blk-lib.c 6.99 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
/*
 * Functions related to generic helpers functions
 */
#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/bio.h>
#include <linux/blkdev.h>
#include <linux/scatterlist.h>

#include "blk.h"

12
13
struct bio_batch {
	atomic_t		done;
14
	int			error;
15
16
17
	struct completion	*wait;
};

18
static void bio_batch_end_io(struct bio *bio)
19
{
20
21
	struct bio_batch *bb = bio->bi_private;

22
23
	if (bio->bi_error && bio->bi_error != -EOPNOTSUPP)
		bb->error = bio->bi_error;
24
25
	if (atomic_dec_and_test(&bb->done))
		complete(bb->wait);
26
27
28
	bio_put(bio);
}

29
30
31
32
33
34
35
/*
 * Ensure that max discard sectors doesn't overflow bi_size and hopefully
 * it is of the proper granularity as long as the granularity is a power
 * of two.
 */
#define MAX_BIO_SECTORS ((1U << 31) >> 9)

36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
/**
 * blkdev_issue_discard - queue a discard
 * @bdev:	blockdev to issue discard for
 * @sector:	start sector
 * @nr_sects:	number of sectors to discard
 * @gfp_mask:	memory allocation flags (for bio_alloc)
 * @flags:	BLKDEV_IFL_* flags to control behaviour
 *
 * Description:
 *    Issue a discard request for the sectors in question.
 */
int blkdev_issue_discard(struct block_device *bdev, sector_t sector,
		sector_t nr_sects, gfp_t gfp_mask, unsigned long flags)
{
	DECLARE_COMPLETION_ONSTACK(wait);
	struct request_queue *q = bdev_get_queue(bdev);
52
	int type = REQ_WRITE | REQ_DISCARD;
53
	struct bio_batch bb;
54
55
	struct bio *bio;
	int ret = 0;
56
	struct blk_plug plug;
57
58
59
60
61
62
63

	if (!q)
		return -ENXIO;

	if (!blk_queue_discard(q))
		return -EOPNOTSUPP;

64
	if (flags & BLKDEV_DISCARD_SECURE) {
Adrian Hunter's avatar
Adrian Hunter committed
65
66
		if (!blk_queue_secdiscard(q))
			return -EOPNOTSUPP;
67
		type |= REQ_SECURE;
Adrian Hunter's avatar
Adrian Hunter committed
68
69
	}

70
	atomic_set(&bb.done, 1);
71
	bb.error = 0;
72
73
	bb.wait = &wait;

74
	blk_start_plug(&plug);
75
	while (nr_sects) {
76
		unsigned int req_sects;
77
		sector_t end_sect;
78

79
		bio = bio_alloc(gfp_mask, 1);
80
81
82
83
84
		if (!bio) {
			ret = -ENOMEM;
			break;
		}

85
		req_sects = min_t(sector_t, nr_sects, MAX_BIO_SECTORS);
86
87
		end_sect = sector + req_sects;

88
		bio->bi_iter.bi_sector = sector;
89
		bio->bi_end_io = bio_batch_end_io;
90
		bio->bi_bdev = bdev;
91
		bio->bi_private = &bb;
92

93
		bio->bi_iter.bi_size = req_sects << 9;
94
95
		nr_sects -= req_sects;
		sector = end_sect;
96

97
		atomic_inc(&bb.done);
98
		submit_bio(type, bio);
99
100
101
102
103
104
105
106

		/*
		 * We can loop for a long time in here, if someone does
		 * full device discards (like mkfs). Be nice and allow
		 * us to schedule out to avoid softlocking if preempt
		 * is disabled.
		 */
		cond_resched();
107
	}
108
	blk_finish_plug(&plug);
109

110
111
	/* Wait for bios in-flight */
	if (!atomic_dec_and_test(&bb.done))
112
		wait_for_completion_io(&wait);
113

114
115
	if (bb.error)
		return bb.error;
116
117
118
	return ret;
}
EXPORT_SYMBOL(blkdev_issue_discard);
119

120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
/**
 * blkdev_issue_write_same - queue a write same operation
 * @bdev:	target blockdev
 * @sector:	start sector
 * @nr_sects:	number of sectors to write
 * @gfp_mask:	memory allocation flags (for bio_alloc)
 * @page:	page containing data to write
 *
 * Description:
 *    Issue a write same request for the sectors in question.
 */
int blkdev_issue_write_same(struct block_device *bdev, sector_t sector,
			    sector_t nr_sects, gfp_t gfp_mask,
			    struct page *page)
{
	DECLARE_COMPLETION_ONSTACK(wait);
	struct request_queue *q = bdev_get_queue(bdev);
	unsigned int max_write_same_sectors;
	struct bio_batch bb;
	struct bio *bio;
	int ret = 0;

	if (!q)
		return -ENXIO;

145
146
	/* Ensure that max_write_same_sectors doesn't overflow bi_size */
	max_write_same_sectors = UINT_MAX >> 9;
147
148

	atomic_set(&bb.done, 1);
149
	bb.error = 0;
150
151
152
153
154
155
156
157
158
	bb.wait = &wait;

	while (nr_sects) {
		bio = bio_alloc(gfp_mask, 1);
		if (!bio) {
			ret = -ENOMEM;
			break;
		}

159
		bio->bi_iter.bi_sector = sector;
160
161
162
163
164
165
166
167
168
		bio->bi_end_io = bio_batch_end_io;
		bio->bi_bdev = bdev;
		bio->bi_private = &bb;
		bio->bi_vcnt = 1;
		bio->bi_io_vec->bv_page = page;
		bio->bi_io_vec->bv_offset = 0;
		bio->bi_io_vec->bv_len = bdev_logical_block_size(bdev);

		if (nr_sects > max_write_same_sectors) {
169
			bio->bi_iter.bi_size = max_write_same_sectors << 9;
170
171
172
			nr_sects -= max_write_same_sectors;
			sector += max_write_same_sectors;
		} else {
173
			bio->bi_iter.bi_size = nr_sects << 9;
174
175
176
177
178
179
180
181
182
			nr_sects = 0;
		}

		atomic_inc(&bb.done);
		submit_bio(REQ_WRITE | REQ_WRITE_SAME, bio);
	}

	/* Wait for bios in-flight */
	if (!atomic_dec_and_test(&bb.done))
183
		wait_for_completion_io(&wait);
184

185
186
	if (bb.error)
		return bb.error;
187
188
189
190
	return ret;
}
EXPORT_SYMBOL(blkdev_issue_write_same);

191
/**
192
 * blkdev_issue_zeroout - generate number of zero filed write bios
193
194
195
196
197
198
199
200
201
 * @bdev:	blockdev to issue
 * @sector:	start sector
 * @nr_sects:	number of sectors to write
 * @gfp_mask:	memory allocation flags (for bio_alloc)
 *
 * Description:
 *  Generate and issue number of bios with zerofiled pages.
 */

202
203
static int __blkdev_issue_zeroout(struct block_device *bdev, sector_t sector,
				  sector_t nr_sects, gfp_t gfp_mask)
204
{
205
	int ret;
206
207
	struct bio *bio;
	struct bio_batch bb;
208
	unsigned int sz;
209
210
	DECLARE_COMPLETION_ONSTACK(wait);

211
	atomic_set(&bb.done, 1);
212
	bb.error = 0;
213
214
	bb.wait = &wait;

215
	ret = 0;
216
217
218
	while (nr_sects != 0) {
		bio = bio_alloc(gfp_mask,
				min(nr_sects, (sector_t)BIO_MAX_PAGES));
219
220
		if (!bio) {
			ret = -ENOMEM;
221
			break;
222
		}
223

224
		bio->bi_iter.bi_sector = sector;
225
226
		bio->bi_bdev   = bdev;
		bio->bi_end_io = bio_batch_end_io;
227
		bio->bi_private = &bb;
228

229
230
		while (nr_sects != 0) {
			sz = min((sector_t) PAGE_SIZE >> 9 , nr_sects);
231
232
233
234
235
236
			ret = bio_add_page(bio, ZERO_PAGE(0), sz << 9, 0);
			nr_sects -= ret >> 9;
			sector += ret >> 9;
			if (ret < (sz << 9))
				break;
		}
237
		ret = 0;
238
		atomic_inc(&bb.done);
239
240
241
		submit_bio(WRITE, bio);
	}

242
	/* Wait for bios in-flight */
243
	if (!atomic_dec_and_test(&bb.done))
244
		wait_for_completion_io(&wait);
245

246
247
	if (bb.error)
		return bb.error;
248
249
	return ret;
}
250
251
252
253
254
255
256

/**
 * blkdev_issue_zeroout - zero-fill a block range
 * @bdev:	blockdev to write
 * @sector:	start sector
 * @nr_sects:	number of sectors to write
 * @gfp_mask:	memory allocation flags (for bio_alloc)
257
 * @discard:	whether to discard the block range
258
259
 *
 * Description:
260
261
262
263
264
265
266
267
268
269
 *  Zero-fill a block range.  If the discard flag is set and the block
 *  device guarantees that subsequent READ operations to the block range
 *  in question will return zeroes, the blocks will be discarded. Should
 *  the discard request fail, if the discard flag is not set, or if
 *  discard_zeroes_data is not supported, this function will resort to
 *  zeroing the blocks manually, thus provisioning (allocating,
 *  anchoring) them. If the block device supports the WRITE SAME command
 *  blkdev_issue_zeroout() will use it to optimize the process of
 *  clearing the block range. Otherwise the zeroing will be performed
 *  using regular WRITE calls.
270
271
272
 */

int blkdev_issue_zeroout(struct block_device *bdev, sector_t sector,
273
			 sector_t nr_sects, gfp_t gfp_mask, bool discard)
274
{
275
276
	struct request_queue *q = bdev_get_queue(bdev);

277
278
279
	if (discard && blk_queue_discard(q) && q->limits.discard_zeroes_data &&
	    blkdev_issue_discard(bdev, sector, nr_sects, gfp_mask, 0) == 0)
		return 0;
280

281
282
283
284
	if (bdev_write_same(bdev) &&
	    blkdev_issue_write_same(bdev, sector, nr_sects, gfp_mask,
				    ZERO_PAGE(0)) == 0)
		return 0;
285
286
287

	return __blkdev_issue_zeroout(bdev, sector, nr_sects, gfp_mask);
}
288
EXPORT_SYMBOL(blkdev_issue_zeroout);