blk-flush.c 15.4 KB
Newer Older
1
/*
2
 * Functions to sequence FLUSH and FUA writes.
3
4
5
6
7
8
9
10
11
12
 *
 * Copyright (C) 2011		Max Planck Institute for Gravitational Physics
 * Copyright (C) 2011		Tejun Heo <tj@kernel.org>
 *
 * This file is released under the GPLv2.
 *
 * REQ_{FLUSH|FUA} requests are decomposed to sequences consisted of three
 * optional steps - PREFLUSH, DATA and POSTFLUSH - according to the request
 * properties and hardware capability.
 *
13
14
 * If a request doesn't have data, only REQ_PREFLUSH makes sense, which
 * indicates a simple flush request.  If there is data, REQ_PREFLUSH indicates
15
16
17
18
19
20
21
22
 * that the device cache should be flushed before the data is executed, and
 * REQ_FUA means that the data must be on non-volatile media on request
 * completion.
 *
 * If the device doesn't have writeback cache, FLUSH and FUA don't make any
 * difference.  The requests are either completed immediately if there's no
 * data or executed as normal requests otherwise.
 *
23
 * If the device has writeback cache and supports FUA, REQ_PREFLUSH is
24
25
 * translated to PREFLUSH but REQ_FUA is passed down directly with DATA.
 *
26
27
 * If the device has writeback cache and doesn't support FUA, REQ_PREFLUSH
 * is translated to PREFLUSH and REQ_FUA to POSTFLUSH.
28
29
30
 *
 * The actual execution of flush is double buffered.  Whenever a request
 * needs to execute PRE or POSTFLUSH, it queues at
31
 * fq->flush_queue[fq->flush_pending_idx].  Once certain criteria are met, a
32
 * REQ_OP_FLUSH is issued and the pending_idx is toggled.  When the flush
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
 * completes, all the requests which were pending are proceeded to the next
 * step.  This allows arbitrary merging of different types of FLUSH/FUA
 * requests.
 *
 * Currently, the following conditions are used to determine when to issue
 * flush.
 *
 * C1. At any given time, only one flush shall be in progress.  This makes
 *     double buffering sufficient.
 *
 * C2. Flush is deferred if any request is executing DATA of its sequence.
 *     This avoids issuing separate POSTFLUSHes for requests which shared
 *     PREFLUSH.
 *
 * C3. The second condition is ignored if there is a request which has
 *     waited longer than FLUSH_PENDING_TIMEOUT.  This is to avoid
 *     starvation in the unlikely case where there are continuous stream of
 *     FUA (without FLUSH) requests.
 *
 * For devices which support FUA, it isn't clear whether C2 (and thus C3)
 * is beneficial.
 *
 * Note that a sequenced FLUSH/FUA request with DATA is completed twice.
 * Once while executing DATA and again after the whole sequence is
 * complete.  The first completion updates the contained bio but doesn't
 * finish it so that the bio submitter is notified only after the whole
 * sequence is complete.  This is implemented by testing REQ_FLUSH_SEQ in
 * req_bio_endio().
 *
 * The above peculiarity requires that each FLUSH/FUA request has only one
 * bio attached to it, which is guaranteed as they aren't allowed to be
 * merged in the usual way.
65
 */
66

67
68
69
70
#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/bio.h>
#include <linux/blkdev.h>
71
#include <linux/gfp.h>
72
#include <linux/blk-mq.h>
73
74

#include "blk.h"
75
#include "blk-mq.h"
76
#include "blk-mq-tag.h"
77

78
79
/* FLUSH/FUA sequences */
enum {
80
81
82
83
84
85
86
87
88
89
90
91
92
	REQ_FSEQ_PREFLUSH	= (1 << 0), /* pre-flushing in progress */
	REQ_FSEQ_DATA		= (1 << 1), /* data write in progress */
	REQ_FSEQ_POSTFLUSH	= (1 << 2), /* post-flushing in progress */
	REQ_FSEQ_DONE		= (1 << 3),

	REQ_FSEQ_ACTIONS	= REQ_FSEQ_PREFLUSH | REQ_FSEQ_DATA |
				  REQ_FSEQ_POSTFLUSH,

	/*
	 * If flush has been pending longer than the following timeout,
	 * it's issued even if flush_data requests are still in flight.
	 */
	FLUSH_PENDING_TIMEOUT	= 5 * HZ,
93
94
};

95
96
static bool blk_kick_flush(struct request_queue *q,
			   struct blk_flush_queue *fq);
97

Jens Axboe's avatar
Jens Axboe committed
98
static unsigned int blk_flush_policy(unsigned long fflags, struct request *rq)
99
{
100
	unsigned int policy = 0;
101

102
103
104
	if (blk_rq_sectors(rq))
		policy |= REQ_FSEQ_DATA;

Jens Axboe's avatar
Jens Axboe committed
105
	if (fflags & (1UL << QUEUE_FLAG_WC)) {
106
		if (rq->cmd_flags & REQ_PREFLUSH)
107
			policy |= REQ_FSEQ_PREFLUSH;
Jens Axboe's avatar
Jens Axboe committed
108
109
		if (!(fflags & (1UL << QUEUE_FLAG_FUA)) &&
		    (rq->cmd_flags & REQ_FUA))
110
			policy |= REQ_FSEQ_POSTFLUSH;
111
	}
112
	return policy;
113
114
}

115
static unsigned int blk_flush_cur_seq(struct request *rq)
116
{
117
118
	return 1 << ffz(rq->flush.seq);
}
119

120
121
static void blk_flush_restore_request(struct request *rq)
{
122
	/*
123
124
125
	 * After flush data completion, @rq->bio is %NULL but we need to
	 * complete the bio again.  @rq->biotail is guaranteed to equal the
	 * original @rq->bio.  Restore it.
126
	 */
127
128
129
130
	rq->bio = rq->biotail;

	/* make @rq a normal request */
	rq->cmd_flags &= ~REQ_FLUSH_SEQ;
131
	rq->end_io = rq->flush.saved_end_io;
132
133
}

134
static bool blk_flush_queue_rq(struct request *rq, bool add_front)
135
{
136
	if (rq->q->mq_ops) {
137
138
139
140
		struct request_queue *q = rq->q;

		blk_mq_add_to_requeue_list(rq, add_front);
		blk_mq_kick_requeue_list(q);
141
142
		return false;
	} else {
143
144
145
146
		if (add_front)
			list_add(&rq->queuelist, &rq->q->queue_head);
		else
			list_add_tail(&rq->queuelist, &rq->q->queue_head);
147
148
		return true;
	}
149
150
}

151
152
153
/**
 * blk_flush_complete_seq - complete flush sequence
 * @rq: FLUSH/FUA request being sequenced
154
 * @fq: flush queue
155
156
157
158
159
160
161
 * @seq: sequences to complete (mask of %REQ_FSEQ_*, can be zero)
 * @error: whether an error occurred
 *
 * @rq just completed @seq part of its flush sequence, record the
 * completion and trigger the next step.
 *
 * CONTEXT:
162
 * spin_lock_irq(q->queue_lock or fq->mq_flush_lock)
163
164
165
166
 *
 * RETURNS:
 * %true if requests were added to the dispatch queue, %false otherwise.
 */
167
168
169
static bool blk_flush_complete_seq(struct request *rq,
				   struct blk_flush_queue *fq,
				   unsigned int seq, int error)
170
{
171
	struct request_queue *q = rq->q;
172
	struct list_head *pending = &fq->flush_queue[fq->flush_pending_idx];
173
	bool queued = false, kicked;
174
175
176
177
178
179
180
181
182
183
184
185
186
187

	BUG_ON(rq->flush.seq & seq);
	rq->flush.seq |= seq;

	if (likely(!error))
		seq = blk_flush_cur_seq(rq);
	else
		seq = REQ_FSEQ_DONE;

	switch (seq) {
	case REQ_FSEQ_PREFLUSH:
	case REQ_FSEQ_POSTFLUSH:
		/* queue for flush */
		if (list_empty(pending))
188
			fq->flush_pending_since = jiffies;
189
190
191
192
		list_move_tail(&rq->flush.list, pending);
		break;

	case REQ_FSEQ_DATA:
193
		list_move_tail(&rq->flush.list, &fq->flush_data_in_flight);
194
		queued = blk_flush_queue_rq(rq, true);
195
196
197
198
199
200
201
202
203
204
205
206
		break;

	case REQ_FSEQ_DONE:
		/*
		 * @rq was previously adjusted by blk_flush_issue() for
		 * flush sequencing and may already have gone through the
		 * flush data request completion path.  Restore @rq for
		 * normal completion and end it.
		 */
		BUG_ON(!list_empty(&rq->queuelist));
		list_del_init(&rq->flush.list);
		blk_flush_restore_request(rq);
207
		if (q->mq_ops)
208
			blk_mq_end_request(rq, error);
209
210
		else
			__blk_end_request_all(rq, error);
211
212
213
214
215
216
		break;

	default:
		BUG();
	}

217
	kicked = blk_kick_flush(q, fq);
218
	return kicked | queued;
219
220
}

221
static void flush_end_io(struct request *flush_rq, int error)
222
{
223
	struct request_queue *q = flush_rq->q;
224
	struct list_head *running;
225
226
	bool queued = false;
	struct request *rq, *n;
227
	unsigned long flags = 0;
228
	struct blk_flush_queue *fq = blk_get_flush_queue(q, flush_rq->mq_ctx);
229

230
	if (q->mq_ops) {
231
232
233
		struct blk_mq_hw_ctx *hctx;

		/* release the tag's ownership to the req cloned from */
234
		spin_lock_irqsave(&fq->mq_flush_lock, flags);
Christoph Hellwig's avatar
Christoph Hellwig committed
235
		hctx = blk_mq_map_queue(q, flush_rq->mq_ctx->cpu);
236
		blk_mq_tag_set_rq(hctx, flush_rq->tag, fq->orig_rq);
237
		flush_rq->tag = -1;
238
	}
239

240
241
	running = &fq->flush_queue[fq->flush_running_idx];
	BUG_ON(fq->flush_pending_idx == fq->flush_running_idx);
242
243

	/* account completion of the flush request */
244
	fq->flush_running_idx ^= 1;
245
246
247

	if (!q->mq_ops)
		elv_completed_request(q, flush_rq);
248
249
250
251
252
253

	/* and push the waiting requests to the next stage */
	list_for_each_entry_safe(rq, n, running, flush.list) {
		unsigned int seq = blk_flush_cur_seq(rq);

		BUG_ON(seq != REQ_FSEQ_PREFLUSH && seq != REQ_FSEQ_POSTFLUSH);
254
		queued |= blk_flush_complete_seq(rq, fq, seq, error);
255
256
	}

257
	/*
258
259
260
261
262
263
264
265
266
	 * Kick the queue to avoid stall for two cases:
	 * 1. Moving a request silently to empty queue_head may stall the
	 * queue.
	 * 2. When flush request is running in non-queueable queue, the
	 * queue is hold. Restart the queue after flush request is finished
	 * to avoid stall.
	 * This function is called from request completion path and calling
	 * directly into request_fn may confuse the driver.  Always use
	 * kblockd.
267
	 */
268
	if (queued || fq->flush_queue_delayed) {
269
270
		WARN_ON(q->mq_ops);
		blk_run_queue_async(q);
271
	}
272
	fq->flush_queue_delayed = 0;
273
	if (q->mq_ops)
274
		spin_unlock_irqrestore(&fq->mq_flush_lock, flags);
275
276
}

277
278
279
/**
 * blk_kick_flush - consider issuing flush request
 * @q: request_queue being kicked
280
 * @fq: flush queue
281
282
283
284
285
 *
 * Flush related states of @q have changed, consider issuing flush request.
 * Please read the comment at the top of this file for more info.
 *
 * CONTEXT:
286
 * spin_lock_irq(q->queue_lock or fq->mq_flush_lock)
287
288
289
290
 *
 * RETURNS:
 * %true if flush was issued, %false otherwise.
 */
291
static bool blk_kick_flush(struct request_queue *q, struct blk_flush_queue *fq)
292
{
293
	struct list_head *pending = &fq->flush_queue[fq->flush_pending_idx];
294
295
	struct request *first_rq =
		list_first_entry(pending, struct request, flush.list);
296
	struct request *flush_rq = fq->flush_rq;
297
298

	/* C1 described at the top of this file */
299
	if (fq->flush_pending_idx != fq->flush_running_idx || list_empty(pending))
300
301
302
		return false;

	/* C2 and C3 */
303
	if (!list_empty(&fq->flush_data_in_flight) &&
304
	    time_before(jiffies,
305
			fq->flush_pending_since + FLUSH_PENDING_TIMEOUT))
306
307
308
309
310
311
		return false;

	/*
	 * Issue flush and toggle pending_idx.  This makes pending_idx
	 * different from running_idx, which means flush is in flight.
	 */
312
	fq->flush_pending_idx ^= 1;
313

314
	blk_rq_init(q, flush_rq);
315
316
317

	/*
	 * Borrow tag from the first request since they can't
318
319
	 * be in flight at the same time. And acquire the tag's
	 * ownership for flush req.
320
321
	 */
	if (q->mq_ops) {
322
323
		struct blk_mq_hw_ctx *hctx;

324
325
		flush_rq->mq_ctx = first_rq->mq_ctx;
		flush_rq->tag = first_rq->tag;
326
327
		fq->orig_rq = first_rq;

Christoph Hellwig's avatar
Christoph Hellwig committed
328
		hctx = blk_mq_map_queue(q, first_rq->mq_ctx->cpu);
329
		blk_mq_tag_set_rq(hctx, first_rq->tag, flush_rq);
330
	}
331

332
	flush_rq->cmd_type = REQ_TYPE_FS;
333
	req_set_op_attrs(flush_rq, REQ_OP_FLUSH, WRITE_FLUSH | REQ_FLUSH_SEQ);
334
335
	flush_rq->rq_disk = first_rq->rq_disk;
	flush_rq->end_io = flush_end_io;
336

337
	return blk_flush_queue_rq(flush_rq, false);
338
339
}

340
static void flush_data_end_io(struct request *rq, int error)
341
{
342
	struct request_queue *q = rq->q;
343
	struct blk_flush_queue *fq = blk_get_flush_queue(q, NULL);
344

345
346
347
348
	/*
	 * After populating an empty queue, kick it to avoid stall.  Read
	 * the comment in flush_end_io().
	 */
349
	if (blk_flush_complete_seq(rq, fq, REQ_FSEQ_DATA, error))
350
		blk_run_queue_async(q);
351
352
}

353
354
355
356
static void mq_flush_data_end_io(struct request *rq, int error)
{
	struct request_queue *q = rq->q;
	struct blk_mq_hw_ctx *hctx;
357
	struct blk_mq_ctx *ctx = rq->mq_ctx;
358
	unsigned long flags;
359
	struct blk_flush_queue *fq = blk_get_flush_queue(q, ctx);
360

Christoph Hellwig's avatar
Christoph Hellwig committed
361
	hctx = blk_mq_map_queue(q, ctx->cpu);
362
363
364
365
366

	/*
	 * After populating an empty queue, kick it to avoid stall.  Read
	 * the comment in flush_end_io().
	 */
367
	spin_lock_irqsave(&fq->mq_flush_lock, flags);
368
	if (blk_flush_complete_seq(rq, fq, REQ_FSEQ_DATA, error))
369
		blk_mq_run_hw_queue(hctx, true);
370
	spin_unlock_irqrestore(&fq->mq_flush_lock, flags);
371
372
}

373
374
375
376
/**
 * blk_insert_flush - insert a new FLUSH/FUA request
 * @rq: request to insert
 *
377
 * To be called from __elv_add_request() for %ELEVATOR_INSERT_FLUSH insertions.
378
 * or __blk_mq_run_hw_queue() to dispatch request.
379
380
381
382
 * @rq is being submitted.  Analyze what needs to be done and put it on the
 * right queue.
 *
 * CONTEXT:
383
 * spin_lock_irq(q->queue_lock) in !mq case
384
385
 */
void blk_insert_flush(struct request *rq)
386
{
387
	struct request_queue *q = rq->q;
Jens Axboe's avatar
Jens Axboe committed
388
	unsigned long fflags = q->queue_flags;	/* may change, cache */
389
	unsigned int policy = blk_flush_policy(fflags, rq);
390
	struct blk_flush_queue *fq = blk_get_flush_queue(q, rq->mq_ctx);
391

392
393
	/*
	 * @policy now records what operations need to be done.  Adjust
394
	 * REQ_PREFLUSH and FUA for the driver.
395
	 */
396
	rq->cmd_flags &= ~REQ_PREFLUSH;
Jens Axboe's avatar
Jens Axboe committed
397
	if (!(fflags & (1UL << QUEUE_FLAG_FUA)))
398
399
		rq->cmd_flags &= ~REQ_FUA;

400
401
402
403
404
405
406
	/*
	 * An empty flush handed down from a stacking driver may
	 * translate into nothing if the underlying device does not
	 * advertise a write-back cache.  In this case, simply
	 * complete the request.
	 */
	if (!policy) {
407
		if (q->mq_ops)
408
			blk_mq_end_request(rq, 0);
409
410
		else
			__blk_end_bidi_request(rq, 0, 0, 0);
411
412
413
		return;
	}

414
	BUG_ON(rq->bio != rq->biotail); /*assumes zero or single bio rq */
415

416
417
418
419
420
421
422
	/*
	 * If there's data but flush is not necessary, the request can be
	 * processed directly without going through flush machinery.  Queue
	 * for normal execution.
	 */
	if ((policy & REQ_FSEQ_DATA) &&
	    !(policy & (REQ_FSEQ_PREFLUSH | REQ_FSEQ_POSTFLUSH))) {
423
		if (q->mq_ops) {
424
			blk_mq_insert_request(rq, false, false, true);
425
		} else
426
			list_add_tail(&rq->queuelist, &q->queue_head);
427
		return;
428
	}
429

430
431
432
433
434
435
	/*
	 * @rq should go through flush machinery.  Mark it part of flush
	 * sequence and submit for further processing.
	 */
	memset(&rq->flush, 0, sizeof(rq->flush));
	INIT_LIST_HEAD(&rq->flush.list);
Tejun Heo's avatar
Tejun Heo committed
436
	rq->cmd_flags |= REQ_FLUSH_SEQ;
437
	rq->flush.saved_end_io = rq->end_io; /* Usually NULL */
438
439
440
	if (q->mq_ops) {
		rq->end_io = mq_flush_data_end_io;

441
		spin_lock_irq(&fq->mq_flush_lock);
442
		blk_flush_complete_seq(rq, fq, REQ_FSEQ_ACTIONS & ~policy, 0);
443
		spin_unlock_irq(&fq->mq_flush_lock);
444
445
		return;
	}
446
447
	rq->end_io = flush_data_end_io;

448
	blk_flush_complete_seq(rq, fq, REQ_FSEQ_ACTIONS & ~policy, 0);
449
450
451
452
453
}

/**
 * blkdev_issue_flush - queue a flush
 * @bdev:	blockdev to issue flush for
454
 * @gfp_mask:	memory allocation flags (for bio_alloc)
455
456
457
458
459
 * @error_sector:	error sector
 *
 * Description:
 *    Issue a flush for the block device in question. Caller can supply
 *    room for storing the error offset in case of a flush error, if they
460
461
 *    wish to. If WAIT flag is not passed then caller may check only what
 *    request was pushed in some internal queue for later handling.
462
 */
463
int blkdev_issue_flush(struct block_device *bdev, gfp_t gfp_mask,
464
		sector_t *error_sector)
465
466
467
{
	struct request_queue *q;
	struct bio *bio;
468
	int ret = 0;
469
470
471
472
473
474
475
476

	if (bdev->bd_disk == NULL)
		return -ENXIO;

	q = bdev_get_queue(bdev);
	if (!q)
		return -ENXIO;

477
478
479
480
	/*
	 * some block devices may not have their queue correctly set up here
	 * (e.g. loop device without a backing file) and so issuing a flush
	 * here will panic. Ensure there is a request function before issuing
481
	 * the flush.
482
483
484
485
	 */
	if (!q->make_request_fn)
		return -ENXIO;

486
	bio = bio_alloc(gfp_mask, 0);
487
	bio->bi_bdev = bdev;
488
	bio_set_op_attrs(bio, REQ_OP_WRITE, WRITE_FLUSH);
489

490
	ret = submit_bio_wait(bio);
491
492
493
494
495
496
497

	/*
	 * The driver must store the error location in ->bi_sector, if
	 * it supports it. For non-stacked drivers, this should be
	 * copied from blk_rq_pos(rq).
	 */
	if (error_sector)
498
		*error_sector = bio->bi_iter.bi_sector;
499
500
501
502
503

	bio_put(bio);
	return ret;
}
EXPORT_SYMBOL(blkdev_issue_flush);
504

505
506
struct blk_flush_queue *blk_alloc_flush_queue(struct request_queue *q,
		int node, int cmd_size)
507
{
508
509
	struct blk_flush_queue *fq;
	int rq_sz = sizeof(struct request);
510

511
	fq = kzalloc_node(sizeof(*fq), GFP_KERNEL, node);
512
513
	if (!fq)
		goto fail;
514

515
516
	if (q->mq_ops) {
		spin_lock_init(&fq->mq_flush_lock);
517
		rq_sz = round_up(rq_sz + cmd_size, cache_line_size());
518
519
	}

520
	fq->flush_rq = kzalloc_node(rq_sz, GFP_KERNEL, node);
521
522
523
524
525
526
527
528
529
530
531
532
533
	if (!fq->flush_rq)
		goto fail_rq;

	INIT_LIST_HEAD(&fq->flush_queue[0]);
	INIT_LIST_HEAD(&fq->flush_queue[1]);
	INIT_LIST_HEAD(&fq->flush_data_in_flight);

	return fq;

 fail_rq:
	kfree(fq);
 fail:
	return NULL;
534
}
535

536
void blk_free_flush_queue(struct blk_flush_queue *fq)
537
{
538
539
540
	/* bio based request queue hasn't flush queue */
	if (!fq)
		return;
541

542
543
544
	kfree(fq->flush_rq);
	kfree(fq);
}