blk-flush.c 15.4 KB
Newer Older
1
/*
2
 * Functions to sequence FLUSH and FUA writes.
3
4
5
6
7
8
9
10
11
12
 *
 * Copyright (C) 2011		Max Planck Institute for Gravitational Physics
 * Copyright (C) 2011		Tejun Heo <tj@kernel.org>
 *
 * This file is released under the GPLv2.
 *
 * REQ_{FLUSH|FUA} requests are decomposed to sequences consisted of three
 * optional steps - PREFLUSH, DATA and POSTFLUSH - according to the request
 * properties and hardware capability.
 *
13
14
 * If a request doesn't have data, only REQ_PREFLUSH makes sense, which
 * indicates a simple flush request.  If there is data, REQ_PREFLUSH indicates
15
16
17
18
19
20
21
22
 * that the device cache should be flushed before the data is executed, and
 * REQ_FUA means that the data must be on non-volatile media on request
 * completion.
 *
 * If the device doesn't have writeback cache, FLUSH and FUA don't make any
 * difference.  The requests are either completed immediately if there's no
 * data or executed as normal requests otherwise.
 *
23
 * If the device has writeback cache and supports FUA, REQ_PREFLUSH is
24
25
 * translated to PREFLUSH but REQ_FUA is passed down directly with DATA.
 *
26
27
 * If the device has writeback cache and doesn't support FUA, REQ_PREFLUSH
 * is translated to PREFLUSH and REQ_FUA to POSTFLUSH.
28
29
30
 *
 * The actual execution of flush is double buffered.  Whenever a request
 * needs to execute PRE or POSTFLUSH, it queues at
31
 * fq->flush_queue[fq->flush_pending_idx].  Once certain criteria are met, a
32
 * REQ_OP_FLUSH is issued and the pending_idx is toggled.  When the flush
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
 * completes, all the requests which were pending are proceeded to the next
 * step.  This allows arbitrary merging of different types of FLUSH/FUA
 * requests.
 *
 * Currently, the following conditions are used to determine when to issue
 * flush.
 *
 * C1. At any given time, only one flush shall be in progress.  This makes
 *     double buffering sufficient.
 *
 * C2. Flush is deferred if any request is executing DATA of its sequence.
 *     This avoids issuing separate POSTFLUSHes for requests which shared
 *     PREFLUSH.
 *
 * C3. The second condition is ignored if there is a request which has
 *     waited longer than FLUSH_PENDING_TIMEOUT.  This is to avoid
 *     starvation in the unlikely case where there are continuous stream of
 *     FUA (without FLUSH) requests.
 *
 * For devices which support FUA, it isn't clear whether C2 (and thus C3)
 * is beneficial.
 *
 * Note that a sequenced FLUSH/FUA request with DATA is completed twice.
 * Once while executing DATA and again after the whole sequence is
 * complete.  The first completion updates the contained bio but doesn't
 * finish it so that the bio submitter is notified only after the whole
59
 * sequence is complete.  This is implemented by testing RQF_FLUSH_SEQ in
60
61
62
63
64
 * req_bio_endio().
 *
 * The above peculiarity requires that each FLUSH/FUA request has only one
 * bio attached to it, which is guaranteed as they aren't allowed to be
 * merged in the usual way.
65
 */
66

67
68
69
70
#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/bio.h>
#include <linux/blkdev.h>
71
#include <linux/gfp.h>
72
#include <linux/blk-mq.h>
73
74

#include "blk.h"
75
#include "blk-mq.h"
76
#include "blk-mq-tag.h"
77

78
79
/* FLUSH/FUA sequences */
enum {
80
81
82
83
84
85
86
87
88
89
90
91
92
	REQ_FSEQ_PREFLUSH	= (1 << 0), /* pre-flushing in progress */
	REQ_FSEQ_DATA		= (1 << 1), /* data write in progress */
	REQ_FSEQ_POSTFLUSH	= (1 << 2), /* post-flushing in progress */
	REQ_FSEQ_DONE		= (1 << 3),

	REQ_FSEQ_ACTIONS	= REQ_FSEQ_PREFLUSH | REQ_FSEQ_DATA |
				  REQ_FSEQ_POSTFLUSH,

	/*
	 * If flush has been pending longer than the following timeout,
	 * it's issued even if flush_data requests are still in flight.
	 */
	FLUSH_PENDING_TIMEOUT	= 5 * HZ,
93
94
};

95
96
static bool blk_kick_flush(struct request_queue *q,
			   struct blk_flush_queue *fq);
97

Jens Axboe's avatar
Jens Axboe committed
98
static unsigned int blk_flush_policy(unsigned long fflags, struct request *rq)
99
{
100
	unsigned int policy = 0;
101

102
103
104
	if (blk_rq_sectors(rq))
		policy |= REQ_FSEQ_DATA;

Jens Axboe's avatar
Jens Axboe committed
105
	if (fflags & (1UL << QUEUE_FLAG_WC)) {
106
		if (rq->cmd_flags & REQ_PREFLUSH)
107
			policy |= REQ_FSEQ_PREFLUSH;
Jens Axboe's avatar
Jens Axboe committed
108
109
		if (!(fflags & (1UL << QUEUE_FLAG_FUA)) &&
		    (rq->cmd_flags & REQ_FUA))
110
			policy |= REQ_FSEQ_POSTFLUSH;
111
	}
112
	return policy;
113
114
}

115
static unsigned int blk_flush_cur_seq(struct request *rq)
116
{
117
118
	return 1 << ffz(rq->flush.seq);
}
119

120
121
static void blk_flush_restore_request(struct request *rq)
{
122
	/*
123
124
125
	 * After flush data completion, @rq->bio is %NULL but we need to
	 * complete the bio again.  @rq->biotail is guaranteed to equal the
	 * original @rq->bio.  Restore it.
126
	 */
127
128
129
	rq->bio = rq->biotail;

	/* make @rq a normal request */
130
	rq->rq_flags &= ~RQF_FLUSH_SEQ;
131
	rq->end_io = rq->flush.saved_end_io;
132
133
}

134
static bool blk_flush_queue_rq(struct request *rq, bool add_front)
135
{
136
	if (rq->q->mq_ops) {
137
138
139
140
		struct request_queue *q = rq->q;

		blk_mq_add_to_requeue_list(rq, add_front);
		blk_mq_kick_requeue_list(q);
141
142
		return false;
	} else {
143
144
145
146
		if (add_front)
			list_add(&rq->queuelist, &rq->q->queue_head);
		else
			list_add_tail(&rq->queuelist, &rq->q->queue_head);
147
148
		return true;
	}
149
150
}

151
152
153
/**
 * blk_flush_complete_seq - complete flush sequence
 * @rq: FLUSH/FUA request being sequenced
154
 * @fq: flush queue
155
156
157
158
159
160
161
 * @seq: sequences to complete (mask of %REQ_FSEQ_*, can be zero)
 * @error: whether an error occurred
 *
 * @rq just completed @seq part of its flush sequence, record the
 * completion and trigger the next step.
 *
 * CONTEXT:
162
 * spin_lock_irq(q->queue_lock or fq->mq_flush_lock)
163
164
165
166
 *
 * RETURNS:
 * %true if requests were added to the dispatch queue, %false otherwise.
 */
167
168
169
static bool blk_flush_complete_seq(struct request *rq,
				   struct blk_flush_queue *fq,
				   unsigned int seq, int error)
170
{
171
	struct request_queue *q = rq->q;
172
	struct list_head *pending = &fq->flush_queue[fq->flush_pending_idx];
173
	bool queued = false, kicked;
174
175
176
177
178
179
180
181
182
183
184
185
186
187

	BUG_ON(rq->flush.seq & seq);
	rq->flush.seq |= seq;

	if (likely(!error))
		seq = blk_flush_cur_seq(rq);
	else
		seq = REQ_FSEQ_DONE;

	switch (seq) {
	case REQ_FSEQ_PREFLUSH:
	case REQ_FSEQ_POSTFLUSH:
		/* queue for flush */
		if (list_empty(pending))
188
			fq->flush_pending_since = jiffies;
189
190
191
192
		list_move_tail(&rq->flush.list, pending);
		break;

	case REQ_FSEQ_DATA:
193
		list_move_tail(&rq->flush.list, &fq->flush_data_in_flight);
194
		queued = blk_flush_queue_rq(rq, true);
195
196
197
198
199
200
201
202
203
204
205
206
		break;

	case REQ_FSEQ_DONE:
		/*
		 * @rq was previously adjusted by blk_flush_issue() for
		 * flush sequencing and may already have gone through the
		 * flush data request completion path.  Restore @rq for
		 * normal completion and end it.
		 */
		BUG_ON(!list_empty(&rq->queuelist));
		list_del_init(&rq->flush.list);
		blk_flush_restore_request(rq);
207
		if (q->mq_ops)
208
			blk_mq_end_request(rq, error);
209
210
		else
			__blk_end_request_all(rq, error);
211
212
213
214
215
216
		break;

	default:
		BUG();
	}

217
	kicked = blk_kick_flush(q, fq);
218
	return kicked | queued;
219
220
}

221
static void flush_end_io(struct request *flush_rq, int error)
222
{
223
	struct request_queue *q = flush_rq->q;
224
	struct list_head *running;
225
226
	bool queued = false;
	struct request *rq, *n;
227
	unsigned long flags = 0;
228
	struct blk_flush_queue *fq = blk_get_flush_queue(q, flush_rq->mq_ctx);
229

230
	if (q->mq_ops) {
231
232
233
		struct blk_mq_hw_ctx *hctx;

		/* release the tag's ownership to the req cloned from */
234
		spin_lock_irqsave(&fq->mq_flush_lock, flags);
Christoph Hellwig's avatar
Christoph Hellwig committed
235
		hctx = blk_mq_map_queue(q, flush_rq->mq_ctx->cpu);
236
		blk_mq_tag_set_rq(hctx, flush_rq->tag, fq->orig_rq);
237
		flush_rq->tag = -1;
238
	}
239

240
241
	running = &fq->flush_queue[fq->flush_running_idx];
	BUG_ON(fq->flush_pending_idx == fq->flush_running_idx);
242
243

	/* account completion of the flush request */
244
	fq->flush_running_idx ^= 1;
245
246
247

	if (!q->mq_ops)
		elv_completed_request(q, flush_rq);
248
249
250
251
252
253

	/* and push the waiting requests to the next stage */
	list_for_each_entry_safe(rq, n, running, flush.list) {
		unsigned int seq = blk_flush_cur_seq(rq);

		BUG_ON(seq != REQ_FSEQ_PREFLUSH && seq != REQ_FSEQ_POSTFLUSH);
254
		queued |= blk_flush_complete_seq(rq, fq, seq, error);
255
256
	}

257
	/*
258
259
260
261
262
263
264
265
266
	 * Kick the queue to avoid stall for two cases:
	 * 1. Moving a request silently to empty queue_head may stall the
	 * queue.
	 * 2. When flush request is running in non-queueable queue, the
	 * queue is hold. Restart the queue after flush request is finished
	 * to avoid stall.
	 * This function is called from request completion path and calling
	 * directly into request_fn may confuse the driver.  Always use
	 * kblockd.
267
	 */
268
	if (queued || fq->flush_queue_delayed) {
269
270
		WARN_ON(q->mq_ops);
		blk_run_queue_async(q);
271
	}
272
	fq->flush_queue_delayed = 0;
273
	if (q->mq_ops)
274
		spin_unlock_irqrestore(&fq->mq_flush_lock, flags);
275
276
}

277
278
279
/**
 * blk_kick_flush - consider issuing flush request
 * @q: request_queue being kicked
280
 * @fq: flush queue
281
282
283
284
285
 *
 * Flush related states of @q have changed, consider issuing flush request.
 * Please read the comment at the top of this file for more info.
 *
 * CONTEXT:
286
 * spin_lock_irq(q->queue_lock or fq->mq_flush_lock)
287
288
289
290
 *
 * RETURNS:
 * %true if flush was issued, %false otherwise.
 */
291
static bool blk_kick_flush(struct request_queue *q, struct blk_flush_queue *fq)
292
{
293
	struct list_head *pending = &fq->flush_queue[fq->flush_pending_idx];
294
295
	struct request *first_rq =
		list_first_entry(pending, struct request, flush.list);
296
	struct request *flush_rq = fq->flush_rq;
297
298

	/* C1 described at the top of this file */
299
	if (fq->flush_pending_idx != fq->flush_running_idx || list_empty(pending))
300
301
302
		return false;

	/* C2 and C3 */
303
	if (!list_empty(&fq->flush_data_in_flight) &&
304
	    time_before(jiffies,
305
			fq->flush_pending_since + FLUSH_PENDING_TIMEOUT))
306
307
308
309
310
311
		return false;

	/*
	 * Issue flush and toggle pending_idx.  This makes pending_idx
	 * different from running_idx, which means flush is in flight.
	 */
312
	fq->flush_pending_idx ^= 1;
313

314
	blk_rq_init(q, flush_rq);
315
316
317

	/*
	 * Borrow tag from the first request since they can't
318
319
	 * be in flight at the same time. And acquire the tag's
	 * ownership for flush req.
320
321
	 */
	if (q->mq_ops) {
322
323
		struct blk_mq_hw_ctx *hctx;

324
325
		flush_rq->mq_ctx = first_rq->mq_ctx;
		flush_rq->tag = first_rq->tag;
326
327
		fq->orig_rq = first_rq;

Christoph Hellwig's avatar
Christoph Hellwig committed
328
		hctx = blk_mq_map_queue(q, first_rq->mq_ctx->cpu);
329
		blk_mq_tag_set_rq(hctx, first_rq->tag, flush_rq);
330
	}
331

332
	flush_rq->cmd_type = REQ_TYPE_FS;
333
	flush_rq->cmd_flags = REQ_OP_FLUSH | REQ_PREFLUSH;
334
	flush_rq->rq_flags |= RQF_FLUSH_SEQ;
335
336
	flush_rq->rq_disk = first_rq->rq_disk;
	flush_rq->end_io = flush_end_io;
337

338
	return blk_flush_queue_rq(flush_rq, false);
339
340
}

341
static void flush_data_end_io(struct request *rq, int error)
342
{
343
	struct request_queue *q = rq->q;
344
	struct blk_flush_queue *fq = blk_get_flush_queue(q, NULL);
345

346
347
348
349
	/*
	 * After populating an empty queue, kick it to avoid stall.  Read
	 * the comment in flush_end_io().
	 */
350
	if (blk_flush_complete_seq(rq, fq, REQ_FSEQ_DATA, error))
351
		blk_run_queue_async(q);
352
353
}

354
355
356
357
static void mq_flush_data_end_io(struct request *rq, int error)
{
	struct request_queue *q = rq->q;
	struct blk_mq_hw_ctx *hctx;
358
	struct blk_mq_ctx *ctx = rq->mq_ctx;
359
	unsigned long flags;
360
	struct blk_flush_queue *fq = blk_get_flush_queue(q, ctx);
361

Christoph Hellwig's avatar
Christoph Hellwig committed
362
	hctx = blk_mq_map_queue(q, ctx->cpu);
363
364
365
366
367

	/*
	 * After populating an empty queue, kick it to avoid stall.  Read
	 * the comment in flush_end_io().
	 */
368
	spin_lock_irqsave(&fq->mq_flush_lock, flags);
369
	if (blk_flush_complete_seq(rq, fq, REQ_FSEQ_DATA, error))
370
		blk_mq_run_hw_queue(hctx, true);
371
	spin_unlock_irqrestore(&fq->mq_flush_lock, flags);
372
373
}

374
375
376
377
/**
 * blk_insert_flush - insert a new FLUSH/FUA request
 * @rq: request to insert
 *
378
 * To be called from __elv_add_request() for %ELEVATOR_INSERT_FLUSH insertions.
379
 * or __blk_mq_run_hw_queue() to dispatch request.
380
381
382
383
 * @rq is being submitted.  Analyze what needs to be done and put it on the
 * right queue.
 *
 * CONTEXT:
384
 * spin_lock_irq(q->queue_lock) in !mq case
385
386
 */
void blk_insert_flush(struct request *rq)
387
{
388
	struct request_queue *q = rq->q;
Jens Axboe's avatar
Jens Axboe committed
389
	unsigned long fflags = q->queue_flags;	/* may change, cache */
390
	unsigned int policy = blk_flush_policy(fflags, rq);
391
	struct blk_flush_queue *fq = blk_get_flush_queue(q, rq->mq_ctx);
392

393
394
	/*
	 * @policy now records what operations need to be done.  Adjust
395
	 * REQ_PREFLUSH and FUA for the driver.
396
	 */
397
	rq->cmd_flags &= ~REQ_PREFLUSH;
Jens Axboe's avatar
Jens Axboe committed
398
	if (!(fflags & (1UL << QUEUE_FLAG_FUA)))
399
400
		rq->cmd_flags &= ~REQ_FUA;

401
402
403
404
405
406
407
	/*
	 * An empty flush handed down from a stacking driver may
	 * translate into nothing if the underlying device does not
	 * advertise a write-back cache.  In this case, simply
	 * complete the request.
	 */
	if (!policy) {
408
		if (q->mq_ops)
409
			blk_mq_end_request(rq, 0);
410
411
		else
			__blk_end_bidi_request(rq, 0, 0, 0);
412
413
414
		return;
	}

415
	BUG_ON(rq->bio != rq->biotail); /*assumes zero or single bio rq */
416

417
418
419
420
421
422
423
	/*
	 * If there's data but flush is not necessary, the request can be
	 * processed directly without going through flush machinery.  Queue
	 * for normal execution.
	 */
	if ((policy & REQ_FSEQ_DATA) &&
	    !(policy & (REQ_FSEQ_PREFLUSH | REQ_FSEQ_POSTFLUSH))) {
424
		if (q->mq_ops) {
425
			blk_mq_insert_request(rq, false, false, true);
426
		} else
427
			list_add_tail(&rq->queuelist, &q->queue_head);
428
		return;
429
	}
430

431
432
433
434
435
436
	/*
	 * @rq should go through flush machinery.  Mark it part of flush
	 * sequence and submit for further processing.
	 */
	memset(&rq->flush, 0, sizeof(rq->flush));
	INIT_LIST_HEAD(&rq->flush.list);
437
	rq->rq_flags |= RQF_FLUSH_SEQ;
438
	rq->flush.saved_end_io = rq->end_io; /* Usually NULL */
439
440
441
	if (q->mq_ops) {
		rq->end_io = mq_flush_data_end_io;

442
		spin_lock_irq(&fq->mq_flush_lock);
443
		blk_flush_complete_seq(rq, fq, REQ_FSEQ_ACTIONS & ~policy, 0);
444
		spin_unlock_irq(&fq->mq_flush_lock);
445
446
		return;
	}
447
448
	rq->end_io = flush_data_end_io;

449
	blk_flush_complete_seq(rq, fq, REQ_FSEQ_ACTIONS & ~policy, 0);
450
451
452
453
454
}

/**
 * blkdev_issue_flush - queue a flush
 * @bdev:	blockdev to issue flush for
455
 * @gfp_mask:	memory allocation flags (for bio_alloc)
456
457
458
459
460
 * @error_sector:	error sector
 *
 * Description:
 *    Issue a flush for the block device in question. Caller can supply
 *    room for storing the error offset in case of a flush error, if they
461
462
 *    wish to. If WAIT flag is not passed then caller may check only what
 *    request was pushed in some internal queue for later handling.
463
 */
464
int blkdev_issue_flush(struct block_device *bdev, gfp_t gfp_mask,
465
		sector_t *error_sector)
466
467
468
{
	struct request_queue *q;
	struct bio *bio;
469
	int ret = 0;
470
471
472
473
474
475
476
477

	if (bdev->bd_disk == NULL)
		return -ENXIO;

	q = bdev_get_queue(bdev);
	if (!q)
		return -ENXIO;

478
479
480
481
	/*
	 * some block devices may not have their queue correctly set up here
	 * (e.g. loop device without a backing file) and so issuing a flush
	 * here will panic. Ensure there is a request function before issuing
482
	 * the flush.
483
484
485
486
	 */
	if (!q->make_request_fn)
		return -ENXIO;

487
	bio = bio_alloc(gfp_mask, 0);
488
	bio->bi_bdev = bdev;
489
	bio->bi_opf = REQ_OP_WRITE | REQ_PREFLUSH;
490

491
	ret = submit_bio_wait(bio);
492
493
494
495
496
497
498

	/*
	 * The driver must store the error location in ->bi_sector, if
	 * it supports it. For non-stacked drivers, this should be
	 * copied from blk_rq_pos(rq).
	 */
	if (error_sector)
499
		*error_sector = bio->bi_iter.bi_sector;
500
501
502
503
504

	bio_put(bio);
	return ret;
}
EXPORT_SYMBOL(blkdev_issue_flush);
505

506
507
struct blk_flush_queue *blk_alloc_flush_queue(struct request_queue *q,
		int node, int cmd_size)
508
{
509
510
	struct blk_flush_queue *fq;
	int rq_sz = sizeof(struct request);
511

512
	fq = kzalloc_node(sizeof(*fq), GFP_KERNEL, node);
513
514
	if (!fq)
		goto fail;
515

516
517
	if (q->mq_ops) {
		spin_lock_init(&fq->mq_flush_lock);
518
		rq_sz = round_up(rq_sz + cmd_size, cache_line_size());
519
520
	}

521
	fq->flush_rq = kzalloc_node(rq_sz, GFP_KERNEL, node);
522
523
524
525
526
527
528
529
530
531
532
533
534
	if (!fq->flush_rq)
		goto fail_rq;

	INIT_LIST_HEAD(&fq->flush_queue[0]);
	INIT_LIST_HEAD(&fq->flush_queue[1]);
	INIT_LIST_HEAD(&fq->flush_data_in_flight);

	return fq;

 fail_rq:
	kfree(fq);
 fail:
	return NULL;
535
}
536

537
void blk_free_flush_queue(struct blk_flush_queue *fq)
538
{
539
540
541
	/* bio based request queue hasn't flush queue */
	if (!fq)
		return;
542

543
544
545
	kfree(fq->flush_rq);
	kfree(fq);
}