blk-flush.c 15.5 KB
Newer Older
1
/*
2
 * Functions to sequence FLUSH and FUA writes.
3
4
5
6
7
8
9
10
11
12
 *
 * Copyright (C) 2011		Max Planck Institute for Gravitational Physics
 * Copyright (C) 2011		Tejun Heo <tj@kernel.org>
 *
 * This file is released under the GPLv2.
 *
 * REQ_{FLUSH|FUA} requests are decomposed to sequences consisted of three
 * optional steps - PREFLUSH, DATA and POSTFLUSH - according to the request
 * properties and hardware capability.
 *
13
14
 * If a request doesn't have data, only REQ_PREFLUSH makes sense, which
 * indicates a simple flush request.  If there is data, REQ_PREFLUSH indicates
15
16
17
18
19
20
21
22
 * that the device cache should be flushed before the data is executed, and
 * REQ_FUA means that the data must be on non-volatile media on request
 * completion.
 *
 * If the device doesn't have writeback cache, FLUSH and FUA don't make any
 * difference.  The requests are either completed immediately if there's no
 * data or executed as normal requests otherwise.
 *
23
 * If the device has writeback cache and supports FUA, REQ_PREFLUSH is
24
25
 * translated to PREFLUSH but REQ_FUA is passed down directly with DATA.
 *
26
27
 * If the device has writeback cache and doesn't support FUA, REQ_PREFLUSH
 * is translated to PREFLUSH and REQ_FUA to POSTFLUSH.
28
29
30
 *
 * The actual execution of flush is double buffered.  Whenever a request
 * needs to execute PRE or POSTFLUSH, it queues at
31
 * fq->flush_queue[fq->flush_pending_idx].  Once certain criteria are met, a
32
 * REQ_OP_FLUSH is issued and the pending_idx is toggled.  When the flush
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
 * completes, all the requests which were pending are proceeded to the next
 * step.  This allows arbitrary merging of different types of FLUSH/FUA
 * requests.
 *
 * Currently, the following conditions are used to determine when to issue
 * flush.
 *
 * C1. At any given time, only one flush shall be in progress.  This makes
 *     double buffering sufficient.
 *
 * C2. Flush is deferred if any request is executing DATA of its sequence.
 *     This avoids issuing separate POSTFLUSHes for requests which shared
 *     PREFLUSH.
 *
 * C3. The second condition is ignored if there is a request which has
 *     waited longer than FLUSH_PENDING_TIMEOUT.  This is to avoid
 *     starvation in the unlikely case where there are continuous stream of
 *     FUA (without FLUSH) requests.
 *
 * For devices which support FUA, it isn't clear whether C2 (and thus C3)
 * is beneficial.
 *
 * Note that a sequenced FLUSH/FUA request with DATA is completed twice.
 * Once while executing DATA and again after the whole sequence is
 * complete.  The first completion updates the contained bio but doesn't
 * finish it so that the bio submitter is notified only after the whole
59
 * sequence is complete.  This is implemented by testing RQF_FLUSH_SEQ in
60
61
62
63
64
 * req_bio_endio().
 *
 * The above peculiarity requires that each FLUSH/FUA request has only one
 * bio attached to it, which is guaranteed as they aren't allowed to be
 * merged in the usual way.
65
 */
66

67
68
69
70
#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/bio.h>
#include <linux/blkdev.h>
71
#include <linux/gfp.h>
72
#include <linux/blk-mq.h>
73
74

#include "blk.h"
75
#include "blk-mq.h"
76
#include "blk-mq-tag.h"
77

78
79
/* FLUSH/FUA sequences */
enum {
80
81
82
83
84
85
86
87
88
89
90
91
92
	REQ_FSEQ_PREFLUSH	= (1 << 0), /* pre-flushing in progress */
	REQ_FSEQ_DATA		= (1 << 1), /* data write in progress */
	REQ_FSEQ_POSTFLUSH	= (1 << 2), /* post-flushing in progress */
	REQ_FSEQ_DONE		= (1 << 3),

	REQ_FSEQ_ACTIONS	= REQ_FSEQ_PREFLUSH | REQ_FSEQ_DATA |
				  REQ_FSEQ_POSTFLUSH,

	/*
	 * If flush has been pending longer than the following timeout,
	 * it's issued even if flush_data requests are still in flight.
	 */
	FLUSH_PENDING_TIMEOUT	= 5 * HZ,
93
94
};

95
96
static bool blk_kick_flush(struct request_queue *q,
			   struct blk_flush_queue *fq);
97

Jens Axboe's avatar
Jens Axboe committed
98
static unsigned int blk_flush_policy(unsigned long fflags, struct request *rq)
99
{
100
	unsigned int policy = 0;
101

102
103
104
	if (blk_rq_sectors(rq))
		policy |= REQ_FSEQ_DATA;

Jens Axboe's avatar
Jens Axboe committed
105
	if (fflags & (1UL << QUEUE_FLAG_WC)) {
106
		if (rq->cmd_flags & REQ_PREFLUSH)
107
			policy |= REQ_FSEQ_PREFLUSH;
Jens Axboe's avatar
Jens Axboe committed
108
109
		if (!(fflags & (1UL << QUEUE_FLAG_FUA)) &&
		    (rq->cmd_flags & REQ_FUA))
110
			policy |= REQ_FSEQ_POSTFLUSH;
111
	}
112
	return policy;
113
114
}

115
static unsigned int blk_flush_cur_seq(struct request *rq)
116
{
117
118
	return 1 << ffz(rq->flush.seq);
}
119

120
121
static void blk_flush_restore_request(struct request *rq)
{
122
	/*
123
124
125
	 * After flush data completion, @rq->bio is %NULL but we need to
	 * complete the bio again.  @rq->biotail is guaranteed to equal the
	 * original @rq->bio.  Restore it.
126
	 */
127
128
129
	rq->bio = rq->biotail;

	/* make @rq a normal request */
130
	rq->rq_flags &= ~RQF_FLUSH_SEQ;
131
	rq->end_io = rq->flush.saved_end_io;
132
133
}

134
static bool blk_flush_queue_rq(struct request *rq, bool add_front)
135
{
136
	if (rq->q->mq_ops) {
137
		blk_mq_add_to_requeue_list(rq, add_front, true);
138
139
		return false;
	} else {
140
141
142
143
		if (add_front)
			list_add(&rq->queuelist, &rq->q->queue_head);
		else
			list_add_tail(&rq->queuelist, &rq->q->queue_head);
144
145
		return true;
	}
146
147
}

148
149
150
/**
 * blk_flush_complete_seq - complete flush sequence
 * @rq: FLUSH/FUA request being sequenced
151
 * @fq: flush queue
152
153
154
155
156
157
158
 * @seq: sequences to complete (mask of %REQ_FSEQ_*, can be zero)
 * @error: whether an error occurred
 *
 * @rq just completed @seq part of its flush sequence, record the
 * completion and trigger the next step.
 *
 * CONTEXT:
159
 * spin_lock_irq(q->queue_lock or fq->mq_flush_lock)
160
161
162
163
 *
 * RETURNS:
 * %true if requests were added to the dispatch queue, %false otherwise.
 */
164
165
166
static bool blk_flush_complete_seq(struct request *rq,
				   struct blk_flush_queue *fq,
				   unsigned int seq, int error)
167
{
168
	struct request_queue *q = rq->q;
169
	struct list_head *pending = &fq->flush_queue[fq->flush_pending_idx];
170
	bool queued = false, kicked;
171
172
173
174
175
176
177
178
179
180
181
182
183
184

	BUG_ON(rq->flush.seq & seq);
	rq->flush.seq |= seq;

	if (likely(!error))
		seq = blk_flush_cur_seq(rq);
	else
		seq = REQ_FSEQ_DONE;

	switch (seq) {
	case REQ_FSEQ_PREFLUSH:
	case REQ_FSEQ_POSTFLUSH:
		/* queue for flush */
		if (list_empty(pending))
185
			fq->flush_pending_since = jiffies;
186
187
188
189
		list_move_tail(&rq->flush.list, pending);
		break;

	case REQ_FSEQ_DATA:
190
		list_move_tail(&rq->flush.list, &fq->flush_data_in_flight);
191
		queued = blk_flush_queue_rq(rq, true);
192
193
194
195
196
197
198
199
200
201
202
203
		break;

	case REQ_FSEQ_DONE:
		/*
		 * @rq was previously adjusted by blk_flush_issue() for
		 * flush sequencing and may already have gone through the
		 * flush data request completion path.  Restore @rq for
		 * normal completion and end it.
		 */
		BUG_ON(!list_empty(&rq->queuelist));
		list_del_init(&rq->flush.list);
		blk_flush_restore_request(rq);
204
		if (q->mq_ops)
205
			blk_mq_end_request(rq, error);
206
207
		else
			__blk_end_request_all(rq, error);
208
209
210
211
212
213
		break;

	default:
		BUG();
	}

214
	kicked = blk_kick_flush(q, fq);
215
	return kicked | queued;
216
217
}

218
static void flush_end_io(struct request *flush_rq, int error)
219
{
220
	struct request_queue *q = flush_rq->q;
221
	struct list_head *running;
222
223
	bool queued = false;
	struct request *rq, *n;
224
	unsigned long flags = 0;
225
	struct blk_flush_queue *fq = blk_get_flush_queue(q, flush_rq->mq_ctx);
226

227
	if (q->mq_ops) {
228
229
230
		struct blk_mq_hw_ctx *hctx;

		/* release the tag's ownership to the req cloned from */
231
		spin_lock_irqsave(&fq->mq_flush_lock, flags);
Christoph Hellwig's avatar
Christoph Hellwig committed
232
		hctx = blk_mq_map_queue(q, flush_rq->mq_ctx->cpu);
233
		blk_mq_tag_set_rq(hctx, flush_rq->tag, fq->orig_rq);
234
		flush_rq->tag = -1;
235
	}
236

237
238
	running = &fq->flush_queue[fq->flush_running_idx];
	BUG_ON(fq->flush_pending_idx == fq->flush_running_idx);
239
240

	/* account completion of the flush request */
241
	fq->flush_running_idx ^= 1;
242
243
244

	if (!q->mq_ops)
		elv_completed_request(q, flush_rq);
245
246
247
248
249
250

	/* and push the waiting requests to the next stage */
	list_for_each_entry_safe(rq, n, running, flush.list) {
		unsigned int seq = blk_flush_cur_seq(rq);

		BUG_ON(seq != REQ_FSEQ_PREFLUSH && seq != REQ_FSEQ_POSTFLUSH);
251
		queued |= blk_flush_complete_seq(rq, fq, seq, error);
252
253
	}

254
	/*
255
256
257
258
259
260
261
262
263
	 * Kick the queue to avoid stall for two cases:
	 * 1. Moving a request silently to empty queue_head may stall the
	 * queue.
	 * 2. When flush request is running in non-queueable queue, the
	 * queue is hold. Restart the queue after flush request is finished
	 * to avoid stall.
	 * This function is called from request completion path and calling
	 * directly into request_fn may confuse the driver.  Always use
	 * kblockd.
264
	 */
265
	if (queued || fq->flush_queue_delayed) {
266
267
		WARN_ON(q->mq_ops);
		blk_run_queue_async(q);
268
	}
269
	fq->flush_queue_delayed = 0;
270
	if (q->mq_ops)
271
		spin_unlock_irqrestore(&fq->mq_flush_lock, flags);
272
273
}

274
275
276
/**
 * blk_kick_flush - consider issuing flush request
 * @q: request_queue being kicked
277
 * @fq: flush queue
278
279
280
281
282
 *
 * Flush related states of @q have changed, consider issuing flush request.
 * Please read the comment at the top of this file for more info.
 *
 * CONTEXT:
283
 * spin_lock_irq(q->queue_lock or fq->mq_flush_lock)
284
285
286
287
 *
 * RETURNS:
 * %true if flush was issued, %false otherwise.
 */
288
static bool blk_kick_flush(struct request_queue *q, struct blk_flush_queue *fq)
289
{
290
	struct list_head *pending = &fq->flush_queue[fq->flush_pending_idx];
291
292
	struct request *first_rq =
		list_first_entry(pending, struct request, flush.list);
293
	struct request *flush_rq = fq->flush_rq;
294
295

	/* C1 described at the top of this file */
296
	if (fq->flush_pending_idx != fq->flush_running_idx || list_empty(pending))
297
298
299
		return false;

	/* C2 and C3 */
300
	if (!list_empty(&fq->flush_data_in_flight) &&
301
	    time_before(jiffies,
302
			fq->flush_pending_since + FLUSH_PENDING_TIMEOUT))
303
304
305
306
307
308
		return false;

	/*
	 * Issue flush and toggle pending_idx.  This makes pending_idx
	 * different from running_idx, which means flush is in flight.
	 */
309
	fq->flush_pending_idx ^= 1;
310

311
	blk_rq_init(q, flush_rq);
312
313
314

	/*
	 * Borrow tag from the first request since they can't
315
316
	 * be in flight at the same time. And acquire the tag's
	 * ownership for flush req.
317
318
	 */
	if (q->mq_ops) {
319
320
		struct blk_mq_hw_ctx *hctx;

321
322
		flush_rq->mq_ctx = first_rq->mq_ctx;
		flush_rq->tag = first_rq->tag;
323
324
		fq->orig_rq = first_rq;

Christoph Hellwig's avatar
Christoph Hellwig committed
325
		hctx = blk_mq_map_queue(q, first_rq->mq_ctx->cpu);
326
		blk_mq_tag_set_rq(hctx, first_rq->tag, flush_rq);
327
	}
328

329
	flush_rq->cmd_type = REQ_TYPE_FS;
330
	flush_rq->cmd_flags = REQ_OP_FLUSH | REQ_PREFLUSH;
331
	flush_rq->rq_flags |= RQF_FLUSH_SEQ;
332
333
	flush_rq->rq_disk = first_rq->rq_disk;
	flush_rq->end_io = flush_end_io;
334

335
	return blk_flush_queue_rq(flush_rq, false);
336
337
}

338
static void flush_data_end_io(struct request *rq, int error)
339
{
340
	struct request_queue *q = rq->q;
341
	struct blk_flush_queue *fq = blk_get_flush_queue(q, NULL);
342

343
344
345
346
	/*
	 * After populating an empty queue, kick it to avoid stall.  Read
	 * the comment in flush_end_io().
	 */
347
	if (blk_flush_complete_seq(rq, fq, REQ_FSEQ_DATA, error))
348
		blk_run_queue_async(q);
349
350
}

351
352
353
354
static void mq_flush_data_end_io(struct request *rq, int error)
{
	struct request_queue *q = rq->q;
	struct blk_mq_hw_ctx *hctx;
355
	struct blk_mq_ctx *ctx = rq->mq_ctx;
356
	unsigned long flags;
357
	struct blk_flush_queue *fq = blk_get_flush_queue(q, ctx);
358

Christoph Hellwig's avatar
Christoph Hellwig committed
359
	hctx = blk_mq_map_queue(q, ctx->cpu);
360
361
362
363
364

	/*
	 * After populating an empty queue, kick it to avoid stall.  Read
	 * the comment in flush_end_io().
	 */
365
	spin_lock_irqsave(&fq->mq_flush_lock, flags);
366
	if (blk_flush_complete_seq(rq, fq, REQ_FSEQ_DATA, error))
367
		blk_mq_run_hw_queue(hctx, true);
368
	spin_unlock_irqrestore(&fq->mq_flush_lock, flags);
369
370
}

371
372
373
374
/**
 * blk_insert_flush - insert a new FLUSH/FUA request
 * @rq: request to insert
 *
375
 * To be called from __elv_add_request() for %ELEVATOR_INSERT_FLUSH insertions.
376
 * or __blk_mq_run_hw_queue() to dispatch request.
377
378
379
380
 * @rq is being submitted.  Analyze what needs to be done and put it on the
 * right queue.
 *
 * CONTEXT:
381
 * spin_lock_irq(q->queue_lock) in !mq case
382
383
 */
void blk_insert_flush(struct request *rq)
384
{
385
	struct request_queue *q = rq->q;
Jens Axboe's avatar
Jens Axboe committed
386
	unsigned long fflags = q->queue_flags;	/* may change, cache */
387
	unsigned int policy = blk_flush_policy(fflags, rq);
388
	struct blk_flush_queue *fq = blk_get_flush_queue(q, rq->mq_ctx);
389

390
391
	/*
	 * @policy now records what operations need to be done.  Adjust
392
	 * REQ_PREFLUSH and FUA for the driver.
393
	 */
394
	rq->cmd_flags &= ~REQ_PREFLUSH;
Jens Axboe's avatar
Jens Axboe committed
395
	if (!(fflags & (1UL << QUEUE_FLAG_FUA)))
396
397
		rq->cmd_flags &= ~REQ_FUA;

398
399
400
401
402
403
404
	/*
	 * REQ_PREFLUSH|REQ_FUA implies REQ_SYNC, so if we clear any
	 * of those flags, we have to set REQ_SYNC to avoid skewing
	 * the request accounting.
	 */
	rq->cmd_flags |= REQ_SYNC;

405
406
407
408
409
410
411
	/*
	 * An empty flush handed down from a stacking driver may
	 * translate into nothing if the underlying device does not
	 * advertise a write-back cache.  In this case, simply
	 * complete the request.
	 */
	if (!policy) {
412
		if (q->mq_ops)
413
			blk_mq_end_request(rq, 0);
414
415
		else
			__blk_end_bidi_request(rq, 0, 0, 0);
416
417
418
		return;
	}

419
	BUG_ON(rq->bio != rq->biotail); /*assumes zero or single bio rq */
420

421
422
423
424
425
426
427
	/*
	 * If there's data but flush is not necessary, the request can be
	 * processed directly without going through flush machinery.  Queue
	 * for normal execution.
	 */
	if ((policy & REQ_FSEQ_DATA) &&
	    !(policy & (REQ_FSEQ_PREFLUSH | REQ_FSEQ_POSTFLUSH))) {
428
		if (q->mq_ops) {
429
			blk_mq_insert_request(rq, false, true, false);
430
		} else
431
			list_add_tail(&rq->queuelist, &q->queue_head);
432
		return;
433
	}
434

435
436
437
438
439
440
	/*
	 * @rq should go through flush machinery.  Mark it part of flush
	 * sequence and submit for further processing.
	 */
	memset(&rq->flush, 0, sizeof(rq->flush));
	INIT_LIST_HEAD(&rq->flush.list);
441
	rq->rq_flags |= RQF_FLUSH_SEQ;
442
	rq->flush.saved_end_io = rq->end_io; /* Usually NULL */
443
444
445
	if (q->mq_ops) {
		rq->end_io = mq_flush_data_end_io;

446
		spin_lock_irq(&fq->mq_flush_lock);
447
		blk_flush_complete_seq(rq, fq, REQ_FSEQ_ACTIONS & ~policy, 0);
448
		spin_unlock_irq(&fq->mq_flush_lock);
449
450
		return;
	}
451
452
	rq->end_io = flush_data_end_io;

453
	blk_flush_complete_seq(rq, fq, REQ_FSEQ_ACTIONS & ~policy, 0);
454
455
456
457
458
}

/**
 * blkdev_issue_flush - queue a flush
 * @bdev:	blockdev to issue flush for
459
 * @gfp_mask:	memory allocation flags (for bio_alloc)
460
461
462
463
464
 * @error_sector:	error sector
 *
 * Description:
 *    Issue a flush for the block device in question. Caller can supply
 *    room for storing the error offset in case of a flush error, if they
465
466
 *    wish to. If WAIT flag is not passed then caller may check only what
 *    request was pushed in some internal queue for later handling.
467
 */
468
int blkdev_issue_flush(struct block_device *bdev, gfp_t gfp_mask,
469
		sector_t *error_sector)
470
471
472
{
	struct request_queue *q;
	struct bio *bio;
473
	int ret = 0;
474
475
476
477
478
479
480
481

	if (bdev->bd_disk == NULL)
		return -ENXIO;

	q = bdev_get_queue(bdev);
	if (!q)
		return -ENXIO;

482
483
484
485
	/*
	 * some block devices may not have their queue correctly set up here
	 * (e.g. loop device without a backing file) and so issuing a flush
	 * here will panic. Ensure there is a request function before issuing
486
	 * the flush.
487
488
489
490
	 */
	if (!q->make_request_fn)
		return -ENXIO;

491
	bio = bio_alloc(gfp_mask, 0);
492
	bio->bi_bdev = bdev;
493
	bio->bi_opf = REQ_OP_WRITE | REQ_PREFLUSH;
494

495
	ret = submit_bio_wait(bio);
496
497
498
499
500
501
502

	/*
	 * The driver must store the error location in ->bi_sector, if
	 * it supports it. For non-stacked drivers, this should be
	 * copied from blk_rq_pos(rq).
	 */
	if (error_sector)
503
		*error_sector = bio->bi_iter.bi_sector;
504
505
506
507
508

	bio_put(bio);
	return ret;
}
EXPORT_SYMBOL(blkdev_issue_flush);
509

510
511
struct blk_flush_queue *blk_alloc_flush_queue(struct request_queue *q,
		int node, int cmd_size)
512
{
513
514
	struct blk_flush_queue *fq;
	int rq_sz = sizeof(struct request);
515

516
	fq = kzalloc_node(sizeof(*fq), GFP_KERNEL, node);
517
518
	if (!fq)
		goto fail;
519

520
521
	if (q->mq_ops) {
		spin_lock_init(&fq->mq_flush_lock);
522
		rq_sz = round_up(rq_sz + cmd_size, cache_line_size());
523
524
	}

525
	fq->flush_rq = kzalloc_node(rq_sz, GFP_KERNEL, node);
526
527
528
529
530
531
532
533
534
535
536
537
538
	if (!fq->flush_rq)
		goto fail_rq;

	INIT_LIST_HEAD(&fq->flush_queue[0]);
	INIT_LIST_HEAD(&fq->flush_queue[1]);
	INIT_LIST_HEAD(&fq->flush_data_in_flight);

	return fq;

 fail_rq:
	kfree(fq);
 fail:
	return NULL;
539
}
540

541
void blk_free_flush_queue(struct blk_flush_queue *fq)
542
{
543
544
545
	/* bio based request queue hasn't flush queue */
	if (!fq)
		return;
546

547
548
549
	kfree(fq->flush_rq);
	kfree(fq);
}