blk-cgroup.c 40.4 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
13
/*
 * Common Block IO controller cgroup interface
 *
 * Based on ideas and code from CFQ, CFS and BFQ:
 * Copyright (C) 2003 Jens Axboe <axboe@kernel.dk>
 *
 * Copyright (C) 2008 Fabio Checconi <fabio@gandalf.sssup.it>
 *		      Paolo Valente <paolo.valente@unimore.it>
 *
 * Copyright (C) 2009 Vivek Goyal <vgoyal@redhat.com>
 * 	              Nauman Rafique <nauman@google.com>
 */
#include <linux/ioprio.h>
14
15
#include <linux/seq_file.h>
#include <linux/kdev_t.h>
16
#include <linux/module.h>
17
#include <linux/err.h>
18
#include <linux/blkdev.h>
19
#include <linux/slab.h>
20
#include <linux/genhd.h>
21
#include <linux/delay.h>
Tejun Heo's avatar
Tejun Heo committed
22
#include <linux/atomic.h>
23
#include "blk-cgroup.h"
24
#include "blk.h"
25

26
27
#define MAX_KEY_LEN 100

28
29
static DEFINE_SPINLOCK(blkio_list_lock);
static LIST_HEAD(blkio_list);
30

31
32
33
static DEFINE_MUTEX(all_q_mutex);
static LIST_HEAD(all_q_list);

34
35
36
37
38
39
40
/* List of groups pending per cpu stats allocation */
static DEFINE_SPINLOCK(alloc_list_lock);
static LIST_HEAD(alloc_list);

static void blkio_stat_alloc_fn(struct work_struct *);
static DECLARE_DELAYED_WORK(blkio_stat_alloc_work, blkio_stat_alloc_fn);

41
struct blkio_cgroup blkio_root_cgroup = { .weight = 2*BLKIO_WEIGHT_DEFAULT };
42
43
EXPORT_SYMBOL_GPL(blkio_root_cgroup);

44
45
static struct blkio_policy_type *blkio_policy[BLKIO_NR_POLICIES];

46
47
48
49
50
struct blkio_cgroup *cgroup_to_blkio_cgroup(struct cgroup *cgroup)
{
	return container_of(cgroup_subsys_state(cgroup, blkio_subsys_id),
			    struct blkio_cgroup, css);
}
51
EXPORT_SYMBOL_GPL(cgroup_to_blkio_cgroup);
52

53
static struct blkio_cgroup *task_blkio_cgroup(struct task_struct *tsk)
54
55
56
57
{
	return container_of(task_subsys_state(tsk, blkio_subsys_id),
			    struct blkio_cgroup, css);
}
58
59
60
61
62
63
64
65

struct blkio_cgroup *bio_blkio_cgroup(struct bio *bio)
{
	if (bio && bio->bi_css)
		return container_of(bio->bi_css, struct blkio_cgroup, css);
	return task_blkio_cgroup(current);
}
EXPORT_SYMBOL_GPL(bio_blkio_cgroup);
66

67
68
static inline void blkio_update_group_weight(struct blkio_group *blkg,
					     int plid, unsigned int weight)
69
70
71
72
73
{
	struct blkio_policy_type *blkiop;

	list_for_each_entry(blkiop, &blkio_list, list) {
		/* If this policy does not own the blkg, do not send updates */
74
		if (blkiop->plid != plid)
75
76
			continue;
		if (blkiop->ops.blkio_update_group_weight_fn)
77
			blkiop->ops.blkio_update_group_weight_fn(blkg->q,
78
							blkg, weight);
79
80
81
	}
}

82
static inline void blkio_update_group_bps(struct blkio_group *blkg, int plid,
83
					  u64 bps, int rw)
84
85
86
87
88
89
{
	struct blkio_policy_type *blkiop;

	list_for_each_entry(blkiop, &blkio_list, list) {

		/* If this policy does not own the blkg, do not send updates */
90
		if (blkiop->plid != plid)
91
92
			continue;

93
		if (rw == READ && blkiop->ops.blkio_update_group_read_bps_fn)
94
			blkiop->ops.blkio_update_group_read_bps_fn(blkg->q,
95
								blkg, bps);
96

97
		if (rw == WRITE && blkiop->ops.blkio_update_group_write_bps_fn)
98
			blkiop->ops.blkio_update_group_write_bps_fn(blkg->q,
99
								blkg, bps);
100
101
102
	}
}

103
104
static inline void blkio_update_group_iops(struct blkio_group *blkg, int plid,
					   u64 iops, int rw)
105
106
107
108
109
110
{
	struct blkio_policy_type *blkiop;

	list_for_each_entry(blkiop, &blkio_list, list) {

		/* If this policy does not own the blkg, do not send updates */
111
		if (blkiop->plid != plid)
112
113
			continue;

114
		if (rw == READ && blkiop->ops.blkio_update_group_read_iops_fn)
115
			blkiop->ops.blkio_update_group_read_iops_fn(blkg->q,
116
								blkg, iops);
117

118
		if (rw == WRITE && blkiop->ops.blkio_update_group_write_iops_fn)
119
			blkiop->ops.blkio_update_group_write_iops_fn(blkg->q,
120
								blkg,iops);
121
122
123
	}
}

124
#ifdef CONFIG_DEBUG_BLK_CGROUP
125
/* This should be called with the queue_lock held. */
126
static void blkio_set_start_group_wait_time(struct blkio_group *blkg,
127
128
					    struct blkio_policy_type *pol,
					    struct blkio_group *curr_blkg)
129
{
130
	struct blkg_policy_data *pd = blkg->pd[pol->plid];
131
132

	if (blkio_blkg_waiting(&pd->stats))
133
134
135
		return;
	if (blkg == curr_blkg)
		return;
136
137
	pd->stats.start_group_wait_time = sched_clock();
	blkio_mark_blkg_waiting(&pd->stats);
138
139
}

140
/* This should be called with the queue_lock held. */
141
142
143
144
145
146
147
148
149
static void blkio_update_group_wait_time(struct blkio_group_stats *stats)
{
	unsigned long long now;

	if (!blkio_blkg_waiting(stats))
		return;

	now = sched_clock();
	if (time_after64(now, stats->start_group_wait_time))
150
151
		blkg_stat_add(&stats->group_wait_time,
			      now - stats->start_group_wait_time);
152
153
154
	blkio_clear_blkg_waiting(stats);
}

155
/* This should be called with the queue_lock held. */
156
157
158
159
160
161
162
163
164
static void blkio_end_empty_time(struct blkio_group_stats *stats)
{
	unsigned long long now;

	if (!blkio_blkg_empty(stats))
		return;

	now = sched_clock();
	if (time_after64(now, stats->start_empty_time))
165
166
		blkg_stat_add(&stats->empty_time,
			      now - stats->start_empty_time);
167
168
169
	blkio_clear_blkg_empty(stats);
}

170
171
void blkiocg_update_set_idle_time_stats(struct blkio_group *blkg,
					struct blkio_policy_type *pol)
172
{
173
	struct blkio_group_stats *stats = &blkg->pd[pol->plid]->stats;
174

175
176
177
178
179
	lockdep_assert_held(blkg->q->queue_lock);
	BUG_ON(blkio_blkg_idling(stats));

	stats->start_idle_time = sched_clock();
	blkio_mark_blkg_idling(stats);
180
181
182
}
EXPORT_SYMBOL_GPL(blkiocg_update_set_idle_time_stats);

183
184
void blkiocg_update_idle_time_stats(struct blkio_group *blkg,
				    struct blkio_policy_type *pol)
185
{
186
187
188
	struct blkio_group_stats *stats = &blkg->pd[pol->plid]->stats;

	lockdep_assert_held(blkg->q->queue_lock);
189
190

	if (blkio_blkg_idling(stats)) {
191
192
		unsigned long long now = sched_clock();

193
194
195
		if (time_after64(now, stats->start_idle_time))
			blkg_stat_add(&stats->idle_time,
				      now - stats->start_idle_time);
196
197
198
199
200
		blkio_clear_blkg_idling(stats);
	}
}
EXPORT_SYMBOL_GPL(blkiocg_update_idle_time_stats);

201
202
void blkiocg_update_avg_queue_size_stats(struct blkio_group *blkg,
					 struct blkio_policy_type *pol)
203
{
204
	struct blkio_group_stats *stats = &blkg->pd[pol->plid]->stats;
205

206
207
	lockdep_assert_held(blkg->q->queue_lock);

208
209
210
	blkg_stat_add(&stats->avg_queue_size_sum,
		      blkg_rwstat_sum(&stats->queued));
	blkg_stat_add(&stats->avg_queue_size_samples, 1);
211
	blkio_update_group_wait_time(stats);
212
}
213
214
EXPORT_SYMBOL_GPL(blkiocg_update_avg_queue_size_stats);

215
216
void blkiocg_set_start_empty_time(struct blkio_group *blkg,
				  struct blkio_policy_type *pol)
Divyesh Shah's avatar
Divyesh Shah committed
217
{
218
	struct blkio_group_stats *stats = &blkg->pd[pol->plid]->stats;
Divyesh Shah's avatar
Divyesh Shah committed
219

220
	lockdep_assert_held(blkg->q->queue_lock);
Divyesh Shah's avatar
Divyesh Shah committed
221

222
	if (blkg_rwstat_sum(&stats->queued))
Divyesh Shah's avatar
Divyesh Shah committed
223
224
225
		return;

	/*
226
227
228
	 * group is already marked empty. This can happen if cfqq got new
	 * request in parent group and moved to this group while being added
	 * to service tree. Just ignore the event and move on.
Divyesh Shah's avatar
Divyesh Shah committed
229
	 */
230
	if (blkio_blkg_empty(stats))
231
232
		return;

Divyesh Shah's avatar
Divyesh Shah committed
233
234
235
236
237
	stats->start_empty_time = sched_clock();
	blkio_mark_blkg_empty(stats);
}
EXPORT_SYMBOL_GPL(blkiocg_set_start_empty_time);

238
void blkiocg_update_dequeue_stats(struct blkio_group *blkg,
239
240
				  struct blkio_policy_type *pol,
				  unsigned long dequeue)
241
{
242
	struct blkg_policy_data *pd = blkg->pd[pol->plid];
243

244
245
	lockdep_assert_held(blkg->q->queue_lock);

246
	blkg_stat_add(&pd->stats.dequeue, dequeue);
247
248
}
EXPORT_SYMBOL_GPL(blkiocg_update_dequeue_stats);
249
250
#else
static inline void blkio_set_start_group_wait_time(struct blkio_group *blkg,
251
252
253
					struct blkio_policy_type *pol,
					struct blkio_group *curr_blkg) { }
static inline void blkio_end_empty_time(struct blkio_group_stats *stats) { }
254
255
#endif

256
void blkiocg_update_io_add_stats(struct blkio_group *blkg,
257
258
259
				 struct blkio_policy_type *pol,
				 struct blkio_group *curr_blkg, bool direction,
				 bool sync)
260
{
261
	struct blkio_group_stats *stats = &blkg->pd[pol->plid]->stats;
262
	int rw = (direction ? REQ_WRITE : 0) | (sync ? REQ_SYNC : 0);
263
264
265

	lockdep_assert_held(blkg->q->queue_lock);

266
	blkg_rwstat_add(&stats->queued, rw, 1);
267
	blkio_end_empty_time(stats);
268
	blkio_set_start_group_wait_time(blkg, pol, curr_blkg);
269
}
270
EXPORT_SYMBOL_GPL(blkiocg_update_io_add_stats);
271

272
void blkiocg_update_io_remove_stats(struct blkio_group *blkg,
273
274
				    struct blkio_policy_type *pol,
				    bool direction, bool sync)
275
{
276
	struct blkio_group_stats *stats = &blkg->pd[pol->plid]->stats;
277
	int rw = (direction ? REQ_WRITE : 0) | (sync ? REQ_SYNC : 0);
278
279

	lockdep_assert_held(blkg->q->queue_lock);
280

281
	blkg_rwstat_add(&stats->queued, rw, -1);
282
}
283
EXPORT_SYMBOL_GPL(blkiocg_update_io_remove_stats);
284

285
286
287
288
void blkiocg_update_timeslice_used(struct blkio_group *blkg,
				   struct blkio_policy_type *pol,
				   unsigned long time,
				   unsigned long unaccounted_time)
289
{
290
291
292
	struct blkio_group_stats *stats = &blkg->pd[pol->plid]->stats;

	lockdep_assert_held(blkg->q->queue_lock);
293

294
	blkg_stat_add(&stats->time, time);
295
#ifdef CONFIG_DEBUG_BLK_CGROUP
296
	blkg_stat_add(&stats->unaccounted_time, unaccounted_time);
297
#endif
298
}
299
EXPORT_SYMBOL_GPL(blkiocg_update_timeslice_used);
300

301
302
303
304
/*
 * should be called under rcu read lock or queue lock to make sure blkg pointer
 * is valid.
 */
305
void blkiocg_update_dispatch_stats(struct blkio_group *blkg,
306
307
				   struct blkio_policy_type *pol,
				   uint64_t bytes, bool direction, bool sync)
308
{
309
	int rw = (direction ? REQ_WRITE : 0) | (sync ? REQ_SYNC : 0);
310
	struct blkg_policy_data *pd = blkg->pd[pol->plid];
311
	struct blkio_group_stats_cpu *stats_cpu;
312
313
	unsigned long flags;

314
315
316
317
	/* If per cpu stats are not allocated yet, don't do any accounting. */
	if (pd->stats_cpu == NULL)
		return;

318
319
320
321
322
323
	/*
	 * Disabling interrupts to provide mutual exclusion between two
	 * writes on same cpu. It probably is not needed for 64bit. Not
	 * optimizing that case yet.
	 */
	local_irq_save(flags);
324

325
	stats_cpu = this_cpu_ptr(pd->stats_cpu);
326

327
328
329
330
	blkg_stat_add(&stats_cpu->sectors, bytes >> 9);
	blkg_rwstat_add(&stats_cpu->serviced, rw, 1);
	blkg_rwstat_add(&stats_cpu->service_bytes, rw, bytes);

331
	local_irq_restore(flags);
332
}
333
EXPORT_SYMBOL_GPL(blkiocg_update_dispatch_stats);
334

335
void blkiocg_update_completion_stats(struct blkio_group *blkg,
336
337
338
339
				     struct blkio_policy_type *pol,
				     uint64_t start_time,
				     uint64_t io_start_time, bool direction,
				     bool sync)
340
{
341
	struct blkio_group_stats *stats = &blkg->pd[pol->plid]->stats;
342
	unsigned long long now = sched_clock();
343
	int rw = (direction ? REQ_WRITE : 0) | (sync ? REQ_SYNC : 0);
344

345
346
	lockdep_assert_held(blkg->q->queue_lock);

347
	if (time_after64(now, io_start_time))
348
		blkg_rwstat_add(&stats->service_time, rw, now - io_start_time);
349
	if (time_after64(io_start_time, start_time))
350
351
		blkg_rwstat_add(&stats->wait_time, rw,
				io_start_time - start_time);
352
}
353
EXPORT_SYMBOL_GPL(blkiocg_update_completion_stats);
354

355
/*  Merged stats are per cpu.  */
356
357
358
void blkiocg_update_io_merged_stats(struct blkio_group *blkg,
				    struct blkio_policy_type *pol,
				    bool direction, bool sync)
Divyesh Shah's avatar
Divyesh Shah committed
359
{
360
	struct blkio_group_stats *stats = &blkg->pd[pol->plid]->stats;
361
	int rw = (direction ? REQ_WRITE : 0) | (sync ? REQ_SYNC : 0);
362
363

	lockdep_assert_held(blkg->q->queue_lock);
Divyesh Shah's avatar
Divyesh Shah committed
364

365
	blkg_rwstat_add(&stats->merged, rw, 1);
Divyesh Shah's avatar
Divyesh Shah committed
366
367
368
}
EXPORT_SYMBOL_GPL(blkiocg_update_io_merged_stats);

369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
/*
 * Worker for allocating per cpu stat for blk groups. This is scheduled on
 * the system_nrt_wq once there are some groups on the alloc_list waiting
 * for allocation.
 */
static void blkio_stat_alloc_fn(struct work_struct *work)
{
	static void *pcpu_stats[BLKIO_NR_POLICIES];
	struct delayed_work *dwork = to_delayed_work(work);
	struct blkio_group *blkg;
	int i;
	bool empty = false;

alloc_stats:
	for (i = 0; i < BLKIO_NR_POLICIES; i++) {
		if (pcpu_stats[i] != NULL)
			continue;

		pcpu_stats[i] = alloc_percpu(struct blkio_group_stats_cpu);

		/* Allocation failed. Try again after some time. */
		if (pcpu_stats[i] == NULL) {
			queue_delayed_work(system_nrt_wq, dwork,
						msecs_to_jiffies(10));
			return;
		}
	}

	spin_lock_irq(&blkio_list_lock);
	spin_lock(&alloc_list_lock);

	/* cgroup got deleted or queue exited. */
	if (!list_empty(&alloc_list)) {
		blkg = list_first_entry(&alloc_list, struct blkio_group,
						alloc_node);
		for (i = 0; i < BLKIO_NR_POLICIES; i++) {
			struct blkg_policy_data *pd = blkg->pd[i];

			if (blkio_policy[i] && pd && !pd->stats_cpu)
				swap(pd->stats_cpu, pcpu_stats[i]);
		}

		list_del_init(&blkg->alloc_node);
	}

	empty = list_empty(&alloc_list);

	spin_unlock(&alloc_list_lock);
	spin_unlock_irq(&blkio_list_lock);

	if (!empty)
		goto alloc_stats;
}

423
424
425
426
427
428
429
430
/**
 * blkg_free - free a blkg
 * @blkg: blkg to free
 *
 * Free @blkg which may be partially allocated.
 */
static void blkg_free(struct blkio_group *blkg)
{
431
	int i;
432
433
434
435

	if (!blkg)
		return;

436
437
438
439
440
441
442
	for (i = 0; i < BLKIO_NR_POLICIES; i++) {
		struct blkg_policy_data *pd = blkg->pd[i];

		if (pd) {
			free_percpu(pd->stats_cpu);
			kfree(pd);
		}
443
	}
444

445
	kfree(blkg);
446
447
448
449
450
451
452
}

/**
 * blkg_alloc - allocate a blkg
 * @blkcg: block cgroup the new blkg is associated with
 * @q: request_queue the new blkg is associated with
 *
453
 * Allocate a new blkg assocating @blkcg and @q.
454
455
 */
static struct blkio_group *blkg_alloc(struct blkio_cgroup *blkcg,
456
				      struct request_queue *q)
457
458
{
	struct blkio_group *blkg;
459
	int i;
460
461
462
463
464
465

	/* alloc and init base part */
	blkg = kzalloc_node(sizeof(*blkg), GFP_ATOMIC, q->node);
	if (!blkg)
		return NULL;

466
	blkg->q = q;
467
	INIT_LIST_HEAD(&blkg->q_node);
468
	INIT_LIST_HEAD(&blkg->alloc_node);
469
	blkg->blkcg = blkcg;
Tejun Heo's avatar
Tejun Heo committed
470
	blkg->refcnt = 1;
471
472
	cgroup_path(blkcg->css.cgroup, blkg->path, sizeof(blkg->path));

473
474
475
	for (i = 0; i < BLKIO_NR_POLICIES; i++) {
		struct blkio_policy_type *pol = blkio_policy[i];
		struct blkg_policy_data *pd;
476

477
478
479
480
481
482
483
484
485
486
		if (!pol)
			continue;

		/* alloc per-policy data and attach it to blkg */
		pd = kzalloc_node(sizeof(*pd) + pol->pdata_size, GFP_ATOMIC,
				  q->node);
		if (!pd) {
			blkg_free(blkg);
			return NULL;
		}
487

488
489
		blkg->pd[i] = pd;
		pd->blkg = blkg;
490
491
	}

492
	/* invoke per-policy init */
493
494
495
496
497
498
499
	for (i = 0; i < BLKIO_NR_POLICIES; i++) {
		struct blkio_policy_type *pol = blkio_policy[i];

		if (pol)
			pol->ops.blkio_init_group_fn(blkg);
	}

500
501
502
	return blkg;
}

503
504
505
506
struct blkio_group *blkg_lookup_create(struct blkio_cgroup *blkcg,
				       struct request_queue *q,
				       bool for_root)
	__releases(q->queue_lock) __acquires(q->queue_lock)
507
{
508
	struct blkio_group *blkg;
509

510
511
512
513
514
515
516
517
518
519
520
521
	WARN_ON_ONCE(!rcu_read_lock_held());
	lockdep_assert_held(q->queue_lock);

	/*
	 * This could be the first entry point of blkcg implementation and
	 * we shouldn't allow anything to go through for a bypassing queue.
	 * The following can be removed if blkg lookup is guaranteed to
	 * fail on a bypassing queue.
	 */
	if (unlikely(blk_queue_bypass(q)) && !for_root)
		return ERR_PTR(blk_queue_dead(q) ? -EINVAL : -EBUSY);

522
	blkg = blkg_lookup(blkcg, q);
523
524
525
	if (blkg)
		return blkg;

526
	/* blkg holds a reference to blkcg */
527
528
529
530
531
532
	if (!css_tryget(&blkcg->css))
		return ERR_PTR(-EINVAL);

	/*
	 * Allocate and initialize.
	 */
533
	blkg = blkg_alloc(blkcg, q);
534
535

	/* did alloc fail? */
536
	if (unlikely(!blkg)) {
537
538
539
540
541
542
		blkg = ERR_PTR(-ENOMEM);
		goto out;
	}

	/* insert */
	spin_lock(&blkcg->lock);
543
	hlist_add_head_rcu(&blkg->blkcg_node, &blkcg->blkg_list);
544
	list_add(&blkg->q_node, &q->blkg_list);
545
	spin_unlock(&blkcg->lock);
546
547
548
549
550
551

	spin_lock(&alloc_list_lock);
	list_add(&blkg->alloc_node, &alloc_list);
	/* Queue per cpu stat allocation from worker thread. */
	queue_delayed_work(system_nrt_wq, &blkio_stat_alloc_work, 0);
	spin_unlock(&alloc_list_lock);
552
553
out:
	return blkg;
554
}
555
EXPORT_SYMBOL_GPL(blkg_lookup_create);
556
557

/* called under rcu_read_lock(). */
558
struct blkio_group *blkg_lookup(struct blkio_cgroup *blkcg,
559
				struct request_queue *q)
560
561
562
563
{
	struct blkio_group *blkg;
	struct hlist_node *n;

564
	hlist_for_each_entry_rcu(blkg, n, &blkcg->blkg_list, blkcg_node)
565
		if (blkg->q == q)
566
567
568
			return blkg;
	return NULL;
}
569
EXPORT_SYMBOL_GPL(blkg_lookup);
570

571
static void blkg_destroy(struct blkio_group *blkg)
572
573
{
	struct request_queue *q = blkg->q;
574
	struct blkio_cgroup *blkcg = blkg->blkcg;
575
576

	lockdep_assert_held(q->queue_lock);
577
	lockdep_assert_held(&blkcg->lock);
578
579

	/* Something wrong if we are trying to remove same group twice */
580
	WARN_ON_ONCE(list_empty(&blkg->q_node));
581
	WARN_ON_ONCE(hlist_unhashed(&blkg->blkcg_node));
582
	list_del_init(&blkg->q_node);
583
	hlist_del_init_rcu(&blkg->blkcg_node);
584

585
586
587
588
	spin_lock(&alloc_list_lock);
	list_del_init(&blkg->alloc_node);
	spin_unlock(&alloc_list_lock);

589
590
591
592
593
594
595
	/*
	 * Put the reference taken at the time of creation so that when all
	 * queues are gone, group can be destroyed.
	 */
	blkg_put(blkg);
}

596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
/*
 * XXX: This updates blkg policy data in-place for root blkg, which is
 * necessary across elevator switch and policy registration as root blkgs
 * aren't shot down.  This broken and racy implementation is temporary.
 * Eventually, blkg shoot down will be replaced by proper in-place update.
 */
void update_root_blkg_pd(struct request_queue *q, enum blkio_policy_id plid)
{
	struct blkio_policy_type *pol = blkio_policy[plid];
	struct blkio_group *blkg = blkg_lookup(&blkio_root_cgroup, q);
	struct blkg_policy_data *pd;

	if (!blkg)
		return;

	kfree(blkg->pd[plid]);
	blkg->pd[plid] = NULL;

	if (!pol)
		return;

	pd = kzalloc(sizeof(*pd) + pol->pdata_size, GFP_KERNEL);
	WARN_ON_ONCE(!pd);

	pd->stats_cpu = alloc_percpu(struct blkio_group_stats_cpu);
	WARN_ON_ONCE(!pd->stats_cpu);

	blkg->pd[plid] = pd;
	pd->blkg = blkg;
	pol->ops.blkio_init_group_fn(blkg);
}
EXPORT_SYMBOL_GPL(update_root_blkg_pd);

629
630
631
632
633
634
635
636
/**
 * blkg_destroy_all - destroy all blkgs associated with a request_queue
 * @q: request_queue of interest
 * @destroy_root: whether to destroy root blkg or not
 *
 * Destroy blkgs associated with @q.  If @destroy_root is %true, all are
 * destroyed; otherwise, root blkg is left alone.
 */
637
void blkg_destroy_all(struct request_queue *q, bool destroy_root)
638
{
639
	struct blkio_group *blkg, *n;
640

641
	spin_lock_irq(q->queue_lock);
642

643
644
	list_for_each_entry_safe(blkg, n, &q->blkg_list, q_node) {
		struct blkio_cgroup *blkcg = blkg->blkcg;
645

646
647
648
		/* skip root? */
		if (!destroy_root && blkg->blkcg == &blkio_root_cgroup)
			continue;
649

650
651
652
		spin_lock(&blkcg->lock);
		blkg_destroy(blkg);
		spin_unlock(&blkcg->lock);
653
	}
654
655

	spin_unlock_irq(q->queue_lock);
656
}
657
EXPORT_SYMBOL_GPL(blkg_destroy_all);
658

Tejun Heo's avatar
Tejun Heo committed
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
static void blkg_rcu_free(struct rcu_head *rcu_head)
{
	blkg_free(container_of(rcu_head, struct blkio_group, rcu_head));
}

void __blkg_release(struct blkio_group *blkg)
{
	/* release the extra blkcg reference this blkg has been holding */
	css_put(&blkg->blkcg->css);

	/*
	 * A group is freed in rcu manner. But having an rcu lock does not
	 * mean that one can access all the fields of blkg and assume these
	 * are valid. For example, don't try to follow throtl_data and
	 * request queue links.
	 *
	 * Having a reference to blkg under an rcu allows acess to only
	 * values local to groups like group stats and group rate limits
	 */
	call_rcu(&blkg->rcu_head, blkg_rcu_free);
}
EXPORT_SYMBOL_GPL(__blkg_release);

682
static void blkio_reset_stats_cpu(struct blkio_group *blkg, int plid)
683
{
684
	struct blkg_policy_data *pd = blkg->pd[plid];
Tejun Heo's avatar
Tejun Heo committed
685
	int cpu;
686
687
688

	if (pd->stats_cpu == NULL)
		return;
Tejun Heo's avatar
Tejun Heo committed
689
690
691
692
693

	for_each_possible_cpu(cpu) {
		struct blkio_group_stats_cpu *sc =
			per_cpu_ptr(pd->stats_cpu, cpu);

694
695
696
		blkg_rwstat_reset(&sc->service_bytes);
		blkg_rwstat_reset(&sc->serviced);
		blkg_stat_reset(&sc->sectors);
697
698
699
	}
}

700
static int
701
blkiocg_reset_stats(struct cgroup *cgroup, struct cftype *cftype, u64 val)
702
{
Tejun Heo's avatar
Tejun Heo committed
703
	struct blkio_cgroup *blkcg = cgroup_to_blkio_cgroup(cgroup);
704
705
706
	struct blkio_group *blkg;
	struct hlist_node *n;

707
	spin_lock(&blkio_list_lock);
708
	spin_lock_irq(&blkcg->lock);
Tejun Heo's avatar
Tejun Heo committed
709
710
711
712
713
714

	/*
	 * Note that stat reset is racy - it doesn't synchronize against
	 * stat updates.  This is a debug feature which shouldn't exist
	 * anyway.  If you get hit by a race, retry.
	 */
715
	hlist_for_each_entry(blkg, n, &blkcg->blkg_list, blkcg_node) {
716
		struct blkio_policy_type *pol;
717

718
719
		list_for_each_entry(pol, &blkio_list, list) {
			struct blkg_policy_data *pd = blkg->pd[pol->plid];
Tejun Heo's avatar
Tejun Heo committed
720
721
722
			struct blkio_group_stats *stats = &pd->stats;

			/* queued stats shouldn't be cleared */
723
724
725
726
			blkg_rwstat_reset(&stats->merged);
			blkg_rwstat_reset(&stats->service_time);
			blkg_rwstat_reset(&stats->wait_time);
			blkg_stat_reset(&stats->time);
727
#ifdef CONFIG_DEBUG_BLK_CGROUP
728
729
730
731
732
733
734
			blkg_stat_reset(&stats->unaccounted_time);
			blkg_stat_reset(&stats->avg_queue_size_sum);
			blkg_stat_reset(&stats->avg_queue_size_samples);
			blkg_stat_reset(&stats->dequeue);
			blkg_stat_reset(&stats->group_wait_time);
			blkg_stat_reset(&stats->idle_time);
			blkg_stat_reset(&stats->empty_time);
735
#endif
736
737
			blkio_reset_stats_cpu(blkg, pol->plid);
		}
738
	}
739

740
	spin_unlock_irq(&blkcg->lock);
741
	spin_unlock(&blkio_list_lock);
742
743
744
	return 0;
}

745
static const char *blkg_dev_name(struct blkio_group *blkg)
746
{
747
748
749
750
	/* some drivers (floppy) instantiate a queue w/o disk registered */
	if (blkg->q->backing_dev_info.dev)
		return dev_name(blkg->q->backing_dev_info.dev);
	return NULL;
751
752
}

753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
/**
 * blkcg_print_blkgs - helper for printing per-blkg data
 * @sf: seq_file to print to
 * @blkcg: blkcg of interest
 * @prfill: fill function to print out a blkg
 * @pol: policy in question
 * @data: data to be passed to @prfill
 * @show_total: to print out sum of prfill return values or not
 *
 * This function invokes @prfill on each blkg of @blkcg if pd for the
 * policy specified by @pol exists.  @prfill is invoked with @sf, the
 * policy data and @data.  If @show_total is %true, the sum of the return
 * values from @prfill is printed with "Total" label at the end.
 *
 * This is to be used to construct print functions for
 * cftype->read_seq_string method.
 */
static void blkcg_print_blkgs(struct seq_file *sf, struct blkio_cgroup *blkcg,
			      u64 (*prfill)(struct seq_file *,
					    struct blkg_policy_data *, int),
			      int pol, int data, bool show_total)
774
{
775
776
777
	struct blkio_group *blkg;
	struct hlist_node *n;
	u64 total = 0;
778

779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
	spin_lock_irq(&blkcg->lock);
	hlist_for_each_entry(blkg, n, &blkcg->blkg_list, blkcg_node)
		if (blkg->pd[pol])
			total += prfill(sf, blkg->pd[pol], data);
	spin_unlock_irq(&blkcg->lock);

	if (show_total)
		seq_printf(sf, "Total %llu\n", (unsigned long long)total);
}

/**
 * __blkg_prfill_u64 - prfill helper for a single u64 value
 * @sf: seq_file to print to
 * @pd: policy data of interest
 * @v: value to print
 *
 * Print @v to @sf for the device assocaited with @pd.
 */
static u64 __blkg_prfill_u64(struct seq_file *sf, struct blkg_policy_data *pd,
			     u64 v)
{
	const char *dname = blkg_dev_name(pd->blkg);

	if (!dname)
		return 0;

	seq_printf(sf, "%s %llu\n", dname, (unsigned long long)v);
	return v;
}

/**
 * __blkg_prfill_rwstat - prfill helper for a blkg_rwstat
 * @sf: seq_file to print to
 * @pd: policy data of interest
 * @rwstat: rwstat to print
 *
 * Print @rwstat to @sf for the device assocaited with @pd.
 */
static u64 __blkg_prfill_rwstat(struct seq_file *sf,
				struct blkg_policy_data *pd,
				const struct blkg_rwstat *rwstat)
{
	static const char *rwstr[] = {
		[BLKG_RWSTAT_READ]	= "Read",
		[BLKG_RWSTAT_WRITE]	= "Write",
		[BLKG_RWSTAT_SYNC]	= "Sync",
		[BLKG_RWSTAT_ASYNC]	= "Async",
	};
	const char *dname = blkg_dev_name(pd->blkg);
	u64 v;
	int i;

	if (!dname)
		return 0;

	for (i = 0; i < BLKG_RWSTAT_NR; i++)
		seq_printf(sf, "%s %s %llu\n", dname, rwstr[i],
			   (unsigned long long)rwstat->cnt[i]);

	v = rwstat->cnt[BLKG_RWSTAT_READ] + rwstat->cnt[BLKG_RWSTAT_WRITE];
	seq_printf(sf, "%s Total %llu\n", dname, (unsigned long long)v);
	return v;
}

static u64 blkg_prfill_stat(struct seq_file *sf, struct blkg_policy_data *pd,
			    int off)
{
	return __blkg_prfill_u64(sf, pd,
				 blkg_stat_read((void *)&pd->stats + off));
}

static u64 blkg_prfill_rwstat(struct seq_file *sf, struct blkg_policy_data *pd,
			      int off)
{
	struct blkg_rwstat rwstat = blkg_rwstat_read((void *)&pd->stats + off);

	return __blkg_prfill_rwstat(sf, pd, &rwstat);
}

/* print blkg_stat specified by BLKCG_STAT_PRIV() */
static int blkcg_print_stat(struct cgroup *cgrp, struct cftype *cft,
			    struct seq_file *sf)
{
	struct blkio_cgroup *blkcg = cgroup_to_blkio_cgroup(cgrp);

	blkcg_print_blkgs(sf, blkcg, blkg_prfill_stat,
			  BLKCG_STAT_POL(cft->private),
			  BLKCG_STAT_OFF(cft->private), false);
	return 0;
}

/* print blkg_rwstat specified by BLKCG_STAT_PRIV() */
static int blkcg_print_rwstat(struct cgroup *cgrp, struct cftype *cft,
			      struct seq_file *sf)
{
	struct blkio_cgroup *blkcg = cgroup_to_blkio_cgroup(cgrp);

	blkcg_print_blkgs(sf, blkcg, blkg_prfill_rwstat,
			  BLKCG_STAT_POL(cft->private),
			  BLKCG_STAT_OFF(cft->private), true);
	return 0;
}

static u64 blkg_prfill_cpu_stat(struct seq_file *sf,
				struct blkg_policy_data *pd, int off)
{
	u64 v = 0;
	int cpu;
887

888
	for_each_possible_cpu(cpu) {
889
		struct blkio_group_stats_cpu *sc =
890
891
			per_cpu_ptr(pd->stats_cpu, cpu);

892
		v += blkg_stat_read((void *)sc + off);
893
894
	}

895
	return __blkg_prfill_u64(sf, pd, v);
896
897
}

898
899
static u64 blkg_prfill_cpu_rwstat(struct seq_file *sf,
				  struct blkg_policy_data *pd, int off)
900
{
901
902
903
904
905
906
	struct blkg_rwstat rwstat = { }, tmp;
	int i, cpu;

	for_each_possible_cpu(cpu) {
		struct blkio_group_stats_cpu *sc =
			per_cpu_ptr(pd->stats_cpu, cpu);
907

908
909
910
		tmp = blkg_rwstat_read((void *)sc + off);
		for (i = 0; i < BLKG_RWSTAT_NR; i++)
			rwstat.cnt[i] += tmp.cnt[i];
911
912
	}

913
914
	return __blkg_prfill_rwstat(sf, pd, &rwstat);
}
915

916
917
918
919
920
921
922
923
924
925
/* print per-cpu blkg_stat specified by BLKCG_STAT_PRIV() */
static int blkcg_print_cpu_stat(struct cgroup *cgrp, struct cftype *cft,
				struct seq_file *sf)
{
	struct blkio_cgroup *blkcg = cgroup_to_blkio_cgroup(cgrp);

	blkcg_print_blkgs(sf, blkcg, blkg_prfill_cpu_stat,
			  BLKCG_STAT_POL(cft->private),
			  BLKCG_STAT_OFF(cft->private), false);
	return 0;
926
927
}

928
929
930
/* print per-cpu blkg_rwstat specified by BLKCG_STAT_PRIV() */
static int blkcg_print_cpu_rwstat(struct cgroup *cgrp, struct cftype *cft,
				  struct seq_file *sf)
931
{
932
	struct blkio_cgroup *blkcg = cgroup_to_blkio_cgroup(cgrp);
933

934
935
936
937
938
	blkcg_print_blkgs(sf, blkcg, blkg_prfill_cpu_rwstat,
			  BLKCG_STAT_POL(cft->private),
			  BLKCG_STAT_OFF(cft->private), true);
	return 0;
}
939

940
941
942
943
944
945
#ifdef CONFIG_DEBUG_BLK_CGROUP
static u64 blkg_prfill_avg_queue_size(struct seq_file *sf,
				      struct blkg_policy_data *pd, int off)
{
	u64 samples = blkg_stat_read(&pd->stats.avg_queue_size_samples);
	u64 v = 0;
946

947
948
949
	if (samples) {
		v = blkg_stat_read(&pd->stats.avg_queue_size_sum);
		do_div(v, samples);
950
	}
951
952
953
	__blkg_prfill_u64(sf, pd, v);
	return 0;
}
954

955
956
957
958
959
/* print avg_queue_size */
static int blkcg_print_avg_queue_size(struct cgroup *cgrp, struct cftype *cft,
				      struct seq_file *sf)
{
	struct blkio_cgroup *blkcg = cgroup_to_blkio_cgroup(cgrp);
960

961
962
963
	blkcg_print_blkgs(sf, blkcg, blkg_prfill_avg_queue_size,
			  BLKIO_POLICY_PROP, 0, false);
	return 0;
964
}
965
#endif	/* CONFIG_DEBUG_BLK_CGROUP */
966

967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
struct blkg_conf_ctx {
	struct gendisk		*disk;
	struct blkio_group	*blkg;
	u64			v;
};

/**
 * blkg_conf_prep - parse and prepare for per-blkg config update
 * @blkcg: target block cgroup
 * @input: input string
 * @ctx: blkg_conf_ctx to be filled
 *
 * Parse per-blkg config update from @input and initialize @ctx with the
 * result.  @ctx->blkg points to the blkg to be updated and @ctx->v the new
 * value.  This function returns with RCU read locked and must be paired
 * with blkg_conf_finish().
 */
static int blkg_conf_prep(struct blkio_cgroup *blkcg, const char *input,
			  struct blkg_conf_ctx *ctx)
	__acquires(rcu)
987
{
988
989
	struct gendisk *disk;
	struct blkio_group *blkg;
Tejun Heo's avatar
Tejun Heo committed
990
991
992
	unsigned int major, minor;
	unsigned long long v;
	int part, ret;
993

Tejun Heo's avatar
Tejun Heo committed
994
995
	if (sscanf(input, "%u:%u %llu", &major, &minor, &v) != 3)
		return -EINVAL;
996

Tejun Heo's avatar
Tejun Heo committed
997
	disk = get_gendisk(MKDEV(major, minor), &part);
Tejun Heo's avatar
Tejun Heo committed
998
	if (!disk || part)
Tejun Heo's avatar
Tejun Heo committed
999
		return -EINVAL;
1000

For faster browsing, not all history is shown. View entire blame