blk-cgroup.h 10.8 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
#ifndef _BLK_CGROUP_H
#define _BLK_CGROUP_H
/*
 * Common Block IO controller cgroup interface
 *
 * Based on ideas and code from CFQ, CFS and BFQ:
 * Copyright (C) 2003 Jens Axboe <axboe@kernel.dk>
 *
 * Copyright (C) 2008 Fabio Checconi <fabio@gandalf.sssup.it>
 *		      Paolo Valente <paolo.valente@unimore.it>
 *
 * Copyright (C) 2009 Vivek Goyal <vgoyal@redhat.com>
 * 	              Nauman Rafique <nauman@google.com>
 */

#include <linux/cgroup.h>
17
#include <linux/u64_stats_sync.h>
18
#include <linux/seq_file.h>
19
#include <linux/radix-tree.h>
20

21
22
23
/* Max limits for throttle policy */
#define THROTL_IOPS_MAX		UINT_MAX

24
25
26
27
28
/* CFQ specific, out here for blkcg->cfq_weight */
#define CFQ_WEIGHT_MIN		10
#define CFQ_WEIGHT_MAX		1000
#define CFQ_WEIGHT_DEFAULT	500

Tejun Heo's avatar
Tejun Heo committed
29
30
#ifdef CONFIG_BLK_CGROUP

31
32
33
34
35
36
37
38
enum blkg_rwstat_type {
	BLKG_RWSTAT_READ,
	BLKG_RWSTAT_WRITE,
	BLKG_RWSTAT_SYNC,
	BLKG_RWSTAT_ASYNC,

	BLKG_RWSTAT_NR,
	BLKG_RWSTAT_TOTAL = BLKG_RWSTAT_NR,
39
40
};

41
42
struct blkcg_gq;

Tejun Heo's avatar
Tejun Heo committed
43
struct blkcg {
44
45
	struct cgroup_subsys_state	css;
	spinlock_t			lock;
46
47
48

	struct radix_tree_root		blkg_tree;
	struct blkcg_gq			*blkg_hint;
49
	struct hlist_head		blkg_list;
Tejun Heo's avatar
Tejun Heo committed
50
51

	/* for policies to test whether associated blkcg has changed */
52
	uint64_t			id;
53

Tejun Heo's avatar
Tejun Heo committed
54
	/* TODO: per-policy storage in blkcg */
55
	unsigned int			cfq_weight;	/* belongs to cfq */
56
57
};

58
59
60
61
62
63
64
65
66
67
struct blkg_stat {
	struct u64_stats_sync		syncp;
	uint64_t			cnt;
};

struct blkg_rwstat {
	struct u64_stats_sync		syncp;
	uint64_t			cnt[BLKG_RWSTAT_NR];
};

68
69
70
71
72
73
74
75
76
77
78
79
80
/*
 * A blkcg_gq (blkg) is association between a block cgroup (blkcg) and a
 * request_queue (q).  This is used by blkcg policies which need to track
 * information per blkcg - q pair.
 *
 * There can be multiple active blkcg policies and each has its private
 * data on each blkg, the size of which is determined by
 * blkcg_policy->pd_size.  blkcg core allocates and frees such areas
 * together with blkg and invokes pd_init/exit_fn() methods.
 *
 * Such private data must embed struct blkg_policy_data (pd) at the
 * beginning and pd_size can't be smaller than pd.
 */
81
82
struct blkg_policy_data {
	/* the blkg this per-policy data belongs to */
Tejun Heo's avatar
Tejun Heo committed
83
	struct blkcg_gq			*blkg;
84

85
	/* used during policy activation */
86
	struct list_head		alloc_node;
87
88
};

Tejun Heo's avatar
Tejun Heo committed
89
90
/* association between a blk cgroup and a request queue */
struct blkcg_gq {
91
	/* Pointer to the associated request_queue */
92
93
94
	struct request_queue		*q;
	struct list_head		q_node;
	struct hlist_node		blkcg_node;
Tejun Heo's avatar
Tejun Heo committed
95
	struct blkcg			*blkcg;
Tejun Heo's avatar
Tejun Heo committed
96
	/* reference count */
97
	int				refcnt;
98

99
	struct blkg_policy_data		*pd[BLKCG_MAX_POLS];
Tejun Heo's avatar
Tejun Heo committed
100

101
	struct rcu_head			rcu_head;
102
103
};

Tejun Heo's avatar
Tejun Heo committed
104
105
106
typedef void (blkcg_pol_init_pd_fn)(struct blkcg_gq *blkg);
typedef void (blkcg_pol_exit_pd_fn)(struct blkcg_gq *blkg);
typedef void (blkcg_pol_reset_pd_stats_fn)(struct blkcg_gq *blkg);
107

Tejun Heo's avatar
Tejun Heo committed
108
struct blkcg_policy {
109
110
	int				plid;
	/* policy specific private data size */
111
	size_t				pd_size;
112
113
	/* cgroup files for the policy */
	struct cftype			*cftypes;
114
115
116
117
118

	/* operations */
	blkcg_pol_init_pd_fn		*pd_init_fn;
	blkcg_pol_exit_pd_fn		*pd_exit_fn;
	blkcg_pol_reset_pd_stats_fn	*pd_reset_stats_fn;
119
120
};

Tejun Heo's avatar
Tejun Heo committed
121
extern struct blkcg blkcg_root;
122

Tejun Heo's avatar
Tejun Heo committed
123
124
125
struct blkcg_gq *blkg_lookup(struct blkcg *blkcg, struct request_queue *q);
struct blkcg_gq *blkg_lookup_create(struct blkcg *blkcg,
				    struct request_queue *q);
126
127
128
int blkcg_init_queue(struct request_queue *q);
void blkcg_drain_queue(struct request_queue *q);
void blkcg_exit_queue(struct request_queue *q);
129

130
/* Blkio controller policy registration */
Tejun Heo's avatar
Tejun Heo committed
131
132
int blkcg_policy_register(struct blkcg_policy *pol);
void blkcg_policy_unregister(struct blkcg_policy *pol);
133
int blkcg_activate_policy(struct request_queue *q,
Tejun Heo's avatar
Tejun Heo committed
134
			  const struct blkcg_policy *pol);
135
void blkcg_deactivate_policy(struct request_queue *q,
Tejun Heo's avatar
Tejun Heo committed
136
			     const struct blkcg_policy *pol);
137

Tejun Heo's avatar
Tejun Heo committed
138
void blkcg_print_blkgs(struct seq_file *sf, struct blkcg *blkcg,
139
140
		       u64 (*prfill)(struct seq_file *,
				     struct blkg_policy_data *, int),
Tejun Heo's avatar
Tejun Heo committed
141
		       const struct blkcg_policy *pol, int data,
142
		       bool show_total);
143
144
u64 __blkg_prfill_u64(struct seq_file *sf, struct blkg_policy_data *pd, u64 v);
u64 __blkg_prfill_rwstat(struct seq_file *sf, struct blkg_policy_data *pd,
145
			 const struct blkg_rwstat *rwstat);
146
147
148
u64 blkg_prfill_stat(struct seq_file *sf, struct blkg_policy_data *pd, int off);
u64 blkg_prfill_rwstat(struct seq_file *sf, struct blkg_policy_data *pd,
		       int off);
149
150

struct blkg_conf_ctx {
151
	struct gendisk			*disk;
Tejun Heo's avatar
Tejun Heo committed
152
	struct blkcg_gq			*blkg;
153
	u64				v;
154
155
};

Tejun Heo's avatar
Tejun Heo committed
156
157
int blkg_conf_prep(struct blkcg *blkcg, const struct blkcg_policy *pol,
		   const char *input, struct blkg_conf_ctx *ctx);
158
159
160
void blkg_conf_finish(struct blkg_conf_ctx *ctx);


161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
static inline struct blkcg *cgroup_to_blkcg(struct cgroup *cgroup)
{
	return container_of(cgroup_subsys_state(cgroup, blkio_subsys_id),
			    struct blkcg, css);
}

static inline struct blkcg *task_blkcg(struct task_struct *tsk)
{
	return container_of(task_subsys_state(tsk, blkio_subsys_id),
			    struct blkcg, css);
}

static inline struct blkcg *bio_blkcg(struct bio *bio)
{
	if (bio && bio->bi_css)
		return container_of(bio->bi_css, struct blkcg, css);
	return task_blkcg(current);
}

180
181
182
183
184
185
186
/**
 * blkg_to_pdata - get policy private data
 * @blkg: blkg of interest
 * @pol: policy of interest
 *
 * Return pointer to private data associated with the @blkg-@pol pair.
 */
187
188
static inline struct blkg_policy_data *blkg_to_pd(struct blkcg_gq *blkg,
						  struct blkcg_policy *pol)
189
{
190
	return blkg ? blkg->pd[pol->plid] : NULL;
191
192
193
194
}

/**
 * pdata_to_blkg - get blkg associated with policy private data
195
 * @pd: policy private data of interest
196
 *
197
 * @pd is policy private data.  Determine the blkg it's associated with.
198
 */
199
static inline struct blkcg_gq *pd_to_blkg(struct blkg_policy_data *pd)
200
{
201
	return pd ? pd->blkg : NULL;
202
203
}

Tejun Heo's avatar
Tejun Heo committed
204
205
206
207
208
209
210
211
/**
 * blkg_path - format cgroup path of blkg
 * @blkg: blkg of interest
 * @buf: target buffer
 * @buflen: target buffer length
 *
 * Format the path of the cgroup of @blkg into @buf.
 */
Tejun Heo's avatar
Tejun Heo committed
212
static inline int blkg_path(struct blkcg_gq *blkg, char *buf, int buflen)
213
{
Tejun Heo's avatar
Tejun Heo committed
214
215
216
217
218
219
220
221
	int ret;

	rcu_read_lock();
	ret = cgroup_path(blkg->blkcg->css.cgroup, buf, buflen);
	rcu_read_unlock();
	if (ret)
		strncpy(buf, "<unavailable>", buflen);
	return ret;
222
223
}

Tejun Heo's avatar
Tejun Heo committed
224
225
226
227
228
229
/**
 * blkg_get - get a blkg reference
 * @blkg: blkg to get
 *
 * The caller should be holding queue_lock and an existing reference.
 */
Tejun Heo's avatar
Tejun Heo committed
230
static inline void blkg_get(struct blkcg_gq *blkg)
Tejun Heo's avatar
Tejun Heo committed
231
232
233
234
235
236
{
	lockdep_assert_held(blkg->q->queue_lock);
	WARN_ON_ONCE(!blkg->refcnt);
	blkg->refcnt++;
}

Tejun Heo's avatar
Tejun Heo committed
237
void __blkg_release(struct blkcg_gq *blkg);
Tejun Heo's avatar
Tejun Heo committed
238
239
240
241
242
243
244

/**
 * blkg_put - put a blkg reference
 * @blkg: blkg to put
 *
 * The caller should be holding queue_lock.
 */
Tejun Heo's avatar
Tejun Heo committed
245
static inline void blkg_put(struct blkcg_gq *blkg)
Tejun Heo's avatar
Tejun Heo committed
246
247
248
249
250
251
252
{
	lockdep_assert_held(blkg->q->queue_lock);
	WARN_ON_ONCE(blkg->refcnt <= 0);
	if (!--blkg->refcnt)
		__blkg_release(blkg);
}

253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
/**
 * blkg_stat_add - add a value to a blkg_stat
 * @stat: target blkg_stat
 * @val: value to add
 *
 * Add @val to @stat.  The caller is responsible for synchronizing calls to
 * this function.
 */
static inline void blkg_stat_add(struct blkg_stat *stat, uint64_t val)
{
	u64_stats_update_begin(&stat->syncp);
	stat->cnt += val;
	u64_stats_update_end(&stat->syncp);
}

/**
 * blkg_stat_read - read the current value of a blkg_stat
 * @stat: blkg_stat to read
 *
 * Read the current value of @stat.  This function can be called without
 * synchroniztion and takes care of u64 atomicity.
 */
static inline uint64_t blkg_stat_read(struct blkg_stat *stat)
{
	unsigned int start;
	uint64_t v;

	do {
		start = u64_stats_fetch_begin(&stat->syncp);
		v = stat->cnt;
	} while (u64_stats_fetch_retry(&stat->syncp, start));

	return v;
}

/**
 * blkg_stat_reset - reset a blkg_stat
 * @stat: blkg_stat to reset
 */
static inline void blkg_stat_reset(struct blkg_stat *stat)
{
	stat->cnt = 0;
}

/**
 * blkg_rwstat_add - add a value to a blkg_rwstat
 * @rwstat: target blkg_rwstat
 * @rw: mask of REQ_{WRITE|SYNC}
 * @val: value to add
 *
 * Add @val to @rwstat.  The counters are chosen according to @rw.  The
 * caller is responsible for synchronizing calls to this function.
 */
static inline void blkg_rwstat_add(struct blkg_rwstat *rwstat,
				   int rw, uint64_t val)
{
	u64_stats_update_begin(&rwstat->syncp);

	if (rw & REQ_WRITE)
		rwstat->cnt[BLKG_RWSTAT_WRITE] += val;
	else
		rwstat->cnt[BLKG_RWSTAT_READ] += val;
	if (rw & REQ_SYNC)
		rwstat->cnt[BLKG_RWSTAT_SYNC] += val;
	else
		rwstat->cnt[BLKG_RWSTAT_ASYNC] += val;

	u64_stats_update_end(&rwstat->syncp);
}

/**
 * blkg_rwstat_read - read the current values of a blkg_rwstat
 * @rwstat: blkg_rwstat to read
 *
 * Read the current snapshot of @rwstat and return it as the return value.
 * This function can be called without synchronization and takes care of
 * u64 atomicity.
 */
331
static inline struct blkg_rwstat blkg_rwstat_read(struct blkg_rwstat *rwstat)
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
{
	unsigned int start;
	struct blkg_rwstat tmp;

	do {
		start = u64_stats_fetch_begin(&rwstat->syncp);
		tmp = *rwstat;
	} while (u64_stats_fetch_retry(&rwstat->syncp, start));

	return tmp;
}

/**
 * blkg_rwstat_sum - read the total count of a blkg_rwstat
 * @rwstat: blkg_rwstat to read
 *
 * Return the total count of @rwstat regardless of the IO direction.  This
 * function can be called without synchronization and takes care of u64
 * atomicity.
 */
static inline uint64_t blkg_rwstat_sum(struct blkg_rwstat *rwstat)
{
	struct blkg_rwstat tmp = blkg_rwstat_read(rwstat);

	return tmp.cnt[BLKG_RWSTAT_READ] + tmp.cnt[BLKG_RWSTAT_WRITE];
}

/**
 * blkg_rwstat_reset - reset a blkg_rwstat
 * @rwstat: blkg_rwstat to reset
 */
static inline void blkg_rwstat_reset(struct blkg_rwstat *rwstat)
{
	memset(rwstat->cnt, 0, sizeof(rwstat->cnt));
}

368
369
370
#else	/* CONFIG_BLK_CGROUP */

struct cgroup;
371
struct blkcg;
372

373
374
375
struct blkg_policy_data {
};

Tejun Heo's avatar
Tejun Heo committed
376
struct blkcg_gq {
377
378
};

Tejun Heo's avatar
Tejun Heo committed
379
struct blkcg_policy {
380
381
};

Tejun Heo's avatar
Tejun Heo committed
382
static inline struct blkcg_gq *blkg_lookup(struct blkcg *blkcg, void *key) { return NULL; }
383
384
385
static inline int blkcg_init_queue(struct request_queue *q) { return 0; }
static inline void blkcg_drain_queue(struct request_queue *q) { }
static inline void blkcg_exit_queue(struct request_queue *q) { }
Tejun Heo's avatar
Tejun Heo committed
386
387
static inline int blkcg_policy_register(struct blkcg_policy *pol) { return 0; }
static inline void blkcg_policy_unregister(struct blkcg_policy *pol) { }
388
static inline int blkcg_activate_policy(struct request_queue *q,
Tejun Heo's avatar
Tejun Heo committed
389
					const struct blkcg_policy *pol) { return 0; }
390
static inline void blkcg_deactivate_policy(struct request_queue *q,
Tejun Heo's avatar
Tejun Heo committed
391
392
					   const struct blkcg_policy *pol) { }

393
394
static inline struct blkcg *cgroup_to_blkcg(struct cgroup *cgroup) { return NULL; }
static inline struct blkcg *bio_blkcg(struct bio *bio) { return NULL; }
395
396
397
static inline struct blkg_policy_data *blkg_to_pd(struct blkcg_gq *blkg,
						  struct blkcg_policy *pol) { return NULL; }
static inline struct blkcg_gq *pd_to_blkg(struct blkg_policy_data *pd) { return NULL; }
Tejun Heo's avatar
Tejun Heo committed
398
399
400
static inline char *blkg_path(struct blkcg_gq *blkg) { return NULL; }
static inline void blkg_get(struct blkcg_gq *blkg) { }
static inline void blkg_put(struct blkcg_gq *blkg) { }
401

402
403
#endif	/* CONFIG_BLK_CGROUP */
#endif	/* _BLK_CGROUP_H */