blk-cgroup.h 10.1 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
#ifndef _BLK_CGROUP_H
#define _BLK_CGROUP_H
/*
 * Common Block IO controller cgroup interface
 *
 * Based on ideas and code from CFQ, CFS and BFQ:
 * Copyright (C) 2003 Jens Axboe <axboe@kernel.dk>
 *
 * Copyright (C) 2008 Fabio Checconi <fabio@gandalf.sssup.it>
 *		      Paolo Valente <paolo.valente@unimore.it>
 *
 * Copyright (C) 2009 Vivek Goyal <vgoyal@redhat.com>
 * 	              Nauman Rafique <nauman@google.com>
 */

#include <linux/cgroup.h>
17
#include <linux/u64_stats_sync.h>
18
#include <linux/seq_file.h>
19

20
21
22
/* Max limits for throttle policy */
#define THROTL_IOPS_MAX		UINT_MAX

23
24
25
26
27
/* CFQ specific, out here for blkcg->cfq_weight */
#define CFQ_WEIGHT_MIN		10
#define CFQ_WEIGHT_MAX		1000
#define CFQ_WEIGHT_DEFAULT	500

Tejun Heo's avatar
Tejun Heo committed
28
29
#ifdef CONFIG_BLK_CGROUP

30
31
32
33
34
35
36
37
enum blkg_rwstat_type {
	BLKG_RWSTAT_READ,
	BLKG_RWSTAT_WRITE,
	BLKG_RWSTAT_SYNC,
	BLKG_RWSTAT_ASYNC,

	BLKG_RWSTAT_NR,
	BLKG_RWSTAT_TOTAL = BLKG_RWSTAT_NR,
38
39
};

40
struct blkio_cgroup {
41
42
43
	struct cgroup_subsys_state	css;
	spinlock_t			lock;
	struct hlist_head		blkg_list;
Tejun Heo's avatar
Tejun Heo committed
44
45

	/* for policies to test whether associated blkcg has changed */
46
	uint64_t			id;
47
48

	/* TODO: per-policy storage in blkio_cgroup */
49
	unsigned int			cfq_weight;	/* belongs to cfq */
50
51
};

52
53
54
55
56
57
58
59
60
61
struct blkg_stat {
	struct u64_stats_sync		syncp;
	uint64_t			cnt;
};

struct blkg_rwstat {
	struct u64_stats_sync		syncp;
	uint64_t			cnt[BLKG_RWSTAT_NR];
};

62
63
64
/* per-blkg per-policy data */
struct blkg_policy_data {
	/* the blkg this per-policy data belongs to */
65
	struct blkio_group		*blkg;
66

67
	/* used during policy activation */
68
	struct list_head		alloc_node;
69

70
	/* pol->pdata_size bytes of private data used by policy impl */
71
	char				pdata[] __aligned(__alignof__(unsigned long long));
72
73
};

74
struct blkio_group {
75
	/* Pointer to the associated request_queue */
76
77
78
79
	struct request_queue		*q;
	struct list_head		q_node;
	struct hlist_node		blkcg_node;
	struct blkio_cgroup		*blkcg;
Tejun Heo's avatar
Tejun Heo committed
80
	/* reference count */
81
	int				refcnt;
82

83
	struct blkg_policy_data		*pd[BLKCG_MAX_POLS];
Tejun Heo's avatar
Tejun Heo committed
84

85
	struct rcu_head			rcu_head;
86
87
};

88
typedef void (blkio_init_group_fn)(struct blkio_group *blkg);
89
90
typedef void (blkio_exit_group_fn)(struct blkio_group *blkg);
typedef void (blkio_reset_group_stats_fn)(struct blkio_group *blkg);
91
92

struct blkio_policy_ops {
93
94
95
	blkio_init_group_fn		*blkio_init_group_fn;
	blkio_exit_group_fn		*blkio_exit_group_fn;
	blkio_reset_group_stats_fn	*blkio_reset_group_stats_fn;
96
97
98
};

struct blkio_policy_type {
99
100
101
102
103
104
	struct blkio_policy_ops		ops;
	int				plid;
	/* policy specific private data size */
	size_t				pdata_size;
	/* cgroup files for the policy */
	struct cftype			*cftypes;
105
106
};

107
108
109
110
111
112
113
114
115
116
117
extern struct blkio_cgroup blkio_root_cgroup;

struct blkio_cgroup *cgroup_to_blkio_cgroup(struct cgroup *cgroup);
struct blkio_cgroup *bio_blkio_cgroup(struct bio *bio);
struct blkio_group *blkg_lookup(struct blkio_cgroup *blkcg,
				struct request_queue *q);
struct blkio_group *blkg_lookup_create(struct blkio_cgroup *blkcg,
				       struct request_queue *q);
int blkcg_init_queue(struct request_queue *q);
void blkcg_drain_queue(struct request_queue *q);
void blkcg_exit_queue(struct request_queue *q);
118

119
/* Blkio controller policy registration */
120
121
122
123
124
125
int blkio_policy_register(struct blkio_policy_type *);
void blkio_policy_unregister(struct blkio_policy_type *);
int blkcg_activate_policy(struct request_queue *q,
			  const struct blkio_policy_type *pol);
void blkcg_deactivate_policy(struct request_queue *q,
			     const struct blkio_policy_type *pol);
126

127
void blkcg_print_blkgs(struct seq_file *sf, struct blkio_cgroup *blkcg,
128
		       u64 (*prfill)(struct seq_file *, void *, int),
129
130
		       const struct blkio_policy_type *pol, int data,
		       bool show_total);
131
132
u64 __blkg_prfill_u64(struct seq_file *sf, void *pdata, u64 v);
u64 __blkg_prfill_rwstat(struct seq_file *sf, void *pdata,
133
			 const struct blkg_rwstat *rwstat);
134
135
u64 blkg_prfill_stat(struct seq_file *sf, void *pdata, int off);
u64 blkg_prfill_rwstat(struct seq_file *sf, void *pdata, int off);
136
137

struct blkg_conf_ctx {
138
139
140
	struct gendisk			*disk;
	struct blkio_group		*blkg;
	u64				v;
141
142
};

143
144
int blkg_conf_prep(struct blkio_cgroup *blkcg,
		   const struct blkio_policy_type *pol, const char *input,
145
146
147
148
		   struct blkg_conf_ctx *ctx);
void blkg_conf_finish(struct blkg_conf_ctx *ctx);


149
150
151
152
153
154
155
156
157
158
/**
 * blkg_to_pdata - get policy private data
 * @blkg: blkg of interest
 * @pol: policy of interest
 *
 * Return pointer to private data associated with the @blkg-@pol pair.
 */
static inline void *blkg_to_pdata(struct blkio_group *blkg,
			      struct blkio_policy_type *pol)
{
159
	return blkg ? blkg->pd[pol->plid]->pdata : NULL;
160
161
162
163
164
165
}

/**
 * pdata_to_blkg - get blkg associated with policy private data
 * @pdata: policy private data of interest
 *
166
 * @pdata is policy private data.  Determine the blkg it's associated with.
167
 */
168
static inline struct blkio_group *pdata_to_blkg(void *pdata)
169
170
171
172
173
174
175
176
177
{
	if (pdata) {
		struct blkg_policy_data *pd =
			container_of(pdata, struct blkg_policy_data, pdata);
		return pd->blkg;
	}
	return NULL;
}

Tejun Heo's avatar
Tejun Heo committed
178
179
180
181
182
183
184
185
186
/**
 * blkg_path - format cgroup path of blkg
 * @blkg: blkg of interest
 * @buf: target buffer
 * @buflen: target buffer length
 *
 * Format the path of the cgroup of @blkg into @buf.
 */
static inline int blkg_path(struct blkio_group *blkg, char *buf, int buflen)
187
{
Tejun Heo's avatar
Tejun Heo committed
188
189
190
191
192
193
194
195
	int ret;

	rcu_read_lock();
	ret = cgroup_path(blkg->blkcg->css.cgroup, buf, buflen);
	rcu_read_unlock();
	if (ret)
		strncpy(buf, "<unavailable>", buflen);
	return ret;
196
197
}

Tejun Heo's avatar
Tejun Heo committed
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
/**
 * blkg_get - get a blkg reference
 * @blkg: blkg to get
 *
 * The caller should be holding queue_lock and an existing reference.
 */
static inline void blkg_get(struct blkio_group *blkg)
{
	lockdep_assert_held(blkg->q->queue_lock);
	WARN_ON_ONCE(!blkg->refcnt);
	blkg->refcnt++;
}

void __blkg_release(struct blkio_group *blkg);

/**
 * blkg_put - put a blkg reference
 * @blkg: blkg to put
 *
 * The caller should be holding queue_lock.
 */
static inline void blkg_put(struct blkio_group *blkg)
{
	lockdep_assert_held(blkg->q->queue_lock);
	WARN_ON_ONCE(blkg->refcnt <= 0);
	if (!--blkg->refcnt)
		__blkg_release(blkg);
}

227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
/**
 * blkg_stat_add - add a value to a blkg_stat
 * @stat: target blkg_stat
 * @val: value to add
 *
 * Add @val to @stat.  The caller is responsible for synchronizing calls to
 * this function.
 */
static inline void blkg_stat_add(struct blkg_stat *stat, uint64_t val)
{
	u64_stats_update_begin(&stat->syncp);
	stat->cnt += val;
	u64_stats_update_end(&stat->syncp);
}

/**
 * blkg_stat_read - read the current value of a blkg_stat
 * @stat: blkg_stat to read
 *
 * Read the current value of @stat.  This function can be called without
 * synchroniztion and takes care of u64 atomicity.
 */
static inline uint64_t blkg_stat_read(struct blkg_stat *stat)
{
	unsigned int start;
	uint64_t v;

	do {
		start = u64_stats_fetch_begin(&stat->syncp);
		v = stat->cnt;
	} while (u64_stats_fetch_retry(&stat->syncp, start));

	return v;
}

/**
 * blkg_stat_reset - reset a blkg_stat
 * @stat: blkg_stat to reset
 */
static inline void blkg_stat_reset(struct blkg_stat *stat)
{
	stat->cnt = 0;
}

/**
 * blkg_rwstat_add - add a value to a blkg_rwstat
 * @rwstat: target blkg_rwstat
 * @rw: mask of REQ_{WRITE|SYNC}
 * @val: value to add
 *
 * Add @val to @rwstat.  The counters are chosen according to @rw.  The
 * caller is responsible for synchronizing calls to this function.
 */
static inline void blkg_rwstat_add(struct blkg_rwstat *rwstat,
				   int rw, uint64_t val)
{
	u64_stats_update_begin(&rwstat->syncp);

	if (rw & REQ_WRITE)
		rwstat->cnt[BLKG_RWSTAT_WRITE] += val;
	else
		rwstat->cnt[BLKG_RWSTAT_READ] += val;
	if (rw & REQ_SYNC)
		rwstat->cnt[BLKG_RWSTAT_SYNC] += val;
	else
		rwstat->cnt[BLKG_RWSTAT_ASYNC] += val;

	u64_stats_update_end(&rwstat->syncp);
}

/**
 * blkg_rwstat_read - read the current values of a blkg_rwstat
 * @rwstat: blkg_rwstat to read
 *
 * Read the current snapshot of @rwstat and return it as the return value.
 * This function can be called without synchronization and takes care of
 * u64 atomicity.
 */
305
static inline struct blkg_rwstat blkg_rwstat_read(struct blkg_rwstat *rwstat)
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
{
	unsigned int start;
	struct blkg_rwstat tmp;

	do {
		start = u64_stats_fetch_begin(&rwstat->syncp);
		tmp = *rwstat;
	} while (u64_stats_fetch_retry(&rwstat->syncp, start));

	return tmp;
}

/**
 * blkg_rwstat_sum - read the total count of a blkg_rwstat
 * @rwstat: blkg_rwstat to read
 *
 * Return the total count of @rwstat regardless of the IO direction.  This
 * function can be called without synchronization and takes care of u64
 * atomicity.
 */
static inline uint64_t blkg_rwstat_sum(struct blkg_rwstat *rwstat)
{
	struct blkg_rwstat tmp = blkg_rwstat_read(rwstat);

	return tmp.cnt[BLKG_RWSTAT_READ] + tmp.cnt[BLKG_RWSTAT_WRITE];
}

/**
 * blkg_rwstat_reset - reset a blkg_rwstat
 * @rwstat: blkg_rwstat to reset
 */
static inline void blkg_rwstat_reset(struct blkg_rwstat *rwstat)
{
	memset(rwstat->cnt, 0, sizeof(rwstat->cnt));
}

342
343
344
#else	/* CONFIG_BLK_CGROUP */

struct cgroup;
345
346
347
348

struct blkio_group {
};

349
350
351
struct blkio_policy_type {
};

352
353
354
static inline struct blkio_cgroup *cgroup_to_blkio_cgroup(struct cgroup *cgroup) { return NULL; }
static inline struct blkio_cgroup *bio_blkio_cgroup(struct bio *bio) { return NULL; }
static inline struct blkio_group *blkg_lookup(struct blkio_cgroup *blkcg, void *key) { return NULL; }
355
356
357
static inline int blkcg_init_queue(struct request_queue *q) { return 0; }
static inline void blkcg_drain_queue(struct request_queue *q) { }
static inline void blkcg_exit_queue(struct request_queue *q) { }
358
static inline int blkio_policy_register(struct blkio_policy_type *blkiop) { return 0; }
359
static inline void blkio_policy_unregister(struct blkio_policy_type *blkiop) { }
360
361
362
363
static inline int blkcg_activate_policy(struct request_queue *q,
					const struct blkio_policy_type *pol) { return 0; }
static inline void blkcg_deactivate_policy(struct request_queue *q,
					   const struct blkio_policy_type *pol) { }
364

365
366
367
368
static inline void *blkg_to_pdata(struct blkio_group *blkg,
				struct blkio_policy_type *pol) { return NULL; }
static inline struct blkio_group *pdata_to_blkg(void *pdata,
				struct blkio_policy_type *pol) { return NULL; }
369
static inline char *blkg_path(struct blkio_group *blkg) { return NULL; }
Tejun Heo's avatar
Tejun Heo committed
370
371
static inline void blkg_get(struct blkio_group *blkg) { }
static inline void blkg_put(struct blkio_group *blkg) { }
372

373
374
#endif	/* CONFIG_BLK_CGROUP */
#endif	/* _BLK_CGROUP_H */