blob: ea84a23d5e680d8acc8dfc67948e5872f0142ba7 [file] [log] [blame]
Vivek Goyal31e4c282009-12-03 12:59:42 -05001/*
2 * Common Block IO controller cgroup interface
3 *
4 * Based on ideas and code from CFQ, CFS and BFQ:
5 * Copyright (C) 2003 Jens Axboe <axboe@kernel.dk>
6 *
7 * Copyright (C) 2008 Fabio Checconi <fabio@gandalf.sssup.it>
8 * Paolo Valente <paolo.valente@unimore.it>
9 *
10 * Copyright (C) 2009 Vivek Goyal <vgoyal@redhat.com>
11 * Nauman Rafique <nauman@google.com>
12 */
13#include <linux/ioprio.h>
Vivek Goyal22084192009-12-03 12:59:49 -050014#include <linux/seq_file.h>
15#include <linux/kdev_t.h>
Vivek Goyal9d6a9862009-12-04 10:36:41 -050016#include <linux/module.h>
Stephen Rothwellaccee782009-12-07 19:29:39 +110017#include <linux/err.h>
Divyesh Shah91952912010-04-01 15:01:41 -070018#include <linux/blkdev.h>
Tejun Heo5a0e3ad2010-03-24 17:04:11 +090019#include <linux/slab.h>
Vivek Goyal31e4c282009-12-03 12:59:42 -050020#include "blk-cgroup.h"
Gui Jianfeng34d0f172010-04-13 16:05:49 +080021#include <linux/genhd.h>
Vivek Goyal3e252062009-12-04 10:36:42 -050022
Divyesh Shah84c124d2010-04-09 08:31:19 +020023#define MAX_KEY_LEN 100
24
Vivek Goyal3e252062009-12-04 10:36:42 -050025static DEFINE_SPINLOCK(blkio_list_lock);
26static LIST_HEAD(blkio_list);
Vivek Goyalb1c35762009-12-03 12:59:47 -050027
Vivek Goyal31e4c282009-12-03 12:59:42 -050028struct blkio_cgroup blkio_root_cgroup = { .weight = 2*BLKIO_WEIGHT_DEFAULT };
Vivek Goyal9d6a9862009-12-04 10:36:41 -050029EXPORT_SYMBOL_GPL(blkio_root_cgroup);
30
Li Zefan761b3ef2012-01-31 13:47:36 +080031static struct cgroup_subsys_state *blkiocg_create(struct cgroup *);
32static int blkiocg_can_attach(struct cgroup *, struct cgroup_taskset *);
33static void blkiocg_attach(struct cgroup *, struct cgroup_taskset *);
34static void blkiocg_destroy(struct cgroup *);
Ben Blum67523c42010-03-10 15:22:11 -080035static int blkiocg_populate(struct cgroup_subsys *, struct cgroup *);
36
Vivek Goyal062a6442010-09-15 17:06:33 -040037/* for encoding cft->private value on file */
38#define BLKIOFILE_PRIVATE(x, val) (((x) << 16) | (val))
39/* What policy owns the file, proportional or throttle */
40#define BLKIOFILE_POLICY(val) (((val) >> 16) & 0xffff)
41#define BLKIOFILE_ATTR(val) ((val) & 0xffff)
42
Ben Blum67523c42010-03-10 15:22:11 -080043struct cgroup_subsys blkio_subsys = {
44 .name = "blkio",
45 .create = blkiocg_create,
Tejun Heobb9d97b2011-12-12 18:12:21 -080046 .can_attach = blkiocg_can_attach,
47 .attach = blkiocg_attach,
Ben Blum67523c42010-03-10 15:22:11 -080048 .destroy = blkiocg_destroy,
49 .populate = blkiocg_populate,
50#ifdef CONFIG_BLK_CGROUP
51 /* note: blkio_subsys_id is otherwise defined in blk-cgroup.h */
52 .subsys_id = blkio_subsys_id,
53#endif
54 .use_id = 1,
55 .module = THIS_MODULE,
56};
57EXPORT_SYMBOL_GPL(blkio_subsys);
58
Gui Jianfeng34d0f172010-04-13 16:05:49 +080059static inline void blkio_policy_insert_node(struct blkio_cgroup *blkcg,
60 struct blkio_policy_node *pn)
61{
62 list_add(&pn->node, &blkcg->policy_list);
63}
64
Vivek Goyal062a6442010-09-15 17:06:33 -040065static inline bool cftype_blkg_same_policy(struct cftype *cft,
66 struct blkio_group *blkg)
67{
68 enum blkio_policy_id plid = BLKIOFILE_POLICY(cft->private);
69
70 if (blkg->plid == plid)
71 return 1;
72
73 return 0;
74}
75
76/* Determines if policy node matches cgroup file being accessed */
77static inline bool pn_matches_cftype(struct cftype *cft,
78 struct blkio_policy_node *pn)
79{
80 enum blkio_policy_id plid = BLKIOFILE_POLICY(cft->private);
81 int fileid = BLKIOFILE_ATTR(cft->private);
82
83 return (plid == pn->plid && fileid == pn->fileid);
84}
85
Gui Jianfeng34d0f172010-04-13 16:05:49 +080086/* Must be called with blkcg->lock held */
87static inline void blkio_policy_delete_node(struct blkio_policy_node *pn)
88{
89 list_del(&pn->node);
90}
91
92/* Must be called with blkcg->lock held */
93static struct blkio_policy_node *
Vivek Goyal062a6442010-09-15 17:06:33 -040094blkio_policy_search_node(const struct blkio_cgroup *blkcg, dev_t dev,
95 enum blkio_policy_id plid, int fileid)
Gui Jianfeng34d0f172010-04-13 16:05:49 +080096{
97 struct blkio_policy_node *pn;
98
99 list_for_each_entry(pn, &blkcg->policy_list, node) {
Vivek Goyal062a6442010-09-15 17:06:33 -0400100 if (pn->dev == dev && pn->plid == plid && pn->fileid == fileid)
Gui Jianfeng34d0f172010-04-13 16:05:49 +0800101 return pn;
102 }
103
104 return NULL;
105}
106
Vivek Goyal31e4c282009-12-03 12:59:42 -0500107struct blkio_cgroup *cgroup_to_blkio_cgroup(struct cgroup *cgroup)
108{
109 return container_of(cgroup_subsys_state(cgroup, blkio_subsys_id),
110 struct blkio_cgroup, css);
111}
Vivek Goyal9d6a9862009-12-04 10:36:41 -0500112EXPORT_SYMBOL_GPL(cgroup_to_blkio_cgroup);
Vivek Goyal31e4c282009-12-03 12:59:42 -0500113
Vivek Goyal70087dc2011-05-16 15:24:08 +0200114struct blkio_cgroup *task_blkio_cgroup(struct task_struct *tsk)
115{
116 return container_of(task_subsys_state(tsk, blkio_subsys_id),
117 struct blkio_cgroup, css);
118}
119EXPORT_SYMBOL_GPL(task_blkio_cgroup);
120
Vivek Goyal062a6442010-09-15 17:06:33 -0400121static inline void
122blkio_update_group_weight(struct blkio_group *blkg, unsigned int weight)
123{
124 struct blkio_policy_type *blkiop;
125
126 list_for_each_entry(blkiop, &blkio_list, list) {
127 /* If this policy does not own the blkg, do not send updates */
128 if (blkiop->plid != blkg->plid)
129 continue;
130 if (blkiop->ops.blkio_update_group_weight_fn)
Vivek Goyalfe071432010-10-01 14:49:49 +0200131 blkiop->ops.blkio_update_group_weight_fn(blkg->key,
132 blkg, weight);
Vivek Goyal062a6442010-09-15 17:06:33 -0400133 }
134}
135
Vivek Goyal4c9eefa2010-09-15 17:06:34 -0400136static inline void blkio_update_group_bps(struct blkio_group *blkg, u64 bps,
137 int fileid)
138{
139 struct blkio_policy_type *blkiop;
140
141 list_for_each_entry(blkiop, &blkio_list, list) {
142
143 /* If this policy does not own the blkg, do not send updates */
144 if (blkiop->plid != blkg->plid)
145 continue;
146
147 if (fileid == BLKIO_THROTL_read_bps_device
148 && blkiop->ops.blkio_update_group_read_bps_fn)
Vivek Goyalfe071432010-10-01 14:49:49 +0200149 blkiop->ops.blkio_update_group_read_bps_fn(blkg->key,
150 blkg, bps);
Vivek Goyal4c9eefa2010-09-15 17:06:34 -0400151
152 if (fileid == BLKIO_THROTL_write_bps_device
153 && blkiop->ops.blkio_update_group_write_bps_fn)
Vivek Goyalfe071432010-10-01 14:49:49 +0200154 blkiop->ops.blkio_update_group_write_bps_fn(blkg->key,
155 blkg, bps);
Vivek Goyal4c9eefa2010-09-15 17:06:34 -0400156 }
157}
158
Vivek Goyal7702e8f2010-09-15 17:06:36 -0400159static inline void blkio_update_group_iops(struct blkio_group *blkg,
160 unsigned int iops, int fileid)
161{
162 struct blkio_policy_type *blkiop;
163
164 list_for_each_entry(blkiop, &blkio_list, list) {
165
166 /* If this policy does not own the blkg, do not send updates */
167 if (blkiop->plid != blkg->plid)
168 continue;
169
170 if (fileid == BLKIO_THROTL_read_iops_device
171 && blkiop->ops.blkio_update_group_read_iops_fn)
Vivek Goyalfe071432010-10-01 14:49:49 +0200172 blkiop->ops.blkio_update_group_read_iops_fn(blkg->key,
173 blkg, iops);
Vivek Goyal7702e8f2010-09-15 17:06:36 -0400174
175 if (fileid == BLKIO_THROTL_write_iops_device
176 && blkiop->ops.blkio_update_group_write_iops_fn)
Vivek Goyalfe071432010-10-01 14:49:49 +0200177 blkiop->ops.blkio_update_group_write_iops_fn(blkg->key,
178 blkg,iops);
Vivek Goyal7702e8f2010-09-15 17:06:36 -0400179 }
180}
181
Divyesh Shah91952912010-04-01 15:01:41 -0700182/*
183 * Add to the appropriate stat variable depending on the request type.
184 * This should be called with the blkg->stats_lock held.
185 */
Divyesh Shah84c124d2010-04-09 08:31:19 +0200186static void blkio_add_stat(uint64_t *stat, uint64_t add, bool direction,
187 bool sync)
Divyesh Shah91952912010-04-01 15:01:41 -0700188{
Divyesh Shah84c124d2010-04-09 08:31:19 +0200189 if (direction)
190 stat[BLKIO_STAT_WRITE] += add;
Divyesh Shah91952912010-04-01 15:01:41 -0700191 else
Divyesh Shah84c124d2010-04-09 08:31:19 +0200192 stat[BLKIO_STAT_READ] += add;
193 if (sync)
194 stat[BLKIO_STAT_SYNC] += add;
Divyesh Shah91952912010-04-01 15:01:41 -0700195 else
Divyesh Shah84c124d2010-04-09 08:31:19 +0200196 stat[BLKIO_STAT_ASYNC] += add;
Divyesh Shah91952912010-04-01 15:01:41 -0700197}
198
Divyesh Shahcdc11842010-04-08 21:15:10 -0700199/*
200 * Decrements the appropriate stat variable if non-zero depending on the
201 * request type. Panics on value being zero.
202 * This should be called with the blkg->stats_lock held.
203 */
204static void blkio_check_and_dec_stat(uint64_t *stat, bool direction, bool sync)
205{
206 if (direction) {
207 BUG_ON(stat[BLKIO_STAT_WRITE] == 0);
208 stat[BLKIO_STAT_WRITE]--;
209 } else {
210 BUG_ON(stat[BLKIO_STAT_READ] == 0);
211 stat[BLKIO_STAT_READ]--;
212 }
213 if (sync) {
214 BUG_ON(stat[BLKIO_STAT_SYNC] == 0);
215 stat[BLKIO_STAT_SYNC]--;
216 } else {
217 BUG_ON(stat[BLKIO_STAT_ASYNC] == 0);
218 stat[BLKIO_STAT_ASYNC]--;
219 }
220}
221
222#ifdef CONFIG_DEBUG_BLK_CGROUP
Divyesh Shah812df482010-04-08 21:15:35 -0700223/* This should be called with the blkg->stats_lock held. */
224static void blkio_set_start_group_wait_time(struct blkio_group *blkg,
225 struct blkio_group *curr_blkg)
226{
227 if (blkio_blkg_waiting(&blkg->stats))
228 return;
229 if (blkg == curr_blkg)
230 return;
231 blkg->stats.start_group_wait_time = sched_clock();
232 blkio_mark_blkg_waiting(&blkg->stats);
233}
234
235/* This should be called with the blkg->stats_lock held. */
236static void blkio_update_group_wait_time(struct blkio_group_stats *stats)
237{
238 unsigned long long now;
239
240 if (!blkio_blkg_waiting(stats))
241 return;
242
243 now = sched_clock();
244 if (time_after64(now, stats->start_group_wait_time))
245 stats->group_wait_time += now - stats->start_group_wait_time;
246 blkio_clear_blkg_waiting(stats);
247}
248
249/* This should be called with the blkg->stats_lock held. */
250static void blkio_end_empty_time(struct blkio_group_stats *stats)
251{
252 unsigned long long now;
253
254 if (!blkio_blkg_empty(stats))
255 return;
256
257 now = sched_clock();
258 if (time_after64(now, stats->start_empty_time))
259 stats->empty_time += now - stats->start_empty_time;
260 blkio_clear_blkg_empty(stats);
261}
262
263void blkiocg_update_set_idle_time_stats(struct blkio_group *blkg)
264{
265 unsigned long flags;
266
267 spin_lock_irqsave(&blkg->stats_lock, flags);
268 BUG_ON(blkio_blkg_idling(&blkg->stats));
269 blkg->stats.start_idle_time = sched_clock();
270 blkio_mark_blkg_idling(&blkg->stats);
271 spin_unlock_irqrestore(&blkg->stats_lock, flags);
272}
273EXPORT_SYMBOL_GPL(blkiocg_update_set_idle_time_stats);
274
275void blkiocg_update_idle_time_stats(struct blkio_group *blkg)
276{
277 unsigned long flags;
278 unsigned long long now;
279 struct blkio_group_stats *stats;
280
281 spin_lock_irqsave(&blkg->stats_lock, flags);
282 stats = &blkg->stats;
283 if (blkio_blkg_idling(stats)) {
284 now = sched_clock();
285 if (time_after64(now, stats->start_idle_time))
286 stats->idle_time += now - stats->start_idle_time;
287 blkio_clear_blkg_idling(stats);
288 }
289 spin_unlock_irqrestore(&blkg->stats_lock, flags);
290}
291EXPORT_SYMBOL_GPL(blkiocg_update_idle_time_stats);
292
Divyesh Shaha11cdaa2010-04-13 19:59:17 +0200293void blkiocg_update_avg_queue_size_stats(struct blkio_group *blkg)
Divyesh Shahcdc11842010-04-08 21:15:10 -0700294{
295 unsigned long flags;
296 struct blkio_group_stats *stats;
297
298 spin_lock_irqsave(&blkg->stats_lock, flags);
299 stats = &blkg->stats;
300 stats->avg_queue_size_sum +=
301 stats->stat_arr[BLKIO_STAT_QUEUED][BLKIO_STAT_READ] +
302 stats->stat_arr[BLKIO_STAT_QUEUED][BLKIO_STAT_WRITE];
303 stats->avg_queue_size_samples++;
Divyesh Shah812df482010-04-08 21:15:35 -0700304 blkio_update_group_wait_time(stats);
Divyesh Shahcdc11842010-04-08 21:15:10 -0700305 spin_unlock_irqrestore(&blkg->stats_lock, flags);
306}
Divyesh Shaha11cdaa2010-04-13 19:59:17 +0200307EXPORT_SYMBOL_GPL(blkiocg_update_avg_queue_size_stats);
308
Vivek Goyale5ff0822010-04-26 19:25:11 +0200309void blkiocg_set_start_empty_time(struct blkio_group *blkg)
Divyesh Shah28baf442010-04-14 11:22:38 +0200310{
311 unsigned long flags;
312 struct blkio_group_stats *stats;
313
314 spin_lock_irqsave(&blkg->stats_lock, flags);
315 stats = &blkg->stats;
316
317 if (stats->stat_arr[BLKIO_STAT_QUEUED][BLKIO_STAT_READ] ||
318 stats->stat_arr[BLKIO_STAT_QUEUED][BLKIO_STAT_WRITE]) {
319 spin_unlock_irqrestore(&blkg->stats_lock, flags);
320 return;
321 }
322
323 /*
Vivek Goyale5ff0822010-04-26 19:25:11 +0200324 * group is already marked empty. This can happen if cfqq got new
325 * request in parent group and moved to this group while being added
326 * to service tree. Just ignore the event and move on.
Divyesh Shah28baf442010-04-14 11:22:38 +0200327 */
Vivek Goyale5ff0822010-04-26 19:25:11 +0200328 if(blkio_blkg_empty(stats)) {
329 spin_unlock_irqrestore(&blkg->stats_lock, flags);
330 return;
331 }
332
Divyesh Shah28baf442010-04-14 11:22:38 +0200333 stats->start_empty_time = sched_clock();
334 blkio_mark_blkg_empty(stats);
335 spin_unlock_irqrestore(&blkg->stats_lock, flags);
336}
337EXPORT_SYMBOL_GPL(blkiocg_set_start_empty_time);
338
Divyesh Shaha11cdaa2010-04-13 19:59:17 +0200339void blkiocg_update_dequeue_stats(struct blkio_group *blkg,
340 unsigned long dequeue)
341{
342 blkg->stats.dequeue += dequeue;
343}
344EXPORT_SYMBOL_GPL(blkiocg_update_dequeue_stats);
Divyesh Shah812df482010-04-08 21:15:35 -0700345#else
346static inline void blkio_set_start_group_wait_time(struct blkio_group *blkg,
347 struct blkio_group *curr_blkg) {}
348static inline void blkio_end_empty_time(struct blkio_group_stats *stats) {}
Divyesh Shahcdc11842010-04-08 21:15:10 -0700349#endif
350
Divyesh Shaha11cdaa2010-04-13 19:59:17 +0200351void blkiocg_update_io_add_stats(struct blkio_group *blkg,
Divyesh Shahcdc11842010-04-08 21:15:10 -0700352 struct blkio_group *curr_blkg, bool direction,
353 bool sync)
354{
355 unsigned long flags;
356
357 spin_lock_irqsave(&blkg->stats_lock, flags);
358 blkio_add_stat(blkg->stats.stat_arr[BLKIO_STAT_QUEUED], 1, direction,
359 sync);
Divyesh Shah812df482010-04-08 21:15:35 -0700360 blkio_end_empty_time(&blkg->stats);
361 blkio_set_start_group_wait_time(blkg, curr_blkg);
Divyesh Shahcdc11842010-04-08 21:15:10 -0700362 spin_unlock_irqrestore(&blkg->stats_lock, flags);
363}
Divyesh Shaha11cdaa2010-04-13 19:59:17 +0200364EXPORT_SYMBOL_GPL(blkiocg_update_io_add_stats);
Divyesh Shahcdc11842010-04-08 21:15:10 -0700365
Divyesh Shaha11cdaa2010-04-13 19:59:17 +0200366void blkiocg_update_io_remove_stats(struct blkio_group *blkg,
Divyesh Shahcdc11842010-04-08 21:15:10 -0700367 bool direction, bool sync)
368{
369 unsigned long flags;
370
371 spin_lock_irqsave(&blkg->stats_lock, flags);
372 blkio_check_and_dec_stat(blkg->stats.stat_arr[BLKIO_STAT_QUEUED],
373 direction, sync);
374 spin_unlock_irqrestore(&blkg->stats_lock, flags);
375}
Divyesh Shaha11cdaa2010-04-13 19:59:17 +0200376EXPORT_SYMBOL_GPL(blkiocg_update_io_remove_stats);
Divyesh Shahcdc11842010-04-08 21:15:10 -0700377
Justin TerAvest167400d2011-03-12 16:54:00 +0100378void blkiocg_update_timeslice_used(struct blkio_group *blkg, unsigned long time,
379 unsigned long unaccounted_time)
Vivek Goyal22084192009-12-03 12:59:49 -0500380{
Divyesh Shah303a3ac2010-04-01 15:01:24 -0700381 unsigned long flags;
382
383 spin_lock_irqsave(&blkg->stats_lock, flags);
384 blkg->stats.time += time;
Vivek Goyala23e6862011-05-19 15:38:20 -0400385#ifdef CONFIG_DEBUG_BLK_CGROUP
Justin TerAvest167400d2011-03-12 16:54:00 +0100386 blkg->stats.unaccounted_time += unaccounted_time;
Vivek Goyala23e6862011-05-19 15:38:20 -0400387#endif
Divyesh Shah303a3ac2010-04-01 15:01:24 -0700388 spin_unlock_irqrestore(&blkg->stats_lock, flags);
Vivek Goyal22084192009-12-03 12:59:49 -0500389}
Divyesh Shah303a3ac2010-04-01 15:01:24 -0700390EXPORT_SYMBOL_GPL(blkiocg_update_timeslice_used);
Vivek Goyal22084192009-12-03 12:59:49 -0500391
Vivek Goyal5624a4e2011-05-19 15:38:28 -0400392/*
393 * should be called under rcu read lock or queue lock to make sure blkg pointer
394 * is valid.
395 */
Divyesh Shah84c124d2010-04-09 08:31:19 +0200396void blkiocg_update_dispatch_stats(struct blkio_group *blkg,
397 uint64_t bytes, bool direction, bool sync)
Divyesh Shah91952912010-04-01 15:01:41 -0700398{
Vivek Goyal5624a4e2011-05-19 15:38:28 -0400399 struct blkio_group_stats_cpu *stats_cpu;
Vivek Goyal575969a2011-05-19 15:38:29 -0400400 unsigned long flags;
401
402 /*
403 * Disabling interrupts to provide mutual exclusion between two
404 * writes on same cpu. It probably is not needed for 64bit. Not
405 * optimizing that case yet.
406 */
407 local_irq_save(flags);
Divyesh Shah91952912010-04-01 15:01:41 -0700408
Vivek Goyal5624a4e2011-05-19 15:38:28 -0400409 stats_cpu = this_cpu_ptr(blkg->stats_cpu);
410
Vivek Goyal575969a2011-05-19 15:38:29 -0400411 u64_stats_update_begin(&stats_cpu->syncp);
Vivek Goyal5624a4e2011-05-19 15:38:28 -0400412 stats_cpu->sectors += bytes >> 9;
413 blkio_add_stat(stats_cpu->stat_arr_cpu[BLKIO_STAT_CPU_SERVICED],
414 1, direction, sync);
415 blkio_add_stat(stats_cpu->stat_arr_cpu[BLKIO_STAT_CPU_SERVICE_BYTES],
416 bytes, direction, sync);
Vivek Goyal575969a2011-05-19 15:38:29 -0400417 u64_stats_update_end(&stats_cpu->syncp);
418 local_irq_restore(flags);
Divyesh Shah91952912010-04-01 15:01:41 -0700419}
Divyesh Shah84c124d2010-04-09 08:31:19 +0200420EXPORT_SYMBOL_GPL(blkiocg_update_dispatch_stats);
Divyesh Shah91952912010-04-01 15:01:41 -0700421
Divyesh Shah84c124d2010-04-09 08:31:19 +0200422void blkiocg_update_completion_stats(struct blkio_group *blkg,
423 uint64_t start_time, uint64_t io_start_time, bool direction, bool sync)
Divyesh Shah91952912010-04-01 15:01:41 -0700424{
425 struct blkio_group_stats *stats;
426 unsigned long flags;
427 unsigned long long now = sched_clock();
428
429 spin_lock_irqsave(&blkg->stats_lock, flags);
430 stats = &blkg->stats;
Divyesh Shah84c124d2010-04-09 08:31:19 +0200431 if (time_after64(now, io_start_time))
432 blkio_add_stat(stats->stat_arr[BLKIO_STAT_SERVICE_TIME],
433 now - io_start_time, direction, sync);
434 if (time_after64(io_start_time, start_time))
435 blkio_add_stat(stats->stat_arr[BLKIO_STAT_WAIT_TIME],
436 io_start_time - start_time, direction, sync);
Divyesh Shah91952912010-04-01 15:01:41 -0700437 spin_unlock_irqrestore(&blkg->stats_lock, flags);
438}
Divyesh Shah84c124d2010-04-09 08:31:19 +0200439EXPORT_SYMBOL_GPL(blkiocg_update_completion_stats);
Divyesh Shah91952912010-04-01 15:01:41 -0700440
Vivek Goyal317389a2011-05-23 10:02:19 +0200441/* Merged stats are per cpu. */
Divyesh Shah812d4022010-04-08 21:14:23 -0700442void blkiocg_update_io_merged_stats(struct blkio_group *blkg, bool direction,
443 bool sync)
444{
Vivek Goyal317389a2011-05-23 10:02:19 +0200445 struct blkio_group_stats_cpu *stats_cpu;
Divyesh Shah812d4022010-04-08 21:14:23 -0700446 unsigned long flags;
447
Vivek Goyal317389a2011-05-23 10:02:19 +0200448 /*
449 * Disabling interrupts to provide mutual exclusion between two
450 * writes on same cpu. It probably is not needed for 64bit. Not
451 * optimizing that case yet.
452 */
453 local_irq_save(flags);
454
455 stats_cpu = this_cpu_ptr(blkg->stats_cpu);
456
457 u64_stats_update_begin(&stats_cpu->syncp);
458 blkio_add_stat(stats_cpu->stat_arr_cpu[BLKIO_STAT_CPU_MERGED], 1,
459 direction, sync);
460 u64_stats_update_end(&stats_cpu->syncp);
461 local_irq_restore(flags);
Divyesh Shah812d4022010-04-08 21:14:23 -0700462}
463EXPORT_SYMBOL_GPL(blkiocg_update_io_merged_stats);
464
Vivek Goyal5624a4e2011-05-19 15:38:28 -0400465/*
466 * This function allocates the per cpu stats for blkio_group. Should be called
467 * from sleepable context as alloc_per_cpu() requires that.
468 */
469int blkio_alloc_blkg_stats(struct blkio_group *blkg)
470{
471 /* Allocate memory for per cpu stats */
472 blkg->stats_cpu = alloc_percpu(struct blkio_group_stats_cpu);
473 if (!blkg->stats_cpu)
474 return -ENOMEM;
475 return 0;
476}
477EXPORT_SYMBOL_GPL(blkio_alloc_blkg_stats);
478
Vivek Goyal31e4c282009-12-03 12:59:42 -0500479void blkiocg_add_blkio_group(struct blkio_cgroup *blkcg,
Vivek Goyal062a6442010-09-15 17:06:33 -0400480 struct blkio_group *blkg, void *key, dev_t dev,
481 enum blkio_policy_id plid)
Vivek Goyal31e4c282009-12-03 12:59:42 -0500482{
483 unsigned long flags;
484
485 spin_lock_irqsave(&blkcg->lock, flags);
Divyesh Shah8d2a91f2010-04-16 08:10:51 +0200486 spin_lock_init(&blkg->stats_lock);
Vivek Goyal31e4c282009-12-03 12:59:42 -0500487 rcu_assign_pointer(blkg->key, key);
Vivek Goyalb1c35762009-12-03 12:59:47 -0500488 blkg->blkcg_id = css_id(&blkcg->css);
Vivek Goyal31e4c282009-12-03 12:59:42 -0500489 hlist_add_head_rcu(&blkg->blkcg_node, &blkcg->blkg_list);
Vivek Goyal062a6442010-09-15 17:06:33 -0400490 blkg->plid = plid;
Vivek Goyal31e4c282009-12-03 12:59:42 -0500491 spin_unlock_irqrestore(&blkcg->lock, flags);
Vivek Goyal2868ef72009-12-03 12:59:48 -0500492 /* Need to take css reference ? */
493 cgroup_path(blkcg->css.cgroup, blkg->path, sizeof(blkg->path));
Vivek Goyal22084192009-12-03 12:59:49 -0500494 blkg->dev = dev;
Vivek Goyal31e4c282009-12-03 12:59:42 -0500495}
Vivek Goyal9d6a9862009-12-04 10:36:41 -0500496EXPORT_SYMBOL_GPL(blkiocg_add_blkio_group);
Vivek Goyal31e4c282009-12-03 12:59:42 -0500497
Vivek Goyalb1c35762009-12-03 12:59:47 -0500498static void __blkiocg_del_blkio_group(struct blkio_group *blkg)
499{
500 hlist_del_init_rcu(&blkg->blkcg_node);
501 blkg->blkcg_id = 0;
502}
503
504/*
505 * returns 0 if blkio_group was still on cgroup list. Otherwise returns 1
506 * indicating that blk_group was unhashed by the time we got to it.
507 */
Vivek Goyal31e4c282009-12-03 12:59:42 -0500508int blkiocg_del_blkio_group(struct blkio_group *blkg)
509{
Vivek Goyalb1c35762009-12-03 12:59:47 -0500510 struct blkio_cgroup *blkcg;
511 unsigned long flags;
512 struct cgroup_subsys_state *css;
513 int ret = 1;
514
515 rcu_read_lock();
516 css = css_lookup(&blkio_subsys, blkg->blkcg_id);
Jens Axboe0f3942a2010-05-03 14:28:55 +0200517 if (css) {
518 blkcg = container_of(css, struct blkio_cgroup, css);
519 spin_lock_irqsave(&blkcg->lock, flags);
520 if (!hlist_unhashed(&blkg->blkcg_node)) {
521 __blkiocg_del_blkio_group(blkg);
522 ret = 0;
523 }
524 spin_unlock_irqrestore(&blkcg->lock, flags);
Vivek Goyalb1c35762009-12-03 12:59:47 -0500525 }
Jens Axboe0f3942a2010-05-03 14:28:55 +0200526
Vivek Goyalb1c35762009-12-03 12:59:47 -0500527 rcu_read_unlock();
528 return ret;
Vivek Goyal31e4c282009-12-03 12:59:42 -0500529}
Vivek Goyal9d6a9862009-12-04 10:36:41 -0500530EXPORT_SYMBOL_GPL(blkiocg_del_blkio_group);
Vivek Goyal31e4c282009-12-03 12:59:42 -0500531
532/* called under rcu_read_lock(). */
533struct blkio_group *blkiocg_lookup_group(struct blkio_cgroup *blkcg, void *key)
534{
535 struct blkio_group *blkg;
536 struct hlist_node *n;
537 void *__key;
538
539 hlist_for_each_entry_rcu(blkg, n, &blkcg->blkg_list, blkcg_node) {
540 __key = blkg->key;
541 if (__key == key)
542 return blkg;
543 }
544
545 return NULL;
546}
Vivek Goyal9d6a9862009-12-04 10:36:41 -0500547EXPORT_SYMBOL_GPL(blkiocg_lookup_group);
Vivek Goyal31e4c282009-12-03 12:59:42 -0500548
Vivek Goyalf0bdc8c2011-05-19 15:38:30 -0400549static void blkio_reset_stats_cpu(struct blkio_group *blkg)
550{
551 struct blkio_group_stats_cpu *stats_cpu;
552 int i, j, k;
553 /*
554 * Note: On 64 bit arch this should not be an issue. This has the
555 * possibility of returning some inconsistent value on 32bit arch
556 * as 64bit update on 32bit is non atomic. Taking care of this
557 * corner case makes code very complicated, like sending IPIs to
558 * cpus, taking care of stats of offline cpus etc.
559 *
560 * reset stats is anyway more of a debug feature and this sounds a
561 * corner case. So I am not complicating the code yet until and
562 * unless this becomes a real issue.
563 */
564 for_each_possible_cpu(i) {
565 stats_cpu = per_cpu_ptr(blkg->stats_cpu, i);
566 stats_cpu->sectors = 0;
567 for(j = 0; j < BLKIO_STAT_CPU_NR; j++)
568 for (k = 0; k < BLKIO_STAT_TOTAL; k++)
569 stats_cpu->stat_arr_cpu[j][k] = 0;
570 }
571}
572
Divyesh Shah303a3ac2010-04-01 15:01:24 -0700573static int
Divyesh Shah84c124d2010-04-09 08:31:19 +0200574blkiocg_reset_stats(struct cgroup *cgroup, struct cftype *cftype, u64 val)
Divyesh Shah303a3ac2010-04-01 15:01:24 -0700575{
576 struct blkio_cgroup *blkcg;
577 struct blkio_group *blkg;
Divyesh Shah812df482010-04-08 21:15:35 -0700578 struct blkio_group_stats *stats;
Divyesh Shah303a3ac2010-04-01 15:01:24 -0700579 struct hlist_node *n;
Divyesh Shahcdc11842010-04-08 21:15:10 -0700580 uint64_t queued[BLKIO_STAT_TOTAL];
581 int i;
Divyesh Shah812df482010-04-08 21:15:35 -0700582#ifdef CONFIG_DEBUG_BLK_CGROUP
583 bool idling, waiting, empty;
584 unsigned long long now = sched_clock();
585#endif
Divyesh Shah303a3ac2010-04-01 15:01:24 -0700586
587 blkcg = cgroup_to_blkio_cgroup(cgroup);
588 spin_lock_irq(&blkcg->lock);
589 hlist_for_each_entry(blkg, n, &blkcg->blkg_list, blkcg_node) {
590 spin_lock(&blkg->stats_lock);
Divyesh Shah812df482010-04-08 21:15:35 -0700591 stats = &blkg->stats;
592#ifdef CONFIG_DEBUG_BLK_CGROUP
593 idling = blkio_blkg_idling(stats);
594 waiting = blkio_blkg_waiting(stats);
595 empty = blkio_blkg_empty(stats);
596#endif
Divyesh Shahcdc11842010-04-08 21:15:10 -0700597 for (i = 0; i < BLKIO_STAT_TOTAL; i++)
Divyesh Shah812df482010-04-08 21:15:35 -0700598 queued[i] = stats->stat_arr[BLKIO_STAT_QUEUED][i];
599 memset(stats, 0, sizeof(struct blkio_group_stats));
Divyesh Shahcdc11842010-04-08 21:15:10 -0700600 for (i = 0; i < BLKIO_STAT_TOTAL; i++)
Divyesh Shah812df482010-04-08 21:15:35 -0700601 stats->stat_arr[BLKIO_STAT_QUEUED][i] = queued[i];
602#ifdef CONFIG_DEBUG_BLK_CGROUP
603 if (idling) {
604 blkio_mark_blkg_idling(stats);
605 stats->start_idle_time = now;
606 }
607 if (waiting) {
608 blkio_mark_blkg_waiting(stats);
609 stats->start_group_wait_time = now;
610 }
611 if (empty) {
612 blkio_mark_blkg_empty(stats);
613 stats->start_empty_time = now;
614 }
615#endif
Divyesh Shah303a3ac2010-04-01 15:01:24 -0700616 spin_unlock(&blkg->stats_lock);
Vivek Goyalf0bdc8c2011-05-19 15:38:30 -0400617
618 /* Reset Per cpu stats which don't take blkg->stats_lock */
619 blkio_reset_stats_cpu(blkg);
Divyesh Shah303a3ac2010-04-01 15:01:24 -0700620 }
Vivek Goyalf0bdc8c2011-05-19 15:38:30 -0400621
Divyesh Shah303a3ac2010-04-01 15:01:24 -0700622 spin_unlock_irq(&blkcg->lock);
623 return 0;
624}
625
Divyesh Shah84c124d2010-04-09 08:31:19 +0200626static void blkio_get_key_name(enum stat_sub_type type, dev_t dev, char *str,
627 int chars_left, bool diskname_only)
Divyesh Shah303a3ac2010-04-01 15:01:24 -0700628{
Divyesh Shah84c124d2010-04-09 08:31:19 +0200629 snprintf(str, chars_left, "%d:%d", MAJOR(dev), MINOR(dev));
Divyesh Shah303a3ac2010-04-01 15:01:24 -0700630 chars_left -= strlen(str);
631 if (chars_left <= 0) {
632 printk(KERN_WARNING
633 "Possibly incorrect cgroup stat display format");
634 return;
635 }
Divyesh Shah84c124d2010-04-09 08:31:19 +0200636 if (diskname_only)
637 return;
Divyesh Shah303a3ac2010-04-01 15:01:24 -0700638 switch (type) {
Divyesh Shah84c124d2010-04-09 08:31:19 +0200639 case BLKIO_STAT_READ:
Divyesh Shah303a3ac2010-04-01 15:01:24 -0700640 strlcat(str, " Read", chars_left);
641 break;
Divyesh Shah84c124d2010-04-09 08:31:19 +0200642 case BLKIO_STAT_WRITE:
Divyesh Shah303a3ac2010-04-01 15:01:24 -0700643 strlcat(str, " Write", chars_left);
644 break;
Divyesh Shah84c124d2010-04-09 08:31:19 +0200645 case BLKIO_STAT_SYNC:
Divyesh Shah303a3ac2010-04-01 15:01:24 -0700646 strlcat(str, " Sync", chars_left);
647 break;
Divyesh Shah84c124d2010-04-09 08:31:19 +0200648 case BLKIO_STAT_ASYNC:
Divyesh Shah303a3ac2010-04-01 15:01:24 -0700649 strlcat(str, " Async", chars_left);
650 break;
Divyesh Shah84c124d2010-04-09 08:31:19 +0200651 case BLKIO_STAT_TOTAL:
Divyesh Shah303a3ac2010-04-01 15:01:24 -0700652 strlcat(str, " Total", chars_left);
653 break;
654 default:
655 strlcat(str, " Invalid", chars_left);
656 }
657}
658
Divyesh Shah84c124d2010-04-09 08:31:19 +0200659static uint64_t blkio_fill_stat(char *str, int chars_left, uint64_t val,
660 struct cgroup_map_cb *cb, dev_t dev)
661{
662 blkio_get_key_name(0, dev, str, chars_left, true);
663 cb->fill(cb, str, val);
664 return val;
665}
Divyesh Shah303a3ac2010-04-01 15:01:24 -0700666
Vivek Goyal5624a4e2011-05-19 15:38:28 -0400667
668static uint64_t blkio_read_stat_cpu(struct blkio_group *blkg,
669 enum stat_type_cpu type, enum stat_sub_type sub_type)
670{
671 int cpu;
672 struct blkio_group_stats_cpu *stats_cpu;
Vivek Goyal575969a2011-05-19 15:38:29 -0400673 u64 val = 0, tval;
Vivek Goyal5624a4e2011-05-19 15:38:28 -0400674
675 for_each_possible_cpu(cpu) {
Vivek Goyal575969a2011-05-19 15:38:29 -0400676 unsigned int start;
Vivek Goyal5624a4e2011-05-19 15:38:28 -0400677 stats_cpu = per_cpu_ptr(blkg->stats_cpu, cpu);
678
Vivek Goyal575969a2011-05-19 15:38:29 -0400679 do {
680 start = u64_stats_fetch_begin(&stats_cpu->syncp);
681 if (type == BLKIO_STAT_CPU_SECTORS)
682 tval = stats_cpu->sectors;
683 else
684 tval = stats_cpu->stat_arr_cpu[type][sub_type];
685 } while(u64_stats_fetch_retry(&stats_cpu->syncp, start));
686
687 val += tval;
Vivek Goyal5624a4e2011-05-19 15:38:28 -0400688 }
689
690 return val;
691}
692
693static uint64_t blkio_get_stat_cpu(struct blkio_group *blkg,
694 struct cgroup_map_cb *cb, dev_t dev, enum stat_type_cpu type)
695{
696 uint64_t disk_total, val;
697 char key_str[MAX_KEY_LEN];
698 enum stat_sub_type sub_type;
699
700 if (type == BLKIO_STAT_CPU_SECTORS) {
701 val = blkio_read_stat_cpu(blkg, type, 0);
702 return blkio_fill_stat(key_str, MAX_KEY_LEN - 1, val, cb, dev);
703 }
704
705 for (sub_type = BLKIO_STAT_READ; sub_type < BLKIO_STAT_TOTAL;
706 sub_type++) {
707 blkio_get_key_name(sub_type, dev, key_str, MAX_KEY_LEN, false);
708 val = blkio_read_stat_cpu(blkg, type, sub_type);
709 cb->fill(cb, key_str, val);
710 }
711
712 disk_total = blkio_read_stat_cpu(blkg, type, BLKIO_STAT_READ) +
713 blkio_read_stat_cpu(blkg, type, BLKIO_STAT_WRITE);
714
715 blkio_get_key_name(BLKIO_STAT_TOTAL, dev, key_str, MAX_KEY_LEN, false);
716 cb->fill(cb, key_str, disk_total);
717 return disk_total;
718}
719
Divyesh Shah84c124d2010-04-09 08:31:19 +0200720/* This should be called with blkg->stats_lock held */
721static uint64_t blkio_get_stat(struct blkio_group *blkg,
722 struct cgroup_map_cb *cb, dev_t dev, enum stat_type type)
Divyesh Shah303a3ac2010-04-01 15:01:24 -0700723{
724 uint64_t disk_total;
725 char key_str[MAX_KEY_LEN];
Divyesh Shah84c124d2010-04-09 08:31:19 +0200726 enum stat_sub_type sub_type;
Divyesh Shah303a3ac2010-04-01 15:01:24 -0700727
Divyesh Shah84c124d2010-04-09 08:31:19 +0200728 if (type == BLKIO_STAT_TIME)
729 return blkio_fill_stat(key_str, MAX_KEY_LEN - 1,
730 blkg->stats.time, cb, dev);
Justin TerAvest9026e522011-03-22 21:26:54 +0100731#ifdef CONFIG_DEBUG_BLK_CGROUP
Justin TerAvest167400d2011-03-12 16:54:00 +0100732 if (type == BLKIO_STAT_UNACCOUNTED_TIME)
733 return blkio_fill_stat(key_str, MAX_KEY_LEN - 1,
734 blkg->stats.unaccounted_time, cb, dev);
Divyesh Shahcdc11842010-04-08 21:15:10 -0700735 if (type == BLKIO_STAT_AVG_QUEUE_SIZE) {
736 uint64_t sum = blkg->stats.avg_queue_size_sum;
737 uint64_t samples = blkg->stats.avg_queue_size_samples;
738 if (samples)
739 do_div(sum, samples);
740 else
741 sum = 0;
742 return blkio_fill_stat(key_str, MAX_KEY_LEN - 1, sum, cb, dev);
743 }
Divyesh Shah812df482010-04-08 21:15:35 -0700744 if (type == BLKIO_STAT_GROUP_WAIT_TIME)
745 return blkio_fill_stat(key_str, MAX_KEY_LEN - 1,
746 blkg->stats.group_wait_time, cb, dev);
747 if (type == BLKIO_STAT_IDLE_TIME)
748 return blkio_fill_stat(key_str, MAX_KEY_LEN - 1,
749 blkg->stats.idle_time, cb, dev);
750 if (type == BLKIO_STAT_EMPTY_TIME)
751 return blkio_fill_stat(key_str, MAX_KEY_LEN - 1,
752 blkg->stats.empty_time, cb, dev);
Divyesh Shah84c124d2010-04-09 08:31:19 +0200753 if (type == BLKIO_STAT_DEQUEUE)
754 return blkio_fill_stat(key_str, MAX_KEY_LEN - 1,
755 blkg->stats.dequeue, cb, dev);
756#endif
757
758 for (sub_type = BLKIO_STAT_READ; sub_type < BLKIO_STAT_TOTAL;
759 sub_type++) {
760 blkio_get_key_name(sub_type, dev, key_str, MAX_KEY_LEN, false);
761 cb->fill(cb, key_str, blkg->stats.stat_arr[type][sub_type]);
Divyesh Shah303a3ac2010-04-01 15:01:24 -0700762 }
Divyesh Shah84c124d2010-04-09 08:31:19 +0200763 disk_total = blkg->stats.stat_arr[type][BLKIO_STAT_READ] +
764 blkg->stats.stat_arr[type][BLKIO_STAT_WRITE];
765 blkio_get_key_name(BLKIO_STAT_TOTAL, dev, key_str, MAX_KEY_LEN, false);
Divyesh Shah303a3ac2010-04-01 15:01:24 -0700766 cb->fill(cb, key_str, disk_total);
767 return disk_total;
768}
769
Gui Jianfeng34d0f172010-04-13 16:05:49 +0800770static int blkio_policy_parse_and_set(char *buf,
Vivek Goyal062a6442010-09-15 17:06:33 -0400771 struct blkio_policy_node *newpn, enum blkio_policy_id plid, int fileid)
Gui Jianfeng34d0f172010-04-13 16:05:49 +0800772{
Tejun Heoece84242011-10-19 14:31:15 +0200773 struct gendisk *disk = NULL;
Gui Jianfeng34d0f172010-04-13 16:05:49 +0800774 char *s[4], *p, *major_s = NULL, *minor_s = NULL;
Wanlong Gaod11bb442011-09-21 10:22:10 +0200775 unsigned long major, minor;
Tejun Heoece84242011-10-19 14:31:15 +0200776 int i = 0, ret = -EINVAL;
777 int part;
Gui Jianfeng34d0f172010-04-13 16:05:49 +0800778 dev_t dev;
Wanlong Gaod11bb442011-09-21 10:22:10 +0200779 u64 temp;
Gui Jianfeng34d0f172010-04-13 16:05:49 +0800780
781 memset(s, 0, sizeof(s));
782
783 while ((p = strsep(&buf, " ")) != NULL) {
784 if (!*p)
785 continue;
786
787 s[i++] = p;
788
789 /* Prevent from inputing too many things */
790 if (i == 3)
791 break;
792 }
793
794 if (i != 2)
Tejun Heoece84242011-10-19 14:31:15 +0200795 goto out;
Gui Jianfeng34d0f172010-04-13 16:05:49 +0800796
797 p = strsep(&s[0], ":");
798 if (p != NULL)
799 major_s = p;
800 else
Tejun Heoece84242011-10-19 14:31:15 +0200801 goto out;
Gui Jianfeng34d0f172010-04-13 16:05:49 +0800802
803 minor_s = s[0];
804 if (!minor_s)
Tejun Heoece84242011-10-19 14:31:15 +0200805 goto out;
Gui Jianfeng34d0f172010-04-13 16:05:49 +0800806
Tejun Heoece84242011-10-19 14:31:15 +0200807 if (strict_strtoul(major_s, 10, &major))
808 goto out;
Gui Jianfeng34d0f172010-04-13 16:05:49 +0800809
Tejun Heoece84242011-10-19 14:31:15 +0200810 if (strict_strtoul(minor_s, 10, &minor))
811 goto out;
Gui Jianfeng34d0f172010-04-13 16:05:49 +0800812
813 dev = MKDEV(major, minor);
814
Tejun Heoece84242011-10-19 14:31:15 +0200815 if (strict_strtoull(s[1], 10, &temp))
816 goto out;
Wanlong Gaod11bb442011-09-21 10:22:10 +0200817
818 /* For rule removal, do not check for device presence. */
819 if (temp) {
Tejun Heoece84242011-10-19 14:31:15 +0200820 disk = get_gendisk(dev, &part);
821 if (!disk || part) {
822 ret = -ENODEV;
823 goto out;
824 }
Wanlong Gaod11bb442011-09-21 10:22:10 +0200825 }
Gui Jianfeng34d0f172010-04-13 16:05:49 +0800826
827 newpn->dev = dev;
828
Vivek Goyal062a6442010-09-15 17:06:33 -0400829 switch (plid) {
830 case BLKIO_POLICY_PROP:
Wanlong Gaod11bb442011-09-21 10:22:10 +0200831 if ((temp < BLKIO_WEIGHT_MIN && temp > 0) ||
832 temp > BLKIO_WEIGHT_MAX)
Tejun Heoece84242011-10-19 14:31:15 +0200833 goto out;
Gui Jianfeng34d0f172010-04-13 16:05:49 +0800834
Vivek Goyal062a6442010-09-15 17:06:33 -0400835 newpn->plid = plid;
836 newpn->fileid = fileid;
Vivek Goyal4c9eefa2010-09-15 17:06:34 -0400837 newpn->val.weight = temp;
838 break;
839 case BLKIO_POLICY_THROTL:
Vivek Goyal7702e8f2010-09-15 17:06:36 -0400840 switch(fileid) {
841 case BLKIO_THROTL_read_bps_device:
842 case BLKIO_THROTL_write_bps_device:
Vivek Goyal7702e8f2010-09-15 17:06:36 -0400843 newpn->plid = plid;
844 newpn->fileid = fileid;
Wanlong Gaod11bb442011-09-21 10:22:10 +0200845 newpn->val.bps = temp;
Vivek Goyal7702e8f2010-09-15 17:06:36 -0400846 break;
847 case BLKIO_THROTL_read_iops_device:
848 case BLKIO_THROTL_write_iops_device:
Wanlong Gaod11bb442011-09-21 10:22:10 +0200849 if (temp > THROTL_IOPS_MAX)
Tejun Heoece84242011-10-19 14:31:15 +0200850 goto out;
Vivek Goyal9355aed2010-10-01 21:16:41 +0200851
Vivek Goyal7702e8f2010-09-15 17:06:36 -0400852 newpn->plid = plid;
853 newpn->fileid = fileid;
Wanlong Gaod11bb442011-09-21 10:22:10 +0200854 newpn->val.iops = (unsigned int)temp;
Vivek Goyal7702e8f2010-09-15 17:06:36 -0400855 break;
856 }
Vivek Goyal062a6442010-09-15 17:06:33 -0400857 break;
858 default:
859 BUG();
860 }
Tejun Heoece84242011-10-19 14:31:15 +0200861 ret = 0;
862out:
863 put_disk(disk);
864 return ret;
Gui Jianfeng34d0f172010-04-13 16:05:49 +0800865}
866
867unsigned int blkcg_get_weight(struct blkio_cgroup *blkcg,
868 dev_t dev)
869{
870 struct blkio_policy_node *pn;
Vivek Goyala38eb632011-10-25 15:48:12 +0200871 unsigned long flags;
872 unsigned int weight;
873
874 spin_lock_irqsave(&blkcg->lock, flags);
Gui Jianfeng34d0f172010-04-13 16:05:49 +0800875
Vivek Goyal062a6442010-09-15 17:06:33 -0400876 pn = blkio_policy_search_node(blkcg, dev, BLKIO_POLICY_PROP,
877 BLKIO_PROP_weight_device);
Gui Jianfeng34d0f172010-04-13 16:05:49 +0800878 if (pn)
Vivek Goyala38eb632011-10-25 15:48:12 +0200879 weight = pn->val.weight;
Gui Jianfeng34d0f172010-04-13 16:05:49 +0800880 else
Vivek Goyala38eb632011-10-25 15:48:12 +0200881 weight = blkcg->weight;
882
883 spin_unlock_irqrestore(&blkcg->lock, flags);
884
885 return weight;
Gui Jianfeng34d0f172010-04-13 16:05:49 +0800886}
887EXPORT_SYMBOL_GPL(blkcg_get_weight);
888
Vivek Goyal4c9eefa2010-09-15 17:06:34 -0400889uint64_t blkcg_get_read_bps(struct blkio_cgroup *blkcg, dev_t dev)
890{
891 struct blkio_policy_node *pn;
Vivek Goyala38eb632011-10-25 15:48:12 +0200892 unsigned long flags;
893 uint64_t bps = -1;
Gui Jianfeng34d0f172010-04-13 16:05:49 +0800894
Vivek Goyala38eb632011-10-25 15:48:12 +0200895 spin_lock_irqsave(&blkcg->lock, flags);
Vivek Goyal4c9eefa2010-09-15 17:06:34 -0400896 pn = blkio_policy_search_node(blkcg, dev, BLKIO_POLICY_THROTL,
897 BLKIO_THROTL_read_bps_device);
898 if (pn)
Vivek Goyala38eb632011-10-25 15:48:12 +0200899 bps = pn->val.bps;
900 spin_unlock_irqrestore(&blkcg->lock, flags);
901
902 return bps;
Vivek Goyal4c9eefa2010-09-15 17:06:34 -0400903}
904
905uint64_t blkcg_get_write_bps(struct blkio_cgroup *blkcg, dev_t dev)
906{
907 struct blkio_policy_node *pn;
Vivek Goyala38eb632011-10-25 15:48:12 +0200908 unsigned long flags;
909 uint64_t bps = -1;
910
911 spin_lock_irqsave(&blkcg->lock, flags);
Vivek Goyal4c9eefa2010-09-15 17:06:34 -0400912 pn = blkio_policy_search_node(blkcg, dev, BLKIO_POLICY_THROTL,
913 BLKIO_THROTL_write_bps_device);
914 if (pn)
Vivek Goyala38eb632011-10-25 15:48:12 +0200915 bps = pn->val.bps;
916 spin_unlock_irqrestore(&blkcg->lock, flags);
917
918 return bps;
Vivek Goyal4c9eefa2010-09-15 17:06:34 -0400919}
920
Vivek Goyal7702e8f2010-09-15 17:06:36 -0400921unsigned int blkcg_get_read_iops(struct blkio_cgroup *blkcg, dev_t dev)
922{
923 struct blkio_policy_node *pn;
Vivek Goyala38eb632011-10-25 15:48:12 +0200924 unsigned long flags;
925 unsigned int iops = -1;
Vivek Goyal7702e8f2010-09-15 17:06:36 -0400926
Vivek Goyala38eb632011-10-25 15:48:12 +0200927 spin_lock_irqsave(&blkcg->lock, flags);
Vivek Goyal7702e8f2010-09-15 17:06:36 -0400928 pn = blkio_policy_search_node(blkcg, dev, BLKIO_POLICY_THROTL,
929 BLKIO_THROTL_read_iops_device);
930 if (pn)
Vivek Goyala38eb632011-10-25 15:48:12 +0200931 iops = pn->val.iops;
932 spin_unlock_irqrestore(&blkcg->lock, flags);
933
934 return iops;
Vivek Goyal7702e8f2010-09-15 17:06:36 -0400935}
936
937unsigned int blkcg_get_write_iops(struct blkio_cgroup *blkcg, dev_t dev)
938{
939 struct blkio_policy_node *pn;
Vivek Goyala38eb632011-10-25 15:48:12 +0200940 unsigned long flags;
941 unsigned int iops = -1;
942
943 spin_lock_irqsave(&blkcg->lock, flags);
Vivek Goyal7702e8f2010-09-15 17:06:36 -0400944 pn = blkio_policy_search_node(blkcg, dev, BLKIO_POLICY_THROTL,
945 BLKIO_THROTL_write_iops_device);
946 if (pn)
Vivek Goyala38eb632011-10-25 15:48:12 +0200947 iops = pn->val.iops;
948 spin_unlock_irqrestore(&blkcg->lock, flags);
949
950 return iops;
Vivek Goyal7702e8f2010-09-15 17:06:36 -0400951}
952
Vivek Goyal062a6442010-09-15 17:06:33 -0400953/* Checks whether user asked for deleting a policy rule */
954static bool blkio_delete_rule_command(struct blkio_policy_node *pn)
955{
956 switch(pn->plid) {
957 case BLKIO_POLICY_PROP:
Vivek Goyal4c9eefa2010-09-15 17:06:34 -0400958 if (pn->val.weight == 0)
959 return 1;
960 break;
961 case BLKIO_POLICY_THROTL:
Vivek Goyal7702e8f2010-09-15 17:06:36 -0400962 switch(pn->fileid) {
963 case BLKIO_THROTL_read_bps_device:
964 case BLKIO_THROTL_write_bps_device:
965 if (pn->val.bps == 0)
966 return 1;
967 break;
968 case BLKIO_THROTL_read_iops_device:
969 case BLKIO_THROTL_write_iops_device:
970 if (pn->val.iops == 0)
971 return 1;
972 }
Vivek Goyal062a6442010-09-15 17:06:33 -0400973 break;
974 default:
975 BUG();
976 }
Gui Jianfeng34d0f172010-04-13 16:05:49 +0800977
Vivek Goyal062a6442010-09-15 17:06:33 -0400978 return 0;
979}
980
981static void blkio_update_policy_rule(struct blkio_policy_node *oldpn,
982 struct blkio_policy_node *newpn)
983{
984 switch(oldpn->plid) {
985 case BLKIO_POLICY_PROP:
Vivek Goyal4c9eefa2010-09-15 17:06:34 -0400986 oldpn->val.weight = newpn->val.weight;
987 break;
988 case BLKIO_POLICY_THROTL:
Vivek Goyal7702e8f2010-09-15 17:06:36 -0400989 switch(newpn->fileid) {
990 case BLKIO_THROTL_read_bps_device:
991 case BLKIO_THROTL_write_bps_device:
992 oldpn->val.bps = newpn->val.bps;
993 break;
994 case BLKIO_THROTL_read_iops_device:
995 case BLKIO_THROTL_write_iops_device:
996 oldpn->val.iops = newpn->val.iops;
997 }
Vivek Goyal062a6442010-09-15 17:06:33 -0400998 break;
999 default:
1000 BUG();
1001 }
1002}
1003
1004/*
Lucas De Marchi25985ed2011-03-30 22:57:33 -03001005 * Some rules/values in blkg have changed. Propagate those to respective
Vivek Goyal062a6442010-09-15 17:06:33 -04001006 * policies.
1007 */
1008static void blkio_update_blkg_policy(struct blkio_cgroup *blkcg,
1009 struct blkio_group *blkg, struct blkio_policy_node *pn)
1010{
Vivek Goyal7702e8f2010-09-15 17:06:36 -04001011 unsigned int weight, iops;
Vivek Goyal4c9eefa2010-09-15 17:06:34 -04001012 u64 bps;
Vivek Goyal062a6442010-09-15 17:06:33 -04001013
1014 switch(pn->plid) {
1015 case BLKIO_POLICY_PROP:
Vivek Goyal4c9eefa2010-09-15 17:06:34 -04001016 weight = pn->val.weight ? pn->val.weight :
Vivek Goyal062a6442010-09-15 17:06:33 -04001017 blkcg->weight;
1018 blkio_update_group_weight(blkg, weight);
1019 break;
Vivek Goyal4c9eefa2010-09-15 17:06:34 -04001020 case BLKIO_POLICY_THROTL:
1021 switch(pn->fileid) {
1022 case BLKIO_THROTL_read_bps_device:
1023 case BLKIO_THROTL_write_bps_device:
1024 bps = pn->val.bps ? pn->val.bps : (-1);
1025 blkio_update_group_bps(blkg, bps, pn->fileid);
1026 break;
Vivek Goyal7702e8f2010-09-15 17:06:36 -04001027 case BLKIO_THROTL_read_iops_device:
1028 case BLKIO_THROTL_write_iops_device:
1029 iops = pn->val.iops ? pn->val.iops : (-1);
1030 blkio_update_group_iops(blkg, iops, pn->fileid);
1031 break;
Vivek Goyal4c9eefa2010-09-15 17:06:34 -04001032 }
1033 break;
Vivek Goyal062a6442010-09-15 17:06:33 -04001034 default:
1035 BUG();
1036 }
1037}
1038
1039/*
Lucas De Marchi25985ed2011-03-30 22:57:33 -03001040 * A policy node rule has been updated. Propagate this update to all the
Vivek Goyal062a6442010-09-15 17:06:33 -04001041 * block groups which might be affected by this update.
1042 */
1043static void blkio_update_policy_node_blkg(struct blkio_cgroup *blkcg,
1044 struct blkio_policy_node *pn)
1045{
1046 struct blkio_group *blkg;
1047 struct hlist_node *n;
1048
1049 spin_lock(&blkio_list_lock);
1050 spin_lock_irq(&blkcg->lock);
1051
1052 hlist_for_each_entry(blkg, n, &blkcg->blkg_list, blkcg_node) {
1053 if (pn->dev != blkg->dev || pn->plid != blkg->plid)
1054 continue;
1055 blkio_update_blkg_policy(blkcg, blkg, pn);
1056 }
1057
1058 spin_unlock_irq(&blkcg->lock);
1059 spin_unlock(&blkio_list_lock);
1060}
1061
1062static int blkiocg_file_write(struct cgroup *cgrp, struct cftype *cft,
1063 const char *buffer)
Gui Jianfeng34d0f172010-04-13 16:05:49 +08001064{
1065 int ret = 0;
1066 char *buf;
1067 struct blkio_policy_node *newpn, *pn;
1068 struct blkio_cgroup *blkcg;
Gui Jianfeng34d0f172010-04-13 16:05:49 +08001069 int keep_newpn = 0;
Vivek Goyal062a6442010-09-15 17:06:33 -04001070 enum blkio_policy_id plid = BLKIOFILE_POLICY(cft->private);
1071 int fileid = BLKIOFILE_ATTR(cft->private);
Gui Jianfeng34d0f172010-04-13 16:05:49 +08001072
1073 buf = kstrdup(buffer, GFP_KERNEL);
1074 if (!buf)
1075 return -ENOMEM;
1076
1077 newpn = kzalloc(sizeof(*newpn), GFP_KERNEL);
1078 if (!newpn) {
1079 ret = -ENOMEM;
1080 goto free_buf;
1081 }
1082
Vivek Goyal062a6442010-09-15 17:06:33 -04001083 ret = blkio_policy_parse_and_set(buf, newpn, plid, fileid);
Gui Jianfeng34d0f172010-04-13 16:05:49 +08001084 if (ret)
1085 goto free_newpn;
1086
1087 blkcg = cgroup_to_blkio_cgroup(cgrp);
1088
1089 spin_lock_irq(&blkcg->lock);
1090
Vivek Goyal062a6442010-09-15 17:06:33 -04001091 pn = blkio_policy_search_node(blkcg, newpn->dev, plid, fileid);
Gui Jianfeng34d0f172010-04-13 16:05:49 +08001092 if (!pn) {
Vivek Goyal062a6442010-09-15 17:06:33 -04001093 if (!blkio_delete_rule_command(newpn)) {
Gui Jianfeng34d0f172010-04-13 16:05:49 +08001094 blkio_policy_insert_node(blkcg, newpn);
1095 keep_newpn = 1;
1096 }
1097 spin_unlock_irq(&blkcg->lock);
1098 goto update_io_group;
1099 }
1100
Vivek Goyal062a6442010-09-15 17:06:33 -04001101 if (blkio_delete_rule_command(newpn)) {
Gui Jianfeng34d0f172010-04-13 16:05:49 +08001102 blkio_policy_delete_node(pn);
Vivek Goyale060f002011-10-25 15:48:12 +02001103 kfree(pn);
Gui Jianfeng34d0f172010-04-13 16:05:49 +08001104 spin_unlock_irq(&blkcg->lock);
1105 goto update_io_group;
1106 }
1107 spin_unlock_irq(&blkcg->lock);
1108
Vivek Goyal062a6442010-09-15 17:06:33 -04001109 blkio_update_policy_rule(pn, newpn);
Gui Jianfeng34d0f172010-04-13 16:05:49 +08001110
1111update_io_group:
Vivek Goyal062a6442010-09-15 17:06:33 -04001112 blkio_update_policy_node_blkg(blkcg, newpn);
Gui Jianfeng34d0f172010-04-13 16:05:49 +08001113
1114free_newpn:
1115 if (!keep_newpn)
1116 kfree(newpn);
1117free_buf:
1118 kfree(buf);
1119 return ret;
1120}
1121
Vivek Goyal062a6442010-09-15 17:06:33 -04001122static void
1123blkio_print_policy_node(struct seq_file *m, struct blkio_policy_node *pn)
Gui Jianfeng34d0f172010-04-13 16:05:49 +08001124{
Vivek Goyal062a6442010-09-15 17:06:33 -04001125 switch(pn->plid) {
1126 case BLKIO_POLICY_PROP:
1127 if (pn->fileid == BLKIO_PROP_weight_device)
1128 seq_printf(m, "%u:%u\t%u\n", MAJOR(pn->dev),
Vivek Goyal4c9eefa2010-09-15 17:06:34 -04001129 MINOR(pn->dev), pn->val.weight);
1130 break;
1131 case BLKIO_POLICY_THROTL:
Vivek Goyal7702e8f2010-09-15 17:06:36 -04001132 switch(pn->fileid) {
1133 case BLKIO_THROTL_read_bps_device:
1134 case BLKIO_THROTL_write_bps_device:
Vivek Goyal4c9eefa2010-09-15 17:06:34 -04001135 seq_printf(m, "%u:%u\t%llu\n", MAJOR(pn->dev),
1136 MINOR(pn->dev), pn->val.bps);
Vivek Goyal7702e8f2010-09-15 17:06:36 -04001137 break;
1138 case BLKIO_THROTL_read_iops_device:
1139 case BLKIO_THROTL_write_iops_device:
1140 seq_printf(m, "%u:%u\t%u\n", MAJOR(pn->dev),
1141 MINOR(pn->dev), pn->val.iops);
1142 break;
1143 }
Vivek Goyal062a6442010-09-15 17:06:33 -04001144 break;
1145 default:
1146 BUG();
1147 }
1148}
1149
1150/* cgroup files which read their data from policy nodes end up here */
1151static void blkio_read_policy_node_files(struct cftype *cft,
1152 struct blkio_cgroup *blkcg, struct seq_file *m)
1153{
Gui Jianfeng34d0f172010-04-13 16:05:49 +08001154 struct blkio_policy_node *pn;
1155
Jens Axboe0f3942a2010-05-03 14:28:55 +02001156 if (!list_empty(&blkcg->policy_list)) {
1157 spin_lock_irq(&blkcg->lock);
1158 list_for_each_entry(pn, &blkcg->policy_list, node) {
Vivek Goyal062a6442010-09-15 17:06:33 -04001159 if (!pn_matches_cftype(cft, pn))
1160 continue;
1161 blkio_print_policy_node(m, pn);
Jens Axboe0f3942a2010-05-03 14:28:55 +02001162 }
1163 spin_unlock_irq(&blkcg->lock);
Gui Jianfeng34d0f172010-04-13 16:05:49 +08001164 }
Vivek Goyal062a6442010-09-15 17:06:33 -04001165}
1166
1167static int blkiocg_file_read(struct cgroup *cgrp, struct cftype *cft,
1168 struct seq_file *m)
1169{
1170 struct blkio_cgroup *blkcg;
1171 enum blkio_policy_id plid = BLKIOFILE_POLICY(cft->private);
1172 int name = BLKIOFILE_ATTR(cft->private);
1173
1174 blkcg = cgroup_to_blkio_cgroup(cgrp);
1175
1176 switch(plid) {
1177 case BLKIO_POLICY_PROP:
1178 switch(name) {
1179 case BLKIO_PROP_weight_device:
1180 blkio_read_policy_node_files(cft, blkcg, m);
1181 return 0;
1182 default:
1183 BUG();
1184 }
1185 break;
Vivek Goyal4c9eefa2010-09-15 17:06:34 -04001186 case BLKIO_POLICY_THROTL:
1187 switch(name){
1188 case BLKIO_THROTL_read_bps_device:
1189 case BLKIO_THROTL_write_bps_device:
Vivek Goyal7702e8f2010-09-15 17:06:36 -04001190 case BLKIO_THROTL_read_iops_device:
1191 case BLKIO_THROTL_write_iops_device:
Vivek Goyal4c9eefa2010-09-15 17:06:34 -04001192 blkio_read_policy_node_files(cft, blkcg, m);
1193 return 0;
1194 default:
1195 BUG();
1196 }
1197 break;
Vivek Goyal062a6442010-09-15 17:06:33 -04001198 default:
1199 BUG();
1200 }
1201
1202 return 0;
1203}
1204
1205static int blkio_read_blkg_stats(struct blkio_cgroup *blkcg,
Vivek Goyal5624a4e2011-05-19 15:38:28 -04001206 struct cftype *cft, struct cgroup_map_cb *cb,
1207 enum stat_type type, bool show_total, bool pcpu)
Vivek Goyal062a6442010-09-15 17:06:33 -04001208{
1209 struct blkio_group *blkg;
1210 struct hlist_node *n;
1211 uint64_t cgroup_total = 0;
1212
1213 rcu_read_lock();
1214 hlist_for_each_entry_rcu(blkg, n, &blkcg->blkg_list, blkcg_node) {
1215 if (blkg->dev) {
1216 if (!cftype_blkg_same_policy(cft, blkg))
1217 continue;
Vivek Goyal5624a4e2011-05-19 15:38:28 -04001218 if (pcpu)
1219 cgroup_total += blkio_get_stat_cpu(blkg, cb,
1220 blkg->dev, type);
1221 else {
1222 spin_lock_irq(&blkg->stats_lock);
1223 cgroup_total += blkio_get_stat(blkg, cb,
1224 blkg->dev, type);
1225 spin_unlock_irq(&blkg->stats_lock);
1226 }
Vivek Goyal062a6442010-09-15 17:06:33 -04001227 }
1228 }
1229 if (show_total)
1230 cb->fill(cb, "Total", cgroup_total);
1231 rcu_read_unlock();
1232 return 0;
1233}
1234
1235/* All map kind of cgroup file get serviced by this function */
1236static int blkiocg_file_read_map(struct cgroup *cgrp, struct cftype *cft,
1237 struct cgroup_map_cb *cb)
1238{
1239 struct blkio_cgroup *blkcg;
1240 enum blkio_policy_id plid = BLKIOFILE_POLICY(cft->private);
1241 int name = BLKIOFILE_ATTR(cft->private);
1242
1243 blkcg = cgroup_to_blkio_cgroup(cgrp);
1244
1245 switch(plid) {
1246 case BLKIO_POLICY_PROP:
1247 switch(name) {
1248 case BLKIO_PROP_time:
1249 return blkio_read_blkg_stats(blkcg, cft, cb,
Vivek Goyal5624a4e2011-05-19 15:38:28 -04001250 BLKIO_STAT_TIME, 0, 0);
Vivek Goyal062a6442010-09-15 17:06:33 -04001251 case BLKIO_PROP_sectors:
1252 return blkio_read_blkg_stats(blkcg, cft, cb,
Vivek Goyal5624a4e2011-05-19 15:38:28 -04001253 BLKIO_STAT_CPU_SECTORS, 0, 1);
Vivek Goyal062a6442010-09-15 17:06:33 -04001254 case BLKIO_PROP_io_service_bytes:
1255 return blkio_read_blkg_stats(blkcg, cft, cb,
Vivek Goyal5624a4e2011-05-19 15:38:28 -04001256 BLKIO_STAT_CPU_SERVICE_BYTES, 1, 1);
Vivek Goyal062a6442010-09-15 17:06:33 -04001257 case BLKIO_PROP_io_serviced:
1258 return blkio_read_blkg_stats(blkcg, cft, cb,
Vivek Goyal5624a4e2011-05-19 15:38:28 -04001259 BLKIO_STAT_CPU_SERVICED, 1, 1);
Vivek Goyal062a6442010-09-15 17:06:33 -04001260 case BLKIO_PROP_io_service_time:
1261 return blkio_read_blkg_stats(blkcg, cft, cb,
Vivek Goyal5624a4e2011-05-19 15:38:28 -04001262 BLKIO_STAT_SERVICE_TIME, 1, 0);
Vivek Goyal062a6442010-09-15 17:06:33 -04001263 case BLKIO_PROP_io_wait_time:
1264 return blkio_read_blkg_stats(blkcg, cft, cb,
Vivek Goyal5624a4e2011-05-19 15:38:28 -04001265 BLKIO_STAT_WAIT_TIME, 1, 0);
Vivek Goyal062a6442010-09-15 17:06:33 -04001266 case BLKIO_PROP_io_merged:
1267 return blkio_read_blkg_stats(blkcg, cft, cb,
Vivek Goyal317389a2011-05-23 10:02:19 +02001268 BLKIO_STAT_CPU_MERGED, 1, 1);
Vivek Goyal062a6442010-09-15 17:06:33 -04001269 case BLKIO_PROP_io_queued:
1270 return blkio_read_blkg_stats(blkcg, cft, cb,
Vivek Goyal5624a4e2011-05-19 15:38:28 -04001271 BLKIO_STAT_QUEUED, 1, 0);
Vivek Goyal062a6442010-09-15 17:06:33 -04001272#ifdef CONFIG_DEBUG_BLK_CGROUP
Justin TerAvest9026e522011-03-22 21:26:54 +01001273 case BLKIO_PROP_unaccounted_time:
1274 return blkio_read_blkg_stats(blkcg, cft, cb,
Vivek Goyal5624a4e2011-05-19 15:38:28 -04001275 BLKIO_STAT_UNACCOUNTED_TIME, 0, 0);
Vivek Goyal062a6442010-09-15 17:06:33 -04001276 case BLKIO_PROP_dequeue:
1277 return blkio_read_blkg_stats(blkcg, cft, cb,
Vivek Goyal5624a4e2011-05-19 15:38:28 -04001278 BLKIO_STAT_DEQUEUE, 0, 0);
Vivek Goyal062a6442010-09-15 17:06:33 -04001279 case BLKIO_PROP_avg_queue_size:
1280 return blkio_read_blkg_stats(blkcg, cft, cb,
Vivek Goyal5624a4e2011-05-19 15:38:28 -04001281 BLKIO_STAT_AVG_QUEUE_SIZE, 0, 0);
Vivek Goyal062a6442010-09-15 17:06:33 -04001282 case BLKIO_PROP_group_wait_time:
1283 return blkio_read_blkg_stats(blkcg, cft, cb,
Vivek Goyal5624a4e2011-05-19 15:38:28 -04001284 BLKIO_STAT_GROUP_WAIT_TIME, 0, 0);
Vivek Goyal062a6442010-09-15 17:06:33 -04001285 case BLKIO_PROP_idle_time:
1286 return blkio_read_blkg_stats(blkcg, cft, cb,
Vivek Goyal5624a4e2011-05-19 15:38:28 -04001287 BLKIO_STAT_IDLE_TIME, 0, 0);
Vivek Goyal062a6442010-09-15 17:06:33 -04001288 case BLKIO_PROP_empty_time:
1289 return blkio_read_blkg_stats(blkcg, cft, cb,
Vivek Goyal5624a4e2011-05-19 15:38:28 -04001290 BLKIO_STAT_EMPTY_TIME, 0, 0);
Vivek Goyal062a6442010-09-15 17:06:33 -04001291#endif
1292 default:
1293 BUG();
1294 }
1295 break;
Vivek Goyal4c9eefa2010-09-15 17:06:34 -04001296 case BLKIO_POLICY_THROTL:
1297 switch(name){
1298 case BLKIO_THROTL_io_service_bytes:
1299 return blkio_read_blkg_stats(blkcg, cft, cb,
Vivek Goyal5624a4e2011-05-19 15:38:28 -04001300 BLKIO_STAT_CPU_SERVICE_BYTES, 1, 1);
Vivek Goyal4c9eefa2010-09-15 17:06:34 -04001301 case BLKIO_THROTL_io_serviced:
1302 return blkio_read_blkg_stats(blkcg, cft, cb,
Vivek Goyal5624a4e2011-05-19 15:38:28 -04001303 BLKIO_STAT_CPU_SERVICED, 1, 1);
Vivek Goyal4c9eefa2010-09-15 17:06:34 -04001304 default:
1305 BUG();
1306 }
1307 break;
Vivek Goyal062a6442010-09-15 17:06:33 -04001308 default:
1309 BUG();
1310 }
1311
1312 return 0;
1313}
1314
1315static int blkio_weight_write(struct blkio_cgroup *blkcg, u64 val)
1316{
1317 struct blkio_group *blkg;
1318 struct hlist_node *n;
1319 struct blkio_policy_node *pn;
1320
1321 if (val < BLKIO_WEIGHT_MIN || val > BLKIO_WEIGHT_MAX)
1322 return -EINVAL;
1323
1324 spin_lock(&blkio_list_lock);
1325 spin_lock_irq(&blkcg->lock);
1326 blkcg->weight = (unsigned int)val;
1327
1328 hlist_for_each_entry(blkg, n, &blkcg->blkg_list, blkcg_node) {
1329 pn = blkio_policy_search_node(blkcg, blkg->dev,
1330 BLKIO_POLICY_PROP, BLKIO_PROP_weight_device);
1331 if (pn)
1332 continue;
1333
1334 blkio_update_group_weight(blkg, blkcg->weight);
1335 }
1336 spin_unlock_irq(&blkcg->lock);
1337 spin_unlock(&blkio_list_lock);
1338 return 0;
1339}
1340
1341static u64 blkiocg_file_read_u64 (struct cgroup *cgrp, struct cftype *cft) {
1342 struct blkio_cgroup *blkcg;
1343 enum blkio_policy_id plid = BLKIOFILE_POLICY(cft->private);
1344 int name = BLKIOFILE_ATTR(cft->private);
1345
1346 blkcg = cgroup_to_blkio_cgroup(cgrp);
1347
1348 switch(plid) {
1349 case BLKIO_POLICY_PROP:
1350 switch(name) {
1351 case BLKIO_PROP_weight:
1352 return (u64)blkcg->weight;
1353 }
1354 break;
1355 default:
1356 BUG();
1357 }
1358 return 0;
1359}
1360
1361static int
1362blkiocg_file_write_u64(struct cgroup *cgrp, struct cftype *cft, u64 val)
1363{
1364 struct blkio_cgroup *blkcg;
1365 enum blkio_policy_id plid = BLKIOFILE_POLICY(cft->private);
1366 int name = BLKIOFILE_ATTR(cft->private);
1367
1368 blkcg = cgroup_to_blkio_cgroup(cgrp);
1369
1370 switch(plid) {
1371 case BLKIO_POLICY_PROP:
1372 switch(name) {
1373 case BLKIO_PROP_weight:
1374 return blkio_weight_write(blkcg, val);
1375 }
1376 break;
1377 default:
1378 BUG();
1379 }
Gui Jianfeng34d0f172010-04-13 16:05:49 +08001380
Gui Jianfeng34d0f172010-04-13 16:05:49 +08001381 return 0;
1382}
1383
Vivek Goyal31e4c282009-12-03 12:59:42 -05001384struct cftype blkio_files[] = {
1385 {
Gui Jianfeng34d0f172010-04-13 16:05:49 +08001386 .name = "weight_device",
Vivek Goyal062a6442010-09-15 17:06:33 -04001387 .private = BLKIOFILE_PRIVATE(BLKIO_POLICY_PROP,
1388 BLKIO_PROP_weight_device),
1389 .read_seq_string = blkiocg_file_read,
1390 .write_string = blkiocg_file_write,
Gui Jianfeng34d0f172010-04-13 16:05:49 +08001391 .max_write_len = 256,
1392 },
1393 {
Vivek Goyal31e4c282009-12-03 12:59:42 -05001394 .name = "weight",
Vivek Goyal062a6442010-09-15 17:06:33 -04001395 .private = BLKIOFILE_PRIVATE(BLKIO_POLICY_PROP,
1396 BLKIO_PROP_weight),
1397 .read_u64 = blkiocg_file_read_u64,
1398 .write_u64 = blkiocg_file_write_u64,
Vivek Goyal31e4c282009-12-03 12:59:42 -05001399 },
Vivek Goyal22084192009-12-03 12:59:49 -05001400 {
1401 .name = "time",
Vivek Goyal13f98252010-10-01 14:49:41 +02001402 .private = BLKIOFILE_PRIVATE(BLKIO_POLICY_PROP,
1403 BLKIO_PROP_time),
1404 .read_map = blkiocg_file_read_map,
Vivek Goyal22084192009-12-03 12:59:49 -05001405 },
1406 {
1407 .name = "sectors",
Vivek Goyal13f98252010-10-01 14:49:41 +02001408 .private = BLKIOFILE_PRIVATE(BLKIO_POLICY_PROP,
1409 BLKIO_PROP_sectors),
1410 .read_map = blkiocg_file_read_map,
Divyesh Shah303a3ac2010-04-01 15:01:24 -07001411 },
1412 {
1413 .name = "io_service_bytes",
Vivek Goyal13f98252010-10-01 14:49:41 +02001414 .private = BLKIOFILE_PRIVATE(BLKIO_POLICY_PROP,
1415 BLKIO_PROP_io_service_bytes),
1416 .read_map = blkiocg_file_read_map,
Divyesh Shah303a3ac2010-04-01 15:01:24 -07001417 },
1418 {
1419 .name = "io_serviced",
Vivek Goyal13f98252010-10-01 14:49:41 +02001420 .private = BLKIOFILE_PRIVATE(BLKIO_POLICY_PROP,
1421 BLKIO_PROP_io_serviced),
1422 .read_map = blkiocg_file_read_map,
Divyesh Shah303a3ac2010-04-01 15:01:24 -07001423 },
1424 {
1425 .name = "io_service_time",
Vivek Goyal13f98252010-10-01 14:49:41 +02001426 .private = BLKIOFILE_PRIVATE(BLKIO_POLICY_PROP,
1427 BLKIO_PROP_io_service_time),
1428 .read_map = blkiocg_file_read_map,
Divyesh Shah303a3ac2010-04-01 15:01:24 -07001429 },
1430 {
1431 .name = "io_wait_time",
Vivek Goyal13f98252010-10-01 14:49:41 +02001432 .private = BLKIOFILE_PRIVATE(BLKIO_POLICY_PROP,
1433 BLKIO_PROP_io_wait_time),
1434 .read_map = blkiocg_file_read_map,
Divyesh Shah84c124d2010-04-09 08:31:19 +02001435 },
1436 {
Divyesh Shah812d4022010-04-08 21:14:23 -07001437 .name = "io_merged",
Vivek Goyal13f98252010-10-01 14:49:41 +02001438 .private = BLKIOFILE_PRIVATE(BLKIO_POLICY_PROP,
1439 BLKIO_PROP_io_merged),
1440 .read_map = blkiocg_file_read_map,
Divyesh Shah812d4022010-04-08 21:14:23 -07001441 },
1442 {
Divyesh Shahcdc11842010-04-08 21:15:10 -07001443 .name = "io_queued",
Vivek Goyal13f98252010-10-01 14:49:41 +02001444 .private = BLKIOFILE_PRIVATE(BLKIO_POLICY_PROP,
1445 BLKIO_PROP_io_queued),
1446 .read_map = blkiocg_file_read_map,
Divyesh Shahcdc11842010-04-08 21:15:10 -07001447 },
1448 {
Divyesh Shah84c124d2010-04-09 08:31:19 +02001449 .name = "reset_stats",
1450 .write_u64 = blkiocg_reset_stats,
Vivek Goyal22084192009-12-03 12:59:49 -05001451 },
Vivek Goyal13f98252010-10-01 14:49:41 +02001452#ifdef CONFIG_BLK_DEV_THROTTLING
1453 {
Vivek Goyal4c9eefa2010-09-15 17:06:34 -04001454 .name = "throttle.read_bps_device",
1455 .private = BLKIOFILE_PRIVATE(BLKIO_POLICY_THROTL,
1456 BLKIO_THROTL_read_bps_device),
1457 .read_seq_string = blkiocg_file_read,
1458 .write_string = blkiocg_file_write,
1459 .max_write_len = 256,
1460 },
1461
1462 {
1463 .name = "throttle.write_bps_device",
1464 .private = BLKIOFILE_PRIVATE(BLKIO_POLICY_THROTL,
1465 BLKIO_THROTL_write_bps_device),
1466 .read_seq_string = blkiocg_file_read,
1467 .write_string = blkiocg_file_write,
1468 .max_write_len = 256,
1469 },
Vivek Goyal7702e8f2010-09-15 17:06:36 -04001470
1471 {
1472 .name = "throttle.read_iops_device",
1473 .private = BLKIOFILE_PRIVATE(BLKIO_POLICY_THROTL,
1474 BLKIO_THROTL_read_iops_device),
1475 .read_seq_string = blkiocg_file_read,
1476 .write_string = blkiocg_file_write,
1477 .max_write_len = 256,
1478 },
1479
1480 {
1481 .name = "throttle.write_iops_device",
1482 .private = BLKIOFILE_PRIVATE(BLKIO_POLICY_THROTL,
1483 BLKIO_THROTL_write_iops_device),
1484 .read_seq_string = blkiocg_file_read,
1485 .write_string = blkiocg_file_write,
1486 .max_write_len = 256,
1487 },
Vivek Goyal4c9eefa2010-09-15 17:06:34 -04001488 {
Vivek Goyal4c9eefa2010-09-15 17:06:34 -04001489 .name = "throttle.io_service_bytes",
1490 .private = BLKIOFILE_PRIVATE(BLKIO_POLICY_THROTL,
1491 BLKIO_THROTL_io_service_bytes),
1492 .read_map = blkiocg_file_read_map,
1493 },
1494 {
Vivek Goyal4c9eefa2010-09-15 17:06:34 -04001495 .name = "throttle.io_serviced",
1496 .private = BLKIOFILE_PRIVATE(BLKIO_POLICY_THROTL,
1497 BLKIO_THROTL_io_serviced),
1498 .read_map = blkiocg_file_read_map,
1499 },
Vivek Goyal13f98252010-10-01 14:49:41 +02001500#endif /* CONFIG_BLK_DEV_THROTTLING */
1501
Vivek Goyal22084192009-12-03 12:59:49 -05001502#ifdef CONFIG_DEBUG_BLK_CGROUP
Divyesh Shahcdc11842010-04-08 21:15:10 -07001503 {
1504 .name = "avg_queue_size",
Vivek Goyal062a6442010-09-15 17:06:33 -04001505 .private = BLKIOFILE_PRIVATE(BLKIO_POLICY_PROP,
1506 BLKIO_PROP_avg_queue_size),
1507 .read_map = blkiocg_file_read_map,
Divyesh Shahcdc11842010-04-08 21:15:10 -07001508 },
1509 {
Divyesh Shah812df482010-04-08 21:15:35 -07001510 .name = "group_wait_time",
Vivek Goyal062a6442010-09-15 17:06:33 -04001511 .private = BLKIOFILE_PRIVATE(BLKIO_POLICY_PROP,
1512 BLKIO_PROP_group_wait_time),
1513 .read_map = blkiocg_file_read_map,
Divyesh Shah812df482010-04-08 21:15:35 -07001514 },
1515 {
1516 .name = "idle_time",
Vivek Goyal062a6442010-09-15 17:06:33 -04001517 .private = BLKIOFILE_PRIVATE(BLKIO_POLICY_PROP,
1518 BLKIO_PROP_idle_time),
1519 .read_map = blkiocg_file_read_map,
Divyesh Shah812df482010-04-08 21:15:35 -07001520 },
1521 {
1522 .name = "empty_time",
Vivek Goyal062a6442010-09-15 17:06:33 -04001523 .private = BLKIOFILE_PRIVATE(BLKIO_POLICY_PROP,
1524 BLKIO_PROP_empty_time),
1525 .read_map = blkiocg_file_read_map,
Divyesh Shah812df482010-04-08 21:15:35 -07001526 },
1527 {
Vivek Goyal22084192009-12-03 12:59:49 -05001528 .name = "dequeue",
Vivek Goyal062a6442010-09-15 17:06:33 -04001529 .private = BLKIOFILE_PRIVATE(BLKIO_POLICY_PROP,
1530 BLKIO_PROP_dequeue),
1531 .read_map = blkiocg_file_read_map,
Divyesh Shahcdc11842010-04-08 21:15:10 -07001532 },
Justin TerAvest9026e522011-03-22 21:26:54 +01001533 {
1534 .name = "unaccounted_time",
1535 .private = BLKIOFILE_PRIVATE(BLKIO_POLICY_PROP,
1536 BLKIO_PROP_unaccounted_time),
1537 .read_map = blkiocg_file_read_map,
1538 },
Vivek Goyal22084192009-12-03 12:59:49 -05001539#endif
Vivek Goyal31e4c282009-12-03 12:59:42 -05001540};
1541
1542static int blkiocg_populate(struct cgroup_subsys *subsys, struct cgroup *cgroup)
1543{
1544 return cgroup_add_files(cgroup, subsys, blkio_files,
1545 ARRAY_SIZE(blkio_files));
1546}
1547
Li Zefan761b3ef2012-01-31 13:47:36 +08001548static void blkiocg_destroy(struct cgroup *cgroup)
Vivek Goyal31e4c282009-12-03 12:59:42 -05001549{
1550 struct blkio_cgroup *blkcg = cgroup_to_blkio_cgroup(cgroup);
Vivek Goyalb1c35762009-12-03 12:59:47 -05001551 unsigned long flags;
1552 struct blkio_group *blkg;
1553 void *key;
Vivek Goyal3e252062009-12-04 10:36:42 -05001554 struct blkio_policy_type *blkiop;
Gui Jianfeng34d0f172010-04-13 16:05:49 +08001555 struct blkio_policy_node *pn, *pntmp;
Vivek Goyal31e4c282009-12-03 12:59:42 -05001556
Vivek Goyalb1c35762009-12-03 12:59:47 -05001557 rcu_read_lock();
Jens Axboe0f3942a2010-05-03 14:28:55 +02001558 do {
1559 spin_lock_irqsave(&blkcg->lock, flags);
Vivek Goyalb1c35762009-12-03 12:59:47 -05001560
Jens Axboe0f3942a2010-05-03 14:28:55 +02001561 if (hlist_empty(&blkcg->blkg_list)) {
1562 spin_unlock_irqrestore(&blkcg->lock, flags);
1563 break;
1564 }
1565
1566 blkg = hlist_entry(blkcg->blkg_list.first, struct blkio_group,
1567 blkcg_node);
1568 key = rcu_dereference(blkg->key);
1569 __blkiocg_del_blkio_group(blkg);
1570
Vivek Goyalb1c35762009-12-03 12:59:47 -05001571 spin_unlock_irqrestore(&blkcg->lock, flags);
Vivek Goyalb1c35762009-12-03 12:59:47 -05001572
Jens Axboe0f3942a2010-05-03 14:28:55 +02001573 /*
1574 * This blkio_group is being unlinked as associated cgroup is
1575 * going away. Let all the IO controlling policies know about
Vivek Goyal61014e92010-10-01 14:49:44 +02001576 * this event.
Jens Axboe0f3942a2010-05-03 14:28:55 +02001577 */
1578 spin_lock(&blkio_list_lock);
Vivek Goyal61014e92010-10-01 14:49:44 +02001579 list_for_each_entry(blkiop, &blkio_list, list) {
1580 if (blkiop->plid != blkg->plid)
1581 continue;
Jens Axboe0f3942a2010-05-03 14:28:55 +02001582 blkiop->ops.blkio_unlink_group_fn(key, blkg);
Vivek Goyal61014e92010-10-01 14:49:44 +02001583 }
Jens Axboe0f3942a2010-05-03 14:28:55 +02001584 spin_unlock(&blkio_list_lock);
1585 } while (1);
Vivek Goyalb1c35762009-12-03 12:59:47 -05001586
Gui Jianfeng34d0f172010-04-13 16:05:49 +08001587 list_for_each_entry_safe(pn, pntmp, &blkcg->policy_list, node) {
1588 blkio_policy_delete_node(pn);
1589 kfree(pn);
1590 }
Jens Axboe0f3942a2010-05-03 14:28:55 +02001591
Vivek Goyal31e4c282009-12-03 12:59:42 -05001592 free_css_id(&blkio_subsys, &blkcg->css);
Vivek Goyalb1c35762009-12-03 12:59:47 -05001593 rcu_read_unlock();
Ben Blum67523c42010-03-10 15:22:11 -08001594 if (blkcg != &blkio_root_cgroup)
1595 kfree(blkcg);
Vivek Goyal31e4c282009-12-03 12:59:42 -05001596}
1597
Li Zefan761b3ef2012-01-31 13:47:36 +08001598static struct cgroup_subsys_state *blkiocg_create(struct cgroup *cgroup)
Vivek Goyal31e4c282009-12-03 12:59:42 -05001599{
Li Zefan03415092010-05-07 08:57:00 +02001600 struct blkio_cgroup *blkcg;
1601 struct cgroup *parent = cgroup->parent;
Vivek Goyal31e4c282009-12-03 12:59:42 -05001602
Li Zefan03415092010-05-07 08:57:00 +02001603 if (!parent) {
Vivek Goyal31e4c282009-12-03 12:59:42 -05001604 blkcg = &blkio_root_cgroup;
1605 goto done;
1606 }
1607
Vivek Goyal31e4c282009-12-03 12:59:42 -05001608 blkcg = kzalloc(sizeof(*blkcg), GFP_KERNEL);
1609 if (!blkcg)
1610 return ERR_PTR(-ENOMEM);
1611
1612 blkcg->weight = BLKIO_WEIGHT_DEFAULT;
1613done:
1614 spin_lock_init(&blkcg->lock);
1615 INIT_HLIST_HEAD(&blkcg->blkg_list);
1616
Gui Jianfeng34d0f172010-04-13 16:05:49 +08001617 INIT_LIST_HEAD(&blkcg->policy_list);
Vivek Goyal31e4c282009-12-03 12:59:42 -05001618 return &blkcg->css;
1619}
1620
1621/*
1622 * We cannot support shared io contexts, as we have no mean to support
1623 * two tasks with the same ioc in two different groups without major rework
1624 * of the main cic data structures. For now we allow a task to change
1625 * its cgroup only if it's the only owner of its ioc.
1626 */
Li Zefan761b3ef2012-01-31 13:47:36 +08001627static int blkiocg_can_attach(struct cgroup *cgrp, struct cgroup_taskset *tset)
Vivek Goyal31e4c282009-12-03 12:59:42 -05001628{
Tejun Heobb9d97b2011-12-12 18:12:21 -08001629 struct task_struct *task;
Vivek Goyal31e4c282009-12-03 12:59:42 -05001630 struct io_context *ioc;
1631 int ret = 0;
1632
1633 /* task_lock() is needed to avoid races with exit_io_context() */
Tejun Heobb9d97b2011-12-12 18:12:21 -08001634 cgroup_taskset_for_each(task, cgrp, tset) {
1635 task_lock(task);
1636 ioc = task->io_context;
1637 if (ioc && atomic_read(&ioc->nr_tasks) > 1)
1638 ret = -EINVAL;
1639 task_unlock(task);
1640 if (ret)
1641 break;
1642 }
Vivek Goyal31e4c282009-12-03 12:59:42 -05001643 return ret;
1644}
1645
Li Zefan761b3ef2012-01-31 13:47:36 +08001646static void blkiocg_attach(struct cgroup *cgrp, struct cgroup_taskset *tset)
Vivek Goyal31e4c282009-12-03 12:59:42 -05001647{
Tejun Heobb9d97b2011-12-12 18:12:21 -08001648 struct task_struct *task;
Vivek Goyal31e4c282009-12-03 12:59:42 -05001649 struct io_context *ioc;
1650
Tejun Heobb9d97b2011-12-12 18:12:21 -08001651 cgroup_taskset_for_each(task, cgrp, tset) {
Linus Torvaldsb3c9dd12012-01-15 12:24:45 -08001652 /* we don't lose anything even if ioc allocation fails */
1653 ioc = get_task_io_context(task, GFP_ATOMIC, NUMA_NO_NODE);
1654 if (ioc) {
1655 ioc_cgroup_changed(ioc);
Tejun Heo11a31222012-02-07 07:51:30 +01001656 put_io_context(ioc);
Linus Torvaldsb3c9dd12012-01-15 12:24:45 -08001657 }
Tejun Heobb9d97b2011-12-12 18:12:21 -08001658 }
Vivek Goyal31e4c282009-12-03 12:59:42 -05001659}
1660
Vivek Goyal3e252062009-12-04 10:36:42 -05001661void blkio_policy_register(struct blkio_policy_type *blkiop)
1662{
1663 spin_lock(&blkio_list_lock);
1664 list_add_tail(&blkiop->list, &blkio_list);
1665 spin_unlock(&blkio_list_lock);
1666}
1667EXPORT_SYMBOL_GPL(blkio_policy_register);
1668
1669void blkio_policy_unregister(struct blkio_policy_type *blkiop)
1670{
1671 spin_lock(&blkio_list_lock);
1672 list_del_init(&blkiop->list);
1673 spin_unlock(&blkio_list_lock);
1674}
1675EXPORT_SYMBOL_GPL(blkio_policy_unregister);
Ben Blum67523c42010-03-10 15:22:11 -08001676
1677static int __init init_cgroup_blkio(void)
1678{
1679 return cgroup_load_subsys(&blkio_subsys);
1680}
1681
1682static void __exit exit_cgroup_blkio(void)
1683{
1684 cgroup_unload_subsys(&blkio_subsys);
1685}
1686
1687module_init(init_cgroup_blkio);
1688module_exit(exit_cgroup_blkio);
1689MODULE_LICENSE("GPL");