blob: e9b7a47f6da02d230d9f1b144c753d3f80d1d720 [file] [log] [blame]
Vivek Goyale43473b2010-09-15 17:06:35 -04001/*
2 * Interface for controlling IO bandwidth on a request queue
3 *
4 * Copyright (C) 2010 Vivek Goyal <vgoyal@redhat.com>
5 */
6
7#include <linux/module.h>
8#include <linux/slab.h>
9#include <linux/blkdev.h>
10#include <linux/bio.h>
11#include <linux/blktrace_api.h>
12#include "blk-cgroup.h"
Tejun Heobc9fcbf2011-10-19 14:31:18 +020013#include "blk.h"
Vivek Goyale43473b2010-09-15 17:06:35 -040014
15/* Max dispatch from a group in 1 round */
16static int throtl_grp_quantum = 8;
17
18/* Total max dispatch from all groups in one round */
19static int throtl_quantum = 32;
20
21/* Throttling is performed over 100ms slice and after that slice is renewed */
22static unsigned long throtl_slice = HZ/10; /* 100 ms */
23
Tejun Heo03814112012-03-05 13:15:14 -080024static struct blkio_policy_type blkio_policy_throtl;
25
Vivek Goyal450adcb2011-03-01 13:40:54 -050026/* A workqueue to queue throttle related work */
27static struct workqueue_struct *kthrotld_workqueue;
28static void throtl_schedule_delayed_work(struct throtl_data *td,
29 unsigned long delay);
30
Vivek Goyale43473b2010-09-15 17:06:35 -040031struct throtl_rb_root {
32 struct rb_root rb;
33 struct rb_node *left;
34 unsigned int count;
35 unsigned long min_disptime;
36};
37
38#define THROTL_RB_ROOT (struct throtl_rb_root) { .rb = RB_ROOT, .left = NULL, \
39 .count = 0, .min_disptime = 0}
40
41#define rb_entry_tg(node) rb_entry((node), struct throtl_grp, rb_node)
42
Tejun Heo8a3d2612012-04-01 14:38:44 -070043/* Per-cpu group stats */
44struct tg_stats_cpu {
45 /* total bytes transferred */
46 struct blkg_rwstat service_bytes;
47 /* total IOs serviced, post merge */
48 struct blkg_rwstat serviced;
49};
50
Vivek Goyale43473b2010-09-15 17:06:35 -040051struct throtl_grp {
Vivek Goyale43473b2010-09-15 17:06:35 -040052 /* active throtl group service_tree member */
53 struct rb_node rb_node;
54
55 /*
56 * Dispatch time in jiffies. This is the estimated time when group
57 * will unthrottle and is ready to dispatch more bio. It is used as
58 * key to sort active groups in service tree.
59 */
60 unsigned long disptime;
61
Vivek Goyale43473b2010-09-15 17:06:35 -040062 unsigned int flags;
63
64 /* Two lists for READ and WRITE */
65 struct bio_list bio_lists[2];
66
67 /* Number of queued bios on READ and WRITE lists */
68 unsigned int nr_queued[2];
69
70 /* bytes per second rate limits */
71 uint64_t bps[2];
72
Vivek Goyal8e89d132010-09-15 17:06:37 -040073 /* IOPS limits */
74 unsigned int iops[2];
75
Vivek Goyale43473b2010-09-15 17:06:35 -040076 /* Number of bytes disptached in current slice */
77 uint64_t bytes_disp[2];
Vivek Goyal8e89d132010-09-15 17:06:37 -040078 /* Number of bio's dispatched in current slice */
79 unsigned int io_disp[2];
Vivek Goyale43473b2010-09-15 17:06:35 -040080
81 /* When did we start a new slice */
82 unsigned long slice_start[2];
83 unsigned long slice_end[2];
Vivek Goyalfe071432010-10-01 14:49:49 +020084
85 /* Some throttle limits got updated for the group */
Andreas Schwab6f037932011-03-30 12:21:56 +020086 int limits_changed;
Tejun Heo8a3d2612012-04-01 14:38:44 -070087
88 /* Per cpu stats pointer */
89 struct tg_stats_cpu __percpu *stats_cpu;
90
91 /* List of tgs waiting for per cpu stats memory to be allocated */
92 struct list_head stats_alloc_node;
Vivek Goyale43473b2010-09-15 17:06:35 -040093};
94
95struct throtl_data
96{
Vivek Goyale43473b2010-09-15 17:06:35 -040097 /* service tree for active throtl groups */
98 struct throtl_rb_root tg_service_tree;
99
Vivek Goyale43473b2010-09-15 17:06:35 -0400100 struct request_queue *queue;
101
102 /* Total Number of queued bios on READ and WRITE lists */
103 unsigned int nr_queued[2];
104
105 /*
Vivek Goyal02977e42010-10-01 14:49:48 +0200106 * number of total undestroyed groups
Vivek Goyale43473b2010-09-15 17:06:35 -0400107 */
108 unsigned int nr_undestroyed_grps;
109
110 /* Work for dispatching throttled bios */
111 struct delayed_work throtl_work;
Vivek Goyalfe071432010-10-01 14:49:49 +0200112
Andreas Schwab6f037932011-03-30 12:21:56 +0200113 int limits_changed;
Vivek Goyale43473b2010-09-15 17:06:35 -0400114};
115
Tejun Heo8a3d2612012-04-01 14:38:44 -0700116/* list and work item to allocate percpu group stats */
117static DEFINE_SPINLOCK(tg_stats_alloc_lock);
118static LIST_HEAD(tg_stats_alloc_list);
119
120static void tg_stats_alloc_fn(struct work_struct *);
121static DECLARE_DELAYED_WORK(tg_stats_alloc_work, tg_stats_alloc_fn);
122
Tejun Heo03814112012-03-05 13:15:14 -0800123static inline struct throtl_grp *blkg_to_tg(struct blkio_group *blkg)
124{
125 return blkg_to_pdata(blkg, &blkio_policy_throtl);
126}
127
128static inline struct blkio_group *tg_to_blkg(struct throtl_grp *tg)
129{
Tejun Heoaaec55a2012-04-01 14:38:42 -0700130 return pdata_to_blkg(tg);
Tejun Heo03814112012-03-05 13:15:14 -0800131}
132
Tejun Heo03d8e112012-04-13 13:11:32 -0700133static inline struct throtl_grp *td_root_tg(struct throtl_data *td)
134{
135 return blkg_to_tg(td->queue->root_blkg);
136}
137
Vivek Goyale43473b2010-09-15 17:06:35 -0400138enum tg_state_flags {
139 THROTL_TG_FLAG_on_rr = 0, /* on round-robin busy list */
140};
141
142#define THROTL_TG_FNS(name) \
143static inline void throtl_mark_tg_##name(struct throtl_grp *tg) \
144{ \
145 (tg)->flags |= (1 << THROTL_TG_FLAG_##name); \
146} \
147static inline void throtl_clear_tg_##name(struct throtl_grp *tg) \
148{ \
149 (tg)->flags &= ~(1 << THROTL_TG_FLAG_##name); \
150} \
151static inline int throtl_tg_##name(const struct throtl_grp *tg) \
152{ \
153 return ((tg)->flags & (1 << THROTL_TG_FLAG_##name)) != 0; \
154}
155
156THROTL_TG_FNS(on_rr);
157
Tejun Heo54e7ed12012-04-16 13:57:23 -0700158#define throtl_log_tg(td, tg, fmt, args...) do { \
159 char __pbuf[128]; \
160 \
161 blkg_path(tg_to_blkg(tg), __pbuf, sizeof(__pbuf)); \
162 blk_add_trace_msg((td)->queue, "throtl %s " fmt, __pbuf, ##args); \
163} while (0)
Vivek Goyale43473b2010-09-15 17:06:35 -0400164
165#define throtl_log(td, fmt, args...) \
166 blk_add_trace_msg((td)->queue, "throtl " fmt, ##args)
167
Joe Perchesd2f31a52011-06-13 20:19:27 +0200168static inline unsigned int total_nr_queued(struct throtl_data *td)
Vivek Goyale43473b2010-09-15 17:06:35 -0400169{
Joe Perchesd2f31a52011-06-13 20:19:27 +0200170 return td->nr_queued[0] + td->nr_queued[1];
Vivek Goyale43473b2010-09-15 17:06:35 -0400171}
172
Tejun Heo8a3d2612012-04-01 14:38:44 -0700173/*
174 * Worker for allocating per cpu stat for tgs. This is scheduled on the
175 * system_nrt_wq once there are some groups on the alloc_list waiting for
176 * allocation.
177 */
178static void tg_stats_alloc_fn(struct work_struct *work)
179{
180 static struct tg_stats_cpu *stats_cpu; /* this fn is non-reentrant */
181 struct delayed_work *dwork = to_delayed_work(work);
182 bool empty = false;
183
184alloc_stats:
185 if (!stats_cpu) {
186 stats_cpu = alloc_percpu(struct tg_stats_cpu);
187 if (!stats_cpu) {
188 /* allocation failed, try again after some time */
189 queue_delayed_work(system_nrt_wq, dwork,
190 msecs_to_jiffies(10));
191 return;
192 }
193 }
194
195 spin_lock_irq(&tg_stats_alloc_lock);
196
197 if (!list_empty(&tg_stats_alloc_list)) {
198 struct throtl_grp *tg = list_first_entry(&tg_stats_alloc_list,
199 struct throtl_grp,
200 stats_alloc_node);
201 swap(tg->stats_cpu, stats_cpu);
202 list_del_init(&tg->stats_alloc_node);
203 }
204
205 empty = list_empty(&tg_stats_alloc_list);
206 spin_unlock_irq(&tg_stats_alloc_lock);
207 if (!empty)
208 goto alloc_stats;
209}
210
Tejun Heo03814112012-03-05 13:15:14 -0800211static void throtl_init_blkio_group(struct blkio_group *blkg)
Vivek Goyala29a1712011-05-19 15:38:19 -0400212{
Tejun Heo03814112012-03-05 13:15:14 -0800213 struct throtl_grp *tg = blkg_to_tg(blkg);
Tejun Heocd1604f2012-03-05 13:15:06 -0800214
Vivek Goyala29a1712011-05-19 15:38:19 -0400215 RB_CLEAR_NODE(&tg->rb_node);
216 bio_list_init(&tg->bio_lists[0]);
217 bio_list_init(&tg->bio_lists[1]);
218 tg->limits_changed = false;
219
Tejun Heoe56da7e2012-03-05 13:15:07 -0800220 tg->bps[READ] = -1;
221 tg->bps[WRITE] = -1;
222 tg->iops[READ] = -1;
223 tg->iops[WRITE] = -1;
Tejun Heo8a3d2612012-04-01 14:38:44 -0700224
225 /*
226 * Ugh... We need to perform per-cpu allocation for tg->stats_cpu
227 * but percpu allocator can't be called from IO path. Queue tg on
228 * tg_stats_alloc_list and allocate from work item.
229 */
230 spin_lock(&tg_stats_alloc_lock);
231 list_add(&tg->stats_alloc_node, &tg_stats_alloc_list);
232 queue_delayed_work(system_nrt_wq, &tg_stats_alloc_work, 0);
233 spin_unlock(&tg_stats_alloc_lock);
234}
235
236static void throtl_exit_blkio_group(struct blkio_group *blkg)
237{
238 struct throtl_grp *tg = blkg_to_tg(blkg);
239
240 spin_lock(&tg_stats_alloc_lock);
241 list_del_init(&tg->stats_alloc_node);
242 spin_unlock(&tg_stats_alloc_lock);
243
244 free_percpu(tg->stats_cpu);
245}
246
247static void throtl_reset_group_stats(struct blkio_group *blkg)
248{
249 struct throtl_grp *tg = blkg_to_tg(blkg);
250 int cpu;
251
252 if (tg->stats_cpu == NULL)
253 return;
254
255 for_each_possible_cpu(cpu) {
256 struct tg_stats_cpu *sc = per_cpu_ptr(tg->stats_cpu, cpu);
257
258 blkg_rwstat_reset(&sc->service_bytes);
259 blkg_rwstat_reset(&sc->serviced);
260 }
Vivek Goyala29a1712011-05-19 15:38:19 -0400261}
262
Vivek Goyalf469a7b2011-05-19 15:38:23 -0400263static struct
Tejun Heocd1604f2012-03-05 13:15:06 -0800264throtl_grp *throtl_lookup_tg(struct throtl_data *td, struct blkio_cgroup *blkcg)
Vivek Goyale43473b2010-09-15 17:06:35 -0400265{
Vivek Goyale43473b2010-09-15 17:06:35 -0400266 /*
Vivek Goyalbe2c6b12011-01-19 08:25:02 -0700267 * This is the common case when there are no blkio cgroups.
Tejun Heocd1604f2012-03-05 13:15:06 -0800268 * Avoid lookup in this case
269 */
Vivek Goyalbe2c6b12011-01-19 08:25:02 -0700270 if (blkcg == &blkio_root_cgroup)
Tejun Heo03d8e112012-04-13 13:11:32 -0700271 return td_root_tg(td);
Vivek Goyale43473b2010-09-15 17:06:35 -0400272
Tejun Heoe8989fa2012-03-05 13:15:20 -0800273 return blkg_to_tg(blkg_lookup(blkcg, td->queue));
Vivek Goyale43473b2010-09-15 17:06:35 -0400274}
275
Tejun Heocd1604f2012-03-05 13:15:06 -0800276static struct throtl_grp *throtl_lookup_create_tg(struct throtl_data *td,
277 struct blkio_cgroup *blkcg)
Vivek Goyale43473b2010-09-15 17:06:35 -0400278{
Vivek Goyalf469a7b2011-05-19 15:38:23 -0400279 struct request_queue *q = td->queue;
Tejun Heocd1604f2012-03-05 13:15:06 -0800280 struct throtl_grp *tg = NULL;
Tejun Heo0a5a7d02012-03-05 13:15:02 -0800281
Vivek Goyalf469a7b2011-05-19 15:38:23 -0400282 /*
Tejun Heocd1604f2012-03-05 13:15:06 -0800283 * This is the common case when there are no blkio cgroups.
284 * Avoid lookup in this case
Vivek Goyalf469a7b2011-05-19 15:38:23 -0400285 */
Tejun Heocd1604f2012-03-05 13:15:06 -0800286 if (blkcg == &blkio_root_cgroup) {
Tejun Heo03d8e112012-04-13 13:11:32 -0700287 tg = td_root_tg(td);
Tejun Heocd1604f2012-03-05 13:15:06 -0800288 } else {
289 struct blkio_group *blkg;
290
Tejun Heo3c96cb32012-04-13 13:11:34 -0700291 blkg = blkg_lookup_create(blkcg, q);
Tejun Heocd1604f2012-03-05 13:15:06 -0800292
293 /* if %NULL and @q is alive, fall back to root_tg */
294 if (!IS_ERR(blkg))
Tejun Heo03814112012-03-05 13:15:14 -0800295 tg = blkg_to_tg(blkg);
Tejun Heocd1604f2012-03-05 13:15:06 -0800296 else if (!blk_queue_dead(q))
Tejun Heo03d8e112012-04-13 13:11:32 -0700297 tg = td_root_tg(td);
Vivek Goyalf469a7b2011-05-19 15:38:23 -0400298 }
299
Vivek Goyale43473b2010-09-15 17:06:35 -0400300 return tg;
301}
302
303static struct throtl_grp *throtl_rb_first(struct throtl_rb_root *root)
304{
305 /* Service tree is empty */
306 if (!root->count)
307 return NULL;
308
309 if (!root->left)
310 root->left = rb_first(&root->rb);
311
312 if (root->left)
313 return rb_entry_tg(root->left);
314
315 return NULL;
316}
317
318static void rb_erase_init(struct rb_node *n, struct rb_root *root)
319{
320 rb_erase(n, root);
321 RB_CLEAR_NODE(n);
322}
323
324static void throtl_rb_erase(struct rb_node *n, struct throtl_rb_root *root)
325{
326 if (root->left == n)
327 root->left = NULL;
328 rb_erase_init(n, &root->rb);
329 --root->count;
330}
331
332static void update_min_dispatch_time(struct throtl_rb_root *st)
333{
334 struct throtl_grp *tg;
335
336 tg = throtl_rb_first(st);
337 if (!tg)
338 return;
339
340 st->min_disptime = tg->disptime;
341}
342
343static void
344tg_service_tree_add(struct throtl_rb_root *st, struct throtl_grp *tg)
345{
346 struct rb_node **node = &st->rb.rb_node;
347 struct rb_node *parent = NULL;
348 struct throtl_grp *__tg;
349 unsigned long key = tg->disptime;
350 int left = 1;
351
352 while (*node != NULL) {
353 parent = *node;
354 __tg = rb_entry_tg(parent);
355
356 if (time_before(key, __tg->disptime))
357 node = &parent->rb_left;
358 else {
359 node = &parent->rb_right;
360 left = 0;
361 }
362 }
363
364 if (left)
365 st->left = &tg->rb_node;
366
367 rb_link_node(&tg->rb_node, parent, node);
368 rb_insert_color(&tg->rb_node, &st->rb);
369}
370
371static void __throtl_enqueue_tg(struct throtl_data *td, struct throtl_grp *tg)
372{
373 struct throtl_rb_root *st = &td->tg_service_tree;
374
375 tg_service_tree_add(st, tg);
376 throtl_mark_tg_on_rr(tg);
377 st->count++;
378}
379
380static void throtl_enqueue_tg(struct throtl_data *td, struct throtl_grp *tg)
381{
382 if (!throtl_tg_on_rr(tg))
383 __throtl_enqueue_tg(td, tg);
384}
385
386static void __throtl_dequeue_tg(struct throtl_data *td, struct throtl_grp *tg)
387{
388 throtl_rb_erase(&tg->rb_node, &td->tg_service_tree);
389 throtl_clear_tg_on_rr(tg);
390}
391
392static void throtl_dequeue_tg(struct throtl_data *td, struct throtl_grp *tg)
393{
394 if (throtl_tg_on_rr(tg))
395 __throtl_dequeue_tg(td, tg);
396}
397
398static void throtl_schedule_next_dispatch(struct throtl_data *td)
399{
400 struct throtl_rb_root *st = &td->tg_service_tree;
401
402 /*
403 * If there are more bios pending, schedule more work.
404 */
405 if (!total_nr_queued(td))
406 return;
407
408 BUG_ON(!st->count);
409
410 update_min_dispatch_time(st);
411
412 if (time_before_eq(st->min_disptime, jiffies))
Vivek Goyal450adcb2011-03-01 13:40:54 -0500413 throtl_schedule_delayed_work(td, 0);
Vivek Goyale43473b2010-09-15 17:06:35 -0400414 else
Vivek Goyal450adcb2011-03-01 13:40:54 -0500415 throtl_schedule_delayed_work(td, (st->min_disptime - jiffies));
Vivek Goyale43473b2010-09-15 17:06:35 -0400416}
417
418static inline void
419throtl_start_new_slice(struct throtl_data *td, struct throtl_grp *tg, bool rw)
420{
421 tg->bytes_disp[rw] = 0;
Vivek Goyal8e89d132010-09-15 17:06:37 -0400422 tg->io_disp[rw] = 0;
Vivek Goyale43473b2010-09-15 17:06:35 -0400423 tg->slice_start[rw] = jiffies;
424 tg->slice_end[rw] = jiffies + throtl_slice;
425 throtl_log_tg(td, tg, "[%c] new slice start=%lu end=%lu jiffies=%lu",
426 rw == READ ? 'R' : 'W', tg->slice_start[rw],
427 tg->slice_end[rw], jiffies);
428}
429
Vivek Goyald1ae8ff2010-12-01 19:34:46 +0100430static inline void throtl_set_slice_end(struct throtl_data *td,
431 struct throtl_grp *tg, bool rw, unsigned long jiffy_end)
432{
433 tg->slice_end[rw] = roundup(jiffy_end, throtl_slice);
434}
435
Vivek Goyale43473b2010-09-15 17:06:35 -0400436static inline void throtl_extend_slice(struct throtl_data *td,
437 struct throtl_grp *tg, bool rw, unsigned long jiffy_end)
438{
439 tg->slice_end[rw] = roundup(jiffy_end, throtl_slice);
440 throtl_log_tg(td, tg, "[%c] extend slice start=%lu end=%lu jiffies=%lu",
441 rw == READ ? 'R' : 'W', tg->slice_start[rw],
442 tg->slice_end[rw], jiffies);
443}
444
445/* Determine if previously allocated or extended slice is complete or not */
446static bool
447throtl_slice_used(struct throtl_data *td, struct throtl_grp *tg, bool rw)
448{
449 if (time_in_range(jiffies, tg->slice_start[rw], tg->slice_end[rw]))
450 return 0;
451
452 return 1;
453}
454
455/* Trim the used slices and adjust slice start accordingly */
456static inline void
457throtl_trim_slice(struct throtl_data *td, struct throtl_grp *tg, bool rw)
458{
Vivek Goyal3aad5d32010-10-01 14:51:14 +0200459 unsigned long nr_slices, time_elapsed, io_trim;
460 u64 bytes_trim, tmp;
Vivek Goyale43473b2010-09-15 17:06:35 -0400461
462 BUG_ON(time_before(tg->slice_end[rw], tg->slice_start[rw]));
463
464 /*
465 * If bps are unlimited (-1), then time slice don't get
466 * renewed. Don't try to trim the slice if slice is used. A new
467 * slice will start when appropriate.
468 */
469 if (throtl_slice_used(td, tg, rw))
470 return;
471
Vivek Goyald1ae8ff2010-12-01 19:34:46 +0100472 /*
473 * A bio has been dispatched. Also adjust slice_end. It might happen
474 * that initially cgroup limit was very low resulting in high
475 * slice_end, but later limit was bumped up and bio was dispached
476 * sooner, then we need to reduce slice_end. A high bogus slice_end
477 * is bad because it does not allow new slice to start.
478 */
479
480 throtl_set_slice_end(td, tg, rw, jiffies + throtl_slice);
481
Vivek Goyale43473b2010-09-15 17:06:35 -0400482 time_elapsed = jiffies - tg->slice_start[rw];
483
484 nr_slices = time_elapsed / throtl_slice;
485
486 if (!nr_slices)
487 return;
Vivek Goyal3aad5d32010-10-01 14:51:14 +0200488 tmp = tg->bps[rw] * throtl_slice * nr_slices;
489 do_div(tmp, HZ);
490 bytes_trim = tmp;
Vivek Goyale43473b2010-09-15 17:06:35 -0400491
Vivek Goyal8e89d132010-09-15 17:06:37 -0400492 io_trim = (tg->iops[rw] * throtl_slice * nr_slices)/HZ;
Vivek Goyale43473b2010-09-15 17:06:35 -0400493
Vivek Goyal8e89d132010-09-15 17:06:37 -0400494 if (!bytes_trim && !io_trim)
Vivek Goyale43473b2010-09-15 17:06:35 -0400495 return;
496
497 if (tg->bytes_disp[rw] >= bytes_trim)
498 tg->bytes_disp[rw] -= bytes_trim;
499 else
500 tg->bytes_disp[rw] = 0;
501
Vivek Goyal8e89d132010-09-15 17:06:37 -0400502 if (tg->io_disp[rw] >= io_trim)
503 tg->io_disp[rw] -= io_trim;
504 else
505 tg->io_disp[rw] = 0;
506
Vivek Goyale43473b2010-09-15 17:06:35 -0400507 tg->slice_start[rw] += nr_slices * throtl_slice;
508
Vivek Goyal3aad5d32010-10-01 14:51:14 +0200509 throtl_log_tg(td, tg, "[%c] trim slice nr=%lu bytes=%llu io=%lu"
Vivek Goyale43473b2010-09-15 17:06:35 -0400510 " start=%lu end=%lu jiffies=%lu",
Vivek Goyal8e89d132010-09-15 17:06:37 -0400511 rw == READ ? 'R' : 'W', nr_slices, bytes_trim, io_trim,
Vivek Goyale43473b2010-09-15 17:06:35 -0400512 tg->slice_start[rw], tg->slice_end[rw], jiffies);
513}
514
Vivek Goyal8e89d132010-09-15 17:06:37 -0400515static bool tg_with_in_iops_limit(struct throtl_data *td, struct throtl_grp *tg,
516 struct bio *bio, unsigned long *wait)
Vivek Goyale43473b2010-09-15 17:06:35 -0400517{
518 bool rw = bio_data_dir(bio);
Vivek Goyal8e89d132010-09-15 17:06:37 -0400519 unsigned int io_allowed;
Vivek Goyale43473b2010-09-15 17:06:35 -0400520 unsigned long jiffy_elapsed, jiffy_wait, jiffy_elapsed_rnd;
Vivek Goyalc49c06e2010-10-01 21:16:42 +0200521 u64 tmp;
Vivek Goyale43473b2010-09-15 17:06:35 -0400522
Vivek Goyal8e89d132010-09-15 17:06:37 -0400523 jiffy_elapsed = jiffy_elapsed_rnd = jiffies - tg->slice_start[rw];
Vivek Goyale43473b2010-09-15 17:06:35 -0400524
Vivek Goyal8e89d132010-09-15 17:06:37 -0400525 /* Slice has just started. Consider one slice interval */
526 if (!jiffy_elapsed)
527 jiffy_elapsed_rnd = throtl_slice;
528
529 jiffy_elapsed_rnd = roundup(jiffy_elapsed_rnd, throtl_slice);
530
Vivek Goyalc49c06e2010-10-01 21:16:42 +0200531 /*
532 * jiffy_elapsed_rnd should not be a big value as minimum iops can be
533 * 1 then at max jiffy elapsed should be equivalent of 1 second as we
534 * will allow dispatch after 1 second and after that slice should
535 * have been trimmed.
536 */
537
538 tmp = (u64)tg->iops[rw] * jiffy_elapsed_rnd;
539 do_div(tmp, HZ);
540
541 if (tmp > UINT_MAX)
542 io_allowed = UINT_MAX;
543 else
544 io_allowed = tmp;
Vivek Goyal8e89d132010-09-15 17:06:37 -0400545
546 if (tg->io_disp[rw] + 1 <= io_allowed) {
Vivek Goyale43473b2010-09-15 17:06:35 -0400547 if (wait)
548 *wait = 0;
549 return 1;
550 }
551
Vivek Goyal8e89d132010-09-15 17:06:37 -0400552 /* Calc approx time to dispatch */
553 jiffy_wait = ((tg->io_disp[rw] + 1) * HZ)/tg->iops[rw] + 1;
554
555 if (jiffy_wait > jiffy_elapsed)
556 jiffy_wait = jiffy_wait - jiffy_elapsed;
557 else
558 jiffy_wait = 1;
559
560 if (wait)
561 *wait = jiffy_wait;
562 return 0;
563}
564
565static bool tg_with_in_bps_limit(struct throtl_data *td, struct throtl_grp *tg,
566 struct bio *bio, unsigned long *wait)
567{
568 bool rw = bio_data_dir(bio);
Vivek Goyal3aad5d32010-10-01 14:51:14 +0200569 u64 bytes_allowed, extra_bytes, tmp;
Vivek Goyal8e89d132010-09-15 17:06:37 -0400570 unsigned long jiffy_elapsed, jiffy_wait, jiffy_elapsed_rnd;
Vivek Goyale43473b2010-09-15 17:06:35 -0400571
572 jiffy_elapsed = jiffy_elapsed_rnd = jiffies - tg->slice_start[rw];
573
574 /* Slice has just started. Consider one slice interval */
575 if (!jiffy_elapsed)
576 jiffy_elapsed_rnd = throtl_slice;
577
578 jiffy_elapsed_rnd = roundup(jiffy_elapsed_rnd, throtl_slice);
579
Vivek Goyal5e901a22010-10-01 21:16:38 +0200580 tmp = tg->bps[rw] * jiffy_elapsed_rnd;
581 do_div(tmp, HZ);
Vivek Goyal3aad5d32010-10-01 14:51:14 +0200582 bytes_allowed = tmp;
Vivek Goyale43473b2010-09-15 17:06:35 -0400583
584 if (tg->bytes_disp[rw] + bio->bi_size <= bytes_allowed) {
585 if (wait)
586 *wait = 0;
587 return 1;
588 }
589
590 /* Calc approx time to dispatch */
591 extra_bytes = tg->bytes_disp[rw] + bio->bi_size - bytes_allowed;
592 jiffy_wait = div64_u64(extra_bytes * HZ, tg->bps[rw]);
593
594 if (!jiffy_wait)
595 jiffy_wait = 1;
596
597 /*
598 * This wait time is without taking into consideration the rounding
599 * up we did. Add that time also.
600 */
601 jiffy_wait = jiffy_wait + (jiffy_elapsed_rnd - jiffy_elapsed);
Vivek Goyale43473b2010-09-15 17:06:35 -0400602 if (wait)
603 *wait = jiffy_wait;
Vivek Goyal8e89d132010-09-15 17:06:37 -0400604 return 0;
605}
Vivek Goyale43473b2010-09-15 17:06:35 -0400606
Vivek Goyalaf75cd32011-05-19 15:38:31 -0400607static bool tg_no_rule_group(struct throtl_grp *tg, bool rw) {
608 if (tg->bps[rw] == -1 && tg->iops[rw] == -1)
609 return 1;
610 return 0;
611}
612
Vivek Goyal8e89d132010-09-15 17:06:37 -0400613/*
614 * Returns whether one can dispatch a bio or not. Also returns approx number
615 * of jiffies to wait before this bio is with-in IO rate and can be dispatched
616 */
617static bool tg_may_dispatch(struct throtl_data *td, struct throtl_grp *tg,
618 struct bio *bio, unsigned long *wait)
619{
620 bool rw = bio_data_dir(bio);
621 unsigned long bps_wait = 0, iops_wait = 0, max_wait = 0;
622
623 /*
624 * Currently whole state machine of group depends on first bio
625 * queued in the group bio list. So one should not be calling
626 * this function with a different bio if there are other bios
627 * queued.
628 */
629 BUG_ON(tg->nr_queued[rw] && bio != bio_list_peek(&tg->bio_lists[rw]));
630
631 /* If tg->bps = -1, then BW is unlimited */
632 if (tg->bps[rw] == -1 && tg->iops[rw] == -1) {
633 if (wait)
634 *wait = 0;
635 return 1;
636 }
637
638 /*
639 * If previous slice expired, start a new one otherwise renew/extend
640 * existing slice to make sure it is at least throtl_slice interval
641 * long since now.
642 */
643 if (throtl_slice_used(td, tg, rw))
644 throtl_start_new_slice(td, tg, rw);
645 else {
646 if (time_before(tg->slice_end[rw], jiffies + throtl_slice))
647 throtl_extend_slice(td, tg, rw, jiffies + throtl_slice);
648 }
649
650 if (tg_with_in_bps_limit(td, tg, bio, &bps_wait)
651 && tg_with_in_iops_limit(td, tg, bio, &iops_wait)) {
652 if (wait)
653 *wait = 0;
654 return 1;
655 }
656
657 max_wait = max(bps_wait, iops_wait);
658
659 if (wait)
660 *wait = max_wait;
661
662 if (time_before(tg->slice_end[rw], jiffies + max_wait))
663 throtl_extend_slice(td, tg, rw, jiffies + max_wait);
Vivek Goyale43473b2010-09-15 17:06:35 -0400664
665 return 0;
666}
667
Tejun Heo629ed0b2012-04-01 14:38:44 -0700668static void throtl_update_dispatch_stats(struct blkio_group *blkg, u64 bytes,
669 int rw)
670{
Tejun Heo8a3d2612012-04-01 14:38:44 -0700671 struct throtl_grp *tg = blkg_to_tg(blkg);
672 struct tg_stats_cpu *stats_cpu;
Tejun Heo629ed0b2012-04-01 14:38:44 -0700673 unsigned long flags;
674
675 /* If per cpu stats are not allocated yet, don't do any accounting. */
Tejun Heo8a3d2612012-04-01 14:38:44 -0700676 if (tg->stats_cpu == NULL)
Tejun Heo629ed0b2012-04-01 14:38:44 -0700677 return;
678
679 /*
680 * Disabling interrupts to provide mutual exclusion between two
681 * writes on same cpu. It probably is not needed for 64bit. Not
682 * optimizing that case yet.
683 */
684 local_irq_save(flags);
685
Tejun Heo8a3d2612012-04-01 14:38:44 -0700686 stats_cpu = this_cpu_ptr(tg->stats_cpu);
Tejun Heo629ed0b2012-04-01 14:38:44 -0700687
Tejun Heo629ed0b2012-04-01 14:38:44 -0700688 blkg_rwstat_add(&stats_cpu->serviced, rw, 1);
689 blkg_rwstat_add(&stats_cpu->service_bytes, rw, bytes);
690
691 local_irq_restore(flags);
692}
693
Vivek Goyale43473b2010-09-15 17:06:35 -0400694static void throtl_charge_bio(struct throtl_grp *tg, struct bio *bio)
695{
696 bool rw = bio_data_dir(bio);
Vivek Goyale43473b2010-09-15 17:06:35 -0400697
698 /* Charge the bio to the group */
699 tg->bytes_disp[rw] += bio->bi_size;
Vivek Goyal8e89d132010-09-15 17:06:37 -0400700 tg->io_disp[rw]++;
Vivek Goyale43473b2010-09-15 17:06:35 -0400701
Tejun Heo629ed0b2012-04-01 14:38:44 -0700702 throtl_update_dispatch_stats(tg_to_blkg(tg), bio->bi_size, bio->bi_rw);
Vivek Goyale43473b2010-09-15 17:06:35 -0400703}
704
705static void throtl_add_bio_tg(struct throtl_data *td, struct throtl_grp *tg,
706 struct bio *bio)
707{
708 bool rw = bio_data_dir(bio);
709
710 bio_list_add(&tg->bio_lists[rw], bio);
711 /* Take a bio reference on tg */
Tejun Heo1adaf3d2012-03-05 13:15:15 -0800712 blkg_get(tg_to_blkg(tg));
Vivek Goyale43473b2010-09-15 17:06:35 -0400713 tg->nr_queued[rw]++;
714 td->nr_queued[rw]++;
715 throtl_enqueue_tg(td, tg);
716}
717
718static void tg_update_disptime(struct throtl_data *td, struct throtl_grp *tg)
719{
720 unsigned long read_wait = -1, write_wait = -1, min_wait = -1, disptime;
721 struct bio *bio;
722
723 if ((bio = bio_list_peek(&tg->bio_lists[READ])))
724 tg_may_dispatch(td, tg, bio, &read_wait);
725
726 if ((bio = bio_list_peek(&tg->bio_lists[WRITE])))
727 tg_may_dispatch(td, tg, bio, &write_wait);
728
729 min_wait = min(read_wait, write_wait);
730 disptime = jiffies + min_wait;
731
Vivek Goyale43473b2010-09-15 17:06:35 -0400732 /* Update dispatch time */
733 throtl_dequeue_tg(td, tg);
734 tg->disptime = disptime;
735 throtl_enqueue_tg(td, tg);
736}
737
738static void tg_dispatch_one_bio(struct throtl_data *td, struct throtl_grp *tg,
739 bool rw, struct bio_list *bl)
740{
741 struct bio *bio;
742
743 bio = bio_list_pop(&tg->bio_lists[rw]);
744 tg->nr_queued[rw]--;
Tejun Heo1adaf3d2012-03-05 13:15:15 -0800745 /* Drop bio reference on blkg */
746 blkg_put(tg_to_blkg(tg));
Vivek Goyale43473b2010-09-15 17:06:35 -0400747
748 BUG_ON(td->nr_queued[rw] <= 0);
749 td->nr_queued[rw]--;
750
751 throtl_charge_bio(tg, bio);
752 bio_list_add(bl, bio);
753 bio->bi_rw |= REQ_THROTTLED;
754
755 throtl_trim_slice(td, tg, rw);
756}
757
758static int throtl_dispatch_tg(struct throtl_data *td, struct throtl_grp *tg,
759 struct bio_list *bl)
760{
761 unsigned int nr_reads = 0, nr_writes = 0;
762 unsigned int max_nr_reads = throtl_grp_quantum*3/4;
Vivek Goyalc2f68052010-11-15 19:32:42 +0100763 unsigned int max_nr_writes = throtl_grp_quantum - max_nr_reads;
Vivek Goyale43473b2010-09-15 17:06:35 -0400764 struct bio *bio;
765
766 /* Try to dispatch 75% READS and 25% WRITES */
767
768 while ((bio = bio_list_peek(&tg->bio_lists[READ]))
769 && tg_may_dispatch(td, tg, bio, NULL)) {
770
771 tg_dispatch_one_bio(td, tg, bio_data_dir(bio), bl);
772 nr_reads++;
773
774 if (nr_reads >= max_nr_reads)
775 break;
776 }
777
778 while ((bio = bio_list_peek(&tg->bio_lists[WRITE]))
779 && tg_may_dispatch(td, tg, bio, NULL)) {
780
781 tg_dispatch_one_bio(td, tg, bio_data_dir(bio), bl);
782 nr_writes++;
783
784 if (nr_writes >= max_nr_writes)
785 break;
786 }
787
788 return nr_reads + nr_writes;
789}
790
791static int throtl_select_dispatch(struct throtl_data *td, struct bio_list *bl)
792{
793 unsigned int nr_disp = 0;
794 struct throtl_grp *tg;
795 struct throtl_rb_root *st = &td->tg_service_tree;
796
797 while (1) {
798 tg = throtl_rb_first(st);
799
800 if (!tg)
801 break;
802
803 if (time_before(jiffies, tg->disptime))
804 break;
805
806 throtl_dequeue_tg(td, tg);
807
808 nr_disp += throtl_dispatch_tg(td, tg, bl);
809
810 if (tg->nr_queued[0] || tg->nr_queued[1]) {
811 tg_update_disptime(td, tg);
812 throtl_enqueue_tg(td, tg);
813 }
814
815 if (nr_disp >= throtl_quantum)
816 break;
817 }
818
819 return nr_disp;
820}
821
Vivek Goyalfe071432010-10-01 14:49:49 +0200822static void throtl_process_limit_change(struct throtl_data *td)
823{
Tejun Heo4eef3042012-03-05 13:15:18 -0800824 struct request_queue *q = td->queue;
825 struct blkio_group *blkg, *n;
Vivek Goyalfe071432010-10-01 14:49:49 +0200826
Vivek Goyalde701c72011-03-07 21:09:32 +0100827 if (!td->limits_changed)
Vivek Goyalfe071432010-10-01 14:49:49 +0200828 return;
829
Vivek Goyalde701c72011-03-07 21:09:32 +0100830 xchg(&td->limits_changed, false);
Vivek Goyalfe071432010-10-01 14:49:49 +0200831
Vivek Goyalde701c72011-03-07 21:09:32 +0100832 throtl_log(td, "limits changed");
Vivek Goyalfe071432010-10-01 14:49:49 +0200833
Tejun Heoe8989fa2012-03-05 13:15:20 -0800834 list_for_each_entry_safe(blkg, n, &q->blkg_list, q_node) {
Tejun Heo4eef3042012-03-05 13:15:18 -0800835 struct throtl_grp *tg = blkg_to_tg(blkg);
836
Vivek Goyalde701c72011-03-07 21:09:32 +0100837 if (!tg->limits_changed)
838 continue;
Vivek Goyalfe071432010-10-01 14:49:49 +0200839
Vivek Goyalde701c72011-03-07 21:09:32 +0100840 if (!xchg(&tg->limits_changed, false))
841 continue;
842
843 throtl_log_tg(td, tg, "limit change rbps=%llu wbps=%llu"
844 " riops=%u wiops=%u", tg->bps[READ], tg->bps[WRITE],
845 tg->iops[READ], tg->iops[WRITE]);
846
Vivek Goyal04521db2011-03-22 21:54:29 +0100847 /*
848 * Restart the slices for both READ and WRITES. It
849 * might happen that a group's limit are dropped
850 * suddenly and we don't want to account recently
851 * dispatched IO with new low rate
852 */
853 throtl_start_new_slice(td, tg, 0);
854 throtl_start_new_slice(td, tg, 1);
855
Vivek Goyalde701c72011-03-07 21:09:32 +0100856 if (throtl_tg_on_rr(tg))
857 tg_update_disptime(td, tg);
858 }
Vivek Goyalfe071432010-10-01 14:49:49 +0200859}
860
Vivek Goyale43473b2010-09-15 17:06:35 -0400861/* Dispatch throttled bios. Should be called without queue lock held. */
862static int throtl_dispatch(struct request_queue *q)
863{
864 struct throtl_data *td = q->td;
865 unsigned int nr_disp = 0;
866 struct bio_list bio_list_on_stack;
867 struct bio *bio;
Vivek Goyal69d60eb2011-03-09 08:27:37 +0100868 struct blk_plug plug;
Vivek Goyale43473b2010-09-15 17:06:35 -0400869
870 spin_lock_irq(q->queue_lock);
871
Vivek Goyalfe071432010-10-01 14:49:49 +0200872 throtl_process_limit_change(td);
873
Vivek Goyale43473b2010-09-15 17:06:35 -0400874 if (!total_nr_queued(td))
875 goto out;
876
877 bio_list_init(&bio_list_on_stack);
878
Joe Perchesd2f31a52011-06-13 20:19:27 +0200879 throtl_log(td, "dispatch nr_queued=%u read=%u write=%u",
Vivek Goyale43473b2010-09-15 17:06:35 -0400880 total_nr_queued(td), td->nr_queued[READ],
881 td->nr_queued[WRITE]);
882
883 nr_disp = throtl_select_dispatch(td, &bio_list_on_stack);
884
885 if (nr_disp)
886 throtl_log(td, "bios disp=%u", nr_disp);
887
888 throtl_schedule_next_dispatch(td);
889out:
890 spin_unlock_irq(q->queue_lock);
891
892 /*
893 * If we dispatched some requests, unplug the queue to make sure
894 * immediate dispatch
895 */
896 if (nr_disp) {
Vivek Goyal69d60eb2011-03-09 08:27:37 +0100897 blk_start_plug(&plug);
Vivek Goyale43473b2010-09-15 17:06:35 -0400898 while((bio = bio_list_pop(&bio_list_on_stack)))
899 generic_make_request(bio);
Vivek Goyal69d60eb2011-03-09 08:27:37 +0100900 blk_finish_plug(&plug);
Vivek Goyale43473b2010-09-15 17:06:35 -0400901 }
902 return nr_disp;
903}
904
905void blk_throtl_work(struct work_struct *work)
906{
907 struct throtl_data *td = container_of(work, struct throtl_data,
908 throtl_work.work);
909 struct request_queue *q = td->queue;
910
911 throtl_dispatch(q);
912}
913
914/* Call with queue lock held */
Vivek Goyal450adcb2011-03-01 13:40:54 -0500915static void
916throtl_schedule_delayed_work(struct throtl_data *td, unsigned long delay)
Vivek Goyale43473b2010-09-15 17:06:35 -0400917{
918
Vivek Goyale43473b2010-09-15 17:06:35 -0400919 struct delayed_work *dwork = &td->throtl_work;
920
Vivek Goyal04521db2011-03-22 21:54:29 +0100921 /* schedule work if limits changed even if no bio is queued */
Joe Perchesd2f31a52011-06-13 20:19:27 +0200922 if (total_nr_queued(td) || td->limits_changed) {
Vivek Goyale43473b2010-09-15 17:06:35 -0400923 /*
924 * We might have a work scheduled to be executed in future.
925 * Cancel that and schedule a new one.
926 */
927 __cancel_delayed_work(dwork);
Vivek Goyal450adcb2011-03-01 13:40:54 -0500928 queue_delayed_work(kthrotld_workqueue, dwork, delay);
Vivek Goyale43473b2010-09-15 17:06:35 -0400929 throtl_log(td, "schedule work. delay=%lu jiffies=%lu",
930 delay, jiffies);
931 }
932}
Vivek Goyale43473b2010-09-15 17:06:35 -0400933
Tejun Heod366e7e2012-04-01 14:38:44 -0700934static u64 tg_prfill_cpu_rwstat(struct seq_file *sf, void *pdata, int off)
Tejun Heo41b38b62012-04-01 14:38:44 -0700935{
Tejun Heod366e7e2012-04-01 14:38:44 -0700936 struct throtl_grp *tg = pdata;
Tejun Heo41b38b62012-04-01 14:38:44 -0700937 struct blkg_rwstat rwstat = { }, tmp;
938 int i, cpu;
939
940 for_each_possible_cpu(cpu) {
Tejun Heo8a3d2612012-04-01 14:38:44 -0700941 struct tg_stats_cpu *sc = per_cpu_ptr(tg->stats_cpu, cpu);
Tejun Heo41b38b62012-04-01 14:38:44 -0700942
943 tmp = blkg_rwstat_read((void *)sc + off);
944 for (i = 0; i < BLKG_RWSTAT_NR; i++)
945 rwstat.cnt[i] += tmp.cnt[i];
946 }
947
Tejun Heod366e7e2012-04-01 14:38:44 -0700948 return __blkg_prfill_rwstat(sf, pdata, &rwstat);
Tejun Heo41b38b62012-04-01 14:38:44 -0700949}
950
Tejun Heo8a3d2612012-04-01 14:38:44 -0700951static int tg_print_cpu_rwstat(struct cgroup *cgrp, struct cftype *cft,
952 struct seq_file *sf)
Tejun Heo41b38b62012-04-01 14:38:44 -0700953{
954 struct blkio_cgroup *blkcg = cgroup_to_blkio_cgroup(cgrp);
955
Tejun Heoec399342012-04-13 13:11:27 -0700956 blkcg_print_blkgs(sf, blkcg, tg_prfill_cpu_rwstat, &blkio_policy_throtl,
Tejun Heo5bc4afb12012-04-01 14:38:45 -0700957 cft->private, true);
Tejun Heo41b38b62012-04-01 14:38:44 -0700958 return 0;
959}
960
Tejun Heod366e7e2012-04-01 14:38:44 -0700961static u64 tg_prfill_conf_u64(struct seq_file *sf, void *pdata, int off)
Tejun Heo60c2bc22012-04-01 14:38:43 -0700962{
Tejun Heod366e7e2012-04-01 14:38:44 -0700963 u64 v = *(u64 *)(pdata + off);
Tejun Heo60c2bc22012-04-01 14:38:43 -0700964
Tejun Heoaf133ce2012-04-01 14:38:44 -0700965 if (v == -1)
Tejun Heo60c2bc22012-04-01 14:38:43 -0700966 return 0;
Tejun Heod366e7e2012-04-01 14:38:44 -0700967 return __blkg_prfill_u64(sf, pdata, v);
Tejun Heo60c2bc22012-04-01 14:38:43 -0700968}
969
Tejun Heod366e7e2012-04-01 14:38:44 -0700970static u64 tg_prfill_conf_uint(struct seq_file *sf, void *pdata, int off)
Tejun Heo60c2bc22012-04-01 14:38:43 -0700971{
Tejun Heod366e7e2012-04-01 14:38:44 -0700972 unsigned int v = *(unsigned int *)(pdata + off);
Tejun Heoaf133ce2012-04-01 14:38:44 -0700973
974 if (v == -1)
975 return 0;
Tejun Heod366e7e2012-04-01 14:38:44 -0700976 return __blkg_prfill_u64(sf, pdata, v);
Tejun Heoaf133ce2012-04-01 14:38:44 -0700977}
978
979static int tg_print_conf_u64(struct cgroup *cgrp, struct cftype *cft,
980 struct seq_file *sf)
981{
982 blkcg_print_blkgs(sf, cgroup_to_blkio_cgroup(cgrp), tg_prfill_conf_u64,
Tejun Heoec399342012-04-13 13:11:27 -0700983 &blkio_policy_throtl, cft->private, false);
Tejun Heo60c2bc22012-04-01 14:38:43 -0700984 return 0;
985}
986
Tejun Heoaf133ce2012-04-01 14:38:44 -0700987static int tg_print_conf_uint(struct cgroup *cgrp, struct cftype *cft,
988 struct seq_file *sf)
Vivek Goyale43473b2010-09-15 17:06:35 -0400989{
Tejun Heoaf133ce2012-04-01 14:38:44 -0700990 blkcg_print_blkgs(sf, cgroup_to_blkio_cgroup(cgrp), tg_prfill_conf_uint,
Tejun Heoec399342012-04-13 13:11:27 -0700991 &blkio_policy_throtl, cft->private, false);
Tejun Heoaf133ce2012-04-01 14:38:44 -0700992 return 0;
Vivek Goyale43473b2010-09-15 17:06:35 -0400993}
994
Tejun Heoaf133ce2012-04-01 14:38:44 -0700995static int tg_set_conf(struct cgroup *cgrp, struct cftype *cft, const char *buf,
996 bool is_u64)
Tejun Heo60c2bc22012-04-01 14:38:43 -0700997{
998 struct blkio_cgroup *blkcg = cgroup_to_blkio_cgroup(cgrp);
Tejun Heo60c2bc22012-04-01 14:38:43 -0700999 struct blkg_conf_ctx ctx;
Tejun Heoaf133ce2012-04-01 14:38:44 -07001000 struct throtl_grp *tg;
Tejun Heoa2b16932012-04-13 13:11:33 -07001001 struct throtl_data *td;
Tejun Heo60c2bc22012-04-01 14:38:43 -07001002 int ret;
1003
Tejun Heoda8b0662012-04-13 13:11:29 -07001004 ret = blkg_conf_prep(blkcg, &blkio_policy_throtl, buf, &ctx);
Tejun Heo60c2bc22012-04-01 14:38:43 -07001005 if (ret)
1006 return ret;
1007
Tejun Heoaf133ce2012-04-01 14:38:44 -07001008 tg = blkg_to_tg(ctx.blkg);
Tejun Heoa2b16932012-04-13 13:11:33 -07001009 td = ctx.blkg->q->td;
Tejun Heoaf133ce2012-04-01 14:38:44 -07001010
Tejun Heoa2b16932012-04-13 13:11:33 -07001011 if (!ctx.v)
1012 ctx.v = -1;
Tejun Heoaf133ce2012-04-01 14:38:44 -07001013
Tejun Heoa2b16932012-04-13 13:11:33 -07001014 if (is_u64)
1015 *(u64 *)((void *)tg + cft->private) = ctx.v;
1016 else
1017 *(unsigned int *)((void *)tg + cft->private) = ctx.v;
Tejun Heoaf133ce2012-04-01 14:38:44 -07001018
Tejun Heoa2b16932012-04-13 13:11:33 -07001019 /* XXX: we don't need the following deferred processing */
1020 xchg(&tg->limits_changed, true);
1021 xchg(&td->limits_changed, true);
1022 throtl_schedule_delayed_work(td, 0);
Tejun Heo60c2bc22012-04-01 14:38:43 -07001023
1024 blkg_conf_finish(&ctx);
Tejun Heoa2b16932012-04-13 13:11:33 -07001025 return 0;
Tejun Heo60c2bc22012-04-01 14:38:43 -07001026}
1027
Tejun Heoaf133ce2012-04-01 14:38:44 -07001028static int tg_set_conf_u64(struct cgroup *cgrp, struct cftype *cft,
1029 const char *buf)
Tejun Heo60c2bc22012-04-01 14:38:43 -07001030{
Tejun Heoaf133ce2012-04-01 14:38:44 -07001031 return tg_set_conf(cgrp, cft, buf, true);
Tejun Heo60c2bc22012-04-01 14:38:43 -07001032}
1033
Tejun Heoaf133ce2012-04-01 14:38:44 -07001034static int tg_set_conf_uint(struct cgroup *cgrp, struct cftype *cft,
1035 const char *buf)
Tejun Heo60c2bc22012-04-01 14:38:43 -07001036{
Tejun Heoaf133ce2012-04-01 14:38:44 -07001037 return tg_set_conf(cgrp, cft, buf, false);
Tejun Heo60c2bc22012-04-01 14:38:43 -07001038}
1039
1040static struct cftype throtl_files[] = {
1041 {
1042 .name = "throttle.read_bps_device",
Tejun Heoaf133ce2012-04-01 14:38:44 -07001043 .private = offsetof(struct throtl_grp, bps[READ]),
1044 .read_seq_string = tg_print_conf_u64,
1045 .write_string = tg_set_conf_u64,
Tejun Heo60c2bc22012-04-01 14:38:43 -07001046 .max_write_len = 256,
1047 },
1048 {
1049 .name = "throttle.write_bps_device",
Tejun Heoaf133ce2012-04-01 14:38:44 -07001050 .private = offsetof(struct throtl_grp, bps[WRITE]),
1051 .read_seq_string = tg_print_conf_u64,
1052 .write_string = tg_set_conf_u64,
Tejun Heo60c2bc22012-04-01 14:38:43 -07001053 .max_write_len = 256,
1054 },
1055 {
1056 .name = "throttle.read_iops_device",
Tejun Heoaf133ce2012-04-01 14:38:44 -07001057 .private = offsetof(struct throtl_grp, iops[READ]),
1058 .read_seq_string = tg_print_conf_uint,
1059 .write_string = tg_set_conf_uint,
Tejun Heo60c2bc22012-04-01 14:38:43 -07001060 .max_write_len = 256,
1061 },
1062 {
1063 .name = "throttle.write_iops_device",
Tejun Heoaf133ce2012-04-01 14:38:44 -07001064 .private = offsetof(struct throtl_grp, iops[WRITE]),
1065 .read_seq_string = tg_print_conf_uint,
1066 .write_string = tg_set_conf_uint,
Tejun Heo60c2bc22012-04-01 14:38:43 -07001067 .max_write_len = 256,
1068 },
1069 {
1070 .name = "throttle.io_service_bytes",
Tejun Heo5bc4afb12012-04-01 14:38:45 -07001071 .private = offsetof(struct tg_stats_cpu, service_bytes),
Tejun Heo8a3d2612012-04-01 14:38:44 -07001072 .read_seq_string = tg_print_cpu_rwstat,
Tejun Heo60c2bc22012-04-01 14:38:43 -07001073 },
1074 {
1075 .name = "throttle.io_serviced",
Tejun Heo5bc4afb12012-04-01 14:38:45 -07001076 .private = offsetof(struct tg_stats_cpu, serviced),
Tejun Heo8a3d2612012-04-01 14:38:44 -07001077 .read_seq_string = tg_print_cpu_rwstat,
Tejun Heo60c2bc22012-04-01 14:38:43 -07001078 },
1079 { } /* terminate */
1080};
1081
Vivek Goyalda527772011-03-02 19:05:33 -05001082static void throtl_shutdown_wq(struct request_queue *q)
Vivek Goyale43473b2010-09-15 17:06:35 -04001083{
1084 struct throtl_data *td = q->td;
1085
1086 cancel_delayed_work_sync(&td->throtl_work);
1087}
1088
1089static struct blkio_policy_type blkio_policy_throtl = {
1090 .ops = {
Tejun Heo03814112012-03-05 13:15:14 -08001091 .blkio_init_group_fn = throtl_init_blkio_group,
Tejun Heo8a3d2612012-04-01 14:38:44 -07001092 .blkio_exit_group_fn = throtl_exit_blkio_group,
1093 .blkio_reset_group_stats_fn = throtl_reset_group_stats,
Vivek Goyale43473b2010-09-15 17:06:35 -04001094 },
Tejun Heo03814112012-03-05 13:15:14 -08001095 .pdata_size = sizeof(struct throtl_grp),
Tejun Heo60c2bc22012-04-01 14:38:43 -07001096 .cftypes = throtl_files,
Vivek Goyale43473b2010-09-15 17:06:35 -04001097};
1098
Tejun Heobc16a4f2011-10-19 14:33:01 +02001099bool blk_throtl_bio(struct request_queue *q, struct bio *bio)
Vivek Goyale43473b2010-09-15 17:06:35 -04001100{
1101 struct throtl_data *td = q->td;
1102 struct throtl_grp *tg;
Vivek Goyale43473b2010-09-15 17:06:35 -04001103 bool rw = bio_data_dir(bio), update_disptime = true;
Vivek Goyalaf75cd32011-05-19 15:38:31 -04001104 struct blkio_cgroup *blkcg;
Tejun Heobc16a4f2011-10-19 14:33:01 +02001105 bool throttled = false;
Vivek Goyale43473b2010-09-15 17:06:35 -04001106
1107 if (bio->bi_rw & REQ_THROTTLED) {
1108 bio->bi_rw &= ~REQ_THROTTLED;
Tejun Heobc16a4f2011-10-19 14:33:01 +02001109 goto out;
Vivek Goyale43473b2010-09-15 17:06:35 -04001110 }
1111
Tejun Heo671058f2012-03-05 13:15:29 -08001112 /* bio_associate_current() needs ioc, try creating */
1113 create_io_context(GFP_ATOMIC, q->node);
1114
Vivek Goyalaf75cd32011-05-19 15:38:31 -04001115 /*
1116 * A throtl_grp pointer retrieved under rcu can be used to access
1117 * basic fields like stats and io rates. If a group has no rules,
1118 * just update the dispatch stats in lockless manner and return.
1119 */
Vivek Goyalaf75cd32011-05-19 15:38:31 -04001120 rcu_read_lock();
Tejun Heo4f85cb92012-03-05 13:15:28 -08001121 blkcg = bio_blkio_cgroup(bio);
Tejun Heocd1604f2012-03-05 13:15:06 -08001122 tg = throtl_lookup_tg(td, blkcg);
Vivek Goyalaf75cd32011-05-19 15:38:31 -04001123 if (tg) {
Vivek Goyalaf75cd32011-05-19 15:38:31 -04001124 if (tg_no_rule_group(tg, rw)) {
Tejun Heo629ed0b2012-04-01 14:38:44 -07001125 throtl_update_dispatch_stats(tg_to_blkg(tg),
1126 bio->bi_size, bio->bi_rw);
Tejun Heo2a7f1242012-03-05 13:15:01 -08001127 goto out_unlock_rcu;
Vivek Goyalaf75cd32011-05-19 15:38:31 -04001128 }
1129 }
Vivek Goyalaf75cd32011-05-19 15:38:31 -04001130
1131 /*
1132 * Either group has not been allocated yet or it is not an unlimited
1133 * IO group
1134 */
Vivek Goyale43473b2010-09-15 17:06:35 -04001135 spin_lock_irq(q->queue_lock);
Tejun Heocd1604f2012-03-05 13:15:06 -08001136 tg = throtl_lookup_create_tg(td, blkcg);
Tejun Heobc16a4f2011-10-19 14:33:01 +02001137 if (unlikely(!tg))
1138 goto out_unlock;
Vivek Goyalf469a7b2011-05-19 15:38:23 -04001139
Vivek Goyale43473b2010-09-15 17:06:35 -04001140 if (tg->nr_queued[rw]) {
1141 /*
1142 * There is already another bio queued in same dir. No
1143 * need to update dispatch time.
1144 */
Vivek Goyal231d7042011-03-07 21:05:14 +01001145 update_disptime = false;
Vivek Goyale43473b2010-09-15 17:06:35 -04001146 goto queue_bio;
Vivek Goyalde701c72011-03-07 21:09:32 +01001147
Vivek Goyale43473b2010-09-15 17:06:35 -04001148 }
1149
1150 /* Bio is with-in rate limit of group */
1151 if (tg_may_dispatch(td, tg, bio, NULL)) {
1152 throtl_charge_bio(tg, bio);
Vivek Goyal04521db2011-03-22 21:54:29 +01001153
1154 /*
1155 * We need to trim slice even when bios are not being queued
1156 * otherwise it might happen that a bio is not queued for
1157 * a long time and slice keeps on extending and trim is not
1158 * called for a long time. Now if limits are reduced suddenly
1159 * we take into account all the IO dispatched so far at new
1160 * low rate and * newly queued IO gets a really long dispatch
1161 * time.
1162 *
1163 * So keep on trimming slice even if bio is not queued.
1164 */
1165 throtl_trim_slice(td, tg, rw);
Tejun Heobc16a4f2011-10-19 14:33:01 +02001166 goto out_unlock;
Vivek Goyale43473b2010-09-15 17:06:35 -04001167 }
1168
1169queue_bio:
Joe Perchesfd16d262011-06-13 10:42:49 +02001170 throtl_log_tg(td, tg, "[%c] bio. bdisp=%llu sz=%u bps=%llu"
Vivek Goyal8e89d132010-09-15 17:06:37 -04001171 " iodisp=%u iops=%u queued=%d/%d",
1172 rw == READ ? 'R' : 'W',
Vivek Goyale43473b2010-09-15 17:06:35 -04001173 tg->bytes_disp[rw], bio->bi_size, tg->bps[rw],
Vivek Goyal8e89d132010-09-15 17:06:37 -04001174 tg->io_disp[rw], tg->iops[rw],
Vivek Goyale43473b2010-09-15 17:06:35 -04001175 tg->nr_queued[READ], tg->nr_queued[WRITE]);
1176
Tejun Heo671058f2012-03-05 13:15:29 -08001177 bio_associate_current(bio);
Vivek Goyale43473b2010-09-15 17:06:35 -04001178 throtl_add_bio_tg(q->td, tg, bio);
Tejun Heobc16a4f2011-10-19 14:33:01 +02001179 throttled = true;
Vivek Goyale43473b2010-09-15 17:06:35 -04001180
1181 if (update_disptime) {
1182 tg_update_disptime(td, tg);
1183 throtl_schedule_next_dispatch(td);
1184 }
1185
Tejun Heobc16a4f2011-10-19 14:33:01 +02001186out_unlock:
Vivek Goyale43473b2010-09-15 17:06:35 -04001187 spin_unlock_irq(q->queue_lock);
Tejun Heo2a7f1242012-03-05 13:15:01 -08001188out_unlock_rcu:
1189 rcu_read_unlock();
Tejun Heobc16a4f2011-10-19 14:33:01 +02001190out:
1191 return throttled;
Vivek Goyale43473b2010-09-15 17:06:35 -04001192}
1193
Tejun Heoc9a929d2011-10-19 14:42:16 +02001194/**
1195 * blk_throtl_drain - drain throttled bios
1196 * @q: request_queue to drain throttled bios for
1197 *
1198 * Dispatch all currently throttled bios on @q through ->make_request_fn().
1199 */
1200void blk_throtl_drain(struct request_queue *q)
1201 __releases(q->queue_lock) __acquires(q->queue_lock)
1202{
1203 struct throtl_data *td = q->td;
1204 struct throtl_rb_root *st = &td->tg_service_tree;
1205 struct throtl_grp *tg;
1206 struct bio_list bl;
1207 struct bio *bio;
1208
Jens Axboe334c2b02011-10-25 15:51:48 +02001209 WARN_ON_ONCE(!queue_is_locked(q));
Tejun Heoc9a929d2011-10-19 14:42:16 +02001210
1211 bio_list_init(&bl);
1212
1213 while ((tg = throtl_rb_first(st))) {
1214 throtl_dequeue_tg(td, tg);
1215
1216 while ((bio = bio_list_peek(&tg->bio_lists[READ])))
1217 tg_dispatch_one_bio(td, tg, bio_data_dir(bio), &bl);
1218 while ((bio = bio_list_peek(&tg->bio_lists[WRITE])))
1219 tg_dispatch_one_bio(td, tg, bio_data_dir(bio), &bl);
1220 }
1221 spin_unlock_irq(q->queue_lock);
1222
1223 while ((bio = bio_list_pop(&bl)))
1224 generic_make_request(bio);
1225
1226 spin_lock_irq(q->queue_lock);
1227}
1228
Vivek Goyale43473b2010-09-15 17:06:35 -04001229int blk_throtl_init(struct request_queue *q)
1230{
1231 struct throtl_data *td;
Tejun Heoa2b16932012-04-13 13:11:33 -07001232 int ret;
Vivek Goyale43473b2010-09-15 17:06:35 -04001233
1234 td = kzalloc_node(sizeof(*td), GFP_KERNEL, q->node);
1235 if (!td)
1236 return -ENOMEM;
1237
Vivek Goyale43473b2010-09-15 17:06:35 -04001238 td->tg_service_tree = THROTL_RB_ROOT;
Vivek Goyalde701c72011-03-07 21:09:32 +01001239 td->limits_changed = false;
Vivek Goyala29a1712011-05-19 15:38:19 -04001240 INIT_DELAYED_WORK(&td->throtl_work, blk_throtl_work);
Vivek Goyale43473b2010-09-15 17:06:35 -04001241
Tejun Heocd1604f2012-03-05 13:15:06 -08001242 q->td = td;
Vivek Goyal29b12582011-05-19 15:38:24 -04001243 td->queue = q;
Vivek Goyal02977e42010-10-01 14:49:48 +02001244
Tejun Heoa2b16932012-04-13 13:11:33 -07001245 /* activate policy */
1246 ret = blkcg_activate_policy(q, &blkio_policy_throtl);
1247 if (ret)
Vivek Goyal29b12582011-05-19 15:38:24 -04001248 kfree(td);
Tejun Heoa2b16932012-04-13 13:11:33 -07001249 return ret;
Vivek Goyale43473b2010-09-15 17:06:35 -04001250}
1251
1252void blk_throtl_exit(struct request_queue *q)
1253{
Tejun Heoc875f4d2012-03-05 13:15:22 -08001254 BUG_ON(!q->td);
Vivek Goyalda527772011-03-02 19:05:33 -05001255 throtl_shutdown_wq(q);
Tejun Heoa2b16932012-04-13 13:11:33 -07001256 blkcg_deactivate_policy(q, &blkio_policy_throtl);
Tejun Heoc9a929d2011-10-19 14:42:16 +02001257 kfree(q->td);
Vivek Goyale43473b2010-09-15 17:06:35 -04001258}
1259
1260static int __init throtl_init(void)
1261{
Vivek Goyal450adcb2011-03-01 13:40:54 -05001262 kthrotld_workqueue = alloc_workqueue("kthrotld", WQ_MEM_RECLAIM, 0);
1263 if (!kthrotld_workqueue)
1264 panic("Failed to create kthrotld\n");
1265
Tejun Heo8bd435b2012-04-13 13:11:28 -07001266 return blkio_policy_register(&blkio_policy_throtl);
Vivek Goyale43473b2010-09-15 17:06:35 -04001267}
1268
1269module_init(throtl_init);