blob: 420eaa150d119519736fe7914999b754497b4f84 [file] [log] [blame]
Vivek Goyale43473b2010-09-15 17:06:35 -04001/*
2 * Interface for controlling IO bandwidth on a request queue
3 *
4 * Copyright (C) 2010 Vivek Goyal <vgoyal@redhat.com>
5 */
6
7#include <linux/module.h>
8#include <linux/slab.h>
9#include <linux/blkdev.h>
10#include <linux/bio.h>
11#include <linux/blktrace_api.h>
12#include "blk-cgroup.h"
Tejun Heobc9fcbf2011-10-19 14:31:18 +020013#include "blk.h"
Vivek Goyale43473b2010-09-15 17:06:35 -040014
15/* Max dispatch from a group in 1 round */
16static int throtl_grp_quantum = 8;
17
18/* Total max dispatch from all groups in one round */
19static int throtl_quantum = 32;
20
21/* Throttling is performed over 100ms slice and after that slice is renewed */
22static unsigned long throtl_slice = HZ/10; /* 100 ms */
23
Tejun Heo3c798392012-04-16 13:57:25 -070024static struct blkcg_policy blkcg_policy_throtl;
Tejun Heo03814112012-03-05 13:15:14 -080025
Vivek Goyal450adcb2011-03-01 13:40:54 -050026/* A workqueue to queue throttle related work */
27static struct workqueue_struct *kthrotld_workqueue;
Vivek Goyal450adcb2011-03-01 13:40:54 -050028
Tejun Heoc9e03322013-05-14 13:52:32 -070029struct throtl_service_queue {
Tejun Heo77216b02013-05-14 13:52:36 -070030 struct throtl_service_queue *parent_sq; /* the parent service_queue */
31
Tejun Heo73f0d492013-05-14 13:52:35 -070032 /*
33 * Bios queued directly to this service_queue or dispatched from
34 * children throtl_grp's.
35 */
36 struct bio_list bio_lists[2]; /* queued bios [READ/WRITE] */
37 unsigned int nr_queued[2]; /* number of queued bios */
38
39 /*
40 * RB tree of active children throtl_grp's, which are sorted by
41 * their ->disptime.
42 */
Tejun Heoc9e03322013-05-14 13:52:32 -070043 struct rb_root pending_tree; /* RB tree of active tgs */
44 struct rb_node *first_pending; /* first node in the tree */
45 unsigned int nr_pending; /* # queued in the tree */
46 unsigned long first_pending_disptime; /* disptime of the first tg */
Vivek Goyale43473b2010-09-15 17:06:35 -040047};
48
Tejun Heo5b2c16a2013-05-14 13:52:32 -070049enum tg_state_flags {
50 THROTL_TG_PENDING = 1 << 0, /* on parent's pending tree */
Tejun Heo0e9f4162013-05-14 13:52:35 -070051 THROTL_TG_WAS_EMPTY = 1 << 1, /* bio_lists[] became non-empty */
Tejun Heo5b2c16a2013-05-14 13:52:32 -070052};
53
Vivek Goyale43473b2010-09-15 17:06:35 -040054#define rb_entry_tg(node) rb_entry((node), struct throtl_grp, rb_node)
55
Tejun Heo8a3d2612012-04-01 14:38:44 -070056/* Per-cpu group stats */
57struct tg_stats_cpu {
58 /* total bytes transferred */
59 struct blkg_rwstat service_bytes;
60 /* total IOs serviced, post merge */
61 struct blkg_rwstat serviced;
62};
63
Vivek Goyale43473b2010-09-15 17:06:35 -040064struct throtl_grp {
Tejun Heof95a04a2012-04-16 13:57:26 -070065 /* must be the first member */
66 struct blkg_policy_data pd;
67
Tejun Heoc9e03322013-05-14 13:52:32 -070068 /* active throtl group service_queue member */
Vivek Goyale43473b2010-09-15 17:06:35 -040069 struct rb_node rb_node;
70
Tejun Heo0f3457f2013-05-14 13:52:32 -070071 /* throtl_data this group belongs to */
72 struct throtl_data *td;
73
Tejun Heo49a2f1e2013-05-14 13:52:34 -070074 /* this group's service queue */
75 struct throtl_service_queue service_queue;
76
Vivek Goyale43473b2010-09-15 17:06:35 -040077 /*
78 * Dispatch time in jiffies. This is the estimated time when group
79 * will unthrottle and is ready to dispatch more bio. It is used as
80 * key to sort active groups in service tree.
81 */
82 unsigned long disptime;
83
Vivek Goyale43473b2010-09-15 17:06:35 -040084 unsigned int flags;
85
Vivek Goyale43473b2010-09-15 17:06:35 -040086 /* bytes per second rate limits */
87 uint64_t bps[2];
88
Vivek Goyal8e89d132010-09-15 17:06:37 -040089 /* IOPS limits */
90 unsigned int iops[2];
91
Vivek Goyale43473b2010-09-15 17:06:35 -040092 /* Number of bytes disptached in current slice */
93 uint64_t bytes_disp[2];
Vivek Goyal8e89d132010-09-15 17:06:37 -040094 /* Number of bio's dispatched in current slice */
95 unsigned int io_disp[2];
Vivek Goyale43473b2010-09-15 17:06:35 -040096
97 /* When did we start a new slice */
98 unsigned long slice_start[2];
99 unsigned long slice_end[2];
Vivek Goyalfe071432010-10-01 14:49:49 +0200100
Tejun Heo8a3d2612012-04-01 14:38:44 -0700101 /* Per cpu stats pointer */
102 struct tg_stats_cpu __percpu *stats_cpu;
103
104 /* List of tgs waiting for per cpu stats memory to be allocated */
105 struct list_head stats_alloc_node;
Vivek Goyale43473b2010-09-15 17:06:35 -0400106};
107
108struct throtl_data
109{
Vivek Goyale43473b2010-09-15 17:06:35 -0400110 /* service tree for active throtl groups */
Tejun Heoc9e03322013-05-14 13:52:32 -0700111 struct throtl_service_queue service_queue;
Vivek Goyale43473b2010-09-15 17:06:35 -0400112
Vivek Goyale43473b2010-09-15 17:06:35 -0400113 struct request_queue *queue;
114
115 /* Total Number of queued bios on READ and WRITE lists */
116 unsigned int nr_queued[2];
117
118 /*
Vivek Goyal02977e42010-10-01 14:49:48 +0200119 * number of total undestroyed groups
Vivek Goyale43473b2010-09-15 17:06:35 -0400120 */
121 unsigned int nr_undestroyed_grps;
122
123 /* Work for dispatching throttled bios */
Tejun Heocb761992013-05-14 13:52:31 -0700124 struct delayed_work dispatch_work;
Vivek Goyale43473b2010-09-15 17:06:35 -0400125};
126
Tejun Heo8a3d2612012-04-01 14:38:44 -0700127/* list and work item to allocate percpu group stats */
128static DEFINE_SPINLOCK(tg_stats_alloc_lock);
129static LIST_HEAD(tg_stats_alloc_list);
130
131static void tg_stats_alloc_fn(struct work_struct *);
132static DECLARE_DELAYED_WORK(tg_stats_alloc_work, tg_stats_alloc_fn);
133
Tejun Heof95a04a2012-04-16 13:57:26 -0700134static inline struct throtl_grp *pd_to_tg(struct blkg_policy_data *pd)
135{
136 return pd ? container_of(pd, struct throtl_grp, pd) : NULL;
137}
138
Tejun Heo3c798392012-04-16 13:57:25 -0700139static inline struct throtl_grp *blkg_to_tg(struct blkcg_gq *blkg)
Tejun Heo03814112012-03-05 13:15:14 -0800140{
Tejun Heof95a04a2012-04-16 13:57:26 -0700141 return pd_to_tg(blkg_to_pd(blkg, &blkcg_policy_throtl));
Tejun Heo03814112012-03-05 13:15:14 -0800142}
143
Tejun Heo3c798392012-04-16 13:57:25 -0700144static inline struct blkcg_gq *tg_to_blkg(struct throtl_grp *tg)
Tejun Heo03814112012-03-05 13:15:14 -0800145{
Tejun Heof95a04a2012-04-16 13:57:26 -0700146 return pd_to_blkg(&tg->pd);
Tejun Heo03814112012-03-05 13:15:14 -0800147}
148
Tejun Heo03d8e112012-04-13 13:11:32 -0700149static inline struct throtl_grp *td_root_tg(struct throtl_data *td)
150{
151 return blkg_to_tg(td->queue->root_blkg);
152}
153
Tejun Heofda6f272013-05-14 13:52:36 -0700154/**
155 * sq_to_tg - return the throl_grp the specified service queue belongs to
156 * @sq: the throtl_service_queue of interest
157 *
158 * Return the throtl_grp @sq belongs to. If @sq is the top-level one
159 * embedded in throtl_data, %NULL is returned.
160 */
161static struct throtl_grp *sq_to_tg(struct throtl_service_queue *sq)
162{
163 if (sq && sq->parent_sq)
164 return container_of(sq, struct throtl_grp, service_queue);
165 else
166 return NULL;
167}
Vivek Goyale43473b2010-09-15 17:06:35 -0400168
Tejun Heofda6f272013-05-14 13:52:36 -0700169/**
170 * sq_to_td - return throtl_data the specified service queue belongs to
171 * @sq: the throtl_service_queue of interest
172 *
173 * A service_queue can be embeded in either a throtl_grp or throtl_data.
174 * Determine the associated throtl_data accordingly and return it.
175 */
176static struct throtl_data *sq_to_td(struct throtl_service_queue *sq)
177{
178 struct throtl_grp *tg = sq_to_tg(sq);
179
180 if (tg)
181 return tg->td;
182 else
183 return container_of(sq, struct throtl_data, service_queue);
184}
185
186/**
187 * throtl_log - log debug message via blktrace
188 * @sq: the service_queue being reported
189 * @fmt: printf format string
190 * @args: printf args
191 *
192 * The messages are prefixed with "throtl BLKG_NAME" if @sq belongs to a
193 * throtl_grp; otherwise, just "throtl".
194 *
195 * TODO: this should be made a function and name formatting should happen
196 * after testing whether blktrace is enabled.
197 */
198#define throtl_log(sq, fmt, args...) do { \
199 struct throtl_grp *__tg = sq_to_tg((sq)); \
200 struct throtl_data *__td = sq_to_td((sq)); \
201 \
202 (void)__td; \
203 if ((__tg)) { \
204 char __pbuf[128]; \
205 \
206 blkg_path(tg_to_blkg(__tg), __pbuf, sizeof(__pbuf)); \
207 blk_add_trace_msg(__td->queue, "throtl %s " fmt, __pbuf, ##args); \
208 } else { \
209 blk_add_trace_msg(__td->queue, "throtl " fmt, ##args); \
210 } \
211} while (0)
Vivek Goyale43473b2010-09-15 17:06:35 -0400212
Tejun Heo8a3d2612012-04-01 14:38:44 -0700213/*
214 * Worker for allocating per cpu stat for tgs. This is scheduled on the
Tejun Heo3b07e9c2012-08-20 14:51:24 -0700215 * system_wq once there are some groups on the alloc_list waiting for
Tejun Heo8a3d2612012-04-01 14:38:44 -0700216 * allocation.
217 */
218static void tg_stats_alloc_fn(struct work_struct *work)
219{
220 static struct tg_stats_cpu *stats_cpu; /* this fn is non-reentrant */
221 struct delayed_work *dwork = to_delayed_work(work);
222 bool empty = false;
223
224alloc_stats:
225 if (!stats_cpu) {
226 stats_cpu = alloc_percpu(struct tg_stats_cpu);
227 if (!stats_cpu) {
228 /* allocation failed, try again after some time */
Tejun Heo3b07e9c2012-08-20 14:51:24 -0700229 schedule_delayed_work(dwork, msecs_to_jiffies(10));
Tejun Heo8a3d2612012-04-01 14:38:44 -0700230 return;
231 }
232 }
233
234 spin_lock_irq(&tg_stats_alloc_lock);
235
236 if (!list_empty(&tg_stats_alloc_list)) {
237 struct throtl_grp *tg = list_first_entry(&tg_stats_alloc_list,
238 struct throtl_grp,
239 stats_alloc_node);
240 swap(tg->stats_cpu, stats_cpu);
241 list_del_init(&tg->stats_alloc_node);
242 }
243
244 empty = list_empty(&tg_stats_alloc_list);
245 spin_unlock_irq(&tg_stats_alloc_lock);
246 if (!empty)
247 goto alloc_stats;
248}
249
Tejun Heo49a2f1e2013-05-14 13:52:34 -0700250/* init a service_queue, assumes the caller zeroed it */
Tejun Heo77216b02013-05-14 13:52:36 -0700251static void throtl_service_queue_init(struct throtl_service_queue *sq,
252 struct throtl_service_queue *parent_sq)
Tejun Heo49a2f1e2013-05-14 13:52:34 -0700253{
Tejun Heo73f0d492013-05-14 13:52:35 -0700254 bio_list_init(&sq->bio_lists[0]);
255 bio_list_init(&sq->bio_lists[1]);
Tejun Heo49a2f1e2013-05-14 13:52:34 -0700256 sq->pending_tree = RB_ROOT;
Tejun Heo77216b02013-05-14 13:52:36 -0700257 sq->parent_sq = parent_sq;
Tejun Heo49a2f1e2013-05-14 13:52:34 -0700258}
259
Tejun Heo3c798392012-04-16 13:57:25 -0700260static void throtl_pd_init(struct blkcg_gq *blkg)
Vivek Goyala29a1712011-05-19 15:38:19 -0400261{
Tejun Heo03814112012-03-05 13:15:14 -0800262 struct throtl_grp *tg = blkg_to_tg(blkg);
Tejun Heo77216b02013-05-14 13:52:36 -0700263 struct throtl_data *td = blkg->q->td;
Tejun Heoff26eaa2012-05-23 12:16:21 +0200264 unsigned long flags;
Tejun Heocd1604f2012-03-05 13:15:06 -0800265
Tejun Heo77216b02013-05-14 13:52:36 -0700266 throtl_service_queue_init(&tg->service_queue, &td->service_queue);
Vivek Goyala29a1712011-05-19 15:38:19 -0400267 RB_CLEAR_NODE(&tg->rb_node);
Tejun Heo77216b02013-05-14 13:52:36 -0700268 tg->td = td;
Vivek Goyala29a1712011-05-19 15:38:19 -0400269
Tejun Heoe56da7e2012-03-05 13:15:07 -0800270 tg->bps[READ] = -1;
271 tg->bps[WRITE] = -1;
272 tg->iops[READ] = -1;
273 tg->iops[WRITE] = -1;
Tejun Heo8a3d2612012-04-01 14:38:44 -0700274
275 /*
276 * Ugh... We need to perform per-cpu allocation for tg->stats_cpu
277 * but percpu allocator can't be called from IO path. Queue tg on
278 * tg_stats_alloc_list and allocate from work item.
279 */
Tejun Heoff26eaa2012-05-23 12:16:21 +0200280 spin_lock_irqsave(&tg_stats_alloc_lock, flags);
Tejun Heo8a3d2612012-04-01 14:38:44 -0700281 list_add(&tg->stats_alloc_node, &tg_stats_alloc_list);
Tejun Heo3b07e9c2012-08-20 14:51:24 -0700282 schedule_delayed_work(&tg_stats_alloc_work, 0);
Tejun Heoff26eaa2012-05-23 12:16:21 +0200283 spin_unlock_irqrestore(&tg_stats_alloc_lock, flags);
Tejun Heo8a3d2612012-04-01 14:38:44 -0700284}
285
Tejun Heo3c798392012-04-16 13:57:25 -0700286static void throtl_pd_exit(struct blkcg_gq *blkg)
Tejun Heo8a3d2612012-04-01 14:38:44 -0700287{
288 struct throtl_grp *tg = blkg_to_tg(blkg);
Tejun Heoff26eaa2012-05-23 12:16:21 +0200289 unsigned long flags;
Tejun Heo8a3d2612012-04-01 14:38:44 -0700290
Tejun Heoff26eaa2012-05-23 12:16:21 +0200291 spin_lock_irqsave(&tg_stats_alloc_lock, flags);
Tejun Heo8a3d2612012-04-01 14:38:44 -0700292 list_del_init(&tg->stats_alloc_node);
Tejun Heoff26eaa2012-05-23 12:16:21 +0200293 spin_unlock_irqrestore(&tg_stats_alloc_lock, flags);
Tejun Heo8a3d2612012-04-01 14:38:44 -0700294
295 free_percpu(tg->stats_cpu);
296}
297
Tejun Heo3c798392012-04-16 13:57:25 -0700298static void throtl_pd_reset_stats(struct blkcg_gq *blkg)
Tejun Heo8a3d2612012-04-01 14:38:44 -0700299{
300 struct throtl_grp *tg = blkg_to_tg(blkg);
301 int cpu;
302
303 if (tg->stats_cpu == NULL)
304 return;
305
306 for_each_possible_cpu(cpu) {
307 struct tg_stats_cpu *sc = per_cpu_ptr(tg->stats_cpu, cpu);
308
309 blkg_rwstat_reset(&sc->service_bytes);
310 blkg_rwstat_reset(&sc->serviced);
311 }
Vivek Goyala29a1712011-05-19 15:38:19 -0400312}
313
Tejun Heo3c798392012-04-16 13:57:25 -0700314static struct throtl_grp *throtl_lookup_tg(struct throtl_data *td,
315 struct blkcg *blkcg)
Vivek Goyale43473b2010-09-15 17:06:35 -0400316{
Vivek Goyale43473b2010-09-15 17:06:35 -0400317 /*
Tejun Heo3c798392012-04-16 13:57:25 -0700318 * This is the common case when there are no blkcgs. Avoid lookup
319 * in this case
Tejun Heocd1604f2012-03-05 13:15:06 -0800320 */
Tejun Heo3c798392012-04-16 13:57:25 -0700321 if (blkcg == &blkcg_root)
Tejun Heo03d8e112012-04-13 13:11:32 -0700322 return td_root_tg(td);
Vivek Goyale43473b2010-09-15 17:06:35 -0400323
Tejun Heoe8989fa2012-03-05 13:15:20 -0800324 return blkg_to_tg(blkg_lookup(blkcg, td->queue));
Vivek Goyale43473b2010-09-15 17:06:35 -0400325}
326
Tejun Heocd1604f2012-03-05 13:15:06 -0800327static struct throtl_grp *throtl_lookup_create_tg(struct throtl_data *td,
Tejun Heo3c798392012-04-16 13:57:25 -0700328 struct blkcg *blkcg)
Vivek Goyale43473b2010-09-15 17:06:35 -0400329{
Vivek Goyalf469a7b2011-05-19 15:38:23 -0400330 struct request_queue *q = td->queue;
Tejun Heocd1604f2012-03-05 13:15:06 -0800331 struct throtl_grp *tg = NULL;
Tejun Heo0a5a7d02012-03-05 13:15:02 -0800332
Vivek Goyalf469a7b2011-05-19 15:38:23 -0400333 /*
Tejun Heo3c798392012-04-16 13:57:25 -0700334 * This is the common case when there are no blkcgs. Avoid lookup
335 * in this case
Vivek Goyalf469a7b2011-05-19 15:38:23 -0400336 */
Tejun Heo3c798392012-04-16 13:57:25 -0700337 if (blkcg == &blkcg_root) {
Tejun Heo03d8e112012-04-13 13:11:32 -0700338 tg = td_root_tg(td);
Tejun Heocd1604f2012-03-05 13:15:06 -0800339 } else {
Tejun Heo3c798392012-04-16 13:57:25 -0700340 struct blkcg_gq *blkg;
Tejun Heocd1604f2012-03-05 13:15:06 -0800341
Tejun Heo3c96cb32012-04-13 13:11:34 -0700342 blkg = blkg_lookup_create(blkcg, q);
Tejun Heocd1604f2012-03-05 13:15:06 -0800343
344 /* if %NULL and @q is alive, fall back to root_tg */
345 if (!IS_ERR(blkg))
Tejun Heo03814112012-03-05 13:15:14 -0800346 tg = blkg_to_tg(blkg);
Bart Van Assche3f3299d2012-11-28 13:42:38 +0100347 else if (!blk_queue_dying(q))
Tejun Heo03d8e112012-04-13 13:11:32 -0700348 tg = td_root_tg(td);
Vivek Goyalf469a7b2011-05-19 15:38:23 -0400349 }
350
Vivek Goyale43473b2010-09-15 17:06:35 -0400351 return tg;
352}
353
Tejun Heo0049af72013-05-14 13:52:33 -0700354static struct throtl_grp *
355throtl_rb_first(struct throtl_service_queue *parent_sq)
Vivek Goyale43473b2010-09-15 17:06:35 -0400356{
357 /* Service tree is empty */
Tejun Heo0049af72013-05-14 13:52:33 -0700358 if (!parent_sq->nr_pending)
Vivek Goyale43473b2010-09-15 17:06:35 -0400359 return NULL;
360
Tejun Heo0049af72013-05-14 13:52:33 -0700361 if (!parent_sq->first_pending)
362 parent_sq->first_pending = rb_first(&parent_sq->pending_tree);
Vivek Goyale43473b2010-09-15 17:06:35 -0400363
Tejun Heo0049af72013-05-14 13:52:33 -0700364 if (parent_sq->first_pending)
365 return rb_entry_tg(parent_sq->first_pending);
Vivek Goyale43473b2010-09-15 17:06:35 -0400366
367 return NULL;
368}
369
370static void rb_erase_init(struct rb_node *n, struct rb_root *root)
371{
372 rb_erase(n, root);
373 RB_CLEAR_NODE(n);
374}
375
Tejun Heo0049af72013-05-14 13:52:33 -0700376static void throtl_rb_erase(struct rb_node *n,
377 struct throtl_service_queue *parent_sq)
Vivek Goyale43473b2010-09-15 17:06:35 -0400378{
Tejun Heo0049af72013-05-14 13:52:33 -0700379 if (parent_sq->first_pending == n)
380 parent_sq->first_pending = NULL;
381 rb_erase_init(n, &parent_sq->pending_tree);
382 --parent_sq->nr_pending;
Vivek Goyale43473b2010-09-15 17:06:35 -0400383}
384
Tejun Heo0049af72013-05-14 13:52:33 -0700385static void update_min_dispatch_time(struct throtl_service_queue *parent_sq)
Vivek Goyale43473b2010-09-15 17:06:35 -0400386{
387 struct throtl_grp *tg;
388
Tejun Heo0049af72013-05-14 13:52:33 -0700389 tg = throtl_rb_first(parent_sq);
Vivek Goyale43473b2010-09-15 17:06:35 -0400390 if (!tg)
391 return;
392
Tejun Heo0049af72013-05-14 13:52:33 -0700393 parent_sq->first_pending_disptime = tg->disptime;
Vivek Goyale43473b2010-09-15 17:06:35 -0400394}
395
Tejun Heo77216b02013-05-14 13:52:36 -0700396static void tg_service_queue_add(struct throtl_grp *tg)
Vivek Goyale43473b2010-09-15 17:06:35 -0400397{
Tejun Heo77216b02013-05-14 13:52:36 -0700398 struct throtl_service_queue *parent_sq = tg->service_queue.parent_sq;
Tejun Heo0049af72013-05-14 13:52:33 -0700399 struct rb_node **node = &parent_sq->pending_tree.rb_node;
Vivek Goyale43473b2010-09-15 17:06:35 -0400400 struct rb_node *parent = NULL;
401 struct throtl_grp *__tg;
402 unsigned long key = tg->disptime;
403 int left = 1;
404
405 while (*node != NULL) {
406 parent = *node;
407 __tg = rb_entry_tg(parent);
408
409 if (time_before(key, __tg->disptime))
410 node = &parent->rb_left;
411 else {
412 node = &parent->rb_right;
413 left = 0;
414 }
415 }
416
417 if (left)
Tejun Heo0049af72013-05-14 13:52:33 -0700418 parent_sq->first_pending = &tg->rb_node;
Vivek Goyale43473b2010-09-15 17:06:35 -0400419
420 rb_link_node(&tg->rb_node, parent, node);
Tejun Heo0049af72013-05-14 13:52:33 -0700421 rb_insert_color(&tg->rb_node, &parent_sq->pending_tree);
Vivek Goyale43473b2010-09-15 17:06:35 -0400422}
423
Tejun Heo77216b02013-05-14 13:52:36 -0700424static void __throtl_enqueue_tg(struct throtl_grp *tg)
Vivek Goyale43473b2010-09-15 17:06:35 -0400425{
Tejun Heo77216b02013-05-14 13:52:36 -0700426 tg_service_queue_add(tg);
Tejun Heo5b2c16a2013-05-14 13:52:32 -0700427 tg->flags |= THROTL_TG_PENDING;
Tejun Heo77216b02013-05-14 13:52:36 -0700428 tg->service_queue.parent_sq->nr_pending++;
Vivek Goyale43473b2010-09-15 17:06:35 -0400429}
430
Tejun Heo77216b02013-05-14 13:52:36 -0700431static void throtl_enqueue_tg(struct throtl_grp *tg)
Vivek Goyale43473b2010-09-15 17:06:35 -0400432{
Tejun Heo5b2c16a2013-05-14 13:52:32 -0700433 if (!(tg->flags & THROTL_TG_PENDING))
Tejun Heo77216b02013-05-14 13:52:36 -0700434 __throtl_enqueue_tg(tg);
Vivek Goyale43473b2010-09-15 17:06:35 -0400435}
436
Tejun Heo77216b02013-05-14 13:52:36 -0700437static void __throtl_dequeue_tg(struct throtl_grp *tg)
Vivek Goyale43473b2010-09-15 17:06:35 -0400438{
Tejun Heo77216b02013-05-14 13:52:36 -0700439 throtl_rb_erase(&tg->rb_node, tg->service_queue.parent_sq);
Tejun Heo5b2c16a2013-05-14 13:52:32 -0700440 tg->flags &= ~THROTL_TG_PENDING;
Vivek Goyale43473b2010-09-15 17:06:35 -0400441}
442
Tejun Heo77216b02013-05-14 13:52:36 -0700443static void throtl_dequeue_tg(struct throtl_grp *tg)
Vivek Goyale43473b2010-09-15 17:06:35 -0400444{
Tejun Heo5b2c16a2013-05-14 13:52:32 -0700445 if (tg->flags & THROTL_TG_PENDING)
Tejun Heo77216b02013-05-14 13:52:36 -0700446 __throtl_dequeue_tg(tg);
Vivek Goyale43473b2010-09-15 17:06:35 -0400447}
448
Tejun Heoa9131a22013-05-14 13:52:31 -0700449/* Call with queue lock held */
450static void throtl_schedule_delayed_work(struct throtl_data *td,
451 unsigned long delay)
452{
453 struct delayed_work *dwork = &td->dispatch_work;
Tejun Heofda6f272013-05-14 13:52:36 -0700454 struct throtl_service_queue *sq = &td->service_queue;
Tejun Heoa9131a22013-05-14 13:52:31 -0700455
Tejun Heo6a525602013-05-14 13:52:32 -0700456 mod_delayed_work(kthrotld_workqueue, dwork, delay);
Tejun Heofda6f272013-05-14 13:52:36 -0700457 throtl_log(sq, "schedule work. delay=%lu jiffies=%lu", delay, jiffies);
Tejun Heoa9131a22013-05-14 13:52:31 -0700458}
459
Vivek Goyale43473b2010-09-15 17:06:35 -0400460static void throtl_schedule_next_dispatch(struct throtl_data *td)
461{
Tejun Heoc9e03322013-05-14 13:52:32 -0700462 struct throtl_service_queue *sq = &td->service_queue;
Vivek Goyale43473b2010-09-15 17:06:35 -0400463
Tejun Heo6a525602013-05-14 13:52:32 -0700464 /* any pending children left? */
Tejun Heoc9e03322013-05-14 13:52:32 -0700465 if (!sq->nr_pending)
Vivek Goyale43473b2010-09-15 17:06:35 -0400466 return;
467
Tejun Heoc9e03322013-05-14 13:52:32 -0700468 update_min_dispatch_time(sq);
Vivek Goyale43473b2010-09-15 17:06:35 -0400469
Tejun Heoc9e03322013-05-14 13:52:32 -0700470 if (time_before_eq(sq->first_pending_disptime, jiffies))
Vivek Goyal450adcb2011-03-01 13:40:54 -0500471 throtl_schedule_delayed_work(td, 0);
Vivek Goyale43473b2010-09-15 17:06:35 -0400472 else
Tejun Heoc9e03322013-05-14 13:52:32 -0700473 throtl_schedule_delayed_work(td, sq->first_pending_disptime - jiffies);
Vivek Goyale43473b2010-09-15 17:06:35 -0400474}
475
Tejun Heo0f3457f2013-05-14 13:52:32 -0700476static inline void throtl_start_new_slice(struct throtl_grp *tg, bool rw)
Vivek Goyale43473b2010-09-15 17:06:35 -0400477{
478 tg->bytes_disp[rw] = 0;
Vivek Goyal8e89d132010-09-15 17:06:37 -0400479 tg->io_disp[rw] = 0;
Vivek Goyale43473b2010-09-15 17:06:35 -0400480 tg->slice_start[rw] = jiffies;
481 tg->slice_end[rw] = jiffies + throtl_slice;
Tejun Heofda6f272013-05-14 13:52:36 -0700482 throtl_log(&tg->service_queue,
483 "[%c] new slice start=%lu end=%lu jiffies=%lu",
484 rw == READ ? 'R' : 'W', tg->slice_start[rw],
485 tg->slice_end[rw], jiffies);
Vivek Goyale43473b2010-09-15 17:06:35 -0400486}
487
Tejun Heo0f3457f2013-05-14 13:52:32 -0700488static inline void throtl_set_slice_end(struct throtl_grp *tg, bool rw,
489 unsigned long jiffy_end)
Vivek Goyald1ae8ff2010-12-01 19:34:46 +0100490{
491 tg->slice_end[rw] = roundup(jiffy_end, throtl_slice);
492}
493
Tejun Heo0f3457f2013-05-14 13:52:32 -0700494static inline void throtl_extend_slice(struct throtl_grp *tg, bool rw,
495 unsigned long jiffy_end)
Vivek Goyale43473b2010-09-15 17:06:35 -0400496{
497 tg->slice_end[rw] = roundup(jiffy_end, throtl_slice);
Tejun Heofda6f272013-05-14 13:52:36 -0700498 throtl_log(&tg->service_queue,
499 "[%c] extend slice start=%lu end=%lu jiffies=%lu",
500 rw == READ ? 'R' : 'W', tg->slice_start[rw],
501 tg->slice_end[rw], jiffies);
Vivek Goyale43473b2010-09-15 17:06:35 -0400502}
503
504/* Determine if previously allocated or extended slice is complete or not */
Tejun Heo0f3457f2013-05-14 13:52:32 -0700505static bool throtl_slice_used(struct throtl_grp *tg, bool rw)
Vivek Goyale43473b2010-09-15 17:06:35 -0400506{
507 if (time_in_range(jiffies, tg->slice_start[rw], tg->slice_end[rw]))
508 return 0;
509
510 return 1;
511}
512
513/* Trim the used slices and adjust slice start accordingly */
Tejun Heo0f3457f2013-05-14 13:52:32 -0700514static inline void throtl_trim_slice(struct throtl_grp *tg, bool rw)
Vivek Goyale43473b2010-09-15 17:06:35 -0400515{
Vivek Goyal3aad5d32010-10-01 14:51:14 +0200516 unsigned long nr_slices, time_elapsed, io_trim;
517 u64 bytes_trim, tmp;
Vivek Goyale43473b2010-09-15 17:06:35 -0400518
519 BUG_ON(time_before(tg->slice_end[rw], tg->slice_start[rw]));
520
521 /*
522 * If bps are unlimited (-1), then time slice don't get
523 * renewed. Don't try to trim the slice if slice is used. A new
524 * slice will start when appropriate.
525 */
Tejun Heo0f3457f2013-05-14 13:52:32 -0700526 if (throtl_slice_used(tg, rw))
Vivek Goyale43473b2010-09-15 17:06:35 -0400527 return;
528
Vivek Goyald1ae8ff2010-12-01 19:34:46 +0100529 /*
530 * A bio has been dispatched. Also adjust slice_end. It might happen
531 * that initially cgroup limit was very low resulting in high
532 * slice_end, but later limit was bumped up and bio was dispached
533 * sooner, then we need to reduce slice_end. A high bogus slice_end
534 * is bad because it does not allow new slice to start.
535 */
536
Tejun Heo0f3457f2013-05-14 13:52:32 -0700537 throtl_set_slice_end(tg, rw, jiffies + throtl_slice);
Vivek Goyald1ae8ff2010-12-01 19:34:46 +0100538
Vivek Goyale43473b2010-09-15 17:06:35 -0400539 time_elapsed = jiffies - tg->slice_start[rw];
540
541 nr_slices = time_elapsed / throtl_slice;
542
543 if (!nr_slices)
544 return;
Vivek Goyal3aad5d32010-10-01 14:51:14 +0200545 tmp = tg->bps[rw] * throtl_slice * nr_slices;
546 do_div(tmp, HZ);
547 bytes_trim = tmp;
Vivek Goyale43473b2010-09-15 17:06:35 -0400548
Vivek Goyal8e89d132010-09-15 17:06:37 -0400549 io_trim = (tg->iops[rw] * throtl_slice * nr_slices)/HZ;
Vivek Goyale43473b2010-09-15 17:06:35 -0400550
Vivek Goyal8e89d132010-09-15 17:06:37 -0400551 if (!bytes_trim && !io_trim)
Vivek Goyale43473b2010-09-15 17:06:35 -0400552 return;
553
554 if (tg->bytes_disp[rw] >= bytes_trim)
555 tg->bytes_disp[rw] -= bytes_trim;
556 else
557 tg->bytes_disp[rw] = 0;
558
Vivek Goyal8e89d132010-09-15 17:06:37 -0400559 if (tg->io_disp[rw] >= io_trim)
560 tg->io_disp[rw] -= io_trim;
561 else
562 tg->io_disp[rw] = 0;
563
Vivek Goyale43473b2010-09-15 17:06:35 -0400564 tg->slice_start[rw] += nr_slices * throtl_slice;
565
Tejun Heofda6f272013-05-14 13:52:36 -0700566 throtl_log(&tg->service_queue,
567 "[%c] trim slice nr=%lu bytes=%llu io=%lu start=%lu end=%lu jiffies=%lu",
568 rw == READ ? 'R' : 'W', nr_slices, bytes_trim, io_trim,
569 tg->slice_start[rw], tg->slice_end[rw], jiffies);
Vivek Goyale43473b2010-09-15 17:06:35 -0400570}
571
Tejun Heo0f3457f2013-05-14 13:52:32 -0700572static bool tg_with_in_iops_limit(struct throtl_grp *tg, struct bio *bio,
573 unsigned long *wait)
Vivek Goyale43473b2010-09-15 17:06:35 -0400574{
575 bool rw = bio_data_dir(bio);
Vivek Goyal8e89d132010-09-15 17:06:37 -0400576 unsigned int io_allowed;
Vivek Goyale43473b2010-09-15 17:06:35 -0400577 unsigned long jiffy_elapsed, jiffy_wait, jiffy_elapsed_rnd;
Vivek Goyalc49c06e2010-10-01 21:16:42 +0200578 u64 tmp;
Vivek Goyale43473b2010-09-15 17:06:35 -0400579
Vivek Goyal8e89d132010-09-15 17:06:37 -0400580 jiffy_elapsed = jiffy_elapsed_rnd = jiffies - tg->slice_start[rw];
Vivek Goyale43473b2010-09-15 17:06:35 -0400581
Vivek Goyal8e89d132010-09-15 17:06:37 -0400582 /* Slice has just started. Consider one slice interval */
583 if (!jiffy_elapsed)
584 jiffy_elapsed_rnd = throtl_slice;
585
586 jiffy_elapsed_rnd = roundup(jiffy_elapsed_rnd, throtl_slice);
587
Vivek Goyalc49c06e2010-10-01 21:16:42 +0200588 /*
589 * jiffy_elapsed_rnd should not be a big value as minimum iops can be
590 * 1 then at max jiffy elapsed should be equivalent of 1 second as we
591 * will allow dispatch after 1 second and after that slice should
592 * have been trimmed.
593 */
594
595 tmp = (u64)tg->iops[rw] * jiffy_elapsed_rnd;
596 do_div(tmp, HZ);
597
598 if (tmp > UINT_MAX)
599 io_allowed = UINT_MAX;
600 else
601 io_allowed = tmp;
Vivek Goyal8e89d132010-09-15 17:06:37 -0400602
603 if (tg->io_disp[rw] + 1 <= io_allowed) {
Vivek Goyale43473b2010-09-15 17:06:35 -0400604 if (wait)
605 *wait = 0;
606 return 1;
607 }
608
Vivek Goyal8e89d132010-09-15 17:06:37 -0400609 /* Calc approx time to dispatch */
610 jiffy_wait = ((tg->io_disp[rw] + 1) * HZ)/tg->iops[rw] + 1;
611
612 if (jiffy_wait > jiffy_elapsed)
613 jiffy_wait = jiffy_wait - jiffy_elapsed;
614 else
615 jiffy_wait = 1;
616
617 if (wait)
618 *wait = jiffy_wait;
619 return 0;
620}
621
Tejun Heo0f3457f2013-05-14 13:52:32 -0700622static bool tg_with_in_bps_limit(struct throtl_grp *tg, struct bio *bio,
623 unsigned long *wait)
Vivek Goyal8e89d132010-09-15 17:06:37 -0400624{
625 bool rw = bio_data_dir(bio);
Vivek Goyal3aad5d32010-10-01 14:51:14 +0200626 u64 bytes_allowed, extra_bytes, tmp;
Vivek Goyal8e89d132010-09-15 17:06:37 -0400627 unsigned long jiffy_elapsed, jiffy_wait, jiffy_elapsed_rnd;
Vivek Goyale43473b2010-09-15 17:06:35 -0400628
629 jiffy_elapsed = jiffy_elapsed_rnd = jiffies - tg->slice_start[rw];
630
631 /* Slice has just started. Consider one slice interval */
632 if (!jiffy_elapsed)
633 jiffy_elapsed_rnd = throtl_slice;
634
635 jiffy_elapsed_rnd = roundup(jiffy_elapsed_rnd, throtl_slice);
636
Vivek Goyal5e901a22010-10-01 21:16:38 +0200637 tmp = tg->bps[rw] * jiffy_elapsed_rnd;
638 do_div(tmp, HZ);
Vivek Goyal3aad5d32010-10-01 14:51:14 +0200639 bytes_allowed = tmp;
Vivek Goyale43473b2010-09-15 17:06:35 -0400640
641 if (tg->bytes_disp[rw] + bio->bi_size <= bytes_allowed) {
642 if (wait)
643 *wait = 0;
644 return 1;
645 }
646
647 /* Calc approx time to dispatch */
648 extra_bytes = tg->bytes_disp[rw] + bio->bi_size - bytes_allowed;
649 jiffy_wait = div64_u64(extra_bytes * HZ, tg->bps[rw]);
650
651 if (!jiffy_wait)
652 jiffy_wait = 1;
653
654 /*
655 * This wait time is without taking into consideration the rounding
656 * up we did. Add that time also.
657 */
658 jiffy_wait = jiffy_wait + (jiffy_elapsed_rnd - jiffy_elapsed);
Vivek Goyale43473b2010-09-15 17:06:35 -0400659 if (wait)
660 *wait = jiffy_wait;
Vivek Goyal8e89d132010-09-15 17:06:37 -0400661 return 0;
662}
Vivek Goyale43473b2010-09-15 17:06:35 -0400663
Vivek Goyalaf75cd32011-05-19 15:38:31 -0400664static bool tg_no_rule_group(struct throtl_grp *tg, bool rw) {
665 if (tg->bps[rw] == -1 && tg->iops[rw] == -1)
666 return 1;
667 return 0;
668}
669
Vivek Goyal8e89d132010-09-15 17:06:37 -0400670/*
671 * Returns whether one can dispatch a bio or not. Also returns approx number
672 * of jiffies to wait before this bio is with-in IO rate and can be dispatched
673 */
Tejun Heo0f3457f2013-05-14 13:52:32 -0700674static bool tg_may_dispatch(struct throtl_grp *tg, struct bio *bio,
675 unsigned long *wait)
Vivek Goyal8e89d132010-09-15 17:06:37 -0400676{
677 bool rw = bio_data_dir(bio);
678 unsigned long bps_wait = 0, iops_wait = 0, max_wait = 0;
679
680 /*
681 * Currently whole state machine of group depends on first bio
682 * queued in the group bio list. So one should not be calling
683 * this function with a different bio if there are other bios
684 * queued.
685 */
Tejun Heo73f0d492013-05-14 13:52:35 -0700686 BUG_ON(tg->service_queue.nr_queued[rw] &&
687 bio != bio_list_peek(&tg->service_queue.bio_lists[rw]));
Vivek Goyal8e89d132010-09-15 17:06:37 -0400688
689 /* If tg->bps = -1, then BW is unlimited */
690 if (tg->bps[rw] == -1 && tg->iops[rw] == -1) {
691 if (wait)
692 *wait = 0;
693 return 1;
694 }
695
696 /*
697 * If previous slice expired, start a new one otherwise renew/extend
698 * existing slice to make sure it is at least throtl_slice interval
699 * long since now.
700 */
Tejun Heo0f3457f2013-05-14 13:52:32 -0700701 if (throtl_slice_used(tg, rw))
702 throtl_start_new_slice(tg, rw);
Vivek Goyal8e89d132010-09-15 17:06:37 -0400703 else {
704 if (time_before(tg->slice_end[rw], jiffies + throtl_slice))
Tejun Heo0f3457f2013-05-14 13:52:32 -0700705 throtl_extend_slice(tg, rw, jiffies + throtl_slice);
Vivek Goyal8e89d132010-09-15 17:06:37 -0400706 }
707
Tejun Heo0f3457f2013-05-14 13:52:32 -0700708 if (tg_with_in_bps_limit(tg, bio, &bps_wait) &&
709 tg_with_in_iops_limit(tg, bio, &iops_wait)) {
Vivek Goyal8e89d132010-09-15 17:06:37 -0400710 if (wait)
711 *wait = 0;
712 return 1;
713 }
714
715 max_wait = max(bps_wait, iops_wait);
716
717 if (wait)
718 *wait = max_wait;
719
720 if (time_before(tg->slice_end[rw], jiffies + max_wait))
Tejun Heo0f3457f2013-05-14 13:52:32 -0700721 throtl_extend_slice(tg, rw, jiffies + max_wait);
Vivek Goyale43473b2010-09-15 17:06:35 -0400722
723 return 0;
724}
725
Tejun Heo3c798392012-04-16 13:57:25 -0700726static void throtl_update_dispatch_stats(struct blkcg_gq *blkg, u64 bytes,
Tejun Heo629ed0b2012-04-01 14:38:44 -0700727 int rw)
728{
Tejun Heo8a3d2612012-04-01 14:38:44 -0700729 struct throtl_grp *tg = blkg_to_tg(blkg);
730 struct tg_stats_cpu *stats_cpu;
Tejun Heo629ed0b2012-04-01 14:38:44 -0700731 unsigned long flags;
732
733 /* If per cpu stats are not allocated yet, don't do any accounting. */
Tejun Heo8a3d2612012-04-01 14:38:44 -0700734 if (tg->stats_cpu == NULL)
Tejun Heo629ed0b2012-04-01 14:38:44 -0700735 return;
736
737 /*
738 * Disabling interrupts to provide mutual exclusion between two
739 * writes on same cpu. It probably is not needed for 64bit. Not
740 * optimizing that case yet.
741 */
742 local_irq_save(flags);
743
Tejun Heo8a3d2612012-04-01 14:38:44 -0700744 stats_cpu = this_cpu_ptr(tg->stats_cpu);
Tejun Heo629ed0b2012-04-01 14:38:44 -0700745
Tejun Heo629ed0b2012-04-01 14:38:44 -0700746 blkg_rwstat_add(&stats_cpu->serviced, rw, 1);
747 blkg_rwstat_add(&stats_cpu->service_bytes, rw, bytes);
748
749 local_irq_restore(flags);
750}
751
Vivek Goyale43473b2010-09-15 17:06:35 -0400752static void throtl_charge_bio(struct throtl_grp *tg, struct bio *bio)
753{
754 bool rw = bio_data_dir(bio);
Vivek Goyale43473b2010-09-15 17:06:35 -0400755
756 /* Charge the bio to the group */
757 tg->bytes_disp[rw] += bio->bi_size;
Vivek Goyal8e89d132010-09-15 17:06:37 -0400758 tg->io_disp[rw]++;
Vivek Goyale43473b2010-09-15 17:06:35 -0400759
Tejun Heo2a0f61e2013-05-14 13:52:36 -0700760 /*
761 * REQ_THROTTLED is used to prevent the same bio to be throttled
762 * more than once as a throttled bio will go through blk-throtl the
763 * second time when it eventually gets issued. Set it when a bio
764 * is being charged to a tg.
765 *
766 * Dispatch stats aren't recursive and each @bio should only be
767 * accounted by the @tg it was originally associated with. Let's
768 * update the stats when setting REQ_THROTTLED for the first time
769 * which is guaranteed to be for the @bio's original tg.
770 */
771 if (!(bio->bi_rw & REQ_THROTTLED)) {
772 bio->bi_rw |= REQ_THROTTLED;
773 throtl_update_dispatch_stats(tg_to_blkg(tg), bio->bi_size,
774 bio->bi_rw);
775 }
Vivek Goyale43473b2010-09-15 17:06:35 -0400776}
777
Tejun Heo77216b02013-05-14 13:52:36 -0700778static void throtl_add_bio_tg(struct bio *bio, struct throtl_grp *tg)
Vivek Goyale43473b2010-09-15 17:06:35 -0400779{
Tejun Heo73f0d492013-05-14 13:52:35 -0700780 struct throtl_service_queue *sq = &tg->service_queue;
Vivek Goyale43473b2010-09-15 17:06:35 -0400781 bool rw = bio_data_dir(bio);
782
Tejun Heo0e9f4162013-05-14 13:52:35 -0700783 /*
784 * If @tg doesn't currently have any bios queued in the same
785 * direction, queueing @bio can change when @tg should be
786 * dispatched. Mark that @tg was empty. This is automatically
787 * cleaered on the next tg_update_disptime().
788 */
789 if (!sq->nr_queued[rw])
790 tg->flags |= THROTL_TG_WAS_EMPTY;
791
Tejun Heo73f0d492013-05-14 13:52:35 -0700792 bio_list_add(&sq->bio_lists[rw], bio);
Vivek Goyale43473b2010-09-15 17:06:35 -0400793 /* Take a bio reference on tg */
Tejun Heo1adaf3d2012-03-05 13:15:15 -0800794 blkg_get(tg_to_blkg(tg));
Tejun Heo73f0d492013-05-14 13:52:35 -0700795 sq->nr_queued[rw]++;
Tejun Heoe2d57e62013-05-14 13:52:33 -0700796 tg->td->nr_queued[rw]++;
Tejun Heo77216b02013-05-14 13:52:36 -0700797 throtl_enqueue_tg(tg);
Vivek Goyale43473b2010-09-15 17:06:35 -0400798}
799
Tejun Heo77216b02013-05-14 13:52:36 -0700800static void tg_update_disptime(struct throtl_grp *tg)
Vivek Goyale43473b2010-09-15 17:06:35 -0400801{
Tejun Heo73f0d492013-05-14 13:52:35 -0700802 struct throtl_service_queue *sq = &tg->service_queue;
Vivek Goyale43473b2010-09-15 17:06:35 -0400803 unsigned long read_wait = -1, write_wait = -1, min_wait = -1, disptime;
804 struct bio *bio;
805
Tejun Heo73f0d492013-05-14 13:52:35 -0700806 if ((bio = bio_list_peek(&sq->bio_lists[READ])))
Tejun Heo0f3457f2013-05-14 13:52:32 -0700807 tg_may_dispatch(tg, bio, &read_wait);
Vivek Goyale43473b2010-09-15 17:06:35 -0400808
Tejun Heo73f0d492013-05-14 13:52:35 -0700809 if ((bio = bio_list_peek(&sq->bio_lists[WRITE])))
Tejun Heo0f3457f2013-05-14 13:52:32 -0700810 tg_may_dispatch(tg, bio, &write_wait);
Vivek Goyale43473b2010-09-15 17:06:35 -0400811
812 min_wait = min(read_wait, write_wait);
813 disptime = jiffies + min_wait;
814
Vivek Goyale43473b2010-09-15 17:06:35 -0400815 /* Update dispatch time */
Tejun Heo77216b02013-05-14 13:52:36 -0700816 throtl_dequeue_tg(tg);
Vivek Goyale43473b2010-09-15 17:06:35 -0400817 tg->disptime = disptime;
Tejun Heo77216b02013-05-14 13:52:36 -0700818 throtl_enqueue_tg(tg);
Tejun Heo0e9f4162013-05-14 13:52:35 -0700819
820 /* see throtl_add_bio_tg() */
821 tg->flags &= ~THROTL_TG_WAS_EMPTY;
Vivek Goyale43473b2010-09-15 17:06:35 -0400822}
823
Tejun Heo77216b02013-05-14 13:52:36 -0700824static void tg_dispatch_one_bio(struct throtl_grp *tg, bool rw)
Vivek Goyale43473b2010-09-15 17:06:35 -0400825{
Tejun Heo73f0d492013-05-14 13:52:35 -0700826 struct throtl_service_queue *sq = &tg->service_queue;
Vivek Goyale43473b2010-09-15 17:06:35 -0400827 struct bio *bio;
828
Tejun Heo73f0d492013-05-14 13:52:35 -0700829 bio = bio_list_pop(&sq->bio_lists[rw]);
830 sq->nr_queued[rw]--;
Tejun Heo1adaf3d2012-03-05 13:15:15 -0800831 /* Drop bio reference on blkg */
832 blkg_put(tg_to_blkg(tg));
Vivek Goyale43473b2010-09-15 17:06:35 -0400833
Tejun Heo0f3457f2013-05-14 13:52:32 -0700834 BUG_ON(tg->td->nr_queued[rw] <= 0);
835 tg->td->nr_queued[rw]--;
Vivek Goyale43473b2010-09-15 17:06:35 -0400836
837 throtl_charge_bio(tg, bio);
Tejun Heo77216b02013-05-14 13:52:36 -0700838 bio_list_add(&sq->parent_sq->bio_lists[rw], bio);
Vivek Goyale43473b2010-09-15 17:06:35 -0400839
Tejun Heo0f3457f2013-05-14 13:52:32 -0700840 throtl_trim_slice(tg, rw);
Vivek Goyale43473b2010-09-15 17:06:35 -0400841}
842
Tejun Heo77216b02013-05-14 13:52:36 -0700843static int throtl_dispatch_tg(struct throtl_grp *tg)
Vivek Goyale43473b2010-09-15 17:06:35 -0400844{
Tejun Heo73f0d492013-05-14 13:52:35 -0700845 struct throtl_service_queue *sq = &tg->service_queue;
Vivek Goyale43473b2010-09-15 17:06:35 -0400846 unsigned int nr_reads = 0, nr_writes = 0;
847 unsigned int max_nr_reads = throtl_grp_quantum*3/4;
Vivek Goyalc2f68052010-11-15 19:32:42 +0100848 unsigned int max_nr_writes = throtl_grp_quantum - max_nr_reads;
Vivek Goyale43473b2010-09-15 17:06:35 -0400849 struct bio *bio;
850
851 /* Try to dispatch 75% READS and 25% WRITES */
852
Tejun Heo73f0d492013-05-14 13:52:35 -0700853 while ((bio = bio_list_peek(&sq->bio_lists[READ])) &&
Tejun Heo0f3457f2013-05-14 13:52:32 -0700854 tg_may_dispatch(tg, bio, NULL)) {
Vivek Goyale43473b2010-09-15 17:06:35 -0400855
Tejun Heo77216b02013-05-14 13:52:36 -0700856 tg_dispatch_one_bio(tg, bio_data_dir(bio));
Vivek Goyale43473b2010-09-15 17:06:35 -0400857 nr_reads++;
858
859 if (nr_reads >= max_nr_reads)
860 break;
861 }
862
Tejun Heo73f0d492013-05-14 13:52:35 -0700863 while ((bio = bio_list_peek(&sq->bio_lists[WRITE])) &&
Tejun Heo0f3457f2013-05-14 13:52:32 -0700864 tg_may_dispatch(tg, bio, NULL)) {
Vivek Goyale43473b2010-09-15 17:06:35 -0400865
Tejun Heo77216b02013-05-14 13:52:36 -0700866 tg_dispatch_one_bio(tg, bio_data_dir(bio));
Vivek Goyale43473b2010-09-15 17:06:35 -0400867 nr_writes++;
868
869 if (nr_writes >= max_nr_writes)
870 break;
871 }
872
873 return nr_reads + nr_writes;
874}
875
Tejun Heo651930b2013-05-14 13:52:35 -0700876static int throtl_select_dispatch(struct throtl_service_queue *parent_sq)
Vivek Goyale43473b2010-09-15 17:06:35 -0400877{
878 unsigned int nr_disp = 0;
Vivek Goyale43473b2010-09-15 17:06:35 -0400879
880 while (1) {
Tejun Heo73f0d492013-05-14 13:52:35 -0700881 struct throtl_grp *tg = throtl_rb_first(parent_sq);
882 struct throtl_service_queue *sq = &tg->service_queue;
Vivek Goyale43473b2010-09-15 17:06:35 -0400883
884 if (!tg)
885 break;
886
887 if (time_before(jiffies, tg->disptime))
888 break;
889
Tejun Heo77216b02013-05-14 13:52:36 -0700890 throtl_dequeue_tg(tg);
Vivek Goyale43473b2010-09-15 17:06:35 -0400891
Tejun Heo77216b02013-05-14 13:52:36 -0700892 nr_disp += throtl_dispatch_tg(tg);
Vivek Goyale43473b2010-09-15 17:06:35 -0400893
Tejun Heo73f0d492013-05-14 13:52:35 -0700894 if (sq->nr_queued[0] || sq->nr_queued[1])
Tejun Heo77216b02013-05-14 13:52:36 -0700895 tg_update_disptime(tg);
Vivek Goyale43473b2010-09-15 17:06:35 -0400896
897 if (nr_disp >= throtl_quantum)
898 break;
899 }
900
901 return nr_disp;
902}
903
Tejun Heocb761992013-05-14 13:52:31 -0700904/* work function to dispatch throttled bios */
905void blk_throtl_dispatch_work_fn(struct work_struct *work)
Vivek Goyale43473b2010-09-15 17:06:35 -0400906{
Tejun Heocb761992013-05-14 13:52:31 -0700907 struct throtl_data *td = container_of(to_delayed_work(work),
908 struct throtl_data, dispatch_work);
Tejun Heo651930b2013-05-14 13:52:35 -0700909 struct throtl_service_queue *sq = &td->service_queue;
Tejun Heocb761992013-05-14 13:52:31 -0700910 struct request_queue *q = td->queue;
Vivek Goyale43473b2010-09-15 17:06:35 -0400911 unsigned int nr_disp = 0;
912 struct bio_list bio_list_on_stack;
913 struct bio *bio;
Vivek Goyal69d60eb2011-03-09 08:27:37 +0100914 struct blk_plug plug;
Tejun Heo651930b2013-05-14 13:52:35 -0700915 int rw;
Vivek Goyale43473b2010-09-15 17:06:35 -0400916
917 spin_lock_irq(q->queue_lock);
918
Vivek Goyale43473b2010-09-15 17:06:35 -0400919 bio_list_init(&bio_list_on_stack);
920
Tejun Heofda6f272013-05-14 13:52:36 -0700921 throtl_log(sq, "dispatch nr_queued=%u read=%u write=%u",
Tejun Heo6a525602013-05-14 13:52:32 -0700922 td->nr_queued[READ] + td->nr_queued[WRITE],
923 td->nr_queued[READ], td->nr_queued[WRITE]);
Vivek Goyale43473b2010-09-15 17:06:35 -0400924
Tejun Heo651930b2013-05-14 13:52:35 -0700925 nr_disp = throtl_select_dispatch(sq);
Vivek Goyale43473b2010-09-15 17:06:35 -0400926
Tejun Heo651930b2013-05-14 13:52:35 -0700927 if (nr_disp) {
928 for (rw = READ; rw <= WRITE; rw++) {
929 bio_list_merge(&bio_list_on_stack, &sq->bio_lists[rw]);
930 bio_list_init(&sq->bio_lists[rw]);
931 }
Tejun Heofda6f272013-05-14 13:52:36 -0700932 throtl_log(sq, "bios disp=%u", nr_disp);
Tejun Heo651930b2013-05-14 13:52:35 -0700933 }
Vivek Goyale43473b2010-09-15 17:06:35 -0400934
935 throtl_schedule_next_dispatch(td);
Tejun Heo6a525602013-05-14 13:52:32 -0700936
Vivek Goyale43473b2010-09-15 17:06:35 -0400937 spin_unlock_irq(q->queue_lock);
938
939 /*
940 * If we dispatched some requests, unplug the queue to make sure
941 * immediate dispatch
942 */
943 if (nr_disp) {
Vivek Goyal69d60eb2011-03-09 08:27:37 +0100944 blk_start_plug(&plug);
Vivek Goyale43473b2010-09-15 17:06:35 -0400945 while((bio = bio_list_pop(&bio_list_on_stack)))
946 generic_make_request(bio);
Vivek Goyal69d60eb2011-03-09 08:27:37 +0100947 blk_finish_plug(&plug);
Vivek Goyale43473b2010-09-15 17:06:35 -0400948 }
Vivek Goyale43473b2010-09-15 17:06:35 -0400949}
950
Tejun Heof95a04a2012-04-16 13:57:26 -0700951static u64 tg_prfill_cpu_rwstat(struct seq_file *sf,
952 struct blkg_policy_data *pd, int off)
Tejun Heo41b38b62012-04-01 14:38:44 -0700953{
Tejun Heof95a04a2012-04-16 13:57:26 -0700954 struct throtl_grp *tg = pd_to_tg(pd);
Tejun Heo41b38b62012-04-01 14:38:44 -0700955 struct blkg_rwstat rwstat = { }, tmp;
956 int i, cpu;
957
958 for_each_possible_cpu(cpu) {
Tejun Heo8a3d2612012-04-01 14:38:44 -0700959 struct tg_stats_cpu *sc = per_cpu_ptr(tg->stats_cpu, cpu);
Tejun Heo41b38b62012-04-01 14:38:44 -0700960
961 tmp = blkg_rwstat_read((void *)sc + off);
962 for (i = 0; i < BLKG_RWSTAT_NR; i++)
963 rwstat.cnt[i] += tmp.cnt[i];
964 }
965
Tejun Heof95a04a2012-04-16 13:57:26 -0700966 return __blkg_prfill_rwstat(sf, pd, &rwstat);
Tejun Heo41b38b62012-04-01 14:38:44 -0700967}
968
Tejun Heo8a3d2612012-04-01 14:38:44 -0700969static int tg_print_cpu_rwstat(struct cgroup *cgrp, struct cftype *cft,
970 struct seq_file *sf)
Tejun Heo41b38b62012-04-01 14:38:44 -0700971{
Tejun Heo3c798392012-04-16 13:57:25 -0700972 struct blkcg *blkcg = cgroup_to_blkcg(cgrp);
Tejun Heo41b38b62012-04-01 14:38:44 -0700973
Tejun Heo3c798392012-04-16 13:57:25 -0700974 blkcg_print_blkgs(sf, blkcg, tg_prfill_cpu_rwstat, &blkcg_policy_throtl,
Tejun Heo5bc4afb12012-04-01 14:38:45 -0700975 cft->private, true);
Tejun Heo41b38b62012-04-01 14:38:44 -0700976 return 0;
977}
978
Tejun Heof95a04a2012-04-16 13:57:26 -0700979static u64 tg_prfill_conf_u64(struct seq_file *sf, struct blkg_policy_data *pd,
980 int off)
Tejun Heo60c2bc22012-04-01 14:38:43 -0700981{
Tejun Heof95a04a2012-04-16 13:57:26 -0700982 struct throtl_grp *tg = pd_to_tg(pd);
983 u64 v = *(u64 *)((void *)tg + off);
Tejun Heo60c2bc22012-04-01 14:38:43 -0700984
Tejun Heoaf133ce2012-04-01 14:38:44 -0700985 if (v == -1)
Tejun Heo60c2bc22012-04-01 14:38:43 -0700986 return 0;
Tejun Heof95a04a2012-04-16 13:57:26 -0700987 return __blkg_prfill_u64(sf, pd, v);
Tejun Heo60c2bc22012-04-01 14:38:43 -0700988}
989
Tejun Heof95a04a2012-04-16 13:57:26 -0700990static u64 tg_prfill_conf_uint(struct seq_file *sf, struct blkg_policy_data *pd,
991 int off)
Tejun Heo60c2bc22012-04-01 14:38:43 -0700992{
Tejun Heof95a04a2012-04-16 13:57:26 -0700993 struct throtl_grp *tg = pd_to_tg(pd);
994 unsigned int v = *(unsigned int *)((void *)tg + off);
Tejun Heoaf133ce2012-04-01 14:38:44 -0700995
996 if (v == -1)
997 return 0;
Tejun Heof95a04a2012-04-16 13:57:26 -0700998 return __blkg_prfill_u64(sf, pd, v);
Tejun Heoaf133ce2012-04-01 14:38:44 -0700999}
1000
1001static int tg_print_conf_u64(struct cgroup *cgrp, struct cftype *cft,
1002 struct seq_file *sf)
1003{
Tejun Heo3c798392012-04-16 13:57:25 -07001004 blkcg_print_blkgs(sf, cgroup_to_blkcg(cgrp), tg_prfill_conf_u64,
1005 &blkcg_policy_throtl, cft->private, false);
Tejun Heo60c2bc22012-04-01 14:38:43 -07001006 return 0;
1007}
1008
Tejun Heoaf133ce2012-04-01 14:38:44 -07001009static int tg_print_conf_uint(struct cgroup *cgrp, struct cftype *cft,
1010 struct seq_file *sf)
Vivek Goyale43473b2010-09-15 17:06:35 -04001011{
Tejun Heo3c798392012-04-16 13:57:25 -07001012 blkcg_print_blkgs(sf, cgroup_to_blkcg(cgrp), tg_prfill_conf_uint,
1013 &blkcg_policy_throtl, cft->private, false);
Tejun Heoaf133ce2012-04-01 14:38:44 -07001014 return 0;
Vivek Goyale43473b2010-09-15 17:06:35 -04001015}
1016
Tejun Heoaf133ce2012-04-01 14:38:44 -07001017static int tg_set_conf(struct cgroup *cgrp, struct cftype *cft, const char *buf,
1018 bool is_u64)
Tejun Heo60c2bc22012-04-01 14:38:43 -07001019{
Tejun Heo3c798392012-04-16 13:57:25 -07001020 struct blkcg *blkcg = cgroup_to_blkcg(cgrp);
Tejun Heo60c2bc22012-04-01 14:38:43 -07001021 struct blkg_conf_ctx ctx;
Tejun Heoaf133ce2012-04-01 14:38:44 -07001022 struct throtl_grp *tg;
Tejun Heoa2b16932012-04-13 13:11:33 -07001023 struct throtl_data *td;
Tejun Heo60c2bc22012-04-01 14:38:43 -07001024 int ret;
1025
Tejun Heo3c798392012-04-16 13:57:25 -07001026 ret = blkg_conf_prep(blkcg, &blkcg_policy_throtl, buf, &ctx);
Tejun Heo60c2bc22012-04-01 14:38:43 -07001027 if (ret)
1028 return ret;
1029
Tejun Heoaf133ce2012-04-01 14:38:44 -07001030 tg = blkg_to_tg(ctx.blkg);
Tejun Heoa2b16932012-04-13 13:11:33 -07001031 td = ctx.blkg->q->td;
Tejun Heoaf133ce2012-04-01 14:38:44 -07001032
Tejun Heoa2b16932012-04-13 13:11:33 -07001033 if (!ctx.v)
1034 ctx.v = -1;
Tejun Heoaf133ce2012-04-01 14:38:44 -07001035
Tejun Heoa2b16932012-04-13 13:11:33 -07001036 if (is_u64)
1037 *(u64 *)((void *)tg + cft->private) = ctx.v;
1038 else
1039 *(unsigned int *)((void *)tg + cft->private) = ctx.v;
Tejun Heoaf133ce2012-04-01 14:38:44 -07001040
Tejun Heofda6f272013-05-14 13:52:36 -07001041 throtl_log(&tg->service_queue,
1042 "limit change rbps=%llu wbps=%llu riops=%u wiops=%u",
1043 tg->bps[READ], tg->bps[WRITE],
1044 tg->iops[READ], tg->iops[WRITE]);
Tejun Heo632b4492013-05-14 13:52:31 -07001045
1046 /*
1047 * We're already holding queue_lock and know @tg is valid. Let's
1048 * apply the new config directly.
1049 *
1050 * Restart the slices for both READ and WRITES. It might happen
1051 * that a group's limit are dropped suddenly and we don't want to
1052 * account recently dispatched IO with new low rate.
1053 */
Tejun Heo0f3457f2013-05-14 13:52:32 -07001054 throtl_start_new_slice(tg, 0);
1055 throtl_start_new_slice(tg, 1);
Tejun Heo632b4492013-05-14 13:52:31 -07001056
Tejun Heo5b2c16a2013-05-14 13:52:32 -07001057 if (tg->flags & THROTL_TG_PENDING) {
Tejun Heo77216b02013-05-14 13:52:36 -07001058 tg_update_disptime(tg);
Tejun Heo632b4492013-05-14 13:52:31 -07001059 throtl_schedule_next_dispatch(td);
1060 }
Tejun Heo60c2bc22012-04-01 14:38:43 -07001061
1062 blkg_conf_finish(&ctx);
Tejun Heoa2b16932012-04-13 13:11:33 -07001063 return 0;
Tejun Heo60c2bc22012-04-01 14:38:43 -07001064}
1065
Tejun Heoaf133ce2012-04-01 14:38:44 -07001066static int tg_set_conf_u64(struct cgroup *cgrp, struct cftype *cft,
1067 const char *buf)
Tejun Heo60c2bc22012-04-01 14:38:43 -07001068{
Tejun Heoaf133ce2012-04-01 14:38:44 -07001069 return tg_set_conf(cgrp, cft, buf, true);
Tejun Heo60c2bc22012-04-01 14:38:43 -07001070}
1071
Tejun Heoaf133ce2012-04-01 14:38:44 -07001072static int tg_set_conf_uint(struct cgroup *cgrp, struct cftype *cft,
1073 const char *buf)
Tejun Heo60c2bc22012-04-01 14:38:43 -07001074{
Tejun Heoaf133ce2012-04-01 14:38:44 -07001075 return tg_set_conf(cgrp, cft, buf, false);
Tejun Heo60c2bc22012-04-01 14:38:43 -07001076}
1077
1078static struct cftype throtl_files[] = {
1079 {
1080 .name = "throttle.read_bps_device",
Tejun Heoaf133ce2012-04-01 14:38:44 -07001081 .private = offsetof(struct throtl_grp, bps[READ]),
1082 .read_seq_string = tg_print_conf_u64,
1083 .write_string = tg_set_conf_u64,
Tejun Heo60c2bc22012-04-01 14:38:43 -07001084 .max_write_len = 256,
1085 },
1086 {
1087 .name = "throttle.write_bps_device",
Tejun Heoaf133ce2012-04-01 14:38:44 -07001088 .private = offsetof(struct throtl_grp, bps[WRITE]),
1089 .read_seq_string = tg_print_conf_u64,
1090 .write_string = tg_set_conf_u64,
Tejun Heo60c2bc22012-04-01 14:38:43 -07001091 .max_write_len = 256,
1092 },
1093 {
1094 .name = "throttle.read_iops_device",
Tejun Heoaf133ce2012-04-01 14:38:44 -07001095 .private = offsetof(struct throtl_grp, iops[READ]),
1096 .read_seq_string = tg_print_conf_uint,
1097 .write_string = tg_set_conf_uint,
Tejun Heo60c2bc22012-04-01 14:38:43 -07001098 .max_write_len = 256,
1099 },
1100 {
1101 .name = "throttle.write_iops_device",
Tejun Heoaf133ce2012-04-01 14:38:44 -07001102 .private = offsetof(struct throtl_grp, iops[WRITE]),
1103 .read_seq_string = tg_print_conf_uint,
1104 .write_string = tg_set_conf_uint,
Tejun Heo60c2bc22012-04-01 14:38:43 -07001105 .max_write_len = 256,
1106 },
1107 {
1108 .name = "throttle.io_service_bytes",
Tejun Heo5bc4afb12012-04-01 14:38:45 -07001109 .private = offsetof(struct tg_stats_cpu, service_bytes),
Tejun Heo8a3d2612012-04-01 14:38:44 -07001110 .read_seq_string = tg_print_cpu_rwstat,
Tejun Heo60c2bc22012-04-01 14:38:43 -07001111 },
1112 {
1113 .name = "throttle.io_serviced",
Tejun Heo5bc4afb12012-04-01 14:38:45 -07001114 .private = offsetof(struct tg_stats_cpu, serviced),
Tejun Heo8a3d2612012-04-01 14:38:44 -07001115 .read_seq_string = tg_print_cpu_rwstat,
Tejun Heo60c2bc22012-04-01 14:38:43 -07001116 },
1117 { } /* terminate */
1118};
1119
Vivek Goyalda527772011-03-02 19:05:33 -05001120static void throtl_shutdown_wq(struct request_queue *q)
Vivek Goyale43473b2010-09-15 17:06:35 -04001121{
1122 struct throtl_data *td = q->td;
1123
Tejun Heocb761992013-05-14 13:52:31 -07001124 cancel_delayed_work_sync(&td->dispatch_work);
Vivek Goyale43473b2010-09-15 17:06:35 -04001125}
1126
Tejun Heo3c798392012-04-16 13:57:25 -07001127static struct blkcg_policy blkcg_policy_throtl = {
Tejun Heof9fcc2d2012-04-16 13:57:27 -07001128 .pd_size = sizeof(struct throtl_grp),
1129 .cftypes = throtl_files,
1130
1131 .pd_init_fn = throtl_pd_init,
1132 .pd_exit_fn = throtl_pd_exit,
1133 .pd_reset_stats_fn = throtl_pd_reset_stats,
Vivek Goyale43473b2010-09-15 17:06:35 -04001134};
1135
Tejun Heobc16a4f2011-10-19 14:33:01 +02001136bool blk_throtl_bio(struct request_queue *q, struct bio *bio)
Vivek Goyale43473b2010-09-15 17:06:35 -04001137{
1138 struct throtl_data *td = q->td;
1139 struct throtl_grp *tg;
Tejun Heo73f0d492013-05-14 13:52:35 -07001140 struct throtl_service_queue *sq;
Tejun Heo0e9f4162013-05-14 13:52:35 -07001141 bool rw = bio_data_dir(bio);
Tejun Heo3c798392012-04-16 13:57:25 -07001142 struct blkcg *blkcg;
Tejun Heobc16a4f2011-10-19 14:33:01 +02001143 bool throttled = false;
Vivek Goyale43473b2010-09-15 17:06:35 -04001144
Tejun Heo2a0f61e2013-05-14 13:52:36 -07001145 /* see throtl_charge_bio() */
1146 if (bio->bi_rw & REQ_THROTTLED)
Tejun Heobc16a4f2011-10-19 14:33:01 +02001147 goto out;
Vivek Goyale43473b2010-09-15 17:06:35 -04001148
Vivek Goyalaf75cd32011-05-19 15:38:31 -04001149 /*
1150 * A throtl_grp pointer retrieved under rcu can be used to access
1151 * basic fields like stats and io rates. If a group has no rules,
1152 * just update the dispatch stats in lockless manner and return.
1153 */
Vivek Goyalaf75cd32011-05-19 15:38:31 -04001154 rcu_read_lock();
Tejun Heo3c798392012-04-16 13:57:25 -07001155 blkcg = bio_blkcg(bio);
Tejun Heocd1604f2012-03-05 13:15:06 -08001156 tg = throtl_lookup_tg(td, blkcg);
Vivek Goyalaf75cd32011-05-19 15:38:31 -04001157 if (tg) {
Vivek Goyalaf75cd32011-05-19 15:38:31 -04001158 if (tg_no_rule_group(tg, rw)) {
Tejun Heo629ed0b2012-04-01 14:38:44 -07001159 throtl_update_dispatch_stats(tg_to_blkg(tg),
1160 bio->bi_size, bio->bi_rw);
Tejun Heo2a7f1242012-03-05 13:15:01 -08001161 goto out_unlock_rcu;
Vivek Goyalaf75cd32011-05-19 15:38:31 -04001162 }
1163 }
Vivek Goyalaf75cd32011-05-19 15:38:31 -04001164
1165 /*
1166 * Either group has not been allocated yet or it is not an unlimited
1167 * IO group
1168 */
Vivek Goyale43473b2010-09-15 17:06:35 -04001169 spin_lock_irq(q->queue_lock);
Tejun Heocd1604f2012-03-05 13:15:06 -08001170 tg = throtl_lookup_create_tg(td, blkcg);
Tejun Heobc16a4f2011-10-19 14:33:01 +02001171 if (unlikely(!tg))
1172 goto out_unlock;
Vivek Goyalf469a7b2011-05-19 15:38:23 -04001173
Tejun Heo73f0d492013-05-14 13:52:35 -07001174 sq = &tg->service_queue;
1175
Tejun Heo0e9f4162013-05-14 13:52:35 -07001176 /* throtl is FIFO - if other bios are already queued, should queue */
1177 if (sq->nr_queued[rw])
Vivek Goyale43473b2010-09-15 17:06:35 -04001178 goto queue_bio;
Vivek Goyalde701c72011-03-07 21:09:32 +01001179
Vivek Goyale43473b2010-09-15 17:06:35 -04001180 /* Bio is with-in rate limit of group */
Tejun Heo0f3457f2013-05-14 13:52:32 -07001181 if (tg_may_dispatch(tg, bio, NULL)) {
Vivek Goyale43473b2010-09-15 17:06:35 -04001182 throtl_charge_bio(tg, bio);
Vivek Goyal04521db2011-03-22 21:54:29 +01001183
1184 /*
1185 * We need to trim slice even when bios are not being queued
1186 * otherwise it might happen that a bio is not queued for
1187 * a long time and slice keeps on extending and trim is not
1188 * called for a long time. Now if limits are reduced suddenly
1189 * we take into account all the IO dispatched so far at new
1190 * low rate and * newly queued IO gets a really long dispatch
1191 * time.
1192 *
1193 * So keep on trimming slice even if bio is not queued.
1194 */
Tejun Heo0f3457f2013-05-14 13:52:32 -07001195 throtl_trim_slice(tg, rw);
Tejun Heobc16a4f2011-10-19 14:33:01 +02001196 goto out_unlock;
Vivek Goyale43473b2010-09-15 17:06:35 -04001197 }
1198
1199queue_bio:
Tejun Heofda6f272013-05-14 13:52:36 -07001200 throtl_log(sq, "[%c] bio. bdisp=%llu sz=%u bps=%llu iodisp=%u iops=%u queued=%d/%d",
1201 rw == READ ? 'R' : 'W',
1202 tg->bytes_disp[rw], bio->bi_size, tg->bps[rw],
1203 tg->io_disp[rw], tg->iops[rw],
1204 sq->nr_queued[READ], sq->nr_queued[WRITE]);
Vivek Goyale43473b2010-09-15 17:06:35 -04001205
Tejun Heo671058f2012-03-05 13:15:29 -08001206 bio_associate_current(bio);
Tejun Heo77216b02013-05-14 13:52:36 -07001207 throtl_add_bio_tg(bio, tg);
Tejun Heobc16a4f2011-10-19 14:33:01 +02001208 throttled = true;
Vivek Goyale43473b2010-09-15 17:06:35 -04001209
Tejun Heo0e9f4162013-05-14 13:52:35 -07001210 /* update @tg's dispatch time if @tg was empty before @bio */
1211 if (tg->flags & THROTL_TG_WAS_EMPTY) {
Tejun Heo77216b02013-05-14 13:52:36 -07001212 tg_update_disptime(tg);
Vivek Goyale43473b2010-09-15 17:06:35 -04001213 throtl_schedule_next_dispatch(td);
1214 }
1215
Tejun Heobc16a4f2011-10-19 14:33:01 +02001216out_unlock:
Vivek Goyale43473b2010-09-15 17:06:35 -04001217 spin_unlock_irq(q->queue_lock);
Tejun Heo2a7f1242012-03-05 13:15:01 -08001218out_unlock_rcu:
1219 rcu_read_unlock();
Tejun Heobc16a4f2011-10-19 14:33:01 +02001220out:
Tejun Heo2a0f61e2013-05-14 13:52:36 -07001221 /*
1222 * As multiple blk-throtls may stack in the same issue path, we
1223 * don't want bios to leave with the flag set. Clear the flag if
1224 * being issued.
1225 */
1226 if (!throttled)
1227 bio->bi_rw &= ~REQ_THROTTLED;
Tejun Heobc16a4f2011-10-19 14:33:01 +02001228 return throttled;
Vivek Goyale43473b2010-09-15 17:06:35 -04001229}
1230
Tejun Heoc9a929d2011-10-19 14:42:16 +02001231/**
1232 * blk_throtl_drain - drain throttled bios
1233 * @q: request_queue to drain throttled bios for
1234 *
1235 * Dispatch all currently throttled bios on @q through ->make_request_fn().
1236 */
1237void blk_throtl_drain(struct request_queue *q)
1238 __releases(q->queue_lock) __acquires(q->queue_lock)
1239{
1240 struct throtl_data *td = q->td;
Tejun Heo0049af72013-05-14 13:52:33 -07001241 struct throtl_service_queue *parent_sq = &td->service_queue;
Tejun Heoc9a929d2011-10-19 14:42:16 +02001242 struct throtl_grp *tg;
Tejun Heoc9a929d2011-10-19 14:42:16 +02001243 struct bio *bio;
Tejun Heo651930b2013-05-14 13:52:35 -07001244 int rw;
Tejun Heoc9a929d2011-10-19 14:42:16 +02001245
Andi Kleen8bcb6c72012-03-30 12:33:28 +02001246 queue_lockdep_assert_held(q);
Tejun Heoc9a929d2011-10-19 14:42:16 +02001247
Tejun Heo0049af72013-05-14 13:52:33 -07001248 while ((tg = throtl_rb_first(parent_sq))) {
Tejun Heo73f0d492013-05-14 13:52:35 -07001249 struct throtl_service_queue *sq = &tg->service_queue;
1250
Tejun Heo77216b02013-05-14 13:52:36 -07001251 throtl_dequeue_tg(tg);
Tejun Heoc9a929d2011-10-19 14:42:16 +02001252
Tejun Heo73f0d492013-05-14 13:52:35 -07001253 while ((bio = bio_list_peek(&sq->bio_lists[READ])))
Tejun Heo77216b02013-05-14 13:52:36 -07001254 tg_dispatch_one_bio(tg, bio_data_dir(bio));
Tejun Heo73f0d492013-05-14 13:52:35 -07001255 while ((bio = bio_list_peek(&sq->bio_lists[WRITE])))
Tejun Heo77216b02013-05-14 13:52:36 -07001256 tg_dispatch_one_bio(tg, bio_data_dir(bio));
Tejun Heoc9a929d2011-10-19 14:42:16 +02001257 }
1258 spin_unlock_irq(q->queue_lock);
1259
Tejun Heo651930b2013-05-14 13:52:35 -07001260 for (rw = READ; rw <= WRITE; rw++)
1261 while ((bio = bio_list_pop(&parent_sq->bio_lists[rw])))
1262 generic_make_request(bio);
Tejun Heoc9a929d2011-10-19 14:42:16 +02001263
1264 spin_lock_irq(q->queue_lock);
1265}
1266
Vivek Goyale43473b2010-09-15 17:06:35 -04001267int blk_throtl_init(struct request_queue *q)
1268{
1269 struct throtl_data *td;
Tejun Heoa2b16932012-04-13 13:11:33 -07001270 int ret;
Vivek Goyale43473b2010-09-15 17:06:35 -04001271
1272 td = kzalloc_node(sizeof(*td), GFP_KERNEL, q->node);
1273 if (!td)
1274 return -ENOMEM;
1275
Tejun Heocb761992013-05-14 13:52:31 -07001276 INIT_DELAYED_WORK(&td->dispatch_work, blk_throtl_dispatch_work_fn);
Tejun Heo77216b02013-05-14 13:52:36 -07001277 throtl_service_queue_init(&td->service_queue, NULL);
Vivek Goyale43473b2010-09-15 17:06:35 -04001278
Tejun Heocd1604f2012-03-05 13:15:06 -08001279 q->td = td;
Vivek Goyal29b12582011-05-19 15:38:24 -04001280 td->queue = q;
Vivek Goyal02977e42010-10-01 14:49:48 +02001281
Tejun Heoa2b16932012-04-13 13:11:33 -07001282 /* activate policy */
Tejun Heo3c798392012-04-16 13:57:25 -07001283 ret = blkcg_activate_policy(q, &blkcg_policy_throtl);
Tejun Heoa2b16932012-04-13 13:11:33 -07001284 if (ret)
Vivek Goyal29b12582011-05-19 15:38:24 -04001285 kfree(td);
Tejun Heoa2b16932012-04-13 13:11:33 -07001286 return ret;
Vivek Goyale43473b2010-09-15 17:06:35 -04001287}
1288
1289void blk_throtl_exit(struct request_queue *q)
1290{
Tejun Heoc875f4d2012-03-05 13:15:22 -08001291 BUG_ON(!q->td);
Vivek Goyalda527772011-03-02 19:05:33 -05001292 throtl_shutdown_wq(q);
Tejun Heo3c798392012-04-16 13:57:25 -07001293 blkcg_deactivate_policy(q, &blkcg_policy_throtl);
Tejun Heoc9a929d2011-10-19 14:42:16 +02001294 kfree(q->td);
Vivek Goyale43473b2010-09-15 17:06:35 -04001295}
1296
1297static int __init throtl_init(void)
1298{
Vivek Goyal450adcb2011-03-01 13:40:54 -05001299 kthrotld_workqueue = alloc_workqueue("kthrotld", WQ_MEM_RECLAIM, 0);
1300 if (!kthrotld_workqueue)
1301 panic("Failed to create kthrotld\n");
1302
Tejun Heo3c798392012-04-16 13:57:25 -07001303 return blkcg_policy_register(&blkcg_policy_throtl);
Vivek Goyale43473b2010-09-15 17:06:35 -04001304}
1305
1306module_init(throtl_init);