blob: 04de88463a986384b54fa57593d2c89e7189e95c [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
Linus Torvalds1da177e2005-04-16 15:20:36 -07002 * CFQ, or complete fairness queueing, disk scheduler.
3 *
4 * Based on ideas from a previously unfinished io
5 * scheduler (round robin per-process disk scheduling) and Andrea Arcangeli.
6 *
Jens Axboe0fe23472006-09-04 15:41:16 +02007 * Copyright (C) 2003 Jens Axboe <axboe@kernel.dk>
Linus Torvalds1da177e2005-04-16 15:20:36 -07008 */
Linus Torvalds1da177e2005-04-16 15:20:36 -07009#include <linux/module.h>
Tejun Heo5a0e3ad2010-03-24 17:04:11 +090010#include <linux/slab.h>
Al Viro1cc9be62006-03-18 12:29:52 -050011#include <linux/blkdev.h>
12#include <linux/elevator.h>
Randy Dunlapad5ebd22009-11-11 13:47:45 +010013#include <linux/jiffies.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070014#include <linux/rbtree.h>
Jens Axboe22e2c502005-06-27 10:55:12 +020015#include <linux/ioprio.h>
Jens Axboe7b679132008-05-30 12:23:07 +020016#include <linux/blktrace_api.h>
Tejun Heoeea8f412015-05-22 17:13:17 -040017#include <linux/blk-cgroup.h>
Tejun Heo6e736be2011-12-14 00:33:38 +010018#include "blk.h"
Linus Torvalds1da177e2005-04-16 15:20:36 -070019
20/*
21 * tunables
22 */
Jens Axboefe094d92008-01-31 13:08:54 +010023/* max queue in one round of service */
Shaohua Liabc3c742010-03-01 09:20:54 +010024static const int cfq_quantum = 8;
Arjan van de Ven64100092006-01-06 09:46:02 +010025static const int cfq_fifo_expire[2] = { HZ / 4, HZ / 8 };
Jens Axboefe094d92008-01-31 13:08:54 +010026/* maximum backwards seek, in KiB */
27static const int cfq_back_max = 16 * 1024;
28/* penalty of a backwards seek */
29static const int cfq_back_penalty = 2;
Arjan van de Ven64100092006-01-06 09:46:02 +010030static const int cfq_slice_sync = HZ / 10;
Jens Axboe3b181522005-06-27 10:56:24 +020031static int cfq_slice_async = HZ / 25;
Arjan van de Ven64100092006-01-06 09:46:02 +010032static const int cfq_slice_async_rq = 2;
Jens Axboecaaa5f92006-06-16 11:23:00 +020033static int cfq_slice_idle = HZ / 125;
Vivek Goyal80bdf0c2010-08-23 12:24:26 +020034static int cfq_group_idle = HZ / 125;
Corrado Zoccolo5db5d642009-10-26 22:44:04 +010035static const int cfq_target_latency = HZ * 3/10; /* 300 ms */
36static const int cfq_hist_divisor = 4;
Jens Axboe22e2c502005-06-27 10:55:12 +020037
Jens Axboed9e76202007-04-20 14:27:50 +020038/*
Jens Axboe08717142008-01-28 11:38:15 +010039 * offset from end of service tree
Jens Axboed9e76202007-04-20 14:27:50 +020040 */
Jens Axboe08717142008-01-28 11:38:15 +010041#define CFQ_IDLE_DELAY (HZ / 5)
Jens Axboed9e76202007-04-20 14:27:50 +020042
43/*
44 * below this threshold, we consider thinktime immediate
45 */
46#define CFQ_MIN_TT (2)
47
Jens Axboe22e2c502005-06-27 10:55:12 +020048#define CFQ_SLICE_SCALE (5)
Aaron Carroll45333d52008-08-26 15:52:36 +020049#define CFQ_HW_QUEUE_MIN (5)
Vivek Goyal25bc6b02009-12-03 12:59:43 -050050#define CFQ_SERVICE_SHIFT 12
Jens Axboe22e2c502005-06-27 10:55:12 +020051
Corrado Zoccolo3dde36d2010-02-27 19:45:39 +010052#define CFQQ_SEEK_THR (sector_t)(8 * 100)
Shaohua Lie9ce3352010-03-19 08:03:04 +010053#define CFQQ_CLOSE_THR (sector_t)(8 * 1024)
Corrado Zoccolo41647e72010-02-27 19:45:40 +010054#define CFQQ_SECT_THR_NONROT (sector_t)(2 * 32)
Corrado Zoccolo3dde36d2010-02-27 19:45:39 +010055#define CFQQ_SEEKY(cfqq) (hweight32(cfqq->seek_history) > 32/8)
Shaohua Liae54abe2010-02-05 13:11:45 +010056
Tejun Heoa612fdd2011-12-14 00:33:41 +010057#define RQ_CIC(rq) icq_to_cic((rq)->elv.icq)
58#define RQ_CFQQ(rq) (struct cfq_queue *) ((rq)->elv.priv[0])
59#define RQ_CFQG(rq) (struct cfq_group *) ((rq)->elv.priv[1])
Linus Torvalds1da177e2005-04-16 15:20:36 -070060
Christoph Lametere18b8902006-12-06 20:33:20 -080061static struct kmem_cache *cfq_pool;
Linus Torvalds1da177e2005-04-16 15:20:36 -070062
Jens Axboe22e2c502005-06-27 10:55:12 +020063#define CFQ_PRIO_LISTS IOPRIO_BE_NR
64#define cfq_class_idle(cfqq) ((cfqq)->ioprio_class == IOPRIO_CLASS_IDLE)
Jens Axboe22e2c502005-06-27 10:55:12 +020065#define cfq_class_rt(cfqq) ((cfqq)->ioprio_class == IOPRIO_CLASS_RT)
66
Jens Axboe206dc692006-03-28 13:03:44 +020067#define sample_valid(samples) ((samples) > 80)
Vivek Goyal1fa8f6d2009-12-03 12:59:41 -050068#define rb_entry_cfqg(node) rb_entry((node), struct cfq_group, rb_node)
Jens Axboe206dc692006-03-28 13:03:44 +020069
Arianna Avanzinie48453c2015-06-05 23:38:42 +020070/* blkio-related constants */
Tejun Heo3ecca622015-08-18 14:55:35 -070071#define CFQ_WEIGHT_LEGACY_MIN 10
72#define CFQ_WEIGHT_LEGACY_DFL 500
73#define CFQ_WEIGHT_LEGACY_MAX 1000
Arianna Avanzinie48453c2015-06-05 23:38:42 +020074
Tejun Heoc5869802011-12-14 00:33:41 +010075struct cfq_ttime {
76 unsigned long last_end_request;
77
78 unsigned long ttime_total;
79 unsigned long ttime_samples;
80 unsigned long ttime_mean;
81};
82
Jens Axboe22e2c502005-06-27 10:55:12 +020083/*
Jens Axboecc09e292007-04-26 12:53:50 +020084 * Most of our rbtree usage is for sorting with min extraction, so
85 * if we cache the leftmost node we don't have to walk down the tree
86 * to find it. Idea borrowed from Ingo Molnars CFS scheduler. We should
87 * move this into the elevator for the rq sorting as well.
88 */
89struct cfq_rb_root {
90 struct rb_root rb;
91 struct rb_node *left;
Corrado Zoccoloaa6f6a32009-10-26 22:44:33 +010092 unsigned count;
Vivek Goyal1fa8f6d2009-12-03 12:59:41 -050093 u64 min_vdisktime;
Shaohua Lif5f2b6c2011-07-12 14:24:55 +020094 struct cfq_ttime ttime;
Jens Axboecc09e292007-04-26 12:53:50 +020095};
Shaohua Lif5f2b6c2011-07-12 14:24:55 +020096#define CFQ_RB_ROOT (struct cfq_rb_root) { .rb = RB_ROOT, \
97 .ttime = {.last_end_request = jiffies,},}
Jens Axboecc09e292007-04-26 12:53:50 +020098
99/*
Jens Axboe6118b702009-06-30 09:34:12 +0200100 * Per process-grouping structure
101 */
102struct cfq_queue {
103 /* reference count */
Shaohua Li30d7b942011-01-07 08:46:59 +0100104 int ref;
Jens Axboe6118b702009-06-30 09:34:12 +0200105 /* various state flags, see below */
106 unsigned int flags;
107 /* parent cfq_data */
108 struct cfq_data *cfqd;
109 /* service_tree member */
110 struct rb_node rb_node;
111 /* service_tree key */
112 unsigned long rb_key;
113 /* prio tree member */
114 struct rb_node p_node;
115 /* prio tree root we belong to, if any */
116 struct rb_root *p_root;
117 /* sorted list of pending requests */
118 struct rb_root sort_list;
119 /* if fifo isn't expired, next request to serve */
120 struct request *next_rq;
121 /* requests queued in sort_list */
122 int queued[2];
123 /* currently allocated requests */
124 int allocated[2];
125 /* fifo list of requests in sort_list */
126 struct list_head fifo;
127
Vivek Goyaldae739e2009-12-03 12:59:45 -0500128 /* time when queue got scheduled in to dispatch first request. */
129 unsigned long dispatch_start;
Vivek Goyalf75edf22009-12-03 12:59:53 -0500130 unsigned int allocated_slice;
Richard Kennedyc4081ba2010-02-22 13:49:24 +0100131 unsigned int slice_dispatch;
Vivek Goyaldae739e2009-12-03 12:59:45 -0500132 /* time when first request from queue completed and slice started. */
133 unsigned long slice_start;
Jens Axboe6118b702009-06-30 09:34:12 +0200134 unsigned long slice_end;
135 long slice_resid;
Jens Axboe6118b702009-06-30 09:34:12 +0200136
Christoph Hellwig65299a32011-08-23 14:50:29 +0200137 /* pending priority requests */
138 int prio_pending;
Jens Axboe6118b702009-06-30 09:34:12 +0200139 /* number of requests that are on the dispatch list or inside driver */
140 int dispatched;
141
142 /* io prio of this group */
143 unsigned short ioprio, org_ioprio;
Justin TerAvest4aede842011-07-12 08:31:45 +0200144 unsigned short ioprio_class;
Jens Axboe6118b702009-06-30 09:34:12 +0200145
Richard Kennedyc4081ba2010-02-22 13:49:24 +0100146 pid_t pid;
147
Corrado Zoccolo3dde36d2010-02-27 19:45:39 +0100148 u32 seek_history;
Jeff Moyerb2c18e12009-10-23 17:14:49 -0400149 sector_t last_request_pos;
150
Corrado Zoccoloaa6f6a32009-10-26 22:44:33 +0100151 struct cfq_rb_root *service_tree;
Jeff Moyerdf5fe3e2009-10-23 17:14:50 -0400152 struct cfq_queue *new_cfqq;
Vivek Goyalcdb16e82009-12-03 12:59:38 -0500153 struct cfq_group *cfqg;
Vivek Goyalc4e78932010-08-23 12:25:03 +0200154 /* Number of sectors dispatched from queue in single dispatch round */
155 unsigned long nr_sectors;
Jens Axboe6118b702009-06-30 09:34:12 +0200156};
157
158/*
Corrado Zoccolo718eee02009-10-26 22:45:29 +0100159 * First index in the service_trees.
Corrado Zoccoloc0324a02009-10-27 19:16:03 +0100160 * IDLE is handled separately, so it has negative index
161 */
Vivek Goyal3bf10fe2012-10-03 16:56:56 -0400162enum wl_class_t {
Corrado Zoccoloc0324a02009-10-27 19:16:03 +0100163 BE_WORKLOAD = 0,
Vivek Goyal615f0252009-12-03 12:59:39 -0500164 RT_WORKLOAD = 1,
165 IDLE_WORKLOAD = 2,
Vivek Goyalb4627322010-10-22 09:48:43 +0200166 CFQ_PRIO_NR,
Corrado Zoccoloc0324a02009-10-27 19:16:03 +0100167};
168
169/*
Corrado Zoccolo718eee02009-10-26 22:45:29 +0100170 * Second index in the service_trees.
171 */
172enum wl_type_t {
173 ASYNC_WORKLOAD = 0,
174 SYNC_NOIDLE_WORKLOAD = 1,
175 SYNC_WORKLOAD = 2
176};
177
Tejun Heo155fead2012-04-01 14:38:44 -0700178struct cfqg_stats {
179#ifdef CONFIG_CFQ_GROUP_IOSCHED
Tejun Heo155fead2012-04-01 14:38:44 -0700180 /* number of ios merged */
181 struct blkg_rwstat merged;
182 /* total time spent on device in ns, may not be accurate w/ queueing */
183 struct blkg_rwstat service_time;
184 /* total time spent waiting in scheduler queue in ns */
185 struct blkg_rwstat wait_time;
186 /* number of IOs queued up */
187 struct blkg_rwstat queued;
Tejun Heo155fead2012-04-01 14:38:44 -0700188 /* total disk time and nr sectors dispatched by this group */
189 struct blkg_stat time;
190#ifdef CONFIG_DEBUG_BLK_CGROUP
191 /* time not charged to this cgroup */
192 struct blkg_stat unaccounted_time;
193 /* sum of number of ios queued across all samples */
194 struct blkg_stat avg_queue_size_sum;
195 /* count of samples taken for average */
196 struct blkg_stat avg_queue_size_samples;
197 /* how many times this group has been removed from service tree */
198 struct blkg_stat dequeue;
199 /* total time spent waiting for it to be assigned a timeslice. */
200 struct blkg_stat group_wait_time;
Tejun Heo3c798392012-04-16 13:57:25 -0700201 /* time spent idling for this blkcg_gq */
Tejun Heo155fead2012-04-01 14:38:44 -0700202 struct blkg_stat idle_time;
203 /* total time with empty current active q with other requests queued */
204 struct blkg_stat empty_time;
205 /* fields after this shouldn't be cleared on stat reset */
206 uint64_t start_group_wait_time;
207 uint64_t start_idle_time;
208 uint64_t start_empty_time;
209 uint16_t flags;
210#endif /* CONFIG_DEBUG_BLK_CGROUP */
211#endif /* CONFIG_CFQ_GROUP_IOSCHED */
212};
213
Arianna Avanzinie48453c2015-06-05 23:38:42 +0200214/* Per-cgroup data */
215struct cfq_group_data {
216 /* must be the first member */
Tejun Heo81437642015-08-18 14:55:15 -0700217 struct blkcg_policy_data cpd;
Arianna Avanzinie48453c2015-06-05 23:38:42 +0200218
219 unsigned int weight;
220 unsigned int leaf_weight;
221};
222
Vivek Goyalcdb16e82009-12-03 12:59:38 -0500223/* This is per cgroup per device grouping structure */
224struct cfq_group {
Tejun Heof95a04a2012-04-16 13:57:26 -0700225 /* must be the first member */
226 struct blkg_policy_data pd;
227
Vivek Goyal1fa8f6d2009-12-03 12:59:41 -0500228 /* group service_tree member */
229 struct rb_node rb_node;
230
231 /* group service_tree key */
232 u64 vdisktime;
Tejun Heoe71357e2013-01-09 08:05:10 -0800233
234 /*
Tejun Heo7918ffb2013-01-09 08:05:11 -0800235 * The number of active cfqgs and sum of their weights under this
236 * cfqg. This covers this cfqg's leaf_weight and all children's
237 * weights, but does not cover weights of further descendants.
238 *
239 * If a cfqg is on the service tree, it's active. An active cfqg
240 * also activates its parent and contributes to the children_weight
241 * of the parent.
242 */
243 int nr_active;
244 unsigned int children_weight;
245
246 /*
Tejun Heo1d3650f2013-01-09 08:05:11 -0800247 * vfraction is the fraction of vdisktime that the tasks in this
248 * cfqg are entitled to. This is determined by compounding the
249 * ratios walking up from this cfqg to the root.
250 *
251 * It is in fixed point w/ CFQ_SERVICE_SHIFT and the sum of all
252 * vfractions on a service tree is approximately 1. The sum may
253 * deviate a bit due to rounding errors and fluctuations caused by
254 * cfqgs entering and leaving the service tree.
255 */
256 unsigned int vfraction;
257
258 /*
Tejun Heoe71357e2013-01-09 08:05:10 -0800259 * There are two weights - (internal) weight is the weight of this
260 * cfqg against the sibling cfqgs. leaf_weight is the wight of
261 * this cfqg against the child cfqgs. For the root cfqg, both
262 * weights are kept in sync for backward compatibility.
263 */
Vivek Goyal25bc6b02009-12-03 12:59:43 -0500264 unsigned int weight;
Justin TerAvest8184f932011-03-17 16:12:36 +0100265 unsigned int new_weight;
Tejun Heo3381cb82012-04-01 14:38:44 -0700266 unsigned int dev_weight;
Vivek Goyal1fa8f6d2009-12-03 12:59:41 -0500267
Tejun Heoe71357e2013-01-09 08:05:10 -0800268 unsigned int leaf_weight;
269 unsigned int new_leaf_weight;
270 unsigned int dev_leaf_weight;
271
Vivek Goyal1fa8f6d2009-12-03 12:59:41 -0500272 /* number of cfqq currently on this group */
273 int nr_cfqq;
274
Jens Axboe22e2c502005-06-27 10:55:12 +0200275 /*
Kyungmin Park4495a7d2011-05-31 10:04:09 +0200276 * Per group busy queues average. Useful for workload slice calc. We
Vivek Goyalb4627322010-10-22 09:48:43 +0200277 * create the array for each prio class but at run time it is used
278 * only for RT and BE class and slot for IDLE class remains unused.
279 * This is primarily done to avoid confusion and a gcc warning.
280 */
281 unsigned int busy_queues_avg[CFQ_PRIO_NR];
282 /*
283 * rr lists of queues with requests. We maintain service trees for
284 * RT and BE classes. These trees are subdivided in subclasses
285 * of SYNC, SYNC_NOIDLE and ASYNC based on workload type. For IDLE
286 * class there is no subclassification and all the cfq queues go on
287 * a single tree service_tree_idle.
Corrado Zoccoloc0324a02009-10-27 19:16:03 +0100288 * Counts are embedded in the cfq_rb_root
Jens Axboe22e2c502005-06-27 10:55:12 +0200289 */
Corrado Zoccolo718eee02009-10-26 22:45:29 +0100290 struct cfq_rb_root service_trees[2][3];
Corrado Zoccoloc0324a02009-10-27 19:16:03 +0100291 struct cfq_rb_root service_tree_idle;
Vivek Goyaldae739e2009-12-03 12:59:45 -0500292
Vivek Goyal4d2ceea2012-10-03 16:56:57 -0400293 unsigned long saved_wl_slice;
294 enum wl_type_t saved_wl_type;
295 enum wl_class_t saved_wl_class;
Tejun Heo4eef3042012-03-05 13:15:18 -0800296
Vivek Goyal80bdf0c2010-08-23 12:24:26 +0200297 /* number of requests that are on the dispatch list or inside driver */
298 int dispatched;
Shaohua Li7700fc42011-07-12 14:24:56 +0200299 struct cfq_ttime ttime;
Tejun Heo0b399202013-01-09 08:05:13 -0800300 struct cfqg_stats stats; /* stats for this cfqg */
Tejun Heo60a83702015-08-18 14:55:05 -0700301
302 /* async queue for each priority case */
303 struct cfq_queue *async_cfqq[2][IOPRIO_BE_NR];
304 struct cfq_queue *async_idle_cfqq;
305
Vivek Goyalcdb16e82009-12-03 12:59:38 -0500306};
307
Tejun Heoc5869802011-12-14 00:33:41 +0100308struct cfq_io_cq {
309 struct io_cq icq; /* must be the first member */
310 struct cfq_queue *cfqq[2];
311 struct cfq_ttime ttime;
Tejun Heo598971b2012-03-19 15:10:58 -0700312 int ioprio; /* the current ioprio */
313#ifdef CONFIG_CFQ_GROUP_IOSCHED
Tejun Heof4da8072014-09-08 08:15:20 +0900314 uint64_t blkcg_serial_nr; /* the current blkcg serial */
Tejun Heo598971b2012-03-19 15:10:58 -0700315#endif
Tejun Heoc5869802011-12-14 00:33:41 +0100316};
317
Vivek Goyalcdb16e82009-12-03 12:59:38 -0500318/*
319 * Per block device queue structure
320 */
321struct cfq_data {
322 struct request_queue *queue;
Vivek Goyal1fa8f6d2009-12-03 12:59:41 -0500323 /* Root service tree for cfq_groups */
324 struct cfq_rb_root grp_service_tree;
Tejun Heof51b8022012-03-05 13:15:05 -0800325 struct cfq_group *root_group;
Vivek Goyalcdb16e82009-12-03 12:59:38 -0500326
Corrado Zoccoloc0324a02009-10-27 19:16:03 +0100327 /*
328 * The priority currently being served
329 */
Vivek Goyal4d2ceea2012-10-03 16:56:57 -0400330 enum wl_class_t serving_wl_class;
331 enum wl_type_t serving_wl_type;
Corrado Zoccolo718eee02009-10-26 22:45:29 +0100332 unsigned long workload_expires;
Vivek Goyalcdb16e82009-12-03 12:59:38 -0500333 struct cfq_group *serving_group;
Jens Axboea36e71f2009-04-15 12:15:11 +0200334
335 /*
336 * Each priority tree is sorted by next_request position. These
337 * trees are used when determining if two or more queues are
338 * interleaving requests (see cfq_close_cooperator).
339 */
340 struct rb_root prio_trees[CFQ_PRIO_LISTS];
341
Jens Axboe22e2c502005-06-27 10:55:12 +0200342 unsigned int busy_queues;
Shaohua Lief8a41d2011-03-07 09:26:29 +0100343 unsigned int busy_sync_queues;
Jens Axboe22e2c502005-06-27 10:55:12 +0200344
Corrado Zoccolo53c583d2010-02-28 19:45:05 +0100345 int rq_in_driver;
346 int rq_in_flight[2];
Aaron Carroll45333d52008-08-26 15:52:36 +0200347
348 /*
349 * queue-depth detection
350 */
351 int rq_queued;
Jens Axboe25776e32006-06-01 10:12:26 +0200352 int hw_tag;
Corrado Zoccoloe459dd02009-11-26 10:02:57 +0100353 /*
354 * hw_tag can be
355 * -1 => indeterminate, (cfq will behave as if NCQ is present, to allow better detection)
356 * 1 => NCQ is present (hw_tag_est_depth is the estimated max depth)
357 * 0 => no NCQ
358 */
359 int hw_tag_est_depth;
360 unsigned int hw_tag_samples;
Jens Axboe22e2c502005-06-27 10:55:12 +0200361
362 /*
Jens Axboe22e2c502005-06-27 10:55:12 +0200363 * idle window management
364 */
365 struct timer_list idle_slice_timer;
Jens Axboe23e018a2009-10-05 08:52:35 +0200366 struct work_struct unplug_work;
Jens Axboe22e2c502005-06-27 10:55:12 +0200367
368 struct cfq_queue *active_queue;
Tejun Heoc5869802011-12-14 00:33:41 +0100369 struct cfq_io_cq *active_cic;
Jens Axboe22e2c502005-06-27 10:55:12 +0200370
Jens Axboe6d048f52007-04-25 12:44:27 +0200371 sector_t last_position;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700372
Linus Torvalds1da177e2005-04-16 15:20:36 -0700373 /*
374 * tunables, see top of file
375 */
376 unsigned int cfq_quantum;
Jens Axboe22e2c502005-06-27 10:55:12 +0200377 unsigned int cfq_fifo_expire[2];
Linus Torvalds1da177e2005-04-16 15:20:36 -0700378 unsigned int cfq_back_penalty;
379 unsigned int cfq_back_max;
Jens Axboe22e2c502005-06-27 10:55:12 +0200380 unsigned int cfq_slice[2];
381 unsigned int cfq_slice_async_rq;
382 unsigned int cfq_slice_idle;
Vivek Goyal80bdf0c2010-08-23 12:24:26 +0200383 unsigned int cfq_group_idle;
Jens Axboe963b72f2009-10-03 19:42:18 +0200384 unsigned int cfq_latency;
Tao Ma5bf14c02012-04-01 14:33:39 -0700385 unsigned int cfq_target_latency;
Al Virod9ff4182006-03-18 13:51:22 -0500386
Jens Axboe6118b702009-06-30 09:34:12 +0200387 /*
388 * Fallback dummy cfqq for extreme OOM conditions
389 */
390 struct cfq_queue oom_cfqq;
Vivek Goyal365722b2009-10-03 15:21:27 +0200391
Corrado Zoccolo573412b2009-12-06 11:48:52 +0100392 unsigned long last_delayed_sync;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700393};
394
Vivek Goyal25fb5162009-12-03 12:59:46 -0500395static struct cfq_group *cfq_get_next_cfqg(struct cfq_data *cfqd);
Tejun Heo60a83702015-08-18 14:55:05 -0700396static void cfq_put_queue(struct cfq_queue *cfqq);
Vivek Goyal25fb5162009-12-03 12:59:46 -0500397
Vivek Goyal34b98d02012-10-03 16:56:58 -0400398static struct cfq_rb_root *st_for(struct cfq_group *cfqg,
Vivek Goyal3bf10fe2012-10-03 16:56:56 -0400399 enum wl_class_t class,
Vivek Goyal65b32a52009-12-16 17:52:59 -0500400 enum wl_type_t type)
Corrado Zoccoloc0324a02009-10-27 19:16:03 +0100401{
Vivek Goyal1fa8f6d2009-12-03 12:59:41 -0500402 if (!cfqg)
403 return NULL;
404
Vivek Goyal3bf10fe2012-10-03 16:56:56 -0400405 if (class == IDLE_WORKLOAD)
Vivek Goyalcdb16e82009-12-03 12:59:38 -0500406 return &cfqg->service_tree_idle;
Corrado Zoccoloc0324a02009-10-27 19:16:03 +0100407
Vivek Goyal3bf10fe2012-10-03 16:56:56 -0400408 return &cfqg->service_trees[class][type];
Corrado Zoccoloc0324a02009-10-27 19:16:03 +0100409}
410
Jens Axboe3b181522005-06-27 10:56:24 +0200411enum cfqq_state_flags {
Jens Axboeb0b8d7492007-01-19 11:35:30 +1100412 CFQ_CFQQ_FLAG_on_rr = 0, /* on round-robin busy list */
413 CFQ_CFQQ_FLAG_wait_request, /* waiting for a request */
Jens Axboeb0291952009-04-07 11:38:31 +0200414 CFQ_CFQQ_FLAG_must_dispatch, /* must be allowed a dispatch */
Jens Axboeb0b8d7492007-01-19 11:35:30 +1100415 CFQ_CFQQ_FLAG_must_alloc_slice, /* per-slice must_alloc flag */
Jens Axboeb0b8d7492007-01-19 11:35:30 +1100416 CFQ_CFQQ_FLAG_fifo_expire, /* FIFO checked in this slice */
417 CFQ_CFQQ_FLAG_idle_window, /* slice idling enabled */
418 CFQ_CFQQ_FLAG_prio_changed, /* task priority has changed */
Jens Axboe44f7c162007-01-19 11:51:58 +1100419 CFQ_CFQQ_FLAG_slice_new, /* no requests dispatched in slice */
Vasily Tarasov91fac312007-04-25 12:29:51 +0200420 CFQ_CFQQ_FLAG_sync, /* synchronous queue */
Jeff Moyerb3b6d042009-10-23 17:14:51 -0400421 CFQ_CFQQ_FLAG_coop, /* cfqq is shared */
Shaohua Liae54abe2010-02-05 13:11:45 +0100422 CFQ_CFQQ_FLAG_split_coop, /* shared cfqq will be splitted */
Corrado Zoccolo76280af2009-11-26 10:02:58 +0100423 CFQ_CFQQ_FLAG_deep, /* sync cfqq experienced large depth */
Vivek Goyalf75edf22009-12-03 12:59:53 -0500424 CFQ_CFQQ_FLAG_wait_busy, /* Waiting for next request */
Jens Axboe3b181522005-06-27 10:56:24 +0200425};
426
427#define CFQ_CFQQ_FNS(name) \
428static inline void cfq_mark_cfqq_##name(struct cfq_queue *cfqq) \
429{ \
Jens Axboefe094d92008-01-31 13:08:54 +0100430 (cfqq)->flags |= (1 << CFQ_CFQQ_FLAG_##name); \
Jens Axboe3b181522005-06-27 10:56:24 +0200431} \
432static inline void cfq_clear_cfqq_##name(struct cfq_queue *cfqq) \
433{ \
Jens Axboefe094d92008-01-31 13:08:54 +0100434 (cfqq)->flags &= ~(1 << CFQ_CFQQ_FLAG_##name); \
Jens Axboe3b181522005-06-27 10:56:24 +0200435} \
436static inline int cfq_cfqq_##name(const struct cfq_queue *cfqq) \
437{ \
Jens Axboefe094d92008-01-31 13:08:54 +0100438 return ((cfqq)->flags & (1 << CFQ_CFQQ_FLAG_##name)) != 0; \
Jens Axboe3b181522005-06-27 10:56:24 +0200439}
440
441CFQ_CFQQ_FNS(on_rr);
442CFQ_CFQQ_FNS(wait_request);
Jens Axboeb0291952009-04-07 11:38:31 +0200443CFQ_CFQQ_FNS(must_dispatch);
Jens Axboe3b181522005-06-27 10:56:24 +0200444CFQ_CFQQ_FNS(must_alloc_slice);
Jens Axboe3b181522005-06-27 10:56:24 +0200445CFQ_CFQQ_FNS(fifo_expire);
446CFQ_CFQQ_FNS(idle_window);
447CFQ_CFQQ_FNS(prio_changed);
Jens Axboe44f7c162007-01-19 11:51:58 +1100448CFQ_CFQQ_FNS(slice_new);
Vasily Tarasov91fac312007-04-25 12:29:51 +0200449CFQ_CFQQ_FNS(sync);
Jens Axboea36e71f2009-04-15 12:15:11 +0200450CFQ_CFQQ_FNS(coop);
Shaohua Liae54abe2010-02-05 13:11:45 +0100451CFQ_CFQQ_FNS(split_coop);
Corrado Zoccolo76280af2009-11-26 10:02:58 +0100452CFQ_CFQQ_FNS(deep);
Vivek Goyalf75edf22009-12-03 12:59:53 -0500453CFQ_CFQQ_FNS(wait_busy);
Jens Axboe3b181522005-06-27 10:56:24 +0200454#undef CFQ_CFQQ_FNS
455
Tejun Heo629ed0b2012-04-01 14:38:44 -0700456#if defined(CONFIG_CFQ_GROUP_IOSCHED) && defined(CONFIG_DEBUG_BLK_CGROUP)
Tejun Heo2ce4d502012-04-01 14:38:43 -0700457
Tejun Heo155fead2012-04-01 14:38:44 -0700458/* cfqg stats flags */
459enum cfqg_stats_flags {
460 CFQG_stats_waiting = 0,
461 CFQG_stats_idling,
462 CFQG_stats_empty,
Tejun Heo629ed0b2012-04-01 14:38:44 -0700463};
464
Tejun Heo155fead2012-04-01 14:38:44 -0700465#define CFQG_FLAG_FNS(name) \
466static inline void cfqg_stats_mark_##name(struct cfqg_stats *stats) \
Tejun Heo629ed0b2012-04-01 14:38:44 -0700467{ \
Tejun Heo155fead2012-04-01 14:38:44 -0700468 stats->flags |= (1 << CFQG_stats_##name); \
Tejun Heo629ed0b2012-04-01 14:38:44 -0700469} \
Tejun Heo155fead2012-04-01 14:38:44 -0700470static inline void cfqg_stats_clear_##name(struct cfqg_stats *stats) \
Tejun Heo629ed0b2012-04-01 14:38:44 -0700471{ \
Tejun Heo155fead2012-04-01 14:38:44 -0700472 stats->flags &= ~(1 << CFQG_stats_##name); \
Tejun Heo629ed0b2012-04-01 14:38:44 -0700473} \
Tejun Heo155fead2012-04-01 14:38:44 -0700474static inline int cfqg_stats_##name(struct cfqg_stats *stats) \
Tejun Heo629ed0b2012-04-01 14:38:44 -0700475{ \
Tejun Heo155fead2012-04-01 14:38:44 -0700476 return (stats->flags & (1 << CFQG_stats_##name)) != 0; \
Tejun Heo629ed0b2012-04-01 14:38:44 -0700477} \
478
Tejun Heo155fead2012-04-01 14:38:44 -0700479CFQG_FLAG_FNS(waiting)
480CFQG_FLAG_FNS(idling)
481CFQG_FLAG_FNS(empty)
482#undef CFQG_FLAG_FNS
Tejun Heo629ed0b2012-04-01 14:38:44 -0700483
484/* This should be called with the queue_lock held. */
Tejun Heo155fead2012-04-01 14:38:44 -0700485static void cfqg_stats_update_group_wait_time(struct cfqg_stats *stats)
Tejun Heo629ed0b2012-04-01 14:38:44 -0700486{
487 unsigned long long now;
488
Tejun Heo155fead2012-04-01 14:38:44 -0700489 if (!cfqg_stats_waiting(stats))
Tejun Heo629ed0b2012-04-01 14:38:44 -0700490 return;
491
492 now = sched_clock();
493 if (time_after64(now, stats->start_group_wait_time))
494 blkg_stat_add(&stats->group_wait_time,
495 now - stats->start_group_wait_time);
Tejun Heo155fead2012-04-01 14:38:44 -0700496 cfqg_stats_clear_waiting(stats);
Tejun Heo629ed0b2012-04-01 14:38:44 -0700497}
498
499/* This should be called with the queue_lock held. */
Tejun Heo155fead2012-04-01 14:38:44 -0700500static void cfqg_stats_set_start_group_wait_time(struct cfq_group *cfqg,
501 struct cfq_group *curr_cfqg)
Tejun Heo629ed0b2012-04-01 14:38:44 -0700502{
Tejun Heo155fead2012-04-01 14:38:44 -0700503 struct cfqg_stats *stats = &cfqg->stats;
Tejun Heo629ed0b2012-04-01 14:38:44 -0700504
Tejun Heo155fead2012-04-01 14:38:44 -0700505 if (cfqg_stats_waiting(stats))
Tejun Heo629ed0b2012-04-01 14:38:44 -0700506 return;
Tejun Heo155fead2012-04-01 14:38:44 -0700507 if (cfqg == curr_cfqg)
Tejun Heo629ed0b2012-04-01 14:38:44 -0700508 return;
Tejun Heo155fead2012-04-01 14:38:44 -0700509 stats->start_group_wait_time = sched_clock();
510 cfqg_stats_mark_waiting(stats);
Tejun Heo629ed0b2012-04-01 14:38:44 -0700511}
512
513/* This should be called with the queue_lock held. */
Tejun Heo155fead2012-04-01 14:38:44 -0700514static void cfqg_stats_end_empty_time(struct cfqg_stats *stats)
Tejun Heo629ed0b2012-04-01 14:38:44 -0700515{
516 unsigned long long now;
517
Tejun Heo155fead2012-04-01 14:38:44 -0700518 if (!cfqg_stats_empty(stats))
Tejun Heo629ed0b2012-04-01 14:38:44 -0700519 return;
520
521 now = sched_clock();
522 if (time_after64(now, stats->start_empty_time))
523 blkg_stat_add(&stats->empty_time,
524 now - stats->start_empty_time);
Tejun Heo155fead2012-04-01 14:38:44 -0700525 cfqg_stats_clear_empty(stats);
Tejun Heo629ed0b2012-04-01 14:38:44 -0700526}
527
Tejun Heo155fead2012-04-01 14:38:44 -0700528static void cfqg_stats_update_dequeue(struct cfq_group *cfqg)
Tejun Heo629ed0b2012-04-01 14:38:44 -0700529{
Tejun Heo155fead2012-04-01 14:38:44 -0700530 blkg_stat_add(&cfqg->stats.dequeue, 1);
Tejun Heo629ed0b2012-04-01 14:38:44 -0700531}
532
Tejun Heo155fead2012-04-01 14:38:44 -0700533static void cfqg_stats_set_start_empty_time(struct cfq_group *cfqg)
Tejun Heo629ed0b2012-04-01 14:38:44 -0700534{
Tejun Heo155fead2012-04-01 14:38:44 -0700535 struct cfqg_stats *stats = &cfqg->stats;
Tejun Heo629ed0b2012-04-01 14:38:44 -0700536
Tejun Heo4d5e80a2013-01-09 08:05:12 -0800537 if (blkg_rwstat_total(&stats->queued))
Tejun Heo629ed0b2012-04-01 14:38:44 -0700538 return;
539
540 /*
541 * group is already marked empty. This can happen if cfqq got new
542 * request in parent group and moved to this group while being added
543 * to service tree. Just ignore the event and move on.
544 */
Tejun Heo155fead2012-04-01 14:38:44 -0700545 if (cfqg_stats_empty(stats))
Tejun Heo629ed0b2012-04-01 14:38:44 -0700546 return;
547
548 stats->start_empty_time = sched_clock();
Tejun Heo155fead2012-04-01 14:38:44 -0700549 cfqg_stats_mark_empty(stats);
Tejun Heo629ed0b2012-04-01 14:38:44 -0700550}
551
Tejun Heo155fead2012-04-01 14:38:44 -0700552static void cfqg_stats_update_idle_time(struct cfq_group *cfqg)
Tejun Heo629ed0b2012-04-01 14:38:44 -0700553{
Tejun Heo155fead2012-04-01 14:38:44 -0700554 struct cfqg_stats *stats = &cfqg->stats;
Tejun Heo629ed0b2012-04-01 14:38:44 -0700555
Tejun Heo155fead2012-04-01 14:38:44 -0700556 if (cfqg_stats_idling(stats)) {
Tejun Heo629ed0b2012-04-01 14:38:44 -0700557 unsigned long long now = sched_clock();
558
559 if (time_after64(now, stats->start_idle_time))
560 blkg_stat_add(&stats->idle_time,
561 now - stats->start_idle_time);
Tejun Heo155fead2012-04-01 14:38:44 -0700562 cfqg_stats_clear_idling(stats);
Tejun Heo629ed0b2012-04-01 14:38:44 -0700563 }
564}
565
Tejun Heo155fead2012-04-01 14:38:44 -0700566static void cfqg_stats_set_start_idle_time(struct cfq_group *cfqg)
Tejun Heo629ed0b2012-04-01 14:38:44 -0700567{
Tejun Heo155fead2012-04-01 14:38:44 -0700568 struct cfqg_stats *stats = &cfqg->stats;
Tejun Heo629ed0b2012-04-01 14:38:44 -0700569
Tejun Heo155fead2012-04-01 14:38:44 -0700570 BUG_ON(cfqg_stats_idling(stats));
Tejun Heo629ed0b2012-04-01 14:38:44 -0700571
572 stats->start_idle_time = sched_clock();
Tejun Heo155fead2012-04-01 14:38:44 -0700573 cfqg_stats_mark_idling(stats);
Tejun Heo629ed0b2012-04-01 14:38:44 -0700574}
575
Tejun Heo155fead2012-04-01 14:38:44 -0700576static void cfqg_stats_update_avg_queue_size(struct cfq_group *cfqg)
Tejun Heo629ed0b2012-04-01 14:38:44 -0700577{
Tejun Heo155fead2012-04-01 14:38:44 -0700578 struct cfqg_stats *stats = &cfqg->stats;
Tejun Heo629ed0b2012-04-01 14:38:44 -0700579
580 blkg_stat_add(&stats->avg_queue_size_sum,
Tejun Heo4d5e80a2013-01-09 08:05:12 -0800581 blkg_rwstat_total(&stats->queued));
Tejun Heo629ed0b2012-04-01 14:38:44 -0700582 blkg_stat_add(&stats->avg_queue_size_samples, 1);
Tejun Heo155fead2012-04-01 14:38:44 -0700583 cfqg_stats_update_group_wait_time(stats);
Tejun Heo629ed0b2012-04-01 14:38:44 -0700584}
585
586#else /* CONFIG_CFQ_GROUP_IOSCHED && CONFIG_DEBUG_BLK_CGROUP */
587
Tejun Heof48ec1d2012-04-13 13:11:25 -0700588static inline void cfqg_stats_set_start_group_wait_time(struct cfq_group *cfqg, struct cfq_group *curr_cfqg) { }
589static inline void cfqg_stats_end_empty_time(struct cfqg_stats *stats) { }
590static inline void cfqg_stats_update_dequeue(struct cfq_group *cfqg) { }
591static inline void cfqg_stats_set_start_empty_time(struct cfq_group *cfqg) { }
592static inline void cfqg_stats_update_idle_time(struct cfq_group *cfqg) { }
593static inline void cfqg_stats_set_start_idle_time(struct cfq_group *cfqg) { }
594static inline void cfqg_stats_update_avg_queue_size(struct cfq_group *cfqg) { }
Tejun Heo629ed0b2012-04-01 14:38:44 -0700595
596#endif /* CONFIG_CFQ_GROUP_IOSCHED && CONFIG_DEBUG_BLK_CGROUP */
597
598#ifdef CONFIG_CFQ_GROUP_IOSCHED
Tejun Heo2ce4d502012-04-01 14:38:43 -0700599
Jens Axboe4ceab712015-06-19 10:13:01 -0600600static inline struct cfq_group *pd_to_cfqg(struct blkg_policy_data *pd)
601{
602 return pd ? container_of(pd, struct cfq_group, pd) : NULL;
603}
604
605static struct cfq_group_data
606*cpd_to_cfqgd(struct blkcg_policy_data *cpd)
607{
Tejun Heo81437642015-08-18 14:55:15 -0700608 return cpd ? container_of(cpd, struct cfq_group_data, cpd) : NULL;
Jens Axboe4ceab712015-06-19 10:13:01 -0600609}
610
611static inline struct blkcg_gq *cfqg_to_blkg(struct cfq_group *cfqg)
612{
613 return pd_to_blkg(&cfqg->pd);
614}
615
Tejun Heoffea73f2012-06-04 10:02:29 +0200616static struct blkcg_policy blkcg_policy_cfq;
617
618static inline struct cfq_group *blkg_to_cfqg(struct blkcg_gq *blkg)
619{
620 return pd_to_cfqg(blkg_to_pd(blkg, &blkcg_policy_cfq));
621}
622
Arianna Avanzinie48453c2015-06-05 23:38:42 +0200623static struct cfq_group_data *blkcg_to_cfqgd(struct blkcg *blkcg)
624{
625 return cpd_to_cfqgd(blkcg_to_cpd(blkcg, &blkcg_policy_cfq));
626}
627
Tejun Heod02f7aa2013-01-09 08:05:11 -0800628static inline struct cfq_group *cfqg_parent(struct cfq_group *cfqg)
Tejun Heo7918ffb2013-01-09 08:05:11 -0800629{
Tejun Heod02f7aa2013-01-09 08:05:11 -0800630 struct blkcg_gq *pblkg = cfqg_to_blkg(cfqg)->parent;
Tejun Heo7918ffb2013-01-09 08:05:11 -0800631
Tejun Heod02f7aa2013-01-09 08:05:11 -0800632 return pblkg ? blkg_to_cfqg(pblkg) : NULL;
Tejun Heo7918ffb2013-01-09 08:05:11 -0800633}
634
Tejun Heoeb7d8c072012-03-23 14:02:53 +0100635static inline void cfqg_get(struct cfq_group *cfqg)
636{
637 return blkg_get(cfqg_to_blkg(cfqg));
638}
639
640static inline void cfqg_put(struct cfq_group *cfqg)
641{
642 return blkg_put(cfqg_to_blkg(cfqg));
643}
644
Tejun Heo54e7ed12012-04-16 13:57:23 -0700645#define cfq_log_cfqq(cfqd, cfqq, fmt, args...) do { \
646 char __pbuf[128]; \
647 \
648 blkg_path(cfqg_to_blkg((cfqq)->cfqg), __pbuf, sizeof(__pbuf)); \
Vivek Goyalb226e5c2012-10-03 16:57:01 -0400649 blk_add_trace_msg((cfqd)->queue, "cfq%d%c%c %s " fmt, (cfqq)->pid, \
650 cfq_cfqq_sync((cfqq)) ? 'S' : 'A', \
651 cfqq_type((cfqq)) == SYNC_NOIDLE_WORKLOAD ? 'N' : ' ',\
Tejun Heo54e7ed12012-04-16 13:57:23 -0700652 __pbuf, ##args); \
653} while (0)
Vivek Goyal2868ef72009-12-03 12:59:48 -0500654
Tejun Heo54e7ed12012-04-16 13:57:23 -0700655#define cfq_log_cfqg(cfqd, cfqg, fmt, args...) do { \
656 char __pbuf[128]; \
657 \
658 blkg_path(cfqg_to_blkg(cfqg), __pbuf, sizeof(__pbuf)); \
659 blk_add_trace_msg((cfqd)->queue, "%s " fmt, __pbuf, ##args); \
660} while (0)
Vivek Goyal2868ef72009-12-03 12:59:48 -0500661
Tejun Heo155fead2012-04-01 14:38:44 -0700662static inline void cfqg_stats_update_io_add(struct cfq_group *cfqg,
663 struct cfq_group *curr_cfqg, int rw)
Tejun Heo2ce4d502012-04-01 14:38:43 -0700664{
Tejun Heo155fead2012-04-01 14:38:44 -0700665 blkg_rwstat_add(&cfqg->stats.queued, rw, 1);
666 cfqg_stats_end_empty_time(&cfqg->stats);
667 cfqg_stats_set_start_group_wait_time(cfqg, curr_cfqg);
Tejun Heo2ce4d502012-04-01 14:38:43 -0700668}
669
Tejun Heo155fead2012-04-01 14:38:44 -0700670static inline void cfqg_stats_update_timeslice_used(struct cfq_group *cfqg,
671 unsigned long time, unsigned long unaccounted_time)
Tejun Heo2ce4d502012-04-01 14:38:43 -0700672{
Tejun Heo155fead2012-04-01 14:38:44 -0700673 blkg_stat_add(&cfqg->stats.time, time);
Tejun Heo629ed0b2012-04-01 14:38:44 -0700674#ifdef CONFIG_DEBUG_BLK_CGROUP
Tejun Heo155fead2012-04-01 14:38:44 -0700675 blkg_stat_add(&cfqg->stats.unaccounted_time, unaccounted_time);
Tejun Heo629ed0b2012-04-01 14:38:44 -0700676#endif
Tejun Heo2ce4d502012-04-01 14:38:43 -0700677}
678
Tejun Heo155fead2012-04-01 14:38:44 -0700679static inline void cfqg_stats_update_io_remove(struct cfq_group *cfqg, int rw)
Tejun Heo2ce4d502012-04-01 14:38:43 -0700680{
Tejun Heo155fead2012-04-01 14:38:44 -0700681 blkg_rwstat_add(&cfqg->stats.queued, rw, -1);
Tejun Heo2ce4d502012-04-01 14:38:43 -0700682}
683
Tejun Heo155fead2012-04-01 14:38:44 -0700684static inline void cfqg_stats_update_io_merged(struct cfq_group *cfqg, int rw)
Tejun Heo2ce4d502012-04-01 14:38:43 -0700685{
Tejun Heo155fead2012-04-01 14:38:44 -0700686 blkg_rwstat_add(&cfqg->stats.merged, rw, 1);
Tejun Heo2ce4d502012-04-01 14:38:43 -0700687}
688
Tejun Heo155fead2012-04-01 14:38:44 -0700689static inline void cfqg_stats_update_completion(struct cfq_group *cfqg,
690 uint64_t start_time, uint64_t io_start_time, int rw)
Tejun Heo2ce4d502012-04-01 14:38:43 -0700691{
Tejun Heo155fead2012-04-01 14:38:44 -0700692 struct cfqg_stats *stats = &cfqg->stats;
Tejun Heo629ed0b2012-04-01 14:38:44 -0700693 unsigned long long now = sched_clock();
Tejun Heo629ed0b2012-04-01 14:38:44 -0700694
695 if (time_after64(now, io_start_time))
696 blkg_rwstat_add(&stats->service_time, rw, now - io_start_time);
697 if (time_after64(io_start_time, start_time))
698 blkg_rwstat_add(&stats->wait_time, rw,
699 io_start_time - start_time);
Tejun Heo2ce4d502012-04-01 14:38:43 -0700700}
701
Tejun Heo689665a2013-01-09 08:05:13 -0800702/* @stats = 0 */
703static void cfqg_stats_reset(struct cfqg_stats *stats)
Tejun Heo155fead2012-04-01 14:38:44 -0700704{
Tejun Heo155fead2012-04-01 14:38:44 -0700705 /* queued stats shouldn't be cleared */
Tejun Heo155fead2012-04-01 14:38:44 -0700706 blkg_rwstat_reset(&stats->merged);
707 blkg_rwstat_reset(&stats->service_time);
708 blkg_rwstat_reset(&stats->wait_time);
709 blkg_stat_reset(&stats->time);
710#ifdef CONFIG_DEBUG_BLK_CGROUP
711 blkg_stat_reset(&stats->unaccounted_time);
712 blkg_stat_reset(&stats->avg_queue_size_sum);
713 blkg_stat_reset(&stats->avg_queue_size_samples);
714 blkg_stat_reset(&stats->dequeue);
715 blkg_stat_reset(&stats->group_wait_time);
716 blkg_stat_reset(&stats->idle_time);
717 blkg_stat_reset(&stats->empty_time);
718#endif
719}
720
Tejun Heo0b399202013-01-09 08:05:13 -0800721/* @to += @from */
Tejun Heoe6269c42015-08-18 14:55:21 -0700722static void cfqg_stats_add_aux(struct cfqg_stats *to, struct cfqg_stats *from)
Tejun Heo0b399202013-01-09 08:05:13 -0800723{
724 /* queued stats shouldn't be cleared */
Tejun Heoe6269c42015-08-18 14:55:21 -0700725 blkg_rwstat_add_aux(&to->merged, &from->merged);
726 blkg_rwstat_add_aux(&to->service_time, &from->service_time);
727 blkg_rwstat_add_aux(&to->wait_time, &from->wait_time);
728 blkg_stat_add_aux(&from->time, &from->time);
Tejun Heo0b399202013-01-09 08:05:13 -0800729#ifdef CONFIG_DEBUG_BLK_CGROUP
Tejun Heoe6269c42015-08-18 14:55:21 -0700730 blkg_stat_add_aux(&to->unaccounted_time, &from->unaccounted_time);
731 blkg_stat_add_aux(&to->avg_queue_size_sum, &from->avg_queue_size_sum);
732 blkg_stat_add_aux(&to->avg_queue_size_samples, &from->avg_queue_size_samples);
733 blkg_stat_add_aux(&to->dequeue, &from->dequeue);
734 blkg_stat_add_aux(&to->group_wait_time, &from->group_wait_time);
735 blkg_stat_add_aux(&to->idle_time, &from->idle_time);
736 blkg_stat_add_aux(&to->empty_time, &from->empty_time);
Tejun Heo0b399202013-01-09 08:05:13 -0800737#endif
738}
739
740/*
Tejun Heoe6269c42015-08-18 14:55:21 -0700741 * Transfer @cfqg's stats to its parent's aux counts so that the ancestors'
Tejun Heo0b399202013-01-09 08:05:13 -0800742 * recursive stats can still account for the amount used by this cfqg after
743 * it's gone.
744 */
745static void cfqg_stats_xfer_dead(struct cfq_group *cfqg)
746{
747 struct cfq_group *parent = cfqg_parent(cfqg);
748
749 lockdep_assert_held(cfqg_to_blkg(cfqg)->q->queue_lock);
750
751 if (unlikely(!parent))
752 return;
753
Tejun Heoe6269c42015-08-18 14:55:21 -0700754 cfqg_stats_add_aux(&parent->stats, &cfqg->stats);
Tejun Heo0b399202013-01-09 08:05:13 -0800755 cfqg_stats_reset(&cfqg->stats);
Tejun Heo0b399202013-01-09 08:05:13 -0800756}
757
Tejun Heoeb7d8c072012-03-23 14:02:53 +0100758#else /* CONFIG_CFQ_GROUP_IOSCHED */
759
Tejun Heod02f7aa2013-01-09 08:05:11 -0800760static inline struct cfq_group *cfqg_parent(struct cfq_group *cfqg) { return NULL; }
Tejun Heoeb7d8c072012-03-23 14:02:53 +0100761static inline void cfqg_get(struct cfq_group *cfqg) { }
762static inline void cfqg_put(struct cfq_group *cfqg) { }
763
Jens Axboe7b679132008-05-30 12:23:07 +0200764#define cfq_log_cfqq(cfqd, cfqq, fmt, args...) \
Vivek Goyalb226e5c2012-10-03 16:57:01 -0400765 blk_add_trace_msg((cfqd)->queue, "cfq%d%c%c " fmt, (cfqq)->pid, \
766 cfq_cfqq_sync((cfqq)) ? 'S' : 'A', \
767 cfqq_type((cfqq)) == SYNC_NOIDLE_WORKLOAD ? 'N' : ' ',\
768 ##args)
Kyungmin Park4495a7d2011-05-31 10:04:09 +0200769#define cfq_log_cfqg(cfqd, cfqg, fmt, args...) do {} while (0)
Tejun Heoeb7d8c072012-03-23 14:02:53 +0100770
Tejun Heo155fead2012-04-01 14:38:44 -0700771static inline void cfqg_stats_update_io_add(struct cfq_group *cfqg,
772 struct cfq_group *curr_cfqg, int rw) { }
773static inline void cfqg_stats_update_timeslice_used(struct cfq_group *cfqg,
774 unsigned long time, unsigned long unaccounted_time) { }
775static inline void cfqg_stats_update_io_remove(struct cfq_group *cfqg, int rw) { }
776static inline void cfqg_stats_update_io_merged(struct cfq_group *cfqg, int rw) { }
Tejun Heo155fead2012-04-01 14:38:44 -0700777static inline void cfqg_stats_update_completion(struct cfq_group *cfqg,
778 uint64_t start_time, uint64_t io_start_time, int rw) { }
Tejun Heo2ce4d502012-04-01 14:38:43 -0700779
Tejun Heoeb7d8c072012-03-23 14:02:53 +0100780#endif /* CONFIG_CFQ_GROUP_IOSCHED */
781
Jens Axboe7b679132008-05-30 12:23:07 +0200782#define cfq_log(cfqd, fmt, args...) \
783 blk_add_trace_msg((cfqd)->queue, "cfq " fmt, ##args)
784
Vivek Goyal615f0252009-12-03 12:59:39 -0500785/* Traverses through cfq group service trees */
786#define for_each_cfqg_st(cfqg, i, j, st) \
787 for (i = 0; i <= IDLE_WORKLOAD; i++) \
788 for (j = 0, st = i < IDLE_WORKLOAD ? &cfqg->service_trees[i][j]\
789 : &cfqg->service_tree_idle; \
790 (i < IDLE_WORKLOAD && j <= SYNC_WORKLOAD) || \
791 (i == IDLE_WORKLOAD && j == 0); \
792 j++, st = i < IDLE_WORKLOAD ? \
793 &cfqg->service_trees[i][j]: NULL) \
794
Shaohua Lif5f2b6c2011-07-12 14:24:55 +0200795static inline bool cfq_io_thinktime_big(struct cfq_data *cfqd,
796 struct cfq_ttime *ttime, bool group_idle)
797{
798 unsigned long slice;
799 if (!sample_valid(ttime->ttime_samples))
800 return false;
801 if (group_idle)
802 slice = cfqd->cfq_group_idle;
803 else
804 slice = cfqd->cfq_slice_idle;
805 return ttime->ttime_mean > slice;
806}
Vivek Goyal615f0252009-12-03 12:59:39 -0500807
Vivek Goyal02b35082010-08-23 12:23:53 +0200808static inline bool iops_mode(struct cfq_data *cfqd)
809{
810 /*
811 * If we are not idling on queues and it is a NCQ drive, parallel
812 * execution of requests is on and measuring time is not possible
813 * in most of the cases until and unless we drive shallower queue
814 * depths and that becomes a performance bottleneck. In such cases
815 * switch to start providing fairness in terms of number of IOs.
816 */
817 if (!cfqd->cfq_slice_idle && cfqd->hw_tag)
818 return true;
819 else
820 return false;
821}
822
Vivek Goyal3bf10fe2012-10-03 16:56:56 -0400823static inline enum wl_class_t cfqq_class(struct cfq_queue *cfqq)
Corrado Zoccoloc0324a02009-10-27 19:16:03 +0100824{
825 if (cfq_class_idle(cfqq))
826 return IDLE_WORKLOAD;
827 if (cfq_class_rt(cfqq))
828 return RT_WORKLOAD;
829 return BE_WORKLOAD;
830}
831
Corrado Zoccolo718eee02009-10-26 22:45:29 +0100832
833static enum wl_type_t cfqq_type(struct cfq_queue *cfqq)
834{
835 if (!cfq_cfqq_sync(cfqq))
836 return ASYNC_WORKLOAD;
837 if (!cfq_cfqq_idle_window(cfqq))
838 return SYNC_NOIDLE_WORKLOAD;
839 return SYNC_WORKLOAD;
840}
841
Vivek Goyal3bf10fe2012-10-03 16:56:56 -0400842static inline int cfq_group_busy_queues_wl(enum wl_class_t wl_class,
Vivek Goyal58ff82f2009-12-03 12:59:44 -0500843 struct cfq_data *cfqd,
844 struct cfq_group *cfqg)
Corrado Zoccoloc0324a02009-10-27 19:16:03 +0100845{
Vivek Goyal3bf10fe2012-10-03 16:56:56 -0400846 if (wl_class == IDLE_WORKLOAD)
Vivek Goyalcdb16e82009-12-03 12:59:38 -0500847 return cfqg->service_tree_idle.count;
848
Vivek Goyal34b98d02012-10-03 16:56:58 -0400849 return cfqg->service_trees[wl_class][ASYNC_WORKLOAD].count +
850 cfqg->service_trees[wl_class][SYNC_NOIDLE_WORKLOAD].count +
851 cfqg->service_trees[wl_class][SYNC_WORKLOAD].count;
Corrado Zoccoloc0324a02009-10-27 19:16:03 +0100852}
853
Vivek Goyalf26bd1f2009-12-03 12:59:54 -0500854static inline int cfqg_busy_async_queues(struct cfq_data *cfqd,
855 struct cfq_group *cfqg)
856{
Vivek Goyal34b98d02012-10-03 16:56:58 -0400857 return cfqg->service_trees[RT_WORKLOAD][ASYNC_WORKLOAD].count +
858 cfqg->service_trees[BE_WORKLOAD][ASYNC_WORKLOAD].count;
Vivek Goyalf26bd1f2009-12-03 12:59:54 -0500859}
860
Jens Axboe165125e2007-07-24 09:28:11 +0200861static void cfq_dispatch_insert(struct request_queue *, struct request *);
Tejun Heo4f85cb92012-03-05 13:15:28 -0800862static struct cfq_queue *cfq_get_queue(struct cfq_data *cfqd, bool is_sync,
Tejun Heo2da8de02015-08-18 14:55:02 -0700863 struct cfq_io_cq *cic, struct bio *bio);
Vasily Tarasov91fac312007-04-25 12:29:51 +0200864
Tejun Heoc5869802011-12-14 00:33:41 +0100865static inline struct cfq_io_cq *icq_to_cic(struct io_cq *icq)
866{
867 /* cic->icq is the first member, %NULL will convert to %NULL */
868 return container_of(icq, struct cfq_io_cq, icq);
869}
870
Tejun Heo47fdd4c2011-12-14 00:33:42 +0100871static inline struct cfq_io_cq *cfq_cic_lookup(struct cfq_data *cfqd,
872 struct io_context *ioc)
873{
874 if (ioc)
875 return icq_to_cic(ioc_lookup_icq(ioc, cfqd->queue));
876 return NULL;
877}
878
Tejun Heoc5869802011-12-14 00:33:41 +0100879static inline struct cfq_queue *cic_to_cfqq(struct cfq_io_cq *cic, bool is_sync)
Vasily Tarasov91fac312007-04-25 12:29:51 +0200880{
Jens Axboea6151c32009-10-07 20:02:57 +0200881 return cic->cfqq[is_sync];
Vasily Tarasov91fac312007-04-25 12:29:51 +0200882}
883
Tejun Heoc5869802011-12-14 00:33:41 +0100884static inline void cic_set_cfqq(struct cfq_io_cq *cic, struct cfq_queue *cfqq,
885 bool is_sync)
Vasily Tarasov91fac312007-04-25 12:29:51 +0200886{
Jens Axboea6151c32009-10-07 20:02:57 +0200887 cic->cfqq[is_sync] = cfqq;
Vasily Tarasov91fac312007-04-25 12:29:51 +0200888}
889
Tejun Heoc5869802011-12-14 00:33:41 +0100890static inline struct cfq_data *cic_to_cfqd(struct cfq_io_cq *cic)
Konstantin Khlebnikovbca4b912010-05-20 23:21:34 +0400891{
Tejun Heoc5869802011-12-14 00:33:41 +0100892 return cic->icq.q->elevator->elevator_data;
Konstantin Khlebnikovbca4b912010-05-20 23:21:34 +0400893}
894
Vasily Tarasov91fac312007-04-25 12:29:51 +0200895/*
896 * We regard a request as SYNC, if it's either a read or has the SYNC bit
897 * set (in which case it could also be direct WRITE).
898 */
Jens Axboea6151c32009-10-07 20:02:57 +0200899static inline bool cfq_bio_sync(struct bio *bio)
Vasily Tarasov91fac312007-04-25 12:29:51 +0200900{
Christoph Hellwig7b6d91d2010-08-07 18:20:39 +0200901 return bio_data_dir(bio) == READ || (bio->bi_rw & REQ_SYNC);
Vasily Tarasov91fac312007-04-25 12:29:51 +0200902}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700903
Linus Torvalds1da177e2005-04-16 15:20:36 -0700904/*
Andrew Morton99f95e52005-06-27 20:14:05 -0700905 * scheduler run of queue, if there are requests pending and no one in the
906 * driver that will restart queueing
907 */
Jens Axboe23e018a2009-10-05 08:52:35 +0200908static inline void cfq_schedule_dispatch(struct cfq_data *cfqd)
Andrew Morton99f95e52005-06-27 20:14:05 -0700909{
Jens Axboe7b679132008-05-30 12:23:07 +0200910 if (cfqd->busy_queues) {
911 cfq_log(cfqd, "schedule dispatch");
Jens Axboe59c3d452014-04-08 09:15:35 -0600912 kblockd_schedule_work(&cfqd->unplug_work);
Jens Axboe7b679132008-05-30 12:23:07 +0200913 }
Andrew Morton99f95e52005-06-27 20:14:05 -0700914}
915
Linus Torvalds1da177e2005-04-16 15:20:36 -0700916/*
Jens Axboe44f7c162007-01-19 11:51:58 +1100917 * Scale schedule slice based on io priority. Use the sync time slice only
918 * if a queue is marked sync and has sync io queued. A sync queue with async
919 * io only, should not get full sync slice length.
920 */
Jens Axboea6151c32009-10-07 20:02:57 +0200921static inline int cfq_prio_slice(struct cfq_data *cfqd, bool sync,
Jens Axboed9e76202007-04-20 14:27:50 +0200922 unsigned short prio)
923{
924 const int base_slice = cfqd->cfq_slice[sync];
925
926 WARN_ON(prio >= IOPRIO_BE_NR);
927
928 return base_slice + (base_slice/CFQ_SLICE_SCALE * (4 - prio));
929}
930
Jens Axboe44f7c162007-01-19 11:51:58 +1100931static inline int
932cfq_prio_to_slice(struct cfq_data *cfqd, struct cfq_queue *cfqq)
933{
Jens Axboed9e76202007-04-20 14:27:50 +0200934 return cfq_prio_slice(cfqd, cfq_cfqq_sync(cfqq), cfqq->ioprio);
Jens Axboe44f7c162007-01-19 11:51:58 +1100935}
936
Tejun Heo1d3650f2013-01-09 08:05:11 -0800937/**
938 * cfqg_scale_charge - scale disk time charge according to cfqg weight
939 * @charge: disk time being charged
940 * @vfraction: vfraction of the cfqg, fixed point w/ CFQ_SERVICE_SHIFT
941 *
942 * Scale @charge according to @vfraction, which is in range (0, 1]. The
943 * scaling is inversely proportional.
944 *
945 * scaled = charge / vfraction
946 *
947 * The result is also in fixed point w/ CFQ_SERVICE_SHIFT.
948 */
949static inline u64 cfqg_scale_charge(unsigned long charge,
950 unsigned int vfraction)
Vivek Goyal25bc6b02009-12-03 12:59:43 -0500951{
Tejun Heo1d3650f2013-01-09 08:05:11 -0800952 u64 c = charge << CFQ_SERVICE_SHIFT; /* make it fixed point */
Vivek Goyal25bc6b02009-12-03 12:59:43 -0500953
Tejun Heo1d3650f2013-01-09 08:05:11 -0800954 /* charge / vfraction */
955 c <<= CFQ_SERVICE_SHIFT;
956 do_div(c, vfraction);
957 return c;
Vivek Goyal25bc6b02009-12-03 12:59:43 -0500958}
959
960static inline u64 max_vdisktime(u64 min_vdisktime, u64 vdisktime)
961{
962 s64 delta = (s64)(vdisktime - min_vdisktime);
963 if (delta > 0)
964 min_vdisktime = vdisktime;
965
966 return min_vdisktime;
967}
968
969static inline u64 min_vdisktime(u64 min_vdisktime, u64 vdisktime)
970{
971 s64 delta = (s64)(vdisktime - min_vdisktime);
972 if (delta < 0)
973 min_vdisktime = vdisktime;
974
975 return min_vdisktime;
976}
977
978static void update_min_vdisktime(struct cfq_rb_root *st)
979{
Vivek Goyal25bc6b02009-12-03 12:59:43 -0500980 struct cfq_group *cfqg;
981
Vivek Goyal25bc6b02009-12-03 12:59:43 -0500982 if (st->left) {
983 cfqg = rb_entry_cfqg(st->left);
Gui Jianfenga6032712011-03-07 09:28:09 +0100984 st->min_vdisktime = max_vdisktime(st->min_vdisktime,
985 cfqg->vdisktime);
Vivek Goyal25bc6b02009-12-03 12:59:43 -0500986 }
Vivek Goyal25bc6b02009-12-03 12:59:43 -0500987}
988
Corrado Zoccolo5db5d642009-10-26 22:44:04 +0100989/*
990 * get averaged number of queues of RT/BE priority.
991 * average is updated, with a formula that gives more weight to higher numbers,
992 * to quickly follows sudden increases and decrease slowly
993 */
994
Vivek Goyal58ff82f2009-12-03 12:59:44 -0500995static inline unsigned cfq_group_get_avg_queues(struct cfq_data *cfqd,
996 struct cfq_group *cfqg, bool rt)
Jens Axboe5869619c2009-10-28 09:27:07 +0100997{
Corrado Zoccolo5db5d642009-10-26 22:44:04 +0100998 unsigned min_q, max_q;
999 unsigned mult = cfq_hist_divisor - 1;
1000 unsigned round = cfq_hist_divisor / 2;
Vivek Goyal58ff82f2009-12-03 12:59:44 -05001001 unsigned busy = cfq_group_busy_queues_wl(rt, cfqd, cfqg);
Corrado Zoccolo5db5d642009-10-26 22:44:04 +01001002
Vivek Goyal58ff82f2009-12-03 12:59:44 -05001003 min_q = min(cfqg->busy_queues_avg[rt], busy);
1004 max_q = max(cfqg->busy_queues_avg[rt], busy);
1005 cfqg->busy_queues_avg[rt] = (mult * max_q + min_q + round) /
Corrado Zoccolo5db5d642009-10-26 22:44:04 +01001006 cfq_hist_divisor;
Vivek Goyal58ff82f2009-12-03 12:59:44 -05001007 return cfqg->busy_queues_avg[rt];
1008}
1009
1010static inline unsigned
1011cfq_group_slice(struct cfq_data *cfqd, struct cfq_group *cfqg)
1012{
Tejun Heo41cad6a2013-01-09 08:05:11 -08001013 return cfqd->cfq_target_latency * cfqg->vfraction >> CFQ_SERVICE_SHIFT;
Corrado Zoccolo5db5d642009-10-26 22:44:04 +01001014}
1015
Shaohua Lic553f8e2011-01-14 08:41:03 +01001016static inline unsigned
Vivek Goyalba5bd522011-01-19 08:25:02 -07001017cfq_scaled_cfqq_slice(struct cfq_data *cfqd, struct cfq_queue *cfqq)
Jens Axboe44f7c162007-01-19 11:51:58 +11001018{
Corrado Zoccolo5db5d642009-10-26 22:44:04 +01001019 unsigned slice = cfq_prio_to_slice(cfqd, cfqq);
1020 if (cfqd->cfq_latency) {
Vivek Goyal58ff82f2009-12-03 12:59:44 -05001021 /*
1022 * interested queues (we consider only the ones with the same
1023 * priority class in the cfq group)
1024 */
1025 unsigned iq = cfq_group_get_avg_queues(cfqd, cfqq->cfqg,
1026 cfq_class_rt(cfqq));
Corrado Zoccolo5db5d642009-10-26 22:44:04 +01001027 unsigned sync_slice = cfqd->cfq_slice[1];
1028 unsigned expect_latency = sync_slice * iq;
Vivek Goyal58ff82f2009-12-03 12:59:44 -05001029 unsigned group_slice = cfq_group_slice(cfqd, cfqq->cfqg);
1030
1031 if (expect_latency > group_slice) {
Corrado Zoccolo5db5d642009-10-26 22:44:04 +01001032 unsigned base_low_slice = 2 * cfqd->cfq_slice_idle;
1033 /* scale low_slice according to IO priority
1034 * and sync vs async */
1035 unsigned low_slice =
1036 min(slice, base_low_slice * slice / sync_slice);
1037 /* the adapted slice value is scaled to fit all iqs
1038 * into the target latency */
Vivek Goyal58ff82f2009-12-03 12:59:44 -05001039 slice = max(slice * group_slice / expect_latency,
Corrado Zoccolo5db5d642009-10-26 22:44:04 +01001040 low_slice);
1041 }
1042 }
Shaohua Lic553f8e2011-01-14 08:41:03 +01001043 return slice;
1044}
1045
1046static inline void
1047cfq_set_prio_slice(struct cfq_data *cfqd, struct cfq_queue *cfqq)
1048{
Vivek Goyalba5bd522011-01-19 08:25:02 -07001049 unsigned slice = cfq_scaled_cfqq_slice(cfqd, cfqq);
Shaohua Lic553f8e2011-01-14 08:41:03 +01001050
Vivek Goyaldae739e2009-12-03 12:59:45 -05001051 cfqq->slice_start = jiffies;
Corrado Zoccolo5db5d642009-10-26 22:44:04 +01001052 cfqq->slice_end = jiffies + slice;
Vivek Goyalf75edf22009-12-03 12:59:53 -05001053 cfqq->allocated_slice = slice;
Jens Axboe7b679132008-05-30 12:23:07 +02001054 cfq_log_cfqq(cfqd, cfqq, "set_slice=%lu", cfqq->slice_end - jiffies);
Jens Axboe44f7c162007-01-19 11:51:58 +11001055}
1056
1057/*
1058 * We need to wrap this check in cfq_cfqq_slice_new(), since ->slice_end
1059 * isn't valid until the first request from the dispatch is activated
1060 * and the slice time set.
1061 */
Jens Axboea6151c32009-10-07 20:02:57 +02001062static inline bool cfq_slice_used(struct cfq_queue *cfqq)
Jens Axboe44f7c162007-01-19 11:51:58 +11001063{
1064 if (cfq_cfqq_slice_new(cfqq))
Shaohua Lic1e44752010-11-08 15:01:02 +01001065 return false;
Jens Axboe44f7c162007-01-19 11:51:58 +11001066 if (time_before(jiffies, cfqq->slice_end))
Shaohua Lic1e44752010-11-08 15:01:02 +01001067 return false;
Jens Axboe44f7c162007-01-19 11:51:58 +11001068
Shaohua Lic1e44752010-11-08 15:01:02 +01001069 return true;
Jens Axboe44f7c162007-01-19 11:51:58 +11001070}
1071
1072/*
Jens Axboe5e705372006-07-13 12:39:25 +02001073 * Lifted from AS - choose which of rq1 and rq2 that is best served now.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001074 * We choose the request that is closest to the head right now. Distance
Andreas Mohre8a99052006-03-28 08:59:49 +02001075 * behind the head is penalized and only allowed to a certain extent.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001076 */
Jens Axboe5e705372006-07-13 12:39:25 +02001077static struct request *
Corrado Zoccolocf7c25c2009-11-08 17:16:46 +01001078cfq_choose_req(struct cfq_data *cfqd, struct request *rq1, struct request *rq2, sector_t last)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001079{
Corrado Zoccolocf7c25c2009-11-08 17:16:46 +01001080 sector_t s1, s2, d1 = 0, d2 = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001081 unsigned long back_max;
Andreas Mohre8a99052006-03-28 08:59:49 +02001082#define CFQ_RQ1_WRAP 0x01 /* request 1 wraps */
1083#define CFQ_RQ2_WRAP 0x02 /* request 2 wraps */
1084 unsigned wrap = 0; /* bit mask: requests behind the disk head? */
Linus Torvalds1da177e2005-04-16 15:20:36 -07001085
Jens Axboe5e705372006-07-13 12:39:25 +02001086 if (rq1 == NULL || rq1 == rq2)
1087 return rq2;
1088 if (rq2 == NULL)
1089 return rq1;
Jens Axboe9c2c38a2005-08-24 14:57:54 +02001090
Namhyung Kim229836b2011-05-24 10:23:21 +02001091 if (rq_is_sync(rq1) != rq_is_sync(rq2))
1092 return rq_is_sync(rq1) ? rq1 : rq2;
1093
Christoph Hellwig65299a32011-08-23 14:50:29 +02001094 if ((rq1->cmd_flags ^ rq2->cmd_flags) & REQ_PRIO)
1095 return rq1->cmd_flags & REQ_PRIO ? rq1 : rq2;
Jens Axboeb53d1ed2011-08-19 08:34:48 +02001096
Tejun Heo83096eb2009-05-07 22:24:39 +09001097 s1 = blk_rq_pos(rq1);
1098 s2 = blk_rq_pos(rq2);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001099
Linus Torvalds1da177e2005-04-16 15:20:36 -07001100 /*
1101 * by definition, 1KiB is 2 sectors
1102 */
1103 back_max = cfqd->cfq_back_max * 2;
1104
1105 /*
1106 * Strict one way elevator _except_ in the case where we allow
1107 * short backward seeks which are biased as twice the cost of a
1108 * similar forward seek.
1109 */
1110 if (s1 >= last)
1111 d1 = s1 - last;
1112 else if (s1 + back_max >= last)
1113 d1 = (last - s1) * cfqd->cfq_back_penalty;
1114 else
Andreas Mohre8a99052006-03-28 08:59:49 +02001115 wrap |= CFQ_RQ1_WRAP;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001116
1117 if (s2 >= last)
1118 d2 = s2 - last;
1119 else if (s2 + back_max >= last)
1120 d2 = (last - s2) * cfqd->cfq_back_penalty;
1121 else
Andreas Mohre8a99052006-03-28 08:59:49 +02001122 wrap |= CFQ_RQ2_WRAP;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001123
1124 /* Found required data */
Linus Torvalds1da177e2005-04-16 15:20:36 -07001125
Andreas Mohre8a99052006-03-28 08:59:49 +02001126 /*
1127 * By doing switch() on the bit mask "wrap" we avoid having to
1128 * check two variables for all permutations: --> faster!
1129 */
1130 switch (wrap) {
Jens Axboe5e705372006-07-13 12:39:25 +02001131 case 0: /* common case for CFQ: rq1 and rq2 not wrapped */
Andreas Mohre8a99052006-03-28 08:59:49 +02001132 if (d1 < d2)
Jens Axboe5e705372006-07-13 12:39:25 +02001133 return rq1;
Andreas Mohre8a99052006-03-28 08:59:49 +02001134 else if (d2 < d1)
Jens Axboe5e705372006-07-13 12:39:25 +02001135 return rq2;
Andreas Mohre8a99052006-03-28 08:59:49 +02001136 else {
1137 if (s1 >= s2)
Jens Axboe5e705372006-07-13 12:39:25 +02001138 return rq1;
Andreas Mohre8a99052006-03-28 08:59:49 +02001139 else
Jens Axboe5e705372006-07-13 12:39:25 +02001140 return rq2;
Andreas Mohre8a99052006-03-28 08:59:49 +02001141 }
1142
1143 case CFQ_RQ2_WRAP:
Jens Axboe5e705372006-07-13 12:39:25 +02001144 return rq1;
Andreas Mohre8a99052006-03-28 08:59:49 +02001145 case CFQ_RQ1_WRAP:
Jens Axboe5e705372006-07-13 12:39:25 +02001146 return rq2;
1147 case (CFQ_RQ1_WRAP|CFQ_RQ2_WRAP): /* both rqs wrapped */
Andreas Mohre8a99052006-03-28 08:59:49 +02001148 default:
1149 /*
1150 * Since both rqs are wrapped,
1151 * start with the one that's further behind head
1152 * (--> only *one* back seek required),
1153 * since back seek takes more time than forward.
1154 */
1155 if (s1 <= s2)
Jens Axboe5e705372006-07-13 12:39:25 +02001156 return rq1;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001157 else
Jens Axboe5e705372006-07-13 12:39:25 +02001158 return rq2;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001159 }
1160}
1161
Jens Axboe498d3aa22007-04-26 12:54:48 +02001162/*
1163 * The below is leftmost cache rbtree addon
1164 */
Jens Axboe08717142008-01-28 11:38:15 +01001165static struct cfq_queue *cfq_rb_first(struct cfq_rb_root *root)
Jens Axboecc09e292007-04-26 12:53:50 +02001166{
Vivek Goyal615f0252009-12-03 12:59:39 -05001167 /* Service tree is empty */
1168 if (!root->count)
1169 return NULL;
1170
Jens Axboecc09e292007-04-26 12:53:50 +02001171 if (!root->left)
1172 root->left = rb_first(&root->rb);
1173
Jens Axboe08717142008-01-28 11:38:15 +01001174 if (root->left)
1175 return rb_entry(root->left, struct cfq_queue, rb_node);
1176
1177 return NULL;
Jens Axboecc09e292007-04-26 12:53:50 +02001178}
1179
Vivek Goyal1fa8f6d2009-12-03 12:59:41 -05001180static struct cfq_group *cfq_rb_first_group(struct cfq_rb_root *root)
1181{
1182 if (!root->left)
1183 root->left = rb_first(&root->rb);
1184
1185 if (root->left)
1186 return rb_entry_cfqg(root->left);
1187
1188 return NULL;
1189}
1190
Jens Axboea36e71f2009-04-15 12:15:11 +02001191static void rb_erase_init(struct rb_node *n, struct rb_root *root)
1192{
1193 rb_erase(n, root);
1194 RB_CLEAR_NODE(n);
1195}
1196
Jens Axboecc09e292007-04-26 12:53:50 +02001197static void cfq_rb_erase(struct rb_node *n, struct cfq_rb_root *root)
1198{
1199 if (root->left == n)
1200 root->left = NULL;
Jens Axboea36e71f2009-04-15 12:15:11 +02001201 rb_erase_init(n, &root->rb);
Corrado Zoccoloaa6f6a32009-10-26 22:44:33 +01001202 --root->count;
Jens Axboecc09e292007-04-26 12:53:50 +02001203}
1204
Linus Torvalds1da177e2005-04-16 15:20:36 -07001205/*
1206 * would be nice to take fifo expire time into account as well
1207 */
Jens Axboe5e705372006-07-13 12:39:25 +02001208static struct request *
1209cfq_find_next_rq(struct cfq_data *cfqd, struct cfq_queue *cfqq,
1210 struct request *last)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001211{
Jens Axboe21183b02006-07-13 12:33:14 +02001212 struct rb_node *rbnext = rb_next(&last->rb_node);
1213 struct rb_node *rbprev = rb_prev(&last->rb_node);
Jens Axboe5e705372006-07-13 12:39:25 +02001214 struct request *next = NULL, *prev = NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001215
Jens Axboe21183b02006-07-13 12:33:14 +02001216 BUG_ON(RB_EMPTY_NODE(&last->rb_node));
Linus Torvalds1da177e2005-04-16 15:20:36 -07001217
1218 if (rbprev)
Jens Axboe5e705372006-07-13 12:39:25 +02001219 prev = rb_entry_rq(rbprev);
Jens Axboe21183b02006-07-13 12:33:14 +02001220
Linus Torvalds1da177e2005-04-16 15:20:36 -07001221 if (rbnext)
Jens Axboe5e705372006-07-13 12:39:25 +02001222 next = rb_entry_rq(rbnext);
Jens Axboe21183b02006-07-13 12:33:14 +02001223 else {
1224 rbnext = rb_first(&cfqq->sort_list);
1225 if (rbnext && rbnext != &last->rb_node)
Jens Axboe5e705372006-07-13 12:39:25 +02001226 next = rb_entry_rq(rbnext);
Jens Axboe21183b02006-07-13 12:33:14 +02001227 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001228
Corrado Zoccolocf7c25c2009-11-08 17:16:46 +01001229 return cfq_choose_req(cfqd, next, prev, blk_rq_pos(last));
Linus Torvalds1da177e2005-04-16 15:20:36 -07001230}
1231
Jens Axboed9e76202007-04-20 14:27:50 +02001232static unsigned long cfq_slice_offset(struct cfq_data *cfqd,
1233 struct cfq_queue *cfqq)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001234{
Jens Axboed9e76202007-04-20 14:27:50 +02001235 /*
1236 * just an approximation, should be ok.
1237 */
Vivek Goyalcdb16e82009-12-03 12:59:38 -05001238 return (cfqq->cfqg->nr_cfqq - 1) * (cfq_prio_slice(cfqd, 1, 0) -
Jens Axboe464191c2009-11-30 09:38:13 +01001239 cfq_prio_slice(cfqd, cfq_cfqq_sync(cfqq), cfqq->ioprio));
Jens Axboed9e76202007-04-20 14:27:50 +02001240}
Linus Torvalds1da177e2005-04-16 15:20:36 -07001241
Vivek Goyal1fa8f6d2009-12-03 12:59:41 -05001242static inline s64
1243cfqg_key(struct cfq_rb_root *st, struct cfq_group *cfqg)
1244{
1245 return cfqg->vdisktime - st->min_vdisktime;
1246}
1247
1248static void
1249__cfq_group_service_tree_add(struct cfq_rb_root *st, struct cfq_group *cfqg)
1250{
1251 struct rb_node **node = &st->rb.rb_node;
1252 struct rb_node *parent = NULL;
1253 struct cfq_group *__cfqg;
1254 s64 key = cfqg_key(st, cfqg);
1255 int left = 1;
1256
1257 while (*node != NULL) {
1258 parent = *node;
1259 __cfqg = rb_entry_cfqg(parent);
1260
1261 if (key < cfqg_key(st, __cfqg))
1262 node = &parent->rb_left;
1263 else {
1264 node = &parent->rb_right;
1265 left = 0;
1266 }
1267 }
1268
1269 if (left)
1270 st->left = &cfqg->rb_node;
1271
1272 rb_link_node(&cfqg->rb_node, parent, node);
1273 rb_insert_color(&cfqg->rb_node, &st->rb);
1274}
1275
Toshiaki Makita7b5af5c2014-08-28 17:14:58 +09001276/*
1277 * This has to be called only on activation of cfqg
1278 */
Vivek Goyal1fa8f6d2009-12-03 12:59:41 -05001279static void
Justin TerAvest8184f932011-03-17 16:12:36 +01001280cfq_update_group_weight(struct cfq_group *cfqg)
1281{
Tejun Heo3381cb82012-04-01 14:38:44 -07001282 if (cfqg->new_weight) {
Justin TerAvest8184f932011-03-17 16:12:36 +01001283 cfqg->weight = cfqg->new_weight;
Tejun Heo3381cb82012-04-01 14:38:44 -07001284 cfqg->new_weight = 0;
Justin TerAvest8184f932011-03-17 16:12:36 +01001285 }
Toshiaki Makitae15693e2014-08-26 20:56:36 +09001286}
1287
1288static void
1289cfq_update_group_leaf_weight(struct cfq_group *cfqg)
1290{
1291 BUG_ON(!RB_EMPTY_NODE(&cfqg->rb_node));
Tejun Heoe71357e2013-01-09 08:05:10 -08001292
1293 if (cfqg->new_leaf_weight) {
1294 cfqg->leaf_weight = cfqg->new_leaf_weight;
1295 cfqg->new_leaf_weight = 0;
1296 }
Justin TerAvest8184f932011-03-17 16:12:36 +01001297}
1298
1299static void
1300cfq_group_service_tree_add(struct cfq_rb_root *st, struct cfq_group *cfqg)
1301{
Tejun Heo1d3650f2013-01-09 08:05:11 -08001302 unsigned int vfr = 1 << CFQ_SERVICE_SHIFT; /* start with 1 */
Tejun Heo7918ffb2013-01-09 08:05:11 -08001303 struct cfq_group *pos = cfqg;
Tejun Heo1d3650f2013-01-09 08:05:11 -08001304 struct cfq_group *parent;
Tejun Heo7918ffb2013-01-09 08:05:11 -08001305 bool propagate;
1306
1307 /* add to the service tree */
Justin TerAvest8184f932011-03-17 16:12:36 +01001308 BUG_ON(!RB_EMPTY_NODE(&cfqg->rb_node));
1309
Toshiaki Makita7b5af5c2014-08-28 17:14:58 +09001310 /*
1311 * Update leaf_weight. We cannot update weight at this point
1312 * because cfqg might already have been activated and is
1313 * contributing its current weight to the parent's child_weight.
1314 */
Toshiaki Makitae15693e2014-08-26 20:56:36 +09001315 cfq_update_group_leaf_weight(cfqg);
Justin TerAvest8184f932011-03-17 16:12:36 +01001316 __cfq_group_service_tree_add(st, cfqg);
Tejun Heo7918ffb2013-01-09 08:05:11 -08001317
1318 /*
Tejun Heo1d3650f2013-01-09 08:05:11 -08001319 * Activate @cfqg and calculate the portion of vfraction @cfqg is
1320 * entitled to. vfraction is calculated by walking the tree
1321 * towards the root calculating the fraction it has at each level.
1322 * The compounded ratio is how much vfraction @cfqg owns.
1323 *
1324 * Start with the proportion tasks in this cfqg has against active
1325 * children cfqgs - its leaf_weight against children_weight.
Tejun Heo7918ffb2013-01-09 08:05:11 -08001326 */
1327 propagate = !pos->nr_active++;
1328 pos->children_weight += pos->leaf_weight;
Tejun Heo1d3650f2013-01-09 08:05:11 -08001329 vfr = vfr * pos->leaf_weight / pos->children_weight;
Tejun Heo7918ffb2013-01-09 08:05:11 -08001330
Tejun Heo1d3650f2013-01-09 08:05:11 -08001331 /*
1332 * Compound ->weight walking up the tree. Both activation and
1333 * vfraction calculation are done in the same loop. Propagation
1334 * stops once an already activated node is met. vfraction
1335 * calculation should always continue to the root.
1336 */
Tejun Heod02f7aa2013-01-09 08:05:11 -08001337 while ((parent = cfqg_parent(pos))) {
Tejun Heo1d3650f2013-01-09 08:05:11 -08001338 if (propagate) {
Toshiaki Makitae15693e2014-08-26 20:56:36 +09001339 cfq_update_group_weight(pos);
Tejun Heo1d3650f2013-01-09 08:05:11 -08001340 propagate = !parent->nr_active++;
1341 parent->children_weight += pos->weight;
1342 }
1343 vfr = vfr * pos->weight / parent->children_weight;
Tejun Heo7918ffb2013-01-09 08:05:11 -08001344 pos = parent;
1345 }
Tejun Heo1d3650f2013-01-09 08:05:11 -08001346
1347 cfqg->vfraction = max_t(unsigned, vfr, 1);
Justin TerAvest8184f932011-03-17 16:12:36 +01001348}
1349
1350static void
1351cfq_group_notify_queue_add(struct cfq_data *cfqd, struct cfq_group *cfqg)
Vivek Goyal1fa8f6d2009-12-03 12:59:41 -05001352{
1353 struct cfq_rb_root *st = &cfqd->grp_service_tree;
1354 struct cfq_group *__cfqg;
1355 struct rb_node *n;
1356
1357 cfqg->nr_cfqq++;
Gui Jianfeng760701b2010-11-30 20:52:47 +01001358 if (!RB_EMPTY_NODE(&cfqg->rb_node))
Vivek Goyal1fa8f6d2009-12-03 12:59:41 -05001359 return;
1360
1361 /*
1362 * Currently put the group at the end. Later implement something
1363 * so that groups get lesser vtime based on their weights, so that
Lucas De Marchi25985ed2011-03-30 22:57:33 -03001364 * if group does not loose all if it was not continuously backlogged.
Vivek Goyal1fa8f6d2009-12-03 12:59:41 -05001365 */
1366 n = rb_last(&st->rb);
1367 if (n) {
1368 __cfqg = rb_entry_cfqg(n);
1369 cfqg->vdisktime = __cfqg->vdisktime + CFQ_IDLE_DELAY;
1370 } else
1371 cfqg->vdisktime = st->min_vdisktime;
Justin TerAvest8184f932011-03-17 16:12:36 +01001372 cfq_group_service_tree_add(st, cfqg);
Vivek Goyal1fa8f6d2009-12-03 12:59:41 -05001373}
1374
1375static void
Justin TerAvest8184f932011-03-17 16:12:36 +01001376cfq_group_service_tree_del(struct cfq_rb_root *st, struct cfq_group *cfqg)
1377{
Tejun Heo7918ffb2013-01-09 08:05:11 -08001378 struct cfq_group *pos = cfqg;
1379 bool propagate;
1380
1381 /*
1382 * Undo activation from cfq_group_service_tree_add(). Deactivate
1383 * @cfqg and propagate deactivation upwards.
1384 */
1385 propagate = !--pos->nr_active;
1386 pos->children_weight -= pos->leaf_weight;
1387
1388 while (propagate) {
Tejun Heod02f7aa2013-01-09 08:05:11 -08001389 struct cfq_group *parent = cfqg_parent(pos);
Tejun Heo7918ffb2013-01-09 08:05:11 -08001390
1391 /* @pos has 0 nr_active at this point */
1392 WARN_ON_ONCE(pos->children_weight);
Tejun Heo1d3650f2013-01-09 08:05:11 -08001393 pos->vfraction = 0;
Tejun Heo7918ffb2013-01-09 08:05:11 -08001394
1395 if (!parent)
1396 break;
1397
1398 propagate = !--parent->nr_active;
1399 parent->children_weight -= pos->weight;
1400 pos = parent;
1401 }
1402
1403 /* remove from the service tree */
Justin TerAvest8184f932011-03-17 16:12:36 +01001404 if (!RB_EMPTY_NODE(&cfqg->rb_node))
1405 cfq_rb_erase(&cfqg->rb_node, st);
1406}
1407
1408static void
1409cfq_group_notify_queue_del(struct cfq_data *cfqd, struct cfq_group *cfqg)
Vivek Goyal1fa8f6d2009-12-03 12:59:41 -05001410{
1411 struct cfq_rb_root *st = &cfqd->grp_service_tree;
1412
1413 BUG_ON(cfqg->nr_cfqq < 1);
1414 cfqg->nr_cfqq--;
Vivek Goyal25bc6b02009-12-03 12:59:43 -05001415
Vivek Goyal1fa8f6d2009-12-03 12:59:41 -05001416 /* If there are other cfq queues under this group, don't delete it */
1417 if (cfqg->nr_cfqq)
1418 return;
1419
Vivek Goyal2868ef72009-12-03 12:59:48 -05001420 cfq_log_cfqg(cfqd, cfqg, "del_from_rr group");
Justin TerAvest8184f932011-03-17 16:12:36 +01001421 cfq_group_service_tree_del(st, cfqg);
Vivek Goyal4d2ceea2012-10-03 16:56:57 -04001422 cfqg->saved_wl_slice = 0;
Tejun Heo155fead2012-04-01 14:38:44 -07001423 cfqg_stats_update_dequeue(cfqg);
Vivek Goyaldae739e2009-12-03 12:59:45 -05001424}
1425
Justin TerAvest167400d2011-03-12 16:54:00 +01001426static inline unsigned int cfq_cfqq_slice_usage(struct cfq_queue *cfqq,
1427 unsigned int *unaccounted_time)
Vivek Goyaldae739e2009-12-03 12:59:45 -05001428{
Vivek Goyalf75edf22009-12-03 12:59:53 -05001429 unsigned int slice_used;
Vivek Goyaldae739e2009-12-03 12:59:45 -05001430
1431 /*
1432 * Queue got expired before even a single request completed or
1433 * got expired immediately after first request completion.
1434 */
1435 if (!cfqq->slice_start || cfqq->slice_start == jiffies) {
1436 /*
1437 * Also charge the seek time incurred to the group, otherwise
1438 * if there are mutiple queues in the group, each can dispatch
1439 * a single request on seeky media and cause lots of seek time
1440 * and group will never know it.
1441 */
1442 slice_used = max_t(unsigned, (jiffies - cfqq->dispatch_start),
1443 1);
1444 } else {
1445 slice_used = jiffies - cfqq->slice_start;
Justin TerAvest167400d2011-03-12 16:54:00 +01001446 if (slice_used > cfqq->allocated_slice) {
1447 *unaccounted_time = slice_used - cfqq->allocated_slice;
Vivek Goyalf75edf22009-12-03 12:59:53 -05001448 slice_used = cfqq->allocated_slice;
Justin TerAvest167400d2011-03-12 16:54:00 +01001449 }
1450 if (time_after(cfqq->slice_start, cfqq->dispatch_start))
1451 *unaccounted_time += cfqq->slice_start -
1452 cfqq->dispatch_start;
Vivek Goyaldae739e2009-12-03 12:59:45 -05001453 }
1454
Vivek Goyaldae739e2009-12-03 12:59:45 -05001455 return slice_used;
1456}
1457
1458static void cfq_group_served(struct cfq_data *cfqd, struct cfq_group *cfqg,
Vivek Goyale5ff0822010-04-26 19:25:11 +02001459 struct cfq_queue *cfqq)
Vivek Goyaldae739e2009-12-03 12:59:45 -05001460{
1461 struct cfq_rb_root *st = &cfqd->grp_service_tree;
Justin TerAvest167400d2011-03-12 16:54:00 +01001462 unsigned int used_sl, charge, unaccounted_sl = 0;
Vivek Goyalf26bd1f2009-12-03 12:59:54 -05001463 int nr_sync = cfqg->nr_cfqq - cfqg_busy_async_queues(cfqd, cfqg)
1464 - cfqg->service_tree_idle.count;
Tejun Heo1d3650f2013-01-09 08:05:11 -08001465 unsigned int vfr;
Vivek Goyaldae739e2009-12-03 12:59:45 -05001466
Vivek Goyalf26bd1f2009-12-03 12:59:54 -05001467 BUG_ON(nr_sync < 0);
Justin TerAvest167400d2011-03-12 16:54:00 +01001468 used_sl = charge = cfq_cfqq_slice_usage(cfqq, &unaccounted_sl);
Vivek Goyalf26bd1f2009-12-03 12:59:54 -05001469
Vivek Goyal02b35082010-08-23 12:23:53 +02001470 if (iops_mode(cfqd))
1471 charge = cfqq->slice_dispatch;
1472 else if (!cfq_cfqq_sync(cfqq) && !nr_sync)
1473 charge = cfqq->allocated_slice;
Vivek Goyaldae739e2009-12-03 12:59:45 -05001474
Tejun Heo1d3650f2013-01-09 08:05:11 -08001475 /*
1476 * Can't update vdisktime while on service tree and cfqg->vfraction
1477 * is valid only while on it. Cache vfr, leave the service tree,
1478 * update vdisktime and go back on. The re-addition to the tree
1479 * will also update the weights as necessary.
1480 */
1481 vfr = cfqg->vfraction;
Justin TerAvest8184f932011-03-17 16:12:36 +01001482 cfq_group_service_tree_del(st, cfqg);
Tejun Heo1d3650f2013-01-09 08:05:11 -08001483 cfqg->vdisktime += cfqg_scale_charge(charge, vfr);
Justin TerAvest8184f932011-03-17 16:12:36 +01001484 cfq_group_service_tree_add(st, cfqg);
Vivek Goyaldae739e2009-12-03 12:59:45 -05001485
1486 /* This group is being expired. Save the context */
1487 if (time_after(cfqd->workload_expires, jiffies)) {
Vivek Goyal4d2ceea2012-10-03 16:56:57 -04001488 cfqg->saved_wl_slice = cfqd->workload_expires
Vivek Goyaldae739e2009-12-03 12:59:45 -05001489 - jiffies;
Vivek Goyal4d2ceea2012-10-03 16:56:57 -04001490 cfqg->saved_wl_type = cfqd->serving_wl_type;
1491 cfqg->saved_wl_class = cfqd->serving_wl_class;
Vivek Goyaldae739e2009-12-03 12:59:45 -05001492 } else
Vivek Goyal4d2ceea2012-10-03 16:56:57 -04001493 cfqg->saved_wl_slice = 0;
Vivek Goyal2868ef72009-12-03 12:59:48 -05001494
1495 cfq_log_cfqg(cfqd, cfqg, "served: vt=%llu min_vt=%llu", cfqg->vdisktime,
1496 st->min_vdisktime);
Joe Perchesfd16d262011-06-13 10:42:49 +02001497 cfq_log_cfqq(cfqq->cfqd, cfqq,
1498 "sl_used=%u disp=%u charge=%u iops=%u sect=%lu",
1499 used_sl, cfqq->slice_dispatch, charge,
1500 iops_mode(cfqd), cfqq->nr_sectors);
Tejun Heo155fead2012-04-01 14:38:44 -07001501 cfqg_stats_update_timeslice_used(cfqg, used_sl, unaccounted_sl);
1502 cfqg_stats_set_start_empty_time(cfqg);
Vivek Goyal1fa8f6d2009-12-03 12:59:41 -05001503}
1504
Tejun Heof51b8022012-03-05 13:15:05 -08001505/**
1506 * cfq_init_cfqg_base - initialize base part of a cfq_group
1507 * @cfqg: cfq_group to initialize
1508 *
1509 * Initialize the base part which is used whether %CONFIG_CFQ_GROUP_IOSCHED
1510 * is enabled or not.
1511 */
1512static void cfq_init_cfqg_base(struct cfq_group *cfqg)
1513{
1514 struct cfq_rb_root *st;
1515 int i, j;
1516
1517 for_each_cfqg_st(cfqg, i, j, st)
1518 *st = CFQ_RB_ROOT;
1519 RB_CLEAR_NODE(&cfqg->rb_node);
1520
1521 cfqg->ttime.last_end_request = jiffies;
1522}
1523
Vivek Goyal25fb5162009-12-03 12:59:46 -05001524#ifdef CONFIG_CFQ_GROUP_IOSCHED
Tejun Heo69d7fde2015-08-18 14:55:36 -07001525static int __cfq_set_weight(struct cgroup_subsys_state *css, u64 val,
1526 bool on_dfl, bool reset_dev, bool is_leaf_weight);
1527
Tejun Heo24bdb8e2015-08-18 14:55:22 -07001528static void cfqg_stats_exit(struct cfqg_stats *stats)
Peter Zijlstra90d38392013-11-12 19:42:14 -08001529{
Tejun Heo24bdb8e2015-08-18 14:55:22 -07001530 blkg_rwstat_exit(&stats->merged);
1531 blkg_rwstat_exit(&stats->service_time);
1532 blkg_rwstat_exit(&stats->wait_time);
1533 blkg_rwstat_exit(&stats->queued);
Tejun Heo24bdb8e2015-08-18 14:55:22 -07001534 blkg_stat_exit(&stats->time);
1535#ifdef CONFIG_DEBUG_BLK_CGROUP
1536 blkg_stat_exit(&stats->unaccounted_time);
1537 blkg_stat_exit(&stats->avg_queue_size_sum);
1538 blkg_stat_exit(&stats->avg_queue_size_samples);
1539 blkg_stat_exit(&stats->dequeue);
1540 blkg_stat_exit(&stats->group_wait_time);
1541 blkg_stat_exit(&stats->idle_time);
1542 blkg_stat_exit(&stats->empty_time);
1543#endif
1544}
1545
1546static int cfqg_stats_init(struct cfqg_stats *stats, gfp_t gfp)
1547{
Tejun Heo77ea7332015-08-18 14:55:24 -07001548 if (blkg_rwstat_init(&stats->merged, gfp) ||
Tejun Heo24bdb8e2015-08-18 14:55:22 -07001549 blkg_rwstat_init(&stats->service_time, gfp) ||
1550 blkg_rwstat_init(&stats->wait_time, gfp) ||
1551 blkg_rwstat_init(&stats->queued, gfp) ||
Tejun Heo24bdb8e2015-08-18 14:55:22 -07001552 blkg_stat_init(&stats->time, gfp))
1553 goto err;
Peter Zijlstra90d38392013-11-12 19:42:14 -08001554
1555#ifdef CONFIG_DEBUG_BLK_CGROUP
Tejun Heo24bdb8e2015-08-18 14:55:22 -07001556 if (blkg_stat_init(&stats->unaccounted_time, gfp) ||
1557 blkg_stat_init(&stats->avg_queue_size_sum, gfp) ||
1558 blkg_stat_init(&stats->avg_queue_size_samples, gfp) ||
1559 blkg_stat_init(&stats->dequeue, gfp) ||
1560 blkg_stat_init(&stats->group_wait_time, gfp) ||
1561 blkg_stat_init(&stats->idle_time, gfp) ||
1562 blkg_stat_init(&stats->empty_time, gfp))
1563 goto err;
Peter Zijlstra90d38392013-11-12 19:42:14 -08001564#endif
Tejun Heo24bdb8e2015-08-18 14:55:22 -07001565 return 0;
1566err:
1567 cfqg_stats_exit(stats);
1568 return -ENOMEM;
Peter Zijlstra90d38392013-11-12 19:42:14 -08001569}
1570
Tejun Heoe4a9bde2015-08-18 14:55:16 -07001571static struct blkcg_policy_data *cfq_cpd_alloc(gfp_t gfp)
1572{
1573 struct cfq_group_data *cgd;
1574
1575 cgd = kzalloc(sizeof(*cgd), GFP_KERNEL);
1576 if (!cgd)
1577 return NULL;
1578 return &cgd->cpd;
1579}
1580
Tejun Heo81437642015-08-18 14:55:15 -07001581static void cfq_cpd_init(struct blkcg_policy_data *cpd)
Arianna Avanzinie48453c2015-06-05 23:38:42 +02001582{
Tejun Heo81437642015-08-18 14:55:15 -07001583 struct cfq_group_data *cgd = cpd_to_cfqgd(cpd);
Tejun Heo69d7fde2015-08-18 14:55:36 -07001584 unsigned int weight = cgroup_on_dfl(blkcg_root.css.cgroup) ?
1585 CGROUP_WEIGHT_DFL : CFQ_WEIGHT_LEGACY_DFL;
Arianna Avanzinie48453c2015-06-05 23:38:42 +02001586
Tejun Heo69d7fde2015-08-18 14:55:36 -07001587 if (cpd_to_blkcg(cpd) == &blkcg_root)
1588 weight *= 2;
1589
1590 cgd->weight = weight;
1591 cgd->leaf_weight = weight;
Arianna Avanzinie48453c2015-06-05 23:38:42 +02001592}
1593
Tejun Heoe4a9bde2015-08-18 14:55:16 -07001594static void cfq_cpd_free(struct blkcg_policy_data *cpd)
1595{
1596 kfree(cpd_to_cfqgd(cpd));
1597}
1598
Tejun Heo69d7fde2015-08-18 14:55:36 -07001599static void cfq_cpd_bind(struct blkcg_policy_data *cpd)
1600{
1601 struct blkcg *blkcg = cpd_to_blkcg(cpd);
1602 bool on_dfl = cgroup_on_dfl(blkcg_root.css.cgroup);
1603 unsigned int weight = on_dfl ? CGROUP_WEIGHT_DFL : CFQ_WEIGHT_LEGACY_DFL;
1604
1605 if (blkcg == &blkcg_root)
1606 weight *= 2;
1607
1608 WARN_ON_ONCE(__cfq_set_weight(&blkcg->css, weight, on_dfl, true, false));
1609 WARN_ON_ONCE(__cfq_set_weight(&blkcg->css, weight, on_dfl, true, true));
1610}
1611
Tejun Heo001bea72015-08-18 14:55:11 -07001612static struct blkg_policy_data *cfq_pd_alloc(gfp_t gfp, int node)
1613{
Tejun Heob2ce2642015-08-18 14:55:13 -07001614 struct cfq_group *cfqg;
1615
1616 cfqg = kzalloc_node(sizeof(*cfqg), gfp, node);
1617 if (!cfqg)
1618 return NULL;
1619
1620 cfq_init_cfqg_base(cfqg);
Tejun Heo24bdb8e2015-08-18 14:55:22 -07001621 if (cfqg_stats_init(&cfqg->stats, gfp)) {
1622 kfree(cfqg);
1623 return NULL;
1624 }
Tejun Heob2ce2642015-08-18 14:55:13 -07001625
1626 return &cfqg->pd;
Tejun Heo001bea72015-08-18 14:55:11 -07001627}
1628
Tejun Heoa9520cd2015-08-18 14:55:14 -07001629static void cfq_pd_init(struct blkg_policy_data *pd)
Vivek Goyalf469a7b2011-05-19 15:38:23 -04001630{
Tejun Heoa9520cd2015-08-18 14:55:14 -07001631 struct cfq_group *cfqg = pd_to_cfqg(pd);
1632 struct cfq_group_data *cgd = blkcg_to_cfqgd(pd->blkg->blkcg);
Vivek Goyal25fb5162009-12-03 12:59:46 -05001633
Arianna Avanzinie48453c2015-06-05 23:38:42 +02001634 cfqg->weight = cgd->weight;
1635 cfqg->leaf_weight = cgd->leaf_weight;
Vivek Goyal25fb5162009-12-03 12:59:46 -05001636}
1637
Tejun Heoa9520cd2015-08-18 14:55:14 -07001638static void cfq_pd_offline(struct blkg_policy_data *pd)
Tejun Heo0b399202013-01-09 08:05:13 -08001639{
Tejun Heoa9520cd2015-08-18 14:55:14 -07001640 struct cfq_group *cfqg = pd_to_cfqg(pd);
Tejun Heo60a83702015-08-18 14:55:05 -07001641 int i;
1642
1643 for (i = 0; i < IOPRIO_BE_NR; i++) {
1644 if (cfqg->async_cfqq[0][i])
1645 cfq_put_queue(cfqg->async_cfqq[0][i]);
1646 if (cfqg->async_cfqq[1][i])
1647 cfq_put_queue(cfqg->async_cfqq[1][i]);
1648 }
1649
1650 if (cfqg->async_idle_cfqq)
1651 cfq_put_queue(cfqg->async_idle_cfqq);
1652
Tejun Heo0b399202013-01-09 08:05:13 -08001653 /*
1654 * @blkg is going offline and will be ignored by
1655 * blkg_[rw]stat_recursive_sum(). Transfer stats to the parent so
1656 * that they don't get lost. If IOs complete after this point, the
1657 * stats for them will be lost. Oh well...
1658 */
Tejun Heo60a83702015-08-18 14:55:05 -07001659 cfqg_stats_xfer_dead(cfqg);
Tejun Heo0b399202013-01-09 08:05:13 -08001660}
1661
Tejun Heo001bea72015-08-18 14:55:11 -07001662static void cfq_pd_free(struct blkg_policy_data *pd)
1663{
Tejun Heo24bdb8e2015-08-18 14:55:22 -07001664 struct cfq_group *cfqg = pd_to_cfqg(pd);
1665
1666 cfqg_stats_exit(&cfqg->stats);
1667 return kfree(cfqg);
Tejun Heo001bea72015-08-18 14:55:11 -07001668}
1669
Tejun Heoa9520cd2015-08-18 14:55:14 -07001670static void cfq_pd_reset_stats(struct blkg_policy_data *pd)
Tejun Heo689665a2013-01-09 08:05:13 -08001671{
Tejun Heoa9520cd2015-08-18 14:55:14 -07001672 struct cfq_group *cfqg = pd_to_cfqg(pd);
Tejun Heo689665a2013-01-09 08:05:13 -08001673
1674 cfqg_stats_reset(&cfqg->stats);
Vivek Goyal25fb5162009-12-03 12:59:46 -05001675}
1676
Tejun Heoae118892015-08-18 14:55:20 -07001677static struct cfq_group *cfq_lookup_cfqg(struct cfq_data *cfqd,
1678 struct blkcg *blkcg)
Vivek Goyal25fb5162009-12-03 12:59:46 -05001679{
Tejun Heoae118892015-08-18 14:55:20 -07001680 struct blkcg_gq *blkg;
Vivek Goyal25fb5162009-12-03 12:59:46 -05001681
Tejun Heoae118892015-08-18 14:55:20 -07001682 blkg = blkg_lookup(blkcg, cfqd->queue);
1683 if (likely(blkg))
1684 return blkg_to_cfqg(blkg);
1685 return NULL;
Vivek Goyal25fb5162009-12-03 12:59:46 -05001686}
1687
1688static void cfq_link_cfqq_cfqg(struct cfq_queue *cfqq, struct cfq_group *cfqg)
1689{
Vivek Goyal25fb5162009-12-03 12:59:46 -05001690 cfqq->cfqg = cfqg;
Vivek Goyalb1c35762009-12-03 12:59:47 -05001691 /* cfqq reference on cfqg */
Tejun Heoeb7d8c072012-03-23 14:02:53 +01001692 cfqg_get(cfqg);
Vivek Goyalb1c35762009-12-03 12:59:47 -05001693}
1694
Tejun Heof95a04a2012-04-16 13:57:26 -07001695static u64 cfqg_prfill_weight_device(struct seq_file *sf,
1696 struct blkg_policy_data *pd, int off)
Tejun Heo60c2bc22012-04-01 14:38:43 -07001697{
Tejun Heof95a04a2012-04-16 13:57:26 -07001698 struct cfq_group *cfqg = pd_to_cfqg(pd);
Tejun Heo3381cb82012-04-01 14:38:44 -07001699
1700 if (!cfqg->dev_weight)
Tejun Heo60c2bc22012-04-01 14:38:43 -07001701 return 0;
Tejun Heof95a04a2012-04-16 13:57:26 -07001702 return __blkg_prfill_u64(sf, pd, cfqg->dev_weight);
Tejun Heo60c2bc22012-04-01 14:38:43 -07001703}
1704
Tejun Heo2da8ca82013-12-05 12:28:04 -05001705static int cfqg_print_weight_device(struct seq_file *sf, void *v)
Tejun Heo60c2bc22012-04-01 14:38:43 -07001706{
Tejun Heo2da8ca82013-12-05 12:28:04 -05001707 blkcg_print_blkgs(sf, css_to_blkcg(seq_css(sf)),
1708 cfqg_prfill_weight_device, &blkcg_policy_cfq,
1709 0, false);
Tejun Heo60c2bc22012-04-01 14:38:43 -07001710 return 0;
1711}
1712
Tejun Heoe71357e2013-01-09 08:05:10 -08001713static u64 cfqg_prfill_leaf_weight_device(struct seq_file *sf,
1714 struct blkg_policy_data *pd, int off)
1715{
1716 struct cfq_group *cfqg = pd_to_cfqg(pd);
1717
1718 if (!cfqg->dev_leaf_weight)
1719 return 0;
1720 return __blkg_prfill_u64(sf, pd, cfqg->dev_leaf_weight);
1721}
1722
Tejun Heo2da8ca82013-12-05 12:28:04 -05001723static int cfqg_print_leaf_weight_device(struct seq_file *sf, void *v)
Tejun Heoe71357e2013-01-09 08:05:10 -08001724{
Tejun Heo2da8ca82013-12-05 12:28:04 -05001725 blkcg_print_blkgs(sf, css_to_blkcg(seq_css(sf)),
1726 cfqg_prfill_leaf_weight_device, &blkcg_policy_cfq,
1727 0, false);
Tejun Heoe71357e2013-01-09 08:05:10 -08001728 return 0;
1729}
1730
Tejun Heo2da8ca82013-12-05 12:28:04 -05001731static int cfq_print_weight(struct seq_file *sf, void *v)
Tejun Heo60c2bc22012-04-01 14:38:43 -07001732{
Arianna Avanzinie48453c2015-06-05 23:38:42 +02001733 struct blkcg *blkcg = css_to_blkcg(seq_css(sf));
Jens Axboe9470e4a2015-06-19 10:19:36 -06001734 struct cfq_group_data *cgd = blkcg_to_cfqgd(blkcg);
1735 unsigned int val = 0;
Arianna Avanzinie48453c2015-06-05 23:38:42 +02001736
Jens Axboe9470e4a2015-06-19 10:19:36 -06001737 if (cgd)
1738 val = cgd->weight;
1739
1740 seq_printf(sf, "%u\n", val);
Tejun Heo60c2bc22012-04-01 14:38:43 -07001741 return 0;
1742}
1743
Tejun Heo2da8ca82013-12-05 12:28:04 -05001744static int cfq_print_leaf_weight(struct seq_file *sf, void *v)
Tejun Heoe71357e2013-01-09 08:05:10 -08001745{
Arianna Avanzinie48453c2015-06-05 23:38:42 +02001746 struct blkcg *blkcg = css_to_blkcg(seq_css(sf));
Jens Axboe9470e4a2015-06-19 10:19:36 -06001747 struct cfq_group_data *cgd = blkcg_to_cfqgd(blkcg);
1748 unsigned int val = 0;
Arianna Avanzinie48453c2015-06-05 23:38:42 +02001749
Jens Axboe9470e4a2015-06-19 10:19:36 -06001750 if (cgd)
1751 val = cgd->leaf_weight;
1752
1753 seq_printf(sf, "%u\n", val);
Tejun Heoe71357e2013-01-09 08:05:10 -08001754 return 0;
1755}
1756
Tejun Heo451af502014-05-13 12:16:21 -04001757static ssize_t __cfqg_set_weight_device(struct kernfs_open_file *of,
1758 char *buf, size_t nbytes, loff_t off,
Tejun Heo2ee867dc2015-08-18 14:55:34 -07001759 bool on_dfl, bool is_leaf_weight)
Tejun Heo60c2bc22012-04-01 14:38:43 -07001760{
Tejun Heo69d7fde2015-08-18 14:55:36 -07001761 unsigned int min = on_dfl ? CGROUP_WEIGHT_MIN : CFQ_WEIGHT_LEGACY_MIN;
1762 unsigned int max = on_dfl ? CGROUP_WEIGHT_MAX : CFQ_WEIGHT_LEGACY_MAX;
Tejun Heo451af502014-05-13 12:16:21 -04001763 struct blkcg *blkcg = css_to_blkcg(of_css(of));
Tejun Heo60c2bc22012-04-01 14:38:43 -07001764 struct blkg_conf_ctx ctx;
Tejun Heo3381cb82012-04-01 14:38:44 -07001765 struct cfq_group *cfqg;
Arianna Avanzinie48453c2015-06-05 23:38:42 +02001766 struct cfq_group_data *cfqgd;
Tejun Heo60c2bc22012-04-01 14:38:43 -07001767 int ret;
Tejun Heo36aa9e52015-08-18 14:55:31 -07001768 u64 v;
Tejun Heo60c2bc22012-04-01 14:38:43 -07001769
Tejun Heo3c798392012-04-16 13:57:25 -07001770 ret = blkg_conf_prep(blkcg, &blkcg_policy_cfq, buf, &ctx);
Tejun Heo60c2bc22012-04-01 14:38:43 -07001771 if (ret)
1772 return ret;
1773
Tejun Heo2ee867dc2015-08-18 14:55:34 -07001774 if (sscanf(ctx.body, "%llu", &v) == 1) {
1775 /* require "default" on dfl */
1776 ret = -ERANGE;
1777 if (!v && on_dfl)
1778 goto out_finish;
1779 } else if (!strcmp(strim(ctx.body), "default")) {
1780 v = 0;
1781 } else {
1782 ret = -EINVAL;
Tejun Heo36aa9e52015-08-18 14:55:31 -07001783 goto out_finish;
Tejun Heo2ee867dc2015-08-18 14:55:34 -07001784 }
Tejun Heo36aa9e52015-08-18 14:55:31 -07001785
Tejun Heo3381cb82012-04-01 14:38:44 -07001786 cfqg = blkg_to_cfqg(ctx.blkg);
Arianna Avanzinie48453c2015-06-05 23:38:42 +02001787 cfqgd = blkcg_to_cfqgd(blkcg);
Jens Axboeae994ea2015-06-20 10:26:50 -06001788
Tejun Heo20386ce2015-08-18 14:55:28 -07001789 ret = -ERANGE;
Tejun Heo69d7fde2015-08-18 14:55:36 -07001790 if (!v || (v >= min && v <= max)) {
Tejun Heoe71357e2013-01-09 08:05:10 -08001791 if (!is_leaf_weight) {
Tejun Heo36aa9e52015-08-18 14:55:31 -07001792 cfqg->dev_weight = v;
1793 cfqg->new_weight = v ?: cfqgd->weight;
Tejun Heoe71357e2013-01-09 08:05:10 -08001794 } else {
Tejun Heo36aa9e52015-08-18 14:55:31 -07001795 cfqg->dev_leaf_weight = v;
1796 cfqg->new_leaf_weight = v ?: cfqgd->leaf_weight;
Tejun Heoe71357e2013-01-09 08:05:10 -08001797 }
Tejun Heo60c2bc22012-04-01 14:38:43 -07001798 ret = 0;
1799 }
Tejun Heo36aa9e52015-08-18 14:55:31 -07001800out_finish:
Tejun Heo60c2bc22012-04-01 14:38:43 -07001801 blkg_conf_finish(&ctx);
Tejun Heo451af502014-05-13 12:16:21 -04001802 return ret ?: nbytes;
Tejun Heo60c2bc22012-04-01 14:38:43 -07001803}
1804
Tejun Heo451af502014-05-13 12:16:21 -04001805static ssize_t cfqg_set_weight_device(struct kernfs_open_file *of,
1806 char *buf, size_t nbytes, loff_t off)
Tejun Heoe71357e2013-01-09 08:05:10 -08001807{
Tejun Heo2ee867dc2015-08-18 14:55:34 -07001808 return __cfqg_set_weight_device(of, buf, nbytes, off, false, false);
Tejun Heoe71357e2013-01-09 08:05:10 -08001809}
1810
Tejun Heo451af502014-05-13 12:16:21 -04001811static ssize_t cfqg_set_leaf_weight_device(struct kernfs_open_file *of,
1812 char *buf, size_t nbytes, loff_t off)
Tejun Heoe71357e2013-01-09 08:05:10 -08001813{
Tejun Heo2ee867dc2015-08-18 14:55:34 -07001814 return __cfqg_set_weight_device(of, buf, nbytes, off, false, true);
Tejun Heoe71357e2013-01-09 08:05:10 -08001815}
1816
Tejun Heodd165eb2015-08-18 14:55:33 -07001817static int __cfq_set_weight(struct cgroup_subsys_state *css, u64 val,
Tejun Heo69d7fde2015-08-18 14:55:36 -07001818 bool on_dfl, bool reset_dev, bool is_leaf_weight)
Tejun Heo60c2bc22012-04-01 14:38:43 -07001819{
Tejun Heo69d7fde2015-08-18 14:55:36 -07001820 unsigned int min = on_dfl ? CGROUP_WEIGHT_MIN : CFQ_WEIGHT_LEGACY_MIN;
1821 unsigned int max = on_dfl ? CGROUP_WEIGHT_MAX : CFQ_WEIGHT_LEGACY_MAX;
Tejun Heo182446d2013-08-08 20:11:24 -04001822 struct blkcg *blkcg = css_to_blkcg(css);
Tejun Heo3c798392012-04-16 13:57:25 -07001823 struct blkcg_gq *blkg;
Arianna Avanzinie48453c2015-06-05 23:38:42 +02001824 struct cfq_group_data *cfqgd;
Jens Axboeae994ea2015-06-20 10:26:50 -06001825 int ret = 0;
Tejun Heo60c2bc22012-04-01 14:38:43 -07001826
Tejun Heo69d7fde2015-08-18 14:55:36 -07001827 if (val < min || val > max)
1828 return -ERANGE;
Tejun Heo60c2bc22012-04-01 14:38:43 -07001829
1830 spin_lock_irq(&blkcg->lock);
Arianna Avanzinie48453c2015-06-05 23:38:42 +02001831 cfqgd = blkcg_to_cfqgd(blkcg);
Jens Axboeae994ea2015-06-20 10:26:50 -06001832 if (!cfqgd) {
1833 ret = -EINVAL;
1834 goto out;
1835 }
Tejun Heoe71357e2013-01-09 08:05:10 -08001836
1837 if (!is_leaf_weight)
Arianna Avanzinie48453c2015-06-05 23:38:42 +02001838 cfqgd->weight = val;
Tejun Heoe71357e2013-01-09 08:05:10 -08001839 else
Arianna Avanzinie48453c2015-06-05 23:38:42 +02001840 cfqgd->leaf_weight = val;
Tejun Heo60c2bc22012-04-01 14:38:43 -07001841
Sasha Levinb67bfe02013-02-27 17:06:00 -08001842 hlist_for_each_entry(blkg, &blkcg->blkg_list, blkcg_node) {
Tejun Heo3381cb82012-04-01 14:38:44 -07001843 struct cfq_group *cfqg = blkg_to_cfqg(blkg);
Tejun Heo60c2bc22012-04-01 14:38:43 -07001844
Tejun Heoe71357e2013-01-09 08:05:10 -08001845 if (!cfqg)
1846 continue;
1847
1848 if (!is_leaf_weight) {
Tejun Heo69d7fde2015-08-18 14:55:36 -07001849 if (reset_dev)
1850 cfqg->dev_weight = 0;
Tejun Heoe71357e2013-01-09 08:05:10 -08001851 if (!cfqg->dev_weight)
Arianna Avanzinie48453c2015-06-05 23:38:42 +02001852 cfqg->new_weight = cfqgd->weight;
Tejun Heoe71357e2013-01-09 08:05:10 -08001853 } else {
Tejun Heo69d7fde2015-08-18 14:55:36 -07001854 if (reset_dev)
1855 cfqg->dev_leaf_weight = 0;
Tejun Heoe71357e2013-01-09 08:05:10 -08001856 if (!cfqg->dev_leaf_weight)
Arianna Avanzinie48453c2015-06-05 23:38:42 +02001857 cfqg->new_leaf_weight = cfqgd->leaf_weight;
Tejun Heoe71357e2013-01-09 08:05:10 -08001858 }
Tejun Heo60c2bc22012-04-01 14:38:43 -07001859 }
1860
Jens Axboeae994ea2015-06-20 10:26:50 -06001861out:
Tejun Heo60c2bc22012-04-01 14:38:43 -07001862 spin_unlock_irq(&blkcg->lock);
Jens Axboeae994ea2015-06-20 10:26:50 -06001863 return ret;
Tejun Heo60c2bc22012-04-01 14:38:43 -07001864}
1865
Tejun Heo182446d2013-08-08 20:11:24 -04001866static int cfq_set_weight(struct cgroup_subsys_state *css, struct cftype *cft,
1867 u64 val)
Tejun Heoe71357e2013-01-09 08:05:10 -08001868{
Tejun Heo69d7fde2015-08-18 14:55:36 -07001869 return __cfq_set_weight(css, val, false, false, false);
Tejun Heoe71357e2013-01-09 08:05:10 -08001870}
1871
Tejun Heo182446d2013-08-08 20:11:24 -04001872static int cfq_set_leaf_weight(struct cgroup_subsys_state *css,
1873 struct cftype *cft, u64 val)
Tejun Heoe71357e2013-01-09 08:05:10 -08001874{
Tejun Heo69d7fde2015-08-18 14:55:36 -07001875 return __cfq_set_weight(css, val, false, false, true);
Tejun Heoe71357e2013-01-09 08:05:10 -08001876}
1877
Tejun Heo2da8ca82013-12-05 12:28:04 -05001878static int cfqg_print_stat(struct seq_file *sf, void *v)
Tejun Heo5bc4afb12012-04-01 14:38:45 -07001879{
Tejun Heo2da8ca82013-12-05 12:28:04 -05001880 blkcg_print_blkgs(sf, css_to_blkcg(seq_css(sf)), blkg_prfill_stat,
1881 &blkcg_policy_cfq, seq_cft(sf)->private, false);
Tejun Heo5bc4afb12012-04-01 14:38:45 -07001882 return 0;
1883}
1884
Tejun Heo2da8ca82013-12-05 12:28:04 -05001885static int cfqg_print_rwstat(struct seq_file *sf, void *v)
Tejun Heo5bc4afb12012-04-01 14:38:45 -07001886{
Tejun Heo2da8ca82013-12-05 12:28:04 -05001887 blkcg_print_blkgs(sf, css_to_blkcg(seq_css(sf)), blkg_prfill_rwstat,
1888 &blkcg_policy_cfq, seq_cft(sf)->private, true);
Tejun Heo5bc4afb12012-04-01 14:38:45 -07001889 return 0;
1890}
1891
Tejun Heo43114012013-01-09 08:05:13 -08001892static u64 cfqg_prfill_stat_recursive(struct seq_file *sf,
1893 struct blkg_policy_data *pd, int off)
1894{
Tejun Heof12c74c2015-08-18 14:55:23 -07001895 u64 sum = blkg_stat_recursive_sum(pd_to_blkg(pd),
1896 &blkcg_policy_cfq, off);
Tejun Heo43114012013-01-09 08:05:13 -08001897 return __blkg_prfill_u64(sf, pd, sum);
1898}
1899
1900static u64 cfqg_prfill_rwstat_recursive(struct seq_file *sf,
1901 struct blkg_policy_data *pd, int off)
1902{
Tejun Heof12c74c2015-08-18 14:55:23 -07001903 struct blkg_rwstat sum = blkg_rwstat_recursive_sum(pd_to_blkg(pd),
1904 &blkcg_policy_cfq, off);
Tejun Heo43114012013-01-09 08:05:13 -08001905 return __blkg_prfill_rwstat(sf, pd, &sum);
1906}
1907
Tejun Heo2da8ca82013-12-05 12:28:04 -05001908static int cfqg_print_stat_recursive(struct seq_file *sf, void *v)
Tejun Heo43114012013-01-09 08:05:13 -08001909{
Tejun Heo2da8ca82013-12-05 12:28:04 -05001910 blkcg_print_blkgs(sf, css_to_blkcg(seq_css(sf)),
1911 cfqg_prfill_stat_recursive, &blkcg_policy_cfq,
1912 seq_cft(sf)->private, false);
Tejun Heo43114012013-01-09 08:05:13 -08001913 return 0;
1914}
1915
Tejun Heo2da8ca82013-12-05 12:28:04 -05001916static int cfqg_print_rwstat_recursive(struct seq_file *sf, void *v)
Tejun Heo43114012013-01-09 08:05:13 -08001917{
Tejun Heo2da8ca82013-12-05 12:28:04 -05001918 blkcg_print_blkgs(sf, css_to_blkcg(seq_css(sf)),
1919 cfqg_prfill_rwstat_recursive, &blkcg_policy_cfq,
1920 seq_cft(sf)->private, true);
Tejun Heo43114012013-01-09 08:05:13 -08001921 return 0;
1922}
1923
Tejun Heo702747c2015-08-18 14:55:25 -07001924static u64 cfqg_prfill_sectors(struct seq_file *sf, struct blkg_policy_data *pd,
1925 int off)
1926{
1927 u64 sum = blkg_rwstat_total(&pd->blkg->stat_bytes);
1928
1929 return __blkg_prfill_u64(sf, pd, sum >> 9);
1930}
1931
1932static int cfqg_print_stat_sectors(struct seq_file *sf, void *v)
1933{
1934 blkcg_print_blkgs(sf, css_to_blkcg(seq_css(sf)),
1935 cfqg_prfill_sectors, &blkcg_policy_cfq, 0, false);
1936 return 0;
1937}
1938
1939static u64 cfqg_prfill_sectors_recursive(struct seq_file *sf,
1940 struct blkg_policy_data *pd, int off)
1941{
1942 struct blkg_rwstat tmp = blkg_rwstat_recursive_sum(pd->blkg, NULL,
1943 offsetof(struct blkcg_gq, stat_bytes));
1944 u64 sum = atomic64_read(&tmp.aux_cnt[BLKG_RWSTAT_READ]) +
1945 atomic64_read(&tmp.aux_cnt[BLKG_RWSTAT_WRITE]);
1946
1947 return __blkg_prfill_u64(sf, pd, sum >> 9);
1948}
1949
1950static int cfqg_print_stat_sectors_recursive(struct seq_file *sf, void *v)
1951{
1952 blkcg_print_blkgs(sf, css_to_blkcg(seq_css(sf)),
1953 cfqg_prfill_sectors_recursive, &blkcg_policy_cfq, 0,
1954 false);
1955 return 0;
1956}
1957
Tejun Heo60c2bc22012-04-01 14:38:43 -07001958#ifdef CONFIG_DEBUG_BLK_CGROUP
Tejun Heof95a04a2012-04-16 13:57:26 -07001959static u64 cfqg_prfill_avg_queue_size(struct seq_file *sf,
1960 struct blkg_policy_data *pd, int off)
Tejun Heo60c2bc22012-04-01 14:38:43 -07001961{
Tejun Heof95a04a2012-04-16 13:57:26 -07001962 struct cfq_group *cfqg = pd_to_cfqg(pd);
Tejun Heo155fead2012-04-01 14:38:44 -07001963 u64 samples = blkg_stat_read(&cfqg->stats.avg_queue_size_samples);
Tejun Heo60c2bc22012-04-01 14:38:43 -07001964 u64 v = 0;
1965
1966 if (samples) {
Tejun Heo155fead2012-04-01 14:38:44 -07001967 v = blkg_stat_read(&cfqg->stats.avg_queue_size_sum);
Anatol Pomozovf3cff252013-09-22 12:43:47 -06001968 v = div64_u64(v, samples);
Tejun Heo60c2bc22012-04-01 14:38:43 -07001969 }
Tejun Heof95a04a2012-04-16 13:57:26 -07001970 __blkg_prfill_u64(sf, pd, v);
Tejun Heo60c2bc22012-04-01 14:38:43 -07001971 return 0;
1972}
1973
1974/* print avg_queue_size */
Tejun Heo2da8ca82013-12-05 12:28:04 -05001975static int cfqg_print_avg_queue_size(struct seq_file *sf, void *v)
Tejun Heo60c2bc22012-04-01 14:38:43 -07001976{
Tejun Heo2da8ca82013-12-05 12:28:04 -05001977 blkcg_print_blkgs(sf, css_to_blkcg(seq_css(sf)),
1978 cfqg_prfill_avg_queue_size, &blkcg_policy_cfq,
1979 0, false);
Tejun Heo60c2bc22012-04-01 14:38:43 -07001980 return 0;
1981}
1982#endif /* CONFIG_DEBUG_BLK_CGROUP */
1983
Tejun Heo880f50e2015-08-18 14:55:30 -07001984static struct cftype cfq_blkcg_legacy_files[] = {
Tejun Heo1d3650f2013-01-09 08:05:11 -08001985 /* on root, weight is mapped to leaf_weight */
Tejun Heo60c2bc22012-04-01 14:38:43 -07001986 {
1987 .name = "weight_device",
Tejun Heo1d3650f2013-01-09 08:05:11 -08001988 .flags = CFTYPE_ONLY_ON_ROOT,
Tejun Heo2da8ca82013-12-05 12:28:04 -05001989 .seq_show = cfqg_print_leaf_weight_device,
Tejun Heo451af502014-05-13 12:16:21 -04001990 .write = cfqg_set_leaf_weight_device,
Tejun Heo1d3650f2013-01-09 08:05:11 -08001991 },
1992 {
1993 .name = "weight",
1994 .flags = CFTYPE_ONLY_ON_ROOT,
Tejun Heo2da8ca82013-12-05 12:28:04 -05001995 .seq_show = cfq_print_leaf_weight,
Tejun Heo1d3650f2013-01-09 08:05:11 -08001996 .write_u64 = cfq_set_leaf_weight,
1997 },
1998
1999 /* no such mapping necessary for !roots */
2000 {
2001 .name = "weight_device",
2002 .flags = CFTYPE_NOT_ON_ROOT,
Tejun Heo2da8ca82013-12-05 12:28:04 -05002003 .seq_show = cfqg_print_weight_device,
Tejun Heo451af502014-05-13 12:16:21 -04002004 .write = cfqg_set_weight_device,
Tejun Heo60c2bc22012-04-01 14:38:43 -07002005 },
2006 {
2007 .name = "weight",
Tejun Heoe71357e2013-01-09 08:05:10 -08002008 .flags = CFTYPE_NOT_ON_ROOT,
Tejun Heo2da8ca82013-12-05 12:28:04 -05002009 .seq_show = cfq_print_weight,
Tejun Heo3381cb82012-04-01 14:38:44 -07002010 .write_u64 = cfq_set_weight,
Tejun Heo60c2bc22012-04-01 14:38:43 -07002011 },
Tejun Heo1d3650f2013-01-09 08:05:11 -08002012
2013 {
2014 .name = "leaf_weight_device",
Tejun Heo2da8ca82013-12-05 12:28:04 -05002015 .seq_show = cfqg_print_leaf_weight_device,
Tejun Heo451af502014-05-13 12:16:21 -04002016 .write = cfqg_set_leaf_weight_device,
Tejun Heoe71357e2013-01-09 08:05:10 -08002017 },
2018 {
2019 .name = "leaf_weight",
Tejun Heo2da8ca82013-12-05 12:28:04 -05002020 .seq_show = cfq_print_leaf_weight,
Tejun Heoe71357e2013-01-09 08:05:10 -08002021 .write_u64 = cfq_set_leaf_weight,
2022 },
2023
Tejun Heo43114012013-01-09 08:05:13 -08002024 /* statistics, covers only the tasks in the cfqg */
Tejun Heo60c2bc22012-04-01 14:38:43 -07002025 {
2026 .name = "time",
Tejun Heo5bc4afb12012-04-01 14:38:45 -07002027 .private = offsetof(struct cfq_group, stats.time),
Tejun Heo2da8ca82013-12-05 12:28:04 -05002028 .seq_show = cfqg_print_stat,
Tejun Heo60c2bc22012-04-01 14:38:43 -07002029 },
2030 {
2031 .name = "sectors",
Tejun Heo702747c2015-08-18 14:55:25 -07002032 .seq_show = cfqg_print_stat_sectors,
Tejun Heo60c2bc22012-04-01 14:38:43 -07002033 },
2034 {
2035 .name = "io_service_bytes",
Tejun Heo77ea7332015-08-18 14:55:24 -07002036 .private = (unsigned long)&blkcg_policy_cfq,
2037 .seq_show = blkg_print_stat_bytes,
Tejun Heo60c2bc22012-04-01 14:38:43 -07002038 },
2039 {
2040 .name = "io_serviced",
Tejun Heo77ea7332015-08-18 14:55:24 -07002041 .private = (unsigned long)&blkcg_policy_cfq,
2042 .seq_show = blkg_print_stat_ios,
Tejun Heo60c2bc22012-04-01 14:38:43 -07002043 },
2044 {
2045 .name = "io_service_time",
Tejun Heo5bc4afb12012-04-01 14:38:45 -07002046 .private = offsetof(struct cfq_group, stats.service_time),
Tejun Heo2da8ca82013-12-05 12:28:04 -05002047 .seq_show = cfqg_print_rwstat,
Tejun Heo60c2bc22012-04-01 14:38:43 -07002048 },
2049 {
2050 .name = "io_wait_time",
Tejun Heo5bc4afb12012-04-01 14:38:45 -07002051 .private = offsetof(struct cfq_group, stats.wait_time),
Tejun Heo2da8ca82013-12-05 12:28:04 -05002052 .seq_show = cfqg_print_rwstat,
Tejun Heo60c2bc22012-04-01 14:38:43 -07002053 },
2054 {
2055 .name = "io_merged",
Tejun Heo5bc4afb12012-04-01 14:38:45 -07002056 .private = offsetof(struct cfq_group, stats.merged),
Tejun Heo2da8ca82013-12-05 12:28:04 -05002057 .seq_show = cfqg_print_rwstat,
Tejun Heo60c2bc22012-04-01 14:38:43 -07002058 },
2059 {
2060 .name = "io_queued",
Tejun Heo5bc4afb12012-04-01 14:38:45 -07002061 .private = offsetof(struct cfq_group, stats.queued),
Tejun Heo2da8ca82013-12-05 12:28:04 -05002062 .seq_show = cfqg_print_rwstat,
Tejun Heo60c2bc22012-04-01 14:38:43 -07002063 },
Tejun Heo43114012013-01-09 08:05:13 -08002064
2065 /* the same statictics which cover the cfqg and its descendants */
2066 {
2067 .name = "time_recursive",
2068 .private = offsetof(struct cfq_group, stats.time),
Tejun Heo2da8ca82013-12-05 12:28:04 -05002069 .seq_show = cfqg_print_stat_recursive,
Tejun Heo43114012013-01-09 08:05:13 -08002070 },
2071 {
2072 .name = "sectors_recursive",
Tejun Heo702747c2015-08-18 14:55:25 -07002073 .seq_show = cfqg_print_stat_sectors_recursive,
Tejun Heo43114012013-01-09 08:05:13 -08002074 },
2075 {
2076 .name = "io_service_bytes_recursive",
Tejun Heo77ea7332015-08-18 14:55:24 -07002077 .private = (unsigned long)&blkcg_policy_cfq,
2078 .seq_show = blkg_print_stat_bytes_recursive,
Tejun Heo43114012013-01-09 08:05:13 -08002079 },
2080 {
2081 .name = "io_serviced_recursive",
Tejun Heo77ea7332015-08-18 14:55:24 -07002082 .private = (unsigned long)&blkcg_policy_cfq,
2083 .seq_show = blkg_print_stat_ios_recursive,
Tejun Heo43114012013-01-09 08:05:13 -08002084 },
2085 {
2086 .name = "io_service_time_recursive",
2087 .private = offsetof(struct cfq_group, stats.service_time),
Tejun Heo2da8ca82013-12-05 12:28:04 -05002088 .seq_show = cfqg_print_rwstat_recursive,
Tejun Heo43114012013-01-09 08:05:13 -08002089 },
2090 {
2091 .name = "io_wait_time_recursive",
2092 .private = offsetof(struct cfq_group, stats.wait_time),
Tejun Heo2da8ca82013-12-05 12:28:04 -05002093 .seq_show = cfqg_print_rwstat_recursive,
Tejun Heo43114012013-01-09 08:05:13 -08002094 },
2095 {
2096 .name = "io_merged_recursive",
2097 .private = offsetof(struct cfq_group, stats.merged),
Tejun Heo2da8ca82013-12-05 12:28:04 -05002098 .seq_show = cfqg_print_rwstat_recursive,
Tejun Heo43114012013-01-09 08:05:13 -08002099 },
2100 {
2101 .name = "io_queued_recursive",
2102 .private = offsetof(struct cfq_group, stats.queued),
Tejun Heo2da8ca82013-12-05 12:28:04 -05002103 .seq_show = cfqg_print_rwstat_recursive,
Tejun Heo43114012013-01-09 08:05:13 -08002104 },
Tejun Heo60c2bc22012-04-01 14:38:43 -07002105#ifdef CONFIG_DEBUG_BLK_CGROUP
2106 {
2107 .name = "avg_queue_size",
Tejun Heo2da8ca82013-12-05 12:28:04 -05002108 .seq_show = cfqg_print_avg_queue_size,
Tejun Heo60c2bc22012-04-01 14:38:43 -07002109 },
2110 {
2111 .name = "group_wait_time",
Tejun Heo5bc4afb12012-04-01 14:38:45 -07002112 .private = offsetof(struct cfq_group, stats.group_wait_time),
Tejun Heo2da8ca82013-12-05 12:28:04 -05002113 .seq_show = cfqg_print_stat,
Tejun Heo60c2bc22012-04-01 14:38:43 -07002114 },
2115 {
2116 .name = "idle_time",
Tejun Heo5bc4afb12012-04-01 14:38:45 -07002117 .private = offsetof(struct cfq_group, stats.idle_time),
Tejun Heo2da8ca82013-12-05 12:28:04 -05002118 .seq_show = cfqg_print_stat,
Tejun Heo60c2bc22012-04-01 14:38:43 -07002119 },
2120 {
2121 .name = "empty_time",
Tejun Heo5bc4afb12012-04-01 14:38:45 -07002122 .private = offsetof(struct cfq_group, stats.empty_time),
Tejun Heo2da8ca82013-12-05 12:28:04 -05002123 .seq_show = cfqg_print_stat,
Tejun Heo60c2bc22012-04-01 14:38:43 -07002124 },
2125 {
2126 .name = "dequeue",
Tejun Heo5bc4afb12012-04-01 14:38:45 -07002127 .private = offsetof(struct cfq_group, stats.dequeue),
Tejun Heo2da8ca82013-12-05 12:28:04 -05002128 .seq_show = cfqg_print_stat,
Tejun Heo60c2bc22012-04-01 14:38:43 -07002129 },
2130 {
2131 .name = "unaccounted_time",
Tejun Heo5bc4afb12012-04-01 14:38:45 -07002132 .private = offsetof(struct cfq_group, stats.unaccounted_time),
Tejun Heo2da8ca82013-12-05 12:28:04 -05002133 .seq_show = cfqg_print_stat,
Tejun Heo60c2bc22012-04-01 14:38:43 -07002134 },
2135#endif /* CONFIG_DEBUG_BLK_CGROUP */
2136 { } /* terminate */
2137};
Tejun Heo2ee867dc2015-08-18 14:55:34 -07002138
2139static int cfq_print_weight_on_dfl(struct seq_file *sf, void *v)
2140{
2141 struct blkcg *blkcg = css_to_blkcg(seq_css(sf));
2142 struct cfq_group_data *cgd = blkcg_to_cfqgd(blkcg);
2143
2144 seq_printf(sf, "default %u\n", cgd->weight);
2145 blkcg_print_blkgs(sf, blkcg, cfqg_prfill_weight_device,
2146 &blkcg_policy_cfq, 0, false);
2147 return 0;
2148}
2149
2150static ssize_t cfq_set_weight_on_dfl(struct kernfs_open_file *of,
2151 char *buf, size_t nbytes, loff_t off)
2152{
2153 char *endp;
2154 int ret;
2155 u64 v;
2156
2157 buf = strim(buf);
2158
2159 /* "WEIGHT" or "default WEIGHT" sets the default weight */
2160 v = simple_strtoull(buf, &endp, 0);
2161 if (*endp == '\0' || sscanf(buf, "default %llu", &v) == 1) {
Tejun Heo69d7fde2015-08-18 14:55:36 -07002162 ret = __cfq_set_weight(of_css(of), v, true, false, false);
Tejun Heo2ee867dc2015-08-18 14:55:34 -07002163 return ret ?: nbytes;
2164 }
2165
2166 /* "MAJ:MIN WEIGHT" */
2167 return __cfqg_set_weight_device(of, buf, nbytes, off, true, false);
2168}
2169
2170static struct cftype cfq_blkcg_files[] = {
2171 {
2172 .name = "weight",
2173 .flags = CFTYPE_NOT_ON_ROOT,
2174 .seq_show = cfq_print_weight_on_dfl,
2175 .write = cfq_set_weight_on_dfl,
2176 },
2177 { } /* terminate */
2178};
2179
Vivek Goyal25fb5162009-12-03 12:59:46 -05002180#else /* GROUP_IOSCHED */
Tejun Heoae118892015-08-18 14:55:20 -07002181static struct cfq_group *cfq_lookup_cfqg(struct cfq_data *cfqd,
2182 struct blkcg *blkcg)
Vivek Goyal25fb5162009-12-03 12:59:46 -05002183{
Tejun Heof51b8022012-03-05 13:15:05 -08002184 return cfqd->root_group;
Vivek Goyal25fb5162009-12-03 12:59:46 -05002185}
Vivek Goyal7f1dc8a2010-04-21 17:44:16 +02002186
Vivek Goyal25fb5162009-12-03 12:59:46 -05002187static inline void
2188cfq_link_cfqq_cfqg(struct cfq_queue *cfqq, struct cfq_group *cfqg) {
2189 cfqq->cfqg = cfqg;
2190}
2191
2192#endif /* GROUP_IOSCHED */
2193
Jens Axboe498d3aa22007-04-26 12:54:48 +02002194/*
Corrado Zoccoloc0324a02009-10-27 19:16:03 +01002195 * The cfqd->service_trees holds all pending cfq_queue's that have
Jens Axboe498d3aa22007-04-26 12:54:48 +02002196 * requests waiting to be processed. It is sorted in the order that
2197 * we will service the queues.
2198 */
Jens Axboea36e71f2009-04-15 12:15:11 +02002199static void cfq_service_tree_add(struct cfq_data *cfqd, struct cfq_queue *cfqq,
Jens Axboea6151c32009-10-07 20:02:57 +02002200 bool add_front)
Jens Axboed9e76202007-04-20 14:27:50 +02002201{
Jens Axboe08717142008-01-28 11:38:15 +01002202 struct rb_node **p, *parent;
2203 struct cfq_queue *__cfqq;
Jens Axboed9e76202007-04-20 14:27:50 +02002204 unsigned long rb_key;
Vivek Goyal34b98d02012-10-03 16:56:58 -04002205 struct cfq_rb_root *st;
Jens Axboe498d3aa22007-04-26 12:54:48 +02002206 int left;
Vivek Goyaldae739e2009-12-03 12:59:45 -05002207 int new_cfqq = 1;
Vivek Goyalae30c282009-12-03 12:59:55 -05002208
Vivek Goyal34b98d02012-10-03 16:56:58 -04002209 st = st_for(cfqq->cfqg, cfqq_class(cfqq), cfqq_type(cfqq));
Jens Axboe08717142008-01-28 11:38:15 +01002210 if (cfq_class_idle(cfqq)) {
2211 rb_key = CFQ_IDLE_DELAY;
Vivek Goyal34b98d02012-10-03 16:56:58 -04002212 parent = rb_last(&st->rb);
Jens Axboe08717142008-01-28 11:38:15 +01002213 if (parent && parent != &cfqq->rb_node) {
2214 __cfqq = rb_entry(parent, struct cfq_queue, rb_node);
2215 rb_key += __cfqq->rb_key;
2216 } else
2217 rb_key += jiffies;
2218 } else if (!add_front) {
Jens Axboeb9c89462009-10-06 20:53:44 +02002219 /*
2220 * Get our rb key offset. Subtract any residual slice
2221 * value carried from last service. A negative resid
2222 * count indicates slice overrun, and this should position
2223 * the next service time further away in the tree.
2224 */
Jens Axboeedd75ff2007-04-19 12:03:34 +02002225 rb_key = cfq_slice_offset(cfqd, cfqq) + jiffies;
Jens Axboeb9c89462009-10-06 20:53:44 +02002226 rb_key -= cfqq->slice_resid;
Jens Axboeedd75ff2007-04-19 12:03:34 +02002227 cfqq->slice_resid = 0;
Corrado Zoccolo48e025e2009-10-05 08:49:23 +02002228 } else {
2229 rb_key = -HZ;
Vivek Goyal34b98d02012-10-03 16:56:58 -04002230 __cfqq = cfq_rb_first(st);
Corrado Zoccolo48e025e2009-10-05 08:49:23 +02002231 rb_key += __cfqq ? __cfqq->rb_key : jiffies;
2232 }
Jens Axboed9e76202007-04-20 14:27:50 +02002233
2234 if (!RB_EMPTY_NODE(&cfqq->rb_node)) {
Vivek Goyaldae739e2009-12-03 12:59:45 -05002235 new_cfqq = 0;
Jens Axboe99f96282007-02-05 11:56:25 +01002236 /*
Jens Axboed9e76202007-04-20 14:27:50 +02002237 * same position, nothing more to do
Jens Axboe99f96282007-02-05 11:56:25 +01002238 */
Vivek Goyal34b98d02012-10-03 16:56:58 -04002239 if (rb_key == cfqq->rb_key && cfqq->service_tree == st)
Jens Axboed9e76202007-04-20 14:27:50 +02002240 return;
Jens Axboe53b037442006-07-28 09:48:51 +02002241
Corrado Zoccoloaa6f6a32009-10-26 22:44:33 +01002242 cfq_rb_erase(&cfqq->rb_node, cfqq->service_tree);
2243 cfqq->service_tree = NULL;
Jens Axboe22e2c502005-06-27 10:55:12 +02002244 }
Jens Axboed9e76202007-04-20 14:27:50 +02002245
Jens Axboe498d3aa22007-04-26 12:54:48 +02002246 left = 1;
Jens Axboe08717142008-01-28 11:38:15 +01002247 parent = NULL;
Vivek Goyal34b98d02012-10-03 16:56:58 -04002248 cfqq->service_tree = st;
2249 p = &st->rb.rb_node;
Jens Axboed9e76202007-04-20 14:27:50 +02002250 while (*p) {
2251 parent = *p;
2252 __cfqq = rb_entry(parent, struct cfq_queue, rb_node);
2253
Jens Axboe0c534e02007-04-18 20:01:57 +02002254 /*
Corrado Zoccoloc0324a02009-10-27 19:16:03 +01002255 * sort by key, that represents service time.
Jens Axboe0c534e02007-04-18 20:01:57 +02002256 */
Corrado Zoccoloc0324a02009-10-27 19:16:03 +01002257 if (time_before(rb_key, __cfqq->rb_key))
Vivek Goyal1f23f122012-10-03 16:57:00 -04002258 p = &parent->rb_left;
Corrado Zoccoloc0324a02009-10-27 19:16:03 +01002259 else {
Vivek Goyal1f23f122012-10-03 16:57:00 -04002260 p = &parent->rb_right;
Jens Axboecc09e292007-04-26 12:53:50 +02002261 left = 0;
Corrado Zoccoloc0324a02009-10-27 19:16:03 +01002262 }
Jens Axboed9e76202007-04-20 14:27:50 +02002263 }
2264
Jens Axboecc09e292007-04-26 12:53:50 +02002265 if (left)
Vivek Goyal34b98d02012-10-03 16:56:58 -04002266 st->left = &cfqq->rb_node;
Jens Axboecc09e292007-04-26 12:53:50 +02002267
Jens Axboed9e76202007-04-20 14:27:50 +02002268 cfqq->rb_key = rb_key;
2269 rb_link_node(&cfqq->rb_node, parent, p);
Vivek Goyal34b98d02012-10-03 16:56:58 -04002270 rb_insert_color(&cfqq->rb_node, &st->rb);
2271 st->count++;
Namhyung Kim20359f22011-05-24 10:23:22 +02002272 if (add_front || !new_cfqq)
Vivek Goyaldae739e2009-12-03 12:59:45 -05002273 return;
Justin TerAvest8184f932011-03-17 16:12:36 +01002274 cfq_group_notify_queue_add(cfqd, cfqq->cfqg);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002275}
2276
Jens Axboea36e71f2009-04-15 12:15:11 +02002277static struct cfq_queue *
Jens Axboef2d1f0a2009-04-23 12:19:38 +02002278cfq_prio_tree_lookup(struct cfq_data *cfqd, struct rb_root *root,
2279 sector_t sector, struct rb_node **ret_parent,
2280 struct rb_node ***rb_link)
Jens Axboea36e71f2009-04-15 12:15:11 +02002281{
Jens Axboea36e71f2009-04-15 12:15:11 +02002282 struct rb_node **p, *parent;
2283 struct cfq_queue *cfqq = NULL;
2284
2285 parent = NULL;
2286 p = &root->rb_node;
2287 while (*p) {
2288 struct rb_node **n;
2289
2290 parent = *p;
2291 cfqq = rb_entry(parent, struct cfq_queue, p_node);
2292
2293 /*
2294 * Sort strictly based on sector. Smallest to the left,
2295 * largest to the right.
2296 */
Tejun Heo2e46e8b2009-05-07 22:24:41 +09002297 if (sector > blk_rq_pos(cfqq->next_rq))
Jens Axboea36e71f2009-04-15 12:15:11 +02002298 n = &(*p)->rb_right;
Tejun Heo2e46e8b2009-05-07 22:24:41 +09002299 else if (sector < blk_rq_pos(cfqq->next_rq))
Jens Axboea36e71f2009-04-15 12:15:11 +02002300 n = &(*p)->rb_left;
2301 else
2302 break;
2303 p = n;
Jens Axboe3ac6c9f2009-04-23 12:14:56 +02002304 cfqq = NULL;
Jens Axboea36e71f2009-04-15 12:15:11 +02002305 }
2306
2307 *ret_parent = parent;
2308 if (rb_link)
2309 *rb_link = p;
Jens Axboe3ac6c9f2009-04-23 12:14:56 +02002310 return cfqq;
Jens Axboea36e71f2009-04-15 12:15:11 +02002311}
2312
2313static void cfq_prio_tree_add(struct cfq_data *cfqd, struct cfq_queue *cfqq)
2314{
Jens Axboea36e71f2009-04-15 12:15:11 +02002315 struct rb_node **p, *parent;
2316 struct cfq_queue *__cfqq;
2317
Jens Axboef2d1f0a2009-04-23 12:19:38 +02002318 if (cfqq->p_root) {
2319 rb_erase(&cfqq->p_node, cfqq->p_root);
2320 cfqq->p_root = NULL;
2321 }
Jens Axboea36e71f2009-04-15 12:15:11 +02002322
2323 if (cfq_class_idle(cfqq))
2324 return;
2325 if (!cfqq->next_rq)
2326 return;
2327
Jens Axboef2d1f0a2009-04-23 12:19:38 +02002328 cfqq->p_root = &cfqd->prio_trees[cfqq->org_ioprio];
Tejun Heo2e46e8b2009-05-07 22:24:41 +09002329 __cfqq = cfq_prio_tree_lookup(cfqd, cfqq->p_root,
2330 blk_rq_pos(cfqq->next_rq), &parent, &p);
Jens Axboe3ac6c9f2009-04-23 12:14:56 +02002331 if (!__cfqq) {
2332 rb_link_node(&cfqq->p_node, parent, p);
Jens Axboef2d1f0a2009-04-23 12:19:38 +02002333 rb_insert_color(&cfqq->p_node, cfqq->p_root);
2334 } else
2335 cfqq->p_root = NULL;
Jens Axboea36e71f2009-04-15 12:15:11 +02002336}
2337
Jens Axboe498d3aa22007-04-26 12:54:48 +02002338/*
2339 * Update cfqq's position in the service tree.
2340 */
Jens Axboeedd75ff2007-04-19 12:03:34 +02002341static void cfq_resort_rr_list(struct cfq_data *cfqd, struct cfq_queue *cfqq)
Jens Axboe6d048f52007-04-25 12:44:27 +02002342{
Jens Axboe6d048f52007-04-25 12:44:27 +02002343 /*
2344 * Resorting requires the cfqq to be on the RR list already.
2345 */
Jens Axboea36e71f2009-04-15 12:15:11 +02002346 if (cfq_cfqq_on_rr(cfqq)) {
Jens Axboeedd75ff2007-04-19 12:03:34 +02002347 cfq_service_tree_add(cfqd, cfqq, 0);
Jens Axboea36e71f2009-04-15 12:15:11 +02002348 cfq_prio_tree_add(cfqd, cfqq);
2349 }
Jens Axboe6d048f52007-04-25 12:44:27 +02002350}
2351
Linus Torvalds1da177e2005-04-16 15:20:36 -07002352/*
2353 * add to busy list of queues for service, trying to be fair in ordering
Jens Axboe22e2c502005-06-27 10:55:12 +02002354 * the pending list according to last request service
Linus Torvalds1da177e2005-04-16 15:20:36 -07002355 */
Jens Axboefebffd62008-01-28 13:19:43 +01002356static void cfq_add_cfqq_rr(struct cfq_data *cfqd, struct cfq_queue *cfqq)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002357{
Jens Axboe7b679132008-05-30 12:23:07 +02002358 cfq_log_cfqq(cfqd, cfqq, "add_to_rr");
Jens Axboe3b181522005-06-27 10:56:24 +02002359 BUG_ON(cfq_cfqq_on_rr(cfqq));
2360 cfq_mark_cfqq_on_rr(cfqq);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002361 cfqd->busy_queues++;
Shaohua Lief8a41d2011-03-07 09:26:29 +01002362 if (cfq_cfqq_sync(cfqq))
2363 cfqd->busy_sync_queues++;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002364
Jens Axboeedd75ff2007-04-19 12:03:34 +02002365 cfq_resort_rr_list(cfqd, cfqq);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002366}
2367
Jens Axboe498d3aa22007-04-26 12:54:48 +02002368/*
2369 * Called when the cfqq no longer has requests pending, remove it from
2370 * the service tree.
2371 */
Jens Axboefebffd62008-01-28 13:19:43 +01002372static void cfq_del_cfqq_rr(struct cfq_data *cfqd, struct cfq_queue *cfqq)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002373{
Jens Axboe7b679132008-05-30 12:23:07 +02002374 cfq_log_cfqq(cfqd, cfqq, "del_from_rr");
Jens Axboe3b181522005-06-27 10:56:24 +02002375 BUG_ON(!cfq_cfqq_on_rr(cfqq));
2376 cfq_clear_cfqq_on_rr(cfqq);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002377
Corrado Zoccoloaa6f6a32009-10-26 22:44:33 +01002378 if (!RB_EMPTY_NODE(&cfqq->rb_node)) {
2379 cfq_rb_erase(&cfqq->rb_node, cfqq->service_tree);
2380 cfqq->service_tree = NULL;
2381 }
Jens Axboef2d1f0a2009-04-23 12:19:38 +02002382 if (cfqq->p_root) {
2383 rb_erase(&cfqq->p_node, cfqq->p_root);
2384 cfqq->p_root = NULL;
2385 }
Jens Axboed9e76202007-04-20 14:27:50 +02002386
Justin TerAvest8184f932011-03-17 16:12:36 +01002387 cfq_group_notify_queue_del(cfqd, cfqq->cfqg);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002388 BUG_ON(!cfqd->busy_queues);
2389 cfqd->busy_queues--;
Shaohua Lief8a41d2011-03-07 09:26:29 +01002390 if (cfq_cfqq_sync(cfqq))
2391 cfqd->busy_sync_queues--;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002392}
2393
2394/*
2395 * rb tree support functions
2396 */
Jens Axboefebffd62008-01-28 13:19:43 +01002397static void cfq_del_rq_rb(struct request *rq)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002398{
Jens Axboe5e705372006-07-13 12:39:25 +02002399 struct cfq_queue *cfqq = RQ_CFQQ(rq);
Jens Axboe5e705372006-07-13 12:39:25 +02002400 const int sync = rq_is_sync(rq);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002401
Jens Axboeb4878f22005-10-20 16:42:29 +02002402 BUG_ON(!cfqq->queued[sync]);
2403 cfqq->queued[sync]--;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002404
Jens Axboe5e705372006-07-13 12:39:25 +02002405 elv_rb_del(&cfqq->sort_list, rq);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002406
Vivek Goyalf04a6422009-12-03 12:59:40 -05002407 if (cfq_cfqq_on_rr(cfqq) && RB_EMPTY_ROOT(&cfqq->sort_list)) {
2408 /*
2409 * Queue will be deleted from service tree when we actually
2410 * expire it later. Right now just remove it from prio tree
2411 * as it is empty.
2412 */
2413 if (cfqq->p_root) {
2414 rb_erase(&cfqq->p_node, cfqq->p_root);
2415 cfqq->p_root = NULL;
2416 }
2417 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07002418}
2419
Jens Axboe5e705372006-07-13 12:39:25 +02002420static void cfq_add_rq_rb(struct request *rq)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002421{
Jens Axboe5e705372006-07-13 12:39:25 +02002422 struct cfq_queue *cfqq = RQ_CFQQ(rq);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002423 struct cfq_data *cfqd = cfqq->cfqd;
Jeff Moyer796d5112011-06-02 21:19:05 +02002424 struct request *prev;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002425
Jens Axboe5380a102006-07-13 12:37:56 +02002426 cfqq->queued[rq_is_sync(rq)]++;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002427
Jeff Moyer796d5112011-06-02 21:19:05 +02002428 elv_rb_add(&cfqq->sort_list, rq);
Jens Axboe5fccbf62006-10-31 14:21:55 +01002429
2430 if (!cfq_cfqq_on_rr(cfqq))
2431 cfq_add_cfqq_rr(cfqd, cfqq);
Jens Axboe5044eed2007-04-25 11:53:48 +02002432
2433 /*
2434 * check if this request is a better next-serve candidate
2435 */
Jens Axboea36e71f2009-04-15 12:15:11 +02002436 prev = cfqq->next_rq;
Corrado Zoccolocf7c25c2009-11-08 17:16:46 +01002437 cfqq->next_rq = cfq_choose_req(cfqd, cfqq->next_rq, rq, cfqd->last_position);
Jens Axboea36e71f2009-04-15 12:15:11 +02002438
2439 /*
2440 * adjust priority tree position, if ->next_rq changes
2441 */
2442 if (prev != cfqq->next_rq)
2443 cfq_prio_tree_add(cfqd, cfqq);
2444
Jens Axboe5044eed2007-04-25 11:53:48 +02002445 BUG_ON(!cfqq->next_rq);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002446}
2447
Jens Axboefebffd62008-01-28 13:19:43 +01002448static void cfq_reposition_rq_rb(struct cfq_queue *cfqq, struct request *rq)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002449{
Jens Axboe5380a102006-07-13 12:37:56 +02002450 elv_rb_del(&cfqq->sort_list, rq);
2451 cfqq->queued[rq_is_sync(rq)]--;
Tejun Heo155fead2012-04-01 14:38:44 -07002452 cfqg_stats_update_io_remove(RQ_CFQG(rq), rq->cmd_flags);
Jens Axboe5e705372006-07-13 12:39:25 +02002453 cfq_add_rq_rb(rq);
Tejun Heo155fead2012-04-01 14:38:44 -07002454 cfqg_stats_update_io_add(RQ_CFQG(rq), cfqq->cfqd->serving_group,
2455 rq->cmd_flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002456}
2457
Jens Axboe206dc692006-03-28 13:03:44 +02002458static struct request *
2459cfq_find_rq_fmerge(struct cfq_data *cfqd, struct bio *bio)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002460{
Jens Axboe206dc692006-03-28 13:03:44 +02002461 struct task_struct *tsk = current;
Tejun Heoc5869802011-12-14 00:33:41 +01002462 struct cfq_io_cq *cic;
Jens Axboe206dc692006-03-28 13:03:44 +02002463 struct cfq_queue *cfqq;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002464
Jens Axboe4ac845a2008-01-24 08:44:49 +01002465 cic = cfq_cic_lookup(cfqd, tsk->io_context);
Vasily Tarasov91fac312007-04-25 12:29:51 +02002466 if (!cic)
2467 return NULL;
2468
2469 cfqq = cic_to_cfqq(cic, cfq_bio_sync(bio));
Kent Overstreetf73a1c72012-09-25 15:05:12 -07002470 if (cfqq)
2471 return elv_rb_find(&cfqq->sort_list, bio_end_sector(bio));
Linus Torvalds1da177e2005-04-16 15:20:36 -07002472
Linus Torvalds1da177e2005-04-16 15:20:36 -07002473 return NULL;
2474}
2475
Jens Axboe165125e2007-07-24 09:28:11 +02002476static void cfq_activate_request(struct request_queue *q, struct request *rq)
Jens Axboeb4878f22005-10-20 16:42:29 +02002477{
2478 struct cfq_data *cfqd = q->elevator->elevator_data;
2479
Corrado Zoccolo53c583d2010-02-28 19:45:05 +01002480 cfqd->rq_in_driver++;
Jens Axboe7b679132008-05-30 12:23:07 +02002481 cfq_log_cfqq(cfqd, RQ_CFQQ(rq), "activate rq, drv=%d",
Corrado Zoccolo53c583d2010-02-28 19:45:05 +01002482 cfqd->rq_in_driver);
Jens Axboe25776e32006-06-01 10:12:26 +02002483
Tejun Heo5b936292009-05-07 22:24:38 +09002484 cfqd->last_position = blk_rq_pos(rq) + blk_rq_sectors(rq);
Jens Axboeb4878f22005-10-20 16:42:29 +02002485}
2486
Jens Axboe165125e2007-07-24 09:28:11 +02002487static void cfq_deactivate_request(struct request_queue *q, struct request *rq)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002488{
Jens Axboe22e2c502005-06-27 10:55:12 +02002489 struct cfq_data *cfqd = q->elevator->elevator_data;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002490
Corrado Zoccolo53c583d2010-02-28 19:45:05 +01002491 WARN_ON(!cfqd->rq_in_driver);
2492 cfqd->rq_in_driver--;
Jens Axboe7b679132008-05-30 12:23:07 +02002493 cfq_log_cfqq(cfqd, RQ_CFQQ(rq), "deactivate rq, drv=%d",
Corrado Zoccolo53c583d2010-02-28 19:45:05 +01002494 cfqd->rq_in_driver);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002495}
2496
Jens Axboeb4878f22005-10-20 16:42:29 +02002497static void cfq_remove_request(struct request *rq)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002498{
Jens Axboe5e705372006-07-13 12:39:25 +02002499 struct cfq_queue *cfqq = RQ_CFQQ(rq);
Jens Axboe21183b02006-07-13 12:33:14 +02002500
Jens Axboe5e705372006-07-13 12:39:25 +02002501 if (cfqq->next_rq == rq)
2502 cfqq->next_rq = cfq_find_next_rq(cfqq->cfqd, cfqq, rq);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002503
Jens Axboeb4878f22005-10-20 16:42:29 +02002504 list_del_init(&rq->queuelist);
Jens Axboe5e705372006-07-13 12:39:25 +02002505 cfq_del_rq_rb(rq);
Jens Axboe374f84a2006-07-23 01:42:19 +02002506
Aaron Carroll45333d52008-08-26 15:52:36 +02002507 cfqq->cfqd->rq_queued--;
Tejun Heo155fead2012-04-01 14:38:44 -07002508 cfqg_stats_update_io_remove(RQ_CFQG(rq), rq->cmd_flags);
Christoph Hellwig65299a32011-08-23 14:50:29 +02002509 if (rq->cmd_flags & REQ_PRIO) {
2510 WARN_ON(!cfqq->prio_pending);
2511 cfqq->prio_pending--;
Jens Axboeb53d1ed2011-08-19 08:34:48 +02002512 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07002513}
2514
Jens Axboe165125e2007-07-24 09:28:11 +02002515static int cfq_merge(struct request_queue *q, struct request **req,
2516 struct bio *bio)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002517{
2518 struct cfq_data *cfqd = q->elevator->elevator_data;
2519 struct request *__rq;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002520
Jens Axboe206dc692006-03-28 13:03:44 +02002521 __rq = cfq_find_rq_fmerge(cfqd, bio);
Jens Axboe22e2c502005-06-27 10:55:12 +02002522 if (__rq && elv_rq_merge_ok(__rq, bio)) {
Jens Axboe98170642006-07-28 09:23:08 +02002523 *req = __rq;
2524 return ELEVATOR_FRONT_MERGE;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002525 }
2526
2527 return ELEVATOR_NO_MERGE;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002528}
2529
Jens Axboe165125e2007-07-24 09:28:11 +02002530static void cfq_merged_request(struct request_queue *q, struct request *req,
Jens Axboe21183b02006-07-13 12:33:14 +02002531 int type)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002532{
Jens Axboe21183b02006-07-13 12:33:14 +02002533 if (type == ELEVATOR_FRONT_MERGE) {
Jens Axboe5e705372006-07-13 12:39:25 +02002534 struct cfq_queue *cfqq = RQ_CFQQ(req);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002535
Jens Axboe5e705372006-07-13 12:39:25 +02002536 cfq_reposition_rq_rb(cfqq, req);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002537 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07002538}
2539
Divyesh Shah812d4022010-04-08 21:14:23 -07002540static void cfq_bio_merged(struct request_queue *q, struct request *req,
2541 struct bio *bio)
2542{
Tejun Heo155fead2012-04-01 14:38:44 -07002543 cfqg_stats_update_io_merged(RQ_CFQG(req), bio->bi_rw);
Divyesh Shah812d4022010-04-08 21:14:23 -07002544}
2545
Linus Torvalds1da177e2005-04-16 15:20:36 -07002546static void
Jens Axboe165125e2007-07-24 09:28:11 +02002547cfq_merged_requests(struct request_queue *q, struct request *rq,
Linus Torvalds1da177e2005-04-16 15:20:36 -07002548 struct request *next)
2549{
Corrado Zoccolocf7c25c2009-11-08 17:16:46 +01002550 struct cfq_queue *cfqq = RQ_CFQQ(rq);
Shaohua Li4a0b75c2011-12-16 14:00:22 +01002551 struct cfq_data *cfqd = q->elevator->elevator_data;
2552
Jens Axboe22e2c502005-06-27 10:55:12 +02002553 /*
2554 * reposition in fifo if next is older than rq
2555 */
2556 if (!list_empty(&rq->queuelist) && !list_empty(&next->queuelist) &&
Jan Kara8b4922d2014-02-24 16:39:52 +01002557 time_before(next->fifo_time, rq->fifo_time) &&
Shaohua Li3d106fba2012-11-06 12:39:51 +01002558 cfqq == RQ_CFQQ(next)) {
Jens Axboe22e2c502005-06-27 10:55:12 +02002559 list_move(&rq->queuelist, &next->queuelist);
Jan Kara8b4922d2014-02-24 16:39:52 +01002560 rq->fifo_time = next->fifo_time;
Jens Axboe30996f42009-10-05 11:03:39 +02002561 }
Jens Axboe22e2c502005-06-27 10:55:12 +02002562
Corrado Zoccolocf7c25c2009-11-08 17:16:46 +01002563 if (cfqq->next_rq == next)
2564 cfqq->next_rq = rq;
Jens Axboeb4878f22005-10-20 16:42:29 +02002565 cfq_remove_request(next);
Tejun Heo155fead2012-04-01 14:38:44 -07002566 cfqg_stats_update_io_merged(RQ_CFQG(rq), next->cmd_flags);
Shaohua Li4a0b75c2011-12-16 14:00:22 +01002567
2568 cfqq = RQ_CFQQ(next);
2569 /*
2570 * all requests of this queue are merged to other queues, delete it
2571 * from the service tree. If it's the active_queue,
2572 * cfq_dispatch_requests() will choose to expire it or do idle
2573 */
2574 if (cfq_cfqq_on_rr(cfqq) && RB_EMPTY_ROOT(&cfqq->sort_list) &&
2575 cfqq != cfqd->active_queue)
2576 cfq_del_cfqq_rr(cfqd, cfqq);
Jens Axboe22e2c502005-06-27 10:55:12 +02002577}
2578
Jens Axboe165125e2007-07-24 09:28:11 +02002579static int cfq_allow_merge(struct request_queue *q, struct request *rq,
Jens Axboeda775262006-12-20 11:04:12 +01002580 struct bio *bio)
2581{
2582 struct cfq_data *cfqd = q->elevator->elevator_data;
Tejun Heoc5869802011-12-14 00:33:41 +01002583 struct cfq_io_cq *cic;
Jens Axboeda775262006-12-20 11:04:12 +01002584 struct cfq_queue *cfqq;
Jens Axboeda775262006-12-20 11:04:12 +01002585
2586 /*
Jens Axboeec8acb62007-01-02 18:32:11 +01002587 * Disallow merge of a sync bio into an async request.
Jens Axboeda775262006-12-20 11:04:12 +01002588 */
Vasily Tarasov91fac312007-04-25 12:29:51 +02002589 if (cfq_bio_sync(bio) && !rq_is_sync(rq))
Jens Axboea6151c32009-10-07 20:02:57 +02002590 return false;
Jens Axboeda775262006-12-20 11:04:12 +01002591
2592 /*
Tejun Heof1a4f4d2011-12-14 00:33:39 +01002593 * Lookup the cfqq that this bio will be queued with and allow
Tejun Heo07c2bd32012-02-08 09:19:42 +01002594 * merge only if rq is queued there.
Jens Axboeda775262006-12-20 11:04:12 +01002595 */
Tejun Heo07c2bd32012-02-08 09:19:42 +01002596 cic = cfq_cic_lookup(cfqd, current->io_context);
2597 if (!cic)
2598 return false;
Jens Axboe719d3402006-12-22 09:38:53 +01002599
Vasily Tarasov91fac312007-04-25 12:29:51 +02002600 cfqq = cic_to_cfqq(cic, cfq_bio_sync(bio));
Jens Axboea6151c32009-10-07 20:02:57 +02002601 return cfqq == RQ_CFQQ(rq);
Jens Axboeda775262006-12-20 11:04:12 +01002602}
2603
Divyesh Shah812df482010-04-08 21:15:35 -07002604static inline void cfq_del_timer(struct cfq_data *cfqd, struct cfq_queue *cfqq)
2605{
2606 del_timer(&cfqd->idle_slice_timer);
Tejun Heo155fead2012-04-01 14:38:44 -07002607 cfqg_stats_update_idle_time(cfqq->cfqg);
Divyesh Shah812df482010-04-08 21:15:35 -07002608}
2609
Jens Axboefebffd62008-01-28 13:19:43 +01002610static void __cfq_set_active_queue(struct cfq_data *cfqd,
2611 struct cfq_queue *cfqq)
Jens Axboe22e2c502005-06-27 10:55:12 +02002612{
2613 if (cfqq) {
Vivek Goyal3bf10fe2012-10-03 16:56:56 -04002614 cfq_log_cfqq(cfqd, cfqq, "set_active wl_class:%d wl_type:%d",
Vivek Goyal4d2ceea2012-10-03 16:56:57 -04002615 cfqd->serving_wl_class, cfqd->serving_wl_type);
Tejun Heo155fead2012-04-01 14:38:44 -07002616 cfqg_stats_update_avg_queue_size(cfqq->cfqg);
Justin TerAvest62a37f62011-03-23 08:25:44 +01002617 cfqq->slice_start = 0;
2618 cfqq->dispatch_start = jiffies;
2619 cfqq->allocated_slice = 0;
2620 cfqq->slice_end = 0;
2621 cfqq->slice_dispatch = 0;
2622 cfqq->nr_sectors = 0;
2623
2624 cfq_clear_cfqq_wait_request(cfqq);
2625 cfq_clear_cfqq_must_dispatch(cfqq);
2626 cfq_clear_cfqq_must_alloc_slice(cfqq);
2627 cfq_clear_cfqq_fifo_expire(cfqq);
2628 cfq_mark_cfqq_slice_new(cfqq);
2629
2630 cfq_del_timer(cfqd, cfqq);
Jens Axboe22e2c502005-06-27 10:55:12 +02002631 }
2632
2633 cfqd->active_queue = cfqq;
2634}
2635
2636/*
Jens Axboe7b14e3b2006-02-28 09:35:11 +01002637 * current cfqq expired its slice (or was too idle), select new one
2638 */
2639static void
2640__cfq_slice_expired(struct cfq_data *cfqd, struct cfq_queue *cfqq,
Vivek Goyale5ff0822010-04-26 19:25:11 +02002641 bool timed_out)
Jens Axboe7b14e3b2006-02-28 09:35:11 +01002642{
Jens Axboe7b679132008-05-30 12:23:07 +02002643 cfq_log_cfqq(cfqd, cfqq, "slice expired t=%d", timed_out);
2644
Jens Axboe7b14e3b2006-02-28 09:35:11 +01002645 if (cfq_cfqq_wait_request(cfqq))
Divyesh Shah812df482010-04-08 21:15:35 -07002646 cfq_del_timer(cfqd, cfqq);
Jens Axboe7b14e3b2006-02-28 09:35:11 +01002647
Jens Axboe7b14e3b2006-02-28 09:35:11 +01002648 cfq_clear_cfqq_wait_request(cfqq);
Vivek Goyalf75edf22009-12-03 12:59:53 -05002649 cfq_clear_cfqq_wait_busy(cfqq);
Jens Axboe7b14e3b2006-02-28 09:35:11 +01002650
2651 /*
Shaohua Liae54abe2010-02-05 13:11:45 +01002652 * If this cfqq is shared between multiple processes, check to
2653 * make sure that those processes are still issuing I/Os within
2654 * the mean seek distance. If not, it may be time to break the
2655 * queues apart again.
2656 */
2657 if (cfq_cfqq_coop(cfqq) && CFQQ_SEEKY(cfqq))
2658 cfq_mark_cfqq_split_coop(cfqq);
2659
2660 /*
Jens Axboe6084cdd2007-04-23 08:25:00 +02002661 * store what was left of this slice, if the queue idled/timed out
Jens Axboe7b14e3b2006-02-28 09:35:11 +01002662 */
Shaohua Lic553f8e2011-01-14 08:41:03 +01002663 if (timed_out) {
2664 if (cfq_cfqq_slice_new(cfqq))
Vivek Goyalba5bd522011-01-19 08:25:02 -07002665 cfqq->slice_resid = cfq_scaled_cfqq_slice(cfqd, cfqq);
Shaohua Lic553f8e2011-01-14 08:41:03 +01002666 else
2667 cfqq->slice_resid = cfqq->slice_end - jiffies;
Jens Axboe7b679132008-05-30 12:23:07 +02002668 cfq_log_cfqq(cfqd, cfqq, "resid=%ld", cfqq->slice_resid);
2669 }
Jens Axboe7b14e3b2006-02-28 09:35:11 +01002670
Vivek Goyale5ff0822010-04-26 19:25:11 +02002671 cfq_group_served(cfqd, cfqq->cfqg, cfqq);
Vivek Goyaldae739e2009-12-03 12:59:45 -05002672
Vivek Goyalf04a6422009-12-03 12:59:40 -05002673 if (cfq_cfqq_on_rr(cfqq) && RB_EMPTY_ROOT(&cfqq->sort_list))
2674 cfq_del_cfqq_rr(cfqd, cfqq);
2675
Jens Axboeedd75ff2007-04-19 12:03:34 +02002676 cfq_resort_rr_list(cfqd, cfqq);
Jens Axboe7b14e3b2006-02-28 09:35:11 +01002677
2678 if (cfqq == cfqd->active_queue)
2679 cfqd->active_queue = NULL;
2680
2681 if (cfqd->active_cic) {
Tejun Heo11a31222012-02-07 07:51:30 +01002682 put_io_context(cfqd->active_cic->icq.ioc);
Jens Axboe7b14e3b2006-02-28 09:35:11 +01002683 cfqd->active_cic = NULL;
2684 }
Jens Axboe7b14e3b2006-02-28 09:35:11 +01002685}
2686
Vivek Goyale5ff0822010-04-26 19:25:11 +02002687static inline void cfq_slice_expired(struct cfq_data *cfqd, bool timed_out)
Jens Axboe7b14e3b2006-02-28 09:35:11 +01002688{
2689 struct cfq_queue *cfqq = cfqd->active_queue;
2690
2691 if (cfqq)
Vivek Goyale5ff0822010-04-26 19:25:11 +02002692 __cfq_slice_expired(cfqd, cfqq, timed_out);
Jens Axboe7b14e3b2006-02-28 09:35:11 +01002693}
2694
Jens Axboe498d3aa22007-04-26 12:54:48 +02002695/*
2696 * Get next queue for service. Unless we have a queue preemption,
2697 * we'll simply select the first cfqq in the service tree.
2698 */
Jens Axboe6d048f52007-04-25 12:44:27 +02002699static struct cfq_queue *cfq_get_next_queue(struct cfq_data *cfqd)
Jens Axboe22e2c502005-06-27 10:55:12 +02002700{
Vivek Goyal34b98d02012-10-03 16:56:58 -04002701 struct cfq_rb_root *st = st_for(cfqd->serving_group,
2702 cfqd->serving_wl_class, cfqd->serving_wl_type);
Jens Axboeedd75ff2007-04-19 12:03:34 +02002703
Vivek Goyalf04a6422009-12-03 12:59:40 -05002704 if (!cfqd->rq_queued)
2705 return NULL;
2706
Vivek Goyal1fa8f6d2009-12-03 12:59:41 -05002707 /* There is nothing to dispatch */
Vivek Goyal34b98d02012-10-03 16:56:58 -04002708 if (!st)
Vivek Goyal1fa8f6d2009-12-03 12:59:41 -05002709 return NULL;
Vivek Goyal34b98d02012-10-03 16:56:58 -04002710 if (RB_EMPTY_ROOT(&st->rb))
Corrado Zoccoloc0324a02009-10-27 19:16:03 +01002711 return NULL;
Vivek Goyal34b98d02012-10-03 16:56:58 -04002712 return cfq_rb_first(st);
Jens Axboe6d048f52007-04-25 12:44:27 +02002713}
2714
Vivek Goyalf04a6422009-12-03 12:59:40 -05002715static struct cfq_queue *cfq_get_next_queue_forced(struct cfq_data *cfqd)
2716{
Vivek Goyal25fb5162009-12-03 12:59:46 -05002717 struct cfq_group *cfqg;
Vivek Goyalf04a6422009-12-03 12:59:40 -05002718 struct cfq_queue *cfqq;
2719 int i, j;
2720 struct cfq_rb_root *st;
2721
2722 if (!cfqd->rq_queued)
2723 return NULL;
2724
Vivek Goyal25fb5162009-12-03 12:59:46 -05002725 cfqg = cfq_get_next_cfqg(cfqd);
2726 if (!cfqg)
2727 return NULL;
2728
Vivek Goyalf04a6422009-12-03 12:59:40 -05002729 for_each_cfqg_st(cfqg, i, j, st)
2730 if ((cfqq = cfq_rb_first(st)) != NULL)
2731 return cfqq;
2732 return NULL;
2733}
2734
Jens Axboe498d3aa22007-04-26 12:54:48 +02002735/*
2736 * Get and set a new active queue for service.
2737 */
Jens Axboea36e71f2009-04-15 12:15:11 +02002738static struct cfq_queue *cfq_set_active_queue(struct cfq_data *cfqd,
2739 struct cfq_queue *cfqq)
Jens Axboe6d048f52007-04-25 12:44:27 +02002740{
Jens Axboee00ef792009-11-04 08:54:55 +01002741 if (!cfqq)
Jens Axboea36e71f2009-04-15 12:15:11 +02002742 cfqq = cfq_get_next_queue(cfqd);
Jens Axboe6d048f52007-04-25 12:44:27 +02002743
Jens Axboe22e2c502005-06-27 10:55:12 +02002744 __cfq_set_active_queue(cfqd, cfqq);
Jens Axboe3b181522005-06-27 10:56:24 +02002745 return cfqq;
Jens Axboe22e2c502005-06-27 10:55:12 +02002746}
2747
Jens Axboed9e76202007-04-20 14:27:50 +02002748static inline sector_t cfq_dist_from_last(struct cfq_data *cfqd,
2749 struct request *rq)
2750{
Tejun Heo83096eb2009-05-07 22:24:39 +09002751 if (blk_rq_pos(rq) >= cfqd->last_position)
2752 return blk_rq_pos(rq) - cfqd->last_position;
Jens Axboed9e76202007-04-20 14:27:50 +02002753 else
Tejun Heo83096eb2009-05-07 22:24:39 +09002754 return cfqd->last_position - blk_rq_pos(rq);
Jens Axboed9e76202007-04-20 14:27:50 +02002755}
2756
Jeff Moyerb2c18e12009-10-23 17:14:49 -04002757static inline int cfq_rq_close(struct cfq_data *cfqd, struct cfq_queue *cfqq,
Shaohua Lie9ce3352010-03-19 08:03:04 +01002758 struct request *rq)
Jens Axboe6d048f52007-04-25 12:44:27 +02002759{
Shaohua Lie9ce3352010-03-19 08:03:04 +01002760 return cfq_dist_from_last(cfqd, rq) <= CFQQ_CLOSE_THR;
Jens Axboe6d048f52007-04-25 12:44:27 +02002761}
2762
Jens Axboea36e71f2009-04-15 12:15:11 +02002763static struct cfq_queue *cfqq_close(struct cfq_data *cfqd,
2764 struct cfq_queue *cur_cfqq)
Jens Axboe6d048f52007-04-25 12:44:27 +02002765{
Jens Axboef2d1f0a2009-04-23 12:19:38 +02002766 struct rb_root *root = &cfqd->prio_trees[cur_cfqq->org_ioprio];
Jens Axboea36e71f2009-04-15 12:15:11 +02002767 struct rb_node *parent, *node;
2768 struct cfq_queue *__cfqq;
2769 sector_t sector = cfqd->last_position;
2770
2771 if (RB_EMPTY_ROOT(root))
2772 return NULL;
2773
2774 /*
2775 * First, if we find a request starting at the end of the last
2776 * request, choose it.
2777 */
Jens Axboef2d1f0a2009-04-23 12:19:38 +02002778 __cfqq = cfq_prio_tree_lookup(cfqd, root, sector, &parent, NULL);
Jens Axboea36e71f2009-04-15 12:15:11 +02002779 if (__cfqq)
2780 return __cfqq;
2781
2782 /*
2783 * If the exact sector wasn't found, the parent of the NULL leaf
2784 * will contain the closest sector.
2785 */
2786 __cfqq = rb_entry(parent, struct cfq_queue, p_node);
Shaohua Lie9ce3352010-03-19 08:03:04 +01002787 if (cfq_rq_close(cfqd, cur_cfqq, __cfqq->next_rq))
Jens Axboea36e71f2009-04-15 12:15:11 +02002788 return __cfqq;
2789
Tejun Heo2e46e8b2009-05-07 22:24:41 +09002790 if (blk_rq_pos(__cfqq->next_rq) < sector)
Jens Axboea36e71f2009-04-15 12:15:11 +02002791 node = rb_next(&__cfqq->p_node);
2792 else
2793 node = rb_prev(&__cfqq->p_node);
2794 if (!node)
2795 return NULL;
2796
2797 __cfqq = rb_entry(node, struct cfq_queue, p_node);
Shaohua Lie9ce3352010-03-19 08:03:04 +01002798 if (cfq_rq_close(cfqd, cur_cfqq, __cfqq->next_rq))
Jens Axboea36e71f2009-04-15 12:15:11 +02002799 return __cfqq;
2800
2801 return NULL;
2802}
2803
2804/*
2805 * cfqd - obvious
2806 * cur_cfqq - passed in so that we don't decide that the current queue is
2807 * closely cooperating with itself.
2808 *
2809 * So, basically we're assuming that that cur_cfqq has dispatched at least
2810 * one request, and that cfqd->last_position reflects a position on the disk
2811 * associated with the I/O issued by cur_cfqq. I'm not sure this is a valid
2812 * assumption.
2813 */
2814static struct cfq_queue *cfq_close_cooperator(struct cfq_data *cfqd,
Jeff Moyerb3b6d042009-10-23 17:14:51 -04002815 struct cfq_queue *cur_cfqq)
Jens Axboea36e71f2009-04-15 12:15:11 +02002816{
2817 struct cfq_queue *cfqq;
2818
Divyesh Shah39c01b22010-03-25 15:45:57 +01002819 if (cfq_class_idle(cur_cfqq))
2820 return NULL;
Jeff Moyere6c5bc72009-10-23 17:14:52 -04002821 if (!cfq_cfqq_sync(cur_cfqq))
2822 return NULL;
2823 if (CFQQ_SEEKY(cur_cfqq))
2824 return NULL;
2825
Jens Axboea36e71f2009-04-15 12:15:11 +02002826 /*
Gui Jianfengb9d8f4c2009-12-08 08:54:17 +01002827 * Don't search priority tree if it's the only queue in the group.
2828 */
2829 if (cur_cfqq->cfqg->nr_cfqq == 1)
2830 return NULL;
2831
2832 /*
Jens Axboed9e76202007-04-20 14:27:50 +02002833 * We should notice if some of the queues are cooperating, eg
2834 * working closely on the same area of the disk. In that case,
2835 * we can group them together and don't waste time idling.
Jens Axboe6d048f52007-04-25 12:44:27 +02002836 */
Jens Axboea36e71f2009-04-15 12:15:11 +02002837 cfqq = cfqq_close(cfqd, cur_cfqq);
2838 if (!cfqq)
2839 return NULL;
2840
Vivek Goyal8682e1f2009-12-03 12:59:50 -05002841 /* If new queue belongs to different cfq_group, don't choose it */
2842 if (cur_cfqq->cfqg != cfqq->cfqg)
2843 return NULL;
2844
Jeff Moyerdf5fe3e2009-10-23 17:14:50 -04002845 /*
2846 * It only makes sense to merge sync queues.
2847 */
2848 if (!cfq_cfqq_sync(cfqq))
2849 return NULL;
Jeff Moyere6c5bc72009-10-23 17:14:52 -04002850 if (CFQQ_SEEKY(cfqq))
2851 return NULL;
Jeff Moyerdf5fe3e2009-10-23 17:14:50 -04002852
Corrado Zoccoloc0324a02009-10-27 19:16:03 +01002853 /*
2854 * Do not merge queues of different priority classes
2855 */
2856 if (cfq_class_rt(cfqq) != cfq_class_rt(cur_cfqq))
2857 return NULL;
2858
Jens Axboea36e71f2009-04-15 12:15:11 +02002859 return cfqq;
Jens Axboe6d048f52007-04-25 12:44:27 +02002860}
2861
Corrado Zoccoloa6d44e92009-10-26 22:45:11 +01002862/*
2863 * Determine whether we should enforce idle window for this queue.
2864 */
2865
2866static bool cfq_should_idle(struct cfq_data *cfqd, struct cfq_queue *cfqq)
2867{
Vivek Goyal3bf10fe2012-10-03 16:56:56 -04002868 enum wl_class_t wl_class = cfqq_class(cfqq);
Vivek Goyal34b98d02012-10-03 16:56:58 -04002869 struct cfq_rb_root *st = cfqq->service_tree;
Corrado Zoccoloa6d44e92009-10-26 22:45:11 +01002870
Vivek Goyal34b98d02012-10-03 16:56:58 -04002871 BUG_ON(!st);
2872 BUG_ON(!st->count);
Vivek Goyalf04a6422009-12-03 12:59:40 -05002873
Vivek Goyalb6508c12010-08-23 12:23:33 +02002874 if (!cfqd->cfq_slice_idle)
2875 return false;
2876
Corrado Zoccoloa6d44e92009-10-26 22:45:11 +01002877 /* We never do for idle class queues. */
Vivek Goyal3bf10fe2012-10-03 16:56:56 -04002878 if (wl_class == IDLE_WORKLOAD)
Corrado Zoccoloa6d44e92009-10-26 22:45:11 +01002879 return false;
2880
2881 /* We do for queues that were marked with idle window flag. */
Shaohua Li3c764b72009-12-04 13:12:06 +01002882 if (cfq_cfqq_idle_window(cfqq) &&
2883 !(blk_queue_nonrot(cfqd->queue) && cfqd->hw_tag))
Corrado Zoccoloa6d44e92009-10-26 22:45:11 +01002884 return true;
2885
2886 /*
2887 * Otherwise, we do only if they are the last ones
2888 * in their service tree.
2889 */
Vivek Goyal34b98d02012-10-03 16:56:58 -04002890 if (st->count == 1 && cfq_cfqq_sync(cfqq) &&
2891 !cfq_io_thinktime_big(cfqd, &st->ttime, false))
Shaohua Lic1e44752010-11-08 15:01:02 +01002892 return true;
Vivek Goyal34b98d02012-10-03 16:56:58 -04002893 cfq_log_cfqq(cfqd, cfqq, "Not idling. st->count:%d", st->count);
Shaohua Lic1e44752010-11-08 15:01:02 +01002894 return false;
Corrado Zoccoloa6d44e92009-10-26 22:45:11 +01002895}
2896
Jens Axboe6d048f52007-04-25 12:44:27 +02002897static void cfq_arm_slice_timer(struct cfq_data *cfqd)
Jens Axboe22e2c502005-06-27 10:55:12 +02002898{
Jens Axboe17926692007-01-19 11:59:30 +11002899 struct cfq_queue *cfqq = cfqd->active_queue;
Tejun Heoc5869802011-12-14 00:33:41 +01002900 struct cfq_io_cq *cic;
Vivek Goyal80bdf0c2010-08-23 12:24:26 +02002901 unsigned long sl, group_idle = 0;
Jens Axboe7b14e3b2006-02-28 09:35:11 +01002902
Jens Axboea68bbddba2008-09-24 13:03:33 +02002903 /*
Jens Axboef7d7b7a2008-09-25 11:37:50 +02002904 * SSD device without seek penalty, disable idling. But only do so
2905 * for devices that support queuing, otherwise we still have a problem
2906 * with sync vs async workloads.
Jens Axboea68bbddba2008-09-24 13:03:33 +02002907 */
Jens Axboef7d7b7a2008-09-25 11:37:50 +02002908 if (blk_queue_nonrot(cfqd->queue) && cfqd->hw_tag)
Jens Axboea68bbddba2008-09-24 13:03:33 +02002909 return;
2910
Jens Axboedd67d052006-06-21 09:36:18 +02002911 WARN_ON(!RB_EMPTY_ROOT(&cfqq->sort_list));
Jens Axboe6d048f52007-04-25 12:44:27 +02002912 WARN_ON(cfq_cfqq_slice_new(cfqq));
Jens Axboe22e2c502005-06-27 10:55:12 +02002913
2914 /*
2915 * idle is disabled, either manually or by past process history
2916 */
Vivek Goyal80bdf0c2010-08-23 12:24:26 +02002917 if (!cfq_should_idle(cfqd, cfqq)) {
2918 /* no queue idling. Check for group idling */
2919 if (cfqd->cfq_group_idle)
2920 group_idle = cfqd->cfq_group_idle;
2921 else
2922 return;
2923 }
Jens Axboe6d048f52007-04-25 12:44:27 +02002924
Jens Axboe22e2c502005-06-27 10:55:12 +02002925 /*
Corrado Zoccolo8e550632009-11-26 10:02:58 +01002926 * still active requests from this queue, don't idle
Jens Axboe7b679132008-05-30 12:23:07 +02002927 */
Corrado Zoccolo8e550632009-11-26 10:02:58 +01002928 if (cfqq->dispatched)
Jens Axboe7b679132008-05-30 12:23:07 +02002929 return;
2930
2931 /*
Jens Axboe22e2c502005-06-27 10:55:12 +02002932 * task has exited, don't wait
2933 */
Jens Axboe206dc692006-03-28 13:03:44 +02002934 cic = cfqd->active_cic;
Tejun Heof6e8d012012-03-05 13:15:26 -08002935 if (!cic || !atomic_read(&cic->icq.ioc->active_ref))
Jens Axboe6d048f52007-04-25 12:44:27 +02002936 return;
2937
Corrado Zoccolo355b6592009-10-08 08:43:32 +02002938 /*
2939 * If our average think time is larger than the remaining time
2940 * slice, then don't idle. This avoids overrunning the allotted
2941 * time slice.
2942 */
Shaohua Li383cd722011-07-12 14:24:35 +02002943 if (sample_valid(cic->ttime.ttime_samples) &&
2944 (cfqq->slice_end - jiffies < cic->ttime.ttime_mean)) {
Joe Perchesfd16d262011-06-13 10:42:49 +02002945 cfq_log_cfqq(cfqd, cfqq, "Not idling. think_time:%lu",
Shaohua Li383cd722011-07-12 14:24:35 +02002946 cic->ttime.ttime_mean);
Corrado Zoccolo355b6592009-10-08 08:43:32 +02002947 return;
Divyesh Shahb1ffe732010-03-25 15:45:03 +01002948 }
Corrado Zoccolo355b6592009-10-08 08:43:32 +02002949
Vivek Goyal80bdf0c2010-08-23 12:24:26 +02002950 /* There are other queues in the group, don't do group idle */
2951 if (group_idle && cfqq->cfqg->nr_cfqq > 1)
2952 return;
2953
Jens Axboe3b181522005-06-27 10:56:24 +02002954 cfq_mark_cfqq_wait_request(cfqq);
Jens Axboe22e2c502005-06-27 10:55:12 +02002955
Vivek Goyal80bdf0c2010-08-23 12:24:26 +02002956 if (group_idle)
2957 sl = cfqd->cfq_group_idle;
2958 else
2959 sl = cfqd->cfq_slice_idle;
Jens Axboe206dc692006-03-28 13:03:44 +02002960
Jens Axboe7b14e3b2006-02-28 09:35:11 +01002961 mod_timer(&cfqd->idle_slice_timer, jiffies + sl);
Tejun Heo155fead2012-04-01 14:38:44 -07002962 cfqg_stats_set_start_idle_time(cfqq->cfqg);
Vivek Goyal80bdf0c2010-08-23 12:24:26 +02002963 cfq_log_cfqq(cfqd, cfqq, "arm_idle: %lu group_idle: %d", sl,
2964 group_idle ? 1 : 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002965}
2966
Jens Axboe498d3aa22007-04-26 12:54:48 +02002967/*
2968 * Move request from internal lists to the request queue dispatch list.
2969 */
Jens Axboe165125e2007-07-24 09:28:11 +02002970static void cfq_dispatch_insert(struct request_queue *q, struct request *rq)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002971{
Jens Axboe3ed9a292007-04-23 08:33:33 +02002972 struct cfq_data *cfqd = q->elevator->elevator_data;
Jens Axboe5e705372006-07-13 12:39:25 +02002973 struct cfq_queue *cfqq = RQ_CFQQ(rq);
Jens Axboe22e2c502005-06-27 10:55:12 +02002974
Jens Axboe7b679132008-05-30 12:23:07 +02002975 cfq_log_cfqq(cfqd, cfqq, "dispatch_insert");
2976
Jeff Moyer06d21882009-09-11 17:08:59 +02002977 cfqq->next_rq = cfq_find_next_rq(cfqd, cfqq, rq);
Jens Axboe5380a102006-07-13 12:37:56 +02002978 cfq_remove_request(rq);
Jens Axboe6d048f52007-04-25 12:44:27 +02002979 cfqq->dispatched++;
Vivek Goyal80bdf0c2010-08-23 12:24:26 +02002980 (RQ_CFQG(rq))->dispatched++;
Jens Axboe5380a102006-07-13 12:37:56 +02002981 elv_dispatch_sort(q, rq);
Jens Axboe3ed9a292007-04-23 08:33:33 +02002982
Corrado Zoccolo53c583d2010-02-28 19:45:05 +01002983 cfqd->rq_in_flight[cfq_cfqq_sync(cfqq)]++;
Vivek Goyalc4e78932010-08-23 12:25:03 +02002984 cfqq->nr_sectors += blk_rq_sectors(rq);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002985}
2986
2987/*
2988 * return expired entry, or NULL to just start from scratch in rbtree
2989 */
Jens Axboefebffd62008-01-28 13:19:43 +01002990static struct request *cfq_check_fifo(struct cfq_queue *cfqq)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002991{
Jens Axboe30996f42009-10-05 11:03:39 +02002992 struct request *rq = NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002993
Jens Axboe3b181522005-06-27 10:56:24 +02002994 if (cfq_cfqq_fifo_expire(cfqq))
Linus Torvalds1da177e2005-04-16 15:20:36 -07002995 return NULL;
Jens Axboecb887412007-01-19 12:01:16 +11002996
2997 cfq_mark_cfqq_fifo_expire(cfqq);
2998
Jens Axboe89850f72006-07-22 16:48:31 +02002999 if (list_empty(&cfqq->fifo))
3000 return NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003001
Jens Axboe89850f72006-07-22 16:48:31 +02003002 rq = rq_entry_fifo(cfqq->fifo.next);
Jan Kara8b4922d2014-02-24 16:39:52 +01003003 if (time_before(jiffies, rq->fifo_time))
Jens Axboe7b679132008-05-30 12:23:07 +02003004 rq = NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003005
Jens Axboe30996f42009-10-05 11:03:39 +02003006 cfq_log_cfqq(cfqq->cfqd, cfqq, "fifo=%p", rq);
Jens Axboe6d048f52007-04-25 12:44:27 +02003007 return rq;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003008}
3009
Jens Axboe22e2c502005-06-27 10:55:12 +02003010static inline int
3011cfq_prio_to_maxrq(struct cfq_data *cfqd, struct cfq_queue *cfqq)
3012{
3013 const int base_rq = cfqd->cfq_slice_async_rq;
3014
3015 WARN_ON(cfqq->ioprio >= IOPRIO_BE_NR);
3016
Namhyung Kimb9f8ce02011-05-24 10:23:21 +02003017 return 2 * base_rq * (IOPRIO_BE_NR - cfqq->ioprio);
Jens Axboe22e2c502005-06-27 10:55:12 +02003018}
3019
3020/*
Jeff Moyerdf5fe3e2009-10-23 17:14:50 -04003021 * Must be called with the queue_lock held.
3022 */
3023static int cfqq_process_refs(struct cfq_queue *cfqq)
3024{
3025 int process_refs, io_refs;
3026
3027 io_refs = cfqq->allocated[READ] + cfqq->allocated[WRITE];
Shaohua Li30d7b942011-01-07 08:46:59 +01003028 process_refs = cfqq->ref - io_refs;
Jeff Moyerdf5fe3e2009-10-23 17:14:50 -04003029 BUG_ON(process_refs < 0);
3030 return process_refs;
3031}
3032
3033static void cfq_setup_merge(struct cfq_queue *cfqq, struct cfq_queue *new_cfqq)
3034{
Jeff Moyere6c5bc72009-10-23 17:14:52 -04003035 int process_refs, new_process_refs;
Jeff Moyerdf5fe3e2009-10-23 17:14:50 -04003036 struct cfq_queue *__cfqq;
3037
Jeff Moyerc10b61f2010-06-17 10:19:11 -04003038 /*
3039 * If there are no process references on the new_cfqq, then it is
3040 * unsafe to follow the ->new_cfqq chain as other cfqq's in the
3041 * chain may have dropped their last reference (not just their
3042 * last process reference).
3043 */
3044 if (!cfqq_process_refs(new_cfqq))
3045 return;
3046
Jeff Moyerdf5fe3e2009-10-23 17:14:50 -04003047 /* Avoid a circular list and skip interim queue merges */
3048 while ((__cfqq = new_cfqq->new_cfqq)) {
3049 if (__cfqq == cfqq)
3050 return;
3051 new_cfqq = __cfqq;
3052 }
3053
3054 process_refs = cfqq_process_refs(cfqq);
Jeff Moyerc10b61f2010-06-17 10:19:11 -04003055 new_process_refs = cfqq_process_refs(new_cfqq);
Jeff Moyerdf5fe3e2009-10-23 17:14:50 -04003056 /*
3057 * If the process for the cfqq has gone away, there is no
3058 * sense in merging the queues.
3059 */
Jeff Moyerc10b61f2010-06-17 10:19:11 -04003060 if (process_refs == 0 || new_process_refs == 0)
Jeff Moyerdf5fe3e2009-10-23 17:14:50 -04003061 return;
3062
Jeff Moyere6c5bc72009-10-23 17:14:52 -04003063 /*
3064 * Merge in the direction of the lesser amount of work.
3065 */
Jeff Moyere6c5bc72009-10-23 17:14:52 -04003066 if (new_process_refs >= process_refs) {
3067 cfqq->new_cfqq = new_cfqq;
Shaohua Li30d7b942011-01-07 08:46:59 +01003068 new_cfqq->ref += process_refs;
Jeff Moyere6c5bc72009-10-23 17:14:52 -04003069 } else {
3070 new_cfqq->new_cfqq = cfqq;
Shaohua Li30d7b942011-01-07 08:46:59 +01003071 cfqq->ref += new_process_refs;
Jeff Moyere6c5bc72009-10-23 17:14:52 -04003072 }
Jeff Moyerdf5fe3e2009-10-23 17:14:50 -04003073}
3074
Vivek Goyal6d816ec2012-10-03 16:56:59 -04003075static enum wl_type_t cfq_choose_wl_type(struct cfq_data *cfqd,
Vivek Goyal3bf10fe2012-10-03 16:56:56 -04003076 struct cfq_group *cfqg, enum wl_class_t wl_class)
Corrado Zoccolo718eee02009-10-26 22:45:29 +01003077{
3078 struct cfq_queue *queue;
3079 int i;
3080 bool key_valid = false;
3081 unsigned long lowest_key = 0;
3082 enum wl_type_t cur_best = SYNC_NOIDLE_WORKLOAD;
3083
Vivek Goyal65b32a52009-12-16 17:52:59 -05003084 for (i = 0; i <= SYNC_WORKLOAD; ++i) {
3085 /* select the one with lowest rb_key */
Vivek Goyal34b98d02012-10-03 16:56:58 -04003086 queue = cfq_rb_first(st_for(cfqg, wl_class, i));
Corrado Zoccolo718eee02009-10-26 22:45:29 +01003087 if (queue &&
3088 (!key_valid || time_before(queue->rb_key, lowest_key))) {
3089 lowest_key = queue->rb_key;
3090 cur_best = i;
3091 key_valid = true;
3092 }
3093 }
3094
3095 return cur_best;
3096}
3097
Vivek Goyal6d816ec2012-10-03 16:56:59 -04003098static void
3099choose_wl_class_and_type(struct cfq_data *cfqd, struct cfq_group *cfqg)
Corrado Zoccolo718eee02009-10-26 22:45:29 +01003100{
Corrado Zoccolo718eee02009-10-26 22:45:29 +01003101 unsigned slice;
3102 unsigned count;
Vivek Goyalcdb16e82009-12-03 12:59:38 -05003103 struct cfq_rb_root *st;
Vivek Goyal58ff82f2009-12-03 12:59:44 -05003104 unsigned group_slice;
Vivek Goyal4d2ceea2012-10-03 16:56:57 -04003105 enum wl_class_t original_class = cfqd->serving_wl_class;
Vivek Goyal1fa8f6d2009-12-03 12:59:41 -05003106
Corrado Zoccolo718eee02009-10-26 22:45:29 +01003107 /* Choose next priority. RT > BE > IDLE */
Vivek Goyal58ff82f2009-12-03 12:59:44 -05003108 if (cfq_group_busy_queues_wl(RT_WORKLOAD, cfqd, cfqg))
Vivek Goyal4d2ceea2012-10-03 16:56:57 -04003109 cfqd->serving_wl_class = RT_WORKLOAD;
Vivek Goyal58ff82f2009-12-03 12:59:44 -05003110 else if (cfq_group_busy_queues_wl(BE_WORKLOAD, cfqd, cfqg))
Vivek Goyal4d2ceea2012-10-03 16:56:57 -04003111 cfqd->serving_wl_class = BE_WORKLOAD;
Corrado Zoccolo718eee02009-10-26 22:45:29 +01003112 else {
Vivek Goyal4d2ceea2012-10-03 16:56:57 -04003113 cfqd->serving_wl_class = IDLE_WORKLOAD;
Corrado Zoccolo718eee02009-10-26 22:45:29 +01003114 cfqd->workload_expires = jiffies + 1;
3115 return;
3116 }
3117
Vivek Goyal4d2ceea2012-10-03 16:56:57 -04003118 if (original_class != cfqd->serving_wl_class)
Shaohua Li writese4ea0c12010-12-13 14:32:22 +01003119 goto new_workload;
3120
Corrado Zoccolo718eee02009-10-26 22:45:29 +01003121 /*
3122 * For RT and BE, we have to choose also the type
3123 * (SYNC, SYNC_NOIDLE, ASYNC), and to compute a workload
3124 * expiration time
3125 */
Vivek Goyal34b98d02012-10-03 16:56:58 -04003126 st = st_for(cfqg, cfqd->serving_wl_class, cfqd->serving_wl_type);
Vivek Goyalcdb16e82009-12-03 12:59:38 -05003127 count = st->count;
Corrado Zoccolo718eee02009-10-26 22:45:29 +01003128
3129 /*
Vivek Goyal65b32a52009-12-16 17:52:59 -05003130 * check workload expiration, and that we still have other queues ready
Corrado Zoccolo718eee02009-10-26 22:45:29 +01003131 */
Vivek Goyal65b32a52009-12-16 17:52:59 -05003132 if (count && !time_after(jiffies, cfqd->workload_expires))
Corrado Zoccolo718eee02009-10-26 22:45:29 +01003133 return;
3134
Shaohua Li writese4ea0c12010-12-13 14:32:22 +01003135new_workload:
Corrado Zoccolo718eee02009-10-26 22:45:29 +01003136 /* otherwise select new workload type */
Vivek Goyal6d816ec2012-10-03 16:56:59 -04003137 cfqd->serving_wl_type = cfq_choose_wl_type(cfqd, cfqg,
Vivek Goyal4d2ceea2012-10-03 16:56:57 -04003138 cfqd->serving_wl_class);
Vivek Goyal34b98d02012-10-03 16:56:58 -04003139 st = st_for(cfqg, cfqd->serving_wl_class, cfqd->serving_wl_type);
Vivek Goyalcdb16e82009-12-03 12:59:38 -05003140 count = st->count;
Corrado Zoccolo718eee02009-10-26 22:45:29 +01003141
3142 /*
3143 * the workload slice is computed as a fraction of target latency
3144 * proportional to the number of queues in that workload, over
3145 * all the queues in the same priority class
3146 */
Vivek Goyal58ff82f2009-12-03 12:59:44 -05003147 group_slice = cfq_group_slice(cfqd, cfqg);
3148
3149 slice = group_slice * count /
Vivek Goyal4d2ceea2012-10-03 16:56:57 -04003150 max_t(unsigned, cfqg->busy_queues_avg[cfqd->serving_wl_class],
3151 cfq_group_busy_queues_wl(cfqd->serving_wl_class, cfqd,
Vivek Goyal3bf10fe2012-10-03 16:56:56 -04003152 cfqg));
Corrado Zoccolo718eee02009-10-26 22:45:29 +01003153
Vivek Goyal4d2ceea2012-10-03 16:56:57 -04003154 if (cfqd->serving_wl_type == ASYNC_WORKLOAD) {
Vivek Goyalf26bd1f2009-12-03 12:59:54 -05003155 unsigned int tmp;
3156
3157 /*
3158 * Async queues are currently system wide. Just taking
3159 * proportion of queues with-in same group will lead to higher
3160 * async ratio system wide as generally root group is going
3161 * to have higher weight. A more accurate thing would be to
3162 * calculate system wide asnc/sync ratio.
3163 */
Tao Ma5bf14c02012-04-01 14:33:39 -07003164 tmp = cfqd->cfq_target_latency *
3165 cfqg_busy_async_queues(cfqd, cfqg);
Vivek Goyalf26bd1f2009-12-03 12:59:54 -05003166 tmp = tmp/cfqd->busy_queues;
3167 slice = min_t(unsigned, slice, tmp);
3168
Corrado Zoccolo718eee02009-10-26 22:45:29 +01003169 /* async workload slice is scaled down according to
3170 * the sync/async slice ratio. */
3171 slice = slice * cfqd->cfq_slice[0] / cfqd->cfq_slice[1];
Vivek Goyalf26bd1f2009-12-03 12:59:54 -05003172 } else
Corrado Zoccolo718eee02009-10-26 22:45:29 +01003173 /* sync workload slice is at least 2 * cfq_slice_idle */
3174 slice = max(slice, 2 * cfqd->cfq_slice_idle);
3175
3176 slice = max_t(unsigned, slice, CFQ_MIN_TT);
Divyesh Shahb1ffe732010-03-25 15:45:03 +01003177 cfq_log(cfqd, "workload slice:%d", slice);
Corrado Zoccolo718eee02009-10-26 22:45:29 +01003178 cfqd->workload_expires = jiffies + slice;
3179}
3180
Vivek Goyal1fa8f6d2009-12-03 12:59:41 -05003181static struct cfq_group *cfq_get_next_cfqg(struct cfq_data *cfqd)
3182{
3183 struct cfq_rb_root *st = &cfqd->grp_service_tree;
Vivek Goyal25bc6b02009-12-03 12:59:43 -05003184 struct cfq_group *cfqg;
Vivek Goyal1fa8f6d2009-12-03 12:59:41 -05003185
3186 if (RB_EMPTY_ROOT(&st->rb))
3187 return NULL;
Vivek Goyal25bc6b02009-12-03 12:59:43 -05003188 cfqg = cfq_rb_first_group(st);
Vivek Goyal25bc6b02009-12-03 12:59:43 -05003189 update_min_vdisktime(st);
3190 return cfqg;
Vivek Goyal1fa8f6d2009-12-03 12:59:41 -05003191}
3192
Vivek Goyalcdb16e82009-12-03 12:59:38 -05003193static void cfq_choose_cfqg(struct cfq_data *cfqd)
3194{
Vivek Goyal1fa8f6d2009-12-03 12:59:41 -05003195 struct cfq_group *cfqg = cfq_get_next_cfqg(cfqd);
3196
3197 cfqd->serving_group = cfqg;
Vivek Goyaldae739e2009-12-03 12:59:45 -05003198
3199 /* Restore the workload type data */
Vivek Goyal4d2ceea2012-10-03 16:56:57 -04003200 if (cfqg->saved_wl_slice) {
3201 cfqd->workload_expires = jiffies + cfqg->saved_wl_slice;
3202 cfqd->serving_wl_type = cfqg->saved_wl_type;
3203 cfqd->serving_wl_class = cfqg->saved_wl_class;
Gui Jianfeng66ae2912009-12-15 10:08:45 +01003204 } else
3205 cfqd->workload_expires = jiffies - 1;
3206
Vivek Goyal6d816ec2012-10-03 16:56:59 -04003207 choose_wl_class_and_type(cfqd, cfqg);
Vivek Goyalcdb16e82009-12-03 12:59:38 -05003208}
3209
Jeff Moyerdf5fe3e2009-10-23 17:14:50 -04003210/*
Jens Axboe498d3aa22007-04-26 12:54:48 +02003211 * Select a queue for service. If we have a current active queue,
3212 * check whether to continue servicing it, or retrieve and set a new one.
Jens Axboe22e2c502005-06-27 10:55:12 +02003213 */
Tejun Heo1b5ed5e12005-11-10 08:49:19 +01003214static struct cfq_queue *cfq_select_queue(struct cfq_data *cfqd)
Jens Axboe22e2c502005-06-27 10:55:12 +02003215{
Jens Axboea36e71f2009-04-15 12:15:11 +02003216 struct cfq_queue *cfqq, *new_cfqq = NULL;
Jens Axboe22e2c502005-06-27 10:55:12 +02003217
3218 cfqq = cfqd->active_queue;
3219 if (!cfqq)
3220 goto new_queue;
3221
Vivek Goyalf04a6422009-12-03 12:59:40 -05003222 if (!cfqd->rq_queued)
3223 return NULL;
Vivek Goyalc244bb52009-12-08 17:52:57 -05003224
3225 /*
3226 * We were waiting for group to get backlogged. Expire the queue
3227 */
3228 if (cfq_cfqq_wait_busy(cfqq) && !RB_EMPTY_ROOT(&cfqq->sort_list))
3229 goto expire;
3230
Jens Axboe22e2c502005-06-27 10:55:12 +02003231 /*
Jens Axboe6d048f52007-04-25 12:44:27 +02003232 * The active queue has run out of time, expire it and select new.
Jens Axboe22e2c502005-06-27 10:55:12 +02003233 */
Vivek Goyal7667aa02009-12-08 17:52:58 -05003234 if (cfq_slice_used(cfqq) && !cfq_cfqq_must_dispatch(cfqq)) {
3235 /*
3236 * If slice had not expired at the completion of last request
3237 * we might not have turned on wait_busy flag. Don't expire
3238 * the queue yet. Allow the group to get backlogged.
3239 *
3240 * The very fact that we have used the slice, that means we
3241 * have been idling all along on this queue and it should be
3242 * ok to wait for this request to complete.
3243 */
Vivek Goyal82bbbf22009-12-10 19:25:41 +01003244 if (cfqq->cfqg->nr_cfqq == 1 && RB_EMPTY_ROOT(&cfqq->sort_list)
3245 && cfqq->dispatched && cfq_should_idle(cfqd, cfqq)) {
3246 cfqq = NULL;
Vivek Goyal7667aa02009-12-08 17:52:58 -05003247 goto keep_queue;
Vivek Goyal82bbbf22009-12-10 19:25:41 +01003248 } else
Vivek Goyal80bdf0c2010-08-23 12:24:26 +02003249 goto check_group_idle;
Vivek Goyal7667aa02009-12-08 17:52:58 -05003250 }
Jens Axboe22e2c502005-06-27 10:55:12 +02003251
3252 /*
Jens Axboe6d048f52007-04-25 12:44:27 +02003253 * The active queue has requests and isn't expired, allow it to
3254 * dispatch.
Jens Axboe22e2c502005-06-27 10:55:12 +02003255 */
Jens Axboedd67d052006-06-21 09:36:18 +02003256 if (!RB_EMPTY_ROOT(&cfqq->sort_list))
Jens Axboe22e2c502005-06-27 10:55:12 +02003257 goto keep_queue;
Jens Axboe6d048f52007-04-25 12:44:27 +02003258
3259 /*
Jens Axboea36e71f2009-04-15 12:15:11 +02003260 * If another queue has a request waiting within our mean seek
3261 * distance, let it run. The expire code will check for close
3262 * cooperators and put the close queue at the front of the service
Jeff Moyerdf5fe3e2009-10-23 17:14:50 -04003263 * tree. If possible, merge the expiring queue with the new cfqq.
Jens Axboea36e71f2009-04-15 12:15:11 +02003264 */
Jeff Moyerb3b6d042009-10-23 17:14:51 -04003265 new_cfqq = cfq_close_cooperator(cfqd, cfqq);
Jeff Moyerdf5fe3e2009-10-23 17:14:50 -04003266 if (new_cfqq) {
3267 if (!cfqq->new_cfqq)
3268 cfq_setup_merge(cfqq, new_cfqq);
Jens Axboea36e71f2009-04-15 12:15:11 +02003269 goto expire;
Jeff Moyerdf5fe3e2009-10-23 17:14:50 -04003270 }
Jens Axboea36e71f2009-04-15 12:15:11 +02003271
3272 /*
Jens Axboe6d048f52007-04-25 12:44:27 +02003273 * No requests pending. If the active queue still has requests in
3274 * flight or is idling for a new request, allow either of these
3275 * conditions to happen (or time out) before selecting a new queue.
3276 */
Vivek Goyal80bdf0c2010-08-23 12:24:26 +02003277 if (timer_pending(&cfqd->idle_slice_timer)) {
3278 cfqq = NULL;
3279 goto keep_queue;
3280 }
3281
Shaohua Li8e1ac662010-11-08 15:01:04 +01003282 /*
3283 * This is a deep seek queue, but the device is much faster than
3284 * the queue can deliver, don't idle
3285 **/
3286 if (CFQQ_SEEKY(cfqq) && cfq_cfqq_idle_window(cfqq) &&
3287 (cfq_cfqq_slice_new(cfqq) ||
3288 (cfqq->slice_end - jiffies > jiffies - cfqq->slice_start))) {
3289 cfq_clear_cfqq_deep(cfqq);
3290 cfq_clear_cfqq_idle_window(cfqq);
3291 }
3292
Vivek Goyal80bdf0c2010-08-23 12:24:26 +02003293 if (cfqq->dispatched && cfq_should_idle(cfqd, cfqq)) {
3294 cfqq = NULL;
3295 goto keep_queue;
3296 }
3297
3298 /*
3299 * If group idle is enabled and there are requests dispatched from
3300 * this group, wait for requests to complete.
3301 */
3302check_group_idle:
Shaohua Li7700fc42011-07-12 14:24:56 +02003303 if (cfqd->cfq_group_idle && cfqq->cfqg->nr_cfqq == 1 &&
3304 cfqq->cfqg->dispatched &&
3305 !cfq_io_thinktime_big(cfqd, &cfqq->cfqg->ttime, true)) {
Jens Axboecaaa5f92006-06-16 11:23:00 +02003306 cfqq = NULL;
3307 goto keep_queue;
Jens Axboe22e2c502005-06-27 10:55:12 +02003308 }
3309
Jens Axboe3b181522005-06-27 10:56:24 +02003310expire:
Vivek Goyale5ff0822010-04-26 19:25:11 +02003311 cfq_slice_expired(cfqd, 0);
Jens Axboe3b181522005-06-27 10:56:24 +02003312new_queue:
Corrado Zoccolo718eee02009-10-26 22:45:29 +01003313 /*
3314 * Current queue expired. Check if we have to switch to a new
3315 * service tree
3316 */
3317 if (!new_cfqq)
Vivek Goyalcdb16e82009-12-03 12:59:38 -05003318 cfq_choose_cfqg(cfqd);
Corrado Zoccolo718eee02009-10-26 22:45:29 +01003319
Jens Axboea36e71f2009-04-15 12:15:11 +02003320 cfqq = cfq_set_active_queue(cfqd, new_cfqq);
Jens Axboe22e2c502005-06-27 10:55:12 +02003321keep_queue:
Jens Axboe3b181522005-06-27 10:56:24 +02003322 return cfqq;
Jens Axboe22e2c502005-06-27 10:55:12 +02003323}
3324
Jens Axboefebffd62008-01-28 13:19:43 +01003325static int __cfq_forced_dispatch_cfqq(struct cfq_queue *cfqq)
Jens Axboed9e76202007-04-20 14:27:50 +02003326{
3327 int dispatched = 0;
3328
3329 while (cfqq->next_rq) {
3330 cfq_dispatch_insert(cfqq->cfqd->queue, cfqq->next_rq);
3331 dispatched++;
3332 }
3333
3334 BUG_ON(!list_empty(&cfqq->fifo));
Vivek Goyalf04a6422009-12-03 12:59:40 -05003335
3336 /* By default cfqq is not expired if it is empty. Do it explicitly */
Vivek Goyale5ff0822010-04-26 19:25:11 +02003337 __cfq_slice_expired(cfqq->cfqd, cfqq, 0);
Jens Axboed9e76202007-04-20 14:27:50 +02003338 return dispatched;
3339}
3340
Jens Axboe498d3aa22007-04-26 12:54:48 +02003341/*
3342 * Drain our current requests. Used for barriers and when switching
3343 * io schedulers on-the-fly.
3344 */
Jens Axboed9e76202007-04-20 14:27:50 +02003345static int cfq_forced_dispatch(struct cfq_data *cfqd)
Tejun Heo1b5ed5e12005-11-10 08:49:19 +01003346{
Jens Axboe08717142008-01-28 11:38:15 +01003347 struct cfq_queue *cfqq;
Jens Axboed9e76202007-04-20 14:27:50 +02003348 int dispatched = 0;
Vivek Goyalcdb16e82009-12-03 12:59:38 -05003349
Divyesh Shah3440c492010-04-09 09:29:57 +02003350 /* Expire the timeslice of the current active queue first */
Vivek Goyale5ff0822010-04-26 19:25:11 +02003351 cfq_slice_expired(cfqd, 0);
Divyesh Shah3440c492010-04-09 09:29:57 +02003352 while ((cfqq = cfq_get_next_queue_forced(cfqd)) != NULL) {
3353 __cfq_set_active_queue(cfqd, cfqq);
Vivek Goyalf04a6422009-12-03 12:59:40 -05003354 dispatched += __cfq_forced_dispatch_cfqq(cfqq);
Divyesh Shah3440c492010-04-09 09:29:57 +02003355 }
Tejun Heo1b5ed5e12005-11-10 08:49:19 +01003356
Tejun Heo1b5ed5e12005-11-10 08:49:19 +01003357 BUG_ON(cfqd->busy_queues);
3358
Jeff Moyer69237152009-06-12 15:29:30 +02003359 cfq_log(cfqd, "forced_dispatch=%d", dispatched);
Tejun Heo1b5ed5e12005-11-10 08:49:19 +01003360 return dispatched;
3361}
3362
Shaohua Liabc3c742010-03-01 09:20:54 +01003363static inline bool cfq_slice_used_soon(struct cfq_data *cfqd,
3364 struct cfq_queue *cfqq)
3365{
3366 /* the queue hasn't finished any request, can't estimate */
3367 if (cfq_cfqq_slice_new(cfqq))
Shaohua Lic1e44752010-11-08 15:01:02 +01003368 return true;
Shaohua Liabc3c742010-03-01 09:20:54 +01003369 if (time_after(jiffies + cfqd->cfq_slice_idle * cfqq->dispatched,
3370 cfqq->slice_end))
Shaohua Lic1e44752010-11-08 15:01:02 +01003371 return true;
Shaohua Liabc3c742010-03-01 09:20:54 +01003372
Shaohua Lic1e44752010-11-08 15:01:02 +01003373 return false;
Shaohua Liabc3c742010-03-01 09:20:54 +01003374}
3375
Jens Axboe0b182d62009-10-06 20:49:37 +02003376static bool cfq_may_dispatch(struct cfq_data *cfqd, struct cfq_queue *cfqq)
Jens Axboe2f5cb732009-04-07 08:51:19 +02003377{
Jens Axboe2f5cb732009-04-07 08:51:19 +02003378 unsigned int max_dispatch;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003379
Jens Axboe2f5cb732009-04-07 08:51:19 +02003380 /*
Jens Axboe5ad531d2009-07-03 12:57:48 +02003381 * Drain async requests before we start sync IO
3382 */
Corrado Zoccolo53c583d2010-02-28 19:45:05 +01003383 if (cfq_should_idle(cfqd, cfqq) && cfqd->rq_in_flight[BLK_RW_ASYNC])
Jens Axboe0b182d62009-10-06 20:49:37 +02003384 return false;
Jens Axboe5ad531d2009-07-03 12:57:48 +02003385
3386 /*
Jens Axboe2f5cb732009-04-07 08:51:19 +02003387 * If this is an async queue and we have sync IO in flight, let it wait
3388 */
Corrado Zoccolo53c583d2010-02-28 19:45:05 +01003389 if (cfqd->rq_in_flight[BLK_RW_SYNC] && !cfq_cfqq_sync(cfqq))
Jens Axboe0b182d62009-10-06 20:49:37 +02003390 return false;
Jens Axboe2f5cb732009-04-07 08:51:19 +02003391
Shaohua Liabc3c742010-03-01 09:20:54 +01003392 max_dispatch = max_t(unsigned int, cfqd->cfq_quantum / 2, 1);
Jens Axboe2f5cb732009-04-07 08:51:19 +02003393 if (cfq_class_idle(cfqq))
3394 max_dispatch = 1;
3395
3396 /*
3397 * Does this cfqq already have too much IO in flight?
3398 */
3399 if (cfqq->dispatched >= max_dispatch) {
Shaohua Lief8a41d2011-03-07 09:26:29 +01003400 bool promote_sync = false;
Jens Axboe2f5cb732009-04-07 08:51:19 +02003401 /*
3402 * idle queue must always only have a single IO in flight
3403 */
Jens Axboe3ed9a292007-04-23 08:33:33 +02003404 if (cfq_class_idle(cfqq))
Jens Axboe0b182d62009-10-06 20:49:37 +02003405 return false;
Jens Axboe3ed9a292007-04-23 08:33:33 +02003406
Jens Axboe2f5cb732009-04-07 08:51:19 +02003407 /*
Li, Shaohuac4ade942011-03-23 08:30:34 +01003408 * If there is only one sync queue
3409 * we can ignore async queue here and give the sync
Shaohua Lief8a41d2011-03-07 09:26:29 +01003410 * queue no dispatch limit. The reason is a sync queue can
3411 * preempt async queue, limiting the sync queue doesn't make
3412 * sense. This is useful for aiostress test.
3413 */
Li, Shaohuac4ade942011-03-23 08:30:34 +01003414 if (cfq_cfqq_sync(cfqq) && cfqd->busy_sync_queues == 1)
3415 promote_sync = true;
Shaohua Lief8a41d2011-03-07 09:26:29 +01003416
3417 /*
Jens Axboe2f5cb732009-04-07 08:51:19 +02003418 * We have other queues, don't allow more IO from this one
3419 */
Shaohua Lief8a41d2011-03-07 09:26:29 +01003420 if (cfqd->busy_queues > 1 && cfq_slice_used_soon(cfqd, cfqq) &&
3421 !promote_sync)
Jens Axboe0b182d62009-10-06 20:49:37 +02003422 return false;
Jens Axboe9ede2092007-01-19 12:11:44 +11003423
Jens Axboe2f5cb732009-04-07 08:51:19 +02003424 /*
Shaohua Li474b18c2009-12-03 12:58:05 +01003425 * Sole queue user, no limit
Vivek Goyal365722b2009-10-03 15:21:27 +02003426 */
Shaohua Lief8a41d2011-03-07 09:26:29 +01003427 if (cfqd->busy_queues == 1 || promote_sync)
Shaohua Liabc3c742010-03-01 09:20:54 +01003428 max_dispatch = -1;
3429 else
3430 /*
3431 * Normally we start throttling cfqq when cfq_quantum/2
3432 * requests have been dispatched. But we can drive
3433 * deeper queue depths at the beginning of slice
3434 * subjected to upper limit of cfq_quantum.
3435 * */
3436 max_dispatch = cfqd->cfq_quantum;
Jens Axboe8e296752009-10-03 16:26:03 +02003437 }
3438
3439 /*
3440 * Async queues must wait a bit before being allowed dispatch.
3441 * We also ramp up the dispatch depth gradually for async IO,
3442 * based on the last sync IO we serviced
3443 */
Jens Axboe963b72f2009-10-03 19:42:18 +02003444 if (!cfq_cfqq_sync(cfqq) && cfqd->cfq_latency) {
Corrado Zoccolo573412b2009-12-06 11:48:52 +01003445 unsigned long last_sync = jiffies - cfqd->last_delayed_sync;
Jens Axboe8e296752009-10-03 16:26:03 +02003446 unsigned int depth;
Vivek Goyal365722b2009-10-03 15:21:27 +02003447
Jens Axboe61f0c1d2009-10-03 19:46:03 +02003448 depth = last_sync / cfqd->cfq_slice[1];
Jens Axboee00c54c2009-10-04 20:36:19 +02003449 if (!depth && !cfqq->dispatched)
3450 depth = 1;
Jens Axboe8e296752009-10-03 16:26:03 +02003451 if (depth < max_dispatch)
3452 max_dispatch = depth;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003453 }
3454
Jens Axboe0b182d62009-10-06 20:49:37 +02003455 /*
3456 * If we're below the current max, allow a dispatch
3457 */
3458 return cfqq->dispatched < max_dispatch;
3459}
3460
3461/*
3462 * Dispatch a request from cfqq, moving them to the request queue
3463 * dispatch list.
3464 */
3465static bool cfq_dispatch_request(struct cfq_data *cfqd, struct cfq_queue *cfqq)
3466{
3467 struct request *rq;
3468
3469 BUG_ON(RB_EMPTY_ROOT(&cfqq->sort_list));
3470
3471 if (!cfq_may_dispatch(cfqd, cfqq))
3472 return false;
3473
3474 /*
3475 * follow expired path, else get first next available
3476 */
3477 rq = cfq_check_fifo(cfqq);
3478 if (!rq)
3479 rq = cfqq->next_rq;
3480
3481 /*
3482 * insert request into driver dispatch list
3483 */
3484 cfq_dispatch_insert(cfqd->queue, rq);
3485
3486 if (!cfqd->active_cic) {
Tejun Heoc5869802011-12-14 00:33:41 +01003487 struct cfq_io_cq *cic = RQ_CIC(rq);
Jens Axboe0b182d62009-10-06 20:49:37 +02003488
Tejun Heoc5869802011-12-14 00:33:41 +01003489 atomic_long_inc(&cic->icq.ioc->refcount);
Jens Axboe0b182d62009-10-06 20:49:37 +02003490 cfqd->active_cic = cic;
3491 }
3492
3493 return true;
3494}
3495
3496/*
3497 * Find the cfqq that we need to service and move a request from that to the
3498 * dispatch list
3499 */
3500static int cfq_dispatch_requests(struct request_queue *q, int force)
3501{
3502 struct cfq_data *cfqd = q->elevator->elevator_data;
3503 struct cfq_queue *cfqq;
3504
3505 if (!cfqd->busy_queues)
3506 return 0;
3507
3508 if (unlikely(force))
3509 return cfq_forced_dispatch(cfqd);
3510
3511 cfqq = cfq_select_queue(cfqd);
3512 if (!cfqq)
Jens Axboe8e296752009-10-03 16:26:03 +02003513 return 0;
3514
Jens Axboe2f5cb732009-04-07 08:51:19 +02003515 /*
Jens Axboe0b182d62009-10-06 20:49:37 +02003516 * Dispatch a request from this cfqq, if it is allowed
Jens Axboe2f5cb732009-04-07 08:51:19 +02003517 */
Jens Axboe0b182d62009-10-06 20:49:37 +02003518 if (!cfq_dispatch_request(cfqd, cfqq))
3519 return 0;
3520
Jens Axboe2f5cb732009-04-07 08:51:19 +02003521 cfqq->slice_dispatch++;
Jens Axboeb0291952009-04-07 11:38:31 +02003522 cfq_clear_cfqq_must_dispatch(cfqq);
Jens Axboe2f5cb732009-04-07 08:51:19 +02003523
3524 /*
3525 * expire an async queue immediately if it has used up its slice. idle
3526 * queue always expire after 1 dispatch round.
3527 */
3528 if (cfqd->busy_queues > 1 && ((!cfq_cfqq_sync(cfqq) &&
3529 cfqq->slice_dispatch >= cfq_prio_to_maxrq(cfqd, cfqq)) ||
3530 cfq_class_idle(cfqq))) {
3531 cfqq->slice_end = jiffies + 1;
Vivek Goyale5ff0822010-04-26 19:25:11 +02003532 cfq_slice_expired(cfqd, 0);
Jens Axboe2f5cb732009-04-07 08:51:19 +02003533 }
3534
Shan Weib217a902009-09-01 10:06:42 +02003535 cfq_log_cfqq(cfqd, cfqq, "dispatched a request");
Jens Axboe2f5cb732009-04-07 08:51:19 +02003536 return 1;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003537}
3538
Linus Torvalds1da177e2005-04-16 15:20:36 -07003539/*
Jens Axboe5e705372006-07-13 12:39:25 +02003540 * task holds one reference to the queue, dropped when task exits. each rq
3541 * in-flight on this queue also holds a reference, dropped when rq is freed.
Linus Torvalds1da177e2005-04-16 15:20:36 -07003542 *
Vivek Goyalb1c35762009-12-03 12:59:47 -05003543 * Each cfq queue took a reference on the parent group. Drop it now.
Linus Torvalds1da177e2005-04-16 15:20:36 -07003544 * queue lock must be held here.
3545 */
3546static void cfq_put_queue(struct cfq_queue *cfqq)
3547{
Jens Axboe22e2c502005-06-27 10:55:12 +02003548 struct cfq_data *cfqd = cfqq->cfqd;
Justin TerAvest0bbfeb82011-03-01 15:05:08 -05003549 struct cfq_group *cfqg;
Jens Axboe22e2c502005-06-27 10:55:12 +02003550
Shaohua Li30d7b942011-01-07 08:46:59 +01003551 BUG_ON(cfqq->ref <= 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003552
Shaohua Li30d7b942011-01-07 08:46:59 +01003553 cfqq->ref--;
3554 if (cfqq->ref)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003555 return;
3556
Jens Axboe7b679132008-05-30 12:23:07 +02003557 cfq_log_cfqq(cfqd, cfqq, "put_queue");
Linus Torvalds1da177e2005-04-16 15:20:36 -07003558 BUG_ON(rb_first(&cfqq->sort_list));
Jens Axboe22e2c502005-06-27 10:55:12 +02003559 BUG_ON(cfqq->allocated[READ] + cfqq->allocated[WRITE]);
Vivek Goyalb1c35762009-12-03 12:59:47 -05003560 cfqg = cfqq->cfqg;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003561
Jens Axboe28f95cbc2007-01-19 12:09:53 +11003562 if (unlikely(cfqd->active_queue == cfqq)) {
Vivek Goyale5ff0822010-04-26 19:25:11 +02003563 __cfq_slice_expired(cfqd, cfqq, 0);
Jens Axboe23e018a2009-10-05 08:52:35 +02003564 cfq_schedule_dispatch(cfqd);
Jens Axboe28f95cbc2007-01-19 12:09:53 +11003565 }
Jens Axboe22e2c502005-06-27 10:55:12 +02003566
Vivek Goyalf04a6422009-12-03 12:59:40 -05003567 BUG_ON(cfq_cfqq_on_rr(cfqq));
Linus Torvalds1da177e2005-04-16 15:20:36 -07003568 kmem_cache_free(cfq_pool, cfqq);
Tejun Heoeb7d8c072012-03-23 14:02:53 +01003569 cfqg_put(cfqg);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003570}
3571
Shaohua Lid02a2c02010-05-25 10:16:53 +02003572static void cfq_put_cooperator(struct cfq_queue *cfqq)
Jens Axboe89850f72006-07-22 16:48:31 +02003573{
Jeff Moyerdf5fe3e2009-10-23 17:14:50 -04003574 struct cfq_queue *__cfqq, *next;
3575
Jeff Moyerdf5fe3e2009-10-23 17:14:50 -04003576 /*
3577 * If this queue was scheduled to merge with another queue, be
3578 * sure to drop the reference taken on that queue (and others in
3579 * the merge chain). See cfq_setup_merge and cfq_merge_cfqqs.
3580 */
3581 __cfqq = cfqq->new_cfqq;
3582 while (__cfqq) {
3583 if (__cfqq == cfqq) {
3584 WARN(1, "cfqq->new_cfqq loop detected\n");
3585 break;
3586 }
3587 next = __cfqq->new_cfqq;
3588 cfq_put_queue(__cfqq);
3589 __cfqq = next;
3590 }
Shaohua Lid02a2c02010-05-25 10:16:53 +02003591}
3592
3593static void cfq_exit_cfqq(struct cfq_data *cfqd, struct cfq_queue *cfqq)
3594{
3595 if (unlikely(cfqq == cfqd->active_queue)) {
3596 __cfq_slice_expired(cfqd, cfqq, 0);
3597 cfq_schedule_dispatch(cfqd);
3598 }
3599
3600 cfq_put_cooperator(cfqq);
Jeff Moyerdf5fe3e2009-10-23 17:14:50 -04003601
Jens Axboe89850f72006-07-22 16:48:31 +02003602 cfq_put_queue(cfqq);
3603}
3604
Tejun Heo9b84cac2011-12-14 00:33:42 +01003605static void cfq_init_icq(struct io_cq *icq)
3606{
3607 struct cfq_io_cq *cic = icq_to_cic(icq);
3608
3609 cic->ttime.last_end_request = jiffies;
3610}
3611
Tejun Heoc5869802011-12-14 00:33:41 +01003612static void cfq_exit_icq(struct io_cq *icq)
Jens Axboe89850f72006-07-22 16:48:31 +02003613{
Tejun Heoc5869802011-12-14 00:33:41 +01003614 struct cfq_io_cq *cic = icq_to_cic(icq);
Tejun Heo283287a2011-12-14 00:33:38 +01003615 struct cfq_data *cfqd = cic_to_cfqd(cic);
Fabio Checconi4faa3c82008-04-10 08:28:01 +02003616
Tejun Heo563180a2015-08-18 14:55:00 -07003617 if (cic_to_cfqq(cic, false)) {
3618 cfq_exit_cfqq(cfqd, cic_to_cfqq(cic, false));
3619 cic_set_cfqq(cic, NULL, false);
Jens Axboe89850f72006-07-22 16:48:31 +02003620 }
3621
Tejun Heo563180a2015-08-18 14:55:00 -07003622 if (cic_to_cfqq(cic, true)) {
3623 cfq_exit_cfqq(cfqd, cic_to_cfqq(cic, true));
3624 cic_set_cfqq(cic, NULL, true);
Jens Axboe89850f72006-07-22 16:48:31 +02003625 }
Jens Axboe89850f72006-07-22 16:48:31 +02003626}
3627
Tejun Heoabede6d2012-03-19 15:10:57 -07003628static void cfq_init_prio_data(struct cfq_queue *cfqq, struct cfq_io_cq *cic)
Jens Axboe22e2c502005-06-27 10:55:12 +02003629{
3630 struct task_struct *tsk = current;
3631 int ioprio_class;
3632
Jens Axboe3b181522005-06-27 10:56:24 +02003633 if (!cfq_cfqq_prio_changed(cfqq))
Jens Axboe22e2c502005-06-27 10:55:12 +02003634 return;
3635
Tejun Heo598971b2012-03-19 15:10:58 -07003636 ioprio_class = IOPRIO_PRIO_CLASS(cic->ioprio);
Jens Axboe22e2c502005-06-27 10:55:12 +02003637 switch (ioprio_class) {
Jens Axboefe094d92008-01-31 13:08:54 +01003638 default:
3639 printk(KERN_ERR "cfq: bad prio %x\n", ioprio_class);
3640 case IOPRIO_CLASS_NONE:
3641 /*
Jens Axboe6d63c272008-05-07 09:51:23 +02003642 * no prio set, inherit CPU scheduling settings
Jens Axboefe094d92008-01-31 13:08:54 +01003643 */
3644 cfqq->ioprio = task_nice_ioprio(tsk);
Jens Axboe6d63c272008-05-07 09:51:23 +02003645 cfqq->ioprio_class = task_nice_ioclass(tsk);
Jens Axboefe094d92008-01-31 13:08:54 +01003646 break;
3647 case IOPRIO_CLASS_RT:
Tejun Heo598971b2012-03-19 15:10:58 -07003648 cfqq->ioprio = IOPRIO_PRIO_DATA(cic->ioprio);
Jens Axboefe094d92008-01-31 13:08:54 +01003649 cfqq->ioprio_class = IOPRIO_CLASS_RT;
3650 break;
3651 case IOPRIO_CLASS_BE:
Tejun Heo598971b2012-03-19 15:10:58 -07003652 cfqq->ioprio = IOPRIO_PRIO_DATA(cic->ioprio);
Jens Axboefe094d92008-01-31 13:08:54 +01003653 cfqq->ioprio_class = IOPRIO_CLASS_BE;
3654 break;
3655 case IOPRIO_CLASS_IDLE:
3656 cfqq->ioprio_class = IOPRIO_CLASS_IDLE;
3657 cfqq->ioprio = 7;
3658 cfq_clear_cfqq_idle_window(cfqq);
3659 break;
Jens Axboe22e2c502005-06-27 10:55:12 +02003660 }
3661
3662 /*
3663 * keep track of original prio settings in case we have to temporarily
3664 * elevate the priority of this queue
3665 */
3666 cfqq->org_ioprio = cfqq->ioprio;
Jens Axboe3b181522005-06-27 10:56:24 +02003667 cfq_clear_cfqq_prio_changed(cfqq);
Jens Axboe22e2c502005-06-27 10:55:12 +02003668}
3669
Tejun Heo598971b2012-03-19 15:10:58 -07003670static void check_ioprio_changed(struct cfq_io_cq *cic, struct bio *bio)
Jens Axboe22e2c502005-06-27 10:55:12 +02003671{
Tejun Heo598971b2012-03-19 15:10:58 -07003672 int ioprio = cic->icq.ioc->ioprio;
Konstantin Khlebnikovbca4b912010-05-20 23:21:34 +04003673 struct cfq_data *cfqd = cic_to_cfqd(cic);
Al Viro478a82b2006-03-18 13:25:24 -05003674 struct cfq_queue *cfqq;
Jens Axboe35e60772006-06-14 09:10:45 +02003675
Tejun Heo598971b2012-03-19 15:10:58 -07003676 /*
3677 * Check whether ioprio has changed. The condition may trigger
3678 * spuriously on a newly created cic but there's no harm.
3679 */
3680 if (unlikely(!cfqd) || likely(cic->ioprio == ioprio))
Jens Axboecaaa5f92006-06-16 11:23:00 +02003681 return;
3682
Tejun Heo563180a2015-08-18 14:55:00 -07003683 cfqq = cic_to_cfqq(cic, false);
Jens Axboecaaa5f92006-06-16 11:23:00 +02003684 if (cfqq) {
Tejun Heo563180a2015-08-18 14:55:00 -07003685 cfq_put_queue(cfqq);
Tejun Heo2da8de02015-08-18 14:55:02 -07003686 cfqq = cfq_get_queue(cfqd, BLK_RW_ASYNC, cic, bio);
Tejun Heo563180a2015-08-18 14:55:00 -07003687 cic_set_cfqq(cic, cfqq, false);
Jens Axboe22e2c502005-06-27 10:55:12 +02003688 }
Jens Axboecaaa5f92006-06-16 11:23:00 +02003689
Tejun Heo563180a2015-08-18 14:55:00 -07003690 cfqq = cic_to_cfqq(cic, true);
Jens Axboecaaa5f92006-06-16 11:23:00 +02003691 if (cfqq)
3692 cfq_mark_cfqq_prio_changed(cfqq);
Tejun Heo598971b2012-03-19 15:10:58 -07003693
3694 cic->ioprio = ioprio;
Jens Axboe22e2c502005-06-27 10:55:12 +02003695}
3696
Jens Axboed5036d72009-06-26 10:44:34 +02003697static void cfq_init_cfqq(struct cfq_data *cfqd, struct cfq_queue *cfqq,
Jens Axboea6151c32009-10-07 20:02:57 +02003698 pid_t pid, bool is_sync)
Jens Axboed5036d72009-06-26 10:44:34 +02003699{
3700 RB_CLEAR_NODE(&cfqq->rb_node);
3701 RB_CLEAR_NODE(&cfqq->p_node);
3702 INIT_LIST_HEAD(&cfqq->fifo);
3703
Shaohua Li30d7b942011-01-07 08:46:59 +01003704 cfqq->ref = 0;
Jens Axboed5036d72009-06-26 10:44:34 +02003705 cfqq->cfqd = cfqd;
3706
3707 cfq_mark_cfqq_prio_changed(cfqq);
3708
3709 if (is_sync) {
3710 if (!cfq_class_idle(cfqq))
3711 cfq_mark_cfqq_idle_window(cfqq);
3712 cfq_mark_cfqq_sync(cfqq);
3713 }
3714 cfqq->pid = pid;
3715}
3716
Vivek Goyal246103332009-12-03 12:59:51 -05003717#ifdef CONFIG_CFQ_GROUP_IOSCHED
Tejun Heo598971b2012-03-19 15:10:58 -07003718static void check_blkcg_changed(struct cfq_io_cq *cic, struct bio *bio)
Vivek Goyal246103332009-12-03 12:59:51 -05003719{
Konstantin Khlebnikovbca4b912010-05-20 23:21:34 +04003720 struct cfq_data *cfqd = cic_to_cfqd(cic);
Tejun Heo60a83702015-08-18 14:55:05 -07003721 struct cfq_queue *cfqq;
Tejun Heof4da8072014-09-08 08:15:20 +09003722 uint64_t serial_nr;
Vivek Goyal246103332009-12-03 12:59:51 -05003723
Tejun Heo598971b2012-03-19 15:10:58 -07003724 rcu_read_lock();
Tejun Heof4da8072014-09-08 08:15:20 +09003725 serial_nr = bio_blkcg(bio)->css.serial_nr;
Tejun Heo598971b2012-03-19 15:10:58 -07003726 rcu_read_unlock();
3727
3728 /*
3729 * Check whether blkcg has changed. The condition may trigger
3730 * spuriously on a newly created cic but there's no harm.
3731 */
Tejun Heof4da8072014-09-08 08:15:20 +09003732 if (unlikely(!cfqd) || likely(cic->blkcg_serial_nr == serial_nr))
Vivek Goyal246103332009-12-03 12:59:51 -05003733 return;
3734
Tejun Heo60a83702015-08-18 14:55:05 -07003735 /*
3736 * Drop reference to queues. New queues will be assigned in new
3737 * group upon arrival of fresh requests.
3738 */
3739 cfqq = cic_to_cfqq(cic, false);
3740 if (cfqq) {
3741 cfq_log_cfqq(cfqd, cfqq, "changed cgroup");
3742 cic_set_cfqq(cic, NULL, false);
3743 cfq_put_queue(cfqq);
3744 }
3745
3746 cfqq = cic_to_cfqq(cic, true);
3747 if (cfqq) {
3748 cfq_log_cfqq(cfqd, cfqq, "changed cgroup");
3749 cic_set_cfqq(cic, NULL, true);
3750 cfq_put_queue(cfqq);
Vivek Goyal246103332009-12-03 12:59:51 -05003751 }
Tejun Heo598971b2012-03-19 15:10:58 -07003752
Tejun Heof4da8072014-09-08 08:15:20 +09003753 cic->blkcg_serial_nr = serial_nr;
Vivek Goyal246103332009-12-03 12:59:51 -05003754}
Tejun Heo598971b2012-03-19 15:10:58 -07003755#else
3756static inline void check_blkcg_changed(struct cfq_io_cq *cic, struct bio *bio) { }
Vivek Goyal246103332009-12-03 12:59:51 -05003757#endif /* CONFIG_CFQ_GROUP_IOSCHED */
3758
Vasily Tarasovc2dea2d2007-07-20 10:06:38 +02003759static struct cfq_queue **
Tejun Heo60a83702015-08-18 14:55:05 -07003760cfq_async_queue_prio(struct cfq_group *cfqg, int ioprio_class, int ioprio)
Vasily Tarasovc2dea2d2007-07-20 10:06:38 +02003761{
Jens Axboefe094d92008-01-31 13:08:54 +01003762 switch (ioprio_class) {
Vasily Tarasovc2dea2d2007-07-20 10:06:38 +02003763 case IOPRIO_CLASS_RT:
Tejun Heo60a83702015-08-18 14:55:05 -07003764 return &cfqg->async_cfqq[0][ioprio];
Tejun Heo598971b2012-03-19 15:10:58 -07003765 case IOPRIO_CLASS_NONE:
3766 ioprio = IOPRIO_NORM;
3767 /* fall through */
Vasily Tarasovc2dea2d2007-07-20 10:06:38 +02003768 case IOPRIO_CLASS_BE:
Tejun Heo60a83702015-08-18 14:55:05 -07003769 return &cfqg->async_cfqq[1][ioprio];
Vasily Tarasovc2dea2d2007-07-20 10:06:38 +02003770 case IOPRIO_CLASS_IDLE:
Tejun Heo60a83702015-08-18 14:55:05 -07003771 return &cfqg->async_idle_cfqq;
Vasily Tarasovc2dea2d2007-07-20 10:06:38 +02003772 default:
3773 BUG();
3774 }
3775}
3776
Jens Axboe15c31be2007-07-10 13:43:25 +02003777static struct cfq_queue *
Tejun Heoabede6d2012-03-19 15:10:57 -07003778cfq_get_queue(struct cfq_data *cfqd, bool is_sync, struct cfq_io_cq *cic,
Tejun Heo2da8de02015-08-18 14:55:02 -07003779 struct bio *bio)
Jens Axboe15c31be2007-07-10 13:43:25 +02003780{
Jeff Moyerc6ce1942015-01-12 15:21:01 -05003781 int ioprio_class = IOPRIO_PRIO_CLASS(cic->ioprio);
3782 int ioprio = IOPRIO_PRIO_DATA(cic->ioprio);
Tejun Heod4aad7f2015-08-18 14:55:04 -07003783 struct cfq_queue **async_cfqq = NULL;
Tejun Heo4ebc1c62015-08-18 14:54:57 -07003784 struct cfq_queue *cfqq;
Tejun Heo322731e2015-08-18 14:55:03 -07003785 struct cfq_group *cfqg;
3786
3787 rcu_read_lock();
Tejun Heoae118892015-08-18 14:55:20 -07003788 cfqg = cfq_lookup_cfqg(cfqd, bio_blkcg(bio));
Tejun Heo322731e2015-08-18 14:55:03 -07003789 if (!cfqg) {
3790 cfqq = &cfqd->oom_cfqq;
3791 goto out;
3792 }
Jens Axboe15c31be2007-07-10 13:43:25 +02003793
Vasily Tarasovc2dea2d2007-07-20 10:06:38 +02003794 if (!is_sync) {
Jeff Moyerc6ce1942015-01-12 15:21:01 -05003795 if (!ioprio_valid(cic->ioprio)) {
3796 struct task_struct *tsk = current;
3797 ioprio = task_nice_ioprio(tsk);
3798 ioprio_class = task_nice_ioclass(tsk);
3799 }
Tejun Heo60a83702015-08-18 14:55:05 -07003800 async_cfqq = cfq_async_queue_prio(cfqg, ioprio_class, ioprio);
Vasily Tarasovc2dea2d2007-07-20 10:06:38 +02003801 cfqq = *async_cfqq;
Tejun Heo4ebc1c62015-08-18 14:54:57 -07003802 if (cfqq)
3803 goto out;
Vasily Tarasovc2dea2d2007-07-20 10:06:38 +02003804 }
3805
Tejun Heod4aad7f2015-08-18 14:55:04 -07003806 cfqq = kmem_cache_alloc_node(cfq_pool, GFP_NOWAIT | __GFP_ZERO,
3807 cfqd->queue->node);
3808 if (!cfqq) {
3809 cfqq = &cfqd->oom_cfqq;
3810 goto out;
3811 }
Jens Axboe15c31be2007-07-10 13:43:25 +02003812
Tejun Heod4aad7f2015-08-18 14:55:04 -07003813 cfq_init_cfqq(cfqd, cfqq, current->pid, is_sync);
3814 cfq_init_prio_data(cfqq, cic);
3815 cfq_link_cfqq_cfqg(cfqq, cfqg);
3816 cfq_log_cfqq(cfqd, cfqq, "alloced");
3817
3818 if (async_cfqq) {
3819 /* a new async queue is created, pin and remember */
Shaohua Li30d7b942011-01-07 08:46:59 +01003820 cfqq->ref++;
Vasily Tarasovc2dea2d2007-07-20 10:06:38 +02003821 *async_cfqq = cfqq;
Jens Axboe15c31be2007-07-10 13:43:25 +02003822 }
Tejun Heo4ebc1c62015-08-18 14:54:57 -07003823out:
Shaohua Li30d7b942011-01-07 08:46:59 +01003824 cfqq->ref++;
Tejun Heo322731e2015-08-18 14:55:03 -07003825 rcu_read_unlock();
Jens Axboe15c31be2007-07-10 13:43:25 +02003826 return cfqq;
3827}
3828
Jens Axboe22e2c502005-06-27 10:55:12 +02003829static void
Shaohua Li383cd722011-07-12 14:24:35 +02003830__cfq_update_io_thinktime(struct cfq_ttime *ttime, unsigned long slice_idle)
Jens Axboe22e2c502005-06-27 10:55:12 +02003831{
Shaohua Li383cd722011-07-12 14:24:35 +02003832 unsigned long elapsed = jiffies - ttime->last_end_request;
3833 elapsed = min(elapsed, 2UL * slice_idle);
Jens Axboe22e2c502005-06-27 10:55:12 +02003834
Shaohua Li383cd722011-07-12 14:24:35 +02003835 ttime->ttime_samples = (7*ttime->ttime_samples + 256) / 8;
3836 ttime->ttime_total = (7*ttime->ttime_total + 256*elapsed) / 8;
3837 ttime->ttime_mean = (ttime->ttime_total + 128) / ttime->ttime_samples;
3838}
3839
3840static void
3841cfq_update_io_thinktime(struct cfq_data *cfqd, struct cfq_queue *cfqq,
Tejun Heoc5869802011-12-14 00:33:41 +01003842 struct cfq_io_cq *cic)
Shaohua Li383cd722011-07-12 14:24:35 +02003843{
Shaohua Lif5f2b6c2011-07-12 14:24:55 +02003844 if (cfq_cfqq_sync(cfqq)) {
Shaohua Li383cd722011-07-12 14:24:35 +02003845 __cfq_update_io_thinktime(&cic->ttime, cfqd->cfq_slice_idle);
Shaohua Lif5f2b6c2011-07-12 14:24:55 +02003846 __cfq_update_io_thinktime(&cfqq->service_tree->ttime,
3847 cfqd->cfq_slice_idle);
3848 }
Shaohua Li7700fc42011-07-12 14:24:56 +02003849#ifdef CONFIG_CFQ_GROUP_IOSCHED
3850 __cfq_update_io_thinktime(&cfqq->cfqg->ttime, cfqd->cfq_group_idle);
3851#endif
Jens Axboe22e2c502005-06-27 10:55:12 +02003852}
3853
Jens Axboe206dc692006-03-28 13:03:44 +02003854static void
Jeff Moyerb2c18e12009-10-23 17:14:49 -04003855cfq_update_io_seektime(struct cfq_data *cfqd, struct cfq_queue *cfqq,
Jens Axboe6d048f52007-04-25 12:44:27 +02003856 struct request *rq)
Jens Axboe206dc692006-03-28 13:03:44 +02003857{
Corrado Zoccolo3dde36d2010-02-27 19:45:39 +01003858 sector_t sdist = 0;
Corrado Zoccolo41647e72010-02-27 19:45:40 +01003859 sector_t n_sec = blk_rq_sectors(rq);
Corrado Zoccolo3dde36d2010-02-27 19:45:39 +01003860 if (cfqq->last_request_pos) {
3861 if (cfqq->last_request_pos < blk_rq_pos(rq))
3862 sdist = blk_rq_pos(rq) - cfqq->last_request_pos;
3863 else
3864 sdist = cfqq->last_request_pos - blk_rq_pos(rq);
3865 }
Jens Axboe206dc692006-03-28 13:03:44 +02003866
Corrado Zoccolo3dde36d2010-02-27 19:45:39 +01003867 cfqq->seek_history <<= 1;
Corrado Zoccolo41647e72010-02-27 19:45:40 +01003868 if (blk_queue_nonrot(cfqd->queue))
3869 cfqq->seek_history |= (n_sec < CFQQ_SECT_THR_NONROT);
3870 else
3871 cfqq->seek_history |= (sdist > CFQQ_SEEK_THR);
Jens Axboe206dc692006-03-28 13:03:44 +02003872}
Jens Axboe22e2c502005-06-27 10:55:12 +02003873
3874/*
3875 * Disable idle window if the process thinks too long or seeks so much that
3876 * it doesn't matter
3877 */
3878static void
3879cfq_update_idle_window(struct cfq_data *cfqd, struct cfq_queue *cfqq,
Tejun Heoc5869802011-12-14 00:33:41 +01003880 struct cfq_io_cq *cic)
Jens Axboe22e2c502005-06-27 10:55:12 +02003881{
Jens Axboe7b679132008-05-30 12:23:07 +02003882 int old_idle, enable_idle;
Jens Axboe1be92f2f2007-04-19 14:32:26 +02003883
Jens Axboe08717142008-01-28 11:38:15 +01003884 /*
3885 * Don't idle for async or idle io prio class
3886 */
3887 if (!cfq_cfqq_sync(cfqq) || cfq_class_idle(cfqq))
Jens Axboe1be92f2f2007-04-19 14:32:26 +02003888 return;
3889
Jens Axboec265a7f2008-06-26 13:49:33 +02003890 enable_idle = old_idle = cfq_cfqq_idle_window(cfqq);
Jens Axboe22e2c502005-06-27 10:55:12 +02003891
Corrado Zoccolo76280af2009-11-26 10:02:58 +01003892 if (cfqq->queued[0] + cfqq->queued[1] >= 4)
3893 cfq_mark_cfqq_deep(cfqq);
3894
Corrado Zoccolo749ef9f2010-09-20 15:24:50 +02003895 if (cfqq->next_rq && (cfqq->next_rq->cmd_flags & REQ_NOIDLE))
3896 enable_idle = 0;
Tejun Heof6e8d012012-03-05 13:15:26 -08003897 else if (!atomic_read(&cic->icq.ioc->active_ref) ||
Tejun Heoc5869802011-12-14 00:33:41 +01003898 !cfqd->cfq_slice_idle ||
3899 (!cfq_cfqq_deep(cfqq) && CFQQ_SEEKY(cfqq)))
Jens Axboe22e2c502005-06-27 10:55:12 +02003900 enable_idle = 0;
Shaohua Li383cd722011-07-12 14:24:35 +02003901 else if (sample_valid(cic->ttime.ttime_samples)) {
3902 if (cic->ttime.ttime_mean > cfqd->cfq_slice_idle)
Jens Axboe22e2c502005-06-27 10:55:12 +02003903 enable_idle = 0;
3904 else
3905 enable_idle = 1;
3906 }
3907
Jens Axboe7b679132008-05-30 12:23:07 +02003908 if (old_idle != enable_idle) {
3909 cfq_log_cfqq(cfqd, cfqq, "idle=%d", enable_idle);
3910 if (enable_idle)
3911 cfq_mark_cfqq_idle_window(cfqq);
3912 else
3913 cfq_clear_cfqq_idle_window(cfqq);
3914 }
Jens Axboe22e2c502005-06-27 10:55:12 +02003915}
3916
Jens Axboe22e2c502005-06-27 10:55:12 +02003917/*
3918 * Check if new_cfqq should preempt the currently active queue. Return 0 for
3919 * no or if we aren't sure, a 1 will cause a preempt.
3920 */
Jens Axboea6151c32009-10-07 20:02:57 +02003921static bool
Jens Axboe22e2c502005-06-27 10:55:12 +02003922cfq_should_preempt(struct cfq_data *cfqd, struct cfq_queue *new_cfqq,
Jens Axboe5e705372006-07-13 12:39:25 +02003923 struct request *rq)
Jens Axboe22e2c502005-06-27 10:55:12 +02003924{
Jens Axboe6d048f52007-04-25 12:44:27 +02003925 struct cfq_queue *cfqq;
Jens Axboe22e2c502005-06-27 10:55:12 +02003926
Jens Axboe6d048f52007-04-25 12:44:27 +02003927 cfqq = cfqd->active_queue;
3928 if (!cfqq)
Jens Axboea6151c32009-10-07 20:02:57 +02003929 return false;
Jens Axboe22e2c502005-06-27 10:55:12 +02003930
Jens Axboe6d048f52007-04-25 12:44:27 +02003931 if (cfq_class_idle(new_cfqq))
Jens Axboea6151c32009-10-07 20:02:57 +02003932 return false;
Jens Axboe22e2c502005-06-27 10:55:12 +02003933
3934 if (cfq_class_idle(cfqq))
Jens Axboea6151c32009-10-07 20:02:57 +02003935 return true;
Jens Axboe1e3335d2007-02-14 19:59:49 +01003936
Jens Axboe22e2c502005-06-27 10:55:12 +02003937 /*
Divyesh Shah875feb62010-01-06 18:58:20 -08003938 * Don't allow a non-RT request to preempt an ongoing RT cfqq timeslice.
3939 */
3940 if (cfq_class_rt(cfqq) && !cfq_class_rt(new_cfqq))
3941 return false;
3942
3943 /*
Jens Axboe374f84a2006-07-23 01:42:19 +02003944 * if the new request is sync, but the currently running queue is
3945 * not, let the sync request have priority.
3946 */
Jens Axboe5e705372006-07-13 12:39:25 +02003947 if (rq_is_sync(rq) && !cfq_cfqq_sync(cfqq))
Jens Axboea6151c32009-10-07 20:02:57 +02003948 return true;
Jens Axboe1e3335d2007-02-14 19:59:49 +01003949
Vivek Goyal8682e1f2009-12-03 12:59:50 -05003950 if (new_cfqq->cfqg != cfqq->cfqg)
3951 return false;
3952
3953 if (cfq_slice_used(cfqq))
3954 return true;
3955
3956 /* Allow preemption only if we are idling on sync-noidle tree */
Vivek Goyal4d2ceea2012-10-03 16:56:57 -04003957 if (cfqd->serving_wl_type == SYNC_NOIDLE_WORKLOAD &&
Vivek Goyal8682e1f2009-12-03 12:59:50 -05003958 cfqq_type(new_cfqq) == SYNC_NOIDLE_WORKLOAD &&
3959 new_cfqq->service_tree->count == 2 &&
3960 RB_EMPTY_ROOT(&cfqq->sort_list))
3961 return true;
3962
Jens Axboe374f84a2006-07-23 01:42:19 +02003963 /*
Jens Axboeb53d1ed2011-08-19 08:34:48 +02003964 * So both queues are sync. Let the new request get disk time if
3965 * it's a metadata request and the current queue is doing regular IO.
3966 */
Christoph Hellwig65299a32011-08-23 14:50:29 +02003967 if ((rq->cmd_flags & REQ_PRIO) && !cfqq->prio_pending)
Jens Axboeb53d1ed2011-08-19 08:34:48 +02003968 return true;
3969
3970 /*
Divyesh Shah3a9a3f62009-01-30 12:46:41 +01003971 * Allow an RT request to pre-empt an ongoing non-RT cfqq timeslice.
3972 */
3973 if (cfq_class_rt(new_cfqq) && !cfq_class_rt(cfqq))
Jens Axboea6151c32009-10-07 20:02:57 +02003974 return true;
Divyesh Shah3a9a3f62009-01-30 12:46:41 +01003975
Shaohua Lid2d59e12010-11-08 15:01:03 +01003976 /* An idle queue should not be idle now for some reason */
3977 if (RB_EMPTY_ROOT(&cfqq->sort_list) && !cfq_should_idle(cfqd, cfqq))
3978 return true;
3979
Jens Axboe1e3335d2007-02-14 19:59:49 +01003980 if (!cfqd->active_cic || !cfq_cfqq_wait_request(cfqq))
Jens Axboea6151c32009-10-07 20:02:57 +02003981 return false;
Jens Axboe1e3335d2007-02-14 19:59:49 +01003982
3983 /*
3984 * if this request is as-good as one we would expect from the
3985 * current cfqq, let it preempt
3986 */
Shaohua Lie9ce3352010-03-19 08:03:04 +01003987 if (cfq_rq_close(cfqd, cfqq, rq))
Jens Axboea6151c32009-10-07 20:02:57 +02003988 return true;
Jens Axboe1e3335d2007-02-14 19:59:49 +01003989
Jens Axboea6151c32009-10-07 20:02:57 +02003990 return false;
Jens Axboe22e2c502005-06-27 10:55:12 +02003991}
3992
3993/*
3994 * cfqq preempts the active queue. if we allowed preempt with no slice left,
3995 * let it have half of its nominal slice.
3996 */
3997static void cfq_preempt_queue(struct cfq_data *cfqd, struct cfq_queue *cfqq)
3998{
Shaohua Lidf0793a2012-01-19 09:20:09 +01003999 enum wl_type_t old_type = cfqq_type(cfqd->active_queue);
4000
Jens Axboe7b679132008-05-30 12:23:07 +02004001 cfq_log_cfqq(cfqd, cfqq, "preempt");
Shaohua Lidf0793a2012-01-19 09:20:09 +01004002 cfq_slice_expired(cfqd, 1);
Jens Axboe22e2c502005-06-27 10:55:12 +02004003
Jens Axboebf572252006-07-19 20:29:12 +02004004 /*
Shaohua Lif8ae6e32011-01-14 08:41:02 +01004005 * workload type is changed, don't save slice, otherwise preempt
4006 * doesn't happen
4007 */
Shaohua Lidf0793a2012-01-19 09:20:09 +01004008 if (old_type != cfqq_type(cfqq))
Vivek Goyal4d2ceea2012-10-03 16:56:57 -04004009 cfqq->cfqg->saved_wl_slice = 0;
Shaohua Lif8ae6e32011-01-14 08:41:02 +01004010
4011 /*
Jens Axboebf572252006-07-19 20:29:12 +02004012 * Put the new queue at the front of the of the current list,
4013 * so we know that it will be selected next.
4014 */
4015 BUG_ON(!cfq_cfqq_on_rr(cfqq));
Jens Axboeedd75ff2007-04-19 12:03:34 +02004016
4017 cfq_service_tree_add(cfqd, cfqq, 1);
Justin TerAvesteda5e0c2011-03-22 21:26:49 +01004018
Justin TerAvest62a37f62011-03-23 08:25:44 +01004019 cfqq->slice_end = 0;
4020 cfq_mark_cfqq_slice_new(cfqq);
Jens Axboe22e2c502005-06-27 10:55:12 +02004021}
4022
4023/*
Jens Axboe5e705372006-07-13 12:39:25 +02004024 * Called when a new fs request (rq) is added (to cfqq). Check if there's
Jens Axboe22e2c502005-06-27 10:55:12 +02004025 * something we should do about it
4026 */
4027static void
Jens Axboe5e705372006-07-13 12:39:25 +02004028cfq_rq_enqueued(struct cfq_data *cfqd, struct cfq_queue *cfqq,
4029 struct request *rq)
Jens Axboe22e2c502005-06-27 10:55:12 +02004030{
Tejun Heoc5869802011-12-14 00:33:41 +01004031 struct cfq_io_cq *cic = RQ_CIC(rq);
Jens Axboe12e9fdd2006-06-01 10:09:56 +02004032
Aaron Carroll45333d52008-08-26 15:52:36 +02004033 cfqd->rq_queued++;
Christoph Hellwig65299a32011-08-23 14:50:29 +02004034 if (rq->cmd_flags & REQ_PRIO)
4035 cfqq->prio_pending++;
Jens Axboe374f84a2006-07-23 01:42:19 +02004036
Shaohua Li383cd722011-07-12 14:24:35 +02004037 cfq_update_io_thinktime(cfqd, cfqq, cic);
Jeff Moyerb2c18e12009-10-23 17:14:49 -04004038 cfq_update_io_seektime(cfqd, cfqq, rq);
Jens Axboe9c2c38a2005-08-24 14:57:54 +02004039 cfq_update_idle_window(cfqd, cfqq, cic);
4040
Jeff Moyerb2c18e12009-10-23 17:14:49 -04004041 cfqq->last_request_pos = blk_rq_pos(rq) + blk_rq_sectors(rq);
Jens Axboe22e2c502005-06-27 10:55:12 +02004042
4043 if (cfqq == cfqd->active_queue) {
4044 /*
Jens Axboeb0291952009-04-07 11:38:31 +02004045 * Remember that we saw a request from this process, but
4046 * don't start queuing just yet. Otherwise we risk seeing lots
4047 * of tiny requests, because we disrupt the normal plugging
Jens Axboed6ceb252009-04-14 14:18:16 +02004048 * and merging. If the request is already larger than a single
4049 * page, let it rip immediately. For that case we assume that
Jens Axboe2d870722009-04-15 12:12:46 +02004050 * merging is already done. Ditto for a busy system that
4051 * has other work pending, don't risk delaying until the
4052 * idle timer unplug to continue working.
Jens Axboe22e2c502005-06-27 10:55:12 +02004053 */
Jens Axboed6ceb252009-04-14 14:18:16 +02004054 if (cfq_cfqq_wait_request(cfqq)) {
Jens Axboe2d870722009-04-15 12:12:46 +02004055 if (blk_rq_bytes(rq) > PAGE_CACHE_SIZE ||
4056 cfqd->busy_queues > 1) {
Divyesh Shah812df482010-04-08 21:15:35 -07004057 cfq_del_timer(cfqd, cfqq);
Gui Jianfeng554554f2009-12-10 09:38:39 +01004058 cfq_clear_cfqq_wait_request(cfqq);
Christoph Hellwig24ecfbe2011-04-18 11:41:33 +02004059 __blk_run_queue(cfqd->queue);
Divyesh Shaha11cdaa2010-04-13 19:59:17 +02004060 } else {
Tejun Heo155fead2012-04-01 14:38:44 -07004061 cfqg_stats_update_idle_time(cfqq->cfqg);
Vivek Goyalbf7919372009-12-03 12:59:37 -05004062 cfq_mark_cfqq_must_dispatch(cfqq);
Divyesh Shaha11cdaa2010-04-13 19:59:17 +02004063 }
Jens Axboed6ceb252009-04-14 14:18:16 +02004064 }
Jens Axboe5e705372006-07-13 12:39:25 +02004065 } else if (cfq_should_preempt(cfqd, cfqq, rq)) {
Jens Axboe22e2c502005-06-27 10:55:12 +02004066 /*
4067 * not the active queue - expire current slice if it is
4068 * idle and has expired it's mean thinktime or this new queue
Divyesh Shah3a9a3f62009-01-30 12:46:41 +01004069 * has some old slice time left and is of higher priority or
4070 * this new queue is RT and the current one is BE
Jens Axboe22e2c502005-06-27 10:55:12 +02004071 */
4072 cfq_preempt_queue(cfqd, cfqq);
Christoph Hellwig24ecfbe2011-04-18 11:41:33 +02004073 __blk_run_queue(cfqd->queue);
Jens Axboe22e2c502005-06-27 10:55:12 +02004074 }
4075}
4076
Jens Axboe165125e2007-07-24 09:28:11 +02004077static void cfq_insert_request(struct request_queue *q, struct request *rq)
Jens Axboe22e2c502005-06-27 10:55:12 +02004078{
Jens Axboeb4878f22005-10-20 16:42:29 +02004079 struct cfq_data *cfqd = q->elevator->elevator_data;
Jens Axboe5e705372006-07-13 12:39:25 +02004080 struct cfq_queue *cfqq = RQ_CFQQ(rq);
Jens Axboe22e2c502005-06-27 10:55:12 +02004081
Jens Axboe7b679132008-05-30 12:23:07 +02004082 cfq_log_cfqq(cfqd, cfqq, "insert_request");
Tejun Heoabede6d2012-03-19 15:10:57 -07004083 cfq_init_prio_data(cfqq, RQ_CIC(rq));
Linus Torvalds1da177e2005-04-16 15:20:36 -07004084
Jan Kara8b4922d2014-02-24 16:39:52 +01004085 rq->fifo_time = jiffies + cfqd->cfq_fifo_expire[rq_is_sync(rq)];
Jens Axboe22e2c502005-06-27 10:55:12 +02004086 list_add_tail(&rq->queuelist, &cfqq->fifo);
Corrado Zoccoloaa6f6a32009-10-26 22:44:33 +01004087 cfq_add_rq_rb(rq);
Tejun Heo155fead2012-04-01 14:38:44 -07004088 cfqg_stats_update_io_add(RQ_CFQG(rq), cfqd->serving_group,
4089 rq->cmd_flags);
Jens Axboe5e705372006-07-13 12:39:25 +02004090 cfq_rq_enqueued(cfqd, cfqq, rq);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004091}
4092
Aaron Carroll45333d52008-08-26 15:52:36 +02004093/*
4094 * Update hw_tag based on peak queue depth over 50 samples under
4095 * sufficient load.
4096 */
4097static void cfq_update_hw_tag(struct cfq_data *cfqd)
4098{
Shaohua Li1a1238a2009-10-27 08:46:23 +01004099 struct cfq_queue *cfqq = cfqd->active_queue;
4100
Corrado Zoccolo53c583d2010-02-28 19:45:05 +01004101 if (cfqd->rq_in_driver > cfqd->hw_tag_est_depth)
4102 cfqd->hw_tag_est_depth = cfqd->rq_in_driver;
Corrado Zoccoloe459dd02009-11-26 10:02:57 +01004103
4104 if (cfqd->hw_tag == 1)
4105 return;
Aaron Carroll45333d52008-08-26 15:52:36 +02004106
4107 if (cfqd->rq_queued <= CFQ_HW_QUEUE_MIN &&
Corrado Zoccolo53c583d2010-02-28 19:45:05 +01004108 cfqd->rq_in_driver <= CFQ_HW_QUEUE_MIN)
Aaron Carroll45333d52008-08-26 15:52:36 +02004109 return;
4110
Shaohua Li1a1238a2009-10-27 08:46:23 +01004111 /*
4112 * If active queue hasn't enough requests and can idle, cfq might not
4113 * dispatch sufficient requests to hardware. Don't zero hw_tag in this
4114 * case
4115 */
4116 if (cfqq && cfq_cfqq_idle_window(cfqq) &&
4117 cfqq->dispatched + cfqq->queued[0] + cfqq->queued[1] <
Corrado Zoccolo53c583d2010-02-28 19:45:05 +01004118 CFQ_HW_QUEUE_MIN && cfqd->rq_in_driver < CFQ_HW_QUEUE_MIN)
Shaohua Li1a1238a2009-10-27 08:46:23 +01004119 return;
4120
Aaron Carroll45333d52008-08-26 15:52:36 +02004121 if (cfqd->hw_tag_samples++ < 50)
4122 return;
4123
Corrado Zoccoloe459dd02009-11-26 10:02:57 +01004124 if (cfqd->hw_tag_est_depth >= CFQ_HW_QUEUE_MIN)
Aaron Carroll45333d52008-08-26 15:52:36 +02004125 cfqd->hw_tag = 1;
4126 else
4127 cfqd->hw_tag = 0;
Aaron Carroll45333d52008-08-26 15:52:36 +02004128}
4129
Vivek Goyal7667aa02009-12-08 17:52:58 -05004130static bool cfq_should_wait_busy(struct cfq_data *cfqd, struct cfq_queue *cfqq)
4131{
Tejun Heoc5869802011-12-14 00:33:41 +01004132 struct cfq_io_cq *cic = cfqd->active_cic;
Vivek Goyal7667aa02009-12-08 17:52:58 -05004133
Justin TerAvest02a8f012011-02-09 14:20:03 +01004134 /* If the queue already has requests, don't wait */
4135 if (!RB_EMPTY_ROOT(&cfqq->sort_list))
4136 return false;
4137
Vivek Goyal7667aa02009-12-08 17:52:58 -05004138 /* If there are other queues in the group, don't wait */
4139 if (cfqq->cfqg->nr_cfqq > 1)
4140 return false;
4141
Shaohua Li7700fc42011-07-12 14:24:56 +02004142 /* the only queue in the group, but think time is big */
4143 if (cfq_io_thinktime_big(cfqd, &cfqq->cfqg->ttime, true))
4144 return false;
4145
Vivek Goyal7667aa02009-12-08 17:52:58 -05004146 if (cfq_slice_used(cfqq))
4147 return true;
4148
4149 /* if slice left is less than think time, wait busy */
Shaohua Li383cd722011-07-12 14:24:35 +02004150 if (cic && sample_valid(cic->ttime.ttime_samples)
4151 && (cfqq->slice_end - jiffies < cic->ttime.ttime_mean))
Vivek Goyal7667aa02009-12-08 17:52:58 -05004152 return true;
4153
4154 /*
4155 * If think times is less than a jiffy than ttime_mean=0 and above
4156 * will not be true. It might happen that slice has not expired yet
4157 * but will expire soon (4-5 ns) during select_queue(). To cover the
4158 * case where think time is less than a jiffy, mark the queue wait
4159 * busy if only 1 jiffy is left in the slice.
4160 */
4161 if (cfqq->slice_end - jiffies == 1)
4162 return true;
4163
4164 return false;
4165}
4166
Jens Axboe165125e2007-07-24 09:28:11 +02004167static void cfq_completed_request(struct request_queue *q, struct request *rq)
Linus Torvalds1da177e2005-04-16 15:20:36 -07004168{
Jens Axboe5e705372006-07-13 12:39:25 +02004169 struct cfq_queue *cfqq = RQ_CFQQ(rq);
Jens Axboeb4878f22005-10-20 16:42:29 +02004170 struct cfq_data *cfqd = cfqq->cfqd;
Jens Axboe5380a102006-07-13 12:37:56 +02004171 const int sync = rq_is_sync(rq);
Jens Axboeb4878f22005-10-20 16:42:29 +02004172 unsigned long now;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004173
Jens Axboeb4878f22005-10-20 16:42:29 +02004174 now = jiffies;
Christoph Hellwig33659eb2010-08-07 18:17:56 +02004175 cfq_log_cfqq(cfqd, cfqq, "complete rqnoidle %d",
4176 !!(rq->cmd_flags & REQ_NOIDLE));
Linus Torvalds1da177e2005-04-16 15:20:36 -07004177
Aaron Carroll45333d52008-08-26 15:52:36 +02004178 cfq_update_hw_tag(cfqd);
4179
Corrado Zoccolo53c583d2010-02-28 19:45:05 +01004180 WARN_ON(!cfqd->rq_in_driver);
Jens Axboe6d048f52007-04-25 12:44:27 +02004181 WARN_ON(!cfqq->dispatched);
Corrado Zoccolo53c583d2010-02-28 19:45:05 +01004182 cfqd->rq_in_driver--;
Jens Axboe6d048f52007-04-25 12:44:27 +02004183 cfqq->dispatched--;
Vivek Goyal80bdf0c2010-08-23 12:24:26 +02004184 (RQ_CFQG(rq))->dispatched--;
Tejun Heo155fead2012-04-01 14:38:44 -07004185 cfqg_stats_update_completion(cfqq->cfqg, rq_start_time_ns(rq),
4186 rq_io_start_time_ns(rq), rq->cmd_flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004187
Corrado Zoccolo53c583d2010-02-28 19:45:05 +01004188 cfqd->rq_in_flight[cfq_cfqq_sync(cfqq)]--;
Jens Axboe3ed9a292007-04-23 08:33:33 +02004189
Vivek Goyal365722b2009-10-03 15:21:27 +02004190 if (sync) {
Vivek Goyal34b98d02012-10-03 16:56:58 -04004191 struct cfq_rb_root *st;
Shaohua Lif5f2b6c2011-07-12 14:24:55 +02004192
Shaohua Li383cd722011-07-12 14:24:35 +02004193 RQ_CIC(rq)->ttime.last_end_request = now;
Shaohua Lif5f2b6c2011-07-12 14:24:55 +02004194
4195 if (cfq_cfqq_on_rr(cfqq))
Vivek Goyal34b98d02012-10-03 16:56:58 -04004196 st = cfqq->service_tree;
Shaohua Lif5f2b6c2011-07-12 14:24:55 +02004197 else
Vivek Goyal34b98d02012-10-03 16:56:58 -04004198 st = st_for(cfqq->cfqg, cfqq_class(cfqq),
4199 cfqq_type(cfqq));
4200
4201 st->ttime.last_end_request = now;
Corrado Zoccolo573412b2009-12-06 11:48:52 +01004202 if (!time_after(rq->start_time + cfqd->cfq_fifo_expire[1], now))
4203 cfqd->last_delayed_sync = now;
Vivek Goyal365722b2009-10-03 15:21:27 +02004204 }
Jens Axboecaaa5f92006-06-16 11:23:00 +02004205
Shaohua Li7700fc42011-07-12 14:24:56 +02004206#ifdef CONFIG_CFQ_GROUP_IOSCHED
4207 cfqq->cfqg->ttime.last_end_request = now;
4208#endif
4209
Jens Axboecaaa5f92006-06-16 11:23:00 +02004210 /*
4211 * If this is the active queue, check if it needs to be expired,
4212 * or if we want to idle in case it has no pending requests.
4213 */
4214 if (cfqd->active_queue == cfqq) {
Jens Axboea36e71f2009-04-15 12:15:11 +02004215 const bool cfqq_empty = RB_EMPTY_ROOT(&cfqq->sort_list);
4216
Jens Axboe44f7c162007-01-19 11:51:58 +11004217 if (cfq_cfqq_slice_new(cfqq)) {
4218 cfq_set_prio_slice(cfqd, cfqq);
4219 cfq_clear_cfqq_slice_new(cfqq);
4220 }
Vivek Goyalf75edf22009-12-03 12:59:53 -05004221
4222 /*
Vivek Goyal7667aa02009-12-08 17:52:58 -05004223 * Should we wait for next request to come in before we expire
4224 * the queue.
Vivek Goyalf75edf22009-12-03 12:59:53 -05004225 */
Vivek Goyal7667aa02009-12-08 17:52:58 -05004226 if (cfq_should_wait_busy(cfqd, cfqq)) {
Vivek Goyal80bdf0c2010-08-23 12:24:26 +02004227 unsigned long extend_sl = cfqd->cfq_slice_idle;
4228 if (!cfqd->cfq_slice_idle)
4229 extend_sl = cfqd->cfq_group_idle;
4230 cfqq->slice_end = jiffies + extend_sl;
Vivek Goyalf75edf22009-12-03 12:59:53 -05004231 cfq_mark_cfqq_wait_busy(cfqq);
Divyesh Shahb1ffe732010-03-25 15:45:03 +01004232 cfq_log_cfqq(cfqd, cfqq, "will busy wait");
Vivek Goyalf75edf22009-12-03 12:59:53 -05004233 }
4234
Jens Axboea36e71f2009-04-15 12:15:11 +02004235 /*
Corrado Zoccolo8e550632009-11-26 10:02:58 +01004236 * Idling is not enabled on:
4237 * - expired queues
4238 * - idle-priority queues
4239 * - async queues
4240 * - queues with still some requests queued
4241 * - when there is a close cooperator
Jens Axboea36e71f2009-04-15 12:15:11 +02004242 */
Jens Axboe08717142008-01-28 11:38:15 +01004243 if (cfq_slice_used(cfqq) || cfq_class_idle(cfqq))
Vivek Goyale5ff0822010-04-26 19:25:11 +02004244 cfq_slice_expired(cfqd, 1);
Corrado Zoccolo8e550632009-11-26 10:02:58 +01004245 else if (sync && cfqq_empty &&
4246 !cfq_close_cooperator(cfqd, cfqq)) {
Corrado Zoccolo749ef9f2010-09-20 15:24:50 +02004247 cfq_arm_slice_timer(cfqd);
Corrado Zoccolo8e550632009-11-26 10:02:58 +01004248 }
Jens Axboecaaa5f92006-06-16 11:23:00 +02004249 }
Jens Axboe6d048f52007-04-25 12:44:27 +02004250
Corrado Zoccolo53c583d2010-02-28 19:45:05 +01004251 if (!cfqd->rq_in_driver)
Jens Axboe23e018a2009-10-05 08:52:35 +02004252 cfq_schedule_dispatch(cfqd);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004253}
4254
Jens Axboe89850f72006-07-22 16:48:31 +02004255static inline int __cfq_may_queue(struct cfq_queue *cfqq)
Jens Axboe22e2c502005-06-27 10:55:12 +02004256{
Jens Axboe1b379d82009-08-11 08:26:11 +02004257 if (cfq_cfqq_wait_request(cfqq) && !cfq_cfqq_must_alloc_slice(cfqq)) {
Jens Axboe3b181522005-06-27 10:56:24 +02004258 cfq_mark_cfqq_must_alloc_slice(cfqq);
Jens Axboe22e2c502005-06-27 10:55:12 +02004259 return ELV_MQUEUE_MUST;
Jens Axboe3b181522005-06-27 10:56:24 +02004260 }
Jens Axboe22e2c502005-06-27 10:55:12 +02004261
4262 return ELV_MQUEUE_MAY;
Jens Axboe22e2c502005-06-27 10:55:12 +02004263}
4264
Jens Axboe165125e2007-07-24 09:28:11 +02004265static int cfq_may_queue(struct request_queue *q, int rw)
Jens Axboe22e2c502005-06-27 10:55:12 +02004266{
4267 struct cfq_data *cfqd = q->elevator->elevator_data;
4268 struct task_struct *tsk = current;
Tejun Heoc5869802011-12-14 00:33:41 +01004269 struct cfq_io_cq *cic;
Jens Axboe22e2c502005-06-27 10:55:12 +02004270 struct cfq_queue *cfqq;
4271
4272 /*
4273 * don't force setup of a queue from here, as a call to may_queue
4274 * does not necessarily imply that a request actually will be queued.
4275 * so just lookup a possibly existing queue, or return 'may queue'
4276 * if that fails
4277 */
Jens Axboe4ac845a2008-01-24 08:44:49 +01004278 cic = cfq_cic_lookup(cfqd, tsk->io_context);
Vasily Tarasov91fac312007-04-25 12:29:51 +02004279 if (!cic)
4280 return ELV_MQUEUE_MAY;
4281
Jens Axboeb0b78f82009-04-08 10:56:08 +02004282 cfqq = cic_to_cfqq(cic, rw_is_sync(rw));
Jens Axboe22e2c502005-06-27 10:55:12 +02004283 if (cfqq) {
Tejun Heoabede6d2012-03-19 15:10:57 -07004284 cfq_init_prio_data(cfqq, cic);
Jens Axboe22e2c502005-06-27 10:55:12 +02004285
Jens Axboe89850f72006-07-22 16:48:31 +02004286 return __cfq_may_queue(cfqq);
Jens Axboe22e2c502005-06-27 10:55:12 +02004287 }
4288
4289 return ELV_MQUEUE_MAY;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004290}
4291
Linus Torvalds1da177e2005-04-16 15:20:36 -07004292/*
4293 * queue lock held here
4294 */
Jens Axboebb37b942006-12-01 10:42:33 +01004295static void cfq_put_request(struct request *rq)
Linus Torvalds1da177e2005-04-16 15:20:36 -07004296{
Jens Axboe5e705372006-07-13 12:39:25 +02004297 struct cfq_queue *cfqq = RQ_CFQQ(rq);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004298
Jens Axboe5e705372006-07-13 12:39:25 +02004299 if (cfqq) {
Jens Axboe22e2c502005-06-27 10:55:12 +02004300 const int rw = rq_data_dir(rq);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004301
Jens Axboe22e2c502005-06-27 10:55:12 +02004302 BUG_ON(!cfqq->allocated[rw]);
4303 cfqq->allocated[rw]--;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004304
Vivek Goyal7f1dc8a2010-04-21 17:44:16 +02004305 /* Put down rq reference on cfqg */
Tejun Heoeb7d8c072012-03-23 14:02:53 +01004306 cfqg_put(RQ_CFQG(rq));
Tejun Heoa612fdd2011-12-14 00:33:41 +01004307 rq->elv.priv[0] = NULL;
4308 rq->elv.priv[1] = NULL;
Vivek Goyal7f1dc8a2010-04-21 17:44:16 +02004309
Linus Torvalds1da177e2005-04-16 15:20:36 -07004310 cfq_put_queue(cfqq);
4311 }
4312}
4313
Jeff Moyerdf5fe3e2009-10-23 17:14:50 -04004314static struct cfq_queue *
Tejun Heoc5869802011-12-14 00:33:41 +01004315cfq_merge_cfqqs(struct cfq_data *cfqd, struct cfq_io_cq *cic,
Jeff Moyerdf5fe3e2009-10-23 17:14:50 -04004316 struct cfq_queue *cfqq)
4317{
4318 cfq_log_cfqq(cfqd, cfqq, "merging with queue %p", cfqq->new_cfqq);
4319 cic_set_cfqq(cic, cfqq->new_cfqq, 1);
Jeff Moyerb3b6d042009-10-23 17:14:51 -04004320 cfq_mark_cfqq_coop(cfqq->new_cfqq);
Jeff Moyerdf5fe3e2009-10-23 17:14:50 -04004321 cfq_put_queue(cfqq);
4322 return cic_to_cfqq(cic, 1);
4323}
4324
Jeff Moyere6c5bc72009-10-23 17:14:52 -04004325/*
4326 * Returns NULL if a new cfqq should be allocated, or the old cfqq if this
4327 * was the last process referring to said cfqq.
4328 */
4329static struct cfq_queue *
Tejun Heoc5869802011-12-14 00:33:41 +01004330split_cfqq(struct cfq_io_cq *cic, struct cfq_queue *cfqq)
Jeff Moyere6c5bc72009-10-23 17:14:52 -04004331{
4332 if (cfqq_process_refs(cfqq) == 1) {
Jeff Moyere6c5bc72009-10-23 17:14:52 -04004333 cfqq->pid = current->pid;
4334 cfq_clear_cfqq_coop(cfqq);
Shaohua Liae54abe2010-02-05 13:11:45 +01004335 cfq_clear_cfqq_split_coop(cfqq);
Jeff Moyere6c5bc72009-10-23 17:14:52 -04004336 return cfqq;
4337 }
4338
4339 cic_set_cfqq(cic, NULL, 1);
Shaohua Lid02a2c02010-05-25 10:16:53 +02004340
4341 cfq_put_cooperator(cfqq);
4342
Jeff Moyere6c5bc72009-10-23 17:14:52 -04004343 cfq_put_queue(cfqq);
4344 return NULL;
4345}
Linus Torvalds1da177e2005-04-16 15:20:36 -07004346/*
Jens Axboe22e2c502005-06-27 10:55:12 +02004347 * Allocate cfq data structures associated with this request.
Linus Torvalds1da177e2005-04-16 15:20:36 -07004348 */
Jens Axboe22e2c502005-06-27 10:55:12 +02004349static int
Tejun Heo852c7882012-03-05 13:15:27 -08004350cfq_set_request(struct request_queue *q, struct request *rq, struct bio *bio,
4351 gfp_t gfp_mask)
Linus Torvalds1da177e2005-04-16 15:20:36 -07004352{
4353 struct cfq_data *cfqd = q->elevator->elevator_data;
Tejun Heof1f8cc92011-12-14 00:33:42 +01004354 struct cfq_io_cq *cic = icq_to_cic(rq->elv.icq);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004355 const int rw = rq_data_dir(rq);
Jens Axboea6151c32009-10-07 20:02:57 +02004356 const bool is_sync = rq_is_sync(rq);
Jens Axboe22e2c502005-06-27 10:55:12 +02004357 struct cfq_queue *cfqq;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004358
Tejun Heo216284c32011-12-14 00:33:38 +01004359 spin_lock_irq(q->queue_lock);
Tejun Heof1f8cc92011-12-14 00:33:42 +01004360
Tejun Heo598971b2012-03-19 15:10:58 -07004361 check_ioprio_changed(cic, bio);
4362 check_blkcg_changed(cic, bio);
Jeff Moyere6c5bc72009-10-23 17:14:52 -04004363new_queue:
Vasily Tarasov91fac312007-04-25 12:29:51 +02004364 cfqq = cic_to_cfqq(cic, is_sync);
Vivek Goyal32f2e802009-07-09 22:13:16 +02004365 if (!cfqq || cfqq == &cfqd->oom_cfqq) {
Tejun Heobce61332015-08-18 14:54:59 -07004366 if (cfqq)
4367 cfq_put_queue(cfqq);
Tejun Heo2da8de02015-08-18 14:55:02 -07004368 cfqq = cfq_get_queue(cfqd, is_sync, cic, bio);
Vasily Tarasov91fac312007-04-25 12:29:51 +02004369 cic_set_cfqq(cic, cfqq, is_sync);
Jeff Moyerdf5fe3e2009-10-23 17:14:50 -04004370 } else {
4371 /*
Jeff Moyere6c5bc72009-10-23 17:14:52 -04004372 * If the queue was seeky for too long, break it apart.
4373 */
Shaohua Liae54abe2010-02-05 13:11:45 +01004374 if (cfq_cfqq_coop(cfqq) && cfq_cfqq_split_coop(cfqq)) {
Jeff Moyere6c5bc72009-10-23 17:14:52 -04004375 cfq_log_cfqq(cfqd, cfqq, "breaking apart cfqq");
4376 cfqq = split_cfqq(cic, cfqq);
4377 if (!cfqq)
4378 goto new_queue;
4379 }
4380
4381 /*
Jeff Moyerdf5fe3e2009-10-23 17:14:50 -04004382 * Check to see if this queue is scheduled to merge with
4383 * another, closely cooperating queue. The merging of
4384 * queues happens here as it must be done in process context.
4385 * The reference on new_cfqq was taken in merge_cfqqs.
4386 */
4387 if (cfqq->new_cfqq)
4388 cfqq = cfq_merge_cfqqs(cfqd, cic, cfqq);
Vasily Tarasov91fac312007-04-25 12:29:51 +02004389 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07004390
4391 cfqq->allocated[rw]++;
Jens Axboe5e705372006-07-13 12:39:25 +02004392
Jens Axboe6fae9c22011-03-01 15:04:39 -05004393 cfqq->ref++;
Tejun Heoeb7d8c072012-03-23 14:02:53 +01004394 cfqg_get(cfqq->cfqg);
Tejun Heoa612fdd2011-12-14 00:33:41 +01004395 rq->elv.priv[0] = cfqq;
Tejun Heo1adaf3d2012-03-05 13:15:15 -08004396 rq->elv.priv[1] = cfqq->cfqg;
Tejun Heo216284c32011-12-14 00:33:38 +01004397 spin_unlock_irq(q->queue_lock);
Jens Axboe5e705372006-07-13 12:39:25 +02004398 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004399}
4400
David Howells65f27f32006-11-22 14:55:48 +00004401static void cfq_kick_queue(struct work_struct *work)
Jens Axboe22e2c502005-06-27 10:55:12 +02004402{
David Howells65f27f32006-11-22 14:55:48 +00004403 struct cfq_data *cfqd =
Jens Axboe23e018a2009-10-05 08:52:35 +02004404 container_of(work, struct cfq_data, unplug_work);
Jens Axboe165125e2007-07-24 09:28:11 +02004405 struct request_queue *q = cfqd->queue;
Jens Axboe22e2c502005-06-27 10:55:12 +02004406
Jens Axboe40bb54d2009-04-15 12:11:10 +02004407 spin_lock_irq(q->queue_lock);
Christoph Hellwig24ecfbe2011-04-18 11:41:33 +02004408 __blk_run_queue(cfqd->queue);
Jens Axboe40bb54d2009-04-15 12:11:10 +02004409 spin_unlock_irq(q->queue_lock);
Jens Axboe22e2c502005-06-27 10:55:12 +02004410}
4411
4412/*
4413 * Timer running if the active_queue is currently idling inside its time slice
4414 */
4415static void cfq_idle_slice_timer(unsigned long data)
4416{
4417 struct cfq_data *cfqd = (struct cfq_data *) data;
4418 struct cfq_queue *cfqq;
4419 unsigned long flags;
Jens Axboe3c6bd2f2007-01-19 12:06:33 +11004420 int timed_out = 1;
Jens Axboe22e2c502005-06-27 10:55:12 +02004421
Jens Axboe7b679132008-05-30 12:23:07 +02004422 cfq_log(cfqd, "idle timer fired");
4423
Jens Axboe22e2c502005-06-27 10:55:12 +02004424 spin_lock_irqsave(cfqd->queue->queue_lock, flags);
4425
Jens Axboefe094d92008-01-31 13:08:54 +01004426 cfqq = cfqd->active_queue;
4427 if (cfqq) {
Jens Axboe3c6bd2f2007-01-19 12:06:33 +11004428 timed_out = 0;
4429
Jens Axboe22e2c502005-06-27 10:55:12 +02004430 /*
Jens Axboeb0291952009-04-07 11:38:31 +02004431 * We saw a request before the queue expired, let it through
4432 */
4433 if (cfq_cfqq_must_dispatch(cfqq))
4434 goto out_kick;
4435
4436 /*
Jens Axboe22e2c502005-06-27 10:55:12 +02004437 * expired
4438 */
Jens Axboe44f7c162007-01-19 11:51:58 +11004439 if (cfq_slice_used(cfqq))
Jens Axboe22e2c502005-06-27 10:55:12 +02004440 goto expire;
4441
4442 /*
4443 * only expire and reinvoke request handler, if there are
4444 * other queues with pending requests
4445 */
Jens Axboecaaa5f92006-06-16 11:23:00 +02004446 if (!cfqd->busy_queues)
Jens Axboe22e2c502005-06-27 10:55:12 +02004447 goto out_cont;
Jens Axboe22e2c502005-06-27 10:55:12 +02004448
4449 /*
4450 * not expired and it has a request pending, let it dispatch
4451 */
Jens Axboe75e50982009-04-07 08:56:14 +02004452 if (!RB_EMPTY_ROOT(&cfqq->sort_list))
Jens Axboe22e2c502005-06-27 10:55:12 +02004453 goto out_kick;
Corrado Zoccolo76280af2009-11-26 10:02:58 +01004454
4455 /*
4456 * Queue depth flag is reset only when the idle didn't succeed
4457 */
4458 cfq_clear_cfqq_deep(cfqq);
Jens Axboe22e2c502005-06-27 10:55:12 +02004459 }
4460expire:
Vivek Goyale5ff0822010-04-26 19:25:11 +02004461 cfq_slice_expired(cfqd, timed_out);
Jens Axboe22e2c502005-06-27 10:55:12 +02004462out_kick:
Jens Axboe23e018a2009-10-05 08:52:35 +02004463 cfq_schedule_dispatch(cfqd);
Jens Axboe22e2c502005-06-27 10:55:12 +02004464out_cont:
4465 spin_unlock_irqrestore(cfqd->queue->queue_lock, flags);
4466}
4467
Jens Axboe3b181522005-06-27 10:56:24 +02004468static void cfq_shutdown_timer_wq(struct cfq_data *cfqd)
4469{
4470 del_timer_sync(&cfqd->idle_slice_timer);
Jens Axboe23e018a2009-10-05 08:52:35 +02004471 cancel_work_sync(&cfqd->unplug_work);
Jens Axboe3b181522005-06-27 10:56:24 +02004472}
Jens Axboe22e2c502005-06-27 10:55:12 +02004473
Jens Axboeb374d182008-10-31 10:05:07 +01004474static void cfq_exit_queue(struct elevator_queue *e)
Linus Torvalds1da177e2005-04-16 15:20:36 -07004475{
Jens Axboe22e2c502005-06-27 10:55:12 +02004476 struct cfq_data *cfqd = e->elevator_data;
Jens Axboe165125e2007-07-24 09:28:11 +02004477 struct request_queue *q = cfqd->queue;
Jens Axboe22e2c502005-06-27 10:55:12 +02004478
Jens Axboe3b181522005-06-27 10:56:24 +02004479 cfq_shutdown_timer_wq(cfqd);
Jens Axboee2d74ac2006-03-28 08:59:01 +02004480
Al Virod9ff4182006-03-18 13:51:22 -05004481 spin_lock_irq(q->queue_lock);
Jens Axboee2d74ac2006-03-28 08:59:01 +02004482
Al Virod9ff4182006-03-18 13:51:22 -05004483 if (cfqd->active_queue)
Vivek Goyale5ff0822010-04-26 19:25:11 +02004484 __cfq_slice_expired(cfqd, cfqd->active_queue, 0);
Jens Axboee2d74ac2006-03-28 08:59:01 +02004485
Tejun Heo03aa2642012-03-05 13:15:19 -08004486 spin_unlock_irq(q->queue_lock);
4487
Al Viroa90d7422006-03-18 12:05:37 -05004488 cfq_shutdown_timer_wq(cfqd);
4489
Tejun Heoffea73f2012-06-04 10:02:29 +02004490#ifdef CONFIG_CFQ_GROUP_IOSCHED
4491 blkcg_deactivate_policy(q, &blkcg_policy_cfq);
4492#else
Tejun Heof51b8022012-03-05 13:15:05 -08004493 kfree(cfqd->root_group);
Vivek Goyal2abae552011-05-23 10:02:19 +02004494#endif
Vivek Goyal56edf7d2011-05-19 15:38:22 -04004495 kfree(cfqd);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004496}
4497
Jianpeng Mad50235b2013-07-03 13:25:24 +02004498static int cfq_init_queue(struct request_queue *q, struct elevator_type *e)
Linus Torvalds1da177e2005-04-16 15:20:36 -07004499{
4500 struct cfq_data *cfqd;
Tejun Heo3c798392012-04-16 13:57:25 -07004501 struct blkcg_gq *blkg __maybe_unused;
Tejun Heoa2b16932012-04-13 13:11:33 -07004502 int i, ret;
Jianpeng Mad50235b2013-07-03 13:25:24 +02004503 struct elevator_queue *eq;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004504
Jianpeng Mad50235b2013-07-03 13:25:24 +02004505 eq = elevator_alloc(q, e);
4506 if (!eq)
Tejun Heob2fab5a2012-03-05 13:14:57 -08004507 return -ENOMEM;
Konstantin Khlebnikov80b15c72010-05-20 23:21:41 +04004508
Joe Perchesc1b511e2013-08-29 15:21:42 -07004509 cfqd = kzalloc_node(sizeof(*cfqd), GFP_KERNEL, q->node);
Jianpeng Mad50235b2013-07-03 13:25:24 +02004510 if (!cfqd) {
4511 kobject_put(&eq->kobj);
4512 return -ENOMEM;
4513 }
4514 eq->elevator_data = cfqd;
4515
Tejun Heof51b8022012-03-05 13:15:05 -08004516 cfqd->queue = q;
Jianpeng Mad50235b2013-07-03 13:25:24 +02004517 spin_lock_irq(q->queue_lock);
4518 q->elevator = eq;
4519 spin_unlock_irq(q->queue_lock);
Tejun Heof51b8022012-03-05 13:15:05 -08004520
Vivek Goyal1fa8f6d2009-12-03 12:59:41 -05004521 /* Init root service tree */
4522 cfqd->grp_service_tree = CFQ_RB_ROOT;
4523
Tejun Heof51b8022012-03-05 13:15:05 -08004524 /* Init root group and prefer root group over other groups by default */
Vivek Goyal25fb5162009-12-03 12:59:46 -05004525#ifdef CONFIG_CFQ_GROUP_IOSCHED
Tejun Heo3c798392012-04-16 13:57:25 -07004526 ret = blkcg_activate_policy(q, &blkcg_policy_cfq);
Tejun Heoa2b16932012-04-13 13:11:33 -07004527 if (ret)
4528 goto out_free;
Vivek Goyal5624a4e2011-05-19 15:38:28 -04004529
Tejun Heoa2b16932012-04-13 13:11:33 -07004530 cfqd->root_group = blkg_to_cfqg(q->root_blkg);
Tejun Heof51b8022012-03-05 13:15:05 -08004531#else
Tejun Heoa2b16932012-04-13 13:11:33 -07004532 ret = -ENOMEM;
Tejun Heof51b8022012-03-05 13:15:05 -08004533 cfqd->root_group = kzalloc_node(sizeof(*cfqd->root_group),
4534 GFP_KERNEL, cfqd->queue->node);
Tejun Heoa2b16932012-04-13 13:11:33 -07004535 if (!cfqd->root_group)
4536 goto out_free;
Vivek Goyal5624a4e2011-05-19 15:38:28 -04004537
Tejun Heoa2b16932012-04-13 13:11:33 -07004538 cfq_init_cfqg_base(cfqd->root_group);
Tejun Heo3ecca622015-08-18 14:55:35 -07004539 cfqd->root_group->weight = 2 * CFQ_WEIGHT_LEGACY_DFL;
4540 cfqd->root_group->leaf_weight = 2 * CFQ_WEIGHT_LEGACY_DFL;
Tejun Heo69d7fde2015-08-18 14:55:36 -07004541#endif
Vivek Goyal5624a4e2011-05-19 15:38:28 -04004542
Jens Axboe26a2ac02009-04-23 12:13:27 +02004543 /*
4544 * Not strictly needed (since RB_ROOT just clears the node and we
4545 * zeroed cfqd on alloc), but better be safe in case someone decides
4546 * to add magic to the rb code
4547 */
4548 for (i = 0; i < CFQ_PRIO_LISTS; i++)
4549 cfqd->prio_trees[i] = RB_ROOT;
4550
Jens Axboe6118b702009-06-30 09:34:12 +02004551 /*
Tejun Heod4aad7f2015-08-18 14:55:04 -07004552 * Our fallback cfqq if cfq_get_queue() runs into OOM issues.
Jens Axboe6118b702009-06-30 09:34:12 +02004553 * Grab a permanent reference to it, so that the normal code flow
Tejun Heof51b8022012-03-05 13:15:05 -08004554 * will not attempt to free it. oom_cfqq is linked to root_group
4555 * but shouldn't hold a reference as it'll never be unlinked. Lose
4556 * the reference from linking right away.
Jens Axboe6118b702009-06-30 09:34:12 +02004557 */
4558 cfq_init_cfqq(cfqd, &cfqd->oom_cfqq, 1, 0);
Shaohua Li30d7b942011-01-07 08:46:59 +01004559 cfqd->oom_cfqq.ref++;
Tejun Heo1adaf3d2012-03-05 13:15:15 -08004560
4561 spin_lock_irq(q->queue_lock);
Tejun Heof51b8022012-03-05 13:15:05 -08004562 cfq_link_cfqq_cfqg(&cfqd->oom_cfqq, cfqd->root_group);
Tejun Heoeb7d8c072012-03-23 14:02:53 +01004563 cfqg_put(cfqd->root_group);
Tejun Heo1adaf3d2012-03-05 13:15:15 -08004564 spin_unlock_irq(q->queue_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004565
Jens Axboe22e2c502005-06-27 10:55:12 +02004566 init_timer(&cfqd->idle_slice_timer);
4567 cfqd->idle_slice_timer.function = cfq_idle_slice_timer;
4568 cfqd->idle_slice_timer.data = (unsigned long) cfqd;
4569
Jens Axboe23e018a2009-10-05 08:52:35 +02004570 INIT_WORK(&cfqd->unplug_work, cfq_kick_queue);
Jens Axboe22e2c502005-06-27 10:55:12 +02004571
Linus Torvalds1da177e2005-04-16 15:20:36 -07004572 cfqd->cfq_quantum = cfq_quantum;
Jens Axboe22e2c502005-06-27 10:55:12 +02004573 cfqd->cfq_fifo_expire[0] = cfq_fifo_expire[0];
4574 cfqd->cfq_fifo_expire[1] = cfq_fifo_expire[1];
Linus Torvalds1da177e2005-04-16 15:20:36 -07004575 cfqd->cfq_back_max = cfq_back_max;
4576 cfqd->cfq_back_penalty = cfq_back_penalty;
Jens Axboe22e2c502005-06-27 10:55:12 +02004577 cfqd->cfq_slice[0] = cfq_slice_async;
4578 cfqd->cfq_slice[1] = cfq_slice_sync;
Tao Ma5bf14c02012-04-01 14:33:39 -07004579 cfqd->cfq_target_latency = cfq_target_latency;
Jens Axboe22e2c502005-06-27 10:55:12 +02004580 cfqd->cfq_slice_async_rq = cfq_slice_async_rq;
Jens Axboe0bb97942015-06-10 08:01:20 -06004581 cfqd->cfq_slice_idle = cfq_slice_idle;
Vivek Goyal80bdf0c2010-08-23 12:24:26 +02004582 cfqd->cfq_group_idle = cfq_group_idle;
Jens Axboe963b72f2009-10-03 19:42:18 +02004583 cfqd->cfq_latency = 1;
Corrado Zoccoloe459dd02009-11-26 10:02:57 +01004584 cfqd->hw_tag = -1;
Corrado Zoccoloedc71132009-12-09 20:56:04 +01004585 /*
4586 * we optimistically start assuming sync ops weren't delayed in last
4587 * second, in order to have larger depth for async operations.
4588 */
Corrado Zoccolo573412b2009-12-06 11:48:52 +01004589 cfqd->last_delayed_sync = jiffies - HZ;
Tejun Heob2fab5a2012-03-05 13:14:57 -08004590 return 0;
Tejun Heoa2b16932012-04-13 13:11:33 -07004591
4592out_free:
4593 kfree(cfqd);
Jianpeng Mad50235b2013-07-03 13:25:24 +02004594 kobject_put(&eq->kobj);
Tejun Heoa2b16932012-04-13 13:11:33 -07004595 return ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004596}
4597
Jens Axboe0bb97942015-06-10 08:01:20 -06004598static void cfq_registered_queue(struct request_queue *q)
4599{
4600 struct elevator_queue *e = q->elevator;
4601 struct cfq_data *cfqd = e->elevator_data;
4602
4603 /*
4604 * Default to IOPS mode with no idling for SSDs
4605 */
4606 if (blk_queue_nonrot(q))
4607 cfqd->cfq_slice_idle = 0;
4608}
4609
Linus Torvalds1da177e2005-04-16 15:20:36 -07004610/*
4611 * sysfs parts below -->
4612 */
Linus Torvalds1da177e2005-04-16 15:20:36 -07004613static ssize_t
4614cfq_var_show(unsigned int var, char *page)
4615{
Masanari Iida176167a2014-04-28 12:38:34 +09004616 return sprintf(page, "%u\n", var);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004617}
4618
4619static ssize_t
4620cfq_var_store(unsigned int *var, const char *page, size_t count)
4621{
4622 char *p = (char *) page;
4623
4624 *var = simple_strtoul(p, &p, 10);
4625 return count;
4626}
4627
Linus Torvalds1da177e2005-04-16 15:20:36 -07004628#define SHOW_FUNCTION(__FUNC, __VAR, __CONV) \
Jens Axboeb374d182008-10-31 10:05:07 +01004629static ssize_t __FUNC(struct elevator_queue *e, char *page) \
Linus Torvalds1da177e2005-04-16 15:20:36 -07004630{ \
Al Viro3d1ab402006-03-18 18:35:43 -05004631 struct cfq_data *cfqd = e->elevator_data; \
Linus Torvalds1da177e2005-04-16 15:20:36 -07004632 unsigned int __data = __VAR; \
4633 if (__CONV) \
4634 __data = jiffies_to_msecs(__data); \
4635 return cfq_var_show(__data, (page)); \
4636}
4637SHOW_FUNCTION(cfq_quantum_show, cfqd->cfq_quantum, 0);
Jens Axboe22e2c502005-06-27 10:55:12 +02004638SHOW_FUNCTION(cfq_fifo_expire_sync_show, cfqd->cfq_fifo_expire[1], 1);
4639SHOW_FUNCTION(cfq_fifo_expire_async_show, cfqd->cfq_fifo_expire[0], 1);
Al Viroe572ec72006-03-18 22:27:18 -05004640SHOW_FUNCTION(cfq_back_seek_max_show, cfqd->cfq_back_max, 0);
4641SHOW_FUNCTION(cfq_back_seek_penalty_show, cfqd->cfq_back_penalty, 0);
Jens Axboe22e2c502005-06-27 10:55:12 +02004642SHOW_FUNCTION(cfq_slice_idle_show, cfqd->cfq_slice_idle, 1);
Vivek Goyal80bdf0c2010-08-23 12:24:26 +02004643SHOW_FUNCTION(cfq_group_idle_show, cfqd->cfq_group_idle, 1);
Jens Axboe22e2c502005-06-27 10:55:12 +02004644SHOW_FUNCTION(cfq_slice_sync_show, cfqd->cfq_slice[1], 1);
4645SHOW_FUNCTION(cfq_slice_async_show, cfqd->cfq_slice[0], 1);
4646SHOW_FUNCTION(cfq_slice_async_rq_show, cfqd->cfq_slice_async_rq, 0);
Jens Axboe963b72f2009-10-03 19:42:18 +02004647SHOW_FUNCTION(cfq_low_latency_show, cfqd->cfq_latency, 0);
Tao Ma5bf14c02012-04-01 14:33:39 -07004648SHOW_FUNCTION(cfq_target_latency_show, cfqd->cfq_target_latency, 1);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004649#undef SHOW_FUNCTION
4650
4651#define STORE_FUNCTION(__FUNC, __PTR, MIN, MAX, __CONV) \
Jens Axboeb374d182008-10-31 10:05:07 +01004652static ssize_t __FUNC(struct elevator_queue *e, const char *page, size_t count) \
Linus Torvalds1da177e2005-04-16 15:20:36 -07004653{ \
Al Viro3d1ab402006-03-18 18:35:43 -05004654 struct cfq_data *cfqd = e->elevator_data; \
Linus Torvalds1da177e2005-04-16 15:20:36 -07004655 unsigned int __data; \
4656 int ret = cfq_var_store(&__data, (page), count); \
4657 if (__data < (MIN)) \
4658 __data = (MIN); \
4659 else if (__data > (MAX)) \
4660 __data = (MAX); \
4661 if (__CONV) \
4662 *(__PTR) = msecs_to_jiffies(__data); \
4663 else \
4664 *(__PTR) = __data; \
4665 return ret; \
4666}
4667STORE_FUNCTION(cfq_quantum_store, &cfqd->cfq_quantum, 1, UINT_MAX, 0);
Jens Axboefe094d92008-01-31 13:08:54 +01004668STORE_FUNCTION(cfq_fifo_expire_sync_store, &cfqd->cfq_fifo_expire[1], 1,
4669 UINT_MAX, 1);
4670STORE_FUNCTION(cfq_fifo_expire_async_store, &cfqd->cfq_fifo_expire[0], 1,
4671 UINT_MAX, 1);
Al Viroe572ec72006-03-18 22:27:18 -05004672STORE_FUNCTION(cfq_back_seek_max_store, &cfqd->cfq_back_max, 0, UINT_MAX, 0);
Jens Axboefe094d92008-01-31 13:08:54 +01004673STORE_FUNCTION(cfq_back_seek_penalty_store, &cfqd->cfq_back_penalty, 1,
4674 UINT_MAX, 0);
Jens Axboe22e2c502005-06-27 10:55:12 +02004675STORE_FUNCTION(cfq_slice_idle_store, &cfqd->cfq_slice_idle, 0, UINT_MAX, 1);
Vivek Goyal80bdf0c2010-08-23 12:24:26 +02004676STORE_FUNCTION(cfq_group_idle_store, &cfqd->cfq_group_idle, 0, UINT_MAX, 1);
Jens Axboe22e2c502005-06-27 10:55:12 +02004677STORE_FUNCTION(cfq_slice_sync_store, &cfqd->cfq_slice[1], 1, UINT_MAX, 1);
4678STORE_FUNCTION(cfq_slice_async_store, &cfqd->cfq_slice[0], 1, UINT_MAX, 1);
Jens Axboefe094d92008-01-31 13:08:54 +01004679STORE_FUNCTION(cfq_slice_async_rq_store, &cfqd->cfq_slice_async_rq, 1,
4680 UINT_MAX, 0);
Jens Axboe963b72f2009-10-03 19:42:18 +02004681STORE_FUNCTION(cfq_low_latency_store, &cfqd->cfq_latency, 0, 1, 0);
Tao Ma5bf14c02012-04-01 14:33:39 -07004682STORE_FUNCTION(cfq_target_latency_store, &cfqd->cfq_target_latency, 1, UINT_MAX, 1);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004683#undef STORE_FUNCTION
4684
Al Viroe572ec72006-03-18 22:27:18 -05004685#define CFQ_ATTR(name) \
4686 __ATTR(name, S_IRUGO|S_IWUSR, cfq_##name##_show, cfq_##name##_store)
Jens Axboe3b181522005-06-27 10:56:24 +02004687
Al Viroe572ec72006-03-18 22:27:18 -05004688static struct elv_fs_entry cfq_attrs[] = {
4689 CFQ_ATTR(quantum),
Al Viroe572ec72006-03-18 22:27:18 -05004690 CFQ_ATTR(fifo_expire_sync),
4691 CFQ_ATTR(fifo_expire_async),
4692 CFQ_ATTR(back_seek_max),
4693 CFQ_ATTR(back_seek_penalty),
4694 CFQ_ATTR(slice_sync),
4695 CFQ_ATTR(slice_async),
4696 CFQ_ATTR(slice_async_rq),
4697 CFQ_ATTR(slice_idle),
Vivek Goyal80bdf0c2010-08-23 12:24:26 +02004698 CFQ_ATTR(group_idle),
Jens Axboe963b72f2009-10-03 19:42:18 +02004699 CFQ_ATTR(low_latency),
Tao Ma5bf14c02012-04-01 14:33:39 -07004700 CFQ_ATTR(target_latency),
Al Viroe572ec72006-03-18 22:27:18 -05004701 __ATTR_NULL
Linus Torvalds1da177e2005-04-16 15:20:36 -07004702};
4703
Linus Torvalds1da177e2005-04-16 15:20:36 -07004704static struct elevator_type iosched_cfq = {
4705 .ops = {
4706 .elevator_merge_fn = cfq_merge,
4707 .elevator_merged_fn = cfq_merged_request,
4708 .elevator_merge_req_fn = cfq_merged_requests,
Jens Axboeda775262006-12-20 11:04:12 +01004709 .elevator_allow_merge_fn = cfq_allow_merge,
Divyesh Shah812d4022010-04-08 21:14:23 -07004710 .elevator_bio_merged_fn = cfq_bio_merged,
Jens Axboeb4878f22005-10-20 16:42:29 +02004711 .elevator_dispatch_fn = cfq_dispatch_requests,
Linus Torvalds1da177e2005-04-16 15:20:36 -07004712 .elevator_add_req_fn = cfq_insert_request,
Jens Axboeb4878f22005-10-20 16:42:29 +02004713 .elevator_activate_req_fn = cfq_activate_request,
Linus Torvalds1da177e2005-04-16 15:20:36 -07004714 .elevator_deactivate_req_fn = cfq_deactivate_request,
Linus Torvalds1da177e2005-04-16 15:20:36 -07004715 .elevator_completed_req_fn = cfq_completed_request,
Jens Axboe21183b02006-07-13 12:33:14 +02004716 .elevator_former_req_fn = elv_rb_former_request,
4717 .elevator_latter_req_fn = elv_rb_latter_request,
Tejun Heo9b84cac2011-12-14 00:33:42 +01004718 .elevator_init_icq_fn = cfq_init_icq,
Tejun Heo7e5a8792011-12-14 00:33:42 +01004719 .elevator_exit_icq_fn = cfq_exit_icq,
Linus Torvalds1da177e2005-04-16 15:20:36 -07004720 .elevator_set_req_fn = cfq_set_request,
4721 .elevator_put_req_fn = cfq_put_request,
4722 .elevator_may_queue_fn = cfq_may_queue,
4723 .elevator_init_fn = cfq_init_queue,
4724 .elevator_exit_fn = cfq_exit_queue,
Jens Axboe0bb97942015-06-10 08:01:20 -06004725 .elevator_registered_fn = cfq_registered_queue,
Linus Torvalds1da177e2005-04-16 15:20:36 -07004726 },
Tejun Heo3d3c2372011-12-14 00:33:42 +01004727 .icq_size = sizeof(struct cfq_io_cq),
4728 .icq_align = __alignof__(struct cfq_io_cq),
Al Viro3d1ab402006-03-18 18:35:43 -05004729 .elevator_attrs = cfq_attrs,
Tejun Heo3d3c2372011-12-14 00:33:42 +01004730 .elevator_name = "cfq",
Linus Torvalds1da177e2005-04-16 15:20:36 -07004731 .elevator_owner = THIS_MODULE,
4732};
4733
Vivek Goyal3e252062009-12-04 10:36:42 -05004734#ifdef CONFIG_CFQ_GROUP_IOSCHED
Tejun Heo3c798392012-04-16 13:57:25 -07004735static struct blkcg_policy blkcg_policy_cfq = {
Tejun Heo2ee867dc2015-08-18 14:55:34 -07004736 .dfl_cftypes = cfq_blkcg_files,
Tejun Heo880f50e2015-08-18 14:55:30 -07004737 .legacy_cftypes = cfq_blkcg_legacy_files,
Tejun Heof9fcc2d2012-04-16 13:57:27 -07004738
Tejun Heoe4a9bde2015-08-18 14:55:16 -07004739 .cpd_alloc_fn = cfq_cpd_alloc,
Arianna Avanzinie48453c2015-06-05 23:38:42 +02004740 .cpd_init_fn = cfq_cpd_init,
Tejun Heoe4a9bde2015-08-18 14:55:16 -07004741 .cpd_free_fn = cfq_cpd_free,
Tejun Heo69d7fde2015-08-18 14:55:36 -07004742 .cpd_bind_fn = cfq_cpd_bind,
Tejun Heoe4a9bde2015-08-18 14:55:16 -07004743
Tejun Heo001bea72015-08-18 14:55:11 -07004744 .pd_alloc_fn = cfq_pd_alloc,
Tejun Heof9fcc2d2012-04-16 13:57:27 -07004745 .pd_init_fn = cfq_pd_init,
Tejun Heo0b399202013-01-09 08:05:13 -08004746 .pd_offline_fn = cfq_pd_offline,
Tejun Heo001bea72015-08-18 14:55:11 -07004747 .pd_free_fn = cfq_pd_free,
Tejun Heof9fcc2d2012-04-16 13:57:27 -07004748 .pd_reset_stats_fn = cfq_pd_reset_stats,
Vivek Goyal3e252062009-12-04 10:36:42 -05004749};
Vivek Goyal3e252062009-12-04 10:36:42 -05004750#endif
4751
Linus Torvalds1da177e2005-04-16 15:20:36 -07004752static int __init cfq_init(void)
4753{
Tejun Heo3d3c2372011-12-14 00:33:42 +01004754 int ret;
4755
Jens Axboe22e2c502005-06-27 10:55:12 +02004756 /*
4757 * could be 0 on HZ < 1000 setups
4758 */
4759 if (!cfq_slice_async)
4760 cfq_slice_async = 1;
4761 if (!cfq_slice_idle)
4762 cfq_slice_idle = 1;
4763
Vivek Goyal80bdf0c2010-08-23 12:24:26 +02004764#ifdef CONFIG_CFQ_GROUP_IOSCHED
4765 if (!cfq_group_idle)
4766 cfq_group_idle = 1;
Tejun Heo8bd435b2012-04-13 13:11:28 -07004767
Tejun Heo3c798392012-04-16 13:57:25 -07004768 ret = blkcg_policy_register(&blkcg_policy_cfq);
Tejun Heo8bd435b2012-04-13 13:11:28 -07004769 if (ret)
4770 return ret;
Tejun Heoffea73f2012-06-04 10:02:29 +02004771#else
4772 cfq_group_idle = 0;
4773#endif
Tejun Heo8bd435b2012-04-13 13:11:28 -07004774
Tejun Heofd794952012-06-04 10:01:38 +02004775 ret = -ENOMEM;
Tejun Heo3d3c2372011-12-14 00:33:42 +01004776 cfq_pool = KMEM_CACHE(cfq_queue, 0);
4777 if (!cfq_pool)
Tejun Heo8bd435b2012-04-13 13:11:28 -07004778 goto err_pol_unreg;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004779
Tejun Heo3d3c2372011-12-14 00:33:42 +01004780 ret = elv_register(&iosched_cfq);
Tejun Heo8bd435b2012-04-13 13:11:28 -07004781 if (ret)
4782 goto err_free_pool;
Tejun Heo3d3c2372011-12-14 00:33:42 +01004783
Adrian Bunk2fdd82b2007-12-12 18:51:56 +01004784 return 0;
Tejun Heo8bd435b2012-04-13 13:11:28 -07004785
4786err_free_pool:
4787 kmem_cache_destroy(cfq_pool);
4788err_pol_unreg:
Tejun Heoffea73f2012-06-04 10:02:29 +02004789#ifdef CONFIG_CFQ_GROUP_IOSCHED
Tejun Heo3c798392012-04-16 13:57:25 -07004790 blkcg_policy_unregister(&blkcg_policy_cfq);
Tejun Heoffea73f2012-06-04 10:02:29 +02004791#endif
Tejun Heo8bd435b2012-04-13 13:11:28 -07004792 return ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004793}
4794
4795static void __exit cfq_exit(void)
4796{
Tejun Heoffea73f2012-06-04 10:02:29 +02004797#ifdef CONFIG_CFQ_GROUP_IOSCHED
Tejun Heo3c798392012-04-16 13:57:25 -07004798 blkcg_policy_unregister(&blkcg_policy_cfq);
Tejun Heoffea73f2012-06-04 10:02:29 +02004799#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -07004800 elv_unregister(&iosched_cfq);
Tejun Heo3d3c2372011-12-14 00:33:42 +01004801 kmem_cache_destroy(cfq_pool);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004802}
4803
4804module_init(cfq_init);
4805module_exit(cfq_exit);
4806
4807MODULE_AUTHOR("Jens Axboe");
4808MODULE_LICENSE("GPL");
4809MODULE_DESCRIPTION("Completely Fair Queueing IO scheduler");