blob: 3baec8cbe7cdf4bd76f89bc6572d65896200adfb [file] [log] [blame]
Tatyana Brokhman16349062012-09-20 10:46:10 +03001/*
2 * ROW (Read Over Write) I/O scheduler.
3 *
Tatyana Brokhmanbd56be32013-01-13 22:04:59 +02004 * Copyright (c) 2012-2013, The Linux Foundation. All rights reserved.
Tatyana Brokhman16349062012-09-20 10:46:10 +03005 *
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License version 2 and
8 * only version 2 as published by the Free Software Foundation.
9 *
10 * This program is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details.
14 */
15
16/* See Documentation/block/row-iosched.txt */
17
18#include <linux/kernel.h>
19#include <linux/fs.h>
20#include <linux/blkdev.h>
21#include <linux/elevator.h>
22#include <linux/bio.h>
23#include <linux/module.h>
24#include <linux/slab.h>
25#include <linux/init.h>
26#include <linux/compiler.h>
27#include <linux/blktrace_api.h>
Tatyana Brokhmance1a8ed2013-01-17 20:56:07 +020028#include <linux/hrtimer.h>
Tatyana Brokhman16349062012-09-20 10:46:10 +030029
30/*
31 * enum row_queue_prio - Priorities of the ROW queues
32 *
33 * This enum defines the priorities (and the number of queues)
Tatyana Brokhmandb7c1532013-01-23 17:15:49 +020034 * the requests will be distributed to. The higher priority -
35 * the bigger is the "bus time" (or the dispatch quantum) given
36 * to that queue.
Tatyana Brokhman16349062012-09-20 10:46:10 +030037 * ROWQ_PRIO_HIGH_READ - is the higher priority queue.
38 *
39 */
40enum row_queue_prio {
41 ROWQ_PRIO_HIGH_READ = 0,
Tatyana Brokhman16349062012-09-20 10:46:10 +030042 ROWQ_PRIO_HIGH_SWRITE,
Tatyana Brokhmandb7c1532013-01-23 17:15:49 +020043 ROWQ_PRIO_REG_READ,
Tatyana Brokhman16349062012-09-20 10:46:10 +030044 ROWQ_PRIO_REG_SWRITE,
45 ROWQ_PRIO_REG_WRITE,
46 ROWQ_PRIO_LOW_READ,
47 ROWQ_PRIO_LOW_SWRITE,
48 ROWQ_MAX_PRIO,
49};
50
Tatyana Brokhmandb7c1532013-01-23 17:15:49 +020051/*
52 * The following indexes define the distribution of ROW queues according to
53 * priorities. Each index defines the first queue in that priority group.
54 */
55#define ROWQ_HIGH_PRIO_IDX ROWQ_PRIO_HIGH_READ
56#define ROWQ_REG_PRIO_IDX ROWQ_PRIO_REG_READ
57#define ROWQ_LOW_PRIO_IDX ROWQ_PRIO_LOW_READ
58
Tatyana Brokhman9375bcc2013-01-12 16:23:18 +020059/**
60 * struct row_queue_params - ROW queue parameters
61 * @idling_enabled: Flag indicating whether idling is enable on
62 * the queue
63 * @quantum: Number of requests to be dispatched from this queue
64 * in a dispatch cycle
65 * @is_urgent: Flags indicating whether the queue can notify on
66 * urgent requests
67 *
68 */
69struct row_queue_params {
70 bool idling_enabled;
71 int quantum;
72 bool is_urgent;
Tatyana Brokhman16349062012-09-20 10:46:10 +030073};
74
Tatyana Brokhman9375bcc2013-01-12 16:23:18 +020075/*
76 * This array holds the default values of the different configurables
77 * for each ROW queue. Each row of the array holds the following values:
78 * {idling_enabled, quantum, is_urgent}
79 * Each row corresponds to a queue with the same index (according to
80 * enum row_queue_prio)
Tatyana Brokhmandb7c1532013-01-23 17:15:49 +020081 * Note: The quantums are valid inside their priority type. For example:
82 * For every 10 high priority read requests, 1 high priority sync
83 * write will be dispatched.
84 * For every 100 regular read requests 1 regular write request will
85 * be dispatched.
Tatyana Brokhman9375bcc2013-01-12 16:23:18 +020086 */
87static const struct row_queue_params row_queues_def[] = {
88/* idling_enabled, quantum, is_urgent */
Tatyana Brokhmandb7c1532013-01-23 17:15:49 +020089 {true, 10, true}, /* ROWQ_PRIO_HIGH_READ */
Tatyana Brokhmanfe6fd2f2013-03-12 21:17:18 +020090 {false, 1, false}, /* ROWQ_PRIO_HIGH_SWRITE */
Tatyana Brokhman9375bcc2013-01-12 16:23:18 +020091 {true, 100, true}, /* ROWQ_PRIO_REG_READ */
Tatyana Brokhman9375bcc2013-01-12 16:23:18 +020092 {false, 1, false}, /* ROWQ_PRIO_REG_SWRITE */
93 {false, 1, false}, /* ROWQ_PRIO_REG_WRITE */
94 {false, 1, false}, /* ROWQ_PRIO_LOW_READ */
95 {false, 1, false} /* ROWQ_PRIO_LOW_SWRITE */
Tatyana Brokhman16349062012-09-20 10:46:10 +030096};
97
Tatyana Brokhmanbfb04f62012-12-06 13:17:19 +020098/* Default values for idling on read queues (in msec) */
99#define ROW_IDLE_TIME_MSEC 5
100#define ROW_READ_FREQ_MSEC 20
Tatyana Brokhman16349062012-09-20 10:46:10 +0300101
102/**
103 * struct rowq_idling_data - parameters for idling on the queue
Tatyana Brokhmanbfb04f62012-12-06 13:17:19 +0200104 * @last_insert_time: time the last request was inserted
105 * to the queue
Tatyana Brokhman16349062012-09-20 10:46:10 +0300106 * @begin_idling: flag indicating wether we should idle
107 *
108 */
109struct rowq_idling_data {
Tatyana Brokhmanbfb04f62012-12-06 13:17:19 +0200110 ktime_t last_insert_time;
Tatyana Brokhman16349062012-09-20 10:46:10 +0300111 bool begin_idling;
112};
113
114/**
115 * struct row_queue - requests grouping structure
116 * @rdata: parent row_data structure
117 * @fifo: fifo of requests
118 * @prio: queue priority (enum row_queue_prio)
119 * @nr_dispatched: number of requests already dispatched in
120 * the current dispatch cycle
Tatyana Brokhmanbd56be32013-01-13 22:04:59 +0200121 * @nr_req: number of requests in queue
Tatyana Brokhman8a970bc2013-01-12 16:21:12 +0200122 * @dispatch quantum: number of requests this queue may
123 * dispatch in a dispatch cycle
Tatyana Brokhman16349062012-09-20 10:46:10 +0300124 * @idle_data: data for idling on queues
125 *
126 */
127struct row_queue {
128 struct row_data *rdata;
129 struct list_head fifo;
130 enum row_queue_prio prio;
131
132 unsigned int nr_dispatched;
Tatyana Brokhman16349062012-09-20 10:46:10 +0300133
Tatyana Brokhmanbd56be32013-01-13 22:04:59 +0200134 unsigned int nr_req;
Tatyana Brokhman8a970bc2013-01-12 16:21:12 +0200135 int disp_quantum;
Tatyana Brokhmanbd56be32013-01-13 22:04:59 +0200136
Tatyana Brokhman16349062012-09-20 10:46:10 +0300137 /* used only for READ queues */
138 struct rowq_idling_data idle_data;
139};
140
141/**
142 * struct idling_data - data for idling on empty rqueue
Tatyana Brokhmance1a8ed2013-01-17 20:56:07 +0200143 * @idle_time_ms: idling duration (msec)
144 * @freq_ms: min time between two requests that
Tatyana Brokhman16349062012-09-20 10:46:10 +0300145 * triger idling (msec)
Tatyana Brokhmance1a8ed2013-01-17 20:56:07 +0200146 * @hr_timer: idling timer
147 * @idle_work: the work to be scheduled when idling timer expires
Tatyana Brokhmandb7c1532013-01-23 17:15:49 +0200148 * @idling_queue_idx: index of the queues we're idling on
Tatyana Brokhman16349062012-09-20 10:46:10 +0300149 *
150 */
151struct idling_data {
Tatyana Brokhmance1a8ed2013-01-17 20:56:07 +0200152 s64 idle_time_ms;
153 s64 freq_ms;
Tatyana Brokhman16349062012-09-20 10:46:10 +0300154
Tatyana Brokhmance1a8ed2013-01-17 20:56:07 +0200155 struct hrtimer hr_timer;
156 struct work_struct idle_work;
Tatyana Brokhmandb7c1532013-01-23 17:15:49 +0200157 enum row_queue_prio idling_queue_idx;
Tatyana Brokhman16349062012-09-20 10:46:10 +0300158};
159
160/**
161 * struct row_queue - Per block device rqueue structure
162 * @dispatch_queue: dispatch rqueue
Tatyana Brokhman8a970bc2013-01-12 16:21:12 +0200163 * @row_queues: array of priority request queues
Tatyana Brokhmandb7c1532013-01-23 17:15:49 +0200164 * @rd_idle_data: data for idling after READ request
Tatyana Brokhman16349062012-09-20 10:46:10 +0300165 * @nr_reqs: nr_reqs[0] holds the number of all READ requests in
166 * scheduler, nr_reqs[1] holds the number of all WRITE
167 * requests in scheduler
Tatyana Brokhmanfe6fd2f2013-03-12 21:17:18 +0200168 * @urgent_in_flight: flag indicating that there is an urgent
169 * request that was dispatched to driver and is yet to
170 * complete.
171 * @pending_urgent_rq: pointer to the pending urgent request
172 * @last_served_ioprio_class: I/O priority class that was last dispatched from
Tatyana Brokhman16349062012-09-20 10:46:10 +0300173 * @cycle_flags: used for marking unserved queueus
174 *
175 */
176struct row_data {
177 struct request_queue *dispatch_queue;
178
Tatyana Brokhman8a970bc2013-01-12 16:21:12 +0200179 struct row_queue row_queues[ROWQ_MAX_PRIO];
Tatyana Brokhman16349062012-09-20 10:46:10 +0300180
Tatyana Brokhmandb7c1532013-01-23 17:15:49 +0200181 struct idling_data rd_idle_data;
Tatyana Brokhman16349062012-09-20 10:46:10 +0300182 unsigned int nr_reqs[2];
Tatyana Brokhmanfe6fd2f2013-03-12 21:17:18 +0200183 bool urgent_in_flight;
184 struct request *pending_urgent_rq;
185 int last_served_ioprio_class;
Tatyana Brokhman16349062012-09-20 10:46:10 +0300186 unsigned int cycle_flags;
187};
188
189#define RQ_ROWQ(rq) ((struct row_queue *) ((rq)->elv.priv[0]))
190
191#define row_log(q, fmt, args...) \
192 blk_add_trace_msg(q, "%s():" fmt , __func__, ##args)
193#define row_log_rowq(rdata, rowq_id, fmt, args...) \
194 blk_add_trace_msg(rdata->dispatch_queue, "rowq%d " fmt, \
195 rowq_id, ##args)
196
197static inline void row_mark_rowq_unserved(struct row_data *rd,
198 enum row_queue_prio qnum)
199{
200 rd->cycle_flags |= (1 << qnum);
201}
202
203static inline void row_clear_rowq_unserved(struct row_data *rd,
204 enum row_queue_prio qnum)
205{
206 rd->cycle_flags &= ~(1 << qnum);
207}
208
209static inline int row_rowq_unserved(struct row_data *rd,
210 enum row_queue_prio qnum)
211{
212 return rd->cycle_flags & (1 << qnum);
213}
214
Tatyana Brokhmanbd56be32013-01-13 22:04:59 +0200215static inline void __maybe_unused row_dump_queues_stat(struct row_data *rd)
216{
217 int i;
218
Tatyana Brokhman8a970bc2013-01-12 16:21:12 +0200219 row_log(rd->dispatch_queue, " Queues status:");
Tatyana Brokhmanbd56be32013-01-13 22:04:59 +0200220 for (i = 0; i < ROWQ_MAX_PRIO; i++)
221 row_log(rd->dispatch_queue,
222 "queue%d: dispatched= %d, nr_req=%d", i,
223 rd->row_queues[i].nr_dispatched,
224 rd->row_queues[i].nr_req);
225}
226
Tatyana Brokhman16349062012-09-20 10:46:10 +0300227/******************** Static helper functions ***********************/
Tatyana Brokhman16349062012-09-20 10:46:10 +0300228static void kick_queue(struct work_struct *work)
229{
Tatyana Brokhman16349062012-09-20 10:46:10 +0300230 struct idling_data *read_data =
Tatyana Brokhmance1a8ed2013-01-17 20:56:07 +0200231 container_of(work, struct idling_data, idle_work);
232 struct row_data *rd =
233 container_of(read_data, struct row_data, rd_idle_data);
234
235 blk_run_queue(rd->dispatch_queue);
236}
237
238
239static enum hrtimer_restart row_idle_hrtimer_fn(struct hrtimer *hr_timer)
240{
241 struct idling_data *read_data =
242 container_of(hr_timer, struct idling_data, hr_timer);
Tatyana Brokhman16349062012-09-20 10:46:10 +0300243 struct row_data *rd =
Tatyana Brokhmandb7c1532013-01-23 17:15:49 +0200244 container_of(read_data, struct row_data, rd_idle_data);
Tatyana Brokhman16349062012-09-20 10:46:10 +0300245
Tatyana Brokhmandb7c1532013-01-23 17:15:49 +0200246 row_log_rowq(rd, rd->rd_idle_data.idling_queue_idx,
247 "Performing delayed work");
Tatyana Brokhman16349062012-09-20 10:46:10 +0300248 /* Mark idling process as done */
Tatyana Brokhmandb7c1532013-01-23 17:15:49 +0200249 rd->row_queues[rd->rd_idle_data.idling_queue_idx].
250 idle_data.begin_idling = false;
251 rd->rd_idle_data.idling_queue_idx = ROWQ_MAX_PRIO;
Tatyana Brokhman16349062012-09-20 10:46:10 +0300252
Tatyana Brokhmandb7c1532013-01-23 17:15:49 +0200253 if (!rd->nr_reqs[READ] && !rd->nr_reqs[WRITE])
Tatyana Brokhman16349062012-09-20 10:46:10 +0300254 row_log(rd->dispatch_queue, "No requests in scheduler");
Tatyana Brokhmance1a8ed2013-01-17 20:56:07 +0200255 else
256 kblockd_schedule_work(rd->dispatch_queue,
257 &read_data->idle_work);
258 return HRTIMER_NORESTART;
Tatyana Brokhman16349062012-09-20 10:46:10 +0300259}
260
Tatyana Brokhman16349062012-09-20 10:46:10 +0300261/******************* Elevator callback functions *********************/
262
263/*
264 * row_add_request() - Add request to the scheduler
265 * @q: requests queue
266 * @rq: request to add
267 *
268 */
269static void row_add_request(struct request_queue *q,
270 struct request *rq)
271{
272 struct row_data *rd = (struct row_data *)q->elevator->elevator_data;
273 struct row_queue *rqueue = RQ_ROWQ(rq);
Tatyana Brokhmandb7c1532013-01-23 17:15:49 +0200274 s64 diff_ms;
Tatyana Brokhman16349062012-09-20 10:46:10 +0300275
276 list_add_tail(&rq->queuelist, &rqueue->fifo);
277 rd->nr_reqs[rq_data_dir(rq)]++;
Tatyana Brokhmanbd56be32013-01-13 22:04:59 +0200278 rqueue->nr_req++;
Tatyana Brokhman16349062012-09-20 10:46:10 +0300279 rq_set_fifo_time(rq, jiffies); /* for statistics*/
280
Tatyana Brokhmanfe6fd2f2013-03-12 21:17:18 +0200281 if (rq->cmd_flags & REQ_URGENT) {
282 WARN_ON(1);
283 blk_dump_rq_flags(rq, "");
284 rq->cmd_flags &= ~REQ_URGENT;
285 }
286
Tatyana Brokhman9375bcc2013-01-12 16:23:18 +0200287 if (row_queues_def[rqueue->prio].idling_enabled) {
Tatyana Brokhmandb7c1532013-01-23 17:15:49 +0200288 if (rd->rd_idle_data.idling_queue_idx == rqueue->prio &&
Tatyana Brokhmance1a8ed2013-01-17 20:56:07 +0200289 hrtimer_active(&rd->rd_idle_data.hr_timer)) {
290 (void)hrtimer_cancel(&rd->rd_idle_data.hr_timer);
Tatyana Brokhmandb7c1532013-01-23 17:15:49 +0200291 row_log_rowq(rd, rqueue->prio,
292 "Canceled delayed work on %d",
293 rd->rd_idle_data.idling_queue_idx);
294 rd->rd_idle_data.idling_queue_idx = ROWQ_MAX_PRIO;
295 }
296 diff_ms = ktime_to_ms(ktime_sub(ktime_get(),
297 rqueue->idle_data.last_insert_time));
298 if (unlikely(diff_ms < 0)) {
Tatyana Brokhmanfe6fd2f2013-03-12 21:17:18 +0200299 pr_err("%s(): time delta error: diff_ms < 0",
300 __func__);
Tatyana Brokhmandb7c1532013-01-23 17:15:49 +0200301 rqueue->idle_data.begin_idling = false;
302 return;
303 }
Tatyana Brokhmance1a8ed2013-01-17 20:56:07 +0200304 if (diff_ms < rd->rd_idle_data.freq_ms) {
Tatyana Brokhman16349062012-09-20 10:46:10 +0300305 rqueue->idle_data.begin_idling = true;
306 row_log_rowq(rd, rqueue->prio, "Enable idling");
Tatyana Brokhmanbfb04f62012-12-06 13:17:19 +0200307 } else {
Tatyana Brokhman16349062012-09-20 10:46:10 +0300308 rqueue->idle_data.begin_idling = false;
Tatyana Brokhmandb7c1532013-01-23 17:15:49 +0200309 row_log_rowq(rd, rqueue->prio, "Disable idling (%ldms)",
310 (long)diff_ms);
Tatyana Brokhmanbfb04f62012-12-06 13:17:19 +0200311 }
Tatyana Brokhman16349062012-09-20 10:46:10 +0300312
Tatyana Brokhmanbfb04f62012-12-06 13:17:19 +0200313 rqueue->idle_data.last_insert_time = ktime_get();
Tatyana Brokhman16349062012-09-20 10:46:10 +0300314 }
Tatyana Brokhman9375bcc2013-01-12 16:23:18 +0200315 if (row_queues_def[rqueue->prio].is_urgent &&
Tatyana Brokhmanfe6fd2f2013-03-12 21:17:18 +0200316 !rd->pending_urgent_rq && !rd->urgent_in_flight) {
317 /* Handle High Priority queues */
318 if (rqueue->prio < ROWQ_REG_PRIO_IDX &&
319 rd->last_served_ioprio_class != IOPRIO_CLASS_RT) {
320 row_log_rowq(rd, rqueue->prio,
321 "added (high prio) urgent request");
322 rq->cmd_flags |= REQ_URGENT;
323 rd->pending_urgent_rq = rq;
324 } else if (row_rowq_unserved(rd, rqueue->prio)) {
325 /* Handle Regular priotity queues */
326 row_log_rowq(rd, rqueue->prio,
327 "added urgent request (total on queue=%d)",
328 rqueue->nr_req);
329 rq->cmd_flags |= REQ_URGENT;
330 WARN_ON(rqueue->nr_req > 1);
331 rd->pending_urgent_rq = rq;
332 }
Tatyana Brokhman0ef81432012-12-20 19:23:58 +0200333 } else
Tatyana Brokhmanbd56be32013-01-13 22:04:59 +0200334 row_log_rowq(rd, rqueue->prio,
335 "added request (total on queue=%d)", rqueue->nr_req);
Tatyana Brokhman16349062012-09-20 10:46:10 +0300336}
337
Tatyana Brokhmanb7bf9ac2012-10-30 08:33:06 +0200338/**
339 * row_reinsert_req() - Reinsert request back to the scheduler
340 * @q: requests queue
341 * @rq: request to add
342 *
343 * Reinsert the given request back to the queue it was
344 * dispatched from as if it was never dispatched.
345 *
346 * Returns 0 on success, error code otherwise
347 */
348static int row_reinsert_req(struct request_queue *q,
349 struct request *rq)
350{
351 struct row_data *rd = q->elevator->elevator_data;
352 struct row_queue *rqueue = RQ_ROWQ(rq);
353
Tatyana Brokhmanfe6fd2f2013-03-12 21:17:18 +0200354 if (!rqueue || rqueue->prio >= ROWQ_MAX_PRIO)
Tatyana Brokhmanb7bf9ac2012-10-30 08:33:06 +0200355 return -EIO;
Tatyana Brokhmanb7bf9ac2012-10-30 08:33:06 +0200356
357 list_add(&rq->queuelist, &rqueue->fifo);
358 rd->nr_reqs[rq_data_dir(rq)]++;
Tatyana Brokhmanbd56be32013-01-13 22:04:59 +0200359 rqueue->nr_req++;
Tatyana Brokhmanb7bf9ac2012-10-30 08:33:06 +0200360
Tatyana Brokhmanbd56be32013-01-13 22:04:59 +0200361 row_log_rowq(rd, rqueue->prio,
Tatyana Brokhmanfe6fd2f2013-03-12 21:17:18 +0200362 "%s request reinserted (total on queue=%d)",
363 (rq_data_dir(rq) == READ ? "READ" : "write"), rqueue->nr_req);
Tatyana Brokhmanb7bf9ac2012-10-30 08:33:06 +0200364
Tatyana Brokhmanfe6fd2f2013-03-12 21:17:18 +0200365 if (rq->cmd_flags & REQ_URGENT) {
366 /*
367 * It's not compliant with the design to re-insert
368 * urgent requests. We want to be able to track this
369 * down.
370 */
371 WARN_ON(1);
372 if (!rd->urgent_in_flight) {
373 pr_err("%s(): no urgent in flight", __func__);
374 } else {
375 rd->urgent_in_flight = false;
376 pr_err("%s(): reinserting URGENT %s req",
377 __func__,
378 (rq_data_dir(rq) == READ ? "READ" : "WRITE"));
379 if (rd->pending_urgent_rq) {
380 pr_err("%s(): urgent rq is pending",
381 __func__);
382 rd->pending_urgent_rq->cmd_flags &= ~REQ_URGENT;
383 }
384 rd->pending_urgent_rq = rq;
385 }
386 }
Tatyana Brokhmanb7bf9ac2012-10-30 08:33:06 +0200387 return 0;
388}
389
Tatyana Brokhman4c3c3cc2013-01-24 15:08:40 +0200390static void row_completed_req(struct request_queue *q, struct request *rq)
391{
392 struct row_data *rd = q->elevator->elevator_data;
393
394 if (rq->cmd_flags & REQ_URGENT) {
Tatyana Brokhmanfe6fd2f2013-03-12 21:17:18 +0200395 if (!rd->urgent_in_flight) {
396 WARN_ON(1);
397 pr_err("%s(): URGENT req but urgent_in_flight = F",
Tatyana Brokhman4c3c3cc2013-01-24 15:08:40 +0200398 __func__);
Tatyana Brokhman4c3c3cc2013-01-24 15:08:40 +0200399 }
Tatyana Brokhmanfe6fd2f2013-03-12 21:17:18 +0200400 rd->urgent_in_flight = false;
401 rq->cmd_flags &= ~REQ_URGENT;
Tatyana Brokhman4c3c3cc2013-01-24 15:08:40 +0200402 }
Tatyana Brokhmanfe6fd2f2013-03-12 21:17:18 +0200403 row_log(q, "completed %s %s req.",
404 (rq->cmd_flags & REQ_URGENT ? "URGENT" : "regular"),
405 (rq_data_dir(rq) == READ ? "READ" : "WRITE"));
Tatyana Brokhman4c3c3cc2013-01-24 15:08:40 +0200406}
407
Tatyana Brokhmanb7bf9ac2012-10-30 08:33:06 +0200408/**
Tatyana Brokhman0ef81432012-12-20 19:23:58 +0200409 * row_urgent_pending() - Return TRUE if there is an urgent
410 * request on scheduler
411 * @q: requests queue
412 */
413static bool row_urgent_pending(struct request_queue *q)
414{
415 struct row_data *rd = q->elevator->elevator_data;
Tatyana Brokhman0ef81432012-12-20 19:23:58 +0200416
Tatyana Brokhmanfe6fd2f2013-03-12 21:17:18 +0200417 if (rd->urgent_in_flight) {
Tatyana Brokhman4c3c3cc2013-01-24 15:08:40 +0200418 row_log(rd->dispatch_queue, "%d urgent requests in flight",
Tatyana Brokhmanfe6fd2f2013-03-12 21:17:18 +0200419 rd->urgent_in_flight);
Tatyana Brokhman4c3c3cc2013-01-24 15:08:40 +0200420 return false;
421 }
422
Tatyana Brokhmanfe6fd2f2013-03-12 21:17:18 +0200423 if (rd->pending_urgent_rq) {
424 row_log(rd->dispatch_queue, "Urgent request pending");
425 return true;
426 }
Shashank Babu Chinta Venkata3df69bf2013-02-26 17:33:55 -0800427
Tatyana Brokhmanfe6fd2f2013-03-12 21:17:18 +0200428 row_log(rd->dispatch_queue, "no urgent request pending/in flight");
Tatyana Brokhman0ef81432012-12-20 19:23:58 +0200429 return false;
430}
431
432/**
Tatyana Brokhman16349062012-09-20 10:46:10 +0300433 * row_remove_request() - Remove given request from scheduler
434 * @q: requests queue
435 * @rq: request to remove
436 *
437 */
Tatyana Brokhmanfe6fd2f2013-03-12 21:17:18 +0200438static void row_remove_request(struct row_data *rd,
Tatyana Brokhman16349062012-09-20 10:46:10 +0300439 struct request *rq)
440{
Tatyana Brokhmanbd56be32013-01-13 22:04:59 +0200441 struct row_queue *rqueue = RQ_ROWQ(rq);
Tatyana Brokhman16349062012-09-20 10:46:10 +0300442
Tatyana Brokhmanfe6fd2f2013-03-12 21:17:18 +0200443 list_del_init(&(rq)->queuelist);
444 if (rd->pending_urgent_rq == rq)
445 rd->pending_urgent_rq = NULL;
446 else
447 BUG_ON(rq->cmd_flags & REQ_URGENT);
Tatyana Brokhmanbd56be32013-01-13 22:04:59 +0200448 rqueue->nr_req--;
Tatyana Brokhman16349062012-09-20 10:46:10 +0300449 rd->nr_reqs[rq_data_dir(rq)]--;
450}
451
452/*
453 * row_dispatch_insert() - move request to dispatch queue
Tatyana Brokhmandb7c1532013-01-23 17:15:49 +0200454 * @rd: pointer to struct row_data
Tatyana Brokhmanfe6fd2f2013-03-12 21:17:18 +0200455 * @rq: the request to dispatch
Tatyana Brokhman16349062012-09-20 10:46:10 +0300456 *
Tatyana Brokhmanfe6fd2f2013-03-12 21:17:18 +0200457 * This function moves the given request to the dispatch queue
Tatyana Brokhman16349062012-09-20 10:46:10 +0300458 *
459 */
Tatyana Brokhmanfe6fd2f2013-03-12 21:17:18 +0200460static void row_dispatch_insert(struct row_data *rd, struct request *rq)
Tatyana Brokhman16349062012-09-20 10:46:10 +0300461{
Tatyana Brokhmanfe6fd2f2013-03-12 21:17:18 +0200462 struct row_queue *rqueue = RQ_ROWQ(rq);
Tatyana Brokhman16349062012-09-20 10:46:10 +0300463
Tatyana Brokhmanfe6fd2f2013-03-12 21:17:18 +0200464 row_remove_request(rd, rq);
465 elv_dispatch_sort(rd->dispatch_queue, rq);
466 if (rq->cmd_flags & REQ_URGENT) {
467 WARN_ON(rd->urgent_in_flight);
468 rd->urgent_in_flight = true;
469 }
470 rqueue->nr_dispatched++;
471 row_clear_rowq_unserved(rd, rqueue->prio);
472 row_log_rowq(rd, rqueue->prio,
473 " Dispatched request %p nr_disp = %d", rq,
474 rqueue->nr_dispatched);
475 if (rqueue->prio < ROWQ_REG_PRIO_IDX)
476 rd->last_served_ioprio_class = IOPRIO_CLASS_RT;
477 else if (rqueue->prio < ROWQ_LOW_PRIO_IDX)
478 rd->last_served_ioprio_class = IOPRIO_CLASS_BE;
479 else
480 rd->last_served_ioprio_class = IOPRIO_CLASS_IDLE;
Tatyana Brokhman16349062012-09-20 10:46:10 +0300481}
482
483/*
Tatyana Brokhmandb7c1532013-01-23 17:15:49 +0200484 * row_get_ioprio_class_to_serve() - Return the next I/O priority
485 * class to dispatch requests from
Tatyana Brokhman16349062012-09-20 10:46:10 +0300486 * @rd: pointer to struct row_data
Tatyana Brokhmandb7c1532013-01-23 17:15:49 +0200487 * @force: flag indicating if forced dispatch
Tatyana Brokhman16349062012-09-20 10:46:10 +0300488 *
Tatyana Brokhmandb7c1532013-01-23 17:15:49 +0200489 * This function returns the next I/O priority class to serve
490 * {IOPRIO_CLASS_NONE, IOPRIO_CLASS_RT, IOPRIO_CLASS_BE, IOPRIO_CLASS_IDLE}.
491 * If there are no more requests in scheduler or if we're idling on some queue
492 * IOPRIO_CLASS_NONE will be returned.
493 * If idling is scheduled on a lower priority queue than the one that needs
494 * to be served, it will be canceled.
Tatyana Brokhman16349062012-09-20 10:46:10 +0300495 *
496 */
Tatyana Brokhmandb7c1532013-01-23 17:15:49 +0200497static int row_get_ioprio_class_to_serve(struct row_data *rd, int force)
Tatyana Brokhman16349062012-09-20 10:46:10 +0300498{
Tatyana Brokhmandb7c1532013-01-23 17:15:49 +0200499 int i;
500 int ret = IOPRIO_CLASS_NONE;
Tatyana Brokhman16349062012-09-20 10:46:10 +0300501
Tatyana Brokhmandb7c1532013-01-23 17:15:49 +0200502 if (!rd->nr_reqs[READ] && !rd->nr_reqs[WRITE]) {
Tatyana Brokhman16349062012-09-20 10:46:10 +0300503 row_log(rd->dispatch_queue, "No more requests in scheduler");
Tatyana Brokhmandb7c1532013-01-23 17:15:49 +0200504 goto check_idling;
Tatyana Brokhman16349062012-09-20 10:46:10 +0300505 }
506
Tatyana Brokhmandb7c1532013-01-23 17:15:49 +0200507 /* First, go over the high priority queues */
508 for (i = 0; i < ROWQ_REG_PRIO_IDX; i++) {
509 if (!list_empty(&rd->row_queues[i].fifo)) {
Tatyana Brokhmance1a8ed2013-01-17 20:56:07 +0200510 if (hrtimer_active(&rd->rd_idle_data.hr_timer)) {
511 (void)hrtimer_cancel(
512 &rd->rd_idle_data.hr_timer);
Tatyana Brokhmandb7c1532013-01-23 17:15:49 +0200513 row_log_rowq(rd,
514 rd->rd_idle_data.idling_queue_idx,
515 "Canceling delayed work on %d. RT pending",
516 rd->rd_idle_data.idling_queue_idx);
517 rd->rd_idle_data.idling_queue_idx =
518 ROWQ_MAX_PRIO;
519 }
520 ret = IOPRIO_CLASS_RT;
521 goto done;
522 }
523 }
Tatyana Brokhman16349062012-09-20 10:46:10 +0300524
525 /*
Tatyana Brokhmandb7c1532013-01-23 17:15:49 +0200526 * At the moment idling is implemented only for READ queues.
527 * If enabled on WRITE, this needs updating
Tatyana Brokhman16349062012-09-20 10:46:10 +0300528 */
Tatyana Brokhmance1a8ed2013-01-17 20:56:07 +0200529 if (hrtimer_active(&rd->rd_idle_data.hr_timer)) {
Tatyana Brokhmandb7c1532013-01-23 17:15:49 +0200530 row_log(rd->dispatch_queue, "Delayed work pending. Exiting");
531 goto done;
532 }
533check_idling:
534 /* Check for (high priority) idling and enable if needed */
535 for (i = 0; i < ROWQ_REG_PRIO_IDX && !force; i++) {
536 if (rd->row_queues[i].idle_data.begin_idling &&
537 row_queues_def[i].idling_enabled)
538 goto initiate_idling;
Tatyana Brokhman16349062012-09-20 10:46:10 +0300539 }
540
Tatyana Brokhmandb7c1532013-01-23 17:15:49 +0200541 /* Regular priority queues */
542 for (i = ROWQ_REG_PRIO_IDX; i < ROWQ_LOW_PRIO_IDX; i++) {
543 if (list_empty(&rd->row_queues[i].fifo)) {
544 /* We can idle only if this is not a forced dispatch */
545 if (rd->row_queues[i].idle_data.begin_idling &&
546 !force && row_queues_def[i].idling_enabled)
547 goto initiate_idling;
548 } else {
549 ret = IOPRIO_CLASS_BE;
550 goto done;
551 }
552 }
553
554 if (rd->nr_reqs[READ] || rd->nr_reqs[WRITE])
555 ret = IOPRIO_CLASS_IDLE;
556 goto done;
557
558initiate_idling:
Tatyana Brokhmance1a8ed2013-01-17 20:56:07 +0200559 hrtimer_start(&rd->rd_idle_data.hr_timer,
560 ktime_set(0, rd->rd_idle_data.idle_time_ms * NSEC_PER_MSEC),
561 HRTIMER_MODE_REL);
562
563 rd->rd_idle_data.idling_queue_idx = i;
564 row_log_rowq(rd, i, "Scheduled delayed work on %d. exiting", i);
565
Tatyana Brokhmandb7c1532013-01-23 17:15:49 +0200566done:
567 return ret;
568}
569
570static void row_restart_cycle(struct row_data *rd,
571 int start_idx, int end_idx)
572{
573 int i;
574
575 row_dump_queues_stat(rd);
576 for (i = start_idx; i < end_idx; i++) {
577 if (rd->row_queues[i].nr_dispatched <
578 rd->row_queues[i].disp_quantum)
579 row_mark_rowq_unserved(rd, i);
580 rd->row_queues[i].nr_dispatched = 0;
581 }
582 row_log(rd->dispatch_queue, "Restarting cycle for class @ %d-%d",
583 start_idx, end_idx);
584}
585
586/*
587 * row_get_next_queue() - selects the next queue to dispatch from
588 * @q: requests queue
589 * @rd: pointer to struct row_data
590 * @start_idx/end_idx: indexes in the row_queues array to select a queue
591 * from.
592 *
593 * Return index of the queues to dispatch from. Error code if fails.
594 *
595 */
596static int row_get_next_queue(struct request_queue *q, struct row_data *rd,
597 int start_idx, int end_idx)
598{
599 int i = start_idx;
600 bool restart = true;
601 int ret = -EIO;
602
603 do {
604 if (list_empty(&rd->row_queues[i].fifo) ||
605 rd->row_queues[i].nr_dispatched >=
606 rd->row_queues[i].disp_quantum) {
607 i++;
608 if (i == end_idx && restart) {
609 /* Restart cycle for this priority class */
610 row_restart_cycle(rd, start_idx, end_idx);
611 i = start_idx;
612 restart = false;
613 }
614 } else {
615 ret = i;
616 break;
617 }
618 } while (i < end_idx);
619
620 return ret;
Tatyana Brokhman16349062012-09-20 10:46:10 +0300621}
622
623/*
624 * row_dispatch_requests() - selects the next request to dispatch
625 * @q: requests queue
Tatyana Brokhmandb7c1532013-01-23 17:15:49 +0200626 * @force: flag indicating if forced dispatch
Tatyana Brokhman16349062012-09-20 10:46:10 +0300627 *
628 * Return 0 if no requests were moved to the dispatch queue.
629 * 1 otherwise
630 *
631 */
632static int row_dispatch_requests(struct request_queue *q, int force)
633{
634 struct row_data *rd = (struct row_data *)q->elevator->elevator_data;
Tatyana Brokhmandb7c1532013-01-23 17:15:49 +0200635 int ret = 0, currq, ioprio_class_to_serve, start_idx, end_idx;
Tatyana Brokhman16349062012-09-20 10:46:10 +0300636
Tatyana Brokhmance1a8ed2013-01-17 20:56:07 +0200637 if (force && hrtimer_active(&rd->rd_idle_data.hr_timer)) {
638 (void)hrtimer_cancel(&rd->rd_idle_data.hr_timer);
Tatyana Brokhmandb7c1532013-01-23 17:15:49 +0200639 row_log_rowq(rd, rd->rd_idle_data.idling_queue_idx,
640 "Canceled delayed work on %d - forced dispatch",
641 rd->rd_idle_data.idling_queue_idx);
642 rd->rd_idle_data.idling_queue_idx = ROWQ_MAX_PRIO;
Tatyana Brokhman16349062012-09-20 10:46:10 +0300643 }
644
Tatyana Brokhmanfe6fd2f2013-03-12 21:17:18 +0200645 if (rd->pending_urgent_rq) {
646 row_log(rd->dispatch_queue, "dispatching urgent request");
647 row_dispatch_insert(rd, rd->pending_urgent_rq);
648 ret = 1;
649 goto done;
650 }
651
Tatyana Brokhmandb7c1532013-01-23 17:15:49 +0200652 ioprio_class_to_serve = row_get_ioprio_class_to_serve(rd, force);
653 row_log(rd->dispatch_queue, "Dispatching from %d priority class",
654 ioprio_class_to_serve);
655
656 switch (ioprio_class_to_serve) {
657 case IOPRIO_CLASS_NONE:
Tatyana Brokhmanfe6fd2f2013-03-12 21:17:18 +0200658 rd->last_served_ioprio_class = IOPRIO_CLASS_NONE;
Tatyana Brokhmandb7c1532013-01-23 17:15:49 +0200659 goto done;
660 case IOPRIO_CLASS_RT:
661 start_idx = ROWQ_HIGH_PRIO_IDX;
662 end_idx = ROWQ_REG_PRIO_IDX;
663 break;
664 case IOPRIO_CLASS_BE:
665 start_idx = ROWQ_REG_PRIO_IDX;
666 end_idx = ROWQ_LOW_PRIO_IDX;
667 break;
668 case IOPRIO_CLASS_IDLE:
669 start_idx = ROWQ_LOW_PRIO_IDX;
670 end_idx = ROWQ_MAX_PRIO;
671 break;
672 default:
673 pr_err("%s(): Invalid I/O priority class", __func__);
Tatyana Brokhman16349062012-09-20 10:46:10 +0300674 goto done;
675 }
676
Tatyana Brokhmandb7c1532013-01-23 17:15:49 +0200677 currq = row_get_next_queue(q, rd, start_idx, end_idx);
Tatyana Brokhman16349062012-09-20 10:46:10 +0300678
Tatyana Brokhmandb7c1532013-01-23 17:15:49 +0200679 /* Dispatch */
680 if (currq >= 0) {
Tatyana Brokhmanfe6fd2f2013-03-12 21:17:18 +0200681 row_dispatch_insert(rd,
682 rq_entry_fifo(rd->row_queues[currq].fifo.next));
Tatyana Brokhmandb7c1532013-01-23 17:15:49 +0200683 ret = 1;
Tatyana Brokhman16349062012-09-20 10:46:10 +0300684 }
Tatyana Brokhman16349062012-09-20 10:46:10 +0300685done:
686 return ret;
687}
688
689/*
690 * row_init_queue() - Init scheduler data structures
691 * @q: requests queue
692 *
693 * Return pointer to struct row_data to be saved in elevator for
694 * this dispatch queue
695 *
696 */
697static void *row_init_queue(struct request_queue *q)
698{
699
700 struct row_data *rdata;
701 int i;
702
703 rdata = kmalloc_node(sizeof(*rdata),
704 GFP_KERNEL | __GFP_ZERO, q->node);
705 if (!rdata)
706 return NULL;
707
Tatyana Brokhman522778f2013-01-24 16:17:27 +0200708 memset(rdata, 0, sizeof(*rdata));
Tatyana Brokhman16349062012-09-20 10:46:10 +0300709 for (i = 0; i < ROWQ_MAX_PRIO; i++) {
Tatyana Brokhman8a970bc2013-01-12 16:21:12 +0200710 INIT_LIST_HEAD(&rdata->row_queues[i].fifo);
Tatyana Brokhman9375bcc2013-01-12 16:23:18 +0200711 rdata->row_queues[i].disp_quantum = row_queues_def[i].quantum;
Tatyana Brokhman8a970bc2013-01-12 16:21:12 +0200712 rdata->row_queues[i].rdata = rdata;
713 rdata->row_queues[i].prio = i;
714 rdata->row_queues[i].idle_data.begin_idling = false;
715 rdata->row_queues[i].idle_data.last_insert_time =
Tatyana Brokhmanbfb04f62012-12-06 13:17:19 +0200716 ktime_set(0, 0);
Tatyana Brokhman16349062012-09-20 10:46:10 +0300717 }
718
719 /*
720 * Currently idling is enabled only for READ queues. If we want to
721 * enable it for write queues also, note that idling frequency will
722 * be the same in both cases
723 */
Tatyana Brokhmance1a8ed2013-01-17 20:56:07 +0200724 rdata->rd_idle_data.idle_time_ms = ROW_IDLE_TIME_MSEC;
725 rdata->rd_idle_data.freq_ms = ROW_READ_FREQ_MSEC;
726 hrtimer_init(&rdata->rd_idle_data.hr_timer,
727 CLOCK_MONOTONIC, HRTIMER_MODE_REL);
728 rdata->rd_idle_data.hr_timer.function = &row_idle_hrtimer_fn;
729
730 INIT_WORK(&rdata->rd_idle_data.idle_work, kick_queue);
Tatyana Brokhmanfe6fd2f2013-03-12 21:17:18 +0200731 rdata->last_served_ioprio_class = IOPRIO_CLASS_NONE;
Tatyana Brokhmandb7c1532013-01-23 17:15:49 +0200732 rdata->rd_idle_data.idling_queue_idx = ROWQ_MAX_PRIO;
Tatyana Brokhman16349062012-09-20 10:46:10 +0300733 rdata->dispatch_queue = q;
734
Tatyana Brokhman16349062012-09-20 10:46:10 +0300735 return rdata;
736}
737
738/*
739 * row_exit_queue() - called on unloading the RAW scheduler
740 * @e: poiner to struct elevator_queue
741 *
742 */
743static void row_exit_queue(struct elevator_queue *e)
744{
745 struct row_data *rd = (struct row_data *)e->elevator_data;
746 int i;
747
748 for (i = 0; i < ROWQ_MAX_PRIO; i++)
Tatyana Brokhman8a970bc2013-01-12 16:21:12 +0200749 BUG_ON(!list_empty(&rd->row_queues[i].fifo));
Tatyana Brokhmance1a8ed2013-01-17 20:56:07 +0200750 if (hrtimer_cancel(&rd->rd_idle_data.hr_timer))
Tatyana Brokhmanfe6fd2f2013-03-12 21:17:18 +0200751 pr_err("%s(): idle timer was active!", __func__);
Tatyana Brokhmandb7c1532013-01-23 17:15:49 +0200752 rd->rd_idle_data.idling_queue_idx = ROWQ_MAX_PRIO;
Tatyana Brokhman16349062012-09-20 10:46:10 +0300753 kfree(rd);
754}
755
756/*
757 * row_merged_requests() - Called when 2 requests are merged
758 * @q: requests queue
759 * @rq: request the two requests were merged into
760 * @next: request that was merged
761 */
762static void row_merged_requests(struct request_queue *q, struct request *rq,
763 struct request *next)
764{
765 struct row_queue *rqueue = RQ_ROWQ(next);
766
767 list_del_init(&next->queuelist);
Tatyana Brokhmanbd56be32013-01-13 22:04:59 +0200768 rqueue->nr_req--;
Tatyana Brokhmanfe6fd2f2013-03-12 21:17:18 +0200769 if (rqueue->rdata->pending_urgent_rq == next) {
770 pr_err("\n\nROW_WARNING: merging pending urgent!");
771 rqueue->rdata->pending_urgent_rq = rq;
772 rq->cmd_flags |= REQ_URGENT;
773 WARN_ON(!(next->cmd_flags & REQ_URGENT));
774 next->cmd_flags &= ~REQ_URGENT;
775 }
Tatyana Brokhman16349062012-09-20 10:46:10 +0300776 rqueue->rdata->nr_reqs[rq_data_dir(rq)]--;
777}
778
779/*
Tatyana Brokhmandb7c1532013-01-23 17:15:49 +0200780 * row_get_queue_prio() - Get queue priority for a given request
Tatyana Brokhman16349062012-09-20 10:46:10 +0300781 *
782 * This is a helping function which purpose is to determine what
783 * ROW queue the given request should be added to (and
Tatyana Brokhmandb7c1532013-01-23 17:15:49 +0200784 * dispatched from later on)
Tatyana Brokhman16349062012-09-20 10:46:10 +0300785 *
Tatyana Brokhman16349062012-09-20 10:46:10 +0300786 */
Tatyana Brokhmanfe6fd2f2013-03-12 21:17:18 +0200787static enum row_queue_prio row_get_queue_prio(struct request *rq,
788 struct row_data *rd)
Tatyana Brokhman16349062012-09-20 10:46:10 +0300789{
790 const int data_dir = rq_data_dir(rq);
791 const bool is_sync = rq_is_sync(rq);
Tatyana Brokhmandb7c1532013-01-23 17:15:49 +0200792 enum row_queue_prio q_type = ROWQ_MAX_PRIO;
793 int ioprio_class = IOPRIO_PRIO_CLASS(rq->elv.icq->ioc->ioprio);
Tatyana Brokhman16349062012-09-20 10:46:10 +0300794
Tatyana Brokhmandb7c1532013-01-23 17:15:49 +0200795 switch (ioprio_class) {
796 case IOPRIO_CLASS_RT:
797 if (data_dir == READ)
798 q_type = ROWQ_PRIO_HIGH_READ;
799 else if (is_sync)
800 q_type = ROWQ_PRIO_HIGH_SWRITE;
801 else {
802 pr_err("%s:%s(): got a simple write from RT_CLASS. How???",
803 rq->rq_disk->disk_name, __func__);
804 q_type = ROWQ_PRIO_REG_WRITE;
805 }
806 break;
807 case IOPRIO_CLASS_IDLE:
808 if (data_dir == READ)
809 q_type = ROWQ_PRIO_LOW_READ;
810 else if (is_sync)
811 q_type = ROWQ_PRIO_LOW_SWRITE;
812 else {
813 pr_err("%s:%s(): got a simple write from IDLE_CLASS. How???",
814 rq->rq_disk->disk_name, __func__);
815 q_type = ROWQ_PRIO_REG_WRITE;
816 }
817 break;
818 case IOPRIO_CLASS_NONE:
819 case IOPRIO_CLASS_BE:
820 default:
821 if (data_dir == READ)
822 q_type = ROWQ_PRIO_REG_READ;
823 else if (is_sync)
824 q_type = ROWQ_PRIO_REG_SWRITE;
825 else
826 q_type = ROWQ_PRIO_REG_WRITE;
827 break;
828 }
829
830 return q_type;
Tatyana Brokhman16349062012-09-20 10:46:10 +0300831}
832
833/*
834 * row_set_request() - Set ROW data structures associated with this request.
835 * @q: requests queue
836 * @rq: pointer to the request
837 * @gfp_mask: ignored
838 *
839 */
840static int
841row_set_request(struct request_queue *q, struct request *rq, gfp_t gfp_mask)
842{
843 struct row_data *rd = (struct row_data *)q->elevator->elevator_data;
844 unsigned long flags;
845
846 spin_lock_irqsave(q->queue_lock, flags);
847 rq->elv.priv[0] =
Tatyana Brokhmanfe6fd2f2013-03-12 21:17:18 +0200848 (void *)(&rd->row_queues[row_get_queue_prio(rq, rd)]);
Tatyana Brokhman16349062012-09-20 10:46:10 +0300849 spin_unlock_irqrestore(q->queue_lock, flags);
850
851 return 0;
852}
853
854/********** Helping sysfs functions/defenitions for ROW attributes ******/
855static ssize_t row_var_show(int var, char *page)
856{
857 return snprintf(page, 100, "%d\n", var);
858}
859
860static ssize_t row_var_store(int *var, const char *page, size_t count)
861{
862 int err;
863 err = kstrtoul(page, 10, (unsigned long *)var);
864
865 return count;
866}
867
868#define SHOW_FUNCTION(__FUNC, __VAR, __CONV) \
869static ssize_t __FUNC(struct elevator_queue *e, char *page) \
870{ \
871 struct row_data *rowd = e->elevator_data; \
872 int __data = __VAR; \
873 if (__CONV) \
874 __data = jiffies_to_msecs(__data); \
875 return row_var_show(__data, (page)); \
876}
877SHOW_FUNCTION(row_hp_read_quantum_show,
878 rowd->row_queues[ROWQ_PRIO_HIGH_READ].disp_quantum, 0);
879SHOW_FUNCTION(row_rp_read_quantum_show,
880 rowd->row_queues[ROWQ_PRIO_REG_READ].disp_quantum, 0);
881SHOW_FUNCTION(row_hp_swrite_quantum_show,
882 rowd->row_queues[ROWQ_PRIO_HIGH_SWRITE].disp_quantum, 0);
883SHOW_FUNCTION(row_rp_swrite_quantum_show,
884 rowd->row_queues[ROWQ_PRIO_REG_SWRITE].disp_quantum, 0);
885SHOW_FUNCTION(row_rp_write_quantum_show,
886 rowd->row_queues[ROWQ_PRIO_REG_WRITE].disp_quantum, 0);
887SHOW_FUNCTION(row_lp_read_quantum_show,
888 rowd->row_queues[ROWQ_PRIO_LOW_READ].disp_quantum, 0);
889SHOW_FUNCTION(row_lp_swrite_quantum_show,
890 rowd->row_queues[ROWQ_PRIO_LOW_SWRITE].disp_quantum, 0);
Tatyana Brokhmance1a8ed2013-01-17 20:56:07 +0200891SHOW_FUNCTION(row_rd_idle_data_show, rowd->rd_idle_data.idle_time_ms, 0);
892SHOW_FUNCTION(row_rd_idle_data_freq_show, rowd->rd_idle_data.freq_ms, 0);
Tatyana Brokhman16349062012-09-20 10:46:10 +0300893#undef SHOW_FUNCTION
894
895#define STORE_FUNCTION(__FUNC, __PTR, MIN, MAX, __CONV) \
896static ssize_t __FUNC(struct elevator_queue *e, \
897 const char *page, size_t count) \
898{ \
899 struct row_data *rowd = e->elevator_data; \
900 int __data; \
901 int ret = row_var_store(&__data, (page), count); \
902 if (__CONV) \
903 __data = (int)msecs_to_jiffies(__data); \
904 if (__data < (MIN)) \
905 __data = (MIN); \
906 else if (__data > (MAX)) \
907 __data = (MAX); \
908 *(__PTR) = __data; \
909 return ret; \
910}
911STORE_FUNCTION(row_hp_read_quantum_store,
Tatyana Brokhman0a0345a2012-10-15 20:50:54 +0200912&rowd->row_queues[ROWQ_PRIO_HIGH_READ].disp_quantum, 1, INT_MAX, 0);
Tatyana Brokhman16349062012-09-20 10:46:10 +0300913STORE_FUNCTION(row_rp_read_quantum_store,
Tatyana Brokhman0a0345a2012-10-15 20:50:54 +0200914 &rowd->row_queues[ROWQ_PRIO_REG_READ].disp_quantum,
915 1, INT_MAX, 0);
Tatyana Brokhman16349062012-09-20 10:46:10 +0300916STORE_FUNCTION(row_hp_swrite_quantum_store,
Tatyana Brokhman0a0345a2012-10-15 20:50:54 +0200917 &rowd->row_queues[ROWQ_PRIO_HIGH_SWRITE].disp_quantum,
918 1, INT_MAX, 0);
Tatyana Brokhman16349062012-09-20 10:46:10 +0300919STORE_FUNCTION(row_rp_swrite_quantum_store,
Tatyana Brokhman0a0345a2012-10-15 20:50:54 +0200920 &rowd->row_queues[ROWQ_PRIO_REG_SWRITE].disp_quantum,
921 1, INT_MAX, 0);
Tatyana Brokhman16349062012-09-20 10:46:10 +0300922STORE_FUNCTION(row_rp_write_quantum_store,
Tatyana Brokhman0a0345a2012-10-15 20:50:54 +0200923 &rowd->row_queues[ROWQ_PRIO_REG_WRITE].disp_quantum,
924 1, INT_MAX, 0);
Tatyana Brokhman16349062012-09-20 10:46:10 +0300925STORE_FUNCTION(row_lp_read_quantum_store,
Tatyana Brokhman0a0345a2012-10-15 20:50:54 +0200926 &rowd->row_queues[ROWQ_PRIO_LOW_READ].disp_quantum,
927 1, INT_MAX, 0);
Tatyana Brokhman16349062012-09-20 10:46:10 +0300928STORE_FUNCTION(row_lp_swrite_quantum_store,
Tatyana Brokhman0a0345a2012-10-15 20:50:54 +0200929 &rowd->row_queues[ROWQ_PRIO_LOW_SWRITE].disp_quantum,
Tatyana Brokhmandb7c1532013-01-23 17:15:49 +0200930 1, INT_MAX, 0);
Tatyana Brokhmance1a8ed2013-01-17 20:56:07 +0200931STORE_FUNCTION(row_rd_idle_data_store, &rowd->rd_idle_data.idle_time_ms,
932 1, INT_MAX, 0);
933STORE_FUNCTION(row_rd_idle_data_freq_store, &rowd->rd_idle_data.freq_ms,
Tatyana Brokhmandb7c1532013-01-23 17:15:49 +0200934 1, INT_MAX, 0);
Tatyana Brokhman16349062012-09-20 10:46:10 +0300935
936#undef STORE_FUNCTION
937
938#define ROW_ATTR(name) \
939 __ATTR(name, S_IRUGO|S_IWUSR, row_##name##_show, \
940 row_##name##_store)
941
942static struct elv_fs_entry row_attrs[] = {
943 ROW_ATTR(hp_read_quantum),
944 ROW_ATTR(rp_read_quantum),
945 ROW_ATTR(hp_swrite_quantum),
946 ROW_ATTR(rp_swrite_quantum),
947 ROW_ATTR(rp_write_quantum),
948 ROW_ATTR(lp_read_quantum),
949 ROW_ATTR(lp_swrite_quantum),
Tatyana Brokhmandb7c1532013-01-23 17:15:49 +0200950 ROW_ATTR(rd_idle_data),
951 ROW_ATTR(rd_idle_data_freq),
Tatyana Brokhman16349062012-09-20 10:46:10 +0300952 __ATTR_NULL
953};
954
955static struct elevator_type iosched_row = {
956 .ops = {
957 .elevator_merge_req_fn = row_merged_requests,
958 .elevator_dispatch_fn = row_dispatch_requests,
959 .elevator_add_req_fn = row_add_request,
Tatyana Brokhmanb7bf9ac2012-10-30 08:33:06 +0200960 .elevator_reinsert_req_fn = row_reinsert_req,
Tatyana Brokhman0ef81432012-12-20 19:23:58 +0200961 .elevator_is_urgent_fn = row_urgent_pending,
Tatyana Brokhman4c3c3cc2013-01-24 15:08:40 +0200962 .elevator_completed_req_fn = row_completed_req,
Tatyana Brokhman16349062012-09-20 10:46:10 +0300963 .elevator_former_req_fn = elv_rb_former_request,
964 .elevator_latter_req_fn = elv_rb_latter_request,
965 .elevator_set_req_fn = row_set_request,
966 .elevator_init_fn = row_init_queue,
967 .elevator_exit_fn = row_exit_queue,
968 },
Tatyana Brokhmandb7c1532013-01-23 17:15:49 +0200969 .icq_size = sizeof(struct io_cq),
970 .icq_align = __alignof__(struct io_cq),
Tatyana Brokhman16349062012-09-20 10:46:10 +0300971 .elevator_attrs = row_attrs,
972 .elevator_name = "row",
973 .elevator_owner = THIS_MODULE,
974};
975
976static int __init row_init(void)
977{
978 elv_register(&iosched_row);
979 return 0;
980}
981
982static void __exit row_exit(void)
983{
984 elv_unregister(&iosched_row);
985}
986
987module_init(row_init);
988module_exit(row_exit);
989
990MODULE_LICENSE("GPLv2");
991MODULE_DESCRIPTION("Read Over Write IO scheduler");