blob: a27555908d359654a619209ff34dad174d018247 [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
2 * linux/drivers/block/elevator.c
3 *
4 * Block device elevator/IO-scheduler.
5 *
6 * Copyright (C) 2000 Andrea Arcangeli <andrea@suse.de> SuSE
7 *
8 * 30042000 Jens Axboe <axboe@suse.de> :
9 *
10 * Split the elevator a bit so that it is possible to choose a different
11 * one or even write a new "plug in". There are three pieces:
12 * - elevator_fn, inserts a new request in the queue list
13 * - elevator_merge_fn, decides whether a new buffer can be merged with
14 * an existing request
15 * - elevator_dequeue_fn, called when a request is taken off the active list
16 *
17 * 20082000 Dave Jones <davej@suse.de> :
18 * Removed tests for max-bomb-segments, which was breaking elvtune
19 * when run without -bN
20 *
21 * Jens:
22 * - Rework again to work with bio instead of buffer_heads
23 * - loose bi_dev comparisons, partition handling is right now
24 * - completely modularize elevator setup and teardown
25 *
26 */
27#include <linux/kernel.h>
28#include <linux/fs.h>
29#include <linux/blkdev.h>
30#include <linux/elevator.h>
31#include <linux/bio.h>
32#include <linux/config.h>
33#include <linux/module.h>
34#include <linux/slab.h>
35#include <linux/init.h>
36#include <linux/compiler.h>
37
38#include <asm/uaccess.h>
39
40static DEFINE_SPINLOCK(elv_list_lock);
41static LIST_HEAD(elv_list);
42
Tejun Heo8922e162005-10-20 16:23:44 +020043static inline sector_t rq_last_sector(struct request *rq)
44{
45 return rq->sector + rq->nr_sectors;
46}
47
Linus Torvalds1da177e2005-04-16 15:20:36 -070048/*
49 * can we safely merge with this request?
50 */
51inline int elv_rq_merge_ok(struct request *rq, struct bio *bio)
52{
53 if (!rq_mergeable(rq))
54 return 0;
55
56 /*
57 * different data direction or already started, don't merge
58 */
59 if (bio_data_dir(bio) != rq_data_dir(rq))
60 return 0;
61
62 /*
63 * same device and no special stuff set, merge is ok
64 */
65 if (rq->rq_disk == bio->bi_bdev->bd_disk &&
66 !rq->waiting && !rq->special)
67 return 1;
68
69 return 0;
70}
71EXPORT_SYMBOL(elv_rq_merge_ok);
72
73inline int elv_try_merge(struct request *__rq, struct bio *bio)
74{
75 int ret = ELEVATOR_NO_MERGE;
76
77 /*
78 * we can merge and sequence is ok, check if it's possible
79 */
80 if (elv_rq_merge_ok(__rq, bio)) {
81 if (__rq->sector + __rq->nr_sectors == bio->bi_sector)
82 ret = ELEVATOR_BACK_MERGE;
83 else if (__rq->sector - bio_sectors(bio) == bio->bi_sector)
84 ret = ELEVATOR_FRONT_MERGE;
85 }
86
87 return ret;
88}
89EXPORT_SYMBOL(elv_try_merge);
90
91inline int elv_try_last_merge(request_queue_t *q, struct bio *bio)
92{
93 if (q->last_merge)
94 return elv_try_merge(q->last_merge, bio);
95
96 return ELEVATOR_NO_MERGE;
97}
98EXPORT_SYMBOL(elv_try_last_merge);
99
100static struct elevator_type *elevator_find(const char *name)
101{
102 struct elevator_type *e = NULL;
103 struct list_head *entry;
104
Linus Torvalds1da177e2005-04-16 15:20:36 -0700105 list_for_each(entry, &elv_list) {
106 struct elevator_type *__e;
107
108 __e = list_entry(entry, struct elevator_type, list);
109
110 if (!strcmp(__e->elevator_name, name)) {
111 e = __e;
112 break;
113 }
114 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700115
116 return e;
117}
118
119static void elevator_put(struct elevator_type *e)
120{
121 module_put(e->elevator_owner);
122}
123
124static struct elevator_type *elevator_get(const char *name)
125{
Tejun Heo2824bc932005-10-20 10:56:41 +0200126 struct elevator_type *e;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700127
Tejun Heo2824bc932005-10-20 10:56:41 +0200128 spin_lock_irq(&elv_list_lock);
129
130 e = elevator_find(name);
131 if (e && !try_module_get(e->elevator_owner))
132 e = NULL;
133
134 spin_unlock_irq(&elv_list_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700135
136 return e;
137}
138
139static int elevator_attach(request_queue_t *q, struct elevator_type *e,
140 struct elevator_queue *eq)
141{
142 int ret = 0;
143
144 memset(eq, 0, sizeof(*eq));
145 eq->ops = &e->ops;
146 eq->elevator_type = e;
147
148 INIT_LIST_HEAD(&q->queue_head);
149 q->last_merge = NULL;
150 q->elevator = eq;
Tejun Heo8922e162005-10-20 16:23:44 +0200151 q->last_sector = 0;
152 q->boundary_rq = NULL;
153 q->max_back_kb = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700154
155 if (eq->ops->elevator_init_fn)
156 ret = eq->ops->elevator_init_fn(q, eq);
157
158 return ret;
159}
160
161static char chosen_elevator[16];
162
163static void elevator_setup_default(void)
164{
Tejun Heo2824bc932005-10-20 10:56:41 +0200165 struct elevator_type *e;
166
Linus Torvalds1da177e2005-04-16 15:20:36 -0700167 /*
168 * check if default is set and exists
169 */
Tejun Heo2824bc932005-10-20 10:56:41 +0200170 if (chosen_elevator[0] && (e = elevator_get(chosen_elevator))) {
171 elevator_put(e);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700172 return;
Tejun Heo2824bc932005-10-20 10:56:41 +0200173 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700174
175#if defined(CONFIG_IOSCHED_AS)
176 strcpy(chosen_elevator, "anticipatory");
177#elif defined(CONFIG_IOSCHED_DEADLINE)
178 strcpy(chosen_elevator, "deadline");
179#elif defined(CONFIG_IOSCHED_CFQ)
180 strcpy(chosen_elevator, "cfq");
181#elif defined(CONFIG_IOSCHED_NOOP)
182 strcpy(chosen_elevator, "noop");
183#else
184#error "You must build at least 1 IO scheduler into the kernel"
185#endif
186}
187
188static int __init elevator_setup(char *str)
189{
190 strncpy(chosen_elevator, str, sizeof(chosen_elevator) - 1);
191 return 0;
192}
193
194__setup("elevator=", elevator_setup);
195
196int elevator_init(request_queue_t *q, char *name)
197{
198 struct elevator_type *e = NULL;
199 struct elevator_queue *eq;
200 int ret = 0;
201
202 elevator_setup_default();
203
204 if (!name)
205 name = chosen_elevator;
206
207 e = elevator_get(name);
208 if (!e)
209 return -EINVAL;
210
211 eq = kmalloc(sizeof(struct elevator_queue), GFP_KERNEL);
212 if (!eq) {
213 elevator_put(e->elevator_type);
214 return -ENOMEM;
215 }
216
217 ret = elevator_attach(q, e, eq);
218 if (ret) {
219 kfree(eq);
220 elevator_put(e->elevator_type);
221 }
222
223 return ret;
224}
225
226void elevator_exit(elevator_t *e)
227{
228 if (e->ops->elevator_exit_fn)
229 e->ops->elevator_exit_fn(e);
230
231 elevator_put(e->elevator_type);
232 e->elevator_type = NULL;
233 kfree(e);
234}
235
Tejun Heo8922e162005-10-20 16:23:44 +0200236/*
237 * Insert rq into dispatch queue of q. Queue lock must be held on
238 * entry. If sort != 0, rq is sort-inserted; otherwise, rq will be
239 * appended to the dispatch queue. To be used by specific elevators.
240 */
241void elv_dispatch_insert(request_queue_t *q, struct request *rq, int sort)
242{
243 sector_t boundary;
244 unsigned max_back;
245 struct list_head *entry;
246
247 if (!sort) {
248 /* Specific elevator is performing sort. Step away. */
249 q->last_sector = rq_last_sector(rq);
250 q->boundary_rq = rq;
251 list_add_tail(&rq->queuelist, &q->queue_head);
252 return;
253 }
254
255 boundary = q->last_sector;
256 max_back = q->max_back_kb * 2;
257 boundary = boundary > max_back ? boundary - max_back : 0;
258
259 list_for_each_prev(entry, &q->queue_head) {
260 struct request *pos = list_entry_rq(entry);
261
262 if (pos->flags & (REQ_SOFTBARRIER|REQ_HARDBARRIER|REQ_STARTED))
263 break;
264 if (rq->sector >= boundary) {
265 if (pos->sector < boundary)
266 continue;
267 } else {
268 if (pos->sector >= boundary)
269 break;
270 }
271 if (rq->sector >= pos->sector)
272 break;
273 }
274
275 list_add(&rq->queuelist, entry);
276}
277
Linus Torvalds1da177e2005-04-16 15:20:36 -0700278int elv_merge(request_queue_t *q, struct request **req, struct bio *bio)
279{
280 elevator_t *e = q->elevator;
281
282 if (e->ops->elevator_merge_fn)
283 return e->ops->elevator_merge_fn(q, req, bio);
284
285 return ELEVATOR_NO_MERGE;
286}
287
288void elv_merged_request(request_queue_t *q, struct request *rq)
289{
290 elevator_t *e = q->elevator;
291
292 if (e->ops->elevator_merged_fn)
293 e->ops->elevator_merged_fn(q, rq);
294}
295
296void elv_merge_requests(request_queue_t *q, struct request *rq,
297 struct request *next)
298{
299 elevator_t *e = q->elevator;
300
301 if (q->last_merge == next)
302 q->last_merge = NULL;
303
304 if (e->ops->elevator_merge_req_fn)
305 e->ops->elevator_merge_req_fn(q, rq, next);
306}
307
Tejun Heo8922e162005-10-20 16:23:44 +0200308void elv_requeue_request(request_queue_t *q, struct request *rq)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700309{
310 elevator_t *e = q->elevator;
311
312 /*
313 * it already went through dequeue, we need to decrement the
314 * in_flight count again
315 */
Tejun Heo8922e162005-10-20 16:23:44 +0200316 if (blk_account_rq(rq)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700317 q->in_flight--;
Tejun Heo8922e162005-10-20 16:23:44 +0200318 if (blk_sorted_rq(rq) && e->ops->elevator_deactivate_req_fn)
319 e->ops->elevator_deactivate_req_fn(q, rq);
320 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700321
322 rq->flags &= ~REQ_STARTED;
323
Linus Torvalds1da177e2005-04-16 15:20:36 -0700324 /*
325 * if this is the flush, requeue the original instead and drop the flush
326 */
327 if (rq->flags & REQ_BAR_FLUSH) {
328 clear_bit(QUEUE_FLAG_FLUSH, &q->queue_flags);
329 rq = rq->end_io_data;
330 }
331
Tejun Heo8922e162005-10-20 16:23:44 +0200332 __elv_add_request(q, rq, ELEVATOR_INSERT_FRONT, 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700333}
334
335void __elv_add_request(request_queue_t *q, struct request *rq, int where,
336 int plug)
337{
Tejun Heo8922e162005-10-20 16:23:44 +0200338 if (rq->flags & (REQ_SOFTBARRIER | REQ_HARDBARRIER)) {
339 /*
340 * barriers implicitly indicate back insertion
341 */
342 if (where == ELEVATOR_INSERT_SORT)
343 where = ELEVATOR_INSERT_BACK;
344
345 /*
346 * this request is scheduling boundary, update last_sector
347 */
348 if (blk_fs_request(rq)) {
349 q->last_sector = rq_last_sector(rq);
350 q->boundary_rq = rq;
351 }
352 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700353
354 if (plug)
355 blk_plug_device(q);
356
357 rq->q = q;
358
Tejun Heo8922e162005-10-20 16:23:44 +0200359 if (unlikely(test_bit(QUEUE_FLAG_DRAIN, &q->queue_flags))) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700360 /*
361 * if drain is set, store the request "locally". when the drain
362 * is finished, the requests will be handed ordered to the io
363 * scheduler
364 */
365 list_add_tail(&rq->queuelist, &q->drain_list);
Tejun Heo8922e162005-10-20 16:23:44 +0200366 return;
367 }
368
369 switch (where) {
370 case ELEVATOR_INSERT_FRONT:
371 rq->flags |= REQ_SOFTBARRIER;
372
373 list_add(&rq->queuelist, &q->queue_head);
374 break;
375
376 case ELEVATOR_INSERT_BACK:
377 rq->flags |= REQ_SOFTBARRIER;
378
379 while (q->elevator->ops->elevator_dispatch_fn(q, 1))
380 ;
381 list_add_tail(&rq->queuelist, &q->queue_head);
382 /*
383 * We kick the queue here for the following reasons.
384 * - The elevator might have returned NULL previously
385 * to delay requests and returned them now. As the
386 * queue wasn't empty before this request, ll_rw_blk
387 * won't run the queue on return, resulting in hang.
388 * - Usually, back inserted requests won't be merged
389 * with anything. There's no point in delaying queue
390 * processing.
391 */
392 blk_remove_plug(q);
393 q->request_fn(q);
394 break;
395
396 case ELEVATOR_INSERT_SORT:
397 BUG_ON(!blk_fs_request(rq));
398 rq->flags |= REQ_SORTED;
399 q->elevator->ops->elevator_add_req_fn(q, rq);
400 break;
401
402 default:
403 printk(KERN_ERR "%s: bad insertion point %d\n",
404 __FUNCTION__, where);
405 BUG();
406 }
407
408 if (blk_queue_plugged(q)) {
409 int nrq = q->rq.count[READ] + q->rq.count[WRITE]
410 - q->in_flight;
411
412 if (nrq >= q->unplug_thresh)
413 __generic_unplug_device(q);
414 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700415}
416
417void elv_add_request(request_queue_t *q, struct request *rq, int where,
418 int plug)
419{
420 unsigned long flags;
421
422 spin_lock_irqsave(q->queue_lock, flags);
423 __elv_add_request(q, rq, where, plug);
424 spin_unlock_irqrestore(q->queue_lock, flags);
425}
426
427static inline struct request *__elv_next_request(request_queue_t *q)
428{
Tejun Heo8922e162005-10-20 16:23:44 +0200429 struct request *rq;
430
431 if (unlikely(list_empty(&q->queue_head) &&
432 !q->elevator->ops->elevator_dispatch_fn(q, 0)))
433 return NULL;
434
435 rq = list_entry_rq(q->queue_head.next);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700436
437 /*
438 * if this is a barrier write and the device has to issue a
439 * flush sequence to support it, check how far we are
440 */
Tejun Heo8922e162005-10-20 16:23:44 +0200441 if (blk_fs_request(rq) && blk_barrier_rq(rq)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700442 BUG_ON(q->ordered == QUEUE_ORDERED_NONE);
443
444 if (q->ordered == QUEUE_ORDERED_FLUSH &&
445 !blk_barrier_preflush(rq))
446 rq = blk_start_pre_flush(q, rq);
447 }
448
449 return rq;
450}
451
452struct request *elv_next_request(request_queue_t *q)
453{
454 struct request *rq;
455 int ret;
456
457 while ((rq = __elv_next_request(q)) != NULL) {
Tejun Heo8922e162005-10-20 16:23:44 +0200458 if (!(rq->flags & REQ_STARTED)) {
459 elevator_t *e = q->elevator;
460
461 /*
462 * This is the first time the device driver
463 * sees this request (possibly after
464 * requeueing). Notify IO scheduler.
465 */
466 if (blk_sorted_rq(rq) &&
467 e->ops->elevator_activate_req_fn)
468 e->ops->elevator_activate_req_fn(q, rq);
469
470 /*
471 * just mark as started even if we don't start
472 * it, a request that has been delayed should
473 * not be passed by new incoming requests
474 */
475 rq->flags |= REQ_STARTED;
476 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700477
478 if (rq == q->last_merge)
479 q->last_merge = NULL;
480
Tejun Heo8922e162005-10-20 16:23:44 +0200481 if (!q->boundary_rq || q->boundary_rq == rq) {
482 q->last_sector = rq_last_sector(rq);
483 q->boundary_rq = NULL;
484 }
485
Linus Torvalds1da177e2005-04-16 15:20:36 -0700486 if ((rq->flags & REQ_DONTPREP) || !q->prep_rq_fn)
487 break;
488
489 ret = q->prep_rq_fn(q, rq);
490 if (ret == BLKPREP_OK) {
491 break;
492 } else if (ret == BLKPREP_DEFER) {
Tejun Heo 2e759cd2005-04-24 02:04:21 -0500493 /*
494 * the request may have been (partially) prepped.
495 * we need to keep this request in the front to
Tejun Heo8922e162005-10-20 16:23:44 +0200496 * avoid resource deadlock. REQ_STARTED will
497 * prevent other fs requests from passing this one.
Tejun Heo 2e759cd2005-04-24 02:04:21 -0500498 */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700499 rq = NULL;
500 break;
501 } else if (ret == BLKPREP_KILL) {
502 int nr_bytes = rq->hard_nr_sectors << 9;
503
504 if (!nr_bytes)
505 nr_bytes = rq->data_len;
506
507 blkdev_dequeue_request(rq);
508 rq->flags |= REQ_QUIET;
509 end_that_request_chunk(rq, 0, nr_bytes);
510 end_that_request_last(rq);
511 } else {
512 printk(KERN_ERR "%s: bad return=%d\n", __FUNCTION__,
513 ret);
514 break;
515 }
516 }
517
518 return rq;
519}
520
Tejun Heo8922e162005-10-20 16:23:44 +0200521void elv_dequeue_request(request_queue_t *q, struct request *rq)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700522{
Tejun Heo8922e162005-10-20 16:23:44 +0200523 BUG_ON(list_empty(&rq->queuelist));
524
525 list_del_init(&rq->queuelist);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700526
527 /*
528 * the time frame between a request being removed from the lists
529 * and to it is freed is accounted as io that is in progress at
Tejun Heo8922e162005-10-20 16:23:44 +0200530 * the driver side.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700531 */
532 if (blk_account_rq(rq))
533 q->in_flight++;
534
535 /*
536 * the main clearing point for q->last_merge is on retrieval of
537 * request by driver (it calls elv_next_request()), but it _can_
538 * also happen here if a request is added to the queue but later
539 * deleted without ever being given to driver (merged with another
540 * request).
541 */
542 if (rq == q->last_merge)
543 q->last_merge = NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700544}
545
546int elv_queue_empty(request_queue_t *q)
547{
548 elevator_t *e = q->elevator;
549
Tejun Heo8922e162005-10-20 16:23:44 +0200550 if (!list_empty(&q->queue_head))
551 return 0;
552
Linus Torvalds1da177e2005-04-16 15:20:36 -0700553 if (e->ops->elevator_queue_empty_fn)
554 return e->ops->elevator_queue_empty_fn(q);
555
Tejun Heo8922e162005-10-20 16:23:44 +0200556 return 1;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700557}
558
559struct request *elv_latter_request(request_queue_t *q, struct request *rq)
560{
561 struct list_head *next;
562
563 elevator_t *e = q->elevator;
564
565 if (e->ops->elevator_latter_req_fn)
566 return e->ops->elevator_latter_req_fn(q, rq);
567
568 next = rq->queuelist.next;
569 if (next != &q->queue_head && next != &rq->queuelist)
570 return list_entry_rq(next);
571
572 return NULL;
573}
574
575struct request *elv_former_request(request_queue_t *q, struct request *rq)
576{
577 struct list_head *prev;
578
579 elevator_t *e = q->elevator;
580
581 if (e->ops->elevator_former_req_fn)
582 return e->ops->elevator_former_req_fn(q, rq);
583
584 prev = rq->queuelist.prev;
585 if (prev != &q->queue_head && prev != &rq->queuelist)
586 return list_entry_rq(prev);
587
588 return NULL;
589}
590
Jens Axboe22e2c502005-06-27 10:55:12 +0200591int elv_set_request(request_queue_t *q, struct request *rq, struct bio *bio,
592 int gfp_mask)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700593{
594 elevator_t *e = q->elevator;
595
596 if (e->ops->elevator_set_req_fn)
Jens Axboe22e2c502005-06-27 10:55:12 +0200597 return e->ops->elevator_set_req_fn(q, rq, bio, gfp_mask);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700598
599 rq->elevator_private = NULL;
600 return 0;
601}
602
603void elv_put_request(request_queue_t *q, struct request *rq)
604{
605 elevator_t *e = q->elevator;
606
607 if (e->ops->elevator_put_req_fn)
608 e->ops->elevator_put_req_fn(q, rq);
609}
610
Jens Axboe22e2c502005-06-27 10:55:12 +0200611int elv_may_queue(request_queue_t *q, int rw, struct bio *bio)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700612{
613 elevator_t *e = q->elevator;
614
615 if (e->ops->elevator_may_queue_fn)
Jens Axboe22e2c502005-06-27 10:55:12 +0200616 return e->ops->elevator_may_queue_fn(q, rw, bio);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700617
618 return ELV_MQUEUE_MAY;
619}
620
621void elv_completed_request(request_queue_t *q, struct request *rq)
622{
623 elevator_t *e = q->elevator;
624
625 /*
626 * request is released from the driver, io must be done
627 */
Tejun Heo8922e162005-10-20 16:23:44 +0200628 if (blk_account_rq(rq)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700629 q->in_flight--;
Tejun Heo8922e162005-10-20 16:23:44 +0200630 if (blk_sorted_rq(rq) && e->ops->elevator_completed_req_fn)
631 e->ops->elevator_completed_req_fn(q, rq);
632 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700633}
634
635int elv_register_queue(struct request_queue *q)
636{
637 elevator_t *e = q->elevator;
638
639 e->kobj.parent = kobject_get(&q->kobj);
640 if (!e->kobj.parent)
641 return -EBUSY;
642
643 snprintf(e->kobj.name, KOBJ_NAME_LEN, "%s", "iosched");
644 e->kobj.ktype = e->elevator_type->elevator_ktype;
645
646 return kobject_register(&e->kobj);
647}
648
649void elv_unregister_queue(struct request_queue *q)
650{
651 if (q) {
652 elevator_t *e = q->elevator;
653 kobject_unregister(&e->kobj);
654 kobject_put(&q->kobj);
655 }
656}
657
658int elv_register(struct elevator_type *e)
659{
Tejun Heo2824bc932005-10-20 10:56:41 +0200660 spin_lock_irq(&elv_list_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700661 if (elevator_find(e->elevator_name))
662 BUG();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700663 list_add_tail(&e->list, &elv_list);
664 spin_unlock_irq(&elv_list_lock);
665
666 printk(KERN_INFO "io scheduler %s registered", e->elevator_name);
667 if (!strcmp(e->elevator_name, chosen_elevator))
668 printk(" (default)");
669 printk("\n");
670 return 0;
671}
672EXPORT_SYMBOL_GPL(elv_register);
673
674void elv_unregister(struct elevator_type *e)
675{
676 spin_lock_irq(&elv_list_lock);
677 list_del_init(&e->list);
678 spin_unlock_irq(&elv_list_lock);
679}
680EXPORT_SYMBOL_GPL(elv_unregister);
681
682/*
683 * switch to new_e io scheduler. be careful not to introduce deadlocks -
684 * we don't free the old io scheduler, before we have allocated what we
685 * need for the new one. this way we have a chance of going back to the old
686 * one, if the new one fails init for some reason. we also do an intermediate
687 * switch to noop to ensure safety with stack-allocated requests, since they
688 * don't originate from the block layer allocator. noop is safe here, because
689 * it never needs to touch the elevator itself for completion events. DRAIN
690 * flags will make sure we don't touch it for additions either.
691 */
692static void elevator_switch(request_queue_t *q, struct elevator_type *new_e)
693{
694 elevator_t *e = kmalloc(sizeof(elevator_t), GFP_KERNEL);
695 struct elevator_type *noop_elevator = NULL;
696 elevator_t *old_elevator;
697
698 if (!e)
699 goto error;
700
701 /*
702 * first step, drain requests from the block freelist
703 */
704 blk_wait_queue_drained(q, 0);
705
706 /*
707 * unregister old elevator data
708 */
709 elv_unregister_queue(q);
710 old_elevator = q->elevator;
711
712 /*
713 * next step, switch to noop since it uses no private rq structures
714 * and doesn't allocate any memory for anything. then wait for any
715 * non-fs requests in-flight
716 */
717 noop_elevator = elevator_get("noop");
718 spin_lock_irq(q->queue_lock);
719 elevator_attach(q, noop_elevator, e);
720 spin_unlock_irq(q->queue_lock);
721
722 blk_wait_queue_drained(q, 1);
723
724 /*
725 * attach and start new elevator
726 */
727 if (elevator_attach(q, new_e, e))
728 goto fail;
729
730 if (elv_register_queue(q))
731 goto fail_register;
732
733 /*
734 * finally exit old elevator and start queue again
735 */
736 elevator_exit(old_elevator);
737 blk_finish_queue_drain(q);
738 elevator_put(noop_elevator);
739 return;
740
741fail_register:
742 /*
743 * switch failed, exit the new io scheduler and reattach the old
744 * one again (along with re-adding the sysfs dir)
745 */
746 elevator_exit(e);
747fail:
748 q->elevator = old_elevator;
749 elv_register_queue(q);
750 blk_finish_queue_drain(q);
751error:
752 if (noop_elevator)
753 elevator_put(noop_elevator);
754 elevator_put(new_e);
755 printk(KERN_ERR "elevator: switch to %s failed\n",new_e->elevator_name);
756}
757
758ssize_t elv_iosched_store(request_queue_t *q, const char *name, size_t count)
759{
760 char elevator_name[ELV_NAME_MAX];
761 struct elevator_type *e;
762
763 memset(elevator_name, 0, sizeof(elevator_name));
764 strncpy(elevator_name, name, sizeof(elevator_name));
765
766 if (elevator_name[strlen(elevator_name) - 1] == '\n')
767 elevator_name[strlen(elevator_name) - 1] = '\0';
768
769 e = elevator_get(elevator_name);
770 if (!e) {
771 printk(KERN_ERR "elevator: type %s not found\n", elevator_name);
772 return -EINVAL;
773 }
774
775 if (!strcmp(elevator_name, q->elevator->elevator_type->elevator_name))
776 return count;
777
778 elevator_switch(q, e);
779 return count;
780}
781
782ssize_t elv_iosched_show(request_queue_t *q, char *name)
783{
784 elevator_t *e = q->elevator;
785 struct elevator_type *elv = e->elevator_type;
786 struct list_head *entry;
787 int len = 0;
788
789 spin_lock_irq(q->queue_lock);
790 list_for_each(entry, &elv_list) {
791 struct elevator_type *__e;
792
793 __e = list_entry(entry, struct elevator_type, list);
794 if (!strcmp(elv->elevator_name, __e->elevator_name))
795 len += sprintf(name+len, "[%s] ", elv->elevator_name);
796 else
797 len += sprintf(name+len, "%s ", __e->elevator_name);
798 }
799 spin_unlock_irq(q->queue_lock);
800
801 len += sprintf(len+name, "\n");
802 return len;
803}
804
Tejun Heo8922e162005-10-20 16:23:44 +0200805EXPORT_SYMBOL(elv_dispatch_insert);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700806EXPORT_SYMBOL(elv_add_request);
807EXPORT_SYMBOL(__elv_add_request);
808EXPORT_SYMBOL(elv_requeue_request);
809EXPORT_SYMBOL(elv_next_request);
Tejun Heo8922e162005-10-20 16:23:44 +0200810EXPORT_SYMBOL(elv_dequeue_request);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700811EXPORT_SYMBOL(elv_queue_empty);
812EXPORT_SYMBOL(elv_completed_request);
813EXPORT_SYMBOL(elevator_exit);
814EXPORT_SYMBOL(elevator_init);