blob: 7e0fab53e930f747462e8534c4a4fc3cc82bcae8 [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
Linus Torvalds1da177e2005-04-16 15:20:36 -07002 * Copyright (C) 1991, 1992 Linus Torvalds
3 * Copyright (C) 1994, Karl Keyte: Added support for disk statistics
4 * Elevator latency, (C) 2000 Andrea Arcangeli <andrea@suse.de> SuSE
5 * Queue request tables / lock, selectable elevator, Jens Axboe <axboe@suse.de>
Jens Axboe6728cb02008-01-31 13:03:55 +01006 * kernel-doc documentation started by NeilBrown <neilb@cse.unsw.edu.au>
7 * - July2000
Linus Torvalds1da177e2005-04-16 15:20:36 -07008 * bio rewrite, highmem i/o, etc, Jens Axboe <axboe@suse.de> - may 2001
9 */
10
11/*
12 * This handles all read/write requests to block devices
13 */
Linus Torvalds1da177e2005-04-16 15:20:36 -070014#include <linux/kernel.h>
15#include <linux/module.h>
16#include <linux/backing-dev.h>
17#include <linux/bio.h>
18#include <linux/blkdev.h>
19#include <linux/highmem.h>
20#include <linux/mm.h>
21#include <linux/kernel_stat.h>
22#include <linux/string.h>
23#include <linux/init.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070024#include <linux/completion.h>
25#include <linux/slab.h>
26#include <linux/swap.h>
27#include <linux/writeback.h>
Andrew Mortonfaccbd4b2006-12-10 02:19:35 -080028#include <linux/task_io_accounting_ops.h>
Jens Axboe2056a782006-03-23 20:00:26 +010029#include <linux/blktrace_api.h>
Akinobu Mitac17bb492006-12-08 02:39:46 -080030#include <linux/fault-inject.h>
Arnaldo Carvalho de Melo5f3ea372008-10-30 08:34:33 +010031#include <trace/block.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070032
Jens Axboe8324aa92008-01-29 14:51:59 +010033#include "blk.h"
34
Ingo Molnar0bfc2452008-11-26 11:59:56 +010035DEFINE_TRACE(block_plug);
36DEFINE_TRACE(block_unplug_io);
37DEFINE_TRACE(block_unplug_timer);
38DEFINE_TRACE(block_getrq);
39DEFINE_TRACE(block_sleeprq);
40DEFINE_TRACE(block_rq_requeue);
41DEFINE_TRACE(block_bio_backmerge);
42DEFINE_TRACE(block_bio_frontmerge);
43DEFINE_TRACE(block_bio_queue);
44DEFINE_TRACE(block_rq_complete);
45DEFINE_TRACE(block_remap); /* Also used in drivers/md/dm.c */
46EXPORT_TRACEPOINT_SYMBOL_GPL(block_remap);
47
Jens Axboe165125e2007-07-24 09:28:11 +020048static int __make_request(struct request_queue *q, struct bio *bio);
Linus Torvalds1da177e2005-04-16 15:20:36 -070049
50/*
51 * For the allocated request tables
52 */
Adrian Bunk5ece6c52008-02-18 13:45:51 +010053static struct kmem_cache *request_cachep;
Linus Torvalds1da177e2005-04-16 15:20:36 -070054
55/*
56 * For queue allocation
57 */
Jens Axboe6728cb02008-01-31 13:03:55 +010058struct kmem_cache *blk_requestq_cachep;
Linus Torvalds1da177e2005-04-16 15:20:36 -070059
60/*
Linus Torvalds1da177e2005-04-16 15:20:36 -070061 * Controlling structure to kblockd
62 */
Jens Axboeff856ba2006-01-09 16:02:34 +010063static struct workqueue_struct *kblockd_workqueue;
Linus Torvalds1da177e2005-04-16 15:20:36 -070064
Jens Axboe26b82562008-01-29 13:54:41 +010065static void drive_stat_acct(struct request *rq, int new_io)
66{
Jens Axboe28f13702008-05-07 10:15:46 +020067 struct hd_struct *part;
Jens Axboe26b82562008-01-29 13:54:41 +010068 int rw = rq_data_dir(rq);
Tejun Heoc9959052008-08-25 19:47:21 +090069 int cpu;
Jens Axboe26b82562008-01-29 13:54:41 +010070
Jerome Marchand26308ea2009-03-27 10:31:51 +010071 if (!blk_fs_request(rq) || !blk_do_io_stat(rq))
Jens Axboe26b82562008-01-29 13:54:41 +010072 return;
73
Tejun Heo074a7ac2008-08-25 19:56:14 +090074 cpu = part_stat_lock();
Tejun Heoe71bf0d2008-09-03 09:03:02 +020075 part = disk_map_sector_rcu(rq->rq_disk, rq->sector);
Tejun Heoc9959052008-08-25 19:47:21 +090076
Jens Axboe28f13702008-05-07 10:15:46 +020077 if (!new_io)
Tejun Heo074a7ac2008-08-25 19:56:14 +090078 part_stat_inc(cpu, part, merges[rw]);
Jens Axboe28f13702008-05-07 10:15:46 +020079 else {
Tejun Heo074a7ac2008-08-25 19:56:14 +090080 part_round_stats(cpu, part);
81 part_inc_in_flight(part);
Jens Axboe26b82562008-01-29 13:54:41 +010082 }
Tejun Heoe71bf0d2008-09-03 09:03:02 +020083
Tejun Heo074a7ac2008-08-25 19:56:14 +090084 part_stat_unlock();
Jens Axboe26b82562008-01-29 13:54:41 +010085}
86
Jens Axboe8324aa92008-01-29 14:51:59 +010087void blk_queue_congestion_threshold(struct request_queue *q)
Linus Torvalds1da177e2005-04-16 15:20:36 -070088{
89 int nr;
90
91 nr = q->nr_requests - (q->nr_requests / 8) + 1;
92 if (nr > q->nr_requests)
93 nr = q->nr_requests;
94 q->nr_congestion_on = nr;
95
96 nr = q->nr_requests - (q->nr_requests / 8) - (q->nr_requests / 16) - 1;
97 if (nr < 1)
98 nr = 1;
99 q->nr_congestion_off = nr;
100}
101
Linus Torvalds1da177e2005-04-16 15:20:36 -0700102/**
103 * blk_get_backing_dev_info - get the address of a queue's backing_dev_info
104 * @bdev: device
105 *
106 * Locates the passed device's request queue and returns the address of its
107 * backing_dev_info
108 *
109 * Will return NULL if the request queue cannot be located.
110 */
111struct backing_dev_info *blk_get_backing_dev_info(struct block_device *bdev)
112{
113 struct backing_dev_info *ret = NULL;
Jens Axboe165125e2007-07-24 09:28:11 +0200114 struct request_queue *q = bdev_get_queue(bdev);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700115
116 if (q)
117 ret = &q->backing_dev_info;
118 return ret;
119}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700120EXPORT_SYMBOL(blk_get_backing_dev_info);
121
FUJITA Tomonori2a4aa302008-04-29 09:54:36 +0200122void blk_rq_init(struct request_queue *q, struct request *rq)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700123{
FUJITA Tomonori1afb20f2008-04-25 12:26:28 +0200124 memset(rq, 0, sizeof(*rq));
125
Linus Torvalds1da177e2005-04-16 15:20:36 -0700126 INIT_LIST_HEAD(&rq->queuelist);
Jens Axboe242f9dc2008-09-14 05:55:09 -0700127 INIT_LIST_HEAD(&rq->timeout_list);
Jens Axboec7c22e42008-09-13 20:26:01 +0200128 rq->cpu = -1;
Jens Axboe63a71382008-02-08 12:41:03 +0100129 rq->q = q;
130 rq->sector = rq->hard_sector = (sector_t) -1;
Jens Axboe2e662b62006-07-13 11:55:04 +0200131 INIT_HLIST_NODE(&rq->hash);
132 RB_CLEAR_NODE(&rq->rb_node);
FUJITA Tomonorid7e3c322008-04-29 09:54:39 +0200133 rq->cmd = rq->__cmd;
Li Zefane2494e12009-04-02 13:43:26 +0800134 rq->cmd_len = BLK_MAX_CDB;
Jens Axboe63a71382008-02-08 12:41:03 +0100135 rq->tag = -1;
Jens Axboe63a71382008-02-08 12:41:03 +0100136 rq->ref_count = 1;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700137}
FUJITA Tomonori2a4aa302008-04-29 09:54:36 +0200138EXPORT_SYMBOL(blk_rq_init);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700139
NeilBrown5bb23a62007-09-27 12:46:13 +0200140static void req_bio_endio(struct request *rq, struct bio *bio,
141 unsigned int nbytes, int error)
Tejun Heo797e7db2006-01-06 09:51:03 +0100142{
Jens Axboe165125e2007-07-24 09:28:11 +0200143 struct request_queue *q = rq->q;
Tejun Heo797e7db2006-01-06 09:51:03 +0100144
NeilBrown5bb23a62007-09-27 12:46:13 +0200145 if (&q->bar_rq != rq) {
146 if (error)
147 clear_bit(BIO_UPTODATE, &bio->bi_flags);
148 else if (!test_bit(BIO_UPTODATE, &bio->bi_flags))
149 error = -EIO;
Tejun Heo797e7db2006-01-06 09:51:03 +0100150
NeilBrown5bb23a62007-09-27 12:46:13 +0200151 if (unlikely(nbytes > bio->bi_size)) {
Jens Axboe6728cb02008-01-31 13:03:55 +0100152 printk(KERN_ERR "%s: want %u bytes done, %u left\n",
Harvey Harrison24c03d42008-05-01 04:35:17 -0700153 __func__, nbytes, bio->bi_size);
NeilBrown5bb23a62007-09-27 12:46:13 +0200154 nbytes = bio->bi_size;
155 }
Tejun Heo797e7db2006-01-06 09:51:03 +0100156
Keith Mannthey08bafc02008-11-25 10:24:35 +0100157 if (unlikely(rq->cmd_flags & REQ_QUIET))
158 set_bit(BIO_QUIET, &bio->bi_flags);
159
NeilBrown5bb23a62007-09-27 12:46:13 +0200160 bio->bi_size -= nbytes;
161 bio->bi_sector += (nbytes >> 9);
Martin K. Petersen7ba1ba12008-06-30 20:04:41 +0200162
163 if (bio_integrity(bio))
164 bio_integrity_advance(bio, nbytes);
165
NeilBrown5bb23a62007-09-27 12:46:13 +0200166 if (bio->bi_size == 0)
NeilBrown6712ecf2007-09-27 12:47:43 +0200167 bio_endio(bio, error);
NeilBrown5bb23a62007-09-27 12:46:13 +0200168 } else {
169
170 /*
171 * Okay, this is the barrier request in progress, just
172 * record the error;
173 */
174 if (error && !q->orderr)
175 q->orderr = error;
176 }
Tejun Heo797e7db2006-01-06 09:51:03 +0100177}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700178
Linus Torvalds1da177e2005-04-16 15:20:36 -0700179void blk_dump_rq_flags(struct request *rq, char *msg)
180{
181 int bit;
182
Jens Axboe6728cb02008-01-31 13:03:55 +0100183 printk(KERN_INFO "%s: dev %s: type=%x, flags=%x\n", msg,
Jens Axboe4aff5e22006-08-10 08:44:47 +0200184 rq->rq_disk ? rq->rq_disk->disk_name : "?", rq->cmd_type,
185 rq->cmd_flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700186
Jens Axboe6728cb02008-01-31 13:03:55 +0100187 printk(KERN_INFO " sector %llu, nr/cnr %lu/%u\n",
188 (unsigned long long)rq->sector,
189 rq->nr_sectors,
190 rq->current_nr_sectors);
191 printk(KERN_INFO " bio %p, biotail %p, buffer %p, data %p, len %u\n",
192 rq->bio, rq->biotail,
193 rq->buffer, rq->data,
194 rq->data_len);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700195
Jens Axboe4aff5e22006-08-10 08:44:47 +0200196 if (blk_pc_request(rq)) {
Jens Axboe6728cb02008-01-31 13:03:55 +0100197 printk(KERN_INFO " cdb: ");
FUJITA Tomonorid34c87e2008-04-29 14:37:52 +0200198 for (bit = 0; bit < BLK_MAX_CDB; bit++)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700199 printk("%02x ", rq->cmd[bit]);
200 printk("\n");
201 }
202}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700203EXPORT_SYMBOL(blk_dump_rq_flags);
204
Linus Torvalds1da177e2005-04-16 15:20:36 -0700205/*
206 * "plug" the device if there are no outstanding requests: this will
207 * force the transfer to start only after we have put all the requests
208 * on the list.
209 *
210 * This is called with interrupts off and no requests on the queue and
211 * with the queue lock held.
212 */
Jens Axboe165125e2007-07-24 09:28:11 +0200213void blk_plug_device(struct request_queue *q)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700214{
215 WARN_ON(!irqs_disabled());
216
217 /*
218 * don't plug a stopped queue, it must be paired with blk_start_queue()
219 * which will restart the queueing
220 */
Coywolf Qi Hunt7daac492006-04-19 10:14:49 +0200221 if (blk_queue_stopped(q))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700222 return;
223
Jens Axboee48ec692008-07-03 13:18:54 +0200224 if (!queue_flag_test_and_set(QUEUE_FLAG_PLUGGED, q)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700225 mod_timer(&q->unplug_timer, jiffies + q->unplug_delay);
Arnaldo Carvalho de Melo5f3ea372008-10-30 08:34:33 +0100226 trace_block_plug(q);
Jens Axboe2056a782006-03-23 20:00:26 +0100227 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700228}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700229EXPORT_SYMBOL(blk_plug_device);
230
Jens Axboe6c5e0c42008-08-01 20:31:32 +0200231/**
232 * blk_plug_device_unlocked - plug a device without queue lock held
233 * @q: The &struct request_queue to plug
234 *
235 * Description:
236 * Like @blk_plug_device(), but grabs the queue lock and disables
237 * interrupts.
238 **/
239void blk_plug_device_unlocked(struct request_queue *q)
240{
241 unsigned long flags;
242
243 spin_lock_irqsave(q->queue_lock, flags);
244 blk_plug_device(q);
245 spin_unlock_irqrestore(q->queue_lock, flags);
246}
247EXPORT_SYMBOL(blk_plug_device_unlocked);
248
Linus Torvalds1da177e2005-04-16 15:20:36 -0700249/*
250 * remove the queue from the plugged list, if present. called with
251 * queue lock held and interrupts disabled.
252 */
Jens Axboe165125e2007-07-24 09:28:11 +0200253int blk_remove_plug(struct request_queue *q)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700254{
255 WARN_ON(!irqs_disabled());
256
Jens Axboee48ec692008-07-03 13:18:54 +0200257 if (!queue_flag_test_and_clear(QUEUE_FLAG_PLUGGED, q))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700258 return 0;
259
260 del_timer(&q->unplug_timer);
261 return 1;
262}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700263EXPORT_SYMBOL(blk_remove_plug);
264
265/*
266 * remove the plug and let it rip..
267 */
Jens Axboe165125e2007-07-24 09:28:11 +0200268void __generic_unplug_device(struct request_queue *q)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700269{
Coywolf Qi Hunt7daac492006-04-19 10:14:49 +0200270 if (unlikely(blk_queue_stopped(q)))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700271 return;
Jens Axboea31a9732008-10-17 13:58:29 +0200272 if (!blk_remove_plug(q) && !blk_queue_nonrot(q))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700273 return;
274
Jens Axboe22e2c502005-06-27 10:55:12 +0200275 q->request_fn(q);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700276}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700277
278/**
279 * generic_unplug_device - fire a request queue
Jens Axboe165125e2007-07-24 09:28:11 +0200280 * @q: The &struct request_queue in question
Linus Torvalds1da177e2005-04-16 15:20:36 -0700281 *
282 * Description:
283 * Linux uses plugging to build bigger requests queues before letting
284 * the device have at them. If a queue is plugged, the I/O scheduler
285 * is still adding and merging requests on the queue. Once the queue
286 * gets unplugged, the request_fn defined for the queue is invoked and
287 * transfers started.
288 **/
Jens Axboe165125e2007-07-24 09:28:11 +0200289void generic_unplug_device(struct request_queue *q)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700290{
Jens Axboedbaf2c02008-05-07 09:48:17 +0200291 if (blk_queue_plugged(q)) {
292 spin_lock_irq(q->queue_lock);
293 __generic_unplug_device(q);
294 spin_unlock_irq(q->queue_lock);
295 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700296}
297EXPORT_SYMBOL(generic_unplug_device);
298
299static void blk_backing_dev_unplug(struct backing_dev_info *bdi,
300 struct page *page)
301{
Jens Axboe165125e2007-07-24 09:28:11 +0200302 struct request_queue *q = bdi->unplug_io_data;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700303
Alan D. Brunelle2ad8b1e2007-11-07 14:26:56 -0500304 blk_unplug(q);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700305}
306
Jens Axboe86db1e22008-01-29 14:53:40 +0100307void blk_unplug_work(struct work_struct *work)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700308{
Jens Axboe165125e2007-07-24 09:28:11 +0200309 struct request_queue *q =
310 container_of(work, struct request_queue, unplug_work);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700311
Arnaldo Carvalho de Melo5f3ea372008-10-30 08:34:33 +0100312 trace_block_unplug_io(q);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700313 q->unplug_fn(q);
314}
315
Jens Axboe86db1e22008-01-29 14:53:40 +0100316void blk_unplug_timeout(unsigned long data)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700317{
Jens Axboe165125e2007-07-24 09:28:11 +0200318 struct request_queue *q = (struct request_queue *)data;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700319
Arnaldo Carvalho de Melo5f3ea372008-10-30 08:34:33 +0100320 trace_block_unplug_timer(q);
Jens Axboe18887ad2008-07-28 13:08:45 +0200321 kblockd_schedule_work(q, &q->unplug_work);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700322}
323
Alan D. Brunelle2ad8b1e2007-11-07 14:26:56 -0500324void blk_unplug(struct request_queue *q)
325{
326 /*
327 * devices don't necessarily have an ->unplug_fn defined
328 */
329 if (q->unplug_fn) {
Arnaldo Carvalho de Melo5f3ea372008-10-30 08:34:33 +0100330 trace_block_unplug_io(q);
Alan D. Brunelle2ad8b1e2007-11-07 14:26:56 -0500331 q->unplug_fn(q);
332 }
333}
334EXPORT_SYMBOL(blk_unplug);
335
Linus Torvalds1da177e2005-04-16 15:20:36 -0700336/**
337 * blk_start_queue - restart a previously stopped queue
Jens Axboe165125e2007-07-24 09:28:11 +0200338 * @q: The &struct request_queue in question
Linus Torvalds1da177e2005-04-16 15:20:36 -0700339 *
340 * Description:
341 * blk_start_queue() will clear the stop flag on the queue, and call
342 * the request_fn for the queue if it was in a stopped state when
343 * entered. Also see blk_stop_queue(). Queue lock must be held.
344 **/
Jens Axboe165125e2007-07-24 09:28:11 +0200345void blk_start_queue(struct request_queue *q)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700346{
Paolo 'Blaisorblade' Giarrussoa038e252006-06-05 12:09:01 +0200347 WARN_ON(!irqs_disabled());
348
Nick Piggin75ad23b2008-04-29 14:48:33 +0200349 queue_flag_clear(QUEUE_FLAG_STOPPED, q);
Tejun Heoa538cd02009-04-23 11:05:17 +0900350 __blk_run_queue(q);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700351}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700352EXPORT_SYMBOL(blk_start_queue);
353
354/**
355 * blk_stop_queue - stop a queue
Jens Axboe165125e2007-07-24 09:28:11 +0200356 * @q: The &struct request_queue in question
Linus Torvalds1da177e2005-04-16 15:20:36 -0700357 *
358 * Description:
359 * The Linux block layer assumes that a block driver will consume all
360 * entries on the request queue when the request_fn strategy is called.
361 * Often this will not happen, because of hardware limitations (queue
362 * depth settings). If a device driver gets a 'queue full' response,
363 * or if it simply chooses not to queue more I/O at one point, it can
364 * call this function to prevent the request_fn from being called until
365 * the driver has signalled it's ready to go again. This happens by calling
366 * blk_start_queue() to restart queue operations. Queue lock must be held.
367 **/
Jens Axboe165125e2007-07-24 09:28:11 +0200368void blk_stop_queue(struct request_queue *q)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700369{
370 blk_remove_plug(q);
Nick Piggin75ad23b2008-04-29 14:48:33 +0200371 queue_flag_set(QUEUE_FLAG_STOPPED, q);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700372}
373EXPORT_SYMBOL(blk_stop_queue);
374
375/**
376 * blk_sync_queue - cancel any pending callbacks on a queue
377 * @q: the queue
378 *
379 * Description:
380 * The block layer may perform asynchronous callback activity
381 * on a queue, such as calling the unplug function after a timeout.
382 * A block device may call blk_sync_queue to ensure that any
383 * such activity is cancelled, thus allowing it to release resources
Michael Opdenacker59c51592007-05-09 08:57:56 +0200384 * that the callbacks might use. The caller must already have made sure
Linus Torvalds1da177e2005-04-16 15:20:36 -0700385 * that its ->make_request_fn will not re-add plugging prior to calling
386 * this function.
387 *
388 */
389void blk_sync_queue(struct request_queue *q)
390{
391 del_timer_sync(&q->unplug_timer);
Jens Axboe70ed28b2008-11-19 14:38:39 +0100392 del_timer_sync(&q->timeout);
Cheng Renquan64d01dc2008-12-03 12:41:39 +0100393 cancel_work_sync(&q->unplug_work);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700394}
395EXPORT_SYMBOL(blk_sync_queue);
396
397/**
Jens Axboe80a4b582008-10-14 09:51:06 +0200398 * __blk_run_queue - run a single device queue
Linus Torvalds1da177e2005-04-16 15:20:36 -0700399 * @q: The queue to run
Jens Axboe80a4b582008-10-14 09:51:06 +0200400 *
401 * Description:
402 * See @blk_run_queue. This variant must be called with the queue lock
403 * held and interrupts disabled.
404 *
Linus Torvalds1da177e2005-04-16 15:20:36 -0700405 */
Nick Piggin75ad23b2008-04-29 14:48:33 +0200406void __blk_run_queue(struct request_queue *q)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700407{
Linus Torvalds1da177e2005-04-16 15:20:36 -0700408 blk_remove_plug(q);
Jens Axboedac07ec2006-05-11 08:20:16 +0200409
Tejun Heoa538cd02009-04-23 11:05:17 +0900410 if (unlikely(blk_queue_stopped(q)))
411 return;
412
413 if (elv_queue_empty(q))
414 return;
415
Jens Axboedac07ec2006-05-11 08:20:16 +0200416 /*
417 * Only recurse once to avoid overrunning the stack, let the unplug
418 * handling reinvoke the handler shortly if we already got there.
419 */
Tejun Heoa538cd02009-04-23 11:05:17 +0900420 if (!queue_flag_test_and_set(QUEUE_FLAG_REENTER, q)) {
421 q->request_fn(q);
422 queue_flag_clear(QUEUE_FLAG_REENTER, q);
423 } else {
424 queue_flag_set(QUEUE_FLAG_PLUGGED, q);
425 kblockd_schedule_work(q, &q->unplug_work);
426 }
Nick Piggin75ad23b2008-04-29 14:48:33 +0200427}
428EXPORT_SYMBOL(__blk_run_queue);
Jens Axboedac07ec2006-05-11 08:20:16 +0200429
Nick Piggin75ad23b2008-04-29 14:48:33 +0200430/**
431 * blk_run_queue - run a single device queue
432 * @q: The queue to run
Jens Axboe80a4b582008-10-14 09:51:06 +0200433 *
434 * Description:
435 * Invoke request handling on this queue, if it has pending work to do.
Tejun Heoa7f55792009-04-23 11:05:17 +0900436 * May be used to restart queueing when a request has completed.
Nick Piggin75ad23b2008-04-29 14:48:33 +0200437 */
438void blk_run_queue(struct request_queue *q)
439{
440 unsigned long flags;
441
442 spin_lock_irqsave(q->queue_lock, flags);
443 __blk_run_queue(q);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700444 spin_unlock_irqrestore(q->queue_lock, flags);
445}
446EXPORT_SYMBOL(blk_run_queue);
447
Jens Axboe165125e2007-07-24 09:28:11 +0200448void blk_put_queue(struct request_queue *q)
Al Viro483f4af2006-03-18 18:34:37 -0500449{
450 kobject_put(&q->kobj);
451}
Al Viro483f4af2006-03-18 18:34:37 -0500452
Jens Axboe6728cb02008-01-31 13:03:55 +0100453void blk_cleanup_queue(struct request_queue *q)
Al Viro483f4af2006-03-18 18:34:37 -0500454{
Jens Axboee3335de92008-09-18 09:22:54 -0700455 /*
456 * We know we have process context here, so we can be a little
457 * cautious and ensure that pending block actions on this device
458 * are done before moving on. Going into this function, we should
459 * not have processes doing IO to this device.
460 */
461 blk_sync_queue(q);
462
Al Viro483f4af2006-03-18 18:34:37 -0500463 mutex_lock(&q->sysfs_lock);
Nick Piggin75ad23b2008-04-29 14:48:33 +0200464 queue_flag_set_unlocked(QUEUE_FLAG_DEAD, q);
Al Viro483f4af2006-03-18 18:34:37 -0500465 mutex_unlock(&q->sysfs_lock);
466
467 if (q->elevator)
468 elevator_exit(q->elevator);
469
470 blk_put_queue(q);
471}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700472EXPORT_SYMBOL(blk_cleanup_queue);
473
Jens Axboe165125e2007-07-24 09:28:11 +0200474static int blk_init_free_list(struct request_queue *q)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700475{
476 struct request_list *rl = &q->rq;
477
Jens Axboe1faa16d2009-04-06 14:48:01 +0200478 rl->count[BLK_RW_SYNC] = rl->count[BLK_RW_ASYNC] = 0;
479 rl->starved[BLK_RW_SYNC] = rl->starved[BLK_RW_ASYNC] = 0;
Tejun Heocb98fc82005-10-28 08:29:39 +0200480 rl->elvpriv = 0;
Jens Axboe1faa16d2009-04-06 14:48:01 +0200481 init_waitqueue_head(&rl->wait[BLK_RW_SYNC]);
482 init_waitqueue_head(&rl->wait[BLK_RW_ASYNC]);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700483
Christoph Lameter19460892005-06-23 00:08:19 -0700484 rl->rq_pool = mempool_create_node(BLKDEV_MIN_RQ, mempool_alloc_slab,
485 mempool_free_slab, request_cachep, q->node);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700486
487 if (!rl->rq_pool)
488 return -ENOMEM;
489
490 return 0;
491}
492
Jens Axboe165125e2007-07-24 09:28:11 +0200493struct request_queue *blk_alloc_queue(gfp_t gfp_mask)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700494{
Christoph Lameter19460892005-06-23 00:08:19 -0700495 return blk_alloc_queue_node(gfp_mask, -1);
496}
497EXPORT_SYMBOL(blk_alloc_queue);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700498
Jens Axboe165125e2007-07-24 09:28:11 +0200499struct request_queue *blk_alloc_queue_node(gfp_t gfp_mask, int node_id)
Christoph Lameter19460892005-06-23 00:08:19 -0700500{
Jens Axboe165125e2007-07-24 09:28:11 +0200501 struct request_queue *q;
Peter Zijlstrae0bf68d2007-10-16 23:25:46 -0700502 int err;
Christoph Lameter19460892005-06-23 00:08:19 -0700503
Jens Axboe8324aa92008-01-29 14:51:59 +0100504 q = kmem_cache_alloc_node(blk_requestq_cachep,
Christoph Lameter94f60302007-07-17 04:03:29 -0700505 gfp_mask | __GFP_ZERO, node_id);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700506 if (!q)
507 return NULL;
508
Peter Zijlstrae0bf68d2007-10-16 23:25:46 -0700509 q->backing_dev_info.unplug_io_fn = blk_backing_dev_unplug;
510 q->backing_dev_info.unplug_io_data = q;
511 err = bdi_init(&q->backing_dev_info);
512 if (err) {
Jens Axboe8324aa92008-01-29 14:51:59 +0100513 kmem_cache_free(blk_requestq_cachep, q);
Peter Zijlstrae0bf68d2007-10-16 23:25:46 -0700514 return NULL;
515 }
516
Linus Torvalds1da177e2005-04-16 15:20:36 -0700517 init_timer(&q->unplug_timer);
Jens Axboe242f9dc2008-09-14 05:55:09 -0700518 setup_timer(&q->timeout, blk_rq_timed_out_timer, (unsigned long) q);
519 INIT_LIST_HEAD(&q->timeout_list);
Peter Zijlstra713ada92008-10-16 13:44:57 +0200520 INIT_WORK(&q->unplug_work, blk_unplug_work);
Al Viro483f4af2006-03-18 18:34:37 -0500521
Jens Axboe8324aa92008-01-29 14:51:59 +0100522 kobject_init(&q->kobj, &blk_queue_ktype);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700523
Al Viro483f4af2006-03-18 18:34:37 -0500524 mutex_init(&q->sysfs_lock);
Neil Browne7e72bf2008-05-14 16:05:54 -0700525 spin_lock_init(&q->__queue_lock);
Al Viro483f4af2006-03-18 18:34:37 -0500526
Linus Torvalds1da177e2005-04-16 15:20:36 -0700527 return q;
528}
Christoph Lameter19460892005-06-23 00:08:19 -0700529EXPORT_SYMBOL(blk_alloc_queue_node);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700530
531/**
532 * blk_init_queue - prepare a request queue for use with a block device
533 * @rfn: The function to be called to process requests that have been
534 * placed on the queue.
535 * @lock: Request queue spin lock
536 *
537 * Description:
538 * If a block device wishes to use the standard request handling procedures,
539 * which sorts requests and coalesces adjacent requests, then it must
540 * call blk_init_queue(). The function @rfn will be called when there
541 * are requests on the queue that need to be processed. If the device
542 * supports plugging, then @rfn may not be called immediately when requests
543 * are available on the queue, but may be called at some time later instead.
544 * Plugged queues are generally unplugged when a buffer belonging to one
545 * of the requests on the queue is needed, or due to memory pressure.
546 *
547 * @rfn is not required, or even expected, to remove all requests off the
548 * queue, but only as many as it can handle at a time. If it does leave
549 * requests on the queue, it is responsible for arranging that the requests
550 * get dealt with eventually.
551 *
552 * The queue spin lock must be held while manipulating the requests on the
Paolo 'Blaisorblade' Giarrussoa038e252006-06-05 12:09:01 +0200553 * request queue; this lock will be taken also from interrupt context, so irq
554 * disabling is needed for it.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700555 *
Randy Dunlap710027a2008-08-19 20:13:11 +0200556 * Function returns a pointer to the initialized request queue, or %NULL if
Linus Torvalds1da177e2005-04-16 15:20:36 -0700557 * it didn't succeed.
558 *
559 * Note:
560 * blk_init_queue() must be paired with a blk_cleanup_queue() call
561 * when the block device is deactivated (such as at module unload).
562 **/
Christoph Lameter19460892005-06-23 00:08:19 -0700563
Jens Axboe165125e2007-07-24 09:28:11 +0200564struct request_queue *blk_init_queue(request_fn_proc *rfn, spinlock_t *lock)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700565{
Christoph Lameter19460892005-06-23 00:08:19 -0700566 return blk_init_queue_node(rfn, lock, -1);
567}
568EXPORT_SYMBOL(blk_init_queue);
569
Jens Axboe165125e2007-07-24 09:28:11 +0200570struct request_queue *
Christoph Lameter19460892005-06-23 00:08:19 -0700571blk_init_queue_node(request_fn_proc *rfn, spinlock_t *lock, int node_id)
572{
Jens Axboe165125e2007-07-24 09:28:11 +0200573 struct request_queue *q = blk_alloc_queue_node(GFP_KERNEL, node_id);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700574
575 if (!q)
576 return NULL;
577
Christoph Lameter19460892005-06-23 00:08:19 -0700578 q->node = node_id;
Al Viro8669aaf2006-03-18 13:50:00 -0500579 if (blk_init_free_list(q)) {
Jens Axboe8324aa92008-01-29 14:51:59 +0100580 kmem_cache_free(blk_requestq_cachep, q);
Al Viro8669aaf2006-03-18 13:50:00 -0500581 return NULL;
582 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700583
152587d2005-04-12 16:22:06 -0500584 /*
585 * if caller didn't supply a lock, they get per-queue locking with
586 * our embedded lock
587 */
Neil Browne7e72bf2008-05-14 16:05:54 -0700588 if (!lock)
152587d2005-04-12 16:22:06 -0500589 lock = &q->__queue_lock;
152587d2005-04-12 16:22:06 -0500590
Linus Torvalds1da177e2005-04-16 15:20:36 -0700591 q->request_fn = rfn;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700592 q->prep_rq_fn = NULL;
593 q->unplug_fn = generic_unplug_device;
Jens Axboebc58ba92009-01-23 10:54:44 +0100594 q->queue_flags = QUEUE_FLAG_DEFAULT;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700595 q->queue_lock = lock;
596
Jens Axboef3b144a2009-03-06 08:48:33 +0100597 /*
598 * This also sets hw/phys segments, boundary and size
599 */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700600 blk_queue_make_request(q, __make_request);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700601
Alan Stern44ec9542007-02-20 11:01:57 -0500602 q->sg_reserved_size = INT_MAX;
603
FUJITA Tomonoriabf54392008-08-16 14:10:05 +0900604 blk_set_cmd_filter_defaults(&q->cmd_filter);
605
Linus Torvalds1da177e2005-04-16 15:20:36 -0700606 /*
607 * all done
608 */
609 if (!elevator_init(q, NULL)) {
610 blk_queue_congestion_threshold(q);
611 return q;
612 }
613
Al Viro8669aaf2006-03-18 13:50:00 -0500614 blk_put_queue(q);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700615 return NULL;
616}
Christoph Lameter19460892005-06-23 00:08:19 -0700617EXPORT_SYMBOL(blk_init_queue_node);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700618
Jens Axboe165125e2007-07-24 09:28:11 +0200619int blk_get_queue(struct request_queue *q)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700620{
Nick Pigginfde6ad22005-06-23 00:08:53 -0700621 if (likely(!test_bit(QUEUE_FLAG_DEAD, &q->queue_flags))) {
Al Viro483f4af2006-03-18 18:34:37 -0500622 kobject_get(&q->kobj);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700623 return 0;
624 }
625
626 return 1;
627}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700628
Jens Axboe165125e2007-07-24 09:28:11 +0200629static inline void blk_free_request(struct request_queue *q, struct request *rq)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700630{
Jens Axboe4aff5e22006-08-10 08:44:47 +0200631 if (rq->cmd_flags & REQ_ELVPRIV)
Tejun Heocb98fc82005-10-28 08:29:39 +0200632 elv_put_request(q, rq);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700633 mempool_free(rq, q->rq.rq_pool);
634}
635
Jens Axboe1ea25ecb2006-07-18 22:24:11 +0200636static struct request *
Jerome Marchand42dad762009-04-22 14:01:49 +0200637blk_alloc_request(struct request_queue *q, int flags, int priv, gfp_t gfp_mask)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700638{
639 struct request *rq = mempool_alloc(q->rq.rq_pool, gfp_mask);
640
641 if (!rq)
642 return NULL;
643
FUJITA Tomonori2a4aa302008-04-29 09:54:36 +0200644 blk_rq_init(q, rq);
FUJITA Tomonori1afb20f2008-04-25 12:26:28 +0200645
Jerome Marchand42dad762009-04-22 14:01:49 +0200646 rq->cmd_flags = flags | REQ_ALLOCED;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700647
Tejun Heocb98fc82005-10-28 08:29:39 +0200648 if (priv) {
Jens Axboecb78b282006-07-28 09:32:57 +0200649 if (unlikely(elv_set_request(q, rq, gfp_mask))) {
Tejun Heocb98fc82005-10-28 08:29:39 +0200650 mempool_free(rq, q->rq.rq_pool);
651 return NULL;
652 }
Jens Axboe4aff5e22006-08-10 08:44:47 +0200653 rq->cmd_flags |= REQ_ELVPRIV;
Tejun Heocb98fc82005-10-28 08:29:39 +0200654 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700655
Tejun Heocb98fc82005-10-28 08:29:39 +0200656 return rq;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700657}
658
659/*
660 * ioc_batching returns true if the ioc is a valid batching request and
661 * should be given priority access to a request.
662 */
Jens Axboe165125e2007-07-24 09:28:11 +0200663static inline int ioc_batching(struct request_queue *q, struct io_context *ioc)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700664{
665 if (!ioc)
666 return 0;
667
668 /*
669 * Make sure the process is able to allocate at least 1 request
670 * even if the batch times out, otherwise we could theoretically
671 * lose wakeups.
672 */
673 return ioc->nr_batch_requests == q->nr_batching ||
674 (ioc->nr_batch_requests > 0
675 && time_before(jiffies, ioc->last_waited + BLK_BATCH_TIME));
676}
677
678/*
679 * ioc_set_batching sets ioc to be a new "batcher" if it is not one. This
680 * will cause the process to be a "batcher" on all queues in the system. This
681 * is the behaviour we want though - once it gets a wakeup it should be given
682 * a nice run.
683 */
Jens Axboe165125e2007-07-24 09:28:11 +0200684static void ioc_set_batching(struct request_queue *q, struct io_context *ioc)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700685{
686 if (!ioc || ioc_batching(q, ioc))
687 return;
688
689 ioc->nr_batch_requests = q->nr_batching;
690 ioc->last_waited = jiffies;
691}
692
Jens Axboe1faa16d2009-04-06 14:48:01 +0200693static void __freed_request(struct request_queue *q, int sync)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700694{
695 struct request_list *rl = &q->rq;
696
Jens Axboe1faa16d2009-04-06 14:48:01 +0200697 if (rl->count[sync] < queue_congestion_off_threshold(q))
698 blk_clear_queue_congested(q, sync);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700699
Jens Axboe1faa16d2009-04-06 14:48:01 +0200700 if (rl->count[sync] + 1 <= q->nr_requests) {
701 if (waitqueue_active(&rl->wait[sync]))
702 wake_up(&rl->wait[sync]);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700703
Jens Axboe1faa16d2009-04-06 14:48:01 +0200704 blk_clear_queue_full(q, sync);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700705 }
706}
707
708/*
709 * A request has just been released. Account for it, update the full and
710 * congestion status, wake up any waiters. Called under q->queue_lock.
711 */
Jens Axboe1faa16d2009-04-06 14:48:01 +0200712static void freed_request(struct request_queue *q, int sync, int priv)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700713{
714 struct request_list *rl = &q->rq;
715
Jens Axboe1faa16d2009-04-06 14:48:01 +0200716 rl->count[sync]--;
Tejun Heocb98fc82005-10-28 08:29:39 +0200717 if (priv)
718 rl->elvpriv--;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700719
Jens Axboe1faa16d2009-04-06 14:48:01 +0200720 __freed_request(q, sync);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700721
Jens Axboe1faa16d2009-04-06 14:48:01 +0200722 if (unlikely(rl->starved[sync ^ 1]))
723 __freed_request(q, sync ^ 1);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700724}
725
Linus Torvalds1da177e2005-04-16 15:20:36 -0700726/*
Nick Piggind6344532005-06-28 20:45:14 -0700727 * Get a free request, queue_lock must be held.
728 * Returns NULL on failure, with queue_lock held.
729 * Returns !NULL on success, with queue_lock *not held*.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700730 */
Jens Axboe165125e2007-07-24 09:28:11 +0200731static struct request *get_request(struct request_queue *q, int rw_flags,
Jens Axboe7749a8d2006-12-13 13:02:26 +0100732 struct bio *bio, gfp_t gfp_mask)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700733{
734 struct request *rq = NULL;
735 struct request_list *rl = &q->rq;
Jens Axboe88ee5ef2005-11-12 11:09:12 +0100736 struct io_context *ioc = NULL;
Jens Axboe1faa16d2009-04-06 14:48:01 +0200737 const bool is_sync = rw_is_sync(rw_flags) != 0;
Jens Axboe88ee5ef2005-11-12 11:09:12 +0100738 int may_queue, priv;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700739
Jens Axboe7749a8d2006-12-13 13:02:26 +0100740 may_queue = elv_may_queue(q, rw_flags);
Jens Axboe88ee5ef2005-11-12 11:09:12 +0100741 if (may_queue == ELV_MQUEUE_NO)
742 goto rq_starved;
743
Jens Axboe1faa16d2009-04-06 14:48:01 +0200744 if (rl->count[is_sync]+1 >= queue_congestion_on_threshold(q)) {
745 if (rl->count[is_sync]+1 >= q->nr_requests) {
Jens Axboeb5deef92006-07-19 23:39:40 +0200746 ioc = current_io_context(GFP_ATOMIC, q->node);
Jens Axboe88ee5ef2005-11-12 11:09:12 +0100747 /*
748 * The queue will fill after this allocation, so set
749 * it as full, and mark this process as "batching".
750 * This process will be allowed to complete a batch of
751 * requests, others will be blocked.
752 */
Jens Axboe1faa16d2009-04-06 14:48:01 +0200753 if (!blk_queue_full(q, is_sync)) {
Jens Axboe88ee5ef2005-11-12 11:09:12 +0100754 ioc_set_batching(q, ioc);
Jens Axboe1faa16d2009-04-06 14:48:01 +0200755 blk_set_queue_full(q, is_sync);
Jens Axboe88ee5ef2005-11-12 11:09:12 +0100756 } else {
757 if (may_queue != ELV_MQUEUE_MUST
758 && !ioc_batching(q, ioc)) {
759 /*
760 * The queue is full and the allocating
761 * process is not a "batcher", and not
762 * exempted by the IO scheduler
763 */
764 goto out;
765 }
766 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700767 }
Jens Axboe1faa16d2009-04-06 14:48:01 +0200768 blk_set_queue_congested(q, is_sync);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700769 }
770
Jens Axboe082cf692005-06-28 16:35:11 +0200771 /*
772 * Only allow batching queuers to allocate up to 50% over the defined
773 * limit of requests, otherwise we could have thousands of requests
774 * allocated with any setting of ->nr_requests
775 */
Jens Axboe1faa16d2009-04-06 14:48:01 +0200776 if (rl->count[is_sync] >= (3 * q->nr_requests / 2))
Jens Axboe082cf692005-06-28 16:35:11 +0200777 goto out;
Hugh Dickinsfd782a42005-06-29 15:15:40 +0100778
Jens Axboe1faa16d2009-04-06 14:48:01 +0200779 rl->count[is_sync]++;
780 rl->starved[is_sync] = 0;
Tejun Heocb98fc82005-10-28 08:29:39 +0200781
Jens Axboe64521d12005-10-28 08:30:39 +0200782 priv = !test_bit(QUEUE_FLAG_ELVSWITCH, &q->queue_flags);
Tejun Heocb98fc82005-10-28 08:29:39 +0200783 if (priv)
784 rl->elvpriv++;
785
Jerome Marchand42dad762009-04-22 14:01:49 +0200786 if (blk_queue_io_stat(q))
787 rw_flags |= REQ_IO_STAT;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700788 spin_unlock_irq(q->queue_lock);
789
Jens Axboe7749a8d2006-12-13 13:02:26 +0100790 rq = blk_alloc_request(q, rw_flags, priv, gfp_mask);
Jens Axboe88ee5ef2005-11-12 11:09:12 +0100791 if (unlikely(!rq)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700792 /*
793 * Allocation failed presumably due to memory. Undo anything
794 * we might have messed up.
795 *
796 * Allocating task should really be put onto the front of the
797 * wait queue, but this is pretty rare.
798 */
799 spin_lock_irq(q->queue_lock);
Jens Axboe1faa16d2009-04-06 14:48:01 +0200800 freed_request(q, is_sync, priv);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700801
802 /*
803 * in the very unlikely event that allocation failed and no
804 * requests for this direction was pending, mark us starved
805 * so that freeing of a request in the other direction will
806 * notice us. another possible fix would be to split the
807 * rq mempool into READ and WRITE
808 */
809rq_starved:
Jens Axboe1faa16d2009-04-06 14:48:01 +0200810 if (unlikely(rl->count[is_sync] == 0))
811 rl->starved[is_sync] = 1;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700812
Linus Torvalds1da177e2005-04-16 15:20:36 -0700813 goto out;
814 }
815
Jens Axboe88ee5ef2005-11-12 11:09:12 +0100816 /*
817 * ioc may be NULL here, and ioc_batching will be false. That's
818 * OK, if the queue is under the request limit then requests need
819 * not count toward the nr_batch_requests limit. There will always
820 * be some limit enforced by BLK_BATCH_TIME.
821 */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700822 if (ioc_batching(q, ioc))
823 ioc->nr_batch_requests--;
Jens Axboe6728cb02008-01-31 13:03:55 +0100824
Jens Axboe1faa16d2009-04-06 14:48:01 +0200825 trace_block_getrq(q, bio, rw_flags & 1);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700826out:
Linus Torvalds1da177e2005-04-16 15:20:36 -0700827 return rq;
828}
829
830/*
831 * No available requests for this queue, unplug the device and wait for some
832 * requests to become available.
Nick Piggind6344532005-06-28 20:45:14 -0700833 *
834 * Called with q->queue_lock held, and returns with it unlocked.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700835 */
Jens Axboe165125e2007-07-24 09:28:11 +0200836static struct request *get_request_wait(struct request_queue *q, int rw_flags,
Jens Axboe22e2c502005-06-27 10:55:12 +0200837 struct bio *bio)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700838{
Jens Axboe1faa16d2009-04-06 14:48:01 +0200839 const bool is_sync = rw_is_sync(rw_flags) != 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700840 struct request *rq;
841
Jens Axboe7749a8d2006-12-13 13:02:26 +0100842 rq = get_request(q, rw_flags, bio, GFP_NOIO);
Nick Piggin450991b2005-06-28 20:45:13 -0700843 while (!rq) {
844 DEFINE_WAIT(wait);
Zhang, Yanmin05caf8d2008-05-22 15:13:29 +0200845 struct io_context *ioc;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700846 struct request_list *rl = &q->rq;
847
Jens Axboe1faa16d2009-04-06 14:48:01 +0200848 prepare_to_wait_exclusive(&rl->wait[is_sync], &wait,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700849 TASK_UNINTERRUPTIBLE);
850
Jens Axboe1faa16d2009-04-06 14:48:01 +0200851 trace_block_sleeprq(q, bio, rw_flags & 1);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700852
Zhang, Yanmin05caf8d2008-05-22 15:13:29 +0200853 __generic_unplug_device(q);
854 spin_unlock_irq(q->queue_lock);
855 io_schedule();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700856
Zhang, Yanmin05caf8d2008-05-22 15:13:29 +0200857 /*
858 * After sleeping, we become a "batching" process and
859 * will be able to allocate at least one request, and
860 * up to a big batch of them for a small period time.
861 * See ioc_batching, ioc_set_batching
862 */
863 ioc = current_io_context(GFP_NOIO, q->node);
864 ioc_set_batching(q, ioc);
Jens Axboe2056a782006-03-23 20:00:26 +0100865
Zhang, Yanmin05caf8d2008-05-22 15:13:29 +0200866 spin_lock_irq(q->queue_lock);
Jens Axboe1faa16d2009-04-06 14:48:01 +0200867 finish_wait(&rl->wait[is_sync], &wait);
Zhang, Yanmin05caf8d2008-05-22 15:13:29 +0200868
869 rq = get_request(q, rw_flags, bio, GFP_NOIO);
870 };
Linus Torvalds1da177e2005-04-16 15:20:36 -0700871
872 return rq;
873}
874
Jens Axboe165125e2007-07-24 09:28:11 +0200875struct request *blk_get_request(struct request_queue *q, int rw, gfp_t gfp_mask)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700876{
877 struct request *rq;
878
879 BUG_ON(rw != READ && rw != WRITE);
880
Nick Piggind6344532005-06-28 20:45:14 -0700881 spin_lock_irq(q->queue_lock);
882 if (gfp_mask & __GFP_WAIT) {
Jens Axboe22e2c502005-06-27 10:55:12 +0200883 rq = get_request_wait(q, rw, NULL);
Nick Piggind6344532005-06-28 20:45:14 -0700884 } else {
Jens Axboe22e2c502005-06-27 10:55:12 +0200885 rq = get_request(q, rw, NULL, gfp_mask);
Nick Piggind6344532005-06-28 20:45:14 -0700886 if (!rq)
887 spin_unlock_irq(q->queue_lock);
888 }
889 /* q->queue_lock is unlocked at this point */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700890
891 return rq;
892}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700893EXPORT_SYMBOL(blk_get_request);
894
895/**
896 * blk_requeue_request - put a request back on queue
897 * @q: request queue where request should be inserted
898 * @rq: request to be inserted
899 *
900 * Description:
901 * Drivers often keep queueing requests until the hardware cannot accept
902 * more, when that condition happens we need to put the request back
903 * on the queue. Must be called with queue lock held.
904 */
Jens Axboe165125e2007-07-24 09:28:11 +0200905void blk_requeue_request(struct request_queue *q, struct request *rq)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700906{
Jens Axboe242f9dc2008-09-14 05:55:09 -0700907 blk_delete_timer(rq);
908 blk_clear_rq_complete(rq);
Arnaldo Carvalho de Melo5f3ea372008-10-30 08:34:33 +0100909 trace_block_rq_requeue(q, rq);
Jens Axboe2056a782006-03-23 20:00:26 +0100910
Linus Torvalds1da177e2005-04-16 15:20:36 -0700911 if (blk_rq_tagged(rq))
912 blk_queue_end_tag(q, rq);
913
914 elv_requeue_request(q, rq);
915}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700916EXPORT_SYMBOL(blk_requeue_request);
917
918/**
Randy Dunlap710027a2008-08-19 20:13:11 +0200919 * blk_insert_request - insert a special request into a request queue
Linus Torvalds1da177e2005-04-16 15:20:36 -0700920 * @q: request queue where request should be inserted
921 * @rq: request to be inserted
922 * @at_head: insert request at head or tail of queue
923 * @data: private data
Linus Torvalds1da177e2005-04-16 15:20:36 -0700924 *
925 * Description:
926 * Many block devices need to execute commands asynchronously, so they don't
927 * block the whole kernel from preemption during request execution. This is
928 * accomplished normally by inserting aritficial requests tagged as
Randy Dunlap710027a2008-08-19 20:13:11 +0200929 * REQ_TYPE_SPECIAL in to the corresponding request queue, and letting them
930 * be scheduled for actual execution by the request queue.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700931 *
932 * We have the option of inserting the head or the tail of the queue.
933 * Typically we use the tail for new ioctls and so forth. We use the head
934 * of the queue for things like a QUEUE_FULL message from a device, or a
935 * host that is unable to accept a particular command.
936 */
Jens Axboe165125e2007-07-24 09:28:11 +0200937void blk_insert_request(struct request_queue *q, struct request *rq,
Tejun Heo 867d1192005-04-24 02:06:05 -0500938 int at_head, void *data)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700939{
Tejun Heo 867d1192005-04-24 02:06:05 -0500940 int where = at_head ? ELEVATOR_INSERT_FRONT : ELEVATOR_INSERT_BACK;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700941 unsigned long flags;
942
943 /*
944 * tell I/O scheduler that this isn't a regular read/write (ie it
945 * must not attempt merges on this) and that it acts as a soft
946 * barrier
947 */
Jens Axboe4aff5e22006-08-10 08:44:47 +0200948 rq->cmd_type = REQ_TYPE_SPECIAL;
949 rq->cmd_flags |= REQ_SOFTBARRIER;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700950
951 rq->special = data;
952
953 spin_lock_irqsave(q->queue_lock, flags);
954
955 /*
956 * If command is tagged, release the tag
957 */
Tejun Heo 867d1192005-04-24 02:06:05 -0500958 if (blk_rq_tagged(rq))
959 blk_queue_end_tag(q, rq);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700960
Jerome Marchandb238b3d2007-10-23 15:05:46 +0200961 drive_stat_acct(rq, 1);
Tejun Heo 867d1192005-04-24 02:06:05 -0500962 __elv_add_request(q, rq, where, 0);
Tejun Heoa7f55792009-04-23 11:05:17 +0900963 __blk_run_queue(q);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700964 spin_unlock_irqrestore(q->queue_lock, flags);
965}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700966EXPORT_SYMBOL(blk_insert_request);
967
Linus Torvalds1da177e2005-04-16 15:20:36 -0700968/*
969 * add-request adds a request to the linked list.
970 * queue lock is held and interrupts disabled, as we muck with the
971 * request queue list.
972 */
Jens Axboe6728cb02008-01-31 13:03:55 +0100973static inline void add_request(struct request_queue *q, struct request *req)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700974{
Jerome Marchandb238b3d2007-10-23 15:05:46 +0200975 drive_stat_acct(req, 1);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700976
Linus Torvalds1da177e2005-04-16 15:20:36 -0700977 /*
978 * elevator indicated where it wants this request to be
979 * inserted at elevator_merge time
980 */
981 __elv_add_request(q, req, ELEVATOR_INSERT_SORT, 0);
982}
Jens Axboe6728cb02008-01-31 13:03:55 +0100983
Tejun Heo074a7ac2008-08-25 19:56:14 +0900984static void part_round_stats_single(int cpu, struct hd_struct *part,
985 unsigned long now)
986{
987 if (now == part->stamp)
988 return;
989
990 if (part->in_flight) {
991 __part_stat_add(cpu, part, time_in_queue,
992 part->in_flight * (now - part->stamp));
993 __part_stat_add(cpu, part, io_ticks, (now - part->stamp));
994 }
995 part->stamp = now;
996}
997
998/**
Randy Dunlap496aa8a2008-10-16 07:46:23 +0200999 * part_round_stats() - Round off the performance stats on a struct disk_stats.
1000 * @cpu: cpu number for stats access
1001 * @part: target partition
Linus Torvalds1da177e2005-04-16 15:20:36 -07001002 *
1003 * The average IO queue length and utilisation statistics are maintained
1004 * by observing the current state of the queue length and the amount of
1005 * time it has been in this state for.
1006 *
1007 * Normally, that accounting is done on IO completion, but that can result
1008 * in more than a second's worth of IO being accounted for within any one
1009 * second, leading to >100% utilisation. To deal with that, we call this
1010 * function to do a round-off before returning the results when reading
1011 * /proc/diskstats. This accounts immediately for all queue usage up to
1012 * the current jiffies and restarts the counters again.
1013 */
Tejun Heoc9959052008-08-25 19:47:21 +09001014void part_round_stats(int cpu, struct hd_struct *part)
Jerome Marchand6f2576a2008-02-08 11:04:35 +01001015{
1016 unsigned long now = jiffies;
1017
Tejun Heo074a7ac2008-08-25 19:56:14 +09001018 if (part->partno)
1019 part_round_stats_single(cpu, &part_to_disk(part)->part0, now);
1020 part_round_stats_single(cpu, part, now);
Jerome Marchand6f2576a2008-02-08 11:04:35 +01001021}
Tejun Heo074a7ac2008-08-25 19:56:14 +09001022EXPORT_SYMBOL_GPL(part_round_stats);
Jerome Marchand6f2576a2008-02-08 11:04:35 +01001023
Linus Torvalds1da177e2005-04-16 15:20:36 -07001024/*
1025 * queue lock must be held
1026 */
Jens Axboe165125e2007-07-24 09:28:11 +02001027void __blk_put_request(struct request_queue *q, struct request *req)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001028{
Linus Torvalds1da177e2005-04-16 15:20:36 -07001029 if (unlikely(!q))
1030 return;
1031 if (unlikely(--req->ref_count))
1032 return;
1033
Tejun Heo8922e162005-10-20 16:23:44 +02001034 elv_completed_request(q, req);
1035
Boaz Harrosh1cd96c22009-03-24 12:35:07 +01001036 /* this is a bio leak */
1037 WARN_ON(req->bio != NULL);
1038
Linus Torvalds1da177e2005-04-16 15:20:36 -07001039 /*
1040 * Request may not have originated from ll_rw_blk. if not,
1041 * it didn't come out of our reserved rq pools
1042 */
Jens Axboe49171e52006-08-10 08:59:11 +02001043 if (req->cmd_flags & REQ_ALLOCED) {
Jens Axboe1faa16d2009-04-06 14:48:01 +02001044 int is_sync = rq_is_sync(req) != 0;
Jens Axboe4aff5e22006-08-10 08:44:47 +02001045 int priv = req->cmd_flags & REQ_ELVPRIV;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001046
Linus Torvalds1da177e2005-04-16 15:20:36 -07001047 BUG_ON(!list_empty(&req->queuelist));
Jens Axboe98170642006-07-28 09:23:08 +02001048 BUG_ON(!hlist_unhashed(&req->hash));
Linus Torvalds1da177e2005-04-16 15:20:36 -07001049
1050 blk_free_request(q, req);
Jens Axboe1faa16d2009-04-06 14:48:01 +02001051 freed_request(q, is_sync, priv);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001052 }
1053}
Mike Christie6e39b69e2005-11-11 05:30:24 -06001054EXPORT_SYMBOL_GPL(__blk_put_request);
1055
Linus Torvalds1da177e2005-04-16 15:20:36 -07001056void blk_put_request(struct request *req)
1057{
Tejun Heo8922e162005-10-20 16:23:44 +02001058 unsigned long flags;
Jens Axboe165125e2007-07-24 09:28:11 +02001059 struct request_queue *q = req->q;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001060
FUJITA Tomonori52a93ba2008-07-15 21:21:45 +02001061 spin_lock_irqsave(q->queue_lock, flags);
1062 __blk_put_request(q, req);
1063 spin_unlock_irqrestore(q->queue_lock, flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001064}
Linus Torvalds1da177e2005-04-16 15:20:36 -07001065EXPORT_SYMBOL(blk_put_request);
1066
Jens Axboe86db1e22008-01-29 14:53:40 +01001067void init_request_from_bio(struct request *req, struct bio *bio)
Tejun Heo52d9e672006-01-06 09:49:58 +01001068{
Jens Axboec7c22e42008-09-13 20:26:01 +02001069 req->cpu = bio->bi_comp_cpu;
Jens Axboe4aff5e22006-08-10 08:44:47 +02001070 req->cmd_type = REQ_TYPE_FS;
Tejun Heo52d9e672006-01-06 09:49:58 +01001071
1072 /*
1073 * inherit FAILFAST from bio (for read-ahead, and explicit FAILFAST)
1074 */
Mike Christie6000a362008-08-19 18:45:30 -05001075 if (bio_rw_ahead(bio))
1076 req->cmd_flags |= (REQ_FAILFAST_DEV | REQ_FAILFAST_TRANSPORT |
1077 REQ_FAILFAST_DRIVER);
1078 if (bio_failfast_dev(bio))
1079 req->cmd_flags |= REQ_FAILFAST_DEV;
1080 if (bio_failfast_transport(bio))
1081 req->cmd_flags |= REQ_FAILFAST_TRANSPORT;
1082 if (bio_failfast_driver(bio))
1083 req->cmd_flags |= REQ_FAILFAST_DRIVER;
Tejun Heo52d9e672006-01-06 09:49:58 +01001084
David Woodhousefb2dce82008-08-05 18:01:53 +01001085 if (unlikely(bio_discard(bio))) {
David Woodhousee17fc0a2008-08-09 16:42:20 +01001086 req->cmd_flags |= REQ_DISCARD;
1087 if (bio_barrier(bio))
1088 req->cmd_flags |= REQ_SOFTBARRIER;
David Woodhousefb2dce82008-08-05 18:01:53 +01001089 req->q->prepare_discard_fn(req->q, req);
David Woodhousee17fc0a2008-08-09 16:42:20 +01001090 } else if (unlikely(bio_barrier(bio)))
Tejun Heoe4025f62009-04-23 11:05:17 +09001091 req->cmd_flags |= REQ_HARDBARRIER;
Tejun Heo52d9e672006-01-06 09:49:58 +01001092
Jens Axboeb31dc662006-06-13 08:26:10 +02001093 if (bio_sync(bio))
Jens Axboe4aff5e22006-08-10 08:44:47 +02001094 req->cmd_flags |= REQ_RW_SYNC;
Jens Axboe5404bc72006-08-10 09:01:02 +02001095 if (bio_rw_meta(bio))
1096 req->cmd_flags |= REQ_RW_META;
Jens Axboeaeb6faf2009-04-06 14:48:07 +02001097 if (bio_noidle(bio))
1098 req->cmd_flags |= REQ_NOIDLE;
Jens Axboeb31dc662006-06-13 08:26:10 +02001099
Tejun Heo52d9e672006-01-06 09:49:58 +01001100 req->errors = 0;
1101 req->hard_sector = req->sector = bio->bi_sector;
Tejun Heo52d9e672006-01-06 09:49:58 +01001102 req->ioprio = bio_prio(bio);
Tejun Heo52d9e672006-01-06 09:49:58 +01001103 req->start_time = jiffies;
NeilBrownbc1c56f2007-08-16 13:31:30 +02001104 blk_rq_bio_prep(req->q, req, bio);
Tejun Heo52d9e672006-01-06 09:49:58 +01001105}
1106
Jens Axboe644b2d92009-04-06 14:48:06 +02001107/*
1108 * Only disabling plugging for non-rotational devices if it does tagging
1109 * as well, otherwise we do need the proper merging
1110 */
1111static inline bool queue_should_plug(struct request_queue *q)
1112{
1113 return !(blk_queue_nonrot(q) && blk_queue_tagged(q));
1114}
1115
Jens Axboe165125e2007-07-24 09:28:11 +02001116static int __make_request(struct request_queue *q, struct bio *bio)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001117{
Nick Piggin450991b2005-06-28 20:45:13 -07001118 struct request *req;
Tejun Heoa7384672008-11-28 13:32:03 +09001119 int el_ret, nr_sectors;
Jens Axboe51da90f2006-07-18 04:14:45 +02001120 const unsigned short prio = bio_prio(bio);
1121 const int sync = bio_sync(bio);
Jens Axboe213d9412009-01-06 09:16:05 +01001122 const int unplug = bio_unplug(bio);
Jens Axboe7749a8d2006-12-13 13:02:26 +01001123 int rw_flags;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001124
Linus Torvalds1da177e2005-04-16 15:20:36 -07001125 nr_sectors = bio_sectors(bio);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001126
1127 /*
1128 * low level driver can indicate that it wants pages above a
1129 * certain limit bounced to low memory (ie for highmem, or even
1130 * ISA dma in theory)
1131 */
1132 blk_queue_bounce(q, &bio);
1133
Linus Torvalds1da177e2005-04-16 15:20:36 -07001134 spin_lock_irq(q->queue_lock);
1135
Tejun Heoa7384672008-11-28 13:32:03 +09001136 if (unlikely(bio_barrier(bio)) || elv_queue_empty(q))
Linus Torvalds1da177e2005-04-16 15:20:36 -07001137 goto get_rq;
1138
1139 el_ret = elv_merge(q, &req, bio);
1140 switch (el_ret) {
Jens Axboe6728cb02008-01-31 13:03:55 +01001141 case ELEVATOR_BACK_MERGE:
1142 BUG_ON(!rq_mergeable(req));
Linus Torvalds1da177e2005-04-16 15:20:36 -07001143
Jens Axboe6728cb02008-01-31 13:03:55 +01001144 if (!ll_back_merge_fn(q, req, bio))
1145 break;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001146
Arnaldo Carvalho de Melo5f3ea372008-10-30 08:34:33 +01001147 trace_block_bio_backmerge(q, bio);
Jens Axboe2056a782006-03-23 20:00:26 +01001148
Jens Axboe6728cb02008-01-31 13:03:55 +01001149 req->biotail->bi_next = bio;
1150 req->biotail = bio;
1151 req->nr_sectors = req->hard_nr_sectors += nr_sectors;
1152 req->ioprio = ioprio_best(req->ioprio, prio);
Jens Axboeab780f12008-08-26 10:25:02 +02001153 if (!blk_rq_cpu_valid(req))
1154 req->cpu = bio->bi_comp_cpu;
Jens Axboe6728cb02008-01-31 13:03:55 +01001155 drive_stat_acct(req, 0);
1156 if (!attempt_back_merge(q, req))
1157 elv_merged_request(q, req, el_ret);
1158 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001159
Jens Axboe6728cb02008-01-31 13:03:55 +01001160 case ELEVATOR_FRONT_MERGE:
1161 BUG_ON(!rq_mergeable(req));
Linus Torvalds1da177e2005-04-16 15:20:36 -07001162
Jens Axboe6728cb02008-01-31 13:03:55 +01001163 if (!ll_front_merge_fn(q, req, bio))
1164 break;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001165
Arnaldo Carvalho de Melo5f3ea372008-10-30 08:34:33 +01001166 trace_block_bio_frontmerge(q, bio);
Jens Axboe2056a782006-03-23 20:00:26 +01001167
Jens Axboe6728cb02008-01-31 13:03:55 +01001168 bio->bi_next = req->bio;
1169 req->bio = bio;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001170
Jens Axboe6728cb02008-01-31 13:03:55 +01001171 /*
1172 * may not be valid. if the low level driver said
1173 * it didn't need a bounce buffer then it better
1174 * not touch req->buffer either...
1175 */
1176 req->buffer = bio_data(bio);
1177 req->current_nr_sectors = bio_cur_sectors(bio);
1178 req->hard_cur_sectors = req->current_nr_sectors;
1179 req->sector = req->hard_sector = bio->bi_sector;
1180 req->nr_sectors = req->hard_nr_sectors += nr_sectors;
1181 req->ioprio = ioprio_best(req->ioprio, prio);
Jens Axboeab780f12008-08-26 10:25:02 +02001182 if (!blk_rq_cpu_valid(req))
1183 req->cpu = bio->bi_comp_cpu;
Jens Axboe6728cb02008-01-31 13:03:55 +01001184 drive_stat_acct(req, 0);
1185 if (!attempt_front_merge(q, req))
1186 elv_merged_request(q, req, el_ret);
1187 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001188
Jens Axboe6728cb02008-01-31 13:03:55 +01001189 /* ELV_NO_MERGE: elevator says don't/can't merge. */
1190 default:
1191 ;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001192 }
1193
Linus Torvalds1da177e2005-04-16 15:20:36 -07001194get_rq:
Nick Piggin450991b2005-06-28 20:45:13 -07001195 /*
Jens Axboe7749a8d2006-12-13 13:02:26 +01001196 * This sync check and mask will be re-done in init_request_from_bio(),
1197 * but we need to set it earlier to expose the sync flag to the
1198 * rq allocator and io schedulers.
1199 */
1200 rw_flags = bio_data_dir(bio);
1201 if (sync)
1202 rw_flags |= REQ_RW_SYNC;
1203
1204 /*
Nick Piggin450991b2005-06-28 20:45:13 -07001205 * Grab a free request. This is might sleep but can not fail.
Nick Piggind6344532005-06-28 20:45:14 -07001206 * Returns with the queue unlocked.
Nick Piggin450991b2005-06-28 20:45:13 -07001207 */
Jens Axboe7749a8d2006-12-13 13:02:26 +01001208 req = get_request_wait(q, rw_flags, bio);
Nick Piggind6344532005-06-28 20:45:14 -07001209
Nick Piggin450991b2005-06-28 20:45:13 -07001210 /*
1211 * After dropping the lock and possibly sleeping here, our request
1212 * may now be mergeable after it had proven unmergeable (above).
1213 * We don't worry about that case for efficiency. It won't happen
1214 * often, and the elevators are able to handle it.
1215 */
Tejun Heo52d9e672006-01-06 09:49:58 +01001216 init_request_from_bio(req, bio);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001217
Nick Piggin450991b2005-06-28 20:45:13 -07001218 spin_lock_irq(q->queue_lock);
Jens Axboec7c22e42008-09-13 20:26:01 +02001219 if (test_bit(QUEUE_FLAG_SAME_COMP, &q->queue_flags) ||
1220 bio_flagged(bio, BIO_CPU_AFFINE))
1221 req->cpu = blk_cpu_to_group(smp_processor_id());
Jens Axboe644b2d92009-04-06 14:48:06 +02001222 if (queue_should_plug(q) && elv_queue_empty(q))
Nick Piggin450991b2005-06-28 20:45:13 -07001223 blk_plug_device(q);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001224 add_request(q, req);
1225out:
Jens Axboe644b2d92009-04-06 14:48:06 +02001226 if (unplug || !queue_should_plug(q))
Linus Torvalds1da177e2005-04-16 15:20:36 -07001227 __generic_unplug_device(q);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001228 spin_unlock_irq(q->queue_lock);
1229 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001230}
1231
1232/*
1233 * If bio->bi_dev is a partition, remap the location
1234 */
1235static inline void blk_partition_remap(struct bio *bio)
1236{
1237 struct block_device *bdev = bio->bi_bdev;
1238
Jens Axboebf2de6f2007-09-27 13:01:25 +02001239 if (bio_sectors(bio) && bdev != bdev->bd_contains) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001240 struct hd_struct *p = bdev->bd_part;
Jens Axboea3623572005-11-01 09:26:16 +01001241
Linus Torvalds1da177e2005-04-16 15:20:36 -07001242 bio->bi_sector += p->start_sect;
1243 bio->bi_bdev = bdev->bd_contains;
Alan D. Brunellec7149d62007-08-07 15:30:23 +02001244
Arnaldo Carvalho de Melo5f3ea372008-10-30 08:34:33 +01001245 trace_block_remap(bdev_get_queue(bio->bi_bdev), bio,
Alan D. Brunellec7149d62007-08-07 15:30:23 +02001246 bdev->bd_dev, bio->bi_sector,
1247 bio->bi_sector - p->start_sect);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001248 }
1249}
1250
Linus Torvalds1da177e2005-04-16 15:20:36 -07001251static void handle_bad_sector(struct bio *bio)
1252{
1253 char b[BDEVNAME_SIZE];
1254
1255 printk(KERN_INFO "attempt to access beyond end of device\n");
1256 printk(KERN_INFO "%s: rw=%ld, want=%Lu, limit=%Lu\n",
1257 bdevname(bio->bi_bdev, b),
1258 bio->bi_rw,
1259 (unsigned long long)bio->bi_sector + bio_sectors(bio),
1260 (long long)(bio->bi_bdev->bd_inode->i_size >> 9));
1261
1262 set_bit(BIO_EOF, &bio->bi_flags);
1263}
1264
Akinobu Mitac17bb492006-12-08 02:39:46 -08001265#ifdef CONFIG_FAIL_MAKE_REQUEST
1266
1267static DECLARE_FAULT_ATTR(fail_make_request);
1268
1269static int __init setup_fail_make_request(char *str)
1270{
1271 return setup_fault_attr(&fail_make_request, str);
1272}
1273__setup("fail_make_request=", setup_fail_make_request);
1274
1275static int should_fail_request(struct bio *bio)
1276{
Tejun Heoeddb2e22008-08-25 19:56:13 +09001277 struct hd_struct *part = bio->bi_bdev->bd_part;
1278
1279 if (part_to_disk(part)->part0.make_it_fail || part->make_it_fail)
Akinobu Mitac17bb492006-12-08 02:39:46 -08001280 return should_fail(&fail_make_request, bio->bi_size);
1281
1282 return 0;
1283}
1284
1285static int __init fail_make_request_debugfs(void)
1286{
1287 return init_fault_attr_dentries(&fail_make_request,
1288 "fail_make_request");
1289}
1290
1291late_initcall(fail_make_request_debugfs);
1292
1293#else /* CONFIG_FAIL_MAKE_REQUEST */
1294
1295static inline int should_fail_request(struct bio *bio)
1296{
1297 return 0;
1298}
1299
1300#endif /* CONFIG_FAIL_MAKE_REQUEST */
1301
Jens Axboec07e2b42007-07-18 13:27:58 +02001302/*
1303 * Check whether this bio extends beyond the end of the device.
1304 */
1305static inline int bio_check_eod(struct bio *bio, unsigned int nr_sectors)
1306{
1307 sector_t maxsector;
1308
1309 if (!nr_sectors)
1310 return 0;
1311
1312 /* Test device or partition size, when known. */
1313 maxsector = bio->bi_bdev->bd_inode->i_size >> 9;
1314 if (maxsector) {
1315 sector_t sector = bio->bi_sector;
1316
1317 if (maxsector < nr_sectors || maxsector - nr_sectors < sector) {
1318 /*
1319 * This may well happen - the kernel calls bread()
1320 * without checking the size of the device, e.g., when
1321 * mounting a device.
1322 */
1323 handle_bad_sector(bio);
1324 return 1;
1325 }
1326 }
1327
1328 return 0;
1329}
1330
Linus Torvalds1da177e2005-04-16 15:20:36 -07001331/**
Randy Dunlap710027a2008-08-19 20:13:11 +02001332 * generic_make_request - hand a buffer to its device driver for I/O
Linus Torvalds1da177e2005-04-16 15:20:36 -07001333 * @bio: The bio describing the location in memory and on the device.
1334 *
1335 * generic_make_request() is used to make I/O requests of block
1336 * devices. It is passed a &struct bio, which describes the I/O that needs
1337 * to be done.
1338 *
1339 * generic_make_request() does not return any status. The
1340 * success/failure status of the request, along with notification of
1341 * completion, is delivered asynchronously through the bio->bi_end_io
1342 * function described (one day) else where.
1343 *
1344 * The caller of generic_make_request must make sure that bi_io_vec
1345 * are set to describe the memory buffer, and that bi_dev and bi_sector are
1346 * set to describe the device address, and the
1347 * bi_end_io and optionally bi_private are set to describe how
1348 * completion notification should be signaled.
1349 *
1350 * generic_make_request and the drivers it calls may use bi_next if this
1351 * bio happens to be merged with someone else, and may change bi_dev and
1352 * bi_sector for remaps as it sees fit. So the values of these fields
1353 * should NOT be depended on after the call to generic_make_request.
1354 */
Neil Brownd89d8792007-05-01 09:53:42 +02001355static inline void __generic_make_request(struct bio *bio)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001356{
Jens Axboe165125e2007-07-24 09:28:11 +02001357 struct request_queue *q;
NeilBrown5ddfe962006-10-30 22:07:21 -08001358 sector_t old_sector;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001359 int ret, nr_sectors = bio_sectors(bio);
Jens Axboe2056a782006-03-23 20:00:26 +01001360 dev_t old_dev;
Jens Axboe51fd77b2007-11-02 08:49:08 +01001361 int err = -EIO;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001362
1363 might_sleep();
Linus Torvalds1da177e2005-04-16 15:20:36 -07001364
Jens Axboec07e2b42007-07-18 13:27:58 +02001365 if (bio_check_eod(bio, nr_sectors))
1366 goto end_io;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001367
1368 /*
1369 * Resolve the mapping until finished. (drivers are
1370 * still free to implement/resolve their own stacking
1371 * by explicitly returning 0)
1372 *
1373 * NOTE: we don't repeat the blk_size check for each new device.
1374 * Stacking drivers are expected to know what they are doing.
1375 */
NeilBrown5ddfe962006-10-30 22:07:21 -08001376 old_sector = -1;
Jens Axboe2056a782006-03-23 20:00:26 +01001377 old_dev = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001378 do {
1379 char b[BDEVNAME_SIZE];
1380
1381 q = bdev_get_queue(bio->bi_bdev);
Tejun Heoa7384672008-11-28 13:32:03 +09001382 if (unlikely(!q)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001383 printk(KERN_ERR
1384 "generic_make_request: Trying to access "
1385 "nonexistent block-device %s (%Lu)\n",
1386 bdevname(bio->bi_bdev, b),
1387 (long long) bio->bi_sector);
Tejun Heoa7384672008-11-28 13:32:03 +09001388 goto end_io;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001389 }
1390
Jens Axboe4fa253f2007-07-18 13:13:10 +02001391 if (unlikely(nr_sectors > q->max_hw_sectors)) {
Jens Axboe6728cb02008-01-31 13:03:55 +01001392 printk(KERN_ERR "bio too big device %s (%u > %u)\n",
Linus Torvalds1da177e2005-04-16 15:20:36 -07001393 bdevname(bio->bi_bdev, b),
1394 bio_sectors(bio),
1395 q->max_hw_sectors);
1396 goto end_io;
1397 }
1398
Nick Pigginfde6ad22005-06-23 00:08:53 -07001399 if (unlikely(test_bit(QUEUE_FLAG_DEAD, &q->queue_flags)))
Linus Torvalds1da177e2005-04-16 15:20:36 -07001400 goto end_io;
1401
Akinobu Mitac17bb492006-12-08 02:39:46 -08001402 if (should_fail_request(bio))
1403 goto end_io;
1404
Linus Torvalds1da177e2005-04-16 15:20:36 -07001405 /*
1406 * If this device has partitions, remap block n
1407 * of partition p to block n+start(p) of the disk.
1408 */
1409 blk_partition_remap(bio);
1410
Martin K. Petersen7ba1ba12008-06-30 20:04:41 +02001411 if (bio_integrity_enabled(bio) && bio_integrity_prep(bio))
1412 goto end_io;
1413
NeilBrown5ddfe962006-10-30 22:07:21 -08001414 if (old_sector != -1)
Arnaldo Carvalho de Melo5f3ea372008-10-30 08:34:33 +01001415 trace_block_remap(q, bio, old_dev, bio->bi_sector,
NeilBrown5ddfe962006-10-30 22:07:21 -08001416 old_sector);
Jens Axboe2056a782006-03-23 20:00:26 +01001417
Arnaldo Carvalho de Melo5f3ea372008-10-30 08:34:33 +01001418 trace_block_bio_queue(q, bio);
Jens Axboe2056a782006-03-23 20:00:26 +01001419
NeilBrown5ddfe962006-10-30 22:07:21 -08001420 old_sector = bio->bi_sector;
Jens Axboe2056a782006-03-23 20:00:26 +01001421 old_dev = bio->bi_bdev->bd_dev;
1422
Jens Axboec07e2b42007-07-18 13:27:58 +02001423 if (bio_check_eod(bio, nr_sectors))
1424 goto end_io;
Tejun Heoa7384672008-11-28 13:32:03 +09001425
1426 if (bio_discard(bio) && !q->prepare_discard_fn) {
Jens Axboe51fd77b2007-11-02 08:49:08 +01001427 err = -EOPNOTSUPP;
1428 goto end_io;
1429 }
Jens Axboecec07072009-01-13 15:28:32 +01001430 if (bio_barrier(bio) && bio_has_data(bio) &&
1431 (q->next_ordered == QUEUE_ORDERED_NONE)) {
1432 err = -EOPNOTSUPP;
1433 goto end_io;
1434 }
NeilBrown5ddfe962006-10-30 22:07:21 -08001435
Linus Torvalds1da177e2005-04-16 15:20:36 -07001436 ret = q->make_request_fn(q, bio);
1437 } while (ret);
Tejun Heoa7384672008-11-28 13:32:03 +09001438
1439 return;
1440
1441end_io:
1442 bio_endio(bio, err);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001443}
1444
Neil Brownd89d8792007-05-01 09:53:42 +02001445/*
1446 * We only want one ->make_request_fn to be active at a time,
1447 * else stack usage with stacked devices could be a problem.
1448 * So use current->bio_{list,tail} to keep a list of requests
1449 * submited by a make_request_fn function.
1450 * current->bio_tail is also used as a flag to say if
1451 * generic_make_request is currently active in this task or not.
1452 * If it is NULL, then no make_request is active. If it is non-NULL,
1453 * then a make_request is active, and new requests should be added
1454 * at the tail
1455 */
1456void generic_make_request(struct bio *bio)
1457{
1458 if (current->bio_tail) {
1459 /* make_request is active */
1460 *(current->bio_tail) = bio;
1461 bio->bi_next = NULL;
1462 current->bio_tail = &bio->bi_next;
1463 return;
1464 }
1465 /* following loop may be a bit non-obvious, and so deserves some
1466 * explanation.
1467 * Before entering the loop, bio->bi_next is NULL (as all callers
1468 * ensure that) so we have a list with a single bio.
1469 * We pretend that we have just taken it off a longer list, so
1470 * we assign bio_list to the next (which is NULL) and bio_tail
1471 * to &bio_list, thus initialising the bio_list of new bios to be
1472 * added. __generic_make_request may indeed add some more bios
1473 * through a recursive call to generic_make_request. If it
1474 * did, we find a non-NULL value in bio_list and re-enter the loop
1475 * from the top. In this case we really did just take the bio
1476 * of the top of the list (no pretending) and so fixup bio_list and
1477 * bio_tail or bi_next, and call into __generic_make_request again.
1478 *
1479 * The loop was structured like this to make only one call to
1480 * __generic_make_request (which is important as it is large and
1481 * inlined) and to keep the structure simple.
1482 */
1483 BUG_ON(bio->bi_next);
1484 do {
1485 current->bio_list = bio->bi_next;
1486 if (bio->bi_next == NULL)
1487 current->bio_tail = &current->bio_list;
1488 else
1489 bio->bi_next = NULL;
1490 __generic_make_request(bio);
1491 bio = current->bio_list;
1492 } while (bio);
1493 current->bio_tail = NULL; /* deactivate */
1494}
Linus Torvalds1da177e2005-04-16 15:20:36 -07001495EXPORT_SYMBOL(generic_make_request);
1496
1497/**
Randy Dunlap710027a2008-08-19 20:13:11 +02001498 * submit_bio - submit a bio to the block device layer for I/O
Linus Torvalds1da177e2005-04-16 15:20:36 -07001499 * @rw: whether to %READ or %WRITE, or maybe to %READA (read ahead)
1500 * @bio: The &struct bio which describes the I/O
1501 *
1502 * submit_bio() is very similar in purpose to generic_make_request(), and
1503 * uses that function to do most of the work. Both are fairly rough
Randy Dunlap710027a2008-08-19 20:13:11 +02001504 * interfaces; @bio must be presetup and ready for I/O.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001505 *
1506 */
1507void submit_bio(int rw, struct bio *bio)
1508{
1509 int count = bio_sectors(bio);
1510
Jens Axboe22e2c502005-06-27 10:55:12 +02001511 bio->bi_rw |= rw;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001512
Jens Axboebf2de6f2007-09-27 13:01:25 +02001513 /*
1514 * If it's a regular read/write or a barrier with data attached,
1515 * go through the normal accounting stuff before submission.
1516 */
Jens Axboea9c701e2008-08-08 11:04:44 +02001517 if (bio_has_data(bio)) {
Jens Axboebf2de6f2007-09-27 13:01:25 +02001518 if (rw & WRITE) {
1519 count_vm_events(PGPGOUT, count);
1520 } else {
1521 task_io_account_read(bio->bi_size);
1522 count_vm_events(PGPGIN, count);
1523 }
1524
1525 if (unlikely(block_dump)) {
1526 char b[BDEVNAME_SIZE];
1527 printk(KERN_DEBUG "%s(%d): %s block %Lu on %s\n",
Pavel Emelyanovba25f9d2007-10-18 23:40:40 -07001528 current->comm, task_pid_nr(current),
Jens Axboebf2de6f2007-09-27 13:01:25 +02001529 (rw & WRITE) ? "WRITE" : "READ",
1530 (unsigned long long)bio->bi_sector,
Jens Axboe6728cb02008-01-31 13:03:55 +01001531 bdevname(bio->bi_bdev, b));
Jens Axboebf2de6f2007-09-27 13:01:25 +02001532 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001533 }
1534
1535 generic_make_request(bio);
1536}
Linus Torvalds1da177e2005-04-16 15:20:36 -07001537EXPORT_SYMBOL(submit_bio);
1538
Kiyoshi Ueda3bcddea2007-12-11 17:52:28 -05001539/**
Kiyoshi Ueda82124d62008-09-18 10:45:38 -04001540 * blk_rq_check_limits - Helper function to check a request for the queue limit
1541 * @q: the queue
1542 * @rq: the request being checked
1543 *
1544 * Description:
1545 * @rq may have been made based on weaker limitations of upper-level queues
1546 * in request stacking drivers, and it may violate the limitation of @q.
1547 * Since the block layer and the underlying device driver trust @rq
1548 * after it is inserted to @q, it should be checked against @q before
1549 * the insertion using this generic function.
1550 *
1551 * This function should also be useful for request stacking drivers
1552 * in some cases below, so export this fuction.
1553 * Request stacking drivers like request-based dm may change the queue
1554 * limits while requests are in the queue (e.g. dm's table swapping).
1555 * Such request stacking drivers should check those requests agaist
1556 * the new queue limits again when they dispatch those requests,
1557 * although such checkings are also done against the old queue limits
1558 * when submitting requests.
1559 */
1560int blk_rq_check_limits(struct request_queue *q, struct request *rq)
1561{
1562 if (rq->nr_sectors > q->max_sectors ||
1563 rq->data_len > q->max_hw_sectors << 9) {
1564 printk(KERN_ERR "%s: over max size limit.\n", __func__);
1565 return -EIO;
1566 }
1567
1568 /*
1569 * queue's settings related to segment counting like q->bounce_pfn
1570 * may differ from that of other stacking queues.
1571 * Recalculate it to check the request correctly on this queue's
1572 * limitation.
1573 */
1574 blk_recalc_rq_segments(rq);
1575 if (rq->nr_phys_segments > q->max_phys_segments ||
1576 rq->nr_phys_segments > q->max_hw_segments) {
1577 printk(KERN_ERR "%s: over max segments limit.\n", __func__);
1578 return -EIO;
1579 }
1580
1581 return 0;
1582}
1583EXPORT_SYMBOL_GPL(blk_rq_check_limits);
1584
1585/**
1586 * blk_insert_cloned_request - Helper for stacking drivers to submit a request
1587 * @q: the queue to submit the request
1588 * @rq: the request being queued
1589 */
1590int blk_insert_cloned_request(struct request_queue *q, struct request *rq)
1591{
1592 unsigned long flags;
1593
1594 if (blk_rq_check_limits(q, rq))
1595 return -EIO;
1596
1597#ifdef CONFIG_FAIL_MAKE_REQUEST
1598 if (rq->rq_disk && rq->rq_disk->part0.make_it_fail &&
1599 should_fail(&fail_make_request, blk_rq_bytes(rq)))
1600 return -EIO;
1601#endif
1602
1603 spin_lock_irqsave(q->queue_lock, flags);
1604
1605 /*
1606 * Submitting request must be dequeued before calling this function
1607 * because it will be linked to another request_queue
1608 */
1609 BUG_ON(blk_queued_rq(rq));
1610
1611 drive_stat_acct(rq, 1);
1612 __elv_add_request(q, rq, ELEVATOR_INSERT_BACK, 0);
1613
1614 spin_unlock_irqrestore(q->queue_lock, flags);
1615
1616 return 0;
1617}
1618EXPORT_SYMBOL_GPL(blk_insert_cloned_request);
1619
1620/**
Tejun Heo53a08802008-12-03 12:41:26 +01001621 * blkdev_dequeue_request - dequeue request and start timeout timer
1622 * @req: request to dequeue
1623 *
1624 * Dequeue @req and start timeout timer on it. This hands off the
1625 * request to the driver.
1626 *
1627 * Block internal functions which don't want to start timer should
1628 * call elv_dequeue_request().
1629 */
1630void blkdev_dequeue_request(struct request *req)
1631{
1632 elv_dequeue_request(req->q, req);
1633
1634 /*
1635 * We are now handing the request to the hardware, add the
1636 * timeout handler.
1637 */
1638 blk_add_timer(req);
1639}
1640EXPORT_SYMBOL(blkdev_dequeue_request);
1641
Jens Axboebc58ba92009-01-23 10:54:44 +01001642static void blk_account_io_completion(struct request *req, unsigned int bytes)
1643{
Jerome Marchand26308ea2009-03-27 10:31:51 +01001644 if (!blk_do_io_stat(req))
Jens Axboebc58ba92009-01-23 10:54:44 +01001645 return;
1646
1647 if (blk_fs_request(req)) {
1648 const int rw = rq_data_dir(req);
1649 struct hd_struct *part;
1650 int cpu;
1651
1652 cpu = part_stat_lock();
1653 part = disk_map_sector_rcu(req->rq_disk, req->sector);
1654 part_stat_add(cpu, part, sectors[rw], bytes >> 9);
1655 part_stat_unlock();
1656 }
1657}
1658
1659static void blk_account_io_done(struct request *req)
1660{
Jerome Marchand26308ea2009-03-27 10:31:51 +01001661 if (!blk_do_io_stat(req))
Jens Axboebc58ba92009-01-23 10:54:44 +01001662 return;
1663
1664 /*
1665 * Account IO completion. bar_rq isn't accounted as a normal
1666 * IO on queueing nor completion. Accounting the containing
1667 * request is enough.
1668 */
1669 if (blk_fs_request(req) && req != &req->q->bar_rq) {
1670 unsigned long duration = jiffies - req->start_time;
1671 const int rw = rq_data_dir(req);
1672 struct hd_struct *part;
1673 int cpu;
1674
1675 cpu = part_stat_lock();
Jerome Marchand26308ea2009-03-27 10:31:51 +01001676 part = disk_map_sector_rcu(req->rq_disk, req->sector);
Jens Axboebc58ba92009-01-23 10:54:44 +01001677
1678 part_stat_inc(cpu, part, ios[rw]);
1679 part_stat_add(cpu, part, ticks[rw], duration);
1680 part_round_stats(cpu, part);
1681 part_dec_in_flight(part);
1682
1683 part_stat_unlock();
1684 }
1685}
1686
Tejun Heo53a08802008-12-03 12:41:26 +01001687/**
Kiyoshi Ueda3bcddea2007-12-11 17:52:28 -05001688 * __end_that_request_first - end I/O on a request
1689 * @req: the request being processed
Randy Dunlap710027a2008-08-19 20:13:11 +02001690 * @error: %0 for success, < %0 for error
Kiyoshi Ueda3bcddea2007-12-11 17:52:28 -05001691 * @nr_bytes: number of bytes to complete
1692 *
1693 * Description:
1694 * Ends I/O on a number of bytes attached to @req, and sets it up
1695 * for the next range of segments (if any) in the cluster.
1696 *
1697 * Return:
Randy Dunlap710027a2008-08-19 20:13:11 +02001698 * %0 - we are done with this request, call end_that_request_last()
1699 * %1 - still buffers pending for this request
Kiyoshi Ueda3bcddea2007-12-11 17:52:28 -05001700 **/
Kiyoshi Ueda5450d3e2007-12-11 17:53:03 -05001701static int __end_that_request_first(struct request *req, int error,
Linus Torvalds1da177e2005-04-16 15:20:36 -07001702 int nr_bytes)
1703{
Kiyoshi Ueda5450d3e2007-12-11 17:53:03 -05001704 int total_bytes, bio_nbytes, next_idx = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001705 struct bio *bio;
1706
Arnaldo Carvalho de Melo5f3ea372008-10-30 08:34:33 +01001707 trace_block_rq_complete(req->q, req);
Jens Axboe2056a782006-03-23 20:00:26 +01001708
Linus Torvalds1da177e2005-04-16 15:20:36 -07001709 /*
Tejun Heo924cec72009-04-19 07:00:41 +09001710 * For fs requests, rq is just carrier of independent bio's
1711 * and each partial completion should be handled separately.
1712 * Reset per-request error on each partial completion.
1713 *
1714 * TODO: tj: This is too subtle. It would be better to let
1715 * low level drivers do what they see fit.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001716 */
Tejun Heo924cec72009-04-19 07:00:41 +09001717 if (blk_fs_request(req))
Linus Torvalds1da177e2005-04-16 15:20:36 -07001718 req->errors = 0;
1719
Jens Axboe6728cb02008-01-31 13:03:55 +01001720 if (error && (blk_fs_request(req) && !(req->cmd_flags & REQ_QUIET))) {
1721 printk(KERN_ERR "end_request: I/O error, dev %s, sector %llu\n",
Linus Torvalds1da177e2005-04-16 15:20:36 -07001722 req->rq_disk ? req->rq_disk->disk_name : "?",
1723 (unsigned long long)req->sector);
1724 }
1725
Jens Axboebc58ba92009-01-23 10:54:44 +01001726 blk_account_io_completion(req, nr_bytes);
Jens Axboed72d9042005-11-01 08:35:42 +01001727
Linus Torvalds1da177e2005-04-16 15:20:36 -07001728 total_bytes = bio_nbytes = 0;
1729 while ((bio = req->bio) != NULL) {
1730 int nbytes;
1731
1732 if (nr_bytes >= bio->bi_size) {
1733 req->bio = bio->bi_next;
1734 nbytes = bio->bi_size;
NeilBrown5bb23a62007-09-27 12:46:13 +02001735 req_bio_endio(req, bio, nbytes, error);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001736 next_idx = 0;
1737 bio_nbytes = 0;
1738 } else {
1739 int idx = bio->bi_idx + next_idx;
1740
1741 if (unlikely(bio->bi_idx >= bio->bi_vcnt)) {
1742 blk_dump_rq_flags(req, "__end_that");
Jens Axboe6728cb02008-01-31 13:03:55 +01001743 printk(KERN_ERR "%s: bio idx %d >= vcnt %d\n",
Harvey Harrison24c03d42008-05-01 04:35:17 -07001744 __func__, bio->bi_idx, bio->bi_vcnt);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001745 break;
1746 }
1747
1748 nbytes = bio_iovec_idx(bio, idx)->bv_len;
1749 BIO_BUG_ON(nbytes > bio->bi_size);
1750
1751 /*
1752 * not a complete bvec done
1753 */
1754 if (unlikely(nbytes > nr_bytes)) {
1755 bio_nbytes += nr_bytes;
1756 total_bytes += nr_bytes;
1757 break;
1758 }
1759
1760 /*
1761 * advance to the next vector
1762 */
1763 next_idx++;
1764 bio_nbytes += nbytes;
1765 }
1766
1767 total_bytes += nbytes;
1768 nr_bytes -= nbytes;
1769
Jens Axboe6728cb02008-01-31 13:03:55 +01001770 bio = req->bio;
1771 if (bio) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001772 /*
1773 * end more in this run, or just return 'not-done'
1774 */
1775 if (unlikely(nr_bytes <= 0))
1776 break;
1777 }
1778 }
1779
1780 /*
1781 * completely done
1782 */
1783 if (!req->bio)
1784 return 0;
1785
1786 /*
1787 * if the request wasn't completed, update state
1788 */
1789 if (bio_nbytes) {
NeilBrown5bb23a62007-09-27 12:46:13 +02001790 req_bio_endio(req, bio, bio_nbytes, error);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001791 bio->bi_idx += next_idx;
1792 bio_iovec(bio)->bv_offset += nr_bytes;
1793 bio_iovec(bio)->bv_len -= nr_bytes;
1794 }
1795
1796 blk_recalc_rq_sectors(req, total_bytes >> 9);
1797 blk_recalc_rq_segments(req);
1798 return 1;
1799}
1800
Linus Torvalds1da177e2005-04-16 15:20:36 -07001801/*
1802 * queue lock must be held
1803 */
Kiyoshi Ueda5450d3e2007-12-11 17:53:03 -05001804static void end_that_request_last(struct request *req, int error)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001805{
Kiyoshi Uedab8286232007-12-11 17:53:24 -05001806 if (blk_rq_tagged(req))
1807 blk_queue_end_tag(req->q, req);
1808
1809 if (blk_queued_rq(req))
Tejun Heo53a08802008-12-03 12:41:26 +01001810 elv_dequeue_request(req->q, req);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001811
1812 if (unlikely(laptop_mode) && blk_fs_request(req))
1813 laptop_io_completion();
1814
Mike Andersone78042e2008-10-30 02:16:20 -07001815 blk_delete_timer(req);
1816
Jens Axboebc58ba92009-01-23 10:54:44 +01001817 blk_account_io_done(req);
Kiyoshi Uedab8286232007-12-11 17:53:24 -05001818
Linus Torvalds1da177e2005-04-16 15:20:36 -07001819 if (req->end_io)
Tejun Heo8ffdc652006-01-06 09:49:03 +01001820 req->end_io(req, error);
Kiyoshi Uedab8286232007-12-11 17:53:24 -05001821 else {
1822 if (blk_bidi_rq(req))
1823 __blk_put_request(req->next_rq->q, req->next_rq);
1824
Linus Torvalds1da177e2005-04-16 15:20:36 -07001825 __blk_put_request(req->q, req);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001826 }
1827}
1828
Kiyoshi Ueda3b113132007-12-11 17:41:17 -05001829/**
1830 * blk_rq_bytes - Returns bytes left to complete in the entire request
Randy Dunlap5d87a052008-02-20 09:01:22 +01001831 * @rq: the request being processed
Kiyoshi Ueda3b113132007-12-11 17:41:17 -05001832 **/
1833unsigned int blk_rq_bytes(struct request *rq)
Jens Axboea0cd1282007-09-21 10:41:07 +02001834{
1835 if (blk_fs_request(rq))
1836 return rq->hard_nr_sectors << 9;
1837
1838 return rq->data_len;
1839}
Kiyoshi Ueda3b113132007-12-11 17:41:17 -05001840EXPORT_SYMBOL_GPL(blk_rq_bytes);
1841
1842/**
1843 * blk_rq_cur_bytes - Returns bytes left to complete in the current segment
Randy Dunlap5d87a052008-02-20 09:01:22 +01001844 * @rq: the request being processed
Kiyoshi Ueda3b113132007-12-11 17:41:17 -05001845 **/
1846unsigned int blk_rq_cur_bytes(struct request *rq)
1847{
1848 if (blk_fs_request(rq))
1849 return rq->current_nr_sectors << 9;
1850
1851 if (rq->bio)
1852 return rq->bio->bi_size;
1853
1854 return rq->data_len;
1855}
1856EXPORT_SYMBOL_GPL(blk_rq_cur_bytes);
Jens Axboea0cd1282007-09-21 10:41:07 +02001857
1858/**
Jens Axboea0cd1282007-09-21 10:41:07 +02001859 * end_request - end I/O on the current segment of the request
Randy Dunlap8f731f72007-10-18 23:39:28 -07001860 * @req: the request being processed
Randy Dunlap710027a2008-08-19 20:13:11 +02001861 * @uptodate: error value or %0/%1 uptodate flag
Jens Axboea0cd1282007-09-21 10:41:07 +02001862 *
1863 * Description:
1864 * Ends I/O on the current segment of a request. If that is the only
1865 * remaining segment, the request is also completed and freed.
1866 *
Randy Dunlap710027a2008-08-19 20:13:11 +02001867 * This is a remnant of how older block drivers handled I/O completions.
1868 * Modern drivers typically end I/O on the full request in one go, unless
Jens Axboea0cd1282007-09-21 10:41:07 +02001869 * they have a residual value to account for. For that case this function
1870 * isn't really useful, unless the residual just happens to be the
1871 * full current segment. In other words, don't use this function in new
Kiyoshi Uedad00e29f2008-10-01 10:14:46 -04001872 * code. Use blk_end_request() or __blk_end_request() to end a request.
Jens Axboea0cd1282007-09-21 10:41:07 +02001873 **/
1874void end_request(struct request *req, int uptodate)
1875{
Kiyoshi Uedad00e29f2008-10-01 10:14:46 -04001876 int error = 0;
1877
1878 if (uptodate <= 0)
1879 error = uptodate ? uptodate : -EIO;
1880
1881 __blk_end_request(req, error, req->hard_cur_sectors << 9);
Jens Axboea0cd1282007-09-21 10:41:07 +02001882}
Linus Torvalds1da177e2005-04-16 15:20:36 -07001883EXPORT_SYMBOL(end_request);
1884
Kiyoshi Ueda32fab442008-09-18 10:45:09 -04001885static int end_that_request_data(struct request *rq, int error,
1886 unsigned int nr_bytes, unsigned int bidi_bytes)
1887{
1888 if (rq->bio) {
1889 if (__end_that_request_first(rq, error, nr_bytes))
1890 return 1;
1891
1892 /* Bidi request must be completed as a whole */
1893 if (blk_bidi_rq(rq) &&
1894 __end_that_request_first(rq->next_rq, error, bidi_bytes))
1895 return 1;
1896 }
1897
1898 return 0;
1899}
1900
Kiyoshi Ueda336cdb42007-12-11 17:40:30 -05001901/**
Kiyoshi Uedae19a3ab2007-12-11 17:51:02 -05001902 * blk_end_io - Generic end_io function to complete a request.
1903 * @rq: the request being processed
Randy Dunlap710027a2008-08-19 20:13:11 +02001904 * @error: %0 for success, < %0 for error
Kiyoshi Uedae3a04fe2007-12-11 17:51:46 -05001905 * @nr_bytes: number of bytes to complete @rq
1906 * @bidi_bytes: number of bytes to complete @rq->next_rq
Kiyoshi Uedae19a3ab2007-12-11 17:51:02 -05001907 * @drv_callback: function called between completion of bios in the request
1908 * and completion of the request.
Randy Dunlap710027a2008-08-19 20:13:11 +02001909 * If the callback returns non %0, this helper returns without
Kiyoshi Uedae19a3ab2007-12-11 17:51:02 -05001910 * completion of the request.
Kiyoshi Ueda336cdb42007-12-11 17:40:30 -05001911 *
1912 * Description:
Kiyoshi Uedae3a04fe2007-12-11 17:51:46 -05001913 * Ends I/O on a number of bytes attached to @rq and @rq->next_rq.
Kiyoshi Ueda336cdb42007-12-11 17:40:30 -05001914 * If @rq has leftover, sets it up for the next range of segments.
1915 *
1916 * Return:
Randy Dunlap710027a2008-08-19 20:13:11 +02001917 * %0 - we are done with this request
1918 * %1 - this request is not freed yet, it still has pending buffers.
Kiyoshi Ueda336cdb42007-12-11 17:40:30 -05001919 **/
Jens Axboe22b13212008-01-31 12:36:19 +01001920static int blk_end_io(struct request *rq, int error, unsigned int nr_bytes,
1921 unsigned int bidi_bytes,
1922 int (drv_callback)(struct request *))
Kiyoshi Ueda336cdb42007-12-11 17:40:30 -05001923{
1924 struct request_queue *q = rq->q;
1925 unsigned long flags = 0UL;
Kiyoshi Ueda336cdb42007-12-11 17:40:30 -05001926
Kiyoshi Ueda32fab442008-09-18 10:45:09 -04001927 if (end_that_request_data(rq, error, nr_bytes, bidi_bytes))
1928 return 1;
Kiyoshi Ueda336cdb42007-12-11 17:40:30 -05001929
Kiyoshi Uedae19a3ab2007-12-11 17:51:02 -05001930 /* Special feature for tricky drivers */
1931 if (drv_callback && drv_callback(rq))
1932 return 1;
1933
Kiyoshi Ueda336cdb42007-12-11 17:40:30 -05001934 add_disk_randomness(rq->rq_disk);
1935
1936 spin_lock_irqsave(q->queue_lock, flags);
Kiyoshi Uedab8286232007-12-11 17:53:24 -05001937 end_that_request_last(rq, error);
Kiyoshi Ueda336cdb42007-12-11 17:40:30 -05001938 spin_unlock_irqrestore(q->queue_lock, flags);
1939
1940 return 0;
1941}
Kiyoshi Uedae19a3ab2007-12-11 17:51:02 -05001942
1943/**
1944 * blk_end_request - Helper function for drivers to complete the request.
1945 * @rq: the request being processed
Randy Dunlap710027a2008-08-19 20:13:11 +02001946 * @error: %0 for success, < %0 for error
Kiyoshi Uedae19a3ab2007-12-11 17:51:02 -05001947 * @nr_bytes: number of bytes to complete
1948 *
1949 * Description:
1950 * Ends I/O on a number of bytes attached to @rq.
1951 * If @rq has leftover, sets it up for the next range of segments.
1952 *
1953 * Return:
Randy Dunlap710027a2008-08-19 20:13:11 +02001954 * %0 - we are done with this request
1955 * %1 - still buffers pending for this request
Kiyoshi Uedae19a3ab2007-12-11 17:51:02 -05001956 **/
Jens Axboe22b13212008-01-31 12:36:19 +01001957int blk_end_request(struct request *rq, int error, unsigned int nr_bytes)
Kiyoshi Uedae19a3ab2007-12-11 17:51:02 -05001958{
Kiyoshi Uedae3a04fe2007-12-11 17:51:46 -05001959 return blk_end_io(rq, error, nr_bytes, 0, NULL);
Kiyoshi Uedae19a3ab2007-12-11 17:51:02 -05001960}
Kiyoshi Ueda336cdb42007-12-11 17:40:30 -05001961EXPORT_SYMBOL_GPL(blk_end_request);
1962
1963/**
1964 * __blk_end_request - Helper function for drivers to complete the request.
1965 * @rq: the request being processed
Randy Dunlap710027a2008-08-19 20:13:11 +02001966 * @error: %0 for success, < %0 for error
Kiyoshi Ueda336cdb42007-12-11 17:40:30 -05001967 * @nr_bytes: number of bytes to complete
1968 *
1969 * Description:
1970 * Must be called with queue lock held unlike blk_end_request().
1971 *
1972 * Return:
Randy Dunlap710027a2008-08-19 20:13:11 +02001973 * %0 - we are done with this request
1974 * %1 - still buffers pending for this request
Kiyoshi Ueda336cdb42007-12-11 17:40:30 -05001975 **/
Jens Axboe22b13212008-01-31 12:36:19 +01001976int __blk_end_request(struct request *rq, int error, unsigned int nr_bytes)
Kiyoshi Ueda336cdb42007-12-11 17:40:30 -05001977{
Jens Axboe60540162008-08-26 13:34:34 +02001978 if (rq->bio && __end_that_request_first(rq, error, nr_bytes))
Jens Axboe051cc392008-08-08 11:06:45 +02001979 return 1;
Kiyoshi Ueda336cdb42007-12-11 17:40:30 -05001980
1981 add_disk_randomness(rq->rq_disk);
1982
Kiyoshi Uedab8286232007-12-11 17:53:24 -05001983 end_that_request_last(rq, error);
Kiyoshi Ueda336cdb42007-12-11 17:40:30 -05001984
1985 return 0;
1986}
1987EXPORT_SYMBOL_GPL(__blk_end_request);
1988
Kiyoshi Uedae19a3ab2007-12-11 17:51:02 -05001989/**
Kiyoshi Uedae3a04fe2007-12-11 17:51:46 -05001990 * blk_end_bidi_request - Helper function for drivers to complete bidi request.
1991 * @rq: the bidi request being processed
Randy Dunlap710027a2008-08-19 20:13:11 +02001992 * @error: %0 for success, < %0 for error
Kiyoshi Uedae3a04fe2007-12-11 17:51:46 -05001993 * @nr_bytes: number of bytes to complete @rq
1994 * @bidi_bytes: number of bytes to complete @rq->next_rq
1995 *
1996 * Description:
1997 * Ends I/O on a number of bytes attached to @rq and @rq->next_rq.
1998 *
1999 * Return:
Randy Dunlap710027a2008-08-19 20:13:11 +02002000 * %0 - we are done with this request
2001 * %1 - still buffers pending for this request
Kiyoshi Uedae3a04fe2007-12-11 17:51:46 -05002002 **/
Jens Axboe22b13212008-01-31 12:36:19 +01002003int blk_end_bidi_request(struct request *rq, int error, unsigned int nr_bytes,
2004 unsigned int bidi_bytes)
Kiyoshi Uedae3a04fe2007-12-11 17:51:46 -05002005{
2006 return blk_end_io(rq, error, nr_bytes, bidi_bytes, NULL);
2007}
2008EXPORT_SYMBOL_GPL(blk_end_bidi_request);
2009
2010/**
Kiyoshi Ueda32fab442008-09-18 10:45:09 -04002011 * blk_update_request - Special helper function for request stacking drivers
2012 * @rq: the request being processed
2013 * @error: %0 for success, < %0 for error
2014 * @nr_bytes: number of bytes to complete @rq
2015 *
2016 * Description:
2017 * Ends I/O on a number of bytes attached to @rq, but doesn't complete
2018 * the request structure even if @rq doesn't have leftover.
2019 * If @rq has leftover, sets it up for the next range of segments.
2020 *
2021 * This special helper function is only for request stacking drivers
2022 * (e.g. request-based dm) so that they can handle partial completion.
2023 * Actual device drivers should use blk_end_request instead.
2024 */
2025void blk_update_request(struct request *rq, int error, unsigned int nr_bytes)
2026{
2027 if (!end_that_request_data(rq, error, nr_bytes, 0)) {
2028 /*
2029 * These members are not updated in end_that_request_data()
2030 * when all bios are completed.
2031 * Update them so that the request stacking driver can find
2032 * how many bytes remain in the request later.
2033 */
2034 rq->nr_sectors = rq->hard_nr_sectors = 0;
2035 rq->current_nr_sectors = rq->hard_cur_sectors = 0;
2036 }
2037}
2038EXPORT_SYMBOL_GPL(blk_update_request);
2039
2040/**
Kiyoshi Uedae19a3ab2007-12-11 17:51:02 -05002041 * blk_end_request_callback - Special helper function for tricky drivers
2042 * @rq: the request being processed
Randy Dunlap710027a2008-08-19 20:13:11 +02002043 * @error: %0 for success, < %0 for error
Kiyoshi Uedae19a3ab2007-12-11 17:51:02 -05002044 * @nr_bytes: number of bytes to complete
2045 * @drv_callback: function called between completion of bios in the request
2046 * and completion of the request.
Randy Dunlap710027a2008-08-19 20:13:11 +02002047 * If the callback returns non %0, this helper returns without
Kiyoshi Uedae19a3ab2007-12-11 17:51:02 -05002048 * completion of the request.
2049 *
2050 * Description:
2051 * Ends I/O on a number of bytes attached to @rq.
2052 * If @rq has leftover, sets it up for the next range of segments.
2053 *
2054 * This special helper function is used only for existing tricky drivers.
2055 * (e.g. cdrom_newpc_intr() of ide-cd)
2056 * This interface will be removed when such drivers are rewritten.
2057 * Don't use this interface in other places anymore.
2058 *
2059 * Return:
Randy Dunlap710027a2008-08-19 20:13:11 +02002060 * %0 - we are done with this request
2061 * %1 - this request is not freed yet.
2062 * this request still has pending buffers or
2063 * the driver doesn't want to finish this request yet.
Kiyoshi Uedae19a3ab2007-12-11 17:51:02 -05002064 **/
Jens Axboe22b13212008-01-31 12:36:19 +01002065int blk_end_request_callback(struct request *rq, int error,
2066 unsigned int nr_bytes,
Kiyoshi Uedae19a3ab2007-12-11 17:51:02 -05002067 int (drv_callback)(struct request *))
2068{
Kiyoshi Uedae3a04fe2007-12-11 17:51:46 -05002069 return blk_end_io(rq, error, nr_bytes, 0, drv_callback);
Kiyoshi Uedae19a3ab2007-12-11 17:51:02 -05002070}
2071EXPORT_SYMBOL_GPL(blk_end_request_callback);
2072
Jens Axboe86db1e22008-01-29 14:53:40 +01002073void blk_rq_bio_prep(struct request_queue *q, struct request *rq,
2074 struct bio *bio)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002075{
David Woodhoused628eae2008-08-09 16:22:17 +01002076 /* Bit 0 (R/W) is identical in rq->cmd_flags and bio->bi_rw, and
2077 we want BIO_RW_AHEAD (bit 1) to imply REQ_FAILFAST (bit 1). */
Jens Axboe4aff5e22006-08-10 08:44:47 +02002078 rq->cmd_flags |= (bio->bi_rw & 3);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002079
David Woodhousefb2dce82008-08-05 18:01:53 +01002080 if (bio_has_data(bio)) {
2081 rq->nr_phys_segments = bio_phys_segments(q, bio);
David Woodhousefb2dce82008-08-05 18:01:53 +01002082 rq->buffer = bio_data(bio);
2083 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07002084 rq->current_nr_sectors = bio_cur_sectors(bio);
2085 rq->hard_cur_sectors = rq->current_nr_sectors;
2086 rq->hard_nr_sectors = rq->nr_sectors = bio_sectors(bio);
Mike Christie0e75f902006-12-01 10:40:55 +01002087 rq->data_len = bio->bi_size;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002088
2089 rq->bio = rq->biotail = bio;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002090
NeilBrown66846572007-08-16 13:31:28 +02002091 if (bio->bi_bdev)
2092 rq->rq_disk = bio->bi_bdev->bd_disk;
2093}
Linus Torvalds1da177e2005-04-16 15:20:36 -07002094
Kiyoshi Uedaef9e3fa2008-10-01 16:12:15 +02002095/**
2096 * blk_lld_busy - Check if underlying low-level drivers of a device are busy
2097 * @q : the queue of the device being checked
2098 *
2099 * Description:
2100 * Check if underlying low-level drivers of a device are busy.
2101 * If the drivers want to export their busy state, they must set own
2102 * exporting function using blk_queue_lld_busy() first.
2103 *
2104 * Basically, this function is used only by request stacking drivers
2105 * to stop dispatching requests to underlying devices when underlying
2106 * devices are busy. This behavior helps more I/O merging on the queue
2107 * of the request stacking driver and prevents I/O throughput regression
2108 * on burst I/O load.
2109 *
2110 * Return:
2111 * 0 - Not busy (The request stacking driver should dispatch request)
2112 * 1 - Busy (The request stacking driver should stop dispatching request)
2113 */
2114int blk_lld_busy(struct request_queue *q)
2115{
2116 if (q->lld_busy_fn)
2117 return q->lld_busy_fn(q);
2118
2119 return 0;
2120}
2121EXPORT_SYMBOL_GPL(blk_lld_busy);
2122
Jens Axboe18887ad2008-07-28 13:08:45 +02002123int kblockd_schedule_work(struct request_queue *q, struct work_struct *work)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002124{
2125 return queue_work(kblockd_workqueue, work);
2126}
Linus Torvalds1da177e2005-04-16 15:20:36 -07002127EXPORT_SYMBOL(kblockd_schedule_work);
2128
Linus Torvalds1da177e2005-04-16 15:20:36 -07002129int __init blk_dev_init(void)
2130{
2131 kblockd_workqueue = create_workqueue("kblockd");
2132 if (!kblockd_workqueue)
2133 panic("Failed to create kblockd\n");
2134
2135 request_cachep = kmem_cache_create("blkdev_requests",
Paul Mundt20c2df82007-07-20 10:11:58 +09002136 sizeof(struct request), 0, SLAB_PANIC, NULL);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002137
Jens Axboe8324aa92008-01-29 14:51:59 +01002138 blk_requestq_cachep = kmem_cache_create("blkdev_queue",
Jens Axboe165125e2007-07-24 09:28:11 +02002139 sizeof(struct request_queue), 0, SLAB_PANIC, NULL);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002140
Linus Torvalds1da177e2005-04-16 15:20:36 -07002141 return 0;
2142}
2143