blob: 5d09f8c56024011588ec1b89e90bff2033a150ca [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
Linus Torvalds1da177e2005-04-16 15:20:36 -07002 * Copyright (C) 1991, 1992 Linus Torvalds
3 * Copyright (C) 1994, Karl Keyte: Added support for disk statistics
4 * Elevator latency, (C) 2000 Andrea Arcangeli <andrea@suse.de> SuSE
5 * Queue request tables / lock, selectable elevator, Jens Axboe <axboe@suse.de>
Jens Axboe6728cb02008-01-31 13:03:55 +01006 * kernel-doc documentation started by NeilBrown <neilb@cse.unsw.edu.au>
7 * - July2000
Linus Torvalds1da177e2005-04-16 15:20:36 -07008 * bio rewrite, highmem i/o, etc, Jens Axboe <axboe@suse.de> - may 2001
9 */
10
11/*
12 * This handles all read/write requests to block devices
13 */
Linus Torvalds1da177e2005-04-16 15:20:36 -070014#include <linux/kernel.h>
15#include <linux/module.h>
16#include <linux/backing-dev.h>
17#include <linux/bio.h>
18#include <linux/blkdev.h>
19#include <linux/highmem.h>
20#include <linux/mm.h>
21#include <linux/kernel_stat.h>
22#include <linux/string.h>
23#include <linux/init.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070024#include <linux/completion.h>
25#include <linux/slab.h>
26#include <linux/swap.h>
27#include <linux/writeback.h>
Andrew Mortonfaccbd4b2006-12-10 02:19:35 -080028#include <linux/task_io_accounting_ops.h>
Jens Axboeff856ba2006-01-09 16:02:34 +010029#include <linux/interrupt.h>
30#include <linux/cpu.h>
Jens Axboe2056a782006-03-23 20:00:26 +010031#include <linux/blktrace_api.h>
Akinobu Mitac17bb492006-12-08 02:39:46 -080032#include <linux/fault-inject.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070033
Jens Axboe8324aa92008-01-29 14:51:59 +010034#include "blk.h"
35
Jens Axboe165125e2007-07-24 09:28:11 +020036static int __make_request(struct request_queue *q, struct bio *bio);
Linus Torvalds1da177e2005-04-16 15:20:36 -070037
38/*
39 * For the allocated request tables
40 */
Adrian Bunk5ece6c52008-02-18 13:45:51 +010041static struct kmem_cache *request_cachep;
Linus Torvalds1da177e2005-04-16 15:20:36 -070042
43/*
44 * For queue allocation
45 */
Jens Axboe6728cb02008-01-31 13:03:55 +010046struct kmem_cache *blk_requestq_cachep;
Linus Torvalds1da177e2005-04-16 15:20:36 -070047
48/*
Linus Torvalds1da177e2005-04-16 15:20:36 -070049 * Controlling structure to kblockd
50 */
Jens Axboeff856ba2006-01-09 16:02:34 +010051static struct workqueue_struct *kblockd_workqueue;
Linus Torvalds1da177e2005-04-16 15:20:36 -070052
Jens Axboeff856ba2006-01-09 16:02:34 +010053static DEFINE_PER_CPU(struct list_head, blk_cpu_done);
54
Jens Axboe26b82562008-01-29 13:54:41 +010055static void drive_stat_acct(struct request *rq, int new_io)
56{
57 int rw = rq_data_dir(rq);
58
59 if (!blk_fs_request(rq) || !rq->rq_disk)
60 return;
61
62 if (!new_io) {
Jerome Marchand6f2576a2008-02-08 11:04:35 +010063 __all_stat_inc(rq->rq_disk, merges[rw], rq->sector);
Jens Axboe26b82562008-01-29 13:54:41 +010064 } else {
Jerome Marchand6f2576a2008-02-08 11:04:35 +010065 struct hd_struct *part = get_part(rq->rq_disk, rq->sector);
Jens Axboe26b82562008-01-29 13:54:41 +010066 disk_round_stats(rq->rq_disk);
67 rq->rq_disk->in_flight++;
Jerome Marchand6f2576a2008-02-08 11:04:35 +010068 if (part) {
69 part_round_stats(part);
70 part->in_flight++;
71 }
Jens Axboe26b82562008-01-29 13:54:41 +010072 }
73}
74
Jens Axboe8324aa92008-01-29 14:51:59 +010075void blk_queue_congestion_threshold(struct request_queue *q)
Linus Torvalds1da177e2005-04-16 15:20:36 -070076{
77 int nr;
78
79 nr = q->nr_requests - (q->nr_requests / 8) + 1;
80 if (nr > q->nr_requests)
81 nr = q->nr_requests;
82 q->nr_congestion_on = nr;
83
84 nr = q->nr_requests - (q->nr_requests / 8) - (q->nr_requests / 16) - 1;
85 if (nr < 1)
86 nr = 1;
87 q->nr_congestion_off = nr;
88}
89
Linus Torvalds1da177e2005-04-16 15:20:36 -070090/**
91 * blk_get_backing_dev_info - get the address of a queue's backing_dev_info
92 * @bdev: device
93 *
94 * Locates the passed device's request queue and returns the address of its
95 * backing_dev_info
96 *
97 * Will return NULL if the request queue cannot be located.
98 */
99struct backing_dev_info *blk_get_backing_dev_info(struct block_device *bdev)
100{
101 struct backing_dev_info *ret = NULL;
Jens Axboe165125e2007-07-24 09:28:11 +0200102 struct request_queue *q = bdev_get_queue(bdev);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700103
104 if (q)
105 ret = &q->backing_dev_info;
106 return ret;
107}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700108EXPORT_SYMBOL(blk_get_backing_dev_info);
109
FUJITA Tomonori2a4aa302008-04-29 09:54:36 +0200110void blk_rq_init(struct request_queue *q, struct request *rq)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700111{
FUJITA Tomonori1afb20f2008-04-25 12:26:28 +0200112 memset(rq, 0, sizeof(*rq));
113
Linus Torvalds1da177e2005-04-16 15:20:36 -0700114 INIT_LIST_HEAD(&rq->queuelist);
Jens Axboeff856ba2006-01-09 16:02:34 +0100115 INIT_LIST_HEAD(&rq->donelist);
Jens Axboe63a71382008-02-08 12:41:03 +0100116 rq->q = q;
117 rq->sector = rq->hard_sector = (sector_t) -1;
Jens Axboe2e662b62006-07-13 11:55:04 +0200118 INIT_HLIST_NODE(&rq->hash);
119 RB_CLEAR_NODE(&rq->rb_node);
FUJITA Tomonorid7e3c322008-04-29 09:54:39 +0200120 rq->cmd = rq->__cmd;
Jens Axboe63a71382008-02-08 12:41:03 +0100121 rq->tag = -1;
Jens Axboe63a71382008-02-08 12:41:03 +0100122 rq->ref_count = 1;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700123}
FUJITA Tomonori2a4aa302008-04-29 09:54:36 +0200124EXPORT_SYMBOL(blk_rq_init);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700125
NeilBrown5bb23a62007-09-27 12:46:13 +0200126static void req_bio_endio(struct request *rq, struct bio *bio,
127 unsigned int nbytes, int error)
Tejun Heo797e7db2006-01-06 09:51:03 +0100128{
Jens Axboe165125e2007-07-24 09:28:11 +0200129 struct request_queue *q = rq->q;
Tejun Heo797e7db2006-01-06 09:51:03 +0100130
NeilBrown5bb23a62007-09-27 12:46:13 +0200131 if (&q->bar_rq != rq) {
132 if (error)
133 clear_bit(BIO_UPTODATE, &bio->bi_flags);
134 else if (!test_bit(BIO_UPTODATE, &bio->bi_flags))
135 error = -EIO;
Tejun Heo797e7db2006-01-06 09:51:03 +0100136
NeilBrown5bb23a62007-09-27 12:46:13 +0200137 if (unlikely(nbytes > bio->bi_size)) {
Jens Axboe6728cb02008-01-31 13:03:55 +0100138 printk(KERN_ERR "%s: want %u bytes done, %u left\n",
NeilBrown5bb23a62007-09-27 12:46:13 +0200139 __FUNCTION__, nbytes, bio->bi_size);
140 nbytes = bio->bi_size;
141 }
Tejun Heo797e7db2006-01-06 09:51:03 +0100142
NeilBrown5bb23a62007-09-27 12:46:13 +0200143 bio->bi_size -= nbytes;
144 bio->bi_sector += (nbytes >> 9);
145 if (bio->bi_size == 0)
NeilBrown6712ecf2007-09-27 12:47:43 +0200146 bio_endio(bio, error);
NeilBrown5bb23a62007-09-27 12:46:13 +0200147 } else {
148
149 /*
150 * Okay, this is the barrier request in progress, just
151 * record the error;
152 */
153 if (error && !q->orderr)
154 q->orderr = error;
155 }
Tejun Heo797e7db2006-01-06 09:51:03 +0100156}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700157
Linus Torvalds1da177e2005-04-16 15:20:36 -0700158void blk_dump_rq_flags(struct request *rq, char *msg)
159{
160 int bit;
161
Jens Axboe6728cb02008-01-31 13:03:55 +0100162 printk(KERN_INFO "%s: dev %s: type=%x, flags=%x\n", msg,
Jens Axboe4aff5e22006-08-10 08:44:47 +0200163 rq->rq_disk ? rq->rq_disk->disk_name : "?", rq->cmd_type,
164 rq->cmd_flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700165
Jens Axboe6728cb02008-01-31 13:03:55 +0100166 printk(KERN_INFO " sector %llu, nr/cnr %lu/%u\n",
167 (unsigned long long)rq->sector,
168 rq->nr_sectors,
169 rq->current_nr_sectors);
170 printk(KERN_INFO " bio %p, biotail %p, buffer %p, data %p, len %u\n",
171 rq->bio, rq->biotail,
172 rq->buffer, rq->data,
173 rq->data_len);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700174
Jens Axboe4aff5e22006-08-10 08:44:47 +0200175 if (blk_pc_request(rq)) {
Jens Axboe6728cb02008-01-31 13:03:55 +0100176 printk(KERN_INFO " cdb: ");
FUJITA Tomonorid34c87e2008-04-29 14:37:52 +0200177 for (bit = 0; bit < BLK_MAX_CDB; bit++)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700178 printk("%02x ", rq->cmd[bit]);
179 printk("\n");
180 }
181}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700182EXPORT_SYMBOL(blk_dump_rq_flags);
183
Linus Torvalds1da177e2005-04-16 15:20:36 -0700184/*
185 * "plug" the device if there are no outstanding requests: this will
186 * force the transfer to start only after we have put all the requests
187 * on the list.
188 *
189 * This is called with interrupts off and no requests on the queue and
190 * with the queue lock held.
191 */
Jens Axboe165125e2007-07-24 09:28:11 +0200192void blk_plug_device(struct request_queue *q)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700193{
194 WARN_ON(!irqs_disabled());
195
196 /*
197 * don't plug a stopped queue, it must be paired with blk_start_queue()
198 * which will restart the queueing
199 */
Coywolf Qi Hunt7daac492006-04-19 10:14:49 +0200200 if (blk_queue_stopped(q))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700201 return;
202
Nick Piggin75ad23b2008-04-29 14:48:33 +0200203 if (!test_bit(QUEUE_FLAG_PLUGGED, &q->queue_flags)) {
204 __set_bit(QUEUE_FLAG_PLUGGED, &q->queue_flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700205 mod_timer(&q->unplug_timer, jiffies + q->unplug_delay);
Jens Axboe2056a782006-03-23 20:00:26 +0100206 blk_add_trace_generic(q, NULL, 0, BLK_TA_PLUG);
207 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700208}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700209EXPORT_SYMBOL(blk_plug_device);
210
211/*
212 * remove the queue from the plugged list, if present. called with
213 * queue lock held and interrupts disabled.
214 */
Jens Axboe165125e2007-07-24 09:28:11 +0200215int blk_remove_plug(struct request_queue *q)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700216{
217 WARN_ON(!irqs_disabled());
218
Nick Piggin75ad23b2008-04-29 14:48:33 +0200219 if (!test_bit(QUEUE_FLAG_PLUGGED, &q->queue_flags))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700220 return 0;
221
Nick Piggin75ad23b2008-04-29 14:48:33 +0200222 queue_flag_clear(QUEUE_FLAG_PLUGGED, q);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700223 del_timer(&q->unplug_timer);
224 return 1;
225}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700226EXPORT_SYMBOL(blk_remove_plug);
227
228/*
229 * remove the plug and let it rip..
230 */
Jens Axboe165125e2007-07-24 09:28:11 +0200231void __generic_unplug_device(struct request_queue *q)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700232{
Coywolf Qi Hunt7daac492006-04-19 10:14:49 +0200233 if (unlikely(blk_queue_stopped(q)))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700234 return;
235
236 if (!blk_remove_plug(q))
237 return;
238
Jens Axboe22e2c502005-06-27 10:55:12 +0200239 q->request_fn(q);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700240}
241EXPORT_SYMBOL(__generic_unplug_device);
242
243/**
244 * generic_unplug_device - fire a request queue
Jens Axboe165125e2007-07-24 09:28:11 +0200245 * @q: The &struct request_queue in question
Linus Torvalds1da177e2005-04-16 15:20:36 -0700246 *
247 * Description:
248 * Linux uses plugging to build bigger requests queues before letting
249 * the device have at them. If a queue is plugged, the I/O scheduler
250 * is still adding and merging requests on the queue. Once the queue
251 * gets unplugged, the request_fn defined for the queue is invoked and
252 * transfers started.
253 **/
Jens Axboe165125e2007-07-24 09:28:11 +0200254void generic_unplug_device(struct request_queue *q)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700255{
256 spin_lock_irq(q->queue_lock);
257 __generic_unplug_device(q);
258 spin_unlock_irq(q->queue_lock);
259}
260EXPORT_SYMBOL(generic_unplug_device);
261
262static void blk_backing_dev_unplug(struct backing_dev_info *bdi,
263 struct page *page)
264{
Jens Axboe165125e2007-07-24 09:28:11 +0200265 struct request_queue *q = bdi->unplug_io_data;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700266
Alan D. Brunelle2ad8b1e2007-11-07 14:26:56 -0500267 blk_unplug(q);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700268}
269
Jens Axboe86db1e22008-01-29 14:53:40 +0100270void blk_unplug_work(struct work_struct *work)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700271{
Jens Axboe165125e2007-07-24 09:28:11 +0200272 struct request_queue *q =
273 container_of(work, struct request_queue, unplug_work);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700274
Jens Axboe2056a782006-03-23 20:00:26 +0100275 blk_add_trace_pdu_int(q, BLK_TA_UNPLUG_IO, NULL,
276 q->rq.count[READ] + q->rq.count[WRITE]);
277
Linus Torvalds1da177e2005-04-16 15:20:36 -0700278 q->unplug_fn(q);
279}
280
Jens Axboe86db1e22008-01-29 14:53:40 +0100281void blk_unplug_timeout(unsigned long data)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700282{
Jens Axboe165125e2007-07-24 09:28:11 +0200283 struct request_queue *q = (struct request_queue *)data;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700284
Jens Axboe2056a782006-03-23 20:00:26 +0100285 blk_add_trace_pdu_int(q, BLK_TA_UNPLUG_TIMER, NULL,
286 q->rq.count[READ] + q->rq.count[WRITE]);
287
Linus Torvalds1da177e2005-04-16 15:20:36 -0700288 kblockd_schedule_work(&q->unplug_work);
289}
290
Alan D. Brunelle2ad8b1e2007-11-07 14:26:56 -0500291void blk_unplug(struct request_queue *q)
292{
293 /*
294 * devices don't necessarily have an ->unplug_fn defined
295 */
296 if (q->unplug_fn) {
297 blk_add_trace_pdu_int(q, BLK_TA_UNPLUG_IO, NULL,
298 q->rq.count[READ] + q->rq.count[WRITE]);
299
300 q->unplug_fn(q);
301 }
302}
303EXPORT_SYMBOL(blk_unplug);
304
Linus Torvalds1da177e2005-04-16 15:20:36 -0700305/**
306 * blk_start_queue - restart a previously stopped queue
Jens Axboe165125e2007-07-24 09:28:11 +0200307 * @q: The &struct request_queue in question
Linus Torvalds1da177e2005-04-16 15:20:36 -0700308 *
309 * Description:
310 * blk_start_queue() will clear the stop flag on the queue, and call
311 * the request_fn for the queue if it was in a stopped state when
312 * entered. Also see blk_stop_queue(). Queue lock must be held.
313 **/
Jens Axboe165125e2007-07-24 09:28:11 +0200314void blk_start_queue(struct request_queue *q)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700315{
Paolo 'Blaisorblade' Giarrussoa038e252006-06-05 12:09:01 +0200316 WARN_ON(!irqs_disabled());
317
Nick Piggin75ad23b2008-04-29 14:48:33 +0200318 queue_flag_clear(QUEUE_FLAG_STOPPED, q);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700319
320 /*
321 * one level of recursion is ok and is much faster than kicking
322 * the unplug handling
323 */
Nick Piggin75ad23b2008-04-29 14:48:33 +0200324 if (!test_bit(QUEUE_FLAG_REENTER, &q->queue_flags)) {
325 queue_flag_set(QUEUE_FLAG_REENTER, q);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700326 q->request_fn(q);
Nick Piggin75ad23b2008-04-29 14:48:33 +0200327 queue_flag_clear(QUEUE_FLAG_REENTER, q);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700328 } else {
329 blk_plug_device(q);
330 kblockd_schedule_work(&q->unplug_work);
331 }
332}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700333EXPORT_SYMBOL(blk_start_queue);
334
335/**
336 * blk_stop_queue - stop a queue
Jens Axboe165125e2007-07-24 09:28:11 +0200337 * @q: The &struct request_queue in question
Linus Torvalds1da177e2005-04-16 15:20:36 -0700338 *
339 * Description:
340 * The Linux block layer assumes that a block driver will consume all
341 * entries on the request queue when the request_fn strategy is called.
342 * Often this will not happen, because of hardware limitations (queue
343 * depth settings). If a device driver gets a 'queue full' response,
344 * or if it simply chooses not to queue more I/O at one point, it can
345 * call this function to prevent the request_fn from being called until
346 * the driver has signalled it's ready to go again. This happens by calling
347 * blk_start_queue() to restart queue operations. Queue lock must be held.
348 **/
Jens Axboe165125e2007-07-24 09:28:11 +0200349void blk_stop_queue(struct request_queue *q)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700350{
351 blk_remove_plug(q);
Nick Piggin75ad23b2008-04-29 14:48:33 +0200352 queue_flag_set(QUEUE_FLAG_STOPPED, q);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700353}
354EXPORT_SYMBOL(blk_stop_queue);
355
356/**
357 * blk_sync_queue - cancel any pending callbacks on a queue
358 * @q: the queue
359 *
360 * Description:
361 * The block layer may perform asynchronous callback activity
362 * on a queue, such as calling the unplug function after a timeout.
363 * A block device may call blk_sync_queue to ensure that any
364 * such activity is cancelled, thus allowing it to release resources
Michael Opdenacker59c51592007-05-09 08:57:56 +0200365 * that the callbacks might use. The caller must already have made sure
Linus Torvalds1da177e2005-04-16 15:20:36 -0700366 * that its ->make_request_fn will not re-add plugging prior to calling
367 * this function.
368 *
369 */
370void blk_sync_queue(struct request_queue *q)
371{
372 del_timer_sync(&q->unplug_timer);
Oleg Nesterovabbeb882007-10-23 15:08:19 +0200373 kblockd_flush_work(&q->unplug_work);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700374}
375EXPORT_SYMBOL(blk_sync_queue);
376
377/**
378 * blk_run_queue - run a single device queue
379 * @q: The queue to run
380 */
Nick Piggin75ad23b2008-04-29 14:48:33 +0200381void __blk_run_queue(struct request_queue *q)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700382{
Linus Torvalds1da177e2005-04-16 15:20:36 -0700383 blk_remove_plug(q);
Jens Axboedac07ec2006-05-11 08:20:16 +0200384
385 /*
386 * Only recurse once to avoid overrunning the stack, let the unplug
387 * handling reinvoke the handler shortly if we already got there.
388 */
389 if (!elv_queue_empty(q)) {
Nick Piggin75ad23b2008-04-29 14:48:33 +0200390 if (!test_bit(QUEUE_FLAG_REENTER, &q->queue_flags)) {
391 queue_flag_set(QUEUE_FLAG_REENTER, q);
Jens Axboedac07ec2006-05-11 08:20:16 +0200392 q->request_fn(q);
Nick Piggin75ad23b2008-04-29 14:48:33 +0200393 queue_flag_clear(QUEUE_FLAG_REENTER, q);
Jens Axboedac07ec2006-05-11 08:20:16 +0200394 } else {
395 blk_plug_device(q);
396 kblockd_schedule_work(&q->unplug_work);
397 }
398 }
Nick Piggin75ad23b2008-04-29 14:48:33 +0200399}
400EXPORT_SYMBOL(__blk_run_queue);
Jens Axboedac07ec2006-05-11 08:20:16 +0200401
Nick Piggin75ad23b2008-04-29 14:48:33 +0200402/**
403 * blk_run_queue - run a single device queue
404 * @q: The queue to run
405 */
406void blk_run_queue(struct request_queue *q)
407{
408 unsigned long flags;
409
410 spin_lock_irqsave(q->queue_lock, flags);
411 __blk_run_queue(q);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700412 spin_unlock_irqrestore(q->queue_lock, flags);
413}
414EXPORT_SYMBOL(blk_run_queue);
415
Jens Axboe165125e2007-07-24 09:28:11 +0200416void blk_put_queue(struct request_queue *q)
Al Viro483f4af2006-03-18 18:34:37 -0500417{
418 kobject_put(&q->kobj);
419}
Al Viro483f4af2006-03-18 18:34:37 -0500420
Jens Axboe6728cb02008-01-31 13:03:55 +0100421void blk_cleanup_queue(struct request_queue *q)
Al Viro483f4af2006-03-18 18:34:37 -0500422{
423 mutex_lock(&q->sysfs_lock);
Nick Piggin75ad23b2008-04-29 14:48:33 +0200424 queue_flag_set_unlocked(QUEUE_FLAG_DEAD, q);
Al Viro483f4af2006-03-18 18:34:37 -0500425 mutex_unlock(&q->sysfs_lock);
426
427 if (q->elevator)
428 elevator_exit(q->elevator);
429
430 blk_put_queue(q);
431}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700432EXPORT_SYMBOL(blk_cleanup_queue);
433
Jens Axboe165125e2007-07-24 09:28:11 +0200434static int blk_init_free_list(struct request_queue *q)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700435{
436 struct request_list *rl = &q->rq;
437
438 rl->count[READ] = rl->count[WRITE] = 0;
439 rl->starved[READ] = rl->starved[WRITE] = 0;
Tejun Heocb98fc82005-10-28 08:29:39 +0200440 rl->elvpriv = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700441 init_waitqueue_head(&rl->wait[READ]);
442 init_waitqueue_head(&rl->wait[WRITE]);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700443
Christoph Lameter19460892005-06-23 00:08:19 -0700444 rl->rq_pool = mempool_create_node(BLKDEV_MIN_RQ, mempool_alloc_slab,
445 mempool_free_slab, request_cachep, q->node);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700446
447 if (!rl->rq_pool)
448 return -ENOMEM;
449
450 return 0;
451}
452
Jens Axboe165125e2007-07-24 09:28:11 +0200453struct request_queue *blk_alloc_queue(gfp_t gfp_mask)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700454{
Christoph Lameter19460892005-06-23 00:08:19 -0700455 return blk_alloc_queue_node(gfp_mask, -1);
456}
457EXPORT_SYMBOL(blk_alloc_queue);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700458
Jens Axboe165125e2007-07-24 09:28:11 +0200459struct request_queue *blk_alloc_queue_node(gfp_t gfp_mask, int node_id)
Christoph Lameter19460892005-06-23 00:08:19 -0700460{
Jens Axboe165125e2007-07-24 09:28:11 +0200461 struct request_queue *q;
Peter Zijlstrae0bf68d2007-10-16 23:25:46 -0700462 int err;
Christoph Lameter19460892005-06-23 00:08:19 -0700463
Jens Axboe8324aa92008-01-29 14:51:59 +0100464 q = kmem_cache_alloc_node(blk_requestq_cachep,
Christoph Lameter94f60302007-07-17 04:03:29 -0700465 gfp_mask | __GFP_ZERO, node_id);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700466 if (!q)
467 return NULL;
468
Peter Zijlstrae0bf68d2007-10-16 23:25:46 -0700469 q->backing_dev_info.unplug_io_fn = blk_backing_dev_unplug;
470 q->backing_dev_info.unplug_io_data = q;
471 err = bdi_init(&q->backing_dev_info);
472 if (err) {
Jens Axboe8324aa92008-01-29 14:51:59 +0100473 kmem_cache_free(blk_requestq_cachep, q);
Peter Zijlstrae0bf68d2007-10-16 23:25:46 -0700474 return NULL;
475 }
476
Linus Torvalds1da177e2005-04-16 15:20:36 -0700477 init_timer(&q->unplug_timer);
Al Viro483f4af2006-03-18 18:34:37 -0500478
Jens Axboe8324aa92008-01-29 14:51:59 +0100479 kobject_init(&q->kobj, &blk_queue_ktype);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700480
Al Viro483f4af2006-03-18 18:34:37 -0500481 mutex_init(&q->sysfs_lock);
482
Linus Torvalds1da177e2005-04-16 15:20:36 -0700483 return q;
484}
Christoph Lameter19460892005-06-23 00:08:19 -0700485EXPORT_SYMBOL(blk_alloc_queue_node);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700486
487/**
488 * blk_init_queue - prepare a request queue for use with a block device
489 * @rfn: The function to be called to process requests that have been
490 * placed on the queue.
491 * @lock: Request queue spin lock
492 *
493 * Description:
494 * If a block device wishes to use the standard request handling procedures,
495 * which sorts requests and coalesces adjacent requests, then it must
496 * call blk_init_queue(). The function @rfn will be called when there
497 * are requests on the queue that need to be processed. If the device
498 * supports plugging, then @rfn may not be called immediately when requests
499 * are available on the queue, but may be called at some time later instead.
500 * Plugged queues are generally unplugged when a buffer belonging to one
501 * of the requests on the queue is needed, or due to memory pressure.
502 *
503 * @rfn is not required, or even expected, to remove all requests off the
504 * queue, but only as many as it can handle at a time. If it does leave
505 * requests on the queue, it is responsible for arranging that the requests
506 * get dealt with eventually.
507 *
508 * The queue spin lock must be held while manipulating the requests on the
Paolo 'Blaisorblade' Giarrussoa038e252006-06-05 12:09:01 +0200509 * request queue; this lock will be taken also from interrupt context, so irq
510 * disabling is needed for it.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700511 *
512 * Function returns a pointer to the initialized request queue, or NULL if
513 * it didn't succeed.
514 *
515 * Note:
516 * blk_init_queue() must be paired with a blk_cleanup_queue() call
517 * when the block device is deactivated (such as at module unload).
518 **/
Christoph Lameter19460892005-06-23 00:08:19 -0700519
Jens Axboe165125e2007-07-24 09:28:11 +0200520struct request_queue *blk_init_queue(request_fn_proc *rfn, spinlock_t *lock)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700521{
Christoph Lameter19460892005-06-23 00:08:19 -0700522 return blk_init_queue_node(rfn, lock, -1);
523}
524EXPORT_SYMBOL(blk_init_queue);
525
Jens Axboe165125e2007-07-24 09:28:11 +0200526struct request_queue *
Christoph Lameter19460892005-06-23 00:08:19 -0700527blk_init_queue_node(request_fn_proc *rfn, spinlock_t *lock, int node_id)
528{
Jens Axboe165125e2007-07-24 09:28:11 +0200529 struct request_queue *q = blk_alloc_queue_node(GFP_KERNEL, node_id);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700530
531 if (!q)
532 return NULL;
533
Christoph Lameter19460892005-06-23 00:08:19 -0700534 q->node = node_id;
Al Viro8669aaf2006-03-18 13:50:00 -0500535 if (blk_init_free_list(q)) {
Jens Axboe8324aa92008-01-29 14:51:59 +0100536 kmem_cache_free(blk_requestq_cachep, q);
Al Viro8669aaf2006-03-18 13:50:00 -0500537 return NULL;
538 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700539
152587d2005-04-12 16:22:06 -0500540 /*
541 * if caller didn't supply a lock, they get per-queue locking with
542 * our embedded lock
543 */
544 if (!lock) {
545 spin_lock_init(&q->__queue_lock);
546 lock = &q->__queue_lock;
547 }
548
Linus Torvalds1da177e2005-04-16 15:20:36 -0700549 q->request_fn = rfn;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700550 q->prep_rq_fn = NULL;
551 q->unplug_fn = generic_unplug_device;
552 q->queue_flags = (1 << QUEUE_FLAG_CLUSTER);
553 q->queue_lock = lock;
554
555 blk_queue_segment_boundary(q, 0xffffffff);
556
557 blk_queue_make_request(q, __make_request);
558 blk_queue_max_segment_size(q, MAX_SEGMENT_SIZE);
559
560 blk_queue_max_hw_segments(q, MAX_HW_SEGMENTS);
561 blk_queue_max_phys_segments(q, MAX_PHYS_SEGMENTS);
562
Alan Stern44ec9542007-02-20 11:01:57 -0500563 q->sg_reserved_size = INT_MAX;
564
Linus Torvalds1da177e2005-04-16 15:20:36 -0700565 /*
566 * all done
567 */
568 if (!elevator_init(q, NULL)) {
569 blk_queue_congestion_threshold(q);
570 return q;
571 }
572
Al Viro8669aaf2006-03-18 13:50:00 -0500573 blk_put_queue(q);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700574 return NULL;
575}
Christoph Lameter19460892005-06-23 00:08:19 -0700576EXPORT_SYMBOL(blk_init_queue_node);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700577
Jens Axboe165125e2007-07-24 09:28:11 +0200578int blk_get_queue(struct request_queue *q)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700579{
Nick Pigginfde6ad22005-06-23 00:08:53 -0700580 if (likely(!test_bit(QUEUE_FLAG_DEAD, &q->queue_flags))) {
Al Viro483f4af2006-03-18 18:34:37 -0500581 kobject_get(&q->kobj);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700582 return 0;
583 }
584
585 return 1;
586}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700587
Jens Axboe165125e2007-07-24 09:28:11 +0200588static inline void blk_free_request(struct request_queue *q, struct request *rq)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700589{
Jens Axboe4aff5e22006-08-10 08:44:47 +0200590 if (rq->cmd_flags & REQ_ELVPRIV)
Tejun Heocb98fc82005-10-28 08:29:39 +0200591 elv_put_request(q, rq);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700592 mempool_free(rq, q->rq.rq_pool);
593}
594
Jens Axboe1ea25ecb2006-07-18 22:24:11 +0200595static struct request *
Jens Axboe165125e2007-07-24 09:28:11 +0200596blk_alloc_request(struct request_queue *q, int rw, int priv, gfp_t gfp_mask)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700597{
598 struct request *rq = mempool_alloc(q->rq.rq_pool, gfp_mask);
599
600 if (!rq)
601 return NULL;
602
FUJITA Tomonori2a4aa302008-04-29 09:54:36 +0200603 blk_rq_init(q, rq);
FUJITA Tomonori1afb20f2008-04-25 12:26:28 +0200604
Linus Torvalds1da177e2005-04-16 15:20:36 -0700605 /*
Jens Axboe4aff5e22006-08-10 08:44:47 +0200606 * first three bits are identical in rq->cmd_flags and bio->bi_rw,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700607 * see bio.h and blkdev.h
608 */
Jens Axboe49171e52006-08-10 08:59:11 +0200609 rq->cmd_flags = rw | REQ_ALLOCED;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700610
Tejun Heocb98fc82005-10-28 08:29:39 +0200611 if (priv) {
Jens Axboecb78b282006-07-28 09:32:57 +0200612 if (unlikely(elv_set_request(q, rq, gfp_mask))) {
Tejun Heocb98fc82005-10-28 08:29:39 +0200613 mempool_free(rq, q->rq.rq_pool);
614 return NULL;
615 }
Jens Axboe4aff5e22006-08-10 08:44:47 +0200616 rq->cmd_flags |= REQ_ELVPRIV;
Tejun Heocb98fc82005-10-28 08:29:39 +0200617 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700618
Tejun Heocb98fc82005-10-28 08:29:39 +0200619 return rq;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700620}
621
622/*
623 * ioc_batching returns true if the ioc is a valid batching request and
624 * should be given priority access to a request.
625 */
Jens Axboe165125e2007-07-24 09:28:11 +0200626static inline int ioc_batching(struct request_queue *q, struct io_context *ioc)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700627{
628 if (!ioc)
629 return 0;
630
631 /*
632 * Make sure the process is able to allocate at least 1 request
633 * even if the batch times out, otherwise we could theoretically
634 * lose wakeups.
635 */
636 return ioc->nr_batch_requests == q->nr_batching ||
637 (ioc->nr_batch_requests > 0
638 && time_before(jiffies, ioc->last_waited + BLK_BATCH_TIME));
639}
640
641/*
642 * ioc_set_batching sets ioc to be a new "batcher" if it is not one. This
643 * will cause the process to be a "batcher" on all queues in the system. This
644 * is the behaviour we want though - once it gets a wakeup it should be given
645 * a nice run.
646 */
Jens Axboe165125e2007-07-24 09:28:11 +0200647static void ioc_set_batching(struct request_queue *q, struct io_context *ioc)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700648{
649 if (!ioc || ioc_batching(q, ioc))
650 return;
651
652 ioc->nr_batch_requests = q->nr_batching;
653 ioc->last_waited = jiffies;
654}
655
Jens Axboe165125e2007-07-24 09:28:11 +0200656static void __freed_request(struct request_queue *q, int rw)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700657{
658 struct request_list *rl = &q->rq;
659
660 if (rl->count[rw] < queue_congestion_off_threshold(q))
Thomas Maier79e2de42006-10-19 23:28:15 -0700661 blk_clear_queue_congested(q, rw);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700662
663 if (rl->count[rw] + 1 <= q->nr_requests) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700664 if (waitqueue_active(&rl->wait[rw]))
665 wake_up(&rl->wait[rw]);
666
667 blk_clear_queue_full(q, rw);
668 }
669}
670
671/*
672 * A request has just been released. Account for it, update the full and
673 * congestion status, wake up any waiters. Called under q->queue_lock.
674 */
Jens Axboe165125e2007-07-24 09:28:11 +0200675static void freed_request(struct request_queue *q, int rw, int priv)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700676{
677 struct request_list *rl = &q->rq;
678
679 rl->count[rw]--;
Tejun Heocb98fc82005-10-28 08:29:39 +0200680 if (priv)
681 rl->elvpriv--;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700682
683 __freed_request(q, rw);
684
685 if (unlikely(rl->starved[rw ^ 1]))
686 __freed_request(q, rw ^ 1);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700687}
688
689#define blkdev_free_rq(list) list_entry((list)->next, struct request, queuelist)
690/*
Nick Piggind6344532005-06-28 20:45:14 -0700691 * Get a free request, queue_lock must be held.
692 * Returns NULL on failure, with queue_lock held.
693 * Returns !NULL on success, with queue_lock *not held*.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700694 */
Jens Axboe165125e2007-07-24 09:28:11 +0200695static struct request *get_request(struct request_queue *q, int rw_flags,
Jens Axboe7749a8d2006-12-13 13:02:26 +0100696 struct bio *bio, gfp_t gfp_mask)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700697{
698 struct request *rq = NULL;
699 struct request_list *rl = &q->rq;
Jens Axboe88ee5ef2005-11-12 11:09:12 +0100700 struct io_context *ioc = NULL;
Jens Axboe7749a8d2006-12-13 13:02:26 +0100701 const int rw = rw_flags & 0x01;
Jens Axboe88ee5ef2005-11-12 11:09:12 +0100702 int may_queue, priv;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700703
Jens Axboe7749a8d2006-12-13 13:02:26 +0100704 may_queue = elv_may_queue(q, rw_flags);
Jens Axboe88ee5ef2005-11-12 11:09:12 +0100705 if (may_queue == ELV_MQUEUE_NO)
706 goto rq_starved;
707
708 if (rl->count[rw]+1 >= queue_congestion_on_threshold(q)) {
709 if (rl->count[rw]+1 >= q->nr_requests) {
Jens Axboeb5deef92006-07-19 23:39:40 +0200710 ioc = current_io_context(GFP_ATOMIC, q->node);
Jens Axboe88ee5ef2005-11-12 11:09:12 +0100711 /*
712 * The queue will fill after this allocation, so set
713 * it as full, and mark this process as "batching".
714 * This process will be allowed to complete a batch of
715 * requests, others will be blocked.
716 */
717 if (!blk_queue_full(q, rw)) {
718 ioc_set_batching(q, ioc);
719 blk_set_queue_full(q, rw);
720 } else {
721 if (may_queue != ELV_MQUEUE_MUST
722 && !ioc_batching(q, ioc)) {
723 /*
724 * The queue is full and the allocating
725 * process is not a "batcher", and not
726 * exempted by the IO scheduler
727 */
728 goto out;
729 }
730 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700731 }
Thomas Maier79e2de42006-10-19 23:28:15 -0700732 blk_set_queue_congested(q, rw);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700733 }
734
Jens Axboe082cf692005-06-28 16:35:11 +0200735 /*
736 * Only allow batching queuers to allocate up to 50% over the defined
737 * limit of requests, otherwise we could have thousands of requests
738 * allocated with any setting of ->nr_requests
739 */
Hugh Dickinsfd782a42005-06-29 15:15:40 +0100740 if (rl->count[rw] >= (3 * q->nr_requests / 2))
Jens Axboe082cf692005-06-28 16:35:11 +0200741 goto out;
Hugh Dickinsfd782a42005-06-29 15:15:40 +0100742
Linus Torvalds1da177e2005-04-16 15:20:36 -0700743 rl->count[rw]++;
744 rl->starved[rw] = 0;
Tejun Heocb98fc82005-10-28 08:29:39 +0200745
Jens Axboe64521d12005-10-28 08:30:39 +0200746 priv = !test_bit(QUEUE_FLAG_ELVSWITCH, &q->queue_flags);
Tejun Heocb98fc82005-10-28 08:29:39 +0200747 if (priv)
748 rl->elvpriv++;
749
Linus Torvalds1da177e2005-04-16 15:20:36 -0700750 spin_unlock_irq(q->queue_lock);
751
Jens Axboe7749a8d2006-12-13 13:02:26 +0100752 rq = blk_alloc_request(q, rw_flags, priv, gfp_mask);
Jens Axboe88ee5ef2005-11-12 11:09:12 +0100753 if (unlikely(!rq)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700754 /*
755 * Allocation failed presumably due to memory. Undo anything
756 * we might have messed up.
757 *
758 * Allocating task should really be put onto the front of the
759 * wait queue, but this is pretty rare.
760 */
761 spin_lock_irq(q->queue_lock);
Tejun Heocb98fc82005-10-28 08:29:39 +0200762 freed_request(q, rw, priv);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700763
764 /*
765 * in the very unlikely event that allocation failed and no
766 * requests for this direction was pending, mark us starved
767 * so that freeing of a request in the other direction will
768 * notice us. another possible fix would be to split the
769 * rq mempool into READ and WRITE
770 */
771rq_starved:
772 if (unlikely(rl->count[rw] == 0))
773 rl->starved[rw] = 1;
774
Linus Torvalds1da177e2005-04-16 15:20:36 -0700775 goto out;
776 }
777
Jens Axboe88ee5ef2005-11-12 11:09:12 +0100778 /*
779 * ioc may be NULL here, and ioc_batching will be false. That's
780 * OK, if the queue is under the request limit then requests need
781 * not count toward the nr_batch_requests limit. There will always
782 * be some limit enforced by BLK_BATCH_TIME.
783 */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700784 if (ioc_batching(q, ioc))
785 ioc->nr_batch_requests--;
Jens Axboe6728cb02008-01-31 13:03:55 +0100786
Jens Axboe2056a782006-03-23 20:00:26 +0100787 blk_add_trace_generic(q, bio, rw, BLK_TA_GETRQ);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700788out:
Linus Torvalds1da177e2005-04-16 15:20:36 -0700789 return rq;
790}
791
792/*
793 * No available requests for this queue, unplug the device and wait for some
794 * requests to become available.
Nick Piggind6344532005-06-28 20:45:14 -0700795 *
796 * Called with q->queue_lock held, and returns with it unlocked.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700797 */
Jens Axboe165125e2007-07-24 09:28:11 +0200798static struct request *get_request_wait(struct request_queue *q, int rw_flags,
Jens Axboe22e2c502005-06-27 10:55:12 +0200799 struct bio *bio)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700800{
Jens Axboe7749a8d2006-12-13 13:02:26 +0100801 const int rw = rw_flags & 0x01;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700802 struct request *rq;
803
Jens Axboe7749a8d2006-12-13 13:02:26 +0100804 rq = get_request(q, rw_flags, bio, GFP_NOIO);
Nick Piggin450991b2005-06-28 20:45:13 -0700805 while (!rq) {
806 DEFINE_WAIT(wait);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700807 struct request_list *rl = &q->rq;
808
809 prepare_to_wait_exclusive(&rl->wait[rw], &wait,
810 TASK_UNINTERRUPTIBLE);
811
Jens Axboe7749a8d2006-12-13 13:02:26 +0100812 rq = get_request(q, rw_flags, bio, GFP_NOIO);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700813
814 if (!rq) {
815 struct io_context *ioc;
816
Jens Axboe2056a782006-03-23 20:00:26 +0100817 blk_add_trace_generic(q, bio, rw, BLK_TA_SLEEPRQ);
818
Nick Piggind6344532005-06-28 20:45:14 -0700819 __generic_unplug_device(q);
820 spin_unlock_irq(q->queue_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700821 io_schedule();
822
823 /*
824 * After sleeping, we become a "batching" process and
825 * will be able to allocate at least one request, and
826 * up to a big batch of them for a small period time.
827 * See ioc_batching, ioc_set_batching
828 */
Jens Axboeb5deef92006-07-19 23:39:40 +0200829 ioc = current_io_context(GFP_NOIO, q->node);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700830 ioc_set_batching(q, ioc);
Nick Piggind6344532005-06-28 20:45:14 -0700831
832 spin_lock_irq(q->queue_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700833 }
834 finish_wait(&rl->wait[rw], &wait);
Nick Piggin450991b2005-06-28 20:45:13 -0700835 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700836
837 return rq;
838}
839
Jens Axboe165125e2007-07-24 09:28:11 +0200840struct request *blk_get_request(struct request_queue *q, int rw, gfp_t gfp_mask)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700841{
842 struct request *rq;
843
844 BUG_ON(rw != READ && rw != WRITE);
845
Nick Piggind6344532005-06-28 20:45:14 -0700846 spin_lock_irq(q->queue_lock);
847 if (gfp_mask & __GFP_WAIT) {
Jens Axboe22e2c502005-06-27 10:55:12 +0200848 rq = get_request_wait(q, rw, NULL);
Nick Piggind6344532005-06-28 20:45:14 -0700849 } else {
Jens Axboe22e2c502005-06-27 10:55:12 +0200850 rq = get_request(q, rw, NULL, gfp_mask);
Nick Piggind6344532005-06-28 20:45:14 -0700851 if (!rq)
852 spin_unlock_irq(q->queue_lock);
853 }
854 /* q->queue_lock is unlocked at this point */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700855
856 return rq;
857}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700858EXPORT_SYMBOL(blk_get_request);
859
860/**
Jens Axboedc72ef42006-07-20 14:54:05 +0200861 * blk_start_queueing - initiate dispatch of requests to device
862 * @q: request queue to kick into gear
863 *
864 * This is basically a helper to remove the need to know whether a queue
865 * is plugged or not if someone just wants to initiate dispatch of requests
866 * for this queue.
867 *
868 * The queue lock must be held with interrupts disabled.
869 */
Jens Axboe165125e2007-07-24 09:28:11 +0200870void blk_start_queueing(struct request_queue *q)
Jens Axboedc72ef42006-07-20 14:54:05 +0200871{
872 if (!blk_queue_plugged(q))
873 q->request_fn(q);
874 else
875 __generic_unplug_device(q);
876}
877EXPORT_SYMBOL(blk_start_queueing);
878
879/**
Linus Torvalds1da177e2005-04-16 15:20:36 -0700880 * blk_requeue_request - put a request back on queue
881 * @q: request queue where request should be inserted
882 * @rq: request to be inserted
883 *
884 * Description:
885 * Drivers often keep queueing requests until the hardware cannot accept
886 * more, when that condition happens we need to put the request back
887 * on the queue. Must be called with queue lock held.
888 */
Jens Axboe165125e2007-07-24 09:28:11 +0200889void blk_requeue_request(struct request_queue *q, struct request *rq)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700890{
Jens Axboe2056a782006-03-23 20:00:26 +0100891 blk_add_trace_rq(q, rq, BLK_TA_REQUEUE);
892
Linus Torvalds1da177e2005-04-16 15:20:36 -0700893 if (blk_rq_tagged(rq))
894 blk_queue_end_tag(q, rq);
895
896 elv_requeue_request(q, rq);
897}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700898EXPORT_SYMBOL(blk_requeue_request);
899
900/**
901 * blk_insert_request - insert a special request in to a request queue
902 * @q: request queue where request should be inserted
903 * @rq: request to be inserted
904 * @at_head: insert request at head or tail of queue
905 * @data: private data
Linus Torvalds1da177e2005-04-16 15:20:36 -0700906 *
907 * Description:
908 * Many block devices need to execute commands asynchronously, so they don't
909 * block the whole kernel from preemption during request execution. This is
910 * accomplished normally by inserting aritficial requests tagged as
911 * REQ_SPECIAL in to the corresponding request queue, and letting them be
912 * scheduled for actual execution by the request queue.
913 *
914 * We have the option of inserting the head or the tail of the queue.
915 * Typically we use the tail for new ioctls and so forth. We use the head
916 * of the queue for things like a QUEUE_FULL message from a device, or a
917 * host that is unable to accept a particular command.
918 */
Jens Axboe165125e2007-07-24 09:28:11 +0200919void blk_insert_request(struct request_queue *q, struct request *rq,
Tejun Heo 867d1192005-04-24 02:06:05 -0500920 int at_head, void *data)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700921{
Tejun Heo 867d1192005-04-24 02:06:05 -0500922 int where = at_head ? ELEVATOR_INSERT_FRONT : ELEVATOR_INSERT_BACK;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700923 unsigned long flags;
924
925 /*
926 * tell I/O scheduler that this isn't a regular read/write (ie it
927 * must not attempt merges on this) and that it acts as a soft
928 * barrier
929 */
Jens Axboe4aff5e22006-08-10 08:44:47 +0200930 rq->cmd_type = REQ_TYPE_SPECIAL;
931 rq->cmd_flags |= REQ_SOFTBARRIER;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700932
933 rq->special = data;
934
935 spin_lock_irqsave(q->queue_lock, flags);
936
937 /*
938 * If command is tagged, release the tag
939 */
Tejun Heo 867d1192005-04-24 02:06:05 -0500940 if (blk_rq_tagged(rq))
941 blk_queue_end_tag(q, rq);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700942
Jerome Marchandb238b3d2007-10-23 15:05:46 +0200943 drive_stat_acct(rq, 1);
Tejun Heo 867d1192005-04-24 02:06:05 -0500944 __elv_add_request(q, rq, where, 0);
Jens Axboedc72ef42006-07-20 14:54:05 +0200945 blk_start_queueing(q);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700946 spin_unlock_irqrestore(q->queue_lock, flags);
947}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700948EXPORT_SYMBOL(blk_insert_request);
949
Linus Torvalds1da177e2005-04-16 15:20:36 -0700950/*
951 * add-request adds a request to the linked list.
952 * queue lock is held and interrupts disabled, as we muck with the
953 * request queue list.
954 */
Jens Axboe6728cb02008-01-31 13:03:55 +0100955static inline void add_request(struct request_queue *q, struct request *req)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700956{
Jerome Marchandb238b3d2007-10-23 15:05:46 +0200957 drive_stat_acct(req, 1);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700958
Linus Torvalds1da177e2005-04-16 15:20:36 -0700959 /*
960 * elevator indicated where it wants this request to be
961 * inserted at elevator_merge time
962 */
963 __elv_add_request(q, req, ELEVATOR_INSERT_SORT, 0);
964}
Jens Axboe6728cb02008-01-31 13:03:55 +0100965
Linus Torvalds1da177e2005-04-16 15:20:36 -0700966/*
967 * disk_round_stats() - Round off the performance stats on a struct
968 * disk_stats.
969 *
970 * The average IO queue length and utilisation statistics are maintained
971 * by observing the current state of the queue length and the amount of
972 * time it has been in this state for.
973 *
974 * Normally, that accounting is done on IO completion, but that can result
975 * in more than a second's worth of IO being accounted for within any one
976 * second, leading to >100% utilisation. To deal with that, we call this
977 * function to do a round-off before returning the results when reading
978 * /proc/diskstats. This accounts immediately for all queue usage up to
979 * the current jiffies and restarts the counters again.
980 */
981void disk_round_stats(struct gendisk *disk)
982{
983 unsigned long now = jiffies;
984
Chen, Kenneth Wb2982642005-10-13 21:49:29 +0200985 if (now == disk->stamp)
986 return;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700987
Chen, Kenneth W20e5c812005-10-13 21:48:42 +0200988 if (disk->in_flight) {
989 __disk_stat_add(disk, time_in_queue,
990 disk->in_flight * (now - disk->stamp));
991 __disk_stat_add(disk, io_ticks, (now - disk->stamp));
992 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700993 disk->stamp = now;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700994}
Jun'ichi "Nick" Nomura3eaf8402006-02-01 03:04:53 -0800995EXPORT_SYMBOL_GPL(disk_round_stats);
996
Jerome Marchand6f2576a2008-02-08 11:04:35 +0100997void part_round_stats(struct hd_struct *part)
998{
999 unsigned long now = jiffies;
1000
1001 if (now == part->stamp)
1002 return;
1003
1004 if (part->in_flight) {
1005 __part_stat_add(part, time_in_queue,
1006 part->in_flight * (now - part->stamp));
1007 __part_stat_add(part, io_ticks, (now - part->stamp));
1008 }
1009 part->stamp = now;
1010}
1011
Linus Torvalds1da177e2005-04-16 15:20:36 -07001012/*
1013 * queue lock must be held
1014 */
Jens Axboe165125e2007-07-24 09:28:11 +02001015void __blk_put_request(struct request_queue *q, struct request *req)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001016{
Linus Torvalds1da177e2005-04-16 15:20:36 -07001017 if (unlikely(!q))
1018 return;
1019 if (unlikely(--req->ref_count))
1020 return;
1021
Tejun Heo8922e162005-10-20 16:23:44 +02001022 elv_completed_request(q, req);
1023
Linus Torvalds1da177e2005-04-16 15:20:36 -07001024 /*
1025 * Request may not have originated from ll_rw_blk. if not,
1026 * it didn't come out of our reserved rq pools
1027 */
Jens Axboe49171e52006-08-10 08:59:11 +02001028 if (req->cmd_flags & REQ_ALLOCED) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001029 int rw = rq_data_dir(req);
Jens Axboe4aff5e22006-08-10 08:44:47 +02001030 int priv = req->cmd_flags & REQ_ELVPRIV;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001031
Linus Torvalds1da177e2005-04-16 15:20:36 -07001032 BUG_ON(!list_empty(&req->queuelist));
Jens Axboe98170642006-07-28 09:23:08 +02001033 BUG_ON(!hlist_unhashed(&req->hash));
Linus Torvalds1da177e2005-04-16 15:20:36 -07001034
1035 blk_free_request(q, req);
Tejun Heocb98fc82005-10-28 08:29:39 +02001036 freed_request(q, rw, priv);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001037 }
1038}
Mike Christie6e39b69e2005-11-11 05:30:24 -06001039EXPORT_SYMBOL_GPL(__blk_put_request);
1040
Linus Torvalds1da177e2005-04-16 15:20:36 -07001041void blk_put_request(struct request *req)
1042{
Tejun Heo8922e162005-10-20 16:23:44 +02001043 unsigned long flags;
Jens Axboe165125e2007-07-24 09:28:11 +02001044 struct request_queue *q = req->q;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001045
Tejun Heo8922e162005-10-20 16:23:44 +02001046 /*
1047 * Gee, IDE calls in w/ NULL q. Fix IDE and remove the
1048 * following if (q) test.
1049 */
1050 if (q) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001051 spin_lock_irqsave(q->queue_lock, flags);
1052 __blk_put_request(q, req);
1053 spin_unlock_irqrestore(q->queue_lock, flags);
1054 }
1055}
Linus Torvalds1da177e2005-04-16 15:20:36 -07001056EXPORT_SYMBOL(blk_put_request);
1057
Jens Axboe86db1e22008-01-29 14:53:40 +01001058void init_request_from_bio(struct request *req, struct bio *bio)
Tejun Heo52d9e672006-01-06 09:49:58 +01001059{
Jens Axboe4aff5e22006-08-10 08:44:47 +02001060 req->cmd_type = REQ_TYPE_FS;
Tejun Heo52d9e672006-01-06 09:49:58 +01001061
1062 /*
1063 * inherit FAILFAST from bio (for read-ahead, and explicit FAILFAST)
1064 */
1065 if (bio_rw_ahead(bio) || bio_failfast(bio))
Jens Axboe4aff5e22006-08-10 08:44:47 +02001066 req->cmd_flags |= REQ_FAILFAST;
Tejun Heo52d9e672006-01-06 09:49:58 +01001067
1068 /*
1069 * REQ_BARRIER implies no merging, but lets make it explicit
1070 */
1071 if (unlikely(bio_barrier(bio)))
Jens Axboe4aff5e22006-08-10 08:44:47 +02001072 req->cmd_flags |= (REQ_HARDBARRIER | REQ_NOMERGE);
Tejun Heo52d9e672006-01-06 09:49:58 +01001073
Jens Axboeb31dc662006-06-13 08:26:10 +02001074 if (bio_sync(bio))
Jens Axboe4aff5e22006-08-10 08:44:47 +02001075 req->cmd_flags |= REQ_RW_SYNC;
Jens Axboe5404bc72006-08-10 09:01:02 +02001076 if (bio_rw_meta(bio))
1077 req->cmd_flags |= REQ_RW_META;
Jens Axboeb31dc662006-06-13 08:26:10 +02001078
Tejun Heo52d9e672006-01-06 09:49:58 +01001079 req->errors = 0;
1080 req->hard_sector = req->sector = bio->bi_sector;
Tejun Heo52d9e672006-01-06 09:49:58 +01001081 req->ioprio = bio_prio(bio);
Tejun Heo52d9e672006-01-06 09:49:58 +01001082 req->start_time = jiffies;
NeilBrownbc1c56f2007-08-16 13:31:30 +02001083 blk_rq_bio_prep(req->q, req, bio);
Tejun Heo52d9e672006-01-06 09:49:58 +01001084}
1085
Jens Axboe165125e2007-07-24 09:28:11 +02001086static int __make_request(struct request_queue *q, struct bio *bio)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001087{
Nick Piggin450991b2005-06-28 20:45:13 -07001088 struct request *req;
Jens Axboe51da90f2006-07-18 04:14:45 +02001089 int el_ret, nr_sectors, barrier, err;
1090 const unsigned short prio = bio_prio(bio);
1091 const int sync = bio_sync(bio);
Jens Axboe7749a8d2006-12-13 13:02:26 +01001092 int rw_flags;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001093
Linus Torvalds1da177e2005-04-16 15:20:36 -07001094 nr_sectors = bio_sectors(bio);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001095
1096 /*
1097 * low level driver can indicate that it wants pages above a
1098 * certain limit bounced to low memory (ie for highmem, or even
1099 * ISA dma in theory)
1100 */
1101 blk_queue_bounce(q, &bio);
1102
Linus Torvalds1da177e2005-04-16 15:20:36 -07001103 barrier = bio_barrier(bio);
Tejun Heo797e7db2006-01-06 09:51:03 +01001104 if (unlikely(barrier) && (q->next_ordered == QUEUE_ORDERED_NONE)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001105 err = -EOPNOTSUPP;
1106 goto end_io;
1107 }
1108
Linus Torvalds1da177e2005-04-16 15:20:36 -07001109 spin_lock_irq(q->queue_lock);
1110
Nick Piggin450991b2005-06-28 20:45:13 -07001111 if (unlikely(barrier) || elv_queue_empty(q))
Linus Torvalds1da177e2005-04-16 15:20:36 -07001112 goto get_rq;
1113
1114 el_ret = elv_merge(q, &req, bio);
1115 switch (el_ret) {
Jens Axboe6728cb02008-01-31 13:03:55 +01001116 case ELEVATOR_BACK_MERGE:
1117 BUG_ON(!rq_mergeable(req));
Linus Torvalds1da177e2005-04-16 15:20:36 -07001118
Jens Axboe6728cb02008-01-31 13:03:55 +01001119 if (!ll_back_merge_fn(q, req, bio))
1120 break;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001121
Jens Axboe6728cb02008-01-31 13:03:55 +01001122 blk_add_trace_bio(q, bio, BLK_TA_BACKMERGE);
Jens Axboe2056a782006-03-23 20:00:26 +01001123
Jens Axboe6728cb02008-01-31 13:03:55 +01001124 req->biotail->bi_next = bio;
1125 req->biotail = bio;
1126 req->nr_sectors = req->hard_nr_sectors += nr_sectors;
1127 req->ioprio = ioprio_best(req->ioprio, prio);
1128 drive_stat_acct(req, 0);
1129 if (!attempt_back_merge(q, req))
1130 elv_merged_request(q, req, el_ret);
1131 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001132
Jens Axboe6728cb02008-01-31 13:03:55 +01001133 case ELEVATOR_FRONT_MERGE:
1134 BUG_ON(!rq_mergeable(req));
Linus Torvalds1da177e2005-04-16 15:20:36 -07001135
Jens Axboe6728cb02008-01-31 13:03:55 +01001136 if (!ll_front_merge_fn(q, req, bio))
1137 break;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001138
Jens Axboe6728cb02008-01-31 13:03:55 +01001139 blk_add_trace_bio(q, bio, BLK_TA_FRONTMERGE);
Jens Axboe2056a782006-03-23 20:00:26 +01001140
Jens Axboe6728cb02008-01-31 13:03:55 +01001141 bio->bi_next = req->bio;
1142 req->bio = bio;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001143
Jens Axboe6728cb02008-01-31 13:03:55 +01001144 /*
1145 * may not be valid. if the low level driver said
1146 * it didn't need a bounce buffer then it better
1147 * not touch req->buffer either...
1148 */
1149 req->buffer = bio_data(bio);
1150 req->current_nr_sectors = bio_cur_sectors(bio);
1151 req->hard_cur_sectors = req->current_nr_sectors;
1152 req->sector = req->hard_sector = bio->bi_sector;
1153 req->nr_sectors = req->hard_nr_sectors += nr_sectors;
1154 req->ioprio = ioprio_best(req->ioprio, prio);
1155 drive_stat_acct(req, 0);
1156 if (!attempt_front_merge(q, req))
1157 elv_merged_request(q, req, el_ret);
1158 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001159
Jens Axboe6728cb02008-01-31 13:03:55 +01001160 /* ELV_NO_MERGE: elevator says don't/can't merge. */
1161 default:
1162 ;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001163 }
1164
Linus Torvalds1da177e2005-04-16 15:20:36 -07001165get_rq:
Nick Piggin450991b2005-06-28 20:45:13 -07001166 /*
Jens Axboe7749a8d2006-12-13 13:02:26 +01001167 * This sync check and mask will be re-done in init_request_from_bio(),
1168 * but we need to set it earlier to expose the sync flag to the
1169 * rq allocator and io schedulers.
1170 */
1171 rw_flags = bio_data_dir(bio);
1172 if (sync)
1173 rw_flags |= REQ_RW_SYNC;
1174
1175 /*
Nick Piggin450991b2005-06-28 20:45:13 -07001176 * Grab a free request. This is might sleep but can not fail.
Nick Piggind6344532005-06-28 20:45:14 -07001177 * Returns with the queue unlocked.
Nick Piggin450991b2005-06-28 20:45:13 -07001178 */
Jens Axboe7749a8d2006-12-13 13:02:26 +01001179 req = get_request_wait(q, rw_flags, bio);
Nick Piggind6344532005-06-28 20:45:14 -07001180
Nick Piggin450991b2005-06-28 20:45:13 -07001181 /*
1182 * After dropping the lock and possibly sleeping here, our request
1183 * may now be mergeable after it had proven unmergeable (above).
1184 * We don't worry about that case for efficiency. It won't happen
1185 * often, and the elevators are able to handle it.
1186 */
Tejun Heo52d9e672006-01-06 09:49:58 +01001187 init_request_from_bio(req, bio);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001188
Nick Piggin450991b2005-06-28 20:45:13 -07001189 spin_lock_irq(q->queue_lock);
1190 if (elv_queue_empty(q))
1191 blk_plug_device(q);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001192 add_request(q, req);
1193out:
Jens Axboe4a534f92005-04-16 15:25:40 -07001194 if (sync)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001195 __generic_unplug_device(q);
1196
1197 spin_unlock_irq(q->queue_lock);
1198 return 0;
1199
1200end_io:
NeilBrown6712ecf2007-09-27 12:47:43 +02001201 bio_endio(bio, err);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001202 return 0;
1203}
1204
1205/*
1206 * If bio->bi_dev is a partition, remap the location
1207 */
1208static inline void blk_partition_remap(struct bio *bio)
1209{
1210 struct block_device *bdev = bio->bi_bdev;
1211
Jens Axboebf2de6f2007-09-27 13:01:25 +02001212 if (bio_sectors(bio) && bdev != bdev->bd_contains) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001213 struct hd_struct *p = bdev->bd_part;
Jens Axboea3623572005-11-01 09:26:16 +01001214
Linus Torvalds1da177e2005-04-16 15:20:36 -07001215 bio->bi_sector += p->start_sect;
1216 bio->bi_bdev = bdev->bd_contains;
Alan D. Brunellec7149d62007-08-07 15:30:23 +02001217
1218 blk_add_trace_remap(bdev_get_queue(bio->bi_bdev), bio,
1219 bdev->bd_dev, bio->bi_sector,
1220 bio->bi_sector - p->start_sect);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001221 }
1222}
1223
Linus Torvalds1da177e2005-04-16 15:20:36 -07001224static void handle_bad_sector(struct bio *bio)
1225{
1226 char b[BDEVNAME_SIZE];
1227
1228 printk(KERN_INFO "attempt to access beyond end of device\n");
1229 printk(KERN_INFO "%s: rw=%ld, want=%Lu, limit=%Lu\n",
1230 bdevname(bio->bi_bdev, b),
1231 bio->bi_rw,
1232 (unsigned long long)bio->bi_sector + bio_sectors(bio),
1233 (long long)(bio->bi_bdev->bd_inode->i_size >> 9));
1234
1235 set_bit(BIO_EOF, &bio->bi_flags);
1236}
1237
Akinobu Mitac17bb492006-12-08 02:39:46 -08001238#ifdef CONFIG_FAIL_MAKE_REQUEST
1239
1240static DECLARE_FAULT_ATTR(fail_make_request);
1241
1242static int __init setup_fail_make_request(char *str)
1243{
1244 return setup_fault_attr(&fail_make_request, str);
1245}
1246__setup("fail_make_request=", setup_fail_make_request);
1247
1248static int should_fail_request(struct bio *bio)
1249{
1250 if ((bio->bi_bdev->bd_disk->flags & GENHD_FL_FAIL) ||
1251 (bio->bi_bdev->bd_part && bio->bi_bdev->bd_part->make_it_fail))
1252 return should_fail(&fail_make_request, bio->bi_size);
1253
1254 return 0;
1255}
1256
1257static int __init fail_make_request_debugfs(void)
1258{
1259 return init_fault_attr_dentries(&fail_make_request,
1260 "fail_make_request");
1261}
1262
1263late_initcall(fail_make_request_debugfs);
1264
1265#else /* CONFIG_FAIL_MAKE_REQUEST */
1266
1267static inline int should_fail_request(struct bio *bio)
1268{
1269 return 0;
1270}
1271
1272#endif /* CONFIG_FAIL_MAKE_REQUEST */
1273
Jens Axboec07e2b42007-07-18 13:27:58 +02001274/*
1275 * Check whether this bio extends beyond the end of the device.
1276 */
1277static inline int bio_check_eod(struct bio *bio, unsigned int nr_sectors)
1278{
1279 sector_t maxsector;
1280
1281 if (!nr_sectors)
1282 return 0;
1283
1284 /* Test device or partition size, when known. */
1285 maxsector = bio->bi_bdev->bd_inode->i_size >> 9;
1286 if (maxsector) {
1287 sector_t sector = bio->bi_sector;
1288
1289 if (maxsector < nr_sectors || maxsector - nr_sectors < sector) {
1290 /*
1291 * This may well happen - the kernel calls bread()
1292 * without checking the size of the device, e.g., when
1293 * mounting a device.
1294 */
1295 handle_bad_sector(bio);
1296 return 1;
1297 }
1298 }
1299
1300 return 0;
1301}
1302
Linus Torvalds1da177e2005-04-16 15:20:36 -07001303/**
1304 * generic_make_request: hand a buffer to its device driver for I/O
1305 * @bio: The bio describing the location in memory and on the device.
1306 *
1307 * generic_make_request() is used to make I/O requests of block
1308 * devices. It is passed a &struct bio, which describes the I/O that needs
1309 * to be done.
1310 *
1311 * generic_make_request() does not return any status. The
1312 * success/failure status of the request, along with notification of
1313 * completion, is delivered asynchronously through the bio->bi_end_io
1314 * function described (one day) else where.
1315 *
1316 * The caller of generic_make_request must make sure that bi_io_vec
1317 * are set to describe the memory buffer, and that bi_dev and bi_sector are
1318 * set to describe the device address, and the
1319 * bi_end_io and optionally bi_private are set to describe how
1320 * completion notification should be signaled.
1321 *
1322 * generic_make_request and the drivers it calls may use bi_next if this
1323 * bio happens to be merged with someone else, and may change bi_dev and
1324 * bi_sector for remaps as it sees fit. So the values of these fields
1325 * should NOT be depended on after the call to generic_make_request.
1326 */
Neil Brownd89d8792007-05-01 09:53:42 +02001327static inline void __generic_make_request(struct bio *bio)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001328{
Jens Axboe165125e2007-07-24 09:28:11 +02001329 struct request_queue *q;
NeilBrown5ddfe962006-10-30 22:07:21 -08001330 sector_t old_sector;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001331 int ret, nr_sectors = bio_sectors(bio);
Jens Axboe2056a782006-03-23 20:00:26 +01001332 dev_t old_dev;
Jens Axboe51fd77b2007-11-02 08:49:08 +01001333 int err = -EIO;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001334
1335 might_sleep();
Linus Torvalds1da177e2005-04-16 15:20:36 -07001336
Jens Axboec07e2b42007-07-18 13:27:58 +02001337 if (bio_check_eod(bio, nr_sectors))
1338 goto end_io;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001339
1340 /*
1341 * Resolve the mapping until finished. (drivers are
1342 * still free to implement/resolve their own stacking
1343 * by explicitly returning 0)
1344 *
1345 * NOTE: we don't repeat the blk_size check for each new device.
1346 * Stacking drivers are expected to know what they are doing.
1347 */
NeilBrown5ddfe962006-10-30 22:07:21 -08001348 old_sector = -1;
Jens Axboe2056a782006-03-23 20:00:26 +01001349 old_dev = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001350 do {
1351 char b[BDEVNAME_SIZE];
1352
1353 q = bdev_get_queue(bio->bi_bdev);
1354 if (!q) {
1355 printk(KERN_ERR
1356 "generic_make_request: Trying to access "
1357 "nonexistent block-device %s (%Lu)\n",
1358 bdevname(bio->bi_bdev, b),
1359 (long long) bio->bi_sector);
1360end_io:
Jens Axboe51fd77b2007-11-02 08:49:08 +01001361 bio_endio(bio, err);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001362 break;
1363 }
1364
Jens Axboe4fa253f2007-07-18 13:13:10 +02001365 if (unlikely(nr_sectors > q->max_hw_sectors)) {
Jens Axboe6728cb02008-01-31 13:03:55 +01001366 printk(KERN_ERR "bio too big device %s (%u > %u)\n",
Linus Torvalds1da177e2005-04-16 15:20:36 -07001367 bdevname(bio->bi_bdev, b),
1368 bio_sectors(bio),
1369 q->max_hw_sectors);
1370 goto end_io;
1371 }
1372
Nick Pigginfde6ad22005-06-23 00:08:53 -07001373 if (unlikely(test_bit(QUEUE_FLAG_DEAD, &q->queue_flags)))
Linus Torvalds1da177e2005-04-16 15:20:36 -07001374 goto end_io;
1375
Akinobu Mitac17bb492006-12-08 02:39:46 -08001376 if (should_fail_request(bio))
1377 goto end_io;
1378
Linus Torvalds1da177e2005-04-16 15:20:36 -07001379 /*
1380 * If this device has partitions, remap block n
1381 * of partition p to block n+start(p) of the disk.
1382 */
1383 blk_partition_remap(bio);
1384
NeilBrown5ddfe962006-10-30 22:07:21 -08001385 if (old_sector != -1)
Jens Axboe4fa253f2007-07-18 13:13:10 +02001386 blk_add_trace_remap(q, bio, old_dev, bio->bi_sector,
NeilBrown5ddfe962006-10-30 22:07:21 -08001387 old_sector);
Jens Axboe2056a782006-03-23 20:00:26 +01001388
1389 blk_add_trace_bio(q, bio, BLK_TA_QUEUE);
1390
NeilBrown5ddfe962006-10-30 22:07:21 -08001391 old_sector = bio->bi_sector;
Jens Axboe2056a782006-03-23 20:00:26 +01001392 old_dev = bio->bi_bdev->bd_dev;
1393
Jens Axboec07e2b42007-07-18 13:27:58 +02001394 if (bio_check_eod(bio, nr_sectors))
1395 goto end_io;
Jens Axboe51fd77b2007-11-02 08:49:08 +01001396 if (bio_empty_barrier(bio) && !q->prepare_flush_fn) {
1397 err = -EOPNOTSUPP;
1398 goto end_io;
1399 }
NeilBrown5ddfe962006-10-30 22:07:21 -08001400
Linus Torvalds1da177e2005-04-16 15:20:36 -07001401 ret = q->make_request_fn(q, bio);
1402 } while (ret);
1403}
1404
Neil Brownd89d8792007-05-01 09:53:42 +02001405/*
1406 * We only want one ->make_request_fn to be active at a time,
1407 * else stack usage with stacked devices could be a problem.
1408 * So use current->bio_{list,tail} to keep a list of requests
1409 * submited by a make_request_fn function.
1410 * current->bio_tail is also used as a flag to say if
1411 * generic_make_request is currently active in this task or not.
1412 * If it is NULL, then no make_request is active. If it is non-NULL,
1413 * then a make_request is active, and new requests should be added
1414 * at the tail
1415 */
1416void generic_make_request(struct bio *bio)
1417{
1418 if (current->bio_tail) {
1419 /* make_request is active */
1420 *(current->bio_tail) = bio;
1421 bio->bi_next = NULL;
1422 current->bio_tail = &bio->bi_next;
1423 return;
1424 }
1425 /* following loop may be a bit non-obvious, and so deserves some
1426 * explanation.
1427 * Before entering the loop, bio->bi_next is NULL (as all callers
1428 * ensure that) so we have a list with a single bio.
1429 * We pretend that we have just taken it off a longer list, so
1430 * we assign bio_list to the next (which is NULL) and bio_tail
1431 * to &bio_list, thus initialising the bio_list of new bios to be
1432 * added. __generic_make_request may indeed add some more bios
1433 * through a recursive call to generic_make_request. If it
1434 * did, we find a non-NULL value in bio_list and re-enter the loop
1435 * from the top. In this case we really did just take the bio
1436 * of the top of the list (no pretending) and so fixup bio_list and
1437 * bio_tail or bi_next, and call into __generic_make_request again.
1438 *
1439 * The loop was structured like this to make only one call to
1440 * __generic_make_request (which is important as it is large and
1441 * inlined) and to keep the structure simple.
1442 */
1443 BUG_ON(bio->bi_next);
1444 do {
1445 current->bio_list = bio->bi_next;
1446 if (bio->bi_next == NULL)
1447 current->bio_tail = &current->bio_list;
1448 else
1449 bio->bi_next = NULL;
1450 __generic_make_request(bio);
1451 bio = current->bio_list;
1452 } while (bio);
1453 current->bio_tail = NULL; /* deactivate */
1454}
Linus Torvalds1da177e2005-04-16 15:20:36 -07001455EXPORT_SYMBOL(generic_make_request);
1456
1457/**
1458 * submit_bio: submit a bio to the block device layer for I/O
1459 * @rw: whether to %READ or %WRITE, or maybe to %READA (read ahead)
1460 * @bio: The &struct bio which describes the I/O
1461 *
1462 * submit_bio() is very similar in purpose to generic_make_request(), and
1463 * uses that function to do most of the work. Both are fairly rough
1464 * interfaces, @bio must be presetup and ready for I/O.
1465 *
1466 */
1467void submit_bio(int rw, struct bio *bio)
1468{
1469 int count = bio_sectors(bio);
1470
Jens Axboe22e2c502005-06-27 10:55:12 +02001471 bio->bi_rw |= rw;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001472
Jens Axboebf2de6f2007-09-27 13:01:25 +02001473 /*
1474 * If it's a regular read/write or a barrier with data attached,
1475 * go through the normal accounting stuff before submission.
1476 */
1477 if (!bio_empty_barrier(bio)) {
1478
1479 BIO_BUG_ON(!bio->bi_size);
1480 BIO_BUG_ON(!bio->bi_io_vec);
1481
1482 if (rw & WRITE) {
1483 count_vm_events(PGPGOUT, count);
1484 } else {
1485 task_io_account_read(bio->bi_size);
1486 count_vm_events(PGPGIN, count);
1487 }
1488
1489 if (unlikely(block_dump)) {
1490 char b[BDEVNAME_SIZE];
1491 printk(KERN_DEBUG "%s(%d): %s block %Lu on %s\n",
Pavel Emelyanovba25f9d2007-10-18 23:40:40 -07001492 current->comm, task_pid_nr(current),
Jens Axboebf2de6f2007-09-27 13:01:25 +02001493 (rw & WRITE) ? "WRITE" : "READ",
1494 (unsigned long long)bio->bi_sector,
Jens Axboe6728cb02008-01-31 13:03:55 +01001495 bdevname(bio->bi_bdev, b));
Jens Axboebf2de6f2007-09-27 13:01:25 +02001496 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001497 }
1498
1499 generic_make_request(bio);
1500}
Linus Torvalds1da177e2005-04-16 15:20:36 -07001501EXPORT_SYMBOL(submit_bio);
1502
Kiyoshi Ueda3bcddea2007-12-11 17:52:28 -05001503/**
1504 * __end_that_request_first - end I/O on a request
1505 * @req: the request being processed
Kiyoshi Ueda5450d3e2007-12-11 17:53:03 -05001506 * @error: 0 for success, < 0 for error
Kiyoshi Ueda3bcddea2007-12-11 17:52:28 -05001507 * @nr_bytes: number of bytes to complete
1508 *
1509 * Description:
1510 * Ends I/O on a number of bytes attached to @req, and sets it up
1511 * for the next range of segments (if any) in the cluster.
1512 *
1513 * Return:
1514 * 0 - we are done with this request, call end_that_request_last()
1515 * 1 - still buffers pending for this request
1516 **/
Kiyoshi Ueda5450d3e2007-12-11 17:53:03 -05001517static int __end_that_request_first(struct request *req, int error,
Linus Torvalds1da177e2005-04-16 15:20:36 -07001518 int nr_bytes)
1519{
Kiyoshi Ueda5450d3e2007-12-11 17:53:03 -05001520 int total_bytes, bio_nbytes, next_idx = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001521 struct bio *bio;
1522
Jens Axboe2056a782006-03-23 20:00:26 +01001523 blk_add_trace_rq(req->q, req, BLK_TA_COMPLETE);
1524
Linus Torvalds1da177e2005-04-16 15:20:36 -07001525 /*
Linus Torvalds1da177e2005-04-16 15:20:36 -07001526 * for a REQ_BLOCK_PC request, we want to carry any eventual
1527 * sense key with us all the way through
1528 */
1529 if (!blk_pc_request(req))
1530 req->errors = 0;
1531
Jens Axboe6728cb02008-01-31 13:03:55 +01001532 if (error && (blk_fs_request(req) && !(req->cmd_flags & REQ_QUIET))) {
1533 printk(KERN_ERR "end_request: I/O error, dev %s, sector %llu\n",
Linus Torvalds1da177e2005-04-16 15:20:36 -07001534 req->rq_disk ? req->rq_disk->disk_name : "?",
1535 (unsigned long long)req->sector);
1536 }
1537
Jens Axboed72d9042005-11-01 08:35:42 +01001538 if (blk_fs_request(req) && req->rq_disk) {
Jens Axboea3623572005-11-01 09:26:16 +01001539 const int rw = rq_data_dir(req);
1540
Jerome Marchand6f2576a2008-02-08 11:04:35 +01001541 all_stat_add(req->rq_disk, sectors[rw],
1542 nr_bytes >> 9, req->sector);
Jens Axboed72d9042005-11-01 08:35:42 +01001543 }
1544
Linus Torvalds1da177e2005-04-16 15:20:36 -07001545 total_bytes = bio_nbytes = 0;
1546 while ((bio = req->bio) != NULL) {
1547 int nbytes;
1548
Jens Axboebf2de6f2007-09-27 13:01:25 +02001549 /*
1550 * For an empty barrier request, the low level driver must
1551 * store a potential error location in ->sector. We pass
1552 * that back up in ->bi_sector.
1553 */
1554 if (blk_empty_barrier(req))
1555 bio->bi_sector = req->sector;
1556
Linus Torvalds1da177e2005-04-16 15:20:36 -07001557 if (nr_bytes >= bio->bi_size) {
1558 req->bio = bio->bi_next;
1559 nbytes = bio->bi_size;
NeilBrown5bb23a62007-09-27 12:46:13 +02001560 req_bio_endio(req, bio, nbytes, error);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001561 next_idx = 0;
1562 bio_nbytes = 0;
1563 } else {
1564 int idx = bio->bi_idx + next_idx;
1565
1566 if (unlikely(bio->bi_idx >= bio->bi_vcnt)) {
1567 blk_dump_rq_flags(req, "__end_that");
Jens Axboe6728cb02008-01-31 13:03:55 +01001568 printk(KERN_ERR "%s: bio idx %d >= vcnt %d\n",
1569 __FUNCTION__, bio->bi_idx,
1570 bio->bi_vcnt);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001571 break;
1572 }
1573
1574 nbytes = bio_iovec_idx(bio, idx)->bv_len;
1575 BIO_BUG_ON(nbytes > bio->bi_size);
1576
1577 /*
1578 * not a complete bvec done
1579 */
1580 if (unlikely(nbytes > nr_bytes)) {
1581 bio_nbytes += nr_bytes;
1582 total_bytes += nr_bytes;
1583 break;
1584 }
1585
1586 /*
1587 * advance to the next vector
1588 */
1589 next_idx++;
1590 bio_nbytes += nbytes;
1591 }
1592
1593 total_bytes += nbytes;
1594 nr_bytes -= nbytes;
1595
Jens Axboe6728cb02008-01-31 13:03:55 +01001596 bio = req->bio;
1597 if (bio) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001598 /*
1599 * end more in this run, or just return 'not-done'
1600 */
1601 if (unlikely(nr_bytes <= 0))
1602 break;
1603 }
1604 }
1605
1606 /*
1607 * completely done
1608 */
1609 if (!req->bio)
1610 return 0;
1611
1612 /*
1613 * if the request wasn't completed, update state
1614 */
1615 if (bio_nbytes) {
NeilBrown5bb23a62007-09-27 12:46:13 +02001616 req_bio_endio(req, bio, bio_nbytes, error);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001617 bio->bi_idx += next_idx;
1618 bio_iovec(bio)->bv_offset += nr_bytes;
1619 bio_iovec(bio)->bv_len -= nr_bytes;
1620 }
1621
1622 blk_recalc_rq_sectors(req, total_bytes >> 9);
1623 blk_recalc_rq_segments(req);
1624 return 1;
1625}
1626
Linus Torvalds1da177e2005-04-16 15:20:36 -07001627/*
Jens Axboeff856ba2006-01-09 16:02:34 +01001628 * splice the completion data to a local structure and hand off to
1629 * process_completion_queue() to complete the requests
1630 */
1631static void blk_done_softirq(struct softirq_action *h)
1632{
Oleg Nesterov626ab0e2006-06-23 02:05:55 -07001633 struct list_head *cpu_list, local_list;
Jens Axboeff856ba2006-01-09 16:02:34 +01001634
1635 local_irq_disable();
1636 cpu_list = &__get_cpu_var(blk_cpu_done);
Oleg Nesterov626ab0e2006-06-23 02:05:55 -07001637 list_replace_init(cpu_list, &local_list);
Jens Axboeff856ba2006-01-09 16:02:34 +01001638 local_irq_enable();
1639
1640 while (!list_empty(&local_list)) {
Jens Axboe6728cb02008-01-31 13:03:55 +01001641 struct request *rq;
Jens Axboeff856ba2006-01-09 16:02:34 +01001642
Jens Axboe6728cb02008-01-31 13:03:55 +01001643 rq = list_entry(local_list.next, struct request, donelist);
Jens Axboeff856ba2006-01-09 16:02:34 +01001644 list_del_init(&rq->donelist);
1645 rq->q->softirq_done_fn(rq);
1646 }
1647}
1648
Jens Axboe6728cb02008-01-31 13:03:55 +01001649static int __cpuinit blk_cpu_notify(struct notifier_block *self,
1650 unsigned long action, void *hcpu)
Jens Axboeff856ba2006-01-09 16:02:34 +01001651{
1652 /*
1653 * If a CPU goes away, splice its entries to the current CPU
1654 * and trigger a run of the softirq
1655 */
Rafael J. Wysocki8bb78442007-05-09 02:35:10 -07001656 if (action == CPU_DEAD || action == CPU_DEAD_FROZEN) {
Jens Axboeff856ba2006-01-09 16:02:34 +01001657 int cpu = (unsigned long) hcpu;
1658
1659 local_irq_disable();
1660 list_splice_init(&per_cpu(blk_cpu_done, cpu),
1661 &__get_cpu_var(blk_cpu_done));
1662 raise_softirq_irqoff(BLOCK_SOFTIRQ);
1663 local_irq_enable();
1664 }
1665
1666 return NOTIFY_OK;
1667}
1668
1669
Satyam Sharmadb47d472007-08-23 09:29:40 +02001670static struct notifier_block blk_cpu_notifier __cpuinitdata = {
Jens Axboeff856ba2006-01-09 16:02:34 +01001671 .notifier_call = blk_cpu_notify,
1672};
1673
Jens Axboeff856ba2006-01-09 16:02:34 +01001674/**
1675 * blk_complete_request - end I/O on a request
1676 * @req: the request being processed
1677 *
1678 * Description:
1679 * Ends all I/O on a request. It does not handle partial completions,
Andreas Mohrd6e05ed2006-06-26 18:35:02 +02001680 * unless the driver actually implements this in its completion callback
Jens Axboe4fa253f2007-07-18 13:13:10 +02001681 * through requeueing. The actual completion happens out-of-order,
Jens Axboeff856ba2006-01-09 16:02:34 +01001682 * through a softirq handler. The user must have registered a completion
1683 * callback through blk_queue_softirq_done().
1684 **/
1685
1686void blk_complete_request(struct request *req)
1687{
1688 struct list_head *cpu_list;
1689 unsigned long flags;
1690
1691 BUG_ON(!req->q->softirq_done_fn);
Jens Axboe6728cb02008-01-31 13:03:55 +01001692
Jens Axboeff856ba2006-01-09 16:02:34 +01001693 local_irq_save(flags);
1694
1695 cpu_list = &__get_cpu_var(blk_cpu_done);
1696 list_add_tail(&req->donelist, cpu_list);
1697 raise_softirq_irqoff(BLOCK_SOFTIRQ);
1698
1699 local_irq_restore(flags);
1700}
Jens Axboeff856ba2006-01-09 16:02:34 +01001701EXPORT_SYMBOL(blk_complete_request);
Jens Axboe6728cb02008-01-31 13:03:55 +01001702
Jens Axboeff856ba2006-01-09 16:02:34 +01001703/*
Linus Torvalds1da177e2005-04-16 15:20:36 -07001704 * queue lock must be held
1705 */
Kiyoshi Ueda5450d3e2007-12-11 17:53:03 -05001706static void end_that_request_last(struct request *req, int error)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001707{
1708 struct gendisk *disk = req->rq_disk;
Tejun Heo8ffdc652006-01-06 09:49:03 +01001709
Kiyoshi Uedab8286232007-12-11 17:53:24 -05001710 if (blk_rq_tagged(req))
1711 blk_queue_end_tag(req->q, req);
1712
1713 if (blk_queued_rq(req))
1714 blkdev_dequeue_request(req);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001715
1716 if (unlikely(laptop_mode) && blk_fs_request(req))
1717 laptop_io_completion();
1718
Jens Axboefd0ff8a2006-05-23 11:23:49 +02001719 /*
1720 * Account IO completion. bar_rq isn't accounted as a normal
1721 * IO on queueing nor completion. Accounting the containing
1722 * request is enough.
1723 */
1724 if (disk && blk_fs_request(req) && req != &req->q->bar_rq) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001725 unsigned long duration = jiffies - req->start_time;
Jens Axboea3623572005-11-01 09:26:16 +01001726 const int rw = rq_data_dir(req);
Jerome Marchand6f2576a2008-02-08 11:04:35 +01001727 struct hd_struct *part = get_part(disk, req->sector);
Jens Axboea3623572005-11-01 09:26:16 +01001728
Jerome Marchand6f2576a2008-02-08 11:04:35 +01001729 __all_stat_inc(disk, ios[rw], req->sector);
1730 __all_stat_add(disk, ticks[rw], duration, req->sector);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001731 disk_round_stats(disk);
1732 disk->in_flight--;
Jerome Marchand6f2576a2008-02-08 11:04:35 +01001733 if (part) {
1734 part_round_stats(part);
1735 part->in_flight--;
1736 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001737 }
Kiyoshi Uedab8286232007-12-11 17:53:24 -05001738
Linus Torvalds1da177e2005-04-16 15:20:36 -07001739 if (req->end_io)
Tejun Heo8ffdc652006-01-06 09:49:03 +01001740 req->end_io(req, error);
Kiyoshi Uedab8286232007-12-11 17:53:24 -05001741 else {
1742 if (blk_bidi_rq(req))
1743 __blk_put_request(req->next_rq->q, req->next_rq);
1744
Linus Torvalds1da177e2005-04-16 15:20:36 -07001745 __blk_put_request(req->q, req);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001746 }
1747}
1748
Linus Torvalds1da177e2005-04-16 15:20:36 -07001749static inline void __end_request(struct request *rq, int uptodate,
Kiyoshi Ueda9e6e39f2007-12-11 17:41:54 -05001750 unsigned int nr_bytes)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001751{
Kiyoshi Ueda9e6e39f2007-12-11 17:41:54 -05001752 int error = 0;
1753
1754 if (uptodate <= 0)
1755 error = uptodate ? uptodate : -EIO;
1756
1757 __blk_end_request(rq, error, nr_bytes);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001758}
1759
Kiyoshi Ueda3b113132007-12-11 17:41:17 -05001760/**
1761 * blk_rq_bytes - Returns bytes left to complete in the entire request
Randy Dunlap5d87a052008-02-20 09:01:22 +01001762 * @rq: the request being processed
Kiyoshi Ueda3b113132007-12-11 17:41:17 -05001763 **/
1764unsigned int blk_rq_bytes(struct request *rq)
Jens Axboea0cd1282007-09-21 10:41:07 +02001765{
1766 if (blk_fs_request(rq))
1767 return rq->hard_nr_sectors << 9;
1768
1769 return rq->data_len;
1770}
Kiyoshi Ueda3b113132007-12-11 17:41:17 -05001771EXPORT_SYMBOL_GPL(blk_rq_bytes);
1772
1773/**
1774 * blk_rq_cur_bytes - Returns bytes left to complete in the current segment
Randy Dunlap5d87a052008-02-20 09:01:22 +01001775 * @rq: the request being processed
Kiyoshi Ueda3b113132007-12-11 17:41:17 -05001776 **/
1777unsigned int blk_rq_cur_bytes(struct request *rq)
1778{
1779 if (blk_fs_request(rq))
1780 return rq->current_nr_sectors << 9;
1781
1782 if (rq->bio)
1783 return rq->bio->bi_size;
1784
1785 return rq->data_len;
1786}
1787EXPORT_SYMBOL_GPL(blk_rq_cur_bytes);
Jens Axboea0cd1282007-09-21 10:41:07 +02001788
1789/**
1790 * end_queued_request - end all I/O on a queued request
1791 * @rq: the request being processed
1792 * @uptodate: error value or 0/1 uptodate flag
1793 *
1794 * Description:
1795 * Ends all I/O on a request, and removes it from the block layer queues.
1796 * Not suitable for normal IO completion, unless the driver still has
1797 * the request attached to the block layer.
1798 *
1799 **/
1800void end_queued_request(struct request *rq, int uptodate)
1801{
Kiyoshi Ueda9e6e39f2007-12-11 17:41:54 -05001802 __end_request(rq, uptodate, blk_rq_bytes(rq));
Jens Axboea0cd1282007-09-21 10:41:07 +02001803}
1804EXPORT_SYMBOL(end_queued_request);
1805
1806/**
1807 * end_dequeued_request - end all I/O on a dequeued request
1808 * @rq: the request being processed
1809 * @uptodate: error value or 0/1 uptodate flag
1810 *
1811 * Description:
1812 * Ends all I/O on a request. The request must already have been
1813 * dequeued using blkdev_dequeue_request(), as is normally the case
1814 * for most drivers.
1815 *
1816 **/
1817void end_dequeued_request(struct request *rq, int uptodate)
1818{
Kiyoshi Ueda9e6e39f2007-12-11 17:41:54 -05001819 __end_request(rq, uptodate, blk_rq_bytes(rq));
Jens Axboea0cd1282007-09-21 10:41:07 +02001820}
1821EXPORT_SYMBOL(end_dequeued_request);
1822
1823
1824/**
1825 * end_request - end I/O on the current segment of the request
Randy Dunlap8f731f72007-10-18 23:39:28 -07001826 * @req: the request being processed
Jens Axboea0cd1282007-09-21 10:41:07 +02001827 * @uptodate: error value or 0/1 uptodate flag
1828 *
1829 * Description:
1830 * Ends I/O on the current segment of a request. If that is the only
1831 * remaining segment, the request is also completed and freed.
1832 *
1833 * This is a remnant of how older block drivers handled IO completions.
1834 * Modern drivers typically end IO on the full request in one go, unless
1835 * they have a residual value to account for. For that case this function
1836 * isn't really useful, unless the residual just happens to be the
1837 * full current segment. In other words, don't use this function in new
1838 * code. Either use end_request_completely(), or the
1839 * end_that_request_chunk() (along with end_that_request_last()) for
1840 * partial completions.
1841 *
1842 **/
1843void end_request(struct request *req, int uptodate)
1844{
Kiyoshi Ueda9e6e39f2007-12-11 17:41:54 -05001845 __end_request(req, uptodate, req->hard_cur_sectors << 9);
Jens Axboea0cd1282007-09-21 10:41:07 +02001846}
Linus Torvalds1da177e2005-04-16 15:20:36 -07001847EXPORT_SYMBOL(end_request);
1848
Kiyoshi Ueda336cdb42007-12-11 17:40:30 -05001849/**
Kiyoshi Uedae19a3ab2007-12-11 17:51:02 -05001850 * blk_end_io - Generic end_io function to complete a request.
1851 * @rq: the request being processed
1852 * @error: 0 for success, < 0 for error
Kiyoshi Uedae3a04fe2007-12-11 17:51:46 -05001853 * @nr_bytes: number of bytes to complete @rq
1854 * @bidi_bytes: number of bytes to complete @rq->next_rq
Kiyoshi Uedae19a3ab2007-12-11 17:51:02 -05001855 * @drv_callback: function called between completion of bios in the request
1856 * and completion of the request.
1857 * If the callback returns non 0, this helper returns without
1858 * completion of the request.
Kiyoshi Ueda336cdb42007-12-11 17:40:30 -05001859 *
1860 * Description:
Kiyoshi Uedae3a04fe2007-12-11 17:51:46 -05001861 * Ends I/O on a number of bytes attached to @rq and @rq->next_rq.
Kiyoshi Ueda336cdb42007-12-11 17:40:30 -05001862 * If @rq has leftover, sets it up for the next range of segments.
1863 *
1864 * Return:
1865 * 0 - we are done with this request
Kiyoshi Uedae19a3ab2007-12-11 17:51:02 -05001866 * 1 - this request is not freed yet, it still has pending buffers.
Kiyoshi Ueda336cdb42007-12-11 17:40:30 -05001867 **/
Jens Axboe22b13212008-01-31 12:36:19 +01001868static int blk_end_io(struct request *rq, int error, unsigned int nr_bytes,
1869 unsigned int bidi_bytes,
1870 int (drv_callback)(struct request *))
Kiyoshi Ueda336cdb42007-12-11 17:40:30 -05001871{
1872 struct request_queue *q = rq->q;
1873 unsigned long flags = 0UL;
Kiyoshi Ueda336cdb42007-12-11 17:40:30 -05001874
1875 if (blk_fs_request(rq) || blk_pc_request(rq)) {
Kiyoshi Ueda5450d3e2007-12-11 17:53:03 -05001876 if (__end_that_request_first(rq, error, nr_bytes))
Kiyoshi Ueda336cdb42007-12-11 17:40:30 -05001877 return 1;
Kiyoshi Uedae3a04fe2007-12-11 17:51:46 -05001878
1879 /* Bidi request must be completed as a whole */
1880 if (blk_bidi_rq(rq) &&
Kiyoshi Ueda5450d3e2007-12-11 17:53:03 -05001881 __end_that_request_first(rq->next_rq, error, bidi_bytes))
Kiyoshi Uedae3a04fe2007-12-11 17:51:46 -05001882 return 1;
Kiyoshi Ueda336cdb42007-12-11 17:40:30 -05001883 }
1884
Kiyoshi Uedae19a3ab2007-12-11 17:51:02 -05001885 /* Special feature for tricky drivers */
1886 if (drv_callback && drv_callback(rq))
1887 return 1;
1888
Kiyoshi Ueda336cdb42007-12-11 17:40:30 -05001889 add_disk_randomness(rq->rq_disk);
1890
1891 spin_lock_irqsave(q->queue_lock, flags);
Kiyoshi Uedab8286232007-12-11 17:53:24 -05001892 end_that_request_last(rq, error);
Kiyoshi Ueda336cdb42007-12-11 17:40:30 -05001893 spin_unlock_irqrestore(q->queue_lock, flags);
1894
1895 return 0;
1896}
Kiyoshi Uedae19a3ab2007-12-11 17:51:02 -05001897
1898/**
1899 * blk_end_request - Helper function for drivers to complete the request.
1900 * @rq: the request being processed
1901 * @error: 0 for success, < 0 for error
1902 * @nr_bytes: number of bytes to complete
1903 *
1904 * Description:
1905 * Ends I/O on a number of bytes attached to @rq.
1906 * If @rq has leftover, sets it up for the next range of segments.
1907 *
1908 * Return:
1909 * 0 - we are done with this request
1910 * 1 - still buffers pending for this request
1911 **/
Jens Axboe22b13212008-01-31 12:36:19 +01001912int blk_end_request(struct request *rq, int error, unsigned int nr_bytes)
Kiyoshi Uedae19a3ab2007-12-11 17:51:02 -05001913{
Kiyoshi Uedae3a04fe2007-12-11 17:51:46 -05001914 return blk_end_io(rq, error, nr_bytes, 0, NULL);
Kiyoshi Uedae19a3ab2007-12-11 17:51:02 -05001915}
Kiyoshi Ueda336cdb42007-12-11 17:40:30 -05001916EXPORT_SYMBOL_GPL(blk_end_request);
1917
1918/**
1919 * __blk_end_request - Helper function for drivers to complete the request.
1920 * @rq: the request being processed
1921 * @error: 0 for success, < 0 for error
1922 * @nr_bytes: number of bytes to complete
1923 *
1924 * Description:
1925 * Must be called with queue lock held unlike blk_end_request().
1926 *
1927 * Return:
1928 * 0 - we are done with this request
1929 * 1 - still buffers pending for this request
1930 **/
Jens Axboe22b13212008-01-31 12:36:19 +01001931int __blk_end_request(struct request *rq, int error, unsigned int nr_bytes)
Kiyoshi Ueda336cdb42007-12-11 17:40:30 -05001932{
Kiyoshi Ueda336cdb42007-12-11 17:40:30 -05001933 if (blk_fs_request(rq) || blk_pc_request(rq)) {
Kiyoshi Ueda5450d3e2007-12-11 17:53:03 -05001934 if (__end_that_request_first(rq, error, nr_bytes))
Kiyoshi Ueda336cdb42007-12-11 17:40:30 -05001935 return 1;
1936 }
1937
1938 add_disk_randomness(rq->rq_disk);
1939
Kiyoshi Uedab8286232007-12-11 17:53:24 -05001940 end_that_request_last(rq, error);
Kiyoshi Ueda336cdb42007-12-11 17:40:30 -05001941
1942 return 0;
1943}
1944EXPORT_SYMBOL_GPL(__blk_end_request);
1945
Kiyoshi Uedae19a3ab2007-12-11 17:51:02 -05001946/**
Kiyoshi Uedae3a04fe2007-12-11 17:51:46 -05001947 * blk_end_bidi_request - Helper function for drivers to complete bidi request.
1948 * @rq: the bidi request being processed
1949 * @error: 0 for success, < 0 for error
1950 * @nr_bytes: number of bytes to complete @rq
1951 * @bidi_bytes: number of bytes to complete @rq->next_rq
1952 *
1953 * Description:
1954 * Ends I/O on a number of bytes attached to @rq and @rq->next_rq.
1955 *
1956 * Return:
1957 * 0 - we are done with this request
1958 * 1 - still buffers pending for this request
1959 **/
Jens Axboe22b13212008-01-31 12:36:19 +01001960int blk_end_bidi_request(struct request *rq, int error, unsigned int nr_bytes,
1961 unsigned int bidi_bytes)
Kiyoshi Uedae3a04fe2007-12-11 17:51:46 -05001962{
1963 return blk_end_io(rq, error, nr_bytes, bidi_bytes, NULL);
1964}
1965EXPORT_SYMBOL_GPL(blk_end_bidi_request);
1966
1967/**
Kiyoshi Uedae19a3ab2007-12-11 17:51:02 -05001968 * blk_end_request_callback - Special helper function for tricky drivers
1969 * @rq: the request being processed
1970 * @error: 0 for success, < 0 for error
1971 * @nr_bytes: number of bytes to complete
1972 * @drv_callback: function called between completion of bios in the request
1973 * and completion of the request.
1974 * If the callback returns non 0, this helper returns without
1975 * completion of the request.
1976 *
1977 * Description:
1978 * Ends I/O on a number of bytes attached to @rq.
1979 * If @rq has leftover, sets it up for the next range of segments.
1980 *
1981 * This special helper function is used only for existing tricky drivers.
1982 * (e.g. cdrom_newpc_intr() of ide-cd)
1983 * This interface will be removed when such drivers are rewritten.
1984 * Don't use this interface in other places anymore.
1985 *
1986 * Return:
1987 * 0 - we are done with this request
1988 * 1 - this request is not freed yet.
1989 * this request still has pending buffers or
1990 * the driver doesn't want to finish this request yet.
1991 **/
Jens Axboe22b13212008-01-31 12:36:19 +01001992int blk_end_request_callback(struct request *rq, int error,
1993 unsigned int nr_bytes,
Kiyoshi Uedae19a3ab2007-12-11 17:51:02 -05001994 int (drv_callback)(struct request *))
1995{
Kiyoshi Uedae3a04fe2007-12-11 17:51:46 -05001996 return blk_end_io(rq, error, nr_bytes, 0, drv_callback);
Kiyoshi Uedae19a3ab2007-12-11 17:51:02 -05001997}
1998EXPORT_SYMBOL_GPL(blk_end_request_callback);
1999
Jens Axboe86db1e22008-01-29 14:53:40 +01002000void blk_rq_bio_prep(struct request_queue *q, struct request *rq,
2001 struct bio *bio)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002002{
Jens Axboe4aff5e22006-08-10 08:44:47 +02002003 /* first two bits are identical in rq->cmd_flags and bio->bi_rw */
2004 rq->cmd_flags |= (bio->bi_rw & 3);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002005
2006 rq->nr_phys_segments = bio_phys_segments(q, bio);
2007 rq->nr_hw_segments = bio_hw_segments(q, bio);
2008 rq->current_nr_sectors = bio_cur_sectors(bio);
2009 rq->hard_cur_sectors = rq->current_nr_sectors;
2010 rq->hard_nr_sectors = rq->nr_sectors = bio_sectors(bio);
2011 rq->buffer = bio_data(bio);
Mike Christie0e75f902006-12-01 10:40:55 +01002012 rq->data_len = bio->bi_size;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002013
2014 rq->bio = rq->biotail = bio;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002015
NeilBrown66846572007-08-16 13:31:28 +02002016 if (bio->bi_bdev)
2017 rq->rq_disk = bio->bi_bdev->bd_disk;
2018}
Linus Torvalds1da177e2005-04-16 15:20:36 -07002019
2020int kblockd_schedule_work(struct work_struct *work)
2021{
2022 return queue_work(kblockd_workqueue, work);
2023}
Linus Torvalds1da177e2005-04-16 15:20:36 -07002024EXPORT_SYMBOL(kblockd_schedule_work);
2025
Andrew Morton19a75d82007-05-09 02:33:56 -07002026void kblockd_flush_work(struct work_struct *work)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002027{
Oleg Nesterov28e53bd2007-05-09 02:34:22 -07002028 cancel_work_sync(work);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002029}
Andrew Morton19a75d82007-05-09 02:33:56 -07002030EXPORT_SYMBOL(kblockd_flush_work);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002031
2032int __init blk_dev_init(void)
2033{
Jens Axboeff856ba2006-01-09 16:02:34 +01002034 int i;
2035
Linus Torvalds1da177e2005-04-16 15:20:36 -07002036 kblockd_workqueue = create_workqueue("kblockd");
2037 if (!kblockd_workqueue)
2038 panic("Failed to create kblockd\n");
2039
2040 request_cachep = kmem_cache_create("blkdev_requests",
Paul Mundt20c2df82007-07-20 10:11:58 +09002041 sizeof(struct request), 0, SLAB_PANIC, NULL);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002042
Jens Axboe8324aa92008-01-29 14:51:59 +01002043 blk_requestq_cachep = kmem_cache_create("blkdev_queue",
Jens Axboe165125e2007-07-24 09:28:11 +02002044 sizeof(struct request_queue), 0, SLAB_PANIC, NULL);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002045
KAMEZAWA Hiroyuki0a945022006-03-28 01:56:37 -08002046 for_each_possible_cpu(i)
Jens Axboeff856ba2006-01-09 16:02:34 +01002047 INIT_LIST_HEAD(&per_cpu(blk_cpu_done, i));
2048
2049 open_softirq(BLOCK_SOFTIRQ, blk_done_softirq, NULL);
Chandra Seetharaman5a67e4c2006-06-27 02:54:11 -07002050 register_hotcpu_notifier(&blk_cpu_notifier);
Jens Axboeff856ba2006-01-09 16:02:34 +01002051
Linus Torvalds1da177e2005-04-16 15:20:36 -07002052 return 0;
2053}
2054