blob: 3bc5579d6f543fa57783e09268707178de26c4b7 [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
Linus Torvalds1da177e2005-04-16 15:20:36 -07002 * Copyright (C) 1991, 1992 Linus Torvalds
3 * Copyright (C) 1994, Karl Keyte: Added support for disk statistics
4 * Elevator latency, (C) 2000 Andrea Arcangeli <andrea@suse.de> SuSE
5 * Queue request tables / lock, selectable elevator, Jens Axboe <axboe@suse.de>
Jens Axboe6728cb02008-01-31 13:03:55 +01006 * kernel-doc documentation started by NeilBrown <neilb@cse.unsw.edu.au>
7 * - July2000
Linus Torvalds1da177e2005-04-16 15:20:36 -07008 * bio rewrite, highmem i/o, etc, Jens Axboe <axboe@suse.de> - may 2001
9 */
10
11/*
12 * This handles all read/write requests to block devices
13 */
Linus Torvalds1da177e2005-04-16 15:20:36 -070014#include <linux/kernel.h>
15#include <linux/module.h>
16#include <linux/backing-dev.h>
17#include <linux/bio.h>
18#include <linux/blkdev.h>
19#include <linux/highmem.h>
20#include <linux/mm.h>
21#include <linux/kernel_stat.h>
22#include <linux/string.h>
23#include <linux/init.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070024#include <linux/completion.h>
25#include <linux/slab.h>
26#include <linux/swap.h>
27#include <linux/writeback.h>
Andrew Mortonfaccbd42006-12-10 02:19:35 -080028#include <linux/task_io_accounting_ops.h>
Akinobu Mitac17bb492006-12-08 02:39:46 -080029#include <linux/fault-inject.h>
Li Zefan55782132009-06-09 13:43:05 +080030
31#define CREATE_TRACE_POINTS
32#include <trace/events/block.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070033
Jens Axboe8324aa92008-01-29 14:51:59 +010034#include "blk.h"
35
Ingo Molnar0bfc2452008-11-26 11:59:56 +010036EXPORT_TRACEPOINT_SYMBOL_GPL(block_remap);
Jun'ichi Nomurab0da3f02009-10-01 21:16:13 +020037EXPORT_TRACEPOINT_SYMBOL_GPL(block_rq_remap);
Li Zefan55782132009-06-09 13:43:05 +080038EXPORT_TRACEPOINT_SYMBOL_GPL(block_bio_complete);
Ingo Molnar0bfc2452008-11-26 11:59:56 +010039
Jens Axboe165125e2007-07-24 09:28:11 +020040static int __make_request(struct request_queue *q, struct bio *bio);
Linus Torvalds1da177e2005-04-16 15:20:36 -070041
42/*
43 * For the allocated request tables
44 */
Adrian Bunk5ece6c52008-02-18 13:45:51 +010045static struct kmem_cache *request_cachep;
Linus Torvalds1da177e2005-04-16 15:20:36 -070046
47/*
48 * For queue allocation
49 */
Jens Axboe6728cb02008-01-31 13:03:55 +010050struct kmem_cache *blk_requestq_cachep;
Linus Torvalds1da177e2005-04-16 15:20:36 -070051
52/*
Linus Torvalds1da177e2005-04-16 15:20:36 -070053 * Controlling structure to kblockd
54 */
Jens Axboeff856ba2006-01-09 16:02:34 +010055static struct workqueue_struct *kblockd_workqueue;
Linus Torvalds1da177e2005-04-16 15:20:36 -070056
Jens Axboe26b82562008-01-29 13:54:41 +010057static void drive_stat_acct(struct request *rq, int new_io)
58{
Jens Axboe28f13702008-05-07 10:15:46 +020059 struct hd_struct *part;
Jens Axboe26b82562008-01-29 13:54:41 +010060 int rw = rq_data_dir(rq);
Tejun Heoc9959052008-08-25 19:47:21 +090061 int cpu;
Jens Axboe26b82562008-01-29 13:54:41 +010062
Jens Axboec2553b52009-04-24 08:10:11 +020063 if (!blk_do_io_stat(rq))
Jens Axboe26b82562008-01-29 13:54:41 +010064 return;
65
Tejun Heo074a7ac2008-08-25 19:56:14 +090066 cpu = part_stat_lock();
Tejun Heo83096eb2009-05-07 22:24:39 +090067 part = disk_map_sector_rcu(rq->rq_disk, blk_rq_pos(rq));
Tejun Heoc9959052008-08-25 19:47:21 +090068
Jens Axboe28f13702008-05-07 10:15:46 +020069 if (!new_io)
Tejun Heo074a7ac2008-08-25 19:56:14 +090070 part_stat_inc(cpu, part, merges[rw]);
Jens Axboe28f13702008-05-07 10:15:46 +020071 else {
Tejun Heo074a7ac2008-08-25 19:56:14 +090072 part_round_stats(cpu, part);
Nikanth Karthikesan316d3152009-10-06 20:16:55 +020073 part_inc_in_flight(part, rw);
Jens Axboe26b82562008-01-29 13:54:41 +010074 }
Tejun Heoe71bf0d2008-09-03 09:03:02 +020075
Tejun Heo074a7ac2008-08-25 19:56:14 +090076 part_stat_unlock();
Jens Axboe26b82562008-01-29 13:54:41 +010077}
78
Jens Axboe8324aa92008-01-29 14:51:59 +010079void blk_queue_congestion_threshold(struct request_queue *q)
Linus Torvalds1da177e2005-04-16 15:20:36 -070080{
81 int nr;
82
83 nr = q->nr_requests - (q->nr_requests / 8) + 1;
84 if (nr > q->nr_requests)
85 nr = q->nr_requests;
86 q->nr_congestion_on = nr;
87
88 nr = q->nr_requests - (q->nr_requests / 8) - (q->nr_requests / 16) - 1;
89 if (nr < 1)
90 nr = 1;
91 q->nr_congestion_off = nr;
92}
93
Linus Torvalds1da177e2005-04-16 15:20:36 -070094/**
95 * blk_get_backing_dev_info - get the address of a queue's backing_dev_info
96 * @bdev: device
97 *
98 * Locates the passed device's request queue and returns the address of its
99 * backing_dev_info
100 *
101 * Will return NULL if the request queue cannot be located.
102 */
103struct backing_dev_info *blk_get_backing_dev_info(struct block_device *bdev)
104{
105 struct backing_dev_info *ret = NULL;
Jens Axboe165125e2007-07-24 09:28:11 +0200106 struct request_queue *q = bdev_get_queue(bdev);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700107
108 if (q)
109 ret = &q->backing_dev_info;
110 return ret;
111}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700112EXPORT_SYMBOL(blk_get_backing_dev_info);
113
FUJITA Tomonori2a4aa302008-04-29 09:54:36 +0200114void blk_rq_init(struct request_queue *q, struct request *rq)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700115{
FUJITA Tomonori1afb20f2008-04-25 12:26:28 +0200116 memset(rq, 0, sizeof(*rq));
117
Linus Torvalds1da177e2005-04-16 15:20:36 -0700118 INIT_LIST_HEAD(&rq->queuelist);
Jens Axboe242f9dc2008-09-14 05:55:09 -0700119 INIT_LIST_HEAD(&rq->timeout_list);
Jens Axboec7c22e42008-09-13 20:26:01 +0200120 rq->cpu = -1;
Jens Axboe63a71382008-02-08 12:41:03 +0100121 rq->q = q;
Tejun Heoa2dec7b2009-05-07 22:24:44 +0900122 rq->__sector = (sector_t) -1;
Jens Axboe2e662b62006-07-13 11:55:04 +0200123 INIT_HLIST_NODE(&rq->hash);
124 RB_CLEAR_NODE(&rq->rb_node);
FUJITA Tomonorid7e3c322008-04-29 09:54:39 +0200125 rq->cmd = rq->__cmd;
Li Zefane2494e12009-04-02 13:43:26 +0800126 rq->cmd_len = BLK_MAX_CDB;
Jens Axboe63a71382008-02-08 12:41:03 +0100127 rq->tag = -1;
Jens Axboe63a71382008-02-08 12:41:03 +0100128 rq->ref_count = 1;
Tejun Heob243ddc2009-04-23 11:05:18 +0900129 rq->start_time = jiffies;
Divyesh Shah91952912010-04-01 15:01:41 -0700130 set_start_time_ns(rq);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700131}
FUJITA Tomonori2a4aa302008-04-29 09:54:36 +0200132EXPORT_SYMBOL(blk_rq_init);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700133
NeilBrown5bb23a62007-09-27 12:46:13 +0200134static void req_bio_endio(struct request *rq, struct bio *bio,
135 unsigned int nbytes, int error)
Tejun Heo797e7db2006-01-06 09:51:03 +0100136{
Jens Axboe165125e2007-07-24 09:28:11 +0200137 struct request_queue *q = rq->q;
Tejun Heo797e7db2006-01-06 09:51:03 +0100138
NeilBrown5bb23a62007-09-27 12:46:13 +0200139 if (&q->bar_rq != rq) {
140 if (error)
141 clear_bit(BIO_UPTODATE, &bio->bi_flags);
142 else if (!test_bit(BIO_UPTODATE, &bio->bi_flags))
143 error = -EIO;
Tejun Heo797e7db2006-01-06 09:51:03 +0100144
NeilBrown5bb23a62007-09-27 12:46:13 +0200145 if (unlikely(nbytes > bio->bi_size)) {
Jens Axboe6728cb02008-01-31 13:03:55 +0100146 printk(KERN_ERR "%s: want %u bytes done, %u left\n",
Harvey Harrison24c03d42008-05-01 04:35:17 -0700147 __func__, nbytes, bio->bi_size);
NeilBrown5bb23a62007-09-27 12:46:13 +0200148 nbytes = bio->bi_size;
149 }
Tejun Heo797e7db2006-01-06 09:51:03 +0100150
Keith Mannthey08bafc02008-11-25 10:24:35 +0100151 if (unlikely(rq->cmd_flags & REQ_QUIET))
152 set_bit(BIO_QUIET, &bio->bi_flags);
153
NeilBrown5bb23a62007-09-27 12:46:13 +0200154 bio->bi_size -= nbytes;
155 bio->bi_sector += (nbytes >> 9);
Martin K. Petersen7ba1ba12008-06-30 20:04:41 +0200156
157 if (bio_integrity(bio))
158 bio_integrity_advance(bio, nbytes);
159
NeilBrown5bb23a62007-09-27 12:46:13 +0200160 if (bio->bi_size == 0)
NeilBrown6712ecf2007-09-27 12:47:43 +0200161 bio_endio(bio, error);
NeilBrown5bb23a62007-09-27 12:46:13 +0200162 } else {
163
164 /*
165 * Okay, this is the barrier request in progress, just
166 * record the error;
167 */
168 if (error && !q->orderr)
169 q->orderr = error;
170 }
Tejun Heo797e7db2006-01-06 09:51:03 +0100171}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700172
Linus Torvalds1da177e2005-04-16 15:20:36 -0700173void blk_dump_rq_flags(struct request *rq, char *msg)
174{
175 int bit;
176
Jens Axboe6728cb02008-01-31 13:03:55 +0100177 printk(KERN_INFO "%s: dev %s: type=%x, flags=%x\n", msg,
Jens Axboe4aff5e22006-08-10 08:44:47 +0200178 rq->rq_disk ? rq->rq_disk->disk_name : "?", rq->cmd_type,
179 rq->cmd_flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700180
Tejun Heo83096eb2009-05-07 22:24:39 +0900181 printk(KERN_INFO " sector %llu, nr/cnr %u/%u\n",
182 (unsigned long long)blk_rq_pos(rq),
183 blk_rq_sectors(rq), blk_rq_cur_sectors(rq));
Tejun Heo731ec492009-04-23 11:05:20 +0900184 printk(KERN_INFO " bio %p, biotail %p, buffer %p, len %u\n",
Tejun Heo2e46e8b2009-05-07 22:24:41 +0900185 rq->bio, rq->biotail, rq->buffer, blk_rq_bytes(rq));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700186
Jens Axboe4aff5e22006-08-10 08:44:47 +0200187 if (blk_pc_request(rq)) {
Jens Axboe6728cb02008-01-31 13:03:55 +0100188 printk(KERN_INFO " cdb: ");
FUJITA Tomonorid34c87e2008-04-29 14:37:52 +0200189 for (bit = 0; bit < BLK_MAX_CDB; bit++)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700190 printk("%02x ", rq->cmd[bit]);
191 printk("\n");
192 }
193}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700194EXPORT_SYMBOL(blk_dump_rq_flags);
195
Linus Torvalds1da177e2005-04-16 15:20:36 -0700196/*
197 * "plug" the device if there are no outstanding requests: this will
198 * force the transfer to start only after we have put all the requests
199 * on the list.
200 *
201 * This is called with interrupts off and no requests on the queue and
202 * with the queue lock held.
203 */
Jens Axboe165125e2007-07-24 09:28:11 +0200204void blk_plug_device(struct request_queue *q)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700205{
206 WARN_ON(!irqs_disabled());
207
208 /*
209 * don't plug a stopped queue, it must be paired with blk_start_queue()
210 * which will restart the queueing
211 */
Coywolf Qi Hunt7daac492006-04-19 10:14:49 +0200212 if (blk_queue_stopped(q))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700213 return;
214
Jens Axboee48ec692008-07-03 13:18:54 +0200215 if (!queue_flag_test_and_set(QUEUE_FLAG_PLUGGED, q)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700216 mod_timer(&q->unplug_timer, jiffies + q->unplug_delay);
Arnaldo Carvalho de Melo5f3ea372008-10-30 08:34:33 +0100217 trace_block_plug(q);
Jens Axboe2056a782006-03-23 20:00:26 +0100218 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700219}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700220EXPORT_SYMBOL(blk_plug_device);
221
Jens Axboe6c5e0c42008-08-01 20:31:32 +0200222/**
223 * blk_plug_device_unlocked - plug a device without queue lock held
224 * @q: The &struct request_queue to plug
225 *
226 * Description:
227 * Like @blk_plug_device(), but grabs the queue lock and disables
228 * interrupts.
229 **/
230void blk_plug_device_unlocked(struct request_queue *q)
231{
232 unsigned long flags;
233
234 spin_lock_irqsave(q->queue_lock, flags);
235 blk_plug_device(q);
236 spin_unlock_irqrestore(q->queue_lock, flags);
237}
238EXPORT_SYMBOL(blk_plug_device_unlocked);
239
Linus Torvalds1da177e2005-04-16 15:20:36 -0700240/*
241 * remove the queue from the plugged list, if present. called with
242 * queue lock held and interrupts disabled.
243 */
Jens Axboe165125e2007-07-24 09:28:11 +0200244int blk_remove_plug(struct request_queue *q)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700245{
246 WARN_ON(!irqs_disabled());
247
Jens Axboee48ec692008-07-03 13:18:54 +0200248 if (!queue_flag_test_and_clear(QUEUE_FLAG_PLUGGED, q))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700249 return 0;
250
251 del_timer(&q->unplug_timer);
252 return 1;
253}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700254EXPORT_SYMBOL(blk_remove_plug);
255
256/*
257 * remove the plug and let it rip..
258 */
Jens Axboe165125e2007-07-24 09:28:11 +0200259void __generic_unplug_device(struct request_queue *q)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700260{
Coywolf Qi Hunt7daac492006-04-19 10:14:49 +0200261 if (unlikely(blk_queue_stopped(q)))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700262 return;
Jens Axboea31a9732008-10-17 13:58:29 +0200263 if (!blk_remove_plug(q) && !blk_queue_nonrot(q))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700264 return;
265
Jens Axboe22e2c502005-06-27 10:55:12 +0200266 q->request_fn(q);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700267}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700268
269/**
270 * generic_unplug_device - fire a request queue
Jens Axboe165125e2007-07-24 09:28:11 +0200271 * @q: The &struct request_queue in question
Linus Torvalds1da177e2005-04-16 15:20:36 -0700272 *
273 * Description:
274 * Linux uses plugging to build bigger requests queues before letting
275 * the device have at them. If a queue is plugged, the I/O scheduler
276 * is still adding and merging requests on the queue. Once the queue
277 * gets unplugged, the request_fn defined for the queue is invoked and
278 * transfers started.
279 **/
Jens Axboe165125e2007-07-24 09:28:11 +0200280void generic_unplug_device(struct request_queue *q)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700281{
Jens Axboedbaf2c02008-05-07 09:48:17 +0200282 if (blk_queue_plugged(q)) {
283 spin_lock_irq(q->queue_lock);
284 __generic_unplug_device(q);
285 spin_unlock_irq(q->queue_lock);
286 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700287}
288EXPORT_SYMBOL(generic_unplug_device);
289
290static void blk_backing_dev_unplug(struct backing_dev_info *bdi,
291 struct page *page)
292{
Jens Axboe165125e2007-07-24 09:28:11 +0200293 struct request_queue *q = bdi->unplug_io_data;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700294
Alan D. Brunelle2ad8b1e2007-11-07 14:26:56 -0500295 blk_unplug(q);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700296}
297
Jens Axboe86db1e22008-01-29 14:53:40 +0100298void blk_unplug_work(struct work_struct *work)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700299{
Jens Axboe165125e2007-07-24 09:28:11 +0200300 struct request_queue *q =
301 container_of(work, struct request_queue, unplug_work);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700302
Arnaldo Carvalho de Melo5f3ea372008-10-30 08:34:33 +0100303 trace_block_unplug_io(q);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700304 q->unplug_fn(q);
305}
306
Jens Axboe86db1e22008-01-29 14:53:40 +0100307void blk_unplug_timeout(unsigned long data)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700308{
Jens Axboe165125e2007-07-24 09:28:11 +0200309 struct request_queue *q = (struct request_queue *)data;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700310
Arnaldo Carvalho de Melo5f3ea372008-10-30 08:34:33 +0100311 trace_block_unplug_timer(q);
Jens Axboe18887ad2008-07-28 13:08:45 +0200312 kblockd_schedule_work(q, &q->unplug_work);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700313}
314
Alan D. Brunelle2ad8b1e2007-11-07 14:26:56 -0500315void blk_unplug(struct request_queue *q)
316{
317 /*
318 * devices don't necessarily have an ->unplug_fn defined
319 */
320 if (q->unplug_fn) {
Arnaldo Carvalho de Melo5f3ea372008-10-30 08:34:33 +0100321 trace_block_unplug_io(q);
Alan D. Brunelle2ad8b1e2007-11-07 14:26:56 -0500322 q->unplug_fn(q);
323 }
324}
325EXPORT_SYMBOL(blk_unplug);
326
Linus Torvalds1da177e2005-04-16 15:20:36 -0700327/**
328 * blk_start_queue - restart a previously stopped queue
Jens Axboe165125e2007-07-24 09:28:11 +0200329 * @q: The &struct request_queue in question
Linus Torvalds1da177e2005-04-16 15:20:36 -0700330 *
331 * Description:
332 * blk_start_queue() will clear the stop flag on the queue, and call
333 * the request_fn for the queue if it was in a stopped state when
334 * entered. Also see blk_stop_queue(). Queue lock must be held.
335 **/
Jens Axboe165125e2007-07-24 09:28:11 +0200336void blk_start_queue(struct request_queue *q)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700337{
Paolo 'Blaisorblade' Giarrussoa038e252006-06-05 12:09:01 +0200338 WARN_ON(!irqs_disabled());
339
Nick Piggin75ad23b2008-04-29 14:48:33 +0200340 queue_flag_clear(QUEUE_FLAG_STOPPED, q);
Tejun Heoa538cd02009-04-23 11:05:17 +0900341 __blk_run_queue(q);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700342}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700343EXPORT_SYMBOL(blk_start_queue);
344
345/**
346 * blk_stop_queue - stop a queue
Jens Axboe165125e2007-07-24 09:28:11 +0200347 * @q: The &struct request_queue in question
Linus Torvalds1da177e2005-04-16 15:20:36 -0700348 *
349 * Description:
350 * The Linux block layer assumes that a block driver will consume all
351 * entries on the request queue when the request_fn strategy is called.
352 * Often this will not happen, because of hardware limitations (queue
353 * depth settings). If a device driver gets a 'queue full' response,
354 * or if it simply chooses not to queue more I/O at one point, it can
355 * call this function to prevent the request_fn from being called until
356 * the driver has signalled it's ready to go again. This happens by calling
357 * blk_start_queue() to restart queue operations. Queue lock must be held.
358 **/
Jens Axboe165125e2007-07-24 09:28:11 +0200359void blk_stop_queue(struct request_queue *q)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700360{
361 blk_remove_plug(q);
Nick Piggin75ad23b2008-04-29 14:48:33 +0200362 queue_flag_set(QUEUE_FLAG_STOPPED, q);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700363}
364EXPORT_SYMBOL(blk_stop_queue);
365
366/**
367 * blk_sync_queue - cancel any pending callbacks on a queue
368 * @q: the queue
369 *
370 * Description:
371 * The block layer may perform asynchronous callback activity
372 * on a queue, such as calling the unplug function after a timeout.
373 * A block device may call blk_sync_queue to ensure that any
374 * such activity is cancelled, thus allowing it to release resources
Michael Opdenacker59c51592007-05-09 08:57:56 +0200375 * that the callbacks might use. The caller must already have made sure
Linus Torvalds1da177e2005-04-16 15:20:36 -0700376 * that its ->make_request_fn will not re-add plugging prior to calling
377 * this function.
378 *
379 */
380void blk_sync_queue(struct request_queue *q)
381{
382 del_timer_sync(&q->unplug_timer);
Jens Axboe70ed28b2008-11-19 14:38:39 +0100383 del_timer_sync(&q->timeout);
Cheng Renquan64d01dc2008-12-03 12:41:39 +0100384 cancel_work_sync(&q->unplug_work);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700385}
386EXPORT_SYMBOL(blk_sync_queue);
387
388/**
Jens Axboe80a4b582008-10-14 09:51:06 +0200389 * __blk_run_queue - run a single device queue
Linus Torvalds1da177e2005-04-16 15:20:36 -0700390 * @q: The queue to run
Jens Axboe80a4b582008-10-14 09:51:06 +0200391 *
392 * Description:
393 * See @blk_run_queue. This variant must be called with the queue lock
394 * held and interrupts disabled.
395 *
Linus Torvalds1da177e2005-04-16 15:20:36 -0700396 */
Nick Piggin75ad23b2008-04-29 14:48:33 +0200397void __blk_run_queue(struct request_queue *q)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700398{
Linus Torvalds1da177e2005-04-16 15:20:36 -0700399 blk_remove_plug(q);
Jens Axboedac07ec2006-05-11 08:20:16 +0200400
Tejun Heoa538cd02009-04-23 11:05:17 +0900401 if (unlikely(blk_queue_stopped(q)))
402 return;
403
404 if (elv_queue_empty(q))
405 return;
406
Jens Axboedac07ec2006-05-11 08:20:16 +0200407 /*
408 * Only recurse once to avoid overrunning the stack, let the unplug
409 * handling reinvoke the handler shortly if we already got there.
410 */
Tejun Heoa538cd02009-04-23 11:05:17 +0900411 if (!queue_flag_test_and_set(QUEUE_FLAG_REENTER, q)) {
412 q->request_fn(q);
413 queue_flag_clear(QUEUE_FLAG_REENTER, q);
414 } else {
415 queue_flag_set(QUEUE_FLAG_PLUGGED, q);
416 kblockd_schedule_work(q, &q->unplug_work);
417 }
Nick Piggin75ad23b2008-04-29 14:48:33 +0200418}
419EXPORT_SYMBOL(__blk_run_queue);
Jens Axboedac07ec2006-05-11 08:20:16 +0200420
Nick Piggin75ad23b2008-04-29 14:48:33 +0200421/**
422 * blk_run_queue - run a single device queue
423 * @q: The queue to run
Jens Axboe80a4b582008-10-14 09:51:06 +0200424 *
425 * Description:
426 * Invoke request handling on this queue, if it has pending work to do.
Tejun Heoa7f55792009-04-23 11:05:17 +0900427 * May be used to restart queueing when a request has completed.
Nick Piggin75ad23b2008-04-29 14:48:33 +0200428 */
429void blk_run_queue(struct request_queue *q)
430{
431 unsigned long flags;
432
433 spin_lock_irqsave(q->queue_lock, flags);
434 __blk_run_queue(q);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700435 spin_unlock_irqrestore(q->queue_lock, flags);
436}
437EXPORT_SYMBOL(blk_run_queue);
438
Jens Axboe165125e2007-07-24 09:28:11 +0200439void blk_put_queue(struct request_queue *q)
Al Viro483f4af2006-03-18 18:34:37 -0500440{
441 kobject_put(&q->kobj);
442}
Al Viro483f4af2006-03-18 18:34:37 -0500443
Jens Axboe6728cb02008-01-31 13:03:55 +0100444void blk_cleanup_queue(struct request_queue *q)
Al Viro483f4af2006-03-18 18:34:37 -0500445{
Jens Axboee3335de2008-09-18 09:22:54 -0700446 /*
447 * We know we have process context here, so we can be a little
448 * cautious and ensure that pending block actions on this device
449 * are done before moving on. Going into this function, we should
450 * not have processes doing IO to this device.
451 */
452 blk_sync_queue(q);
453
Matthew Garrett31373d02010-04-06 14:25:14 +0200454 del_timer_sync(&q->backing_dev_info.laptop_mode_wb_timer);
Al Viro483f4af2006-03-18 18:34:37 -0500455 mutex_lock(&q->sysfs_lock);
Nick Piggin75ad23b2008-04-29 14:48:33 +0200456 queue_flag_set_unlocked(QUEUE_FLAG_DEAD, q);
Al Viro483f4af2006-03-18 18:34:37 -0500457 mutex_unlock(&q->sysfs_lock);
458
459 if (q->elevator)
460 elevator_exit(q->elevator);
461
462 blk_put_queue(q);
463}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700464EXPORT_SYMBOL(blk_cleanup_queue);
465
Jens Axboe165125e2007-07-24 09:28:11 +0200466static int blk_init_free_list(struct request_queue *q)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700467{
468 struct request_list *rl = &q->rq;
469
Jens Axboe1faa16d2009-04-06 14:48:01 +0200470 rl->count[BLK_RW_SYNC] = rl->count[BLK_RW_ASYNC] = 0;
471 rl->starved[BLK_RW_SYNC] = rl->starved[BLK_RW_ASYNC] = 0;
Tejun Heocb98fc82005-10-28 08:29:39 +0200472 rl->elvpriv = 0;
Jens Axboe1faa16d2009-04-06 14:48:01 +0200473 init_waitqueue_head(&rl->wait[BLK_RW_SYNC]);
474 init_waitqueue_head(&rl->wait[BLK_RW_ASYNC]);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700475
Christoph Lameter19460892005-06-23 00:08:19 -0700476 rl->rq_pool = mempool_create_node(BLKDEV_MIN_RQ, mempool_alloc_slab,
477 mempool_free_slab, request_cachep, q->node);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700478
479 if (!rl->rq_pool)
480 return -ENOMEM;
481
482 return 0;
483}
484
Jens Axboe165125e2007-07-24 09:28:11 +0200485struct request_queue *blk_alloc_queue(gfp_t gfp_mask)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700486{
Christoph Lameter19460892005-06-23 00:08:19 -0700487 return blk_alloc_queue_node(gfp_mask, -1);
488}
489EXPORT_SYMBOL(blk_alloc_queue);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700490
Jens Axboe165125e2007-07-24 09:28:11 +0200491struct request_queue *blk_alloc_queue_node(gfp_t gfp_mask, int node_id)
Christoph Lameter19460892005-06-23 00:08:19 -0700492{
Jens Axboe165125e2007-07-24 09:28:11 +0200493 struct request_queue *q;
Peter Zijlstrae0bf68d2007-10-16 23:25:46 -0700494 int err;
Christoph Lameter19460892005-06-23 00:08:19 -0700495
Jens Axboe8324aa92008-01-29 14:51:59 +0100496 q = kmem_cache_alloc_node(blk_requestq_cachep,
Christoph Lameter94f60302007-07-17 04:03:29 -0700497 gfp_mask | __GFP_ZERO, node_id);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700498 if (!q)
499 return NULL;
500
Peter Zijlstrae0bf68d2007-10-16 23:25:46 -0700501 q->backing_dev_info.unplug_io_fn = blk_backing_dev_unplug;
502 q->backing_dev_info.unplug_io_data = q;
Jens Axboe0989a022009-06-12 14:42:56 +0200503 q->backing_dev_info.ra_pages =
504 (VM_MAX_READAHEAD * 1024) / PAGE_CACHE_SIZE;
505 q->backing_dev_info.state = 0;
506 q->backing_dev_info.capabilities = BDI_CAP_MAP_COPY;
Jens Axboed9938312009-06-12 14:45:52 +0200507 q->backing_dev_info.name = "block";
Jens Axboe0989a022009-06-12 14:42:56 +0200508
Peter Zijlstrae0bf68d2007-10-16 23:25:46 -0700509 err = bdi_init(&q->backing_dev_info);
510 if (err) {
Jens Axboe8324aa92008-01-29 14:51:59 +0100511 kmem_cache_free(blk_requestq_cachep, q);
Peter Zijlstrae0bf68d2007-10-16 23:25:46 -0700512 return NULL;
513 }
514
Matthew Garrett31373d02010-04-06 14:25:14 +0200515 setup_timer(&q->backing_dev_info.laptop_mode_wb_timer,
516 laptop_mode_timer_fn, (unsigned long) q);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700517 init_timer(&q->unplug_timer);
Jens Axboe242f9dc2008-09-14 05:55:09 -0700518 setup_timer(&q->timeout, blk_rq_timed_out_timer, (unsigned long) q);
519 INIT_LIST_HEAD(&q->timeout_list);
Peter Zijlstra713ada92008-10-16 13:44:57 +0200520 INIT_WORK(&q->unplug_work, blk_unplug_work);
Al Viro483f4af2006-03-18 18:34:37 -0500521
Jens Axboe8324aa92008-01-29 14:51:59 +0100522 kobject_init(&q->kobj, &blk_queue_ktype);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700523
Al Viro483f4af2006-03-18 18:34:37 -0500524 mutex_init(&q->sysfs_lock);
Neil Browne7e72bf2008-05-14 16:05:54 -0700525 spin_lock_init(&q->__queue_lock);
Al Viro483f4af2006-03-18 18:34:37 -0500526
Linus Torvalds1da177e2005-04-16 15:20:36 -0700527 return q;
528}
Christoph Lameter19460892005-06-23 00:08:19 -0700529EXPORT_SYMBOL(blk_alloc_queue_node);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700530
531/**
532 * blk_init_queue - prepare a request queue for use with a block device
533 * @rfn: The function to be called to process requests that have been
534 * placed on the queue.
535 * @lock: Request queue spin lock
536 *
537 * Description:
538 * If a block device wishes to use the standard request handling procedures,
539 * which sorts requests and coalesces adjacent requests, then it must
540 * call blk_init_queue(). The function @rfn will be called when there
541 * are requests on the queue that need to be processed. If the device
542 * supports plugging, then @rfn may not be called immediately when requests
543 * are available on the queue, but may be called at some time later instead.
544 * Plugged queues are generally unplugged when a buffer belonging to one
545 * of the requests on the queue is needed, or due to memory pressure.
546 *
547 * @rfn is not required, or even expected, to remove all requests off the
548 * queue, but only as many as it can handle at a time. If it does leave
549 * requests on the queue, it is responsible for arranging that the requests
550 * get dealt with eventually.
551 *
552 * The queue spin lock must be held while manipulating the requests on the
Paolo 'Blaisorblade' Giarrussoa038e252006-06-05 12:09:01 +0200553 * request queue; this lock will be taken also from interrupt context, so irq
554 * disabling is needed for it.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700555 *
Randy Dunlap710027a2008-08-19 20:13:11 +0200556 * Function returns a pointer to the initialized request queue, or %NULL if
Linus Torvalds1da177e2005-04-16 15:20:36 -0700557 * it didn't succeed.
558 *
559 * Note:
560 * blk_init_queue() must be paired with a blk_cleanup_queue() call
561 * when the block device is deactivated (such as at module unload).
562 **/
Christoph Lameter19460892005-06-23 00:08:19 -0700563
Jens Axboe165125e2007-07-24 09:28:11 +0200564struct request_queue *blk_init_queue(request_fn_proc *rfn, spinlock_t *lock)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700565{
Christoph Lameter19460892005-06-23 00:08:19 -0700566 return blk_init_queue_node(rfn, lock, -1);
567}
568EXPORT_SYMBOL(blk_init_queue);
569
Jens Axboe165125e2007-07-24 09:28:11 +0200570struct request_queue *
Christoph Lameter19460892005-06-23 00:08:19 -0700571blk_init_queue_node(request_fn_proc *rfn, spinlock_t *lock, int node_id)
572{
Jens Axboe165125e2007-07-24 09:28:11 +0200573 struct request_queue *q = blk_alloc_queue_node(GFP_KERNEL, node_id);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700574
Mike Snitzer01effb02010-05-11 08:57:42 +0200575 return blk_init_allocated_queue_node(q, rfn, lock, node_id);
576}
577EXPORT_SYMBOL(blk_init_queue_node);
578
579struct request_queue *
580blk_init_allocated_queue(struct request_queue *q, request_fn_proc *rfn,
581 spinlock_t *lock)
582{
583 return blk_init_allocated_queue_node(q, rfn, lock, -1);
584}
585EXPORT_SYMBOL(blk_init_allocated_queue);
586
587struct request_queue *
588blk_init_allocated_queue_node(struct request_queue *q, request_fn_proc *rfn,
589 spinlock_t *lock, int node_id)
590{
Linus Torvalds1da177e2005-04-16 15:20:36 -0700591 if (!q)
592 return NULL;
593
Christoph Lameter19460892005-06-23 00:08:19 -0700594 q->node = node_id;
Al Viro8669aaf2006-03-18 13:50:00 -0500595 if (blk_init_free_list(q)) {
Jens Axboe8324aa92008-01-29 14:51:59 +0100596 kmem_cache_free(blk_requestq_cachep, q);
Al Viro8669aaf2006-03-18 13:50:00 -0500597 return NULL;
598 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700599
600 q->request_fn = rfn;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700601 q->prep_rq_fn = NULL;
602 q->unplug_fn = generic_unplug_device;
Jens Axboebc58ba92009-01-23 10:54:44 +0100603 q->queue_flags = QUEUE_FLAG_DEFAULT;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700604 q->queue_lock = lock;
605
Jens Axboef3b144a2009-03-06 08:48:33 +0100606 /*
607 * This also sets hw/phys segments, boundary and size
608 */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700609 blk_queue_make_request(q, __make_request);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700610
Alan Stern44ec9542007-02-20 11:01:57 -0500611 q->sg_reserved_size = INT_MAX;
612
Linus Torvalds1da177e2005-04-16 15:20:36 -0700613 /*
614 * all done
615 */
616 if (!elevator_init(q, NULL)) {
617 blk_queue_congestion_threshold(q);
618 return q;
619 }
620
Al Viro8669aaf2006-03-18 13:50:00 -0500621 blk_put_queue(q);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700622 return NULL;
623}
Mike Snitzer01effb02010-05-11 08:57:42 +0200624EXPORT_SYMBOL(blk_init_allocated_queue_node);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700625
Jens Axboe165125e2007-07-24 09:28:11 +0200626int blk_get_queue(struct request_queue *q)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700627{
Nick Pigginfde6ad22005-06-23 00:08:53 -0700628 if (likely(!test_bit(QUEUE_FLAG_DEAD, &q->queue_flags))) {
Al Viro483f4af2006-03-18 18:34:37 -0500629 kobject_get(&q->kobj);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700630 return 0;
631 }
632
633 return 1;
634}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700635
Jens Axboe165125e2007-07-24 09:28:11 +0200636static inline void blk_free_request(struct request_queue *q, struct request *rq)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700637{
Jens Axboe4aff5e22006-08-10 08:44:47 +0200638 if (rq->cmd_flags & REQ_ELVPRIV)
Tejun Heocb98fc82005-10-28 08:29:39 +0200639 elv_put_request(q, rq);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700640 mempool_free(rq, q->rq.rq_pool);
641}
642
Jens Axboe1ea25ec2006-07-18 22:24:11 +0200643static struct request *
Jerome Marchand42dad762009-04-22 14:01:49 +0200644blk_alloc_request(struct request_queue *q, int flags, int priv, gfp_t gfp_mask)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700645{
646 struct request *rq = mempool_alloc(q->rq.rq_pool, gfp_mask);
647
648 if (!rq)
649 return NULL;
650
FUJITA Tomonori2a4aa302008-04-29 09:54:36 +0200651 blk_rq_init(q, rq);
FUJITA Tomonori1afb20f2008-04-25 12:26:28 +0200652
Jerome Marchand42dad762009-04-22 14:01:49 +0200653 rq->cmd_flags = flags | REQ_ALLOCED;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700654
Tejun Heocb98fc82005-10-28 08:29:39 +0200655 if (priv) {
Jens Axboecb78b282006-07-28 09:32:57 +0200656 if (unlikely(elv_set_request(q, rq, gfp_mask))) {
Tejun Heocb98fc82005-10-28 08:29:39 +0200657 mempool_free(rq, q->rq.rq_pool);
658 return NULL;
659 }
Jens Axboe4aff5e22006-08-10 08:44:47 +0200660 rq->cmd_flags |= REQ_ELVPRIV;
Tejun Heocb98fc82005-10-28 08:29:39 +0200661 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700662
Tejun Heocb98fc82005-10-28 08:29:39 +0200663 return rq;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700664}
665
666/*
667 * ioc_batching returns true if the ioc is a valid batching request and
668 * should be given priority access to a request.
669 */
Jens Axboe165125e2007-07-24 09:28:11 +0200670static inline int ioc_batching(struct request_queue *q, struct io_context *ioc)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700671{
672 if (!ioc)
673 return 0;
674
675 /*
676 * Make sure the process is able to allocate at least 1 request
677 * even if the batch times out, otherwise we could theoretically
678 * lose wakeups.
679 */
680 return ioc->nr_batch_requests == q->nr_batching ||
681 (ioc->nr_batch_requests > 0
682 && time_before(jiffies, ioc->last_waited + BLK_BATCH_TIME));
683}
684
685/*
686 * ioc_set_batching sets ioc to be a new "batcher" if it is not one. This
687 * will cause the process to be a "batcher" on all queues in the system. This
688 * is the behaviour we want though - once it gets a wakeup it should be given
689 * a nice run.
690 */
Jens Axboe165125e2007-07-24 09:28:11 +0200691static void ioc_set_batching(struct request_queue *q, struct io_context *ioc)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700692{
693 if (!ioc || ioc_batching(q, ioc))
694 return;
695
696 ioc->nr_batch_requests = q->nr_batching;
697 ioc->last_waited = jiffies;
698}
699
Jens Axboe1faa16d2009-04-06 14:48:01 +0200700static void __freed_request(struct request_queue *q, int sync)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700701{
702 struct request_list *rl = &q->rq;
703
Jens Axboe1faa16d2009-04-06 14:48:01 +0200704 if (rl->count[sync] < queue_congestion_off_threshold(q))
705 blk_clear_queue_congested(q, sync);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700706
Jens Axboe1faa16d2009-04-06 14:48:01 +0200707 if (rl->count[sync] + 1 <= q->nr_requests) {
708 if (waitqueue_active(&rl->wait[sync]))
709 wake_up(&rl->wait[sync]);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700710
Jens Axboe1faa16d2009-04-06 14:48:01 +0200711 blk_clear_queue_full(q, sync);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700712 }
713}
714
715/*
716 * A request has just been released. Account for it, update the full and
717 * congestion status, wake up any waiters. Called under q->queue_lock.
718 */
Jens Axboe1faa16d2009-04-06 14:48:01 +0200719static void freed_request(struct request_queue *q, int sync, int priv)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700720{
721 struct request_list *rl = &q->rq;
722
Jens Axboe1faa16d2009-04-06 14:48:01 +0200723 rl->count[sync]--;
Tejun Heocb98fc82005-10-28 08:29:39 +0200724 if (priv)
725 rl->elvpriv--;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700726
Jens Axboe1faa16d2009-04-06 14:48:01 +0200727 __freed_request(q, sync);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700728
Jens Axboe1faa16d2009-04-06 14:48:01 +0200729 if (unlikely(rl->starved[sync ^ 1]))
730 __freed_request(q, sync ^ 1);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700731}
732
Linus Torvalds1da177e2005-04-16 15:20:36 -0700733/*
Nick Piggind6344532005-06-28 20:45:14 -0700734 * Get a free request, queue_lock must be held.
735 * Returns NULL on failure, with queue_lock held.
736 * Returns !NULL on success, with queue_lock *not held*.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700737 */
Jens Axboe165125e2007-07-24 09:28:11 +0200738static struct request *get_request(struct request_queue *q, int rw_flags,
Jens Axboe7749a8d2006-12-13 13:02:26 +0100739 struct bio *bio, gfp_t gfp_mask)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700740{
741 struct request *rq = NULL;
742 struct request_list *rl = &q->rq;
Jens Axboe88ee5ef2005-11-12 11:09:12 +0100743 struct io_context *ioc = NULL;
Jens Axboe1faa16d2009-04-06 14:48:01 +0200744 const bool is_sync = rw_is_sync(rw_flags) != 0;
Jens Axboe88ee5ef2005-11-12 11:09:12 +0100745 int may_queue, priv;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700746
Jens Axboe7749a8d2006-12-13 13:02:26 +0100747 may_queue = elv_may_queue(q, rw_flags);
Jens Axboe88ee5ef2005-11-12 11:09:12 +0100748 if (may_queue == ELV_MQUEUE_NO)
749 goto rq_starved;
750
Jens Axboe1faa16d2009-04-06 14:48:01 +0200751 if (rl->count[is_sync]+1 >= queue_congestion_on_threshold(q)) {
752 if (rl->count[is_sync]+1 >= q->nr_requests) {
Jens Axboeb5deef92006-07-19 23:39:40 +0200753 ioc = current_io_context(GFP_ATOMIC, q->node);
Jens Axboe88ee5ef2005-11-12 11:09:12 +0100754 /*
755 * The queue will fill after this allocation, so set
756 * it as full, and mark this process as "batching".
757 * This process will be allowed to complete a batch of
758 * requests, others will be blocked.
759 */
Jens Axboe1faa16d2009-04-06 14:48:01 +0200760 if (!blk_queue_full(q, is_sync)) {
Jens Axboe88ee5ef2005-11-12 11:09:12 +0100761 ioc_set_batching(q, ioc);
Jens Axboe1faa16d2009-04-06 14:48:01 +0200762 blk_set_queue_full(q, is_sync);
Jens Axboe88ee5ef2005-11-12 11:09:12 +0100763 } else {
764 if (may_queue != ELV_MQUEUE_MUST
765 && !ioc_batching(q, ioc)) {
766 /*
767 * The queue is full and the allocating
768 * process is not a "batcher", and not
769 * exempted by the IO scheduler
770 */
771 goto out;
772 }
773 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700774 }
Jens Axboe1faa16d2009-04-06 14:48:01 +0200775 blk_set_queue_congested(q, is_sync);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700776 }
777
Jens Axboe082cf692005-06-28 16:35:11 +0200778 /*
779 * Only allow batching queuers to allocate up to 50% over the defined
780 * limit of requests, otherwise we could have thousands of requests
781 * allocated with any setting of ->nr_requests
782 */
Jens Axboe1faa16d2009-04-06 14:48:01 +0200783 if (rl->count[is_sync] >= (3 * q->nr_requests / 2))
Jens Axboe082cf692005-06-28 16:35:11 +0200784 goto out;
Hugh Dickinsfd782a42005-06-29 15:15:40 +0100785
Jens Axboe1faa16d2009-04-06 14:48:01 +0200786 rl->count[is_sync]++;
787 rl->starved[is_sync] = 0;
Tejun Heocb98fc82005-10-28 08:29:39 +0200788
Jens Axboe64521d12005-10-28 08:30:39 +0200789 priv = !test_bit(QUEUE_FLAG_ELVSWITCH, &q->queue_flags);
Tejun Heocb98fc82005-10-28 08:29:39 +0200790 if (priv)
791 rl->elvpriv++;
792
Jerome Marchand42dad762009-04-22 14:01:49 +0200793 if (blk_queue_io_stat(q))
794 rw_flags |= REQ_IO_STAT;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700795 spin_unlock_irq(q->queue_lock);
796
Jens Axboe7749a8d2006-12-13 13:02:26 +0100797 rq = blk_alloc_request(q, rw_flags, priv, gfp_mask);
Jens Axboe88ee5ef2005-11-12 11:09:12 +0100798 if (unlikely(!rq)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700799 /*
800 * Allocation failed presumably due to memory. Undo anything
801 * we might have messed up.
802 *
803 * Allocating task should really be put onto the front of the
804 * wait queue, but this is pretty rare.
805 */
806 spin_lock_irq(q->queue_lock);
Jens Axboe1faa16d2009-04-06 14:48:01 +0200807 freed_request(q, is_sync, priv);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700808
809 /*
810 * in the very unlikely event that allocation failed and no
811 * requests for this direction was pending, mark us starved
812 * so that freeing of a request in the other direction will
813 * notice us. another possible fix would be to split the
814 * rq mempool into READ and WRITE
815 */
816rq_starved:
Jens Axboe1faa16d2009-04-06 14:48:01 +0200817 if (unlikely(rl->count[is_sync] == 0))
818 rl->starved[is_sync] = 1;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700819
Linus Torvalds1da177e2005-04-16 15:20:36 -0700820 goto out;
821 }
822
Jens Axboe88ee5ef2005-11-12 11:09:12 +0100823 /*
824 * ioc may be NULL here, and ioc_batching will be false. That's
825 * OK, if the queue is under the request limit then requests need
826 * not count toward the nr_batch_requests limit. There will always
827 * be some limit enforced by BLK_BATCH_TIME.
828 */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700829 if (ioc_batching(q, ioc))
830 ioc->nr_batch_requests--;
Jens Axboe6728cb02008-01-31 13:03:55 +0100831
Jens Axboe1faa16d2009-04-06 14:48:01 +0200832 trace_block_getrq(q, bio, rw_flags & 1);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700833out:
Linus Torvalds1da177e2005-04-16 15:20:36 -0700834 return rq;
835}
836
837/*
838 * No available requests for this queue, unplug the device and wait for some
839 * requests to become available.
Nick Piggind6344532005-06-28 20:45:14 -0700840 *
841 * Called with q->queue_lock held, and returns with it unlocked.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700842 */
Jens Axboe165125e2007-07-24 09:28:11 +0200843static struct request *get_request_wait(struct request_queue *q, int rw_flags,
Jens Axboe22e2c502005-06-27 10:55:12 +0200844 struct bio *bio)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700845{
Jens Axboe1faa16d2009-04-06 14:48:01 +0200846 const bool is_sync = rw_is_sync(rw_flags) != 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700847 struct request *rq;
848
Jens Axboe7749a8d2006-12-13 13:02:26 +0100849 rq = get_request(q, rw_flags, bio, GFP_NOIO);
Nick Piggin450991b2005-06-28 20:45:13 -0700850 while (!rq) {
851 DEFINE_WAIT(wait);
Zhang, Yanmin05caf8d2008-05-22 15:13:29 +0200852 struct io_context *ioc;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700853 struct request_list *rl = &q->rq;
854
Jens Axboe1faa16d2009-04-06 14:48:01 +0200855 prepare_to_wait_exclusive(&rl->wait[is_sync], &wait,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700856 TASK_UNINTERRUPTIBLE);
857
Jens Axboe1faa16d2009-04-06 14:48:01 +0200858 trace_block_sleeprq(q, bio, rw_flags & 1);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700859
Zhang, Yanmin05caf8d2008-05-22 15:13:29 +0200860 __generic_unplug_device(q);
861 spin_unlock_irq(q->queue_lock);
862 io_schedule();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700863
Zhang, Yanmin05caf8d2008-05-22 15:13:29 +0200864 /*
865 * After sleeping, we become a "batching" process and
866 * will be able to allocate at least one request, and
867 * up to a big batch of them for a small period time.
868 * See ioc_batching, ioc_set_batching
869 */
870 ioc = current_io_context(GFP_NOIO, q->node);
871 ioc_set_batching(q, ioc);
Jens Axboe2056a782006-03-23 20:00:26 +0100872
Zhang, Yanmin05caf8d2008-05-22 15:13:29 +0200873 spin_lock_irq(q->queue_lock);
Jens Axboe1faa16d2009-04-06 14:48:01 +0200874 finish_wait(&rl->wait[is_sync], &wait);
Zhang, Yanmin05caf8d2008-05-22 15:13:29 +0200875
876 rq = get_request(q, rw_flags, bio, GFP_NOIO);
877 };
Linus Torvalds1da177e2005-04-16 15:20:36 -0700878
879 return rq;
880}
881
Jens Axboe165125e2007-07-24 09:28:11 +0200882struct request *blk_get_request(struct request_queue *q, int rw, gfp_t gfp_mask)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700883{
884 struct request *rq;
885
886 BUG_ON(rw != READ && rw != WRITE);
887
Nick Piggind6344532005-06-28 20:45:14 -0700888 spin_lock_irq(q->queue_lock);
889 if (gfp_mask & __GFP_WAIT) {
Jens Axboe22e2c502005-06-27 10:55:12 +0200890 rq = get_request_wait(q, rw, NULL);
Nick Piggind6344532005-06-28 20:45:14 -0700891 } else {
Jens Axboe22e2c502005-06-27 10:55:12 +0200892 rq = get_request(q, rw, NULL, gfp_mask);
Nick Piggind6344532005-06-28 20:45:14 -0700893 if (!rq)
894 spin_unlock_irq(q->queue_lock);
895 }
896 /* q->queue_lock is unlocked at this point */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700897
898 return rq;
899}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700900EXPORT_SYMBOL(blk_get_request);
901
902/**
Boaz Harrosh79eb63e2009-05-17 18:57:15 +0300903 * blk_make_request - given a bio, allocate a corresponding struct request.
Randy Dunlap8ebf9752009-06-11 20:00:41 -0700904 * @q: target request queue
Boaz Harrosh79eb63e2009-05-17 18:57:15 +0300905 * @bio: The bio describing the memory mappings that will be submitted for IO.
906 * It may be a chained-bio properly constructed by block/bio layer.
Randy Dunlap8ebf9752009-06-11 20:00:41 -0700907 * @gfp_mask: gfp flags to be used for memory allocation
Jens Axboedc72ef42006-07-20 14:54:05 +0200908 *
Boaz Harrosh79eb63e2009-05-17 18:57:15 +0300909 * blk_make_request is the parallel of generic_make_request for BLOCK_PC
910 * type commands. Where the struct request needs to be farther initialized by
911 * the caller. It is passed a &struct bio, which describes the memory info of
912 * the I/O transfer.
913 *
914 * The caller of blk_make_request must make sure that bi_io_vec
915 * are set to describe the memory buffers. That bio_data_dir() will return
916 * the needed direction of the request. (And all bio's in the passed bio-chain
917 * are properly set accordingly)
918 *
919 * If called under none-sleepable conditions, mapped bio buffers must not
920 * need bouncing, by calling the appropriate masked or flagged allocator,
921 * suitable for the target device. Otherwise the call to blk_queue_bounce will
922 * BUG.
Jens Axboe53674ac2009-05-19 19:52:35 +0200923 *
924 * WARNING: When allocating/cloning a bio-chain, careful consideration should be
925 * given to how you allocate bios. In particular, you cannot use __GFP_WAIT for
926 * anything but the first bio in the chain. Otherwise you risk waiting for IO
927 * completion of a bio that hasn't been submitted yet, thus resulting in a
928 * deadlock. Alternatively bios should be allocated using bio_kmalloc() instead
929 * of bio_alloc(), as that avoids the mempool deadlock.
930 * If possible a big IO should be split into smaller parts when allocation
931 * fails. Partial allocation should not be an error, or you risk a live-lock.
Jens Axboedc72ef42006-07-20 14:54:05 +0200932 */
Boaz Harrosh79eb63e2009-05-17 18:57:15 +0300933struct request *blk_make_request(struct request_queue *q, struct bio *bio,
934 gfp_t gfp_mask)
Jens Axboedc72ef42006-07-20 14:54:05 +0200935{
Boaz Harrosh79eb63e2009-05-17 18:57:15 +0300936 struct request *rq = blk_get_request(q, bio_data_dir(bio), gfp_mask);
937
938 if (unlikely(!rq))
939 return ERR_PTR(-ENOMEM);
940
941 for_each_bio(bio) {
942 struct bio *bounce_bio = bio;
943 int ret;
944
945 blk_queue_bounce(q, &bounce_bio);
946 ret = blk_rq_append_bio(q, rq, bounce_bio);
947 if (unlikely(ret)) {
948 blk_put_request(rq);
949 return ERR_PTR(ret);
950 }
951 }
952
953 return rq;
Jens Axboedc72ef42006-07-20 14:54:05 +0200954}
Boaz Harrosh79eb63e2009-05-17 18:57:15 +0300955EXPORT_SYMBOL(blk_make_request);
Jens Axboedc72ef42006-07-20 14:54:05 +0200956
957/**
Linus Torvalds1da177e2005-04-16 15:20:36 -0700958 * blk_requeue_request - put a request back on queue
959 * @q: request queue where request should be inserted
960 * @rq: request to be inserted
961 *
962 * Description:
963 * Drivers often keep queueing requests until the hardware cannot accept
964 * more, when that condition happens we need to put the request back
965 * on the queue. Must be called with queue lock held.
966 */
Jens Axboe165125e2007-07-24 09:28:11 +0200967void blk_requeue_request(struct request_queue *q, struct request *rq)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700968{
Jens Axboe242f9dc2008-09-14 05:55:09 -0700969 blk_delete_timer(rq);
970 blk_clear_rq_complete(rq);
Arnaldo Carvalho de Melo5f3ea372008-10-30 08:34:33 +0100971 trace_block_rq_requeue(q, rq);
Jens Axboe2056a782006-03-23 20:00:26 +0100972
Linus Torvalds1da177e2005-04-16 15:20:36 -0700973 if (blk_rq_tagged(rq))
974 blk_queue_end_tag(q, rq);
975
James Bottomleyba396a62009-05-27 14:17:08 +0200976 BUG_ON(blk_queued_rq(rq));
977
Linus Torvalds1da177e2005-04-16 15:20:36 -0700978 elv_requeue_request(q, rq);
979}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700980EXPORT_SYMBOL(blk_requeue_request);
981
982/**
Randy Dunlap710027a2008-08-19 20:13:11 +0200983 * blk_insert_request - insert a special request into a request queue
Linus Torvalds1da177e2005-04-16 15:20:36 -0700984 * @q: request queue where request should be inserted
985 * @rq: request to be inserted
986 * @at_head: insert request at head or tail of queue
987 * @data: private data
Linus Torvalds1da177e2005-04-16 15:20:36 -0700988 *
989 * Description:
990 * Many block devices need to execute commands asynchronously, so they don't
991 * block the whole kernel from preemption during request execution. This is
992 * accomplished normally by inserting aritficial requests tagged as
Randy Dunlap710027a2008-08-19 20:13:11 +0200993 * REQ_TYPE_SPECIAL in to the corresponding request queue, and letting them
994 * be scheduled for actual execution by the request queue.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700995 *
996 * We have the option of inserting the head or the tail of the queue.
997 * Typically we use the tail for new ioctls and so forth. We use the head
998 * of the queue for things like a QUEUE_FULL message from a device, or a
999 * host that is unable to accept a particular command.
1000 */
Jens Axboe165125e2007-07-24 09:28:11 +02001001void blk_insert_request(struct request_queue *q, struct request *rq,
Tejun Heo 867d1192005-04-24 02:06:05 -05001002 int at_head, void *data)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001003{
Tejun Heo 867d1192005-04-24 02:06:05 -05001004 int where = at_head ? ELEVATOR_INSERT_FRONT : ELEVATOR_INSERT_BACK;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001005 unsigned long flags;
1006
1007 /*
1008 * tell I/O scheduler that this isn't a regular read/write (ie it
1009 * must not attempt merges on this) and that it acts as a soft
1010 * barrier
1011 */
Jens Axboe4aff5e22006-08-10 08:44:47 +02001012 rq->cmd_type = REQ_TYPE_SPECIAL;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001013
1014 rq->special = data;
1015
1016 spin_lock_irqsave(q->queue_lock, flags);
1017
1018 /*
1019 * If command is tagged, release the tag
1020 */
Tejun Heo 867d1192005-04-24 02:06:05 -05001021 if (blk_rq_tagged(rq))
1022 blk_queue_end_tag(q, rq);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001023
Jerome Marchandb238b3d2007-10-23 15:05:46 +02001024 drive_stat_acct(rq, 1);
Tejun Heo 867d1192005-04-24 02:06:05 -05001025 __elv_add_request(q, rq, where, 0);
Tejun Heoa7f55792009-04-23 11:05:17 +09001026 __blk_run_queue(q);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001027 spin_unlock_irqrestore(q->queue_lock, flags);
1028}
Linus Torvalds1da177e2005-04-16 15:20:36 -07001029EXPORT_SYMBOL(blk_insert_request);
1030
Linus Torvalds1da177e2005-04-16 15:20:36 -07001031/*
1032 * add-request adds a request to the linked list.
1033 * queue lock is held and interrupts disabled, as we muck with the
1034 * request queue list.
1035 */
Jens Axboe6728cb02008-01-31 13:03:55 +01001036static inline void add_request(struct request_queue *q, struct request *req)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001037{
Jerome Marchandb238b3d2007-10-23 15:05:46 +02001038 drive_stat_acct(req, 1);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001039
Linus Torvalds1da177e2005-04-16 15:20:36 -07001040 /*
1041 * elevator indicated where it wants this request to be
1042 * inserted at elevator_merge time
1043 */
1044 __elv_add_request(q, req, ELEVATOR_INSERT_SORT, 0);
1045}
Jens Axboe6728cb02008-01-31 13:03:55 +01001046
Tejun Heo074a7ac2008-08-25 19:56:14 +09001047static void part_round_stats_single(int cpu, struct hd_struct *part,
1048 unsigned long now)
1049{
1050 if (now == part->stamp)
1051 return;
1052
Nikanth Karthikesan316d3152009-10-06 20:16:55 +02001053 if (part_in_flight(part)) {
Tejun Heo074a7ac2008-08-25 19:56:14 +09001054 __part_stat_add(cpu, part, time_in_queue,
Nikanth Karthikesan316d3152009-10-06 20:16:55 +02001055 part_in_flight(part) * (now - part->stamp));
Tejun Heo074a7ac2008-08-25 19:56:14 +09001056 __part_stat_add(cpu, part, io_ticks, (now - part->stamp));
1057 }
1058 part->stamp = now;
1059}
1060
1061/**
Randy Dunlap496aa8a2008-10-16 07:46:23 +02001062 * part_round_stats() - Round off the performance stats on a struct disk_stats.
1063 * @cpu: cpu number for stats access
1064 * @part: target partition
Linus Torvalds1da177e2005-04-16 15:20:36 -07001065 *
1066 * The average IO queue length and utilisation statistics are maintained
1067 * by observing the current state of the queue length and the amount of
1068 * time it has been in this state for.
1069 *
1070 * Normally, that accounting is done on IO completion, but that can result
1071 * in more than a second's worth of IO being accounted for within any one
1072 * second, leading to >100% utilisation. To deal with that, we call this
1073 * function to do a round-off before returning the results when reading
1074 * /proc/diskstats. This accounts immediately for all queue usage up to
1075 * the current jiffies and restarts the counters again.
1076 */
Tejun Heoc9959052008-08-25 19:47:21 +09001077void part_round_stats(int cpu, struct hd_struct *part)
Jerome Marchand6f2576a2008-02-08 11:04:35 +01001078{
1079 unsigned long now = jiffies;
1080
Tejun Heo074a7ac2008-08-25 19:56:14 +09001081 if (part->partno)
1082 part_round_stats_single(cpu, &part_to_disk(part)->part0, now);
1083 part_round_stats_single(cpu, part, now);
Jerome Marchand6f2576a2008-02-08 11:04:35 +01001084}
Tejun Heo074a7ac2008-08-25 19:56:14 +09001085EXPORT_SYMBOL_GPL(part_round_stats);
Jerome Marchand6f2576a2008-02-08 11:04:35 +01001086
Linus Torvalds1da177e2005-04-16 15:20:36 -07001087/*
1088 * queue lock must be held
1089 */
Jens Axboe165125e2007-07-24 09:28:11 +02001090void __blk_put_request(struct request_queue *q, struct request *req)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001091{
Linus Torvalds1da177e2005-04-16 15:20:36 -07001092 if (unlikely(!q))
1093 return;
1094 if (unlikely(--req->ref_count))
1095 return;
1096
Tejun Heo8922e162005-10-20 16:23:44 +02001097 elv_completed_request(q, req);
1098
Boaz Harrosh1cd96c22009-03-24 12:35:07 +01001099 /* this is a bio leak */
1100 WARN_ON(req->bio != NULL);
1101
Linus Torvalds1da177e2005-04-16 15:20:36 -07001102 /*
1103 * Request may not have originated from ll_rw_blk. if not,
1104 * it didn't come out of our reserved rq pools
1105 */
Jens Axboe49171e52006-08-10 08:59:11 +02001106 if (req->cmd_flags & REQ_ALLOCED) {
Jens Axboe1faa16d2009-04-06 14:48:01 +02001107 int is_sync = rq_is_sync(req) != 0;
Jens Axboe4aff5e22006-08-10 08:44:47 +02001108 int priv = req->cmd_flags & REQ_ELVPRIV;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001109
Linus Torvalds1da177e2005-04-16 15:20:36 -07001110 BUG_ON(!list_empty(&req->queuelist));
Jens Axboe98170642006-07-28 09:23:08 +02001111 BUG_ON(!hlist_unhashed(&req->hash));
Linus Torvalds1da177e2005-04-16 15:20:36 -07001112
1113 blk_free_request(q, req);
Jens Axboe1faa16d2009-04-06 14:48:01 +02001114 freed_request(q, is_sync, priv);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001115 }
1116}
Mike Christie6e39b692005-11-11 05:30:24 -06001117EXPORT_SYMBOL_GPL(__blk_put_request);
1118
Linus Torvalds1da177e2005-04-16 15:20:36 -07001119void blk_put_request(struct request *req)
1120{
Tejun Heo8922e162005-10-20 16:23:44 +02001121 unsigned long flags;
Jens Axboe165125e2007-07-24 09:28:11 +02001122 struct request_queue *q = req->q;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001123
FUJITA Tomonori52a93ba2008-07-15 21:21:45 +02001124 spin_lock_irqsave(q->queue_lock, flags);
1125 __blk_put_request(q, req);
1126 spin_unlock_irqrestore(q->queue_lock, flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001127}
Linus Torvalds1da177e2005-04-16 15:20:36 -07001128EXPORT_SYMBOL(blk_put_request);
1129
Jens Axboe86db1e22008-01-29 14:53:40 +01001130void init_request_from_bio(struct request *req, struct bio *bio)
Tejun Heo52d9e672006-01-06 09:49:58 +01001131{
Jens Axboec7c22e42008-09-13 20:26:01 +02001132 req->cpu = bio->bi_comp_cpu;
Jens Axboe4aff5e22006-08-10 08:44:47 +02001133 req->cmd_type = REQ_TYPE_FS;
Tejun Heo52d9e672006-01-06 09:49:58 +01001134
1135 /*
Tejun Heoa82afdf2009-07-03 17:48:16 +09001136 * Inherit FAILFAST from bio (for read-ahead, and explicit
1137 * FAILFAST). FAILFAST flags are identical for req and bio.
Tejun Heo52d9e672006-01-06 09:49:58 +01001138 */
Jens Axboe1f98a132009-09-11 14:32:04 +02001139 if (bio_rw_flagged(bio, BIO_RW_AHEAD))
Tejun Heoa82afdf2009-07-03 17:48:16 +09001140 req->cmd_flags |= REQ_FAILFAST_MASK;
1141 else
1142 req->cmd_flags |= bio->bi_rw & REQ_FAILFAST_MASK;
Tejun Heo52d9e672006-01-06 09:49:58 +01001143
Jens Axboe1f98a132009-09-11 14:32:04 +02001144 if (unlikely(bio_rw_flagged(bio, BIO_RW_DISCARD))) {
David Woodhousee17fc0a2008-08-09 16:42:20 +01001145 req->cmd_flags |= REQ_DISCARD;
Jens Axboe1f98a132009-09-11 14:32:04 +02001146 if (bio_rw_flagged(bio, BIO_RW_BARRIER))
David Woodhousee17fc0a2008-08-09 16:42:20 +01001147 req->cmd_flags |= REQ_SOFTBARRIER;
Jens Axboe1f98a132009-09-11 14:32:04 +02001148 } else if (unlikely(bio_rw_flagged(bio, BIO_RW_BARRIER)))
Tejun Heoe4025f62009-04-23 11:05:17 +09001149 req->cmd_flags |= REQ_HARDBARRIER;
Tejun Heo52d9e672006-01-06 09:49:58 +01001150
Jens Axboe1f98a132009-09-11 14:32:04 +02001151 if (bio_rw_flagged(bio, BIO_RW_SYNCIO))
Jens Axboe4aff5e22006-08-10 08:44:47 +02001152 req->cmd_flags |= REQ_RW_SYNC;
Jens Axboe1f98a132009-09-11 14:32:04 +02001153 if (bio_rw_flagged(bio, BIO_RW_META))
Jens Axboe5404bc72006-08-10 09:01:02 +02001154 req->cmd_flags |= REQ_RW_META;
Jens Axboe1f98a132009-09-11 14:32:04 +02001155 if (bio_rw_flagged(bio, BIO_RW_NOIDLE))
Jens Axboeaeb6faf2009-04-06 14:48:07 +02001156 req->cmd_flags |= REQ_NOIDLE;
Jens Axboeb31dc662006-06-13 08:26:10 +02001157
Tejun Heo52d9e672006-01-06 09:49:58 +01001158 req->errors = 0;
Tejun Heoa2dec7b2009-05-07 22:24:44 +09001159 req->__sector = bio->bi_sector;
Tejun Heo52d9e672006-01-06 09:49:58 +01001160 req->ioprio = bio_prio(bio);
NeilBrownbc1c56f2007-08-16 13:31:30 +02001161 blk_rq_bio_prep(req->q, req, bio);
Tejun Heo52d9e672006-01-06 09:49:58 +01001162}
1163
Jens Axboe644b2d92009-04-06 14:48:06 +02001164/*
1165 * Only disabling plugging for non-rotational devices if it does tagging
1166 * as well, otherwise we do need the proper merging
1167 */
1168static inline bool queue_should_plug(struct request_queue *q)
1169{
Jens Axboe79da0642010-02-23 08:40:43 +01001170 return !(blk_queue_nonrot(q) && blk_queue_tagged(q));
Jens Axboe644b2d92009-04-06 14:48:06 +02001171}
1172
Jens Axboe165125e2007-07-24 09:28:11 +02001173static int __make_request(struct request_queue *q, struct bio *bio)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001174{
Nick Piggin450991b2005-06-28 20:45:13 -07001175 struct request *req;
Tejun Heo2e46e8b2009-05-07 22:24:41 +09001176 int el_ret;
1177 unsigned int bytes = bio->bi_size;
Jens Axboe51da90f2006-07-18 04:14:45 +02001178 const unsigned short prio = bio_prio(bio);
Jens Axboe1f98a132009-09-11 14:32:04 +02001179 const bool sync = bio_rw_flagged(bio, BIO_RW_SYNCIO);
1180 const bool unplug = bio_rw_flagged(bio, BIO_RW_UNPLUG);
Tejun Heo80a761f2009-07-03 17:48:17 +09001181 const unsigned int ff = bio->bi_rw & REQ_FAILFAST_MASK;
Jens Axboe7749a8d2006-12-13 13:02:26 +01001182 int rw_flags;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001183
Mark McLoughlin6cafb122009-10-24 14:14:31 +02001184 if (bio_rw_flagged(bio, BIO_RW_BARRIER) &&
NeilBrowndb64f682009-06-30 09:35:44 +02001185 (q->next_ordered == QUEUE_ORDERED_NONE)) {
1186 bio_endio(bio, -EOPNOTSUPP);
1187 return 0;
1188 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001189 /*
1190 * low level driver can indicate that it wants pages above a
1191 * certain limit bounced to low memory (ie for highmem, or even
1192 * ISA dma in theory)
1193 */
1194 blk_queue_bounce(q, &bio);
1195
Linus Torvalds1da177e2005-04-16 15:20:36 -07001196 spin_lock_irq(q->queue_lock);
1197
Jens Axboe1f98a132009-09-11 14:32:04 +02001198 if (unlikely(bio_rw_flagged(bio, BIO_RW_BARRIER)) || elv_queue_empty(q))
Linus Torvalds1da177e2005-04-16 15:20:36 -07001199 goto get_rq;
1200
1201 el_ret = elv_merge(q, &req, bio);
1202 switch (el_ret) {
Jens Axboe6728cb02008-01-31 13:03:55 +01001203 case ELEVATOR_BACK_MERGE:
1204 BUG_ON(!rq_mergeable(req));
Linus Torvalds1da177e2005-04-16 15:20:36 -07001205
Jens Axboe6728cb02008-01-31 13:03:55 +01001206 if (!ll_back_merge_fn(q, req, bio))
1207 break;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001208
Arnaldo Carvalho de Melo5f3ea372008-10-30 08:34:33 +01001209 trace_block_bio_backmerge(q, bio);
Jens Axboe2056a782006-03-23 20:00:26 +01001210
Tejun Heo80a761f2009-07-03 17:48:17 +09001211 if ((req->cmd_flags & REQ_FAILFAST_MASK) != ff)
1212 blk_rq_set_mixed_merge(req);
1213
Jens Axboe6728cb02008-01-31 13:03:55 +01001214 req->biotail->bi_next = bio;
1215 req->biotail = bio;
Tejun Heoa2dec7b2009-05-07 22:24:44 +09001216 req->__data_len += bytes;
Jens Axboe6728cb02008-01-31 13:03:55 +01001217 req->ioprio = ioprio_best(req->ioprio, prio);
Jens Axboeab780f12008-08-26 10:25:02 +02001218 if (!blk_rq_cpu_valid(req))
1219 req->cpu = bio->bi_comp_cpu;
Jens Axboe6728cb02008-01-31 13:03:55 +01001220 drive_stat_acct(req, 0);
Divyesh Shah812d4022010-04-08 21:14:23 -07001221 elv_bio_merged(q, req, bio);
Jens Axboe6728cb02008-01-31 13:03:55 +01001222 if (!attempt_back_merge(q, req))
1223 elv_merged_request(q, req, el_ret);
1224 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001225
Jens Axboe6728cb02008-01-31 13:03:55 +01001226 case ELEVATOR_FRONT_MERGE:
1227 BUG_ON(!rq_mergeable(req));
Linus Torvalds1da177e2005-04-16 15:20:36 -07001228
Jens Axboe6728cb02008-01-31 13:03:55 +01001229 if (!ll_front_merge_fn(q, req, bio))
1230 break;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001231
Arnaldo Carvalho de Melo5f3ea372008-10-30 08:34:33 +01001232 trace_block_bio_frontmerge(q, bio);
Jens Axboe2056a782006-03-23 20:00:26 +01001233
Tejun Heo80a761f2009-07-03 17:48:17 +09001234 if ((req->cmd_flags & REQ_FAILFAST_MASK) != ff) {
1235 blk_rq_set_mixed_merge(req);
1236 req->cmd_flags &= ~REQ_FAILFAST_MASK;
1237 req->cmd_flags |= ff;
1238 }
1239
Jens Axboe6728cb02008-01-31 13:03:55 +01001240 bio->bi_next = req->bio;
1241 req->bio = bio;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001242
Jens Axboe6728cb02008-01-31 13:03:55 +01001243 /*
1244 * may not be valid. if the low level driver said
1245 * it didn't need a bounce buffer then it better
1246 * not touch req->buffer either...
1247 */
1248 req->buffer = bio_data(bio);
Tejun Heoa2dec7b2009-05-07 22:24:44 +09001249 req->__sector = bio->bi_sector;
1250 req->__data_len += bytes;
Jens Axboe6728cb02008-01-31 13:03:55 +01001251 req->ioprio = ioprio_best(req->ioprio, prio);
Jens Axboeab780f12008-08-26 10:25:02 +02001252 if (!blk_rq_cpu_valid(req))
1253 req->cpu = bio->bi_comp_cpu;
Jens Axboe6728cb02008-01-31 13:03:55 +01001254 drive_stat_acct(req, 0);
Divyesh Shah812d4022010-04-08 21:14:23 -07001255 elv_bio_merged(q, req, bio);
Jens Axboe6728cb02008-01-31 13:03:55 +01001256 if (!attempt_front_merge(q, req))
1257 elv_merged_request(q, req, el_ret);
1258 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001259
Jens Axboe6728cb02008-01-31 13:03:55 +01001260 /* ELV_NO_MERGE: elevator says don't/can't merge. */
1261 default:
1262 ;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001263 }
1264
Linus Torvalds1da177e2005-04-16 15:20:36 -07001265get_rq:
Nick Piggin450991b2005-06-28 20:45:13 -07001266 /*
Jens Axboe7749a8d2006-12-13 13:02:26 +01001267 * This sync check and mask will be re-done in init_request_from_bio(),
1268 * but we need to set it earlier to expose the sync flag to the
1269 * rq allocator and io schedulers.
1270 */
1271 rw_flags = bio_data_dir(bio);
1272 if (sync)
1273 rw_flags |= REQ_RW_SYNC;
1274
1275 /*
Nick Piggin450991b2005-06-28 20:45:13 -07001276 * Grab a free request. This is might sleep but can not fail.
Nick Piggind6344532005-06-28 20:45:14 -07001277 * Returns with the queue unlocked.
Nick Piggin450991b2005-06-28 20:45:13 -07001278 */
Jens Axboe7749a8d2006-12-13 13:02:26 +01001279 req = get_request_wait(q, rw_flags, bio);
Nick Piggind6344532005-06-28 20:45:14 -07001280
Nick Piggin450991b2005-06-28 20:45:13 -07001281 /*
1282 * After dropping the lock and possibly sleeping here, our request
1283 * may now be mergeable after it had proven unmergeable (above).
1284 * We don't worry about that case for efficiency. It won't happen
1285 * often, and the elevators are able to handle it.
1286 */
Tejun Heo52d9e672006-01-06 09:49:58 +01001287 init_request_from_bio(req, bio);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001288
Nick Piggin450991b2005-06-28 20:45:13 -07001289 spin_lock_irq(q->queue_lock);
Jens Axboec7c22e42008-09-13 20:26:01 +02001290 if (test_bit(QUEUE_FLAG_SAME_COMP, &q->queue_flags) ||
1291 bio_flagged(bio, BIO_CPU_AFFINE))
1292 req->cpu = blk_cpu_to_group(smp_processor_id());
Jens Axboe644b2d92009-04-06 14:48:06 +02001293 if (queue_should_plug(q) && elv_queue_empty(q))
Nick Piggin450991b2005-06-28 20:45:13 -07001294 blk_plug_device(q);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001295 add_request(q, req);
1296out:
Jens Axboe644b2d92009-04-06 14:48:06 +02001297 if (unplug || !queue_should_plug(q))
Linus Torvalds1da177e2005-04-16 15:20:36 -07001298 __generic_unplug_device(q);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001299 spin_unlock_irq(q->queue_lock);
1300 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001301}
1302
1303/*
1304 * If bio->bi_dev is a partition, remap the location
1305 */
1306static inline void blk_partition_remap(struct bio *bio)
1307{
1308 struct block_device *bdev = bio->bi_bdev;
1309
Jens Axboebf2de6f2007-09-27 13:01:25 +02001310 if (bio_sectors(bio) && bdev != bdev->bd_contains) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001311 struct hd_struct *p = bdev->bd_part;
Jens Axboea3623572005-11-01 09:26:16 +01001312
Linus Torvalds1da177e2005-04-16 15:20:36 -07001313 bio->bi_sector += p->start_sect;
1314 bio->bi_bdev = bdev->bd_contains;
Alan D. Brunellec7149d62007-08-07 15:30:23 +02001315
Arnaldo Carvalho de Melo5f3ea372008-10-30 08:34:33 +01001316 trace_block_remap(bdev_get_queue(bio->bi_bdev), bio,
Alan D. Brunelle22a7c312009-05-04 16:35:08 -04001317 bdev->bd_dev,
Alan D. Brunellec7149d62007-08-07 15:30:23 +02001318 bio->bi_sector - p->start_sect);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001319 }
1320}
1321
Linus Torvalds1da177e2005-04-16 15:20:36 -07001322static void handle_bad_sector(struct bio *bio)
1323{
1324 char b[BDEVNAME_SIZE];
1325
1326 printk(KERN_INFO "attempt to access beyond end of device\n");
1327 printk(KERN_INFO "%s: rw=%ld, want=%Lu, limit=%Lu\n",
1328 bdevname(bio->bi_bdev, b),
1329 bio->bi_rw,
1330 (unsigned long long)bio->bi_sector + bio_sectors(bio),
1331 (long long)(bio->bi_bdev->bd_inode->i_size >> 9));
1332
1333 set_bit(BIO_EOF, &bio->bi_flags);
1334}
1335
Akinobu Mitac17bb492006-12-08 02:39:46 -08001336#ifdef CONFIG_FAIL_MAKE_REQUEST
1337
1338static DECLARE_FAULT_ATTR(fail_make_request);
1339
1340static int __init setup_fail_make_request(char *str)
1341{
1342 return setup_fault_attr(&fail_make_request, str);
1343}
1344__setup("fail_make_request=", setup_fail_make_request);
1345
1346static int should_fail_request(struct bio *bio)
1347{
Tejun Heoeddb2e22008-08-25 19:56:13 +09001348 struct hd_struct *part = bio->bi_bdev->bd_part;
1349
1350 if (part_to_disk(part)->part0.make_it_fail || part->make_it_fail)
Akinobu Mitac17bb492006-12-08 02:39:46 -08001351 return should_fail(&fail_make_request, bio->bi_size);
1352
1353 return 0;
1354}
1355
1356static int __init fail_make_request_debugfs(void)
1357{
1358 return init_fault_attr_dentries(&fail_make_request,
1359 "fail_make_request");
1360}
1361
1362late_initcall(fail_make_request_debugfs);
1363
1364#else /* CONFIG_FAIL_MAKE_REQUEST */
1365
1366static inline int should_fail_request(struct bio *bio)
1367{
1368 return 0;
1369}
1370
1371#endif /* CONFIG_FAIL_MAKE_REQUEST */
1372
Jens Axboec07e2b42007-07-18 13:27:58 +02001373/*
1374 * Check whether this bio extends beyond the end of the device.
1375 */
1376static inline int bio_check_eod(struct bio *bio, unsigned int nr_sectors)
1377{
1378 sector_t maxsector;
1379
1380 if (!nr_sectors)
1381 return 0;
1382
1383 /* Test device or partition size, when known. */
1384 maxsector = bio->bi_bdev->bd_inode->i_size >> 9;
1385 if (maxsector) {
1386 sector_t sector = bio->bi_sector;
1387
1388 if (maxsector < nr_sectors || maxsector - nr_sectors < sector) {
1389 /*
1390 * This may well happen - the kernel calls bread()
1391 * without checking the size of the device, e.g., when
1392 * mounting a device.
1393 */
1394 handle_bad_sector(bio);
1395 return 1;
1396 }
1397 }
1398
1399 return 0;
1400}
1401
Linus Torvalds1da177e2005-04-16 15:20:36 -07001402/**
Randy Dunlap710027a2008-08-19 20:13:11 +02001403 * generic_make_request - hand a buffer to its device driver for I/O
Linus Torvalds1da177e2005-04-16 15:20:36 -07001404 * @bio: The bio describing the location in memory and on the device.
1405 *
1406 * generic_make_request() is used to make I/O requests of block
1407 * devices. It is passed a &struct bio, which describes the I/O that needs
1408 * to be done.
1409 *
1410 * generic_make_request() does not return any status. The
1411 * success/failure status of the request, along with notification of
1412 * completion, is delivered asynchronously through the bio->bi_end_io
1413 * function described (one day) else where.
1414 *
1415 * The caller of generic_make_request must make sure that bi_io_vec
1416 * are set to describe the memory buffer, and that bi_dev and bi_sector are
1417 * set to describe the device address, and the
1418 * bi_end_io and optionally bi_private are set to describe how
1419 * completion notification should be signaled.
1420 *
1421 * generic_make_request and the drivers it calls may use bi_next if this
1422 * bio happens to be merged with someone else, and may change bi_dev and
1423 * bi_sector for remaps as it sees fit. So the values of these fields
1424 * should NOT be depended on after the call to generic_make_request.
1425 */
Neil Brownd89d8792007-05-01 09:53:42 +02001426static inline void __generic_make_request(struct bio *bio)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001427{
Jens Axboe165125e2007-07-24 09:28:11 +02001428 struct request_queue *q;
NeilBrown5ddfe962006-10-30 22:07:21 -08001429 sector_t old_sector;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001430 int ret, nr_sectors = bio_sectors(bio);
Jens Axboe2056a782006-03-23 20:00:26 +01001431 dev_t old_dev;
Jens Axboe51fd77b2007-11-02 08:49:08 +01001432 int err = -EIO;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001433
1434 might_sleep();
Linus Torvalds1da177e2005-04-16 15:20:36 -07001435
Jens Axboec07e2b42007-07-18 13:27:58 +02001436 if (bio_check_eod(bio, nr_sectors))
1437 goto end_io;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001438
1439 /*
1440 * Resolve the mapping until finished. (drivers are
1441 * still free to implement/resolve their own stacking
1442 * by explicitly returning 0)
1443 *
1444 * NOTE: we don't repeat the blk_size check for each new device.
1445 * Stacking drivers are expected to know what they are doing.
1446 */
NeilBrown5ddfe962006-10-30 22:07:21 -08001447 old_sector = -1;
Jens Axboe2056a782006-03-23 20:00:26 +01001448 old_dev = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001449 do {
1450 char b[BDEVNAME_SIZE];
1451
1452 q = bdev_get_queue(bio->bi_bdev);
Tejun Heoa7384672008-11-28 13:32:03 +09001453 if (unlikely(!q)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001454 printk(KERN_ERR
1455 "generic_make_request: Trying to access "
1456 "nonexistent block-device %s (%Lu)\n",
1457 bdevname(bio->bi_bdev, b),
1458 (long long) bio->bi_sector);
Tejun Heoa7384672008-11-28 13:32:03 +09001459 goto end_io;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001460 }
1461
Christoph Hellwig67efc922009-09-30 13:54:20 +02001462 if (unlikely(!bio_rw_flagged(bio, BIO_RW_DISCARD) &&
1463 nr_sectors > queue_max_hw_sectors(q))) {
Jens Axboe6728cb02008-01-31 13:03:55 +01001464 printk(KERN_ERR "bio too big device %s (%u > %u)\n",
Martin K. Petersenae03bf62009-05-22 17:17:50 -04001465 bdevname(bio->bi_bdev, b),
1466 bio_sectors(bio),
1467 queue_max_hw_sectors(q));
Linus Torvalds1da177e2005-04-16 15:20:36 -07001468 goto end_io;
1469 }
1470
Nick Pigginfde6ad22005-06-23 00:08:53 -07001471 if (unlikely(test_bit(QUEUE_FLAG_DEAD, &q->queue_flags)))
Linus Torvalds1da177e2005-04-16 15:20:36 -07001472 goto end_io;
1473
Akinobu Mitac17bb492006-12-08 02:39:46 -08001474 if (should_fail_request(bio))
1475 goto end_io;
1476
Linus Torvalds1da177e2005-04-16 15:20:36 -07001477 /*
1478 * If this device has partitions, remap block n
1479 * of partition p to block n+start(p) of the disk.
1480 */
1481 blk_partition_remap(bio);
1482
Martin K. Petersen7ba1ba12008-06-30 20:04:41 +02001483 if (bio_integrity_enabled(bio) && bio_integrity_prep(bio))
1484 goto end_io;
1485
NeilBrown5ddfe962006-10-30 22:07:21 -08001486 if (old_sector != -1)
Alan D. Brunelle22a7c312009-05-04 16:35:08 -04001487 trace_block_remap(q, bio, old_dev, old_sector);
Jens Axboe2056a782006-03-23 20:00:26 +01001488
NeilBrown5ddfe962006-10-30 22:07:21 -08001489 old_sector = bio->bi_sector;
Jens Axboe2056a782006-03-23 20:00:26 +01001490 old_dev = bio->bi_bdev->bd_dev;
1491
Jens Axboec07e2b42007-07-18 13:27:58 +02001492 if (bio_check_eod(bio, nr_sectors))
1493 goto end_io;
Tejun Heoa7384672008-11-28 13:32:03 +09001494
Jens Axboe1f98a132009-09-11 14:32:04 +02001495 if (bio_rw_flagged(bio, BIO_RW_DISCARD) &&
Christoph Hellwigc15227d2009-09-30 13:52:12 +02001496 !blk_queue_discard(q)) {
Jens Axboe51fd77b2007-11-02 08:49:08 +01001497 err = -EOPNOTSUPP;
1498 goto end_io;
1499 }
NeilBrown5ddfe962006-10-30 22:07:21 -08001500
Minchan Kim01edede2009-09-08 21:56:38 +02001501 trace_block_bio_queue(q, bio);
1502
Linus Torvalds1da177e2005-04-16 15:20:36 -07001503 ret = q->make_request_fn(q, bio);
1504 } while (ret);
Tejun Heoa7384672008-11-28 13:32:03 +09001505
1506 return;
1507
1508end_io:
1509 bio_endio(bio, err);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001510}
1511
Neil Brownd89d8792007-05-01 09:53:42 +02001512/*
1513 * We only want one ->make_request_fn to be active at a time,
1514 * else stack usage with stacked devices could be a problem.
Akinobu Mitabddd87c2010-02-23 08:55:42 +01001515 * So use current->bio_list to keep a list of requests
Neil Brownd89d8792007-05-01 09:53:42 +02001516 * submited by a make_request_fn function.
Akinobu Mitabddd87c2010-02-23 08:55:42 +01001517 * current->bio_list is also used as a flag to say if
Neil Brownd89d8792007-05-01 09:53:42 +02001518 * generic_make_request is currently active in this task or not.
1519 * If it is NULL, then no make_request is active. If it is non-NULL,
1520 * then a make_request is active, and new requests should be added
1521 * at the tail
1522 */
1523void generic_make_request(struct bio *bio)
1524{
Akinobu Mitabddd87c2010-02-23 08:55:42 +01001525 struct bio_list bio_list_on_stack;
1526
1527 if (current->bio_list) {
Neil Brownd89d8792007-05-01 09:53:42 +02001528 /* make_request is active */
Akinobu Mitabddd87c2010-02-23 08:55:42 +01001529 bio_list_add(current->bio_list, bio);
Neil Brownd89d8792007-05-01 09:53:42 +02001530 return;
1531 }
1532 /* following loop may be a bit non-obvious, and so deserves some
1533 * explanation.
1534 * Before entering the loop, bio->bi_next is NULL (as all callers
1535 * ensure that) so we have a list with a single bio.
1536 * We pretend that we have just taken it off a longer list, so
Akinobu Mitabddd87c2010-02-23 08:55:42 +01001537 * we assign bio_list to a pointer to the bio_list_on_stack,
1538 * thus initialising the bio_list of new bios to be
Neil Brownd89d8792007-05-01 09:53:42 +02001539 * added. __generic_make_request may indeed add some more bios
1540 * through a recursive call to generic_make_request. If it
1541 * did, we find a non-NULL value in bio_list and re-enter the loop
1542 * from the top. In this case we really did just take the bio
Akinobu Mitabddd87c2010-02-23 08:55:42 +01001543 * of the top of the list (no pretending) and so remove it from
1544 * bio_list, and call into __generic_make_request again.
Neil Brownd89d8792007-05-01 09:53:42 +02001545 *
1546 * The loop was structured like this to make only one call to
1547 * __generic_make_request (which is important as it is large and
1548 * inlined) and to keep the structure simple.
1549 */
1550 BUG_ON(bio->bi_next);
Akinobu Mitabddd87c2010-02-23 08:55:42 +01001551 bio_list_init(&bio_list_on_stack);
1552 current->bio_list = &bio_list_on_stack;
Neil Brownd89d8792007-05-01 09:53:42 +02001553 do {
Neil Brownd89d8792007-05-01 09:53:42 +02001554 __generic_make_request(bio);
Akinobu Mitabddd87c2010-02-23 08:55:42 +01001555 bio = bio_list_pop(current->bio_list);
Neil Brownd89d8792007-05-01 09:53:42 +02001556 } while (bio);
Akinobu Mitabddd87c2010-02-23 08:55:42 +01001557 current->bio_list = NULL; /* deactivate */
Neil Brownd89d8792007-05-01 09:53:42 +02001558}
Linus Torvalds1da177e2005-04-16 15:20:36 -07001559EXPORT_SYMBOL(generic_make_request);
1560
1561/**
Randy Dunlap710027a2008-08-19 20:13:11 +02001562 * submit_bio - submit a bio to the block device layer for I/O
Linus Torvalds1da177e2005-04-16 15:20:36 -07001563 * @rw: whether to %READ or %WRITE, or maybe to %READA (read ahead)
1564 * @bio: The &struct bio which describes the I/O
1565 *
1566 * submit_bio() is very similar in purpose to generic_make_request(), and
1567 * uses that function to do most of the work. Both are fairly rough
Randy Dunlap710027a2008-08-19 20:13:11 +02001568 * interfaces; @bio must be presetup and ready for I/O.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001569 *
1570 */
1571void submit_bio(int rw, struct bio *bio)
1572{
1573 int count = bio_sectors(bio);
1574
Jens Axboe22e2c502005-06-27 10:55:12 +02001575 bio->bi_rw |= rw;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001576
Jens Axboebf2de6f2007-09-27 13:01:25 +02001577 /*
1578 * If it's a regular read/write or a barrier with data attached,
1579 * go through the normal accounting stuff before submission.
1580 */
Jens Axboea9c701e2008-08-08 11:04:44 +02001581 if (bio_has_data(bio)) {
Jens Axboebf2de6f2007-09-27 13:01:25 +02001582 if (rw & WRITE) {
1583 count_vm_events(PGPGOUT, count);
1584 } else {
1585 task_io_account_read(bio->bi_size);
1586 count_vm_events(PGPGIN, count);
1587 }
1588
1589 if (unlikely(block_dump)) {
1590 char b[BDEVNAME_SIZE];
1591 printk(KERN_DEBUG "%s(%d): %s block %Lu on %s\n",
Pavel Emelyanovba25f9d2007-10-18 23:40:40 -07001592 current->comm, task_pid_nr(current),
Jens Axboebf2de6f2007-09-27 13:01:25 +02001593 (rw & WRITE) ? "WRITE" : "READ",
1594 (unsigned long long)bio->bi_sector,
Jens Axboe6728cb02008-01-31 13:03:55 +01001595 bdevname(bio->bi_bdev, b));
Jens Axboebf2de6f2007-09-27 13:01:25 +02001596 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001597 }
1598
1599 generic_make_request(bio);
1600}
Linus Torvalds1da177e2005-04-16 15:20:36 -07001601EXPORT_SYMBOL(submit_bio);
1602
Kiyoshi Ueda3bcddea2007-12-11 17:52:28 -05001603/**
Kiyoshi Ueda82124d62008-09-18 10:45:38 -04001604 * blk_rq_check_limits - Helper function to check a request for the queue limit
1605 * @q: the queue
1606 * @rq: the request being checked
1607 *
1608 * Description:
1609 * @rq may have been made based on weaker limitations of upper-level queues
1610 * in request stacking drivers, and it may violate the limitation of @q.
1611 * Since the block layer and the underlying device driver trust @rq
1612 * after it is inserted to @q, it should be checked against @q before
1613 * the insertion using this generic function.
1614 *
1615 * This function should also be useful for request stacking drivers
1616 * in some cases below, so export this fuction.
1617 * Request stacking drivers like request-based dm may change the queue
1618 * limits while requests are in the queue (e.g. dm's table swapping).
1619 * Such request stacking drivers should check those requests agaist
1620 * the new queue limits again when they dispatch those requests,
1621 * although such checkings are also done against the old queue limits
1622 * when submitting requests.
1623 */
1624int blk_rq_check_limits(struct request_queue *q, struct request *rq)
1625{
Martin K. Petersenae03bf62009-05-22 17:17:50 -04001626 if (blk_rq_sectors(rq) > queue_max_sectors(q) ||
1627 blk_rq_bytes(rq) > queue_max_hw_sectors(q) << 9) {
Kiyoshi Ueda82124d62008-09-18 10:45:38 -04001628 printk(KERN_ERR "%s: over max size limit.\n", __func__);
1629 return -EIO;
1630 }
1631
1632 /*
1633 * queue's settings related to segment counting like q->bounce_pfn
1634 * may differ from that of other stacking queues.
1635 * Recalculate it to check the request correctly on this queue's
1636 * limitation.
1637 */
1638 blk_recalc_rq_segments(rq);
Martin K. Petersen8a783622010-02-26 00:20:39 -05001639 if (rq->nr_phys_segments > queue_max_segments(q)) {
Kiyoshi Ueda82124d62008-09-18 10:45:38 -04001640 printk(KERN_ERR "%s: over max segments limit.\n", __func__);
1641 return -EIO;
1642 }
1643
1644 return 0;
1645}
1646EXPORT_SYMBOL_GPL(blk_rq_check_limits);
1647
1648/**
1649 * blk_insert_cloned_request - Helper for stacking drivers to submit a request
1650 * @q: the queue to submit the request
1651 * @rq: the request being queued
1652 */
1653int blk_insert_cloned_request(struct request_queue *q, struct request *rq)
1654{
1655 unsigned long flags;
1656
1657 if (blk_rq_check_limits(q, rq))
1658 return -EIO;
1659
1660#ifdef CONFIG_FAIL_MAKE_REQUEST
1661 if (rq->rq_disk && rq->rq_disk->part0.make_it_fail &&
1662 should_fail(&fail_make_request, blk_rq_bytes(rq)))
1663 return -EIO;
1664#endif
1665
1666 spin_lock_irqsave(q->queue_lock, flags);
1667
1668 /*
1669 * Submitting request must be dequeued before calling this function
1670 * because it will be linked to another request_queue
1671 */
1672 BUG_ON(blk_queued_rq(rq));
1673
1674 drive_stat_acct(rq, 1);
1675 __elv_add_request(q, rq, ELEVATOR_INSERT_BACK, 0);
1676
1677 spin_unlock_irqrestore(q->queue_lock, flags);
1678
1679 return 0;
1680}
1681EXPORT_SYMBOL_GPL(blk_insert_cloned_request);
1682
Tejun Heo80a761f2009-07-03 17:48:17 +09001683/**
1684 * blk_rq_err_bytes - determine number of bytes till the next failure boundary
1685 * @rq: request to examine
1686 *
1687 * Description:
1688 * A request could be merge of IOs which require different failure
1689 * handling. This function determines the number of bytes which
1690 * can be failed from the beginning of the request without
1691 * crossing into area which need to be retried further.
1692 *
1693 * Return:
1694 * The number of bytes to fail.
1695 *
1696 * Context:
1697 * queue_lock must be held.
1698 */
1699unsigned int blk_rq_err_bytes(const struct request *rq)
1700{
1701 unsigned int ff = rq->cmd_flags & REQ_FAILFAST_MASK;
1702 unsigned int bytes = 0;
1703 struct bio *bio;
1704
1705 if (!(rq->cmd_flags & REQ_MIXED_MERGE))
1706 return blk_rq_bytes(rq);
1707
1708 /*
1709 * Currently the only 'mixing' which can happen is between
1710 * different fastfail types. We can safely fail portions
1711 * which have all the failfast bits that the first one has -
1712 * the ones which are at least as eager to fail as the first
1713 * one.
1714 */
1715 for (bio = rq->bio; bio; bio = bio->bi_next) {
1716 if ((bio->bi_rw & ff) != ff)
1717 break;
1718 bytes += bio->bi_size;
1719 }
1720
1721 /* this could lead to infinite loop */
1722 BUG_ON(blk_rq_bytes(rq) && !bytes);
1723 return bytes;
1724}
1725EXPORT_SYMBOL_GPL(blk_rq_err_bytes);
1726
Jens Axboebc58ba92009-01-23 10:54:44 +01001727static void blk_account_io_completion(struct request *req, unsigned int bytes)
1728{
Jens Axboec2553b52009-04-24 08:10:11 +02001729 if (blk_do_io_stat(req)) {
Jens Axboebc58ba92009-01-23 10:54:44 +01001730 const int rw = rq_data_dir(req);
1731 struct hd_struct *part;
1732 int cpu;
1733
1734 cpu = part_stat_lock();
Tejun Heo83096eb2009-05-07 22:24:39 +09001735 part = disk_map_sector_rcu(req->rq_disk, blk_rq_pos(req));
Jens Axboebc58ba92009-01-23 10:54:44 +01001736 part_stat_add(cpu, part, sectors[rw], bytes >> 9);
1737 part_stat_unlock();
1738 }
1739}
1740
1741static void blk_account_io_done(struct request *req)
1742{
Jens Axboebc58ba92009-01-23 10:54:44 +01001743 /*
1744 * Account IO completion. bar_rq isn't accounted as a normal
1745 * IO on queueing nor completion. Accounting the containing
1746 * request is enough.
1747 */
Jens Axboec2553b52009-04-24 08:10:11 +02001748 if (blk_do_io_stat(req) && req != &req->q->bar_rq) {
Jens Axboebc58ba92009-01-23 10:54:44 +01001749 unsigned long duration = jiffies - req->start_time;
1750 const int rw = rq_data_dir(req);
1751 struct hd_struct *part;
1752 int cpu;
1753
1754 cpu = part_stat_lock();
Tejun Heo83096eb2009-05-07 22:24:39 +09001755 part = disk_map_sector_rcu(req->rq_disk, blk_rq_pos(req));
Jens Axboebc58ba92009-01-23 10:54:44 +01001756
1757 part_stat_inc(cpu, part, ios[rw]);
1758 part_stat_add(cpu, part, ticks[rw], duration);
1759 part_round_stats(cpu, part);
Nikanth Karthikesan316d3152009-10-06 20:16:55 +02001760 part_dec_in_flight(part, rw);
Jens Axboebc58ba92009-01-23 10:54:44 +01001761
1762 part_stat_unlock();
1763 }
1764}
1765
Tejun Heo53a08802008-12-03 12:41:26 +01001766/**
Tejun Heo9934c8c2009-05-08 11:54:16 +09001767 * blk_peek_request - peek at the top of a request queue
1768 * @q: request queue to peek at
Kiyoshi Ueda3bcddea2007-12-11 17:52:28 -05001769 *
1770 * Description:
Tejun Heo9934c8c2009-05-08 11:54:16 +09001771 * Return the request at the top of @q. The returned request
1772 * should be started using blk_start_request() before LLD starts
1773 * processing it.
Kiyoshi Ueda3bcddea2007-12-11 17:52:28 -05001774 *
1775 * Return:
Tejun Heo9934c8c2009-05-08 11:54:16 +09001776 * Pointer to the request at the top of @q if available. Null
1777 * otherwise.
1778 *
1779 * Context:
1780 * queue_lock must be held.
1781 */
1782struct request *blk_peek_request(struct request_queue *q)
Tejun Heo158dbda2009-04-23 11:05:18 +09001783{
1784 struct request *rq;
1785 int ret;
1786
1787 while ((rq = __elv_next_request(q)) != NULL) {
1788 if (!(rq->cmd_flags & REQ_STARTED)) {
1789 /*
1790 * This is the first time the device driver
1791 * sees this request (possibly after
1792 * requeueing). Notify IO scheduler.
1793 */
1794 if (blk_sorted_rq(rq))
1795 elv_activate_rq(q, rq);
1796
1797 /*
1798 * just mark as started even if we don't start
1799 * it, a request that has been delayed should
1800 * not be passed by new incoming requests
1801 */
1802 rq->cmd_flags |= REQ_STARTED;
1803 trace_block_rq_issue(q, rq);
1804 }
1805
1806 if (!q->boundary_rq || q->boundary_rq == rq) {
1807 q->end_sector = rq_end_sector(rq);
1808 q->boundary_rq = NULL;
1809 }
1810
1811 if (rq->cmd_flags & REQ_DONTPREP)
1812 break;
1813
Tejun Heo2e46e8b2009-05-07 22:24:41 +09001814 if (q->dma_drain_size && blk_rq_bytes(rq)) {
Tejun Heo158dbda2009-04-23 11:05:18 +09001815 /*
1816 * make sure space for the drain appears we
1817 * know we can do this because max_hw_segments
1818 * has been adjusted to be one fewer than the
1819 * device can handle
1820 */
1821 rq->nr_phys_segments++;
1822 }
1823
1824 if (!q->prep_rq_fn)
1825 break;
1826
1827 ret = q->prep_rq_fn(q, rq);
1828 if (ret == BLKPREP_OK) {
1829 break;
1830 } else if (ret == BLKPREP_DEFER) {
1831 /*
1832 * the request may have been (partially) prepped.
1833 * we need to keep this request in the front to
1834 * avoid resource deadlock. REQ_STARTED will
1835 * prevent other fs requests from passing this one.
1836 */
Tejun Heo2e46e8b2009-05-07 22:24:41 +09001837 if (q->dma_drain_size && blk_rq_bytes(rq) &&
Tejun Heo158dbda2009-04-23 11:05:18 +09001838 !(rq->cmd_flags & REQ_DONTPREP)) {
1839 /*
1840 * remove the space for the drain we added
1841 * so that we don't add it again
1842 */
1843 --rq->nr_phys_segments;
1844 }
1845
1846 rq = NULL;
1847 break;
1848 } else if (ret == BLKPREP_KILL) {
1849 rq->cmd_flags |= REQ_QUIET;
James Bottomleyc143dc92009-05-30 06:43:49 +02001850 /*
1851 * Mark this request as started so we don't trigger
1852 * any debug logic in the end I/O path.
1853 */
1854 blk_start_request(rq);
Tejun Heo40cbbb72009-04-23 11:05:19 +09001855 __blk_end_request_all(rq, -EIO);
Tejun Heo158dbda2009-04-23 11:05:18 +09001856 } else {
1857 printk(KERN_ERR "%s: bad return=%d\n", __func__, ret);
1858 break;
1859 }
1860 }
1861
1862 return rq;
1863}
Tejun Heo9934c8c2009-05-08 11:54:16 +09001864EXPORT_SYMBOL(blk_peek_request);
Tejun Heo158dbda2009-04-23 11:05:18 +09001865
Tejun Heo9934c8c2009-05-08 11:54:16 +09001866void blk_dequeue_request(struct request *rq)
Tejun Heo158dbda2009-04-23 11:05:18 +09001867{
Tejun Heo9934c8c2009-05-08 11:54:16 +09001868 struct request_queue *q = rq->q;
1869
Tejun Heo158dbda2009-04-23 11:05:18 +09001870 BUG_ON(list_empty(&rq->queuelist));
1871 BUG_ON(ELV_ON_HASH(rq));
1872
1873 list_del_init(&rq->queuelist);
1874
1875 /*
1876 * the time frame between a request being removed from the lists
1877 * and to it is freed is accounted as io that is in progress at
1878 * the driver side.
1879 */
Divyesh Shah91952912010-04-01 15:01:41 -07001880 if (blk_account_rq(rq)) {
Jens Axboe0a7ae2f2009-05-20 08:54:31 +02001881 q->in_flight[rq_is_sync(rq)]++;
Divyesh Shah91952912010-04-01 15:01:41 -07001882 set_io_start_time_ns(rq);
1883 }
Tejun Heo158dbda2009-04-23 11:05:18 +09001884}
1885
Tejun Heo5efccd12009-04-23 11:05:18 +09001886/**
Tejun Heo9934c8c2009-05-08 11:54:16 +09001887 * blk_start_request - start request processing on the driver
1888 * @req: request to dequeue
1889 *
1890 * Description:
1891 * Dequeue @req and start timeout timer on it. This hands off the
1892 * request to the driver.
1893 *
1894 * Block internal functions which don't want to start timer should
1895 * call blk_dequeue_request().
1896 *
1897 * Context:
1898 * queue_lock must be held.
1899 */
1900void blk_start_request(struct request *req)
1901{
1902 blk_dequeue_request(req);
1903
1904 /*
Tejun Heo5f49f632009-05-19 18:33:05 +09001905 * We are now handing the request to the hardware, initialize
1906 * resid_len to full count and add the timeout handler.
Tejun Heo9934c8c2009-05-08 11:54:16 +09001907 */
Tejun Heo5f49f632009-05-19 18:33:05 +09001908 req->resid_len = blk_rq_bytes(req);
FUJITA Tomonoridbb66c42009-06-09 05:47:10 +02001909 if (unlikely(blk_bidi_rq(req)))
1910 req->next_rq->resid_len = blk_rq_bytes(req->next_rq);
1911
Tejun Heo9934c8c2009-05-08 11:54:16 +09001912 blk_add_timer(req);
1913}
1914EXPORT_SYMBOL(blk_start_request);
1915
1916/**
1917 * blk_fetch_request - fetch a request from a request queue
1918 * @q: request queue to fetch a request from
1919 *
1920 * Description:
1921 * Return the request at the top of @q. The request is started on
1922 * return and LLD can start processing it immediately.
1923 *
1924 * Return:
1925 * Pointer to the request at the top of @q if available. Null
1926 * otherwise.
1927 *
1928 * Context:
1929 * queue_lock must be held.
1930 */
1931struct request *blk_fetch_request(struct request_queue *q)
1932{
1933 struct request *rq;
1934
1935 rq = blk_peek_request(q);
1936 if (rq)
1937 blk_start_request(rq);
1938 return rq;
1939}
1940EXPORT_SYMBOL(blk_fetch_request);
1941
1942/**
Tejun Heo2e60e022009-04-23 11:05:18 +09001943 * blk_update_request - Special helper function for request stacking drivers
Randy Dunlap8ebf9752009-06-11 20:00:41 -07001944 * @req: the request being processed
Kiyoshi Ueda3bcddea2007-12-11 17:52:28 -05001945 * @error: %0 for success, < %0 for error
Randy Dunlap8ebf9752009-06-11 20:00:41 -07001946 * @nr_bytes: number of bytes to complete @req
Kiyoshi Ueda3bcddea2007-12-11 17:52:28 -05001947 *
1948 * Description:
Randy Dunlap8ebf9752009-06-11 20:00:41 -07001949 * Ends I/O on a number of bytes attached to @req, but doesn't complete
1950 * the request structure even if @req doesn't have leftover.
1951 * If @req has leftover, sets it up for the next range of segments.
Tejun Heo2e60e022009-04-23 11:05:18 +09001952 *
1953 * This special helper function is only for request stacking drivers
1954 * (e.g. request-based dm) so that they can handle partial completion.
1955 * Actual device drivers should use blk_end_request instead.
1956 *
1957 * Passing the result of blk_rq_bytes() as @nr_bytes guarantees
1958 * %false return from this function.
Kiyoshi Ueda3bcddea2007-12-11 17:52:28 -05001959 *
1960 * Return:
Tejun Heo2e60e022009-04-23 11:05:18 +09001961 * %false - this request doesn't have any more data
1962 * %true - this request has more data
Kiyoshi Ueda3bcddea2007-12-11 17:52:28 -05001963 **/
Tejun Heo2e60e022009-04-23 11:05:18 +09001964bool blk_update_request(struct request *req, int error, unsigned int nr_bytes)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001965{
Kiyoshi Ueda5450d3e2007-12-11 17:53:03 -05001966 int total_bytes, bio_nbytes, next_idx = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001967 struct bio *bio;
1968
Tejun Heo2e60e022009-04-23 11:05:18 +09001969 if (!req->bio)
1970 return false;
1971
Arnaldo Carvalho de Melo5f3ea372008-10-30 08:34:33 +01001972 trace_block_rq_complete(req->q, req);
Jens Axboe2056a782006-03-23 20:00:26 +01001973
Linus Torvalds1da177e2005-04-16 15:20:36 -07001974 /*
Tejun Heo6f414692009-04-19 07:00:41 +09001975 * For fs requests, rq is just carrier of independent bio's
1976 * and each partial completion should be handled separately.
1977 * Reset per-request error on each partial completion.
1978 *
1979 * TODO: tj: This is too subtle. It would be better to let
1980 * low level drivers do what they see fit.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001981 */
Tejun Heo6f414692009-04-19 07:00:41 +09001982 if (blk_fs_request(req))
Linus Torvalds1da177e2005-04-16 15:20:36 -07001983 req->errors = 0;
1984
Jens Axboe6728cb02008-01-31 13:03:55 +01001985 if (error && (blk_fs_request(req) && !(req->cmd_flags & REQ_QUIET))) {
1986 printk(KERN_ERR "end_request: I/O error, dev %s, sector %llu\n",
Linus Torvalds1da177e2005-04-16 15:20:36 -07001987 req->rq_disk ? req->rq_disk->disk_name : "?",
Tejun Heo83096eb2009-05-07 22:24:39 +09001988 (unsigned long long)blk_rq_pos(req));
Linus Torvalds1da177e2005-04-16 15:20:36 -07001989 }
1990
Jens Axboebc58ba92009-01-23 10:54:44 +01001991 blk_account_io_completion(req, nr_bytes);
Jens Axboed72d9042005-11-01 08:35:42 +01001992
Linus Torvalds1da177e2005-04-16 15:20:36 -07001993 total_bytes = bio_nbytes = 0;
1994 while ((bio = req->bio) != NULL) {
1995 int nbytes;
1996
1997 if (nr_bytes >= bio->bi_size) {
1998 req->bio = bio->bi_next;
1999 nbytes = bio->bi_size;
NeilBrown5bb23a62007-09-27 12:46:13 +02002000 req_bio_endio(req, bio, nbytes, error);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002001 next_idx = 0;
2002 bio_nbytes = 0;
2003 } else {
2004 int idx = bio->bi_idx + next_idx;
2005
Kazuhisa Ichikawaaf498d72009-05-12 13:27:45 +02002006 if (unlikely(idx >= bio->bi_vcnt)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07002007 blk_dump_rq_flags(req, "__end_that");
Jens Axboe6728cb02008-01-31 13:03:55 +01002008 printk(KERN_ERR "%s: bio idx %d >= vcnt %d\n",
Kazuhisa Ichikawaaf498d72009-05-12 13:27:45 +02002009 __func__, idx, bio->bi_vcnt);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002010 break;
2011 }
2012
2013 nbytes = bio_iovec_idx(bio, idx)->bv_len;
2014 BIO_BUG_ON(nbytes > bio->bi_size);
2015
2016 /*
2017 * not a complete bvec done
2018 */
2019 if (unlikely(nbytes > nr_bytes)) {
2020 bio_nbytes += nr_bytes;
2021 total_bytes += nr_bytes;
2022 break;
2023 }
2024
2025 /*
2026 * advance to the next vector
2027 */
2028 next_idx++;
2029 bio_nbytes += nbytes;
2030 }
2031
2032 total_bytes += nbytes;
2033 nr_bytes -= nbytes;
2034
Jens Axboe6728cb02008-01-31 13:03:55 +01002035 bio = req->bio;
2036 if (bio) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07002037 /*
2038 * end more in this run, or just return 'not-done'
2039 */
2040 if (unlikely(nr_bytes <= 0))
2041 break;
2042 }
2043 }
2044
2045 /*
2046 * completely done
2047 */
Tejun Heo2e60e022009-04-23 11:05:18 +09002048 if (!req->bio) {
2049 /*
2050 * Reset counters so that the request stacking driver
2051 * can find how many bytes remain in the request
2052 * later.
2053 */
Tejun Heoa2dec7b2009-05-07 22:24:44 +09002054 req->__data_len = 0;
Tejun Heo2e60e022009-04-23 11:05:18 +09002055 return false;
2056 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07002057
2058 /*
2059 * if the request wasn't completed, update state
2060 */
2061 if (bio_nbytes) {
NeilBrown5bb23a62007-09-27 12:46:13 +02002062 req_bio_endio(req, bio, bio_nbytes, error);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002063 bio->bi_idx += next_idx;
2064 bio_iovec(bio)->bv_offset += nr_bytes;
2065 bio_iovec(bio)->bv_len -= nr_bytes;
2066 }
2067
Tejun Heoa2dec7b2009-05-07 22:24:44 +09002068 req->__data_len -= total_bytes;
Tejun Heo2e46e8b2009-05-07 22:24:41 +09002069 req->buffer = bio_data(req->bio);
2070
2071 /* update sector only for requests with clear definition of sector */
2072 if (blk_fs_request(req) || blk_discard_rq(req))
Tejun Heoa2dec7b2009-05-07 22:24:44 +09002073 req->__sector += total_bytes >> 9;
Tejun Heo2e46e8b2009-05-07 22:24:41 +09002074
Tejun Heo80a761f2009-07-03 17:48:17 +09002075 /* mixed attributes always follow the first bio */
2076 if (req->cmd_flags & REQ_MIXED_MERGE) {
2077 req->cmd_flags &= ~REQ_FAILFAST_MASK;
2078 req->cmd_flags |= req->bio->bi_rw & REQ_FAILFAST_MASK;
2079 }
2080
Tejun Heo2e46e8b2009-05-07 22:24:41 +09002081 /*
2082 * If total number of sectors is less than the first segment
2083 * size, something has gone terribly wrong.
2084 */
2085 if (blk_rq_bytes(req) < blk_rq_cur_bytes(req)) {
2086 printk(KERN_ERR "blk: request botched\n");
Tejun Heoa2dec7b2009-05-07 22:24:44 +09002087 req->__data_len = blk_rq_cur_bytes(req);
Tejun Heo2e46e8b2009-05-07 22:24:41 +09002088 }
2089
2090 /* recalculate the number of segments */
Linus Torvalds1da177e2005-04-16 15:20:36 -07002091 blk_recalc_rq_segments(req);
Tejun Heo2e46e8b2009-05-07 22:24:41 +09002092
Tejun Heo2e60e022009-04-23 11:05:18 +09002093 return true;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002094}
Tejun Heo2e60e022009-04-23 11:05:18 +09002095EXPORT_SYMBOL_GPL(blk_update_request);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002096
Tejun Heo2e60e022009-04-23 11:05:18 +09002097static bool blk_update_bidi_request(struct request *rq, int error,
2098 unsigned int nr_bytes,
2099 unsigned int bidi_bytes)
Tejun Heo5efccd12009-04-23 11:05:18 +09002100{
Tejun Heo2e60e022009-04-23 11:05:18 +09002101 if (blk_update_request(rq, error, nr_bytes))
2102 return true;
Tejun Heo5efccd12009-04-23 11:05:18 +09002103
Tejun Heo2e60e022009-04-23 11:05:18 +09002104 /* Bidi request must be completed as a whole */
2105 if (unlikely(blk_bidi_rq(rq)) &&
2106 blk_update_request(rq->next_rq, error, bidi_bytes))
2107 return true;
Tejun Heo5efccd12009-04-23 11:05:18 +09002108
Tejun Heo2e60e022009-04-23 11:05:18 +09002109 add_disk_randomness(rq->rq_disk);
2110
2111 return false;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002112}
2113
Linus Torvalds1da177e2005-04-16 15:20:36 -07002114/*
2115 * queue lock must be held
2116 */
Tejun Heo2e60e022009-04-23 11:05:18 +09002117static void blk_finish_request(struct request *req, int error)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002118{
Kiyoshi Uedab8286232007-12-11 17:53:24 -05002119 if (blk_rq_tagged(req))
2120 blk_queue_end_tag(req->q, req);
2121
James Bottomleyba396a62009-05-27 14:17:08 +02002122 BUG_ON(blk_queued_rq(req));
Linus Torvalds1da177e2005-04-16 15:20:36 -07002123
2124 if (unlikely(laptop_mode) && blk_fs_request(req))
Matthew Garrett31373d02010-04-06 14:25:14 +02002125 laptop_io_completion(&req->q->backing_dev_info);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002126
Mike Andersone78042e2008-10-30 02:16:20 -07002127 blk_delete_timer(req);
2128
Jens Axboebc58ba92009-01-23 10:54:44 +01002129 blk_account_io_done(req);
Kiyoshi Uedab8286232007-12-11 17:53:24 -05002130
Linus Torvalds1da177e2005-04-16 15:20:36 -07002131 if (req->end_io)
Tejun Heo8ffdc652006-01-06 09:49:03 +01002132 req->end_io(req, error);
Kiyoshi Uedab8286232007-12-11 17:53:24 -05002133 else {
2134 if (blk_bidi_rq(req))
2135 __blk_put_request(req->next_rq->q, req->next_rq);
2136
Linus Torvalds1da177e2005-04-16 15:20:36 -07002137 __blk_put_request(req->q, req);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002138 }
2139}
2140
Kiyoshi Ueda3b113132007-12-11 17:41:17 -05002141/**
Tejun Heo2e60e022009-04-23 11:05:18 +09002142 * blk_end_bidi_request - Complete a bidi request
2143 * @rq: the request to complete
Randy Dunlap710027a2008-08-19 20:13:11 +02002144 * @error: %0 for success, < %0 for error
Kiyoshi Uedae3a04fe2007-12-11 17:51:46 -05002145 * @nr_bytes: number of bytes to complete @rq
2146 * @bidi_bytes: number of bytes to complete @rq->next_rq
Kiyoshi Ueda336cdb42007-12-11 17:40:30 -05002147 *
2148 * Description:
2149 * Ends I/O on a number of bytes attached to @rq and @rq->next_rq.
Tejun Heo2e60e022009-04-23 11:05:18 +09002150 * Drivers that supports bidi can safely call this member for any
2151 * type of request, bidi or uni. In the later case @bidi_bytes is
2152 * just ignored.
Kiyoshi Ueda336cdb42007-12-11 17:40:30 -05002153 *
2154 * Return:
Tejun Heo2e60e022009-04-23 11:05:18 +09002155 * %false - we are done with this request
2156 * %true - still buffers pending for this request
Kiyoshi Ueda336cdb42007-12-11 17:40:30 -05002157 **/
FUJITA Tomonorib1f74492009-05-11 17:56:09 +09002158static bool blk_end_bidi_request(struct request *rq, int error,
2159 unsigned int nr_bytes, unsigned int bidi_bytes)
Kiyoshi Ueda336cdb42007-12-11 17:40:30 -05002160{
2161 struct request_queue *q = rq->q;
Tejun Heo2e60e022009-04-23 11:05:18 +09002162 unsigned long flags;
Kiyoshi Ueda336cdb42007-12-11 17:40:30 -05002163
Tejun Heo2e60e022009-04-23 11:05:18 +09002164 if (blk_update_bidi_request(rq, error, nr_bytes, bidi_bytes))
2165 return true;
Kiyoshi Uedae19a3ab2007-12-11 17:51:02 -05002166
Kiyoshi Ueda336cdb42007-12-11 17:40:30 -05002167 spin_lock_irqsave(q->queue_lock, flags);
Tejun Heo2e60e022009-04-23 11:05:18 +09002168 blk_finish_request(rq, error);
Kiyoshi Ueda336cdb42007-12-11 17:40:30 -05002169 spin_unlock_irqrestore(q->queue_lock, flags);
2170
Tejun Heo2e60e022009-04-23 11:05:18 +09002171 return false;
Kiyoshi Uedae3a04fe2007-12-11 17:51:46 -05002172}
Kiyoshi Uedae3a04fe2007-12-11 17:51:46 -05002173
2174/**
Tejun Heo2e60e022009-04-23 11:05:18 +09002175 * __blk_end_bidi_request - Complete a bidi request with queue lock held
2176 * @rq: the request to complete
2177 * @error: %0 for success, < %0 for error
2178 * @nr_bytes: number of bytes to complete @rq
2179 * @bidi_bytes: number of bytes to complete @rq->next_rq
Tejun Heo5efccd12009-04-23 11:05:18 +09002180 *
2181 * Description:
Tejun Heo2e60e022009-04-23 11:05:18 +09002182 * Identical to blk_end_bidi_request() except that queue lock is
2183 * assumed to be locked on entry and remains so on return.
Tejun Heo5efccd12009-04-23 11:05:18 +09002184 *
Tejun Heo2e60e022009-04-23 11:05:18 +09002185 * Return:
2186 * %false - we are done with this request
2187 * %true - still buffers pending for this request
Tejun Heo5efccd12009-04-23 11:05:18 +09002188 **/
FUJITA Tomonorib1f74492009-05-11 17:56:09 +09002189static bool __blk_end_bidi_request(struct request *rq, int error,
2190 unsigned int nr_bytes, unsigned int bidi_bytes)
Tejun Heo5efccd12009-04-23 11:05:18 +09002191{
Tejun Heo2e60e022009-04-23 11:05:18 +09002192 if (blk_update_bidi_request(rq, error, nr_bytes, bidi_bytes))
2193 return true;
Tejun Heo5efccd12009-04-23 11:05:18 +09002194
Tejun Heo2e60e022009-04-23 11:05:18 +09002195 blk_finish_request(rq, error);
Tejun Heo5efccd12009-04-23 11:05:18 +09002196
Tejun Heo2e60e022009-04-23 11:05:18 +09002197 return false;
Kiyoshi Ueda336cdb42007-12-11 17:40:30 -05002198}
2199
2200/**
2201 * blk_end_request - Helper function for drivers to complete the request.
2202 * @rq: the request being processed
2203 * @error: %0 for success, < %0 for error
2204 * @nr_bytes: number of bytes to complete
2205 *
2206 * Description:
2207 * Ends I/O on a number of bytes attached to @rq.
2208 * If @rq has leftover, sets it up for the next range of segments.
2209 *
2210 * Return:
FUJITA Tomonorib1f74492009-05-11 17:56:09 +09002211 * %false - we are done with this request
2212 * %true - still buffers pending for this request
Kiyoshi Ueda336cdb42007-12-11 17:40:30 -05002213 **/
FUJITA Tomonorib1f74492009-05-11 17:56:09 +09002214bool blk_end_request(struct request *rq, int error, unsigned int nr_bytes)
Kiyoshi Ueda336cdb42007-12-11 17:40:30 -05002215{
FUJITA Tomonorib1f74492009-05-11 17:56:09 +09002216 return blk_end_bidi_request(rq, error, nr_bytes, 0);
Kiyoshi Ueda336cdb42007-12-11 17:40:30 -05002217}
Jens Axboe56ad1742009-07-28 22:11:24 +02002218EXPORT_SYMBOL(blk_end_request);
Kiyoshi Ueda336cdb42007-12-11 17:40:30 -05002219
2220/**
FUJITA Tomonorib1f74492009-05-11 17:56:09 +09002221 * blk_end_request_all - Helper function for drives to finish the request.
2222 * @rq: the request to finish
Randy Dunlap8ebf9752009-06-11 20:00:41 -07002223 * @error: %0 for success, < %0 for error
FUJITA Tomonorib1f74492009-05-11 17:56:09 +09002224 *
2225 * Description:
2226 * Completely finish @rq.
2227 */
2228void blk_end_request_all(struct request *rq, int error)
2229{
2230 bool pending;
2231 unsigned int bidi_bytes = 0;
2232
2233 if (unlikely(blk_bidi_rq(rq)))
2234 bidi_bytes = blk_rq_bytes(rq->next_rq);
2235
2236 pending = blk_end_bidi_request(rq, error, blk_rq_bytes(rq), bidi_bytes);
2237 BUG_ON(pending);
2238}
Jens Axboe56ad1742009-07-28 22:11:24 +02002239EXPORT_SYMBOL(blk_end_request_all);
FUJITA Tomonorib1f74492009-05-11 17:56:09 +09002240
2241/**
2242 * blk_end_request_cur - Helper function to finish the current request chunk.
2243 * @rq: the request to finish the current chunk for
Randy Dunlap8ebf9752009-06-11 20:00:41 -07002244 * @error: %0 for success, < %0 for error
FUJITA Tomonorib1f74492009-05-11 17:56:09 +09002245 *
2246 * Description:
2247 * Complete the current consecutively mapped chunk from @rq.
2248 *
2249 * Return:
2250 * %false - we are done with this request
2251 * %true - still buffers pending for this request
2252 */
2253bool blk_end_request_cur(struct request *rq, int error)
2254{
2255 return blk_end_request(rq, error, blk_rq_cur_bytes(rq));
2256}
Jens Axboe56ad1742009-07-28 22:11:24 +02002257EXPORT_SYMBOL(blk_end_request_cur);
FUJITA Tomonorib1f74492009-05-11 17:56:09 +09002258
2259/**
Tejun Heo80a761f2009-07-03 17:48:17 +09002260 * blk_end_request_err - Finish a request till the next failure boundary.
2261 * @rq: the request to finish till the next failure boundary for
2262 * @error: must be negative errno
2263 *
2264 * Description:
2265 * Complete @rq till the next failure boundary.
2266 *
2267 * Return:
2268 * %false - we are done with this request
2269 * %true - still buffers pending for this request
2270 */
2271bool blk_end_request_err(struct request *rq, int error)
2272{
2273 WARN_ON(error >= 0);
2274 return blk_end_request(rq, error, blk_rq_err_bytes(rq));
2275}
2276EXPORT_SYMBOL_GPL(blk_end_request_err);
2277
2278/**
Kiyoshi Uedae19a3ab2007-12-11 17:51:02 -05002279 * __blk_end_request - Helper function for drivers to complete the request.
Kiyoshi Ueda336cdb42007-12-11 17:40:30 -05002280 * @rq: the request being processed
Kiyoshi Uedae19a3ab2007-12-11 17:51:02 -05002281 * @error: %0 for success, < %0 for error
2282 * @nr_bytes: number of bytes to complete
Kiyoshi Ueda336cdb42007-12-11 17:40:30 -05002283 *
2284 * Description:
2285 * Must be called with queue lock held unlike blk_end_request().
2286 *
2287 * Return:
FUJITA Tomonorib1f74492009-05-11 17:56:09 +09002288 * %false - we are done with this request
2289 * %true - still buffers pending for this request
Kiyoshi Ueda336cdb42007-12-11 17:40:30 -05002290 **/
FUJITA Tomonorib1f74492009-05-11 17:56:09 +09002291bool __blk_end_request(struct request *rq, int error, unsigned int nr_bytes)
Kiyoshi Ueda336cdb42007-12-11 17:40:30 -05002292{
FUJITA Tomonorib1f74492009-05-11 17:56:09 +09002293 return __blk_end_bidi_request(rq, error, nr_bytes, 0);
Kiyoshi Ueda336cdb42007-12-11 17:40:30 -05002294}
Jens Axboe56ad1742009-07-28 22:11:24 +02002295EXPORT_SYMBOL(__blk_end_request);
Kiyoshi Ueda336cdb42007-12-11 17:40:30 -05002296
2297/**
FUJITA Tomonorib1f74492009-05-11 17:56:09 +09002298 * __blk_end_request_all - Helper function for drives to finish the request.
2299 * @rq: the request to finish
Randy Dunlap8ebf9752009-06-11 20:00:41 -07002300 * @error: %0 for success, < %0 for error
Kiyoshi Ueda336cdb42007-12-11 17:40:30 -05002301 *
2302 * Description:
FUJITA Tomonorib1f74492009-05-11 17:56:09 +09002303 * Completely finish @rq. Must be called with queue lock held.
Kiyoshi Ueda32fab442008-09-18 10:45:09 -04002304 */
FUJITA Tomonorib1f74492009-05-11 17:56:09 +09002305void __blk_end_request_all(struct request *rq, int error)
Kiyoshi Ueda32fab442008-09-18 10:45:09 -04002306{
FUJITA Tomonorib1f74492009-05-11 17:56:09 +09002307 bool pending;
2308 unsigned int bidi_bytes = 0;
2309
2310 if (unlikely(blk_bidi_rq(rq)))
2311 bidi_bytes = blk_rq_bytes(rq->next_rq);
2312
2313 pending = __blk_end_bidi_request(rq, error, blk_rq_bytes(rq), bidi_bytes);
2314 BUG_ON(pending);
Kiyoshi Ueda32fab442008-09-18 10:45:09 -04002315}
Jens Axboe56ad1742009-07-28 22:11:24 +02002316EXPORT_SYMBOL(__blk_end_request_all);
Kiyoshi Ueda32fab442008-09-18 10:45:09 -04002317
2318/**
FUJITA Tomonorib1f74492009-05-11 17:56:09 +09002319 * __blk_end_request_cur - Helper function to finish the current request chunk.
2320 * @rq: the request to finish the current chunk for
Randy Dunlap8ebf9752009-06-11 20:00:41 -07002321 * @error: %0 for success, < %0 for error
Kiyoshi Uedae19a3ab2007-12-11 17:51:02 -05002322 *
2323 * Description:
FUJITA Tomonorib1f74492009-05-11 17:56:09 +09002324 * Complete the current consecutively mapped chunk from @rq. Must
2325 * be called with queue lock held.
Kiyoshi Uedae19a3ab2007-12-11 17:51:02 -05002326 *
2327 * Return:
FUJITA Tomonorib1f74492009-05-11 17:56:09 +09002328 * %false - we are done with this request
2329 * %true - still buffers pending for this request
2330 */
2331bool __blk_end_request_cur(struct request *rq, int error)
Kiyoshi Uedae19a3ab2007-12-11 17:51:02 -05002332{
FUJITA Tomonorib1f74492009-05-11 17:56:09 +09002333 return __blk_end_request(rq, error, blk_rq_cur_bytes(rq));
Kiyoshi Uedae19a3ab2007-12-11 17:51:02 -05002334}
Jens Axboe56ad1742009-07-28 22:11:24 +02002335EXPORT_SYMBOL(__blk_end_request_cur);
Kiyoshi Uedae19a3ab2007-12-11 17:51:02 -05002336
Tejun Heo80a761f2009-07-03 17:48:17 +09002337/**
2338 * __blk_end_request_err - Finish a request till the next failure boundary.
2339 * @rq: the request to finish till the next failure boundary for
2340 * @error: must be negative errno
2341 *
2342 * Description:
2343 * Complete @rq till the next failure boundary. Must be called
2344 * with queue lock held.
2345 *
2346 * Return:
2347 * %false - we are done with this request
2348 * %true - still buffers pending for this request
2349 */
2350bool __blk_end_request_err(struct request *rq, int error)
2351{
2352 WARN_ON(error >= 0);
2353 return __blk_end_request(rq, error, blk_rq_err_bytes(rq));
2354}
2355EXPORT_SYMBOL_GPL(__blk_end_request_err);
2356
Jens Axboe86db1e22008-01-29 14:53:40 +01002357void blk_rq_bio_prep(struct request_queue *q, struct request *rq,
2358 struct bio *bio)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002359{
Tejun Heoa82afdf2009-07-03 17:48:16 +09002360 /* Bit 0 (R/W) is identical in rq->cmd_flags and bio->bi_rw */
2361 rq->cmd_flags |= bio->bi_rw & REQ_RW;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002362
David Woodhousefb2dce82008-08-05 18:01:53 +01002363 if (bio_has_data(bio)) {
2364 rq->nr_phys_segments = bio_phys_segments(q, bio);
David Woodhousefb2dce82008-08-05 18:01:53 +01002365 rq->buffer = bio_data(bio);
2366 }
Tejun Heoa2dec7b2009-05-07 22:24:44 +09002367 rq->__data_len = bio->bi_size;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002368 rq->bio = rq->biotail = bio;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002369
NeilBrown66846572007-08-16 13:31:28 +02002370 if (bio->bi_bdev)
2371 rq->rq_disk = bio->bi_bdev->bd_disk;
2372}
Linus Torvalds1da177e2005-04-16 15:20:36 -07002373
Ilya Loginov2d4dc892009-11-26 09:16:19 +01002374#if ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE
2375/**
2376 * rq_flush_dcache_pages - Helper function to flush all pages in a request
2377 * @rq: the request to be flushed
2378 *
2379 * Description:
2380 * Flush all pages in @rq.
2381 */
2382void rq_flush_dcache_pages(struct request *rq)
2383{
2384 struct req_iterator iter;
2385 struct bio_vec *bvec;
2386
2387 rq_for_each_segment(bvec, rq, iter)
2388 flush_dcache_page(bvec->bv_page);
2389}
2390EXPORT_SYMBOL_GPL(rq_flush_dcache_pages);
2391#endif
2392
Kiyoshi Uedaef9e3fa2008-10-01 16:12:15 +02002393/**
2394 * blk_lld_busy - Check if underlying low-level drivers of a device are busy
2395 * @q : the queue of the device being checked
2396 *
2397 * Description:
2398 * Check if underlying low-level drivers of a device are busy.
2399 * If the drivers want to export their busy state, they must set own
2400 * exporting function using blk_queue_lld_busy() first.
2401 *
2402 * Basically, this function is used only by request stacking drivers
2403 * to stop dispatching requests to underlying devices when underlying
2404 * devices are busy. This behavior helps more I/O merging on the queue
2405 * of the request stacking driver and prevents I/O throughput regression
2406 * on burst I/O load.
2407 *
2408 * Return:
2409 * 0 - Not busy (The request stacking driver should dispatch request)
2410 * 1 - Busy (The request stacking driver should stop dispatching request)
2411 */
2412int blk_lld_busy(struct request_queue *q)
2413{
2414 if (q->lld_busy_fn)
2415 return q->lld_busy_fn(q);
2416
2417 return 0;
2418}
2419EXPORT_SYMBOL_GPL(blk_lld_busy);
2420
Kiyoshi Uedab0fd2712009-06-11 13:10:16 +02002421/**
2422 * blk_rq_unprep_clone - Helper function to free all bios in a cloned request
2423 * @rq: the clone request to be cleaned up
2424 *
2425 * Description:
2426 * Free all bios in @rq for a cloned request.
2427 */
2428void blk_rq_unprep_clone(struct request *rq)
2429{
2430 struct bio *bio;
2431
2432 while ((bio = rq->bio) != NULL) {
2433 rq->bio = bio->bi_next;
2434
2435 bio_put(bio);
2436 }
2437}
2438EXPORT_SYMBOL_GPL(blk_rq_unprep_clone);
2439
2440/*
2441 * Copy attributes of the original request to the clone request.
2442 * The actual data parts (e.g. ->cmd, ->buffer, ->sense) are not copied.
2443 */
2444static void __blk_rq_prep_clone(struct request *dst, struct request *src)
2445{
2446 dst->cpu = src->cpu;
2447 dst->cmd_flags = (rq_data_dir(src) | REQ_NOMERGE);
2448 dst->cmd_type = src->cmd_type;
2449 dst->__sector = blk_rq_pos(src);
2450 dst->__data_len = blk_rq_bytes(src);
2451 dst->nr_phys_segments = src->nr_phys_segments;
2452 dst->ioprio = src->ioprio;
2453 dst->extra_len = src->extra_len;
2454}
2455
2456/**
2457 * blk_rq_prep_clone - Helper function to setup clone request
2458 * @rq: the request to be setup
2459 * @rq_src: original request to be cloned
2460 * @bs: bio_set that bios for clone are allocated from
2461 * @gfp_mask: memory allocation mask for bio
2462 * @bio_ctr: setup function to be called for each clone bio.
2463 * Returns %0 for success, non %0 for failure.
2464 * @data: private data to be passed to @bio_ctr
2465 *
2466 * Description:
2467 * Clones bios in @rq_src to @rq, and copies attributes of @rq_src to @rq.
2468 * The actual data parts of @rq_src (e.g. ->cmd, ->buffer, ->sense)
2469 * are not copied, and copying such parts is the caller's responsibility.
2470 * Also, pages which the original bios are pointing to are not copied
2471 * and the cloned bios just point same pages.
2472 * So cloned bios must be completed before original bios, which means
2473 * the caller must complete @rq before @rq_src.
2474 */
2475int blk_rq_prep_clone(struct request *rq, struct request *rq_src,
2476 struct bio_set *bs, gfp_t gfp_mask,
2477 int (*bio_ctr)(struct bio *, struct bio *, void *),
2478 void *data)
2479{
2480 struct bio *bio, *bio_src;
2481
2482 if (!bs)
2483 bs = fs_bio_set;
2484
2485 blk_rq_init(NULL, rq);
2486
2487 __rq_for_each_bio(bio_src, rq_src) {
2488 bio = bio_alloc_bioset(gfp_mask, bio_src->bi_max_vecs, bs);
2489 if (!bio)
2490 goto free_and_out;
2491
2492 __bio_clone(bio, bio_src);
2493
2494 if (bio_integrity(bio_src) &&
Martin K. Petersen7878cba2009-06-26 15:37:49 +02002495 bio_integrity_clone(bio, bio_src, gfp_mask, bs))
Kiyoshi Uedab0fd2712009-06-11 13:10:16 +02002496 goto free_and_out;
2497
2498 if (bio_ctr && bio_ctr(bio, bio_src, data))
2499 goto free_and_out;
2500
2501 if (rq->bio) {
2502 rq->biotail->bi_next = bio;
2503 rq->biotail = bio;
2504 } else
2505 rq->bio = rq->biotail = bio;
2506 }
2507
2508 __blk_rq_prep_clone(rq, rq_src);
2509
2510 return 0;
2511
2512free_and_out:
2513 if (bio)
2514 bio_free(bio, bs);
2515 blk_rq_unprep_clone(rq);
2516
2517 return -ENOMEM;
2518}
2519EXPORT_SYMBOL_GPL(blk_rq_prep_clone);
2520
Jens Axboe18887ad2008-07-28 13:08:45 +02002521int kblockd_schedule_work(struct request_queue *q, struct work_struct *work)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002522{
2523 return queue_work(kblockd_workqueue, work);
2524}
Linus Torvalds1da177e2005-04-16 15:20:36 -07002525EXPORT_SYMBOL(kblockd_schedule_work);
2526
Linus Torvalds1da177e2005-04-16 15:20:36 -07002527int __init blk_dev_init(void)
2528{
Nikanth Karthikesan9eb55b02009-04-27 14:53:54 +02002529 BUILD_BUG_ON(__REQ_NR_BITS > 8 *
2530 sizeof(((struct request *)0)->cmd_flags));
2531
Linus Torvalds1da177e2005-04-16 15:20:36 -07002532 kblockd_workqueue = create_workqueue("kblockd");
2533 if (!kblockd_workqueue)
2534 panic("Failed to create kblockd\n");
2535
2536 request_cachep = kmem_cache_create("blkdev_requests",
Paul Mundt20c2df82007-07-20 10:11:58 +09002537 sizeof(struct request), 0, SLAB_PANIC, NULL);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002538
Jens Axboe8324aa92008-01-29 14:51:59 +01002539 blk_requestq_cachep = kmem_cache_create("blkdev_queue",
Jens Axboe165125e2007-07-24 09:28:11 +02002540 sizeof(struct request_queue), 0, SLAB_PANIC, NULL);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002541
Linus Torvalds1da177e2005-04-16 15:20:36 -07002542 return 0;
2543}