blob: e56e7685af9cf352b086580cfe6886c64895d0fa [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
Jens Axboe0fe23472006-09-04 15:41:16 +02002 * Copyright (C) 2001 Jens Axboe <axboe@kernel.dk>
Linus Torvalds1da177e2005-04-16 15:20:36 -07003 *
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License version 2 as
6 * published by the Free Software Foundation.
7 *
8 * This program is distributed in the hope that it will be useful,
9 * but WITHOUT ANY WARRANTY; without even the implied warranty of
10 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 * GNU General Public License for more details.
12 *
13 * You should have received a copy of the GNU General Public Licens
14 * along with this program; if not, write to the Free Software
15 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-
16 *
17 */
18#include <linux/mm.h>
19#include <linux/swap.h>
20#include <linux/bio.h>
21#include <linux/blkdev.h>
22#include <linux/slab.h>
23#include <linux/init.h>
24#include <linux/kernel.h>
25#include <linux/module.h>
26#include <linux/mempool.h>
27#include <linux/workqueue.h>
Jens Axboe2056a782006-03-23 20:00:26 +010028#include <linux/blktrace_api.h>
James Bottomley f1970ba2005-06-20 14:06:52 +020029#include <scsi/sg.h> /* for struct sg_iovec */
Linus Torvalds1da177e2005-04-16 15:20:36 -070030
Christoph Lametere18b8902006-12-06 20:33:20 -080031static struct kmem_cache *bio_slab __read_mostly;
Linus Torvalds1da177e2005-04-16 15:20:36 -070032
Eric Dumazetfa3536c2006-03-26 01:37:24 -080033mempool_t *bio_split_pool __read_mostly;
Linus Torvalds1da177e2005-04-16 15:20:36 -070034
Linus Torvalds1da177e2005-04-16 15:20:36 -070035/*
36 * if you change this list, also change bvec_alloc or things will
37 * break badly! cannot be bigger than what you can fit into an
38 * unsigned short
39 */
40
41#define BV(x) { .nr_vecs = x, .name = "biovec-"__stringify(x) }
Christoph Lameter6c036522005-07-07 17:56:59 -070042static struct biovec_slab bvec_slabs[BIOVEC_NR_POOLS] __read_mostly = {
Linus Torvalds1da177e2005-04-16 15:20:36 -070043 BV(1), BV(4), BV(16), BV(64), BV(128), BV(BIO_MAX_PAGES),
44};
45#undef BV
46
47/*
Linus Torvalds1da177e2005-04-16 15:20:36 -070048 * fs_bio_set is the bio_set containing bio and iovec memory pools used by
49 * IO code that does not need private memory pools.
50 */
Martin K. Petersen51d654e2008-06-17 18:59:56 +020051struct bio_set *fs_bio_set;
Linus Torvalds1da177e2005-04-16 15:20:36 -070052
Martin K. Petersen7ba1ba12008-06-30 20:04:41 +020053unsigned int bvec_nr_vecs(unsigned short idx)
54{
55 return bvec_slabs[idx].nr_vecs;
56}
57
Martin K. Petersen51d654e2008-06-17 18:59:56 +020058struct bio_vec *bvec_alloc_bs(gfp_t gfp_mask, int nr, unsigned long *idx, struct bio_set *bs)
Linus Torvalds1da177e2005-04-16 15:20:36 -070059{
60 struct bio_vec *bvl;
Linus Torvalds1da177e2005-04-16 15:20:36 -070061
62 /*
Jens Axboe0a0d96b2008-09-11 13:17:37 +020063 * If 'bs' is given, lookup the pool and do the mempool alloc.
64 * If not, this is a bio_kmalloc() allocation and just do a
65 * kzalloc() for the exact number of vecs right away.
Linus Torvalds1da177e2005-04-16 15:20:36 -070066 */
Jens Axboe0a0d96b2008-09-11 13:17:37 +020067 if (bs) {
68 /*
69 * see comment near bvec_array define!
70 */
71 switch (nr) {
72 case 1:
73 *idx = 0;
74 break;
75 case 2 ... 4:
76 *idx = 1;
77 break;
78 case 5 ... 16:
79 *idx = 2;
80 break;
81 case 17 ... 64:
82 *idx = 3;
83 break;
84 case 65 ... 128:
85 *idx = 4;
86 break;
87 case 129 ... BIO_MAX_PAGES:
88 *idx = 5;
89 break;
Linus Torvalds1da177e2005-04-16 15:20:36 -070090 default:
91 return NULL;
Jens Axboe0a0d96b2008-09-11 13:17:37 +020092 }
Linus Torvalds1da177e2005-04-16 15:20:36 -070093
Jens Axboe0a0d96b2008-09-11 13:17:37 +020094 /*
95 * idx now points to the pool we want to allocate from
96 */
97 bvl = mempool_alloc(bs->bvec_pools[*idx], gfp_mask);
98 if (bvl)
99 memset(bvl, 0,
100 bvec_nr_vecs(*idx) * sizeof(struct bio_vec));
101 } else
102 bvl = kzalloc(nr * sizeof(struct bio_vec), gfp_mask);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700103
104 return bvl;
105}
106
Peter Osterlund36763472005-09-06 15:16:42 -0700107void bio_free(struct bio *bio, struct bio_set *bio_set)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700108{
Jens Axboe992c5dd2007-07-18 13:18:08 +0200109 if (bio->bi_io_vec) {
110 const int pool_idx = BIO_POOL_IDX(bio);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700111
Jens Axboe992c5dd2007-07-18 13:18:08 +0200112 BIO_BUG_ON(pool_idx >= BIOVEC_NR_POOLS);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700113
Jens Axboe992c5dd2007-07-18 13:18:08 +0200114 mempool_free(bio->bi_io_vec, bio_set->bvec_pools[pool_idx]);
115 }
116
Martin K. Petersen7ba1ba12008-06-30 20:04:41 +0200117 if (bio_integrity(bio))
118 bio_integrity_free(bio, bio_set);
119
Peter Osterlund36763472005-09-06 15:16:42 -0700120 mempool_free(bio, bio_set->bio_pool);
121}
122
123/*
124 * default destructor for a bio allocated with bio_alloc_bioset()
125 */
126static void bio_fs_destructor(struct bio *bio)
127{
128 bio_free(bio, fs_bio_set);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700129}
130
Jens Axboe0a0d96b2008-09-11 13:17:37 +0200131static void bio_kmalloc_destructor(struct bio *bio)
132{
133 kfree(bio->bi_io_vec);
134 kfree(bio);
135}
136
Arjan van de Ven858119e2006-01-14 13:20:43 -0800137void bio_init(struct bio *bio)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700138{
Jens Axboe2b94de52007-07-18 13:14:03 +0200139 memset(bio, 0, sizeof(*bio));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700140 bio->bi_flags = 1 << BIO_UPTODATE;
Jens Axboec7c22e42008-09-13 20:26:01 +0200141 bio->bi_comp_cpu = -1;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700142 atomic_set(&bio->bi_cnt, 1);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700143}
144
145/**
146 * bio_alloc_bioset - allocate a bio for I/O
147 * @gfp_mask: the GFP_ mask given to the slab allocator
148 * @nr_iovecs: number of iovecs to pre-allocate
Jens Axboe0a0d96b2008-09-11 13:17:37 +0200149 * @bs: the bio_set to allocate from. If %NULL, just use kmalloc
Linus Torvalds1da177e2005-04-16 15:20:36 -0700150 *
151 * Description:
Jens Axboe0a0d96b2008-09-11 13:17:37 +0200152 * bio_alloc_bioset will first try its own mempool to satisfy the allocation.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700153 * If %__GFP_WAIT is set then we will block on the internal pool waiting
Jens Axboe0a0d96b2008-09-11 13:17:37 +0200154 * for a &struct bio to become free. If a %NULL @bs is passed in, we will
155 * fall back to just using @kmalloc to allocate the required memory.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700156 *
157 * allocate bio and iovecs from the memory pools specified by the
Jens Axboe0a0d96b2008-09-11 13:17:37 +0200158 * bio_set structure, or @kmalloc if none given.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700159 **/
Al Virodd0fc662005-10-07 07:46:04 +0100160struct bio *bio_alloc_bioset(gfp_t gfp_mask, int nr_iovecs, struct bio_set *bs)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700161{
Jens Axboe0a0d96b2008-09-11 13:17:37 +0200162 struct bio *bio;
163
164 if (bs)
165 bio = mempool_alloc(bs->bio_pool, gfp_mask);
166 else
167 bio = kmalloc(sizeof(*bio), gfp_mask);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700168
169 if (likely(bio)) {
170 struct bio_vec *bvl = NULL;
171
172 bio_init(bio);
173 if (likely(nr_iovecs)) {
Jens Axboeeeae1d42008-05-07 13:26:27 +0200174 unsigned long uninitialized_var(idx);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700175
176 bvl = bvec_alloc_bs(gfp_mask, nr_iovecs, &idx, bs);
177 if (unlikely(!bvl)) {
Jens Axboe0a0d96b2008-09-11 13:17:37 +0200178 if (bs)
179 mempool_free(bio, bs->bio_pool);
180 else
181 kfree(bio);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700182 bio = NULL;
183 goto out;
184 }
185 bio->bi_flags |= idx << BIO_POOL_OFFSET;
Denis ChengRq1ac0ae02008-08-04 11:56:30 +0200186 bio->bi_max_vecs = bvec_nr_vecs(idx);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700187 }
188 bio->bi_io_vec = bvl;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700189 }
190out:
191 return bio;
192}
193
Al Virodd0fc662005-10-07 07:46:04 +0100194struct bio *bio_alloc(gfp_t gfp_mask, int nr_iovecs)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700195{
Peter Osterlund36763472005-09-06 15:16:42 -0700196 struct bio *bio = bio_alloc_bioset(gfp_mask, nr_iovecs, fs_bio_set);
197
198 if (bio)
199 bio->bi_destructor = bio_fs_destructor;
200
201 return bio;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700202}
203
Jens Axboe0a0d96b2008-09-11 13:17:37 +0200204/*
205 * Like bio_alloc(), but doesn't use a mempool backing. This means that
206 * it CAN fail, but while bio_alloc() can only be used for allocations
207 * that have a short (finite) life span, bio_kmalloc() should be used
208 * for more permanent bio allocations (like allocating some bio's for
209 * initalization or setup purposes).
210 */
211struct bio *bio_kmalloc(gfp_t gfp_mask, int nr_iovecs)
212{
213 struct bio *bio = bio_alloc_bioset(gfp_mask, nr_iovecs, NULL);
214
215 if (bio)
216 bio->bi_destructor = bio_kmalloc_destructor;
217
218 return bio;
219}
220
Linus Torvalds1da177e2005-04-16 15:20:36 -0700221void zero_fill_bio(struct bio *bio)
222{
223 unsigned long flags;
224 struct bio_vec *bv;
225 int i;
226
227 bio_for_each_segment(bv, bio, i) {
228 char *data = bvec_kmap_irq(bv, &flags);
229 memset(data, 0, bv->bv_len);
230 flush_dcache_page(bv->bv_page);
231 bvec_kunmap_irq(data, &flags);
232 }
233}
234EXPORT_SYMBOL(zero_fill_bio);
235
236/**
237 * bio_put - release a reference to a bio
238 * @bio: bio to release reference to
239 *
240 * Description:
241 * Put a reference to a &struct bio, either one you have gotten with
242 * bio_alloc or bio_get. The last put of a bio will free it.
243 **/
244void bio_put(struct bio *bio)
245{
246 BIO_BUG_ON(!atomic_read(&bio->bi_cnt));
247
248 /*
249 * last put frees it
250 */
251 if (atomic_dec_and_test(&bio->bi_cnt)) {
252 bio->bi_next = NULL;
253 bio->bi_destructor(bio);
254 }
255}
256
Jens Axboe165125e2007-07-24 09:28:11 +0200257inline int bio_phys_segments(struct request_queue *q, struct bio *bio)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700258{
259 if (unlikely(!bio_flagged(bio, BIO_SEG_VALID)))
260 blk_recount_segments(q, bio);
261
262 return bio->bi_phys_segments;
263}
264
Linus Torvalds1da177e2005-04-16 15:20:36 -0700265/**
266 * __bio_clone - clone a bio
267 * @bio: destination bio
268 * @bio_src: bio to clone
269 *
270 * Clone a &bio. Caller will own the returned bio, but not
271 * the actual data it points to. Reference count of returned
272 * bio will be one.
273 */
Arjan van de Ven858119e2006-01-14 13:20:43 -0800274void __bio_clone(struct bio *bio, struct bio *bio_src)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700275{
Andrew Mortone525e152005-08-07 09:42:12 -0700276 memcpy(bio->bi_io_vec, bio_src->bi_io_vec,
277 bio_src->bi_max_vecs * sizeof(struct bio_vec));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700278
Jens Axboe5d840702008-01-25 12:44:44 +0100279 /*
280 * most users will be overriding ->bi_bdev with a new target,
281 * so we don't set nor calculate new physical/hw segment counts here
282 */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700283 bio->bi_sector = bio_src->bi_sector;
284 bio->bi_bdev = bio_src->bi_bdev;
285 bio->bi_flags |= 1 << BIO_CLONED;
286 bio->bi_rw = bio_src->bi_rw;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700287 bio->bi_vcnt = bio_src->bi_vcnt;
288 bio->bi_size = bio_src->bi_size;
Andrew Mortona5453be2005-07-28 01:07:18 -0700289 bio->bi_idx = bio_src->bi_idx;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700290}
291
292/**
293 * bio_clone - clone a bio
294 * @bio: bio to clone
295 * @gfp_mask: allocation priority
296 *
297 * Like __bio_clone, only also allocates the returned bio
298 */
Al Virodd0fc662005-10-07 07:46:04 +0100299struct bio *bio_clone(struct bio *bio, gfp_t gfp_mask)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700300{
301 struct bio *b = bio_alloc_bioset(gfp_mask, bio->bi_max_vecs, fs_bio_set);
302
Martin K. Petersen7ba1ba12008-06-30 20:04:41 +0200303 if (!b)
304 return NULL;
305
306 b->bi_destructor = bio_fs_destructor;
307 __bio_clone(b, bio);
308
309 if (bio_integrity(bio)) {
310 int ret;
311
312 ret = bio_integrity_clone(b, bio, fs_bio_set);
313
314 if (ret < 0)
315 return NULL;
Peter Osterlund36763472005-09-06 15:16:42 -0700316 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700317
318 return b;
319}
320
321/**
322 * bio_get_nr_vecs - return approx number of vecs
323 * @bdev: I/O target
324 *
325 * Return the approximate number of pages we can send to this target.
326 * There's no guarantee that you will be able to fit this number of pages
327 * into a bio, it does not account for dynamic restrictions that vary
328 * on offset.
329 */
330int bio_get_nr_vecs(struct block_device *bdev)
331{
Jens Axboe165125e2007-07-24 09:28:11 +0200332 struct request_queue *q = bdev_get_queue(bdev);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700333 int nr_pages;
334
335 nr_pages = ((q->max_sectors << 9) + PAGE_SIZE - 1) >> PAGE_SHIFT;
336 if (nr_pages > q->max_phys_segments)
337 nr_pages = q->max_phys_segments;
338 if (nr_pages > q->max_hw_segments)
339 nr_pages = q->max_hw_segments;
340
341 return nr_pages;
342}
343
Jens Axboe165125e2007-07-24 09:28:11 +0200344static int __bio_add_page(struct request_queue *q, struct bio *bio, struct page
Mike Christiedefd94b2005-12-05 02:37:06 -0600345 *page, unsigned int len, unsigned int offset,
346 unsigned short max_sectors)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700347{
348 int retried_segments = 0;
349 struct bio_vec *bvec;
350
351 /*
352 * cloned bio must not modify vec list
353 */
354 if (unlikely(bio_flagged(bio, BIO_CLONED)))
355 return 0;
356
Jens Axboe80cfd542006-01-06 09:43:28 +0100357 if (((bio->bi_size + len) >> 9) > max_sectors)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700358 return 0;
359
Jens Axboe80cfd542006-01-06 09:43:28 +0100360 /*
361 * For filesystems with a blocksize smaller than the pagesize
362 * we will often be called with the same page as last time and
363 * a consecutive offset. Optimize this special case.
364 */
365 if (bio->bi_vcnt > 0) {
366 struct bio_vec *prev = &bio->bi_io_vec[bio->bi_vcnt - 1];
367
368 if (page == prev->bv_page &&
369 offset == prev->bv_offset + prev->bv_len) {
370 prev->bv_len += len;
Alasdair G Kergoncc371e62008-07-03 09:53:43 +0200371
372 if (q->merge_bvec_fn) {
373 struct bvec_merge_data bvm = {
374 .bi_bdev = bio->bi_bdev,
375 .bi_sector = bio->bi_sector,
376 .bi_size = bio->bi_size,
377 .bi_rw = bio->bi_rw,
378 };
379
380 if (q->merge_bvec_fn(q, &bvm, prev) < len) {
381 prev->bv_len -= len;
382 return 0;
383 }
Jens Axboe80cfd542006-01-06 09:43:28 +0100384 }
385
386 goto done;
387 }
388 }
389
390 if (bio->bi_vcnt >= bio->bi_max_vecs)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700391 return 0;
392
393 /*
394 * we might lose a segment or two here, but rather that than
395 * make this too complex.
396 */
397
398 while (bio->bi_phys_segments >= q->max_phys_segments
Mikulas Patocka5df97b92008-08-15 10:20:02 +0200399 || bio->bi_phys_segments >= q->max_hw_segments) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700400
401 if (retried_segments)
402 return 0;
403
404 retried_segments = 1;
405 blk_recount_segments(q, bio);
406 }
407
408 /*
409 * setup the new entry, we might clear it again later if we
410 * cannot add the page
411 */
412 bvec = &bio->bi_io_vec[bio->bi_vcnt];
413 bvec->bv_page = page;
414 bvec->bv_len = len;
415 bvec->bv_offset = offset;
416
417 /*
418 * if queue has other restrictions (eg varying max sector size
419 * depending on offset), it can specify a merge_bvec_fn in the
420 * queue to get further control
421 */
422 if (q->merge_bvec_fn) {
Alasdair G Kergoncc371e62008-07-03 09:53:43 +0200423 struct bvec_merge_data bvm = {
424 .bi_bdev = bio->bi_bdev,
425 .bi_sector = bio->bi_sector,
426 .bi_size = bio->bi_size,
427 .bi_rw = bio->bi_rw,
428 };
429
Linus Torvalds1da177e2005-04-16 15:20:36 -0700430 /*
431 * merge_bvec_fn() returns number of bytes it can accept
432 * at this offset
433 */
Alasdair G Kergoncc371e62008-07-03 09:53:43 +0200434 if (q->merge_bvec_fn(q, &bvm, bvec) < len) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700435 bvec->bv_page = NULL;
436 bvec->bv_len = 0;
437 bvec->bv_offset = 0;
438 return 0;
439 }
440 }
441
442 /* If we may be able to merge these biovecs, force a recount */
Mikulas Patockab8b3e162008-08-15 10:15:19 +0200443 if (bio->bi_vcnt && (BIOVEC_PHYS_MERGEABLE(bvec-1, bvec)))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700444 bio->bi_flags &= ~(1 << BIO_SEG_VALID);
445
446 bio->bi_vcnt++;
447 bio->bi_phys_segments++;
Jens Axboe80cfd542006-01-06 09:43:28 +0100448 done:
Linus Torvalds1da177e2005-04-16 15:20:36 -0700449 bio->bi_size += len;
450 return len;
451}
452
453/**
Mike Christie6e68af62005-11-11 05:30:27 -0600454 * bio_add_pc_page - attempt to add page to bio
Jens Axboefddfdea2006-01-31 15:24:34 +0100455 * @q: the target queue
Mike Christie6e68af62005-11-11 05:30:27 -0600456 * @bio: destination bio
457 * @page: page to add
458 * @len: vec entry length
459 * @offset: vec entry offset
460 *
461 * Attempt to add a page to the bio_vec maplist. This can fail for a
462 * number of reasons, such as the bio being full or target block
463 * device limitations. The target block device must allow bio's
464 * smaller than PAGE_SIZE, so it is always possible to add a single
465 * page to an empty bio. This should only be used by REQ_PC bios.
466 */
Jens Axboe165125e2007-07-24 09:28:11 +0200467int bio_add_pc_page(struct request_queue *q, struct bio *bio, struct page *page,
Mike Christie6e68af62005-11-11 05:30:27 -0600468 unsigned int len, unsigned int offset)
469{
Mike Christiedefd94b2005-12-05 02:37:06 -0600470 return __bio_add_page(q, bio, page, len, offset, q->max_hw_sectors);
Mike Christie6e68af62005-11-11 05:30:27 -0600471}
472
473/**
Linus Torvalds1da177e2005-04-16 15:20:36 -0700474 * bio_add_page - attempt to add page to bio
475 * @bio: destination bio
476 * @page: page to add
477 * @len: vec entry length
478 * @offset: vec entry offset
479 *
480 * Attempt to add a page to the bio_vec maplist. This can fail for a
481 * number of reasons, such as the bio being full or target block
482 * device limitations. The target block device must allow bio's
483 * smaller than PAGE_SIZE, so it is always possible to add a single
484 * page to an empty bio.
485 */
486int bio_add_page(struct bio *bio, struct page *page, unsigned int len,
487 unsigned int offset)
488{
Mike Christiedefd94b2005-12-05 02:37:06 -0600489 struct request_queue *q = bdev_get_queue(bio->bi_bdev);
490 return __bio_add_page(q, bio, page, len, offset, q->max_sectors);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700491}
492
493struct bio_map_data {
494 struct bio_vec *iovecs;
FUJITA Tomonoric5dec1c2008-04-11 12:56:49 +0200495 struct sg_iovec *sgvecs;
FUJITA Tomonori152e2832008-08-28 16:17:06 +0900496 int nr_sgvecs;
497 int is_our_pages;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700498};
499
FUJITA Tomonoric5dec1c2008-04-11 12:56:49 +0200500static void bio_set_map_data(struct bio_map_data *bmd, struct bio *bio,
FUJITA Tomonori152e2832008-08-28 16:17:06 +0900501 struct sg_iovec *iov, int iov_count,
502 int is_our_pages)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700503{
504 memcpy(bmd->iovecs, bio->bi_io_vec, sizeof(struct bio_vec) * bio->bi_vcnt);
FUJITA Tomonoric5dec1c2008-04-11 12:56:49 +0200505 memcpy(bmd->sgvecs, iov, sizeof(struct sg_iovec) * iov_count);
506 bmd->nr_sgvecs = iov_count;
FUJITA Tomonori152e2832008-08-28 16:17:06 +0900507 bmd->is_our_pages = is_our_pages;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700508 bio->bi_private = bmd;
509}
510
511static void bio_free_map_data(struct bio_map_data *bmd)
512{
513 kfree(bmd->iovecs);
FUJITA Tomonoric5dec1c2008-04-11 12:56:49 +0200514 kfree(bmd->sgvecs);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700515 kfree(bmd);
516}
517
FUJITA Tomonori76029ff2008-08-25 20:36:08 +0200518static struct bio_map_data *bio_alloc_map_data(int nr_segs, int iov_count,
519 gfp_t gfp_mask)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700520{
FUJITA Tomonori76029ff2008-08-25 20:36:08 +0200521 struct bio_map_data *bmd = kmalloc(sizeof(*bmd), gfp_mask);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700522
523 if (!bmd)
524 return NULL;
525
FUJITA Tomonori76029ff2008-08-25 20:36:08 +0200526 bmd->iovecs = kmalloc(sizeof(struct bio_vec) * nr_segs, gfp_mask);
FUJITA Tomonoric5dec1c2008-04-11 12:56:49 +0200527 if (!bmd->iovecs) {
528 kfree(bmd);
529 return NULL;
530 }
531
FUJITA Tomonori76029ff2008-08-25 20:36:08 +0200532 bmd->sgvecs = kmalloc(sizeof(struct sg_iovec) * iov_count, gfp_mask);
FUJITA Tomonoric5dec1c2008-04-11 12:56:49 +0200533 if (bmd->sgvecs)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700534 return bmd;
535
FUJITA Tomonoric5dec1c2008-04-11 12:56:49 +0200536 kfree(bmd->iovecs);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700537 kfree(bmd);
538 return NULL;
539}
540
FUJITA Tomonoriaefcc282008-08-25 20:36:08 +0200541static int __bio_copy_iov(struct bio *bio, struct bio_vec *iovecs,
FUJITA Tomonori152e2832008-08-28 16:17:06 +0900542 struct sg_iovec *iov, int iov_count, int uncopy,
543 int do_free_page)
FUJITA Tomonoric5dec1c2008-04-11 12:56:49 +0200544{
545 int ret = 0, i;
546 struct bio_vec *bvec;
547 int iov_idx = 0;
548 unsigned int iov_off = 0;
549 int read = bio_data_dir(bio) == READ;
550
551 __bio_for_each_segment(bvec, bio, i, 0) {
552 char *bv_addr = page_address(bvec->bv_page);
FUJITA Tomonoriaefcc282008-08-25 20:36:08 +0200553 unsigned int bv_len = iovecs[i].bv_len;
FUJITA Tomonoric5dec1c2008-04-11 12:56:49 +0200554
555 while (bv_len && iov_idx < iov_count) {
556 unsigned int bytes;
557 char *iov_addr;
558
559 bytes = min_t(unsigned int,
560 iov[iov_idx].iov_len - iov_off, bv_len);
561 iov_addr = iov[iov_idx].iov_base + iov_off;
562
563 if (!ret) {
564 if (!read && !uncopy)
565 ret = copy_from_user(bv_addr, iov_addr,
566 bytes);
567 if (read && uncopy)
568 ret = copy_to_user(iov_addr, bv_addr,
569 bytes);
570
571 if (ret)
572 ret = -EFAULT;
573 }
574
575 bv_len -= bytes;
576 bv_addr += bytes;
577 iov_addr += bytes;
578 iov_off += bytes;
579
580 if (iov[iov_idx].iov_len == iov_off) {
581 iov_idx++;
582 iov_off = 0;
583 }
584 }
585
FUJITA Tomonori152e2832008-08-28 16:17:06 +0900586 if (do_free_page)
FUJITA Tomonoric5dec1c2008-04-11 12:56:49 +0200587 __free_page(bvec->bv_page);
588 }
589
590 return ret;
591}
592
Linus Torvalds1da177e2005-04-16 15:20:36 -0700593/**
594 * bio_uncopy_user - finish previously mapped bio
595 * @bio: bio being terminated
596 *
597 * Free pages allocated from bio_copy_user() and write back data
598 * to user space in case of a read.
599 */
600int bio_uncopy_user(struct bio *bio)
601{
602 struct bio_map_data *bmd = bio->bi_private;
FUJITA Tomonori81882762008-09-02 16:20:19 +0900603 int ret = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700604
FUJITA Tomonori81882762008-09-02 16:20:19 +0900605 if (!bio_flagged(bio, BIO_NULL_MAPPED))
606 ret = __bio_copy_iov(bio, bmd->iovecs, bmd->sgvecs,
607 bmd->nr_sgvecs, 1, bmd->is_our_pages);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700608 bio_free_map_data(bmd);
609 bio_put(bio);
610 return ret;
611}
612
613/**
FUJITA Tomonoric5dec1c2008-04-11 12:56:49 +0200614 * bio_copy_user_iov - copy user data to bio
Linus Torvalds1da177e2005-04-16 15:20:36 -0700615 * @q: destination block queue
FUJITA Tomonori152e2832008-08-28 16:17:06 +0900616 * @map_data: pointer to the rq_map_data holding pages (if necessary)
FUJITA Tomonoric5dec1c2008-04-11 12:56:49 +0200617 * @iov: the iovec.
618 * @iov_count: number of elements in the iovec
Linus Torvalds1da177e2005-04-16 15:20:36 -0700619 * @write_to_vm: bool indicating writing to pages or not
FUJITA Tomonoria3bce902008-08-28 16:17:05 +0900620 * @gfp_mask: memory allocation flags
Linus Torvalds1da177e2005-04-16 15:20:36 -0700621 *
622 * Prepares and returns a bio for indirect user io, bouncing data
623 * to/from kernel pages as necessary. Must be paired with
624 * call bio_uncopy_user() on io completion.
625 */
FUJITA Tomonori152e2832008-08-28 16:17:06 +0900626struct bio *bio_copy_user_iov(struct request_queue *q,
627 struct rq_map_data *map_data,
628 struct sg_iovec *iov, int iov_count,
629 int write_to_vm, gfp_t gfp_mask)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700630{
Linus Torvalds1da177e2005-04-16 15:20:36 -0700631 struct bio_map_data *bmd;
632 struct bio_vec *bvec;
633 struct page *page;
634 struct bio *bio;
635 int i, ret;
FUJITA Tomonoric5dec1c2008-04-11 12:56:49 +0200636 int nr_pages = 0;
637 unsigned int len = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700638
FUJITA Tomonoric5dec1c2008-04-11 12:56:49 +0200639 for (i = 0; i < iov_count; i++) {
640 unsigned long uaddr;
641 unsigned long end;
642 unsigned long start;
643
644 uaddr = (unsigned long)iov[i].iov_base;
645 end = (uaddr + iov[i].iov_len + PAGE_SIZE - 1) >> PAGE_SHIFT;
646 start = uaddr >> PAGE_SHIFT;
647
648 nr_pages += end - start;
649 len += iov[i].iov_len;
650 }
651
FUJITA Tomonoria3bce902008-08-28 16:17:05 +0900652 bmd = bio_alloc_map_data(nr_pages, iov_count, gfp_mask);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700653 if (!bmd)
654 return ERR_PTR(-ENOMEM);
655
Linus Torvalds1da177e2005-04-16 15:20:36 -0700656 ret = -ENOMEM;
FUJITA Tomonoria3bce902008-08-28 16:17:05 +0900657 bio = bio_alloc(gfp_mask, nr_pages);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700658 if (!bio)
659 goto out_bmd;
660
661 bio->bi_rw |= (!write_to_vm << BIO_RW);
662
663 ret = 0;
FUJITA Tomonori152e2832008-08-28 16:17:06 +0900664 i = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700665 while (len) {
FUJITA Tomonori152e2832008-08-28 16:17:06 +0900666 unsigned int bytes;
667
668 if (map_data)
669 bytes = 1U << (PAGE_SHIFT + map_data->page_order);
670 else
671 bytes = PAGE_SIZE;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700672
673 if (bytes > len)
674 bytes = len;
675
FUJITA Tomonori152e2832008-08-28 16:17:06 +0900676 if (map_data) {
677 if (i == map_data->nr_entries) {
678 ret = -ENOMEM;
679 break;
680 }
681 page = map_data->pages[i++];
682 } else
683 page = alloc_page(q->bounce_gfp | gfp_mask);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700684 if (!page) {
685 ret = -ENOMEM;
686 break;
687 }
688
Mike Christie0e75f902006-12-01 10:40:55 +0100689 if (bio_add_pc_page(q, bio, page, bytes, 0) < bytes)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700690 break;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700691
692 len -= bytes;
693 }
694
695 if (ret)
696 goto cleanup;
697
698 /*
699 * success
700 */
701 if (!write_to_vm) {
FUJITA Tomonori152e2832008-08-28 16:17:06 +0900702 ret = __bio_copy_iov(bio, bio->bi_io_vec, iov, iov_count, 0, 0);
FUJITA Tomonoric5dec1c2008-04-11 12:56:49 +0200703 if (ret)
704 goto cleanup;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700705 }
706
FUJITA Tomonori152e2832008-08-28 16:17:06 +0900707 bio_set_map_data(bmd, bio, iov, iov_count, map_data ? 0 : 1);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700708 return bio;
709cleanup:
FUJITA Tomonori152e2832008-08-28 16:17:06 +0900710 if (!map_data)
711 bio_for_each_segment(bvec, bio, i)
712 __free_page(bvec->bv_page);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700713
714 bio_put(bio);
715out_bmd:
716 bio_free_map_data(bmd);
717 return ERR_PTR(ret);
718}
719
FUJITA Tomonoric5dec1c2008-04-11 12:56:49 +0200720/**
721 * bio_copy_user - copy user data to bio
722 * @q: destination block queue
FUJITA Tomonori152e2832008-08-28 16:17:06 +0900723 * @map_data: pointer to the rq_map_data holding pages (if necessary)
FUJITA Tomonoric5dec1c2008-04-11 12:56:49 +0200724 * @uaddr: start of user address
725 * @len: length in bytes
726 * @write_to_vm: bool indicating writing to pages or not
FUJITA Tomonoria3bce902008-08-28 16:17:05 +0900727 * @gfp_mask: memory allocation flags
FUJITA Tomonoric5dec1c2008-04-11 12:56:49 +0200728 *
729 * Prepares and returns a bio for indirect user io, bouncing data
730 * to/from kernel pages as necessary. Must be paired with
731 * call bio_uncopy_user() on io completion.
732 */
FUJITA Tomonori152e2832008-08-28 16:17:06 +0900733struct bio *bio_copy_user(struct request_queue *q, struct rq_map_data *map_data,
734 unsigned long uaddr, unsigned int len,
735 int write_to_vm, gfp_t gfp_mask)
FUJITA Tomonoric5dec1c2008-04-11 12:56:49 +0200736{
737 struct sg_iovec iov;
738
739 iov.iov_base = (void __user *)uaddr;
740 iov.iov_len = len;
741
FUJITA Tomonori152e2832008-08-28 16:17:06 +0900742 return bio_copy_user_iov(q, map_data, &iov, 1, write_to_vm, gfp_mask);
FUJITA Tomonoric5dec1c2008-04-11 12:56:49 +0200743}
744
Jens Axboe165125e2007-07-24 09:28:11 +0200745static struct bio *__bio_map_user_iov(struct request_queue *q,
James Bottomley f1970ba2005-06-20 14:06:52 +0200746 struct block_device *bdev,
747 struct sg_iovec *iov, int iov_count,
FUJITA Tomonoria3bce902008-08-28 16:17:05 +0900748 int write_to_vm, gfp_t gfp_mask)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700749{
James Bottomley f1970ba2005-06-20 14:06:52 +0200750 int i, j;
751 int nr_pages = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700752 struct page **pages;
753 struct bio *bio;
James Bottomley f1970ba2005-06-20 14:06:52 +0200754 int cur_page = 0;
755 int ret, offset;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700756
James Bottomley f1970ba2005-06-20 14:06:52 +0200757 for (i = 0; i < iov_count; i++) {
758 unsigned long uaddr = (unsigned long)iov[i].iov_base;
759 unsigned long len = iov[i].iov_len;
760 unsigned long end = (uaddr + len + PAGE_SIZE - 1) >> PAGE_SHIFT;
761 unsigned long start = uaddr >> PAGE_SHIFT;
762
763 nr_pages += end - start;
764 /*
Mike Christiead2d7222006-12-01 10:40:20 +0100765 * buffer must be aligned to at least hardsector size for now
James Bottomley f1970ba2005-06-20 14:06:52 +0200766 */
Mike Christiead2d7222006-12-01 10:40:20 +0100767 if (uaddr & queue_dma_alignment(q))
James Bottomley f1970ba2005-06-20 14:06:52 +0200768 return ERR_PTR(-EINVAL);
769 }
770
771 if (!nr_pages)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700772 return ERR_PTR(-EINVAL);
773
FUJITA Tomonoria3bce902008-08-28 16:17:05 +0900774 bio = bio_alloc(gfp_mask, nr_pages);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700775 if (!bio)
776 return ERR_PTR(-ENOMEM);
777
778 ret = -ENOMEM;
FUJITA Tomonoria3bce902008-08-28 16:17:05 +0900779 pages = kcalloc(nr_pages, sizeof(struct page *), gfp_mask);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700780 if (!pages)
781 goto out;
782
James Bottomley f1970ba2005-06-20 14:06:52 +0200783 for (i = 0; i < iov_count; i++) {
784 unsigned long uaddr = (unsigned long)iov[i].iov_base;
785 unsigned long len = iov[i].iov_len;
786 unsigned long end = (uaddr + len + PAGE_SIZE - 1) >> PAGE_SHIFT;
787 unsigned long start = uaddr >> PAGE_SHIFT;
788 const int local_nr_pages = end - start;
789 const int page_limit = cur_page + local_nr_pages;
790
Nick Pigginf5dd33c2008-07-25 19:45:25 -0700791 ret = get_user_pages_fast(uaddr, local_nr_pages,
792 write_to_vm, &pages[cur_page]);
Jens Axboe99172152006-06-16 13:02:29 +0200793 if (ret < local_nr_pages) {
794 ret = -EFAULT;
James Bottomley f1970ba2005-06-20 14:06:52 +0200795 goto out_unmap;
Jens Axboe99172152006-06-16 13:02:29 +0200796 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700797
James Bottomley f1970ba2005-06-20 14:06:52 +0200798 offset = uaddr & ~PAGE_MASK;
799 for (j = cur_page; j < page_limit; j++) {
800 unsigned int bytes = PAGE_SIZE - offset;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700801
James Bottomley f1970ba2005-06-20 14:06:52 +0200802 if (len <= 0)
803 break;
804
805 if (bytes > len)
806 bytes = len;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700807
James Bottomley f1970ba2005-06-20 14:06:52 +0200808 /*
809 * sorry...
810 */
Mike Christiedefd94b2005-12-05 02:37:06 -0600811 if (bio_add_pc_page(q, bio, pages[j], bytes, offset) <
812 bytes)
James Bottomley f1970ba2005-06-20 14:06:52 +0200813 break;
814
815 len -= bytes;
816 offset = 0;
817 }
818
819 cur_page = j;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700820 /*
James Bottomley f1970ba2005-06-20 14:06:52 +0200821 * release the pages we didn't map into the bio, if any
Linus Torvalds1da177e2005-04-16 15:20:36 -0700822 */
James Bottomley f1970ba2005-06-20 14:06:52 +0200823 while (j < page_limit)
824 page_cache_release(pages[j++]);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700825 }
826
Linus Torvalds1da177e2005-04-16 15:20:36 -0700827 kfree(pages);
828
829 /*
830 * set data direction, and check if mapped pages need bouncing
831 */
832 if (!write_to_vm)
833 bio->bi_rw |= (1 << BIO_RW);
834
James Bottomley f1970ba2005-06-20 14:06:52 +0200835 bio->bi_bdev = bdev;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700836 bio->bi_flags |= (1 << BIO_USER_MAPPED);
837 return bio;
James Bottomley f1970ba2005-06-20 14:06:52 +0200838
839 out_unmap:
840 for (i = 0; i < nr_pages; i++) {
841 if(!pages[i])
842 break;
843 page_cache_release(pages[i]);
844 }
845 out:
Linus Torvalds1da177e2005-04-16 15:20:36 -0700846 kfree(pages);
847 bio_put(bio);
848 return ERR_PTR(ret);
849}
850
851/**
852 * bio_map_user - map user address into bio
Jens Axboe165125e2007-07-24 09:28:11 +0200853 * @q: the struct request_queue for the bio
Linus Torvalds1da177e2005-04-16 15:20:36 -0700854 * @bdev: destination block device
855 * @uaddr: start of user address
856 * @len: length in bytes
857 * @write_to_vm: bool indicating writing to pages or not
FUJITA Tomonoria3bce902008-08-28 16:17:05 +0900858 * @gfp_mask: memory allocation flags
Linus Torvalds1da177e2005-04-16 15:20:36 -0700859 *
860 * Map the user space address into a bio suitable for io to a block
861 * device. Returns an error pointer in case of error.
862 */
Jens Axboe165125e2007-07-24 09:28:11 +0200863struct bio *bio_map_user(struct request_queue *q, struct block_device *bdev,
FUJITA Tomonoria3bce902008-08-28 16:17:05 +0900864 unsigned long uaddr, unsigned int len, int write_to_vm,
865 gfp_t gfp_mask)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700866{
James Bottomley f1970ba2005-06-20 14:06:52 +0200867 struct sg_iovec iov;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700868
viro@ZenIV.linux.org.uk3f703532005-09-09 16:53:56 +0100869 iov.iov_base = (void __user *)uaddr;
James Bottomley f1970ba2005-06-20 14:06:52 +0200870 iov.iov_len = len;
871
FUJITA Tomonoria3bce902008-08-28 16:17:05 +0900872 return bio_map_user_iov(q, bdev, &iov, 1, write_to_vm, gfp_mask);
James Bottomley f1970ba2005-06-20 14:06:52 +0200873}
874
875/**
876 * bio_map_user_iov - map user sg_iovec table into bio
Jens Axboe165125e2007-07-24 09:28:11 +0200877 * @q: the struct request_queue for the bio
James Bottomley f1970ba2005-06-20 14:06:52 +0200878 * @bdev: destination block device
879 * @iov: the iovec.
880 * @iov_count: number of elements in the iovec
881 * @write_to_vm: bool indicating writing to pages or not
FUJITA Tomonoria3bce902008-08-28 16:17:05 +0900882 * @gfp_mask: memory allocation flags
James Bottomley f1970ba2005-06-20 14:06:52 +0200883 *
884 * Map the user space address into a bio suitable for io to a block
885 * device. Returns an error pointer in case of error.
886 */
Jens Axboe165125e2007-07-24 09:28:11 +0200887struct bio *bio_map_user_iov(struct request_queue *q, struct block_device *bdev,
James Bottomley f1970ba2005-06-20 14:06:52 +0200888 struct sg_iovec *iov, int iov_count,
FUJITA Tomonoria3bce902008-08-28 16:17:05 +0900889 int write_to_vm, gfp_t gfp_mask)
James Bottomley f1970ba2005-06-20 14:06:52 +0200890{
891 struct bio *bio;
James Bottomley f1970ba2005-06-20 14:06:52 +0200892
FUJITA Tomonoria3bce902008-08-28 16:17:05 +0900893 bio = __bio_map_user_iov(q, bdev, iov, iov_count, write_to_vm,
894 gfp_mask);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700895 if (IS_ERR(bio))
896 return bio;
897
898 /*
899 * subtle -- if __bio_map_user() ended up bouncing a bio,
900 * it would normally disappear when its bi_end_io is run.
901 * however, we need it for the unmap, so grab an extra
902 * reference to it
903 */
904 bio_get(bio);
905
Mike Christie0e75f902006-12-01 10:40:55 +0100906 return bio;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700907}
908
909static void __bio_unmap_user(struct bio *bio)
910{
911 struct bio_vec *bvec;
912 int i;
913
914 /*
915 * make sure we dirty pages we wrote to
916 */
917 __bio_for_each_segment(bvec, bio, i, 0) {
918 if (bio_data_dir(bio) == READ)
919 set_page_dirty_lock(bvec->bv_page);
920
921 page_cache_release(bvec->bv_page);
922 }
923
924 bio_put(bio);
925}
926
927/**
928 * bio_unmap_user - unmap a bio
929 * @bio: the bio being unmapped
930 *
931 * Unmap a bio previously mapped by bio_map_user(). Must be called with
932 * a process context.
933 *
934 * bio_unmap_user() may sleep.
935 */
936void bio_unmap_user(struct bio *bio)
937{
938 __bio_unmap_user(bio);
939 bio_put(bio);
940}
941
NeilBrown6712ecf2007-09-27 12:47:43 +0200942static void bio_map_kern_endio(struct bio *bio, int err)
Jens Axboeb8238252005-06-20 14:05:27 +0200943{
Jens Axboeb8238252005-06-20 14:05:27 +0200944 bio_put(bio);
Jens Axboeb8238252005-06-20 14:05:27 +0200945}
946
947
Jens Axboe165125e2007-07-24 09:28:11 +0200948static struct bio *__bio_map_kern(struct request_queue *q, void *data,
Al Viro27496a82005-10-21 03:20:48 -0400949 unsigned int len, gfp_t gfp_mask)
Mike Christie df46b9a2005-06-20 14:04:44 +0200950{
951 unsigned long kaddr = (unsigned long)data;
952 unsigned long end = (kaddr + len + PAGE_SIZE - 1) >> PAGE_SHIFT;
953 unsigned long start = kaddr >> PAGE_SHIFT;
954 const int nr_pages = end - start;
955 int offset, i;
956 struct bio *bio;
957
958 bio = bio_alloc(gfp_mask, nr_pages);
959 if (!bio)
960 return ERR_PTR(-ENOMEM);
961
962 offset = offset_in_page(kaddr);
963 for (i = 0; i < nr_pages; i++) {
964 unsigned int bytes = PAGE_SIZE - offset;
965
966 if (len <= 0)
967 break;
968
969 if (bytes > len)
970 bytes = len;
971
Mike Christiedefd94b2005-12-05 02:37:06 -0600972 if (bio_add_pc_page(q, bio, virt_to_page(data), bytes,
973 offset) < bytes)
Mike Christie df46b9a2005-06-20 14:04:44 +0200974 break;
975
976 data += bytes;
977 len -= bytes;
978 offset = 0;
979 }
980
Jens Axboeb8238252005-06-20 14:05:27 +0200981 bio->bi_end_io = bio_map_kern_endio;
Mike Christie df46b9a2005-06-20 14:04:44 +0200982 return bio;
983}
984
985/**
986 * bio_map_kern - map kernel address into bio
Jens Axboe165125e2007-07-24 09:28:11 +0200987 * @q: the struct request_queue for the bio
Mike Christie df46b9a2005-06-20 14:04:44 +0200988 * @data: pointer to buffer to map
989 * @len: length in bytes
990 * @gfp_mask: allocation flags for bio allocation
991 *
992 * Map the kernel address into a bio suitable for io to a block
993 * device. Returns an error pointer in case of error.
994 */
Jens Axboe165125e2007-07-24 09:28:11 +0200995struct bio *bio_map_kern(struct request_queue *q, void *data, unsigned int len,
Al Viro27496a82005-10-21 03:20:48 -0400996 gfp_t gfp_mask)
Mike Christie df46b9a2005-06-20 14:04:44 +0200997{
998 struct bio *bio;
999
1000 bio = __bio_map_kern(q, data, len, gfp_mask);
1001 if (IS_ERR(bio))
1002 return bio;
1003
1004 if (bio->bi_size == len)
1005 return bio;
1006
1007 /*
1008 * Don't support partial mappings.
1009 */
1010 bio_put(bio);
1011 return ERR_PTR(-EINVAL);
1012}
1013
FUJITA Tomonori68154e92008-04-25 12:47:50 +02001014static void bio_copy_kern_endio(struct bio *bio, int err)
1015{
1016 struct bio_vec *bvec;
1017 const int read = bio_data_dir(bio) == READ;
FUJITA Tomonori76029ff2008-08-25 20:36:08 +02001018 struct bio_map_data *bmd = bio->bi_private;
FUJITA Tomonori68154e92008-04-25 12:47:50 +02001019 int i;
FUJITA Tomonori76029ff2008-08-25 20:36:08 +02001020 char *p = bmd->sgvecs[0].iov_base;
FUJITA Tomonori68154e92008-04-25 12:47:50 +02001021
1022 __bio_for_each_segment(bvec, bio, i, 0) {
1023 char *addr = page_address(bvec->bv_page);
FUJITA Tomonori76029ff2008-08-25 20:36:08 +02001024 int len = bmd->iovecs[i].bv_len;
FUJITA Tomonori68154e92008-04-25 12:47:50 +02001025
1026 if (read && !err)
FUJITA Tomonori76029ff2008-08-25 20:36:08 +02001027 memcpy(p, addr, len);
FUJITA Tomonori68154e92008-04-25 12:47:50 +02001028
1029 __free_page(bvec->bv_page);
FUJITA Tomonori76029ff2008-08-25 20:36:08 +02001030 p += len;
FUJITA Tomonori68154e92008-04-25 12:47:50 +02001031 }
1032
FUJITA Tomonori76029ff2008-08-25 20:36:08 +02001033 bio_free_map_data(bmd);
FUJITA Tomonori68154e92008-04-25 12:47:50 +02001034 bio_put(bio);
1035}
1036
1037/**
1038 * bio_copy_kern - copy kernel address into bio
1039 * @q: the struct request_queue for the bio
1040 * @data: pointer to buffer to copy
1041 * @len: length in bytes
1042 * @gfp_mask: allocation flags for bio and page allocation
Randy Dunlapffee0252008-04-30 09:08:54 +02001043 * @reading: data direction is READ
FUJITA Tomonori68154e92008-04-25 12:47:50 +02001044 *
1045 * copy the kernel address into a bio suitable for io to a block
1046 * device. Returns an error pointer in case of error.
1047 */
1048struct bio *bio_copy_kern(struct request_queue *q, void *data, unsigned int len,
1049 gfp_t gfp_mask, int reading)
1050{
FUJITA Tomonori68154e92008-04-25 12:47:50 +02001051 struct bio *bio;
1052 struct bio_vec *bvec;
FUJITA Tomonori4d8ab622008-08-28 15:05:57 +09001053 int i;
FUJITA Tomonori68154e92008-04-25 12:47:50 +02001054
FUJITA Tomonori4d8ab622008-08-28 15:05:57 +09001055 bio = bio_copy_user(q, NULL, (unsigned long)data, len, 1, gfp_mask);
1056 if (IS_ERR(bio))
1057 return bio;
FUJITA Tomonori68154e92008-04-25 12:47:50 +02001058
1059 if (!reading) {
1060 void *p = data;
1061
1062 bio_for_each_segment(bvec, bio, i) {
1063 char *addr = page_address(bvec->bv_page);
1064
1065 memcpy(addr, p, bvec->bv_len);
1066 p += bvec->bv_len;
1067 }
1068 }
1069
FUJITA Tomonori68154e92008-04-25 12:47:50 +02001070 bio->bi_end_io = bio_copy_kern_endio;
FUJITA Tomonori76029ff2008-08-25 20:36:08 +02001071
FUJITA Tomonori68154e92008-04-25 12:47:50 +02001072 return bio;
FUJITA Tomonori68154e92008-04-25 12:47:50 +02001073}
1074
Linus Torvalds1da177e2005-04-16 15:20:36 -07001075/*
1076 * bio_set_pages_dirty() and bio_check_pages_dirty() are support functions
1077 * for performing direct-IO in BIOs.
1078 *
1079 * The problem is that we cannot run set_page_dirty() from interrupt context
1080 * because the required locks are not interrupt-safe. So what we can do is to
1081 * mark the pages dirty _before_ performing IO. And in interrupt context,
1082 * check that the pages are still dirty. If so, fine. If not, redirty them
1083 * in process context.
1084 *
1085 * We special-case compound pages here: normally this means reads into hugetlb
1086 * pages. The logic in here doesn't really work right for compound pages
1087 * because the VM does not uniformly chase down the head page in all cases.
1088 * But dirtiness of compound pages is pretty meaningless anyway: the VM doesn't
1089 * handle them at all. So we skip compound pages here at an early stage.
1090 *
1091 * Note that this code is very hard to test under normal circumstances because
1092 * direct-io pins the pages with get_user_pages(). This makes
1093 * is_page_cache_freeable return false, and the VM will not clean the pages.
1094 * But other code (eg, pdflush) could clean the pages if they are mapped
1095 * pagecache.
1096 *
1097 * Simply disabling the call to bio_set_pages_dirty() is a good way to test the
1098 * deferred bio dirtying paths.
1099 */
1100
1101/*
1102 * bio_set_pages_dirty() will mark all the bio's pages as dirty.
1103 */
1104void bio_set_pages_dirty(struct bio *bio)
1105{
1106 struct bio_vec *bvec = bio->bi_io_vec;
1107 int i;
1108
1109 for (i = 0; i < bio->bi_vcnt; i++) {
1110 struct page *page = bvec[i].bv_page;
1111
1112 if (page && !PageCompound(page))
1113 set_page_dirty_lock(page);
1114 }
1115}
1116
Adrian Bunk86b6c7a2008-02-18 13:48:32 +01001117static void bio_release_pages(struct bio *bio)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001118{
1119 struct bio_vec *bvec = bio->bi_io_vec;
1120 int i;
1121
1122 for (i = 0; i < bio->bi_vcnt; i++) {
1123 struct page *page = bvec[i].bv_page;
1124
1125 if (page)
1126 put_page(page);
1127 }
1128}
1129
1130/*
1131 * bio_check_pages_dirty() will check that all the BIO's pages are still dirty.
1132 * If they are, then fine. If, however, some pages are clean then they must
1133 * have been written out during the direct-IO read. So we take another ref on
1134 * the BIO and the offending pages and re-dirty the pages in process context.
1135 *
1136 * It is expected that bio_check_pages_dirty() will wholly own the BIO from
1137 * here on. It will run one page_cache_release() against each page and will
1138 * run one bio_put() against the BIO.
1139 */
1140
David Howells65f27f32006-11-22 14:55:48 +00001141static void bio_dirty_fn(struct work_struct *work);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001142
David Howells65f27f32006-11-22 14:55:48 +00001143static DECLARE_WORK(bio_dirty_work, bio_dirty_fn);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001144static DEFINE_SPINLOCK(bio_dirty_lock);
1145static struct bio *bio_dirty_list;
1146
1147/*
1148 * This runs in process context
1149 */
David Howells65f27f32006-11-22 14:55:48 +00001150static void bio_dirty_fn(struct work_struct *work)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001151{
1152 unsigned long flags;
1153 struct bio *bio;
1154
1155 spin_lock_irqsave(&bio_dirty_lock, flags);
1156 bio = bio_dirty_list;
1157 bio_dirty_list = NULL;
1158 spin_unlock_irqrestore(&bio_dirty_lock, flags);
1159
1160 while (bio) {
1161 struct bio *next = bio->bi_private;
1162
1163 bio_set_pages_dirty(bio);
1164 bio_release_pages(bio);
1165 bio_put(bio);
1166 bio = next;
1167 }
1168}
1169
1170void bio_check_pages_dirty(struct bio *bio)
1171{
1172 struct bio_vec *bvec = bio->bi_io_vec;
1173 int nr_clean_pages = 0;
1174 int i;
1175
1176 for (i = 0; i < bio->bi_vcnt; i++) {
1177 struct page *page = bvec[i].bv_page;
1178
1179 if (PageDirty(page) || PageCompound(page)) {
1180 page_cache_release(page);
1181 bvec[i].bv_page = NULL;
1182 } else {
1183 nr_clean_pages++;
1184 }
1185 }
1186
1187 if (nr_clean_pages) {
1188 unsigned long flags;
1189
1190 spin_lock_irqsave(&bio_dirty_lock, flags);
1191 bio->bi_private = bio_dirty_list;
1192 bio_dirty_list = bio;
1193 spin_unlock_irqrestore(&bio_dirty_lock, flags);
1194 schedule_work(&bio_dirty_work);
1195 } else {
1196 bio_put(bio);
1197 }
1198}
1199
1200/**
1201 * bio_endio - end I/O on a bio
1202 * @bio: bio
Linus Torvalds1da177e2005-04-16 15:20:36 -07001203 * @error: error, if any
1204 *
1205 * Description:
NeilBrown6712ecf2007-09-27 12:47:43 +02001206 * bio_endio() will end I/O on the whole bio. bio_endio() is the
NeilBrown5bb23a62007-09-27 12:46:13 +02001207 * preferred way to end I/O on a bio, it takes care of clearing
1208 * BIO_UPTODATE on error. @error is 0 on success, and and one of the
1209 * established -Exxxx (-EIO, for instance) error values in case
1210 * something went wrong. Noone should call bi_end_io() directly on a
1211 * bio unless they own it and thus know that it has an end_io
1212 * function.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001213 **/
NeilBrown6712ecf2007-09-27 12:47:43 +02001214void bio_endio(struct bio *bio, int error)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001215{
1216 if (error)
1217 clear_bit(BIO_UPTODATE, &bio->bi_flags);
NeilBrown9cc54d42007-09-27 12:46:12 +02001218 else if (!test_bit(BIO_UPTODATE, &bio->bi_flags))
1219 error = -EIO;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001220
NeilBrown5bb23a62007-09-27 12:46:13 +02001221 if (bio->bi_end_io)
NeilBrown6712ecf2007-09-27 12:47:43 +02001222 bio->bi_end_io(bio, error);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001223}
1224
1225void bio_pair_release(struct bio_pair *bp)
1226{
1227 if (atomic_dec_and_test(&bp->cnt)) {
1228 struct bio *master = bp->bio1.bi_private;
1229
NeilBrown6712ecf2007-09-27 12:47:43 +02001230 bio_endio(master, bp->error);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001231 mempool_free(bp, bp->bio2.bi_private);
1232 }
1233}
1234
NeilBrown6712ecf2007-09-27 12:47:43 +02001235static void bio_pair_end_1(struct bio *bi, int err)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001236{
1237 struct bio_pair *bp = container_of(bi, struct bio_pair, bio1);
1238
1239 if (err)
1240 bp->error = err;
1241
Linus Torvalds1da177e2005-04-16 15:20:36 -07001242 bio_pair_release(bp);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001243}
1244
NeilBrown6712ecf2007-09-27 12:47:43 +02001245static void bio_pair_end_2(struct bio *bi, int err)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001246{
1247 struct bio_pair *bp = container_of(bi, struct bio_pair, bio2);
1248
1249 if (err)
1250 bp->error = err;
1251
Linus Torvalds1da177e2005-04-16 15:20:36 -07001252 bio_pair_release(bp);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001253}
1254
1255/*
1256 * split a bio - only worry about a bio with a single page
1257 * in it's iovec
1258 */
1259struct bio_pair *bio_split(struct bio *bi, mempool_t *pool, int first_sectors)
1260{
1261 struct bio_pair *bp = mempool_alloc(pool, GFP_NOIO);
1262
1263 if (!bp)
1264 return bp;
1265
Jens Axboe2056a782006-03-23 20:00:26 +01001266 blk_add_trace_pdu_int(bdev_get_queue(bi->bi_bdev), BLK_TA_SPLIT, bi,
1267 bi->bi_sector + first_sectors);
1268
Linus Torvalds1da177e2005-04-16 15:20:36 -07001269 BUG_ON(bi->bi_vcnt != 1);
1270 BUG_ON(bi->bi_idx != 0);
1271 atomic_set(&bp->cnt, 3);
1272 bp->error = 0;
1273 bp->bio1 = *bi;
1274 bp->bio2 = *bi;
1275 bp->bio2.bi_sector += first_sectors;
1276 bp->bio2.bi_size -= first_sectors << 9;
1277 bp->bio1.bi_size = first_sectors << 9;
1278
1279 bp->bv1 = bi->bi_io_vec[0];
1280 bp->bv2 = bi->bi_io_vec[0];
1281 bp->bv2.bv_offset += first_sectors << 9;
1282 bp->bv2.bv_len -= first_sectors << 9;
1283 bp->bv1.bv_len = first_sectors << 9;
1284
1285 bp->bio1.bi_io_vec = &bp->bv1;
1286 bp->bio2.bi_io_vec = &bp->bv2;
1287
NeilBrowna2eb0c12006-05-22 22:35:27 -07001288 bp->bio1.bi_max_vecs = 1;
1289 bp->bio2.bi_max_vecs = 1;
1290
Linus Torvalds1da177e2005-04-16 15:20:36 -07001291 bp->bio1.bi_end_io = bio_pair_end_1;
1292 bp->bio2.bi_end_io = bio_pair_end_2;
1293
1294 bp->bio1.bi_private = bi;
1295 bp->bio2.bi_private = pool;
1296
Martin K. Petersen7ba1ba12008-06-30 20:04:41 +02001297 if (bio_integrity(bi))
1298 bio_integrity_split(bi, bp, first_sectors);
1299
Linus Torvalds1da177e2005-04-16 15:20:36 -07001300 return bp;
1301}
1302
Linus Torvalds1da177e2005-04-16 15:20:36 -07001303
1304/*
1305 * create memory pools for biovec's in a bio_set.
1306 * use the global biovec slabs created for general use.
1307 */
Jens Axboe59725112007-04-02 10:06:42 +02001308static int biovec_create_pools(struct bio_set *bs, int pool_entries)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001309{
1310 int i;
1311
1312 for (i = 0; i < BIOVEC_NR_POOLS; i++) {
1313 struct biovec_slab *bp = bvec_slabs + i;
1314 mempool_t **bvp = bs->bvec_pools + i;
1315
Matthew Dobson93d23412006-03-26 01:37:50 -08001316 *bvp = mempool_create_slab_pool(pool_entries, bp->slab);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001317 if (!*bvp)
1318 return -ENOMEM;
1319 }
1320 return 0;
1321}
1322
1323static void biovec_free_pools(struct bio_set *bs)
1324{
1325 int i;
1326
1327 for (i = 0; i < BIOVEC_NR_POOLS; i++) {
1328 mempool_t *bvp = bs->bvec_pools[i];
1329
1330 if (bvp)
1331 mempool_destroy(bvp);
1332 }
1333
1334}
1335
1336void bioset_free(struct bio_set *bs)
1337{
1338 if (bs->bio_pool)
1339 mempool_destroy(bs->bio_pool);
1340
Martin K. Petersen7ba1ba12008-06-30 20:04:41 +02001341 bioset_integrity_free(bs);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001342 biovec_free_pools(bs);
1343
1344 kfree(bs);
1345}
1346
Jens Axboe59725112007-04-02 10:06:42 +02001347struct bio_set *bioset_create(int bio_pool_size, int bvec_pool_size)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001348{
Oliver Neukum11b0b5a2006-03-25 03:08:13 -08001349 struct bio_set *bs = kzalloc(sizeof(*bs), GFP_KERNEL);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001350
1351 if (!bs)
1352 return NULL;
1353
Matthew Dobson93d23412006-03-26 01:37:50 -08001354 bs->bio_pool = mempool_create_slab_pool(bio_pool_size, bio_slab);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001355 if (!bs->bio_pool)
1356 goto bad;
1357
Martin K. Petersen7ba1ba12008-06-30 20:04:41 +02001358 if (bioset_integrity_create(bs, bio_pool_size))
1359 goto bad;
1360
Jens Axboe59725112007-04-02 10:06:42 +02001361 if (!biovec_create_pools(bs, bvec_pool_size))
Linus Torvalds1da177e2005-04-16 15:20:36 -07001362 return bs;
1363
1364bad:
1365 bioset_free(bs);
1366 return NULL;
1367}
1368
1369static void __init biovec_init_slabs(void)
1370{
1371 int i;
1372
1373 for (i = 0; i < BIOVEC_NR_POOLS; i++) {
1374 int size;
1375 struct biovec_slab *bvs = bvec_slabs + i;
1376
1377 size = bvs->nr_vecs * sizeof(struct bio_vec);
1378 bvs->slab = kmem_cache_create(bvs->name, size, 0,
Paul Mundt20c2df82007-07-20 10:11:58 +09001379 SLAB_HWCACHE_ALIGN|SLAB_PANIC, NULL);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001380 }
1381}
1382
1383static int __init init_bio(void)
1384{
Christoph Lameter0a31bd52007-05-06 14:49:57 -07001385 bio_slab = KMEM_CACHE(bio, SLAB_HWCACHE_ALIGN|SLAB_PANIC);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001386
Martin K. Petersen7ba1ba12008-06-30 20:04:41 +02001387 bio_integrity_init_slab();
Linus Torvalds1da177e2005-04-16 15:20:36 -07001388 biovec_init_slabs();
1389
Jens Axboe59725112007-04-02 10:06:42 +02001390 fs_bio_set = bioset_create(BIO_POOL_SIZE, 2);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001391 if (!fs_bio_set)
1392 panic("bio: can't allocate bios\n");
1393
Matthew Dobson0eaae62a2006-03-26 01:37:47 -08001394 bio_split_pool = mempool_create_kmalloc_pool(BIO_SPLIT_ENTRIES,
1395 sizeof(struct bio_pair));
Linus Torvalds1da177e2005-04-16 15:20:36 -07001396 if (!bio_split_pool)
1397 panic("bio: can't create split pool\n");
1398
1399 return 0;
1400}
1401
1402subsys_initcall(init_bio);
1403
1404EXPORT_SYMBOL(bio_alloc);
Jens Axboe0a0d96b2008-09-11 13:17:37 +02001405EXPORT_SYMBOL(bio_kmalloc);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001406EXPORT_SYMBOL(bio_put);
Peter Osterlund36763472005-09-06 15:16:42 -07001407EXPORT_SYMBOL(bio_free);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001408EXPORT_SYMBOL(bio_endio);
1409EXPORT_SYMBOL(bio_init);
1410EXPORT_SYMBOL(__bio_clone);
1411EXPORT_SYMBOL(bio_clone);
1412EXPORT_SYMBOL(bio_phys_segments);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001413EXPORT_SYMBOL(bio_add_page);
Mike Christie6e68af62005-11-11 05:30:27 -06001414EXPORT_SYMBOL(bio_add_pc_page);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001415EXPORT_SYMBOL(bio_get_nr_vecs);
Jens Axboe40044ce2008-03-17 21:14:40 +01001416EXPORT_SYMBOL(bio_map_user);
1417EXPORT_SYMBOL(bio_unmap_user);
Mike Christie df46b9a2005-06-20 14:04:44 +02001418EXPORT_SYMBOL(bio_map_kern);
FUJITA Tomonori68154e92008-04-25 12:47:50 +02001419EXPORT_SYMBOL(bio_copy_kern);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001420EXPORT_SYMBOL(bio_pair_release);
1421EXPORT_SYMBOL(bio_split);
1422EXPORT_SYMBOL(bio_split_pool);
1423EXPORT_SYMBOL(bio_copy_user);
1424EXPORT_SYMBOL(bio_uncopy_user);
1425EXPORT_SYMBOL(bioset_create);
1426EXPORT_SYMBOL(bioset_free);
1427EXPORT_SYMBOL(bio_alloc_bioset);