blob: 3d2e9ad2472843630e83824de84ce532e4bd3763 [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
Jens Axboe0fe23472006-09-04 15:41:16 +02002 * Copyright (C) 2001 Jens Axboe <axboe@kernel.dk>
Linus Torvalds1da177e2005-04-16 15:20:36 -07003 *
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License version 2 as
6 * published by the Free Software Foundation.
7 *
8 * This program is distributed in the hope that it will be useful,
9 * but WITHOUT ANY WARRANTY; without even the implied warranty of
10 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 * GNU General Public License for more details.
12 *
13 * You should have received a copy of the GNU General Public Licens
14 * along with this program; if not, write to the Free Software
15 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-
16 *
17 */
18#include <linux/mm.h>
19#include <linux/swap.h>
20#include <linux/bio.h>
21#include <linux/blkdev.h>
22#include <linux/slab.h>
23#include <linux/init.h>
24#include <linux/kernel.h>
25#include <linux/module.h>
26#include <linux/mempool.h>
27#include <linux/workqueue.h>
Jens Axboe2056a782006-03-23 20:00:26 +010028#include <linux/blktrace_api.h>
James Bottomley f1970ba2005-06-20 14:06:52 +020029#include <scsi/sg.h> /* for struct sg_iovec */
Linus Torvalds1da177e2005-04-16 15:20:36 -070030
Christoph Lametere18b8902006-12-06 20:33:20 -080031static struct kmem_cache *bio_slab __read_mostly;
Linus Torvalds1da177e2005-04-16 15:20:36 -070032
Eric Dumazetfa3536c2006-03-26 01:37:24 -080033mempool_t *bio_split_pool __read_mostly;
Linus Torvalds1da177e2005-04-16 15:20:36 -070034
Linus Torvalds1da177e2005-04-16 15:20:36 -070035/*
36 * if you change this list, also change bvec_alloc or things will
37 * break badly! cannot be bigger than what you can fit into an
38 * unsigned short
39 */
40
41#define BV(x) { .nr_vecs = x, .name = "biovec-"__stringify(x) }
Christoph Lameter6c036522005-07-07 17:56:59 -070042static struct biovec_slab bvec_slabs[BIOVEC_NR_POOLS] __read_mostly = {
Linus Torvalds1da177e2005-04-16 15:20:36 -070043 BV(1), BV(4), BV(16), BV(64), BV(128), BV(BIO_MAX_PAGES),
44};
45#undef BV
46
47/*
Linus Torvalds1da177e2005-04-16 15:20:36 -070048 * fs_bio_set is the bio_set containing bio and iovec memory pools used by
49 * IO code that does not need private memory pools.
50 */
Martin K. Petersen51d654e2008-06-17 18:59:56 +020051struct bio_set *fs_bio_set;
Linus Torvalds1da177e2005-04-16 15:20:36 -070052
Martin K. Petersen7ba1ba12008-06-30 20:04:41 +020053unsigned int bvec_nr_vecs(unsigned short idx)
54{
55 return bvec_slabs[idx].nr_vecs;
56}
57
Martin K. Petersen51d654e2008-06-17 18:59:56 +020058struct bio_vec *bvec_alloc_bs(gfp_t gfp_mask, int nr, unsigned long *idx, struct bio_set *bs)
Linus Torvalds1da177e2005-04-16 15:20:36 -070059{
60 struct bio_vec *bvl;
Linus Torvalds1da177e2005-04-16 15:20:36 -070061
62 /*
63 * see comment near bvec_array define!
64 */
65 switch (nr) {
66 case 1 : *idx = 0; break;
67 case 2 ... 4: *idx = 1; break;
68 case 5 ... 16: *idx = 2; break;
69 case 17 ... 64: *idx = 3; break;
70 case 65 ... 128: *idx = 4; break;
71 case 129 ... BIO_MAX_PAGES: *idx = 5; break;
72 default:
73 return NULL;
74 }
75 /*
76 * idx now points to the pool we want to allocate from
77 */
78
Linus Torvalds1da177e2005-04-16 15:20:36 -070079 bvl = mempool_alloc(bs->bvec_pools[*idx], gfp_mask);
Denis ChengRq1ac0ae02008-08-04 11:56:30 +020080 if (bvl)
81 memset(bvl, 0, bvec_nr_vecs(*idx) * sizeof(struct bio_vec));
Linus Torvalds1da177e2005-04-16 15:20:36 -070082
83 return bvl;
84}
85
Peter Osterlund36763472005-09-06 15:16:42 -070086void bio_free(struct bio *bio, struct bio_set *bio_set)
Linus Torvalds1da177e2005-04-16 15:20:36 -070087{
Jens Axboe992c5dd2007-07-18 13:18:08 +020088 if (bio->bi_io_vec) {
89 const int pool_idx = BIO_POOL_IDX(bio);
Linus Torvalds1da177e2005-04-16 15:20:36 -070090
Jens Axboe992c5dd2007-07-18 13:18:08 +020091 BIO_BUG_ON(pool_idx >= BIOVEC_NR_POOLS);
Linus Torvalds1da177e2005-04-16 15:20:36 -070092
Jens Axboe992c5dd2007-07-18 13:18:08 +020093 mempool_free(bio->bi_io_vec, bio_set->bvec_pools[pool_idx]);
94 }
95
Martin K. Petersen7ba1ba12008-06-30 20:04:41 +020096 if (bio_integrity(bio))
97 bio_integrity_free(bio, bio_set);
98
Peter Osterlund36763472005-09-06 15:16:42 -070099 mempool_free(bio, bio_set->bio_pool);
100}
101
102/*
103 * default destructor for a bio allocated with bio_alloc_bioset()
104 */
105static void bio_fs_destructor(struct bio *bio)
106{
107 bio_free(bio, fs_bio_set);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700108}
109
Arjan van de Ven858119e2006-01-14 13:20:43 -0800110void bio_init(struct bio *bio)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700111{
Jens Axboe2b94de52007-07-18 13:14:03 +0200112 memset(bio, 0, sizeof(*bio));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700113 bio->bi_flags = 1 << BIO_UPTODATE;
Jens Axboec7c22e42008-09-13 20:26:01 +0200114 bio->bi_comp_cpu = -1;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700115 atomic_set(&bio->bi_cnt, 1);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700116}
117
118/**
119 * bio_alloc_bioset - allocate a bio for I/O
120 * @gfp_mask: the GFP_ mask given to the slab allocator
121 * @nr_iovecs: number of iovecs to pre-allocate
Martin Waitz67be2dd2005-05-01 08:59:26 -0700122 * @bs: the bio_set to allocate from
Linus Torvalds1da177e2005-04-16 15:20:36 -0700123 *
124 * Description:
125 * bio_alloc_bioset will first try it's on mempool to satisfy the allocation.
126 * If %__GFP_WAIT is set then we will block on the internal pool waiting
127 * for a &struct bio to become free.
128 *
129 * allocate bio and iovecs from the memory pools specified by the
130 * bio_set structure.
131 **/
Al Virodd0fc662005-10-07 07:46:04 +0100132struct bio *bio_alloc_bioset(gfp_t gfp_mask, int nr_iovecs, struct bio_set *bs)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700133{
134 struct bio *bio = mempool_alloc(bs->bio_pool, gfp_mask);
135
136 if (likely(bio)) {
137 struct bio_vec *bvl = NULL;
138
139 bio_init(bio);
140 if (likely(nr_iovecs)) {
Jens Axboeeeae1d42008-05-07 13:26:27 +0200141 unsigned long uninitialized_var(idx);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700142
143 bvl = bvec_alloc_bs(gfp_mask, nr_iovecs, &idx, bs);
144 if (unlikely(!bvl)) {
145 mempool_free(bio, bs->bio_pool);
146 bio = NULL;
147 goto out;
148 }
149 bio->bi_flags |= idx << BIO_POOL_OFFSET;
Denis ChengRq1ac0ae02008-08-04 11:56:30 +0200150 bio->bi_max_vecs = bvec_nr_vecs(idx);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700151 }
152 bio->bi_io_vec = bvl;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700153 }
154out:
155 return bio;
156}
157
Al Virodd0fc662005-10-07 07:46:04 +0100158struct bio *bio_alloc(gfp_t gfp_mask, int nr_iovecs)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700159{
Peter Osterlund36763472005-09-06 15:16:42 -0700160 struct bio *bio = bio_alloc_bioset(gfp_mask, nr_iovecs, fs_bio_set);
161
162 if (bio)
163 bio->bi_destructor = bio_fs_destructor;
164
165 return bio;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700166}
167
168void zero_fill_bio(struct bio *bio)
169{
170 unsigned long flags;
171 struct bio_vec *bv;
172 int i;
173
174 bio_for_each_segment(bv, bio, i) {
175 char *data = bvec_kmap_irq(bv, &flags);
176 memset(data, 0, bv->bv_len);
177 flush_dcache_page(bv->bv_page);
178 bvec_kunmap_irq(data, &flags);
179 }
180}
181EXPORT_SYMBOL(zero_fill_bio);
182
183/**
184 * bio_put - release a reference to a bio
185 * @bio: bio to release reference to
186 *
187 * Description:
188 * Put a reference to a &struct bio, either one you have gotten with
189 * bio_alloc or bio_get. The last put of a bio will free it.
190 **/
191void bio_put(struct bio *bio)
192{
193 BIO_BUG_ON(!atomic_read(&bio->bi_cnt));
194
195 /*
196 * last put frees it
197 */
198 if (atomic_dec_and_test(&bio->bi_cnt)) {
199 bio->bi_next = NULL;
200 bio->bi_destructor(bio);
201 }
202}
203
Jens Axboe165125e2007-07-24 09:28:11 +0200204inline int bio_phys_segments(struct request_queue *q, struct bio *bio)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700205{
206 if (unlikely(!bio_flagged(bio, BIO_SEG_VALID)))
207 blk_recount_segments(q, bio);
208
209 return bio->bi_phys_segments;
210}
211
Linus Torvalds1da177e2005-04-16 15:20:36 -0700212/**
213 * __bio_clone - clone a bio
214 * @bio: destination bio
215 * @bio_src: bio to clone
216 *
217 * Clone a &bio. Caller will own the returned bio, but not
218 * the actual data it points to. Reference count of returned
219 * bio will be one.
220 */
Arjan van de Ven858119e2006-01-14 13:20:43 -0800221void __bio_clone(struct bio *bio, struct bio *bio_src)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700222{
Andrew Mortone525e152005-08-07 09:42:12 -0700223 memcpy(bio->bi_io_vec, bio_src->bi_io_vec,
224 bio_src->bi_max_vecs * sizeof(struct bio_vec));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700225
Jens Axboe5d840702008-01-25 12:44:44 +0100226 /*
227 * most users will be overriding ->bi_bdev with a new target,
228 * so we don't set nor calculate new physical/hw segment counts here
229 */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700230 bio->bi_sector = bio_src->bi_sector;
231 bio->bi_bdev = bio_src->bi_bdev;
232 bio->bi_flags |= 1 << BIO_CLONED;
233 bio->bi_rw = bio_src->bi_rw;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700234 bio->bi_vcnt = bio_src->bi_vcnt;
235 bio->bi_size = bio_src->bi_size;
Andrew Mortona5453be2005-07-28 01:07:18 -0700236 bio->bi_idx = bio_src->bi_idx;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700237}
238
239/**
240 * bio_clone - clone a bio
241 * @bio: bio to clone
242 * @gfp_mask: allocation priority
243 *
244 * Like __bio_clone, only also allocates the returned bio
245 */
Al Virodd0fc662005-10-07 07:46:04 +0100246struct bio *bio_clone(struct bio *bio, gfp_t gfp_mask)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700247{
248 struct bio *b = bio_alloc_bioset(gfp_mask, bio->bi_max_vecs, fs_bio_set);
249
Martin K. Petersen7ba1ba12008-06-30 20:04:41 +0200250 if (!b)
251 return NULL;
252
253 b->bi_destructor = bio_fs_destructor;
254 __bio_clone(b, bio);
255
256 if (bio_integrity(bio)) {
257 int ret;
258
259 ret = bio_integrity_clone(b, bio, fs_bio_set);
260
261 if (ret < 0)
262 return NULL;
Peter Osterlund36763472005-09-06 15:16:42 -0700263 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700264
265 return b;
266}
267
268/**
269 * bio_get_nr_vecs - return approx number of vecs
270 * @bdev: I/O target
271 *
272 * Return the approximate number of pages we can send to this target.
273 * There's no guarantee that you will be able to fit this number of pages
274 * into a bio, it does not account for dynamic restrictions that vary
275 * on offset.
276 */
277int bio_get_nr_vecs(struct block_device *bdev)
278{
Jens Axboe165125e2007-07-24 09:28:11 +0200279 struct request_queue *q = bdev_get_queue(bdev);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700280 int nr_pages;
281
282 nr_pages = ((q->max_sectors << 9) + PAGE_SIZE - 1) >> PAGE_SHIFT;
283 if (nr_pages > q->max_phys_segments)
284 nr_pages = q->max_phys_segments;
285 if (nr_pages > q->max_hw_segments)
286 nr_pages = q->max_hw_segments;
287
288 return nr_pages;
289}
290
Jens Axboe165125e2007-07-24 09:28:11 +0200291static int __bio_add_page(struct request_queue *q, struct bio *bio, struct page
Mike Christiedefd94b2005-12-05 02:37:06 -0600292 *page, unsigned int len, unsigned int offset,
293 unsigned short max_sectors)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700294{
295 int retried_segments = 0;
296 struct bio_vec *bvec;
297
298 /*
299 * cloned bio must not modify vec list
300 */
301 if (unlikely(bio_flagged(bio, BIO_CLONED)))
302 return 0;
303
Jens Axboe80cfd542006-01-06 09:43:28 +0100304 if (((bio->bi_size + len) >> 9) > max_sectors)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700305 return 0;
306
Jens Axboe80cfd542006-01-06 09:43:28 +0100307 /*
308 * For filesystems with a blocksize smaller than the pagesize
309 * we will often be called with the same page as last time and
310 * a consecutive offset. Optimize this special case.
311 */
312 if (bio->bi_vcnt > 0) {
313 struct bio_vec *prev = &bio->bi_io_vec[bio->bi_vcnt - 1];
314
315 if (page == prev->bv_page &&
316 offset == prev->bv_offset + prev->bv_len) {
317 prev->bv_len += len;
Alasdair G Kergoncc371e62008-07-03 09:53:43 +0200318
319 if (q->merge_bvec_fn) {
320 struct bvec_merge_data bvm = {
321 .bi_bdev = bio->bi_bdev,
322 .bi_sector = bio->bi_sector,
323 .bi_size = bio->bi_size,
324 .bi_rw = bio->bi_rw,
325 };
326
327 if (q->merge_bvec_fn(q, &bvm, prev) < len) {
328 prev->bv_len -= len;
329 return 0;
330 }
Jens Axboe80cfd542006-01-06 09:43:28 +0100331 }
332
333 goto done;
334 }
335 }
336
337 if (bio->bi_vcnt >= bio->bi_max_vecs)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700338 return 0;
339
340 /*
341 * we might lose a segment or two here, but rather that than
342 * make this too complex.
343 */
344
345 while (bio->bi_phys_segments >= q->max_phys_segments
Mikulas Patocka5df97b92008-08-15 10:20:02 +0200346 || bio->bi_phys_segments >= q->max_hw_segments) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700347
348 if (retried_segments)
349 return 0;
350
351 retried_segments = 1;
352 blk_recount_segments(q, bio);
353 }
354
355 /*
356 * setup the new entry, we might clear it again later if we
357 * cannot add the page
358 */
359 bvec = &bio->bi_io_vec[bio->bi_vcnt];
360 bvec->bv_page = page;
361 bvec->bv_len = len;
362 bvec->bv_offset = offset;
363
364 /*
365 * if queue has other restrictions (eg varying max sector size
366 * depending on offset), it can specify a merge_bvec_fn in the
367 * queue to get further control
368 */
369 if (q->merge_bvec_fn) {
Alasdair G Kergoncc371e62008-07-03 09:53:43 +0200370 struct bvec_merge_data bvm = {
371 .bi_bdev = bio->bi_bdev,
372 .bi_sector = bio->bi_sector,
373 .bi_size = bio->bi_size,
374 .bi_rw = bio->bi_rw,
375 };
376
Linus Torvalds1da177e2005-04-16 15:20:36 -0700377 /*
378 * merge_bvec_fn() returns number of bytes it can accept
379 * at this offset
380 */
Alasdair G Kergoncc371e62008-07-03 09:53:43 +0200381 if (q->merge_bvec_fn(q, &bvm, bvec) < len) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700382 bvec->bv_page = NULL;
383 bvec->bv_len = 0;
384 bvec->bv_offset = 0;
385 return 0;
386 }
387 }
388
389 /* If we may be able to merge these biovecs, force a recount */
Mikulas Patockab8b3e162008-08-15 10:15:19 +0200390 if (bio->bi_vcnt && (BIOVEC_PHYS_MERGEABLE(bvec-1, bvec)))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700391 bio->bi_flags &= ~(1 << BIO_SEG_VALID);
392
393 bio->bi_vcnt++;
394 bio->bi_phys_segments++;
Jens Axboe80cfd542006-01-06 09:43:28 +0100395 done:
Linus Torvalds1da177e2005-04-16 15:20:36 -0700396 bio->bi_size += len;
397 return len;
398}
399
400/**
Mike Christie6e68af62005-11-11 05:30:27 -0600401 * bio_add_pc_page - attempt to add page to bio
Jens Axboefddfdea2006-01-31 15:24:34 +0100402 * @q: the target queue
Mike Christie6e68af62005-11-11 05:30:27 -0600403 * @bio: destination bio
404 * @page: page to add
405 * @len: vec entry length
406 * @offset: vec entry offset
407 *
408 * Attempt to add a page to the bio_vec maplist. This can fail for a
409 * number of reasons, such as the bio being full or target block
410 * device limitations. The target block device must allow bio's
411 * smaller than PAGE_SIZE, so it is always possible to add a single
412 * page to an empty bio. This should only be used by REQ_PC bios.
413 */
Jens Axboe165125e2007-07-24 09:28:11 +0200414int bio_add_pc_page(struct request_queue *q, struct bio *bio, struct page *page,
Mike Christie6e68af62005-11-11 05:30:27 -0600415 unsigned int len, unsigned int offset)
416{
Mike Christiedefd94b2005-12-05 02:37:06 -0600417 return __bio_add_page(q, bio, page, len, offset, q->max_hw_sectors);
Mike Christie6e68af62005-11-11 05:30:27 -0600418}
419
420/**
Linus Torvalds1da177e2005-04-16 15:20:36 -0700421 * bio_add_page - attempt to add page to bio
422 * @bio: destination bio
423 * @page: page to add
424 * @len: vec entry length
425 * @offset: vec entry offset
426 *
427 * Attempt to add a page to the bio_vec maplist. This can fail for a
428 * number of reasons, such as the bio being full or target block
429 * device limitations. The target block device must allow bio's
430 * smaller than PAGE_SIZE, so it is always possible to add a single
431 * page to an empty bio.
432 */
433int bio_add_page(struct bio *bio, struct page *page, unsigned int len,
434 unsigned int offset)
435{
Mike Christiedefd94b2005-12-05 02:37:06 -0600436 struct request_queue *q = bdev_get_queue(bio->bi_bdev);
437 return __bio_add_page(q, bio, page, len, offset, q->max_sectors);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700438}
439
440struct bio_map_data {
441 struct bio_vec *iovecs;
FUJITA Tomonoric5dec1c2008-04-11 12:56:49 +0200442 int nr_sgvecs;
443 struct sg_iovec *sgvecs;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700444};
445
FUJITA Tomonoric5dec1c2008-04-11 12:56:49 +0200446static void bio_set_map_data(struct bio_map_data *bmd, struct bio *bio,
447 struct sg_iovec *iov, int iov_count)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700448{
449 memcpy(bmd->iovecs, bio->bi_io_vec, sizeof(struct bio_vec) * bio->bi_vcnt);
FUJITA Tomonoric5dec1c2008-04-11 12:56:49 +0200450 memcpy(bmd->sgvecs, iov, sizeof(struct sg_iovec) * iov_count);
451 bmd->nr_sgvecs = iov_count;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700452 bio->bi_private = bmd;
453}
454
455static void bio_free_map_data(struct bio_map_data *bmd)
456{
457 kfree(bmd->iovecs);
FUJITA Tomonoric5dec1c2008-04-11 12:56:49 +0200458 kfree(bmd->sgvecs);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700459 kfree(bmd);
460}
461
FUJITA Tomonori76029ff2008-08-25 20:36:08 +0200462static struct bio_map_data *bio_alloc_map_data(int nr_segs, int iov_count,
463 gfp_t gfp_mask)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700464{
FUJITA Tomonori76029ff2008-08-25 20:36:08 +0200465 struct bio_map_data *bmd = kmalloc(sizeof(*bmd), gfp_mask);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700466
467 if (!bmd)
468 return NULL;
469
FUJITA Tomonori76029ff2008-08-25 20:36:08 +0200470 bmd->iovecs = kmalloc(sizeof(struct bio_vec) * nr_segs, gfp_mask);
FUJITA Tomonoric5dec1c2008-04-11 12:56:49 +0200471 if (!bmd->iovecs) {
472 kfree(bmd);
473 return NULL;
474 }
475
FUJITA Tomonori76029ff2008-08-25 20:36:08 +0200476 bmd->sgvecs = kmalloc(sizeof(struct sg_iovec) * iov_count, gfp_mask);
FUJITA Tomonoric5dec1c2008-04-11 12:56:49 +0200477 if (bmd->sgvecs)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700478 return bmd;
479
FUJITA Tomonoric5dec1c2008-04-11 12:56:49 +0200480 kfree(bmd->iovecs);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700481 kfree(bmd);
482 return NULL;
483}
484
FUJITA Tomonoriaefcc282008-08-25 20:36:08 +0200485static int __bio_copy_iov(struct bio *bio, struct bio_vec *iovecs,
486 struct sg_iovec *iov, int iov_count, int uncopy)
FUJITA Tomonoric5dec1c2008-04-11 12:56:49 +0200487{
488 int ret = 0, i;
489 struct bio_vec *bvec;
490 int iov_idx = 0;
491 unsigned int iov_off = 0;
492 int read = bio_data_dir(bio) == READ;
493
494 __bio_for_each_segment(bvec, bio, i, 0) {
495 char *bv_addr = page_address(bvec->bv_page);
FUJITA Tomonoriaefcc282008-08-25 20:36:08 +0200496 unsigned int bv_len = iovecs[i].bv_len;
FUJITA Tomonoric5dec1c2008-04-11 12:56:49 +0200497
498 while (bv_len && iov_idx < iov_count) {
499 unsigned int bytes;
500 char *iov_addr;
501
502 bytes = min_t(unsigned int,
503 iov[iov_idx].iov_len - iov_off, bv_len);
504 iov_addr = iov[iov_idx].iov_base + iov_off;
505
506 if (!ret) {
507 if (!read && !uncopy)
508 ret = copy_from_user(bv_addr, iov_addr,
509 bytes);
510 if (read && uncopy)
511 ret = copy_to_user(iov_addr, bv_addr,
512 bytes);
513
514 if (ret)
515 ret = -EFAULT;
516 }
517
518 bv_len -= bytes;
519 bv_addr += bytes;
520 iov_addr += bytes;
521 iov_off += bytes;
522
523 if (iov[iov_idx].iov_len == iov_off) {
524 iov_idx++;
525 iov_off = 0;
526 }
527 }
528
529 if (uncopy)
530 __free_page(bvec->bv_page);
531 }
532
533 return ret;
534}
535
Linus Torvalds1da177e2005-04-16 15:20:36 -0700536/**
537 * bio_uncopy_user - finish previously mapped bio
538 * @bio: bio being terminated
539 *
540 * Free pages allocated from bio_copy_user() and write back data
541 * to user space in case of a read.
542 */
543int bio_uncopy_user(struct bio *bio)
544{
545 struct bio_map_data *bmd = bio->bi_private;
FUJITA Tomonoric5dec1c2008-04-11 12:56:49 +0200546 int ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700547
FUJITA Tomonoriaefcc282008-08-25 20:36:08 +0200548 ret = __bio_copy_iov(bio, bmd->iovecs, bmd->sgvecs, bmd->nr_sgvecs, 1);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700549
Linus Torvalds1da177e2005-04-16 15:20:36 -0700550 bio_free_map_data(bmd);
551 bio_put(bio);
552 return ret;
553}
554
555/**
FUJITA Tomonoric5dec1c2008-04-11 12:56:49 +0200556 * bio_copy_user_iov - copy user data to bio
Linus Torvalds1da177e2005-04-16 15:20:36 -0700557 * @q: destination block queue
FUJITA Tomonoric5dec1c2008-04-11 12:56:49 +0200558 * @iov: the iovec.
559 * @iov_count: number of elements in the iovec
Linus Torvalds1da177e2005-04-16 15:20:36 -0700560 * @write_to_vm: bool indicating writing to pages or not
FUJITA Tomonoria3bce902008-08-28 16:17:05 +0900561 * @gfp_mask: memory allocation flags
Linus Torvalds1da177e2005-04-16 15:20:36 -0700562 *
563 * Prepares and returns a bio for indirect user io, bouncing data
564 * to/from kernel pages as necessary. Must be paired with
565 * call bio_uncopy_user() on io completion.
566 */
FUJITA Tomonoric5dec1c2008-04-11 12:56:49 +0200567struct bio *bio_copy_user_iov(struct request_queue *q, struct sg_iovec *iov,
FUJITA Tomonoria3bce902008-08-28 16:17:05 +0900568 int iov_count, int write_to_vm, gfp_t gfp_mask)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700569{
Linus Torvalds1da177e2005-04-16 15:20:36 -0700570 struct bio_map_data *bmd;
571 struct bio_vec *bvec;
572 struct page *page;
573 struct bio *bio;
574 int i, ret;
FUJITA Tomonoric5dec1c2008-04-11 12:56:49 +0200575 int nr_pages = 0;
576 unsigned int len = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700577
FUJITA Tomonoric5dec1c2008-04-11 12:56:49 +0200578 for (i = 0; i < iov_count; i++) {
579 unsigned long uaddr;
580 unsigned long end;
581 unsigned long start;
582
583 uaddr = (unsigned long)iov[i].iov_base;
584 end = (uaddr + iov[i].iov_len + PAGE_SIZE - 1) >> PAGE_SHIFT;
585 start = uaddr >> PAGE_SHIFT;
586
587 nr_pages += end - start;
588 len += iov[i].iov_len;
589 }
590
FUJITA Tomonoria3bce902008-08-28 16:17:05 +0900591 bmd = bio_alloc_map_data(nr_pages, iov_count, gfp_mask);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700592 if (!bmd)
593 return ERR_PTR(-ENOMEM);
594
Linus Torvalds1da177e2005-04-16 15:20:36 -0700595 ret = -ENOMEM;
FUJITA Tomonoria3bce902008-08-28 16:17:05 +0900596 bio = bio_alloc(gfp_mask, nr_pages);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700597 if (!bio)
598 goto out_bmd;
599
600 bio->bi_rw |= (!write_to_vm << BIO_RW);
601
602 ret = 0;
603 while (len) {
604 unsigned int bytes = PAGE_SIZE;
605
606 if (bytes > len)
607 bytes = len;
608
FUJITA Tomonoria3bce902008-08-28 16:17:05 +0900609 page = alloc_page(q->bounce_gfp | gfp_mask);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700610 if (!page) {
611 ret = -ENOMEM;
612 break;
613 }
614
Mike Christie0e75f902006-12-01 10:40:55 +0100615 if (bio_add_pc_page(q, bio, page, bytes, 0) < bytes)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700616 break;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700617
618 len -= bytes;
619 }
620
621 if (ret)
622 goto cleanup;
623
624 /*
625 * success
626 */
627 if (!write_to_vm) {
FUJITA Tomonoriaefcc282008-08-25 20:36:08 +0200628 ret = __bio_copy_iov(bio, bio->bi_io_vec, iov, iov_count, 0);
FUJITA Tomonoric5dec1c2008-04-11 12:56:49 +0200629 if (ret)
630 goto cleanup;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700631 }
632
FUJITA Tomonoric5dec1c2008-04-11 12:56:49 +0200633 bio_set_map_data(bmd, bio, iov, iov_count);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700634 return bio;
635cleanup:
636 bio_for_each_segment(bvec, bio, i)
637 __free_page(bvec->bv_page);
638
639 bio_put(bio);
640out_bmd:
641 bio_free_map_data(bmd);
642 return ERR_PTR(ret);
643}
644
FUJITA Tomonoric5dec1c2008-04-11 12:56:49 +0200645/**
646 * bio_copy_user - copy user data to bio
647 * @q: destination block queue
648 * @uaddr: start of user address
649 * @len: length in bytes
650 * @write_to_vm: bool indicating writing to pages or not
FUJITA Tomonoria3bce902008-08-28 16:17:05 +0900651 * @gfp_mask: memory allocation flags
FUJITA Tomonoric5dec1c2008-04-11 12:56:49 +0200652 *
653 * Prepares and returns a bio for indirect user io, bouncing data
654 * to/from kernel pages as necessary. Must be paired with
655 * call bio_uncopy_user() on io completion.
656 */
657struct bio *bio_copy_user(struct request_queue *q, unsigned long uaddr,
FUJITA Tomonoria3bce902008-08-28 16:17:05 +0900658 unsigned int len, int write_to_vm, gfp_t gfp_mask)
FUJITA Tomonoric5dec1c2008-04-11 12:56:49 +0200659{
660 struct sg_iovec iov;
661
662 iov.iov_base = (void __user *)uaddr;
663 iov.iov_len = len;
664
FUJITA Tomonoria3bce902008-08-28 16:17:05 +0900665 return bio_copy_user_iov(q, &iov, 1, write_to_vm, gfp_mask);
FUJITA Tomonoric5dec1c2008-04-11 12:56:49 +0200666}
667
Jens Axboe165125e2007-07-24 09:28:11 +0200668static struct bio *__bio_map_user_iov(struct request_queue *q,
James Bottomley f1970ba2005-06-20 14:06:52 +0200669 struct block_device *bdev,
670 struct sg_iovec *iov, int iov_count,
FUJITA Tomonoria3bce902008-08-28 16:17:05 +0900671 int write_to_vm, gfp_t gfp_mask)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700672{
James Bottomley f1970ba2005-06-20 14:06:52 +0200673 int i, j;
674 int nr_pages = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700675 struct page **pages;
676 struct bio *bio;
James Bottomley f1970ba2005-06-20 14:06:52 +0200677 int cur_page = 0;
678 int ret, offset;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700679
James Bottomley f1970ba2005-06-20 14:06:52 +0200680 for (i = 0; i < iov_count; i++) {
681 unsigned long uaddr = (unsigned long)iov[i].iov_base;
682 unsigned long len = iov[i].iov_len;
683 unsigned long end = (uaddr + len + PAGE_SIZE - 1) >> PAGE_SHIFT;
684 unsigned long start = uaddr >> PAGE_SHIFT;
685
686 nr_pages += end - start;
687 /*
Mike Christiead2d7222006-12-01 10:40:20 +0100688 * buffer must be aligned to at least hardsector size for now
James Bottomley f1970ba2005-06-20 14:06:52 +0200689 */
Mike Christiead2d7222006-12-01 10:40:20 +0100690 if (uaddr & queue_dma_alignment(q))
James Bottomley f1970ba2005-06-20 14:06:52 +0200691 return ERR_PTR(-EINVAL);
692 }
693
694 if (!nr_pages)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700695 return ERR_PTR(-EINVAL);
696
FUJITA Tomonoria3bce902008-08-28 16:17:05 +0900697 bio = bio_alloc(gfp_mask, nr_pages);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700698 if (!bio)
699 return ERR_PTR(-ENOMEM);
700
701 ret = -ENOMEM;
FUJITA Tomonoria3bce902008-08-28 16:17:05 +0900702 pages = kcalloc(nr_pages, sizeof(struct page *), gfp_mask);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700703 if (!pages)
704 goto out;
705
James Bottomley f1970ba2005-06-20 14:06:52 +0200706 for (i = 0; i < iov_count; i++) {
707 unsigned long uaddr = (unsigned long)iov[i].iov_base;
708 unsigned long len = iov[i].iov_len;
709 unsigned long end = (uaddr + len + PAGE_SIZE - 1) >> PAGE_SHIFT;
710 unsigned long start = uaddr >> PAGE_SHIFT;
711 const int local_nr_pages = end - start;
712 const int page_limit = cur_page + local_nr_pages;
713
Nick Pigginf5dd33c2008-07-25 19:45:25 -0700714 ret = get_user_pages_fast(uaddr, local_nr_pages,
715 write_to_vm, &pages[cur_page]);
Jens Axboe99172152006-06-16 13:02:29 +0200716 if (ret < local_nr_pages) {
717 ret = -EFAULT;
James Bottomley f1970ba2005-06-20 14:06:52 +0200718 goto out_unmap;
Jens Axboe99172152006-06-16 13:02:29 +0200719 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700720
James Bottomley f1970ba2005-06-20 14:06:52 +0200721 offset = uaddr & ~PAGE_MASK;
722 for (j = cur_page; j < page_limit; j++) {
723 unsigned int bytes = PAGE_SIZE - offset;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700724
James Bottomley f1970ba2005-06-20 14:06:52 +0200725 if (len <= 0)
726 break;
727
728 if (bytes > len)
729 bytes = len;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700730
James Bottomley f1970ba2005-06-20 14:06:52 +0200731 /*
732 * sorry...
733 */
Mike Christiedefd94b2005-12-05 02:37:06 -0600734 if (bio_add_pc_page(q, bio, pages[j], bytes, offset) <
735 bytes)
James Bottomley f1970ba2005-06-20 14:06:52 +0200736 break;
737
738 len -= bytes;
739 offset = 0;
740 }
741
742 cur_page = j;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700743 /*
James Bottomley f1970ba2005-06-20 14:06:52 +0200744 * release the pages we didn't map into the bio, if any
Linus Torvalds1da177e2005-04-16 15:20:36 -0700745 */
James Bottomley f1970ba2005-06-20 14:06:52 +0200746 while (j < page_limit)
747 page_cache_release(pages[j++]);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700748 }
749
Linus Torvalds1da177e2005-04-16 15:20:36 -0700750 kfree(pages);
751
752 /*
753 * set data direction, and check if mapped pages need bouncing
754 */
755 if (!write_to_vm)
756 bio->bi_rw |= (1 << BIO_RW);
757
James Bottomley f1970ba2005-06-20 14:06:52 +0200758 bio->bi_bdev = bdev;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700759 bio->bi_flags |= (1 << BIO_USER_MAPPED);
760 return bio;
James Bottomley f1970ba2005-06-20 14:06:52 +0200761
762 out_unmap:
763 for (i = 0; i < nr_pages; i++) {
764 if(!pages[i])
765 break;
766 page_cache_release(pages[i]);
767 }
768 out:
Linus Torvalds1da177e2005-04-16 15:20:36 -0700769 kfree(pages);
770 bio_put(bio);
771 return ERR_PTR(ret);
772}
773
774/**
775 * bio_map_user - map user address into bio
Jens Axboe165125e2007-07-24 09:28:11 +0200776 * @q: the struct request_queue for the bio
Linus Torvalds1da177e2005-04-16 15:20:36 -0700777 * @bdev: destination block device
778 * @uaddr: start of user address
779 * @len: length in bytes
780 * @write_to_vm: bool indicating writing to pages or not
FUJITA Tomonoria3bce902008-08-28 16:17:05 +0900781 * @gfp_mask: memory allocation flags
Linus Torvalds1da177e2005-04-16 15:20:36 -0700782 *
783 * Map the user space address into a bio suitable for io to a block
784 * device. Returns an error pointer in case of error.
785 */
Jens Axboe165125e2007-07-24 09:28:11 +0200786struct bio *bio_map_user(struct request_queue *q, struct block_device *bdev,
FUJITA Tomonoria3bce902008-08-28 16:17:05 +0900787 unsigned long uaddr, unsigned int len, int write_to_vm,
788 gfp_t gfp_mask)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700789{
James Bottomley f1970ba2005-06-20 14:06:52 +0200790 struct sg_iovec iov;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700791
viro@ZenIV.linux.org.uk3f703532005-09-09 16:53:56 +0100792 iov.iov_base = (void __user *)uaddr;
James Bottomley f1970ba2005-06-20 14:06:52 +0200793 iov.iov_len = len;
794
FUJITA Tomonoria3bce902008-08-28 16:17:05 +0900795 return bio_map_user_iov(q, bdev, &iov, 1, write_to_vm, gfp_mask);
James Bottomley f1970ba2005-06-20 14:06:52 +0200796}
797
798/**
799 * bio_map_user_iov - map user sg_iovec table into bio
Jens Axboe165125e2007-07-24 09:28:11 +0200800 * @q: the struct request_queue for the bio
James Bottomley f1970ba2005-06-20 14:06:52 +0200801 * @bdev: destination block device
802 * @iov: the iovec.
803 * @iov_count: number of elements in the iovec
804 * @write_to_vm: bool indicating writing to pages or not
FUJITA Tomonoria3bce902008-08-28 16:17:05 +0900805 * @gfp_mask: memory allocation flags
James Bottomley f1970ba2005-06-20 14:06:52 +0200806 *
807 * Map the user space address into a bio suitable for io to a block
808 * device. Returns an error pointer in case of error.
809 */
Jens Axboe165125e2007-07-24 09:28:11 +0200810struct bio *bio_map_user_iov(struct request_queue *q, struct block_device *bdev,
James Bottomley f1970ba2005-06-20 14:06:52 +0200811 struct sg_iovec *iov, int iov_count,
FUJITA Tomonoria3bce902008-08-28 16:17:05 +0900812 int write_to_vm, gfp_t gfp_mask)
James Bottomley f1970ba2005-06-20 14:06:52 +0200813{
814 struct bio *bio;
James Bottomley f1970ba2005-06-20 14:06:52 +0200815
FUJITA Tomonoria3bce902008-08-28 16:17:05 +0900816 bio = __bio_map_user_iov(q, bdev, iov, iov_count, write_to_vm,
817 gfp_mask);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700818 if (IS_ERR(bio))
819 return bio;
820
821 /*
822 * subtle -- if __bio_map_user() ended up bouncing a bio,
823 * it would normally disappear when its bi_end_io is run.
824 * however, we need it for the unmap, so grab an extra
825 * reference to it
826 */
827 bio_get(bio);
828
Mike Christie0e75f902006-12-01 10:40:55 +0100829 return bio;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700830}
831
832static void __bio_unmap_user(struct bio *bio)
833{
834 struct bio_vec *bvec;
835 int i;
836
837 /*
838 * make sure we dirty pages we wrote to
839 */
840 __bio_for_each_segment(bvec, bio, i, 0) {
841 if (bio_data_dir(bio) == READ)
842 set_page_dirty_lock(bvec->bv_page);
843
844 page_cache_release(bvec->bv_page);
845 }
846
847 bio_put(bio);
848}
849
850/**
851 * bio_unmap_user - unmap a bio
852 * @bio: the bio being unmapped
853 *
854 * Unmap a bio previously mapped by bio_map_user(). Must be called with
855 * a process context.
856 *
857 * bio_unmap_user() may sleep.
858 */
859void bio_unmap_user(struct bio *bio)
860{
861 __bio_unmap_user(bio);
862 bio_put(bio);
863}
864
NeilBrown6712ecf2007-09-27 12:47:43 +0200865static void bio_map_kern_endio(struct bio *bio, int err)
Jens Axboeb8238252005-06-20 14:05:27 +0200866{
Jens Axboeb8238252005-06-20 14:05:27 +0200867 bio_put(bio);
Jens Axboeb8238252005-06-20 14:05:27 +0200868}
869
870
Jens Axboe165125e2007-07-24 09:28:11 +0200871static struct bio *__bio_map_kern(struct request_queue *q, void *data,
Al Viro27496a82005-10-21 03:20:48 -0400872 unsigned int len, gfp_t gfp_mask)
Mike Christie df46b9a2005-06-20 14:04:44 +0200873{
874 unsigned long kaddr = (unsigned long)data;
875 unsigned long end = (kaddr + len + PAGE_SIZE - 1) >> PAGE_SHIFT;
876 unsigned long start = kaddr >> PAGE_SHIFT;
877 const int nr_pages = end - start;
878 int offset, i;
879 struct bio *bio;
880
881 bio = bio_alloc(gfp_mask, nr_pages);
882 if (!bio)
883 return ERR_PTR(-ENOMEM);
884
885 offset = offset_in_page(kaddr);
886 for (i = 0; i < nr_pages; i++) {
887 unsigned int bytes = PAGE_SIZE - offset;
888
889 if (len <= 0)
890 break;
891
892 if (bytes > len)
893 bytes = len;
894
Mike Christiedefd94b2005-12-05 02:37:06 -0600895 if (bio_add_pc_page(q, bio, virt_to_page(data), bytes,
896 offset) < bytes)
Mike Christie df46b9a2005-06-20 14:04:44 +0200897 break;
898
899 data += bytes;
900 len -= bytes;
901 offset = 0;
902 }
903
Jens Axboeb8238252005-06-20 14:05:27 +0200904 bio->bi_end_io = bio_map_kern_endio;
Mike Christie df46b9a2005-06-20 14:04:44 +0200905 return bio;
906}
907
908/**
909 * bio_map_kern - map kernel address into bio
Jens Axboe165125e2007-07-24 09:28:11 +0200910 * @q: the struct request_queue for the bio
Mike Christie df46b9a2005-06-20 14:04:44 +0200911 * @data: pointer to buffer to map
912 * @len: length in bytes
913 * @gfp_mask: allocation flags for bio allocation
914 *
915 * Map the kernel address into a bio suitable for io to a block
916 * device. Returns an error pointer in case of error.
917 */
Jens Axboe165125e2007-07-24 09:28:11 +0200918struct bio *bio_map_kern(struct request_queue *q, void *data, unsigned int len,
Al Viro27496a82005-10-21 03:20:48 -0400919 gfp_t gfp_mask)
Mike Christie df46b9a2005-06-20 14:04:44 +0200920{
921 struct bio *bio;
922
923 bio = __bio_map_kern(q, data, len, gfp_mask);
924 if (IS_ERR(bio))
925 return bio;
926
927 if (bio->bi_size == len)
928 return bio;
929
930 /*
931 * Don't support partial mappings.
932 */
933 bio_put(bio);
934 return ERR_PTR(-EINVAL);
935}
936
FUJITA Tomonori68154e92008-04-25 12:47:50 +0200937static void bio_copy_kern_endio(struct bio *bio, int err)
938{
939 struct bio_vec *bvec;
940 const int read = bio_data_dir(bio) == READ;
FUJITA Tomonori76029ff2008-08-25 20:36:08 +0200941 struct bio_map_data *bmd = bio->bi_private;
FUJITA Tomonori68154e92008-04-25 12:47:50 +0200942 int i;
FUJITA Tomonori76029ff2008-08-25 20:36:08 +0200943 char *p = bmd->sgvecs[0].iov_base;
FUJITA Tomonori68154e92008-04-25 12:47:50 +0200944
945 __bio_for_each_segment(bvec, bio, i, 0) {
946 char *addr = page_address(bvec->bv_page);
FUJITA Tomonori76029ff2008-08-25 20:36:08 +0200947 int len = bmd->iovecs[i].bv_len;
FUJITA Tomonori68154e92008-04-25 12:47:50 +0200948
949 if (read && !err)
FUJITA Tomonori76029ff2008-08-25 20:36:08 +0200950 memcpy(p, addr, len);
FUJITA Tomonori68154e92008-04-25 12:47:50 +0200951
952 __free_page(bvec->bv_page);
FUJITA Tomonori76029ff2008-08-25 20:36:08 +0200953 p += len;
FUJITA Tomonori68154e92008-04-25 12:47:50 +0200954 }
955
FUJITA Tomonori76029ff2008-08-25 20:36:08 +0200956 bio_free_map_data(bmd);
FUJITA Tomonori68154e92008-04-25 12:47:50 +0200957 bio_put(bio);
958}
959
960/**
961 * bio_copy_kern - copy kernel address into bio
962 * @q: the struct request_queue for the bio
963 * @data: pointer to buffer to copy
964 * @len: length in bytes
965 * @gfp_mask: allocation flags for bio and page allocation
Randy Dunlapffee0252008-04-30 09:08:54 +0200966 * @reading: data direction is READ
FUJITA Tomonori68154e92008-04-25 12:47:50 +0200967 *
968 * copy the kernel address into a bio suitable for io to a block
969 * device. Returns an error pointer in case of error.
970 */
971struct bio *bio_copy_kern(struct request_queue *q, void *data, unsigned int len,
972 gfp_t gfp_mask, int reading)
973{
974 unsigned long kaddr = (unsigned long)data;
975 unsigned long end = (kaddr + len + PAGE_SIZE - 1) >> PAGE_SHIFT;
976 unsigned long start = kaddr >> PAGE_SHIFT;
977 const int nr_pages = end - start;
978 struct bio *bio;
979 struct bio_vec *bvec;
FUJITA Tomonori76029ff2008-08-25 20:36:08 +0200980 struct bio_map_data *bmd;
FUJITA Tomonori68154e92008-04-25 12:47:50 +0200981 int i, ret;
FUJITA Tomonori76029ff2008-08-25 20:36:08 +0200982 struct sg_iovec iov;
FUJITA Tomonori68154e92008-04-25 12:47:50 +0200983
FUJITA Tomonori76029ff2008-08-25 20:36:08 +0200984 iov.iov_base = data;
985 iov.iov_len = len;
986
987 bmd = bio_alloc_map_data(nr_pages, 1, gfp_mask);
988 if (!bmd)
989 return ERR_PTR(-ENOMEM);
990
991 ret = -ENOMEM;
FUJITA Tomonori68154e92008-04-25 12:47:50 +0200992 bio = bio_alloc(gfp_mask, nr_pages);
993 if (!bio)
FUJITA Tomonori76029ff2008-08-25 20:36:08 +0200994 goto out_bmd;
FUJITA Tomonori68154e92008-04-25 12:47:50 +0200995
996 while (len) {
997 struct page *page;
998 unsigned int bytes = PAGE_SIZE;
999
1000 if (bytes > len)
1001 bytes = len;
1002
1003 page = alloc_page(q->bounce_gfp | gfp_mask);
1004 if (!page) {
1005 ret = -ENOMEM;
1006 goto cleanup;
1007 }
1008
1009 if (bio_add_pc_page(q, bio, page, bytes, 0) < bytes) {
1010 ret = -EINVAL;
1011 goto cleanup;
1012 }
1013
1014 len -= bytes;
1015 }
1016
1017 if (!reading) {
1018 void *p = data;
1019
1020 bio_for_each_segment(bvec, bio, i) {
1021 char *addr = page_address(bvec->bv_page);
1022
1023 memcpy(addr, p, bvec->bv_len);
1024 p += bvec->bv_len;
1025 }
1026 }
1027
FUJITA Tomonori76029ff2008-08-25 20:36:08 +02001028 bio->bi_private = bmd;
FUJITA Tomonori68154e92008-04-25 12:47:50 +02001029 bio->bi_end_io = bio_copy_kern_endio;
FUJITA Tomonori76029ff2008-08-25 20:36:08 +02001030
1031 bio_set_map_data(bmd, bio, &iov, 1);
FUJITA Tomonori68154e92008-04-25 12:47:50 +02001032 return bio;
1033cleanup:
1034 bio_for_each_segment(bvec, bio, i)
1035 __free_page(bvec->bv_page);
1036
1037 bio_put(bio);
FUJITA Tomonori76029ff2008-08-25 20:36:08 +02001038out_bmd:
1039 bio_free_map_data(bmd);
FUJITA Tomonori68154e92008-04-25 12:47:50 +02001040
1041 return ERR_PTR(ret);
1042}
1043
Linus Torvalds1da177e2005-04-16 15:20:36 -07001044/*
1045 * bio_set_pages_dirty() and bio_check_pages_dirty() are support functions
1046 * for performing direct-IO in BIOs.
1047 *
1048 * The problem is that we cannot run set_page_dirty() from interrupt context
1049 * because the required locks are not interrupt-safe. So what we can do is to
1050 * mark the pages dirty _before_ performing IO. And in interrupt context,
1051 * check that the pages are still dirty. If so, fine. If not, redirty them
1052 * in process context.
1053 *
1054 * We special-case compound pages here: normally this means reads into hugetlb
1055 * pages. The logic in here doesn't really work right for compound pages
1056 * because the VM does not uniformly chase down the head page in all cases.
1057 * But dirtiness of compound pages is pretty meaningless anyway: the VM doesn't
1058 * handle them at all. So we skip compound pages here at an early stage.
1059 *
1060 * Note that this code is very hard to test under normal circumstances because
1061 * direct-io pins the pages with get_user_pages(). This makes
1062 * is_page_cache_freeable return false, and the VM will not clean the pages.
1063 * But other code (eg, pdflush) could clean the pages if they are mapped
1064 * pagecache.
1065 *
1066 * Simply disabling the call to bio_set_pages_dirty() is a good way to test the
1067 * deferred bio dirtying paths.
1068 */
1069
1070/*
1071 * bio_set_pages_dirty() will mark all the bio's pages as dirty.
1072 */
1073void bio_set_pages_dirty(struct bio *bio)
1074{
1075 struct bio_vec *bvec = bio->bi_io_vec;
1076 int i;
1077
1078 for (i = 0; i < bio->bi_vcnt; i++) {
1079 struct page *page = bvec[i].bv_page;
1080
1081 if (page && !PageCompound(page))
1082 set_page_dirty_lock(page);
1083 }
1084}
1085
Adrian Bunk86b6c7a2008-02-18 13:48:32 +01001086static void bio_release_pages(struct bio *bio)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001087{
1088 struct bio_vec *bvec = bio->bi_io_vec;
1089 int i;
1090
1091 for (i = 0; i < bio->bi_vcnt; i++) {
1092 struct page *page = bvec[i].bv_page;
1093
1094 if (page)
1095 put_page(page);
1096 }
1097}
1098
1099/*
1100 * bio_check_pages_dirty() will check that all the BIO's pages are still dirty.
1101 * If they are, then fine. If, however, some pages are clean then they must
1102 * have been written out during the direct-IO read. So we take another ref on
1103 * the BIO and the offending pages and re-dirty the pages in process context.
1104 *
1105 * It is expected that bio_check_pages_dirty() will wholly own the BIO from
1106 * here on. It will run one page_cache_release() against each page and will
1107 * run one bio_put() against the BIO.
1108 */
1109
David Howells65f27f32006-11-22 14:55:48 +00001110static void bio_dirty_fn(struct work_struct *work);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001111
David Howells65f27f32006-11-22 14:55:48 +00001112static DECLARE_WORK(bio_dirty_work, bio_dirty_fn);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001113static DEFINE_SPINLOCK(bio_dirty_lock);
1114static struct bio *bio_dirty_list;
1115
1116/*
1117 * This runs in process context
1118 */
David Howells65f27f32006-11-22 14:55:48 +00001119static void bio_dirty_fn(struct work_struct *work)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001120{
1121 unsigned long flags;
1122 struct bio *bio;
1123
1124 spin_lock_irqsave(&bio_dirty_lock, flags);
1125 bio = bio_dirty_list;
1126 bio_dirty_list = NULL;
1127 spin_unlock_irqrestore(&bio_dirty_lock, flags);
1128
1129 while (bio) {
1130 struct bio *next = bio->bi_private;
1131
1132 bio_set_pages_dirty(bio);
1133 bio_release_pages(bio);
1134 bio_put(bio);
1135 bio = next;
1136 }
1137}
1138
1139void bio_check_pages_dirty(struct bio *bio)
1140{
1141 struct bio_vec *bvec = bio->bi_io_vec;
1142 int nr_clean_pages = 0;
1143 int i;
1144
1145 for (i = 0; i < bio->bi_vcnt; i++) {
1146 struct page *page = bvec[i].bv_page;
1147
1148 if (PageDirty(page) || PageCompound(page)) {
1149 page_cache_release(page);
1150 bvec[i].bv_page = NULL;
1151 } else {
1152 nr_clean_pages++;
1153 }
1154 }
1155
1156 if (nr_clean_pages) {
1157 unsigned long flags;
1158
1159 spin_lock_irqsave(&bio_dirty_lock, flags);
1160 bio->bi_private = bio_dirty_list;
1161 bio_dirty_list = bio;
1162 spin_unlock_irqrestore(&bio_dirty_lock, flags);
1163 schedule_work(&bio_dirty_work);
1164 } else {
1165 bio_put(bio);
1166 }
1167}
1168
1169/**
1170 * bio_endio - end I/O on a bio
1171 * @bio: bio
Linus Torvalds1da177e2005-04-16 15:20:36 -07001172 * @error: error, if any
1173 *
1174 * Description:
NeilBrown6712ecf2007-09-27 12:47:43 +02001175 * bio_endio() will end I/O on the whole bio. bio_endio() is the
NeilBrown5bb23a62007-09-27 12:46:13 +02001176 * preferred way to end I/O on a bio, it takes care of clearing
1177 * BIO_UPTODATE on error. @error is 0 on success, and and one of the
1178 * established -Exxxx (-EIO, for instance) error values in case
1179 * something went wrong. Noone should call bi_end_io() directly on a
1180 * bio unless they own it and thus know that it has an end_io
1181 * function.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001182 **/
NeilBrown6712ecf2007-09-27 12:47:43 +02001183void bio_endio(struct bio *bio, int error)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001184{
1185 if (error)
1186 clear_bit(BIO_UPTODATE, &bio->bi_flags);
NeilBrown9cc54d42007-09-27 12:46:12 +02001187 else if (!test_bit(BIO_UPTODATE, &bio->bi_flags))
1188 error = -EIO;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001189
NeilBrown5bb23a62007-09-27 12:46:13 +02001190 if (bio->bi_end_io)
NeilBrown6712ecf2007-09-27 12:47:43 +02001191 bio->bi_end_io(bio, error);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001192}
1193
1194void bio_pair_release(struct bio_pair *bp)
1195{
1196 if (atomic_dec_and_test(&bp->cnt)) {
1197 struct bio *master = bp->bio1.bi_private;
1198
NeilBrown6712ecf2007-09-27 12:47:43 +02001199 bio_endio(master, bp->error);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001200 mempool_free(bp, bp->bio2.bi_private);
1201 }
1202}
1203
NeilBrown6712ecf2007-09-27 12:47:43 +02001204static void bio_pair_end_1(struct bio *bi, int err)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001205{
1206 struct bio_pair *bp = container_of(bi, struct bio_pair, bio1);
1207
1208 if (err)
1209 bp->error = err;
1210
Linus Torvalds1da177e2005-04-16 15:20:36 -07001211 bio_pair_release(bp);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001212}
1213
NeilBrown6712ecf2007-09-27 12:47:43 +02001214static void bio_pair_end_2(struct bio *bi, int err)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001215{
1216 struct bio_pair *bp = container_of(bi, struct bio_pair, bio2);
1217
1218 if (err)
1219 bp->error = err;
1220
Linus Torvalds1da177e2005-04-16 15:20:36 -07001221 bio_pair_release(bp);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001222}
1223
1224/*
1225 * split a bio - only worry about a bio with a single page
1226 * in it's iovec
1227 */
1228struct bio_pair *bio_split(struct bio *bi, mempool_t *pool, int first_sectors)
1229{
1230 struct bio_pair *bp = mempool_alloc(pool, GFP_NOIO);
1231
1232 if (!bp)
1233 return bp;
1234
Jens Axboe2056a782006-03-23 20:00:26 +01001235 blk_add_trace_pdu_int(bdev_get_queue(bi->bi_bdev), BLK_TA_SPLIT, bi,
1236 bi->bi_sector + first_sectors);
1237
Linus Torvalds1da177e2005-04-16 15:20:36 -07001238 BUG_ON(bi->bi_vcnt != 1);
1239 BUG_ON(bi->bi_idx != 0);
1240 atomic_set(&bp->cnt, 3);
1241 bp->error = 0;
1242 bp->bio1 = *bi;
1243 bp->bio2 = *bi;
1244 bp->bio2.bi_sector += first_sectors;
1245 bp->bio2.bi_size -= first_sectors << 9;
1246 bp->bio1.bi_size = first_sectors << 9;
1247
1248 bp->bv1 = bi->bi_io_vec[0];
1249 bp->bv2 = bi->bi_io_vec[0];
1250 bp->bv2.bv_offset += first_sectors << 9;
1251 bp->bv2.bv_len -= first_sectors << 9;
1252 bp->bv1.bv_len = first_sectors << 9;
1253
1254 bp->bio1.bi_io_vec = &bp->bv1;
1255 bp->bio2.bi_io_vec = &bp->bv2;
1256
NeilBrowna2eb0c12006-05-22 22:35:27 -07001257 bp->bio1.bi_max_vecs = 1;
1258 bp->bio2.bi_max_vecs = 1;
1259
Linus Torvalds1da177e2005-04-16 15:20:36 -07001260 bp->bio1.bi_end_io = bio_pair_end_1;
1261 bp->bio2.bi_end_io = bio_pair_end_2;
1262
1263 bp->bio1.bi_private = bi;
1264 bp->bio2.bi_private = pool;
1265
Martin K. Petersen7ba1ba12008-06-30 20:04:41 +02001266 if (bio_integrity(bi))
1267 bio_integrity_split(bi, bp, first_sectors);
1268
Linus Torvalds1da177e2005-04-16 15:20:36 -07001269 return bp;
1270}
1271
Linus Torvalds1da177e2005-04-16 15:20:36 -07001272
1273/*
1274 * create memory pools for biovec's in a bio_set.
1275 * use the global biovec slabs created for general use.
1276 */
Jens Axboe59725112007-04-02 10:06:42 +02001277static int biovec_create_pools(struct bio_set *bs, int pool_entries)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001278{
1279 int i;
1280
1281 for (i = 0; i < BIOVEC_NR_POOLS; i++) {
1282 struct biovec_slab *bp = bvec_slabs + i;
1283 mempool_t **bvp = bs->bvec_pools + i;
1284
Matthew Dobson93d23412006-03-26 01:37:50 -08001285 *bvp = mempool_create_slab_pool(pool_entries, bp->slab);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001286 if (!*bvp)
1287 return -ENOMEM;
1288 }
1289 return 0;
1290}
1291
1292static void biovec_free_pools(struct bio_set *bs)
1293{
1294 int i;
1295
1296 for (i = 0; i < BIOVEC_NR_POOLS; i++) {
1297 mempool_t *bvp = bs->bvec_pools[i];
1298
1299 if (bvp)
1300 mempool_destroy(bvp);
1301 }
1302
1303}
1304
1305void bioset_free(struct bio_set *bs)
1306{
1307 if (bs->bio_pool)
1308 mempool_destroy(bs->bio_pool);
1309
Martin K. Petersen7ba1ba12008-06-30 20:04:41 +02001310 bioset_integrity_free(bs);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001311 biovec_free_pools(bs);
1312
1313 kfree(bs);
1314}
1315
Jens Axboe59725112007-04-02 10:06:42 +02001316struct bio_set *bioset_create(int bio_pool_size, int bvec_pool_size)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001317{
Oliver Neukum11b0b5a2006-03-25 03:08:13 -08001318 struct bio_set *bs = kzalloc(sizeof(*bs), GFP_KERNEL);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001319
1320 if (!bs)
1321 return NULL;
1322
Matthew Dobson93d23412006-03-26 01:37:50 -08001323 bs->bio_pool = mempool_create_slab_pool(bio_pool_size, bio_slab);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001324 if (!bs->bio_pool)
1325 goto bad;
1326
Martin K. Petersen7ba1ba12008-06-30 20:04:41 +02001327 if (bioset_integrity_create(bs, bio_pool_size))
1328 goto bad;
1329
Jens Axboe59725112007-04-02 10:06:42 +02001330 if (!biovec_create_pools(bs, bvec_pool_size))
Linus Torvalds1da177e2005-04-16 15:20:36 -07001331 return bs;
1332
1333bad:
1334 bioset_free(bs);
1335 return NULL;
1336}
1337
1338static void __init biovec_init_slabs(void)
1339{
1340 int i;
1341
1342 for (i = 0; i < BIOVEC_NR_POOLS; i++) {
1343 int size;
1344 struct biovec_slab *bvs = bvec_slabs + i;
1345
1346 size = bvs->nr_vecs * sizeof(struct bio_vec);
1347 bvs->slab = kmem_cache_create(bvs->name, size, 0,
Paul Mundt20c2df82007-07-20 10:11:58 +09001348 SLAB_HWCACHE_ALIGN|SLAB_PANIC, NULL);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001349 }
1350}
1351
1352static int __init init_bio(void)
1353{
Christoph Lameter0a31bd52007-05-06 14:49:57 -07001354 bio_slab = KMEM_CACHE(bio, SLAB_HWCACHE_ALIGN|SLAB_PANIC);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001355
Martin K. Petersen7ba1ba12008-06-30 20:04:41 +02001356 bio_integrity_init_slab();
Linus Torvalds1da177e2005-04-16 15:20:36 -07001357 biovec_init_slabs();
1358
Jens Axboe59725112007-04-02 10:06:42 +02001359 fs_bio_set = bioset_create(BIO_POOL_SIZE, 2);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001360 if (!fs_bio_set)
1361 panic("bio: can't allocate bios\n");
1362
Matthew Dobson0eaae62a2006-03-26 01:37:47 -08001363 bio_split_pool = mempool_create_kmalloc_pool(BIO_SPLIT_ENTRIES,
1364 sizeof(struct bio_pair));
Linus Torvalds1da177e2005-04-16 15:20:36 -07001365 if (!bio_split_pool)
1366 panic("bio: can't create split pool\n");
1367
1368 return 0;
1369}
1370
1371subsys_initcall(init_bio);
1372
1373EXPORT_SYMBOL(bio_alloc);
1374EXPORT_SYMBOL(bio_put);
Peter Osterlund36763472005-09-06 15:16:42 -07001375EXPORT_SYMBOL(bio_free);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001376EXPORT_SYMBOL(bio_endio);
1377EXPORT_SYMBOL(bio_init);
1378EXPORT_SYMBOL(__bio_clone);
1379EXPORT_SYMBOL(bio_clone);
1380EXPORT_SYMBOL(bio_phys_segments);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001381EXPORT_SYMBOL(bio_add_page);
Mike Christie6e68af62005-11-11 05:30:27 -06001382EXPORT_SYMBOL(bio_add_pc_page);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001383EXPORT_SYMBOL(bio_get_nr_vecs);
Jens Axboe40044ce2008-03-17 21:14:40 +01001384EXPORT_SYMBOL(bio_map_user);
1385EXPORT_SYMBOL(bio_unmap_user);
Mike Christie df46b9a2005-06-20 14:04:44 +02001386EXPORT_SYMBOL(bio_map_kern);
FUJITA Tomonori68154e92008-04-25 12:47:50 +02001387EXPORT_SYMBOL(bio_copy_kern);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001388EXPORT_SYMBOL(bio_pair_release);
1389EXPORT_SYMBOL(bio_split);
1390EXPORT_SYMBOL(bio_split_pool);
1391EXPORT_SYMBOL(bio_copy_user);
1392EXPORT_SYMBOL(bio_uncopy_user);
1393EXPORT_SYMBOL(bioset_create);
1394EXPORT_SYMBOL(bioset_free);
1395EXPORT_SYMBOL(bio_alloc_bioset);