blob: 060859c6909283fb210045def5950311a36a7e65 [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
Jens Axboe0fe23472006-09-04 15:41:16 +02002 * Copyright (C) 2001 Jens Axboe <axboe@kernel.dk>
Linus Torvalds1da177e2005-04-16 15:20:36 -07003 *
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License version 2 as
6 * published by the Free Software Foundation.
7 *
8 * This program is distributed in the hope that it will be useful,
9 * but WITHOUT ANY WARRANTY; without even the implied warranty of
10 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 * GNU General Public License for more details.
12 *
13 * You should have received a copy of the GNU General Public Licens
14 * along with this program; if not, write to the Free Software
15 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-
16 *
17 */
18#include <linux/mm.h>
19#include <linux/swap.h>
20#include <linux/bio.h>
21#include <linux/blkdev.h>
22#include <linux/slab.h>
23#include <linux/init.h>
24#include <linux/kernel.h>
25#include <linux/module.h>
26#include <linux/mempool.h>
27#include <linux/workqueue.h>
Jens Axboe2056a782006-03-23 20:00:26 +010028#include <linux/blktrace_api.h>
Arnaldo Carvalho de Melo5f3ea372008-10-30 08:34:33 +010029#include <trace/block.h>
James Bottomley f1970ba2005-06-20 14:06:52 +020030#include <scsi/sg.h> /* for struct sg_iovec */
Linus Torvalds1da177e2005-04-16 15:20:36 -070031
Christoph Lametere18b8902006-12-06 20:33:20 -080032static struct kmem_cache *bio_slab __read_mostly;
Linus Torvalds1da177e2005-04-16 15:20:36 -070033
Denis ChengRq6feef532008-10-09 08:57:05 +020034static mempool_t *bio_split_pool __read_mostly;
Linus Torvalds1da177e2005-04-16 15:20:36 -070035
Linus Torvalds1da177e2005-04-16 15:20:36 -070036/*
37 * if you change this list, also change bvec_alloc or things will
38 * break badly! cannot be bigger than what you can fit into an
39 * unsigned short
40 */
41
42#define BV(x) { .nr_vecs = x, .name = "biovec-"__stringify(x) }
Christoph Lameter6c036522005-07-07 17:56:59 -070043static struct biovec_slab bvec_slabs[BIOVEC_NR_POOLS] __read_mostly = {
Linus Torvalds1da177e2005-04-16 15:20:36 -070044 BV(1), BV(4), BV(16), BV(64), BV(128), BV(BIO_MAX_PAGES),
45};
46#undef BV
47
48/*
Linus Torvalds1da177e2005-04-16 15:20:36 -070049 * fs_bio_set is the bio_set containing bio and iovec memory pools used by
50 * IO code that does not need private memory pools.
51 */
Martin K. Petersen51d654e2008-06-17 18:59:56 +020052struct bio_set *fs_bio_set;
Linus Torvalds1da177e2005-04-16 15:20:36 -070053
Martin K. Petersen7ba1ba12008-06-30 20:04:41 +020054unsigned int bvec_nr_vecs(unsigned short idx)
55{
56 return bvec_slabs[idx].nr_vecs;
57}
58
Martin K. Petersen51d654e2008-06-17 18:59:56 +020059struct bio_vec *bvec_alloc_bs(gfp_t gfp_mask, int nr, unsigned long *idx, struct bio_set *bs)
Linus Torvalds1da177e2005-04-16 15:20:36 -070060{
61 struct bio_vec *bvl;
Linus Torvalds1da177e2005-04-16 15:20:36 -070062
63 /*
Jens Axboe0a0d96b2008-09-11 13:17:37 +020064 * If 'bs' is given, lookup the pool and do the mempool alloc.
65 * If not, this is a bio_kmalloc() allocation and just do a
66 * kzalloc() for the exact number of vecs right away.
Linus Torvalds1da177e2005-04-16 15:20:36 -070067 */
Jens Axboe0a0d96b2008-09-11 13:17:37 +020068 if (bs) {
69 /*
70 * see comment near bvec_array define!
71 */
72 switch (nr) {
73 case 1:
74 *idx = 0;
75 break;
76 case 2 ... 4:
77 *idx = 1;
78 break;
79 case 5 ... 16:
80 *idx = 2;
81 break;
82 case 17 ... 64:
83 *idx = 3;
84 break;
85 case 65 ... 128:
86 *idx = 4;
87 break;
88 case 129 ... BIO_MAX_PAGES:
89 *idx = 5;
90 break;
Linus Torvalds1da177e2005-04-16 15:20:36 -070091 default:
92 return NULL;
Jens Axboe0a0d96b2008-09-11 13:17:37 +020093 }
Linus Torvalds1da177e2005-04-16 15:20:36 -070094
Jens Axboe0a0d96b2008-09-11 13:17:37 +020095 /*
96 * idx now points to the pool we want to allocate from
97 */
98 bvl = mempool_alloc(bs->bvec_pools[*idx], gfp_mask);
99 if (bvl)
100 memset(bvl, 0,
101 bvec_nr_vecs(*idx) * sizeof(struct bio_vec));
102 } else
103 bvl = kzalloc(nr * sizeof(struct bio_vec), gfp_mask);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700104
105 return bvl;
106}
107
Peter Osterlund36763472005-09-06 15:16:42 -0700108void bio_free(struct bio *bio, struct bio_set *bio_set)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700109{
Jens Axboe992c5dd2007-07-18 13:18:08 +0200110 if (bio->bi_io_vec) {
111 const int pool_idx = BIO_POOL_IDX(bio);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700112
Jens Axboe992c5dd2007-07-18 13:18:08 +0200113 BIO_BUG_ON(pool_idx >= BIOVEC_NR_POOLS);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700114
Jens Axboe992c5dd2007-07-18 13:18:08 +0200115 mempool_free(bio->bi_io_vec, bio_set->bvec_pools[pool_idx]);
116 }
117
Martin K. Petersen7ba1ba12008-06-30 20:04:41 +0200118 if (bio_integrity(bio))
119 bio_integrity_free(bio, bio_set);
120
Peter Osterlund36763472005-09-06 15:16:42 -0700121 mempool_free(bio, bio_set->bio_pool);
122}
123
124/*
125 * default destructor for a bio allocated with bio_alloc_bioset()
126 */
127static void bio_fs_destructor(struct bio *bio)
128{
129 bio_free(bio, fs_bio_set);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700130}
131
Jens Axboe0a0d96b2008-09-11 13:17:37 +0200132static void bio_kmalloc_destructor(struct bio *bio)
133{
134 kfree(bio->bi_io_vec);
135 kfree(bio);
136}
137
Arjan van de Ven858119e2006-01-14 13:20:43 -0800138void bio_init(struct bio *bio)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700139{
Jens Axboe2b94de52007-07-18 13:14:03 +0200140 memset(bio, 0, sizeof(*bio));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700141 bio->bi_flags = 1 << BIO_UPTODATE;
Jens Axboec7c22e42008-09-13 20:26:01 +0200142 bio->bi_comp_cpu = -1;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700143 atomic_set(&bio->bi_cnt, 1);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700144}
145
146/**
147 * bio_alloc_bioset - allocate a bio for I/O
148 * @gfp_mask: the GFP_ mask given to the slab allocator
149 * @nr_iovecs: number of iovecs to pre-allocate
Jens Axboe0a0d96b2008-09-11 13:17:37 +0200150 * @bs: the bio_set to allocate from. If %NULL, just use kmalloc
Linus Torvalds1da177e2005-04-16 15:20:36 -0700151 *
152 * Description:
Jens Axboe0a0d96b2008-09-11 13:17:37 +0200153 * bio_alloc_bioset will first try its own mempool to satisfy the allocation.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700154 * If %__GFP_WAIT is set then we will block on the internal pool waiting
Jens Axboe0a0d96b2008-09-11 13:17:37 +0200155 * for a &struct bio to become free. If a %NULL @bs is passed in, we will
156 * fall back to just using @kmalloc to allocate the required memory.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700157 *
158 * allocate bio and iovecs from the memory pools specified by the
Jens Axboe0a0d96b2008-09-11 13:17:37 +0200159 * bio_set structure, or @kmalloc if none given.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700160 **/
Al Virodd0fc662005-10-07 07:46:04 +0100161struct bio *bio_alloc_bioset(gfp_t gfp_mask, int nr_iovecs, struct bio_set *bs)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700162{
Jens Axboe0a0d96b2008-09-11 13:17:37 +0200163 struct bio *bio;
164
165 if (bs)
166 bio = mempool_alloc(bs->bio_pool, gfp_mask);
167 else
168 bio = kmalloc(sizeof(*bio), gfp_mask);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700169
170 if (likely(bio)) {
171 struct bio_vec *bvl = NULL;
172
173 bio_init(bio);
174 if (likely(nr_iovecs)) {
Jens Axboeeeae1d42008-05-07 13:26:27 +0200175 unsigned long uninitialized_var(idx);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700176
177 bvl = bvec_alloc_bs(gfp_mask, nr_iovecs, &idx, bs);
178 if (unlikely(!bvl)) {
Jens Axboe0a0d96b2008-09-11 13:17:37 +0200179 if (bs)
180 mempool_free(bio, bs->bio_pool);
181 else
182 kfree(bio);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700183 bio = NULL;
184 goto out;
185 }
186 bio->bi_flags |= idx << BIO_POOL_OFFSET;
Denis ChengRq1ac0ae02008-08-04 11:56:30 +0200187 bio->bi_max_vecs = bvec_nr_vecs(idx);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700188 }
189 bio->bi_io_vec = bvl;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700190 }
191out:
192 return bio;
193}
194
Al Virodd0fc662005-10-07 07:46:04 +0100195struct bio *bio_alloc(gfp_t gfp_mask, int nr_iovecs)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700196{
Peter Osterlund36763472005-09-06 15:16:42 -0700197 struct bio *bio = bio_alloc_bioset(gfp_mask, nr_iovecs, fs_bio_set);
198
199 if (bio)
200 bio->bi_destructor = bio_fs_destructor;
201
202 return bio;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700203}
204
Jens Axboe0a0d96b2008-09-11 13:17:37 +0200205/*
206 * Like bio_alloc(), but doesn't use a mempool backing. This means that
207 * it CAN fail, but while bio_alloc() can only be used for allocations
208 * that have a short (finite) life span, bio_kmalloc() should be used
209 * for more permanent bio allocations (like allocating some bio's for
210 * initalization or setup purposes).
211 */
212struct bio *bio_kmalloc(gfp_t gfp_mask, int nr_iovecs)
213{
214 struct bio *bio = bio_alloc_bioset(gfp_mask, nr_iovecs, NULL);
215
216 if (bio)
217 bio->bi_destructor = bio_kmalloc_destructor;
218
219 return bio;
220}
221
Linus Torvalds1da177e2005-04-16 15:20:36 -0700222void zero_fill_bio(struct bio *bio)
223{
224 unsigned long flags;
225 struct bio_vec *bv;
226 int i;
227
228 bio_for_each_segment(bv, bio, i) {
229 char *data = bvec_kmap_irq(bv, &flags);
230 memset(data, 0, bv->bv_len);
231 flush_dcache_page(bv->bv_page);
232 bvec_kunmap_irq(data, &flags);
233 }
234}
235EXPORT_SYMBOL(zero_fill_bio);
236
237/**
238 * bio_put - release a reference to a bio
239 * @bio: bio to release reference to
240 *
241 * Description:
242 * Put a reference to a &struct bio, either one you have gotten with
243 * bio_alloc or bio_get. The last put of a bio will free it.
244 **/
245void bio_put(struct bio *bio)
246{
247 BIO_BUG_ON(!atomic_read(&bio->bi_cnt));
248
249 /*
250 * last put frees it
251 */
252 if (atomic_dec_and_test(&bio->bi_cnt)) {
253 bio->bi_next = NULL;
254 bio->bi_destructor(bio);
255 }
256}
257
Jens Axboe165125e2007-07-24 09:28:11 +0200258inline int bio_phys_segments(struct request_queue *q, struct bio *bio)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700259{
260 if (unlikely(!bio_flagged(bio, BIO_SEG_VALID)))
261 blk_recount_segments(q, bio);
262
263 return bio->bi_phys_segments;
264}
265
Linus Torvalds1da177e2005-04-16 15:20:36 -0700266/**
267 * __bio_clone - clone a bio
268 * @bio: destination bio
269 * @bio_src: bio to clone
270 *
271 * Clone a &bio. Caller will own the returned bio, but not
272 * the actual data it points to. Reference count of returned
273 * bio will be one.
274 */
Arjan van de Ven858119e2006-01-14 13:20:43 -0800275void __bio_clone(struct bio *bio, struct bio *bio_src)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700276{
Andrew Mortone525e152005-08-07 09:42:12 -0700277 memcpy(bio->bi_io_vec, bio_src->bi_io_vec,
278 bio_src->bi_max_vecs * sizeof(struct bio_vec));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700279
Jens Axboe5d840702008-01-25 12:44:44 +0100280 /*
281 * most users will be overriding ->bi_bdev with a new target,
282 * so we don't set nor calculate new physical/hw segment counts here
283 */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700284 bio->bi_sector = bio_src->bi_sector;
285 bio->bi_bdev = bio_src->bi_bdev;
286 bio->bi_flags |= 1 << BIO_CLONED;
287 bio->bi_rw = bio_src->bi_rw;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700288 bio->bi_vcnt = bio_src->bi_vcnt;
289 bio->bi_size = bio_src->bi_size;
Andrew Mortona5453be2005-07-28 01:07:18 -0700290 bio->bi_idx = bio_src->bi_idx;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700291}
292
293/**
294 * bio_clone - clone a bio
295 * @bio: bio to clone
296 * @gfp_mask: allocation priority
297 *
298 * Like __bio_clone, only also allocates the returned bio
299 */
Al Virodd0fc662005-10-07 07:46:04 +0100300struct bio *bio_clone(struct bio *bio, gfp_t gfp_mask)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700301{
302 struct bio *b = bio_alloc_bioset(gfp_mask, bio->bi_max_vecs, fs_bio_set);
303
Martin K. Petersen7ba1ba12008-06-30 20:04:41 +0200304 if (!b)
305 return NULL;
306
307 b->bi_destructor = bio_fs_destructor;
308 __bio_clone(b, bio);
309
310 if (bio_integrity(bio)) {
311 int ret;
312
313 ret = bio_integrity_clone(b, bio, fs_bio_set);
314
315 if (ret < 0)
316 return NULL;
Peter Osterlund36763472005-09-06 15:16:42 -0700317 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700318
319 return b;
320}
321
322/**
323 * bio_get_nr_vecs - return approx number of vecs
324 * @bdev: I/O target
325 *
326 * Return the approximate number of pages we can send to this target.
327 * There's no guarantee that you will be able to fit this number of pages
328 * into a bio, it does not account for dynamic restrictions that vary
329 * on offset.
330 */
331int bio_get_nr_vecs(struct block_device *bdev)
332{
Jens Axboe165125e2007-07-24 09:28:11 +0200333 struct request_queue *q = bdev_get_queue(bdev);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700334 int nr_pages;
335
336 nr_pages = ((q->max_sectors << 9) + PAGE_SIZE - 1) >> PAGE_SHIFT;
337 if (nr_pages > q->max_phys_segments)
338 nr_pages = q->max_phys_segments;
339 if (nr_pages > q->max_hw_segments)
340 nr_pages = q->max_hw_segments;
341
342 return nr_pages;
343}
344
Jens Axboe165125e2007-07-24 09:28:11 +0200345static int __bio_add_page(struct request_queue *q, struct bio *bio, struct page
Mike Christiedefd94b2005-12-05 02:37:06 -0600346 *page, unsigned int len, unsigned int offset,
347 unsigned short max_sectors)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700348{
349 int retried_segments = 0;
350 struct bio_vec *bvec;
351
352 /*
353 * cloned bio must not modify vec list
354 */
355 if (unlikely(bio_flagged(bio, BIO_CLONED)))
356 return 0;
357
Jens Axboe80cfd542006-01-06 09:43:28 +0100358 if (((bio->bi_size + len) >> 9) > max_sectors)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700359 return 0;
360
Jens Axboe80cfd542006-01-06 09:43:28 +0100361 /*
362 * For filesystems with a blocksize smaller than the pagesize
363 * we will often be called with the same page as last time and
364 * a consecutive offset. Optimize this special case.
365 */
366 if (bio->bi_vcnt > 0) {
367 struct bio_vec *prev = &bio->bi_io_vec[bio->bi_vcnt - 1];
368
369 if (page == prev->bv_page &&
370 offset == prev->bv_offset + prev->bv_len) {
371 prev->bv_len += len;
Alasdair G Kergoncc371e62008-07-03 09:53:43 +0200372
373 if (q->merge_bvec_fn) {
374 struct bvec_merge_data bvm = {
375 .bi_bdev = bio->bi_bdev,
376 .bi_sector = bio->bi_sector,
377 .bi_size = bio->bi_size,
378 .bi_rw = bio->bi_rw,
379 };
380
381 if (q->merge_bvec_fn(q, &bvm, prev) < len) {
382 prev->bv_len -= len;
383 return 0;
384 }
Jens Axboe80cfd542006-01-06 09:43:28 +0100385 }
386
387 goto done;
388 }
389 }
390
391 if (bio->bi_vcnt >= bio->bi_max_vecs)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700392 return 0;
393
394 /*
395 * we might lose a segment or two here, but rather that than
396 * make this too complex.
397 */
398
399 while (bio->bi_phys_segments >= q->max_phys_segments
Mikulas Patocka5df97b92008-08-15 10:20:02 +0200400 || bio->bi_phys_segments >= q->max_hw_segments) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700401
402 if (retried_segments)
403 return 0;
404
405 retried_segments = 1;
406 blk_recount_segments(q, bio);
407 }
408
409 /*
410 * setup the new entry, we might clear it again later if we
411 * cannot add the page
412 */
413 bvec = &bio->bi_io_vec[bio->bi_vcnt];
414 bvec->bv_page = page;
415 bvec->bv_len = len;
416 bvec->bv_offset = offset;
417
418 /*
419 * if queue has other restrictions (eg varying max sector size
420 * depending on offset), it can specify a merge_bvec_fn in the
421 * queue to get further control
422 */
423 if (q->merge_bvec_fn) {
Alasdair G Kergoncc371e62008-07-03 09:53:43 +0200424 struct bvec_merge_data bvm = {
425 .bi_bdev = bio->bi_bdev,
426 .bi_sector = bio->bi_sector,
427 .bi_size = bio->bi_size,
428 .bi_rw = bio->bi_rw,
429 };
430
Linus Torvalds1da177e2005-04-16 15:20:36 -0700431 /*
432 * merge_bvec_fn() returns number of bytes it can accept
433 * at this offset
434 */
Alasdair G Kergoncc371e62008-07-03 09:53:43 +0200435 if (q->merge_bvec_fn(q, &bvm, bvec) < len) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700436 bvec->bv_page = NULL;
437 bvec->bv_len = 0;
438 bvec->bv_offset = 0;
439 return 0;
440 }
441 }
442
443 /* If we may be able to merge these biovecs, force a recount */
Mikulas Patockab8b3e162008-08-15 10:15:19 +0200444 if (bio->bi_vcnt && (BIOVEC_PHYS_MERGEABLE(bvec-1, bvec)))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700445 bio->bi_flags &= ~(1 << BIO_SEG_VALID);
446
447 bio->bi_vcnt++;
448 bio->bi_phys_segments++;
Jens Axboe80cfd542006-01-06 09:43:28 +0100449 done:
Linus Torvalds1da177e2005-04-16 15:20:36 -0700450 bio->bi_size += len;
451 return len;
452}
453
454/**
Mike Christie6e68af62005-11-11 05:30:27 -0600455 * bio_add_pc_page - attempt to add page to bio
Jens Axboefddfdea2006-01-31 15:24:34 +0100456 * @q: the target queue
Mike Christie6e68af62005-11-11 05:30:27 -0600457 * @bio: destination bio
458 * @page: page to add
459 * @len: vec entry length
460 * @offset: vec entry offset
461 *
462 * Attempt to add a page to the bio_vec maplist. This can fail for a
463 * number of reasons, such as the bio being full or target block
464 * device limitations. The target block device must allow bio's
465 * smaller than PAGE_SIZE, so it is always possible to add a single
466 * page to an empty bio. This should only be used by REQ_PC bios.
467 */
Jens Axboe165125e2007-07-24 09:28:11 +0200468int bio_add_pc_page(struct request_queue *q, struct bio *bio, struct page *page,
Mike Christie6e68af62005-11-11 05:30:27 -0600469 unsigned int len, unsigned int offset)
470{
Mike Christiedefd94b2005-12-05 02:37:06 -0600471 return __bio_add_page(q, bio, page, len, offset, q->max_hw_sectors);
Mike Christie6e68af62005-11-11 05:30:27 -0600472}
473
474/**
Linus Torvalds1da177e2005-04-16 15:20:36 -0700475 * bio_add_page - attempt to add page to bio
476 * @bio: destination bio
477 * @page: page to add
478 * @len: vec entry length
479 * @offset: vec entry offset
480 *
481 * Attempt to add a page to the bio_vec maplist. This can fail for a
482 * number of reasons, such as the bio being full or target block
483 * device limitations. The target block device must allow bio's
484 * smaller than PAGE_SIZE, so it is always possible to add a single
485 * page to an empty bio.
486 */
487int bio_add_page(struct bio *bio, struct page *page, unsigned int len,
488 unsigned int offset)
489{
Mike Christiedefd94b2005-12-05 02:37:06 -0600490 struct request_queue *q = bdev_get_queue(bio->bi_bdev);
491 return __bio_add_page(q, bio, page, len, offset, q->max_sectors);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700492}
493
494struct bio_map_data {
495 struct bio_vec *iovecs;
FUJITA Tomonoric5dec1c2008-04-11 12:56:49 +0200496 struct sg_iovec *sgvecs;
FUJITA Tomonori152e2832008-08-28 16:17:06 +0900497 int nr_sgvecs;
498 int is_our_pages;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700499};
500
FUJITA Tomonoric5dec1c2008-04-11 12:56:49 +0200501static void bio_set_map_data(struct bio_map_data *bmd, struct bio *bio,
FUJITA Tomonori152e2832008-08-28 16:17:06 +0900502 struct sg_iovec *iov, int iov_count,
503 int is_our_pages)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700504{
505 memcpy(bmd->iovecs, bio->bi_io_vec, sizeof(struct bio_vec) * bio->bi_vcnt);
FUJITA Tomonoric5dec1c2008-04-11 12:56:49 +0200506 memcpy(bmd->sgvecs, iov, sizeof(struct sg_iovec) * iov_count);
507 bmd->nr_sgvecs = iov_count;
FUJITA Tomonori152e2832008-08-28 16:17:06 +0900508 bmd->is_our_pages = is_our_pages;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700509 bio->bi_private = bmd;
510}
511
512static void bio_free_map_data(struct bio_map_data *bmd)
513{
514 kfree(bmd->iovecs);
FUJITA Tomonoric5dec1c2008-04-11 12:56:49 +0200515 kfree(bmd->sgvecs);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700516 kfree(bmd);
517}
518
FUJITA Tomonori76029ff2008-08-25 20:36:08 +0200519static struct bio_map_data *bio_alloc_map_data(int nr_segs, int iov_count,
520 gfp_t gfp_mask)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700521{
FUJITA Tomonori76029ff2008-08-25 20:36:08 +0200522 struct bio_map_data *bmd = kmalloc(sizeof(*bmd), gfp_mask);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700523
524 if (!bmd)
525 return NULL;
526
FUJITA Tomonori76029ff2008-08-25 20:36:08 +0200527 bmd->iovecs = kmalloc(sizeof(struct bio_vec) * nr_segs, gfp_mask);
FUJITA Tomonoric5dec1c2008-04-11 12:56:49 +0200528 if (!bmd->iovecs) {
529 kfree(bmd);
530 return NULL;
531 }
532
FUJITA Tomonori76029ff2008-08-25 20:36:08 +0200533 bmd->sgvecs = kmalloc(sizeof(struct sg_iovec) * iov_count, gfp_mask);
FUJITA Tomonoric5dec1c2008-04-11 12:56:49 +0200534 if (bmd->sgvecs)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700535 return bmd;
536
FUJITA Tomonoric5dec1c2008-04-11 12:56:49 +0200537 kfree(bmd->iovecs);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700538 kfree(bmd);
539 return NULL;
540}
541
FUJITA Tomonoriaefcc282008-08-25 20:36:08 +0200542static int __bio_copy_iov(struct bio *bio, struct bio_vec *iovecs,
FUJITA Tomonori152e2832008-08-28 16:17:06 +0900543 struct sg_iovec *iov, int iov_count, int uncopy,
544 int do_free_page)
FUJITA Tomonoric5dec1c2008-04-11 12:56:49 +0200545{
546 int ret = 0, i;
547 struct bio_vec *bvec;
548 int iov_idx = 0;
549 unsigned int iov_off = 0;
550 int read = bio_data_dir(bio) == READ;
551
552 __bio_for_each_segment(bvec, bio, i, 0) {
553 char *bv_addr = page_address(bvec->bv_page);
FUJITA Tomonoriaefcc282008-08-25 20:36:08 +0200554 unsigned int bv_len = iovecs[i].bv_len;
FUJITA Tomonoric5dec1c2008-04-11 12:56:49 +0200555
556 while (bv_len && iov_idx < iov_count) {
557 unsigned int bytes;
558 char *iov_addr;
559
560 bytes = min_t(unsigned int,
561 iov[iov_idx].iov_len - iov_off, bv_len);
562 iov_addr = iov[iov_idx].iov_base + iov_off;
563
564 if (!ret) {
565 if (!read && !uncopy)
566 ret = copy_from_user(bv_addr, iov_addr,
567 bytes);
568 if (read && uncopy)
569 ret = copy_to_user(iov_addr, bv_addr,
570 bytes);
571
572 if (ret)
573 ret = -EFAULT;
574 }
575
576 bv_len -= bytes;
577 bv_addr += bytes;
578 iov_addr += bytes;
579 iov_off += bytes;
580
581 if (iov[iov_idx].iov_len == iov_off) {
582 iov_idx++;
583 iov_off = 0;
584 }
585 }
586
FUJITA Tomonori152e2832008-08-28 16:17:06 +0900587 if (do_free_page)
FUJITA Tomonoric5dec1c2008-04-11 12:56:49 +0200588 __free_page(bvec->bv_page);
589 }
590
591 return ret;
592}
593
Linus Torvalds1da177e2005-04-16 15:20:36 -0700594/**
595 * bio_uncopy_user - finish previously mapped bio
596 * @bio: bio being terminated
597 *
598 * Free pages allocated from bio_copy_user() and write back data
599 * to user space in case of a read.
600 */
601int bio_uncopy_user(struct bio *bio)
602{
603 struct bio_map_data *bmd = bio->bi_private;
FUJITA Tomonori81882762008-09-02 16:20:19 +0900604 int ret = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700605
FUJITA Tomonori81882762008-09-02 16:20:19 +0900606 if (!bio_flagged(bio, BIO_NULL_MAPPED))
607 ret = __bio_copy_iov(bio, bmd->iovecs, bmd->sgvecs,
608 bmd->nr_sgvecs, 1, bmd->is_our_pages);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700609 bio_free_map_data(bmd);
610 bio_put(bio);
611 return ret;
612}
613
614/**
FUJITA Tomonoric5dec1c2008-04-11 12:56:49 +0200615 * bio_copy_user_iov - copy user data to bio
Linus Torvalds1da177e2005-04-16 15:20:36 -0700616 * @q: destination block queue
FUJITA Tomonori152e2832008-08-28 16:17:06 +0900617 * @map_data: pointer to the rq_map_data holding pages (if necessary)
FUJITA Tomonoric5dec1c2008-04-11 12:56:49 +0200618 * @iov: the iovec.
619 * @iov_count: number of elements in the iovec
Linus Torvalds1da177e2005-04-16 15:20:36 -0700620 * @write_to_vm: bool indicating writing to pages or not
FUJITA Tomonoria3bce902008-08-28 16:17:05 +0900621 * @gfp_mask: memory allocation flags
Linus Torvalds1da177e2005-04-16 15:20:36 -0700622 *
623 * Prepares and returns a bio for indirect user io, bouncing data
624 * to/from kernel pages as necessary. Must be paired with
625 * call bio_uncopy_user() on io completion.
626 */
FUJITA Tomonori152e2832008-08-28 16:17:06 +0900627struct bio *bio_copy_user_iov(struct request_queue *q,
628 struct rq_map_data *map_data,
629 struct sg_iovec *iov, int iov_count,
630 int write_to_vm, gfp_t gfp_mask)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700631{
Linus Torvalds1da177e2005-04-16 15:20:36 -0700632 struct bio_map_data *bmd;
633 struct bio_vec *bvec;
634 struct page *page;
635 struct bio *bio;
636 int i, ret;
FUJITA Tomonoric5dec1c2008-04-11 12:56:49 +0200637 int nr_pages = 0;
638 unsigned int len = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700639
FUJITA Tomonoric5dec1c2008-04-11 12:56:49 +0200640 for (i = 0; i < iov_count; i++) {
641 unsigned long uaddr;
642 unsigned long end;
643 unsigned long start;
644
645 uaddr = (unsigned long)iov[i].iov_base;
646 end = (uaddr + iov[i].iov_len + PAGE_SIZE - 1) >> PAGE_SHIFT;
647 start = uaddr >> PAGE_SHIFT;
648
649 nr_pages += end - start;
650 len += iov[i].iov_len;
651 }
652
FUJITA Tomonoria3bce902008-08-28 16:17:05 +0900653 bmd = bio_alloc_map_data(nr_pages, iov_count, gfp_mask);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700654 if (!bmd)
655 return ERR_PTR(-ENOMEM);
656
Linus Torvalds1da177e2005-04-16 15:20:36 -0700657 ret = -ENOMEM;
FUJITA Tomonoria3bce902008-08-28 16:17:05 +0900658 bio = bio_alloc(gfp_mask, nr_pages);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700659 if (!bio)
660 goto out_bmd;
661
662 bio->bi_rw |= (!write_to_vm << BIO_RW);
663
664 ret = 0;
FUJITA Tomonori152e2832008-08-28 16:17:06 +0900665 i = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700666 while (len) {
FUJITA Tomonori152e2832008-08-28 16:17:06 +0900667 unsigned int bytes;
668
669 if (map_data)
670 bytes = 1U << (PAGE_SHIFT + map_data->page_order);
671 else
672 bytes = PAGE_SIZE;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700673
674 if (bytes > len)
675 bytes = len;
676
FUJITA Tomonori152e2832008-08-28 16:17:06 +0900677 if (map_data) {
678 if (i == map_data->nr_entries) {
679 ret = -ENOMEM;
680 break;
681 }
682 page = map_data->pages[i++];
683 } else
684 page = alloc_page(q->bounce_gfp | gfp_mask);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700685 if (!page) {
686 ret = -ENOMEM;
687 break;
688 }
689
Mike Christie0e75f902006-12-01 10:40:55 +0100690 if (bio_add_pc_page(q, bio, page, bytes, 0) < bytes)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700691 break;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700692
693 len -= bytes;
694 }
695
696 if (ret)
697 goto cleanup;
698
699 /*
700 * success
701 */
702 if (!write_to_vm) {
FUJITA Tomonori152e2832008-08-28 16:17:06 +0900703 ret = __bio_copy_iov(bio, bio->bi_io_vec, iov, iov_count, 0, 0);
FUJITA Tomonoric5dec1c2008-04-11 12:56:49 +0200704 if (ret)
705 goto cleanup;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700706 }
707
FUJITA Tomonori152e2832008-08-28 16:17:06 +0900708 bio_set_map_data(bmd, bio, iov, iov_count, map_data ? 0 : 1);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700709 return bio;
710cleanup:
FUJITA Tomonori152e2832008-08-28 16:17:06 +0900711 if (!map_data)
712 bio_for_each_segment(bvec, bio, i)
713 __free_page(bvec->bv_page);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700714
715 bio_put(bio);
716out_bmd:
717 bio_free_map_data(bmd);
718 return ERR_PTR(ret);
719}
720
FUJITA Tomonoric5dec1c2008-04-11 12:56:49 +0200721/**
722 * bio_copy_user - copy user data to bio
723 * @q: destination block queue
FUJITA Tomonori152e2832008-08-28 16:17:06 +0900724 * @map_data: pointer to the rq_map_data holding pages (if necessary)
FUJITA Tomonoric5dec1c2008-04-11 12:56:49 +0200725 * @uaddr: start of user address
726 * @len: length in bytes
727 * @write_to_vm: bool indicating writing to pages or not
FUJITA Tomonoria3bce902008-08-28 16:17:05 +0900728 * @gfp_mask: memory allocation flags
FUJITA Tomonoric5dec1c2008-04-11 12:56:49 +0200729 *
730 * Prepares and returns a bio for indirect user io, bouncing data
731 * to/from kernel pages as necessary. Must be paired with
732 * call bio_uncopy_user() on io completion.
733 */
FUJITA Tomonori152e2832008-08-28 16:17:06 +0900734struct bio *bio_copy_user(struct request_queue *q, struct rq_map_data *map_data,
735 unsigned long uaddr, unsigned int len,
736 int write_to_vm, gfp_t gfp_mask)
FUJITA Tomonoric5dec1c2008-04-11 12:56:49 +0200737{
738 struct sg_iovec iov;
739
740 iov.iov_base = (void __user *)uaddr;
741 iov.iov_len = len;
742
FUJITA Tomonori152e2832008-08-28 16:17:06 +0900743 return bio_copy_user_iov(q, map_data, &iov, 1, write_to_vm, gfp_mask);
FUJITA Tomonoric5dec1c2008-04-11 12:56:49 +0200744}
745
Jens Axboe165125e2007-07-24 09:28:11 +0200746static struct bio *__bio_map_user_iov(struct request_queue *q,
James Bottomley f1970ba2005-06-20 14:06:52 +0200747 struct block_device *bdev,
748 struct sg_iovec *iov, int iov_count,
FUJITA Tomonoria3bce902008-08-28 16:17:05 +0900749 int write_to_vm, gfp_t gfp_mask)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700750{
James Bottomley f1970ba2005-06-20 14:06:52 +0200751 int i, j;
752 int nr_pages = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700753 struct page **pages;
754 struct bio *bio;
James Bottomley f1970ba2005-06-20 14:06:52 +0200755 int cur_page = 0;
756 int ret, offset;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700757
James Bottomley f1970ba2005-06-20 14:06:52 +0200758 for (i = 0; i < iov_count; i++) {
759 unsigned long uaddr = (unsigned long)iov[i].iov_base;
760 unsigned long len = iov[i].iov_len;
761 unsigned long end = (uaddr + len + PAGE_SIZE - 1) >> PAGE_SHIFT;
762 unsigned long start = uaddr >> PAGE_SHIFT;
763
764 nr_pages += end - start;
765 /*
Mike Christiead2d7222006-12-01 10:40:20 +0100766 * buffer must be aligned to at least hardsector size for now
James Bottomley f1970ba2005-06-20 14:06:52 +0200767 */
Mike Christiead2d7222006-12-01 10:40:20 +0100768 if (uaddr & queue_dma_alignment(q))
James Bottomley f1970ba2005-06-20 14:06:52 +0200769 return ERR_PTR(-EINVAL);
770 }
771
772 if (!nr_pages)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700773 return ERR_PTR(-EINVAL);
774
FUJITA Tomonoria3bce902008-08-28 16:17:05 +0900775 bio = bio_alloc(gfp_mask, nr_pages);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700776 if (!bio)
777 return ERR_PTR(-ENOMEM);
778
779 ret = -ENOMEM;
FUJITA Tomonoria3bce902008-08-28 16:17:05 +0900780 pages = kcalloc(nr_pages, sizeof(struct page *), gfp_mask);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700781 if (!pages)
782 goto out;
783
James Bottomley f1970ba2005-06-20 14:06:52 +0200784 for (i = 0; i < iov_count; i++) {
785 unsigned long uaddr = (unsigned long)iov[i].iov_base;
786 unsigned long len = iov[i].iov_len;
787 unsigned long end = (uaddr + len + PAGE_SIZE - 1) >> PAGE_SHIFT;
788 unsigned long start = uaddr >> PAGE_SHIFT;
789 const int local_nr_pages = end - start;
790 const int page_limit = cur_page + local_nr_pages;
791
Nick Pigginf5dd33c2008-07-25 19:45:25 -0700792 ret = get_user_pages_fast(uaddr, local_nr_pages,
793 write_to_vm, &pages[cur_page]);
Jens Axboe99172152006-06-16 13:02:29 +0200794 if (ret < local_nr_pages) {
795 ret = -EFAULT;
James Bottomley f1970ba2005-06-20 14:06:52 +0200796 goto out_unmap;
Jens Axboe99172152006-06-16 13:02:29 +0200797 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700798
James Bottomley f1970ba2005-06-20 14:06:52 +0200799 offset = uaddr & ~PAGE_MASK;
800 for (j = cur_page; j < page_limit; j++) {
801 unsigned int bytes = PAGE_SIZE - offset;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700802
James Bottomley f1970ba2005-06-20 14:06:52 +0200803 if (len <= 0)
804 break;
805
806 if (bytes > len)
807 bytes = len;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700808
James Bottomley f1970ba2005-06-20 14:06:52 +0200809 /*
810 * sorry...
811 */
Mike Christiedefd94b2005-12-05 02:37:06 -0600812 if (bio_add_pc_page(q, bio, pages[j], bytes, offset) <
813 bytes)
James Bottomley f1970ba2005-06-20 14:06:52 +0200814 break;
815
816 len -= bytes;
817 offset = 0;
818 }
819
820 cur_page = j;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700821 /*
James Bottomley f1970ba2005-06-20 14:06:52 +0200822 * release the pages we didn't map into the bio, if any
Linus Torvalds1da177e2005-04-16 15:20:36 -0700823 */
James Bottomley f1970ba2005-06-20 14:06:52 +0200824 while (j < page_limit)
825 page_cache_release(pages[j++]);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700826 }
827
Linus Torvalds1da177e2005-04-16 15:20:36 -0700828 kfree(pages);
829
830 /*
831 * set data direction, and check if mapped pages need bouncing
832 */
833 if (!write_to_vm)
834 bio->bi_rw |= (1 << BIO_RW);
835
James Bottomley f1970ba2005-06-20 14:06:52 +0200836 bio->bi_bdev = bdev;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700837 bio->bi_flags |= (1 << BIO_USER_MAPPED);
838 return bio;
James Bottomley f1970ba2005-06-20 14:06:52 +0200839
840 out_unmap:
841 for (i = 0; i < nr_pages; i++) {
842 if(!pages[i])
843 break;
844 page_cache_release(pages[i]);
845 }
846 out:
Linus Torvalds1da177e2005-04-16 15:20:36 -0700847 kfree(pages);
848 bio_put(bio);
849 return ERR_PTR(ret);
850}
851
852/**
853 * bio_map_user - map user address into bio
Jens Axboe165125e2007-07-24 09:28:11 +0200854 * @q: the struct request_queue for the bio
Linus Torvalds1da177e2005-04-16 15:20:36 -0700855 * @bdev: destination block device
856 * @uaddr: start of user address
857 * @len: length in bytes
858 * @write_to_vm: bool indicating writing to pages or not
FUJITA Tomonoria3bce902008-08-28 16:17:05 +0900859 * @gfp_mask: memory allocation flags
Linus Torvalds1da177e2005-04-16 15:20:36 -0700860 *
861 * Map the user space address into a bio suitable for io to a block
862 * device. Returns an error pointer in case of error.
863 */
Jens Axboe165125e2007-07-24 09:28:11 +0200864struct bio *bio_map_user(struct request_queue *q, struct block_device *bdev,
FUJITA Tomonoria3bce902008-08-28 16:17:05 +0900865 unsigned long uaddr, unsigned int len, int write_to_vm,
866 gfp_t gfp_mask)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700867{
James Bottomley f1970ba2005-06-20 14:06:52 +0200868 struct sg_iovec iov;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700869
viro@ZenIV.linux.org.uk3f703532005-09-09 16:53:56 +0100870 iov.iov_base = (void __user *)uaddr;
James Bottomley f1970ba2005-06-20 14:06:52 +0200871 iov.iov_len = len;
872
FUJITA Tomonoria3bce902008-08-28 16:17:05 +0900873 return bio_map_user_iov(q, bdev, &iov, 1, write_to_vm, gfp_mask);
James Bottomley f1970ba2005-06-20 14:06:52 +0200874}
875
876/**
877 * bio_map_user_iov - map user sg_iovec table into bio
Jens Axboe165125e2007-07-24 09:28:11 +0200878 * @q: the struct request_queue for the bio
James Bottomley f1970ba2005-06-20 14:06:52 +0200879 * @bdev: destination block device
880 * @iov: the iovec.
881 * @iov_count: number of elements in the iovec
882 * @write_to_vm: bool indicating writing to pages or not
FUJITA Tomonoria3bce902008-08-28 16:17:05 +0900883 * @gfp_mask: memory allocation flags
James Bottomley f1970ba2005-06-20 14:06:52 +0200884 *
885 * Map the user space address into a bio suitable for io to a block
886 * device. Returns an error pointer in case of error.
887 */
Jens Axboe165125e2007-07-24 09:28:11 +0200888struct bio *bio_map_user_iov(struct request_queue *q, struct block_device *bdev,
James Bottomley f1970ba2005-06-20 14:06:52 +0200889 struct sg_iovec *iov, int iov_count,
FUJITA Tomonoria3bce902008-08-28 16:17:05 +0900890 int write_to_vm, gfp_t gfp_mask)
James Bottomley f1970ba2005-06-20 14:06:52 +0200891{
892 struct bio *bio;
James Bottomley f1970ba2005-06-20 14:06:52 +0200893
FUJITA Tomonoria3bce902008-08-28 16:17:05 +0900894 bio = __bio_map_user_iov(q, bdev, iov, iov_count, write_to_vm,
895 gfp_mask);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700896 if (IS_ERR(bio))
897 return bio;
898
899 /*
900 * subtle -- if __bio_map_user() ended up bouncing a bio,
901 * it would normally disappear when its bi_end_io is run.
902 * however, we need it for the unmap, so grab an extra
903 * reference to it
904 */
905 bio_get(bio);
906
Mike Christie0e75f902006-12-01 10:40:55 +0100907 return bio;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700908}
909
910static void __bio_unmap_user(struct bio *bio)
911{
912 struct bio_vec *bvec;
913 int i;
914
915 /*
916 * make sure we dirty pages we wrote to
917 */
918 __bio_for_each_segment(bvec, bio, i, 0) {
919 if (bio_data_dir(bio) == READ)
920 set_page_dirty_lock(bvec->bv_page);
921
922 page_cache_release(bvec->bv_page);
923 }
924
925 bio_put(bio);
926}
927
928/**
929 * bio_unmap_user - unmap a bio
930 * @bio: the bio being unmapped
931 *
932 * Unmap a bio previously mapped by bio_map_user(). Must be called with
933 * a process context.
934 *
935 * bio_unmap_user() may sleep.
936 */
937void bio_unmap_user(struct bio *bio)
938{
939 __bio_unmap_user(bio);
940 bio_put(bio);
941}
942
NeilBrown6712ecf2007-09-27 12:47:43 +0200943static void bio_map_kern_endio(struct bio *bio, int err)
Jens Axboeb8238252005-06-20 14:05:27 +0200944{
Jens Axboeb8238252005-06-20 14:05:27 +0200945 bio_put(bio);
Jens Axboeb8238252005-06-20 14:05:27 +0200946}
947
948
Jens Axboe165125e2007-07-24 09:28:11 +0200949static struct bio *__bio_map_kern(struct request_queue *q, void *data,
Al Viro27496a82005-10-21 03:20:48 -0400950 unsigned int len, gfp_t gfp_mask)
Mike Christie df46b9a2005-06-20 14:04:44 +0200951{
952 unsigned long kaddr = (unsigned long)data;
953 unsigned long end = (kaddr + len + PAGE_SIZE - 1) >> PAGE_SHIFT;
954 unsigned long start = kaddr >> PAGE_SHIFT;
955 const int nr_pages = end - start;
956 int offset, i;
957 struct bio *bio;
958
959 bio = bio_alloc(gfp_mask, nr_pages);
960 if (!bio)
961 return ERR_PTR(-ENOMEM);
962
963 offset = offset_in_page(kaddr);
964 for (i = 0; i < nr_pages; i++) {
965 unsigned int bytes = PAGE_SIZE - offset;
966
967 if (len <= 0)
968 break;
969
970 if (bytes > len)
971 bytes = len;
972
Mike Christiedefd94b2005-12-05 02:37:06 -0600973 if (bio_add_pc_page(q, bio, virt_to_page(data), bytes,
974 offset) < bytes)
Mike Christie df46b9a2005-06-20 14:04:44 +0200975 break;
976
977 data += bytes;
978 len -= bytes;
979 offset = 0;
980 }
981
Jens Axboeb8238252005-06-20 14:05:27 +0200982 bio->bi_end_io = bio_map_kern_endio;
Mike Christie df46b9a2005-06-20 14:04:44 +0200983 return bio;
984}
985
986/**
987 * bio_map_kern - map kernel address into bio
Jens Axboe165125e2007-07-24 09:28:11 +0200988 * @q: the struct request_queue for the bio
Mike Christie df46b9a2005-06-20 14:04:44 +0200989 * @data: pointer to buffer to map
990 * @len: length in bytes
991 * @gfp_mask: allocation flags for bio allocation
992 *
993 * Map the kernel address into a bio suitable for io to a block
994 * device. Returns an error pointer in case of error.
995 */
Jens Axboe165125e2007-07-24 09:28:11 +0200996struct bio *bio_map_kern(struct request_queue *q, void *data, unsigned int len,
Al Viro27496a82005-10-21 03:20:48 -0400997 gfp_t gfp_mask)
Mike Christie df46b9a2005-06-20 14:04:44 +0200998{
999 struct bio *bio;
1000
1001 bio = __bio_map_kern(q, data, len, gfp_mask);
1002 if (IS_ERR(bio))
1003 return bio;
1004
1005 if (bio->bi_size == len)
1006 return bio;
1007
1008 /*
1009 * Don't support partial mappings.
1010 */
1011 bio_put(bio);
1012 return ERR_PTR(-EINVAL);
1013}
1014
FUJITA Tomonori68154e92008-04-25 12:47:50 +02001015static void bio_copy_kern_endio(struct bio *bio, int err)
1016{
1017 struct bio_vec *bvec;
1018 const int read = bio_data_dir(bio) == READ;
FUJITA Tomonori76029ff2008-08-25 20:36:08 +02001019 struct bio_map_data *bmd = bio->bi_private;
FUJITA Tomonori68154e92008-04-25 12:47:50 +02001020 int i;
FUJITA Tomonori76029ff2008-08-25 20:36:08 +02001021 char *p = bmd->sgvecs[0].iov_base;
FUJITA Tomonori68154e92008-04-25 12:47:50 +02001022
1023 __bio_for_each_segment(bvec, bio, i, 0) {
1024 char *addr = page_address(bvec->bv_page);
FUJITA Tomonori76029ff2008-08-25 20:36:08 +02001025 int len = bmd->iovecs[i].bv_len;
FUJITA Tomonori68154e92008-04-25 12:47:50 +02001026
1027 if (read && !err)
FUJITA Tomonori76029ff2008-08-25 20:36:08 +02001028 memcpy(p, addr, len);
FUJITA Tomonori68154e92008-04-25 12:47:50 +02001029
1030 __free_page(bvec->bv_page);
FUJITA Tomonori76029ff2008-08-25 20:36:08 +02001031 p += len;
FUJITA Tomonori68154e92008-04-25 12:47:50 +02001032 }
1033
FUJITA Tomonori76029ff2008-08-25 20:36:08 +02001034 bio_free_map_data(bmd);
FUJITA Tomonori68154e92008-04-25 12:47:50 +02001035 bio_put(bio);
1036}
1037
1038/**
1039 * bio_copy_kern - copy kernel address into bio
1040 * @q: the struct request_queue for the bio
1041 * @data: pointer to buffer to copy
1042 * @len: length in bytes
1043 * @gfp_mask: allocation flags for bio and page allocation
Randy Dunlapffee0252008-04-30 09:08:54 +02001044 * @reading: data direction is READ
FUJITA Tomonori68154e92008-04-25 12:47:50 +02001045 *
1046 * copy the kernel address into a bio suitable for io to a block
1047 * device. Returns an error pointer in case of error.
1048 */
1049struct bio *bio_copy_kern(struct request_queue *q, void *data, unsigned int len,
1050 gfp_t gfp_mask, int reading)
1051{
FUJITA Tomonori68154e92008-04-25 12:47:50 +02001052 struct bio *bio;
1053 struct bio_vec *bvec;
FUJITA Tomonori4d8ab622008-08-28 15:05:57 +09001054 int i;
FUJITA Tomonori68154e92008-04-25 12:47:50 +02001055
FUJITA Tomonori4d8ab622008-08-28 15:05:57 +09001056 bio = bio_copy_user(q, NULL, (unsigned long)data, len, 1, gfp_mask);
1057 if (IS_ERR(bio))
1058 return bio;
FUJITA Tomonori68154e92008-04-25 12:47:50 +02001059
1060 if (!reading) {
1061 void *p = data;
1062
1063 bio_for_each_segment(bvec, bio, i) {
1064 char *addr = page_address(bvec->bv_page);
1065
1066 memcpy(addr, p, bvec->bv_len);
1067 p += bvec->bv_len;
1068 }
1069 }
1070
FUJITA Tomonori68154e92008-04-25 12:47:50 +02001071 bio->bi_end_io = bio_copy_kern_endio;
FUJITA Tomonori76029ff2008-08-25 20:36:08 +02001072
FUJITA Tomonori68154e92008-04-25 12:47:50 +02001073 return bio;
FUJITA Tomonori68154e92008-04-25 12:47:50 +02001074}
1075
Linus Torvalds1da177e2005-04-16 15:20:36 -07001076/*
1077 * bio_set_pages_dirty() and bio_check_pages_dirty() are support functions
1078 * for performing direct-IO in BIOs.
1079 *
1080 * The problem is that we cannot run set_page_dirty() from interrupt context
1081 * because the required locks are not interrupt-safe. So what we can do is to
1082 * mark the pages dirty _before_ performing IO. And in interrupt context,
1083 * check that the pages are still dirty. If so, fine. If not, redirty them
1084 * in process context.
1085 *
1086 * We special-case compound pages here: normally this means reads into hugetlb
1087 * pages. The logic in here doesn't really work right for compound pages
1088 * because the VM does not uniformly chase down the head page in all cases.
1089 * But dirtiness of compound pages is pretty meaningless anyway: the VM doesn't
1090 * handle them at all. So we skip compound pages here at an early stage.
1091 *
1092 * Note that this code is very hard to test under normal circumstances because
1093 * direct-io pins the pages with get_user_pages(). This makes
1094 * is_page_cache_freeable return false, and the VM will not clean the pages.
1095 * But other code (eg, pdflush) could clean the pages if they are mapped
1096 * pagecache.
1097 *
1098 * Simply disabling the call to bio_set_pages_dirty() is a good way to test the
1099 * deferred bio dirtying paths.
1100 */
1101
1102/*
1103 * bio_set_pages_dirty() will mark all the bio's pages as dirty.
1104 */
1105void bio_set_pages_dirty(struct bio *bio)
1106{
1107 struct bio_vec *bvec = bio->bi_io_vec;
1108 int i;
1109
1110 for (i = 0; i < bio->bi_vcnt; i++) {
1111 struct page *page = bvec[i].bv_page;
1112
1113 if (page && !PageCompound(page))
1114 set_page_dirty_lock(page);
1115 }
1116}
1117
Adrian Bunk86b6c7a2008-02-18 13:48:32 +01001118static void bio_release_pages(struct bio *bio)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001119{
1120 struct bio_vec *bvec = bio->bi_io_vec;
1121 int i;
1122
1123 for (i = 0; i < bio->bi_vcnt; i++) {
1124 struct page *page = bvec[i].bv_page;
1125
1126 if (page)
1127 put_page(page);
1128 }
1129}
1130
1131/*
1132 * bio_check_pages_dirty() will check that all the BIO's pages are still dirty.
1133 * If they are, then fine. If, however, some pages are clean then they must
1134 * have been written out during the direct-IO read. So we take another ref on
1135 * the BIO and the offending pages and re-dirty the pages in process context.
1136 *
1137 * It is expected that bio_check_pages_dirty() will wholly own the BIO from
1138 * here on. It will run one page_cache_release() against each page and will
1139 * run one bio_put() against the BIO.
1140 */
1141
David Howells65f27f32006-11-22 14:55:48 +00001142static void bio_dirty_fn(struct work_struct *work);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001143
David Howells65f27f32006-11-22 14:55:48 +00001144static DECLARE_WORK(bio_dirty_work, bio_dirty_fn);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001145static DEFINE_SPINLOCK(bio_dirty_lock);
1146static struct bio *bio_dirty_list;
1147
1148/*
1149 * This runs in process context
1150 */
David Howells65f27f32006-11-22 14:55:48 +00001151static void bio_dirty_fn(struct work_struct *work)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001152{
1153 unsigned long flags;
1154 struct bio *bio;
1155
1156 spin_lock_irqsave(&bio_dirty_lock, flags);
1157 bio = bio_dirty_list;
1158 bio_dirty_list = NULL;
1159 spin_unlock_irqrestore(&bio_dirty_lock, flags);
1160
1161 while (bio) {
1162 struct bio *next = bio->bi_private;
1163
1164 bio_set_pages_dirty(bio);
1165 bio_release_pages(bio);
1166 bio_put(bio);
1167 bio = next;
1168 }
1169}
1170
1171void bio_check_pages_dirty(struct bio *bio)
1172{
1173 struct bio_vec *bvec = bio->bi_io_vec;
1174 int nr_clean_pages = 0;
1175 int i;
1176
1177 for (i = 0; i < bio->bi_vcnt; i++) {
1178 struct page *page = bvec[i].bv_page;
1179
1180 if (PageDirty(page) || PageCompound(page)) {
1181 page_cache_release(page);
1182 bvec[i].bv_page = NULL;
1183 } else {
1184 nr_clean_pages++;
1185 }
1186 }
1187
1188 if (nr_clean_pages) {
1189 unsigned long flags;
1190
1191 spin_lock_irqsave(&bio_dirty_lock, flags);
1192 bio->bi_private = bio_dirty_list;
1193 bio_dirty_list = bio;
1194 spin_unlock_irqrestore(&bio_dirty_lock, flags);
1195 schedule_work(&bio_dirty_work);
1196 } else {
1197 bio_put(bio);
1198 }
1199}
1200
1201/**
1202 * bio_endio - end I/O on a bio
1203 * @bio: bio
Linus Torvalds1da177e2005-04-16 15:20:36 -07001204 * @error: error, if any
1205 *
1206 * Description:
NeilBrown6712ecf2007-09-27 12:47:43 +02001207 * bio_endio() will end I/O on the whole bio. bio_endio() is the
NeilBrown5bb23a62007-09-27 12:46:13 +02001208 * preferred way to end I/O on a bio, it takes care of clearing
1209 * BIO_UPTODATE on error. @error is 0 on success, and and one of the
1210 * established -Exxxx (-EIO, for instance) error values in case
1211 * something went wrong. Noone should call bi_end_io() directly on a
1212 * bio unless they own it and thus know that it has an end_io
1213 * function.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001214 **/
NeilBrown6712ecf2007-09-27 12:47:43 +02001215void bio_endio(struct bio *bio, int error)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001216{
1217 if (error)
1218 clear_bit(BIO_UPTODATE, &bio->bi_flags);
NeilBrown9cc54d42007-09-27 12:46:12 +02001219 else if (!test_bit(BIO_UPTODATE, &bio->bi_flags))
1220 error = -EIO;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001221
NeilBrown5bb23a62007-09-27 12:46:13 +02001222 if (bio->bi_end_io)
NeilBrown6712ecf2007-09-27 12:47:43 +02001223 bio->bi_end_io(bio, error);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001224}
1225
1226void bio_pair_release(struct bio_pair *bp)
1227{
1228 if (atomic_dec_and_test(&bp->cnt)) {
1229 struct bio *master = bp->bio1.bi_private;
1230
NeilBrown6712ecf2007-09-27 12:47:43 +02001231 bio_endio(master, bp->error);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001232 mempool_free(bp, bp->bio2.bi_private);
1233 }
1234}
1235
NeilBrown6712ecf2007-09-27 12:47:43 +02001236static void bio_pair_end_1(struct bio *bi, int err)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001237{
1238 struct bio_pair *bp = container_of(bi, struct bio_pair, bio1);
1239
1240 if (err)
1241 bp->error = err;
1242
Linus Torvalds1da177e2005-04-16 15:20:36 -07001243 bio_pair_release(bp);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001244}
1245
NeilBrown6712ecf2007-09-27 12:47:43 +02001246static void bio_pair_end_2(struct bio *bi, int err)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001247{
1248 struct bio_pair *bp = container_of(bi, struct bio_pair, bio2);
1249
1250 if (err)
1251 bp->error = err;
1252
Linus Torvalds1da177e2005-04-16 15:20:36 -07001253 bio_pair_release(bp);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001254}
1255
1256/*
1257 * split a bio - only worry about a bio with a single page
1258 * in it's iovec
1259 */
Denis ChengRq6feef532008-10-09 08:57:05 +02001260struct bio_pair *bio_split(struct bio *bi, int first_sectors)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001261{
Denis ChengRq6feef532008-10-09 08:57:05 +02001262 struct bio_pair *bp = mempool_alloc(bio_split_pool, GFP_NOIO);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001263
1264 if (!bp)
1265 return bp;
1266
Arnaldo Carvalho de Melo5f3ea372008-10-30 08:34:33 +01001267 trace_block_split(bdev_get_queue(bi->bi_bdev), bi,
Jens Axboe2056a782006-03-23 20:00:26 +01001268 bi->bi_sector + first_sectors);
1269
Linus Torvalds1da177e2005-04-16 15:20:36 -07001270 BUG_ON(bi->bi_vcnt != 1);
1271 BUG_ON(bi->bi_idx != 0);
1272 atomic_set(&bp->cnt, 3);
1273 bp->error = 0;
1274 bp->bio1 = *bi;
1275 bp->bio2 = *bi;
1276 bp->bio2.bi_sector += first_sectors;
1277 bp->bio2.bi_size -= first_sectors << 9;
1278 bp->bio1.bi_size = first_sectors << 9;
1279
1280 bp->bv1 = bi->bi_io_vec[0];
1281 bp->bv2 = bi->bi_io_vec[0];
1282 bp->bv2.bv_offset += first_sectors << 9;
1283 bp->bv2.bv_len -= first_sectors << 9;
1284 bp->bv1.bv_len = first_sectors << 9;
1285
1286 bp->bio1.bi_io_vec = &bp->bv1;
1287 bp->bio2.bi_io_vec = &bp->bv2;
1288
NeilBrowna2eb0c12006-05-22 22:35:27 -07001289 bp->bio1.bi_max_vecs = 1;
1290 bp->bio2.bi_max_vecs = 1;
1291
Linus Torvalds1da177e2005-04-16 15:20:36 -07001292 bp->bio1.bi_end_io = bio_pair_end_1;
1293 bp->bio2.bi_end_io = bio_pair_end_2;
1294
1295 bp->bio1.bi_private = bi;
Denis ChengRq6feef532008-10-09 08:57:05 +02001296 bp->bio2.bi_private = bio_split_pool;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001297
Martin K. Petersen7ba1ba12008-06-30 20:04:41 +02001298 if (bio_integrity(bi))
1299 bio_integrity_split(bi, bp, first_sectors);
1300
Linus Torvalds1da177e2005-04-16 15:20:36 -07001301 return bp;
1302}
1303
Martin K. Petersenad3316b2008-10-01 22:42:53 -04001304/**
1305 * bio_sector_offset - Find hardware sector offset in bio
1306 * @bio: bio to inspect
1307 * @index: bio_vec index
1308 * @offset: offset in bv_page
1309 *
1310 * Return the number of hardware sectors between beginning of bio
1311 * and an end point indicated by a bio_vec index and an offset
1312 * within that vector's page.
1313 */
1314sector_t bio_sector_offset(struct bio *bio, unsigned short index,
1315 unsigned int offset)
1316{
1317 unsigned int sector_sz = queue_hardsect_size(bio->bi_bdev->bd_disk->queue);
1318 struct bio_vec *bv;
1319 sector_t sectors;
1320 int i;
1321
1322 sectors = 0;
1323
1324 if (index >= bio->bi_idx)
1325 index = bio->bi_vcnt - 1;
1326
1327 __bio_for_each_segment(bv, bio, i, 0) {
1328 if (i == index) {
1329 if (offset > bv->bv_offset)
1330 sectors += (offset - bv->bv_offset) / sector_sz;
1331 break;
1332 }
1333
1334 sectors += bv->bv_len / sector_sz;
1335 }
1336
1337 return sectors;
1338}
1339EXPORT_SYMBOL(bio_sector_offset);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001340
1341/*
1342 * create memory pools for biovec's in a bio_set.
1343 * use the global biovec slabs created for general use.
1344 */
Jens Axboe59725112007-04-02 10:06:42 +02001345static int biovec_create_pools(struct bio_set *bs, int pool_entries)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001346{
1347 int i;
1348
1349 for (i = 0; i < BIOVEC_NR_POOLS; i++) {
1350 struct biovec_slab *bp = bvec_slabs + i;
1351 mempool_t **bvp = bs->bvec_pools + i;
1352
Matthew Dobson93d23412006-03-26 01:37:50 -08001353 *bvp = mempool_create_slab_pool(pool_entries, bp->slab);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001354 if (!*bvp)
1355 return -ENOMEM;
1356 }
1357 return 0;
1358}
1359
1360static void biovec_free_pools(struct bio_set *bs)
1361{
1362 int i;
1363
1364 for (i = 0; i < BIOVEC_NR_POOLS; i++) {
1365 mempool_t *bvp = bs->bvec_pools[i];
1366
1367 if (bvp)
1368 mempool_destroy(bvp);
1369 }
1370
1371}
1372
1373void bioset_free(struct bio_set *bs)
1374{
1375 if (bs->bio_pool)
1376 mempool_destroy(bs->bio_pool);
1377
Martin K. Petersen7ba1ba12008-06-30 20:04:41 +02001378 bioset_integrity_free(bs);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001379 biovec_free_pools(bs);
1380
1381 kfree(bs);
1382}
1383
Jens Axboe59725112007-04-02 10:06:42 +02001384struct bio_set *bioset_create(int bio_pool_size, int bvec_pool_size)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001385{
Oliver Neukum11b0b5a2006-03-25 03:08:13 -08001386 struct bio_set *bs = kzalloc(sizeof(*bs), GFP_KERNEL);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001387
1388 if (!bs)
1389 return NULL;
1390
Matthew Dobson93d23412006-03-26 01:37:50 -08001391 bs->bio_pool = mempool_create_slab_pool(bio_pool_size, bio_slab);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001392 if (!bs->bio_pool)
1393 goto bad;
1394
Martin K. Petersen7ba1ba12008-06-30 20:04:41 +02001395 if (bioset_integrity_create(bs, bio_pool_size))
1396 goto bad;
1397
Jens Axboe59725112007-04-02 10:06:42 +02001398 if (!biovec_create_pools(bs, bvec_pool_size))
Linus Torvalds1da177e2005-04-16 15:20:36 -07001399 return bs;
1400
1401bad:
1402 bioset_free(bs);
1403 return NULL;
1404}
1405
1406static void __init biovec_init_slabs(void)
1407{
1408 int i;
1409
1410 for (i = 0; i < BIOVEC_NR_POOLS; i++) {
1411 int size;
1412 struct biovec_slab *bvs = bvec_slabs + i;
1413
1414 size = bvs->nr_vecs * sizeof(struct bio_vec);
1415 bvs->slab = kmem_cache_create(bvs->name, size, 0,
Paul Mundt20c2df82007-07-20 10:11:58 +09001416 SLAB_HWCACHE_ALIGN|SLAB_PANIC, NULL);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001417 }
1418}
1419
1420static int __init init_bio(void)
1421{
Christoph Lameter0a31bd52007-05-06 14:49:57 -07001422 bio_slab = KMEM_CACHE(bio, SLAB_HWCACHE_ALIGN|SLAB_PANIC);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001423
Martin K. Petersen7ba1ba12008-06-30 20:04:41 +02001424 bio_integrity_init_slab();
Linus Torvalds1da177e2005-04-16 15:20:36 -07001425 biovec_init_slabs();
1426
Jens Axboe59725112007-04-02 10:06:42 +02001427 fs_bio_set = bioset_create(BIO_POOL_SIZE, 2);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001428 if (!fs_bio_set)
1429 panic("bio: can't allocate bios\n");
1430
Matthew Dobson0eaae62a2006-03-26 01:37:47 -08001431 bio_split_pool = mempool_create_kmalloc_pool(BIO_SPLIT_ENTRIES,
1432 sizeof(struct bio_pair));
Linus Torvalds1da177e2005-04-16 15:20:36 -07001433 if (!bio_split_pool)
1434 panic("bio: can't create split pool\n");
1435
1436 return 0;
1437}
1438
1439subsys_initcall(init_bio);
1440
1441EXPORT_SYMBOL(bio_alloc);
Jens Axboe0a0d96b2008-09-11 13:17:37 +02001442EXPORT_SYMBOL(bio_kmalloc);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001443EXPORT_SYMBOL(bio_put);
Peter Osterlund36763472005-09-06 15:16:42 -07001444EXPORT_SYMBOL(bio_free);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001445EXPORT_SYMBOL(bio_endio);
1446EXPORT_SYMBOL(bio_init);
1447EXPORT_SYMBOL(__bio_clone);
1448EXPORT_SYMBOL(bio_clone);
1449EXPORT_SYMBOL(bio_phys_segments);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001450EXPORT_SYMBOL(bio_add_page);
Mike Christie6e68af62005-11-11 05:30:27 -06001451EXPORT_SYMBOL(bio_add_pc_page);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001452EXPORT_SYMBOL(bio_get_nr_vecs);
Jens Axboe40044ce2008-03-17 21:14:40 +01001453EXPORT_SYMBOL(bio_map_user);
1454EXPORT_SYMBOL(bio_unmap_user);
Mike Christie df46b9a2005-06-20 14:04:44 +02001455EXPORT_SYMBOL(bio_map_kern);
FUJITA Tomonori68154e92008-04-25 12:47:50 +02001456EXPORT_SYMBOL(bio_copy_kern);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001457EXPORT_SYMBOL(bio_pair_release);
1458EXPORT_SYMBOL(bio_split);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001459EXPORT_SYMBOL(bio_copy_user);
1460EXPORT_SYMBOL(bio_uncopy_user);
1461EXPORT_SYMBOL(bioset_create);
1462EXPORT_SYMBOL(bioset_free);
1463EXPORT_SYMBOL(bio_alloc_bioset);