blob: 4d94eb8bcbccc224bd0206cb435b1f052ddaaebf [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
2 * 2.5 block I/O model
3 *
4 * Copyright (C) 2001 Jens Axboe <axboe@suse.de>
5 *
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License version 2 as
8 * published by the Free Software Foundation.
9 *
10 * This program is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
Tejun Heo7cc01582010-08-03 13:14:58 +020012 *
Linus Torvalds1da177e2005-04-16 15:20:36 -070013 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 * GNU General Public License for more details.
15 *
16 * You should have received a copy of the GNU General Public Licens
17 * along with this program; if not, write to the Free Software
18 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-
19 */
20#ifndef __LINUX_BIO_H
21#define __LINUX_BIO_H
22
23#include <linux/highmem.h>
24#include <linux/mempool.h>
Jens Axboe22e2c502005-06-27 10:55:12 +020025#include <linux/ioprio.h>
Paul Gortmaker187f1882011-11-23 20:12:59 -050026#include <linux/bug.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070027
David Howells02a5e0a2007-08-11 22:34:32 +020028#ifdef CONFIG_BLOCK
29
Linus Torvalds1da177e2005-04-16 15:20:36 -070030#include <asm/io.h>
31
Tejun Heo7cc01582010-08-03 13:14:58 +020032/* struct bio, bio_vec and BIO_* flags are defined in blk_types.h */
33#include <linux/blk_types.h>
34
Linus Torvalds1da177e2005-04-16 15:20:36 -070035#define BIO_DEBUG
36
37#ifdef BIO_DEBUG
38#define BIO_BUG_ON BUG_ON
39#else
40#define BIO_BUG_ON
41#endif
42
Alexey Dobriyand84a8472006-06-25 05:49:32 -070043#define BIO_MAX_PAGES 256
Linus Torvalds1da177e2005-04-16 15:20:36 -070044#define BIO_MAX_SIZE (BIO_MAX_PAGES << PAGE_CACHE_SHIFT)
45#define BIO_MAX_SECTORS (BIO_MAX_SIZE >> 9)
46
47/*
Jens Axboe22e2c502005-06-27 10:55:12 +020048 * upper 16 bits of bi_rw define the io priority of this bio
49 */
50#define BIO_PRIO_SHIFT (8 * sizeof(unsigned long) - IOPRIO_BITS)
51#define bio_prio(bio) ((bio)->bi_rw >> BIO_PRIO_SHIFT)
52#define bio_prio_valid(bio) ioprio_valid(bio_prio(bio))
53
54#define bio_set_prio(bio, prio) do { \
55 WARN_ON(prio >= (1 << IOPRIO_BITS)); \
56 (bio)->bi_rw &= ((1UL << BIO_PRIO_SHIFT) - 1); \
57 (bio)->bi_rw |= ((unsigned long) (prio) << BIO_PRIO_SHIFT); \
58} while (0)
59
60/*
Linus Torvalds1da177e2005-04-16 15:20:36 -070061 * various member access, note that bio_data should of course not be used
62 * on highmem page vectors
63 */
64#define bio_iovec_idx(bio, idx) (&((bio)->bi_io_vec[(idx)]))
65#define bio_iovec(bio) bio_iovec_idx((bio), (bio)->bi_idx)
66#define bio_page(bio) bio_iovec((bio))->bv_page
67#define bio_offset(bio) bio_iovec((bio))->bv_offset
68#define bio_segments(bio) ((bio)->bi_vcnt - (bio)->bi_idx)
69#define bio_sectors(bio) ((bio)->bi_size >> 9)
Jens Axboebf2de6f2007-09-27 13:01:25 +020070
Tejun Heo2e46e8b2009-05-07 22:24:41 +090071static inline unsigned int bio_cur_bytes(struct bio *bio)
Jens Axboebf2de6f2007-09-27 13:01:25 +020072{
73 if (bio->bi_vcnt)
Tejun Heo2e46e8b2009-05-07 22:24:41 +090074 return bio_iovec(bio)->bv_len;
David Woodhousefb2dce82008-08-05 18:01:53 +010075 else /* dataless requests such as discard */
Tejun Heo2e46e8b2009-05-07 22:24:41 +090076 return bio->bi_size;
Jens Axboebf2de6f2007-09-27 13:01:25 +020077}
78
79static inline void *bio_data(struct bio *bio)
80{
81 if (bio->bi_vcnt)
82 return page_address(bio_page(bio)) + bio_offset(bio);
83
84 return NULL;
85}
Linus Torvalds1da177e2005-04-16 15:20:36 -070086
Jens Axboe392ddc32008-12-23 12:42:54 +010087static inline int bio_has_allocated_vec(struct bio *bio)
88{
89 return bio->bi_io_vec && bio->bi_io_vec != bio->bi_inline_vecs;
90}
91
Linus Torvalds1da177e2005-04-16 15:20:36 -070092/*
93 * will die
94 */
95#define bio_to_phys(bio) (page_to_phys(bio_page((bio))) + (unsigned long) bio_offset((bio)))
96#define bvec_to_phys(bv) (page_to_phys((bv)->bv_page) + (unsigned long) (bv)->bv_offset)
97
98/*
99 * queues that have highmem support enabled may still need to revert to
100 * PIO transfers occasionally and thus map high pages temporarily. For
101 * permanent PIO fall back, user is probably better off disabling highmem
102 * I/O completely on that queue (see ide-dma for example)
103 */
104#define __bio_kmap_atomic(bio, idx, kmtype) \
Cong Wange8e3c3d2011-11-25 23:14:27 +0800105 (kmap_atomic(bio_iovec_idx((bio), (idx))->bv_page) + \
Linus Torvalds1da177e2005-04-16 15:20:36 -0700106 bio_iovec_idx((bio), (idx))->bv_offset)
107
Cong Wange8e3c3d2011-11-25 23:14:27 +0800108#define __bio_kunmap_atomic(addr, kmtype) kunmap_atomic(addr)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700109
110/*
111 * merge helpers etc
112 */
113
114#define __BVEC_END(bio) bio_iovec_idx((bio), (bio)->bi_vcnt - 1)
115#define __BVEC_START(bio) bio_iovec_idx((bio), (bio)->bi_idx)
116
Jeremy Fitzhardingef92131c2008-10-29 14:10:51 +0100117/* Default implementation of BIOVEC_PHYS_MERGEABLE */
118#define __BIOVEC_PHYS_MERGEABLE(vec1, vec2) \
119 ((bvec_to_phys((vec1)) + (vec1)->bv_len) == bvec_to_phys((vec2)))
120
Linus Torvalds1da177e2005-04-16 15:20:36 -0700121/*
122 * allow arch override, for eg virtualized architectures (put in asm/io.h)
123 */
124#ifndef BIOVEC_PHYS_MERGEABLE
125#define BIOVEC_PHYS_MERGEABLE(vec1, vec2) \
Jeremy Fitzhardingef92131c2008-10-29 14:10:51 +0100126 __BIOVEC_PHYS_MERGEABLE(vec1, vec2)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700127#endif
128
Linus Torvalds1da177e2005-04-16 15:20:36 -0700129#define __BIO_SEG_BOUNDARY(addr1, addr2, mask) \
130 (((addr1) | (mask)) == (((addr2) - 1) | (mask)))
131#define BIOVEC_SEG_BOUNDARY(q, b1, b2) \
Martin K. Petersenae03bf62009-05-22 17:17:50 -0400132 __BIO_SEG_BOUNDARY(bvec_to_phys((b1)), bvec_to_phys((b2)) + (b2)->bv_len, queue_segment_boundary((q)))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700133#define BIO_SEG_BOUNDARY(q, b1, b2) \
134 BIOVEC_SEG_BOUNDARY((q), __BVEC_END((b1)), __BVEC_START((b2)))
135
NeilBrown6712ecf2007-09-27 12:47:43 +0200136#define bio_io_error(bio) bio_endio((bio), -EIO)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700137
138/*
139 * drivers should not use the __ version unless they _really_ want to
140 * run through the entire bio and not just pending pieces
141 */
142#define __bio_for_each_segment(bvl, bio, i, start_idx) \
143 for (bvl = bio_iovec_idx((bio), (start_idx)), i = (start_idx); \
144 i < (bio)->bi_vcnt; \
145 bvl++, i++)
146
147#define bio_for_each_segment(bvl, bio, i) \
148 __bio_for_each_segment(bvl, bio, i, (bio)->bi_idx)
149
150/*
151 * get a reference to a bio, so it won't disappear. the intended use is
152 * something like:
153 *
154 * bio_get(bio);
155 * submit_bio(rw, bio);
156 * if (bio->bi_flags ...)
157 * do_something
158 * bio_put(bio);
159 *
160 * without the bio_get(), it could potentially complete I/O before submit_bio
161 * returns. and then bio would be freed memory when if (bio->bi_flags ...)
162 * runs
163 */
164#define bio_get(bio) atomic_inc(&(bio)->bi_cnt)
165
Martin K. Petersen7ba1ba12008-06-30 20:04:41 +0200166#if defined(CONFIG_BLK_DEV_INTEGRITY)
167/*
168 * bio integrity payload
169 */
170struct bio_integrity_payload {
171 struct bio *bip_bio; /* parent bio */
Martin K. Petersen7ba1ba12008-06-30 20:04:41 +0200172
173 sector_t bip_sector; /* virtual start sector */
174
175 void *bip_buf; /* generated integrity data */
176 bio_end_io_t *bip_end_io; /* saved I/O completion fn */
177
Martin K. Petersen7ba1ba12008-06-30 20:04:41 +0200178 unsigned int bip_size;
179
Martin K. Petersen7878cba2009-06-26 15:37:49 +0200180 unsigned short bip_slab; /* slab the bip came from */
Martin K. Petersen7ba1ba12008-06-30 20:04:41 +0200181 unsigned short bip_vcnt; /* # of integrity bio_vecs */
182 unsigned short bip_idx; /* current bip_vec index */
183
184 struct work_struct bip_work; /* I/O completion */
Martin K. Petersen7878cba2009-06-26 15:37:49 +0200185 struct bio_vec bip_vec[0]; /* embedded bvec array */
Martin K. Petersen7ba1ba12008-06-30 20:04:41 +0200186};
187#endif /* CONFIG_BLK_DEV_INTEGRITY */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700188
189/*
190 * A bio_pair is used when we need to split a bio.
191 * This can only happen for a bio that refers to just one
192 * page of data, and in the unusual situation when the
193 * page crosses a chunk/device boundary
194 *
195 * The address of the master bio is stored in bio1.bi_private
196 * The address of the pool the pair was allocated from is stored
197 * in bio2.bi_private
198 */
199struct bio_pair {
Martin K. Petersen7ba1ba12008-06-30 20:04:41 +0200200 struct bio bio1, bio2;
201 struct bio_vec bv1, bv2;
202#if defined(CONFIG_BLK_DEV_INTEGRITY)
203 struct bio_integrity_payload bip1, bip2;
204 struct bio_vec iv1, iv2;
205#endif
206 atomic_t cnt;
207 int error;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700208};
Denis ChengRq6feef532008-10-09 08:57:05 +0200209extern struct bio_pair *bio_split(struct bio *bi, int first_sectors);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700210extern void bio_pair_release(struct bio_pair *dbio);
211
Jens Axboebb799ca2008-12-10 15:35:05 +0100212extern struct bio_set *bioset_create(unsigned int, unsigned int);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700213extern void bioset_free(struct bio_set *);
214
Dan Carpenter121f0992011-11-16 09:21:50 +0100215extern struct bio *bio_alloc(gfp_t, unsigned int);
216extern struct bio *bio_kmalloc(gfp_t, unsigned int);
Al Virodd0fc662005-10-07 07:46:04 +0100217extern struct bio *bio_alloc_bioset(gfp_t, int, struct bio_set *);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700218extern void bio_put(struct bio *);
Peter Osterlund36763472005-09-06 15:16:42 -0700219extern void bio_free(struct bio *, struct bio_set *);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700220
NeilBrown6712ecf2007-09-27 12:47:43 +0200221extern void bio_endio(struct bio *, int);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700222struct request_queue;
223extern int bio_phys_segments(struct request_queue *, struct bio *);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700224
225extern void __bio_clone(struct bio *, struct bio *);
Al Virodd0fc662005-10-07 07:46:04 +0100226extern struct bio *bio_clone(struct bio *, gfp_t);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700227
228extern void bio_init(struct bio *);
229
230extern int bio_add_page(struct bio *, struct page *, unsigned int,unsigned int);
Mike Christie6e68af62005-11-11 05:30:27 -0600231extern int bio_add_pc_page(struct request_queue *, struct bio *, struct page *,
232 unsigned int, unsigned int);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700233extern int bio_get_nr_vecs(struct block_device *);
Martin K. Petersenad3316b2008-10-01 22:42:53 -0400234extern sector_t bio_sector_offset(struct bio *, unsigned short, unsigned int);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700235extern struct bio *bio_map_user(struct request_queue *, struct block_device *,
FUJITA Tomonoria3bce902008-08-28 16:17:05 +0900236 unsigned long, unsigned int, int, gfp_t);
James Bottomley f1970ba2005-06-20 14:06:52 +0200237struct sg_iovec;
FUJITA Tomonori152e2832008-08-28 16:17:06 +0900238struct rq_map_data;
James Bottomley f1970ba2005-06-20 14:06:52 +0200239extern struct bio *bio_map_user_iov(struct request_queue *,
240 struct block_device *,
FUJITA Tomonoria3bce902008-08-28 16:17:05 +0900241 struct sg_iovec *, int, int, gfp_t);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700242extern void bio_unmap_user(struct bio *);
Mike Christie df46b9a2005-06-20 14:04:44 +0200243extern struct bio *bio_map_kern(struct request_queue *, void *, unsigned int,
Al Viro27496a82005-10-21 03:20:48 -0400244 gfp_t);
FUJITA Tomonori68154e92008-04-25 12:47:50 +0200245extern struct bio *bio_copy_kern(struct request_queue *, void *, unsigned int,
246 gfp_t, int);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700247extern void bio_set_pages_dirty(struct bio *bio);
248extern void bio_check_pages_dirty(struct bio *bio);
Ilya Loginov2d4dc892009-11-26 09:16:19 +0100249
250#ifndef ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE
251# error "You should define ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE for your platform"
252#endif
253#if ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE
254extern void bio_flush_dcache_pages(struct bio *bi);
255#else
256static inline void bio_flush_dcache_pages(struct bio *bi)
257{
258}
259#endif
260
FUJITA Tomonori152e2832008-08-28 16:17:06 +0900261extern struct bio *bio_copy_user(struct request_queue *, struct rq_map_data *,
262 unsigned long, unsigned int, int, gfp_t);
263extern struct bio *bio_copy_user_iov(struct request_queue *,
264 struct rq_map_data *, struct sg_iovec *,
FUJITA Tomonoria3bce902008-08-28 16:17:05 +0900265 int, int, gfp_t);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700266extern int bio_uncopy_user(struct bio *);
267void zero_fill_bio(struct bio *bio);
Martin K. Petersen51d654e2008-06-17 18:59:56 +0200268extern struct bio_vec *bvec_alloc_bs(gfp_t, int, unsigned long *, struct bio_set *);
Jens Axboebb799ca2008-12-10 15:35:05 +0100269extern void bvec_free_bs(struct bio_set *, struct bio_vec *, unsigned int);
Martin K. Petersen7ba1ba12008-06-30 20:04:41 +0200270extern unsigned int bvec_nr_vecs(unsigned short idx);
Martin K. Petersen51d654e2008-06-17 18:59:56 +0200271
272/*
273 * bio_set is used to allow other portions of the IO system to
274 * allocate their own private memory pools for bio and iovec structures.
275 * These memory pools in turn all allocate from the bio_slab
276 * and the bvec_slabs[].
277 */
278#define BIO_POOL_SIZE 2
279#define BIOVEC_NR_POOLS 6
Jens Axboe7ff93452008-12-11 11:53:43 +0100280#define BIOVEC_MAX_IDX (BIOVEC_NR_POOLS - 1)
Martin K. Petersen51d654e2008-06-17 18:59:56 +0200281
282struct bio_set {
Jens Axboe1b434492008-10-22 20:32:58 +0200283 struct kmem_cache *bio_slab;
Jens Axboebb799ca2008-12-10 15:35:05 +0100284 unsigned int front_pad;
285
Martin K. Petersen51d654e2008-06-17 18:59:56 +0200286 mempool_t *bio_pool;
Martin K. Petersen7878cba2009-06-26 15:37:49 +0200287#if defined(CONFIG_BLK_DEV_INTEGRITY)
288 mempool_t *bio_integrity_pool;
289#endif
Jens Axboe7ff93452008-12-11 11:53:43 +0100290 mempool_t *bvec_pool;
Martin K. Petersen51d654e2008-06-17 18:59:56 +0200291};
292
293struct biovec_slab {
294 int nr_vecs;
295 char *name;
296 struct kmem_cache *slab;
297};
298
299extern struct bio_set *fs_bio_set;
300
301/*
302 * a small number of entries is fine, not going to be performance critical.
303 * basically we just need to survive
304 */
305#define BIO_SPLIT_ENTRIES 2
Linus Torvalds1da177e2005-04-16 15:20:36 -0700306
307#ifdef CONFIG_HIGHMEM
308/*
Alberto Bertogli20b636b2009-02-02 12:41:07 +0100309 * remember never ever reenable interrupts between a bvec_kmap_irq and
310 * bvec_kunmap_irq!
Linus Torvalds1da177e2005-04-16 15:20:36 -0700311 */
Alberto Bertogli4f570f92009-11-02 11:40:16 +0100312static inline char *bvec_kmap_irq(struct bio_vec *bvec, unsigned long *flags)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700313{
314 unsigned long addr;
315
316 /*
317 * might not be a highmem page, but the preempt/irq count
318 * balancing is a lot nicer this way
319 */
320 local_irq_save(*flags);
Cong Wange8e3c3d2011-11-25 23:14:27 +0800321 addr = (unsigned long) kmap_atomic(bvec->bv_page);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700322
323 BUG_ON(addr & ~PAGE_MASK);
324
325 return (char *) addr + bvec->bv_offset;
326}
327
Alberto Bertogli4f570f92009-11-02 11:40:16 +0100328static inline void bvec_kunmap_irq(char *buffer, unsigned long *flags)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700329{
330 unsigned long ptr = (unsigned long) buffer & PAGE_MASK;
331
Cong Wange8e3c3d2011-11-25 23:14:27 +0800332 kunmap_atomic((void *) ptr);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700333 local_irq_restore(*flags);
334}
335
336#else
Geert Uytterhoeven11a691b2010-10-21 10:32:29 +0200337static inline char *bvec_kmap_irq(struct bio_vec *bvec, unsigned long *flags)
338{
339 return page_address(bvec->bv_page) + bvec->bv_offset;
340}
341
342static inline void bvec_kunmap_irq(char *buffer, unsigned long *flags)
343{
344 *flags = 0;
345}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700346#endif
347
Adrian Bunkc2d08da2005-09-10 00:27:18 -0700348static inline char *__bio_kmap_irq(struct bio *bio, unsigned short idx,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700349 unsigned long *flags)
350{
351 return bvec_kmap_irq(bio_iovec_idx(bio, idx), flags);
352}
353#define __bio_kunmap_irq(buf, flags) bvec_kunmap_irq(buf, flags)
354
355#define bio_kmap_irq(bio, flags) \
356 __bio_kmap_irq((bio), (bio)->bi_idx, (flags))
357#define bio_kunmap_irq(buf,flags) __bio_kunmap_irq(buf, flags)
358
Jens Axboe7a67f632008-08-08 11:17:12 +0200359/*
360 * Check whether this bio carries any data or not. A NULL bio is allowed.
361 */
362static inline int bio_has_data(struct bio *bio)
363{
364 return bio && bio->bi_io_vec != NULL;
365}
366
Christoph Hellwig8f3d8ba2009-04-07 19:55:13 +0200367/*
Akinobu Mitae6863072009-04-17 08:41:21 +0200368 * BIO list management for use by remapping drivers (e.g. DM or MD) and loop.
Christoph Hellwig8f3d8ba2009-04-07 19:55:13 +0200369 *
370 * A bio_list anchors a singly-linked list of bios chained through the bi_next
371 * member of the bio. The bio_list also caches the last list member to allow
372 * fast access to the tail.
373 */
374struct bio_list {
375 struct bio *head;
376 struct bio *tail;
377};
378
379static inline int bio_list_empty(const struct bio_list *bl)
380{
381 return bl->head == NULL;
382}
383
384static inline void bio_list_init(struct bio_list *bl)
385{
386 bl->head = bl->tail = NULL;
387}
388
389#define bio_list_for_each(bio, bl) \
390 for (bio = (bl)->head; bio; bio = bio->bi_next)
391
392static inline unsigned bio_list_size(const struct bio_list *bl)
393{
394 unsigned sz = 0;
395 struct bio *bio;
396
397 bio_list_for_each(bio, bl)
398 sz++;
399
400 return sz;
401}
402
403static inline void bio_list_add(struct bio_list *bl, struct bio *bio)
404{
405 bio->bi_next = NULL;
406
407 if (bl->tail)
408 bl->tail->bi_next = bio;
409 else
410 bl->head = bio;
411
412 bl->tail = bio;
413}
414
415static inline void bio_list_add_head(struct bio_list *bl, struct bio *bio)
416{
417 bio->bi_next = bl->head;
418
419 bl->head = bio;
420
421 if (!bl->tail)
422 bl->tail = bio;
423}
424
425static inline void bio_list_merge(struct bio_list *bl, struct bio_list *bl2)
426{
427 if (!bl2->head)
428 return;
429
430 if (bl->tail)
431 bl->tail->bi_next = bl2->head;
432 else
433 bl->head = bl2->head;
434
435 bl->tail = bl2->tail;
436}
437
438static inline void bio_list_merge_head(struct bio_list *bl,
439 struct bio_list *bl2)
440{
441 if (!bl2->head)
442 return;
443
444 if (bl->head)
445 bl2->tail->bi_next = bl->head;
446 else
447 bl->tail = bl2->tail;
448
449 bl->head = bl2->head;
450}
451
Geert Uytterhoeven13685a12009-06-10 04:38:40 +0000452static inline struct bio *bio_list_peek(struct bio_list *bl)
453{
454 return bl->head;
455}
456
Christoph Hellwig8f3d8ba2009-04-07 19:55:13 +0200457static inline struct bio *bio_list_pop(struct bio_list *bl)
458{
459 struct bio *bio = bl->head;
460
461 if (bio) {
462 bl->head = bl->head->bi_next;
463 if (!bl->head)
464 bl->tail = NULL;
465
466 bio->bi_next = NULL;
467 }
468
469 return bio;
470}
471
472static inline struct bio *bio_list_get(struct bio_list *bl)
473{
474 struct bio *bio = bl->head;
475
476 bl->head = bl->tail = NULL;
477
478 return bio;
479}
480
Martin K. Petersen7ba1ba12008-06-30 20:04:41 +0200481#if defined(CONFIG_BLK_DEV_INTEGRITY)
482
483#define bip_vec_idx(bip, idx) (&(bip->bip_vec[(idx)]))
484#define bip_vec(bip) bip_vec_idx(bip, 0)
485
486#define __bip_for_each_vec(bvl, bip, i, start_idx) \
487 for (bvl = bip_vec_idx((bip), (start_idx)), i = (start_idx); \
488 i < (bip)->bip_vcnt; \
489 bvl++, i++)
490
491#define bip_for_each_vec(bvl, bip, i) \
492 __bip_for_each_vec(bvl, bip, i, (bip)->bip_idx)
493
Martin K. Petersen13f05c82010-09-10 20:50:10 +0200494#define bio_for_each_integrity_vec(_bvl, _bio, _iter) \
495 for_each_bio(_bio) \
496 bip_for_each_vec(_bvl, _bio->bi_integrity, _iter)
497
Alberto Bertogli8deaf722008-10-02 12:46:53 +0200498#define bio_integrity(bio) (bio->bi_integrity != NULL)
Martin K. Petersen7ba1ba12008-06-30 20:04:41 +0200499
Martin K. Petersen7878cba2009-06-26 15:37:49 +0200500extern struct bio_integrity_payload *bio_integrity_alloc_bioset(struct bio *, gfp_t, unsigned int, struct bio_set *);
Martin K. Petersen7ba1ba12008-06-30 20:04:41 +0200501extern struct bio_integrity_payload *bio_integrity_alloc(struct bio *, gfp_t, unsigned int);
Martin K. Petersen7878cba2009-06-26 15:37:49 +0200502extern void bio_integrity_free(struct bio *, struct bio_set *);
Martin K. Petersen7ba1ba12008-06-30 20:04:41 +0200503extern int bio_integrity_add_page(struct bio *, struct page *, unsigned int, unsigned int);
504extern int bio_integrity_enabled(struct bio *bio);
505extern int bio_integrity_set_tag(struct bio *, void *, unsigned int);
506extern int bio_integrity_get_tag(struct bio *, void *, unsigned int);
507extern int bio_integrity_prep(struct bio *);
508extern void bio_integrity_endio(struct bio *, int);
509extern void bio_integrity_advance(struct bio *, unsigned int);
510extern void bio_integrity_trim(struct bio *, unsigned int, unsigned int);
511extern void bio_integrity_split(struct bio *, struct bio_pair *, int);
Martin K. Petersen7878cba2009-06-26 15:37:49 +0200512extern int bio_integrity_clone(struct bio *, struct bio *, gfp_t, struct bio_set *);
513extern int bioset_integrity_create(struct bio_set *, int);
514extern void bioset_integrity_free(struct bio_set *);
515extern void bio_integrity_init(void);
Martin K. Petersen7ba1ba12008-06-30 20:04:41 +0200516
517#else /* CONFIG_BLK_DEV_INTEGRITY */
518
Martin K. Petersen6898e3b2012-01-13 08:15:33 +0100519static inline int bio_integrity(struct bio *bio)
520{
521 return 0;
522}
523
524static inline int bio_integrity_enabled(struct bio *bio)
525{
526 return 0;
527}
528
529static inline int bioset_integrity_create(struct bio_set *bs, int pool_size)
530{
531 return 0;
532}
533
534static inline void bioset_integrity_free (struct bio_set *bs)
535{
536 return;
537}
538
539static inline int bio_integrity_prep(struct bio *bio)
540{
541 return 0;
542}
543
544static inline void bio_integrity_free(struct bio *bio, struct bio_set *bs)
545{
546 return;
547}
548
Stephen Rothwell0c614e22011-11-16 09:21:48 +0100549static inline int bio_integrity_clone(struct bio *bio, struct bio *bio_src,
550 gfp_t gfp_mask, struct bio_set *bs)
551{
552 return 0;
553}
Martin K. Petersen6898e3b2012-01-13 08:15:33 +0100554
555static inline void bio_integrity_split(struct bio *bio, struct bio_pair *bp,
556 int sectors)
557{
558 return;
559}
560
561static inline void bio_integrity_advance(struct bio *bio,
562 unsigned int bytes_done)
563{
564 return;
565}
566
567static inline void bio_integrity_trim(struct bio *bio, unsigned int offset,
568 unsigned int sectors)
569{
570 return;
571}
572
573static inline void bio_integrity_init(void)
574{
575 return;
576}
Martin K. Petersen7ba1ba12008-06-30 20:04:41 +0200577
578#endif /* CONFIG_BLK_DEV_INTEGRITY */
579
David Howells02a5e0a2007-08-11 22:34:32 +0200580#endif /* CONFIG_BLOCK */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700581#endif /* __LINUX_BIO_H */