blob: 20507eb7c97926ba4bf2884bae92d4b1ceef2b6e [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
2 * 2.5 block I/O model
3 *
4 * Copyright (C) 2001 Jens Axboe <axboe@suse.de>
5 *
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License version 2 as
8 * published by the Free Software Foundation.
9 *
10 * This program is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
Tejun Heo7cc01582010-08-03 13:14:58 +020012 *
Linus Torvalds1da177e2005-04-16 15:20:36 -070013 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 * GNU General Public License for more details.
15 *
16 * You should have received a copy of the GNU General Public Licens
17 * along with this program; if not, write to the Free Software
18 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-
19 */
20#ifndef __LINUX_BIO_H
21#define __LINUX_BIO_H
22
23#include <linux/highmem.h>
24#include <linux/mempool.h>
Jens Axboe22e2c502005-06-27 10:55:12 +020025#include <linux/ioprio.h>
Paul Gortmaker187f1882011-11-23 20:12:59 -050026#include <linux/bug.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070027
David Howells02a5e0a2007-08-11 22:34:32 +020028#ifdef CONFIG_BLOCK
29
Linus Torvalds1da177e2005-04-16 15:20:36 -070030#include <asm/io.h>
31
Tejun Heo7cc01582010-08-03 13:14:58 +020032/* struct bio, bio_vec and BIO_* flags are defined in blk_types.h */
33#include <linux/blk_types.h>
34
Linus Torvalds1da177e2005-04-16 15:20:36 -070035#define BIO_DEBUG
36
37#ifdef BIO_DEBUG
38#define BIO_BUG_ON BUG_ON
39#else
40#define BIO_BUG_ON
41#endif
42
Alexey Dobriyand84a8472006-06-25 05:49:32 -070043#define BIO_MAX_PAGES 256
Linus Torvalds1da177e2005-04-16 15:20:36 -070044#define BIO_MAX_SIZE (BIO_MAX_PAGES << PAGE_CACHE_SHIFT)
45#define BIO_MAX_SECTORS (BIO_MAX_SIZE >> 9)
46
47/*
Jens Axboe22e2c502005-06-27 10:55:12 +020048 * upper 16 bits of bi_rw define the io priority of this bio
49 */
50#define BIO_PRIO_SHIFT (8 * sizeof(unsigned long) - IOPRIO_BITS)
51#define bio_prio(bio) ((bio)->bi_rw >> BIO_PRIO_SHIFT)
52#define bio_prio_valid(bio) ioprio_valid(bio_prio(bio))
53
54#define bio_set_prio(bio, prio) do { \
55 WARN_ON(prio >= (1 << IOPRIO_BITS)); \
56 (bio)->bi_rw &= ((1UL << BIO_PRIO_SHIFT) - 1); \
57 (bio)->bi_rw |= ((unsigned long) (prio) << BIO_PRIO_SHIFT); \
58} while (0)
59
60/*
Linus Torvalds1da177e2005-04-16 15:20:36 -070061 * various member access, note that bio_data should of course not be used
62 * on highmem page vectors
63 */
64#define bio_iovec_idx(bio, idx) (&((bio)->bi_io_vec[(idx)]))
65#define bio_iovec(bio) bio_iovec_idx((bio), (bio)->bi_idx)
66#define bio_page(bio) bio_iovec((bio))->bv_page
67#define bio_offset(bio) bio_iovec((bio))->bv_offset
68#define bio_segments(bio) ((bio)->bi_vcnt - (bio)->bi_idx)
69#define bio_sectors(bio) ((bio)->bi_size >> 9)
Kent Overstreetf73a1c72012-09-25 15:05:12 -070070#define bio_end_sector(bio) ((bio)->bi_sector + bio_sectors((bio)))
Jens Axboebf2de6f2007-09-27 13:01:25 +020071
Tejun Heo2e46e8b2009-05-07 22:24:41 +090072static inline unsigned int bio_cur_bytes(struct bio *bio)
Jens Axboebf2de6f2007-09-27 13:01:25 +020073{
74 if (bio->bi_vcnt)
Tejun Heo2e46e8b2009-05-07 22:24:41 +090075 return bio_iovec(bio)->bv_len;
David Woodhousefb2dce82008-08-05 18:01:53 +010076 else /* dataless requests such as discard */
Tejun Heo2e46e8b2009-05-07 22:24:41 +090077 return bio->bi_size;
Jens Axboebf2de6f2007-09-27 13:01:25 +020078}
79
80static inline void *bio_data(struct bio *bio)
81{
82 if (bio->bi_vcnt)
83 return page_address(bio_page(bio)) + bio_offset(bio);
84
85 return NULL;
86}
Linus Torvalds1da177e2005-04-16 15:20:36 -070087
Jens Axboe392ddc32008-12-23 12:42:54 +010088static inline int bio_has_allocated_vec(struct bio *bio)
89{
90 return bio->bi_io_vec && bio->bi_io_vec != bio->bi_inline_vecs;
91}
92
Linus Torvalds1da177e2005-04-16 15:20:36 -070093/*
94 * will die
95 */
96#define bio_to_phys(bio) (page_to_phys(bio_page((bio))) + (unsigned long) bio_offset((bio)))
97#define bvec_to_phys(bv) (page_to_phys((bv)->bv_page) + (unsigned long) (bv)->bv_offset)
98
99/*
100 * queues that have highmem support enabled may still need to revert to
101 * PIO transfers occasionally and thus map high pages temporarily. For
102 * permanent PIO fall back, user is probably better off disabling highmem
103 * I/O completely on that queue (see ide-dma for example)
104 */
105#define __bio_kmap_atomic(bio, idx, kmtype) \
Cong Wange8e3c3d2011-11-25 23:14:27 +0800106 (kmap_atomic(bio_iovec_idx((bio), (idx))->bv_page) + \
Linus Torvalds1da177e2005-04-16 15:20:36 -0700107 bio_iovec_idx((bio), (idx))->bv_offset)
108
Cong Wange8e3c3d2011-11-25 23:14:27 +0800109#define __bio_kunmap_atomic(addr, kmtype) kunmap_atomic(addr)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700110
111/*
112 * merge helpers etc
113 */
114
115#define __BVEC_END(bio) bio_iovec_idx((bio), (bio)->bi_vcnt - 1)
116#define __BVEC_START(bio) bio_iovec_idx((bio), (bio)->bi_idx)
117
Jeremy Fitzhardingef92131c2008-10-29 14:10:51 +0100118/* Default implementation of BIOVEC_PHYS_MERGEABLE */
119#define __BIOVEC_PHYS_MERGEABLE(vec1, vec2) \
120 ((bvec_to_phys((vec1)) + (vec1)->bv_len) == bvec_to_phys((vec2)))
121
Linus Torvalds1da177e2005-04-16 15:20:36 -0700122/*
123 * allow arch override, for eg virtualized architectures (put in asm/io.h)
124 */
125#ifndef BIOVEC_PHYS_MERGEABLE
126#define BIOVEC_PHYS_MERGEABLE(vec1, vec2) \
Jeremy Fitzhardingef92131c2008-10-29 14:10:51 +0100127 __BIOVEC_PHYS_MERGEABLE(vec1, vec2)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700128#endif
129
Linus Torvalds1da177e2005-04-16 15:20:36 -0700130#define __BIO_SEG_BOUNDARY(addr1, addr2, mask) \
131 (((addr1) | (mask)) == (((addr2) - 1) | (mask)))
132#define BIOVEC_SEG_BOUNDARY(q, b1, b2) \
Martin K. Petersenae03bf62009-05-22 17:17:50 -0400133 __BIO_SEG_BOUNDARY(bvec_to_phys((b1)), bvec_to_phys((b2)) + (b2)->bv_len, queue_segment_boundary((q)))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700134#define BIO_SEG_BOUNDARY(q, b1, b2) \
135 BIOVEC_SEG_BOUNDARY((q), __BVEC_END((b1)), __BVEC_START((b2)))
136
NeilBrown6712ecf2007-09-27 12:47:43 +0200137#define bio_io_error(bio) bio_endio((bio), -EIO)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700138
139/*
140 * drivers should not use the __ version unless they _really_ want to
141 * run through the entire bio and not just pending pieces
142 */
143#define __bio_for_each_segment(bvl, bio, i, start_idx) \
144 for (bvl = bio_iovec_idx((bio), (start_idx)), i = (start_idx); \
145 i < (bio)->bi_vcnt; \
146 bvl++, i++)
147
148#define bio_for_each_segment(bvl, bio, i) \
149 __bio_for_each_segment(bvl, bio, i, (bio)->bi_idx)
150
151/*
152 * get a reference to a bio, so it won't disappear. the intended use is
153 * something like:
154 *
155 * bio_get(bio);
156 * submit_bio(rw, bio);
157 * if (bio->bi_flags ...)
158 * do_something
159 * bio_put(bio);
160 *
161 * without the bio_get(), it could potentially complete I/O before submit_bio
162 * returns. and then bio would be freed memory when if (bio->bi_flags ...)
163 * runs
164 */
165#define bio_get(bio) atomic_inc(&(bio)->bi_cnt)
166
Martin K. Petersen7ba1ba12008-06-30 20:04:41 +0200167#if defined(CONFIG_BLK_DEV_INTEGRITY)
168/*
169 * bio integrity payload
170 */
171struct bio_integrity_payload {
172 struct bio *bip_bio; /* parent bio */
Martin K. Petersen7ba1ba12008-06-30 20:04:41 +0200173
174 sector_t bip_sector; /* virtual start sector */
175
176 void *bip_buf; /* generated integrity data */
177 bio_end_io_t *bip_end_io; /* saved I/O completion fn */
178
Martin K. Petersen7ba1ba12008-06-30 20:04:41 +0200179 unsigned int bip_size;
180
Martin K. Petersen7878cba2009-06-26 15:37:49 +0200181 unsigned short bip_slab; /* slab the bip came from */
Martin K. Petersen7ba1ba12008-06-30 20:04:41 +0200182 unsigned short bip_vcnt; /* # of integrity bio_vecs */
183 unsigned short bip_idx; /* current bip_vec index */
184
185 struct work_struct bip_work; /* I/O completion */
Kent Overstreet6fda9812012-10-12 13:18:27 -0700186
187 struct bio_vec *bip_vec;
188 struct bio_vec bip_inline_vecs[0];/* embedded bvec array */
Martin K. Petersen7ba1ba12008-06-30 20:04:41 +0200189};
190#endif /* CONFIG_BLK_DEV_INTEGRITY */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700191
192/*
193 * A bio_pair is used when we need to split a bio.
194 * This can only happen for a bio that refers to just one
195 * page of data, and in the unusual situation when the
196 * page crosses a chunk/device boundary
197 *
198 * The address of the master bio is stored in bio1.bi_private
199 * The address of the pool the pair was allocated from is stored
200 * in bio2.bi_private
201 */
202struct bio_pair {
Martin K. Petersen7ba1ba12008-06-30 20:04:41 +0200203 struct bio bio1, bio2;
204 struct bio_vec bv1, bv2;
205#if defined(CONFIG_BLK_DEV_INTEGRITY)
206 struct bio_integrity_payload bip1, bip2;
207 struct bio_vec iv1, iv2;
208#endif
209 atomic_t cnt;
210 int error;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700211};
Denis ChengRq6feef532008-10-09 08:57:05 +0200212extern struct bio_pair *bio_split(struct bio *bi, int first_sectors);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700213extern void bio_pair_release(struct bio_pair *dbio);
214
Jens Axboebb799ca2008-12-10 15:35:05 +0100215extern struct bio_set *bioset_create(unsigned int, unsigned int);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700216extern void bioset_free(struct bio_set *);
Kent Overstreet9f060e22012-10-12 15:29:33 -0700217extern mempool_t *biovec_create_pool(struct bio_set *bs, int pool_entries);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700218
Al Virodd0fc662005-10-07 07:46:04 +0100219extern struct bio *bio_alloc_bioset(gfp_t, int, struct bio_set *);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700220extern void bio_put(struct bio *);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700221
Kent Overstreetbf800ef2012-09-06 15:35:02 -0700222extern void __bio_clone(struct bio *, struct bio *);
223extern struct bio *bio_clone_bioset(struct bio *, gfp_t, struct bio_set *bs);
224
Kent Overstreet3f86a822012-09-06 15:35:01 -0700225extern struct bio_set *fs_bio_set;
226
227static inline struct bio *bio_alloc(gfp_t gfp_mask, unsigned int nr_iovecs)
228{
229 return bio_alloc_bioset(gfp_mask, nr_iovecs, fs_bio_set);
230}
231
Kent Overstreetbf800ef2012-09-06 15:35:02 -0700232static inline struct bio *bio_clone(struct bio *bio, gfp_t gfp_mask)
233{
234 return bio_clone_bioset(bio, gfp_mask, fs_bio_set);
235}
236
Kent Overstreet3f86a822012-09-06 15:35:01 -0700237static inline struct bio *bio_kmalloc(gfp_t gfp_mask, unsigned int nr_iovecs)
238{
239 return bio_alloc_bioset(gfp_mask, nr_iovecs, NULL);
240}
241
Kent Overstreetbf800ef2012-09-06 15:35:02 -0700242static inline struct bio *bio_clone_kmalloc(struct bio *bio, gfp_t gfp_mask)
243{
244 return bio_clone_bioset(bio, gfp_mask, NULL);
245
246}
247
NeilBrown6712ecf2007-09-27 12:47:43 +0200248extern void bio_endio(struct bio *, int);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700249struct request_queue;
250extern int bio_phys_segments(struct request_queue *, struct bio *);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700251
Kent Overstreet054bdf62012-09-28 13:17:55 -0700252extern void bio_advance(struct bio *, unsigned);
253
Linus Torvalds1da177e2005-04-16 15:20:36 -0700254extern void bio_init(struct bio *);
Kent Overstreetf44b48c2012-09-06 15:34:58 -0700255extern void bio_reset(struct bio *);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700256
257extern int bio_add_page(struct bio *, struct page *, unsigned int,unsigned int);
Mike Christie6e68af62005-11-11 05:30:27 -0600258extern int bio_add_pc_page(struct request_queue *, struct bio *, struct page *,
259 unsigned int, unsigned int);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700260extern int bio_get_nr_vecs(struct block_device *);
Martin K. Petersenad3316b2008-10-01 22:42:53 -0400261extern sector_t bio_sector_offset(struct bio *, unsigned short, unsigned int);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700262extern struct bio *bio_map_user(struct request_queue *, struct block_device *,
FUJITA Tomonoria3bce902008-08-28 16:17:05 +0900263 unsigned long, unsigned int, int, gfp_t);
James Bottomley f1970ba2005-06-20 14:06:52 +0200264struct sg_iovec;
FUJITA Tomonori152e2832008-08-28 16:17:06 +0900265struct rq_map_data;
James Bottomley f1970ba2005-06-20 14:06:52 +0200266extern struct bio *bio_map_user_iov(struct request_queue *,
267 struct block_device *,
FUJITA Tomonoria3bce902008-08-28 16:17:05 +0900268 struct sg_iovec *, int, int, gfp_t);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700269extern void bio_unmap_user(struct bio *);
Mike Christie df46b9a2005-06-20 14:04:44 +0200270extern struct bio *bio_map_kern(struct request_queue *, void *, unsigned int,
Al Viro27496a82005-10-21 03:20:48 -0400271 gfp_t);
FUJITA Tomonori68154e92008-04-25 12:47:50 +0200272extern struct bio *bio_copy_kern(struct request_queue *, void *, unsigned int,
273 gfp_t, int);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700274extern void bio_set_pages_dirty(struct bio *bio);
275extern void bio_check_pages_dirty(struct bio *bio);
Ilya Loginov2d4dc892009-11-26 09:16:19 +0100276
277#ifndef ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE
278# error "You should define ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE for your platform"
279#endif
280#if ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE
281extern void bio_flush_dcache_pages(struct bio *bi);
282#else
283static inline void bio_flush_dcache_pages(struct bio *bi)
284{
285}
286#endif
287
FUJITA Tomonori152e2832008-08-28 16:17:06 +0900288extern struct bio *bio_copy_user(struct request_queue *, struct rq_map_data *,
289 unsigned long, unsigned int, int, gfp_t);
290extern struct bio *bio_copy_user_iov(struct request_queue *,
291 struct rq_map_data *, struct sg_iovec *,
FUJITA Tomonoria3bce902008-08-28 16:17:05 +0900292 int, int, gfp_t);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700293extern int bio_uncopy_user(struct bio *);
294void zero_fill_bio(struct bio *bio);
Kent Overstreet9f060e22012-10-12 15:29:33 -0700295extern struct bio_vec *bvec_alloc(gfp_t, int, unsigned long *, mempool_t *);
296extern void bvec_free(mempool_t *, struct bio_vec *, unsigned int);
Martin K. Petersen7ba1ba12008-06-30 20:04:41 +0200297extern unsigned int bvec_nr_vecs(unsigned short idx);
Martin K. Petersen51d654e2008-06-17 18:59:56 +0200298
Tejun Heo852c7882012-03-05 13:15:27 -0800299#ifdef CONFIG_BLK_CGROUP
300int bio_associate_current(struct bio *bio);
301void bio_disassociate_task(struct bio *bio);
302#else /* CONFIG_BLK_CGROUP */
303static inline int bio_associate_current(struct bio *bio) { return -ENOENT; }
304static inline void bio_disassociate_task(struct bio *bio) { }
305#endif /* CONFIG_BLK_CGROUP */
306
Linus Torvalds1da177e2005-04-16 15:20:36 -0700307#ifdef CONFIG_HIGHMEM
308/*
Alberto Bertogli20b636b2009-02-02 12:41:07 +0100309 * remember never ever reenable interrupts between a bvec_kmap_irq and
310 * bvec_kunmap_irq!
Linus Torvalds1da177e2005-04-16 15:20:36 -0700311 */
Alberto Bertogli4f570f92009-11-02 11:40:16 +0100312static inline char *bvec_kmap_irq(struct bio_vec *bvec, unsigned long *flags)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700313{
314 unsigned long addr;
315
316 /*
317 * might not be a highmem page, but the preempt/irq count
318 * balancing is a lot nicer this way
319 */
320 local_irq_save(*flags);
Cong Wange8e3c3d2011-11-25 23:14:27 +0800321 addr = (unsigned long) kmap_atomic(bvec->bv_page);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700322
323 BUG_ON(addr & ~PAGE_MASK);
324
325 return (char *) addr + bvec->bv_offset;
326}
327
Alberto Bertogli4f570f92009-11-02 11:40:16 +0100328static inline void bvec_kunmap_irq(char *buffer, unsigned long *flags)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700329{
330 unsigned long ptr = (unsigned long) buffer & PAGE_MASK;
331
Cong Wange8e3c3d2011-11-25 23:14:27 +0800332 kunmap_atomic((void *) ptr);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700333 local_irq_restore(*flags);
334}
335
336#else
Geert Uytterhoeven11a691b2010-10-21 10:32:29 +0200337static inline char *bvec_kmap_irq(struct bio_vec *bvec, unsigned long *flags)
338{
339 return page_address(bvec->bv_page) + bvec->bv_offset;
340}
341
342static inline void bvec_kunmap_irq(char *buffer, unsigned long *flags)
343{
344 *flags = 0;
345}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700346#endif
347
Adrian Bunkc2d08da2005-09-10 00:27:18 -0700348static inline char *__bio_kmap_irq(struct bio *bio, unsigned short idx,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700349 unsigned long *flags)
350{
351 return bvec_kmap_irq(bio_iovec_idx(bio, idx), flags);
352}
353#define __bio_kunmap_irq(buf, flags) bvec_kunmap_irq(buf, flags)
354
355#define bio_kmap_irq(bio, flags) \
356 __bio_kmap_irq((bio), (bio)->bi_idx, (flags))
357#define bio_kunmap_irq(buf,flags) __bio_kunmap_irq(buf, flags)
358
Jens Axboe7a67f632008-08-08 11:17:12 +0200359/*
360 * Check whether this bio carries any data or not. A NULL bio is allowed.
361 */
Martin K. Petersene2a60da2012-09-18 12:19:25 -0400362static inline bool bio_has_data(struct bio *bio)
Jens Axboe7a67f632008-08-08 11:17:12 +0200363{
Martin K. Petersene2a60da2012-09-18 12:19:25 -0400364 if (bio && bio->bi_vcnt)
365 return true;
366
367 return false;
368}
369
370static inline bool bio_is_rw(struct bio *bio)
371{
372 if (!bio_has_data(bio))
373 return false;
374
Martin K. Petersen4363ac72012-09-18 12:19:27 -0400375 if (bio->bi_rw & REQ_WRITE_SAME)
376 return false;
377
Martin K. Petersene2a60da2012-09-18 12:19:25 -0400378 return true;
379}
380
381static inline bool bio_mergeable(struct bio *bio)
382{
383 if (bio->bi_rw & REQ_NOMERGE_FLAGS)
384 return false;
385
386 return true;
Jens Axboe7a67f632008-08-08 11:17:12 +0200387}
388
Christoph Hellwig8f3d8ba2009-04-07 19:55:13 +0200389/*
Akinobu Mitae6863072009-04-17 08:41:21 +0200390 * BIO list management for use by remapping drivers (e.g. DM or MD) and loop.
Christoph Hellwig8f3d8ba2009-04-07 19:55:13 +0200391 *
392 * A bio_list anchors a singly-linked list of bios chained through the bi_next
393 * member of the bio. The bio_list also caches the last list member to allow
394 * fast access to the tail.
395 */
396struct bio_list {
397 struct bio *head;
398 struct bio *tail;
399};
400
401static inline int bio_list_empty(const struct bio_list *bl)
402{
403 return bl->head == NULL;
404}
405
406static inline void bio_list_init(struct bio_list *bl)
407{
408 bl->head = bl->tail = NULL;
409}
410
411#define bio_list_for_each(bio, bl) \
412 for (bio = (bl)->head; bio; bio = bio->bi_next)
413
414static inline unsigned bio_list_size(const struct bio_list *bl)
415{
416 unsigned sz = 0;
417 struct bio *bio;
418
419 bio_list_for_each(bio, bl)
420 sz++;
421
422 return sz;
423}
424
425static inline void bio_list_add(struct bio_list *bl, struct bio *bio)
426{
427 bio->bi_next = NULL;
428
429 if (bl->tail)
430 bl->tail->bi_next = bio;
431 else
432 bl->head = bio;
433
434 bl->tail = bio;
435}
436
437static inline void bio_list_add_head(struct bio_list *bl, struct bio *bio)
438{
439 bio->bi_next = bl->head;
440
441 bl->head = bio;
442
443 if (!bl->tail)
444 bl->tail = bio;
445}
446
447static inline void bio_list_merge(struct bio_list *bl, struct bio_list *bl2)
448{
449 if (!bl2->head)
450 return;
451
452 if (bl->tail)
453 bl->tail->bi_next = bl2->head;
454 else
455 bl->head = bl2->head;
456
457 bl->tail = bl2->tail;
458}
459
460static inline void bio_list_merge_head(struct bio_list *bl,
461 struct bio_list *bl2)
462{
463 if (!bl2->head)
464 return;
465
466 if (bl->head)
467 bl2->tail->bi_next = bl->head;
468 else
469 bl->tail = bl2->tail;
470
471 bl->head = bl2->head;
472}
473
Geert Uytterhoeven13685a12009-06-10 04:38:40 +0000474static inline struct bio *bio_list_peek(struct bio_list *bl)
475{
476 return bl->head;
477}
478
Christoph Hellwig8f3d8ba2009-04-07 19:55:13 +0200479static inline struct bio *bio_list_pop(struct bio_list *bl)
480{
481 struct bio *bio = bl->head;
482
483 if (bio) {
484 bl->head = bl->head->bi_next;
485 if (!bl->head)
486 bl->tail = NULL;
487
488 bio->bi_next = NULL;
489 }
490
491 return bio;
492}
493
494static inline struct bio *bio_list_get(struct bio_list *bl)
495{
496 struct bio *bio = bl->head;
497
498 bl->head = bl->tail = NULL;
499
500 return bio;
501}
502
Kent Overstreet57fb2332012-08-24 04:56:11 -0700503/*
504 * bio_set is used to allow other portions of the IO system to
505 * allocate their own private memory pools for bio and iovec structures.
506 * These memory pools in turn all allocate from the bio_slab
507 * and the bvec_slabs[].
508 */
509#define BIO_POOL_SIZE 2
510#define BIOVEC_NR_POOLS 6
511#define BIOVEC_MAX_IDX (BIOVEC_NR_POOLS - 1)
512
513struct bio_set {
514 struct kmem_cache *bio_slab;
515 unsigned int front_pad;
516
517 mempool_t *bio_pool;
Kent Overstreet9f060e22012-10-12 15:29:33 -0700518 mempool_t *bvec_pool;
Kent Overstreet57fb2332012-08-24 04:56:11 -0700519#if defined(CONFIG_BLK_DEV_INTEGRITY)
520 mempool_t *bio_integrity_pool;
Kent Overstreet9f060e22012-10-12 15:29:33 -0700521 mempool_t *bvec_integrity_pool;
Kent Overstreet57fb2332012-08-24 04:56:11 -0700522#endif
Kent Overstreetdf2cb6d2012-09-10 14:33:46 -0700523
524 /*
525 * Deadlock avoidance for stacking block drivers: see comments in
526 * bio_alloc_bioset() for details
527 */
528 spinlock_t rescue_lock;
529 struct bio_list rescue_list;
530 struct work_struct rescue_work;
531 struct workqueue_struct *rescue_workqueue;
Kent Overstreet57fb2332012-08-24 04:56:11 -0700532};
533
534struct biovec_slab {
535 int nr_vecs;
536 char *name;
537 struct kmem_cache *slab;
538};
539
540/*
541 * a small number of entries is fine, not going to be performance critical.
542 * basically we just need to survive
543 */
544#define BIO_SPLIT_ENTRIES 2
545
Martin K. Petersen7ba1ba12008-06-30 20:04:41 +0200546#if defined(CONFIG_BLK_DEV_INTEGRITY)
547
548#define bip_vec_idx(bip, idx) (&(bip->bip_vec[(idx)]))
549#define bip_vec(bip) bip_vec_idx(bip, 0)
550
551#define __bip_for_each_vec(bvl, bip, i, start_idx) \
552 for (bvl = bip_vec_idx((bip), (start_idx)), i = (start_idx); \
553 i < (bip)->bip_vcnt; \
554 bvl++, i++)
555
556#define bip_for_each_vec(bvl, bip, i) \
557 __bip_for_each_vec(bvl, bip, i, (bip)->bip_idx)
558
Martin K. Petersen13f05c82010-09-10 20:50:10 +0200559#define bio_for_each_integrity_vec(_bvl, _bio, _iter) \
560 for_each_bio(_bio) \
561 bip_for_each_vec(_bvl, _bio->bi_integrity, _iter)
562
Alberto Bertogli8deaf722008-10-02 12:46:53 +0200563#define bio_integrity(bio) (bio->bi_integrity != NULL)
Martin K. Petersen7ba1ba12008-06-30 20:04:41 +0200564
Martin K. Petersen7ba1ba12008-06-30 20:04:41 +0200565extern struct bio_integrity_payload *bio_integrity_alloc(struct bio *, gfp_t, unsigned int);
Kent Overstreet1e2a410f2012-09-06 15:34:56 -0700566extern void bio_integrity_free(struct bio *);
Martin K. Petersen7ba1ba12008-06-30 20:04:41 +0200567extern int bio_integrity_add_page(struct bio *, struct page *, unsigned int, unsigned int);
568extern int bio_integrity_enabled(struct bio *bio);
569extern int bio_integrity_set_tag(struct bio *, void *, unsigned int);
570extern int bio_integrity_get_tag(struct bio *, void *, unsigned int);
571extern int bio_integrity_prep(struct bio *);
572extern void bio_integrity_endio(struct bio *, int);
573extern void bio_integrity_advance(struct bio *, unsigned int);
574extern void bio_integrity_trim(struct bio *, unsigned int, unsigned int);
575extern void bio_integrity_split(struct bio *, struct bio_pair *, int);
Kent Overstreet1e2a410f2012-09-06 15:34:56 -0700576extern int bio_integrity_clone(struct bio *, struct bio *, gfp_t);
Martin K. Petersen7878cba2009-06-26 15:37:49 +0200577extern int bioset_integrity_create(struct bio_set *, int);
578extern void bioset_integrity_free(struct bio_set *);
579extern void bio_integrity_init(void);
Martin K. Petersen7ba1ba12008-06-30 20:04:41 +0200580
581#else /* CONFIG_BLK_DEV_INTEGRITY */
582
Martin K. Petersen6898e3b2012-01-13 08:15:33 +0100583static inline int bio_integrity(struct bio *bio)
584{
585 return 0;
586}
587
588static inline int bio_integrity_enabled(struct bio *bio)
589{
590 return 0;
591}
592
593static inline int bioset_integrity_create(struct bio_set *bs, int pool_size)
594{
595 return 0;
596}
597
598static inline void bioset_integrity_free (struct bio_set *bs)
599{
600 return;
601}
602
603static inline int bio_integrity_prep(struct bio *bio)
604{
605 return 0;
606}
607
Kent Overstreet1e2a410f2012-09-06 15:34:56 -0700608static inline void bio_integrity_free(struct bio *bio)
Martin K. Petersen6898e3b2012-01-13 08:15:33 +0100609{
610 return;
611}
612
Stephen Rothwell0c614e22011-11-16 09:21:48 +0100613static inline int bio_integrity_clone(struct bio *bio, struct bio *bio_src,
Kent Overstreet1e2a410f2012-09-06 15:34:56 -0700614 gfp_t gfp_mask)
Stephen Rothwell0c614e22011-11-16 09:21:48 +0100615{
616 return 0;
617}
Martin K. Petersen6898e3b2012-01-13 08:15:33 +0100618
619static inline void bio_integrity_split(struct bio *bio, struct bio_pair *bp,
620 int sectors)
621{
622 return;
623}
624
625static inline void bio_integrity_advance(struct bio *bio,
626 unsigned int bytes_done)
627{
628 return;
629}
630
631static inline void bio_integrity_trim(struct bio *bio, unsigned int offset,
632 unsigned int sectors)
633{
634 return;
635}
636
637static inline void bio_integrity_init(void)
638{
639 return;
640}
Martin K. Petersen7ba1ba12008-06-30 20:04:41 +0200641
642#endif /* CONFIG_BLK_DEV_INTEGRITY */
643
David Howells02a5e0a2007-08-11 22:34:32 +0200644#endif /* CONFIG_BLOCK */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700645#endif /* __LINUX_BIO_H */