blob: b6d2d0f1019b3975640a5b0388825b44112be7de [file] [log] [blame]
David Howells831058d2006-08-29 19:06:00 +01001/* bounce buffer handling for block devices
2 *
3 * - Split from highmem.c
4 */
5
6#include <linux/mm.h>
7#include <linux/module.h>
8#include <linux/swap.h>
9#include <linux/bio.h>
10#include <linux/pagemap.h>
11#include <linux/mempool.h>
12#include <linux/blkdev.h>
13#include <linux/init.h>
14#include <linux/hash.h>
15#include <linux/highmem.h>
16#include <linux/blktrace_api.h>
17#include <asm/tlbflush.h>
18
19#define POOL_SIZE 64
20#define ISA_POOL_SIZE 16
21
22static mempool_t *page_pool, *isa_page_pool;
23
24#ifdef CONFIG_HIGHMEM
25static __init int init_emergency_pool(void)
26{
27 struct sysinfo i;
28 si_meminfo(&i);
29 si_swapinfo(&i);
30
31 if (!i.totalhigh)
32 return 0;
33
34 page_pool = mempool_create_page_pool(POOL_SIZE, 0);
35 BUG_ON(!page_pool);
36 printk("highmem bounce pool size: %d pages\n", POOL_SIZE);
37
38 return 0;
39}
40
41__initcall(init_emergency_pool);
42
43/*
44 * highmem version, map in to vec
45 */
46static void bounce_copy_vec(struct bio_vec *to, unsigned char *vfrom)
47{
48 unsigned long flags;
49 unsigned char *vto;
50
51 local_irq_save(flags);
52 vto = kmap_atomic(to->bv_page, KM_BOUNCE_READ);
53 memcpy(vto + to->bv_offset, vfrom, to->bv_len);
54 kunmap_atomic(vto, KM_BOUNCE_READ);
55 local_irq_restore(flags);
56}
57
58#else /* CONFIG_HIGHMEM */
59
60#define bounce_copy_vec(to, vfrom) \
61 memcpy(page_address((to)->bv_page) + (to)->bv_offset, vfrom, (to)->bv_len)
62
63#endif /* CONFIG_HIGHMEM */
64
65/*
66 * allocate pages in the DMA region for the ISA pool
67 */
68static void *mempool_alloc_pages_isa(gfp_t gfp_mask, void *data)
69{
70 return mempool_alloc_pages(gfp_mask | GFP_DMA, data);
71}
72
73/*
74 * gets called "every" time someone init's a queue with BLK_BOUNCE_ISA
75 * as the max address, so check if the pool has already been created.
76 */
77int init_emergency_isa_pool(void)
78{
79 if (isa_page_pool)
80 return 0;
81
82 isa_page_pool = mempool_create(ISA_POOL_SIZE, mempool_alloc_pages_isa,
83 mempool_free_pages, (void *) 0);
84 BUG_ON(!isa_page_pool);
85
86 printk("isa bounce pool size: %d pages\n", ISA_POOL_SIZE);
87 return 0;
88}
89
90/*
91 * Simple bounce buffer support for highmem pages. Depending on the
92 * queue gfp mask set, *to may or may not be a highmem page. kmap it
93 * always, it will do the Right Thing
94 */
95static void copy_to_high_bio_irq(struct bio *to, struct bio *from)
96{
97 unsigned char *vfrom;
98 struct bio_vec *tovec, *fromvec;
99 int i;
100
101 __bio_for_each_segment(tovec, to, i, 0) {
102 fromvec = from->bi_io_vec + i;
103
104 /*
105 * not bounced
106 */
107 if (tovec->bv_page == fromvec->bv_page)
108 continue;
109
110 /*
111 * fromvec->bv_offset and fromvec->bv_len might have been
112 * modified by the block layer, so use the original copy,
113 * bounce_copy_vec already uses tovec->bv_len
114 */
115 vfrom = page_address(fromvec->bv_page) + tovec->bv_offset;
116
117 flush_dcache_page(tovec->bv_page);
118 bounce_copy_vec(tovec, vfrom);
119 }
120}
121
122static void bounce_end_io(struct bio *bio, mempool_t *pool, int err)
123{
124 struct bio *bio_orig = bio->bi_private;
125 struct bio_vec *bvec, *org_vec;
126 int i;
127
128 if (test_bit(BIO_EOPNOTSUPP, &bio->bi_flags))
129 set_bit(BIO_EOPNOTSUPP, &bio_orig->bi_flags);
130
131 /*
132 * free up bounce indirect pages used
133 */
134 __bio_for_each_segment(bvec, bio, i, 0) {
135 org_vec = bio_orig->bi_io_vec + i;
136 if (bvec->bv_page == org_vec->bv_page)
137 continue;
138
139 dec_zone_page_state(bvec->bv_page, NR_BOUNCE);
140 mempool_free(bvec->bv_page, pool);
141 }
142
NeilBrown6712ecf2007-09-27 12:47:43 +0200143 bio_endio(bio_orig, err);
David Howells831058d2006-08-29 19:06:00 +0100144 bio_put(bio);
145}
146
NeilBrown6712ecf2007-09-27 12:47:43 +0200147static void bounce_end_io_write(struct bio *bio, int err)
David Howells831058d2006-08-29 19:06:00 +0100148{
David Howells831058d2006-08-29 19:06:00 +0100149 bounce_end_io(bio, page_pool, err);
David Howells831058d2006-08-29 19:06:00 +0100150}
151
NeilBrown6712ecf2007-09-27 12:47:43 +0200152static void bounce_end_io_write_isa(struct bio *bio, int err)
David Howells831058d2006-08-29 19:06:00 +0100153{
David Howells831058d2006-08-29 19:06:00 +0100154
155 bounce_end_io(bio, isa_page_pool, err);
David Howells831058d2006-08-29 19:06:00 +0100156}
157
158static void __bounce_end_io_read(struct bio *bio, mempool_t *pool, int err)
159{
160 struct bio *bio_orig = bio->bi_private;
161
162 if (test_bit(BIO_UPTODATE, &bio->bi_flags))
163 copy_to_high_bio_irq(bio_orig, bio);
164
165 bounce_end_io(bio, pool, err);
166}
167
NeilBrown6712ecf2007-09-27 12:47:43 +0200168static void bounce_end_io_read(struct bio *bio, int err)
David Howells831058d2006-08-29 19:06:00 +0100169{
David Howells831058d2006-08-29 19:06:00 +0100170 __bounce_end_io_read(bio, page_pool, err);
David Howells831058d2006-08-29 19:06:00 +0100171}
172
NeilBrown6712ecf2007-09-27 12:47:43 +0200173static void bounce_end_io_read_isa(struct bio *bio, int err)
David Howells831058d2006-08-29 19:06:00 +0100174{
David Howells831058d2006-08-29 19:06:00 +0100175 __bounce_end_io_read(bio, isa_page_pool, err);
David Howells831058d2006-08-29 19:06:00 +0100176}
177
Jens Axboe165125e2007-07-24 09:28:11 +0200178static void __blk_queue_bounce(struct request_queue *q, struct bio **bio_orig,
David Howells831058d2006-08-29 19:06:00 +0100179 mempool_t *pool)
180{
181 struct page *page;
182 struct bio *bio = NULL;
183 int i, rw = bio_data_dir(*bio_orig);
184 struct bio_vec *to, *from;
185
186 bio_for_each_segment(from, *bio_orig, i) {
187 page = from->bv_page;
188
189 /*
190 * is destination page below bounce pfn?
191 */
Vasily Tarasovf772b3d2007-03-27 08:52:47 +0200192 if (page_to_pfn(page) <= q->bounce_pfn)
David Howells831058d2006-08-29 19:06:00 +0100193 continue;
194
195 /*
196 * irk, bounce it
197 */
198 if (!bio)
199 bio = bio_alloc(GFP_NOIO, (*bio_orig)->bi_vcnt);
200
201 to = bio->bi_io_vec + i;
202
203 to->bv_page = mempool_alloc(pool, q->bounce_gfp);
204 to->bv_len = from->bv_len;
205 to->bv_offset = from->bv_offset;
206 inc_zone_page_state(to->bv_page, NR_BOUNCE);
207
208 if (rw == WRITE) {
209 char *vto, *vfrom;
210
211 flush_dcache_page(from->bv_page);
212 vto = page_address(to->bv_page) + to->bv_offset;
213 vfrom = kmap(from->bv_page) + from->bv_offset;
214 memcpy(vto, vfrom, to->bv_len);
215 kunmap(from->bv_page);
216 }
217 }
218
219 /*
220 * no pages bounced
221 */
222 if (!bio)
223 return;
224
Jens Axboec43a5082007-01-12 12:20:26 +0100225 blk_add_trace_bio(q, *bio_orig, BLK_TA_BOUNCE);
226
David Howells831058d2006-08-29 19:06:00 +0100227 /*
228 * at least one page was bounced, fill in possible non-highmem
229 * pages
230 */
231 __bio_for_each_segment(from, *bio_orig, i, 0) {
232 to = bio_iovec_idx(bio, i);
233 if (!to->bv_page) {
234 to->bv_page = from->bv_page;
235 to->bv_len = from->bv_len;
236 to->bv_offset = from->bv_offset;
237 }
238 }
239
240 bio->bi_bdev = (*bio_orig)->bi_bdev;
241 bio->bi_flags |= (1 << BIO_BOUNCED);
242 bio->bi_sector = (*bio_orig)->bi_sector;
243 bio->bi_rw = (*bio_orig)->bi_rw;
244
245 bio->bi_vcnt = (*bio_orig)->bi_vcnt;
246 bio->bi_idx = (*bio_orig)->bi_idx;
247 bio->bi_size = (*bio_orig)->bi_size;
248
249 if (pool == page_pool) {
250 bio->bi_end_io = bounce_end_io_write;
251 if (rw == READ)
252 bio->bi_end_io = bounce_end_io_read;
253 } else {
254 bio->bi_end_io = bounce_end_io_write_isa;
255 if (rw == READ)
256 bio->bi_end_io = bounce_end_io_read_isa;
257 }
258
259 bio->bi_private = *bio_orig;
260 *bio_orig = bio;
261}
262
Jens Axboe165125e2007-07-24 09:28:11 +0200263void blk_queue_bounce(struct request_queue *q, struct bio **bio_orig)
David Howells831058d2006-08-29 19:06:00 +0100264{
265 mempool_t *pool;
266
267 /*
Jens Axboebf2de6f2007-09-27 13:01:25 +0200268 * Data-less bio, nothing to bounce
269 */
270 if (bio_empty_barrier(*bio_orig))
271 return;
272
273 /*
David Howells831058d2006-08-29 19:06:00 +0100274 * for non-isa bounce case, just check if the bounce pfn is equal
275 * to or bigger than the highest pfn in the system -- in that case,
276 * don't waste time iterating over bio segments
277 */
278 if (!(q->bounce_gfp & GFP_DMA)) {
279 if (q->bounce_pfn >= blk_max_pfn)
280 return;
281 pool = page_pool;
282 } else {
283 BUG_ON(!isa_page_pool);
284 pool = isa_page_pool;
285 }
286
David Howells831058d2006-08-29 19:06:00 +0100287 /*
288 * slow path
289 */
290 __blk_queue_bounce(q, bio_orig, pool);
291}
292
293EXPORT_SYMBOL(blk_queue_bounce);