blob: ce62324c78e7dbc734a2793c39a89de38238e27b [file] [log] [blame]
Chris Masonc8b97812008-10-29 14:49:59 -04001/*
2 * Copyright (C) 2008 Oracle. All rights reserved.
3 *
4 * This program is free software; you can redistribute it and/or
5 * modify it under the terms of the GNU General Public
6 * License v2 as published by the Free Software Foundation.
7 *
8 * This program is distributed in the hope that it will be useful,
9 * but WITHOUT ANY WARRANTY; without even the implied warranty of
10 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
11 * General Public License for more details.
12 *
13 * You should have received a copy of the GNU General Public
14 * License along with this program; if not, write to the
15 * Free Software Foundation, Inc., 59 Temple Place - Suite 330,
16 * Boston, MA 021110-1307, USA.
17 */
18
19#include <linux/kernel.h>
20#include <linux/bio.h>
21#include <linux/buffer_head.h>
22#include <linux/file.h>
23#include <linux/fs.h>
24#include <linux/pagemap.h>
25#include <linux/highmem.h>
26#include <linux/time.h>
27#include <linux/init.h>
28#include <linux/string.h>
Chris Masonc8b97812008-10-29 14:49:59 -040029#include <linux/backing-dev.h>
30#include <linux/mpage.h>
31#include <linux/swap.h>
32#include <linux/writeback.h>
33#include <linux/bit_spinlock.h>
Tejun Heo5a0e3ad2010-03-24 17:04:11 +090034#include <linux/slab.h>
Chris Masonc8b97812008-10-29 14:49:59 -040035#include "ctree.h"
36#include "disk-io.h"
37#include "transaction.h"
38#include "btrfs_inode.h"
39#include "volumes.h"
40#include "ordered-data.h"
Chris Masonc8b97812008-10-29 14:49:59 -040041#include "compression.h"
42#include "extent_io.h"
43#include "extent_map.h"
44
45struct compressed_bio {
46 /* number of bios pending for this compressed extent */
47 atomic_t pending_bios;
48
49 /* the pages with the compressed data on them */
50 struct page **compressed_pages;
51
52 /* inode that owns this data */
53 struct inode *inode;
54
55 /* starting offset in the inode for our pages */
56 u64 start;
57
58 /* number of bytes in the inode we're working on */
59 unsigned long len;
60
61 /* number of bytes on disk */
62 unsigned long compressed_len;
63
Li Zefan261507a02010-12-17 14:21:50 +080064 /* the compression algorithm for this bio */
65 int compress_type;
66
Chris Masonc8b97812008-10-29 14:49:59 -040067 /* number of compressed pages in the array */
68 unsigned long nr_pages;
69
70 /* IO errors */
71 int errors;
Chris Masond20f7042008-12-08 16:58:54 -050072 int mirror_num;
Chris Masonc8b97812008-10-29 14:49:59 -040073
74 /* for reads, this is the bio we are copying the data into */
75 struct bio *orig_bio;
Chris Masond20f7042008-12-08 16:58:54 -050076
77 /*
78 * the start of a variable length array of checksums only
79 * used by reads
80 */
81 u32 sums;
Chris Masonc8b97812008-10-29 14:49:59 -040082};
83
Eric Sandeen48a3b632013-04-25 20:41:01 +000084static int btrfs_decompress_biovec(int type, struct page **pages_in,
85 u64 disk_start, struct bio_vec *bvec,
86 int vcnt, size_t srclen);
87
Chris Masond20f7042008-12-08 16:58:54 -050088static inline int compressed_bio_size(struct btrfs_root *root,
89 unsigned long disk_size)
90{
David Sterba6c417612011-04-13 15:41:04 +020091 u16 csum_size = btrfs_super_csum_size(root->fs_info->super_copy);
92
Chris Masond20f7042008-12-08 16:58:54 -050093 return sizeof(struct compressed_bio) +
David Sterbaed6078f2014-06-05 01:59:57 +020094 (DIV_ROUND_UP(disk_size, root->sectorsize)) * csum_size;
Chris Masond20f7042008-12-08 16:58:54 -050095}
96
Chris Masonc8b97812008-10-29 14:49:59 -040097static struct bio *compressed_bio_alloc(struct block_device *bdev,
98 u64 first_byte, gfp_t gfp_flags)
99{
Chris Masonc8b97812008-10-29 14:49:59 -0400100 int nr_vecs;
101
102 nr_vecs = bio_get_nr_vecs(bdev);
Miao Xie88f794e2010-11-22 03:02:55 +0000103 return btrfs_bio_alloc(bdev, first_byte >> 9, nr_vecs, gfp_flags);
Chris Masonc8b97812008-10-29 14:49:59 -0400104}
105
Chris Masond20f7042008-12-08 16:58:54 -0500106static int check_compressed_csum(struct inode *inode,
107 struct compressed_bio *cb,
108 u64 disk_start)
109{
110 int ret;
Chris Masond20f7042008-12-08 16:58:54 -0500111 struct page *page;
112 unsigned long i;
113 char *kaddr;
114 u32 csum;
115 u32 *cb_sum = &cb->sums;
116
Christoph Hellwig6cbff002009-04-17 10:37:41 +0200117 if (BTRFS_I(inode)->flags & BTRFS_INODE_NODATASUM)
Chris Masond20f7042008-12-08 16:58:54 -0500118 return 0;
119
120 for (i = 0; i < cb->nr_pages; i++) {
121 page = cb->compressed_pages[i];
122 csum = ~(u32)0;
123
Cong Wang7ac687d2011-11-25 23:14:28 +0800124 kaddr = kmap_atomic(page);
Liu Bob0496682013-03-14 14:57:45 +0000125 csum = btrfs_csum_data(kaddr, csum, PAGE_CACHE_SIZE);
Chris Masond20f7042008-12-08 16:58:54 -0500126 btrfs_csum_final(csum, (char *)&csum);
Cong Wang7ac687d2011-11-25 23:14:28 +0800127 kunmap_atomic(kaddr);
Chris Masond20f7042008-12-08 16:58:54 -0500128
129 if (csum != *cb_sum) {
Frank Holtonefe120a2013-12-20 11:37:06 -0500130 btrfs_info(BTRFS_I(inode)->root->fs_info,
131 "csum failed ino %llu extent %llu csum %u wanted %u mirror %d",
132 btrfs_ino(inode), disk_start, csum, *cb_sum,
133 cb->mirror_num);
Chris Masond20f7042008-12-08 16:58:54 -0500134 ret = -EIO;
135 goto fail;
136 }
137 cb_sum++;
138
139 }
140 ret = 0;
141fail:
142 return ret;
143}
144
Chris Masonc8b97812008-10-29 14:49:59 -0400145/* when we finish reading compressed pages from the disk, we
146 * decompress them and then run the bio end_io routines on the
147 * decompressed pages (in the inode address space).
148 *
149 * This allows the checksumming and other IO error handling routines
150 * to work normally
151 *
152 * The compressed pages are freed here, and it must be run
153 * in process context
154 */
155static void end_compressed_bio_read(struct bio *bio, int err)
156{
Chris Masonc8b97812008-10-29 14:49:59 -0400157 struct compressed_bio *cb = bio->bi_private;
158 struct inode *inode;
159 struct page *page;
160 unsigned long index;
161 int ret;
162
163 if (err)
164 cb->errors = 1;
165
166 /* if there are more bios still pending for this compressed
167 * extent, just exit
168 */
169 if (!atomic_dec_and_test(&cb->pending_bios))
170 goto out;
171
Chris Masond20f7042008-12-08 16:58:54 -0500172 inode = cb->inode;
Kent Overstreet4f024f32013-10-11 15:44:27 -0700173 ret = check_compressed_csum(inode, cb,
174 (u64)bio->bi_iter.bi_sector << 9);
Chris Masond20f7042008-12-08 16:58:54 -0500175 if (ret)
176 goto csum_failed;
177
Chris Masonc8b97812008-10-29 14:49:59 -0400178 /* ok, we're the last bio for this extent, lets start
179 * the decompression.
180 */
Li Zefan261507a02010-12-17 14:21:50 +0800181 ret = btrfs_decompress_biovec(cb->compress_type,
182 cb->compressed_pages,
183 cb->start,
184 cb->orig_bio->bi_io_vec,
185 cb->orig_bio->bi_vcnt,
186 cb->compressed_len);
Chris Masond20f7042008-12-08 16:58:54 -0500187csum_failed:
Chris Masonc8b97812008-10-29 14:49:59 -0400188 if (ret)
189 cb->errors = 1;
190
191 /* release the compressed pages */
192 index = 0;
193 for (index = 0; index < cb->nr_pages; index++) {
194 page = cb->compressed_pages[index];
195 page->mapping = NULL;
196 page_cache_release(page);
197 }
198
199 /* do io completion on the original bio */
Chris Mason771ed682008-11-06 22:02:51 -0500200 if (cb->errors) {
Chris Masonc8b97812008-10-29 14:49:59 -0400201 bio_io_error(cb->orig_bio);
Chris Masond20f7042008-12-08 16:58:54 -0500202 } else {
Kent Overstreet2c30c712013-11-07 12:20:26 -0800203 int i;
204 struct bio_vec *bvec;
Chris Masond20f7042008-12-08 16:58:54 -0500205
206 /*
207 * we have verified the checksum already, set page
208 * checked so the end_io handlers know about it
209 */
Kent Overstreet2c30c712013-11-07 12:20:26 -0800210 bio_for_each_segment_all(bvec, cb->orig_bio, i)
Chris Masond20f7042008-12-08 16:58:54 -0500211 SetPageChecked(bvec->bv_page);
Kent Overstreet2c30c712013-11-07 12:20:26 -0800212
Chris Masonc8b97812008-10-29 14:49:59 -0400213 bio_endio(cb->orig_bio, 0);
Chris Masond20f7042008-12-08 16:58:54 -0500214 }
Chris Masonc8b97812008-10-29 14:49:59 -0400215
216 /* finally free the cb struct */
217 kfree(cb->compressed_pages);
218 kfree(cb);
219out:
220 bio_put(bio);
221}
222
223/*
224 * Clear the writeback bits on all of the file
225 * pages for a compressed write
226 */
Filipe Manana7bdcefc2014-10-07 01:48:26 +0100227static noinline void end_compressed_writeback(struct inode *inode,
228 const struct compressed_bio *cb)
Chris Masonc8b97812008-10-29 14:49:59 -0400229{
Filipe Manana7bdcefc2014-10-07 01:48:26 +0100230 unsigned long index = cb->start >> PAGE_CACHE_SHIFT;
231 unsigned long end_index = (cb->start + cb->len - 1) >> PAGE_CACHE_SHIFT;
Chris Masonc8b97812008-10-29 14:49:59 -0400232 struct page *pages[16];
233 unsigned long nr_pages = end_index - index + 1;
234 int i;
235 int ret;
236
Filipe Manana7bdcefc2014-10-07 01:48:26 +0100237 if (cb->errors)
238 mapping_set_error(inode->i_mapping, -EIO);
239
Chris Masond3977122009-01-05 21:25:51 -0500240 while (nr_pages > 0) {
Chris Masonc8b97812008-10-29 14:49:59 -0400241 ret = find_get_pages_contig(inode->i_mapping, index,
Chris Mason5b050f02008-11-11 09:34:41 -0500242 min_t(unsigned long,
243 nr_pages, ARRAY_SIZE(pages)), pages);
Chris Masonc8b97812008-10-29 14:49:59 -0400244 if (ret == 0) {
245 nr_pages -= 1;
246 index += 1;
247 continue;
248 }
249 for (i = 0; i < ret; i++) {
Filipe Manana7bdcefc2014-10-07 01:48:26 +0100250 if (cb->errors)
251 SetPageError(pages[i]);
Chris Masonc8b97812008-10-29 14:49:59 -0400252 end_page_writeback(pages[i]);
253 page_cache_release(pages[i]);
254 }
255 nr_pages -= ret;
256 index += ret;
257 }
258 /* the inode may be gone now */
Chris Masonc8b97812008-10-29 14:49:59 -0400259}
260
261/*
262 * do the cleanup once all the compressed pages hit the disk.
263 * This will clear writeback on the file pages and free the compressed
264 * pages.
265 *
266 * This also calls the writeback end hooks for the file pages so that
267 * metadata and checksums can be updated in the file.
268 */
269static void end_compressed_bio_write(struct bio *bio, int err)
270{
271 struct extent_io_tree *tree;
272 struct compressed_bio *cb = bio->bi_private;
273 struct inode *inode;
274 struct page *page;
275 unsigned long index;
276
277 if (err)
278 cb->errors = 1;
279
280 /* if there are more bios still pending for this compressed
281 * extent, just exit
282 */
283 if (!atomic_dec_and_test(&cb->pending_bios))
284 goto out;
285
286 /* ok, we're the last bio for this extent, step one is to
287 * call back into the FS and do all the end_io operations
288 */
289 inode = cb->inode;
290 tree = &BTRFS_I(inode)->io_tree;
Chris Mason70b99e62008-10-31 12:46:39 -0400291 cb->compressed_pages[0]->mapping = cb->inode->i_mapping;
Chris Masonc8b97812008-10-29 14:49:59 -0400292 tree->ops->writepage_end_io_hook(cb->compressed_pages[0],
293 cb->start,
294 cb->start + cb->len - 1,
Filipe Manana7bdcefc2014-10-07 01:48:26 +0100295 NULL,
296 err ? 0 : 1);
Chris Mason70b99e62008-10-31 12:46:39 -0400297 cb->compressed_pages[0]->mapping = NULL;
Chris Masonc8b97812008-10-29 14:49:59 -0400298
Filipe Manana7bdcefc2014-10-07 01:48:26 +0100299 end_compressed_writeback(inode, cb);
Chris Masonc8b97812008-10-29 14:49:59 -0400300 /* note, our inode could be gone now */
301
302 /*
303 * release the compressed pages, these came from alloc_page and
304 * are not attached to the inode at all
305 */
306 index = 0;
307 for (index = 0; index < cb->nr_pages; index++) {
308 page = cb->compressed_pages[index];
309 page->mapping = NULL;
310 page_cache_release(page);
311 }
312
313 /* finally free the cb struct */
314 kfree(cb->compressed_pages);
315 kfree(cb);
316out:
317 bio_put(bio);
318}
319
320/*
321 * worker function to build and submit bios for previously compressed pages.
322 * The corresponding pages in the inode should be marked for writeback
323 * and the compressed pages should have a reference on them for dropping
324 * when the IO is complete.
325 *
326 * This also checksums the file bytes and gets things ready for
327 * the end io hooks.
328 */
329int btrfs_submit_compressed_write(struct inode *inode, u64 start,
330 unsigned long len, u64 disk_start,
331 unsigned long compressed_len,
332 struct page **compressed_pages,
333 unsigned long nr_pages)
334{
335 struct bio *bio = NULL;
336 struct btrfs_root *root = BTRFS_I(inode)->root;
337 struct compressed_bio *cb;
338 unsigned long bytes_left;
339 struct extent_io_tree *io_tree = &BTRFS_I(inode)->io_tree;
David Sterba306e16c2011-04-19 14:29:38 +0200340 int pg_index = 0;
Chris Masonc8b97812008-10-29 14:49:59 -0400341 struct page *page;
342 u64 first_byte = disk_start;
343 struct block_device *bdev;
344 int ret;
Li Zefane55179b2011-07-14 03:16:47 +0000345 int skip_sum = BTRFS_I(inode)->flags & BTRFS_INODE_NODATASUM;
Chris Masonc8b97812008-10-29 14:49:59 -0400346
347 WARN_ON(start & ((u64)PAGE_CACHE_SIZE - 1));
Chris Masond20f7042008-12-08 16:58:54 -0500348 cb = kmalloc(compressed_bio_size(root, compressed_len), GFP_NOFS);
Yoshinori Sanodac97e52011-02-15 12:01:42 +0000349 if (!cb)
350 return -ENOMEM;
Chris Masonc8b97812008-10-29 14:49:59 -0400351 atomic_set(&cb->pending_bios, 0);
352 cb->errors = 0;
353 cb->inode = inode;
354 cb->start = start;
355 cb->len = len;
Chris Masond20f7042008-12-08 16:58:54 -0500356 cb->mirror_num = 0;
Chris Masonc8b97812008-10-29 14:49:59 -0400357 cb->compressed_pages = compressed_pages;
358 cb->compressed_len = compressed_len;
359 cb->orig_bio = NULL;
360 cb->nr_pages = nr_pages;
361
362 bdev = BTRFS_I(inode)->root->fs_info->fs_devices->latest_bdev;
363
Chris Masonc8b97812008-10-29 14:49:59 -0400364 bio = compressed_bio_alloc(bdev, first_byte, GFP_NOFS);
Dulshani Gunawardhana67871252013-10-31 10:33:04 +0530365 if (!bio) {
Yoshinori Sanodac97e52011-02-15 12:01:42 +0000366 kfree(cb);
367 return -ENOMEM;
368 }
Chris Masonc8b97812008-10-29 14:49:59 -0400369 bio->bi_private = cb;
370 bio->bi_end_io = end_compressed_bio_write;
371 atomic_inc(&cb->pending_bios);
372
373 /* create and submit bios for the compressed pages */
374 bytes_left = compressed_len;
David Sterba306e16c2011-04-19 14:29:38 +0200375 for (pg_index = 0; pg_index < cb->nr_pages; pg_index++) {
376 page = compressed_pages[pg_index];
Chris Masonc8b97812008-10-29 14:49:59 -0400377 page->mapping = inode->i_mapping;
Kent Overstreet4f024f32013-10-11 15:44:27 -0700378 if (bio->bi_iter.bi_size)
David Woodhouse64a16702009-07-15 23:29:37 +0100379 ret = io_tree->ops->merge_bio_hook(WRITE, page, 0,
Chris Masonc8b97812008-10-29 14:49:59 -0400380 PAGE_CACHE_SIZE,
381 bio, 0);
382 else
383 ret = 0;
384
Chris Mason70b99e62008-10-31 12:46:39 -0400385 page->mapping = NULL;
Chris Masonc8b97812008-10-29 14:49:59 -0400386 if (ret || bio_add_page(bio, page, PAGE_CACHE_SIZE, 0) <
387 PAGE_CACHE_SIZE) {
388 bio_get(bio);
389
Chris Masonaf09abf2008-11-07 12:35:44 -0500390 /*
391 * inc the count before we submit the bio so
392 * we know the end IO handler won't happen before
393 * we inc the count. Otherwise, the cb might get
394 * freed before we're done setting it up
395 */
396 atomic_inc(&cb->pending_bios);
David Sterbabfebd8b2014-07-30 00:25:45 +0200397 ret = btrfs_bio_wq_end_io(root->fs_info, bio,
398 BTRFS_WQ_ENDIO_DATA);
Jeff Mahoney79787ea2012-03-12 16:03:00 +0100399 BUG_ON(ret); /* -ENOMEM */
Chris Masonc8b97812008-10-29 14:49:59 -0400400
Li Zefane55179b2011-07-14 03:16:47 +0000401 if (!skip_sum) {
402 ret = btrfs_csum_one_bio(root, inode, bio,
403 start, 1);
Jeff Mahoney79787ea2012-03-12 16:03:00 +0100404 BUG_ON(ret); /* -ENOMEM */
Li Zefane55179b2011-07-14 03:16:47 +0000405 }
Chris Masond20f7042008-12-08 16:58:54 -0500406
Chris Masonc8b97812008-10-29 14:49:59 -0400407 ret = btrfs_map_bio(root, WRITE, bio, 0, 1);
Jeff Mahoney79787ea2012-03-12 16:03:00 +0100408 BUG_ON(ret); /* -ENOMEM */
Chris Masonc8b97812008-10-29 14:49:59 -0400409
410 bio_put(bio);
411
412 bio = compressed_bio_alloc(bdev, first_byte, GFP_NOFS);
Tsutomu Itohe627ee72012-04-12 16:03:56 -0400413 BUG_ON(!bio);
Chris Masonc8b97812008-10-29 14:49:59 -0400414 bio->bi_private = cb;
415 bio->bi_end_io = end_compressed_bio_write;
416 bio_add_page(bio, page, PAGE_CACHE_SIZE, 0);
417 }
Chris Masoncfbc2462008-10-30 13:22:14 -0400418 if (bytes_left < PAGE_CACHE_SIZE) {
Frank Holtonefe120a2013-12-20 11:37:06 -0500419 btrfs_info(BTRFS_I(inode)->root->fs_info,
420 "bytes left %lu compress len %lu nr %lu",
Chris Masoncfbc2462008-10-30 13:22:14 -0400421 bytes_left, cb->compressed_len, cb->nr_pages);
422 }
Chris Masonc8b97812008-10-29 14:49:59 -0400423 bytes_left -= PAGE_CACHE_SIZE;
424 first_byte += PAGE_CACHE_SIZE;
Chris Mason771ed682008-11-06 22:02:51 -0500425 cond_resched();
Chris Masonc8b97812008-10-29 14:49:59 -0400426 }
427 bio_get(bio);
428
David Sterbabfebd8b2014-07-30 00:25:45 +0200429 ret = btrfs_bio_wq_end_io(root->fs_info, bio, BTRFS_WQ_ENDIO_DATA);
Jeff Mahoney79787ea2012-03-12 16:03:00 +0100430 BUG_ON(ret); /* -ENOMEM */
Chris Masonc8b97812008-10-29 14:49:59 -0400431
Li Zefane55179b2011-07-14 03:16:47 +0000432 if (!skip_sum) {
433 ret = btrfs_csum_one_bio(root, inode, bio, start, 1);
Jeff Mahoney79787ea2012-03-12 16:03:00 +0100434 BUG_ON(ret); /* -ENOMEM */
Li Zefane55179b2011-07-14 03:16:47 +0000435 }
Chris Masond20f7042008-12-08 16:58:54 -0500436
Chris Masonc8b97812008-10-29 14:49:59 -0400437 ret = btrfs_map_bio(root, WRITE, bio, 0, 1);
Jeff Mahoney79787ea2012-03-12 16:03:00 +0100438 BUG_ON(ret); /* -ENOMEM */
Chris Masonc8b97812008-10-29 14:49:59 -0400439
440 bio_put(bio);
441 return 0;
442}
443
Chris Mason771ed682008-11-06 22:02:51 -0500444static noinline int add_ra_bio_pages(struct inode *inode,
445 u64 compressed_end,
446 struct compressed_bio *cb)
447{
448 unsigned long end_index;
David Sterba306e16c2011-04-19 14:29:38 +0200449 unsigned long pg_index;
Chris Mason771ed682008-11-06 22:02:51 -0500450 u64 last_offset;
451 u64 isize = i_size_read(inode);
452 int ret;
453 struct page *page;
454 unsigned long nr_pages = 0;
455 struct extent_map *em;
456 struct address_space *mapping = inode->i_mapping;
Chris Mason771ed682008-11-06 22:02:51 -0500457 struct extent_map_tree *em_tree;
458 struct extent_io_tree *tree;
459 u64 end;
460 int misses = 0;
461
462 page = cb->orig_bio->bi_io_vec[cb->orig_bio->bi_vcnt - 1].bv_page;
463 last_offset = (page_offset(page) + PAGE_CACHE_SIZE);
464 em_tree = &BTRFS_I(inode)->extent_tree;
465 tree = &BTRFS_I(inode)->io_tree;
466
467 if (isize == 0)
468 return 0;
469
470 end_index = (i_size_read(inode) - 1) >> PAGE_CACHE_SHIFT;
471
Chris Masond3977122009-01-05 21:25:51 -0500472 while (last_offset < compressed_end) {
David Sterba306e16c2011-04-19 14:29:38 +0200473 pg_index = last_offset >> PAGE_CACHE_SHIFT;
Chris Mason771ed682008-11-06 22:02:51 -0500474
David Sterba306e16c2011-04-19 14:29:38 +0200475 if (pg_index > end_index)
Chris Mason771ed682008-11-06 22:02:51 -0500476 break;
477
478 rcu_read_lock();
David Sterba306e16c2011-04-19 14:29:38 +0200479 page = radix_tree_lookup(&mapping->page_tree, pg_index);
Chris Mason771ed682008-11-06 22:02:51 -0500480 rcu_read_unlock();
Johannes Weiner0cd61442014-04-03 14:47:46 -0700481 if (page && !radix_tree_exceptional_entry(page)) {
Chris Mason771ed682008-11-06 22:02:51 -0500482 misses++;
483 if (misses > 4)
484 break;
485 goto next;
486 }
487
Nick Piggin28ecb6092010-03-17 13:31:04 +0000488 page = __page_cache_alloc(mapping_gfp_mask(mapping) &
489 ~__GFP_FS);
Chris Mason771ed682008-11-06 22:02:51 -0500490 if (!page)
491 break;
492
David Sterba306e16c2011-04-19 14:29:38 +0200493 if (add_to_page_cache_lru(page, mapping, pg_index,
Nick Piggin28ecb6092010-03-17 13:31:04 +0000494 GFP_NOFS)) {
Chris Mason771ed682008-11-06 22:02:51 -0500495 page_cache_release(page);
496 goto next;
497 }
498
Chris Mason771ed682008-11-06 22:02:51 -0500499 end = last_offset + PAGE_CACHE_SIZE - 1;
500 /*
501 * at this point, we have a locked page in the page cache
502 * for these bytes in the file. But, we have to make
503 * sure they map to this compressed extent on disk.
504 */
505 set_page_extent_mapped(page);
Jeff Mahoneyd0082372012-03-01 14:57:19 +0100506 lock_extent(tree, last_offset, end);
Chris Mason890871b2009-09-02 16:24:52 -0400507 read_lock(&em_tree->lock);
Chris Mason771ed682008-11-06 22:02:51 -0500508 em = lookup_extent_mapping(em_tree, last_offset,
509 PAGE_CACHE_SIZE);
Chris Mason890871b2009-09-02 16:24:52 -0400510 read_unlock(&em_tree->lock);
Chris Mason771ed682008-11-06 22:02:51 -0500511
512 if (!em || last_offset < em->start ||
513 (last_offset + PAGE_CACHE_SIZE > extent_map_end(em)) ||
Kent Overstreet4f024f32013-10-11 15:44:27 -0700514 (em->block_start >> 9) != cb->orig_bio->bi_iter.bi_sector) {
Chris Mason771ed682008-11-06 22:02:51 -0500515 free_extent_map(em);
Jeff Mahoneyd0082372012-03-01 14:57:19 +0100516 unlock_extent(tree, last_offset, end);
Chris Mason771ed682008-11-06 22:02:51 -0500517 unlock_page(page);
518 page_cache_release(page);
519 break;
520 }
521 free_extent_map(em);
522
523 if (page->index == end_index) {
524 char *userpage;
525 size_t zero_offset = isize & (PAGE_CACHE_SIZE - 1);
526
527 if (zero_offset) {
528 int zeros;
529 zeros = PAGE_CACHE_SIZE - zero_offset;
Cong Wang7ac687d2011-11-25 23:14:28 +0800530 userpage = kmap_atomic(page);
Chris Mason771ed682008-11-06 22:02:51 -0500531 memset(userpage + zero_offset, 0, zeros);
532 flush_dcache_page(page);
Cong Wang7ac687d2011-11-25 23:14:28 +0800533 kunmap_atomic(userpage);
Chris Mason771ed682008-11-06 22:02:51 -0500534 }
535 }
536
537 ret = bio_add_page(cb->orig_bio, page,
538 PAGE_CACHE_SIZE, 0);
539
540 if (ret == PAGE_CACHE_SIZE) {
541 nr_pages++;
542 page_cache_release(page);
543 } else {
Jeff Mahoneyd0082372012-03-01 14:57:19 +0100544 unlock_extent(tree, last_offset, end);
Chris Mason771ed682008-11-06 22:02:51 -0500545 unlock_page(page);
546 page_cache_release(page);
547 break;
548 }
549next:
550 last_offset += PAGE_CACHE_SIZE;
551 }
Chris Mason771ed682008-11-06 22:02:51 -0500552 return 0;
553}
554
Chris Masonc8b97812008-10-29 14:49:59 -0400555/*
556 * for a compressed read, the bio we get passed has all the inode pages
557 * in it. We don't actually do IO on those pages but allocate new ones
558 * to hold the compressed pages on disk.
559 *
Kent Overstreet4f024f32013-10-11 15:44:27 -0700560 * bio->bi_iter.bi_sector points to the compressed extent on disk
Chris Masonc8b97812008-10-29 14:49:59 -0400561 * bio->bi_io_vec points to all of the inode pages
562 * bio->bi_vcnt is a count of pages
563 *
564 * After the compressed pages are read, we copy the bytes into the
565 * bio we were passed and then call the bio end_io calls
566 */
567int btrfs_submit_compressed_read(struct inode *inode, struct bio *bio,
568 int mirror_num, unsigned long bio_flags)
569{
570 struct extent_io_tree *tree;
571 struct extent_map_tree *em_tree;
572 struct compressed_bio *cb;
573 struct btrfs_root *root = BTRFS_I(inode)->root;
574 unsigned long uncompressed_len = bio->bi_vcnt * PAGE_CACHE_SIZE;
575 unsigned long compressed_len;
576 unsigned long nr_pages;
David Sterba306e16c2011-04-19 14:29:38 +0200577 unsigned long pg_index;
Chris Masonc8b97812008-10-29 14:49:59 -0400578 struct page *page;
579 struct block_device *bdev;
580 struct bio *comp_bio;
Kent Overstreet4f024f32013-10-11 15:44:27 -0700581 u64 cur_disk_byte = (u64)bio->bi_iter.bi_sector << 9;
Chris Masone04ca622008-11-10 11:44:58 -0500582 u64 em_len;
583 u64 em_start;
Chris Masonc8b97812008-10-29 14:49:59 -0400584 struct extent_map *em;
liubo6b82ce82011-01-26 06:21:39 +0000585 int ret = -ENOMEM;
Josef Bacik15e30042012-10-05 13:39:50 -0400586 int faili = 0;
Chris Masond20f7042008-12-08 16:58:54 -0500587 u32 *sums;
Chris Masonc8b97812008-10-29 14:49:59 -0400588
589 tree = &BTRFS_I(inode)->io_tree;
590 em_tree = &BTRFS_I(inode)->extent_tree;
591
592 /* we need the actual starting offset of this extent in the file */
Chris Mason890871b2009-09-02 16:24:52 -0400593 read_lock(&em_tree->lock);
Chris Masonc8b97812008-10-29 14:49:59 -0400594 em = lookup_extent_mapping(em_tree,
595 page_offset(bio->bi_io_vec->bv_page),
596 PAGE_CACHE_SIZE);
Chris Mason890871b2009-09-02 16:24:52 -0400597 read_unlock(&em_tree->lock);
Tsutomu Itoh285190d2012-02-16 16:23:58 +0900598 if (!em)
599 return -EIO;
Chris Masonc8b97812008-10-29 14:49:59 -0400600
Chris Masond20f7042008-12-08 16:58:54 -0500601 compressed_len = em->block_len;
602 cb = kmalloc(compressed_bio_size(root, compressed_len), GFP_NOFS);
liubo6b82ce82011-01-26 06:21:39 +0000603 if (!cb)
604 goto out;
605
Chris Masonc8b97812008-10-29 14:49:59 -0400606 atomic_set(&cb->pending_bios, 0);
607 cb->errors = 0;
608 cb->inode = inode;
Chris Masond20f7042008-12-08 16:58:54 -0500609 cb->mirror_num = mirror_num;
610 sums = &cb->sums;
Chris Masonc8b97812008-10-29 14:49:59 -0400611
Yan Zhengff5b7ee2008-11-10 07:34:43 -0500612 cb->start = em->orig_start;
Chris Masone04ca622008-11-10 11:44:58 -0500613 em_len = em->len;
614 em_start = em->start;
Chris Masond20f7042008-12-08 16:58:54 -0500615
Chris Masonc8b97812008-10-29 14:49:59 -0400616 free_extent_map(em);
Chris Masone04ca622008-11-10 11:44:58 -0500617 em = NULL;
Chris Masonc8b97812008-10-29 14:49:59 -0400618
619 cb->len = uncompressed_len;
620 cb->compressed_len = compressed_len;
Li Zefan261507a02010-12-17 14:21:50 +0800621 cb->compress_type = extent_compress_type(bio_flags);
Chris Masonc8b97812008-10-29 14:49:59 -0400622 cb->orig_bio = bio;
623
David Sterbaed6078f2014-06-05 01:59:57 +0200624 nr_pages = DIV_ROUND_UP(compressed_len, PAGE_CACHE_SIZE);
David Sterba31e818f2015-02-20 18:00:26 +0100625 cb->compressed_pages = kcalloc(nr_pages, sizeof(struct page *),
Chris Masonc8b97812008-10-29 14:49:59 -0400626 GFP_NOFS);
liubo6b82ce82011-01-26 06:21:39 +0000627 if (!cb->compressed_pages)
628 goto fail1;
629
Chris Masonc8b97812008-10-29 14:49:59 -0400630 bdev = BTRFS_I(inode)->root->fs_info->fs_devices->latest_bdev;
631
David Sterba306e16c2011-04-19 14:29:38 +0200632 for (pg_index = 0; pg_index < nr_pages; pg_index++) {
633 cb->compressed_pages[pg_index] = alloc_page(GFP_NOFS |
Chris Masonc8b97812008-10-29 14:49:59 -0400634 __GFP_HIGHMEM);
Josef Bacik15e30042012-10-05 13:39:50 -0400635 if (!cb->compressed_pages[pg_index]) {
636 faili = pg_index - 1;
637 ret = -ENOMEM;
liubo6b82ce82011-01-26 06:21:39 +0000638 goto fail2;
Josef Bacik15e30042012-10-05 13:39:50 -0400639 }
Chris Masonc8b97812008-10-29 14:49:59 -0400640 }
Josef Bacik15e30042012-10-05 13:39:50 -0400641 faili = nr_pages - 1;
Chris Masonc8b97812008-10-29 14:49:59 -0400642 cb->nr_pages = nr_pages;
643
Mark Fasheh4b384312013-08-06 11:42:50 -0700644 /* In the parent-locked case, we only locked the range we are
645 * interested in. In all other cases, we can opportunistically
646 * cache decompressed data that goes beyond the requested range. */
647 if (!(bio_flags & EXTENT_BIO_PARENT_LOCKED))
648 add_ra_bio_pages(inode, em_start + em_len, cb);
Chris Mason771ed682008-11-06 22:02:51 -0500649
Chris Mason771ed682008-11-06 22:02:51 -0500650 /* include any pages we added in add_ra-bio_pages */
651 uncompressed_len = bio->bi_vcnt * PAGE_CACHE_SIZE;
652 cb->len = uncompressed_len;
653
Chris Masonc8b97812008-10-29 14:49:59 -0400654 comp_bio = compressed_bio_alloc(bdev, cur_disk_byte, GFP_NOFS);
liubo6b82ce82011-01-26 06:21:39 +0000655 if (!comp_bio)
656 goto fail2;
Chris Masonc8b97812008-10-29 14:49:59 -0400657 comp_bio->bi_private = cb;
658 comp_bio->bi_end_io = end_compressed_bio_read;
659 atomic_inc(&cb->pending_bios);
660
David Sterba306e16c2011-04-19 14:29:38 +0200661 for (pg_index = 0; pg_index < nr_pages; pg_index++) {
662 page = cb->compressed_pages[pg_index];
Chris Masonc8b97812008-10-29 14:49:59 -0400663 page->mapping = inode->i_mapping;
Chris Masond20f7042008-12-08 16:58:54 -0500664 page->index = em_start >> PAGE_CACHE_SHIFT;
665
Kent Overstreet4f024f32013-10-11 15:44:27 -0700666 if (comp_bio->bi_iter.bi_size)
David Woodhouse64a16702009-07-15 23:29:37 +0100667 ret = tree->ops->merge_bio_hook(READ, page, 0,
Chris Masonc8b97812008-10-29 14:49:59 -0400668 PAGE_CACHE_SIZE,
669 comp_bio, 0);
670 else
671 ret = 0;
672
Chris Mason70b99e62008-10-31 12:46:39 -0400673 page->mapping = NULL;
Chris Masonc8b97812008-10-29 14:49:59 -0400674 if (ret || bio_add_page(comp_bio, page, PAGE_CACHE_SIZE, 0) <
675 PAGE_CACHE_SIZE) {
676 bio_get(comp_bio);
677
David Sterbabfebd8b2014-07-30 00:25:45 +0200678 ret = btrfs_bio_wq_end_io(root->fs_info, comp_bio,
679 BTRFS_WQ_ENDIO_DATA);
Jeff Mahoney79787ea2012-03-12 16:03:00 +0100680 BUG_ON(ret); /* -ENOMEM */
Chris Masonc8b97812008-10-29 14:49:59 -0400681
Chris Masonaf09abf2008-11-07 12:35:44 -0500682 /*
683 * inc the count before we submit the bio so
684 * we know the end IO handler won't happen before
685 * we inc the count. Otherwise, the cb might get
686 * freed before we're done setting it up
687 */
688 atomic_inc(&cb->pending_bios);
689
Christoph Hellwig6cbff002009-04-17 10:37:41 +0200690 if (!(BTRFS_I(inode)->flags & BTRFS_INODE_NODATASUM)) {
Tsutomu Itohc2db1072011-03-01 06:48:31 +0000691 ret = btrfs_lookup_bio_sums(root, inode,
692 comp_bio, sums);
Jeff Mahoney79787ea2012-03-12 16:03:00 +0100693 BUG_ON(ret); /* -ENOMEM */
Chris Masond20f7042008-12-08 16:58:54 -0500694 }
David Sterbaed6078f2014-06-05 01:59:57 +0200695 sums += DIV_ROUND_UP(comp_bio->bi_iter.bi_size,
696 root->sectorsize);
Chris Masond20f7042008-12-08 16:58:54 -0500697
698 ret = btrfs_map_bio(root, READ, comp_bio,
699 mirror_num, 0);
Stefan Behrens61891922012-11-05 18:51:52 +0100700 if (ret)
701 bio_endio(comp_bio, ret);
Chris Masonc8b97812008-10-29 14:49:59 -0400702
703 bio_put(comp_bio);
704
705 comp_bio = compressed_bio_alloc(bdev, cur_disk_byte,
706 GFP_NOFS);
Tsutomu Itohe627ee72012-04-12 16:03:56 -0400707 BUG_ON(!comp_bio);
Chris Mason771ed682008-11-06 22:02:51 -0500708 comp_bio->bi_private = cb;
709 comp_bio->bi_end_io = end_compressed_bio_read;
710
711 bio_add_page(comp_bio, page, PAGE_CACHE_SIZE, 0);
Chris Masonc8b97812008-10-29 14:49:59 -0400712 }
713 cur_disk_byte += PAGE_CACHE_SIZE;
714 }
715 bio_get(comp_bio);
716
David Sterbabfebd8b2014-07-30 00:25:45 +0200717 ret = btrfs_bio_wq_end_io(root->fs_info, comp_bio,
718 BTRFS_WQ_ENDIO_DATA);
Jeff Mahoney79787ea2012-03-12 16:03:00 +0100719 BUG_ON(ret); /* -ENOMEM */
Chris Masonc8b97812008-10-29 14:49:59 -0400720
Tsutomu Itohc2db1072011-03-01 06:48:31 +0000721 if (!(BTRFS_I(inode)->flags & BTRFS_INODE_NODATASUM)) {
722 ret = btrfs_lookup_bio_sums(root, inode, comp_bio, sums);
Jeff Mahoney79787ea2012-03-12 16:03:00 +0100723 BUG_ON(ret); /* -ENOMEM */
Tsutomu Itohc2db1072011-03-01 06:48:31 +0000724 }
Chris Masond20f7042008-12-08 16:58:54 -0500725
726 ret = btrfs_map_bio(root, READ, comp_bio, mirror_num, 0);
Stefan Behrens61891922012-11-05 18:51:52 +0100727 if (ret)
728 bio_endio(comp_bio, ret);
Chris Masonc8b97812008-10-29 14:49:59 -0400729
730 bio_put(comp_bio);
731 return 0;
liubo6b82ce82011-01-26 06:21:39 +0000732
733fail2:
Josef Bacik15e30042012-10-05 13:39:50 -0400734 while (faili >= 0) {
735 __free_page(cb->compressed_pages[faili]);
736 faili--;
737 }
liubo6b82ce82011-01-26 06:21:39 +0000738
739 kfree(cb->compressed_pages);
740fail1:
741 kfree(cb);
742out:
743 free_extent_map(em);
744 return ret;
Chris Masonc8b97812008-10-29 14:49:59 -0400745}
Li Zefan261507a02010-12-17 14:21:50 +0800746
747static struct list_head comp_idle_workspace[BTRFS_COMPRESS_TYPES];
748static spinlock_t comp_workspace_lock[BTRFS_COMPRESS_TYPES];
749static int comp_num_workspace[BTRFS_COMPRESS_TYPES];
750static atomic_t comp_alloc_workspace[BTRFS_COMPRESS_TYPES];
751static wait_queue_head_t comp_workspace_wait[BTRFS_COMPRESS_TYPES];
752
David Sterbae8c9f182015-01-02 18:23:10 +0100753static const struct btrfs_compress_op * const btrfs_compress_op[] = {
Li Zefan261507a02010-12-17 14:21:50 +0800754 &btrfs_zlib_compress,
Li Zefana6fa6fa2010-10-25 15:12:26 +0800755 &btrfs_lzo_compress,
Li Zefan261507a02010-12-17 14:21:50 +0800756};
757
Jeff Mahoney143bede2012-03-01 14:56:26 +0100758void __init btrfs_init_compress(void)
Li Zefan261507a02010-12-17 14:21:50 +0800759{
760 int i;
761
762 for (i = 0; i < BTRFS_COMPRESS_TYPES; i++) {
763 INIT_LIST_HEAD(&comp_idle_workspace[i]);
764 spin_lock_init(&comp_workspace_lock[i]);
765 atomic_set(&comp_alloc_workspace[i], 0);
766 init_waitqueue_head(&comp_workspace_wait[i]);
767 }
Li Zefan261507a02010-12-17 14:21:50 +0800768}
769
770/*
771 * this finds an available workspace or allocates a new one
772 * ERR_PTR is returned if things go bad.
773 */
774static struct list_head *find_workspace(int type)
775{
776 struct list_head *workspace;
777 int cpus = num_online_cpus();
778 int idx = type - 1;
779
780 struct list_head *idle_workspace = &comp_idle_workspace[idx];
781 spinlock_t *workspace_lock = &comp_workspace_lock[idx];
782 atomic_t *alloc_workspace = &comp_alloc_workspace[idx];
783 wait_queue_head_t *workspace_wait = &comp_workspace_wait[idx];
784 int *num_workspace = &comp_num_workspace[idx];
785again:
786 spin_lock(workspace_lock);
787 if (!list_empty(idle_workspace)) {
788 workspace = idle_workspace->next;
789 list_del(workspace);
790 (*num_workspace)--;
791 spin_unlock(workspace_lock);
792 return workspace;
793
794 }
795 if (atomic_read(alloc_workspace) > cpus) {
796 DEFINE_WAIT(wait);
797
798 spin_unlock(workspace_lock);
799 prepare_to_wait(workspace_wait, &wait, TASK_UNINTERRUPTIBLE);
800 if (atomic_read(alloc_workspace) > cpus && !*num_workspace)
801 schedule();
802 finish_wait(workspace_wait, &wait);
803 goto again;
804 }
805 atomic_inc(alloc_workspace);
806 spin_unlock(workspace_lock);
807
808 workspace = btrfs_compress_op[idx]->alloc_workspace();
809 if (IS_ERR(workspace)) {
810 atomic_dec(alloc_workspace);
811 wake_up(workspace_wait);
812 }
813 return workspace;
814}
815
816/*
817 * put a workspace struct back on the list or free it if we have enough
818 * idle ones sitting around
819 */
820static void free_workspace(int type, struct list_head *workspace)
821{
822 int idx = type - 1;
823 struct list_head *idle_workspace = &comp_idle_workspace[idx];
824 spinlock_t *workspace_lock = &comp_workspace_lock[idx];
825 atomic_t *alloc_workspace = &comp_alloc_workspace[idx];
826 wait_queue_head_t *workspace_wait = &comp_workspace_wait[idx];
827 int *num_workspace = &comp_num_workspace[idx];
828
829 spin_lock(workspace_lock);
830 if (*num_workspace < num_online_cpus()) {
Sergey Senozhatskyc39aa702014-06-25 00:00:44 +0900831 list_add(workspace, idle_workspace);
Li Zefan261507a02010-12-17 14:21:50 +0800832 (*num_workspace)++;
833 spin_unlock(workspace_lock);
834 goto wake;
835 }
836 spin_unlock(workspace_lock);
837
838 btrfs_compress_op[idx]->free_workspace(workspace);
839 atomic_dec(alloc_workspace);
840wake:
Josef Bacik66657b32012-08-01 15:36:24 -0400841 smp_mb();
Li Zefan261507a02010-12-17 14:21:50 +0800842 if (waitqueue_active(workspace_wait))
843 wake_up(workspace_wait);
844}
845
846/*
847 * cleanup function for module exit
848 */
849static void free_workspaces(void)
850{
851 struct list_head *workspace;
852 int i;
853
854 for (i = 0; i < BTRFS_COMPRESS_TYPES; i++) {
855 while (!list_empty(&comp_idle_workspace[i])) {
856 workspace = comp_idle_workspace[i].next;
857 list_del(workspace);
858 btrfs_compress_op[i]->free_workspace(workspace);
859 atomic_dec(&comp_alloc_workspace[i]);
860 }
861 }
862}
863
864/*
865 * given an address space and start/len, compress the bytes.
866 *
867 * pages are allocated to hold the compressed result and stored
868 * in 'pages'
869 *
870 * out_pages is used to return the number of pages allocated. There
871 * may be pages allocated even if we return an error
872 *
873 * total_in is used to return the number of bytes actually read. It
874 * may be smaller then len if we had to exit early because we
875 * ran out of room in the pages array or because we cross the
876 * max_out threshold.
877 *
878 * total_out is used to return the total number of compressed bytes
879 *
880 * max_out tells us the max number of bytes that we're allowed to
881 * stuff into pages
882 */
883int btrfs_compress_pages(int type, struct address_space *mapping,
884 u64 start, unsigned long len,
885 struct page **pages,
886 unsigned long nr_dest_pages,
887 unsigned long *out_pages,
888 unsigned long *total_in,
889 unsigned long *total_out,
890 unsigned long max_out)
891{
892 struct list_head *workspace;
893 int ret;
894
895 workspace = find_workspace(type);
896 if (IS_ERR(workspace))
Zach Brown774bcb32014-05-09 17:15:09 -0400897 return PTR_ERR(workspace);
Li Zefan261507a02010-12-17 14:21:50 +0800898
899 ret = btrfs_compress_op[type-1]->compress_pages(workspace, mapping,
900 start, len, pages,
901 nr_dest_pages, out_pages,
902 total_in, total_out,
903 max_out);
904 free_workspace(type, workspace);
905 return ret;
906}
907
908/*
909 * pages_in is an array of pages with compressed data.
910 *
911 * disk_start is the starting logical offset of this array in the file
912 *
913 * bvec is a bio_vec of pages from the file that we want to decompress into
914 *
915 * vcnt is the count of pages in the biovec
916 *
917 * srclen is the number of bytes in pages_in
918 *
919 * The basic idea is that we have a bio that was created by readpages.
920 * The pages in the bio are for the uncompressed data, and they may not
921 * be contiguous. They all correspond to the range of bytes covered by
922 * the compressed extent.
923 */
Eric Sandeen48a3b632013-04-25 20:41:01 +0000924static int btrfs_decompress_biovec(int type, struct page **pages_in,
925 u64 disk_start, struct bio_vec *bvec,
926 int vcnt, size_t srclen)
Li Zefan261507a02010-12-17 14:21:50 +0800927{
928 struct list_head *workspace;
929 int ret;
930
931 workspace = find_workspace(type);
932 if (IS_ERR(workspace))
Zach Brown774bcb32014-05-09 17:15:09 -0400933 return PTR_ERR(workspace);
Li Zefan261507a02010-12-17 14:21:50 +0800934
935 ret = btrfs_compress_op[type-1]->decompress_biovec(workspace, pages_in,
936 disk_start,
937 bvec, vcnt, srclen);
938 free_workspace(type, workspace);
939 return ret;
940}
941
942/*
943 * a less complex decompression routine. Our compressed data fits in a
944 * single page, and we want to read a single page out of it.
945 * start_byte tells us the offset into the compressed data we're interested in
946 */
947int btrfs_decompress(int type, unsigned char *data_in, struct page *dest_page,
948 unsigned long start_byte, size_t srclen, size_t destlen)
949{
950 struct list_head *workspace;
951 int ret;
952
953 workspace = find_workspace(type);
954 if (IS_ERR(workspace))
Zach Brown774bcb32014-05-09 17:15:09 -0400955 return PTR_ERR(workspace);
Li Zefan261507a02010-12-17 14:21:50 +0800956
957 ret = btrfs_compress_op[type-1]->decompress(workspace, data_in,
958 dest_page, start_byte,
959 srclen, destlen);
960
961 free_workspace(type, workspace);
962 return ret;
963}
964
Alexey Charkov8e4eef7a2011-02-02 21:15:35 +0000965void btrfs_exit_compress(void)
Li Zefan261507a02010-12-17 14:21:50 +0800966{
967 free_workspaces();
968}
Li Zefan3a39c182010-11-08 15:22:19 +0800969
970/*
971 * Copy uncompressed data from working buffer to pages.
972 *
973 * buf_start is the byte offset we're of the start of our workspace buffer.
974 *
975 * total_out is the last byte of the buffer
976 */
977int btrfs_decompress_buf2page(char *buf, unsigned long buf_start,
978 unsigned long total_out, u64 disk_start,
979 struct bio_vec *bvec, int vcnt,
David Sterba306e16c2011-04-19 14:29:38 +0200980 unsigned long *pg_index,
Li Zefan3a39c182010-11-08 15:22:19 +0800981 unsigned long *pg_offset)
982{
983 unsigned long buf_offset;
984 unsigned long current_buf_start;
985 unsigned long start_byte;
986 unsigned long working_bytes = total_out - buf_start;
987 unsigned long bytes;
988 char *kaddr;
David Sterba306e16c2011-04-19 14:29:38 +0200989 struct page *page_out = bvec[*pg_index].bv_page;
Li Zefan3a39c182010-11-08 15:22:19 +0800990
991 /*
992 * start byte is the first byte of the page we're currently
993 * copying into relative to the start of the compressed data.
994 */
995 start_byte = page_offset(page_out) - disk_start;
996
997 /* we haven't yet hit data corresponding to this page */
998 if (total_out <= start_byte)
999 return 1;
1000
1001 /*
1002 * the start of the data we care about is offset into
1003 * the middle of our working buffer
1004 */
1005 if (total_out > start_byte && buf_start < start_byte) {
1006 buf_offset = start_byte - buf_start;
1007 working_bytes -= buf_offset;
1008 } else {
1009 buf_offset = 0;
1010 }
1011 current_buf_start = buf_start;
1012
1013 /* copy bytes from the working buffer into the pages */
1014 while (working_bytes > 0) {
1015 bytes = min(PAGE_CACHE_SIZE - *pg_offset,
1016 PAGE_CACHE_SIZE - buf_offset);
1017 bytes = min(bytes, working_bytes);
Cong Wang7ac687d2011-11-25 23:14:28 +08001018 kaddr = kmap_atomic(page_out);
Li Zefan3a39c182010-11-08 15:22:19 +08001019 memcpy(kaddr + *pg_offset, buf + buf_offset, bytes);
Cong Wang7ac687d2011-11-25 23:14:28 +08001020 kunmap_atomic(kaddr);
Li Zefan3a39c182010-11-08 15:22:19 +08001021 flush_dcache_page(page_out);
1022
1023 *pg_offset += bytes;
1024 buf_offset += bytes;
1025 working_bytes -= bytes;
1026 current_buf_start += bytes;
1027
1028 /* check if we need to pick another page */
1029 if (*pg_offset == PAGE_CACHE_SIZE) {
David Sterba306e16c2011-04-19 14:29:38 +02001030 (*pg_index)++;
1031 if (*pg_index >= vcnt)
Li Zefan3a39c182010-11-08 15:22:19 +08001032 return 0;
1033
David Sterba306e16c2011-04-19 14:29:38 +02001034 page_out = bvec[*pg_index].bv_page;
Li Zefan3a39c182010-11-08 15:22:19 +08001035 *pg_offset = 0;
1036 start_byte = page_offset(page_out) - disk_start;
1037
1038 /*
1039 * make sure our new page is covered by this
1040 * working buffer
1041 */
1042 if (total_out <= start_byte)
1043 return 1;
1044
1045 /*
1046 * the next page in the biovec might not be adjacent
1047 * to the last page, but it might still be found
1048 * inside this working buffer. bump our offset pointer
1049 */
1050 if (total_out > start_byte &&
1051 current_buf_start < start_byte) {
1052 buf_offset = start_byte - buf_start;
1053 working_bytes = total_out - start_byte;
1054 current_buf_start = buf_start + buf_offset;
1055 }
1056 }
1057 }
1058
1059 return 1;
1060}
Chris Mason2f19cad2014-11-30 08:56:33 -05001061
1062/*
1063 * When uncompressing data, we need to make sure and zero any parts of
1064 * the biovec that were not filled in by the decompression code. pg_index
1065 * and pg_offset indicate the last page and the last offset of that page
1066 * that have been filled in. This will zero everything remaining in the
1067 * biovec.
1068 */
1069void btrfs_clear_biovec_end(struct bio_vec *bvec, int vcnt,
1070 unsigned long pg_index,
1071 unsigned long pg_offset)
1072{
1073 while (pg_index < vcnt) {
1074 struct page *page = bvec[pg_index].bv_page;
1075 unsigned long off = bvec[pg_index].bv_offset;
1076 unsigned long len = bvec[pg_index].bv_len;
1077
1078 if (pg_offset < off)
1079 pg_offset = off;
1080 if (pg_offset < off + len) {
1081 unsigned long bytes = off + len - pg_offset;
1082 char *kaddr;
1083
1084 kaddr = kmap_atomic(page);
1085 memset(kaddr + pg_offset, 0, bytes);
1086 kunmap_atomic(kaddr);
1087 }
1088 pg_index++;
1089 pg_offset = 0;
1090 }
1091}