blob: 583d98bd065ed83ca979a2786b59ae4342380c47 [file] [log] [blame]
Chris Masond1310b22008-01-24 16:13:08 -05001#include <linux/bitops.h>
2#include <linux/slab.h>
3#include <linux/bio.h>
4#include <linux/mm.h>
Chris Masond1310b22008-01-24 16:13:08 -05005#include <linux/pagemap.h>
6#include <linux/page-flags.h>
Chris Masond1310b22008-01-24 16:13:08 -05007#include <linux/spinlock.h>
8#include <linux/blkdev.h>
9#include <linux/swap.h>
Chris Masond1310b22008-01-24 16:13:08 -050010#include <linux/writeback.h>
11#include <linux/pagevec.h>
Linus Torvalds268bb0c2011-05-20 12:50:29 -070012#include <linux/prefetch.h>
Dan Magenheimer90a887c2011-05-26 10:01:56 -060013#include <linux/cleancache.h>
Chris Masond1310b22008-01-24 16:13:08 -050014#include "extent_io.h"
15#include "extent_map.h"
David Woodhouse2db04962008-08-07 11:19:43 -040016#include "compat.h"
David Woodhouse902b22f2008-08-20 08:51:49 -040017#include "ctree.h"
18#include "btrfs_inode.h"
Jan Schmidt4a54c8c2011-07-22 15:41:52 +020019#include "volumes.h"
Stefan Behrens21adbd52011-11-09 13:44:05 +010020#include "check-integrity.h"
Josef Bacik0b32f4b2012-03-13 09:38:00 -040021#include "locking.h"
Josef Bacik606686e2012-06-04 14:03:51 -040022#include "rcu-string.h"
Chris Masond1310b22008-01-24 16:13:08 -050023
Chris Masond1310b22008-01-24 16:13:08 -050024static struct kmem_cache *extent_state_cache;
25static struct kmem_cache *extent_buffer_cache;
Chris Mason9be33952013-05-17 18:30:14 -040026static struct bio_set *btrfs_bioset;
Chris Masond1310b22008-01-24 16:13:08 -050027
Eric Sandeen6d49ba12013-04-22 16:12:31 +000028#ifdef CONFIG_BTRFS_DEBUG
Chris Masond1310b22008-01-24 16:13:08 -050029static LIST_HEAD(buffers);
30static LIST_HEAD(states);
Chris Mason4bef0842008-09-08 11:18:08 -040031
Chris Masond3977122009-01-05 21:25:51 -050032static DEFINE_SPINLOCK(leak_lock);
Eric Sandeen6d49ba12013-04-22 16:12:31 +000033
34static inline
35void btrfs_leak_debug_add(struct list_head *new, struct list_head *head)
36{
37 unsigned long flags;
38
39 spin_lock_irqsave(&leak_lock, flags);
40 list_add(new, head);
41 spin_unlock_irqrestore(&leak_lock, flags);
42}
43
44static inline
45void btrfs_leak_debug_del(struct list_head *entry)
46{
47 unsigned long flags;
48
49 spin_lock_irqsave(&leak_lock, flags);
50 list_del(entry);
51 spin_unlock_irqrestore(&leak_lock, flags);
52}
53
54static inline
55void btrfs_leak_debug_check(void)
56{
57 struct extent_state *state;
58 struct extent_buffer *eb;
59
60 while (!list_empty(&states)) {
61 state = list_entry(states.next, struct extent_state, leak_list);
62 printk(KERN_ERR "btrfs state leak: start %llu end %llu "
63 "state %lu in tree %p refs %d\n",
64 (unsigned long long)state->start,
65 (unsigned long long)state->end,
66 state->state, state->tree, atomic_read(&state->refs));
67 list_del(&state->leak_list);
68 kmem_cache_free(extent_state_cache, state);
69 }
70
71 while (!list_empty(&buffers)) {
72 eb = list_entry(buffers.next, struct extent_buffer, leak_list);
73 printk(KERN_ERR "btrfs buffer leak start %llu len %lu "
74 "refs %d\n", (unsigned long long)eb->start,
75 eb->len, atomic_read(&eb->refs));
76 list_del(&eb->leak_list);
77 kmem_cache_free(extent_buffer_cache, eb);
78 }
79}
David Sterba8d599ae2013-04-30 15:22:23 +000080
81#define btrfs_debug_check_extent_io_range(inode, start, end) \
82 __btrfs_debug_check_extent_io_range(__func__, (inode), (start), (end))
83static inline void __btrfs_debug_check_extent_io_range(const char *caller,
84 struct inode *inode, u64 start, u64 end)
85{
86 u64 isize = i_size_read(inode);
87
88 if (end >= PAGE_SIZE && (end % 2) == 0 && end != isize - 1) {
89 printk_ratelimited(KERN_DEBUG
90 "btrfs: %s: ino %llu isize %llu odd range [%llu,%llu]\n",
91 caller,
92 (unsigned long long)btrfs_ino(inode),
93 (unsigned long long)isize,
94 (unsigned long long)start,
95 (unsigned long long)end);
96 }
97}
Eric Sandeen6d49ba12013-04-22 16:12:31 +000098#else
99#define btrfs_leak_debug_add(new, head) do {} while (0)
100#define btrfs_leak_debug_del(entry) do {} while (0)
101#define btrfs_leak_debug_check() do {} while (0)
David Sterba8d599ae2013-04-30 15:22:23 +0000102#define btrfs_debug_check_extent_io_range(c, s, e) do {} while (0)
Chris Mason4bef0842008-09-08 11:18:08 -0400103#endif
Chris Masond1310b22008-01-24 16:13:08 -0500104
Chris Masond1310b22008-01-24 16:13:08 -0500105#define BUFFER_LRU_MAX 64
106
107struct tree_entry {
108 u64 start;
109 u64 end;
Chris Masond1310b22008-01-24 16:13:08 -0500110 struct rb_node rb_node;
111};
112
113struct extent_page_data {
114 struct bio *bio;
115 struct extent_io_tree *tree;
116 get_extent_t *get_extent;
Josef Bacikde0022b2012-09-25 14:25:58 -0400117 unsigned long bio_flags;
Chris Mason771ed682008-11-06 22:02:51 -0500118
119 /* tells writepage not to lock the state bits for this range
120 * it still does the unlocking
121 */
Chris Masonffbd5172009-04-20 15:50:09 -0400122 unsigned int extent_locked:1;
123
124 /* tells the submit_bio code to use a WRITE_SYNC */
125 unsigned int sync_io:1;
Chris Masond1310b22008-01-24 16:13:08 -0500126};
127
Josef Bacik0b32f4b2012-03-13 09:38:00 -0400128static noinline void flush_write_bio(void *data);
Jeff Mahoneyc2d904e2011-10-03 23:22:32 -0400129static inline struct btrfs_fs_info *
130tree_fs_info(struct extent_io_tree *tree)
131{
132 return btrfs_sb(tree->mapping->host->i_sb);
133}
Josef Bacik0b32f4b2012-03-13 09:38:00 -0400134
Chris Masond1310b22008-01-24 16:13:08 -0500135int __init extent_io_init(void)
136{
David Sterba837e1972012-09-07 03:00:48 -0600137 extent_state_cache = kmem_cache_create("btrfs_extent_state",
Christoph Hellwig9601e3f2009-04-13 15:33:09 +0200138 sizeof(struct extent_state), 0,
139 SLAB_RECLAIM_ACCOUNT | SLAB_MEM_SPREAD, NULL);
Chris Masond1310b22008-01-24 16:13:08 -0500140 if (!extent_state_cache)
141 return -ENOMEM;
142
David Sterba837e1972012-09-07 03:00:48 -0600143 extent_buffer_cache = kmem_cache_create("btrfs_extent_buffer",
Christoph Hellwig9601e3f2009-04-13 15:33:09 +0200144 sizeof(struct extent_buffer), 0,
145 SLAB_RECLAIM_ACCOUNT | SLAB_MEM_SPREAD, NULL);
Chris Masond1310b22008-01-24 16:13:08 -0500146 if (!extent_buffer_cache)
147 goto free_state_cache;
Chris Mason9be33952013-05-17 18:30:14 -0400148
149 btrfs_bioset = bioset_create(BIO_POOL_SIZE,
150 offsetof(struct btrfs_io_bio, bio));
151 if (!btrfs_bioset)
152 goto free_buffer_cache;
Chris Masond1310b22008-01-24 16:13:08 -0500153 return 0;
154
Chris Mason9be33952013-05-17 18:30:14 -0400155free_buffer_cache:
156 kmem_cache_destroy(extent_buffer_cache);
157 extent_buffer_cache = NULL;
158
Chris Masond1310b22008-01-24 16:13:08 -0500159free_state_cache:
160 kmem_cache_destroy(extent_state_cache);
Chris Mason9be33952013-05-17 18:30:14 -0400161 extent_state_cache = NULL;
Chris Masond1310b22008-01-24 16:13:08 -0500162 return -ENOMEM;
163}
164
165void extent_io_exit(void)
166{
Eric Sandeen6d49ba12013-04-22 16:12:31 +0000167 btrfs_leak_debug_check();
Kirill A. Shutemov8c0a8532012-09-26 11:33:07 +1000168
169 /*
170 * Make sure all delayed rcu free are flushed before we
171 * destroy caches.
172 */
173 rcu_barrier();
Chris Masond1310b22008-01-24 16:13:08 -0500174 if (extent_state_cache)
175 kmem_cache_destroy(extent_state_cache);
176 if (extent_buffer_cache)
177 kmem_cache_destroy(extent_buffer_cache);
Chris Mason9be33952013-05-17 18:30:14 -0400178 if (btrfs_bioset)
179 bioset_free(btrfs_bioset);
Chris Masond1310b22008-01-24 16:13:08 -0500180}
181
182void extent_io_tree_init(struct extent_io_tree *tree,
David Sterbaf993c882011-04-20 23:35:57 +0200183 struct address_space *mapping)
Chris Masond1310b22008-01-24 16:13:08 -0500184{
Eric Paris6bef4d32010-02-23 19:43:04 +0000185 tree->state = RB_ROOT;
Miao Xie19fe0a82010-10-26 20:57:29 -0400186 INIT_RADIX_TREE(&tree->buffer, GFP_ATOMIC);
Chris Masond1310b22008-01-24 16:13:08 -0500187 tree->ops = NULL;
188 tree->dirty_bytes = 0;
Chris Mason70dec802008-01-29 09:59:12 -0500189 spin_lock_init(&tree->lock);
Chris Mason6af118ce2008-07-22 11:18:07 -0400190 spin_lock_init(&tree->buffer_lock);
Chris Masond1310b22008-01-24 16:13:08 -0500191 tree->mapping = mapping;
Chris Masond1310b22008-01-24 16:13:08 -0500192}
Chris Masond1310b22008-01-24 16:13:08 -0500193
Christoph Hellwigb2950862008-12-02 09:54:17 -0500194static struct extent_state *alloc_extent_state(gfp_t mask)
Chris Masond1310b22008-01-24 16:13:08 -0500195{
196 struct extent_state *state;
Chris Masond1310b22008-01-24 16:13:08 -0500197
198 state = kmem_cache_alloc(extent_state_cache, mask);
Peter2b114d12008-04-01 11:21:40 -0400199 if (!state)
Chris Masond1310b22008-01-24 16:13:08 -0500200 return state;
201 state->state = 0;
Chris Masond1310b22008-01-24 16:13:08 -0500202 state->private = 0;
Chris Mason70dec802008-01-29 09:59:12 -0500203 state->tree = NULL;
Eric Sandeen6d49ba12013-04-22 16:12:31 +0000204 btrfs_leak_debug_add(&state->leak_list, &states);
Chris Masond1310b22008-01-24 16:13:08 -0500205 atomic_set(&state->refs, 1);
206 init_waitqueue_head(&state->wq);
Jeff Mahoney143bede2012-03-01 14:56:26 +0100207 trace_alloc_extent_state(state, mask, _RET_IP_);
Chris Masond1310b22008-01-24 16:13:08 -0500208 return state;
209}
Chris Masond1310b22008-01-24 16:13:08 -0500210
Chris Mason4845e442010-05-25 20:56:50 -0400211void free_extent_state(struct extent_state *state)
Chris Masond1310b22008-01-24 16:13:08 -0500212{
Chris Masond1310b22008-01-24 16:13:08 -0500213 if (!state)
214 return;
215 if (atomic_dec_and_test(&state->refs)) {
Chris Mason70dec802008-01-29 09:59:12 -0500216 WARN_ON(state->tree);
Eric Sandeen6d49ba12013-04-22 16:12:31 +0000217 btrfs_leak_debug_del(&state->leak_list);
Jeff Mahoney143bede2012-03-01 14:56:26 +0100218 trace_free_extent_state(state, _RET_IP_);
Chris Masond1310b22008-01-24 16:13:08 -0500219 kmem_cache_free(extent_state_cache, state);
220 }
221}
Chris Masond1310b22008-01-24 16:13:08 -0500222
223static struct rb_node *tree_insert(struct rb_root *root, u64 offset,
224 struct rb_node *node)
225{
Chris Masond3977122009-01-05 21:25:51 -0500226 struct rb_node **p = &root->rb_node;
227 struct rb_node *parent = NULL;
Chris Masond1310b22008-01-24 16:13:08 -0500228 struct tree_entry *entry;
229
Chris Masond3977122009-01-05 21:25:51 -0500230 while (*p) {
Chris Masond1310b22008-01-24 16:13:08 -0500231 parent = *p;
232 entry = rb_entry(parent, struct tree_entry, rb_node);
233
234 if (offset < entry->start)
235 p = &(*p)->rb_left;
236 else if (offset > entry->end)
237 p = &(*p)->rb_right;
238 else
239 return parent;
240 }
241
Chris Masond1310b22008-01-24 16:13:08 -0500242 rb_link_node(node, parent, p);
243 rb_insert_color(node, root);
244 return NULL;
245}
246
Chris Mason80ea96b2008-02-01 14:51:59 -0500247static struct rb_node *__etree_search(struct extent_io_tree *tree, u64 offset,
Chris Masond1310b22008-01-24 16:13:08 -0500248 struct rb_node **prev_ret,
249 struct rb_node **next_ret)
250{
Chris Mason80ea96b2008-02-01 14:51:59 -0500251 struct rb_root *root = &tree->state;
Chris Masond3977122009-01-05 21:25:51 -0500252 struct rb_node *n = root->rb_node;
Chris Masond1310b22008-01-24 16:13:08 -0500253 struct rb_node *prev = NULL;
254 struct rb_node *orig_prev = NULL;
255 struct tree_entry *entry;
256 struct tree_entry *prev_entry = NULL;
257
Chris Masond3977122009-01-05 21:25:51 -0500258 while (n) {
Chris Masond1310b22008-01-24 16:13:08 -0500259 entry = rb_entry(n, struct tree_entry, rb_node);
260 prev = n;
261 prev_entry = entry;
262
263 if (offset < entry->start)
264 n = n->rb_left;
265 else if (offset > entry->end)
266 n = n->rb_right;
Chris Masond3977122009-01-05 21:25:51 -0500267 else
Chris Masond1310b22008-01-24 16:13:08 -0500268 return n;
269 }
270
271 if (prev_ret) {
272 orig_prev = prev;
Chris Masond3977122009-01-05 21:25:51 -0500273 while (prev && offset > prev_entry->end) {
Chris Masond1310b22008-01-24 16:13:08 -0500274 prev = rb_next(prev);
275 prev_entry = rb_entry(prev, struct tree_entry, rb_node);
276 }
277 *prev_ret = prev;
278 prev = orig_prev;
279 }
280
281 if (next_ret) {
282 prev_entry = rb_entry(prev, struct tree_entry, rb_node);
Chris Masond3977122009-01-05 21:25:51 -0500283 while (prev && offset < prev_entry->start) {
Chris Masond1310b22008-01-24 16:13:08 -0500284 prev = rb_prev(prev);
285 prev_entry = rb_entry(prev, struct tree_entry, rb_node);
286 }
287 *next_ret = prev;
288 }
289 return NULL;
290}
291
Chris Mason80ea96b2008-02-01 14:51:59 -0500292static inline struct rb_node *tree_search(struct extent_io_tree *tree,
293 u64 offset)
Chris Masond1310b22008-01-24 16:13:08 -0500294{
Chris Mason70dec802008-01-29 09:59:12 -0500295 struct rb_node *prev = NULL;
Chris Masond1310b22008-01-24 16:13:08 -0500296 struct rb_node *ret;
Chris Mason70dec802008-01-29 09:59:12 -0500297
Chris Mason80ea96b2008-02-01 14:51:59 -0500298 ret = __etree_search(tree, offset, &prev, NULL);
Chris Masond3977122009-01-05 21:25:51 -0500299 if (!ret)
Chris Masond1310b22008-01-24 16:13:08 -0500300 return prev;
301 return ret;
302}
303
Josef Bacik9ed74f22009-09-11 16:12:44 -0400304static void merge_cb(struct extent_io_tree *tree, struct extent_state *new,
305 struct extent_state *other)
306{
307 if (tree->ops && tree->ops->merge_extent_hook)
308 tree->ops->merge_extent_hook(tree->mapping->host, new,
309 other);
310}
311
Chris Masond1310b22008-01-24 16:13:08 -0500312/*
313 * utility function to look for merge candidates inside a given range.
314 * Any extents with matching state are merged together into a single
315 * extent in the tree. Extents with EXTENT_IO in their state field
316 * are not merged because the end_io handlers need to be able to do
317 * operations on them without sleeping (or doing allocations/splits).
318 *
319 * This should be called with the tree lock held.
320 */
Jeff Mahoney1bf85042011-07-21 16:56:09 +0000321static void merge_state(struct extent_io_tree *tree,
322 struct extent_state *state)
Chris Masond1310b22008-01-24 16:13:08 -0500323{
324 struct extent_state *other;
325 struct rb_node *other_node;
326
Zheng Yan5b21f2e2008-09-26 10:05:38 -0400327 if (state->state & (EXTENT_IOBITS | EXTENT_BOUNDARY))
Jeff Mahoney1bf85042011-07-21 16:56:09 +0000328 return;
Chris Masond1310b22008-01-24 16:13:08 -0500329
330 other_node = rb_prev(&state->rb_node);
331 if (other_node) {
332 other = rb_entry(other_node, struct extent_state, rb_node);
333 if (other->end == state->start - 1 &&
334 other->state == state->state) {
Josef Bacik9ed74f22009-09-11 16:12:44 -0400335 merge_cb(tree, state, other);
Chris Masond1310b22008-01-24 16:13:08 -0500336 state->start = other->start;
Chris Mason70dec802008-01-29 09:59:12 -0500337 other->tree = NULL;
Chris Masond1310b22008-01-24 16:13:08 -0500338 rb_erase(&other->rb_node, &tree->state);
339 free_extent_state(other);
340 }
341 }
342 other_node = rb_next(&state->rb_node);
343 if (other_node) {
344 other = rb_entry(other_node, struct extent_state, rb_node);
345 if (other->start == state->end + 1 &&
346 other->state == state->state) {
Josef Bacik9ed74f22009-09-11 16:12:44 -0400347 merge_cb(tree, state, other);
Josef Bacikdf98b6e2011-06-20 14:53:48 -0400348 state->end = other->end;
349 other->tree = NULL;
350 rb_erase(&other->rb_node, &tree->state);
351 free_extent_state(other);
Chris Masond1310b22008-01-24 16:13:08 -0500352 }
353 }
Chris Masond1310b22008-01-24 16:13:08 -0500354}
355
Jeff Mahoney1bf85042011-07-21 16:56:09 +0000356static void set_state_cb(struct extent_io_tree *tree,
David Sterba41074882013-04-29 13:38:46 +0000357 struct extent_state *state, unsigned long *bits)
Chris Mason291d6732008-01-29 15:55:23 -0500358{
Jeff Mahoney1bf85042011-07-21 16:56:09 +0000359 if (tree->ops && tree->ops->set_bit_hook)
360 tree->ops->set_bit_hook(tree->mapping->host, state, bits);
Chris Mason291d6732008-01-29 15:55:23 -0500361}
362
363static void clear_state_cb(struct extent_io_tree *tree,
David Sterba41074882013-04-29 13:38:46 +0000364 struct extent_state *state, unsigned long *bits)
Chris Mason291d6732008-01-29 15:55:23 -0500365{
Josef Bacik9ed74f22009-09-11 16:12:44 -0400366 if (tree->ops && tree->ops->clear_bit_hook)
367 tree->ops->clear_bit_hook(tree->mapping->host, state, bits);
Chris Mason291d6732008-01-29 15:55:23 -0500368}
369
Xiao Guangrong3150b692011-07-14 03:19:08 +0000370static void set_state_bits(struct extent_io_tree *tree,
David Sterba41074882013-04-29 13:38:46 +0000371 struct extent_state *state, unsigned long *bits);
Xiao Guangrong3150b692011-07-14 03:19:08 +0000372
Chris Masond1310b22008-01-24 16:13:08 -0500373/*
374 * insert an extent_state struct into the tree. 'bits' are set on the
375 * struct before it is inserted.
376 *
377 * This may return -EEXIST if the extent is already there, in which case the
378 * state struct is freed.
379 *
380 * The tree lock is not taken internally. This is a utility function and
381 * probably isn't what you want to call (see set/clear_extent_bit).
382 */
383static int insert_state(struct extent_io_tree *tree,
384 struct extent_state *state, u64 start, u64 end,
David Sterba41074882013-04-29 13:38:46 +0000385 unsigned long *bits)
Chris Masond1310b22008-01-24 16:13:08 -0500386{
387 struct rb_node *node;
388
Julia Lawall31b1a2b2012-11-03 10:58:34 +0000389 if (end < start)
390 WARN(1, KERN_ERR "btrfs end < start %llu %llu\n",
Chris Masond3977122009-01-05 21:25:51 -0500391 (unsigned long long)end,
392 (unsigned long long)start);
Chris Masond1310b22008-01-24 16:13:08 -0500393 state->start = start;
394 state->end = end;
Josef Bacik9ed74f22009-09-11 16:12:44 -0400395
Xiao Guangrong3150b692011-07-14 03:19:08 +0000396 set_state_bits(tree, state, bits);
397
Chris Masond1310b22008-01-24 16:13:08 -0500398 node = tree_insert(&tree->state, end, &state->rb_node);
399 if (node) {
400 struct extent_state *found;
401 found = rb_entry(node, struct extent_state, rb_node);
Chris Masond3977122009-01-05 21:25:51 -0500402 printk(KERN_ERR "btrfs found node %llu %llu on insert of "
403 "%llu %llu\n", (unsigned long long)found->start,
404 (unsigned long long)found->end,
405 (unsigned long long)start, (unsigned long long)end);
Chris Masond1310b22008-01-24 16:13:08 -0500406 return -EEXIST;
407 }
Chris Mason70dec802008-01-29 09:59:12 -0500408 state->tree = tree;
Chris Masond1310b22008-01-24 16:13:08 -0500409 merge_state(tree, state);
410 return 0;
411}
412
Jeff Mahoney1bf85042011-07-21 16:56:09 +0000413static void split_cb(struct extent_io_tree *tree, struct extent_state *orig,
Josef Bacik9ed74f22009-09-11 16:12:44 -0400414 u64 split)
415{
416 if (tree->ops && tree->ops->split_extent_hook)
Jeff Mahoney1bf85042011-07-21 16:56:09 +0000417 tree->ops->split_extent_hook(tree->mapping->host, orig, split);
Josef Bacik9ed74f22009-09-11 16:12:44 -0400418}
419
Chris Masond1310b22008-01-24 16:13:08 -0500420/*
421 * split a given extent state struct in two, inserting the preallocated
422 * struct 'prealloc' as the newly created second half. 'split' indicates an
423 * offset inside 'orig' where it should be split.
424 *
425 * Before calling,
426 * the tree has 'orig' at [orig->start, orig->end]. After calling, there
427 * are two extent state structs in the tree:
428 * prealloc: [orig->start, split - 1]
429 * orig: [ split, orig->end ]
430 *
431 * The tree locks are not taken by this function. They need to be held
432 * by the caller.
433 */
434static int split_state(struct extent_io_tree *tree, struct extent_state *orig,
435 struct extent_state *prealloc, u64 split)
436{
437 struct rb_node *node;
Josef Bacik9ed74f22009-09-11 16:12:44 -0400438
439 split_cb(tree, orig, split);
440
Chris Masond1310b22008-01-24 16:13:08 -0500441 prealloc->start = orig->start;
442 prealloc->end = split - 1;
443 prealloc->state = orig->state;
444 orig->start = split;
445
446 node = tree_insert(&tree->state, prealloc->end, &prealloc->rb_node);
447 if (node) {
Chris Masond1310b22008-01-24 16:13:08 -0500448 free_extent_state(prealloc);
449 return -EEXIST;
450 }
Chris Mason70dec802008-01-29 09:59:12 -0500451 prealloc->tree = tree;
Chris Masond1310b22008-01-24 16:13:08 -0500452 return 0;
453}
454
Li Zefancdc6a392012-03-12 16:39:48 +0800455static struct extent_state *next_state(struct extent_state *state)
456{
457 struct rb_node *next = rb_next(&state->rb_node);
458 if (next)
459 return rb_entry(next, struct extent_state, rb_node);
460 else
461 return NULL;
462}
463
Chris Masond1310b22008-01-24 16:13:08 -0500464/*
465 * utility function to clear some bits in an extent state struct.
Wang Sheng-Hui1b303fc2012-04-06 14:35:18 +0800466 * it will optionally wake up any one waiting on this state (wake == 1).
Chris Masond1310b22008-01-24 16:13:08 -0500467 *
468 * If no bits are set on the state struct after clearing things, the
469 * struct is freed and removed from the tree
470 */
Li Zefancdc6a392012-03-12 16:39:48 +0800471static struct extent_state *clear_state_bit(struct extent_io_tree *tree,
472 struct extent_state *state,
David Sterba41074882013-04-29 13:38:46 +0000473 unsigned long *bits, int wake)
Chris Masond1310b22008-01-24 16:13:08 -0500474{
Li Zefancdc6a392012-03-12 16:39:48 +0800475 struct extent_state *next;
David Sterba41074882013-04-29 13:38:46 +0000476 unsigned long bits_to_clear = *bits & ~EXTENT_CTLBITS;
Chris Masond1310b22008-01-24 16:13:08 -0500477
Yan, Zheng0ca1f7c2010-05-16 10:48:47 -0400478 if ((bits_to_clear & EXTENT_DIRTY) && (state->state & EXTENT_DIRTY)) {
Chris Masond1310b22008-01-24 16:13:08 -0500479 u64 range = state->end - state->start + 1;
480 WARN_ON(range > tree->dirty_bytes);
481 tree->dirty_bytes -= range;
482 }
Chris Mason291d6732008-01-29 15:55:23 -0500483 clear_state_cb(tree, state, bits);
Josef Bacik32c00af2009-10-08 13:34:05 -0400484 state->state &= ~bits_to_clear;
Chris Masond1310b22008-01-24 16:13:08 -0500485 if (wake)
486 wake_up(&state->wq);
Yan, Zheng0ca1f7c2010-05-16 10:48:47 -0400487 if (state->state == 0) {
Li Zefancdc6a392012-03-12 16:39:48 +0800488 next = next_state(state);
Chris Mason70dec802008-01-29 09:59:12 -0500489 if (state->tree) {
Chris Masond1310b22008-01-24 16:13:08 -0500490 rb_erase(&state->rb_node, &tree->state);
Chris Mason70dec802008-01-29 09:59:12 -0500491 state->tree = NULL;
Chris Masond1310b22008-01-24 16:13:08 -0500492 free_extent_state(state);
493 } else {
494 WARN_ON(1);
495 }
496 } else {
497 merge_state(tree, state);
Li Zefancdc6a392012-03-12 16:39:48 +0800498 next = next_state(state);
Chris Masond1310b22008-01-24 16:13:08 -0500499 }
Li Zefancdc6a392012-03-12 16:39:48 +0800500 return next;
Chris Masond1310b22008-01-24 16:13:08 -0500501}
502
Xiao Guangrong82337672011-04-20 06:44:57 +0000503static struct extent_state *
504alloc_extent_state_atomic(struct extent_state *prealloc)
505{
506 if (!prealloc)
507 prealloc = alloc_extent_state(GFP_ATOMIC);
508
509 return prealloc;
510}
511
Eric Sandeen48a3b632013-04-25 20:41:01 +0000512static void extent_io_tree_panic(struct extent_io_tree *tree, int err)
Jeff Mahoneyc2d904e2011-10-03 23:22:32 -0400513{
514 btrfs_panic(tree_fs_info(tree), err, "Locking error: "
515 "Extent tree was modified by another "
516 "thread while locked.");
517}
518
Chris Masond1310b22008-01-24 16:13:08 -0500519/*
520 * clear some bits on a range in the tree. This may require splitting
521 * or inserting elements in the tree, so the gfp mask is used to
522 * indicate which allocations or sleeping are allowed.
523 *
524 * pass 'wake' == 1 to kick any sleepers, and 'delete' == 1 to remove
525 * the given range from the tree regardless of state (ie for truncate).
526 *
527 * the range [start, end] is inclusive.
528 *
Jeff Mahoney6763af82012-03-01 14:56:29 +0100529 * This takes the tree lock, and returns 0 on success and < 0 on error.
Chris Masond1310b22008-01-24 16:13:08 -0500530 */
531int clear_extent_bit(struct extent_io_tree *tree, u64 start, u64 end,
David Sterba41074882013-04-29 13:38:46 +0000532 unsigned long bits, int wake, int delete,
Chris Mason2c64c532009-09-02 15:04:12 -0400533 struct extent_state **cached_state,
534 gfp_t mask)
Chris Masond1310b22008-01-24 16:13:08 -0500535{
536 struct extent_state *state;
Chris Mason2c64c532009-09-02 15:04:12 -0400537 struct extent_state *cached;
Chris Masond1310b22008-01-24 16:13:08 -0500538 struct extent_state *prealloc = NULL;
539 struct rb_node *node;
Yan Zheng5c939df2009-05-27 09:16:03 -0400540 u64 last_end;
Chris Masond1310b22008-01-24 16:13:08 -0500541 int err;
Josef Bacik2ac55d42010-02-03 19:33:23 +0000542 int clear = 0;
Chris Masond1310b22008-01-24 16:13:08 -0500543
David Sterba8d599ae2013-04-30 15:22:23 +0000544 btrfs_debug_check_extent_io_range(tree->mapping->host, start, end);
545
Josef Bacik7ee9e442013-06-21 16:37:03 -0400546 if (bits & EXTENT_DELALLOC)
547 bits |= EXTENT_NORESERVE;
548
Yan, Zheng0ca1f7c2010-05-16 10:48:47 -0400549 if (delete)
550 bits |= ~EXTENT_CTLBITS;
551 bits |= EXTENT_FIRST_DELALLOC;
552
Josef Bacik2ac55d42010-02-03 19:33:23 +0000553 if (bits & (EXTENT_IOBITS | EXTENT_BOUNDARY))
554 clear = 1;
Chris Masond1310b22008-01-24 16:13:08 -0500555again:
556 if (!prealloc && (mask & __GFP_WAIT)) {
557 prealloc = alloc_extent_state(mask);
558 if (!prealloc)
559 return -ENOMEM;
560 }
561
Chris Masoncad321a2008-12-17 14:51:42 -0500562 spin_lock(&tree->lock);
Chris Mason2c64c532009-09-02 15:04:12 -0400563 if (cached_state) {
564 cached = *cached_state;
Josef Bacik2ac55d42010-02-03 19:33:23 +0000565
566 if (clear) {
567 *cached_state = NULL;
568 cached_state = NULL;
569 }
570
Josef Bacikdf98b6e2011-06-20 14:53:48 -0400571 if (cached && cached->tree && cached->start <= start &&
572 cached->end > start) {
Josef Bacik2ac55d42010-02-03 19:33:23 +0000573 if (clear)
574 atomic_dec(&cached->refs);
Chris Mason2c64c532009-09-02 15:04:12 -0400575 state = cached;
Chris Mason42daec22009-09-23 19:51:09 -0400576 goto hit_next;
Chris Mason2c64c532009-09-02 15:04:12 -0400577 }
Josef Bacik2ac55d42010-02-03 19:33:23 +0000578 if (clear)
579 free_extent_state(cached);
Chris Mason2c64c532009-09-02 15:04:12 -0400580 }
Chris Masond1310b22008-01-24 16:13:08 -0500581 /*
582 * this search will find the extents that end after
583 * our range starts
584 */
Chris Mason80ea96b2008-02-01 14:51:59 -0500585 node = tree_search(tree, start);
Chris Masond1310b22008-01-24 16:13:08 -0500586 if (!node)
587 goto out;
588 state = rb_entry(node, struct extent_state, rb_node);
Chris Mason2c64c532009-09-02 15:04:12 -0400589hit_next:
Chris Masond1310b22008-01-24 16:13:08 -0500590 if (state->start > end)
591 goto out;
592 WARN_ON(state->end < start);
Yan Zheng5c939df2009-05-27 09:16:03 -0400593 last_end = state->end;
Chris Masond1310b22008-01-24 16:13:08 -0500594
Liu Bo04493142012-02-16 18:34:37 +0800595 /* the state doesn't have the wanted bits, go ahead */
Li Zefancdc6a392012-03-12 16:39:48 +0800596 if (!(state->state & bits)) {
597 state = next_state(state);
Liu Bo04493142012-02-16 18:34:37 +0800598 goto next;
Li Zefancdc6a392012-03-12 16:39:48 +0800599 }
Liu Bo04493142012-02-16 18:34:37 +0800600
Chris Masond1310b22008-01-24 16:13:08 -0500601 /*
602 * | ---- desired range ---- |
603 * | state | or
604 * | ------------- state -------------- |
605 *
606 * We need to split the extent we found, and may flip
607 * bits on second half.
608 *
609 * If the extent we found extends past our range, we
610 * just split and search again. It'll get split again
611 * the next time though.
612 *
613 * If the extent we found is inside our range, we clear
614 * the desired bit on it.
615 */
616
617 if (state->start < start) {
Xiao Guangrong82337672011-04-20 06:44:57 +0000618 prealloc = alloc_extent_state_atomic(prealloc);
619 BUG_ON(!prealloc);
Chris Masond1310b22008-01-24 16:13:08 -0500620 err = split_state(tree, state, prealloc, start);
Jeff Mahoneyc2d904e2011-10-03 23:22:32 -0400621 if (err)
622 extent_io_tree_panic(tree, err);
623
Chris Masond1310b22008-01-24 16:13:08 -0500624 prealloc = NULL;
625 if (err)
626 goto out;
627 if (state->end <= end) {
Liu Bod1ac6e42012-05-10 18:10:39 +0800628 state = clear_state_bit(tree, state, &bits, wake);
629 goto next;
Chris Masond1310b22008-01-24 16:13:08 -0500630 }
631 goto search_again;
632 }
633 /*
634 * | ---- desired range ---- |
635 * | state |
636 * We need to split the extent, and clear the bit
637 * on the first half
638 */
639 if (state->start <= end && state->end > end) {
Xiao Guangrong82337672011-04-20 06:44:57 +0000640 prealloc = alloc_extent_state_atomic(prealloc);
641 BUG_ON(!prealloc);
Chris Masond1310b22008-01-24 16:13:08 -0500642 err = split_state(tree, state, prealloc, end + 1);
Jeff Mahoneyc2d904e2011-10-03 23:22:32 -0400643 if (err)
644 extent_io_tree_panic(tree, err);
645
Chris Masond1310b22008-01-24 16:13:08 -0500646 if (wake)
647 wake_up(&state->wq);
Chris Mason42daec22009-09-23 19:51:09 -0400648
Jeff Mahoney6763af82012-03-01 14:56:29 +0100649 clear_state_bit(tree, prealloc, &bits, wake);
Josef Bacik9ed74f22009-09-11 16:12:44 -0400650
Chris Masond1310b22008-01-24 16:13:08 -0500651 prealloc = NULL;
652 goto out;
653 }
Chris Mason42daec22009-09-23 19:51:09 -0400654
Li Zefancdc6a392012-03-12 16:39:48 +0800655 state = clear_state_bit(tree, state, &bits, wake);
Liu Bo04493142012-02-16 18:34:37 +0800656next:
Yan Zheng5c939df2009-05-27 09:16:03 -0400657 if (last_end == (u64)-1)
658 goto out;
659 start = last_end + 1;
Li Zefancdc6a392012-03-12 16:39:48 +0800660 if (start <= end && state && !need_resched())
Liu Bo692e5752012-02-16 18:34:36 +0800661 goto hit_next;
Chris Masond1310b22008-01-24 16:13:08 -0500662 goto search_again;
663
664out:
Chris Masoncad321a2008-12-17 14:51:42 -0500665 spin_unlock(&tree->lock);
Chris Masond1310b22008-01-24 16:13:08 -0500666 if (prealloc)
667 free_extent_state(prealloc);
668
Jeff Mahoney6763af82012-03-01 14:56:29 +0100669 return 0;
Chris Masond1310b22008-01-24 16:13:08 -0500670
671search_again:
672 if (start > end)
673 goto out;
Chris Masoncad321a2008-12-17 14:51:42 -0500674 spin_unlock(&tree->lock);
Chris Masond1310b22008-01-24 16:13:08 -0500675 if (mask & __GFP_WAIT)
676 cond_resched();
677 goto again;
678}
Chris Masond1310b22008-01-24 16:13:08 -0500679
Jeff Mahoney143bede2012-03-01 14:56:26 +0100680static void wait_on_state(struct extent_io_tree *tree,
681 struct extent_state *state)
Christoph Hellwig641f5212008-12-02 06:36:10 -0500682 __releases(tree->lock)
683 __acquires(tree->lock)
Chris Masond1310b22008-01-24 16:13:08 -0500684{
685 DEFINE_WAIT(wait);
686 prepare_to_wait(&state->wq, &wait, TASK_UNINTERRUPTIBLE);
Chris Masoncad321a2008-12-17 14:51:42 -0500687 spin_unlock(&tree->lock);
Chris Masond1310b22008-01-24 16:13:08 -0500688 schedule();
Chris Masoncad321a2008-12-17 14:51:42 -0500689 spin_lock(&tree->lock);
Chris Masond1310b22008-01-24 16:13:08 -0500690 finish_wait(&state->wq, &wait);
Chris Masond1310b22008-01-24 16:13:08 -0500691}
692
693/*
694 * waits for one or more bits to clear on a range in the state tree.
695 * The range [start, end] is inclusive.
696 * The tree lock is taken by this function
697 */
David Sterba41074882013-04-29 13:38:46 +0000698static void wait_extent_bit(struct extent_io_tree *tree, u64 start, u64 end,
699 unsigned long bits)
Chris Masond1310b22008-01-24 16:13:08 -0500700{
701 struct extent_state *state;
702 struct rb_node *node;
703
David Sterba8d599ae2013-04-30 15:22:23 +0000704 btrfs_debug_check_extent_io_range(tree->mapping->host, start, end);
705
Chris Masoncad321a2008-12-17 14:51:42 -0500706 spin_lock(&tree->lock);
Chris Masond1310b22008-01-24 16:13:08 -0500707again:
708 while (1) {
709 /*
710 * this search will find all the extents that end after
711 * our range starts
712 */
Chris Mason80ea96b2008-02-01 14:51:59 -0500713 node = tree_search(tree, start);
Chris Masond1310b22008-01-24 16:13:08 -0500714 if (!node)
715 break;
716
717 state = rb_entry(node, struct extent_state, rb_node);
718
719 if (state->start > end)
720 goto out;
721
722 if (state->state & bits) {
723 start = state->start;
724 atomic_inc(&state->refs);
725 wait_on_state(tree, state);
726 free_extent_state(state);
727 goto again;
728 }
729 start = state->end + 1;
730
731 if (start > end)
732 break;
733
Xiao Guangrongded91f02011-07-14 03:19:27 +0000734 cond_resched_lock(&tree->lock);
Chris Masond1310b22008-01-24 16:13:08 -0500735 }
736out:
Chris Masoncad321a2008-12-17 14:51:42 -0500737 spin_unlock(&tree->lock);
Chris Masond1310b22008-01-24 16:13:08 -0500738}
Chris Masond1310b22008-01-24 16:13:08 -0500739
Jeff Mahoney1bf85042011-07-21 16:56:09 +0000740static void set_state_bits(struct extent_io_tree *tree,
Chris Masond1310b22008-01-24 16:13:08 -0500741 struct extent_state *state,
David Sterba41074882013-04-29 13:38:46 +0000742 unsigned long *bits)
Chris Masond1310b22008-01-24 16:13:08 -0500743{
David Sterba41074882013-04-29 13:38:46 +0000744 unsigned long bits_to_set = *bits & ~EXTENT_CTLBITS;
Josef Bacik9ed74f22009-09-11 16:12:44 -0400745
Jeff Mahoney1bf85042011-07-21 16:56:09 +0000746 set_state_cb(tree, state, bits);
Yan, Zheng0ca1f7c2010-05-16 10:48:47 -0400747 if ((bits_to_set & EXTENT_DIRTY) && !(state->state & EXTENT_DIRTY)) {
Chris Masond1310b22008-01-24 16:13:08 -0500748 u64 range = state->end - state->start + 1;
749 tree->dirty_bytes += range;
750 }
Yan, Zheng0ca1f7c2010-05-16 10:48:47 -0400751 state->state |= bits_to_set;
Chris Masond1310b22008-01-24 16:13:08 -0500752}
753
Chris Mason2c64c532009-09-02 15:04:12 -0400754static void cache_state(struct extent_state *state,
755 struct extent_state **cached_ptr)
756{
757 if (cached_ptr && !(*cached_ptr)) {
758 if (state->state & (EXTENT_IOBITS | EXTENT_BOUNDARY)) {
759 *cached_ptr = state;
760 atomic_inc(&state->refs);
761 }
762 }
763}
764
Arne Jansen507903b2011-04-06 10:02:20 +0000765static void uncache_state(struct extent_state **cached_ptr)
766{
767 if (cached_ptr && (*cached_ptr)) {
768 struct extent_state *state = *cached_ptr;
Chris Mason109b36a2011-04-12 13:57:39 -0400769 *cached_ptr = NULL;
770 free_extent_state(state);
Arne Jansen507903b2011-04-06 10:02:20 +0000771 }
772}
773
Chris Masond1310b22008-01-24 16:13:08 -0500774/*
Chris Mason1edbb732009-09-02 13:24:36 -0400775 * set some bits on a range in the tree. This may require allocations or
776 * sleeping, so the gfp mask is used to indicate what is allowed.
Chris Masond1310b22008-01-24 16:13:08 -0500777 *
Chris Mason1edbb732009-09-02 13:24:36 -0400778 * If any of the exclusive bits are set, this will fail with -EEXIST if some
779 * part of the range already has the desired bits set. The start of the
780 * existing range is returned in failed_start in this case.
Chris Masond1310b22008-01-24 16:13:08 -0500781 *
Chris Mason1edbb732009-09-02 13:24:36 -0400782 * [start, end] is inclusive This takes the tree lock.
Chris Masond1310b22008-01-24 16:13:08 -0500783 */
Chris Mason1edbb732009-09-02 13:24:36 -0400784
Jeff Mahoney3fbe5c02012-03-01 14:57:19 +0100785static int __must_check
786__set_extent_bit(struct extent_io_tree *tree, u64 start, u64 end,
David Sterba41074882013-04-29 13:38:46 +0000787 unsigned long bits, unsigned long exclusive_bits,
788 u64 *failed_start, struct extent_state **cached_state,
789 gfp_t mask)
Chris Masond1310b22008-01-24 16:13:08 -0500790{
791 struct extent_state *state;
792 struct extent_state *prealloc = NULL;
793 struct rb_node *node;
Chris Masond1310b22008-01-24 16:13:08 -0500794 int err = 0;
Chris Masond1310b22008-01-24 16:13:08 -0500795 u64 last_start;
796 u64 last_end;
Chris Mason42daec22009-09-23 19:51:09 -0400797
David Sterba8d599ae2013-04-30 15:22:23 +0000798 btrfs_debug_check_extent_io_range(tree->mapping->host, start, end);
799
Yan, Zheng0ca1f7c2010-05-16 10:48:47 -0400800 bits |= EXTENT_FIRST_DELALLOC;
Chris Masond1310b22008-01-24 16:13:08 -0500801again:
802 if (!prealloc && (mask & __GFP_WAIT)) {
803 prealloc = alloc_extent_state(mask);
Xiao Guangrong82337672011-04-20 06:44:57 +0000804 BUG_ON(!prealloc);
Chris Masond1310b22008-01-24 16:13:08 -0500805 }
806
Chris Masoncad321a2008-12-17 14:51:42 -0500807 spin_lock(&tree->lock);
Chris Mason9655d292009-09-02 15:22:30 -0400808 if (cached_state && *cached_state) {
809 state = *cached_state;
Josef Bacikdf98b6e2011-06-20 14:53:48 -0400810 if (state->start <= start && state->end > start &&
811 state->tree) {
Chris Mason9655d292009-09-02 15:22:30 -0400812 node = &state->rb_node;
813 goto hit_next;
814 }
815 }
Chris Masond1310b22008-01-24 16:13:08 -0500816 /*
817 * this search will find all the extents that end after
818 * our range starts.
819 */
Chris Mason80ea96b2008-02-01 14:51:59 -0500820 node = tree_search(tree, start);
Chris Masond1310b22008-01-24 16:13:08 -0500821 if (!node) {
Xiao Guangrong82337672011-04-20 06:44:57 +0000822 prealloc = alloc_extent_state_atomic(prealloc);
823 BUG_ON(!prealloc);
Yan, Zheng0ca1f7c2010-05-16 10:48:47 -0400824 err = insert_state(tree, prealloc, start, end, &bits);
Jeff Mahoneyc2d904e2011-10-03 23:22:32 -0400825 if (err)
826 extent_io_tree_panic(tree, err);
827
Chris Masond1310b22008-01-24 16:13:08 -0500828 prealloc = NULL;
Chris Masond1310b22008-01-24 16:13:08 -0500829 goto out;
830 }
Chris Masond1310b22008-01-24 16:13:08 -0500831 state = rb_entry(node, struct extent_state, rb_node);
Chris Mason40431d62009-08-05 12:57:59 -0400832hit_next:
Chris Masond1310b22008-01-24 16:13:08 -0500833 last_start = state->start;
834 last_end = state->end;
835
836 /*
837 * | ---- desired range ---- |
838 * | state |
839 *
840 * Just lock what we found and keep going
841 */
842 if (state->start == start && state->end <= end) {
Chris Mason1edbb732009-09-02 13:24:36 -0400843 if (state->state & exclusive_bits) {
Chris Masond1310b22008-01-24 16:13:08 -0500844 *failed_start = state->start;
845 err = -EEXIST;
846 goto out;
847 }
Chris Mason42daec22009-09-23 19:51:09 -0400848
Jeff Mahoney1bf85042011-07-21 16:56:09 +0000849 set_state_bits(tree, state, &bits);
Chris Mason2c64c532009-09-02 15:04:12 -0400850 cache_state(state, cached_state);
Chris Masond1310b22008-01-24 16:13:08 -0500851 merge_state(tree, state);
Yan Zheng5c939df2009-05-27 09:16:03 -0400852 if (last_end == (u64)-1)
853 goto out;
854 start = last_end + 1;
Liu Bod1ac6e42012-05-10 18:10:39 +0800855 state = next_state(state);
856 if (start < end && state && state->start == start &&
857 !need_resched())
858 goto hit_next;
Chris Masond1310b22008-01-24 16:13:08 -0500859 goto search_again;
860 }
861
862 /*
863 * | ---- desired range ---- |
864 * | state |
865 * or
866 * | ------------- state -------------- |
867 *
868 * We need to split the extent we found, and may flip bits on
869 * second half.
870 *
871 * If the extent we found extends past our
872 * range, we just split and search again. It'll get split
873 * again the next time though.
874 *
875 * If the extent we found is inside our range, we set the
876 * desired bit on it.
877 */
878 if (state->start < start) {
Chris Mason1edbb732009-09-02 13:24:36 -0400879 if (state->state & exclusive_bits) {
Chris Masond1310b22008-01-24 16:13:08 -0500880 *failed_start = start;
881 err = -EEXIST;
882 goto out;
883 }
Xiao Guangrong82337672011-04-20 06:44:57 +0000884
885 prealloc = alloc_extent_state_atomic(prealloc);
886 BUG_ON(!prealloc);
Chris Masond1310b22008-01-24 16:13:08 -0500887 err = split_state(tree, state, prealloc, start);
Jeff Mahoneyc2d904e2011-10-03 23:22:32 -0400888 if (err)
889 extent_io_tree_panic(tree, err);
890
Chris Masond1310b22008-01-24 16:13:08 -0500891 prealloc = NULL;
892 if (err)
893 goto out;
894 if (state->end <= end) {
Jeff Mahoney1bf85042011-07-21 16:56:09 +0000895 set_state_bits(tree, state, &bits);
Chris Mason2c64c532009-09-02 15:04:12 -0400896 cache_state(state, cached_state);
Chris Masond1310b22008-01-24 16:13:08 -0500897 merge_state(tree, state);
Yan Zheng5c939df2009-05-27 09:16:03 -0400898 if (last_end == (u64)-1)
899 goto out;
900 start = last_end + 1;
Liu Bod1ac6e42012-05-10 18:10:39 +0800901 state = next_state(state);
902 if (start < end && state && state->start == start &&
903 !need_resched())
904 goto hit_next;
Chris Masond1310b22008-01-24 16:13:08 -0500905 }
906 goto search_again;
907 }
908 /*
909 * | ---- desired range ---- |
910 * | state | or | state |
911 *
912 * There's a hole, we need to insert something in it and
913 * ignore the extent we found.
914 */
915 if (state->start > start) {
916 u64 this_end;
917 if (end < last_start)
918 this_end = end;
919 else
Chris Masond3977122009-01-05 21:25:51 -0500920 this_end = last_start - 1;
Xiao Guangrong82337672011-04-20 06:44:57 +0000921
922 prealloc = alloc_extent_state_atomic(prealloc);
923 BUG_ON(!prealloc);
Xiao Guangrongc7f895a2011-04-20 06:45:49 +0000924
925 /*
926 * Avoid to free 'prealloc' if it can be merged with
927 * the later extent.
928 */
Chris Masond1310b22008-01-24 16:13:08 -0500929 err = insert_state(tree, prealloc, start, this_end,
Yan, Zheng0ca1f7c2010-05-16 10:48:47 -0400930 &bits);
Jeff Mahoneyc2d904e2011-10-03 23:22:32 -0400931 if (err)
932 extent_io_tree_panic(tree, err);
933
Chris Mason2c64c532009-09-02 15:04:12 -0400934 cache_state(prealloc, cached_state);
Chris Masond1310b22008-01-24 16:13:08 -0500935 prealloc = NULL;
Chris Masond1310b22008-01-24 16:13:08 -0500936 start = this_end + 1;
937 goto search_again;
938 }
939 /*
940 * | ---- desired range ---- |
941 * | state |
942 * We need to split the extent, and set the bit
943 * on the first half
944 */
945 if (state->start <= end && state->end > end) {
Chris Mason1edbb732009-09-02 13:24:36 -0400946 if (state->state & exclusive_bits) {
Chris Masond1310b22008-01-24 16:13:08 -0500947 *failed_start = start;
948 err = -EEXIST;
949 goto out;
950 }
Xiao Guangrong82337672011-04-20 06:44:57 +0000951
952 prealloc = alloc_extent_state_atomic(prealloc);
953 BUG_ON(!prealloc);
Chris Masond1310b22008-01-24 16:13:08 -0500954 err = split_state(tree, state, prealloc, end + 1);
Jeff Mahoneyc2d904e2011-10-03 23:22:32 -0400955 if (err)
956 extent_io_tree_panic(tree, err);
Chris Masond1310b22008-01-24 16:13:08 -0500957
Jeff Mahoney1bf85042011-07-21 16:56:09 +0000958 set_state_bits(tree, prealloc, &bits);
Chris Mason2c64c532009-09-02 15:04:12 -0400959 cache_state(prealloc, cached_state);
Chris Masond1310b22008-01-24 16:13:08 -0500960 merge_state(tree, prealloc);
961 prealloc = NULL;
962 goto out;
963 }
964
965 goto search_again;
966
967out:
Chris Masoncad321a2008-12-17 14:51:42 -0500968 spin_unlock(&tree->lock);
Chris Masond1310b22008-01-24 16:13:08 -0500969 if (prealloc)
970 free_extent_state(prealloc);
971
972 return err;
973
974search_again:
975 if (start > end)
976 goto out;
Chris Masoncad321a2008-12-17 14:51:42 -0500977 spin_unlock(&tree->lock);
Chris Masond1310b22008-01-24 16:13:08 -0500978 if (mask & __GFP_WAIT)
979 cond_resched();
980 goto again;
981}
Chris Masond1310b22008-01-24 16:13:08 -0500982
David Sterba41074882013-04-29 13:38:46 +0000983int set_extent_bit(struct extent_io_tree *tree, u64 start, u64 end,
984 unsigned long bits, u64 * failed_start,
985 struct extent_state **cached_state, gfp_t mask)
Jeff Mahoney3fbe5c02012-03-01 14:57:19 +0100986{
987 return __set_extent_bit(tree, start, end, bits, 0, failed_start,
988 cached_state, mask);
989}
990
991
Josef Bacik462d6fa2011-09-26 13:56:12 -0400992/**
Liu Bo10983f22012-07-11 15:26:19 +0800993 * convert_extent_bit - convert all bits in a given range from one bit to
994 * another
Josef Bacik462d6fa2011-09-26 13:56:12 -0400995 * @tree: the io tree to search
996 * @start: the start offset in bytes
997 * @end: the end offset in bytes (inclusive)
998 * @bits: the bits to set in this range
999 * @clear_bits: the bits to clear in this range
Josef Bacike6138872012-09-27 17:07:30 -04001000 * @cached_state: state that we're going to cache
Josef Bacik462d6fa2011-09-26 13:56:12 -04001001 * @mask: the allocation mask
1002 *
1003 * This will go through and set bits for the given range. If any states exist
1004 * already in this range they are set with the given bit and cleared of the
1005 * clear_bits. This is only meant to be used by things that are mergeable, ie
1006 * converting from say DELALLOC to DIRTY. This is not meant to be used with
1007 * boundary bits like LOCK.
1008 */
1009int convert_extent_bit(struct extent_io_tree *tree, u64 start, u64 end,
David Sterba41074882013-04-29 13:38:46 +00001010 unsigned long bits, unsigned long clear_bits,
Josef Bacike6138872012-09-27 17:07:30 -04001011 struct extent_state **cached_state, gfp_t mask)
Josef Bacik462d6fa2011-09-26 13:56:12 -04001012{
1013 struct extent_state *state;
1014 struct extent_state *prealloc = NULL;
1015 struct rb_node *node;
1016 int err = 0;
1017 u64 last_start;
1018 u64 last_end;
1019
David Sterba8d599ae2013-04-30 15:22:23 +00001020 btrfs_debug_check_extent_io_range(tree->mapping->host, start, end);
1021
Josef Bacik462d6fa2011-09-26 13:56:12 -04001022again:
1023 if (!prealloc && (mask & __GFP_WAIT)) {
1024 prealloc = alloc_extent_state(mask);
1025 if (!prealloc)
1026 return -ENOMEM;
1027 }
1028
1029 spin_lock(&tree->lock);
Josef Bacike6138872012-09-27 17:07:30 -04001030 if (cached_state && *cached_state) {
1031 state = *cached_state;
1032 if (state->start <= start && state->end > start &&
1033 state->tree) {
1034 node = &state->rb_node;
1035 goto hit_next;
1036 }
1037 }
1038
Josef Bacik462d6fa2011-09-26 13:56:12 -04001039 /*
1040 * this search will find all the extents that end after
1041 * our range starts.
1042 */
1043 node = tree_search(tree, start);
1044 if (!node) {
1045 prealloc = alloc_extent_state_atomic(prealloc);
Liu Bo1cf4ffd2011-12-07 20:08:40 -05001046 if (!prealloc) {
1047 err = -ENOMEM;
1048 goto out;
1049 }
Josef Bacik462d6fa2011-09-26 13:56:12 -04001050 err = insert_state(tree, prealloc, start, end, &bits);
1051 prealloc = NULL;
Jeff Mahoneyc2d904e2011-10-03 23:22:32 -04001052 if (err)
1053 extent_io_tree_panic(tree, err);
Josef Bacik462d6fa2011-09-26 13:56:12 -04001054 goto out;
1055 }
1056 state = rb_entry(node, struct extent_state, rb_node);
1057hit_next:
1058 last_start = state->start;
1059 last_end = state->end;
1060
1061 /*
1062 * | ---- desired range ---- |
1063 * | state |
1064 *
1065 * Just lock what we found and keep going
1066 */
1067 if (state->start == start && state->end <= end) {
Josef Bacik462d6fa2011-09-26 13:56:12 -04001068 set_state_bits(tree, state, &bits);
Josef Bacike6138872012-09-27 17:07:30 -04001069 cache_state(state, cached_state);
Liu Bod1ac6e42012-05-10 18:10:39 +08001070 state = clear_state_bit(tree, state, &clear_bits, 0);
Josef Bacik462d6fa2011-09-26 13:56:12 -04001071 if (last_end == (u64)-1)
1072 goto out;
Josef Bacik462d6fa2011-09-26 13:56:12 -04001073 start = last_end + 1;
Liu Bod1ac6e42012-05-10 18:10:39 +08001074 if (start < end && state && state->start == start &&
1075 !need_resched())
1076 goto hit_next;
Josef Bacik462d6fa2011-09-26 13:56:12 -04001077 goto search_again;
1078 }
1079
1080 /*
1081 * | ---- desired range ---- |
1082 * | state |
1083 * or
1084 * | ------------- state -------------- |
1085 *
1086 * We need to split the extent we found, and may flip bits on
1087 * second half.
1088 *
1089 * If the extent we found extends past our
1090 * range, we just split and search again. It'll get split
1091 * again the next time though.
1092 *
1093 * If the extent we found is inside our range, we set the
1094 * desired bit on it.
1095 */
1096 if (state->start < start) {
1097 prealloc = alloc_extent_state_atomic(prealloc);
Liu Bo1cf4ffd2011-12-07 20:08:40 -05001098 if (!prealloc) {
1099 err = -ENOMEM;
1100 goto out;
1101 }
Josef Bacik462d6fa2011-09-26 13:56:12 -04001102 err = split_state(tree, state, prealloc, start);
Jeff Mahoneyc2d904e2011-10-03 23:22:32 -04001103 if (err)
1104 extent_io_tree_panic(tree, err);
Josef Bacik462d6fa2011-09-26 13:56:12 -04001105 prealloc = NULL;
1106 if (err)
1107 goto out;
1108 if (state->end <= end) {
1109 set_state_bits(tree, state, &bits);
Josef Bacike6138872012-09-27 17:07:30 -04001110 cache_state(state, cached_state);
Liu Bod1ac6e42012-05-10 18:10:39 +08001111 state = clear_state_bit(tree, state, &clear_bits, 0);
Josef Bacik462d6fa2011-09-26 13:56:12 -04001112 if (last_end == (u64)-1)
1113 goto out;
1114 start = last_end + 1;
Liu Bod1ac6e42012-05-10 18:10:39 +08001115 if (start < end && state && state->start == start &&
1116 !need_resched())
1117 goto hit_next;
Josef Bacik462d6fa2011-09-26 13:56:12 -04001118 }
1119 goto search_again;
1120 }
1121 /*
1122 * | ---- desired range ---- |
1123 * | state | or | state |
1124 *
1125 * There's a hole, we need to insert something in it and
1126 * ignore the extent we found.
1127 */
1128 if (state->start > start) {
1129 u64 this_end;
1130 if (end < last_start)
1131 this_end = end;
1132 else
1133 this_end = last_start - 1;
1134
1135 prealloc = alloc_extent_state_atomic(prealloc);
Liu Bo1cf4ffd2011-12-07 20:08:40 -05001136 if (!prealloc) {
1137 err = -ENOMEM;
1138 goto out;
1139 }
Josef Bacik462d6fa2011-09-26 13:56:12 -04001140
1141 /*
1142 * Avoid to free 'prealloc' if it can be merged with
1143 * the later extent.
1144 */
1145 err = insert_state(tree, prealloc, start, this_end,
1146 &bits);
Jeff Mahoneyc2d904e2011-10-03 23:22:32 -04001147 if (err)
1148 extent_io_tree_panic(tree, err);
Josef Bacike6138872012-09-27 17:07:30 -04001149 cache_state(prealloc, cached_state);
Josef Bacik462d6fa2011-09-26 13:56:12 -04001150 prealloc = NULL;
1151 start = this_end + 1;
1152 goto search_again;
1153 }
1154 /*
1155 * | ---- desired range ---- |
1156 * | state |
1157 * We need to split the extent, and set the bit
1158 * on the first half
1159 */
1160 if (state->start <= end && state->end > end) {
1161 prealloc = alloc_extent_state_atomic(prealloc);
Liu Bo1cf4ffd2011-12-07 20:08:40 -05001162 if (!prealloc) {
1163 err = -ENOMEM;
1164 goto out;
1165 }
Josef Bacik462d6fa2011-09-26 13:56:12 -04001166
1167 err = split_state(tree, state, prealloc, end + 1);
Jeff Mahoneyc2d904e2011-10-03 23:22:32 -04001168 if (err)
1169 extent_io_tree_panic(tree, err);
Josef Bacik462d6fa2011-09-26 13:56:12 -04001170
1171 set_state_bits(tree, prealloc, &bits);
Josef Bacike6138872012-09-27 17:07:30 -04001172 cache_state(prealloc, cached_state);
Josef Bacik462d6fa2011-09-26 13:56:12 -04001173 clear_state_bit(tree, prealloc, &clear_bits, 0);
Josef Bacik462d6fa2011-09-26 13:56:12 -04001174 prealloc = NULL;
1175 goto out;
1176 }
1177
1178 goto search_again;
1179
1180out:
1181 spin_unlock(&tree->lock);
1182 if (prealloc)
1183 free_extent_state(prealloc);
1184
1185 return err;
1186
1187search_again:
1188 if (start > end)
1189 goto out;
1190 spin_unlock(&tree->lock);
1191 if (mask & __GFP_WAIT)
1192 cond_resched();
1193 goto again;
1194}
1195
Chris Masond1310b22008-01-24 16:13:08 -05001196/* wrappers around set/clear extent bit */
1197int set_extent_dirty(struct extent_io_tree *tree, u64 start, u64 end,
1198 gfp_t mask)
1199{
Jeff Mahoney3fbe5c02012-03-01 14:57:19 +01001200 return set_extent_bit(tree, start, end, EXTENT_DIRTY, NULL,
Chris Mason2c64c532009-09-02 15:04:12 -04001201 NULL, mask);
Chris Masond1310b22008-01-24 16:13:08 -05001202}
Chris Masond1310b22008-01-24 16:13:08 -05001203
1204int set_extent_bits(struct extent_io_tree *tree, u64 start, u64 end,
David Sterba41074882013-04-29 13:38:46 +00001205 unsigned long bits, gfp_t mask)
Chris Masond1310b22008-01-24 16:13:08 -05001206{
Jeff Mahoney3fbe5c02012-03-01 14:57:19 +01001207 return set_extent_bit(tree, start, end, bits, NULL,
Chris Mason2c64c532009-09-02 15:04:12 -04001208 NULL, mask);
Chris Masond1310b22008-01-24 16:13:08 -05001209}
Chris Masond1310b22008-01-24 16:13:08 -05001210
1211int clear_extent_bits(struct extent_io_tree *tree, u64 start, u64 end,
David Sterba41074882013-04-29 13:38:46 +00001212 unsigned long bits, gfp_t mask)
Chris Masond1310b22008-01-24 16:13:08 -05001213{
Chris Mason2c64c532009-09-02 15:04:12 -04001214 return clear_extent_bit(tree, start, end, bits, 0, 0, NULL, mask);
Chris Masond1310b22008-01-24 16:13:08 -05001215}
Chris Masond1310b22008-01-24 16:13:08 -05001216
1217int set_extent_delalloc(struct extent_io_tree *tree, u64 start, u64 end,
Josef Bacik2ac55d42010-02-03 19:33:23 +00001218 struct extent_state **cached_state, gfp_t mask)
Chris Masond1310b22008-01-24 16:13:08 -05001219{
1220 return set_extent_bit(tree, start, end,
Liu Bofee187d2011-09-29 15:55:28 +08001221 EXTENT_DELALLOC | EXTENT_UPTODATE,
Jeff Mahoney3fbe5c02012-03-01 14:57:19 +01001222 NULL, cached_state, mask);
Chris Masond1310b22008-01-24 16:13:08 -05001223}
Chris Masond1310b22008-01-24 16:13:08 -05001224
Liu Bo9e8a4a82012-09-05 19:10:51 -06001225int set_extent_defrag(struct extent_io_tree *tree, u64 start, u64 end,
1226 struct extent_state **cached_state, gfp_t mask)
1227{
1228 return set_extent_bit(tree, start, end,
1229 EXTENT_DELALLOC | EXTENT_UPTODATE | EXTENT_DEFRAG,
1230 NULL, cached_state, mask);
1231}
1232
Chris Masond1310b22008-01-24 16:13:08 -05001233int clear_extent_dirty(struct extent_io_tree *tree, u64 start, u64 end,
1234 gfp_t mask)
1235{
1236 return clear_extent_bit(tree, start, end,
Josef Bacik32c00af2009-10-08 13:34:05 -04001237 EXTENT_DIRTY | EXTENT_DELALLOC |
Yan, Zheng0ca1f7c2010-05-16 10:48:47 -04001238 EXTENT_DO_ACCOUNTING, 0, 0, NULL, mask);
Chris Masond1310b22008-01-24 16:13:08 -05001239}
Chris Masond1310b22008-01-24 16:13:08 -05001240
1241int set_extent_new(struct extent_io_tree *tree, u64 start, u64 end,
1242 gfp_t mask)
1243{
Jeff Mahoney3fbe5c02012-03-01 14:57:19 +01001244 return set_extent_bit(tree, start, end, EXTENT_NEW, NULL,
Chris Mason2c64c532009-09-02 15:04:12 -04001245 NULL, mask);
Chris Masond1310b22008-01-24 16:13:08 -05001246}
Chris Masond1310b22008-01-24 16:13:08 -05001247
Chris Masond1310b22008-01-24 16:13:08 -05001248int set_extent_uptodate(struct extent_io_tree *tree, u64 start, u64 end,
Arne Jansen507903b2011-04-06 10:02:20 +00001249 struct extent_state **cached_state, gfp_t mask)
Chris Masond1310b22008-01-24 16:13:08 -05001250{
Liu Bo6b67a322013-03-28 08:30:28 +00001251 return set_extent_bit(tree, start, end, EXTENT_UPTODATE, NULL,
Jeff Mahoney3fbe5c02012-03-01 14:57:19 +01001252 cached_state, mask);
Chris Masond1310b22008-01-24 16:13:08 -05001253}
Chris Masond1310b22008-01-24 16:13:08 -05001254
Josef Bacik5fd02042012-05-02 14:00:54 -04001255int clear_extent_uptodate(struct extent_io_tree *tree, u64 start, u64 end,
1256 struct extent_state **cached_state, gfp_t mask)
Chris Masond1310b22008-01-24 16:13:08 -05001257{
Chris Mason2c64c532009-09-02 15:04:12 -04001258 return clear_extent_bit(tree, start, end, EXTENT_UPTODATE, 0, 0,
Josef Bacik2ac55d42010-02-03 19:33:23 +00001259 cached_state, mask);
Chris Masond1310b22008-01-24 16:13:08 -05001260}
Chris Masond1310b22008-01-24 16:13:08 -05001261
Chris Masond352ac62008-09-29 15:18:18 -04001262/*
1263 * either insert or lock state struct between start and end use mask to tell
1264 * us if waiting is desired.
1265 */
Chris Mason1edbb732009-09-02 13:24:36 -04001266int lock_extent_bits(struct extent_io_tree *tree, u64 start, u64 end,
David Sterba41074882013-04-29 13:38:46 +00001267 unsigned long bits, struct extent_state **cached_state)
Chris Masond1310b22008-01-24 16:13:08 -05001268{
1269 int err;
1270 u64 failed_start;
1271 while (1) {
Jeff Mahoney3fbe5c02012-03-01 14:57:19 +01001272 err = __set_extent_bit(tree, start, end, EXTENT_LOCKED | bits,
1273 EXTENT_LOCKED, &failed_start,
1274 cached_state, GFP_NOFS);
Jeff Mahoneyd0082372012-03-01 14:57:19 +01001275 if (err == -EEXIST) {
Chris Masond1310b22008-01-24 16:13:08 -05001276 wait_extent_bit(tree, failed_start, end, EXTENT_LOCKED);
1277 start = failed_start;
Jeff Mahoneyd0082372012-03-01 14:57:19 +01001278 } else
Chris Masond1310b22008-01-24 16:13:08 -05001279 break;
Chris Masond1310b22008-01-24 16:13:08 -05001280 WARN_ON(start > end);
1281 }
1282 return err;
1283}
Chris Masond1310b22008-01-24 16:13:08 -05001284
Jeff Mahoneyd0082372012-03-01 14:57:19 +01001285int lock_extent(struct extent_io_tree *tree, u64 start, u64 end)
Chris Mason1edbb732009-09-02 13:24:36 -04001286{
Jeff Mahoneyd0082372012-03-01 14:57:19 +01001287 return lock_extent_bits(tree, start, end, 0, NULL);
Chris Mason1edbb732009-09-02 13:24:36 -04001288}
1289
Jeff Mahoneyd0082372012-03-01 14:57:19 +01001290int try_lock_extent(struct extent_io_tree *tree, u64 start, u64 end)
Josef Bacik25179202008-10-29 14:49:05 -04001291{
1292 int err;
1293 u64 failed_start;
1294
Jeff Mahoney3fbe5c02012-03-01 14:57:19 +01001295 err = __set_extent_bit(tree, start, end, EXTENT_LOCKED, EXTENT_LOCKED,
1296 &failed_start, NULL, GFP_NOFS);
Yan Zheng66435582008-10-30 14:19:50 -04001297 if (err == -EEXIST) {
1298 if (failed_start > start)
1299 clear_extent_bit(tree, start, failed_start - 1,
Jeff Mahoneyd0082372012-03-01 14:57:19 +01001300 EXTENT_LOCKED, 1, 0, NULL, GFP_NOFS);
Josef Bacik25179202008-10-29 14:49:05 -04001301 return 0;
Yan Zheng66435582008-10-30 14:19:50 -04001302 }
Josef Bacik25179202008-10-29 14:49:05 -04001303 return 1;
1304}
Josef Bacik25179202008-10-29 14:49:05 -04001305
Chris Mason2c64c532009-09-02 15:04:12 -04001306int unlock_extent_cached(struct extent_io_tree *tree, u64 start, u64 end,
1307 struct extent_state **cached, gfp_t mask)
1308{
1309 return clear_extent_bit(tree, start, end, EXTENT_LOCKED, 1, 0, cached,
1310 mask);
1311}
1312
Jeff Mahoneyd0082372012-03-01 14:57:19 +01001313int unlock_extent(struct extent_io_tree *tree, u64 start, u64 end)
Chris Masond1310b22008-01-24 16:13:08 -05001314{
Chris Mason2c64c532009-09-02 15:04:12 -04001315 return clear_extent_bit(tree, start, end, EXTENT_LOCKED, 1, 0, NULL,
Jeff Mahoneyd0082372012-03-01 14:57:19 +01001316 GFP_NOFS);
Chris Masond1310b22008-01-24 16:13:08 -05001317}
Chris Masond1310b22008-01-24 16:13:08 -05001318
Chris Mason4adaa612013-03-26 13:07:00 -04001319int extent_range_clear_dirty_for_io(struct inode *inode, u64 start, u64 end)
1320{
1321 unsigned long index = start >> PAGE_CACHE_SHIFT;
1322 unsigned long end_index = end >> PAGE_CACHE_SHIFT;
1323 struct page *page;
1324
1325 while (index <= end_index) {
1326 page = find_get_page(inode->i_mapping, index);
1327 BUG_ON(!page); /* Pages should be in the extent_io_tree */
1328 clear_page_dirty_for_io(page);
1329 page_cache_release(page);
1330 index++;
1331 }
1332 return 0;
1333}
1334
1335int extent_range_redirty_for_io(struct inode *inode, u64 start, u64 end)
1336{
1337 unsigned long index = start >> PAGE_CACHE_SHIFT;
1338 unsigned long end_index = end >> PAGE_CACHE_SHIFT;
1339 struct page *page;
1340
1341 while (index <= end_index) {
1342 page = find_get_page(inode->i_mapping, index);
1343 BUG_ON(!page); /* Pages should be in the extent_io_tree */
1344 account_page_redirty(page);
1345 __set_page_dirty_nobuffers(page);
1346 page_cache_release(page);
1347 index++;
1348 }
1349 return 0;
1350}
1351
Chris Masond1310b22008-01-24 16:13:08 -05001352/*
Chris Masond1310b22008-01-24 16:13:08 -05001353 * helper function to set both pages and extents in the tree writeback
1354 */
Christoph Hellwigb2950862008-12-02 09:54:17 -05001355static int set_range_writeback(struct extent_io_tree *tree, u64 start, u64 end)
Chris Masond1310b22008-01-24 16:13:08 -05001356{
1357 unsigned long index = start >> PAGE_CACHE_SHIFT;
1358 unsigned long end_index = end >> PAGE_CACHE_SHIFT;
1359 struct page *page;
1360
1361 while (index <= end_index) {
1362 page = find_get_page(tree->mapping, index);
Jeff Mahoney79787ea2012-03-12 16:03:00 +01001363 BUG_ON(!page); /* Pages should be in the extent_io_tree */
Chris Masond1310b22008-01-24 16:13:08 -05001364 set_page_writeback(page);
1365 page_cache_release(page);
1366 index++;
1367 }
Chris Masond1310b22008-01-24 16:13:08 -05001368 return 0;
1369}
Chris Masond1310b22008-01-24 16:13:08 -05001370
Chris Masond352ac62008-09-29 15:18:18 -04001371/* find the first state struct with 'bits' set after 'start', and
1372 * return it. tree->lock must be held. NULL will returned if
1373 * nothing was found after 'start'
1374 */
Eric Sandeen48a3b632013-04-25 20:41:01 +00001375static struct extent_state *
1376find_first_extent_bit_state(struct extent_io_tree *tree,
David Sterba41074882013-04-29 13:38:46 +00001377 u64 start, unsigned long bits)
Chris Masond7fc6402008-02-18 12:12:38 -05001378{
1379 struct rb_node *node;
1380 struct extent_state *state;
1381
1382 /*
1383 * this search will find all the extents that end after
1384 * our range starts.
1385 */
1386 node = tree_search(tree, start);
Chris Masond3977122009-01-05 21:25:51 -05001387 if (!node)
Chris Masond7fc6402008-02-18 12:12:38 -05001388 goto out;
Chris Masond7fc6402008-02-18 12:12:38 -05001389
Chris Masond3977122009-01-05 21:25:51 -05001390 while (1) {
Chris Masond7fc6402008-02-18 12:12:38 -05001391 state = rb_entry(node, struct extent_state, rb_node);
Chris Masond3977122009-01-05 21:25:51 -05001392 if (state->end >= start && (state->state & bits))
Chris Masond7fc6402008-02-18 12:12:38 -05001393 return state;
Chris Masond3977122009-01-05 21:25:51 -05001394
Chris Masond7fc6402008-02-18 12:12:38 -05001395 node = rb_next(node);
1396 if (!node)
1397 break;
1398 }
1399out:
1400 return NULL;
1401}
Chris Masond7fc6402008-02-18 12:12:38 -05001402
Chris Masond352ac62008-09-29 15:18:18 -04001403/*
Xiao Guangrong69261c42011-07-14 03:19:45 +00001404 * find the first offset in the io tree with 'bits' set. zero is
1405 * returned if we find something, and *start_ret and *end_ret are
1406 * set to reflect the state struct that was found.
1407 *
Wang Sheng-Hui477d7ea2012-04-06 14:35:47 +08001408 * If nothing was found, 1 is returned. If found something, return 0.
Xiao Guangrong69261c42011-07-14 03:19:45 +00001409 */
1410int find_first_extent_bit(struct extent_io_tree *tree, u64 start,
David Sterba41074882013-04-29 13:38:46 +00001411 u64 *start_ret, u64 *end_ret, unsigned long bits,
Josef Bacike6138872012-09-27 17:07:30 -04001412 struct extent_state **cached_state)
Xiao Guangrong69261c42011-07-14 03:19:45 +00001413{
1414 struct extent_state *state;
Josef Bacike6138872012-09-27 17:07:30 -04001415 struct rb_node *n;
Xiao Guangrong69261c42011-07-14 03:19:45 +00001416 int ret = 1;
1417
1418 spin_lock(&tree->lock);
Josef Bacike6138872012-09-27 17:07:30 -04001419 if (cached_state && *cached_state) {
1420 state = *cached_state;
1421 if (state->end == start - 1 && state->tree) {
1422 n = rb_next(&state->rb_node);
1423 while (n) {
1424 state = rb_entry(n, struct extent_state,
1425 rb_node);
1426 if (state->state & bits)
1427 goto got_it;
1428 n = rb_next(n);
1429 }
1430 free_extent_state(*cached_state);
1431 *cached_state = NULL;
1432 goto out;
1433 }
1434 free_extent_state(*cached_state);
1435 *cached_state = NULL;
1436 }
1437
Xiao Guangrong69261c42011-07-14 03:19:45 +00001438 state = find_first_extent_bit_state(tree, start, bits);
Josef Bacike6138872012-09-27 17:07:30 -04001439got_it:
Xiao Guangrong69261c42011-07-14 03:19:45 +00001440 if (state) {
Josef Bacike6138872012-09-27 17:07:30 -04001441 cache_state(state, cached_state);
Xiao Guangrong69261c42011-07-14 03:19:45 +00001442 *start_ret = state->start;
1443 *end_ret = state->end;
1444 ret = 0;
1445 }
Josef Bacike6138872012-09-27 17:07:30 -04001446out:
Xiao Guangrong69261c42011-07-14 03:19:45 +00001447 spin_unlock(&tree->lock);
1448 return ret;
1449}
1450
1451/*
Chris Masond352ac62008-09-29 15:18:18 -04001452 * find a contiguous range of bytes in the file marked as delalloc, not
1453 * more than 'max_bytes'. start and end are used to return the range,
1454 *
1455 * 1 is returned if we find something, 0 if nothing was in the tree
1456 */
Chris Masonc8b97812008-10-29 14:49:59 -04001457static noinline u64 find_delalloc_range(struct extent_io_tree *tree,
Josef Bacikc2a128d2010-02-02 21:19:11 +00001458 u64 *start, u64 *end, u64 max_bytes,
1459 struct extent_state **cached_state)
Chris Masond1310b22008-01-24 16:13:08 -05001460{
1461 struct rb_node *node;
1462 struct extent_state *state;
1463 u64 cur_start = *start;
1464 u64 found = 0;
1465 u64 total_bytes = 0;
1466
Chris Masoncad321a2008-12-17 14:51:42 -05001467 spin_lock(&tree->lock);
Chris Masonc8b97812008-10-29 14:49:59 -04001468
Chris Masond1310b22008-01-24 16:13:08 -05001469 /*
1470 * this search will find all the extents that end after
1471 * our range starts.
1472 */
Chris Mason80ea96b2008-02-01 14:51:59 -05001473 node = tree_search(tree, cur_start);
Peter2b114d12008-04-01 11:21:40 -04001474 if (!node) {
Chris Mason3b951512008-04-17 11:29:12 -04001475 if (!found)
1476 *end = (u64)-1;
Chris Masond1310b22008-01-24 16:13:08 -05001477 goto out;
1478 }
1479
Chris Masond3977122009-01-05 21:25:51 -05001480 while (1) {
Chris Masond1310b22008-01-24 16:13:08 -05001481 state = rb_entry(node, struct extent_state, rb_node);
Zheng Yan5b21f2e2008-09-26 10:05:38 -04001482 if (found && (state->start != cur_start ||
1483 (state->state & EXTENT_BOUNDARY))) {
Chris Masond1310b22008-01-24 16:13:08 -05001484 goto out;
1485 }
1486 if (!(state->state & EXTENT_DELALLOC)) {
1487 if (!found)
1488 *end = state->end;
1489 goto out;
1490 }
Josef Bacikc2a128d2010-02-02 21:19:11 +00001491 if (!found) {
Chris Masond1310b22008-01-24 16:13:08 -05001492 *start = state->start;
Josef Bacikc2a128d2010-02-02 21:19:11 +00001493 *cached_state = state;
1494 atomic_inc(&state->refs);
1495 }
Chris Masond1310b22008-01-24 16:13:08 -05001496 found++;
1497 *end = state->end;
1498 cur_start = state->end + 1;
1499 node = rb_next(node);
1500 if (!node)
1501 break;
1502 total_bytes += state->end - state->start + 1;
1503 if (total_bytes >= max_bytes)
1504 break;
1505 }
1506out:
Chris Masoncad321a2008-12-17 14:51:42 -05001507 spin_unlock(&tree->lock);
Chris Masond1310b22008-01-24 16:13:08 -05001508 return found;
1509}
1510
Jeff Mahoney143bede2012-03-01 14:56:26 +01001511static noinline void __unlock_for_delalloc(struct inode *inode,
1512 struct page *locked_page,
1513 u64 start, u64 end)
Chris Masonc8b97812008-10-29 14:49:59 -04001514{
1515 int ret;
1516 struct page *pages[16];
1517 unsigned long index = start >> PAGE_CACHE_SHIFT;
1518 unsigned long end_index = end >> PAGE_CACHE_SHIFT;
1519 unsigned long nr_pages = end_index - index + 1;
1520 int i;
1521
1522 if (index == locked_page->index && end_index == index)
Jeff Mahoney143bede2012-03-01 14:56:26 +01001523 return;
Chris Masonc8b97812008-10-29 14:49:59 -04001524
Chris Masond3977122009-01-05 21:25:51 -05001525 while (nr_pages > 0) {
Chris Masonc8b97812008-10-29 14:49:59 -04001526 ret = find_get_pages_contig(inode->i_mapping, index,
Chris Mason5b050f02008-11-11 09:34:41 -05001527 min_t(unsigned long, nr_pages,
1528 ARRAY_SIZE(pages)), pages);
Chris Masonc8b97812008-10-29 14:49:59 -04001529 for (i = 0; i < ret; i++) {
1530 if (pages[i] != locked_page)
1531 unlock_page(pages[i]);
1532 page_cache_release(pages[i]);
1533 }
1534 nr_pages -= ret;
1535 index += ret;
1536 cond_resched();
1537 }
Chris Masonc8b97812008-10-29 14:49:59 -04001538}
1539
1540static noinline int lock_delalloc_pages(struct inode *inode,
1541 struct page *locked_page,
1542 u64 delalloc_start,
1543 u64 delalloc_end)
1544{
1545 unsigned long index = delalloc_start >> PAGE_CACHE_SHIFT;
1546 unsigned long start_index = index;
1547 unsigned long end_index = delalloc_end >> PAGE_CACHE_SHIFT;
1548 unsigned long pages_locked = 0;
1549 struct page *pages[16];
1550 unsigned long nrpages;
1551 int ret;
1552 int i;
1553
1554 /* the caller is responsible for locking the start index */
1555 if (index == locked_page->index && index == end_index)
1556 return 0;
1557
1558 /* skip the page at the start index */
1559 nrpages = end_index - index + 1;
Chris Masond3977122009-01-05 21:25:51 -05001560 while (nrpages > 0) {
Chris Masonc8b97812008-10-29 14:49:59 -04001561 ret = find_get_pages_contig(inode->i_mapping, index,
Chris Mason5b050f02008-11-11 09:34:41 -05001562 min_t(unsigned long,
1563 nrpages, ARRAY_SIZE(pages)), pages);
Chris Masonc8b97812008-10-29 14:49:59 -04001564 if (ret == 0) {
1565 ret = -EAGAIN;
1566 goto done;
1567 }
1568 /* now we have an array of pages, lock them all */
1569 for (i = 0; i < ret; i++) {
1570 /*
1571 * the caller is taking responsibility for
1572 * locked_page
1573 */
Chris Mason771ed682008-11-06 22:02:51 -05001574 if (pages[i] != locked_page) {
Chris Masonc8b97812008-10-29 14:49:59 -04001575 lock_page(pages[i]);
Chris Masonf2b1c412008-11-10 07:31:30 -05001576 if (!PageDirty(pages[i]) ||
1577 pages[i]->mapping != inode->i_mapping) {
Chris Mason771ed682008-11-06 22:02:51 -05001578 ret = -EAGAIN;
1579 unlock_page(pages[i]);
1580 page_cache_release(pages[i]);
1581 goto done;
1582 }
1583 }
Chris Masonc8b97812008-10-29 14:49:59 -04001584 page_cache_release(pages[i]);
Chris Mason771ed682008-11-06 22:02:51 -05001585 pages_locked++;
Chris Masonc8b97812008-10-29 14:49:59 -04001586 }
Chris Masonc8b97812008-10-29 14:49:59 -04001587 nrpages -= ret;
1588 index += ret;
1589 cond_resched();
1590 }
1591 ret = 0;
1592done:
1593 if (ret && pages_locked) {
1594 __unlock_for_delalloc(inode, locked_page,
1595 delalloc_start,
1596 ((u64)(start_index + pages_locked - 1)) <<
1597 PAGE_CACHE_SHIFT);
1598 }
1599 return ret;
1600}
1601
1602/*
1603 * find a contiguous range of bytes in the file marked as delalloc, not
1604 * more than 'max_bytes'. start and end are used to return the range,
1605 *
1606 * 1 is returned if we find something, 0 if nothing was in the tree
1607 */
1608static noinline u64 find_lock_delalloc_range(struct inode *inode,
1609 struct extent_io_tree *tree,
1610 struct page *locked_page,
1611 u64 *start, u64 *end,
1612 u64 max_bytes)
1613{
1614 u64 delalloc_start;
1615 u64 delalloc_end;
1616 u64 found;
Chris Mason9655d292009-09-02 15:22:30 -04001617 struct extent_state *cached_state = NULL;
Chris Masonc8b97812008-10-29 14:49:59 -04001618 int ret;
1619 int loops = 0;
1620
1621again:
1622 /* step one, find a bunch of delalloc bytes starting at start */
1623 delalloc_start = *start;
1624 delalloc_end = 0;
1625 found = find_delalloc_range(tree, &delalloc_start, &delalloc_end,
Josef Bacikc2a128d2010-02-02 21:19:11 +00001626 max_bytes, &cached_state);
Chris Mason70b99e62008-10-31 12:46:39 -04001627 if (!found || delalloc_end <= *start) {
Chris Masonc8b97812008-10-29 14:49:59 -04001628 *start = delalloc_start;
1629 *end = delalloc_end;
Josef Bacikc2a128d2010-02-02 21:19:11 +00001630 free_extent_state(cached_state);
Chris Masonc8b97812008-10-29 14:49:59 -04001631 return found;
1632 }
1633
1634 /*
Chris Mason70b99e62008-10-31 12:46:39 -04001635 * start comes from the offset of locked_page. We have to lock
1636 * pages in order, so we can't process delalloc bytes before
1637 * locked_page
1638 */
Chris Masond3977122009-01-05 21:25:51 -05001639 if (delalloc_start < *start)
Chris Mason70b99e62008-10-31 12:46:39 -04001640 delalloc_start = *start;
Chris Mason70b99e62008-10-31 12:46:39 -04001641
1642 /*
Chris Masonc8b97812008-10-29 14:49:59 -04001643 * make sure to limit the number of pages we try to lock down
1644 * if we're looping.
1645 */
Chris Masond3977122009-01-05 21:25:51 -05001646 if (delalloc_end + 1 - delalloc_start > max_bytes && loops)
Chris Mason771ed682008-11-06 22:02:51 -05001647 delalloc_end = delalloc_start + PAGE_CACHE_SIZE - 1;
Chris Masond3977122009-01-05 21:25:51 -05001648
Chris Masonc8b97812008-10-29 14:49:59 -04001649 /* step two, lock all the pages after the page that has start */
1650 ret = lock_delalloc_pages(inode, locked_page,
1651 delalloc_start, delalloc_end);
1652 if (ret == -EAGAIN) {
1653 /* some of the pages are gone, lets avoid looping by
1654 * shortening the size of the delalloc range we're searching
1655 */
Chris Mason9655d292009-09-02 15:22:30 -04001656 free_extent_state(cached_state);
Chris Masonc8b97812008-10-29 14:49:59 -04001657 if (!loops) {
1658 unsigned long offset = (*start) & (PAGE_CACHE_SIZE - 1);
1659 max_bytes = PAGE_CACHE_SIZE - offset;
1660 loops = 1;
1661 goto again;
1662 } else {
1663 found = 0;
1664 goto out_failed;
1665 }
1666 }
Jeff Mahoney79787ea2012-03-12 16:03:00 +01001667 BUG_ON(ret); /* Only valid values are 0 and -EAGAIN */
Chris Masonc8b97812008-10-29 14:49:59 -04001668
1669 /* step three, lock the state bits for the whole range */
Jeff Mahoneyd0082372012-03-01 14:57:19 +01001670 lock_extent_bits(tree, delalloc_start, delalloc_end, 0, &cached_state);
Chris Masonc8b97812008-10-29 14:49:59 -04001671
1672 /* then test to make sure it is all still delalloc */
1673 ret = test_range_bit(tree, delalloc_start, delalloc_end,
Chris Mason9655d292009-09-02 15:22:30 -04001674 EXTENT_DELALLOC, 1, cached_state);
Chris Masonc8b97812008-10-29 14:49:59 -04001675 if (!ret) {
Chris Mason9655d292009-09-02 15:22:30 -04001676 unlock_extent_cached(tree, delalloc_start, delalloc_end,
1677 &cached_state, GFP_NOFS);
Chris Masonc8b97812008-10-29 14:49:59 -04001678 __unlock_for_delalloc(inode, locked_page,
1679 delalloc_start, delalloc_end);
1680 cond_resched();
1681 goto again;
1682 }
Chris Mason9655d292009-09-02 15:22:30 -04001683 free_extent_state(cached_state);
Chris Masonc8b97812008-10-29 14:49:59 -04001684 *start = delalloc_start;
1685 *end = delalloc_end;
1686out_failed:
1687 return found;
1688}
1689
1690int extent_clear_unlock_delalloc(struct inode *inode,
1691 struct extent_io_tree *tree,
1692 u64 start, u64 end, struct page *locked_page,
Chris Masona791e352009-10-08 11:27:10 -04001693 unsigned long op)
Chris Masonc8b97812008-10-29 14:49:59 -04001694{
1695 int ret;
1696 struct page *pages[16];
1697 unsigned long index = start >> PAGE_CACHE_SHIFT;
1698 unsigned long end_index = end >> PAGE_CACHE_SHIFT;
1699 unsigned long nr_pages = end_index - index + 1;
1700 int i;
David Sterba41074882013-04-29 13:38:46 +00001701 unsigned long clear_bits = 0;
Chris Masonc8b97812008-10-29 14:49:59 -04001702
Chris Masona791e352009-10-08 11:27:10 -04001703 if (op & EXTENT_CLEAR_UNLOCK)
Chris Mason771ed682008-11-06 22:02:51 -05001704 clear_bits |= EXTENT_LOCKED;
Chris Masona791e352009-10-08 11:27:10 -04001705 if (op & EXTENT_CLEAR_DIRTY)
Chris Masonc8b97812008-10-29 14:49:59 -04001706 clear_bits |= EXTENT_DIRTY;
1707
Chris Masona791e352009-10-08 11:27:10 -04001708 if (op & EXTENT_CLEAR_DELALLOC)
Chris Mason771ed682008-11-06 22:02:51 -05001709 clear_bits |= EXTENT_DELALLOC;
1710
Chris Mason2c64c532009-09-02 15:04:12 -04001711 clear_extent_bit(tree, start, end, clear_bits, 1, 0, NULL, GFP_NOFS);
Josef Bacik32c00af2009-10-08 13:34:05 -04001712 if (!(op & (EXTENT_CLEAR_UNLOCK_PAGE | EXTENT_CLEAR_DIRTY |
1713 EXTENT_SET_WRITEBACK | EXTENT_END_WRITEBACK |
1714 EXTENT_SET_PRIVATE2)))
Chris Mason771ed682008-11-06 22:02:51 -05001715 return 0;
Chris Masonc8b97812008-10-29 14:49:59 -04001716
Chris Masond3977122009-01-05 21:25:51 -05001717 while (nr_pages > 0) {
Chris Masonc8b97812008-10-29 14:49:59 -04001718 ret = find_get_pages_contig(inode->i_mapping, index,
Chris Mason5b050f02008-11-11 09:34:41 -05001719 min_t(unsigned long,
1720 nr_pages, ARRAY_SIZE(pages)), pages);
Chris Masonc8b97812008-10-29 14:49:59 -04001721 for (i = 0; i < ret; i++) {
Chris Mason8b62b722009-09-02 16:53:46 -04001722
Chris Masona791e352009-10-08 11:27:10 -04001723 if (op & EXTENT_SET_PRIVATE2)
Chris Mason8b62b722009-09-02 16:53:46 -04001724 SetPagePrivate2(pages[i]);
1725
Chris Masonc8b97812008-10-29 14:49:59 -04001726 if (pages[i] == locked_page) {
1727 page_cache_release(pages[i]);
1728 continue;
1729 }
Chris Masona791e352009-10-08 11:27:10 -04001730 if (op & EXTENT_CLEAR_DIRTY)
Chris Masonc8b97812008-10-29 14:49:59 -04001731 clear_page_dirty_for_io(pages[i]);
Chris Masona791e352009-10-08 11:27:10 -04001732 if (op & EXTENT_SET_WRITEBACK)
Chris Masonc8b97812008-10-29 14:49:59 -04001733 set_page_writeback(pages[i]);
Chris Masona791e352009-10-08 11:27:10 -04001734 if (op & EXTENT_END_WRITEBACK)
Chris Masonc8b97812008-10-29 14:49:59 -04001735 end_page_writeback(pages[i]);
Chris Masona791e352009-10-08 11:27:10 -04001736 if (op & EXTENT_CLEAR_UNLOCK_PAGE)
Chris Mason771ed682008-11-06 22:02:51 -05001737 unlock_page(pages[i]);
Chris Masonc8b97812008-10-29 14:49:59 -04001738 page_cache_release(pages[i]);
1739 }
1740 nr_pages -= ret;
1741 index += ret;
1742 cond_resched();
1743 }
1744 return 0;
1745}
Chris Masonc8b97812008-10-29 14:49:59 -04001746
Chris Masond352ac62008-09-29 15:18:18 -04001747/*
1748 * count the number of bytes in the tree that have a given bit(s)
1749 * set. This can be fairly slow, except for EXTENT_DIRTY which is
1750 * cached. The total number found is returned.
1751 */
Chris Masond1310b22008-01-24 16:13:08 -05001752u64 count_range_bits(struct extent_io_tree *tree,
1753 u64 *start, u64 search_end, u64 max_bytes,
Chris Masonec29ed52011-02-23 16:23:20 -05001754 unsigned long bits, int contig)
Chris Masond1310b22008-01-24 16:13:08 -05001755{
1756 struct rb_node *node;
1757 struct extent_state *state;
1758 u64 cur_start = *start;
1759 u64 total_bytes = 0;
Chris Masonec29ed52011-02-23 16:23:20 -05001760 u64 last = 0;
Chris Masond1310b22008-01-24 16:13:08 -05001761 int found = 0;
1762
1763 if (search_end <= cur_start) {
Chris Masond1310b22008-01-24 16:13:08 -05001764 WARN_ON(1);
1765 return 0;
1766 }
1767
Chris Masoncad321a2008-12-17 14:51:42 -05001768 spin_lock(&tree->lock);
Chris Masond1310b22008-01-24 16:13:08 -05001769 if (cur_start == 0 && bits == EXTENT_DIRTY) {
1770 total_bytes = tree->dirty_bytes;
1771 goto out;
1772 }
1773 /*
1774 * this search will find all the extents that end after
1775 * our range starts.
1776 */
Chris Mason80ea96b2008-02-01 14:51:59 -05001777 node = tree_search(tree, cur_start);
Chris Masond3977122009-01-05 21:25:51 -05001778 if (!node)
Chris Masond1310b22008-01-24 16:13:08 -05001779 goto out;
Chris Masond1310b22008-01-24 16:13:08 -05001780
Chris Masond3977122009-01-05 21:25:51 -05001781 while (1) {
Chris Masond1310b22008-01-24 16:13:08 -05001782 state = rb_entry(node, struct extent_state, rb_node);
1783 if (state->start > search_end)
1784 break;
Chris Masonec29ed52011-02-23 16:23:20 -05001785 if (contig && found && state->start > last + 1)
1786 break;
1787 if (state->end >= cur_start && (state->state & bits) == bits) {
Chris Masond1310b22008-01-24 16:13:08 -05001788 total_bytes += min(search_end, state->end) + 1 -
1789 max(cur_start, state->start);
1790 if (total_bytes >= max_bytes)
1791 break;
1792 if (!found) {
Josef Bacikaf60bed2011-05-04 11:11:17 -04001793 *start = max(cur_start, state->start);
Chris Masond1310b22008-01-24 16:13:08 -05001794 found = 1;
1795 }
Chris Masonec29ed52011-02-23 16:23:20 -05001796 last = state->end;
1797 } else if (contig && found) {
1798 break;
Chris Masond1310b22008-01-24 16:13:08 -05001799 }
1800 node = rb_next(node);
1801 if (!node)
1802 break;
1803 }
1804out:
Chris Masoncad321a2008-12-17 14:51:42 -05001805 spin_unlock(&tree->lock);
Chris Masond1310b22008-01-24 16:13:08 -05001806 return total_bytes;
1807}
Christoph Hellwigb2950862008-12-02 09:54:17 -05001808
Chris Masond352ac62008-09-29 15:18:18 -04001809/*
1810 * set the private field for a given byte offset in the tree. If there isn't
1811 * an extent_state there already, this does nothing.
1812 */
Chris Masond1310b22008-01-24 16:13:08 -05001813int set_state_private(struct extent_io_tree *tree, u64 start, u64 private)
1814{
1815 struct rb_node *node;
1816 struct extent_state *state;
1817 int ret = 0;
1818
Chris Masoncad321a2008-12-17 14:51:42 -05001819 spin_lock(&tree->lock);
Chris Masond1310b22008-01-24 16:13:08 -05001820 /*
1821 * this search will find all the extents that end after
1822 * our range starts.
1823 */
Chris Mason80ea96b2008-02-01 14:51:59 -05001824 node = tree_search(tree, start);
Peter2b114d12008-04-01 11:21:40 -04001825 if (!node) {
Chris Masond1310b22008-01-24 16:13:08 -05001826 ret = -ENOENT;
1827 goto out;
1828 }
1829 state = rb_entry(node, struct extent_state, rb_node);
1830 if (state->start != start) {
1831 ret = -ENOENT;
1832 goto out;
1833 }
1834 state->private = private;
1835out:
Chris Masoncad321a2008-12-17 14:51:42 -05001836 spin_unlock(&tree->lock);
Chris Masond1310b22008-01-24 16:13:08 -05001837 return ret;
1838}
1839
Miao Xiee4100d92013-04-05 07:20:56 +00001840void extent_cache_csums_dio(struct extent_io_tree *tree, u64 start, u32 csums[],
1841 int count)
1842{
1843 struct rb_node *node;
1844 struct extent_state *state;
1845
1846 spin_lock(&tree->lock);
1847 /*
1848 * this search will find all the extents that end after
1849 * our range starts.
1850 */
1851 node = tree_search(tree, start);
1852 BUG_ON(!node);
1853
1854 state = rb_entry(node, struct extent_state, rb_node);
1855 BUG_ON(state->start != start);
1856
1857 while (count) {
1858 state->private = *csums++;
1859 count--;
1860 state = next_state(state);
1861 }
1862 spin_unlock(&tree->lock);
1863}
1864
1865static inline u64 __btrfs_get_bio_offset(struct bio *bio, int bio_index)
1866{
1867 struct bio_vec *bvec = bio->bi_io_vec + bio_index;
1868
1869 return page_offset(bvec->bv_page) + bvec->bv_offset;
1870}
1871
1872void extent_cache_csums(struct extent_io_tree *tree, struct bio *bio, int bio_index,
1873 u32 csums[], int count)
1874{
1875 struct rb_node *node;
1876 struct extent_state *state = NULL;
1877 u64 start;
1878
1879 spin_lock(&tree->lock);
1880 do {
1881 start = __btrfs_get_bio_offset(bio, bio_index);
1882 if (state == NULL || state->start != start) {
1883 node = tree_search(tree, start);
1884 BUG_ON(!node);
1885
1886 state = rb_entry(node, struct extent_state, rb_node);
1887 BUG_ON(state->start != start);
1888 }
1889 state->private = *csums++;
1890 count--;
1891 bio_index++;
1892
1893 state = next_state(state);
1894 } while (count);
1895 spin_unlock(&tree->lock);
1896}
1897
Chris Masond1310b22008-01-24 16:13:08 -05001898int get_state_private(struct extent_io_tree *tree, u64 start, u64 *private)
1899{
1900 struct rb_node *node;
1901 struct extent_state *state;
1902 int ret = 0;
1903
Chris Masoncad321a2008-12-17 14:51:42 -05001904 spin_lock(&tree->lock);
Chris Masond1310b22008-01-24 16:13:08 -05001905 /*
1906 * this search will find all the extents that end after
1907 * our range starts.
1908 */
Chris Mason80ea96b2008-02-01 14:51:59 -05001909 node = tree_search(tree, start);
Peter2b114d12008-04-01 11:21:40 -04001910 if (!node) {
Chris Masond1310b22008-01-24 16:13:08 -05001911 ret = -ENOENT;
1912 goto out;
1913 }
1914 state = rb_entry(node, struct extent_state, rb_node);
1915 if (state->start != start) {
1916 ret = -ENOENT;
1917 goto out;
1918 }
1919 *private = state->private;
1920out:
Chris Masoncad321a2008-12-17 14:51:42 -05001921 spin_unlock(&tree->lock);
Chris Masond1310b22008-01-24 16:13:08 -05001922 return ret;
1923}
1924
1925/*
1926 * searches a range in the state tree for a given mask.
Chris Mason70dec802008-01-29 09:59:12 -05001927 * If 'filled' == 1, this returns 1 only if every extent in the tree
Chris Masond1310b22008-01-24 16:13:08 -05001928 * has the bits set. Otherwise, 1 is returned if any bit in the
1929 * range is found set.
1930 */
1931int test_range_bit(struct extent_io_tree *tree, u64 start, u64 end,
David Sterba41074882013-04-29 13:38:46 +00001932 unsigned long bits, int filled, struct extent_state *cached)
Chris Masond1310b22008-01-24 16:13:08 -05001933{
1934 struct extent_state *state = NULL;
1935 struct rb_node *node;
1936 int bitset = 0;
Chris Masond1310b22008-01-24 16:13:08 -05001937
Chris Masoncad321a2008-12-17 14:51:42 -05001938 spin_lock(&tree->lock);
Josef Bacikdf98b6e2011-06-20 14:53:48 -04001939 if (cached && cached->tree && cached->start <= start &&
1940 cached->end > start)
Chris Mason9655d292009-09-02 15:22:30 -04001941 node = &cached->rb_node;
1942 else
1943 node = tree_search(tree, start);
Chris Masond1310b22008-01-24 16:13:08 -05001944 while (node && start <= end) {
1945 state = rb_entry(node, struct extent_state, rb_node);
1946
1947 if (filled && state->start > start) {
1948 bitset = 0;
1949 break;
1950 }
1951
1952 if (state->start > end)
1953 break;
1954
1955 if (state->state & bits) {
1956 bitset = 1;
1957 if (!filled)
1958 break;
1959 } else if (filled) {
1960 bitset = 0;
1961 break;
1962 }
Chris Mason46562cec2009-09-23 20:23:16 -04001963
1964 if (state->end == (u64)-1)
1965 break;
1966
Chris Masond1310b22008-01-24 16:13:08 -05001967 start = state->end + 1;
1968 if (start > end)
1969 break;
1970 node = rb_next(node);
1971 if (!node) {
1972 if (filled)
1973 bitset = 0;
1974 break;
1975 }
1976 }
Chris Masoncad321a2008-12-17 14:51:42 -05001977 spin_unlock(&tree->lock);
Chris Masond1310b22008-01-24 16:13:08 -05001978 return bitset;
1979}
Chris Masond1310b22008-01-24 16:13:08 -05001980
1981/*
1982 * helper function to set a given page up to date if all the
1983 * extents in the tree for that page are up to date
1984 */
Jeff Mahoney143bede2012-03-01 14:56:26 +01001985static void check_page_uptodate(struct extent_io_tree *tree, struct page *page)
Chris Masond1310b22008-01-24 16:13:08 -05001986{
Miao Xie4eee4fa2012-12-21 09:17:45 +00001987 u64 start = page_offset(page);
Chris Masond1310b22008-01-24 16:13:08 -05001988 u64 end = start + PAGE_CACHE_SIZE - 1;
Chris Mason9655d292009-09-02 15:22:30 -04001989 if (test_range_bit(tree, start, end, EXTENT_UPTODATE, 1, NULL))
Chris Masond1310b22008-01-24 16:13:08 -05001990 SetPageUptodate(page);
Chris Masond1310b22008-01-24 16:13:08 -05001991}
1992
1993/*
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02001994 * When IO fails, either with EIO or csum verification fails, we
1995 * try other mirrors that might have a good copy of the data. This
1996 * io_failure_record is used to record state as we go through all the
1997 * mirrors. If another mirror has good data, the page is set up to date
1998 * and things continue. If a good mirror can't be found, the original
1999 * bio end_io callback is called to indicate things have failed.
2000 */
2001struct io_failure_record {
2002 struct page *page;
2003 u64 start;
2004 u64 len;
2005 u64 logical;
2006 unsigned long bio_flags;
2007 int this_mirror;
2008 int failed_mirror;
2009 int in_validation;
2010};
2011
2012static int free_io_failure(struct inode *inode, struct io_failure_record *rec,
2013 int did_repair)
2014{
2015 int ret;
2016 int err = 0;
2017 struct extent_io_tree *failure_tree = &BTRFS_I(inode)->io_failure_tree;
2018
2019 set_state_private(failure_tree, rec->start, 0);
2020 ret = clear_extent_bits(failure_tree, rec->start,
2021 rec->start + rec->len - 1,
2022 EXTENT_LOCKED | EXTENT_DIRTY, GFP_NOFS);
2023 if (ret)
2024 err = ret;
2025
David Woodhouse53b381b2013-01-29 18:40:14 -05002026 ret = clear_extent_bits(&BTRFS_I(inode)->io_tree, rec->start,
2027 rec->start + rec->len - 1,
2028 EXTENT_DAMAGED, GFP_NOFS);
2029 if (ret && !err)
2030 err = ret;
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002031
2032 kfree(rec);
2033 return err;
2034}
2035
2036static void repair_io_failure_callback(struct bio *bio, int err)
2037{
2038 complete(bio->bi_private);
2039}
2040
2041/*
2042 * this bypasses the standard btrfs submit functions deliberately, as
2043 * the standard behavior is to write all copies in a raid setup. here we only
2044 * want to write the one bad copy. so we do the mapping for ourselves and issue
2045 * submit_bio directly.
Stefan Behrens3ec706c2012-11-05 15:46:42 +01002046 * to avoid any synchronization issues, wait for the data after writing, which
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002047 * actually prevents the read that triggered the error from finishing.
2048 * currently, there can be no more than two copies of every data bit. thus,
2049 * exactly one rewrite is required.
2050 */
Stefan Behrens3ec706c2012-11-05 15:46:42 +01002051int repair_io_failure(struct btrfs_fs_info *fs_info, u64 start,
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002052 u64 length, u64 logical, struct page *page,
2053 int mirror_num)
2054{
2055 struct bio *bio;
2056 struct btrfs_device *dev;
2057 DECLARE_COMPLETION_ONSTACK(compl);
2058 u64 map_length = 0;
2059 u64 sector;
2060 struct btrfs_bio *bbio = NULL;
David Woodhouse53b381b2013-01-29 18:40:14 -05002061 struct btrfs_mapping_tree *map_tree = &fs_info->mapping_tree;
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002062 int ret;
2063
2064 BUG_ON(!mirror_num);
2065
David Woodhouse53b381b2013-01-29 18:40:14 -05002066 /* we can't repair anything in raid56 yet */
2067 if (btrfs_is_parity_mirror(map_tree, logical, length, mirror_num))
2068 return 0;
2069
Chris Mason9be33952013-05-17 18:30:14 -04002070 bio = btrfs_io_bio_alloc(GFP_NOFS, 1);
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002071 if (!bio)
2072 return -EIO;
2073 bio->bi_private = &compl;
2074 bio->bi_end_io = repair_io_failure_callback;
2075 bio->bi_size = 0;
2076 map_length = length;
2077
Stefan Behrens3ec706c2012-11-05 15:46:42 +01002078 ret = btrfs_map_block(fs_info, WRITE, logical,
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002079 &map_length, &bbio, mirror_num);
2080 if (ret) {
2081 bio_put(bio);
2082 return -EIO;
2083 }
2084 BUG_ON(mirror_num != bbio->mirror_num);
2085 sector = bbio->stripes[mirror_num-1].physical >> 9;
2086 bio->bi_sector = sector;
2087 dev = bbio->stripes[mirror_num-1].dev;
2088 kfree(bbio);
2089 if (!dev || !dev->bdev || !dev->writeable) {
2090 bio_put(bio);
2091 return -EIO;
2092 }
2093 bio->bi_bdev = dev->bdev;
Miao Xie4eee4fa2012-12-21 09:17:45 +00002094 bio_add_page(bio, page, length, start - page_offset(page));
Stefan Behrens21adbd52011-11-09 13:44:05 +01002095 btrfsic_submit_bio(WRITE_SYNC, bio);
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002096 wait_for_completion(&compl);
2097
2098 if (!test_bit(BIO_UPTODATE, &bio->bi_flags)) {
2099 /* try to remap that extent elsewhere? */
2100 bio_put(bio);
Stefan Behrens442a4f62012-05-25 16:06:08 +02002101 btrfs_dev_stat_inc_and_print(dev, BTRFS_DEV_STAT_WRITE_ERRS);
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002102 return -EIO;
2103 }
2104
Anand Jaind5b025d2012-07-02 22:05:21 -06002105 printk_ratelimited_in_rcu(KERN_INFO "btrfs read error corrected: ino %lu off %llu "
Josef Bacik606686e2012-06-04 14:03:51 -04002106 "(dev %s sector %llu)\n", page->mapping->host->i_ino,
2107 start, rcu_str_deref(dev->name), sector);
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002108
2109 bio_put(bio);
2110 return 0;
2111}
2112
Josef Bacikea466792012-03-26 21:57:36 -04002113int repair_eb_io_failure(struct btrfs_root *root, struct extent_buffer *eb,
2114 int mirror_num)
2115{
Josef Bacikea466792012-03-26 21:57:36 -04002116 u64 start = eb->start;
2117 unsigned long i, num_pages = num_extent_pages(eb->start, eb->len);
Chris Masond95603b2012-04-12 15:55:15 -04002118 int ret = 0;
Josef Bacikea466792012-03-26 21:57:36 -04002119
2120 for (i = 0; i < num_pages; i++) {
2121 struct page *p = extent_buffer_page(eb, i);
Stefan Behrens3ec706c2012-11-05 15:46:42 +01002122 ret = repair_io_failure(root->fs_info, start, PAGE_CACHE_SIZE,
Josef Bacikea466792012-03-26 21:57:36 -04002123 start, p, mirror_num);
2124 if (ret)
2125 break;
2126 start += PAGE_CACHE_SIZE;
2127 }
2128
2129 return ret;
2130}
2131
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002132/*
2133 * each time an IO finishes, we do a fast check in the IO failure tree
2134 * to see if we need to process or clean up an io_failure_record
2135 */
2136static int clean_io_failure(u64 start, struct page *page)
2137{
2138 u64 private;
2139 u64 private_failure;
2140 struct io_failure_record *failrec;
Stefan Behrens3ec706c2012-11-05 15:46:42 +01002141 struct btrfs_fs_info *fs_info;
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002142 struct extent_state *state;
2143 int num_copies;
2144 int did_repair = 0;
2145 int ret;
2146 struct inode *inode = page->mapping->host;
2147
2148 private = 0;
2149 ret = count_range_bits(&BTRFS_I(inode)->io_failure_tree, &private,
2150 (u64)-1, 1, EXTENT_DIRTY, 0);
2151 if (!ret)
2152 return 0;
2153
2154 ret = get_state_private(&BTRFS_I(inode)->io_failure_tree, start,
2155 &private_failure);
2156 if (ret)
2157 return 0;
2158
2159 failrec = (struct io_failure_record *)(unsigned long) private_failure;
2160 BUG_ON(!failrec->this_mirror);
2161
2162 if (failrec->in_validation) {
2163 /* there was no real error, just free the record */
2164 pr_debug("clean_io_failure: freeing dummy error at %llu\n",
2165 failrec->start);
2166 did_repair = 1;
2167 goto out;
2168 }
2169
2170 spin_lock(&BTRFS_I(inode)->io_tree.lock);
2171 state = find_first_extent_bit_state(&BTRFS_I(inode)->io_tree,
2172 failrec->start,
2173 EXTENT_LOCKED);
2174 spin_unlock(&BTRFS_I(inode)->io_tree.lock);
2175
2176 if (state && state->start == failrec->start) {
Stefan Behrens3ec706c2012-11-05 15:46:42 +01002177 fs_info = BTRFS_I(inode)->root->fs_info;
2178 num_copies = btrfs_num_copies(fs_info, failrec->logical,
2179 failrec->len);
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002180 if (num_copies > 1) {
Stefan Behrens3ec706c2012-11-05 15:46:42 +01002181 ret = repair_io_failure(fs_info, start, failrec->len,
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002182 failrec->logical, page,
2183 failrec->failed_mirror);
2184 did_repair = !ret;
2185 }
David Woodhouse53b381b2013-01-29 18:40:14 -05002186 ret = 0;
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002187 }
2188
2189out:
2190 if (!ret)
2191 ret = free_io_failure(inode, failrec, did_repair);
2192
2193 return ret;
2194}
2195
2196/*
2197 * this is a generic handler for readpage errors (default
2198 * readpage_io_failed_hook). if other copies exist, read those and write back
2199 * good data to the failed position. does not investigate in remapping the
2200 * failed extent elsewhere, hoping the device will be smart enough to do this as
2201 * needed
2202 */
2203
2204static int bio_readpage_error(struct bio *failed_bio, struct page *page,
2205 u64 start, u64 end, int failed_mirror,
2206 struct extent_state *state)
2207{
2208 struct io_failure_record *failrec = NULL;
2209 u64 private;
2210 struct extent_map *em;
2211 struct inode *inode = page->mapping->host;
2212 struct extent_io_tree *failure_tree = &BTRFS_I(inode)->io_failure_tree;
2213 struct extent_io_tree *tree = &BTRFS_I(inode)->io_tree;
2214 struct extent_map_tree *em_tree = &BTRFS_I(inode)->extent_tree;
2215 struct bio *bio;
2216 int num_copies;
2217 int ret;
2218 int read_mode;
2219 u64 logical;
2220
2221 BUG_ON(failed_bio->bi_rw & REQ_WRITE);
2222
2223 ret = get_state_private(failure_tree, start, &private);
2224 if (ret) {
2225 failrec = kzalloc(sizeof(*failrec), GFP_NOFS);
2226 if (!failrec)
2227 return -ENOMEM;
2228 failrec->start = start;
2229 failrec->len = end - start + 1;
2230 failrec->this_mirror = 0;
2231 failrec->bio_flags = 0;
2232 failrec->in_validation = 0;
2233
2234 read_lock(&em_tree->lock);
2235 em = lookup_extent_mapping(em_tree, start, failrec->len);
2236 if (!em) {
2237 read_unlock(&em_tree->lock);
2238 kfree(failrec);
2239 return -EIO;
2240 }
2241
2242 if (em->start > start || em->start + em->len < start) {
2243 free_extent_map(em);
2244 em = NULL;
2245 }
2246 read_unlock(&em_tree->lock);
2247
Tsutomu Itoh7a2d6a62012-10-01 03:07:15 -06002248 if (!em) {
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002249 kfree(failrec);
2250 return -EIO;
2251 }
2252 logical = start - em->start;
2253 logical = em->block_start + logical;
2254 if (test_bit(EXTENT_FLAG_COMPRESSED, &em->flags)) {
2255 logical = em->block_start;
2256 failrec->bio_flags = EXTENT_BIO_COMPRESSED;
2257 extent_set_compress_type(&failrec->bio_flags,
2258 em->compress_type);
2259 }
2260 pr_debug("bio_readpage_error: (new) logical=%llu, start=%llu, "
2261 "len=%llu\n", logical, start, failrec->len);
2262 failrec->logical = logical;
2263 free_extent_map(em);
2264
2265 /* set the bits in the private failure tree */
2266 ret = set_extent_bits(failure_tree, start, end,
2267 EXTENT_LOCKED | EXTENT_DIRTY, GFP_NOFS);
2268 if (ret >= 0)
2269 ret = set_state_private(failure_tree, start,
2270 (u64)(unsigned long)failrec);
2271 /* set the bits in the inode's tree */
2272 if (ret >= 0)
2273 ret = set_extent_bits(tree, start, end, EXTENT_DAMAGED,
2274 GFP_NOFS);
2275 if (ret < 0) {
2276 kfree(failrec);
2277 return ret;
2278 }
2279 } else {
2280 failrec = (struct io_failure_record *)(unsigned long)private;
2281 pr_debug("bio_readpage_error: (found) logical=%llu, "
2282 "start=%llu, len=%llu, validation=%d\n",
2283 failrec->logical, failrec->start, failrec->len,
2284 failrec->in_validation);
2285 /*
2286 * when data can be on disk more than twice, add to failrec here
2287 * (e.g. with a list for failed_mirror) to make
2288 * clean_io_failure() clean all those errors at once.
2289 */
2290 }
Stefan Behrens5d964052012-11-05 14:59:07 +01002291 num_copies = btrfs_num_copies(BTRFS_I(inode)->root->fs_info,
2292 failrec->logical, failrec->len);
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002293 if (num_copies == 1) {
2294 /*
2295 * we only have a single copy of the data, so don't bother with
2296 * all the retry and error correction code that follows. no
2297 * matter what the error is, it is very likely to persist.
2298 */
2299 pr_debug("bio_readpage_error: cannot repair, num_copies == 1. "
2300 "state=%p, num_copies=%d, next_mirror %d, "
2301 "failed_mirror %d\n", state, num_copies,
2302 failrec->this_mirror, failed_mirror);
2303 free_io_failure(inode, failrec, 0);
2304 return -EIO;
2305 }
2306
2307 if (!state) {
2308 spin_lock(&tree->lock);
2309 state = find_first_extent_bit_state(tree, failrec->start,
2310 EXTENT_LOCKED);
2311 if (state && state->start != failrec->start)
2312 state = NULL;
2313 spin_unlock(&tree->lock);
2314 }
2315
2316 /*
2317 * there are two premises:
2318 * a) deliver good data to the caller
2319 * b) correct the bad sectors on disk
2320 */
2321 if (failed_bio->bi_vcnt > 1) {
2322 /*
2323 * to fulfill b), we need to know the exact failing sectors, as
2324 * we don't want to rewrite any more than the failed ones. thus,
2325 * we need separate read requests for the failed bio
2326 *
2327 * if the following BUG_ON triggers, our validation request got
2328 * merged. we need separate requests for our algorithm to work.
2329 */
2330 BUG_ON(failrec->in_validation);
2331 failrec->in_validation = 1;
2332 failrec->this_mirror = failed_mirror;
2333 read_mode = READ_SYNC | REQ_FAILFAST_DEV;
2334 } else {
2335 /*
2336 * we're ready to fulfill a) and b) alongside. get a good copy
2337 * of the failed sector and if we succeed, we have setup
2338 * everything for repair_io_failure to do the rest for us.
2339 */
2340 if (failrec->in_validation) {
2341 BUG_ON(failrec->this_mirror != failed_mirror);
2342 failrec->in_validation = 0;
2343 failrec->this_mirror = 0;
2344 }
2345 failrec->failed_mirror = failed_mirror;
2346 failrec->this_mirror++;
2347 if (failrec->this_mirror == failed_mirror)
2348 failrec->this_mirror++;
2349 read_mode = READ_SYNC;
2350 }
2351
2352 if (!state || failrec->this_mirror > num_copies) {
2353 pr_debug("bio_readpage_error: (fail) state=%p, num_copies=%d, "
2354 "next_mirror %d, failed_mirror %d\n", state,
2355 num_copies, failrec->this_mirror, failed_mirror);
2356 free_io_failure(inode, failrec, 0);
2357 return -EIO;
2358 }
2359
Chris Mason9be33952013-05-17 18:30:14 -04002360 bio = btrfs_io_bio_alloc(GFP_NOFS, 1);
Tsutomu Itohe627ee72012-04-12 16:03:56 -04002361 if (!bio) {
2362 free_io_failure(inode, failrec, 0);
2363 return -EIO;
2364 }
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002365 bio->bi_private = state;
2366 bio->bi_end_io = failed_bio->bi_end_io;
2367 bio->bi_sector = failrec->logical >> 9;
2368 bio->bi_bdev = BTRFS_I(inode)->root->fs_info->fs_devices->latest_bdev;
2369 bio->bi_size = 0;
2370
2371 bio_add_page(bio, page, failrec->len, start - page_offset(page));
2372
2373 pr_debug("bio_readpage_error: submitting new read[%#x] to "
2374 "this_mirror=%d, num_copies=%d, in_validation=%d\n", read_mode,
2375 failrec->this_mirror, num_copies, failrec->in_validation);
2376
Tsutomu Itoh013bd4c2012-02-16 10:11:40 +09002377 ret = tree->ops->submit_bio_hook(inode, read_mode, bio,
2378 failrec->this_mirror,
2379 failrec->bio_flags, 0);
2380 return ret;
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002381}
2382
Chris Masond1310b22008-01-24 16:13:08 -05002383/* lots and lots of room for performance fixes in the end_bio funcs */
2384
Jeff Mahoney87826df2012-02-15 16:23:57 +01002385int end_extent_writepage(struct page *page, int err, u64 start, u64 end)
2386{
2387 int uptodate = (err == 0);
2388 struct extent_io_tree *tree;
2389 int ret;
2390
2391 tree = &BTRFS_I(page->mapping->host)->io_tree;
2392
2393 if (tree->ops && tree->ops->writepage_end_io_hook) {
2394 ret = tree->ops->writepage_end_io_hook(page, start,
2395 end, NULL, uptodate);
2396 if (ret)
2397 uptodate = 0;
2398 }
2399
Jeff Mahoney87826df2012-02-15 16:23:57 +01002400 if (!uptodate) {
Jeff Mahoney87826df2012-02-15 16:23:57 +01002401 ClearPageUptodate(page);
2402 SetPageError(page);
2403 }
2404 return 0;
2405}
2406
Chris Masond1310b22008-01-24 16:13:08 -05002407/*
2408 * after a writepage IO is done, we need to:
2409 * clear the uptodate bits on error
2410 * clear the writeback bits in the extent tree for this IO
2411 * end_page_writeback if the page has no more pending IO
2412 *
2413 * Scheduling is not allowed, so the extent state tree is expected
2414 * to have one and only one object corresponding to this IO.
2415 */
Chris Masond1310b22008-01-24 16:13:08 -05002416static void end_bio_extent_writepage(struct bio *bio, int err)
Chris Masond1310b22008-01-24 16:13:08 -05002417{
Chris Masond1310b22008-01-24 16:13:08 -05002418 struct bio_vec *bvec = bio->bi_io_vec + bio->bi_vcnt - 1;
David Woodhouse902b22f2008-08-20 08:51:49 -04002419 struct extent_io_tree *tree;
Chris Masond1310b22008-01-24 16:13:08 -05002420 u64 start;
2421 u64 end;
Chris Masond1310b22008-01-24 16:13:08 -05002422
Chris Masond1310b22008-01-24 16:13:08 -05002423 do {
2424 struct page *page = bvec->bv_page;
David Woodhouse902b22f2008-08-20 08:51:49 -04002425 tree = &BTRFS_I(page->mapping->host)->io_tree;
2426
Alexandre Oliva17a5adc2013-05-15 11:38:55 -04002427 /* We always issue full-page reads, but if some block
2428 * in a page fails to read, blk_update_request() will
2429 * advance bv_offset and adjust bv_len to compensate.
2430 * Print a warning for nonzero offsets, and an error
2431 * if they don't add up to a full page. */
2432 if (bvec->bv_offset || bvec->bv_len != PAGE_CACHE_SIZE)
2433 printk("%s page write in btrfs with offset %u and length %u\n",
2434 bvec->bv_offset + bvec->bv_len != PAGE_CACHE_SIZE
2435 ? KERN_ERR "partial" : KERN_INFO "incomplete",
2436 bvec->bv_offset, bvec->bv_len);
Chris Masond1310b22008-01-24 16:13:08 -05002437
Alexandre Oliva17a5adc2013-05-15 11:38:55 -04002438 start = page_offset(page);
2439 end = start + bvec->bv_offset + bvec->bv_len - 1;
Chris Masond1310b22008-01-24 16:13:08 -05002440
2441 if (--bvec >= bio->bi_io_vec)
2442 prefetchw(&bvec->bv_page->flags);
Chris Mason1259ab72008-05-12 13:39:03 -04002443
Jeff Mahoney87826df2012-02-15 16:23:57 +01002444 if (end_extent_writepage(page, err, start, end))
2445 continue;
Chris Mason70dec802008-01-29 09:59:12 -05002446
Alexandre Oliva17a5adc2013-05-15 11:38:55 -04002447 end_page_writeback(page);
Chris Masond1310b22008-01-24 16:13:08 -05002448 } while (bvec >= bio->bi_io_vec);
Chris Mason2b1f55b2008-09-24 11:48:04 -04002449
Chris Masond1310b22008-01-24 16:13:08 -05002450 bio_put(bio);
Chris Masond1310b22008-01-24 16:13:08 -05002451}
2452
2453/*
2454 * after a readpage IO is done, we need to:
2455 * clear the uptodate bits on error
2456 * set the uptodate bits if things worked
2457 * set the page up to date if all extents in the tree are uptodate
2458 * clear the lock bit in the extent tree
2459 * unlock the page if there are no other extents locked for it
2460 *
2461 * Scheduling is not allowed, so the extent state tree is expected
2462 * to have one and only one object corresponding to this IO.
2463 */
Chris Masond1310b22008-01-24 16:13:08 -05002464static void end_bio_extent_readpage(struct bio *bio, int err)
Chris Masond1310b22008-01-24 16:13:08 -05002465{
2466 int uptodate = test_bit(BIO_UPTODATE, &bio->bi_flags);
Chris Mason4125bf72010-02-03 18:18:45 +00002467 struct bio_vec *bvec_end = bio->bi_io_vec + bio->bi_vcnt - 1;
2468 struct bio_vec *bvec = bio->bi_io_vec;
David Woodhouse902b22f2008-08-20 08:51:49 -04002469 struct extent_io_tree *tree;
Chris Masond1310b22008-01-24 16:13:08 -05002470 u64 start;
2471 u64 end;
Josef Bacik5cf1ab52012-04-16 09:42:26 -04002472 int mirror;
Chris Masond1310b22008-01-24 16:13:08 -05002473 int ret;
2474
Chris Masond20f7042008-12-08 16:58:54 -05002475 if (err)
2476 uptodate = 0;
2477
Chris Masond1310b22008-01-24 16:13:08 -05002478 do {
2479 struct page *page = bvec->bv_page;
Arne Jansen507903b2011-04-06 10:02:20 +00002480 struct extent_state *cached = NULL;
2481 struct extent_state *state;
Chris Mason9be33952013-05-17 18:30:14 -04002482 struct btrfs_io_bio *io_bio = btrfs_io_bio(bio);
Josef Bacika71754f2013-06-17 17:14:39 -04002483 struct inode *inode = page->mapping->host;
Arne Jansen507903b2011-04-06 10:02:20 +00002484
Kent Overstreetbe3940c2012-09-11 14:23:05 -06002485 pr_debug("end_bio_extent_readpage: bi_sector=%llu, err=%d, "
Chris Mason9be33952013-05-17 18:30:14 -04002486 "mirror=%lu\n", (u64)bio->bi_sector, err,
2487 io_bio->mirror_num);
Josef Bacika71754f2013-06-17 17:14:39 -04002488 tree = &BTRFS_I(inode)->io_tree;
David Woodhouse902b22f2008-08-20 08:51:49 -04002489
Alexandre Oliva17a5adc2013-05-15 11:38:55 -04002490 /* We always issue full-page reads, but if some block
2491 * in a page fails to read, blk_update_request() will
2492 * advance bv_offset and adjust bv_len to compensate.
2493 * Print a warning for nonzero offsets, and an error
2494 * if they don't add up to a full page. */
2495 if (bvec->bv_offset || bvec->bv_len != PAGE_CACHE_SIZE)
2496 printk("%s page read in btrfs with offset %u and length %u\n",
2497 bvec->bv_offset + bvec->bv_len != PAGE_CACHE_SIZE
2498 ? KERN_ERR "partial" : KERN_INFO "incomplete",
2499 bvec->bv_offset, bvec->bv_len);
Chris Masond1310b22008-01-24 16:13:08 -05002500
Alexandre Oliva17a5adc2013-05-15 11:38:55 -04002501 start = page_offset(page);
2502 end = start + bvec->bv_offset + bvec->bv_len - 1;
Chris Masond1310b22008-01-24 16:13:08 -05002503
Chris Mason4125bf72010-02-03 18:18:45 +00002504 if (++bvec <= bvec_end)
Chris Masond1310b22008-01-24 16:13:08 -05002505 prefetchw(&bvec->bv_page->flags);
2506
Arne Jansen507903b2011-04-06 10:02:20 +00002507 spin_lock(&tree->lock);
Chris Mason0d399202011-04-16 06:55:39 -04002508 state = find_first_extent_bit_state(tree, start, EXTENT_LOCKED);
Chris Mason109b36a2011-04-12 13:57:39 -04002509 if (state && state->start == start) {
Arne Jansen507903b2011-04-06 10:02:20 +00002510 /*
2511 * take a reference on the state, unlock will drop
2512 * the ref
2513 */
2514 cache_state(state, &cached);
2515 }
2516 spin_unlock(&tree->lock);
2517
Chris Mason9be33952013-05-17 18:30:14 -04002518 mirror = io_bio->mirror_num;
Chris Masond1310b22008-01-24 16:13:08 -05002519 if (uptodate && tree->ops && tree->ops->readpage_end_io_hook) {
Chris Mason70dec802008-01-29 09:59:12 -05002520 ret = tree->ops->readpage_end_io_hook(page, start, end,
Josef Bacik5cf1ab52012-04-16 09:42:26 -04002521 state, mirror);
Stefan Behrens5ee08442012-08-27 08:30:03 -06002522 if (ret)
Chris Masond1310b22008-01-24 16:13:08 -05002523 uptodate = 0;
Stefan Behrens5ee08442012-08-27 08:30:03 -06002524 else
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002525 clean_io_failure(start, page);
Chris Masond1310b22008-01-24 16:13:08 -05002526 }
Josef Bacikea466792012-03-26 21:57:36 -04002527
Josef Bacikea466792012-03-26 21:57:36 -04002528 if (!uptodate && tree->ops && tree->ops->readpage_io_failed_hook) {
Josef Bacik5cf1ab52012-04-16 09:42:26 -04002529 ret = tree->ops->readpage_io_failed_hook(page, mirror);
Josef Bacikea466792012-03-26 21:57:36 -04002530 if (!ret && !err &&
2531 test_bit(BIO_UPTODATE, &bio->bi_flags))
2532 uptodate = 1;
2533 } else if (!uptodate) {
Jan Schmidtf4a8e652011-12-01 09:30:36 -05002534 /*
2535 * The generic bio_readpage_error handles errors the
2536 * following way: If possible, new read requests are
2537 * created and submitted and will end up in
2538 * end_bio_extent_readpage as well (if we're lucky, not
2539 * in the !uptodate case). In that case it returns 0 and
2540 * we just go on with the next page in our bio. If it
2541 * can't handle the error it will return -EIO and we
2542 * remain responsible for that page.
2543 */
Josef Bacik5cf1ab52012-04-16 09:42:26 -04002544 ret = bio_readpage_error(bio, page, start, end, mirror, NULL);
Chris Mason7e383262008-04-09 16:28:12 -04002545 if (ret == 0) {
Chris Mason3b951512008-04-17 11:29:12 -04002546 uptodate =
2547 test_bit(BIO_UPTODATE, &bio->bi_flags);
Chris Masond20f7042008-12-08 16:58:54 -05002548 if (err)
2549 uptodate = 0;
Arne Jansen507903b2011-04-06 10:02:20 +00002550 uncache_state(&cached);
Chris Mason7e383262008-04-09 16:28:12 -04002551 continue;
2552 }
2553 }
Chris Mason70dec802008-01-29 09:59:12 -05002554
Josef Bacik0b32f4b2012-03-13 09:38:00 -04002555 if (uptodate && tree->track_uptodate) {
Arne Jansen507903b2011-04-06 10:02:20 +00002556 set_extent_uptodate(tree, start, end, &cached,
David Woodhouse902b22f2008-08-20 08:51:49 -04002557 GFP_ATOMIC);
Chris Mason771ed682008-11-06 22:02:51 -05002558 }
Arne Jansen507903b2011-04-06 10:02:20 +00002559 unlock_extent_cached(tree, start, end, &cached, GFP_ATOMIC);
Chris Masond1310b22008-01-24 16:13:08 -05002560
Alexandre Oliva17a5adc2013-05-15 11:38:55 -04002561 if (uptodate) {
Josef Bacika71754f2013-06-17 17:14:39 -04002562 loff_t i_size = i_size_read(inode);
2563 pgoff_t end_index = i_size >> PAGE_CACHE_SHIFT;
2564 unsigned offset;
2565
2566 /* Zero out the end if this page straddles i_size */
2567 offset = i_size & (PAGE_CACHE_SIZE-1);
2568 if (page->index == end_index && offset)
2569 zero_user_segment(page, offset, PAGE_CACHE_SIZE);
Alexandre Oliva17a5adc2013-05-15 11:38:55 -04002570 SetPageUptodate(page);
Chris Mason70dec802008-01-29 09:59:12 -05002571 } else {
Alexandre Oliva17a5adc2013-05-15 11:38:55 -04002572 ClearPageUptodate(page);
2573 SetPageError(page);
Chris Mason70dec802008-01-29 09:59:12 -05002574 }
Alexandre Oliva17a5adc2013-05-15 11:38:55 -04002575 unlock_page(page);
Chris Mason4125bf72010-02-03 18:18:45 +00002576 } while (bvec <= bvec_end);
Chris Masond1310b22008-01-24 16:13:08 -05002577
2578 bio_put(bio);
Chris Masond1310b22008-01-24 16:13:08 -05002579}
2580
Chris Mason9be33952013-05-17 18:30:14 -04002581/*
2582 * this allocates from the btrfs_bioset. We're returning a bio right now
2583 * but you can call btrfs_io_bio for the appropriate container_of magic
2584 */
Miao Xie88f794e2010-11-22 03:02:55 +00002585struct bio *
2586btrfs_bio_alloc(struct block_device *bdev, u64 first_sector, int nr_vecs,
2587 gfp_t gfp_flags)
Chris Masond1310b22008-01-24 16:13:08 -05002588{
2589 struct bio *bio;
2590
Chris Mason9be33952013-05-17 18:30:14 -04002591 bio = bio_alloc_bioset(gfp_flags, nr_vecs, btrfs_bioset);
Chris Masond1310b22008-01-24 16:13:08 -05002592
2593 if (bio == NULL && (current->flags & PF_MEMALLOC)) {
Chris Mason9be33952013-05-17 18:30:14 -04002594 while (!bio && (nr_vecs /= 2)) {
2595 bio = bio_alloc_bioset(gfp_flags,
2596 nr_vecs, btrfs_bioset);
2597 }
Chris Masond1310b22008-01-24 16:13:08 -05002598 }
2599
2600 if (bio) {
Chris Masone1c4b742008-04-22 13:26:46 -04002601 bio->bi_size = 0;
Chris Masond1310b22008-01-24 16:13:08 -05002602 bio->bi_bdev = bdev;
2603 bio->bi_sector = first_sector;
2604 }
2605 return bio;
2606}
2607
Chris Mason9be33952013-05-17 18:30:14 -04002608struct bio *btrfs_bio_clone(struct bio *bio, gfp_t gfp_mask)
2609{
2610 return bio_clone_bioset(bio, gfp_mask, btrfs_bioset);
2611}
2612
2613
2614/* this also allocates from the btrfs_bioset */
2615struct bio *btrfs_io_bio_alloc(gfp_t gfp_mask, unsigned int nr_iovecs)
2616{
2617 return bio_alloc_bioset(gfp_mask, nr_iovecs, btrfs_bioset);
2618}
2619
2620
Jeff Mahoney355808c2011-10-03 23:23:14 -04002621static int __must_check submit_one_bio(int rw, struct bio *bio,
2622 int mirror_num, unsigned long bio_flags)
Chris Masond1310b22008-01-24 16:13:08 -05002623{
Chris Masond1310b22008-01-24 16:13:08 -05002624 int ret = 0;
Chris Mason70dec802008-01-29 09:59:12 -05002625 struct bio_vec *bvec = bio->bi_io_vec + bio->bi_vcnt - 1;
2626 struct page *page = bvec->bv_page;
2627 struct extent_io_tree *tree = bio->bi_private;
Chris Mason70dec802008-01-29 09:59:12 -05002628 u64 start;
Chris Mason70dec802008-01-29 09:59:12 -05002629
Miao Xie4eee4fa2012-12-21 09:17:45 +00002630 start = page_offset(page) + bvec->bv_offset;
Chris Mason70dec802008-01-29 09:59:12 -05002631
David Woodhouse902b22f2008-08-20 08:51:49 -04002632 bio->bi_private = NULL;
Chris Masond1310b22008-01-24 16:13:08 -05002633
2634 bio_get(bio);
2635
Chris Mason065631f2008-02-20 12:07:25 -05002636 if (tree->ops && tree->ops->submit_bio_hook)
liubo6b82ce82011-01-26 06:21:39 +00002637 ret = tree->ops->submit_bio_hook(page->mapping->host, rw, bio,
Chris Masoneaf25d92010-05-25 09:48:28 -04002638 mirror_num, bio_flags, start);
Chris Mason0b86a832008-03-24 15:01:56 -04002639 else
Stefan Behrens21adbd52011-11-09 13:44:05 +01002640 btrfsic_submit_bio(rw, bio);
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002641
Chris Masond1310b22008-01-24 16:13:08 -05002642 if (bio_flagged(bio, BIO_EOPNOTSUPP))
2643 ret = -EOPNOTSUPP;
2644 bio_put(bio);
2645 return ret;
2646}
2647
David Woodhouse64a16702009-07-15 23:29:37 +01002648static int merge_bio(int rw, struct extent_io_tree *tree, struct page *page,
Jeff Mahoney3444a972011-10-03 23:23:13 -04002649 unsigned long offset, size_t size, struct bio *bio,
2650 unsigned long bio_flags)
2651{
2652 int ret = 0;
2653 if (tree->ops && tree->ops->merge_bio_hook)
David Woodhouse64a16702009-07-15 23:29:37 +01002654 ret = tree->ops->merge_bio_hook(rw, page, offset, size, bio,
Jeff Mahoney3444a972011-10-03 23:23:13 -04002655 bio_flags);
2656 BUG_ON(ret < 0);
2657 return ret;
2658
2659}
2660
Chris Masond1310b22008-01-24 16:13:08 -05002661static int submit_extent_page(int rw, struct extent_io_tree *tree,
2662 struct page *page, sector_t sector,
2663 size_t size, unsigned long offset,
2664 struct block_device *bdev,
2665 struct bio **bio_ret,
2666 unsigned long max_pages,
Chris Masonf1885912008-04-09 16:28:12 -04002667 bio_end_io_t end_io_func,
Chris Masonc8b97812008-10-29 14:49:59 -04002668 int mirror_num,
2669 unsigned long prev_bio_flags,
2670 unsigned long bio_flags)
Chris Masond1310b22008-01-24 16:13:08 -05002671{
2672 int ret = 0;
2673 struct bio *bio;
2674 int nr;
Chris Masonc8b97812008-10-29 14:49:59 -04002675 int contig = 0;
2676 int this_compressed = bio_flags & EXTENT_BIO_COMPRESSED;
2677 int old_compressed = prev_bio_flags & EXTENT_BIO_COMPRESSED;
Chris Mason5b050f02008-11-11 09:34:41 -05002678 size_t page_size = min_t(size_t, size, PAGE_CACHE_SIZE);
Chris Masond1310b22008-01-24 16:13:08 -05002679
2680 if (bio_ret && *bio_ret) {
2681 bio = *bio_ret;
Chris Masonc8b97812008-10-29 14:49:59 -04002682 if (old_compressed)
2683 contig = bio->bi_sector == sector;
2684 else
Kent Overstreetf73a1c72012-09-25 15:05:12 -07002685 contig = bio_end_sector(bio) == sector;
Chris Masonc8b97812008-10-29 14:49:59 -04002686
2687 if (prev_bio_flags != bio_flags || !contig ||
David Woodhouse64a16702009-07-15 23:29:37 +01002688 merge_bio(rw, tree, page, offset, page_size, bio, bio_flags) ||
Chris Masonc8b97812008-10-29 14:49:59 -04002689 bio_add_page(bio, page, page_size, offset) < page_size) {
2690 ret = submit_one_bio(rw, bio, mirror_num,
2691 prev_bio_flags);
Jeff Mahoney79787ea2012-03-12 16:03:00 +01002692 if (ret < 0)
2693 return ret;
Chris Masond1310b22008-01-24 16:13:08 -05002694 bio = NULL;
2695 } else {
2696 return 0;
2697 }
2698 }
Chris Masonc8b97812008-10-29 14:49:59 -04002699 if (this_compressed)
2700 nr = BIO_MAX_PAGES;
2701 else
2702 nr = bio_get_nr_vecs(bdev);
2703
Miao Xie88f794e2010-11-22 03:02:55 +00002704 bio = btrfs_bio_alloc(bdev, sector, nr, GFP_NOFS | __GFP_HIGH);
Tsutomu Itoh5df67082011-02-01 09:17:35 +00002705 if (!bio)
2706 return -ENOMEM;
Chris Mason70dec802008-01-29 09:59:12 -05002707
Chris Masonc8b97812008-10-29 14:49:59 -04002708 bio_add_page(bio, page, page_size, offset);
Chris Masond1310b22008-01-24 16:13:08 -05002709 bio->bi_end_io = end_io_func;
2710 bio->bi_private = tree;
Chris Mason70dec802008-01-29 09:59:12 -05002711
Chris Masond3977122009-01-05 21:25:51 -05002712 if (bio_ret)
Chris Masond1310b22008-01-24 16:13:08 -05002713 *bio_ret = bio;
Chris Masond3977122009-01-05 21:25:51 -05002714 else
Chris Masonc8b97812008-10-29 14:49:59 -04002715 ret = submit_one_bio(rw, bio, mirror_num, bio_flags);
Chris Masond1310b22008-01-24 16:13:08 -05002716
2717 return ret;
2718}
2719
Eric Sandeen48a3b632013-04-25 20:41:01 +00002720static void attach_extent_buffer_page(struct extent_buffer *eb,
2721 struct page *page)
Josef Bacik4f2de97a2012-03-07 16:20:05 -05002722{
2723 if (!PagePrivate(page)) {
2724 SetPagePrivate(page);
2725 page_cache_get(page);
2726 set_page_private(page, (unsigned long)eb);
2727 } else {
2728 WARN_ON(page->private != (unsigned long)eb);
2729 }
2730}
2731
Chris Masond1310b22008-01-24 16:13:08 -05002732void set_page_extent_mapped(struct page *page)
2733{
2734 if (!PagePrivate(page)) {
2735 SetPagePrivate(page);
Chris Masond1310b22008-01-24 16:13:08 -05002736 page_cache_get(page);
Chris Mason6af118ce2008-07-22 11:18:07 -04002737 set_page_private(page, EXTENT_PAGE_PRIVATE);
Chris Masond1310b22008-01-24 16:13:08 -05002738 }
2739}
2740
Chris Masond1310b22008-01-24 16:13:08 -05002741/*
2742 * basic readpage implementation. Locked extent state structs are inserted
2743 * into the tree that are removed when the IO is done (by the end_io
2744 * handlers)
Jeff Mahoney79787ea2012-03-12 16:03:00 +01002745 * XXX JDM: This needs looking at to ensure proper page locking
Chris Masond1310b22008-01-24 16:13:08 -05002746 */
2747static int __extent_read_full_page(struct extent_io_tree *tree,
2748 struct page *page,
2749 get_extent_t *get_extent,
Chris Masonc8b97812008-10-29 14:49:59 -04002750 struct bio **bio, int mirror_num,
Josef Bacikd4c7ca82013-04-19 19:49:09 -04002751 unsigned long *bio_flags, int rw)
Chris Masond1310b22008-01-24 16:13:08 -05002752{
2753 struct inode *inode = page->mapping->host;
Miao Xie4eee4fa2012-12-21 09:17:45 +00002754 u64 start = page_offset(page);
Chris Masond1310b22008-01-24 16:13:08 -05002755 u64 page_end = start + PAGE_CACHE_SIZE - 1;
2756 u64 end;
2757 u64 cur = start;
2758 u64 extent_offset;
2759 u64 last_byte = i_size_read(inode);
2760 u64 block_start;
2761 u64 cur_end;
2762 sector_t sector;
2763 struct extent_map *em;
2764 struct block_device *bdev;
Josef Bacik11c65dc2010-05-23 11:07:21 -04002765 struct btrfs_ordered_extent *ordered;
Chris Masond1310b22008-01-24 16:13:08 -05002766 int ret;
2767 int nr = 0;
David Sterba306e16c2011-04-19 14:29:38 +02002768 size_t pg_offset = 0;
Chris Masond1310b22008-01-24 16:13:08 -05002769 size_t iosize;
Chris Masonc8b97812008-10-29 14:49:59 -04002770 size_t disk_io_size;
Chris Masond1310b22008-01-24 16:13:08 -05002771 size_t blocksize = inode->i_sb->s_blocksize;
Chris Masonc8b97812008-10-29 14:49:59 -04002772 unsigned long this_bio_flag = 0;
Chris Masond1310b22008-01-24 16:13:08 -05002773
2774 set_page_extent_mapped(page);
2775
Dan Magenheimer90a887c2011-05-26 10:01:56 -06002776 if (!PageUptodate(page)) {
2777 if (cleancache_get_page(page) == 0) {
2778 BUG_ON(blocksize != PAGE_SIZE);
2779 goto out;
2780 }
2781 }
2782
Chris Masond1310b22008-01-24 16:13:08 -05002783 end = page_end;
Josef Bacik11c65dc2010-05-23 11:07:21 -04002784 while (1) {
Jeff Mahoneyd0082372012-03-01 14:57:19 +01002785 lock_extent(tree, start, end);
Josef Bacik11c65dc2010-05-23 11:07:21 -04002786 ordered = btrfs_lookup_ordered_extent(inode, start);
2787 if (!ordered)
2788 break;
Jeff Mahoneyd0082372012-03-01 14:57:19 +01002789 unlock_extent(tree, start, end);
Josef Bacik11c65dc2010-05-23 11:07:21 -04002790 btrfs_start_ordered_extent(inode, ordered, 1);
2791 btrfs_put_ordered_extent(ordered);
2792 }
Chris Masond1310b22008-01-24 16:13:08 -05002793
Chris Masonc8b97812008-10-29 14:49:59 -04002794 if (page->index == last_byte >> PAGE_CACHE_SHIFT) {
2795 char *userpage;
2796 size_t zero_offset = last_byte & (PAGE_CACHE_SIZE - 1);
2797
2798 if (zero_offset) {
2799 iosize = PAGE_CACHE_SIZE - zero_offset;
Cong Wang7ac687d2011-11-25 23:14:28 +08002800 userpage = kmap_atomic(page);
Chris Masonc8b97812008-10-29 14:49:59 -04002801 memset(userpage + zero_offset, 0, iosize);
2802 flush_dcache_page(page);
Cong Wang7ac687d2011-11-25 23:14:28 +08002803 kunmap_atomic(userpage);
Chris Masonc8b97812008-10-29 14:49:59 -04002804 }
2805 }
Chris Masond1310b22008-01-24 16:13:08 -05002806 while (cur <= end) {
Josef Bacikc8f2f242013-02-11 11:33:00 -05002807 unsigned long pnr = (last_byte >> PAGE_CACHE_SHIFT) + 1;
2808
Chris Masond1310b22008-01-24 16:13:08 -05002809 if (cur >= last_byte) {
2810 char *userpage;
Arne Jansen507903b2011-04-06 10:02:20 +00002811 struct extent_state *cached = NULL;
2812
David Sterba306e16c2011-04-19 14:29:38 +02002813 iosize = PAGE_CACHE_SIZE - pg_offset;
Cong Wang7ac687d2011-11-25 23:14:28 +08002814 userpage = kmap_atomic(page);
David Sterba306e16c2011-04-19 14:29:38 +02002815 memset(userpage + pg_offset, 0, iosize);
Chris Masond1310b22008-01-24 16:13:08 -05002816 flush_dcache_page(page);
Cong Wang7ac687d2011-11-25 23:14:28 +08002817 kunmap_atomic(userpage);
Chris Masond1310b22008-01-24 16:13:08 -05002818 set_extent_uptodate(tree, cur, cur + iosize - 1,
Arne Jansen507903b2011-04-06 10:02:20 +00002819 &cached, GFP_NOFS);
2820 unlock_extent_cached(tree, cur, cur + iosize - 1,
2821 &cached, GFP_NOFS);
Chris Masond1310b22008-01-24 16:13:08 -05002822 break;
2823 }
David Sterba306e16c2011-04-19 14:29:38 +02002824 em = get_extent(inode, page, pg_offset, cur,
Chris Masond1310b22008-01-24 16:13:08 -05002825 end - cur + 1, 0);
David Sterbac7040052011-04-19 18:00:01 +02002826 if (IS_ERR_OR_NULL(em)) {
Chris Masond1310b22008-01-24 16:13:08 -05002827 SetPageError(page);
Jeff Mahoneyd0082372012-03-01 14:57:19 +01002828 unlock_extent(tree, cur, end);
Chris Masond1310b22008-01-24 16:13:08 -05002829 break;
2830 }
Chris Masond1310b22008-01-24 16:13:08 -05002831 extent_offset = cur - em->start;
2832 BUG_ON(extent_map_end(em) <= cur);
2833 BUG_ON(end < cur);
2834
Li Zefan261507a02010-12-17 14:21:50 +08002835 if (test_bit(EXTENT_FLAG_COMPRESSED, &em->flags)) {
Chris Masonc8b97812008-10-29 14:49:59 -04002836 this_bio_flag = EXTENT_BIO_COMPRESSED;
Li Zefan261507a02010-12-17 14:21:50 +08002837 extent_set_compress_type(&this_bio_flag,
2838 em->compress_type);
2839 }
Chris Masonc8b97812008-10-29 14:49:59 -04002840
Chris Masond1310b22008-01-24 16:13:08 -05002841 iosize = min(extent_map_end(em) - cur, end - cur + 1);
2842 cur_end = min(extent_map_end(em) - 1, end);
Qu Wenruofda28322013-02-26 08:10:22 +00002843 iosize = ALIGN(iosize, blocksize);
Chris Masonc8b97812008-10-29 14:49:59 -04002844 if (this_bio_flag & EXTENT_BIO_COMPRESSED) {
2845 disk_io_size = em->block_len;
2846 sector = em->block_start >> 9;
2847 } else {
2848 sector = (em->block_start + extent_offset) >> 9;
2849 disk_io_size = iosize;
2850 }
Chris Masond1310b22008-01-24 16:13:08 -05002851 bdev = em->bdev;
2852 block_start = em->block_start;
Yan Zhengd899e052008-10-30 14:25:28 -04002853 if (test_bit(EXTENT_FLAG_PREALLOC, &em->flags))
2854 block_start = EXTENT_MAP_HOLE;
Chris Masond1310b22008-01-24 16:13:08 -05002855 free_extent_map(em);
2856 em = NULL;
2857
2858 /* we've found a hole, just zero and go on */
2859 if (block_start == EXTENT_MAP_HOLE) {
2860 char *userpage;
Arne Jansen507903b2011-04-06 10:02:20 +00002861 struct extent_state *cached = NULL;
2862
Cong Wang7ac687d2011-11-25 23:14:28 +08002863 userpage = kmap_atomic(page);
David Sterba306e16c2011-04-19 14:29:38 +02002864 memset(userpage + pg_offset, 0, iosize);
Chris Masond1310b22008-01-24 16:13:08 -05002865 flush_dcache_page(page);
Cong Wang7ac687d2011-11-25 23:14:28 +08002866 kunmap_atomic(userpage);
Chris Masond1310b22008-01-24 16:13:08 -05002867
2868 set_extent_uptodate(tree, cur, cur + iosize - 1,
Arne Jansen507903b2011-04-06 10:02:20 +00002869 &cached, GFP_NOFS);
2870 unlock_extent_cached(tree, cur, cur + iosize - 1,
2871 &cached, GFP_NOFS);
Chris Masond1310b22008-01-24 16:13:08 -05002872 cur = cur + iosize;
David Sterba306e16c2011-04-19 14:29:38 +02002873 pg_offset += iosize;
Chris Masond1310b22008-01-24 16:13:08 -05002874 continue;
2875 }
2876 /* the get_extent function already copied into the page */
Chris Mason9655d292009-09-02 15:22:30 -04002877 if (test_range_bit(tree, cur, cur_end,
2878 EXTENT_UPTODATE, 1, NULL)) {
Chris Masona1b32a52008-09-05 16:09:51 -04002879 check_page_uptodate(tree, page);
Jeff Mahoneyd0082372012-03-01 14:57:19 +01002880 unlock_extent(tree, cur, cur + iosize - 1);
Chris Masond1310b22008-01-24 16:13:08 -05002881 cur = cur + iosize;
David Sterba306e16c2011-04-19 14:29:38 +02002882 pg_offset += iosize;
Chris Masond1310b22008-01-24 16:13:08 -05002883 continue;
2884 }
Chris Mason70dec802008-01-29 09:59:12 -05002885 /* we have an inline extent but it didn't get marked up
2886 * to date. Error out
2887 */
2888 if (block_start == EXTENT_MAP_INLINE) {
2889 SetPageError(page);
Jeff Mahoneyd0082372012-03-01 14:57:19 +01002890 unlock_extent(tree, cur, cur + iosize - 1);
Chris Mason70dec802008-01-29 09:59:12 -05002891 cur = cur + iosize;
David Sterba306e16c2011-04-19 14:29:38 +02002892 pg_offset += iosize;
Chris Mason70dec802008-01-29 09:59:12 -05002893 continue;
2894 }
Chris Masond1310b22008-01-24 16:13:08 -05002895
Josef Bacikc8f2f242013-02-11 11:33:00 -05002896 pnr -= page->index;
Josef Bacikd4c7ca82013-04-19 19:49:09 -04002897 ret = submit_extent_page(rw, tree, page,
David Sterba306e16c2011-04-19 14:29:38 +02002898 sector, disk_io_size, pg_offset,
Chris Mason89642222008-07-24 09:41:53 -04002899 bdev, bio, pnr,
Chris Masonc8b97812008-10-29 14:49:59 -04002900 end_bio_extent_readpage, mirror_num,
2901 *bio_flags,
2902 this_bio_flag);
Josef Bacikc8f2f242013-02-11 11:33:00 -05002903 if (!ret) {
2904 nr++;
2905 *bio_flags = this_bio_flag;
2906 } else {
Chris Masond1310b22008-01-24 16:13:08 -05002907 SetPageError(page);
Josef Bacikedd33c92012-10-05 16:40:32 -04002908 unlock_extent(tree, cur, cur + iosize - 1);
2909 }
Chris Masond1310b22008-01-24 16:13:08 -05002910 cur = cur + iosize;
David Sterba306e16c2011-04-19 14:29:38 +02002911 pg_offset += iosize;
Chris Masond1310b22008-01-24 16:13:08 -05002912 }
Dan Magenheimer90a887c2011-05-26 10:01:56 -06002913out:
Chris Masond1310b22008-01-24 16:13:08 -05002914 if (!nr) {
2915 if (!PageError(page))
2916 SetPageUptodate(page);
2917 unlock_page(page);
2918 }
2919 return 0;
2920}
2921
2922int extent_read_full_page(struct extent_io_tree *tree, struct page *page,
Jan Schmidt8ddc7d92011-06-13 20:02:58 +02002923 get_extent_t *get_extent, int mirror_num)
Chris Masond1310b22008-01-24 16:13:08 -05002924{
2925 struct bio *bio = NULL;
Chris Masonc8b97812008-10-29 14:49:59 -04002926 unsigned long bio_flags = 0;
Chris Masond1310b22008-01-24 16:13:08 -05002927 int ret;
2928
Jan Schmidt8ddc7d92011-06-13 20:02:58 +02002929 ret = __extent_read_full_page(tree, page, get_extent, &bio, mirror_num,
Josef Bacikd4c7ca82013-04-19 19:49:09 -04002930 &bio_flags, READ);
Chris Masond1310b22008-01-24 16:13:08 -05002931 if (bio)
Jan Schmidt8ddc7d92011-06-13 20:02:58 +02002932 ret = submit_one_bio(READ, bio, mirror_num, bio_flags);
Chris Masond1310b22008-01-24 16:13:08 -05002933 return ret;
2934}
Chris Masond1310b22008-01-24 16:13:08 -05002935
Chris Mason11c83492009-04-20 15:50:09 -04002936static noinline void update_nr_written(struct page *page,
2937 struct writeback_control *wbc,
2938 unsigned long nr_written)
2939{
2940 wbc->nr_to_write -= nr_written;
2941 if (wbc->range_cyclic || (wbc->nr_to_write > 0 &&
2942 wbc->range_start == 0 && wbc->range_end == LLONG_MAX))
2943 page->mapping->writeback_index = page->index + nr_written;
2944}
2945
Chris Masond1310b22008-01-24 16:13:08 -05002946/*
2947 * the writepage semantics are similar to regular writepage. extent
2948 * records are inserted to lock ranges in the tree, and as dirty areas
2949 * are found, they are marked writeback. Then the lock bits are removed
2950 * and the end_io handler clears the writeback ranges
2951 */
2952static int __extent_writepage(struct page *page, struct writeback_control *wbc,
2953 void *data)
2954{
2955 struct inode *inode = page->mapping->host;
2956 struct extent_page_data *epd = data;
2957 struct extent_io_tree *tree = epd->tree;
Miao Xie4eee4fa2012-12-21 09:17:45 +00002958 u64 start = page_offset(page);
Chris Masond1310b22008-01-24 16:13:08 -05002959 u64 delalloc_start;
2960 u64 page_end = start + PAGE_CACHE_SIZE - 1;
2961 u64 end;
2962 u64 cur = start;
2963 u64 extent_offset;
2964 u64 last_byte = i_size_read(inode);
2965 u64 block_start;
2966 u64 iosize;
2967 sector_t sector;
Chris Mason2c64c532009-09-02 15:04:12 -04002968 struct extent_state *cached_state = NULL;
Chris Masond1310b22008-01-24 16:13:08 -05002969 struct extent_map *em;
2970 struct block_device *bdev;
2971 int ret;
2972 int nr = 0;
Chris Mason7f3c74f2008-07-18 12:01:11 -04002973 size_t pg_offset = 0;
Chris Masond1310b22008-01-24 16:13:08 -05002974 size_t blocksize;
2975 loff_t i_size = i_size_read(inode);
2976 unsigned long end_index = i_size >> PAGE_CACHE_SHIFT;
2977 u64 nr_delalloc;
2978 u64 delalloc_end;
Chris Masonc8b97812008-10-29 14:49:59 -04002979 int page_started;
2980 int compressed;
Chris Masonffbd5172009-04-20 15:50:09 -04002981 int write_flags;
Chris Mason771ed682008-11-06 22:02:51 -05002982 unsigned long nr_written = 0;
Josef Bacik9e487102011-08-01 12:08:18 -04002983 bool fill_delalloc = true;
Chris Masond1310b22008-01-24 16:13:08 -05002984
Chris Masonffbd5172009-04-20 15:50:09 -04002985 if (wbc->sync_mode == WB_SYNC_ALL)
Jens Axboe721a9602011-03-09 11:56:30 +01002986 write_flags = WRITE_SYNC;
Chris Masonffbd5172009-04-20 15:50:09 -04002987 else
2988 write_flags = WRITE;
2989
liubo1abe9b82011-03-24 11:18:59 +00002990 trace___extent_writepage(page, inode, wbc);
2991
Chris Masond1310b22008-01-24 16:13:08 -05002992 WARN_ON(!PageLocked(page));
Chris Masonbf0da8c2011-11-04 12:29:37 -04002993
2994 ClearPageError(page);
2995
Chris Mason7f3c74f2008-07-18 12:01:11 -04002996 pg_offset = i_size & (PAGE_CACHE_SIZE - 1);
Chris Mason211c17f2008-05-15 09:13:45 -04002997 if (page->index > end_index ||
Chris Mason7f3c74f2008-07-18 12:01:11 -04002998 (page->index == end_index && !pg_offset)) {
Lukas Czernerd47992f2013-05-21 23:17:23 -04002999 page->mapping->a_ops->invalidatepage(page, 0, PAGE_CACHE_SIZE);
Chris Masond1310b22008-01-24 16:13:08 -05003000 unlock_page(page);
3001 return 0;
3002 }
3003
3004 if (page->index == end_index) {
3005 char *userpage;
3006
Cong Wang7ac687d2011-11-25 23:14:28 +08003007 userpage = kmap_atomic(page);
Chris Mason7f3c74f2008-07-18 12:01:11 -04003008 memset(userpage + pg_offset, 0,
3009 PAGE_CACHE_SIZE - pg_offset);
Cong Wang7ac687d2011-11-25 23:14:28 +08003010 kunmap_atomic(userpage);
Chris Mason211c17f2008-05-15 09:13:45 -04003011 flush_dcache_page(page);
Chris Masond1310b22008-01-24 16:13:08 -05003012 }
Chris Mason7f3c74f2008-07-18 12:01:11 -04003013 pg_offset = 0;
Chris Masond1310b22008-01-24 16:13:08 -05003014
3015 set_page_extent_mapped(page);
3016
Josef Bacik9e487102011-08-01 12:08:18 -04003017 if (!tree->ops || !tree->ops->fill_delalloc)
3018 fill_delalloc = false;
3019
Chris Masond1310b22008-01-24 16:13:08 -05003020 delalloc_start = start;
3021 delalloc_end = 0;
Chris Masonc8b97812008-10-29 14:49:59 -04003022 page_started = 0;
Josef Bacik9e487102011-08-01 12:08:18 -04003023 if (!epd->extent_locked && fill_delalloc) {
Chris Masonf85d7d6c2009-09-18 16:03:16 -04003024 u64 delalloc_to_write = 0;
Chris Mason11c83492009-04-20 15:50:09 -04003025 /*
3026 * make sure the wbc mapping index is at least updated
3027 * to this page.
3028 */
3029 update_nr_written(page, wbc, 0);
3030
Chris Masond3977122009-01-05 21:25:51 -05003031 while (delalloc_end < page_end) {
Chris Mason771ed682008-11-06 22:02:51 -05003032 nr_delalloc = find_lock_delalloc_range(inode, tree,
Chris Masonc8b97812008-10-29 14:49:59 -04003033 page,
3034 &delalloc_start,
Chris Masond1310b22008-01-24 16:13:08 -05003035 &delalloc_end,
3036 128 * 1024 * 1024);
Chris Mason771ed682008-11-06 22:02:51 -05003037 if (nr_delalloc == 0) {
3038 delalloc_start = delalloc_end + 1;
3039 continue;
3040 }
Tsutomu Itoh013bd4c2012-02-16 10:11:40 +09003041 ret = tree->ops->fill_delalloc(inode, page,
3042 delalloc_start,
3043 delalloc_end,
3044 &page_started,
3045 &nr_written);
Jeff Mahoney79787ea2012-03-12 16:03:00 +01003046 /* File system has been set read-only */
3047 if (ret) {
3048 SetPageError(page);
3049 goto done;
3050 }
Chris Masonf85d7d6c2009-09-18 16:03:16 -04003051 /*
3052 * delalloc_end is already one less than the total
3053 * length, so we don't subtract one from
3054 * PAGE_CACHE_SIZE
3055 */
3056 delalloc_to_write += (delalloc_end - delalloc_start +
3057 PAGE_CACHE_SIZE) >>
3058 PAGE_CACHE_SHIFT;
Chris Masond1310b22008-01-24 16:13:08 -05003059 delalloc_start = delalloc_end + 1;
Chris Masond1310b22008-01-24 16:13:08 -05003060 }
Chris Masonf85d7d6c2009-09-18 16:03:16 -04003061 if (wbc->nr_to_write < delalloc_to_write) {
3062 int thresh = 8192;
3063
3064 if (delalloc_to_write < thresh * 2)
3065 thresh = delalloc_to_write;
3066 wbc->nr_to_write = min_t(u64, delalloc_to_write,
3067 thresh);
3068 }
Chris Masonc8b97812008-10-29 14:49:59 -04003069
Chris Mason771ed682008-11-06 22:02:51 -05003070 /* did the fill delalloc function already unlock and start
3071 * the IO?
3072 */
3073 if (page_started) {
3074 ret = 0;
Chris Mason11c83492009-04-20 15:50:09 -04003075 /*
3076 * we've unlocked the page, so we can't update
3077 * the mapping's writeback index, just update
3078 * nr_to_write.
3079 */
3080 wbc->nr_to_write -= nr_written;
3081 goto done_unlocked;
Chris Mason771ed682008-11-06 22:02:51 -05003082 }
Chris Masonc8b97812008-10-29 14:49:59 -04003083 }
Chris Mason247e7432008-07-17 12:53:51 -04003084 if (tree->ops && tree->ops->writepage_start_hook) {
Chris Masonc8b97812008-10-29 14:49:59 -04003085 ret = tree->ops->writepage_start_hook(page, start,
3086 page_end);
Jeff Mahoney87826df2012-02-15 16:23:57 +01003087 if (ret) {
3088 /* Fixup worker will requeue */
3089 if (ret == -EBUSY)
3090 wbc->pages_skipped++;
3091 else
3092 redirty_page_for_writepage(wbc, page);
Chris Mason11c83492009-04-20 15:50:09 -04003093 update_nr_written(page, wbc, nr_written);
Chris Mason247e7432008-07-17 12:53:51 -04003094 unlock_page(page);
Chris Mason771ed682008-11-06 22:02:51 -05003095 ret = 0;
Chris Mason11c83492009-04-20 15:50:09 -04003096 goto done_unlocked;
Chris Mason247e7432008-07-17 12:53:51 -04003097 }
3098 }
3099
Chris Mason11c83492009-04-20 15:50:09 -04003100 /*
3101 * we don't want to touch the inode after unlocking the page,
3102 * so we update the mapping writeback index now
3103 */
3104 update_nr_written(page, wbc, nr_written + 1);
Chris Mason771ed682008-11-06 22:02:51 -05003105
Chris Masond1310b22008-01-24 16:13:08 -05003106 end = page_end;
Chris Masond1310b22008-01-24 16:13:08 -05003107 if (last_byte <= start) {
Chris Masone6dcd2d2008-07-17 12:53:50 -04003108 if (tree->ops && tree->ops->writepage_end_io_hook)
3109 tree->ops->writepage_end_io_hook(page, start,
3110 page_end, NULL, 1);
Chris Masond1310b22008-01-24 16:13:08 -05003111 goto done;
3112 }
3113
Chris Masond1310b22008-01-24 16:13:08 -05003114 blocksize = inode->i_sb->s_blocksize;
3115
3116 while (cur <= end) {
3117 if (cur >= last_byte) {
Chris Masone6dcd2d2008-07-17 12:53:50 -04003118 if (tree->ops && tree->ops->writepage_end_io_hook)
3119 tree->ops->writepage_end_io_hook(page, cur,
3120 page_end, NULL, 1);
Chris Masond1310b22008-01-24 16:13:08 -05003121 break;
3122 }
Chris Mason7f3c74f2008-07-18 12:01:11 -04003123 em = epd->get_extent(inode, page, pg_offset, cur,
Chris Masond1310b22008-01-24 16:13:08 -05003124 end - cur + 1, 1);
David Sterbac7040052011-04-19 18:00:01 +02003125 if (IS_ERR_OR_NULL(em)) {
Chris Masond1310b22008-01-24 16:13:08 -05003126 SetPageError(page);
3127 break;
3128 }
3129
3130 extent_offset = cur - em->start;
3131 BUG_ON(extent_map_end(em) <= cur);
3132 BUG_ON(end < cur);
3133 iosize = min(extent_map_end(em) - cur, end - cur + 1);
Qu Wenruofda28322013-02-26 08:10:22 +00003134 iosize = ALIGN(iosize, blocksize);
Chris Masond1310b22008-01-24 16:13:08 -05003135 sector = (em->block_start + extent_offset) >> 9;
3136 bdev = em->bdev;
3137 block_start = em->block_start;
Chris Masonc8b97812008-10-29 14:49:59 -04003138 compressed = test_bit(EXTENT_FLAG_COMPRESSED, &em->flags);
Chris Masond1310b22008-01-24 16:13:08 -05003139 free_extent_map(em);
3140 em = NULL;
3141
Chris Masonc8b97812008-10-29 14:49:59 -04003142 /*
3143 * compressed and inline extents are written through other
3144 * paths in the FS
3145 */
3146 if (compressed || block_start == EXTENT_MAP_HOLE ||
Chris Masond1310b22008-01-24 16:13:08 -05003147 block_start == EXTENT_MAP_INLINE) {
Chris Masonc8b97812008-10-29 14:49:59 -04003148 /*
3149 * end_io notification does not happen here for
3150 * compressed extents
3151 */
3152 if (!compressed && tree->ops &&
3153 tree->ops->writepage_end_io_hook)
Chris Masone6dcd2d2008-07-17 12:53:50 -04003154 tree->ops->writepage_end_io_hook(page, cur,
3155 cur + iosize - 1,
3156 NULL, 1);
Chris Masonc8b97812008-10-29 14:49:59 -04003157 else if (compressed) {
3158 /* we don't want to end_page_writeback on
3159 * a compressed extent. this happens
3160 * elsewhere
3161 */
3162 nr++;
3163 }
3164
3165 cur += iosize;
Chris Mason7f3c74f2008-07-18 12:01:11 -04003166 pg_offset += iosize;
Chris Masond1310b22008-01-24 16:13:08 -05003167 continue;
3168 }
Chris Masond1310b22008-01-24 16:13:08 -05003169 /* leave this out until we have a page_mkwrite call */
3170 if (0 && !test_range_bit(tree, cur, cur + iosize - 1,
Chris Mason9655d292009-09-02 15:22:30 -04003171 EXTENT_DIRTY, 0, NULL)) {
Chris Masond1310b22008-01-24 16:13:08 -05003172 cur = cur + iosize;
Chris Mason7f3c74f2008-07-18 12:01:11 -04003173 pg_offset += iosize;
Chris Masond1310b22008-01-24 16:13:08 -05003174 continue;
3175 }
Chris Masonc8b97812008-10-29 14:49:59 -04003176
Chris Masond1310b22008-01-24 16:13:08 -05003177 if (tree->ops && tree->ops->writepage_io_hook) {
3178 ret = tree->ops->writepage_io_hook(page, cur,
3179 cur + iosize - 1);
3180 } else {
3181 ret = 0;
3182 }
Chris Mason1259ab72008-05-12 13:39:03 -04003183 if (ret) {
Chris Masond1310b22008-01-24 16:13:08 -05003184 SetPageError(page);
Chris Mason1259ab72008-05-12 13:39:03 -04003185 } else {
Chris Masond1310b22008-01-24 16:13:08 -05003186 unsigned long max_nr = end_index + 1;
Chris Mason7f3c74f2008-07-18 12:01:11 -04003187
Chris Masond1310b22008-01-24 16:13:08 -05003188 set_range_writeback(tree, cur, cur + iosize - 1);
3189 if (!PageWriteback(page)) {
Chris Masond3977122009-01-05 21:25:51 -05003190 printk(KERN_ERR "btrfs warning page %lu not "
3191 "writeback, cur %llu end %llu\n",
3192 page->index, (unsigned long long)cur,
Chris Masond1310b22008-01-24 16:13:08 -05003193 (unsigned long long)end);
3194 }
3195
Chris Masonffbd5172009-04-20 15:50:09 -04003196 ret = submit_extent_page(write_flags, tree, page,
3197 sector, iosize, pg_offset,
3198 bdev, &epd->bio, max_nr,
Chris Masonc8b97812008-10-29 14:49:59 -04003199 end_bio_extent_writepage,
3200 0, 0, 0);
Chris Masond1310b22008-01-24 16:13:08 -05003201 if (ret)
3202 SetPageError(page);
3203 }
3204 cur = cur + iosize;
Chris Mason7f3c74f2008-07-18 12:01:11 -04003205 pg_offset += iosize;
Chris Masond1310b22008-01-24 16:13:08 -05003206 nr++;
3207 }
3208done:
3209 if (nr == 0) {
3210 /* make sure the mapping tag for page dirty gets cleared */
3211 set_page_writeback(page);
3212 end_page_writeback(page);
3213 }
Chris Masond1310b22008-01-24 16:13:08 -05003214 unlock_page(page);
Chris Mason771ed682008-11-06 22:02:51 -05003215
Chris Mason11c83492009-04-20 15:50:09 -04003216done_unlocked:
3217
Chris Mason2c64c532009-09-02 15:04:12 -04003218 /* drop our reference on any cached states */
3219 free_extent_state(cached_state);
Chris Masond1310b22008-01-24 16:13:08 -05003220 return 0;
3221}
3222
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003223static int eb_wait(void *word)
3224{
3225 io_schedule();
3226 return 0;
3227}
3228
Josef Bacikfd8b2b62013-04-24 16:41:19 -04003229void wait_on_extent_buffer_writeback(struct extent_buffer *eb)
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003230{
3231 wait_on_bit(&eb->bflags, EXTENT_BUFFER_WRITEBACK, eb_wait,
3232 TASK_UNINTERRUPTIBLE);
3233}
3234
3235static int lock_extent_buffer_for_io(struct extent_buffer *eb,
3236 struct btrfs_fs_info *fs_info,
3237 struct extent_page_data *epd)
3238{
3239 unsigned long i, num_pages;
3240 int flush = 0;
3241 int ret = 0;
3242
3243 if (!btrfs_try_tree_write_lock(eb)) {
3244 flush = 1;
3245 flush_write_bio(epd);
3246 btrfs_tree_lock(eb);
3247 }
3248
3249 if (test_bit(EXTENT_BUFFER_WRITEBACK, &eb->bflags)) {
3250 btrfs_tree_unlock(eb);
3251 if (!epd->sync_io)
3252 return 0;
3253 if (!flush) {
3254 flush_write_bio(epd);
3255 flush = 1;
3256 }
Chris Masona098d8e2012-03-21 12:09:56 -04003257 while (1) {
3258 wait_on_extent_buffer_writeback(eb);
3259 btrfs_tree_lock(eb);
3260 if (!test_bit(EXTENT_BUFFER_WRITEBACK, &eb->bflags))
3261 break;
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003262 btrfs_tree_unlock(eb);
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003263 }
3264 }
3265
Josef Bacik51561ff2012-07-20 16:25:24 -04003266 /*
3267 * We need to do this to prevent races in people who check if the eb is
3268 * under IO since we can end up having no IO bits set for a short period
3269 * of time.
3270 */
3271 spin_lock(&eb->refs_lock);
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003272 if (test_and_clear_bit(EXTENT_BUFFER_DIRTY, &eb->bflags)) {
3273 set_bit(EXTENT_BUFFER_WRITEBACK, &eb->bflags);
Josef Bacik51561ff2012-07-20 16:25:24 -04003274 spin_unlock(&eb->refs_lock);
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003275 btrfs_set_header_flag(eb, BTRFS_HEADER_FLAG_WRITTEN);
Miao Xiee2d84522013-01-29 10:09:20 +00003276 __percpu_counter_add(&fs_info->dirty_metadata_bytes,
3277 -eb->len,
3278 fs_info->dirty_metadata_batch);
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003279 ret = 1;
Josef Bacik51561ff2012-07-20 16:25:24 -04003280 } else {
3281 spin_unlock(&eb->refs_lock);
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003282 }
3283
3284 btrfs_tree_unlock(eb);
3285
3286 if (!ret)
3287 return ret;
3288
3289 num_pages = num_extent_pages(eb->start, eb->len);
3290 for (i = 0; i < num_pages; i++) {
3291 struct page *p = extent_buffer_page(eb, i);
3292
3293 if (!trylock_page(p)) {
3294 if (!flush) {
3295 flush_write_bio(epd);
3296 flush = 1;
3297 }
3298 lock_page(p);
3299 }
3300 }
3301
3302 return ret;
3303}
3304
3305static void end_extent_buffer_writeback(struct extent_buffer *eb)
3306{
3307 clear_bit(EXTENT_BUFFER_WRITEBACK, &eb->bflags);
3308 smp_mb__after_clear_bit();
3309 wake_up_bit(&eb->bflags, EXTENT_BUFFER_WRITEBACK);
3310}
3311
3312static void end_bio_extent_buffer_writepage(struct bio *bio, int err)
3313{
3314 int uptodate = err == 0;
3315 struct bio_vec *bvec = bio->bi_io_vec + bio->bi_vcnt - 1;
3316 struct extent_buffer *eb;
3317 int done;
3318
3319 do {
3320 struct page *page = bvec->bv_page;
3321
3322 bvec--;
3323 eb = (struct extent_buffer *)page->private;
3324 BUG_ON(!eb);
3325 done = atomic_dec_and_test(&eb->io_pages);
3326
3327 if (!uptodate || test_bit(EXTENT_BUFFER_IOERR, &eb->bflags)) {
3328 set_bit(EXTENT_BUFFER_IOERR, &eb->bflags);
3329 ClearPageUptodate(page);
3330 SetPageError(page);
3331 }
3332
3333 end_page_writeback(page);
3334
3335 if (!done)
3336 continue;
3337
3338 end_extent_buffer_writeback(eb);
3339 } while (bvec >= bio->bi_io_vec);
3340
3341 bio_put(bio);
3342
3343}
3344
3345static int write_one_eb(struct extent_buffer *eb,
3346 struct btrfs_fs_info *fs_info,
3347 struct writeback_control *wbc,
3348 struct extent_page_data *epd)
3349{
3350 struct block_device *bdev = fs_info->fs_devices->latest_bdev;
3351 u64 offset = eb->start;
3352 unsigned long i, num_pages;
Josef Bacikde0022b2012-09-25 14:25:58 -04003353 unsigned long bio_flags = 0;
Josef Bacikd4c7ca82013-04-19 19:49:09 -04003354 int rw = (epd->sync_io ? WRITE_SYNC : WRITE) | REQ_META;
Josef Bacikd7dbe9e2012-04-23 14:00:51 -04003355 int ret = 0;
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003356
3357 clear_bit(EXTENT_BUFFER_IOERR, &eb->bflags);
3358 num_pages = num_extent_pages(eb->start, eb->len);
3359 atomic_set(&eb->io_pages, num_pages);
Josef Bacikde0022b2012-09-25 14:25:58 -04003360 if (btrfs_header_owner(eb) == BTRFS_TREE_LOG_OBJECTID)
3361 bio_flags = EXTENT_BIO_TREE_LOG;
3362
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003363 for (i = 0; i < num_pages; i++) {
3364 struct page *p = extent_buffer_page(eb, i);
3365
3366 clear_page_dirty_for_io(p);
3367 set_page_writeback(p);
3368 ret = submit_extent_page(rw, eb->tree, p, offset >> 9,
3369 PAGE_CACHE_SIZE, 0, bdev, &epd->bio,
3370 -1, end_bio_extent_buffer_writepage,
Josef Bacikde0022b2012-09-25 14:25:58 -04003371 0, epd->bio_flags, bio_flags);
3372 epd->bio_flags = bio_flags;
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003373 if (ret) {
3374 set_bit(EXTENT_BUFFER_IOERR, &eb->bflags);
3375 SetPageError(p);
3376 if (atomic_sub_and_test(num_pages - i, &eb->io_pages))
3377 end_extent_buffer_writeback(eb);
3378 ret = -EIO;
3379 break;
3380 }
3381 offset += PAGE_CACHE_SIZE;
3382 update_nr_written(p, wbc, 1);
3383 unlock_page(p);
3384 }
3385
3386 if (unlikely(ret)) {
3387 for (; i < num_pages; i++) {
3388 struct page *p = extent_buffer_page(eb, i);
3389 unlock_page(p);
3390 }
3391 }
3392
3393 return ret;
3394}
3395
3396int btree_write_cache_pages(struct address_space *mapping,
3397 struct writeback_control *wbc)
3398{
3399 struct extent_io_tree *tree = &BTRFS_I(mapping->host)->io_tree;
3400 struct btrfs_fs_info *fs_info = BTRFS_I(mapping->host)->root->fs_info;
3401 struct extent_buffer *eb, *prev_eb = NULL;
3402 struct extent_page_data epd = {
3403 .bio = NULL,
3404 .tree = tree,
3405 .extent_locked = 0,
3406 .sync_io = wbc->sync_mode == WB_SYNC_ALL,
Josef Bacikde0022b2012-09-25 14:25:58 -04003407 .bio_flags = 0,
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003408 };
3409 int ret = 0;
3410 int done = 0;
3411 int nr_to_write_done = 0;
3412 struct pagevec pvec;
3413 int nr_pages;
3414 pgoff_t index;
3415 pgoff_t end; /* Inclusive */
3416 int scanned = 0;
3417 int tag;
3418
3419 pagevec_init(&pvec, 0);
3420 if (wbc->range_cyclic) {
3421 index = mapping->writeback_index; /* Start from prev offset */
3422 end = -1;
3423 } else {
3424 index = wbc->range_start >> PAGE_CACHE_SHIFT;
3425 end = wbc->range_end >> PAGE_CACHE_SHIFT;
3426 scanned = 1;
3427 }
3428 if (wbc->sync_mode == WB_SYNC_ALL)
3429 tag = PAGECACHE_TAG_TOWRITE;
3430 else
3431 tag = PAGECACHE_TAG_DIRTY;
3432retry:
3433 if (wbc->sync_mode == WB_SYNC_ALL)
3434 tag_pages_for_writeback(mapping, index, end);
3435 while (!done && !nr_to_write_done && (index <= end) &&
3436 (nr_pages = pagevec_lookup_tag(&pvec, mapping, &index, tag,
3437 min(end - index, (pgoff_t)PAGEVEC_SIZE-1) + 1))) {
3438 unsigned i;
3439
3440 scanned = 1;
3441 for (i = 0; i < nr_pages; i++) {
3442 struct page *page = pvec.pages[i];
3443
3444 if (!PagePrivate(page))
3445 continue;
3446
3447 if (!wbc->range_cyclic && page->index > end) {
3448 done = 1;
3449 break;
3450 }
3451
Josef Bacikb5bae262012-09-14 13:43:01 -04003452 spin_lock(&mapping->private_lock);
3453 if (!PagePrivate(page)) {
3454 spin_unlock(&mapping->private_lock);
3455 continue;
3456 }
3457
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003458 eb = (struct extent_buffer *)page->private;
Josef Bacikb5bae262012-09-14 13:43:01 -04003459
3460 /*
3461 * Shouldn't happen and normally this would be a BUG_ON
3462 * but no sense in crashing the users box for something
3463 * we can survive anyway.
3464 */
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003465 if (!eb) {
Josef Bacikb5bae262012-09-14 13:43:01 -04003466 spin_unlock(&mapping->private_lock);
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003467 WARN_ON(1);
3468 continue;
3469 }
3470
Josef Bacikb5bae262012-09-14 13:43:01 -04003471 if (eb == prev_eb) {
3472 spin_unlock(&mapping->private_lock);
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003473 continue;
3474 }
3475
Josef Bacikb5bae262012-09-14 13:43:01 -04003476 ret = atomic_inc_not_zero(&eb->refs);
3477 spin_unlock(&mapping->private_lock);
3478 if (!ret)
3479 continue;
3480
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003481 prev_eb = eb;
3482 ret = lock_extent_buffer_for_io(eb, fs_info, &epd);
3483 if (!ret) {
3484 free_extent_buffer(eb);
3485 continue;
3486 }
3487
3488 ret = write_one_eb(eb, fs_info, wbc, &epd);
3489 if (ret) {
3490 done = 1;
3491 free_extent_buffer(eb);
3492 break;
3493 }
3494 free_extent_buffer(eb);
3495
3496 /*
3497 * the filesystem may choose to bump up nr_to_write.
3498 * We have to make sure to honor the new nr_to_write
3499 * at any time
3500 */
3501 nr_to_write_done = wbc->nr_to_write <= 0;
3502 }
3503 pagevec_release(&pvec);
3504 cond_resched();
3505 }
3506 if (!scanned && !done) {
3507 /*
3508 * We hit the last page and there is more work to be done: wrap
3509 * back to the start of the file
3510 */
3511 scanned = 1;
3512 index = 0;
3513 goto retry;
3514 }
3515 flush_write_bio(&epd);
3516 return ret;
3517}
3518
Chris Masond1310b22008-01-24 16:13:08 -05003519/**
Chris Mason4bef0842008-09-08 11:18:08 -04003520 * write_cache_pages - walk the list of dirty pages of the given address space and write all of them.
Chris Masond1310b22008-01-24 16:13:08 -05003521 * @mapping: address space structure to write
3522 * @wbc: subtract the number of written pages from *@wbc->nr_to_write
3523 * @writepage: function called for each page
3524 * @data: data passed to writepage function
3525 *
3526 * If a page is already under I/O, write_cache_pages() skips it, even
3527 * if it's dirty. This is desirable behaviour for memory-cleaning writeback,
3528 * but it is INCORRECT for data-integrity system calls such as fsync(). fsync()
3529 * and msync() need to guarantee that all the data which was dirty at the time
3530 * the call was made get new I/O started against them. If wbc->sync_mode is
3531 * WB_SYNC_ALL then we were called for data integrity and we must wait for
3532 * existing IO to complete.
3533 */
Christoph Hellwigb2950862008-12-02 09:54:17 -05003534static int extent_write_cache_pages(struct extent_io_tree *tree,
Chris Mason4bef0842008-09-08 11:18:08 -04003535 struct address_space *mapping,
3536 struct writeback_control *wbc,
Chris Masond2c3f4f2008-11-19 12:44:22 -05003537 writepage_t writepage, void *data,
3538 void (*flush_fn)(void *))
Chris Masond1310b22008-01-24 16:13:08 -05003539{
Josef Bacik7fd1a3f2012-06-27 17:18:41 -04003540 struct inode *inode = mapping->host;
Chris Masond1310b22008-01-24 16:13:08 -05003541 int ret = 0;
3542 int done = 0;
Chris Masonf85d7d6c2009-09-18 16:03:16 -04003543 int nr_to_write_done = 0;
Chris Masond1310b22008-01-24 16:13:08 -05003544 struct pagevec pvec;
3545 int nr_pages;
3546 pgoff_t index;
3547 pgoff_t end; /* Inclusive */
3548 int scanned = 0;
Josef Bacikf7aaa062011-07-15 21:26:38 +00003549 int tag;
Chris Masond1310b22008-01-24 16:13:08 -05003550
Josef Bacik7fd1a3f2012-06-27 17:18:41 -04003551 /*
3552 * We have to hold onto the inode so that ordered extents can do their
3553 * work when the IO finishes. The alternative to this is failing to add
3554 * an ordered extent if the igrab() fails there and that is a huge pain
3555 * to deal with, so instead just hold onto the inode throughout the
3556 * writepages operation. If it fails here we are freeing up the inode
3557 * anyway and we'd rather not waste our time writing out stuff that is
3558 * going to be truncated anyway.
3559 */
3560 if (!igrab(inode))
3561 return 0;
3562
Chris Masond1310b22008-01-24 16:13:08 -05003563 pagevec_init(&pvec, 0);
3564 if (wbc->range_cyclic) {
3565 index = mapping->writeback_index; /* Start from prev offset */
3566 end = -1;
3567 } else {
3568 index = wbc->range_start >> PAGE_CACHE_SHIFT;
3569 end = wbc->range_end >> PAGE_CACHE_SHIFT;
Chris Masond1310b22008-01-24 16:13:08 -05003570 scanned = 1;
3571 }
Josef Bacikf7aaa062011-07-15 21:26:38 +00003572 if (wbc->sync_mode == WB_SYNC_ALL)
3573 tag = PAGECACHE_TAG_TOWRITE;
3574 else
3575 tag = PAGECACHE_TAG_DIRTY;
Chris Masond1310b22008-01-24 16:13:08 -05003576retry:
Josef Bacikf7aaa062011-07-15 21:26:38 +00003577 if (wbc->sync_mode == WB_SYNC_ALL)
3578 tag_pages_for_writeback(mapping, index, end);
Chris Masonf85d7d6c2009-09-18 16:03:16 -04003579 while (!done && !nr_to_write_done && (index <= end) &&
Josef Bacikf7aaa062011-07-15 21:26:38 +00003580 (nr_pages = pagevec_lookup_tag(&pvec, mapping, &index, tag,
3581 min(end - index, (pgoff_t)PAGEVEC_SIZE-1) + 1))) {
Chris Masond1310b22008-01-24 16:13:08 -05003582 unsigned i;
3583
3584 scanned = 1;
3585 for (i = 0; i < nr_pages; i++) {
3586 struct page *page = pvec.pages[i];
3587
3588 /*
3589 * At this point we hold neither mapping->tree_lock nor
3590 * lock on the page itself: the page may be truncated or
3591 * invalidated (changing page->mapping to NULL), or even
3592 * swizzled back from swapper_space to tmpfs file
3593 * mapping
3594 */
Josef Bacikc8f2f242013-02-11 11:33:00 -05003595 if (!trylock_page(page)) {
3596 flush_fn(data);
3597 lock_page(page);
Chris Mason01d658f2011-11-01 10:08:06 -04003598 }
Chris Masond1310b22008-01-24 16:13:08 -05003599
3600 if (unlikely(page->mapping != mapping)) {
3601 unlock_page(page);
3602 continue;
3603 }
3604
3605 if (!wbc->range_cyclic && page->index > end) {
3606 done = 1;
3607 unlock_page(page);
3608 continue;
3609 }
3610
Chris Masond2c3f4f2008-11-19 12:44:22 -05003611 if (wbc->sync_mode != WB_SYNC_NONE) {
Chris Mason0e6bd952008-11-20 10:46:35 -05003612 if (PageWriteback(page))
3613 flush_fn(data);
Chris Masond1310b22008-01-24 16:13:08 -05003614 wait_on_page_writeback(page);
Chris Masond2c3f4f2008-11-19 12:44:22 -05003615 }
Chris Masond1310b22008-01-24 16:13:08 -05003616
3617 if (PageWriteback(page) ||
3618 !clear_page_dirty_for_io(page)) {
3619 unlock_page(page);
3620 continue;
3621 }
3622
3623 ret = (*writepage)(page, wbc, data);
3624
3625 if (unlikely(ret == AOP_WRITEPAGE_ACTIVATE)) {
3626 unlock_page(page);
3627 ret = 0;
3628 }
Chris Masonf85d7d6c2009-09-18 16:03:16 -04003629 if (ret)
Chris Masond1310b22008-01-24 16:13:08 -05003630 done = 1;
Chris Masonf85d7d6c2009-09-18 16:03:16 -04003631
3632 /*
3633 * the filesystem may choose to bump up nr_to_write.
3634 * We have to make sure to honor the new nr_to_write
3635 * at any time
3636 */
3637 nr_to_write_done = wbc->nr_to_write <= 0;
Chris Masond1310b22008-01-24 16:13:08 -05003638 }
3639 pagevec_release(&pvec);
3640 cond_resched();
3641 }
3642 if (!scanned && !done) {
3643 /*
3644 * We hit the last page and there is more work to be done: wrap
3645 * back to the start of the file
3646 */
3647 scanned = 1;
3648 index = 0;
3649 goto retry;
3650 }
Josef Bacik7fd1a3f2012-06-27 17:18:41 -04003651 btrfs_add_delayed_iput(inode);
Chris Masond1310b22008-01-24 16:13:08 -05003652 return ret;
3653}
Chris Masond1310b22008-01-24 16:13:08 -05003654
Chris Masonffbd5172009-04-20 15:50:09 -04003655static void flush_epd_write_bio(struct extent_page_data *epd)
3656{
3657 if (epd->bio) {
Jeff Mahoney355808c2011-10-03 23:23:14 -04003658 int rw = WRITE;
3659 int ret;
3660
Chris Masonffbd5172009-04-20 15:50:09 -04003661 if (epd->sync_io)
Jeff Mahoney355808c2011-10-03 23:23:14 -04003662 rw = WRITE_SYNC;
3663
Josef Bacikde0022b2012-09-25 14:25:58 -04003664 ret = submit_one_bio(rw, epd->bio, 0, epd->bio_flags);
Jeff Mahoney79787ea2012-03-12 16:03:00 +01003665 BUG_ON(ret < 0); /* -ENOMEM */
Chris Masonffbd5172009-04-20 15:50:09 -04003666 epd->bio = NULL;
3667 }
3668}
3669
Chris Masond2c3f4f2008-11-19 12:44:22 -05003670static noinline void flush_write_bio(void *data)
3671{
3672 struct extent_page_data *epd = data;
Chris Masonffbd5172009-04-20 15:50:09 -04003673 flush_epd_write_bio(epd);
Chris Masond2c3f4f2008-11-19 12:44:22 -05003674}
3675
Chris Masond1310b22008-01-24 16:13:08 -05003676int extent_write_full_page(struct extent_io_tree *tree, struct page *page,
3677 get_extent_t *get_extent,
3678 struct writeback_control *wbc)
3679{
3680 int ret;
Chris Masond1310b22008-01-24 16:13:08 -05003681 struct extent_page_data epd = {
3682 .bio = NULL,
3683 .tree = tree,
3684 .get_extent = get_extent,
Chris Mason771ed682008-11-06 22:02:51 -05003685 .extent_locked = 0,
Chris Masonffbd5172009-04-20 15:50:09 -04003686 .sync_io = wbc->sync_mode == WB_SYNC_ALL,
Josef Bacikde0022b2012-09-25 14:25:58 -04003687 .bio_flags = 0,
Chris Masond1310b22008-01-24 16:13:08 -05003688 };
Chris Masond1310b22008-01-24 16:13:08 -05003689
Chris Masond1310b22008-01-24 16:13:08 -05003690 ret = __extent_writepage(page, wbc, &epd);
3691
Chris Masonffbd5172009-04-20 15:50:09 -04003692 flush_epd_write_bio(&epd);
Chris Masond1310b22008-01-24 16:13:08 -05003693 return ret;
3694}
Chris Masond1310b22008-01-24 16:13:08 -05003695
Chris Mason771ed682008-11-06 22:02:51 -05003696int extent_write_locked_range(struct extent_io_tree *tree, struct inode *inode,
3697 u64 start, u64 end, get_extent_t *get_extent,
3698 int mode)
3699{
3700 int ret = 0;
3701 struct address_space *mapping = inode->i_mapping;
3702 struct page *page;
3703 unsigned long nr_pages = (end - start + PAGE_CACHE_SIZE) >>
3704 PAGE_CACHE_SHIFT;
3705
3706 struct extent_page_data epd = {
3707 .bio = NULL,
3708 .tree = tree,
3709 .get_extent = get_extent,
3710 .extent_locked = 1,
Chris Masonffbd5172009-04-20 15:50:09 -04003711 .sync_io = mode == WB_SYNC_ALL,
Josef Bacikde0022b2012-09-25 14:25:58 -04003712 .bio_flags = 0,
Chris Mason771ed682008-11-06 22:02:51 -05003713 };
3714 struct writeback_control wbc_writepages = {
Chris Mason771ed682008-11-06 22:02:51 -05003715 .sync_mode = mode,
Chris Mason771ed682008-11-06 22:02:51 -05003716 .nr_to_write = nr_pages * 2,
3717 .range_start = start,
3718 .range_end = end + 1,
3719 };
3720
Chris Masond3977122009-01-05 21:25:51 -05003721 while (start <= end) {
Chris Mason771ed682008-11-06 22:02:51 -05003722 page = find_get_page(mapping, start >> PAGE_CACHE_SHIFT);
3723 if (clear_page_dirty_for_io(page))
3724 ret = __extent_writepage(page, &wbc_writepages, &epd);
3725 else {
3726 if (tree->ops && tree->ops->writepage_end_io_hook)
3727 tree->ops->writepage_end_io_hook(page, start,
3728 start + PAGE_CACHE_SIZE - 1,
3729 NULL, 1);
3730 unlock_page(page);
3731 }
3732 page_cache_release(page);
3733 start += PAGE_CACHE_SIZE;
3734 }
3735
Chris Masonffbd5172009-04-20 15:50:09 -04003736 flush_epd_write_bio(&epd);
Chris Mason771ed682008-11-06 22:02:51 -05003737 return ret;
3738}
Chris Masond1310b22008-01-24 16:13:08 -05003739
3740int extent_writepages(struct extent_io_tree *tree,
3741 struct address_space *mapping,
3742 get_extent_t *get_extent,
3743 struct writeback_control *wbc)
3744{
3745 int ret = 0;
3746 struct extent_page_data epd = {
3747 .bio = NULL,
3748 .tree = tree,
3749 .get_extent = get_extent,
Chris Mason771ed682008-11-06 22:02:51 -05003750 .extent_locked = 0,
Chris Masonffbd5172009-04-20 15:50:09 -04003751 .sync_io = wbc->sync_mode == WB_SYNC_ALL,
Josef Bacikde0022b2012-09-25 14:25:58 -04003752 .bio_flags = 0,
Chris Masond1310b22008-01-24 16:13:08 -05003753 };
3754
Chris Mason4bef0842008-09-08 11:18:08 -04003755 ret = extent_write_cache_pages(tree, mapping, wbc,
Chris Masond2c3f4f2008-11-19 12:44:22 -05003756 __extent_writepage, &epd,
3757 flush_write_bio);
Chris Masonffbd5172009-04-20 15:50:09 -04003758 flush_epd_write_bio(&epd);
Chris Masond1310b22008-01-24 16:13:08 -05003759 return ret;
3760}
Chris Masond1310b22008-01-24 16:13:08 -05003761
3762int extent_readpages(struct extent_io_tree *tree,
3763 struct address_space *mapping,
3764 struct list_head *pages, unsigned nr_pages,
3765 get_extent_t get_extent)
3766{
3767 struct bio *bio = NULL;
3768 unsigned page_idx;
Chris Masonc8b97812008-10-29 14:49:59 -04003769 unsigned long bio_flags = 0;
Liu Bo67c96842012-07-20 21:43:09 -06003770 struct page *pagepool[16];
3771 struct page *page;
3772 int i = 0;
3773 int nr = 0;
Chris Masond1310b22008-01-24 16:13:08 -05003774
Chris Masond1310b22008-01-24 16:13:08 -05003775 for (page_idx = 0; page_idx < nr_pages; page_idx++) {
Liu Bo67c96842012-07-20 21:43:09 -06003776 page = list_entry(pages->prev, struct page, lru);
Chris Masond1310b22008-01-24 16:13:08 -05003777
3778 prefetchw(&page->flags);
3779 list_del(&page->lru);
Liu Bo67c96842012-07-20 21:43:09 -06003780 if (add_to_page_cache_lru(page, mapping,
Itaru Kitayama43e817a2011-04-25 19:43:51 -04003781 page->index, GFP_NOFS)) {
Liu Bo67c96842012-07-20 21:43:09 -06003782 page_cache_release(page);
3783 continue;
Chris Masond1310b22008-01-24 16:13:08 -05003784 }
Liu Bo67c96842012-07-20 21:43:09 -06003785
3786 pagepool[nr++] = page;
3787 if (nr < ARRAY_SIZE(pagepool))
3788 continue;
3789 for (i = 0; i < nr; i++) {
3790 __extent_read_full_page(tree, pagepool[i], get_extent,
Josef Bacikd4c7ca82013-04-19 19:49:09 -04003791 &bio, 0, &bio_flags, READ);
Liu Bo67c96842012-07-20 21:43:09 -06003792 page_cache_release(pagepool[i]);
3793 }
3794 nr = 0;
Chris Masond1310b22008-01-24 16:13:08 -05003795 }
Liu Bo67c96842012-07-20 21:43:09 -06003796 for (i = 0; i < nr; i++) {
3797 __extent_read_full_page(tree, pagepool[i], get_extent,
Josef Bacikd4c7ca82013-04-19 19:49:09 -04003798 &bio, 0, &bio_flags, READ);
Liu Bo67c96842012-07-20 21:43:09 -06003799 page_cache_release(pagepool[i]);
3800 }
3801
Chris Masond1310b22008-01-24 16:13:08 -05003802 BUG_ON(!list_empty(pages));
3803 if (bio)
Jeff Mahoney79787ea2012-03-12 16:03:00 +01003804 return submit_one_bio(READ, bio, 0, bio_flags);
Chris Masond1310b22008-01-24 16:13:08 -05003805 return 0;
3806}
Chris Masond1310b22008-01-24 16:13:08 -05003807
3808/*
3809 * basic invalidatepage code, this waits on any locked or writeback
3810 * ranges corresponding to the page, and then deletes any extent state
3811 * records from the tree
3812 */
3813int extent_invalidatepage(struct extent_io_tree *tree,
3814 struct page *page, unsigned long offset)
3815{
Josef Bacik2ac55d42010-02-03 19:33:23 +00003816 struct extent_state *cached_state = NULL;
Miao Xie4eee4fa2012-12-21 09:17:45 +00003817 u64 start = page_offset(page);
Chris Masond1310b22008-01-24 16:13:08 -05003818 u64 end = start + PAGE_CACHE_SIZE - 1;
3819 size_t blocksize = page->mapping->host->i_sb->s_blocksize;
3820
Qu Wenruofda28322013-02-26 08:10:22 +00003821 start += ALIGN(offset, blocksize);
Chris Masond1310b22008-01-24 16:13:08 -05003822 if (start > end)
3823 return 0;
3824
Jeff Mahoneyd0082372012-03-01 14:57:19 +01003825 lock_extent_bits(tree, start, end, 0, &cached_state);
Chris Mason1edbb732009-09-02 13:24:36 -04003826 wait_on_page_writeback(page);
Chris Masond1310b22008-01-24 16:13:08 -05003827 clear_extent_bit(tree, start, end,
Josef Bacik32c00af2009-10-08 13:34:05 -04003828 EXTENT_LOCKED | EXTENT_DIRTY | EXTENT_DELALLOC |
3829 EXTENT_DO_ACCOUNTING,
Josef Bacik2ac55d42010-02-03 19:33:23 +00003830 1, 1, &cached_state, GFP_NOFS);
Chris Masond1310b22008-01-24 16:13:08 -05003831 return 0;
3832}
Chris Masond1310b22008-01-24 16:13:08 -05003833
3834/*
Chris Mason7b13b7b2008-04-18 10:29:50 -04003835 * a helper for releasepage, this tests for areas of the page that
3836 * are locked or under IO and drops the related state bits if it is safe
3837 * to drop the page.
3838 */
Eric Sandeen48a3b632013-04-25 20:41:01 +00003839static int try_release_extent_state(struct extent_map_tree *map,
3840 struct extent_io_tree *tree,
3841 struct page *page, gfp_t mask)
Chris Mason7b13b7b2008-04-18 10:29:50 -04003842{
Miao Xie4eee4fa2012-12-21 09:17:45 +00003843 u64 start = page_offset(page);
Chris Mason7b13b7b2008-04-18 10:29:50 -04003844 u64 end = start + PAGE_CACHE_SIZE - 1;
3845 int ret = 1;
3846
Chris Mason211f90e2008-07-18 11:56:15 -04003847 if (test_range_bit(tree, start, end,
Chris Mason8b62b722009-09-02 16:53:46 -04003848 EXTENT_IOBITS, 0, NULL))
Chris Mason7b13b7b2008-04-18 10:29:50 -04003849 ret = 0;
3850 else {
3851 if ((mask & GFP_NOFS) == GFP_NOFS)
3852 mask = GFP_NOFS;
Chris Mason11ef1602009-09-23 20:28:46 -04003853 /*
3854 * at this point we can safely clear everything except the
3855 * locked bit and the nodatasum bit
3856 */
Chris Masone3f24cc2011-02-14 12:52:08 -05003857 ret = clear_extent_bit(tree, start, end,
Chris Mason11ef1602009-09-23 20:28:46 -04003858 ~(EXTENT_LOCKED | EXTENT_NODATASUM),
3859 0, 0, NULL, mask);
Chris Masone3f24cc2011-02-14 12:52:08 -05003860
3861 /* if clear_extent_bit failed for enomem reasons,
3862 * we can't allow the release to continue.
3863 */
3864 if (ret < 0)
3865 ret = 0;
3866 else
3867 ret = 1;
Chris Mason7b13b7b2008-04-18 10:29:50 -04003868 }
3869 return ret;
3870}
Chris Mason7b13b7b2008-04-18 10:29:50 -04003871
3872/*
Chris Masond1310b22008-01-24 16:13:08 -05003873 * a helper for releasepage. As long as there are no locked extents
3874 * in the range corresponding to the page, both state records and extent
3875 * map records are removed
3876 */
3877int try_release_extent_mapping(struct extent_map_tree *map,
Chris Mason70dec802008-01-29 09:59:12 -05003878 struct extent_io_tree *tree, struct page *page,
3879 gfp_t mask)
Chris Masond1310b22008-01-24 16:13:08 -05003880{
3881 struct extent_map *em;
Miao Xie4eee4fa2012-12-21 09:17:45 +00003882 u64 start = page_offset(page);
Chris Masond1310b22008-01-24 16:13:08 -05003883 u64 end = start + PAGE_CACHE_SIZE - 1;
Chris Mason7b13b7b2008-04-18 10:29:50 -04003884
Chris Mason70dec802008-01-29 09:59:12 -05003885 if ((mask & __GFP_WAIT) &&
3886 page->mapping->host->i_size > 16 * 1024 * 1024) {
Yan39b56372008-02-15 10:40:50 -05003887 u64 len;
Chris Mason70dec802008-01-29 09:59:12 -05003888 while (start <= end) {
Yan39b56372008-02-15 10:40:50 -05003889 len = end - start + 1;
Chris Mason890871b2009-09-02 16:24:52 -04003890 write_lock(&map->lock);
Yan39b56372008-02-15 10:40:50 -05003891 em = lookup_extent_mapping(map, start, len);
Tsutomu Itoh285190d2012-02-16 16:23:58 +09003892 if (!em) {
Chris Mason890871b2009-09-02 16:24:52 -04003893 write_unlock(&map->lock);
Chris Mason70dec802008-01-29 09:59:12 -05003894 break;
3895 }
Chris Mason7f3c74f2008-07-18 12:01:11 -04003896 if (test_bit(EXTENT_FLAG_PINNED, &em->flags) ||
3897 em->start != start) {
Chris Mason890871b2009-09-02 16:24:52 -04003898 write_unlock(&map->lock);
Chris Mason70dec802008-01-29 09:59:12 -05003899 free_extent_map(em);
3900 break;
3901 }
3902 if (!test_range_bit(tree, em->start,
3903 extent_map_end(em) - 1,
Chris Mason8b62b722009-09-02 16:53:46 -04003904 EXTENT_LOCKED | EXTENT_WRITEBACK,
Chris Mason9655d292009-09-02 15:22:30 -04003905 0, NULL)) {
Chris Mason70dec802008-01-29 09:59:12 -05003906 remove_extent_mapping(map, em);
3907 /* once for the rb tree */
3908 free_extent_map(em);
3909 }
3910 start = extent_map_end(em);
Chris Mason890871b2009-09-02 16:24:52 -04003911 write_unlock(&map->lock);
Chris Mason70dec802008-01-29 09:59:12 -05003912
3913 /* once for us */
Chris Masond1310b22008-01-24 16:13:08 -05003914 free_extent_map(em);
3915 }
Chris Masond1310b22008-01-24 16:13:08 -05003916 }
Chris Mason7b13b7b2008-04-18 10:29:50 -04003917 return try_release_extent_state(map, tree, page, mask);
Chris Masond1310b22008-01-24 16:13:08 -05003918}
Chris Masond1310b22008-01-24 16:13:08 -05003919
Chris Masonec29ed52011-02-23 16:23:20 -05003920/*
3921 * helper function for fiemap, which doesn't want to see any holes.
3922 * This maps until we find something past 'last'
3923 */
3924static struct extent_map *get_extent_skip_holes(struct inode *inode,
3925 u64 offset,
3926 u64 last,
3927 get_extent_t *get_extent)
3928{
3929 u64 sectorsize = BTRFS_I(inode)->root->sectorsize;
3930 struct extent_map *em;
3931 u64 len;
3932
3933 if (offset >= last)
3934 return NULL;
3935
3936 while(1) {
3937 len = last - offset;
3938 if (len == 0)
3939 break;
Qu Wenruofda28322013-02-26 08:10:22 +00003940 len = ALIGN(len, sectorsize);
Chris Masonec29ed52011-02-23 16:23:20 -05003941 em = get_extent(inode, NULL, 0, offset, len, 0);
David Sterbac7040052011-04-19 18:00:01 +02003942 if (IS_ERR_OR_NULL(em))
Chris Masonec29ed52011-02-23 16:23:20 -05003943 return em;
3944
3945 /* if this isn't a hole return it */
3946 if (!test_bit(EXTENT_FLAG_VACANCY, &em->flags) &&
3947 em->block_start != EXTENT_MAP_HOLE) {
3948 return em;
3949 }
3950
3951 /* this is a hole, advance to the next extent */
3952 offset = extent_map_end(em);
3953 free_extent_map(em);
3954 if (offset >= last)
3955 break;
3956 }
3957 return NULL;
3958}
3959
Yehuda Sadeh1506fcc2009-01-21 14:39:14 -05003960int extent_fiemap(struct inode *inode, struct fiemap_extent_info *fieinfo,
3961 __u64 start, __u64 len, get_extent_t *get_extent)
3962{
Josef Bacik975f84f2010-11-23 19:36:57 +00003963 int ret = 0;
Yehuda Sadeh1506fcc2009-01-21 14:39:14 -05003964 u64 off = start;
3965 u64 max = start + len;
3966 u32 flags = 0;
Josef Bacik975f84f2010-11-23 19:36:57 +00003967 u32 found_type;
3968 u64 last;
Chris Masonec29ed52011-02-23 16:23:20 -05003969 u64 last_for_get_extent = 0;
Yehuda Sadeh1506fcc2009-01-21 14:39:14 -05003970 u64 disko = 0;
Chris Masonec29ed52011-02-23 16:23:20 -05003971 u64 isize = i_size_read(inode);
Josef Bacik975f84f2010-11-23 19:36:57 +00003972 struct btrfs_key found_key;
Yehuda Sadeh1506fcc2009-01-21 14:39:14 -05003973 struct extent_map *em = NULL;
Josef Bacik2ac55d42010-02-03 19:33:23 +00003974 struct extent_state *cached_state = NULL;
Josef Bacik975f84f2010-11-23 19:36:57 +00003975 struct btrfs_path *path;
3976 struct btrfs_file_extent_item *item;
Yehuda Sadeh1506fcc2009-01-21 14:39:14 -05003977 int end = 0;
Chris Masonec29ed52011-02-23 16:23:20 -05003978 u64 em_start = 0;
3979 u64 em_len = 0;
3980 u64 em_end = 0;
Yehuda Sadeh1506fcc2009-01-21 14:39:14 -05003981 unsigned long emflags;
Yehuda Sadeh1506fcc2009-01-21 14:39:14 -05003982
3983 if (len == 0)
3984 return -EINVAL;
3985
Josef Bacik975f84f2010-11-23 19:36:57 +00003986 path = btrfs_alloc_path();
3987 if (!path)
3988 return -ENOMEM;
3989 path->leave_spinning = 1;
3990
Josef Bacik4d479cf2011-11-17 11:34:31 -05003991 start = ALIGN(start, BTRFS_I(inode)->root->sectorsize);
3992 len = ALIGN(len, BTRFS_I(inode)->root->sectorsize);
3993
Chris Masonec29ed52011-02-23 16:23:20 -05003994 /*
3995 * lookup the last file extent. We're not using i_size here
3996 * because there might be preallocation past i_size
3997 */
Josef Bacik975f84f2010-11-23 19:36:57 +00003998 ret = btrfs_lookup_file_extent(NULL, BTRFS_I(inode)->root,
Li Zefan33345d012011-04-20 10:31:50 +08003999 path, btrfs_ino(inode), -1, 0);
Josef Bacik975f84f2010-11-23 19:36:57 +00004000 if (ret < 0) {
4001 btrfs_free_path(path);
4002 return ret;
4003 }
4004 WARN_ON(!ret);
4005 path->slots[0]--;
4006 item = btrfs_item_ptr(path->nodes[0], path->slots[0],
4007 struct btrfs_file_extent_item);
4008 btrfs_item_key_to_cpu(path->nodes[0], &found_key, path->slots[0]);
4009 found_type = btrfs_key_type(&found_key);
4010
Chris Masonec29ed52011-02-23 16:23:20 -05004011 /* No extents, but there might be delalloc bits */
Li Zefan33345d012011-04-20 10:31:50 +08004012 if (found_key.objectid != btrfs_ino(inode) ||
Josef Bacik975f84f2010-11-23 19:36:57 +00004013 found_type != BTRFS_EXTENT_DATA_KEY) {
Chris Masonec29ed52011-02-23 16:23:20 -05004014 /* have to trust i_size as the end */
4015 last = (u64)-1;
4016 last_for_get_extent = isize;
4017 } else {
4018 /*
4019 * remember the start of the last extent. There are a
4020 * bunch of different factors that go into the length of the
4021 * extent, so its much less complex to remember where it started
4022 */
4023 last = found_key.offset;
4024 last_for_get_extent = last + 1;
Josef Bacik975f84f2010-11-23 19:36:57 +00004025 }
Josef Bacik975f84f2010-11-23 19:36:57 +00004026 btrfs_free_path(path);
4027
Chris Masonec29ed52011-02-23 16:23:20 -05004028 /*
4029 * we might have some extents allocated but more delalloc past those
4030 * extents. so, we trust isize unless the start of the last extent is
4031 * beyond isize
4032 */
4033 if (last < isize) {
4034 last = (u64)-1;
4035 last_for_get_extent = isize;
4036 }
4037
Liu Boa52f4cd2013-05-01 16:23:41 +00004038 lock_extent_bits(&BTRFS_I(inode)->io_tree, start, start + len - 1, 0,
Jeff Mahoneyd0082372012-03-01 14:57:19 +01004039 &cached_state);
Chris Masonec29ed52011-02-23 16:23:20 -05004040
Josef Bacik4d479cf2011-11-17 11:34:31 -05004041 em = get_extent_skip_holes(inode, start, last_for_get_extent,
Chris Masonec29ed52011-02-23 16:23:20 -05004042 get_extent);
Yehuda Sadeh1506fcc2009-01-21 14:39:14 -05004043 if (!em)
4044 goto out;
4045 if (IS_ERR(em)) {
4046 ret = PTR_ERR(em);
4047 goto out;
4048 }
Josef Bacik975f84f2010-11-23 19:36:57 +00004049
Yehuda Sadeh1506fcc2009-01-21 14:39:14 -05004050 while (!end) {
Chris Masonea8efc72011-03-08 11:54:40 -05004051 u64 offset_in_extent;
Yehuda Sadeh1506fcc2009-01-21 14:39:14 -05004052
Chris Masonea8efc72011-03-08 11:54:40 -05004053 /* break if the extent we found is outside the range */
4054 if (em->start >= max || extent_map_end(em) < off)
4055 break;
4056
4057 /*
4058 * get_extent may return an extent that starts before our
4059 * requested range. We have to make sure the ranges
4060 * we return to fiemap always move forward and don't
4061 * overlap, so adjust the offsets here
4062 */
4063 em_start = max(em->start, off);
4064
4065 /*
4066 * record the offset from the start of the extent
4067 * for adjusting the disk offset below
4068 */
4069 offset_in_extent = em_start - em->start;
Chris Masonec29ed52011-02-23 16:23:20 -05004070 em_end = extent_map_end(em);
Chris Masonea8efc72011-03-08 11:54:40 -05004071 em_len = em_end - em_start;
Chris Masonec29ed52011-02-23 16:23:20 -05004072 emflags = em->flags;
Yehuda Sadeh1506fcc2009-01-21 14:39:14 -05004073 disko = 0;
4074 flags = 0;
4075
Chris Masonea8efc72011-03-08 11:54:40 -05004076 /*
4077 * bump off for our next call to get_extent
4078 */
4079 off = extent_map_end(em);
4080 if (off >= max)
4081 end = 1;
4082
Heiko Carstens93dbfad2009-04-03 10:33:45 -04004083 if (em->block_start == EXTENT_MAP_LAST_BYTE) {
Yehuda Sadeh1506fcc2009-01-21 14:39:14 -05004084 end = 1;
4085 flags |= FIEMAP_EXTENT_LAST;
Heiko Carstens93dbfad2009-04-03 10:33:45 -04004086 } else if (em->block_start == EXTENT_MAP_INLINE) {
Yehuda Sadeh1506fcc2009-01-21 14:39:14 -05004087 flags |= (FIEMAP_EXTENT_DATA_INLINE |
4088 FIEMAP_EXTENT_NOT_ALIGNED);
Heiko Carstens93dbfad2009-04-03 10:33:45 -04004089 } else if (em->block_start == EXTENT_MAP_DELALLOC) {
Yehuda Sadeh1506fcc2009-01-21 14:39:14 -05004090 flags |= (FIEMAP_EXTENT_DELALLOC |
4091 FIEMAP_EXTENT_UNKNOWN);
Heiko Carstens93dbfad2009-04-03 10:33:45 -04004092 } else {
Chris Masonea8efc72011-03-08 11:54:40 -05004093 disko = em->block_start + offset_in_extent;
Yehuda Sadeh1506fcc2009-01-21 14:39:14 -05004094 }
4095 if (test_bit(EXTENT_FLAG_COMPRESSED, &em->flags))
4096 flags |= FIEMAP_EXTENT_ENCODED;
4097
Yehuda Sadeh1506fcc2009-01-21 14:39:14 -05004098 free_extent_map(em);
4099 em = NULL;
Chris Masonec29ed52011-02-23 16:23:20 -05004100 if ((em_start >= last) || em_len == (u64)-1 ||
4101 (last == (u64)-1 && isize <= em_end)) {
Yehuda Sadeh1506fcc2009-01-21 14:39:14 -05004102 flags |= FIEMAP_EXTENT_LAST;
4103 end = 1;
4104 }
4105
Chris Masonec29ed52011-02-23 16:23:20 -05004106 /* now scan forward to see if this is really the last extent. */
4107 em = get_extent_skip_holes(inode, off, last_for_get_extent,
4108 get_extent);
4109 if (IS_ERR(em)) {
4110 ret = PTR_ERR(em);
4111 goto out;
4112 }
4113 if (!em) {
Josef Bacik975f84f2010-11-23 19:36:57 +00004114 flags |= FIEMAP_EXTENT_LAST;
4115 end = 1;
4116 }
Chris Masonec29ed52011-02-23 16:23:20 -05004117 ret = fiemap_fill_next_extent(fieinfo, em_start, disko,
4118 em_len, flags);
4119 if (ret)
4120 goto out_free;
Yehuda Sadeh1506fcc2009-01-21 14:39:14 -05004121 }
4122out_free:
4123 free_extent_map(em);
4124out:
Liu Boa52f4cd2013-05-01 16:23:41 +00004125 unlock_extent_cached(&BTRFS_I(inode)->io_tree, start, start + len - 1,
Josef Bacik2ac55d42010-02-03 19:33:23 +00004126 &cached_state, GFP_NOFS);
Yehuda Sadeh1506fcc2009-01-21 14:39:14 -05004127 return ret;
4128}
4129
Chris Mason727011e2010-08-06 13:21:20 -04004130static void __free_extent_buffer(struct extent_buffer *eb)
4131{
Eric Sandeen6d49ba12013-04-22 16:12:31 +00004132 btrfs_leak_debug_del(&eb->leak_list);
Chris Mason727011e2010-08-06 13:21:20 -04004133 kmem_cache_free(extent_buffer_cache, eb);
4134}
4135
Chris Masond1310b22008-01-24 16:13:08 -05004136static struct extent_buffer *__alloc_extent_buffer(struct extent_io_tree *tree,
4137 u64 start,
4138 unsigned long len,
4139 gfp_t mask)
4140{
4141 struct extent_buffer *eb = NULL;
4142
Chris Masond1310b22008-01-24 16:13:08 -05004143 eb = kmem_cache_zalloc(extent_buffer_cache, mask);
Tsutomu Itoh91ca3382011-01-05 02:32:22 +00004144 if (eb == NULL)
4145 return NULL;
Chris Masond1310b22008-01-24 16:13:08 -05004146 eb->start = start;
4147 eb->len = len;
Josef Bacik4f2de97a2012-03-07 16:20:05 -05004148 eb->tree = tree;
Jan Schmidt815a51c2012-05-16 17:00:02 +02004149 eb->bflags = 0;
Chris Masonbd681512011-07-16 15:23:14 -04004150 rwlock_init(&eb->lock);
4151 atomic_set(&eb->write_locks, 0);
4152 atomic_set(&eb->read_locks, 0);
4153 atomic_set(&eb->blocking_readers, 0);
4154 atomic_set(&eb->blocking_writers, 0);
4155 atomic_set(&eb->spinning_readers, 0);
4156 atomic_set(&eb->spinning_writers, 0);
Arne Jansen5b25f702011-09-13 10:55:48 +02004157 eb->lock_nested = 0;
Chris Masonbd681512011-07-16 15:23:14 -04004158 init_waitqueue_head(&eb->write_lock_wq);
4159 init_waitqueue_head(&eb->read_lock_wq);
Chris Masonb4ce94d2009-02-04 09:25:08 -05004160
Eric Sandeen6d49ba12013-04-22 16:12:31 +00004161 btrfs_leak_debug_add(&eb->leak_list, &buffers);
4162
Josef Bacik3083ee22012-03-09 16:01:49 -05004163 spin_lock_init(&eb->refs_lock);
Chris Masond1310b22008-01-24 16:13:08 -05004164 atomic_set(&eb->refs, 1);
Josef Bacik0b32f4b2012-03-13 09:38:00 -04004165 atomic_set(&eb->io_pages, 0);
Chris Mason727011e2010-08-06 13:21:20 -04004166
David Sterbab8dae312013-02-28 14:54:18 +00004167 /*
4168 * Sanity checks, currently the maximum is 64k covered by 16x 4k pages
4169 */
4170 BUILD_BUG_ON(BTRFS_MAX_METADATA_BLOCKSIZE
4171 > MAX_INLINE_EXTENT_BUFFER_SIZE);
4172 BUG_ON(len > MAX_INLINE_EXTENT_BUFFER_SIZE);
Chris Masond1310b22008-01-24 16:13:08 -05004173
4174 return eb;
4175}
4176
Jan Schmidt815a51c2012-05-16 17:00:02 +02004177struct extent_buffer *btrfs_clone_extent_buffer(struct extent_buffer *src)
4178{
4179 unsigned long i;
4180 struct page *p;
4181 struct extent_buffer *new;
4182 unsigned long num_pages = num_extent_pages(src->start, src->len);
4183
4184 new = __alloc_extent_buffer(NULL, src->start, src->len, GFP_ATOMIC);
4185 if (new == NULL)
4186 return NULL;
4187
4188 for (i = 0; i < num_pages; i++) {
4189 p = alloc_page(GFP_ATOMIC);
4190 BUG_ON(!p);
4191 attach_extent_buffer_page(new, p);
4192 WARN_ON(PageDirty(p));
4193 SetPageUptodate(p);
4194 new->pages[i] = p;
4195 }
4196
4197 copy_extent_buffer(new, src, 0, 0, src->len);
4198 set_bit(EXTENT_BUFFER_UPTODATE, &new->bflags);
4199 set_bit(EXTENT_BUFFER_DUMMY, &new->bflags);
4200
4201 return new;
4202}
4203
4204struct extent_buffer *alloc_dummy_extent_buffer(u64 start, unsigned long len)
4205{
4206 struct extent_buffer *eb;
4207 unsigned long num_pages = num_extent_pages(0, len);
4208 unsigned long i;
4209
4210 eb = __alloc_extent_buffer(NULL, start, len, GFP_ATOMIC);
4211 if (!eb)
4212 return NULL;
4213
4214 for (i = 0; i < num_pages; i++) {
4215 eb->pages[i] = alloc_page(GFP_ATOMIC);
4216 if (!eb->pages[i])
4217 goto err;
4218 }
4219 set_extent_buffer_uptodate(eb);
4220 btrfs_set_header_nritems(eb, 0);
4221 set_bit(EXTENT_BUFFER_DUMMY, &eb->bflags);
4222
4223 return eb;
4224err:
Stefan Behrens84167d12012-10-11 07:25:16 -06004225 for (; i > 0; i--)
4226 __free_page(eb->pages[i - 1]);
Jan Schmidt815a51c2012-05-16 17:00:02 +02004227 __free_extent_buffer(eb);
4228 return NULL;
4229}
4230
Josef Bacik0b32f4b2012-03-13 09:38:00 -04004231static int extent_buffer_under_io(struct extent_buffer *eb)
Chris Masond1310b22008-01-24 16:13:08 -05004232{
Josef Bacik0b32f4b2012-03-13 09:38:00 -04004233 return (atomic_read(&eb->io_pages) ||
4234 test_bit(EXTENT_BUFFER_WRITEBACK, &eb->bflags) ||
4235 test_bit(EXTENT_BUFFER_DIRTY, &eb->bflags));
Chris Masond1310b22008-01-24 16:13:08 -05004236}
4237
Miao Xie897ca6e2010-10-26 20:57:29 -04004238/*
4239 * Helper for releasing extent buffer page.
4240 */
4241static void btrfs_release_extent_buffer_page(struct extent_buffer *eb,
4242 unsigned long start_idx)
4243{
4244 unsigned long index;
Wang Sheng-Hui39bab872012-04-06 14:35:31 +08004245 unsigned long num_pages;
Miao Xie897ca6e2010-10-26 20:57:29 -04004246 struct page *page;
Jan Schmidt815a51c2012-05-16 17:00:02 +02004247 int mapped = !test_bit(EXTENT_BUFFER_DUMMY, &eb->bflags);
Miao Xie897ca6e2010-10-26 20:57:29 -04004248
Josef Bacik0b32f4b2012-03-13 09:38:00 -04004249 BUG_ON(extent_buffer_under_io(eb));
Miao Xie897ca6e2010-10-26 20:57:29 -04004250
Wang Sheng-Hui39bab872012-04-06 14:35:31 +08004251 num_pages = num_extent_pages(eb->start, eb->len);
4252 index = start_idx + num_pages;
Miao Xie897ca6e2010-10-26 20:57:29 -04004253 if (start_idx >= index)
4254 return;
4255
4256 do {
4257 index--;
4258 page = extent_buffer_page(eb, index);
Jan Schmidt815a51c2012-05-16 17:00:02 +02004259 if (page && mapped) {
Josef Bacik4f2de97a2012-03-07 16:20:05 -05004260 spin_lock(&page->mapping->private_lock);
4261 /*
4262 * We do this since we'll remove the pages after we've
4263 * removed the eb from the radix tree, so we could race
4264 * and have this page now attached to the new eb. So
4265 * only clear page_private if it's still connected to
4266 * this eb.
4267 */
4268 if (PagePrivate(page) &&
4269 page->private == (unsigned long)eb) {
Josef Bacik0b32f4b2012-03-13 09:38:00 -04004270 BUG_ON(test_bit(EXTENT_BUFFER_DIRTY, &eb->bflags));
Josef Bacik3083ee22012-03-09 16:01:49 -05004271 BUG_ON(PageDirty(page));
4272 BUG_ON(PageWriteback(page));
Josef Bacik4f2de97a2012-03-07 16:20:05 -05004273 /*
4274 * We need to make sure we haven't be attached
4275 * to a new eb.
4276 */
4277 ClearPagePrivate(page);
4278 set_page_private(page, 0);
4279 /* One for the page private */
4280 page_cache_release(page);
4281 }
4282 spin_unlock(&page->mapping->private_lock);
4283
Jan Schmidt815a51c2012-05-16 17:00:02 +02004284 }
4285 if (page) {
Josef Bacik4f2de97a2012-03-07 16:20:05 -05004286 /* One for when we alloced the page */
Miao Xie897ca6e2010-10-26 20:57:29 -04004287 page_cache_release(page);
Josef Bacik4f2de97a2012-03-07 16:20:05 -05004288 }
Miao Xie897ca6e2010-10-26 20:57:29 -04004289 } while (index != start_idx);
4290}
4291
4292/*
4293 * Helper for releasing the extent buffer.
4294 */
4295static inline void btrfs_release_extent_buffer(struct extent_buffer *eb)
4296{
4297 btrfs_release_extent_buffer_page(eb, 0);
4298 __free_extent_buffer(eb);
4299}
4300
Josef Bacik0b32f4b2012-03-13 09:38:00 -04004301static void check_buffer_tree_ref(struct extent_buffer *eb)
4302{
Chris Mason242e18c2013-01-29 17:49:37 -05004303 int refs;
Josef Bacik0b32f4b2012-03-13 09:38:00 -04004304 /* the ref bit is tricky. We have to make sure it is set
4305 * if we have the buffer dirty. Otherwise the
4306 * code to free a buffer can end up dropping a dirty
4307 * page
4308 *
4309 * Once the ref bit is set, it won't go away while the
4310 * buffer is dirty or in writeback, and it also won't
4311 * go away while we have the reference count on the
4312 * eb bumped.
4313 *
4314 * We can't just set the ref bit without bumping the
4315 * ref on the eb because free_extent_buffer might
4316 * see the ref bit and try to clear it. If this happens
4317 * free_extent_buffer might end up dropping our original
4318 * ref by mistake and freeing the page before we are able
4319 * to add one more ref.
4320 *
4321 * So bump the ref count first, then set the bit. If someone
4322 * beat us to it, drop the ref we added.
4323 */
Chris Mason242e18c2013-01-29 17:49:37 -05004324 refs = atomic_read(&eb->refs);
4325 if (refs >= 2 && test_bit(EXTENT_BUFFER_TREE_REF, &eb->bflags))
4326 return;
4327
Josef Bacik594831c2012-07-20 16:11:08 -04004328 spin_lock(&eb->refs_lock);
4329 if (!test_and_set_bit(EXTENT_BUFFER_TREE_REF, &eb->bflags))
Josef Bacik0b32f4b2012-03-13 09:38:00 -04004330 atomic_inc(&eb->refs);
Josef Bacik594831c2012-07-20 16:11:08 -04004331 spin_unlock(&eb->refs_lock);
Josef Bacik0b32f4b2012-03-13 09:38:00 -04004332}
4333
Josef Bacik5df42352012-03-15 18:24:42 -04004334static void mark_extent_buffer_accessed(struct extent_buffer *eb)
4335{
4336 unsigned long num_pages, i;
4337
Josef Bacik0b32f4b2012-03-13 09:38:00 -04004338 check_buffer_tree_ref(eb);
4339
Josef Bacik5df42352012-03-15 18:24:42 -04004340 num_pages = num_extent_pages(eb->start, eb->len);
4341 for (i = 0; i < num_pages; i++) {
4342 struct page *p = extent_buffer_page(eb, i);
4343 mark_page_accessed(p);
4344 }
4345}
4346
Chris Masond1310b22008-01-24 16:13:08 -05004347struct extent_buffer *alloc_extent_buffer(struct extent_io_tree *tree,
Chris Mason727011e2010-08-06 13:21:20 -04004348 u64 start, unsigned long len)
Chris Masond1310b22008-01-24 16:13:08 -05004349{
4350 unsigned long num_pages = num_extent_pages(start, len);
4351 unsigned long i;
4352 unsigned long index = start >> PAGE_CACHE_SHIFT;
4353 struct extent_buffer *eb;
Chris Mason6af118ce2008-07-22 11:18:07 -04004354 struct extent_buffer *exists = NULL;
Chris Masond1310b22008-01-24 16:13:08 -05004355 struct page *p;
4356 struct address_space *mapping = tree->mapping;
4357 int uptodate = 1;
Miao Xie19fe0a82010-10-26 20:57:29 -04004358 int ret;
Chris Masond1310b22008-01-24 16:13:08 -05004359
Miao Xie19fe0a82010-10-26 20:57:29 -04004360 rcu_read_lock();
4361 eb = radix_tree_lookup(&tree->buffer, start >> PAGE_CACHE_SHIFT);
4362 if (eb && atomic_inc_not_zero(&eb->refs)) {
4363 rcu_read_unlock();
Josef Bacik5df42352012-03-15 18:24:42 -04004364 mark_extent_buffer_accessed(eb);
Chris Mason6af118ce2008-07-22 11:18:07 -04004365 return eb;
4366 }
Miao Xie19fe0a82010-10-26 20:57:29 -04004367 rcu_read_unlock();
Chris Mason6af118ce2008-07-22 11:18:07 -04004368
David Sterbaba144192011-04-21 01:12:06 +02004369 eb = __alloc_extent_buffer(tree, start, len, GFP_NOFS);
Peter2b114d12008-04-01 11:21:40 -04004370 if (!eb)
Chris Masond1310b22008-01-24 16:13:08 -05004371 return NULL;
4372
Chris Mason727011e2010-08-06 13:21:20 -04004373 for (i = 0; i < num_pages; i++, index++) {
Chris Masona6591712011-07-19 12:04:14 -04004374 p = find_or_create_page(mapping, index, GFP_NOFS);
Josef Bacik4804b382012-10-05 16:43:45 -04004375 if (!p)
Chris Mason6af118ce2008-07-22 11:18:07 -04004376 goto free_eb;
Josef Bacik4f2de97a2012-03-07 16:20:05 -05004377
4378 spin_lock(&mapping->private_lock);
4379 if (PagePrivate(p)) {
4380 /*
4381 * We could have already allocated an eb for this page
4382 * and attached one so lets see if we can get a ref on
4383 * the existing eb, and if we can we know it's good and
4384 * we can just return that one, else we know we can just
4385 * overwrite page->private.
4386 */
4387 exists = (struct extent_buffer *)p->private;
4388 if (atomic_inc_not_zero(&exists->refs)) {
4389 spin_unlock(&mapping->private_lock);
4390 unlock_page(p);
Josef Bacik17de39a2012-05-04 15:16:06 -04004391 page_cache_release(p);
Josef Bacik5df42352012-03-15 18:24:42 -04004392 mark_extent_buffer_accessed(exists);
Josef Bacik4f2de97a2012-03-07 16:20:05 -05004393 goto free_eb;
4394 }
4395
Josef Bacik0b32f4b2012-03-13 09:38:00 -04004396 /*
Josef Bacik4f2de97a2012-03-07 16:20:05 -05004397 * Do this so attach doesn't complain and we need to
4398 * drop the ref the old guy had.
4399 */
4400 ClearPagePrivate(p);
Josef Bacik0b32f4b2012-03-13 09:38:00 -04004401 WARN_ON(PageDirty(p));
Josef Bacik4f2de97a2012-03-07 16:20:05 -05004402 page_cache_release(p);
Chris Masond1310b22008-01-24 16:13:08 -05004403 }
Josef Bacik4f2de97a2012-03-07 16:20:05 -05004404 attach_extent_buffer_page(eb, p);
4405 spin_unlock(&mapping->private_lock);
Josef Bacik0b32f4b2012-03-13 09:38:00 -04004406 WARN_ON(PageDirty(p));
Chris Masond1310b22008-01-24 16:13:08 -05004407 mark_page_accessed(p);
Chris Mason727011e2010-08-06 13:21:20 -04004408 eb->pages[i] = p;
Chris Masond1310b22008-01-24 16:13:08 -05004409 if (!PageUptodate(p))
4410 uptodate = 0;
Chris Masoneb14ab82011-02-10 12:35:00 -05004411
4412 /*
4413 * see below about how we avoid a nasty race with release page
4414 * and why we unlock later
4415 */
Chris Masond1310b22008-01-24 16:13:08 -05004416 }
4417 if (uptodate)
Chris Masonb4ce94d2009-02-04 09:25:08 -05004418 set_bit(EXTENT_BUFFER_UPTODATE, &eb->bflags);
Josef Bacik115391d2012-03-09 09:51:43 -05004419again:
Miao Xie19fe0a82010-10-26 20:57:29 -04004420 ret = radix_tree_preload(GFP_NOFS & ~__GFP_HIGHMEM);
4421 if (ret)
4422 goto free_eb;
4423
Chris Mason6af118ce2008-07-22 11:18:07 -04004424 spin_lock(&tree->buffer_lock);
Miao Xie19fe0a82010-10-26 20:57:29 -04004425 ret = radix_tree_insert(&tree->buffer, start >> PAGE_CACHE_SHIFT, eb);
4426 if (ret == -EEXIST) {
4427 exists = radix_tree_lookup(&tree->buffer,
4428 start >> PAGE_CACHE_SHIFT);
Josef Bacik115391d2012-03-09 09:51:43 -05004429 if (!atomic_inc_not_zero(&exists->refs)) {
4430 spin_unlock(&tree->buffer_lock);
4431 radix_tree_preload_end();
Josef Bacik115391d2012-03-09 09:51:43 -05004432 exists = NULL;
4433 goto again;
4434 }
Chris Mason6af118ce2008-07-22 11:18:07 -04004435 spin_unlock(&tree->buffer_lock);
Miao Xie19fe0a82010-10-26 20:57:29 -04004436 radix_tree_preload_end();
Josef Bacik5df42352012-03-15 18:24:42 -04004437 mark_extent_buffer_accessed(exists);
Chris Mason6af118ce2008-07-22 11:18:07 -04004438 goto free_eb;
4439 }
Chris Mason6af118ce2008-07-22 11:18:07 -04004440 /* add one reference for the tree */
Josef Bacik0b32f4b2012-03-13 09:38:00 -04004441 check_buffer_tree_ref(eb);
Yan, Zhengf044ba72010-02-04 08:46:56 +00004442 spin_unlock(&tree->buffer_lock);
Miao Xie19fe0a82010-10-26 20:57:29 -04004443 radix_tree_preload_end();
Chris Masoneb14ab82011-02-10 12:35:00 -05004444
4445 /*
4446 * there is a race where release page may have
4447 * tried to find this extent buffer in the radix
4448 * but failed. It will tell the VM it is safe to
4449 * reclaim the, and it will clear the page private bit.
4450 * We must make sure to set the page private bit properly
4451 * after the extent buffer is in the radix tree so
4452 * it doesn't get lost
4453 */
Chris Mason727011e2010-08-06 13:21:20 -04004454 SetPageChecked(eb->pages[0]);
4455 for (i = 1; i < num_pages; i++) {
4456 p = extent_buffer_page(eb, i);
Chris Mason727011e2010-08-06 13:21:20 -04004457 ClearPageChecked(p);
4458 unlock_page(p);
4459 }
4460 unlock_page(eb->pages[0]);
Chris Masond1310b22008-01-24 16:13:08 -05004461 return eb;
4462
Chris Mason6af118ce2008-07-22 11:18:07 -04004463free_eb:
Chris Mason727011e2010-08-06 13:21:20 -04004464 for (i = 0; i < num_pages; i++) {
4465 if (eb->pages[i])
4466 unlock_page(eb->pages[i]);
4467 }
Chris Masoneb14ab82011-02-10 12:35:00 -05004468
Josef Bacik17de39a2012-05-04 15:16:06 -04004469 WARN_ON(!atomic_dec_and_test(&eb->refs));
Miao Xie897ca6e2010-10-26 20:57:29 -04004470 btrfs_release_extent_buffer(eb);
Chris Mason6af118ce2008-07-22 11:18:07 -04004471 return exists;
Chris Masond1310b22008-01-24 16:13:08 -05004472}
Chris Masond1310b22008-01-24 16:13:08 -05004473
4474struct extent_buffer *find_extent_buffer(struct extent_io_tree *tree,
David Sterbaf09d1f62011-04-21 01:08:01 +02004475 u64 start, unsigned long len)
Chris Masond1310b22008-01-24 16:13:08 -05004476{
Chris Masond1310b22008-01-24 16:13:08 -05004477 struct extent_buffer *eb;
Chris Masond1310b22008-01-24 16:13:08 -05004478
Miao Xie19fe0a82010-10-26 20:57:29 -04004479 rcu_read_lock();
4480 eb = radix_tree_lookup(&tree->buffer, start >> PAGE_CACHE_SHIFT);
4481 if (eb && atomic_inc_not_zero(&eb->refs)) {
4482 rcu_read_unlock();
Josef Bacik5df42352012-03-15 18:24:42 -04004483 mark_extent_buffer_accessed(eb);
Miao Xie19fe0a82010-10-26 20:57:29 -04004484 return eb;
4485 }
4486 rcu_read_unlock();
Josef Bacik0f9dd462008-09-23 13:14:11 -04004487
Miao Xie19fe0a82010-10-26 20:57:29 -04004488 return NULL;
Chris Masond1310b22008-01-24 16:13:08 -05004489}
Chris Masond1310b22008-01-24 16:13:08 -05004490
Josef Bacik3083ee22012-03-09 16:01:49 -05004491static inline void btrfs_release_extent_buffer_rcu(struct rcu_head *head)
4492{
4493 struct extent_buffer *eb =
4494 container_of(head, struct extent_buffer, rcu_head);
4495
4496 __free_extent_buffer(eb);
4497}
4498
Josef Bacik3083ee22012-03-09 16:01:49 -05004499/* Expects to have eb->eb_lock already held */
David Sterbaf7a52a42013-04-26 14:56:29 +00004500static int release_extent_buffer(struct extent_buffer *eb)
Josef Bacik3083ee22012-03-09 16:01:49 -05004501{
4502 WARN_ON(atomic_read(&eb->refs) == 0);
4503 if (atomic_dec_and_test(&eb->refs)) {
Jan Schmidt815a51c2012-05-16 17:00:02 +02004504 if (test_bit(EXTENT_BUFFER_DUMMY, &eb->bflags)) {
4505 spin_unlock(&eb->refs_lock);
4506 } else {
4507 struct extent_io_tree *tree = eb->tree;
Josef Bacik3083ee22012-03-09 16:01:49 -05004508
Jan Schmidt815a51c2012-05-16 17:00:02 +02004509 spin_unlock(&eb->refs_lock);
Josef Bacik3083ee22012-03-09 16:01:49 -05004510
Jan Schmidt815a51c2012-05-16 17:00:02 +02004511 spin_lock(&tree->buffer_lock);
4512 radix_tree_delete(&tree->buffer,
4513 eb->start >> PAGE_CACHE_SHIFT);
4514 spin_unlock(&tree->buffer_lock);
4515 }
Josef Bacik3083ee22012-03-09 16:01:49 -05004516
4517 /* Should be safe to release our pages at this point */
4518 btrfs_release_extent_buffer_page(eb, 0);
Josef Bacik3083ee22012-03-09 16:01:49 -05004519 call_rcu(&eb->rcu_head, btrfs_release_extent_buffer_rcu);
Josef Bacike64860a2012-07-20 16:05:36 -04004520 return 1;
Josef Bacik3083ee22012-03-09 16:01:49 -05004521 }
4522 spin_unlock(&eb->refs_lock);
Josef Bacike64860a2012-07-20 16:05:36 -04004523
4524 return 0;
Josef Bacik3083ee22012-03-09 16:01:49 -05004525}
4526
Chris Masond1310b22008-01-24 16:13:08 -05004527void free_extent_buffer(struct extent_buffer *eb)
4528{
Chris Mason242e18c2013-01-29 17:49:37 -05004529 int refs;
4530 int old;
Chris Masond1310b22008-01-24 16:13:08 -05004531 if (!eb)
4532 return;
4533
Chris Mason242e18c2013-01-29 17:49:37 -05004534 while (1) {
4535 refs = atomic_read(&eb->refs);
4536 if (refs <= 3)
4537 break;
4538 old = atomic_cmpxchg(&eb->refs, refs, refs - 1);
4539 if (old == refs)
4540 return;
4541 }
4542
Josef Bacik3083ee22012-03-09 16:01:49 -05004543 spin_lock(&eb->refs_lock);
4544 if (atomic_read(&eb->refs) == 2 &&
Jan Schmidt815a51c2012-05-16 17:00:02 +02004545 test_bit(EXTENT_BUFFER_DUMMY, &eb->bflags))
4546 atomic_dec(&eb->refs);
4547
4548 if (atomic_read(&eb->refs) == 2 &&
Josef Bacik3083ee22012-03-09 16:01:49 -05004549 test_bit(EXTENT_BUFFER_STALE, &eb->bflags) &&
Josef Bacik0b32f4b2012-03-13 09:38:00 -04004550 !extent_buffer_under_io(eb) &&
Josef Bacik3083ee22012-03-09 16:01:49 -05004551 test_and_clear_bit(EXTENT_BUFFER_TREE_REF, &eb->bflags))
4552 atomic_dec(&eb->refs);
Chris Masond1310b22008-01-24 16:13:08 -05004553
Josef Bacik3083ee22012-03-09 16:01:49 -05004554 /*
4555 * I know this is terrible, but it's temporary until we stop tracking
4556 * the uptodate bits and such for the extent buffers.
4557 */
David Sterbaf7a52a42013-04-26 14:56:29 +00004558 release_extent_buffer(eb);
Chris Masond1310b22008-01-24 16:13:08 -05004559}
Chris Masond1310b22008-01-24 16:13:08 -05004560
Josef Bacik3083ee22012-03-09 16:01:49 -05004561void free_extent_buffer_stale(struct extent_buffer *eb)
4562{
4563 if (!eb)
Chris Masond1310b22008-01-24 16:13:08 -05004564 return;
4565
Josef Bacik3083ee22012-03-09 16:01:49 -05004566 spin_lock(&eb->refs_lock);
4567 set_bit(EXTENT_BUFFER_STALE, &eb->bflags);
4568
Josef Bacik0b32f4b2012-03-13 09:38:00 -04004569 if (atomic_read(&eb->refs) == 2 && !extent_buffer_under_io(eb) &&
Josef Bacik3083ee22012-03-09 16:01:49 -05004570 test_and_clear_bit(EXTENT_BUFFER_TREE_REF, &eb->bflags))
4571 atomic_dec(&eb->refs);
David Sterbaf7a52a42013-04-26 14:56:29 +00004572 release_extent_buffer(eb);
Chris Masond1310b22008-01-24 16:13:08 -05004573}
4574
Chris Mason1d4284b2012-03-28 20:31:37 -04004575void clear_extent_buffer_dirty(struct extent_buffer *eb)
Chris Masond1310b22008-01-24 16:13:08 -05004576{
Chris Masond1310b22008-01-24 16:13:08 -05004577 unsigned long i;
4578 unsigned long num_pages;
4579 struct page *page;
4580
Chris Masond1310b22008-01-24 16:13:08 -05004581 num_pages = num_extent_pages(eb->start, eb->len);
4582
4583 for (i = 0; i < num_pages; i++) {
4584 page = extent_buffer_page(eb, i);
Chris Masonb9473432009-03-13 11:00:37 -04004585 if (!PageDirty(page))
Chris Masond2c3f4f2008-11-19 12:44:22 -05004586 continue;
4587
Chris Masona61e6f22008-07-22 11:18:08 -04004588 lock_page(page);
Chris Masoneb14ab82011-02-10 12:35:00 -05004589 WARN_ON(!PagePrivate(page));
4590
Chris Masond1310b22008-01-24 16:13:08 -05004591 clear_page_dirty_for_io(page);
Sven Wegener0ee0fda2008-07-30 16:54:26 -04004592 spin_lock_irq(&page->mapping->tree_lock);
Chris Masond1310b22008-01-24 16:13:08 -05004593 if (!PageDirty(page)) {
4594 radix_tree_tag_clear(&page->mapping->page_tree,
4595 page_index(page),
4596 PAGECACHE_TAG_DIRTY);
4597 }
Sven Wegener0ee0fda2008-07-30 16:54:26 -04004598 spin_unlock_irq(&page->mapping->tree_lock);
Chris Masonbf0da8c2011-11-04 12:29:37 -04004599 ClearPageError(page);
Chris Masona61e6f22008-07-22 11:18:08 -04004600 unlock_page(page);
Chris Masond1310b22008-01-24 16:13:08 -05004601 }
Josef Bacik0b32f4b2012-03-13 09:38:00 -04004602 WARN_ON(atomic_read(&eb->refs) == 0);
Chris Masond1310b22008-01-24 16:13:08 -05004603}
Chris Masond1310b22008-01-24 16:13:08 -05004604
Josef Bacik0b32f4b2012-03-13 09:38:00 -04004605int set_extent_buffer_dirty(struct extent_buffer *eb)
Chris Masond1310b22008-01-24 16:13:08 -05004606{
4607 unsigned long i;
4608 unsigned long num_pages;
Chris Masonb9473432009-03-13 11:00:37 -04004609 int was_dirty = 0;
Chris Masond1310b22008-01-24 16:13:08 -05004610
Josef Bacik0b32f4b2012-03-13 09:38:00 -04004611 check_buffer_tree_ref(eb);
4612
Chris Masonb9473432009-03-13 11:00:37 -04004613 was_dirty = test_and_set_bit(EXTENT_BUFFER_DIRTY, &eb->bflags);
Josef Bacik0b32f4b2012-03-13 09:38:00 -04004614
Chris Masond1310b22008-01-24 16:13:08 -05004615 num_pages = num_extent_pages(eb->start, eb->len);
Josef Bacik3083ee22012-03-09 16:01:49 -05004616 WARN_ON(atomic_read(&eb->refs) == 0);
Josef Bacik0b32f4b2012-03-13 09:38:00 -04004617 WARN_ON(!test_bit(EXTENT_BUFFER_TREE_REF, &eb->bflags));
4618
Chris Masonb9473432009-03-13 11:00:37 -04004619 for (i = 0; i < num_pages; i++)
Josef Bacik0b32f4b2012-03-13 09:38:00 -04004620 set_page_dirty(extent_buffer_page(eb, i));
Chris Masonb9473432009-03-13 11:00:37 -04004621 return was_dirty;
Chris Masond1310b22008-01-24 16:13:08 -05004622}
Chris Masond1310b22008-01-24 16:13:08 -05004623
Josef Bacik0b32f4b2012-03-13 09:38:00 -04004624int clear_extent_buffer_uptodate(struct extent_buffer *eb)
Chris Mason1259ab72008-05-12 13:39:03 -04004625{
4626 unsigned long i;
4627 struct page *page;
4628 unsigned long num_pages;
4629
Chris Masonb4ce94d2009-02-04 09:25:08 -05004630 clear_bit(EXTENT_BUFFER_UPTODATE, &eb->bflags);
Josef Bacik0b32f4b2012-03-13 09:38:00 -04004631 num_pages = num_extent_pages(eb->start, eb->len);
Chris Mason1259ab72008-05-12 13:39:03 -04004632 for (i = 0; i < num_pages; i++) {
4633 page = extent_buffer_page(eb, i);
Chris Mason33958dc2008-07-30 10:29:12 -04004634 if (page)
4635 ClearPageUptodate(page);
Chris Mason1259ab72008-05-12 13:39:03 -04004636 }
4637 return 0;
4638}
4639
Josef Bacik0b32f4b2012-03-13 09:38:00 -04004640int set_extent_buffer_uptodate(struct extent_buffer *eb)
Chris Masond1310b22008-01-24 16:13:08 -05004641{
4642 unsigned long i;
4643 struct page *page;
4644 unsigned long num_pages;
4645
Josef Bacik0b32f4b2012-03-13 09:38:00 -04004646 set_bit(EXTENT_BUFFER_UPTODATE, &eb->bflags);
Chris Masond1310b22008-01-24 16:13:08 -05004647 num_pages = num_extent_pages(eb->start, eb->len);
Chris Masond1310b22008-01-24 16:13:08 -05004648 for (i = 0; i < num_pages; i++) {
4649 page = extent_buffer_page(eb, i);
Chris Masond1310b22008-01-24 16:13:08 -05004650 SetPageUptodate(page);
4651 }
4652 return 0;
4653}
Chris Masond1310b22008-01-24 16:13:08 -05004654
Josef Bacik0b32f4b2012-03-13 09:38:00 -04004655int extent_buffer_uptodate(struct extent_buffer *eb)
Chris Masond1310b22008-01-24 16:13:08 -05004656{
Josef Bacik0b32f4b2012-03-13 09:38:00 -04004657 return test_bit(EXTENT_BUFFER_UPTODATE, &eb->bflags);
Chris Masond1310b22008-01-24 16:13:08 -05004658}
Chris Masond1310b22008-01-24 16:13:08 -05004659
4660int read_extent_buffer_pages(struct extent_io_tree *tree,
Arne Jansenbb82ab82011-06-10 14:06:53 +02004661 struct extent_buffer *eb, u64 start, int wait,
Chris Masonf1885912008-04-09 16:28:12 -04004662 get_extent_t *get_extent, int mirror_num)
Chris Masond1310b22008-01-24 16:13:08 -05004663{
4664 unsigned long i;
4665 unsigned long start_i;
4666 struct page *page;
4667 int err;
4668 int ret = 0;
Chris Masonce9adaa2008-04-09 16:28:12 -04004669 int locked_pages = 0;
4670 int all_uptodate = 1;
Chris Masond1310b22008-01-24 16:13:08 -05004671 unsigned long num_pages;
Chris Mason727011e2010-08-06 13:21:20 -04004672 unsigned long num_reads = 0;
Chris Masona86c12c2008-02-07 10:50:54 -05004673 struct bio *bio = NULL;
Chris Masonc8b97812008-10-29 14:49:59 -04004674 unsigned long bio_flags = 0;
Chris Masona86c12c2008-02-07 10:50:54 -05004675
Chris Masonb4ce94d2009-02-04 09:25:08 -05004676 if (test_bit(EXTENT_BUFFER_UPTODATE, &eb->bflags))
Chris Masond1310b22008-01-24 16:13:08 -05004677 return 0;
4678
Chris Masond1310b22008-01-24 16:13:08 -05004679 if (start) {
4680 WARN_ON(start < eb->start);
4681 start_i = (start >> PAGE_CACHE_SHIFT) -
4682 (eb->start >> PAGE_CACHE_SHIFT);
4683 } else {
4684 start_i = 0;
4685 }
4686
4687 num_pages = num_extent_pages(eb->start, eb->len);
4688 for (i = start_i; i < num_pages; i++) {
4689 page = extent_buffer_page(eb, i);
Arne Jansenbb82ab82011-06-10 14:06:53 +02004690 if (wait == WAIT_NONE) {
David Woodhouse2db04962008-08-07 11:19:43 -04004691 if (!trylock_page(page))
Chris Masonce9adaa2008-04-09 16:28:12 -04004692 goto unlock_exit;
Chris Masond1310b22008-01-24 16:13:08 -05004693 } else {
4694 lock_page(page);
4695 }
Chris Masonce9adaa2008-04-09 16:28:12 -04004696 locked_pages++;
Chris Mason727011e2010-08-06 13:21:20 -04004697 if (!PageUptodate(page)) {
4698 num_reads++;
Chris Masonce9adaa2008-04-09 16:28:12 -04004699 all_uptodate = 0;
Chris Mason727011e2010-08-06 13:21:20 -04004700 }
Chris Masonce9adaa2008-04-09 16:28:12 -04004701 }
4702 if (all_uptodate) {
4703 if (start_i == 0)
Chris Masonb4ce94d2009-02-04 09:25:08 -05004704 set_bit(EXTENT_BUFFER_UPTODATE, &eb->bflags);
Chris Masonce9adaa2008-04-09 16:28:12 -04004705 goto unlock_exit;
4706 }
4707
Josef Bacikea466792012-03-26 21:57:36 -04004708 clear_bit(EXTENT_BUFFER_IOERR, &eb->bflags);
Josef Bacik5cf1ab52012-04-16 09:42:26 -04004709 eb->read_mirror = 0;
Josef Bacik0b32f4b2012-03-13 09:38:00 -04004710 atomic_set(&eb->io_pages, num_reads);
Chris Masonce9adaa2008-04-09 16:28:12 -04004711 for (i = start_i; i < num_pages; i++) {
4712 page = extent_buffer_page(eb, i);
Chris Masonce9adaa2008-04-09 16:28:12 -04004713 if (!PageUptodate(page)) {
Chris Masonf1885912008-04-09 16:28:12 -04004714 ClearPageError(page);
Chris Masona86c12c2008-02-07 10:50:54 -05004715 err = __extent_read_full_page(tree, page,
Chris Masonf1885912008-04-09 16:28:12 -04004716 get_extent, &bio,
Josef Bacikd4c7ca82013-04-19 19:49:09 -04004717 mirror_num, &bio_flags,
4718 READ | REQ_META);
Chris Masond3977122009-01-05 21:25:51 -05004719 if (err)
Chris Masond1310b22008-01-24 16:13:08 -05004720 ret = err;
Chris Masond1310b22008-01-24 16:13:08 -05004721 } else {
4722 unlock_page(page);
4723 }
4724 }
4725
Jeff Mahoney355808c2011-10-03 23:23:14 -04004726 if (bio) {
Josef Bacikd4c7ca82013-04-19 19:49:09 -04004727 err = submit_one_bio(READ | REQ_META, bio, mirror_num,
4728 bio_flags);
Jeff Mahoney79787ea2012-03-12 16:03:00 +01004729 if (err)
4730 return err;
Jeff Mahoney355808c2011-10-03 23:23:14 -04004731 }
Chris Masona86c12c2008-02-07 10:50:54 -05004732
Arne Jansenbb82ab82011-06-10 14:06:53 +02004733 if (ret || wait != WAIT_COMPLETE)
Chris Masond1310b22008-01-24 16:13:08 -05004734 return ret;
Chris Masond3977122009-01-05 21:25:51 -05004735
Chris Masond1310b22008-01-24 16:13:08 -05004736 for (i = start_i; i < num_pages; i++) {
4737 page = extent_buffer_page(eb, i);
4738 wait_on_page_locked(page);
Chris Masond3977122009-01-05 21:25:51 -05004739 if (!PageUptodate(page))
Chris Masond1310b22008-01-24 16:13:08 -05004740 ret = -EIO;
Chris Masond1310b22008-01-24 16:13:08 -05004741 }
Chris Masond3977122009-01-05 21:25:51 -05004742
Chris Masond1310b22008-01-24 16:13:08 -05004743 return ret;
Chris Masonce9adaa2008-04-09 16:28:12 -04004744
4745unlock_exit:
4746 i = start_i;
Chris Masond3977122009-01-05 21:25:51 -05004747 while (locked_pages > 0) {
Chris Masonce9adaa2008-04-09 16:28:12 -04004748 page = extent_buffer_page(eb, i);
4749 i++;
4750 unlock_page(page);
4751 locked_pages--;
4752 }
4753 return ret;
Chris Masond1310b22008-01-24 16:13:08 -05004754}
Chris Masond1310b22008-01-24 16:13:08 -05004755
4756void read_extent_buffer(struct extent_buffer *eb, void *dstv,
4757 unsigned long start,
4758 unsigned long len)
4759{
4760 size_t cur;
4761 size_t offset;
4762 struct page *page;
4763 char *kaddr;
4764 char *dst = (char *)dstv;
4765 size_t start_offset = eb->start & ((u64)PAGE_CACHE_SIZE - 1);
4766 unsigned long i = (start_offset + start) >> PAGE_CACHE_SHIFT;
Chris Masond1310b22008-01-24 16:13:08 -05004767
4768 WARN_ON(start > eb->len);
4769 WARN_ON(start + len > eb->start + eb->len);
4770
4771 offset = (start_offset + start) & ((unsigned long)PAGE_CACHE_SIZE - 1);
4772
Chris Masond3977122009-01-05 21:25:51 -05004773 while (len > 0) {
Chris Masond1310b22008-01-24 16:13:08 -05004774 page = extent_buffer_page(eb, i);
Chris Masond1310b22008-01-24 16:13:08 -05004775
4776 cur = min(len, (PAGE_CACHE_SIZE - offset));
Chris Masona6591712011-07-19 12:04:14 -04004777 kaddr = page_address(page);
Chris Masond1310b22008-01-24 16:13:08 -05004778 memcpy(dst, kaddr + offset, cur);
Chris Masond1310b22008-01-24 16:13:08 -05004779
4780 dst += cur;
4781 len -= cur;
4782 offset = 0;
4783 i++;
4784 }
4785}
Chris Masond1310b22008-01-24 16:13:08 -05004786
4787int map_private_extent_buffer(struct extent_buffer *eb, unsigned long start,
Chris Masona6591712011-07-19 12:04:14 -04004788 unsigned long min_len, char **map,
Chris Masond1310b22008-01-24 16:13:08 -05004789 unsigned long *map_start,
Chris Masona6591712011-07-19 12:04:14 -04004790 unsigned long *map_len)
Chris Masond1310b22008-01-24 16:13:08 -05004791{
4792 size_t offset = start & (PAGE_CACHE_SIZE - 1);
4793 char *kaddr;
4794 struct page *p;
4795 size_t start_offset = eb->start & ((u64)PAGE_CACHE_SIZE - 1);
4796 unsigned long i = (start_offset + start) >> PAGE_CACHE_SHIFT;
4797 unsigned long end_i = (start_offset + start + min_len - 1) >>
4798 PAGE_CACHE_SHIFT;
4799
4800 if (i != end_i)
4801 return -EINVAL;
4802
4803 if (i == 0) {
4804 offset = start_offset;
4805 *map_start = 0;
4806 } else {
4807 offset = 0;
4808 *map_start = ((u64)i << PAGE_CACHE_SHIFT) - start_offset;
4809 }
Chris Masond3977122009-01-05 21:25:51 -05004810
Chris Masond1310b22008-01-24 16:13:08 -05004811 if (start + min_len > eb->len) {
Julia Lawall31b1a2b2012-11-03 10:58:34 +00004812 WARN(1, KERN_ERR "btrfs bad mapping eb start %llu len %lu, "
Chris Masond3977122009-01-05 21:25:51 -05004813 "wanted %lu %lu\n", (unsigned long long)eb->start,
4814 eb->len, start, min_len);
Josef Bacik850265332011-03-15 14:52:12 -04004815 return -EINVAL;
Chris Masond1310b22008-01-24 16:13:08 -05004816 }
4817
4818 p = extent_buffer_page(eb, i);
Chris Masona6591712011-07-19 12:04:14 -04004819 kaddr = page_address(p);
Chris Masond1310b22008-01-24 16:13:08 -05004820 *map = kaddr + offset;
4821 *map_len = PAGE_CACHE_SIZE - offset;
4822 return 0;
4823}
Chris Masond1310b22008-01-24 16:13:08 -05004824
Chris Masond1310b22008-01-24 16:13:08 -05004825int memcmp_extent_buffer(struct extent_buffer *eb, const void *ptrv,
4826 unsigned long start,
4827 unsigned long len)
4828{
4829 size_t cur;
4830 size_t offset;
4831 struct page *page;
4832 char *kaddr;
4833 char *ptr = (char *)ptrv;
4834 size_t start_offset = eb->start & ((u64)PAGE_CACHE_SIZE - 1);
4835 unsigned long i = (start_offset + start) >> PAGE_CACHE_SHIFT;
4836 int ret = 0;
4837
4838 WARN_ON(start > eb->len);
4839 WARN_ON(start + len > eb->start + eb->len);
4840
4841 offset = (start_offset + start) & ((unsigned long)PAGE_CACHE_SIZE - 1);
4842
Chris Masond3977122009-01-05 21:25:51 -05004843 while (len > 0) {
Chris Masond1310b22008-01-24 16:13:08 -05004844 page = extent_buffer_page(eb, i);
Chris Masond1310b22008-01-24 16:13:08 -05004845
4846 cur = min(len, (PAGE_CACHE_SIZE - offset));
4847
Chris Masona6591712011-07-19 12:04:14 -04004848 kaddr = page_address(page);
Chris Masond1310b22008-01-24 16:13:08 -05004849 ret = memcmp(ptr, kaddr + offset, cur);
Chris Masond1310b22008-01-24 16:13:08 -05004850 if (ret)
4851 break;
4852
4853 ptr += cur;
4854 len -= cur;
4855 offset = 0;
4856 i++;
4857 }
4858 return ret;
4859}
Chris Masond1310b22008-01-24 16:13:08 -05004860
4861void write_extent_buffer(struct extent_buffer *eb, const void *srcv,
4862 unsigned long start, unsigned long len)
4863{
4864 size_t cur;
4865 size_t offset;
4866 struct page *page;
4867 char *kaddr;
4868 char *src = (char *)srcv;
4869 size_t start_offset = eb->start & ((u64)PAGE_CACHE_SIZE - 1);
4870 unsigned long i = (start_offset + start) >> PAGE_CACHE_SHIFT;
4871
4872 WARN_ON(start > eb->len);
4873 WARN_ON(start + len > eb->start + eb->len);
4874
4875 offset = (start_offset + start) & ((unsigned long)PAGE_CACHE_SIZE - 1);
4876
Chris Masond3977122009-01-05 21:25:51 -05004877 while (len > 0) {
Chris Masond1310b22008-01-24 16:13:08 -05004878 page = extent_buffer_page(eb, i);
4879 WARN_ON(!PageUptodate(page));
4880
4881 cur = min(len, PAGE_CACHE_SIZE - offset);
Chris Masona6591712011-07-19 12:04:14 -04004882 kaddr = page_address(page);
Chris Masond1310b22008-01-24 16:13:08 -05004883 memcpy(kaddr + offset, src, cur);
Chris Masond1310b22008-01-24 16:13:08 -05004884
4885 src += cur;
4886 len -= cur;
4887 offset = 0;
4888 i++;
4889 }
4890}
Chris Masond1310b22008-01-24 16:13:08 -05004891
4892void memset_extent_buffer(struct extent_buffer *eb, char c,
4893 unsigned long start, unsigned long len)
4894{
4895 size_t cur;
4896 size_t offset;
4897 struct page *page;
4898 char *kaddr;
4899 size_t start_offset = eb->start & ((u64)PAGE_CACHE_SIZE - 1);
4900 unsigned long i = (start_offset + start) >> PAGE_CACHE_SHIFT;
4901
4902 WARN_ON(start > eb->len);
4903 WARN_ON(start + len > eb->start + eb->len);
4904
4905 offset = (start_offset + start) & ((unsigned long)PAGE_CACHE_SIZE - 1);
4906
Chris Masond3977122009-01-05 21:25:51 -05004907 while (len > 0) {
Chris Masond1310b22008-01-24 16:13:08 -05004908 page = extent_buffer_page(eb, i);
4909 WARN_ON(!PageUptodate(page));
4910
4911 cur = min(len, PAGE_CACHE_SIZE - offset);
Chris Masona6591712011-07-19 12:04:14 -04004912 kaddr = page_address(page);
Chris Masond1310b22008-01-24 16:13:08 -05004913 memset(kaddr + offset, c, cur);
Chris Masond1310b22008-01-24 16:13:08 -05004914
4915 len -= cur;
4916 offset = 0;
4917 i++;
4918 }
4919}
Chris Masond1310b22008-01-24 16:13:08 -05004920
4921void copy_extent_buffer(struct extent_buffer *dst, struct extent_buffer *src,
4922 unsigned long dst_offset, unsigned long src_offset,
4923 unsigned long len)
4924{
4925 u64 dst_len = dst->len;
4926 size_t cur;
4927 size_t offset;
4928 struct page *page;
4929 char *kaddr;
4930 size_t start_offset = dst->start & ((u64)PAGE_CACHE_SIZE - 1);
4931 unsigned long i = (start_offset + dst_offset) >> PAGE_CACHE_SHIFT;
4932
4933 WARN_ON(src->len != dst_len);
4934
4935 offset = (start_offset + dst_offset) &
4936 ((unsigned long)PAGE_CACHE_SIZE - 1);
4937
Chris Masond3977122009-01-05 21:25:51 -05004938 while (len > 0) {
Chris Masond1310b22008-01-24 16:13:08 -05004939 page = extent_buffer_page(dst, i);
4940 WARN_ON(!PageUptodate(page));
4941
4942 cur = min(len, (unsigned long)(PAGE_CACHE_SIZE - offset));
4943
Chris Masona6591712011-07-19 12:04:14 -04004944 kaddr = page_address(page);
Chris Masond1310b22008-01-24 16:13:08 -05004945 read_extent_buffer(src, kaddr + offset, src_offset, cur);
Chris Masond1310b22008-01-24 16:13:08 -05004946
4947 src_offset += cur;
4948 len -= cur;
4949 offset = 0;
4950 i++;
4951 }
4952}
Chris Masond1310b22008-01-24 16:13:08 -05004953
4954static void move_pages(struct page *dst_page, struct page *src_page,
4955 unsigned long dst_off, unsigned long src_off,
4956 unsigned long len)
4957{
Chris Masona6591712011-07-19 12:04:14 -04004958 char *dst_kaddr = page_address(dst_page);
Chris Masond1310b22008-01-24 16:13:08 -05004959 if (dst_page == src_page) {
4960 memmove(dst_kaddr + dst_off, dst_kaddr + src_off, len);
4961 } else {
Chris Masona6591712011-07-19 12:04:14 -04004962 char *src_kaddr = page_address(src_page);
Chris Masond1310b22008-01-24 16:13:08 -05004963 char *p = dst_kaddr + dst_off + len;
4964 char *s = src_kaddr + src_off + len;
4965
4966 while (len--)
4967 *--p = *--s;
Chris Masond1310b22008-01-24 16:13:08 -05004968 }
Chris Masond1310b22008-01-24 16:13:08 -05004969}
4970
Sergei Trofimovich33872062011-04-11 21:52:52 +00004971static inline bool areas_overlap(unsigned long src, unsigned long dst, unsigned long len)
4972{
4973 unsigned long distance = (src > dst) ? src - dst : dst - src;
4974 return distance < len;
4975}
4976
Chris Masond1310b22008-01-24 16:13:08 -05004977static void copy_pages(struct page *dst_page, struct page *src_page,
4978 unsigned long dst_off, unsigned long src_off,
4979 unsigned long len)
4980{
Chris Masona6591712011-07-19 12:04:14 -04004981 char *dst_kaddr = page_address(dst_page);
Chris Masond1310b22008-01-24 16:13:08 -05004982 char *src_kaddr;
Chris Mason727011e2010-08-06 13:21:20 -04004983 int must_memmove = 0;
Chris Masond1310b22008-01-24 16:13:08 -05004984
Sergei Trofimovich33872062011-04-11 21:52:52 +00004985 if (dst_page != src_page) {
Chris Masona6591712011-07-19 12:04:14 -04004986 src_kaddr = page_address(src_page);
Sergei Trofimovich33872062011-04-11 21:52:52 +00004987 } else {
Chris Masond1310b22008-01-24 16:13:08 -05004988 src_kaddr = dst_kaddr;
Chris Mason727011e2010-08-06 13:21:20 -04004989 if (areas_overlap(src_off, dst_off, len))
4990 must_memmove = 1;
Sergei Trofimovich33872062011-04-11 21:52:52 +00004991 }
Chris Masond1310b22008-01-24 16:13:08 -05004992
Chris Mason727011e2010-08-06 13:21:20 -04004993 if (must_memmove)
4994 memmove(dst_kaddr + dst_off, src_kaddr + src_off, len);
4995 else
4996 memcpy(dst_kaddr + dst_off, src_kaddr + src_off, len);
Chris Masond1310b22008-01-24 16:13:08 -05004997}
4998
4999void memcpy_extent_buffer(struct extent_buffer *dst, unsigned long dst_offset,
5000 unsigned long src_offset, unsigned long len)
5001{
5002 size_t cur;
5003 size_t dst_off_in_page;
5004 size_t src_off_in_page;
5005 size_t start_offset = dst->start & ((u64)PAGE_CACHE_SIZE - 1);
5006 unsigned long dst_i;
5007 unsigned long src_i;
5008
5009 if (src_offset + len > dst->len) {
Chris Masond3977122009-01-05 21:25:51 -05005010 printk(KERN_ERR "btrfs memmove bogus src_offset %lu move "
5011 "len %lu dst len %lu\n", src_offset, len, dst->len);
Chris Masond1310b22008-01-24 16:13:08 -05005012 BUG_ON(1);
5013 }
5014 if (dst_offset + len > dst->len) {
Chris Masond3977122009-01-05 21:25:51 -05005015 printk(KERN_ERR "btrfs memmove bogus dst_offset %lu move "
5016 "len %lu dst len %lu\n", dst_offset, len, dst->len);
Chris Masond1310b22008-01-24 16:13:08 -05005017 BUG_ON(1);
5018 }
5019
Chris Masond3977122009-01-05 21:25:51 -05005020 while (len > 0) {
Chris Masond1310b22008-01-24 16:13:08 -05005021 dst_off_in_page = (start_offset + dst_offset) &
5022 ((unsigned long)PAGE_CACHE_SIZE - 1);
5023 src_off_in_page = (start_offset + src_offset) &
5024 ((unsigned long)PAGE_CACHE_SIZE - 1);
5025
5026 dst_i = (start_offset + dst_offset) >> PAGE_CACHE_SHIFT;
5027 src_i = (start_offset + src_offset) >> PAGE_CACHE_SHIFT;
5028
5029 cur = min(len, (unsigned long)(PAGE_CACHE_SIZE -
5030 src_off_in_page));
5031 cur = min_t(unsigned long, cur,
5032 (unsigned long)(PAGE_CACHE_SIZE - dst_off_in_page));
5033
5034 copy_pages(extent_buffer_page(dst, dst_i),
5035 extent_buffer_page(dst, src_i),
5036 dst_off_in_page, src_off_in_page, cur);
5037
5038 src_offset += cur;
5039 dst_offset += cur;
5040 len -= cur;
5041 }
5042}
Chris Masond1310b22008-01-24 16:13:08 -05005043
5044void memmove_extent_buffer(struct extent_buffer *dst, unsigned long dst_offset,
5045 unsigned long src_offset, unsigned long len)
5046{
5047 size_t cur;
5048 size_t dst_off_in_page;
5049 size_t src_off_in_page;
5050 unsigned long dst_end = dst_offset + len - 1;
5051 unsigned long src_end = src_offset + len - 1;
5052 size_t start_offset = dst->start & ((u64)PAGE_CACHE_SIZE - 1);
5053 unsigned long dst_i;
5054 unsigned long src_i;
5055
5056 if (src_offset + len > dst->len) {
Chris Masond3977122009-01-05 21:25:51 -05005057 printk(KERN_ERR "btrfs memmove bogus src_offset %lu move "
5058 "len %lu len %lu\n", src_offset, len, dst->len);
Chris Masond1310b22008-01-24 16:13:08 -05005059 BUG_ON(1);
5060 }
5061 if (dst_offset + len > dst->len) {
Chris Masond3977122009-01-05 21:25:51 -05005062 printk(KERN_ERR "btrfs memmove bogus dst_offset %lu move "
5063 "len %lu len %lu\n", dst_offset, len, dst->len);
Chris Masond1310b22008-01-24 16:13:08 -05005064 BUG_ON(1);
5065 }
Chris Mason727011e2010-08-06 13:21:20 -04005066 if (dst_offset < src_offset) {
Chris Masond1310b22008-01-24 16:13:08 -05005067 memcpy_extent_buffer(dst, dst_offset, src_offset, len);
5068 return;
5069 }
Chris Masond3977122009-01-05 21:25:51 -05005070 while (len > 0) {
Chris Masond1310b22008-01-24 16:13:08 -05005071 dst_i = (start_offset + dst_end) >> PAGE_CACHE_SHIFT;
5072 src_i = (start_offset + src_end) >> PAGE_CACHE_SHIFT;
5073
5074 dst_off_in_page = (start_offset + dst_end) &
5075 ((unsigned long)PAGE_CACHE_SIZE - 1);
5076 src_off_in_page = (start_offset + src_end) &
5077 ((unsigned long)PAGE_CACHE_SIZE - 1);
5078
5079 cur = min_t(unsigned long, len, src_off_in_page + 1);
5080 cur = min(cur, dst_off_in_page + 1);
5081 move_pages(extent_buffer_page(dst, dst_i),
5082 extent_buffer_page(dst, src_i),
5083 dst_off_in_page - cur + 1,
5084 src_off_in_page - cur + 1, cur);
5085
5086 dst_end -= cur;
5087 src_end -= cur;
5088 len -= cur;
5089 }
5090}
Chris Mason6af118ce2008-07-22 11:18:07 -04005091
David Sterbaf7a52a42013-04-26 14:56:29 +00005092int try_release_extent_buffer(struct page *page)
Miao Xie19fe0a82010-10-26 20:57:29 -04005093{
Chris Mason6af118ce2008-07-22 11:18:07 -04005094 struct extent_buffer *eb;
Miao Xie897ca6e2010-10-26 20:57:29 -04005095
Miao Xie19fe0a82010-10-26 20:57:29 -04005096 /*
Josef Bacik3083ee22012-03-09 16:01:49 -05005097 * We need to make sure noboody is attaching this page to an eb right
5098 * now.
Miao Xie19fe0a82010-10-26 20:57:29 -04005099 */
Josef Bacik3083ee22012-03-09 16:01:49 -05005100 spin_lock(&page->mapping->private_lock);
5101 if (!PagePrivate(page)) {
5102 spin_unlock(&page->mapping->private_lock);
5103 return 1;
Miao Xie19fe0a82010-10-26 20:57:29 -04005104 }
5105
Josef Bacik3083ee22012-03-09 16:01:49 -05005106 eb = (struct extent_buffer *)page->private;
5107 BUG_ON(!eb);
Miao Xie19fe0a82010-10-26 20:57:29 -04005108
Josef Bacik0b32f4b2012-03-13 09:38:00 -04005109 /*
Josef Bacik3083ee22012-03-09 16:01:49 -05005110 * This is a little awful but should be ok, we need to make sure that
5111 * the eb doesn't disappear out from under us while we're looking at
5112 * this page.
5113 */
5114 spin_lock(&eb->refs_lock);
Josef Bacik0b32f4b2012-03-13 09:38:00 -04005115 if (atomic_read(&eb->refs) != 1 || extent_buffer_under_io(eb)) {
Josef Bacik3083ee22012-03-09 16:01:49 -05005116 spin_unlock(&eb->refs_lock);
5117 spin_unlock(&page->mapping->private_lock);
5118 return 0;
5119 }
5120 spin_unlock(&page->mapping->private_lock);
5121
Josef Bacik3083ee22012-03-09 16:01:49 -05005122 /*
5123 * If tree ref isn't set then we know the ref on this eb is a real ref,
5124 * so just return, this page will likely be freed soon anyway.
5125 */
5126 if (!test_and_clear_bit(EXTENT_BUFFER_TREE_REF, &eb->bflags)) {
5127 spin_unlock(&eb->refs_lock);
5128 return 0;
5129 }
Josef Bacik3083ee22012-03-09 16:01:49 -05005130
David Sterbaf7a52a42013-04-26 14:56:29 +00005131 return release_extent_buffer(eb);
Chris Mason6af118ce2008-07-22 11:18:07 -04005132}