blob: a389820d158b5b7ae64a2c6dfecabbb6ed3ed9bb [file] [log] [blame]
Chris Masond1310b22008-01-24 16:13:08 -05001#include <linux/bitops.h>
2#include <linux/slab.h>
3#include <linux/bio.h>
4#include <linux/mm.h>
Chris Masond1310b22008-01-24 16:13:08 -05005#include <linux/pagemap.h>
6#include <linux/page-flags.h>
Chris Masond1310b22008-01-24 16:13:08 -05007#include <linux/spinlock.h>
8#include <linux/blkdev.h>
9#include <linux/swap.h>
Chris Masond1310b22008-01-24 16:13:08 -050010#include <linux/writeback.h>
11#include <linux/pagevec.h>
Linus Torvalds268bb0c2011-05-20 12:50:29 -070012#include <linux/prefetch.h>
Dan Magenheimer90a887c2011-05-26 10:01:56 -060013#include <linux/cleancache.h>
Chris Masond1310b22008-01-24 16:13:08 -050014#include "extent_io.h"
15#include "extent_map.h"
David Woodhouse902b22f2008-08-20 08:51:49 -040016#include "ctree.h"
17#include "btrfs_inode.h"
Jan Schmidt4a54c8c2011-07-22 15:41:52 +020018#include "volumes.h"
Stefan Behrens21adbd52011-11-09 13:44:05 +010019#include "check-integrity.h"
Josef Bacik0b32f4b2012-03-13 09:38:00 -040020#include "locking.h"
Josef Bacik606686e2012-06-04 14:03:51 -040021#include "rcu-string.h"
Liu Bofe09e162013-09-22 12:54:23 +080022#include "backref.h"
Chris Masond1310b22008-01-24 16:13:08 -050023
Chris Masond1310b22008-01-24 16:13:08 -050024static struct kmem_cache *extent_state_cache;
25static struct kmem_cache *extent_buffer_cache;
Chris Mason9be33952013-05-17 18:30:14 -040026static struct bio_set *btrfs_bioset;
Chris Masond1310b22008-01-24 16:13:08 -050027
Eric Sandeen6d49ba12013-04-22 16:12:31 +000028#ifdef CONFIG_BTRFS_DEBUG
Chris Masond1310b22008-01-24 16:13:08 -050029static LIST_HEAD(buffers);
30static LIST_HEAD(states);
Chris Mason4bef0842008-09-08 11:18:08 -040031
Chris Masond3977122009-01-05 21:25:51 -050032static DEFINE_SPINLOCK(leak_lock);
Eric Sandeen6d49ba12013-04-22 16:12:31 +000033
34static inline
35void btrfs_leak_debug_add(struct list_head *new, struct list_head *head)
36{
37 unsigned long flags;
38
39 spin_lock_irqsave(&leak_lock, flags);
40 list_add(new, head);
41 spin_unlock_irqrestore(&leak_lock, flags);
42}
43
44static inline
45void btrfs_leak_debug_del(struct list_head *entry)
46{
47 unsigned long flags;
48
49 spin_lock_irqsave(&leak_lock, flags);
50 list_del(entry);
51 spin_unlock_irqrestore(&leak_lock, flags);
52}
53
54static inline
55void btrfs_leak_debug_check(void)
56{
57 struct extent_state *state;
58 struct extent_buffer *eb;
59
60 while (!list_empty(&states)) {
61 state = list_entry(states.next, struct extent_state, leak_list);
Frank Holtonefe120a2013-12-20 11:37:06 -050062 printk(KERN_ERR "BTRFS: state leak: start %llu end %llu "
Eric Sandeen6d49ba12013-04-22 16:12:31 +000063 "state %lu in tree %p refs %d\n",
Geert Uytterhoevenc1c9ff72013-08-20 13:20:07 +020064 state->start, state->end, state->state, state->tree,
65 atomic_read(&state->refs));
Eric Sandeen6d49ba12013-04-22 16:12:31 +000066 list_del(&state->leak_list);
67 kmem_cache_free(extent_state_cache, state);
68 }
69
70 while (!list_empty(&buffers)) {
71 eb = list_entry(buffers.next, struct extent_buffer, leak_list);
Frank Holtonefe120a2013-12-20 11:37:06 -050072 printk(KERN_ERR "BTRFS: buffer leak start %llu len %lu "
Geert Uytterhoevenc1c9ff72013-08-20 13:20:07 +020073 "refs %d\n",
74 eb->start, eb->len, atomic_read(&eb->refs));
Eric Sandeen6d49ba12013-04-22 16:12:31 +000075 list_del(&eb->leak_list);
76 kmem_cache_free(extent_buffer_cache, eb);
77 }
78}
David Sterba8d599ae2013-04-30 15:22:23 +000079
Josef Bacika5dee372013-12-13 10:02:44 -050080#define btrfs_debug_check_extent_io_range(tree, start, end) \
81 __btrfs_debug_check_extent_io_range(__func__, (tree), (start), (end))
David Sterba8d599ae2013-04-30 15:22:23 +000082static inline void __btrfs_debug_check_extent_io_range(const char *caller,
Josef Bacika5dee372013-12-13 10:02:44 -050083 struct extent_io_tree *tree, u64 start, u64 end)
David Sterba8d599ae2013-04-30 15:22:23 +000084{
Josef Bacika5dee372013-12-13 10:02:44 -050085 struct inode *inode;
86 u64 isize;
David Sterba8d599ae2013-04-30 15:22:23 +000087
Josef Bacika5dee372013-12-13 10:02:44 -050088 if (!tree->mapping)
89 return;
90
91 inode = tree->mapping->host;
92 isize = i_size_read(inode);
David Sterba8d599ae2013-04-30 15:22:23 +000093 if (end >= PAGE_SIZE && (end % 2) == 0 && end != isize - 1) {
94 printk_ratelimited(KERN_DEBUG
Frank Holtonefe120a2013-12-20 11:37:06 -050095 "BTRFS: %s: ino %llu isize %llu odd range [%llu,%llu]\n",
Geert Uytterhoevenc1c9ff72013-08-20 13:20:07 +020096 caller, btrfs_ino(inode), isize, start, end);
David Sterba8d599ae2013-04-30 15:22:23 +000097 }
98}
Eric Sandeen6d49ba12013-04-22 16:12:31 +000099#else
100#define btrfs_leak_debug_add(new, head) do {} while (0)
101#define btrfs_leak_debug_del(entry) do {} while (0)
102#define btrfs_leak_debug_check() do {} while (0)
David Sterba8d599ae2013-04-30 15:22:23 +0000103#define btrfs_debug_check_extent_io_range(c, s, e) do {} while (0)
Chris Mason4bef0842008-09-08 11:18:08 -0400104#endif
Chris Masond1310b22008-01-24 16:13:08 -0500105
Chris Masond1310b22008-01-24 16:13:08 -0500106#define BUFFER_LRU_MAX 64
107
108struct tree_entry {
109 u64 start;
110 u64 end;
Chris Masond1310b22008-01-24 16:13:08 -0500111 struct rb_node rb_node;
112};
113
114struct extent_page_data {
115 struct bio *bio;
116 struct extent_io_tree *tree;
117 get_extent_t *get_extent;
Josef Bacikde0022b2012-09-25 14:25:58 -0400118 unsigned long bio_flags;
Chris Mason771ed682008-11-06 22:02:51 -0500119
120 /* tells writepage not to lock the state bits for this range
121 * it still does the unlocking
122 */
Chris Masonffbd5172009-04-20 15:50:09 -0400123 unsigned int extent_locked:1;
124
125 /* tells the submit_bio code to use a WRITE_SYNC */
126 unsigned int sync_io:1;
Chris Masond1310b22008-01-24 16:13:08 -0500127};
128
Josef Bacik0b32f4b2012-03-13 09:38:00 -0400129static noinline void flush_write_bio(void *data);
Jeff Mahoneyc2d904e2011-10-03 23:22:32 -0400130static inline struct btrfs_fs_info *
131tree_fs_info(struct extent_io_tree *tree)
132{
Josef Bacika5dee372013-12-13 10:02:44 -0500133 if (!tree->mapping)
134 return NULL;
Jeff Mahoneyc2d904e2011-10-03 23:22:32 -0400135 return btrfs_sb(tree->mapping->host->i_sb);
136}
Josef Bacik0b32f4b2012-03-13 09:38:00 -0400137
Chris Masond1310b22008-01-24 16:13:08 -0500138int __init extent_io_init(void)
139{
David Sterba837e1972012-09-07 03:00:48 -0600140 extent_state_cache = kmem_cache_create("btrfs_extent_state",
Christoph Hellwig9601e3f2009-04-13 15:33:09 +0200141 sizeof(struct extent_state), 0,
142 SLAB_RECLAIM_ACCOUNT | SLAB_MEM_SPREAD, NULL);
Chris Masond1310b22008-01-24 16:13:08 -0500143 if (!extent_state_cache)
144 return -ENOMEM;
145
David Sterba837e1972012-09-07 03:00:48 -0600146 extent_buffer_cache = kmem_cache_create("btrfs_extent_buffer",
Christoph Hellwig9601e3f2009-04-13 15:33:09 +0200147 sizeof(struct extent_buffer), 0,
148 SLAB_RECLAIM_ACCOUNT | SLAB_MEM_SPREAD, NULL);
Chris Masond1310b22008-01-24 16:13:08 -0500149 if (!extent_buffer_cache)
150 goto free_state_cache;
Chris Mason9be33952013-05-17 18:30:14 -0400151
152 btrfs_bioset = bioset_create(BIO_POOL_SIZE,
153 offsetof(struct btrfs_io_bio, bio));
154 if (!btrfs_bioset)
155 goto free_buffer_cache;
Darrick J. Wongb208c2f2013-09-19 20:37:07 -0700156
157 if (bioset_integrity_create(btrfs_bioset, BIO_POOL_SIZE))
158 goto free_bioset;
159
Chris Masond1310b22008-01-24 16:13:08 -0500160 return 0;
161
Darrick J. Wongb208c2f2013-09-19 20:37:07 -0700162free_bioset:
163 bioset_free(btrfs_bioset);
164 btrfs_bioset = NULL;
165
Chris Mason9be33952013-05-17 18:30:14 -0400166free_buffer_cache:
167 kmem_cache_destroy(extent_buffer_cache);
168 extent_buffer_cache = NULL;
169
Chris Masond1310b22008-01-24 16:13:08 -0500170free_state_cache:
171 kmem_cache_destroy(extent_state_cache);
Chris Mason9be33952013-05-17 18:30:14 -0400172 extent_state_cache = NULL;
Chris Masond1310b22008-01-24 16:13:08 -0500173 return -ENOMEM;
174}
175
176void extent_io_exit(void)
177{
Eric Sandeen6d49ba12013-04-22 16:12:31 +0000178 btrfs_leak_debug_check();
Kirill A. Shutemov8c0a8532012-09-26 11:33:07 +1000179
180 /*
181 * Make sure all delayed rcu free are flushed before we
182 * destroy caches.
183 */
184 rcu_barrier();
Chris Masond1310b22008-01-24 16:13:08 -0500185 if (extent_state_cache)
186 kmem_cache_destroy(extent_state_cache);
187 if (extent_buffer_cache)
188 kmem_cache_destroy(extent_buffer_cache);
Chris Mason9be33952013-05-17 18:30:14 -0400189 if (btrfs_bioset)
190 bioset_free(btrfs_bioset);
Chris Masond1310b22008-01-24 16:13:08 -0500191}
192
193void extent_io_tree_init(struct extent_io_tree *tree,
David Sterbaf993c882011-04-20 23:35:57 +0200194 struct address_space *mapping)
Chris Masond1310b22008-01-24 16:13:08 -0500195{
Eric Paris6bef4d32010-02-23 19:43:04 +0000196 tree->state = RB_ROOT;
Chris Masond1310b22008-01-24 16:13:08 -0500197 tree->ops = NULL;
198 tree->dirty_bytes = 0;
Chris Mason70dec802008-01-29 09:59:12 -0500199 spin_lock_init(&tree->lock);
Chris Masond1310b22008-01-24 16:13:08 -0500200 tree->mapping = mapping;
Chris Masond1310b22008-01-24 16:13:08 -0500201}
Chris Masond1310b22008-01-24 16:13:08 -0500202
Christoph Hellwigb2950862008-12-02 09:54:17 -0500203static struct extent_state *alloc_extent_state(gfp_t mask)
Chris Masond1310b22008-01-24 16:13:08 -0500204{
205 struct extent_state *state;
Chris Masond1310b22008-01-24 16:13:08 -0500206
207 state = kmem_cache_alloc(extent_state_cache, mask);
Peter2b114d12008-04-01 11:21:40 -0400208 if (!state)
Chris Masond1310b22008-01-24 16:13:08 -0500209 return state;
210 state->state = 0;
Chris Masond1310b22008-01-24 16:13:08 -0500211 state->private = 0;
Chris Mason70dec802008-01-29 09:59:12 -0500212 state->tree = NULL;
Eric Sandeen6d49ba12013-04-22 16:12:31 +0000213 btrfs_leak_debug_add(&state->leak_list, &states);
Chris Masond1310b22008-01-24 16:13:08 -0500214 atomic_set(&state->refs, 1);
215 init_waitqueue_head(&state->wq);
Jeff Mahoney143bede2012-03-01 14:56:26 +0100216 trace_alloc_extent_state(state, mask, _RET_IP_);
Chris Masond1310b22008-01-24 16:13:08 -0500217 return state;
218}
Chris Masond1310b22008-01-24 16:13:08 -0500219
Chris Mason4845e442010-05-25 20:56:50 -0400220void free_extent_state(struct extent_state *state)
Chris Masond1310b22008-01-24 16:13:08 -0500221{
Chris Masond1310b22008-01-24 16:13:08 -0500222 if (!state)
223 return;
224 if (atomic_dec_and_test(&state->refs)) {
Chris Mason70dec802008-01-29 09:59:12 -0500225 WARN_ON(state->tree);
Eric Sandeen6d49ba12013-04-22 16:12:31 +0000226 btrfs_leak_debug_del(&state->leak_list);
Jeff Mahoney143bede2012-03-01 14:56:26 +0100227 trace_free_extent_state(state, _RET_IP_);
Chris Masond1310b22008-01-24 16:13:08 -0500228 kmem_cache_free(extent_state_cache, state);
229 }
230}
Chris Masond1310b22008-01-24 16:13:08 -0500231
Filipe Mananaf2071b22014-02-12 15:05:53 +0000232static struct rb_node *tree_insert(struct rb_root *root,
233 struct rb_node *search_start,
234 u64 offset,
Filipe David Borba Manana12cfbad2013-11-26 15:41:47 +0000235 struct rb_node *node,
236 struct rb_node ***p_in,
237 struct rb_node **parent_in)
Chris Masond1310b22008-01-24 16:13:08 -0500238{
Filipe Mananaf2071b22014-02-12 15:05:53 +0000239 struct rb_node **p;
Chris Masond3977122009-01-05 21:25:51 -0500240 struct rb_node *parent = NULL;
Chris Masond1310b22008-01-24 16:13:08 -0500241 struct tree_entry *entry;
242
Filipe David Borba Manana12cfbad2013-11-26 15:41:47 +0000243 if (p_in && parent_in) {
244 p = *p_in;
245 parent = *parent_in;
246 goto do_insert;
247 }
248
Filipe Mananaf2071b22014-02-12 15:05:53 +0000249 p = search_start ? &search_start : &root->rb_node;
Chris Masond3977122009-01-05 21:25:51 -0500250 while (*p) {
Chris Masond1310b22008-01-24 16:13:08 -0500251 parent = *p;
252 entry = rb_entry(parent, struct tree_entry, rb_node);
253
254 if (offset < entry->start)
255 p = &(*p)->rb_left;
256 else if (offset > entry->end)
257 p = &(*p)->rb_right;
258 else
259 return parent;
260 }
261
Filipe David Borba Manana12cfbad2013-11-26 15:41:47 +0000262do_insert:
Chris Masond1310b22008-01-24 16:13:08 -0500263 rb_link_node(node, parent, p);
264 rb_insert_color(node, root);
265 return NULL;
266}
267
Chris Mason80ea96b2008-02-01 14:51:59 -0500268static struct rb_node *__etree_search(struct extent_io_tree *tree, u64 offset,
Filipe David Borba Manana12cfbad2013-11-26 15:41:47 +0000269 struct rb_node **prev_ret,
270 struct rb_node **next_ret,
271 struct rb_node ***p_ret,
272 struct rb_node **parent_ret)
Chris Masond1310b22008-01-24 16:13:08 -0500273{
Chris Mason80ea96b2008-02-01 14:51:59 -0500274 struct rb_root *root = &tree->state;
Filipe David Borba Manana12cfbad2013-11-26 15:41:47 +0000275 struct rb_node **n = &root->rb_node;
Chris Masond1310b22008-01-24 16:13:08 -0500276 struct rb_node *prev = NULL;
277 struct rb_node *orig_prev = NULL;
278 struct tree_entry *entry;
279 struct tree_entry *prev_entry = NULL;
280
Filipe David Borba Manana12cfbad2013-11-26 15:41:47 +0000281 while (*n) {
282 prev = *n;
283 entry = rb_entry(prev, struct tree_entry, rb_node);
Chris Masond1310b22008-01-24 16:13:08 -0500284 prev_entry = entry;
285
286 if (offset < entry->start)
Filipe David Borba Manana12cfbad2013-11-26 15:41:47 +0000287 n = &(*n)->rb_left;
Chris Masond1310b22008-01-24 16:13:08 -0500288 else if (offset > entry->end)
Filipe David Borba Manana12cfbad2013-11-26 15:41:47 +0000289 n = &(*n)->rb_right;
Chris Masond3977122009-01-05 21:25:51 -0500290 else
Filipe David Borba Manana12cfbad2013-11-26 15:41:47 +0000291 return *n;
Chris Masond1310b22008-01-24 16:13:08 -0500292 }
293
Filipe David Borba Manana12cfbad2013-11-26 15:41:47 +0000294 if (p_ret)
295 *p_ret = n;
296 if (parent_ret)
297 *parent_ret = prev;
298
Chris Masond1310b22008-01-24 16:13:08 -0500299 if (prev_ret) {
300 orig_prev = prev;
Chris Masond3977122009-01-05 21:25:51 -0500301 while (prev && offset > prev_entry->end) {
Chris Masond1310b22008-01-24 16:13:08 -0500302 prev = rb_next(prev);
303 prev_entry = rb_entry(prev, struct tree_entry, rb_node);
304 }
305 *prev_ret = prev;
306 prev = orig_prev;
307 }
308
309 if (next_ret) {
310 prev_entry = rb_entry(prev, struct tree_entry, rb_node);
Chris Masond3977122009-01-05 21:25:51 -0500311 while (prev && offset < prev_entry->start) {
Chris Masond1310b22008-01-24 16:13:08 -0500312 prev = rb_prev(prev);
313 prev_entry = rb_entry(prev, struct tree_entry, rb_node);
314 }
315 *next_ret = prev;
316 }
317 return NULL;
318}
319
Filipe David Borba Manana12cfbad2013-11-26 15:41:47 +0000320static inline struct rb_node *
321tree_search_for_insert(struct extent_io_tree *tree,
322 u64 offset,
323 struct rb_node ***p_ret,
324 struct rb_node **parent_ret)
Chris Masond1310b22008-01-24 16:13:08 -0500325{
Chris Mason70dec802008-01-29 09:59:12 -0500326 struct rb_node *prev = NULL;
Chris Masond1310b22008-01-24 16:13:08 -0500327 struct rb_node *ret;
Chris Mason70dec802008-01-29 09:59:12 -0500328
Filipe David Borba Manana12cfbad2013-11-26 15:41:47 +0000329 ret = __etree_search(tree, offset, &prev, NULL, p_ret, parent_ret);
Chris Masond3977122009-01-05 21:25:51 -0500330 if (!ret)
Chris Masond1310b22008-01-24 16:13:08 -0500331 return prev;
332 return ret;
333}
334
Filipe David Borba Manana12cfbad2013-11-26 15:41:47 +0000335static inline struct rb_node *tree_search(struct extent_io_tree *tree,
336 u64 offset)
337{
338 return tree_search_for_insert(tree, offset, NULL, NULL);
339}
340
Josef Bacik9ed74f22009-09-11 16:12:44 -0400341static void merge_cb(struct extent_io_tree *tree, struct extent_state *new,
342 struct extent_state *other)
343{
344 if (tree->ops && tree->ops->merge_extent_hook)
345 tree->ops->merge_extent_hook(tree->mapping->host, new,
346 other);
347}
348
Chris Masond1310b22008-01-24 16:13:08 -0500349/*
350 * utility function to look for merge candidates inside a given range.
351 * Any extents with matching state are merged together into a single
352 * extent in the tree. Extents with EXTENT_IO in their state field
353 * are not merged because the end_io handlers need to be able to do
354 * operations on them without sleeping (or doing allocations/splits).
355 *
356 * This should be called with the tree lock held.
357 */
Jeff Mahoney1bf85042011-07-21 16:56:09 +0000358static void merge_state(struct extent_io_tree *tree,
359 struct extent_state *state)
Chris Masond1310b22008-01-24 16:13:08 -0500360{
361 struct extent_state *other;
362 struct rb_node *other_node;
363
Zheng Yan5b21f2e2008-09-26 10:05:38 -0400364 if (state->state & (EXTENT_IOBITS | EXTENT_BOUNDARY))
Jeff Mahoney1bf85042011-07-21 16:56:09 +0000365 return;
Chris Masond1310b22008-01-24 16:13:08 -0500366
367 other_node = rb_prev(&state->rb_node);
368 if (other_node) {
369 other = rb_entry(other_node, struct extent_state, rb_node);
370 if (other->end == state->start - 1 &&
371 other->state == state->state) {
Josef Bacik9ed74f22009-09-11 16:12:44 -0400372 merge_cb(tree, state, other);
Chris Masond1310b22008-01-24 16:13:08 -0500373 state->start = other->start;
Chris Mason70dec802008-01-29 09:59:12 -0500374 other->tree = NULL;
Chris Masond1310b22008-01-24 16:13:08 -0500375 rb_erase(&other->rb_node, &tree->state);
376 free_extent_state(other);
377 }
378 }
379 other_node = rb_next(&state->rb_node);
380 if (other_node) {
381 other = rb_entry(other_node, struct extent_state, rb_node);
382 if (other->start == state->end + 1 &&
383 other->state == state->state) {
Josef Bacik9ed74f22009-09-11 16:12:44 -0400384 merge_cb(tree, state, other);
Josef Bacikdf98b6e2011-06-20 14:53:48 -0400385 state->end = other->end;
386 other->tree = NULL;
387 rb_erase(&other->rb_node, &tree->state);
388 free_extent_state(other);
Chris Masond1310b22008-01-24 16:13:08 -0500389 }
390 }
Chris Masond1310b22008-01-24 16:13:08 -0500391}
392
Jeff Mahoney1bf85042011-07-21 16:56:09 +0000393static void set_state_cb(struct extent_io_tree *tree,
David Sterba41074882013-04-29 13:38:46 +0000394 struct extent_state *state, unsigned long *bits)
Chris Mason291d6732008-01-29 15:55:23 -0500395{
Jeff Mahoney1bf85042011-07-21 16:56:09 +0000396 if (tree->ops && tree->ops->set_bit_hook)
397 tree->ops->set_bit_hook(tree->mapping->host, state, bits);
Chris Mason291d6732008-01-29 15:55:23 -0500398}
399
400static void clear_state_cb(struct extent_io_tree *tree,
David Sterba41074882013-04-29 13:38:46 +0000401 struct extent_state *state, unsigned long *bits)
Chris Mason291d6732008-01-29 15:55:23 -0500402{
Josef Bacik9ed74f22009-09-11 16:12:44 -0400403 if (tree->ops && tree->ops->clear_bit_hook)
404 tree->ops->clear_bit_hook(tree->mapping->host, state, bits);
Chris Mason291d6732008-01-29 15:55:23 -0500405}
406
Xiao Guangrong3150b692011-07-14 03:19:08 +0000407static void set_state_bits(struct extent_io_tree *tree,
David Sterba41074882013-04-29 13:38:46 +0000408 struct extent_state *state, unsigned long *bits);
Xiao Guangrong3150b692011-07-14 03:19:08 +0000409
Chris Masond1310b22008-01-24 16:13:08 -0500410/*
411 * insert an extent_state struct into the tree. 'bits' are set on the
412 * struct before it is inserted.
413 *
414 * This may return -EEXIST if the extent is already there, in which case the
415 * state struct is freed.
416 *
417 * The tree lock is not taken internally. This is a utility function and
418 * probably isn't what you want to call (see set/clear_extent_bit).
419 */
420static int insert_state(struct extent_io_tree *tree,
421 struct extent_state *state, u64 start, u64 end,
Filipe David Borba Manana12cfbad2013-11-26 15:41:47 +0000422 struct rb_node ***p,
423 struct rb_node **parent,
David Sterba41074882013-04-29 13:38:46 +0000424 unsigned long *bits)
Chris Masond1310b22008-01-24 16:13:08 -0500425{
426 struct rb_node *node;
427
Julia Lawall31b1a2b2012-11-03 10:58:34 +0000428 if (end < start)
Frank Holtonefe120a2013-12-20 11:37:06 -0500429 WARN(1, KERN_ERR "BTRFS: end < start %llu %llu\n",
Geert Uytterhoevenc1c9ff72013-08-20 13:20:07 +0200430 end, start);
Chris Masond1310b22008-01-24 16:13:08 -0500431 state->start = start;
432 state->end = end;
Josef Bacik9ed74f22009-09-11 16:12:44 -0400433
Xiao Guangrong3150b692011-07-14 03:19:08 +0000434 set_state_bits(tree, state, bits);
435
Filipe Mananaf2071b22014-02-12 15:05:53 +0000436 node = tree_insert(&tree->state, NULL, end, &state->rb_node, p, parent);
Chris Masond1310b22008-01-24 16:13:08 -0500437 if (node) {
438 struct extent_state *found;
439 found = rb_entry(node, struct extent_state, rb_node);
Frank Holtonefe120a2013-12-20 11:37:06 -0500440 printk(KERN_ERR "BTRFS: found node %llu %llu on insert of "
Geert Uytterhoevenc1c9ff72013-08-20 13:20:07 +0200441 "%llu %llu\n",
442 found->start, found->end, start, end);
Chris Masond1310b22008-01-24 16:13:08 -0500443 return -EEXIST;
444 }
Chris Mason70dec802008-01-29 09:59:12 -0500445 state->tree = tree;
Chris Masond1310b22008-01-24 16:13:08 -0500446 merge_state(tree, state);
447 return 0;
448}
449
Jeff Mahoney1bf85042011-07-21 16:56:09 +0000450static void split_cb(struct extent_io_tree *tree, struct extent_state *orig,
Josef Bacik9ed74f22009-09-11 16:12:44 -0400451 u64 split)
452{
453 if (tree->ops && tree->ops->split_extent_hook)
Jeff Mahoney1bf85042011-07-21 16:56:09 +0000454 tree->ops->split_extent_hook(tree->mapping->host, orig, split);
Josef Bacik9ed74f22009-09-11 16:12:44 -0400455}
456
Chris Masond1310b22008-01-24 16:13:08 -0500457/*
458 * split a given extent state struct in two, inserting the preallocated
459 * struct 'prealloc' as the newly created second half. 'split' indicates an
460 * offset inside 'orig' where it should be split.
461 *
462 * Before calling,
463 * the tree has 'orig' at [orig->start, orig->end]. After calling, there
464 * are two extent state structs in the tree:
465 * prealloc: [orig->start, split - 1]
466 * orig: [ split, orig->end ]
467 *
468 * The tree locks are not taken by this function. They need to be held
469 * by the caller.
470 */
471static int split_state(struct extent_io_tree *tree, struct extent_state *orig,
472 struct extent_state *prealloc, u64 split)
473{
474 struct rb_node *node;
Josef Bacik9ed74f22009-09-11 16:12:44 -0400475
476 split_cb(tree, orig, split);
477
Chris Masond1310b22008-01-24 16:13:08 -0500478 prealloc->start = orig->start;
479 prealloc->end = split - 1;
480 prealloc->state = orig->state;
481 orig->start = split;
482
Filipe Mananaf2071b22014-02-12 15:05:53 +0000483 node = tree_insert(&tree->state, &orig->rb_node, prealloc->end,
484 &prealloc->rb_node, NULL, NULL);
Chris Masond1310b22008-01-24 16:13:08 -0500485 if (node) {
Chris Masond1310b22008-01-24 16:13:08 -0500486 free_extent_state(prealloc);
487 return -EEXIST;
488 }
Chris Mason70dec802008-01-29 09:59:12 -0500489 prealloc->tree = tree;
Chris Masond1310b22008-01-24 16:13:08 -0500490 return 0;
491}
492
Li Zefancdc6a392012-03-12 16:39:48 +0800493static struct extent_state *next_state(struct extent_state *state)
494{
495 struct rb_node *next = rb_next(&state->rb_node);
496 if (next)
497 return rb_entry(next, struct extent_state, rb_node);
498 else
499 return NULL;
500}
501
Chris Masond1310b22008-01-24 16:13:08 -0500502/*
503 * utility function to clear some bits in an extent state struct.
Wang Sheng-Hui1b303fc2012-04-06 14:35:18 +0800504 * it will optionally wake up any one waiting on this state (wake == 1).
Chris Masond1310b22008-01-24 16:13:08 -0500505 *
506 * If no bits are set on the state struct after clearing things, the
507 * struct is freed and removed from the tree
508 */
Li Zefancdc6a392012-03-12 16:39:48 +0800509static struct extent_state *clear_state_bit(struct extent_io_tree *tree,
510 struct extent_state *state,
David Sterba41074882013-04-29 13:38:46 +0000511 unsigned long *bits, int wake)
Chris Masond1310b22008-01-24 16:13:08 -0500512{
Li Zefancdc6a392012-03-12 16:39:48 +0800513 struct extent_state *next;
David Sterba41074882013-04-29 13:38:46 +0000514 unsigned long bits_to_clear = *bits & ~EXTENT_CTLBITS;
Chris Masond1310b22008-01-24 16:13:08 -0500515
Yan, Zheng0ca1f7c2010-05-16 10:48:47 -0400516 if ((bits_to_clear & EXTENT_DIRTY) && (state->state & EXTENT_DIRTY)) {
Chris Masond1310b22008-01-24 16:13:08 -0500517 u64 range = state->end - state->start + 1;
518 WARN_ON(range > tree->dirty_bytes);
519 tree->dirty_bytes -= range;
520 }
Chris Mason291d6732008-01-29 15:55:23 -0500521 clear_state_cb(tree, state, bits);
Josef Bacik32c00af2009-10-08 13:34:05 -0400522 state->state &= ~bits_to_clear;
Chris Masond1310b22008-01-24 16:13:08 -0500523 if (wake)
524 wake_up(&state->wq);
Yan, Zheng0ca1f7c2010-05-16 10:48:47 -0400525 if (state->state == 0) {
Li Zefancdc6a392012-03-12 16:39:48 +0800526 next = next_state(state);
Chris Mason70dec802008-01-29 09:59:12 -0500527 if (state->tree) {
Chris Masond1310b22008-01-24 16:13:08 -0500528 rb_erase(&state->rb_node, &tree->state);
Chris Mason70dec802008-01-29 09:59:12 -0500529 state->tree = NULL;
Chris Masond1310b22008-01-24 16:13:08 -0500530 free_extent_state(state);
531 } else {
532 WARN_ON(1);
533 }
534 } else {
535 merge_state(tree, state);
Li Zefancdc6a392012-03-12 16:39:48 +0800536 next = next_state(state);
Chris Masond1310b22008-01-24 16:13:08 -0500537 }
Li Zefancdc6a392012-03-12 16:39:48 +0800538 return next;
Chris Masond1310b22008-01-24 16:13:08 -0500539}
540
Xiao Guangrong82337672011-04-20 06:44:57 +0000541static struct extent_state *
542alloc_extent_state_atomic(struct extent_state *prealloc)
543{
544 if (!prealloc)
545 prealloc = alloc_extent_state(GFP_ATOMIC);
546
547 return prealloc;
548}
549
Eric Sandeen48a3b632013-04-25 20:41:01 +0000550static void extent_io_tree_panic(struct extent_io_tree *tree, int err)
Jeff Mahoneyc2d904e2011-10-03 23:22:32 -0400551{
552 btrfs_panic(tree_fs_info(tree), err, "Locking error: "
553 "Extent tree was modified by another "
554 "thread while locked.");
555}
556
Chris Masond1310b22008-01-24 16:13:08 -0500557/*
558 * clear some bits on a range in the tree. This may require splitting
559 * or inserting elements in the tree, so the gfp mask is used to
560 * indicate which allocations or sleeping are allowed.
561 *
562 * pass 'wake' == 1 to kick any sleepers, and 'delete' == 1 to remove
563 * the given range from the tree regardless of state (ie for truncate).
564 *
565 * the range [start, end] is inclusive.
566 *
Jeff Mahoney6763af82012-03-01 14:56:29 +0100567 * This takes the tree lock, and returns 0 on success and < 0 on error.
Chris Masond1310b22008-01-24 16:13:08 -0500568 */
569int clear_extent_bit(struct extent_io_tree *tree, u64 start, u64 end,
David Sterba41074882013-04-29 13:38:46 +0000570 unsigned long bits, int wake, int delete,
Chris Mason2c64c532009-09-02 15:04:12 -0400571 struct extent_state **cached_state,
572 gfp_t mask)
Chris Masond1310b22008-01-24 16:13:08 -0500573{
574 struct extent_state *state;
Chris Mason2c64c532009-09-02 15:04:12 -0400575 struct extent_state *cached;
Chris Masond1310b22008-01-24 16:13:08 -0500576 struct extent_state *prealloc = NULL;
577 struct rb_node *node;
Yan Zheng5c939df2009-05-27 09:16:03 -0400578 u64 last_end;
Chris Masond1310b22008-01-24 16:13:08 -0500579 int err;
Josef Bacik2ac55d42010-02-03 19:33:23 +0000580 int clear = 0;
Chris Masond1310b22008-01-24 16:13:08 -0500581
Josef Bacika5dee372013-12-13 10:02:44 -0500582 btrfs_debug_check_extent_io_range(tree, start, end);
David Sterba8d599ae2013-04-30 15:22:23 +0000583
Josef Bacik7ee9e442013-06-21 16:37:03 -0400584 if (bits & EXTENT_DELALLOC)
585 bits |= EXTENT_NORESERVE;
586
Yan, Zheng0ca1f7c2010-05-16 10:48:47 -0400587 if (delete)
588 bits |= ~EXTENT_CTLBITS;
589 bits |= EXTENT_FIRST_DELALLOC;
590
Josef Bacik2ac55d42010-02-03 19:33:23 +0000591 if (bits & (EXTENT_IOBITS | EXTENT_BOUNDARY))
592 clear = 1;
Chris Masond1310b22008-01-24 16:13:08 -0500593again:
594 if (!prealloc && (mask & __GFP_WAIT)) {
595 prealloc = alloc_extent_state(mask);
596 if (!prealloc)
597 return -ENOMEM;
598 }
599
Chris Masoncad321a2008-12-17 14:51:42 -0500600 spin_lock(&tree->lock);
Chris Mason2c64c532009-09-02 15:04:12 -0400601 if (cached_state) {
602 cached = *cached_state;
Josef Bacik2ac55d42010-02-03 19:33:23 +0000603
604 if (clear) {
605 *cached_state = NULL;
606 cached_state = NULL;
607 }
608
Josef Bacikdf98b6e2011-06-20 14:53:48 -0400609 if (cached && cached->tree && cached->start <= start &&
610 cached->end > start) {
Josef Bacik2ac55d42010-02-03 19:33:23 +0000611 if (clear)
612 atomic_dec(&cached->refs);
Chris Mason2c64c532009-09-02 15:04:12 -0400613 state = cached;
Chris Mason42daec22009-09-23 19:51:09 -0400614 goto hit_next;
Chris Mason2c64c532009-09-02 15:04:12 -0400615 }
Josef Bacik2ac55d42010-02-03 19:33:23 +0000616 if (clear)
617 free_extent_state(cached);
Chris Mason2c64c532009-09-02 15:04:12 -0400618 }
Chris Masond1310b22008-01-24 16:13:08 -0500619 /*
620 * this search will find the extents that end after
621 * our range starts
622 */
Chris Mason80ea96b2008-02-01 14:51:59 -0500623 node = tree_search(tree, start);
Chris Masond1310b22008-01-24 16:13:08 -0500624 if (!node)
625 goto out;
626 state = rb_entry(node, struct extent_state, rb_node);
Chris Mason2c64c532009-09-02 15:04:12 -0400627hit_next:
Chris Masond1310b22008-01-24 16:13:08 -0500628 if (state->start > end)
629 goto out;
630 WARN_ON(state->end < start);
Yan Zheng5c939df2009-05-27 09:16:03 -0400631 last_end = state->end;
Chris Masond1310b22008-01-24 16:13:08 -0500632
Liu Bo04493142012-02-16 18:34:37 +0800633 /* the state doesn't have the wanted bits, go ahead */
Li Zefancdc6a392012-03-12 16:39:48 +0800634 if (!(state->state & bits)) {
635 state = next_state(state);
Liu Bo04493142012-02-16 18:34:37 +0800636 goto next;
Li Zefancdc6a392012-03-12 16:39:48 +0800637 }
Liu Bo04493142012-02-16 18:34:37 +0800638
Chris Masond1310b22008-01-24 16:13:08 -0500639 /*
640 * | ---- desired range ---- |
641 * | state | or
642 * | ------------- state -------------- |
643 *
644 * We need to split the extent we found, and may flip
645 * bits on second half.
646 *
647 * If the extent we found extends past our range, we
648 * just split and search again. It'll get split again
649 * the next time though.
650 *
651 * If the extent we found is inside our range, we clear
652 * the desired bit on it.
653 */
654
655 if (state->start < start) {
Xiao Guangrong82337672011-04-20 06:44:57 +0000656 prealloc = alloc_extent_state_atomic(prealloc);
657 BUG_ON(!prealloc);
Chris Masond1310b22008-01-24 16:13:08 -0500658 err = split_state(tree, state, prealloc, start);
Jeff Mahoneyc2d904e2011-10-03 23:22:32 -0400659 if (err)
660 extent_io_tree_panic(tree, err);
661
Chris Masond1310b22008-01-24 16:13:08 -0500662 prealloc = NULL;
663 if (err)
664 goto out;
665 if (state->end <= end) {
Liu Bod1ac6e42012-05-10 18:10:39 +0800666 state = clear_state_bit(tree, state, &bits, wake);
667 goto next;
Chris Masond1310b22008-01-24 16:13:08 -0500668 }
669 goto search_again;
670 }
671 /*
672 * | ---- desired range ---- |
673 * | state |
674 * We need to split the extent, and clear the bit
675 * on the first half
676 */
677 if (state->start <= end && state->end > end) {
Xiao Guangrong82337672011-04-20 06:44:57 +0000678 prealloc = alloc_extent_state_atomic(prealloc);
679 BUG_ON(!prealloc);
Chris Masond1310b22008-01-24 16:13:08 -0500680 err = split_state(tree, state, prealloc, end + 1);
Jeff Mahoneyc2d904e2011-10-03 23:22:32 -0400681 if (err)
682 extent_io_tree_panic(tree, err);
683
Chris Masond1310b22008-01-24 16:13:08 -0500684 if (wake)
685 wake_up(&state->wq);
Chris Mason42daec22009-09-23 19:51:09 -0400686
Jeff Mahoney6763af82012-03-01 14:56:29 +0100687 clear_state_bit(tree, prealloc, &bits, wake);
Josef Bacik9ed74f22009-09-11 16:12:44 -0400688
Chris Masond1310b22008-01-24 16:13:08 -0500689 prealloc = NULL;
690 goto out;
691 }
Chris Mason42daec22009-09-23 19:51:09 -0400692
Li Zefancdc6a392012-03-12 16:39:48 +0800693 state = clear_state_bit(tree, state, &bits, wake);
Liu Bo04493142012-02-16 18:34:37 +0800694next:
Yan Zheng5c939df2009-05-27 09:16:03 -0400695 if (last_end == (u64)-1)
696 goto out;
697 start = last_end + 1;
Li Zefancdc6a392012-03-12 16:39:48 +0800698 if (start <= end && state && !need_resched())
Liu Bo692e5752012-02-16 18:34:36 +0800699 goto hit_next;
Chris Masond1310b22008-01-24 16:13:08 -0500700 goto search_again;
701
702out:
Chris Masoncad321a2008-12-17 14:51:42 -0500703 spin_unlock(&tree->lock);
Chris Masond1310b22008-01-24 16:13:08 -0500704 if (prealloc)
705 free_extent_state(prealloc);
706
Jeff Mahoney6763af82012-03-01 14:56:29 +0100707 return 0;
Chris Masond1310b22008-01-24 16:13:08 -0500708
709search_again:
710 if (start > end)
711 goto out;
Chris Masoncad321a2008-12-17 14:51:42 -0500712 spin_unlock(&tree->lock);
Chris Masond1310b22008-01-24 16:13:08 -0500713 if (mask & __GFP_WAIT)
714 cond_resched();
715 goto again;
716}
Chris Masond1310b22008-01-24 16:13:08 -0500717
Jeff Mahoney143bede2012-03-01 14:56:26 +0100718static void wait_on_state(struct extent_io_tree *tree,
719 struct extent_state *state)
Christoph Hellwig641f5212008-12-02 06:36:10 -0500720 __releases(tree->lock)
721 __acquires(tree->lock)
Chris Masond1310b22008-01-24 16:13:08 -0500722{
723 DEFINE_WAIT(wait);
724 prepare_to_wait(&state->wq, &wait, TASK_UNINTERRUPTIBLE);
Chris Masoncad321a2008-12-17 14:51:42 -0500725 spin_unlock(&tree->lock);
Chris Masond1310b22008-01-24 16:13:08 -0500726 schedule();
Chris Masoncad321a2008-12-17 14:51:42 -0500727 spin_lock(&tree->lock);
Chris Masond1310b22008-01-24 16:13:08 -0500728 finish_wait(&state->wq, &wait);
Chris Masond1310b22008-01-24 16:13:08 -0500729}
730
731/*
732 * waits for one or more bits to clear on a range in the state tree.
733 * The range [start, end] is inclusive.
734 * The tree lock is taken by this function
735 */
David Sterba41074882013-04-29 13:38:46 +0000736static void wait_extent_bit(struct extent_io_tree *tree, u64 start, u64 end,
737 unsigned long bits)
Chris Masond1310b22008-01-24 16:13:08 -0500738{
739 struct extent_state *state;
740 struct rb_node *node;
741
Josef Bacika5dee372013-12-13 10:02:44 -0500742 btrfs_debug_check_extent_io_range(tree, start, end);
David Sterba8d599ae2013-04-30 15:22:23 +0000743
Chris Masoncad321a2008-12-17 14:51:42 -0500744 spin_lock(&tree->lock);
Chris Masond1310b22008-01-24 16:13:08 -0500745again:
746 while (1) {
747 /*
748 * this search will find all the extents that end after
749 * our range starts
750 */
Chris Mason80ea96b2008-02-01 14:51:59 -0500751 node = tree_search(tree, start);
Filipe Mananac50d3e72014-03-31 14:53:25 +0100752process_node:
Chris Masond1310b22008-01-24 16:13:08 -0500753 if (!node)
754 break;
755
756 state = rb_entry(node, struct extent_state, rb_node);
757
758 if (state->start > end)
759 goto out;
760
761 if (state->state & bits) {
762 start = state->start;
763 atomic_inc(&state->refs);
764 wait_on_state(tree, state);
765 free_extent_state(state);
766 goto again;
767 }
768 start = state->end + 1;
769
770 if (start > end)
771 break;
772
Filipe Mananac50d3e72014-03-31 14:53:25 +0100773 if (!cond_resched_lock(&tree->lock)) {
774 node = rb_next(node);
775 goto process_node;
776 }
Chris Masond1310b22008-01-24 16:13:08 -0500777 }
778out:
Chris Masoncad321a2008-12-17 14:51:42 -0500779 spin_unlock(&tree->lock);
Chris Masond1310b22008-01-24 16:13:08 -0500780}
Chris Masond1310b22008-01-24 16:13:08 -0500781
Jeff Mahoney1bf85042011-07-21 16:56:09 +0000782static void set_state_bits(struct extent_io_tree *tree,
Chris Masond1310b22008-01-24 16:13:08 -0500783 struct extent_state *state,
David Sterba41074882013-04-29 13:38:46 +0000784 unsigned long *bits)
Chris Masond1310b22008-01-24 16:13:08 -0500785{
David Sterba41074882013-04-29 13:38:46 +0000786 unsigned long bits_to_set = *bits & ~EXTENT_CTLBITS;
Josef Bacik9ed74f22009-09-11 16:12:44 -0400787
Jeff Mahoney1bf85042011-07-21 16:56:09 +0000788 set_state_cb(tree, state, bits);
Yan, Zheng0ca1f7c2010-05-16 10:48:47 -0400789 if ((bits_to_set & EXTENT_DIRTY) && !(state->state & EXTENT_DIRTY)) {
Chris Masond1310b22008-01-24 16:13:08 -0500790 u64 range = state->end - state->start + 1;
791 tree->dirty_bytes += range;
792 }
Yan, Zheng0ca1f7c2010-05-16 10:48:47 -0400793 state->state |= bits_to_set;
Chris Masond1310b22008-01-24 16:13:08 -0500794}
795
Chris Mason2c64c532009-09-02 15:04:12 -0400796static void cache_state(struct extent_state *state,
797 struct extent_state **cached_ptr)
798{
799 if (cached_ptr && !(*cached_ptr)) {
800 if (state->state & (EXTENT_IOBITS | EXTENT_BOUNDARY)) {
801 *cached_ptr = state;
802 atomic_inc(&state->refs);
803 }
804 }
805}
806
Chris Masond1310b22008-01-24 16:13:08 -0500807/*
Chris Mason1edbb732009-09-02 13:24:36 -0400808 * set some bits on a range in the tree. This may require allocations or
809 * sleeping, so the gfp mask is used to indicate what is allowed.
Chris Masond1310b22008-01-24 16:13:08 -0500810 *
Chris Mason1edbb732009-09-02 13:24:36 -0400811 * If any of the exclusive bits are set, this will fail with -EEXIST if some
812 * part of the range already has the desired bits set. The start of the
813 * existing range is returned in failed_start in this case.
Chris Masond1310b22008-01-24 16:13:08 -0500814 *
Chris Mason1edbb732009-09-02 13:24:36 -0400815 * [start, end] is inclusive This takes the tree lock.
Chris Masond1310b22008-01-24 16:13:08 -0500816 */
Chris Mason1edbb732009-09-02 13:24:36 -0400817
Jeff Mahoney3fbe5c02012-03-01 14:57:19 +0100818static int __must_check
819__set_extent_bit(struct extent_io_tree *tree, u64 start, u64 end,
David Sterba41074882013-04-29 13:38:46 +0000820 unsigned long bits, unsigned long exclusive_bits,
821 u64 *failed_start, struct extent_state **cached_state,
822 gfp_t mask)
Chris Masond1310b22008-01-24 16:13:08 -0500823{
824 struct extent_state *state;
825 struct extent_state *prealloc = NULL;
826 struct rb_node *node;
Filipe David Borba Manana12cfbad2013-11-26 15:41:47 +0000827 struct rb_node **p;
828 struct rb_node *parent;
Chris Masond1310b22008-01-24 16:13:08 -0500829 int err = 0;
Chris Masond1310b22008-01-24 16:13:08 -0500830 u64 last_start;
831 u64 last_end;
Chris Mason42daec22009-09-23 19:51:09 -0400832
Josef Bacika5dee372013-12-13 10:02:44 -0500833 btrfs_debug_check_extent_io_range(tree, start, end);
David Sterba8d599ae2013-04-30 15:22:23 +0000834
Yan, Zheng0ca1f7c2010-05-16 10:48:47 -0400835 bits |= EXTENT_FIRST_DELALLOC;
Chris Masond1310b22008-01-24 16:13:08 -0500836again:
837 if (!prealloc && (mask & __GFP_WAIT)) {
838 prealloc = alloc_extent_state(mask);
Xiao Guangrong82337672011-04-20 06:44:57 +0000839 BUG_ON(!prealloc);
Chris Masond1310b22008-01-24 16:13:08 -0500840 }
841
Chris Masoncad321a2008-12-17 14:51:42 -0500842 spin_lock(&tree->lock);
Chris Mason9655d292009-09-02 15:22:30 -0400843 if (cached_state && *cached_state) {
844 state = *cached_state;
Josef Bacikdf98b6e2011-06-20 14:53:48 -0400845 if (state->start <= start && state->end > start &&
846 state->tree) {
Chris Mason9655d292009-09-02 15:22:30 -0400847 node = &state->rb_node;
848 goto hit_next;
849 }
850 }
Chris Masond1310b22008-01-24 16:13:08 -0500851 /*
852 * this search will find all the extents that end after
853 * our range starts.
854 */
Filipe David Borba Manana12cfbad2013-11-26 15:41:47 +0000855 node = tree_search_for_insert(tree, start, &p, &parent);
Chris Masond1310b22008-01-24 16:13:08 -0500856 if (!node) {
Xiao Guangrong82337672011-04-20 06:44:57 +0000857 prealloc = alloc_extent_state_atomic(prealloc);
858 BUG_ON(!prealloc);
Filipe David Borba Manana12cfbad2013-11-26 15:41:47 +0000859 err = insert_state(tree, prealloc, start, end,
860 &p, &parent, &bits);
Jeff Mahoneyc2d904e2011-10-03 23:22:32 -0400861 if (err)
862 extent_io_tree_panic(tree, err);
863
Filipe David Borba Mananac42ac0b2013-11-26 15:01:34 +0000864 cache_state(prealloc, cached_state);
Chris Masond1310b22008-01-24 16:13:08 -0500865 prealloc = NULL;
Chris Masond1310b22008-01-24 16:13:08 -0500866 goto out;
867 }
Chris Masond1310b22008-01-24 16:13:08 -0500868 state = rb_entry(node, struct extent_state, rb_node);
Chris Mason40431d62009-08-05 12:57:59 -0400869hit_next:
Chris Masond1310b22008-01-24 16:13:08 -0500870 last_start = state->start;
871 last_end = state->end;
872
873 /*
874 * | ---- desired range ---- |
875 * | state |
876 *
877 * Just lock what we found and keep going
878 */
879 if (state->start == start && state->end <= end) {
Chris Mason1edbb732009-09-02 13:24:36 -0400880 if (state->state & exclusive_bits) {
Chris Masond1310b22008-01-24 16:13:08 -0500881 *failed_start = state->start;
882 err = -EEXIST;
883 goto out;
884 }
Chris Mason42daec22009-09-23 19:51:09 -0400885
Jeff Mahoney1bf85042011-07-21 16:56:09 +0000886 set_state_bits(tree, state, &bits);
Chris Mason2c64c532009-09-02 15:04:12 -0400887 cache_state(state, cached_state);
Chris Masond1310b22008-01-24 16:13:08 -0500888 merge_state(tree, state);
Yan Zheng5c939df2009-05-27 09:16:03 -0400889 if (last_end == (u64)-1)
890 goto out;
891 start = last_end + 1;
Liu Bod1ac6e42012-05-10 18:10:39 +0800892 state = next_state(state);
893 if (start < end && state && state->start == start &&
894 !need_resched())
895 goto hit_next;
Chris Masond1310b22008-01-24 16:13:08 -0500896 goto search_again;
897 }
898
899 /*
900 * | ---- desired range ---- |
901 * | state |
902 * or
903 * | ------------- state -------------- |
904 *
905 * We need to split the extent we found, and may flip bits on
906 * second half.
907 *
908 * If the extent we found extends past our
909 * range, we just split and search again. It'll get split
910 * again the next time though.
911 *
912 * If the extent we found is inside our range, we set the
913 * desired bit on it.
914 */
915 if (state->start < start) {
Chris Mason1edbb732009-09-02 13:24:36 -0400916 if (state->state & exclusive_bits) {
Chris Masond1310b22008-01-24 16:13:08 -0500917 *failed_start = start;
918 err = -EEXIST;
919 goto out;
920 }
Xiao Guangrong82337672011-04-20 06:44:57 +0000921
922 prealloc = alloc_extent_state_atomic(prealloc);
923 BUG_ON(!prealloc);
Chris Masond1310b22008-01-24 16:13:08 -0500924 err = split_state(tree, state, prealloc, start);
Jeff Mahoneyc2d904e2011-10-03 23:22:32 -0400925 if (err)
926 extent_io_tree_panic(tree, err);
927
Chris Masond1310b22008-01-24 16:13:08 -0500928 prealloc = NULL;
929 if (err)
930 goto out;
931 if (state->end <= end) {
Jeff Mahoney1bf85042011-07-21 16:56:09 +0000932 set_state_bits(tree, state, &bits);
Chris Mason2c64c532009-09-02 15:04:12 -0400933 cache_state(state, cached_state);
Chris Masond1310b22008-01-24 16:13:08 -0500934 merge_state(tree, state);
Yan Zheng5c939df2009-05-27 09:16:03 -0400935 if (last_end == (u64)-1)
936 goto out;
937 start = last_end + 1;
Liu Bod1ac6e42012-05-10 18:10:39 +0800938 state = next_state(state);
939 if (start < end && state && state->start == start &&
940 !need_resched())
941 goto hit_next;
Chris Masond1310b22008-01-24 16:13:08 -0500942 }
943 goto search_again;
944 }
945 /*
946 * | ---- desired range ---- |
947 * | state | or | state |
948 *
949 * There's a hole, we need to insert something in it and
950 * ignore the extent we found.
951 */
952 if (state->start > start) {
953 u64 this_end;
954 if (end < last_start)
955 this_end = end;
956 else
Chris Masond3977122009-01-05 21:25:51 -0500957 this_end = last_start - 1;
Xiao Guangrong82337672011-04-20 06:44:57 +0000958
959 prealloc = alloc_extent_state_atomic(prealloc);
960 BUG_ON(!prealloc);
Xiao Guangrongc7f895a2011-04-20 06:45:49 +0000961
962 /*
963 * Avoid to free 'prealloc' if it can be merged with
964 * the later extent.
965 */
Chris Masond1310b22008-01-24 16:13:08 -0500966 err = insert_state(tree, prealloc, start, this_end,
Filipe David Borba Manana12cfbad2013-11-26 15:41:47 +0000967 NULL, NULL, &bits);
Jeff Mahoneyc2d904e2011-10-03 23:22:32 -0400968 if (err)
969 extent_io_tree_panic(tree, err);
970
Chris Mason2c64c532009-09-02 15:04:12 -0400971 cache_state(prealloc, cached_state);
Chris Masond1310b22008-01-24 16:13:08 -0500972 prealloc = NULL;
Chris Masond1310b22008-01-24 16:13:08 -0500973 start = this_end + 1;
974 goto search_again;
975 }
976 /*
977 * | ---- desired range ---- |
978 * | state |
979 * We need to split the extent, and set the bit
980 * on the first half
981 */
982 if (state->start <= end && state->end > end) {
Chris Mason1edbb732009-09-02 13:24:36 -0400983 if (state->state & exclusive_bits) {
Chris Masond1310b22008-01-24 16:13:08 -0500984 *failed_start = start;
985 err = -EEXIST;
986 goto out;
987 }
Xiao Guangrong82337672011-04-20 06:44:57 +0000988
989 prealloc = alloc_extent_state_atomic(prealloc);
990 BUG_ON(!prealloc);
Chris Masond1310b22008-01-24 16:13:08 -0500991 err = split_state(tree, state, prealloc, end + 1);
Jeff Mahoneyc2d904e2011-10-03 23:22:32 -0400992 if (err)
993 extent_io_tree_panic(tree, err);
Chris Masond1310b22008-01-24 16:13:08 -0500994
Jeff Mahoney1bf85042011-07-21 16:56:09 +0000995 set_state_bits(tree, prealloc, &bits);
Chris Mason2c64c532009-09-02 15:04:12 -0400996 cache_state(prealloc, cached_state);
Chris Masond1310b22008-01-24 16:13:08 -0500997 merge_state(tree, prealloc);
998 prealloc = NULL;
999 goto out;
1000 }
1001
1002 goto search_again;
1003
1004out:
Chris Masoncad321a2008-12-17 14:51:42 -05001005 spin_unlock(&tree->lock);
Chris Masond1310b22008-01-24 16:13:08 -05001006 if (prealloc)
1007 free_extent_state(prealloc);
1008
1009 return err;
1010
1011search_again:
1012 if (start > end)
1013 goto out;
Chris Masoncad321a2008-12-17 14:51:42 -05001014 spin_unlock(&tree->lock);
Chris Masond1310b22008-01-24 16:13:08 -05001015 if (mask & __GFP_WAIT)
1016 cond_resched();
1017 goto again;
1018}
Chris Masond1310b22008-01-24 16:13:08 -05001019
David Sterba41074882013-04-29 13:38:46 +00001020int set_extent_bit(struct extent_io_tree *tree, u64 start, u64 end,
1021 unsigned long bits, u64 * failed_start,
1022 struct extent_state **cached_state, gfp_t mask)
Jeff Mahoney3fbe5c02012-03-01 14:57:19 +01001023{
1024 return __set_extent_bit(tree, start, end, bits, 0, failed_start,
1025 cached_state, mask);
1026}
1027
1028
Josef Bacik462d6fa2011-09-26 13:56:12 -04001029/**
Liu Bo10983f22012-07-11 15:26:19 +08001030 * convert_extent_bit - convert all bits in a given range from one bit to
1031 * another
Josef Bacik462d6fa2011-09-26 13:56:12 -04001032 * @tree: the io tree to search
1033 * @start: the start offset in bytes
1034 * @end: the end offset in bytes (inclusive)
1035 * @bits: the bits to set in this range
1036 * @clear_bits: the bits to clear in this range
Josef Bacike6138872012-09-27 17:07:30 -04001037 * @cached_state: state that we're going to cache
Josef Bacik462d6fa2011-09-26 13:56:12 -04001038 * @mask: the allocation mask
1039 *
1040 * This will go through and set bits for the given range. If any states exist
1041 * already in this range they are set with the given bit and cleared of the
1042 * clear_bits. This is only meant to be used by things that are mergeable, ie
1043 * converting from say DELALLOC to DIRTY. This is not meant to be used with
1044 * boundary bits like LOCK.
1045 */
1046int convert_extent_bit(struct extent_io_tree *tree, u64 start, u64 end,
David Sterba41074882013-04-29 13:38:46 +00001047 unsigned long bits, unsigned long clear_bits,
Josef Bacike6138872012-09-27 17:07:30 -04001048 struct extent_state **cached_state, gfp_t mask)
Josef Bacik462d6fa2011-09-26 13:56:12 -04001049{
1050 struct extent_state *state;
1051 struct extent_state *prealloc = NULL;
1052 struct rb_node *node;
Filipe David Borba Manana12cfbad2013-11-26 15:41:47 +00001053 struct rb_node **p;
1054 struct rb_node *parent;
Josef Bacik462d6fa2011-09-26 13:56:12 -04001055 int err = 0;
1056 u64 last_start;
1057 u64 last_end;
1058
Josef Bacika5dee372013-12-13 10:02:44 -05001059 btrfs_debug_check_extent_io_range(tree, start, end);
David Sterba8d599ae2013-04-30 15:22:23 +00001060
Josef Bacik462d6fa2011-09-26 13:56:12 -04001061again:
1062 if (!prealloc && (mask & __GFP_WAIT)) {
1063 prealloc = alloc_extent_state(mask);
1064 if (!prealloc)
1065 return -ENOMEM;
1066 }
1067
1068 spin_lock(&tree->lock);
Josef Bacike6138872012-09-27 17:07:30 -04001069 if (cached_state && *cached_state) {
1070 state = *cached_state;
1071 if (state->start <= start && state->end > start &&
1072 state->tree) {
1073 node = &state->rb_node;
1074 goto hit_next;
1075 }
1076 }
1077
Josef Bacik462d6fa2011-09-26 13:56:12 -04001078 /*
1079 * this search will find all the extents that end after
1080 * our range starts.
1081 */
Filipe David Borba Manana12cfbad2013-11-26 15:41:47 +00001082 node = tree_search_for_insert(tree, start, &p, &parent);
Josef Bacik462d6fa2011-09-26 13:56:12 -04001083 if (!node) {
1084 prealloc = alloc_extent_state_atomic(prealloc);
Liu Bo1cf4ffd2011-12-07 20:08:40 -05001085 if (!prealloc) {
1086 err = -ENOMEM;
1087 goto out;
1088 }
Filipe David Borba Manana12cfbad2013-11-26 15:41:47 +00001089 err = insert_state(tree, prealloc, start, end,
1090 &p, &parent, &bits);
Jeff Mahoneyc2d904e2011-10-03 23:22:32 -04001091 if (err)
1092 extent_io_tree_panic(tree, err);
Filipe David Borba Mananac42ac0b2013-11-26 15:01:34 +00001093 cache_state(prealloc, cached_state);
1094 prealloc = NULL;
Josef Bacik462d6fa2011-09-26 13:56:12 -04001095 goto out;
1096 }
1097 state = rb_entry(node, struct extent_state, rb_node);
1098hit_next:
1099 last_start = state->start;
1100 last_end = state->end;
1101
1102 /*
1103 * | ---- desired range ---- |
1104 * | state |
1105 *
1106 * Just lock what we found and keep going
1107 */
1108 if (state->start == start && state->end <= end) {
Josef Bacik462d6fa2011-09-26 13:56:12 -04001109 set_state_bits(tree, state, &bits);
Josef Bacike6138872012-09-27 17:07:30 -04001110 cache_state(state, cached_state);
Liu Bod1ac6e42012-05-10 18:10:39 +08001111 state = clear_state_bit(tree, state, &clear_bits, 0);
Josef Bacik462d6fa2011-09-26 13:56:12 -04001112 if (last_end == (u64)-1)
1113 goto out;
Josef Bacik462d6fa2011-09-26 13:56:12 -04001114 start = last_end + 1;
Liu Bod1ac6e42012-05-10 18:10:39 +08001115 if (start < end && state && state->start == start &&
1116 !need_resched())
1117 goto hit_next;
Josef Bacik462d6fa2011-09-26 13:56:12 -04001118 goto search_again;
1119 }
1120
1121 /*
1122 * | ---- desired range ---- |
1123 * | state |
1124 * or
1125 * | ------------- state -------------- |
1126 *
1127 * We need to split the extent we found, and may flip bits on
1128 * second half.
1129 *
1130 * If the extent we found extends past our
1131 * range, we just split and search again. It'll get split
1132 * again the next time though.
1133 *
1134 * If the extent we found is inside our range, we set the
1135 * desired bit on it.
1136 */
1137 if (state->start < start) {
1138 prealloc = alloc_extent_state_atomic(prealloc);
Liu Bo1cf4ffd2011-12-07 20:08:40 -05001139 if (!prealloc) {
1140 err = -ENOMEM;
1141 goto out;
1142 }
Josef Bacik462d6fa2011-09-26 13:56:12 -04001143 err = split_state(tree, state, prealloc, start);
Jeff Mahoneyc2d904e2011-10-03 23:22:32 -04001144 if (err)
1145 extent_io_tree_panic(tree, err);
Josef Bacik462d6fa2011-09-26 13:56:12 -04001146 prealloc = NULL;
1147 if (err)
1148 goto out;
1149 if (state->end <= end) {
1150 set_state_bits(tree, state, &bits);
Josef Bacike6138872012-09-27 17:07:30 -04001151 cache_state(state, cached_state);
Liu Bod1ac6e42012-05-10 18:10:39 +08001152 state = clear_state_bit(tree, state, &clear_bits, 0);
Josef Bacik462d6fa2011-09-26 13:56:12 -04001153 if (last_end == (u64)-1)
1154 goto out;
1155 start = last_end + 1;
Liu Bod1ac6e42012-05-10 18:10:39 +08001156 if (start < end && state && state->start == start &&
1157 !need_resched())
1158 goto hit_next;
Josef Bacik462d6fa2011-09-26 13:56:12 -04001159 }
1160 goto search_again;
1161 }
1162 /*
1163 * | ---- desired range ---- |
1164 * | state | or | state |
1165 *
1166 * There's a hole, we need to insert something in it and
1167 * ignore the extent we found.
1168 */
1169 if (state->start > start) {
1170 u64 this_end;
1171 if (end < last_start)
1172 this_end = end;
1173 else
1174 this_end = last_start - 1;
1175
1176 prealloc = alloc_extent_state_atomic(prealloc);
Liu Bo1cf4ffd2011-12-07 20:08:40 -05001177 if (!prealloc) {
1178 err = -ENOMEM;
1179 goto out;
1180 }
Josef Bacik462d6fa2011-09-26 13:56:12 -04001181
1182 /*
1183 * Avoid to free 'prealloc' if it can be merged with
1184 * the later extent.
1185 */
1186 err = insert_state(tree, prealloc, start, this_end,
Filipe David Borba Manana12cfbad2013-11-26 15:41:47 +00001187 NULL, NULL, &bits);
Jeff Mahoneyc2d904e2011-10-03 23:22:32 -04001188 if (err)
1189 extent_io_tree_panic(tree, err);
Josef Bacike6138872012-09-27 17:07:30 -04001190 cache_state(prealloc, cached_state);
Josef Bacik462d6fa2011-09-26 13:56:12 -04001191 prealloc = NULL;
1192 start = this_end + 1;
1193 goto search_again;
1194 }
1195 /*
1196 * | ---- desired range ---- |
1197 * | state |
1198 * We need to split the extent, and set the bit
1199 * on the first half
1200 */
1201 if (state->start <= end && state->end > end) {
1202 prealloc = alloc_extent_state_atomic(prealloc);
Liu Bo1cf4ffd2011-12-07 20:08:40 -05001203 if (!prealloc) {
1204 err = -ENOMEM;
1205 goto out;
1206 }
Josef Bacik462d6fa2011-09-26 13:56:12 -04001207
1208 err = split_state(tree, state, prealloc, end + 1);
Jeff Mahoneyc2d904e2011-10-03 23:22:32 -04001209 if (err)
1210 extent_io_tree_panic(tree, err);
Josef Bacik462d6fa2011-09-26 13:56:12 -04001211
1212 set_state_bits(tree, prealloc, &bits);
Josef Bacike6138872012-09-27 17:07:30 -04001213 cache_state(prealloc, cached_state);
Josef Bacik462d6fa2011-09-26 13:56:12 -04001214 clear_state_bit(tree, prealloc, &clear_bits, 0);
Josef Bacik462d6fa2011-09-26 13:56:12 -04001215 prealloc = NULL;
1216 goto out;
1217 }
1218
1219 goto search_again;
1220
1221out:
1222 spin_unlock(&tree->lock);
1223 if (prealloc)
1224 free_extent_state(prealloc);
1225
1226 return err;
1227
1228search_again:
1229 if (start > end)
1230 goto out;
1231 spin_unlock(&tree->lock);
1232 if (mask & __GFP_WAIT)
1233 cond_resched();
1234 goto again;
1235}
1236
Chris Masond1310b22008-01-24 16:13:08 -05001237/* wrappers around set/clear extent bit */
1238int set_extent_dirty(struct extent_io_tree *tree, u64 start, u64 end,
1239 gfp_t mask)
1240{
Jeff Mahoney3fbe5c02012-03-01 14:57:19 +01001241 return set_extent_bit(tree, start, end, EXTENT_DIRTY, NULL,
Chris Mason2c64c532009-09-02 15:04:12 -04001242 NULL, mask);
Chris Masond1310b22008-01-24 16:13:08 -05001243}
Chris Masond1310b22008-01-24 16:13:08 -05001244
1245int set_extent_bits(struct extent_io_tree *tree, u64 start, u64 end,
David Sterba41074882013-04-29 13:38:46 +00001246 unsigned long bits, gfp_t mask)
Chris Masond1310b22008-01-24 16:13:08 -05001247{
Jeff Mahoney3fbe5c02012-03-01 14:57:19 +01001248 return set_extent_bit(tree, start, end, bits, NULL,
Chris Mason2c64c532009-09-02 15:04:12 -04001249 NULL, mask);
Chris Masond1310b22008-01-24 16:13:08 -05001250}
Chris Masond1310b22008-01-24 16:13:08 -05001251
1252int clear_extent_bits(struct extent_io_tree *tree, u64 start, u64 end,
David Sterba41074882013-04-29 13:38:46 +00001253 unsigned long bits, gfp_t mask)
Chris Masond1310b22008-01-24 16:13:08 -05001254{
Chris Mason2c64c532009-09-02 15:04:12 -04001255 return clear_extent_bit(tree, start, end, bits, 0, 0, NULL, mask);
Chris Masond1310b22008-01-24 16:13:08 -05001256}
Chris Masond1310b22008-01-24 16:13:08 -05001257
1258int set_extent_delalloc(struct extent_io_tree *tree, u64 start, u64 end,
Josef Bacik2ac55d42010-02-03 19:33:23 +00001259 struct extent_state **cached_state, gfp_t mask)
Chris Masond1310b22008-01-24 16:13:08 -05001260{
1261 return set_extent_bit(tree, start, end,
Liu Bofee187d2011-09-29 15:55:28 +08001262 EXTENT_DELALLOC | EXTENT_UPTODATE,
Jeff Mahoney3fbe5c02012-03-01 14:57:19 +01001263 NULL, cached_state, mask);
Chris Masond1310b22008-01-24 16:13:08 -05001264}
Chris Masond1310b22008-01-24 16:13:08 -05001265
Liu Bo9e8a4a82012-09-05 19:10:51 -06001266int set_extent_defrag(struct extent_io_tree *tree, u64 start, u64 end,
1267 struct extent_state **cached_state, gfp_t mask)
1268{
1269 return set_extent_bit(tree, start, end,
1270 EXTENT_DELALLOC | EXTENT_UPTODATE | EXTENT_DEFRAG,
1271 NULL, cached_state, mask);
1272}
1273
Chris Masond1310b22008-01-24 16:13:08 -05001274int clear_extent_dirty(struct extent_io_tree *tree, u64 start, u64 end,
1275 gfp_t mask)
1276{
1277 return clear_extent_bit(tree, start, end,
Josef Bacik32c00af2009-10-08 13:34:05 -04001278 EXTENT_DIRTY | EXTENT_DELALLOC |
Yan, Zheng0ca1f7c2010-05-16 10:48:47 -04001279 EXTENT_DO_ACCOUNTING, 0, 0, NULL, mask);
Chris Masond1310b22008-01-24 16:13:08 -05001280}
Chris Masond1310b22008-01-24 16:13:08 -05001281
1282int set_extent_new(struct extent_io_tree *tree, u64 start, u64 end,
1283 gfp_t mask)
1284{
Jeff Mahoney3fbe5c02012-03-01 14:57:19 +01001285 return set_extent_bit(tree, start, end, EXTENT_NEW, NULL,
Chris Mason2c64c532009-09-02 15:04:12 -04001286 NULL, mask);
Chris Masond1310b22008-01-24 16:13:08 -05001287}
Chris Masond1310b22008-01-24 16:13:08 -05001288
Chris Masond1310b22008-01-24 16:13:08 -05001289int set_extent_uptodate(struct extent_io_tree *tree, u64 start, u64 end,
Arne Jansen507903b2011-04-06 10:02:20 +00001290 struct extent_state **cached_state, gfp_t mask)
Chris Masond1310b22008-01-24 16:13:08 -05001291{
Liu Bo6b67a322013-03-28 08:30:28 +00001292 return set_extent_bit(tree, start, end, EXTENT_UPTODATE, NULL,
Jeff Mahoney3fbe5c02012-03-01 14:57:19 +01001293 cached_state, mask);
Chris Masond1310b22008-01-24 16:13:08 -05001294}
Chris Masond1310b22008-01-24 16:13:08 -05001295
Josef Bacik5fd02042012-05-02 14:00:54 -04001296int clear_extent_uptodate(struct extent_io_tree *tree, u64 start, u64 end,
1297 struct extent_state **cached_state, gfp_t mask)
Chris Masond1310b22008-01-24 16:13:08 -05001298{
Chris Mason2c64c532009-09-02 15:04:12 -04001299 return clear_extent_bit(tree, start, end, EXTENT_UPTODATE, 0, 0,
Josef Bacik2ac55d42010-02-03 19:33:23 +00001300 cached_state, mask);
Chris Masond1310b22008-01-24 16:13:08 -05001301}
Chris Masond1310b22008-01-24 16:13:08 -05001302
Chris Masond352ac62008-09-29 15:18:18 -04001303/*
1304 * either insert or lock state struct between start and end use mask to tell
1305 * us if waiting is desired.
1306 */
Chris Mason1edbb732009-09-02 13:24:36 -04001307int lock_extent_bits(struct extent_io_tree *tree, u64 start, u64 end,
David Sterba41074882013-04-29 13:38:46 +00001308 unsigned long bits, struct extent_state **cached_state)
Chris Masond1310b22008-01-24 16:13:08 -05001309{
1310 int err;
1311 u64 failed_start;
1312 while (1) {
Jeff Mahoney3fbe5c02012-03-01 14:57:19 +01001313 err = __set_extent_bit(tree, start, end, EXTENT_LOCKED | bits,
1314 EXTENT_LOCKED, &failed_start,
1315 cached_state, GFP_NOFS);
Jeff Mahoneyd0082372012-03-01 14:57:19 +01001316 if (err == -EEXIST) {
Chris Masond1310b22008-01-24 16:13:08 -05001317 wait_extent_bit(tree, failed_start, end, EXTENT_LOCKED);
1318 start = failed_start;
Jeff Mahoneyd0082372012-03-01 14:57:19 +01001319 } else
Chris Masond1310b22008-01-24 16:13:08 -05001320 break;
Chris Masond1310b22008-01-24 16:13:08 -05001321 WARN_ON(start > end);
1322 }
1323 return err;
1324}
Chris Masond1310b22008-01-24 16:13:08 -05001325
Jeff Mahoneyd0082372012-03-01 14:57:19 +01001326int lock_extent(struct extent_io_tree *tree, u64 start, u64 end)
Chris Mason1edbb732009-09-02 13:24:36 -04001327{
Jeff Mahoneyd0082372012-03-01 14:57:19 +01001328 return lock_extent_bits(tree, start, end, 0, NULL);
Chris Mason1edbb732009-09-02 13:24:36 -04001329}
1330
Jeff Mahoneyd0082372012-03-01 14:57:19 +01001331int try_lock_extent(struct extent_io_tree *tree, u64 start, u64 end)
Josef Bacik25179202008-10-29 14:49:05 -04001332{
1333 int err;
1334 u64 failed_start;
1335
Jeff Mahoney3fbe5c02012-03-01 14:57:19 +01001336 err = __set_extent_bit(tree, start, end, EXTENT_LOCKED, EXTENT_LOCKED,
1337 &failed_start, NULL, GFP_NOFS);
Yan Zheng66435582008-10-30 14:19:50 -04001338 if (err == -EEXIST) {
1339 if (failed_start > start)
1340 clear_extent_bit(tree, start, failed_start - 1,
Jeff Mahoneyd0082372012-03-01 14:57:19 +01001341 EXTENT_LOCKED, 1, 0, NULL, GFP_NOFS);
Josef Bacik25179202008-10-29 14:49:05 -04001342 return 0;
Yan Zheng66435582008-10-30 14:19:50 -04001343 }
Josef Bacik25179202008-10-29 14:49:05 -04001344 return 1;
1345}
Josef Bacik25179202008-10-29 14:49:05 -04001346
Chris Mason2c64c532009-09-02 15:04:12 -04001347int unlock_extent_cached(struct extent_io_tree *tree, u64 start, u64 end,
1348 struct extent_state **cached, gfp_t mask)
1349{
1350 return clear_extent_bit(tree, start, end, EXTENT_LOCKED, 1, 0, cached,
1351 mask);
1352}
1353
Jeff Mahoneyd0082372012-03-01 14:57:19 +01001354int unlock_extent(struct extent_io_tree *tree, u64 start, u64 end)
Chris Masond1310b22008-01-24 16:13:08 -05001355{
Chris Mason2c64c532009-09-02 15:04:12 -04001356 return clear_extent_bit(tree, start, end, EXTENT_LOCKED, 1, 0, NULL,
Jeff Mahoneyd0082372012-03-01 14:57:19 +01001357 GFP_NOFS);
Chris Masond1310b22008-01-24 16:13:08 -05001358}
Chris Masond1310b22008-01-24 16:13:08 -05001359
Chris Mason4adaa612013-03-26 13:07:00 -04001360int extent_range_clear_dirty_for_io(struct inode *inode, u64 start, u64 end)
1361{
1362 unsigned long index = start >> PAGE_CACHE_SHIFT;
1363 unsigned long end_index = end >> PAGE_CACHE_SHIFT;
1364 struct page *page;
1365
1366 while (index <= end_index) {
1367 page = find_get_page(inode->i_mapping, index);
1368 BUG_ON(!page); /* Pages should be in the extent_io_tree */
1369 clear_page_dirty_for_io(page);
1370 page_cache_release(page);
1371 index++;
1372 }
1373 return 0;
1374}
1375
1376int extent_range_redirty_for_io(struct inode *inode, u64 start, u64 end)
1377{
1378 unsigned long index = start >> PAGE_CACHE_SHIFT;
1379 unsigned long end_index = end >> PAGE_CACHE_SHIFT;
1380 struct page *page;
1381
1382 while (index <= end_index) {
1383 page = find_get_page(inode->i_mapping, index);
1384 BUG_ON(!page); /* Pages should be in the extent_io_tree */
1385 account_page_redirty(page);
1386 __set_page_dirty_nobuffers(page);
1387 page_cache_release(page);
1388 index++;
1389 }
1390 return 0;
1391}
1392
Chris Masond1310b22008-01-24 16:13:08 -05001393/*
Chris Masond1310b22008-01-24 16:13:08 -05001394 * helper function to set both pages and extents in the tree writeback
1395 */
Christoph Hellwigb2950862008-12-02 09:54:17 -05001396static int set_range_writeback(struct extent_io_tree *tree, u64 start, u64 end)
Chris Masond1310b22008-01-24 16:13:08 -05001397{
1398 unsigned long index = start >> PAGE_CACHE_SHIFT;
1399 unsigned long end_index = end >> PAGE_CACHE_SHIFT;
1400 struct page *page;
1401
1402 while (index <= end_index) {
1403 page = find_get_page(tree->mapping, index);
Jeff Mahoney79787ea2012-03-12 16:03:00 +01001404 BUG_ON(!page); /* Pages should be in the extent_io_tree */
Chris Masond1310b22008-01-24 16:13:08 -05001405 set_page_writeback(page);
1406 page_cache_release(page);
1407 index++;
1408 }
Chris Masond1310b22008-01-24 16:13:08 -05001409 return 0;
1410}
Chris Masond1310b22008-01-24 16:13:08 -05001411
Chris Masond352ac62008-09-29 15:18:18 -04001412/* find the first state struct with 'bits' set after 'start', and
1413 * return it. tree->lock must be held. NULL will returned if
1414 * nothing was found after 'start'
1415 */
Eric Sandeen48a3b632013-04-25 20:41:01 +00001416static struct extent_state *
1417find_first_extent_bit_state(struct extent_io_tree *tree,
David Sterba41074882013-04-29 13:38:46 +00001418 u64 start, unsigned long bits)
Chris Masond7fc6402008-02-18 12:12:38 -05001419{
1420 struct rb_node *node;
1421 struct extent_state *state;
1422
1423 /*
1424 * this search will find all the extents that end after
1425 * our range starts.
1426 */
1427 node = tree_search(tree, start);
Chris Masond3977122009-01-05 21:25:51 -05001428 if (!node)
Chris Masond7fc6402008-02-18 12:12:38 -05001429 goto out;
Chris Masond7fc6402008-02-18 12:12:38 -05001430
Chris Masond3977122009-01-05 21:25:51 -05001431 while (1) {
Chris Masond7fc6402008-02-18 12:12:38 -05001432 state = rb_entry(node, struct extent_state, rb_node);
Chris Masond3977122009-01-05 21:25:51 -05001433 if (state->end >= start && (state->state & bits))
Chris Masond7fc6402008-02-18 12:12:38 -05001434 return state;
Chris Masond3977122009-01-05 21:25:51 -05001435
Chris Masond7fc6402008-02-18 12:12:38 -05001436 node = rb_next(node);
1437 if (!node)
1438 break;
1439 }
1440out:
1441 return NULL;
1442}
Chris Masond7fc6402008-02-18 12:12:38 -05001443
Chris Masond352ac62008-09-29 15:18:18 -04001444/*
Xiao Guangrong69261c42011-07-14 03:19:45 +00001445 * find the first offset in the io tree with 'bits' set. zero is
1446 * returned if we find something, and *start_ret and *end_ret are
1447 * set to reflect the state struct that was found.
1448 *
Wang Sheng-Hui477d7ea2012-04-06 14:35:47 +08001449 * If nothing was found, 1 is returned. If found something, return 0.
Xiao Guangrong69261c42011-07-14 03:19:45 +00001450 */
1451int find_first_extent_bit(struct extent_io_tree *tree, u64 start,
David Sterba41074882013-04-29 13:38:46 +00001452 u64 *start_ret, u64 *end_ret, unsigned long bits,
Josef Bacike6138872012-09-27 17:07:30 -04001453 struct extent_state **cached_state)
Xiao Guangrong69261c42011-07-14 03:19:45 +00001454{
1455 struct extent_state *state;
Josef Bacike6138872012-09-27 17:07:30 -04001456 struct rb_node *n;
Xiao Guangrong69261c42011-07-14 03:19:45 +00001457 int ret = 1;
1458
1459 spin_lock(&tree->lock);
Josef Bacike6138872012-09-27 17:07:30 -04001460 if (cached_state && *cached_state) {
1461 state = *cached_state;
1462 if (state->end == start - 1 && state->tree) {
1463 n = rb_next(&state->rb_node);
1464 while (n) {
1465 state = rb_entry(n, struct extent_state,
1466 rb_node);
1467 if (state->state & bits)
1468 goto got_it;
1469 n = rb_next(n);
1470 }
1471 free_extent_state(*cached_state);
1472 *cached_state = NULL;
1473 goto out;
1474 }
1475 free_extent_state(*cached_state);
1476 *cached_state = NULL;
1477 }
1478
Xiao Guangrong69261c42011-07-14 03:19:45 +00001479 state = find_first_extent_bit_state(tree, start, bits);
Josef Bacike6138872012-09-27 17:07:30 -04001480got_it:
Xiao Guangrong69261c42011-07-14 03:19:45 +00001481 if (state) {
Josef Bacike6138872012-09-27 17:07:30 -04001482 cache_state(state, cached_state);
Xiao Guangrong69261c42011-07-14 03:19:45 +00001483 *start_ret = state->start;
1484 *end_ret = state->end;
1485 ret = 0;
1486 }
Josef Bacike6138872012-09-27 17:07:30 -04001487out:
Xiao Guangrong69261c42011-07-14 03:19:45 +00001488 spin_unlock(&tree->lock);
1489 return ret;
1490}
1491
1492/*
Chris Masond352ac62008-09-29 15:18:18 -04001493 * find a contiguous range of bytes in the file marked as delalloc, not
1494 * more than 'max_bytes'. start and end are used to return the range,
1495 *
1496 * 1 is returned if we find something, 0 if nothing was in the tree
1497 */
Chris Masonc8b97812008-10-29 14:49:59 -04001498static noinline u64 find_delalloc_range(struct extent_io_tree *tree,
Josef Bacikc2a128d2010-02-02 21:19:11 +00001499 u64 *start, u64 *end, u64 max_bytes,
1500 struct extent_state **cached_state)
Chris Masond1310b22008-01-24 16:13:08 -05001501{
1502 struct rb_node *node;
1503 struct extent_state *state;
1504 u64 cur_start = *start;
1505 u64 found = 0;
1506 u64 total_bytes = 0;
1507
Chris Masoncad321a2008-12-17 14:51:42 -05001508 spin_lock(&tree->lock);
Chris Masonc8b97812008-10-29 14:49:59 -04001509
Chris Masond1310b22008-01-24 16:13:08 -05001510 /*
1511 * this search will find all the extents that end after
1512 * our range starts.
1513 */
Chris Mason80ea96b2008-02-01 14:51:59 -05001514 node = tree_search(tree, cur_start);
Peter2b114d12008-04-01 11:21:40 -04001515 if (!node) {
Chris Mason3b951512008-04-17 11:29:12 -04001516 if (!found)
1517 *end = (u64)-1;
Chris Masond1310b22008-01-24 16:13:08 -05001518 goto out;
1519 }
1520
Chris Masond3977122009-01-05 21:25:51 -05001521 while (1) {
Chris Masond1310b22008-01-24 16:13:08 -05001522 state = rb_entry(node, struct extent_state, rb_node);
Zheng Yan5b21f2e2008-09-26 10:05:38 -04001523 if (found && (state->start != cur_start ||
1524 (state->state & EXTENT_BOUNDARY))) {
Chris Masond1310b22008-01-24 16:13:08 -05001525 goto out;
1526 }
1527 if (!(state->state & EXTENT_DELALLOC)) {
1528 if (!found)
1529 *end = state->end;
1530 goto out;
1531 }
Josef Bacikc2a128d2010-02-02 21:19:11 +00001532 if (!found) {
Chris Masond1310b22008-01-24 16:13:08 -05001533 *start = state->start;
Josef Bacikc2a128d2010-02-02 21:19:11 +00001534 *cached_state = state;
1535 atomic_inc(&state->refs);
1536 }
Chris Masond1310b22008-01-24 16:13:08 -05001537 found++;
1538 *end = state->end;
1539 cur_start = state->end + 1;
1540 node = rb_next(node);
Chris Masond1310b22008-01-24 16:13:08 -05001541 total_bytes += state->end - state->start + 1;
Josef Bacik7bf811a52013-10-07 22:11:09 -04001542 if (total_bytes >= max_bytes)
Josef Bacik573aeca2013-08-30 14:38:49 -04001543 break;
Josef Bacik573aeca2013-08-30 14:38:49 -04001544 if (!node)
Chris Masond1310b22008-01-24 16:13:08 -05001545 break;
1546 }
1547out:
Chris Masoncad321a2008-12-17 14:51:42 -05001548 spin_unlock(&tree->lock);
Chris Masond1310b22008-01-24 16:13:08 -05001549 return found;
1550}
1551
Jeff Mahoney143bede2012-03-01 14:56:26 +01001552static noinline void __unlock_for_delalloc(struct inode *inode,
1553 struct page *locked_page,
1554 u64 start, u64 end)
Chris Masonc8b97812008-10-29 14:49:59 -04001555{
1556 int ret;
1557 struct page *pages[16];
1558 unsigned long index = start >> PAGE_CACHE_SHIFT;
1559 unsigned long end_index = end >> PAGE_CACHE_SHIFT;
1560 unsigned long nr_pages = end_index - index + 1;
1561 int i;
1562
1563 if (index == locked_page->index && end_index == index)
Jeff Mahoney143bede2012-03-01 14:56:26 +01001564 return;
Chris Masonc8b97812008-10-29 14:49:59 -04001565
Chris Masond3977122009-01-05 21:25:51 -05001566 while (nr_pages > 0) {
Chris Masonc8b97812008-10-29 14:49:59 -04001567 ret = find_get_pages_contig(inode->i_mapping, index,
Chris Mason5b050f02008-11-11 09:34:41 -05001568 min_t(unsigned long, nr_pages,
1569 ARRAY_SIZE(pages)), pages);
Chris Masonc8b97812008-10-29 14:49:59 -04001570 for (i = 0; i < ret; i++) {
1571 if (pages[i] != locked_page)
1572 unlock_page(pages[i]);
1573 page_cache_release(pages[i]);
1574 }
1575 nr_pages -= ret;
1576 index += ret;
1577 cond_resched();
1578 }
Chris Masonc8b97812008-10-29 14:49:59 -04001579}
1580
1581static noinline int lock_delalloc_pages(struct inode *inode,
1582 struct page *locked_page,
1583 u64 delalloc_start,
1584 u64 delalloc_end)
1585{
1586 unsigned long index = delalloc_start >> PAGE_CACHE_SHIFT;
1587 unsigned long start_index = index;
1588 unsigned long end_index = delalloc_end >> PAGE_CACHE_SHIFT;
1589 unsigned long pages_locked = 0;
1590 struct page *pages[16];
1591 unsigned long nrpages;
1592 int ret;
1593 int i;
1594
1595 /* the caller is responsible for locking the start index */
1596 if (index == locked_page->index && index == end_index)
1597 return 0;
1598
1599 /* skip the page at the start index */
1600 nrpages = end_index - index + 1;
Chris Masond3977122009-01-05 21:25:51 -05001601 while (nrpages > 0) {
Chris Masonc8b97812008-10-29 14:49:59 -04001602 ret = find_get_pages_contig(inode->i_mapping, index,
Chris Mason5b050f02008-11-11 09:34:41 -05001603 min_t(unsigned long,
1604 nrpages, ARRAY_SIZE(pages)), pages);
Chris Masonc8b97812008-10-29 14:49:59 -04001605 if (ret == 0) {
1606 ret = -EAGAIN;
1607 goto done;
1608 }
1609 /* now we have an array of pages, lock them all */
1610 for (i = 0; i < ret; i++) {
1611 /*
1612 * the caller is taking responsibility for
1613 * locked_page
1614 */
Chris Mason771ed682008-11-06 22:02:51 -05001615 if (pages[i] != locked_page) {
Chris Masonc8b97812008-10-29 14:49:59 -04001616 lock_page(pages[i]);
Chris Masonf2b1c412008-11-10 07:31:30 -05001617 if (!PageDirty(pages[i]) ||
1618 pages[i]->mapping != inode->i_mapping) {
Chris Mason771ed682008-11-06 22:02:51 -05001619 ret = -EAGAIN;
1620 unlock_page(pages[i]);
1621 page_cache_release(pages[i]);
1622 goto done;
1623 }
1624 }
Chris Masonc8b97812008-10-29 14:49:59 -04001625 page_cache_release(pages[i]);
Chris Mason771ed682008-11-06 22:02:51 -05001626 pages_locked++;
Chris Masonc8b97812008-10-29 14:49:59 -04001627 }
Chris Masonc8b97812008-10-29 14:49:59 -04001628 nrpages -= ret;
1629 index += ret;
1630 cond_resched();
1631 }
1632 ret = 0;
1633done:
1634 if (ret && pages_locked) {
1635 __unlock_for_delalloc(inode, locked_page,
1636 delalloc_start,
1637 ((u64)(start_index + pages_locked - 1)) <<
1638 PAGE_CACHE_SHIFT);
1639 }
1640 return ret;
1641}
1642
1643/*
1644 * find a contiguous range of bytes in the file marked as delalloc, not
1645 * more than 'max_bytes'. start and end are used to return the range,
1646 *
1647 * 1 is returned if we find something, 0 if nothing was in the tree
1648 */
Josef Bacik294e30f2013-10-09 12:00:56 -04001649STATIC u64 find_lock_delalloc_range(struct inode *inode,
1650 struct extent_io_tree *tree,
1651 struct page *locked_page, u64 *start,
1652 u64 *end, u64 max_bytes)
Chris Masonc8b97812008-10-29 14:49:59 -04001653{
1654 u64 delalloc_start;
1655 u64 delalloc_end;
1656 u64 found;
Chris Mason9655d292009-09-02 15:22:30 -04001657 struct extent_state *cached_state = NULL;
Chris Masonc8b97812008-10-29 14:49:59 -04001658 int ret;
1659 int loops = 0;
1660
1661again:
1662 /* step one, find a bunch of delalloc bytes starting at start */
1663 delalloc_start = *start;
1664 delalloc_end = 0;
1665 found = find_delalloc_range(tree, &delalloc_start, &delalloc_end,
Josef Bacikc2a128d2010-02-02 21:19:11 +00001666 max_bytes, &cached_state);
Chris Mason70b99e62008-10-31 12:46:39 -04001667 if (!found || delalloc_end <= *start) {
Chris Masonc8b97812008-10-29 14:49:59 -04001668 *start = delalloc_start;
1669 *end = delalloc_end;
Josef Bacikc2a128d2010-02-02 21:19:11 +00001670 free_extent_state(cached_state);
Liu Bo385fe0b2013-10-01 23:49:49 +08001671 return 0;
Chris Masonc8b97812008-10-29 14:49:59 -04001672 }
1673
1674 /*
Chris Mason70b99e62008-10-31 12:46:39 -04001675 * start comes from the offset of locked_page. We have to lock
1676 * pages in order, so we can't process delalloc bytes before
1677 * locked_page
1678 */
Chris Masond3977122009-01-05 21:25:51 -05001679 if (delalloc_start < *start)
Chris Mason70b99e62008-10-31 12:46:39 -04001680 delalloc_start = *start;
Chris Mason70b99e62008-10-31 12:46:39 -04001681
1682 /*
Chris Masonc8b97812008-10-29 14:49:59 -04001683 * make sure to limit the number of pages we try to lock down
Chris Masonc8b97812008-10-29 14:49:59 -04001684 */
Josef Bacik7bf811a52013-10-07 22:11:09 -04001685 if (delalloc_end + 1 - delalloc_start > max_bytes)
1686 delalloc_end = delalloc_start + max_bytes - 1;
Chris Masond3977122009-01-05 21:25:51 -05001687
Chris Masonc8b97812008-10-29 14:49:59 -04001688 /* step two, lock all the pages after the page that has start */
1689 ret = lock_delalloc_pages(inode, locked_page,
1690 delalloc_start, delalloc_end);
1691 if (ret == -EAGAIN) {
1692 /* some of the pages are gone, lets avoid looping by
1693 * shortening the size of the delalloc range we're searching
1694 */
Chris Mason9655d292009-09-02 15:22:30 -04001695 free_extent_state(cached_state);
Chris Mason7d788742014-05-21 05:49:54 -07001696 cached_state = NULL;
Chris Masonc8b97812008-10-29 14:49:59 -04001697 if (!loops) {
Josef Bacik7bf811a52013-10-07 22:11:09 -04001698 max_bytes = PAGE_CACHE_SIZE;
Chris Masonc8b97812008-10-29 14:49:59 -04001699 loops = 1;
1700 goto again;
1701 } else {
1702 found = 0;
1703 goto out_failed;
1704 }
1705 }
Jeff Mahoney79787ea2012-03-12 16:03:00 +01001706 BUG_ON(ret); /* Only valid values are 0 and -EAGAIN */
Chris Masonc8b97812008-10-29 14:49:59 -04001707
1708 /* step three, lock the state bits for the whole range */
Jeff Mahoneyd0082372012-03-01 14:57:19 +01001709 lock_extent_bits(tree, delalloc_start, delalloc_end, 0, &cached_state);
Chris Masonc8b97812008-10-29 14:49:59 -04001710
1711 /* then test to make sure it is all still delalloc */
1712 ret = test_range_bit(tree, delalloc_start, delalloc_end,
Chris Mason9655d292009-09-02 15:22:30 -04001713 EXTENT_DELALLOC, 1, cached_state);
Chris Masonc8b97812008-10-29 14:49:59 -04001714 if (!ret) {
Chris Mason9655d292009-09-02 15:22:30 -04001715 unlock_extent_cached(tree, delalloc_start, delalloc_end,
1716 &cached_state, GFP_NOFS);
Chris Masonc8b97812008-10-29 14:49:59 -04001717 __unlock_for_delalloc(inode, locked_page,
1718 delalloc_start, delalloc_end);
1719 cond_resched();
1720 goto again;
1721 }
Chris Mason9655d292009-09-02 15:22:30 -04001722 free_extent_state(cached_state);
Chris Masonc8b97812008-10-29 14:49:59 -04001723 *start = delalloc_start;
1724 *end = delalloc_end;
1725out_failed:
1726 return found;
1727}
1728
Josef Bacikc2790a22013-07-29 11:20:47 -04001729int extent_clear_unlock_delalloc(struct inode *inode, u64 start, u64 end,
1730 struct page *locked_page,
1731 unsigned long clear_bits,
1732 unsigned long page_ops)
Chris Masonc8b97812008-10-29 14:49:59 -04001733{
Josef Bacikc2790a22013-07-29 11:20:47 -04001734 struct extent_io_tree *tree = &BTRFS_I(inode)->io_tree;
Chris Masonc8b97812008-10-29 14:49:59 -04001735 int ret;
1736 struct page *pages[16];
1737 unsigned long index = start >> PAGE_CACHE_SHIFT;
1738 unsigned long end_index = end >> PAGE_CACHE_SHIFT;
1739 unsigned long nr_pages = end_index - index + 1;
1740 int i;
Chris Mason771ed682008-11-06 22:02:51 -05001741
Chris Mason2c64c532009-09-02 15:04:12 -04001742 clear_extent_bit(tree, start, end, clear_bits, 1, 0, NULL, GFP_NOFS);
Josef Bacikc2790a22013-07-29 11:20:47 -04001743 if (page_ops == 0)
Chris Mason771ed682008-11-06 22:02:51 -05001744 return 0;
Chris Masonc8b97812008-10-29 14:49:59 -04001745
Chris Masond3977122009-01-05 21:25:51 -05001746 while (nr_pages > 0) {
Chris Masonc8b97812008-10-29 14:49:59 -04001747 ret = find_get_pages_contig(inode->i_mapping, index,
Chris Mason5b050f02008-11-11 09:34:41 -05001748 min_t(unsigned long,
1749 nr_pages, ARRAY_SIZE(pages)), pages);
Chris Masonc8b97812008-10-29 14:49:59 -04001750 for (i = 0; i < ret; i++) {
Chris Mason8b62b722009-09-02 16:53:46 -04001751
Josef Bacikc2790a22013-07-29 11:20:47 -04001752 if (page_ops & PAGE_SET_PRIVATE2)
Chris Mason8b62b722009-09-02 16:53:46 -04001753 SetPagePrivate2(pages[i]);
1754
Chris Masonc8b97812008-10-29 14:49:59 -04001755 if (pages[i] == locked_page) {
1756 page_cache_release(pages[i]);
1757 continue;
1758 }
Josef Bacikc2790a22013-07-29 11:20:47 -04001759 if (page_ops & PAGE_CLEAR_DIRTY)
Chris Masonc8b97812008-10-29 14:49:59 -04001760 clear_page_dirty_for_io(pages[i]);
Josef Bacikc2790a22013-07-29 11:20:47 -04001761 if (page_ops & PAGE_SET_WRITEBACK)
Chris Masonc8b97812008-10-29 14:49:59 -04001762 set_page_writeback(pages[i]);
Josef Bacikc2790a22013-07-29 11:20:47 -04001763 if (page_ops & PAGE_END_WRITEBACK)
Chris Masonc8b97812008-10-29 14:49:59 -04001764 end_page_writeback(pages[i]);
Josef Bacikc2790a22013-07-29 11:20:47 -04001765 if (page_ops & PAGE_UNLOCK)
Chris Mason771ed682008-11-06 22:02:51 -05001766 unlock_page(pages[i]);
Chris Masonc8b97812008-10-29 14:49:59 -04001767 page_cache_release(pages[i]);
1768 }
1769 nr_pages -= ret;
1770 index += ret;
1771 cond_resched();
1772 }
1773 return 0;
1774}
Chris Masonc8b97812008-10-29 14:49:59 -04001775
Chris Masond352ac62008-09-29 15:18:18 -04001776/*
1777 * count the number of bytes in the tree that have a given bit(s)
1778 * set. This can be fairly slow, except for EXTENT_DIRTY which is
1779 * cached. The total number found is returned.
1780 */
Chris Masond1310b22008-01-24 16:13:08 -05001781u64 count_range_bits(struct extent_io_tree *tree,
1782 u64 *start, u64 search_end, u64 max_bytes,
Chris Masonec29ed52011-02-23 16:23:20 -05001783 unsigned long bits, int contig)
Chris Masond1310b22008-01-24 16:13:08 -05001784{
1785 struct rb_node *node;
1786 struct extent_state *state;
1787 u64 cur_start = *start;
1788 u64 total_bytes = 0;
Chris Masonec29ed52011-02-23 16:23:20 -05001789 u64 last = 0;
Chris Masond1310b22008-01-24 16:13:08 -05001790 int found = 0;
1791
Dulshani Gunawardhanafae7f212013-10-31 10:30:08 +05301792 if (WARN_ON(search_end <= cur_start))
Chris Masond1310b22008-01-24 16:13:08 -05001793 return 0;
Chris Masond1310b22008-01-24 16:13:08 -05001794
Chris Masoncad321a2008-12-17 14:51:42 -05001795 spin_lock(&tree->lock);
Chris Masond1310b22008-01-24 16:13:08 -05001796 if (cur_start == 0 && bits == EXTENT_DIRTY) {
1797 total_bytes = tree->dirty_bytes;
1798 goto out;
1799 }
1800 /*
1801 * this search will find all the extents that end after
1802 * our range starts.
1803 */
Chris Mason80ea96b2008-02-01 14:51:59 -05001804 node = tree_search(tree, cur_start);
Chris Masond3977122009-01-05 21:25:51 -05001805 if (!node)
Chris Masond1310b22008-01-24 16:13:08 -05001806 goto out;
Chris Masond1310b22008-01-24 16:13:08 -05001807
Chris Masond3977122009-01-05 21:25:51 -05001808 while (1) {
Chris Masond1310b22008-01-24 16:13:08 -05001809 state = rb_entry(node, struct extent_state, rb_node);
1810 if (state->start > search_end)
1811 break;
Chris Masonec29ed52011-02-23 16:23:20 -05001812 if (contig && found && state->start > last + 1)
1813 break;
1814 if (state->end >= cur_start && (state->state & bits) == bits) {
Chris Masond1310b22008-01-24 16:13:08 -05001815 total_bytes += min(search_end, state->end) + 1 -
1816 max(cur_start, state->start);
1817 if (total_bytes >= max_bytes)
1818 break;
1819 if (!found) {
Josef Bacikaf60bed2011-05-04 11:11:17 -04001820 *start = max(cur_start, state->start);
Chris Masond1310b22008-01-24 16:13:08 -05001821 found = 1;
1822 }
Chris Masonec29ed52011-02-23 16:23:20 -05001823 last = state->end;
1824 } else if (contig && found) {
1825 break;
Chris Masond1310b22008-01-24 16:13:08 -05001826 }
1827 node = rb_next(node);
1828 if (!node)
1829 break;
1830 }
1831out:
Chris Masoncad321a2008-12-17 14:51:42 -05001832 spin_unlock(&tree->lock);
Chris Masond1310b22008-01-24 16:13:08 -05001833 return total_bytes;
1834}
Christoph Hellwigb2950862008-12-02 09:54:17 -05001835
Chris Masond352ac62008-09-29 15:18:18 -04001836/*
1837 * set the private field for a given byte offset in the tree. If there isn't
1838 * an extent_state there already, this does nothing.
1839 */
Sergei Trofimovich171170c2013-08-14 23:27:46 +03001840static int set_state_private(struct extent_io_tree *tree, u64 start, u64 private)
Chris Masond1310b22008-01-24 16:13:08 -05001841{
1842 struct rb_node *node;
1843 struct extent_state *state;
1844 int ret = 0;
1845
Chris Masoncad321a2008-12-17 14:51:42 -05001846 spin_lock(&tree->lock);
Chris Masond1310b22008-01-24 16:13:08 -05001847 /*
1848 * this search will find all the extents that end after
1849 * our range starts.
1850 */
Chris Mason80ea96b2008-02-01 14:51:59 -05001851 node = tree_search(tree, start);
Peter2b114d12008-04-01 11:21:40 -04001852 if (!node) {
Chris Masond1310b22008-01-24 16:13:08 -05001853 ret = -ENOENT;
1854 goto out;
1855 }
1856 state = rb_entry(node, struct extent_state, rb_node);
1857 if (state->start != start) {
1858 ret = -ENOENT;
1859 goto out;
1860 }
1861 state->private = private;
1862out:
Chris Masoncad321a2008-12-17 14:51:42 -05001863 spin_unlock(&tree->lock);
Chris Masond1310b22008-01-24 16:13:08 -05001864 return ret;
1865}
1866
1867int get_state_private(struct extent_io_tree *tree, u64 start, u64 *private)
1868{
1869 struct rb_node *node;
1870 struct extent_state *state;
1871 int ret = 0;
1872
Chris Masoncad321a2008-12-17 14:51:42 -05001873 spin_lock(&tree->lock);
Chris Masond1310b22008-01-24 16:13:08 -05001874 /*
1875 * this search will find all the extents that end after
1876 * our range starts.
1877 */
Chris Mason80ea96b2008-02-01 14:51:59 -05001878 node = tree_search(tree, start);
Peter2b114d12008-04-01 11:21:40 -04001879 if (!node) {
Chris Masond1310b22008-01-24 16:13:08 -05001880 ret = -ENOENT;
1881 goto out;
1882 }
1883 state = rb_entry(node, struct extent_state, rb_node);
1884 if (state->start != start) {
1885 ret = -ENOENT;
1886 goto out;
1887 }
1888 *private = state->private;
1889out:
Chris Masoncad321a2008-12-17 14:51:42 -05001890 spin_unlock(&tree->lock);
Chris Masond1310b22008-01-24 16:13:08 -05001891 return ret;
1892}
1893
1894/*
1895 * searches a range in the state tree for a given mask.
Chris Mason70dec802008-01-29 09:59:12 -05001896 * If 'filled' == 1, this returns 1 only if every extent in the tree
Chris Masond1310b22008-01-24 16:13:08 -05001897 * has the bits set. Otherwise, 1 is returned if any bit in the
1898 * range is found set.
1899 */
1900int test_range_bit(struct extent_io_tree *tree, u64 start, u64 end,
David Sterba41074882013-04-29 13:38:46 +00001901 unsigned long bits, int filled, struct extent_state *cached)
Chris Masond1310b22008-01-24 16:13:08 -05001902{
1903 struct extent_state *state = NULL;
1904 struct rb_node *node;
1905 int bitset = 0;
Chris Masond1310b22008-01-24 16:13:08 -05001906
Chris Masoncad321a2008-12-17 14:51:42 -05001907 spin_lock(&tree->lock);
Josef Bacikdf98b6e2011-06-20 14:53:48 -04001908 if (cached && cached->tree && cached->start <= start &&
1909 cached->end > start)
Chris Mason9655d292009-09-02 15:22:30 -04001910 node = &cached->rb_node;
1911 else
1912 node = tree_search(tree, start);
Chris Masond1310b22008-01-24 16:13:08 -05001913 while (node && start <= end) {
1914 state = rb_entry(node, struct extent_state, rb_node);
1915
1916 if (filled && state->start > start) {
1917 bitset = 0;
1918 break;
1919 }
1920
1921 if (state->start > end)
1922 break;
1923
1924 if (state->state & bits) {
1925 bitset = 1;
1926 if (!filled)
1927 break;
1928 } else if (filled) {
1929 bitset = 0;
1930 break;
1931 }
Chris Mason46562cec2009-09-23 20:23:16 -04001932
1933 if (state->end == (u64)-1)
1934 break;
1935
Chris Masond1310b22008-01-24 16:13:08 -05001936 start = state->end + 1;
1937 if (start > end)
1938 break;
1939 node = rb_next(node);
1940 if (!node) {
1941 if (filled)
1942 bitset = 0;
1943 break;
1944 }
1945 }
Chris Masoncad321a2008-12-17 14:51:42 -05001946 spin_unlock(&tree->lock);
Chris Masond1310b22008-01-24 16:13:08 -05001947 return bitset;
1948}
Chris Masond1310b22008-01-24 16:13:08 -05001949
1950/*
1951 * helper function to set a given page up to date if all the
1952 * extents in the tree for that page are up to date
1953 */
Jeff Mahoney143bede2012-03-01 14:56:26 +01001954static void check_page_uptodate(struct extent_io_tree *tree, struct page *page)
Chris Masond1310b22008-01-24 16:13:08 -05001955{
Miao Xie4eee4fa2012-12-21 09:17:45 +00001956 u64 start = page_offset(page);
Chris Masond1310b22008-01-24 16:13:08 -05001957 u64 end = start + PAGE_CACHE_SIZE - 1;
Chris Mason9655d292009-09-02 15:22:30 -04001958 if (test_range_bit(tree, start, end, EXTENT_UPTODATE, 1, NULL))
Chris Masond1310b22008-01-24 16:13:08 -05001959 SetPageUptodate(page);
Chris Masond1310b22008-01-24 16:13:08 -05001960}
1961
1962/*
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02001963 * When IO fails, either with EIO or csum verification fails, we
1964 * try other mirrors that might have a good copy of the data. This
1965 * io_failure_record is used to record state as we go through all the
1966 * mirrors. If another mirror has good data, the page is set up to date
1967 * and things continue. If a good mirror can't be found, the original
1968 * bio end_io callback is called to indicate things have failed.
1969 */
1970struct io_failure_record {
1971 struct page *page;
1972 u64 start;
1973 u64 len;
1974 u64 logical;
1975 unsigned long bio_flags;
1976 int this_mirror;
1977 int failed_mirror;
1978 int in_validation;
1979};
1980
1981static int free_io_failure(struct inode *inode, struct io_failure_record *rec,
1982 int did_repair)
1983{
1984 int ret;
1985 int err = 0;
1986 struct extent_io_tree *failure_tree = &BTRFS_I(inode)->io_failure_tree;
1987
1988 set_state_private(failure_tree, rec->start, 0);
1989 ret = clear_extent_bits(failure_tree, rec->start,
1990 rec->start + rec->len - 1,
1991 EXTENT_LOCKED | EXTENT_DIRTY, GFP_NOFS);
1992 if (ret)
1993 err = ret;
1994
David Woodhouse53b381b2013-01-29 18:40:14 -05001995 ret = clear_extent_bits(&BTRFS_I(inode)->io_tree, rec->start,
1996 rec->start + rec->len - 1,
1997 EXTENT_DAMAGED, GFP_NOFS);
1998 if (ret && !err)
1999 err = ret;
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002000
2001 kfree(rec);
2002 return err;
2003}
2004
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002005/*
2006 * this bypasses the standard btrfs submit functions deliberately, as
2007 * the standard behavior is to write all copies in a raid setup. here we only
2008 * want to write the one bad copy. so we do the mapping for ourselves and issue
2009 * submit_bio directly.
Stefan Behrens3ec706c2012-11-05 15:46:42 +01002010 * to avoid any synchronization issues, wait for the data after writing, which
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002011 * actually prevents the read that triggered the error from finishing.
2012 * currently, there can be no more than two copies of every data bit. thus,
2013 * exactly one rewrite is required.
2014 */
Stefan Behrens3ec706c2012-11-05 15:46:42 +01002015int repair_io_failure(struct btrfs_fs_info *fs_info, u64 start,
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002016 u64 length, u64 logical, struct page *page,
2017 int mirror_num)
2018{
2019 struct bio *bio;
2020 struct btrfs_device *dev;
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002021 u64 map_length = 0;
2022 u64 sector;
2023 struct btrfs_bio *bbio = NULL;
David Woodhouse53b381b2013-01-29 18:40:14 -05002024 struct btrfs_mapping_tree *map_tree = &fs_info->mapping_tree;
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002025 int ret;
2026
Ilya Dryomov908960c2013-11-03 19:06:39 +02002027 ASSERT(!(fs_info->sb->s_flags & MS_RDONLY));
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002028 BUG_ON(!mirror_num);
2029
David Woodhouse53b381b2013-01-29 18:40:14 -05002030 /* we can't repair anything in raid56 yet */
2031 if (btrfs_is_parity_mirror(map_tree, logical, length, mirror_num))
2032 return 0;
2033
Chris Mason9be33952013-05-17 18:30:14 -04002034 bio = btrfs_io_bio_alloc(GFP_NOFS, 1);
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002035 if (!bio)
2036 return -EIO;
Kent Overstreet4f024f32013-10-11 15:44:27 -07002037 bio->bi_iter.bi_size = 0;
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002038 map_length = length;
2039
Stefan Behrens3ec706c2012-11-05 15:46:42 +01002040 ret = btrfs_map_block(fs_info, WRITE, logical,
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002041 &map_length, &bbio, mirror_num);
2042 if (ret) {
2043 bio_put(bio);
2044 return -EIO;
2045 }
2046 BUG_ON(mirror_num != bbio->mirror_num);
2047 sector = bbio->stripes[mirror_num-1].physical >> 9;
Kent Overstreet4f024f32013-10-11 15:44:27 -07002048 bio->bi_iter.bi_sector = sector;
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002049 dev = bbio->stripes[mirror_num-1].dev;
2050 kfree(bbio);
2051 if (!dev || !dev->bdev || !dev->writeable) {
2052 bio_put(bio);
2053 return -EIO;
2054 }
2055 bio->bi_bdev = dev->bdev;
Miao Xie4eee4fa2012-12-21 09:17:45 +00002056 bio_add_page(bio, page, length, start - page_offset(page));
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002057
Kent Overstreet33879d42013-11-23 22:33:32 -08002058 if (btrfsic_submit_bio_wait(WRITE_SYNC, bio)) {
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002059 /* try to remap that extent elsewhere? */
2060 bio_put(bio);
Stefan Behrens442a4f62012-05-25 16:06:08 +02002061 btrfs_dev_stat_inc_and_print(dev, BTRFS_DEV_STAT_WRITE_ERRS);
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002062 return -EIO;
2063 }
2064
Frank Holtonefe120a2013-12-20 11:37:06 -05002065 printk_ratelimited_in_rcu(KERN_INFO
2066 "BTRFS: read error corrected: ino %lu off %llu "
2067 "(dev %s sector %llu)\n", page->mapping->host->i_ino,
2068 start, rcu_str_deref(dev->name), sector);
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002069
2070 bio_put(bio);
2071 return 0;
2072}
2073
Josef Bacikea466792012-03-26 21:57:36 -04002074int repair_eb_io_failure(struct btrfs_root *root, struct extent_buffer *eb,
2075 int mirror_num)
2076{
Josef Bacikea466792012-03-26 21:57:36 -04002077 u64 start = eb->start;
2078 unsigned long i, num_pages = num_extent_pages(eb->start, eb->len);
Chris Masond95603b2012-04-12 15:55:15 -04002079 int ret = 0;
Josef Bacikea466792012-03-26 21:57:36 -04002080
Ilya Dryomov908960c2013-11-03 19:06:39 +02002081 if (root->fs_info->sb->s_flags & MS_RDONLY)
2082 return -EROFS;
2083
Josef Bacikea466792012-03-26 21:57:36 -04002084 for (i = 0; i < num_pages; i++) {
2085 struct page *p = extent_buffer_page(eb, i);
Stefan Behrens3ec706c2012-11-05 15:46:42 +01002086 ret = repair_io_failure(root->fs_info, start, PAGE_CACHE_SIZE,
Josef Bacikea466792012-03-26 21:57:36 -04002087 start, p, mirror_num);
2088 if (ret)
2089 break;
2090 start += PAGE_CACHE_SIZE;
2091 }
2092
2093 return ret;
2094}
2095
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002096/*
2097 * each time an IO finishes, we do a fast check in the IO failure tree
2098 * to see if we need to process or clean up an io_failure_record
2099 */
2100static int clean_io_failure(u64 start, struct page *page)
2101{
2102 u64 private;
2103 u64 private_failure;
2104 struct io_failure_record *failrec;
Ilya Dryomov908960c2013-11-03 19:06:39 +02002105 struct inode *inode = page->mapping->host;
2106 struct btrfs_fs_info *fs_info = BTRFS_I(inode)->root->fs_info;
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002107 struct extent_state *state;
2108 int num_copies;
2109 int did_repair = 0;
2110 int ret;
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002111
2112 private = 0;
2113 ret = count_range_bits(&BTRFS_I(inode)->io_failure_tree, &private,
2114 (u64)-1, 1, EXTENT_DIRTY, 0);
2115 if (!ret)
2116 return 0;
2117
2118 ret = get_state_private(&BTRFS_I(inode)->io_failure_tree, start,
2119 &private_failure);
2120 if (ret)
2121 return 0;
2122
2123 failrec = (struct io_failure_record *)(unsigned long) private_failure;
2124 BUG_ON(!failrec->this_mirror);
2125
2126 if (failrec->in_validation) {
2127 /* there was no real error, just free the record */
2128 pr_debug("clean_io_failure: freeing dummy error at %llu\n",
2129 failrec->start);
2130 did_repair = 1;
2131 goto out;
2132 }
Ilya Dryomov908960c2013-11-03 19:06:39 +02002133 if (fs_info->sb->s_flags & MS_RDONLY)
2134 goto out;
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002135
2136 spin_lock(&BTRFS_I(inode)->io_tree.lock);
2137 state = find_first_extent_bit_state(&BTRFS_I(inode)->io_tree,
2138 failrec->start,
2139 EXTENT_LOCKED);
2140 spin_unlock(&BTRFS_I(inode)->io_tree.lock);
2141
Miao Xie883d0de2013-07-25 19:22:35 +08002142 if (state && state->start <= failrec->start &&
2143 state->end >= failrec->start + failrec->len - 1) {
Stefan Behrens3ec706c2012-11-05 15:46:42 +01002144 num_copies = btrfs_num_copies(fs_info, failrec->logical,
2145 failrec->len);
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002146 if (num_copies > 1) {
Stefan Behrens3ec706c2012-11-05 15:46:42 +01002147 ret = repair_io_failure(fs_info, start, failrec->len,
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002148 failrec->logical, page,
2149 failrec->failed_mirror);
2150 did_repair = !ret;
2151 }
David Woodhouse53b381b2013-01-29 18:40:14 -05002152 ret = 0;
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002153 }
2154
2155out:
2156 if (!ret)
2157 ret = free_io_failure(inode, failrec, did_repair);
2158
2159 return ret;
2160}
2161
2162/*
2163 * this is a generic handler for readpage errors (default
2164 * readpage_io_failed_hook). if other copies exist, read those and write back
2165 * good data to the failed position. does not investigate in remapping the
2166 * failed extent elsewhere, hoping the device will be smart enough to do this as
2167 * needed
2168 */
2169
Miao Xiefacc8a222013-07-25 19:22:34 +08002170static int bio_readpage_error(struct bio *failed_bio, u64 phy_offset,
2171 struct page *page, u64 start, u64 end,
2172 int failed_mirror)
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002173{
2174 struct io_failure_record *failrec = NULL;
2175 u64 private;
2176 struct extent_map *em;
2177 struct inode *inode = page->mapping->host;
2178 struct extent_io_tree *failure_tree = &BTRFS_I(inode)->io_failure_tree;
2179 struct extent_io_tree *tree = &BTRFS_I(inode)->io_tree;
2180 struct extent_map_tree *em_tree = &BTRFS_I(inode)->extent_tree;
2181 struct bio *bio;
Miao Xiefacc8a222013-07-25 19:22:34 +08002182 struct btrfs_io_bio *btrfs_failed_bio;
2183 struct btrfs_io_bio *btrfs_bio;
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002184 int num_copies;
2185 int ret;
2186 int read_mode;
2187 u64 logical;
2188
2189 BUG_ON(failed_bio->bi_rw & REQ_WRITE);
2190
2191 ret = get_state_private(failure_tree, start, &private);
2192 if (ret) {
2193 failrec = kzalloc(sizeof(*failrec), GFP_NOFS);
2194 if (!failrec)
2195 return -ENOMEM;
2196 failrec->start = start;
2197 failrec->len = end - start + 1;
2198 failrec->this_mirror = 0;
2199 failrec->bio_flags = 0;
2200 failrec->in_validation = 0;
2201
2202 read_lock(&em_tree->lock);
2203 em = lookup_extent_mapping(em_tree, start, failrec->len);
2204 if (!em) {
2205 read_unlock(&em_tree->lock);
2206 kfree(failrec);
2207 return -EIO;
2208 }
2209
Filipe David Borba Manana68ba9902013-11-25 03:22:07 +00002210 if (em->start > start || em->start + em->len <= start) {
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002211 free_extent_map(em);
2212 em = NULL;
2213 }
2214 read_unlock(&em_tree->lock);
2215
Tsutomu Itoh7a2d6a62012-10-01 03:07:15 -06002216 if (!em) {
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002217 kfree(failrec);
2218 return -EIO;
2219 }
2220 logical = start - em->start;
2221 logical = em->block_start + logical;
2222 if (test_bit(EXTENT_FLAG_COMPRESSED, &em->flags)) {
2223 logical = em->block_start;
2224 failrec->bio_flags = EXTENT_BIO_COMPRESSED;
2225 extent_set_compress_type(&failrec->bio_flags,
2226 em->compress_type);
2227 }
2228 pr_debug("bio_readpage_error: (new) logical=%llu, start=%llu, "
2229 "len=%llu\n", logical, start, failrec->len);
2230 failrec->logical = logical;
2231 free_extent_map(em);
2232
2233 /* set the bits in the private failure tree */
2234 ret = set_extent_bits(failure_tree, start, end,
2235 EXTENT_LOCKED | EXTENT_DIRTY, GFP_NOFS);
2236 if (ret >= 0)
2237 ret = set_state_private(failure_tree, start,
2238 (u64)(unsigned long)failrec);
2239 /* set the bits in the inode's tree */
2240 if (ret >= 0)
2241 ret = set_extent_bits(tree, start, end, EXTENT_DAMAGED,
2242 GFP_NOFS);
2243 if (ret < 0) {
2244 kfree(failrec);
2245 return ret;
2246 }
2247 } else {
2248 failrec = (struct io_failure_record *)(unsigned long)private;
2249 pr_debug("bio_readpage_error: (found) logical=%llu, "
2250 "start=%llu, len=%llu, validation=%d\n",
2251 failrec->logical, failrec->start, failrec->len,
2252 failrec->in_validation);
2253 /*
2254 * when data can be on disk more than twice, add to failrec here
2255 * (e.g. with a list for failed_mirror) to make
2256 * clean_io_failure() clean all those errors at once.
2257 */
2258 }
Stefan Behrens5d964052012-11-05 14:59:07 +01002259 num_copies = btrfs_num_copies(BTRFS_I(inode)->root->fs_info,
2260 failrec->logical, failrec->len);
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002261 if (num_copies == 1) {
2262 /*
2263 * we only have a single copy of the data, so don't bother with
2264 * all the retry and error correction code that follows. no
2265 * matter what the error is, it is very likely to persist.
2266 */
Miao Xie09a7f7a2013-07-25 19:22:32 +08002267 pr_debug("bio_readpage_error: cannot repair, num_copies=%d, next_mirror %d, failed_mirror %d\n",
2268 num_copies, failrec->this_mirror, failed_mirror);
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002269 free_io_failure(inode, failrec, 0);
2270 return -EIO;
2271 }
2272
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002273 /*
2274 * there are two premises:
2275 * a) deliver good data to the caller
2276 * b) correct the bad sectors on disk
2277 */
2278 if (failed_bio->bi_vcnt > 1) {
2279 /*
2280 * to fulfill b), we need to know the exact failing sectors, as
2281 * we don't want to rewrite any more than the failed ones. thus,
2282 * we need separate read requests for the failed bio
2283 *
2284 * if the following BUG_ON triggers, our validation request got
2285 * merged. we need separate requests for our algorithm to work.
2286 */
2287 BUG_ON(failrec->in_validation);
2288 failrec->in_validation = 1;
2289 failrec->this_mirror = failed_mirror;
2290 read_mode = READ_SYNC | REQ_FAILFAST_DEV;
2291 } else {
2292 /*
2293 * we're ready to fulfill a) and b) alongside. get a good copy
2294 * of the failed sector and if we succeed, we have setup
2295 * everything for repair_io_failure to do the rest for us.
2296 */
2297 if (failrec->in_validation) {
2298 BUG_ON(failrec->this_mirror != failed_mirror);
2299 failrec->in_validation = 0;
2300 failrec->this_mirror = 0;
2301 }
2302 failrec->failed_mirror = failed_mirror;
2303 failrec->this_mirror++;
2304 if (failrec->this_mirror == failed_mirror)
2305 failrec->this_mirror++;
2306 read_mode = READ_SYNC;
2307 }
2308
Miao Xiefacc8a222013-07-25 19:22:34 +08002309 if (failrec->this_mirror > num_copies) {
2310 pr_debug("bio_readpage_error: (fail) num_copies=%d, next_mirror %d, failed_mirror %d\n",
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002311 num_copies, failrec->this_mirror, failed_mirror);
2312 free_io_failure(inode, failrec, 0);
2313 return -EIO;
2314 }
2315
Chris Mason9be33952013-05-17 18:30:14 -04002316 bio = btrfs_io_bio_alloc(GFP_NOFS, 1);
Tsutomu Itohe627ee72012-04-12 16:03:56 -04002317 if (!bio) {
2318 free_io_failure(inode, failrec, 0);
2319 return -EIO;
2320 }
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002321 bio->bi_end_io = failed_bio->bi_end_io;
Kent Overstreet4f024f32013-10-11 15:44:27 -07002322 bio->bi_iter.bi_sector = failrec->logical >> 9;
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002323 bio->bi_bdev = BTRFS_I(inode)->root->fs_info->fs_devices->latest_bdev;
Kent Overstreet4f024f32013-10-11 15:44:27 -07002324 bio->bi_iter.bi_size = 0;
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002325
Miao Xiefacc8a222013-07-25 19:22:34 +08002326 btrfs_failed_bio = btrfs_io_bio(failed_bio);
2327 if (btrfs_failed_bio->csum) {
2328 struct btrfs_fs_info *fs_info = BTRFS_I(inode)->root->fs_info;
2329 u16 csum_size = btrfs_super_csum_size(fs_info->super_copy);
2330
2331 btrfs_bio = btrfs_io_bio(bio);
2332 btrfs_bio->csum = btrfs_bio->csum_inline;
2333 phy_offset >>= inode->i_sb->s_blocksize_bits;
2334 phy_offset *= csum_size;
2335 memcpy(btrfs_bio->csum, btrfs_failed_bio->csum + phy_offset,
2336 csum_size);
2337 }
2338
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002339 bio_add_page(bio, page, failrec->len, start - page_offset(page));
2340
2341 pr_debug("bio_readpage_error: submitting new read[%#x] to "
2342 "this_mirror=%d, num_copies=%d, in_validation=%d\n", read_mode,
2343 failrec->this_mirror, num_copies, failrec->in_validation);
2344
Tsutomu Itoh013bd4c2012-02-16 10:11:40 +09002345 ret = tree->ops->submit_bio_hook(inode, read_mode, bio,
2346 failrec->this_mirror,
2347 failrec->bio_flags, 0);
2348 return ret;
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002349}
2350
Chris Masond1310b22008-01-24 16:13:08 -05002351/* lots and lots of room for performance fixes in the end_bio funcs */
2352
Jeff Mahoney87826df2012-02-15 16:23:57 +01002353int end_extent_writepage(struct page *page, int err, u64 start, u64 end)
2354{
2355 int uptodate = (err == 0);
2356 struct extent_io_tree *tree;
Eric Sandeen3e2426b2014-06-12 00:39:58 -05002357 int ret = 0;
Jeff Mahoney87826df2012-02-15 16:23:57 +01002358
2359 tree = &BTRFS_I(page->mapping->host)->io_tree;
2360
2361 if (tree->ops && tree->ops->writepage_end_io_hook) {
2362 ret = tree->ops->writepage_end_io_hook(page, start,
2363 end, NULL, uptodate);
2364 if (ret)
2365 uptodate = 0;
2366 }
2367
Jeff Mahoney87826df2012-02-15 16:23:57 +01002368 if (!uptodate) {
Jeff Mahoney87826df2012-02-15 16:23:57 +01002369 ClearPageUptodate(page);
2370 SetPageError(page);
Liu Bo5dca6ee2014-05-12 12:47:36 +08002371 ret = ret < 0 ? ret : -EIO;
2372 mapping_set_error(page->mapping, ret);
Jeff Mahoney87826df2012-02-15 16:23:57 +01002373 }
2374 return 0;
2375}
2376
Chris Masond1310b22008-01-24 16:13:08 -05002377/*
2378 * after a writepage IO is done, we need to:
2379 * clear the uptodate bits on error
2380 * clear the writeback bits in the extent tree for this IO
2381 * end_page_writeback if the page has no more pending IO
2382 *
2383 * Scheduling is not allowed, so the extent state tree is expected
2384 * to have one and only one object corresponding to this IO.
2385 */
Chris Masond1310b22008-01-24 16:13:08 -05002386static void end_bio_extent_writepage(struct bio *bio, int err)
Chris Masond1310b22008-01-24 16:13:08 -05002387{
Kent Overstreet2c30c712013-11-07 12:20:26 -08002388 struct bio_vec *bvec;
Chris Masond1310b22008-01-24 16:13:08 -05002389 u64 start;
2390 u64 end;
Kent Overstreet2c30c712013-11-07 12:20:26 -08002391 int i;
Chris Masond1310b22008-01-24 16:13:08 -05002392
Kent Overstreet2c30c712013-11-07 12:20:26 -08002393 bio_for_each_segment_all(bvec, bio, i) {
Chris Masond1310b22008-01-24 16:13:08 -05002394 struct page *page = bvec->bv_page;
David Woodhouse902b22f2008-08-20 08:51:49 -04002395
Alexandre Oliva17a5adc2013-05-15 11:38:55 -04002396 /* We always issue full-page reads, but if some block
2397 * in a page fails to read, blk_update_request() will
2398 * advance bv_offset and adjust bv_len to compensate.
2399 * Print a warning for nonzero offsets, and an error
2400 * if they don't add up to a full page. */
Frank Holtonefe120a2013-12-20 11:37:06 -05002401 if (bvec->bv_offset || bvec->bv_len != PAGE_CACHE_SIZE) {
2402 if (bvec->bv_offset + bvec->bv_len != PAGE_CACHE_SIZE)
2403 btrfs_err(BTRFS_I(page->mapping->host)->root->fs_info,
2404 "partial page write in btrfs with offset %u and length %u",
2405 bvec->bv_offset, bvec->bv_len);
2406 else
2407 btrfs_info(BTRFS_I(page->mapping->host)->root->fs_info,
2408 "incomplete page write in btrfs with offset %u and "
2409 "length %u",
2410 bvec->bv_offset, bvec->bv_len);
2411 }
Chris Masond1310b22008-01-24 16:13:08 -05002412
Alexandre Oliva17a5adc2013-05-15 11:38:55 -04002413 start = page_offset(page);
2414 end = start + bvec->bv_offset + bvec->bv_len - 1;
Chris Masond1310b22008-01-24 16:13:08 -05002415
Jeff Mahoney87826df2012-02-15 16:23:57 +01002416 if (end_extent_writepage(page, err, start, end))
2417 continue;
Chris Mason70dec802008-01-29 09:59:12 -05002418
Alexandre Oliva17a5adc2013-05-15 11:38:55 -04002419 end_page_writeback(page);
Kent Overstreet2c30c712013-11-07 12:20:26 -08002420 }
Chris Mason2b1f55b2008-09-24 11:48:04 -04002421
Chris Masond1310b22008-01-24 16:13:08 -05002422 bio_put(bio);
Chris Masond1310b22008-01-24 16:13:08 -05002423}
2424
Miao Xie883d0de2013-07-25 19:22:35 +08002425static void
2426endio_readpage_release_extent(struct extent_io_tree *tree, u64 start, u64 len,
2427 int uptodate)
2428{
2429 struct extent_state *cached = NULL;
2430 u64 end = start + len - 1;
2431
2432 if (uptodate && tree->track_uptodate)
2433 set_extent_uptodate(tree, start, end, &cached, GFP_ATOMIC);
2434 unlock_extent_cached(tree, start, end, &cached, GFP_ATOMIC);
2435}
2436
Chris Masond1310b22008-01-24 16:13:08 -05002437/*
2438 * after a readpage IO is done, we need to:
2439 * clear the uptodate bits on error
2440 * set the uptodate bits if things worked
2441 * set the page up to date if all extents in the tree are uptodate
2442 * clear the lock bit in the extent tree
2443 * unlock the page if there are no other extents locked for it
2444 *
2445 * Scheduling is not allowed, so the extent state tree is expected
2446 * to have one and only one object corresponding to this IO.
2447 */
Chris Masond1310b22008-01-24 16:13:08 -05002448static void end_bio_extent_readpage(struct bio *bio, int err)
Chris Masond1310b22008-01-24 16:13:08 -05002449{
Kent Overstreet2c30c712013-11-07 12:20:26 -08002450 struct bio_vec *bvec;
Chris Masond1310b22008-01-24 16:13:08 -05002451 int uptodate = test_bit(BIO_UPTODATE, &bio->bi_flags);
Miao Xiefacc8a222013-07-25 19:22:34 +08002452 struct btrfs_io_bio *io_bio = btrfs_io_bio(bio);
David Woodhouse902b22f2008-08-20 08:51:49 -04002453 struct extent_io_tree *tree;
Miao Xiefacc8a222013-07-25 19:22:34 +08002454 u64 offset = 0;
Chris Masond1310b22008-01-24 16:13:08 -05002455 u64 start;
2456 u64 end;
Miao Xiefacc8a222013-07-25 19:22:34 +08002457 u64 len;
Miao Xie883d0de2013-07-25 19:22:35 +08002458 u64 extent_start = 0;
2459 u64 extent_len = 0;
Josef Bacik5cf1ab52012-04-16 09:42:26 -04002460 int mirror;
Chris Masond1310b22008-01-24 16:13:08 -05002461 int ret;
Kent Overstreet2c30c712013-11-07 12:20:26 -08002462 int i;
Chris Masond1310b22008-01-24 16:13:08 -05002463
Chris Masond20f7042008-12-08 16:58:54 -05002464 if (err)
2465 uptodate = 0;
2466
Kent Overstreet2c30c712013-11-07 12:20:26 -08002467 bio_for_each_segment_all(bvec, bio, i) {
Chris Masond1310b22008-01-24 16:13:08 -05002468 struct page *page = bvec->bv_page;
Josef Bacika71754f2013-06-17 17:14:39 -04002469 struct inode *inode = page->mapping->host;
Arne Jansen507903b2011-04-06 10:02:20 +00002470
Kent Overstreetbe3940c2012-09-11 14:23:05 -06002471 pr_debug("end_bio_extent_readpage: bi_sector=%llu, err=%d, "
Kent Overstreet4f024f32013-10-11 15:44:27 -07002472 "mirror=%lu\n", (u64)bio->bi_iter.bi_sector, err,
Chris Mason9be33952013-05-17 18:30:14 -04002473 io_bio->mirror_num);
Josef Bacika71754f2013-06-17 17:14:39 -04002474 tree = &BTRFS_I(inode)->io_tree;
David Woodhouse902b22f2008-08-20 08:51:49 -04002475
Alexandre Oliva17a5adc2013-05-15 11:38:55 -04002476 /* We always issue full-page reads, but if some block
2477 * in a page fails to read, blk_update_request() will
2478 * advance bv_offset and adjust bv_len to compensate.
2479 * Print a warning for nonzero offsets, and an error
2480 * if they don't add up to a full page. */
Frank Holtonefe120a2013-12-20 11:37:06 -05002481 if (bvec->bv_offset || bvec->bv_len != PAGE_CACHE_SIZE) {
2482 if (bvec->bv_offset + bvec->bv_len != PAGE_CACHE_SIZE)
2483 btrfs_err(BTRFS_I(page->mapping->host)->root->fs_info,
2484 "partial page read in btrfs with offset %u and length %u",
2485 bvec->bv_offset, bvec->bv_len);
2486 else
2487 btrfs_info(BTRFS_I(page->mapping->host)->root->fs_info,
2488 "incomplete page read in btrfs with offset %u and "
2489 "length %u",
2490 bvec->bv_offset, bvec->bv_len);
2491 }
Chris Masond1310b22008-01-24 16:13:08 -05002492
Alexandre Oliva17a5adc2013-05-15 11:38:55 -04002493 start = page_offset(page);
2494 end = start + bvec->bv_offset + bvec->bv_len - 1;
Miao Xiefacc8a222013-07-25 19:22:34 +08002495 len = bvec->bv_len;
Chris Masond1310b22008-01-24 16:13:08 -05002496
Chris Mason9be33952013-05-17 18:30:14 -04002497 mirror = io_bio->mirror_num;
Miao Xief2a09da2013-07-25 19:22:33 +08002498 if (likely(uptodate && tree->ops &&
2499 tree->ops->readpage_end_io_hook)) {
Miao Xiefacc8a222013-07-25 19:22:34 +08002500 ret = tree->ops->readpage_end_io_hook(io_bio, offset,
2501 page, start, end,
2502 mirror);
Stefan Behrens5ee08442012-08-27 08:30:03 -06002503 if (ret)
Chris Masond1310b22008-01-24 16:13:08 -05002504 uptodate = 0;
Stefan Behrens5ee08442012-08-27 08:30:03 -06002505 else
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002506 clean_io_failure(start, page);
Chris Masond1310b22008-01-24 16:13:08 -05002507 }
Josef Bacikea466792012-03-26 21:57:36 -04002508
Miao Xief2a09da2013-07-25 19:22:33 +08002509 if (likely(uptodate))
2510 goto readpage_ok;
2511
2512 if (tree->ops && tree->ops->readpage_io_failed_hook) {
Josef Bacik5cf1ab52012-04-16 09:42:26 -04002513 ret = tree->ops->readpage_io_failed_hook(page, mirror);
Josef Bacikea466792012-03-26 21:57:36 -04002514 if (!ret && !err &&
2515 test_bit(BIO_UPTODATE, &bio->bi_flags))
2516 uptodate = 1;
Miao Xief2a09da2013-07-25 19:22:33 +08002517 } else {
Jan Schmidtf4a8e652011-12-01 09:30:36 -05002518 /*
2519 * The generic bio_readpage_error handles errors the
2520 * following way: If possible, new read requests are
2521 * created and submitted and will end up in
2522 * end_bio_extent_readpage as well (if we're lucky, not
2523 * in the !uptodate case). In that case it returns 0 and
2524 * we just go on with the next page in our bio. If it
2525 * can't handle the error it will return -EIO and we
2526 * remain responsible for that page.
2527 */
Miao Xiefacc8a222013-07-25 19:22:34 +08002528 ret = bio_readpage_error(bio, offset, page, start, end,
2529 mirror);
Chris Mason7e383262008-04-09 16:28:12 -04002530 if (ret == 0) {
Chris Mason3b951512008-04-17 11:29:12 -04002531 uptodate =
2532 test_bit(BIO_UPTODATE, &bio->bi_flags);
Chris Masond20f7042008-12-08 16:58:54 -05002533 if (err)
2534 uptodate = 0;
Chris Mason7e383262008-04-09 16:28:12 -04002535 continue;
2536 }
2537 }
Miao Xief2a09da2013-07-25 19:22:33 +08002538readpage_ok:
Miao Xie883d0de2013-07-25 19:22:35 +08002539 if (likely(uptodate)) {
Josef Bacika71754f2013-06-17 17:14:39 -04002540 loff_t i_size = i_size_read(inode);
2541 pgoff_t end_index = i_size >> PAGE_CACHE_SHIFT;
2542 unsigned offset;
2543
2544 /* Zero out the end if this page straddles i_size */
2545 offset = i_size & (PAGE_CACHE_SIZE-1);
2546 if (page->index == end_index && offset)
2547 zero_user_segment(page, offset, PAGE_CACHE_SIZE);
Alexandre Oliva17a5adc2013-05-15 11:38:55 -04002548 SetPageUptodate(page);
Chris Mason70dec802008-01-29 09:59:12 -05002549 } else {
Alexandre Oliva17a5adc2013-05-15 11:38:55 -04002550 ClearPageUptodate(page);
2551 SetPageError(page);
Chris Mason70dec802008-01-29 09:59:12 -05002552 }
Alexandre Oliva17a5adc2013-05-15 11:38:55 -04002553 unlock_page(page);
Miao Xiefacc8a222013-07-25 19:22:34 +08002554 offset += len;
Miao Xie883d0de2013-07-25 19:22:35 +08002555
2556 if (unlikely(!uptodate)) {
2557 if (extent_len) {
2558 endio_readpage_release_extent(tree,
2559 extent_start,
2560 extent_len, 1);
2561 extent_start = 0;
2562 extent_len = 0;
2563 }
2564 endio_readpage_release_extent(tree, start,
2565 end - start + 1, 0);
2566 } else if (!extent_len) {
2567 extent_start = start;
2568 extent_len = end + 1 - start;
2569 } else if (extent_start + extent_len == start) {
2570 extent_len += end + 1 - start;
2571 } else {
2572 endio_readpage_release_extent(tree, extent_start,
2573 extent_len, uptodate);
2574 extent_start = start;
2575 extent_len = end + 1 - start;
2576 }
Kent Overstreet2c30c712013-11-07 12:20:26 -08002577 }
Chris Masond1310b22008-01-24 16:13:08 -05002578
Miao Xie883d0de2013-07-25 19:22:35 +08002579 if (extent_len)
2580 endio_readpage_release_extent(tree, extent_start, extent_len,
2581 uptodate);
Miao Xiefacc8a222013-07-25 19:22:34 +08002582 if (io_bio->end_io)
2583 io_bio->end_io(io_bio, err);
Chris Masond1310b22008-01-24 16:13:08 -05002584 bio_put(bio);
Chris Masond1310b22008-01-24 16:13:08 -05002585}
2586
Chris Mason9be33952013-05-17 18:30:14 -04002587/*
2588 * this allocates from the btrfs_bioset. We're returning a bio right now
2589 * but you can call btrfs_io_bio for the appropriate container_of magic
2590 */
Miao Xie88f794e2010-11-22 03:02:55 +00002591struct bio *
2592btrfs_bio_alloc(struct block_device *bdev, u64 first_sector, int nr_vecs,
2593 gfp_t gfp_flags)
Chris Masond1310b22008-01-24 16:13:08 -05002594{
Miao Xiefacc8a222013-07-25 19:22:34 +08002595 struct btrfs_io_bio *btrfs_bio;
Chris Masond1310b22008-01-24 16:13:08 -05002596 struct bio *bio;
2597
Chris Mason9be33952013-05-17 18:30:14 -04002598 bio = bio_alloc_bioset(gfp_flags, nr_vecs, btrfs_bioset);
Chris Masond1310b22008-01-24 16:13:08 -05002599
2600 if (bio == NULL && (current->flags & PF_MEMALLOC)) {
Chris Mason9be33952013-05-17 18:30:14 -04002601 while (!bio && (nr_vecs /= 2)) {
2602 bio = bio_alloc_bioset(gfp_flags,
2603 nr_vecs, btrfs_bioset);
2604 }
Chris Masond1310b22008-01-24 16:13:08 -05002605 }
2606
2607 if (bio) {
2608 bio->bi_bdev = bdev;
Kent Overstreet4f024f32013-10-11 15:44:27 -07002609 bio->bi_iter.bi_sector = first_sector;
Miao Xiefacc8a222013-07-25 19:22:34 +08002610 btrfs_bio = btrfs_io_bio(bio);
2611 btrfs_bio->csum = NULL;
2612 btrfs_bio->csum_allocated = NULL;
2613 btrfs_bio->end_io = NULL;
Chris Masond1310b22008-01-24 16:13:08 -05002614 }
2615 return bio;
2616}
2617
Chris Mason9be33952013-05-17 18:30:14 -04002618struct bio *btrfs_bio_clone(struct bio *bio, gfp_t gfp_mask)
2619{
2620 return bio_clone_bioset(bio, gfp_mask, btrfs_bioset);
2621}
2622
2623
2624/* this also allocates from the btrfs_bioset */
2625struct bio *btrfs_io_bio_alloc(gfp_t gfp_mask, unsigned int nr_iovecs)
2626{
Miao Xiefacc8a222013-07-25 19:22:34 +08002627 struct btrfs_io_bio *btrfs_bio;
2628 struct bio *bio;
2629
2630 bio = bio_alloc_bioset(gfp_mask, nr_iovecs, btrfs_bioset);
2631 if (bio) {
2632 btrfs_bio = btrfs_io_bio(bio);
2633 btrfs_bio->csum = NULL;
2634 btrfs_bio->csum_allocated = NULL;
2635 btrfs_bio->end_io = NULL;
2636 }
2637 return bio;
Chris Mason9be33952013-05-17 18:30:14 -04002638}
2639
2640
Jeff Mahoney355808c2011-10-03 23:23:14 -04002641static int __must_check submit_one_bio(int rw, struct bio *bio,
2642 int mirror_num, unsigned long bio_flags)
Chris Masond1310b22008-01-24 16:13:08 -05002643{
Chris Masond1310b22008-01-24 16:13:08 -05002644 int ret = 0;
Chris Mason70dec802008-01-29 09:59:12 -05002645 struct bio_vec *bvec = bio->bi_io_vec + bio->bi_vcnt - 1;
2646 struct page *page = bvec->bv_page;
2647 struct extent_io_tree *tree = bio->bi_private;
Chris Mason70dec802008-01-29 09:59:12 -05002648 u64 start;
Chris Mason70dec802008-01-29 09:59:12 -05002649
Miao Xie4eee4fa2012-12-21 09:17:45 +00002650 start = page_offset(page) + bvec->bv_offset;
Chris Mason70dec802008-01-29 09:59:12 -05002651
David Woodhouse902b22f2008-08-20 08:51:49 -04002652 bio->bi_private = NULL;
Chris Masond1310b22008-01-24 16:13:08 -05002653
2654 bio_get(bio);
2655
Chris Mason065631f2008-02-20 12:07:25 -05002656 if (tree->ops && tree->ops->submit_bio_hook)
liubo6b82ce82011-01-26 06:21:39 +00002657 ret = tree->ops->submit_bio_hook(page->mapping->host, rw, bio,
Chris Masoneaf25d92010-05-25 09:48:28 -04002658 mirror_num, bio_flags, start);
Chris Mason0b86a832008-03-24 15:01:56 -04002659 else
Stefan Behrens21adbd52011-11-09 13:44:05 +01002660 btrfsic_submit_bio(rw, bio);
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002661
Chris Masond1310b22008-01-24 16:13:08 -05002662 if (bio_flagged(bio, BIO_EOPNOTSUPP))
2663 ret = -EOPNOTSUPP;
2664 bio_put(bio);
2665 return ret;
2666}
2667
David Woodhouse64a16702009-07-15 23:29:37 +01002668static int merge_bio(int rw, struct extent_io_tree *tree, struct page *page,
Jeff Mahoney3444a972011-10-03 23:23:13 -04002669 unsigned long offset, size_t size, struct bio *bio,
2670 unsigned long bio_flags)
2671{
2672 int ret = 0;
2673 if (tree->ops && tree->ops->merge_bio_hook)
David Woodhouse64a16702009-07-15 23:29:37 +01002674 ret = tree->ops->merge_bio_hook(rw, page, offset, size, bio,
Jeff Mahoney3444a972011-10-03 23:23:13 -04002675 bio_flags);
2676 BUG_ON(ret < 0);
2677 return ret;
2678
2679}
2680
Chris Masond1310b22008-01-24 16:13:08 -05002681static int submit_extent_page(int rw, struct extent_io_tree *tree,
2682 struct page *page, sector_t sector,
2683 size_t size, unsigned long offset,
2684 struct block_device *bdev,
2685 struct bio **bio_ret,
2686 unsigned long max_pages,
Chris Masonf1885912008-04-09 16:28:12 -04002687 bio_end_io_t end_io_func,
Chris Masonc8b97812008-10-29 14:49:59 -04002688 int mirror_num,
2689 unsigned long prev_bio_flags,
2690 unsigned long bio_flags)
Chris Masond1310b22008-01-24 16:13:08 -05002691{
2692 int ret = 0;
2693 struct bio *bio;
2694 int nr;
Chris Masonc8b97812008-10-29 14:49:59 -04002695 int contig = 0;
2696 int this_compressed = bio_flags & EXTENT_BIO_COMPRESSED;
2697 int old_compressed = prev_bio_flags & EXTENT_BIO_COMPRESSED;
Chris Mason5b050f02008-11-11 09:34:41 -05002698 size_t page_size = min_t(size_t, size, PAGE_CACHE_SIZE);
Chris Masond1310b22008-01-24 16:13:08 -05002699
2700 if (bio_ret && *bio_ret) {
2701 bio = *bio_ret;
Chris Masonc8b97812008-10-29 14:49:59 -04002702 if (old_compressed)
Kent Overstreet4f024f32013-10-11 15:44:27 -07002703 contig = bio->bi_iter.bi_sector == sector;
Chris Masonc8b97812008-10-29 14:49:59 -04002704 else
Kent Overstreetf73a1c72012-09-25 15:05:12 -07002705 contig = bio_end_sector(bio) == sector;
Chris Masonc8b97812008-10-29 14:49:59 -04002706
2707 if (prev_bio_flags != bio_flags || !contig ||
David Woodhouse64a16702009-07-15 23:29:37 +01002708 merge_bio(rw, tree, page, offset, page_size, bio, bio_flags) ||
Chris Masonc8b97812008-10-29 14:49:59 -04002709 bio_add_page(bio, page, page_size, offset) < page_size) {
2710 ret = submit_one_bio(rw, bio, mirror_num,
2711 prev_bio_flags);
Jeff Mahoney79787ea2012-03-12 16:03:00 +01002712 if (ret < 0)
2713 return ret;
Chris Masond1310b22008-01-24 16:13:08 -05002714 bio = NULL;
2715 } else {
2716 return 0;
2717 }
2718 }
Chris Masonc8b97812008-10-29 14:49:59 -04002719 if (this_compressed)
2720 nr = BIO_MAX_PAGES;
2721 else
2722 nr = bio_get_nr_vecs(bdev);
2723
Miao Xie88f794e2010-11-22 03:02:55 +00002724 bio = btrfs_bio_alloc(bdev, sector, nr, GFP_NOFS | __GFP_HIGH);
Tsutomu Itoh5df67082011-02-01 09:17:35 +00002725 if (!bio)
2726 return -ENOMEM;
Chris Mason70dec802008-01-29 09:59:12 -05002727
Chris Masonc8b97812008-10-29 14:49:59 -04002728 bio_add_page(bio, page, page_size, offset);
Chris Masond1310b22008-01-24 16:13:08 -05002729 bio->bi_end_io = end_io_func;
2730 bio->bi_private = tree;
Chris Mason70dec802008-01-29 09:59:12 -05002731
Chris Masond3977122009-01-05 21:25:51 -05002732 if (bio_ret)
Chris Masond1310b22008-01-24 16:13:08 -05002733 *bio_ret = bio;
Chris Masond3977122009-01-05 21:25:51 -05002734 else
Chris Masonc8b97812008-10-29 14:49:59 -04002735 ret = submit_one_bio(rw, bio, mirror_num, bio_flags);
Chris Masond1310b22008-01-24 16:13:08 -05002736
2737 return ret;
2738}
2739
Eric Sandeen48a3b632013-04-25 20:41:01 +00002740static void attach_extent_buffer_page(struct extent_buffer *eb,
2741 struct page *page)
Josef Bacik4f2de97a2012-03-07 16:20:05 -05002742{
2743 if (!PagePrivate(page)) {
2744 SetPagePrivate(page);
2745 page_cache_get(page);
2746 set_page_private(page, (unsigned long)eb);
2747 } else {
2748 WARN_ON(page->private != (unsigned long)eb);
2749 }
2750}
2751
Chris Masond1310b22008-01-24 16:13:08 -05002752void set_page_extent_mapped(struct page *page)
2753{
2754 if (!PagePrivate(page)) {
2755 SetPagePrivate(page);
Chris Masond1310b22008-01-24 16:13:08 -05002756 page_cache_get(page);
Chris Mason6af118ce2008-07-22 11:18:07 -04002757 set_page_private(page, EXTENT_PAGE_PRIVATE);
Chris Masond1310b22008-01-24 16:13:08 -05002758 }
2759}
2760
Miao Xie125bac012013-07-25 19:22:37 +08002761static struct extent_map *
2762__get_extent_map(struct inode *inode, struct page *page, size_t pg_offset,
2763 u64 start, u64 len, get_extent_t *get_extent,
2764 struct extent_map **em_cached)
2765{
2766 struct extent_map *em;
2767
2768 if (em_cached && *em_cached) {
2769 em = *em_cached;
Filipe Mananacbc0e922014-02-25 14:15:12 +00002770 if (extent_map_in_tree(em) && start >= em->start &&
Miao Xie125bac012013-07-25 19:22:37 +08002771 start < extent_map_end(em)) {
2772 atomic_inc(&em->refs);
2773 return em;
2774 }
2775
2776 free_extent_map(em);
2777 *em_cached = NULL;
2778 }
2779
2780 em = get_extent(inode, page, pg_offset, start, len, 0);
2781 if (em_cached && !IS_ERR_OR_NULL(em)) {
2782 BUG_ON(*em_cached);
2783 atomic_inc(&em->refs);
2784 *em_cached = em;
2785 }
2786 return em;
2787}
Chris Masond1310b22008-01-24 16:13:08 -05002788/*
2789 * basic readpage implementation. Locked extent state structs are inserted
2790 * into the tree that are removed when the IO is done (by the end_io
2791 * handlers)
Jeff Mahoney79787ea2012-03-12 16:03:00 +01002792 * XXX JDM: This needs looking at to ensure proper page locking
Chris Masond1310b22008-01-24 16:13:08 -05002793 */
Miao Xie99740902013-07-25 19:22:36 +08002794static int __do_readpage(struct extent_io_tree *tree,
2795 struct page *page,
2796 get_extent_t *get_extent,
Miao Xie125bac012013-07-25 19:22:37 +08002797 struct extent_map **em_cached,
Miao Xie99740902013-07-25 19:22:36 +08002798 struct bio **bio, int mirror_num,
2799 unsigned long *bio_flags, int rw)
Chris Masond1310b22008-01-24 16:13:08 -05002800{
2801 struct inode *inode = page->mapping->host;
Miao Xie4eee4fa2012-12-21 09:17:45 +00002802 u64 start = page_offset(page);
Chris Masond1310b22008-01-24 16:13:08 -05002803 u64 page_end = start + PAGE_CACHE_SIZE - 1;
2804 u64 end;
2805 u64 cur = start;
2806 u64 extent_offset;
2807 u64 last_byte = i_size_read(inode);
2808 u64 block_start;
2809 u64 cur_end;
2810 sector_t sector;
2811 struct extent_map *em;
2812 struct block_device *bdev;
2813 int ret;
2814 int nr = 0;
Mark Fasheh4b384312013-08-06 11:42:50 -07002815 int parent_locked = *bio_flags & EXTENT_BIO_PARENT_LOCKED;
David Sterba306e16c2011-04-19 14:29:38 +02002816 size_t pg_offset = 0;
Chris Masond1310b22008-01-24 16:13:08 -05002817 size_t iosize;
Chris Masonc8b97812008-10-29 14:49:59 -04002818 size_t disk_io_size;
Chris Masond1310b22008-01-24 16:13:08 -05002819 size_t blocksize = inode->i_sb->s_blocksize;
Mark Fasheh4b384312013-08-06 11:42:50 -07002820 unsigned long this_bio_flag = *bio_flags & EXTENT_BIO_PARENT_LOCKED;
Chris Masond1310b22008-01-24 16:13:08 -05002821
2822 set_page_extent_mapped(page);
2823
Miao Xie99740902013-07-25 19:22:36 +08002824 end = page_end;
Dan Magenheimer90a887c2011-05-26 10:01:56 -06002825 if (!PageUptodate(page)) {
2826 if (cleancache_get_page(page) == 0) {
2827 BUG_ON(blocksize != PAGE_SIZE);
Miao Xie99740902013-07-25 19:22:36 +08002828 unlock_extent(tree, start, end);
Dan Magenheimer90a887c2011-05-26 10:01:56 -06002829 goto out;
2830 }
2831 }
2832
Chris Masonc8b97812008-10-29 14:49:59 -04002833 if (page->index == last_byte >> PAGE_CACHE_SHIFT) {
2834 char *userpage;
2835 size_t zero_offset = last_byte & (PAGE_CACHE_SIZE - 1);
2836
2837 if (zero_offset) {
2838 iosize = PAGE_CACHE_SIZE - zero_offset;
Cong Wang7ac687d2011-11-25 23:14:28 +08002839 userpage = kmap_atomic(page);
Chris Masonc8b97812008-10-29 14:49:59 -04002840 memset(userpage + zero_offset, 0, iosize);
2841 flush_dcache_page(page);
Cong Wang7ac687d2011-11-25 23:14:28 +08002842 kunmap_atomic(userpage);
Chris Masonc8b97812008-10-29 14:49:59 -04002843 }
2844 }
Chris Masond1310b22008-01-24 16:13:08 -05002845 while (cur <= end) {
Josef Bacikc8f2f242013-02-11 11:33:00 -05002846 unsigned long pnr = (last_byte >> PAGE_CACHE_SHIFT) + 1;
2847
Chris Masond1310b22008-01-24 16:13:08 -05002848 if (cur >= last_byte) {
2849 char *userpage;
Arne Jansen507903b2011-04-06 10:02:20 +00002850 struct extent_state *cached = NULL;
2851
David Sterba306e16c2011-04-19 14:29:38 +02002852 iosize = PAGE_CACHE_SIZE - pg_offset;
Cong Wang7ac687d2011-11-25 23:14:28 +08002853 userpage = kmap_atomic(page);
David Sterba306e16c2011-04-19 14:29:38 +02002854 memset(userpage + pg_offset, 0, iosize);
Chris Masond1310b22008-01-24 16:13:08 -05002855 flush_dcache_page(page);
Cong Wang7ac687d2011-11-25 23:14:28 +08002856 kunmap_atomic(userpage);
Chris Masond1310b22008-01-24 16:13:08 -05002857 set_extent_uptodate(tree, cur, cur + iosize - 1,
Arne Jansen507903b2011-04-06 10:02:20 +00002858 &cached, GFP_NOFS);
Mark Fasheh4b384312013-08-06 11:42:50 -07002859 if (!parent_locked)
2860 unlock_extent_cached(tree, cur,
2861 cur + iosize - 1,
2862 &cached, GFP_NOFS);
Chris Masond1310b22008-01-24 16:13:08 -05002863 break;
2864 }
Miao Xie125bac012013-07-25 19:22:37 +08002865 em = __get_extent_map(inode, page, pg_offset, cur,
2866 end - cur + 1, get_extent, em_cached);
David Sterbac7040052011-04-19 18:00:01 +02002867 if (IS_ERR_OR_NULL(em)) {
Chris Masond1310b22008-01-24 16:13:08 -05002868 SetPageError(page);
Mark Fasheh4b384312013-08-06 11:42:50 -07002869 if (!parent_locked)
2870 unlock_extent(tree, cur, end);
Chris Masond1310b22008-01-24 16:13:08 -05002871 break;
2872 }
Chris Masond1310b22008-01-24 16:13:08 -05002873 extent_offset = cur - em->start;
2874 BUG_ON(extent_map_end(em) <= cur);
2875 BUG_ON(end < cur);
2876
Li Zefan261507a02010-12-17 14:21:50 +08002877 if (test_bit(EXTENT_FLAG_COMPRESSED, &em->flags)) {
Mark Fasheh4b384312013-08-06 11:42:50 -07002878 this_bio_flag |= EXTENT_BIO_COMPRESSED;
Li Zefan261507a02010-12-17 14:21:50 +08002879 extent_set_compress_type(&this_bio_flag,
2880 em->compress_type);
2881 }
Chris Masonc8b97812008-10-29 14:49:59 -04002882
Chris Masond1310b22008-01-24 16:13:08 -05002883 iosize = min(extent_map_end(em) - cur, end - cur + 1);
2884 cur_end = min(extent_map_end(em) - 1, end);
Qu Wenruofda28322013-02-26 08:10:22 +00002885 iosize = ALIGN(iosize, blocksize);
Chris Masonc8b97812008-10-29 14:49:59 -04002886 if (this_bio_flag & EXTENT_BIO_COMPRESSED) {
2887 disk_io_size = em->block_len;
2888 sector = em->block_start >> 9;
2889 } else {
2890 sector = (em->block_start + extent_offset) >> 9;
2891 disk_io_size = iosize;
2892 }
Chris Masond1310b22008-01-24 16:13:08 -05002893 bdev = em->bdev;
2894 block_start = em->block_start;
Yan Zhengd899e052008-10-30 14:25:28 -04002895 if (test_bit(EXTENT_FLAG_PREALLOC, &em->flags))
2896 block_start = EXTENT_MAP_HOLE;
Chris Masond1310b22008-01-24 16:13:08 -05002897 free_extent_map(em);
2898 em = NULL;
2899
2900 /* we've found a hole, just zero and go on */
2901 if (block_start == EXTENT_MAP_HOLE) {
2902 char *userpage;
Arne Jansen507903b2011-04-06 10:02:20 +00002903 struct extent_state *cached = NULL;
2904
Cong Wang7ac687d2011-11-25 23:14:28 +08002905 userpage = kmap_atomic(page);
David Sterba306e16c2011-04-19 14:29:38 +02002906 memset(userpage + pg_offset, 0, iosize);
Chris Masond1310b22008-01-24 16:13:08 -05002907 flush_dcache_page(page);
Cong Wang7ac687d2011-11-25 23:14:28 +08002908 kunmap_atomic(userpage);
Chris Masond1310b22008-01-24 16:13:08 -05002909
2910 set_extent_uptodate(tree, cur, cur + iosize - 1,
Arne Jansen507903b2011-04-06 10:02:20 +00002911 &cached, GFP_NOFS);
2912 unlock_extent_cached(tree, cur, cur + iosize - 1,
2913 &cached, GFP_NOFS);
Chris Masond1310b22008-01-24 16:13:08 -05002914 cur = cur + iosize;
David Sterba306e16c2011-04-19 14:29:38 +02002915 pg_offset += iosize;
Chris Masond1310b22008-01-24 16:13:08 -05002916 continue;
2917 }
2918 /* the get_extent function already copied into the page */
Chris Mason9655d292009-09-02 15:22:30 -04002919 if (test_range_bit(tree, cur, cur_end,
2920 EXTENT_UPTODATE, 1, NULL)) {
Chris Masona1b32a52008-09-05 16:09:51 -04002921 check_page_uptodate(tree, page);
Mark Fasheh4b384312013-08-06 11:42:50 -07002922 if (!parent_locked)
2923 unlock_extent(tree, cur, cur + iosize - 1);
Chris Masond1310b22008-01-24 16:13:08 -05002924 cur = cur + iosize;
David Sterba306e16c2011-04-19 14:29:38 +02002925 pg_offset += iosize;
Chris Masond1310b22008-01-24 16:13:08 -05002926 continue;
2927 }
Chris Mason70dec802008-01-29 09:59:12 -05002928 /* we have an inline extent but it didn't get marked up
2929 * to date. Error out
2930 */
2931 if (block_start == EXTENT_MAP_INLINE) {
2932 SetPageError(page);
Mark Fasheh4b384312013-08-06 11:42:50 -07002933 if (!parent_locked)
2934 unlock_extent(tree, cur, cur + iosize - 1);
Chris Mason70dec802008-01-29 09:59:12 -05002935 cur = cur + iosize;
David Sterba306e16c2011-04-19 14:29:38 +02002936 pg_offset += iosize;
Chris Mason70dec802008-01-29 09:59:12 -05002937 continue;
2938 }
Chris Masond1310b22008-01-24 16:13:08 -05002939
Josef Bacikc8f2f242013-02-11 11:33:00 -05002940 pnr -= page->index;
Josef Bacikd4c7ca82013-04-19 19:49:09 -04002941 ret = submit_extent_page(rw, tree, page,
David Sterba306e16c2011-04-19 14:29:38 +02002942 sector, disk_io_size, pg_offset,
Chris Mason89642222008-07-24 09:41:53 -04002943 bdev, bio, pnr,
Chris Masonc8b97812008-10-29 14:49:59 -04002944 end_bio_extent_readpage, mirror_num,
2945 *bio_flags,
2946 this_bio_flag);
Josef Bacikc8f2f242013-02-11 11:33:00 -05002947 if (!ret) {
2948 nr++;
2949 *bio_flags = this_bio_flag;
2950 } else {
Chris Masond1310b22008-01-24 16:13:08 -05002951 SetPageError(page);
Mark Fasheh4b384312013-08-06 11:42:50 -07002952 if (!parent_locked)
2953 unlock_extent(tree, cur, cur + iosize - 1);
Josef Bacikedd33c92012-10-05 16:40:32 -04002954 }
Chris Masond1310b22008-01-24 16:13:08 -05002955 cur = cur + iosize;
David Sterba306e16c2011-04-19 14:29:38 +02002956 pg_offset += iosize;
Chris Masond1310b22008-01-24 16:13:08 -05002957 }
Dan Magenheimer90a887c2011-05-26 10:01:56 -06002958out:
Chris Masond1310b22008-01-24 16:13:08 -05002959 if (!nr) {
2960 if (!PageError(page))
2961 SetPageUptodate(page);
2962 unlock_page(page);
2963 }
2964 return 0;
2965}
2966
Miao Xie99740902013-07-25 19:22:36 +08002967static inline void __do_contiguous_readpages(struct extent_io_tree *tree,
2968 struct page *pages[], int nr_pages,
2969 u64 start, u64 end,
2970 get_extent_t *get_extent,
Miao Xie125bac012013-07-25 19:22:37 +08002971 struct extent_map **em_cached,
Miao Xie99740902013-07-25 19:22:36 +08002972 struct bio **bio, int mirror_num,
2973 unsigned long *bio_flags, int rw)
2974{
2975 struct inode *inode;
2976 struct btrfs_ordered_extent *ordered;
2977 int index;
2978
2979 inode = pages[0]->mapping->host;
2980 while (1) {
2981 lock_extent(tree, start, end);
2982 ordered = btrfs_lookup_ordered_range(inode, start,
2983 end - start + 1);
2984 if (!ordered)
2985 break;
2986 unlock_extent(tree, start, end);
2987 btrfs_start_ordered_extent(inode, ordered, 1);
2988 btrfs_put_ordered_extent(ordered);
2989 }
2990
2991 for (index = 0; index < nr_pages; index++) {
Miao Xie125bac012013-07-25 19:22:37 +08002992 __do_readpage(tree, pages[index], get_extent, em_cached, bio,
2993 mirror_num, bio_flags, rw);
Miao Xie99740902013-07-25 19:22:36 +08002994 page_cache_release(pages[index]);
2995 }
2996}
2997
2998static void __extent_readpages(struct extent_io_tree *tree,
2999 struct page *pages[],
3000 int nr_pages, get_extent_t *get_extent,
Miao Xie125bac012013-07-25 19:22:37 +08003001 struct extent_map **em_cached,
Miao Xie99740902013-07-25 19:22:36 +08003002 struct bio **bio, int mirror_num,
3003 unsigned long *bio_flags, int rw)
3004{
Stefan Behrens35a36212013-08-14 18:12:25 +02003005 u64 start = 0;
Miao Xie99740902013-07-25 19:22:36 +08003006 u64 end = 0;
3007 u64 page_start;
3008 int index;
Stefan Behrens35a36212013-08-14 18:12:25 +02003009 int first_index = 0;
Miao Xie99740902013-07-25 19:22:36 +08003010
3011 for (index = 0; index < nr_pages; index++) {
3012 page_start = page_offset(pages[index]);
3013 if (!end) {
3014 start = page_start;
3015 end = start + PAGE_CACHE_SIZE - 1;
3016 first_index = index;
3017 } else if (end + 1 == page_start) {
3018 end += PAGE_CACHE_SIZE;
3019 } else {
3020 __do_contiguous_readpages(tree, &pages[first_index],
3021 index - first_index, start,
Miao Xie125bac012013-07-25 19:22:37 +08003022 end, get_extent, em_cached,
3023 bio, mirror_num, bio_flags,
3024 rw);
Miao Xie99740902013-07-25 19:22:36 +08003025 start = page_start;
3026 end = start + PAGE_CACHE_SIZE - 1;
3027 first_index = index;
3028 }
3029 }
3030
3031 if (end)
3032 __do_contiguous_readpages(tree, &pages[first_index],
3033 index - first_index, start,
Miao Xie125bac012013-07-25 19:22:37 +08003034 end, get_extent, em_cached, bio,
Miao Xie99740902013-07-25 19:22:36 +08003035 mirror_num, bio_flags, rw);
3036}
3037
3038static int __extent_read_full_page(struct extent_io_tree *tree,
3039 struct page *page,
3040 get_extent_t *get_extent,
3041 struct bio **bio, int mirror_num,
3042 unsigned long *bio_flags, int rw)
3043{
3044 struct inode *inode = page->mapping->host;
3045 struct btrfs_ordered_extent *ordered;
3046 u64 start = page_offset(page);
3047 u64 end = start + PAGE_CACHE_SIZE - 1;
3048 int ret;
3049
3050 while (1) {
3051 lock_extent(tree, start, end);
3052 ordered = btrfs_lookup_ordered_extent(inode, start);
3053 if (!ordered)
3054 break;
3055 unlock_extent(tree, start, end);
3056 btrfs_start_ordered_extent(inode, ordered, 1);
3057 btrfs_put_ordered_extent(ordered);
3058 }
3059
Miao Xie125bac012013-07-25 19:22:37 +08003060 ret = __do_readpage(tree, page, get_extent, NULL, bio, mirror_num,
3061 bio_flags, rw);
Miao Xie99740902013-07-25 19:22:36 +08003062 return ret;
3063}
3064
Chris Masond1310b22008-01-24 16:13:08 -05003065int extent_read_full_page(struct extent_io_tree *tree, struct page *page,
Jan Schmidt8ddc7d92011-06-13 20:02:58 +02003066 get_extent_t *get_extent, int mirror_num)
Chris Masond1310b22008-01-24 16:13:08 -05003067{
3068 struct bio *bio = NULL;
Chris Masonc8b97812008-10-29 14:49:59 -04003069 unsigned long bio_flags = 0;
Chris Masond1310b22008-01-24 16:13:08 -05003070 int ret;
3071
Jan Schmidt8ddc7d92011-06-13 20:02:58 +02003072 ret = __extent_read_full_page(tree, page, get_extent, &bio, mirror_num,
Josef Bacikd4c7ca82013-04-19 19:49:09 -04003073 &bio_flags, READ);
Chris Masond1310b22008-01-24 16:13:08 -05003074 if (bio)
Jan Schmidt8ddc7d92011-06-13 20:02:58 +02003075 ret = submit_one_bio(READ, bio, mirror_num, bio_flags);
Chris Masond1310b22008-01-24 16:13:08 -05003076 return ret;
3077}
Chris Masond1310b22008-01-24 16:13:08 -05003078
Mark Fasheh4b384312013-08-06 11:42:50 -07003079int extent_read_full_page_nolock(struct extent_io_tree *tree, struct page *page,
3080 get_extent_t *get_extent, int mirror_num)
3081{
3082 struct bio *bio = NULL;
3083 unsigned long bio_flags = EXTENT_BIO_PARENT_LOCKED;
3084 int ret;
3085
3086 ret = __do_readpage(tree, page, get_extent, NULL, &bio, mirror_num,
3087 &bio_flags, READ);
3088 if (bio)
3089 ret = submit_one_bio(READ, bio, mirror_num, bio_flags);
3090 return ret;
3091}
3092
Chris Mason11c83492009-04-20 15:50:09 -04003093static noinline void update_nr_written(struct page *page,
3094 struct writeback_control *wbc,
3095 unsigned long nr_written)
3096{
3097 wbc->nr_to_write -= nr_written;
3098 if (wbc->range_cyclic || (wbc->nr_to_write > 0 &&
3099 wbc->range_start == 0 && wbc->range_end == LLONG_MAX))
3100 page->mapping->writeback_index = page->index + nr_written;
3101}
3102
Chris Masond1310b22008-01-24 16:13:08 -05003103/*
Chris Mason40f76582014-05-21 13:35:51 -07003104 * helper for __extent_writepage, doing all of the delayed allocation setup.
3105 *
3106 * This returns 1 if our fill_delalloc function did all the work required
3107 * to write the page (copy into inline extent). In this case the IO has
3108 * been started and the page is already unlocked.
3109 *
3110 * This returns 0 if all went well (page still locked)
3111 * This returns < 0 if there were errors (page still locked)
Chris Masond1310b22008-01-24 16:13:08 -05003112 */
Chris Mason40f76582014-05-21 13:35:51 -07003113static noinline_for_stack int writepage_delalloc(struct inode *inode,
3114 struct page *page, struct writeback_control *wbc,
3115 struct extent_page_data *epd,
3116 u64 delalloc_start,
3117 unsigned long *nr_written)
Chris Masond1310b22008-01-24 16:13:08 -05003118{
Chris Mason40f76582014-05-21 13:35:51 -07003119 struct extent_io_tree *tree = epd->tree;
3120 u64 page_end = delalloc_start + PAGE_CACHE_SIZE - 1;
3121 u64 nr_delalloc;
3122 u64 delalloc_to_write = 0;
3123 u64 delalloc_end = 0;
3124 int ret;
3125 int page_started = 0;
3126
3127 if (epd->extent_locked || !tree->ops || !tree->ops->fill_delalloc)
3128 return 0;
3129
3130 while (delalloc_end < page_end) {
3131 nr_delalloc = find_lock_delalloc_range(inode, tree,
3132 page,
3133 &delalloc_start,
3134 &delalloc_end,
3135 128 * 1024 * 1024);
3136 if (nr_delalloc == 0) {
3137 delalloc_start = delalloc_end + 1;
3138 continue;
3139 }
3140 ret = tree->ops->fill_delalloc(inode, page,
3141 delalloc_start,
3142 delalloc_end,
3143 &page_started,
3144 nr_written);
3145 /* File system has been set read-only */
3146 if (ret) {
3147 SetPageError(page);
3148 /* fill_delalloc should be return < 0 for error
3149 * but just in case, we use > 0 here meaning the
3150 * IO is started, so we don't want to return > 0
3151 * unless things are going well.
3152 */
3153 ret = ret < 0 ? ret : -EIO;
3154 goto done;
3155 }
3156 /*
3157 * delalloc_end is already one less than the total
3158 * length, so we don't subtract one from
3159 * PAGE_CACHE_SIZE
3160 */
3161 delalloc_to_write += (delalloc_end - delalloc_start +
3162 PAGE_CACHE_SIZE) >>
3163 PAGE_CACHE_SHIFT;
3164 delalloc_start = delalloc_end + 1;
3165 }
3166 if (wbc->nr_to_write < delalloc_to_write) {
3167 int thresh = 8192;
3168
3169 if (delalloc_to_write < thresh * 2)
3170 thresh = delalloc_to_write;
3171 wbc->nr_to_write = min_t(u64, delalloc_to_write,
3172 thresh);
3173 }
3174
3175 /* did the fill delalloc function already unlock and start
3176 * the IO?
3177 */
3178 if (page_started) {
3179 /*
3180 * we've unlocked the page, so we can't update
3181 * the mapping's writeback index, just update
3182 * nr_to_write.
3183 */
3184 wbc->nr_to_write -= *nr_written;
3185 return 1;
3186 }
3187
3188 ret = 0;
3189
3190done:
3191 return ret;
3192}
3193
3194/*
3195 * helper for __extent_writepage. This calls the writepage start hooks,
3196 * and does the loop to map the page into extents and bios.
3197 *
3198 * We return 1 if the IO is started and the page is unlocked,
3199 * 0 if all went well (page still locked)
3200 * < 0 if there were errors (page still locked)
3201 */
3202static noinline_for_stack int __extent_writepage_io(struct inode *inode,
3203 struct page *page,
3204 struct writeback_control *wbc,
3205 struct extent_page_data *epd,
3206 loff_t i_size,
3207 unsigned long nr_written,
3208 int write_flags, int *nr_ret)
3209{
Chris Masond1310b22008-01-24 16:13:08 -05003210 struct extent_io_tree *tree = epd->tree;
Miao Xie4eee4fa2012-12-21 09:17:45 +00003211 u64 start = page_offset(page);
Chris Masond1310b22008-01-24 16:13:08 -05003212 u64 page_end = start + PAGE_CACHE_SIZE - 1;
3213 u64 end;
3214 u64 cur = start;
3215 u64 extent_offset;
Chris Masond1310b22008-01-24 16:13:08 -05003216 u64 block_start;
3217 u64 iosize;
3218 sector_t sector;
Chris Mason2c64c532009-09-02 15:04:12 -04003219 struct extent_state *cached_state = NULL;
Chris Masond1310b22008-01-24 16:13:08 -05003220 struct extent_map *em;
3221 struct block_device *bdev;
Chris Mason7f3c74f2008-07-18 12:01:11 -04003222 size_t pg_offset = 0;
Chris Masond1310b22008-01-24 16:13:08 -05003223 size_t blocksize;
Chris Mason40f76582014-05-21 13:35:51 -07003224 int ret = 0;
3225 int nr = 0;
3226 bool compressed;
Chris Masond1310b22008-01-24 16:13:08 -05003227
Chris Mason247e7432008-07-17 12:53:51 -04003228 if (tree->ops && tree->ops->writepage_start_hook) {
Chris Masonc8b97812008-10-29 14:49:59 -04003229 ret = tree->ops->writepage_start_hook(page, start,
3230 page_end);
Jeff Mahoney87826df2012-02-15 16:23:57 +01003231 if (ret) {
3232 /* Fixup worker will requeue */
3233 if (ret == -EBUSY)
3234 wbc->pages_skipped++;
3235 else
3236 redirty_page_for_writepage(wbc, page);
Chris Mason40f76582014-05-21 13:35:51 -07003237
Chris Mason11c83492009-04-20 15:50:09 -04003238 update_nr_written(page, wbc, nr_written);
Chris Mason247e7432008-07-17 12:53:51 -04003239 unlock_page(page);
Chris Mason40f76582014-05-21 13:35:51 -07003240 ret = 1;
Chris Mason11c83492009-04-20 15:50:09 -04003241 goto done_unlocked;
Chris Mason247e7432008-07-17 12:53:51 -04003242 }
3243 }
3244
Chris Mason11c83492009-04-20 15:50:09 -04003245 /*
3246 * we don't want to touch the inode after unlocking the page,
3247 * so we update the mapping writeback index now
3248 */
3249 update_nr_written(page, wbc, nr_written + 1);
Chris Mason771ed682008-11-06 22:02:51 -05003250
Chris Masond1310b22008-01-24 16:13:08 -05003251 end = page_end;
Chris Mason40f76582014-05-21 13:35:51 -07003252 if (i_size <= start) {
Chris Masone6dcd2d2008-07-17 12:53:50 -04003253 if (tree->ops && tree->ops->writepage_end_io_hook)
3254 tree->ops->writepage_end_io_hook(page, start,
3255 page_end, NULL, 1);
Chris Masond1310b22008-01-24 16:13:08 -05003256 goto done;
3257 }
3258
Chris Masond1310b22008-01-24 16:13:08 -05003259 blocksize = inode->i_sb->s_blocksize;
3260
3261 while (cur <= end) {
Chris Mason40f76582014-05-21 13:35:51 -07003262 u64 em_end;
3263 if (cur >= i_size) {
Chris Masone6dcd2d2008-07-17 12:53:50 -04003264 if (tree->ops && tree->ops->writepage_end_io_hook)
3265 tree->ops->writepage_end_io_hook(page, cur,
3266 page_end, NULL, 1);
Chris Masond1310b22008-01-24 16:13:08 -05003267 break;
3268 }
Chris Mason7f3c74f2008-07-18 12:01:11 -04003269 em = epd->get_extent(inode, page, pg_offset, cur,
Chris Masond1310b22008-01-24 16:13:08 -05003270 end - cur + 1, 1);
David Sterbac7040052011-04-19 18:00:01 +02003271 if (IS_ERR_OR_NULL(em)) {
Chris Masond1310b22008-01-24 16:13:08 -05003272 SetPageError(page);
Filipe Manana61391d52014-05-09 17:17:40 +01003273 ret = PTR_ERR_OR_ZERO(em);
Chris Masond1310b22008-01-24 16:13:08 -05003274 break;
3275 }
3276
3277 extent_offset = cur - em->start;
Chris Mason40f76582014-05-21 13:35:51 -07003278 em_end = extent_map_end(em);
3279 BUG_ON(em_end <= cur);
Chris Masond1310b22008-01-24 16:13:08 -05003280 BUG_ON(end < cur);
Chris Mason40f76582014-05-21 13:35:51 -07003281 iosize = min(em_end - cur, end - cur + 1);
Qu Wenruofda28322013-02-26 08:10:22 +00003282 iosize = ALIGN(iosize, blocksize);
Chris Masond1310b22008-01-24 16:13:08 -05003283 sector = (em->block_start + extent_offset) >> 9;
3284 bdev = em->bdev;
3285 block_start = em->block_start;
Chris Masonc8b97812008-10-29 14:49:59 -04003286 compressed = test_bit(EXTENT_FLAG_COMPRESSED, &em->flags);
Chris Masond1310b22008-01-24 16:13:08 -05003287 free_extent_map(em);
3288 em = NULL;
3289
Chris Masonc8b97812008-10-29 14:49:59 -04003290 /*
3291 * compressed and inline extents are written through other
3292 * paths in the FS
3293 */
3294 if (compressed || block_start == EXTENT_MAP_HOLE ||
Chris Masond1310b22008-01-24 16:13:08 -05003295 block_start == EXTENT_MAP_INLINE) {
Chris Masonc8b97812008-10-29 14:49:59 -04003296 /*
3297 * end_io notification does not happen here for
3298 * compressed extents
3299 */
3300 if (!compressed && tree->ops &&
3301 tree->ops->writepage_end_io_hook)
Chris Masone6dcd2d2008-07-17 12:53:50 -04003302 tree->ops->writepage_end_io_hook(page, cur,
3303 cur + iosize - 1,
3304 NULL, 1);
Chris Masonc8b97812008-10-29 14:49:59 -04003305 else if (compressed) {
3306 /* we don't want to end_page_writeback on
3307 * a compressed extent. this happens
3308 * elsewhere
3309 */
3310 nr++;
3311 }
3312
3313 cur += iosize;
Chris Mason7f3c74f2008-07-18 12:01:11 -04003314 pg_offset += iosize;
Chris Masond1310b22008-01-24 16:13:08 -05003315 continue;
3316 }
Chris Masonc8b97812008-10-29 14:49:59 -04003317
Chris Masond1310b22008-01-24 16:13:08 -05003318 if (tree->ops && tree->ops->writepage_io_hook) {
3319 ret = tree->ops->writepage_io_hook(page, cur,
3320 cur + iosize - 1);
3321 } else {
3322 ret = 0;
3323 }
Chris Mason1259ab72008-05-12 13:39:03 -04003324 if (ret) {
Chris Masond1310b22008-01-24 16:13:08 -05003325 SetPageError(page);
Chris Mason1259ab72008-05-12 13:39:03 -04003326 } else {
Chris Mason40f76582014-05-21 13:35:51 -07003327 unsigned long max_nr = (i_size >> PAGE_CACHE_SHIFT) + 1;
Chris Mason7f3c74f2008-07-18 12:01:11 -04003328
Chris Masond1310b22008-01-24 16:13:08 -05003329 set_range_writeback(tree, cur, cur + iosize - 1);
3330 if (!PageWriteback(page)) {
Frank Holtonefe120a2013-12-20 11:37:06 -05003331 btrfs_err(BTRFS_I(inode)->root->fs_info,
3332 "page %lu not writeback, cur %llu end %llu",
Geert Uytterhoevenc1c9ff72013-08-20 13:20:07 +02003333 page->index, cur, end);
Chris Masond1310b22008-01-24 16:13:08 -05003334 }
3335
Chris Masonffbd5172009-04-20 15:50:09 -04003336 ret = submit_extent_page(write_flags, tree, page,
3337 sector, iosize, pg_offset,
3338 bdev, &epd->bio, max_nr,
Chris Masonc8b97812008-10-29 14:49:59 -04003339 end_bio_extent_writepage,
3340 0, 0, 0);
Chris Masond1310b22008-01-24 16:13:08 -05003341 if (ret)
3342 SetPageError(page);
3343 }
3344 cur = cur + iosize;
Chris Mason7f3c74f2008-07-18 12:01:11 -04003345 pg_offset += iosize;
Chris Masond1310b22008-01-24 16:13:08 -05003346 nr++;
3347 }
3348done:
Chris Mason40f76582014-05-21 13:35:51 -07003349 *nr_ret = nr;
Chris Mason771ed682008-11-06 22:02:51 -05003350
Chris Mason11c83492009-04-20 15:50:09 -04003351done_unlocked:
3352
Chris Mason2c64c532009-09-02 15:04:12 -04003353 /* drop our reference on any cached states */
3354 free_extent_state(cached_state);
Chris Mason40f76582014-05-21 13:35:51 -07003355 return ret;
3356}
3357
3358/*
3359 * the writepage semantics are similar to regular writepage. extent
3360 * records are inserted to lock ranges in the tree, and as dirty areas
3361 * are found, they are marked writeback. Then the lock bits are removed
3362 * and the end_io handler clears the writeback ranges
3363 */
3364static int __extent_writepage(struct page *page, struct writeback_control *wbc,
3365 void *data)
3366{
3367 struct inode *inode = page->mapping->host;
3368 struct extent_page_data *epd = data;
3369 u64 start = page_offset(page);
3370 u64 page_end = start + PAGE_CACHE_SIZE - 1;
3371 int ret;
3372 int nr = 0;
3373 size_t pg_offset = 0;
3374 loff_t i_size = i_size_read(inode);
3375 unsigned long end_index = i_size >> PAGE_CACHE_SHIFT;
3376 int write_flags;
3377 unsigned long nr_written = 0;
3378
3379 if (wbc->sync_mode == WB_SYNC_ALL)
3380 write_flags = WRITE_SYNC;
3381 else
3382 write_flags = WRITE;
3383
3384 trace___extent_writepage(page, inode, wbc);
3385
3386 WARN_ON(!PageLocked(page));
3387
3388 ClearPageError(page);
3389
3390 pg_offset = i_size & (PAGE_CACHE_SIZE - 1);
3391 if (page->index > end_index ||
3392 (page->index == end_index && !pg_offset)) {
3393 page->mapping->a_ops->invalidatepage(page, 0, PAGE_CACHE_SIZE);
3394 unlock_page(page);
3395 return 0;
3396 }
3397
3398 if (page->index == end_index) {
3399 char *userpage;
3400
3401 userpage = kmap_atomic(page);
3402 memset(userpage + pg_offset, 0,
3403 PAGE_CACHE_SIZE - pg_offset);
3404 kunmap_atomic(userpage);
3405 flush_dcache_page(page);
3406 }
3407
3408 pg_offset = 0;
3409
3410 set_page_extent_mapped(page);
3411
3412 ret = writepage_delalloc(inode, page, wbc, epd, start, &nr_written);
3413 if (ret == 1)
3414 goto done_unlocked;
3415 if (ret)
3416 goto done;
3417
3418 ret = __extent_writepage_io(inode, page, wbc, epd,
3419 i_size, nr_written, write_flags, &nr);
3420 if (ret == 1)
3421 goto done_unlocked;
3422
3423done:
Chris Masone6dcd2d2008-07-17 12:53:50 -04003424 if (nr == 0) {
Chris Masond1310b22008-01-24 16:13:08 -05003425 /* make sure the mapping tag for page dirty gets cleared */
Chris Mason771ed682008-11-06 22:02:51 -05003426 set_page_writeback(page);
3427 end_page_writeback(page);
3428 }
Filipe Manana61391d52014-05-09 17:17:40 +01003429 if (PageError(page)) {
3430 ret = ret < 0 ? ret : -EIO;
3431 end_extent_writepage(page, ret, start, page_end);
3432 }
Christoph Hellwigb2950862008-12-02 09:54:17 -05003433 unlock_page(page);
Chris Mason40f76582014-05-21 13:35:51 -07003434 return ret;
Chris Mason4bef0842008-09-08 11:18:08 -04003435
3436done_unlocked:
Chris Masond1310b22008-01-24 16:13:08 -05003437 return 0;
3438}
3439
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003440static int eb_wait(void *word)
3441{
3442 io_schedule();
3443 return 0;
3444}
3445
Josef Bacikfd8b2b62013-04-24 16:41:19 -04003446void wait_on_extent_buffer_writeback(struct extent_buffer *eb)
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003447{
3448 wait_on_bit(&eb->bflags, EXTENT_BUFFER_WRITEBACK, eb_wait,
3449 TASK_UNINTERRUPTIBLE);
3450}
3451
Chris Mason0e378df2014-05-19 20:55:27 -07003452static noinline_for_stack int
3453lock_extent_buffer_for_io(struct extent_buffer *eb,
3454 struct btrfs_fs_info *fs_info,
3455 struct extent_page_data *epd)
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003456{
3457 unsigned long i, num_pages;
3458 int flush = 0;
3459 int ret = 0;
3460
3461 if (!btrfs_try_tree_write_lock(eb)) {
3462 flush = 1;
3463 flush_write_bio(epd);
3464 btrfs_tree_lock(eb);
3465 }
3466
3467 if (test_bit(EXTENT_BUFFER_WRITEBACK, &eb->bflags)) {
3468 btrfs_tree_unlock(eb);
3469 if (!epd->sync_io)
3470 return 0;
3471 if (!flush) {
3472 flush_write_bio(epd);
3473 flush = 1;
3474 }
Chris Masona098d8e2012-03-21 12:09:56 -04003475 while (1) {
3476 wait_on_extent_buffer_writeback(eb);
3477 btrfs_tree_lock(eb);
3478 if (!test_bit(EXTENT_BUFFER_WRITEBACK, &eb->bflags))
3479 break;
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003480 btrfs_tree_unlock(eb);
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003481 }
3482 }
3483
Josef Bacik51561ff2012-07-20 16:25:24 -04003484 /*
3485 * We need to do this to prevent races in people who check if the eb is
3486 * under IO since we can end up having no IO bits set for a short period
3487 * of time.
3488 */
3489 spin_lock(&eb->refs_lock);
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003490 if (test_and_clear_bit(EXTENT_BUFFER_DIRTY, &eb->bflags)) {
3491 set_bit(EXTENT_BUFFER_WRITEBACK, &eb->bflags);
Josef Bacik51561ff2012-07-20 16:25:24 -04003492 spin_unlock(&eb->refs_lock);
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003493 btrfs_set_header_flag(eb, BTRFS_HEADER_FLAG_WRITTEN);
Miao Xiee2d84522013-01-29 10:09:20 +00003494 __percpu_counter_add(&fs_info->dirty_metadata_bytes,
3495 -eb->len,
3496 fs_info->dirty_metadata_batch);
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003497 ret = 1;
Josef Bacik51561ff2012-07-20 16:25:24 -04003498 } else {
3499 spin_unlock(&eb->refs_lock);
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003500 }
3501
3502 btrfs_tree_unlock(eb);
3503
3504 if (!ret)
3505 return ret;
3506
3507 num_pages = num_extent_pages(eb->start, eb->len);
3508 for (i = 0; i < num_pages; i++) {
3509 struct page *p = extent_buffer_page(eb, i);
3510
3511 if (!trylock_page(p)) {
3512 if (!flush) {
3513 flush_write_bio(epd);
3514 flush = 1;
3515 }
3516 lock_page(p);
3517 }
3518 }
3519
3520 return ret;
3521}
3522
3523static void end_extent_buffer_writeback(struct extent_buffer *eb)
3524{
3525 clear_bit(EXTENT_BUFFER_WRITEBACK, &eb->bflags);
Peter Zijlstra4e857c52014-03-17 18:06:10 +01003526 smp_mb__after_atomic();
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003527 wake_up_bit(&eb->bflags, EXTENT_BUFFER_WRITEBACK);
3528}
3529
3530static void end_bio_extent_buffer_writepage(struct bio *bio, int err)
3531{
Kent Overstreet2c30c712013-11-07 12:20:26 -08003532 struct bio_vec *bvec;
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003533 struct extent_buffer *eb;
Kent Overstreet2c30c712013-11-07 12:20:26 -08003534 int i, done;
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003535
Kent Overstreet2c30c712013-11-07 12:20:26 -08003536 bio_for_each_segment_all(bvec, bio, i) {
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003537 struct page *page = bvec->bv_page;
3538
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003539 eb = (struct extent_buffer *)page->private;
3540 BUG_ON(!eb);
3541 done = atomic_dec_and_test(&eb->io_pages);
3542
Kent Overstreet2c30c712013-11-07 12:20:26 -08003543 if (err || test_bit(EXTENT_BUFFER_IOERR, &eb->bflags)) {
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003544 set_bit(EXTENT_BUFFER_IOERR, &eb->bflags);
3545 ClearPageUptodate(page);
3546 SetPageError(page);
3547 }
3548
3549 end_page_writeback(page);
3550
3551 if (!done)
3552 continue;
3553
3554 end_extent_buffer_writeback(eb);
Kent Overstreet2c30c712013-11-07 12:20:26 -08003555 }
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003556
3557 bio_put(bio);
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003558}
3559
Chris Mason0e378df2014-05-19 20:55:27 -07003560static noinline_for_stack int write_one_eb(struct extent_buffer *eb,
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003561 struct btrfs_fs_info *fs_info,
3562 struct writeback_control *wbc,
3563 struct extent_page_data *epd)
3564{
3565 struct block_device *bdev = fs_info->fs_devices->latest_bdev;
Josef Bacikf28491e2013-12-16 13:24:27 -05003566 struct extent_io_tree *tree = &BTRFS_I(fs_info->btree_inode)->io_tree;
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003567 u64 offset = eb->start;
3568 unsigned long i, num_pages;
Josef Bacikde0022b2012-09-25 14:25:58 -04003569 unsigned long bio_flags = 0;
Josef Bacikd4c7ca82013-04-19 19:49:09 -04003570 int rw = (epd->sync_io ? WRITE_SYNC : WRITE) | REQ_META;
Josef Bacikd7dbe9e2012-04-23 14:00:51 -04003571 int ret = 0;
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003572
3573 clear_bit(EXTENT_BUFFER_IOERR, &eb->bflags);
3574 num_pages = num_extent_pages(eb->start, eb->len);
3575 atomic_set(&eb->io_pages, num_pages);
Josef Bacikde0022b2012-09-25 14:25:58 -04003576 if (btrfs_header_owner(eb) == BTRFS_TREE_LOG_OBJECTID)
3577 bio_flags = EXTENT_BIO_TREE_LOG;
3578
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003579 for (i = 0; i < num_pages; i++) {
3580 struct page *p = extent_buffer_page(eb, i);
3581
3582 clear_page_dirty_for_io(p);
3583 set_page_writeback(p);
Josef Bacikf28491e2013-12-16 13:24:27 -05003584 ret = submit_extent_page(rw, tree, p, offset >> 9,
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003585 PAGE_CACHE_SIZE, 0, bdev, &epd->bio,
3586 -1, end_bio_extent_buffer_writepage,
Josef Bacikde0022b2012-09-25 14:25:58 -04003587 0, epd->bio_flags, bio_flags);
3588 epd->bio_flags = bio_flags;
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003589 if (ret) {
3590 set_bit(EXTENT_BUFFER_IOERR, &eb->bflags);
3591 SetPageError(p);
3592 if (atomic_sub_and_test(num_pages - i, &eb->io_pages))
3593 end_extent_buffer_writeback(eb);
3594 ret = -EIO;
3595 break;
3596 }
3597 offset += PAGE_CACHE_SIZE;
3598 update_nr_written(p, wbc, 1);
3599 unlock_page(p);
3600 }
3601
3602 if (unlikely(ret)) {
3603 for (; i < num_pages; i++) {
3604 struct page *p = extent_buffer_page(eb, i);
3605 unlock_page(p);
3606 }
3607 }
3608
3609 return ret;
3610}
3611
3612int btree_write_cache_pages(struct address_space *mapping,
3613 struct writeback_control *wbc)
3614{
3615 struct extent_io_tree *tree = &BTRFS_I(mapping->host)->io_tree;
3616 struct btrfs_fs_info *fs_info = BTRFS_I(mapping->host)->root->fs_info;
3617 struct extent_buffer *eb, *prev_eb = NULL;
3618 struct extent_page_data epd = {
3619 .bio = NULL,
3620 .tree = tree,
3621 .extent_locked = 0,
3622 .sync_io = wbc->sync_mode == WB_SYNC_ALL,
Josef Bacikde0022b2012-09-25 14:25:58 -04003623 .bio_flags = 0,
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003624 };
3625 int ret = 0;
3626 int done = 0;
3627 int nr_to_write_done = 0;
3628 struct pagevec pvec;
3629 int nr_pages;
3630 pgoff_t index;
3631 pgoff_t end; /* Inclusive */
3632 int scanned = 0;
3633 int tag;
3634
3635 pagevec_init(&pvec, 0);
3636 if (wbc->range_cyclic) {
3637 index = mapping->writeback_index; /* Start from prev offset */
3638 end = -1;
3639 } else {
3640 index = wbc->range_start >> PAGE_CACHE_SHIFT;
3641 end = wbc->range_end >> PAGE_CACHE_SHIFT;
3642 scanned = 1;
3643 }
3644 if (wbc->sync_mode == WB_SYNC_ALL)
3645 tag = PAGECACHE_TAG_TOWRITE;
3646 else
3647 tag = PAGECACHE_TAG_DIRTY;
3648retry:
3649 if (wbc->sync_mode == WB_SYNC_ALL)
3650 tag_pages_for_writeback(mapping, index, end);
3651 while (!done && !nr_to_write_done && (index <= end) &&
3652 (nr_pages = pagevec_lookup_tag(&pvec, mapping, &index, tag,
3653 min(end - index, (pgoff_t)PAGEVEC_SIZE-1) + 1))) {
3654 unsigned i;
3655
3656 scanned = 1;
3657 for (i = 0; i < nr_pages; i++) {
3658 struct page *page = pvec.pages[i];
3659
3660 if (!PagePrivate(page))
3661 continue;
3662
3663 if (!wbc->range_cyclic && page->index > end) {
3664 done = 1;
3665 break;
3666 }
3667
Josef Bacikb5bae262012-09-14 13:43:01 -04003668 spin_lock(&mapping->private_lock);
3669 if (!PagePrivate(page)) {
3670 spin_unlock(&mapping->private_lock);
3671 continue;
3672 }
3673
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003674 eb = (struct extent_buffer *)page->private;
Josef Bacikb5bae262012-09-14 13:43:01 -04003675
3676 /*
3677 * Shouldn't happen and normally this would be a BUG_ON
3678 * but no sense in crashing the users box for something
3679 * we can survive anyway.
3680 */
Dulshani Gunawardhanafae7f212013-10-31 10:30:08 +05303681 if (WARN_ON(!eb)) {
Josef Bacikb5bae262012-09-14 13:43:01 -04003682 spin_unlock(&mapping->private_lock);
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003683 continue;
3684 }
3685
Josef Bacikb5bae262012-09-14 13:43:01 -04003686 if (eb == prev_eb) {
3687 spin_unlock(&mapping->private_lock);
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003688 continue;
3689 }
3690
Josef Bacikb5bae262012-09-14 13:43:01 -04003691 ret = atomic_inc_not_zero(&eb->refs);
3692 spin_unlock(&mapping->private_lock);
3693 if (!ret)
3694 continue;
3695
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003696 prev_eb = eb;
3697 ret = lock_extent_buffer_for_io(eb, fs_info, &epd);
3698 if (!ret) {
3699 free_extent_buffer(eb);
3700 continue;
3701 }
3702
3703 ret = write_one_eb(eb, fs_info, wbc, &epd);
3704 if (ret) {
3705 done = 1;
3706 free_extent_buffer(eb);
3707 break;
3708 }
3709 free_extent_buffer(eb);
3710
3711 /*
3712 * the filesystem may choose to bump up nr_to_write.
3713 * We have to make sure to honor the new nr_to_write
3714 * at any time
3715 */
3716 nr_to_write_done = wbc->nr_to_write <= 0;
3717 }
3718 pagevec_release(&pvec);
3719 cond_resched();
3720 }
3721 if (!scanned && !done) {
3722 /*
3723 * We hit the last page and there is more work to be done: wrap
3724 * back to the start of the file
3725 */
3726 scanned = 1;
3727 index = 0;
3728 goto retry;
3729 }
3730 flush_write_bio(&epd);
3731 return ret;
3732}
3733
Chris Masond1310b22008-01-24 16:13:08 -05003734/**
3735 * write_cache_pages - walk the list of dirty pages of the given address space and write all of them.
3736 * @mapping: address space structure to write
3737 * @wbc: subtract the number of written pages from *@wbc->nr_to_write
3738 * @writepage: function called for each page
3739 * @data: data passed to writepage function
3740 *
3741 * If a page is already under I/O, write_cache_pages() skips it, even
3742 * if it's dirty. This is desirable behaviour for memory-cleaning writeback,
3743 * but it is INCORRECT for data-integrity system calls such as fsync(). fsync()
3744 * and msync() need to guarantee that all the data which was dirty at the time
3745 * the call was made get new I/O started against them. If wbc->sync_mode is
3746 * WB_SYNC_ALL then we were called for data integrity and we must wait for
3747 * existing IO to complete.
3748 */
Chris Mason4bef0842008-09-08 11:18:08 -04003749static int extent_write_cache_pages(struct extent_io_tree *tree,
3750 struct address_space *mapping,
3751 struct writeback_control *wbc,
Chris Masond2c3f4f2008-11-19 12:44:22 -05003752 writepage_t writepage, void *data,
3753 void (*flush_fn)(void *))
Chris Masond1310b22008-01-24 16:13:08 -05003754{
Josef Bacik7fd1a3f2012-06-27 17:18:41 -04003755 struct inode *inode = mapping->host;
Chris Masond1310b22008-01-24 16:13:08 -05003756 int ret = 0;
3757 int done = 0;
Filipe Manana61391d52014-05-09 17:17:40 +01003758 int err = 0;
Chris Masonf85d7d6c2009-09-18 16:03:16 -04003759 int nr_to_write_done = 0;
Chris Masond1310b22008-01-24 16:13:08 -05003760 struct pagevec pvec;
3761 int nr_pages;
3762 pgoff_t index;
3763 pgoff_t end; /* Inclusive */
3764 int scanned = 0;
Josef Bacikf7aaa062011-07-15 21:26:38 +00003765 int tag;
Chris Masond1310b22008-01-24 16:13:08 -05003766
Josef Bacik7fd1a3f2012-06-27 17:18:41 -04003767 /*
3768 * We have to hold onto the inode so that ordered extents can do their
3769 * work when the IO finishes. The alternative to this is failing to add
3770 * an ordered extent if the igrab() fails there and that is a huge pain
3771 * to deal with, so instead just hold onto the inode throughout the
3772 * writepages operation. If it fails here we are freeing up the inode
3773 * anyway and we'd rather not waste our time writing out stuff that is
3774 * going to be truncated anyway.
3775 */
3776 if (!igrab(inode))
3777 return 0;
3778
Chris Masond1310b22008-01-24 16:13:08 -05003779 pagevec_init(&pvec, 0);
3780 if (wbc->range_cyclic) {
3781 index = mapping->writeback_index; /* Start from prev offset */
3782 end = -1;
3783 } else {
3784 index = wbc->range_start >> PAGE_CACHE_SHIFT;
3785 end = wbc->range_end >> PAGE_CACHE_SHIFT;
Chris Masond1310b22008-01-24 16:13:08 -05003786 scanned = 1;
3787 }
Josef Bacikf7aaa062011-07-15 21:26:38 +00003788 if (wbc->sync_mode == WB_SYNC_ALL)
3789 tag = PAGECACHE_TAG_TOWRITE;
3790 else
3791 tag = PAGECACHE_TAG_DIRTY;
Chris Masond1310b22008-01-24 16:13:08 -05003792retry:
Josef Bacikf7aaa062011-07-15 21:26:38 +00003793 if (wbc->sync_mode == WB_SYNC_ALL)
3794 tag_pages_for_writeback(mapping, index, end);
Chris Masonf85d7d6c2009-09-18 16:03:16 -04003795 while (!done && !nr_to_write_done && (index <= end) &&
Josef Bacikf7aaa062011-07-15 21:26:38 +00003796 (nr_pages = pagevec_lookup_tag(&pvec, mapping, &index, tag,
3797 min(end - index, (pgoff_t)PAGEVEC_SIZE-1) + 1))) {
Chris Masond1310b22008-01-24 16:13:08 -05003798 unsigned i;
3799
3800 scanned = 1;
3801 for (i = 0; i < nr_pages; i++) {
3802 struct page *page = pvec.pages[i];
3803
3804 /*
3805 * At this point we hold neither mapping->tree_lock nor
3806 * lock on the page itself: the page may be truncated or
3807 * invalidated (changing page->mapping to NULL), or even
3808 * swizzled back from swapper_space to tmpfs file
3809 * mapping
3810 */
Josef Bacikc8f2f242013-02-11 11:33:00 -05003811 if (!trylock_page(page)) {
3812 flush_fn(data);
3813 lock_page(page);
Chris Mason01d658f2011-11-01 10:08:06 -04003814 }
Chris Masond1310b22008-01-24 16:13:08 -05003815
3816 if (unlikely(page->mapping != mapping)) {
3817 unlock_page(page);
3818 continue;
3819 }
3820
3821 if (!wbc->range_cyclic && page->index > end) {
3822 done = 1;
3823 unlock_page(page);
3824 continue;
3825 }
3826
Chris Masond2c3f4f2008-11-19 12:44:22 -05003827 if (wbc->sync_mode != WB_SYNC_NONE) {
Chris Mason0e6bd952008-11-20 10:46:35 -05003828 if (PageWriteback(page))
3829 flush_fn(data);
Chris Masond1310b22008-01-24 16:13:08 -05003830 wait_on_page_writeback(page);
Chris Masond2c3f4f2008-11-19 12:44:22 -05003831 }
Chris Masond1310b22008-01-24 16:13:08 -05003832
3833 if (PageWriteback(page) ||
3834 !clear_page_dirty_for_io(page)) {
3835 unlock_page(page);
3836 continue;
3837 }
3838
3839 ret = (*writepage)(page, wbc, data);
3840
3841 if (unlikely(ret == AOP_WRITEPAGE_ACTIVATE)) {
3842 unlock_page(page);
3843 ret = 0;
3844 }
Filipe Manana61391d52014-05-09 17:17:40 +01003845 if (!err && ret < 0)
3846 err = ret;
Chris Masonf85d7d6c2009-09-18 16:03:16 -04003847
3848 /*
3849 * the filesystem may choose to bump up nr_to_write.
3850 * We have to make sure to honor the new nr_to_write
3851 * at any time
3852 */
3853 nr_to_write_done = wbc->nr_to_write <= 0;
Chris Masond1310b22008-01-24 16:13:08 -05003854 }
3855 pagevec_release(&pvec);
3856 cond_resched();
3857 }
Filipe Manana61391d52014-05-09 17:17:40 +01003858 if (!scanned && !done && !err) {
Chris Masond1310b22008-01-24 16:13:08 -05003859 /*
3860 * We hit the last page and there is more work to be done: wrap
3861 * back to the start of the file
3862 */
3863 scanned = 1;
3864 index = 0;
3865 goto retry;
3866 }
Josef Bacik7fd1a3f2012-06-27 17:18:41 -04003867 btrfs_add_delayed_iput(inode);
Filipe Manana61391d52014-05-09 17:17:40 +01003868 return err;
Chris Masond1310b22008-01-24 16:13:08 -05003869}
Chris Masond1310b22008-01-24 16:13:08 -05003870
Chris Masonffbd5172009-04-20 15:50:09 -04003871static void flush_epd_write_bio(struct extent_page_data *epd)
3872{
3873 if (epd->bio) {
Jeff Mahoney355808c2011-10-03 23:23:14 -04003874 int rw = WRITE;
3875 int ret;
3876
Chris Masonffbd5172009-04-20 15:50:09 -04003877 if (epd->sync_io)
Jeff Mahoney355808c2011-10-03 23:23:14 -04003878 rw = WRITE_SYNC;
3879
Josef Bacikde0022b2012-09-25 14:25:58 -04003880 ret = submit_one_bio(rw, epd->bio, 0, epd->bio_flags);
Jeff Mahoney79787ea2012-03-12 16:03:00 +01003881 BUG_ON(ret < 0); /* -ENOMEM */
Chris Masonffbd5172009-04-20 15:50:09 -04003882 epd->bio = NULL;
3883 }
3884}
3885
Chris Masond2c3f4f2008-11-19 12:44:22 -05003886static noinline void flush_write_bio(void *data)
3887{
3888 struct extent_page_data *epd = data;
Chris Masonffbd5172009-04-20 15:50:09 -04003889 flush_epd_write_bio(epd);
Chris Masond2c3f4f2008-11-19 12:44:22 -05003890}
3891
Chris Masond1310b22008-01-24 16:13:08 -05003892int extent_write_full_page(struct extent_io_tree *tree, struct page *page,
3893 get_extent_t *get_extent,
3894 struct writeback_control *wbc)
3895{
3896 int ret;
Chris Masond1310b22008-01-24 16:13:08 -05003897 struct extent_page_data epd = {
3898 .bio = NULL,
3899 .tree = tree,
3900 .get_extent = get_extent,
Chris Mason771ed682008-11-06 22:02:51 -05003901 .extent_locked = 0,
Chris Masonffbd5172009-04-20 15:50:09 -04003902 .sync_io = wbc->sync_mode == WB_SYNC_ALL,
Josef Bacikde0022b2012-09-25 14:25:58 -04003903 .bio_flags = 0,
Chris Masond1310b22008-01-24 16:13:08 -05003904 };
Chris Masond1310b22008-01-24 16:13:08 -05003905
Chris Masond1310b22008-01-24 16:13:08 -05003906 ret = __extent_writepage(page, wbc, &epd);
3907
Chris Masonffbd5172009-04-20 15:50:09 -04003908 flush_epd_write_bio(&epd);
Chris Masond1310b22008-01-24 16:13:08 -05003909 return ret;
3910}
Chris Masond1310b22008-01-24 16:13:08 -05003911
Chris Mason771ed682008-11-06 22:02:51 -05003912int extent_write_locked_range(struct extent_io_tree *tree, struct inode *inode,
3913 u64 start, u64 end, get_extent_t *get_extent,
3914 int mode)
3915{
3916 int ret = 0;
3917 struct address_space *mapping = inode->i_mapping;
3918 struct page *page;
3919 unsigned long nr_pages = (end - start + PAGE_CACHE_SIZE) >>
3920 PAGE_CACHE_SHIFT;
3921
3922 struct extent_page_data epd = {
3923 .bio = NULL,
3924 .tree = tree,
3925 .get_extent = get_extent,
3926 .extent_locked = 1,
Chris Masonffbd5172009-04-20 15:50:09 -04003927 .sync_io = mode == WB_SYNC_ALL,
Josef Bacikde0022b2012-09-25 14:25:58 -04003928 .bio_flags = 0,
Chris Mason771ed682008-11-06 22:02:51 -05003929 };
3930 struct writeback_control wbc_writepages = {
Chris Mason771ed682008-11-06 22:02:51 -05003931 .sync_mode = mode,
Chris Mason771ed682008-11-06 22:02:51 -05003932 .nr_to_write = nr_pages * 2,
3933 .range_start = start,
3934 .range_end = end + 1,
3935 };
3936
Chris Masond3977122009-01-05 21:25:51 -05003937 while (start <= end) {
Chris Mason771ed682008-11-06 22:02:51 -05003938 page = find_get_page(mapping, start >> PAGE_CACHE_SHIFT);
3939 if (clear_page_dirty_for_io(page))
3940 ret = __extent_writepage(page, &wbc_writepages, &epd);
3941 else {
3942 if (tree->ops && tree->ops->writepage_end_io_hook)
3943 tree->ops->writepage_end_io_hook(page, start,
3944 start + PAGE_CACHE_SIZE - 1,
3945 NULL, 1);
3946 unlock_page(page);
3947 }
3948 page_cache_release(page);
3949 start += PAGE_CACHE_SIZE;
3950 }
3951
Chris Masonffbd5172009-04-20 15:50:09 -04003952 flush_epd_write_bio(&epd);
Chris Mason771ed682008-11-06 22:02:51 -05003953 return ret;
3954}
Chris Masond1310b22008-01-24 16:13:08 -05003955
3956int extent_writepages(struct extent_io_tree *tree,
3957 struct address_space *mapping,
3958 get_extent_t *get_extent,
3959 struct writeback_control *wbc)
3960{
3961 int ret = 0;
3962 struct extent_page_data epd = {
3963 .bio = NULL,
3964 .tree = tree,
3965 .get_extent = get_extent,
Chris Mason771ed682008-11-06 22:02:51 -05003966 .extent_locked = 0,
Chris Masonffbd5172009-04-20 15:50:09 -04003967 .sync_io = wbc->sync_mode == WB_SYNC_ALL,
Josef Bacikde0022b2012-09-25 14:25:58 -04003968 .bio_flags = 0,
Chris Masond1310b22008-01-24 16:13:08 -05003969 };
3970
Chris Mason4bef0842008-09-08 11:18:08 -04003971 ret = extent_write_cache_pages(tree, mapping, wbc,
Chris Masond2c3f4f2008-11-19 12:44:22 -05003972 __extent_writepage, &epd,
3973 flush_write_bio);
Chris Masonffbd5172009-04-20 15:50:09 -04003974 flush_epd_write_bio(&epd);
Chris Masond1310b22008-01-24 16:13:08 -05003975 return ret;
3976}
Chris Masond1310b22008-01-24 16:13:08 -05003977
3978int extent_readpages(struct extent_io_tree *tree,
3979 struct address_space *mapping,
3980 struct list_head *pages, unsigned nr_pages,
3981 get_extent_t get_extent)
3982{
3983 struct bio *bio = NULL;
3984 unsigned page_idx;
Chris Masonc8b97812008-10-29 14:49:59 -04003985 unsigned long bio_flags = 0;
Liu Bo67c96842012-07-20 21:43:09 -06003986 struct page *pagepool[16];
3987 struct page *page;
Miao Xie125bac012013-07-25 19:22:37 +08003988 struct extent_map *em_cached = NULL;
Liu Bo67c96842012-07-20 21:43:09 -06003989 int nr = 0;
Chris Masond1310b22008-01-24 16:13:08 -05003990
Chris Masond1310b22008-01-24 16:13:08 -05003991 for (page_idx = 0; page_idx < nr_pages; page_idx++) {
Liu Bo67c96842012-07-20 21:43:09 -06003992 page = list_entry(pages->prev, struct page, lru);
Chris Masond1310b22008-01-24 16:13:08 -05003993
3994 prefetchw(&page->flags);
3995 list_del(&page->lru);
Liu Bo67c96842012-07-20 21:43:09 -06003996 if (add_to_page_cache_lru(page, mapping,
Itaru Kitayama43e817a2011-04-25 19:43:51 -04003997 page->index, GFP_NOFS)) {
Liu Bo67c96842012-07-20 21:43:09 -06003998 page_cache_release(page);
3999 continue;
Chris Masond1310b22008-01-24 16:13:08 -05004000 }
Liu Bo67c96842012-07-20 21:43:09 -06004001
4002 pagepool[nr++] = page;
4003 if (nr < ARRAY_SIZE(pagepool))
4004 continue;
Miao Xie125bac012013-07-25 19:22:37 +08004005 __extent_readpages(tree, pagepool, nr, get_extent, &em_cached,
Miao Xie99740902013-07-25 19:22:36 +08004006 &bio, 0, &bio_flags, READ);
Liu Bo67c96842012-07-20 21:43:09 -06004007 nr = 0;
Chris Masond1310b22008-01-24 16:13:08 -05004008 }
Miao Xie99740902013-07-25 19:22:36 +08004009 if (nr)
Miao Xie125bac012013-07-25 19:22:37 +08004010 __extent_readpages(tree, pagepool, nr, get_extent, &em_cached,
Miao Xie99740902013-07-25 19:22:36 +08004011 &bio, 0, &bio_flags, READ);
Liu Bo67c96842012-07-20 21:43:09 -06004012
Miao Xie125bac012013-07-25 19:22:37 +08004013 if (em_cached)
4014 free_extent_map(em_cached);
4015
Chris Masond1310b22008-01-24 16:13:08 -05004016 BUG_ON(!list_empty(pages));
4017 if (bio)
Jeff Mahoney79787ea2012-03-12 16:03:00 +01004018 return submit_one_bio(READ, bio, 0, bio_flags);
Chris Masond1310b22008-01-24 16:13:08 -05004019 return 0;
4020}
Chris Masond1310b22008-01-24 16:13:08 -05004021
4022/*
4023 * basic invalidatepage code, this waits on any locked or writeback
4024 * ranges corresponding to the page, and then deletes any extent state
4025 * records from the tree
4026 */
4027int extent_invalidatepage(struct extent_io_tree *tree,
4028 struct page *page, unsigned long offset)
4029{
Josef Bacik2ac55d42010-02-03 19:33:23 +00004030 struct extent_state *cached_state = NULL;
Miao Xie4eee4fa2012-12-21 09:17:45 +00004031 u64 start = page_offset(page);
Chris Masond1310b22008-01-24 16:13:08 -05004032 u64 end = start + PAGE_CACHE_SIZE - 1;
4033 size_t blocksize = page->mapping->host->i_sb->s_blocksize;
4034
Qu Wenruofda28322013-02-26 08:10:22 +00004035 start += ALIGN(offset, blocksize);
Chris Masond1310b22008-01-24 16:13:08 -05004036 if (start > end)
4037 return 0;
4038
Jeff Mahoneyd0082372012-03-01 14:57:19 +01004039 lock_extent_bits(tree, start, end, 0, &cached_state);
Chris Mason1edbb732009-09-02 13:24:36 -04004040 wait_on_page_writeback(page);
Chris Masond1310b22008-01-24 16:13:08 -05004041 clear_extent_bit(tree, start, end,
Josef Bacik32c00af2009-10-08 13:34:05 -04004042 EXTENT_LOCKED | EXTENT_DIRTY | EXTENT_DELALLOC |
4043 EXTENT_DO_ACCOUNTING,
Josef Bacik2ac55d42010-02-03 19:33:23 +00004044 1, 1, &cached_state, GFP_NOFS);
Chris Masond1310b22008-01-24 16:13:08 -05004045 return 0;
4046}
Chris Masond1310b22008-01-24 16:13:08 -05004047
4048/*
Chris Mason7b13b7b2008-04-18 10:29:50 -04004049 * a helper for releasepage, this tests for areas of the page that
4050 * are locked or under IO and drops the related state bits if it is safe
4051 * to drop the page.
4052 */
Eric Sandeen48a3b632013-04-25 20:41:01 +00004053static int try_release_extent_state(struct extent_map_tree *map,
4054 struct extent_io_tree *tree,
4055 struct page *page, gfp_t mask)
Chris Mason7b13b7b2008-04-18 10:29:50 -04004056{
Miao Xie4eee4fa2012-12-21 09:17:45 +00004057 u64 start = page_offset(page);
Chris Mason7b13b7b2008-04-18 10:29:50 -04004058 u64 end = start + PAGE_CACHE_SIZE - 1;
4059 int ret = 1;
4060
Chris Mason211f90e2008-07-18 11:56:15 -04004061 if (test_range_bit(tree, start, end,
Chris Mason8b62b722009-09-02 16:53:46 -04004062 EXTENT_IOBITS, 0, NULL))
Chris Mason7b13b7b2008-04-18 10:29:50 -04004063 ret = 0;
4064 else {
4065 if ((mask & GFP_NOFS) == GFP_NOFS)
4066 mask = GFP_NOFS;
Chris Mason11ef1602009-09-23 20:28:46 -04004067 /*
4068 * at this point we can safely clear everything except the
4069 * locked bit and the nodatasum bit
4070 */
Chris Masone3f24cc2011-02-14 12:52:08 -05004071 ret = clear_extent_bit(tree, start, end,
Chris Mason11ef1602009-09-23 20:28:46 -04004072 ~(EXTENT_LOCKED | EXTENT_NODATASUM),
4073 0, 0, NULL, mask);
Chris Masone3f24cc2011-02-14 12:52:08 -05004074
4075 /* if clear_extent_bit failed for enomem reasons,
4076 * we can't allow the release to continue.
4077 */
4078 if (ret < 0)
4079 ret = 0;
4080 else
4081 ret = 1;
Chris Mason7b13b7b2008-04-18 10:29:50 -04004082 }
4083 return ret;
4084}
Chris Mason7b13b7b2008-04-18 10:29:50 -04004085
4086/*
Chris Masond1310b22008-01-24 16:13:08 -05004087 * a helper for releasepage. As long as there are no locked extents
4088 * in the range corresponding to the page, both state records and extent
4089 * map records are removed
4090 */
4091int try_release_extent_mapping(struct extent_map_tree *map,
Chris Mason70dec802008-01-29 09:59:12 -05004092 struct extent_io_tree *tree, struct page *page,
4093 gfp_t mask)
Chris Masond1310b22008-01-24 16:13:08 -05004094{
4095 struct extent_map *em;
Miao Xie4eee4fa2012-12-21 09:17:45 +00004096 u64 start = page_offset(page);
Chris Masond1310b22008-01-24 16:13:08 -05004097 u64 end = start + PAGE_CACHE_SIZE - 1;
Chris Mason7b13b7b2008-04-18 10:29:50 -04004098
Chris Mason70dec802008-01-29 09:59:12 -05004099 if ((mask & __GFP_WAIT) &&
4100 page->mapping->host->i_size > 16 * 1024 * 1024) {
Yan39b56372008-02-15 10:40:50 -05004101 u64 len;
Chris Mason70dec802008-01-29 09:59:12 -05004102 while (start <= end) {
Yan39b56372008-02-15 10:40:50 -05004103 len = end - start + 1;
Chris Mason890871b2009-09-02 16:24:52 -04004104 write_lock(&map->lock);
Yan39b56372008-02-15 10:40:50 -05004105 em = lookup_extent_mapping(map, start, len);
Tsutomu Itoh285190d2012-02-16 16:23:58 +09004106 if (!em) {
Chris Mason890871b2009-09-02 16:24:52 -04004107 write_unlock(&map->lock);
Chris Mason70dec802008-01-29 09:59:12 -05004108 break;
4109 }
Chris Mason7f3c74f2008-07-18 12:01:11 -04004110 if (test_bit(EXTENT_FLAG_PINNED, &em->flags) ||
4111 em->start != start) {
Chris Mason890871b2009-09-02 16:24:52 -04004112 write_unlock(&map->lock);
Chris Mason70dec802008-01-29 09:59:12 -05004113 free_extent_map(em);
4114 break;
4115 }
4116 if (!test_range_bit(tree, em->start,
4117 extent_map_end(em) - 1,
Chris Mason8b62b722009-09-02 16:53:46 -04004118 EXTENT_LOCKED | EXTENT_WRITEBACK,
Chris Mason9655d292009-09-02 15:22:30 -04004119 0, NULL)) {
Chris Mason70dec802008-01-29 09:59:12 -05004120 remove_extent_mapping(map, em);
4121 /* once for the rb tree */
4122 free_extent_map(em);
4123 }
4124 start = extent_map_end(em);
Chris Mason890871b2009-09-02 16:24:52 -04004125 write_unlock(&map->lock);
Chris Mason70dec802008-01-29 09:59:12 -05004126
4127 /* once for us */
Chris Masond1310b22008-01-24 16:13:08 -05004128 free_extent_map(em);
4129 }
Chris Masond1310b22008-01-24 16:13:08 -05004130 }
Chris Mason7b13b7b2008-04-18 10:29:50 -04004131 return try_release_extent_state(map, tree, page, mask);
Chris Masond1310b22008-01-24 16:13:08 -05004132}
Chris Masond1310b22008-01-24 16:13:08 -05004133
Chris Masonec29ed52011-02-23 16:23:20 -05004134/*
4135 * helper function for fiemap, which doesn't want to see any holes.
4136 * This maps until we find something past 'last'
4137 */
4138static struct extent_map *get_extent_skip_holes(struct inode *inode,
4139 u64 offset,
4140 u64 last,
4141 get_extent_t *get_extent)
4142{
4143 u64 sectorsize = BTRFS_I(inode)->root->sectorsize;
4144 struct extent_map *em;
4145 u64 len;
4146
4147 if (offset >= last)
4148 return NULL;
4149
Dulshani Gunawardhana67871252013-10-31 10:33:04 +05304150 while (1) {
Chris Masonec29ed52011-02-23 16:23:20 -05004151 len = last - offset;
4152 if (len == 0)
4153 break;
Qu Wenruofda28322013-02-26 08:10:22 +00004154 len = ALIGN(len, sectorsize);
Chris Masonec29ed52011-02-23 16:23:20 -05004155 em = get_extent(inode, NULL, 0, offset, len, 0);
David Sterbac7040052011-04-19 18:00:01 +02004156 if (IS_ERR_OR_NULL(em))
Chris Masonec29ed52011-02-23 16:23:20 -05004157 return em;
4158
4159 /* if this isn't a hole return it */
4160 if (!test_bit(EXTENT_FLAG_VACANCY, &em->flags) &&
4161 em->block_start != EXTENT_MAP_HOLE) {
4162 return em;
4163 }
4164
4165 /* this is a hole, advance to the next extent */
4166 offset = extent_map_end(em);
4167 free_extent_map(em);
4168 if (offset >= last)
4169 break;
4170 }
4171 return NULL;
4172}
4173
Liu Bofe09e162013-09-22 12:54:23 +08004174static noinline int count_ext_ref(u64 inum, u64 offset, u64 root_id, void *ctx)
4175{
4176 unsigned long cnt = *((unsigned long *)ctx);
4177
4178 cnt++;
4179 *((unsigned long *)ctx) = cnt;
4180
4181 /* Now we're sure that the extent is shared. */
4182 if (cnt > 1)
4183 return 1;
4184 return 0;
4185}
4186
Yehuda Sadeh1506fcc2009-01-21 14:39:14 -05004187int extent_fiemap(struct inode *inode, struct fiemap_extent_info *fieinfo,
4188 __u64 start, __u64 len, get_extent_t *get_extent)
4189{
Josef Bacik975f84f2010-11-23 19:36:57 +00004190 int ret = 0;
Yehuda Sadeh1506fcc2009-01-21 14:39:14 -05004191 u64 off = start;
4192 u64 max = start + len;
4193 u32 flags = 0;
Josef Bacik975f84f2010-11-23 19:36:57 +00004194 u32 found_type;
4195 u64 last;
Chris Masonec29ed52011-02-23 16:23:20 -05004196 u64 last_for_get_extent = 0;
Yehuda Sadeh1506fcc2009-01-21 14:39:14 -05004197 u64 disko = 0;
Chris Masonec29ed52011-02-23 16:23:20 -05004198 u64 isize = i_size_read(inode);
Josef Bacik975f84f2010-11-23 19:36:57 +00004199 struct btrfs_key found_key;
Yehuda Sadeh1506fcc2009-01-21 14:39:14 -05004200 struct extent_map *em = NULL;
Josef Bacik2ac55d42010-02-03 19:33:23 +00004201 struct extent_state *cached_state = NULL;
Josef Bacik975f84f2010-11-23 19:36:57 +00004202 struct btrfs_path *path;
Yehuda Sadeh1506fcc2009-01-21 14:39:14 -05004203 int end = 0;
Chris Masonec29ed52011-02-23 16:23:20 -05004204 u64 em_start = 0;
4205 u64 em_len = 0;
4206 u64 em_end = 0;
Yehuda Sadeh1506fcc2009-01-21 14:39:14 -05004207
4208 if (len == 0)
4209 return -EINVAL;
4210
Josef Bacik975f84f2010-11-23 19:36:57 +00004211 path = btrfs_alloc_path();
4212 if (!path)
4213 return -ENOMEM;
4214 path->leave_spinning = 1;
4215
Josef Bacik4d479cf2011-11-17 11:34:31 -05004216 start = ALIGN(start, BTRFS_I(inode)->root->sectorsize);
4217 len = ALIGN(len, BTRFS_I(inode)->root->sectorsize);
4218
Chris Masonec29ed52011-02-23 16:23:20 -05004219 /*
4220 * lookup the last file extent. We're not using i_size here
4221 * because there might be preallocation past i_size
4222 */
Josef Bacik975f84f2010-11-23 19:36:57 +00004223 ret = btrfs_lookup_file_extent(NULL, BTRFS_I(inode)->root,
Li Zefan33345d012011-04-20 10:31:50 +08004224 path, btrfs_ino(inode), -1, 0);
Josef Bacik975f84f2010-11-23 19:36:57 +00004225 if (ret < 0) {
4226 btrfs_free_path(path);
4227 return ret;
4228 }
4229 WARN_ON(!ret);
4230 path->slots[0]--;
Josef Bacik975f84f2010-11-23 19:36:57 +00004231 btrfs_item_key_to_cpu(path->nodes[0], &found_key, path->slots[0]);
4232 found_type = btrfs_key_type(&found_key);
4233
Chris Masonec29ed52011-02-23 16:23:20 -05004234 /* No extents, but there might be delalloc bits */
Li Zefan33345d012011-04-20 10:31:50 +08004235 if (found_key.objectid != btrfs_ino(inode) ||
Josef Bacik975f84f2010-11-23 19:36:57 +00004236 found_type != BTRFS_EXTENT_DATA_KEY) {
Chris Masonec29ed52011-02-23 16:23:20 -05004237 /* have to trust i_size as the end */
4238 last = (u64)-1;
4239 last_for_get_extent = isize;
4240 } else {
4241 /*
4242 * remember the start of the last extent. There are a
4243 * bunch of different factors that go into the length of the
4244 * extent, so its much less complex to remember where it started
4245 */
4246 last = found_key.offset;
4247 last_for_get_extent = last + 1;
Josef Bacik975f84f2010-11-23 19:36:57 +00004248 }
Liu Bofe09e162013-09-22 12:54:23 +08004249 btrfs_release_path(path);
Josef Bacik975f84f2010-11-23 19:36:57 +00004250
Chris Masonec29ed52011-02-23 16:23:20 -05004251 /*
4252 * we might have some extents allocated but more delalloc past those
4253 * extents. so, we trust isize unless the start of the last extent is
4254 * beyond isize
4255 */
4256 if (last < isize) {
4257 last = (u64)-1;
4258 last_for_get_extent = isize;
4259 }
4260
Liu Boa52f4cd2013-05-01 16:23:41 +00004261 lock_extent_bits(&BTRFS_I(inode)->io_tree, start, start + len - 1, 0,
Jeff Mahoneyd0082372012-03-01 14:57:19 +01004262 &cached_state);
Chris Masonec29ed52011-02-23 16:23:20 -05004263
Josef Bacik4d479cf2011-11-17 11:34:31 -05004264 em = get_extent_skip_holes(inode, start, last_for_get_extent,
Chris Masonec29ed52011-02-23 16:23:20 -05004265 get_extent);
Yehuda Sadeh1506fcc2009-01-21 14:39:14 -05004266 if (!em)
4267 goto out;
4268 if (IS_ERR(em)) {
4269 ret = PTR_ERR(em);
4270 goto out;
4271 }
Josef Bacik975f84f2010-11-23 19:36:57 +00004272
Yehuda Sadeh1506fcc2009-01-21 14:39:14 -05004273 while (!end) {
Josef Bacikb76bb702013-07-05 13:52:51 -04004274 u64 offset_in_extent = 0;
Yehuda Sadeh1506fcc2009-01-21 14:39:14 -05004275
Chris Masonea8efc72011-03-08 11:54:40 -05004276 /* break if the extent we found is outside the range */
4277 if (em->start >= max || extent_map_end(em) < off)
4278 break;
4279
4280 /*
4281 * get_extent may return an extent that starts before our
4282 * requested range. We have to make sure the ranges
4283 * we return to fiemap always move forward and don't
4284 * overlap, so adjust the offsets here
4285 */
4286 em_start = max(em->start, off);
4287
4288 /*
4289 * record the offset from the start of the extent
Josef Bacikb76bb702013-07-05 13:52:51 -04004290 * for adjusting the disk offset below. Only do this if the
4291 * extent isn't compressed since our in ram offset may be past
4292 * what we have actually allocated on disk.
Chris Masonea8efc72011-03-08 11:54:40 -05004293 */
Josef Bacikb76bb702013-07-05 13:52:51 -04004294 if (!test_bit(EXTENT_FLAG_COMPRESSED, &em->flags))
4295 offset_in_extent = em_start - em->start;
Chris Masonec29ed52011-02-23 16:23:20 -05004296 em_end = extent_map_end(em);
Chris Masonea8efc72011-03-08 11:54:40 -05004297 em_len = em_end - em_start;
Yehuda Sadeh1506fcc2009-01-21 14:39:14 -05004298 disko = 0;
4299 flags = 0;
4300
Chris Masonea8efc72011-03-08 11:54:40 -05004301 /*
4302 * bump off for our next call to get_extent
4303 */
4304 off = extent_map_end(em);
4305 if (off >= max)
4306 end = 1;
4307
Heiko Carstens93dbfad2009-04-03 10:33:45 -04004308 if (em->block_start == EXTENT_MAP_LAST_BYTE) {
Yehuda Sadeh1506fcc2009-01-21 14:39:14 -05004309 end = 1;
4310 flags |= FIEMAP_EXTENT_LAST;
Heiko Carstens93dbfad2009-04-03 10:33:45 -04004311 } else if (em->block_start == EXTENT_MAP_INLINE) {
Yehuda Sadeh1506fcc2009-01-21 14:39:14 -05004312 flags |= (FIEMAP_EXTENT_DATA_INLINE |
4313 FIEMAP_EXTENT_NOT_ALIGNED);
Heiko Carstens93dbfad2009-04-03 10:33:45 -04004314 } else if (em->block_start == EXTENT_MAP_DELALLOC) {
Yehuda Sadeh1506fcc2009-01-21 14:39:14 -05004315 flags |= (FIEMAP_EXTENT_DELALLOC |
4316 FIEMAP_EXTENT_UNKNOWN);
Heiko Carstens93dbfad2009-04-03 10:33:45 -04004317 } else {
Liu Bofe09e162013-09-22 12:54:23 +08004318 unsigned long ref_cnt = 0;
4319
Chris Masonea8efc72011-03-08 11:54:40 -05004320 disko = em->block_start + offset_in_extent;
Liu Bofe09e162013-09-22 12:54:23 +08004321
4322 /*
4323 * As btrfs supports shared space, this information
4324 * can be exported to userspace tools via
4325 * flag FIEMAP_EXTENT_SHARED.
4326 */
4327 ret = iterate_inodes_from_logical(
4328 em->block_start,
4329 BTRFS_I(inode)->root->fs_info,
4330 path, count_ext_ref, &ref_cnt);
4331 if (ret < 0 && ret != -ENOENT)
4332 goto out_free;
4333
4334 if (ref_cnt > 1)
4335 flags |= FIEMAP_EXTENT_SHARED;
Yehuda Sadeh1506fcc2009-01-21 14:39:14 -05004336 }
4337 if (test_bit(EXTENT_FLAG_COMPRESSED, &em->flags))
4338 flags |= FIEMAP_EXTENT_ENCODED;
4339
Yehuda Sadeh1506fcc2009-01-21 14:39:14 -05004340 free_extent_map(em);
4341 em = NULL;
Chris Masonec29ed52011-02-23 16:23:20 -05004342 if ((em_start >= last) || em_len == (u64)-1 ||
4343 (last == (u64)-1 && isize <= em_end)) {
Yehuda Sadeh1506fcc2009-01-21 14:39:14 -05004344 flags |= FIEMAP_EXTENT_LAST;
4345 end = 1;
4346 }
4347
Chris Masonec29ed52011-02-23 16:23:20 -05004348 /* now scan forward to see if this is really the last extent. */
4349 em = get_extent_skip_holes(inode, off, last_for_get_extent,
4350 get_extent);
4351 if (IS_ERR(em)) {
4352 ret = PTR_ERR(em);
4353 goto out;
4354 }
4355 if (!em) {
Josef Bacik975f84f2010-11-23 19:36:57 +00004356 flags |= FIEMAP_EXTENT_LAST;
4357 end = 1;
4358 }
Chris Masonec29ed52011-02-23 16:23:20 -05004359 ret = fiemap_fill_next_extent(fieinfo, em_start, disko,
4360 em_len, flags);
4361 if (ret)
4362 goto out_free;
Yehuda Sadeh1506fcc2009-01-21 14:39:14 -05004363 }
4364out_free:
4365 free_extent_map(em);
4366out:
Liu Bofe09e162013-09-22 12:54:23 +08004367 btrfs_free_path(path);
Liu Boa52f4cd2013-05-01 16:23:41 +00004368 unlock_extent_cached(&BTRFS_I(inode)->io_tree, start, start + len - 1,
Josef Bacik2ac55d42010-02-03 19:33:23 +00004369 &cached_state, GFP_NOFS);
Yehuda Sadeh1506fcc2009-01-21 14:39:14 -05004370 return ret;
4371}
4372
Chris Mason727011e2010-08-06 13:21:20 -04004373static void __free_extent_buffer(struct extent_buffer *eb)
4374{
Eric Sandeen6d49ba12013-04-22 16:12:31 +00004375 btrfs_leak_debug_del(&eb->leak_list);
Chris Mason727011e2010-08-06 13:21:20 -04004376 kmem_cache_free(extent_buffer_cache, eb);
4377}
4378
Josef Bacika26e8c92014-03-28 17:07:27 -04004379int extent_buffer_under_io(struct extent_buffer *eb)
Chris Masond1310b22008-01-24 16:13:08 -05004380{
Josef Bacik0b32f4b2012-03-13 09:38:00 -04004381 return (atomic_read(&eb->io_pages) ||
4382 test_bit(EXTENT_BUFFER_WRITEBACK, &eb->bflags) ||
4383 test_bit(EXTENT_BUFFER_DIRTY, &eb->bflags));
Chris Masond1310b22008-01-24 16:13:08 -05004384}
4385
Miao Xie897ca6e2010-10-26 20:57:29 -04004386/*
4387 * Helper for releasing extent buffer page.
4388 */
4389static void btrfs_release_extent_buffer_page(struct extent_buffer *eb,
4390 unsigned long start_idx)
4391{
4392 unsigned long index;
Wang Sheng-Hui39bab872012-04-06 14:35:31 +08004393 unsigned long num_pages;
Miao Xie897ca6e2010-10-26 20:57:29 -04004394 struct page *page;
Jan Schmidt815a51c2012-05-16 17:00:02 +02004395 int mapped = !test_bit(EXTENT_BUFFER_DUMMY, &eb->bflags);
Miao Xie897ca6e2010-10-26 20:57:29 -04004396
Josef Bacik0b32f4b2012-03-13 09:38:00 -04004397 BUG_ON(extent_buffer_under_io(eb));
Miao Xie897ca6e2010-10-26 20:57:29 -04004398
Wang Sheng-Hui39bab872012-04-06 14:35:31 +08004399 num_pages = num_extent_pages(eb->start, eb->len);
4400 index = start_idx + num_pages;
Miao Xie897ca6e2010-10-26 20:57:29 -04004401 if (start_idx >= index)
4402 return;
4403
4404 do {
4405 index--;
4406 page = extent_buffer_page(eb, index);
Jan Schmidt815a51c2012-05-16 17:00:02 +02004407 if (page && mapped) {
Josef Bacik4f2de97a2012-03-07 16:20:05 -05004408 spin_lock(&page->mapping->private_lock);
4409 /*
4410 * We do this since we'll remove the pages after we've
4411 * removed the eb from the radix tree, so we could race
4412 * and have this page now attached to the new eb. So
4413 * only clear page_private if it's still connected to
4414 * this eb.
4415 */
4416 if (PagePrivate(page) &&
4417 page->private == (unsigned long)eb) {
Josef Bacik0b32f4b2012-03-13 09:38:00 -04004418 BUG_ON(test_bit(EXTENT_BUFFER_DIRTY, &eb->bflags));
Josef Bacik3083ee22012-03-09 16:01:49 -05004419 BUG_ON(PageDirty(page));
4420 BUG_ON(PageWriteback(page));
Josef Bacik4f2de97a2012-03-07 16:20:05 -05004421 /*
4422 * We need to make sure we haven't be attached
4423 * to a new eb.
4424 */
4425 ClearPagePrivate(page);
4426 set_page_private(page, 0);
4427 /* One for the page private */
4428 page_cache_release(page);
4429 }
4430 spin_unlock(&page->mapping->private_lock);
4431
Jan Schmidt815a51c2012-05-16 17:00:02 +02004432 }
4433 if (page) {
Josef Bacik4f2de97a2012-03-07 16:20:05 -05004434 /* One for when we alloced the page */
Miao Xie897ca6e2010-10-26 20:57:29 -04004435 page_cache_release(page);
Josef Bacik4f2de97a2012-03-07 16:20:05 -05004436 }
Miao Xie897ca6e2010-10-26 20:57:29 -04004437 } while (index != start_idx);
4438}
4439
4440/*
4441 * Helper for releasing the extent buffer.
4442 */
4443static inline void btrfs_release_extent_buffer(struct extent_buffer *eb)
4444{
4445 btrfs_release_extent_buffer_page(eb, 0);
4446 __free_extent_buffer(eb);
4447}
4448
Josef Bacikf28491e2013-12-16 13:24:27 -05004449static struct extent_buffer *
4450__alloc_extent_buffer(struct btrfs_fs_info *fs_info, u64 start,
4451 unsigned long len, gfp_t mask)
Josef Bacikdb7f3432013-08-07 14:54:37 -04004452{
4453 struct extent_buffer *eb = NULL;
4454
4455 eb = kmem_cache_zalloc(extent_buffer_cache, mask);
4456 if (eb == NULL)
4457 return NULL;
4458 eb->start = start;
4459 eb->len = len;
Josef Bacikf28491e2013-12-16 13:24:27 -05004460 eb->fs_info = fs_info;
Josef Bacikdb7f3432013-08-07 14:54:37 -04004461 eb->bflags = 0;
4462 rwlock_init(&eb->lock);
4463 atomic_set(&eb->write_locks, 0);
4464 atomic_set(&eb->read_locks, 0);
4465 atomic_set(&eb->blocking_readers, 0);
4466 atomic_set(&eb->blocking_writers, 0);
4467 atomic_set(&eb->spinning_readers, 0);
4468 atomic_set(&eb->spinning_writers, 0);
4469 eb->lock_nested = 0;
4470 init_waitqueue_head(&eb->write_lock_wq);
4471 init_waitqueue_head(&eb->read_lock_wq);
4472
4473 btrfs_leak_debug_add(&eb->leak_list, &buffers);
4474
4475 spin_lock_init(&eb->refs_lock);
4476 atomic_set(&eb->refs, 1);
4477 atomic_set(&eb->io_pages, 0);
4478
4479 /*
4480 * Sanity checks, currently the maximum is 64k covered by 16x 4k pages
4481 */
4482 BUILD_BUG_ON(BTRFS_MAX_METADATA_BLOCKSIZE
4483 > MAX_INLINE_EXTENT_BUFFER_SIZE);
4484 BUG_ON(len > MAX_INLINE_EXTENT_BUFFER_SIZE);
4485
4486 return eb;
4487}
4488
4489struct extent_buffer *btrfs_clone_extent_buffer(struct extent_buffer *src)
4490{
4491 unsigned long i;
4492 struct page *p;
4493 struct extent_buffer *new;
4494 unsigned long num_pages = num_extent_pages(src->start, src->len);
4495
Josef Bacik9ec72672013-08-07 16:57:23 -04004496 new = __alloc_extent_buffer(NULL, src->start, src->len, GFP_NOFS);
Josef Bacikdb7f3432013-08-07 14:54:37 -04004497 if (new == NULL)
4498 return NULL;
4499
4500 for (i = 0; i < num_pages; i++) {
Josef Bacik9ec72672013-08-07 16:57:23 -04004501 p = alloc_page(GFP_NOFS);
Josef Bacikdb7f3432013-08-07 14:54:37 -04004502 if (!p) {
4503 btrfs_release_extent_buffer(new);
4504 return NULL;
4505 }
4506 attach_extent_buffer_page(new, p);
4507 WARN_ON(PageDirty(p));
4508 SetPageUptodate(p);
4509 new->pages[i] = p;
4510 }
4511
4512 copy_extent_buffer(new, src, 0, 0, src->len);
4513 set_bit(EXTENT_BUFFER_UPTODATE, &new->bflags);
4514 set_bit(EXTENT_BUFFER_DUMMY, &new->bflags);
4515
4516 return new;
4517}
4518
4519struct extent_buffer *alloc_dummy_extent_buffer(u64 start, unsigned long len)
4520{
4521 struct extent_buffer *eb;
4522 unsigned long num_pages = num_extent_pages(0, len);
4523 unsigned long i;
4524
Josef Bacik9ec72672013-08-07 16:57:23 -04004525 eb = __alloc_extent_buffer(NULL, start, len, GFP_NOFS);
Josef Bacikdb7f3432013-08-07 14:54:37 -04004526 if (!eb)
4527 return NULL;
4528
4529 for (i = 0; i < num_pages; i++) {
Josef Bacik9ec72672013-08-07 16:57:23 -04004530 eb->pages[i] = alloc_page(GFP_NOFS);
Josef Bacikdb7f3432013-08-07 14:54:37 -04004531 if (!eb->pages[i])
4532 goto err;
4533 }
4534 set_extent_buffer_uptodate(eb);
4535 btrfs_set_header_nritems(eb, 0);
4536 set_bit(EXTENT_BUFFER_DUMMY, &eb->bflags);
4537
4538 return eb;
4539err:
4540 for (; i > 0; i--)
4541 __free_page(eb->pages[i - 1]);
4542 __free_extent_buffer(eb);
4543 return NULL;
4544}
4545
Josef Bacik0b32f4b2012-03-13 09:38:00 -04004546static void check_buffer_tree_ref(struct extent_buffer *eb)
4547{
Chris Mason242e18c2013-01-29 17:49:37 -05004548 int refs;
Josef Bacik0b32f4b2012-03-13 09:38:00 -04004549 /* the ref bit is tricky. We have to make sure it is set
4550 * if we have the buffer dirty. Otherwise the
4551 * code to free a buffer can end up dropping a dirty
4552 * page
4553 *
4554 * Once the ref bit is set, it won't go away while the
4555 * buffer is dirty or in writeback, and it also won't
4556 * go away while we have the reference count on the
4557 * eb bumped.
4558 *
4559 * We can't just set the ref bit without bumping the
4560 * ref on the eb because free_extent_buffer might
4561 * see the ref bit and try to clear it. If this happens
4562 * free_extent_buffer might end up dropping our original
4563 * ref by mistake and freeing the page before we are able
4564 * to add one more ref.
4565 *
4566 * So bump the ref count first, then set the bit. If someone
4567 * beat us to it, drop the ref we added.
4568 */
Chris Mason242e18c2013-01-29 17:49:37 -05004569 refs = atomic_read(&eb->refs);
4570 if (refs >= 2 && test_bit(EXTENT_BUFFER_TREE_REF, &eb->bflags))
4571 return;
4572
Josef Bacik594831c2012-07-20 16:11:08 -04004573 spin_lock(&eb->refs_lock);
4574 if (!test_and_set_bit(EXTENT_BUFFER_TREE_REF, &eb->bflags))
Josef Bacik0b32f4b2012-03-13 09:38:00 -04004575 atomic_inc(&eb->refs);
Josef Bacik594831c2012-07-20 16:11:08 -04004576 spin_unlock(&eb->refs_lock);
Josef Bacik0b32f4b2012-03-13 09:38:00 -04004577}
4578
Mel Gorman2457aec2014-06-04 16:10:31 -07004579static void mark_extent_buffer_accessed(struct extent_buffer *eb,
4580 struct page *accessed)
Josef Bacik5df42352012-03-15 18:24:42 -04004581{
4582 unsigned long num_pages, i;
4583
Josef Bacik0b32f4b2012-03-13 09:38:00 -04004584 check_buffer_tree_ref(eb);
4585
Josef Bacik5df42352012-03-15 18:24:42 -04004586 num_pages = num_extent_pages(eb->start, eb->len);
4587 for (i = 0; i < num_pages; i++) {
4588 struct page *p = extent_buffer_page(eb, i);
Mel Gorman2457aec2014-06-04 16:10:31 -07004589 if (p != accessed)
4590 mark_page_accessed(p);
Josef Bacik5df42352012-03-15 18:24:42 -04004591 }
4592}
4593
Josef Bacikf28491e2013-12-16 13:24:27 -05004594struct extent_buffer *find_extent_buffer(struct btrfs_fs_info *fs_info,
4595 u64 start)
Chandra Seetharaman452c75c2013-10-07 10:45:25 -05004596{
4597 struct extent_buffer *eb;
4598
4599 rcu_read_lock();
Josef Bacikf28491e2013-12-16 13:24:27 -05004600 eb = radix_tree_lookup(&fs_info->buffer_radix,
4601 start >> PAGE_CACHE_SHIFT);
Chandra Seetharaman452c75c2013-10-07 10:45:25 -05004602 if (eb && atomic_inc_not_zero(&eb->refs)) {
4603 rcu_read_unlock();
Mel Gorman2457aec2014-06-04 16:10:31 -07004604 mark_extent_buffer_accessed(eb, NULL);
Chandra Seetharaman452c75c2013-10-07 10:45:25 -05004605 return eb;
4606 }
4607 rcu_read_unlock();
4608
4609 return NULL;
4610}
4611
Josef Bacikfaa2dbf2014-05-07 17:06:09 -04004612#ifdef CONFIG_BTRFS_FS_RUN_SANITY_TESTS
4613struct extent_buffer *alloc_test_extent_buffer(struct btrfs_fs_info *fs_info,
4614 u64 start, unsigned long len)
4615{
4616 struct extent_buffer *eb, *exists = NULL;
4617 int ret;
4618
4619 eb = find_extent_buffer(fs_info, start);
4620 if (eb)
4621 return eb;
4622 eb = alloc_dummy_extent_buffer(start, len);
4623 if (!eb)
4624 return NULL;
4625 eb->fs_info = fs_info;
4626again:
4627 ret = radix_tree_preload(GFP_NOFS & ~__GFP_HIGHMEM);
4628 if (ret)
4629 goto free_eb;
4630 spin_lock(&fs_info->buffer_lock);
4631 ret = radix_tree_insert(&fs_info->buffer_radix,
4632 start >> PAGE_CACHE_SHIFT, eb);
4633 spin_unlock(&fs_info->buffer_lock);
4634 radix_tree_preload_end();
4635 if (ret == -EEXIST) {
4636 exists = find_extent_buffer(fs_info, start);
4637 if (exists)
4638 goto free_eb;
4639 else
4640 goto again;
4641 }
4642 check_buffer_tree_ref(eb);
4643 set_bit(EXTENT_BUFFER_IN_TREE, &eb->bflags);
4644
4645 /*
4646 * We will free dummy extent buffer's if they come into
4647 * free_extent_buffer with a ref count of 2, but if we are using this we
4648 * want the buffers to stay in memory until we're done with them, so
4649 * bump the ref count again.
4650 */
4651 atomic_inc(&eb->refs);
4652 return eb;
4653free_eb:
4654 btrfs_release_extent_buffer(eb);
4655 return exists;
4656}
4657#endif
4658
Josef Bacikf28491e2013-12-16 13:24:27 -05004659struct extent_buffer *alloc_extent_buffer(struct btrfs_fs_info *fs_info,
Chris Mason727011e2010-08-06 13:21:20 -04004660 u64 start, unsigned long len)
Chris Masond1310b22008-01-24 16:13:08 -05004661{
4662 unsigned long num_pages = num_extent_pages(start, len);
4663 unsigned long i;
4664 unsigned long index = start >> PAGE_CACHE_SHIFT;
4665 struct extent_buffer *eb;
Chris Mason6af118ce2008-07-22 11:18:07 -04004666 struct extent_buffer *exists = NULL;
Chris Masond1310b22008-01-24 16:13:08 -05004667 struct page *p;
Josef Bacikf28491e2013-12-16 13:24:27 -05004668 struct address_space *mapping = fs_info->btree_inode->i_mapping;
Chris Masond1310b22008-01-24 16:13:08 -05004669 int uptodate = 1;
Miao Xie19fe0a82010-10-26 20:57:29 -04004670 int ret;
Chris Masond1310b22008-01-24 16:13:08 -05004671
Josef Bacikf28491e2013-12-16 13:24:27 -05004672 eb = find_extent_buffer(fs_info, start);
Chandra Seetharaman452c75c2013-10-07 10:45:25 -05004673 if (eb)
Chris Mason6af118ce2008-07-22 11:18:07 -04004674 return eb;
Chris Mason6af118ce2008-07-22 11:18:07 -04004675
Josef Bacikf28491e2013-12-16 13:24:27 -05004676 eb = __alloc_extent_buffer(fs_info, start, len, GFP_NOFS);
Peter2b114d12008-04-01 11:21:40 -04004677 if (!eb)
Chris Masond1310b22008-01-24 16:13:08 -05004678 return NULL;
4679
Chris Mason727011e2010-08-06 13:21:20 -04004680 for (i = 0; i < num_pages; i++, index++) {
Chris Masona6591712011-07-19 12:04:14 -04004681 p = find_or_create_page(mapping, index, GFP_NOFS);
Josef Bacik4804b382012-10-05 16:43:45 -04004682 if (!p)
Chris Mason6af118ce2008-07-22 11:18:07 -04004683 goto free_eb;
Josef Bacik4f2de97a2012-03-07 16:20:05 -05004684
4685 spin_lock(&mapping->private_lock);
4686 if (PagePrivate(p)) {
4687 /*
4688 * We could have already allocated an eb for this page
4689 * and attached one so lets see if we can get a ref on
4690 * the existing eb, and if we can we know it's good and
4691 * we can just return that one, else we know we can just
4692 * overwrite page->private.
4693 */
4694 exists = (struct extent_buffer *)p->private;
4695 if (atomic_inc_not_zero(&exists->refs)) {
4696 spin_unlock(&mapping->private_lock);
4697 unlock_page(p);
Josef Bacik17de39a2012-05-04 15:16:06 -04004698 page_cache_release(p);
Mel Gorman2457aec2014-06-04 16:10:31 -07004699 mark_extent_buffer_accessed(exists, p);
Josef Bacik4f2de97a2012-03-07 16:20:05 -05004700 goto free_eb;
4701 }
4702
Josef Bacik0b32f4b2012-03-13 09:38:00 -04004703 /*
Josef Bacik4f2de97a2012-03-07 16:20:05 -05004704 * Do this so attach doesn't complain and we need to
4705 * drop the ref the old guy had.
4706 */
4707 ClearPagePrivate(p);
Josef Bacik0b32f4b2012-03-13 09:38:00 -04004708 WARN_ON(PageDirty(p));
Josef Bacik4f2de97a2012-03-07 16:20:05 -05004709 page_cache_release(p);
Chris Masond1310b22008-01-24 16:13:08 -05004710 }
Josef Bacik4f2de97a2012-03-07 16:20:05 -05004711 attach_extent_buffer_page(eb, p);
4712 spin_unlock(&mapping->private_lock);
Josef Bacik0b32f4b2012-03-13 09:38:00 -04004713 WARN_ON(PageDirty(p));
Chris Mason727011e2010-08-06 13:21:20 -04004714 eb->pages[i] = p;
Chris Masond1310b22008-01-24 16:13:08 -05004715 if (!PageUptodate(p))
4716 uptodate = 0;
Chris Masoneb14ab82011-02-10 12:35:00 -05004717
4718 /*
4719 * see below about how we avoid a nasty race with release page
4720 * and why we unlock later
4721 */
Chris Masond1310b22008-01-24 16:13:08 -05004722 }
4723 if (uptodate)
Chris Masonb4ce94d2009-02-04 09:25:08 -05004724 set_bit(EXTENT_BUFFER_UPTODATE, &eb->bflags);
Josef Bacik115391d2012-03-09 09:51:43 -05004725again:
Miao Xie19fe0a82010-10-26 20:57:29 -04004726 ret = radix_tree_preload(GFP_NOFS & ~__GFP_HIGHMEM);
4727 if (ret)
4728 goto free_eb;
4729
Josef Bacikf28491e2013-12-16 13:24:27 -05004730 spin_lock(&fs_info->buffer_lock);
4731 ret = radix_tree_insert(&fs_info->buffer_radix,
4732 start >> PAGE_CACHE_SHIFT, eb);
4733 spin_unlock(&fs_info->buffer_lock);
Chandra Seetharaman452c75c2013-10-07 10:45:25 -05004734 radix_tree_preload_end();
Miao Xie19fe0a82010-10-26 20:57:29 -04004735 if (ret == -EEXIST) {
Josef Bacikf28491e2013-12-16 13:24:27 -05004736 exists = find_extent_buffer(fs_info, start);
Chandra Seetharaman452c75c2013-10-07 10:45:25 -05004737 if (exists)
4738 goto free_eb;
4739 else
Josef Bacik115391d2012-03-09 09:51:43 -05004740 goto again;
Chris Mason6af118ce2008-07-22 11:18:07 -04004741 }
Chris Mason6af118ce2008-07-22 11:18:07 -04004742 /* add one reference for the tree */
Josef Bacik0b32f4b2012-03-13 09:38:00 -04004743 check_buffer_tree_ref(eb);
Josef Bacik34b41ac2013-12-13 10:41:51 -05004744 set_bit(EXTENT_BUFFER_IN_TREE, &eb->bflags);
Chris Masoneb14ab82011-02-10 12:35:00 -05004745
4746 /*
4747 * there is a race where release page may have
4748 * tried to find this extent buffer in the radix
4749 * but failed. It will tell the VM it is safe to
4750 * reclaim the, and it will clear the page private bit.
4751 * We must make sure to set the page private bit properly
4752 * after the extent buffer is in the radix tree so
4753 * it doesn't get lost
4754 */
Chris Mason727011e2010-08-06 13:21:20 -04004755 SetPageChecked(eb->pages[0]);
4756 for (i = 1; i < num_pages; i++) {
4757 p = extent_buffer_page(eb, i);
Chris Mason727011e2010-08-06 13:21:20 -04004758 ClearPageChecked(p);
4759 unlock_page(p);
4760 }
4761 unlock_page(eb->pages[0]);
Chris Masond1310b22008-01-24 16:13:08 -05004762 return eb;
4763
Chris Mason6af118ce2008-07-22 11:18:07 -04004764free_eb:
Chris Mason727011e2010-08-06 13:21:20 -04004765 for (i = 0; i < num_pages; i++) {
4766 if (eb->pages[i])
4767 unlock_page(eb->pages[i]);
4768 }
Chris Masoneb14ab82011-02-10 12:35:00 -05004769
Josef Bacik17de39a2012-05-04 15:16:06 -04004770 WARN_ON(!atomic_dec_and_test(&eb->refs));
Miao Xie897ca6e2010-10-26 20:57:29 -04004771 btrfs_release_extent_buffer(eb);
Chris Mason6af118ce2008-07-22 11:18:07 -04004772 return exists;
Chris Masond1310b22008-01-24 16:13:08 -05004773}
Chris Masond1310b22008-01-24 16:13:08 -05004774
Josef Bacik3083ee22012-03-09 16:01:49 -05004775static inline void btrfs_release_extent_buffer_rcu(struct rcu_head *head)
4776{
4777 struct extent_buffer *eb =
4778 container_of(head, struct extent_buffer, rcu_head);
4779
4780 __free_extent_buffer(eb);
4781}
4782
Josef Bacik3083ee22012-03-09 16:01:49 -05004783/* Expects to have eb->eb_lock already held */
David Sterbaf7a52a42013-04-26 14:56:29 +00004784static int release_extent_buffer(struct extent_buffer *eb)
Josef Bacik3083ee22012-03-09 16:01:49 -05004785{
4786 WARN_ON(atomic_read(&eb->refs) == 0);
4787 if (atomic_dec_and_test(&eb->refs)) {
Josef Bacik34b41ac2013-12-13 10:41:51 -05004788 if (test_and_clear_bit(EXTENT_BUFFER_IN_TREE, &eb->bflags)) {
Josef Bacikf28491e2013-12-16 13:24:27 -05004789 struct btrfs_fs_info *fs_info = eb->fs_info;
Josef Bacik3083ee22012-03-09 16:01:49 -05004790
Jan Schmidt815a51c2012-05-16 17:00:02 +02004791 spin_unlock(&eb->refs_lock);
Josef Bacik3083ee22012-03-09 16:01:49 -05004792
Josef Bacikf28491e2013-12-16 13:24:27 -05004793 spin_lock(&fs_info->buffer_lock);
4794 radix_tree_delete(&fs_info->buffer_radix,
Jan Schmidt815a51c2012-05-16 17:00:02 +02004795 eb->start >> PAGE_CACHE_SHIFT);
Josef Bacikf28491e2013-12-16 13:24:27 -05004796 spin_unlock(&fs_info->buffer_lock);
Josef Bacik34b41ac2013-12-13 10:41:51 -05004797 } else {
4798 spin_unlock(&eb->refs_lock);
Jan Schmidt815a51c2012-05-16 17:00:02 +02004799 }
Josef Bacik3083ee22012-03-09 16:01:49 -05004800
4801 /* Should be safe to release our pages at this point */
4802 btrfs_release_extent_buffer_page(eb, 0);
Josef Bacik3083ee22012-03-09 16:01:49 -05004803 call_rcu(&eb->rcu_head, btrfs_release_extent_buffer_rcu);
Josef Bacike64860a2012-07-20 16:05:36 -04004804 return 1;
Josef Bacik3083ee22012-03-09 16:01:49 -05004805 }
4806 spin_unlock(&eb->refs_lock);
Josef Bacike64860a2012-07-20 16:05:36 -04004807
4808 return 0;
Josef Bacik3083ee22012-03-09 16:01:49 -05004809}
4810
Chris Masond1310b22008-01-24 16:13:08 -05004811void free_extent_buffer(struct extent_buffer *eb)
4812{
Chris Mason242e18c2013-01-29 17:49:37 -05004813 int refs;
4814 int old;
Chris Masond1310b22008-01-24 16:13:08 -05004815 if (!eb)
4816 return;
4817
Chris Mason242e18c2013-01-29 17:49:37 -05004818 while (1) {
4819 refs = atomic_read(&eb->refs);
4820 if (refs <= 3)
4821 break;
4822 old = atomic_cmpxchg(&eb->refs, refs, refs - 1);
4823 if (old == refs)
4824 return;
4825 }
4826
Josef Bacik3083ee22012-03-09 16:01:49 -05004827 spin_lock(&eb->refs_lock);
4828 if (atomic_read(&eb->refs) == 2 &&
Jan Schmidt815a51c2012-05-16 17:00:02 +02004829 test_bit(EXTENT_BUFFER_DUMMY, &eb->bflags))
4830 atomic_dec(&eb->refs);
4831
4832 if (atomic_read(&eb->refs) == 2 &&
Josef Bacik3083ee22012-03-09 16:01:49 -05004833 test_bit(EXTENT_BUFFER_STALE, &eb->bflags) &&
Josef Bacik0b32f4b2012-03-13 09:38:00 -04004834 !extent_buffer_under_io(eb) &&
Josef Bacik3083ee22012-03-09 16:01:49 -05004835 test_and_clear_bit(EXTENT_BUFFER_TREE_REF, &eb->bflags))
4836 atomic_dec(&eb->refs);
Chris Masond1310b22008-01-24 16:13:08 -05004837
Josef Bacik3083ee22012-03-09 16:01:49 -05004838 /*
4839 * I know this is terrible, but it's temporary until we stop tracking
4840 * the uptodate bits and such for the extent buffers.
4841 */
David Sterbaf7a52a42013-04-26 14:56:29 +00004842 release_extent_buffer(eb);
Chris Masond1310b22008-01-24 16:13:08 -05004843}
Chris Masond1310b22008-01-24 16:13:08 -05004844
Josef Bacik3083ee22012-03-09 16:01:49 -05004845void free_extent_buffer_stale(struct extent_buffer *eb)
4846{
4847 if (!eb)
Chris Masond1310b22008-01-24 16:13:08 -05004848 return;
4849
Josef Bacik3083ee22012-03-09 16:01:49 -05004850 spin_lock(&eb->refs_lock);
4851 set_bit(EXTENT_BUFFER_STALE, &eb->bflags);
4852
Josef Bacik0b32f4b2012-03-13 09:38:00 -04004853 if (atomic_read(&eb->refs) == 2 && !extent_buffer_under_io(eb) &&
Josef Bacik3083ee22012-03-09 16:01:49 -05004854 test_and_clear_bit(EXTENT_BUFFER_TREE_REF, &eb->bflags))
4855 atomic_dec(&eb->refs);
David Sterbaf7a52a42013-04-26 14:56:29 +00004856 release_extent_buffer(eb);
Chris Masond1310b22008-01-24 16:13:08 -05004857}
4858
Chris Mason1d4284b2012-03-28 20:31:37 -04004859void clear_extent_buffer_dirty(struct extent_buffer *eb)
Chris Masond1310b22008-01-24 16:13:08 -05004860{
Chris Masond1310b22008-01-24 16:13:08 -05004861 unsigned long i;
4862 unsigned long num_pages;
4863 struct page *page;
4864
Chris Masond1310b22008-01-24 16:13:08 -05004865 num_pages = num_extent_pages(eb->start, eb->len);
4866
4867 for (i = 0; i < num_pages; i++) {
4868 page = extent_buffer_page(eb, i);
Chris Masonb9473432009-03-13 11:00:37 -04004869 if (!PageDirty(page))
Chris Masond2c3f4f2008-11-19 12:44:22 -05004870 continue;
4871
Chris Masona61e6f22008-07-22 11:18:08 -04004872 lock_page(page);
Chris Masoneb14ab82011-02-10 12:35:00 -05004873 WARN_ON(!PagePrivate(page));
4874
Chris Masond1310b22008-01-24 16:13:08 -05004875 clear_page_dirty_for_io(page);
Sven Wegener0ee0fda2008-07-30 16:54:26 -04004876 spin_lock_irq(&page->mapping->tree_lock);
Chris Masond1310b22008-01-24 16:13:08 -05004877 if (!PageDirty(page)) {
4878 radix_tree_tag_clear(&page->mapping->page_tree,
4879 page_index(page),
4880 PAGECACHE_TAG_DIRTY);
4881 }
Sven Wegener0ee0fda2008-07-30 16:54:26 -04004882 spin_unlock_irq(&page->mapping->tree_lock);
Chris Masonbf0da8c2011-11-04 12:29:37 -04004883 ClearPageError(page);
Chris Masona61e6f22008-07-22 11:18:08 -04004884 unlock_page(page);
Chris Masond1310b22008-01-24 16:13:08 -05004885 }
Josef Bacik0b32f4b2012-03-13 09:38:00 -04004886 WARN_ON(atomic_read(&eb->refs) == 0);
Chris Masond1310b22008-01-24 16:13:08 -05004887}
Chris Masond1310b22008-01-24 16:13:08 -05004888
Josef Bacik0b32f4b2012-03-13 09:38:00 -04004889int set_extent_buffer_dirty(struct extent_buffer *eb)
Chris Masond1310b22008-01-24 16:13:08 -05004890{
4891 unsigned long i;
4892 unsigned long num_pages;
Chris Masonb9473432009-03-13 11:00:37 -04004893 int was_dirty = 0;
Chris Masond1310b22008-01-24 16:13:08 -05004894
Josef Bacik0b32f4b2012-03-13 09:38:00 -04004895 check_buffer_tree_ref(eb);
4896
Chris Masonb9473432009-03-13 11:00:37 -04004897 was_dirty = test_and_set_bit(EXTENT_BUFFER_DIRTY, &eb->bflags);
Josef Bacik0b32f4b2012-03-13 09:38:00 -04004898
Chris Masond1310b22008-01-24 16:13:08 -05004899 num_pages = num_extent_pages(eb->start, eb->len);
Josef Bacik3083ee22012-03-09 16:01:49 -05004900 WARN_ON(atomic_read(&eb->refs) == 0);
Josef Bacik0b32f4b2012-03-13 09:38:00 -04004901 WARN_ON(!test_bit(EXTENT_BUFFER_TREE_REF, &eb->bflags));
4902
Chris Masonb9473432009-03-13 11:00:37 -04004903 for (i = 0; i < num_pages; i++)
Josef Bacik0b32f4b2012-03-13 09:38:00 -04004904 set_page_dirty(extent_buffer_page(eb, i));
Chris Masonb9473432009-03-13 11:00:37 -04004905 return was_dirty;
Chris Masond1310b22008-01-24 16:13:08 -05004906}
Chris Masond1310b22008-01-24 16:13:08 -05004907
Josef Bacik0b32f4b2012-03-13 09:38:00 -04004908int clear_extent_buffer_uptodate(struct extent_buffer *eb)
Chris Mason1259ab72008-05-12 13:39:03 -04004909{
4910 unsigned long i;
4911 struct page *page;
4912 unsigned long num_pages;
4913
Chris Masonb4ce94d2009-02-04 09:25:08 -05004914 clear_bit(EXTENT_BUFFER_UPTODATE, &eb->bflags);
Josef Bacik0b32f4b2012-03-13 09:38:00 -04004915 num_pages = num_extent_pages(eb->start, eb->len);
Chris Mason1259ab72008-05-12 13:39:03 -04004916 for (i = 0; i < num_pages; i++) {
4917 page = extent_buffer_page(eb, i);
Chris Mason33958dc2008-07-30 10:29:12 -04004918 if (page)
4919 ClearPageUptodate(page);
Chris Mason1259ab72008-05-12 13:39:03 -04004920 }
4921 return 0;
4922}
4923
Josef Bacik0b32f4b2012-03-13 09:38:00 -04004924int set_extent_buffer_uptodate(struct extent_buffer *eb)
Chris Masond1310b22008-01-24 16:13:08 -05004925{
4926 unsigned long i;
4927 struct page *page;
4928 unsigned long num_pages;
4929
Josef Bacik0b32f4b2012-03-13 09:38:00 -04004930 set_bit(EXTENT_BUFFER_UPTODATE, &eb->bflags);
Chris Masond1310b22008-01-24 16:13:08 -05004931 num_pages = num_extent_pages(eb->start, eb->len);
Chris Masond1310b22008-01-24 16:13:08 -05004932 for (i = 0; i < num_pages; i++) {
4933 page = extent_buffer_page(eb, i);
Chris Masond1310b22008-01-24 16:13:08 -05004934 SetPageUptodate(page);
4935 }
4936 return 0;
4937}
Chris Masond1310b22008-01-24 16:13:08 -05004938
Josef Bacik0b32f4b2012-03-13 09:38:00 -04004939int extent_buffer_uptodate(struct extent_buffer *eb)
Chris Masond1310b22008-01-24 16:13:08 -05004940{
Josef Bacik0b32f4b2012-03-13 09:38:00 -04004941 return test_bit(EXTENT_BUFFER_UPTODATE, &eb->bflags);
Chris Masond1310b22008-01-24 16:13:08 -05004942}
Chris Masond1310b22008-01-24 16:13:08 -05004943
4944int read_extent_buffer_pages(struct extent_io_tree *tree,
Arne Jansenbb82ab82011-06-10 14:06:53 +02004945 struct extent_buffer *eb, u64 start, int wait,
Chris Masonf1885912008-04-09 16:28:12 -04004946 get_extent_t *get_extent, int mirror_num)
Chris Masond1310b22008-01-24 16:13:08 -05004947{
4948 unsigned long i;
4949 unsigned long start_i;
4950 struct page *page;
4951 int err;
4952 int ret = 0;
Chris Masonce9adaa2008-04-09 16:28:12 -04004953 int locked_pages = 0;
4954 int all_uptodate = 1;
Chris Masond1310b22008-01-24 16:13:08 -05004955 unsigned long num_pages;
Chris Mason727011e2010-08-06 13:21:20 -04004956 unsigned long num_reads = 0;
Chris Masona86c12c2008-02-07 10:50:54 -05004957 struct bio *bio = NULL;
Chris Masonc8b97812008-10-29 14:49:59 -04004958 unsigned long bio_flags = 0;
Chris Masona86c12c2008-02-07 10:50:54 -05004959
Chris Masonb4ce94d2009-02-04 09:25:08 -05004960 if (test_bit(EXTENT_BUFFER_UPTODATE, &eb->bflags))
Chris Masond1310b22008-01-24 16:13:08 -05004961 return 0;
4962
Chris Masond1310b22008-01-24 16:13:08 -05004963 if (start) {
4964 WARN_ON(start < eb->start);
4965 start_i = (start >> PAGE_CACHE_SHIFT) -
4966 (eb->start >> PAGE_CACHE_SHIFT);
4967 } else {
4968 start_i = 0;
4969 }
4970
4971 num_pages = num_extent_pages(eb->start, eb->len);
4972 for (i = start_i; i < num_pages; i++) {
4973 page = extent_buffer_page(eb, i);
Arne Jansenbb82ab82011-06-10 14:06:53 +02004974 if (wait == WAIT_NONE) {
David Woodhouse2db04962008-08-07 11:19:43 -04004975 if (!trylock_page(page))
Chris Masonce9adaa2008-04-09 16:28:12 -04004976 goto unlock_exit;
Chris Masond1310b22008-01-24 16:13:08 -05004977 } else {
4978 lock_page(page);
4979 }
Chris Masonce9adaa2008-04-09 16:28:12 -04004980 locked_pages++;
Chris Mason727011e2010-08-06 13:21:20 -04004981 if (!PageUptodate(page)) {
4982 num_reads++;
Chris Masonce9adaa2008-04-09 16:28:12 -04004983 all_uptodate = 0;
Chris Mason727011e2010-08-06 13:21:20 -04004984 }
Chris Masonce9adaa2008-04-09 16:28:12 -04004985 }
4986 if (all_uptodate) {
4987 if (start_i == 0)
Chris Masonb4ce94d2009-02-04 09:25:08 -05004988 set_bit(EXTENT_BUFFER_UPTODATE, &eb->bflags);
Chris Masonce9adaa2008-04-09 16:28:12 -04004989 goto unlock_exit;
4990 }
4991
Josef Bacikea466792012-03-26 21:57:36 -04004992 clear_bit(EXTENT_BUFFER_IOERR, &eb->bflags);
Josef Bacik5cf1ab52012-04-16 09:42:26 -04004993 eb->read_mirror = 0;
Josef Bacik0b32f4b2012-03-13 09:38:00 -04004994 atomic_set(&eb->io_pages, num_reads);
Chris Masonce9adaa2008-04-09 16:28:12 -04004995 for (i = start_i; i < num_pages; i++) {
4996 page = extent_buffer_page(eb, i);
Chris Masonce9adaa2008-04-09 16:28:12 -04004997 if (!PageUptodate(page)) {
Chris Masonf1885912008-04-09 16:28:12 -04004998 ClearPageError(page);
Chris Masona86c12c2008-02-07 10:50:54 -05004999 err = __extent_read_full_page(tree, page,
Chris Masonf1885912008-04-09 16:28:12 -04005000 get_extent, &bio,
Josef Bacikd4c7ca82013-04-19 19:49:09 -04005001 mirror_num, &bio_flags,
5002 READ | REQ_META);
Chris Masond3977122009-01-05 21:25:51 -05005003 if (err)
Chris Masond1310b22008-01-24 16:13:08 -05005004 ret = err;
Chris Masond1310b22008-01-24 16:13:08 -05005005 } else {
5006 unlock_page(page);
5007 }
5008 }
5009
Jeff Mahoney355808c2011-10-03 23:23:14 -04005010 if (bio) {
Josef Bacikd4c7ca82013-04-19 19:49:09 -04005011 err = submit_one_bio(READ | REQ_META, bio, mirror_num,
5012 bio_flags);
Jeff Mahoney79787ea2012-03-12 16:03:00 +01005013 if (err)
5014 return err;
Jeff Mahoney355808c2011-10-03 23:23:14 -04005015 }
Chris Masona86c12c2008-02-07 10:50:54 -05005016
Arne Jansenbb82ab82011-06-10 14:06:53 +02005017 if (ret || wait != WAIT_COMPLETE)
Chris Masond1310b22008-01-24 16:13:08 -05005018 return ret;
Chris Masond3977122009-01-05 21:25:51 -05005019
Chris Masond1310b22008-01-24 16:13:08 -05005020 for (i = start_i; i < num_pages; i++) {
5021 page = extent_buffer_page(eb, i);
5022 wait_on_page_locked(page);
Chris Masond3977122009-01-05 21:25:51 -05005023 if (!PageUptodate(page))
Chris Masond1310b22008-01-24 16:13:08 -05005024 ret = -EIO;
Chris Masond1310b22008-01-24 16:13:08 -05005025 }
Chris Masond3977122009-01-05 21:25:51 -05005026
Chris Masond1310b22008-01-24 16:13:08 -05005027 return ret;
Chris Masonce9adaa2008-04-09 16:28:12 -04005028
5029unlock_exit:
5030 i = start_i;
Chris Masond3977122009-01-05 21:25:51 -05005031 while (locked_pages > 0) {
Chris Masonce9adaa2008-04-09 16:28:12 -04005032 page = extent_buffer_page(eb, i);
5033 i++;
5034 unlock_page(page);
5035 locked_pages--;
5036 }
5037 return ret;
Chris Masond1310b22008-01-24 16:13:08 -05005038}
Chris Masond1310b22008-01-24 16:13:08 -05005039
5040void read_extent_buffer(struct extent_buffer *eb, void *dstv,
5041 unsigned long start,
5042 unsigned long len)
5043{
5044 size_t cur;
5045 size_t offset;
5046 struct page *page;
5047 char *kaddr;
5048 char *dst = (char *)dstv;
5049 size_t start_offset = eb->start & ((u64)PAGE_CACHE_SIZE - 1);
5050 unsigned long i = (start_offset + start) >> PAGE_CACHE_SHIFT;
Chris Masond1310b22008-01-24 16:13:08 -05005051
5052 WARN_ON(start > eb->len);
5053 WARN_ON(start + len > eb->start + eb->len);
5054
Geert Uytterhoeven778746b2013-08-20 13:20:16 +02005055 offset = (start_offset + start) & (PAGE_CACHE_SIZE - 1);
Chris Masond1310b22008-01-24 16:13:08 -05005056
Chris Masond3977122009-01-05 21:25:51 -05005057 while (len > 0) {
Chris Masond1310b22008-01-24 16:13:08 -05005058 page = extent_buffer_page(eb, i);
Chris Masond1310b22008-01-24 16:13:08 -05005059
5060 cur = min(len, (PAGE_CACHE_SIZE - offset));
Chris Masona6591712011-07-19 12:04:14 -04005061 kaddr = page_address(page);
Chris Masond1310b22008-01-24 16:13:08 -05005062 memcpy(dst, kaddr + offset, cur);
Chris Masond1310b22008-01-24 16:13:08 -05005063
5064 dst += cur;
5065 len -= cur;
5066 offset = 0;
5067 i++;
5068 }
5069}
Chris Masond1310b22008-01-24 16:13:08 -05005070
Gerhard Heift550ac1d2014-01-30 16:24:01 +01005071int read_extent_buffer_to_user(struct extent_buffer *eb, void __user *dstv,
5072 unsigned long start,
5073 unsigned long len)
5074{
5075 size_t cur;
5076 size_t offset;
5077 struct page *page;
5078 char *kaddr;
5079 char __user *dst = (char __user *)dstv;
5080 size_t start_offset = eb->start & ((u64)PAGE_CACHE_SIZE - 1);
5081 unsigned long i = (start_offset + start) >> PAGE_CACHE_SHIFT;
5082 int ret = 0;
5083
5084 WARN_ON(start > eb->len);
5085 WARN_ON(start + len > eb->start + eb->len);
5086
5087 offset = (start_offset + start) & (PAGE_CACHE_SIZE - 1);
5088
5089 while (len > 0) {
5090 page = extent_buffer_page(eb, i);
5091
5092 cur = min(len, (PAGE_CACHE_SIZE - offset));
5093 kaddr = page_address(page);
5094 if (copy_to_user(dst, kaddr + offset, cur)) {
5095 ret = -EFAULT;
5096 break;
5097 }
5098
5099 dst += cur;
5100 len -= cur;
5101 offset = 0;
5102 i++;
5103 }
5104
5105 return ret;
5106}
5107
Chris Masond1310b22008-01-24 16:13:08 -05005108int map_private_extent_buffer(struct extent_buffer *eb, unsigned long start,
Chris Masona6591712011-07-19 12:04:14 -04005109 unsigned long min_len, char **map,
Chris Masond1310b22008-01-24 16:13:08 -05005110 unsigned long *map_start,
Chris Masona6591712011-07-19 12:04:14 -04005111 unsigned long *map_len)
Chris Masond1310b22008-01-24 16:13:08 -05005112{
5113 size_t offset = start & (PAGE_CACHE_SIZE - 1);
5114 char *kaddr;
5115 struct page *p;
5116 size_t start_offset = eb->start & ((u64)PAGE_CACHE_SIZE - 1);
5117 unsigned long i = (start_offset + start) >> PAGE_CACHE_SHIFT;
5118 unsigned long end_i = (start_offset + start + min_len - 1) >>
5119 PAGE_CACHE_SHIFT;
5120
5121 if (i != end_i)
5122 return -EINVAL;
5123
5124 if (i == 0) {
5125 offset = start_offset;
5126 *map_start = 0;
5127 } else {
5128 offset = 0;
5129 *map_start = ((u64)i << PAGE_CACHE_SHIFT) - start_offset;
5130 }
Chris Masond3977122009-01-05 21:25:51 -05005131
Chris Masond1310b22008-01-24 16:13:08 -05005132 if (start + min_len > eb->len) {
Julia Lawall31b1a2b2012-11-03 10:58:34 +00005133 WARN(1, KERN_ERR "btrfs bad mapping eb start %llu len %lu, "
Geert Uytterhoevenc1c9ff72013-08-20 13:20:07 +02005134 "wanted %lu %lu\n",
5135 eb->start, eb->len, start, min_len);
Josef Bacik850265332011-03-15 14:52:12 -04005136 return -EINVAL;
Chris Masond1310b22008-01-24 16:13:08 -05005137 }
5138
5139 p = extent_buffer_page(eb, i);
Chris Masona6591712011-07-19 12:04:14 -04005140 kaddr = page_address(p);
Chris Masond1310b22008-01-24 16:13:08 -05005141 *map = kaddr + offset;
5142 *map_len = PAGE_CACHE_SIZE - offset;
5143 return 0;
5144}
Chris Masond1310b22008-01-24 16:13:08 -05005145
Chris Masond1310b22008-01-24 16:13:08 -05005146int memcmp_extent_buffer(struct extent_buffer *eb, const void *ptrv,
5147 unsigned long start,
5148 unsigned long len)
5149{
5150 size_t cur;
5151 size_t offset;
5152 struct page *page;
5153 char *kaddr;
5154 char *ptr = (char *)ptrv;
5155 size_t start_offset = eb->start & ((u64)PAGE_CACHE_SIZE - 1);
5156 unsigned long i = (start_offset + start) >> PAGE_CACHE_SHIFT;
5157 int ret = 0;
5158
5159 WARN_ON(start > eb->len);
5160 WARN_ON(start + len > eb->start + eb->len);
5161
Geert Uytterhoeven778746b2013-08-20 13:20:16 +02005162 offset = (start_offset + start) & (PAGE_CACHE_SIZE - 1);
Chris Masond1310b22008-01-24 16:13:08 -05005163
Chris Masond3977122009-01-05 21:25:51 -05005164 while (len > 0) {
Chris Masond1310b22008-01-24 16:13:08 -05005165 page = extent_buffer_page(eb, i);
Chris Masond1310b22008-01-24 16:13:08 -05005166
5167 cur = min(len, (PAGE_CACHE_SIZE - offset));
5168
Chris Masona6591712011-07-19 12:04:14 -04005169 kaddr = page_address(page);
Chris Masond1310b22008-01-24 16:13:08 -05005170 ret = memcmp(ptr, kaddr + offset, cur);
Chris Masond1310b22008-01-24 16:13:08 -05005171 if (ret)
5172 break;
5173
5174 ptr += cur;
5175 len -= cur;
5176 offset = 0;
5177 i++;
5178 }
5179 return ret;
5180}
Chris Masond1310b22008-01-24 16:13:08 -05005181
5182void write_extent_buffer(struct extent_buffer *eb, const void *srcv,
5183 unsigned long start, unsigned long len)
5184{
5185 size_t cur;
5186 size_t offset;
5187 struct page *page;
5188 char *kaddr;
5189 char *src = (char *)srcv;
5190 size_t start_offset = eb->start & ((u64)PAGE_CACHE_SIZE - 1);
5191 unsigned long i = (start_offset + start) >> PAGE_CACHE_SHIFT;
5192
5193 WARN_ON(start > eb->len);
5194 WARN_ON(start + len > eb->start + eb->len);
5195
Geert Uytterhoeven778746b2013-08-20 13:20:16 +02005196 offset = (start_offset + start) & (PAGE_CACHE_SIZE - 1);
Chris Masond1310b22008-01-24 16:13:08 -05005197
Chris Masond3977122009-01-05 21:25:51 -05005198 while (len > 0) {
Chris Masond1310b22008-01-24 16:13:08 -05005199 page = extent_buffer_page(eb, i);
5200 WARN_ON(!PageUptodate(page));
5201
5202 cur = min(len, PAGE_CACHE_SIZE - offset);
Chris Masona6591712011-07-19 12:04:14 -04005203 kaddr = page_address(page);
Chris Masond1310b22008-01-24 16:13:08 -05005204 memcpy(kaddr + offset, src, cur);
Chris Masond1310b22008-01-24 16:13:08 -05005205
5206 src += cur;
5207 len -= cur;
5208 offset = 0;
5209 i++;
5210 }
5211}
Chris Masond1310b22008-01-24 16:13:08 -05005212
5213void memset_extent_buffer(struct extent_buffer *eb, char c,
5214 unsigned long start, unsigned long len)
5215{
5216 size_t cur;
5217 size_t offset;
5218 struct page *page;
5219 char *kaddr;
5220 size_t start_offset = eb->start & ((u64)PAGE_CACHE_SIZE - 1);
5221 unsigned long i = (start_offset + start) >> PAGE_CACHE_SHIFT;
5222
5223 WARN_ON(start > eb->len);
5224 WARN_ON(start + len > eb->start + eb->len);
5225
Geert Uytterhoeven778746b2013-08-20 13:20:16 +02005226 offset = (start_offset + start) & (PAGE_CACHE_SIZE - 1);
Chris Masond1310b22008-01-24 16:13:08 -05005227
Chris Masond3977122009-01-05 21:25:51 -05005228 while (len > 0) {
Chris Masond1310b22008-01-24 16:13:08 -05005229 page = extent_buffer_page(eb, i);
5230 WARN_ON(!PageUptodate(page));
5231
5232 cur = min(len, PAGE_CACHE_SIZE - offset);
Chris Masona6591712011-07-19 12:04:14 -04005233 kaddr = page_address(page);
Chris Masond1310b22008-01-24 16:13:08 -05005234 memset(kaddr + offset, c, cur);
Chris Masond1310b22008-01-24 16:13:08 -05005235
5236 len -= cur;
5237 offset = 0;
5238 i++;
5239 }
5240}
Chris Masond1310b22008-01-24 16:13:08 -05005241
5242void copy_extent_buffer(struct extent_buffer *dst, struct extent_buffer *src,
5243 unsigned long dst_offset, unsigned long src_offset,
5244 unsigned long len)
5245{
5246 u64 dst_len = dst->len;
5247 size_t cur;
5248 size_t offset;
5249 struct page *page;
5250 char *kaddr;
5251 size_t start_offset = dst->start & ((u64)PAGE_CACHE_SIZE - 1);
5252 unsigned long i = (start_offset + dst_offset) >> PAGE_CACHE_SHIFT;
5253
5254 WARN_ON(src->len != dst_len);
5255
5256 offset = (start_offset + dst_offset) &
Geert Uytterhoeven778746b2013-08-20 13:20:16 +02005257 (PAGE_CACHE_SIZE - 1);
Chris Masond1310b22008-01-24 16:13:08 -05005258
Chris Masond3977122009-01-05 21:25:51 -05005259 while (len > 0) {
Chris Masond1310b22008-01-24 16:13:08 -05005260 page = extent_buffer_page(dst, i);
5261 WARN_ON(!PageUptodate(page));
5262
5263 cur = min(len, (unsigned long)(PAGE_CACHE_SIZE - offset));
5264
Chris Masona6591712011-07-19 12:04:14 -04005265 kaddr = page_address(page);
Chris Masond1310b22008-01-24 16:13:08 -05005266 read_extent_buffer(src, kaddr + offset, src_offset, cur);
Chris Masond1310b22008-01-24 16:13:08 -05005267
5268 src_offset += cur;
5269 len -= cur;
5270 offset = 0;
5271 i++;
5272 }
5273}
Chris Masond1310b22008-01-24 16:13:08 -05005274
Sergei Trofimovich33872062011-04-11 21:52:52 +00005275static inline bool areas_overlap(unsigned long src, unsigned long dst, unsigned long len)
5276{
5277 unsigned long distance = (src > dst) ? src - dst : dst - src;
5278 return distance < len;
5279}
5280
Chris Masond1310b22008-01-24 16:13:08 -05005281static void copy_pages(struct page *dst_page, struct page *src_page,
5282 unsigned long dst_off, unsigned long src_off,
5283 unsigned long len)
5284{
Chris Masona6591712011-07-19 12:04:14 -04005285 char *dst_kaddr = page_address(dst_page);
Chris Masond1310b22008-01-24 16:13:08 -05005286 char *src_kaddr;
Chris Mason727011e2010-08-06 13:21:20 -04005287 int must_memmove = 0;
Chris Masond1310b22008-01-24 16:13:08 -05005288
Sergei Trofimovich33872062011-04-11 21:52:52 +00005289 if (dst_page != src_page) {
Chris Masona6591712011-07-19 12:04:14 -04005290 src_kaddr = page_address(src_page);
Sergei Trofimovich33872062011-04-11 21:52:52 +00005291 } else {
Chris Masond1310b22008-01-24 16:13:08 -05005292 src_kaddr = dst_kaddr;
Chris Mason727011e2010-08-06 13:21:20 -04005293 if (areas_overlap(src_off, dst_off, len))
5294 must_memmove = 1;
Sergei Trofimovich33872062011-04-11 21:52:52 +00005295 }
Chris Masond1310b22008-01-24 16:13:08 -05005296
Chris Mason727011e2010-08-06 13:21:20 -04005297 if (must_memmove)
5298 memmove(dst_kaddr + dst_off, src_kaddr + src_off, len);
5299 else
5300 memcpy(dst_kaddr + dst_off, src_kaddr + src_off, len);
Chris Masond1310b22008-01-24 16:13:08 -05005301}
5302
5303void memcpy_extent_buffer(struct extent_buffer *dst, unsigned long dst_offset,
5304 unsigned long src_offset, unsigned long len)
5305{
5306 size_t cur;
5307 size_t dst_off_in_page;
5308 size_t src_off_in_page;
5309 size_t start_offset = dst->start & ((u64)PAGE_CACHE_SIZE - 1);
5310 unsigned long dst_i;
5311 unsigned long src_i;
5312
5313 if (src_offset + len > dst->len) {
Frank Holtonefe120a2013-12-20 11:37:06 -05005314 printk(KERN_ERR "BTRFS: memmove bogus src_offset %lu move "
Chris Masond3977122009-01-05 21:25:51 -05005315 "len %lu dst len %lu\n", src_offset, len, dst->len);
Chris Masond1310b22008-01-24 16:13:08 -05005316 BUG_ON(1);
5317 }
5318 if (dst_offset + len > dst->len) {
Frank Holtonefe120a2013-12-20 11:37:06 -05005319 printk(KERN_ERR "BTRFS: memmove bogus dst_offset %lu move "
Chris Masond3977122009-01-05 21:25:51 -05005320 "len %lu dst len %lu\n", dst_offset, len, dst->len);
Chris Masond1310b22008-01-24 16:13:08 -05005321 BUG_ON(1);
5322 }
5323
Chris Masond3977122009-01-05 21:25:51 -05005324 while (len > 0) {
Chris Masond1310b22008-01-24 16:13:08 -05005325 dst_off_in_page = (start_offset + dst_offset) &
Geert Uytterhoeven778746b2013-08-20 13:20:16 +02005326 (PAGE_CACHE_SIZE - 1);
Chris Masond1310b22008-01-24 16:13:08 -05005327 src_off_in_page = (start_offset + src_offset) &
Geert Uytterhoeven778746b2013-08-20 13:20:16 +02005328 (PAGE_CACHE_SIZE - 1);
Chris Masond1310b22008-01-24 16:13:08 -05005329
5330 dst_i = (start_offset + dst_offset) >> PAGE_CACHE_SHIFT;
5331 src_i = (start_offset + src_offset) >> PAGE_CACHE_SHIFT;
5332
5333 cur = min(len, (unsigned long)(PAGE_CACHE_SIZE -
5334 src_off_in_page));
5335 cur = min_t(unsigned long, cur,
5336 (unsigned long)(PAGE_CACHE_SIZE - dst_off_in_page));
5337
5338 copy_pages(extent_buffer_page(dst, dst_i),
5339 extent_buffer_page(dst, src_i),
5340 dst_off_in_page, src_off_in_page, cur);
5341
5342 src_offset += cur;
5343 dst_offset += cur;
5344 len -= cur;
5345 }
5346}
Chris Masond1310b22008-01-24 16:13:08 -05005347
5348void memmove_extent_buffer(struct extent_buffer *dst, unsigned long dst_offset,
5349 unsigned long src_offset, unsigned long len)
5350{
5351 size_t cur;
5352 size_t dst_off_in_page;
5353 size_t src_off_in_page;
5354 unsigned long dst_end = dst_offset + len - 1;
5355 unsigned long src_end = src_offset + len - 1;
5356 size_t start_offset = dst->start & ((u64)PAGE_CACHE_SIZE - 1);
5357 unsigned long dst_i;
5358 unsigned long src_i;
5359
5360 if (src_offset + len > dst->len) {
Frank Holtonefe120a2013-12-20 11:37:06 -05005361 printk(KERN_ERR "BTRFS: memmove bogus src_offset %lu move "
Chris Masond3977122009-01-05 21:25:51 -05005362 "len %lu len %lu\n", src_offset, len, dst->len);
Chris Masond1310b22008-01-24 16:13:08 -05005363 BUG_ON(1);
5364 }
5365 if (dst_offset + len > dst->len) {
Frank Holtonefe120a2013-12-20 11:37:06 -05005366 printk(KERN_ERR "BTRFS: memmove bogus dst_offset %lu move "
Chris Masond3977122009-01-05 21:25:51 -05005367 "len %lu len %lu\n", dst_offset, len, dst->len);
Chris Masond1310b22008-01-24 16:13:08 -05005368 BUG_ON(1);
5369 }
Chris Mason727011e2010-08-06 13:21:20 -04005370 if (dst_offset < src_offset) {
Chris Masond1310b22008-01-24 16:13:08 -05005371 memcpy_extent_buffer(dst, dst_offset, src_offset, len);
5372 return;
5373 }
Chris Masond3977122009-01-05 21:25:51 -05005374 while (len > 0) {
Chris Masond1310b22008-01-24 16:13:08 -05005375 dst_i = (start_offset + dst_end) >> PAGE_CACHE_SHIFT;
5376 src_i = (start_offset + src_end) >> PAGE_CACHE_SHIFT;
5377
5378 dst_off_in_page = (start_offset + dst_end) &
Geert Uytterhoeven778746b2013-08-20 13:20:16 +02005379 (PAGE_CACHE_SIZE - 1);
Chris Masond1310b22008-01-24 16:13:08 -05005380 src_off_in_page = (start_offset + src_end) &
Geert Uytterhoeven778746b2013-08-20 13:20:16 +02005381 (PAGE_CACHE_SIZE - 1);
Chris Masond1310b22008-01-24 16:13:08 -05005382
5383 cur = min_t(unsigned long, len, src_off_in_page + 1);
5384 cur = min(cur, dst_off_in_page + 1);
Zach Brown1877e1a2013-10-16 12:10:33 -07005385 copy_pages(extent_buffer_page(dst, dst_i),
Chris Masond1310b22008-01-24 16:13:08 -05005386 extent_buffer_page(dst, src_i),
5387 dst_off_in_page - cur + 1,
5388 src_off_in_page - cur + 1, cur);
5389
5390 dst_end -= cur;
5391 src_end -= cur;
5392 len -= cur;
5393 }
5394}
Chris Mason6af118ce2008-07-22 11:18:07 -04005395
David Sterbaf7a52a42013-04-26 14:56:29 +00005396int try_release_extent_buffer(struct page *page)
Miao Xie19fe0a82010-10-26 20:57:29 -04005397{
Chris Mason6af118ce2008-07-22 11:18:07 -04005398 struct extent_buffer *eb;
Miao Xie897ca6e2010-10-26 20:57:29 -04005399
Miao Xie19fe0a82010-10-26 20:57:29 -04005400 /*
Josef Bacik3083ee22012-03-09 16:01:49 -05005401 * We need to make sure noboody is attaching this page to an eb right
5402 * now.
Miao Xie19fe0a82010-10-26 20:57:29 -04005403 */
Josef Bacik3083ee22012-03-09 16:01:49 -05005404 spin_lock(&page->mapping->private_lock);
5405 if (!PagePrivate(page)) {
5406 spin_unlock(&page->mapping->private_lock);
5407 return 1;
Miao Xie19fe0a82010-10-26 20:57:29 -04005408 }
5409
Josef Bacik3083ee22012-03-09 16:01:49 -05005410 eb = (struct extent_buffer *)page->private;
5411 BUG_ON(!eb);
Miao Xie19fe0a82010-10-26 20:57:29 -04005412
Josef Bacik0b32f4b2012-03-13 09:38:00 -04005413 /*
Josef Bacik3083ee22012-03-09 16:01:49 -05005414 * This is a little awful but should be ok, we need to make sure that
5415 * the eb doesn't disappear out from under us while we're looking at
5416 * this page.
5417 */
5418 spin_lock(&eb->refs_lock);
Josef Bacik0b32f4b2012-03-13 09:38:00 -04005419 if (atomic_read(&eb->refs) != 1 || extent_buffer_under_io(eb)) {
Josef Bacik3083ee22012-03-09 16:01:49 -05005420 spin_unlock(&eb->refs_lock);
5421 spin_unlock(&page->mapping->private_lock);
5422 return 0;
5423 }
5424 spin_unlock(&page->mapping->private_lock);
5425
Josef Bacik3083ee22012-03-09 16:01:49 -05005426 /*
5427 * If tree ref isn't set then we know the ref on this eb is a real ref,
5428 * so just return, this page will likely be freed soon anyway.
5429 */
5430 if (!test_and_clear_bit(EXTENT_BUFFER_TREE_REF, &eb->bflags)) {
5431 spin_unlock(&eb->refs_lock);
5432 return 0;
5433 }
Josef Bacik3083ee22012-03-09 16:01:49 -05005434
David Sterbaf7a52a42013-04-26 14:56:29 +00005435 return release_extent_buffer(eb);
Chris Mason6af118ce2008-07-22 11:18:07 -04005436}