blob: 619592d86c2a0b6cd847c5d109ee9c1d0adf1e9c [file] [log] [blame]
Chris Masond1310b22008-01-24 16:13:08 -05001#include <linux/bitops.h>
2#include <linux/slab.h>
3#include <linux/bio.h>
4#include <linux/mm.h>
Chris Masond1310b22008-01-24 16:13:08 -05005#include <linux/pagemap.h>
6#include <linux/page-flags.h>
Chris Masond1310b22008-01-24 16:13:08 -05007#include <linux/spinlock.h>
8#include <linux/blkdev.h>
9#include <linux/swap.h>
Chris Masond1310b22008-01-24 16:13:08 -050010#include <linux/writeback.h>
11#include <linux/pagevec.h>
Linus Torvalds268bb0c2011-05-20 12:50:29 -070012#include <linux/prefetch.h>
Dan Magenheimer90a887c2011-05-26 10:01:56 -060013#include <linux/cleancache.h>
Chris Masond1310b22008-01-24 16:13:08 -050014#include "extent_io.h"
15#include "extent_map.h"
David Woodhouse902b22f2008-08-20 08:51:49 -040016#include "ctree.h"
17#include "btrfs_inode.h"
Jan Schmidt4a54c8c2011-07-22 15:41:52 +020018#include "volumes.h"
Stefan Behrens21adbd52011-11-09 13:44:05 +010019#include "check-integrity.h"
Josef Bacik0b32f4b2012-03-13 09:38:00 -040020#include "locking.h"
Josef Bacik606686e2012-06-04 14:03:51 -040021#include "rcu-string.h"
Liu Bofe09e162013-09-22 12:54:23 +080022#include "backref.h"
Chris Masond1310b22008-01-24 16:13:08 -050023
Chris Masond1310b22008-01-24 16:13:08 -050024static struct kmem_cache *extent_state_cache;
25static struct kmem_cache *extent_buffer_cache;
Chris Mason9be33952013-05-17 18:30:14 -040026static struct bio_set *btrfs_bioset;
Chris Masond1310b22008-01-24 16:13:08 -050027
Filipe Manana27a35072014-07-06 20:09:59 +010028static inline bool extent_state_in_tree(const struct extent_state *state)
29{
30 return !RB_EMPTY_NODE(&state->rb_node);
31}
32
Eric Sandeen6d49ba12013-04-22 16:12:31 +000033#ifdef CONFIG_BTRFS_DEBUG
Chris Masond1310b22008-01-24 16:13:08 -050034static LIST_HEAD(buffers);
35static LIST_HEAD(states);
Chris Mason4bef0842008-09-08 11:18:08 -040036
Chris Masond3977122009-01-05 21:25:51 -050037static DEFINE_SPINLOCK(leak_lock);
Eric Sandeen6d49ba12013-04-22 16:12:31 +000038
39static inline
40void btrfs_leak_debug_add(struct list_head *new, struct list_head *head)
41{
42 unsigned long flags;
43
44 spin_lock_irqsave(&leak_lock, flags);
45 list_add(new, head);
46 spin_unlock_irqrestore(&leak_lock, flags);
47}
48
49static inline
50void btrfs_leak_debug_del(struct list_head *entry)
51{
52 unsigned long flags;
53
54 spin_lock_irqsave(&leak_lock, flags);
55 list_del(entry);
56 spin_unlock_irqrestore(&leak_lock, flags);
57}
58
59static inline
60void btrfs_leak_debug_check(void)
61{
62 struct extent_state *state;
63 struct extent_buffer *eb;
64
65 while (!list_empty(&states)) {
66 state = list_entry(states.next, struct extent_state, leak_list);
Filipe Manana27a35072014-07-06 20:09:59 +010067 pr_err("BTRFS: state leak: start %llu end %llu state %lu in tree %d refs %d\n",
68 state->start, state->end, state->state,
69 extent_state_in_tree(state),
Geert Uytterhoevenc1c9ff72013-08-20 13:20:07 +020070 atomic_read(&state->refs));
Eric Sandeen6d49ba12013-04-22 16:12:31 +000071 list_del(&state->leak_list);
72 kmem_cache_free(extent_state_cache, state);
73 }
74
75 while (!list_empty(&buffers)) {
76 eb = list_entry(buffers.next, struct extent_buffer, leak_list);
Frank Holtonefe120a2013-12-20 11:37:06 -050077 printk(KERN_ERR "BTRFS: buffer leak start %llu len %lu "
Geert Uytterhoevenc1c9ff72013-08-20 13:20:07 +020078 "refs %d\n",
79 eb->start, eb->len, atomic_read(&eb->refs));
Eric Sandeen6d49ba12013-04-22 16:12:31 +000080 list_del(&eb->leak_list);
81 kmem_cache_free(extent_buffer_cache, eb);
82 }
83}
David Sterba8d599ae2013-04-30 15:22:23 +000084
Josef Bacika5dee372013-12-13 10:02:44 -050085#define btrfs_debug_check_extent_io_range(tree, start, end) \
86 __btrfs_debug_check_extent_io_range(__func__, (tree), (start), (end))
David Sterba8d599ae2013-04-30 15:22:23 +000087static inline void __btrfs_debug_check_extent_io_range(const char *caller,
Josef Bacika5dee372013-12-13 10:02:44 -050088 struct extent_io_tree *tree, u64 start, u64 end)
David Sterba8d599ae2013-04-30 15:22:23 +000089{
Josef Bacika5dee372013-12-13 10:02:44 -050090 struct inode *inode;
91 u64 isize;
David Sterba8d599ae2013-04-30 15:22:23 +000092
Josef Bacika5dee372013-12-13 10:02:44 -050093 if (!tree->mapping)
94 return;
95
96 inode = tree->mapping->host;
97 isize = i_size_read(inode);
David Sterba8d599ae2013-04-30 15:22:23 +000098 if (end >= PAGE_SIZE && (end % 2) == 0 && end != isize - 1) {
99 printk_ratelimited(KERN_DEBUG
Frank Holtonefe120a2013-12-20 11:37:06 -0500100 "BTRFS: %s: ino %llu isize %llu odd range [%llu,%llu]\n",
Geert Uytterhoevenc1c9ff72013-08-20 13:20:07 +0200101 caller, btrfs_ino(inode), isize, start, end);
David Sterba8d599ae2013-04-30 15:22:23 +0000102 }
103}
Eric Sandeen6d49ba12013-04-22 16:12:31 +0000104#else
105#define btrfs_leak_debug_add(new, head) do {} while (0)
106#define btrfs_leak_debug_del(entry) do {} while (0)
107#define btrfs_leak_debug_check() do {} while (0)
David Sterba8d599ae2013-04-30 15:22:23 +0000108#define btrfs_debug_check_extent_io_range(c, s, e) do {} while (0)
Chris Mason4bef0842008-09-08 11:18:08 -0400109#endif
Chris Masond1310b22008-01-24 16:13:08 -0500110
Chris Masond1310b22008-01-24 16:13:08 -0500111#define BUFFER_LRU_MAX 64
112
113struct tree_entry {
114 u64 start;
115 u64 end;
Chris Masond1310b22008-01-24 16:13:08 -0500116 struct rb_node rb_node;
117};
118
119struct extent_page_data {
120 struct bio *bio;
121 struct extent_io_tree *tree;
122 get_extent_t *get_extent;
Josef Bacikde0022b2012-09-25 14:25:58 -0400123 unsigned long bio_flags;
Chris Mason771ed682008-11-06 22:02:51 -0500124
125 /* tells writepage not to lock the state bits for this range
126 * it still does the unlocking
127 */
Chris Masonffbd5172009-04-20 15:50:09 -0400128 unsigned int extent_locked:1;
129
130 /* tells the submit_bio code to use a WRITE_SYNC */
131 unsigned int sync_io:1;
Chris Masond1310b22008-01-24 16:13:08 -0500132};
133
Josef Bacik0b32f4b2012-03-13 09:38:00 -0400134static noinline void flush_write_bio(void *data);
Jeff Mahoneyc2d904e2011-10-03 23:22:32 -0400135static inline struct btrfs_fs_info *
136tree_fs_info(struct extent_io_tree *tree)
137{
Josef Bacika5dee372013-12-13 10:02:44 -0500138 if (!tree->mapping)
139 return NULL;
Jeff Mahoneyc2d904e2011-10-03 23:22:32 -0400140 return btrfs_sb(tree->mapping->host->i_sb);
141}
Josef Bacik0b32f4b2012-03-13 09:38:00 -0400142
Chris Masond1310b22008-01-24 16:13:08 -0500143int __init extent_io_init(void)
144{
David Sterba837e1972012-09-07 03:00:48 -0600145 extent_state_cache = kmem_cache_create("btrfs_extent_state",
Christoph Hellwig9601e3f2009-04-13 15:33:09 +0200146 sizeof(struct extent_state), 0,
147 SLAB_RECLAIM_ACCOUNT | SLAB_MEM_SPREAD, NULL);
Chris Masond1310b22008-01-24 16:13:08 -0500148 if (!extent_state_cache)
149 return -ENOMEM;
150
David Sterba837e1972012-09-07 03:00:48 -0600151 extent_buffer_cache = kmem_cache_create("btrfs_extent_buffer",
Christoph Hellwig9601e3f2009-04-13 15:33:09 +0200152 sizeof(struct extent_buffer), 0,
153 SLAB_RECLAIM_ACCOUNT | SLAB_MEM_SPREAD, NULL);
Chris Masond1310b22008-01-24 16:13:08 -0500154 if (!extent_buffer_cache)
155 goto free_state_cache;
Chris Mason9be33952013-05-17 18:30:14 -0400156
157 btrfs_bioset = bioset_create(BIO_POOL_SIZE,
158 offsetof(struct btrfs_io_bio, bio));
159 if (!btrfs_bioset)
160 goto free_buffer_cache;
Darrick J. Wongb208c2f2013-09-19 20:37:07 -0700161
162 if (bioset_integrity_create(btrfs_bioset, BIO_POOL_SIZE))
163 goto free_bioset;
164
Chris Masond1310b22008-01-24 16:13:08 -0500165 return 0;
166
Darrick J. Wongb208c2f2013-09-19 20:37:07 -0700167free_bioset:
168 bioset_free(btrfs_bioset);
169 btrfs_bioset = NULL;
170
Chris Mason9be33952013-05-17 18:30:14 -0400171free_buffer_cache:
172 kmem_cache_destroy(extent_buffer_cache);
173 extent_buffer_cache = NULL;
174
Chris Masond1310b22008-01-24 16:13:08 -0500175free_state_cache:
176 kmem_cache_destroy(extent_state_cache);
Chris Mason9be33952013-05-17 18:30:14 -0400177 extent_state_cache = NULL;
Chris Masond1310b22008-01-24 16:13:08 -0500178 return -ENOMEM;
179}
180
181void extent_io_exit(void)
182{
Eric Sandeen6d49ba12013-04-22 16:12:31 +0000183 btrfs_leak_debug_check();
Kirill A. Shutemov8c0a8532012-09-26 11:33:07 +1000184
185 /*
186 * Make sure all delayed rcu free are flushed before we
187 * destroy caches.
188 */
189 rcu_barrier();
Chris Masond1310b22008-01-24 16:13:08 -0500190 if (extent_state_cache)
191 kmem_cache_destroy(extent_state_cache);
192 if (extent_buffer_cache)
193 kmem_cache_destroy(extent_buffer_cache);
Chris Mason9be33952013-05-17 18:30:14 -0400194 if (btrfs_bioset)
195 bioset_free(btrfs_bioset);
Chris Masond1310b22008-01-24 16:13:08 -0500196}
197
198void extent_io_tree_init(struct extent_io_tree *tree,
David Sterbaf993c882011-04-20 23:35:57 +0200199 struct address_space *mapping)
Chris Masond1310b22008-01-24 16:13:08 -0500200{
Eric Paris6bef4d32010-02-23 19:43:04 +0000201 tree->state = RB_ROOT;
Chris Masond1310b22008-01-24 16:13:08 -0500202 tree->ops = NULL;
203 tree->dirty_bytes = 0;
Chris Mason70dec802008-01-29 09:59:12 -0500204 spin_lock_init(&tree->lock);
Chris Masond1310b22008-01-24 16:13:08 -0500205 tree->mapping = mapping;
Chris Masond1310b22008-01-24 16:13:08 -0500206}
Chris Masond1310b22008-01-24 16:13:08 -0500207
Christoph Hellwigb2950862008-12-02 09:54:17 -0500208static struct extent_state *alloc_extent_state(gfp_t mask)
Chris Masond1310b22008-01-24 16:13:08 -0500209{
210 struct extent_state *state;
Chris Masond1310b22008-01-24 16:13:08 -0500211
212 state = kmem_cache_alloc(extent_state_cache, mask);
Peter2b114d12008-04-01 11:21:40 -0400213 if (!state)
Chris Masond1310b22008-01-24 16:13:08 -0500214 return state;
215 state->state = 0;
Chris Masond1310b22008-01-24 16:13:08 -0500216 state->private = 0;
Filipe Manana27a35072014-07-06 20:09:59 +0100217 RB_CLEAR_NODE(&state->rb_node);
Eric Sandeen6d49ba12013-04-22 16:12:31 +0000218 btrfs_leak_debug_add(&state->leak_list, &states);
Chris Masond1310b22008-01-24 16:13:08 -0500219 atomic_set(&state->refs, 1);
220 init_waitqueue_head(&state->wq);
Jeff Mahoney143bede2012-03-01 14:56:26 +0100221 trace_alloc_extent_state(state, mask, _RET_IP_);
Chris Masond1310b22008-01-24 16:13:08 -0500222 return state;
223}
Chris Masond1310b22008-01-24 16:13:08 -0500224
Chris Mason4845e442010-05-25 20:56:50 -0400225void free_extent_state(struct extent_state *state)
Chris Masond1310b22008-01-24 16:13:08 -0500226{
Chris Masond1310b22008-01-24 16:13:08 -0500227 if (!state)
228 return;
229 if (atomic_dec_and_test(&state->refs)) {
Filipe Manana27a35072014-07-06 20:09:59 +0100230 WARN_ON(extent_state_in_tree(state));
Eric Sandeen6d49ba12013-04-22 16:12:31 +0000231 btrfs_leak_debug_del(&state->leak_list);
Jeff Mahoney143bede2012-03-01 14:56:26 +0100232 trace_free_extent_state(state, _RET_IP_);
Chris Masond1310b22008-01-24 16:13:08 -0500233 kmem_cache_free(extent_state_cache, state);
234 }
235}
Chris Masond1310b22008-01-24 16:13:08 -0500236
Filipe Mananaf2071b22014-02-12 15:05:53 +0000237static struct rb_node *tree_insert(struct rb_root *root,
238 struct rb_node *search_start,
239 u64 offset,
Filipe David Borba Manana12cfbad2013-11-26 15:41:47 +0000240 struct rb_node *node,
241 struct rb_node ***p_in,
242 struct rb_node **parent_in)
Chris Masond1310b22008-01-24 16:13:08 -0500243{
Filipe Mananaf2071b22014-02-12 15:05:53 +0000244 struct rb_node **p;
Chris Masond3977122009-01-05 21:25:51 -0500245 struct rb_node *parent = NULL;
Chris Masond1310b22008-01-24 16:13:08 -0500246 struct tree_entry *entry;
247
Filipe David Borba Manana12cfbad2013-11-26 15:41:47 +0000248 if (p_in && parent_in) {
249 p = *p_in;
250 parent = *parent_in;
251 goto do_insert;
252 }
253
Filipe Mananaf2071b22014-02-12 15:05:53 +0000254 p = search_start ? &search_start : &root->rb_node;
Chris Masond3977122009-01-05 21:25:51 -0500255 while (*p) {
Chris Masond1310b22008-01-24 16:13:08 -0500256 parent = *p;
257 entry = rb_entry(parent, struct tree_entry, rb_node);
258
259 if (offset < entry->start)
260 p = &(*p)->rb_left;
261 else if (offset > entry->end)
262 p = &(*p)->rb_right;
263 else
264 return parent;
265 }
266
Filipe David Borba Manana12cfbad2013-11-26 15:41:47 +0000267do_insert:
Chris Masond1310b22008-01-24 16:13:08 -0500268 rb_link_node(node, parent, p);
269 rb_insert_color(node, root);
270 return NULL;
271}
272
Chris Mason80ea96b2008-02-01 14:51:59 -0500273static struct rb_node *__etree_search(struct extent_io_tree *tree, u64 offset,
Filipe David Borba Manana12cfbad2013-11-26 15:41:47 +0000274 struct rb_node **prev_ret,
275 struct rb_node **next_ret,
276 struct rb_node ***p_ret,
277 struct rb_node **parent_ret)
Chris Masond1310b22008-01-24 16:13:08 -0500278{
Chris Mason80ea96b2008-02-01 14:51:59 -0500279 struct rb_root *root = &tree->state;
Filipe David Borba Manana12cfbad2013-11-26 15:41:47 +0000280 struct rb_node **n = &root->rb_node;
Chris Masond1310b22008-01-24 16:13:08 -0500281 struct rb_node *prev = NULL;
282 struct rb_node *orig_prev = NULL;
283 struct tree_entry *entry;
284 struct tree_entry *prev_entry = NULL;
285
Filipe David Borba Manana12cfbad2013-11-26 15:41:47 +0000286 while (*n) {
287 prev = *n;
288 entry = rb_entry(prev, struct tree_entry, rb_node);
Chris Masond1310b22008-01-24 16:13:08 -0500289 prev_entry = entry;
290
291 if (offset < entry->start)
Filipe David Borba Manana12cfbad2013-11-26 15:41:47 +0000292 n = &(*n)->rb_left;
Chris Masond1310b22008-01-24 16:13:08 -0500293 else if (offset > entry->end)
Filipe David Borba Manana12cfbad2013-11-26 15:41:47 +0000294 n = &(*n)->rb_right;
Chris Masond3977122009-01-05 21:25:51 -0500295 else
Filipe David Borba Manana12cfbad2013-11-26 15:41:47 +0000296 return *n;
Chris Masond1310b22008-01-24 16:13:08 -0500297 }
298
Filipe David Borba Manana12cfbad2013-11-26 15:41:47 +0000299 if (p_ret)
300 *p_ret = n;
301 if (parent_ret)
302 *parent_ret = prev;
303
Chris Masond1310b22008-01-24 16:13:08 -0500304 if (prev_ret) {
305 orig_prev = prev;
Chris Masond3977122009-01-05 21:25:51 -0500306 while (prev && offset > prev_entry->end) {
Chris Masond1310b22008-01-24 16:13:08 -0500307 prev = rb_next(prev);
308 prev_entry = rb_entry(prev, struct tree_entry, rb_node);
309 }
310 *prev_ret = prev;
311 prev = orig_prev;
312 }
313
314 if (next_ret) {
315 prev_entry = rb_entry(prev, struct tree_entry, rb_node);
Chris Masond3977122009-01-05 21:25:51 -0500316 while (prev && offset < prev_entry->start) {
Chris Masond1310b22008-01-24 16:13:08 -0500317 prev = rb_prev(prev);
318 prev_entry = rb_entry(prev, struct tree_entry, rb_node);
319 }
320 *next_ret = prev;
321 }
322 return NULL;
323}
324
Filipe David Borba Manana12cfbad2013-11-26 15:41:47 +0000325static inline struct rb_node *
326tree_search_for_insert(struct extent_io_tree *tree,
327 u64 offset,
328 struct rb_node ***p_ret,
329 struct rb_node **parent_ret)
Chris Masond1310b22008-01-24 16:13:08 -0500330{
Chris Mason70dec802008-01-29 09:59:12 -0500331 struct rb_node *prev = NULL;
Chris Masond1310b22008-01-24 16:13:08 -0500332 struct rb_node *ret;
Chris Mason70dec802008-01-29 09:59:12 -0500333
Filipe David Borba Manana12cfbad2013-11-26 15:41:47 +0000334 ret = __etree_search(tree, offset, &prev, NULL, p_ret, parent_ret);
Chris Masond3977122009-01-05 21:25:51 -0500335 if (!ret)
Chris Masond1310b22008-01-24 16:13:08 -0500336 return prev;
337 return ret;
338}
339
Filipe David Borba Manana12cfbad2013-11-26 15:41:47 +0000340static inline struct rb_node *tree_search(struct extent_io_tree *tree,
341 u64 offset)
342{
343 return tree_search_for_insert(tree, offset, NULL, NULL);
344}
345
Josef Bacik9ed74f22009-09-11 16:12:44 -0400346static void merge_cb(struct extent_io_tree *tree, struct extent_state *new,
347 struct extent_state *other)
348{
349 if (tree->ops && tree->ops->merge_extent_hook)
350 tree->ops->merge_extent_hook(tree->mapping->host, new,
351 other);
352}
353
Chris Masond1310b22008-01-24 16:13:08 -0500354/*
355 * utility function to look for merge candidates inside a given range.
356 * Any extents with matching state are merged together into a single
357 * extent in the tree. Extents with EXTENT_IO in their state field
358 * are not merged because the end_io handlers need to be able to do
359 * operations on them without sleeping (or doing allocations/splits).
360 *
361 * This should be called with the tree lock held.
362 */
Jeff Mahoney1bf85042011-07-21 16:56:09 +0000363static void merge_state(struct extent_io_tree *tree,
364 struct extent_state *state)
Chris Masond1310b22008-01-24 16:13:08 -0500365{
366 struct extent_state *other;
367 struct rb_node *other_node;
368
Zheng Yan5b21f2e2008-09-26 10:05:38 -0400369 if (state->state & (EXTENT_IOBITS | EXTENT_BOUNDARY))
Jeff Mahoney1bf85042011-07-21 16:56:09 +0000370 return;
Chris Masond1310b22008-01-24 16:13:08 -0500371
372 other_node = rb_prev(&state->rb_node);
373 if (other_node) {
374 other = rb_entry(other_node, struct extent_state, rb_node);
375 if (other->end == state->start - 1 &&
376 other->state == state->state) {
Josef Bacik9ed74f22009-09-11 16:12:44 -0400377 merge_cb(tree, state, other);
Chris Masond1310b22008-01-24 16:13:08 -0500378 state->start = other->start;
Chris Masond1310b22008-01-24 16:13:08 -0500379 rb_erase(&other->rb_node, &tree->state);
Filipe Manana27a35072014-07-06 20:09:59 +0100380 RB_CLEAR_NODE(&other->rb_node);
Chris Masond1310b22008-01-24 16:13:08 -0500381 free_extent_state(other);
382 }
383 }
384 other_node = rb_next(&state->rb_node);
385 if (other_node) {
386 other = rb_entry(other_node, struct extent_state, rb_node);
387 if (other->start == state->end + 1 &&
388 other->state == state->state) {
Josef Bacik9ed74f22009-09-11 16:12:44 -0400389 merge_cb(tree, state, other);
Josef Bacikdf98b6e2011-06-20 14:53:48 -0400390 state->end = other->end;
Josef Bacikdf98b6e2011-06-20 14:53:48 -0400391 rb_erase(&other->rb_node, &tree->state);
Filipe Manana27a35072014-07-06 20:09:59 +0100392 RB_CLEAR_NODE(&other->rb_node);
Josef Bacikdf98b6e2011-06-20 14:53:48 -0400393 free_extent_state(other);
Chris Masond1310b22008-01-24 16:13:08 -0500394 }
395 }
Chris Masond1310b22008-01-24 16:13:08 -0500396}
397
Jeff Mahoney1bf85042011-07-21 16:56:09 +0000398static void set_state_cb(struct extent_io_tree *tree,
David Sterba41074882013-04-29 13:38:46 +0000399 struct extent_state *state, unsigned long *bits)
Chris Mason291d6732008-01-29 15:55:23 -0500400{
Jeff Mahoney1bf85042011-07-21 16:56:09 +0000401 if (tree->ops && tree->ops->set_bit_hook)
402 tree->ops->set_bit_hook(tree->mapping->host, state, bits);
Chris Mason291d6732008-01-29 15:55:23 -0500403}
404
405static void clear_state_cb(struct extent_io_tree *tree,
David Sterba41074882013-04-29 13:38:46 +0000406 struct extent_state *state, unsigned long *bits)
Chris Mason291d6732008-01-29 15:55:23 -0500407{
Josef Bacik9ed74f22009-09-11 16:12:44 -0400408 if (tree->ops && tree->ops->clear_bit_hook)
409 tree->ops->clear_bit_hook(tree->mapping->host, state, bits);
Chris Mason291d6732008-01-29 15:55:23 -0500410}
411
Xiao Guangrong3150b692011-07-14 03:19:08 +0000412static void set_state_bits(struct extent_io_tree *tree,
David Sterba41074882013-04-29 13:38:46 +0000413 struct extent_state *state, unsigned long *bits);
Xiao Guangrong3150b692011-07-14 03:19:08 +0000414
Chris Masond1310b22008-01-24 16:13:08 -0500415/*
416 * insert an extent_state struct into the tree. 'bits' are set on the
417 * struct before it is inserted.
418 *
419 * This may return -EEXIST if the extent is already there, in which case the
420 * state struct is freed.
421 *
422 * The tree lock is not taken internally. This is a utility function and
423 * probably isn't what you want to call (see set/clear_extent_bit).
424 */
425static int insert_state(struct extent_io_tree *tree,
426 struct extent_state *state, u64 start, u64 end,
Filipe David Borba Manana12cfbad2013-11-26 15:41:47 +0000427 struct rb_node ***p,
428 struct rb_node **parent,
David Sterba41074882013-04-29 13:38:46 +0000429 unsigned long *bits)
Chris Masond1310b22008-01-24 16:13:08 -0500430{
431 struct rb_node *node;
432
Julia Lawall31b1a2b2012-11-03 10:58:34 +0000433 if (end < start)
Frank Holtonefe120a2013-12-20 11:37:06 -0500434 WARN(1, KERN_ERR "BTRFS: end < start %llu %llu\n",
Geert Uytterhoevenc1c9ff72013-08-20 13:20:07 +0200435 end, start);
Chris Masond1310b22008-01-24 16:13:08 -0500436 state->start = start;
437 state->end = end;
Josef Bacik9ed74f22009-09-11 16:12:44 -0400438
Xiao Guangrong3150b692011-07-14 03:19:08 +0000439 set_state_bits(tree, state, bits);
440
Filipe Mananaf2071b22014-02-12 15:05:53 +0000441 node = tree_insert(&tree->state, NULL, end, &state->rb_node, p, parent);
Chris Masond1310b22008-01-24 16:13:08 -0500442 if (node) {
443 struct extent_state *found;
444 found = rb_entry(node, struct extent_state, rb_node);
Frank Holtonefe120a2013-12-20 11:37:06 -0500445 printk(KERN_ERR "BTRFS: found node %llu %llu on insert of "
Geert Uytterhoevenc1c9ff72013-08-20 13:20:07 +0200446 "%llu %llu\n",
447 found->start, found->end, start, end);
Chris Masond1310b22008-01-24 16:13:08 -0500448 return -EEXIST;
449 }
450 merge_state(tree, state);
451 return 0;
452}
453
Jeff Mahoney1bf85042011-07-21 16:56:09 +0000454static void split_cb(struct extent_io_tree *tree, struct extent_state *orig,
Josef Bacik9ed74f22009-09-11 16:12:44 -0400455 u64 split)
456{
457 if (tree->ops && tree->ops->split_extent_hook)
Jeff Mahoney1bf85042011-07-21 16:56:09 +0000458 tree->ops->split_extent_hook(tree->mapping->host, orig, split);
Josef Bacik9ed74f22009-09-11 16:12:44 -0400459}
460
Chris Masond1310b22008-01-24 16:13:08 -0500461/*
462 * split a given extent state struct in two, inserting the preallocated
463 * struct 'prealloc' as the newly created second half. 'split' indicates an
464 * offset inside 'orig' where it should be split.
465 *
466 * Before calling,
467 * the tree has 'orig' at [orig->start, orig->end]. After calling, there
468 * are two extent state structs in the tree:
469 * prealloc: [orig->start, split - 1]
470 * orig: [ split, orig->end ]
471 *
472 * The tree locks are not taken by this function. They need to be held
473 * by the caller.
474 */
475static int split_state(struct extent_io_tree *tree, struct extent_state *orig,
476 struct extent_state *prealloc, u64 split)
477{
478 struct rb_node *node;
Josef Bacik9ed74f22009-09-11 16:12:44 -0400479
480 split_cb(tree, orig, split);
481
Chris Masond1310b22008-01-24 16:13:08 -0500482 prealloc->start = orig->start;
483 prealloc->end = split - 1;
484 prealloc->state = orig->state;
485 orig->start = split;
486
Filipe Mananaf2071b22014-02-12 15:05:53 +0000487 node = tree_insert(&tree->state, &orig->rb_node, prealloc->end,
488 &prealloc->rb_node, NULL, NULL);
Chris Masond1310b22008-01-24 16:13:08 -0500489 if (node) {
Chris Masond1310b22008-01-24 16:13:08 -0500490 free_extent_state(prealloc);
491 return -EEXIST;
492 }
493 return 0;
494}
495
Li Zefancdc6a392012-03-12 16:39:48 +0800496static struct extent_state *next_state(struct extent_state *state)
497{
498 struct rb_node *next = rb_next(&state->rb_node);
499 if (next)
500 return rb_entry(next, struct extent_state, rb_node);
501 else
502 return NULL;
503}
504
Chris Masond1310b22008-01-24 16:13:08 -0500505/*
506 * utility function to clear some bits in an extent state struct.
Wang Sheng-Hui1b303fc2012-04-06 14:35:18 +0800507 * it will optionally wake up any one waiting on this state (wake == 1).
Chris Masond1310b22008-01-24 16:13:08 -0500508 *
509 * If no bits are set on the state struct after clearing things, the
510 * struct is freed and removed from the tree
511 */
Li Zefancdc6a392012-03-12 16:39:48 +0800512static struct extent_state *clear_state_bit(struct extent_io_tree *tree,
513 struct extent_state *state,
David Sterba41074882013-04-29 13:38:46 +0000514 unsigned long *bits, int wake)
Chris Masond1310b22008-01-24 16:13:08 -0500515{
Li Zefancdc6a392012-03-12 16:39:48 +0800516 struct extent_state *next;
David Sterba41074882013-04-29 13:38:46 +0000517 unsigned long bits_to_clear = *bits & ~EXTENT_CTLBITS;
Chris Masond1310b22008-01-24 16:13:08 -0500518
Yan, Zheng0ca1f7c2010-05-16 10:48:47 -0400519 if ((bits_to_clear & EXTENT_DIRTY) && (state->state & EXTENT_DIRTY)) {
Chris Masond1310b22008-01-24 16:13:08 -0500520 u64 range = state->end - state->start + 1;
521 WARN_ON(range > tree->dirty_bytes);
522 tree->dirty_bytes -= range;
523 }
Chris Mason291d6732008-01-29 15:55:23 -0500524 clear_state_cb(tree, state, bits);
Josef Bacik32c00af2009-10-08 13:34:05 -0400525 state->state &= ~bits_to_clear;
Chris Masond1310b22008-01-24 16:13:08 -0500526 if (wake)
527 wake_up(&state->wq);
Yan, Zheng0ca1f7c2010-05-16 10:48:47 -0400528 if (state->state == 0) {
Li Zefancdc6a392012-03-12 16:39:48 +0800529 next = next_state(state);
Filipe Manana27a35072014-07-06 20:09:59 +0100530 if (extent_state_in_tree(state)) {
Chris Masond1310b22008-01-24 16:13:08 -0500531 rb_erase(&state->rb_node, &tree->state);
Filipe Manana27a35072014-07-06 20:09:59 +0100532 RB_CLEAR_NODE(&state->rb_node);
Chris Masond1310b22008-01-24 16:13:08 -0500533 free_extent_state(state);
534 } else {
535 WARN_ON(1);
536 }
537 } else {
538 merge_state(tree, state);
Li Zefancdc6a392012-03-12 16:39:48 +0800539 next = next_state(state);
Chris Masond1310b22008-01-24 16:13:08 -0500540 }
Li Zefancdc6a392012-03-12 16:39:48 +0800541 return next;
Chris Masond1310b22008-01-24 16:13:08 -0500542}
543
Xiao Guangrong82337672011-04-20 06:44:57 +0000544static struct extent_state *
545alloc_extent_state_atomic(struct extent_state *prealloc)
546{
547 if (!prealloc)
548 prealloc = alloc_extent_state(GFP_ATOMIC);
549
550 return prealloc;
551}
552
Eric Sandeen48a3b632013-04-25 20:41:01 +0000553static void extent_io_tree_panic(struct extent_io_tree *tree, int err)
Jeff Mahoneyc2d904e2011-10-03 23:22:32 -0400554{
555 btrfs_panic(tree_fs_info(tree), err, "Locking error: "
556 "Extent tree was modified by another "
557 "thread while locked.");
558}
559
Chris Masond1310b22008-01-24 16:13:08 -0500560/*
561 * clear some bits on a range in the tree. This may require splitting
562 * or inserting elements in the tree, so the gfp mask is used to
563 * indicate which allocations or sleeping are allowed.
564 *
565 * pass 'wake' == 1 to kick any sleepers, and 'delete' == 1 to remove
566 * the given range from the tree regardless of state (ie for truncate).
567 *
568 * the range [start, end] is inclusive.
569 *
Jeff Mahoney6763af82012-03-01 14:56:29 +0100570 * This takes the tree lock, and returns 0 on success and < 0 on error.
Chris Masond1310b22008-01-24 16:13:08 -0500571 */
572int clear_extent_bit(struct extent_io_tree *tree, u64 start, u64 end,
David Sterba41074882013-04-29 13:38:46 +0000573 unsigned long bits, int wake, int delete,
Chris Mason2c64c532009-09-02 15:04:12 -0400574 struct extent_state **cached_state,
575 gfp_t mask)
Chris Masond1310b22008-01-24 16:13:08 -0500576{
577 struct extent_state *state;
Chris Mason2c64c532009-09-02 15:04:12 -0400578 struct extent_state *cached;
Chris Masond1310b22008-01-24 16:13:08 -0500579 struct extent_state *prealloc = NULL;
580 struct rb_node *node;
Yan Zheng5c939df2009-05-27 09:16:03 -0400581 u64 last_end;
Chris Masond1310b22008-01-24 16:13:08 -0500582 int err;
Josef Bacik2ac55d42010-02-03 19:33:23 +0000583 int clear = 0;
Chris Masond1310b22008-01-24 16:13:08 -0500584
Josef Bacika5dee372013-12-13 10:02:44 -0500585 btrfs_debug_check_extent_io_range(tree, start, end);
David Sterba8d599ae2013-04-30 15:22:23 +0000586
Josef Bacik7ee9e442013-06-21 16:37:03 -0400587 if (bits & EXTENT_DELALLOC)
588 bits |= EXTENT_NORESERVE;
589
Yan, Zheng0ca1f7c2010-05-16 10:48:47 -0400590 if (delete)
591 bits |= ~EXTENT_CTLBITS;
592 bits |= EXTENT_FIRST_DELALLOC;
593
Josef Bacik2ac55d42010-02-03 19:33:23 +0000594 if (bits & (EXTENT_IOBITS | EXTENT_BOUNDARY))
595 clear = 1;
Chris Masond1310b22008-01-24 16:13:08 -0500596again:
597 if (!prealloc && (mask & __GFP_WAIT)) {
Filipe Mananac7bc6312014-11-03 14:12:57 +0000598 /*
599 * Don't care for allocation failure here because we might end
600 * up not needing the pre-allocated extent state at all, which
601 * is the case if we only have in the tree extent states that
602 * cover our input range and don't cover too any other range.
603 * If we end up needing a new extent state we allocate it later.
604 */
Chris Masond1310b22008-01-24 16:13:08 -0500605 prealloc = alloc_extent_state(mask);
Chris Masond1310b22008-01-24 16:13:08 -0500606 }
607
Chris Masoncad321a2008-12-17 14:51:42 -0500608 spin_lock(&tree->lock);
Chris Mason2c64c532009-09-02 15:04:12 -0400609 if (cached_state) {
610 cached = *cached_state;
Josef Bacik2ac55d42010-02-03 19:33:23 +0000611
612 if (clear) {
613 *cached_state = NULL;
614 cached_state = NULL;
615 }
616
Filipe Manana27a35072014-07-06 20:09:59 +0100617 if (cached && extent_state_in_tree(cached) &&
618 cached->start <= start && cached->end > start) {
Josef Bacik2ac55d42010-02-03 19:33:23 +0000619 if (clear)
620 atomic_dec(&cached->refs);
Chris Mason2c64c532009-09-02 15:04:12 -0400621 state = cached;
Chris Mason42daec22009-09-23 19:51:09 -0400622 goto hit_next;
Chris Mason2c64c532009-09-02 15:04:12 -0400623 }
Josef Bacik2ac55d42010-02-03 19:33:23 +0000624 if (clear)
625 free_extent_state(cached);
Chris Mason2c64c532009-09-02 15:04:12 -0400626 }
Chris Masond1310b22008-01-24 16:13:08 -0500627 /*
628 * this search will find the extents that end after
629 * our range starts
630 */
Chris Mason80ea96b2008-02-01 14:51:59 -0500631 node = tree_search(tree, start);
Chris Masond1310b22008-01-24 16:13:08 -0500632 if (!node)
633 goto out;
634 state = rb_entry(node, struct extent_state, rb_node);
Chris Mason2c64c532009-09-02 15:04:12 -0400635hit_next:
Chris Masond1310b22008-01-24 16:13:08 -0500636 if (state->start > end)
637 goto out;
638 WARN_ON(state->end < start);
Yan Zheng5c939df2009-05-27 09:16:03 -0400639 last_end = state->end;
Chris Masond1310b22008-01-24 16:13:08 -0500640
Liu Bo04493142012-02-16 18:34:37 +0800641 /* the state doesn't have the wanted bits, go ahead */
Li Zefancdc6a392012-03-12 16:39:48 +0800642 if (!(state->state & bits)) {
643 state = next_state(state);
Liu Bo04493142012-02-16 18:34:37 +0800644 goto next;
Li Zefancdc6a392012-03-12 16:39:48 +0800645 }
Liu Bo04493142012-02-16 18:34:37 +0800646
Chris Masond1310b22008-01-24 16:13:08 -0500647 /*
648 * | ---- desired range ---- |
649 * | state | or
650 * | ------------- state -------------- |
651 *
652 * We need to split the extent we found, and may flip
653 * bits on second half.
654 *
655 * If the extent we found extends past our range, we
656 * just split and search again. It'll get split again
657 * the next time though.
658 *
659 * If the extent we found is inside our range, we clear
660 * the desired bit on it.
661 */
662
663 if (state->start < start) {
Xiao Guangrong82337672011-04-20 06:44:57 +0000664 prealloc = alloc_extent_state_atomic(prealloc);
665 BUG_ON(!prealloc);
Chris Masond1310b22008-01-24 16:13:08 -0500666 err = split_state(tree, state, prealloc, start);
Jeff Mahoneyc2d904e2011-10-03 23:22:32 -0400667 if (err)
668 extent_io_tree_panic(tree, err);
669
Chris Masond1310b22008-01-24 16:13:08 -0500670 prealloc = NULL;
671 if (err)
672 goto out;
673 if (state->end <= end) {
Liu Bod1ac6e42012-05-10 18:10:39 +0800674 state = clear_state_bit(tree, state, &bits, wake);
675 goto next;
Chris Masond1310b22008-01-24 16:13:08 -0500676 }
677 goto search_again;
678 }
679 /*
680 * | ---- desired range ---- |
681 * | state |
682 * We need to split the extent, and clear the bit
683 * on the first half
684 */
685 if (state->start <= end && state->end > end) {
Xiao Guangrong82337672011-04-20 06:44:57 +0000686 prealloc = alloc_extent_state_atomic(prealloc);
687 BUG_ON(!prealloc);
Chris Masond1310b22008-01-24 16:13:08 -0500688 err = split_state(tree, state, prealloc, end + 1);
Jeff Mahoneyc2d904e2011-10-03 23:22:32 -0400689 if (err)
690 extent_io_tree_panic(tree, err);
691
Chris Masond1310b22008-01-24 16:13:08 -0500692 if (wake)
693 wake_up(&state->wq);
Chris Mason42daec22009-09-23 19:51:09 -0400694
Jeff Mahoney6763af82012-03-01 14:56:29 +0100695 clear_state_bit(tree, prealloc, &bits, wake);
Josef Bacik9ed74f22009-09-11 16:12:44 -0400696
Chris Masond1310b22008-01-24 16:13:08 -0500697 prealloc = NULL;
698 goto out;
699 }
Chris Mason42daec22009-09-23 19:51:09 -0400700
Li Zefancdc6a392012-03-12 16:39:48 +0800701 state = clear_state_bit(tree, state, &bits, wake);
Liu Bo04493142012-02-16 18:34:37 +0800702next:
Yan Zheng5c939df2009-05-27 09:16:03 -0400703 if (last_end == (u64)-1)
704 goto out;
705 start = last_end + 1;
Li Zefancdc6a392012-03-12 16:39:48 +0800706 if (start <= end && state && !need_resched())
Liu Bo692e5752012-02-16 18:34:36 +0800707 goto hit_next;
Chris Masond1310b22008-01-24 16:13:08 -0500708 goto search_again;
709
710out:
Chris Masoncad321a2008-12-17 14:51:42 -0500711 spin_unlock(&tree->lock);
Chris Masond1310b22008-01-24 16:13:08 -0500712 if (prealloc)
713 free_extent_state(prealloc);
714
Jeff Mahoney6763af82012-03-01 14:56:29 +0100715 return 0;
Chris Masond1310b22008-01-24 16:13:08 -0500716
717search_again:
718 if (start > end)
719 goto out;
Chris Masoncad321a2008-12-17 14:51:42 -0500720 spin_unlock(&tree->lock);
Chris Masond1310b22008-01-24 16:13:08 -0500721 if (mask & __GFP_WAIT)
722 cond_resched();
723 goto again;
724}
Chris Masond1310b22008-01-24 16:13:08 -0500725
Jeff Mahoney143bede2012-03-01 14:56:26 +0100726static void wait_on_state(struct extent_io_tree *tree,
727 struct extent_state *state)
Christoph Hellwig641f5212008-12-02 06:36:10 -0500728 __releases(tree->lock)
729 __acquires(tree->lock)
Chris Masond1310b22008-01-24 16:13:08 -0500730{
731 DEFINE_WAIT(wait);
732 prepare_to_wait(&state->wq, &wait, TASK_UNINTERRUPTIBLE);
Chris Masoncad321a2008-12-17 14:51:42 -0500733 spin_unlock(&tree->lock);
Chris Masond1310b22008-01-24 16:13:08 -0500734 schedule();
Chris Masoncad321a2008-12-17 14:51:42 -0500735 spin_lock(&tree->lock);
Chris Masond1310b22008-01-24 16:13:08 -0500736 finish_wait(&state->wq, &wait);
Chris Masond1310b22008-01-24 16:13:08 -0500737}
738
739/*
740 * waits for one or more bits to clear on a range in the state tree.
741 * The range [start, end] is inclusive.
742 * The tree lock is taken by this function
743 */
David Sterba41074882013-04-29 13:38:46 +0000744static void wait_extent_bit(struct extent_io_tree *tree, u64 start, u64 end,
745 unsigned long bits)
Chris Masond1310b22008-01-24 16:13:08 -0500746{
747 struct extent_state *state;
748 struct rb_node *node;
749
Josef Bacika5dee372013-12-13 10:02:44 -0500750 btrfs_debug_check_extent_io_range(tree, start, end);
David Sterba8d599ae2013-04-30 15:22:23 +0000751
Chris Masoncad321a2008-12-17 14:51:42 -0500752 spin_lock(&tree->lock);
Chris Masond1310b22008-01-24 16:13:08 -0500753again:
754 while (1) {
755 /*
756 * this search will find all the extents that end after
757 * our range starts
758 */
Chris Mason80ea96b2008-02-01 14:51:59 -0500759 node = tree_search(tree, start);
Filipe Mananac50d3e72014-03-31 14:53:25 +0100760process_node:
Chris Masond1310b22008-01-24 16:13:08 -0500761 if (!node)
762 break;
763
764 state = rb_entry(node, struct extent_state, rb_node);
765
766 if (state->start > end)
767 goto out;
768
769 if (state->state & bits) {
770 start = state->start;
771 atomic_inc(&state->refs);
772 wait_on_state(tree, state);
773 free_extent_state(state);
774 goto again;
775 }
776 start = state->end + 1;
777
778 if (start > end)
779 break;
780
Filipe Mananac50d3e72014-03-31 14:53:25 +0100781 if (!cond_resched_lock(&tree->lock)) {
782 node = rb_next(node);
783 goto process_node;
784 }
Chris Masond1310b22008-01-24 16:13:08 -0500785 }
786out:
Chris Masoncad321a2008-12-17 14:51:42 -0500787 spin_unlock(&tree->lock);
Chris Masond1310b22008-01-24 16:13:08 -0500788}
Chris Masond1310b22008-01-24 16:13:08 -0500789
Jeff Mahoney1bf85042011-07-21 16:56:09 +0000790static void set_state_bits(struct extent_io_tree *tree,
Chris Masond1310b22008-01-24 16:13:08 -0500791 struct extent_state *state,
David Sterba41074882013-04-29 13:38:46 +0000792 unsigned long *bits)
Chris Masond1310b22008-01-24 16:13:08 -0500793{
David Sterba41074882013-04-29 13:38:46 +0000794 unsigned long bits_to_set = *bits & ~EXTENT_CTLBITS;
Josef Bacik9ed74f22009-09-11 16:12:44 -0400795
Jeff Mahoney1bf85042011-07-21 16:56:09 +0000796 set_state_cb(tree, state, bits);
Yan, Zheng0ca1f7c2010-05-16 10:48:47 -0400797 if ((bits_to_set & EXTENT_DIRTY) && !(state->state & EXTENT_DIRTY)) {
Chris Masond1310b22008-01-24 16:13:08 -0500798 u64 range = state->end - state->start + 1;
799 tree->dirty_bytes += range;
800 }
Yan, Zheng0ca1f7c2010-05-16 10:48:47 -0400801 state->state |= bits_to_set;
Chris Masond1310b22008-01-24 16:13:08 -0500802}
803
Filipe Mananae38e2ed2014-10-13 12:28:38 +0100804static void cache_state_if_flags(struct extent_state *state,
805 struct extent_state **cached_ptr,
806 const u64 flags)
Chris Mason2c64c532009-09-02 15:04:12 -0400807{
808 if (cached_ptr && !(*cached_ptr)) {
Filipe Mananae38e2ed2014-10-13 12:28:38 +0100809 if (!flags || (state->state & flags)) {
Chris Mason2c64c532009-09-02 15:04:12 -0400810 *cached_ptr = state;
811 atomic_inc(&state->refs);
812 }
813 }
814}
815
Filipe Mananae38e2ed2014-10-13 12:28:38 +0100816static void cache_state(struct extent_state *state,
817 struct extent_state **cached_ptr)
818{
819 return cache_state_if_flags(state, cached_ptr,
820 EXTENT_IOBITS | EXTENT_BOUNDARY);
821}
822
Chris Masond1310b22008-01-24 16:13:08 -0500823/*
Chris Mason1edbb732009-09-02 13:24:36 -0400824 * set some bits on a range in the tree. This may require allocations or
825 * sleeping, so the gfp mask is used to indicate what is allowed.
Chris Masond1310b22008-01-24 16:13:08 -0500826 *
Chris Mason1edbb732009-09-02 13:24:36 -0400827 * If any of the exclusive bits are set, this will fail with -EEXIST if some
828 * part of the range already has the desired bits set. The start of the
829 * existing range is returned in failed_start in this case.
Chris Masond1310b22008-01-24 16:13:08 -0500830 *
Chris Mason1edbb732009-09-02 13:24:36 -0400831 * [start, end] is inclusive This takes the tree lock.
Chris Masond1310b22008-01-24 16:13:08 -0500832 */
Chris Mason1edbb732009-09-02 13:24:36 -0400833
Jeff Mahoney3fbe5c02012-03-01 14:57:19 +0100834static int __must_check
835__set_extent_bit(struct extent_io_tree *tree, u64 start, u64 end,
David Sterba41074882013-04-29 13:38:46 +0000836 unsigned long bits, unsigned long exclusive_bits,
837 u64 *failed_start, struct extent_state **cached_state,
838 gfp_t mask)
Chris Masond1310b22008-01-24 16:13:08 -0500839{
840 struct extent_state *state;
841 struct extent_state *prealloc = NULL;
842 struct rb_node *node;
Filipe David Borba Manana12cfbad2013-11-26 15:41:47 +0000843 struct rb_node **p;
844 struct rb_node *parent;
Chris Masond1310b22008-01-24 16:13:08 -0500845 int err = 0;
Chris Masond1310b22008-01-24 16:13:08 -0500846 u64 last_start;
847 u64 last_end;
Chris Mason42daec22009-09-23 19:51:09 -0400848
Josef Bacika5dee372013-12-13 10:02:44 -0500849 btrfs_debug_check_extent_io_range(tree, start, end);
David Sterba8d599ae2013-04-30 15:22:23 +0000850
Yan, Zheng0ca1f7c2010-05-16 10:48:47 -0400851 bits |= EXTENT_FIRST_DELALLOC;
Chris Masond1310b22008-01-24 16:13:08 -0500852again:
853 if (!prealloc && (mask & __GFP_WAIT)) {
854 prealloc = alloc_extent_state(mask);
Xiao Guangrong82337672011-04-20 06:44:57 +0000855 BUG_ON(!prealloc);
Chris Masond1310b22008-01-24 16:13:08 -0500856 }
857
Chris Masoncad321a2008-12-17 14:51:42 -0500858 spin_lock(&tree->lock);
Chris Mason9655d292009-09-02 15:22:30 -0400859 if (cached_state && *cached_state) {
860 state = *cached_state;
Josef Bacikdf98b6e2011-06-20 14:53:48 -0400861 if (state->start <= start && state->end > start &&
Filipe Manana27a35072014-07-06 20:09:59 +0100862 extent_state_in_tree(state)) {
Chris Mason9655d292009-09-02 15:22:30 -0400863 node = &state->rb_node;
864 goto hit_next;
865 }
866 }
Chris Masond1310b22008-01-24 16:13:08 -0500867 /*
868 * this search will find all the extents that end after
869 * our range starts.
870 */
Filipe David Borba Manana12cfbad2013-11-26 15:41:47 +0000871 node = tree_search_for_insert(tree, start, &p, &parent);
Chris Masond1310b22008-01-24 16:13:08 -0500872 if (!node) {
Xiao Guangrong82337672011-04-20 06:44:57 +0000873 prealloc = alloc_extent_state_atomic(prealloc);
874 BUG_ON(!prealloc);
Filipe David Borba Manana12cfbad2013-11-26 15:41:47 +0000875 err = insert_state(tree, prealloc, start, end,
876 &p, &parent, &bits);
Jeff Mahoneyc2d904e2011-10-03 23:22:32 -0400877 if (err)
878 extent_io_tree_panic(tree, err);
879
Filipe David Borba Mananac42ac0b2013-11-26 15:01:34 +0000880 cache_state(prealloc, cached_state);
Chris Masond1310b22008-01-24 16:13:08 -0500881 prealloc = NULL;
Chris Masond1310b22008-01-24 16:13:08 -0500882 goto out;
883 }
Chris Masond1310b22008-01-24 16:13:08 -0500884 state = rb_entry(node, struct extent_state, rb_node);
Chris Mason40431d62009-08-05 12:57:59 -0400885hit_next:
Chris Masond1310b22008-01-24 16:13:08 -0500886 last_start = state->start;
887 last_end = state->end;
888
889 /*
890 * | ---- desired range ---- |
891 * | state |
892 *
893 * Just lock what we found and keep going
894 */
895 if (state->start == start && state->end <= end) {
Chris Mason1edbb732009-09-02 13:24:36 -0400896 if (state->state & exclusive_bits) {
Chris Masond1310b22008-01-24 16:13:08 -0500897 *failed_start = state->start;
898 err = -EEXIST;
899 goto out;
900 }
Chris Mason42daec22009-09-23 19:51:09 -0400901
Jeff Mahoney1bf85042011-07-21 16:56:09 +0000902 set_state_bits(tree, state, &bits);
Chris Mason2c64c532009-09-02 15:04:12 -0400903 cache_state(state, cached_state);
Chris Masond1310b22008-01-24 16:13:08 -0500904 merge_state(tree, state);
Yan Zheng5c939df2009-05-27 09:16:03 -0400905 if (last_end == (u64)-1)
906 goto out;
907 start = last_end + 1;
Liu Bod1ac6e42012-05-10 18:10:39 +0800908 state = next_state(state);
909 if (start < end && state && state->start == start &&
910 !need_resched())
911 goto hit_next;
Chris Masond1310b22008-01-24 16:13:08 -0500912 goto search_again;
913 }
914
915 /*
916 * | ---- desired range ---- |
917 * | state |
918 * or
919 * | ------------- state -------------- |
920 *
921 * We need to split the extent we found, and may flip bits on
922 * second half.
923 *
924 * If the extent we found extends past our
925 * range, we just split and search again. It'll get split
926 * again the next time though.
927 *
928 * If the extent we found is inside our range, we set the
929 * desired bit on it.
930 */
931 if (state->start < start) {
Chris Mason1edbb732009-09-02 13:24:36 -0400932 if (state->state & exclusive_bits) {
Chris Masond1310b22008-01-24 16:13:08 -0500933 *failed_start = start;
934 err = -EEXIST;
935 goto out;
936 }
Xiao Guangrong82337672011-04-20 06:44:57 +0000937
938 prealloc = alloc_extent_state_atomic(prealloc);
939 BUG_ON(!prealloc);
Chris Masond1310b22008-01-24 16:13:08 -0500940 err = split_state(tree, state, prealloc, start);
Jeff Mahoneyc2d904e2011-10-03 23:22:32 -0400941 if (err)
942 extent_io_tree_panic(tree, err);
943
Chris Masond1310b22008-01-24 16:13:08 -0500944 prealloc = NULL;
945 if (err)
946 goto out;
947 if (state->end <= end) {
Jeff Mahoney1bf85042011-07-21 16:56:09 +0000948 set_state_bits(tree, state, &bits);
Chris Mason2c64c532009-09-02 15:04:12 -0400949 cache_state(state, cached_state);
Chris Masond1310b22008-01-24 16:13:08 -0500950 merge_state(tree, state);
Yan Zheng5c939df2009-05-27 09:16:03 -0400951 if (last_end == (u64)-1)
952 goto out;
953 start = last_end + 1;
Liu Bod1ac6e42012-05-10 18:10:39 +0800954 state = next_state(state);
955 if (start < end && state && state->start == start &&
956 !need_resched())
957 goto hit_next;
Chris Masond1310b22008-01-24 16:13:08 -0500958 }
959 goto search_again;
960 }
961 /*
962 * | ---- desired range ---- |
963 * | state | or | state |
964 *
965 * There's a hole, we need to insert something in it and
966 * ignore the extent we found.
967 */
968 if (state->start > start) {
969 u64 this_end;
970 if (end < last_start)
971 this_end = end;
972 else
Chris Masond3977122009-01-05 21:25:51 -0500973 this_end = last_start - 1;
Xiao Guangrong82337672011-04-20 06:44:57 +0000974
975 prealloc = alloc_extent_state_atomic(prealloc);
976 BUG_ON(!prealloc);
Xiao Guangrongc7f895a2011-04-20 06:45:49 +0000977
978 /*
979 * Avoid to free 'prealloc' if it can be merged with
980 * the later extent.
981 */
Chris Masond1310b22008-01-24 16:13:08 -0500982 err = insert_state(tree, prealloc, start, this_end,
Filipe David Borba Manana12cfbad2013-11-26 15:41:47 +0000983 NULL, NULL, &bits);
Jeff Mahoneyc2d904e2011-10-03 23:22:32 -0400984 if (err)
985 extent_io_tree_panic(tree, err);
986
Chris Mason2c64c532009-09-02 15:04:12 -0400987 cache_state(prealloc, cached_state);
Chris Masond1310b22008-01-24 16:13:08 -0500988 prealloc = NULL;
Chris Masond1310b22008-01-24 16:13:08 -0500989 start = this_end + 1;
990 goto search_again;
991 }
992 /*
993 * | ---- desired range ---- |
994 * | state |
995 * We need to split the extent, and set the bit
996 * on the first half
997 */
998 if (state->start <= end && state->end > end) {
Chris Mason1edbb732009-09-02 13:24:36 -0400999 if (state->state & exclusive_bits) {
Chris Masond1310b22008-01-24 16:13:08 -05001000 *failed_start = start;
1001 err = -EEXIST;
1002 goto out;
1003 }
Xiao Guangrong82337672011-04-20 06:44:57 +00001004
1005 prealloc = alloc_extent_state_atomic(prealloc);
1006 BUG_ON(!prealloc);
Chris Masond1310b22008-01-24 16:13:08 -05001007 err = split_state(tree, state, prealloc, end + 1);
Jeff Mahoneyc2d904e2011-10-03 23:22:32 -04001008 if (err)
1009 extent_io_tree_panic(tree, err);
Chris Masond1310b22008-01-24 16:13:08 -05001010
Jeff Mahoney1bf85042011-07-21 16:56:09 +00001011 set_state_bits(tree, prealloc, &bits);
Chris Mason2c64c532009-09-02 15:04:12 -04001012 cache_state(prealloc, cached_state);
Chris Masond1310b22008-01-24 16:13:08 -05001013 merge_state(tree, prealloc);
1014 prealloc = NULL;
1015 goto out;
1016 }
1017
1018 goto search_again;
1019
1020out:
Chris Masoncad321a2008-12-17 14:51:42 -05001021 spin_unlock(&tree->lock);
Chris Masond1310b22008-01-24 16:13:08 -05001022 if (prealloc)
1023 free_extent_state(prealloc);
1024
1025 return err;
1026
1027search_again:
1028 if (start > end)
1029 goto out;
Chris Masoncad321a2008-12-17 14:51:42 -05001030 spin_unlock(&tree->lock);
Chris Masond1310b22008-01-24 16:13:08 -05001031 if (mask & __GFP_WAIT)
1032 cond_resched();
1033 goto again;
1034}
Chris Masond1310b22008-01-24 16:13:08 -05001035
David Sterba41074882013-04-29 13:38:46 +00001036int set_extent_bit(struct extent_io_tree *tree, u64 start, u64 end,
1037 unsigned long bits, u64 * failed_start,
1038 struct extent_state **cached_state, gfp_t mask)
Jeff Mahoney3fbe5c02012-03-01 14:57:19 +01001039{
1040 return __set_extent_bit(tree, start, end, bits, 0, failed_start,
1041 cached_state, mask);
1042}
1043
1044
Josef Bacik462d6fa2011-09-26 13:56:12 -04001045/**
Liu Bo10983f22012-07-11 15:26:19 +08001046 * convert_extent_bit - convert all bits in a given range from one bit to
1047 * another
Josef Bacik462d6fa2011-09-26 13:56:12 -04001048 * @tree: the io tree to search
1049 * @start: the start offset in bytes
1050 * @end: the end offset in bytes (inclusive)
1051 * @bits: the bits to set in this range
1052 * @clear_bits: the bits to clear in this range
Josef Bacike6138872012-09-27 17:07:30 -04001053 * @cached_state: state that we're going to cache
Josef Bacik462d6fa2011-09-26 13:56:12 -04001054 * @mask: the allocation mask
1055 *
1056 * This will go through and set bits for the given range. If any states exist
1057 * already in this range they are set with the given bit and cleared of the
1058 * clear_bits. This is only meant to be used by things that are mergeable, ie
1059 * converting from say DELALLOC to DIRTY. This is not meant to be used with
1060 * boundary bits like LOCK.
1061 */
1062int convert_extent_bit(struct extent_io_tree *tree, u64 start, u64 end,
David Sterba41074882013-04-29 13:38:46 +00001063 unsigned long bits, unsigned long clear_bits,
Josef Bacike6138872012-09-27 17:07:30 -04001064 struct extent_state **cached_state, gfp_t mask)
Josef Bacik462d6fa2011-09-26 13:56:12 -04001065{
1066 struct extent_state *state;
1067 struct extent_state *prealloc = NULL;
1068 struct rb_node *node;
Filipe David Borba Manana12cfbad2013-11-26 15:41:47 +00001069 struct rb_node **p;
1070 struct rb_node *parent;
Josef Bacik462d6fa2011-09-26 13:56:12 -04001071 int err = 0;
1072 u64 last_start;
1073 u64 last_end;
Filipe Mananac8fd3de2014-10-13 12:28:39 +01001074 bool first_iteration = true;
Josef Bacik462d6fa2011-09-26 13:56:12 -04001075
Josef Bacika5dee372013-12-13 10:02:44 -05001076 btrfs_debug_check_extent_io_range(tree, start, end);
David Sterba8d599ae2013-04-30 15:22:23 +00001077
Josef Bacik462d6fa2011-09-26 13:56:12 -04001078again:
1079 if (!prealloc && (mask & __GFP_WAIT)) {
Filipe Mananac8fd3de2014-10-13 12:28:39 +01001080 /*
1081 * Best effort, don't worry if extent state allocation fails
1082 * here for the first iteration. We might have a cached state
1083 * that matches exactly the target range, in which case no
1084 * extent state allocations are needed. We'll only know this
1085 * after locking the tree.
1086 */
Josef Bacik462d6fa2011-09-26 13:56:12 -04001087 prealloc = alloc_extent_state(mask);
Filipe Mananac8fd3de2014-10-13 12:28:39 +01001088 if (!prealloc && !first_iteration)
Josef Bacik462d6fa2011-09-26 13:56:12 -04001089 return -ENOMEM;
1090 }
1091
1092 spin_lock(&tree->lock);
Josef Bacike6138872012-09-27 17:07:30 -04001093 if (cached_state && *cached_state) {
1094 state = *cached_state;
1095 if (state->start <= start && state->end > start &&
Filipe Manana27a35072014-07-06 20:09:59 +01001096 extent_state_in_tree(state)) {
Josef Bacike6138872012-09-27 17:07:30 -04001097 node = &state->rb_node;
1098 goto hit_next;
1099 }
1100 }
1101
Josef Bacik462d6fa2011-09-26 13:56:12 -04001102 /*
1103 * this search will find all the extents that end after
1104 * our range starts.
1105 */
Filipe David Borba Manana12cfbad2013-11-26 15:41:47 +00001106 node = tree_search_for_insert(tree, start, &p, &parent);
Josef Bacik462d6fa2011-09-26 13:56:12 -04001107 if (!node) {
1108 prealloc = alloc_extent_state_atomic(prealloc);
Liu Bo1cf4ffd2011-12-07 20:08:40 -05001109 if (!prealloc) {
1110 err = -ENOMEM;
1111 goto out;
1112 }
Filipe David Borba Manana12cfbad2013-11-26 15:41:47 +00001113 err = insert_state(tree, prealloc, start, end,
1114 &p, &parent, &bits);
Jeff Mahoneyc2d904e2011-10-03 23:22:32 -04001115 if (err)
1116 extent_io_tree_panic(tree, err);
Filipe David Borba Mananac42ac0b2013-11-26 15:01:34 +00001117 cache_state(prealloc, cached_state);
1118 prealloc = NULL;
Josef Bacik462d6fa2011-09-26 13:56:12 -04001119 goto out;
1120 }
1121 state = rb_entry(node, struct extent_state, rb_node);
1122hit_next:
1123 last_start = state->start;
1124 last_end = state->end;
1125
1126 /*
1127 * | ---- desired range ---- |
1128 * | state |
1129 *
1130 * Just lock what we found and keep going
1131 */
1132 if (state->start == start && state->end <= end) {
Josef Bacik462d6fa2011-09-26 13:56:12 -04001133 set_state_bits(tree, state, &bits);
Josef Bacike6138872012-09-27 17:07:30 -04001134 cache_state(state, cached_state);
Liu Bod1ac6e42012-05-10 18:10:39 +08001135 state = clear_state_bit(tree, state, &clear_bits, 0);
Josef Bacik462d6fa2011-09-26 13:56:12 -04001136 if (last_end == (u64)-1)
1137 goto out;
Josef Bacik462d6fa2011-09-26 13:56:12 -04001138 start = last_end + 1;
Liu Bod1ac6e42012-05-10 18:10:39 +08001139 if (start < end && state && state->start == start &&
1140 !need_resched())
1141 goto hit_next;
Josef Bacik462d6fa2011-09-26 13:56:12 -04001142 goto search_again;
1143 }
1144
1145 /*
1146 * | ---- desired range ---- |
1147 * | state |
1148 * or
1149 * | ------------- state -------------- |
1150 *
1151 * We need to split the extent we found, and may flip bits on
1152 * second half.
1153 *
1154 * If the extent we found extends past our
1155 * range, we just split and search again. It'll get split
1156 * again the next time though.
1157 *
1158 * If the extent we found is inside our range, we set the
1159 * desired bit on it.
1160 */
1161 if (state->start < start) {
1162 prealloc = alloc_extent_state_atomic(prealloc);
Liu Bo1cf4ffd2011-12-07 20:08:40 -05001163 if (!prealloc) {
1164 err = -ENOMEM;
1165 goto out;
1166 }
Josef Bacik462d6fa2011-09-26 13:56:12 -04001167 err = split_state(tree, state, prealloc, start);
Jeff Mahoneyc2d904e2011-10-03 23:22:32 -04001168 if (err)
1169 extent_io_tree_panic(tree, err);
Josef Bacik462d6fa2011-09-26 13:56:12 -04001170 prealloc = NULL;
1171 if (err)
1172 goto out;
1173 if (state->end <= end) {
1174 set_state_bits(tree, state, &bits);
Josef Bacike6138872012-09-27 17:07:30 -04001175 cache_state(state, cached_state);
Liu Bod1ac6e42012-05-10 18:10:39 +08001176 state = clear_state_bit(tree, state, &clear_bits, 0);
Josef Bacik462d6fa2011-09-26 13:56:12 -04001177 if (last_end == (u64)-1)
1178 goto out;
1179 start = last_end + 1;
Liu Bod1ac6e42012-05-10 18:10:39 +08001180 if (start < end && state && state->start == start &&
1181 !need_resched())
1182 goto hit_next;
Josef Bacik462d6fa2011-09-26 13:56:12 -04001183 }
1184 goto search_again;
1185 }
1186 /*
1187 * | ---- desired range ---- |
1188 * | state | or | state |
1189 *
1190 * There's a hole, we need to insert something in it and
1191 * ignore the extent we found.
1192 */
1193 if (state->start > start) {
1194 u64 this_end;
1195 if (end < last_start)
1196 this_end = end;
1197 else
1198 this_end = last_start - 1;
1199
1200 prealloc = alloc_extent_state_atomic(prealloc);
Liu Bo1cf4ffd2011-12-07 20:08:40 -05001201 if (!prealloc) {
1202 err = -ENOMEM;
1203 goto out;
1204 }
Josef Bacik462d6fa2011-09-26 13:56:12 -04001205
1206 /*
1207 * Avoid to free 'prealloc' if it can be merged with
1208 * the later extent.
1209 */
1210 err = insert_state(tree, prealloc, start, this_end,
Filipe David Borba Manana12cfbad2013-11-26 15:41:47 +00001211 NULL, NULL, &bits);
Jeff Mahoneyc2d904e2011-10-03 23:22:32 -04001212 if (err)
1213 extent_io_tree_panic(tree, err);
Josef Bacike6138872012-09-27 17:07:30 -04001214 cache_state(prealloc, cached_state);
Josef Bacik462d6fa2011-09-26 13:56:12 -04001215 prealloc = NULL;
1216 start = this_end + 1;
1217 goto search_again;
1218 }
1219 /*
1220 * | ---- desired range ---- |
1221 * | state |
1222 * We need to split the extent, and set the bit
1223 * on the first half
1224 */
1225 if (state->start <= end && state->end > end) {
1226 prealloc = alloc_extent_state_atomic(prealloc);
Liu Bo1cf4ffd2011-12-07 20:08:40 -05001227 if (!prealloc) {
1228 err = -ENOMEM;
1229 goto out;
1230 }
Josef Bacik462d6fa2011-09-26 13:56:12 -04001231
1232 err = split_state(tree, state, prealloc, end + 1);
Jeff Mahoneyc2d904e2011-10-03 23:22:32 -04001233 if (err)
1234 extent_io_tree_panic(tree, err);
Josef Bacik462d6fa2011-09-26 13:56:12 -04001235
1236 set_state_bits(tree, prealloc, &bits);
Josef Bacike6138872012-09-27 17:07:30 -04001237 cache_state(prealloc, cached_state);
Josef Bacik462d6fa2011-09-26 13:56:12 -04001238 clear_state_bit(tree, prealloc, &clear_bits, 0);
Josef Bacik462d6fa2011-09-26 13:56:12 -04001239 prealloc = NULL;
1240 goto out;
1241 }
1242
1243 goto search_again;
1244
1245out:
1246 spin_unlock(&tree->lock);
1247 if (prealloc)
1248 free_extent_state(prealloc);
1249
1250 return err;
1251
1252search_again:
1253 if (start > end)
1254 goto out;
1255 spin_unlock(&tree->lock);
1256 if (mask & __GFP_WAIT)
1257 cond_resched();
Filipe Mananac8fd3de2014-10-13 12:28:39 +01001258 first_iteration = false;
Josef Bacik462d6fa2011-09-26 13:56:12 -04001259 goto again;
1260}
1261
Chris Masond1310b22008-01-24 16:13:08 -05001262/* wrappers around set/clear extent bit */
1263int set_extent_dirty(struct extent_io_tree *tree, u64 start, u64 end,
1264 gfp_t mask)
1265{
Jeff Mahoney3fbe5c02012-03-01 14:57:19 +01001266 return set_extent_bit(tree, start, end, EXTENT_DIRTY, NULL,
Chris Mason2c64c532009-09-02 15:04:12 -04001267 NULL, mask);
Chris Masond1310b22008-01-24 16:13:08 -05001268}
Chris Masond1310b22008-01-24 16:13:08 -05001269
1270int set_extent_bits(struct extent_io_tree *tree, u64 start, u64 end,
David Sterba41074882013-04-29 13:38:46 +00001271 unsigned long bits, gfp_t mask)
Chris Masond1310b22008-01-24 16:13:08 -05001272{
Jeff Mahoney3fbe5c02012-03-01 14:57:19 +01001273 return set_extent_bit(tree, start, end, bits, NULL,
Chris Mason2c64c532009-09-02 15:04:12 -04001274 NULL, mask);
Chris Masond1310b22008-01-24 16:13:08 -05001275}
Chris Masond1310b22008-01-24 16:13:08 -05001276
1277int clear_extent_bits(struct extent_io_tree *tree, u64 start, u64 end,
David Sterba41074882013-04-29 13:38:46 +00001278 unsigned long bits, gfp_t mask)
Chris Masond1310b22008-01-24 16:13:08 -05001279{
Chris Mason2c64c532009-09-02 15:04:12 -04001280 return clear_extent_bit(tree, start, end, bits, 0, 0, NULL, mask);
Chris Masond1310b22008-01-24 16:13:08 -05001281}
Chris Masond1310b22008-01-24 16:13:08 -05001282
1283int set_extent_delalloc(struct extent_io_tree *tree, u64 start, u64 end,
Josef Bacik2ac55d42010-02-03 19:33:23 +00001284 struct extent_state **cached_state, gfp_t mask)
Chris Masond1310b22008-01-24 16:13:08 -05001285{
1286 return set_extent_bit(tree, start, end,
Liu Bofee187d2011-09-29 15:55:28 +08001287 EXTENT_DELALLOC | EXTENT_UPTODATE,
Jeff Mahoney3fbe5c02012-03-01 14:57:19 +01001288 NULL, cached_state, mask);
Chris Masond1310b22008-01-24 16:13:08 -05001289}
Chris Masond1310b22008-01-24 16:13:08 -05001290
Liu Bo9e8a4a82012-09-05 19:10:51 -06001291int set_extent_defrag(struct extent_io_tree *tree, u64 start, u64 end,
1292 struct extent_state **cached_state, gfp_t mask)
1293{
1294 return set_extent_bit(tree, start, end,
1295 EXTENT_DELALLOC | EXTENT_UPTODATE | EXTENT_DEFRAG,
1296 NULL, cached_state, mask);
1297}
1298
Chris Masond1310b22008-01-24 16:13:08 -05001299int clear_extent_dirty(struct extent_io_tree *tree, u64 start, u64 end,
1300 gfp_t mask)
1301{
1302 return clear_extent_bit(tree, start, end,
Josef Bacik32c00af2009-10-08 13:34:05 -04001303 EXTENT_DIRTY | EXTENT_DELALLOC |
Yan, Zheng0ca1f7c2010-05-16 10:48:47 -04001304 EXTENT_DO_ACCOUNTING, 0, 0, NULL, mask);
Chris Masond1310b22008-01-24 16:13:08 -05001305}
Chris Masond1310b22008-01-24 16:13:08 -05001306
1307int set_extent_new(struct extent_io_tree *tree, u64 start, u64 end,
1308 gfp_t mask)
1309{
Jeff Mahoney3fbe5c02012-03-01 14:57:19 +01001310 return set_extent_bit(tree, start, end, EXTENT_NEW, NULL,
Chris Mason2c64c532009-09-02 15:04:12 -04001311 NULL, mask);
Chris Masond1310b22008-01-24 16:13:08 -05001312}
Chris Masond1310b22008-01-24 16:13:08 -05001313
Chris Masond1310b22008-01-24 16:13:08 -05001314int set_extent_uptodate(struct extent_io_tree *tree, u64 start, u64 end,
Arne Jansen507903b2011-04-06 10:02:20 +00001315 struct extent_state **cached_state, gfp_t mask)
Chris Masond1310b22008-01-24 16:13:08 -05001316{
Liu Bo6b67a322013-03-28 08:30:28 +00001317 return set_extent_bit(tree, start, end, EXTENT_UPTODATE, NULL,
Jeff Mahoney3fbe5c02012-03-01 14:57:19 +01001318 cached_state, mask);
Chris Masond1310b22008-01-24 16:13:08 -05001319}
Chris Masond1310b22008-01-24 16:13:08 -05001320
Josef Bacik5fd02042012-05-02 14:00:54 -04001321int clear_extent_uptodate(struct extent_io_tree *tree, u64 start, u64 end,
1322 struct extent_state **cached_state, gfp_t mask)
Chris Masond1310b22008-01-24 16:13:08 -05001323{
Chris Mason2c64c532009-09-02 15:04:12 -04001324 return clear_extent_bit(tree, start, end, EXTENT_UPTODATE, 0, 0,
Josef Bacik2ac55d42010-02-03 19:33:23 +00001325 cached_state, mask);
Chris Masond1310b22008-01-24 16:13:08 -05001326}
Chris Masond1310b22008-01-24 16:13:08 -05001327
Chris Masond352ac62008-09-29 15:18:18 -04001328/*
1329 * either insert or lock state struct between start and end use mask to tell
1330 * us if waiting is desired.
1331 */
Chris Mason1edbb732009-09-02 13:24:36 -04001332int lock_extent_bits(struct extent_io_tree *tree, u64 start, u64 end,
David Sterba41074882013-04-29 13:38:46 +00001333 unsigned long bits, struct extent_state **cached_state)
Chris Masond1310b22008-01-24 16:13:08 -05001334{
1335 int err;
1336 u64 failed_start;
1337 while (1) {
Jeff Mahoney3fbe5c02012-03-01 14:57:19 +01001338 err = __set_extent_bit(tree, start, end, EXTENT_LOCKED | bits,
1339 EXTENT_LOCKED, &failed_start,
1340 cached_state, GFP_NOFS);
Jeff Mahoneyd0082372012-03-01 14:57:19 +01001341 if (err == -EEXIST) {
Chris Masond1310b22008-01-24 16:13:08 -05001342 wait_extent_bit(tree, failed_start, end, EXTENT_LOCKED);
1343 start = failed_start;
Jeff Mahoneyd0082372012-03-01 14:57:19 +01001344 } else
Chris Masond1310b22008-01-24 16:13:08 -05001345 break;
Chris Masond1310b22008-01-24 16:13:08 -05001346 WARN_ON(start > end);
1347 }
1348 return err;
1349}
Chris Masond1310b22008-01-24 16:13:08 -05001350
Jeff Mahoneyd0082372012-03-01 14:57:19 +01001351int lock_extent(struct extent_io_tree *tree, u64 start, u64 end)
Chris Mason1edbb732009-09-02 13:24:36 -04001352{
Jeff Mahoneyd0082372012-03-01 14:57:19 +01001353 return lock_extent_bits(tree, start, end, 0, NULL);
Chris Mason1edbb732009-09-02 13:24:36 -04001354}
1355
Jeff Mahoneyd0082372012-03-01 14:57:19 +01001356int try_lock_extent(struct extent_io_tree *tree, u64 start, u64 end)
Josef Bacik25179202008-10-29 14:49:05 -04001357{
1358 int err;
1359 u64 failed_start;
1360
Jeff Mahoney3fbe5c02012-03-01 14:57:19 +01001361 err = __set_extent_bit(tree, start, end, EXTENT_LOCKED, EXTENT_LOCKED,
1362 &failed_start, NULL, GFP_NOFS);
Yan Zheng66435582008-10-30 14:19:50 -04001363 if (err == -EEXIST) {
1364 if (failed_start > start)
1365 clear_extent_bit(tree, start, failed_start - 1,
Jeff Mahoneyd0082372012-03-01 14:57:19 +01001366 EXTENT_LOCKED, 1, 0, NULL, GFP_NOFS);
Josef Bacik25179202008-10-29 14:49:05 -04001367 return 0;
Yan Zheng66435582008-10-30 14:19:50 -04001368 }
Josef Bacik25179202008-10-29 14:49:05 -04001369 return 1;
1370}
Josef Bacik25179202008-10-29 14:49:05 -04001371
Chris Mason2c64c532009-09-02 15:04:12 -04001372int unlock_extent_cached(struct extent_io_tree *tree, u64 start, u64 end,
1373 struct extent_state **cached, gfp_t mask)
1374{
1375 return clear_extent_bit(tree, start, end, EXTENT_LOCKED, 1, 0, cached,
1376 mask);
1377}
1378
Jeff Mahoneyd0082372012-03-01 14:57:19 +01001379int unlock_extent(struct extent_io_tree *tree, u64 start, u64 end)
Chris Masond1310b22008-01-24 16:13:08 -05001380{
Chris Mason2c64c532009-09-02 15:04:12 -04001381 return clear_extent_bit(tree, start, end, EXTENT_LOCKED, 1, 0, NULL,
Jeff Mahoneyd0082372012-03-01 14:57:19 +01001382 GFP_NOFS);
Chris Masond1310b22008-01-24 16:13:08 -05001383}
Chris Masond1310b22008-01-24 16:13:08 -05001384
Chris Mason4adaa612013-03-26 13:07:00 -04001385int extent_range_clear_dirty_for_io(struct inode *inode, u64 start, u64 end)
1386{
1387 unsigned long index = start >> PAGE_CACHE_SHIFT;
1388 unsigned long end_index = end >> PAGE_CACHE_SHIFT;
1389 struct page *page;
1390
1391 while (index <= end_index) {
1392 page = find_get_page(inode->i_mapping, index);
1393 BUG_ON(!page); /* Pages should be in the extent_io_tree */
1394 clear_page_dirty_for_io(page);
1395 page_cache_release(page);
1396 index++;
1397 }
1398 return 0;
1399}
1400
1401int extent_range_redirty_for_io(struct inode *inode, u64 start, u64 end)
1402{
1403 unsigned long index = start >> PAGE_CACHE_SHIFT;
1404 unsigned long end_index = end >> PAGE_CACHE_SHIFT;
1405 struct page *page;
1406
1407 while (index <= end_index) {
1408 page = find_get_page(inode->i_mapping, index);
1409 BUG_ON(!page); /* Pages should be in the extent_io_tree */
1410 account_page_redirty(page);
1411 __set_page_dirty_nobuffers(page);
1412 page_cache_release(page);
1413 index++;
1414 }
1415 return 0;
1416}
1417
Chris Masond1310b22008-01-24 16:13:08 -05001418/*
Chris Masond1310b22008-01-24 16:13:08 -05001419 * helper function to set both pages and extents in the tree writeback
1420 */
Christoph Hellwigb2950862008-12-02 09:54:17 -05001421static int set_range_writeback(struct extent_io_tree *tree, u64 start, u64 end)
Chris Masond1310b22008-01-24 16:13:08 -05001422{
1423 unsigned long index = start >> PAGE_CACHE_SHIFT;
1424 unsigned long end_index = end >> PAGE_CACHE_SHIFT;
1425 struct page *page;
1426
1427 while (index <= end_index) {
1428 page = find_get_page(tree->mapping, index);
Jeff Mahoney79787ea2012-03-12 16:03:00 +01001429 BUG_ON(!page); /* Pages should be in the extent_io_tree */
Chris Masond1310b22008-01-24 16:13:08 -05001430 set_page_writeback(page);
1431 page_cache_release(page);
1432 index++;
1433 }
Chris Masond1310b22008-01-24 16:13:08 -05001434 return 0;
1435}
Chris Masond1310b22008-01-24 16:13:08 -05001436
Chris Masond352ac62008-09-29 15:18:18 -04001437/* find the first state struct with 'bits' set after 'start', and
1438 * return it. tree->lock must be held. NULL will returned if
1439 * nothing was found after 'start'
1440 */
Eric Sandeen48a3b632013-04-25 20:41:01 +00001441static struct extent_state *
1442find_first_extent_bit_state(struct extent_io_tree *tree,
David Sterba41074882013-04-29 13:38:46 +00001443 u64 start, unsigned long bits)
Chris Masond7fc6402008-02-18 12:12:38 -05001444{
1445 struct rb_node *node;
1446 struct extent_state *state;
1447
1448 /*
1449 * this search will find all the extents that end after
1450 * our range starts.
1451 */
1452 node = tree_search(tree, start);
Chris Masond3977122009-01-05 21:25:51 -05001453 if (!node)
Chris Masond7fc6402008-02-18 12:12:38 -05001454 goto out;
Chris Masond7fc6402008-02-18 12:12:38 -05001455
Chris Masond3977122009-01-05 21:25:51 -05001456 while (1) {
Chris Masond7fc6402008-02-18 12:12:38 -05001457 state = rb_entry(node, struct extent_state, rb_node);
Chris Masond3977122009-01-05 21:25:51 -05001458 if (state->end >= start && (state->state & bits))
Chris Masond7fc6402008-02-18 12:12:38 -05001459 return state;
Chris Masond3977122009-01-05 21:25:51 -05001460
Chris Masond7fc6402008-02-18 12:12:38 -05001461 node = rb_next(node);
1462 if (!node)
1463 break;
1464 }
1465out:
1466 return NULL;
1467}
Chris Masond7fc6402008-02-18 12:12:38 -05001468
Chris Masond352ac62008-09-29 15:18:18 -04001469/*
Xiao Guangrong69261c42011-07-14 03:19:45 +00001470 * find the first offset in the io tree with 'bits' set. zero is
1471 * returned if we find something, and *start_ret and *end_ret are
1472 * set to reflect the state struct that was found.
1473 *
Wang Sheng-Hui477d7ea2012-04-06 14:35:47 +08001474 * If nothing was found, 1 is returned. If found something, return 0.
Xiao Guangrong69261c42011-07-14 03:19:45 +00001475 */
1476int find_first_extent_bit(struct extent_io_tree *tree, u64 start,
David Sterba41074882013-04-29 13:38:46 +00001477 u64 *start_ret, u64 *end_ret, unsigned long bits,
Josef Bacike6138872012-09-27 17:07:30 -04001478 struct extent_state **cached_state)
Xiao Guangrong69261c42011-07-14 03:19:45 +00001479{
1480 struct extent_state *state;
Josef Bacike6138872012-09-27 17:07:30 -04001481 struct rb_node *n;
Xiao Guangrong69261c42011-07-14 03:19:45 +00001482 int ret = 1;
1483
1484 spin_lock(&tree->lock);
Josef Bacike6138872012-09-27 17:07:30 -04001485 if (cached_state && *cached_state) {
1486 state = *cached_state;
Filipe Manana27a35072014-07-06 20:09:59 +01001487 if (state->end == start - 1 && extent_state_in_tree(state)) {
Josef Bacike6138872012-09-27 17:07:30 -04001488 n = rb_next(&state->rb_node);
1489 while (n) {
1490 state = rb_entry(n, struct extent_state,
1491 rb_node);
1492 if (state->state & bits)
1493 goto got_it;
1494 n = rb_next(n);
1495 }
1496 free_extent_state(*cached_state);
1497 *cached_state = NULL;
1498 goto out;
1499 }
1500 free_extent_state(*cached_state);
1501 *cached_state = NULL;
1502 }
1503
Xiao Guangrong69261c42011-07-14 03:19:45 +00001504 state = find_first_extent_bit_state(tree, start, bits);
Josef Bacike6138872012-09-27 17:07:30 -04001505got_it:
Xiao Guangrong69261c42011-07-14 03:19:45 +00001506 if (state) {
Filipe Mananae38e2ed2014-10-13 12:28:38 +01001507 cache_state_if_flags(state, cached_state, 0);
Xiao Guangrong69261c42011-07-14 03:19:45 +00001508 *start_ret = state->start;
1509 *end_ret = state->end;
1510 ret = 0;
1511 }
Josef Bacike6138872012-09-27 17:07:30 -04001512out:
Xiao Guangrong69261c42011-07-14 03:19:45 +00001513 spin_unlock(&tree->lock);
1514 return ret;
1515}
1516
1517/*
Chris Masond352ac62008-09-29 15:18:18 -04001518 * find a contiguous range of bytes in the file marked as delalloc, not
1519 * more than 'max_bytes'. start and end are used to return the range,
1520 *
1521 * 1 is returned if we find something, 0 if nothing was in the tree
1522 */
Chris Masonc8b97812008-10-29 14:49:59 -04001523static noinline u64 find_delalloc_range(struct extent_io_tree *tree,
Josef Bacikc2a128d2010-02-02 21:19:11 +00001524 u64 *start, u64 *end, u64 max_bytes,
1525 struct extent_state **cached_state)
Chris Masond1310b22008-01-24 16:13:08 -05001526{
1527 struct rb_node *node;
1528 struct extent_state *state;
1529 u64 cur_start = *start;
1530 u64 found = 0;
1531 u64 total_bytes = 0;
1532
Chris Masoncad321a2008-12-17 14:51:42 -05001533 spin_lock(&tree->lock);
Chris Masonc8b97812008-10-29 14:49:59 -04001534
Chris Masond1310b22008-01-24 16:13:08 -05001535 /*
1536 * this search will find all the extents that end after
1537 * our range starts.
1538 */
Chris Mason80ea96b2008-02-01 14:51:59 -05001539 node = tree_search(tree, cur_start);
Peter2b114d12008-04-01 11:21:40 -04001540 if (!node) {
Chris Mason3b951512008-04-17 11:29:12 -04001541 if (!found)
1542 *end = (u64)-1;
Chris Masond1310b22008-01-24 16:13:08 -05001543 goto out;
1544 }
1545
Chris Masond3977122009-01-05 21:25:51 -05001546 while (1) {
Chris Masond1310b22008-01-24 16:13:08 -05001547 state = rb_entry(node, struct extent_state, rb_node);
Zheng Yan5b21f2e2008-09-26 10:05:38 -04001548 if (found && (state->start != cur_start ||
1549 (state->state & EXTENT_BOUNDARY))) {
Chris Masond1310b22008-01-24 16:13:08 -05001550 goto out;
1551 }
1552 if (!(state->state & EXTENT_DELALLOC)) {
1553 if (!found)
1554 *end = state->end;
1555 goto out;
1556 }
Josef Bacikc2a128d2010-02-02 21:19:11 +00001557 if (!found) {
Chris Masond1310b22008-01-24 16:13:08 -05001558 *start = state->start;
Josef Bacikc2a128d2010-02-02 21:19:11 +00001559 *cached_state = state;
1560 atomic_inc(&state->refs);
1561 }
Chris Masond1310b22008-01-24 16:13:08 -05001562 found++;
1563 *end = state->end;
1564 cur_start = state->end + 1;
1565 node = rb_next(node);
Chris Masond1310b22008-01-24 16:13:08 -05001566 total_bytes += state->end - state->start + 1;
Josef Bacik7bf811a52013-10-07 22:11:09 -04001567 if (total_bytes >= max_bytes)
Josef Bacik573aeca2013-08-30 14:38:49 -04001568 break;
Josef Bacik573aeca2013-08-30 14:38:49 -04001569 if (!node)
Chris Masond1310b22008-01-24 16:13:08 -05001570 break;
1571 }
1572out:
Chris Masoncad321a2008-12-17 14:51:42 -05001573 spin_unlock(&tree->lock);
Chris Masond1310b22008-01-24 16:13:08 -05001574 return found;
1575}
1576
Jeff Mahoney143bede2012-03-01 14:56:26 +01001577static noinline void __unlock_for_delalloc(struct inode *inode,
1578 struct page *locked_page,
1579 u64 start, u64 end)
Chris Masonc8b97812008-10-29 14:49:59 -04001580{
1581 int ret;
1582 struct page *pages[16];
1583 unsigned long index = start >> PAGE_CACHE_SHIFT;
1584 unsigned long end_index = end >> PAGE_CACHE_SHIFT;
1585 unsigned long nr_pages = end_index - index + 1;
1586 int i;
1587
1588 if (index == locked_page->index && end_index == index)
Jeff Mahoney143bede2012-03-01 14:56:26 +01001589 return;
Chris Masonc8b97812008-10-29 14:49:59 -04001590
Chris Masond3977122009-01-05 21:25:51 -05001591 while (nr_pages > 0) {
Chris Masonc8b97812008-10-29 14:49:59 -04001592 ret = find_get_pages_contig(inode->i_mapping, index,
Chris Mason5b050f02008-11-11 09:34:41 -05001593 min_t(unsigned long, nr_pages,
1594 ARRAY_SIZE(pages)), pages);
Chris Masonc8b97812008-10-29 14:49:59 -04001595 for (i = 0; i < ret; i++) {
1596 if (pages[i] != locked_page)
1597 unlock_page(pages[i]);
1598 page_cache_release(pages[i]);
1599 }
1600 nr_pages -= ret;
1601 index += ret;
1602 cond_resched();
1603 }
Chris Masonc8b97812008-10-29 14:49:59 -04001604}
1605
1606static noinline int lock_delalloc_pages(struct inode *inode,
1607 struct page *locked_page,
1608 u64 delalloc_start,
1609 u64 delalloc_end)
1610{
1611 unsigned long index = delalloc_start >> PAGE_CACHE_SHIFT;
1612 unsigned long start_index = index;
1613 unsigned long end_index = delalloc_end >> PAGE_CACHE_SHIFT;
1614 unsigned long pages_locked = 0;
1615 struct page *pages[16];
1616 unsigned long nrpages;
1617 int ret;
1618 int i;
1619
1620 /* the caller is responsible for locking the start index */
1621 if (index == locked_page->index && index == end_index)
1622 return 0;
1623
1624 /* skip the page at the start index */
1625 nrpages = end_index - index + 1;
Chris Masond3977122009-01-05 21:25:51 -05001626 while (nrpages > 0) {
Chris Masonc8b97812008-10-29 14:49:59 -04001627 ret = find_get_pages_contig(inode->i_mapping, index,
Chris Mason5b050f02008-11-11 09:34:41 -05001628 min_t(unsigned long,
1629 nrpages, ARRAY_SIZE(pages)), pages);
Chris Masonc8b97812008-10-29 14:49:59 -04001630 if (ret == 0) {
1631 ret = -EAGAIN;
1632 goto done;
1633 }
1634 /* now we have an array of pages, lock them all */
1635 for (i = 0; i < ret; i++) {
1636 /*
1637 * the caller is taking responsibility for
1638 * locked_page
1639 */
Chris Mason771ed682008-11-06 22:02:51 -05001640 if (pages[i] != locked_page) {
Chris Masonc8b97812008-10-29 14:49:59 -04001641 lock_page(pages[i]);
Chris Masonf2b1c412008-11-10 07:31:30 -05001642 if (!PageDirty(pages[i]) ||
1643 pages[i]->mapping != inode->i_mapping) {
Chris Mason771ed682008-11-06 22:02:51 -05001644 ret = -EAGAIN;
1645 unlock_page(pages[i]);
1646 page_cache_release(pages[i]);
1647 goto done;
1648 }
1649 }
Chris Masonc8b97812008-10-29 14:49:59 -04001650 page_cache_release(pages[i]);
Chris Mason771ed682008-11-06 22:02:51 -05001651 pages_locked++;
Chris Masonc8b97812008-10-29 14:49:59 -04001652 }
Chris Masonc8b97812008-10-29 14:49:59 -04001653 nrpages -= ret;
1654 index += ret;
1655 cond_resched();
1656 }
1657 ret = 0;
1658done:
1659 if (ret && pages_locked) {
1660 __unlock_for_delalloc(inode, locked_page,
1661 delalloc_start,
1662 ((u64)(start_index + pages_locked - 1)) <<
1663 PAGE_CACHE_SHIFT);
1664 }
1665 return ret;
1666}
1667
1668/*
1669 * find a contiguous range of bytes in the file marked as delalloc, not
1670 * more than 'max_bytes'. start and end are used to return the range,
1671 *
1672 * 1 is returned if we find something, 0 if nothing was in the tree
1673 */
Josef Bacik294e30f2013-10-09 12:00:56 -04001674STATIC u64 find_lock_delalloc_range(struct inode *inode,
1675 struct extent_io_tree *tree,
1676 struct page *locked_page, u64 *start,
1677 u64 *end, u64 max_bytes)
Chris Masonc8b97812008-10-29 14:49:59 -04001678{
1679 u64 delalloc_start;
1680 u64 delalloc_end;
1681 u64 found;
Chris Mason9655d292009-09-02 15:22:30 -04001682 struct extent_state *cached_state = NULL;
Chris Masonc8b97812008-10-29 14:49:59 -04001683 int ret;
1684 int loops = 0;
1685
1686again:
1687 /* step one, find a bunch of delalloc bytes starting at start */
1688 delalloc_start = *start;
1689 delalloc_end = 0;
1690 found = find_delalloc_range(tree, &delalloc_start, &delalloc_end,
Josef Bacikc2a128d2010-02-02 21:19:11 +00001691 max_bytes, &cached_state);
Chris Mason70b99e62008-10-31 12:46:39 -04001692 if (!found || delalloc_end <= *start) {
Chris Masonc8b97812008-10-29 14:49:59 -04001693 *start = delalloc_start;
1694 *end = delalloc_end;
Josef Bacikc2a128d2010-02-02 21:19:11 +00001695 free_extent_state(cached_state);
Liu Bo385fe0b2013-10-01 23:49:49 +08001696 return 0;
Chris Masonc8b97812008-10-29 14:49:59 -04001697 }
1698
1699 /*
Chris Mason70b99e62008-10-31 12:46:39 -04001700 * start comes from the offset of locked_page. We have to lock
1701 * pages in order, so we can't process delalloc bytes before
1702 * locked_page
1703 */
Chris Masond3977122009-01-05 21:25:51 -05001704 if (delalloc_start < *start)
Chris Mason70b99e62008-10-31 12:46:39 -04001705 delalloc_start = *start;
Chris Mason70b99e62008-10-31 12:46:39 -04001706
1707 /*
Chris Masonc8b97812008-10-29 14:49:59 -04001708 * make sure to limit the number of pages we try to lock down
Chris Masonc8b97812008-10-29 14:49:59 -04001709 */
Josef Bacik7bf811a52013-10-07 22:11:09 -04001710 if (delalloc_end + 1 - delalloc_start > max_bytes)
1711 delalloc_end = delalloc_start + max_bytes - 1;
Chris Masond3977122009-01-05 21:25:51 -05001712
Chris Masonc8b97812008-10-29 14:49:59 -04001713 /* step two, lock all the pages after the page that has start */
1714 ret = lock_delalloc_pages(inode, locked_page,
1715 delalloc_start, delalloc_end);
1716 if (ret == -EAGAIN) {
1717 /* some of the pages are gone, lets avoid looping by
1718 * shortening the size of the delalloc range we're searching
1719 */
Chris Mason9655d292009-09-02 15:22:30 -04001720 free_extent_state(cached_state);
Chris Mason7d788742014-05-21 05:49:54 -07001721 cached_state = NULL;
Chris Masonc8b97812008-10-29 14:49:59 -04001722 if (!loops) {
Josef Bacik7bf811a52013-10-07 22:11:09 -04001723 max_bytes = PAGE_CACHE_SIZE;
Chris Masonc8b97812008-10-29 14:49:59 -04001724 loops = 1;
1725 goto again;
1726 } else {
1727 found = 0;
1728 goto out_failed;
1729 }
1730 }
Jeff Mahoney79787ea2012-03-12 16:03:00 +01001731 BUG_ON(ret); /* Only valid values are 0 and -EAGAIN */
Chris Masonc8b97812008-10-29 14:49:59 -04001732
1733 /* step three, lock the state bits for the whole range */
Jeff Mahoneyd0082372012-03-01 14:57:19 +01001734 lock_extent_bits(tree, delalloc_start, delalloc_end, 0, &cached_state);
Chris Masonc8b97812008-10-29 14:49:59 -04001735
1736 /* then test to make sure it is all still delalloc */
1737 ret = test_range_bit(tree, delalloc_start, delalloc_end,
Chris Mason9655d292009-09-02 15:22:30 -04001738 EXTENT_DELALLOC, 1, cached_state);
Chris Masonc8b97812008-10-29 14:49:59 -04001739 if (!ret) {
Chris Mason9655d292009-09-02 15:22:30 -04001740 unlock_extent_cached(tree, delalloc_start, delalloc_end,
1741 &cached_state, GFP_NOFS);
Chris Masonc8b97812008-10-29 14:49:59 -04001742 __unlock_for_delalloc(inode, locked_page,
1743 delalloc_start, delalloc_end);
1744 cond_resched();
1745 goto again;
1746 }
Chris Mason9655d292009-09-02 15:22:30 -04001747 free_extent_state(cached_state);
Chris Masonc8b97812008-10-29 14:49:59 -04001748 *start = delalloc_start;
1749 *end = delalloc_end;
1750out_failed:
1751 return found;
1752}
1753
Josef Bacikc2790a22013-07-29 11:20:47 -04001754int extent_clear_unlock_delalloc(struct inode *inode, u64 start, u64 end,
1755 struct page *locked_page,
1756 unsigned long clear_bits,
1757 unsigned long page_ops)
Chris Masonc8b97812008-10-29 14:49:59 -04001758{
Josef Bacikc2790a22013-07-29 11:20:47 -04001759 struct extent_io_tree *tree = &BTRFS_I(inode)->io_tree;
Chris Masonc8b97812008-10-29 14:49:59 -04001760 int ret;
1761 struct page *pages[16];
1762 unsigned long index = start >> PAGE_CACHE_SHIFT;
1763 unsigned long end_index = end >> PAGE_CACHE_SHIFT;
1764 unsigned long nr_pages = end_index - index + 1;
1765 int i;
Chris Mason771ed682008-11-06 22:02:51 -05001766
Chris Mason2c64c532009-09-02 15:04:12 -04001767 clear_extent_bit(tree, start, end, clear_bits, 1, 0, NULL, GFP_NOFS);
Josef Bacikc2790a22013-07-29 11:20:47 -04001768 if (page_ops == 0)
Chris Mason771ed682008-11-06 22:02:51 -05001769 return 0;
Chris Masonc8b97812008-10-29 14:49:59 -04001770
Filipe Manana704de492014-10-06 22:14:22 +01001771 if ((page_ops & PAGE_SET_ERROR) && nr_pages > 0)
1772 mapping_set_error(inode->i_mapping, -EIO);
1773
Chris Masond3977122009-01-05 21:25:51 -05001774 while (nr_pages > 0) {
Chris Masonc8b97812008-10-29 14:49:59 -04001775 ret = find_get_pages_contig(inode->i_mapping, index,
Chris Mason5b050f02008-11-11 09:34:41 -05001776 min_t(unsigned long,
1777 nr_pages, ARRAY_SIZE(pages)), pages);
Chris Masonc8b97812008-10-29 14:49:59 -04001778 for (i = 0; i < ret; i++) {
Chris Mason8b62b722009-09-02 16:53:46 -04001779
Josef Bacikc2790a22013-07-29 11:20:47 -04001780 if (page_ops & PAGE_SET_PRIVATE2)
Chris Mason8b62b722009-09-02 16:53:46 -04001781 SetPagePrivate2(pages[i]);
1782
Chris Masonc8b97812008-10-29 14:49:59 -04001783 if (pages[i] == locked_page) {
1784 page_cache_release(pages[i]);
1785 continue;
1786 }
Josef Bacikc2790a22013-07-29 11:20:47 -04001787 if (page_ops & PAGE_CLEAR_DIRTY)
Chris Masonc8b97812008-10-29 14:49:59 -04001788 clear_page_dirty_for_io(pages[i]);
Josef Bacikc2790a22013-07-29 11:20:47 -04001789 if (page_ops & PAGE_SET_WRITEBACK)
Chris Masonc8b97812008-10-29 14:49:59 -04001790 set_page_writeback(pages[i]);
Filipe Manana704de492014-10-06 22:14:22 +01001791 if (page_ops & PAGE_SET_ERROR)
1792 SetPageError(pages[i]);
Josef Bacikc2790a22013-07-29 11:20:47 -04001793 if (page_ops & PAGE_END_WRITEBACK)
Chris Masonc8b97812008-10-29 14:49:59 -04001794 end_page_writeback(pages[i]);
Josef Bacikc2790a22013-07-29 11:20:47 -04001795 if (page_ops & PAGE_UNLOCK)
Chris Mason771ed682008-11-06 22:02:51 -05001796 unlock_page(pages[i]);
Chris Masonc8b97812008-10-29 14:49:59 -04001797 page_cache_release(pages[i]);
1798 }
1799 nr_pages -= ret;
1800 index += ret;
1801 cond_resched();
1802 }
1803 return 0;
1804}
Chris Masonc8b97812008-10-29 14:49:59 -04001805
Chris Masond352ac62008-09-29 15:18:18 -04001806/*
1807 * count the number of bytes in the tree that have a given bit(s)
1808 * set. This can be fairly slow, except for EXTENT_DIRTY which is
1809 * cached. The total number found is returned.
1810 */
Chris Masond1310b22008-01-24 16:13:08 -05001811u64 count_range_bits(struct extent_io_tree *tree,
1812 u64 *start, u64 search_end, u64 max_bytes,
Chris Masonec29ed52011-02-23 16:23:20 -05001813 unsigned long bits, int contig)
Chris Masond1310b22008-01-24 16:13:08 -05001814{
1815 struct rb_node *node;
1816 struct extent_state *state;
1817 u64 cur_start = *start;
1818 u64 total_bytes = 0;
Chris Masonec29ed52011-02-23 16:23:20 -05001819 u64 last = 0;
Chris Masond1310b22008-01-24 16:13:08 -05001820 int found = 0;
1821
Dulshani Gunawardhanafae7f212013-10-31 10:30:08 +05301822 if (WARN_ON(search_end <= cur_start))
Chris Masond1310b22008-01-24 16:13:08 -05001823 return 0;
Chris Masond1310b22008-01-24 16:13:08 -05001824
Chris Masoncad321a2008-12-17 14:51:42 -05001825 spin_lock(&tree->lock);
Chris Masond1310b22008-01-24 16:13:08 -05001826 if (cur_start == 0 && bits == EXTENT_DIRTY) {
1827 total_bytes = tree->dirty_bytes;
1828 goto out;
1829 }
1830 /*
1831 * this search will find all the extents that end after
1832 * our range starts.
1833 */
Chris Mason80ea96b2008-02-01 14:51:59 -05001834 node = tree_search(tree, cur_start);
Chris Masond3977122009-01-05 21:25:51 -05001835 if (!node)
Chris Masond1310b22008-01-24 16:13:08 -05001836 goto out;
Chris Masond1310b22008-01-24 16:13:08 -05001837
Chris Masond3977122009-01-05 21:25:51 -05001838 while (1) {
Chris Masond1310b22008-01-24 16:13:08 -05001839 state = rb_entry(node, struct extent_state, rb_node);
1840 if (state->start > search_end)
1841 break;
Chris Masonec29ed52011-02-23 16:23:20 -05001842 if (contig && found && state->start > last + 1)
1843 break;
1844 if (state->end >= cur_start && (state->state & bits) == bits) {
Chris Masond1310b22008-01-24 16:13:08 -05001845 total_bytes += min(search_end, state->end) + 1 -
1846 max(cur_start, state->start);
1847 if (total_bytes >= max_bytes)
1848 break;
1849 if (!found) {
Josef Bacikaf60bed2011-05-04 11:11:17 -04001850 *start = max(cur_start, state->start);
Chris Masond1310b22008-01-24 16:13:08 -05001851 found = 1;
1852 }
Chris Masonec29ed52011-02-23 16:23:20 -05001853 last = state->end;
1854 } else if (contig && found) {
1855 break;
Chris Masond1310b22008-01-24 16:13:08 -05001856 }
1857 node = rb_next(node);
1858 if (!node)
1859 break;
1860 }
1861out:
Chris Masoncad321a2008-12-17 14:51:42 -05001862 spin_unlock(&tree->lock);
Chris Masond1310b22008-01-24 16:13:08 -05001863 return total_bytes;
1864}
Christoph Hellwigb2950862008-12-02 09:54:17 -05001865
Chris Masond352ac62008-09-29 15:18:18 -04001866/*
1867 * set the private field for a given byte offset in the tree. If there isn't
1868 * an extent_state there already, this does nothing.
1869 */
Sergei Trofimovich171170c2013-08-14 23:27:46 +03001870static int set_state_private(struct extent_io_tree *tree, u64 start, u64 private)
Chris Masond1310b22008-01-24 16:13:08 -05001871{
1872 struct rb_node *node;
1873 struct extent_state *state;
1874 int ret = 0;
1875
Chris Masoncad321a2008-12-17 14:51:42 -05001876 spin_lock(&tree->lock);
Chris Masond1310b22008-01-24 16:13:08 -05001877 /*
1878 * this search will find all the extents that end after
1879 * our range starts.
1880 */
Chris Mason80ea96b2008-02-01 14:51:59 -05001881 node = tree_search(tree, start);
Peter2b114d12008-04-01 11:21:40 -04001882 if (!node) {
Chris Masond1310b22008-01-24 16:13:08 -05001883 ret = -ENOENT;
1884 goto out;
1885 }
1886 state = rb_entry(node, struct extent_state, rb_node);
1887 if (state->start != start) {
1888 ret = -ENOENT;
1889 goto out;
1890 }
1891 state->private = private;
1892out:
Chris Masoncad321a2008-12-17 14:51:42 -05001893 spin_unlock(&tree->lock);
Chris Masond1310b22008-01-24 16:13:08 -05001894 return ret;
1895}
1896
1897int get_state_private(struct extent_io_tree *tree, u64 start, u64 *private)
1898{
1899 struct rb_node *node;
1900 struct extent_state *state;
1901 int ret = 0;
1902
Chris Masoncad321a2008-12-17 14:51:42 -05001903 spin_lock(&tree->lock);
Chris Masond1310b22008-01-24 16:13:08 -05001904 /*
1905 * this search will find all the extents that end after
1906 * our range starts.
1907 */
Chris Mason80ea96b2008-02-01 14:51:59 -05001908 node = tree_search(tree, start);
Peter2b114d12008-04-01 11:21:40 -04001909 if (!node) {
Chris Masond1310b22008-01-24 16:13:08 -05001910 ret = -ENOENT;
1911 goto out;
1912 }
1913 state = rb_entry(node, struct extent_state, rb_node);
1914 if (state->start != start) {
1915 ret = -ENOENT;
1916 goto out;
1917 }
1918 *private = state->private;
1919out:
Chris Masoncad321a2008-12-17 14:51:42 -05001920 spin_unlock(&tree->lock);
Chris Masond1310b22008-01-24 16:13:08 -05001921 return ret;
1922}
1923
1924/*
1925 * searches a range in the state tree for a given mask.
Chris Mason70dec802008-01-29 09:59:12 -05001926 * If 'filled' == 1, this returns 1 only if every extent in the tree
Chris Masond1310b22008-01-24 16:13:08 -05001927 * has the bits set. Otherwise, 1 is returned if any bit in the
1928 * range is found set.
1929 */
1930int test_range_bit(struct extent_io_tree *tree, u64 start, u64 end,
David Sterba41074882013-04-29 13:38:46 +00001931 unsigned long bits, int filled, struct extent_state *cached)
Chris Masond1310b22008-01-24 16:13:08 -05001932{
1933 struct extent_state *state = NULL;
1934 struct rb_node *node;
1935 int bitset = 0;
Chris Masond1310b22008-01-24 16:13:08 -05001936
Chris Masoncad321a2008-12-17 14:51:42 -05001937 spin_lock(&tree->lock);
Filipe Manana27a35072014-07-06 20:09:59 +01001938 if (cached && extent_state_in_tree(cached) && cached->start <= start &&
Josef Bacikdf98b6e2011-06-20 14:53:48 -04001939 cached->end > start)
Chris Mason9655d292009-09-02 15:22:30 -04001940 node = &cached->rb_node;
1941 else
1942 node = tree_search(tree, start);
Chris Masond1310b22008-01-24 16:13:08 -05001943 while (node && start <= end) {
1944 state = rb_entry(node, struct extent_state, rb_node);
1945
1946 if (filled && state->start > start) {
1947 bitset = 0;
1948 break;
1949 }
1950
1951 if (state->start > end)
1952 break;
1953
1954 if (state->state & bits) {
1955 bitset = 1;
1956 if (!filled)
1957 break;
1958 } else if (filled) {
1959 bitset = 0;
1960 break;
1961 }
Chris Mason46562cec2009-09-23 20:23:16 -04001962
1963 if (state->end == (u64)-1)
1964 break;
1965
Chris Masond1310b22008-01-24 16:13:08 -05001966 start = state->end + 1;
1967 if (start > end)
1968 break;
1969 node = rb_next(node);
1970 if (!node) {
1971 if (filled)
1972 bitset = 0;
1973 break;
1974 }
1975 }
Chris Masoncad321a2008-12-17 14:51:42 -05001976 spin_unlock(&tree->lock);
Chris Masond1310b22008-01-24 16:13:08 -05001977 return bitset;
1978}
Chris Masond1310b22008-01-24 16:13:08 -05001979
1980/*
1981 * helper function to set a given page up to date if all the
1982 * extents in the tree for that page are up to date
1983 */
Jeff Mahoney143bede2012-03-01 14:56:26 +01001984static void check_page_uptodate(struct extent_io_tree *tree, struct page *page)
Chris Masond1310b22008-01-24 16:13:08 -05001985{
Miao Xie4eee4fa2012-12-21 09:17:45 +00001986 u64 start = page_offset(page);
Chris Masond1310b22008-01-24 16:13:08 -05001987 u64 end = start + PAGE_CACHE_SIZE - 1;
Chris Mason9655d292009-09-02 15:22:30 -04001988 if (test_range_bit(tree, start, end, EXTENT_UPTODATE, 1, NULL))
Chris Masond1310b22008-01-24 16:13:08 -05001989 SetPageUptodate(page);
Chris Masond1310b22008-01-24 16:13:08 -05001990}
1991
Miao Xie8b110e32014-09-12 18:44:03 +08001992int free_io_failure(struct inode *inode, struct io_failure_record *rec)
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02001993{
1994 int ret;
1995 int err = 0;
1996 struct extent_io_tree *failure_tree = &BTRFS_I(inode)->io_failure_tree;
1997
1998 set_state_private(failure_tree, rec->start, 0);
1999 ret = clear_extent_bits(failure_tree, rec->start,
2000 rec->start + rec->len - 1,
2001 EXTENT_LOCKED | EXTENT_DIRTY, GFP_NOFS);
2002 if (ret)
2003 err = ret;
2004
David Woodhouse53b381b2013-01-29 18:40:14 -05002005 ret = clear_extent_bits(&BTRFS_I(inode)->io_tree, rec->start,
2006 rec->start + rec->len - 1,
2007 EXTENT_DAMAGED, GFP_NOFS);
2008 if (ret && !err)
2009 err = ret;
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002010
2011 kfree(rec);
2012 return err;
2013}
2014
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002015/*
2016 * this bypasses the standard btrfs submit functions deliberately, as
2017 * the standard behavior is to write all copies in a raid setup. here we only
2018 * want to write the one bad copy. so we do the mapping for ourselves and issue
2019 * submit_bio directly.
Stefan Behrens3ec706c2012-11-05 15:46:42 +01002020 * to avoid any synchronization issues, wait for the data after writing, which
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002021 * actually prevents the read that triggered the error from finishing.
2022 * currently, there can be no more than two copies of every data bit. thus,
2023 * exactly one rewrite is required.
2024 */
Miao Xie1203b682014-09-12 18:44:01 +08002025int repair_io_failure(struct inode *inode, u64 start, u64 length, u64 logical,
2026 struct page *page, unsigned int pg_offset, int mirror_num)
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002027{
Miao Xie1203b682014-09-12 18:44:01 +08002028 struct btrfs_fs_info *fs_info = BTRFS_I(inode)->root->fs_info;
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002029 struct bio *bio;
2030 struct btrfs_device *dev;
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002031 u64 map_length = 0;
2032 u64 sector;
2033 struct btrfs_bio *bbio = NULL;
David Woodhouse53b381b2013-01-29 18:40:14 -05002034 struct btrfs_mapping_tree *map_tree = &fs_info->mapping_tree;
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002035 int ret;
2036
Ilya Dryomov908960c2013-11-03 19:06:39 +02002037 ASSERT(!(fs_info->sb->s_flags & MS_RDONLY));
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002038 BUG_ON(!mirror_num);
2039
David Woodhouse53b381b2013-01-29 18:40:14 -05002040 /* we can't repair anything in raid56 yet */
2041 if (btrfs_is_parity_mirror(map_tree, logical, length, mirror_num))
2042 return 0;
2043
Chris Mason9be33952013-05-17 18:30:14 -04002044 bio = btrfs_io_bio_alloc(GFP_NOFS, 1);
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002045 if (!bio)
2046 return -EIO;
Kent Overstreet4f024f32013-10-11 15:44:27 -07002047 bio->bi_iter.bi_size = 0;
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002048 map_length = length;
2049
Stefan Behrens3ec706c2012-11-05 15:46:42 +01002050 ret = btrfs_map_block(fs_info, WRITE, logical,
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002051 &map_length, &bbio, mirror_num);
2052 if (ret) {
2053 bio_put(bio);
2054 return -EIO;
2055 }
2056 BUG_ON(mirror_num != bbio->mirror_num);
2057 sector = bbio->stripes[mirror_num-1].physical >> 9;
Kent Overstreet4f024f32013-10-11 15:44:27 -07002058 bio->bi_iter.bi_sector = sector;
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002059 dev = bbio->stripes[mirror_num-1].dev;
2060 kfree(bbio);
2061 if (!dev || !dev->bdev || !dev->writeable) {
2062 bio_put(bio);
2063 return -EIO;
2064 }
2065 bio->bi_bdev = dev->bdev;
Miao Xieffdd2012014-09-12 18:44:00 +08002066 bio_add_page(bio, page, length, pg_offset);
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002067
Kent Overstreet33879d42013-11-23 22:33:32 -08002068 if (btrfsic_submit_bio_wait(WRITE_SYNC, bio)) {
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002069 /* try to remap that extent elsewhere? */
2070 bio_put(bio);
Stefan Behrens442a4f62012-05-25 16:06:08 +02002071 btrfs_dev_stat_inc_and_print(dev, BTRFS_DEV_STAT_WRITE_ERRS);
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002072 return -EIO;
2073 }
2074
Frank Holtonefe120a2013-12-20 11:37:06 -05002075 printk_ratelimited_in_rcu(KERN_INFO
Miao Xie1203b682014-09-12 18:44:01 +08002076 "BTRFS: read error corrected: ino %llu off %llu (dev %s sector %llu)\n",
2077 btrfs_ino(inode), start,
2078 rcu_str_deref(dev->name), sector);
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002079 bio_put(bio);
2080 return 0;
2081}
2082
Josef Bacikea466792012-03-26 21:57:36 -04002083int repair_eb_io_failure(struct btrfs_root *root, struct extent_buffer *eb,
2084 int mirror_num)
2085{
Josef Bacikea466792012-03-26 21:57:36 -04002086 u64 start = eb->start;
2087 unsigned long i, num_pages = num_extent_pages(eb->start, eb->len);
Chris Masond95603b2012-04-12 15:55:15 -04002088 int ret = 0;
Josef Bacikea466792012-03-26 21:57:36 -04002089
Ilya Dryomov908960c2013-11-03 19:06:39 +02002090 if (root->fs_info->sb->s_flags & MS_RDONLY)
2091 return -EROFS;
2092
Josef Bacikea466792012-03-26 21:57:36 -04002093 for (i = 0; i < num_pages; i++) {
David Sterbafb85fc92014-07-31 01:03:53 +02002094 struct page *p = eb->pages[i];
Miao Xie1203b682014-09-12 18:44:01 +08002095
2096 ret = repair_io_failure(root->fs_info->btree_inode, start,
2097 PAGE_CACHE_SIZE, start, p,
2098 start - page_offset(p), mirror_num);
Josef Bacikea466792012-03-26 21:57:36 -04002099 if (ret)
2100 break;
2101 start += PAGE_CACHE_SIZE;
2102 }
2103
2104 return ret;
2105}
2106
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002107/*
2108 * each time an IO finishes, we do a fast check in the IO failure tree
2109 * to see if we need to process or clean up an io_failure_record
2110 */
Miao Xie8b110e32014-09-12 18:44:03 +08002111int clean_io_failure(struct inode *inode, u64 start, struct page *page,
2112 unsigned int pg_offset)
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002113{
2114 u64 private;
2115 u64 private_failure;
2116 struct io_failure_record *failrec;
Ilya Dryomov908960c2013-11-03 19:06:39 +02002117 struct btrfs_fs_info *fs_info = BTRFS_I(inode)->root->fs_info;
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002118 struct extent_state *state;
2119 int num_copies;
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002120 int ret;
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002121
2122 private = 0;
2123 ret = count_range_bits(&BTRFS_I(inode)->io_failure_tree, &private,
2124 (u64)-1, 1, EXTENT_DIRTY, 0);
2125 if (!ret)
2126 return 0;
2127
2128 ret = get_state_private(&BTRFS_I(inode)->io_failure_tree, start,
2129 &private_failure);
2130 if (ret)
2131 return 0;
2132
2133 failrec = (struct io_failure_record *)(unsigned long) private_failure;
2134 BUG_ON(!failrec->this_mirror);
2135
2136 if (failrec->in_validation) {
2137 /* there was no real error, just free the record */
2138 pr_debug("clean_io_failure: freeing dummy error at %llu\n",
2139 failrec->start);
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002140 goto out;
2141 }
Ilya Dryomov908960c2013-11-03 19:06:39 +02002142 if (fs_info->sb->s_flags & MS_RDONLY)
2143 goto out;
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002144
2145 spin_lock(&BTRFS_I(inode)->io_tree.lock);
2146 state = find_first_extent_bit_state(&BTRFS_I(inode)->io_tree,
2147 failrec->start,
2148 EXTENT_LOCKED);
2149 spin_unlock(&BTRFS_I(inode)->io_tree.lock);
2150
Miao Xie883d0de2013-07-25 19:22:35 +08002151 if (state && state->start <= failrec->start &&
2152 state->end >= failrec->start + failrec->len - 1) {
Stefan Behrens3ec706c2012-11-05 15:46:42 +01002153 num_copies = btrfs_num_copies(fs_info, failrec->logical,
2154 failrec->len);
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002155 if (num_copies > 1) {
Miao Xie1203b682014-09-12 18:44:01 +08002156 repair_io_failure(inode, start, failrec->len,
Miao Xie454ff3d2014-09-12 18:43:58 +08002157 failrec->logical, page,
Miao Xie1203b682014-09-12 18:44:01 +08002158 pg_offset, failrec->failed_mirror);
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002159 }
2160 }
2161
2162out:
Miao Xie454ff3d2014-09-12 18:43:58 +08002163 free_io_failure(inode, failrec);
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002164
Miao Xie454ff3d2014-09-12 18:43:58 +08002165 return 0;
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002166}
2167
Miao Xief6124962014-09-12 18:44:04 +08002168/*
2169 * Can be called when
2170 * - hold extent lock
2171 * - under ordered extent
2172 * - the inode is freeing
2173 */
2174void btrfs_free_io_failure_record(struct inode *inode, u64 start, u64 end)
2175{
2176 struct extent_io_tree *failure_tree = &BTRFS_I(inode)->io_failure_tree;
2177 struct io_failure_record *failrec;
2178 struct extent_state *state, *next;
2179
2180 if (RB_EMPTY_ROOT(&failure_tree->state))
2181 return;
2182
2183 spin_lock(&failure_tree->lock);
2184 state = find_first_extent_bit_state(failure_tree, start, EXTENT_DIRTY);
2185 while (state) {
2186 if (state->start > end)
2187 break;
2188
2189 ASSERT(state->end <= end);
2190
2191 next = next_state(state);
2192
2193 failrec = (struct io_failure_record *)state->private;
2194 free_extent_state(state);
2195 kfree(failrec);
2196
2197 state = next;
2198 }
2199 spin_unlock(&failure_tree->lock);
2200}
2201
Miao Xie2fe63032014-09-12 18:43:59 +08002202int btrfs_get_io_failure_record(struct inode *inode, u64 start, u64 end,
2203 struct io_failure_record **failrec_ret)
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002204{
Miao Xie2fe63032014-09-12 18:43:59 +08002205 struct io_failure_record *failrec;
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002206 u64 private;
2207 struct extent_map *em;
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002208 struct extent_io_tree *failure_tree = &BTRFS_I(inode)->io_failure_tree;
2209 struct extent_io_tree *tree = &BTRFS_I(inode)->io_tree;
2210 struct extent_map_tree *em_tree = &BTRFS_I(inode)->extent_tree;
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002211 int ret;
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002212 u64 logical;
2213
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002214 ret = get_state_private(failure_tree, start, &private);
2215 if (ret) {
2216 failrec = kzalloc(sizeof(*failrec), GFP_NOFS);
2217 if (!failrec)
2218 return -ENOMEM;
Miao Xie2fe63032014-09-12 18:43:59 +08002219
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002220 failrec->start = start;
2221 failrec->len = end - start + 1;
2222 failrec->this_mirror = 0;
2223 failrec->bio_flags = 0;
2224 failrec->in_validation = 0;
2225
2226 read_lock(&em_tree->lock);
2227 em = lookup_extent_mapping(em_tree, start, failrec->len);
2228 if (!em) {
2229 read_unlock(&em_tree->lock);
2230 kfree(failrec);
2231 return -EIO;
2232 }
2233
Filipe David Borba Manana68ba9902013-11-25 03:22:07 +00002234 if (em->start > start || em->start + em->len <= start) {
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002235 free_extent_map(em);
2236 em = NULL;
2237 }
2238 read_unlock(&em_tree->lock);
Tsutomu Itoh7a2d6a62012-10-01 03:07:15 -06002239 if (!em) {
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002240 kfree(failrec);
2241 return -EIO;
2242 }
Miao Xie2fe63032014-09-12 18:43:59 +08002243
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002244 logical = start - em->start;
2245 logical = em->block_start + logical;
2246 if (test_bit(EXTENT_FLAG_COMPRESSED, &em->flags)) {
2247 logical = em->block_start;
2248 failrec->bio_flags = EXTENT_BIO_COMPRESSED;
2249 extent_set_compress_type(&failrec->bio_flags,
2250 em->compress_type);
2251 }
Miao Xie2fe63032014-09-12 18:43:59 +08002252
2253 pr_debug("Get IO Failure Record: (new) logical=%llu, start=%llu, len=%llu\n",
2254 logical, start, failrec->len);
2255
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002256 failrec->logical = logical;
2257 free_extent_map(em);
2258
2259 /* set the bits in the private failure tree */
2260 ret = set_extent_bits(failure_tree, start, end,
2261 EXTENT_LOCKED | EXTENT_DIRTY, GFP_NOFS);
2262 if (ret >= 0)
2263 ret = set_state_private(failure_tree, start,
2264 (u64)(unsigned long)failrec);
2265 /* set the bits in the inode's tree */
2266 if (ret >= 0)
2267 ret = set_extent_bits(tree, start, end, EXTENT_DAMAGED,
2268 GFP_NOFS);
2269 if (ret < 0) {
2270 kfree(failrec);
2271 return ret;
2272 }
2273 } else {
2274 failrec = (struct io_failure_record *)(unsigned long)private;
Miao Xie2fe63032014-09-12 18:43:59 +08002275 pr_debug("Get IO Failure Record: (found) logical=%llu, start=%llu, len=%llu, validation=%d\n",
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002276 failrec->logical, failrec->start, failrec->len,
2277 failrec->in_validation);
2278 /*
2279 * when data can be on disk more than twice, add to failrec here
2280 * (e.g. with a list for failed_mirror) to make
2281 * clean_io_failure() clean all those errors at once.
2282 */
2283 }
Miao Xie2fe63032014-09-12 18:43:59 +08002284
2285 *failrec_ret = failrec;
2286
2287 return 0;
2288}
2289
2290int btrfs_check_repairable(struct inode *inode, struct bio *failed_bio,
2291 struct io_failure_record *failrec, int failed_mirror)
2292{
2293 int num_copies;
2294
Stefan Behrens5d964052012-11-05 14:59:07 +01002295 num_copies = btrfs_num_copies(BTRFS_I(inode)->root->fs_info,
2296 failrec->logical, failrec->len);
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002297 if (num_copies == 1) {
2298 /*
2299 * we only have a single copy of the data, so don't bother with
2300 * all the retry and error correction code that follows. no
2301 * matter what the error is, it is very likely to persist.
2302 */
Miao Xie2fe63032014-09-12 18:43:59 +08002303 pr_debug("Check Repairable: cannot repair, num_copies=%d, next_mirror %d, failed_mirror %d\n",
Miao Xie09a7f7a2013-07-25 19:22:32 +08002304 num_copies, failrec->this_mirror, failed_mirror);
Miao Xie2fe63032014-09-12 18:43:59 +08002305 return 0;
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002306 }
2307
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002308 /*
2309 * there are two premises:
2310 * a) deliver good data to the caller
2311 * b) correct the bad sectors on disk
2312 */
2313 if (failed_bio->bi_vcnt > 1) {
2314 /*
2315 * to fulfill b), we need to know the exact failing sectors, as
2316 * we don't want to rewrite any more than the failed ones. thus,
2317 * we need separate read requests for the failed bio
2318 *
2319 * if the following BUG_ON triggers, our validation request got
2320 * merged. we need separate requests for our algorithm to work.
2321 */
2322 BUG_ON(failrec->in_validation);
2323 failrec->in_validation = 1;
2324 failrec->this_mirror = failed_mirror;
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002325 } else {
2326 /*
2327 * we're ready to fulfill a) and b) alongside. get a good copy
2328 * of the failed sector and if we succeed, we have setup
2329 * everything for repair_io_failure to do the rest for us.
2330 */
2331 if (failrec->in_validation) {
2332 BUG_ON(failrec->this_mirror != failed_mirror);
2333 failrec->in_validation = 0;
2334 failrec->this_mirror = 0;
2335 }
2336 failrec->failed_mirror = failed_mirror;
2337 failrec->this_mirror++;
2338 if (failrec->this_mirror == failed_mirror)
2339 failrec->this_mirror++;
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002340 }
2341
Miao Xiefacc8a222013-07-25 19:22:34 +08002342 if (failrec->this_mirror > num_copies) {
Miao Xie2fe63032014-09-12 18:43:59 +08002343 pr_debug("Check Repairable: (fail) num_copies=%d, next_mirror %d, failed_mirror %d\n",
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002344 num_copies, failrec->this_mirror, failed_mirror);
Miao Xie2fe63032014-09-12 18:43:59 +08002345 return 0;
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002346 }
2347
Miao Xie2fe63032014-09-12 18:43:59 +08002348 return 1;
2349}
2350
2351
2352struct bio *btrfs_create_repair_bio(struct inode *inode, struct bio *failed_bio,
2353 struct io_failure_record *failrec,
2354 struct page *page, int pg_offset, int icsum,
Miao Xie8b110e32014-09-12 18:44:03 +08002355 bio_end_io_t *endio_func, void *data)
Miao Xie2fe63032014-09-12 18:43:59 +08002356{
2357 struct bio *bio;
2358 struct btrfs_io_bio *btrfs_failed_bio;
2359 struct btrfs_io_bio *btrfs_bio;
2360
Chris Mason9be33952013-05-17 18:30:14 -04002361 bio = btrfs_io_bio_alloc(GFP_NOFS, 1);
Miao Xie2fe63032014-09-12 18:43:59 +08002362 if (!bio)
2363 return NULL;
2364
2365 bio->bi_end_io = endio_func;
Kent Overstreet4f024f32013-10-11 15:44:27 -07002366 bio->bi_iter.bi_sector = failrec->logical >> 9;
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002367 bio->bi_bdev = BTRFS_I(inode)->root->fs_info->fs_devices->latest_bdev;
Kent Overstreet4f024f32013-10-11 15:44:27 -07002368 bio->bi_iter.bi_size = 0;
Miao Xie8b110e32014-09-12 18:44:03 +08002369 bio->bi_private = data;
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002370
Miao Xiefacc8a222013-07-25 19:22:34 +08002371 btrfs_failed_bio = btrfs_io_bio(failed_bio);
2372 if (btrfs_failed_bio->csum) {
2373 struct btrfs_fs_info *fs_info = BTRFS_I(inode)->root->fs_info;
2374 u16 csum_size = btrfs_super_csum_size(fs_info->super_copy);
2375
2376 btrfs_bio = btrfs_io_bio(bio);
2377 btrfs_bio->csum = btrfs_bio->csum_inline;
Miao Xie2fe63032014-09-12 18:43:59 +08002378 icsum *= csum_size;
2379 memcpy(btrfs_bio->csum, btrfs_failed_bio->csum + icsum,
Miao Xiefacc8a222013-07-25 19:22:34 +08002380 csum_size);
2381 }
2382
Miao Xie2fe63032014-09-12 18:43:59 +08002383 bio_add_page(bio, page, failrec->len, pg_offset);
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002384
Miao Xie2fe63032014-09-12 18:43:59 +08002385 return bio;
2386}
2387
2388/*
2389 * this is a generic handler for readpage errors (default
2390 * readpage_io_failed_hook). if other copies exist, read those and write back
2391 * good data to the failed position. does not investigate in remapping the
2392 * failed extent elsewhere, hoping the device will be smart enough to do this as
2393 * needed
2394 */
2395
2396static int bio_readpage_error(struct bio *failed_bio, u64 phy_offset,
2397 struct page *page, u64 start, u64 end,
2398 int failed_mirror)
2399{
2400 struct io_failure_record *failrec;
2401 struct inode *inode = page->mapping->host;
2402 struct extent_io_tree *tree = &BTRFS_I(inode)->io_tree;
2403 struct bio *bio;
2404 int read_mode;
2405 int ret;
2406
2407 BUG_ON(failed_bio->bi_rw & REQ_WRITE);
2408
2409 ret = btrfs_get_io_failure_record(inode, start, end, &failrec);
2410 if (ret)
2411 return ret;
2412
2413 ret = btrfs_check_repairable(inode, failed_bio, failrec, failed_mirror);
2414 if (!ret) {
2415 free_io_failure(inode, failrec);
2416 return -EIO;
2417 }
2418
2419 if (failed_bio->bi_vcnt > 1)
2420 read_mode = READ_SYNC | REQ_FAILFAST_DEV;
2421 else
2422 read_mode = READ_SYNC;
2423
2424 phy_offset >>= inode->i_sb->s_blocksize_bits;
2425 bio = btrfs_create_repair_bio(inode, failed_bio, failrec, page,
2426 start - page_offset(page),
Miao Xie8b110e32014-09-12 18:44:03 +08002427 (int)phy_offset, failed_bio->bi_end_io,
2428 NULL);
Miao Xie2fe63032014-09-12 18:43:59 +08002429 if (!bio) {
2430 free_io_failure(inode, failrec);
2431 return -EIO;
2432 }
2433
2434 pr_debug("Repair Read Error: submitting new read[%#x] to this_mirror=%d, in_validation=%d\n",
2435 read_mode, failrec->this_mirror, failrec->in_validation);
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002436
Tsutomu Itoh013bd4c2012-02-16 10:11:40 +09002437 ret = tree->ops->submit_bio_hook(inode, read_mode, bio,
2438 failrec->this_mirror,
2439 failrec->bio_flags, 0);
Miao Xie6c387ab2014-09-12 18:43:57 +08002440 if (ret) {
Miao Xie454ff3d2014-09-12 18:43:58 +08002441 free_io_failure(inode, failrec);
Miao Xie6c387ab2014-09-12 18:43:57 +08002442 bio_put(bio);
2443 }
2444
Tsutomu Itoh013bd4c2012-02-16 10:11:40 +09002445 return ret;
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002446}
2447
Chris Masond1310b22008-01-24 16:13:08 -05002448/* lots and lots of room for performance fixes in the end_bio funcs */
2449
Jeff Mahoney87826df2012-02-15 16:23:57 +01002450int end_extent_writepage(struct page *page, int err, u64 start, u64 end)
2451{
2452 int uptodate = (err == 0);
2453 struct extent_io_tree *tree;
Eric Sandeen3e2426b2014-06-12 00:39:58 -05002454 int ret = 0;
Jeff Mahoney87826df2012-02-15 16:23:57 +01002455
2456 tree = &BTRFS_I(page->mapping->host)->io_tree;
2457
2458 if (tree->ops && tree->ops->writepage_end_io_hook) {
2459 ret = tree->ops->writepage_end_io_hook(page, start,
2460 end, NULL, uptodate);
2461 if (ret)
2462 uptodate = 0;
2463 }
2464
Jeff Mahoney87826df2012-02-15 16:23:57 +01002465 if (!uptodate) {
Jeff Mahoney87826df2012-02-15 16:23:57 +01002466 ClearPageUptodate(page);
2467 SetPageError(page);
Liu Bo5dca6ee2014-05-12 12:47:36 +08002468 ret = ret < 0 ? ret : -EIO;
2469 mapping_set_error(page->mapping, ret);
Jeff Mahoney87826df2012-02-15 16:23:57 +01002470 }
2471 return 0;
2472}
2473
Chris Masond1310b22008-01-24 16:13:08 -05002474/*
2475 * after a writepage IO is done, we need to:
2476 * clear the uptodate bits on error
2477 * clear the writeback bits in the extent tree for this IO
2478 * end_page_writeback if the page has no more pending IO
2479 *
2480 * Scheduling is not allowed, so the extent state tree is expected
2481 * to have one and only one object corresponding to this IO.
2482 */
Chris Masond1310b22008-01-24 16:13:08 -05002483static void end_bio_extent_writepage(struct bio *bio, int err)
Chris Masond1310b22008-01-24 16:13:08 -05002484{
Kent Overstreet2c30c712013-11-07 12:20:26 -08002485 struct bio_vec *bvec;
Chris Masond1310b22008-01-24 16:13:08 -05002486 u64 start;
2487 u64 end;
Kent Overstreet2c30c712013-11-07 12:20:26 -08002488 int i;
Chris Masond1310b22008-01-24 16:13:08 -05002489
Kent Overstreet2c30c712013-11-07 12:20:26 -08002490 bio_for_each_segment_all(bvec, bio, i) {
Chris Masond1310b22008-01-24 16:13:08 -05002491 struct page *page = bvec->bv_page;
David Woodhouse902b22f2008-08-20 08:51:49 -04002492
Alexandre Oliva17a5adc2013-05-15 11:38:55 -04002493 /* We always issue full-page reads, but if some block
2494 * in a page fails to read, blk_update_request() will
2495 * advance bv_offset and adjust bv_len to compensate.
2496 * Print a warning for nonzero offsets, and an error
2497 * if they don't add up to a full page. */
Frank Holtonefe120a2013-12-20 11:37:06 -05002498 if (bvec->bv_offset || bvec->bv_len != PAGE_CACHE_SIZE) {
2499 if (bvec->bv_offset + bvec->bv_len != PAGE_CACHE_SIZE)
2500 btrfs_err(BTRFS_I(page->mapping->host)->root->fs_info,
2501 "partial page write in btrfs with offset %u and length %u",
2502 bvec->bv_offset, bvec->bv_len);
2503 else
2504 btrfs_info(BTRFS_I(page->mapping->host)->root->fs_info,
2505 "incomplete page write in btrfs with offset %u and "
2506 "length %u",
2507 bvec->bv_offset, bvec->bv_len);
2508 }
Chris Masond1310b22008-01-24 16:13:08 -05002509
Alexandre Oliva17a5adc2013-05-15 11:38:55 -04002510 start = page_offset(page);
2511 end = start + bvec->bv_offset + bvec->bv_len - 1;
Chris Masond1310b22008-01-24 16:13:08 -05002512
Jeff Mahoney87826df2012-02-15 16:23:57 +01002513 if (end_extent_writepage(page, err, start, end))
2514 continue;
Chris Mason70dec802008-01-29 09:59:12 -05002515
Alexandre Oliva17a5adc2013-05-15 11:38:55 -04002516 end_page_writeback(page);
Kent Overstreet2c30c712013-11-07 12:20:26 -08002517 }
Chris Mason2b1f55b2008-09-24 11:48:04 -04002518
Chris Masond1310b22008-01-24 16:13:08 -05002519 bio_put(bio);
Chris Masond1310b22008-01-24 16:13:08 -05002520}
2521
Miao Xie883d0de2013-07-25 19:22:35 +08002522static void
2523endio_readpage_release_extent(struct extent_io_tree *tree, u64 start, u64 len,
2524 int uptodate)
2525{
2526 struct extent_state *cached = NULL;
2527 u64 end = start + len - 1;
2528
2529 if (uptodate && tree->track_uptodate)
2530 set_extent_uptodate(tree, start, end, &cached, GFP_ATOMIC);
2531 unlock_extent_cached(tree, start, end, &cached, GFP_ATOMIC);
2532}
2533
Chris Masond1310b22008-01-24 16:13:08 -05002534/*
2535 * after a readpage IO is done, we need to:
2536 * clear the uptodate bits on error
2537 * set the uptodate bits if things worked
2538 * set the page up to date if all extents in the tree are uptodate
2539 * clear the lock bit in the extent tree
2540 * unlock the page if there are no other extents locked for it
2541 *
2542 * Scheduling is not allowed, so the extent state tree is expected
2543 * to have one and only one object corresponding to this IO.
2544 */
Chris Masond1310b22008-01-24 16:13:08 -05002545static void end_bio_extent_readpage(struct bio *bio, int err)
Chris Masond1310b22008-01-24 16:13:08 -05002546{
Kent Overstreet2c30c712013-11-07 12:20:26 -08002547 struct bio_vec *bvec;
Chris Masond1310b22008-01-24 16:13:08 -05002548 int uptodate = test_bit(BIO_UPTODATE, &bio->bi_flags);
Miao Xiefacc8a222013-07-25 19:22:34 +08002549 struct btrfs_io_bio *io_bio = btrfs_io_bio(bio);
David Woodhouse902b22f2008-08-20 08:51:49 -04002550 struct extent_io_tree *tree;
Miao Xiefacc8a222013-07-25 19:22:34 +08002551 u64 offset = 0;
Chris Masond1310b22008-01-24 16:13:08 -05002552 u64 start;
2553 u64 end;
Miao Xiefacc8a222013-07-25 19:22:34 +08002554 u64 len;
Miao Xie883d0de2013-07-25 19:22:35 +08002555 u64 extent_start = 0;
2556 u64 extent_len = 0;
Josef Bacik5cf1ab52012-04-16 09:42:26 -04002557 int mirror;
Chris Masond1310b22008-01-24 16:13:08 -05002558 int ret;
Kent Overstreet2c30c712013-11-07 12:20:26 -08002559 int i;
Chris Masond1310b22008-01-24 16:13:08 -05002560
Chris Masond20f7042008-12-08 16:58:54 -05002561 if (err)
2562 uptodate = 0;
2563
Kent Overstreet2c30c712013-11-07 12:20:26 -08002564 bio_for_each_segment_all(bvec, bio, i) {
Chris Masond1310b22008-01-24 16:13:08 -05002565 struct page *page = bvec->bv_page;
Josef Bacika71754f2013-06-17 17:14:39 -04002566 struct inode *inode = page->mapping->host;
Arne Jansen507903b2011-04-06 10:02:20 +00002567
Kent Overstreetbe3940c2012-09-11 14:23:05 -06002568 pr_debug("end_bio_extent_readpage: bi_sector=%llu, err=%d, "
Miao Xiec1dc0892014-09-12 18:43:56 +08002569 "mirror=%u\n", (u64)bio->bi_iter.bi_sector, err,
Chris Mason9be33952013-05-17 18:30:14 -04002570 io_bio->mirror_num);
Josef Bacika71754f2013-06-17 17:14:39 -04002571 tree = &BTRFS_I(inode)->io_tree;
David Woodhouse902b22f2008-08-20 08:51:49 -04002572
Alexandre Oliva17a5adc2013-05-15 11:38:55 -04002573 /* We always issue full-page reads, but if some block
2574 * in a page fails to read, blk_update_request() will
2575 * advance bv_offset and adjust bv_len to compensate.
2576 * Print a warning for nonzero offsets, and an error
2577 * if they don't add up to a full page. */
Frank Holtonefe120a2013-12-20 11:37:06 -05002578 if (bvec->bv_offset || bvec->bv_len != PAGE_CACHE_SIZE) {
2579 if (bvec->bv_offset + bvec->bv_len != PAGE_CACHE_SIZE)
2580 btrfs_err(BTRFS_I(page->mapping->host)->root->fs_info,
2581 "partial page read in btrfs with offset %u and length %u",
2582 bvec->bv_offset, bvec->bv_len);
2583 else
2584 btrfs_info(BTRFS_I(page->mapping->host)->root->fs_info,
2585 "incomplete page read in btrfs with offset %u and "
2586 "length %u",
2587 bvec->bv_offset, bvec->bv_len);
2588 }
Chris Masond1310b22008-01-24 16:13:08 -05002589
Alexandre Oliva17a5adc2013-05-15 11:38:55 -04002590 start = page_offset(page);
2591 end = start + bvec->bv_offset + bvec->bv_len - 1;
Miao Xiefacc8a222013-07-25 19:22:34 +08002592 len = bvec->bv_len;
Chris Masond1310b22008-01-24 16:13:08 -05002593
Chris Mason9be33952013-05-17 18:30:14 -04002594 mirror = io_bio->mirror_num;
Miao Xief2a09da2013-07-25 19:22:33 +08002595 if (likely(uptodate && tree->ops &&
2596 tree->ops->readpage_end_io_hook)) {
Miao Xiefacc8a222013-07-25 19:22:34 +08002597 ret = tree->ops->readpage_end_io_hook(io_bio, offset,
2598 page, start, end,
2599 mirror);
Stefan Behrens5ee08442012-08-27 08:30:03 -06002600 if (ret)
Chris Masond1310b22008-01-24 16:13:08 -05002601 uptodate = 0;
Stefan Behrens5ee08442012-08-27 08:30:03 -06002602 else
Miao Xie1203b682014-09-12 18:44:01 +08002603 clean_io_failure(inode, start, page, 0);
Chris Masond1310b22008-01-24 16:13:08 -05002604 }
Josef Bacikea466792012-03-26 21:57:36 -04002605
Miao Xief2a09da2013-07-25 19:22:33 +08002606 if (likely(uptodate))
2607 goto readpage_ok;
2608
2609 if (tree->ops && tree->ops->readpage_io_failed_hook) {
Josef Bacik5cf1ab52012-04-16 09:42:26 -04002610 ret = tree->ops->readpage_io_failed_hook(page, mirror);
Josef Bacikea466792012-03-26 21:57:36 -04002611 if (!ret && !err &&
2612 test_bit(BIO_UPTODATE, &bio->bi_flags))
2613 uptodate = 1;
Miao Xief2a09da2013-07-25 19:22:33 +08002614 } else {
Jan Schmidtf4a8e652011-12-01 09:30:36 -05002615 /*
2616 * The generic bio_readpage_error handles errors the
2617 * following way: If possible, new read requests are
2618 * created and submitted and will end up in
2619 * end_bio_extent_readpage as well (if we're lucky, not
2620 * in the !uptodate case). In that case it returns 0 and
2621 * we just go on with the next page in our bio. If it
2622 * can't handle the error it will return -EIO and we
2623 * remain responsible for that page.
2624 */
Miao Xiefacc8a222013-07-25 19:22:34 +08002625 ret = bio_readpage_error(bio, offset, page, start, end,
2626 mirror);
Chris Mason7e383262008-04-09 16:28:12 -04002627 if (ret == 0) {
Chris Mason3b951512008-04-17 11:29:12 -04002628 uptodate =
2629 test_bit(BIO_UPTODATE, &bio->bi_flags);
Chris Masond20f7042008-12-08 16:58:54 -05002630 if (err)
2631 uptodate = 0;
Liu Bo38c1c2e2014-08-19 23:33:13 +08002632 offset += len;
Chris Mason7e383262008-04-09 16:28:12 -04002633 continue;
2634 }
2635 }
Miao Xief2a09da2013-07-25 19:22:33 +08002636readpage_ok:
Miao Xie883d0de2013-07-25 19:22:35 +08002637 if (likely(uptodate)) {
Josef Bacika71754f2013-06-17 17:14:39 -04002638 loff_t i_size = i_size_read(inode);
2639 pgoff_t end_index = i_size >> PAGE_CACHE_SHIFT;
Liu Boa583c022014-08-19 23:32:22 +08002640 unsigned off;
Josef Bacika71754f2013-06-17 17:14:39 -04002641
2642 /* Zero out the end if this page straddles i_size */
Liu Boa583c022014-08-19 23:32:22 +08002643 off = i_size & (PAGE_CACHE_SIZE-1);
2644 if (page->index == end_index && off)
2645 zero_user_segment(page, off, PAGE_CACHE_SIZE);
Alexandre Oliva17a5adc2013-05-15 11:38:55 -04002646 SetPageUptodate(page);
Chris Mason70dec802008-01-29 09:59:12 -05002647 } else {
Alexandre Oliva17a5adc2013-05-15 11:38:55 -04002648 ClearPageUptodate(page);
2649 SetPageError(page);
Chris Mason70dec802008-01-29 09:59:12 -05002650 }
Alexandre Oliva17a5adc2013-05-15 11:38:55 -04002651 unlock_page(page);
Miao Xiefacc8a222013-07-25 19:22:34 +08002652 offset += len;
Miao Xie883d0de2013-07-25 19:22:35 +08002653
2654 if (unlikely(!uptodate)) {
2655 if (extent_len) {
2656 endio_readpage_release_extent(tree,
2657 extent_start,
2658 extent_len, 1);
2659 extent_start = 0;
2660 extent_len = 0;
2661 }
2662 endio_readpage_release_extent(tree, start,
2663 end - start + 1, 0);
2664 } else if (!extent_len) {
2665 extent_start = start;
2666 extent_len = end + 1 - start;
2667 } else if (extent_start + extent_len == start) {
2668 extent_len += end + 1 - start;
2669 } else {
2670 endio_readpage_release_extent(tree, extent_start,
2671 extent_len, uptodate);
2672 extent_start = start;
2673 extent_len = end + 1 - start;
2674 }
Kent Overstreet2c30c712013-11-07 12:20:26 -08002675 }
Chris Masond1310b22008-01-24 16:13:08 -05002676
Miao Xie883d0de2013-07-25 19:22:35 +08002677 if (extent_len)
2678 endio_readpage_release_extent(tree, extent_start, extent_len,
2679 uptodate);
Miao Xiefacc8a222013-07-25 19:22:34 +08002680 if (io_bio->end_io)
2681 io_bio->end_io(io_bio, err);
Chris Masond1310b22008-01-24 16:13:08 -05002682 bio_put(bio);
Chris Masond1310b22008-01-24 16:13:08 -05002683}
2684
Chris Mason9be33952013-05-17 18:30:14 -04002685/*
2686 * this allocates from the btrfs_bioset. We're returning a bio right now
2687 * but you can call btrfs_io_bio for the appropriate container_of magic
2688 */
Miao Xie88f794e2010-11-22 03:02:55 +00002689struct bio *
2690btrfs_bio_alloc(struct block_device *bdev, u64 first_sector, int nr_vecs,
2691 gfp_t gfp_flags)
Chris Masond1310b22008-01-24 16:13:08 -05002692{
Miao Xiefacc8a222013-07-25 19:22:34 +08002693 struct btrfs_io_bio *btrfs_bio;
Chris Masond1310b22008-01-24 16:13:08 -05002694 struct bio *bio;
2695
Chris Mason9be33952013-05-17 18:30:14 -04002696 bio = bio_alloc_bioset(gfp_flags, nr_vecs, btrfs_bioset);
Chris Masond1310b22008-01-24 16:13:08 -05002697
2698 if (bio == NULL && (current->flags & PF_MEMALLOC)) {
Chris Mason9be33952013-05-17 18:30:14 -04002699 while (!bio && (nr_vecs /= 2)) {
2700 bio = bio_alloc_bioset(gfp_flags,
2701 nr_vecs, btrfs_bioset);
2702 }
Chris Masond1310b22008-01-24 16:13:08 -05002703 }
2704
2705 if (bio) {
2706 bio->bi_bdev = bdev;
Kent Overstreet4f024f32013-10-11 15:44:27 -07002707 bio->bi_iter.bi_sector = first_sector;
Miao Xiefacc8a222013-07-25 19:22:34 +08002708 btrfs_bio = btrfs_io_bio(bio);
2709 btrfs_bio->csum = NULL;
2710 btrfs_bio->csum_allocated = NULL;
2711 btrfs_bio->end_io = NULL;
Chris Masond1310b22008-01-24 16:13:08 -05002712 }
2713 return bio;
2714}
2715
Chris Mason9be33952013-05-17 18:30:14 -04002716struct bio *btrfs_bio_clone(struct bio *bio, gfp_t gfp_mask)
2717{
Miao Xie23ea8e52014-09-12 18:43:54 +08002718 struct btrfs_io_bio *btrfs_bio;
2719 struct bio *new;
Chris Mason9be33952013-05-17 18:30:14 -04002720
Miao Xie23ea8e52014-09-12 18:43:54 +08002721 new = bio_clone_bioset(bio, gfp_mask, btrfs_bioset);
2722 if (new) {
2723 btrfs_bio = btrfs_io_bio(new);
2724 btrfs_bio->csum = NULL;
2725 btrfs_bio->csum_allocated = NULL;
2726 btrfs_bio->end_io = NULL;
2727 }
2728 return new;
2729}
Chris Mason9be33952013-05-17 18:30:14 -04002730
2731/* this also allocates from the btrfs_bioset */
2732struct bio *btrfs_io_bio_alloc(gfp_t gfp_mask, unsigned int nr_iovecs)
2733{
Miao Xiefacc8a222013-07-25 19:22:34 +08002734 struct btrfs_io_bio *btrfs_bio;
2735 struct bio *bio;
2736
2737 bio = bio_alloc_bioset(gfp_mask, nr_iovecs, btrfs_bioset);
2738 if (bio) {
2739 btrfs_bio = btrfs_io_bio(bio);
2740 btrfs_bio->csum = NULL;
2741 btrfs_bio->csum_allocated = NULL;
2742 btrfs_bio->end_io = NULL;
2743 }
2744 return bio;
Chris Mason9be33952013-05-17 18:30:14 -04002745}
2746
2747
Jeff Mahoney355808c2011-10-03 23:23:14 -04002748static int __must_check submit_one_bio(int rw, struct bio *bio,
2749 int mirror_num, unsigned long bio_flags)
Chris Masond1310b22008-01-24 16:13:08 -05002750{
Chris Masond1310b22008-01-24 16:13:08 -05002751 int ret = 0;
Chris Mason70dec802008-01-29 09:59:12 -05002752 struct bio_vec *bvec = bio->bi_io_vec + bio->bi_vcnt - 1;
2753 struct page *page = bvec->bv_page;
2754 struct extent_io_tree *tree = bio->bi_private;
Chris Mason70dec802008-01-29 09:59:12 -05002755 u64 start;
Chris Mason70dec802008-01-29 09:59:12 -05002756
Miao Xie4eee4fa2012-12-21 09:17:45 +00002757 start = page_offset(page) + bvec->bv_offset;
Chris Mason70dec802008-01-29 09:59:12 -05002758
David Woodhouse902b22f2008-08-20 08:51:49 -04002759 bio->bi_private = NULL;
Chris Masond1310b22008-01-24 16:13:08 -05002760
2761 bio_get(bio);
2762
Chris Mason065631f2008-02-20 12:07:25 -05002763 if (tree->ops && tree->ops->submit_bio_hook)
liubo6b82ce82011-01-26 06:21:39 +00002764 ret = tree->ops->submit_bio_hook(page->mapping->host, rw, bio,
Chris Masoneaf25d92010-05-25 09:48:28 -04002765 mirror_num, bio_flags, start);
Chris Mason0b86a832008-03-24 15:01:56 -04002766 else
Stefan Behrens21adbd52011-11-09 13:44:05 +01002767 btrfsic_submit_bio(rw, bio);
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002768
Chris Masond1310b22008-01-24 16:13:08 -05002769 if (bio_flagged(bio, BIO_EOPNOTSUPP))
2770 ret = -EOPNOTSUPP;
2771 bio_put(bio);
2772 return ret;
2773}
2774
David Woodhouse64a16702009-07-15 23:29:37 +01002775static int merge_bio(int rw, struct extent_io_tree *tree, struct page *page,
Jeff Mahoney3444a972011-10-03 23:23:13 -04002776 unsigned long offset, size_t size, struct bio *bio,
2777 unsigned long bio_flags)
2778{
2779 int ret = 0;
2780 if (tree->ops && tree->ops->merge_bio_hook)
David Woodhouse64a16702009-07-15 23:29:37 +01002781 ret = tree->ops->merge_bio_hook(rw, page, offset, size, bio,
Jeff Mahoney3444a972011-10-03 23:23:13 -04002782 bio_flags);
2783 BUG_ON(ret < 0);
2784 return ret;
2785
2786}
2787
Chris Masond1310b22008-01-24 16:13:08 -05002788static int submit_extent_page(int rw, struct extent_io_tree *tree,
2789 struct page *page, sector_t sector,
2790 size_t size, unsigned long offset,
2791 struct block_device *bdev,
2792 struct bio **bio_ret,
2793 unsigned long max_pages,
Chris Masonf1885912008-04-09 16:28:12 -04002794 bio_end_io_t end_io_func,
Chris Masonc8b97812008-10-29 14:49:59 -04002795 int mirror_num,
2796 unsigned long prev_bio_flags,
2797 unsigned long bio_flags)
Chris Masond1310b22008-01-24 16:13:08 -05002798{
2799 int ret = 0;
2800 struct bio *bio;
2801 int nr;
Chris Masonc8b97812008-10-29 14:49:59 -04002802 int contig = 0;
2803 int this_compressed = bio_flags & EXTENT_BIO_COMPRESSED;
2804 int old_compressed = prev_bio_flags & EXTENT_BIO_COMPRESSED;
Chris Mason5b050f02008-11-11 09:34:41 -05002805 size_t page_size = min_t(size_t, size, PAGE_CACHE_SIZE);
Chris Masond1310b22008-01-24 16:13:08 -05002806
2807 if (bio_ret && *bio_ret) {
2808 bio = *bio_ret;
Chris Masonc8b97812008-10-29 14:49:59 -04002809 if (old_compressed)
Kent Overstreet4f024f32013-10-11 15:44:27 -07002810 contig = bio->bi_iter.bi_sector == sector;
Chris Masonc8b97812008-10-29 14:49:59 -04002811 else
Kent Overstreetf73a1c72012-09-25 15:05:12 -07002812 contig = bio_end_sector(bio) == sector;
Chris Masonc8b97812008-10-29 14:49:59 -04002813
2814 if (prev_bio_flags != bio_flags || !contig ||
David Woodhouse64a16702009-07-15 23:29:37 +01002815 merge_bio(rw, tree, page, offset, page_size, bio, bio_flags) ||
Chris Masonc8b97812008-10-29 14:49:59 -04002816 bio_add_page(bio, page, page_size, offset) < page_size) {
2817 ret = submit_one_bio(rw, bio, mirror_num,
2818 prev_bio_flags);
Jeff Mahoney79787ea2012-03-12 16:03:00 +01002819 if (ret < 0)
2820 return ret;
Chris Masond1310b22008-01-24 16:13:08 -05002821 bio = NULL;
2822 } else {
2823 return 0;
2824 }
2825 }
Chris Masonc8b97812008-10-29 14:49:59 -04002826 if (this_compressed)
2827 nr = BIO_MAX_PAGES;
2828 else
2829 nr = bio_get_nr_vecs(bdev);
2830
Miao Xie88f794e2010-11-22 03:02:55 +00002831 bio = btrfs_bio_alloc(bdev, sector, nr, GFP_NOFS | __GFP_HIGH);
Tsutomu Itoh5df67082011-02-01 09:17:35 +00002832 if (!bio)
2833 return -ENOMEM;
Chris Mason70dec802008-01-29 09:59:12 -05002834
Chris Masonc8b97812008-10-29 14:49:59 -04002835 bio_add_page(bio, page, page_size, offset);
Chris Masond1310b22008-01-24 16:13:08 -05002836 bio->bi_end_io = end_io_func;
2837 bio->bi_private = tree;
Chris Mason70dec802008-01-29 09:59:12 -05002838
Chris Masond3977122009-01-05 21:25:51 -05002839 if (bio_ret)
Chris Masond1310b22008-01-24 16:13:08 -05002840 *bio_ret = bio;
Chris Masond3977122009-01-05 21:25:51 -05002841 else
Chris Masonc8b97812008-10-29 14:49:59 -04002842 ret = submit_one_bio(rw, bio, mirror_num, bio_flags);
Chris Masond1310b22008-01-24 16:13:08 -05002843
2844 return ret;
2845}
2846
Eric Sandeen48a3b632013-04-25 20:41:01 +00002847static void attach_extent_buffer_page(struct extent_buffer *eb,
2848 struct page *page)
Josef Bacik4f2de97a2012-03-07 16:20:05 -05002849{
2850 if (!PagePrivate(page)) {
2851 SetPagePrivate(page);
2852 page_cache_get(page);
2853 set_page_private(page, (unsigned long)eb);
2854 } else {
2855 WARN_ON(page->private != (unsigned long)eb);
2856 }
2857}
2858
Chris Masond1310b22008-01-24 16:13:08 -05002859void set_page_extent_mapped(struct page *page)
2860{
2861 if (!PagePrivate(page)) {
2862 SetPagePrivate(page);
Chris Masond1310b22008-01-24 16:13:08 -05002863 page_cache_get(page);
Chris Mason6af118ce2008-07-22 11:18:07 -04002864 set_page_private(page, EXTENT_PAGE_PRIVATE);
Chris Masond1310b22008-01-24 16:13:08 -05002865 }
2866}
2867
Miao Xie125bac012013-07-25 19:22:37 +08002868static struct extent_map *
2869__get_extent_map(struct inode *inode, struct page *page, size_t pg_offset,
2870 u64 start, u64 len, get_extent_t *get_extent,
2871 struct extent_map **em_cached)
2872{
2873 struct extent_map *em;
2874
2875 if (em_cached && *em_cached) {
2876 em = *em_cached;
Filipe Mananacbc0e922014-02-25 14:15:12 +00002877 if (extent_map_in_tree(em) && start >= em->start &&
Miao Xie125bac012013-07-25 19:22:37 +08002878 start < extent_map_end(em)) {
2879 atomic_inc(&em->refs);
2880 return em;
2881 }
2882
2883 free_extent_map(em);
2884 *em_cached = NULL;
2885 }
2886
2887 em = get_extent(inode, page, pg_offset, start, len, 0);
2888 if (em_cached && !IS_ERR_OR_NULL(em)) {
2889 BUG_ON(*em_cached);
2890 atomic_inc(&em->refs);
2891 *em_cached = em;
2892 }
2893 return em;
2894}
Chris Masond1310b22008-01-24 16:13:08 -05002895/*
2896 * basic readpage implementation. Locked extent state structs are inserted
2897 * into the tree that are removed when the IO is done (by the end_io
2898 * handlers)
Jeff Mahoney79787ea2012-03-12 16:03:00 +01002899 * XXX JDM: This needs looking at to ensure proper page locking
Chris Masond1310b22008-01-24 16:13:08 -05002900 */
Miao Xie99740902013-07-25 19:22:36 +08002901static int __do_readpage(struct extent_io_tree *tree,
2902 struct page *page,
2903 get_extent_t *get_extent,
Miao Xie125bac012013-07-25 19:22:37 +08002904 struct extent_map **em_cached,
Miao Xie99740902013-07-25 19:22:36 +08002905 struct bio **bio, int mirror_num,
2906 unsigned long *bio_flags, int rw)
Chris Masond1310b22008-01-24 16:13:08 -05002907{
2908 struct inode *inode = page->mapping->host;
Miao Xie4eee4fa2012-12-21 09:17:45 +00002909 u64 start = page_offset(page);
Chris Masond1310b22008-01-24 16:13:08 -05002910 u64 page_end = start + PAGE_CACHE_SIZE - 1;
2911 u64 end;
2912 u64 cur = start;
2913 u64 extent_offset;
2914 u64 last_byte = i_size_read(inode);
2915 u64 block_start;
2916 u64 cur_end;
2917 sector_t sector;
2918 struct extent_map *em;
2919 struct block_device *bdev;
2920 int ret;
2921 int nr = 0;
Mark Fasheh4b384312013-08-06 11:42:50 -07002922 int parent_locked = *bio_flags & EXTENT_BIO_PARENT_LOCKED;
David Sterba306e16c2011-04-19 14:29:38 +02002923 size_t pg_offset = 0;
Chris Masond1310b22008-01-24 16:13:08 -05002924 size_t iosize;
Chris Masonc8b97812008-10-29 14:49:59 -04002925 size_t disk_io_size;
Chris Masond1310b22008-01-24 16:13:08 -05002926 size_t blocksize = inode->i_sb->s_blocksize;
Mark Fasheh4b384312013-08-06 11:42:50 -07002927 unsigned long this_bio_flag = *bio_flags & EXTENT_BIO_PARENT_LOCKED;
Chris Masond1310b22008-01-24 16:13:08 -05002928
2929 set_page_extent_mapped(page);
2930
Miao Xie99740902013-07-25 19:22:36 +08002931 end = page_end;
Dan Magenheimer90a887c2011-05-26 10:01:56 -06002932 if (!PageUptodate(page)) {
2933 if (cleancache_get_page(page) == 0) {
2934 BUG_ON(blocksize != PAGE_SIZE);
Miao Xie99740902013-07-25 19:22:36 +08002935 unlock_extent(tree, start, end);
Dan Magenheimer90a887c2011-05-26 10:01:56 -06002936 goto out;
2937 }
2938 }
2939
Chris Masonc8b97812008-10-29 14:49:59 -04002940 if (page->index == last_byte >> PAGE_CACHE_SHIFT) {
2941 char *userpage;
2942 size_t zero_offset = last_byte & (PAGE_CACHE_SIZE - 1);
2943
2944 if (zero_offset) {
2945 iosize = PAGE_CACHE_SIZE - zero_offset;
Cong Wang7ac687d2011-11-25 23:14:28 +08002946 userpage = kmap_atomic(page);
Chris Masonc8b97812008-10-29 14:49:59 -04002947 memset(userpage + zero_offset, 0, iosize);
2948 flush_dcache_page(page);
Cong Wang7ac687d2011-11-25 23:14:28 +08002949 kunmap_atomic(userpage);
Chris Masonc8b97812008-10-29 14:49:59 -04002950 }
2951 }
Chris Masond1310b22008-01-24 16:13:08 -05002952 while (cur <= end) {
Josef Bacikc8f2f242013-02-11 11:33:00 -05002953 unsigned long pnr = (last_byte >> PAGE_CACHE_SHIFT) + 1;
2954
Chris Masond1310b22008-01-24 16:13:08 -05002955 if (cur >= last_byte) {
2956 char *userpage;
Arne Jansen507903b2011-04-06 10:02:20 +00002957 struct extent_state *cached = NULL;
2958
David Sterba306e16c2011-04-19 14:29:38 +02002959 iosize = PAGE_CACHE_SIZE - pg_offset;
Cong Wang7ac687d2011-11-25 23:14:28 +08002960 userpage = kmap_atomic(page);
David Sterba306e16c2011-04-19 14:29:38 +02002961 memset(userpage + pg_offset, 0, iosize);
Chris Masond1310b22008-01-24 16:13:08 -05002962 flush_dcache_page(page);
Cong Wang7ac687d2011-11-25 23:14:28 +08002963 kunmap_atomic(userpage);
Chris Masond1310b22008-01-24 16:13:08 -05002964 set_extent_uptodate(tree, cur, cur + iosize - 1,
Arne Jansen507903b2011-04-06 10:02:20 +00002965 &cached, GFP_NOFS);
Mark Fasheh4b384312013-08-06 11:42:50 -07002966 if (!parent_locked)
2967 unlock_extent_cached(tree, cur,
2968 cur + iosize - 1,
2969 &cached, GFP_NOFS);
Chris Masond1310b22008-01-24 16:13:08 -05002970 break;
2971 }
Miao Xie125bac012013-07-25 19:22:37 +08002972 em = __get_extent_map(inode, page, pg_offset, cur,
2973 end - cur + 1, get_extent, em_cached);
David Sterbac7040052011-04-19 18:00:01 +02002974 if (IS_ERR_OR_NULL(em)) {
Chris Masond1310b22008-01-24 16:13:08 -05002975 SetPageError(page);
Mark Fasheh4b384312013-08-06 11:42:50 -07002976 if (!parent_locked)
2977 unlock_extent(tree, cur, end);
Chris Masond1310b22008-01-24 16:13:08 -05002978 break;
2979 }
Chris Masond1310b22008-01-24 16:13:08 -05002980 extent_offset = cur - em->start;
2981 BUG_ON(extent_map_end(em) <= cur);
2982 BUG_ON(end < cur);
2983
Li Zefan261507a02010-12-17 14:21:50 +08002984 if (test_bit(EXTENT_FLAG_COMPRESSED, &em->flags)) {
Mark Fasheh4b384312013-08-06 11:42:50 -07002985 this_bio_flag |= EXTENT_BIO_COMPRESSED;
Li Zefan261507a02010-12-17 14:21:50 +08002986 extent_set_compress_type(&this_bio_flag,
2987 em->compress_type);
2988 }
Chris Masonc8b97812008-10-29 14:49:59 -04002989
Chris Masond1310b22008-01-24 16:13:08 -05002990 iosize = min(extent_map_end(em) - cur, end - cur + 1);
2991 cur_end = min(extent_map_end(em) - 1, end);
Qu Wenruofda28322013-02-26 08:10:22 +00002992 iosize = ALIGN(iosize, blocksize);
Chris Masonc8b97812008-10-29 14:49:59 -04002993 if (this_bio_flag & EXTENT_BIO_COMPRESSED) {
2994 disk_io_size = em->block_len;
2995 sector = em->block_start >> 9;
2996 } else {
2997 sector = (em->block_start + extent_offset) >> 9;
2998 disk_io_size = iosize;
2999 }
Chris Masond1310b22008-01-24 16:13:08 -05003000 bdev = em->bdev;
3001 block_start = em->block_start;
Yan Zhengd899e052008-10-30 14:25:28 -04003002 if (test_bit(EXTENT_FLAG_PREALLOC, &em->flags))
3003 block_start = EXTENT_MAP_HOLE;
Chris Masond1310b22008-01-24 16:13:08 -05003004 free_extent_map(em);
3005 em = NULL;
3006
3007 /* we've found a hole, just zero and go on */
3008 if (block_start == EXTENT_MAP_HOLE) {
3009 char *userpage;
Arne Jansen507903b2011-04-06 10:02:20 +00003010 struct extent_state *cached = NULL;
3011
Cong Wang7ac687d2011-11-25 23:14:28 +08003012 userpage = kmap_atomic(page);
David Sterba306e16c2011-04-19 14:29:38 +02003013 memset(userpage + pg_offset, 0, iosize);
Chris Masond1310b22008-01-24 16:13:08 -05003014 flush_dcache_page(page);
Cong Wang7ac687d2011-11-25 23:14:28 +08003015 kunmap_atomic(userpage);
Chris Masond1310b22008-01-24 16:13:08 -05003016
3017 set_extent_uptodate(tree, cur, cur + iosize - 1,
Arne Jansen507903b2011-04-06 10:02:20 +00003018 &cached, GFP_NOFS);
3019 unlock_extent_cached(tree, cur, cur + iosize - 1,
3020 &cached, GFP_NOFS);
Chris Masond1310b22008-01-24 16:13:08 -05003021 cur = cur + iosize;
David Sterba306e16c2011-04-19 14:29:38 +02003022 pg_offset += iosize;
Chris Masond1310b22008-01-24 16:13:08 -05003023 continue;
3024 }
3025 /* the get_extent function already copied into the page */
Chris Mason9655d292009-09-02 15:22:30 -04003026 if (test_range_bit(tree, cur, cur_end,
3027 EXTENT_UPTODATE, 1, NULL)) {
Chris Masona1b32a52008-09-05 16:09:51 -04003028 check_page_uptodate(tree, page);
Mark Fasheh4b384312013-08-06 11:42:50 -07003029 if (!parent_locked)
3030 unlock_extent(tree, cur, cur + iosize - 1);
Chris Masond1310b22008-01-24 16:13:08 -05003031 cur = cur + iosize;
David Sterba306e16c2011-04-19 14:29:38 +02003032 pg_offset += iosize;
Chris Masond1310b22008-01-24 16:13:08 -05003033 continue;
3034 }
Chris Mason70dec802008-01-29 09:59:12 -05003035 /* we have an inline extent but it didn't get marked up
3036 * to date. Error out
3037 */
3038 if (block_start == EXTENT_MAP_INLINE) {
3039 SetPageError(page);
Mark Fasheh4b384312013-08-06 11:42:50 -07003040 if (!parent_locked)
3041 unlock_extent(tree, cur, cur + iosize - 1);
Chris Mason70dec802008-01-29 09:59:12 -05003042 cur = cur + iosize;
David Sterba306e16c2011-04-19 14:29:38 +02003043 pg_offset += iosize;
Chris Mason70dec802008-01-29 09:59:12 -05003044 continue;
3045 }
Chris Masond1310b22008-01-24 16:13:08 -05003046
Josef Bacikc8f2f242013-02-11 11:33:00 -05003047 pnr -= page->index;
Josef Bacikd4c7ca82013-04-19 19:49:09 -04003048 ret = submit_extent_page(rw, tree, page,
David Sterba306e16c2011-04-19 14:29:38 +02003049 sector, disk_io_size, pg_offset,
Chris Mason89642222008-07-24 09:41:53 -04003050 bdev, bio, pnr,
Chris Masonc8b97812008-10-29 14:49:59 -04003051 end_bio_extent_readpage, mirror_num,
3052 *bio_flags,
3053 this_bio_flag);
Josef Bacikc8f2f242013-02-11 11:33:00 -05003054 if (!ret) {
3055 nr++;
3056 *bio_flags = this_bio_flag;
3057 } else {
Chris Masond1310b22008-01-24 16:13:08 -05003058 SetPageError(page);
Mark Fasheh4b384312013-08-06 11:42:50 -07003059 if (!parent_locked)
3060 unlock_extent(tree, cur, cur + iosize - 1);
Josef Bacikedd33c92012-10-05 16:40:32 -04003061 }
Chris Masond1310b22008-01-24 16:13:08 -05003062 cur = cur + iosize;
David Sterba306e16c2011-04-19 14:29:38 +02003063 pg_offset += iosize;
Chris Masond1310b22008-01-24 16:13:08 -05003064 }
Dan Magenheimer90a887c2011-05-26 10:01:56 -06003065out:
Chris Masond1310b22008-01-24 16:13:08 -05003066 if (!nr) {
3067 if (!PageError(page))
3068 SetPageUptodate(page);
3069 unlock_page(page);
3070 }
3071 return 0;
3072}
3073
Miao Xie99740902013-07-25 19:22:36 +08003074static inline void __do_contiguous_readpages(struct extent_io_tree *tree,
3075 struct page *pages[], int nr_pages,
3076 u64 start, u64 end,
3077 get_extent_t *get_extent,
Miao Xie125bac012013-07-25 19:22:37 +08003078 struct extent_map **em_cached,
Miao Xie99740902013-07-25 19:22:36 +08003079 struct bio **bio, int mirror_num,
3080 unsigned long *bio_flags, int rw)
3081{
3082 struct inode *inode;
3083 struct btrfs_ordered_extent *ordered;
3084 int index;
3085
3086 inode = pages[0]->mapping->host;
3087 while (1) {
3088 lock_extent(tree, start, end);
3089 ordered = btrfs_lookup_ordered_range(inode, start,
3090 end - start + 1);
3091 if (!ordered)
3092 break;
3093 unlock_extent(tree, start, end);
3094 btrfs_start_ordered_extent(inode, ordered, 1);
3095 btrfs_put_ordered_extent(ordered);
3096 }
3097
3098 for (index = 0; index < nr_pages; index++) {
Miao Xie125bac012013-07-25 19:22:37 +08003099 __do_readpage(tree, pages[index], get_extent, em_cached, bio,
3100 mirror_num, bio_flags, rw);
Miao Xie99740902013-07-25 19:22:36 +08003101 page_cache_release(pages[index]);
3102 }
3103}
3104
3105static void __extent_readpages(struct extent_io_tree *tree,
3106 struct page *pages[],
3107 int nr_pages, get_extent_t *get_extent,
Miao Xie125bac012013-07-25 19:22:37 +08003108 struct extent_map **em_cached,
Miao Xie99740902013-07-25 19:22:36 +08003109 struct bio **bio, int mirror_num,
3110 unsigned long *bio_flags, int rw)
3111{
Stefan Behrens35a36212013-08-14 18:12:25 +02003112 u64 start = 0;
Miao Xie99740902013-07-25 19:22:36 +08003113 u64 end = 0;
3114 u64 page_start;
3115 int index;
Stefan Behrens35a36212013-08-14 18:12:25 +02003116 int first_index = 0;
Miao Xie99740902013-07-25 19:22:36 +08003117
3118 for (index = 0; index < nr_pages; index++) {
3119 page_start = page_offset(pages[index]);
3120 if (!end) {
3121 start = page_start;
3122 end = start + PAGE_CACHE_SIZE - 1;
3123 first_index = index;
3124 } else if (end + 1 == page_start) {
3125 end += PAGE_CACHE_SIZE;
3126 } else {
3127 __do_contiguous_readpages(tree, &pages[first_index],
3128 index - first_index, start,
Miao Xie125bac012013-07-25 19:22:37 +08003129 end, get_extent, em_cached,
3130 bio, mirror_num, bio_flags,
3131 rw);
Miao Xie99740902013-07-25 19:22:36 +08003132 start = page_start;
3133 end = start + PAGE_CACHE_SIZE - 1;
3134 first_index = index;
3135 }
3136 }
3137
3138 if (end)
3139 __do_contiguous_readpages(tree, &pages[first_index],
3140 index - first_index, start,
Miao Xie125bac012013-07-25 19:22:37 +08003141 end, get_extent, em_cached, bio,
Miao Xie99740902013-07-25 19:22:36 +08003142 mirror_num, bio_flags, rw);
3143}
3144
3145static int __extent_read_full_page(struct extent_io_tree *tree,
3146 struct page *page,
3147 get_extent_t *get_extent,
3148 struct bio **bio, int mirror_num,
3149 unsigned long *bio_flags, int rw)
3150{
3151 struct inode *inode = page->mapping->host;
3152 struct btrfs_ordered_extent *ordered;
3153 u64 start = page_offset(page);
3154 u64 end = start + PAGE_CACHE_SIZE - 1;
3155 int ret;
3156
3157 while (1) {
3158 lock_extent(tree, start, end);
3159 ordered = btrfs_lookup_ordered_extent(inode, start);
3160 if (!ordered)
3161 break;
3162 unlock_extent(tree, start, end);
3163 btrfs_start_ordered_extent(inode, ordered, 1);
3164 btrfs_put_ordered_extent(ordered);
3165 }
3166
Miao Xie125bac012013-07-25 19:22:37 +08003167 ret = __do_readpage(tree, page, get_extent, NULL, bio, mirror_num,
3168 bio_flags, rw);
Miao Xie99740902013-07-25 19:22:36 +08003169 return ret;
3170}
3171
Chris Masond1310b22008-01-24 16:13:08 -05003172int extent_read_full_page(struct extent_io_tree *tree, struct page *page,
Jan Schmidt8ddc7d92011-06-13 20:02:58 +02003173 get_extent_t *get_extent, int mirror_num)
Chris Masond1310b22008-01-24 16:13:08 -05003174{
3175 struct bio *bio = NULL;
Chris Masonc8b97812008-10-29 14:49:59 -04003176 unsigned long bio_flags = 0;
Chris Masond1310b22008-01-24 16:13:08 -05003177 int ret;
3178
Jan Schmidt8ddc7d92011-06-13 20:02:58 +02003179 ret = __extent_read_full_page(tree, page, get_extent, &bio, mirror_num,
Josef Bacikd4c7ca82013-04-19 19:49:09 -04003180 &bio_flags, READ);
Chris Masond1310b22008-01-24 16:13:08 -05003181 if (bio)
Jan Schmidt8ddc7d92011-06-13 20:02:58 +02003182 ret = submit_one_bio(READ, bio, mirror_num, bio_flags);
Chris Masond1310b22008-01-24 16:13:08 -05003183 return ret;
3184}
Chris Masond1310b22008-01-24 16:13:08 -05003185
Mark Fasheh4b384312013-08-06 11:42:50 -07003186int extent_read_full_page_nolock(struct extent_io_tree *tree, struct page *page,
3187 get_extent_t *get_extent, int mirror_num)
3188{
3189 struct bio *bio = NULL;
3190 unsigned long bio_flags = EXTENT_BIO_PARENT_LOCKED;
3191 int ret;
3192
3193 ret = __do_readpage(tree, page, get_extent, NULL, &bio, mirror_num,
3194 &bio_flags, READ);
3195 if (bio)
3196 ret = submit_one_bio(READ, bio, mirror_num, bio_flags);
3197 return ret;
3198}
3199
Chris Mason11c83492009-04-20 15:50:09 -04003200static noinline void update_nr_written(struct page *page,
3201 struct writeback_control *wbc,
3202 unsigned long nr_written)
3203{
3204 wbc->nr_to_write -= nr_written;
3205 if (wbc->range_cyclic || (wbc->nr_to_write > 0 &&
3206 wbc->range_start == 0 && wbc->range_end == LLONG_MAX))
3207 page->mapping->writeback_index = page->index + nr_written;
3208}
3209
Chris Masond1310b22008-01-24 16:13:08 -05003210/*
Chris Mason40f76582014-05-21 13:35:51 -07003211 * helper for __extent_writepage, doing all of the delayed allocation setup.
3212 *
3213 * This returns 1 if our fill_delalloc function did all the work required
3214 * to write the page (copy into inline extent). In this case the IO has
3215 * been started and the page is already unlocked.
3216 *
3217 * This returns 0 if all went well (page still locked)
3218 * This returns < 0 if there were errors (page still locked)
Chris Masond1310b22008-01-24 16:13:08 -05003219 */
Chris Mason40f76582014-05-21 13:35:51 -07003220static noinline_for_stack int writepage_delalloc(struct inode *inode,
3221 struct page *page, struct writeback_control *wbc,
3222 struct extent_page_data *epd,
3223 u64 delalloc_start,
3224 unsigned long *nr_written)
Chris Masond1310b22008-01-24 16:13:08 -05003225{
Chris Mason40f76582014-05-21 13:35:51 -07003226 struct extent_io_tree *tree = epd->tree;
3227 u64 page_end = delalloc_start + PAGE_CACHE_SIZE - 1;
3228 u64 nr_delalloc;
3229 u64 delalloc_to_write = 0;
3230 u64 delalloc_end = 0;
3231 int ret;
3232 int page_started = 0;
3233
3234 if (epd->extent_locked || !tree->ops || !tree->ops->fill_delalloc)
3235 return 0;
3236
3237 while (delalloc_end < page_end) {
3238 nr_delalloc = find_lock_delalloc_range(inode, tree,
3239 page,
3240 &delalloc_start,
3241 &delalloc_end,
3242 128 * 1024 * 1024);
3243 if (nr_delalloc == 0) {
3244 delalloc_start = delalloc_end + 1;
3245 continue;
3246 }
3247 ret = tree->ops->fill_delalloc(inode, page,
3248 delalloc_start,
3249 delalloc_end,
3250 &page_started,
3251 nr_written);
3252 /* File system has been set read-only */
3253 if (ret) {
3254 SetPageError(page);
3255 /* fill_delalloc should be return < 0 for error
3256 * but just in case, we use > 0 here meaning the
3257 * IO is started, so we don't want to return > 0
3258 * unless things are going well.
3259 */
3260 ret = ret < 0 ? ret : -EIO;
3261 goto done;
3262 }
3263 /*
3264 * delalloc_end is already one less than the total
3265 * length, so we don't subtract one from
3266 * PAGE_CACHE_SIZE
3267 */
3268 delalloc_to_write += (delalloc_end - delalloc_start +
3269 PAGE_CACHE_SIZE) >>
3270 PAGE_CACHE_SHIFT;
3271 delalloc_start = delalloc_end + 1;
3272 }
3273 if (wbc->nr_to_write < delalloc_to_write) {
3274 int thresh = 8192;
3275
3276 if (delalloc_to_write < thresh * 2)
3277 thresh = delalloc_to_write;
3278 wbc->nr_to_write = min_t(u64, delalloc_to_write,
3279 thresh);
3280 }
3281
3282 /* did the fill delalloc function already unlock and start
3283 * the IO?
3284 */
3285 if (page_started) {
3286 /*
3287 * we've unlocked the page, so we can't update
3288 * the mapping's writeback index, just update
3289 * nr_to_write.
3290 */
3291 wbc->nr_to_write -= *nr_written;
3292 return 1;
3293 }
3294
3295 ret = 0;
3296
3297done:
3298 return ret;
3299}
3300
3301/*
3302 * helper for __extent_writepage. This calls the writepage start hooks,
3303 * and does the loop to map the page into extents and bios.
3304 *
3305 * We return 1 if the IO is started and the page is unlocked,
3306 * 0 if all went well (page still locked)
3307 * < 0 if there were errors (page still locked)
3308 */
3309static noinline_for_stack int __extent_writepage_io(struct inode *inode,
3310 struct page *page,
3311 struct writeback_control *wbc,
3312 struct extent_page_data *epd,
3313 loff_t i_size,
3314 unsigned long nr_written,
3315 int write_flags, int *nr_ret)
3316{
Chris Masond1310b22008-01-24 16:13:08 -05003317 struct extent_io_tree *tree = epd->tree;
Miao Xie4eee4fa2012-12-21 09:17:45 +00003318 u64 start = page_offset(page);
Chris Masond1310b22008-01-24 16:13:08 -05003319 u64 page_end = start + PAGE_CACHE_SIZE - 1;
3320 u64 end;
3321 u64 cur = start;
3322 u64 extent_offset;
Chris Masond1310b22008-01-24 16:13:08 -05003323 u64 block_start;
3324 u64 iosize;
3325 sector_t sector;
Chris Mason2c64c532009-09-02 15:04:12 -04003326 struct extent_state *cached_state = NULL;
Chris Masond1310b22008-01-24 16:13:08 -05003327 struct extent_map *em;
3328 struct block_device *bdev;
Chris Mason7f3c74f2008-07-18 12:01:11 -04003329 size_t pg_offset = 0;
Chris Masond1310b22008-01-24 16:13:08 -05003330 size_t blocksize;
Chris Mason40f76582014-05-21 13:35:51 -07003331 int ret = 0;
3332 int nr = 0;
3333 bool compressed;
Chris Masond1310b22008-01-24 16:13:08 -05003334
Chris Mason247e7432008-07-17 12:53:51 -04003335 if (tree->ops && tree->ops->writepage_start_hook) {
Chris Masonc8b97812008-10-29 14:49:59 -04003336 ret = tree->ops->writepage_start_hook(page, start,
3337 page_end);
Jeff Mahoney87826df2012-02-15 16:23:57 +01003338 if (ret) {
3339 /* Fixup worker will requeue */
3340 if (ret == -EBUSY)
3341 wbc->pages_skipped++;
3342 else
3343 redirty_page_for_writepage(wbc, page);
Chris Mason40f76582014-05-21 13:35:51 -07003344
Chris Mason11c83492009-04-20 15:50:09 -04003345 update_nr_written(page, wbc, nr_written);
Chris Mason247e7432008-07-17 12:53:51 -04003346 unlock_page(page);
Chris Mason40f76582014-05-21 13:35:51 -07003347 ret = 1;
Chris Mason11c83492009-04-20 15:50:09 -04003348 goto done_unlocked;
Chris Mason247e7432008-07-17 12:53:51 -04003349 }
3350 }
3351
Chris Mason11c83492009-04-20 15:50:09 -04003352 /*
3353 * we don't want to touch the inode after unlocking the page,
3354 * so we update the mapping writeback index now
3355 */
3356 update_nr_written(page, wbc, nr_written + 1);
Chris Mason771ed682008-11-06 22:02:51 -05003357
Chris Masond1310b22008-01-24 16:13:08 -05003358 end = page_end;
Chris Mason40f76582014-05-21 13:35:51 -07003359 if (i_size <= start) {
Chris Masone6dcd2d2008-07-17 12:53:50 -04003360 if (tree->ops && tree->ops->writepage_end_io_hook)
3361 tree->ops->writepage_end_io_hook(page, start,
3362 page_end, NULL, 1);
Chris Masond1310b22008-01-24 16:13:08 -05003363 goto done;
3364 }
3365
Chris Masond1310b22008-01-24 16:13:08 -05003366 blocksize = inode->i_sb->s_blocksize;
3367
3368 while (cur <= end) {
Chris Mason40f76582014-05-21 13:35:51 -07003369 u64 em_end;
3370 if (cur >= i_size) {
Chris Masone6dcd2d2008-07-17 12:53:50 -04003371 if (tree->ops && tree->ops->writepage_end_io_hook)
3372 tree->ops->writepage_end_io_hook(page, cur,
3373 page_end, NULL, 1);
Chris Masond1310b22008-01-24 16:13:08 -05003374 break;
3375 }
Chris Mason7f3c74f2008-07-18 12:01:11 -04003376 em = epd->get_extent(inode, page, pg_offset, cur,
Chris Masond1310b22008-01-24 16:13:08 -05003377 end - cur + 1, 1);
David Sterbac7040052011-04-19 18:00:01 +02003378 if (IS_ERR_OR_NULL(em)) {
Chris Masond1310b22008-01-24 16:13:08 -05003379 SetPageError(page);
Filipe Manana61391d52014-05-09 17:17:40 +01003380 ret = PTR_ERR_OR_ZERO(em);
Chris Masond1310b22008-01-24 16:13:08 -05003381 break;
3382 }
3383
3384 extent_offset = cur - em->start;
Chris Mason40f76582014-05-21 13:35:51 -07003385 em_end = extent_map_end(em);
3386 BUG_ON(em_end <= cur);
Chris Masond1310b22008-01-24 16:13:08 -05003387 BUG_ON(end < cur);
Chris Mason40f76582014-05-21 13:35:51 -07003388 iosize = min(em_end - cur, end - cur + 1);
Qu Wenruofda28322013-02-26 08:10:22 +00003389 iosize = ALIGN(iosize, blocksize);
Chris Masond1310b22008-01-24 16:13:08 -05003390 sector = (em->block_start + extent_offset) >> 9;
3391 bdev = em->bdev;
3392 block_start = em->block_start;
Chris Masonc8b97812008-10-29 14:49:59 -04003393 compressed = test_bit(EXTENT_FLAG_COMPRESSED, &em->flags);
Chris Masond1310b22008-01-24 16:13:08 -05003394 free_extent_map(em);
3395 em = NULL;
3396
Chris Masonc8b97812008-10-29 14:49:59 -04003397 /*
3398 * compressed and inline extents are written through other
3399 * paths in the FS
3400 */
3401 if (compressed || block_start == EXTENT_MAP_HOLE ||
Chris Masond1310b22008-01-24 16:13:08 -05003402 block_start == EXTENT_MAP_INLINE) {
Chris Masonc8b97812008-10-29 14:49:59 -04003403 /*
3404 * end_io notification does not happen here for
3405 * compressed extents
3406 */
3407 if (!compressed && tree->ops &&
3408 tree->ops->writepage_end_io_hook)
Chris Masone6dcd2d2008-07-17 12:53:50 -04003409 tree->ops->writepage_end_io_hook(page, cur,
3410 cur + iosize - 1,
3411 NULL, 1);
Chris Masonc8b97812008-10-29 14:49:59 -04003412 else if (compressed) {
3413 /* we don't want to end_page_writeback on
3414 * a compressed extent. this happens
3415 * elsewhere
3416 */
3417 nr++;
3418 }
3419
3420 cur += iosize;
Chris Mason7f3c74f2008-07-18 12:01:11 -04003421 pg_offset += iosize;
Chris Masond1310b22008-01-24 16:13:08 -05003422 continue;
3423 }
Chris Masonc8b97812008-10-29 14:49:59 -04003424
Chris Masond1310b22008-01-24 16:13:08 -05003425 if (tree->ops && tree->ops->writepage_io_hook) {
3426 ret = tree->ops->writepage_io_hook(page, cur,
3427 cur + iosize - 1);
3428 } else {
3429 ret = 0;
3430 }
Chris Mason1259ab72008-05-12 13:39:03 -04003431 if (ret) {
Chris Masond1310b22008-01-24 16:13:08 -05003432 SetPageError(page);
Chris Mason1259ab72008-05-12 13:39:03 -04003433 } else {
Chris Mason40f76582014-05-21 13:35:51 -07003434 unsigned long max_nr = (i_size >> PAGE_CACHE_SHIFT) + 1;
Chris Mason7f3c74f2008-07-18 12:01:11 -04003435
Chris Masond1310b22008-01-24 16:13:08 -05003436 set_range_writeback(tree, cur, cur + iosize - 1);
3437 if (!PageWriteback(page)) {
Frank Holtonefe120a2013-12-20 11:37:06 -05003438 btrfs_err(BTRFS_I(inode)->root->fs_info,
3439 "page %lu not writeback, cur %llu end %llu",
Geert Uytterhoevenc1c9ff72013-08-20 13:20:07 +02003440 page->index, cur, end);
Chris Masond1310b22008-01-24 16:13:08 -05003441 }
3442
Chris Masonffbd5172009-04-20 15:50:09 -04003443 ret = submit_extent_page(write_flags, tree, page,
3444 sector, iosize, pg_offset,
3445 bdev, &epd->bio, max_nr,
Chris Masonc8b97812008-10-29 14:49:59 -04003446 end_bio_extent_writepage,
3447 0, 0, 0);
Chris Masond1310b22008-01-24 16:13:08 -05003448 if (ret)
3449 SetPageError(page);
3450 }
3451 cur = cur + iosize;
Chris Mason7f3c74f2008-07-18 12:01:11 -04003452 pg_offset += iosize;
Chris Masond1310b22008-01-24 16:13:08 -05003453 nr++;
3454 }
3455done:
Chris Mason40f76582014-05-21 13:35:51 -07003456 *nr_ret = nr;
Chris Mason771ed682008-11-06 22:02:51 -05003457
Chris Mason11c83492009-04-20 15:50:09 -04003458done_unlocked:
3459
Chris Mason2c64c532009-09-02 15:04:12 -04003460 /* drop our reference on any cached states */
3461 free_extent_state(cached_state);
Chris Mason40f76582014-05-21 13:35:51 -07003462 return ret;
3463}
3464
3465/*
3466 * the writepage semantics are similar to regular writepage. extent
3467 * records are inserted to lock ranges in the tree, and as dirty areas
3468 * are found, they are marked writeback. Then the lock bits are removed
3469 * and the end_io handler clears the writeback ranges
3470 */
3471static int __extent_writepage(struct page *page, struct writeback_control *wbc,
3472 void *data)
3473{
3474 struct inode *inode = page->mapping->host;
3475 struct extent_page_data *epd = data;
3476 u64 start = page_offset(page);
3477 u64 page_end = start + PAGE_CACHE_SIZE - 1;
3478 int ret;
3479 int nr = 0;
3480 size_t pg_offset = 0;
3481 loff_t i_size = i_size_read(inode);
3482 unsigned long end_index = i_size >> PAGE_CACHE_SHIFT;
3483 int write_flags;
3484 unsigned long nr_written = 0;
3485
3486 if (wbc->sync_mode == WB_SYNC_ALL)
3487 write_flags = WRITE_SYNC;
3488 else
3489 write_flags = WRITE;
3490
3491 trace___extent_writepage(page, inode, wbc);
3492
3493 WARN_ON(!PageLocked(page));
3494
3495 ClearPageError(page);
3496
3497 pg_offset = i_size & (PAGE_CACHE_SIZE - 1);
3498 if (page->index > end_index ||
3499 (page->index == end_index && !pg_offset)) {
3500 page->mapping->a_ops->invalidatepage(page, 0, PAGE_CACHE_SIZE);
3501 unlock_page(page);
3502 return 0;
3503 }
3504
3505 if (page->index == end_index) {
3506 char *userpage;
3507
3508 userpage = kmap_atomic(page);
3509 memset(userpage + pg_offset, 0,
3510 PAGE_CACHE_SIZE - pg_offset);
3511 kunmap_atomic(userpage);
3512 flush_dcache_page(page);
3513 }
3514
3515 pg_offset = 0;
3516
3517 set_page_extent_mapped(page);
3518
3519 ret = writepage_delalloc(inode, page, wbc, epd, start, &nr_written);
3520 if (ret == 1)
3521 goto done_unlocked;
3522 if (ret)
3523 goto done;
3524
3525 ret = __extent_writepage_io(inode, page, wbc, epd,
3526 i_size, nr_written, write_flags, &nr);
3527 if (ret == 1)
3528 goto done_unlocked;
3529
3530done:
Chris Masone6dcd2d2008-07-17 12:53:50 -04003531 if (nr == 0) {
Chris Masond1310b22008-01-24 16:13:08 -05003532 /* make sure the mapping tag for page dirty gets cleared */
Chris Mason771ed682008-11-06 22:02:51 -05003533 set_page_writeback(page);
3534 end_page_writeback(page);
3535 }
Filipe Manana61391d52014-05-09 17:17:40 +01003536 if (PageError(page)) {
3537 ret = ret < 0 ? ret : -EIO;
3538 end_extent_writepage(page, ret, start, page_end);
3539 }
Christoph Hellwigb2950862008-12-02 09:54:17 -05003540 unlock_page(page);
Chris Mason40f76582014-05-21 13:35:51 -07003541 return ret;
Chris Mason4bef0842008-09-08 11:18:08 -04003542
3543done_unlocked:
Chris Masond1310b22008-01-24 16:13:08 -05003544 return 0;
3545}
3546
Josef Bacikfd8b2b62013-04-24 16:41:19 -04003547void wait_on_extent_buffer_writeback(struct extent_buffer *eb)
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003548{
NeilBrown74316202014-07-07 15:16:04 +10003549 wait_on_bit_io(&eb->bflags, EXTENT_BUFFER_WRITEBACK,
3550 TASK_UNINTERRUPTIBLE);
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003551}
3552
Chris Mason0e378df2014-05-19 20:55:27 -07003553static noinline_for_stack int
3554lock_extent_buffer_for_io(struct extent_buffer *eb,
3555 struct btrfs_fs_info *fs_info,
3556 struct extent_page_data *epd)
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003557{
3558 unsigned long i, num_pages;
3559 int flush = 0;
3560 int ret = 0;
3561
3562 if (!btrfs_try_tree_write_lock(eb)) {
3563 flush = 1;
3564 flush_write_bio(epd);
3565 btrfs_tree_lock(eb);
3566 }
3567
3568 if (test_bit(EXTENT_BUFFER_WRITEBACK, &eb->bflags)) {
3569 btrfs_tree_unlock(eb);
3570 if (!epd->sync_io)
3571 return 0;
3572 if (!flush) {
3573 flush_write_bio(epd);
3574 flush = 1;
3575 }
Chris Masona098d8e2012-03-21 12:09:56 -04003576 while (1) {
3577 wait_on_extent_buffer_writeback(eb);
3578 btrfs_tree_lock(eb);
3579 if (!test_bit(EXTENT_BUFFER_WRITEBACK, &eb->bflags))
3580 break;
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003581 btrfs_tree_unlock(eb);
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003582 }
3583 }
3584
Josef Bacik51561ff2012-07-20 16:25:24 -04003585 /*
3586 * We need to do this to prevent races in people who check if the eb is
3587 * under IO since we can end up having no IO bits set for a short period
3588 * of time.
3589 */
3590 spin_lock(&eb->refs_lock);
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003591 if (test_and_clear_bit(EXTENT_BUFFER_DIRTY, &eb->bflags)) {
3592 set_bit(EXTENT_BUFFER_WRITEBACK, &eb->bflags);
Josef Bacik51561ff2012-07-20 16:25:24 -04003593 spin_unlock(&eb->refs_lock);
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003594 btrfs_set_header_flag(eb, BTRFS_HEADER_FLAG_WRITTEN);
Miao Xiee2d84522013-01-29 10:09:20 +00003595 __percpu_counter_add(&fs_info->dirty_metadata_bytes,
3596 -eb->len,
3597 fs_info->dirty_metadata_batch);
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003598 ret = 1;
Josef Bacik51561ff2012-07-20 16:25:24 -04003599 } else {
3600 spin_unlock(&eb->refs_lock);
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003601 }
3602
3603 btrfs_tree_unlock(eb);
3604
3605 if (!ret)
3606 return ret;
3607
3608 num_pages = num_extent_pages(eb->start, eb->len);
3609 for (i = 0; i < num_pages; i++) {
David Sterbafb85fc92014-07-31 01:03:53 +02003610 struct page *p = eb->pages[i];
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003611
3612 if (!trylock_page(p)) {
3613 if (!flush) {
3614 flush_write_bio(epd);
3615 flush = 1;
3616 }
3617 lock_page(p);
3618 }
3619 }
3620
3621 return ret;
3622}
3623
3624static void end_extent_buffer_writeback(struct extent_buffer *eb)
3625{
3626 clear_bit(EXTENT_BUFFER_WRITEBACK, &eb->bflags);
Peter Zijlstra4e857c52014-03-17 18:06:10 +01003627 smp_mb__after_atomic();
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003628 wake_up_bit(&eb->bflags, EXTENT_BUFFER_WRITEBACK);
3629}
3630
Filipe Manana656f30d2014-09-26 12:25:56 +01003631static void set_btree_ioerr(struct page *page)
3632{
3633 struct extent_buffer *eb = (struct extent_buffer *)page->private;
3634 struct btrfs_inode *btree_ino = BTRFS_I(eb->fs_info->btree_inode);
3635
3636 SetPageError(page);
3637 if (test_and_set_bit(EXTENT_BUFFER_WRITE_ERR, &eb->bflags))
3638 return;
3639
3640 /*
3641 * If writeback for a btree extent that doesn't belong to a log tree
3642 * failed, increment the counter transaction->eb_write_errors.
3643 * We do this because while the transaction is running and before it's
3644 * committing (when we call filemap_fdata[write|wait]_range against
3645 * the btree inode), we might have
3646 * btree_inode->i_mapping->a_ops->writepages() called by the VM - if it
3647 * returns an error or an error happens during writeback, when we're
3648 * committing the transaction we wouldn't know about it, since the pages
3649 * can be no longer dirty nor marked anymore for writeback (if a
3650 * subsequent modification to the extent buffer didn't happen before the
3651 * transaction commit), which makes filemap_fdata[write|wait]_range not
3652 * able to find the pages tagged with SetPageError at transaction
3653 * commit time. So if this happens we must abort the transaction,
3654 * otherwise we commit a super block with btree roots that point to
3655 * btree nodes/leafs whose content on disk is invalid - either garbage
3656 * or the content of some node/leaf from a past generation that got
3657 * cowed or deleted and is no longer valid.
3658 *
3659 * Note: setting AS_EIO/AS_ENOSPC in the btree inode's i_mapping would
3660 * not be enough - we need to distinguish between log tree extents vs
3661 * non-log tree extents, and the next filemap_fdatawait_range() call
3662 * will catch and clear such errors in the mapping - and that call might
3663 * be from a log sync and not from a transaction commit. Also, checking
3664 * for the eb flag EXTENT_BUFFER_WRITE_ERR at transaction commit time is
3665 * not done and would not be reliable - the eb might have been released
3666 * from memory and reading it back again means that flag would not be
3667 * set (since it's a runtime flag, not persisted on disk).
3668 *
3669 * Using the flags below in the btree inode also makes us achieve the
3670 * goal of AS_EIO/AS_ENOSPC when writepages() returns success, started
3671 * writeback for all dirty pages and before filemap_fdatawait_range()
3672 * is called, the writeback for all dirty pages had already finished
3673 * with errors - because we were not using AS_EIO/AS_ENOSPC,
3674 * filemap_fdatawait_range() would return success, as it could not know
3675 * that writeback errors happened (the pages were no longer tagged for
3676 * writeback).
3677 */
3678 switch (eb->log_index) {
3679 case -1:
3680 set_bit(BTRFS_INODE_BTREE_ERR, &btree_ino->runtime_flags);
3681 break;
3682 case 0:
3683 set_bit(BTRFS_INODE_BTREE_LOG1_ERR, &btree_ino->runtime_flags);
3684 break;
3685 case 1:
3686 set_bit(BTRFS_INODE_BTREE_LOG2_ERR, &btree_ino->runtime_flags);
3687 break;
3688 default:
3689 BUG(); /* unexpected, logic error */
3690 }
3691}
3692
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003693static void end_bio_extent_buffer_writepage(struct bio *bio, int err)
3694{
Kent Overstreet2c30c712013-11-07 12:20:26 -08003695 struct bio_vec *bvec;
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003696 struct extent_buffer *eb;
Kent Overstreet2c30c712013-11-07 12:20:26 -08003697 int i, done;
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003698
Kent Overstreet2c30c712013-11-07 12:20:26 -08003699 bio_for_each_segment_all(bvec, bio, i) {
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003700 struct page *page = bvec->bv_page;
3701
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003702 eb = (struct extent_buffer *)page->private;
3703 BUG_ON(!eb);
3704 done = atomic_dec_and_test(&eb->io_pages);
3705
Filipe Manana656f30d2014-09-26 12:25:56 +01003706 if (err || test_bit(EXTENT_BUFFER_WRITE_ERR, &eb->bflags)) {
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003707 ClearPageUptodate(page);
Filipe Manana656f30d2014-09-26 12:25:56 +01003708 set_btree_ioerr(page);
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003709 }
3710
3711 end_page_writeback(page);
3712
3713 if (!done)
3714 continue;
3715
3716 end_extent_buffer_writeback(eb);
Kent Overstreet2c30c712013-11-07 12:20:26 -08003717 }
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003718
3719 bio_put(bio);
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003720}
3721
Chris Mason0e378df2014-05-19 20:55:27 -07003722static noinline_for_stack int write_one_eb(struct extent_buffer *eb,
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003723 struct btrfs_fs_info *fs_info,
3724 struct writeback_control *wbc,
3725 struct extent_page_data *epd)
3726{
3727 struct block_device *bdev = fs_info->fs_devices->latest_bdev;
Josef Bacikf28491e2013-12-16 13:24:27 -05003728 struct extent_io_tree *tree = &BTRFS_I(fs_info->btree_inode)->io_tree;
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003729 u64 offset = eb->start;
3730 unsigned long i, num_pages;
Josef Bacikde0022b2012-09-25 14:25:58 -04003731 unsigned long bio_flags = 0;
Josef Bacikd4c7ca82013-04-19 19:49:09 -04003732 int rw = (epd->sync_io ? WRITE_SYNC : WRITE) | REQ_META;
Josef Bacikd7dbe9e2012-04-23 14:00:51 -04003733 int ret = 0;
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003734
Filipe Manana656f30d2014-09-26 12:25:56 +01003735 clear_bit(EXTENT_BUFFER_WRITE_ERR, &eb->bflags);
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003736 num_pages = num_extent_pages(eb->start, eb->len);
3737 atomic_set(&eb->io_pages, num_pages);
Josef Bacikde0022b2012-09-25 14:25:58 -04003738 if (btrfs_header_owner(eb) == BTRFS_TREE_LOG_OBJECTID)
3739 bio_flags = EXTENT_BIO_TREE_LOG;
3740
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003741 for (i = 0; i < num_pages; i++) {
David Sterbafb85fc92014-07-31 01:03:53 +02003742 struct page *p = eb->pages[i];
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003743
3744 clear_page_dirty_for_io(p);
3745 set_page_writeback(p);
Josef Bacikf28491e2013-12-16 13:24:27 -05003746 ret = submit_extent_page(rw, tree, p, offset >> 9,
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003747 PAGE_CACHE_SIZE, 0, bdev, &epd->bio,
3748 -1, end_bio_extent_buffer_writepage,
Josef Bacikde0022b2012-09-25 14:25:58 -04003749 0, epd->bio_flags, bio_flags);
3750 epd->bio_flags = bio_flags;
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003751 if (ret) {
Filipe Manana656f30d2014-09-26 12:25:56 +01003752 set_btree_ioerr(p);
Filipe Manana55e3bd22014-09-22 17:41:04 +01003753 end_page_writeback(p);
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003754 if (atomic_sub_and_test(num_pages - i, &eb->io_pages))
3755 end_extent_buffer_writeback(eb);
3756 ret = -EIO;
3757 break;
3758 }
3759 offset += PAGE_CACHE_SIZE;
3760 update_nr_written(p, wbc, 1);
3761 unlock_page(p);
3762 }
3763
3764 if (unlikely(ret)) {
3765 for (; i < num_pages; i++) {
Chris Masonbbf65cf2014-10-04 09:56:45 -07003766 struct page *p = eb->pages[i];
Liu Bo81465022014-09-23 22:22:33 +08003767 clear_page_dirty_for_io(p);
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003768 unlock_page(p);
3769 }
3770 }
3771
3772 return ret;
3773}
3774
3775int btree_write_cache_pages(struct address_space *mapping,
3776 struct writeback_control *wbc)
3777{
3778 struct extent_io_tree *tree = &BTRFS_I(mapping->host)->io_tree;
3779 struct btrfs_fs_info *fs_info = BTRFS_I(mapping->host)->root->fs_info;
3780 struct extent_buffer *eb, *prev_eb = NULL;
3781 struct extent_page_data epd = {
3782 .bio = NULL,
3783 .tree = tree,
3784 .extent_locked = 0,
3785 .sync_io = wbc->sync_mode == WB_SYNC_ALL,
Josef Bacikde0022b2012-09-25 14:25:58 -04003786 .bio_flags = 0,
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003787 };
3788 int ret = 0;
3789 int done = 0;
3790 int nr_to_write_done = 0;
3791 struct pagevec pvec;
3792 int nr_pages;
3793 pgoff_t index;
3794 pgoff_t end; /* Inclusive */
3795 int scanned = 0;
3796 int tag;
3797
3798 pagevec_init(&pvec, 0);
3799 if (wbc->range_cyclic) {
3800 index = mapping->writeback_index; /* Start from prev offset */
3801 end = -1;
3802 } else {
3803 index = wbc->range_start >> PAGE_CACHE_SHIFT;
3804 end = wbc->range_end >> PAGE_CACHE_SHIFT;
3805 scanned = 1;
3806 }
3807 if (wbc->sync_mode == WB_SYNC_ALL)
3808 tag = PAGECACHE_TAG_TOWRITE;
3809 else
3810 tag = PAGECACHE_TAG_DIRTY;
3811retry:
3812 if (wbc->sync_mode == WB_SYNC_ALL)
3813 tag_pages_for_writeback(mapping, index, end);
3814 while (!done && !nr_to_write_done && (index <= end) &&
3815 (nr_pages = pagevec_lookup_tag(&pvec, mapping, &index, tag,
3816 min(end - index, (pgoff_t)PAGEVEC_SIZE-1) + 1))) {
3817 unsigned i;
3818
3819 scanned = 1;
3820 for (i = 0; i < nr_pages; i++) {
3821 struct page *page = pvec.pages[i];
3822
3823 if (!PagePrivate(page))
3824 continue;
3825
3826 if (!wbc->range_cyclic && page->index > end) {
3827 done = 1;
3828 break;
3829 }
3830
Josef Bacikb5bae262012-09-14 13:43:01 -04003831 spin_lock(&mapping->private_lock);
3832 if (!PagePrivate(page)) {
3833 spin_unlock(&mapping->private_lock);
3834 continue;
3835 }
3836
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003837 eb = (struct extent_buffer *)page->private;
Josef Bacikb5bae262012-09-14 13:43:01 -04003838
3839 /*
3840 * Shouldn't happen and normally this would be a BUG_ON
3841 * but no sense in crashing the users box for something
3842 * we can survive anyway.
3843 */
Dulshani Gunawardhanafae7f212013-10-31 10:30:08 +05303844 if (WARN_ON(!eb)) {
Josef Bacikb5bae262012-09-14 13:43:01 -04003845 spin_unlock(&mapping->private_lock);
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003846 continue;
3847 }
3848
Josef Bacikb5bae262012-09-14 13:43:01 -04003849 if (eb == prev_eb) {
3850 spin_unlock(&mapping->private_lock);
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003851 continue;
3852 }
3853
Josef Bacikb5bae262012-09-14 13:43:01 -04003854 ret = atomic_inc_not_zero(&eb->refs);
3855 spin_unlock(&mapping->private_lock);
3856 if (!ret)
3857 continue;
3858
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003859 prev_eb = eb;
3860 ret = lock_extent_buffer_for_io(eb, fs_info, &epd);
3861 if (!ret) {
3862 free_extent_buffer(eb);
3863 continue;
3864 }
3865
3866 ret = write_one_eb(eb, fs_info, wbc, &epd);
3867 if (ret) {
3868 done = 1;
3869 free_extent_buffer(eb);
3870 break;
3871 }
3872 free_extent_buffer(eb);
3873
3874 /*
3875 * the filesystem may choose to bump up nr_to_write.
3876 * We have to make sure to honor the new nr_to_write
3877 * at any time
3878 */
3879 nr_to_write_done = wbc->nr_to_write <= 0;
3880 }
3881 pagevec_release(&pvec);
3882 cond_resched();
3883 }
3884 if (!scanned && !done) {
3885 /*
3886 * We hit the last page and there is more work to be done: wrap
3887 * back to the start of the file
3888 */
3889 scanned = 1;
3890 index = 0;
3891 goto retry;
3892 }
3893 flush_write_bio(&epd);
3894 return ret;
3895}
3896
Chris Masond1310b22008-01-24 16:13:08 -05003897/**
3898 * write_cache_pages - walk the list of dirty pages of the given address space and write all of them.
3899 * @mapping: address space structure to write
3900 * @wbc: subtract the number of written pages from *@wbc->nr_to_write
3901 * @writepage: function called for each page
3902 * @data: data passed to writepage function
3903 *
3904 * If a page is already under I/O, write_cache_pages() skips it, even
3905 * if it's dirty. This is desirable behaviour for memory-cleaning writeback,
3906 * but it is INCORRECT for data-integrity system calls such as fsync(). fsync()
3907 * and msync() need to guarantee that all the data which was dirty at the time
3908 * the call was made get new I/O started against them. If wbc->sync_mode is
3909 * WB_SYNC_ALL then we were called for data integrity and we must wait for
3910 * existing IO to complete.
3911 */
Chris Mason4bef0842008-09-08 11:18:08 -04003912static int extent_write_cache_pages(struct extent_io_tree *tree,
3913 struct address_space *mapping,
3914 struct writeback_control *wbc,
Chris Masond2c3f4f2008-11-19 12:44:22 -05003915 writepage_t writepage, void *data,
3916 void (*flush_fn)(void *))
Chris Masond1310b22008-01-24 16:13:08 -05003917{
Josef Bacik7fd1a3f2012-06-27 17:18:41 -04003918 struct inode *inode = mapping->host;
Chris Masond1310b22008-01-24 16:13:08 -05003919 int ret = 0;
3920 int done = 0;
Filipe Manana61391d52014-05-09 17:17:40 +01003921 int err = 0;
Chris Masonf85d7d6c2009-09-18 16:03:16 -04003922 int nr_to_write_done = 0;
Chris Masond1310b22008-01-24 16:13:08 -05003923 struct pagevec pvec;
3924 int nr_pages;
3925 pgoff_t index;
3926 pgoff_t end; /* Inclusive */
3927 int scanned = 0;
Josef Bacikf7aaa062011-07-15 21:26:38 +00003928 int tag;
Chris Masond1310b22008-01-24 16:13:08 -05003929
Josef Bacik7fd1a3f2012-06-27 17:18:41 -04003930 /*
3931 * We have to hold onto the inode so that ordered extents can do their
3932 * work when the IO finishes. The alternative to this is failing to add
3933 * an ordered extent if the igrab() fails there and that is a huge pain
3934 * to deal with, so instead just hold onto the inode throughout the
3935 * writepages operation. If it fails here we are freeing up the inode
3936 * anyway and we'd rather not waste our time writing out stuff that is
3937 * going to be truncated anyway.
3938 */
3939 if (!igrab(inode))
3940 return 0;
3941
Chris Masond1310b22008-01-24 16:13:08 -05003942 pagevec_init(&pvec, 0);
3943 if (wbc->range_cyclic) {
3944 index = mapping->writeback_index; /* Start from prev offset */
3945 end = -1;
3946 } else {
3947 index = wbc->range_start >> PAGE_CACHE_SHIFT;
3948 end = wbc->range_end >> PAGE_CACHE_SHIFT;
Chris Masond1310b22008-01-24 16:13:08 -05003949 scanned = 1;
3950 }
Josef Bacikf7aaa062011-07-15 21:26:38 +00003951 if (wbc->sync_mode == WB_SYNC_ALL)
3952 tag = PAGECACHE_TAG_TOWRITE;
3953 else
3954 tag = PAGECACHE_TAG_DIRTY;
Chris Masond1310b22008-01-24 16:13:08 -05003955retry:
Josef Bacikf7aaa062011-07-15 21:26:38 +00003956 if (wbc->sync_mode == WB_SYNC_ALL)
3957 tag_pages_for_writeback(mapping, index, end);
Chris Masonf85d7d6c2009-09-18 16:03:16 -04003958 while (!done && !nr_to_write_done && (index <= end) &&
Josef Bacikf7aaa062011-07-15 21:26:38 +00003959 (nr_pages = pagevec_lookup_tag(&pvec, mapping, &index, tag,
3960 min(end - index, (pgoff_t)PAGEVEC_SIZE-1) + 1))) {
Chris Masond1310b22008-01-24 16:13:08 -05003961 unsigned i;
3962
3963 scanned = 1;
3964 for (i = 0; i < nr_pages; i++) {
3965 struct page *page = pvec.pages[i];
3966
3967 /*
3968 * At this point we hold neither mapping->tree_lock nor
3969 * lock on the page itself: the page may be truncated or
3970 * invalidated (changing page->mapping to NULL), or even
3971 * swizzled back from swapper_space to tmpfs file
3972 * mapping
3973 */
Josef Bacikc8f2f242013-02-11 11:33:00 -05003974 if (!trylock_page(page)) {
3975 flush_fn(data);
3976 lock_page(page);
Chris Mason01d658f2011-11-01 10:08:06 -04003977 }
Chris Masond1310b22008-01-24 16:13:08 -05003978
3979 if (unlikely(page->mapping != mapping)) {
3980 unlock_page(page);
3981 continue;
3982 }
3983
3984 if (!wbc->range_cyclic && page->index > end) {
3985 done = 1;
3986 unlock_page(page);
3987 continue;
3988 }
3989
Chris Masond2c3f4f2008-11-19 12:44:22 -05003990 if (wbc->sync_mode != WB_SYNC_NONE) {
Chris Mason0e6bd952008-11-20 10:46:35 -05003991 if (PageWriteback(page))
3992 flush_fn(data);
Chris Masond1310b22008-01-24 16:13:08 -05003993 wait_on_page_writeback(page);
Chris Masond2c3f4f2008-11-19 12:44:22 -05003994 }
Chris Masond1310b22008-01-24 16:13:08 -05003995
3996 if (PageWriteback(page) ||
3997 !clear_page_dirty_for_io(page)) {
3998 unlock_page(page);
3999 continue;
4000 }
4001
4002 ret = (*writepage)(page, wbc, data);
4003
4004 if (unlikely(ret == AOP_WRITEPAGE_ACTIVATE)) {
4005 unlock_page(page);
4006 ret = 0;
4007 }
Filipe Manana61391d52014-05-09 17:17:40 +01004008 if (!err && ret < 0)
4009 err = ret;
Chris Masonf85d7d6c2009-09-18 16:03:16 -04004010
4011 /*
4012 * the filesystem may choose to bump up nr_to_write.
4013 * We have to make sure to honor the new nr_to_write
4014 * at any time
4015 */
4016 nr_to_write_done = wbc->nr_to_write <= 0;
Chris Masond1310b22008-01-24 16:13:08 -05004017 }
4018 pagevec_release(&pvec);
4019 cond_resched();
4020 }
Filipe Manana61391d52014-05-09 17:17:40 +01004021 if (!scanned && !done && !err) {
Chris Masond1310b22008-01-24 16:13:08 -05004022 /*
4023 * We hit the last page and there is more work to be done: wrap
4024 * back to the start of the file
4025 */
4026 scanned = 1;
4027 index = 0;
4028 goto retry;
4029 }
Josef Bacik7fd1a3f2012-06-27 17:18:41 -04004030 btrfs_add_delayed_iput(inode);
Filipe Manana61391d52014-05-09 17:17:40 +01004031 return err;
Chris Masond1310b22008-01-24 16:13:08 -05004032}
Chris Masond1310b22008-01-24 16:13:08 -05004033
Chris Masonffbd5172009-04-20 15:50:09 -04004034static void flush_epd_write_bio(struct extent_page_data *epd)
4035{
4036 if (epd->bio) {
Jeff Mahoney355808c2011-10-03 23:23:14 -04004037 int rw = WRITE;
4038 int ret;
4039
Chris Masonffbd5172009-04-20 15:50:09 -04004040 if (epd->sync_io)
Jeff Mahoney355808c2011-10-03 23:23:14 -04004041 rw = WRITE_SYNC;
4042
Josef Bacikde0022b2012-09-25 14:25:58 -04004043 ret = submit_one_bio(rw, epd->bio, 0, epd->bio_flags);
Jeff Mahoney79787ea2012-03-12 16:03:00 +01004044 BUG_ON(ret < 0); /* -ENOMEM */
Chris Masonffbd5172009-04-20 15:50:09 -04004045 epd->bio = NULL;
4046 }
4047}
4048
Chris Masond2c3f4f2008-11-19 12:44:22 -05004049static noinline void flush_write_bio(void *data)
4050{
4051 struct extent_page_data *epd = data;
Chris Masonffbd5172009-04-20 15:50:09 -04004052 flush_epd_write_bio(epd);
Chris Masond2c3f4f2008-11-19 12:44:22 -05004053}
4054
Chris Masond1310b22008-01-24 16:13:08 -05004055int extent_write_full_page(struct extent_io_tree *tree, struct page *page,
4056 get_extent_t *get_extent,
4057 struct writeback_control *wbc)
4058{
4059 int ret;
Chris Masond1310b22008-01-24 16:13:08 -05004060 struct extent_page_data epd = {
4061 .bio = NULL,
4062 .tree = tree,
4063 .get_extent = get_extent,
Chris Mason771ed682008-11-06 22:02:51 -05004064 .extent_locked = 0,
Chris Masonffbd5172009-04-20 15:50:09 -04004065 .sync_io = wbc->sync_mode == WB_SYNC_ALL,
Josef Bacikde0022b2012-09-25 14:25:58 -04004066 .bio_flags = 0,
Chris Masond1310b22008-01-24 16:13:08 -05004067 };
Chris Masond1310b22008-01-24 16:13:08 -05004068
Chris Masond1310b22008-01-24 16:13:08 -05004069 ret = __extent_writepage(page, wbc, &epd);
4070
Chris Masonffbd5172009-04-20 15:50:09 -04004071 flush_epd_write_bio(&epd);
Chris Masond1310b22008-01-24 16:13:08 -05004072 return ret;
4073}
Chris Masond1310b22008-01-24 16:13:08 -05004074
Chris Mason771ed682008-11-06 22:02:51 -05004075int extent_write_locked_range(struct extent_io_tree *tree, struct inode *inode,
4076 u64 start, u64 end, get_extent_t *get_extent,
4077 int mode)
4078{
4079 int ret = 0;
4080 struct address_space *mapping = inode->i_mapping;
4081 struct page *page;
4082 unsigned long nr_pages = (end - start + PAGE_CACHE_SIZE) >>
4083 PAGE_CACHE_SHIFT;
4084
4085 struct extent_page_data epd = {
4086 .bio = NULL,
4087 .tree = tree,
4088 .get_extent = get_extent,
4089 .extent_locked = 1,
Chris Masonffbd5172009-04-20 15:50:09 -04004090 .sync_io = mode == WB_SYNC_ALL,
Josef Bacikde0022b2012-09-25 14:25:58 -04004091 .bio_flags = 0,
Chris Mason771ed682008-11-06 22:02:51 -05004092 };
4093 struct writeback_control wbc_writepages = {
Chris Mason771ed682008-11-06 22:02:51 -05004094 .sync_mode = mode,
Chris Mason771ed682008-11-06 22:02:51 -05004095 .nr_to_write = nr_pages * 2,
4096 .range_start = start,
4097 .range_end = end + 1,
4098 };
4099
Chris Masond3977122009-01-05 21:25:51 -05004100 while (start <= end) {
Chris Mason771ed682008-11-06 22:02:51 -05004101 page = find_get_page(mapping, start >> PAGE_CACHE_SHIFT);
4102 if (clear_page_dirty_for_io(page))
4103 ret = __extent_writepage(page, &wbc_writepages, &epd);
4104 else {
4105 if (tree->ops && tree->ops->writepage_end_io_hook)
4106 tree->ops->writepage_end_io_hook(page, start,
4107 start + PAGE_CACHE_SIZE - 1,
4108 NULL, 1);
4109 unlock_page(page);
4110 }
4111 page_cache_release(page);
4112 start += PAGE_CACHE_SIZE;
4113 }
4114
Chris Masonffbd5172009-04-20 15:50:09 -04004115 flush_epd_write_bio(&epd);
Chris Mason771ed682008-11-06 22:02:51 -05004116 return ret;
4117}
Chris Masond1310b22008-01-24 16:13:08 -05004118
4119int extent_writepages(struct extent_io_tree *tree,
4120 struct address_space *mapping,
4121 get_extent_t *get_extent,
4122 struct writeback_control *wbc)
4123{
4124 int ret = 0;
4125 struct extent_page_data epd = {
4126 .bio = NULL,
4127 .tree = tree,
4128 .get_extent = get_extent,
Chris Mason771ed682008-11-06 22:02:51 -05004129 .extent_locked = 0,
Chris Masonffbd5172009-04-20 15:50:09 -04004130 .sync_io = wbc->sync_mode == WB_SYNC_ALL,
Josef Bacikde0022b2012-09-25 14:25:58 -04004131 .bio_flags = 0,
Chris Masond1310b22008-01-24 16:13:08 -05004132 };
4133
Chris Mason4bef0842008-09-08 11:18:08 -04004134 ret = extent_write_cache_pages(tree, mapping, wbc,
Chris Masond2c3f4f2008-11-19 12:44:22 -05004135 __extent_writepage, &epd,
4136 flush_write_bio);
Chris Masonffbd5172009-04-20 15:50:09 -04004137 flush_epd_write_bio(&epd);
Chris Masond1310b22008-01-24 16:13:08 -05004138 return ret;
4139}
Chris Masond1310b22008-01-24 16:13:08 -05004140
4141int extent_readpages(struct extent_io_tree *tree,
4142 struct address_space *mapping,
4143 struct list_head *pages, unsigned nr_pages,
4144 get_extent_t get_extent)
4145{
4146 struct bio *bio = NULL;
4147 unsigned page_idx;
Chris Masonc8b97812008-10-29 14:49:59 -04004148 unsigned long bio_flags = 0;
Liu Bo67c96842012-07-20 21:43:09 -06004149 struct page *pagepool[16];
4150 struct page *page;
Miao Xie125bac012013-07-25 19:22:37 +08004151 struct extent_map *em_cached = NULL;
Liu Bo67c96842012-07-20 21:43:09 -06004152 int nr = 0;
Chris Masond1310b22008-01-24 16:13:08 -05004153
Chris Masond1310b22008-01-24 16:13:08 -05004154 for (page_idx = 0; page_idx < nr_pages; page_idx++) {
Liu Bo67c96842012-07-20 21:43:09 -06004155 page = list_entry(pages->prev, struct page, lru);
Chris Masond1310b22008-01-24 16:13:08 -05004156
4157 prefetchw(&page->flags);
4158 list_del(&page->lru);
Liu Bo67c96842012-07-20 21:43:09 -06004159 if (add_to_page_cache_lru(page, mapping,
Itaru Kitayama43e817a2011-04-25 19:43:51 -04004160 page->index, GFP_NOFS)) {
Liu Bo67c96842012-07-20 21:43:09 -06004161 page_cache_release(page);
4162 continue;
Chris Masond1310b22008-01-24 16:13:08 -05004163 }
Liu Bo67c96842012-07-20 21:43:09 -06004164
4165 pagepool[nr++] = page;
4166 if (nr < ARRAY_SIZE(pagepool))
4167 continue;
Miao Xie125bac012013-07-25 19:22:37 +08004168 __extent_readpages(tree, pagepool, nr, get_extent, &em_cached,
Miao Xie99740902013-07-25 19:22:36 +08004169 &bio, 0, &bio_flags, READ);
Liu Bo67c96842012-07-20 21:43:09 -06004170 nr = 0;
Chris Masond1310b22008-01-24 16:13:08 -05004171 }
Miao Xie99740902013-07-25 19:22:36 +08004172 if (nr)
Miao Xie125bac012013-07-25 19:22:37 +08004173 __extent_readpages(tree, pagepool, nr, get_extent, &em_cached,
Miao Xie99740902013-07-25 19:22:36 +08004174 &bio, 0, &bio_flags, READ);
Liu Bo67c96842012-07-20 21:43:09 -06004175
Miao Xie125bac012013-07-25 19:22:37 +08004176 if (em_cached)
4177 free_extent_map(em_cached);
4178
Chris Masond1310b22008-01-24 16:13:08 -05004179 BUG_ON(!list_empty(pages));
4180 if (bio)
Jeff Mahoney79787ea2012-03-12 16:03:00 +01004181 return submit_one_bio(READ, bio, 0, bio_flags);
Chris Masond1310b22008-01-24 16:13:08 -05004182 return 0;
4183}
Chris Masond1310b22008-01-24 16:13:08 -05004184
4185/*
4186 * basic invalidatepage code, this waits on any locked or writeback
4187 * ranges corresponding to the page, and then deletes any extent state
4188 * records from the tree
4189 */
4190int extent_invalidatepage(struct extent_io_tree *tree,
4191 struct page *page, unsigned long offset)
4192{
Josef Bacik2ac55d42010-02-03 19:33:23 +00004193 struct extent_state *cached_state = NULL;
Miao Xie4eee4fa2012-12-21 09:17:45 +00004194 u64 start = page_offset(page);
Chris Masond1310b22008-01-24 16:13:08 -05004195 u64 end = start + PAGE_CACHE_SIZE - 1;
4196 size_t blocksize = page->mapping->host->i_sb->s_blocksize;
4197
Qu Wenruofda28322013-02-26 08:10:22 +00004198 start += ALIGN(offset, blocksize);
Chris Masond1310b22008-01-24 16:13:08 -05004199 if (start > end)
4200 return 0;
4201
Jeff Mahoneyd0082372012-03-01 14:57:19 +01004202 lock_extent_bits(tree, start, end, 0, &cached_state);
Chris Mason1edbb732009-09-02 13:24:36 -04004203 wait_on_page_writeback(page);
Chris Masond1310b22008-01-24 16:13:08 -05004204 clear_extent_bit(tree, start, end,
Josef Bacik32c00af2009-10-08 13:34:05 -04004205 EXTENT_LOCKED | EXTENT_DIRTY | EXTENT_DELALLOC |
4206 EXTENT_DO_ACCOUNTING,
Josef Bacik2ac55d42010-02-03 19:33:23 +00004207 1, 1, &cached_state, GFP_NOFS);
Chris Masond1310b22008-01-24 16:13:08 -05004208 return 0;
4209}
Chris Masond1310b22008-01-24 16:13:08 -05004210
4211/*
Chris Mason7b13b7b2008-04-18 10:29:50 -04004212 * a helper for releasepage, this tests for areas of the page that
4213 * are locked or under IO and drops the related state bits if it is safe
4214 * to drop the page.
4215 */
Eric Sandeen48a3b632013-04-25 20:41:01 +00004216static int try_release_extent_state(struct extent_map_tree *map,
4217 struct extent_io_tree *tree,
4218 struct page *page, gfp_t mask)
Chris Mason7b13b7b2008-04-18 10:29:50 -04004219{
Miao Xie4eee4fa2012-12-21 09:17:45 +00004220 u64 start = page_offset(page);
Chris Mason7b13b7b2008-04-18 10:29:50 -04004221 u64 end = start + PAGE_CACHE_SIZE - 1;
4222 int ret = 1;
4223
Chris Mason211f90e2008-07-18 11:56:15 -04004224 if (test_range_bit(tree, start, end,
Chris Mason8b62b722009-09-02 16:53:46 -04004225 EXTENT_IOBITS, 0, NULL))
Chris Mason7b13b7b2008-04-18 10:29:50 -04004226 ret = 0;
4227 else {
4228 if ((mask & GFP_NOFS) == GFP_NOFS)
4229 mask = GFP_NOFS;
Chris Mason11ef1602009-09-23 20:28:46 -04004230 /*
4231 * at this point we can safely clear everything except the
4232 * locked bit and the nodatasum bit
4233 */
Chris Masone3f24cc2011-02-14 12:52:08 -05004234 ret = clear_extent_bit(tree, start, end,
Chris Mason11ef1602009-09-23 20:28:46 -04004235 ~(EXTENT_LOCKED | EXTENT_NODATASUM),
4236 0, 0, NULL, mask);
Chris Masone3f24cc2011-02-14 12:52:08 -05004237
4238 /* if clear_extent_bit failed for enomem reasons,
4239 * we can't allow the release to continue.
4240 */
4241 if (ret < 0)
4242 ret = 0;
4243 else
4244 ret = 1;
Chris Mason7b13b7b2008-04-18 10:29:50 -04004245 }
4246 return ret;
4247}
Chris Mason7b13b7b2008-04-18 10:29:50 -04004248
4249/*
Chris Masond1310b22008-01-24 16:13:08 -05004250 * a helper for releasepage. As long as there are no locked extents
4251 * in the range corresponding to the page, both state records and extent
4252 * map records are removed
4253 */
4254int try_release_extent_mapping(struct extent_map_tree *map,
Chris Mason70dec802008-01-29 09:59:12 -05004255 struct extent_io_tree *tree, struct page *page,
4256 gfp_t mask)
Chris Masond1310b22008-01-24 16:13:08 -05004257{
4258 struct extent_map *em;
Miao Xie4eee4fa2012-12-21 09:17:45 +00004259 u64 start = page_offset(page);
Chris Masond1310b22008-01-24 16:13:08 -05004260 u64 end = start + PAGE_CACHE_SIZE - 1;
Chris Mason7b13b7b2008-04-18 10:29:50 -04004261
Chris Mason70dec802008-01-29 09:59:12 -05004262 if ((mask & __GFP_WAIT) &&
4263 page->mapping->host->i_size > 16 * 1024 * 1024) {
Yan39b56372008-02-15 10:40:50 -05004264 u64 len;
Chris Mason70dec802008-01-29 09:59:12 -05004265 while (start <= end) {
Yan39b56372008-02-15 10:40:50 -05004266 len = end - start + 1;
Chris Mason890871b2009-09-02 16:24:52 -04004267 write_lock(&map->lock);
Yan39b56372008-02-15 10:40:50 -05004268 em = lookup_extent_mapping(map, start, len);
Tsutomu Itoh285190d2012-02-16 16:23:58 +09004269 if (!em) {
Chris Mason890871b2009-09-02 16:24:52 -04004270 write_unlock(&map->lock);
Chris Mason70dec802008-01-29 09:59:12 -05004271 break;
4272 }
Chris Mason7f3c74f2008-07-18 12:01:11 -04004273 if (test_bit(EXTENT_FLAG_PINNED, &em->flags) ||
4274 em->start != start) {
Chris Mason890871b2009-09-02 16:24:52 -04004275 write_unlock(&map->lock);
Chris Mason70dec802008-01-29 09:59:12 -05004276 free_extent_map(em);
4277 break;
4278 }
4279 if (!test_range_bit(tree, em->start,
4280 extent_map_end(em) - 1,
Chris Mason8b62b722009-09-02 16:53:46 -04004281 EXTENT_LOCKED | EXTENT_WRITEBACK,
Chris Mason9655d292009-09-02 15:22:30 -04004282 0, NULL)) {
Chris Mason70dec802008-01-29 09:59:12 -05004283 remove_extent_mapping(map, em);
4284 /* once for the rb tree */
4285 free_extent_map(em);
4286 }
4287 start = extent_map_end(em);
Chris Mason890871b2009-09-02 16:24:52 -04004288 write_unlock(&map->lock);
Chris Mason70dec802008-01-29 09:59:12 -05004289
4290 /* once for us */
Chris Masond1310b22008-01-24 16:13:08 -05004291 free_extent_map(em);
4292 }
Chris Masond1310b22008-01-24 16:13:08 -05004293 }
Chris Mason7b13b7b2008-04-18 10:29:50 -04004294 return try_release_extent_state(map, tree, page, mask);
Chris Masond1310b22008-01-24 16:13:08 -05004295}
Chris Masond1310b22008-01-24 16:13:08 -05004296
Chris Masonec29ed52011-02-23 16:23:20 -05004297/*
4298 * helper function for fiemap, which doesn't want to see any holes.
4299 * This maps until we find something past 'last'
4300 */
4301static struct extent_map *get_extent_skip_holes(struct inode *inode,
4302 u64 offset,
4303 u64 last,
4304 get_extent_t *get_extent)
4305{
4306 u64 sectorsize = BTRFS_I(inode)->root->sectorsize;
4307 struct extent_map *em;
4308 u64 len;
4309
4310 if (offset >= last)
4311 return NULL;
4312
Dulshani Gunawardhana67871252013-10-31 10:33:04 +05304313 while (1) {
Chris Masonec29ed52011-02-23 16:23:20 -05004314 len = last - offset;
4315 if (len == 0)
4316 break;
Qu Wenruofda28322013-02-26 08:10:22 +00004317 len = ALIGN(len, sectorsize);
Chris Masonec29ed52011-02-23 16:23:20 -05004318 em = get_extent(inode, NULL, 0, offset, len, 0);
David Sterbac7040052011-04-19 18:00:01 +02004319 if (IS_ERR_OR_NULL(em))
Chris Masonec29ed52011-02-23 16:23:20 -05004320 return em;
4321
4322 /* if this isn't a hole return it */
4323 if (!test_bit(EXTENT_FLAG_VACANCY, &em->flags) &&
4324 em->block_start != EXTENT_MAP_HOLE) {
4325 return em;
4326 }
4327
4328 /* this is a hole, advance to the next extent */
4329 offset = extent_map_end(em);
4330 free_extent_map(em);
4331 if (offset >= last)
4332 break;
4333 }
4334 return NULL;
4335}
4336
Yehuda Sadeh1506fcc2009-01-21 14:39:14 -05004337int extent_fiemap(struct inode *inode, struct fiemap_extent_info *fieinfo,
4338 __u64 start, __u64 len, get_extent_t *get_extent)
4339{
Josef Bacik975f84f2010-11-23 19:36:57 +00004340 int ret = 0;
Yehuda Sadeh1506fcc2009-01-21 14:39:14 -05004341 u64 off = start;
4342 u64 max = start + len;
4343 u32 flags = 0;
Josef Bacik975f84f2010-11-23 19:36:57 +00004344 u32 found_type;
4345 u64 last;
Chris Masonec29ed52011-02-23 16:23:20 -05004346 u64 last_for_get_extent = 0;
Yehuda Sadeh1506fcc2009-01-21 14:39:14 -05004347 u64 disko = 0;
Chris Masonec29ed52011-02-23 16:23:20 -05004348 u64 isize = i_size_read(inode);
Josef Bacik975f84f2010-11-23 19:36:57 +00004349 struct btrfs_key found_key;
Yehuda Sadeh1506fcc2009-01-21 14:39:14 -05004350 struct extent_map *em = NULL;
Josef Bacik2ac55d42010-02-03 19:33:23 +00004351 struct extent_state *cached_state = NULL;
Josef Bacik975f84f2010-11-23 19:36:57 +00004352 struct btrfs_path *path;
Josef Bacikdc046b12014-09-10 16:20:45 -04004353 struct btrfs_root *root = BTRFS_I(inode)->root;
Yehuda Sadeh1506fcc2009-01-21 14:39:14 -05004354 int end = 0;
Chris Masonec29ed52011-02-23 16:23:20 -05004355 u64 em_start = 0;
4356 u64 em_len = 0;
4357 u64 em_end = 0;
Yehuda Sadeh1506fcc2009-01-21 14:39:14 -05004358
4359 if (len == 0)
4360 return -EINVAL;
4361
Josef Bacik975f84f2010-11-23 19:36:57 +00004362 path = btrfs_alloc_path();
4363 if (!path)
4364 return -ENOMEM;
4365 path->leave_spinning = 1;
4366
Qu Wenruo2c919432014-07-18 09:55:43 +08004367 start = round_down(start, BTRFS_I(inode)->root->sectorsize);
4368 len = round_up(max, BTRFS_I(inode)->root->sectorsize) - start;
Josef Bacik4d479cf2011-11-17 11:34:31 -05004369
Chris Masonec29ed52011-02-23 16:23:20 -05004370 /*
4371 * lookup the last file extent. We're not using i_size here
4372 * because there might be preallocation past i_size
4373 */
Josef Bacikdc046b12014-09-10 16:20:45 -04004374 ret = btrfs_lookup_file_extent(NULL, root, path, btrfs_ino(inode), -1,
4375 0);
Josef Bacik975f84f2010-11-23 19:36:57 +00004376 if (ret < 0) {
4377 btrfs_free_path(path);
4378 return ret;
4379 }
4380 WARN_ON(!ret);
4381 path->slots[0]--;
Josef Bacik975f84f2010-11-23 19:36:57 +00004382 btrfs_item_key_to_cpu(path->nodes[0], &found_key, path->slots[0]);
David Sterba962a2982014-06-04 18:41:45 +02004383 found_type = found_key.type;
Josef Bacik975f84f2010-11-23 19:36:57 +00004384
Chris Masonec29ed52011-02-23 16:23:20 -05004385 /* No extents, but there might be delalloc bits */
Li Zefan33345d012011-04-20 10:31:50 +08004386 if (found_key.objectid != btrfs_ino(inode) ||
Josef Bacik975f84f2010-11-23 19:36:57 +00004387 found_type != BTRFS_EXTENT_DATA_KEY) {
Chris Masonec29ed52011-02-23 16:23:20 -05004388 /* have to trust i_size as the end */
4389 last = (u64)-1;
4390 last_for_get_extent = isize;
4391 } else {
4392 /*
4393 * remember the start of the last extent. There are a
4394 * bunch of different factors that go into the length of the
4395 * extent, so its much less complex to remember where it started
4396 */
4397 last = found_key.offset;
4398 last_for_get_extent = last + 1;
Josef Bacik975f84f2010-11-23 19:36:57 +00004399 }
Liu Bofe09e162013-09-22 12:54:23 +08004400 btrfs_release_path(path);
Josef Bacik975f84f2010-11-23 19:36:57 +00004401
Chris Masonec29ed52011-02-23 16:23:20 -05004402 /*
4403 * we might have some extents allocated but more delalloc past those
4404 * extents. so, we trust isize unless the start of the last extent is
4405 * beyond isize
4406 */
4407 if (last < isize) {
4408 last = (u64)-1;
4409 last_for_get_extent = isize;
4410 }
4411
Liu Boa52f4cd2013-05-01 16:23:41 +00004412 lock_extent_bits(&BTRFS_I(inode)->io_tree, start, start + len - 1, 0,
Jeff Mahoneyd0082372012-03-01 14:57:19 +01004413 &cached_state);
Chris Masonec29ed52011-02-23 16:23:20 -05004414
Josef Bacik4d479cf2011-11-17 11:34:31 -05004415 em = get_extent_skip_holes(inode, start, last_for_get_extent,
Chris Masonec29ed52011-02-23 16:23:20 -05004416 get_extent);
Yehuda Sadeh1506fcc2009-01-21 14:39:14 -05004417 if (!em)
4418 goto out;
4419 if (IS_ERR(em)) {
4420 ret = PTR_ERR(em);
4421 goto out;
4422 }
Josef Bacik975f84f2010-11-23 19:36:57 +00004423
Yehuda Sadeh1506fcc2009-01-21 14:39:14 -05004424 while (!end) {
Josef Bacikb76bb702013-07-05 13:52:51 -04004425 u64 offset_in_extent = 0;
Yehuda Sadeh1506fcc2009-01-21 14:39:14 -05004426
Chris Masonea8efc72011-03-08 11:54:40 -05004427 /* break if the extent we found is outside the range */
4428 if (em->start >= max || extent_map_end(em) < off)
4429 break;
4430
4431 /*
4432 * get_extent may return an extent that starts before our
4433 * requested range. We have to make sure the ranges
4434 * we return to fiemap always move forward and don't
4435 * overlap, so adjust the offsets here
4436 */
4437 em_start = max(em->start, off);
4438
4439 /*
4440 * record the offset from the start of the extent
Josef Bacikb76bb702013-07-05 13:52:51 -04004441 * for adjusting the disk offset below. Only do this if the
4442 * extent isn't compressed since our in ram offset may be past
4443 * what we have actually allocated on disk.
Chris Masonea8efc72011-03-08 11:54:40 -05004444 */
Josef Bacikb76bb702013-07-05 13:52:51 -04004445 if (!test_bit(EXTENT_FLAG_COMPRESSED, &em->flags))
4446 offset_in_extent = em_start - em->start;
Chris Masonec29ed52011-02-23 16:23:20 -05004447 em_end = extent_map_end(em);
Chris Masonea8efc72011-03-08 11:54:40 -05004448 em_len = em_end - em_start;
Yehuda Sadeh1506fcc2009-01-21 14:39:14 -05004449 disko = 0;
4450 flags = 0;
4451
Chris Masonea8efc72011-03-08 11:54:40 -05004452 /*
4453 * bump off for our next call to get_extent
4454 */
4455 off = extent_map_end(em);
4456 if (off >= max)
4457 end = 1;
4458
Heiko Carstens93dbfad2009-04-03 10:33:45 -04004459 if (em->block_start == EXTENT_MAP_LAST_BYTE) {
Yehuda Sadeh1506fcc2009-01-21 14:39:14 -05004460 end = 1;
4461 flags |= FIEMAP_EXTENT_LAST;
Heiko Carstens93dbfad2009-04-03 10:33:45 -04004462 } else if (em->block_start == EXTENT_MAP_INLINE) {
Yehuda Sadeh1506fcc2009-01-21 14:39:14 -05004463 flags |= (FIEMAP_EXTENT_DATA_INLINE |
4464 FIEMAP_EXTENT_NOT_ALIGNED);
Heiko Carstens93dbfad2009-04-03 10:33:45 -04004465 } else if (em->block_start == EXTENT_MAP_DELALLOC) {
Yehuda Sadeh1506fcc2009-01-21 14:39:14 -05004466 flags |= (FIEMAP_EXTENT_DELALLOC |
4467 FIEMAP_EXTENT_UNKNOWN);
Josef Bacikdc046b12014-09-10 16:20:45 -04004468 } else if (fieinfo->fi_extents_max) {
4469 u64 bytenr = em->block_start -
4470 (em->start - em->orig_start);
Liu Bofe09e162013-09-22 12:54:23 +08004471
Chris Masonea8efc72011-03-08 11:54:40 -05004472 disko = em->block_start + offset_in_extent;
Liu Bofe09e162013-09-22 12:54:23 +08004473
4474 /*
4475 * As btrfs supports shared space, this information
4476 * can be exported to userspace tools via
Josef Bacikdc046b12014-09-10 16:20:45 -04004477 * flag FIEMAP_EXTENT_SHARED. If fi_extents_max == 0
4478 * then we're just getting a count and we can skip the
4479 * lookup stuff.
Liu Bofe09e162013-09-22 12:54:23 +08004480 */
Josef Bacikdc046b12014-09-10 16:20:45 -04004481 ret = btrfs_check_shared(NULL, root->fs_info,
4482 root->objectid,
4483 btrfs_ino(inode), bytenr);
4484 if (ret < 0)
Liu Bofe09e162013-09-22 12:54:23 +08004485 goto out_free;
Josef Bacikdc046b12014-09-10 16:20:45 -04004486 if (ret)
Liu Bofe09e162013-09-22 12:54:23 +08004487 flags |= FIEMAP_EXTENT_SHARED;
Josef Bacikdc046b12014-09-10 16:20:45 -04004488 ret = 0;
Yehuda Sadeh1506fcc2009-01-21 14:39:14 -05004489 }
4490 if (test_bit(EXTENT_FLAG_COMPRESSED, &em->flags))
4491 flags |= FIEMAP_EXTENT_ENCODED;
4492
Yehuda Sadeh1506fcc2009-01-21 14:39:14 -05004493 free_extent_map(em);
4494 em = NULL;
Chris Masonec29ed52011-02-23 16:23:20 -05004495 if ((em_start >= last) || em_len == (u64)-1 ||
4496 (last == (u64)-1 && isize <= em_end)) {
Yehuda Sadeh1506fcc2009-01-21 14:39:14 -05004497 flags |= FIEMAP_EXTENT_LAST;
4498 end = 1;
4499 }
4500
Chris Masonec29ed52011-02-23 16:23:20 -05004501 /* now scan forward to see if this is really the last extent. */
4502 em = get_extent_skip_holes(inode, off, last_for_get_extent,
4503 get_extent);
4504 if (IS_ERR(em)) {
4505 ret = PTR_ERR(em);
4506 goto out;
4507 }
4508 if (!em) {
Josef Bacik975f84f2010-11-23 19:36:57 +00004509 flags |= FIEMAP_EXTENT_LAST;
4510 end = 1;
4511 }
Chris Masonec29ed52011-02-23 16:23:20 -05004512 ret = fiemap_fill_next_extent(fieinfo, em_start, disko,
4513 em_len, flags);
4514 if (ret)
4515 goto out_free;
Yehuda Sadeh1506fcc2009-01-21 14:39:14 -05004516 }
4517out_free:
4518 free_extent_map(em);
4519out:
Liu Bofe09e162013-09-22 12:54:23 +08004520 btrfs_free_path(path);
Liu Boa52f4cd2013-05-01 16:23:41 +00004521 unlock_extent_cached(&BTRFS_I(inode)->io_tree, start, start + len - 1,
Josef Bacik2ac55d42010-02-03 19:33:23 +00004522 &cached_state, GFP_NOFS);
Yehuda Sadeh1506fcc2009-01-21 14:39:14 -05004523 return ret;
4524}
4525
Chris Mason727011e2010-08-06 13:21:20 -04004526static void __free_extent_buffer(struct extent_buffer *eb)
4527{
Eric Sandeen6d49ba12013-04-22 16:12:31 +00004528 btrfs_leak_debug_del(&eb->leak_list);
Chris Mason727011e2010-08-06 13:21:20 -04004529 kmem_cache_free(extent_buffer_cache, eb);
4530}
4531
Josef Bacika26e8c92014-03-28 17:07:27 -04004532int extent_buffer_under_io(struct extent_buffer *eb)
Chris Masond1310b22008-01-24 16:13:08 -05004533{
Josef Bacik0b32f4b2012-03-13 09:38:00 -04004534 return (atomic_read(&eb->io_pages) ||
4535 test_bit(EXTENT_BUFFER_WRITEBACK, &eb->bflags) ||
4536 test_bit(EXTENT_BUFFER_DIRTY, &eb->bflags));
Chris Masond1310b22008-01-24 16:13:08 -05004537}
4538
Miao Xie897ca6e2010-10-26 20:57:29 -04004539/*
4540 * Helper for releasing extent buffer page.
4541 */
David Sterbaa50924e2014-07-31 00:51:36 +02004542static void btrfs_release_extent_buffer_page(struct extent_buffer *eb)
Miao Xie897ca6e2010-10-26 20:57:29 -04004543{
4544 unsigned long index;
4545 struct page *page;
Jan Schmidt815a51c2012-05-16 17:00:02 +02004546 int mapped = !test_bit(EXTENT_BUFFER_DUMMY, &eb->bflags);
Miao Xie897ca6e2010-10-26 20:57:29 -04004547
Josef Bacik0b32f4b2012-03-13 09:38:00 -04004548 BUG_ON(extent_buffer_under_io(eb));
Miao Xie897ca6e2010-10-26 20:57:29 -04004549
David Sterbaa50924e2014-07-31 00:51:36 +02004550 index = num_extent_pages(eb->start, eb->len);
4551 if (index == 0)
Miao Xie897ca6e2010-10-26 20:57:29 -04004552 return;
4553
4554 do {
4555 index--;
David Sterbafb85fc92014-07-31 01:03:53 +02004556 page = eb->pages[index];
Jan Schmidt815a51c2012-05-16 17:00:02 +02004557 if (page && mapped) {
Josef Bacik4f2de97a2012-03-07 16:20:05 -05004558 spin_lock(&page->mapping->private_lock);
4559 /*
4560 * We do this since we'll remove the pages after we've
4561 * removed the eb from the radix tree, so we could race
4562 * and have this page now attached to the new eb. So
4563 * only clear page_private if it's still connected to
4564 * this eb.
4565 */
4566 if (PagePrivate(page) &&
4567 page->private == (unsigned long)eb) {
Josef Bacik0b32f4b2012-03-13 09:38:00 -04004568 BUG_ON(test_bit(EXTENT_BUFFER_DIRTY, &eb->bflags));
Josef Bacik3083ee22012-03-09 16:01:49 -05004569 BUG_ON(PageDirty(page));
4570 BUG_ON(PageWriteback(page));
Josef Bacik4f2de97a2012-03-07 16:20:05 -05004571 /*
4572 * We need to make sure we haven't be attached
4573 * to a new eb.
4574 */
4575 ClearPagePrivate(page);
4576 set_page_private(page, 0);
4577 /* One for the page private */
4578 page_cache_release(page);
4579 }
4580 spin_unlock(&page->mapping->private_lock);
4581
Jan Schmidt815a51c2012-05-16 17:00:02 +02004582 }
4583 if (page) {
Josef Bacik4f2de97a2012-03-07 16:20:05 -05004584 /* One for when we alloced the page */
Miao Xie897ca6e2010-10-26 20:57:29 -04004585 page_cache_release(page);
Josef Bacik4f2de97a2012-03-07 16:20:05 -05004586 }
David Sterbaa50924e2014-07-31 00:51:36 +02004587 } while (index != 0);
Miao Xie897ca6e2010-10-26 20:57:29 -04004588}
4589
4590/*
4591 * Helper for releasing the extent buffer.
4592 */
4593static inline void btrfs_release_extent_buffer(struct extent_buffer *eb)
4594{
David Sterbaa50924e2014-07-31 00:51:36 +02004595 btrfs_release_extent_buffer_page(eb);
Miao Xie897ca6e2010-10-26 20:57:29 -04004596 __free_extent_buffer(eb);
4597}
4598
Josef Bacikf28491e2013-12-16 13:24:27 -05004599static struct extent_buffer *
4600__alloc_extent_buffer(struct btrfs_fs_info *fs_info, u64 start,
David Sterba23d79d82014-06-15 02:55:29 +02004601 unsigned long len)
Josef Bacikdb7f3432013-08-07 14:54:37 -04004602{
4603 struct extent_buffer *eb = NULL;
4604
David Sterba23d79d82014-06-15 02:55:29 +02004605 eb = kmem_cache_zalloc(extent_buffer_cache, GFP_NOFS);
Josef Bacikdb7f3432013-08-07 14:54:37 -04004606 if (eb == NULL)
4607 return NULL;
4608 eb->start = start;
4609 eb->len = len;
Josef Bacikf28491e2013-12-16 13:24:27 -05004610 eb->fs_info = fs_info;
Josef Bacikdb7f3432013-08-07 14:54:37 -04004611 eb->bflags = 0;
4612 rwlock_init(&eb->lock);
4613 atomic_set(&eb->write_locks, 0);
4614 atomic_set(&eb->read_locks, 0);
4615 atomic_set(&eb->blocking_readers, 0);
4616 atomic_set(&eb->blocking_writers, 0);
4617 atomic_set(&eb->spinning_readers, 0);
4618 atomic_set(&eb->spinning_writers, 0);
4619 eb->lock_nested = 0;
4620 init_waitqueue_head(&eb->write_lock_wq);
4621 init_waitqueue_head(&eb->read_lock_wq);
4622
4623 btrfs_leak_debug_add(&eb->leak_list, &buffers);
4624
4625 spin_lock_init(&eb->refs_lock);
4626 atomic_set(&eb->refs, 1);
4627 atomic_set(&eb->io_pages, 0);
4628
4629 /*
4630 * Sanity checks, currently the maximum is 64k covered by 16x 4k pages
4631 */
4632 BUILD_BUG_ON(BTRFS_MAX_METADATA_BLOCKSIZE
4633 > MAX_INLINE_EXTENT_BUFFER_SIZE);
4634 BUG_ON(len > MAX_INLINE_EXTENT_BUFFER_SIZE);
4635
4636 return eb;
4637}
4638
4639struct extent_buffer *btrfs_clone_extent_buffer(struct extent_buffer *src)
4640{
4641 unsigned long i;
4642 struct page *p;
4643 struct extent_buffer *new;
4644 unsigned long num_pages = num_extent_pages(src->start, src->len);
4645
David Sterba23d79d82014-06-15 02:55:29 +02004646 new = __alloc_extent_buffer(NULL, src->start, src->len);
Josef Bacikdb7f3432013-08-07 14:54:37 -04004647 if (new == NULL)
4648 return NULL;
4649
4650 for (i = 0; i < num_pages; i++) {
Josef Bacik9ec72672013-08-07 16:57:23 -04004651 p = alloc_page(GFP_NOFS);
Josef Bacikdb7f3432013-08-07 14:54:37 -04004652 if (!p) {
4653 btrfs_release_extent_buffer(new);
4654 return NULL;
4655 }
4656 attach_extent_buffer_page(new, p);
4657 WARN_ON(PageDirty(p));
4658 SetPageUptodate(p);
4659 new->pages[i] = p;
4660 }
4661
4662 copy_extent_buffer(new, src, 0, 0, src->len);
4663 set_bit(EXTENT_BUFFER_UPTODATE, &new->bflags);
4664 set_bit(EXTENT_BUFFER_DUMMY, &new->bflags);
4665
4666 return new;
4667}
4668
4669struct extent_buffer *alloc_dummy_extent_buffer(u64 start, unsigned long len)
4670{
4671 struct extent_buffer *eb;
4672 unsigned long num_pages = num_extent_pages(0, len);
4673 unsigned long i;
4674
David Sterba23d79d82014-06-15 02:55:29 +02004675 eb = __alloc_extent_buffer(NULL, start, len);
Josef Bacikdb7f3432013-08-07 14:54:37 -04004676 if (!eb)
4677 return NULL;
4678
4679 for (i = 0; i < num_pages; i++) {
Josef Bacik9ec72672013-08-07 16:57:23 -04004680 eb->pages[i] = alloc_page(GFP_NOFS);
Josef Bacikdb7f3432013-08-07 14:54:37 -04004681 if (!eb->pages[i])
4682 goto err;
4683 }
4684 set_extent_buffer_uptodate(eb);
4685 btrfs_set_header_nritems(eb, 0);
4686 set_bit(EXTENT_BUFFER_DUMMY, &eb->bflags);
4687
4688 return eb;
4689err:
4690 for (; i > 0; i--)
4691 __free_page(eb->pages[i - 1]);
4692 __free_extent_buffer(eb);
4693 return NULL;
4694}
4695
Josef Bacik0b32f4b2012-03-13 09:38:00 -04004696static void check_buffer_tree_ref(struct extent_buffer *eb)
4697{
Chris Mason242e18c2013-01-29 17:49:37 -05004698 int refs;
Josef Bacik0b32f4b2012-03-13 09:38:00 -04004699 /* the ref bit is tricky. We have to make sure it is set
4700 * if we have the buffer dirty. Otherwise the
4701 * code to free a buffer can end up dropping a dirty
4702 * page
4703 *
4704 * Once the ref bit is set, it won't go away while the
4705 * buffer is dirty or in writeback, and it also won't
4706 * go away while we have the reference count on the
4707 * eb bumped.
4708 *
4709 * We can't just set the ref bit without bumping the
4710 * ref on the eb because free_extent_buffer might
4711 * see the ref bit and try to clear it. If this happens
4712 * free_extent_buffer might end up dropping our original
4713 * ref by mistake and freeing the page before we are able
4714 * to add one more ref.
4715 *
4716 * So bump the ref count first, then set the bit. If someone
4717 * beat us to it, drop the ref we added.
4718 */
Chris Mason242e18c2013-01-29 17:49:37 -05004719 refs = atomic_read(&eb->refs);
4720 if (refs >= 2 && test_bit(EXTENT_BUFFER_TREE_REF, &eb->bflags))
4721 return;
4722
Josef Bacik594831c2012-07-20 16:11:08 -04004723 spin_lock(&eb->refs_lock);
4724 if (!test_and_set_bit(EXTENT_BUFFER_TREE_REF, &eb->bflags))
Josef Bacik0b32f4b2012-03-13 09:38:00 -04004725 atomic_inc(&eb->refs);
Josef Bacik594831c2012-07-20 16:11:08 -04004726 spin_unlock(&eb->refs_lock);
Josef Bacik0b32f4b2012-03-13 09:38:00 -04004727}
4728
Mel Gorman2457aec2014-06-04 16:10:31 -07004729static void mark_extent_buffer_accessed(struct extent_buffer *eb,
4730 struct page *accessed)
Josef Bacik5df42352012-03-15 18:24:42 -04004731{
4732 unsigned long num_pages, i;
4733
Josef Bacik0b32f4b2012-03-13 09:38:00 -04004734 check_buffer_tree_ref(eb);
4735
Josef Bacik5df42352012-03-15 18:24:42 -04004736 num_pages = num_extent_pages(eb->start, eb->len);
4737 for (i = 0; i < num_pages; i++) {
David Sterbafb85fc92014-07-31 01:03:53 +02004738 struct page *p = eb->pages[i];
4739
Mel Gorman2457aec2014-06-04 16:10:31 -07004740 if (p != accessed)
4741 mark_page_accessed(p);
Josef Bacik5df42352012-03-15 18:24:42 -04004742 }
4743}
4744
Josef Bacikf28491e2013-12-16 13:24:27 -05004745struct extent_buffer *find_extent_buffer(struct btrfs_fs_info *fs_info,
4746 u64 start)
Chandra Seetharaman452c75c2013-10-07 10:45:25 -05004747{
4748 struct extent_buffer *eb;
4749
4750 rcu_read_lock();
Josef Bacikf28491e2013-12-16 13:24:27 -05004751 eb = radix_tree_lookup(&fs_info->buffer_radix,
4752 start >> PAGE_CACHE_SHIFT);
Chandra Seetharaman452c75c2013-10-07 10:45:25 -05004753 if (eb && atomic_inc_not_zero(&eb->refs)) {
4754 rcu_read_unlock();
Mel Gorman2457aec2014-06-04 16:10:31 -07004755 mark_extent_buffer_accessed(eb, NULL);
Chandra Seetharaman452c75c2013-10-07 10:45:25 -05004756 return eb;
4757 }
4758 rcu_read_unlock();
4759
4760 return NULL;
4761}
4762
Josef Bacikfaa2dbf2014-05-07 17:06:09 -04004763#ifdef CONFIG_BTRFS_FS_RUN_SANITY_TESTS
4764struct extent_buffer *alloc_test_extent_buffer(struct btrfs_fs_info *fs_info,
4765 u64 start, unsigned long len)
4766{
4767 struct extent_buffer *eb, *exists = NULL;
4768 int ret;
4769
4770 eb = find_extent_buffer(fs_info, start);
4771 if (eb)
4772 return eb;
4773 eb = alloc_dummy_extent_buffer(start, len);
4774 if (!eb)
4775 return NULL;
4776 eb->fs_info = fs_info;
4777again:
4778 ret = radix_tree_preload(GFP_NOFS & ~__GFP_HIGHMEM);
4779 if (ret)
4780 goto free_eb;
4781 spin_lock(&fs_info->buffer_lock);
4782 ret = radix_tree_insert(&fs_info->buffer_radix,
4783 start >> PAGE_CACHE_SHIFT, eb);
4784 spin_unlock(&fs_info->buffer_lock);
4785 radix_tree_preload_end();
4786 if (ret == -EEXIST) {
4787 exists = find_extent_buffer(fs_info, start);
4788 if (exists)
4789 goto free_eb;
4790 else
4791 goto again;
4792 }
4793 check_buffer_tree_ref(eb);
4794 set_bit(EXTENT_BUFFER_IN_TREE, &eb->bflags);
4795
4796 /*
4797 * We will free dummy extent buffer's if they come into
4798 * free_extent_buffer with a ref count of 2, but if we are using this we
4799 * want the buffers to stay in memory until we're done with them, so
4800 * bump the ref count again.
4801 */
4802 atomic_inc(&eb->refs);
4803 return eb;
4804free_eb:
4805 btrfs_release_extent_buffer(eb);
4806 return exists;
4807}
4808#endif
4809
Josef Bacikf28491e2013-12-16 13:24:27 -05004810struct extent_buffer *alloc_extent_buffer(struct btrfs_fs_info *fs_info,
Chris Mason727011e2010-08-06 13:21:20 -04004811 u64 start, unsigned long len)
Chris Masond1310b22008-01-24 16:13:08 -05004812{
4813 unsigned long num_pages = num_extent_pages(start, len);
4814 unsigned long i;
4815 unsigned long index = start >> PAGE_CACHE_SHIFT;
4816 struct extent_buffer *eb;
Chris Mason6af118ce2008-07-22 11:18:07 -04004817 struct extent_buffer *exists = NULL;
Chris Masond1310b22008-01-24 16:13:08 -05004818 struct page *p;
Josef Bacikf28491e2013-12-16 13:24:27 -05004819 struct address_space *mapping = fs_info->btree_inode->i_mapping;
Chris Masond1310b22008-01-24 16:13:08 -05004820 int uptodate = 1;
Miao Xie19fe0a82010-10-26 20:57:29 -04004821 int ret;
Chris Masond1310b22008-01-24 16:13:08 -05004822
Josef Bacikf28491e2013-12-16 13:24:27 -05004823 eb = find_extent_buffer(fs_info, start);
Chandra Seetharaman452c75c2013-10-07 10:45:25 -05004824 if (eb)
Chris Mason6af118ce2008-07-22 11:18:07 -04004825 return eb;
Chris Mason6af118ce2008-07-22 11:18:07 -04004826
David Sterba23d79d82014-06-15 02:55:29 +02004827 eb = __alloc_extent_buffer(fs_info, start, len);
Peter2b114d12008-04-01 11:21:40 -04004828 if (!eb)
Chris Masond1310b22008-01-24 16:13:08 -05004829 return NULL;
4830
Chris Mason727011e2010-08-06 13:21:20 -04004831 for (i = 0; i < num_pages; i++, index++) {
Chris Masona6591712011-07-19 12:04:14 -04004832 p = find_or_create_page(mapping, index, GFP_NOFS);
Josef Bacik4804b382012-10-05 16:43:45 -04004833 if (!p)
Chris Mason6af118ce2008-07-22 11:18:07 -04004834 goto free_eb;
Josef Bacik4f2de97a2012-03-07 16:20:05 -05004835
4836 spin_lock(&mapping->private_lock);
4837 if (PagePrivate(p)) {
4838 /*
4839 * We could have already allocated an eb for this page
4840 * and attached one so lets see if we can get a ref on
4841 * the existing eb, and if we can we know it's good and
4842 * we can just return that one, else we know we can just
4843 * overwrite page->private.
4844 */
4845 exists = (struct extent_buffer *)p->private;
4846 if (atomic_inc_not_zero(&exists->refs)) {
4847 spin_unlock(&mapping->private_lock);
4848 unlock_page(p);
Josef Bacik17de39a2012-05-04 15:16:06 -04004849 page_cache_release(p);
Mel Gorman2457aec2014-06-04 16:10:31 -07004850 mark_extent_buffer_accessed(exists, p);
Josef Bacik4f2de97a2012-03-07 16:20:05 -05004851 goto free_eb;
4852 }
4853
Josef Bacik0b32f4b2012-03-13 09:38:00 -04004854 /*
Josef Bacik4f2de97a2012-03-07 16:20:05 -05004855 * Do this so attach doesn't complain and we need to
4856 * drop the ref the old guy had.
4857 */
4858 ClearPagePrivate(p);
Josef Bacik0b32f4b2012-03-13 09:38:00 -04004859 WARN_ON(PageDirty(p));
Josef Bacik4f2de97a2012-03-07 16:20:05 -05004860 page_cache_release(p);
Chris Masond1310b22008-01-24 16:13:08 -05004861 }
Josef Bacik4f2de97a2012-03-07 16:20:05 -05004862 attach_extent_buffer_page(eb, p);
4863 spin_unlock(&mapping->private_lock);
Josef Bacik0b32f4b2012-03-13 09:38:00 -04004864 WARN_ON(PageDirty(p));
Chris Mason727011e2010-08-06 13:21:20 -04004865 eb->pages[i] = p;
Chris Masond1310b22008-01-24 16:13:08 -05004866 if (!PageUptodate(p))
4867 uptodate = 0;
Chris Masoneb14ab82011-02-10 12:35:00 -05004868
4869 /*
4870 * see below about how we avoid a nasty race with release page
4871 * and why we unlock later
4872 */
Chris Masond1310b22008-01-24 16:13:08 -05004873 }
4874 if (uptodate)
Chris Masonb4ce94d2009-02-04 09:25:08 -05004875 set_bit(EXTENT_BUFFER_UPTODATE, &eb->bflags);
Josef Bacik115391d2012-03-09 09:51:43 -05004876again:
Miao Xie19fe0a82010-10-26 20:57:29 -04004877 ret = radix_tree_preload(GFP_NOFS & ~__GFP_HIGHMEM);
4878 if (ret)
4879 goto free_eb;
4880
Josef Bacikf28491e2013-12-16 13:24:27 -05004881 spin_lock(&fs_info->buffer_lock);
4882 ret = radix_tree_insert(&fs_info->buffer_radix,
4883 start >> PAGE_CACHE_SHIFT, eb);
4884 spin_unlock(&fs_info->buffer_lock);
Chandra Seetharaman452c75c2013-10-07 10:45:25 -05004885 radix_tree_preload_end();
Miao Xie19fe0a82010-10-26 20:57:29 -04004886 if (ret == -EEXIST) {
Josef Bacikf28491e2013-12-16 13:24:27 -05004887 exists = find_extent_buffer(fs_info, start);
Chandra Seetharaman452c75c2013-10-07 10:45:25 -05004888 if (exists)
4889 goto free_eb;
4890 else
Josef Bacik115391d2012-03-09 09:51:43 -05004891 goto again;
Chris Mason6af118ce2008-07-22 11:18:07 -04004892 }
Chris Mason6af118ce2008-07-22 11:18:07 -04004893 /* add one reference for the tree */
Josef Bacik0b32f4b2012-03-13 09:38:00 -04004894 check_buffer_tree_ref(eb);
Josef Bacik34b41ac2013-12-13 10:41:51 -05004895 set_bit(EXTENT_BUFFER_IN_TREE, &eb->bflags);
Chris Masoneb14ab82011-02-10 12:35:00 -05004896
4897 /*
4898 * there is a race where release page may have
4899 * tried to find this extent buffer in the radix
4900 * but failed. It will tell the VM it is safe to
4901 * reclaim the, and it will clear the page private bit.
4902 * We must make sure to set the page private bit properly
4903 * after the extent buffer is in the radix tree so
4904 * it doesn't get lost
4905 */
Chris Mason727011e2010-08-06 13:21:20 -04004906 SetPageChecked(eb->pages[0]);
4907 for (i = 1; i < num_pages; i++) {
David Sterbafb85fc92014-07-31 01:03:53 +02004908 p = eb->pages[i];
Chris Mason727011e2010-08-06 13:21:20 -04004909 ClearPageChecked(p);
4910 unlock_page(p);
4911 }
4912 unlock_page(eb->pages[0]);
Chris Masond1310b22008-01-24 16:13:08 -05004913 return eb;
4914
Chris Mason6af118ce2008-07-22 11:18:07 -04004915free_eb:
Chris Mason727011e2010-08-06 13:21:20 -04004916 for (i = 0; i < num_pages; i++) {
4917 if (eb->pages[i])
4918 unlock_page(eb->pages[i]);
4919 }
Chris Masoneb14ab82011-02-10 12:35:00 -05004920
Josef Bacik17de39a2012-05-04 15:16:06 -04004921 WARN_ON(!atomic_dec_and_test(&eb->refs));
Miao Xie897ca6e2010-10-26 20:57:29 -04004922 btrfs_release_extent_buffer(eb);
Chris Mason6af118ce2008-07-22 11:18:07 -04004923 return exists;
Chris Masond1310b22008-01-24 16:13:08 -05004924}
Chris Masond1310b22008-01-24 16:13:08 -05004925
Josef Bacik3083ee22012-03-09 16:01:49 -05004926static inline void btrfs_release_extent_buffer_rcu(struct rcu_head *head)
4927{
4928 struct extent_buffer *eb =
4929 container_of(head, struct extent_buffer, rcu_head);
4930
4931 __free_extent_buffer(eb);
4932}
4933
Josef Bacik3083ee22012-03-09 16:01:49 -05004934/* Expects to have eb->eb_lock already held */
David Sterbaf7a52a42013-04-26 14:56:29 +00004935static int release_extent_buffer(struct extent_buffer *eb)
Josef Bacik3083ee22012-03-09 16:01:49 -05004936{
4937 WARN_ON(atomic_read(&eb->refs) == 0);
4938 if (atomic_dec_and_test(&eb->refs)) {
Josef Bacik34b41ac2013-12-13 10:41:51 -05004939 if (test_and_clear_bit(EXTENT_BUFFER_IN_TREE, &eb->bflags)) {
Josef Bacikf28491e2013-12-16 13:24:27 -05004940 struct btrfs_fs_info *fs_info = eb->fs_info;
Josef Bacik3083ee22012-03-09 16:01:49 -05004941
Jan Schmidt815a51c2012-05-16 17:00:02 +02004942 spin_unlock(&eb->refs_lock);
Josef Bacik3083ee22012-03-09 16:01:49 -05004943
Josef Bacikf28491e2013-12-16 13:24:27 -05004944 spin_lock(&fs_info->buffer_lock);
4945 radix_tree_delete(&fs_info->buffer_radix,
Jan Schmidt815a51c2012-05-16 17:00:02 +02004946 eb->start >> PAGE_CACHE_SHIFT);
Josef Bacikf28491e2013-12-16 13:24:27 -05004947 spin_unlock(&fs_info->buffer_lock);
Josef Bacik34b41ac2013-12-13 10:41:51 -05004948 } else {
4949 spin_unlock(&eb->refs_lock);
Jan Schmidt815a51c2012-05-16 17:00:02 +02004950 }
Josef Bacik3083ee22012-03-09 16:01:49 -05004951
4952 /* Should be safe to release our pages at this point */
David Sterbaa50924e2014-07-31 00:51:36 +02004953 btrfs_release_extent_buffer_page(eb);
Josef Bacik3083ee22012-03-09 16:01:49 -05004954 call_rcu(&eb->rcu_head, btrfs_release_extent_buffer_rcu);
Josef Bacike64860a2012-07-20 16:05:36 -04004955 return 1;
Josef Bacik3083ee22012-03-09 16:01:49 -05004956 }
4957 spin_unlock(&eb->refs_lock);
Josef Bacike64860a2012-07-20 16:05:36 -04004958
4959 return 0;
Josef Bacik3083ee22012-03-09 16:01:49 -05004960}
4961
Chris Masond1310b22008-01-24 16:13:08 -05004962void free_extent_buffer(struct extent_buffer *eb)
4963{
Chris Mason242e18c2013-01-29 17:49:37 -05004964 int refs;
4965 int old;
Chris Masond1310b22008-01-24 16:13:08 -05004966 if (!eb)
4967 return;
4968
Chris Mason242e18c2013-01-29 17:49:37 -05004969 while (1) {
4970 refs = atomic_read(&eb->refs);
4971 if (refs <= 3)
4972 break;
4973 old = atomic_cmpxchg(&eb->refs, refs, refs - 1);
4974 if (old == refs)
4975 return;
4976 }
4977
Josef Bacik3083ee22012-03-09 16:01:49 -05004978 spin_lock(&eb->refs_lock);
4979 if (atomic_read(&eb->refs) == 2 &&
Jan Schmidt815a51c2012-05-16 17:00:02 +02004980 test_bit(EXTENT_BUFFER_DUMMY, &eb->bflags))
4981 atomic_dec(&eb->refs);
4982
4983 if (atomic_read(&eb->refs) == 2 &&
Josef Bacik3083ee22012-03-09 16:01:49 -05004984 test_bit(EXTENT_BUFFER_STALE, &eb->bflags) &&
Josef Bacik0b32f4b2012-03-13 09:38:00 -04004985 !extent_buffer_under_io(eb) &&
Josef Bacik3083ee22012-03-09 16:01:49 -05004986 test_and_clear_bit(EXTENT_BUFFER_TREE_REF, &eb->bflags))
4987 atomic_dec(&eb->refs);
Chris Masond1310b22008-01-24 16:13:08 -05004988
Josef Bacik3083ee22012-03-09 16:01:49 -05004989 /*
4990 * I know this is terrible, but it's temporary until we stop tracking
4991 * the uptodate bits and such for the extent buffers.
4992 */
David Sterbaf7a52a42013-04-26 14:56:29 +00004993 release_extent_buffer(eb);
Chris Masond1310b22008-01-24 16:13:08 -05004994}
Chris Masond1310b22008-01-24 16:13:08 -05004995
Josef Bacik3083ee22012-03-09 16:01:49 -05004996void free_extent_buffer_stale(struct extent_buffer *eb)
4997{
4998 if (!eb)
Chris Masond1310b22008-01-24 16:13:08 -05004999 return;
5000
Josef Bacik3083ee22012-03-09 16:01:49 -05005001 spin_lock(&eb->refs_lock);
5002 set_bit(EXTENT_BUFFER_STALE, &eb->bflags);
5003
Josef Bacik0b32f4b2012-03-13 09:38:00 -04005004 if (atomic_read(&eb->refs) == 2 && !extent_buffer_under_io(eb) &&
Josef Bacik3083ee22012-03-09 16:01:49 -05005005 test_and_clear_bit(EXTENT_BUFFER_TREE_REF, &eb->bflags))
5006 atomic_dec(&eb->refs);
David Sterbaf7a52a42013-04-26 14:56:29 +00005007 release_extent_buffer(eb);
Chris Masond1310b22008-01-24 16:13:08 -05005008}
5009
Chris Mason1d4284b2012-03-28 20:31:37 -04005010void clear_extent_buffer_dirty(struct extent_buffer *eb)
Chris Masond1310b22008-01-24 16:13:08 -05005011{
Chris Masond1310b22008-01-24 16:13:08 -05005012 unsigned long i;
5013 unsigned long num_pages;
5014 struct page *page;
5015
Chris Masond1310b22008-01-24 16:13:08 -05005016 num_pages = num_extent_pages(eb->start, eb->len);
5017
5018 for (i = 0; i < num_pages; i++) {
David Sterbafb85fc92014-07-31 01:03:53 +02005019 page = eb->pages[i];
Chris Masonb9473432009-03-13 11:00:37 -04005020 if (!PageDirty(page))
Chris Masond2c3f4f2008-11-19 12:44:22 -05005021 continue;
5022
Chris Masona61e6f22008-07-22 11:18:08 -04005023 lock_page(page);
Chris Masoneb14ab82011-02-10 12:35:00 -05005024 WARN_ON(!PagePrivate(page));
5025
Chris Masond1310b22008-01-24 16:13:08 -05005026 clear_page_dirty_for_io(page);
Sven Wegener0ee0fda2008-07-30 16:54:26 -04005027 spin_lock_irq(&page->mapping->tree_lock);
Chris Masond1310b22008-01-24 16:13:08 -05005028 if (!PageDirty(page)) {
5029 radix_tree_tag_clear(&page->mapping->page_tree,
5030 page_index(page),
5031 PAGECACHE_TAG_DIRTY);
5032 }
Sven Wegener0ee0fda2008-07-30 16:54:26 -04005033 spin_unlock_irq(&page->mapping->tree_lock);
Chris Masonbf0da8c2011-11-04 12:29:37 -04005034 ClearPageError(page);
Chris Masona61e6f22008-07-22 11:18:08 -04005035 unlock_page(page);
Chris Masond1310b22008-01-24 16:13:08 -05005036 }
Josef Bacik0b32f4b2012-03-13 09:38:00 -04005037 WARN_ON(atomic_read(&eb->refs) == 0);
Chris Masond1310b22008-01-24 16:13:08 -05005038}
Chris Masond1310b22008-01-24 16:13:08 -05005039
Josef Bacik0b32f4b2012-03-13 09:38:00 -04005040int set_extent_buffer_dirty(struct extent_buffer *eb)
Chris Masond1310b22008-01-24 16:13:08 -05005041{
5042 unsigned long i;
5043 unsigned long num_pages;
Chris Masonb9473432009-03-13 11:00:37 -04005044 int was_dirty = 0;
Chris Masond1310b22008-01-24 16:13:08 -05005045
Josef Bacik0b32f4b2012-03-13 09:38:00 -04005046 check_buffer_tree_ref(eb);
5047
Chris Masonb9473432009-03-13 11:00:37 -04005048 was_dirty = test_and_set_bit(EXTENT_BUFFER_DIRTY, &eb->bflags);
Josef Bacik0b32f4b2012-03-13 09:38:00 -04005049
Chris Masond1310b22008-01-24 16:13:08 -05005050 num_pages = num_extent_pages(eb->start, eb->len);
Josef Bacik3083ee22012-03-09 16:01:49 -05005051 WARN_ON(atomic_read(&eb->refs) == 0);
Josef Bacik0b32f4b2012-03-13 09:38:00 -04005052 WARN_ON(!test_bit(EXTENT_BUFFER_TREE_REF, &eb->bflags));
5053
Chris Masonb9473432009-03-13 11:00:37 -04005054 for (i = 0; i < num_pages; i++)
David Sterbafb85fc92014-07-31 01:03:53 +02005055 set_page_dirty(eb->pages[i]);
Chris Masonb9473432009-03-13 11:00:37 -04005056 return was_dirty;
Chris Masond1310b22008-01-24 16:13:08 -05005057}
Chris Masond1310b22008-01-24 16:13:08 -05005058
Josef Bacik0b32f4b2012-03-13 09:38:00 -04005059int clear_extent_buffer_uptodate(struct extent_buffer *eb)
Chris Mason1259ab72008-05-12 13:39:03 -04005060{
5061 unsigned long i;
5062 struct page *page;
5063 unsigned long num_pages;
5064
Chris Masonb4ce94d2009-02-04 09:25:08 -05005065 clear_bit(EXTENT_BUFFER_UPTODATE, &eb->bflags);
Josef Bacik0b32f4b2012-03-13 09:38:00 -04005066 num_pages = num_extent_pages(eb->start, eb->len);
Chris Mason1259ab72008-05-12 13:39:03 -04005067 for (i = 0; i < num_pages; i++) {
David Sterbafb85fc92014-07-31 01:03:53 +02005068 page = eb->pages[i];
Chris Mason33958dc2008-07-30 10:29:12 -04005069 if (page)
5070 ClearPageUptodate(page);
Chris Mason1259ab72008-05-12 13:39:03 -04005071 }
5072 return 0;
5073}
5074
Josef Bacik0b32f4b2012-03-13 09:38:00 -04005075int set_extent_buffer_uptodate(struct extent_buffer *eb)
Chris Masond1310b22008-01-24 16:13:08 -05005076{
5077 unsigned long i;
5078 struct page *page;
5079 unsigned long num_pages;
5080
Josef Bacik0b32f4b2012-03-13 09:38:00 -04005081 set_bit(EXTENT_BUFFER_UPTODATE, &eb->bflags);
Chris Masond1310b22008-01-24 16:13:08 -05005082 num_pages = num_extent_pages(eb->start, eb->len);
Chris Masond1310b22008-01-24 16:13:08 -05005083 for (i = 0; i < num_pages; i++) {
David Sterbafb85fc92014-07-31 01:03:53 +02005084 page = eb->pages[i];
Chris Masond1310b22008-01-24 16:13:08 -05005085 SetPageUptodate(page);
5086 }
5087 return 0;
5088}
Chris Masond1310b22008-01-24 16:13:08 -05005089
Josef Bacik0b32f4b2012-03-13 09:38:00 -04005090int extent_buffer_uptodate(struct extent_buffer *eb)
Chris Masond1310b22008-01-24 16:13:08 -05005091{
Josef Bacik0b32f4b2012-03-13 09:38:00 -04005092 return test_bit(EXTENT_BUFFER_UPTODATE, &eb->bflags);
Chris Masond1310b22008-01-24 16:13:08 -05005093}
Chris Masond1310b22008-01-24 16:13:08 -05005094
5095int read_extent_buffer_pages(struct extent_io_tree *tree,
Arne Jansenbb82ab82011-06-10 14:06:53 +02005096 struct extent_buffer *eb, u64 start, int wait,
Chris Masonf1885912008-04-09 16:28:12 -04005097 get_extent_t *get_extent, int mirror_num)
Chris Masond1310b22008-01-24 16:13:08 -05005098{
5099 unsigned long i;
5100 unsigned long start_i;
5101 struct page *page;
5102 int err;
5103 int ret = 0;
Chris Masonce9adaa2008-04-09 16:28:12 -04005104 int locked_pages = 0;
5105 int all_uptodate = 1;
Chris Masond1310b22008-01-24 16:13:08 -05005106 unsigned long num_pages;
Chris Mason727011e2010-08-06 13:21:20 -04005107 unsigned long num_reads = 0;
Chris Masona86c12c2008-02-07 10:50:54 -05005108 struct bio *bio = NULL;
Chris Masonc8b97812008-10-29 14:49:59 -04005109 unsigned long bio_flags = 0;
Chris Masona86c12c2008-02-07 10:50:54 -05005110
Chris Masonb4ce94d2009-02-04 09:25:08 -05005111 if (test_bit(EXTENT_BUFFER_UPTODATE, &eb->bflags))
Chris Masond1310b22008-01-24 16:13:08 -05005112 return 0;
5113
Chris Masond1310b22008-01-24 16:13:08 -05005114 if (start) {
5115 WARN_ON(start < eb->start);
5116 start_i = (start >> PAGE_CACHE_SHIFT) -
5117 (eb->start >> PAGE_CACHE_SHIFT);
5118 } else {
5119 start_i = 0;
5120 }
5121
5122 num_pages = num_extent_pages(eb->start, eb->len);
5123 for (i = start_i; i < num_pages; i++) {
David Sterbafb85fc92014-07-31 01:03:53 +02005124 page = eb->pages[i];
Arne Jansenbb82ab82011-06-10 14:06:53 +02005125 if (wait == WAIT_NONE) {
David Woodhouse2db04962008-08-07 11:19:43 -04005126 if (!trylock_page(page))
Chris Masonce9adaa2008-04-09 16:28:12 -04005127 goto unlock_exit;
Chris Masond1310b22008-01-24 16:13:08 -05005128 } else {
5129 lock_page(page);
5130 }
Chris Masonce9adaa2008-04-09 16:28:12 -04005131 locked_pages++;
Chris Mason727011e2010-08-06 13:21:20 -04005132 if (!PageUptodate(page)) {
5133 num_reads++;
Chris Masonce9adaa2008-04-09 16:28:12 -04005134 all_uptodate = 0;
Chris Mason727011e2010-08-06 13:21:20 -04005135 }
Chris Masonce9adaa2008-04-09 16:28:12 -04005136 }
5137 if (all_uptodate) {
5138 if (start_i == 0)
Chris Masonb4ce94d2009-02-04 09:25:08 -05005139 set_bit(EXTENT_BUFFER_UPTODATE, &eb->bflags);
Chris Masonce9adaa2008-04-09 16:28:12 -04005140 goto unlock_exit;
5141 }
5142
Filipe Manana656f30d2014-09-26 12:25:56 +01005143 clear_bit(EXTENT_BUFFER_READ_ERR, &eb->bflags);
Josef Bacik5cf1ab52012-04-16 09:42:26 -04005144 eb->read_mirror = 0;
Josef Bacik0b32f4b2012-03-13 09:38:00 -04005145 atomic_set(&eb->io_pages, num_reads);
Chris Masonce9adaa2008-04-09 16:28:12 -04005146 for (i = start_i; i < num_pages; i++) {
David Sterbafb85fc92014-07-31 01:03:53 +02005147 page = eb->pages[i];
Chris Masonce9adaa2008-04-09 16:28:12 -04005148 if (!PageUptodate(page)) {
Chris Masonf1885912008-04-09 16:28:12 -04005149 ClearPageError(page);
Chris Masona86c12c2008-02-07 10:50:54 -05005150 err = __extent_read_full_page(tree, page,
Chris Masonf1885912008-04-09 16:28:12 -04005151 get_extent, &bio,
Josef Bacikd4c7ca82013-04-19 19:49:09 -04005152 mirror_num, &bio_flags,
5153 READ | REQ_META);
Chris Masond3977122009-01-05 21:25:51 -05005154 if (err)
Chris Masond1310b22008-01-24 16:13:08 -05005155 ret = err;
Chris Masond1310b22008-01-24 16:13:08 -05005156 } else {
5157 unlock_page(page);
5158 }
5159 }
5160
Jeff Mahoney355808c2011-10-03 23:23:14 -04005161 if (bio) {
Josef Bacikd4c7ca82013-04-19 19:49:09 -04005162 err = submit_one_bio(READ | REQ_META, bio, mirror_num,
5163 bio_flags);
Jeff Mahoney79787ea2012-03-12 16:03:00 +01005164 if (err)
5165 return err;
Jeff Mahoney355808c2011-10-03 23:23:14 -04005166 }
Chris Masona86c12c2008-02-07 10:50:54 -05005167
Arne Jansenbb82ab82011-06-10 14:06:53 +02005168 if (ret || wait != WAIT_COMPLETE)
Chris Masond1310b22008-01-24 16:13:08 -05005169 return ret;
Chris Masond3977122009-01-05 21:25:51 -05005170
Chris Masond1310b22008-01-24 16:13:08 -05005171 for (i = start_i; i < num_pages; i++) {
David Sterbafb85fc92014-07-31 01:03:53 +02005172 page = eb->pages[i];
Chris Masond1310b22008-01-24 16:13:08 -05005173 wait_on_page_locked(page);
Chris Masond3977122009-01-05 21:25:51 -05005174 if (!PageUptodate(page))
Chris Masond1310b22008-01-24 16:13:08 -05005175 ret = -EIO;
Chris Masond1310b22008-01-24 16:13:08 -05005176 }
Chris Masond3977122009-01-05 21:25:51 -05005177
Chris Masond1310b22008-01-24 16:13:08 -05005178 return ret;
Chris Masonce9adaa2008-04-09 16:28:12 -04005179
5180unlock_exit:
5181 i = start_i;
Chris Masond3977122009-01-05 21:25:51 -05005182 while (locked_pages > 0) {
David Sterbafb85fc92014-07-31 01:03:53 +02005183 page = eb->pages[i];
Chris Masonce9adaa2008-04-09 16:28:12 -04005184 i++;
5185 unlock_page(page);
5186 locked_pages--;
5187 }
5188 return ret;
Chris Masond1310b22008-01-24 16:13:08 -05005189}
Chris Masond1310b22008-01-24 16:13:08 -05005190
5191void read_extent_buffer(struct extent_buffer *eb, void *dstv,
5192 unsigned long start,
5193 unsigned long len)
5194{
5195 size_t cur;
5196 size_t offset;
5197 struct page *page;
5198 char *kaddr;
5199 char *dst = (char *)dstv;
5200 size_t start_offset = eb->start & ((u64)PAGE_CACHE_SIZE - 1);
5201 unsigned long i = (start_offset + start) >> PAGE_CACHE_SHIFT;
Chris Masond1310b22008-01-24 16:13:08 -05005202
5203 WARN_ON(start > eb->len);
5204 WARN_ON(start + len > eb->start + eb->len);
5205
Geert Uytterhoeven778746b2013-08-20 13:20:16 +02005206 offset = (start_offset + start) & (PAGE_CACHE_SIZE - 1);
Chris Masond1310b22008-01-24 16:13:08 -05005207
Chris Masond3977122009-01-05 21:25:51 -05005208 while (len > 0) {
David Sterbafb85fc92014-07-31 01:03:53 +02005209 page = eb->pages[i];
Chris Masond1310b22008-01-24 16:13:08 -05005210
5211 cur = min(len, (PAGE_CACHE_SIZE - offset));
Chris Masona6591712011-07-19 12:04:14 -04005212 kaddr = page_address(page);
Chris Masond1310b22008-01-24 16:13:08 -05005213 memcpy(dst, kaddr + offset, cur);
Chris Masond1310b22008-01-24 16:13:08 -05005214
5215 dst += cur;
5216 len -= cur;
5217 offset = 0;
5218 i++;
5219 }
5220}
Chris Masond1310b22008-01-24 16:13:08 -05005221
Gerhard Heift550ac1d2014-01-30 16:24:01 +01005222int read_extent_buffer_to_user(struct extent_buffer *eb, void __user *dstv,
5223 unsigned long start,
5224 unsigned long len)
5225{
5226 size_t cur;
5227 size_t offset;
5228 struct page *page;
5229 char *kaddr;
5230 char __user *dst = (char __user *)dstv;
5231 size_t start_offset = eb->start & ((u64)PAGE_CACHE_SIZE - 1);
5232 unsigned long i = (start_offset + start) >> PAGE_CACHE_SHIFT;
5233 int ret = 0;
5234
5235 WARN_ON(start > eb->len);
5236 WARN_ON(start + len > eb->start + eb->len);
5237
5238 offset = (start_offset + start) & (PAGE_CACHE_SIZE - 1);
5239
5240 while (len > 0) {
David Sterbafb85fc92014-07-31 01:03:53 +02005241 page = eb->pages[i];
Gerhard Heift550ac1d2014-01-30 16:24:01 +01005242
5243 cur = min(len, (PAGE_CACHE_SIZE - offset));
5244 kaddr = page_address(page);
5245 if (copy_to_user(dst, kaddr + offset, cur)) {
5246 ret = -EFAULT;
5247 break;
5248 }
5249
5250 dst += cur;
5251 len -= cur;
5252 offset = 0;
5253 i++;
5254 }
5255
5256 return ret;
5257}
5258
Chris Masond1310b22008-01-24 16:13:08 -05005259int map_private_extent_buffer(struct extent_buffer *eb, unsigned long start,
Chris Masona6591712011-07-19 12:04:14 -04005260 unsigned long min_len, char **map,
Chris Masond1310b22008-01-24 16:13:08 -05005261 unsigned long *map_start,
Chris Masona6591712011-07-19 12:04:14 -04005262 unsigned long *map_len)
Chris Masond1310b22008-01-24 16:13:08 -05005263{
5264 size_t offset = start & (PAGE_CACHE_SIZE - 1);
5265 char *kaddr;
5266 struct page *p;
5267 size_t start_offset = eb->start & ((u64)PAGE_CACHE_SIZE - 1);
5268 unsigned long i = (start_offset + start) >> PAGE_CACHE_SHIFT;
5269 unsigned long end_i = (start_offset + start + min_len - 1) >>
5270 PAGE_CACHE_SHIFT;
5271
5272 if (i != end_i)
5273 return -EINVAL;
5274
5275 if (i == 0) {
5276 offset = start_offset;
5277 *map_start = 0;
5278 } else {
5279 offset = 0;
5280 *map_start = ((u64)i << PAGE_CACHE_SHIFT) - start_offset;
5281 }
Chris Masond3977122009-01-05 21:25:51 -05005282
Chris Masond1310b22008-01-24 16:13:08 -05005283 if (start + min_len > eb->len) {
Julia Lawall31b1a2b2012-11-03 10:58:34 +00005284 WARN(1, KERN_ERR "btrfs bad mapping eb start %llu len %lu, "
Geert Uytterhoevenc1c9ff72013-08-20 13:20:07 +02005285 "wanted %lu %lu\n",
5286 eb->start, eb->len, start, min_len);
Josef Bacik850265332011-03-15 14:52:12 -04005287 return -EINVAL;
Chris Masond1310b22008-01-24 16:13:08 -05005288 }
5289
David Sterbafb85fc92014-07-31 01:03:53 +02005290 p = eb->pages[i];
Chris Masona6591712011-07-19 12:04:14 -04005291 kaddr = page_address(p);
Chris Masond1310b22008-01-24 16:13:08 -05005292 *map = kaddr + offset;
5293 *map_len = PAGE_CACHE_SIZE - offset;
5294 return 0;
5295}
Chris Masond1310b22008-01-24 16:13:08 -05005296
Chris Masond1310b22008-01-24 16:13:08 -05005297int memcmp_extent_buffer(struct extent_buffer *eb, const void *ptrv,
5298 unsigned long start,
5299 unsigned long len)
5300{
5301 size_t cur;
5302 size_t offset;
5303 struct page *page;
5304 char *kaddr;
5305 char *ptr = (char *)ptrv;
5306 size_t start_offset = eb->start & ((u64)PAGE_CACHE_SIZE - 1);
5307 unsigned long i = (start_offset + start) >> PAGE_CACHE_SHIFT;
5308 int ret = 0;
5309
5310 WARN_ON(start > eb->len);
5311 WARN_ON(start + len > eb->start + eb->len);
5312
Geert Uytterhoeven778746b2013-08-20 13:20:16 +02005313 offset = (start_offset + start) & (PAGE_CACHE_SIZE - 1);
Chris Masond1310b22008-01-24 16:13:08 -05005314
Chris Masond3977122009-01-05 21:25:51 -05005315 while (len > 0) {
David Sterbafb85fc92014-07-31 01:03:53 +02005316 page = eb->pages[i];
Chris Masond1310b22008-01-24 16:13:08 -05005317
5318 cur = min(len, (PAGE_CACHE_SIZE - offset));
5319
Chris Masona6591712011-07-19 12:04:14 -04005320 kaddr = page_address(page);
Chris Masond1310b22008-01-24 16:13:08 -05005321 ret = memcmp(ptr, kaddr + offset, cur);
Chris Masond1310b22008-01-24 16:13:08 -05005322 if (ret)
5323 break;
5324
5325 ptr += cur;
5326 len -= cur;
5327 offset = 0;
5328 i++;
5329 }
5330 return ret;
5331}
Chris Masond1310b22008-01-24 16:13:08 -05005332
5333void write_extent_buffer(struct extent_buffer *eb, const void *srcv,
5334 unsigned long start, unsigned long len)
5335{
5336 size_t cur;
5337 size_t offset;
5338 struct page *page;
5339 char *kaddr;
5340 char *src = (char *)srcv;
5341 size_t start_offset = eb->start & ((u64)PAGE_CACHE_SIZE - 1);
5342 unsigned long i = (start_offset + start) >> PAGE_CACHE_SHIFT;
5343
5344 WARN_ON(start > eb->len);
5345 WARN_ON(start + len > eb->start + eb->len);
5346
Geert Uytterhoeven778746b2013-08-20 13:20:16 +02005347 offset = (start_offset + start) & (PAGE_CACHE_SIZE - 1);
Chris Masond1310b22008-01-24 16:13:08 -05005348
Chris Masond3977122009-01-05 21:25:51 -05005349 while (len > 0) {
David Sterbafb85fc92014-07-31 01:03:53 +02005350 page = eb->pages[i];
Chris Masond1310b22008-01-24 16:13:08 -05005351 WARN_ON(!PageUptodate(page));
5352
5353 cur = min(len, PAGE_CACHE_SIZE - offset);
Chris Masona6591712011-07-19 12:04:14 -04005354 kaddr = page_address(page);
Chris Masond1310b22008-01-24 16:13:08 -05005355 memcpy(kaddr + offset, src, cur);
Chris Masond1310b22008-01-24 16:13:08 -05005356
5357 src += cur;
5358 len -= cur;
5359 offset = 0;
5360 i++;
5361 }
5362}
Chris Masond1310b22008-01-24 16:13:08 -05005363
5364void memset_extent_buffer(struct extent_buffer *eb, char c,
5365 unsigned long start, unsigned long len)
5366{
5367 size_t cur;
5368 size_t offset;
5369 struct page *page;
5370 char *kaddr;
5371 size_t start_offset = eb->start & ((u64)PAGE_CACHE_SIZE - 1);
5372 unsigned long i = (start_offset + start) >> PAGE_CACHE_SHIFT;
5373
5374 WARN_ON(start > eb->len);
5375 WARN_ON(start + len > eb->start + eb->len);
5376
Geert Uytterhoeven778746b2013-08-20 13:20:16 +02005377 offset = (start_offset + start) & (PAGE_CACHE_SIZE - 1);
Chris Masond1310b22008-01-24 16:13:08 -05005378
Chris Masond3977122009-01-05 21:25:51 -05005379 while (len > 0) {
David Sterbafb85fc92014-07-31 01:03:53 +02005380 page = eb->pages[i];
Chris Masond1310b22008-01-24 16:13:08 -05005381 WARN_ON(!PageUptodate(page));
5382
5383 cur = min(len, PAGE_CACHE_SIZE - offset);
Chris Masona6591712011-07-19 12:04:14 -04005384 kaddr = page_address(page);
Chris Masond1310b22008-01-24 16:13:08 -05005385 memset(kaddr + offset, c, cur);
Chris Masond1310b22008-01-24 16:13:08 -05005386
5387 len -= cur;
5388 offset = 0;
5389 i++;
5390 }
5391}
Chris Masond1310b22008-01-24 16:13:08 -05005392
5393void copy_extent_buffer(struct extent_buffer *dst, struct extent_buffer *src,
5394 unsigned long dst_offset, unsigned long src_offset,
5395 unsigned long len)
5396{
5397 u64 dst_len = dst->len;
5398 size_t cur;
5399 size_t offset;
5400 struct page *page;
5401 char *kaddr;
5402 size_t start_offset = dst->start & ((u64)PAGE_CACHE_SIZE - 1);
5403 unsigned long i = (start_offset + dst_offset) >> PAGE_CACHE_SHIFT;
5404
5405 WARN_ON(src->len != dst_len);
5406
5407 offset = (start_offset + dst_offset) &
Geert Uytterhoeven778746b2013-08-20 13:20:16 +02005408 (PAGE_CACHE_SIZE - 1);
Chris Masond1310b22008-01-24 16:13:08 -05005409
Chris Masond3977122009-01-05 21:25:51 -05005410 while (len > 0) {
David Sterbafb85fc92014-07-31 01:03:53 +02005411 page = dst->pages[i];
Chris Masond1310b22008-01-24 16:13:08 -05005412 WARN_ON(!PageUptodate(page));
5413
5414 cur = min(len, (unsigned long)(PAGE_CACHE_SIZE - offset));
5415
Chris Masona6591712011-07-19 12:04:14 -04005416 kaddr = page_address(page);
Chris Masond1310b22008-01-24 16:13:08 -05005417 read_extent_buffer(src, kaddr + offset, src_offset, cur);
Chris Masond1310b22008-01-24 16:13:08 -05005418
5419 src_offset += cur;
5420 len -= cur;
5421 offset = 0;
5422 i++;
5423 }
5424}
Chris Masond1310b22008-01-24 16:13:08 -05005425
Sergei Trofimovich33872062011-04-11 21:52:52 +00005426static inline bool areas_overlap(unsigned long src, unsigned long dst, unsigned long len)
5427{
5428 unsigned long distance = (src > dst) ? src - dst : dst - src;
5429 return distance < len;
5430}
5431
Chris Masond1310b22008-01-24 16:13:08 -05005432static void copy_pages(struct page *dst_page, struct page *src_page,
5433 unsigned long dst_off, unsigned long src_off,
5434 unsigned long len)
5435{
Chris Masona6591712011-07-19 12:04:14 -04005436 char *dst_kaddr = page_address(dst_page);
Chris Masond1310b22008-01-24 16:13:08 -05005437 char *src_kaddr;
Chris Mason727011e2010-08-06 13:21:20 -04005438 int must_memmove = 0;
Chris Masond1310b22008-01-24 16:13:08 -05005439
Sergei Trofimovich33872062011-04-11 21:52:52 +00005440 if (dst_page != src_page) {
Chris Masona6591712011-07-19 12:04:14 -04005441 src_kaddr = page_address(src_page);
Sergei Trofimovich33872062011-04-11 21:52:52 +00005442 } else {
Chris Masond1310b22008-01-24 16:13:08 -05005443 src_kaddr = dst_kaddr;
Chris Mason727011e2010-08-06 13:21:20 -04005444 if (areas_overlap(src_off, dst_off, len))
5445 must_memmove = 1;
Sergei Trofimovich33872062011-04-11 21:52:52 +00005446 }
Chris Masond1310b22008-01-24 16:13:08 -05005447
Chris Mason727011e2010-08-06 13:21:20 -04005448 if (must_memmove)
5449 memmove(dst_kaddr + dst_off, src_kaddr + src_off, len);
5450 else
5451 memcpy(dst_kaddr + dst_off, src_kaddr + src_off, len);
Chris Masond1310b22008-01-24 16:13:08 -05005452}
5453
5454void memcpy_extent_buffer(struct extent_buffer *dst, unsigned long dst_offset,
5455 unsigned long src_offset, unsigned long len)
5456{
5457 size_t cur;
5458 size_t dst_off_in_page;
5459 size_t src_off_in_page;
5460 size_t start_offset = dst->start & ((u64)PAGE_CACHE_SIZE - 1);
5461 unsigned long dst_i;
5462 unsigned long src_i;
5463
5464 if (src_offset + len > dst->len) {
Frank Holtonefe120a2013-12-20 11:37:06 -05005465 printk(KERN_ERR "BTRFS: memmove bogus src_offset %lu move "
Chris Masond3977122009-01-05 21:25:51 -05005466 "len %lu dst len %lu\n", src_offset, len, dst->len);
Chris Masond1310b22008-01-24 16:13:08 -05005467 BUG_ON(1);
5468 }
5469 if (dst_offset + len > dst->len) {
Frank Holtonefe120a2013-12-20 11:37:06 -05005470 printk(KERN_ERR "BTRFS: memmove bogus dst_offset %lu move "
Chris Masond3977122009-01-05 21:25:51 -05005471 "len %lu dst len %lu\n", dst_offset, len, dst->len);
Chris Masond1310b22008-01-24 16:13:08 -05005472 BUG_ON(1);
5473 }
5474
Chris Masond3977122009-01-05 21:25:51 -05005475 while (len > 0) {
Chris Masond1310b22008-01-24 16:13:08 -05005476 dst_off_in_page = (start_offset + dst_offset) &
Geert Uytterhoeven778746b2013-08-20 13:20:16 +02005477 (PAGE_CACHE_SIZE - 1);
Chris Masond1310b22008-01-24 16:13:08 -05005478 src_off_in_page = (start_offset + src_offset) &
Geert Uytterhoeven778746b2013-08-20 13:20:16 +02005479 (PAGE_CACHE_SIZE - 1);
Chris Masond1310b22008-01-24 16:13:08 -05005480
5481 dst_i = (start_offset + dst_offset) >> PAGE_CACHE_SHIFT;
5482 src_i = (start_offset + src_offset) >> PAGE_CACHE_SHIFT;
5483
5484 cur = min(len, (unsigned long)(PAGE_CACHE_SIZE -
5485 src_off_in_page));
5486 cur = min_t(unsigned long, cur,
5487 (unsigned long)(PAGE_CACHE_SIZE - dst_off_in_page));
5488
David Sterbafb85fc92014-07-31 01:03:53 +02005489 copy_pages(dst->pages[dst_i], dst->pages[src_i],
Chris Masond1310b22008-01-24 16:13:08 -05005490 dst_off_in_page, src_off_in_page, cur);
5491
5492 src_offset += cur;
5493 dst_offset += cur;
5494 len -= cur;
5495 }
5496}
Chris Masond1310b22008-01-24 16:13:08 -05005497
5498void memmove_extent_buffer(struct extent_buffer *dst, unsigned long dst_offset,
5499 unsigned long src_offset, unsigned long len)
5500{
5501 size_t cur;
5502 size_t dst_off_in_page;
5503 size_t src_off_in_page;
5504 unsigned long dst_end = dst_offset + len - 1;
5505 unsigned long src_end = src_offset + len - 1;
5506 size_t start_offset = dst->start & ((u64)PAGE_CACHE_SIZE - 1);
5507 unsigned long dst_i;
5508 unsigned long src_i;
5509
5510 if (src_offset + len > dst->len) {
Frank Holtonefe120a2013-12-20 11:37:06 -05005511 printk(KERN_ERR "BTRFS: memmove bogus src_offset %lu move "
Chris Masond3977122009-01-05 21:25:51 -05005512 "len %lu len %lu\n", src_offset, len, dst->len);
Chris Masond1310b22008-01-24 16:13:08 -05005513 BUG_ON(1);
5514 }
5515 if (dst_offset + len > dst->len) {
Frank Holtonefe120a2013-12-20 11:37:06 -05005516 printk(KERN_ERR "BTRFS: memmove bogus dst_offset %lu move "
Chris Masond3977122009-01-05 21:25:51 -05005517 "len %lu len %lu\n", dst_offset, len, dst->len);
Chris Masond1310b22008-01-24 16:13:08 -05005518 BUG_ON(1);
5519 }
Chris Mason727011e2010-08-06 13:21:20 -04005520 if (dst_offset < src_offset) {
Chris Masond1310b22008-01-24 16:13:08 -05005521 memcpy_extent_buffer(dst, dst_offset, src_offset, len);
5522 return;
5523 }
Chris Masond3977122009-01-05 21:25:51 -05005524 while (len > 0) {
Chris Masond1310b22008-01-24 16:13:08 -05005525 dst_i = (start_offset + dst_end) >> PAGE_CACHE_SHIFT;
5526 src_i = (start_offset + src_end) >> PAGE_CACHE_SHIFT;
5527
5528 dst_off_in_page = (start_offset + dst_end) &
Geert Uytterhoeven778746b2013-08-20 13:20:16 +02005529 (PAGE_CACHE_SIZE - 1);
Chris Masond1310b22008-01-24 16:13:08 -05005530 src_off_in_page = (start_offset + src_end) &
Geert Uytterhoeven778746b2013-08-20 13:20:16 +02005531 (PAGE_CACHE_SIZE - 1);
Chris Masond1310b22008-01-24 16:13:08 -05005532
5533 cur = min_t(unsigned long, len, src_off_in_page + 1);
5534 cur = min(cur, dst_off_in_page + 1);
David Sterbafb85fc92014-07-31 01:03:53 +02005535 copy_pages(dst->pages[dst_i], dst->pages[src_i],
Chris Masond1310b22008-01-24 16:13:08 -05005536 dst_off_in_page - cur + 1,
5537 src_off_in_page - cur + 1, cur);
5538
5539 dst_end -= cur;
5540 src_end -= cur;
5541 len -= cur;
5542 }
5543}
Chris Mason6af118ce2008-07-22 11:18:07 -04005544
David Sterbaf7a52a42013-04-26 14:56:29 +00005545int try_release_extent_buffer(struct page *page)
Miao Xie19fe0a82010-10-26 20:57:29 -04005546{
Chris Mason6af118ce2008-07-22 11:18:07 -04005547 struct extent_buffer *eb;
Miao Xie897ca6e2010-10-26 20:57:29 -04005548
Miao Xie19fe0a82010-10-26 20:57:29 -04005549 /*
Josef Bacik3083ee22012-03-09 16:01:49 -05005550 * We need to make sure noboody is attaching this page to an eb right
5551 * now.
Miao Xie19fe0a82010-10-26 20:57:29 -04005552 */
Josef Bacik3083ee22012-03-09 16:01:49 -05005553 spin_lock(&page->mapping->private_lock);
5554 if (!PagePrivate(page)) {
5555 spin_unlock(&page->mapping->private_lock);
5556 return 1;
Miao Xie19fe0a82010-10-26 20:57:29 -04005557 }
5558
Josef Bacik3083ee22012-03-09 16:01:49 -05005559 eb = (struct extent_buffer *)page->private;
5560 BUG_ON(!eb);
Miao Xie19fe0a82010-10-26 20:57:29 -04005561
Josef Bacik0b32f4b2012-03-13 09:38:00 -04005562 /*
Josef Bacik3083ee22012-03-09 16:01:49 -05005563 * This is a little awful but should be ok, we need to make sure that
5564 * the eb doesn't disappear out from under us while we're looking at
5565 * this page.
5566 */
5567 spin_lock(&eb->refs_lock);
Josef Bacik0b32f4b2012-03-13 09:38:00 -04005568 if (atomic_read(&eb->refs) != 1 || extent_buffer_under_io(eb)) {
Josef Bacik3083ee22012-03-09 16:01:49 -05005569 spin_unlock(&eb->refs_lock);
5570 spin_unlock(&page->mapping->private_lock);
5571 return 0;
5572 }
5573 spin_unlock(&page->mapping->private_lock);
5574
Josef Bacik3083ee22012-03-09 16:01:49 -05005575 /*
5576 * If tree ref isn't set then we know the ref on this eb is a real ref,
5577 * so just return, this page will likely be freed soon anyway.
5578 */
5579 if (!test_and_clear_bit(EXTENT_BUFFER_TREE_REF, &eb->bflags)) {
5580 spin_unlock(&eb->refs_lock);
5581 return 0;
5582 }
Josef Bacik3083ee22012-03-09 16:01:49 -05005583
David Sterbaf7a52a42013-04-26 14:56:29 +00005584 return release_extent_buffer(eb);
Chris Mason6af118ce2008-07-22 11:18:07 -04005585}