blob: bb318b4e5dd0c47b11780059a7d434c6484a2814 [file] [log] [blame]
Chris Masond1310b22008-01-24 16:13:08 -05001#include <linux/bitops.h>
2#include <linux/slab.h>
3#include <linux/bio.h>
4#include <linux/mm.h>
Chris Masond1310b22008-01-24 16:13:08 -05005#include <linux/pagemap.h>
6#include <linux/page-flags.h>
Chris Masond1310b22008-01-24 16:13:08 -05007#include <linux/spinlock.h>
8#include <linux/blkdev.h>
9#include <linux/swap.h>
Chris Masond1310b22008-01-24 16:13:08 -050010#include <linux/writeback.h>
11#include <linux/pagevec.h>
Linus Torvalds268bb0c2011-05-20 12:50:29 -070012#include <linux/prefetch.h>
Dan Magenheimer90a887c2011-05-26 10:01:56 -060013#include <linux/cleancache.h>
Chris Masond1310b22008-01-24 16:13:08 -050014#include "extent_io.h"
15#include "extent_map.h"
David Woodhouse902b22f2008-08-20 08:51:49 -040016#include "ctree.h"
17#include "btrfs_inode.h"
Jan Schmidt4a54c8c2011-07-22 15:41:52 +020018#include "volumes.h"
Stefan Behrens21adbd52011-11-09 13:44:05 +010019#include "check-integrity.h"
Josef Bacik0b32f4b2012-03-13 09:38:00 -040020#include "locking.h"
Josef Bacik606686e2012-06-04 14:03:51 -040021#include "rcu-string.h"
Liu Bofe09e162013-09-22 12:54:23 +080022#include "backref.h"
Chris Masond1310b22008-01-24 16:13:08 -050023
Chris Masond1310b22008-01-24 16:13:08 -050024static struct kmem_cache *extent_state_cache;
25static struct kmem_cache *extent_buffer_cache;
Chris Mason9be33952013-05-17 18:30:14 -040026static struct bio_set *btrfs_bioset;
Chris Masond1310b22008-01-24 16:13:08 -050027
Filipe Manana27a35072014-07-06 20:09:59 +010028static inline bool extent_state_in_tree(const struct extent_state *state)
29{
30 return !RB_EMPTY_NODE(&state->rb_node);
31}
32
Eric Sandeen6d49ba12013-04-22 16:12:31 +000033#ifdef CONFIG_BTRFS_DEBUG
Chris Masond1310b22008-01-24 16:13:08 -050034static LIST_HEAD(buffers);
35static LIST_HEAD(states);
Chris Mason4bef0842008-09-08 11:18:08 -040036
Chris Masond3977122009-01-05 21:25:51 -050037static DEFINE_SPINLOCK(leak_lock);
Eric Sandeen6d49ba12013-04-22 16:12:31 +000038
39static inline
40void btrfs_leak_debug_add(struct list_head *new, struct list_head *head)
41{
42 unsigned long flags;
43
44 spin_lock_irqsave(&leak_lock, flags);
45 list_add(new, head);
46 spin_unlock_irqrestore(&leak_lock, flags);
47}
48
49static inline
50void btrfs_leak_debug_del(struct list_head *entry)
51{
52 unsigned long flags;
53
54 spin_lock_irqsave(&leak_lock, flags);
55 list_del(entry);
56 spin_unlock_irqrestore(&leak_lock, flags);
57}
58
59static inline
60void btrfs_leak_debug_check(void)
61{
62 struct extent_state *state;
63 struct extent_buffer *eb;
64
65 while (!list_empty(&states)) {
66 state = list_entry(states.next, struct extent_state, leak_list);
David Sterba9ee49a042015-01-14 19:52:13 +010067 pr_err("BTRFS: state leak: start %llu end %llu state %u in tree %d refs %d\n",
Filipe Manana27a35072014-07-06 20:09:59 +010068 state->start, state->end, state->state,
69 extent_state_in_tree(state),
Geert Uytterhoevenc1c9ff72013-08-20 13:20:07 +020070 atomic_read(&state->refs));
Eric Sandeen6d49ba12013-04-22 16:12:31 +000071 list_del(&state->leak_list);
72 kmem_cache_free(extent_state_cache, state);
73 }
74
75 while (!list_empty(&buffers)) {
76 eb = list_entry(buffers.next, struct extent_buffer, leak_list);
Frank Holtonefe120a2013-12-20 11:37:06 -050077 printk(KERN_ERR "BTRFS: buffer leak start %llu len %lu "
Geert Uytterhoevenc1c9ff72013-08-20 13:20:07 +020078 "refs %d\n",
79 eb->start, eb->len, atomic_read(&eb->refs));
Eric Sandeen6d49ba12013-04-22 16:12:31 +000080 list_del(&eb->leak_list);
81 kmem_cache_free(extent_buffer_cache, eb);
82 }
83}
David Sterba8d599ae2013-04-30 15:22:23 +000084
Josef Bacika5dee372013-12-13 10:02:44 -050085#define btrfs_debug_check_extent_io_range(tree, start, end) \
86 __btrfs_debug_check_extent_io_range(__func__, (tree), (start), (end))
David Sterba8d599ae2013-04-30 15:22:23 +000087static inline void __btrfs_debug_check_extent_io_range(const char *caller,
Josef Bacika5dee372013-12-13 10:02:44 -050088 struct extent_io_tree *tree, u64 start, u64 end)
David Sterba8d599ae2013-04-30 15:22:23 +000089{
Josef Bacika5dee372013-12-13 10:02:44 -050090 struct inode *inode;
91 u64 isize;
David Sterba8d599ae2013-04-30 15:22:23 +000092
Josef Bacika5dee372013-12-13 10:02:44 -050093 if (!tree->mapping)
94 return;
95
96 inode = tree->mapping->host;
97 isize = i_size_read(inode);
David Sterba8d599ae2013-04-30 15:22:23 +000098 if (end >= PAGE_SIZE && (end % 2) == 0 && end != isize - 1) {
David Sterba94647322015-10-08 11:01:36 +020099 btrfs_debug_rl(BTRFS_I(inode)->root->fs_info,
100 "%s: ino %llu isize %llu odd range [%llu,%llu]",
Geert Uytterhoevenc1c9ff72013-08-20 13:20:07 +0200101 caller, btrfs_ino(inode), isize, start, end);
David Sterba8d599ae2013-04-30 15:22:23 +0000102 }
103}
Eric Sandeen6d49ba12013-04-22 16:12:31 +0000104#else
105#define btrfs_leak_debug_add(new, head) do {} while (0)
106#define btrfs_leak_debug_del(entry) do {} while (0)
107#define btrfs_leak_debug_check() do {} while (0)
David Sterba8d599ae2013-04-30 15:22:23 +0000108#define btrfs_debug_check_extent_io_range(c, s, e) do {} while (0)
Chris Mason4bef0842008-09-08 11:18:08 -0400109#endif
Chris Masond1310b22008-01-24 16:13:08 -0500110
Chris Masond1310b22008-01-24 16:13:08 -0500111#define BUFFER_LRU_MAX 64
112
113struct tree_entry {
114 u64 start;
115 u64 end;
Chris Masond1310b22008-01-24 16:13:08 -0500116 struct rb_node rb_node;
117};
118
119struct extent_page_data {
120 struct bio *bio;
121 struct extent_io_tree *tree;
122 get_extent_t *get_extent;
Josef Bacikde0022b2012-09-25 14:25:58 -0400123 unsigned long bio_flags;
Chris Mason771ed682008-11-06 22:02:51 -0500124
125 /* tells writepage not to lock the state bits for this range
126 * it still does the unlocking
127 */
Chris Masonffbd5172009-04-20 15:50:09 -0400128 unsigned int extent_locked:1;
129
130 /* tells the submit_bio code to use a WRITE_SYNC */
131 unsigned int sync_io:1;
Chris Masond1310b22008-01-24 16:13:08 -0500132};
133
Josef Bacik0b32f4b2012-03-13 09:38:00 -0400134static noinline void flush_write_bio(void *data);
Jeff Mahoneyc2d904e2011-10-03 23:22:32 -0400135static inline struct btrfs_fs_info *
136tree_fs_info(struct extent_io_tree *tree)
137{
Josef Bacika5dee372013-12-13 10:02:44 -0500138 if (!tree->mapping)
139 return NULL;
Jeff Mahoneyc2d904e2011-10-03 23:22:32 -0400140 return btrfs_sb(tree->mapping->host->i_sb);
141}
Josef Bacik0b32f4b2012-03-13 09:38:00 -0400142
Chris Masond1310b22008-01-24 16:13:08 -0500143int __init extent_io_init(void)
144{
David Sterba837e1972012-09-07 03:00:48 -0600145 extent_state_cache = kmem_cache_create("btrfs_extent_state",
Christoph Hellwig9601e3f2009-04-13 15:33:09 +0200146 sizeof(struct extent_state), 0,
147 SLAB_RECLAIM_ACCOUNT | SLAB_MEM_SPREAD, NULL);
Chris Masond1310b22008-01-24 16:13:08 -0500148 if (!extent_state_cache)
149 return -ENOMEM;
150
David Sterba837e1972012-09-07 03:00:48 -0600151 extent_buffer_cache = kmem_cache_create("btrfs_extent_buffer",
Christoph Hellwig9601e3f2009-04-13 15:33:09 +0200152 sizeof(struct extent_buffer), 0,
153 SLAB_RECLAIM_ACCOUNT | SLAB_MEM_SPREAD, NULL);
Chris Masond1310b22008-01-24 16:13:08 -0500154 if (!extent_buffer_cache)
155 goto free_state_cache;
Chris Mason9be33952013-05-17 18:30:14 -0400156
157 btrfs_bioset = bioset_create(BIO_POOL_SIZE,
158 offsetof(struct btrfs_io_bio, bio));
159 if (!btrfs_bioset)
160 goto free_buffer_cache;
Darrick J. Wongb208c2f2013-09-19 20:37:07 -0700161
162 if (bioset_integrity_create(btrfs_bioset, BIO_POOL_SIZE))
163 goto free_bioset;
164
Chris Masond1310b22008-01-24 16:13:08 -0500165 return 0;
166
Darrick J. Wongb208c2f2013-09-19 20:37:07 -0700167free_bioset:
168 bioset_free(btrfs_bioset);
169 btrfs_bioset = NULL;
170
Chris Mason9be33952013-05-17 18:30:14 -0400171free_buffer_cache:
172 kmem_cache_destroy(extent_buffer_cache);
173 extent_buffer_cache = NULL;
174
Chris Masond1310b22008-01-24 16:13:08 -0500175free_state_cache:
176 kmem_cache_destroy(extent_state_cache);
Chris Mason9be33952013-05-17 18:30:14 -0400177 extent_state_cache = NULL;
Chris Masond1310b22008-01-24 16:13:08 -0500178 return -ENOMEM;
179}
180
181void extent_io_exit(void)
182{
Eric Sandeen6d49ba12013-04-22 16:12:31 +0000183 btrfs_leak_debug_check();
Kirill A. Shutemov8c0a8532012-09-26 11:33:07 +1000184
185 /*
186 * Make sure all delayed rcu free are flushed before we
187 * destroy caches.
188 */
189 rcu_barrier();
Chris Masond1310b22008-01-24 16:13:08 -0500190 if (extent_state_cache)
191 kmem_cache_destroy(extent_state_cache);
192 if (extent_buffer_cache)
193 kmem_cache_destroy(extent_buffer_cache);
Chris Mason9be33952013-05-17 18:30:14 -0400194 if (btrfs_bioset)
195 bioset_free(btrfs_bioset);
Chris Masond1310b22008-01-24 16:13:08 -0500196}
197
198void extent_io_tree_init(struct extent_io_tree *tree,
David Sterbaf993c882011-04-20 23:35:57 +0200199 struct address_space *mapping)
Chris Masond1310b22008-01-24 16:13:08 -0500200{
Eric Paris6bef4d32010-02-23 19:43:04 +0000201 tree->state = RB_ROOT;
Chris Masond1310b22008-01-24 16:13:08 -0500202 tree->ops = NULL;
203 tree->dirty_bytes = 0;
Chris Mason70dec802008-01-29 09:59:12 -0500204 spin_lock_init(&tree->lock);
Chris Masond1310b22008-01-24 16:13:08 -0500205 tree->mapping = mapping;
Chris Masond1310b22008-01-24 16:13:08 -0500206}
Chris Masond1310b22008-01-24 16:13:08 -0500207
Christoph Hellwigb2950862008-12-02 09:54:17 -0500208static struct extent_state *alloc_extent_state(gfp_t mask)
Chris Masond1310b22008-01-24 16:13:08 -0500209{
210 struct extent_state *state;
Chris Masond1310b22008-01-24 16:13:08 -0500211
212 state = kmem_cache_alloc(extent_state_cache, mask);
Peter2b114d12008-04-01 11:21:40 -0400213 if (!state)
Chris Masond1310b22008-01-24 16:13:08 -0500214 return state;
215 state->state = 0;
Chris Masond1310b22008-01-24 16:13:08 -0500216 state->private = 0;
Filipe Manana27a35072014-07-06 20:09:59 +0100217 RB_CLEAR_NODE(&state->rb_node);
Eric Sandeen6d49ba12013-04-22 16:12:31 +0000218 btrfs_leak_debug_add(&state->leak_list, &states);
Chris Masond1310b22008-01-24 16:13:08 -0500219 atomic_set(&state->refs, 1);
220 init_waitqueue_head(&state->wq);
Jeff Mahoney143bede2012-03-01 14:56:26 +0100221 trace_alloc_extent_state(state, mask, _RET_IP_);
Chris Masond1310b22008-01-24 16:13:08 -0500222 return state;
223}
Chris Masond1310b22008-01-24 16:13:08 -0500224
Chris Mason4845e442010-05-25 20:56:50 -0400225void free_extent_state(struct extent_state *state)
Chris Masond1310b22008-01-24 16:13:08 -0500226{
Chris Masond1310b22008-01-24 16:13:08 -0500227 if (!state)
228 return;
229 if (atomic_dec_and_test(&state->refs)) {
Filipe Manana27a35072014-07-06 20:09:59 +0100230 WARN_ON(extent_state_in_tree(state));
Eric Sandeen6d49ba12013-04-22 16:12:31 +0000231 btrfs_leak_debug_del(&state->leak_list);
Jeff Mahoney143bede2012-03-01 14:56:26 +0100232 trace_free_extent_state(state, _RET_IP_);
Chris Masond1310b22008-01-24 16:13:08 -0500233 kmem_cache_free(extent_state_cache, state);
234 }
235}
Chris Masond1310b22008-01-24 16:13:08 -0500236
Filipe Mananaf2071b22014-02-12 15:05:53 +0000237static struct rb_node *tree_insert(struct rb_root *root,
238 struct rb_node *search_start,
239 u64 offset,
Filipe David Borba Manana12cfbad2013-11-26 15:41:47 +0000240 struct rb_node *node,
241 struct rb_node ***p_in,
242 struct rb_node **parent_in)
Chris Masond1310b22008-01-24 16:13:08 -0500243{
Filipe Mananaf2071b22014-02-12 15:05:53 +0000244 struct rb_node **p;
Chris Masond3977122009-01-05 21:25:51 -0500245 struct rb_node *parent = NULL;
Chris Masond1310b22008-01-24 16:13:08 -0500246 struct tree_entry *entry;
247
Filipe David Borba Manana12cfbad2013-11-26 15:41:47 +0000248 if (p_in && parent_in) {
249 p = *p_in;
250 parent = *parent_in;
251 goto do_insert;
252 }
253
Filipe Mananaf2071b22014-02-12 15:05:53 +0000254 p = search_start ? &search_start : &root->rb_node;
Chris Masond3977122009-01-05 21:25:51 -0500255 while (*p) {
Chris Masond1310b22008-01-24 16:13:08 -0500256 parent = *p;
257 entry = rb_entry(parent, struct tree_entry, rb_node);
258
259 if (offset < entry->start)
260 p = &(*p)->rb_left;
261 else if (offset > entry->end)
262 p = &(*p)->rb_right;
263 else
264 return parent;
265 }
266
Filipe David Borba Manana12cfbad2013-11-26 15:41:47 +0000267do_insert:
Chris Masond1310b22008-01-24 16:13:08 -0500268 rb_link_node(node, parent, p);
269 rb_insert_color(node, root);
270 return NULL;
271}
272
Chris Mason80ea96b2008-02-01 14:51:59 -0500273static struct rb_node *__etree_search(struct extent_io_tree *tree, u64 offset,
Filipe David Borba Manana12cfbad2013-11-26 15:41:47 +0000274 struct rb_node **prev_ret,
275 struct rb_node **next_ret,
276 struct rb_node ***p_ret,
277 struct rb_node **parent_ret)
Chris Masond1310b22008-01-24 16:13:08 -0500278{
Chris Mason80ea96b2008-02-01 14:51:59 -0500279 struct rb_root *root = &tree->state;
Filipe David Borba Manana12cfbad2013-11-26 15:41:47 +0000280 struct rb_node **n = &root->rb_node;
Chris Masond1310b22008-01-24 16:13:08 -0500281 struct rb_node *prev = NULL;
282 struct rb_node *orig_prev = NULL;
283 struct tree_entry *entry;
284 struct tree_entry *prev_entry = NULL;
285
Filipe David Borba Manana12cfbad2013-11-26 15:41:47 +0000286 while (*n) {
287 prev = *n;
288 entry = rb_entry(prev, struct tree_entry, rb_node);
Chris Masond1310b22008-01-24 16:13:08 -0500289 prev_entry = entry;
290
291 if (offset < entry->start)
Filipe David Borba Manana12cfbad2013-11-26 15:41:47 +0000292 n = &(*n)->rb_left;
Chris Masond1310b22008-01-24 16:13:08 -0500293 else if (offset > entry->end)
Filipe David Borba Manana12cfbad2013-11-26 15:41:47 +0000294 n = &(*n)->rb_right;
Chris Masond3977122009-01-05 21:25:51 -0500295 else
Filipe David Borba Manana12cfbad2013-11-26 15:41:47 +0000296 return *n;
Chris Masond1310b22008-01-24 16:13:08 -0500297 }
298
Filipe David Borba Manana12cfbad2013-11-26 15:41:47 +0000299 if (p_ret)
300 *p_ret = n;
301 if (parent_ret)
302 *parent_ret = prev;
303
Chris Masond1310b22008-01-24 16:13:08 -0500304 if (prev_ret) {
305 orig_prev = prev;
Chris Masond3977122009-01-05 21:25:51 -0500306 while (prev && offset > prev_entry->end) {
Chris Masond1310b22008-01-24 16:13:08 -0500307 prev = rb_next(prev);
308 prev_entry = rb_entry(prev, struct tree_entry, rb_node);
309 }
310 *prev_ret = prev;
311 prev = orig_prev;
312 }
313
314 if (next_ret) {
315 prev_entry = rb_entry(prev, struct tree_entry, rb_node);
Chris Masond3977122009-01-05 21:25:51 -0500316 while (prev && offset < prev_entry->start) {
Chris Masond1310b22008-01-24 16:13:08 -0500317 prev = rb_prev(prev);
318 prev_entry = rb_entry(prev, struct tree_entry, rb_node);
319 }
320 *next_ret = prev;
321 }
322 return NULL;
323}
324
Filipe David Borba Manana12cfbad2013-11-26 15:41:47 +0000325static inline struct rb_node *
326tree_search_for_insert(struct extent_io_tree *tree,
327 u64 offset,
328 struct rb_node ***p_ret,
329 struct rb_node **parent_ret)
Chris Masond1310b22008-01-24 16:13:08 -0500330{
Chris Mason70dec802008-01-29 09:59:12 -0500331 struct rb_node *prev = NULL;
Chris Masond1310b22008-01-24 16:13:08 -0500332 struct rb_node *ret;
Chris Mason70dec802008-01-29 09:59:12 -0500333
Filipe David Borba Manana12cfbad2013-11-26 15:41:47 +0000334 ret = __etree_search(tree, offset, &prev, NULL, p_ret, parent_ret);
Chris Masond3977122009-01-05 21:25:51 -0500335 if (!ret)
Chris Masond1310b22008-01-24 16:13:08 -0500336 return prev;
337 return ret;
338}
339
Filipe David Borba Manana12cfbad2013-11-26 15:41:47 +0000340static inline struct rb_node *tree_search(struct extent_io_tree *tree,
341 u64 offset)
342{
343 return tree_search_for_insert(tree, offset, NULL, NULL);
344}
345
Josef Bacik9ed74f22009-09-11 16:12:44 -0400346static void merge_cb(struct extent_io_tree *tree, struct extent_state *new,
347 struct extent_state *other)
348{
349 if (tree->ops && tree->ops->merge_extent_hook)
350 tree->ops->merge_extent_hook(tree->mapping->host, new,
351 other);
352}
353
Chris Masond1310b22008-01-24 16:13:08 -0500354/*
355 * utility function to look for merge candidates inside a given range.
356 * Any extents with matching state are merged together into a single
357 * extent in the tree. Extents with EXTENT_IO in their state field
358 * are not merged because the end_io handlers need to be able to do
359 * operations on them without sleeping (or doing allocations/splits).
360 *
361 * This should be called with the tree lock held.
362 */
Jeff Mahoney1bf85042011-07-21 16:56:09 +0000363static void merge_state(struct extent_io_tree *tree,
364 struct extent_state *state)
Chris Masond1310b22008-01-24 16:13:08 -0500365{
366 struct extent_state *other;
367 struct rb_node *other_node;
368
Zheng Yan5b21f2e2008-09-26 10:05:38 -0400369 if (state->state & (EXTENT_IOBITS | EXTENT_BOUNDARY))
Jeff Mahoney1bf85042011-07-21 16:56:09 +0000370 return;
Chris Masond1310b22008-01-24 16:13:08 -0500371
372 other_node = rb_prev(&state->rb_node);
373 if (other_node) {
374 other = rb_entry(other_node, struct extent_state, rb_node);
375 if (other->end == state->start - 1 &&
376 other->state == state->state) {
Josef Bacik9ed74f22009-09-11 16:12:44 -0400377 merge_cb(tree, state, other);
Chris Masond1310b22008-01-24 16:13:08 -0500378 state->start = other->start;
Chris Masond1310b22008-01-24 16:13:08 -0500379 rb_erase(&other->rb_node, &tree->state);
Filipe Manana27a35072014-07-06 20:09:59 +0100380 RB_CLEAR_NODE(&other->rb_node);
Chris Masond1310b22008-01-24 16:13:08 -0500381 free_extent_state(other);
382 }
383 }
384 other_node = rb_next(&state->rb_node);
385 if (other_node) {
386 other = rb_entry(other_node, struct extent_state, rb_node);
387 if (other->start == state->end + 1 &&
388 other->state == state->state) {
Josef Bacik9ed74f22009-09-11 16:12:44 -0400389 merge_cb(tree, state, other);
Josef Bacikdf98b6e2011-06-20 14:53:48 -0400390 state->end = other->end;
Josef Bacikdf98b6e2011-06-20 14:53:48 -0400391 rb_erase(&other->rb_node, &tree->state);
Filipe Manana27a35072014-07-06 20:09:59 +0100392 RB_CLEAR_NODE(&other->rb_node);
Josef Bacikdf98b6e2011-06-20 14:53:48 -0400393 free_extent_state(other);
Chris Masond1310b22008-01-24 16:13:08 -0500394 }
395 }
Chris Masond1310b22008-01-24 16:13:08 -0500396}
397
Jeff Mahoney1bf85042011-07-21 16:56:09 +0000398static void set_state_cb(struct extent_io_tree *tree,
David Sterba9ee49a042015-01-14 19:52:13 +0100399 struct extent_state *state, unsigned *bits)
Chris Mason291d6732008-01-29 15:55:23 -0500400{
Jeff Mahoney1bf85042011-07-21 16:56:09 +0000401 if (tree->ops && tree->ops->set_bit_hook)
402 tree->ops->set_bit_hook(tree->mapping->host, state, bits);
Chris Mason291d6732008-01-29 15:55:23 -0500403}
404
405static void clear_state_cb(struct extent_io_tree *tree,
David Sterba9ee49a042015-01-14 19:52:13 +0100406 struct extent_state *state, unsigned *bits)
Chris Mason291d6732008-01-29 15:55:23 -0500407{
Josef Bacik9ed74f22009-09-11 16:12:44 -0400408 if (tree->ops && tree->ops->clear_bit_hook)
409 tree->ops->clear_bit_hook(tree->mapping->host, state, bits);
Chris Mason291d6732008-01-29 15:55:23 -0500410}
411
Xiao Guangrong3150b692011-07-14 03:19:08 +0000412static void set_state_bits(struct extent_io_tree *tree,
David Sterba9ee49a042015-01-14 19:52:13 +0100413 struct extent_state *state, unsigned *bits);
Xiao Guangrong3150b692011-07-14 03:19:08 +0000414
Chris Masond1310b22008-01-24 16:13:08 -0500415/*
416 * insert an extent_state struct into the tree. 'bits' are set on the
417 * struct before it is inserted.
418 *
419 * This may return -EEXIST if the extent is already there, in which case the
420 * state struct is freed.
421 *
422 * The tree lock is not taken internally. This is a utility function and
423 * probably isn't what you want to call (see set/clear_extent_bit).
424 */
425static int insert_state(struct extent_io_tree *tree,
426 struct extent_state *state, u64 start, u64 end,
Filipe David Borba Manana12cfbad2013-11-26 15:41:47 +0000427 struct rb_node ***p,
428 struct rb_node **parent,
David Sterba9ee49a042015-01-14 19:52:13 +0100429 unsigned *bits)
Chris Masond1310b22008-01-24 16:13:08 -0500430{
431 struct rb_node *node;
432
Julia Lawall31b1a2b2012-11-03 10:58:34 +0000433 if (end < start)
Frank Holtonefe120a2013-12-20 11:37:06 -0500434 WARN(1, KERN_ERR "BTRFS: end < start %llu %llu\n",
Geert Uytterhoevenc1c9ff72013-08-20 13:20:07 +0200435 end, start);
Chris Masond1310b22008-01-24 16:13:08 -0500436 state->start = start;
437 state->end = end;
Josef Bacik9ed74f22009-09-11 16:12:44 -0400438
Xiao Guangrong3150b692011-07-14 03:19:08 +0000439 set_state_bits(tree, state, bits);
440
Filipe Mananaf2071b22014-02-12 15:05:53 +0000441 node = tree_insert(&tree->state, NULL, end, &state->rb_node, p, parent);
Chris Masond1310b22008-01-24 16:13:08 -0500442 if (node) {
443 struct extent_state *found;
444 found = rb_entry(node, struct extent_state, rb_node);
Frank Holtonefe120a2013-12-20 11:37:06 -0500445 printk(KERN_ERR "BTRFS: found node %llu %llu on insert of "
Geert Uytterhoevenc1c9ff72013-08-20 13:20:07 +0200446 "%llu %llu\n",
447 found->start, found->end, start, end);
Chris Masond1310b22008-01-24 16:13:08 -0500448 return -EEXIST;
449 }
450 merge_state(tree, state);
451 return 0;
452}
453
Jeff Mahoney1bf85042011-07-21 16:56:09 +0000454static void split_cb(struct extent_io_tree *tree, struct extent_state *orig,
Josef Bacik9ed74f22009-09-11 16:12:44 -0400455 u64 split)
456{
457 if (tree->ops && tree->ops->split_extent_hook)
Jeff Mahoney1bf85042011-07-21 16:56:09 +0000458 tree->ops->split_extent_hook(tree->mapping->host, orig, split);
Josef Bacik9ed74f22009-09-11 16:12:44 -0400459}
460
Chris Masond1310b22008-01-24 16:13:08 -0500461/*
462 * split a given extent state struct in two, inserting the preallocated
463 * struct 'prealloc' as the newly created second half. 'split' indicates an
464 * offset inside 'orig' where it should be split.
465 *
466 * Before calling,
467 * the tree has 'orig' at [orig->start, orig->end]. After calling, there
468 * are two extent state structs in the tree:
469 * prealloc: [orig->start, split - 1]
470 * orig: [ split, orig->end ]
471 *
472 * The tree locks are not taken by this function. They need to be held
473 * by the caller.
474 */
475static int split_state(struct extent_io_tree *tree, struct extent_state *orig,
476 struct extent_state *prealloc, u64 split)
477{
478 struct rb_node *node;
Josef Bacik9ed74f22009-09-11 16:12:44 -0400479
480 split_cb(tree, orig, split);
481
Chris Masond1310b22008-01-24 16:13:08 -0500482 prealloc->start = orig->start;
483 prealloc->end = split - 1;
484 prealloc->state = orig->state;
485 orig->start = split;
486
Filipe Mananaf2071b22014-02-12 15:05:53 +0000487 node = tree_insert(&tree->state, &orig->rb_node, prealloc->end,
488 &prealloc->rb_node, NULL, NULL);
Chris Masond1310b22008-01-24 16:13:08 -0500489 if (node) {
Chris Masond1310b22008-01-24 16:13:08 -0500490 free_extent_state(prealloc);
491 return -EEXIST;
492 }
493 return 0;
494}
495
Li Zefancdc6a392012-03-12 16:39:48 +0800496static struct extent_state *next_state(struct extent_state *state)
497{
498 struct rb_node *next = rb_next(&state->rb_node);
499 if (next)
500 return rb_entry(next, struct extent_state, rb_node);
501 else
502 return NULL;
503}
504
Chris Masond1310b22008-01-24 16:13:08 -0500505/*
506 * utility function to clear some bits in an extent state struct.
Wang Sheng-Hui1b303fc2012-04-06 14:35:18 +0800507 * it will optionally wake up any one waiting on this state (wake == 1).
Chris Masond1310b22008-01-24 16:13:08 -0500508 *
509 * If no bits are set on the state struct after clearing things, the
510 * struct is freed and removed from the tree
511 */
Li Zefancdc6a392012-03-12 16:39:48 +0800512static struct extent_state *clear_state_bit(struct extent_io_tree *tree,
513 struct extent_state *state,
David Sterba9ee49a042015-01-14 19:52:13 +0100514 unsigned *bits, int wake)
Chris Masond1310b22008-01-24 16:13:08 -0500515{
Li Zefancdc6a392012-03-12 16:39:48 +0800516 struct extent_state *next;
David Sterba9ee49a042015-01-14 19:52:13 +0100517 unsigned bits_to_clear = *bits & ~EXTENT_CTLBITS;
Chris Masond1310b22008-01-24 16:13:08 -0500518
Yan, Zheng0ca1f7c2010-05-16 10:48:47 -0400519 if ((bits_to_clear & EXTENT_DIRTY) && (state->state & EXTENT_DIRTY)) {
Chris Masond1310b22008-01-24 16:13:08 -0500520 u64 range = state->end - state->start + 1;
521 WARN_ON(range > tree->dirty_bytes);
522 tree->dirty_bytes -= range;
523 }
Chris Mason291d6732008-01-29 15:55:23 -0500524 clear_state_cb(tree, state, bits);
Josef Bacik32c00af2009-10-08 13:34:05 -0400525 state->state &= ~bits_to_clear;
Chris Masond1310b22008-01-24 16:13:08 -0500526 if (wake)
527 wake_up(&state->wq);
Yan, Zheng0ca1f7c2010-05-16 10:48:47 -0400528 if (state->state == 0) {
Li Zefancdc6a392012-03-12 16:39:48 +0800529 next = next_state(state);
Filipe Manana27a35072014-07-06 20:09:59 +0100530 if (extent_state_in_tree(state)) {
Chris Masond1310b22008-01-24 16:13:08 -0500531 rb_erase(&state->rb_node, &tree->state);
Filipe Manana27a35072014-07-06 20:09:59 +0100532 RB_CLEAR_NODE(&state->rb_node);
Chris Masond1310b22008-01-24 16:13:08 -0500533 free_extent_state(state);
534 } else {
535 WARN_ON(1);
536 }
537 } else {
538 merge_state(tree, state);
Li Zefancdc6a392012-03-12 16:39:48 +0800539 next = next_state(state);
Chris Masond1310b22008-01-24 16:13:08 -0500540 }
Li Zefancdc6a392012-03-12 16:39:48 +0800541 return next;
Chris Masond1310b22008-01-24 16:13:08 -0500542}
543
Xiao Guangrong82337672011-04-20 06:44:57 +0000544static struct extent_state *
545alloc_extent_state_atomic(struct extent_state *prealloc)
546{
547 if (!prealloc)
548 prealloc = alloc_extent_state(GFP_ATOMIC);
549
550 return prealloc;
551}
552
Eric Sandeen48a3b632013-04-25 20:41:01 +0000553static void extent_io_tree_panic(struct extent_io_tree *tree, int err)
Jeff Mahoneyc2d904e2011-10-03 23:22:32 -0400554{
555 btrfs_panic(tree_fs_info(tree), err, "Locking error: "
556 "Extent tree was modified by another "
557 "thread while locked.");
558}
559
Chris Masond1310b22008-01-24 16:13:08 -0500560/*
561 * clear some bits on a range in the tree. This may require splitting
562 * or inserting elements in the tree, so the gfp mask is used to
563 * indicate which allocations or sleeping are allowed.
564 *
565 * pass 'wake' == 1 to kick any sleepers, and 'delete' == 1 to remove
566 * the given range from the tree regardless of state (ie for truncate).
567 *
568 * the range [start, end] is inclusive.
569 *
Jeff Mahoney6763af82012-03-01 14:56:29 +0100570 * This takes the tree lock, and returns 0 on success and < 0 on error.
Chris Masond1310b22008-01-24 16:13:08 -0500571 */
572int clear_extent_bit(struct extent_io_tree *tree, u64 start, u64 end,
David Sterba9ee49a042015-01-14 19:52:13 +0100573 unsigned bits, int wake, int delete,
Chris Mason2c64c532009-09-02 15:04:12 -0400574 struct extent_state **cached_state,
575 gfp_t mask)
Chris Masond1310b22008-01-24 16:13:08 -0500576{
577 struct extent_state *state;
Chris Mason2c64c532009-09-02 15:04:12 -0400578 struct extent_state *cached;
Chris Masond1310b22008-01-24 16:13:08 -0500579 struct extent_state *prealloc = NULL;
580 struct rb_node *node;
Yan Zheng5c939df2009-05-27 09:16:03 -0400581 u64 last_end;
Chris Masond1310b22008-01-24 16:13:08 -0500582 int err;
Josef Bacik2ac55d42010-02-03 19:33:23 +0000583 int clear = 0;
Chris Masond1310b22008-01-24 16:13:08 -0500584
Josef Bacika5dee372013-12-13 10:02:44 -0500585 btrfs_debug_check_extent_io_range(tree, start, end);
David Sterba8d599ae2013-04-30 15:22:23 +0000586
Josef Bacik7ee9e442013-06-21 16:37:03 -0400587 if (bits & EXTENT_DELALLOC)
588 bits |= EXTENT_NORESERVE;
589
Yan, Zheng0ca1f7c2010-05-16 10:48:47 -0400590 if (delete)
591 bits |= ~EXTENT_CTLBITS;
592 bits |= EXTENT_FIRST_DELALLOC;
593
Josef Bacik2ac55d42010-02-03 19:33:23 +0000594 if (bits & (EXTENT_IOBITS | EXTENT_BOUNDARY))
595 clear = 1;
Chris Masond1310b22008-01-24 16:13:08 -0500596again:
597 if (!prealloc && (mask & __GFP_WAIT)) {
Filipe Mananac7bc6312014-11-03 14:12:57 +0000598 /*
599 * Don't care for allocation failure here because we might end
600 * up not needing the pre-allocated extent state at all, which
601 * is the case if we only have in the tree extent states that
602 * cover our input range and don't cover too any other range.
603 * If we end up needing a new extent state we allocate it later.
604 */
Chris Masond1310b22008-01-24 16:13:08 -0500605 prealloc = alloc_extent_state(mask);
Chris Masond1310b22008-01-24 16:13:08 -0500606 }
607
Chris Masoncad321a2008-12-17 14:51:42 -0500608 spin_lock(&tree->lock);
Chris Mason2c64c532009-09-02 15:04:12 -0400609 if (cached_state) {
610 cached = *cached_state;
Josef Bacik2ac55d42010-02-03 19:33:23 +0000611
612 if (clear) {
613 *cached_state = NULL;
614 cached_state = NULL;
615 }
616
Filipe Manana27a35072014-07-06 20:09:59 +0100617 if (cached && extent_state_in_tree(cached) &&
618 cached->start <= start && cached->end > start) {
Josef Bacik2ac55d42010-02-03 19:33:23 +0000619 if (clear)
620 atomic_dec(&cached->refs);
Chris Mason2c64c532009-09-02 15:04:12 -0400621 state = cached;
Chris Mason42daec22009-09-23 19:51:09 -0400622 goto hit_next;
Chris Mason2c64c532009-09-02 15:04:12 -0400623 }
Josef Bacik2ac55d42010-02-03 19:33:23 +0000624 if (clear)
625 free_extent_state(cached);
Chris Mason2c64c532009-09-02 15:04:12 -0400626 }
Chris Masond1310b22008-01-24 16:13:08 -0500627 /*
628 * this search will find the extents that end after
629 * our range starts
630 */
Chris Mason80ea96b2008-02-01 14:51:59 -0500631 node = tree_search(tree, start);
Chris Masond1310b22008-01-24 16:13:08 -0500632 if (!node)
633 goto out;
634 state = rb_entry(node, struct extent_state, rb_node);
Chris Mason2c64c532009-09-02 15:04:12 -0400635hit_next:
Chris Masond1310b22008-01-24 16:13:08 -0500636 if (state->start > end)
637 goto out;
638 WARN_ON(state->end < start);
Yan Zheng5c939df2009-05-27 09:16:03 -0400639 last_end = state->end;
Chris Masond1310b22008-01-24 16:13:08 -0500640
Liu Bo04493142012-02-16 18:34:37 +0800641 /* the state doesn't have the wanted bits, go ahead */
Li Zefancdc6a392012-03-12 16:39:48 +0800642 if (!(state->state & bits)) {
643 state = next_state(state);
Liu Bo04493142012-02-16 18:34:37 +0800644 goto next;
Li Zefancdc6a392012-03-12 16:39:48 +0800645 }
Liu Bo04493142012-02-16 18:34:37 +0800646
Chris Masond1310b22008-01-24 16:13:08 -0500647 /*
648 * | ---- desired range ---- |
649 * | state | or
650 * | ------------- state -------------- |
651 *
652 * We need to split the extent we found, and may flip
653 * bits on second half.
654 *
655 * If the extent we found extends past our range, we
656 * just split and search again. It'll get split again
657 * the next time though.
658 *
659 * If the extent we found is inside our range, we clear
660 * the desired bit on it.
661 */
662
663 if (state->start < start) {
Xiao Guangrong82337672011-04-20 06:44:57 +0000664 prealloc = alloc_extent_state_atomic(prealloc);
665 BUG_ON(!prealloc);
Chris Masond1310b22008-01-24 16:13:08 -0500666 err = split_state(tree, state, prealloc, start);
Jeff Mahoneyc2d904e2011-10-03 23:22:32 -0400667 if (err)
668 extent_io_tree_panic(tree, err);
669
Chris Masond1310b22008-01-24 16:13:08 -0500670 prealloc = NULL;
671 if (err)
672 goto out;
673 if (state->end <= end) {
Liu Bod1ac6e42012-05-10 18:10:39 +0800674 state = clear_state_bit(tree, state, &bits, wake);
675 goto next;
Chris Masond1310b22008-01-24 16:13:08 -0500676 }
677 goto search_again;
678 }
679 /*
680 * | ---- desired range ---- |
681 * | state |
682 * We need to split the extent, and clear the bit
683 * on the first half
684 */
685 if (state->start <= end && state->end > end) {
Xiao Guangrong82337672011-04-20 06:44:57 +0000686 prealloc = alloc_extent_state_atomic(prealloc);
687 BUG_ON(!prealloc);
Chris Masond1310b22008-01-24 16:13:08 -0500688 err = split_state(tree, state, prealloc, end + 1);
Jeff Mahoneyc2d904e2011-10-03 23:22:32 -0400689 if (err)
690 extent_io_tree_panic(tree, err);
691
Chris Masond1310b22008-01-24 16:13:08 -0500692 if (wake)
693 wake_up(&state->wq);
Chris Mason42daec22009-09-23 19:51:09 -0400694
Jeff Mahoney6763af82012-03-01 14:56:29 +0100695 clear_state_bit(tree, prealloc, &bits, wake);
Josef Bacik9ed74f22009-09-11 16:12:44 -0400696
Chris Masond1310b22008-01-24 16:13:08 -0500697 prealloc = NULL;
698 goto out;
699 }
Chris Mason42daec22009-09-23 19:51:09 -0400700
Li Zefancdc6a392012-03-12 16:39:48 +0800701 state = clear_state_bit(tree, state, &bits, wake);
Liu Bo04493142012-02-16 18:34:37 +0800702next:
Yan Zheng5c939df2009-05-27 09:16:03 -0400703 if (last_end == (u64)-1)
704 goto out;
705 start = last_end + 1;
Li Zefancdc6a392012-03-12 16:39:48 +0800706 if (start <= end && state && !need_resched())
Liu Bo692e5752012-02-16 18:34:36 +0800707 goto hit_next;
Chris Masond1310b22008-01-24 16:13:08 -0500708 goto search_again;
709
710out:
Chris Masoncad321a2008-12-17 14:51:42 -0500711 spin_unlock(&tree->lock);
Chris Masond1310b22008-01-24 16:13:08 -0500712 if (prealloc)
713 free_extent_state(prealloc);
714
Jeff Mahoney6763af82012-03-01 14:56:29 +0100715 return 0;
Chris Masond1310b22008-01-24 16:13:08 -0500716
717search_again:
718 if (start > end)
719 goto out;
Chris Masoncad321a2008-12-17 14:51:42 -0500720 spin_unlock(&tree->lock);
Chris Masond1310b22008-01-24 16:13:08 -0500721 if (mask & __GFP_WAIT)
722 cond_resched();
723 goto again;
724}
Chris Masond1310b22008-01-24 16:13:08 -0500725
Jeff Mahoney143bede2012-03-01 14:56:26 +0100726static void wait_on_state(struct extent_io_tree *tree,
727 struct extent_state *state)
Christoph Hellwig641f5212008-12-02 06:36:10 -0500728 __releases(tree->lock)
729 __acquires(tree->lock)
Chris Masond1310b22008-01-24 16:13:08 -0500730{
731 DEFINE_WAIT(wait);
732 prepare_to_wait(&state->wq, &wait, TASK_UNINTERRUPTIBLE);
Chris Masoncad321a2008-12-17 14:51:42 -0500733 spin_unlock(&tree->lock);
Chris Masond1310b22008-01-24 16:13:08 -0500734 schedule();
Chris Masoncad321a2008-12-17 14:51:42 -0500735 spin_lock(&tree->lock);
Chris Masond1310b22008-01-24 16:13:08 -0500736 finish_wait(&state->wq, &wait);
Chris Masond1310b22008-01-24 16:13:08 -0500737}
738
739/*
740 * waits for one or more bits to clear on a range in the state tree.
741 * The range [start, end] is inclusive.
742 * The tree lock is taken by this function
743 */
David Sterba41074882013-04-29 13:38:46 +0000744static void wait_extent_bit(struct extent_io_tree *tree, u64 start, u64 end,
745 unsigned long bits)
Chris Masond1310b22008-01-24 16:13:08 -0500746{
747 struct extent_state *state;
748 struct rb_node *node;
749
Josef Bacika5dee372013-12-13 10:02:44 -0500750 btrfs_debug_check_extent_io_range(tree, start, end);
David Sterba8d599ae2013-04-30 15:22:23 +0000751
Chris Masoncad321a2008-12-17 14:51:42 -0500752 spin_lock(&tree->lock);
Chris Masond1310b22008-01-24 16:13:08 -0500753again:
754 while (1) {
755 /*
756 * this search will find all the extents that end after
757 * our range starts
758 */
Chris Mason80ea96b2008-02-01 14:51:59 -0500759 node = tree_search(tree, start);
Filipe Mananac50d3e72014-03-31 14:53:25 +0100760process_node:
Chris Masond1310b22008-01-24 16:13:08 -0500761 if (!node)
762 break;
763
764 state = rb_entry(node, struct extent_state, rb_node);
765
766 if (state->start > end)
767 goto out;
768
769 if (state->state & bits) {
770 start = state->start;
771 atomic_inc(&state->refs);
772 wait_on_state(tree, state);
773 free_extent_state(state);
774 goto again;
775 }
776 start = state->end + 1;
777
778 if (start > end)
779 break;
780
Filipe Mananac50d3e72014-03-31 14:53:25 +0100781 if (!cond_resched_lock(&tree->lock)) {
782 node = rb_next(node);
783 goto process_node;
784 }
Chris Masond1310b22008-01-24 16:13:08 -0500785 }
786out:
Chris Masoncad321a2008-12-17 14:51:42 -0500787 spin_unlock(&tree->lock);
Chris Masond1310b22008-01-24 16:13:08 -0500788}
Chris Masond1310b22008-01-24 16:13:08 -0500789
Jeff Mahoney1bf85042011-07-21 16:56:09 +0000790static void set_state_bits(struct extent_io_tree *tree,
Chris Masond1310b22008-01-24 16:13:08 -0500791 struct extent_state *state,
David Sterba9ee49a042015-01-14 19:52:13 +0100792 unsigned *bits)
Chris Masond1310b22008-01-24 16:13:08 -0500793{
David Sterba9ee49a042015-01-14 19:52:13 +0100794 unsigned bits_to_set = *bits & ~EXTENT_CTLBITS;
Josef Bacik9ed74f22009-09-11 16:12:44 -0400795
Jeff Mahoney1bf85042011-07-21 16:56:09 +0000796 set_state_cb(tree, state, bits);
Yan, Zheng0ca1f7c2010-05-16 10:48:47 -0400797 if ((bits_to_set & EXTENT_DIRTY) && !(state->state & EXTENT_DIRTY)) {
Chris Masond1310b22008-01-24 16:13:08 -0500798 u64 range = state->end - state->start + 1;
799 tree->dirty_bytes += range;
800 }
Yan, Zheng0ca1f7c2010-05-16 10:48:47 -0400801 state->state |= bits_to_set;
Chris Masond1310b22008-01-24 16:13:08 -0500802}
803
Filipe Mananae38e2ed2014-10-13 12:28:38 +0100804static void cache_state_if_flags(struct extent_state *state,
805 struct extent_state **cached_ptr,
David Sterba9ee49a042015-01-14 19:52:13 +0100806 unsigned flags)
Chris Mason2c64c532009-09-02 15:04:12 -0400807{
808 if (cached_ptr && !(*cached_ptr)) {
Filipe Mananae38e2ed2014-10-13 12:28:38 +0100809 if (!flags || (state->state & flags)) {
Chris Mason2c64c532009-09-02 15:04:12 -0400810 *cached_ptr = state;
811 atomic_inc(&state->refs);
812 }
813 }
814}
815
Filipe Mananae38e2ed2014-10-13 12:28:38 +0100816static void cache_state(struct extent_state *state,
817 struct extent_state **cached_ptr)
818{
819 return cache_state_if_flags(state, cached_ptr,
820 EXTENT_IOBITS | EXTENT_BOUNDARY);
821}
822
Chris Masond1310b22008-01-24 16:13:08 -0500823/*
Chris Mason1edbb732009-09-02 13:24:36 -0400824 * set some bits on a range in the tree. This may require allocations or
825 * sleeping, so the gfp mask is used to indicate what is allowed.
Chris Masond1310b22008-01-24 16:13:08 -0500826 *
Chris Mason1edbb732009-09-02 13:24:36 -0400827 * If any of the exclusive bits are set, this will fail with -EEXIST if some
828 * part of the range already has the desired bits set. The start of the
829 * existing range is returned in failed_start in this case.
Chris Masond1310b22008-01-24 16:13:08 -0500830 *
Chris Mason1edbb732009-09-02 13:24:36 -0400831 * [start, end] is inclusive This takes the tree lock.
Chris Masond1310b22008-01-24 16:13:08 -0500832 */
Chris Mason1edbb732009-09-02 13:24:36 -0400833
Jeff Mahoney3fbe5c02012-03-01 14:57:19 +0100834static int __must_check
835__set_extent_bit(struct extent_io_tree *tree, u64 start, u64 end,
David Sterba9ee49a042015-01-14 19:52:13 +0100836 unsigned bits, unsigned exclusive_bits,
David Sterba41074882013-04-29 13:38:46 +0000837 u64 *failed_start, struct extent_state **cached_state,
838 gfp_t mask)
Chris Masond1310b22008-01-24 16:13:08 -0500839{
840 struct extent_state *state;
841 struct extent_state *prealloc = NULL;
842 struct rb_node *node;
Filipe David Borba Manana12cfbad2013-11-26 15:41:47 +0000843 struct rb_node **p;
844 struct rb_node *parent;
Chris Masond1310b22008-01-24 16:13:08 -0500845 int err = 0;
Chris Masond1310b22008-01-24 16:13:08 -0500846 u64 last_start;
847 u64 last_end;
Chris Mason42daec22009-09-23 19:51:09 -0400848
Josef Bacika5dee372013-12-13 10:02:44 -0500849 btrfs_debug_check_extent_io_range(tree, start, end);
David Sterba8d599ae2013-04-30 15:22:23 +0000850
Yan, Zheng0ca1f7c2010-05-16 10:48:47 -0400851 bits |= EXTENT_FIRST_DELALLOC;
Chris Masond1310b22008-01-24 16:13:08 -0500852again:
853 if (!prealloc && (mask & __GFP_WAIT)) {
854 prealloc = alloc_extent_state(mask);
Xiao Guangrong82337672011-04-20 06:44:57 +0000855 BUG_ON(!prealloc);
Chris Masond1310b22008-01-24 16:13:08 -0500856 }
857
Chris Masoncad321a2008-12-17 14:51:42 -0500858 spin_lock(&tree->lock);
Chris Mason9655d292009-09-02 15:22:30 -0400859 if (cached_state && *cached_state) {
860 state = *cached_state;
Josef Bacikdf98b6e2011-06-20 14:53:48 -0400861 if (state->start <= start && state->end > start &&
Filipe Manana27a35072014-07-06 20:09:59 +0100862 extent_state_in_tree(state)) {
Chris Mason9655d292009-09-02 15:22:30 -0400863 node = &state->rb_node;
864 goto hit_next;
865 }
866 }
Chris Masond1310b22008-01-24 16:13:08 -0500867 /*
868 * this search will find all the extents that end after
869 * our range starts.
870 */
Filipe David Borba Manana12cfbad2013-11-26 15:41:47 +0000871 node = tree_search_for_insert(tree, start, &p, &parent);
Chris Masond1310b22008-01-24 16:13:08 -0500872 if (!node) {
Xiao Guangrong82337672011-04-20 06:44:57 +0000873 prealloc = alloc_extent_state_atomic(prealloc);
874 BUG_ON(!prealloc);
Filipe David Borba Manana12cfbad2013-11-26 15:41:47 +0000875 err = insert_state(tree, prealloc, start, end,
876 &p, &parent, &bits);
Jeff Mahoneyc2d904e2011-10-03 23:22:32 -0400877 if (err)
878 extent_io_tree_panic(tree, err);
879
Filipe David Borba Mananac42ac0b2013-11-26 15:01:34 +0000880 cache_state(prealloc, cached_state);
Chris Masond1310b22008-01-24 16:13:08 -0500881 prealloc = NULL;
Chris Masond1310b22008-01-24 16:13:08 -0500882 goto out;
883 }
Chris Masond1310b22008-01-24 16:13:08 -0500884 state = rb_entry(node, struct extent_state, rb_node);
Chris Mason40431d62009-08-05 12:57:59 -0400885hit_next:
Chris Masond1310b22008-01-24 16:13:08 -0500886 last_start = state->start;
887 last_end = state->end;
888
889 /*
890 * | ---- desired range ---- |
891 * | state |
892 *
893 * Just lock what we found and keep going
894 */
895 if (state->start == start && state->end <= end) {
Chris Mason1edbb732009-09-02 13:24:36 -0400896 if (state->state & exclusive_bits) {
Chris Masond1310b22008-01-24 16:13:08 -0500897 *failed_start = state->start;
898 err = -EEXIST;
899 goto out;
900 }
Chris Mason42daec22009-09-23 19:51:09 -0400901
Jeff Mahoney1bf85042011-07-21 16:56:09 +0000902 set_state_bits(tree, state, &bits);
Chris Mason2c64c532009-09-02 15:04:12 -0400903 cache_state(state, cached_state);
Chris Masond1310b22008-01-24 16:13:08 -0500904 merge_state(tree, state);
Yan Zheng5c939df2009-05-27 09:16:03 -0400905 if (last_end == (u64)-1)
906 goto out;
907 start = last_end + 1;
Liu Bod1ac6e42012-05-10 18:10:39 +0800908 state = next_state(state);
909 if (start < end && state && state->start == start &&
910 !need_resched())
911 goto hit_next;
Chris Masond1310b22008-01-24 16:13:08 -0500912 goto search_again;
913 }
914
915 /*
916 * | ---- desired range ---- |
917 * | state |
918 * or
919 * | ------------- state -------------- |
920 *
921 * We need to split the extent we found, and may flip bits on
922 * second half.
923 *
924 * If the extent we found extends past our
925 * range, we just split and search again. It'll get split
926 * again the next time though.
927 *
928 * If the extent we found is inside our range, we set the
929 * desired bit on it.
930 */
931 if (state->start < start) {
Chris Mason1edbb732009-09-02 13:24:36 -0400932 if (state->state & exclusive_bits) {
Chris Masond1310b22008-01-24 16:13:08 -0500933 *failed_start = start;
934 err = -EEXIST;
935 goto out;
936 }
Xiao Guangrong82337672011-04-20 06:44:57 +0000937
938 prealloc = alloc_extent_state_atomic(prealloc);
939 BUG_ON(!prealloc);
Chris Masond1310b22008-01-24 16:13:08 -0500940 err = split_state(tree, state, prealloc, start);
Jeff Mahoneyc2d904e2011-10-03 23:22:32 -0400941 if (err)
942 extent_io_tree_panic(tree, err);
943
Chris Masond1310b22008-01-24 16:13:08 -0500944 prealloc = NULL;
945 if (err)
946 goto out;
947 if (state->end <= end) {
Jeff Mahoney1bf85042011-07-21 16:56:09 +0000948 set_state_bits(tree, state, &bits);
Chris Mason2c64c532009-09-02 15:04:12 -0400949 cache_state(state, cached_state);
Chris Masond1310b22008-01-24 16:13:08 -0500950 merge_state(tree, state);
Yan Zheng5c939df2009-05-27 09:16:03 -0400951 if (last_end == (u64)-1)
952 goto out;
953 start = last_end + 1;
Liu Bod1ac6e42012-05-10 18:10:39 +0800954 state = next_state(state);
955 if (start < end && state && state->start == start &&
956 !need_resched())
957 goto hit_next;
Chris Masond1310b22008-01-24 16:13:08 -0500958 }
959 goto search_again;
960 }
961 /*
962 * | ---- desired range ---- |
963 * | state | or | state |
964 *
965 * There's a hole, we need to insert something in it and
966 * ignore the extent we found.
967 */
968 if (state->start > start) {
969 u64 this_end;
970 if (end < last_start)
971 this_end = end;
972 else
Chris Masond3977122009-01-05 21:25:51 -0500973 this_end = last_start - 1;
Xiao Guangrong82337672011-04-20 06:44:57 +0000974
975 prealloc = alloc_extent_state_atomic(prealloc);
976 BUG_ON(!prealloc);
Xiao Guangrongc7f895a2011-04-20 06:45:49 +0000977
978 /*
979 * Avoid to free 'prealloc' if it can be merged with
980 * the later extent.
981 */
Chris Masond1310b22008-01-24 16:13:08 -0500982 err = insert_state(tree, prealloc, start, this_end,
Filipe David Borba Manana12cfbad2013-11-26 15:41:47 +0000983 NULL, NULL, &bits);
Jeff Mahoneyc2d904e2011-10-03 23:22:32 -0400984 if (err)
985 extent_io_tree_panic(tree, err);
986
Chris Mason2c64c532009-09-02 15:04:12 -0400987 cache_state(prealloc, cached_state);
Chris Masond1310b22008-01-24 16:13:08 -0500988 prealloc = NULL;
Chris Masond1310b22008-01-24 16:13:08 -0500989 start = this_end + 1;
990 goto search_again;
991 }
992 /*
993 * | ---- desired range ---- |
994 * | state |
995 * We need to split the extent, and set the bit
996 * on the first half
997 */
998 if (state->start <= end && state->end > end) {
Chris Mason1edbb732009-09-02 13:24:36 -0400999 if (state->state & exclusive_bits) {
Chris Masond1310b22008-01-24 16:13:08 -05001000 *failed_start = start;
1001 err = -EEXIST;
1002 goto out;
1003 }
Xiao Guangrong82337672011-04-20 06:44:57 +00001004
1005 prealloc = alloc_extent_state_atomic(prealloc);
1006 BUG_ON(!prealloc);
Chris Masond1310b22008-01-24 16:13:08 -05001007 err = split_state(tree, state, prealloc, end + 1);
Jeff Mahoneyc2d904e2011-10-03 23:22:32 -04001008 if (err)
1009 extent_io_tree_panic(tree, err);
Chris Masond1310b22008-01-24 16:13:08 -05001010
Jeff Mahoney1bf85042011-07-21 16:56:09 +00001011 set_state_bits(tree, prealloc, &bits);
Chris Mason2c64c532009-09-02 15:04:12 -04001012 cache_state(prealloc, cached_state);
Chris Masond1310b22008-01-24 16:13:08 -05001013 merge_state(tree, prealloc);
1014 prealloc = NULL;
1015 goto out;
1016 }
1017
1018 goto search_again;
1019
1020out:
Chris Masoncad321a2008-12-17 14:51:42 -05001021 spin_unlock(&tree->lock);
Chris Masond1310b22008-01-24 16:13:08 -05001022 if (prealloc)
1023 free_extent_state(prealloc);
1024
1025 return err;
1026
1027search_again:
1028 if (start > end)
1029 goto out;
Chris Masoncad321a2008-12-17 14:51:42 -05001030 spin_unlock(&tree->lock);
Chris Masond1310b22008-01-24 16:13:08 -05001031 if (mask & __GFP_WAIT)
1032 cond_resched();
1033 goto again;
1034}
Chris Masond1310b22008-01-24 16:13:08 -05001035
David Sterba41074882013-04-29 13:38:46 +00001036int set_extent_bit(struct extent_io_tree *tree, u64 start, u64 end,
David Sterba9ee49a042015-01-14 19:52:13 +01001037 unsigned bits, u64 * failed_start,
David Sterba41074882013-04-29 13:38:46 +00001038 struct extent_state **cached_state, gfp_t mask)
Jeff Mahoney3fbe5c02012-03-01 14:57:19 +01001039{
1040 return __set_extent_bit(tree, start, end, bits, 0, failed_start,
1041 cached_state, mask);
1042}
1043
1044
Josef Bacik462d6fa2011-09-26 13:56:12 -04001045/**
Liu Bo10983f22012-07-11 15:26:19 +08001046 * convert_extent_bit - convert all bits in a given range from one bit to
1047 * another
Josef Bacik462d6fa2011-09-26 13:56:12 -04001048 * @tree: the io tree to search
1049 * @start: the start offset in bytes
1050 * @end: the end offset in bytes (inclusive)
1051 * @bits: the bits to set in this range
1052 * @clear_bits: the bits to clear in this range
Josef Bacike6138872012-09-27 17:07:30 -04001053 * @cached_state: state that we're going to cache
Josef Bacik462d6fa2011-09-26 13:56:12 -04001054 * @mask: the allocation mask
1055 *
1056 * This will go through and set bits for the given range. If any states exist
1057 * already in this range they are set with the given bit and cleared of the
1058 * clear_bits. This is only meant to be used by things that are mergeable, ie
1059 * converting from say DELALLOC to DIRTY. This is not meant to be used with
1060 * boundary bits like LOCK.
1061 */
1062int convert_extent_bit(struct extent_io_tree *tree, u64 start, u64 end,
David Sterba9ee49a042015-01-14 19:52:13 +01001063 unsigned bits, unsigned clear_bits,
Josef Bacike6138872012-09-27 17:07:30 -04001064 struct extent_state **cached_state, gfp_t mask)
Josef Bacik462d6fa2011-09-26 13:56:12 -04001065{
1066 struct extent_state *state;
1067 struct extent_state *prealloc = NULL;
1068 struct rb_node *node;
Filipe David Borba Manana12cfbad2013-11-26 15:41:47 +00001069 struct rb_node **p;
1070 struct rb_node *parent;
Josef Bacik462d6fa2011-09-26 13:56:12 -04001071 int err = 0;
1072 u64 last_start;
1073 u64 last_end;
Filipe Mananac8fd3de2014-10-13 12:28:39 +01001074 bool first_iteration = true;
Josef Bacik462d6fa2011-09-26 13:56:12 -04001075
Josef Bacika5dee372013-12-13 10:02:44 -05001076 btrfs_debug_check_extent_io_range(tree, start, end);
David Sterba8d599ae2013-04-30 15:22:23 +00001077
Josef Bacik462d6fa2011-09-26 13:56:12 -04001078again:
1079 if (!prealloc && (mask & __GFP_WAIT)) {
Filipe Mananac8fd3de2014-10-13 12:28:39 +01001080 /*
1081 * Best effort, don't worry if extent state allocation fails
1082 * here for the first iteration. We might have a cached state
1083 * that matches exactly the target range, in which case no
1084 * extent state allocations are needed. We'll only know this
1085 * after locking the tree.
1086 */
Josef Bacik462d6fa2011-09-26 13:56:12 -04001087 prealloc = alloc_extent_state(mask);
Filipe Mananac8fd3de2014-10-13 12:28:39 +01001088 if (!prealloc && !first_iteration)
Josef Bacik462d6fa2011-09-26 13:56:12 -04001089 return -ENOMEM;
1090 }
1091
1092 spin_lock(&tree->lock);
Josef Bacike6138872012-09-27 17:07:30 -04001093 if (cached_state && *cached_state) {
1094 state = *cached_state;
1095 if (state->start <= start && state->end > start &&
Filipe Manana27a35072014-07-06 20:09:59 +01001096 extent_state_in_tree(state)) {
Josef Bacike6138872012-09-27 17:07:30 -04001097 node = &state->rb_node;
1098 goto hit_next;
1099 }
1100 }
1101
Josef Bacik462d6fa2011-09-26 13:56:12 -04001102 /*
1103 * this search will find all the extents that end after
1104 * our range starts.
1105 */
Filipe David Borba Manana12cfbad2013-11-26 15:41:47 +00001106 node = tree_search_for_insert(tree, start, &p, &parent);
Josef Bacik462d6fa2011-09-26 13:56:12 -04001107 if (!node) {
1108 prealloc = alloc_extent_state_atomic(prealloc);
Liu Bo1cf4ffd2011-12-07 20:08:40 -05001109 if (!prealloc) {
1110 err = -ENOMEM;
1111 goto out;
1112 }
Filipe David Borba Manana12cfbad2013-11-26 15:41:47 +00001113 err = insert_state(tree, prealloc, start, end,
1114 &p, &parent, &bits);
Jeff Mahoneyc2d904e2011-10-03 23:22:32 -04001115 if (err)
1116 extent_io_tree_panic(tree, err);
Filipe David Borba Mananac42ac0b2013-11-26 15:01:34 +00001117 cache_state(prealloc, cached_state);
1118 prealloc = NULL;
Josef Bacik462d6fa2011-09-26 13:56:12 -04001119 goto out;
1120 }
1121 state = rb_entry(node, struct extent_state, rb_node);
1122hit_next:
1123 last_start = state->start;
1124 last_end = state->end;
1125
1126 /*
1127 * | ---- desired range ---- |
1128 * | state |
1129 *
1130 * Just lock what we found and keep going
1131 */
1132 if (state->start == start && state->end <= end) {
Josef Bacik462d6fa2011-09-26 13:56:12 -04001133 set_state_bits(tree, state, &bits);
Josef Bacike6138872012-09-27 17:07:30 -04001134 cache_state(state, cached_state);
Liu Bod1ac6e42012-05-10 18:10:39 +08001135 state = clear_state_bit(tree, state, &clear_bits, 0);
Josef Bacik462d6fa2011-09-26 13:56:12 -04001136 if (last_end == (u64)-1)
1137 goto out;
Josef Bacik462d6fa2011-09-26 13:56:12 -04001138 start = last_end + 1;
Liu Bod1ac6e42012-05-10 18:10:39 +08001139 if (start < end && state && state->start == start &&
1140 !need_resched())
1141 goto hit_next;
Josef Bacik462d6fa2011-09-26 13:56:12 -04001142 goto search_again;
1143 }
1144
1145 /*
1146 * | ---- desired range ---- |
1147 * | state |
1148 * or
1149 * | ------------- state -------------- |
1150 *
1151 * We need to split the extent we found, and may flip bits on
1152 * second half.
1153 *
1154 * If the extent we found extends past our
1155 * range, we just split and search again. It'll get split
1156 * again the next time though.
1157 *
1158 * If the extent we found is inside our range, we set the
1159 * desired bit on it.
1160 */
1161 if (state->start < start) {
1162 prealloc = alloc_extent_state_atomic(prealloc);
Liu Bo1cf4ffd2011-12-07 20:08:40 -05001163 if (!prealloc) {
1164 err = -ENOMEM;
1165 goto out;
1166 }
Josef Bacik462d6fa2011-09-26 13:56:12 -04001167 err = split_state(tree, state, prealloc, start);
Jeff Mahoneyc2d904e2011-10-03 23:22:32 -04001168 if (err)
1169 extent_io_tree_panic(tree, err);
Josef Bacik462d6fa2011-09-26 13:56:12 -04001170 prealloc = NULL;
1171 if (err)
1172 goto out;
1173 if (state->end <= end) {
1174 set_state_bits(tree, state, &bits);
Josef Bacike6138872012-09-27 17:07:30 -04001175 cache_state(state, cached_state);
Liu Bod1ac6e42012-05-10 18:10:39 +08001176 state = clear_state_bit(tree, state, &clear_bits, 0);
Josef Bacik462d6fa2011-09-26 13:56:12 -04001177 if (last_end == (u64)-1)
1178 goto out;
1179 start = last_end + 1;
Liu Bod1ac6e42012-05-10 18:10:39 +08001180 if (start < end && state && state->start == start &&
1181 !need_resched())
1182 goto hit_next;
Josef Bacik462d6fa2011-09-26 13:56:12 -04001183 }
1184 goto search_again;
1185 }
1186 /*
1187 * | ---- desired range ---- |
1188 * | state | or | state |
1189 *
1190 * There's a hole, we need to insert something in it and
1191 * ignore the extent we found.
1192 */
1193 if (state->start > start) {
1194 u64 this_end;
1195 if (end < last_start)
1196 this_end = end;
1197 else
1198 this_end = last_start - 1;
1199
1200 prealloc = alloc_extent_state_atomic(prealloc);
Liu Bo1cf4ffd2011-12-07 20:08:40 -05001201 if (!prealloc) {
1202 err = -ENOMEM;
1203 goto out;
1204 }
Josef Bacik462d6fa2011-09-26 13:56:12 -04001205
1206 /*
1207 * Avoid to free 'prealloc' if it can be merged with
1208 * the later extent.
1209 */
1210 err = insert_state(tree, prealloc, start, this_end,
Filipe David Borba Manana12cfbad2013-11-26 15:41:47 +00001211 NULL, NULL, &bits);
Jeff Mahoneyc2d904e2011-10-03 23:22:32 -04001212 if (err)
1213 extent_io_tree_panic(tree, err);
Josef Bacike6138872012-09-27 17:07:30 -04001214 cache_state(prealloc, cached_state);
Josef Bacik462d6fa2011-09-26 13:56:12 -04001215 prealloc = NULL;
1216 start = this_end + 1;
1217 goto search_again;
1218 }
1219 /*
1220 * | ---- desired range ---- |
1221 * | state |
1222 * We need to split the extent, and set the bit
1223 * on the first half
1224 */
1225 if (state->start <= end && state->end > end) {
1226 prealloc = alloc_extent_state_atomic(prealloc);
Liu Bo1cf4ffd2011-12-07 20:08:40 -05001227 if (!prealloc) {
1228 err = -ENOMEM;
1229 goto out;
1230 }
Josef Bacik462d6fa2011-09-26 13:56:12 -04001231
1232 err = split_state(tree, state, prealloc, end + 1);
Jeff Mahoneyc2d904e2011-10-03 23:22:32 -04001233 if (err)
1234 extent_io_tree_panic(tree, err);
Josef Bacik462d6fa2011-09-26 13:56:12 -04001235
1236 set_state_bits(tree, prealloc, &bits);
Josef Bacike6138872012-09-27 17:07:30 -04001237 cache_state(prealloc, cached_state);
Josef Bacik462d6fa2011-09-26 13:56:12 -04001238 clear_state_bit(tree, prealloc, &clear_bits, 0);
Josef Bacik462d6fa2011-09-26 13:56:12 -04001239 prealloc = NULL;
1240 goto out;
1241 }
1242
1243 goto search_again;
1244
1245out:
1246 spin_unlock(&tree->lock);
1247 if (prealloc)
1248 free_extent_state(prealloc);
1249
1250 return err;
1251
1252search_again:
1253 if (start > end)
1254 goto out;
1255 spin_unlock(&tree->lock);
1256 if (mask & __GFP_WAIT)
1257 cond_resched();
Filipe Mananac8fd3de2014-10-13 12:28:39 +01001258 first_iteration = false;
Josef Bacik462d6fa2011-09-26 13:56:12 -04001259 goto again;
1260}
1261
Chris Masond1310b22008-01-24 16:13:08 -05001262/* wrappers around set/clear extent bit */
1263int set_extent_dirty(struct extent_io_tree *tree, u64 start, u64 end,
1264 gfp_t mask)
1265{
Jeff Mahoney3fbe5c02012-03-01 14:57:19 +01001266 return set_extent_bit(tree, start, end, EXTENT_DIRTY, NULL,
Chris Mason2c64c532009-09-02 15:04:12 -04001267 NULL, mask);
Chris Masond1310b22008-01-24 16:13:08 -05001268}
Chris Masond1310b22008-01-24 16:13:08 -05001269
1270int set_extent_bits(struct extent_io_tree *tree, u64 start, u64 end,
David Sterba9ee49a042015-01-14 19:52:13 +01001271 unsigned bits, gfp_t mask)
Chris Masond1310b22008-01-24 16:13:08 -05001272{
Jeff Mahoney3fbe5c02012-03-01 14:57:19 +01001273 return set_extent_bit(tree, start, end, bits, NULL,
Chris Mason2c64c532009-09-02 15:04:12 -04001274 NULL, mask);
Chris Masond1310b22008-01-24 16:13:08 -05001275}
Chris Masond1310b22008-01-24 16:13:08 -05001276
1277int clear_extent_bits(struct extent_io_tree *tree, u64 start, u64 end,
David Sterba9ee49a042015-01-14 19:52:13 +01001278 unsigned bits, gfp_t mask)
Chris Masond1310b22008-01-24 16:13:08 -05001279{
Filipe Manana0f318712015-05-14 20:41:07 +01001280 int wake = 0;
1281
1282 if (bits & EXTENT_LOCKED)
1283 wake = 1;
1284
1285 return clear_extent_bit(tree, start, end, bits, wake, 0, NULL, mask);
Chris Masond1310b22008-01-24 16:13:08 -05001286}
Chris Masond1310b22008-01-24 16:13:08 -05001287
1288int set_extent_delalloc(struct extent_io_tree *tree, u64 start, u64 end,
Josef Bacik2ac55d42010-02-03 19:33:23 +00001289 struct extent_state **cached_state, gfp_t mask)
Chris Masond1310b22008-01-24 16:13:08 -05001290{
1291 return set_extent_bit(tree, start, end,
Liu Bofee187d2011-09-29 15:55:28 +08001292 EXTENT_DELALLOC | EXTENT_UPTODATE,
Jeff Mahoney3fbe5c02012-03-01 14:57:19 +01001293 NULL, cached_state, mask);
Chris Masond1310b22008-01-24 16:13:08 -05001294}
Chris Masond1310b22008-01-24 16:13:08 -05001295
Liu Bo9e8a4a82012-09-05 19:10:51 -06001296int set_extent_defrag(struct extent_io_tree *tree, u64 start, u64 end,
1297 struct extent_state **cached_state, gfp_t mask)
1298{
1299 return set_extent_bit(tree, start, end,
1300 EXTENT_DELALLOC | EXTENT_UPTODATE | EXTENT_DEFRAG,
1301 NULL, cached_state, mask);
1302}
1303
Chris Masond1310b22008-01-24 16:13:08 -05001304int clear_extent_dirty(struct extent_io_tree *tree, u64 start, u64 end,
1305 gfp_t mask)
1306{
1307 return clear_extent_bit(tree, start, end,
Josef Bacik32c00af2009-10-08 13:34:05 -04001308 EXTENT_DIRTY | EXTENT_DELALLOC |
Yan, Zheng0ca1f7c2010-05-16 10:48:47 -04001309 EXTENT_DO_ACCOUNTING, 0, 0, NULL, mask);
Chris Masond1310b22008-01-24 16:13:08 -05001310}
Chris Masond1310b22008-01-24 16:13:08 -05001311
1312int set_extent_new(struct extent_io_tree *tree, u64 start, u64 end,
1313 gfp_t mask)
1314{
Jeff Mahoney3fbe5c02012-03-01 14:57:19 +01001315 return set_extent_bit(tree, start, end, EXTENT_NEW, NULL,
Chris Mason2c64c532009-09-02 15:04:12 -04001316 NULL, mask);
Chris Masond1310b22008-01-24 16:13:08 -05001317}
Chris Masond1310b22008-01-24 16:13:08 -05001318
Chris Masond1310b22008-01-24 16:13:08 -05001319int set_extent_uptodate(struct extent_io_tree *tree, u64 start, u64 end,
Arne Jansen507903b2011-04-06 10:02:20 +00001320 struct extent_state **cached_state, gfp_t mask)
Chris Masond1310b22008-01-24 16:13:08 -05001321{
Liu Bo6b67a322013-03-28 08:30:28 +00001322 return set_extent_bit(tree, start, end, EXTENT_UPTODATE, NULL,
Jeff Mahoney3fbe5c02012-03-01 14:57:19 +01001323 cached_state, mask);
Chris Masond1310b22008-01-24 16:13:08 -05001324}
Chris Masond1310b22008-01-24 16:13:08 -05001325
Josef Bacik5fd02042012-05-02 14:00:54 -04001326int clear_extent_uptodate(struct extent_io_tree *tree, u64 start, u64 end,
1327 struct extent_state **cached_state, gfp_t mask)
Chris Masond1310b22008-01-24 16:13:08 -05001328{
Chris Mason2c64c532009-09-02 15:04:12 -04001329 return clear_extent_bit(tree, start, end, EXTENT_UPTODATE, 0, 0,
Josef Bacik2ac55d42010-02-03 19:33:23 +00001330 cached_state, mask);
Chris Masond1310b22008-01-24 16:13:08 -05001331}
Chris Masond1310b22008-01-24 16:13:08 -05001332
Chris Masond352ac62008-09-29 15:18:18 -04001333/*
1334 * either insert or lock state struct between start and end use mask to tell
1335 * us if waiting is desired.
1336 */
Chris Mason1edbb732009-09-02 13:24:36 -04001337int lock_extent_bits(struct extent_io_tree *tree, u64 start, u64 end,
David Sterba9ee49a042015-01-14 19:52:13 +01001338 unsigned bits, struct extent_state **cached_state)
Chris Masond1310b22008-01-24 16:13:08 -05001339{
1340 int err;
1341 u64 failed_start;
David Sterba9ee49a042015-01-14 19:52:13 +01001342
Chris Masond1310b22008-01-24 16:13:08 -05001343 while (1) {
Jeff Mahoney3fbe5c02012-03-01 14:57:19 +01001344 err = __set_extent_bit(tree, start, end, EXTENT_LOCKED | bits,
1345 EXTENT_LOCKED, &failed_start,
1346 cached_state, GFP_NOFS);
Jeff Mahoneyd0082372012-03-01 14:57:19 +01001347 if (err == -EEXIST) {
Chris Masond1310b22008-01-24 16:13:08 -05001348 wait_extent_bit(tree, failed_start, end, EXTENT_LOCKED);
1349 start = failed_start;
Jeff Mahoneyd0082372012-03-01 14:57:19 +01001350 } else
Chris Masond1310b22008-01-24 16:13:08 -05001351 break;
Chris Masond1310b22008-01-24 16:13:08 -05001352 WARN_ON(start > end);
1353 }
1354 return err;
1355}
Chris Masond1310b22008-01-24 16:13:08 -05001356
Jeff Mahoneyd0082372012-03-01 14:57:19 +01001357int lock_extent(struct extent_io_tree *tree, u64 start, u64 end)
Chris Mason1edbb732009-09-02 13:24:36 -04001358{
Jeff Mahoneyd0082372012-03-01 14:57:19 +01001359 return lock_extent_bits(tree, start, end, 0, NULL);
Chris Mason1edbb732009-09-02 13:24:36 -04001360}
1361
Jeff Mahoneyd0082372012-03-01 14:57:19 +01001362int try_lock_extent(struct extent_io_tree *tree, u64 start, u64 end)
Josef Bacik25179202008-10-29 14:49:05 -04001363{
1364 int err;
1365 u64 failed_start;
1366
Jeff Mahoney3fbe5c02012-03-01 14:57:19 +01001367 err = __set_extent_bit(tree, start, end, EXTENT_LOCKED, EXTENT_LOCKED,
1368 &failed_start, NULL, GFP_NOFS);
Yan Zheng66435582008-10-30 14:19:50 -04001369 if (err == -EEXIST) {
1370 if (failed_start > start)
1371 clear_extent_bit(tree, start, failed_start - 1,
Jeff Mahoneyd0082372012-03-01 14:57:19 +01001372 EXTENT_LOCKED, 1, 0, NULL, GFP_NOFS);
Josef Bacik25179202008-10-29 14:49:05 -04001373 return 0;
Yan Zheng66435582008-10-30 14:19:50 -04001374 }
Josef Bacik25179202008-10-29 14:49:05 -04001375 return 1;
1376}
Josef Bacik25179202008-10-29 14:49:05 -04001377
Chris Mason2c64c532009-09-02 15:04:12 -04001378int unlock_extent_cached(struct extent_io_tree *tree, u64 start, u64 end,
1379 struct extent_state **cached, gfp_t mask)
1380{
1381 return clear_extent_bit(tree, start, end, EXTENT_LOCKED, 1, 0, cached,
1382 mask);
1383}
1384
Jeff Mahoneyd0082372012-03-01 14:57:19 +01001385int unlock_extent(struct extent_io_tree *tree, u64 start, u64 end)
Chris Masond1310b22008-01-24 16:13:08 -05001386{
Chris Mason2c64c532009-09-02 15:04:12 -04001387 return clear_extent_bit(tree, start, end, EXTENT_LOCKED, 1, 0, NULL,
Jeff Mahoneyd0082372012-03-01 14:57:19 +01001388 GFP_NOFS);
Chris Masond1310b22008-01-24 16:13:08 -05001389}
Chris Masond1310b22008-01-24 16:13:08 -05001390
Chris Mason4adaa612013-03-26 13:07:00 -04001391int extent_range_clear_dirty_for_io(struct inode *inode, u64 start, u64 end)
1392{
1393 unsigned long index = start >> PAGE_CACHE_SHIFT;
1394 unsigned long end_index = end >> PAGE_CACHE_SHIFT;
1395 struct page *page;
1396
1397 while (index <= end_index) {
1398 page = find_get_page(inode->i_mapping, index);
1399 BUG_ON(!page); /* Pages should be in the extent_io_tree */
1400 clear_page_dirty_for_io(page);
1401 page_cache_release(page);
1402 index++;
1403 }
1404 return 0;
1405}
1406
1407int extent_range_redirty_for_io(struct inode *inode, u64 start, u64 end)
1408{
1409 unsigned long index = start >> PAGE_CACHE_SHIFT;
1410 unsigned long end_index = end >> PAGE_CACHE_SHIFT;
1411 struct page *page;
1412
1413 while (index <= end_index) {
1414 page = find_get_page(inode->i_mapping, index);
1415 BUG_ON(!page); /* Pages should be in the extent_io_tree */
Chris Mason4adaa612013-03-26 13:07:00 -04001416 __set_page_dirty_nobuffers(page);
Konstantin Khebnikov8d386332015-02-11 15:26:55 -08001417 account_page_redirty(page);
Chris Mason4adaa612013-03-26 13:07:00 -04001418 page_cache_release(page);
1419 index++;
1420 }
1421 return 0;
1422}
1423
Chris Masond1310b22008-01-24 16:13:08 -05001424/*
Chris Masond1310b22008-01-24 16:13:08 -05001425 * helper function to set both pages and extents in the tree writeback
1426 */
Christoph Hellwigb2950862008-12-02 09:54:17 -05001427static int set_range_writeback(struct extent_io_tree *tree, u64 start, u64 end)
Chris Masond1310b22008-01-24 16:13:08 -05001428{
1429 unsigned long index = start >> PAGE_CACHE_SHIFT;
1430 unsigned long end_index = end >> PAGE_CACHE_SHIFT;
1431 struct page *page;
1432
1433 while (index <= end_index) {
1434 page = find_get_page(tree->mapping, index);
Jeff Mahoney79787ea2012-03-12 16:03:00 +01001435 BUG_ON(!page); /* Pages should be in the extent_io_tree */
Chris Masond1310b22008-01-24 16:13:08 -05001436 set_page_writeback(page);
1437 page_cache_release(page);
1438 index++;
1439 }
Chris Masond1310b22008-01-24 16:13:08 -05001440 return 0;
1441}
Chris Masond1310b22008-01-24 16:13:08 -05001442
Chris Masond352ac62008-09-29 15:18:18 -04001443/* find the first state struct with 'bits' set after 'start', and
1444 * return it. tree->lock must be held. NULL will returned if
1445 * nothing was found after 'start'
1446 */
Eric Sandeen48a3b632013-04-25 20:41:01 +00001447static struct extent_state *
1448find_first_extent_bit_state(struct extent_io_tree *tree,
David Sterba9ee49a042015-01-14 19:52:13 +01001449 u64 start, unsigned bits)
Chris Masond7fc6402008-02-18 12:12:38 -05001450{
1451 struct rb_node *node;
1452 struct extent_state *state;
1453
1454 /*
1455 * this search will find all the extents that end after
1456 * our range starts.
1457 */
1458 node = tree_search(tree, start);
Chris Masond3977122009-01-05 21:25:51 -05001459 if (!node)
Chris Masond7fc6402008-02-18 12:12:38 -05001460 goto out;
Chris Masond7fc6402008-02-18 12:12:38 -05001461
Chris Masond3977122009-01-05 21:25:51 -05001462 while (1) {
Chris Masond7fc6402008-02-18 12:12:38 -05001463 state = rb_entry(node, struct extent_state, rb_node);
Chris Masond3977122009-01-05 21:25:51 -05001464 if (state->end >= start && (state->state & bits))
Chris Masond7fc6402008-02-18 12:12:38 -05001465 return state;
Chris Masond3977122009-01-05 21:25:51 -05001466
Chris Masond7fc6402008-02-18 12:12:38 -05001467 node = rb_next(node);
1468 if (!node)
1469 break;
1470 }
1471out:
1472 return NULL;
1473}
Chris Masond7fc6402008-02-18 12:12:38 -05001474
Chris Masond352ac62008-09-29 15:18:18 -04001475/*
Xiao Guangrong69261c42011-07-14 03:19:45 +00001476 * find the first offset in the io tree with 'bits' set. zero is
1477 * returned if we find something, and *start_ret and *end_ret are
1478 * set to reflect the state struct that was found.
1479 *
Wang Sheng-Hui477d7ea2012-04-06 14:35:47 +08001480 * If nothing was found, 1 is returned. If found something, return 0.
Xiao Guangrong69261c42011-07-14 03:19:45 +00001481 */
1482int find_first_extent_bit(struct extent_io_tree *tree, u64 start,
David Sterba9ee49a042015-01-14 19:52:13 +01001483 u64 *start_ret, u64 *end_ret, unsigned bits,
Josef Bacike6138872012-09-27 17:07:30 -04001484 struct extent_state **cached_state)
Xiao Guangrong69261c42011-07-14 03:19:45 +00001485{
1486 struct extent_state *state;
Josef Bacike6138872012-09-27 17:07:30 -04001487 struct rb_node *n;
Xiao Guangrong69261c42011-07-14 03:19:45 +00001488 int ret = 1;
1489
1490 spin_lock(&tree->lock);
Josef Bacike6138872012-09-27 17:07:30 -04001491 if (cached_state && *cached_state) {
1492 state = *cached_state;
Filipe Manana27a35072014-07-06 20:09:59 +01001493 if (state->end == start - 1 && extent_state_in_tree(state)) {
Josef Bacike6138872012-09-27 17:07:30 -04001494 n = rb_next(&state->rb_node);
1495 while (n) {
1496 state = rb_entry(n, struct extent_state,
1497 rb_node);
1498 if (state->state & bits)
1499 goto got_it;
1500 n = rb_next(n);
1501 }
1502 free_extent_state(*cached_state);
1503 *cached_state = NULL;
1504 goto out;
1505 }
1506 free_extent_state(*cached_state);
1507 *cached_state = NULL;
1508 }
1509
Xiao Guangrong69261c42011-07-14 03:19:45 +00001510 state = find_first_extent_bit_state(tree, start, bits);
Josef Bacike6138872012-09-27 17:07:30 -04001511got_it:
Xiao Guangrong69261c42011-07-14 03:19:45 +00001512 if (state) {
Filipe Mananae38e2ed2014-10-13 12:28:38 +01001513 cache_state_if_flags(state, cached_state, 0);
Xiao Guangrong69261c42011-07-14 03:19:45 +00001514 *start_ret = state->start;
1515 *end_ret = state->end;
1516 ret = 0;
1517 }
Josef Bacike6138872012-09-27 17:07:30 -04001518out:
Xiao Guangrong69261c42011-07-14 03:19:45 +00001519 spin_unlock(&tree->lock);
1520 return ret;
1521}
1522
1523/*
Chris Masond352ac62008-09-29 15:18:18 -04001524 * find a contiguous range of bytes in the file marked as delalloc, not
1525 * more than 'max_bytes'. start and end are used to return the range,
1526 *
1527 * 1 is returned if we find something, 0 if nothing was in the tree
1528 */
Chris Masonc8b97812008-10-29 14:49:59 -04001529static noinline u64 find_delalloc_range(struct extent_io_tree *tree,
Josef Bacikc2a128d2010-02-02 21:19:11 +00001530 u64 *start, u64 *end, u64 max_bytes,
1531 struct extent_state **cached_state)
Chris Masond1310b22008-01-24 16:13:08 -05001532{
1533 struct rb_node *node;
1534 struct extent_state *state;
1535 u64 cur_start = *start;
1536 u64 found = 0;
1537 u64 total_bytes = 0;
1538
Chris Masoncad321a2008-12-17 14:51:42 -05001539 spin_lock(&tree->lock);
Chris Masonc8b97812008-10-29 14:49:59 -04001540
Chris Masond1310b22008-01-24 16:13:08 -05001541 /*
1542 * this search will find all the extents that end after
1543 * our range starts.
1544 */
Chris Mason80ea96b2008-02-01 14:51:59 -05001545 node = tree_search(tree, cur_start);
Peter2b114d12008-04-01 11:21:40 -04001546 if (!node) {
Chris Mason3b951512008-04-17 11:29:12 -04001547 if (!found)
1548 *end = (u64)-1;
Chris Masond1310b22008-01-24 16:13:08 -05001549 goto out;
1550 }
1551
Chris Masond3977122009-01-05 21:25:51 -05001552 while (1) {
Chris Masond1310b22008-01-24 16:13:08 -05001553 state = rb_entry(node, struct extent_state, rb_node);
Zheng Yan5b21f2e2008-09-26 10:05:38 -04001554 if (found && (state->start != cur_start ||
1555 (state->state & EXTENT_BOUNDARY))) {
Chris Masond1310b22008-01-24 16:13:08 -05001556 goto out;
1557 }
1558 if (!(state->state & EXTENT_DELALLOC)) {
1559 if (!found)
1560 *end = state->end;
1561 goto out;
1562 }
Josef Bacikc2a128d2010-02-02 21:19:11 +00001563 if (!found) {
Chris Masond1310b22008-01-24 16:13:08 -05001564 *start = state->start;
Josef Bacikc2a128d2010-02-02 21:19:11 +00001565 *cached_state = state;
1566 atomic_inc(&state->refs);
1567 }
Chris Masond1310b22008-01-24 16:13:08 -05001568 found++;
1569 *end = state->end;
1570 cur_start = state->end + 1;
1571 node = rb_next(node);
Chris Masond1310b22008-01-24 16:13:08 -05001572 total_bytes += state->end - state->start + 1;
Josef Bacik7bf811a52013-10-07 22:11:09 -04001573 if (total_bytes >= max_bytes)
Josef Bacik573aeca2013-08-30 14:38:49 -04001574 break;
Josef Bacik573aeca2013-08-30 14:38:49 -04001575 if (!node)
Chris Masond1310b22008-01-24 16:13:08 -05001576 break;
1577 }
1578out:
Chris Masoncad321a2008-12-17 14:51:42 -05001579 spin_unlock(&tree->lock);
Chris Masond1310b22008-01-24 16:13:08 -05001580 return found;
1581}
1582
Jeff Mahoney143bede2012-03-01 14:56:26 +01001583static noinline void __unlock_for_delalloc(struct inode *inode,
1584 struct page *locked_page,
1585 u64 start, u64 end)
Chris Masonc8b97812008-10-29 14:49:59 -04001586{
1587 int ret;
1588 struct page *pages[16];
1589 unsigned long index = start >> PAGE_CACHE_SHIFT;
1590 unsigned long end_index = end >> PAGE_CACHE_SHIFT;
1591 unsigned long nr_pages = end_index - index + 1;
1592 int i;
1593
1594 if (index == locked_page->index && end_index == index)
Jeff Mahoney143bede2012-03-01 14:56:26 +01001595 return;
Chris Masonc8b97812008-10-29 14:49:59 -04001596
Chris Masond3977122009-01-05 21:25:51 -05001597 while (nr_pages > 0) {
Chris Masonc8b97812008-10-29 14:49:59 -04001598 ret = find_get_pages_contig(inode->i_mapping, index,
Chris Mason5b050f02008-11-11 09:34:41 -05001599 min_t(unsigned long, nr_pages,
1600 ARRAY_SIZE(pages)), pages);
Chris Masonc8b97812008-10-29 14:49:59 -04001601 for (i = 0; i < ret; i++) {
1602 if (pages[i] != locked_page)
1603 unlock_page(pages[i]);
1604 page_cache_release(pages[i]);
1605 }
1606 nr_pages -= ret;
1607 index += ret;
1608 cond_resched();
1609 }
Chris Masonc8b97812008-10-29 14:49:59 -04001610}
1611
1612static noinline int lock_delalloc_pages(struct inode *inode,
1613 struct page *locked_page,
1614 u64 delalloc_start,
1615 u64 delalloc_end)
1616{
1617 unsigned long index = delalloc_start >> PAGE_CACHE_SHIFT;
1618 unsigned long start_index = index;
1619 unsigned long end_index = delalloc_end >> PAGE_CACHE_SHIFT;
1620 unsigned long pages_locked = 0;
1621 struct page *pages[16];
1622 unsigned long nrpages;
1623 int ret;
1624 int i;
1625
1626 /* the caller is responsible for locking the start index */
1627 if (index == locked_page->index && index == end_index)
1628 return 0;
1629
1630 /* skip the page at the start index */
1631 nrpages = end_index - index + 1;
Chris Masond3977122009-01-05 21:25:51 -05001632 while (nrpages > 0) {
Chris Masonc8b97812008-10-29 14:49:59 -04001633 ret = find_get_pages_contig(inode->i_mapping, index,
Chris Mason5b050f02008-11-11 09:34:41 -05001634 min_t(unsigned long,
1635 nrpages, ARRAY_SIZE(pages)), pages);
Chris Masonc8b97812008-10-29 14:49:59 -04001636 if (ret == 0) {
1637 ret = -EAGAIN;
1638 goto done;
1639 }
1640 /* now we have an array of pages, lock them all */
1641 for (i = 0; i < ret; i++) {
1642 /*
1643 * the caller is taking responsibility for
1644 * locked_page
1645 */
Chris Mason771ed682008-11-06 22:02:51 -05001646 if (pages[i] != locked_page) {
Chris Masonc8b97812008-10-29 14:49:59 -04001647 lock_page(pages[i]);
Chris Masonf2b1c412008-11-10 07:31:30 -05001648 if (!PageDirty(pages[i]) ||
1649 pages[i]->mapping != inode->i_mapping) {
Chris Mason771ed682008-11-06 22:02:51 -05001650 ret = -EAGAIN;
1651 unlock_page(pages[i]);
1652 page_cache_release(pages[i]);
1653 goto done;
1654 }
1655 }
Chris Masonc8b97812008-10-29 14:49:59 -04001656 page_cache_release(pages[i]);
Chris Mason771ed682008-11-06 22:02:51 -05001657 pages_locked++;
Chris Masonc8b97812008-10-29 14:49:59 -04001658 }
Chris Masonc8b97812008-10-29 14:49:59 -04001659 nrpages -= ret;
1660 index += ret;
1661 cond_resched();
1662 }
1663 ret = 0;
1664done:
1665 if (ret && pages_locked) {
1666 __unlock_for_delalloc(inode, locked_page,
1667 delalloc_start,
1668 ((u64)(start_index + pages_locked - 1)) <<
1669 PAGE_CACHE_SHIFT);
1670 }
1671 return ret;
1672}
1673
1674/*
1675 * find a contiguous range of bytes in the file marked as delalloc, not
1676 * more than 'max_bytes'. start and end are used to return the range,
1677 *
1678 * 1 is returned if we find something, 0 if nothing was in the tree
1679 */
Josef Bacik294e30f2013-10-09 12:00:56 -04001680STATIC u64 find_lock_delalloc_range(struct inode *inode,
1681 struct extent_io_tree *tree,
1682 struct page *locked_page, u64 *start,
1683 u64 *end, u64 max_bytes)
Chris Masonc8b97812008-10-29 14:49:59 -04001684{
1685 u64 delalloc_start;
1686 u64 delalloc_end;
1687 u64 found;
Chris Mason9655d292009-09-02 15:22:30 -04001688 struct extent_state *cached_state = NULL;
Chris Masonc8b97812008-10-29 14:49:59 -04001689 int ret;
1690 int loops = 0;
1691
1692again:
1693 /* step one, find a bunch of delalloc bytes starting at start */
1694 delalloc_start = *start;
1695 delalloc_end = 0;
1696 found = find_delalloc_range(tree, &delalloc_start, &delalloc_end,
Josef Bacikc2a128d2010-02-02 21:19:11 +00001697 max_bytes, &cached_state);
Chris Mason70b99e62008-10-31 12:46:39 -04001698 if (!found || delalloc_end <= *start) {
Chris Masonc8b97812008-10-29 14:49:59 -04001699 *start = delalloc_start;
1700 *end = delalloc_end;
Josef Bacikc2a128d2010-02-02 21:19:11 +00001701 free_extent_state(cached_state);
Liu Bo385fe0b2013-10-01 23:49:49 +08001702 return 0;
Chris Masonc8b97812008-10-29 14:49:59 -04001703 }
1704
1705 /*
Chris Mason70b99e62008-10-31 12:46:39 -04001706 * start comes from the offset of locked_page. We have to lock
1707 * pages in order, so we can't process delalloc bytes before
1708 * locked_page
1709 */
Chris Masond3977122009-01-05 21:25:51 -05001710 if (delalloc_start < *start)
Chris Mason70b99e62008-10-31 12:46:39 -04001711 delalloc_start = *start;
Chris Mason70b99e62008-10-31 12:46:39 -04001712
1713 /*
Chris Masonc8b97812008-10-29 14:49:59 -04001714 * make sure to limit the number of pages we try to lock down
Chris Masonc8b97812008-10-29 14:49:59 -04001715 */
Josef Bacik7bf811a52013-10-07 22:11:09 -04001716 if (delalloc_end + 1 - delalloc_start > max_bytes)
1717 delalloc_end = delalloc_start + max_bytes - 1;
Chris Masond3977122009-01-05 21:25:51 -05001718
Chris Masonc8b97812008-10-29 14:49:59 -04001719 /* step two, lock all the pages after the page that has start */
1720 ret = lock_delalloc_pages(inode, locked_page,
1721 delalloc_start, delalloc_end);
1722 if (ret == -EAGAIN) {
1723 /* some of the pages are gone, lets avoid looping by
1724 * shortening the size of the delalloc range we're searching
1725 */
Chris Mason9655d292009-09-02 15:22:30 -04001726 free_extent_state(cached_state);
Chris Mason7d788742014-05-21 05:49:54 -07001727 cached_state = NULL;
Chris Masonc8b97812008-10-29 14:49:59 -04001728 if (!loops) {
Josef Bacik7bf811a52013-10-07 22:11:09 -04001729 max_bytes = PAGE_CACHE_SIZE;
Chris Masonc8b97812008-10-29 14:49:59 -04001730 loops = 1;
1731 goto again;
1732 } else {
1733 found = 0;
1734 goto out_failed;
1735 }
1736 }
Jeff Mahoney79787ea2012-03-12 16:03:00 +01001737 BUG_ON(ret); /* Only valid values are 0 and -EAGAIN */
Chris Masonc8b97812008-10-29 14:49:59 -04001738
1739 /* step three, lock the state bits for the whole range */
Jeff Mahoneyd0082372012-03-01 14:57:19 +01001740 lock_extent_bits(tree, delalloc_start, delalloc_end, 0, &cached_state);
Chris Masonc8b97812008-10-29 14:49:59 -04001741
1742 /* then test to make sure it is all still delalloc */
1743 ret = test_range_bit(tree, delalloc_start, delalloc_end,
Chris Mason9655d292009-09-02 15:22:30 -04001744 EXTENT_DELALLOC, 1, cached_state);
Chris Masonc8b97812008-10-29 14:49:59 -04001745 if (!ret) {
Chris Mason9655d292009-09-02 15:22:30 -04001746 unlock_extent_cached(tree, delalloc_start, delalloc_end,
1747 &cached_state, GFP_NOFS);
Chris Masonc8b97812008-10-29 14:49:59 -04001748 __unlock_for_delalloc(inode, locked_page,
1749 delalloc_start, delalloc_end);
1750 cond_resched();
1751 goto again;
1752 }
Chris Mason9655d292009-09-02 15:22:30 -04001753 free_extent_state(cached_state);
Chris Masonc8b97812008-10-29 14:49:59 -04001754 *start = delalloc_start;
1755 *end = delalloc_end;
1756out_failed:
1757 return found;
1758}
1759
Josef Bacikc2790a22013-07-29 11:20:47 -04001760int extent_clear_unlock_delalloc(struct inode *inode, u64 start, u64 end,
1761 struct page *locked_page,
David Sterba9ee49a042015-01-14 19:52:13 +01001762 unsigned clear_bits,
Josef Bacikc2790a22013-07-29 11:20:47 -04001763 unsigned long page_ops)
Chris Masonc8b97812008-10-29 14:49:59 -04001764{
Josef Bacikc2790a22013-07-29 11:20:47 -04001765 struct extent_io_tree *tree = &BTRFS_I(inode)->io_tree;
Chris Masonc8b97812008-10-29 14:49:59 -04001766 int ret;
1767 struct page *pages[16];
1768 unsigned long index = start >> PAGE_CACHE_SHIFT;
1769 unsigned long end_index = end >> PAGE_CACHE_SHIFT;
1770 unsigned long nr_pages = end_index - index + 1;
1771 int i;
Chris Mason771ed682008-11-06 22:02:51 -05001772
Chris Mason2c64c532009-09-02 15:04:12 -04001773 clear_extent_bit(tree, start, end, clear_bits, 1, 0, NULL, GFP_NOFS);
Josef Bacikc2790a22013-07-29 11:20:47 -04001774 if (page_ops == 0)
Chris Mason771ed682008-11-06 22:02:51 -05001775 return 0;
Chris Masonc8b97812008-10-29 14:49:59 -04001776
Filipe Manana704de492014-10-06 22:14:22 +01001777 if ((page_ops & PAGE_SET_ERROR) && nr_pages > 0)
1778 mapping_set_error(inode->i_mapping, -EIO);
1779
Chris Masond3977122009-01-05 21:25:51 -05001780 while (nr_pages > 0) {
Chris Masonc8b97812008-10-29 14:49:59 -04001781 ret = find_get_pages_contig(inode->i_mapping, index,
Chris Mason5b050f02008-11-11 09:34:41 -05001782 min_t(unsigned long,
1783 nr_pages, ARRAY_SIZE(pages)), pages);
Chris Masonc8b97812008-10-29 14:49:59 -04001784 for (i = 0; i < ret; i++) {
Chris Mason8b62b722009-09-02 16:53:46 -04001785
Josef Bacikc2790a22013-07-29 11:20:47 -04001786 if (page_ops & PAGE_SET_PRIVATE2)
Chris Mason8b62b722009-09-02 16:53:46 -04001787 SetPagePrivate2(pages[i]);
1788
Chris Masonc8b97812008-10-29 14:49:59 -04001789 if (pages[i] == locked_page) {
1790 page_cache_release(pages[i]);
1791 continue;
1792 }
Josef Bacikc2790a22013-07-29 11:20:47 -04001793 if (page_ops & PAGE_CLEAR_DIRTY)
Chris Masonc8b97812008-10-29 14:49:59 -04001794 clear_page_dirty_for_io(pages[i]);
Josef Bacikc2790a22013-07-29 11:20:47 -04001795 if (page_ops & PAGE_SET_WRITEBACK)
Chris Masonc8b97812008-10-29 14:49:59 -04001796 set_page_writeback(pages[i]);
Filipe Manana704de492014-10-06 22:14:22 +01001797 if (page_ops & PAGE_SET_ERROR)
1798 SetPageError(pages[i]);
Josef Bacikc2790a22013-07-29 11:20:47 -04001799 if (page_ops & PAGE_END_WRITEBACK)
Chris Masonc8b97812008-10-29 14:49:59 -04001800 end_page_writeback(pages[i]);
Josef Bacikc2790a22013-07-29 11:20:47 -04001801 if (page_ops & PAGE_UNLOCK)
Chris Mason771ed682008-11-06 22:02:51 -05001802 unlock_page(pages[i]);
Chris Masonc8b97812008-10-29 14:49:59 -04001803 page_cache_release(pages[i]);
1804 }
1805 nr_pages -= ret;
1806 index += ret;
1807 cond_resched();
1808 }
1809 return 0;
1810}
Chris Masonc8b97812008-10-29 14:49:59 -04001811
Chris Masond352ac62008-09-29 15:18:18 -04001812/*
1813 * count the number of bytes in the tree that have a given bit(s)
1814 * set. This can be fairly slow, except for EXTENT_DIRTY which is
1815 * cached. The total number found is returned.
1816 */
Chris Masond1310b22008-01-24 16:13:08 -05001817u64 count_range_bits(struct extent_io_tree *tree,
1818 u64 *start, u64 search_end, u64 max_bytes,
David Sterba9ee49a042015-01-14 19:52:13 +01001819 unsigned bits, int contig)
Chris Masond1310b22008-01-24 16:13:08 -05001820{
1821 struct rb_node *node;
1822 struct extent_state *state;
1823 u64 cur_start = *start;
1824 u64 total_bytes = 0;
Chris Masonec29ed52011-02-23 16:23:20 -05001825 u64 last = 0;
Chris Masond1310b22008-01-24 16:13:08 -05001826 int found = 0;
1827
Dulshani Gunawardhanafae7f212013-10-31 10:30:08 +05301828 if (WARN_ON(search_end <= cur_start))
Chris Masond1310b22008-01-24 16:13:08 -05001829 return 0;
Chris Masond1310b22008-01-24 16:13:08 -05001830
Chris Masoncad321a2008-12-17 14:51:42 -05001831 spin_lock(&tree->lock);
Chris Masond1310b22008-01-24 16:13:08 -05001832 if (cur_start == 0 && bits == EXTENT_DIRTY) {
1833 total_bytes = tree->dirty_bytes;
1834 goto out;
1835 }
1836 /*
1837 * this search will find all the extents that end after
1838 * our range starts.
1839 */
Chris Mason80ea96b2008-02-01 14:51:59 -05001840 node = tree_search(tree, cur_start);
Chris Masond3977122009-01-05 21:25:51 -05001841 if (!node)
Chris Masond1310b22008-01-24 16:13:08 -05001842 goto out;
Chris Masond1310b22008-01-24 16:13:08 -05001843
Chris Masond3977122009-01-05 21:25:51 -05001844 while (1) {
Chris Masond1310b22008-01-24 16:13:08 -05001845 state = rb_entry(node, struct extent_state, rb_node);
1846 if (state->start > search_end)
1847 break;
Chris Masonec29ed52011-02-23 16:23:20 -05001848 if (contig && found && state->start > last + 1)
1849 break;
1850 if (state->end >= cur_start && (state->state & bits) == bits) {
Chris Masond1310b22008-01-24 16:13:08 -05001851 total_bytes += min(search_end, state->end) + 1 -
1852 max(cur_start, state->start);
1853 if (total_bytes >= max_bytes)
1854 break;
1855 if (!found) {
Josef Bacikaf60bed2011-05-04 11:11:17 -04001856 *start = max(cur_start, state->start);
Chris Masond1310b22008-01-24 16:13:08 -05001857 found = 1;
1858 }
Chris Masonec29ed52011-02-23 16:23:20 -05001859 last = state->end;
1860 } else if (contig && found) {
1861 break;
Chris Masond1310b22008-01-24 16:13:08 -05001862 }
1863 node = rb_next(node);
1864 if (!node)
1865 break;
1866 }
1867out:
Chris Masoncad321a2008-12-17 14:51:42 -05001868 spin_unlock(&tree->lock);
Chris Masond1310b22008-01-24 16:13:08 -05001869 return total_bytes;
1870}
Christoph Hellwigb2950862008-12-02 09:54:17 -05001871
Chris Masond352ac62008-09-29 15:18:18 -04001872/*
1873 * set the private field for a given byte offset in the tree. If there isn't
1874 * an extent_state there already, this does nothing.
1875 */
Sergei Trofimovich171170c2013-08-14 23:27:46 +03001876static int set_state_private(struct extent_io_tree *tree, u64 start, u64 private)
Chris Masond1310b22008-01-24 16:13:08 -05001877{
1878 struct rb_node *node;
1879 struct extent_state *state;
1880 int ret = 0;
1881
Chris Masoncad321a2008-12-17 14:51:42 -05001882 spin_lock(&tree->lock);
Chris Masond1310b22008-01-24 16:13:08 -05001883 /*
1884 * this search will find all the extents that end after
1885 * our range starts.
1886 */
Chris Mason80ea96b2008-02-01 14:51:59 -05001887 node = tree_search(tree, start);
Peter2b114d12008-04-01 11:21:40 -04001888 if (!node) {
Chris Masond1310b22008-01-24 16:13:08 -05001889 ret = -ENOENT;
1890 goto out;
1891 }
1892 state = rb_entry(node, struct extent_state, rb_node);
1893 if (state->start != start) {
1894 ret = -ENOENT;
1895 goto out;
1896 }
1897 state->private = private;
1898out:
Chris Masoncad321a2008-12-17 14:51:42 -05001899 spin_unlock(&tree->lock);
Chris Masond1310b22008-01-24 16:13:08 -05001900 return ret;
1901}
1902
1903int get_state_private(struct extent_io_tree *tree, u64 start, u64 *private)
1904{
1905 struct rb_node *node;
1906 struct extent_state *state;
1907 int ret = 0;
1908
Chris Masoncad321a2008-12-17 14:51:42 -05001909 spin_lock(&tree->lock);
Chris Masond1310b22008-01-24 16:13:08 -05001910 /*
1911 * this search will find all the extents that end after
1912 * our range starts.
1913 */
Chris Mason80ea96b2008-02-01 14:51:59 -05001914 node = tree_search(tree, start);
Peter2b114d12008-04-01 11:21:40 -04001915 if (!node) {
Chris Masond1310b22008-01-24 16:13:08 -05001916 ret = -ENOENT;
1917 goto out;
1918 }
1919 state = rb_entry(node, struct extent_state, rb_node);
1920 if (state->start != start) {
1921 ret = -ENOENT;
1922 goto out;
1923 }
1924 *private = state->private;
1925out:
Chris Masoncad321a2008-12-17 14:51:42 -05001926 spin_unlock(&tree->lock);
Chris Masond1310b22008-01-24 16:13:08 -05001927 return ret;
1928}
1929
1930/*
1931 * searches a range in the state tree for a given mask.
Chris Mason70dec802008-01-29 09:59:12 -05001932 * If 'filled' == 1, this returns 1 only if every extent in the tree
Chris Masond1310b22008-01-24 16:13:08 -05001933 * has the bits set. Otherwise, 1 is returned if any bit in the
1934 * range is found set.
1935 */
1936int test_range_bit(struct extent_io_tree *tree, u64 start, u64 end,
David Sterba9ee49a042015-01-14 19:52:13 +01001937 unsigned bits, int filled, struct extent_state *cached)
Chris Masond1310b22008-01-24 16:13:08 -05001938{
1939 struct extent_state *state = NULL;
1940 struct rb_node *node;
1941 int bitset = 0;
Chris Masond1310b22008-01-24 16:13:08 -05001942
Chris Masoncad321a2008-12-17 14:51:42 -05001943 spin_lock(&tree->lock);
Filipe Manana27a35072014-07-06 20:09:59 +01001944 if (cached && extent_state_in_tree(cached) && cached->start <= start &&
Josef Bacikdf98b6e2011-06-20 14:53:48 -04001945 cached->end > start)
Chris Mason9655d292009-09-02 15:22:30 -04001946 node = &cached->rb_node;
1947 else
1948 node = tree_search(tree, start);
Chris Masond1310b22008-01-24 16:13:08 -05001949 while (node && start <= end) {
1950 state = rb_entry(node, struct extent_state, rb_node);
1951
1952 if (filled && state->start > start) {
1953 bitset = 0;
1954 break;
1955 }
1956
1957 if (state->start > end)
1958 break;
1959
1960 if (state->state & bits) {
1961 bitset = 1;
1962 if (!filled)
1963 break;
1964 } else if (filled) {
1965 bitset = 0;
1966 break;
1967 }
Chris Mason46562ce2009-09-23 20:23:16 -04001968
1969 if (state->end == (u64)-1)
1970 break;
1971
Chris Masond1310b22008-01-24 16:13:08 -05001972 start = state->end + 1;
1973 if (start > end)
1974 break;
1975 node = rb_next(node);
1976 if (!node) {
1977 if (filled)
1978 bitset = 0;
1979 break;
1980 }
1981 }
Chris Masoncad321a2008-12-17 14:51:42 -05001982 spin_unlock(&tree->lock);
Chris Masond1310b22008-01-24 16:13:08 -05001983 return bitset;
1984}
Chris Masond1310b22008-01-24 16:13:08 -05001985
1986/*
1987 * helper function to set a given page up to date if all the
1988 * extents in the tree for that page are up to date
1989 */
Jeff Mahoney143bede2012-03-01 14:56:26 +01001990static void check_page_uptodate(struct extent_io_tree *tree, struct page *page)
Chris Masond1310b22008-01-24 16:13:08 -05001991{
Miao Xie4eee4fa2012-12-21 09:17:45 +00001992 u64 start = page_offset(page);
Chris Masond1310b22008-01-24 16:13:08 -05001993 u64 end = start + PAGE_CACHE_SIZE - 1;
Chris Mason9655d292009-09-02 15:22:30 -04001994 if (test_range_bit(tree, start, end, EXTENT_UPTODATE, 1, NULL))
Chris Masond1310b22008-01-24 16:13:08 -05001995 SetPageUptodate(page);
Chris Masond1310b22008-01-24 16:13:08 -05001996}
1997
Miao Xie8b110e32014-09-12 18:44:03 +08001998int free_io_failure(struct inode *inode, struct io_failure_record *rec)
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02001999{
2000 int ret;
2001 int err = 0;
2002 struct extent_io_tree *failure_tree = &BTRFS_I(inode)->io_failure_tree;
2003
2004 set_state_private(failure_tree, rec->start, 0);
2005 ret = clear_extent_bits(failure_tree, rec->start,
2006 rec->start + rec->len - 1,
2007 EXTENT_LOCKED | EXTENT_DIRTY, GFP_NOFS);
2008 if (ret)
2009 err = ret;
2010
David Woodhouse53b381b2013-01-29 18:40:14 -05002011 ret = clear_extent_bits(&BTRFS_I(inode)->io_tree, rec->start,
2012 rec->start + rec->len - 1,
2013 EXTENT_DAMAGED, GFP_NOFS);
2014 if (ret && !err)
2015 err = ret;
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002016
2017 kfree(rec);
2018 return err;
2019}
2020
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002021/*
2022 * this bypasses the standard btrfs submit functions deliberately, as
2023 * the standard behavior is to write all copies in a raid setup. here we only
2024 * want to write the one bad copy. so we do the mapping for ourselves and issue
2025 * submit_bio directly.
Stefan Behrens3ec706c2012-11-05 15:46:42 +01002026 * to avoid any synchronization issues, wait for the data after writing, which
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002027 * actually prevents the read that triggered the error from finishing.
2028 * currently, there can be no more than two copies of every data bit. thus,
2029 * exactly one rewrite is required.
2030 */
Miao Xie1203b682014-09-12 18:44:01 +08002031int repair_io_failure(struct inode *inode, u64 start, u64 length, u64 logical,
2032 struct page *page, unsigned int pg_offset, int mirror_num)
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002033{
Miao Xie1203b682014-09-12 18:44:01 +08002034 struct btrfs_fs_info *fs_info = BTRFS_I(inode)->root->fs_info;
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002035 struct bio *bio;
2036 struct btrfs_device *dev;
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002037 u64 map_length = 0;
2038 u64 sector;
2039 struct btrfs_bio *bbio = NULL;
David Woodhouse53b381b2013-01-29 18:40:14 -05002040 struct btrfs_mapping_tree *map_tree = &fs_info->mapping_tree;
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002041 int ret;
2042
Ilya Dryomov908960c2013-11-03 19:06:39 +02002043 ASSERT(!(fs_info->sb->s_flags & MS_RDONLY));
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002044 BUG_ON(!mirror_num);
2045
David Woodhouse53b381b2013-01-29 18:40:14 -05002046 /* we can't repair anything in raid56 yet */
2047 if (btrfs_is_parity_mirror(map_tree, logical, length, mirror_num))
2048 return 0;
2049
Chris Mason9be33952013-05-17 18:30:14 -04002050 bio = btrfs_io_bio_alloc(GFP_NOFS, 1);
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002051 if (!bio)
2052 return -EIO;
Kent Overstreet4f024f32013-10-11 15:44:27 -07002053 bio->bi_iter.bi_size = 0;
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002054 map_length = length;
2055
Stefan Behrens3ec706c2012-11-05 15:46:42 +01002056 ret = btrfs_map_block(fs_info, WRITE, logical,
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002057 &map_length, &bbio, mirror_num);
2058 if (ret) {
2059 bio_put(bio);
2060 return -EIO;
2061 }
2062 BUG_ON(mirror_num != bbio->mirror_num);
2063 sector = bbio->stripes[mirror_num-1].physical >> 9;
Kent Overstreet4f024f32013-10-11 15:44:27 -07002064 bio->bi_iter.bi_sector = sector;
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002065 dev = bbio->stripes[mirror_num-1].dev;
Zhao Lei6e9606d2015-01-20 15:11:34 +08002066 btrfs_put_bbio(bbio);
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002067 if (!dev || !dev->bdev || !dev->writeable) {
2068 bio_put(bio);
2069 return -EIO;
2070 }
2071 bio->bi_bdev = dev->bdev;
Miao Xieffdd2012014-09-12 18:44:00 +08002072 bio_add_page(bio, page, length, pg_offset);
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002073
Kent Overstreet33879d42013-11-23 22:33:32 -08002074 if (btrfsic_submit_bio_wait(WRITE_SYNC, bio)) {
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002075 /* try to remap that extent elsewhere? */
2076 bio_put(bio);
Stefan Behrens442a4f62012-05-25 16:06:08 +02002077 btrfs_dev_stat_inc_and_print(dev, BTRFS_DEV_STAT_WRITE_ERRS);
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002078 return -EIO;
2079 }
2080
David Sterbab14af3b2015-10-08 10:43:10 +02002081 btrfs_info_rl_in_rcu(fs_info,
2082 "read error corrected: ino %llu off %llu (dev %s sector %llu)",
Miao Xie1203b682014-09-12 18:44:01 +08002083 btrfs_ino(inode), start,
2084 rcu_str_deref(dev->name), sector);
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002085 bio_put(bio);
2086 return 0;
2087}
2088
Josef Bacikea466792012-03-26 21:57:36 -04002089int repair_eb_io_failure(struct btrfs_root *root, struct extent_buffer *eb,
2090 int mirror_num)
2091{
Josef Bacikea466792012-03-26 21:57:36 -04002092 u64 start = eb->start;
2093 unsigned long i, num_pages = num_extent_pages(eb->start, eb->len);
Chris Masond95603b2012-04-12 15:55:15 -04002094 int ret = 0;
Josef Bacikea466792012-03-26 21:57:36 -04002095
Ilya Dryomov908960c2013-11-03 19:06:39 +02002096 if (root->fs_info->sb->s_flags & MS_RDONLY)
2097 return -EROFS;
2098
Josef Bacikea466792012-03-26 21:57:36 -04002099 for (i = 0; i < num_pages; i++) {
David Sterbafb85fc92014-07-31 01:03:53 +02002100 struct page *p = eb->pages[i];
Miao Xie1203b682014-09-12 18:44:01 +08002101
2102 ret = repair_io_failure(root->fs_info->btree_inode, start,
2103 PAGE_CACHE_SIZE, start, p,
2104 start - page_offset(p), mirror_num);
Josef Bacikea466792012-03-26 21:57:36 -04002105 if (ret)
2106 break;
2107 start += PAGE_CACHE_SIZE;
2108 }
2109
2110 return ret;
2111}
2112
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002113/*
2114 * each time an IO finishes, we do a fast check in the IO failure tree
2115 * to see if we need to process or clean up an io_failure_record
2116 */
Miao Xie8b110e32014-09-12 18:44:03 +08002117int clean_io_failure(struct inode *inode, u64 start, struct page *page,
2118 unsigned int pg_offset)
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002119{
2120 u64 private;
2121 u64 private_failure;
2122 struct io_failure_record *failrec;
Ilya Dryomov908960c2013-11-03 19:06:39 +02002123 struct btrfs_fs_info *fs_info = BTRFS_I(inode)->root->fs_info;
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002124 struct extent_state *state;
2125 int num_copies;
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002126 int ret;
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002127
2128 private = 0;
2129 ret = count_range_bits(&BTRFS_I(inode)->io_failure_tree, &private,
2130 (u64)-1, 1, EXTENT_DIRTY, 0);
2131 if (!ret)
2132 return 0;
2133
2134 ret = get_state_private(&BTRFS_I(inode)->io_failure_tree, start,
2135 &private_failure);
2136 if (ret)
2137 return 0;
2138
2139 failrec = (struct io_failure_record *)(unsigned long) private_failure;
2140 BUG_ON(!failrec->this_mirror);
2141
2142 if (failrec->in_validation) {
2143 /* there was no real error, just free the record */
2144 pr_debug("clean_io_failure: freeing dummy error at %llu\n",
2145 failrec->start);
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002146 goto out;
2147 }
Ilya Dryomov908960c2013-11-03 19:06:39 +02002148 if (fs_info->sb->s_flags & MS_RDONLY)
2149 goto out;
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002150
2151 spin_lock(&BTRFS_I(inode)->io_tree.lock);
2152 state = find_first_extent_bit_state(&BTRFS_I(inode)->io_tree,
2153 failrec->start,
2154 EXTENT_LOCKED);
2155 spin_unlock(&BTRFS_I(inode)->io_tree.lock);
2156
Miao Xie883d0de2013-07-25 19:22:35 +08002157 if (state && state->start <= failrec->start &&
2158 state->end >= failrec->start + failrec->len - 1) {
Stefan Behrens3ec706c2012-11-05 15:46:42 +01002159 num_copies = btrfs_num_copies(fs_info, failrec->logical,
2160 failrec->len);
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002161 if (num_copies > 1) {
Miao Xie1203b682014-09-12 18:44:01 +08002162 repair_io_failure(inode, start, failrec->len,
Miao Xie454ff3d2014-09-12 18:43:58 +08002163 failrec->logical, page,
Miao Xie1203b682014-09-12 18:44:01 +08002164 pg_offset, failrec->failed_mirror);
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002165 }
2166 }
2167
2168out:
Miao Xie454ff3d2014-09-12 18:43:58 +08002169 free_io_failure(inode, failrec);
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002170
Miao Xie454ff3d2014-09-12 18:43:58 +08002171 return 0;
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002172}
2173
Miao Xief6124962014-09-12 18:44:04 +08002174/*
2175 * Can be called when
2176 * - hold extent lock
2177 * - under ordered extent
2178 * - the inode is freeing
2179 */
2180void btrfs_free_io_failure_record(struct inode *inode, u64 start, u64 end)
2181{
2182 struct extent_io_tree *failure_tree = &BTRFS_I(inode)->io_failure_tree;
2183 struct io_failure_record *failrec;
2184 struct extent_state *state, *next;
2185
2186 if (RB_EMPTY_ROOT(&failure_tree->state))
2187 return;
2188
2189 spin_lock(&failure_tree->lock);
2190 state = find_first_extent_bit_state(failure_tree, start, EXTENT_DIRTY);
2191 while (state) {
2192 if (state->start > end)
2193 break;
2194
2195 ASSERT(state->end <= end);
2196
2197 next = next_state(state);
2198
Satoru Takeuchi6e1103a2014-12-25 18:21:41 +09002199 failrec = (struct io_failure_record *)(unsigned long)state->private;
Miao Xief6124962014-09-12 18:44:04 +08002200 free_extent_state(state);
2201 kfree(failrec);
2202
2203 state = next;
2204 }
2205 spin_unlock(&failure_tree->lock);
2206}
2207
Miao Xie2fe63032014-09-12 18:43:59 +08002208int btrfs_get_io_failure_record(struct inode *inode, u64 start, u64 end,
2209 struct io_failure_record **failrec_ret)
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002210{
Miao Xie2fe63032014-09-12 18:43:59 +08002211 struct io_failure_record *failrec;
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002212 u64 private;
2213 struct extent_map *em;
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002214 struct extent_io_tree *failure_tree = &BTRFS_I(inode)->io_failure_tree;
2215 struct extent_io_tree *tree = &BTRFS_I(inode)->io_tree;
2216 struct extent_map_tree *em_tree = &BTRFS_I(inode)->extent_tree;
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002217 int ret;
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002218 u64 logical;
2219
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002220 ret = get_state_private(failure_tree, start, &private);
2221 if (ret) {
2222 failrec = kzalloc(sizeof(*failrec), GFP_NOFS);
2223 if (!failrec)
2224 return -ENOMEM;
Miao Xie2fe63032014-09-12 18:43:59 +08002225
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002226 failrec->start = start;
2227 failrec->len = end - start + 1;
2228 failrec->this_mirror = 0;
2229 failrec->bio_flags = 0;
2230 failrec->in_validation = 0;
2231
2232 read_lock(&em_tree->lock);
2233 em = lookup_extent_mapping(em_tree, start, failrec->len);
2234 if (!em) {
2235 read_unlock(&em_tree->lock);
2236 kfree(failrec);
2237 return -EIO;
2238 }
2239
Filipe David Borba Manana68ba9902013-11-25 03:22:07 +00002240 if (em->start > start || em->start + em->len <= start) {
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002241 free_extent_map(em);
2242 em = NULL;
2243 }
2244 read_unlock(&em_tree->lock);
Tsutomu Itoh7a2d6a62012-10-01 03:07:15 -06002245 if (!em) {
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002246 kfree(failrec);
2247 return -EIO;
2248 }
Miao Xie2fe63032014-09-12 18:43:59 +08002249
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002250 logical = start - em->start;
2251 logical = em->block_start + logical;
2252 if (test_bit(EXTENT_FLAG_COMPRESSED, &em->flags)) {
2253 logical = em->block_start;
2254 failrec->bio_flags = EXTENT_BIO_COMPRESSED;
2255 extent_set_compress_type(&failrec->bio_flags,
2256 em->compress_type);
2257 }
Miao Xie2fe63032014-09-12 18:43:59 +08002258
2259 pr_debug("Get IO Failure Record: (new) logical=%llu, start=%llu, len=%llu\n",
2260 logical, start, failrec->len);
2261
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002262 failrec->logical = logical;
2263 free_extent_map(em);
2264
2265 /* set the bits in the private failure tree */
2266 ret = set_extent_bits(failure_tree, start, end,
2267 EXTENT_LOCKED | EXTENT_DIRTY, GFP_NOFS);
2268 if (ret >= 0)
2269 ret = set_state_private(failure_tree, start,
2270 (u64)(unsigned long)failrec);
2271 /* set the bits in the inode's tree */
2272 if (ret >= 0)
2273 ret = set_extent_bits(tree, start, end, EXTENT_DAMAGED,
2274 GFP_NOFS);
2275 if (ret < 0) {
2276 kfree(failrec);
2277 return ret;
2278 }
2279 } else {
2280 failrec = (struct io_failure_record *)(unsigned long)private;
Miao Xie2fe63032014-09-12 18:43:59 +08002281 pr_debug("Get IO Failure Record: (found) logical=%llu, start=%llu, len=%llu, validation=%d\n",
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002282 failrec->logical, failrec->start, failrec->len,
2283 failrec->in_validation);
2284 /*
2285 * when data can be on disk more than twice, add to failrec here
2286 * (e.g. with a list for failed_mirror) to make
2287 * clean_io_failure() clean all those errors at once.
2288 */
2289 }
Miao Xie2fe63032014-09-12 18:43:59 +08002290
2291 *failrec_ret = failrec;
2292
2293 return 0;
2294}
2295
2296int btrfs_check_repairable(struct inode *inode, struct bio *failed_bio,
2297 struct io_failure_record *failrec, int failed_mirror)
2298{
2299 int num_copies;
2300
Stefan Behrens5d964052012-11-05 14:59:07 +01002301 num_copies = btrfs_num_copies(BTRFS_I(inode)->root->fs_info,
2302 failrec->logical, failrec->len);
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002303 if (num_copies == 1) {
2304 /*
2305 * we only have a single copy of the data, so don't bother with
2306 * all the retry and error correction code that follows. no
2307 * matter what the error is, it is very likely to persist.
2308 */
Miao Xie2fe63032014-09-12 18:43:59 +08002309 pr_debug("Check Repairable: cannot repair, num_copies=%d, next_mirror %d, failed_mirror %d\n",
Miao Xie09a7f7a2013-07-25 19:22:32 +08002310 num_copies, failrec->this_mirror, failed_mirror);
Miao Xie2fe63032014-09-12 18:43:59 +08002311 return 0;
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002312 }
2313
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002314 /*
2315 * there are two premises:
2316 * a) deliver good data to the caller
2317 * b) correct the bad sectors on disk
2318 */
2319 if (failed_bio->bi_vcnt > 1) {
2320 /*
2321 * to fulfill b), we need to know the exact failing sectors, as
2322 * we don't want to rewrite any more than the failed ones. thus,
2323 * we need separate read requests for the failed bio
2324 *
2325 * if the following BUG_ON triggers, our validation request got
2326 * merged. we need separate requests for our algorithm to work.
2327 */
2328 BUG_ON(failrec->in_validation);
2329 failrec->in_validation = 1;
2330 failrec->this_mirror = failed_mirror;
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002331 } else {
2332 /*
2333 * we're ready to fulfill a) and b) alongside. get a good copy
2334 * of the failed sector and if we succeed, we have setup
2335 * everything for repair_io_failure to do the rest for us.
2336 */
2337 if (failrec->in_validation) {
2338 BUG_ON(failrec->this_mirror != failed_mirror);
2339 failrec->in_validation = 0;
2340 failrec->this_mirror = 0;
2341 }
2342 failrec->failed_mirror = failed_mirror;
2343 failrec->this_mirror++;
2344 if (failrec->this_mirror == failed_mirror)
2345 failrec->this_mirror++;
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002346 }
2347
Miao Xiefacc8a222013-07-25 19:22:34 +08002348 if (failrec->this_mirror > num_copies) {
Miao Xie2fe63032014-09-12 18:43:59 +08002349 pr_debug("Check Repairable: (fail) num_copies=%d, next_mirror %d, failed_mirror %d\n",
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002350 num_copies, failrec->this_mirror, failed_mirror);
Miao Xie2fe63032014-09-12 18:43:59 +08002351 return 0;
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002352 }
2353
Miao Xie2fe63032014-09-12 18:43:59 +08002354 return 1;
2355}
2356
2357
2358struct bio *btrfs_create_repair_bio(struct inode *inode, struct bio *failed_bio,
2359 struct io_failure_record *failrec,
2360 struct page *page, int pg_offset, int icsum,
Miao Xie8b110e32014-09-12 18:44:03 +08002361 bio_end_io_t *endio_func, void *data)
Miao Xie2fe63032014-09-12 18:43:59 +08002362{
2363 struct bio *bio;
2364 struct btrfs_io_bio *btrfs_failed_bio;
2365 struct btrfs_io_bio *btrfs_bio;
2366
Chris Mason9be33952013-05-17 18:30:14 -04002367 bio = btrfs_io_bio_alloc(GFP_NOFS, 1);
Miao Xie2fe63032014-09-12 18:43:59 +08002368 if (!bio)
2369 return NULL;
2370
2371 bio->bi_end_io = endio_func;
Kent Overstreet4f024f32013-10-11 15:44:27 -07002372 bio->bi_iter.bi_sector = failrec->logical >> 9;
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002373 bio->bi_bdev = BTRFS_I(inode)->root->fs_info->fs_devices->latest_bdev;
Kent Overstreet4f024f32013-10-11 15:44:27 -07002374 bio->bi_iter.bi_size = 0;
Miao Xie8b110e32014-09-12 18:44:03 +08002375 bio->bi_private = data;
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002376
Miao Xiefacc8a222013-07-25 19:22:34 +08002377 btrfs_failed_bio = btrfs_io_bio(failed_bio);
2378 if (btrfs_failed_bio->csum) {
2379 struct btrfs_fs_info *fs_info = BTRFS_I(inode)->root->fs_info;
2380 u16 csum_size = btrfs_super_csum_size(fs_info->super_copy);
2381
2382 btrfs_bio = btrfs_io_bio(bio);
2383 btrfs_bio->csum = btrfs_bio->csum_inline;
Miao Xie2fe63032014-09-12 18:43:59 +08002384 icsum *= csum_size;
2385 memcpy(btrfs_bio->csum, btrfs_failed_bio->csum + icsum,
Miao Xiefacc8a222013-07-25 19:22:34 +08002386 csum_size);
2387 }
2388
Miao Xie2fe63032014-09-12 18:43:59 +08002389 bio_add_page(bio, page, failrec->len, pg_offset);
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002390
Miao Xie2fe63032014-09-12 18:43:59 +08002391 return bio;
2392}
2393
2394/*
2395 * this is a generic handler for readpage errors (default
2396 * readpage_io_failed_hook). if other copies exist, read those and write back
2397 * good data to the failed position. does not investigate in remapping the
2398 * failed extent elsewhere, hoping the device will be smart enough to do this as
2399 * needed
2400 */
2401
2402static int bio_readpage_error(struct bio *failed_bio, u64 phy_offset,
2403 struct page *page, u64 start, u64 end,
2404 int failed_mirror)
2405{
2406 struct io_failure_record *failrec;
2407 struct inode *inode = page->mapping->host;
2408 struct extent_io_tree *tree = &BTRFS_I(inode)->io_tree;
2409 struct bio *bio;
2410 int read_mode;
2411 int ret;
2412
2413 BUG_ON(failed_bio->bi_rw & REQ_WRITE);
2414
2415 ret = btrfs_get_io_failure_record(inode, start, end, &failrec);
2416 if (ret)
2417 return ret;
2418
2419 ret = btrfs_check_repairable(inode, failed_bio, failrec, failed_mirror);
2420 if (!ret) {
2421 free_io_failure(inode, failrec);
2422 return -EIO;
2423 }
2424
2425 if (failed_bio->bi_vcnt > 1)
2426 read_mode = READ_SYNC | REQ_FAILFAST_DEV;
2427 else
2428 read_mode = READ_SYNC;
2429
2430 phy_offset >>= inode->i_sb->s_blocksize_bits;
2431 bio = btrfs_create_repair_bio(inode, failed_bio, failrec, page,
2432 start - page_offset(page),
Miao Xie8b110e32014-09-12 18:44:03 +08002433 (int)phy_offset, failed_bio->bi_end_io,
2434 NULL);
Miao Xie2fe63032014-09-12 18:43:59 +08002435 if (!bio) {
2436 free_io_failure(inode, failrec);
2437 return -EIO;
2438 }
2439
2440 pr_debug("Repair Read Error: submitting new read[%#x] to this_mirror=%d, in_validation=%d\n",
2441 read_mode, failrec->this_mirror, failrec->in_validation);
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002442
Tsutomu Itoh013bd4c2012-02-16 10:11:40 +09002443 ret = tree->ops->submit_bio_hook(inode, read_mode, bio,
2444 failrec->this_mirror,
2445 failrec->bio_flags, 0);
Miao Xie6c387ab2014-09-12 18:43:57 +08002446 if (ret) {
Miao Xie454ff3d2014-09-12 18:43:58 +08002447 free_io_failure(inode, failrec);
Miao Xie6c387ab2014-09-12 18:43:57 +08002448 bio_put(bio);
2449 }
2450
Tsutomu Itoh013bd4c2012-02-16 10:11:40 +09002451 return ret;
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002452}
2453
Chris Masond1310b22008-01-24 16:13:08 -05002454/* lots and lots of room for performance fixes in the end_bio funcs */
2455
Jeff Mahoney87826df2012-02-15 16:23:57 +01002456int end_extent_writepage(struct page *page, int err, u64 start, u64 end)
2457{
2458 int uptodate = (err == 0);
2459 struct extent_io_tree *tree;
Eric Sandeen3e2426b2014-06-12 00:39:58 -05002460 int ret = 0;
Jeff Mahoney87826df2012-02-15 16:23:57 +01002461
2462 tree = &BTRFS_I(page->mapping->host)->io_tree;
2463
2464 if (tree->ops && tree->ops->writepage_end_io_hook) {
2465 ret = tree->ops->writepage_end_io_hook(page, start,
2466 end, NULL, uptodate);
2467 if (ret)
2468 uptodate = 0;
2469 }
2470
Jeff Mahoney87826df2012-02-15 16:23:57 +01002471 if (!uptodate) {
Jeff Mahoney87826df2012-02-15 16:23:57 +01002472 ClearPageUptodate(page);
2473 SetPageError(page);
Liu Bo5dca6ee2014-05-12 12:47:36 +08002474 ret = ret < 0 ? ret : -EIO;
2475 mapping_set_error(page->mapping, ret);
Jeff Mahoney87826df2012-02-15 16:23:57 +01002476 }
2477 return 0;
2478}
2479
Chris Masond1310b22008-01-24 16:13:08 -05002480/*
2481 * after a writepage IO is done, we need to:
2482 * clear the uptodate bits on error
2483 * clear the writeback bits in the extent tree for this IO
2484 * end_page_writeback if the page has no more pending IO
2485 *
2486 * Scheduling is not allowed, so the extent state tree is expected
2487 * to have one and only one object corresponding to this IO.
2488 */
Christoph Hellwig4246a0b2015-07-20 15:29:37 +02002489static void end_bio_extent_writepage(struct bio *bio)
Chris Masond1310b22008-01-24 16:13:08 -05002490{
Kent Overstreet2c30c712013-11-07 12:20:26 -08002491 struct bio_vec *bvec;
Chris Masond1310b22008-01-24 16:13:08 -05002492 u64 start;
2493 u64 end;
Kent Overstreet2c30c712013-11-07 12:20:26 -08002494 int i;
Chris Masond1310b22008-01-24 16:13:08 -05002495
Kent Overstreet2c30c712013-11-07 12:20:26 -08002496 bio_for_each_segment_all(bvec, bio, i) {
Chris Masond1310b22008-01-24 16:13:08 -05002497 struct page *page = bvec->bv_page;
David Woodhouse902b22f2008-08-20 08:51:49 -04002498
Alexandre Oliva17a5adc2013-05-15 11:38:55 -04002499 /* We always issue full-page reads, but if some block
2500 * in a page fails to read, blk_update_request() will
2501 * advance bv_offset and adjust bv_len to compensate.
2502 * Print a warning for nonzero offsets, and an error
2503 * if they don't add up to a full page. */
Frank Holtonefe120a2013-12-20 11:37:06 -05002504 if (bvec->bv_offset || bvec->bv_len != PAGE_CACHE_SIZE) {
2505 if (bvec->bv_offset + bvec->bv_len != PAGE_CACHE_SIZE)
2506 btrfs_err(BTRFS_I(page->mapping->host)->root->fs_info,
2507 "partial page write in btrfs with offset %u and length %u",
2508 bvec->bv_offset, bvec->bv_len);
2509 else
2510 btrfs_info(BTRFS_I(page->mapping->host)->root->fs_info,
2511 "incomplete page write in btrfs with offset %u and "
2512 "length %u",
2513 bvec->bv_offset, bvec->bv_len);
2514 }
Chris Masond1310b22008-01-24 16:13:08 -05002515
Alexandre Oliva17a5adc2013-05-15 11:38:55 -04002516 start = page_offset(page);
2517 end = start + bvec->bv_offset + bvec->bv_len - 1;
Chris Masond1310b22008-01-24 16:13:08 -05002518
Christoph Hellwig4246a0b2015-07-20 15:29:37 +02002519 if (end_extent_writepage(page, bio->bi_error, start, end))
Jeff Mahoney87826df2012-02-15 16:23:57 +01002520 continue;
Chris Mason70dec802008-01-29 09:59:12 -05002521
Alexandre Oliva17a5adc2013-05-15 11:38:55 -04002522 end_page_writeback(page);
Kent Overstreet2c30c712013-11-07 12:20:26 -08002523 }
Chris Mason2b1f55b2008-09-24 11:48:04 -04002524
Chris Masond1310b22008-01-24 16:13:08 -05002525 bio_put(bio);
Chris Masond1310b22008-01-24 16:13:08 -05002526}
2527
Miao Xie883d0de2013-07-25 19:22:35 +08002528static void
2529endio_readpage_release_extent(struct extent_io_tree *tree, u64 start, u64 len,
2530 int uptodate)
2531{
2532 struct extent_state *cached = NULL;
2533 u64 end = start + len - 1;
2534
2535 if (uptodate && tree->track_uptodate)
2536 set_extent_uptodate(tree, start, end, &cached, GFP_ATOMIC);
2537 unlock_extent_cached(tree, start, end, &cached, GFP_ATOMIC);
2538}
2539
Chris Masond1310b22008-01-24 16:13:08 -05002540/*
2541 * after a readpage IO is done, we need to:
2542 * clear the uptodate bits on error
2543 * set the uptodate bits if things worked
2544 * set the page up to date if all extents in the tree are uptodate
2545 * clear the lock bit in the extent tree
2546 * unlock the page if there are no other extents locked for it
2547 *
2548 * Scheduling is not allowed, so the extent state tree is expected
2549 * to have one and only one object corresponding to this IO.
2550 */
Christoph Hellwig4246a0b2015-07-20 15:29:37 +02002551static void end_bio_extent_readpage(struct bio *bio)
Chris Masond1310b22008-01-24 16:13:08 -05002552{
Kent Overstreet2c30c712013-11-07 12:20:26 -08002553 struct bio_vec *bvec;
Christoph Hellwig4246a0b2015-07-20 15:29:37 +02002554 int uptodate = !bio->bi_error;
Miao Xiefacc8a222013-07-25 19:22:34 +08002555 struct btrfs_io_bio *io_bio = btrfs_io_bio(bio);
David Woodhouse902b22f2008-08-20 08:51:49 -04002556 struct extent_io_tree *tree;
Miao Xiefacc8a222013-07-25 19:22:34 +08002557 u64 offset = 0;
Chris Masond1310b22008-01-24 16:13:08 -05002558 u64 start;
2559 u64 end;
Miao Xiefacc8a222013-07-25 19:22:34 +08002560 u64 len;
Miao Xie883d0de2013-07-25 19:22:35 +08002561 u64 extent_start = 0;
2562 u64 extent_len = 0;
Josef Bacik5cf1ab52012-04-16 09:42:26 -04002563 int mirror;
Chris Masond1310b22008-01-24 16:13:08 -05002564 int ret;
Kent Overstreet2c30c712013-11-07 12:20:26 -08002565 int i;
Chris Masond1310b22008-01-24 16:13:08 -05002566
Kent Overstreet2c30c712013-11-07 12:20:26 -08002567 bio_for_each_segment_all(bvec, bio, i) {
Chris Masond1310b22008-01-24 16:13:08 -05002568 struct page *page = bvec->bv_page;
Josef Bacika71754f2013-06-17 17:14:39 -04002569 struct inode *inode = page->mapping->host;
Arne Jansen507903b2011-04-06 10:02:20 +00002570
Kent Overstreetbe3940c2012-09-11 14:23:05 -06002571 pr_debug("end_bio_extent_readpage: bi_sector=%llu, err=%d, "
Christoph Hellwig4246a0b2015-07-20 15:29:37 +02002572 "mirror=%u\n", (u64)bio->bi_iter.bi_sector,
2573 bio->bi_error, io_bio->mirror_num);
Josef Bacika71754f2013-06-17 17:14:39 -04002574 tree = &BTRFS_I(inode)->io_tree;
David Woodhouse902b22f2008-08-20 08:51:49 -04002575
Alexandre Oliva17a5adc2013-05-15 11:38:55 -04002576 /* We always issue full-page reads, but if some block
2577 * in a page fails to read, blk_update_request() will
2578 * advance bv_offset and adjust bv_len to compensate.
2579 * Print a warning for nonzero offsets, and an error
2580 * if they don't add up to a full page. */
Frank Holtonefe120a2013-12-20 11:37:06 -05002581 if (bvec->bv_offset || bvec->bv_len != PAGE_CACHE_SIZE) {
2582 if (bvec->bv_offset + bvec->bv_len != PAGE_CACHE_SIZE)
2583 btrfs_err(BTRFS_I(page->mapping->host)->root->fs_info,
2584 "partial page read in btrfs with offset %u and length %u",
2585 bvec->bv_offset, bvec->bv_len);
2586 else
2587 btrfs_info(BTRFS_I(page->mapping->host)->root->fs_info,
2588 "incomplete page read in btrfs with offset %u and "
2589 "length %u",
2590 bvec->bv_offset, bvec->bv_len);
2591 }
Chris Masond1310b22008-01-24 16:13:08 -05002592
Alexandre Oliva17a5adc2013-05-15 11:38:55 -04002593 start = page_offset(page);
2594 end = start + bvec->bv_offset + bvec->bv_len - 1;
Miao Xiefacc8a222013-07-25 19:22:34 +08002595 len = bvec->bv_len;
Chris Masond1310b22008-01-24 16:13:08 -05002596
Chris Mason9be33952013-05-17 18:30:14 -04002597 mirror = io_bio->mirror_num;
Miao Xief2a09da2013-07-25 19:22:33 +08002598 if (likely(uptodate && tree->ops &&
2599 tree->ops->readpage_end_io_hook)) {
Miao Xiefacc8a222013-07-25 19:22:34 +08002600 ret = tree->ops->readpage_end_io_hook(io_bio, offset,
2601 page, start, end,
2602 mirror);
Stefan Behrens5ee08442012-08-27 08:30:03 -06002603 if (ret)
Chris Masond1310b22008-01-24 16:13:08 -05002604 uptodate = 0;
Stefan Behrens5ee08442012-08-27 08:30:03 -06002605 else
Miao Xie1203b682014-09-12 18:44:01 +08002606 clean_io_failure(inode, start, page, 0);
Chris Masond1310b22008-01-24 16:13:08 -05002607 }
Josef Bacikea466792012-03-26 21:57:36 -04002608
Miao Xief2a09da2013-07-25 19:22:33 +08002609 if (likely(uptodate))
2610 goto readpage_ok;
2611
2612 if (tree->ops && tree->ops->readpage_io_failed_hook) {
Josef Bacik5cf1ab52012-04-16 09:42:26 -04002613 ret = tree->ops->readpage_io_failed_hook(page, mirror);
Christoph Hellwig4246a0b2015-07-20 15:29:37 +02002614 if (!ret && !bio->bi_error)
Josef Bacikea466792012-03-26 21:57:36 -04002615 uptodate = 1;
Miao Xief2a09da2013-07-25 19:22:33 +08002616 } else {
Jan Schmidtf4a8e652011-12-01 09:30:36 -05002617 /*
2618 * The generic bio_readpage_error handles errors the
2619 * following way: If possible, new read requests are
2620 * created and submitted and will end up in
2621 * end_bio_extent_readpage as well (if we're lucky, not
2622 * in the !uptodate case). In that case it returns 0 and
2623 * we just go on with the next page in our bio. If it
2624 * can't handle the error it will return -EIO and we
2625 * remain responsible for that page.
2626 */
Miao Xiefacc8a222013-07-25 19:22:34 +08002627 ret = bio_readpage_error(bio, offset, page, start, end,
2628 mirror);
Chris Mason7e383262008-04-09 16:28:12 -04002629 if (ret == 0) {
Christoph Hellwig4246a0b2015-07-20 15:29:37 +02002630 uptodate = !bio->bi_error;
Liu Bo38c1c2e2014-08-19 23:33:13 +08002631 offset += len;
Chris Mason7e383262008-04-09 16:28:12 -04002632 continue;
2633 }
2634 }
Miao Xief2a09da2013-07-25 19:22:33 +08002635readpage_ok:
Miao Xie883d0de2013-07-25 19:22:35 +08002636 if (likely(uptodate)) {
Josef Bacika71754f2013-06-17 17:14:39 -04002637 loff_t i_size = i_size_read(inode);
2638 pgoff_t end_index = i_size >> PAGE_CACHE_SHIFT;
Liu Boa583c022014-08-19 23:32:22 +08002639 unsigned off;
Josef Bacika71754f2013-06-17 17:14:39 -04002640
2641 /* Zero out the end if this page straddles i_size */
Liu Boa583c022014-08-19 23:32:22 +08002642 off = i_size & (PAGE_CACHE_SIZE-1);
2643 if (page->index == end_index && off)
2644 zero_user_segment(page, off, PAGE_CACHE_SIZE);
Alexandre Oliva17a5adc2013-05-15 11:38:55 -04002645 SetPageUptodate(page);
Chris Mason70dec802008-01-29 09:59:12 -05002646 } else {
Alexandre Oliva17a5adc2013-05-15 11:38:55 -04002647 ClearPageUptodate(page);
2648 SetPageError(page);
Chris Mason70dec802008-01-29 09:59:12 -05002649 }
Alexandre Oliva17a5adc2013-05-15 11:38:55 -04002650 unlock_page(page);
Miao Xiefacc8a222013-07-25 19:22:34 +08002651 offset += len;
Miao Xie883d0de2013-07-25 19:22:35 +08002652
2653 if (unlikely(!uptodate)) {
2654 if (extent_len) {
2655 endio_readpage_release_extent(tree,
2656 extent_start,
2657 extent_len, 1);
2658 extent_start = 0;
2659 extent_len = 0;
2660 }
2661 endio_readpage_release_extent(tree, start,
2662 end - start + 1, 0);
2663 } else if (!extent_len) {
2664 extent_start = start;
2665 extent_len = end + 1 - start;
2666 } else if (extent_start + extent_len == start) {
2667 extent_len += end + 1 - start;
2668 } else {
2669 endio_readpage_release_extent(tree, extent_start,
2670 extent_len, uptodate);
2671 extent_start = start;
2672 extent_len = end + 1 - start;
2673 }
Kent Overstreet2c30c712013-11-07 12:20:26 -08002674 }
Chris Masond1310b22008-01-24 16:13:08 -05002675
Miao Xie883d0de2013-07-25 19:22:35 +08002676 if (extent_len)
2677 endio_readpage_release_extent(tree, extent_start, extent_len,
2678 uptodate);
Miao Xiefacc8a222013-07-25 19:22:34 +08002679 if (io_bio->end_io)
Christoph Hellwig4246a0b2015-07-20 15:29:37 +02002680 io_bio->end_io(io_bio, bio->bi_error);
Chris Masond1310b22008-01-24 16:13:08 -05002681 bio_put(bio);
Chris Masond1310b22008-01-24 16:13:08 -05002682}
2683
Chris Mason9be33952013-05-17 18:30:14 -04002684/*
2685 * this allocates from the btrfs_bioset. We're returning a bio right now
2686 * but you can call btrfs_io_bio for the appropriate container_of magic
2687 */
Miao Xie88f794e2010-11-22 03:02:55 +00002688struct bio *
2689btrfs_bio_alloc(struct block_device *bdev, u64 first_sector, int nr_vecs,
2690 gfp_t gfp_flags)
Chris Masond1310b22008-01-24 16:13:08 -05002691{
Miao Xiefacc8a222013-07-25 19:22:34 +08002692 struct btrfs_io_bio *btrfs_bio;
Chris Masond1310b22008-01-24 16:13:08 -05002693 struct bio *bio;
2694
Chris Mason9be33952013-05-17 18:30:14 -04002695 bio = bio_alloc_bioset(gfp_flags, nr_vecs, btrfs_bioset);
Chris Masond1310b22008-01-24 16:13:08 -05002696
2697 if (bio == NULL && (current->flags & PF_MEMALLOC)) {
Chris Mason9be33952013-05-17 18:30:14 -04002698 while (!bio && (nr_vecs /= 2)) {
2699 bio = bio_alloc_bioset(gfp_flags,
2700 nr_vecs, btrfs_bioset);
2701 }
Chris Masond1310b22008-01-24 16:13:08 -05002702 }
2703
2704 if (bio) {
2705 bio->bi_bdev = bdev;
Kent Overstreet4f024f32013-10-11 15:44:27 -07002706 bio->bi_iter.bi_sector = first_sector;
Miao Xiefacc8a222013-07-25 19:22:34 +08002707 btrfs_bio = btrfs_io_bio(bio);
2708 btrfs_bio->csum = NULL;
2709 btrfs_bio->csum_allocated = NULL;
2710 btrfs_bio->end_io = NULL;
Chris Masond1310b22008-01-24 16:13:08 -05002711 }
2712 return bio;
2713}
2714
Chris Mason9be33952013-05-17 18:30:14 -04002715struct bio *btrfs_bio_clone(struct bio *bio, gfp_t gfp_mask)
2716{
Miao Xie23ea8e52014-09-12 18:43:54 +08002717 struct btrfs_io_bio *btrfs_bio;
2718 struct bio *new;
Chris Mason9be33952013-05-17 18:30:14 -04002719
Miao Xie23ea8e52014-09-12 18:43:54 +08002720 new = bio_clone_bioset(bio, gfp_mask, btrfs_bioset);
2721 if (new) {
2722 btrfs_bio = btrfs_io_bio(new);
2723 btrfs_bio->csum = NULL;
2724 btrfs_bio->csum_allocated = NULL;
2725 btrfs_bio->end_io = NULL;
Chris Mason3a9508b2015-08-21 10:05:39 -07002726
2727#ifdef CONFIG_BLK_CGROUP
Chris Masonda2f0f72015-07-02 13:57:22 -07002728 /* FIXME, put this into bio_clone_bioset */
2729 if (bio->bi_css)
2730 bio_associate_blkcg(new, bio->bi_css);
Chris Mason3a9508b2015-08-21 10:05:39 -07002731#endif
Miao Xie23ea8e52014-09-12 18:43:54 +08002732 }
2733 return new;
2734}
Chris Mason9be33952013-05-17 18:30:14 -04002735
2736/* this also allocates from the btrfs_bioset */
2737struct bio *btrfs_io_bio_alloc(gfp_t gfp_mask, unsigned int nr_iovecs)
2738{
Miao Xiefacc8a222013-07-25 19:22:34 +08002739 struct btrfs_io_bio *btrfs_bio;
2740 struct bio *bio;
2741
2742 bio = bio_alloc_bioset(gfp_mask, nr_iovecs, btrfs_bioset);
2743 if (bio) {
2744 btrfs_bio = btrfs_io_bio(bio);
2745 btrfs_bio->csum = NULL;
2746 btrfs_bio->csum_allocated = NULL;
2747 btrfs_bio->end_io = NULL;
2748 }
2749 return bio;
Chris Mason9be33952013-05-17 18:30:14 -04002750}
2751
2752
Jeff Mahoney355808c2011-10-03 23:23:14 -04002753static int __must_check submit_one_bio(int rw, struct bio *bio,
2754 int mirror_num, unsigned long bio_flags)
Chris Masond1310b22008-01-24 16:13:08 -05002755{
Chris Masond1310b22008-01-24 16:13:08 -05002756 int ret = 0;
Chris Mason70dec802008-01-29 09:59:12 -05002757 struct bio_vec *bvec = bio->bi_io_vec + bio->bi_vcnt - 1;
2758 struct page *page = bvec->bv_page;
2759 struct extent_io_tree *tree = bio->bi_private;
Chris Mason70dec802008-01-29 09:59:12 -05002760 u64 start;
Chris Mason70dec802008-01-29 09:59:12 -05002761
Miao Xie4eee4fa2012-12-21 09:17:45 +00002762 start = page_offset(page) + bvec->bv_offset;
Chris Mason70dec802008-01-29 09:59:12 -05002763
David Woodhouse902b22f2008-08-20 08:51:49 -04002764 bio->bi_private = NULL;
Chris Masond1310b22008-01-24 16:13:08 -05002765
2766 bio_get(bio);
2767
Chris Mason065631f2008-02-20 12:07:25 -05002768 if (tree->ops && tree->ops->submit_bio_hook)
liubo6b82ce82011-01-26 06:21:39 +00002769 ret = tree->ops->submit_bio_hook(page->mapping->host, rw, bio,
Chris Masoneaf25d92010-05-25 09:48:28 -04002770 mirror_num, bio_flags, start);
Chris Mason0b86a832008-03-24 15:01:56 -04002771 else
Stefan Behrens21adbd52011-11-09 13:44:05 +01002772 btrfsic_submit_bio(rw, bio);
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002773
Chris Masond1310b22008-01-24 16:13:08 -05002774 bio_put(bio);
2775 return ret;
2776}
2777
David Woodhouse64a16702009-07-15 23:29:37 +01002778static int merge_bio(int rw, struct extent_io_tree *tree, struct page *page,
Jeff Mahoney3444a972011-10-03 23:23:13 -04002779 unsigned long offset, size_t size, struct bio *bio,
2780 unsigned long bio_flags)
2781{
2782 int ret = 0;
2783 if (tree->ops && tree->ops->merge_bio_hook)
David Woodhouse64a16702009-07-15 23:29:37 +01002784 ret = tree->ops->merge_bio_hook(rw, page, offset, size, bio,
Jeff Mahoney3444a972011-10-03 23:23:13 -04002785 bio_flags);
2786 BUG_ON(ret < 0);
2787 return ret;
2788
2789}
2790
Chris Masond1310b22008-01-24 16:13:08 -05002791static int submit_extent_page(int rw, struct extent_io_tree *tree,
Chris Masonda2f0f72015-07-02 13:57:22 -07002792 struct writeback_control *wbc,
Chris Masond1310b22008-01-24 16:13:08 -05002793 struct page *page, sector_t sector,
2794 size_t size, unsigned long offset,
2795 struct block_device *bdev,
2796 struct bio **bio_ret,
2797 unsigned long max_pages,
Chris Masonf1885912008-04-09 16:28:12 -04002798 bio_end_io_t end_io_func,
Chris Masonc8b97812008-10-29 14:49:59 -04002799 int mirror_num,
2800 unsigned long prev_bio_flags,
Filipe Manana005efed2015-09-14 09:09:31 +01002801 unsigned long bio_flags,
2802 bool force_bio_submit)
Chris Masond1310b22008-01-24 16:13:08 -05002803{
2804 int ret = 0;
2805 struct bio *bio;
Chris Masonc8b97812008-10-29 14:49:59 -04002806 int contig = 0;
Chris Masonc8b97812008-10-29 14:49:59 -04002807 int old_compressed = prev_bio_flags & EXTENT_BIO_COMPRESSED;
Chris Mason5b050f02008-11-11 09:34:41 -05002808 size_t page_size = min_t(size_t, size, PAGE_CACHE_SIZE);
Chris Masond1310b22008-01-24 16:13:08 -05002809
2810 if (bio_ret && *bio_ret) {
2811 bio = *bio_ret;
Chris Masonc8b97812008-10-29 14:49:59 -04002812 if (old_compressed)
Kent Overstreet4f024f32013-10-11 15:44:27 -07002813 contig = bio->bi_iter.bi_sector == sector;
Chris Masonc8b97812008-10-29 14:49:59 -04002814 else
Kent Overstreetf73a1c72012-09-25 15:05:12 -07002815 contig = bio_end_sector(bio) == sector;
Chris Masonc8b97812008-10-29 14:49:59 -04002816
2817 if (prev_bio_flags != bio_flags || !contig ||
Filipe Manana005efed2015-09-14 09:09:31 +01002818 force_bio_submit ||
David Woodhouse64a16702009-07-15 23:29:37 +01002819 merge_bio(rw, tree, page, offset, page_size, bio, bio_flags) ||
Chris Masonc8b97812008-10-29 14:49:59 -04002820 bio_add_page(bio, page, page_size, offset) < page_size) {
2821 ret = submit_one_bio(rw, bio, mirror_num,
2822 prev_bio_flags);
Naohiro Aota289454a2015-01-06 01:01:03 +09002823 if (ret < 0) {
2824 *bio_ret = NULL;
Jeff Mahoney79787ea2012-03-12 16:03:00 +01002825 return ret;
Naohiro Aota289454a2015-01-06 01:01:03 +09002826 }
Chris Masond1310b22008-01-24 16:13:08 -05002827 bio = NULL;
2828 } else {
Chris Masonda2f0f72015-07-02 13:57:22 -07002829 if (wbc)
2830 wbc_account_io(wbc, page, page_size);
Chris Masond1310b22008-01-24 16:13:08 -05002831 return 0;
2832 }
2833 }
Chris Masonc8b97812008-10-29 14:49:59 -04002834
Kent Overstreetb54ffb72015-05-19 14:31:01 +02002835 bio = btrfs_bio_alloc(bdev, sector, BIO_MAX_PAGES,
2836 GFP_NOFS | __GFP_HIGH);
Tsutomu Itoh5df67082011-02-01 09:17:35 +00002837 if (!bio)
2838 return -ENOMEM;
Chris Mason70dec802008-01-29 09:59:12 -05002839
Chris Masonc8b97812008-10-29 14:49:59 -04002840 bio_add_page(bio, page, page_size, offset);
Chris Masond1310b22008-01-24 16:13:08 -05002841 bio->bi_end_io = end_io_func;
2842 bio->bi_private = tree;
Chris Masonda2f0f72015-07-02 13:57:22 -07002843 if (wbc) {
2844 wbc_init_bio(wbc, bio);
2845 wbc_account_io(wbc, page, page_size);
2846 }
Chris Mason70dec802008-01-29 09:59:12 -05002847
Chris Masond3977122009-01-05 21:25:51 -05002848 if (bio_ret)
Chris Masond1310b22008-01-24 16:13:08 -05002849 *bio_ret = bio;
Chris Masond3977122009-01-05 21:25:51 -05002850 else
Chris Masonc8b97812008-10-29 14:49:59 -04002851 ret = submit_one_bio(rw, bio, mirror_num, bio_flags);
Chris Masond1310b22008-01-24 16:13:08 -05002852
2853 return ret;
2854}
2855
Eric Sandeen48a3b632013-04-25 20:41:01 +00002856static void attach_extent_buffer_page(struct extent_buffer *eb,
2857 struct page *page)
Josef Bacik4f2de97a2012-03-07 16:20:05 -05002858{
2859 if (!PagePrivate(page)) {
2860 SetPagePrivate(page);
2861 page_cache_get(page);
2862 set_page_private(page, (unsigned long)eb);
2863 } else {
2864 WARN_ON(page->private != (unsigned long)eb);
2865 }
2866}
2867
Chris Masond1310b22008-01-24 16:13:08 -05002868void set_page_extent_mapped(struct page *page)
2869{
2870 if (!PagePrivate(page)) {
2871 SetPagePrivate(page);
Chris Masond1310b22008-01-24 16:13:08 -05002872 page_cache_get(page);
Chris Mason6af118ce2008-07-22 11:18:07 -04002873 set_page_private(page, EXTENT_PAGE_PRIVATE);
Chris Masond1310b22008-01-24 16:13:08 -05002874 }
2875}
2876
Miao Xie125bac012013-07-25 19:22:37 +08002877static struct extent_map *
2878__get_extent_map(struct inode *inode, struct page *page, size_t pg_offset,
2879 u64 start, u64 len, get_extent_t *get_extent,
2880 struct extent_map **em_cached)
2881{
2882 struct extent_map *em;
2883
2884 if (em_cached && *em_cached) {
2885 em = *em_cached;
Filipe Mananacbc0e922014-02-25 14:15:12 +00002886 if (extent_map_in_tree(em) && start >= em->start &&
Miao Xie125bac012013-07-25 19:22:37 +08002887 start < extent_map_end(em)) {
2888 atomic_inc(&em->refs);
2889 return em;
2890 }
2891
2892 free_extent_map(em);
2893 *em_cached = NULL;
2894 }
2895
2896 em = get_extent(inode, page, pg_offset, start, len, 0);
2897 if (em_cached && !IS_ERR_OR_NULL(em)) {
2898 BUG_ON(*em_cached);
2899 atomic_inc(&em->refs);
2900 *em_cached = em;
2901 }
2902 return em;
2903}
Chris Masond1310b22008-01-24 16:13:08 -05002904/*
2905 * basic readpage implementation. Locked extent state structs are inserted
2906 * into the tree that are removed when the IO is done (by the end_io
2907 * handlers)
Jeff Mahoney79787ea2012-03-12 16:03:00 +01002908 * XXX JDM: This needs looking at to ensure proper page locking
Chris Masond1310b22008-01-24 16:13:08 -05002909 */
Miao Xie99740902013-07-25 19:22:36 +08002910static int __do_readpage(struct extent_io_tree *tree,
2911 struct page *page,
2912 get_extent_t *get_extent,
Miao Xie125bac012013-07-25 19:22:37 +08002913 struct extent_map **em_cached,
Miao Xie99740902013-07-25 19:22:36 +08002914 struct bio **bio, int mirror_num,
Filipe Manana005efed2015-09-14 09:09:31 +01002915 unsigned long *bio_flags, int rw,
2916 u64 *prev_em_start)
Chris Masond1310b22008-01-24 16:13:08 -05002917{
2918 struct inode *inode = page->mapping->host;
Miao Xie4eee4fa2012-12-21 09:17:45 +00002919 u64 start = page_offset(page);
Chris Masond1310b22008-01-24 16:13:08 -05002920 u64 page_end = start + PAGE_CACHE_SIZE - 1;
2921 u64 end;
2922 u64 cur = start;
2923 u64 extent_offset;
2924 u64 last_byte = i_size_read(inode);
2925 u64 block_start;
2926 u64 cur_end;
2927 sector_t sector;
2928 struct extent_map *em;
2929 struct block_device *bdev;
2930 int ret;
2931 int nr = 0;
Mark Fasheh4b384312013-08-06 11:42:50 -07002932 int parent_locked = *bio_flags & EXTENT_BIO_PARENT_LOCKED;
David Sterba306e16c2011-04-19 14:29:38 +02002933 size_t pg_offset = 0;
Chris Masond1310b22008-01-24 16:13:08 -05002934 size_t iosize;
Chris Masonc8b97812008-10-29 14:49:59 -04002935 size_t disk_io_size;
Chris Masond1310b22008-01-24 16:13:08 -05002936 size_t blocksize = inode->i_sb->s_blocksize;
Mark Fasheh4b384312013-08-06 11:42:50 -07002937 unsigned long this_bio_flag = *bio_flags & EXTENT_BIO_PARENT_LOCKED;
Chris Masond1310b22008-01-24 16:13:08 -05002938
2939 set_page_extent_mapped(page);
2940
Miao Xie99740902013-07-25 19:22:36 +08002941 end = page_end;
Dan Magenheimer90a887c2011-05-26 10:01:56 -06002942 if (!PageUptodate(page)) {
2943 if (cleancache_get_page(page) == 0) {
2944 BUG_ON(blocksize != PAGE_SIZE);
Miao Xie99740902013-07-25 19:22:36 +08002945 unlock_extent(tree, start, end);
Dan Magenheimer90a887c2011-05-26 10:01:56 -06002946 goto out;
2947 }
2948 }
2949
Chris Masonc8b97812008-10-29 14:49:59 -04002950 if (page->index == last_byte >> PAGE_CACHE_SHIFT) {
2951 char *userpage;
2952 size_t zero_offset = last_byte & (PAGE_CACHE_SIZE - 1);
2953
2954 if (zero_offset) {
2955 iosize = PAGE_CACHE_SIZE - zero_offset;
Cong Wang7ac687d2011-11-25 23:14:28 +08002956 userpage = kmap_atomic(page);
Chris Masonc8b97812008-10-29 14:49:59 -04002957 memset(userpage + zero_offset, 0, iosize);
2958 flush_dcache_page(page);
Cong Wang7ac687d2011-11-25 23:14:28 +08002959 kunmap_atomic(userpage);
Chris Masonc8b97812008-10-29 14:49:59 -04002960 }
2961 }
Chris Masond1310b22008-01-24 16:13:08 -05002962 while (cur <= end) {
Josef Bacikc8f2f242013-02-11 11:33:00 -05002963 unsigned long pnr = (last_byte >> PAGE_CACHE_SHIFT) + 1;
Filipe Manana005efed2015-09-14 09:09:31 +01002964 bool force_bio_submit = false;
Josef Bacikc8f2f242013-02-11 11:33:00 -05002965
Chris Masond1310b22008-01-24 16:13:08 -05002966 if (cur >= last_byte) {
2967 char *userpage;
Arne Jansen507903b2011-04-06 10:02:20 +00002968 struct extent_state *cached = NULL;
2969
David Sterba306e16c2011-04-19 14:29:38 +02002970 iosize = PAGE_CACHE_SIZE - pg_offset;
Cong Wang7ac687d2011-11-25 23:14:28 +08002971 userpage = kmap_atomic(page);
David Sterba306e16c2011-04-19 14:29:38 +02002972 memset(userpage + pg_offset, 0, iosize);
Chris Masond1310b22008-01-24 16:13:08 -05002973 flush_dcache_page(page);
Cong Wang7ac687d2011-11-25 23:14:28 +08002974 kunmap_atomic(userpage);
Chris Masond1310b22008-01-24 16:13:08 -05002975 set_extent_uptodate(tree, cur, cur + iosize - 1,
Arne Jansen507903b2011-04-06 10:02:20 +00002976 &cached, GFP_NOFS);
Mark Fasheh4b384312013-08-06 11:42:50 -07002977 if (!parent_locked)
2978 unlock_extent_cached(tree, cur,
2979 cur + iosize - 1,
2980 &cached, GFP_NOFS);
Chris Masond1310b22008-01-24 16:13:08 -05002981 break;
2982 }
Miao Xie125bac012013-07-25 19:22:37 +08002983 em = __get_extent_map(inode, page, pg_offset, cur,
2984 end - cur + 1, get_extent, em_cached);
David Sterbac7040052011-04-19 18:00:01 +02002985 if (IS_ERR_OR_NULL(em)) {
Chris Masond1310b22008-01-24 16:13:08 -05002986 SetPageError(page);
Mark Fasheh4b384312013-08-06 11:42:50 -07002987 if (!parent_locked)
2988 unlock_extent(tree, cur, end);
Chris Masond1310b22008-01-24 16:13:08 -05002989 break;
2990 }
Chris Masond1310b22008-01-24 16:13:08 -05002991 extent_offset = cur - em->start;
2992 BUG_ON(extent_map_end(em) <= cur);
2993 BUG_ON(end < cur);
2994
Li Zefan261507a02010-12-17 14:21:50 +08002995 if (test_bit(EXTENT_FLAG_COMPRESSED, &em->flags)) {
Mark Fasheh4b384312013-08-06 11:42:50 -07002996 this_bio_flag |= EXTENT_BIO_COMPRESSED;
Li Zefan261507a02010-12-17 14:21:50 +08002997 extent_set_compress_type(&this_bio_flag,
2998 em->compress_type);
2999 }
Chris Masonc8b97812008-10-29 14:49:59 -04003000
Chris Masond1310b22008-01-24 16:13:08 -05003001 iosize = min(extent_map_end(em) - cur, end - cur + 1);
3002 cur_end = min(extent_map_end(em) - 1, end);
Qu Wenruofda28322013-02-26 08:10:22 +00003003 iosize = ALIGN(iosize, blocksize);
Chris Masonc8b97812008-10-29 14:49:59 -04003004 if (this_bio_flag & EXTENT_BIO_COMPRESSED) {
3005 disk_io_size = em->block_len;
3006 sector = em->block_start >> 9;
3007 } else {
3008 sector = (em->block_start + extent_offset) >> 9;
3009 disk_io_size = iosize;
3010 }
Chris Masond1310b22008-01-24 16:13:08 -05003011 bdev = em->bdev;
3012 block_start = em->block_start;
Yan Zhengd899e052008-10-30 14:25:28 -04003013 if (test_bit(EXTENT_FLAG_PREALLOC, &em->flags))
3014 block_start = EXTENT_MAP_HOLE;
Filipe Manana005efed2015-09-14 09:09:31 +01003015
3016 /*
3017 * If we have a file range that points to a compressed extent
3018 * and it's followed by a consecutive file range that points to
3019 * to the same compressed extent (possibly with a different
3020 * offset and/or length, so it either points to the whole extent
3021 * or only part of it), we must make sure we do not submit a
3022 * single bio to populate the pages for the 2 ranges because
3023 * this makes the compressed extent read zero out the pages
3024 * belonging to the 2nd range. Imagine the following scenario:
3025 *
3026 * File layout
3027 * [0 - 8K] [8K - 24K]
3028 * | |
3029 * | |
3030 * points to extent X, points to extent X,
3031 * offset 4K, length of 8K offset 0, length 16K
3032 *
3033 * [extent X, compressed length = 4K uncompressed length = 16K]
3034 *
3035 * If the bio to read the compressed extent covers both ranges,
3036 * it will decompress extent X into the pages belonging to the
3037 * first range and then it will stop, zeroing out the remaining
3038 * pages that belong to the other range that points to extent X.
3039 * So here we make sure we submit 2 bios, one for the first
3040 * range and another one for the third range. Both will target
3041 * the same physical extent from disk, but we can't currently
3042 * make the compressed bio endio callback populate the pages
3043 * for both ranges because each compressed bio is tightly
3044 * coupled with a single extent map, and each range can have
3045 * an extent map with a different offset value relative to the
3046 * uncompressed data of our extent and different lengths. This
3047 * is a corner case so we prioritize correctness over
3048 * non-optimal behavior (submitting 2 bios for the same extent).
3049 */
3050 if (test_bit(EXTENT_FLAG_COMPRESSED, &em->flags) &&
3051 prev_em_start && *prev_em_start != (u64)-1 &&
3052 *prev_em_start != em->orig_start)
3053 force_bio_submit = true;
3054
3055 if (prev_em_start)
3056 *prev_em_start = em->orig_start;
3057
Chris Masond1310b22008-01-24 16:13:08 -05003058 free_extent_map(em);
3059 em = NULL;
3060
3061 /* we've found a hole, just zero and go on */
3062 if (block_start == EXTENT_MAP_HOLE) {
3063 char *userpage;
Arne Jansen507903b2011-04-06 10:02:20 +00003064 struct extent_state *cached = NULL;
3065
Cong Wang7ac687d2011-11-25 23:14:28 +08003066 userpage = kmap_atomic(page);
David Sterba306e16c2011-04-19 14:29:38 +02003067 memset(userpage + pg_offset, 0, iosize);
Chris Masond1310b22008-01-24 16:13:08 -05003068 flush_dcache_page(page);
Cong Wang7ac687d2011-11-25 23:14:28 +08003069 kunmap_atomic(userpage);
Chris Masond1310b22008-01-24 16:13:08 -05003070
3071 set_extent_uptodate(tree, cur, cur + iosize - 1,
Arne Jansen507903b2011-04-06 10:02:20 +00003072 &cached, GFP_NOFS);
3073 unlock_extent_cached(tree, cur, cur + iosize - 1,
3074 &cached, GFP_NOFS);
Chris Masond1310b22008-01-24 16:13:08 -05003075 cur = cur + iosize;
David Sterba306e16c2011-04-19 14:29:38 +02003076 pg_offset += iosize;
Chris Masond1310b22008-01-24 16:13:08 -05003077 continue;
3078 }
3079 /* the get_extent function already copied into the page */
Chris Mason9655d292009-09-02 15:22:30 -04003080 if (test_range_bit(tree, cur, cur_end,
3081 EXTENT_UPTODATE, 1, NULL)) {
Chris Masona1b32a52008-09-05 16:09:51 -04003082 check_page_uptodate(tree, page);
Mark Fasheh4b384312013-08-06 11:42:50 -07003083 if (!parent_locked)
3084 unlock_extent(tree, cur, cur + iosize - 1);
Chris Masond1310b22008-01-24 16:13:08 -05003085 cur = cur + iosize;
David Sterba306e16c2011-04-19 14:29:38 +02003086 pg_offset += iosize;
Chris Masond1310b22008-01-24 16:13:08 -05003087 continue;
3088 }
Chris Mason70dec802008-01-29 09:59:12 -05003089 /* we have an inline extent but it didn't get marked up
3090 * to date. Error out
3091 */
3092 if (block_start == EXTENT_MAP_INLINE) {
3093 SetPageError(page);
Mark Fasheh4b384312013-08-06 11:42:50 -07003094 if (!parent_locked)
3095 unlock_extent(tree, cur, cur + iosize - 1);
Chris Mason70dec802008-01-29 09:59:12 -05003096 cur = cur + iosize;
David Sterba306e16c2011-04-19 14:29:38 +02003097 pg_offset += iosize;
Chris Mason70dec802008-01-29 09:59:12 -05003098 continue;
3099 }
Chris Masond1310b22008-01-24 16:13:08 -05003100
Josef Bacikc8f2f242013-02-11 11:33:00 -05003101 pnr -= page->index;
Chris Masonda2f0f72015-07-02 13:57:22 -07003102 ret = submit_extent_page(rw, tree, NULL, page,
David Sterba306e16c2011-04-19 14:29:38 +02003103 sector, disk_io_size, pg_offset,
Chris Mason89642222008-07-24 09:41:53 -04003104 bdev, bio, pnr,
Chris Masonc8b97812008-10-29 14:49:59 -04003105 end_bio_extent_readpage, mirror_num,
3106 *bio_flags,
Filipe Manana005efed2015-09-14 09:09:31 +01003107 this_bio_flag,
3108 force_bio_submit);
Josef Bacikc8f2f242013-02-11 11:33:00 -05003109 if (!ret) {
3110 nr++;
3111 *bio_flags = this_bio_flag;
3112 } else {
Chris Masond1310b22008-01-24 16:13:08 -05003113 SetPageError(page);
Mark Fasheh4b384312013-08-06 11:42:50 -07003114 if (!parent_locked)
3115 unlock_extent(tree, cur, cur + iosize - 1);
Josef Bacikedd33c92012-10-05 16:40:32 -04003116 }
Chris Masond1310b22008-01-24 16:13:08 -05003117 cur = cur + iosize;
David Sterba306e16c2011-04-19 14:29:38 +02003118 pg_offset += iosize;
Chris Masond1310b22008-01-24 16:13:08 -05003119 }
Dan Magenheimer90a887c2011-05-26 10:01:56 -06003120out:
Chris Masond1310b22008-01-24 16:13:08 -05003121 if (!nr) {
3122 if (!PageError(page))
3123 SetPageUptodate(page);
3124 unlock_page(page);
3125 }
3126 return 0;
3127}
3128
Miao Xie99740902013-07-25 19:22:36 +08003129static inline void __do_contiguous_readpages(struct extent_io_tree *tree,
3130 struct page *pages[], int nr_pages,
3131 u64 start, u64 end,
3132 get_extent_t *get_extent,
Miao Xie125bac012013-07-25 19:22:37 +08003133 struct extent_map **em_cached,
Miao Xie99740902013-07-25 19:22:36 +08003134 struct bio **bio, int mirror_num,
3135 unsigned long *bio_flags, int rw)
3136{
3137 struct inode *inode;
3138 struct btrfs_ordered_extent *ordered;
3139 int index;
Filipe Manana005efed2015-09-14 09:09:31 +01003140 u64 prev_em_start = (u64)-1;
Miao Xie99740902013-07-25 19:22:36 +08003141
3142 inode = pages[0]->mapping->host;
3143 while (1) {
3144 lock_extent(tree, start, end);
3145 ordered = btrfs_lookup_ordered_range(inode, start,
3146 end - start + 1);
3147 if (!ordered)
3148 break;
3149 unlock_extent(tree, start, end);
3150 btrfs_start_ordered_extent(inode, ordered, 1);
3151 btrfs_put_ordered_extent(ordered);
3152 }
3153
3154 for (index = 0; index < nr_pages; index++) {
Miao Xie125bac012013-07-25 19:22:37 +08003155 __do_readpage(tree, pages[index], get_extent, em_cached, bio,
Filipe Manana005efed2015-09-14 09:09:31 +01003156 mirror_num, bio_flags, rw, &prev_em_start);
Miao Xie99740902013-07-25 19:22:36 +08003157 page_cache_release(pages[index]);
3158 }
3159}
3160
3161static void __extent_readpages(struct extent_io_tree *tree,
3162 struct page *pages[],
3163 int nr_pages, get_extent_t *get_extent,
Miao Xie125bac012013-07-25 19:22:37 +08003164 struct extent_map **em_cached,
Miao Xie99740902013-07-25 19:22:36 +08003165 struct bio **bio, int mirror_num,
3166 unsigned long *bio_flags, int rw)
3167{
Stefan Behrens35a36212013-08-14 18:12:25 +02003168 u64 start = 0;
Miao Xie99740902013-07-25 19:22:36 +08003169 u64 end = 0;
3170 u64 page_start;
3171 int index;
Stefan Behrens35a36212013-08-14 18:12:25 +02003172 int first_index = 0;
Miao Xie99740902013-07-25 19:22:36 +08003173
3174 for (index = 0; index < nr_pages; index++) {
3175 page_start = page_offset(pages[index]);
3176 if (!end) {
3177 start = page_start;
3178 end = start + PAGE_CACHE_SIZE - 1;
3179 first_index = index;
3180 } else if (end + 1 == page_start) {
3181 end += PAGE_CACHE_SIZE;
3182 } else {
3183 __do_contiguous_readpages(tree, &pages[first_index],
3184 index - first_index, start,
Miao Xie125bac012013-07-25 19:22:37 +08003185 end, get_extent, em_cached,
3186 bio, mirror_num, bio_flags,
3187 rw);
Miao Xie99740902013-07-25 19:22:36 +08003188 start = page_start;
3189 end = start + PAGE_CACHE_SIZE - 1;
3190 first_index = index;
3191 }
3192 }
3193
3194 if (end)
3195 __do_contiguous_readpages(tree, &pages[first_index],
3196 index - first_index, start,
Miao Xie125bac012013-07-25 19:22:37 +08003197 end, get_extent, em_cached, bio,
Miao Xie99740902013-07-25 19:22:36 +08003198 mirror_num, bio_flags, rw);
3199}
3200
3201static int __extent_read_full_page(struct extent_io_tree *tree,
3202 struct page *page,
3203 get_extent_t *get_extent,
3204 struct bio **bio, int mirror_num,
3205 unsigned long *bio_flags, int rw)
3206{
3207 struct inode *inode = page->mapping->host;
3208 struct btrfs_ordered_extent *ordered;
3209 u64 start = page_offset(page);
3210 u64 end = start + PAGE_CACHE_SIZE - 1;
3211 int ret;
3212
3213 while (1) {
3214 lock_extent(tree, start, end);
3215 ordered = btrfs_lookup_ordered_extent(inode, start);
3216 if (!ordered)
3217 break;
3218 unlock_extent(tree, start, end);
3219 btrfs_start_ordered_extent(inode, ordered, 1);
3220 btrfs_put_ordered_extent(ordered);
3221 }
3222
Miao Xie125bac012013-07-25 19:22:37 +08003223 ret = __do_readpage(tree, page, get_extent, NULL, bio, mirror_num,
Filipe Manana005efed2015-09-14 09:09:31 +01003224 bio_flags, rw, NULL);
Miao Xie99740902013-07-25 19:22:36 +08003225 return ret;
3226}
3227
Chris Masond1310b22008-01-24 16:13:08 -05003228int extent_read_full_page(struct extent_io_tree *tree, struct page *page,
Jan Schmidt8ddc7d92011-06-13 20:02:58 +02003229 get_extent_t *get_extent, int mirror_num)
Chris Masond1310b22008-01-24 16:13:08 -05003230{
3231 struct bio *bio = NULL;
Chris Masonc8b97812008-10-29 14:49:59 -04003232 unsigned long bio_flags = 0;
Chris Masond1310b22008-01-24 16:13:08 -05003233 int ret;
3234
Jan Schmidt8ddc7d92011-06-13 20:02:58 +02003235 ret = __extent_read_full_page(tree, page, get_extent, &bio, mirror_num,
Josef Bacikd4c7ca82013-04-19 19:49:09 -04003236 &bio_flags, READ);
Chris Masond1310b22008-01-24 16:13:08 -05003237 if (bio)
Jan Schmidt8ddc7d92011-06-13 20:02:58 +02003238 ret = submit_one_bio(READ, bio, mirror_num, bio_flags);
Chris Masond1310b22008-01-24 16:13:08 -05003239 return ret;
3240}
Chris Masond1310b22008-01-24 16:13:08 -05003241
Mark Fasheh4b384312013-08-06 11:42:50 -07003242int extent_read_full_page_nolock(struct extent_io_tree *tree, struct page *page,
3243 get_extent_t *get_extent, int mirror_num)
3244{
3245 struct bio *bio = NULL;
3246 unsigned long bio_flags = EXTENT_BIO_PARENT_LOCKED;
3247 int ret;
3248
3249 ret = __do_readpage(tree, page, get_extent, NULL, &bio, mirror_num,
Filipe Manana005efed2015-09-14 09:09:31 +01003250 &bio_flags, READ, NULL);
Mark Fasheh4b384312013-08-06 11:42:50 -07003251 if (bio)
3252 ret = submit_one_bio(READ, bio, mirror_num, bio_flags);
3253 return ret;
3254}
3255
Chris Mason11c83492009-04-20 15:50:09 -04003256static noinline void update_nr_written(struct page *page,
3257 struct writeback_control *wbc,
3258 unsigned long nr_written)
3259{
3260 wbc->nr_to_write -= nr_written;
3261 if (wbc->range_cyclic || (wbc->nr_to_write > 0 &&
3262 wbc->range_start == 0 && wbc->range_end == LLONG_MAX))
3263 page->mapping->writeback_index = page->index + nr_written;
3264}
3265
Chris Masond1310b22008-01-24 16:13:08 -05003266/*
Chris Mason40f76582014-05-21 13:35:51 -07003267 * helper for __extent_writepage, doing all of the delayed allocation setup.
3268 *
3269 * This returns 1 if our fill_delalloc function did all the work required
3270 * to write the page (copy into inline extent). In this case the IO has
3271 * been started and the page is already unlocked.
3272 *
3273 * This returns 0 if all went well (page still locked)
3274 * This returns < 0 if there were errors (page still locked)
Chris Masond1310b22008-01-24 16:13:08 -05003275 */
Chris Mason40f76582014-05-21 13:35:51 -07003276static noinline_for_stack int writepage_delalloc(struct inode *inode,
3277 struct page *page, struct writeback_control *wbc,
3278 struct extent_page_data *epd,
3279 u64 delalloc_start,
3280 unsigned long *nr_written)
Chris Masond1310b22008-01-24 16:13:08 -05003281{
Chris Mason40f76582014-05-21 13:35:51 -07003282 struct extent_io_tree *tree = epd->tree;
3283 u64 page_end = delalloc_start + PAGE_CACHE_SIZE - 1;
3284 u64 nr_delalloc;
3285 u64 delalloc_to_write = 0;
3286 u64 delalloc_end = 0;
3287 int ret;
3288 int page_started = 0;
3289
3290 if (epd->extent_locked || !tree->ops || !tree->ops->fill_delalloc)
3291 return 0;
3292
3293 while (delalloc_end < page_end) {
3294 nr_delalloc = find_lock_delalloc_range(inode, tree,
3295 page,
3296 &delalloc_start,
3297 &delalloc_end,
Josef Bacikdcab6a32015-02-11 15:08:59 -05003298 BTRFS_MAX_EXTENT_SIZE);
Chris Mason40f76582014-05-21 13:35:51 -07003299 if (nr_delalloc == 0) {
3300 delalloc_start = delalloc_end + 1;
3301 continue;
3302 }
3303 ret = tree->ops->fill_delalloc(inode, page,
3304 delalloc_start,
3305 delalloc_end,
3306 &page_started,
3307 nr_written);
3308 /* File system has been set read-only */
3309 if (ret) {
3310 SetPageError(page);
3311 /* fill_delalloc should be return < 0 for error
3312 * but just in case, we use > 0 here meaning the
3313 * IO is started, so we don't want to return > 0
3314 * unless things are going well.
3315 */
3316 ret = ret < 0 ? ret : -EIO;
3317 goto done;
3318 }
3319 /*
3320 * delalloc_end is already one less than the total
3321 * length, so we don't subtract one from
3322 * PAGE_CACHE_SIZE
3323 */
3324 delalloc_to_write += (delalloc_end - delalloc_start +
3325 PAGE_CACHE_SIZE) >>
3326 PAGE_CACHE_SHIFT;
3327 delalloc_start = delalloc_end + 1;
3328 }
3329 if (wbc->nr_to_write < delalloc_to_write) {
3330 int thresh = 8192;
3331
3332 if (delalloc_to_write < thresh * 2)
3333 thresh = delalloc_to_write;
3334 wbc->nr_to_write = min_t(u64, delalloc_to_write,
3335 thresh);
3336 }
3337
3338 /* did the fill delalloc function already unlock and start
3339 * the IO?
3340 */
3341 if (page_started) {
3342 /*
3343 * we've unlocked the page, so we can't update
3344 * the mapping's writeback index, just update
3345 * nr_to_write.
3346 */
3347 wbc->nr_to_write -= *nr_written;
3348 return 1;
3349 }
3350
3351 ret = 0;
3352
3353done:
3354 return ret;
3355}
3356
3357/*
3358 * helper for __extent_writepage. This calls the writepage start hooks,
3359 * and does the loop to map the page into extents and bios.
3360 *
3361 * We return 1 if the IO is started and the page is unlocked,
3362 * 0 if all went well (page still locked)
3363 * < 0 if there were errors (page still locked)
3364 */
3365static noinline_for_stack int __extent_writepage_io(struct inode *inode,
3366 struct page *page,
3367 struct writeback_control *wbc,
3368 struct extent_page_data *epd,
3369 loff_t i_size,
3370 unsigned long nr_written,
3371 int write_flags, int *nr_ret)
3372{
Chris Masond1310b22008-01-24 16:13:08 -05003373 struct extent_io_tree *tree = epd->tree;
Miao Xie4eee4fa2012-12-21 09:17:45 +00003374 u64 start = page_offset(page);
Chris Masond1310b22008-01-24 16:13:08 -05003375 u64 page_end = start + PAGE_CACHE_SIZE - 1;
3376 u64 end;
3377 u64 cur = start;
3378 u64 extent_offset;
Chris Masond1310b22008-01-24 16:13:08 -05003379 u64 block_start;
3380 u64 iosize;
3381 sector_t sector;
Chris Mason2c64c532009-09-02 15:04:12 -04003382 struct extent_state *cached_state = NULL;
Chris Masond1310b22008-01-24 16:13:08 -05003383 struct extent_map *em;
3384 struct block_device *bdev;
Chris Mason7f3c74f2008-07-18 12:01:11 -04003385 size_t pg_offset = 0;
Chris Masond1310b22008-01-24 16:13:08 -05003386 size_t blocksize;
Chris Mason40f76582014-05-21 13:35:51 -07003387 int ret = 0;
3388 int nr = 0;
3389 bool compressed;
Chris Masond1310b22008-01-24 16:13:08 -05003390
Chris Mason247e7432008-07-17 12:53:51 -04003391 if (tree->ops && tree->ops->writepage_start_hook) {
Chris Masonc8b97812008-10-29 14:49:59 -04003392 ret = tree->ops->writepage_start_hook(page, start,
3393 page_end);
Jeff Mahoney87826df2012-02-15 16:23:57 +01003394 if (ret) {
3395 /* Fixup worker will requeue */
3396 if (ret == -EBUSY)
3397 wbc->pages_skipped++;
3398 else
3399 redirty_page_for_writepage(wbc, page);
Chris Mason40f76582014-05-21 13:35:51 -07003400
Chris Mason11c83492009-04-20 15:50:09 -04003401 update_nr_written(page, wbc, nr_written);
Chris Mason247e7432008-07-17 12:53:51 -04003402 unlock_page(page);
Chris Mason40f76582014-05-21 13:35:51 -07003403 ret = 1;
Chris Mason11c83492009-04-20 15:50:09 -04003404 goto done_unlocked;
Chris Mason247e7432008-07-17 12:53:51 -04003405 }
3406 }
3407
Chris Mason11c83492009-04-20 15:50:09 -04003408 /*
3409 * we don't want to touch the inode after unlocking the page,
3410 * so we update the mapping writeback index now
3411 */
3412 update_nr_written(page, wbc, nr_written + 1);
Chris Mason771ed682008-11-06 22:02:51 -05003413
Chris Masond1310b22008-01-24 16:13:08 -05003414 end = page_end;
Chris Mason40f76582014-05-21 13:35:51 -07003415 if (i_size <= start) {
Chris Masone6dcd2d2008-07-17 12:53:50 -04003416 if (tree->ops && tree->ops->writepage_end_io_hook)
3417 tree->ops->writepage_end_io_hook(page, start,
3418 page_end, NULL, 1);
Chris Masond1310b22008-01-24 16:13:08 -05003419 goto done;
3420 }
3421
Chris Masond1310b22008-01-24 16:13:08 -05003422 blocksize = inode->i_sb->s_blocksize;
3423
3424 while (cur <= end) {
Chris Mason40f76582014-05-21 13:35:51 -07003425 u64 em_end;
3426 if (cur >= i_size) {
Chris Masone6dcd2d2008-07-17 12:53:50 -04003427 if (tree->ops && tree->ops->writepage_end_io_hook)
3428 tree->ops->writepage_end_io_hook(page, cur,
3429 page_end, NULL, 1);
Chris Masond1310b22008-01-24 16:13:08 -05003430 break;
3431 }
Chris Mason7f3c74f2008-07-18 12:01:11 -04003432 em = epd->get_extent(inode, page, pg_offset, cur,
Chris Masond1310b22008-01-24 16:13:08 -05003433 end - cur + 1, 1);
David Sterbac7040052011-04-19 18:00:01 +02003434 if (IS_ERR_OR_NULL(em)) {
Chris Masond1310b22008-01-24 16:13:08 -05003435 SetPageError(page);
Filipe Manana61391d52014-05-09 17:17:40 +01003436 ret = PTR_ERR_OR_ZERO(em);
Chris Masond1310b22008-01-24 16:13:08 -05003437 break;
3438 }
3439
3440 extent_offset = cur - em->start;
Chris Mason40f76582014-05-21 13:35:51 -07003441 em_end = extent_map_end(em);
3442 BUG_ON(em_end <= cur);
Chris Masond1310b22008-01-24 16:13:08 -05003443 BUG_ON(end < cur);
Chris Mason40f76582014-05-21 13:35:51 -07003444 iosize = min(em_end - cur, end - cur + 1);
Qu Wenruofda28322013-02-26 08:10:22 +00003445 iosize = ALIGN(iosize, blocksize);
Chris Masond1310b22008-01-24 16:13:08 -05003446 sector = (em->block_start + extent_offset) >> 9;
3447 bdev = em->bdev;
3448 block_start = em->block_start;
Chris Masonc8b97812008-10-29 14:49:59 -04003449 compressed = test_bit(EXTENT_FLAG_COMPRESSED, &em->flags);
Chris Masond1310b22008-01-24 16:13:08 -05003450 free_extent_map(em);
3451 em = NULL;
3452
Chris Masonc8b97812008-10-29 14:49:59 -04003453 /*
3454 * compressed and inline extents are written through other
3455 * paths in the FS
3456 */
3457 if (compressed || block_start == EXTENT_MAP_HOLE ||
Chris Masond1310b22008-01-24 16:13:08 -05003458 block_start == EXTENT_MAP_INLINE) {
Chris Masonc8b97812008-10-29 14:49:59 -04003459 /*
3460 * end_io notification does not happen here for
3461 * compressed extents
3462 */
3463 if (!compressed && tree->ops &&
3464 tree->ops->writepage_end_io_hook)
Chris Masone6dcd2d2008-07-17 12:53:50 -04003465 tree->ops->writepage_end_io_hook(page, cur,
3466 cur + iosize - 1,
3467 NULL, 1);
Chris Masonc8b97812008-10-29 14:49:59 -04003468 else if (compressed) {
3469 /* we don't want to end_page_writeback on
3470 * a compressed extent. this happens
3471 * elsewhere
3472 */
3473 nr++;
3474 }
3475
3476 cur += iosize;
Chris Mason7f3c74f2008-07-18 12:01:11 -04003477 pg_offset += iosize;
Chris Masond1310b22008-01-24 16:13:08 -05003478 continue;
3479 }
Chris Masonc8b97812008-10-29 14:49:59 -04003480
Chris Masond1310b22008-01-24 16:13:08 -05003481 if (tree->ops && tree->ops->writepage_io_hook) {
3482 ret = tree->ops->writepage_io_hook(page, cur,
3483 cur + iosize - 1);
3484 } else {
3485 ret = 0;
3486 }
Chris Mason1259ab72008-05-12 13:39:03 -04003487 if (ret) {
Chris Masond1310b22008-01-24 16:13:08 -05003488 SetPageError(page);
Chris Mason1259ab72008-05-12 13:39:03 -04003489 } else {
Chris Mason40f76582014-05-21 13:35:51 -07003490 unsigned long max_nr = (i_size >> PAGE_CACHE_SHIFT) + 1;
Chris Mason7f3c74f2008-07-18 12:01:11 -04003491
Chris Masond1310b22008-01-24 16:13:08 -05003492 set_range_writeback(tree, cur, cur + iosize - 1);
3493 if (!PageWriteback(page)) {
Frank Holtonefe120a2013-12-20 11:37:06 -05003494 btrfs_err(BTRFS_I(inode)->root->fs_info,
3495 "page %lu not writeback, cur %llu end %llu",
Geert Uytterhoevenc1c9ff72013-08-20 13:20:07 +02003496 page->index, cur, end);
Chris Masond1310b22008-01-24 16:13:08 -05003497 }
3498
Chris Masonda2f0f72015-07-02 13:57:22 -07003499 ret = submit_extent_page(write_flags, tree, wbc, page,
Chris Masonffbd5172009-04-20 15:50:09 -04003500 sector, iosize, pg_offset,
3501 bdev, &epd->bio, max_nr,
Chris Masonc8b97812008-10-29 14:49:59 -04003502 end_bio_extent_writepage,
Filipe Manana005efed2015-09-14 09:09:31 +01003503 0, 0, 0, false);
Chris Masond1310b22008-01-24 16:13:08 -05003504 if (ret)
3505 SetPageError(page);
3506 }
3507 cur = cur + iosize;
Chris Mason7f3c74f2008-07-18 12:01:11 -04003508 pg_offset += iosize;
Chris Masond1310b22008-01-24 16:13:08 -05003509 nr++;
3510 }
3511done:
Chris Mason40f76582014-05-21 13:35:51 -07003512 *nr_ret = nr;
Chris Mason771ed682008-11-06 22:02:51 -05003513
Chris Mason11c83492009-04-20 15:50:09 -04003514done_unlocked:
3515
Chris Mason2c64c532009-09-02 15:04:12 -04003516 /* drop our reference on any cached states */
3517 free_extent_state(cached_state);
Chris Mason40f76582014-05-21 13:35:51 -07003518 return ret;
3519}
3520
3521/*
3522 * the writepage semantics are similar to regular writepage. extent
3523 * records are inserted to lock ranges in the tree, and as dirty areas
3524 * are found, they are marked writeback. Then the lock bits are removed
3525 * and the end_io handler clears the writeback ranges
3526 */
3527static int __extent_writepage(struct page *page, struct writeback_control *wbc,
3528 void *data)
3529{
3530 struct inode *inode = page->mapping->host;
3531 struct extent_page_data *epd = data;
3532 u64 start = page_offset(page);
3533 u64 page_end = start + PAGE_CACHE_SIZE - 1;
3534 int ret;
3535 int nr = 0;
3536 size_t pg_offset = 0;
3537 loff_t i_size = i_size_read(inode);
3538 unsigned long end_index = i_size >> PAGE_CACHE_SHIFT;
3539 int write_flags;
3540 unsigned long nr_written = 0;
3541
3542 if (wbc->sync_mode == WB_SYNC_ALL)
3543 write_flags = WRITE_SYNC;
3544 else
3545 write_flags = WRITE;
3546
3547 trace___extent_writepage(page, inode, wbc);
3548
3549 WARN_ON(!PageLocked(page));
3550
3551 ClearPageError(page);
3552
3553 pg_offset = i_size & (PAGE_CACHE_SIZE - 1);
3554 if (page->index > end_index ||
3555 (page->index == end_index && !pg_offset)) {
3556 page->mapping->a_ops->invalidatepage(page, 0, PAGE_CACHE_SIZE);
3557 unlock_page(page);
3558 return 0;
3559 }
3560
3561 if (page->index == end_index) {
3562 char *userpage;
3563
3564 userpage = kmap_atomic(page);
3565 memset(userpage + pg_offset, 0,
3566 PAGE_CACHE_SIZE - pg_offset);
3567 kunmap_atomic(userpage);
3568 flush_dcache_page(page);
3569 }
3570
3571 pg_offset = 0;
3572
3573 set_page_extent_mapped(page);
3574
3575 ret = writepage_delalloc(inode, page, wbc, epd, start, &nr_written);
3576 if (ret == 1)
3577 goto done_unlocked;
3578 if (ret)
3579 goto done;
3580
3581 ret = __extent_writepage_io(inode, page, wbc, epd,
3582 i_size, nr_written, write_flags, &nr);
3583 if (ret == 1)
3584 goto done_unlocked;
3585
3586done:
Chris Masone6dcd2d2008-07-17 12:53:50 -04003587 if (nr == 0) {
Chris Masond1310b22008-01-24 16:13:08 -05003588 /* make sure the mapping tag for page dirty gets cleared */
Chris Mason771ed682008-11-06 22:02:51 -05003589 set_page_writeback(page);
3590 end_page_writeback(page);
3591 }
Filipe Manana61391d52014-05-09 17:17:40 +01003592 if (PageError(page)) {
3593 ret = ret < 0 ? ret : -EIO;
3594 end_extent_writepage(page, ret, start, page_end);
3595 }
Christoph Hellwigb2950862008-12-02 09:54:17 -05003596 unlock_page(page);
Chris Mason40f76582014-05-21 13:35:51 -07003597 return ret;
Chris Mason4bef0842008-09-08 11:18:08 -04003598
3599done_unlocked:
Chris Masond1310b22008-01-24 16:13:08 -05003600 return 0;
3601}
3602
Josef Bacikfd8b2b62013-04-24 16:41:19 -04003603void wait_on_extent_buffer_writeback(struct extent_buffer *eb)
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003604{
NeilBrown74316202014-07-07 15:16:04 +10003605 wait_on_bit_io(&eb->bflags, EXTENT_BUFFER_WRITEBACK,
3606 TASK_UNINTERRUPTIBLE);
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003607}
3608
Chris Mason0e378df2014-05-19 20:55:27 -07003609static noinline_for_stack int
3610lock_extent_buffer_for_io(struct extent_buffer *eb,
3611 struct btrfs_fs_info *fs_info,
3612 struct extent_page_data *epd)
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003613{
3614 unsigned long i, num_pages;
3615 int flush = 0;
3616 int ret = 0;
3617
3618 if (!btrfs_try_tree_write_lock(eb)) {
3619 flush = 1;
3620 flush_write_bio(epd);
3621 btrfs_tree_lock(eb);
3622 }
3623
3624 if (test_bit(EXTENT_BUFFER_WRITEBACK, &eb->bflags)) {
3625 btrfs_tree_unlock(eb);
3626 if (!epd->sync_io)
3627 return 0;
3628 if (!flush) {
3629 flush_write_bio(epd);
3630 flush = 1;
3631 }
Chris Masona098d8e2012-03-21 12:09:56 -04003632 while (1) {
3633 wait_on_extent_buffer_writeback(eb);
3634 btrfs_tree_lock(eb);
3635 if (!test_bit(EXTENT_BUFFER_WRITEBACK, &eb->bflags))
3636 break;
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003637 btrfs_tree_unlock(eb);
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003638 }
3639 }
3640
Josef Bacik51561ff2012-07-20 16:25:24 -04003641 /*
3642 * We need to do this to prevent races in people who check if the eb is
3643 * under IO since we can end up having no IO bits set for a short period
3644 * of time.
3645 */
3646 spin_lock(&eb->refs_lock);
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003647 if (test_and_clear_bit(EXTENT_BUFFER_DIRTY, &eb->bflags)) {
3648 set_bit(EXTENT_BUFFER_WRITEBACK, &eb->bflags);
Josef Bacik51561ff2012-07-20 16:25:24 -04003649 spin_unlock(&eb->refs_lock);
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003650 btrfs_set_header_flag(eb, BTRFS_HEADER_FLAG_WRITTEN);
Miao Xiee2d84522013-01-29 10:09:20 +00003651 __percpu_counter_add(&fs_info->dirty_metadata_bytes,
3652 -eb->len,
3653 fs_info->dirty_metadata_batch);
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003654 ret = 1;
Josef Bacik51561ff2012-07-20 16:25:24 -04003655 } else {
3656 spin_unlock(&eb->refs_lock);
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003657 }
3658
3659 btrfs_tree_unlock(eb);
3660
3661 if (!ret)
3662 return ret;
3663
3664 num_pages = num_extent_pages(eb->start, eb->len);
3665 for (i = 0; i < num_pages; i++) {
David Sterbafb85fc92014-07-31 01:03:53 +02003666 struct page *p = eb->pages[i];
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003667
3668 if (!trylock_page(p)) {
3669 if (!flush) {
3670 flush_write_bio(epd);
3671 flush = 1;
3672 }
3673 lock_page(p);
3674 }
3675 }
3676
3677 return ret;
3678}
3679
3680static void end_extent_buffer_writeback(struct extent_buffer *eb)
3681{
3682 clear_bit(EXTENT_BUFFER_WRITEBACK, &eb->bflags);
Peter Zijlstra4e857c52014-03-17 18:06:10 +01003683 smp_mb__after_atomic();
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003684 wake_up_bit(&eb->bflags, EXTENT_BUFFER_WRITEBACK);
3685}
3686
Filipe Manana656f30d2014-09-26 12:25:56 +01003687static void set_btree_ioerr(struct page *page)
3688{
3689 struct extent_buffer *eb = (struct extent_buffer *)page->private;
3690 struct btrfs_inode *btree_ino = BTRFS_I(eb->fs_info->btree_inode);
3691
3692 SetPageError(page);
3693 if (test_and_set_bit(EXTENT_BUFFER_WRITE_ERR, &eb->bflags))
3694 return;
3695
3696 /*
3697 * If writeback for a btree extent that doesn't belong to a log tree
3698 * failed, increment the counter transaction->eb_write_errors.
3699 * We do this because while the transaction is running and before it's
3700 * committing (when we call filemap_fdata[write|wait]_range against
3701 * the btree inode), we might have
3702 * btree_inode->i_mapping->a_ops->writepages() called by the VM - if it
3703 * returns an error or an error happens during writeback, when we're
3704 * committing the transaction we wouldn't know about it, since the pages
3705 * can be no longer dirty nor marked anymore for writeback (if a
3706 * subsequent modification to the extent buffer didn't happen before the
3707 * transaction commit), which makes filemap_fdata[write|wait]_range not
3708 * able to find the pages tagged with SetPageError at transaction
3709 * commit time. So if this happens we must abort the transaction,
3710 * otherwise we commit a super block with btree roots that point to
3711 * btree nodes/leafs whose content on disk is invalid - either garbage
3712 * or the content of some node/leaf from a past generation that got
3713 * cowed or deleted and is no longer valid.
3714 *
3715 * Note: setting AS_EIO/AS_ENOSPC in the btree inode's i_mapping would
3716 * not be enough - we need to distinguish between log tree extents vs
3717 * non-log tree extents, and the next filemap_fdatawait_range() call
3718 * will catch and clear such errors in the mapping - and that call might
3719 * be from a log sync and not from a transaction commit. Also, checking
3720 * for the eb flag EXTENT_BUFFER_WRITE_ERR at transaction commit time is
3721 * not done and would not be reliable - the eb might have been released
3722 * from memory and reading it back again means that flag would not be
3723 * set (since it's a runtime flag, not persisted on disk).
3724 *
3725 * Using the flags below in the btree inode also makes us achieve the
3726 * goal of AS_EIO/AS_ENOSPC when writepages() returns success, started
3727 * writeback for all dirty pages and before filemap_fdatawait_range()
3728 * is called, the writeback for all dirty pages had already finished
3729 * with errors - because we were not using AS_EIO/AS_ENOSPC,
3730 * filemap_fdatawait_range() would return success, as it could not know
3731 * that writeback errors happened (the pages were no longer tagged for
3732 * writeback).
3733 */
3734 switch (eb->log_index) {
3735 case -1:
3736 set_bit(BTRFS_INODE_BTREE_ERR, &btree_ino->runtime_flags);
3737 break;
3738 case 0:
3739 set_bit(BTRFS_INODE_BTREE_LOG1_ERR, &btree_ino->runtime_flags);
3740 break;
3741 case 1:
3742 set_bit(BTRFS_INODE_BTREE_LOG2_ERR, &btree_ino->runtime_flags);
3743 break;
3744 default:
3745 BUG(); /* unexpected, logic error */
3746 }
3747}
3748
Christoph Hellwig4246a0b2015-07-20 15:29:37 +02003749static void end_bio_extent_buffer_writepage(struct bio *bio)
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003750{
Kent Overstreet2c30c712013-11-07 12:20:26 -08003751 struct bio_vec *bvec;
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003752 struct extent_buffer *eb;
Kent Overstreet2c30c712013-11-07 12:20:26 -08003753 int i, done;
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003754
Kent Overstreet2c30c712013-11-07 12:20:26 -08003755 bio_for_each_segment_all(bvec, bio, i) {
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003756 struct page *page = bvec->bv_page;
3757
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003758 eb = (struct extent_buffer *)page->private;
3759 BUG_ON(!eb);
3760 done = atomic_dec_and_test(&eb->io_pages);
3761
Christoph Hellwig4246a0b2015-07-20 15:29:37 +02003762 if (bio->bi_error ||
3763 test_bit(EXTENT_BUFFER_WRITE_ERR, &eb->bflags)) {
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003764 ClearPageUptodate(page);
Filipe Manana656f30d2014-09-26 12:25:56 +01003765 set_btree_ioerr(page);
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003766 }
3767
3768 end_page_writeback(page);
3769
3770 if (!done)
3771 continue;
3772
3773 end_extent_buffer_writeback(eb);
Kent Overstreet2c30c712013-11-07 12:20:26 -08003774 }
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003775
3776 bio_put(bio);
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003777}
3778
Chris Mason0e378df2014-05-19 20:55:27 -07003779static noinline_for_stack int write_one_eb(struct extent_buffer *eb,
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003780 struct btrfs_fs_info *fs_info,
3781 struct writeback_control *wbc,
3782 struct extent_page_data *epd)
3783{
3784 struct block_device *bdev = fs_info->fs_devices->latest_bdev;
Josef Bacikf28491e2013-12-16 13:24:27 -05003785 struct extent_io_tree *tree = &BTRFS_I(fs_info->btree_inode)->io_tree;
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003786 u64 offset = eb->start;
3787 unsigned long i, num_pages;
Josef Bacikde0022b2012-09-25 14:25:58 -04003788 unsigned long bio_flags = 0;
Josef Bacikd4c7ca82013-04-19 19:49:09 -04003789 int rw = (epd->sync_io ? WRITE_SYNC : WRITE) | REQ_META;
Josef Bacikd7dbe9e2012-04-23 14:00:51 -04003790 int ret = 0;
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003791
Filipe Manana656f30d2014-09-26 12:25:56 +01003792 clear_bit(EXTENT_BUFFER_WRITE_ERR, &eb->bflags);
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003793 num_pages = num_extent_pages(eb->start, eb->len);
3794 atomic_set(&eb->io_pages, num_pages);
Josef Bacikde0022b2012-09-25 14:25:58 -04003795 if (btrfs_header_owner(eb) == BTRFS_TREE_LOG_OBJECTID)
3796 bio_flags = EXTENT_BIO_TREE_LOG;
3797
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003798 for (i = 0; i < num_pages; i++) {
David Sterbafb85fc92014-07-31 01:03:53 +02003799 struct page *p = eb->pages[i];
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003800
3801 clear_page_dirty_for_io(p);
3802 set_page_writeback(p);
Chris Masonda2f0f72015-07-02 13:57:22 -07003803 ret = submit_extent_page(rw, tree, wbc, p, offset >> 9,
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003804 PAGE_CACHE_SIZE, 0, bdev, &epd->bio,
3805 -1, end_bio_extent_buffer_writepage,
Filipe Manana005efed2015-09-14 09:09:31 +01003806 0, epd->bio_flags, bio_flags, false);
Josef Bacikde0022b2012-09-25 14:25:58 -04003807 epd->bio_flags = bio_flags;
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003808 if (ret) {
Filipe Manana656f30d2014-09-26 12:25:56 +01003809 set_btree_ioerr(p);
Filipe Manana55e3bd22014-09-22 17:41:04 +01003810 end_page_writeback(p);
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003811 if (atomic_sub_and_test(num_pages - i, &eb->io_pages))
3812 end_extent_buffer_writeback(eb);
3813 ret = -EIO;
3814 break;
3815 }
3816 offset += PAGE_CACHE_SIZE;
3817 update_nr_written(p, wbc, 1);
3818 unlock_page(p);
3819 }
3820
3821 if (unlikely(ret)) {
3822 for (; i < num_pages; i++) {
Chris Masonbbf65cf2014-10-04 09:56:45 -07003823 struct page *p = eb->pages[i];
Liu Bo81465022014-09-23 22:22:33 +08003824 clear_page_dirty_for_io(p);
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003825 unlock_page(p);
3826 }
3827 }
3828
3829 return ret;
3830}
3831
3832int btree_write_cache_pages(struct address_space *mapping,
3833 struct writeback_control *wbc)
3834{
3835 struct extent_io_tree *tree = &BTRFS_I(mapping->host)->io_tree;
3836 struct btrfs_fs_info *fs_info = BTRFS_I(mapping->host)->root->fs_info;
3837 struct extent_buffer *eb, *prev_eb = NULL;
3838 struct extent_page_data epd = {
3839 .bio = NULL,
3840 .tree = tree,
3841 .extent_locked = 0,
3842 .sync_io = wbc->sync_mode == WB_SYNC_ALL,
Josef Bacikde0022b2012-09-25 14:25:58 -04003843 .bio_flags = 0,
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003844 };
3845 int ret = 0;
3846 int done = 0;
3847 int nr_to_write_done = 0;
3848 struct pagevec pvec;
3849 int nr_pages;
3850 pgoff_t index;
3851 pgoff_t end; /* Inclusive */
3852 int scanned = 0;
3853 int tag;
3854
3855 pagevec_init(&pvec, 0);
3856 if (wbc->range_cyclic) {
3857 index = mapping->writeback_index; /* Start from prev offset */
3858 end = -1;
3859 } else {
3860 index = wbc->range_start >> PAGE_CACHE_SHIFT;
3861 end = wbc->range_end >> PAGE_CACHE_SHIFT;
3862 scanned = 1;
3863 }
3864 if (wbc->sync_mode == WB_SYNC_ALL)
3865 tag = PAGECACHE_TAG_TOWRITE;
3866 else
3867 tag = PAGECACHE_TAG_DIRTY;
3868retry:
3869 if (wbc->sync_mode == WB_SYNC_ALL)
3870 tag_pages_for_writeback(mapping, index, end);
3871 while (!done && !nr_to_write_done && (index <= end) &&
3872 (nr_pages = pagevec_lookup_tag(&pvec, mapping, &index, tag,
3873 min(end - index, (pgoff_t)PAGEVEC_SIZE-1) + 1))) {
3874 unsigned i;
3875
3876 scanned = 1;
3877 for (i = 0; i < nr_pages; i++) {
3878 struct page *page = pvec.pages[i];
3879
3880 if (!PagePrivate(page))
3881 continue;
3882
3883 if (!wbc->range_cyclic && page->index > end) {
3884 done = 1;
3885 break;
3886 }
3887
Josef Bacikb5bae262012-09-14 13:43:01 -04003888 spin_lock(&mapping->private_lock);
3889 if (!PagePrivate(page)) {
3890 spin_unlock(&mapping->private_lock);
3891 continue;
3892 }
3893
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003894 eb = (struct extent_buffer *)page->private;
Josef Bacikb5bae262012-09-14 13:43:01 -04003895
3896 /*
3897 * Shouldn't happen and normally this would be a BUG_ON
3898 * but no sense in crashing the users box for something
3899 * we can survive anyway.
3900 */
Dulshani Gunawardhanafae7f212013-10-31 10:30:08 +05303901 if (WARN_ON(!eb)) {
Josef Bacikb5bae262012-09-14 13:43:01 -04003902 spin_unlock(&mapping->private_lock);
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003903 continue;
3904 }
3905
Josef Bacikb5bae262012-09-14 13:43:01 -04003906 if (eb == prev_eb) {
3907 spin_unlock(&mapping->private_lock);
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003908 continue;
3909 }
3910
Josef Bacikb5bae262012-09-14 13:43:01 -04003911 ret = atomic_inc_not_zero(&eb->refs);
3912 spin_unlock(&mapping->private_lock);
3913 if (!ret)
3914 continue;
3915
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003916 prev_eb = eb;
3917 ret = lock_extent_buffer_for_io(eb, fs_info, &epd);
3918 if (!ret) {
3919 free_extent_buffer(eb);
3920 continue;
3921 }
3922
3923 ret = write_one_eb(eb, fs_info, wbc, &epd);
3924 if (ret) {
3925 done = 1;
3926 free_extent_buffer(eb);
3927 break;
3928 }
3929 free_extent_buffer(eb);
3930
3931 /*
3932 * the filesystem may choose to bump up nr_to_write.
3933 * We have to make sure to honor the new nr_to_write
3934 * at any time
3935 */
3936 nr_to_write_done = wbc->nr_to_write <= 0;
3937 }
3938 pagevec_release(&pvec);
3939 cond_resched();
3940 }
3941 if (!scanned && !done) {
3942 /*
3943 * We hit the last page and there is more work to be done: wrap
3944 * back to the start of the file
3945 */
3946 scanned = 1;
3947 index = 0;
3948 goto retry;
3949 }
3950 flush_write_bio(&epd);
3951 return ret;
3952}
3953
Chris Masond1310b22008-01-24 16:13:08 -05003954/**
3955 * write_cache_pages - walk the list of dirty pages of the given address space and write all of them.
3956 * @mapping: address space structure to write
3957 * @wbc: subtract the number of written pages from *@wbc->nr_to_write
3958 * @writepage: function called for each page
3959 * @data: data passed to writepage function
3960 *
3961 * If a page is already under I/O, write_cache_pages() skips it, even
3962 * if it's dirty. This is desirable behaviour for memory-cleaning writeback,
3963 * but it is INCORRECT for data-integrity system calls such as fsync(). fsync()
3964 * and msync() need to guarantee that all the data which was dirty at the time
3965 * the call was made get new I/O started against them. If wbc->sync_mode is
3966 * WB_SYNC_ALL then we were called for data integrity and we must wait for
3967 * existing IO to complete.
3968 */
Chris Mason4bef0842008-09-08 11:18:08 -04003969static int extent_write_cache_pages(struct extent_io_tree *tree,
3970 struct address_space *mapping,
3971 struct writeback_control *wbc,
Chris Masond2c3f4f2008-11-19 12:44:22 -05003972 writepage_t writepage, void *data,
3973 void (*flush_fn)(void *))
Chris Masond1310b22008-01-24 16:13:08 -05003974{
Josef Bacik7fd1a3f2012-06-27 17:18:41 -04003975 struct inode *inode = mapping->host;
Chris Masond1310b22008-01-24 16:13:08 -05003976 int ret = 0;
3977 int done = 0;
Filipe Manana61391d52014-05-09 17:17:40 +01003978 int err = 0;
Chris Masonf85d7d6c2009-09-18 16:03:16 -04003979 int nr_to_write_done = 0;
Chris Masond1310b22008-01-24 16:13:08 -05003980 struct pagevec pvec;
3981 int nr_pages;
3982 pgoff_t index;
3983 pgoff_t end; /* Inclusive */
3984 int scanned = 0;
Josef Bacikf7aaa062011-07-15 21:26:38 +00003985 int tag;
Chris Masond1310b22008-01-24 16:13:08 -05003986
Josef Bacik7fd1a3f2012-06-27 17:18:41 -04003987 /*
3988 * We have to hold onto the inode so that ordered extents can do their
3989 * work when the IO finishes. The alternative to this is failing to add
3990 * an ordered extent if the igrab() fails there and that is a huge pain
3991 * to deal with, so instead just hold onto the inode throughout the
3992 * writepages operation. If it fails here we are freeing up the inode
3993 * anyway and we'd rather not waste our time writing out stuff that is
3994 * going to be truncated anyway.
3995 */
3996 if (!igrab(inode))
3997 return 0;
3998
Chris Masond1310b22008-01-24 16:13:08 -05003999 pagevec_init(&pvec, 0);
4000 if (wbc->range_cyclic) {
4001 index = mapping->writeback_index; /* Start from prev offset */
4002 end = -1;
4003 } else {
4004 index = wbc->range_start >> PAGE_CACHE_SHIFT;
4005 end = wbc->range_end >> PAGE_CACHE_SHIFT;
Chris Masond1310b22008-01-24 16:13:08 -05004006 scanned = 1;
4007 }
Josef Bacikf7aaa062011-07-15 21:26:38 +00004008 if (wbc->sync_mode == WB_SYNC_ALL)
4009 tag = PAGECACHE_TAG_TOWRITE;
4010 else
4011 tag = PAGECACHE_TAG_DIRTY;
Chris Masond1310b22008-01-24 16:13:08 -05004012retry:
Josef Bacikf7aaa062011-07-15 21:26:38 +00004013 if (wbc->sync_mode == WB_SYNC_ALL)
4014 tag_pages_for_writeback(mapping, index, end);
Chris Masonf85d7d6c2009-09-18 16:03:16 -04004015 while (!done && !nr_to_write_done && (index <= end) &&
Josef Bacikf7aaa062011-07-15 21:26:38 +00004016 (nr_pages = pagevec_lookup_tag(&pvec, mapping, &index, tag,
4017 min(end - index, (pgoff_t)PAGEVEC_SIZE-1) + 1))) {
Chris Masond1310b22008-01-24 16:13:08 -05004018 unsigned i;
4019
4020 scanned = 1;
4021 for (i = 0; i < nr_pages; i++) {
4022 struct page *page = pvec.pages[i];
4023
4024 /*
4025 * At this point we hold neither mapping->tree_lock nor
4026 * lock on the page itself: the page may be truncated or
4027 * invalidated (changing page->mapping to NULL), or even
4028 * swizzled back from swapper_space to tmpfs file
4029 * mapping
4030 */
Josef Bacikc8f2f242013-02-11 11:33:00 -05004031 if (!trylock_page(page)) {
4032 flush_fn(data);
4033 lock_page(page);
Chris Mason01d658f2011-11-01 10:08:06 -04004034 }
Chris Masond1310b22008-01-24 16:13:08 -05004035
4036 if (unlikely(page->mapping != mapping)) {
4037 unlock_page(page);
4038 continue;
4039 }
4040
4041 if (!wbc->range_cyclic && page->index > end) {
4042 done = 1;
4043 unlock_page(page);
4044 continue;
4045 }
4046
Chris Masond2c3f4f2008-11-19 12:44:22 -05004047 if (wbc->sync_mode != WB_SYNC_NONE) {
Chris Mason0e6bd952008-11-20 10:46:35 -05004048 if (PageWriteback(page))
4049 flush_fn(data);
Chris Masond1310b22008-01-24 16:13:08 -05004050 wait_on_page_writeback(page);
Chris Masond2c3f4f2008-11-19 12:44:22 -05004051 }
Chris Masond1310b22008-01-24 16:13:08 -05004052
4053 if (PageWriteback(page) ||
4054 !clear_page_dirty_for_io(page)) {
4055 unlock_page(page);
4056 continue;
4057 }
4058
4059 ret = (*writepage)(page, wbc, data);
4060
4061 if (unlikely(ret == AOP_WRITEPAGE_ACTIVATE)) {
4062 unlock_page(page);
4063 ret = 0;
4064 }
Filipe Manana61391d52014-05-09 17:17:40 +01004065 if (!err && ret < 0)
4066 err = ret;
Chris Masonf85d7d6c2009-09-18 16:03:16 -04004067
4068 /*
4069 * the filesystem may choose to bump up nr_to_write.
4070 * We have to make sure to honor the new nr_to_write
4071 * at any time
4072 */
4073 nr_to_write_done = wbc->nr_to_write <= 0;
Chris Masond1310b22008-01-24 16:13:08 -05004074 }
4075 pagevec_release(&pvec);
4076 cond_resched();
4077 }
Filipe Manana61391d52014-05-09 17:17:40 +01004078 if (!scanned && !done && !err) {
Chris Masond1310b22008-01-24 16:13:08 -05004079 /*
4080 * We hit the last page and there is more work to be done: wrap
4081 * back to the start of the file
4082 */
4083 scanned = 1;
4084 index = 0;
4085 goto retry;
4086 }
Josef Bacik7fd1a3f2012-06-27 17:18:41 -04004087 btrfs_add_delayed_iput(inode);
Filipe Manana61391d52014-05-09 17:17:40 +01004088 return err;
Chris Masond1310b22008-01-24 16:13:08 -05004089}
Chris Masond1310b22008-01-24 16:13:08 -05004090
Chris Masonffbd5172009-04-20 15:50:09 -04004091static void flush_epd_write_bio(struct extent_page_data *epd)
4092{
4093 if (epd->bio) {
Jeff Mahoney355808c2011-10-03 23:23:14 -04004094 int rw = WRITE;
4095 int ret;
4096
Chris Masonffbd5172009-04-20 15:50:09 -04004097 if (epd->sync_io)
Jeff Mahoney355808c2011-10-03 23:23:14 -04004098 rw = WRITE_SYNC;
4099
Josef Bacikde0022b2012-09-25 14:25:58 -04004100 ret = submit_one_bio(rw, epd->bio, 0, epd->bio_flags);
Jeff Mahoney79787ea2012-03-12 16:03:00 +01004101 BUG_ON(ret < 0); /* -ENOMEM */
Chris Masonffbd5172009-04-20 15:50:09 -04004102 epd->bio = NULL;
4103 }
4104}
4105
Chris Masond2c3f4f2008-11-19 12:44:22 -05004106static noinline void flush_write_bio(void *data)
4107{
4108 struct extent_page_data *epd = data;
Chris Masonffbd5172009-04-20 15:50:09 -04004109 flush_epd_write_bio(epd);
Chris Masond2c3f4f2008-11-19 12:44:22 -05004110}
4111
Chris Masond1310b22008-01-24 16:13:08 -05004112int extent_write_full_page(struct extent_io_tree *tree, struct page *page,
4113 get_extent_t *get_extent,
4114 struct writeback_control *wbc)
4115{
4116 int ret;
Chris Masond1310b22008-01-24 16:13:08 -05004117 struct extent_page_data epd = {
4118 .bio = NULL,
4119 .tree = tree,
4120 .get_extent = get_extent,
Chris Mason771ed682008-11-06 22:02:51 -05004121 .extent_locked = 0,
Chris Masonffbd5172009-04-20 15:50:09 -04004122 .sync_io = wbc->sync_mode == WB_SYNC_ALL,
Josef Bacikde0022b2012-09-25 14:25:58 -04004123 .bio_flags = 0,
Chris Masond1310b22008-01-24 16:13:08 -05004124 };
Chris Masond1310b22008-01-24 16:13:08 -05004125
Chris Masond1310b22008-01-24 16:13:08 -05004126 ret = __extent_writepage(page, wbc, &epd);
4127
Chris Masonffbd5172009-04-20 15:50:09 -04004128 flush_epd_write_bio(&epd);
Chris Masond1310b22008-01-24 16:13:08 -05004129 return ret;
4130}
Chris Masond1310b22008-01-24 16:13:08 -05004131
Chris Mason771ed682008-11-06 22:02:51 -05004132int extent_write_locked_range(struct extent_io_tree *tree, struct inode *inode,
4133 u64 start, u64 end, get_extent_t *get_extent,
4134 int mode)
4135{
4136 int ret = 0;
4137 struct address_space *mapping = inode->i_mapping;
4138 struct page *page;
4139 unsigned long nr_pages = (end - start + PAGE_CACHE_SIZE) >>
4140 PAGE_CACHE_SHIFT;
4141
4142 struct extent_page_data epd = {
4143 .bio = NULL,
4144 .tree = tree,
4145 .get_extent = get_extent,
4146 .extent_locked = 1,
Chris Masonffbd5172009-04-20 15:50:09 -04004147 .sync_io = mode == WB_SYNC_ALL,
Josef Bacikde0022b2012-09-25 14:25:58 -04004148 .bio_flags = 0,
Chris Mason771ed682008-11-06 22:02:51 -05004149 };
4150 struct writeback_control wbc_writepages = {
Chris Mason771ed682008-11-06 22:02:51 -05004151 .sync_mode = mode,
Chris Mason771ed682008-11-06 22:02:51 -05004152 .nr_to_write = nr_pages * 2,
4153 .range_start = start,
4154 .range_end = end + 1,
4155 };
4156
Chris Masond3977122009-01-05 21:25:51 -05004157 while (start <= end) {
Chris Mason771ed682008-11-06 22:02:51 -05004158 page = find_get_page(mapping, start >> PAGE_CACHE_SHIFT);
4159 if (clear_page_dirty_for_io(page))
4160 ret = __extent_writepage(page, &wbc_writepages, &epd);
4161 else {
4162 if (tree->ops && tree->ops->writepage_end_io_hook)
4163 tree->ops->writepage_end_io_hook(page, start,
4164 start + PAGE_CACHE_SIZE - 1,
4165 NULL, 1);
4166 unlock_page(page);
4167 }
4168 page_cache_release(page);
4169 start += PAGE_CACHE_SIZE;
4170 }
4171
Chris Masonffbd5172009-04-20 15:50:09 -04004172 flush_epd_write_bio(&epd);
Chris Mason771ed682008-11-06 22:02:51 -05004173 return ret;
4174}
Chris Masond1310b22008-01-24 16:13:08 -05004175
4176int extent_writepages(struct extent_io_tree *tree,
4177 struct address_space *mapping,
4178 get_extent_t *get_extent,
4179 struct writeback_control *wbc)
4180{
4181 int ret = 0;
4182 struct extent_page_data epd = {
4183 .bio = NULL,
4184 .tree = tree,
4185 .get_extent = get_extent,
Chris Mason771ed682008-11-06 22:02:51 -05004186 .extent_locked = 0,
Chris Masonffbd5172009-04-20 15:50:09 -04004187 .sync_io = wbc->sync_mode == WB_SYNC_ALL,
Josef Bacikde0022b2012-09-25 14:25:58 -04004188 .bio_flags = 0,
Chris Masond1310b22008-01-24 16:13:08 -05004189 };
4190
Chris Mason4bef0842008-09-08 11:18:08 -04004191 ret = extent_write_cache_pages(tree, mapping, wbc,
Chris Masond2c3f4f2008-11-19 12:44:22 -05004192 __extent_writepage, &epd,
4193 flush_write_bio);
Chris Masonffbd5172009-04-20 15:50:09 -04004194 flush_epd_write_bio(&epd);
Chris Masond1310b22008-01-24 16:13:08 -05004195 return ret;
4196}
Chris Masond1310b22008-01-24 16:13:08 -05004197
4198int extent_readpages(struct extent_io_tree *tree,
4199 struct address_space *mapping,
4200 struct list_head *pages, unsigned nr_pages,
4201 get_extent_t get_extent)
4202{
4203 struct bio *bio = NULL;
4204 unsigned page_idx;
Chris Masonc8b97812008-10-29 14:49:59 -04004205 unsigned long bio_flags = 0;
Liu Bo67c96842012-07-20 21:43:09 -06004206 struct page *pagepool[16];
4207 struct page *page;
Miao Xie125bac012013-07-25 19:22:37 +08004208 struct extent_map *em_cached = NULL;
Liu Bo67c96842012-07-20 21:43:09 -06004209 int nr = 0;
Chris Masond1310b22008-01-24 16:13:08 -05004210
Chris Masond1310b22008-01-24 16:13:08 -05004211 for (page_idx = 0; page_idx < nr_pages; page_idx++) {
Liu Bo67c96842012-07-20 21:43:09 -06004212 page = list_entry(pages->prev, struct page, lru);
Chris Masond1310b22008-01-24 16:13:08 -05004213
4214 prefetchw(&page->flags);
4215 list_del(&page->lru);
Liu Bo67c96842012-07-20 21:43:09 -06004216 if (add_to_page_cache_lru(page, mapping,
Itaru Kitayama43e817a2011-04-25 19:43:51 -04004217 page->index, GFP_NOFS)) {
Liu Bo67c96842012-07-20 21:43:09 -06004218 page_cache_release(page);
4219 continue;
Chris Masond1310b22008-01-24 16:13:08 -05004220 }
Liu Bo67c96842012-07-20 21:43:09 -06004221
4222 pagepool[nr++] = page;
4223 if (nr < ARRAY_SIZE(pagepool))
4224 continue;
Miao Xie125bac012013-07-25 19:22:37 +08004225 __extent_readpages(tree, pagepool, nr, get_extent, &em_cached,
Miao Xie99740902013-07-25 19:22:36 +08004226 &bio, 0, &bio_flags, READ);
Liu Bo67c96842012-07-20 21:43:09 -06004227 nr = 0;
Chris Masond1310b22008-01-24 16:13:08 -05004228 }
Miao Xie99740902013-07-25 19:22:36 +08004229 if (nr)
Miao Xie125bac012013-07-25 19:22:37 +08004230 __extent_readpages(tree, pagepool, nr, get_extent, &em_cached,
Miao Xie99740902013-07-25 19:22:36 +08004231 &bio, 0, &bio_flags, READ);
Liu Bo67c96842012-07-20 21:43:09 -06004232
Miao Xie125bac012013-07-25 19:22:37 +08004233 if (em_cached)
4234 free_extent_map(em_cached);
4235
Chris Masond1310b22008-01-24 16:13:08 -05004236 BUG_ON(!list_empty(pages));
4237 if (bio)
Jeff Mahoney79787ea2012-03-12 16:03:00 +01004238 return submit_one_bio(READ, bio, 0, bio_flags);
Chris Masond1310b22008-01-24 16:13:08 -05004239 return 0;
4240}
Chris Masond1310b22008-01-24 16:13:08 -05004241
4242/*
4243 * basic invalidatepage code, this waits on any locked or writeback
4244 * ranges corresponding to the page, and then deletes any extent state
4245 * records from the tree
4246 */
4247int extent_invalidatepage(struct extent_io_tree *tree,
4248 struct page *page, unsigned long offset)
4249{
Josef Bacik2ac55d42010-02-03 19:33:23 +00004250 struct extent_state *cached_state = NULL;
Miao Xie4eee4fa2012-12-21 09:17:45 +00004251 u64 start = page_offset(page);
Chris Masond1310b22008-01-24 16:13:08 -05004252 u64 end = start + PAGE_CACHE_SIZE - 1;
4253 size_t blocksize = page->mapping->host->i_sb->s_blocksize;
4254
Qu Wenruofda28322013-02-26 08:10:22 +00004255 start += ALIGN(offset, blocksize);
Chris Masond1310b22008-01-24 16:13:08 -05004256 if (start > end)
4257 return 0;
4258
Jeff Mahoneyd0082372012-03-01 14:57:19 +01004259 lock_extent_bits(tree, start, end, 0, &cached_state);
Chris Mason1edbb732009-09-02 13:24:36 -04004260 wait_on_page_writeback(page);
Chris Masond1310b22008-01-24 16:13:08 -05004261 clear_extent_bit(tree, start, end,
Josef Bacik32c00af2009-10-08 13:34:05 -04004262 EXTENT_LOCKED | EXTENT_DIRTY | EXTENT_DELALLOC |
4263 EXTENT_DO_ACCOUNTING,
Josef Bacik2ac55d42010-02-03 19:33:23 +00004264 1, 1, &cached_state, GFP_NOFS);
Chris Masond1310b22008-01-24 16:13:08 -05004265 return 0;
4266}
Chris Masond1310b22008-01-24 16:13:08 -05004267
4268/*
Chris Mason7b13b7b2008-04-18 10:29:50 -04004269 * a helper for releasepage, this tests for areas of the page that
4270 * are locked or under IO and drops the related state bits if it is safe
4271 * to drop the page.
4272 */
Eric Sandeen48a3b632013-04-25 20:41:01 +00004273static int try_release_extent_state(struct extent_map_tree *map,
4274 struct extent_io_tree *tree,
4275 struct page *page, gfp_t mask)
Chris Mason7b13b7b2008-04-18 10:29:50 -04004276{
Miao Xie4eee4fa2012-12-21 09:17:45 +00004277 u64 start = page_offset(page);
Chris Mason7b13b7b2008-04-18 10:29:50 -04004278 u64 end = start + PAGE_CACHE_SIZE - 1;
4279 int ret = 1;
4280
Chris Mason211f90e2008-07-18 11:56:15 -04004281 if (test_range_bit(tree, start, end,
Chris Mason8b62b722009-09-02 16:53:46 -04004282 EXTENT_IOBITS, 0, NULL))
Chris Mason7b13b7b2008-04-18 10:29:50 -04004283 ret = 0;
4284 else {
4285 if ((mask & GFP_NOFS) == GFP_NOFS)
4286 mask = GFP_NOFS;
Chris Mason11ef1602009-09-23 20:28:46 -04004287 /*
4288 * at this point we can safely clear everything except the
4289 * locked bit and the nodatasum bit
4290 */
Chris Masone3f24cc2011-02-14 12:52:08 -05004291 ret = clear_extent_bit(tree, start, end,
Chris Mason11ef1602009-09-23 20:28:46 -04004292 ~(EXTENT_LOCKED | EXTENT_NODATASUM),
4293 0, 0, NULL, mask);
Chris Masone3f24cc2011-02-14 12:52:08 -05004294
4295 /* if clear_extent_bit failed for enomem reasons,
4296 * we can't allow the release to continue.
4297 */
4298 if (ret < 0)
4299 ret = 0;
4300 else
4301 ret = 1;
Chris Mason7b13b7b2008-04-18 10:29:50 -04004302 }
4303 return ret;
4304}
Chris Mason7b13b7b2008-04-18 10:29:50 -04004305
4306/*
Chris Masond1310b22008-01-24 16:13:08 -05004307 * a helper for releasepage. As long as there are no locked extents
4308 * in the range corresponding to the page, both state records and extent
4309 * map records are removed
4310 */
4311int try_release_extent_mapping(struct extent_map_tree *map,
Chris Mason70dec802008-01-29 09:59:12 -05004312 struct extent_io_tree *tree, struct page *page,
4313 gfp_t mask)
Chris Masond1310b22008-01-24 16:13:08 -05004314{
4315 struct extent_map *em;
Miao Xie4eee4fa2012-12-21 09:17:45 +00004316 u64 start = page_offset(page);
Chris Masond1310b22008-01-24 16:13:08 -05004317 u64 end = start + PAGE_CACHE_SIZE - 1;
Chris Mason7b13b7b2008-04-18 10:29:50 -04004318
Chris Mason70dec802008-01-29 09:59:12 -05004319 if ((mask & __GFP_WAIT) &&
4320 page->mapping->host->i_size > 16 * 1024 * 1024) {
Yan39b56372008-02-15 10:40:50 -05004321 u64 len;
Chris Mason70dec802008-01-29 09:59:12 -05004322 while (start <= end) {
Yan39b56372008-02-15 10:40:50 -05004323 len = end - start + 1;
Chris Mason890871b2009-09-02 16:24:52 -04004324 write_lock(&map->lock);
Yan39b56372008-02-15 10:40:50 -05004325 em = lookup_extent_mapping(map, start, len);
Tsutomu Itoh285190d2012-02-16 16:23:58 +09004326 if (!em) {
Chris Mason890871b2009-09-02 16:24:52 -04004327 write_unlock(&map->lock);
Chris Mason70dec802008-01-29 09:59:12 -05004328 break;
4329 }
Chris Mason7f3c74f2008-07-18 12:01:11 -04004330 if (test_bit(EXTENT_FLAG_PINNED, &em->flags) ||
4331 em->start != start) {
Chris Mason890871b2009-09-02 16:24:52 -04004332 write_unlock(&map->lock);
Chris Mason70dec802008-01-29 09:59:12 -05004333 free_extent_map(em);
4334 break;
4335 }
4336 if (!test_range_bit(tree, em->start,
4337 extent_map_end(em) - 1,
Chris Mason8b62b722009-09-02 16:53:46 -04004338 EXTENT_LOCKED | EXTENT_WRITEBACK,
Chris Mason9655d292009-09-02 15:22:30 -04004339 0, NULL)) {
Chris Mason70dec802008-01-29 09:59:12 -05004340 remove_extent_mapping(map, em);
4341 /* once for the rb tree */
4342 free_extent_map(em);
4343 }
4344 start = extent_map_end(em);
Chris Mason890871b2009-09-02 16:24:52 -04004345 write_unlock(&map->lock);
Chris Mason70dec802008-01-29 09:59:12 -05004346
4347 /* once for us */
Chris Masond1310b22008-01-24 16:13:08 -05004348 free_extent_map(em);
4349 }
Chris Masond1310b22008-01-24 16:13:08 -05004350 }
Chris Mason7b13b7b2008-04-18 10:29:50 -04004351 return try_release_extent_state(map, tree, page, mask);
Chris Masond1310b22008-01-24 16:13:08 -05004352}
Chris Masond1310b22008-01-24 16:13:08 -05004353
Chris Masonec29ed52011-02-23 16:23:20 -05004354/*
4355 * helper function for fiemap, which doesn't want to see any holes.
4356 * This maps until we find something past 'last'
4357 */
4358static struct extent_map *get_extent_skip_holes(struct inode *inode,
4359 u64 offset,
4360 u64 last,
4361 get_extent_t *get_extent)
4362{
4363 u64 sectorsize = BTRFS_I(inode)->root->sectorsize;
4364 struct extent_map *em;
4365 u64 len;
4366
4367 if (offset >= last)
4368 return NULL;
4369
Dulshani Gunawardhana67871252013-10-31 10:33:04 +05304370 while (1) {
Chris Masonec29ed52011-02-23 16:23:20 -05004371 len = last - offset;
4372 if (len == 0)
4373 break;
Qu Wenruofda28322013-02-26 08:10:22 +00004374 len = ALIGN(len, sectorsize);
Chris Masonec29ed52011-02-23 16:23:20 -05004375 em = get_extent(inode, NULL, 0, offset, len, 0);
David Sterbac7040052011-04-19 18:00:01 +02004376 if (IS_ERR_OR_NULL(em))
Chris Masonec29ed52011-02-23 16:23:20 -05004377 return em;
4378
4379 /* if this isn't a hole return it */
4380 if (!test_bit(EXTENT_FLAG_VACANCY, &em->flags) &&
4381 em->block_start != EXTENT_MAP_HOLE) {
4382 return em;
4383 }
4384
4385 /* this is a hole, advance to the next extent */
4386 offset = extent_map_end(em);
4387 free_extent_map(em);
4388 if (offset >= last)
4389 break;
4390 }
4391 return NULL;
4392}
4393
Yehuda Sadeh1506fcc2009-01-21 14:39:14 -05004394int extent_fiemap(struct inode *inode, struct fiemap_extent_info *fieinfo,
4395 __u64 start, __u64 len, get_extent_t *get_extent)
4396{
Josef Bacik975f84f2010-11-23 19:36:57 +00004397 int ret = 0;
Yehuda Sadeh1506fcc2009-01-21 14:39:14 -05004398 u64 off = start;
4399 u64 max = start + len;
4400 u32 flags = 0;
Josef Bacik975f84f2010-11-23 19:36:57 +00004401 u32 found_type;
4402 u64 last;
Chris Masonec29ed52011-02-23 16:23:20 -05004403 u64 last_for_get_extent = 0;
Yehuda Sadeh1506fcc2009-01-21 14:39:14 -05004404 u64 disko = 0;
Chris Masonec29ed52011-02-23 16:23:20 -05004405 u64 isize = i_size_read(inode);
Josef Bacik975f84f2010-11-23 19:36:57 +00004406 struct btrfs_key found_key;
Yehuda Sadeh1506fcc2009-01-21 14:39:14 -05004407 struct extent_map *em = NULL;
Josef Bacik2ac55d42010-02-03 19:33:23 +00004408 struct extent_state *cached_state = NULL;
Josef Bacik975f84f2010-11-23 19:36:57 +00004409 struct btrfs_path *path;
Josef Bacikdc046b12014-09-10 16:20:45 -04004410 struct btrfs_root *root = BTRFS_I(inode)->root;
Yehuda Sadeh1506fcc2009-01-21 14:39:14 -05004411 int end = 0;
Chris Masonec29ed52011-02-23 16:23:20 -05004412 u64 em_start = 0;
4413 u64 em_len = 0;
4414 u64 em_end = 0;
Yehuda Sadeh1506fcc2009-01-21 14:39:14 -05004415
4416 if (len == 0)
4417 return -EINVAL;
4418
Josef Bacik975f84f2010-11-23 19:36:57 +00004419 path = btrfs_alloc_path();
4420 if (!path)
4421 return -ENOMEM;
4422 path->leave_spinning = 1;
4423
Qu Wenruo2c919432014-07-18 09:55:43 +08004424 start = round_down(start, BTRFS_I(inode)->root->sectorsize);
4425 len = round_up(max, BTRFS_I(inode)->root->sectorsize) - start;
Josef Bacik4d479cf2011-11-17 11:34:31 -05004426
Chris Masonec29ed52011-02-23 16:23:20 -05004427 /*
4428 * lookup the last file extent. We're not using i_size here
4429 * because there might be preallocation past i_size
4430 */
Josef Bacikdc046b12014-09-10 16:20:45 -04004431 ret = btrfs_lookup_file_extent(NULL, root, path, btrfs_ino(inode), -1,
4432 0);
Josef Bacik975f84f2010-11-23 19:36:57 +00004433 if (ret < 0) {
4434 btrfs_free_path(path);
4435 return ret;
4436 }
4437 WARN_ON(!ret);
4438 path->slots[0]--;
Josef Bacik975f84f2010-11-23 19:36:57 +00004439 btrfs_item_key_to_cpu(path->nodes[0], &found_key, path->slots[0]);
David Sterba962a2982014-06-04 18:41:45 +02004440 found_type = found_key.type;
Josef Bacik975f84f2010-11-23 19:36:57 +00004441
Chris Masonec29ed52011-02-23 16:23:20 -05004442 /* No extents, but there might be delalloc bits */
Li Zefan33345d012011-04-20 10:31:50 +08004443 if (found_key.objectid != btrfs_ino(inode) ||
Josef Bacik975f84f2010-11-23 19:36:57 +00004444 found_type != BTRFS_EXTENT_DATA_KEY) {
Chris Masonec29ed52011-02-23 16:23:20 -05004445 /* have to trust i_size as the end */
4446 last = (u64)-1;
4447 last_for_get_extent = isize;
4448 } else {
4449 /*
4450 * remember the start of the last extent. There are a
4451 * bunch of different factors that go into the length of the
4452 * extent, so its much less complex to remember where it started
4453 */
4454 last = found_key.offset;
4455 last_for_get_extent = last + 1;
Josef Bacik975f84f2010-11-23 19:36:57 +00004456 }
Liu Bofe09e162013-09-22 12:54:23 +08004457 btrfs_release_path(path);
Josef Bacik975f84f2010-11-23 19:36:57 +00004458
Chris Masonec29ed52011-02-23 16:23:20 -05004459 /*
4460 * we might have some extents allocated but more delalloc past those
4461 * extents. so, we trust isize unless the start of the last extent is
4462 * beyond isize
4463 */
4464 if (last < isize) {
4465 last = (u64)-1;
4466 last_for_get_extent = isize;
4467 }
4468
Liu Boa52f4cd2013-05-01 16:23:41 +00004469 lock_extent_bits(&BTRFS_I(inode)->io_tree, start, start + len - 1, 0,
Jeff Mahoneyd0082372012-03-01 14:57:19 +01004470 &cached_state);
Chris Masonec29ed52011-02-23 16:23:20 -05004471
Josef Bacik4d479cf2011-11-17 11:34:31 -05004472 em = get_extent_skip_holes(inode, start, last_for_get_extent,
Chris Masonec29ed52011-02-23 16:23:20 -05004473 get_extent);
Yehuda Sadeh1506fcc2009-01-21 14:39:14 -05004474 if (!em)
4475 goto out;
4476 if (IS_ERR(em)) {
4477 ret = PTR_ERR(em);
4478 goto out;
4479 }
Josef Bacik975f84f2010-11-23 19:36:57 +00004480
Yehuda Sadeh1506fcc2009-01-21 14:39:14 -05004481 while (!end) {
Josef Bacikb76bb702013-07-05 13:52:51 -04004482 u64 offset_in_extent = 0;
Yehuda Sadeh1506fcc2009-01-21 14:39:14 -05004483
Chris Masonea8efc72011-03-08 11:54:40 -05004484 /* break if the extent we found is outside the range */
4485 if (em->start >= max || extent_map_end(em) < off)
4486 break;
4487
4488 /*
4489 * get_extent may return an extent that starts before our
4490 * requested range. We have to make sure the ranges
4491 * we return to fiemap always move forward and don't
4492 * overlap, so adjust the offsets here
4493 */
4494 em_start = max(em->start, off);
4495
4496 /*
4497 * record the offset from the start of the extent
Josef Bacikb76bb702013-07-05 13:52:51 -04004498 * for adjusting the disk offset below. Only do this if the
4499 * extent isn't compressed since our in ram offset may be past
4500 * what we have actually allocated on disk.
Chris Masonea8efc72011-03-08 11:54:40 -05004501 */
Josef Bacikb76bb702013-07-05 13:52:51 -04004502 if (!test_bit(EXTENT_FLAG_COMPRESSED, &em->flags))
4503 offset_in_extent = em_start - em->start;
Chris Masonec29ed52011-02-23 16:23:20 -05004504 em_end = extent_map_end(em);
Chris Masonea8efc72011-03-08 11:54:40 -05004505 em_len = em_end - em_start;
Yehuda Sadeh1506fcc2009-01-21 14:39:14 -05004506 disko = 0;
4507 flags = 0;
4508
Chris Masonea8efc72011-03-08 11:54:40 -05004509 /*
4510 * bump off for our next call to get_extent
4511 */
4512 off = extent_map_end(em);
4513 if (off >= max)
4514 end = 1;
4515
Heiko Carstens93dbfad2009-04-03 10:33:45 -04004516 if (em->block_start == EXTENT_MAP_LAST_BYTE) {
Yehuda Sadeh1506fcc2009-01-21 14:39:14 -05004517 end = 1;
4518 flags |= FIEMAP_EXTENT_LAST;
Heiko Carstens93dbfad2009-04-03 10:33:45 -04004519 } else if (em->block_start == EXTENT_MAP_INLINE) {
Yehuda Sadeh1506fcc2009-01-21 14:39:14 -05004520 flags |= (FIEMAP_EXTENT_DATA_INLINE |
4521 FIEMAP_EXTENT_NOT_ALIGNED);
Heiko Carstens93dbfad2009-04-03 10:33:45 -04004522 } else if (em->block_start == EXTENT_MAP_DELALLOC) {
Yehuda Sadeh1506fcc2009-01-21 14:39:14 -05004523 flags |= (FIEMAP_EXTENT_DELALLOC |
4524 FIEMAP_EXTENT_UNKNOWN);
Josef Bacikdc046b12014-09-10 16:20:45 -04004525 } else if (fieinfo->fi_extents_max) {
4526 u64 bytenr = em->block_start -
4527 (em->start - em->orig_start);
Liu Bofe09e162013-09-22 12:54:23 +08004528
Chris Masonea8efc72011-03-08 11:54:40 -05004529 disko = em->block_start + offset_in_extent;
Liu Bofe09e162013-09-22 12:54:23 +08004530
4531 /*
4532 * As btrfs supports shared space, this information
4533 * can be exported to userspace tools via
Josef Bacikdc046b12014-09-10 16:20:45 -04004534 * flag FIEMAP_EXTENT_SHARED. If fi_extents_max == 0
4535 * then we're just getting a count and we can skip the
4536 * lookup stuff.
Liu Bofe09e162013-09-22 12:54:23 +08004537 */
Josef Bacikdc046b12014-09-10 16:20:45 -04004538 ret = btrfs_check_shared(NULL, root->fs_info,
4539 root->objectid,
4540 btrfs_ino(inode), bytenr);
4541 if (ret < 0)
Liu Bofe09e162013-09-22 12:54:23 +08004542 goto out_free;
Josef Bacikdc046b12014-09-10 16:20:45 -04004543 if (ret)
Liu Bofe09e162013-09-22 12:54:23 +08004544 flags |= FIEMAP_EXTENT_SHARED;
Josef Bacikdc046b12014-09-10 16:20:45 -04004545 ret = 0;
Yehuda Sadeh1506fcc2009-01-21 14:39:14 -05004546 }
4547 if (test_bit(EXTENT_FLAG_COMPRESSED, &em->flags))
4548 flags |= FIEMAP_EXTENT_ENCODED;
Josef Bacik0d2b2372015-05-19 10:44:04 -04004549 if (test_bit(EXTENT_FLAG_PREALLOC, &em->flags))
4550 flags |= FIEMAP_EXTENT_UNWRITTEN;
Yehuda Sadeh1506fcc2009-01-21 14:39:14 -05004551
Yehuda Sadeh1506fcc2009-01-21 14:39:14 -05004552 free_extent_map(em);
4553 em = NULL;
Chris Masonec29ed52011-02-23 16:23:20 -05004554 if ((em_start >= last) || em_len == (u64)-1 ||
4555 (last == (u64)-1 && isize <= em_end)) {
Yehuda Sadeh1506fcc2009-01-21 14:39:14 -05004556 flags |= FIEMAP_EXTENT_LAST;
4557 end = 1;
4558 }
4559
Chris Masonec29ed52011-02-23 16:23:20 -05004560 /* now scan forward to see if this is really the last extent. */
4561 em = get_extent_skip_holes(inode, off, last_for_get_extent,
4562 get_extent);
4563 if (IS_ERR(em)) {
4564 ret = PTR_ERR(em);
4565 goto out;
4566 }
4567 if (!em) {
Josef Bacik975f84f2010-11-23 19:36:57 +00004568 flags |= FIEMAP_EXTENT_LAST;
4569 end = 1;
4570 }
Chris Masonec29ed52011-02-23 16:23:20 -05004571 ret = fiemap_fill_next_extent(fieinfo, em_start, disko,
4572 em_len, flags);
Chengyu Song26e726a2015-03-24 18:12:56 -04004573 if (ret) {
4574 if (ret == 1)
4575 ret = 0;
Chris Masonec29ed52011-02-23 16:23:20 -05004576 goto out_free;
Chengyu Song26e726a2015-03-24 18:12:56 -04004577 }
Yehuda Sadeh1506fcc2009-01-21 14:39:14 -05004578 }
4579out_free:
4580 free_extent_map(em);
4581out:
Liu Bofe09e162013-09-22 12:54:23 +08004582 btrfs_free_path(path);
Liu Boa52f4cd2013-05-01 16:23:41 +00004583 unlock_extent_cached(&BTRFS_I(inode)->io_tree, start, start + len - 1,
Josef Bacik2ac55d42010-02-03 19:33:23 +00004584 &cached_state, GFP_NOFS);
Yehuda Sadeh1506fcc2009-01-21 14:39:14 -05004585 return ret;
4586}
4587
Chris Mason727011e2010-08-06 13:21:20 -04004588static void __free_extent_buffer(struct extent_buffer *eb)
4589{
Eric Sandeen6d49ba12013-04-22 16:12:31 +00004590 btrfs_leak_debug_del(&eb->leak_list);
Chris Mason727011e2010-08-06 13:21:20 -04004591 kmem_cache_free(extent_buffer_cache, eb);
4592}
4593
Josef Bacika26e8c92014-03-28 17:07:27 -04004594int extent_buffer_under_io(struct extent_buffer *eb)
Chris Masond1310b22008-01-24 16:13:08 -05004595{
Josef Bacik0b32f4b2012-03-13 09:38:00 -04004596 return (atomic_read(&eb->io_pages) ||
4597 test_bit(EXTENT_BUFFER_WRITEBACK, &eb->bflags) ||
4598 test_bit(EXTENT_BUFFER_DIRTY, &eb->bflags));
Chris Masond1310b22008-01-24 16:13:08 -05004599}
4600
Miao Xie897ca6e92010-10-26 20:57:29 -04004601/*
4602 * Helper for releasing extent buffer page.
4603 */
David Sterbaa50924e2014-07-31 00:51:36 +02004604static void btrfs_release_extent_buffer_page(struct extent_buffer *eb)
Miao Xie897ca6e92010-10-26 20:57:29 -04004605{
4606 unsigned long index;
4607 struct page *page;
Jan Schmidt815a51c2012-05-16 17:00:02 +02004608 int mapped = !test_bit(EXTENT_BUFFER_DUMMY, &eb->bflags);
Miao Xie897ca6e92010-10-26 20:57:29 -04004609
Josef Bacik0b32f4b2012-03-13 09:38:00 -04004610 BUG_ON(extent_buffer_under_io(eb));
Miao Xie897ca6e92010-10-26 20:57:29 -04004611
David Sterbaa50924e2014-07-31 00:51:36 +02004612 index = num_extent_pages(eb->start, eb->len);
4613 if (index == 0)
Miao Xie897ca6e92010-10-26 20:57:29 -04004614 return;
4615
4616 do {
4617 index--;
David Sterbafb85fc92014-07-31 01:03:53 +02004618 page = eb->pages[index];
Forrest Liu5d2361d2015-02-09 17:31:45 +08004619 if (!page)
4620 continue;
4621 if (mapped)
Josef Bacik4f2de97a2012-03-07 16:20:05 -05004622 spin_lock(&page->mapping->private_lock);
Forrest Liu5d2361d2015-02-09 17:31:45 +08004623 /*
4624 * We do this since we'll remove the pages after we've
4625 * removed the eb from the radix tree, so we could race
4626 * and have this page now attached to the new eb. So
4627 * only clear page_private if it's still connected to
4628 * this eb.
4629 */
4630 if (PagePrivate(page) &&
4631 page->private == (unsigned long)eb) {
4632 BUG_ON(test_bit(EXTENT_BUFFER_DIRTY, &eb->bflags));
4633 BUG_ON(PageDirty(page));
4634 BUG_ON(PageWriteback(page));
Josef Bacik4f2de97a2012-03-07 16:20:05 -05004635 /*
Forrest Liu5d2361d2015-02-09 17:31:45 +08004636 * We need to make sure we haven't be attached
4637 * to a new eb.
Josef Bacik4f2de97a2012-03-07 16:20:05 -05004638 */
Forrest Liu5d2361d2015-02-09 17:31:45 +08004639 ClearPagePrivate(page);
4640 set_page_private(page, 0);
4641 /* One for the page private */
Miao Xie897ca6e92010-10-26 20:57:29 -04004642 page_cache_release(page);
Josef Bacik4f2de97a2012-03-07 16:20:05 -05004643 }
Forrest Liu5d2361d2015-02-09 17:31:45 +08004644
4645 if (mapped)
4646 spin_unlock(&page->mapping->private_lock);
4647
4648 /* One for when we alloced the page */
4649 page_cache_release(page);
David Sterbaa50924e2014-07-31 00:51:36 +02004650 } while (index != 0);
Miao Xie897ca6e92010-10-26 20:57:29 -04004651}
4652
4653/*
4654 * Helper for releasing the extent buffer.
4655 */
4656static inline void btrfs_release_extent_buffer(struct extent_buffer *eb)
4657{
David Sterbaa50924e2014-07-31 00:51:36 +02004658 btrfs_release_extent_buffer_page(eb);
Miao Xie897ca6e92010-10-26 20:57:29 -04004659 __free_extent_buffer(eb);
4660}
4661
Josef Bacikf28491e2013-12-16 13:24:27 -05004662static struct extent_buffer *
4663__alloc_extent_buffer(struct btrfs_fs_info *fs_info, u64 start,
David Sterba23d79d82014-06-15 02:55:29 +02004664 unsigned long len)
Josef Bacikdb7f3432013-08-07 14:54:37 -04004665{
4666 struct extent_buffer *eb = NULL;
4667
Michal Hockod1b5c562015-08-19 14:17:40 +02004668 eb = kmem_cache_zalloc(extent_buffer_cache, GFP_NOFS|__GFP_NOFAIL);
Josef Bacikdb7f3432013-08-07 14:54:37 -04004669 eb->start = start;
4670 eb->len = len;
Josef Bacikf28491e2013-12-16 13:24:27 -05004671 eb->fs_info = fs_info;
Josef Bacikdb7f3432013-08-07 14:54:37 -04004672 eb->bflags = 0;
4673 rwlock_init(&eb->lock);
4674 atomic_set(&eb->write_locks, 0);
4675 atomic_set(&eb->read_locks, 0);
4676 atomic_set(&eb->blocking_readers, 0);
4677 atomic_set(&eb->blocking_writers, 0);
4678 atomic_set(&eb->spinning_readers, 0);
4679 atomic_set(&eb->spinning_writers, 0);
4680 eb->lock_nested = 0;
4681 init_waitqueue_head(&eb->write_lock_wq);
4682 init_waitqueue_head(&eb->read_lock_wq);
4683
4684 btrfs_leak_debug_add(&eb->leak_list, &buffers);
4685
4686 spin_lock_init(&eb->refs_lock);
4687 atomic_set(&eb->refs, 1);
4688 atomic_set(&eb->io_pages, 0);
4689
4690 /*
4691 * Sanity checks, currently the maximum is 64k covered by 16x 4k pages
4692 */
4693 BUILD_BUG_ON(BTRFS_MAX_METADATA_BLOCKSIZE
4694 > MAX_INLINE_EXTENT_BUFFER_SIZE);
4695 BUG_ON(len > MAX_INLINE_EXTENT_BUFFER_SIZE);
4696
4697 return eb;
4698}
4699
4700struct extent_buffer *btrfs_clone_extent_buffer(struct extent_buffer *src)
4701{
4702 unsigned long i;
4703 struct page *p;
4704 struct extent_buffer *new;
4705 unsigned long num_pages = num_extent_pages(src->start, src->len);
4706
David Sterba3f556f72014-06-15 03:20:26 +02004707 new = __alloc_extent_buffer(src->fs_info, src->start, src->len);
Josef Bacikdb7f3432013-08-07 14:54:37 -04004708 if (new == NULL)
4709 return NULL;
4710
4711 for (i = 0; i < num_pages; i++) {
Josef Bacik9ec72672013-08-07 16:57:23 -04004712 p = alloc_page(GFP_NOFS);
Josef Bacikdb7f3432013-08-07 14:54:37 -04004713 if (!p) {
4714 btrfs_release_extent_buffer(new);
4715 return NULL;
4716 }
4717 attach_extent_buffer_page(new, p);
4718 WARN_ON(PageDirty(p));
4719 SetPageUptodate(p);
4720 new->pages[i] = p;
4721 }
4722
4723 copy_extent_buffer(new, src, 0, 0, src->len);
4724 set_bit(EXTENT_BUFFER_UPTODATE, &new->bflags);
4725 set_bit(EXTENT_BUFFER_DUMMY, &new->bflags);
4726
4727 return new;
4728}
4729
David Sterba3f556f72014-06-15 03:20:26 +02004730struct extent_buffer *alloc_dummy_extent_buffer(struct btrfs_fs_info *fs_info,
4731 u64 start)
Josef Bacikdb7f3432013-08-07 14:54:37 -04004732{
4733 struct extent_buffer *eb;
David Sterba3f556f72014-06-15 03:20:26 +02004734 unsigned long len;
4735 unsigned long num_pages;
Josef Bacikdb7f3432013-08-07 14:54:37 -04004736 unsigned long i;
4737
David Sterba3f556f72014-06-15 03:20:26 +02004738 if (!fs_info) {
4739 /*
4740 * Called only from tests that don't always have a fs_info
4741 * available, but we know that nodesize is 4096
4742 */
4743 len = 4096;
4744 } else {
4745 len = fs_info->tree_root->nodesize;
4746 }
4747 num_pages = num_extent_pages(0, len);
4748
4749 eb = __alloc_extent_buffer(fs_info, start, len);
Josef Bacikdb7f3432013-08-07 14:54:37 -04004750 if (!eb)
4751 return NULL;
4752
4753 for (i = 0; i < num_pages; i++) {
Josef Bacik9ec72672013-08-07 16:57:23 -04004754 eb->pages[i] = alloc_page(GFP_NOFS);
Josef Bacikdb7f3432013-08-07 14:54:37 -04004755 if (!eb->pages[i])
4756 goto err;
4757 }
4758 set_extent_buffer_uptodate(eb);
4759 btrfs_set_header_nritems(eb, 0);
4760 set_bit(EXTENT_BUFFER_DUMMY, &eb->bflags);
4761
4762 return eb;
4763err:
4764 for (; i > 0; i--)
4765 __free_page(eb->pages[i - 1]);
4766 __free_extent_buffer(eb);
4767 return NULL;
4768}
4769
Josef Bacik0b32f4b2012-03-13 09:38:00 -04004770static void check_buffer_tree_ref(struct extent_buffer *eb)
4771{
Chris Mason242e18c2013-01-29 17:49:37 -05004772 int refs;
Josef Bacik0b32f4b2012-03-13 09:38:00 -04004773 /* the ref bit is tricky. We have to make sure it is set
4774 * if we have the buffer dirty. Otherwise the
4775 * code to free a buffer can end up dropping a dirty
4776 * page
4777 *
4778 * Once the ref bit is set, it won't go away while the
4779 * buffer is dirty or in writeback, and it also won't
4780 * go away while we have the reference count on the
4781 * eb bumped.
4782 *
4783 * We can't just set the ref bit without bumping the
4784 * ref on the eb because free_extent_buffer might
4785 * see the ref bit and try to clear it. If this happens
4786 * free_extent_buffer might end up dropping our original
4787 * ref by mistake and freeing the page before we are able
4788 * to add one more ref.
4789 *
4790 * So bump the ref count first, then set the bit. If someone
4791 * beat us to it, drop the ref we added.
4792 */
Chris Mason242e18c2013-01-29 17:49:37 -05004793 refs = atomic_read(&eb->refs);
4794 if (refs >= 2 && test_bit(EXTENT_BUFFER_TREE_REF, &eb->bflags))
4795 return;
4796
Josef Bacik594831c2012-07-20 16:11:08 -04004797 spin_lock(&eb->refs_lock);
4798 if (!test_and_set_bit(EXTENT_BUFFER_TREE_REF, &eb->bflags))
Josef Bacik0b32f4b2012-03-13 09:38:00 -04004799 atomic_inc(&eb->refs);
Josef Bacik594831c2012-07-20 16:11:08 -04004800 spin_unlock(&eb->refs_lock);
Josef Bacik0b32f4b2012-03-13 09:38:00 -04004801}
4802
Mel Gorman2457aec2014-06-04 16:10:31 -07004803static void mark_extent_buffer_accessed(struct extent_buffer *eb,
4804 struct page *accessed)
Josef Bacik5df42352012-03-15 18:24:42 -04004805{
4806 unsigned long num_pages, i;
4807
Josef Bacik0b32f4b2012-03-13 09:38:00 -04004808 check_buffer_tree_ref(eb);
4809
Josef Bacik5df42352012-03-15 18:24:42 -04004810 num_pages = num_extent_pages(eb->start, eb->len);
4811 for (i = 0; i < num_pages; i++) {
David Sterbafb85fc92014-07-31 01:03:53 +02004812 struct page *p = eb->pages[i];
4813
Mel Gorman2457aec2014-06-04 16:10:31 -07004814 if (p != accessed)
4815 mark_page_accessed(p);
Josef Bacik5df42352012-03-15 18:24:42 -04004816 }
4817}
4818
Josef Bacikf28491e2013-12-16 13:24:27 -05004819struct extent_buffer *find_extent_buffer(struct btrfs_fs_info *fs_info,
4820 u64 start)
Chandra Seetharaman452c75c2013-10-07 10:45:25 -05004821{
4822 struct extent_buffer *eb;
4823
4824 rcu_read_lock();
Josef Bacikf28491e2013-12-16 13:24:27 -05004825 eb = radix_tree_lookup(&fs_info->buffer_radix,
4826 start >> PAGE_CACHE_SHIFT);
Chandra Seetharaman452c75c2013-10-07 10:45:25 -05004827 if (eb && atomic_inc_not_zero(&eb->refs)) {
4828 rcu_read_unlock();
Filipe Manana062c19e2015-04-23 11:28:48 +01004829 /*
4830 * Lock our eb's refs_lock to avoid races with
4831 * free_extent_buffer. When we get our eb it might be flagged
4832 * with EXTENT_BUFFER_STALE and another task running
4833 * free_extent_buffer might have seen that flag set,
4834 * eb->refs == 2, that the buffer isn't under IO (dirty and
4835 * writeback flags not set) and it's still in the tree (flag
4836 * EXTENT_BUFFER_TREE_REF set), therefore being in the process
4837 * of decrementing the extent buffer's reference count twice.
4838 * So here we could race and increment the eb's reference count,
4839 * clear its stale flag, mark it as dirty and drop our reference
4840 * before the other task finishes executing free_extent_buffer,
4841 * which would later result in an attempt to free an extent
4842 * buffer that is dirty.
4843 */
4844 if (test_bit(EXTENT_BUFFER_STALE, &eb->bflags)) {
4845 spin_lock(&eb->refs_lock);
4846 spin_unlock(&eb->refs_lock);
4847 }
Mel Gorman2457aec2014-06-04 16:10:31 -07004848 mark_extent_buffer_accessed(eb, NULL);
Chandra Seetharaman452c75c2013-10-07 10:45:25 -05004849 return eb;
4850 }
4851 rcu_read_unlock();
4852
4853 return NULL;
4854}
4855
Josef Bacikfaa2dbf2014-05-07 17:06:09 -04004856#ifdef CONFIG_BTRFS_FS_RUN_SANITY_TESTS
4857struct extent_buffer *alloc_test_extent_buffer(struct btrfs_fs_info *fs_info,
David Sterbace3e6982014-06-15 03:00:04 +02004858 u64 start)
Josef Bacikfaa2dbf2014-05-07 17:06:09 -04004859{
4860 struct extent_buffer *eb, *exists = NULL;
4861 int ret;
4862
4863 eb = find_extent_buffer(fs_info, start);
4864 if (eb)
4865 return eb;
David Sterba3f556f72014-06-15 03:20:26 +02004866 eb = alloc_dummy_extent_buffer(fs_info, start);
Josef Bacikfaa2dbf2014-05-07 17:06:09 -04004867 if (!eb)
4868 return NULL;
4869 eb->fs_info = fs_info;
4870again:
4871 ret = radix_tree_preload(GFP_NOFS & ~__GFP_HIGHMEM);
4872 if (ret)
4873 goto free_eb;
4874 spin_lock(&fs_info->buffer_lock);
4875 ret = radix_tree_insert(&fs_info->buffer_radix,
4876 start >> PAGE_CACHE_SHIFT, eb);
4877 spin_unlock(&fs_info->buffer_lock);
4878 radix_tree_preload_end();
4879 if (ret == -EEXIST) {
4880 exists = find_extent_buffer(fs_info, start);
4881 if (exists)
4882 goto free_eb;
4883 else
4884 goto again;
4885 }
4886 check_buffer_tree_ref(eb);
4887 set_bit(EXTENT_BUFFER_IN_TREE, &eb->bflags);
4888
4889 /*
4890 * We will free dummy extent buffer's if they come into
4891 * free_extent_buffer with a ref count of 2, but if we are using this we
4892 * want the buffers to stay in memory until we're done with them, so
4893 * bump the ref count again.
4894 */
4895 atomic_inc(&eb->refs);
4896 return eb;
4897free_eb:
4898 btrfs_release_extent_buffer(eb);
4899 return exists;
4900}
4901#endif
4902
Josef Bacikf28491e2013-12-16 13:24:27 -05004903struct extent_buffer *alloc_extent_buffer(struct btrfs_fs_info *fs_info,
David Sterbace3e6982014-06-15 03:00:04 +02004904 u64 start)
Chris Masond1310b22008-01-24 16:13:08 -05004905{
David Sterbace3e6982014-06-15 03:00:04 +02004906 unsigned long len = fs_info->tree_root->nodesize;
Chris Masond1310b22008-01-24 16:13:08 -05004907 unsigned long num_pages = num_extent_pages(start, len);
4908 unsigned long i;
4909 unsigned long index = start >> PAGE_CACHE_SHIFT;
4910 struct extent_buffer *eb;
Chris Mason6af118ce2008-07-22 11:18:07 -04004911 struct extent_buffer *exists = NULL;
Chris Masond1310b22008-01-24 16:13:08 -05004912 struct page *p;
Josef Bacikf28491e2013-12-16 13:24:27 -05004913 struct address_space *mapping = fs_info->btree_inode->i_mapping;
Chris Masond1310b22008-01-24 16:13:08 -05004914 int uptodate = 1;
Miao Xie19fe0a82010-10-26 20:57:29 -04004915 int ret;
Chris Masond1310b22008-01-24 16:13:08 -05004916
Josef Bacikf28491e2013-12-16 13:24:27 -05004917 eb = find_extent_buffer(fs_info, start);
Chandra Seetharaman452c75c2013-10-07 10:45:25 -05004918 if (eb)
Chris Mason6af118ce2008-07-22 11:18:07 -04004919 return eb;
Chris Mason6af118ce2008-07-22 11:18:07 -04004920
David Sterba23d79d82014-06-15 02:55:29 +02004921 eb = __alloc_extent_buffer(fs_info, start, len);
Peter2b114d12008-04-01 11:21:40 -04004922 if (!eb)
Chris Masond1310b22008-01-24 16:13:08 -05004923 return NULL;
4924
Chris Mason727011e2010-08-06 13:21:20 -04004925 for (i = 0; i < num_pages; i++, index++) {
Michal Hockod1b5c562015-08-19 14:17:40 +02004926 p = find_or_create_page(mapping, index, GFP_NOFS|__GFP_NOFAIL);
Josef Bacik4804b382012-10-05 16:43:45 -04004927 if (!p)
Chris Mason6af118ce2008-07-22 11:18:07 -04004928 goto free_eb;
Josef Bacik4f2de97a2012-03-07 16:20:05 -05004929
4930 spin_lock(&mapping->private_lock);
4931 if (PagePrivate(p)) {
4932 /*
4933 * We could have already allocated an eb for this page
4934 * and attached one so lets see if we can get a ref on
4935 * the existing eb, and if we can we know it's good and
4936 * we can just return that one, else we know we can just
4937 * overwrite page->private.
4938 */
4939 exists = (struct extent_buffer *)p->private;
4940 if (atomic_inc_not_zero(&exists->refs)) {
4941 spin_unlock(&mapping->private_lock);
4942 unlock_page(p);
Josef Bacik17de39a2012-05-04 15:16:06 -04004943 page_cache_release(p);
Mel Gorman2457aec2014-06-04 16:10:31 -07004944 mark_extent_buffer_accessed(exists, p);
Josef Bacik4f2de97a2012-03-07 16:20:05 -05004945 goto free_eb;
4946 }
Omar Sandoval5ca64f42015-02-24 02:47:05 -08004947 exists = NULL;
Josef Bacik4f2de97a2012-03-07 16:20:05 -05004948
Josef Bacik0b32f4b2012-03-13 09:38:00 -04004949 /*
Josef Bacik4f2de97a2012-03-07 16:20:05 -05004950 * Do this so attach doesn't complain and we need to
4951 * drop the ref the old guy had.
4952 */
4953 ClearPagePrivate(p);
Josef Bacik0b32f4b2012-03-13 09:38:00 -04004954 WARN_ON(PageDirty(p));
Josef Bacik4f2de97a2012-03-07 16:20:05 -05004955 page_cache_release(p);
Chris Masond1310b22008-01-24 16:13:08 -05004956 }
Josef Bacik4f2de97a2012-03-07 16:20:05 -05004957 attach_extent_buffer_page(eb, p);
4958 spin_unlock(&mapping->private_lock);
Josef Bacik0b32f4b2012-03-13 09:38:00 -04004959 WARN_ON(PageDirty(p));
Chris Mason727011e2010-08-06 13:21:20 -04004960 eb->pages[i] = p;
Chris Masond1310b22008-01-24 16:13:08 -05004961 if (!PageUptodate(p))
4962 uptodate = 0;
Chris Masoneb14ab82011-02-10 12:35:00 -05004963
4964 /*
4965 * see below about how we avoid a nasty race with release page
4966 * and why we unlock later
4967 */
Chris Masond1310b22008-01-24 16:13:08 -05004968 }
4969 if (uptodate)
Chris Masonb4ce94d2009-02-04 09:25:08 -05004970 set_bit(EXTENT_BUFFER_UPTODATE, &eb->bflags);
Josef Bacik115391d2012-03-09 09:51:43 -05004971again:
Miao Xie19fe0a82010-10-26 20:57:29 -04004972 ret = radix_tree_preload(GFP_NOFS & ~__GFP_HIGHMEM);
4973 if (ret)
4974 goto free_eb;
4975
Josef Bacikf28491e2013-12-16 13:24:27 -05004976 spin_lock(&fs_info->buffer_lock);
4977 ret = radix_tree_insert(&fs_info->buffer_radix,
4978 start >> PAGE_CACHE_SHIFT, eb);
4979 spin_unlock(&fs_info->buffer_lock);
Chandra Seetharaman452c75c2013-10-07 10:45:25 -05004980 radix_tree_preload_end();
Miao Xie19fe0a82010-10-26 20:57:29 -04004981 if (ret == -EEXIST) {
Josef Bacikf28491e2013-12-16 13:24:27 -05004982 exists = find_extent_buffer(fs_info, start);
Chandra Seetharaman452c75c2013-10-07 10:45:25 -05004983 if (exists)
4984 goto free_eb;
4985 else
Josef Bacik115391d2012-03-09 09:51:43 -05004986 goto again;
Chris Mason6af118ce2008-07-22 11:18:07 -04004987 }
Chris Mason6af118ce2008-07-22 11:18:07 -04004988 /* add one reference for the tree */
Josef Bacik0b32f4b2012-03-13 09:38:00 -04004989 check_buffer_tree_ref(eb);
Josef Bacik34b41ac2013-12-13 10:41:51 -05004990 set_bit(EXTENT_BUFFER_IN_TREE, &eb->bflags);
Chris Masoneb14ab82011-02-10 12:35:00 -05004991
4992 /*
4993 * there is a race where release page may have
4994 * tried to find this extent buffer in the radix
4995 * but failed. It will tell the VM it is safe to
4996 * reclaim the, and it will clear the page private bit.
4997 * We must make sure to set the page private bit properly
4998 * after the extent buffer is in the radix tree so
4999 * it doesn't get lost
5000 */
Chris Mason727011e2010-08-06 13:21:20 -04005001 SetPageChecked(eb->pages[0]);
5002 for (i = 1; i < num_pages; i++) {
David Sterbafb85fc92014-07-31 01:03:53 +02005003 p = eb->pages[i];
Chris Mason727011e2010-08-06 13:21:20 -04005004 ClearPageChecked(p);
5005 unlock_page(p);
5006 }
5007 unlock_page(eb->pages[0]);
Chris Masond1310b22008-01-24 16:13:08 -05005008 return eb;
5009
Chris Mason6af118ce2008-07-22 11:18:07 -04005010free_eb:
Omar Sandoval5ca64f42015-02-24 02:47:05 -08005011 WARN_ON(!atomic_dec_and_test(&eb->refs));
Chris Mason727011e2010-08-06 13:21:20 -04005012 for (i = 0; i < num_pages; i++) {
5013 if (eb->pages[i])
5014 unlock_page(eb->pages[i]);
5015 }
Chris Masoneb14ab82011-02-10 12:35:00 -05005016
Miao Xie897ca6e92010-10-26 20:57:29 -04005017 btrfs_release_extent_buffer(eb);
Chris Mason6af118ce2008-07-22 11:18:07 -04005018 return exists;
Chris Masond1310b22008-01-24 16:13:08 -05005019}
Chris Masond1310b22008-01-24 16:13:08 -05005020
Josef Bacik3083ee22012-03-09 16:01:49 -05005021static inline void btrfs_release_extent_buffer_rcu(struct rcu_head *head)
5022{
5023 struct extent_buffer *eb =
5024 container_of(head, struct extent_buffer, rcu_head);
5025
5026 __free_extent_buffer(eb);
5027}
5028
Josef Bacik3083ee22012-03-09 16:01:49 -05005029/* Expects to have eb->eb_lock already held */
David Sterbaf7a52a42013-04-26 14:56:29 +00005030static int release_extent_buffer(struct extent_buffer *eb)
Josef Bacik3083ee22012-03-09 16:01:49 -05005031{
5032 WARN_ON(atomic_read(&eb->refs) == 0);
5033 if (atomic_dec_and_test(&eb->refs)) {
Josef Bacik34b41ac2013-12-13 10:41:51 -05005034 if (test_and_clear_bit(EXTENT_BUFFER_IN_TREE, &eb->bflags)) {
Josef Bacikf28491e2013-12-16 13:24:27 -05005035 struct btrfs_fs_info *fs_info = eb->fs_info;
Josef Bacik3083ee22012-03-09 16:01:49 -05005036
Jan Schmidt815a51c2012-05-16 17:00:02 +02005037 spin_unlock(&eb->refs_lock);
Josef Bacik3083ee22012-03-09 16:01:49 -05005038
Josef Bacikf28491e2013-12-16 13:24:27 -05005039 spin_lock(&fs_info->buffer_lock);
5040 radix_tree_delete(&fs_info->buffer_radix,
Jan Schmidt815a51c2012-05-16 17:00:02 +02005041 eb->start >> PAGE_CACHE_SHIFT);
Josef Bacikf28491e2013-12-16 13:24:27 -05005042 spin_unlock(&fs_info->buffer_lock);
Josef Bacik34b41ac2013-12-13 10:41:51 -05005043 } else {
5044 spin_unlock(&eb->refs_lock);
Jan Schmidt815a51c2012-05-16 17:00:02 +02005045 }
Josef Bacik3083ee22012-03-09 16:01:49 -05005046
5047 /* Should be safe to release our pages at this point */
David Sterbaa50924e2014-07-31 00:51:36 +02005048 btrfs_release_extent_buffer_page(eb);
Josef Bacikbcb7e442015-03-16 17:38:02 -04005049#ifdef CONFIG_BTRFS_FS_RUN_SANITY_TESTS
5050 if (unlikely(test_bit(EXTENT_BUFFER_DUMMY, &eb->bflags))) {
5051 __free_extent_buffer(eb);
5052 return 1;
5053 }
5054#endif
Josef Bacik3083ee22012-03-09 16:01:49 -05005055 call_rcu(&eb->rcu_head, btrfs_release_extent_buffer_rcu);
Josef Bacike64860a2012-07-20 16:05:36 -04005056 return 1;
Josef Bacik3083ee22012-03-09 16:01:49 -05005057 }
5058 spin_unlock(&eb->refs_lock);
Josef Bacike64860a2012-07-20 16:05:36 -04005059
5060 return 0;
Josef Bacik3083ee22012-03-09 16:01:49 -05005061}
5062
Chris Masond1310b22008-01-24 16:13:08 -05005063void free_extent_buffer(struct extent_buffer *eb)
5064{
Chris Mason242e18c2013-01-29 17:49:37 -05005065 int refs;
5066 int old;
Chris Masond1310b22008-01-24 16:13:08 -05005067 if (!eb)
5068 return;
5069
Chris Mason242e18c2013-01-29 17:49:37 -05005070 while (1) {
5071 refs = atomic_read(&eb->refs);
5072 if (refs <= 3)
5073 break;
5074 old = atomic_cmpxchg(&eb->refs, refs, refs - 1);
5075 if (old == refs)
5076 return;
5077 }
5078
Josef Bacik3083ee22012-03-09 16:01:49 -05005079 spin_lock(&eb->refs_lock);
5080 if (atomic_read(&eb->refs) == 2 &&
Jan Schmidt815a51c2012-05-16 17:00:02 +02005081 test_bit(EXTENT_BUFFER_DUMMY, &eb->bflags))
5082 atomic_dec(&eb->refs);
5083
5084 if (atomic_read(&eb->refs) == 2 &&
Josef Bacik3083ee22012-03-09 16:01:49 -05005085 test_bit(EXTENT_BUFFER_STALE, &eb->bflags) &&
Josef Bacik0b32f4b2012-03-13 09:38:00 -04005086 !extent_buffer_under_io(eb) &&
Josef Bacik3083ee22012-03-09 16:01:49 -05005087 test_and_clear_bit(EXTENT_BUFFER_TREE_REF, &eb->bflags))
5088 atomic_dec(&eb->refs);
Chris Masond1310b22008-01-24 16:13:08 -05005089
Josef Bacik3083ee22012-03-09 16:01:49 -05005090 /*
5091 * I know this is terrible, but it's temporary until we stop tracking
5092 * the uptodate bits and such for the extent buffers.
5093 */
David Sterbaf7a52a42013-04-26 14:56:29 +00005094 release_extent_buffer(eb);
Chris Masond1310b22008-01-24 16:13:08 -05005095}
Chris Masond1310b22008-01-24 16:13:08 -05005096
Josef Bacik3083ee22012-03-09 16:01:49 -05005097void free_extent_buffer_stale(struct extent_buffer *eb)
5098{
5099 if (!eb)
Chris Masond1310b22008-01-24 16:13:08 -05005100 return;
5101
Josef Bacik3083ee22012-03-09 16:01:49 -05005102 spin_lock(&eb->refs_lock);
5103 set_bit(EXTENT_BUFFER_STALE, &eb->bflags);
5104
Josef Bacik0b32f4b2012-03-13 09:38:00 -04005105 if (atomic_read(&eb->refs) == 2 && !extent_buffer_under_io(eb) &&
Josef Bacik3083ee22012-03-09 16:01:49 -05005106 test_and_clear_bit(EXTENT_BUFFER_TREE_REF, &eb->bflags))
5107 atomic_dec(&eb->refs);
David Sterbaf7a52a42013-04-26 14:56:29 +00005108 release_extent_buffer(eb);
Chris Masond1310b22008-01-24 16:13:08 -05005109}
5110
Chris Mason1d4284b2012-03-28 20:31:37 -04005111void clear_extent_buffer_dirty(struct extent_buffer *eb)
Chris Masond1310b22008-01-24 16:13:08 -05005112{
Chris Masond1310b22008-01-24 16:13:08 -05005113 unsigned long i;
5114 unsigned long num_pages;
5115 struct page *page;
5116
Chris Masond1310b22008-01-24 16:13:08 -05005117 num_pages = num_extent_pages(eb->start, eb->len);
5118
5119 for (i = 0; i < num_pages; i++) {
David Sterbafb85fc92014-07-31 01:03:53 +02005120 page = eb->pages[i];
Chris Masonb9473432009-03-13 11:00:37 -04005121 if (!PageDirty(page))
Chris Masond2c3f4f2008-11-19 12:44:22 -05005122 continue;
5123
Chris Masona61e6f22008-07-22 11:18:08 -04005124 lock_page(page);
Chris Masoneb14ab82011-02-10 12:35:00 -05005125 WARN_ON(!PagePrivate(page));
5126
Chris Masond1310b22008-01-24 16:13:08 -05005127 clear_page_dirty_for_io(page);
Sven Wegener0ee0fda2008-07-30 16:54:26 -04005128 spin_lock_irq(&page->mapping->tree_lock);
Chris Masond1310b22008-01-24 16:13:08 -05005129 if (!PageDirty(page)) {
5130 radix_tree_tag_clear(&page->mapping->page_tree,
5131 page_index(page),
5132 PAGECACHE_TAG_DIRTY);
5133 }
Sven Wegener0ee0fda2008-07-30 16:54:26 -04005134 spin_unlock_irq(&page->mapping->tree_lock);
Chris Masonbf0da8c2011-11-04 12:29:37 -04005135 ClearPageError(page);
Chris Masona61e6f22008-07-22 11:18:08 -04005136 unlock_page(page);
Chris Masond1310b22008-01-24 16:13:08 -05005137 }
Josef Bacik0b32f4b2012-03-13 09:38:00 -04005138 WARN_ON(atomic_read(&eb->refs) == 0);
Chris Masond1310b22008-01-24 16:13:08 -05005139}
Chris Masond1310b22008-01-24 16:13:08 -05005140
Josef Bacik0b32f4b2012-03-13 09:38:00 -04005141int set_extent_buffer_dirty(struct extent_buffer *eb)
Chris Masond1310b22008-01-24 16:13:08 -05005142{
5143 unsigned long i;
5144 unsigned long num_pages;
Chris Masonb9473432009-03-13 11:00:37 -04005145 int was_dirty = 0;
Chris Masond1310b22008-01-24 16:13:08 -05005146
Josef Bacik0b32f4b2012-03-13 09:38:00 -04005147 check_buffer_tree_ref(eb);
5148
Chris Masonb9473432009-03-13 11:00:37 -04005149 was_dirty = test_and_set_bit(EXTENT_BUFFER_DIRTY, &eb->bflags);
Josef Bacik0b32f4b2012-03-13 09:38:00 -04005150
Chris Masond1310b22008-01-24 16:13:08 -05005151 num_pages = num_extent_pages(eb->start, eb->len);
Josef Bacik3083ee22012-03-09 16:01:49 -05005152 WARN_ON(atomic_read(&eb->refs) == 0);
Josef Bacik0b32f4b2012-03-13 09:38:00 -04005153 WARN_ON(!test_bit(EXTENT_BUFFER_TREE_REF, &eb->bflags));
5154
Chris Masonb9473432009-03-13 11:00:37 -04005155 for (i = 0; i < num_pages; i++)
David Sterbafb85fc92014-07-31 01:03:53 +02005156 set_page_dirty(eb->pages[i]);
Chris Masonb9473432009-03-13 11:00:37 -04005157 return was_dirty;
Chris Masond1310b22008-01-24 16:13:08 -05005158}
Chris Masond1310b22008-01-24 16:13:08 -05005159
Josef Bacik0b32f4b2012-03-13 09:38:00 -04005160int clear_extent_buffer_uptodate(struct extent_buffer *eb)
Chris Mason1259ab72008-05-12 13:39:03 -04005161{
5162 unsigned long i;
5163 struct page *page;
5164 unsigned long num_pages;
5165
Chris Masonb4ce94d2009-02-04 09:25:08 -05005166 clear_bit(EXTENT_BUFFER_UPTODATE, &eb->bflags);
Josef Bacik0b32f4b2012-03-13 09:38:00 -04005167 num_pages = num_extent_pages(eb->start, eb->len);
Chris Mason1259ab72008-05-12 13:39:03 -04005168 for (i = 0; i < num_pages; i++) {
David Sterbafb85fc92014-07-31 01:03:53 +02005169 page = eb->pages[i];
Chris Mason33958dc2008-07-30 10:29:12 -04005170 if (page)
5171 ClearPageUptodate(page);
Chris Mason1259ab72008-05-12 13:39:03 -04005172 }
5173 return 0;
5174}
5175
Josef Bacik0b32f4b2012-03-13 09:38:00 -04005176int set_extent_buffer_uptodate(struct extent_buffer *eb)
Chris Masond1310b22008-01-24 16:13:08 -05005177{
5178 unsigned long i;
5179 struct page *page;
5180 unsigned long num_pages;
5181
Josef Bacik0b32f4b2012-03-13 09:38:00 -04005182 set_bit(EXTENT_BUFFER_UPTODATE, &eb->bflags);
Chris Masond1310b22008-01-24 16:13:08 -05005183 num_pages = num_extent_pages(eb->start, eb->len);
Chris Masond1310b22008-01-24 16:13:08 -05005184 for (i = 0; i < num_pages; i++) {
David Sterbafb85fc92014-07-31 01:03:53 +02005185 page = eb->pages[i];
Chris Masond1310b22008-01-24 16:13:08 -05005186 SetPageUptodate(page);
5187 }
5188 return 0;
5189}
Chris Masond1310b22008-01-24 16:13:08 -05005190
Josef Bacik0b32f4b2012-03-13 09:38:00 -04005191int extent_buffer_uptodate(struct extent_buffer *eb)
Chris Masond1310b22008-01-24 16:13:08 -05005192{
Josef Bacik0b32f4b2012-03-13 09:38:00 -04005193 return test_bit(EXTENT_BUFFER_UPTODATE, &eb->bflags);
Chris Masond1310b22008-01-24 16:13:08 -05005194}
Chris Masond1310b22008-01-24 16:13:08 -05005195
5196int read_extent_buffer_pages(struct extent_io_tree *tree,
Arne Jansenbb82ab82011-06-10 14:06:53 +02005197 struct extent_buffer *eb, u64 start, int wait,
Chris Masonf1885912008-04-09 16:28:12 -04005198 get_extent_t *get_extent, int mirror_num)
Chris Masond1310b22008-01-24 16:13:08 -05005199{
5200 unsigned long i;
5201 unsigned long start_i;
5202 struct page *page;
5203 int err;
5204 int ret = 0;
Chris Masonce9adaa2008-04-09 16:28:12 -04005205 int locked_pages = 0;
5206 int all_uptodate = 1;
Chris Masond1310b22008-01-24 16:13:08 -05005207 unsigned long num_pages;
Chris Mason727011e2010-08-06 13:21:20 -04005208 unsigned long num_reads = 0;
Chris Masona86c12c2008-02-07 10:50:54 -05005209 struct bio *bio = NULL;
Chris Masonc8b97812008-10-29 14:49:59 -04005210 unsigned long bio_flags = 0;
Chris Masona86c12c2008-02-07 10:50:54 -05005211
Chris Masonb4ce94d2009-02-04 09:25:08 -05005212 if (test_bit(EXTENT_BUFFER_UPTODATE, &eb->bflags))
Chris Masond1310b22008-01-24 16:13:08 -05005213 return 0;
5214
Chris Masond1310b22008-01-24 16:13:08 -05005215 if (start) {
5216 WARN_ON(start < eb->start);
5217 start_i = (start >> PAGE_CACHE_SHIFT) -
5218 (eb->start >> PAGE_CACHE_SHIFT);
5219 } else {
5220 start_i = 0;
5221 }
5222
5223 num_pages = num_extent_pages(eb->start, eb->len);
5224 for (i = start_i; i < num_pages; i++) {
David Sterbafb85fc92014-07-31 01:03:53 +02005225 page = eb->pages[i];
Arne Jansenbb82ab82011-06-10 14:06:53 +02005226 if (wait == WAIT_NONE) {
David Woodhouse2db04962008-08-07 11:19:43 -04005227 if (!trylock_page(page))
Chris Masonce9adaa2008-04-09 16:28:12 -04005228 goto unlock_exit;
Chris Masond1310b22008-01-24 16:13:08 -05005229 } else {
5230 lock_page(page);
5231 }
Chris Masonce9adaa2008-04-09 16:28:12 -04005232 locked_pages++;
Chris Mason727011e2010-08-06 13:21:20 -04005233 if (!PageUptodate(page)) {
5234 num_reads++;
Chris Masonce9adaa2008-04-09 16:28:12 -04005235 all_uptodate = 0;
Chris Mason727011e2010-08-06 13:21:20 -04005236 }
Chris Masonce9adaa2008-04-09 16:28:12 -04005237 }
5238 if (all_uptodate) {
5239 if (start_i == 0)
Chris Masonb4ce94d2009-02-04 09:25:08 -05005240 set_bit(EXTENT_BUFFER_UPTODATE, &eb->bflags);
Chris Masonce9adaa2008-04-09 16:28:12 -04005241 goto unlock_exit;
5242 }
5243
Filipe Manana656f30d2014-09-26 12:25:56 +01005244 clear_bit(EXTENT_BUFFER_READ_ERR, &eb->bflags);
Josef Bacik5cf1ab52012-04-16 09:42:26 -04005245 eb->read_mirror = 0;
Josef Bacik0b32f4b2012-03-13 09:38:00 -04005246 atomic_set(&eb->io_pages, num_reads);
Chris Masonce9adaa2008-04-09 16:28:12 -04005247 for (i = start_i; i < num_pages; i++) {
David Sterbafb85fc92014-07-31 01:03:53 +02005248 page = eb->pages[i];
Chris Masonce9adaa2008-04-09 16:28:12 -04005249 if (!PageUptodate(page)) {
Chris Masonf1885912008-04-09 16:28:12 -04005250 ClearPageError(page);
Chris Masona86c12c2008-02-07 10:50:54 -05005251 err = __extent_read_full_page(tree, page,
Chris Masonf1885912008-04-09 16:28:12 -04005252 get_extent, &bio,
Josef Bacikd4c7ca82013-04-19 19:49:09 -04005253 mirror_num, &bio_flags,
5254 READ | REQ_META);
Chris Masond3977122009-01-05 21:25:51 -05005255 if (err)
Chris Masond1310b22008-01-24 16:13:08 -05005256 ret = err;
Chris Masond1310b22008-01-24 16:13:08 -05005257 } else {
5258 unlock_page(page);
5259 }
5260 }
5261
Jeff Mahoney355808c2011-10-03 23:23:14 -04005262 if (bio) {
Josef Bacikd4c7ca82013-04-19 19:49:09 -04005263 err = submit_one_bio(READ | REQ_META, bio, mirror_num,
5264 bio_flags);
Jeff Mahoney79787ea2012-03-12 16:03:00 +01005265 if (err)
5266 return err;
Jeff Mahoney355808c2011-10-03 23:23:14 -04005267 }
Chris Masona86c12c2008-02-07 10:50:54 -05005268
Arne Jansenbb82ab82011-06-10 14:06:53 +02005269 if (ret || wait != WAIT_COMPLETE)
Chris Masond1310b22008-01-24 16:13:08 -05005270 return ret;
Chris Masond3977122009-01-05 21:25:51 -05005271
Chris Masond1310b22008-01-24 16:13:08 -05005272 for (i = start_i; i < num_pages; i++) {
David Sterbafb85fc92014-07-31 01:03:53 +02005273 page = eb->pages[i];
Chris Masond1310b22008-01-24 16:13:08 -05005274 wait_on_page_locked(page);
Chris Masond3977122009-01-05 21:25:51 -05005275 if (!PageUptodate(page))
Chris Masond1310b22008-01-24 16:13:08 -05005276 ret = -EIO;
Chris Masond1310b22008-01-24 16:13:08 -05005277 }
Chris Masond3977122009-01-05 21:25:51 -05005278
Chris Masond1310b22008-01-24 16:13:08 -05005279 return ret;
Chris Masonce9adaa2008-04-09 16:28:12 -04005280
5281unlock_exit:
5282 i = start_i;
Chris Masond3977122009-01-05 21:25:51 -05005283 while (locked_pages > 0) {
David Sterbafb85fc92014-07-31 01:03:53 +02005284 page = eb->pages[i];
Chris Masonce9adaa2008-04-09 16:28:12 -04005285 i++;
5286 unlock_page(page);
5287 locked_pages--;
5288 }
5289 return ret;
Chris Masond1310b22008-01-24 16:13:08 -05005290}
Chris Masond1310b22008-01-24 16:13:08 -05005291
5292void read_extent_buffer(struct extent_buffer *eb, void *dstv,
5293 unsigned long start,
5294 unsigned long len)
5295{
5296 size_t cur;
5297 size_t offset;
5298 struct page *page;
5299 char *kaddr;
5300 char *dst = (char *)dstv;
5301 size_t start_offset = eb->start & ((u64)PAGE_CACHE_SIZE - 1);
5302 unsigned long i = (start_offset + start) >> PAGE_CACHE_SHIFT;
Chris Masond1310b22008-01-24 16:13:08 -05005303
5304 WARN_ON(start > eb->len);
5305 WARN_ON(start + len > eb->start + eb->len);
5306
Geert Uytterhoeven778746b2013-08-20 13:20:16 +02005307 offset = (start_offset + start) & (PAGE_CACHE_SIZE - 1);
Chris Masond1310b22008-01-24 16:13:08 -05005308
Chris Masond3977122009-01-05 21:25:51 -05005309 while (len > 0) {
David Sterbafb85fc92014-07-31 01:03:53 +02005310 page = eb->pages[i];
Chris Masond1310b22008-01-24 16:13:08 -05005311
5312 cur = min(len, (PAGE_CACHE_SIZE - offset));
Chris Masona6591712011-07-19 12:04:14 -04005313 kaddr = page_address(page);
Chris Masond1310b22008-01-24 16:13:08 -05005314 memcpy(dst, kaddr + offset, cur);
Chris Masond1310b22008-01-24 16:13:08 -05005315
5316 dst += cur;
5317 len -= cur;
5318 offset = 0;
5319 i++;
5320 }
5321}
Chris Masond1310b22008-01-24 16:13:08 -05005322
Gerhard Heift550ac1d2014-01-30 16:24:01 +01005323int read_extent_buffer_to_user(struct extent_buffer *eb, void __user *dstv,
5324 unsigned long start,
5325 unsigned long len)
5326{
5327 size_t cur;
5328 size_t offset;
5329 struct page *page;
5330 char *kaddr;
5331 char __user *dst = (char __user *)dstv;
5332 size_t start_offset = eb->start & ((u64)PAGE_CACHE_SIZE - 1);
5333 unsigned long i = (start_offset + start) >> PAGE_CACHE_SHIFT;
5334 int ret = 0;
5335
5336 WARN_ON(start > eb->len);
5337 WARN_ON(start + len > eb->start + eb->len);
5338
5339 offset = (start_offset + start) & (PAGE_CACHE_SIZE - 1);
5340
5341 while (len > 0) {
David Sterbafb85fc92014-07-31 01:03:53 +02005342 page = eb->pages[i];
Gerhard Heift550ac1d2014-01-30 16:24:01 +01005343
5344 cur = min(len, (PAGE_CACHE_SIZE - offset));
5345 kaddr = page_address(page);
5346 if (copy_to_user(dst, kaddr + offset, cur)) {
5347 ret = -EFAULT;
5348 break;
5349 }
5350
5351 dst += cur;
5352 len -= cur;
5353 offset = 0;
5354 i++;
5355 }
5356
5357 return ret;
5358}
5359
Chris Masond1310b22008-01-24 16:13:08 -05005360int map_private_extent_buffer(struct extent_buffer *eb, unsigned long start,
Chris Masona6591712011-07-19 12:04:14 -04005361 unsigned long min_len, char **map,
Chris Masond1310b22008-01-24 16:13:08 -05005362 unsigned long *map_start,
Chris Masona6591712011-07-19 12:04:14 -04005363 unsigned long *map_len)
Chris Masond1310b22008-01-24 16:13:08 -05005364{
5365 size_t offset = start & (PAGE_CACHE_SIZE - 1);
5366 char *kaddr;
5367 struct page *p;
5368 size_t start_offset = eb->start & ((u64)PAGE_CACHE_SIZE - 1);
5369 unsigned long i = (start_offset + start) >> PAGE_CACHE_SHIFT;
5370 unsigned long end_i = (start_offset + start + min_len - 1) >>
5371 PAGE_CACHE_SHIFT;
5372
5373 if (i != end_i)
5374 return -EINVAL;
5375
5376 if (i == 0) {
5377 offset = start_offset;
5378 *map_start = 0;
5379 } else {
5380 offset = 0;
5381 *map_start = ((u64)i << PAGE_CACHE_SHIFT) - start_offset;
5382 }
Chris Masond3977122009-01-05 21:25:51 -05005383
Chris Masond1310b22008-01-24 16:13:08 -05005384 if (start + min_len > eb->len) {
Julia Lawall31b1a2b2012-11-03 10:58:34 +00005385 WARN(1, KERN_ERR "btrfs bad mapping eb start %llu len %lu, "
Geert Uytterhoevenc1c9ff72013-08-20 13:20:07 +02005386 "wanted %lu %lu\n",
5387 eb->start, eb->len, start, min_len);
Josef Bacik850265332011-03-15 14:52:12 -04005388 return -EINVAL;
Chris Masond1310b22008-01-24 16:13:08 -05005389 }
5390
David Sterbafb85fc92014-07-31 01:03:53 +02005391 p = eb->pages[i];
Chris Masona6591712011-07-19 12:04:14 -04005392 kaddr = page_address(p);
Chris Masond1310b22008-01-24 16:13:08 -05005393 *map = kaddr + offset;
5394 *map_len = PAGE_CACHE_SIZE - offset;
5395 return 0;
5396}
Chris Masond1310b22008-01-24 16:13:08 -05005397
Chris Masond1310b22008-01-24 16:13:08 -05005398int memcmp_extent_buffer(struct extent_buffer *eb, const void *ptrv,
5399 unsigned long start,
5400 unsigned long len)
5401{
5402 size_t cur;
5403 size_t offset;
5404 struct page *page;
5405 char *kaddr;
5406 char *ptr = (char *)ptrv;
5407 size_t start_offset = eb->start & ((u64)PAGE_CACHE_SIZE - 1);
5408 unsigned long i = (start_offset + start) >> PAGE_CACHE_SHIFT;
5409 int ret = 0;
5410
5411 WARN_ON(start > eb->len);
5412 WARN_ON(start + len > eb->start + eb->len);
5413
Geert Uytterhoeven778746b2013-08-20 13:20:16 +02005414 offset = (start_offset + start) & (PAGE_CACHE_SIZE - 1);
Chris Masond1310b22008-01-24 16:13:08 -05005415
Chris Masond3977122009-01-05 21:25:51 -05005416 while (len > 0) {
David Sterbafb85fc92014-07-31 01:03:53 +02005417 page = eb->pages[i];
Chris Masond1310b22008-01-24 16:13:08 -05005418
5419 cur = min(len, (PAGE_CACHE_SIZE - offset));
5420
Chris Masona6591712011-07-19 12:04:14 -04005421 kaddr = page_address(page);
Chris Masond1310b22008-01-24 16:13:08 -05005422 ret = memcmp(ptr, kaddr + offset, cur);
Chris Masond1310b22008-01-24 16:13:08 -05005423 if (ret)
5424 break;
5425
5426 ptr += cur;
5427 len -= cur;
5428 offset = 0;
5429 i++;
5430 }
5431 return ret;
5432}
Chris Masond1310b22008-01-24 16:13:08 -05005433
5434void write_extent_buffer(struct extent_buffer *eb, const void *srcv,
5435 unsigned long start, unsigned long len)
5436{
5437 size_t cur;
5438 size_t offset;
5439 struct page *page;
5440 char *kaddr;
5441 char *src = (char *)srcv;
5442 size_t start_offset = eb->start & ((u64)PAGE_CACHE_SIZE - 1);
5443 unsigned long i = (start_offset + start) >> PAGE_CACHE_SHIFT;
5444
5445 WARN_ON(start > eb->len);
5446 WARN_ON(start + len > eb->start + eb->len);
5447
Geert Uytterhoeven778746b2013-08-20 13:20:16 +02005448 offset = (start_offset + start) & (PAGE_CACHE_SIZE - 1);
Chris Masond1310b22008-01-24 16:13:08 -05005449
Chris Masond3977122009-01-05 21:25:51 -05005450 while (len > 0) {
David Sterbafb85fc92014-07-31 01:03:53 +02005451 page = eb->pages[i];
Chris Masond1310b22008-01-24 16:13:08 -05005452 WARN_ON(!PageUptodate(page));
5453
5454 cur = min(len, PAGE_CACHE_SIZE - offset);
Chris Masona6591712011-07-19 12:04:14 -04005455 kaddr = page_address(page);
Chris Masond1310b22008-01-24 16:13:08 -05005456 memcpy(kaddr + offset, src, cur);
Chris Masond1310b22008-01-24 16:13:08 -05005457
5458 src += cur;
5459 len -= cur;
5460 offset = 0;
5461 i++;
5462 }
5463}
Chris Masond1310b22008-01-24 16:13:08 -05005464
5465void memset_extent_buffer(struct extent_buffer *eb, char c,
5466 unsigned long start, unsigned long len)
5467{
5468 size_t cur;
5469 size_t offset;
5470 struct page *page;
5471 char *kaddr;
5472 size_t start_offset = eb->start & ((u64)PAGE_CACHE_SIZE - 1);
5473 unsigned long i = (start_offset + start) >> PAGE_CACHE_SHIFT;
5474
5475 WARN_ON(start > eb->len);
5476 WARN_ON(start + len > eb->start + eb->len);
5477
Geert Uytterhoeven778746b2013-08-20 13:20:16 +02005478 offset = (start_offset + start) & (PAGE_CACHE_SIZE - 1);
Chris Masond1310b22008-01-24 16:13:08 -05005479
Chris Masond3977122009-01-05 21:25:51 -05005480 while (len > 0) {
David Sterbafb85fc92014-07-31 01:03:53 +02005481 page = eb->pages[i];
Chris Masond1310b22008-01-24 16:13:08 -05005482 WARN_ON(!PageUptodate(page));
5483
5484 cur = min(len, PAGE_CACHE_SIZE - offset);
Chris Masona6591712011-07-19 12:04:14 -04005485 kaddr = page_address(page);
Chris Masond1310b22008-01-24 16:13:08 -05005486 memset(kaddr + offset, c, cur);
Chris Masond1310b22008-01-24 16:13:08 -05005487
5488 len -= cur;
5489 offset = 0;
5490 i++;
5491 }
5492}
Chris Masond1310b22008-01-24 16:13:08 -05005493
5494void copy_extent_buffer(struct extent_buffer *dst, struct extent_buffer *src,
5495 unsigned long dst_offset, unsigned long src_offset,
5496 unsigned long len)
5497{
5498 u64 dst_len = dst->len;
5499 size_t cur;
5500 size_t offset;
5501 struct page *page;
5502 char *kaddr;
5503 size_t start_offset = dst->start & ((u64)PAGE_CACHE_SIZE - 1);
5504 unsigned long i = (start_offset + dst_offset) >> PAGE_CACHE_SHIFT;
5505
5506 WARN_ON(src->len != dst_len);
5507
5508 offset = (start_offset + dst_offset) &
Geert Uytterhoeven778746b2013-08-20 13:20:16 +02005509 (PAGE_CACHE_SIZE - 1);
Chris Masond1310b22008-01-24 16:13:08 -05005510
Chris Masond3977122009-01-05 21:25:51 -05005511 while (len > 0) {
David Sterbafb85fc92014-07-31 01:03:53 +02005512 page = dst->pages[i];
Chris Masond1310b22008-01-24 16:13:08 -05005513 WARN_ON(!PageUptodate(page));
5514
5515 cur = min(len, (unsigned long)(PAGE_CACHE_SIZE - offset));
5516
Chris Masona6591712011-07-19 12:04:14 -04005517 kaddr = page_address(page);
Chris Masond1310b22008-01-24 16:13:08 -05005518 read_extent_buffer(src, kaddr + offset, src_offset, cur);
Chris Masond1310b22008-01-24 16:13:08 -05005519
5520 src_offset += cur;
5521 len -= cur;
5522 offset = 0;
5523 i++;
5524 }
5525}
Chris Masond1310b22008-01-24 16:13:08 -05005526
Sergei Trofimovich33872062011-04-11 21:52:52 +00005527static inline bool areas_overlap(unsigned long src, unsigned long dst, unsigned long len)
5528{
5529 unsigned long distance = (src > dst) ? src - dst : dst - src;
5530 return distance < len;
5531}
5532
Chris Masond1310b22008-01-24 16:13:08 -05005533static void copy_pages(struct page *dst_page, struct page *src_page,
5534 unsigned long dst_off, unsigned long src_off,
5535 unsigned long len)
5536{
Chris Masona6591712011-07-19 12:04:14 -04005537 char *dst_kaddr = page_address(dst_page);
Chris Masond1310b22008-01-24 16:13:08 -05005538 char *src_kaddr;
Chris Mason727011e2010-08-06 13:21:20 -04005539 int must_memmove = 0;
Chris Masond1310b22008-01-24 16:13:08 -05005540
Sergei Trofimovich33872062011-04-11 21:52:52 +00005541 if (dst_page != src_page) {
Chris Masona6591712011-07-19 12:04:14 -04005542 src_kaddr = page_address(src_page);
Sergei Trofimovich33872062011-04-11 21:52:52 +00005543 } else {
Chris Masond1310b22008-01-24 16:13:08 -05005544 src_kaddr = dst_kaddr;
Chris Mason727011e2010-08-06 13:21:20 -04005545 if (areas_overlap(src_off, dst_off, len))
5546 must_memmove = 1;
Sergei Trofimovich33872062011-04-11 21:52:52 +00005547 }
Chris Masond1310b22008-01-24 16:13:08 -05005548
Chris Mason727011e2010-08-06 13:21:20 -04005549 if (must_memmove)
5550 memmove(dst_kaddr + dst_off, src_kaddr + src_off, len);
5551 else
5552 memcpy(dst_kaddr + dst_off, src_kaddr + src_off, len);
Chris Masond1310b22008-01-24 16:13:08 -05005553}
5554
5555void memcpy_extent_buffer(struct extent_buffer *dst, unsigned long dst_offset,
5556 unsigned long src_offset, unsigned long len)
5557{
5558 size_t cur;
5559 size_t dst_off_in_page;
5560 size_t src_off_in_page;
5561 size_t start_offset = dst->start & ((u64)PAGE_CACHE_SIZE - 1);
5562 unsigned long dst_i;
5563 unsigned long src_i;
5564
5565 if (src_offset + len > dst->len) {
Frank Holtonefe120a2013-12-20 11:37:06 -05005566 printk(KERN_ERR "BTRFS: memmove bogus src_offset %lu move "
Chris Masond3977122009-01-05 21:25:51 -05005567 "len %lu dst len %lu\n", src_offset, len, dst->len);
Chris Masond1310b22008-01-24 16:13:08 -05005568 BUG_ON(1);
5569 }
5570 if (dst_offset + len > dst->len) {
Frank Holtonefe120a2013-12-20 11:37:06 -05005571 printk(KERN_ERR "BTRFS: memmove bogus dst_offset %lu move "
Chris Masond3977122009-01-05 21:25:51 -05005572 "len %lu dst len %lu\n", dst_offset, len, dst->len);
Chris Masond1310b22008-01-24 16:13:08 -05005573 BUG_ON(1);
5574 }
5575
Chris Masond3977122009-01-05 21:25:51 -05005576 while (len > 0) {
Chris Masond1310b22008-01-24 16:13:08 -05005577 dst_off_in_page = (start_offset + dst_offset) &
Geert Uytterhoeven778746b2013-08-20 13:20:16 +02005578 (PAGE_CACHE_SIZE - 1);
Chris Masond1310b22008-01-24 16:13:08 -05005579 src_off_in_page = (start_offset + src_offset) &
Geert Uytterhoeven778746b2013-08-20 13:20:16 +02005580 (PAGE_CACHE_SIZE - 1);
Chris Masond1310b22008-01-24 16:13:08 -05005581
5582 dst_i = (start_offset + dst_offset) >> PAGE_CACHE_SHIFT;
5583 src_i = (start_offset + src_offset) >> PAGE_CACHE_SHIFT;
5584
5585 cur = min(len, (unsigned long)(PAGE_CACHE_SIZE -
5586 src_off_in_page));
5587 cur = min_t(unsigned long, cur,
5588 (unsigned long)(PAGE_CACHE_SIZE - dst_off_in_page));
5589
David Sterbafb85fc92014-07-31 01:03:53 +02005590 copy_pages(dst->pages[dst_i], dst->pages[src_i],
Chris Masond1310b22008-01-24 16:13:08 -05005591 dst_off_in_page, src_off_in_page, cur);
5592
5593 src_offset += cur;
5594 dst_offset += cur;
5595 len -= cur;
5596 }
5597}
Chris Masond1310b22008-01-24 16:13:08 -05005598
5599void memmove_extent_buffer(struct extent_buffer *dst, unsigned long dst_offset,
5600 unsigned long src_offset, unsigned long len)
5601{
5602 size_t cur;
5603 size_t dst_off_in_page;
5604 size_t src_off_in_page;
5605 unsigned long dst_end = dst_offset + len - 1;
5606 unsigned long src_end = src_offset + len - 1;
5607 size_t start_offset = dst->start & ((u64)PAGE_CACHE_SIZE - 1);
5608 unsigned long dst_i;
5609 unsigned long src_i;
5610
5611 if (src_offset + len > dst->len) {
Frank Holtonefe120a2013-12-20 11:37:06 -05005612 printk(KERN_ERR "BTRFS: memmove bogus src_offset %lu move "
Chris Masond3977122009-01-05 21:25:51 -05005613 "len %lu len %lu\n", src_offset, len, dst->len);
Chris Masond1310b22008-01-24 16:13:08 -05005614 BUG_ON(1);
5615 }
5616 if (dst_offset + len > dst->len) {
Frank Holtonefe120a2013-12-20 11:37:06 -05005617 printk(KERN_ERR "BTRFS: memmove bogus dst_offset %lu move "
Chris Masond3977122009-01-05 21:25:51 -05005618 "len %lu len %lu\n", dst_offset, len, dst->len);
Chris Masond1310b22008-01-24 16:13:08 -05005619 BUG_ON(1);
5620 }
Chris Mason727011e2010-08-06 13:21:20 -04005621 if (dst_offset < src_offset) {
Chris Masond1310b22008-01-24 16:13:08 -05005622 memcpy_extent_buffer(dst, dst_offset, src_offset, len);
5623 return;
5624 }
Chris Masond3977122009-01-05 21:25:51 -05005625 while (len > 0) {
Chris Masond1310b22008-01-24 16:13:08 -05005626 dst_i = (start_offset + dst_end) >> PAGE_CACHE_SHIFT;
5627 src_i = (start_offset + src_end) >> PAGE_CACHE_SHIFT;
5628
5629 dst_off_in_page = (start_offset + dst_end) &
Geert Uytterhoeven778746b2013-08-20 13:20:16 +02005630 (PAGE_CACHE_SIZE - 1);
Chris Masond1310b22008-01-24 16:13:08 -05005631 src_off_in_page = (start_offset + src_end) &
Geert Uytterhoeven778746b2013-08-20 13:20:16 +02005632 (PAGE_CACHE_SIZE - 1);
Chris Masond1310b22008-01-24 16:13:08 -05005633
5634 cur = min_t(unsigned long, len, src_off_in_page + 1);
5635 cur = min(cur, dst_off_in_page + 1);
David Sterbafb85fc92014-07-31 01:03:53 +02005636 copy_pages(dst->pages[dst_i], dst->pages[src_i],
Chris Masond1310b22008-01-24 16:13:08 -05005637 dst_off_in_page - cur + 1,
5638 src_off_in_page - cur + 1, cur);
5639
5640 dst_end -= cur;
5641 src_end -= cur;
5642 len -= cur;
5643 }
5644}
Chris Mason6af118ce2008-07-22 11:18:07 -04005645
David Sterbaf7a52a42013-04-26 14:56:29 +00005646int try_release_extent_buffer(struct page *page)
Miao Xie19fe0a82010-10-26 20:57:29 -04005647{
Chris Mason6af118ce2008-07-22 11:18:07 -04005648 struct extent_buffer *eb;
Miao Xie897ca6e92010-10-26 20:57:29 -04005649
Miao Xie19fe0a82010-10-26 20:57:29 -04005650 /*
Josef Bacik3083ee22012-03-09 16:01:49 -05005651 * We need to make sure noboody is attaching this page to an eb right
5652 * now.
Miao Xie19fe0a82010-10-26 20:57:29 -04005653 */
Josef Bacik3083ee22012-03-09 16:01:49 -05005654 spin_lock(&page->mapping->private_lock);
5655 if (!PagePrivate(page)) {
5656 spin_unlock(&page->mapping->private_lock);
5657 return 1;
Miao Xie19fe0a82010-10-26 20:57:29 -04005658 }
5659
Josef Bacik3083ee22012-03-09 16:01:49 -05005660 eb = (struct extent_buffer *)page->private;
5661 BUG_ON(!eb);
Miao Xie19fe0a82010-10-26 20:57:29 -04005662
Josef Bacik0b32f4b2012-03-13 09:38:00 -04005663 /*
Josef Bacik3083ee22012-03-09 16:01:49 -05005664 * This is a little awful but should be ok, we need to make sure that
5665 * the eb doesn't disappear out from under us while we're looking at
5666 * this page.
5667 */
5668 spin_lock(&eb->refs_lock);
Josef Bacik0b32f4b2012-03-13 09:38:00 -04005669 if (atomic_read(&eb->refs) != 1 || extent_buffer_under_io(eb)) {
Josef Bacik3083ee22012-03-09 16:01:49 -05005670 spin_unlock(&eb->refs_lock);
5671 spin_unlock(&page->mapping->private_lock);
5672 return 0;
5673 }
5674 spin_unlock(&page->mapping->private_lock);
5675
Josef Bacik3083ee22012-03-09 16:01:49 -05005676 /*
5677 * If tree ref isn't set then we know the ref on this eb is a real ref,
5678 * so just return, this page will likely be freed soon anyway.
5679 */
5680 if (!test_and_clear_bit(EXTENT_BUFFER_TREE_REF, &eb->bflags)) {
5681 spin_unlock(&eb->refs_lock);
5682 return 0;
5683 }
Josef Bacik3083ee22012-03-09 16:01:49 -05005684
David Sterbaf7a52a42013-04-26 14:56:29 +00005685 return release_extent_buffer(eb);
Chris Mason6af118ce2008-07-22 11:18:07 -04005686}