blob: 032abfbebe76e6e4ba41e25d7c9326079ed71f68 [file] [log] [blame]
Chris Masond1310b22008-01-24 16:13:08 -05001#include <linux/bitops.h>
2#include <linux/slab.h>
3#include <linux/bio.h>
4#include <linux/mm.h>
Chris Masond1310b22008-01-24 16:13:08 -05005#include <linux/pagemap.h>
6#include <linux/page-flags.h>
Chris Masond1310b22008-01-24 16:13:08 -05007#include <linux/spinlock.h>
8#include <linux/blkdev.h>
9#include <linux/swap.h>
Chris Masond1310b22008-01-24 16:13:08 -050010#include <linux/writeback.h>
11#include <linux/pagevec.h>
Linus Torvalds268bb0c2011-05-20 12:50:29 -070012#include <linux/prefetch.h>
Dan Magenheimer90a887c2011-05-26 10:01:56 -060013#include <linux/cleancache.h>
Chris Masond1310b22008-01-24 16:13:08 -050014#include "extent_io.h"
15#include "extent_map.h"
David Woodhouse902b22f2008-08-20 08:51:49 -040016#include "ctree.h"
17#include "btrfs_inode.h"
Jan Schmidt4a54c8c2011-07-22 15:41:52 +020018#include "volumes.h"
Stefan Behrens21adbd52011-11-09 13:44:05 +010019#include "check-integrity.h"
Josef Bacik0b32f4b2012-03-13 09:38:00 -040020#include "locking.h"
Josef Bacik606686e2012-06-04 14:03:51 -040021#include "rcu-string.h"
Liu Bofe09e162013-09-22 12:54:23 +080022#include "backref.h"
Chris Masond1310b22008-01-24 16:13:08 -050023
Chris Masond1310b22008-01-24 16:13:08 -050024static struct kmem_cache *extent_state_cache;
25static struct kmem_cache *extent_buffer_cache;
Chris Mason9be33952013-05-17 18:30:14 -040026static struct bio_set *btrfs_bioset;
Chris Masond1310b22008-01-24 16:13:08 -050027
Filipe Manana27a35072014-07-06 20:09:59 +010028static inline bool extent_state_in_tree(const struct extent_state *state)
29{
30 return !RB_EMPTY_NODE(&state->rb_node);
31}
32
Eric Sandeen6d49ba12013-04-22 16:12:31 +000033#ifdef CONFIG_BTRFS_DEBUG
Chris Masond1310b22008-01-24 16:13:08 -050034static LIST_HEAD(buffers);
35static LIST_HEAD(states);
Chris Mason4bef0842008-09-08 11:18:08 -040036
Chris Masond3977122009-01-05 21:25:51 -050037static DEFINE_SPINLOCK(leak_lock);
Eric Sandeen6d49ba12013-04-22 16:12:31 +000038
39static inline
40void btrfs_leak_debug_add(struct list_head *new, struct list_head *head)
41{
42 unsigned long flags;
43
44 spin_lock_irqsave(&leak_lock, flags);
45 list_add(new, head);
46 spin_unlock_irqrestore(&leak_lock, flags);
47}
48
49static inline
50void btrfs_leak_debug_del(struct list_head *entry)
51{
52 unsigned long flags;
53
54 spin_lock_irqsave(&leak_lock, flags);
55 list_del(entry);
56 spin_unlock_irqrestore(&leak_lock, flags);
57}
58
59static inline
60void btrfs_leak_debug_check(void)
61{
62 struct extent_state *state;
63 struct extent_buffer *eb;
64
65 while (!list_empty(&states)) {
66 state = list_entry(states.next, struct extent_state, leak_list);
David Sterba9ee49a042015-01-14 19:52:13 +010067 pr_err("BTRFS: state leak: start %llu end %llu state %u in tree %d refs %d\n",
Filipe Manana27a35072014-07-06 20:09:59 +010068 state->start, state->end, state->state,
69 extent_state_in_tree(state),
Geert Uytterhoevenc1c9ff72013-08-20 13:20:07 +020070 atomic_read(&state->refs));
Eric Sandeen6d49ba12013-04-22 16:12:31 +000071 list_del(&state->leak_list);
72 kmem_cache_free(extent_state_cache, state);
73 }
74
75 while (!list_empty(&buffers)) {
76 eb = list_entry(buffers.next, struct extent_buffer, leak_list);
Frank Holtonefe120a2013-12-20 11:37:06 -050077 printk(KERN_ERR "BTRFS: buffer leak start %llu len %lu "
Geert Uytterhoevenc1c9ff72013-08-20 13:20:07 +020078 "refs %d\n",
79 eb->start, eb->len, atomic_read(&eb->refs));
Eric Sandeen6d49ba12013-04-22 16:12:31 +000080 list_del(&eb->leak_list);
81 kmem_cache_free(extent_buffer_cache, eb);
82 }
83}
David Sterba8d599ae2013-04-30 15:22:23 +000084
Josef Bacika5dee372013-12-13 10:02:44 -050085#define btrfs_debug_check_extent_io_range(tree, start, end) \
86 __btrfs_debug_check_extent_io_range(__func__, (tree), (start), (end))
David Sterba8d599ae2013-04-30 15:22:23 +000087static inline void __btrfs_debug_check_extent_io_range(const char *caller,
Josef Bacika5dee372013-12-13 10:02:44 -050088 struct extent_io_tree *tree, u64 start, u64 end)
David Sterba8d599ae2013-04-30 15:22:23 +000089{
Josef Bacika5dee372013-12-13 10:02:44 -050090 struct inode *inode;
91 u64 isize;
David Sterba8d599ae2013-04-30 15:22:23 +000092
Josef Bacika5dee372013-12-13 10:02:44 -050093 if (!tree->mapping)
94 return;
95
96 inode = tree->mapping->host;
97 isize = i_size_read(inode);
David Sterba8d599ae2013-04-30 15:22:23 +000098 if (end >= PAGE_SIZE && (end % 2) == 0 && end != isize - 1) {
99 printk_ratelimited(KERN_DEBUG
Frank Holtonefe120a2013-12-20 11:37:06 -0500100 "BTRFS: %s: ino %llu isize %llu odd range [%llu,%llu]\n",
Geert Uytterhoevenc1c9ff72013-08-20 13:20:07 +0200101 caller, btrfs_ino(inode), isize, start, end);
David Sterba8d599ae2013-04-30 15:22:23 +0000102 }
103}
Eric Sandeen6d49ba12013-04-22 16:12:31 +0000104#else
105#define btrfs_leak_debug_add(new, head) do {} while (0)
106#define btrfs_leak_debug_del(entry) do {} while (0)
107#define btrfs_leak_debug_check() do {} while (0)
David Sterba8d599ae2013-04-30 15:22:23 +0000108#define btrfs_debug_check_extent_io_range(c, s, e) do {} while (0)
Chris Mason4bef0842008-09-08 11:18:08 -0400109#endif
Chris Masond1310b22008-01-24 16:13:08 -0500110
Chris Masond1310b22008-01-24 16:13:08 -0500111#define BUFFER_LRU_MAX 64
112
113struct tree_entry {
114 u64 start;
115 u64 end;
Chris Masond1310b22008-01-24 16:13:08 -0500116 struct rb_node rb_node;
117};
118
119struct extent_page_data {
120 struct bio *bio;
121 struct extent_io_tree *tree;
122 get_extent_t *get_extent;
Josef Bacikde0022b2012-09-25 14:25:58 -0400123 unsigned long bio_flags;
Chris Mason771ed682008-11-06 22:02:51 -0500124
125 /* tells writepage not to lock the state bits for this range
126 * it still does the unlocking
127 */
Chris Masonffbd5172009-04-20 15:50:09 -0400128 unsigned int extent_locked:1;
129
130 /* tells the submit_bio code to use a WRITE_SYNC */
131 unsigned int sync_io:1;
Chris Masond1310b22008-01-24 16:13:08 -0500132};
133
Josef Bacik0b32f4b2012-03-13 09:38:00 -0400134static noinline void flush_write_bio(void *data);
Jeff Mahoneyc2d904e2011-10-03 23:22:32 -0400135static inline struct btrfs_fs_info *
136tree_fs_info(struct extent_io_tree *tree)
137{
Josef Bacika5dee372013-12-13 10:02:44 -0500138 if (!tree->mapping)
139 return NULL;
Jeff Mahoneyc2d904e2011-10-03 23:22:32 -0400140 return btrfs_sb(tree->mapping->host->i_sb);
141}
Josef Bacik0b32f4b2012-03-13 09:38:00 -0400142
Chris Masond1310b22008-01-24 16:13:08 -0500143int __init extent_io_init(void)
144{
David Sterba837e1972012-09-07 03:00:48 -0600145 extent_state_cache = kmem_cache_create("btrfs_extent_state",
Christoph Hellwig9601e3f2009-04-13 15:33:09 +0200146 sizeof(struct extent_state), 0,
147 SLAB_RECLAIM_ACCOUNT | SLAB_MEM_SPREAD, NULL);
Chris Masond1310b22008-01-24 16:13:08 -0500148 if (!extent_state_cache)
149 return -ENOMEM;
150
David Sterba837e1972012-09-07 03:00:48 -0600151 extent_buffer_cache = kmem_cache_create("btrfs_extent_buffer",
Christoph Hellwig9601e3f2009-04-13 15:33:09 +0200152 sizeof(struct extent_buffer), 0,
153 SLAB_RECLAIM_ACCOUNT | SLAB_MEM_SPREAD, NULL);
Chris Masond1310b22008-01-24 16:13:08 -0500154 if (!extent_buffer_cache)
155 goto free_state_cache;
Chris Mason9be33952013-05-17 18:30:14 -0400156
157 btrfs_bioset = bioset_create(BIO_POOL_SIZE,
158 offsetof(struct btrfs_io_bio, bio));
159 if (!btrfs_bioset)
160 goto free_buffer_cache;
Darrick J. Wongb208c2f2013-09-19 20:37:07 -0700161
162 if (bioset_integrity_create(btrfs_bioset, BIO_POOL_SIZE))
163 goto free_bioset;
164
Chris Masond1310b22008-01-24 16:13:08 -0500165 return 0;
166
Darrick J. Wongb208c2f2013-09-19 20:37:07 -0700167free_bioset:
168 bioset_free(btrfs_bioset);
169 btrfs_bioset = NULL;
170
Chris Mason9be33952013-05-17 18:30:14 -0400171free_buffer_cache:
172 kmem_cache_destroy(extent_buffer_cache);
173 extent_buffer_cache = NULL;
174
Chris Masond1310b22008-01-24 16:13:08 -0500175free_state_cache:
176 kmem_cache_destroy(extent_state_cache);
Chris Mason9be33952013-05-17 18:30:14 -0400177 extent_state_cache = NULL;
Chris Masond1310b22008-01-24 16:13:08 -0500178 return -ENOMEM;
179}
180
181void extent_io_exit(void)
182{
Eric Sandeen6d49ba12013-04-22 16:12:31 +0000183 btrfs_leak_debug_check();
Kirill A. Shutemov8c0a8532012-09-26 11:33:07 +1000184
185 /*
186 * Make sure all delayed rcu free are flushed before we
187 * destroy caches.
188 */
189 rcu_barrier();
Chris Masond1310b22008-01-24 16:13:08 -0500190 if (extent_state_cache)
191 kmem_cache_destroy(extent_state_cache);
192 if (extent_buffer_cache)
193 kmem_cache_destroy(extent_buffer_cache);
Chris Mason9be33952013-05-17 18:30:14 -0400194 if (btrfs_bioset)
195 bioset_free(btrfs_bioset);
Chris Masond1310b22008-01-24 16:13:08 -0500196}
197
198void extent_io_tree_init(struct extent_io_tree *tree,
David Sterbaf993c882011-04-20 23:35:57 +0200199 struct address_space *mapping)
Chris Masond1310b22008-01-24 16:13:08 -0500200{
Eric Paris6bef4d32010-02-23 19:43:04 +0000201 tree->state = RB_ROOT;
Chris Masond1310b22008-01-24 16:13:08 -0500202 tree->ops = NULL;
203 tree->dirty_bytes = 0;
Chris Mason70dec802008-01-29 09:59:12 -0500204 spin_lock_init(&tree->lock);
Chris Masond1310b22008-01-24 16:13:08 -0500205 tree->mapping = mapping;
Chris Masond1310b22008-01-24 16:13:08 -0500206}
Chris Masond1310b22008-01-24 16:13:08 -0500207
Christoph Hellwigb2950862008-12-02 09:54:17 -0500208static struct extent_state *alloc_extent_state(gfp_t mask)
Chris Masond1310b22008-01-24 16:13:08 -0500209{
210 struct extent_state *state;
Chris Masond1310b22008-01-24 16:13:08 -0500211
212 state = kmem_cache_alloc(extent_state_cache, mask);
Peter2b114d12008-04-01 11:21:40 -0400213 if (!state)
Chris Masond1310b22008-01-24 16:13:08 -0500214 return state;
215 state->state = 0;
Chris Masond1310b22008-01-24 16:13:08 -0500216 state->private = 0;
Filipe Manana27a35072014-07-06 20:09:59 +0100217 RB_CLEAR_NODE(&state->rb_node);
Eric Sandeen6d49ba12013-04-22 16:12:31 +0000218 btrfs_leak_debug_add(&state->leak_list, &states);
Chris Masond1310b22008-01-24 16:13:08 -0500219 atomic_set(&state->refs, 1);
220 init_waitqueue_head(&state->wq);
Jeff Mahoney143bede2012-03-01 14:56:26 +0100221 trace_alloc_extent_state(state, mask, _RET_IP_);
Chris Masond1310b22008-01-24 16:13:08 -0500222 return state;
223}
Chris Masond1310b22008-01-24 16:13:08 -0500224
Chris Mason4845e442010-05-25 20:56:50 -0400225void free_extent_state(struct extent_state *state)
Chris Masond1310b22008-01-24 16:13:08 -0500226{
Chris Masond1310b22008-01-24 16:13:08 -0500227 if (!state)
228 return;
229 if (atomic_dec_and_test(&state->refs)) {
Filipe Manana27a35072014-07-06 20:09:59 +0100230 WARN_ON(extent_state_in_tree(state));
Eric Sandeen6d49ba12013-04-22 16:12:31 +0000231 btrfs_leak_debug_del(&state->leak_list);
Jeff Mahoney143bede2012-03-01 14:56:26 +0100232 trace_free_extent_state(state, _RET_IP_);
Chris Masond1310b22008-01-24 16:13:08 -0500233 kmem_cache_free(extent_state_cache, state);
234 }
235}
Chris Masond1310b22008-01-24 16:13:08 -0500236
Filipe Mananaf2071b22014-02-12 15:05:53 +0000237static struct rb_node *tree_insert(struct rb_root *root,
238 struct rb_node *search_start,
239 u64 offset,
Filipe David Borba Manana12cfbad2013-11-26 15:41:47 +0000240 struct rb_node *node,
241 struct rb_node ***p_in,
242 struct rb_node **parent_in)
Chris Masond1310b22008-01-24 16:13:08 -0500243{
Filipe Mananaf2071b22014-02-12 15:05:53 +0000244 struct rb_node **p;
Chris Masond3977122009-01-05 21:25:51 -0500245 struct rb_node *parent = NULL;
Chris Masond1310b22008-01-24 16:13:08 -0500246 struct tree_entry *entry;
247
Filipe David Borba Manana12cfbad2013-11-26 15:41:47 +0000248 if (p_in && parent_in) {
249 p = *p_in;
250 parent = *parent_in;
251 goto do_insert;
252 }
253
Filipe Mananaf2071b22014-02-12 15:05:53 +0000254 p = search_start ? &search_start : &root->rb_node;
Chris Masond3977122009-01-05 21:25:51 -0500255 while (*p) {
Chris Masond1310b22008-01-24 16:13:08 -0500256 parent = *p;
257 entry = rb_entry(parent, struct tree_entry, rb_node);
258
259 if (offset < entry->start)
260 p = &(*p)->rb_left;
261 else if (offset > entry->end)
262 p = &(*p)->rb_right;
263 else
264 return parent;
265 }
266
Filipe David Borba Manana12cfbad2013-11-26 15:41:47 +0000267do_insert:
Chris Masond1310b22008-01-24 16:13:08 -0500268 rb_link_node(node, parent, p);
269 rb_insert_color(node, root);
270 return NULL;
271}
272
Chris Mason80ea96b2008-02-01 14:51:59 -0500273static struct rb_node *__etree_search(struct extent_io_tree *tree, u64 offset,
Filipe David Borba Manana12cfbad2013-11-26 15:41:47 +0000274 struct rb_node **prev_ret,
275 struct rb_node **next_ret,
276 struct rb_node ***p_ret,
277 struct rb_node **parent_ret)
Chris Masond1310b22008-01-24 16:13:08 -0500278{
Chris Mason80ea96b2008-02-01 14:51:59 -0500279 struct rb_root *root = &tree->state;
Filipe David Borba Manana12cfbad2013-11-26 15:41:47 +0000280 struct rb_node **n = &root->rb_node;
Chris Masond1310b22008-01-24 16:13:08 -0500281 struct rb_node *prev = NULL;
282 struct rb_node *orig_prev = NULL;
283 struct tree_entry *entry;
284 struct tree_entry *prev_entry = NULL;
285
Filipe David Borba Manana12cfbad2013-11-26 15:41:47 +0000286 while (*n) {
287 prev = *n;
288 entry = rb_entry(prev, struct tree_entry, rb_node);
Chris Masond1310b22008-01-24 16:13:08 -0500289 prev_entry = entry;
290
291 if (offset < entry->start)
Filipe David Borba Manana12cfbad2013-11-26 15:41:47 +0000292 n = &(*n)->rb_left;
Chris Masond1310b22008-01-24 16:13:08 -0500293 else if (offset > entry->end)
Filipe David Borba Manana12cfbad2013-11-26 15:41:47 +0000294 n = &(*n)->rb_right;
Chris Masond3977122009-01-05 21:25:51 -0500295 else
Filipe David Borba Manana12cfbad2013-11-26 15:41:47 +0000296 return *n;
Chris Masond1310b22008-01-24 16:13:08 -0500297 }
298
Filipe David Borba Manana12cfbad2013-11-26 15:41:47 +0000299 if (p_ret)
300 *p_ret = n;
301 if (parent_ret)
302 *parent_ret = prev;
303
Chris Masond1310b22008-01-24 16:13:08 -0500304 if (prev_ret) {
305 orig_prev = prev;
Chris Masond3977122009-01-05 21:25:51 -0500306 while (prev && offset > prev_entry->end) {
Chris Masond1310b22008-01-24 16:13:08 -0500307 prev = rb_next(prev);
308 prev_entry = rb_entry(prev, struct tree_entry, rb_node);
309 }
310 *prev_ret = prev;
311 prev = orig_prev;
312 }
313
314 if (next_ret) {
315 prev_entry = rb_entry(prev, struct tree_entry, rb_node);
Chris Masond3977122009-01-05 21:25:51 -0500316 while (prev && offset < prev_entry->start) {
Chris Masond1310b22008-01-24 16:13:08 -0500317 prev = rb_prev(prev);
318 prev_entry = rb_entry(prev, struct tree_entry, rb_node);
319 }
320 *next_ret = prev;
321 }
322 return NULL;
323}
324
Filipe David Borba Manana12cfbad2013-11-26 15:41:47 +0000325static inline struct rb_node *
326tree_search_for_insert(struct extent_io_tree *tree,
327 u64 offset,
328 struct rb_node ***p_ret,
329 struct rb_node **parent_ret)
Chris Masond1310b22008-01-24 16:13:08 -0500330{
Chris Mason70dec802008-01-29 09:59:12 -0500331 struct rb_node *prev = NULL;
Chris Masond1310b22008-01-24 16:13:08 -0500332 struct rb_node *ret;
Chris Mason70dec802008-01-29 09:59:12 -0500333
Filipe David Borba Manana12cfbad2013-11-26 15:41:47 +0000334 ret = __etree_search(tree, offset, &prev, NULL, p_ret, parent_ret);
Chris Masond3977122009-01-05 21:25:51 -0500335 if (!ret)
Chris Masond1310b22008-01-24 16:13:08 -0500336 return prev;
337 return ret;
338}
339
Filipe David Borba Manana12cfbad2013-11-26 15:41:47 +0000340static inline struct rb_node *tree_search(struct extent_io_tree *tree,
341 u64 offset)
342{
343 return tree_search_for_insert(tree, offset, NULL, NULL);
344}
345
Josef Bacik9ed74f22009-09-11 16:12:44 -0400346static void merge_cb(struct extent_io_tree *tree, struct extent_state *new,
347 struct extent_state *other)
348{
349 if (tree->ops && tree->ops->merge_extent_hook)
350 tree->ops->merge_extent_hook(tree->mapping->host, new,
351 other);
352}
353
Chris Masond1310b22008-01-24 16:13:08 -0500354/*
355 * utility function to look for merge candidates inside a given range.
356 * Any extents with matching state are merged together into a single
357 * extent in the tree. Extents with EXTENT_IO in their state field
358 * are not merged because the end_io handlers need to be able to do
359 * operations on them without sleeping (or doing allocations/splits).
360 *
361 * This should be called with the tree lock held.
362 */
Jeff Mahoney1bf85042011-07-21 16:56:09 +0000363static void merge_state(struct extent_io_tree *tree,
364 struct extent_state *state)
Chris Masond1310b22008-01-24 16:13:08 -0500365{
366 struct extent_state *other;
367 struct rb_node *other_node;
368
Zheng Yan5b21f2e2008-09-26 10:05:38 -0400369 if (state->state & (EXTENT_IOBITS | EXTENT_BOUNDARY))
Jeff Mahoney1bf85042011-07-21 16:56:09 +0000370 return;
Chris Masond1310b22008-01-24 16:13:08 -0500371
372 other_node = rb_prev(&state->rb_node);
373 if (other_node) {
374 other = rb_entry(other_node, struct extent_state, rb_node);
375 if (other->end == state->start - 1 &&
376 other->state == state->state) {
Josef Bacik9ed74f22009-09-11 16:12:44 -0400377 merge_cb(tree, state, other);
Chris Masond1310b22008-01-24 16:13:08 -0500378 state->start = other->start;
Chris Masond1310b22008-01-24 16:13:08 -0500379 rb_erase(&other->rb_node, &tree->state);
Filipe Manana27a35072014-07-06 20:09:59 +0100380 RB_CLEAR_NODE(&other->rb_node);
Chris Masond1310b22008-01-24 16:13:08 -0500381 free_extent_state(other);
382 }
383 }
384 other_node = rb_next(&state->rb_node);
385 if (other_node) {
386 other = rb_entry(other_node, struct extent_state, rb_node);
387 if (other->start == state->end + 1 &&
388 other->state == state->state) {
Josef Bacik9ed74f22009-09-11 16:12:44 -0400389 merge_cb(tree, state, other);
Josef Bacikdf98b6e2011-06-20 14:53:48 -0400390 state->end = other->end;
Josef Bacikdf98b6e2011-06-20 14:53:48 -0400391 rb_erase(&other->rb_node, &tree->state);
Filipe Manana27a35072014-07-06 20:09:59 +0100392 RB_CLEAR_NODE(&other->rb_node);
Josef Bacikdf98b6e2011-06-20 14:53:48 -0400393 free_extent_state(other);
Chris Masond1310b22008-01-24 16:13:08 -0500394 }
395 }
Chris Masond1310b22008-01-24 16:13:08 -0500396}
397
Jeff Mahoney1bf85042011-07-21 16:56:09 +0000398static void set_state_cb(struct extent_io_tree *tree,
David Sterba9ee49a042015-01-14 19:52:13 +0100399 struct extent_state *state, unsigned *bits)
Chris Mason291d6732008-01-29 15:55:23 -0500400{
Jeff Mahoney1bf85042011-07-21 16:56:09 +0000401 if (tree->ops && tree->ops->set_bit_hook)
402 tree->ops->set_bit_hook(tree->mapping->host, state, bits);
Chris Mason291d6732008-01-29 15:55:23 -0500403}
404
405static void clear_state_cb(struct extent_io_tree *tree,
David Sterba9ee49a042015-01-14 19:52:13 +0100406 struct extent_state *state, unsigned *bits)
Chris Mason291d6732008-01-29 15:55:23 -0500407{
Josef Bacik9ed74f22009-09-11 16:12:44 -0400408 if (tree->ops && tree->ops->clear_bit_hook)
409 tree->ops->clear_bit_hook(tree->mapping->host, state, bits);
Chris Mason291d6732008-01-29 15:55:23 -0500410}
411
Xiao Guangrong3150b692011-07-14 03:19:08 +0000412static void set_state_bits(struct extent_io_tree *tree,
David Sterba9ee49a042015-01-14 19:52:13 +0100413 struct extent_state *state, unsigned *bits);
Xiao Guangrong3150b692011-07-14 03:19:08 +0000414
Chris Masond1310b22008-01-24 16:13:08 -0500415/*
416 * insert an extent_state struct into the tree. 'bits' are set on the
417 * struct before it is inserted.
418 *
419 * This may return -EEXIST if the extent is already there, in which case the
420 * state struct is freed.
421 *
422 * The tree lock is not taken internally. This is a utility function and
423 * probably isn't what you want to call (see set/clear_extent_bit).
424 */
425static int insert_state(struct extent_io_tree *tree,
426 struct extent_state *state, u64 start, u64 end,
Filipe David Borba Manana12cfbad2013-11-26 15:41:47 +0000427 struct rb_node ***p,
428 struct rb_node **parent,
David Sterba9ee49a042015-01-14 19:52:13 +0100429 unsigned *bits)
Chris Masond1310b22008-01-24 16:13:08 -0500430{
431 struct rb_node *node;
432
Julia Lawall31b1a2b2012-11-03 10:58:34 +0000433 if (end < start)
Frank Holtonefe120a2013-12-20 11:37:06 -0500434 WARN(1, KERN_ERR "BTRFS: end < start %llu %llu\n",
Geert Uytterhoevenc1c9ff72013-08-20 13:20:07 +0200435 end, start);
Chris Masond1310b22008-01-24 16:13:08 -0500436 state->start = start;
437 state->end = end;
Josef Bacik9ed74f22009-09-11 16:12:44 -0400438
Xiao Guangrong3150b692011-07-14 03:19:08 +0000439 set_state_bits(tree, state, bits);
440
Filipe Mananaf2071b22014-02-12 15:05:53 +0000441 node = tree_insert(&tree->state, NULL, end, &state->rb_node, p, parent);
Chris Masond1310b22008-01-24 16:13:08 -0500442 if (node) {
443 struct extent_state *found;
444 found = rb_entry(node, struct extent_state, rb_node);
Frank Holtonefe120a2013-12-20 11:37:06 -0500445 printk(KERN_ERR "BTRFS: found node %llu %llu on insert of "
Geert Uytterhoevenc1c9ff72013-08-20 13:20:07 +0200446 "%llu %llu\n",
447 found->start, found->end, start, end);
Chris Masond1310b22008-01-24 16:13:08 -0500448 return -EEXIST;
449 }
450 merge_state(tree, state);
451 return 0;
452}
453
Jeff Mahoney1bf85042011-07-21 16:56:09 +0000454static void split_cb(struct extent_io_tree *tree, struct extent_state *orig,
Josef Bacik9ed74f22009-09-11 16:12:44 -0400455 u64 split)
456{
457 if (tree->ops && tree->ops->split_extent_hook)
Jeff Mahoney1bf85042011-07-21 16:56:09 +0000458 tree->ops->split_extent_hook(tree->mapping->host, orig, split);
Josef Bacik9ed74f22009-09-11 16:12:44 -0400459}
460
Chris Masond1310b22008-01-24 16:13:08 -0500461/*
462 * split a given extent state struct in two, inserting the preallocated
463 * struct 'prealloc' as the newly created second half. 'split' indicates an
464 * offset inside 'orig' where it should be split.
465 *
466 * Before calling,
467 * the tree has 'orig' at [orig->start, orig->end]. After calling, there
468 * are two extent state structs in the tree:
469 * prealloc: [orig->start, split - 1]
470 * orig: [ split, orig->end ]
471 *
472 * The tree locks are not taken by this function. They need to be held
473 * by the caller.
474 */
475static int split_state(struct extent_io_tree *tree, struct extent_state *orig,
476 struct extent_state *prealloc, u64 split)
477{
478 struct rb_node *node;
Josef Bacik9ed74f22009-09-11 16:12:44 -0400479
480 split_cb(tree, orig, split);
481
Chris Masond1310b22008-01-24 16:13:08 -0500482 prealloc->start = orig->start;
483 prealloc->end = split - 1;
484 prealloc->state = orig->state;
485 orig->start = split;
486
Filipe Mananaf2071b22014-02-12 15:05:53 +0000487 node = tree_insert(&tree->state, &orig->rb_node, prealloc->end,
488 &prealloc->rb_node, NULL, NULL);
Chris Masond1310b22008-01-24 16:13:08 -0500489 if (node) {
Chris Masond1310b22008-01-24 16:13:08 -0500490 free_extent_state(prealloc);
491 return -EEXIST;
492 }
493 return 0;
494}
495
Li Zefancdc6a392012-03-12 16:39:48 +0800496static struct extent_state *next_state(struct extent_state *state)
497{
498 struct rb_node *next = rb_next(&state->rb_node);
499 if (next)
500 return rb_entry(next, struct extent_state, rb_node);
501 else
502 return NULL;
503}
504
Chris Masond1310b22008-01-24 16:13:08 -0500505/*
506 * utility function to clear some bits in an extent state struct.
Wang Sheng-Hui1b303fc2012-04-06 14:35:18 +0800507 * it will optionally wake up any one waiting on this state (wake == 1).
Chris Masond1310b22008-01-24 16:13:08 -0500508 *
509 * If no bits are set on the state struct after clearing things, the
510 * struct is freed and removed from the tree
511 */
Li Zefancdc6a392012-03-12 16:39:48 +0800512static struct extent_state *clear_state_bit(struct extent_io_tree *tree,
513 struct extent_state *state,
David Sterba9ee49a042015-01-14 19:52:13 +0100514 unsigned *bits, int wake)
Chris Masond1310b22008-01-24 16:13:08 -0500515{
Li Zefancdc6a392012-03-12 16:39:48 +0800516 struct extent_state *next;
David Sterba9ee49a042015-01-14 19:52:13 +0100517 unsigned bits_to_clear = *bits & ~EXTENT_CTLBITS;
Chris Masond1310b22008-01-24 16:13:08 -0500518
Yan, Zheng0ca1f7c2010-05-16 10:48:47 -0400519 if ((bits_to_clear & EXTENT_DIRTY) && (state->state & EXTENT_DIRTY)) {
Chris Masond1310b22008-01-24 16:13:08 -0500520 u64 range = state->end - state->start + 1;
521 WARN_ON(range > tree->dirty_bytes);
522 tree->dirty_bytes -= range;
523 }
Chris Mason291d6732008-01-29 15:55:23 -0500524 clear_state_cb(tree, state, bits);
Josef Bacik32c00af2009-10-08 13:34:05 -0400525 state->state &= ~bits_to_clear;
Chris Masond1310b22008-01-24 16:13:08 -0500526 if (wake)
527 wake_up(&state->wq);
Yan, Zheng0ca1f7c2010-05-16 10:48:47 -0400528 if (state->state == 0) {
Li Zefancdc6a392012-03-12 16:39:48 +0800529 next = next_state(state);
Filipe Manana27a35072014-07-06 20:09:59 +0100530 if (extent_state_in_tree(state)) {
Chris Masond1310b22008-01-24 16:13:08 -0500531 rb_erase(&state->rb_node, &tree->state);
Filipe Manana27a35072014-07-06 20:09:59 +0100532 RB_CLEAR_NODE(&state->rb_node);
Chris Masond1310b22008-01-24 16:13:08 -0500533 free_extent_state(state);
534 } else {
535 WARN_ON(1);
536 }
537 } else {
538 merge_state(tree, state);
Li Zefancdc6a392012-03-12 16:39:48 +0800539 next = next_state(state);
Chris Masond1310b22008-01-24 16:13:08 -0500540 }
Li Zefancdc6a392012-03-12 16:39:48 +0800541 return next;
Chris Masond1310b22008-01-24 16:13:08 -0500542}
543
Xiao Guangrong82337672011-04-20 06:44:57 +0000544static struct extent_state *
545alloc_extent_state_atomic(struct extent_state *prealloc)
546{
547 if (!prealloc)
548 prealloc = alloc_extent_state(GFP_ATOMIC);
549
550 return prealloc;
551}
552
Eric Sandeen48a3b632013-04-25 20:41:01 +0000553static void extent_io_tree_panic(struct extent_io_tree *tree, int err)
Jeff Mahoneyc2d904e2011-10-03 23:22:32 -0400554{
555 btrfs_panic(tree_fs_info(tree), err, "Locking error: "
556 "Extent tree was modified by another "
557 "thread while locked.");
558}
559
Chris Masond1310b22008-01-24 16:13:08 -0500560/*
561 * clear some bits on a range in the tree. This may require splitting
562 * or inserting elements in the tree, so the gfp mask is used to
563 * indicate which allocations or sleeping are allowed.
564 *
565 * pass 'wake' == 1 to kick any sleepers, and 'delete' == 1 to remove
566 * the given range from the tree regardless of state (ie for truncate).
567 *
568 * the range [start, end] is inclusive.
569 *
Jeff Mahoney6763af82012-03-01 14:56:29 +0100570 * This takes the tree lock, and returns 0 on success and < 0 on error.
Chris Masond1310b22008-01-24 16:13:08 -0500571 */
572int clear_extent_bit(struct extent_io_tree *tree, u64 start, u64 end,
David Sterba9ee49a042015-01-14 19:52:13 +0100573 unsigned bits, int wake, int delete,
Chris Mason2c64c532009-09-02 15:04:12 -0400574 struct extent_state **cached_state,
575 gfp_t mask)
Chris Masond1310b22008-01-24 16:13:08 -0500576{
577 struct extent_state *state;
Chris Mason2c64c532009-09-02 15:04:12 -0400578 struct extent_state *cached;
Chris Masond1310b22008-01-24 16:13:08 -0500579 struct extent_state *prealloc = NULL;
580 struct rb_node *node;
Yan Zheng5c939df2009-05-27 09:16:03 -0400581 u64 last_end;
Chris Masond1310b22008-01-24 16:13:08 -0500582 int err;
Josef Bacik2ac55d42010-02-03 19:33:23 +0000583 int clear = 0;
Chris Masond1310b22008-01-24 16:13:08 -0500584
Josef Bacika5dee372013-12-13 10:02:44 -0500585 btrfs_debug_check_extent_io_range(tree, start, end);
David Sterba8d599ae2013-04-30 15:22:23 +0000586
Josef Bacik7ee9e442013-06-21 16:37:03 -0400587 if (bits & EXTENT_DELALLOC)
588 bits |= EXTENT_NORESERVE;
589
Yan, Zheng0ca1f7c2010-05-16 10:48:47 -0400590 if (delete)
591 bits |= ~EXTENT_CTLBITS;
592 bits |= EXTENT_FIRST_DELALLOC;
593
Josef Bacik2ac55d42010-02-03 19:33:23 +0000594 if (bits & (EXTENT_IOBITS | EXTENT_BOUNDARY))
595 clear = 1;
Chris Masond1310b22008-01-24 16:13:08 -0500596again:
Mel Gormand0164ad2015-11-06 16:28:21 -0800597 if (!prealloc && gfpflags_allow_blocking(mask)) {
Filipe Mananac7bc6312014-11-03 14:12:57 +0000598 /*
599 * Don't care for allocation failure here because we might end
600 * up not needing the pre-allocated extent state at all, which
601 * is the case if we only have in the tree extent states that
602 * cover our input range and don't cover too any other range.
603 * If we end up needing a new extent state we allocate it later.
604 */
Chris Masond1310b22008-01-24 16:13:08 -0500605 prealloc = alloc_extent_state(mask);
Chris Masond1310b22008-01-24 16:13:08 -0500606 }
607
Chris Masoncad321a2008-12-17 14:51:42 -0500608 spin_lock(&tree->lock);
Chris Mason2c64c532009-09-02 15:04:12 -0400609 if (cached_state) {
610 cached = *cached_state;
Josef Bacik2ac55d42010-02-03 19:33:23 +0000611
612 if (clear) {
613 *cached_state = NULL;
614 cached_state = NULL;
615 }
616
Filipe Manana27a35072014-07-06 20:09:59 +0100617 if (cached && extent_state_in_tree(cached) &&
618 cached->start <= start && cached->end > start) {
Josef Bacik2ac55d42010-02-03 19:33:23 +0000619 if (clear)
620 atomic_dec(&cached->refs);
Chris Mason2c64c532009-09-02 15:04:12 -0400621 state = cached;
Chris Mason42daec22009-09-23 19:51:09 -0400622 goto hit_next;
Chris Mason2c64c532009-09-02 15:04:12 -0400623 }
Josef Bacik2ac55d42010-02-03 19:33:23 +0000624 if (clear)
625 free_extent_state(cached);
Chris Mason2c64c532009-09-02 15:04:12 -0400626 }
Chris Masond1310b22008-01-24 16:13:08 -0500627 /*
628 * this search will find the extents that end after
629 * our range starts
630 */
Chris Mason80ea96b2008-02-01 14:51:59 -0500631 node = tree_search(tree, start);
Chris Masond1310b22008-01-24 16:13:08 -0500632 if (!node)
633 goto out;
634 state = rb_entry(node, struct extent_state, rb_node);
Chris Mason2c64c532009-09-02 15:04:12 -0400635hit_next:
Chris Masond1310b22008-01-24 16:13:08 -0500636 if (state->start > end)
637 goto out;
638 WARN_ON(state->end < start);
Yan Zheng5c939df2009-05-27 09:16:03 -0400639 last_end = state->end;
Chris Masond1310b22008-01-24 16:13:08 -0500640
Liu Bo04493142012-02-16 18:34:37 +0800641 /* the state doesn't have the wanted bits, go ahead */
Li Zefancdc6a392012-03-12 16:39:48 +0800642 if (!(state->state & bits)) {
643 state = next_state(state);
Liu Bo04493142012-02-16 18:34:37 +0800644 goto next;
Li Zefancdc6a392012-03-12 16:39:48 +0800645 }
Liu Bo04493142012-02-16 18:34:37 +0800646
Chris Masond1310b22008-01-24 16:13:08 -0500647 /*
648 * | ---- desired range ---- |
649 * | state | or
650 * | ------------- state -------------- |
651 *
652 * We need to split the extent we found, and may flip
653 * bits on second half.
654 *
655 * If the extent we found extends past our range, we
656 * just split and search again. It'll get split again
657 * the next time though.
658 *
659 * If the extent we found is inside our range, we clear
660 * the desired bit on it.
661 */
662
663 if (state->start < start) {
Xiao Guangrong82337672011-04-20 06:44:57 +0000664 prealloc = alloc_extent_state_atomic(prealloc);
665 BUG_ON(!prealloc);
Chris Masond1310b22008-01-24 16:13:08 -0500666 err = split_state(tree, state, prealloc, start);
Jeff Mahoneyc2d904e2011-10-03 23:22:32 -0400667 if (err)
668 extent_io_tree_panic(tree, err);
669
Chris Masond1310b22008-01-24 16:13:08 -0500670 prealloc = NULL;
671 if (err)
672 goto out;
673 if (state->end <= end) {
Liu Bod1ac6e42012-05-10 18:10:39 +0800674 state = clear_state_bit(tree, state, &bits, wake);
675 goto next;
Chris Masond1310b22008-01-24 16:13:08 -0500676 }
677 goto search_again;
678 }
679 /*
680 * | ---- desired range ---- |
681 * | state |
682 * We need to split the extent, and clear the bit
683 * on the first half
684 */
685 if (state->start <= end && state->end > end) {
Xiao Guangrong82337672011-04-20 06:44:57 +0000686 prealloc = alloc_extent_state_atomic(prealloc);
687 BUG_ON(!prealloc);
Chris Masond1310b22008-01-24 16:13:08 -0500688 err = split_state(tree, state, prealloc, end + 1);
Jeff Mahoneyc2d904e2011-10-03 23:22:32 -0400689 if (err)
690 extent_io_tree_panic(tree, err);
691
Chris Masond1310b22008-01-24 16:13:08 -0500692 if (wake)
693 wake_up(&state->wq);
Chris Mason42daec22009-09-23 19:51:09 -0400694
Jeff Mahoney6763af82012-03-01 14:56:29 +0100695 clear_state_bit(tree, prealloc, &bits, wake);
Josef Bacik9ed74f22009-09-11 16:12:44 -0400696
Chris Masond1310b22008-01-24 16:13:08 -0500697 prealloc = NULL;
698 goto out;
699 }
Chris Mason42daec22009-09-23 19:51:09 -0400700
Li Zefancdc6a392012-03-12 16:39:48 +0800701 state = clear_state_bit(tree, state, &bits, wake);
Liu Bo04493142012-02-16 18:34:37 +0800702next:
Yan Zheng5c939df2009-05-27 09:16:03 -0400703 if (last_end == (u64)-1)
704 goto out;
705 start = last_end + 1;
Li Zefancdc6a392012-03-12 16:39:48 +0800706 if (start <= end && state && !need_resched())
Liu Bo692e5752012-02-16 18:34:36 +0800707 goto hit_next;
Chris Masond1310b22008-01-24 16:13:08 -0500708 goto search_again;
709
710out:
Chris Masoncad321a2008-12-17 14:51:42 -0500711 spin_unlock(&tree->lock);
Chris Masond1310b22008-01-24 16:13:08 -0500712 if (prealloc)
713 free_extent_state(prealloc);
714
Jeff Mahoney6763af82012-03-01 14:56:29 +0100715 return 0;
Chris Masond1310b22008-01-24 16:13:08 -0500716
717search_again:
718 if (start > end)
719 goto out;
Chris Masoncad321a2008-12-17 14:51:42 -0500720 spin_unlock(&tree->lock);
Mel Gormand0164ad2015-11-06 16:28:21 -0800721 if (gfpflags_allow_blocking(mask))
Chris Masond1310b22008-01-24 16:13:08 -0500722 cond_resched();
723 goto again;
724}
Chris Masond1310b22008-01-24 16:13:08 -0500725
Jeff Mahoney143bede2012-03-01 14:56:26 +0100726static void wait_on_state(struct extent_io_tree *tree,
727 struct extent_state *state)
Christoph Hellwig641f5212008-12-02 06:36:10 -0500728 __releases(tree->lock)
729 __acquires(tree->lock)
Chris Masond1310b22008-01-24 16:13:08 -0500730{
731 DEFINE_WAIT(wait);
732 prepare_to_wait(&state->wq, &wait, TASK_UNINTERRUPTIBLE);
Chris Masoncad321a2008-12-17 14:51:42 -0500733 spin_unlock(&tree->lock);
Chris Masond1310b22008-01-24 16:13:08 -0500734 schedule();
Chris Masoncad321a2008-12-17 14:51:42 -0500735 spin_lock(&tree->lock);
Chris Masond1310b22008-01-24 16:13:08 -0500736 finish_wait(&state->wq, &wait);
Chris Masond1310b22008-01-24 16:13:08 -0500737}
738
739/*
740 * waits for one or more bits to clear on a range in the state tree.
741 * The range [start, end] is inclusive.
742 * The tree lock is taken by this function
743 */
David Sterba41074882013-04-29 13:38:46 +0000744static void wait_extent_bit(struct extent_io_tree *tree, u64 start, u64 end,
745 unsigned long bits)
Chris Masond1310b22008-01-24 16:13:08 -0500746{
747 struct extent_state *state;
748 struct rb_node *node;
749
Josef Bacika5dee372013-12-13 10:02:44 -0500750 btrfs_debug_check_extent_io_range(tree, start, end);
David Sterba8d599ae2013-04-30 15:22:23 +0000751
Chris Masoncad321a2008-12-17 14:51:42 -0500752 spin_lock(&tree->lock);
Chris Masond1310b22008-01-24 16:13:08 -0500753again:
754 while (1) {
755 /*
756 * this search will find all the extents that end after
757 * our range starts
758 */
Chris Mason80ea96b2008-02-01 14:51:59 -0500759 node = tree_search(tree, start);
Filipe Mananac50d3e72014-03-31 14:53:25 +0100760process_node:
Chris Masond1310b22008-01-24 16:13:08 -0500761 if (!node)
762 break;
763
764 state = rb_entry(node, struct extent_state, rb_node);
765
766 if (state->start > end)
767 goto out;
768
769 if (state->state & bits) {
770 start = state->start;
771 atomic_inc(&state->refs);
772 wait_on_state(tree, state);
773 free_extent_state(state);
774 goto again;
775 }
776 start = state->end + 1;
777
778 if (start > end)
779 break;
780
Filipe Mananac50d3e72014-03-31 14:53:25 +0100781 if (!cond_resched_lock(&tree->lock)) {
782 node = rb_next(node);
783 goto process_node;
784 }
Chris Masond1310b22008-01-24 16:13:08 -0500785 }
786out:
Chris Masoncad321a2008-12-17 14:51:42 -0500787 spin_unlock(&tree->lock);
Chris Masond1310b22008-01-24 16:13:08 -0500788}
Chris Masond1310b22008-01-24 16:13:08 -0500789
Jeff Mahoney1bf85042011-07-21 16:56:09 +0000790static void set_state_bits(struct extent_io_tree *tree,
Chris Masond1310b22008-01-24 16:13:08 -0500791 struct extent_state *state,
David Sterba9ee49a042015-01-14 19:52:13 +0100792 unsigned *bits)
Chris Masond1310b22008-01-24 16:13:08 -0500793{
David Sterba9ee49a042015-01-14 19:52:13 +0100794 unsigned bits_to_set = *bits & ~EXTENT_CTLBITS;
Josef Bacik9ed74f22009-09-11 16:12:44 -0400795
Jeff Mahoney1bf85042011-07-21 16:56:09 +0000796 set_state_cb(tree, state, bits);
Yan, Zheng0ca1f7c2010-05-16 10:48:47 -0400797 if ((bits_to_set & EXTENT_DIRTY) && !(state->state & EXTENT_DIRTY)) {
Chris Masond1310b22008-01-24 16:13:08 -0500798 u64 range = state->end - state->start + 1;
799 tree->dirty_bytes += range;
800 }
Yan, Zheng0ca1f7c2010-05-16 10:48:47 -0400801 state->state |= bits_to_set;
Chris Masond1310b22008-01-24 16:13:08 -0500802}
803
Filipe Mananae38e2ed2014-10-13 12:28:38 +0100804static void cache_state_if_flags(struct extent_state *state,
805 struct extent_state **cached_ptr,
David Sterba9ee49a042015-01-14 19:52:13 +0100806 unsigned flags)
Chris Mason2c64c532009-09-02 15:04:12 -0400807{
808 if (cached_ptr && !(*cached_ptr)) {
Filipe Mananae38e2ed2014-10-13 12:28:38 +0100809 if (!flags || (state->state & flags)) {
Chris Mason2c64c532009-09-02 15:04:12 -0400810 *cached_ptr = state;
811 atomic_inc(&state->refs);
812 }
813 }
814}
815
Filipe Mananae38e2ed2014-10-13 12:28:38 +0100816static void cache_state(struct extent_state *state,
817 struct extent_state **cached_ptr)
818{
819 return cache_state_if_flags(state, cached_ptr,
820 EXTENT_IOBITS | EXTENT_BOUNDARY);
821}
822
Chris Masond1310b22008-01-24 16:13:08 -0500823/*
Chris Mason1edbb732009-09-02 13:24:36 -0400824 * set some bits on a range in the tree. This may require allocations or
825 * sleeping, so the gfp mask is used to indicate what is allowed.
Chris Masond1310b22008-01-24 16:13:08 -0500826 *
Chris Mason1edbb732009-09-02 13:24:36 -0400827 * If any of the exclusive bits are set, this will fail with -EEXIST if some
828 * part of the range already has the desired bits set. The start of the
829 * existing range is returned in failed_start in this case.
Chris Masond1310b22008-01-24 16:13:08 -0500830 *
Chris Mason1edbb732009-09-02 13:24:36 -0400831 * [start, end] is inclusive This takes the tree lock.
Chris Masond1310b22008-01-24 16:13:08 -0500832 */
Chris Mason1edbb732009-09-02 13:24:36 -0400833
Jeff Mahoney3fbe5c02012-03-01 14:57:19 +0100834static int __must_check
835__set_extent_bit(struct extent_io_tree *tree, u64 start, u64 end,
David Sterba9ee49a042015-01-14 19:52:13 +0100836 unsigned bits, unsigned exclusive_bits,
David Sterba41074882013-04-29 13:38:46 +0000837 u64 *failed_start, struct extent_state **cached_state,
838 gfp_t mask)
Chris Masond1310b22008-01-24 16:13:08 -0500839{
840 struct extent_state *state;
841 struct extent_state *prealloc = NULL;
842 struct rb_node *node;
Filipe David Borba Manana12cfbad2013-11-26 15:41:47 +0000843 struct rb_node **p;
844 struct rb_node *parent;
Chris Masond1310b22008-01-24 16:13:08 -0500845 int err = 0;
Chris Masond1310b22008-01-24 16:13:08 -0500846 u64 last_start;
847 u64 last_end;
Chris Mason42daec22009-09-23 19:51:09 -0400848
Josef Bacika5dee372013-12-13 10:02:44 -0500849 btrfs_debug_check_extent_io_range(tree, start, end);
David Sterba8d599ae2013-04-30 15:22:23 +0000850
Yan, Zheng0ca1f7c2010-05-16 10:48:47 -0400851 bits |= EXTENT_FIRST_DELALLOC;
Chris Masond1310b22008-01-24 16:13:08 -0500852again:
Mel Gormand0164ad2015-11-06 16:28:21 -0800853 if (!prealloc && gfpflags_allow_blocking(mask)) {
Chris Masond1310b22008-01-24 16:13:08 -0500854 prealloc = alloc_extent_state(mask);
Xiao Guangrong82337672011-04-20 06:44:57 +0000855 BUG_ON(!prealloc);
Chris Masond1310b22008-01-24 16:13:08 -0500856 }
857
Chris Masoncad321a2008-12-17 14:51:42 -0500858 spin_lock(&tree->lock);
Chris Mason9655d292009-09-02 15:22:30 -0400859 if (cached_state && *cached_state) {
860 state = *cached_state;
Josef Bacikdf98b6e2011-06-20 14:53:48 -0400861 if (state->start <= start && state->end > start &&
Filipe Manana27a35072014-07-06 20:09:59 +0100862 extent_state_in_tree(state)) {
Chris Mason9655d292009-09-02 15:22:30 -0400863 node = &state->rb_node;
864 goto hit_next;
865 }
866 }
Chris Masond1310b22008-01-24 16:13:08 -0500867 /*
868 * this search will find all the extents that end after
869 * our range starts.
870 */
Filipe David Borba Manana12cfbad2013-11-26 15:41:47 +0000871 node = tree_search_for_insert(tree, start, &p, &parent);
Chris Masond1310b22008-01-24 16:13:08 -0500872 if (!node) {
Xiao Guangrong82337672011-04-20 06:44:57 +0000873 prealloc = alloc_extent_state_atomic(prealloc);
874 BUG_ON(!prealloc);
Filipe David Borba Manana12cfbad2013-11-26 15:41:47 +0000875 err = insert_state(tree, prealloc, start, end,
876 &p, &parent, &bits);
Jeff Mahoneyc2d904e2011-10-03 23:22:32 -0400877 if (err)
878 extent_io_tree_panic(tree, err);
879
Filipe David Borba Mananac42ac0b2013-11-26 15:01:34 +0000880 cache_state(prealloc, cached_state);
Chris Masond1310b22008-01-24 16:13:08 -0500881 prealloc = NULL;
Chris Masond1310b22008-01-24 16:13:08 -0500882 goto out;
883 }
Chris Masond1310b22008-01-24 16:13:08 -0500884 state = rb_entry(node, struct extent_state, rb_node);
Chris Mason40431d62009-08-05 12:57:59 -0400885hit_next:
Chris Masond1310b22008-01-24 16:13:08 -0500886 last_start = state->start;
887 last_end = state->end;
888
889 /*
890 * | ---- desired range ---- |
891 * | state |
892 *
893 * Just lock what we found and keep going
894 */
895 if (state->start == start && state->end <= end) {
Chris Mason1edbb732009-09-02 13:24:36 -0400896 if (state->state & exclusive_bits) {
Chris Masond1310b22008-01-24 16:13:08 -0500897 *failed_start = state->start;
898 err = -EEXIST;
899 goto out;
900 }
Chris Mason42daec22009-09-23 19:51:09 -0400901
Jeff Mahoney1bf85042011-07-21 16:56:09 +0000902 set_state_bits(tree, state, &bits);
Chris Mason2c64c532009-09-02 15:04:12 -0400903 cache_state(state, cached_state);
Chris Masond1310b22008-01-24 16:13:08 -0500904 merge_state(tree, state);
Yan Zheng5c939df2009-05-27 09:16:03 -0400905 if (last_end == (u64)-1)
906 goto out;
907 start = last_end + 1;
Liu Bod1ac6e42012-05-10 18:10:39 +0800908 state = next_state(state);
909 if (start < end && state && state->start == start &&
910 !need_resched())
911 goto hit_next;
Chris Masond1310b22008-01-24 16:13:08 -0500912 goto search_again;
913 }
914
915 /*
916 * | ---- desired range ---- |
917 * | state |
918 * or
919 * | ------------- state -------------- |
920 *
921 * We need to split the extent we found, and may flip bits on
922 * second half.
923 *
924 * If the extent we found extends past our
925 * range, we just split and search again. It'll get split
926 * again the next time though.
927 *
928 * If the extent we found is inside our range, we set the
929 * desired bit on it.
930 */
931 if (state->start < start) {
Chris Mason1edbb732009-09-02 13:24:36 -0400932 if (state->state & exclusive_bits) {
Chris Masond1310b22008-01-24 16:13:08 -0500933 *failed_start = start;
934 err = -EEXIST;
935 goto out;
936 }
Xiao Guangrong82337672011-04-20 06:44:57 +0000937
938 prealloc = alloc_extent_state_atomic(prealloc);
939 BUG_ON(!prealloc);
Chris Masond1310b22008-01-24 16:13:08 -0500940 err = split_state(tree, state, prealloc, start);
Jeff Mahoneyc2d904e2011-10-03 23:22:32 -0400941 if (err)
942 extent_io_tree_panic(tree, err);
943
Chris Masond1310b22008-01-24 16:13:08 -0500944 prealloc = NULL;
945 if (err)
946 goto out;
947 if (state->end <= end) {
Jeff Mahoney1bf85042011-07-21 16:56:09 +0000948 set_state_bits(tree, state, &bits);
Chris Mason2c64c532009-09-02 15:04:12 -0400949 cache_state(state, cached_state);
Chris Masond1310b22008-01-24 16:13:08 -0500950 merge_state(tree, state);
Yan Zheng5c939df2009-05-27 09:16:03 -0400951 if (last_end == (u64)-1)
952 goto out;
953 start = last_end + 1;
Liu Bod1ac6e42012-05-10 18:10:39 +0800954 state = next_state(state);
955 if (start < end && state && state->start == start &&
956 !need_resched())
957 goto hit_next;
Chris Masond1310b22008-01-24 16:13:08 -0500958 }
959 goto search_again;
960 }
961 /*
962 * | ---- desired range ---- |
963 * | state | or | state |
964 *
965 * There's a hole, we need to insert something in it and
966 * ignore the extent we found.
967 */
968 if (state->start > start) {
969 u64 this_end;
970 if (end < last_start)
971 this_end = end;
972 else
Chris Masond3977122009-01-05 21:25:51 -0500973 this_end = last_start - 1;
Xiao Guangrong82337672011-04-20 06:44:57 +0000974
975 prealloc = alloc_extent_state_atomic(prealloc);
976 BUG_ON(!prealloc);
Xiao Guangrongc7f895a2011-04-20 06:45:49 +0000977
978 /*
979 * Avoid to free 'prealloc' if it can be merged with
980 * the later extent.
981 */
Chris Masond1310b22008-01-24 16:13:08 -0500982 err = insert_state(tree, prealloc, start, this_end,
Filipe David Borba Manana12cfbad2013-11-26 15:41:47 +0000983 NULL, NULL, &bits);
Jeff Mahoneyc2d904e2011-10-03 23:22:32 -0400984 if (err)
985 extent_io_tree_panic(tree, err);
986
Chris Mason2c64c532009-09-02 15:04:12 -0400987 cache_state(prealloc, cached_state);
Chris Masond1310b22008-01-24 16:13:08 -0500988 prealloc = NULL;
Chris Masond1310b22008-01-24 16:13:08 -0500989 start = this_end + 1;
990 goto search_again;
991 }
992 /*
993 * | ---- desired range ---- |
994 * | state |
995 * We need to split the extent, and set the bit
996 * on the first half
997 */
998 if (state->start <= end && state->end > end) {
Chris Mason1edbb732009-09-02 13:24:36 -0400999 if (state->state & exclusive_bits) {
Chris Masond1310b22008-01-24 16:13:08 -05001000 *failed_start = start;
1001 err = -EEXIST;
1002 goto out;
1003 }
Xiao Guangrong82337672011-04-20 06:44:57 +00001004
1005 prealloc = alloc_extent_state_atomic(prealloc);
1006 BUG_ON(!prealloc);
Chris Masond1310b22008-01-24 16:13:08 -05001007 err = split_state(tree, state, prealloc, end + 1);
Jeff Mahoneyc2d904e2011-10-03 23:22:32 -04001008 if (err)
1009 extent_io_tree_panic(tree, err);
Chris Masond1310b22008-01-24 16:13:08 -05001010
Jeff Mahoney1bf85042011-07-21 16:56:09 +00001011 set_state_bits(tree, prealloc, &bits);
Chris Mason2c64c532009-09-02 15:04:12 -04001012 cache_state(prealloc, cached_state);
Chris Masond1310b22008-01-24 16:13:08 -05001013 merge_state(tree, prealloc);
1014 prealloc = NULL;
1015 goto out;
1016 }
1017
1018 goto search_again;
1019
1020out:
Chris Masoncad321a2008-12-17 14:51:42 -05001021 spin_unlock(&tree->lock);
Chris Masond1310b22008-01-24 16:13:08 -05001022 if (prealloc)
1023 free_extent_state(prealloc);
1024
1025 return err;
1026
1027search_again:
1028 if (start > end)
1029 goto out;
Chris Masoncad321a2008-12-17 14:51:42 -05001030 spin_unlock(&tree->lock);
Mel Gormand0164ad2015-11-06 16:28:21 -08001031 if (gfpflags_allow_blocking(mask))
Chris Masond1310b22008-01-24 16:13:08 -05001032 cond_resched();
1033 goto again;
1034}
Chris Masond1310b22008-01-24 16:13:08 -05001035
David Sterba41074882013-04-29 13:38:46 +00001036int set_extent_bit(struct extent_io_tree *tree, u64 start, u64 end,
David Sterba9ee49a042015-01-14 19:52:13 +01001037 unsigned bits, u64 * failed_start,
David Sterba41074882013-04-29 13:38:46 +00001038 struct extent_state **cached_state, gfp_t mask)
Jeff Mahoney3fbe5c02012-03-01 14:57:19 +01001039{
1040 return __set_extent_bit(tree, start, end, bits, 0, failed_start,
1041 cached_state, mask);
1042}
1043
1044
Josef Bacik462d6fa2011-09-26 13:56:12 -04001045/**
Liu Bo10983f22012-07-11 15:26:19 +08001046 * convert_extent_bit - convert all bits in a given range from one bit to
1047 * another
Josef Bacik462d6fa2011-09-26 13:56:12 -04001048 * @tree: the io tree to search
1049 * @start: the start offset in bytes
1050 * @end: the end offset in bytes (inclusive)
1051 * @bits: the bits to set in this range
1052 * @clear_bits: the bits to clear in this range
Josef Bacike6138872012-09-27 17:07:30 -04001053 * @cached_state: state that we're going to cache
Josef Bacik462d6fa2011-09-26 13:56:12 -04001054 * @mask: the allocation mask
1055 *
1056 * This will go through and set bits for the given range. If any states exist
1057 * already in this range they are set with the given bit and cleared of the
1058 * clear_bits. This is only meant to be used by things that are mergeable, ie
1059 * converting from say DELALLOC to DIRTY. This is not meant to be used with
1060 * boundary bits like LOCK.
1061 */
1062int convert_extent_bit(struct extent_io_tree *tree, u64 start, u64 end,
David Sterba9ee49a042015-01-14 19:52:13 +01001063 unsigned bits, unsigned clear_bits,
Josef Bacike6138872012-09-27 17:07:30 -04001064 struct extent_state **cached_state, gfp_t mask)
Josef Bacik462d6fa2011-09-26 13:56:12 -04001065{
1066 struct extent_state *state;
1067 struct extent_state *prealloc = NULL;
1068 struct rb_node *node;
Filipe David Borba Manana12cfbad2013-11-26 15:41:47 +00001069 struct rb_node **p;
1070 struct rb_node *parent;
Josef Bacik462d6fa2011-09-26 13:56:12 -04001071 int err = 0;
1072 u64 last_start;
1073 u64 last_end;
Filipe Mananac8fd3de2014-10-13 12:28:39 +01001074 bool first_iteration = true;
Josef Bacik462d6fa2011-09-26 13:56:12 -04001075
Josef Bacika5dee372013-12-13 10:02:44 -05001076 btrfs_debug_check_extent_io_range(tree, start, end);
David Sterba8d599ae2013-04-30 15:22:23 +00001077
Josef Bacik462d6fa2011-09-26 13:56:12 -04001078again:
Mel Gormand0164ad2015-11-06 16:28:21 -08001079 if (!prealloc && gfpflags_allow_blocking(mask)) {
Filipe Mananac8fd3de2014-10-13 12:28:39 +01001080 /*
1081 * Best effort, don't worry if extent state allocation fails
1082 * here for the first iteration. We might have a cached state
1083 * that matches exactly the target range, in which case no
1084 * extent state allocations are needed. We'll only know this
1085 * after locking the tree.
1086 */
Josef Bacik462d6fa2011-09-26 13:56:12 -04001087 prealloc = alloc_extent_state(mask);
Filipe Mananac8fd3de2014-10-13 12:28:39 +01001088 if (!prealloc && !first_iteration)
Josef Bacik462d6fa2011-09-26 13:56:12 -04001089 return -ENOMEM;
1090 }
1091
1092 spin_lock(&tree->lock);
Josef Bacike6138872012-09-27 17:07:30 -04001093 if (cached_state && *cached_state) {
1094 state = *cached_state;
1095 if (state->start <= start && state->end > start &&
Filipe Manana27a35072014-07-06 20:09:59 +01001096 extent_state_in_tree(state)) {
Josef Bacike6138872012-09-27 17:07:30 -04001097 node = &state->rb_node;
1098 goto hit_next;
1099 }
1100 }
1101
Josef Bacik462d6fa2011-09-26 13:56:12 -04001102 /*
1103 * this search will find all the extents that end after
1104 * our range starts.
1105 */
Filipe David Borba Manana12cfbad2013-11-26 15:41:47 +00001106 node = tree_search_for_insert(tree, start, &p, &parent);
Josef Bacik462d6fa2011-09-26 13:56:12 -04001107 if (!node) {
1108 prealloc = alloc_extent_state_atomic(prealloc);
Liu Bo1cf4ffd2011-12-07 20:08:40 -05001109 if (!prealloc) {
1110 err = -ENOMEM;
1111 goto out;
1112 }
Filipe David Borba Manana12cfbad2013-11-26 15:41:47 +00001113 err = insert_state(tree, prealloc, start, end,
1114 &p, &parent, &bits);
Jeff Mahoneyc2d904e2011-10-03 23:22:32 -04001115 if (err)
1116 extent_io_tree_panic(tree, err);
Filipe David Borba Mananac42ac0b2013-11-26 15:01:34 +00001117 cache_state(prealloc, cached_state);
1118 prealloc = NULL;
Josef Bacik462d6fa2011-09-26 13:56:12 -04001119 goto out;
1120 }
1121 state = rb_entry(node, struct extent_state, rb_node);
1122hit_next:
1123 last_start = state->start;
1124 last_end = state->end;
1125
1126 /*
1127 * | ---- desired range ---- |
1128 * | state |
1129 *
1130 * Just lock what we found and keep going
1131 */
1132 if (state->start == start && state->end <= end) {
Josef Bacik462d6fa2011-09-26 13:56:12 -04001133 set_state_bits(tree, state, &bits);
Josef Bacike6138872012-09-27 17:07:30 -04001134 cache_state(state, cached_state);
Liu Bod1ac6e42012-05-10 18:10:39 +08001135 state = clear_state_bit(tree, state, &clear_bits, 0);
Josef Bacik462d6fa2011-09-26 13:56:12 -04001136 if (last_end == (u64)-1)
1137 goto out;
Josef Bacik462d6fa2011-09-26 13:56:12 -04001138 start = last_end + 1;
Liu Bod1ac6e42012-05-10 18:10:39 +08001139 if (start < end && state && state->start == start &&
1140 !need_resched())
1141 goto hit_next;
Josef Bacik462d6fa2011-09-26 13:56:12 -04001142 goto search_again;
1143 }
1144
1145 /*
1146 * | ---- desired range ---- |
1147 * | state |
1148 * or
1149 * | ------------- state -------------- |
1150 *
1151 * We need to split the extent we found, and may flip bits on
1152 * second half.
1153 *
1154 * If the extent we found extends past our
1155 * range, we just split and search again. It'll get split
1156 * again the next time though.
1157 *
1158 * If the extent we found is inside our range, we set the
1159 * desired bit on it.
1160 */
1161 if (state->start < start) {
1162 prealloc = alloc_extent_state_atomic(prealloc);
Liu Bo1cf4ffd2011-12-07 20:08:40 -05001163 if (!prealloc) {
1164 err = -ENOMEM;
1165 goto out;
1166 }
Josef Bacik462d6fa2011-09-26 13:56:12 -04001167 err = split_state(tree, state, prealloc, start);
Jeff Mahoneyc2d904e2011-10-03 23:22:32 -04001168 if (err)
1169 extent_io_tree_panic(tree, err);
Josef Bacik462d6fa2011-09-26 13:56:12 -04001170 prealloc = NULL;
1171 if (err)
1172 goto out;
1173 if (state->end <= end) {
1174 set_state_bits(tree, state, &bits);
Josef Bacike6138872012-09-27 17:07:30 -04001175 cache_state(state, cached_state);
Liu Bod1ac6e42012-05-10 18:10:39 +08001176 state = clear_state_bit(tree, state, &clear_bits, 0);
Josef Bacik462d6fa2011-09-26 13:56:12 -04001177 if (last_end == (u64)-1)
1178 goto out;
1179 start = last_end + 1;
Liu Bod1ac6e42012-05-10 18:10:39 +08001180 if (start < end && state && state->start == start &&
1181 !need_resched())
1182 goto hit_next;
Josef Bacik462d6fa2011-09-26 13:56:12 -04001183 }
1184 goto search_again;
1185 }
1186 /*
1187 * | ---- desired range ---- |
1188 * | state | or | state |
1189 *
1190 * There's a hole, we need to insert something in it and
1191 * ignore the extent we found.
1192 */
1193 if (state->start > start) {
1194 u64 this_end;
1195 if (end < last_start)
1196 this_end = end;
1197 else
1198 this_end = last_start - 1;
1199
1200 prealloc = alloc_extent_state_atomic(prealloc);
Liu Bo1cf4ffd2011-12-07 20:08:40 -05001201 if (!prealloc) {
1202 err = -ENOMEM;
1203 goto out;
1204 }
Josef Bacik462d6fa2011-09-26 13:56:12 -04001205
1206 /*
1207 * Avoid to free 'prealloc' if it can be merged with
1208 * the later extent.
1209 */
1210 err = insert_state(tree, prealloc, start, this_end,
Filipe David Borba Manana12cfbad2013-11-26 15:41:47 +00001211 NULL, NULL, &bits);
Jeff Mahoneyc2d904e2011-10-03 23:22:32 -04001212 if (err)
1213 extent_io_tree_panic(tree, err);
Josef Bacike6138872012-09-27 17:07:30 -04001214 cache_state(prealloc, cached_state);
Josef Bacik462d6fa2011-09-26 13:56:12 -04001215 prealloc = NULL;
1216 start = this_end + 1;
1217 goto search_again;
1218 }
1219 /*
1220 * | ---- desired range ---- |
1221 * | state |
1222 * We need to split the extent, and set the bit
1223 * on the first half
1224 */
1225 if (state->start <= end && state->end > end) {
1226 prealloc = alloc_extent_state_atomic(prealloc);
Liu Bo1cf4ffd2011-12-07 20:08:40 -05001227 if (!prealloc) {
1228 err = -ENOMEM;
1229 goto out;
1230 }
Josef Bacik462d6fa2011-09-26 13:56:12 -04001231
1232 err = split_state(tree, state, prealloc, end + 1);
Jeff Mahoneyc2d904e2011-10-03 23:22:32 -04001233 if (err)
1234 extent_io_tree_panic(tree, err);
Josef Bacik462d6fa2011-09-26 13:56:12 -04001235
1236 set_state_bits(tree, prealloc, &bits);
Josef Bacike6138872012-09-27 17:07:30 -04001237 cache_state(prealloc, cached_state);
Josef Bacik462d6fa2011-09-26 13:56:12 -04001238 clear_state_bit(tree, prealloc, &clear_bits, 0);
Josef Bacik462d6fa2011-09-26 13:56:12 -04001239 prealloc = NULL;
1240 goto out;
1241 }
1242
1243 goto search_again;
1244
1245out:
1246 spin_unlock(&tree->lock);
1247 if (prealloc)
1248 free_extent_state(prealloc);
1249
1250 return err;
1251
1252search_again:
1253 if (start > end)
1254 goto out;
1255 spin_unlock(&tree->lock);
Mel Gormand0164ad2015-11-06 16:28:21 -08001256 if (gfpflags_allow_blocking(mask))
Josef Bacik462d6fa2011-09-26 13:56:12 -04001257 cond_resched();
Filipe Mananac8fd3de2014-10-13 12:28:39 +01001258 first_iteration = false;
Josef Bacik462d6fa2011-09-26 13:56:12 -04001259 goto again;
1260}
1261
Chris Masond1310b22008-01-24 16:13:08 -05001262/* wrappers around set/clear extent bit */
1263int set_extent_dirty(struct extent_io_tree *tree, u64 start, u64 end,
1264 gfp_t mask)
1265{
Jeff Mahoney3fbe5c02012-03-01 14:57:19 +01001266 return set_extent_bit(tree, start, end, EXTENT_DIRTY, NULL,
Chris Mason2c64c532009-09-02 15:04:12 -04001267 NULL, mask);
Chris Masond1310b22008-01-24 16:13:08 -05001268}
Chris Masond1310b22008-01-24 16:13:08 -05001269
1270int set_extent_bits(struct extent_io_tree *tree, u64 start, u64 end,
David Sterba9ee49a042015-01-14 19:52:13 +01001271 unsigned bits, gfp_t mask)
Chris Masond1310b22008-01-24 16:13:08 -05001272{
Jeff Mahoney3fbe5c02012-03-01 14:57:19 +01001273 return set_extent_bit(tree, start, end, bits, NULL,
Chris Mason2c64c532009-09-02 15:04:12 -04001274 NULL, mask);
Chris Masond1310b22008-01-24 16:13:08 -05001275}
Chris Masond1310b22008-01-24 16:13:08 -05001276
1277int clear_extent_bits(struct extent_io_tree *tree, u64 start, u64 end,
David Sterba9ee49a042015-01-14 19:52:13 +01001278 unsigned bits, gfp_t mask)
Chris Masond1310b22008-01-24 16:13:08 -05001279{
Filipe Manana0f318712015-05-14 20:41:07 +01001280 int wake = 0;
1281
1282 if (bits & EXTENT_LOCKED)
1283 wake = 1;
1284
1285 return clear_extent_bit(tree, start, end, bits, wake, 0, NULL, mask);
Chris Masond1310b22008-01-24 16:13:08 -05001286}
Chris Masond1310b22008-01-24 16:13:08 -05001287
1288int set_extent_delalloc(struct extent_io_tree *tree, u64 start, u64 end,
Josef Bacik2ac55d42010-02-03 19:33:23 +00001289 struct extent_state **cached_state, gfp_t mask)
Chris Masond1310b22008-01-24 16:13:08 -05001290{
1291 return set_extent_bit(tree, start, end,
Liu Bofee187d2011-09-29 15:55:28 +08001292 EXTENT_DELALLOC | EXTENT_UPTODATE,
Jeff Mahoney3fbe5c02012-03-01 14:57:19 +01001293 NULL, cached_state, mask);
Chris Masond1310b22008-01-24 16:13:08 -05001294}
Chris Masond1310b22008-01-24 16:13:08 -05001295
Liu Bo9e8a4a82012-09-05 19:10:51 -06001296int set_extent_defrag(struct extent_io_tree *tree, u64 start, u64 end,
1297 struct extent_state **cached_state, gfp_t mask)
1298{
1299 return set_extent_bit(tree, start, end,
1300 EXTENT_DELALLOC | EXTENT_UPTODATE | EXTENT_DEFRAG,
1301 NULL, cached_state, mask);
1302}
1303
Chris Masond1310b22008-01-24 16:13:08 -05001304int clear_extent_dirty(struct extent_io_tree *tree, u64 start, u64 end,
1305 gfp_t mask)
1306{
1307 return clear_extent_bit(tree, start, end,
Josef Bacik32c00af2009-10-08 13:34:05 -04001308 EXTENT_DIRTY | EXTENT_DELALLOC |
Yan, Zheng0ca1f7c2010-05-16 10:48:47 -04001309 EXTENT_DO_ACCOUNTING, 0, 0, NULL, mask);
Chris Masond1310b22008-01-24 16:13:08 -05001310}
Chris Masond1310b22008-01-24 16:13:08 -05001311
1312int set_extent_new(struct extent_io_tree *tree, u64 start, u64 end,
1313 gfp_t mask)
1314{
Jeff Mahoney3fbe5c02012-03-01 14:57:19 +01001315 return set_extent_bit(tree, start, end, EXTENT_NEW, NULL,
Chris Mason2c64c532009-09-02 15:04:12 -04001316 NULL, mask);
Chris Masond1310b22008-01-24 16:13:08 -05001317}
Chris Masond1310b22008-01-24 16:13:08 -05001318
Chris Masond1310b22008-01-24 16:13:08 -05001319int set_extent_uptodate(struct extent_io_tree *tree, u64 start, u64 end,
Arne Jansen507903b2011-04-06 10:02:20 +00001320 struct extent_state **cached_state, gfp_t mask)
Chris Masond1310b22008-01-24 16:13:08 -05001321{
Liu Bo6b67a322013-03-28 08:30:28 +00001322 return set_extent_bit(tree, start, end, EXTENT_UPTODATE, NULL,
Jeff Mahoney3fbe5c02012-03-01 14:57:19 +01001323 cached_state, mask);
Chris Masond1310b22008-01-24 16:13:08 -05001324}
Chris Masond1310b22008-01-24 16:13:08 -05001325
Josef Bacik5fd02042012-05-02 14:00:54 -04001326int clear_extent_uptodate(struct extent_io_tree *tree, u64 start, u64 end,
1327 struct extent_state **cached_state, gfp_t mask)
Chris Masond1310b22008-01-24 16:13:08 -05001328{
Chris Mason2c64c532009-09-02 15:04:12 -04001329 return clear_extent_bit(tree, start, end, EXTENT_UPTODATE, 0, 0,
Josef Bacik2ac55d42010-02-03 19:33:23 +00001330 cached_state, mask);
Chris Masond1310b22008-01-24 16:13:08 -05001331}
Chris Masond1310b22008-01-24 16:13:08 -05001332
Chris Masond352ac62008-09-29 15:18:18 -04001333/*
1334 * either insert or lock state struct between start and end use mask to tell
1335 * us if waiting is desired.
1336 */
Chris Mason1edbb732009-09-02 13:24:36 -04001337int lock_extent_bits(struct extent_io_tree *tree, u64 start, u64 end,
David Sterba9ee49a042015-01-14 19:52:13 +01001338 unsigned bits, struct extent_state **cached_state)
Chris Masond1310b22008-01-24 16:13:08 -05001339{
1340 int err;
1341 u64 failed_start;
David Sterba9ee49a042015-01-14 19:52:13 +01001342
Chris Masond1310b22008-01-24 16:13:08 -05001343 while (1) {
Jeff Mahoney3fbe5c02012-03-01 14:57:19 +01001344 err = __set_extent_bit(tree, start, end, EXTENT_LOCKED | bits,
1345 EXTENT_LOCKED, &failed_start,
1346 cached_state, GFP_NOFS);
Jeff Mahoneyd0082372012-03-01 14:57:19 +01001347 if (err == -EEXIST) {
Chris Masond1310b22008-01-24 16:13:08 -05001348 wait_extent_bit(tree, failed_start, end, EXTENT_LOCKED);
1349 start = failed_start;
Jeff Mahoneyd0082372012-03-01 14:57:19 +01001350 } else
Chris Masond1310b22008-01-24 16:13:08 -05001351 break;
Chris Masond1310b22008-01-24 16:13:08 -05001352 WARN_ON(start > end);
1353 }
1354 return err;
1355}
Chris Masond1310b22008-01-24 16:13:08 -05001356
Jeff Mahoneyd0082372012-03-01 14:57:19 +01001357int lock_extent(struct extent_io_tree *tree, u64 start, u64 end)
Chris Mason1edbb732009-09-02 13:24:36 -04001358{
Jeff Mahoneyd0082372012-03-01 14:57:19 +01001359 return lock_extent_bits(tree, start, end, 0, NULL);
Chris Mason1edbb732009-09-02 13:24:36 -04001360}
1361
Jeff Mahoneyd0082372012-03-01 14:57:19 +01001362int try_lock_extent(struct extent_io_tree *tree, u64 start, u64 end)
Josef Bacik25179202008-10-29 14:49:05 -04001363{
1364 int err;
1365 u64 failed_start;
1366
Jeff Mahoney3fbe5c02012-03-01 14:57:19 +01001367 err = __set_extent_bit(tree, start, end, EXTENT_LOCKED, EXTENT_LOCKED,
1368 &failed_start, NULL, GFP_NOFS);
Yan Zheng66435582008-10-30 14:19:50 -04001369 if (err == -EEXIST) {
1370 if (failed_start > start)
1371 clear_extent_bit(tree, start, failed_start - 1,
Jeff Mahoneyd0082372012-03-01 14:57:19 +01001372 EXTENT_LOCKED, 1, 0, NULL, GFP_NOFS);
Josef Bacik25179202008-10-29 14:49:05 -04001373 return 0;
Yan Zheng66435582008-10-30 14:19:50 -04001374 }
Josef Bacik25179202008-10-29 14:49:05 -04001375 return 1;
1376}
Josef Bacik25179202008-10-29 14:49:05 -04001377
Chris Mason2c64c532009-09-02 15:04:12 -04001378int unlock_extent_cached(struct extent_io_tree *tree, u64 start, u64 end,
1379 struct extent_state **cached, gfp_t mask)
1380{
1381 return clear_extent_bit(tree, start, end, EXTENT_LOCKED, 1, 0, cached,
1382 mask);
1383}
1384
Jeff Mahoneyd0082372012-03-01 14:57:19 +01001385int unlock_extent(struct extent_io_tree *tree, u64 start, u64 end)
Chris Masond1310b22008-01-24 16:13:08 -05001386{
Chris Mason2c64c532009-09-02 15:04:12 -04001387 return clear_extent_bit(tree, start, end, EXTENT_LOCKED, 1, 0, NULL,
Jeff Mahoneyd0082372012-03-01 14:57:19 +01001388 GFP_NOFS);
Chris Masond1310b22008-01-24 16:13:08 -05001389}
Chris Masond1310b22008-01-24 16:13:08 -05001390
Chris Mason4adaa612013-03-26 13:07:00 -04001391int extent_range_clear_dirty_for_io(struct inode *inode, u64 start, u64 end)
1392{
1393 unsigned long index = start >> PAGE_CACHE_SHIFT;
1394 unsigned long end_index = end >> PAGE_CACHE_SHIFT;
1395 struct page *page;
1396
1397 while (index <= end_index) {
1398 page = find_get_page(inode->i_mapping, index);
1399 BUG_ON(!page); /* Pages should be in the extent_io_tree */
1400 clear_page_dirty_for_io(page);
1401 page_cache_release(page);
1402 index++;
1403 }
1404 return 0;
1405}
1406
1407int extent_range_redirty_for_io(struct inode *inode, u64 start, u64 end)
1408{
1409 unsigned long index = start >> PAGE_CACHE_SHIFT;
1410 unsigned long end_index = end >> PAGE_CACHE_SHIFT;
1411 struct page *page;
1412
1413 while (index <= end_index) {
1414 page = find_get_page(inode->i_mapping, index);
1415 BUG_ON(!page); /* Pages should be in the extent_io_tree */
Chris Mason4adaa612013-03-26 13:07:00 -04001416 __set_page_dirty_nobuffers(page);
Konstantin Khebnikov8d386332015-02-11 15:26:55 -08001417 account_page_redirty(page);
Chris Mason4adaa612013-03-26 13:07:00 -04001418 page_cache_release(page);
1419 index++;
1420 }
1421 return 0;
1422}
1423
Chris Masond1310b22008-01-24 16:13:08 -05001424/*
Chris Masond1310b22008-01-24 16:13:08 -05001425 * helper function to set both pages and extents in the tree writeback
1426 */
Christoph Hellwigb2950862008-12-02 09:54:17 -05001427static int set_range_writeback(struct extent_io_tree *tree, u64 start, u64 end)
Chris Masond1310b22008-01-24 16:13:08 -05001428{
1429 unsigned long index = start >> PAGE_CACHE_SHIFT;
1430 unsigned long end_index = end >> PAGE_CACHE_SHIFT;
1431 struct page *page;
1432
1433 while (index <= end_index) {
1434 page = find_get_page(tree->mapping, index);
Jeff Mahoney79787ea2012-03-12 16:03:00 +01001435 BUG_ON(!page); /* Pages should be in the extent_io_tree */
Chris Masond1310b22008-01-24 16:13:08 -05001436 set_page_writeback(page);
1437 page_cache_release(page);
1438 index++;
1439 }
Chris Masond1310b22008-01-24 16:13:08 -05001440 return 0;
1441}
Chris Masond1310b22008-01-24 16:13:08 -05001442
Chris Masond352ac62008-09-29 15:18:18 -04001443/* find the first state struct with 'bits' set after 'start', and
1444 * return it. tree->lock must be held. NULL will returned if
1445 * nothing was found after 'start'
1446 */
Eric Sandeen48a3b632013-04-25 20:41:01 +00001447static struct extent_state *
1448find_first_extent_bit_state(struct extent_io_tree *tree,
David Sterba9ee49a042015-01-14 19:52:13 +01001449 u64 start, unsigned bits)
Chris Masond7fc6402008-02-18 12:12:38 -05001450{
1451 struct rb_node *node;
1452 struct extent_state *state;
1453
1454 /*
1455 * this search will find all the extents that end after
1456 * our range starts.
1457 */
1458 node = tree_search(tree, start);
Chris Masond3977122009-01-05 21:25:51 -05001459 if (!node)
Chris Masond7fc6402008-02-18 12:12:38 -05001460 goto out;
Chris Masond7fc6402008-02-18 12:12:38 -05001461
Chris Masond3977122009-01-05 21:25:51 -05001462 while (1) {
Chris Masond7fc6402008-02-18 12:12:38 -05001463 state = rb_entry(node, struct extent_state, rb_node);
Chris Masond3977122009-01-05 21:25:51 -05001464 if (state->end >= start && (state->state & bits))
Chris Masond7fc6402008-02-18 12:12:38 -05001465 return state;
Chris Masond3977122009-01-05 21:25:51 -05001466
Chris Masond7fc6402008-02-18 12:12:38 -05001467 node = rb_next(node);
1468 if (!node)
1469 break;
1470 }
1471out:
1472 return NULL;
1473}
Chris Masond7fc6402008-02-18 12:12:38 -05001474
Chris Masond352ac62008-09-29 15:18:18 -04001475/*
Xiao Guangrong69261c42011-07-14 03:19:45 +00001476 * find the first offset in the io tree with 'bits' set. zero is
1477 * returned if we find something, and *start_ret and *end_ret are
1478 * set to reflect the state struct that was found.
1479 *
Wang Sheng-Hui477d7ea2012-04-06 14:35:47 +08001480 * If nothing was found, 1 is returned. If found something, return 0.
Xiao Guangrong69261c42011-07-14 03:19:45 +00001481 */
1482int find_first_extent_bit(struct extent_io_tree *tree, u64 start,
David Sterba9ee49a042015-01-14 19:52:13 +01001483 u64 *start_ret, u64 *end_ret, unsigned bits,
Josef Bacike6138872012-09-27 17:07:30 -04001484 struct extent_state **cached_state)
Xiao Guangrong69261c42011-07-14 03:19:45 +00001485{
1486 struct extent_state *state;
Josef Bacike6138872012-09-27 17:07:30 -04001487 struct rb_node *n;
Xiao Guangrong69261c42011-07-14 03:19:45 +00001488 int ret = 1;
1489
1490 spin_lock(&tree->lock);
Josef Bacike6138872012-09-27 17:07:30 -04001491 if (cached_state && *cached_state) {
1492 state = *cached_state;
Filipe Manana27a35072014-07-06 20:09:59 +01001493 if (state->end == start - 1 && extent_state_in_tree(state)) {
Josef Bacike6138872012-09-27 17:07:30 -04001494 n = rb_next(&state->rb_node);
1495 while (n) {
1496 state = rb_entry(n, struct extent_state,
1497 rb_node);
1498 if (state->state & bits)
1499 goto got_it;
1500 n = rb_next(n);
1501 }
1502 free_extent_state(*cached_state);
1503 *cached_state = NULL;
1504 goto out;
1505 }
1506 free_extent_state(*cached_state);
1507 *cached_state = NULL;
1508 }
1509
Xiao Guangrong69261c42011-07-14 03:19:45 +00001510 state = find_first_extent_bit_state(tree, start, bits);
Josef Bacike6138872012-09-27 17:07:30 -04001511got_it:
Xiao Guangrong69261c42011-07-14 03:19:45 +00001512 if (state) {
Filipe Mananae38e2ed2014-10-13 12:28:38 +01001513 cache_state_if_flags(state, cached_state, 0);
Xiao Guangrong69261c42011-07-14 03:19:45 +00001514 *start_ret = state->start;
1515 *end_ret = state->end;
1516 ret = 0;
1517 }
Josef Bacike6138872012-09-27 17:07:30 -04001518out:
Xiao Guangrong69261c42011-07-14 03:19:45 +00001519 spin_unlock(&tree->lock);
1520 return ret;
1521}
1522
1523/*
Chris Masond352ac62008-09-29 15:18:18 -04001524 * find a contiguous range of bytes in the file marked as delalloc, not
1525 * more than 'max_bytes'. start and end are used to return the range,
1526 *
1527 * 1 is returned if we find something, 0 if nothing was in the tree
1528 */
Chris Masonc8b97812008-10-29 14:49:59 -04001529static noinline u64 find_delalloc_range(struct extent_io_tree *tree,
Josef Bacikc2a128d2010-02-02 21:19:11 +00001530 u64 *start, u64 *end, u64 max_bytes,
1531 struct extent_state **cached_state)
Chris Masond1310b22008-01-24 16:13:08 -05001532{
1533 struct rb_node *node;
1534 struct extent_state *state;
1535 u64 cur_start = *start;
1536 u64 found = 0;
1537 u64 total_bytes = 0;
1538
Chris Masoncad321a2008-12-17 14:51:42 -05001539 spin_lock(&tree->lock);
Chris Masonc8b97812008-10-29 14:49:59 -04001540
Chris Masond1310b22008-01-24 16:13:08 -05001541 /*
1542 * this search will find all the extents that end after
1543 * our range starts.
1544 */
Chris Mason80ea96b2008-02-01 14:51:59 -05001545 node = tree_search(tree, cur_start);
Peter2b114d12008-04-01 11:21:40 -04001546 if (!node) {
Chris Mason3b951512008-04-17 11:29:12 -04001547 if (!found)
1548 *end = (u64)-1;
Chris Masond1310b22008-01-24 16:13:08 -05001549 goto out;
1550 }
1551
Chris Masond3977122009-01-05 21:25:51 -05001552 while (1) {
Chris Masond1310b22008-01-24 16:13:08 -05001553 state = rb_entry(node, struct extent_state, rb_node);
Zheng Yan5b21f2e2008-09-26 10:05:38 -04001554 if (found && (state->start != cur_start ||
1555 (state->state & EXTENT_BOUNDARY))) {
Chris Masond1310b22008-01-24 16:13:08 -05001556 goto out;
1557 }
1558 if (!(state->state & EXTENT_DELALLOC)) {
1559 if (!found)
1560 *end = state->end;
1561 goto out;
1562 }
Josef Bacikc2a128d2010-02-02 21:19:11 +00001563 if (!found) {
Chris Masond1310b22008-01-24 16:13:08 -05001564 *start = state->start;
Josef Bacikc2a128d2010-02-02 21:19:11 +00001565 *cached_state = state;
1566 atomic_inc(&state->refs);
1567 }
Chris Masond1310b22008-01-24 16:13:08 -05001568 found++;
1569 *end = state->end;
1570 cur_start = state->end + 1;
1571 node = rb_next(node);
Chris Masond1310b22008-01-24 16:13:08 -05001572 total_bytes += state->end - state->start + 1;
Josef Bacik7bf811a52013-10-07 22:11:09 -04001573 if (total_bytes >= max_bytes)
Josef Bacik573aeca2013-08-30 14:38:49 -04001574 break;
Josef Bacik573aeca2013-08-30 14:38:49 -04001575 if (!node)
Chris Masond1310b22008-01-24 16:13:08 -05001576 break;
1577 }
1578out:
Chris Masoncad321a2008-12-17 14:51:42 -05001579 spin_unlock(&tree->lock);
Chris Masond1310b22008-01-24 16:13:08 -05001580 return found;
1581}
1582
Jeff Mahoney143bede2012-03-01 14:56:26 +01001583static noinline void __unlock_for_delalloc(struct inode *inode,
1584 struct page *locked_page,
1585 u64 start, u64 end)
Chris Masonc8b97812008-10-29 14:49:59 -04001586{
1587 int ret;
1588 struct page *pages[16];
1589 unsigned long index = start >> PAGE_CACHE_SHIFT;
1590 unsigned long end_index = end >> PAGE_CACHE_SHIFT;
1591 unsigned long nr_pages = end_index - index + 1;
1592 int i;
1593
1594 if (index == locked_page->index && end_index == index)
Jeff Mahoney143bede2012-03-01 14:56:26 +01001595 return;
Chris Masonc8b97812008-10-29 14:49:59 -04001596
Chris Masond3977122009-01-05 21:25:51 -05001597 while (nr_pages > 0) {
Chris Masonc8b97812008-10-29 14:49:59 -04001598 ret = find_get_pages_contig(inode->i_mapping, index,
Chris Mason5b050f02008-11-11 09:34:41 -05001599 min_t(unsigned long, nr_pages,
1600 ARRAY_SIZE(pages)), pages);
Chris Masonc8b97812008-10-29 14:49:59 -04001601 for (i = 0; i < ret; i++) {
1602 if (pages[i] != locked_page)
1603 unlock_page(pages[i]);
1604 page_cache_release(pages[i]);
1605 }
1606 nr_pages -= ret;
1607 index += ret;
1608 cond_resched();
1609 }
Chris Masonc8b97812008-10-29 14:49:59 -04001610}
1611
1612static noinline int lock_delalloc_pages(struct inode *inode,
1613 struct page *locked_page,
1614 u64 delalloc_start,
1615 u64 delalloc_end)
1616{
1617 unsigned long index = delalloc_start >> PAGE_CACHE_SHIFT;
1618 unsigned long start_index = index;
1619 unsigned long end_index = delalloc_end >> PAGE_CACHE_SHIFT;
1620 unsigned long pages_locked = 0;
1621 struct page *pages[16];
1622 unsigned long nrpages;
1623 int ret;
1624 int i;
1625
1626 /* the caller is responsible for locking the start index */
1627 if (index == locked_page->index && index == end_index)
1628 return 0;
1629
1630 /* skip the page at the start index */
1631 nrpages = end_index - index + 1;
Chris Masond3977122009-01-05 21:25:51 -05001632 while (nrpages > 0) {
Chris Masonc8b97812008-10-29 14:49:59 -04001633 ret = find_get_pages_contig(inode->i_mapping, index,
Chris Mason5b050f02008-11-11 09:34:41 -05001634 min_t(unsigned long,
1635 nrpages, ARRAY_SIZE(pages)), pages);
Chris Masonc8b97812008-10-29 14:49:59 -04001636 if (ret == 0) {
1637 ret = -EAGAIN;
1638 goto done;
1639 }
1640 /* now we have an array of pages, lock them all */
1641 for (i = 0; i < ret; i++) {
1642 /*
1643 * the caller is taking responsibility for
1644 * locked_page
1645 */
Chris Mason771ed682008-11-06 22:02:51 -05001646 if (pages[i] != locked_page) {
Chris Masonc8b97812008-10-29 14:49:59 -04001647 lock_page(pages[i]);
Chris Masonf2b1c412008-11-10 07:31:30 -05001648 if (!PageDirty(pages[i]) ||
1649 pages[i]->mapping != inode->i_mapping) {
Chris Mason771ed682008-11-06 22:02:51 -05001650 ret = -EAGAIN;
1651 unlock_page(pages[i]);
1652 page_cache_release(pages[i]);
1653 goto done;
1654 }
1655 }
Chris Masonc8b97812008-10-29 14:49:59 -04001656 page_cache_release(pages[i]);
Chris Mason771ed682008-11-06 22:02:51 -05001657 pages_locked++;
Chris Masonc8b97812008-10-29 14:49:59 -04001658 }
Chris Masonc8b97812008-10-29 14:49:59 -04001659 nrpages -= ret;
1660 index += ret;
1661 cond_resched();
1662 }
1663 ret = 0;
1664done:
1665 if (ret && pages_locked) {
1666 __unlock_for_delalloc(inode, locked_page,
1667 delalloc_start,
1668 ((u64)(start_index + pages_locked - 1)) <<
1669 PAGE_CACHE_SHIFT);
1670 }
1671 return ret;
1672}
1673
1674/*
1675 * find a contiguous range of bytes in the file marked as delalloc, not
1676 * more than 'max_bytes'. start and end are used to return the range,
1677 *
1678 * 1 is returned if we find something, 0 if nothing was in the tree
1679 */
Josef Bacik294e30f2013-10-09 12:00:56 -04001680STATIC u64 find_lock_delalloc_range(struct inode *inode,
1681 struct extent_io_tree *tree,
1682 struct page *locked_page, u64 *start,
1683 u64 *end, u64 max_bytes)
Chris Masonc8b97812008-10-29 14:49:59 -04001684{
1685 u64 delalloc_start;
1686 u64 delalloc_end;
1687 u64 found;
Chris Mason9655d292009-09-02 15:22:30 -04001688 struct extent_state *cached_state = NULL;
Chris Masonc8b97812008-10-29 14:49:59 -04001689 int ret;
1690 int loops = 0;
1691
1692again:
1693 /* step one, find a bunch of delalloc bytes starting at start */
1694 delalloc_start = *start;
1695 delalloc_end = 0;
1696 found = find_delalloc_range(tree, &delalloc_start, &delalloc_end,
Josef Bacikc2a128d2010-02-02 21:19:11 +00001697 max_bytes, &cached_state);
Chris Mason70b99e62008-10-31 12:46:39 -04001698 if (!found || delalloc_end <= *start) {
Chris Masonc8b97812008-10-29 14:49:59 -04001699 *start = delalloc_start;
1700 *end = delalloc_end;
Josef Bacikc2a128d2010-02-02 21:19:11 +00001701 free_extent_state(cached_state);
Liu Bo385fe0b2013-10-01 23:49:49 +08001702 return 0;
Chris Masonc8b97812008-10-29 14:49:59 -04001703 }
1704
1705 /*
Chris Mason70b99e62008-10-31 12:46:39 -04001706 * start comes from the offset of locked_page. We have to lock
1707 * pages in order, so we can't process delalloc bytes before
1708 * locked_page
1709 */
Chris Masond3977122009-01-05 21:25:51 -05001710 if (delalloc_start < *start)
Chris Mason70b99e62008-10-31 12:46:39 -04001711 delalloc_start = *start;
Chris Mason70b99e62008-10-31 12:46:39 -04001712
1713 /*
Chris Masonc8b97812008-10-29 14:49:59 -04001714 * make sure to limit the number of pages we try to lock down
Chris Masonc8b97812008-10-29 14:49:59 -04001715 */
Josef Bacik7bf811a52013-10-07 22:11:09 -04001716 if (delalloc_end + 1 - delalloc_start > max_bytes)
1717 delalloc_end = delalloc_start + max_bytes - 1;
Chris Masond3977122009-01-05 21:25:51 -05001718
Chris Masonc8b97812008-10-29 14:49:59 -04001719 /* step two, lock all the pages after the page that has start */
1720 ret = lock_delalloc_pages(inode, locked_page,
1721 delalloc_start, delalloc_end);
1722 if (ret == -EAGAIN) {
1723 /* some of the pages are gone, lets avoid looping by
1724 * shortening the size of the delalloc range we're searching
1725 */
Chris Mason9655d292009-09-02 15:22:30 -04001726 free_extent_state(cached_state);
Chris Mason7d788742014-05-21 05:49:54 -07001727 cached_state = NULL;
Chris Masonc8b97812008-10-29 14:49:59 -04001728 if (!loops) {
Josef Bacik7bf811a52013-10-07 22:11:09 -04001729 max_bytes = PAGE_CACHE_SIZE;
Chris Masonc8b97812008-10-29 14:49:59 -04001730 loops = 1;
1731 goto again;
1732 } else {
1733 found = 0;
1734 goto out_failed;
1735 }
1736 }
Jeff Mahoney79787ea2012-03-12 16:03:00 +01001737 BUG_ON(ret); /* Only valid values are 0 and -EAGAIN */
Chris Masonc8b97812008-10-29 14:49:59 -04001738
1739 /* step three, lock the state bits for the whole range */
Jeff Mahoneyd0082372012-03-01 14:57:19 +01001740 lock_extent_bits(tree, delalloc_start, delalloc_end, 0, &cached_state);
Chris Masonc8b97812008-10-29 14:49:59 -04001741
1742 /* then test to make sure it is all still delalloc */
1743 ret = test_range_bit(tree, delalloc_start, delalloc_end,
Chris Mason9655d292009-09-02 15:22:30 -04001744 EXTENT_DELALLOC, 1, cached_state);
Chris Masonc8b97812008-10-29 14:49:59 -04001745 if (!ret) {
Chris Mason9655d292009-09-02 15:22:30 -04001746 unlock_extent_cached(tree, delalloc_start, delalloc_end,
1747 &cached_state, GFP_NOFS);
Chris Masonc8b97812008-10-29 14:49:59 -04001748 __unlock_for_delalloc(inode, locked_page,
1749 delalloc_start, delalloc_end);
1750 cond_resched();
1751 goto again;
1752 }
Chris Mason9655d292009-09-02 15:22:30 -04001753 free_extent_state(cached_state);
Chris Masonc8b97812008-10-29 14:49:59 -04001754 *start = delalloc_start;
1755 *end = delalloc_end;
1756out_failed:
1757 return found;
1758}
1759
Josef Bacikc2790a22013-07-29 11:20:47 -04001760int extent_clear_unlock_delalloc(struct inode *inode, u64 start, u64 end,
1761 struct page *locked_page,
David Sterba9ee49a042015-01-14 19:52:13 +01001762 unsigned clear_bits,
Josef Bacikc2790a22013-07-29 11:20:47 -04001763 unsigned long page_ops)
Chris Masonc8b97812008-10-29 14:49:59 -04001764{
Josef Bacikc2790a22013-07-29 11:20:47 -04001765 struct extent_io_tree *tree = &BTRFS_I(inode)->io_tree;
Chris Masonc8b97812008-10-29 14:49:59 -04001766 int ret;
1767 struct page *pages[16];
1768 unsigned long index = start >> PAGE_CACHE_SHIFT;
1769 unsigned long end_index = end >> PAGE_CACHE_SHIFT;
1770 unsigned long nr_pages = end_index - index + 1;
1771 int i;
Chris Mason771ed682008-11-06 22:02:51 -05001772
Chris Mason2c64c532009-09-02 15:04:12 -04001773 clear_extent_bit(tree, start, end, clear_bits, 1, 0, NULL, GFP_NOFS);
Josef Bacikc2790a22013-07-29 11:20:47 -04001774 if (page_ops == 0)
Chris Mason771ed682008-11-06 22:02:51 -05001775 return 0;
Chris Masonc8b97812008-10-29 14:49:59 -04001776
Filipe Manana704de492014-10-06 22:14:22 +01001777 if ((page_ops & PAGE_SET_ERROR) && nr_pages > 0)
1778 mapping_set_error(inode->i_mapping, -EIO);
1779
Chris Masond3977122009-01-05 21:25:51 -05001780 while (nr_pages > 0) {
Chris Masonc8b97812008-10-29 14:49:59 -04001781 ret = find_get_pages_contig(inode->i_mapping, index,
Chris Mason5b050f02008-11-11 09:34:41 -05001782 min_t(unsigned long,
1783 nr_pages, ARRAY_SIZE(pages)), pages);
Chris Masonc8b97812008-10-29 14:49:59 -04001784 for (i = 0; i < ret; i++) {
Chris Mason8b62b722009-09-02 16:53:46 -04001785
Josef Bacikc2790a22013-07-29 11:20:47 -04001786 if (page_ops & PAGE_SET_PRIVATE2)
Chris Mason8b62b722009-09-02 16:53:46 -04001787 SetPagePrivate2(pages[i]);
1788
Chris Masonc8b97812008-10-29 14:49:59 -04001789 if (pages[i] == locked_page) {
1790 page_cache_release(pages[i]);
1791 continue;
1792 }
Josef Bacikc2790a22013-07-29 11:20:47 -04001793 if (page_ops & PAGE_CLEAR_DIRTY)
Chris Masonc8b97812008-10-29 14:49:59 -04001794 clear_page_dirty_for_io(pages[i]);
Josef Bacikc2790a22013-07-29 11:20:47 -04001795 if (page_ops & PAGE_SET_WRITEBACK)
Chris Masonc8b97812008-10-29 14:49:59 -04001796 set_page_writeback(pages[i]);
Filipe Manana704de492014-10-06 22:14:22 +01001797 if (page_ops & PAGE_SET_ERROR)
1798 SetPageError(pages[i]);
Josef Bacikc2790a22013-07-29 11:20:47 -04001799 if (page_ops & PAGE_END_WRITEBACK)
Chris Masonc8b97812008-10-29 14:49:59 -04001800 end_page_writeback(pages[i]);
Josef Bacikc2790a22013-07-29 11:20:47 -04001801 if (page_ops & PAGE_UNLOCK)
Chris Mason771ed682008-11-06 22:02:51 -05001802 unlock_page(pages[i]);
Chris Masonc8b97812008-10-29 14:49:59 -04001803 page_cache_release(pages[i]);
1804 }
1805 nr_pages -= ret;
1806 index += ret;
1807 cond_resched();
1808 }
1809 return 0;
1810}
Chris Masonc8b97812008-10-29 14:49:59 -04001811
Chris Masond352ac62008-09-29 15:18:18 -04001812/*
1813 * count the number of bytes in the tree that have a given bit(s)
1814 * set. This can be fairly slow, except for EXTENT_DIRTY which is
1815 * cached. The total number found is returned.
1816 */
Chris Masond1310b22008-01-24 16:13:08 -05001817u64 count_range_bits(struct extent_io_tree *tree,
1818 u64 *start, u64 search_end, u64 max_bytes,
David Sterba9ee49a042015-01-14 19:52:13 +01001819 unsigned bits, int contig)
Chris Masond1310b22008-01-24 16:13:08 -05001820{
1821 struct rb_node *node;
1822 struct extent_state *state;
1823 u64 cur_start = *start;
1824 u64 total_bytes = 0;
Chris Masonec29ed52011-02-23 16:23:20 -05001825 u64 last = 0;
Chris Masond1310b22008-01-24 16:13:08 -05001826 int found = 0;
1827
Dulshani Gunawardhanafae7f212013-10-31 10:30:08 +05301828 if (WARN_ON(search_end <= cur_start))
Chris Masond1310b22008-01-24 16:13:08 -05001829 return 0;
Chris Masond1310b22008-01-24 16:13:08 -05001830
Chris Masoncad321a2008-12-17 14:51:42 -05001831 spin_lock(&tree->lock);
Chris Masond1310b22008-01-24 16:13:08 -05001832 if (cur_start == 0 && bits == EXTENT_DIRTY) {
1833 total_bytes = tree->dirty_bytes;
1834 goto out;
1835 }
1836 /*
1837 * this search will find all the extents that end after
1838 * our range starts.
1839 */
Chris Mason80ea96b2008-02-01 14:51:59 -05001840 node = tree_search(tree, cur_start);
Chris Masond3977122009-01-05 21:25:51 -05001841 if (!node)
Chris Masond1310b22008-01-24 16:13:08 -05001842 goto out;
Chris Masond1310b22008-01-24 16:13:08 -05001843
Chris Masond3977122009-01-05 21:25:51 -05001844 while (1) {
Chris Masond1310b22008-01-24 16:13:08 -05001845 state = rb_entry(node, struct extent_state, rb_node);
1846 if (state->start > search_end)
1847 break;
Chris Masonec29ed52011-02-23 16:23:20 -05001848 if (contig && found && state->start > last + 1)
1849 break;
1850 if (state->end >= cur_start && (state->state & bits) == bits) {
Chris Masond1310b22008-01-24 16:13:08 -05001851 total_bytes += min(search_end, state->end) + 1 -
1852 max(cur_start, state->start);
1853 if (total_bytes >= max_bytes)
1854 break;
1855 if (!found) {
Josef Bacikaf60bed2011-05-04 11:11:17 -04001856 *start = max(cur_start, state->start);
Chris Masond1310b22008-01-24 16:13:08 -05001857 found = 1;
1858 }
Chris Masonec29ed52011-02-23 16:23:20 -05001859 last = state->end;
1860 } else if (contig && found) {
1861 break;
Chris Masond1310b22008-01-24 16:13:08 -05001862 }
1863 node = rb_next(node);
1864 if (!node)
1865 break;
1866 }
1867out:
Chris Masoncad321a2008-12-17 14:51:42 -05001868 spin_unlock(&tree->lock);
Chris Masond1310b22008-01-24 16:13:08 -05001869 return total_bytes;
1870}
Christoph Hellwigb2950862008-12-02 09:54:17 -05001871
Chris Masond352ac62008-09-29 15:18:18 -04001872/*
1873 * set the private field for a given byte offset in the tree. If there isn't
1874 * an extent_state there already, this does nothing.
1875 */
Sergei Trofimovich171170c2013-08-14 23:27:46 +03001876static int set_state_private(struct extent_io_tree *tree, u64 start, u64 private)
Chris Masond1310b22008-01-24 16:13:08 -05001877{
1878 struct rb_node *node;
1879 struct extent_state *state;
1880 int ret = 0;
1881
Chris Masoncad321a2008-12-17 14:51:42 -05001882 spin_lock(&tree->lock);
Chris Masond1310b22008-01-24 16:13:08 -05001883 /*
1884 * this search will find all the extents that end after
1885 * our range starts.
1886 */
Chris Mason80ea96b2008-02-01 14:51:59 -05001887 node = tree_search(tree, start);
Peter2b114d12008-04-01 11:21:40 -04001888 if (!node) {
Chris Masond1310b22008-01-24 16:13:08 -05001889 ret = -ENOENT;
1890 goto out;
1891 }
1892 state = rb_entry(node, struct extent_state, rb_node);
1893 if (state->start != start) {
1894 ret = -ENOENT;
1895 goto out;
1896 }
1897 state->private = private;
1898out:
Chris Masoncad321a2008-12-17 14:51:42 -05001899 spin_unlock(&tree->lock);
Chris Masond1310b22008-01-24 16:13:08 -05001900 return ret;
1901}
1902
1903int get_state_private(struct extent_io_tree *tree, u64 start, u64 *private)
1904{
1905 struct rb_node *node;
1906 struct extent_state *state;
1907 int ret = 0;
1908
Chris Masoncad321a2008-12-17 14:51:42 -05001909 spin_lock(&tree->lock);
Chris Masond1310b22008-01-24 16:13:08 -05001910 /*
1911 * this search will find all the extents that end after
1912 * our range starts.
1913 */
Chris Mason80ea96b2008-02-01 14:51:59 -05001914 node = tree_search(tree, start);
Peter2b114d12008-04-01 11:21:40 -04001915 if (!node) {
Chris Masond1310b22008-01-24 16:13:08 -05001916 ret = -ENOENT;
1917 goto out;
1918 }
1919 state = rb_entry(node, struct extent_state, rb_node);
1920 if (state->start != start) {
1921 ret = -ENOENT;
1922 goto out;
1923 }
1924 *private = state->private;
1925out:
Chris Masoncad321a2008-12-17 14:51:42 -05001926 spin_unlock(&tree->lock);
Chris Masond1310b22008-01-24 16:13:08 -05001927 return ret;
1928}
1929
1930/*
1931 * searches a range in the state tree for a given mask.
Chris Mason70dec802008-01-29 09:59:12 -05001932 * If 'filled' == 1, this returns 1 only if every extent in the tree
Chris Masond1310b22008-01-24 16:13:08 -05001933 * has the bits set. Otherwise, 1 is returned if any bit in the
1934 * range is found set.
1935 */
1936int test_range_bit(struct extent_io_tree *tree, u64 start, u64 end,
David Sterba9ee49a042015-01-14 19:52:13 +01001937 unsigned bits, int filled, struct extent_state *cached)
Chris Masond1310b22008-01-24 16:13:08 -05001938{
1939 struct extent_state *state = NULL;
1940 struct rb_node *node;
1941 int bitset = 0;
Chris Masond1310b22008-01-24 16:13:08 -05001942
Chris Masoncad321a2008-12-17 14:51:42 -05001943 spin_lock(&tree->lock);
Filipe Manana27a35072014-07-06 20:09:59 +01001944 if (cached && extent_state_in_tree(cached) && cached->start <= start &&
Josef Bacikdf98b6e2011-06-20 14:53:48 -04001945 cached->end > start)
Chris Mason9655d292009-09-02 15:22:30 -04001946 node = &cached->rb_node;
1947 else
1948 node = tree_search(tree, start);
Chris Masond1310b22008-01-24 16:13:08 -05001949 while (node && start <= end) {
1950 state = rb_entry(node, struct extent_state, rb_node);
1951
1952 if (filled && state->start > start) {
1953 bitset = 0;
1954 break;
1955 }
1956
1957 if (state->start > end)
1958 break;
1959
1960 if (state->state & bits) {
1961 bitset = 1;
1962 if (!filled)
1963 break;
1964 } else if (filled) {
1965 bitset = 0;
1966 break;
1967 }
Chris Mason46562cec2009-09-23 20:23:16 -04001968
1969 if (state->end == (u64)-1)
1970 break;
1971
Chris Masond1310b22008-01-24 16:13:08 -05001972 start = state->end + 1;
1973 if (start > end)
1974 break;
1975 node = rb_next(node);
1976 if (!node) {
1977 if (filled)
1978 bitset = 0;
1979 break;
1980 }
1981 }
Chris Masoncad321a2008-12-17 14:51:42 -05001982 spin_unlock(&tree->lock);
Chris Masond1310b22008-01-24 16:13:08 -05001983 return bitset;
1984}
Chris Masond1310b22008-01-24 16:13:08 -05001985
1986/*
1987 * helper function to set a given page up to date if all the
1988 * extents in the tree for that page are up to date
1989 */
Jeff Mahoney143bede2012-03-01 14:56:26 +01001990static void check_page_uptodate(struct extent_io_tree *tree, struct page *page)
Chris Masond1310b22008-01-24 16:13:08 -05001991{
Miao Xie4eee4fa2012-12-21 09:17:45 +00001992 u64 start = page_offset(page);
Chris Masond1310b22008-01-24 16:13:08 -05001993 u64 end = start + PAGE_CACHE_SIZE - 1;
Chris Mason9655d292009-09-02 15:22:30 -04001994 if (test_range_bit(tree, start, end, EXTENT_UPTODATE, 1, NULL))
Chris Masond1310b22008-01-24 16:13:08 -05001995 SetPageUptodate(page);
Chris Masond1310b22008-01-24 16:13:08 -05001996}
1997
Miao Xie8b110e32014-09-12 18:44:03 +08001998int free_io_failure(struct inode *inode, struct io_failure_record *rec)
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02001999{
2000 int ret;
2001 int err = 0;
2002 struct extent_io_tree *failure_tree = &BTRFS_I(inode)->io_failure_tree;
2003
2004 set_state_private(failure_tree, rec->start, 0);
2005 ret = clear_extent_bits(failure_tree, rec->start,
2006 rec->start + rec->len - 1,
2007 EXTENT_LOCKED | EXTENT_DIRTY, GFP_NOFS);
2008 if (ret)
2009 err = ret;
2010
David Woodhouse53b381b2013-01-29 18:40:14 -05002011 ret = clear_extent_bits(&BTRFS_I(inode)->io_tree, rec->start,
2012 rec->start + rec->len - 1,
2013 EXTENT_DAMAGED, GFP_NOFS);
2014 if (ret && !err)
2015 err = ret;
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002016
2017 kfree(rec);
2018 return err;
2019}
2020
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002021/*
2022 * this bypasses the standard btrfs submit functions deliberately, as
2023 * the standard behavior is to write all copies in a raid setup. here we only
2024 * want to write the one bad copy. so we do the mapping for ourselves and issue
2025 * submit_bio directly.
Stefan Behrens3ec706c2012-11-05 15:46:42 +01002026 * to avoid any synchronization issues, wait for the data after writing, which
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002027 * actually prevents the read that triggered the error from finishing.
2028 * currently, there can be no more than two copies of every data bit. thus,
2029 * exactly one rewrite is required.
2030 */
Miao Xie1203b682014-09-12 18:44:01 +08002031int repair_io_failure(struct inode *inode, u64 start, u64 length, u64 logical,
2032 struct page *page, unsigned int pg_offset, int mirror_num)
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002033{
Miao Xie1203b682014-09-12 18:44:01 +08002034 struct btrfs_fs_info *fs_info = BTRFS_I(inode)->root->fs_info;
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002035 struct bio *bio;
2036 struct btrfs_device *dev;
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002037 u64 map_length = 0;
2038 u64 sector;
2039 struct btrfs_bio *bbio = NULL;
David Woodhouse53b381b2013-01-29 18:40:14 -05002040 struct btrfs_mapping_tree *map_tree = &fs_info->mapping_tree;
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002041 int ret;
2042
Ilya Dryomov908960c2013-11-03 19:06:39 +02002043 ASSERT(!(fs_info->sb->s_flags & MS_RDONLY));
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002044 BUG_ON(!mirror_num);
2045
David Woodhouse53b381b2013-01-29 18:40:14 -05002046 /* we can't repair anything in raid56 yet */
2047 if (btrfs_is_parity_mirror(map_tree, logical, length, mirror_num))
2048 return 0;
2049
Chris Mason9be33952013-05-17 18:30:14 -04002050 bio = btrfs_io_bio_alloc(GFP_NOFS, 1);
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002051 if (!bio)
2052 return -EIO;
Kent Overstreet4f024f32013-10-11 15:44:27 -07002053 bio->bi_iter.bi_size = 0;
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002054 map_length = length;
2055
Stefan Behrens3ec706c2012-11-05 15:46:42 +01002056 ret = btrfs_map_block(fs_info, WRITE, logical,
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002057 &map_length, &bbio, mirror_num);
2058 if (ret) {
2059 bio_put(bio);
2060 return -EIO;
2061 }
2062 BUG_ON(mirror_num != bbio->mirror_num);
2063 sector = bbio->stripes[mirror_num-1].physical >> 9;
Kent Overstreet4f024f32013-10-11 15:44:27 -07002064 bio->bi_iter.bi_sector = sector;
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002065 dev = bbio->stripes[mirror_num-1].dev;
Zhao Lei6e9606d2015-01-20 15:11:34 +08002066 btrfs_put_bbio(bbio);
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002067 if (!dev || !dev->bdev || !dev->writeable) {
2068 bio_put(bio);
2069 return -EIO;
2070 }
2071 bio->bi_bdev = dev->bdev;
Miao Xieffdd2012014-09-12 18:44:00 +08002072 bio_add_page(bio, page, length, pg_offset);
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002073
Kent Overstreet33879d42013-11-23 22:33:32 -08002074 if (btrfsic_submit_bio_wait(WRITE_SYNC, bio)) {
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002075 /* try to remap that extent elsewhere? */
2076 bio_put(bio);
Stefan Behrens442a4f62012-05-25 16:06:08 +02002077 btrfs_dev_stat_inc_and_print(dev, BTRFS_DEV_STAT_WRITE_ERRS);
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002078 return -EIO;
2079 }
2080
Frank Holtonefe120a2013-12-20 11:37:06 -05002081 printk_ratelimited_in_rcu(KERN_INFO
Miao Xie1203b682014-09-12 18:44:01 +08002082 "BTRFS: read error corrected: ino %llu off %llu (dev %s sector %llu)\n",
2083 btrfs_ino(inode), start,
2084 rcu_str_deref(dev->name), sector);
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002085 bio_put(bio);
2086 return 0;
2087}
2088
Josef Bacikea466792012-03-26 21:57:36 -04002089int repair_eb_io_failure(struct btrfs_root *root, struct extent_buffer *eb,
2090 int mirror_num)
2091{
Josef Bacikea466792012-03-26 21:57:36 -04002092 u64 start = eb->start;
2093 unsigned long i, num_pages = num_extent_pages(eb->start, eb->len);
Chris Masond95603b2012-04-12 15:55:15 -04002094 int ret = 0;
Josef Bacikea466792012-03-26 21:57:36 -04002095
Ilya Dryomov908960c2013-11-03 19:06:39 +02002096 if (root->fs_info->sb->s_flags & MS_RDONLY)
2097 return -EROFS;
2098
Josef Bacikea466792012-03-26 21:57:36 -04002099 for (i = 0; i < num_pages; i++) {
David Sterbafb85fc92014-07-31 01:03:53 +02002100 struct page *p = eb->pages[i];
Miao Xie1203b682014-09-12 18:44:01 +08002101
2102 ret = repair_io_failure(root->fs_info->btree_inode, start,
2103 PAGE_CACHE_SIZE, start, p,
2104 start - page_offset(p), mirror_num);
Josef Bacikea466792012-03-26 21:57:36 -04002105 if (ret)
2106 break;
2107 start += PAGE_CACHE_SIZE;
2108 }
2109
2110 return ret;
2111}
2112
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002113/*
2114 * each time an IO finishes, we do a fast check in the IO failure tree
2115 * to see if we need to process or clean up an io_failure_record
2116 */
Miao Xie8b110e32014-09-12 18:44:03 +08002117int clean_io_failure(struct inode *inode, u64 start, struct page *page,
2118 unsigned int pg_offset)
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002119{
2120 u64 private;
2121 u64 private_failure;
2122 struct io_failure_record *failrec;
Ilya Dryomov908960c2013-11-03 19:06:39 +02002123 struct btrfs_fs_info *fs_info = BTRFS_I(inode)->root->fs_info;
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002124 struct extent_state *state;
2125 int num_copies;
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002126 int ret;
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002127
2128 private = 0;
2129 ret = count_range_bits(&BTRFS_I(inode)->io_failure_tree, &private,
2130 (u64)-1, 1, EXTENT_DIRTY, 0);
2131 if (!ret)
2132 return 0;
2133
2134 ret = get_state_private(&BTRFS_I(inode)->io_failure_tree, start,
2135 &private_failure);
2136 if (ret)
2137 return 0;
2138
2139 failrec = (struct io_failure_record *)(unsigned long) private_failure;
2140 BUG_ON(!failrec->this_mirror);
2141
2142 if (failrec->in_validation) {
2143 /* there was no real error, just free the record */
2144 pr_debug("clean_io_failure: freeing dummy error at %llu\n",
2145 failrec->start);
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002146 goto out;
2147 }
Ilya Dryomov908960c2013-11-03 19:06:39 +02002148 if (fs_info->sb->s_flags & MS_RDONLY)
2149 goto out;
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002150
2151 spin_lock(&BTRFS_I(inode)->io_tree.lock);
2152 state = find_first_extent_bit_state(&BTRFS_I(inode)->io_tree,
2153 failrec->start,
2154 EXTENT_LOCKED);
2155 spin_unlock(&BTRFS_I(inode)->io_tree.lock);
2156
Miao Xie883d0de2013-07-25 19:22:35 +08002157 if (state && state->start <= failrec->start &&
2158 state->end >= failrec->start + failrec->len - 1) {
Stefan Behrens3ec706c2012-11-05 15:46:42 +01002159 num_copies = btrfs_num_copies(fs_info, failrec->logical,
2160 failrec->len);
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002161 if (num_copies > 1) {
Miao Xie1203b682014-09-12 18:44:01 +08002162 repair_io_failure(inode, start, failrec->len,
Miao Xie454ff3d2014-09-12 18:43:58 +08002163 failrec->logical, page,
Miao Xie1203b682014-09-12 18:44:01 +08002164 pg_offset, failrec->failed_mirror);
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002165 }
2166 }
2167
2168out:
Miao Xie454ff3d2014-09-12 18:43:58 +08002169 free_io_failure(inode, failrec);
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002170
Miao Xie454ff3d2014-09-12 18:43:58 +08002171 return 0;
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002172}
2173
Miao Xief6124962014-09-12 18:44:04 +08002174/*
2175 * Can be called when
2176 * - hold extent lock
2177 * - under ordered extent
2178 * - the inode is freeing
2179 */
2180void btrfs_free_io_failure_record(struct inode *inode, u64 start, u64 end)
2181{
2182 struct extent_io_tree *failure_tree = &BTRFS_I(inode)->io_failure_tree;
2183 struct io_failure_record *failrec;
2184 struct extent_state *state, *next;
2185
2186 if (RB_EMPTY_ROOT(&failure_tree->state))
2187 return;
2188
2189 spin_lock(&failure_tree->lock);
2190 state = find_first_extent_bit_state(failure_tree, start, EXTENT_DIRTY);
2191 while (state) {
2192 if (state->start > end)
2193 break;
2194
2195 ASSERT(state->end <= end);
2196
2197 next = next_state(state);
2198
Satoru Takeuchi6e1103a2014-12-25 18:21:41 +09002199 failrec = (struct io_failure_record *)(unsigned long)state->private;
Miao Xief6124962014-09-12 18:44:04 +08002200 free_extent_state(state);
2201 kfree(failrec);
2202
2203 state = next;
2204 }
2205 spin_unlock(&failure_tree->lock);
2206}
2207
Miao Xie2fe63032014-09-12 18:43:59 +08002208int btrfs_get_io_failure_record(struct inode *inode, u64 start, u64 end,
2209 struct io_failure_record **failrec_ret)
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002210{
Miao Xie2fe63032014-09-12 18:43:59 +08002211 struct io_failure_record *failrec;
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002212 u64 private;
2213 struct extent_map *em;
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002214 struct extent_io_tree *failure_tree = &BTRFS_I(inode)->io_failure_tree;
2215 struct extent_io_tree *tree = &BTRFS_I(inode)->io_tree;
2216 struct extent_map_tree *em_tree = &BTRFS_I(inode)->extent_tree;
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002217 int ret;
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002218 u64 logical;
2219
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002220 ret = get_state_private(failure_tree, start, &private);
2221 if (ret) {
2222 failrec = kzalloc(sizeof(*failrec), GFP_NOFS);
2223 if (!failrec)
2224 return -ENOMEM;
Miao Xie2fe63032014-09-12 18:43:59 +08002225
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002226 failrec->start = start;
2227 failrec->len = end - start + 1;
2228 failrec->this_mirror = 0;
2229 failrec->bio_flags = 0;
2230 failrec->in_validation = 0;
2231
2232 read_lock(&em_tree->lock);
2233 em = lookup_extent_mapping(em_tree, start, failrec->len);
2234 if (!em) {
2235 read_unlock(&em_tree->lock);
2236 kfree(failrec);
2237 return -EIO;
2238 }
2239
Filipe David Borba Manana68ba9902013-11-25 03:22:07 +00002240 if (em->start > start || em->start + em->len <= start) {
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002241 free_extent_map(em);
2242 em = NULL;
2243 }
2244 read_unlock(&em_tree->lock);
Tsutomu Itoh7a2d6a62012-10-01 03:07:15 -06002245 if (!em) {
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002246 kfree(failrec);
2247 return -EIO;
2248 }
Miao Xie2fe63032014-09-12 18:43:59 +08002249
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002250 logical = start - em->start;
2251 logical = em->block_start + logical;
2252 if (test_bit(EXTENT_FLAG_COMPRESSED, &em->flags)) {
2253 logical = em->block_start;
2254 failrec->bio_flags = EXTENT_BIO_COMPRESSED;
2255 extent_set_compress_type(&failrec->bio_flags,
2256 em->compress_type);
2257 }
Miao Xie2fe63032014-09-12 18:43:59 +08002258
2259 pr_debug("Get IO Failure Record: (new) logical=%llu, start=%llu, len=%llu\n",
2260 logical, start, failrec->len);
2261
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002262 failrec->logical = logical;
2263 free_extent_map(em);
2264
2265 /* set the bits in the private failure tree */
2266 ret = set_extent_bits(failure_tree, start, end,
2267 EXTENT_LOCKED | EXTENT_DIRTY, GFP_NOFS);
2268 if (ret >= 0)
2269 ret = set_state_private(failure_tree, start,
2270 (u64)(unsigned long)failrec);
2271 /* set the bits in the inode's tree */
2272 if (ret >= 0)
2273 ret = set_extent_bits(tree, start, end, EXTENT_DAMAGED,
2274 GFP_NOFS);
2275 if (ret < 0) {
2276 kfree(failrec);
2277 return ret;
2278 }
2279 } else {
2280 failrec = (struct io_failure_record *)(unsigned long)private;
Miao Xie2fe63032014-09-12 18:43:59 +08002281 pr_debug("Get IO Failure Record: (found) logical=%llu, start=%llu, len=%llu, validation=%d\n",
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002282 failrec->logical, failrec->start, failrec->len,
2283 failrec->in_validation);
2284 /*
2285 * when data can be on disk more than twice, add to failrec here
2286 * (e.g. with a list for failed_mirror) to make
2287 * clean_io_failure() clean all those errors at once.
2288 */
2289 }
Miao Xie2fe63032014-09-12 18:43:59 +08002290
2291 *failrec_ret = failrec;
2292
2293 return 0;
2294}
2295
2296int btrfs_check_repairable(struct inode *inode, struct bio *failed_bio,
2297 struct io_failure_record *failrec, int failed_mirror)
2298{
2299 int num_copies;
2300
Stefan Behrens5d964052012-11-05 14:59:07 +01002301 num_copies = btrfs_num_copies(BTRFS_I(inode)->root->fs_info,
2302 failrec->logical, failrec->len);
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002303 if (num_copies == 1) {
2304 /*
2305 * we only have a single copy of the data, so don't bother with
2306 * all the retry and error correction code that follows. no
2307 * matter what the error is, it is very likely to persist.
2308 */
Miao Xie2fe63032014-09-12 18:43:59 +08002309 pr_debug("Check Repairable: cannot repair, num_copies=%d, next_mirror %d, failed_mirror %d\n",
Miao Xie09a7f7a2013-07-25 19:22:32 +08002310 num_copies, failrec->this_mirror, failed_mirror);
Miao Xie2fe63032014-09-12 18:43:59 +08002311 return 0;
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002312 }
2313
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002314 /*
2315 * there are two premises:
2316 * a) deliver good data to the caller
2317 * b) correct the bad sectors on disk
2318 */
2319 if (failed_bio->bi_vcnt > 1) {
2320 /*
2321 * to fulfill b), we need to know the exact failing sectors, as
2322 * we don't want to rewrite any more than the failed ones. thus,
2323 * we need separate read requests for the failed bio
2324 *
2325 * if the following BUG_ON triggers, our validation request got
2326 * merged. we need separate requests for our algorithm to work.
2327 */
2328 BUG_ON(failrec->in_validation);
2329 failrec->in_validation = 1;
2330 failrec->this_mirror = failed_mirror;
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002331 } else {
2332 /*
2333 * we're ready to fulfill a) and b) alongside. get a good copy
2334 * of the failed sector and if we succeed, we have setup
2335 * everything for repair_io_failure to do the rest for us.
2336 */
2337 if (failrec->in_validation) {
2338 BUG_ON(failrec->this_mirror != failed_mirror);
2339 failrec->in_validation = 0;
2340 failrec->this_mirror = 0;
2341 }
2342 failrec->failed_mirror = failed_mirror;
2343 failrec->this_mirror++;
2344 if (failrec->this_mirror == failed_mirror)
2345 failrec->this_mirror++;
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002346 }
2347
Miao Xiefacc8a222013-07-25 19:22:34 +08002348 if (failrec->this_mirror > num_copies) {
Miao Xie2fe63032014-09-12 18:43:59 +08002349 pr_debug("Check Repairable: (fail) num_copies=%d, next_mirror %d, failed_mirror %d\n",
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002350 num_copies, failrec->this_mirror, failed_mirror);
Miao Xie2fe63032014-09-12 18:43:59 +08002351 return 0;
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002352 }
2353
Miao Xie2fe63032014-09-12 18:43:59 +08002354 return 1;
2355}
2356
2357
2358struct bio *btrfs_create_repair_bio(struct inode *inode, struct bio *failed_bio,
2359 struct io_failure_record *failrec,
2360 struct page *page, int pg_offset, int icsum,
Miao Xie8b110e32014-09-12 18:44:03 +08002361 bio_end_io_t *endio_func, void *data)
Miao Xie2fe63032014-09-12 18:43:59 +08002362{
2363 struct bio *bio;
2364 struct btrfs_io_bio *btrfs_failed_bio;
2365 struct btrfs_io_bio *btrfs_bio;
2366
Chris Mason9be33952013-05-17 18:30:14 -04002367 bio = btrfs_io_bio_alloc(GFP_NOFS, 1);
Miao Xie2fe63032014-09-12 18:43:59 +08002368 if (!bio)
2369 return NULL;
2370
2371 bio->bi_end_io = endio_func;
Kent Overstreet4f024f32013-10-11 15:44:27 -07002372 bio->bi_iter.bi_sector = failrec->logical >> 9;
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002373 bio->bi_bdev = BTRFS_I(inode)->root->fs_info->fs_devices->latest_bdev;
Kent Overstreet4f024f32013-10-11 15:44:27 -07002374 bio->bi_iter.bi_size = 0;
Miao Xie8b110e32014-09-12 18:44:03 +08002375 bio->bi_private = data;
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002376
Miao Xiefacc8a222013-07-25 19:22:34 +08002377 btrfs_failed_bio = btrfs_io_bio(failed_bio);
2378 if (btrfs_failed_bio->csum) {
2379 struct btrfs_fs_info *fs_info = BTRFS_I(inode)->root->fs_info;
2380 u16 csum_size = btrfs_super_csum_size(fs_info->super_copy);
2381
2382 btrfs_bio = btrfs_io_bio(bio);
2383 btrfs_bio->csum = btrfs_bio->csum_inline;
Miao Xie2fe63032014-09-12 18:43:59 +08002384 icsum *= csum_size;
2385 memcpy(btrfs_bio->csum, btrfs_failed_bio->csum + icsum,
Miao Xiefacc8a222013-07-25 19:22:34 +08002386 csum_size);
2387 }
2388
Miao Xie2fe63032014-09-12 18:43:59 +08002389 bio_add_page(bio, page, failrec->len, pg_offset);
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002390
Miao Xie2fe63032014-09-12 18:43:59 +08002391 return bio;
2392}
2393
2394/*
2395 * this is a generic handler for readpage errors (default
2396 * readpage_io_failed_hook). if other copies exist, read those and write back
2397 * good data to the failed position. does not investigate in remapping the
2398 * failed extent elsewhere, hoping the device will be smart enough to do this as
2399 * needed
2400 */
2401
2402static int bio_readpage_error(struct bio *failed_bio, u64 phy_offset,
2403 struct page *page, u64 start, u64 end,
2404 int failed_mirror)
2405{
2406 struct io_failure_record *failrec;
2407 struct inode *inode = page->mapping->host;
2408 struct extent_io_tree *tree = &BTRFS_I(inode)->io_tree;
2409 struct bio *bio;
2410 int read_mode;
2411 int ret;
2412
2413 BUG_ON(failed_bio->bi_rw & REQ_WRITE);
2414
2415 ret = btrfs_get_io_failure_record(inode, start, end, &failrec);
2416 if (ret)
2417 return ret;
2418
2419 ret = btrfs_check_repairable(inode, failed_bio, failrec, failed_mirror);
2420 if (!ret) {
2421 free_io_failure(inode, failrec);
2422 return -EIO;
2423 }
2424
2425 if (failed_bio->bi_vcnt > 1)
2426 read_mode = READ_SYNC | REQ_FAILFAST_DEV;
2427 else
2428 read_mode = READ_SYNC;
2429
2430 phy_offset >>= inode->i_sb->s_blocksize_bits;
2431 bio = btrfs_create_repair_bio(inode, failed_bio, failrec, page,
2432 start - page_offset(page),
Miao Xie8b110e32014-09-12 18:44:03 +08002433 (int)phy_offset, failed_bio->bi_end_io,
2434 NULL);
Miao Xie2fe63032014-09-12 18:43:59 +08002435 if (!bio) {
2436 free_io_failure(inode, failrec);
2437 return -EIO;
2438 }
2439
2440 pr_debug("Repair Read Error: submitting new read[%#x] to this_mirror=%d, in_validation=%d\n",
2441 read_mode, failrec->this_mirror, failrec->in_validation);
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002442
Tsutomu Itoh013bd4c2012-02-16 10:11:40 +09002443 ret = tree->ops->submit_bio_hook(inode, read_mode, bio,
2444 failrec->this_mirror,
2445 failrec->bio_flags, 0);
Miao Xie6c387ab2014-09-12 18:43:57 +08002446 if (ret) {
Miao Xie454ff3d2014-09-12 18:43:58 +08002447 free_io_failure(inode, failrec);
Miao Xie6c387ab2014-09-12 18:43:57 +08002448 bio_put(bio);
2449 }
2450
Tsutomu Itoh013bd4c2012-02-16 10:11:40 +09002451 return ret;
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002452}
2453
Chris Masond1310b22008-01-24 16:13:08 -05002454/* lots and lots of room for performance fixes in the end_bio funcs */
2455
Jeff Mahoney87826df2012-02-15 16:23:57 +01002456int end_extent_writepage(struct page *page, int err, u64 start, u64 end)
2457{
2458 int uptodate = (err == 0);
2459 struct extent_io_tree *tree;
Eric Sandeen3e2426b2014-06-12 00:39:58 -05002460 int ret = 0;
Jeff Mahoney87826df2012-02-15 16:23:57 +01002461
2462 tree = &BTRFS_I(page->mapping->host)->io_tree;
2463
2464 if (tree->ops && tree->ops->writepage_end_io_hook) {
2465 ret = tree->ops->writepage_end_io_hook(page, start,
2466 end, NULL, uptodate);
2467 if (ret)
2468 uptodate = 0;
2469 }
2470
Jeff Mahoney87826df2012-02-15 16:23:57 +01002471 if (!uptodate) {
Jeff Mahoney87826df2012-02-15 16:23:57 +01002472 ClearPageUptodate(page);
2473 SetPageError(page);
Liu Bo5dca6ee2014-05-12 12:47:36 +08002474 ret = ret < 0 ? ret : -EIO;
2475 mapping_set_error(page->mapping, ret);
Jeff Mahoney87826df2012-02-15 16:23:57 +01002476 }
2477 return 0;
2478}
2479
Chris Masond1310b22008-01-24 16:13:08 -05002480/*
2481 * after a writepage IO is done, we need to:
2482 * clear the uptodate bits on error
2483 * clear the writeback bits in the extent tree for this IO
2484 * end_page_writeback if the page has no more pending IO
2485 *
2486 * Scheduling is not allowed, so the extent state tree is expected
2487 * to have one and only one object corresponding to this IO.
2488 */
Christoph Hellwig4246a0b2015-07-20 15:29:37 +02002489static void end_bio_extent_writepage(struct bio *bio)
Chris Masond1310b22008-01-24 16:13:08 -05002490{
Kent Overstreet2c30c712013-11-07 12:20:26 -08002491 struct bio_vec *bvec;
Chris Masond1310b22008-01-24 16:13:08 -05002492 u64 start;
2493 u64 end;
Kent Overstreet2c30c712013-11-07 12:20:26 -08002494 int i;
Chris Masond1310b22008-01-24 16:13:08 -05002495
Kent Overstreet2c30c712013-11-07 12:20:26 -08002496 bio_for_each_segment_all(bvec, bio, i) {
Chris Masond1310b22008-01-24 16:13:08 -05002497 struct page *page = bvec->bv_page;
David Woodhouse902b22f2008-08-20 08:51:49 -04002498
Alexandre Oliva17a5adc2013-05-15 11:38:55 -04002499 /* We always issue full-page reads, but if some block
2500 * in a page fails to read, blk_update_request() will
2501 * advance bv_offset and adjust bv_len to compensate.
2502 * Print a warning for nonzero offsets, and an error
2503 * if they don't add up to a full page. */
Frank Holtonefe120a2013-12-20 11:37:06 -05002504 if (bvec->bv_offset || bvec->bv_len != PAGE_CACHE_SIZE) {
2505 if (bvec->bv_offset + bvec->bv_len != PAGE_CACHE_SIZE)
2506 btrfs_err(BTRFS_I(page->mapping->host)->root->fs_info,
2507 "partial page write in btrfs with offset %u and length %u",
2508 bvec->bv_offset, bvec->bv_len);
2509 else
2510 btrfs_info(BTRFS_I(page->mapping->host)->root->fs_info,
2511 "incomplete page write in btrfs with offset %u and "
2512 "length %u",
2513 bvec->bv_offset, bvec->bv_len);
2514 }
Chris Masond1310b22008-01-24 16:13:08 -05002515
Alexandre Oliva17a5adc2013-05-15 11:38:55 -04002516 start = page_offset(page);
2517 end = start + bvec->bv_offset + bvec->bv_len - 1;
Chris Masond1310b22008-01-24 16:13:08 -05002518
Christoph Hellwig4246a0b2015-07-20 15:29:37 +02002519 if (end_extent_writepage(page, bio->bi_error, start, end))
Jeff Mahoney87826df2012-02-15 16:23:57 +01002520 continue;
Chris Mason70dec802008-01-29 09:59:12 -05002521
Alexandre Oliva17a5adc2013-05-15 11:38:55 -04002522 end_page_writeback(page);
Kent Overstreet2c30c712013-11-07 12:20:26 -08002523 }
Chris Mason2b1f55b2008-09-24 11:48:04 -04002524
Chris Masond1310b22008-01-24 16:13:08 -05002525 bio_put(bio);
Chris Masond1310b22008-01-24 16:13:08 -05002526}
2527
Miao Xie883d0de2013-07-25 19:22:35 +08002528static void
2529endio_readpage_release_extent(struct extent_io_tree *tree, u64 start, u64 len,
2530 int uptodate)
2531{
2532 struct extent_state *cached = NULL;
2533 u64 end = start + len - 1;
2534
2535 if (uptodate && tree->track_uptodate)
2536 set_extent_uptodate(tree, start, end, &cached, GFP_ATOMIC);
2537 unlock_extent_cached(tree, start, end, &cached, GFP_ATOMIC);
2538}
2539
Chris Masond1310b22008-01-24 16:13:08 -05002540/*
2541 * after a readpage IO is done, we need to:
2542 * clear the uptodate bits on error
2543 * set the uptodate bits if things worked
2544 * set the page up to date if all extents in the tree are uptodate
2545 * clear the lock bit in the extent tree
2546 * unlock the page if there are no other extents locked for it
2547 *
2548 * Scheduling is not allowed, so the extent state tree is expected
2549 * to have one and only one object corresponding to this IO.
2550 */
Christoph Hellwig4246a0b2015-07-20 15:29:37 +02002551static void end_bio_extent_readpage(struct bio *bio)
Chris Masond1310b22008-01-24 16:13:08 -05002552{
Kent Overstreet2c30c712013-11-07 12:20:26 -08002553 struct bio_vec *bvec;
Christoph Hellwig4246a0b2015-07-20 15:29:37 +02002554 int uptodate = !bio->bi_error;
Miao Xiefacc8a222013-07-25 19:22:34 +08002555 struct btrfs_io_bio *io_bio = btrfs_io_bio(bio);
David Woodhouse902b22f2008-08-20 08:51:49 -04002556 struct extent_io_tree *tree;
Miao Xiefacc8a222013-07-25 19:22:34 +08002557 u64 offset = 0;
Chris Masond1310b22008-01-24 16:13:08 -05002558 u64 start;
2559 u64 end;
Miao Xiefacc8a222013-07-25 19:22:34 +08002560 u64 len;
Miao Xie883d0de2013-07-25 19:22:35 +08002561 u64 extent_start = 0;
2562 u64 extent_len = 0;
Josef Bacik5cf1ab52012-04-16 09:42:26 -04002563 int mirror;
Chris Masond1310b22008-01-24 16:13:08 -05002564 int ret;
Kent Overstreet2c30c712013-11-07 12:20:26 -08002565 int i;
Chris Masond1310b22008-01-24 16:13:08 -05002566
Kent Overstreet2c30c712013-11-07 12:20:26 -08002567 bio_for_each_segment_all(bvec, bio, i) {
Chris Masond1310b22008-01-24 16:13:08 -05002568 struct page *page = bvec->bv_page;
Josef Bacika71754f2013-06-17 17:14:39 -04002569 struct inode *inode = page->mapping->host;
Arne Jansen507903b2011-04-06 10:02:20 +00002570
Kent Overstreetbe3940c2012-09-11 14:23:05 -06002571 pr_debug("end_bio_extent_readpage: bi_sector=%llu, err=%d, "
Christoph Hellwig4246a0b2015-07-20 15:29:37 +02002572 "mirror=%u\n", (u64)bio->bi_iter.bi_sector,
2573 bio->bi_error, io_bio->mirror_num);
Josef Bacika71754f2013-06-17 17:14:39 -04002574 tree = &BTRFS_I(inode)->io_tree;
David Woodhouse902b22f2008-08-20 08:51:49 -04002575
Alexandre Oliva17a5adc2013-05-15 11:38:55 -04002576 /* We always issue full-page reads, but if some block
2577 * in a page fails to read, blk_update_request() will
2578 * advance bv_offset and adjust bv_len to compensate.
2579 * Print a warning for nonzero offsets, and an error
2580 * if they don't add up to a full page. */
Frank Holtonefe120a2013-12-20 11:37:06 -05002581 if (bvec->bv_offset || bvec->bv_len != PAGE_CACHE_SIZE) {
2582 if (bvec->bv_offset + bvec->bv_len != PAGE_CACHE_SIZE)
2583 btrfs_err(BTRFS_I(page->mapping->host)->root->fs_info,
2584 "partial page read in btrfs with offset %u and length %u",
2585 bvec->bv_offset, bvec->bv_len);
2586 else
2587 btrfs_info(BTRFS_I(page->mapping->host)->root->fs_info,
2588 "incomplete page read in btrfs with offset %u and "
2589 "length %u",
2590 bvec->bv_offset, bvec->bv_len);
2591 }
Chris Masond1310b22008-01-24 16:13:08 -05002592
Alexandre Oliva17a5adc2013-05-15 11:38:55 -04002593 start = page_offset(page);
2594 end = start + bvec->bv_offset + bvec->bv_len - 1;
Miao Xiefacc8a222013-07-25 19:22:34 +08002595 len = bvec->bv_len;
Chris Masond1310b22008-01-24 16:13:08 -05002596
Chris Mason9be33952013-05-17 18:30:14 -04002597 mirror = io_bio->mirror_num;
Miao Xief2a09da2013-07-25 19:22:33 +08002598 if (likely(uptodate && tree->ops &&
2599 tree->ops->readpage_end_io_hook)) {
Miao Xiefacc8a222013-07-25 19:22:34 +08002600 ret = tree->ops->readpage_end_io_hook(io_bio, offset,
2601 page, start, end,
2602 mirror);
Stefan Behrens5ee08442012-08-27 08:30:03 -06002603 if (ret)
Chris Masond1310b22008-01-24 16:13:08 -05002604 uptodate = 0;
Stefan Behrens5ee08442012-08-27 08:30:03 -06002605 else
Miao Xie1203b682014-09-12 18:44:01 +08002606 clean_io_failure(inode, start, page, 0);
Chris Masond1310b22008-01-24 16:13:08 -05002607 }
Josef Bacikea466792012-03-26 21:57:36 -04002608
Miao Xief2a09da2013-07-25 19:22:33 +08002609 if (likely(uptodate))
2610 goto readpage_ok;
2611
2612 if (tree->ops && tree->ops->readpage_io_failed_hook) {
Josef Bacik5cf1ab52012-04-16 09:42:26 -04002613 ret = tree->ops->readpage_io_failed_hook(page, mirror);
Christoph Hellwig4246a0b2015-07-20 15:29:37 +02002614 if (!ret && !bio->bi_error)
Josef Bacikea466792012-03-26 21:57:36 -04002615 uptodate = 1;
Miao Xief2a09da2013-07-25 19:22:33 +08002616 } else {
Jan Schmidtf4a8e652011-12-01 09:30:36 -05002617 /*
2618 * The generic bio_readpage_error handles errors the
2619 * following way: If possible, new read requests are
2620 * created and submitted and will end up in
2621 * end_bio_extent_readpage as well (if we're lucky, not
2622 * in the !uptodate case). In that case it returns 0 and
2623 * we just go on with the next page in our bio. If it
2624 * can't handle the error it will return -EIO and we
2625 * remain responsible for that page.
2626 */
Miao Xiefacc8a222013-07-25 19:22:34 +08002627 ret = bio_readpage_error(bio, offset, page, start, end,
2628 mirror);
Chris Mason7e383262008-04-09 16:28:12 -04002629 if (ret == 0) {
Christoph Hellwig4246a0b2015-07-20 15:29:37 +02002630 uptodate = !bio->bi_error;
Liu Bo38c1c2e2014-08-19 23:33:13 +08002631 offset += len;
Chris Mason7e383262008-04-09 16:28:12 -04002632 continue;
2633 }
2634 }
Miao Xief2a09da2013-07-25 19:22:33 +08002635readpage_ok:
Miao Xie883d0de2013-07-25 19:22:35 +08002636 if (likely(uptodate)) {
Josef Bacika71754f2013-06-17 17:14:39 -04002637 loff_t i_size = i_size_read(inode);
2638 pgoff_t end_index = i_size >> PAGE_CACHE_SHIFT;
Liu Boa583c022014-08-19 23:32:22 +08002639 unsigned off;
Josef Bacika71754f2013-06-17 17:14:39 -04002640
2641 /* Zero out the end if this page straddles i_size */
Liu Boa583c022014-08-19 23:32:22 +08002642 off = i_size & (PAGE_CACHE_SIZE-1);
2643 if (page->index == end_index && off)
2644 zero_user_segment(page, off, PAGE_CACHE_SIZE);
Alexandre Oliva17a5adc2013-05-15 11:38:55 -04002645 SetPageUptodate(page);
Chris Mason70dec802008-01-29 09:59:12 -05002646 } else {
Alexandre Oliva17a5adc2013-05-15 11:38:55 -04002647 ClearPageUptodate(page);
2648 SetPageError(page);
Chris Mason70dec802008-01-29 09:59:12 -05002649 }
Alexandre Oliva17a5adc2013-05-15 11:38:55 -04002650 unlock_page(page);
Miao Xiefacc8a222013-07-25 19:22:34 +08002651 offset += len;
Miao Xie883d0de2013-07-25 19:22:35 +08002652
2653 if (unlikely(!uptodate)) {
2654 if (extent_len) {
2655 endio_readpage_release_extent(tree,
2656 extent_start,
2657 extent_len, 1);
2658 extent_start = 0;
2659 extent_len = 0;
2660 }
2661 endio_readpage_release_extent(tree, start,
2662 end - start + 1, 0);
2663 } else if (!extent_len) {
2664 extent_start = start;
2665 extent_len = end + 1 - start;
2666 } else if (extent_start + extent_len == start) {
2667 extent_len += end + 1 - start;
2668 } else {
2669 endio_readpage_release_extent(tree, extent_start,
2670 extent_len, uptodate);
2671 extent_start = start;
2672 extent_len = end + 1 - start;
2673 }
Kent Overstreet2c30c712013-11-07 12:20:26 -08002674 }
Chris Masond1310b22008-01-24 16:13:08 -05002675
Miao Xie883d0de2013-07-25 19:22:35 +08002676 if (extent_len)
2677 endio_readpage_release_extent(tree, extent_start, extent_len,
2678 uptodate);
Miao Xiefacc8a222013-07-25 19:22:34 +08002679 if (io_bio->end_io)
Christoph Hellwig4246a0b2015-07-20 15:29:37 +02002680 io_bio->end_io(io_bio, bio->bi_error);
Chris Masond1310b22008-01-24 16:13:08 -05002681 bio_put(bio);
Chris Masond1310b22008-01-24 16:13:08 -05002682}
2683
Chris Mason9be33952013-05-17 18:30:14 -04002684/*
2685 * this allocates from the btrfs_bioset. We're returning a bio right now
2686 * but you can call btrfs_io_bio for the appropriate container_of magic
2687 */
Miao Xie88f794e2010-11-22 03:02:55 +00002688struct bio *
2689btrfs_bio_alloc(struct block_device *bdev, u64 first_sector, int nr_vecs,
2690 gfp_t gfp_flags)
Chris Masond1310b22008-01-24 16:13:08 -05002691{
Miao Xiefacc8a222013-07-25 19:22:34 +08002692 struct btrfs_io_bio *btrfs_bio;
Chris Masond1310b22008-01-24 16:13:08 -05002693 struct bio *bio;
2694
Chris Mason9be33952013-05-17 18:30:14 -04002695 bio = bio_alloc_bioset(gfp_flags, nr_vecs, btrfs_bioset);
Chris Masond1310b22008-01-24 16:13:08 -05002696
2697 if (bio == NULL && (current->flags & PF_MEMALLOC)) {
Chris Mason9be33952013-05-17 18:30:14 -04002698 while (!bio && (nr_vecs /= 2)) {
2699 bio = bio_alloc_bioset(gfp_flags,
2700 nr_vecs, btrfs_bioset);
2701 }
Chris Masond1310b22008-01-24 16:13:08 -05002702 }
2703
2704 if (bio) {
2705 bio->bi_bdev = bdev;
Kent Overstreet4f024f32013-10-11 15:44:27 -07002706 bio->bi_iter.bi_sector = first_sector;
Miao Xiefacc8a222013-07-25 19:22:34 +08002707 btrfs_bio = btrfs_io_bio(bio);
2708 btrfs_bio->csum = NULL;
2709 btrfs_bio->csum_allocated = NULL;
2710 btrfs_bio->end_io = NULL;
Chris Masond1310b22008-01-24 16:13:08 -05002711 }
2712 return bio;
2713}
2714
Chris Mason9be33952013-05-17 18:30:14 -04002715struct bio *btrfs_bio_clone(struct bio *bio, gfp_t gfp_mask)
2716{
Miao Xie23ea8e52014-09-12 18:43:54 +08002717 struct btrfs_io_bio *btrfs_bio;
2718 struct bio *new;
Chris Mason9be33952013-05-17 18:30:14 -04002719
Miao Xie23ea8e52014-09-12 18:43:54 +08002720 new = bio_clone_bioset(bio, gfp_mask, btrfs_bioset);
2721 if (new) {
2722 btrfs_bio = btrfs_io_bio(new);
2723 btrfs_bio->csum = NULL;
2724 btrfs_bio->csum_allocated = NULL;
2725 btrfs_bio->end_io = NULL;
Chris Mason3a9508b2015-08-21 10:05:39 -07002726
2727#ifdef CONFIG_BLK_CGROUP
Chris Masonda2f0f72015-07-02 13:57:22 -07002728 /* FIXME, put this into bio_clone_bioset */
2729 if (bio->bi_css)
2730 bio_associate_blkcg(new, bio->bi_css);
Chris Mason3a9508b2015-08-21 10:05:39 -07002731#endif
Miao Xie23ea8e52014-09-12 18:43:54 +08002732 }
2733 return new;
2734}
Chris Mason9be33952013-05-17 18:30:14 -04002735
2736/* this also allocates from the btrfs_bioset */
2737struct bio *btrfs_io_bio_alloc(gfp_t gfp_mask, unsigned int nr_iovecs)
2738{
Miao Xiefacc8a222013-07-25 19:22:34 +08002739 struct btrfs_io_bio *btrfs_bio;
2740 struct bio *bio;
2741
2742 bio = bio_alloc_bioset(gfp_mask, nr_iovecs, btrfs_bioset);
2743 if (bio) {
2744 btrfs_bio = btrfs_io_bio(bio);
2745 btrfs_bio->csum = NULL;
2746 btrfs_bio->csum_allocated = NULL;
2747 btrfs_bio->end_io = NULL;
2748 }
2749 return bio;
Chris Mason9be33952013-05-17 18:30:14 -04002750}
2751
2752
Jeff Mahoney355808c2011-10-03 23:23:14 -04002753static int __must_check submit_one_bio(int rw, struct bio *bio,
2754 int mirror_num, unsigned long bio_flags)
Chris Masond1310b22008-01-24 16:13:08 -05002755{
Chris Masond1310b22008-01-24 16:13:08 -05002756 int ret = 0;
Chris Mason70dec802008-01-29 09:59:12 -05002757 struct bio_vec *bvec = bio->bi_io_vec + bio->bi_vcnt - 1;
2758 struct page *page = bvec->bv_page;
2759 struct extent_io_tree *tree = bio->bi_private;
Chris Mason70dec802008-01-29 09:59:12 -05002760 u64 start;
Chris Mason70dec802008-01-29 09:59:12 -05002761
Miao Xie4eee4fa2012-12-21 09:17:45 +00002762 start = page_offset(page) + bvec->bv_offset;
Chris Mason70dec802008-01-29 09:59:12 -05002763
David Woodhouse902b22f2008-08-20 08:51:49 -04002764 bio->bi_private = NULL;
Chris Masond1310b22008-01-24 16:13:08 -05002765
2766 bio_get(bio);
2767
Chris Mason065631f2008-02-20 12:07:25 -05002768 if (tree->ops && tree->ops->submit_bio_hook)
liubo6b82ce82011-01-26 06:21:39 +00002769 ret = tree->ops->submit_bio_hook(page->mapping->host, rw, bio,
Chris Masoneaf25d92010-05-25 09:48:28 -04002770 mirror_num, bio_flags, start);
Chris Mason0b86a832008-03-24 15:01:56 -04002771 else
Stefan Behrens21adbd52011-11-09 13:44:05 +01002772 btrfsic_submit_bio(rw, bio);
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002773
Chris Masond1310b22008-01-24 16:13:08 -05002774 bio_put(bio);
2775 return ret;
2776}
2777
David Woodhouse64a16702009-07-15 23:29:37 +01002778static int merge_bio(int rw, struct extent_io_tree *tree, struct page *page,
Jeff Mahoney3444a972011-10-03 23:23:13 -04002779 unsigned long offset, size_t size, struct bio *bio,
2780 unsigned long bio_flags)
2781{
2782 int ret = 0;
2783 if (tree->ops && tree->ops->merge_bio_hook)
David Woodhouse64a16702009-07-15 23:29:37 +01002784 ret = tree->ops->merge_bio_hook(rw, page, offset, size, bio,
Jeff Mahoney3444a972011-10-03 23:23:13 -04002785 bio_flags);
2786 BUG_ON(ret < 0);
2787 return ret;
2788
2789}
2790
Chris Masond1310b22008-01-24 16:13:08 -05002791static int submit_extent_page(int rw, struct extent_io_tree *tree,
Chris Masonda2f0f72015-07-02 13:57:22 -07002792 struct writeback_control *wbc,
Chris Masond1310b22008-01-24 16:13:08 -05002793 struct page *page, sector_t sector,
2794 size_t size, unsigned long offset,
2795 struct block_device *bdev,
2796 struct bio **bio_ret,
2797 unsigned long max_pages,
Chris Masonf1885912008-04-09 16:28:12 -04002798 bio_end_io_t end_io_func,
Chris Masonc8b97812008-10-29 14:49:59 -04002799 int mirror_num,
2800 unsigned long prev_bio_flags,
Filipe Manana005efed2015-09-14 09:09:31 +01002801 unsigned long bio_flags,
2802 bool force_bio_submit)
Chris Masond1310b22008-01-24 16:13:08 -05002803{
2804 int ret = 0;
2805 struct bio *bio;
Chris Masonc8b97812008-10-29 14:49:59 -04002806 int contig = 0;
Chris Masonc8b97812008-10-29 14:49:59 -04002807 int old_compressed = prev_bio_flags & EXTENT_BIO_COMPRESSED;
Chris Mason5b050f02008-11-11 09:34:41 -05002808 size_t page_size = min_t(size_t, size, PAGE_CACHE_SIZE);
Chris Masond1310b22008-01-24 16:13:08 -05002809
2810 if (bio_ret && *bio_ret) {
2811 bio = *bio_ret;
Chris Masonc8b97812008-10-29 14:49:59 -04002812 if (old_compressed)
Kent Overstreet4f024f32013-10-11 15:44:27 -07002813 contig = bio->bi_iter.bi_sector == sector;
Chris Masonc8b97812008-10-29 14:49:59 -04002814 else
Kent Overstreetf73a1c72012-09-25 15:05:12 -07002815 contig = bio_end_sector(bio) == sector;
Chris Masonc8b97812008-10-29 14:49:59 -04002816
2817 if (prev_bio_flags != bio_flags || !contig ||
Filipe Manana005efed2015-09-14 09:09:31 +01002818 force_bio_submit ||
David Woodhouse64a16702009-07-15 23:29:37 +01002819 merge_bio(rw, tree, page, offset, page_size, bio, bio_flags) ||
Chris Masonc8b97812008-10-29 14:49:59 -04002820 bio_add_page(bio, page, page_size, offset) < page_size) {
2821 ret = submit_one_bio(rw, bio, mirror_num,
2822 prev_bio_flags);
Naohiro Aota289454a2015-01-06 01:01:03 +09002823 if (ret < 0) {
2824 *bio_ret = NULL;
Jeff Mahoney79787ea2012-03-12 16:03:00 +01002825 return ret;
Naohiro Aota289454a2015-01-06 01:01:03 +09002826 }
Chris Masond1310b22008-01-24 16:13:08 -05002827 bio = NULL;
2828 } else {
Chris Masonda2f0f72015-07-02 13:57:22 -07002829 if (wbc)
2830 wbc_account_io(wbc, page, page_size);
Chris Masond1310b22008-01-24 16:13:08 -05002831 return 0;
2832 }
2833 }
Chris Masonc8b97812008-10-29 14:49:59 -04002834
Kent Overstreetb54ffb72015-05-19 14:31:01 +02002835 bio = btrfs_bio_alloc(bdev, sector, BIO_MAX_PAGES,
2836 GFP_NOFS | __GFP_HIGH);
Tsutomu Itoh5df67082011-02-01 09:17:35 +00002837 if (!bio)
2838 return -ENOMEM;
Chris Mason70dec802008-01-29 09:59:12 -05002839
Chris Masonc8b97812008-10-29 14:49:59 -04002840 bio_add_page(bio, page, page_size, offset);
Chris Masond1310b22008-01-24 16:13:08 -05002841 bio->bi_end_io = end_io_func;
2842 bio->bi_private = tree;
Chris Masonda2f0f72015-07-02 13:57:22 -07002843 if (wbc) {
2844 wbc_init_bio(wbc, bio);
2845 wbc_account_io(wbc, page, page_size);
2846 }
Chris Mason70dec802008-01-29 09:59:12 -05002847
Chris Masond3977122009-01-05 21:25:51 -05002848 if (bio_ret)
Chris Masond1310b22008-01-24 16:13:08 -05002849 *bio_ret = bio;
Chris Masond3977122009-01-05 21:25:51 -05002850 else
Chris Masonc8b97812008-10-29 14:49:59 -04002851 ret = submit_one_bio(rw, bio, mirror_num, bio_flags);
Chris Masond1310b22008-01-24 16:13:08 -05002852
2853 return ret;
2854}
2855
Eric Sandeen48a3b632013-04-25 20:41:01 +00002856static void attach_extent_buffer_page(struct extent_buffer *eb,
2857 struct page *page)
Josef Bacik4f2de97a2012-03-07 16:20:05 -05002858{
2859 if (!PagePrivate(page)) {
2860 SetPagePrivate(page);
2861 page_cache_get(page);
2862 set_page_private(page, (unsigned long)eb);
2863 } else {
2864 WARN_ON(page->private != (unsigned long)eb);
2865 }
2866}
2867
Chris Masond1310b22008-01-24 16:13:08 -05002868void set_page_extent_mapped(struct page *page)
2869{
2870 if (!PagePrivate(page)) {
2871 SetPagePrivate(page);
Chris Masond1310b22008-01-24 16:13:08 -05002872 page_cache_get(page);
Chris Mason6af118ce2008-07-22 11:18:07 -04002873 set_page_private(page, EXTENT_PAGE_PRIVATE);
Chris Masond1310b22008-01-24 16:13:08 -05002874 }
2875}
2876
Miao Xie125bac012013-07-25 19:22:37 +08002877static struct extent_map *
2878__get_extent_map(struct inode *inode, struct page *page, size_t pg_offset,
2879 u64 start, u64 len, get_extent_t *get_extent,
2880 struct extent_map **em_cached)
2881{
2882 struct extent_map *em;
2883
2884 if (em_cached && *em_cached) {
2885 em = *em_cached;
Filipe Mananacbc0e922014-02-25 14:15:12 +00002886 if (extent_map_in_tree(em) && start >= em->start &&
Miao Xie125bac012013-07-25 19:22:37 +08002887 start < extent_map_end(em)) {
2888 atomic_inc(&em->refs);
2889 return em;
2890 }
2891
2892 free_extent_map(em);
2893 *em_cached = NULL;
2894 }
2895
2896 em = get_extent(inode, page, pg_offset, start, len, 0);
2897 if (em_cached && !IS_ERR_OR_NULL(em)) {
2898 BUG_ON(*em_cached);
2899 atomic_inc(&em->refs);
2900 *em_cached = em;
2901 }
2902 return em;
2903}
Chris Masond1310b22008-01-24 16:13:08 -05002904/*
2905 * basic readpage implementation. Locked extent state structs are inserted
2906 * into the tree that are removed when the IO is done (by the end_io
2907 * handlers)
Jeff Mahoney79787ea2012-03-12 16:03:00 +01002908 * XXX JDM: This needs looking at to ensure proper page locking
Chris Masond1310b22008-01-24 16:13:08 -05002909 */
Miao Xie99740902013-07-25 19:22:36 +08002910static int __do_readpage(struct extent_io_tree *tree,
2911 struct page *page,
2912 get_extent_t *get_extent,
Miao Xie125bac012013-07-25 19:22:37 +08002913 struct extent_map **em_cached,
Miao Xie99740902013-07-25 19:22:36 +08002914 struct bio **bio, int mirror_num,
Filipe Manana005efed2015-09-14 09:09:31 +01002915 unsigned long *bio_flags, int rw,
2916 u64 *prev_em_start)
Chris Masond1310b22008-01-24 16:13:08 -05002917{
2918 struct inode *inode = page->mapping->host;
Miao Xie4eee4fa2012-12-21 09:17:45 +00002919 u64 start = page_offset(page);
Chris Masond1310b22008-01-24 16:13:08 -05002920 u64 page_end = start + PAGE_CACHE_SIZE - 1;
2921 u64 end;
2922 u64 cur = start;
2923 u64 extent_offset;
2924 u64 last_byte = i_size_read(inode);
2925 u64 block_start;
2926 u64 cur_end;
2927 sector_t sector;
2928 struct extent_map *em;
2929 struct block_device *bdev;
2930 int ret;
2931 int nr = 0;
Mark Fasheh4b384312013-08-06 11:42:50 -07002932 int parent_locked = *bio_flags & EXTENT_BIO_PARENT_LOCKED;
David Sterba306e16c2011-04-19 14:29:38 +02002933 size_t pg_offset = 0;
Chris Masond1310b22008-01-24 16:13:08 -05002934 size_t iosize;
Chris Masonc8b97812008-10-29 14:49:59 -04002935 size_t disk_io_size;
Chris Masond1310b22008-01-24 16:13:08 -05002936 size_t blocksize = inode->i_sb->s_blocksize;
Mark Fasheh4b384312013-08-06 11:42:50 -07002937 unsigned long this_bio_flag = *bio_flags & EXTENT_BIO_PARENT_LOCKED;
Chris Masond1310b22008-01-24 16:13:08 -05002938
2939 set_page_extent_mapped(page);
2940
Miao Xie99740902013-07-25 19:22:36 +08002941 end = page_end;
Dan Magenheimer90a887c2011-05-26 10:01:56 -06002942 if (!PageUptodate(page)) {
2943 if (cleancache_get_page(page) == 0) {
2944 BUG_ON(blocksize != PAGE_SIZE);
Miao Xie99740902013-07-25 19:22:36 +08002945 unlock_extent(tree, start, end);
Dan Magenheimer90a887c2011-05-26 10:01:56 -06002946 goto out;
2947 }
2948 }
2949
Chris Masonc8b97812008-10-29 14:49:59 -04002950 if (page->index == last_byte >> PAGE_CACHE_SHIFT) {
2951 char *userpage;
2952 size_t zero_offset = last_byte & (PAGE_CACHE_SIZE - 1);
2953
2954 if (zero_offset) {
2955 iosize = PAGE_CACHE_SIZE - zero_offset;
Cong Wang7ac687d2011-11-25 23:14:28 +08002956 userpage = kmap_atomic(page);
Chris Masonc8b97812008-10-29 14:49:59 -04002957 memset(userpage + zero_offset, 0, iosize);
2958 flush_dcache_page(page);
Cong Wang7ac687d2011-11-25 23:14:28 +08002959 kunmap_atomic(userpage);
Chris Masonc8b97812008-10-29 14:49:59 -04002960 }
2961 }
Chris Masond1310b22008-01-24 16:13:08 -05002962 while (cur <= end) {
Josef Bacikc8f2f242013-02-11 11:33:00 -05002963 unsigned long pnr = (last_byte >> PAGE_CACHE_SHIFT) + 1;
Filipe Manana005efed2015-09-14 09:09:31 +01002964 bool force_bio_submit = false;
Josef Bacikc8f2f242013-02-11 11:33:00 -05002965
Chris Masond1310b22008-01-24 16:13:08 -05002966 if (cur >= last_byte) {
2967 char *userpage;
Arne Jansen507903b2011-04-06 10:02:20 +00002968 struct extent_state *cached = NULL;
2969
David Sterba306e16c2011-04-19 14:29:38 +02002970 iosize = PAGE_CACHE_SIZE - pg_offset;
Cong Wang7ac687d2011-11-25 23:14:28 +08002971 userpage = kmap_atomic(page);
David Sterba306e16c2011-04-19 14:29:38 +02002972 memset(userpage + pg_offset, 0, iosize);
Chris Masond1310b22008-01-24 16:13:08 -05002973 flush_dcache_page(page);
Cong Wang7ac687d2011-11-25 23:14:28 +08002974 kunmap_atomic(userpage);
Chris Masond1310b22008-01-24 16:13:08 -05002975 set_extent_uptodate(tree, cur, cur + iosize - 1,
Arne Jansen507903b2011-04-06 10:02:20 +00002976 &cached, GFP_NOFS);
Mark Fasheh4b384312013-08-06 11:42:50 -07002977 if (!parent_locked)
2978 unlock_extent_cached(tree, cur,
2979 cur + iosize - 1,
2980 &cached, GFP_NOFS);
Chris Masond1310b22008-01-24 16:13:08 -05002981 break;
2982 }
Miao Xie125bac012013-07-25 19:22:37 +08002983 em = __get_extent_map(inode, page, pg_offset, cur,
2984 end - cur + 1, get_extent, em_cached);
David Sterbac7040052011-04-19 18:00:01 +02002985 if (IS_ERR_OR_NULL(em)) {
Chris Masond1310b22008-01-24 16:13:08 -05002986 SetPageError(page);
Mark Fasheh4b384312013-08-06 11:42:50 -07002987 if (!parent_locked)
2988 unlock_extent(tree, cur, end);
Chris Masond1310b22008-01-24 16:13:08 -05002989 break;
2990 }
Chris Masond1310b22008-01-24 16:13:08 -05002991 extent_offset = cur - em->start;
2992 BUG_ON(extent_map_end(em) <= cur);
2993 BUG_ON(end < cur);
2994
Li Zefan261507a02010-12-17 14:21:50 +08002995 if (test_bit(EXTENT_FLAG_COMPRESSED, &em->flags)) {
Mark Fasheh4b384312013-08-06 11:42:50 -07002996 this_bio_flag |= EXTENT_BIO_COMPRESSED;
Li Zefan261507a02010-12-17 14:21:50 +08002997 extent_set_compress_type(&this_bio_flag,
2998 em->compress_type);
2999 }
Chris Masonc8b97812008-10-29 14:49:59 -04003000
Chris Masond1310b22008-01-24 16:13:08 -05003001 iosize = min(extent_map_end(em) - cur, end - cur + 1);
3002 cur_end = min(extent_map_end(em) - 1, end);
Qu Wenruofda28322013-02-26 08:10:22 +00003003 iosize = ALIGN(iosize, blocksize);
Chris Masonc8b97812008-10-29 14:49:59 -04003004 if (this_bio_flag & EXTENT_BIO_COMPRESSED) {
3005 disk_io_size = em->block_len;
3006 sector = em->block_start >> 9;
3007 } else {
3008 sector = (em->block_start + extent_offset) >> 9;
3009 disk_io_size = iosize;
3010 }
Chris Masond1310b22008-01-24 16:13:08 -05003011 bdev = em->bdev;
3012 block_start = em->block_start;
Yan Zhengd899e052008-10-30 14:25:28 -04003013 if (test_bit(EXTENT_FLAG_PREALLOC, &em->flags))
3014 block_start = EXTENT_MAP_HOLE;
Filipe Manana005efed2015-09-14 09:09:31 +01003015
3016 /*
3017 * If we have a file range that points to a compressed extent
3018 * and it's followed by a consecutive file range that points to
3019 * to the same compressed extent (possibly with a different
3020 * offset and/or length, so it either points to the whole extent
3021 * or only part of it), we must make sure we do not submit a
3022 * single bio to populate the pages for the 2 ranges because
3023 * this makes the compressed extent read zero out the pages
3024 * belonging to the 2nd range. Imagine the following scenario:
3025 *
3026 * File layout
3027 * [0 - 8K] [8K - 24K]
3028 * | |
3029 * | |
3030 * points to extent X, points to extent X,
3031 * offset 4K, length of 8K offset 0, length 16K
3032 *
3033 * [extent X, compressed length = 4K uncompressed length = 16K]
3034 *
3035 * If the bio to read the compressed extent covers both ranges,
3036 * it will decompress extent X into the pages belonging to the
3037 * first range and then it will stop, zeroing out the remaining
3038 * pages that belong to the other range that points to extent X.
3039 * So here we make sure we submit 2 bios, one for the first
3040 * range and another one for the third range. Both will target
3041 * the same physical extent from disk, but we can't currently
3042 * make the compressed bio endio callback populate the pages
3043 * for both ranges because each compressed bio is tightly
3044 * coupled with a single extent map, and each range can have
3045 * an extent map with a different offset value relative to the
3046 * uncompressed data of our extent and different lengths. This
3047 * is a corner case so we prioritize correctness over
3048 * non-optimal behavior (submitting 2 bios for the same extent).
3049 */
3050 if (test_bit(EXTENT_FLAG_COMPRESSED, &em->flags) &&
3051 prev_em_start && *prev_em_start != (u64)-1 &&
3052 *prev_em_start != em->orig_start)
3053 force_bio_submit = true;
3054
3055 if (prev_em_start)
3056 *prev_em_start = em->orig_start;
3057
Chris Masond1310b22008-01-24 16:13:08 -05003058 free_extent_map(em);
3059 em = NULL;
3060
3061 /* we've found a hole, just zero and go on */
3062 if (block_start == EXTENT_MAP_HOLE) {
3063 char *userpage;
Arne Jansen507903b2011-04-06 10:02:20 +00003064 struct extent_state *cached = NULL;
3065
Cong Wang7ac687d2011-11-25 23:14:28 +08003066 userpage = kmap_atomic(page);
David Sterba306e16c2011-04-19 14:29:38 +02003067 memset(userpage + pg_offset, 0, iosize);
Chris Masond1310b22008-01-24 16:13:08 -05003068 flush_dcache_page(page);
Cong Wang7ac687d2011-11-25 23:14:28 +08003069 kunmap_atomic(userpage);
Chris Masond1310b22008-01-24 16:13:08 -05003070
3071 set_extent_uptodate(tree, cur, cur + iosize - 1,
Arne Jansen507903b2011-04-06 10:02:20 +00003072 &cached, GFP_NOFS);
3073 unlock_extent_cached(tree, cur, cur + iosize - 1,
3074 &cached, GFP_NOFS);
Chris Masond1310b22008-01-24 16:13:08 -05003075 cur = cur + iosize;
David Sterba306e16c2011-04-19 14:29:38 +02003076 pg_offset += iosize;
Chris Masond1310b22008-01-24 16:13:08 -05003077 continue;
3078 }
3079 /* the get_extent function already copied into the page */
Chris Mason9655d292009-09-02 15:22:30 -04003080 if (test_range_bit(tree, cur, cur_end,
3081 EXTENT_UPTODATE, 1, NULL)) {
Chris Masona1b32a52008-09-05 16:09:51 -04003082 check_page_uptodate(tree, page);
Mark Fasheh4b384312013-08-06 11:42:50 -07003083 if (!parent_locked)
3084 unlock_extent(tree, cur, cur + iosize - 1);
Chris Masond1310b22008-01-24 16:13:08 -05003085 cur = cur + iosize;
David Sterba306e16c2011-04-19 14:29:38 +02003086 pg_offset += iosize;
Chris Masond1310b22008-01-24 16:13:08 -05003087 continue;
3088 }
Chris Mason70dec802008-01-29 09:59:12 -05003089 /* we have an inline extent but it didn't get marked up
3090 * to date. Error out
3091 */
3092 if (block_start == EXTENT_MAP_INLINE) {
3093 SetPageError(page);
Mark Fasheh4b384312013-08-06 11:42:50 -07003094 if (!parent_locked)
3095 unlock_extent(tree, cur, cur + iosize - 1);
Chris Mason70dec802008-01-29 09:59:12 -05003096 cur = cur + iosize;
David Sterba306e16c2011-04-19 14:29:38 +02003097 pg_offset += iosize;
Chris Mason70dec802008-01-29 09:59:12 -05003098 continue;
3099 }
Chris Masond1310b22008-01-24 16:13:08 -05003100
Josef Bacikc8f2f242013-02-11 11:33:00 -05003101 pnr -= page->index;
Chris Masonda2f0f72015-07-02 13:57:22 -07003102 ret = submit_extent_page(rw, tree, NULL, page,
David Sterba306e16c2011-04-19 14:29:38 +02003103 sector, disk_io_size, pg_offset,
Chris Mason89642222008-07-24 09:41:53 -04003104 bdev, bio, pnr,
Chris Masonc8b97812008-10-29 14:49:59 -04003105 end_bio_extent_readpage, mirror_num,
3106 *bio_flags,
Filipe Manana005efed2015-09-14 09:09:31 +01003107 this_bio_flag,
3108 force_bio_submit);
Josef Bacikc8f2f242013-02-11 11:33:00 -05003109 if (!ret) {
3110 nr++;
3111 *bio_flags = this_bio_flag;
3112 } else {
Chris Masond1310b22008-01-24 16:13:08 -05003113 SetPageError(page);
Mark Fasheh4b384312013-08-06 11:42:50 -07003114 if (!parent_locked)
3115 unlock_extent(tree, cur, cur + iosize - 1);
Josef Bacikedd33c92012-10-05 16:40:32 -04003116 }
Chris Masond1310b22008-01-24 16:13:08 -05003117 cur = cur + iosize;
David Sterba306e16c2011-04-19 14:29:38 +02003118 pg_offset += iosize;
Chris Masond1310b22008-01-24 16:13:08 -05003119 }
Dan Magenheimer90a887c2011-05-26 10:01:56 -06003120out:
Chris Masond1310b22008-01-24 16:13:08 -05003121 if (!nr) {
3122 if (!PageError(page))
3123 SetPageUptodate(page);
3124 unlock_page(page);
3125 }
3126 return 0;
3127}
3128
Miao Xie99740902013-07-25 19:22:36 +08003129static inline void __do_contiguous_readpages(struct extent_io_tree *tree,
3130 struct page *pages[], int nr_pages,
3131 u64 start, u64 end,
3132 get_extent_t *get_extent,
Miao Xie125bac012013-07-25 19:22:37 +08003133 struct extent_map **em_cached,
Miao Xie99740902013-07-25 19:22:36 +08003134 struct bio **bio, int mirror_num,
Filipe Manana808f80b2015-09-28 09:56:26 +01003135 unsigned long *bio_flags, int rw,
3136 u64 *prev_em_start)
Miao Xie99740902013-07-25 19:22:36 +08003137{
3138 struct inode *inode;
3139 struct btrfs_ordered_extent *ordered;
3140 int index;
3141
3142 inode = pages[0]->mapping->host;
3143 while (1) {
3144 lock_extent(tree, start, end);
3145 ordered = btrfs_lookup_ordered_range(inode, start,
3146 end - start + 1);
3147 if (!ordered)
3148 break;
3149 unlock_extent(tree, start, end);
3150 btrfs_start_ordered_extent(inode, ordered, 1);
3151 btrfs_put_ordered_extent(ordered);
3152 }
3153
3154 for (index = 0; index < nr_pages; index++) {
Miao Xie125bac012013-07-25 19:22:37 +08003155 __do_readpage(tree, pages[index], get_extent, em_cached, bio,
Filipe Manana808f80b2015-09-28 09:56:26 +01003156 mirror_num, bio_flags, rw, prev_em_start);
Miao Xie99740902013-07-25 19:22:36 +08003157 page_cache_release(pages[index]);
3158 }
3159}
3160
3161static void __extent_readpages(struct extent_io_tree *tree,
3162 struct page *pages[],
3163 int nr_pages, get_extent_t *get_extent,
Miao Xie125bac012013-07-25 19:22:37 +08003164 struct extent_map **em_cached,
Miao Xie99740902013-07-25 19:22:36 +08003165 struct bio **bio, int mirror_num,
Filipe Manana808f80b2015-09-28 09:56:26 +01003166 unsigned long *bio_flags, int rw,
3167 u64 *prev_em_start)
Miao Xie99740902013-07-25 19:22:36 +08003168{
Stefan Behrens35a36212013-08-14 18:12:25 +02003169 u64 start = 0;
Miao Xie99740902013-07-25 19:22:36 +08003170 u64 end = 0;
3171 u64 page_start;
3172 int index;
Stefan Behrens35a36212013-08-14 18:12:25 +02003173 int first_index = 0;
Miao Xie99740902013-07-25 19:22:36 +08003174
3175 for (index = 0; index < nr_pages; index++) {
3176 page_start = page_offset(pages[index]);
3177 if (!end) {
3178 start = page_start;
3179 end = start + PAGE_CACHE_SIZE - 1;
3180 first_index = index;
3181 } else if (end + 1 == page_start) {
3182 end += PAGE_CACHE_SIZE;
3183 } else {
3184 __do_contiguous_readpages(tree, &pages[first_index],
3185 index - first_index, start,
Miao Xie125bac012013-07-25 19:22:37 +08003186 end, get_extent, em_cached,
3187 bio, mirror_num, bio_flags,
Filipe Manana808f80b2015-09-28 09:56:26 +01003188 rw, prev_em_start);
Miao Xie99740902013-07-25 19:22:36 +08003189 start = page_start;
3190 end = start + PAGE_CACHE_SIZE - 1;
3191 first_index = index;
3192 }
3193 }
3194
3195 if (end)
3196 __do_contiguous_readpages(tree, &pages[first_index],
3197 index - first_index, start,
Miao Xie125bac012013-07-25 19:22:37 +08003198 end, get_extent, em_cached, bio,
Filipe Manana808f80b2015-09-28 09:56:26 +01003199 mirror_num, bio_flags, rw,
3200 prev_em_start);
Miao Xie99740902013-07-25 19:22:36 +08003201}
3202
3203static int __extent_read_full_page(struct extent_io_tree *tree,
3204 struct page *page,
3205 get_extent_t *get_extent,
3206 struct bio **bio, int mirror_num,
3207 unsigned long *bio_flags, int rw)
3208{
3209 struct inode *inode = page->mapping->host;
3210 struct btrfs_ordered_extent *ordered;
3211 u64 start = page_offset(page);
3212 u64 end = start + PAGE_CACHE_SIZE - 1;
3213 int ret;
3214
3215 while (1) {
3216 lock_extent(tree, start, end);
3217 ordered = btrfs_lookup_ordered_extent(inode, start);
3218 if (!ordered)
3219 break;
3220 unlock_extent(tree, start, end);
3221 btrfs_start_ordered_extent(inode, ordered, 1);
3222 btrfs_put_ordered_extent(ordered);
3223 }
3224
Miao Xie125bac012013-07-25 19:22:37 +08003225 ret = __do_readpage(tree, page, get_extent, NULL, bio, mirror_num,
Filipe Manana005efed2015-09-14 09:09:31 +01003226 bio_flags, rw, NULL);
Miao Xie99740902013-07-25 19:22:36 +08003227 return ret;
3228}
3229
Chris Masond1310b22008-01-24 16:13:08 -05003230int extent_read_full_page(struct extent_io_tree *tree, struct page *page,
Jan Schmidt8ddc7d92011-06-13 20:02:58 +02003231 get_extent_t *get_extent, int mirror_num)
Chris Masond1310b22008-01-24 16:13:08 -05003232{
3233 struct bio *bio = NULL;
Chris Masonc8b97812008-10-29 14:49:59 -04003234 unsigned long bio_flags = 0;
Chris Masond1310b22008-01-24 16:13:08 -05003235 int ret;
3236
Jan Schmidt8ddc7d92011-06-13 20:02:58 +02003237 ret = __extent_read_full_page(tree, page, get_extent, &bio, mirror_num,
Josef Bacikd4c7ca82013-04-19 19:49:09 -04003238 &bio_flags, READ);
Chris Masond1310b22008-01-24 16:13:08 -05003239 if (bio)
Jan Schmidt8ddc7d92011-06-13 20:02:58 +02003240 ret = submit_one_bio(READ, bio, mirror_num, bio_flags);
Chris Masond1310b22008-01-24 16:13:08 -05003241 return ret;
3242}
Chris Masond1310b22008-01-24 16:13:08 -05003243
Mark Fasheh4b384312013-08-06 11:42:50 -07003244int extent_read_full_page_nolock(struct extent_io_tree *tree, struct page *page,
3245 get_extent_t *get_extent, int mirror_num)
3246{
3247 struct bio *bio = NULL;
3248 unsigned long bio_flags = EXTENT_BIO_PARENT_LOCKED;
3249 int ret;
3250
3251 ret = __do_readpage(tree, page, get_extent, NULL, &bio, mirror_num,
Filipe Manana005efed2015-09-14 09:09:31 +01003252 &bio_flags, READ, NULL);
Mark Fasheh4b384312013-08-06 11:42:50 -07003253 if (bio)
3254 ret = submit_one_bio(READ, bio, mirror_num, bio_flags);
3255 return ret;
3256}
3257
Chris Mason11c83492009-04-20 15:50:09 -04003258static noinline void update_nr_written(struct page *page,
3259 struct writeback_control *wbc,
3260 unsigned long nr_written)
3261{
3262 wbc->nr_to_write -= nr_written;
3263 if (wbc->range_cyclic || (wbc->nr_to_write > 0 &&
3264 wbc->range_start == 0 && wbc->range_end == LLONG_MAX))
3265 page->mapping->writeback_index = page->index + nr_written;
3266}
3267
Chris Masond1310b22008-01-24 16:13:08 -05003268/*
Chris Mason40f76582014-05-21 13:35:51 -07003269 * helper for __extent_writepage, doing all of the delayed allocation setup.
3270 *
3271 * This returns 1 if our fill_delalloc function did all the work required
3272 * to write the page (copy into inline extent). In this case the IO has
3273 * been started and the page is already unlocked.
3274 *
3275 * This returns 0 if all went well (page still locked)
3276 * This returns < 0 if there were errors (page still locked)
Chris Masond1310b22008-01-24 16:13:08 -05003277 */
Chris Mason40f76582014-05-21 13:35:51 -07003278static noinline_for_stack int writepage_delalloc(struct inode *inode,
3279 struct page *page, struct writeback_control *wbc,
3280 struct extent_page_data *epd,
3281 u64 delalloc_start,
3282 unsigned long *nr_written)
Chris Masond1310b22008-01-24 16:13:08 -05003283{
Chris Mason40f76582014-05-21 13:35:51 -07003284 struct extent_io_tree *tree = epd->tree;
3285 u64 page_end = delalloc_start + PAGE_CACHE_SIZE - 1;
3286 u64 nr_delalloc;
3287 u64 delalloc_to_write = 0;
3288 u64 delalloc_end = 0;
3289 int ret;
3290 int page_started = 0;
3291
3292 if (epd->extent_locked || !tree->ops || !tree->ops->fill_delalloc)
3293 return 0;
3294
3295 while (delalloc_end < page_end) {
3296 nr_delalloc = find_lock_delalloc_range(inode, tree,
3297 page,
3298 &delalloc_start,
3299 &delalloc_end,
Josef Bacikdcab6a32015-02-11 15:08:59 -05003300 BTRFS_MAX_EXTENT_SIZE);
Chris Mason40f76582014-05-21 13:35:51 -07003301 if (nr_delalloc == 0) {
3302 delalloc_start = delalloc_end + 1;
3303 continue;
3304 }
3305 ret = tree->ops->fill_delalloc(inode, page,
3306 delalloc_start,
3307 delalloc_end,
3308 &page_started,
3309 nr_written);
3310 /* File system has been set read-only */
3311 if (ret) {
3312 SetPageError(page);
3313 /* fill_delalloc should be return < 0 for error
3314 * but just in case, we use > 0 here meaning the
3315 * IO is started, so we don't want to return > 0
3316 * unless things are going well.
3317 */
3318 ret = ret < 0 ? ret : -EIO;
3319 goto done;
3320 }
3321 /*
3322 * delalloc_end is already one less than the total
3323 * length, so we don't subtract one from
3324 * PAGE_CACHE_SIZE
3325 */
3326 delalloc_to_write += (delalloc_end - delalloc_start +
3327 PAGE_CACHE_SIZE) >>
3328 PAGE_CACHE_SHIFT;
3329 delalloc_start = delalloc_end + 1;
3330 }
3331 if (wbc->nr_to_write < delalloc_to_write) {
3332 int thresh = 8192;
3333
3334 if (delalloc_to_write < thresh * 2)
3335 thresh = delalloc_to_write;
3336 wbc->nr_to_write = min_t(u64, delalloc_to_write,
3337 thresh);
3338 }
3339
3340 /* did the fill delalloc function already unlock and start
3341 * the IO?
3342 */
3343 if (page_started) {
3344 /*
3345 * we've unlocked the page, so we can't update
3346 * the mapping's writeback index, just update
3347 * nr_to_write.
3348 */
3349 wbc->nr_to_write -= *nr_written;
3350 return 1;
3351 }
3352
3353 ret = 0;
3354
3355done:
3356 return ret;
3357}
3358
3359/*
3360 * helper for __extent_writepage. This calls the writepage start hooks,
3361 * and does the loop to map the page into extents and bios.
3362 *
3363 * We return 1 if the IO is started and the page is unlocked,
3364 * 0 if all went well (page still locked)
3365 * < 0 if there were errors (page still locked)
3366 */
3367static noinline_for_stack int __extent_writepage_io(struct inode *inode,
3368 struct page *page,
3369 struct writeback_control *wbc,
3370 struct extent_page_data *epd,
3371 loff_t i_size,
3372 unsigned long nr_written,
3373 int write_flags, int *nr_ret)
3374{
Chris Masond1310b22008-01-24 16:13:08 -05003375 struct extent_io_tree *tree = epd->tree;
Miao Xie4eee4fa2012-12-21 09:17:45 +00003376 u64 start = page_offset(page);
Chris Masond1310b22008-01-24 16:13:08 -05003377 u64 page_end = start + PAGE_CACHE_SIZE - 1;
3378 u64 end;
3379 u64 cur = start;
3380 u64 extent_offset;
Chris Masond1310b22008-01-24 16:13:08 -05003381 u64 block_start;
3382 u64 iosize;
3383 sector_t sector;
Chris Mason2c64c532009-09-02 15:04:12 -04003384 struct extent_state *cached_state = NULL;
Chris Masond1310b22008-01-24 16:13:08 -05003385 struct extent_map *em;
3386 struct block_device *bdev;
Chris Mason7f3c74f2008-07-18 12:01:11 -04003387 size_t pg_offset = 0;
Chris Masond1310b22008-01-24 16:13:08 -05003388 size_t blocksize;
Chris Mason40f76582014-05-21 13:35:51 -07003389 int ret = 0;
3390 int nr = 0;
3391 bool compressed;
Chris Masond1310b22008-01-24 16:13:08 -05003392
Chris Mason247e7432008-07-17 12:53:51 -04003393 if (tree->ops && tree->ops->writepage_start_hook) {
Chris Masonc8b97812008-10-29 14:49:59 -04003394 ret = tree->ops->writepage_start_hook(page, start,
3395 page_end);
Jeff Mahoney87826df2012-02-15 16:23:57 +01003396 if (ret) {
3397 /* Fixup worker will requeue */
3398 if (ret == -EBUSY)
3399 wbc->pages_skipped++;
3400 else
3401 redirty_page_for_writepage(wbc, page);
Chris Mason40f76582014-05-21 13:35:51 -07003402
Chris Mason11c83492009-04-20 15:50:09 -04003403 update_nr_written(page, wbc, nr_written);
Chris Mason247e7432008-07-17 12:53:51 -04003404 unlock_page(page);
Chris Mason40f76582014-05-21 13:35:51 -07003405 ret = 1;
Chris Mason11c83492009-04-20 15:50:09 -04003406 goto done_unlocked;
Chris Mason247e7432008-07-17 12:53:51 -04003407 }
3408 }
3409
Chris Mason11c83492009-04-20 15:50:09 -04003410 /*
3411 * we don't want to touch the inode after unlocking the page,
3412 * so we update the mapping writeback index now
3413 */
3414 update_nr_written(page, wbc, nr_written + 1);
Chris Mason771ed682008-11-06 22:02:51 -05003415
Chris Masond1310b22008-01-24 16:13:08 -05003416 end = page_end;
Chris Mason40f76582014-05-21 13:35:51 -07003417 if (i_size <= start) {
Chris Masone6dcd2d2008-07-17 12:53:50 -04003418 if (tree->ops && tree->ops->writepage_end_io_hook)
3419 tree->ops->writepage_end_io_hook(page, start,
3420 page_end, NULL, 1);
Chris Masond1310b22008-01-24 16:13:08 -05003421 goto done;
3422 }
3423
Chris Masond1310b22008-01-24 16:13:08 -05003424 blocksize = inode->i_sb->s_blocksize;
3425
3426 while (cur <= end) {
Chris Mason40f76582014-05-21 13:35:51 -07003427 u64 em_end;
3428 if (cur >= i_size) {
Chris Masone6dcd2d2008-07-17 12:53:50 -04003429 if (tree->ops && tree->ops->writepage_end_io_hook)
3430 tree->ops->writepage_end_io_hook(page, cur,
3431 page_end, NULL, 1);
Chris Masond1310b22008-01-24 16:13:08 -05003432 break;
3433 }
Chris Mason7f3c74f2008-07-18 12:01:11 -04003434 em = epd->get_extent(inode, page, pg_offset, cur,
Chris Masond1310b22008-01-24 16:13:08 -05003435 end - cur + 1, 1);
David Sterbac7040052011-04-19 18:00:01 +02003436 if (IS_ERR_OR_NULL(em)) {
Chris Masond1310b22008-01-24 16:13:08 -05003437 SetPageError(page);
Filipe Manana61391d52014-05-09 17:17:40 +01003438 ret = PTR_ERR_OR_ZERO(em);
Chris Masond1310b22008-01-24 16:13:08 -05003439 break;
3440 }
3441
3442 extent_offset = cur - em->start;
Chris Mason40f76582014-05-21 13:35:51 -07003443 em_end = extent_map_end(em);
3444 BUG_ON(em_end <= cur);
Chris Masond1310b22008-01-24 16:13:08 -05003445 BUG_ON(end < cur);
Chris Mason40f76582014-05-21 13:35:51 -07003446 iosize = min(em_end - cur, end - cur + 1);
Qu Wenruofda28322013-02-26 08:10:22 +00003447 iosize = ALIGN(iosize, blocksize);
Chris Masond1310b22008-01-24 16:13:08 -05003448 sector = (em->block_start + extent_offset) >> 9;
3449 bdev = em->bdev;
3450 block_start = em->block_start;
Chris Masonc8b97812008-10-29 14:49:59 -04003451 compressed = test_bit(EXTENT_FLAG_COMPRESSED, &em->flags);
Chris Masond1310b22008-01-24 16:13:08 -05003452 free_extent_map(em);
3453 em = NULL;
3454
Chris Masonc8b97812008-10-29 14:49:59 -04003455 /*
3456 * compressed and inline extents are written through other
3457 * paths in the FS
3458 */
3459 if (compressed || block_start == EXTENT_MAP_HOLE ||
Chris Masond1310b22008-01-24 16:13:08 -05003460 block_start == EXTENT_MAP_INLINE) {
Chris Masonc8b97812008-10-29 14:49:59 -04003461 /*
3462 * end_io notification does not happen here for
3463 * compressed extents
3464 */
3465 if (!compressed && tree->ops &&
3466 tree->ops->writepage_end_io_hook)
Chris Masone6dcd2d2008-07-17 12:53:50 -04003467 tree->ops->writepage_end_io_hook(page, cur,
3468 cur + iosize - 1,
3469 NULL, 1);
Chris Masonc8b97812008-10-29 14:49:59 -04003470 else if (compressed) {
3471 /* we don't want to end_page_writeback on
3472 * a compressed extent. this happens
3473 * elsewhere
3474 */
3475 nr++;
3476 }
3477
3478 cur += iosize;
Chris Mason7f3c74f2008-07-18 12:01:11 -04003479 pg_offset += iosize;
Chris Masond1310b22008-01-24 16:13:08 -05003480 continue;
3481 }
Chris Masonc8b97812008-10-29 14:49:59 -04003482
Chris Masond1310b22008-01-24 16:13:08 -05003483 if (tree->ops && tree->ops->writepage_io_hook) {
3484 ret = tree->ops->writepage_io_hook(page, cur,
3485 cur + iosize - 1);
3486 } else {
3487 ret = 0;
3488 }
Chris Mason1259ab72008-05-12 13:39:03 -04003489 if (ret) {
Chris Masond1310b22008-01-24 16:13:08 -05003490 SetPageError(page);
Chris Mason1259ab72008-05-12 13:39:03 -04003491 } else {
Chris Mason40f76582014-05-21 13:35:51 -07003492 unsigned long max_nr = (i_size >> PAGE_CACHE_SHIFT) + 1;
Chris Mason7f3c74f2008-07-18 12:01:11 -04003493
Chris Masond1310b22008-01-24 16:13:08 -05003494 set_range_writeback(tree, cur, cur + iosize - 1);
3495 if (!PageWriteback(page)) {
Frank Holtonefe120a2013-12-20 11:37:06 -05003496 btrfs_err(BTRFS_I(inode)->root->fs_info,
3497 "page %lu not writeback, cur %llu end %llu",
Geert Uytterhoevenc1c9ff72013-08-20 13:20:07 +02003498 page->index, cur, end);
Chris Masond1310b22008-01-24 16:13:08 -05003499 }
3500
Chris Masonda2f0f72015-07-02 13:57:22 -07003501 ret = submit_extent_page(write_flags, tree, wbc, page,
Chris Masonffbd5172009-04-20 15:50:09 -04003502 sector, iosize, pg_offset,
3503 bdev, &epd->bio, max_nr,
Chris Masonc8b97812008-10-29 14:49:59 -04003504 end_bio_extent_writepage,
Filipe Manana005efed2015-09-14 09:09:31 +01003505 0, 0, 0, false);
Chris Masond1310b22008-01-24 16:13:08 -05003506 if (ret)
3507 SetPageError(page);
3508 }
3509 cur = cur + iosize;
Chris Mason7f3c74f2008-07-18 12:01:11 -04003510 pg_offset += iosize;
Chris Masond1310b22008-01-24 16:13:08 -05003511 nr++;
3512 }
3513done:
Chris Mason40f76582014-05-21 13:35:51 -07003514 *nr_ret = nr;
Chris Mason771ed682008-11-06 22:02:51 -05003515
Chris Mason11c83492009-04-20 15:50:09 -04003516done_unlocked:
3517
Chris Mason2c64c532009-09-02 15:04:12 -04003518 /* drop our reference on any cached states */
3519 free_extent_state(cached_state);
Chris Mason40f76582014-05-21 13:35:51 -07003520 return ret;
3521}
3522
3523/*
3524 * the writepage semantics are similar to regular writepage. extent
3525 * records are inserted to lock ranges in the tree, and as dirty areas
3526 * are found, they are marked writeback. Then the lock bits are removed
3527 * and the end_io handler clears the writeback ranges
3528 */
3529static int __extent_writepage(struct page *page, struct writeback_control *wbc,
3530 void *data)
3531{
3532 struct inode *inode = page->mapping->host;
3533 struct extent_page_data *epd = data;
3534 u64 start = page_offset(page);
3535 u64 page_end = start + PAGE_CACHE_SIZE - 1;
3536 int ret;
3537 int nr = 0;
3538 size_t pg_offset = 0;
3539 loff_t i_size = i_size_read(inode);
3540 unsigned long end_index = i_size >> PAGE_CACHE_SHIFT;
3541 int write_flags;
3542 unsigned long nr_written = 0;
3543
3544 if (wbc->sync_mode == WB_SYNC_ALL)
3545 write_flags = WRITE_SYNC;
3546 else
3547 write_flags = WRITE;
3548
3549 trace___extent_writepage(page, inode, wbc);
3550
3551 WARN_ON(!PageLocked(page));
3552
3553 ClearPageError(page);
3554
3555 pg_offset = i_size & (PAGE_CACHE_SIZE - 1);
3556 if (page->index > end_index ||
3557 (page->index == end_index && !pg_offset)) {
3558 page->mapping->a_ops->invalidatepage(page, 0, PAGE_CACHE_SIZE);
3559 unlock_page(page);
3560 return 0;
3561 }
3562
3563 if (page->index == end_index) {
3564 char *userpage;
3565
3566 userpage = kmap_atomic(page);
3567 memset(userpage + pg_offset, 0,
3568 PAGE_CACHE_SIZE - pg_offset);
3569 kunmap_atomic(userpage);
3570 flush_dcache_page(page);
3571 }
3572
3573 pg_offset = 0;
3574
3575 set_page_extent_mapped(page);
3576
3577 ret = writepage_delalloc(inode, page, wbc, epd, start, &nr_written);
3578 if (ret == 1)
3579 goto done_unlocked;
3580 if (ret)
3581 goto done;
3582
3583 ret = __extent_writepage_io(inode, page, wbc, epd,
3584 i_size, nr_written, write_flags, &nr);
3585 if (ret == 1)
3586 goto done_unlocked;
3587
3588done:
Chris Masone6dcd2d2008-07-17 12:53:50 -04003589 if (nr == 0) {
Chris Masond1310b22008-01-24 16:13:08 -05003590 /* make sure the mapping tag for page dirty gets cleared */
Chris Mason771ed682008-11-06 22:02:51 -05003591 set_page_writeback(page);
3592 end_page_writeback(page);
3593 }
Filipe Manana61391d52014-05-09 17:17:40 +01003594 if (PageError(page)) {
3595 ret = ret < 0 ? ret : -EIO;
3596 end_extent_writepage(page, ret, start, page_end);
3597 }
Christoph Hellwigb2950862008-12-02 09:54:17 -05003598 unlock_page(page);
Chris Mason40f76582014-05-21 13:35:51 -07003599 return ret;
Chris Mason4bef0842008-09-08 11:18:08 -04003600
3601done_unlocked:
Chris Masond1310b22008-01-24 16:13:08 -05003602 return 0;
3603}
3604
Josef Bacikfd8b2b62013-04-24 16:41:19 -04003605void wait_on_extent_buffer_writeback(struct extent_buffer *eb)
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003606{
NeilBrown74316202014-07-07 15:16:04 +10003607 wait_on_bit_io(&eb->bflags, EXTENT_BUFFER_WRITEBACK,
3608 TASK_UNINTERRUPTIBLE);
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003609}
3610
Chris Mason0e378df2014-05-19 20:55:27 -07003611static noinline_for_stack int
3612lock_extent_buffer_for_io(struct extent_buffer *eb,
3613 struct btrfs_fs_info *fs_info,
3614 struct extent_page_data *epd)
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003615{
3616 unsigned long i, num_pages;
3617 int flush = 0;
3618 int ret = 0;
3619
3620 if (!btrfs_try_tree_write_lock(eb)) {
3621 flush = 1;
3622 flush_write_bio(epd);
3623 btrfs_tree_lock(eb);
3624 }
3625
3626 if (test_bit(EXTENT_BUFFER_WRITEBACK, &eb->bflags)) {
3627 btrfs_tree_unlock(eb);
3628 if (!epd->sync_io)
3629 return 0;
3630 if (!flush) {
3631 flush_write_bio(epd);
3632 flush = 1;
3633 }
Chris Masona098d8e2012-03-21 12:09:56 -04003634 while (1) {
3635 wait_on_extent_buffer_writeback(eb);
3636 btrfs_tree_lock(eb);
3637 if (!test_bit(EXTENT_BUFFER_WRITEBACK, &eb->bflags))
3638 break;
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003639 btrfs_tree_unlock(eb);
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003640 }
3641 }
3642
Josef Bacik51561ff2012-07-20 16:25:24 -04003643 /*
3644 * We need to do this to prevent races in people who check if the eb is
3645 * under IO since we can end up having no IO bits set for a short period
3646 * of time.
3647 */
3648 spin_lock(&eb->refs_lock);
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003649 if (test_and_clear_bit(EXTENT_BUFFER_DIRTY, &eb->bflags)) {
3650 set_bit(EXTENT_BUFFER_WRITEBACK, &eb->bflags);
Josef Bacik51561ff2012-07-20 16:25:24 -04003651 spin_unlock(&eb->refs_lock);
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003652 btrfs_set_header_flag(eb, BTRFS_HEADER_FLAG_WRITTEN);
Miao Xiee2d84522013-01-29 10:09:20 +00003653 __percpu_counter_add(&fs_info->dirty_metadata_bytes,
3654 -eb->len,
3655 fs_info->dirty_metadata_batch);
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003656 ret = 1;
Josef Bacik51561ff2012-07-20 16:25:24 -04003657 } else {
3658 spin_unlock(&eb->refs_lock);
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003659 }
3660
3661 btrfs_tree_unlock(eb);
3662
3663 if (!ret)
3664 return ret;
3665
3666 num_pages = num_extent_pages(eb->start, eb->len);
3667 for (i = 0; i < num_pages; i++) {
David Sterbafb85fc92014-07-31 01:03:53 +02003668 struct page *p = eb->pages[i];
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003669
3670 if (!trylock_page(p)) {
3671 if (!flush) {
3672 flush_write_bio(epd);
3673 flush = 1;
3674 }
3675 lock_page(p);
3676 }
3677 }
3678
3679 return ret;
3680}
3681
3682static void end_extent_buffer_writeback(struct extent_buffer *eb)
3683{
3684 clear_bit(EXTENT_BUFFER_WRITEBACK, &eb->bflags);
Peter Zijlstra4e857c52014-03-17 18:06:10 +01003685 smp_mb__after_atomic();
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003686 wake_up_bit(&eb->bflags, EXTENT_BUFFER_WRITEBACK);
3687}
3688
Filipe Manana656f30d2014-09-26 12:25:56 +01003689static void set_btree_ioerr(struct page *page)
3690{
3691 struct extent_buffer *eb = (struct extent_buffer *)page->private;
3692 struct btrfs_inode *btree_ino = BTRFS_I(eb->fs_info->btree_inode);
3693
3694 SetPageError(page);
3695 if (test_and_set_bit(EXTENT_BUFFER_WRITE_ERR, &eb->bflags))
3696 return;
3697
3698 /*
3699 * If writeback for a btree extent that doesn't belong to a log tree
3700 * failed, increment the counter transaction->eb_write_errors.
3701 * We do this because while the transaction is running and before it's
3702 * committing (when we call filemap_fdata[write|wait]_range against
3703 * the btree inode), we might have
3704 * btree_inode->i_mapping->a_ops->writepages() called by the VM - if it
3705 * returns an error or an error happens during writeback, when we're
3706 * committing the transaction we wouldn't know about it, since the pages
3707 * can be no longer dirty nor marked anymore for writeback (if a
3708 * subsequent modification to the extent buffer didn't happen before the
3709 * transaction commit), which makes filemap_fdata[write|wait]_range not
3710 * able to find the pages tagged with SetPageError at transaction
3711 * commit time. So if this happens we must abort the transaction,
3712 * otherwise we commit a super block with btree roots that point to
3713 * btree nodes/leafs whose content on disk is invalid - either garbage
3714 * or the content of some node/leaf from a past generation that got
3715 * cowed or deleted and is no longer valid.
3716 *
3717 * Note: setting AS_EIO/AS_ENOSPC in the btree inode's i_mapping would
3718 * not be enough - we need to distinguish between log tree extents vs
3719 * non-log tree extents, and the next filemap_fdatawait_range() call
3720 * will catch and clear such errors in the mapping - and that call might
3721 * be from a log sync and not from a transaction commit. Also, checking
3722 * for the eb flag EXTENT_BUFFER_WRITE_ERR at transaction commit time is
3723 * not done and would not be reliable - the eb might have been released
3724 * from memory and reading it back again means that flag would not be
3725 * set (since it's a runtime flag, not persisted on disk).
3726 *
3727 * Using the flags below in the btree inode also makes us achieve the
3728 * goal of AS_EIO/AS_ENOSPC when writepages() returns success, started
3729 * writeback for all dirty pages and before filemap_fdatawait_range()
3730 * is called, the writeback for all dirty pages had already finished
3731 * with errors - because we were not using AS_EIO/AS_ENOSPC,
3732 * filemap_fdatawait_range() would return success, as it could not know
3733 * that writeback errors happened (the pages were no longer tagged for
3734 * writeback).
3735 */
3736 switch (eb->log_index) {
3737 case -1:
3738 set_bit(BTRFS_INODE_BTREE_ERR, &btree_ino->runtime_flags);
3739 break;
3740 case 0:
3741 set_bit(BTRFS_INODE_BTREE_LOG1_ERR, &btree_ino->runtime_flags);
3742 break;
3743 case 1:
3744 set_bit(BTRFS_INODE_BTREE_LOG2_ERR, &btree_ino->runtime_flags);
3745 break;
3746 default:
3747 BUG(); /* unexpected, logic error */
3748 }
3749}
3750
Christoph Hellwig4246a0b2015-07-20 15:29:37 +02003751static void end_bio_extent_buffer_writepage(struct bio *bio)
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003752{
Kent Overstreet2c30c712013-11-07 12:20:26 -08003753 struct bio_vec *bvec;
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003754 struct extent_buffer *eb;
Kent Overstreet2c30c712013-11-07 12:20:26 -08003755 int i, done;
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003756
Kent Overstreet2c30c712013-11-07 12:20:26 -08003757 bio_for_each_segment_all(bvec, bio, i) {
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003758 struct page *page = bvec->bv_page;
3759
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003760 eb = (struct extent_buffer *)page->private;
3761 BUG_ON(!eb);
3762 done = atomic_dec_and_test(&eb->io_pages);
3763
Christoph Hellwig4246a0b2015-07-20 15:29:37 +02003764 if (bio->bi_error ||
3765 test_bit(EXTENT_BUFFER_WRITE_ERR, &eb->bflags)) {
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003766 ClearPageUptodate(page);
Filipe Manana656f30d2014-09-26 12:25:56 +01003767 set_btree_ioerr(page);
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003768 }
3769
3770 end_page_writeback(page);
3771
3772 if (!done)
3773 continue;
3774
3775 end_extent_buffer_writeback(eb);
Kent Overstreet2c30c712013-11-07 12:20:26 -08003776 }
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003777
3778 bio_put(bio);
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003779}
3780
Chris Mason0e378df2014-05-19 20:55:27 -07003781static noinline_for_stack int write_one_eb(struct extent_buffer *eb,
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003782 struct btrfs_fs_info *fs_info,
3783 struct writeback_control *wbc,
3784 struct extent_page_data *epd)
3785{
3786 struct block_device *bdev = fs_info->fs_devices->latest_bdev;
Josef Bacikf28491e2013-12-16 13:24:27 -05003787 struct extent_io_tree *tree = &BTRFS_I(fs_info->btree_inode)->io_tree;
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003788 u64 offset = eb->start;
3789 unsigned long i, num_pages;
Josef Bacikde0022b2012-09-25 14:25:58 -04003790 unsigned long bio_flags = 0;
Josef Bacikd4c7ca82013-04-19 19:49:09 -04003791 int rw = (epd->sync_io ? WRITE_SYNC : WRITE) | REQ_META;
Josef Bacikd7dbe9e2012-04-23 14:00:51 -04003792 int ret = 0;
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003793
Filipe Manana656f30d2014-09-26 12:25:56 +01003794 clear_bit(EXTENT_BUFFER_WRITE_ERR, &eb->bflags);
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003795 num_pages = num_extent_pages(eb->start, eb->len);
3796 atomic_set(&eb->io_pages, num_pages);
Josef Bacikde0022b2012-09-25 14:25:58 -04003797 if (btrfs_header_owner(eb) == BTRFS_TREE_LOG_OBJECTID)
3798 bio_flags = EXTENT_BIO_TREE_LOG;
3799
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003800 for (i = 0; i < num_pages; i++) {
David Sterbafb85fc92014-07-31 01:03:53 +02003801 struct page *p = eb->pages[i];
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003802
3803 clear_page_dirty_for_io(p);
3804 set_page_writeback(p);
Chris Masonda2f0f72015-07-02 13:57:22 -07003805 ret = submit_extent_page(rw, tree, wbc, p, offset >> 9,
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003806 PAGE_CACHE_SIZE, 0, bdev, &epd->bio,
3807 -1, end_bio_extent_buffer_writepage,
Filipe Manana005efed2015-09-14 09:09:31 +01003808 0, epd->bio_flags, bio_flags, false);
Josef Bacikde0022b2012-09-25 14:25:58 -04003809 epd->bio_flags = bio_flags;
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003810 if (ret) {
Filipe Manana656f30d2014-09-26 12:25:56 +01003811 set_btree_ioerr(p);
Filipe Manana55e3bd22014-09-22 17:41:04 +01003812 end_page_writeback(p);
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003813 if (atomic_sub_and_test(num_pages - i, &eb->io_pages))
3814 end_extent_buffer_writeback(eb);
3815 ret = -EIO;
3816 break;
3817 }
3818 offset += PAGE_CACHE_SIZE;
3819 update_nr_written(p, wbc, 1);
3820 unlock_page(p);
3821 }
3822
3823 if (unlikely(ret)) {
3824 for (; i < num_pages; i++) {
Chris Masonbbf65cf2014-10-04 09:56:45 -07003825 struct page *p = eb->pages[i];
Liu Bo81465022014-09-23 22:22:33 +08003826 clear_page_dirty_for_io(p);
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003827 unlock_page(p);
3828 }
3829 }
3830
3831 return ret;
3832}
3833
3834int btree_write_cache_pages(struct address_space *mapping,
3835 struct writeback_control *wbc)
3836{
3837 struct extent_io_tree *tree = &BTRFS_I(mapping->host)->io_tree;
3838 struct btrfs_fs_info *fs_info = BTRFS_I(mapping->host)->root->fs_info;
3839 struct extent_buffer *eb, *prev_eb = NULL;
3840 struct extent_page_data epd = {
3841 .bio = NULL,
3842 .tree = tree,
3843 .extent_locked = 0,
3844 .sync_io = wbc->sync_mode == WB_SYNC_ALL,
Josef Bacikde0022b2012-09-25 14:25:58 -04003845 .bio_flags = 0,
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003846 };
3847 int ret = 0;
3848 int done = 0;
3849 int nr_to_write_done = 0;
3850 struct pagevec pvec;
3851 int nr_pages;
3852 pgoff_t index;
3853 pgoff_t end; /* Inclusive */
3854 int scanned = 0;
3855 int tag;
3856
3857 pagevec_init(&pvec, 0);
3858 if (wbc->range_cyclic) {
3859 index = mapping->writeback_index; /* Start from prev offset */
3860 end = -1;
3861 } else {
3862 index = wbc->range_start >> PAGE_CACHE_SHIFT;
3863 end = wbc->range_end >> PAGE_CACHE_SHIFT;
3864 scanned = 1;
3865 }
3866 if (wbc->sync_mode == WB_SYNC_ALL)
3867 tag = PAGECACHE_TAG_TOWRITE;
3868 else
3869 tag = PAGECACHE_TAG_DIRTY;
3870retry:
3871 if (wbc->sync_mode == WB_SYNC_ALL)
3872 tag_pages_for_writeback(mapping, index, end);
3873 while (!done && !nr_to_write_done && (index <= end) &&
3874 (nr_pages = pagevec_lookup_tag(&pvec, mapping, &index, tag,
3875 min(end - index, (pgoff_t)PAGEVEC_SIZE-1) + 1))) {
3876 unsigned i;
3877
3878 scanned = 1;
3879 for (i = 0; i < nr_pages; i++) {
3880 struct page *page = pvec.pages[i];
3881
3882 if (!PagePrivate(page))
3883 continue;
3884
3885 if (!wbc->range_cyclic && page->index > end) {
3886 done = 1;
3887 break;
3888 }
3889
Josef Bacikb5bae262012-09-14 13:43:01 -04003890 spin_lock(&mapping->private_lock);
3891 if (!PagePrivate(page)) {
3892 spin_unlock(&mapping->private_lock);
3893 continue;
3894 }
3895
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003896 eb = (struct extent_buffer *)page->private;
Josef Bacikb5bae262012-09-14 13:43:01 -04003897
3898 /*
3899 * Shouldn't happen and normally this would be a BUG_ON
3900 * but no sense in crashing the users box for something
3901 * we can survive anyway.
3902 */
Dulshani Gunawardhanafae7f212013-10-31 10:30:08 +05303903 if (WARN_ON(!eb)) {
Josef Bacikb5bae262012-09-14 13:43:01 -04003904 spin_unlock(&mapping->private_lock);
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003905 continue;
3906 }
3907
Josef Bacikb5bae262012-09-14 13:43:01 -04003908 if (eb == prev_eb) {
3909 spin_unlock(&mapping->private_lock);
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003910 continue;
3911 }
3912
Josef Bacikb5bae262012-09-14 13:43:01 -04003913 ret = atomic_inc_not_zero(&eb->refs);
3914 spin_unlock(&mapping->private_lock);
3915 if (!ret)
3916 continue;
3917
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003918 prev_eb = eb;
3919 ret = lock_extent_buffer_for_io(eb, fs_info, &epd);
3920 if (!ret) {
3921 free_extent_buffer(eb);
3922 continue;
3923 }
3924
3925 ret = write_one_eb(eb, fs_info, wbc, &epd);
3926 if (ret) {
3927 done = 1;
3928 free_extent_buffer(eb);
3929 break;
3930 }
3931 free_extent_buffer(eb);
3932
3933 /*
3934 * the filesystem may choose to bump up nr_to_write.
3935 * We have to make sure to honor the new nr_to_write
3936 * at any time
3937 */
3938 nr_to_write_done = wbc->nr_to_write <= 0;
3939 }
3940 pagevec_release(&pvec);
3941 cond_resched();
3942 }
3943 if (!scanned && !done) {
3944 /*
3945 * We hit the last page and there is more work to be done: wrap
3946 * back to the start of the file
3947 */
3948 scanned = 1;
3949 index = 0;
3950 goto retry;
3951 }
3952 flush_write_bio(&epd);
3953 return ret;
3954}
3955
Chris Masond1310b22008-01-24 16:13:08 -05003956/**
3957 * write_cache_pages - walk the list of dirty pages of the given address space and write all of them.
3958 * @mapping: address space structure to write
3959 * @wbc: subtract the number of written pages from *@wbc->nr_to_write
3960 * @writepage: function called for each page
3961 * @data: data passed to writepage function
3962 *
3963 * If a page is already under I/O, write_cache_pages() skips it, even
3964 * if it's dirty. This is desirable behaviour for memory-cleaning writeback,
3965 * but it is INCORRECT for data-integrity system calls such as fsync(). fsync()
3966 * and msync() need to guarantee that all the data which was dirty at the time
3967 * the call was made get new I/O started against them. If wbc->sync_mode is
3968 * WB_SYNC_ALL then we were called for data integrity and we must wait for
3969 * existing IO to complete.
3970 */
Chris Mason4bef0842008-09-08 11:18:08 -04003971static int extent_write_cache_pages(struct extent_io_tree *tree,
3972 struct address_space *mapping,
3973 struct writeback_control *wbc,
Chris Masond2c3f4f2008-11-19 12:44:22 -05003974 writepage_t writepage, void *data,
3975 void (*flush_fn)(void *))
Chris Masond1310b22008-01-24 16:13:08 -05003976{
Josef Bacik7fd1a3f2012-06-27 17:18:41 -04003977 struct inode *inode = mapping->host;
Chris Masond1310b22008-01-24 16:13:08 -05003978 int ret = 0;
3979 int done = 0;
Filipe Manana61391d52014-05-09 17:17:40 +01003980 int err = 0;
Chris Masonf85d7d6c2009-09-18 16:03:16 -04003981 int nr_to_write_done = 0;
Chris Masond1310b22008-01-24 16:13:08 -05003982 struct pagevec pvec;
3983 int nr_pages;
3984 pgoff_t index;
3985 pgoff_t end; /* Inclusive */
3986 int scanned = 0;
Josef Bacikf7aaa062011-07-15 21:26:38 +00003987 int tag;
Chris Masond1310b22008-01-24 16:13:08 -05003988
Josef Bacik7fd1a3f2012-06-27 17:18:41 -04003989 /*
3990 * We have to hold onto the inode so that ordered extents can do their
3991 * work when the IO finishes. The alternative to this is failing to add
3992 * an ordered extent if the igrab() fails there and that is a huge pain
3993 * to deal with, so instead just hold onto the inode throughout the
3994 * writepages operation. If it fails here we are freeing up the inode
3995 * anyway and we'd rather not waste our time writing out stuff that is
3996 * going to be truncated anyway.
3997 */
3998 if (!igrab(inode))
3999 return 0;
4000
Chris Masond1310b22008-01-24 16:13:08 -05004001 pagevec_init(&pvec, 0);
4002 if (wbc->range_cyclic) {
4003 index = mapping->writeback_index; /* Start from prev offset */
4004 end = -1;
4005 } else {
4006 index = wbc->range_start >> PAGE_CACHE_SHIFT;
4007 end = wbc->range_end >> PAGE_CACHE_SHIFT;
Chris Masond1310b22008-01-24 16:13:08 -05004008 scanned = 1;
4009 }
Josef Bacikf7aaa062011-07-15 21:26:38 +00004010 if (wbc->sync_mode == WB_SYNC_ALL)
4011 tag = PAGECACHE_TAG_TOWRITE;
4012 else
4013 tag = PAGECACHE_TAG_DIRTY;
Chris Masond1310b22008-01-24 16:13:08 -05004014retry:
Josef Bacikf7aaa062011-07-15 21:26:38 +00004015 if (wbc->sync_mode == WB_SYNC_ALL)
4016 tag_pages_for_writeback(mapping, index, end);
Chris Masonf85d7d6c2009-09-18 16:03:16 -04004017 while (!done && !nr_to_write_done && (index <= end) &&
Josef Bacikf7aaa062011-07-15 21:26:38 +00004018 (nr_pages = pagevec_lookup_tag(&pvec, mapping, &index, tag,
4019 min(end - index, (pgoff_t)PAGEVEC_SIZE-1) + 1))) {
Chris Masond1310b22008-01-24 16:13:08 -05004020 unsigned i;
4021
4022 scanned = 1;
4023 for (i = 0; i < nr_pages; i++) {
4024 struct page *page = pvec.pages[i];
4025
4026 /*
4027 * At this point we hold neither mapping->tree_lock nor
4028 * lock on the page itself: the page may be truncated or
4029 * invalidated (changing page->mapping to NULL), or even
4030 * swizzled back from swapper_space to tmpfs file
4031 * mapping
4032 */
Josef Bacikc8f2f242013-02-11 11:33:00 -05004033 if (!trylock_page(page)) {
4034 flush_fn(data);
4035 lock_page(page);
Chris Mason01d658f2011-11-01 10:08:06 -04004036 }
Chris Masond1310b22008-01-24 16:13:08 -05004037
4038 if (unlikely(page->mapping != mapping)) {
4039 unlock_page(page);
4040 continue;
4041 }
4042
4043 if (!wbc->range_cyclic && page->index > end) {
4044 done = 1;
4045 unlock_page(page);
4046 continue;
4047 }
4048
Chris Masond2c3f4f2008-11-19 12:44:22 -05004049 if (wbc->sync_mode != WB_SYNC_NONE) {
Chris Mason0e6bd952008-11-20 10:46:35 -05004050 if (PageWriteback(page))
4051 flush_fn(data);
Chris Masond1310b22008-01-24 16:13:08 -05004052 wait_on_page_writeback(page);
Chris Masond2c3f4f2008-11-19 12:44:22 -05004053 }
Chris Masond1310b22008-01-24 16:13:08 -05004054
4055 if (PageWriteback(page) ||
4056 !clear_page_dirty_for_io(page)) {
4057 unlock_page(page);
4058 continue;
4059 }
4060
4061 ret = (*writepage)(page, wbc, data);
4062
4063 if (unlikely(ret == AOP_WRITEPAGE_ACTIVATE)) {
4064 unlock_page(page);
4065 ret = 0;
4066 }
Filipe Manana61391d52014-05-09 17:17:40 +01004067 if (!err && ret < 0)
4068 err = ret;
Chris Masonf85d7d6c2009-09-18 16:03:16 -04004069
4070 /*
4071 * the filesystem may choose to bump up nr_to_write.
4072 * We have to make sure to honor the new nr_to_write
4073 * at any time
4074 */
4075 nr_to_write_done = wbc->nr_to_write <= 0;
Chris Masond1310b22008-01-24 16:13:08 -05004076 }
4077 pagevec_release(&pvec);
4078 cond_resched();
4079 }
Filipe Manana61391d52014-05-09 17:17:40 +01004080 if (!scanned && !done && !err) {
Chris Masond1310b22008-01-24 16:13:08 -05004081 /*
4082 * We hit the last page and there is more work to be done: wrap
4083 * back to the start of the file
4084 */
4085 scanned = 1;
4086 index = 0;
4087 goto retry;
4088 }
Josef Bacik7fd1a3f2012-06-27 17:18:41 -04004089 btrfs_add_delayed_iput(inode);
Filipe Manana61391d52014-05-09 17:17:40 +01004090 return err;
Chris Masond1310b22008-01-24 16:13:08 -05004091}
Chris Masond1310b22008-01-24 16:13:08 -05004092
Chris Masonffbd5172009-04-20 15:50:09 -04004093static void flush_epd_write_bio(struct extent_page_data *epd)
4094{
4095 if (epd->bio) {
Jeff Mahoney355808c2011-10-03 23:23:14 -04004096 int rw = WRITE;
4097 int ret;
4098
Chris Masonffbd5172009-04-20 15:50:09 -04004099 if (epd->sync_io)
Jeff Mahoney355808c2011-10-03 23:23:14 -04004100 rw = WRITE_SYNC;
4101
Josef Bacikde0022b2012-09-25 14:25:58 -04004102 ret = submit_one_bio(rw, epd->bio, 0, epd->bio_flags);
Jeff Mahoney79787ea2012-03-12 16:03:00 +01004103 BUG_ON(ret < 0); /* -ENOMEM */
Chris Masonffbd5172009-04-20 15:50:09 -04004104 epd->bio = NULL;
4105 }
4106}
4107
Chris Masond2c3f4f2008-11-19 12:44:22 -05004108static noinline void flush_write_bio(void *data)
4109{
4110 struct extent_page_data *epd = data;
Chris Masonffbd5172009-04-20 15:50:09 -04004111 flush_epd_write_bio(epd);
Chris Masond2c3f4f2008-11-19 12:44:22 -05004112}
4113
Chris Masond1310b22008-01-24 16:13:08 -05004114int extent_write_full_page(struct extent_io_tree *tree, struct page *page,
4115 get_extent_t *get_extent,
4116 struct writeback_control *wbc)
4117{
4118 int ret;
Chris Masond1310b22008-01-24 16:13:08 -05004119 struct extent_page_data epd = {
4120 .bio = NULL,
4121 .tree = tree,
4122 .get_extent = get_extent,
Chris Mason771ed682008-11-06 22:02:51 -05004123 .extent_locked = 0,
Chris Masonffbd5172009-04-20 15:50:09 -04004124 .sync_io = wbc->sync_mode == WB_SYNC_ALL,
Josef Bacikde0022b2012-09-25 14:25:58 -04004125 .bio_flags = 0,
Chris Masond1310b22008-01-24 16:13:08 -05004126 };
Chris Masond1310b22008-01-24 16:13:08 -05004127
Chris Masond1310b22008-01-24 16:13:08 -05004128 ret = __extent_writepage(page, wbc, &epd);
4129
Chris Masonffbd5172009-04-20 15:50:09 -04004130 flush_epd_write_bio(&epd);
Chris Masond1310b22008-01-24 16:13:08 -05004131 return ret;
4132}
Chris Masond1310b22008-01-24 16:13:08 -05004133
Chris Mason771ed682008-11-06 22:02:51 -05004134int extent_write_locked_range(struct extent_io_tree *tree, struct inode *inode,
4135 u64 start, u64 end, get_extent_t *get_extent,
4136 int mode)
4137{
4138 int ret = 0;
4139 struct address_space *mapping = inode->i_mapping;
4140 struct page *page;
4141 unsigned long nr_pages = (end - start + PAGE_CACHE_SIZE) >>
4142 PAGE_CACHE_SHIFT;
4143
4144 struct extent_page_data epd = {
4145 .bio = NULL,
4146 .tree = tree,
4147 .get_extent = get_extent,
4148 .extent_locked = 1,
Chris Masonffbd5172009-04-20 15:50:09 -04004149 .sync_io = mode == WB_SYNC_ALL,
Josef Bacikde0022b2012-09-25 14:25:58 -04004150 .bio_flags = 0,
Chris Mason771ed682008-11-06 22:02:51 -05004151 };
4152 struct writeback_control wbc_writepages = {
Chris Mason771ed682008-11-06 22:02:51 -05004153 .sync_mode = mode,
Chris Mason771ed682008-11-06 22:02:51 -05004154 .nr_to_write = nr_pages * 2,
4155 .range_start = start,
4156 .range_end = end + 1,
4157 };
4158
Chris Masond3977122009-01-05 21:25:51 -05004159 while (start <= end) {
Chris Mason771ed682008-11-06 22:02:51 -05004160 page = find_get_page(mapping, start >> PAGE_CACHE_SHIFT);
4161 if (clear_page_dirty_for_io(page))
4162 ret = __extent_writepage(page, &wbc_writepages, &epd);
4163 else {
4164 if (tree->ops && tree->ops->writepage_end_io_hook)
4165 tree->ops->writepage_end_io_hook(page, start,
4166 start + PAGE_CACHE_SIZE - 1,
4167 NULL, 1);
4168 unlock_page(page);
4169 }
4170 page_cache_release(page);
4171 start += PAGE_CACHE_SIZE;
4172 }
4173
Chris Masonffbd5172009-04-20 15:50:09 -04004174 flush_epd_write_bio(&epd);
Chris Mason771ed682008-11-06 22:02:51 -05004175 return ret;
4176}
Chris Masond1310b22008-01-24 16:13:08 -05004177
4178int extent_writepages(struct extent_io_tree *tree,
4179 struct address_space *mapping,
4180 get_extent_t *get_extent,
4181 struct writeback_control *wbc)
4182{
4183 int ret = 0;
4184 struct extent_page_data epd = {
4185 .bio = NULL,
4186 .tree = tree,
4187 .get_extent = get_extent,
Chris Mason771ed682008-11-06 22:02:51 -05004188 .extent_locked = 0,
Chris Masonffbd5172009-04-20 15:50:09 -04004189 .sync_io = wbc->sync_mode == WB_SYNC_ALL,
Josef Bacikde0022b2012-09-25 14:25:58 -04004190 .bio_flags = 0,
Chris Masond1310b22008-01-24 16:13:08 -05004191 };
4192
Chris Mason4bef0842008-09-08 11:18:08 -04004193 ret = extent_write_cache_pages(tree, mapping, wbc,
Chris Masond2c3f4f2008-11-19 12:44:22 -05004194 __extent_writepage, &epd,
4195 flush_write_bio);
Chris Masonffbd5172009-04-20 15:50:09 -04004196 flush_epd_write_bio(&epd);
Chris Masond1310b22008-01-24 16:13:08 -05004197 return ret;
4198}
Chris Masond1310b22008-01-24 16:13:08 -05004199
4200int extent_readpages(struct extent_io_tree *tree,
4201 struct address_space *mapping,
4202 struct list_head *pages, unsigned nr_pages,
4203 get_extent_t get_extent)
4204{
4205 struct bio *bio = NULL;
4206 unsigned page_idx;
Chris Masonc8b97812008-10-29 14:49:59 -04004207 unsigned long bio_flags = 0;
Liu Bo67c96842012-07-20 21:43:09 -06004208 struct page *pagepool[16];
4209 struct page *page;
Miao Xie125bac012013-07-25 19:22:37 +08004210 struct extent_map *em_cached = NULL;
Liu Bo67c96842012-07-20 21:43:09 -06004211 int nr = 0;
Filipe Manana808f80b2015-09-28 09:56:26 +01004212 u64 prev_em_start = (u64)-1;
Chris Masond1310b22008-01-24 16:13:08 -05004213
Chris Masond1310b22008-01-24 16:13:08 -05004214 for (page_idx = 0; page_idx < nr_pages; page_idx++) {
Liu Bo67c96842012-07-20 21:43:09 -06004215 page = list_entry(pages->prev, struct page, lru);
Chris Masond1310b22008-01-24 16:13:08 -05004216
4217 prefetchw(&page->flags);
4218 list_del(&page->lru);
Liu Bo67c96842012-07-20 21:43:09 -06004219 if (add_to_page_cache_lru(page, mapping,
Itaru Kitayama43e817a2011-04-25 19:43:51 -04004220 page->index, GFP_NOFS)) {
Liu Bo67c96842012-07-20 21:43:09 -06004221 page_cache_release(page);
4222 continue;
Chris Masond1310b22008-01-24 16:13:08 -05004223 }
Liu Bo67c96842012-07-20 21:43:09 -06004224
4225 pagepool[nr++] = page;
4226 if (nr < ARRAY_SIZE(pagepool))
4227 continue;
Miao Xie125bac012013-07-25 19:22:37 +08004228 __extent_readpages(tree, pagepool, nr, get_extent, &em_cached,
Filipe Manana808f80b2015-09-28 09:56:26 +01004229 &bio, 0, &bio_flags, READ, &prev_em_start);
Liu Bo67c96842012-07-20 21:43:09 -06004230 nr = 0;
Chris Masond1310b22008-01-24 16:13:08 -05004231 }
Miao Xie99740902013-07-25 19:22:36 +08004232 if (nr)
Miao Xie125bac012013-07-25 19:22:37 +08004233 __extent_readpages(tree, pagepool, nr, get_extent, &em_cached,
Filipe Manana808f80b2015-09-28 09:56:26 +01004234 &bio, 0, &bio_flags, READ, &prev_em_start);
Liu Bo67c96842012-07-20 21:43:09 -06004235
Miao Xie125bac012013-07-25 19:22:37 +08004236 if (em_cached)
4237 free_extent_map(em_cached);
4238
Chris Masond1310b22008-01-24 16:13:08 -05004239 BUG_ON(!list_empty(pages));
4240 if (bio)
Jeff Mahoney79787ea2012-03-12 16:03:00 +01004241 return submit_one_bio(READ, bio, 0, bio_flags);
Chris Masond1310b22008-01-24 16:13:08 -05004242 return 0;
4243}
Chris Masond1310b22008-01-24 16:13:08 -05004244
4245/*
4246 * basic invalidatepage code, this waits on any locked or writeback
4247 * ranges corresponding to the page, and then deletes any extent state
4248 * records from the tree
4249 */
4250int extent_invalidatepage(struct extent_io_tree *tree,
4251 struct page *page, unsigned long offset)
4252{
Josef Bacik2ac55d42010-02-03 19:33:23 +00004253 struct extent_state *cached_state = NULL;
Miao Xie4eee4fa2012-12-21 09:17:45 +00004254 u64 start = page_offset(page);
Chris Masond1310b22008-01-24 16:13:08 -05004255 u64 end = start + PAGE_CACHE_SIZE - 1;
4256 size_t blocksize = page->mapping->host->i_sb->s_blocksize;
4257
Qu Wenruofda28322013-02-26 08:10:22 +00004258 start += ALIGN(offset, blocksize);
Chris Masond1310b22008-01-24 16:13:08 -05004259 if (start > end)
4260 return 0;
4261
Jeff Mahoneyd0082372012-03-01 14:57:19 +01004262 lock_extent_bits(tree, start, end, 0, &cached_state);
Chris Mason1edbb732009-09-02 13:24:36 -04004263 wait_on_page_writeback(page);
Chris Masond1310b22008-01-24 16:13:08 -05004264 clear_extent_bit(tree, start, end,
Josef Bacik32c00af2009-10-08 13:34:05 -04004265 EXTENT_LOCKED | EXTENT_DIRTY | EXTENT_DELALLOC |
4266 EXTENT_DO_ACCOUNTING,
Josef Bacik2ac55d42010-02-03 19:33:23 +00004267 1, 1, &cached_state, GFP_NOFS);
Chris Masond1310b22008-01-24 16:13:08 -05004268 return 0;
4269}
Chris Masond1310b22008-01-24 16:13:08 -05004270
4271/*
Chris Mason7b13b7b2008-04-18 10:29:50 -04004272 * a helper for releasepage, this tests for areas of the page that
4273 * are locked or under IO and drops the related state bits if it is safe
4274 * to drop the page.
4275 */
Eric Sandeen48a3b632013-04-25 20:41:01 +00004276static int try_release_extent_state(struct extent_map_tree *map,
4277 struct extent_io_tree *tree,
4278 struct page *page, gfp_t mask)
Chris Mason7b13b7b2008-04-18 10:29:50 -04004279{
Miao Xie4eee4fa2012-12-21 09:17:45 +00004280 u64 start = page_offset(page);
Chris Mason7b13b7b2008-04-18 10:29:50 -04004281 u64 end = start + PAGE_CACHE_SIZE - 1;
4282 int ret = 1;
4283
Chris Mason211f90e2008-07-18 11:56:15 -04004284 if (test_range_bit(tree, start, end,
Chris Mason8b62b722009-09-02 16:53:46 -04004285 EXTENT_IOBITS, 0, NULL))
Chris Mason7b13b7b2008-04-18 10:29:50 -04004286 ret = 0;
4287 else {
4288 if ((mask & GFP_NOFS) == GFP_NOFS)
4289 mask = GFP_NOFS;
Chris Mason11ef1602009-09-23 20:28:46 -04004290 /*
4291 * at this point we can safely clear everything except the
4292 * locked bit and the nodatasum bit
4293 */
Chris Masone3f24cc2011-02-14 12:52:08 -05004294 ret = clear_extent_bit(tree, start, end,
Chris Mason11ef1602009-09-23 20:28:46 -04004295 ~(EXTENT_LOCKED | EXTENT_NODATASUM),
4296 0, 0, NULL, mask);
Chris Masone3f24cc2011-02-14 12:52:08 -05004297
4298 /* if clear_extent_bit failed for enomem reasons,
4299 * we can't allow the release to continue.
4300 */
4301 if (ret < 0)
4302 ret = 0;
4303 else
4304 ret = 1;
Chris Mason7b13b7b2008-04-18 10:29:50 -04004305 }
4306 return ret;
4307}
Chris Mason7b13b7b2008-04-18 10:29:50 -04004308
4309/*
Chris Masond1310b22008-01-24 16:13:08 -05004310 * a helper for releasepage. As long as there are no locked extents
4311 * in the range corresponding to the page, both state records and extent
4312 * map records are removed
4313 */
4314int try_release_extent_mapping(struct extent_map_tree *map,
Chris Mason70dec802008-01-29 09:59:12 -05004315 struct extent_io_tree *tree, struct page *page,
4316 gfp_t mask)
Chris Masond1310b22008-01-24 16:13:08 -05004317{
4318 struct extent_map *em;
Miao Xie4eee4fa2012-12-21 09:17:45 +00004319 u64 start = page_offset(page);
Chris Masond1310b22008-01-24 16:13:08 -05004320 u64 end = start + PAGE_CACHE_SIZE - 1;
Chris Mason7b13b7b2008-04-18 10:29:50 -04004321
Mel Gormand0164ad2015-11-06 16:28:21 -08004322 if (gfpflags_allow_blocking(mask) &&
Chris Mason70dec802008-01-29 09:59:12 -05004323 page->mapping->host->i_size > 16 * 1024 * 1024) {
Yan39b56372008-02-15 10:40:50 -05004324 u64 len;
Chris Mason70dec802008-01-29 09:59:12 -05004325 while (start <= end) {
Yan39b56372008-02-15 10:40:50 -05004326 len = end - start + 1;
Chris Mason890871b2009-09-02 16:24:52 -04004327 write_lock(&map->lock);
Yan39b56372008-02-15 10:40:50 -05004328 em = lookup_extent_mapping(map, start, len);
Tsutomu Itoh285190d2012-02-16 16:23:58 +09004329 if (!em) {
Chris Mason890871b2009-09-02 16:24:52 -04004330 write_unlock(&map->lock);
Chris Mason70dec802008-01-29 09:59:12 -05004331 break;
4332 }
Chris Mason7f3c74f2008-07-18 12:01:11 -04004333 if (test_bit(EXTENT_FLAG_PINNED, &em->flags) ||
4334 em->start != start) {
Chris Mason890871b2009-09-02 16:24:52 -04004335 write_unlock(&map->lock);
Chris Mason70dec802008-01-29 09:59:12 -05004336 free_extent_map(em);
4337 break;
4338 }
4339 if (!test_range_bit(tree, em->start,
4340 extent_map_end(em) - 1,
Chris Mason8b62b722009-09-02 16:53:46 -04004341 EXTENT_LOCKED | EXTENT_WRITEBACK,
Chris Mason9655d292009-09-02 15:22:30 -04004342 0, NULL)) {
Chris Mason70dec802008-01-29 09:59:12 -05004343 remove_extent_mapping(map, em);
4344 /* once for the rb tree */
4345 free_extent_map(em);
4346 }
4347 start = extent_map_end(em);
Chris Mason890871b2009-09-02 16:24:52 -04004348 write_unlock(&map->lock);
Chris Mason70dec802008-01-29 09:59:12 -05004349
4350 /* once for us */
Chris Masond1310b22008-01-24 16:13:08 -05004351 free_extent_map(em);
4352 }
Chris Masond1310b22008-01-24 16:13:08 -05004353 }
Chris Mason7b13b7b2008-04-18 10:29:50 -04004354 return try_release_extent_state(map, tree, page, mask);
Chris Masond1310b22008-01-24 16:13:08 -05004355}
Chris Masond1310b22008-01-24 16:13:08 -05004356
Chris Masonec29ed52011-02-23 16:23:20 -05004357/*
4358 * helper function for fiemap, which doesn't want to see any holes.
4359 * This maps until we find something past 'last'
4360 */
4361static struct extent_map *get_extent_skip_holes(struct inode *inode,
4362 u64 offset,
4363 u64 last,
4364 get_extent_t *get_extent)
4365{
4366 u64 sectorsize = BTRFS_I(inode)->root->sectorsize;
4367 struct extent_map *em;
4368 u64 len;
4369
4370 if (offset >= last)
4371 return NULL;
4372
Dulshani Gunawardhana67871252013-10-31 10:33:04 +05304373 while (1) {
Chris Masonec29ed52011-02-23 16:23:20 -05004374 len = last - offset;
4375 if (len == 0)
4376 break;
Qu Wenruofda28322013-02-26 08:10:22 +00004377 len = ALIGN(len, sectorsize);
Chris Masonec29ed52011-02-23 16:23:20 -05004378 em = get_extent(inode, NULL, 0, offset, len, 0);
David Sterbac7040052011-04-19 18:00:01 +02004379 if (IS_ERR_OR_NULL(em))
Chris Masonec29ed52011-02-23 16:23:20 -05004380 return em;
4381
4382 /* if this isn't a hole return it */
4383 if (!test_bit(EXTENT_FLAG_VACANCY, &em->flags) &&
4384 em->block_start != EXTENT_MAP_HOLE) {
4385 return em;
4386 }
4387
4388 /* this is a hole, advance to the next extent */
4389 offset = extent_map_end(em);
4390 free_extent_map(em);
4391 if (offset >= last)
4392 break;
4393 }
4394 return NULL;
4395}
4396
Yehuda Sadeh1506fcc2009-01-21 14:39:14 -05004397int extent_fiemap(struct inode *inode, struct fiemap_extent_info *fieinfo,
4398 __u64 start, __u64 len, get_extent_t *get_extent)
4399{
Josef Bacik975f84f2010-11-23 19:36:57 +00004400 int ret = 0;
Yehuda Sadeh1506fcc2009-01-21 14:39:14 -05004401 u64 off = start;
4402 u64 max = start + len;
4403 u32 flags = 0;
Josef Bacik975f84f2010-11-23 19:36:57 +00004404 u32 found_type;
4405 u64 last;
Chris Masonec29ed52011-02-23 16:23:20 -05004406 u64 last_for_get_extent = 0;
Yehuda Sadeh1506fcc2009-01-21 14:39:14 -05004407 u64 disko = 0;
Chris Masonec29ed52011-02-23 16:23:20 -05004408 u64 isize = i_size_read(inode);
Josef Bacik975f84f2010-11-23 19:36:57 +00004409 struct btrfs_key found_key;
Yehuda Sadeh1506fcc2009-01-21 14:39:14 -05004410 struct extent_map *em = NULL;
Josef Bacik2ac55d42010-02-03 19:33:23 +00004411 struct extent_state *cached_state = NULL;
Josef Bacik975f84f2010-11-23 19:36:57 +00004412 struct btrfs_path *path;
Josef Bacikdc046b12014-09-10 16:20:45 -04004413 struct btrfs_root *root = BTRFS_I(inode)->root;
Yehuda Sadeh1506fcc2009-01-21 14:39:14 -05004414 int end = 0;
Chris Masonec29ed52011-02-23 16:23:20 -05004415 u64 em_start = 0;
4416 u64 em_len = 0;
4417 u64 em_end = 0;
Yehuda Sadeh1506fcc2009-01-21 14:39:14 -05004418
4419 if (len == 0)
4420 return -EINVAL;
4421
Josef Bacik975f84f2010-11-23 19:36:57 +00004422 path = btrfs_alloc_path();
4423 if (!path)
4424 return -ENOMEM;
4425 path->leave_spinning = 1;
4426
Qu Wenruo2c919432014-07-18 09:55:43 +08004427 start = round_down(start, BTRFS_I(inode)->root->sectorsize);
4428 len = round_up(max, BTRFS_I(inode)->root->sectorsize) - start;
Josef Bacik4d479cf2011-11-17 11:34:31 -05004429
Chris Masonec29ed52011-02-23 16:23:20 -05004430 /*
4431 * lookup the last file extent. We're not using i_size here
4432 * because there might be preallocation past i_size
4433 */
Josef Bacikdc046b12014-09-10 16:20:45 -04004434 ret = btrfs_lookup_file_extent(NULL, root, path, btrfs_ino(inode), -1,
4435 0);
Josef Bacik975f84f2010-11-23 19:36:57 +00004436 if (ret < 0) {
4437 btrfs_free_path(path);
4438 return ret;
4439 }
4440 WARN_ON(!ret);
4441 path->slots[0]--;
Josef Bacik975f84f2010-11-23 19:36:57 +00004442 btrfs_item_key_to_cpu(path->nodes[0], &found_key, path->slots[0]);
David Sterba962a2982014-06-04 18:41:45 +02004443 found_type = found_key.type;
Josef Bacik975f84f2010-11-23 19:36:57 +00004444
Chris Masonec29ed52011-02-23 16:23:20 -05004445 /* No extents, but there might be delalloc bits */
Li Zefan33345d012011-04-20 10:31:50 +08004446 if (found_key.objectid != btrfs_ino(inode) ||
Josef Bacik975f84f2010-11-23 19:36:57 +00004447 found_type != BTRFS_EXTENT_DATA_KEY) {
Chris Masonec29ed52011-02-23 16:23:20 -05004448 /* have to trust i_size as the end */
4449 last = (u64)-1;
4450 last_for_get_extent = isize;
4451 } else {
4452 /*
4453 * remember the start of the last extent. There are a
4454 * bunch of different factors that go into the length of the
4455 * extent, so its much less complex to remember where it started
4456 */
4457 last = found_key.offset;
4458 last_for_get_extent = last + 1;
Josef Bacik975f84f2010-11-23 19:36:57 +00004459 }
Liu Bofe09e162013-09-22 12:54:23 +08004460 btrfs_release_path(path);
Josef Bacik975f84f2010-11-23 19:36:57 +00004461
Chris Masonec29ed52011-02-23 16:23:20 -05004462 /*
4463 * we might have some extents allocated but more delalloc past those
4464 * extents. so, we trust isize unless the start of the last extent is
4465 * beyond isize
4466 */
4467 if (last < isize) {
4468 last = (u64)-1;
4469 last_for_get_extent = isize;
4470 }
4471
Liu Boa52f4cd2013-05-01 16:23:41 +00004472 lock_extent_bits(&BTRFS_I(inode)->io_tree, start, start + len - 1, 0,
Jeff Mahoneyd0082372012-03-01 14:57:19 +01004473 &cached_state);
Chris Masonec29ed52011-02-23 16:23:20 -05004474
Josef Bacik4d479cf2011-11-17 11:34:31 -05004475 em = get_extent_skip_holes(inode, start, last_for_get_extent,
Chris Masonec29ed52011-02-23 16:23:20 -05004476 get_extent);
Yehuda Sadeh1506fcc2009-01-21 14:39:14 -05004477 if (!em)
4478 goto out;
4479 if (IS_ERR(em)) {
4480 ret = PTR_ERR(em);
4481 goto out;
4482 }
Josef Bacik975f84f2010-11-23 19:36:57 +00004483
Yehuda Sadeh1506fcc2009-01-21 14:39:14 -05004484 while (!end) {
Josef Bacikb76bb702013-07-05 13:52:51 -04004485 u64 offset_in_extent = 0;
Yehuda Sadeh1506fcc2009-01-21 14:39:14 -05004486
Chris Masonea8efc72011-03-08 11:54:40 -05004487 /* break if the extent we found is outside the range */
4488 if (em->start >= max || extent_map_end(em) < off)
4489 break;
4490
4491 /*
4492 * get_extent may return an extent that starts before our
4493 * requested range. We have to make sure the ranges
4494 * we return to fiemap always move forward and don't
4495 * overlap, so adjust the offsets here
4496 */
4497 em_start = max(em->start, off);
4498
4499 /*
4500 * record the offset from the start of the extent
Josef Bacikb76bb702013-07-05 13:52:51 -04004501 * for adjusting the disk offset below. Only do this if the
4502 * extent isn't compressed since our in ram offset may be past
4503 * what we have actually allocated on disk.
Chris Masonea8efc72011-03-08 11:54:40 -05004504 */
Josef Bacikb76bb702013-07-05 13:52:51 -04004505 if (!test_bit(EXTENT_FLAG_COMPRESSED, &em->flags))
4506 offset_in_extent = em_start - em->start;
Chris Masonec29ed52011-02-23 16:23:20 -05004507 em_end = extent_map_end(em);
Chris Masonea8efc72011-03-08 11:54:40 -05004508 em_len = em_end - em_start;
Yehuda Sadeh1506fcc2009-01-21 14:39:14 -05004509 disko = 0;
4510 flags = 0;
4511
Chris Masonea8efc72011-03-08 11:54:40 -05004512 /*
4513 * bump off for our next call to get_extent
4514 */
4515 off = extent_map_end(em);
4516 if (off >= max)
4517 end = 1;
4518
Heiko Carstens93dbfad2009-04-03 10:33:45 -04004519 if (em->block_start == EXTENT_MAP_LAST_BYTE) {
Yehuda Sadeh1506fcc2009-01-21 14:39:14 -05004520 end = 1;
4521 flags |= FIEMAP_EXTENT_LAST;
Heiko Carstens93dbfad2009-04-03 10:33:45 -04004522 } else if (em->block_start == EXTENT_MAP_INLINE) {
Yehuda Sadeh1506fcc2009-01-21 14:39:14 -05004523 flags |= (FIEMAP_EXTENT_DATA_INLINE |
4524 FIEMAP_EXTENT_NOT_ALIGNED);
Heiko Carstens93dbfad2009-04-03 10:33:45 -04004525 } else if (em->block_start == EXTENT_MAP_DELALLOC) {
Yehuda Sadeh1506fcc2009-01-21 14:39:14 -05004526 flags |= (FIEMAP_EXTENT_DELALLOC |
4527 FIEMAP_EXTENT_UNKNOWN);
Josef Bacikdc046b12014-09-10 16:20:45 -04004528 } else if (fieinfo->fi_extents_max) {
4529 u64 bytenr = em->block_start -
4530 (em->start - em->orig_start);
Liu Bofe09e162013-09-22 12:54:23 +08004531
Chris Masonea8efc72011-03-08 11:54:40 -05004532 disko = em->block_start + offset_in_extent;
Liu Bofe09e162013-09-22 12:54:23 +08004533
4534 /*
4535 * As btrfs supports shared space, this information
4536 * can be exported to userspace tools via
Josef Bacikdc046b12014-09-10 16:20:45 -04004537 * flag FIEMAP_EXTENT_SHARED. If fi_extents_max == 0
4538 * then we're just getting a count and we can skip the
4539 * lookup stuff.
Liu Bofe09e162013-09-22 12:54:23 +08004540 */
Josef Bacikdc046b12014-09-10 16:20:45 -04004541 ret = btrfs_check_shared(NULL, root->fs_info,
4542 root->objectid,
4543 btrfs_ino(inode), bytenr);
4544 if (ret < 0)
Liu Bofe09e162013-09-22 12:54:23 +08004545 goto out_free;
Josef Bacikdc046b12014-09-10 16:20:45 -04004546 if (ret)
Liu Bofe09e162013-09-22 12:54:23 +08004547 flags |= FIEMAP_EXTENT_SHARED;
Josef Bacikdc046b12014-09-10 16:20:45 -04004548 ret = 0;
Yehuda Sadeh1506fcc2009-01-21 14:39:14 -05004549 }
4550 if (test_bit(EXTENT_FLAG_COMPRESSED, &em->flags))
4551 flags |= FIEMAP_EXTENT_ENCODED;
Josef Bacik0d2b2372015-05-19 10:44:04 -04004552 if (test_bit(EXTENT_FLAG_PREALLOC, &em->flags))
4553 flags |= FIEMAP_EXTENT_UNWRITTEN;
Yehuda Sadeh1506fcc2009-01-21 14:39:14 -05004554
Yehuda Sadeh1506fcc2009-01-21 14:39:14 -05004555 free_extent_map(em);
4556 em = NULL;
Chris Masonec29ed52011-02-23 16:23:20 -05004557 if ((em_start >= last) || em_len == (u64)-1 ||
4558 (last == (u64)-1 && isize <= em_end)) {
Yehuda Sadeh1506fcc2009-01-21 14:39:14 -05004559 flags |= FIEMAP_EXTENT_LAST;
4560 end = 1;
4561 }
4562
Chris Masonec29ed52011-02-23 16:23:20 -05004563 /* now scan forward to see if this is really the last extent. */
4564 em = get_extent_skip_holes(inode, off, last_for_get_extent,
4565 get_extent);
4566 if (IS_ERR(em)) {
4567 ret = PTR_ERR(em);
4568 goto out;
4569 }
4570 if (!em) {
Josef Bacik975f84f2010-11-23 19:36:57 +00004571 flags |= FIEMAP_EXTENT_LAST;
4572 end = 1;
4573 }
Chris Masonec29ed52011-02-23 16:23:20 -05004574 ret = fiemap_fill_next_extent(fieinfo, em_start, disko,
4575 em_len, flags);
Chengyu Song26e726a2015-03-24 18:12:56 -04004576 if (ret) {
4577 if (ret == 1)
4578 ret = 0;
Chris Masonec29ed52011-02-23 16:23:20 -05004579 goto out_free;
Chengyu Song26e726a2015-03-24 18:12:56 -04004580 }
Yehuda Sadeh1506fcc2009-01-21 14:39:14 -05004581 }
4582out_free:
4583 free_extent_map(em);
4584out:
Liu Bofe09e162013-09-22 12:54:23 +08004585 btrfs_free_path(path);
Liu Boa52f4cd2013-05-01 16:23:41 +00004586 unlock_extent_cached(&BTRFS_I(inode)->io_tree, start, start + len - 1,
Josef Bacik2ac55d42010-02-03 19:33:23 +00004587 &cached_state, GFP_NOFS);
Yehuda Sadeh1506fcc2009-01-21 14:39:14 -05004588 return ret;
4589}
4590
Chris Mason727011e2010-08-06 13:21:20 -04004591static void __free_extent_buffer(struct extent_buffer *eb)
4592{
Eric Sandeen6d49ba12013-04-22 16:12:31 +00004593 btrfs_leak_debug_del(&eb->leak_list);
Chris Mason727011e2010-08-06 13:21:20 -04004594 kmem_cache_free(extent_buffer_cache, eb);
4595}
4596
Josef Bacika26e8c92014-03-28 17:07:27 -04004597int extent_buffer_under_io(struct extent_buffer *eb)
Chris Masond1310b22008-01-24 16:13:08 -05004598{
Josef Bacik0b32f4b2012-03-13 09:38:00 -04004599 return (atomic_read(&eb->io_pages) ||
4600 test_bit(EXTENT_BUFFER_WRITEBACK, &eb->bflags) ||
4601 test_bit(EXTENT_BUFFER_DIRTY, &eb->bflags));
Chris Masond1310b22008-01-24 16:13:08 -05004602}
4603
Miao Xie897ca6e2010-10-26 20:57:29 -04004604/*
4605 * Helper for releasing extent buffer page.
4606 */
David Sterbaa50924e2014-07-31 00:51:36 +02004607static void btrfs_release_extent_buffer_page(struct extent_buffer *eb)
Miao Xie897ca6e2010-10-26 20:57:29 -04004608{
4609 unsigned long index;
4610 struct page *page;
Jan Schmidt815a51c2012-05-16 17:00:02 +02004611 int mapped = !test_bit(EXTENT_BUFFER_DUMMY, &eb->bflags);
Miao Xie897ca6e2010-10-26 20:57:29 -04004612
Josef Bacik0b32f4b2012-03-13 09:38:00 -04004613 BUG_ON(extent_buffer_under_io(eb));
Miao Xie897ca6e2010-10-26 20:57:29 -04004614
David Sterbaa50924e2014-07-31 00:51:36 +02004615 index = num_extent_pages(eb->start, eb->len);
4616 if (index == 0)
Miao Xie897ca6e2010-10-26 20:57:29 -04004617 return;
4618
4619 do {
4620 index--;
David Sterbafb85fc92014-07-31 01:03:53 +02004621 page = eb->pages[index];
Forrest Liu5d2361d2015-02-09 17:31:45 +08004622 if (!page)
4623 continue;
4624 if (mapped)
Josef Bacik4f2de97a2012-03-07 16:20:05 -05004625 spin_lock(&page->mapping->private_lock);
Forrest Liu5d2361d2015-02-09 17:31:45 +08004626 /*
4627 * We do this since we'll remove the pages after we've
4628 * removed the eb from the radix tree, so we could race
4629 * and have this page now attached to the new eb. So
4630 * only clear page_private if it's still connected to
4631 * this eb.
4632 */
4633 if (PagePrivate(page) &&
4634 page->private == (unsigned long)eb) {
4635 BUG_ON(test_bit(EXTENT_BUFFER_DIRTY, &eb->bflags));
4636 BUG_ON(PageDirty(page));
4637 BUG_ON(PageWriteback(page));
Josef Bacik4f2de97a2012-03-07 16:20:05 -05004638 /*
Forrest Liu5d2361d2015-02-09 17:31:45 +08004639 * We need to make sure we haven't be attached
4640 * to a new eb.
Josef Bacik4f2de97a2012-03-07 16:20:05 -05004641 */
Forrest Liu5d2361d2015-02-09 17:31:45 +08004642 ClearPagePrivate(page);
4643 set_page_private(page, 0);
4644 /* One for the page private */
Miao Xie897ca6e2010-10-26 20:57:29 -04004645 page_cache_release(page);
Josef Bacik4f2de97a2012-03-07 16:20:05 -05004646 }
Forrest Liu5d2361d2015-02-09 17:31:45 +08004647
4648 if (mapped)
4649 spin_unlock(&page->mapping->private_lock);
4650
4651 /* One for when we alloced the page */
4652 page_cache_release(page);
David Sterbaa50924e2014-07-31 00:51:36 +02004653 } while (index != 0);
Miao Xie897ca6e2010-10-26 20:57:29 -04004654}
4655
4656/*
4657 * Helper for releasing the extent buffer.
4658 */
4659static inline void btrfs_release_extent_buffer(struct extent_buffer *eb)
4660{
David Sterbaa50924e2014-07-31 00:51:36 +02004661 btrfs_release_extent_buffer_page(eb);
Miao Xie897ca6e2010-10-26 20:57:29 -04004662 __free_extent_buffer(eb);
4663}
4664
Josef Bacikf28491e2013-12-16 13:24:27 -05004665static struct extent_buffer *
4666__alloc_extent_buffer(struct btrfs_fs_info *fs_info, u64 start,
David Sterba23d79d82014-06-15 02:55:29 +02004667 unsigned long len)
Josef Bacikdb7f3432013-08-07 14:54:37 -04004668{
4669 struct extent_buffer *eb = NULL;
4670
Michal Hockod1b5c562015-08-19 14:17:40 +02004671 eb = kmem_cache_zalloc(extent_buffer_cache, GFP_NOFS|__GFP_NOFAIL);
Josef Bacikdb7f3432013-08-07 14:54:37 -04004672 eb->start = start;
4673 eb->len = len;
Josef Bacikf28491e2013-12-16 13:24:27 -05004674 eb->fs_info = fs_info;
Josef Bacikdb7f3432013-08-07 14:54:37 -04004675 eb->bflags = 0;
4676 rwlock_init(&eb->lock);
4677 atomic_set(&eb->write_locks, 0);
4678 atomic_set(&eb->read_locks, 0);
4679 atomic_set(&eb->blocking_readers, 0);
4680 atomic_set(&eb->blocking_writers, 0);
4681 atomic_set(&eb->spinning_readers, 0);
4682 atomic_set(&eb->spinning_writers, 0);
4683 eb->lock_nested = 0;
4684 init_waitqueue_head(&eb->write_lock_wq);
4685 init_waitqueue_head(&eb->read_lock_wq);
4686
4687 btrfs_leak_debug_add(&eb->leak_list, &buffers);
4688
4689 spin_lock_init(&eb->refs_lock);
4690 atomic_set(&eb->refs, 1);
4691 atomic_set(&eb->io_pages, 0);
4692
4693 /*
4694 * Sanity checks, currently the maximum is 64k covered by 16x 4k pages
4695 */
4696 BUILD_BUG_ON(BTRFS_MAX_METADATA_BLOCKSIZE
4697 > MAX_INLINE_EXTENT_BUFFER_SIZE);
4698 BUG_ON(len > MAX_INLINE_EXTENT_BUFFER_SIZE);
4699
4700 return eb;
4701}
4702
4703struct extent_buffer *btrfs_clone_extent_buffer(struct extent_buffer *src)
4704{
4705 unsigned long i;
4706 struct page *p;
4707 struct extent_buffer *new;
4708 unsigned long num_pages = num_extent_pages(src->start, src->len);
4709
David Sterba3f556f72014-06-15 03:20:26 +02004710 new = __alloc_extent_buffer(src->fs_info, src->start, src->len);
Josef Bacikdb7f3432013-08-07 14:54:37 -04004711 if (new == NULL)
4712 return NULL;
4713
4714 for (i = 0; i < num_pages; i++) {
Josef Bacik9ec72672013-08-07 16:57:23 -04004715 p = alloc_page(GFP_NOFS);
Josef Bacikdb7f3432013-08-07 14:54:37 -04004716 if (!p) {
4717 btrfs_release_extent_buffer(new);
4718 return NULL;
4719 }
4720 attach_extent_buffer_page(new, p);
4721 WARN_ON(PageDirty(p));
4722 SetPageUptodate(p);
4723 new->pages[i] = p;
4724 }
4725
4726 copy_extent_buffer(new, src, 0, 0, src->len);
4727 set_bit(EXTENT_BUFFER_UPTODATE, &new->bflags);
4728 set_bit(EXTENT_BUFFER_DUMMY, &new->bflags);
4729
4730 return new;
4731}
4732
David Sterba3f556f72014-06-15 03:20:26 +02004733struct extent_buffer *alloc_dummy_extent_buffer(struct btrfs_fs_info *fs_info,
4734 u64 start)
Josef Bacikdb7f3432013-08-07 14:54:37 -04004735{
4736 struct extent_buffer *eb;
David Sterba3f556f72014-06-15 03:20:26 +02004737 unsigned long len;
4738 unsigned long num_pages;
Josef Bacikdb7f3432013-08-07 14:54:37 -04004739 unsigned long i;
4740
David Sterba3f556f72014-06-15 03:20:26 +02004741 if (!fs_info) {
4742 /*
4743 * Called only from tests that don't always have a fs_info
4744 * available, but we know that nodesize is 4096
4745 */
4746 len = 4096;
4747 } else {
4748 len = fs_info->tree_root->nodesize;
4749 }
4750 num_pages = num_extent_pages(0, len);
4751
4752 eb = __alloc_extent_buffer(fs_info, start, len);
Josef Bacikdb7f3432013-08-07 14:54:37 -04004753 if (!eb)
4754 return NULL;
4755
4756 for (i = 0; i < num_pages; i++) {
Josef Bacik9ec72672013-08-07 16:57:23 -04004757 eb->pages[i] = alloc_page(GFP_NOFS);
Josef Bacikdb7f3432013-08-07 14:54:37 -04004758 if (!eb->pages[i])
4759 goto err;
4760 }
4761 set_extent_buffer_uptodate(eb);
4762 btrfs_set_header_nritems(eb, 0);
4763 set_bit(EXTENT_BUFFER_DUMMY, &eb->bflags);
4764
4765 return eb;
4766err:
4767 for (; i > 0; i--)
4768 __free_page(eb->pages[i - 1]);
4769 __free_extent_buffer(eb);
4770 return NULL;
4771}
4772
Josef Bacik0b32f4b2012-03-13 09:38:00 -04004773static void check_buffer_tree_ref(struct extent_buffer *eb)
4774{
Chris Mason242e18c2013-01-29 17:49:37 -05004775 int refs;
Josef Bacik0b32f4b2012-03-13 09:38:00 -04004776 /* the ref bit is tricky. We have to make sure it is set
4777 * if we have the buffer dirty. Otherwise the
4778 * code to free a buffer can end up dropping a dirty
4779 * page
4780 *
4781 * Once the ref bit is set, it won't go away while the
4782 * buffer is dirty or in writeback, and it also won't
4783 * go away while we have the reference count on the
4784 * eb bumped.
4785 *
4786 * We can't just set the ref bit without bumping the
4787 * ref on the eb because free_extent_buffer might
4788 * see the ref bit and try to clear it. If this happens
4789 * free_extent_buffer might end up dropping our original
4790 * ref by mistake and freeing the page before we are able
4791 * to add one more ref.
4792 *
4793 * So bump the ref count first, then set the bit. If someone
4794 * beat us to it, drop the ref we added.
4795 */
Chris Mason242e18c2013-01-29 17:49:37 -05004796 refs = atomic_read(&eb->refs);
4797 if (refs >= 2 && test_bit(EXTENT_BUFFER_TREE_REF, &eb->bflags))
4798 return;
4799
Josef Bacik594831c2012-07-20 16:11:08 -04004800 spin_lock(&eb->refs_lock);
4801 if (!test_and_set_bit(EXTENT_BUFFER_TREE_REF, &eb->bflags))
Josef Bacik0b32f4b2012-03-13 09:38:00 -04004802 atomic_inc(&eb->refs);
Josef Bacik594831c2012-07-20 16:11:08 -04004803 spin_unlock(&eb->refs_lock);
Josef Bacik0b32f4b2012-03-13 09:38:00 -04004804}
4805
Mel Gorman2457aec2014-06-04 16:10:31 -07004806static void mark_extent_buffer_accessed(struct extent_buffer *eb,
4807 struct page *accessed)
Josef Bacik5df42352012-03-15 18:24:42 -04004808{
4809 unsigned long num_pages, i;
4810
Josef Bacik0b32f4b2012-03-13 09:38:00 -04004811 check_buffer_tree_ref(eb);
4812
Josef Bacik5df42352012-03-15 18:24:42 -04004813 num_pages = num_extent_pages(eb->start, eb->len);
4814 for (i = 0; i < num_pages; i++) {
David Sterbafb85fc92014-07-31 01:03:53 +02004815 struct page *p = eb->pages[i];
4816
Mel Gorman2457aec2014-06-04 16:10:31 -07004817 if (p != accessed)
4818 mark_page_accessed(p);
Josef Bacik5df42352012-03-15 18:24:42 -04004819 }
4820}
4821
Josef Bacikf28491e2013-12-16 13:24:27 -05004822struct extent_buffer *find_extent_buffer(struct btrfs_fs_info *fs_info,
4823 u64 start)
Chandra Seetharaman452c75c2013-10-07 10:45:25 -05004824{
4825 struct extent_buffer *eb;
4826
4827 rcu_read_lock();
Josef Bacikf28491e2013-12-16 13:24:27 -05004828 eb = radix_tree_lookup(&fs_info->buffer_radix,
4829 start >> PAGE_CACHE_SHIFT);
Chandra Seetharaman452c75c2013-10-07 10:45:25 -05004830 if (eb && atomic_inc_not_zero(&eb->refs)) {
4831 rcu_read_unlock();
Filipe Manana062c19e2015-04-23 11:28:48 +01004832 /*
4833 * Lock our eb's refs_lock to avoid races with
4834 * free_extent_buffer. When we get our eb it might be flagged
4835 * with EXTENT_BUFFER_STALE and another task running
4836 * free_extent_buffer might have seen that flag set,
4837 * eb->refs == 2, that the buffer isn't under IO (dirty and
4838 * writeback flags not set) and it's still in the tree (flag
4839 * EXTENT_BUFFER_TREE_REF set), therefore being in the process
4840 * of decrementing the extent buffer's reference count twice.
4841 * So here we could race and increment the eb's reference count,
4842 * clear its stale flag, mark it as dirty and drop our reference
4843 * before the other task finishes executing free_extent_buffer,
4844 * which would later result in an attempt to free an extent
4845 * buffer that is dirty.
4846 */
4847 if (test_bit(EXTENT_BUFFER_STALE, &eb->bflags)) {
4848 spin_lock(&eb->refs_lock);
4849 spin_unlock(&eb->refs_lock);
4850 }
Mel Gorman2457aec2014-06-04 16:10:31 -07004851 mark_extent_buffer_accessed(eb, NULL);
Chandra Seetharaman452c75c2013-10-07 10:45:25 -05004852 return eb;
4853 }
4854 rcu_read_unlock();
4855
4856 return NULL;
4857}
4858
Josef Bacikfaa2dbf2014-05-07 17:06:09 -04004859#ifdef CONFIG_BTRFS_FS_RUN_SANITY_TESTS
4860struct extent_buffer *alloc_test_extent_buffer(struct btrfs_fs_info *fs_info,
David Sterbace3e6982014-06-15 03:00:04 +02004861 u64 start)
Josef Bacikfaa2dbf2014-05-07 17:06:09 -04004862{
4863 struct extent_buffer *eb, *exists = NULL;
4864 int ret;
4865
4866 eb = find_extent_buffer(fs_info, start);
4867 if (eb)
4868 return eb;
David Sterba3f556f72014-06-15 03:20:26 +02004869 eb = alloc_dummy_extent_buffer(fs_info, start);
Josef Bacikfaa2dbf2014-05-07 17:06:09 -04004870 if (!eb)
4871 return NULL;
4872 eb->fs_info = fs_info;
4873again:
4874 ret = radix_tree_preload(GFP_NOFS & ~__GFP_HIGHMEM);
4875 if (ret)
4876 goto free_eb;
4877 spin_lock(&fs_info->buffer_lock);
4878 ret = radix_tree_insert(&fs_info->buffer_radix,
4879 start >> PAGE_CACHE_SHIFT, eb);
4880 spin_unlock(&fs_info->buffer_lock);
4881 radix_tree_preload_end();
4882 if (ret == -EEXIST) {
4883 exists = find_extent_buffer(fs_info, start);
4884 if (exists)
4885 goto free_eb;
4886 else
4887 goto again;
4888 }
4889 check_buffer_tree_ref(eb);
4890 set_bit(EXTENT_BUFFER_IN_TREE, &eb->bflags);
4891
4892 /*
4893 * We will free dummy extent buffer's if they come into
4894 * free_extent_buffer with a ref count of 2, but if we are using this we
4895 * want the buffers to stay in memory until we're done with them, so
4896 * bump the ref count again.
4897 */
4898 atomic_inc(&eb->refs);
4899 return eb;
4900free_eb:
4901 btrfs_release_extent_buffer(eb);
4902 return exists;
4903}
4904#endif
4905
Josef Bacikf28491e2013-12-16 13:24:27 -05004906struct extent_buffer *alloc_extent_buffer(struct btrfs_fs_info *fs_info,
David Sterbace3e6982014-06-15 03:00:04 +02004907 u64 start)
Chris Masond1310b22008-01-24 16:13:08 -05004908{
David Sterbace3e6982014-06-15 03:00:04 +02004909 unsigned long len = fs_info->tree_root->nodesize;
Chris Masond1310b22008-01-24 16:13:08 -05004910 unsigned long num_pages = num_extent_pages(start, len);
4911 unsigned long i;
4912 unsigned long index = start >> PAGE_CACHE_SHIFT;
4913 struct extent_buffer *eb;
Chris Mason6af118ce2008-07-22 11:18:07 -04004914 struct extent_buffer *exists = NULL;
Chris Masond1310b22008-01-24 16:13:08 -05004915 struct page *p;
Josef Bacikf28491e2013-12-16 13:24:27 -05004916 struct address_space *mapping = fs_info->btree_inode->i_mapping;
Chris Masond1310b22008-01-24 16:13:08 -05004917 int uptodate = 1;
Miao Xie19fe0a82010-10-26 20:57:29 -04004918 int ret;
Chris Masond1310b22008-01-24 16:13:08 -05004919
Josef Bacikf28491e2013-12-16 13:24:27 -05004920 eb = find_extent_buffer(fs_info, start);
Chandra Seetharaman452c75c2013-10-07 10:45:25 -05004921 if (eb)
Chris Mason6af118ce2008-07-22 11:18:07 -04004922 return eb;
Chris Mason6af118ce2008-07-22 11:18:07 -04004923
David Sterba23d79d82014-06-15 02:55:29 +02004924 eb = __alloc_extent_buffer(fs_info, start, len);
Peter2b114d12008-04-01 11:21:40 -04004925 if (!eb)
Chris Masond1310b22008-01-24 16:13:08 -05004926 return NULL;
4927
Chris Mason727011e2010-08-06 13:21:20 -04004928 for (i = 0; i < num_pages; i++, index++) {
Michal Hockod1b5c562015-08-19 14:17:40 +02004929 p = find_or_create_page(mapping, index, GFP_NOFS|__GFP_NOFAIL);
Josef Bacik4804b382012-10-05 16:43:45 -04004930 if (!p)
Chris Mason6af118ce2008-07-22 11:18:07 -04004931 goto free_eb;
Josef Bacik4f2de97a2012-03-07 16:20:05 -05004932
4933 spin_lock(&mapping->private_lock);
4934 if (PagePrivate(p)) {
4935 /*
4936 * We could have already allocated an eb for this page
4937 * and attached one so lets see if we can get a ref on
4938 * the existing eb, and if we can we know it's good and
4939 * we can just return that one, else we know we can just
4940 * overwrite page->private.
4941 */
4942 exists = (struct extent_buffer *)p->private;
4943 if (atomic_inc_not_zero(&exists->refs)) {
4944 spin_unlock(&mapping->private_lock);
4945 unlock_page(p);
Josef Bacik17de39a2012-05-04 15:16:06 -04004946 page_cache_release(p);
Mel Gorman2457aec2014-06-04 16:10:31 -07004947 mark_extent_buffer_accessed(exists, p);
Josef Bacik4f2de97a2012-03-07 16:20:05 -05004948 goto free_eb;
4949 }
Omar Sandoval5ca64f42015-02-24 02:47:05 -08004950 exists = NULL;
Josef Bacik4f2de97a2012-03-07 16:20:05 -05004951
Josef Bacik0b32f4b2012-03-13 09:38:00 -04004952 /*
Josef Bacik4f2de97a2012-03-07 16:20:05 -05004953 * Do this so attach doesn't complain and we need to
4954 * drop the ref the old guy had.
4955 */
4956 ClearPagePrivate(p);
Josef Bacik0b32f4b2012-03-13 09:38:00 -04004957 WARN_ON(PageDirty(p));
Josef Bacik4f2de97a2012-03-07 16:20:05 -05004958 page_cache_release(p);
Chris Masond1310b22008-01-24 16:13:08 -05004959 }
Josef Bacik4f2de97a2012-03-07 16:20:05 -05004960 attach_extent_buffer_page(eb, p);
4961 spin_unlock(&mapping->private_lock);
Josef Bacik0b32f4b2012-03-13 09:38:00 -04004962 WARN_ON(PageDirty(p));
Chris Mason727011e2010-08-06 13:21:20 -04004963 eb->pages[i] = p;
Chris Masond1310b22008-01-24 16:13:08 -05004964 if (!PageUptodate(p))
4965 uptodate = 0;
Chris Masoneb14ab82011-02-10 12:35:00 -05004966
4967 /*
4968 * see below about how we avoid a nasty race with release page
4969 * and why we unlock later
4970 */
Chris Masond1310b22008-01-24 16:13:08 -05004971 }
4972 if (uptodate)
Chris Masonb4ce94d2009-02-04 09:25:08 -05004973 set_bit(EXTENT_BUFFER_UPTODATE, &eb->bflags);
Josef Bacik115391d2012-03-09 09:51:43 -05004974again:
Miao Xie19fe0a82010-10-26 20:57:29 -04004975 ret = radix_tree_preload(GFP_NOFS & ~__GFP_HIGHMEM);
4976 if (ret)
4977 goto free_eb;
4978
Josef Bacikf28491e2013-12-16 13:24:27 -05004979 spin_lock(&fs_info->buffer_lock);
4980 ret = radix_tree_insert(&fs_info->buffer_radix,
4981 start >> PAGE_CACHE_SHIFT, eb);
4982 spin_unlock(&fs_info->buffer_lock);
Chandra Seetharaman452c75c2013-10-07 10:45:25 -05004983 radix_tree_preload_end();
Miao Xie19fe0a82010-10-26 20:57:29 -04004984 if (ret == -EEXIST) {
Josef Bacikf28491e2013-12-16 13:24:27 -05004985 exists = find_extent_buffer(fs_info, start);
Chandra Seetharaman452c75c2013-10-07 10:45:25 -05004986 if (exists)
4987 goto free_eb;
4988 else
Josef Bacik115391d2012-03-09 09:51:43 -05004989 goto again;
Chris Mason6af118ce2008-07-22 11:18:07 -04004990 }
Chris Mason6af118ce2008-07-22 11:18:07 -04004991 /* add one reference for the tree */
Josef Bacik0b32f4b2012-03-13 09:38:00 -04004992 check_buffer_tree_ref(eb);
Josef Bacik34b41ac2013-12-13 10:41:51 -05004993 set_bit(EXTENT_BUFFER_IN_TREE, &eb->bflags);
Chris Masoneb14ab82011-02-10 12:35:00 -05004994
4995 /*
4996 * there is a race where release page may have
4997 * tried to find this extent buffer in the radix
4998 * but failed. It will tell the VM it is safe to
4999 * reclaim the, and it will clear the page private bit.
5000 * We must make sure to set the page private bit properly
5001 * after the extent buffer is in the radix tree so
5002 * it doesn't get lost
5003 */
Chris Mason727011e2010-08-06 13:21:20 -04005004 SetPageChecked(eb->pages[0]);
5005 for (i = 1; i < num_pages; i++) {
David Sterbafb85fc92014-07-31 01:03:53 +02005006 p = eb->pages[i];
Chris Mason727011e2010-08-06 13:21:20 -04005007 ClearPageChecked(p);
5008 unlock_page(p);
5009 }
5010 unlock_page(eb->pages[0]);
Chris Masond1310b22008-01-24 16:13:08 -05005011 return eb;
5012
Chris Mason6af118ce2008-07-22 11:18:07 -04005013free_eb:
Omar Sandoval5ca64f42015-02-24 02:47:05 -08005014 WARN_ON(!atomic_dec_and_test(&eb->refs));
Chris Mason727011e2010-08-06 13:21:20 -04005015 for (i = 0; i < num_pages; i++) {
5016 if (eb->pages[i])
5017 unlock_page(eb->pages[i]);
5018 }
Chris Masoneb14ab82011-02-10 12:35:00 -05005019
Miao Xie897ca6e2010-10-26 20:57:29 -04005020 btrfs_release_extent_buffer(eb);
Chris Mason6af118ce2008-07-22 11:18:07 -04005021 return exists;
Chris Masond1310b22008-01-24 16:13:08 -05005022}
Chris Masond1310b22008-01-24 16:13:08 -05005023
Josef Bacik3083ee22012-03-09 16:01:49 -05005024static inline void btrfs_release_extent_buffer_rcu(struct rcu_head *head)
5025{
5026 struct extent_buffer *eb =
5027 container_of(head, struct extent_buffer, rcu_head);
5028
5029 __free_extent_buffer(eb);
5030}
5031
Josef Bacik3083ee22012-03-09 16:01:49 -05005032/* Expects to have eb->eb_lock already held */
David Sterbaf7a52a42013-04-26 14:56:29 +00005033static int release_extent_buffer(struct extent_buffer *eb)
Josef Bacik3083ee22012-03-09 16:01:49 -05005034{
5035 WARN_ON(atomic_read(&eb->refs) == 0);
5036 if (atomic_dec_and_test(&eb->refs)) {
Josef Bacik34b41ac2013-12-13 10:41:51 -05005037 if (test_and_clear_bit(EXTENT_BUFFER_IN_TREE, &eb->bflags)) {
Josef Bacikf28491e2013-12-16 13:24:27 -05005038 struct btrfs_fs_info *fs_info = eb->fs_info;
Josef Bacik3083ee22012-03-09 16:01:49 -05005039
Jan Schmidt815a51c2012-05-16 17:00:02 +02005040 spin_unlock(&eb->refs_lock);
Josef Bacik3083ee22012-03-09 16:01:49 -05005041
Josef Bacikf28491e2013-12-16 13:24:27 -05005042 spin_lock(&fs_info->buffer_lock);
5043 radix_tree_delete(&fs_info->buffer_radix,
Jan Schmidt815a51c2012-05-16 17:00:02 +02005044 eb->start >> PAGE_CACHE_SHIFT);
Josef Bacikf28491e2013-12-16 13:24:27 -05005045 spin_unlock(&fs_info->buffer_lock);
Josef Bacik34b41ac2013-12-13 10:41:51 -05005046 } else {
5047 spin_unlock(&eb->refs_lock);
Jan Schmidt815a51c2012-05-16 17:00:02 +02005048 }
Josef Bacik3083ee22012-03-09 16:01:49 -05005049
5050 /* Should be safe to release our pages at this point */
David Sterbaa50924e2014-07-31 00:51:36 +02005051 btrfs_release_extent_buffer_page(eb);
Josef Bacikbcb7e442015-03-16 17:38:02 -04005052#ifdef CONFIG_BTRFS_FS_RUN_SANITY_TESTS
5053 if (unlikely(test_bit(EXTENT_BUFFER_DUMMY, &eb->bflags))) {
5054 __free_extent_buffer(eb);
5055 return 1;
5056 }
5057#endif
Josef Bacik3083ee22012-03-09 16:01:49 -05005058 call_rcu(&eb->rcu_head, btrfs_release_extent_buffer_rcu);
Josef Bacike64860a2012-07-20 16:05:36 -04005059 return 1;
Josef Bacik3083ee22012-03-09 16:01:49 -05005060 }
5061 spin_unlock(&eb->refs_lock);
Josef Bacike64860a2012-07-20 16:05:36 -04005062
5063 return 0;
Josef Bacik3083ee22012-03-09 16:01:49 -05005064}
5065
Chris Masond1310b22008-01-24 16:13:08 -05005066void free_extent_buffer(struct extent_buffer *eb)
5067{
Chris Mason242e18c2013-01-29 17:49:37 -05005068 int refs;
5069 int old;
Chris Masond1310b22008-01-24 16:13:08 -05005070 if (!eb)
5071 return;
5072
Chris Mason242e18c2013-01-29 17:49:37 -05005073 while (1) {
5074 refs = atomic_read(&eb->refs);
5075 if (refs <= 3)
5076 break;
5077 old = atomic_cmpxchg(&eb->refs, refs, refs - 1);
5078 if (old == refs)
5079 return;
5080 }
5081
Josef Bacik3083ee22012-03-09 16:01:49 -05005082 spin_lock(&eb->refs_lock);
5083 if (atomic_read(&eb->refs) == 2 &&
Jan Schmidt815a51c2012-05-16 17:00:02 +02005084 test_bit(EXTENT_BUFFER_DUMMY, &eb->bflags))
5085 atomic_dec(&eb->refs);
5086
5087 if (atomic_read(&eb->refs) == 2 &&
Josef Bacik3083ee22012-03-09 16:01:49 -05005088 test_bit(EXTENT_BUFFER_STALE, &eb->bflags) &&
Josef Bacik0b32f4b2012-03-13 09:38:00 -04005089 !extent_buffer_under_io(eb) &&
Josef Bacik3083ee22012-03-09 16:01:49 -05005090 test_and_clear_bit(EXTENT_BUFFER_TREE_REF, &eb->bflags))
5091 atomic_dec(&eb->refs);
Chris Masond1310b22008-01-24 16:13:08 -05005092
Josef Bacik3083ee22012-03-09 16:01:49 -05005093 /*
5094 * I know this is terrible, but it's temporary until we stop tracking
5095 * the uptodate bits and such for the extent buffers.
5096 */
David Sterbaf7a52a42013-04-26 14:56:29 +00005097 release_extent_buffer(eb);
Chris Masond1310b22008-01-24 16:13:08 -05005098}
Chris Masond1310b22008-01-24 16:13:08 -05005099
Josef Bacik3083ee22012-03-09 16:01:49 -05005100void free_extent_buffer_stale(struct extent_buffer *eb)
5101{
5102 if (!eb)
Chris Masond1310b22008-01-24 16:13:08 -05005103 return;
5104
Josef Bacik3083ee22012-03-09 16:01:49 -05005105 spin_lock(&eb->refs_lock);
5106 set_bit(EXTENT_BUFFER_STALE, &eb->bflags);
5107
Josef Bacik0b32f4b2012-03-13 09:38:00 -04005108 if (atomic_read(&eb->refs) == 2 && !extent_buffer_under_io(eb) &&
Josef Bacik3083ee22012-03-09 16:01:49 -05005109 test_and_clear_bit(EXTENT_BUFFER_TREE_REF, &eb->bflags))
5110 atomic_dec(&eb->refs);
David Sterbaf7a52a42013-04-26 14:56:29 +00005111 release_extent_buffer(eb);
Chris Masond1310b22008-01-24 16:13:08 -05005112}
5113
Chris Mason1d4284b2012-03-28 20:31:37 -04005114void clear_extent_buffer_dirty(struct extent_buffer *eb)
Chris Masond1310b22008-01-24 16:13:08 -05005115{
Chris Masond1310b22008-01-24 16:13:08 -05005116 unsigned long i;
5117 unsigned long num_pages;
5118 struct page *page;
5119
Chris Masond1310b22008-01-24 16:13:08 -05005120 num_pages = num_extent_pages(eb->start, eb->len);
5121
5122 for (i = 0; i < num_pages; i++) {
David Sterbafb85fc92014-07-31 01:03:53 +02005123 page = eb->pages[i];
Chris Masonb9473432009-03-13 11:00:37 -04005124 if (!PageDirty(page))
Chris Masond2c3f4f2008-11-19 12:44:22 -05005125 continue;
5126
Chris Masona61e6f22008-07-22 11:18:08 -04005127 lock_page(page);
Chris Masoneb14ab82011-02-10 12:35:00 -05005128 WARN_ON(!PagePrivate(page));
5129
Chris Masond1310b22008-01-24 16:13:08 -05005130 clear_page_dirty_for_io(page);
Sven Wegener0ee0fda2008-07-30 16:54:26 -04005131 spin_lock_irq(&page->mapping->tree_lock);
Chris Masond1310b22008-01-24 16:13:08 -05005132 if (!PageDirty(page)) {
5133 radix_tree_tag_clear(&page->mapping->page_tree,
5134 page_index(page),
5135 PAGECACHE_TAG_DIRTY);
5136 }
Sven Wegener0ee0fda2008-07-30 16:54:26 -04005137 spin_unlock_irq(&page->mapping->tree_lock);
Chris Masonbf0da8c2011-11-04 12:29:37 -04005138 ClearPageError(page);
Chris Masona61e6f22008-07-22 11:18:08 -04005139 unlock_page(page);
Chris Masond1310b22008-01-24 16:13:08 -05005140 }
Josef Bacik0b32f4b2012-03-13 09:38:00 -04005141 WARN_ON(atomic_read(&eb->refs) == 0);
Chris Masond1310b22008-01-24 16:13:08 -05005142}
Chris Masond1310b22008-01-24 16:13:08 -05005143
Josef Bacik0b32f4b2012-03-13 09:38:00 -04005144int set_extent_buffer_dirty(struct extent_buffer *eb)
Chris Masond1310b22008-01-24 16:13:08 -05005145{
5146 unsigned long i;
5147 unsigned long num_pages;
Chris Masonb9473432009-03-13 11:00:37 -04005148 int was_dirty = 0;
Chris Masond1310b22008-01-24 16:13:08 -05005149
Josef Bacik0b32f4b2012-03-13 09:38:00 -04005150 check_buffer_tree_ref(eb);
5151
Chris Masonb9473432009-03-13 11:00:37 -04005152 was_dirty = test_and_set_bit(EXTENT_BUFFER_DIRTY, &eb->bflags);
Josef Bacik0b32f4b2012-03-13 09:38:00 -04005153
Chris Masond1310b22008-01-24 16:13:08 -05005154 num_pages = num_extent_pages(eb->start, eb->len);
Josef Bacik3083ee22012-03-09 16:01:49 -05005155 WARN_ON(atomic_read(&eb->refs) == 0);
Josef Bacik0b32f4b2012-03-13 09:38:00 -04005156 WARN_ON(!test_bit(EXTENT_BUFFER_TREE_REF, &eb->bflags));
5157
Chris Masonb9473432009-03-13 11:00:37 -04005158 for (i = 0; i < num_pages; i++)
David Sterbafb85fc92014-07-31 01:03:53 +02005159 set_page_dirty(eb->pages[i]);
Chris Masonb9473432009-03-13 11:00:37 -04005160 return was_dirty;
Chris Masond1310b22008-01-24 16:13:08 -05005161}
Chris Masond1310b22008-01-24 16:13:08 -05005162
Josef Bacik0b32f4b2012-03-13 09:38:00 -04005163int clear_extent_buffer_uptodate(struct extent_buffer *eb)
Chris Mason1259ab72008-05-12 13:39:03 -04005164{
5165 unsigned long i;
5166 struct page *page;
5167 unsigned long num_pages;
5168
Chris Masonb4ce94d2009-02-04 09:25:08 -05005169 clear_bit(EXTENT_BUFFER_UPTODATE, &eb->bflags);
Josef Bacik0b32f4b2012-03-13 09:38:00 -04005170 num_pages = num_extent_pages(eb->start, eb->len);
Chris Mason1259ab72008-05-12 13:39:03 -04005171 for (i = 0; i < num_pages; i++) {
David Sterbafb85fc92014-07-31 01:03:53 +02005172 page = eb->pages[i];
Chris Mason33958dc2008-07-30 10:29:12 -04005173 if (page)
5174 ClearPageUptodate(page);
Chris Mason1259ab72008-05-12 13:39:03 -04005175 }
5176 return 0;
5177}
5178
Josef Bacik0b32f4b2012-03-13 09:38:00 -04005179int set_extent_buffer_uptodate(struct extent_buffer *eb)
Chris Masond1310b22008-01-24 16:13:08 -05005180{
5181 unsigned long i;
5182 struct page *page;
5183 unsigned long num_pages;
5184
Josef Bacik0b32f4b2012-03-13 09:38:00 -04005185 set_bit(EXTENT_BUFFER_UPTODATE, &eb->bflags);
Chris Masond1310b22008-01-24 16:13:08 -05005186 num_pages = num_extent_pages(eb->start, eb->len);
Chris Masond1310b22008-01-24 16:13:08 -05005187 for (i = 0; i < num_pages; i++) {
David Sterbafb85fc92014-07-31 01:03:53 +02005188 page = eb->pages[i];
Chris Masond1310b22008-01-24 16:13:08 -05005189 SetPageUptodate(page);
5190 }
5191 return 0;
5192}
Chris Masond1310b22008-01-24 16:13:08 -05005193
Josef Bacik0b32f4b2012-03-13 09:38:00 -04005194int extent_buffer_uptodate(struct extent_buffer *eb)
Chris Masond1310b22008-01-24 16:13:08 -05005195{
Josef Bacik0b32f4b2012-03-13 09:38:00 -04005196 return test_bit(EXTENT_BUFFER_UPTODATE, &eb->bflags);
Chris Masond1310b22008-01-24 16:13:08 -05005197}
Chris Masond1310b22008-01-24 16:13:08 -05005198
5199int read_extent_buffer_pages(struct extent_io_tree *tree,
Arne Jansenbb82ab82011-06-10 14:06:53 +02005200 struct extent_buffer *eb, u64 start, int wait,
Chris Masonf1885912008-04-09 16:28:12 -04005201 get_extent_t *get_extent, int mirror_num)
Chris Masond1310b22008-01-24 16:13:08 -05005202{
5203 unsigned long i;
5204 unsigned long start_i;
5205 struct page *page;
5206 int err;
5207 int ret = 0;
Chris Masonce9adaa2008-04-09 16:28:12 -04005208 int locked_pages = 0;
5209 int all_uptodate = 1;
Chris Masond1310b22008-01-24 16:13:08 -05005210 unsigned long num_pages;
Chris Mason727011e2010-08-06 13:21:20 -04005211 unsigned long num_reads = 0;
Chris Masona86c12c2008-02-07 10:50:54 -05005212 struct bio *bio = NULL;
Chris Masonc8b97812008-10-29 14:49:59 -04005213 unsigned long bio_flags = 0;
Chris Masona86c12c2008-02-07 10:50:54 -05005214
Chris Masonb4ce94d2009-02-04 09:25:08 -05005215 if (test_bit(EXTENT_BUFFER_UPTODATE, &eb->bflags))
Chris Masond1310b22008-01-24 16:13:08 -05005216 return 0;
5217
Chris Masond1310b22008-01-24 16:13:08 -05005218 if (start) {
5219 WARN_ON(start < eb->start);
5220 start_i = (start >> PAGE_CACHE_SHIFT) -
5221 (eb->start >> PAGE_CACHE_SHIFT);
5222 } else {
5223 start_i = 0;
5224 }
5225
5226 num_pages = num_extent_pages(eb->start, eb->len);
5227 for (i = start_i; i < num_pages; i++) {
David Sterbafb85fc92014-07-31 01:03:53 +02005228 page = eb->pages[i];
Arne Jansenbb82ab82011-06-10 14:06:53 +02005229 if (wait == WAIT_NONE) {
David Woodhouse2db04962008-08-07 11:19:43 -04005230 if (!trylock_page(page))
Chris Masonce9adaa2008-04-09 16:28:12 -04005231 goto unlock_exit;
Chris Masond1310b22008-01-24 16:13:08 -05005232 } else {
5233 lock_page(page);
5234 }
Chris Masonce9adaa2008-04-09 16:28:12 -04005235 locked_pages++;
Chris Mason727011e2010-08-06 13:21:20 -04005236 if (!PageUptodate(page)) {
5237 num_reads++;
Chris Masonce9adaa2008-04-09 16:28:12 -04005238 all_uptodate = 0;
Chris Mason727011e2010-08-06 13:21:20 -04005239 }
Chris Masonce9adaa2008-04-09 16:28:12 -04005240 }
5241 if (all_uptodate) {
5242 if (start_i == 0)
Chris Masonb4ce94d2009-02-04 09:25:08 -05005243 set_bit(EXTENT_BUFFER_UPTODATE, &eb->bflags);
Chris Masonce9adaa2008-04-09 16:28:12 -04005244 goto unlock_exit;
5245 }
5246
Filipe Manana656f30d2014-09-26 12:25:56 +01005247 clear_bit(EXTENT_BUFFER_READ_ERR, &eb->bflags);
Josef Bacik5cf1ab52012-04-16 09:42:26 -04005248 eb->read_mirror = 0;
Josef Bacik0b32f4b2012-03-13 09:38:00 -04005249 atomic_set(&eb->io_pages, num_reads);
Chris Masonce9adaa2008-04-09 16:28:12 -04005250 for (i = start_i; i < num_pages; i++) {
David Sterbafb85fc92014-07-31 01:03:53 +02005251 page = eb->pages[i];
Chris Masonce9adaa2008-04-09 16:28:12 -04005252 if (!PageUptodate(page)) {
Chris Masonf1885912008-04-09 16:28:12 -04005253 ClearPageError(page);
Chris Masona86c12c2008-02-07 10:50:54 -05005254 err = __extent_read_full_page(tree, page,
Chris Masonf1885912008-04-09 16:28:12 -04005255 get_extent, &bio,
Josef Bacikd4c7ca82013-04-19 19:49:09 -04005256 mirror_num, &bio_flags,
5257 READ | REQ_META);
Chris Masond3977122009-01-05 21:25:51 -05005258 if (err)
Chris Masond1310b22008-01-24 16:13:08 -05005259 ret = err;
Chris Masond1310b22008-01-24 16:13:08 -05005260 } else {
5261 unlock_page(page);
5262 }
5263 }
5264
Jeff Mahoney355808c2011-10-03 23:23:14 -04005265 if (bio) {
Josef Bacikd4c7ca82013-04-19 19:49:09 -04005266 err = submit_one_bio(READ | REQ_META, bio, mirror_num,
5267 bio_flags);
Jeff Mahoney79787ea2012-03-12 16:03:00 +01005268 if (err)
5269 return err;
Jeff Mahoney355808c2011-10-03 23:23:14 -04005270 }
Chris Masona86c12c2008-02-07 10:50:54 -05005271
Arne Jansenbb82ab82011-06-10 14:06:53 +02005272 if (ret || wait != WAIT_COMPLETE)
Chris Masond1310b22008-01-24 16:13:08 -05005273 return ret;
Chris Masond3977122009-01-05 21:25:51 -05005274
Chris Masond1310b22008-01-24 16:13:08 -05005275 for (i = start_i; i < num_pages; i++) {
David Sterbafb85fc92014-07-31 01:03:53 +02005276 page = eb->pages[i];
Chris Masond1310b22008-01-24 16:13:08 -05005277 wait_on_page_locked(page);
Chris Masond3977122009-01-05 21:25:51 -05005278 if (!PageUptodate(page))
Chris Masond1310b22008-01-24 16:13:08 -05005279 ret = -EIO;
Chris Masond1310b22008-01-24 16:13:08 -05005280 }
Chris Masond3977122009-01-05 21:25:51 -05005281
Chris Masond1310b22008-01-24 16:13:08 -05005282 return ret;
Chris Masonce9adaa2008-04-09 16:28:12 -04005283
5284unlock_exit:
5285 i = start_i;
Chris Masond3977122009-01-05 21:25:51 -05005286 while (locked_pages > 0) {
David Sterbafb85fc92014-07-31 01:03:53 +02005287 page = eb->pages[i];
Chris Masonce9adaa2008-04-09 16:28:12 -04005288 i++;
5289 unlock_page(page);
5290 locked_pages--;
5291 }
5292 return ret;
Chris Masond1310b22008-01-24 16:13:08 -05005293}
Chris Masond1310b22008-01-24 16:13:08 -05005294
5295void read_extent_buffer(struct extent_buffer *eb, void *dstv,
5296 unsigned long start,
5297 unsigned long len)
5298{
5299 size_t cur;
5300 size_t offset;
5301 struct page *page;
5302 char *kaddr;
5303 char *dst = (char *)dstv;
5304 size_t start_offset = eb->start & ((u64)PAGE_CACHE_SIZE - 1);
5305 unsigned long i = (start_offset + start) >> PAGE_CACHE_SHIFT;
Chris Masond1310b22008-01-24 16:13:08 -05005306
5307 WARN_ON(start > eb->len);
5308 WARN_ON(start + len > eb->start + eb->len);
5309
Geert Uytterhoeven778746b2013-08-20 13:20:16 +02005310 offset = (start_offset + start) & (PAGE_CACHE_SIZE - 1);
Chris Masond1310b22008-01-24 16:13:08 -05005311
Chris Masond3977122009-01-05 21:25:51 -05005312 while (len > 0) {
David Sterbafb85fc92014-07-31 01:03:53 +02005313 page = eb->pages[i];
Chris Masond1310b22008-01-24 16:13:08 -05005314
5315 cur = min(len, (PAGE_CACHE_SIZE - offset));
Chris Masona6591712011-07-19 12:04:14 -04005316 kaddr = page_address(page);
Chris Masond1310b22008-01-24 16:13:08 -05005317 memcpy(dst, kaddr + offset, cur);
Chris Masond1310b22008-01-24 16:13:08 -05005318
5319 dst += cur;
5320 len -= cur;
5321 offset = 0;
5322 i++;
5323 }
5324}
Chris Masond1310b22008-01-24 16:13:08 -05005325
Gerhard Heift550ac1d2014-01-30 16:24:01 +01005326int read_extent_buffer_to_user(struct extent_buffer *eb, void __user *dstv,
5327 unsigned long start,
5328 unsigned long len)
5329{
5330 size_t cur;
5331 size_t offset;
5332 struct page *page;
5333 char *kaddr;
5334 char __user *dst = (char __user *)dstv;
5335 size_t start_offset = eb->start & ((u64)PAGE_CACHE_SIZE - 1);
5336 unsigned long i = (start_offset + start) >> PAGE_CACHE_SHIFT;
5337 int ret = 0;
5338
5339 WARN_ON(start > eb->len);
5340 WARN_ON(start + len > eb->start + eb->len);
5341
5342 offset = (start_offset + start) & (PAGE_CACHE_SIZE - 1);
5343
5344 while (len > 0) {
David Sterbafb85fc92014-07-31 01:03:53 +02005345 page = eb->pages[i];
Gerhard Heift550ac1d2014-01-30 16:24:01 +01005346
5347 cur = min(len, (PAGE_CACHE_SIZE - offset));
5348 kaddr = page_address(page);
5349 if (copy_to_user(dst, kaddr + offset, cur)) {
5350 ret = -EFAULT;
5351 break;
5352 }
5353
5354 dst += cur;
5355 len -= cur;
5356 offset = 0;
5357 i++;
5358 }
5359
5360 return ret;
5361}
5362
Chris Masond1310b22008-01-24 16:13:08 -05005363int map_private_extent_buffer(struct extent_buffer *eb, unsigned long start,
Chris Masona6591712011-07-19 12:04:14 -04005364 unsigned long min_len, char **map,
Chris Masond1310b22008-01-24 16:13:08 -05005365 unsigned long *map_start,
Chris Masona6591712011-07-19 12:04:14 -04005366 unsigned long *map_len)
Chris Masond1310b22008-01-24 16:13:08 -05005367{
5368 size_t offset = start & (PAGE_CACHE_SIZE - 1);
5369 char *kaddr;
5370 struct page *p;
5371 size_t start_offset = eb->start & ((u64)PAGE_CACHE_SIZE - 1);
5372 unsigned long i = (start_offset + start) >> PAGE_CACHE_SHIFT;
5373 unsigned long end_i = (start_offset + start + min_len - 1) >>
5374 PAGE_CACHE_SHIFT;
5375
5376 if (i != end_i)
5377 return -EINVAL;
5378
5379 if (i == 0) {
5380 offset = start_offset;
5381 *map_start = 0;
5382 } else {
5383 offset = 0;
5384 *map_start = ((u64)i << PAGE_CACHE_SHIFT) - start_offset;
5385 }
Chris Masond3977122009-01-05 21:25:51 -05005386
Chris Masond1310b22008-01-24 16:13:08 -05005387 if (start + min_len > eb->len) {
Julia Lawall31b1a2b2012-11-03 10:58:34 +00005388 WARN(1, KERN_ERR "btrfs bad mapping eb start %llu len %lu, "
Geert Uytterhoevenc1c9ff72013-08-20 13:20:07 +02005389 "wanted %lu %lu\n",
5390 eb->start, eb->len, start, min_len);
Josef Bacik850265332011-03-15 14:52:12 -04005391 return -EINVAL;
Chris Masond1310b22008-01-24 16:13:08 -05005392 }
5393
David Sterbafb85fc92014-07-31 01:03:53 +02005394 p = eb->pages[i];
Chris Masona6591712011-07-19 12:04:14 -04005395 kaddr = page_address(p);
Chris Masond1310b22008-01-24 16:13:08 -05005396 *map = kaddr + offset;
5397 *map_len = PAGE_CACHE_SIZE - offset;
5398 return 0;
5399}
Chris Masond1310b22008-01-24 16:13:08 -05005400
Chris Masond1310b22008-01-24 16:13:08 -05005401int memcmp_extent_buffer(struct extent_buffer *eb, const void *ptrv,
5402 unsigned long start,
5403 unsigned long len)
5404{
5405 size_t cur;
5406 size_t offset;
5407 struct page *page;
5408 char *kaddr;
5409 char *ptr = (char *)ptrv;
5410 size_t start_offset = eb->start & ((u64)PAGE_CACHE_SIZE - 1);
5411 unsigned long i = (start_offset + start) >> PAGE_CACHE_SHIFT;
5412 int ret = 0;
5413
5414 WARN_ON(start > eb->len);
5415 WARN_ON(start + len > eb->start + eb->len);
5416
Geert Uytterhoeven778746b2013-08-20 13:20:16 +02005417 offset = (start_offset + start) & (PAGE_CACHE_SIZE - 1);
Chris Masond1310b22008-01-24 16:13:08 -05005418
Chris Masond3977122009-01-05 21:25:51 -05005419 while (len > 0) {
David Sterbafb85fc92014-07-31 01:03:53 +02005420 page = eb->pages[i];
Chris Masond1310b22008-01-24 16:13:08 -05005421
5422 cur = min(len, (PAGE_CACHE_SIZE - offset));
5423
Chris Masona6591712011-07-19 12:04:14 -04005424 kaddr = page_address(page);
Chris Masond1310b22008-01-24 16:13:08 -05005425 ret = memcmp(ptr, kaddr + offset, cur);
Chris Masond1310b22008-01-24 16:13:08 -05005426 if (ret)
5427 break;
5428
5429 ptr += cur;
5430 len -= cur;
5431 offset = 0;
5432 i++;
5433 }
5434 return ret;
5435}
Chris Masond1310b22008-01-24 16:13:08 -05005436
5437void write_extent_buffer(struct extent_buffer *eb, const void *srcv,
5438 unsigned long start, unsigned long len)
5439{
5440 size_t cur;
5441 size_t offset;
5442 struct page *page;
5443 char *kaddr;
5444 char *src = (char *)srcv;
5445 size_t start_offset = eb->start & ((u64)PAGE_CACHE_SIZE - 1);
5446 unsigned long i = (start_offset + start) >> PAGE_CACHE_SHIFT;
5447
5448 WARN_ON(start > eb->len);
5449 WARN_ON(start + len > eb->start + eb->len);
5450
Geert Uytterhoeven778746b2013-08-20 13:20:16 +02005451 offset = (start_offset + start) & (PAGE_CACHE_SIZE - 1);
Chris Masond1310b22008-01-24 16:13:08 -05005452
Chris Masond3977122009-01-05 21:25:51 -05005453 while (len > 0) {
David Sterbafb85fc92014-07-31 01:03:53 +02005454 page = eb->pages[i];
Chris Masond1310b22008-01-24 16:13:08 -05005455 WARN_ON(!PageUptodate(page));
5456
5457 cur = min(len, PAGE_CACHE_SIZE - offset);
Chris Masona6591712011-07-19 12:04:14 -04005458 kaddr = page_address(page);
Chris Masond1310b22008-01-24 16:13:08 -05005459 memcpy(kaddr + offset, src, cur);
Chris Masond1310b22008-01-24 16:13:08 -05005460
5461 src += cur;
5462 len -= cur;
5463 offset = 0;
5464 i++;
5465 }
5466}
Chris Masond1310b22008-01-24 16:13:08 -05005467
5468void memset_extent_buffer(struct extent_buffer *eb, char c,
5469 unsigned long start, unsigned long len)
5470{
5471 size_t cur;
5472 size_t offset;
5473 struct page *page;
5474 char *kaddr;
5475 size_t start_offset = eb->start & ((u64)PAGE_CACHE_SIZE - 1);
5476 unsigned long i = (start_offset + start) >> PAGE_CACHE_SHIFT;
5477
5478 WARN_ON(start > eb->len);
5479 WARN_ON(start + len > eb->start + eb->len);
5480
Geert Uytterhoeven778746b2013-08-20 13:20:16 +02005481 offset = (start_offset + start) & (PAGE_CACHE_SIZE - 1);
Chris Masond1310b22008-01-24 16:13:08 -05005482
Chris Masond3977122009-01-05 21:25:51 -05005483 while (len > 0) {
David Sterbafb85fc92014-07-31 01:03:53 +02005484 page = eb->pages[i];
Chris Masond1310b22008-01-24 16:13:08 -05005485 WARN_ON(!PageUptodate(page));
5486
5487 cur = min(len, PAGE_CACHE_SIZE - offset);
Chris Masona6591712011-07-19 12:04:14 -04005488 kaddr = page_address(page);
Chris Masond1310b22008-01-24 16:13:08 -05005489 memset(kaddr + offset, c, cur);
Chris Masond1310b22008-01-24 16:13:08 -05005490
5491 len -= cur;
5492 offset = 0;
5493 i++;
5494 }
5495}
Chris Masond1310b22008-01-24 16:13:08 -05005496
5497void copy_extent_buffer(struct extent_buffer *dst, struct extent_buffer *src,
5498 unsigned long dst_offset, unsigned long src_offset,
5499 unsigned long len)
5500{
5501 u64 dst_len = dst->len;
5502 size_t cur;
5503 size_t offset;
5504 struct page *page;
5505 char *kaddr;
5506 size_t start_offset = dst->start & ((u64)PAGE_CACHE_SIZE - 1);
5507 unsigned long i = (start_offset + dst_offset) >> PAGE_CACHE_SHIFT;
5508
5509 WARN_ON(src->len != dst_len);
5510
5511 offset = (start_offset + dst_offset) &
Geert Uytterhoeven778746b2013-08-20 13:20:16 +02005512 (PAGE_CACHE_SIZE - 1);
Chris Masond1310b22008-01-24 16:13:08 -05005513
Chris Masond3977122009-01-05 21:25:51 -05005514 while (len > 0) {
David Sterbafb85fc92014-07-31 01:03:53 +02005515 page = dst->pages[i];
Chris Masond1310b22008-01-24 16:13:08 -05005516 WARN_ON(!PageUptodate(page));
5517
5518 cur = min(len, (unsigned long)(PAGE_CACHE_SIZE - offset));
5519
Chris Masona6591712011-07-19 12:04:14 -04005520 kaddr = page_address(page);
Chris Masond1310b22008-01-24 16:13:08 -05005521 read_extent_buffer(src, kaddr + offset, src_offset, cur);
Chris Masond1310b22008-01-24 16:13:08 -05005522
5523 src_offset += cur;
5524 len -= cur;
5525 offset = 0;
5526 i++;
5527 }
5528}
Chris Masond1310b22008-01-24 16:13:08 -05005529
Sergei Trofimovich33872062011-04-11 21:52:52 +00005530static inline bool areas_overlap(unsigned long src, unsigned long dst, unsigned long len)
5531{
5532 unsigned long distance = (src > dst) ? src - dst : dst - src;
5533 return distance < len;
5534}
5535
Chris Masond1310b22008-01-24 16:13:08 -05005536static void copy_pages(struct page *dst_page, struct page *src_page,
5537 unsigned long dst_off, unsigned long src_off,
5538 unsigned long len)
5539{
Chris Masona6591712011-07-19 12:04:14 -04005540 char *dst_kaddr = page_address(dst_page);
Chris Masond1310b22008-01-24 16:13:08 -05005541 char *src_kaddr;
Chris Mason727011e2010-08-06 13:21:20 -04005542 int must_memmove = 0;
Chris Masond1310b22008-01-24 16:13:08 -05005543
Sergei Trofimovich33872062011-04-11 21:52:52 +00005544 if (dst_page != src_page) {
Chris Masona6591712011-07-19 12:04:14 -04005545 src_kaddr = page_address(src_page);
Sergei Trofimovich33872062011-04-11 21:52:52 +00005546 } else {
Chris Masond1310b22008-01-24 16:13:08 -05005547 src_kaddr = dst_kaddr;
Chris Mason727011e2010-08-06 13:21:20 -04005548 if (areas_overlap(src_off, dst_off, len))
5549 must_memmove = 1;
Sergei Trofimovich33872062011-04-11 21:52:52 +00005550 }
Chris Masond1310b22008-01-24 16:13:08 -05005551
Chris Mason727011e2010-08-06 13:21:20 -04005552 if (must_memmove)
5553 memmove(dst_kaddr + dst_off, src_kaddr + src_off, len);
5554 else
5555 memcpy(dst_kaddr + dst_off, src_kaddr + src_off, len);
Chris Masond1310b22008-01-24 16:13:08 -05005556}
5557
5558void memcpy_extent_buffer(struct extent_buffer *dst, unsigned long dst_offset,
5559 unsigned long src_offset, unsigned long len)
5560{
5561 size_t cur;
5562 size_t dst_off_in_page;
5563 size_t src_off_in_page;
5564 size_t start_offset = dst->start & ((u64)PAGE_CACHE_SIZE - 1);
5565 unsigned long dst_i;
5566 unsigned long src_i;
5567
5568 if (src_offset + len > dst->len) {
Frank Holtonefe120a2013-12-20 11:37:06 -05005569 printk(KERN_ERR "BTRFS: memmove bogus src_offset %lu move "
Chris Masond3977122009-01-05 21:25:51 -05005570 "len %lu dst len %lu\n", src_offset, len, dst->len);
Chris Masond1310b22008-01-24 16:13:08 -05005571 BUG_ON(1);
5572 }
5573 if (dst_offset + len > dst->len) {
Frank Holtonefe120a2013-12-20 11:37:06 -05005574 printk(KERN_ERR "BTRFS: memmove bogus dst_offset %lu move "
Chris Masond3977122009-01-05 21:25:51 -05005575 "len %lu dst len %lu\n", dst_offset, len, dst->len);
Chris Masond1310b22008-01-24 16:13:08 -05005576 BUG_ON(1);
5577 }
5578
Chris Masond3977122009-01-05 21:25:51 -05005579 while (len > 0) {
Chris Masond1310b22008-01-24 16:13:08 -05005580 dst_off_in_page = (start_offset + dst_offset) &
Geert Uytterhoeven778746b2013-08-20 13:20:16 +02005581 (PAGE_CACHE_SIZE - 1);
Chris Masond1310b22008-01-24 16:13:08 -05005582 src_off_in_page = (start_offset + src_offset) &
Geert Uytterhoeven778746b2013-08-20 13:20:16 +02005583 (PAGE_CACHE_SIZE - 1);
Chris Masond1310b22008-01-24 16:13:08 -05005584
5585 dst_i = (start_offset + dst_offset) >> PAGE_CACHE_SHIFT;
5586 src_i = (start_offset + src_offset) >> PAGE_CACHE_SHIFT;
5587
5588 cur = min(len, (unsigned long)(PAGE_CACHE_SIZE -
5589 src_off_in_page));
5590 cur = min_t(unsigned long, cur,
5591 (unsigned long)(PAGE_CACHE_SIZE - dst_off_in_page));
5592
David Sterbafb85fc92014-07-31 01:03:53 +02005593 copy_pages(dst->pages[dst_i], dst->pages[src_i],
Chris Masond1310b22008-01-24 16:13:08 -05005594 dst_off_in_page, src_off_in_page, cur);
5595
5596 src_offset += cur;
5597 dst_offset += cur;
5598 len -= cur;
5599 }
5600}
Chris Masond1310b22008-01-24 16:13:08 -05005601
5602void memmove_extent_buffer(struct extent_buffer *dst, unsigned long dst_offset,
5603 unsigned long src_offset, unsigned long len)
5604{
5605 size_t cur;
5606 size_t dst_off_in_page;
5607 size_t src_off_in_page;
5608 unsigned long dst_end = dst_offset + len - 1;
5609 unsigned long src_end = src_offset + len - 1;
5610 size_t start_offset = dst->start & ((u64)PAGE_CACHE_SIZE - 1);
5611 unsigned long dst_i;
5612 unsigned long src_i;
5613
5614 if (src_offset + len > dst->len) {
Frank Holtonefe120a2013-12-20 11:37:06 -05005615 printk(KERN_ERR "BTRFS: memmove bogus src_offset %lu move "
Chris Masond3977122009-01-05 21:25:51 -05005616 "len %lu len %lu\n", src_offset, len, dst->len);
Chris Masond1310b22008-01-24 16:13:08 -05005617 BUG_ON(1);
5618 }
5619 if (dst_offset + len > dst->len) {
Frank Holtonefe120a2013-12-20 11:37:06 -05005620 printk(KERN_ERR "BTRFS: memmove bogus dst_offset %lu move "
Chris Masond3977122009-01-05 21:25:51 -05005621 "len %lu len %lu\n", dst_offset, len, dst->len);
Chris Masond1310b22008-01-24 16:13:08 -05005622 BUG_ON(1);
5623 }
Chris Mason727011e2010-08-06 13:21:20 -04005624 if (dst_offset < src_offset) {
Chris Masond1310b22008-01-24 16:13:08 -05005625 memcpy_extent_buffer(dst, dst_offset, src_offset, len);
5626 return;
5627 }
Chris Masond3977122009-01-05 21:25:51 -05005628 while (len > 0) {
Chris Masond1310b22008-01-24 16:13:08 -05005629 dst_i = (start_offset + dst_end) >> PAGE_CACHE_SHIFT;
5630 src_i = (start_offset + src_end) >> PAGE_CACHE_SHIFT;
5631
5632 dst_off_in_page = (start_offset + dst_end) &
Geert Uytterhoeven778746b2013-08-20 13:20:16 +02005633 (PAGE_CACHE_SIZE - 1);
Chris Masond1310b22008-01-24 16:13:08 -05005634 src_off_in_page = (start_offset + src_end) &
Geert Uytterhoeven778746b2013-08-20 13:20:16 +02005635 (PAGE_CACHE_SIZE - 1);
Chris Masond1310b22008-01-24 16:13:08 -05005636
5637 cur = min_t(unsigned long, len, src_off_in_page + 1);
5638 cur = min(cur, dst_off_in_page + 1);
David Sterbafb85fc92014-07-31 01:03:53 +02005639 copy_pages(dst->pages[dst_i], dst->pages[src_i],
Chris Masond1310b22008-01-24 16:13:08 -05005640 dst_off_in_page - cur + 1,
5641 src_off_in_page - cur + 1, cur);
5642
5643 dst_end -= cur;
5644 src_end -= cur;
5645 len -= cur;
5646 }
5647}
Chris Mason6af118ce2008-07-22 11:18:07 -04005648
David Sterbaf7a52a42013-04-26 14:56:29 +00005649int try_release_extent_buffer(struct page *page)
Miao Xie19fe0a82010-10-26 20:57:29 -04005650{
Chris Mason6af118ce2008-07-22 11:18:07 -04005651 struct extent_buffer *eb;
Miao Xie897ca6e2010-10-26 20:57:29 -04005652
Miao Xie19fe0a82010-10-26 20:57:29 -04005653 /*
Josef Bacik3083ee22012-03-09 16:01:49 -05005654 * We need to make sure noboody is attaching this page to an eb right
5655 * now.
Miao Xie19fe0a82010-10-26 20:57:29 -04005656 */
Josef Bacik3083ee22012-03-09 16:01:49 -05005657 spin_lock(&page->mapping->private_lock);
5658 if (!PagePrivate(page)) {
5659 spin_unlock(&page->mapping->private_lock);
5660 return 1;
Miao Xie19fe0a82010-10-26 20:57:29 -04005661 }
5662
Josef Bacik3083ee22012-03-09 16:01:49 -05005663 eb = (struct extent_buffer *)page->private;
5664 BUG_ON(!eb);
Miao Xie19fe0a82010-10-26 20:57:29 -04005665
Josef Bacik0b32f4b2012-03-13 09:38:00 -04005666 /*
Josef Bacik3083ee22012-03-09 16:01:49 -05005667 * This is a little awful but should be ok, we need to make sure that
5668 * the eb doesn't disappear out from under us while we're looking at
5669 * this page.
5670 */
5671 spin_lock(&eb->refs_lock);
Josef Bacik0b32f4b2012-03-13 09:38:00 -04005672 if (atomic_read(&eb->refs) != 1 || extent_buffer_under_io(eb)) {
Josef Bacik3083ee22012-03-09 16:01:49 -05005673 spin_unlock(&eb->refs_lock);
5674 spin_unlock(&page->mapping->private_lock);
5675 return 0;
5676 }
5677 spin_unlock(&page->mapping->private_lock);
5678
Josef Bacik3083ee22012-03-09 16:01:49 -05005679 /*
5680 * If tree ref isn't set then we know the ref on this eb is a real ref,
5681 * so just return, this page will likely be freed soon anyway.
5682 */
5683 if (!test_and_clear_bit(EXTENT_BUFFER_TREE_REF, &eb->bflags)) {
5684 spin_unlock(&eb->refs_lock);
5685 return 0;
5686 }
Josef Bacik3083ee22012-03-09 16:01:49 -05005687
David Sterbaf7a52a42013-04-26 14:56:29 +00005688 return release_extent_buffer(eb);
Chris Mason6af118ce2008-07-22 11:18:07 -04005689}