blob: 4d901200be136962583ce5f5f30d10362e764f9d [file] [log] [blame]
Chris Masond1310b22008-01-24 16:13:08 -05001#include <linux/bitops.h>
2#include <linux/slab.h>
3#include <linux/bio.h>
4#include <linux/mm.h>
Chris Masond1310b22008-01-24 16:13:08 -05005#include <linux/pagemap.h>
6#include <linux/page-flags.h>
Chris Masond1310b22008-01-24 16:13:08 -05007#include <linux/spinlock.h>
8#include <linux/blkdev.h>
9#include <linux/swap.h>
Chris Masond1310b22008-01-24 16:13:08 -050010#include <linux/writeback.h>
11#include <linux/pagevec.h>
Linus Torvalds268bb0c2011-05-20 12:50:29 -070012#include <linux/prefetch.h>
Dan Magenheimer90a887c2011-05-26 10:01:56 -060013#include <linux/cleancache.h>
Chris Masond1310b22008-01-24 16:13:08 -050014#include "extent_io.h"
15#include "extent_map.h"
David Woodhouse902b22f2008-08-20 08:51:49 -040016#include "ctree.h"
17#include "btrfs_inode.h"
Jan Schmidt4a54c8c2011-07-22 15:41:52 +020018#include "volumes.h"
Stefan Behrens21adbd52011-11-09 13:44:05 +010019#include "check-integrity.h"
Josef Bacik0b32f4b2012-03-13 09:38:00 -040020#include "locking.h"
Josef Bacik606686e2012-06-04 14:03:51 -040021#include "rcu-string.h"
Liu Bofe09e162013-09-22 12:54:23 +080022#include "backref.h"
Lu Fengqiafce7722016-06-13 09:36:46 +080023#include "transaction.h"
Chris Masond1310b22008-01-24 16:13:08 -050024
Chris Masond1310b22008-01-24 16:13:08 -050025static struct kmem_cache *extent_state_cache;
26static struct kmem_cache *extent_buffer_cache;
Chris Mason9be33952013-05-17 18:30:14 -040027static struct bio_set *btrfs_bioset;
Chris Masond1310b22008-01-24 16:13:08 -050028
Filipe Manana27a35072014-07-06 20:09:59 +010029static inline bool extent_state_in_tree(const struct extent_state *state)
30{
31 return !RB_EMPTY_NODE(&state->rb_node);
32}
33
Eric Sandeen6d49ba12013-04-22 16:12:31 +000034#ifdef CONFIG_BTRFS_DEBUG
Chris Masond1310b22008-01-24 16:13:08 -050035static LIST_HEAD(buffers);
36static LIST_HEAD(states);
Chris Mason4bef0842008-09-08 11:18:08 -040037
Chris Masond3977122009-01-05 21:25:51 -050038static DEFINE_SPINLOCK(leak_lock);
Eric Sandeen6d49ba12013-04-22 16:12:31 +000039
40static inline
41void btrfs_leak_debug_add(struct list_head *new, struct list_head *head)
42{
43 unsigned long flags;
44
45 spin_lock_irqsave(&leak_lock, flags);
46 list_add(new, head);
47 spin_unlock_irqrestore(&leak_lock, flags);
48}
49
50static inline
51void btrfs_leak_debug_del(struct list_head *entry)
52{
53 unsigned long flags;
54
55 spin_lock_irqsave(&leak_lock, flags);
56 list_del(entry);
57 spin_unlock_irqrestore(&leak_lock, flags);
58}
59
60static inline
61void btrfs_leak_debug_check(void)
62{
63 struct extent_state *state;
64 struct extent_buffer *eb;
65
66 while (!list_empty(&states)) {
67 state = list_entry(states.next, struct extent_state, leak_list);
David Sterba9ee49a042015-01-14 19:52:13 +010068 pr_err("BTRFS: state leak: start %llu end %llu state %u in tree %d refs %d\n",
Filipe Manana27a35072014-07-06 20:09:59 +010069 state->start, state->end, state->state,
70 extent_state_in_tree(state),
Geert Uytterhoevenc1c9ff72013-08-20 13:20:07 +020071 atomic_read(&state->refs));
Eric Sandeen6d49ba12013-04-22 16:12:31 +000072 list_del(&state->leak_list);
73 kmem_cache_free(extent_state_cache, state);
74 }
75
76 while (!list_empty(&buffers)) {
77 eb = list_entry(buffers.next, struct extent_buffer, leak_list);
Jeff Mahoney62e85572016-09-20 10:05:01 -040078 pr_err("BTRFS: buffer leak start %llu len %lu refs %d\n",
Geert Uytterhoevenc1c9ff72013-08-20 13:20:07 +020079 eb->start, eb->len, atomic_read(&eb->refs));
Eric Sandeen6d49ba12013-04-22 16:12:31 +000080 list_del(&eb->leak_list);
81 kmem_cache_free(extent_buffer_cache, eb);
82 }
83}
David Sterba8d599ae2013-04-30 15:22:23 +000084
Josef Bacika5dee372013-12-13 10:02:44 -050085#define btrfs_debug_check_extent_io_range(tree, start, end) \
86 __btrfs_debug_check_extent_io_range(__func__, (tree), (start), (end))
David Sterba8d599ae2013-04-30 15:22:23 +000087static inline void __btrfs_debug_check_extent_io_range(const char *caller,
Josef Bacika5dee372013-12-13 10:02:44 -050088 struct extent_io_tree *tree, u64 start, u64 end)
David Sterba8d599ae2013-04-30 15:22:23 +000089{
Josef Bacika5dee372013-12-13 10:02:44 -050090 struct inode *inode;
91 u64 isize;
David Sterba8d599ae2013-04-30 15:22:23 +000092
Josef Bacika5dee372013-12-13 10:02:44 -050093 if (!tree->mapping)
94 return;
95
96 inode = tree->mapping->host;
97 isize = i_size_read(inode);
David Sterba8d599ae2013-04-30 15:22:23 +000098 if (end >= PAGE_SIZE && (end % 2) == 0 && end != isize - 1) {
David Sterba94647322015-10-08 11:01:36 +020099 btrfs_debug_rl(BTRFS_I(inode)->root->fs_info,
100 "%s: ino %llu isize %llu odd range [%llu,%llu]",
Geert Uytterhoevenc1c9ff72013-08-20 13:20:07 +0200101 caller, btrfs_ino(inode), isize, start, end);
David Sterba8d599ae2013-04-30 15:22:23 +0000102 }
103}
Eric Sandeen6d49ba12013-04-22 16:12:31 +0000104#else
105#define btrfs_leak_debug_add(new, head) do {} while (0)
106#define btrfs_leak_debug_del(entry) do {} while (0)
107#define btrfs_leak_debug_check() do {} while (0)
David Sterba8d599ae2013-04-30 15:22:23 +0000108#define btrfs_debug_check_extent_io_range(c, s, e) do {} while (0)
Chris Mason4bef0842008-09-08 11:18:08 -0400109#endif
Chris Masond1310b22008-01-24 16:13:08 -0500110
Chris Masond1310b22008-01-24 16:13:08 -0500111#define BUFFER_LRU_MAX 64
112
113struct tree_entry {
114 u64 start;
115 u64 end;
Chris Masond1310b22008-01-24 16:13:08 -0500116 struct rb_node rb_node;
117};
118
119struct extent_page_data {
120 struct bio *bio;
121 struct extent_io_tree *tree;
122 get_extent_t *get_extent;
Josef Bacikde0022b2012-09-25 14:25:58 -0400123 unsigned long bio_flags;
Chris Mason771ed682008-11-06 22:02:51 -0500124
125 /* tells writepage not to lock the state bits for this range
126 * it still does the unlocking
127 */
Chris Masonffbd5172009-04-20 15:50:09 -0400128 unsigned int extent_locked:1;
129
130 /* tells the submit_bio code to use a WRITE_SYNC */
131 unsigned int sync_io:1;
Chris Masond1310b22008-01-24 16:13:08 -0500132};
133
Qu Wenruod38ed272015-10-12 14:53:37 +0800134static void add_extent_changeset(struct extent_state *state, unsigned bits,
135 struct extent_changeset *changeset,
136 int set)
137{
138 int ret;
139
140 if (!changeset)
141 return;
142 if (set && (state->state & bits) == bits)
143 return;
Qu Wenruofefdc552015-10-12 15:35:38 +0800144 if (!set && (state->state & bits) == 0)
145 return;
Qu Wenruod38ed272015-10-12 14:53:37 +0800146 changeset->bytes_changed += state->end - state->start + 1;
147 ret = ulist_add(changeset->range_changed, state->start, state->end,
148 GFP_ATOMIC);
149 /* ENOMEM */
150 BUG_ON(ret < 0);
151}
152
Josef Bacik0b32f4b2012-03-13 09:38:00 -0400153static noinline void flush_write_bio(void *data);
Jeff Mahoneyc2d904e2011-10-03 23:22:32 -0400154static inline struct btrfs_fs_info *
155tree_fs_info(struct extent_io_tree *tree)
156{
Josef Bacika5dee372013-12-13 10:02:44 -0500157 if (!tree->mapping)
158 return NULL;
Jeff Mahoneyc2d904e2011-10-03 23:22:32 -0400159 return btrfs_sb(tree->mapping->host->i_sb);
160}
Josef Bacik0b32f4b2012-03-13 09:38:00 -0400161
Chris Masond1310b22008-01-24 16:13:08 -0500162int __init extent_io_init(void)
163{
David Sterba837e1972012-09-07 03:00:48 -0600164 extent_state_cache = kmem_cache_create("btrfs_extent_state",
Christoph Hellwig9601e3f2009-04-13 15:33:09 +0200165 sizeof(struct extent_state), 0,
Nikolay Borisovfba4b692016-06-23 21:17:08 +0300166 SLAB_MEM_SPREAD, NULL);
Chris Masond1310b22008-01-24 16:13:08 -0500167 if (!extent_state_cache)
168 return -ENOMEM;
169
David Sterba837e1972012-09-07 03:00:48 -0600170 extent_buffer_cache = kmem_cache_create("btrfs_extent_buffer",
Christoph Hellwig9601e3f2009-04-13 15:33:09 +0200171 sizeof(struct extent_buffer), 0,
Nikolay Borisovfba4b692016-06-23 21:17:08 +0300172 SLAB_MEM_SPREAD, NULL);
Chris Masond1310b22008-01-24 16:13:08 -0500173 if (!extent_buffer_cache)
174 goto free_state_cache;
Chris Mason9be33952013-05-17 18:30:14 -0400175
176 btrfs_bioset = bioset_create(BIO_POOL_SIZE,
177 offsetof(struct btrfs_io_bio, bio));
178 if (!btrfs_bioset)
179 goto free_buffer_cache;
Darrick J. Wongb208c2f2013-09-19 20:37:07 -0700180
181 if (bioset_integrity_create(btrfs_bioset, BIO_POOL_SIZE))
182 goto free_bioset;
183
Chris Masond1310b22008-01-24 16:13:08 -0500184 return 0;
185
Darrick J. Wongb208c2f2013-09-19 20:37:07 -0700186free_bioset:
187 bioset_free(btrfs_bioset);
188 btrfs_bioset = NULL;
189
Chris Mason9be33952013-05-17 18:30:14 -0400190free_buffer_cache:
191 kmem_cache_destroy(extent_buffer_cache);
192 extent_buffer_cache = NULL;
193
Chris Masond1310b22008-01-24 16:13:08 -0500194free_state_cache:
195 kmem_cache_destroy(extent_state_cache);
Chris Mason9be33952013-05-17 18:30:14 -0400196 extent_state_cache = NULL;
Chris Masond1310b22008-01-24 16:13:08 -0500197 return -ENOMEM;
198}
199
200void extent_io_exit(void)
201{
Eric Sandeen6d49ba12013-04-22 16:12:31 +0000202 btrfs_leak_debug_check();
Kirill A. Shutemov8c0a8532012-09-26 11:33:07 +1000203
204 /*
205 * Make sure all delayed rcu free are flushed before we
206 * destroy caches.
207 */
208 rcu_barrier();
Kinglong Mee5598e902016-01-29 21:36:35 +0800209 kmem_cache_destroy(extent_state_cache);
210 kmem_cache_destroy(extent_buffer_cache);
Chris Mason9be33952013-05-17 18:30:14 -0400211 if (btrfs_bioset)
212 bioset_free(btrfs_bioset);
Chris Masond1310b22008-01-24 16:13:08 -0500213}
214
215void extent_io_tree_init(struct extent_io_tree *tree,
David Sterbaf993c882011-04-20 23:35:57 +0200216 struct address_space *mapping)
Chris Masond1310b22008-01-24 16:13:08 -0500217{
Eric Paris6bef4d32010-02-23 19:43:04 +0000218 tree->state = RB_ROOT;
Chris Masond1310b22008-01-24 16:13:08 -0500219 tree->ops = NULL;
220 tree->dirty_bytes = 0;
Chris Mason70dec802008-01-29 09:59:12 -0500221 spin_lock_init(&tree->lock);
Chris Masond1310b22008-01-24 16:13:08 -0500222 tree->mapping = mapping;
Chris Masond1310b22008-01-24 16:13:08 -0500223}
Chris Masond1310b22008-01-24 16:13:08 -0500224
Christoph Hellwigb2950862008-12-02 09:54:17 -0500225static struct extent_state *alloc_extent_state(gfp_t mask)
Chris Masond1310b22008-01-24 16:13:08 -0500226{
227 struct extent_state *state;
Chris Masond1310b22008-01-24 16:13:08 -0500228
229 state = kmem_cache_alloc(extent_state_cache, mask);
Peter2b114d12008-04-01 11:21:40 -0400230 if (!state)
Chris Masond1310b22008-01-24 16:13:08 -0500231 return state;
232 state->state = 0;
David Sterba47dc1962016-02-11 13:24:13 +0100233 state->failrec = NULL;
Filipe Manana27a35072014-07-06 20:09:59 +0100234 RB_CLEAR_NODE(&state->rb_node);
Eric Sandeen6d49ba12013-04-22 16:12:31 +0000235 btrfs_leak_debug_add(&state->leak_list, &states);
Chris Masond1310b22008-01-24 16:13:08 -0500236 atomic_set(&state->refs, 1);
237 init_waitqueue_head(&state->wq);
Jeff Mahoney143bede2012-03-01 14:56:26 +0100238 trace_alloc_extent_state(state, mask, _RET_IP_);
Chris Masond1310b22008-01-24 16:13:08 -0500239 return state;
240}
Chris Masond1310b22008-01-24 16:13:08 -0500241
Chris Mason4845e442010-05-25 20:56:50 -0400242void free_extent_state(struct extent_state *state)
Chris Masond1310b22008-01-24 16:13:08 -0500243{
Chris Masond1310b22008-01-24 16:13:08 -0500244 if (!state)
245 return;
246 if (atomic_dec_and_test(&state->refs)) {
Filipe Manana27a35072014-07-06 20:09:59 +0100247 WARN_ON(extent_state_in_tree(state));
Eric Sandeen6d49ba12013-04-22 16:12:31 +0000248 btrfs_leak_debug_del(&state->leak_list);
Jeff Mahoney143bede2012-03-01 14:56:26 +0100249 trace_free_extent_state(state, _RET_IP_);
Chris Masond1310b22008-01-24 16:13:08 -0500250 kmem_cache_free(extent_state_cache, state);
251 }
252}
Chris Masond1310b22008-01-24 16:13:08 -0500253
Filipe Mananaf2071b22014-02-12 15:05:53 +0000254static struct rb_node *tree_insert(struct rb_root *root,
255 struct rb_node *search_start,
256 u64 offset,
Filipe David Borba Manana12cfbad2013-11-26 15:41:47 +0000257 struct rb_node *node,
258 struct rb_node ***p_in,
259 struct rb_node **parent_in)
Chris Masond1310b22008-01-24 16:13:08 -0500260{
Filipe Mananaf2071b22014-02-12 15:05:53 +0000261 struct rb_node **p;
Chris Masond3977122009-01-05 21:25:51 -0500262 struct rb_node *parent = NULL;
Chris Masond1310b22008-01-24 16:13:08 -0500263 struct tree_entry *entry;
264
Filipe David Borba Manana12cfbad2013-11-26 15:41:47 +0000265 if (p_in && parent_in) {
266 p = *p_in;
267 parent = *parent_in;
268 goto do_insert;
269 }
270
Filipe Mananaf2071b22014-02-12 15:05:53 +0000271 p = search_start ? &search_start : &root->rb_node;
Chris Masond3977122009-01-05 21:25:51 -0500272 while (*p) {
Chris Masond1310b22008-01-24 16:13:08 -0500273 parent = *p;
274 entry = rb_entry(parent, struct tree_entry, rb_node);
275
276 if (offset < entry->start)
277 p = &(*p)->rb_left;
278 else if (offset > entry->end)
279 p = &(*p)->rb_right;
280 else
281 return parent;
282 }
283
Filipe David Borba Manana12cfbad2013-11-26 15:41:47 +0000284do_insert:
Chris Masond1310b22008-01-24 16:13:08 -0500285 rb_link_node(node, parent, p);
286 rb_insert_color(node, root);
287 return NULL;
288}
289
Chris Mason80ea96b2008-02-01 14:51:59 -0500290static struct rb_node *__etree_search(struct extent_io_tree *tree, u64 offset,
Filipe David Borba Manana12cfbad2013-11-26 15:41:47 +0000291 struct rb_node **prev_ret,
292 struct rb_node **next_ret,
293 struct rb_node ***p_ret,
294 struct rb_node **parent_ret)
Chris Masond1310b22008-01-24 16:13:08 -0500295{
Chris Mason80ea96b2008-02-01 14:51:59 -0500296 struct rb_root *root = &tree->state;
Filipe David Borba Manana12cfbad2013-11-26 15:41:47 +0000297 struct rb_node **n = &root->rb_node;
Chris Masond1310b22008-01-24 16:13:08 -0500298 struct rb_node *prev = NULL;
299 struct rb_node *orig_prev = NULL;
300 struct tree_entry *entry;
301 struct tree_entry *prev_entry = NULL;
302
Filipe David Borba Manana12cfbad2013-11-26 15:41:47 +0000303 while (*n) {
304 prev = *n;
305 entry = rb_entry(prev, struct tree_entry, rb_node);
Chris Masond1310b22008-01-24 16:13:08 -0500306 prev_entry = entry;
307
308 if (offset < entry->start)
Filipe David Borba Manana12cfbad2013-11-26 15:41:47 +0000309 n = &(*n)->rb_left;
Chris Masond1310b22008-01-24 16:13:08 -0500310 else if (offset > entry->end)
Filipe David Borba Manana12cfbad2013-11-26 15:41:47 +0000311 n = &(*n)->rb_right;
Chris Masond3977122009-01-05 21:25:51 -0500312 else
Filipe David Borba Manana12cfbad2013-11-26 15:41:47 +0000313 return *n;
Chris Masond1310b22008-01-24 16:13:08 -0500314 }
315
Filipe David Borba Manana12cfbad2013-11-26 15:41:47 +0000316 if (p_ret)
317 *p_ret = n;
318 if (parent_ret)
319 *parent_ret = prev;
320
Chris Masond1310b22008-01-24 16:13:08 -0500321 if (prev_ret) {
322 orig_prev = prev;
Chris Masond3977122009-01-05 21:25:51 -0500323 while (prev && offset > prev_entry->end) {
Chris Masond1310b22008-01-24 16:13:08 -0500324 prev = rb_next(prev);
325 prev_entry = rb_entry(prev, struct tree_entry, rb_node);
326 }
327 *prev_ret = prev;
328 prev = orig_prev;
329 }
330
331 if (next_ret) {
332 prev_entry = rb_entry(prev, struct tree_entry, rb_node);
Chris Masond3977122009-01-05 21:25:51 -0500333 while (prev && offset < prev_entry->start) {
Chris Masond1310b22008-01-24 16:13:08 -0500334 prev = rb_prev(prev);
335 prev_entry = rb_entry(prev, struct tree_entry, rb_node);
336 }
337 *next_ret = prev;
338 }
339 return NULL;
340}
341
Filipe David Borba Manana12cfbad2013-11-26 15:41:47 +0000342static inline struct rb_node *
343tree_search_for_insert(struct extent_io_tree *tree,
344 u64 offset,
345 struct rb_node ***p_ret,
346 struct rb_node **parent_ret)
Chris Masond1310b22008-01-24 16:13:08 -0500347{
Chris Mason70dec802008-01-29 09:59:12 -0500348 struct rb_node *prev = NULL;
Chris Masond1310b22008-01-24 16:13:08 -0500349 struct rb_node *ret;
Chris Mason70dec802008-01-29 09:59:12 -0500350
Filipe David Borba Manana12cfbad2013-11-26 15:41:47 +0000351 ret = __etree_search(tree, offset, &prev, NULL, p_ret, parent_ret);
Chris Masond3977122009-01-05 21:25:51 -0500352 if (!ret)
Chris Masond1310b22008-01-24 16:13:08 -0500353 return prev;
354 return ret;
355}
356
Filipe David Borba Manana12cfbad2013-11-26 15:41:47 +0000357static inline struct rb_node *tree_search(struct extent_io_tree *tree,
358 u64 offset)
359{
360 return tree_search_for_insert(tree, offset, NULL, NULL);
361}
362
Josef Bacik9ed74f22009-09-11 16:12:44 -0400363static void merge_cb(struct extent_io_tree *tree, struct extent_state *new,
364 struct extent_state *other)
365{
366 if (tree->ops && tree->ops->merge_extent_hook)
367 tree->ops->merge_extent_hook(tree->mapping->host, new,
368 other);
369}
370
Chris Masond1310b22008-01-24 16:13:08 -0500371/*
372 * utility function to look for merge candidates inside a given range.
373 * Any extents with matching state are merged together into a single
374 * extent in the tree. Extents with EXTENT_IO in their state field
375 * are not merged because the end_io handlers need to be able to do
376 * operations on them without sleeping (or doing allocations/splits).
377 *
378 * This should be called with the tree lock held.
379 */
Jeff Mahoney1bf85042011-07-21 16:56:09 +0000380static void merge_state(struct extent_io_tree *tree,
381 struct extent_state *state)
Chris Masond1310b22008-01-24 16:13:08 -0500382{
383 struct extent_state *other;
384 struct rb_node *other_node;
385
Zheng Yan5b21f2e2008-09-26 10:05:38 -0400386 if (state->state & (EXTENT_IOBITS | EXTENT_BOUNDARY))
Jeff Mahoney1bf85042011-07-21 16:56:09 +0000387 return;
Chris Masond1310b22008-01-24 16:13:08 -0500388
389 other_node = rb_prev(&state->rb_node);
390 if (other_node) {
391 other = rb_entry(other_node, struct extent_state, rb_node);
392 if (other->end == state->start - 1 &&
393 other->state == state->state) {
Josef Bacik9ed74f22009-09-11 16:12:44 -0400394 merge_cb(tree, state, other);
Chris Masond1310b22008-01-24 16:13:08 -0500395 state->start = other->start;
Chris Masond1310b22008-01-24 16:13:08 -0500396 rb_erase(&other->rb_node, &tree->state);
Filipe Manana27a35072014-07-06 20:09:59 +0100397 RB_CLEAR_NODE(&other->rb_node);
Chris Masond1310b22008-01-24 16:13:08 -0500398 free_extent_state(other);
399 }
400 }
401 other_node = rb_next(&state->rb_node);
402 if (other_node) {
403 other = rb_entry(other_node, struct extent_state, rb_node);
404 if (other->start == state->end + 1 &&
405 other->state == state->state) {
Josef Bacik9ed74f22009-09-11 16:12:44 -0400406 merge_cb(tree, state, other);
Josef Bacikdf98b6e2011-06-20 14:53:48 -0400407 state->end = other->end;
Josef Bacikdf98b6e2011-06-20 14:53:48 -0400408 rb_erase(&other->rb_node, &tree->state);
Filipe Manana27a35072014-07-06 20:09:59 +0100409 RB_CLEAR_NODE(&other->rb_node);
Josef Bacikdf98b6e2011-06-20 14:53:48 -0400410 free_extent_state(other);
Chris Masond1310b22008-01-24 16:13:08 -0500411 }
412 }
Chris Masond1310b22008-01-24 16:13:08 -0500413}
414
Jeff Mahoney1bf85042011-07-21 16:56:09 +0000415static void set_state_cb(struct extent_io_tree *tree,
David Sterba9ee49a042015-01-14 19:52:13 +0100416 struct extent_state *state, unsigned *bits)
Chris Mason291d6732008-01-29 15:55:23 -0500417{
Jeff Mahoney1bf85042011-07-21 16:56:09 +0000418 if (tree->ops && tree->ops->set_bit_hook)
419 tree->ops->set_bit_hook(tree->mapping->host, state, bits);
Chris Mason291d6732008-01-29 15:55:23 -0500420}
421
422static void clear_state_cb(struct extent_io_tree *tree,
David Sterba9ee49a042015-01-14 19:52:13 +0100423 struct extent_state *state, unsigned *bits)
Chris Mason291d6732008-01-29 15:55:23 -0500424{
Josef Bacik9ed74f22009-09-11 16:12:44 -0400425 if (tree->ops && tree->ops->clear_bit_hook)
426 tree->ops->clear_bit_hook(tree->mapping->host, state, bits);
Chris Mason291d6732008-01-29 15:55:23 -0500427}
428
Xiao Guangrong3150b692011-07-14 03:19:08 +0000429static void set_state_bits(struct extent_io_tree *tree,
Qu Wenruod38ed272015-10-12 14:53:37 +0800430 struct extent_state *state, unsigned *bits,
431 struct extent_changeset *changeset);
Xiao Guangrong3150b692011-07-14 03:19:08 +0000432
Chris Masond1310b22008-01-24 16:13:08 -0500433/*
434 * insert an extent_state struct into the tree. 'bits' are set on the
435 * struct before it is inserted.
436 *
437 * This may return -EEXIST if the extent is already there, in which case the
438 * state struct is freed.
439 *
440 * The tree lock is not taken internally. This is a utility function and
441 * probably isn't what you want to call (see set/clear_extent_bit).
442 */
443static int insert_state(struct extent_io_tree *tree,
444 struct extent_state *state, u64 start, u64 end,
Filipe David Borba Manana12cfbad2013-11-26 15:41:47 +0000445 struct rb_node ***p,
446 struct rb_node **parent,
Qu Wenruod38ed272015-10-12 14:53:37 +0800447 unsigned *bits, struct extent_changeset *changeset)
Chris Masond1310b22008-01-24 16:13:08 -0500448{
449 struct rb_node *node;
450
Julia Lawall31b1a2b2012-11-03 10:58:34 +0000451 if (end < start)
Frank Holtonefe120a2013-12-20 11:37:06 -0500452 WARN(1, KERN_ERR "BTRFS: end < start %llu %llu\n",
Geert Uytterhoevenc1c9ff72013-08-20 13:20:07 +0200453 end, start);
Chris Masond1310b22008-01-24 16:13:08 -0500454 state->start = start;
455 state->end = end;
Josef Bacik9ed74f22009-09-11 16:12:44 -0400456
Qu Wenruod38ed272015-10-12 14:53:37 +0800457 set_state_bits(tree, state, bits, changeset);
Xiao Guangrong3150b692011-07-14 03:19:08 +0000458
Filipe Mananaf2071b22014-02-12 15:05:53 +0000459 node = tree_insert(&tree->state, NULL, end, &state->rb_node, p, parent);
Chris Masond1310b22008-01-24 16:13:08 -0500460 if (node) {
461 struct extent_state *found;
462 found = rb_entry(node, struct extent_state, rb_node);
Jeff Mahoney62e85572016-09-20 10:05:01 -0400463 pr_err("BTRFS: found node %llu %llu on insert of %llu %llu\n",
Geert Uytterhoevenc1c9ff72013-08-20 13:20:07 +0200464 found->start, found->end, start, end);
Chris Masond1310b22008-01-24 16:13:08 -0500465 return -EEXIST;
466 }
467 merge_state(tree, state);
468 return 0;
469}
470
Jeff Mahoney1bf85042011-07-21 16:56:09 +0000471static void split_cb(struct extent_io_tree *tree, struct extent_state *orig,
Josef Bacik9ed74f22009-09-11 16:12:44 -0400472 u64 split)
473{
474 if (tree->ops && tree->ops->split_extent_hook)
Jeff Mahoney1bf85042011-07-21 16:56:09 +0000475 tree->ops->split_extent_hook(tree->mapping->host, orig, split);
Josef Bacik9ed74f22009-09-11 16:12:44 -0400476}
477
Chris Masond1310b22008-01-24 16:13:08 -0500478/*
479 * split a given extent state struct in two, inserting the preallocated
480 * struct 'prealloc' as the newly created second half. 'split' indicates an
481 * offset inside 'orig' where it should be split.
482 *
483 * Before calling,
484 * the tree has 'orig' at [orig->start, orig->end]. After calling, there
485 * are two extent state structs in the tree:
486 * prealloc: [orig->start, split - 1]
487 * orig: [ split, orig->end ]
488 *
489 * The tree locks are not taken by this function. They need to be held
490 * by the caller.
491 */
492static int split_state(struct extent_io_tree *tree, struct extent_state *orig,
493 struct extent_state *prealloc, u64 split)
494{
495 struct rb_node *node;
Josef Bacik9ed74f22009-09-11 16:12:44 -0400496
497 split_cb(tree, orig, split);
498
Chris Masond1310b22008-01-24 16:13:08 -0500499 prealloc->start = orig->start;
500 prealloc->end = split - 1;
501 prealloc->state = orig->state;
502 orig->start = split;
503
Filipe Mananaf2071b22014-02-12 15:05:53 +0000504 node = tree_insert(&tree->state, &orig->rb_node, prealloc->end,
505 &prealloc->rb_node, NULL, NULL);
Chris Masond1310b22008-01-24 16:13:08 -0500506 if (node) {
Chris Masond1310b22008-01-24 16:13:08 -0500507 free_extent_state(prealloc);
508 return -EEXIST;
509 }
510 return 0;
511}
512
Li Zefancdc6a392012-03-12 16:39:48 +0800513static struct extent_state *next_state(struct extent_state *state)
514{
515 struct rb_node *next = rb_next(&state->rb_node);
516 if (next)
517 return rb_entry(next, struct extent_state, rb_node);
518 else
519 return NULL;
520}
521
Chris Masond1310b22008-01-24 16:13:08 -0500522/*
523 * utility function to clear some bits in an extent state struct.
Wang Sheng-Hui1b303fc2012-04-06 14:35:18 +0800524 * it will optionally wake up any one waiting on this state (wake == 1).
Chris Masond1310b22008-01-24 16:13:08 -0500525 *
526 * If no bits are set on the state struct after clearing things, the
527 * struct is freed and removed from the tree
528 */
Li Zefancdc6a392012-03-12 16:39:48 +0800529static struct extent_state *clear_state_bit(struct extent_io_tree *tree,
530 struct extent_state *state,
Qu Wenruofefdc552015-10-12 15:35:38 +0800531 unsigned *bits, int wake,
532 struct extent_changeset *changeset)
Chris Masond1310b22008-01-24 16:13:08 -0500533{
Li Zefancdc6a392012-03-12 16:39:48 +0800534 struct extent_state *next;
David Sterba9ee49a042015-01-14 19:52:13 +0100535 unsigned bits_to_clear = *bits & ~EXTENT_CTLBITS;
Chris Masond1310b22008-01-24 16:13:08 -0500536
Yan, Zheng0ca1f7c2010-05-16 10:48:47 -0400537 if ((bits_to_clear & EXTENT_DIRTY) && (state->state & EXTENT_DIRTY)) {
Chris Masond1310b22008-01-24 16:13:08 -0500538 u64 range = state->end - state->start + 1;
539 WARN_ON(range > tree->dirty_bytes);
540 tree->dirty_bytes -= range;
541 }
Chris Mason291d6732008-01-29 15:55:23 -0500542 clear_state_cb(tree, state, bits);
Qu Wenruofefdc552015-10-12 15:35:38 +0800543 add_extent_changeset(state, bits_to_clear, changeset, 0);
Josef Bacik32c00af2009-10-08 13:34:05 -0400544 state->state &= ~bits_to_clear;
Chris Masond1310b22008-01-24 16:13:08 -0500545 if (wake)
546 wake_up(&state->wq);
Yan, Zheng0ca1f7c2010-05-16 10:48:47 -0400547 if (state->state == 0) {
Li Zefancdc6a392012-03-12 16:39:48 +0800548 next = next_state(state);
Filipe Manana27a35072014-07-06 20:09:59 +0100549 if (extent_state_in_tree(state)) {
Chris Masond1310b22008-01-24 16:13:08 -0500550 rb_erase(&state->rb_node, &tree->state);
Filipe Manana27a35072014-07-06 20:09:59 +0100551 RB_CLEAR_NODE(&state->rb_node);
Chris Masond1310b22008-01-24 16:13:08 -0500552 free_extent_state(state);
553 } else {
554 WARN_ON(1);
555 }
556 } else {
557 merge_state(tree, state);
Li Zefancdc6a392012-03-12 16:39:48 +0800558 next = next_state(state);
Chris Masond1310b22008-01-24 16:13:08 -0500559 }
Li Zefancdc6a392012-03-12 16:39:48 +0800560 return next;
Chris Masond1310b22008-01-24 16:13:08 -0500561}
562
Xiao Guangrong82337672011-04-20 06:44:57 +0000563static struct extent_state *
564alloc_extent_state_atomic(struct extent_state *prealloc)
565{
566 if (!prealloc)
567 prealloc = alloc_extent_state(GFP_ATOMIC);
568
569 return prealloc;
570}
571
Eric Sandeen48a3b632013-04-25 20:41:01 +0000572static void extent_io_tree_panic(struct extent_io_tree *tree, int err)
Jeff Mahoneyc2d904e2011-10-03 23:22:32 -0400573{
Jeff Mahoney5d163e02016-09-20 10:05:00 -0400574 btrfs_panic(tree_fs_info(tree), err,
575 "Locking error: Extent tree was modified by another thread while locked.");
Jeff Mahoneyc2d904e2011-10-03 23:22:32 -0400576}
577
Chris Masond1310b22008-01-24 16:13:08 -0500578/*
579 * clear some bits on a range in the tree. This may require splitting
580 * or inserting elements in the tree, so the gfp mask is used to
581 * indicate which allocations or sleeping are allowed.
582 *
583 * pass 'wake' == 1 to kick any sleepers, and 'delete' == 1 to remove
584 * the given range from the tree regardless of state (ie for truncate).
585 *
586 * the range [start, end] is inclusive.
587 *
Jeff Mahoney6763af82012-03-01 14:56:29 +0100588 * This takes the tree lock, and returns 0 on success and < 0 on error.
Chris Masond1310b22008-01-24 16:13:08 -0500589 */
Qu Wenruofefdc552015-10-12 15:35:38 +0800590static int __clear_extent_bit(struct extent_io_tree *tree, u64 start, u64 end,
591 unsigned bits, int wake, int delete,
592 struct extent_state **cached_state,
593 gfp_t mask, struct extent_changeset *changeset)
Chris Masond1310b22008-01-24 16:13:08 -0500594{
595 struct extent_state *state;
Chris Mason2c64c532009-09-02 15:04:12 -0400596 struct extent_state *cached;
Chris Masond1310b22008-01-24 16:13:08 -0500597 struct extent_state *prealloc = NULL;
598 struct rb_node *node;
Yan Zheng5c939df2009-05-27 09:16:03 -0400599 u64 last_end;
Chris Masond1310b22008-01-24 16:13:08 -0500600 int err;
Josef Bacik2ac55d42010-02-03 19:33:23 +0000601 int clear = 0;
Chris Masond1310b22008-01-24 16:13:08 -0500602
Josef Bacika5dee372013-12-13 10:02:44 -0500603 btrfs_debug_check_extent_io_range(tree, start, end);
David Sterba8d599ae2013-04-30 15:22:23 +0000604
Josef Bacik7ee9e442013-06-21 16:37:03 -0400605 if (bits & EXTENT_DELALLOC)
606 bits |= EXTENT_NORESERVE;
607
Yan, Zheng0ca1f7c2010-05-16 10:48:47 -0400608 if (delete)
609 bits |= ~EXTENT_CTLBITS;
610 bits |= EXTENT_FIRST_DELALLOC;
611
Josef Bacik2ac55d42010-02-03 19:33:23 +0000612 if (bits & (EXTENT_IOBITS | EXTENT_BOUNDARY))
613 clear = 1;
Chris Masond1310b22008-01-24 16:13:08 -0500614again:
Mel Gormand0164ad2015-11-06 16:28:21 -0800615 if (!prealloc && gfpflags_allow_blocking(mask)) {
Filipe Mananac7bc6312014-11-03 14:12:57 +0000616 /*
617 * Don't care for allocation failure here because we might end
618 * up not needing the pre-allocated extent state at all, which
619 * is the case if we only have in the tree extent states that
620 * cover our input range and don't cover too any other range.
621 * If we end up needing a new extent state we allocate it later.
622 */
Chris Masond1310b22008-01-24 16:13:08 -0500623 prealloc = alloc_extent_state(mask);
Chris Masond1310b22008-01-24 16:13:08 -0500624 }
625
Chris Masoncad321a2008-12-17 14:51:42 -0500626 spin_lock(&tree->lock);
Chris Mason2c64c532009-09-02 15:04:12 -0400627 if (cached_state) {
628 cached = *cached_state;
Josef Bacik2ac55d42010-02-03 19:33:23 +0000629
630 if (clear) {
631 *cached_state = NULL;
632 cached_state = NULL;
633 }
634
Filipe Manana27a35072014-07-06 20:09:59 +0100635 if (cached && extent_state_in_tree(cached) &&
636 cached->start <= start && cached->end > start) {
Josef Bacik2ac55d42010-02-03 19:33:23 +0000637 if (clear)
638 atomic_dec(&cached->refs);
Chris Mason2c64c532009-09-02 15:04:12 -0400639 state = cached;
Chris Mason42daec22009-09-23 19:51:09 -0400640 goto hit_next;
Chris Mason2c64c532009-09-02 15:04:12 -0400641 }
Josef Bacik2ac55d42010-02-03 19:33:23 +0000642 if (clear)
643 free_extent_state(cached);
Chris Mason2c64c532009-09-02 15:04:12 -0400644 }
Chris Masond1310b22008-01-24 16:13:08 -0500645 /*
646 * this search will find the extents that end after
647 * our range starts
648 */
Chris Mason80ea96b2008-02-01 14:51:59 -0500649 node = tree_search(tree, start);
Chris Masond1310b22008-01-24 16:13:08 -0500650 if (!node)
651 goto out;
652 state = rb_entry(node, struct extent_state, rb_node);
Chris Mason2c64c532009-09-02 15:04:12 -0400653hit_next:
Chris Masond1310b22008-01-24 16:13:08 -0500654 if (state->start > end)
655 goto out;
656 WARN_ON(state->end < start);
Yan Zheng5c939df2009-05-27 09:16:03 -0400657 last_end = state->end;
Chris Masond1310b22008-01-24 16:13:08 -0500658
Liu Bo04493142012-02-16 18:34:37 +0800659 /* the state doesn't have the wanted bits, go ahead */
Li Zefancdc6a392012-03-12 16:39:48 +0800660 if (!(state->state & bits)) {
661 state = next_state(state);
Liu Bo04493142012-02-16 18:34:37 +0800662 goto next;
Li Zefancdc6a392012-03-12 16:39:48 +0800663 }
Liu Bo04493142012-02-16 18:34:37 +0800664
Chris Masond1310b22008-01-24 16:13:08 -0500665 /*
666 * | ---- desired range ---- |
667 * | state | or
668 * | ------------- state -------------- |
669 *
670 * We need to split the extent we found, and may flip
671 * bits on second half.
672 *
673 * If the extent we found extends past our range, we
674 * just split and search again. It'll get split again
675 * the next time though.
676 *
677 * If the extent we found is inside our range, we clear
678 * the desired bit on it.
679 */
680
681 if (state->start < start) {
Xiao Guangrong82337672011-04-20 06:44:57 +0000682 prealloc = alloc_extent_state_atomic(prealloc);
683 BUG_ON(!prealloc);
Chris Masond1310b22008-01-24 16:13:08 -0500684 err = split_state(tree, state, prealloc, start);
Jeff Mahoneyc2d904e2011-10-03 23:22:32 -0400685 if (err)
686 extent_io_tree_panic(tree, err);
687
Chris Masond1310b22008-01-24 16:13:08 -0500688 prealloc = NULL;
689 if (err)
690 goto out;
691 if (state->end <= end) {
Qu Wenruofefdc552015-10-12 15:35:38 +0800692 state = clear_state_bit(tree, state, &bits, wake,
693 changeset);
Liu Bod1ac6e42012-05-10 18:10:39 +0800694 goto next;
Chris Masond1310b22008-01-24 16:13:08 -0500695 }
696 goto search_again;
697 }
698 /*
699 * | ---- desired range ---- |
700 * | state |
701 * We need to split the extent, and clear the bit
702 * on the first half
703 */
704 if (state->start <= end && state->end > end) {
Xiao Guangrong82337672011-04-20 06:44:57 +0000705 prealloc = alloc_extent_state_atomic(prealloc);
706 BUG_ON(!prealloc);
Chris Masond1310b22008-01-24 16:13:08 -0500707 err = split_state(tree, state, prealloc, end + 1);
Jeff Mahoneyc2d904e2011-10-03 23:22:32 -0400708 if (err)
709 extent_io_tree_panic(tree, err);
710
Chris Masond1310b22008-01-24 16:13:08 -0500711 if (wake)
712 wake_up(&state->wq);
Chris Mason42daec22009-09-23 19:51:09 -0400713
Qu Wenruofefdc552015-10-12 15:35:38 +0800714 clear_state_bit(tree, prealloc, &bits, wake, changeset);
Josef Bacik9ed74f22009-09-11 16:12:44 -0400715
Chris Masond1310b22008-01-24 16:13:08 -0500716 prealloc = NULL;
717 goto out;
718 }
Chris Mason42daec22009-09-23 19:51:09 -0400719
Qu Wenruofefdc552015-10-12 15:35:38 +0800720 state = clear_state_bit(tree, state, &bits, wake, changeset);
Liu Bo04493142012-02-16 18:34:37 +0800721next:
Yan Zheng5c939df2009-05-27 09:16:03 -0400722 if (last_end == (u64)-1)
723 goto out;
724 start = last_end + 1;
Li Zefancdc6a392012-03-12 16:39:48 +0800725 if (start <= end && state && !need_resched())
Liu Bo692e5752012-02-16 18:34:36 +0800726 goto hit_next;
Chris Masond1310b22008-01-24 16:13:08 -0500727
728search_again:
729 if (start > end)
730 goto out;
Chris Masoncad321a2008-12-17 14:51:42 -0500731 spin_unlock(&tree->lock);
Mel Gormand0164ad2015-11-06 16:28:21 -0800732 if (gfpflags_allow_blocking(mask))
Chris Masond1310b22008-01-24 16:13:08 -0500733 cond_resched();
734 goto again;
David Sterba7ab5cb22016-04-27 01:02:15 +0200735
736out:
737 spin_unlock(&tree->lock);
738 if (prealloc)
739 free_extent_state(prealloc);
740
741 return 0;
742
Chris Masond1310b22008-01-24 16:13:08 -0500743}
Chris Masond1310b22008-01-24 16:13:08 -0500744
Jeff Mahoney143bede2012-03-01 14:56:26 +0100745static void wait_on_state(struct extent_io_tree *tree,
746 struct extent_state *state)
Christoph Hellwig641f5212008-12-02 06:36:10 -0500747 __releases(tree->lock)
748 __acquires(tree->lock)
Chris Masond1310b22008-01-24 16:13:08 -0500749{
750 DEFINE_WAIT(wait);
751 prepare_to_wait(&state->wq, &wait, TASK_UNINTERRUPTIBLE);
Chris Masoncad321a2008-12-17 14:51:42 -0500752 spin_unlock(&tree->lock);
Chris Masond1310b22008-01-24 16:13:08 -0500753 schedule();
Chris Masoncad321a2008-12-17 14:51:42 -0500754 spin_lock(&tree->lock);
Chris Masond1310b22008-01-24 16:13:08 -0500755 finish_wait(&state->wq, &wait);
Chris Masond1310b22008-01-24 16:13:08 -0500756}
757
758/*
759 * waits for one or more bits to clear on a range in the state tree.
760 * The range [start, end] is inclusive.
761 * The tree lock is taken by this function
762 */
David Sterba41074882013-04-29 13:38:46 +0000763static void wait_extent_bit(struct extent_io_tree *tree, u64 start, u64 end,
764 unsigned long bits)
Chris Masond1310b22008-01-24 16:13:08 -0500765{
766 struct extent_state *state;
767 struct rb_node *node;
768
Josef Bacika5dee372013-12-13 10:02:44 -0500769 btrfs_debug_check_extent_io_range(tree, start, end);
David Sterba8d599ae2013-04-30 15:22:23 +0000770
Chris Masoncad321a2008-12-17 14:51:42 -0500771 spin_lock(&tree->lock);
Chris Masond1310b22008-01-24 16:13:08 -0500772again:
773 while (1) {
774 /*
775 * this search will find all the extents that end after
776 * our range starts
777 */
Chris Mason80ea96b2008-02-01 14:51:59 -0500778 node = tree_search(tree, start);
Filipe Mananac50d3e72014-03-31 14:53:25 +0100779process_node:
Chris Masond1310b22008-01-24 16:13:08 -0500780 if (!node)
781 break;
782
783 state = rb_entry(node, struct extent_state, rb_node);
784
785 if (state->start > end)
786 goto out;
787
788 if (state->state & bits) {
789 start = state->start;
790 atomic_inc(&state->refs);
791 wait_on_state(tree, state);
792 free_extent_state(state);
793 goto again;
794 }
795 start = state->end + 1;
796
797 if (start > end)
798 break;
799
Filipe Mananac50d3e72014-03-31 14:53:25 +0100800 if (!cond_resched_lock(&tree->lock)) {
801 node = rb_next(node);
802 goto process_node;
803 }
Chris Masond1310b22008-01-24 16:13:08 -0500804 }
805out:
Chris Masoncad321a2008-12-17 14:51:42 -0500806 spin_unlock(&tree->lock);
Chris Masond1310b22008-01-24 16:13:08 -0500807}
Chris Masond1310b22008-01-24 16:13:08 -0500808
Jeff Mahoney1bf85042011-07-21 16:56:09 +0000809static void set_state_bits(struct extent_io_tree *tree,
Chris Masond1310b22008-01-24 16:13:08 -0500810 struct extent_state *state,
Qu Wenruod38ed272015-10-12 14:53:37 +0800811 unsigned *bits, struct extent_changeset *changeset)
Chris Masond1310b22008-01-24 16:13:08 -0500812{
David Sterba9ee49a042015-01-14 19:52:13 +0100813 unsigned bits_to_set = *bits & ~EXTENT_CTLBITS;
Josef Bacik9ed74f22009-09-11 16:12:44 -0400814
Jeff Mahoney1bf85042011-07-21 16:56:09 +0000815 set_state_cb(tree, state, bits);
Yan, Zheng0ca1f7c2010-05-16 10:48:47 -0400816 if ((bits_to_set & EXTENT_DIRTY) && !(state->state & EXTENT_DIRTY)) {
Chris Masond1310b22008-01-24 16:13:08 -0500817 u64 range = state->end - state->start + 1;
818 tree->dirty_bytes += range;
819 }
Qu Wenruod38ed272015-10-12 14:53:37 +0800820 add_extent_changeset(state, bits_to_set, changeset, 1);
Yan, Zheng0ca1f7c2010-05-16 10:48:47 -0400821 state->state |= bits_to_set;
Chris Masond1310b22008-01-24 16:13:08 -0500822}
823
Filipe Mananae38e2ed2014-10-13 12:28:38 +0100824static void cache_state_if_flags(struct extent_state *state,
825 struct extent_state **cached_ptr,
David Sterba9ee49a042015-01-14 19:52:13 +0100826 unsigned flags)
Chris Mason2c64c532009-09-02 15:04:12 -0400827{
828 if (cached_ptr && !(*cached_ptr)) {
Filipe Mananae38e2ed2014-10-13 12:28:38 +0100829 if (!flags || (state->state & flags)) {
Chris Mason2c64c532009-09-02 15:04:12 -0400830 *cached_ptr = state;
831 atomic_inc(&state->refs);
832 }
833 }
834}
835
Filipe Mananae38e2ed2014-10-13 12:28:38 +0100836static void cache_state(struct extent_state *state,
837 struct extent_state **cached_ptr)
838{
839 return cache_state_if_flags(state, cached_ptr,
840 EXTENT_IOBITS | EXTENT_BOUNDARY);
841}
842
Chris Masond1310b22008-01-24 16:13:08 -0500843/*
Chris Mason1edbb732009-09-02 13:24:36 -0400844 * set some bits on a range in the tree. This may require allocations or
845 * sleeping, so the gfp mask is used to indicate what is allowed.
Chris Masond1310b22008-01-24 16:13:08 -0500846 *
Chris Mason1edbb732009-09-02 13:24:36 -0400847 * If any of the exclusive bits are set, this will fail with -EEXIST if some
848 * part of the range already has the desired bits set. The start of the
849 * existing range is returned in failed_start in this case.
Chris Masond1310b22008-01-24 16:13:08 -0500850 *
Chris Mason1edbb732009-09-02 13:24:36 -0400851 * [start, end] is inclusive This takes the tree lock.
Chris Masond1310b22008-01-24 16:13:08 -0500852 */
Chris Mason1edbb732009-09-02 13:24:36 -0400853
Jeff Mahoney3fbe5c02012-03-01 14:57:19 +0100854static int __must_check
855__set_extent_bit(struct extent_io_tree *tree, u64 start, u64 end,
David Sterba9ee49a042015-01-14 19:52:13 +0100856 unsigned bits, unsigned exclusive_bits,
David Sterba41074882013-04-29 13:38:46 +0000857 u64 *failed_start, struct extent_state **cached_state,
Qu Wenruod38ed272015-10-12 14:53:37 +0800858 gfp_t mask, struct extent_changeset *changeset)
Chris Masond1310b22008-01-24 16:13:08 -0500859{
860 struct extent_state *state;
861 struct extent_state *prealloc = NULL;
862 struct rb_node *node;
Filipe David Borba Manana12cfbad2013-11-26 15:41:47 +0000863 struct rb_node **p;
864 struct rb_node *parent;
Chris Masond1310b22008-01-24 16:13:08 -0500865 int err = 0;
Chris Masond1310b22008-01-24 16:13:08 -0500866 u64 last_start;
867 u64 last_end;
Chris Mason42daec22009-09-23 19:51:09 -0400868
Josef Bacika5dee372013-12-13 10:02:44 -0500869 btrfs_debug_check_extent_io_range(tree, start, end);
David Sterba8d599ae2013-04-30 15:22:23 +0000870
Yan, Zheng0ca1f7c2010-05-16 10:48:47 -0400871 bits |= EXTENT_FIRST_DELALLOC;
Chris Masond1310b22008-01-24 16:13:08 -0500872again:
Mel Gormand0164ad2015-11-06 16:28:21 -0800873 if (!prealloc && gfpflags_allow_blocking(mask)) {
David Sterba059f7912016-04-27 01:03:45 +0200874 /*
875 * Don't care for allocation failure here because we might end
876 * up not needing the pre-allocated extent state at all, which
877 * is the case if we only have in the tree extent states that
878 * cover our input range and don't cover too any other range.
879 * If we end up needing a new extent state we allocate it later.
880 */
Chris Masond1310b22008-01-24 16:13:08 -0500881 prealloc = alloc_extent_state(mask);
Chris Masond1310b22008-01-24 16:13:08 -0500882 }
883
Chris Masoncad321a2008-12-17 14:51:42 -0500884 spin_lock(&tree->lock);
Chris Mason9655d292009-09-02 15:22:30 -0400885 if (cached_state && *cached_state) {
886 state = *cached_state;
Josef Bacikdf98b6e2011-06-20 14:53:48 -0400887 if (state->start <= start && state->end > start &&
Filipe Manana27a35072014-07-06 20:09:59 +0100888 extent_state_in_tree(state)) {
Chris Mason9655d292009-09-02 15:22:30 -0400889 node = &state->rb_node;
890 goto hit_next;
891 }
892 }
Chris Masond1310b22008-01-24 16:13:08 -0500893 /*
894 * this search will find all the extents that end after
895 * our range starts.
896 */
Filipe David Borba Manana12cfbad2013-11-26 15:41:47 +0000897 node = tree_search_for_insert(tree, start, &p, &parent);
Chris Masond1310b22008-01-24 16:13:08 -0500898 if (!node) {
Xiao Guangrong82337672011-04-20 06:44:57 +0000899 prealloc = alloc_extent_state_atomic(prealloc);
900 BUG_ON(!prealloc);
Filipe David Borba Manana12cfbad2013-11-26 15:41:47 +0000901 err = insert_state(tree, prealloc, start, end,
Qu Wenruod38ed272015-10-12 14:53:37 +0800902 &p, &parent, &bits, changeset);
Jeff Mahoneyc2d904e2011-10-03 23:22:32 -0400903 if (err)
904 extent_io_tree_panic(tree, err);
905
Filipe David Borba Mananac42ac0b2013-11-26 15:01:34 +0000906 cache_state(prealloc, cached_state);
Chris Masond1310b22008-01-24 16:13:08 -0500907 prealloc = NULL;
Chris Masond1310b22008-01-24 16:13:08 -0500908 goto out;
909 }
Chris Masond1310b22008-01-24 16:13:08 -0500910 state = rb_entry(node, struct extent_state, rb_node);
Chris Mason40431d62009-08-05 12:57:59 -0400911hit_next:
Chris Masond1310b22008-01-24 16:13:08 -0500912 last_start = state->start;
913 last_end = state->end;
914
915 /*
916 * | ---- desired range ---- |
917 * | state |
918 *
919 * Just lock what we found and keep going
920 */
921 if (state->start == start && state->end <= end) {
Chris Mason1edbb732009-09-02 13:24:36 -0400922 if (state->state & exclusive_bits) {
Chris Masond1310b22008-01-24 16:13:08 -0500923 *failed_start = state->start;
924 err = -EEXIST;
925 goto out;
926 }
Chris Mason42daec22009-09-23 19:51:09 -0400927
Qu Wenruod38ed272015-10-12 14:53:37 +0800928 set_state_bits(tree, state, &bits, changeset);
Chris Mason2c64c532009-09-02 15:04:12 -0400929 cache_state(state, cached_state);
Chris Masond1310b22008-01-24 16:13:08 -0500930 merge_state(tree, state);
Yan Zheng5c939df2009-05-27 09:16:03 -0400931 if (last_end == (u64)-1)
932 goto out;
933 start = last_end + 1;
Liu Bod1ac6e42012-05-10 18:10:39 +0800934 state = next_state(state);
935 if (start < end && state && state->start == start &&
936 !need_resched())
937 goto hit_next;
Chris Masond1310b22008-01-24 16:13:08 -0500938 goto search_again;
939 }
940
941 /*
942 * | ---- desired range ---- |
943 * | state |
944 * or
945 * | ------------- state -------------- |
946 *
947 * We need to split the extent we found, and may flip bits on
948 * second half.
949 *
950 * If the extent we found extends past our
951 * range, we just split and search again. It'll get split
952 * again the next time though.
953 *
954 * If the extent we found is inside our range, we set the
955 * desired bit on it.
956 */
957 if (state->start < start) {
Chris Mason1edbb732009-09-02 13:24:36 -0400958 if (state->state & exclusive_bits) {
Chris Masond1310b22008-01-24 16:13:08 -0500959 *failed_start = start;
960 err = -EEXIST;
961 goto out;
962 }
Xiao Guangrong82337672011-04-20 06:44:57 +0000963
964 prealloc = alloc_extent_state_atomic(prealloc);
965 BUG_ON(!prealloc);
Chris Masond1310b22008-01-24 16:13:08 -0500966 err = split_state(tree, state, prealloc, start);
Jeff Mahoneyc2d904e2011-10-03 23:22:32 -0400967 if (err)
968 extent_io_tree_panic(tree, err);
969
Chris Masond1310b22008-01-24 16:13:08 -0500970 prealloc = NULL;
971 if (err)
972 goto out;
973 if (state->end <= end) {
Qu Wenruod38ed272015-10-12 14:53:37 +0800974 set_state_bits(tree, state, &bits, changeset);
Chris Mason2c64c532009-09-02 15:04:12 -0400975 cache_state(state, cached_state);
Chris Masond1310b22008-01-24 16:13:08 -0500976 merge_state(tree, state);
Yan Zheng5c939df2009-05-27 09:16:03 -0400977 if (last_end == (u64)-1)
978 goto out;
979 start = last_end + 1;
Liu Bod1ac6e42012-05-10 18:10:39 +0800980 state = next_state(state);
981 if (start < end && state && state->start == start &&
982 !need_resched())
983 goto hit_next;
Chris Masond1310b22008-01-24 16:13:08 -0500984 }
985 goto search_again;
986 }
987 /*
988 * | ---- desired range ---- |
989 * | state | or | state |
990 *
991 * There's a hole, we need to insert something in it and
992 * ignore the extent we found.
993 */
994 if (state->start > start) {
995 u64 this_end;
996 if (end < last_start)
997 this_end = end;
998 else
Chris Masond3977122009-01-05 21:25:51 -0500999 this_end = last_start - 1;
Xiao Guangrong82337672011-04-20 06:44:57 +00001000
1001 prealloc = alloc_extent_state_atomic(prealloc);
1002 BUG_ON(!prealloc);
Xiao Guangrongc7f895a2011-04-20 06:45:49 +00001003
1004 /*
1005 * Avoid to free 'prealloc' if it can be merged with
1006 * the later extent.
1007 */
Chris Masond1310b22008-01-24 16:13:08 -05001008 err = insert_state(tree, prealloc, start, this_end,
Qu Wenruod38ed272015-10-12 14:53:37 +08001009 NULL, NULL, &bits, changeset);
Jeff Mahoneyc2d904e2011-10-03 23:22:32 -04001010 if (err)
1011 extent_io_tree_panic(tree, err);
1012
Chris Mason2c64c532009-09-02 15:04:12 -04001013 cache_state(prealloc, cached_state);
Chris Masond1310b22008-01-24 16:13:08 -05001014 prealloc = NULL;
Chris Masond1310b22008-01-24 16:13:08 -05001015 start = this_end + 1;
1016 goto search_again;
1017 }
1018 /*
1019 * | ---- desired range ---- |
1020 * | state |
1021 * We need to split the extent, and set the bit
1022 * on the first half
1023 */
1024 if (state->start <= end && state->end > end) {
Chris Mason1edbb732009-09-02 13:24:36 -04001025 if (state->state & exclusive_bits) {
Chris Masond1310b22008-01-24 16:13:08 -05001026 *failed_start = start;
1027 err = -EEXIST;
1028 goto out;
1029 }
Xiao Guangrong82337672011-04-20 06:44:57 +00001030
1031 prealloc = alloc_extent_state_atomic(prealloc);
1032 BUG_ON(!prealloc);
Chris Masond1310b22008-01-24 16:13:08 -05001033 err = split_state(tree, state, prealloc, end + 1);
Jeff Mahoneyc2d904e2011-10-03 23:22:32 -04001034 if (err)
1035 extent_io_tree_panic(tree, err);
Chris Masond1310b22008-01-24 16:13:08 -05001036
Qu Wenruod38ed272015-10-12 14:53:37 +08001037 set_state_bits(tree, prealloc, &bits, changeset);
Chris Mason2c64c532009-09-02 15:04:12 -04001038 cache_state(prealloc, cached_state);
Chris Masond1310b22008-01-24 16:13:08 -05001039 merge_state(tree, prealloc);
1040 prealloc = NULL;
1041 goto out;
1042 }
1043
David Sterbab5a4ba12016-04-27 01:02:15 +02001044search_again:
1045 if (start > end)
1046 goto out;
1047 spin_unlock(&tree->lock);
1048 if (gfpflags_allow_blocking(mask))
1049 cond_resched();
1050 goto again;
Chris Masond1310b22008-01-24 16:13:08 -05001051
1052out:
Chris Masoncad321a2008-12-17 14:51:42 -05001053 spin_unlock(&tree->lock);
Chris Masond1310b22008-01-24 16:13:08 -05001054 if (prealloc)
1055 free_extent_state(prealloc);
1056
1057 return err;
1058
Chris Masond1310b22008-01-24 16:13:08 -05001059}
Chris Masond1310b22008-01-24 16:13:08 -05001060
David Sterba41074882013-04-29 13:38:46 +00001061int set_extent_bit(struct extent_io_tree *tree, u64 start, u64 end,
David Sterba9ee49a042015-01-14 19:52:13 +01001062 unsigned bits, u64 * failed_start,
David Sterba41074882013-04-29 13:38:46 +00001063 struct extent_state **cached_state, gfp_t mask)
Jeff Mahoney3fbe5c02012-03-01 14:57:19 +01001064{
1065 return __set_extent_bit(tree, start, end, bits, 0, failed_start,
Qu Wenruod38ed272015-10-12 14:53:37 +08001066 cached_state, mask, NULL);
Jeff Mahoney3fbe5c02012-03-01 14:57:19 +01001067}
1068
1069
Josef Bacik462d6fa2011-09-26 13:56:12 -04001070/**
Liu Bo10983f22012-07-11 15:26:19 +08001071 * convert_extent_bit - convert all bits in a given range from one bit to
1072 * another
Josef Bacik462d6fa2011-09-26 13:56:12 -04001073 * @tree: the io tree to search
1074 * @start: the start offset in bytes
1075 * @end: the end offset in bytes (inclusive)
1076 * @bits: the bits to set in this range
1077 * @clear_bits: the bits to clear in this range
Josef Bacike6138872012-09-27 17:07:30 -04001078 * @cached_state: state that we're going to cache
Josef Bacik462d6fa2011-09-26 13:56:12 -04001079 *
1080 * This will go through and set bits for the given range. If any states exist
1081 * already in this range they are set with the given bit and cleared of the
1082 * clear_bits. This is only meant to be used by things that are mergeable, ie
1083 * converting from say DELALLOC to DIRTY. This is not meant to be used with
1084 * boundary bits like LOCK.
David Sterba210aa272016-04-26 23:54:39 +02001085 *
1086 * All allocations are done with GFP_NOFS.
Josef Bacik462d6fa2011-09-26 13:56:12 -04001087 */
1088int convert_extent_bit(struct extent_io_tree *tree, u64 start, u64 end,
David Sterba9ee49a042015-01-14 19:52:13 +01001089 unsigned bits, unsigned clear_bits,
David Sterba210aa272016-04-26 23:54:39 +02001090 struct extent_state **cached_state)
Josef Bacik462d6fa2011-09-26 13:56:12 -04001091{
1092 struct extent_state *state;
1093 struct extent_state *prealloc = NULL;
1094 struct rb_node *node;
Filipe David Borba Manana12cfbad2013-11-26 15:41:47 +00001095 struct rb_node **p;
1096 struct rb_node *parent;
Josef Bacik462d6fa2011-09-26 13:56:12 -04001097 int err = 0;
1098 u64 last_start;
1099 u64 last_end;
Filipe Mananac8fd3de2014-10-13 12:28:39 +01001100 bool first_iteration = true;
Josef Bacik462d6fa2011-09-26 13:56:12 -04001101
Josef Bacika5dee372013-12-13 10:02:44 -05001102 btrfs_debug_check_extent_io_range(tree, start, end);
David Sterba8d599ae2013-04-30 15:22:23 +00001103
Josef Bacik462d6fa2011-09-26 13:56:12 -04001104again:
David Sterba210aa272016-04-26 23:54:39 +02001105 if (!prealloc) {
Filipe Mananac8fd3de2014-10-13 12:28:39 +01001106 /*
1107 * Best effort, don't worry if extent state allocation fails
1108 * here for the first iteration. We might have a cached state
1109 * that matches exactly the target range, in which case no
1110 * extent state allocations are needed. We'll only know this
1111 * after locking the tree.
1112 */
David Sterba210aa272016-04-26 23:54:39 +02001113 prealloc = alloc_extent_state(GFP_NOFS);
Filipe Mananac8fd3de2014-10-13 12:28:39 +01001114 if (!prealloc && !first_iteration)
Josef Bacik462d6fa2011-09-26 13:56:12 -04001115 return -ENOMEM;
1116 }
1117
1118 spin_lock(&tree->lock);
Josef Bacike6138872012-09-27 17:07:30 -04001119 if (cached_state && *cached_state) {
1120 state = *cached_state;
1121 if (state->start <= start && state->end > start &&
Filipe Manana27a35072014-07-06 20:09:59 +01001122 extent_state_in_tree(state)) {
Josef Bacike6138872012-09-27 17:07:30 -04001123 node = &state->rb_node;
1124 goto hit_next;
1125 }
1126 }
1127
Josef Bacik462d6fa2011-09-26 13:56:12 -04001128 /*
1129 * this search will find all the extents that end after
1130 * our range starts.
1131 */
Filipe David Borba Manana12cfbad2013-11-26 15:41:47 +00001132 node = tree_search_for_insert(tree, start, &p, &parent);
Josef Bacik462d6fa2011-09-26 13:56:12 -04001133 if (!node) {
1134 prealloc = alloc_extent_state_atomic(prealloc);
Liu Bo1cf4ffd2011-12-07 20:08:40 -05001135 if (!prealloc) {
1136 err = -ENOMEM;
1137 goto out;
1138 }
Filipe David Borba Manana12cfbad2013-11-26 15:41:47 +00001139 err = insert_state(tree, prealloc, start, end,
Qu Wenruod38ed272015-10-12 14:53:37 +08001140 &p, &parent, &bits, NULL);
Jeff Mahoneyc2d904e2011-10-03 23:22:32 -04001141 if (err)
1142 extent_io_tree_panic(tree, err);
Filipe David Borba Mananac42ac0b2013-11-26 15:01:34 +00001143 cache_state(prealloc, cached_state);
1144 prealloc = NULL;
Josef Bacik462d6fa2011-09-26 13:56:12 -04001145 goto out;
1146 }
1147 state = rb_entry(node, struct extent_state, rb_node);
1148hit_next:
1149 last_start = state->start;
1150 last_end = state->end;
1151
1152 /*
1153 * | ---- desired range ---- |
1154 * | state |
1155 *
1156 * Just lock what we found and keep going
1157 */
1158 if (state->start == start && state->end <= end) {
Qu Wenruod38ed272015-10-12 14:53:37 +08001159 set_state_bits(tree, state, &bits, NULL);
Josef Bacike6138872012-09-27 17:07:30 -04001160 cache_state(state, cached_state);
Qu Wenruofefdc552015-10-12 15:35:38 +08001161 state = clear_state_bit(tree, state, &clear_bits, 0, NULL);
Josef Bacik462d6fa2011-09-26 13:56:12 -04001162 if (last_end == (u64)-1)
1163 goto out;
Josef Bacik462d6fa2011-09-26 13:56:12 -04001164 start = last_end + 1;
Liu Bod1ac6e42012-05-10 18:10:39 +08001165 if (start < end && state && state->start == start &&
1166 !need_resched())
1167 goto hit_next;
Josef Bacik462d6fa2011-09-26 13:56:12 -04001168 goto search_again;
1169 }
1170
1171 /*
1172 * | ---- desired range ---- |
1173 * | state |
1174 * or
1175 * | ------------- state -------------- |
1176 *
1177 * We need to split the extent we found, and may flip bits on
1178 * second half.
1179 *
1180 * If the extent we found extends past our
1181 * range, we just split and search again. It'll get split
1182 * again the next time though.
1183 *
1184 * If the extent we found is inside our range, we set the
1185 * desired bit on it.
1186 */
1187 if (state->start < start) {
1188 prealloc = alloc_extent_state_atomic(prealloc);
Liu Bo1cf4ffd2011-12-07 20:08:40 -05001189 if (!prealloc) {
1190 err = -ENOMEM;
1191 goto out;
1192 }
Josef Bacik462d6fa2011-09-26 13:56:12 -04001193 err = split_state(tree, state, prealloc, start);
Jeff Mahoneyc2d904e2011-10-03 23:22:32 -04001194 if (err)
1195 extent_io_tree_panic(tree, err);
Josef Bacik462d6fa2011-09-26 13:56:12 -04001196 prealloc = NULL;
1197 if (err)
1198 goto out;
1199 if (state->end <= end) {
Qu Wenruod38ed272015-10-12 14:53:37 +08001200 set_state_bits(tree, state, &bits, NULL);
Josef Bacike6138872012-09-27 17:07:30 -04001201 cache_state(state, cached_state);
Qu Wenruofefdc552015-10-12 15:35:38 +08001202 state = clear_state_bit(tree, state, &clear_bits, 0,
1203 NULL);
Josef Bacik462d6fa2011-09-26 13:56:12 -04001204 if (last_end == (u64)-1)
1205 goto out;
1206 start = last_end + 1;
Liu Bod1ac6e42012-05-10 18:10:39 +08001207 if (start < end && state && state->start == start &&
1208 !need_resched())
1209 goto hit_next;
Josef Bacik462d6fa2011-09-26 13:56:12 -04001210 }
1211 goto search_again;
1212 }
1213 /*
1214 * | ---- desired range ---- |
1215 * | state | or | state |
1216 *
1217 * There's a hole, we need to insert something in it and
1218 * ignore the extent we found.
1219 */
1220 if (state->start > start) {
1221 u64 this_end;
1222 if (end < last_start)
1223 this_end = end;
1224 else
1225 this_end = last_start - 1;
1226
1227 prealloc = alloc_extent_state_atomic(prealloc);
Liu Bo1cf4ffd2011-12-07 20:08:40 -05001228 if (!prealloc) {
1229 err = -ENOMEM;
1230 goto out;
1231 }
Josef Bacik462d6fa2011-09-26 13:56:12 -04001232
1233 /*
1234 * Avoid to free 'prealloc' if it can be merged with
1235 * the later extent.
1236 */
1237 err = insert_state(tree, prealloc, start, this_end,
Qu Wenruod38ed272015-10-12 14:53:37 +08001238 NULL, NULL, &bits, NULL);
Jeff Mahoneyc2d904e2011-10-03 23:22:32 -04001239 if (err)
1240 extent_io_tree_panic(tree, err);
Josef Bacike6138872012-09-27 17:07:30 -04001241 cache_state(prealloc, cached_state);
Josef Bacik462d6fa2011-09-26 13:56:12 -04001242 prealloc = NULL;
1243 start = this_end + 1;
1244 goto search_again;
1245 }
1246 /*
1247 * | ---- desired range ---- |
1248 * | state |
1249 * We need to split the extent, and set the bit
1250 * on the first half
1251 */
1252 if (state->start <= end && state->end > end) {
1253 prealloc = alloc_extent_state_atomic(prealloc);
Liu Bo1cf4ffd2011-12-07 20:08:40 -05001254 if (!prealloc) {
1255 err = -ENOMEM;
1256 goto out;
1257 }
Josef Bacik462d6fa2011-09-26 13:56:12 -04001258
1259 err = split_state(tree, state, prealloc, end + 1);
Jeff Mahoneyc2d904e2011-10-03 23:22:32 -04001260 if (err)
1261 extent_io_tree_panic(tree, err);
Josef Bacik462d6fa2011-09-26 13:56:12 -04001262
Qu Wenruod38ed272015-10-12 14:53:37 +08001263 set_state_bits(tree, prealloc, &bits, NULL);
Josef Bacike6138872012-09-27 17:07:30 -04001264 cache_state(prealloc, cached_state);
Qu Wenruofefdc552015-10-12 15:35:38 +08001265 clear_state_bit(tree, prealloc, &clear_bits, 0, NULL);
Josef Bacik462d6fa2011-09-26 13:56:12 -04001266 prealloc = NULL;
1267 goto out;
1268 }
1269
Josef Bacik462d6fa2011-09-26 13:56:12 -04001270search_again:
1271 if (start > end)
1272 goto out;
1273 spin_unlock(&tree->lock);
David Sterba210aa272016-04-26 23:54:39 +02001274 cond_resched();
Filipe Mananac8fd3de2014-10-13 12:28:39 +01001275 first_iteration = false;
Josef Bacik462d6fa2011-09-26 13:56:12 -04001276 goto again;
Josef Bacik462d6fa2011-09-26 13:56:12 -04001277
1278out:
1279 spin_unlock(&tree->lock);
1280 if (prealloc)
1281 free_extent_state(prealloc);
1282
1283 return err;
1284}
1285
Chris Masond1310b22008-01-24 16:13:08 -05001286/* wrappers around set/clear extent bit */
Qu Wenruod38ed272015-10-12 14:53:37 +08001287int set_record_extent_bits(struct extent_io_tree *tree, u64 start, u64 end,
David Sterba2c53b912016-04-26 23:54:39 +02001288 unsigned bits, struct extent_changeset *changeset)
Qu Wenruod38ed272015-10-12 14:53:37 +08001289{
1290 /*
1291 * We don't support EXTENT_LOCKED yet, as current changeset will
1292 * record any bits changed, so for EXTENT_LOCKED case, it will
1293 * either fail with -EEXIST or changeset will record the whole
1294 * range.
1295 */
1296 BUG_ON(bits & EXTENT_LOCKED);
1297
David Sterba2c53b912016-04-26 23:54:39 +02001298 return __set_extent_bit(tree, start, end, bits, 0, NULL, NULL, GFP_NOFS,
Qu Wenruod38ed272015-10-12 14:53:37 +08001299 changeset);
1300}
1301
Qu Wenruofefdc552015-10-12 15:35:38 +08001302int clear_extent_bit(struct extent_io_tree *tree, u64 start, u64 end,
1303 unsigned bits, int wake, int delete,
1304 struct extent_state **cached, gfp_t mask)
1305{
1306 return __clear_extent_bit(tree, start, end, bits, wake, delete,
1307 cached, mask, NULL);
1308}
1309
Qu Wenruofefdc552015-10-12 15:35:38 +08001310int clear_record_extent_bits(struct extent_io_tree *tree, u64 start, u64 end,
David Sterbaf734c442016-04-26 23:54:39 +02001311 unsigned bits, struct extent_changeset *changeset)
Qu Wenruofefdc552015-10-12 15:35:38 +08001312{
1313 /*
1314 * Don't support EXTENT_LOCKED case, same reason as
1315 * set_record_extent_bits().
1316 */
1317 BUG_ON(bits & EXTENT_LOCKED);
1318
David Sterbaf734c442016-04-26 23:54:39 +02001319 return __clear_extent_bit(tree, start, end, bits, 0, 0, NULL, GFP_NOFS,
Qu Wenruofefdc552015-10-12 15:35:38 +08001320 changeset);
1321}
1322
Chris Masond352ac62008-09-29 15:18:18 -04001323/*
1324 * either insert or lock state struct between start and end use mask to tell
1325 * us if waiting is desired.
1326 */
Chris Mason1edbb732009-09-02 13:24:36 -04001327int lock_extent_bits(struct extent_io_tree *tree, u64 start, u64 end,
David Sterbaff13db42015-12-03 14:30:40 +01001328 struct extent_state **cached_state)
Chris Masond1310b22008-01-24 16:13:08 -05001329{
1330 int err;
1331 u64 failed_start;
David Sterba9ee49a042015-01-14 19:52:13 +01001332
Chris Masond1310b22008-01-24 16:13:08 -05001333 while (1) {
David Sterbaff13db42015-12-03 14:30:40 +01001334 err = __set_extent_bit(tree, start, end, EXTENT_LOCKED,
Jeff Mahoney3fbe5c02012-03-01 14:57:19 +01001335 EXTENT_LOCKED, &failed_start,
Qu Wenruod38ed272015-10-12 14:53:37 +08001336 cached_state, GFP_NOFS, NULL);
Jeff Mahoneyd0082372012-03-01 14:57:19 +01001337 if (err == -EEXIST) {
Chris Masond1310b22008-01-24 16:13:08 -05001338 wait_extent_bit(tree, failed_start, end, EXTENT_LOCKED);
1339 start = failed_start;
Jeff Mahoneyd0082372012-03-01 14:57:19 +01001340 } else
Chris Masond1310b22008-01-24 16:13:08 -05001341 break;
Chris Masond1310b22008-01-24 16:13:08 -05001342 WARN_ON(start > end);
1343 }
1344 return err;
1345}
Chris Masond1310b22008-01-24 16:13:08 -05001346
Jeff Mahoneyd0082372012-03-01 14:57:19 +01001347int try_lock_extent(struct extent_io_tree *tree, u64 start, u64 end)
Josef Bacik25179202008-10-29 14:49:05 -04001348{
1349 int err;
1350 u64 failed_start;
1351
Jeff Mahoney3fbe5c02012-03-01 14:57:19 +01001352 err = __set_extent_bit(tree, start, end, EXTENT_LOCKED, EXTENT_LOCKED,
Qu Wenruod38ed272015-10-12 14:53:37 +08001353 &failed_start, NULL, GFP_NOFS, NULL);
Yan Zheng66435582008-10-30 14:19:50 -04001354 if (err == -EEXIST) {
1355 if (failed_start > start)
1356 clear_extent_bit(tree, start, failed_start - 1,
Jeff Mahoneyd0082372012-03-01 14:57:19 +01001357 EXTENT_LOCKED, 1, 0, NULL, GFP_NOFS);
Josef Bacik25179202008-10-29 14:49:05 -04001358 return 0;
Yan Zheng66435582008-10-30 14:19:50 -04001359 }
Josef Bacik25179202008-10-29 14:49:05 -04001360 return 1;
1361}
Josef Bacik25179202008-10-29 14:49:05 -04001362
David Sterbabd1fa4f2015-12-03 13:08:59 +01001363void extent_range_clear_dirty_for_io(struct inode *inode, u64 start, u64 end)
Chris Mason4adaa612013-03-26 13:07:00 -04001364{
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03001365 unsigned long index = start >> PAGE_SHIFT;
1366 unsigned long end_index = end >> PAGE_SHIFT;
Chris Mason4adaa612013-03-26 13:07:00 -04001367 struct page *page;
1368
1369 while (index <= end_index) {
1370 page = find_get_page(inode->i_mapping, index);
1371 BUG_ON(!page); /* Pages should be in the extent_io_tree */
1372 clear_page_dirty_for_io(page);
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03001373 put_page(page);
Chris Mason4adaa612013-03-26 13:07:00 -04001374 index++;
1375 }
Chris Mason4adaa612013-03-26 13:07:00 -04001376}
1377
David Sterbaf6311572015-12-03 13:08:59 +01001378void extent_range_redirty_for_io(struct inode *inode, u64 start, u64 end)
Chris Mason4adaa612013-03-26 13:07:00 -04001379{
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03001380 unsigned long index = start >> PAGE_SHIFT;
1381 unsigned long end_index = end >> PAGE_SHIFT;
Chris Mason4adaa612013-03-26 13:07:00 -04001382 struct page *page;
1383
1384 while (index <= end_index) {
1385 page = find_get_page(inode->i_mapping, index);
1386 BUG_ON(!page); /* Pages should be in the extent_io_tree */
Chris Mason4adaa612013-03-26 13:07:00 -04001387 __set_page_dirty_nobuffers(page);
Konstantin Khebnikov8d386332015-02-11 15:26:55 -08001388 account_page_redirty(page);
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03001389 put_page(page);
Chris Mason4adaa612013-03-26 13:07:00 -04001390 index++;
1391 }
Chris Mason4adaa612013-03-26 13:07:00 -04001392}
1393
Chris Masond1310b22008-01-24 16:13:08 -05001394/*
Chris Masond1310b22008-01-24 16:13:08 -05001395 * helper function to set both pages and extents in the tree writeback
1396 */
David Sterba35de6db2015-12-03 13:08:59 +01001397static void set_range_writeback(struct extent_io_tree *tree, u64 start, u64 end)
Chris Masond1310b22008-01-24 16:13:08 -05001398{
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03001399 unsigned long index = start >> PAGE_SHIFT;
1400 unsigned long end_index = end >> PAGE_SHIFT;
Chris Masond1310b22008-01-24 16:13:08 -05001401 struct page *page;
1402
1403 while (index <= end_index) {
1404 page = find_get_page(tree->mapping, index);
Jeff Mahoney79787ea2012-03-12 16:03:00 +01001405 BUG_ON(!page); /* Pages should be in the extent_io_tree */
Chris Masond1310b22008-01-24 16:13:08 -05001406 set_page_writeback(page);
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03001407 put_page(page);
Chris Masond1310b22008-01-24 16:13:08 -05001408 index++;
1409 }
Chris Masond1310b22008-01-24 16:13:08 -05001410}
Chris Masond1310b22008-01-24 16:13:08 -05001411
Chris Masond352ac62008-09-29 15:18:18 -04001412/* find the first state struct with 'bits' set after 'start', and
1413 * return it. tree->lock must be held. NULL will returned if
1414 * nothing was found after 'start'
1415 */
Eric Sandeen48a3b632013-04-25 20:41:01 +00001416static struct extent_state *
1417find_first_extent_bit_state(struct extent_io_tree *tree,
David Sterba9ee49a042015-01-14 19:52:13 +01001418 u64 start, unsigned bits)
Chris Masond7fc6402008-02-18 12:12:38 -05001419{
1420 struct rb_node *node;
1421 struct extent_state *state;
1422
1423 /*
1424 * this search will find all the extents that end after
1425 * our range starts.
1426 */
1427 node = tree_search(tree, start);
Chris Masond3977122009-01-05 21:25:51 -05001428 if (!node)
Chris Masond7fc6402008-02-18 12:12:38 -05001429 goto out;
Chris Masond7fc6402008-02-18 12:12:38 -05001430
Chris Masond3977122009-01-05 21:25:51 -05001431 while (1) {
Chris Masond7fc6402008-02-18 12:12:38 -05001432 state = rb_entry(node, struct extent_state, rb_node);
Chris Masond3977122009-01-05 21:25:51 -05001433 if (state->end >= start && (state->state & bits))
Chris Masond7fc6402008-02-18 12:12:38 -05001434 return state;
Chris Masond3977122009-01-05 21:25:51 -05001435
Chris Masond7fc6402008-02-18 12:12:38 -05001436 node = rb_next(node);
1437 if (!node)
1438 break;
1439 }
1440out:
1441 return NULL;
1442}
Chris Masond7fc6402008-02-18 12:12:38 -05001443
Chris Masond352ac62008-09-29 15:18:18 -04001444/*
Xiao Guangrong69261c42011-07-14 03:19:45 +00001445 * find the first offset in the io tree with 'bits' set. zero is
1446 * returned if we find something, and *start_ret and *end_ret are
1447 * set to reflect the state struct that was found.
1448 *
Wang Sheng-Hui477d7ea2012-04-06 14:35:47 +08001449 * If nothing was found, 1 is returned. If found something, return 0.
Xiao Guangrong69261c42011-07-14 03:19:45 +00001450 */
1451int find_first_extent_bit(struct extent_io_tree *tree, u64 start,
David Sterba9ee49a042015-01-14 19:52:13 +01001452 u64 *start_ret, u64 *end_ret, unsigned bits,
Josef Bacike6138872012-09-27 17:07:30 -04001453 struct extent_state **cached_state)
Xiao Guangrong69261c42011-07-14 03:19:45 +00001454{
1455 struct extent_state *state;
Josef Bacike6138872012-09-27 17:07:30 -04001456 struct rb_node *n;
Xiao Guangrong69261c42011-07-14 03:19:45 +00001457 int ret = 1;
1458
1459 spin_lock(&tree->lock);
Josef Bacike6138872012-09-27 17:07:30 -04001460 if (cached_state && *cached_state) {
1461 state = *cached_state;
Filipe Manana27a35072014-07-06 20:09:59 +01001462 if (state->end == start - 1 && extent_state_in_tree(state)) {
Josef Bacike6138872012-09-27 17:07:30 -04001463 n = rb_next(&state->rb_node);
1464 while (n) {
1465 state = rb_entry(n, struct extent_state,
1466 rb_node);
1467 if (state->state & bits)
1468 goto got_it;
1469 n = rb_next(n);
1470 }
1471 free_extent_state(*cached_state);
1472 *cached_state = NULL;
1473 goto out;
1474 }
1475 free_extent_state(*cached_state);
1476 *cached_state = NULL;
1477 }
1478
Xiao Guangrong69261c42011-07-14 03:19:45 +00001479 state = find_first_extent_bit_state(tree, start, bits);
Josef Bacike6138872012-09-27 17:07:30 -04001480got_it:
Xiao Guangrong69261c42011-07-14 03:19:45 +00001481 if (state) {
Filipe Mananae38e2ed2014-10-13 12:28:38 +01001482 cache_state_if_flags(state, cached_state, 0);
Xiao Guangrong69261c42011-07-14 03:19:45 +00001483 *start_ret = state->start;
1484 *end_ret = state->end;
1485 ret = 0;
1486 }
Josef Bacike6138872012-09-27 17:07:30 -04001487out:
Xiao Guangrong69261c42011-07-14 03:19:45 +00001488 spin_unlock(&tree->lock);
1489 return ret;
1490}
1491
1492/*
Chris Masond352ac62008-09-29 15:18:18 -04001493 * find a contiguous range of bytes in the file marked as delalloc, not
1494 * more than 'max_bytes'. start and end are used to return the range,
1495 *
1496 * 1 is returned if we find something, 0 if nothing was in the tree
1497 */
Chris Masonc8b97812008-10-29 14:49:59 -04001498static noinline u64 find_delalloc_range(struct extent_io_tree *tree,
Josef Bacikc2a128d2010-02-02 21:19:11 +00001499 u64 *start, u64 *end, u64 max_bytes,
1500 struct extent_state **cached_state)
Chris Masond1310b22008-01-24 16:13:08 -05001501{
1502 struct rb_node *node;
1503 struct extent_state *state;
1504 u64 cur_start = *start;
1505 u64 found = 0;
1506 u64 total_bytes = 0;
1507
Chris Masoncad321a2008-12-17 14:51:42 -05001508 spin_lock(&tree->lock);
Chris Masonc8b97812008-10-29 14:49:59 -04001509
Chris Masond1310b22008-01-24 16:13:08 -05001510 /*
1511 * this search will find all the extents that end after
1512 * our range starts.
1513 */
Chris Mason80ea96b2008-02-01 14:51:59 -05001514 node = tree_search(tree, cur_start);
Peter2b114d12008-04-01 11:21:40 -04001515 if (!node) {
Chris Mason3b951512008-04-17 11:29:12 -04001516 if (!found)
1517 *end = (u64)-1;
Chris Masond1310b22008-01-24 16:13:08 -05001518 goto out;
1519 }
1520
Chris Masond3977122009-01-05 21:25:51 -05001521 while (1) {
Chris Masond1310b22008-01-24 16:13:08 -05001522 state = rb_entry(node, struct extent_state, rb_node);
Zheng Yan5b21f2e2008-09-26 10:05:38 -04001523 if (found && (state->start != cur_start ||
1524 (state->state & EXTENT_BOUNDARY))) {
Chris Masond1310b22008-01-24 16:13:08 -05001525 goto out;
1526 }
1527 if (!(state->state & EXTENT_DELALLOC)) {
1528 if (!found)
1529 *end = state->end;
1530 goto out;
1531 }
Josef Bacikc2a128d2010-02-02 21:19:11 +00001532 if (!found) {
Chris Masond1310b22008-01-24 16:13:08 -05001533 *start = state->start;
Josef Bacikc2a128d2010-02-02 21:19:11 +00001534 *cached_state = state;
1535 atomic_inc(&state->refs);
1536 }
Chris Masond1310b22008-01-24 16:13:08 -05001537 found++;
1538 *end = state->end;
1539 cur_start = state->end + 1;
1540 node = rb_next(node);
Chris Masond1310b22008-01-24 16:13:08 -05001541 total_bytes += state->end - state->start + 1;
Josef Bacik7bf811a52013-10-07 22:11:09 -04001542 if (total_bytes >= max_bytes)
Josef Bacik573aeca2013-08-30 14:38:49 -04001543 break;
Josef Bacik573aeca2013-08-30 14:38:49 -04001544 if (!node)
Chris Masond1310b22008-01-24 16:13:08 -05001545 break;
1546 }
1547out:
Chris Masoncad321a2008-12-17 14:51:42 -05001548 spin_unlock(&tree->lock);
Chris Masond1310b22008-01-24 16:13:08 -05001549 return found;
1550}
1551
Jeff Mahoney143bede2012-03-01 14:56:26 +01001552static noinline void __unlock_for_delalloc(struct inode *inode,
1553 struct page *locked_page,
1554 u64 start, u64 end)
Chris Masonc8b97812008-10-29 14:49:59 -04001555{
1556 int ret;
1557 struct page *pages[16];
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03001558 unsigned long index = start >> PAGE_SHIFT;
1559 unsigned long end_index = end >> PAGE_SHIFT;
Chris Masonc8b97812008-10-29 14:49:59 -04001560 unsigned long nr_pages = end_index - index + 1;
1561 int i;
1562
1563 if (index == locked_page->index && end_index == index)
Jeff Mahoney143bede2012-03-01 14:56:26 +01001564 return;
Chris Masonc8b97812008-10-29 14:49:59 -04001565
Chris Masond3977122009-01-05 21:25:51 -05001566 while (nr_pages > 0) {
Chris Masonc8b97812008-10-29 14:49:59 -04001567 ret = find_get_pages_contig(inode->i_mapping, index,
Chris Mason5b050f02008-11-11 09:34:41 -05001568 min_t(unsigned long, nr_pages,
1569 ARRAY_SIZE(pages)), pages);
Chris Masonc8b97812008-10-29 14:49:59 -04001570 for (i = 0; i < ret; i++) {
1571 if (pages[i] != locked_page)
1572 unlock_page(pages[i]);
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03001573 put_page(pages[i]);
Chris Masonc8b97812008-10-29 14:49:59 -04001574 }
1575 nr_pages -= ret;
1576 index += ret;
1577 cond_resched();
1578 }
Chris Masonc8b97812008-10-29 14:49:59 -04001579}
1580
1581static noinline int lock_delalloc_pages(struct inode *inode,
1582 struct page *locked_page,
1583 u64 delalloc_start,
1584 u64 delalloc_end)
1585{
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03001586 unsigned long index = delalloc_start >> PAGE_SHIFT;
Chris Masonc8b97812008-10-29 14:49:59 -04001587 unsigned long start_index = index;
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03001588 unsigned long end_index = delalloc_end >> PAGE_SHIFT;
Chris Masonc8b97812008-10-29 14:49:59 -04001589 unsigned long pages_locked = 0;
1590 struct page *pages[16];
1591 unsigned long nrpages;
1592 int ret;
1593 int i;
1594
1595 /* the caller is responsible for locking the start index */
1596 if (index == locked_page->index && index == end_index)
1597 return 0;
1598
1599 /* skip the page at the start index */
1600 nrpages = end_index - index + 1;
Chris Masond3977122009-01-05 21:25:51 -05001601 while (nrpages > 0) {
Chris Masonc8b97812008-10-29 14:49:59 -04001602 ret = find_get_pages_contig(inode->i_mapping, index,
Chris Mason5b050f02008-11-11 09:34:41 -05001603 min_t(unsigned long,
1604 nrpages, ARRAY_SIZE(pages)), pages);
Chris Masonc8b97812008-10-29 14:49:59 -04001605 if (ret == 0) {
1606 ret = -EAGAIN;
1607 goto done;
1608 }
1609 /* now we have an array of pages, lock them all */
1610 for (i = 0; i < ret; i++) {
1611 /*
1612 * the caller is taking responsibility for
1613 * locked_page
1614 */
Chris Mason771ed682008-11-06 22:02:51 -05001615 if (pages[i] != locked_page) {
Chris Masonc8b97812008-10-29 14:49:59 -04001616 lock_page(pages[i]);
Chris Masonf2b1c412008-11-10 07:31:30 -05001617 if (!PageDirty(pages[i]) ||
1618 pages[i]->mapping != inode->i_mapping) {
Chris Mason771ed682008-11-06 22:02:51 -05001619 ret = -EAGAIN;
1620 unlock_page(pages[i]);
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03001621 put_page(pages[i]);
Chris Mason771ed682008-11-06 22:02:51 -05001622 goto done;
1623 }
1624 }
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03001625 put_page(pages[i]);
Chris Mason771ed682008-11-06 22:02:51 -05001626 pages_locked++;
Chris Masonc8b97812008-10-29 14:49:59 -04001627 }
Chris Masonc8b97812008-10-29 14:49:59 -04001628 nrpages -= ret;
1629 index += ret;
1630 cond_resched();
1631 }
1632 ret = 0;
1633done:
1634 if (ret && pages_locked) {
1635 __unlock_for_delalloc(inode, locked_page,
1636 delalloc_start,
1637 ((u64)(start_index + pages_locked - 1)) <<
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03001638 PAGE_SHIFT);
Chris Masonc8b97812008-10-29 14:49:59 -04001639 }
1640 return ret;
1641}
1642
1643/*
1644 * find a contiguous range of bytes in the file marked as delalloc, not
1645 * more than 'max_bytes'. start and end are used to return the range,
1646 *
1647 * 1 is returned if we find something, 0 if nothing was in the tree
1648 */
Josef Bacik294e30f2013-10-09 12:00:56 -04001649STATIC u64 find_lock_delalloc_range(struct inode *inode,
1650 struct extent_io_tree *tree,
1651 struct page *locked_page, u64 *start,
1652 u64 *end, u64 max_bytes)
Chris Masonc8b97812008-10-29 14:49:59 -04001653{
1654 u64 delalloc_start;
1655 u64 delalloc_end;
1656 u64 found;
Chris Mason9655d292009-09-02 15:22:30 -04001657 struct extent_state *cached_state = NULL;
Chris Masonc8b97812008-10-29 14:49:59 -04001658 int ret;
1659 int loops = 0;
1660
1661again:
1662 /* step one, find a bunch of delalloc bytes starting at start */
1663 delalloc_start = *start;
1664 delalloc_end = 0;
1665 found = find_delalloc_range(tree, &delalloc_start, &delalloc_end,
Josef Bacikc2a128d2010-02-02 21:19:11 +00001666 max_bytes, &cached_state);
Chris Mason70b99e62008-10-31 12:46:39 -04001667 if (!found || delalloc_end <= *start) {
Chris Masonc8b97812008-10-29 14:49:59 -04001668 *start = delalloc_start;
1669 *end = delalloc_end;
Josef Bacikc2a128d2010-02-02 21:19:11 +00001670 free_extent_state(cached_state);
Liu Bo385fe0b2013-10-01 23:49:49 +08001671 return 0;
Chris Masonc8b97812008-10-29 14:49:59 -04001672 }
1673
1674 /*
Chris Mason70b99e62008-10-31 12:46:39 -04001675 * start comes from the offset of locked_page. We have to lock
1676 * pages in order, so we can't process delalloc bytes before
1677 * locked_page
1678 */
Chris Masond3977122009-01-05 21:25:51 -05001679 if (delalloc_start < *start)
Chris Mason70b99e62008-10-31 12:46:39 -04001680 delalloc_start = *start;
Chris Mason70b99e62008-10-31 12:46:39 -04001681
1682 /*
Chris Masonc8b97812008-10-29 14:49:59 -04001683 * make sure to limit the number of pages we try to lock down
Chris Masonc8b97812008-10-29 14:49:59 -04001684 */
Josef Bacik7bf811a52013-10-07 22:11:09 -04001685 if (delalloc_end + 1 - delalloc_start > max_bytes)
1686 delalloc_end = delalloc_start + max_bytes - 1;
Chris Masond3977122009-01-05 21:25:51 -05001687
Chris Masonc8b97812008-10-29 14:49:59 -04001688 /* step two, lock all the pages after the page that has start */
1689 ret = lock_delalloc_pages(inode, locked_page,
1690 delalloc_start, delalloc_end);
1691 if (ret == -EAGAIN) {
1692 /* some of the pages are gone, lets avoid looping by
1693 * shortening the size of the delalloc range we're searching
1694 */
Chris Mason9655d292009-09-02 15:22:30 -04001695 free_extent_state(cached_state);
Chris Mason7d788742014-05-21 05:49:54 -07001696 cached_state = NULL;
Chris Masonc8b97812008-10-29 14:49:59 -04001697 if (!loops) {
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03001698 max_bytes = PAGE_SIZE;
Chris Masonc8b97812008-10-29 14:49:59 -04001699 loops = 1;
1700 goto again;
1701 } else {
1702 found = 0;
1703 goto out_failed;
1704 }
1705 }
Jeff Mahoney79787ea2012-03-12 16:03:00 +01001706 BUG_ON(ret); /* Only valid values are 0 and -EAGAIN */
Chris Masonc8b97812008-10-29 14:49:59 -04001707
1708 /* step three, lock the state bits for the whole range */
David Sterbaff13db42015-12-03 14:30:40 +01001709 lock_extent_bits(tree, delalloc_start, delalloc_end, &cached_state);
Chris Masonc8b97812008-10-29 14:49:59 -04001710
1711 /* then test to make sure it is all still delalloc */
1712 ret = test_range_bit(tree, delalloc_start, delalloc_end,
Chris Mason9655d292009-09-02 15:22:30 -04001713 EXTENT_DELALLOC, 1, cached_state);
Chris Masonc8b97812008-10-29 14:49:59 -04001714 if (!ret) {
Chris Mason9655d292009-09-02 15:22:30 -04001715 unlock_extent_cached(tree, delalloc_start, delalloc_end,
1716 &cached_state, GFP_NOFS);
Chris Masonc8b97812008-10-29 14:49:59 -04001717 __unlock_for_delalloc(inode, locked_page,
1718 delalloc_start, delalloc_end);
1719 cond_resched();
1720 goto again;
1721 }
Chris Mason9655d292009-09-02 15:22:30 -04001722 free_extent_state(cached_state);
Chris Masonc8b97812008-10-29 14:49:59 -04001723 *start = delalloc_start;
1724 *end = delalloc_end;
1725out_failed:
1726 return found;
1727}
1728
David Sterbaa9d93e12015-12-03 13:08:59 +01001729void extent_clear_unlock_delalloc(struct inode *inode, u64 start, u64 end,
Qu Wenruoba8b04c2016-07-19 16:50:36 +08001730 u64 delalloc_end, struct page *locked_page,
David Sterba9ee49a042015-01-14 19:52:13 +01001731 unsigned clear_bits,
Josef Bacikc2790a22013-07-29 11:20:47 -04001732 unsigned long page_ops)
Chris Masonc8b97812008-10-29 14:49:59 -04001733{
Josef Bacikc2790a22013-07-29 11:20:47 -04001734 struct extent_io_tree *tree = &BTRFS_I(inode)->io_tree;
Chris Masonc8b97812008-10-29 14:49:59 -04001735 int ret;
1736 struct page *pages[16];
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03001737 unsigned long index = start >> PAGE_SHIFT;
1738 unsigned long end_index = end >> PAGE_SHIFT;
Chris Masonc8b97812008-10-29 14:49:59 -04001739 unsigned long nr_pages = end_index - index + 1;
1740 int i;
Chris Mason771ed682008-11-06 22:02:51 -05001741
Chris Mason2c64c532009-09-02 15:04:12 -04001742 clear_extent_bit(tree, start, end, clear_bits, 1, 0, NULL, GFP_NOFS);
Josef Bacikc2790a22013-07-29 11:20:47 -04001743 if (page_ops == 0)
David Sterbaa9d93e12015-12-03 13:08:59 +01001744 return;
Chris Masonc8b97812008-10-29 14:49:59 -04001745
Filipe Manana704de492014-10-06 22:14:22 +01001746 if ((page_ops & PAGE_SET_ERROR) && nr_pages > 0)
1747 mapping_set_error(inode->i_mapping, -EIO);
1748
Chris Masond3977122009-01-05 21:25:51 -05001749 while (nr_pages > 0) {
Chris Masonc8b97812008-10-29 14:49:59 -04001750 ret = find_get_pages_contig(inode->i_mapping, index,
Chris Mason5b050f02008-11-11 09:34:41 -05001751 min_t(unsigned long,
1752 nr_pages, ARRAY_SIZE(pages)), pages);
Chris Masonc8b97812008-10-29 14:49:59 -04001753 for (i = 0; i < ret; i++) {
Chris Mason8b62b722009-09-02 16:53:46 -04001754
Josef Bacikc2790a22013-07-29 11:20:47 -04001755 if (page_ops & PAGE_SET_PRIVATE2)
Chris Mason8b62b722009-09-02 16:53:46 -04001756 SetPagePrivate2(pages[i]);
1757
Chris Masonc8b97812008-10-29 14:49:59 -04001758 if (pages[i] == locked_page) {
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03001759 put_page(pages[i]);
Chris Masonc8b97812008-10-29 14:49:59 -04001760 continue;
1761 }
Josef Bacikc2790a22013-07-29 11:20:47 -04001762 if (page_ops & PAGE_CLEAR_DIRTY)
Chris Masonc8b97812008-10-29 14:49:59 -04001763 clear_page_dirty_for_io(pages[i]);
Josef Bacikc2790a22013-07-29 11:20:47 -04001764 if (page_ops & PAGE_SET_WRITEBACK)
Chris Masonc8b97812008-10-29 14:49:59 -04001765 set_page_writeback(pages[i]);
Filipe Manana704de492014-10-06 22:14:22 +01001766 if (page_ops & PAGE_SET_ERROR)
1767 SetPageError(pages[i]);
Josef Bacikc2790a22013-07-29 11:20:47 -04001768 if (page_ops & PAGE_END_WRITEBACK)
Chris Masonc8b97812008-10-29 14:49:59 -04001769 end_page_writeback(pages[i]);
Josef Bacikc2790a22013-07-29 11:20:47 -04001770 if (page_ops & PAGE_UNLOCK)
Chris Mason771ed682008-11-06 22:02:51 -05001771 unlock_page(pages[i]);
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03001772 put_page(pages[i]);
Chris Masonc8b97812008-10-29 14:49:59 -04001773 }
1774 nr_pages -= ret;
1775 index += ret;
1776 cond_resched();
1777 }
Chris Masonc8b97812008-10-29 14:49:59 -04001778}
Chris Masonc8b97812008-10-29 14:49:59 -04001779
Chris Masond352ac62008-09-29 15:18:18 -04001780/*
1781 * count the number of bytes in the tree that have a given bit(s)
1782 * set. This can be fairly slow, except for EXTENT_DIRTY which is
1783 * cached. The total number found is returned.
1784 */
Chris Masond1310b22008-01-24 16:13:08 -05001785u64 count_range_bits(struct extent_io_tree *tree,
1786 u64 *start, u64 search_end, u64 max_bytes,
David Sterba9ee49a042015-01-14 19:52:13 +01001787 unsigned bits, int contig)
Chris Masond1310b22008-01-24 16:13:08 -05001788{
1789 struct rb_node *node;
1790 struct extent_state *state;
1791 u64 cur_start = *start;
1792 u64 total_bytes = 0;
Chris Masonec29ed52011-02-23 16:23:20 -05001793 u64 last = 0;
Chris Masond1310b22008-01-24 16:13:08 -05001794 int found = 0;
1795
Dulshani Gunawardhanafae7f212013-10-31 10:30:08 +05301796 if (WARN_ON(search_end <= cur_start))
Chris Masond1310b22008-01-24 16:13:08 -05001797 return 0;
Chris Masond1310b22008-01-24 16:13:08 -05001798
Chris Masoncad321a2008-12-17 14:51:42 -05001799 spin_lock(&tree->lock);
Chris Masond1310b22008-01-24 16:13:08 -05001800 if (cur_start == 0 && bits == EXTENT_DIRTY) {
1801 total_bytes = tree->dirty_bytes;
1802 goto out;
1803 }
1804 /*
1805 * this search will find all the extents that end after
1806 * our range starts.
1807 */
Chris Mason80ea96b2008-02-01 14:51:59 -05001808 node = tree_search(tree, cur_start);
Chris Masond3977122009-01-05 21:25:51 -05001809 if (!node)
Chris Masond1310b22008-01-24 16:13:08 -05001810 goto out;
Chris Masond1310b22008-01-24 16:13:08 -05001811
Chris Masond3977122009-01-05 21:25:51 -05001812 while (1) {
Chris Masond1310b22008-01-24 16:13:08 -05001813 state = rb_entry(node, struct extent_state, rb_node);
1814 if (state->start > search_end)
1815 break;
Chris Masonec29ed52011-02-23 16:23:20 -05001816 if (contig && found && state->start > last + 1)
1817 break;
1818 if (state->end >= cur_start && (state->state & bits) == bits) {
Chris Masond1310b22008-01-24 16:13:08 -05001819 total_bytes += min(search_end, state->end) + 1 -
1820 max(cur_start, state->start);
1821 if (total_bytes >= max_bytes)
1822 break;
1823 if (!found) {
Josef Bacikaf60bed2011-05-04 11:11:17 -04001824 *start = max(cur_start, state->start);
Chris Masond1310b22008-01-24 16:13:08 -05001825 found = 1;
1826 }
Chris Masonec29ed52011-02-23 16:23:20 -05001827 last = state->end;
1828 } else if (contig && found) {
1829 break;
Chris Masond1310b22008-01-24 16:13:08 -05001830 }
1831 node = rb_next(node);
1832 if (!node)
1833 break;
1834 }
1835out:
Chris Masoncad321a2008-12-17 14:51:42 -05001836 spin_unlock(&tree->lock);
Chris Masond1310b22008-01-24 16:13:08 -05001837 return total_bytes;
1838}
Christoph Hellwigb2950862008-12-02 09:54:17 -05001839
Chris Masond352ac62008-09-29 15:18:18 -04001840/*
1841 * set the private field for a given byte offset in the tree. If there isn't
1842 * an extent_state there already, this does nothing.
1843 */
Arnd Bergmannf827ba92016-02-22 22:53:20 +01001844static noinline int set_state_failrec(struct extent_io_tree *tree, u64 start,
David Sterba47dc1962016-02-11 13:24:13 +01001845 struct io_failure_record *failrec)
Chris Masond1310b22008-01-24 16:13:08 -05001846{
1847 struct rb_node *node;
1848 struct extent_state *state;
1849 int ret = 0;
1850
Chris Masoncad321a2008-12-17 14:51:42 -05001851 spin_lock(&tree->lock);
Chris Masond1310b22008-01-24 16:13:08 -05001852 /*
1853 * this search will find all the extents that end after
1854 * our range starts.
1855 */
Chris Mason80ea96b2008-02-01 14:51:59 -05001856 node = tree_search(tree, start);
Peter2b114d12008-04-01 11:21:40 -04001857 if (!node) {
Chris Masond1310b22008-01-24 16:13:08 -05001858 ret = -ENOENT;
1859 goto out;
1860 }
1861 state = rb_entry(node, struct extent_state, rb_node);
1862 if (state->start != start) {
1863 ret = -ENOENT;
1864 goto out;
1865 }
David Sterba47dc1962016-02-11 13:24:13 +01001866 state->failrec = failrec;
Chris Masond1310b22008-01-24 16:13:08 -05001867out:
Chris Masoncad321a2008-12-17 14:51:42 -05001868 spin_unlock(&tree->lock);
Chris Masond1310b22008-01-24 16:13:08 -05001869 return ret;
1870}
1871
Arnd Bergmannf827ba92016-02-22 22:53:20 +01001872static noinline int get_state_failrec(struct extent_io_tree *tree, u64 start,
David Sterba47dc1962016-02-11 13:24:13 +01001873 struct io_failure_record **failrec)
Chris Masond1310b22008-01-24 16:13:08 -05001874{
1875 struct rb_node *node;
1876 struct extent_state *state;
1877 int ret = 0;
1878
Chris Masoncad321a2008-12-17 14:51:42 -05001879 spin_lock(&tree->lock);
Chris Masond1310b22008-01-24 16:13:08 -05001880 /*
1881 * this search will find all the extents that end after
1882 * our range starts.
1883 */
Chris Mason80ea96b2008-02-01 14:51:59 -05001884 node = tree_search(tree, start);
Peter2b114d12008-04-01 11:21:40 -04001885 if (!node) {
Chris Masond1310b22008-01-24 16:13:08 -05001886 ret = -ENOENT;
1887 goto out;
1888 }
1889 state = rb_entry(node, struct extent_state, rb_node);
1890 if (state->start != start) {
1891 ret = -ENOENT;
1892 goto out;
1893 }
David Sterba47dc1962016-02-11 13:24:13 +01001894 *failrec = state->failrec;
Chris Masond1310b22008-01-24 16:13:08 -05001895out:
Chris Masoncad321a2008-12-17 14:51:42 -05001896 spin_unlock(&tree->lock);
Chris Masond1310b22008-01-24 16:13:08 -05001897 return ret;
1898}
1899
1900/*
1901 * searches a range in the state tree for a given mask.
Chris Mason70dec802008-01-29 09:59:12 -05001902 * If 'filled' == 1, this returns 1 only if every extent in the tree
Chris Masond1310b22008-01-24 16:13:08 -05001903 * has the bits set. Otherwise, 1 is returned if any bit in the
1904 * range is found set.
1905 */
1906int test_range_bit(struct extent_io_tree *tree, u64 start, u64 end,
David Sterba9ee49a042015-01-14 19:52:13 +01001907 unsigned bits, int filled, struct extent_state *cached)
Chris Masond1310b22008-01-24 16:13:08 -05001908{
1909 struct extent_state *state = NULL;
1910 struct rb_node *node;
1911 int bitset = 0;
Chris Masond1310b22008-01-24 16:13:08 -05001912
Chris Masoncad321a2008-12-17 14:51:42 -05001913 spin_lock(&tree->lock);
Filipe Manana27a35072014-07-06 20:09:59 +01001914 if (cached && extent_state_in_tree(cached) && cached->start <= start &&
Josef Bacikdf98b6e2011-06-20 14:53:48 -04001915 cached->end > start)
Chris Mason9655d292009-09-02 15:22:30 -04001916 node = &cached->rb_node;
1917 else
1918 node = tree_search(tree, start);
Chris Masond1310b22008-01-24 16:13:08 -05001919 while (node && start <= end) {
1920 state = rb_entry(node, struct extent_state, rb_node);
1921
1922 if (filled && state->start > start) {
1923 bitset = 0;
1924 break;
1925 }
1926
1927 if (state->start > end)
1928 break;
1929
1930 if (state->state & bits) {
1931 bitset = 1;
1932 if (!filled)
1933 break;
1934 } else if (filled) {
1935 bitset = 0;
1936 break;
1937 }
Chris Mason46562cec2009-09-23 20:23:16 -04001938
1939 if (state->end == (u64)-1)
1940 break;
1941
Chris Masond1310b22008-01-24 16:13:08 -05001942 start = state->end + 1;
1943 if (start > end)
1944 break;
1945 node = rb_next(node);
1946 if (!node) {
1947 if (filled)
1948 bitset = 0;
1949 break;
1950 }
1951 }
Chris Masoncad321a2008-12-17 14:51:42 -05001952 spin_unlock(&tree->lock);
Chris Masond1310b22008-01-24 16:13:08 -05001953 return bitset;
1954}
Chris Masond1310b22008-01-24 16:13:08 -05001955
1956/*
1957 * helper function to set a given page up to date if all the
1958 * extents in the tree for that page are up to date
1959 */
Jeff Mahoney143bede2012-03-01 14:56:26 +01001960static void check_page_uptodate(struct extent_io_tree *tree, struct page *page)
Chris Masond1310b22008-01-24 16:13:08 -05001961{
Miao Xie4eee4fa2012-12-21 09:17:45 +00001962 u64 start = page_offset(page);
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03001963 u64 end = start + PAGE_SIZE - 1;
Chris Mason9655d292009-09-02 15:22:30 -04001964 if (test_range_bit(tree, start, end, EXTENT_UPTODATE, 1, NULL))
Chris Masond1310b22008-01-24 16:13:08 -05001965 SetPageUptodate(page);
Chris Masond1310b22008-01-24 16:13:08 -05001966}
1967
Miao Xie8b110e32014-09-12 18:44:03 +08001968int free_io_failure(struct inode *inode, struct io_failure_record *rec)
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02001969{
1970 int ret;
1971 int err = 0;
1972 struct extent_io_tree *failure_tree = &BTRFS_I(inode)->io_failure_tree;
1973
David Sterba47dc1962016-02-11 13:24:13 +01001974 set_state_failrec(failure_tree, rec->start, NULL);
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02001975 ret = clear_extent_bits(failure_tree, rec->start,
1976 rec->start + rec->len - 1,
David Sterba91166212016-04-26 23:54:39 +02001977 EXTENT_LOCKED | EXTENT_DIRTY);
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02001978 if (ret)
1979 err = ret;
1980
David Woodhouse53b381b2013-01-29 18:40:14 -05001981 ret = clear_extent_bits(&BTRFS_I(inode)->io_tree, rec->start,
1982 rec->start + rec->len - 1,
David Sterba91166212016-04-26 23:54:39 +02001983 EXTENT_DAMAGED);
David Woodhouse53b381b2013-01-29 18:40:14 -05001984 if (ret && !err)
1985 err = ret;
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02001986
1987 kfree(rec);
1988 return err;
1989}
1990
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02001991/*
1992 * this bypasses the standard btrfs submit functions deliberately, as
1993 * the standard behavior is to write all copies in a raid setup. here we only
1994 * want to write the one bad copy. so we do the mapping for ourselves and issue
1995 * submit_bio directly.
Stefan Behrens3ec706c2012-11-05 15:46:42 +01001996 * to avoid any synchronization issues, wait for the data after writing, which
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02001997 * actually prevents the read that triggered the error from finishing.
1998 * currently, there can be no more than two copies of every data bit. thus,
1999 * exactly one rewrite is required.
2000 */
Miao Xie1203b682014-09-12 18:44:01 +08002001int repair_io_failure(struct inode *inode, u64 start, u64 length, u64 logical,
2002 struct page *page, unsigned int pg_offset, int mirror_num)
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002003{
Miao Xie1203b682014-09-12 18:44:01 +08002004 struct btrfs_fs_info *fs_info = BTRFS_I(inode)->root->fs_info;
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002005 struct bio *bio;
2006 struct btrfs_device *dev;
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002007 u64 map_length = 0;
2008 u64 sector;
2009 struct btrfs_bio *bbio = NULL;
David Woodhouse53b381b2013-01-29 18:40:14 -05002010 struct btrfs_mapping_tree *map_tree = &fs_info->mapping_tree;
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002011 int ret;
2012
Ilya Dryomov908960c2013-11-03 19:06:39 +02002013 ASSERT(!(fs_info->sb->s_flags & MS_RDONLY));
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002014 BUG_ON(!mirror_num);
2015
David Woodhouse53b381b2013-01-29 18:40:14 -05002016 /* we can't repair anything in raid56 yet */
2017 if (btrfs_is_parity_mirror(map_tree, logical, length, mirror_num))
2018 return 0;
2019
Chris Mason9be33952013-05-17 18:30:14 -04002020 bio = btrfs_io_bio_alloc(GFP_NOFS, 1);
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002021 if (!bio)
2022 return -EIO;
Kent Overstreet4f024f32013-10-11 15:44:27 -07002023 bio->bi_iter.bi_size = 0;
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002024 map_length = length;
2025
Filipe Mananab5de8d02016-05-27 22:21:27 +01002026 /*
2027 * Avoid races with device replace and make sure our bbio has devices
2028 * associated to its stripes that don't go away while we are doing the
2029 * read repair operation.
2030 */
2031 btrfs_bio_counter_inc_blocked(fs_info);
Stefan Behrens3ec706c2012-11-05 15:46:42 +01002032 ret = btrfs_map_block(fs_info, WRITE, logical,
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002033 &map_length, &bbio, mirror_num);
2034 if (ret) {
Filipe Mananab5de8d02016-05-27 22:21:27 +01002035 btrfs_bio_counter_dec(fs_info);
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002036 bio_put(bio);
2037 return -EIO;
2038 }
2039 BUG_ON(mirror_num != bbio->mirror_num);
2040 sector = bbio->stripes[mirror_num-1].physical >> 9;
Kent Overstreet4f024f32013-10-11 15:44:27 -07002041 bio->bi_iter.bi_sector = sector;
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002042 dev = bbio->stripes[mirror_num-1].dev;
Zhao Lei6e9606d2015-01-20 15:11:34 +08002043 btrfs_put_bbio(bbio);
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002044 if (!dev || !dev->bdev || !dev->writeable) {
Filipe Mananab5de8d02016-05-27 22:21:27 +01002045 btrfs_bio_counter_dec(fs_info);
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002046 bio_put(bio);
2047 return -EIO;
2048 }
2049 bio->bi_bdev = dev->bdev;
Shaun Tancheffb571bc62016-07-30 16:45:48 -05002050 bio_set_op_attrs(bio, REQ_OP_WRITE, WRITE_SYNC);
Miao Xieffdd2012014-09-12 18:44:00 +08002051 bio_add_page(bio, page, length, pg_offset);
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002052
Mike Christie4e49ea42016-06-05 14:31:41 -05002053 if (btrfsic_submit_bio_wait(bio)) {
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002054 /* try to remap that extent elsewhere? */
Filipe Mananab5de8d02016-05-27 22:21:27 +01002055 btrfs_bio_counter_dec(fs_info);
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002056 bio_put(bio);
Stefan Behrens442a4f62012-05-25 16:06:08 +02002057 btrfs_dev_stat_inc_and_print(dev, BTRFS_DEV_STAT_WRITE_ERRS);
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002058 return -EIO;
2059 }
2060
David Sterbab14af3b2015-10-08 10:43:10 +02002061 btrfs_info_rl_in_rcu(fs_info,
2062 "read error corrected: ino %llu off %llu (dev %s sector %llu)",
Miao Xie1203b682014-09-12 18:44:01 +08002063 btrfs_ino(inode), start,
2064 rcu_str_deref(dev->name), sector);
Filipe Mananab5de8d02016-05-27 22:21:27 +01002065 btrfs_bio_counter_dec(fs_info);
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002066 bio_put(bio);
2067 return 0;
2068}
2069
Josef Bacikea466792012-03-26 21:57:36 -04002070int repair_eb_io_failure(struct btrfs_root *root, struct extent_buffer *eb,
2071 int mirror_num)
2072{
Josef Bacikea466792012-03-26 21:57:36 -04002073 u64 start = eb->start;
2074 unsigned long i, num_pages = num_extent_pages(eb->start, eb->len);
Chris Masond95603b2012-04-12 15:55:15 -04002075 int ret = 0;
Josef Bacikea466792012-03-26 21:57:36 -04002076
Ilya Dryomov908960c2013-11-03 19:06:39 +02002077 if (root->fs_info->sb->s_flags & MS_RDONLY)
2078 return -EROFS;
2079
Josef Bacikea466792012-03-26 21:57:36 -04002080 for (i = 0; i < num_pages; i++) {
David Sterbafb85fc92014-07-31 01:03:53 +02002081 struct page *p = eb->pages[i];
Miao Xie1203b682014-09-12 18:44:01 +08002082
2083 ret = repair_io_failure(root->fs_info->btree_inode, start,
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03002084 PAGE_SIZE, start, p,
Miao Xie1203b682014-09-12 18:44:01 +08002085 start - page_offset(p), mirror_num);
Josef Bacikea466792012-03-26 21:57:36 -04002086 if (ret)
2087 break;
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03002088 start += PAGE_SIZE;
Josef Bacikea466792012-03-26 21:57:36 -04002089 }
2090
2091 return ret;
2092}
2093
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002094/*
2095 * each time an IO finishes, we do a fast check in the IO failure tree
2096 * to see if we need to process or clean up an io_failure_record
2097 */
Miao Xie8b110e32014-09-12 18:44:03 +08002098int clean_io_failure(struct inode *inode, u64 start, struct page *page,
2099 unsigned int pg_offset)
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002100{
2101 u64 private;
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002102 struct io_failure_record *failrec;
Ilya Dryomov908960c2013-11-03 19:06:39 +02002103 struct btrfs_fs_info *fs_info = BTRFS_I(inode)->root->fs_info;
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002104 struct extent_state *state;
2105 int num_copies;
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002106 int ret;
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002107
2108 private = 0;
2109 ret = count_range_bits(&BTRFS_I(inode)->io_failure_tree, &private,
2110 (u64)-1, 1, EXTENT_DIRTY, 0);
2111 if (!ret)
2112 return 0;
2113
David Sterba47dc1962016-02-11 13:24:13 +01002114 ret = get_state_failrec(&BTRFS_I(inode)->io_failure_tree, start,
2115 &failrec);
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002116 if (ret)
2117 return 0;
2118
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002119 BUG_ON(!failrec->this_mirror);
2120
2121 if (failrec->in_validation) {
2122 /* there was no real error, just free the record */
Jeff Mahoneyab8d0fc2016-09-20 10:05:02 -04002123 btrfs_debug(fs_info,
2124 "clean_io_failure: freeing dummy error at %llu",
2125 failrec->start);
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002126 goto out;
2127 }
Ilya Dryomov908960c2013-11-03 19:06:39 +02002128 if (fs_info->sb->s_flags & MS_RDONLY)
2129 goto out;
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002130
2131 spin_lock(&BTRFS_I(inode)->io_tree.lock);
2132 state = find_first_extent_bit_state(&BTRFS_I(inode)->io_tree,
2133 failrec->start,
2134 EXTENT_LOCKED);
2135 spin_unlock(&BTRFS_I(inode)->io_tree.lock);
2136
Miao Xie883d0de2013-07-25 19:22:35 +08002137 if (state && state->start <= failrec->start &&
2138 state->end >= failrec->start + failrec->len - 1) {
Stefan Behrens3ec706c2012-11-05 15:46:42 +01002139 num_copies = btrfs_num_copies(fs_info, failrec->logical,
2140 failrec->len);
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002141 if (num_copies > 1) {
Miao Xie1203b682014-09-12 18:44:01 +08002142 repair_io_failure(inode, start, failrec->len,
Miao Xie454ff3d2014-09-12 18:43:58 +08002143 failrec->logical, page,
Miao Xie1203b682014-09-12 18:44:01 +08002144 pg_offset, failrec->failed_mirror);
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002145 }
2146 }
2147
2148out:
Miao Xie454ff3d2014-09-12 18:43:58 +08002149 free_io_failure(inode, failrec);
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002150
Miao Xie454ff3d2014-09-12 18:43:58 +08002151 return 0;
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002152}
2153
Miao Xief6124962014-09-12 18:44:04 +08002154/*
2155 * Can be called when
2156 * - hold extent lock
2157 * - under ordered extent
2158 * - the inode is freeing
2159 */
2160void btrfs_free_io_failure_record(struct inode *inode, u64 start, u64 end)
2161{
2162 struct extent_io_tree *failure_tree = &BTRFS_I(inode)->io_failure_tree;
2163 struct io_failure_record *failrec;
2164 struct extent_state *state, *next;
2165
2166 if (RB_EMPTY_ROOT(&failure_tree->state))
2167 return;
2168
2169 spin_lock(&failure_tree->lock);
2170 state = find_first_extent_bit_state(failure_tree, start, EXTENT_DIRTY);
2171 while (state) {
2172 if (state->start > end)
2173 break;
2174
2175 ASSERT(state->end <= end);
2176
2177 next = next_state(state);
2178
David Sterba47dc1962016-02-11 13:24:13 +01002179 failrec = state->failrec;
Miao Xief6124962014-09-12 18:44:04 +08002180 free_extent_state(state);
2181 kfree(failrec);
2182
2183 state = next;
2184 }
2185 spin_unlock(&failure_tree->lock);
2186}
2187
Miao Xie2fe63032014-09-12 18:43:59 +08002188int btrfs_get_io_failure_record(struct inode *inode, u64 start, u64 end,
David Sterba47dc1962016-02-11 13:24:13 +01002189 struct io_failure_record **failrec_ret)
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002190{
Jeff Mahoneyab8d0fc2016-09-20 10:05:02 -04002191 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
Miao Xie2fe63032014-09-12 18:43:59 +08002192 struct io_failure_record *failrec;
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002193 struct extent_map *em;
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002194 struct extent_io_tree *failure_tree = &BTRFS_I(inode)->io_failure_tree;
2195 struct extent_io_tree *tree = &BTRFS_I(inode)->io_tree;
2196 struct extent_map_tree *em_tree = &BTRFS_I(inode)->extent_tree;
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002197 int ret;
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002198 u64 logical;
2199
David Sterba47dc1962016-02-11 13:24:13 +01002200 ret = get_state_failrec(failure_tree, start, &failrec);
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002201 if (ret) {
2202 failrec = kzalloc(sizeof(*failrec), GFP_NOFS);
2203 if (!failrec)
2204 return -ENOMEM;
Miao Xie2fe63032014-09-12 18:43:59 +08002205
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002206 failrec->start = start;
2207 failrec->len = end - start + 1;
2208 failrec->this_mirror = 0;
2209 failrec->bio_flags = 0;
2210 failrec->in_validation = 0;
2211
2212 read_lock(&em_tree->lock);
2213 em = lookup_extent_mapping(em_tree, start, failrec->len);
2214 if (!em) {
2215 read_unlock(&em_tree->lock);
2216 kfree(failrec);
2217 return -EIO;
2218 }
2219
Filipe David Borba Manana68ba9902013-11-25 03:22:07 +00002220 if (em->start > start || em->start + em->len <= start) {
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002221 free_extent_map(em);
2222 em = NULL;
2223 }
2224 read_unlock(&em_tree->lock);
Tsutomu Itoh7a2d6a62012-10-01 03:07:15 -06002225 if (!em) {
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002226 kfree(failrec);
2227 return -EIO;
2228 }
Miao Xie2fe63032014-09-12 18:43:59 +08002229
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002230 logical = start - em->start;
2231 logical = em->block_start + logical;
2232 if (test_bit(EXTENT_FLAG_COMPRESSED, &em->flags)) {
2233 logical = em->block_start;
2234 failrec->bio_flags = EXTENT_BIO_COMPRESSED;
2235 extent_set_compress_type(&failrec->bio_flags,
2236 em->compress_type);
2237 }
Miao Xie2fe63032014-09-12 18:43:59 +08002238
Jeff Mahoneyab8d0fc2016-09-20 10:05:02 -04002239 btrfs_debug(fs_info,
2240 "Get IO Failure Record: (new) logical=%llu, start=%llu, len=%llu",
2241 logical, start, failrec->len);
Miao Xie2fe63032014-09-12 18:43:59 +08002242
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002243 failrec->logical = logical;
2244 free_extent_map(em);
2245
2246 /* set the bits in the private failure tree */
2247 ret = set_extent_bits(failure_tree, start, end,
David Sterbaceeb0ae2016-04-26 23:54:39 +02002248 EXTENT_LOCKED | EXTENT_DIRTY);
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002249 if (ret >= 0)
David Sterba47dc1962016-02-11 13:24:13 +01002250 ret = set_state_failrec(failure_tree, start, failrec);
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002251 /* set the bits in the inode's tree */
2252 if (ret >= 0)
David Sterbaceeb0ae2016-04-26 23:54:39 +02002253 ret = set_extent_bits(tree, start, end, EXTENT_DAMAGED);
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002254 if (ret < 0) {
2255 kfree(failrec);
2256 return ret;
2257 }
2258 } else {
Jeff Mahoneyab8d0fc2016-09-20 10:05:02 -04002259 btrfs_debug(fs_info,
2260 "Get IO Failure Record: (found) logical=%llu, start=%llu, len=%llu, validation=%d",
2261 failrec->logical, failrec->start, failrec->len,
2262 failrec->in_validation);
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002263 /*
2264 * when data can be on disk more than twice, add to failrec here
2265 * (e.g. with a list for failed_mirror) to make
2266 * clean_io_failure() clean all those errors at once.
2267 */
2268 }
Miao Xie2fe63032014-09-12 18:43:59 +08002269
2270 *failrec_ret = failrec;
2271
2272 return 0;
2273}
2274
2275int btrfs_check_repairable(struct inode *inode, struct bio *failed_bio,
2276 struct io_failure_record *failrec, int failed_mirror)
2277{
Jeff Mahoneyab8d0fc2016-09-20 10:05:02 -04002278 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
Miao Xie2fe63032014-09-12 18:43:59 +08002279 int num_copies;
2280
Jeff Mahoneyab8d0fc2016-09-20 10:05:02 -04002281 num_copies = btrfs_num_copies(fs_info, failrec->logical, failrec->len);
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002282 if (num_copies == 1) {
2283 /*
2284 * we only have a single copy of the data, so don't bother with
2285 * all the retry and error correction code that follows. no
2286 * matter what the error is, it is very likely to persist.
2287 */
Jeff Mahoneyab8d0fc2016-09-20 10:05:02 -04002288 btrfs_debug(fs_info,
2289 "Check Repairable: cannot repair, num_copies=%d, next_mirror %d, failed_mirror %d",
2290 num_copies, failrec->this_mirror, failed_mirror);
Miao Xie2fe63032014-09-12 18:43:59 +08002291 return 0;
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002292 }
2293
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002294 /*
2295 * there are two premises:
2296 * a) deliver good data to the caller
2297 * b) correct the bad sectors on disk
2298 */
2299 if (failed_bio->bi_vcnt > 1) {
2300 /*
2301 * to fulfill b), we need to know the exact failing sectors, as
2302 * we don't want to rewrite any more than the failed ones. thus,
2303 * we need separate read requests for the failed bio
2304 *
2305 * if the following BUG_ON triggers, our validation request got
2306 * merged. we need separate requests for our algorithm to work.
2307 */
2308 BUG_ON(failrec->in_validation);
2309 failrec->in_validation = 1;
2310 failrec->this_mirror = failed_mirror;
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002311 } else {
2312 /*
2313 * we're ready to fulfill a) and b) alongside. get a good copy
2314 * of the failed sector and if we succeed, we have setup
2315 * everything for repair_io_failure to do the rest for us.
2316 */
2317 if (failrec->in_validation) {
2318 BUG_ON(failrec->this_mirror != failed_mirror);
2319 failrec->in_validation = 0;
2320 failrec->this_mirror = 0;
2321 }
2322 failrec->failed_mirror = failed_mirror;
2323 failrec->this_mirror++;
2324 if (failrec->this_mirror == failed_mirror)
2325 failrec->this_mirror++;
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002326 }
2327
Miao Xiefacc8a222013-07-25 19:22:34 +08002328 if (failrec->this_mirror > num_copies) {
Jeff Mahoneyab8d0fc2016-09-20 10:05:02 -04002329 btrfs_debug(fs_info,
2330 "Check Repairable: (fail) num_copies=%d, next_mirror %d, failed_mirror %d",
2331 num_copies, failrec->this_mirror, failed_mirror);
Miao Xie2fe63032014-09-12 18:43:59 +08002332 return 0;
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002333 }
2334
Miao Xie2fe63032014-09-12 18:43:59 +08002335 return 1;
2336}
2337
2338
2339struct bio *btrfs_create_repair_bio(struct inode *inode, struct bio *failed_bio,
2340 struct io_failure_record *failrec,
2341 struct page *page, int pg_offset, int icsum,
Miao Xie8b110e32014-09-12 18:44:03 +08002342 bio_end_io_t *endio_func, void *data)
Miao Xie2fe63032014-09-12 18:43:59 +08002343{
2344 struct bio *bio;
2345 struct btrfs_io_bio *btrfs_failed_bio;
2346 struct btrfs_io_bio *btrfs_bio;
2347
Chris Mason9be33952013-05-17 18:30:14 -04002348 bio = btrfs_io_bio_alloc(GFP_NOFS, 1);
Miao Xie2fe63032014-09-12 18:43:59 +08002349 if (!bio)
2350 return NULL;
2351
2352 bio->bi_end_io = endio_func;
Kent Overstreet4f024f32013-10-11 15:44:27 -07002353 bio->bi_iter.bi_sector = failrec->logical >> 9;
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002354 bio->bi_bdev = BTRFS_I(inode)->root->fs_info->fs_devices->latest_bdev;
Kent Overstreet4f024f32013-10-11 15:44:27 -07002355 bio->bi_iter.bi_size = 0;
Miao Xie8b110e32014-09-12 18:44:03 +08002356 bio->bi_private = data;
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002357
Miao Xiefacc8a222013-07-25 19:22:34 +08002358 btrfs_failed_bio = btrfs_io_bio(failed_bio);
2359 if (btrfs_failed_bio->csum) {
2360 struct btrfs_fs_info *fs_info = BTRFS_I(inode)->root->fs_info;
2361 u16 csum_size = btrfs_super_csum_size(fs_info->super_copy);
2362
2363 btrfs_bio = btrfs_io_bio(bio);
2364 btrfs_bio->csum = btrfs_bio->csum_inline;
Miao Xie2fe63032014-09-12 18:43:59 +08002365 icsum *= csum_size;
2366 memcpy(btrfs_bio->csum, btrfs_failed_bio->csum + icsum,
Miao Xiefacc8a222013-07-25 19:22:34 +08002367 csum_size);
2368 }
2369
Miao Xie2fe63032014-09-12 18:43:59 +08002370 bio_add_page(bio, page, failrec->len, pg_offset);
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002371
Miao Xie2fe63032014-09-12 18:43:59 +08002372 return bio;
2373}
2374
2375/*
2376 * this is a generic handler for readpage errors (default
2377 * readpage_io_failed_hook). if other copies exist, read those and write back
2378 * good data to the failed position. does not investigate in remapping the
2379 * failed extent elsewhere, hoping the device will be smart enough to do this as
2380 * needed
2381 */
2382
2383static int bio_readpage_error(struct bio *failed_bio, u64 phy_offset,
2384 struct page *page, u64 start, u64 end,
2385 int failed_mirror)
2386{
2387 struct io_failure_record *failrec;
2388 struct inode *inode = page->mapping->host;
2389 struct extent_io_tree *tree = &BTRFS_I(inode)->io_tree;
2390 struct bio *bio;
2391 int read_mode;
2392 int ret;
2393
Mike Christie1f7ad752016-06-05 14:31:51 -05002394 BUG_ON(bio_op(failed_bio) == REQ_OP_WRITE);
Miao Xie2fe63032014-09-12 18:43:59 +08002395
2396 ret = btrfs_get_io_failure_record(inode, start, end, &failrec);
2397 if (ret)
2398 return ret;
2399
2400 ret = btrfs_check_repairable(inode, failed_bio, failrec, failed_mirror);
2401 if (!ret) {
2402 free_io_failure(inode, failrec);
2403 return -EIO;
2404 }
2405
2406 if (failed_bio->bi_vcnt > 1)
2407 read_mode = READ_SYNC | REQ_FAILFAST_DEV;
2408 else
2409 read_mode = READ_SYNC;
2410
2411 phy_offset >>= inode->i_sb->s_blocksize_bits;
2412 bio = btrfs_create_repair_bio(inode, failed_bio, failrec, page,
2413 start - page_offset(page),
Miao Xie8b110e32014-09-12 18:44:03 +08002414 (int)phy_offset, failed_bio->bi_end_io,
2415 NULL);
Miao Xie2fe63032014-09-12 18:43:59 +08002416 if (!bio) {
2417 free_io_failure(inode, failrec);
2418 return -EIO;
2419 }
Mike Christie1f7ad752016-06-05 14:31:51 -05002420 bio_set_op_attrs(bio, REQ_OP_READ, read_mode);
Miao Xie2fe63032014-09-12 18:43:59 +08002421
Jeff Mahoneyab8d0fc2016-09-20 10:05:02 -04002422 btrfs_debug(btrfs_sb(inode->i_sb),
2423 "Repair Read Error: submitting new read[%#x] to this_mirror=%d, in_validation=%d",
2424 read_mode, failrec->this_mirror, failrec->in_validation);
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002425
Mike Christie81a75f62016-06-05 14:31:54 -05002426 ret = tree->ops->submit_bio_hook(inode, bio, failrec->this_mirror,
Tsutomu Itoh013bd4c2012-02-16 10:11:40 +09002427 failrec->bio_flags, 0);
Miao Xie6c387ab2014-09-12 18:43:57 +08002428 if (ret) {
Miao Xie454ff3d2014-09-12 18:43:58 +08002429 free_io_failure(inode, failrec);
Miao Xie6c387ab2014-09-12 18:43:57 +08002430 bio_put(bio);
2431 }
2432
Tsutomu Itoh013bd4c2012-02-16 10:11:40 +09002433 return ret;
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002434}
2435
Chris Masond1310b22008-01-24 16:13:08 -05002436/* lots and lots of room for performance fixes in the end_bio funcs */
2437
David Sterbab5227c02015-12-03 13:08:59 +01002438void end_extent_writepage(struct page *page, int err, u64 start, u64 end)
Jeff Mahoney87826df2012-02-15 16:23:57 +01002439{
2440 int uptodate = (err == 0);
2441 struct extent_io_tree *tree;
Eric Sandeen3e2426b2014-06-12 00:39:58 -05002442 int ret = 0;
Jeff Mahoney87826df2012-02-15 16:23:57 +01002443
2444 tree = &BTRFS_I(page->mapping->host)->io_tree;
2445
2446 if (tree->ops && tree->ops->writepage_end_io_hook) {
2447 ret = tree->ops->writepage_end_io_hook(page, start,
2448 end, NULL, uptodate);
2449 if (ret)
2450 uptodate = 0;
2451 }
2452
Jeff Mahoney87826df2012-02-15 16:23:57 +01002453 if (!uptodate) {
Jeff Mahoney87826df2012-02-15 16:23:57 +01002454 ClearPageUptodate(page);
2455 SetPageError(page);
Colin Ian Kinge09fd7c2017-05-09 18:14:01 +01002456 ret = err < 0 ? err : -EIO;
Liu Bo5dca6ee2014-05-12 12:47:36 +08002457 mapping_set_error(page->mapping, ret);
Jeff Mahoney87826df2012-02-15 16:23:57 +01002458 }
Jeff Mahoney87826df2012-02-15 16:23:57 +01002459}
2460
Chris Masond1310b22008-01-24 16:13:08 -05002461/*
2462 * after a writepage IO is done, we need to:
2463 * clear the uptodate bits on error
2464 * clear the writeback bits in the extent tree for this IO
2465 * end_page_writeback if the page has no more pending IO
2466 *
2467 * Scheduling is not allowed, so the extent state tree is expected
2468 * to have one and only one object corresponding to this IO.
2469 */
Christoph Hellwig4246a0b2015-07-20 15:29:37 +02002470static void end_bio_extent_writepage(struct bio *bio)
Chris Masond1310b22008-01-24 16:13:08 -05002471{
Kent Overstreet2c30c712013-11-07 12:20:26 -08002472 struct bio_vec *bvec;
Chris Masond1310b22008-01-24 16:13:08 -05002473 u64 start;
2474 u64 end;
Kent Overstreet2c30c712013-11-07 12:20:26 -08002475 int i;
Chris Masond1310b22008-01-24 16:13:08 -05002476
Kent Overstreet2c30c712013-11-07 12:20:26 -08002477 bio_for_each_segment_all(bvec, bio, i) {
Chris Masond1310b22008-01-24 16:13:08 -05002478 struct page *page = bvec->bv_page;
David Woodhouse902b22f2008-08-20 08:51:49 -04002479
Alexandre Oliva17a5adc2013-05-15 11:38:55 -04002480 /* We always issue full-page reads, but if some block
2481 * in a page fails to read, blk_update_request() will
2482 * advance bv_offset and adjust bv_len to compensate.
2483 * Print a warning for nonzero offsets, and an error
2484 * if they don't add up to a full page. */
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03002485 if (bvec->bv_offset || bvec->bv_len != PAGE_SIZE) {
2486 if (bvec->bv_offset + bvec->bv_len != PAGE_SIZE)
Frank Holtonefe120a2013-12-20 11:37:06 -05002487 btrfs_err(BTRFS_I(page->mapping->host)->root->fs_info,
2488 "partial page write in btrfs with offset %u and length %u",
2489 bvec->bv_offset, bvec->bv_len);
2490 else
2491 btrfs_info(BTRFS_I(page->mapping->host)->root->fs_info,
Jeff Mahoney5d163e02016-09-20 10:05:00 -04002492 "incomplete page write in btrfs with offset %u and length %u",
Frank Holtonefe120a2013-12-20 11:37:06 -05002493 bvec->bv_offset, bvec->bv_len);
2494 }
Chris Masond1310b22008-01-24 16:13:08 -05002495
Alexandre Oliva17a5adc2013-05-15 11:38:55 -04002496 start = page_offset(page);
2497 end = start + bvec->bv_offset + bvec->bv_len - 1;
Chris Masond1310b22008-01-24 16:13:08 -05002498
David Sterbab5227c02015-12-03 13:08:59 +01002499 end_extent_writepage(page, bio->bi_error, start, end);
Alexandre Oliva17a5adc2013-05-15 11:38:55 -04002500 end_page_writeback(page);
Kent Overstreet2c30c712013-11-07 12:20:26 -08002501 }
Chris Mason2b1f55b2008-09-24 11:48:04 -04002502
Chris Masond1310b22008-01-24 16:13:08 -05002503 bio_put(bio);
Chris Masond1310b22008-01-24 16:13:08 -05002504}
2505
Miao Xie883d0de2013-07-25 19:22:35 +08002506static void
2507endio_readpage_release_extent(struct extent_io_tree *tree, u64 start, u64 len,
2508 int uptodate)
2509{
2510 struct extent_state *cached = NULL;
2511 u64 end = start + len - 1;
2512
2513 if (uptodate && tree->track_uptodate)
2514 set_extent_uptodate(tree, start, end, &cached, GFP_ATOMIC);
2515 unlock_extent_cached(tree, start, end, &cached, GFP_ATOMIC);
2516}
2517
Chris Masond1310b22008-01-24 16:13:08 -05002518/*
2519 * after a readpage IO is done, we need to:
2520 * clear the uptodate bits on error
2521 * set the uptodate bits if things worked
2522 * set the page up to date if all extents in the tree are uptodate
2523 * clear the lock bit in the extent tree
2524 * unlock the page if there are no other extents locked for it
2525 *
2526 * Scheduling is not allowed, so the extent state tree is expected
2527 * to have one and only one object corresponding to this IO.
2528 */
Christoph Hellwig4246a0b2015-07-20 15:29:37 +02002529static void end_bio_extent_readpage(struct bio *bio)
Chris Masond1310b22008-01-24 16:13:08 -05002530{
Kent Overstreet2c30c712013-11-07 12:20:26 -08002531 struct bio_vec *bvec;
Christoph Hellwig4246a0b2015-07-20 15:29:37 +02002532 int uptodate = !bio->bi_error;
Miao Xiefacc8a222013-07-25 19:22:34 +08002533 struct btrfs_io_bio *io_bio = btrfs_io_bio(bio);
David Woodhouse902b22f2008-08-20 08:51:49 -04002534 struct extent_io_tree *tree;
Miao Xiefacc8a222013-07-25 19:22:34 +08002535 u64 offset = 0;
Chris Masond1310b22008-01-24 16:13:08 -05002536 u64 start;
2537 u64 end;
Miao Xiefacc8a222013-07-25 19:22:34 +08002538 u64 len;
Miao Xie883d0de2013-07-25 19:22:35 +08002539 u64 extent_start = 0;
2540 u64 extent_len = 0;
Josef Bacik5cf1ab52012-04-16 09:42:26 -04002541 int mirror;
Chris Masond1310b22008-01-24 16:13:08 -05002542 int ret;
Kent Overstreet2c30c712013-11-07 12:20:26 -08002543 int i;
Chris Masond1310b22008-01-24 16:13:08 -05002544
Kent Overstreet2c30c712013-11-07 12:20:26 -08002545 bio_for_each_segment_all(bvec, bio, i) {
Chris Masond1310b22008-01-24 16:13:08 -05002546 struct page *page = bvec->bv_page;
Josef Bacika71754f2013-06-17 17:14:39 -04002547 struct inode *inode = page->mapping->host;
Jeff Mahoneyab8d0fc2016-09-20 10:05:02 -04002548 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
Arne Jansen507903b2011-04-06 10:02:20 +00002549
Jeff Mahoneyab8d0fc2016-09-20 10:05:02 -04002550 btrfs_debug(fs_info,
2551 "end_bio_extent_readpage: bi_sector=%llu, err=%d, mirror=%u",
2552 (u64)bio->bi_iter.bi_sector, bio->bi_error,
2553 io_bio->mirror_num);
Josef Bacika71754f2013-06-17 17:14:39 -04002554 tree = &BTRFS_I(inode)->io_tree;
David Woodhouse902b22f2008-08-20 08:51:49 -04002555
Alexandre Oliva17a5adc2013-05-15 11:38:55 -04002556 /* We always issue full-page reads, but if some block
2557 * in a page fails to read, blk_update_request() will
2558 * advance bv_offset and adjust bv_len to compensate.
2559 * Print a warning for nonzero offsets, and an error
2560 * if they don't add up to a full page. */
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03002561 if (bvec->bv_offset || bvec->bv_len != PAGE_SIZE) {
2562 if (bvec->bv_offset + bvec->bv_len != PAGE_SIZE)
Jeff Mahoneyab8d0fc2016-09-20 10:05:02 -04002563 btrfs_err(fs_info,
2564 "partial page read in btrfs with offset %u and length %u",
Frank Holtonefe120a2013-12-20 11:37:06 -05002565 bvec->bv_offset, bvec->bv_len);
2566 else
Jeff Mahoneyab8d0fc2016-09-20 10:05:02 -04002567 btrfs_info(fs_info,
2568 "incomplete page read in btrfs with offset %u and length %u",
Frank Holtonefe120a2013-12-20 11:37:06 -05002569 bvec->bv_offset, bvec->bv_len);
2570 }
Chris Masond1310b22008-01-24 16:13:08 -05002571
Alexandre Oliva17a5adc2013-05-15 11:38:55 -04002572 start = page_offset(page);
2573 end = start + bvec->bv_offset + bvec->bv_len - 1;
Miao Xiefacc8a222013-07-25 19:22:34 +08002574 len = bvec->bv_len;
Chris Masond1310b22008-01-24 16:13:08 -05002575
Chris Mason9be33952013-05-17 18:30:14 -04002576 mirror = io_bio->mirror_num;
Miao Xief2a09da2013-07-25 19:22:33 +08002577 if (likely(uptodate && tree->ops &&
2578 tree->ops->readpage_end_io_hook)) {
Miao Xiefacc8a222013-07-25 19:22:34 +08002579 ret = tree->ops->readpage_end_io_hook(io_bio, offset,
2580 page, start, end,
2581 mirror);
Stefan Behrens5ee08442012-08-27 08:30:03 -06002582 if (ret)
Chris Masond1310b22008-01-24 16:13:08 -05002583 uptodate = 0;
Stefan Behrens5ee08442012-08-27 08:30:03 -06002584 else
Miao Xie1203b682014-09-12 18:44:01 +08002585 clean_io_failure(inode, start, page, 0);
Chris Masond1310b22008-01-24 16:13:08 -05002586 }
Josef Bacikea466792012-03-26 21:57:36 -04002587
Miao Xief2a09da2013-07-25 19:22:33 +08002588 if (likely(uptodate))
2589 goto readpage_ok;
2590
2591 if (tree->ops && tree->ops->readpage_io_failed_hook) {
Josef Bacik5cf1ab52012-04-16 09:42:26 -04002592 ret = tree->ops->readpage_io_failed_hook(page, mirror);
Christoph Hellwig4246a0b2015-07-20 15:29:37 +02002593 if (!ret && !bio->bi_error)
Josef Bacikea466792012-03-26 21:57:36 -04002594 uptodate = 1;
Miao Xief2a09da2013-07-25 19:22:33 +08002595 } else {
Jan Schmidtf4a8e652011-12-01 09:30:36 -05002596 /*
2597 * The generic bio_readpage_error handles errors the
2598 * following way: If possible, new read requests are
2599 * created and submitted and will end up in
2600 * end_bio_extent_readpage as well (if we're lucky, not
2601 * in the !uptodate case). In that case it returns 0 and
2602 * we just go on with the next page in our bio. If it
2603 * can't handle the error it will return -EIO and we
2604 * remain responsible for that page.
2605 */
Miao Xiefacc8a222013-07-25 19:22:34 +08002606 ret = bio_readpage_error(bio, offset, page, start, end,
2607 mirror);
Chris Mason7e383262008-04-09 16:28:12 -04002608 if (ret == 0) {
Christoph Hellwig4246a0b2015-07-20 15:29:37 +02002609 uptodate = !bio->bi_error;
Liu Bo38c1c2e2014-08-19 23:33:13 +08002610 offset += len;
Chris Mason7e383262008-04-09 16:28:12 -04002611 continue;
2612 }
2613 }
Miao Xief2a09da2013-07-25 19:22:33 +08002614readpage_ok:
Miao Xie883d0de2013-07-25 19:22:35 +08002615 if (likely(uptodate)) {
Josef Bacika71754f2013-06-17 17:14:39 -04002616 loff_t i_size = i_size_read(inode);
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03002617 pgoff_t end_index = i_size >> PAGE_SHIFT;
Liu Boa583c022014-08-19 23:32:22 +08002618 unsigned off;
Josef Bacika71754f2013-06-17 17:14:39 -04002619
2620 /* Zero out the end if this page straddles i_size */
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03002621 off = i_size & (PAGE_SIZE-1);
Liu Boa583c022014-08-19 23:32:22 +08002622 if (page->index == end_index && off)
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03002623 zero_user_segment(page, off, PAGE_SIZE);
Alexandre Oliva17a5adc2013-05-15 11:38:55 -04002624 SetPageUptodate(page);
Chris Mason70dec802008-01-29 09:59:12 -05002625 } else {
Alexandre Oliva17a5adc2013-05-15 11:38:55 -04002626 ClearPageUptodate(page);
2627 SetPageError(page);
Chris Mason70dec802008-01-29 09:59:12 -05002628 }
Alexandre Oliva17a5adc2013-05-15 11:38:55 -04002629 unlock_page(page);
Miao Xiefacc8a222013-07-25 19:22:34 +08002630 offset += len;
Miao Xie883d0de2013-07-25 19:22:35 +08002631
2632 if (unlikely(!uptodate)) {
2633 if (extent_len) {
2634 endio_readpage_release_extent(tree,
2635 extent_start,
2636 extent_len, 1);
2637 extent_start = 0;
2638 extent_len = 0;
2639 }
2640 endio_readpage_release_extent(tree, start,
2641 end - start + 1, 0);
2642 } else if (!extent_len) {
2643 extent_start = start;
2644 extent_len = end + 1 - start;
2645 } else if (extent_start + extent_len == start) {
2646 extent_len += end + 1 - start;
2647 } else {
2648 endio_readpage_release_extent(tree, extent_start,
2649 extent_len, uptodate);
2650 extent_start = start;
2651 extent_len = end + 1 - start;
2652 }
Kent Overstreet2c30c712013-11-07 12:20:26 -08002653 }
Chris Masond1310b22008-01-24 16:13:08 -05002654
Miao Xie883d0de2013-07-25 19:22:35 +08002655 if (extent_len)
2656 endio_readpage_release_extent(tree, extent_start, extent_len,
2657 uptodate);
Miao Xiefacc8a222013-07-25 19:22:34 +08002658 if (io_bio->end_io)
Christoph Hellwig4246a0b2015-07-20 15:29:37 +02002659 io_bio->end_io(io_bio, bio->bi_error);
Chris Masond1310b22008-01-24 16:13:08 -05002660 bio_put(bio);
Chris Masond1310b22008-01-24 16:13:08 -05002661}
2662
Chris Mason9be33952013-05-17 18:30:14 -04002663/*
2664 * this allocates from the btrfs_bioset. We're returning a bio right now
2665 * but you can call btrfs_io_bio for the appropriate container_of magic
2666 */
Miao Xie88f794e2010-11-22 03:02:55 +00002667struct bio *
2668btrfs_bio_alloc(struct block_device *bdev, u64 first_sector, int nr_vecs,
2669 gfp_t gfp_flags)
Chris Masond1310b22008-01-24 16:13:08 -05002670{
Miao Xiefacc8a222013-07-25 19:22:34 +08002671 struct btrfs_io_bio *btrfs_bio;
Chris Masond1310b22008-01-24 16:13:08 -05002672 struct bio *bio;
2673
Chris Mason9be33952013-05-17 18:30:14 -04002674 bio = bio_alloc_bioset(gfp_flags, nr_vecs, btrfs_bioset);
Chris Masond1310b22008-01-24 16:13:08 -05002675
2676 if (bio == NULL && (current->flags & PF_MEMALLOC)) {
Chris Mason9be33952013-05-17 18:30:14 -04002677 while (!bio && (nr_vecs /= 2)) {
2678 bio = bio_alloc_bioset(gfp_flags,
2679 nr_vecs, btrfs_bioset);
2680 }
Chris Masond1310b22008-01-24 16:13:08 -05002681 }
2682
2683 if (bio) {
2684 bio->bi_bdev = bdev;
Kent Overstreet4f024f32013-10-11 15:44:27 -07002685 bio->bi_iter.bi_sector = first_sector;
Miao Xiefacc8a222013-07-25 19:22:34 +08002686 btrfs_bio = btrfs_io_bio(bio);
2687 btrfs_bio->csum = NULL;
2688 btrfs_bio->csum_allocated = NULL;
2689 btrfs_bio->end_io = NULL;
Chris Masond1310b22008-01-24 16:13:08 -05002690 }
2691 return bio;
2692}
2693
Chris Mason9be33952013-05-17 18:30:14 -04002694struct bio *btrfs_bio_clone(struct bio *bio, gfp_t gfp_mask)
2695{
Miao Xie23ea8e52014-09-12 18:43:54 +08002696 struct btrfs_io_bio *btrfs_bio;
2697 struct bio *new;
Chris Mason9be33952013-05-17 18:30:14 -04002698
Miao Xie23ea8e52014-09-12 18:43:54 +08002699 new = bio_clone_bioset(bio, gfp_mask, btrfs_bioset);
2700 if (new) {
2701 btrfs_bio = btrfs_io_bio(new);
2702 btrfs_bio->csum = NULL;
2703 btrfs_bio->csum_allocated = NULL;
2704 btrfs_bio->end_io = NULL;
2705 }
2706 return new;
2707}
Chris Mason9be33952013-05-17 18:30:14 -04002708
2709/* this also allocates from the btrfs_bioset */
2710struct bio *btrfs_io_bio_alloc(gfp_t gfp_mask, unsigned int nr_iovecs)
2711{
Miao Xiefacc8a222013-07-25 19:22:34 +08002712 struct btrfs_io_bio *btrfs_bio;
2713 struct bio *bio;
2714
2715 bio = bio_alloc_bioset(gfp_mask, nr_iovecs, btrfs_bioset);
2716 if (bio) {
2717 btrfs_bio = btrfs_io_bio(bio);
2718 btrfs_bio->csum = NULL;
2719 btrfs_bio->csum_allocated = NULL;
2720 btrfs_bio->end_io = NULL;
2721 }
2722 return bio;
Chris Mason9be33952013-05-17 18:30:14 -04002723}
2724
2725
Mike Christie1f7ad752016-06-05 14:31:51 -05002726static int __must_check submit_one_bio(struct bio *bio, int mirror_num,
2727 unsigned long bio_flags)
Chris Masond1310b22008-01-24 16:13:08 -05002728{
Chris Masond1310b22008-01-24 16:13:08 -05002729 int ret = 0;
Chris Mason70dec802008-01-29 09:59:12 -05002730 struct bio_vec *bvec = bio->bi_io_vec + bio->bi_vcnt - 1;
2731 struct page *page = bvec->bv_page;
2732 struct extent_io_tree *tree = bio->bi_private;
Chris Mason70dec802008-01-29 09:59:12 -05002733 u64 start;
Chris Mason70dec802008-01-29 09:59:12 -05002734
Miao Xie4eee4fa2012-12-21 09:17:45 +00002735 start = page_offset(page) + bvec->bv_offset;
Chris Mason70dec802008-01-29 09:59:12 -05002736
David Woodhouse902b22f2008-08-20 08:51:49 -04002737 bio->bi_private = NULL;
Chris Masond1310b22008-01-24 16:13:08 -05002738 bio_get(bio);
2739
Chris Mason065631f2008-02-20 12:07:25 -05002740 if (tree->ops && tree->ops->submit_bio_hook)
Mike Christie81a75f62016-06-05 14:31:54 -05002741 ret = tree->ops->submit_bio_hook(page->mapping->host, bio,
Chris Masoneaf25d92010-05-25 09:48:28 -04002742 mirror_num, bio_flags, start);
Chris Mason0b86a832008-03-24 15:01:56 -04002743 else
Mike Christie4e49ea42016-06-05 14:31:41 -05002744 btrfsic_submit_bio(bio);
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002745
Chris Masond1310b22008-01-24 16:13:08 -05002746 bio_put(bio);
2747 return ret;
2748}
2749
Mike Christie1f7ad752016-06-05 14:31:51 -05002750static int merge_bio(struct extent_io_tree *tree, struct page *page,
Jeff Mahoney3444a972011-10-03 23:23:13 -04002751 unsigned long offset, size_t size, struct bio *bio,
2752 unsigned long bio_flags)
2753{
2754 int ret = 0;
2755 if (tree->ops && tree->ops->merge_bio_hook)
Mike Christie81a75f62016-06-05 14:31:54 -05002756 ret = tree->ops->merge_bio_hook(page, offset, size, bio,
Jeff Mahoney3444a972011-10-03 23:23:13 -04002757 bio_flags);
Jeff Mahoney3444a972011-10-03 23:23:13 -04002758 return ret;
2759
2760}
2761
Mike Christie1f7ad752016-06-05 14:31:51 -05002762static int submit_extent_page(int op, int op_flags, struct extent_io_tree *tree,
Chris Masonda2f0f72015-07-02 13:57:22 -07002763 struct writeback_control *wbc,
Chris Masond1310b22008-01-24 16:13:08 -05002764 struct page *page, sector_t sector,
2765 size_t size, unsigned long offset,
2766 struct block_device *bdev,
2767 struct bio **bio_ret,
2768 unsigned long max_pages,
Chris Masonf1885912008-04-09 16:28:12 -04002769 bio_end_io_t end_io_func,
Chris Masonc8b97812008-10-29 14:49:59 -04002770 int mirror_num,
2771 unsigned long prev_bio_flags,
Filipe Manana005efed2015-09-14 09:09:31 +01002772 unsigned long bio_flags,
2773 bool force_bio_submit)
Chris Masond1310b22008-01-24 16:13:08 -05002774{
2775 int ret = 0;
2776 struct bio *bio;
Chris Masonc8b97812008-10-29 14:49:59 -04002777 int contig = 0;
Chris Masonc8b97812008-10-29 14:49:59 -04002778 int old_compressed = prev_bio_flags & EXTENT_BIO_COMPRESSED;
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03002779 size_t page_size = min_t(size_t, size, PAGE_SIZE);
Chris Masond1310b22008-01-24 16:13:08 -05002780
2781 if (bio_ret && *bio_ret) {
2782 bio = *bio_ret;
Chris Masonc8b97812008-10-29 14:49:59 -04002783 if (old_compressed)
Kent Overstreet4f024f32013-10-11 15:44:27 -07002784 contig = bio->bi_iter.bi_sector == sector;
Chris Masonc8b97812008-10-29 14:49:59 -04002785 else
Kent Overstreetf73a1c72012-09-25 15:05:12 -07002786 contig = bio_end_sector(bio) == sector;
Chris Masonc8b97812008-10-29 14:49:59 -04002787
2788 if (prev_bio_flags != bio_flags || !contig ||
Filipe Manana005efed2015-09-14 09:09:31 +01002789 force_bio_submit ||
Mike Christie1f7ad752016-06-05 14:31:51 -05002790 merge_bio(tree, page, offset, page_size, bio, bio_flags) ||
Chris Masonc8b97812008-10-29 14:49:59 -04002791 bio_add_page(bio, page, page_size, offset) < page_size) {
Mike Christie1f7ad752016-06-05 14:31:51 -05002792 ret = submit_one_bio(bio, mirror_num, prev_bio_flags);
Naohiro Aota289454a2015-01-06 01:01:03 +09002793 if (ret < 0) {
2794 *bio_ret = NULL;
Jeff Mahoney79787ea2012-03-12 16:03:00 +01002795 return ret;
Naohiro Aota289454a2015-01-06 01:01:03 +09002796 }
Chris Masond1310b22008-01-24 16:13:08 -05002797 bio = NULL;
2798 } else {
Chris Masonda2f0f72015-07-02 13:57:22 -07002799 if (wbc)
2800 wbc_account_io(wbc, page, page_size);
Chris Masond1310b22008-01-24 16:13:08 -05002801 return 0;
2802 }
2803 }
Chris Masonc8b97812008-10-29 14:49:59 -04002804
Kent Overstreetb54ffb72015-05-19 14:31:01 +02002805 bio = btrfs_bio_alloc(bdev, sector, BIO_MAX_PAGES,
2806 GFP_NOFS | __GFP_HIGH);
Tsutomu Itoh5df67082011-02-01 09:17:35 +00002807 if (!bio)
2808 return -ENOMEM;
Chris Mason70dec802008-01-29 09:59:12 -05002809
Chris Masonc8b97812008-10-29 14:49:59 -04002810 bio_add_page(bio, page, page_size, offset);
Chris Masond1310b22008-01-24 16:13:08 -05002811 bio->bi_end_io = end_io_func;
2812 bio->bi_private = tree;
Mike Christie1f7ad752016-06-05 14:31:51 -05002813 bio_set_op_attrs(bio, op, op_flags);
Chris Masonda2f0f72015-07-02 13:57:22 -07002814 if (wbc) {
2815 wbc_init_bio(wbc, bio);
2816 wbc_account_io(wbc, page, page_size);
2817 }
Chris Mason70dec802008-01-29 09:59:12 -05002818
Chris Masond3977122009-01-05 21:25:51 -05002819 if (bio_ret)
Chris Masond1310b22008-01-24 16:13:08 -05002820 *bio_ret = bio;
Chris Masond3977122009-01-05 21:25:51 -05002821 else
Mike Christie1f7ad752016-06-05 14:31:51 -05002822 ret = submit_one_bio(bio, mirror_num, bio_flags);
Chris Masond1310b22008-01-24 16:13:08 -05002823
2824 return ret;
2825}
2826
Eric Sandeen48a3b632013-04-25 20:41:01 +00002827static void attach_extent_buffer_page(struct extent_buffer *eb,
2828 struct page *page)
Josef Bacik4f2de97a2012-03-07 16:20:05 -05002829{
2830 if (!PagePrivate(page)) {
2831 SetPagePrivate(page);
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03002832 get_page(page);
Josef Bacik4f2de97a2012-03-07 16:20:05 -05002833 set_page_private(page, (unsigned long)eb);
2834 } else {
2835 WARN_ON(page->private != (unsigned long)eb);
2836 }
2837}
2838
Chris Masond1310b22008-01-24 16:13:08 -05002839void set_page_extent_mapped(struct page *page)
2840{
2841 if (!PagePrivate(page)) {
2842 SetPagePrivate(page);
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03002843 get_page(page);
Chris Mason6af118ce2008-07-22 11:18:07 -04002844 set_page_private(page, EXTENT_PAGE_PRIVATE);
Chris Masond1310b22008-01-24 16:13:08 -05002845 }
2846}
2847
Miao Xie125bac012013-07-25 19:22:37 +08002848static struct extent_map *
2849__get_extent_map(struct inode *inode, struct page *page, size_t pg_offset,
2850 u64 start, u64 len, get_extent_t *get_extent,
2851 struct extent_map **em_cached)
2852{
2853 struct extent_map *em;
2854
2855 if (em_cached && *em_cached) {
2856 em = *em_cached;
Filipe Mananacbc0e922014-02-25 14:15:12 +00002857 if (extent_map_in_tree(em) && start >= em->start &&
Miao Xie125bac012013-07-25 19:22:37 +08002858 start < extent_map_end(em)) {
2859 atomic_inc(&em->refs);
2860 return em;
2861 }
2862
2863 free_extent_map(em);
2864 *em_cached = NULL;
2865 }
2866
2867 em = get_extent(inode, page, pg_offset, start, len, 0);
2868 if (em_cached && !IS_ERR_OR_NULL(em)) {
2869 BUG_ON(*em_cached);
2870 atomic_inc(&em->refs);
2871 *em_cached = em;
2872 }
2873 return em;
2874}
Chris Masond1310b22008-01-24 16:13:08 -05002875/*
2876 * basic readpage implementation. Locked extent state structs are inserted
2877 * into the tree that are removed when the IO is done (by the end_io
2878 * handlers)
Jeff Mahoney79787ea2012-03-12 16:03:00 +01002879 * XXX JDM: This needs looking at to ensure proper page locking
Liu Bobaf863b2016-07-11 10:39:07 -07002880 * return 0 on success, otherwise return error
Chris Masond1310b22008-01-24 16:13:08 -05002881 */
Miao Xie99740902013-07-25 19:22:36 +08002882static int __do_readpage(struct extent_io_tree *tree,
2883 struct page *page,
2884 get_extent_t *get_extent,
Miao Xie125bac012013-07-25 19:22:37 +08002885 struct extent_map **em_cached,
Miao Xie99740902013-07-25 19:22:36 +08002886 struct bio **bio, int mirror_num,
Mike Christie1f7ad752016-06-05 14:31:51 -05002887 unsigned long *bio_flags, int read_flags,
Filipe Manana005efed2015-09-14 09:09:31 +01002888 u64 *prev_em_start)
Chris Masond1310b22008-01-24 16:13:08 -05002889{
2890 struct inode *inode = page->mapping->host;
Miao Xie4eee4fa2012-12-21 09:17:45 +00002891 u64 start = page_offset(page);
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03002892 u64 page_end = start + PAGE_SIZE - 1;
Chris Masond1310b22008-01-24 16:13:08 -05002893 u64 end;
2894 u64 cur = start;
2895 u64 extent_offset;
2896 u64 last_byte = i_size_read(inode);
2897 u64 block_start;
2898 u64 cur_end;
2899 sector_t sector;
2900 struct extent_map *em;
2901 struct block_device *bdev;
Liu Bobaf863b2016-07-11 10:39:07 -07002902 int ret = 0;
Chris Masond1310b22008-01-24 16:13:08 -05002903 int nr = 0;
David Sterba306e16c2011-04-19 14:29:38 +02002904 size_t pg_offset = 0;
Chris Masond1310b22008-01-24 16:13:08 -05002905 size_t iosize;
Chris Masonc8b97812008-10-29 14:49:59 -04002906 size_t disk_io_size;
Chris Masond1310b22008-01-24 16:13:08 -05002907 size_t blocksize = inode->i_sb->s_blocksize;
Filipe Manana7f042a82016-01-27 19:17:20 +00002908 unsigned long this_bio_flag = 0;
Chris Masond1310b22008-01-24 16:13:08 -05002909
2910 set_page_extent_mapped(page);
2911
Miao Xie99740902013-07-25 19:22:36 +08002912 end = page_end;
Dan Magenheimer90a887c2011-05-26 10:01:56 -06002913 if (!PageUptodate(page)) {
2914 if (cleancache_get_page(page) == 0) {
2915 BUG_ON(blocksize != PAGE_SIZE);
Miao Xie99740902013-07-25 19:22:36 +08002916 unlock_extent(tree, start, end);
Dan Magenheimer90a887c2011-05-26 10:01:56 -06002917 goto out;
2918 }
2919 }
2920
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03002921 if (page->index == last_byte >> PAGE_SHIFT) {
Chris Masonc8b97812008-10-29 14:49:59 -04002922 char *userpage;
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03002923 size_t zero_offset = last_byte & (PAGE_SIZE - 1);
Chris Masonc8b97812008-10-29 14:49:59 -04002924
2925 if (zero_offset) {
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03002926 iosize = PAGE_SIZE - zero_offset;
Cong Wang7ac687d2011-11-25 23:14:28 +08002927 userpage = kmap_atomic(page);
Chris Masonc8b97812008-10-29 14:49:59 -04002928 memset(userpage + zero_offset, 0, iosize);
2929 flush_dcache_page(page);
Cong Wang7ac687d2011-11-25 23:14:28 +08002930 kunmap_atomic(userpage);
Chris Masonc8b97812008-10-29 14:49:59 -04002931 }
2932 }
Chris Masond1310b22008-01-24 16:13:08 -05002933 while (cur <= end) {
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03002934 unsigned long pnr = (last_byte >> PAGE_SHIFT) + 1;
Filipe Manana005efed2015-09-14 09:09:31 +01002935 bool force_bio_submit = false;
Josef Bacikc8f2f242013-02-11 11:33:00 -05002936
Chris Masond1310b22008-01-24 16:13:08 -05002937 if (cur >= last_byte) {
2938 char *userpage;
Arne Jansen507903b2011-04-06 10:02:20 +00002939 struct extent_state *cached = NULL;
2940
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03002941 iosize = PAGE_SIZE - pg_offset;
Cong Wang7ac687d2011-11-25 23:14:28 +08002942 userpage = kmap_atomic(page);
David Sterba306e16c2011-04-19 14:29:38 +02002943 memset(userpage + pg_offset, 0, iosize);
Chris Masond1310b22008-01-24 16:13:08 -05002944 flush_dcache_page(page);
Cong Wang7ac687d2011-11-25 23:14:28 +08002945 kunmap_atomic(userpage);
Chris Masond1310b22008-01-24 16:13:08 -05002946 set_extent_uptodate(tree, cur, cur + iosize - 1,
Arne Jansen507903b2011-04-06 10:02:20 +00002947 &cached, GFP_NOFS);
Filipe Manana7f042a82016-01-27 19:17:20 +00002948 unlock_extent_cached(tree, cur,
2949 cur + iosize - 1,
2950 &cached, GFP_NOFS);
Chris Masond1310b22008-01-24 16:13:08 -05002951 break;
2952 }
Miao Xie125bac012013-07-25 19:22:37 +08002953 em = __get_extent_map(inode, page, pg_offset, cur,
2954 end - cur + 1, get_extent, em_cached);
David Sterbac7040052011-04-19 18:00:01 +02002955 if (IS_ERR_OR_NULL(em)) {
Chris Masond1310b22008-01-24 16:13:08 -05002956 SetPageError(page);
Filipe Manana7f042a82016-01-27 19:17:20 +00002957 unlock_extent(tree, cur, end);
Chris Masond1310b22008-01-24 16:13:08 -05002958 break;
2959 }
Chris Masond1310b22008-01-24 16:13:08 -05002960 extent_offset = cur - em->start;
2961 BUG_ON(extent_map_end(em) <= cur);
2962 BUG_ON(end < cur);
2963
Li Zefan261507a02010-12-17 14:21:50 +08002964 if (test_bit(EXTENT_FLAG_COMPRESSED, &em->flags)) {
Mark Fasheh4b384312013-08-06 11:42:50 -07002965 this_bio_flag |= EXTENT_BIO_COMPRESSED;
Li Zefan261507a02010-12-17 14:21:50 +08002966 extent_set_compress_type(&this_bio_flag,
2967 em->compress_type);
2968 }
Chris Masonc8b97812008-10-29 14:49:59 -04002969
Chris Masond1310b22008-01-24 16:13:08 -05002970 iosize = min(extent_map_end(em) - cur, end - cur + 1);
2971 cur_end = min(extent_map_end(em) - 1, end);
Qu Wenruofda28322013-02-26 08:10:22 +00002972 iosize = ALIGN(iosize, blocksize);
Chris Masonc8b97812008-10-29 14:49:59 -04002973 if (this_bio_flag & EXTENT_BIO_COMPRESSED) {
2974 disk_io_size = em->block_len;
2975 sector = em->block_start >> 9;
2976 } else {
2977 sector = (em->block_start + extent_offset) >> 9;
2978 disk_io_size = iosize;
2979 }
Chris Masond1310b22008-01-24 16:13:08 -05002980 bdev = em->bdev;
2981 block_start = em->block_start;
Yan Zhengd899e052008-10-30 14:25:28 -04002982 if (test_bit(EXTENT_FLAG_PREALLOC, &em->flags))
2983 block_start = EXTENT_MAP_HOLE;
Filipe Manana005efed2015-09-14 09:09:31 +01002984
2985 /*
2986 * If we have a file range that points to a compressed extent
2987 * and it's followed by a consecutive file range that points to
2988 * to the same compressed extent (possibly with a different
2989 * offset and/or length, so it either points to the whole extent
2990 * or only part of it), we must make sure we do not submit a
2991 * single bio to populate the pages for the 2 ranges because
2992 * this makes the compressed extent read zero out the pages
2993 * belonging to the 2nd range. Imagine the following scenario:
2994 *
2995 * File layout
2996 * [0 - 8K] [8K - 24K]
2997 * | |
2998 * | |
2999 * points to extent X, points to extent X,
3000 * offset 4K, length of 8K offset 0, length 16K
3001 *
3002 * [extent X, compressed length = 4K uncompressed length = 16K]
3003 *
3004 * If the bio to read the compressed extent covers both ranges,
3005 * it will decompress extent X into the pages belonging to the
3006 * first range and then it will stop, zeroing out the remaining
3007 * pages that belong to the other range that points to extent X.
3008 * So here we make sure we submit 2 bios, one for the first
3009 * range and another one for the third range. Both will target
3010 * the same physical extent from disk, but we can't currently
3011 * make the compressed bio endio callback populate the pages
3012 * for both ranges because each compressed bio is tightly
3013 * coupled with a single extent map, and each range can have
3014 * an extent map with a different offset value relative to the
3015 * uncompressed data of our extent and different lengths. This
3016 * is a corner case so we prioritize correctness over
3017 * non-optimal behavior (submitting 2 bios for the same extent).
3018 */
3019 if (test_bit(EXTENT_FLAG_COMPRESSED, &em->flags) &&
3020 prev_em_start && *prev_em_start != (u64)-1 &&
Filipe Mananafc3a73f2019-02-14 15:17:20 +00003021 *prev_em_start != em->start)
Filipe Manana005efed2015-09-14 09:09:31 +01003022 force_bio_submit = true;
3023
3024 if (prev_em_start)
Filipe Mananafc3a73f2019-02-14 15:17:20 +00003025 *prev_em_start = em->start;
Filipe Manana005efed2015-09-14 09:09:31 +01003026
Chris Masond1310b22008-01-24 16:13:08 -05003027 free_extent_map(em);
3028 em = NULL;
3029
3030 /* we've found a hole, just zero and go on */
3031 if (block_start == EXTENT_MAP_HOLE) {
3032 char *userpage;
Arne Jansen507903b2011-04-06 10:02:20 +00003033 struct extent_state *cached = NULL;
3034
Cong Wang7ac687d2011-11-25 23:14:28 +08003035 userpage = kmap_atomic(page);
David Sterba306e16c2011-04-19 14:29:38 +02003036 memset(userpage + pg_offset, 0, iosize);
Chris Masond1310b22008-01-24 16:13:08 -05003037 flush_dcache_page(page);
Cong Wang7ac687d2011-11-25 23:14:28 +08003038 kunmap_atomic(userpage);
Chris Masond1310b22008-01-24 16:13:08 -05003039
3040 set_extent_uptodate(tree, cur, cur + iosize - 1,
Arne Jansen507903b2011-04-06 10:02:20 +00003041 &cached, GFP_NOFS);
Filipe Manana7f042a82016-01-27 19:17:20 +00003042 unlock_extent_cached(tree, cur,
3043 cur + iosize - 1,
3044 &cached, GFP_NOFS);
Chris Masond1310b22008-01-24 16:13:08 -05003045 cur = cur + iosize;
David Sterba306e16c2011-04-19 14:29:38 +02003046 pg_offset += iosize;
Chris Masond1310b22008-01-24 16:13:08 -05003047 continue;
3048 }
3049 /* the get_extent function already copied into the page */
Chris Mason9655d292009-09-02 15:22:30 -04003050 if (test_range_bit(tree, cur, cur_end,
3051 EXTENT_UPTODATE, 1, NULL)) {
Chris Masona1b32a52008-09-05 16:09:51 -04003052 check_page_uptodate(tree, page);
Filipe Manana7f042a82016-01-27 19:17:20 +00003053 unlock_extent(tree, cur, cur + iosize - 1);
Chris Masond1310b22008-01-24 16:13:08 -05003054 cur = cur + iosize;
David Sterba306e16c2011-04-19 14:29:38 +02003055 pg_offset += iosize;
Chris Masond1310b22008-01-24 16:13:08 -05003056 continue;
3057 }
Chris Mason70dec802008-01-29 09:59:12 -05003058 /* we have an inline extent but it didn't get marked up
3059 * to date. Error out
3060 */
3061 if (block_start == EXTENT_MAP_INLINE) {
3062 SetPageError(page);
Filipe Manana7f042a82016-01-27 19:17:20 +00003063 unlock_extent(tree, cur, cur + iosize - 1);
Chris Mason70dec802008-01-29 09:59:12 -05003064 cur = cur + iosize;
David Sterba306e16c2011-04-19 14:29:38 +02003065 pg_offset += iosize;
Chris Mason70dec802008-01-29 09:59:12 -05003066 continue;
3067 }
Chris Masond1310b22008-01-24 16:13:08 -05003068
Josef Bacikc8f2f242013-02-11 11:33:00 -05003069 pnr -= page->index;
Mike Christie1f7ad752016-06-05 14:31:51 -05003070 ret = submit_extent_page(REQ_OP_READ, read_flags, tree, NULL,
3071 page, sector, disk_io_size, pg_offset,
Chris Mason89642222008-07-24 09:41:53 -04003072 bdev, bio, pnr,
Chris Masonc8b97812008-10-29 14:49:59 -04003073 end_bio_extent_readpage, mirror_num,
3074 *bio_flags,
Filipe Manana005efed2015-09-14 09:09:31 +01003075 this_bio_flag,
3076 force_bio_submit);
Josef Bacikc8f2f242013-02-11 11:33:00 -05003077 if (!ret) {
3078 nr++;
3079 *bio_flags = this_bio_flag;
3080 } else {
Chris Masond1310b22008-01-24 16:13:08 -05003081 SetPageError(page);
Filipe Manana7f042a82016-01-27 19:17:20 +00003082 unlock_extent(tree, cur, cur + iosize - 1);
Liu Bobaf863b2016-07-11 10:39:07 -07003083 goto out;
Josef Bacikedd33c92012-10-05 16:40:32 -04003084 }
Chris Masond1310b22008-01-24 16:13:08 -05003085 cur = cur + iosize;
David Sterba306e16c2011-04-19 14:29:38 +02003086 pg_offset += iosize;
Chris Masond1310b22008-01-24 16:13:08 -05003087 }
Dan Magenheimer90a887c2011-05-26 10:01:56 -06003088out:
Chris Masond1310b22008-01-24 16:13:08 -05003089 if (!nr) {
3090 if (!PageError(page))
3091 SetPageUptodate(page);
3092 unlock_page(page);
3093 }
Liu Bobaf863b2016-07-11 10:39:07 -07003094 return ret;
Chris Masond1310b22008-01-24 16:13:08 -05003095}
3096
Miao Xie99740902013-07-25 19:22:36 +08003097static inline void __do_contiguous_readpages(struct extent_io_tree *tree,
3098 struct page *pages[], int nr_pages,
3099 u64 start, u64 end,
3100 get_extent_t *get_extent,
Miao Xie125bac012013-07-25 19:22:37 +08003101 struct extent_map **em_cached,
Miao Xie99740902013-07-25 19:22:36 +08003102 struct bio **bio, int mirror_num,
Mike Christie1f7ad752016-06-05 14:31:51 -05003103 unsigned long *bio_flags,
Filipe Manana808f80b2015-09-28 09:56:26 +01003104 u64 *prev_em_start)
Miao Xie99740902013-07-25 19:22:36 +08003105{
3106 struct inode *inode;
3107 struct btrfs_ordered_extent *ordered;
3108 int index;
3109
3110 inode = pages[0]->mapping->host;
3111 while (1) {
3112 lock_extent(tree, start, end);
3113 ordered = btrfs_lookup_ordered_range(inode, start,
3114 end - start + 1);
3115 if (!ordered)
3116 break;
3117 unlock_extent(tree, start, end);
3118 btrfs_start_ordered_extent(inode, ordered, 1);
3119 btrfs_put_ordered_extent(ordered);
3120 }
3121
3122 for (index = 0; index < nr_pages; index++) {
Miao Xie125bac012013-07-25 19:22:37 +08003123 __do_readpage(tree, pages[index], get_extent, em_cached, bio,
Mike Christie1f7ad752016-06-05 14:31:51 -05003124 mirror_num, bio_flags, 0, prev_em_start);
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03003125 put_page(pages[index]);
Miao Xie99740902013-07-25 19:22:36 +08003126 }
3127}
3128
3129static void __extent_readpages(struct extent_io_tree *tree,
3130 struct page *pages[],
3131 int nr_pages, get_extent_t *get_extent,
Miao Xie125bac012013-07-25 19:22:37 +08003132 struct extent_map **em_cached,
Miao Xie99740902013-07-25 19:22:36 +08003133 struct bio **bio, int mirror_num,
Mike Christie1f7ad752016-06-05 14:31:51 -05003134 unsigned long *bio_flags,
Filipe Manana808f80b2015-09-28 09:56:26 +01003135 u64 *prev_em_start)
Miao Xie99740902013-07-25 19:22:36 +08003136{
Stefan Behrens35a36212013-08-14 18:12:25 +02003137 u64 start = 0;
Miao Xie99740902013-07-25 19:22:36 +08003138 u64 end = 0;
3139 u64 page_start;
3140 int index;
Stefan Behrens35a36212013-08-14 18:12:25 +02003141 int first_index = 0;
Miao Xie99740902013-07-25 19:22:36 +08003142
3143 for (index = 0; index < nr_pages; index++) {
3144 page_start = page_offset(pages[index]);
3145 if (!end) {
3146 start = page_start;
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03003147 end = start + PAGE_SIZE - 1;
Miao Xie99740902013-07-25 19:22:36 +08003148 first_index = index;
3149 } else if (end + 1 == page_start) {
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03003150 end += PAGE_SIZE;
Miao Xie99740902013-07-25 19:22:36 +08003151 } else {
3152 __do_contiguous_readpages(tree, &pages[first_index],
3153 index - first_index, start,
Miao Xie125bac012013-07-25 19:22:37 +08003154 end, get_extent, em_cached,
3155 bio, mirror_num, bio_flags,
Mike Christie1f7ad752016-06-05 14:31:51 -05003156 prev_em_start);
Miao Xie99740902013-07-25 19:22:36 +08003157 start = page_start;
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03003158 end = start + PAGE_SIZE - 1;
Miao Xie99740902013-07-25 19:22:36 +08003159 first_index = index;
3160 }
3161 }
3162
3163 if (end)
3164 __do_contiguous_readpages(tree, &pages[first_index],
3165 index - first_index, start,
Miao Xie125bac012013-07-25 19:22:37 +08003166 end, get_extent, em_cached, bio,
Mike Christie1f7ad752016-06-05 14:31:51 -05003167 mirror_num, bio_flags,
Filipe Manana808f80b2015-09-28 09:56:26 +01003168 prev_em_start);
Miao Xie99740902013-07-25 19:22:36 +08003169}
3170
3171static int __extent_read_full_page(struct extent_io_tree *tree,
3172 struct page *page,
3173 get_extent_t *get_extent,
3174 struct bio **bio, int mirror_num,
Mike Christie1f7ad752016-06-05 14:31:51 -05003175 unsigned long *bio_flags, int read_flags)
Miao Xie99740902013-07-25 19:22:36 +08003176{
3177 struct inode *inode = page->mapping->host;
3178 struct btrfs_ordered_extent *ordered;
3179 u64 start = page_offset(page);
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03003180 u64 end = start + PAGE_SIZE - 1;
Miao Xie99740902013-07-25 19:22:36 +08003181 int ret;
3182
3183 while (1) {
3184 lock_extent(tree, start, end);
Chandan Rajendradbfdb6d2016-01-21 15:55:58 +05303185 ordered = btrfs_lookup_ordered_range(inode, start,
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03003186 PAGE_SIZE);
Miao Xie99740902013-07-25 19:22:36 +08003187 if (!ordered)
3188 break;
3189 unlock_extent(tree, start, end);
3190 btrfs_start_ordered_extent(inode, ordered, 1);
3191 btrfs_put_ordered_extent(ordered);
3192 }
3193
Miao Xie125bac012013-07-25 19:22:37 +08003194 ret = __do_readpage(tree, page, get_extent, NULL, bio, mirror_num,
Mike Christie1f7ad752016-06-05 14:31:51 -05003195 bio_flags, read_flags, NULL);
Miao Xie99740902013-07-25 19:22:36 +08003196 return ret;
3197}
3198
Chris Masond1310b22008-01-24 16:13:08 -05003199int extent_read_full_page(struct extent_io_tree *tree, struct page *page,
Jan Schmidt8ddc7d92011-06-13 20:02:58 +02003200 get_extent_t *get_extent, int mirror_num)
Chris Masond1310b22008-01-24 16:13:08 -05003201{
3202 struct bio *bio = NULL;
Chris Masonc8b97812008-10-29 14:49:59 -04003203 unsigned long bio_flags = 0;
Chris Masond1310b22008-01-24 16:13:08 -05003204 int ret;
3205
Jan Schmidt8ddc7d92011-06-13 20:02:58 +02003206 ret = __extent_read_full_page(tree, page, get_extent, &bio, mirror_num,
Mike Christie1f7ad752016-06-05 14:31:51 -05003207 &bio_flags, 0);
Chris Masond1310b22008-01-24 16:13:08 -05003208 if (bio)
Mike Christie1f7ad752016-06-05 14:31:51 -05003209 ret = submit_one_bio(bio, mirror_num, bio_flags);
Chris Masond1310b22008-01-24 16:13:08 -05003210 return ret;
3211}
Chris Masond1310b22008-01-24 16:13:08 -05003212
Liu Boa91326672016-03-07 16:56:21 -08003213static void update_nr_written(struct page *page, struct writeback_control *wbc,
3214 unsigned long nr_written)
Chris Mason11c83492009-04-20 15:50:09 -04003215{
3216 wbc->nr_to_write -= nr_written;
Chris Mason11c83492009-04-20 15:50:09 -04003217}
3218
Chris Masond1310b22008-01-24 16:13:08 -05003219/*
Chris Mason40f76582014-05-21 13:35:51 -07003220 * helper for __extent_writepage, doing all of the delayed allocation setup.
3221 *
3222 * This returns 1 if our fill_delalloc function did all the work required
3223 * to write the page (copy into inline extent). In this case the IO has
3224 * been started and the page is already unlocked.
3225 *
3226 * This returns 0 if all went well (page still locked)
3227 * This returns < 0 if there were errors (page still locked)
Chris Masond1310b22008-01-24 16:13:08 -05003228 */
Chris Mason40f76582014-05-21 13:35:51 -07003229static noinline_for_stack int writepage_delalloc(struct inode *inode,
3230 struct page *page, struct writeback_control *wbc,
3231 struct extent_page_data *epd,
3232 u64 delalloc_start,
3233 unsigned long *nr_written)
Chris Masond1310b22008-01-24 16:13:08 -05003234{
Chris Mason40f76582014-05-21 13:35:51 -07003235 struct extent_io_tree *tree = epd->tree;
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03003236 u64 page_end = delalloc_start + PAGE_SIZE - 1;
Chris Mason40f76582014-05-21 13:35:51 -07003237 u64 nr_delalloc;
3238 u64 delalloc_to_write = 0;
3239 u64 delalloc_end = 0;
3240 int ret;
3241 int page_started = 0;
3242
3243 if (epd->extent_locked || !tree->ops || !tree->ops->fill_delalloc)
3244 return 0;
3245
3246 while (delalloc_end < page_end) {
3247 nr_delalloc = find_lock_delalloc_range(inode, tree,
3248 page,
3249 &delalloc_start,
3250 &delalloc_end,
Josef Bacikdcab6a32015-02-11 15:08:59 -05003251 BTRFS_MAX_EXTENT_SIZE);
Chris Mason40f76582014-05-21 13:35:51 -07003252 if (nr_delalloc == 0) {
3253 delalloc_start = delalloc_end + 1;
3254 continue;
3255 }
3256 ret = tree->ops->fill_delalloc(inode, page,
3257 delalloc_start,
3258 delalloc_end,
3259 &page_started,
3260 nr_written);
3261 /* File system has been set read-only */
3262 if (ret) {
3263 SetPageError(page);
3264 /* fill_delalloc should be return < 0 for error
3265 * but just in case, we use > 0 here meaning the
3266 * IO is started, so we don't want to return > 0
3267 * unless things are going well.
3268 */
3269 ret = ret < 0 ? ret : -EIO;
3270 goto done;
3271 }
3272 /*
Kirill A. Shutemovea1754a2016-04-01 15:29:48 +03003273 * delalloc_end is already one less than the total length, so
3274 * we don't subtract one from PAGE_SIZE
Chris Mason40f76582014-05-21 13:35:51 -07003275 */
3276 delalloc_to_write += (delalloc_end - delalloc_start +
Kirill A. Shutemovea1754a2016-04-01 15:29:48 +03003277 PAGE_SIZE) >> PAGE_SHIFT;
Chris Mason40f76582014-05-21 13:35:51 -07003278 delalloc_start = delalloc_end + 1;
3279 }
3280 if (wbc->nr_to_write < delalloc_to_write) {
3281 int thresh = 8192;
3282
3283 if (delalloc_to_write < thresh * 2)
3284 thresh = delalloc_to_write;
3285 wbc->nr_to_write = min_t(u64, delalloc_to_write,
3286 thresh);
3287 }
3288
3289 /* did the fill delalloc function already unlock and start
3290 * the IO?
3291 */
3292 if (page_started) {
3293 /*
3294 * we've unlocked the page, so we can't update
3295 * the mapping's writeback index, just update
3296 * nr_to_write.
3297 */
3298 wbc->nr_to_write -= *nr_written;
3299 return 1;
3300 }
3301
3302 ret = 0;
3303
3304done:
3305 return ret;
3306}
3307
3308/*
3309 * helper for __extent_writepage. This calls the writepage start hooks,
3310 * and does the loop to map the page into extents and bios.
3311 *
3312 * We return 1 if the IO is started and the page is unlocked,
3313 * 0 if all went well (page still locked)
3314 * < 0 if there were errors (page still locked)
3315 */
3316static noinline_for_stack int __extent_writepage_io(struct inode *inode,
3317 struct page *page,
3318 struct writeback_control *wbc,
3319 struct extent_page_data *epd,
3320 loff_t i_size,
3321 unsigned long nr_written,
3322 int write_flags, int *nr_ret)
3323{
Chris Masond1310b22008-01-24 16:13:08 -05003324 struct extent_io_tree *tree = epd->tree;
Miao Xie4eee4fa2012-12-21 09:17:45 +00003325 u64 start = page_offset(page);
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03003326 u64 page_end = start + PAGE_SIZE - 1;
Chris Masond1310b22008-01-24 16:13:08 -05003327 u64 end;
3328 u64 cur = start;
3329 u64 extent_offset;
Chris Masond1310b22008-01-24 16:13:08 -05003330 u64 block_start;
3331 u64 iosize;
3332 sector_t sector;
Chris Mason2c64c532009-09-02 15:04:12 -04003333 struct extent_state *cached_state = NULL;
Chris Masond1310b22008-01-24 16:13:08 -05003334 struct extent_map *em;
3335 struct block_device *bdev;
Chris Mason7f3c74f2008-07-18 12:01:11 -04003336 size_t pg_offset = 0;
Chris Masond1310b22008-01-24 16:13:08 -05003337 size_t blocksize;
Chris Mason40f76582014-05-21 13:35:51 -07003338 int ret = 0;
3339 int nr = 0;
3340 bool compressed;
Chris Masond1310b22008-01-24 16:13:08 -05003341
Chris Mason247e7432008-07-17 12:53:51 -04003342 if (tree->ops && tree->ops->writepage_start_hook) {
Chris Masonc8b97812008-10-29 14:49:59 -04003343 ret = tree->ops->writepage_start_hook(page, start,
3344 page_end);
Jeff Mahoney87826df2012-02-15 16:23:57 +01003345 if (ret) {
3346 /* Fixup worker will requeue */
3347 if (ret == -EBUSY)
3348 wbc->pages_skipped++;
3349 else
3350 redirty_page_for_writepage(wbc, page);
Chris Mason40f76582014-05-21 13:35:51 -07003351
Chris Mason11c83492009-04-20 15:50:09 -04003352 update_nr_written(page, wbc, nr_written);
Chris Mason247e7432008-07-17 12:53:51 -04003353 unlock_page(page);
Chris Mason40f76582014-05-21 13:35:51 -07003354 ret = 1;
Chris Mason11c83492009-04-20 15:50:09 -04003355 goto done_unlocked;
Chris Mason247e7432008-07-17 12:53:51 -04003356 }
3357 }
3358
Chris Mason11c83492009-04-20 15:50:09 -04003359 /*
3360 * we don't want to touch the inode after unlocking the page,
3361 * so we update the mapping writeback index now
3362 */
3363 update_nr_written(page, wbc, nr_written + 1);
Chris Mason771ed682008-11-06 22:02:51 -05003364
Chris Masond1310b22008-01-24 16:13:08 -05003365 end = page_end;
Chris Mason40f76582014-05-21 13:35:51 -07003366 if (i_size <= start) {
Chris Masone6dcd2d2008-07-17 12:53:50 -04003367 if (tree->ops && tree->ops->writepage_end_io_hook)
3368 tree->ops->writepage_end_io_hook(page, start,
3369 page_end, NULL, 1);
Chris Masond1310b22008-01-24 16:13:08 -05003370 goto done;
3371 }
3372
Chris Masond1310b22008-01-24 16:13:08 -05003373 blocksize = inode->i_sb->s_blocksize;
3374
3375 while (cur <= end) {
Chris Mason40f76582014-05-21 13:35:51 -07003376 u64 em_end;
David Sterba58409ed2016-05-04 11:46:10 +02003377 unsigned long max_nr;
3378
Chris Mason40f76582014-05-21 13:35:51 -07003379 if (cur >= i_size) {
Chris Masone6dcd2d2008-07-17 12:53:50 -04003380 if (tree->ops && tree->ops->writepage_end_io_hook)
3381 tree->ops->writepage_end_io_hook(page, cur,
3382 page_end, NULL, 1);
Chris Masond1310b22008-01-24 16:13:08 -05003383 break;
3384 }
Chris Mason7f3c74f2008-07-18 12:01:11 -04003385 em = epd->get_extent(inode, page, pg_offset, cur,
Chris Masond1310b22008-01-24 16:13:08 -05003386 end - cur + 1, 1);
David Sterbac7040052011-04-19 18:00:01 +02003387 if (IS_ERR_OR_NULL(em)) {
Chris Masond1310b22008-01-24 16:13:08 -05003388 SetPageError(page);
Filipe Manana61391d52014-05-09 17:17:40 +01003389 ret = PTR_ERR_OR_ZERO(em);
Chris Masond1310b22008-01-24 16:13:08 -05003390 break;
3391 }
3392
3393 extent_offset = cur - em->start;
Chris Mason40f76582014-05-21 13:35:51 -07003394 em_end = extent_map_end(em);
3395 BUG_ON(em_end <= cur);
Chris Masond1310b22008-01-24 16:13:08 -05003396 BUG_ON(end < cur);
Chris Mason40f76582014-05-21 13:35:51 -07003397 iosize = min(em_end - cur, end - cur + 1);
Qu Wenruofda28322013-02-26 08:10:22 +00003398 iosize = ALIGN(iosize, blocksize);
Chris Masond1310b22008-01-24 16:13:08 -05003399 sector = (em->block_start + extent_offset) >> 9;
3400 bdev = em->bdev;
3401 block_start = em->block_start;
Chris Masonc8b97812008-10-29 14:49:59 -04003402 compressed = test_bit(EXTENT_FLAG_COMPRESSED, &em->flags);
Chris Masond1310b22008-01-24 16:13:08 -05003403 free_extent_map(em);
3404 em = NULL;
3405
Chris Masonc8b97812008-10-29 14:49:59 -04003406 /*
3407 * compressed and inline extents are written through other
3408 * paths in the FS
3409 */
3410 if (compressed || block_start == EXTENT_MAP_HOLE ||
Chris Masond1310b22008-01-24 16:13:08 -05003411 block_start == EXTENT_MAP_INLINE) {
Chris Masonc8b97812008-10-29 14:49:59 -04003412 /*
3413 * end_io notification does not happen here for
3414 * compressed extents
3415 */
3416 if (!compressed && tree->ops &&
3417 tree->ops->writepage_end_io_hook)
Chris Masone6dcd2d2008-07-17 12:53:50 -04003418 tree->ops->writepage_end_io_hook(page, cur,
3419 cur + iosize - 1,
3420 NULL, 1);
Chris Masonc8b97812008-10-29 14:49:59 -04003421 else if (compressed) {
3422 /* we don't want to end_page_writeback on
3423 * a compressed extent. this happens
3424 * elsewhere
3425 */
3426 nr++;
3427 }
3428
3429 cur += iosize;
Chris Mason7f3c74f2008-07-18 12:01:11 -04003430 pg_offset += iosize;
Chris Masond1310b22008-01-24 16:13:08 -05003431 continue;
3432 }
Chris Masonc8b97812008-10-29 14:49:59 -04003433
David Sterba58409ed2016-05-04 11:46:10 +02003434 max_nr = (i_size >> PAGE_SHIFT) + 1;
3435
3436 set_range_writeback(tree, cur, cur + iosize - 1);
3437 if (!PageWriteback(page)) {
3438 btrfs_err(BTRFS_I(inode)->root->fs_info,
3439 "page %lu not writeback, cur %llu end %llu",
3440 page->index, cur, end);
Chris Masond1310b22008-01-24 16:13:08 -05003441 }
David Sterba58409ed2016-05-04 11:46:10 +02003442
Mike Christie1f7ad752016-06-05 14:31:51 -05003443 ret = submit_extent_page(REQ_OP_WRITE, write_flags, tree, wbc,
3444 page, sector, iosize, pg_offset,
David Sterba58409ed2016-05-04 11:46:10 +02003445 bdev, &epd->bio, max_nr,
3446 end_bio_extent_writepage,
3447 0, 0, 0, false);
3448 if (ret)
Chris Masond1310b22008-01-24 16:13:08 -05003449 SetPageError(page);
Chris Mason7f3c74f2008-07-18 12:01:11 -04003450
Chris Masond1310b22008-01-24 16:13:08 -05003451 cur = cur + iosize;
Chris Mason7f3c74f2008-07-18 12:01:11 -04003452 pg_offset += iosize;
Chris Masond1310b22008-01-24 16:13:08 -05003453 nr++;
3454 }
3455done:
Chris Mason40f76582014-05-21 13:35:51 -07003456 *nr_ret = nr;
Chris Mason771ed682008-11-06 22:02:51 -05003457
Chris Mason11c83492009-04-20 15:50:09 -04003458done_unlocked:
3459
Chris Mason2c64c532009-09-02 15:04:12 -04003460 /* drop our reference on any cached states */
3461 free_extent_state(cached_state);
Chris Mason40f76582014-05-21 13:35:51 -07003462 return ret;
3463}
3464
3465/*
3466 * the writepage semantics are similar to regular writepage. extent
3467 * records are inserted to lock ranges in the tree, and as dirty areas
3468 * are found, they are marked writeback. Then the lock bits are removed
3469 * and the end_io handler clears the writeback ranges
3470 */
3471static int __extent_writepage(struct page *page, struct writeback_control *wbc,
3472 void *data)
3473{
3474 struct inode *inode = page->mapping->host;
3475 struct extent_page_data *epd = data;
3476 u64 start = page_offset(page);
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03003477 u64 page_end = start + PAGE_SIZE - 1;
Chris Mason40f76582014-05-21 13:35:51 -07003478 int ret;
3479 int nr = 0;
3480 size_t pg_offset = 0;
3481 loff_t i_size = i_size_read(inode);
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03003482 unsigned long end_index = i_size >> PAGE_SHIFT;
Mike Christie1f7ad752016-06-05 14:31:51 -05003483 int write_flags = 0;
Chris Mason40f76582014-05-21 13:35:51 -07003484 unsigned long nr_written = 0;
3485
3486 if (wbc->sync_mode == WB_SYNC_ALL)
3487 write_flags = WRITE_SYNC;
Chris Mason40f76582014-05-21 13:35:51 -07003488
3489 trace___extent_writepage(page, inode, wbc);
3490
3491 WARN_ON(!PageLocked(page));
3492
3493 ClearPageError(page);
3494
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03003495 pg_offset = i_size & (PAGE_SIZE - 1);
Chris Mason40f76582014-05-21 13:35:51 -07003496 if (page->index > end_index ||
3497 (page->index == end_index && !pg_offset)) {
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03003498 page->mapping->a_ops->invalidatepage(page, 0, PAGE_SIZE);
Chris Mason40f76582014-05-21 13:35:51 -07003499 unlock_page(page);
3500 return 0;
3501 }
3502
3503 if (page->index == end_index) {
3504 char *userpage;
3505
3506 userpage = kmap_atomic(page);
3507 memset(userpage + pg_offset, 0,
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03003508 PAGE_SIZE - pg_offset);
Chris Mason40f76582014-05-21 13:35:51 -07003509 kunmap_atomic(userpage);
3510 flush_dcache_page(page);
3511 }
3512
3513 pg_offset = 0;
3514
3515 set_page_extent_mapped(page);
3516
3517 ret = writepage_delalloc(inode, page, wbc, epd, start, &nr_written);
3518 if (ret == 1)
3519 goto done_unlocked;
3520 if (ret)
3521 goto done;
3522
3523 ret = __extent_writepage_io(inode, page, wbc, epd,
3524 i_size, nr_written, write_flags, &nr);
3525 if (ret == 1)
3526 goto done_unlocked;
3527
3528done:
Chris Masone6dcd2d2008-07-17 12:53:50 -04003529 if (nr == 0) {
Chris Masond1310b22008-01-24 16:13:08 -05003530 /* make sure the mapping tag for page dirty gets cleared */
Chris Mason771ed682008-11-06 22:02:51 -05003531 set_page_writeback(page);
3532 end_page_writeback(page);
3533 }
Filipe Manana61391d52014-05-09 17:17:40 +01003534 if (PageError(page)) {
3535 ret = ret < 0 ? ret : -EIO;
3536 end_extent_writepage(page, ret, start, page_end);
3537 }
Christoph Hellwigb2950862008-12-02 09:54:17 -05003538 unlock_page(page);
Chris Mason40f76582014-05-21 13:35:51 -07003539 return ret;
Chris Mason4bef0842008-09-08 11:18:08 -04003540
3541done_unlocked:
Chris Masond1310b22008-01-24 16:13:08 -05003542 return 0;
3543}
3544
Josef Bacikfd8b2b62013-04-24 16:41:19 -04003545void wait_on_extent_buffer_writeback(struct extent_buffer *eb)
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003546{
NeilBrown74316202014-07-07 15:16:04 +10003547 wait_on_bit_io(&eb->bflags, EXTENT_BUFFER_WRITEBACK,
3548 TASK_UNINTERRUPTIBLE);
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003549}
3550
Chris Mason0e378df2014-05-19 20:55:27 -07003551static noinline_for_stack int
3552lock_extent_buffer_for_io(struct extent_buffer *eb,
3553 struct btrfs_fs_info *fs_info,
3554 struct extent_page_data *epd)
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003555{
3556 unsigned long i, num_pages;
3557 int flush = 0;
3558 int ret = 0;
3559
3560 if (!btrfs_try_tree_write_lock(eb)) {
3561 flush = 1;
3562 flush_write_bio(epd);
3563 btrfs_tree_lock(eb);
3564 }
3565
3566 if (test_bit(EXTENT_BUFFER_WRITEBACK, &eb->bflags)) {
3567 btrfs_tree_unlock(eb);
3568 if (!epd->sync_io)
3569 return 0;
3570 if (!flush) {
3571 flush_write_bio(epd);
3572 flush = 1;
3573 }
Chris Masona098d8e2012-03-21 12:09:56 -04003574 while (1) {
3575 wait_on_extent_buffer_writeback(eb);
3576 btrfs_tree_lock(eb);
3577 if (!test_bit(EXTENT_BUFFER_WRITEBACK, &eb->bflags))
3578 break;
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003579 btrfs_tree_unlock(eb);
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003580 }
3581 }
3582
Josef Bacik51561ff2012-07-20 16:25:24 -04003583 /*
3584 * We need to do this to prevent races in people who check if the eb is
3585 * under IO since we can end up having no IO bits set for a short period
3586 * of time.
3587 */
3588 spin_lock(&eb->refs_lock);
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003589 if (test_and_clear_bit(EXTENT_BUFFER_DIRTY, &eb->bflags)) {
3590 set_bit(EXTENT_BUFFER_WRITEBACK, &eb->bflags);
Josef Bacik51561ff2012-07-20 16:25:24 -04003591 spin_unlock(&eb->refs_lock);
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003592 btrfs_set_header_flag(eb, BTRFS_HEADER_FLAG_WRITTEN);
Miao Xiee2d84522013-01-29 10:09:20 +00003593 __percpu_counter_add(&fs_info->dirty_metadata_bytes,
3594 -eb->len,
3595 fs_info->dirty_metadata_batch);
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003596 ret = 1;
Josef Bacik51561ff2012-07-20 16:25:24 -04003597 } else {
3598 spin_unlock(&eb->refs_lock);
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003599 }
3600
3601 btrfs_tree_unlock(eb);
3602
3603 if (!ret)
3604 return ret;
3605
3606 num_pages = num_extent_pages(eb->start, eb->len);
3607 for (i = 0; i < num_pages; i++) {
David Sterbafb85fc92014-07-31 01:03:53 +02003608 struct page *p = eb->pages[i];
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003609
3610 if (!trylock_page(p)) {
3611 if (!flush) {
3612 flush_write_bio(epd);
3613 flush = 1;
3614 }
3615 lock_page(p);
3616 }
3617 }
3618
3619 return ret;
3620}
3621
3622static void end_extent_buffer_writeback(struct extent_buffer *eb)
3623{
3624 clear_bit(EXTENT_BUFFER_WRITEBACK, &eb->bflags);
Peter Zijlstra4e857c52014-03-17 18:06:10 +01003625 smp_mb__after_atomic();
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003626 wake_up_bit(&eb->bflags, EXTENT_BUFFER_WRITEBACK);
3627}
3628
Filipe Manana656f30d2014-09-26 12:25:56 +01003629static void set_btree_ioerr(struct page *page)
3630{
3631 struct extent_buffer *eb = (struct extent_buffer *)page->private;
Filipe Manana656f30d2014-09-26 12:25:56 +01003632
3633 SetPageError(page);
3634 if (test_and_set_bit(EXTENT_BUFFER_WRITE_ERR, &eb->bflags))
3635 return;
3636
3637 /*
3638 * If writeback for a btree extent that doesn't belong to a log tree
3639 * failed, increment the counter transaction->eb_write_errors.
3640 * We do this because while the transaction is running and before it's
3641 * committing (when we call filemap_fdata[write|wait]_range against
3642 * the btree inode), we might have
3643 * btree_inode->i_mapping->a_ops->writepages() called by the VM - if it
3644 * returns an error or an error happens during writeback, when we're
3645 * committing the transaction we wouldn't know about it, since the pages
3646 * can be no longer dirty nor marked anymore for writeback (if a
3647 * subsequent modification to the extent buffer didn't happen before the
3648 * transaction commit), which makes filemap_fdata[write|wait]_range not
3649 * able to find the pages tagged with SetPageError at transaction
3650 * commit time. So if this happens we must abort the transaction,
3651 * otherwise we commit a super block with btree roots that point to
3652 * btree nodes/leafs whose content on disk is invalid - either garbage
3653 * or the content of some node/leaf from a past generation that got
3654 * cowed or deleted and is no longer valid.
3655 *
3656 * Note: setting AS_EIO/AS_ENOSPC in the btree inode's i_mapping would
3657 * not be enough - we need to distinguish between log tree extents vs
3658 * non-log tree extents, and the next filemap_fdatawait_range() call
3659 * will catch and clear such errors in the mapping - and that call might
3660 * be from a log sync and not from a transaction commit. Also, checking
3661 * for the eb flag EXTENT_BUFFER_WRITE_ERR at transaction commit time is
3662 * not done and would not be reliable - the eb might have been released
3663 * from memory and reading it back again means that flag would not be
3664 * set (since it's a runtime flag, not persisted on disk).
3665 *
3666 * Using the flags below in the btree inode also makes us achieve the
3667 * goal of AS_EIO/AS_ENOSPC when writepages() returns success, started
3668 * writeback for all dirty pages and before filemap_fdatawait_range()
3669 * is called, the writeback for all dirty pages had already finished
3670 * with errors - because we were not using AS_EIO/AS_ENOSPC,
3671 * filemap_fdatawait_range() would return success, as it could not know
3672 * that writeback errors happened (the pages were no longer tagged for
3673 * writeback).
3674 */
3675 switch (eb->log_index) {
3676 case -1:
Josef Bacikafcdd122016-09-02 15:40:02 -04003677 set_bit(BTRFS_FS_BTREE_ERR, &eb->fs_info->flags);
Filipe Manana656f30d2014-09-26 12:25:56 +01003678 break;
3679 case 0:
Josef Bacikafcdd122016-09-02 15:40:02 -04003680 set_bit(BTRFS_FS_LOG1_ERR, &eb->fs_info->flags);
Filipe Manana656f30d2014-09-26 12:25:56 +01003681 break;
3682 case 1:
Josef Bacikafcdd122016-09-02 15:40:02 -04003683 set_bit(BTRFS_FS_LOG2_ERR, &eb->fs_info->flags);
Filipe Manana656f30d2014-09-26 12:25:56 +01003684 break;
3685 default:
3686 BUG(); /* unexpected, logic error */
3687 }
3688}
3689
Christoph Hellwig4246a0b2015-07-20 15:29:37 +02003690static void end_bio_extent_buffer_writepage(struct bio *bio)
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003691{
Kent Overstreet2c30c712013-11-07 12:20:26 -08003692 struct bio_vec *bvec;
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003693 struct extent_buffer *eb;
Kent Overstreet2c30c712013-11-07 12:20:26 -08003694 int i, done;
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003695
Kent Overstreet2c30c712013-11-07 12:20:26 -08003696 bio_for_each_segment_all(bvec, bio, i) {
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003697 struct page *page = bvec->bv_page;
3698
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003699 eb = (struct extent_buffer *)page->private;
3700 BUG_ON(!eb);
3701 done = atomic_dec_and_test(&eb->io_pages);
3702
Christoph Hellwig4246a0b2015-07-20 15:29:37 +02003703 if (bio->bi_error ||
3704 test_bit(EXTENT_BUFFER_WRITE_ERR, &eb->bflags)) {
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003705 ClearPageUptodate(page);
Filipe Manana656f30d2014-09-26 12:25:56 +01003706 set_btree_ioerr(page);
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003707 }
3708
3709 end_page_writeback(page);
3710
3711 if (!done)
3712 continue;
3713
3714 end_extent_buffer_writeback(eb);
Kent Overstreet2c30c712013-11-07 12:20:26 -08003715 }
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003716
3717 bio_put(bio);
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003718}
3719
Chris Mason0e378df2014-05-19 20:55:27 -07003720static noinline_for_stack int write_one_eb(struct extent_buffer *eb,
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003721 struct btrfs_fs_info *fs_info,
3722 struct writeback_control *wbc,
3723 struct extent_page_data *epd)
3724{
3725 struct block_device *bdev = fs_info->fs_devices->latest_bdev;
Josef Bacikf28491e2013-12-16 13:24:27 -05003726 struct extent_io_tree *tree = &BTRFS_I(fs_info->btree_inode)->io_tree;
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003727 u64 offset = eb->start;
Liu Bo851cd172016-09-23 13:44:44 -07003728 u32 nritems;
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003729 unsigned long i, num_pages;
Josef Bacikde0022b2012-09-25 14:25:58 -04003730 unsigned long bio_flags = 0;
Liu Bo851cd172016-09-23 13:44:44 -07003731 unsigned long start, end;
Mike Christie1f7ad752016-06-05 14:31:51 -05003732 int write_flags = (epd->sync_io ? WRITE_SYNC : 0) | REQ_META;
Josef Bacikd7dbe9e2012-04-23 14:00:51 -04003733 int ret = 0;
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003734
Filipe Manana656f30d2014-09-26 12:25:56 +01003735 clear_bit(EXTENT_BUFFER_WRITE_ERR, &eb->bflags);
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003736 num_pages = num_extent_pages(eb->start, eb->len);
3737 atomic_set(&eb->io_pages, num_pages);
Josef Bacikde0022b2012-09-25 14:25:58 -04003738 if (btrfs_header_owner(eb) == BTRFS_TREE_LOG_OBJECTID)
3739 bio_flags = EXTENT_BIO_TREE_LOG;
3740
Liu Bo851cd172016-09-23 13:44:44 -07003741 /* set btree blocks beyond nritems with 0 to avoid stale content. */
3742 nritems = btrfs_header_nritems(eb);
Liu Bo3eb548e2016-09-14 17:22:57 -07003743 if (btrfs_header_level(eb) > 0) {
Liu Bo3eb548e2016-09-14 17:22:57 -07003744 end = btrfs_node_key_ptr_offset(nritems);
3745
3746 memset_extent_buffer(eb, 0, end, eb->len - end);
Liu Bo851cd172016-09-23 13:44:44 -07003747 } else {
3748 /*
3749 * leaf:
3750 * header 0 1 2 .. N ... data_N .. data_2 data_1 data_0
3751 */
3752 start = btrfs_item_nr_offset(nritems);
3753 end = btrfs_leaf_data(eb) +
3754 leaf_data_end(fs_info->tree_root, eb);
3755 memset_extent_buffer(eb, 0, start, end - start);
Liu Bo3eb548e2016-09-14 17:22:57 -07003756 }
3757
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003758 for (i = 0; i < num_pages; i++) {
David Sterbafb85fc92014-07-31 01:03:53 +02003759 struct page *p = eb->pages[i];
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003760
3761 clear_page_dirty_for_io(p);
3762 set_page_writeback(p);
Mike Christie1f7ad752016-06-05 14:31:51 -05003763 ret = submit_extent_page(REQ_OP_WRITE, write_flags, tree, wbc,
3764 p, offset >> 9, PAGE_SIZE, 0, bdev,
3765 &epd->bio, -1,
3766 end_bio_extent_buffer_writepage,
Filipe Manana005efed2015-09-14 09:09:31 +01003767 0, epd->bio_flags, bio_flags, false);
Josef Bacikde0022b2012-09-25 14:25:58 -04003768 epd->bio_flags = bio_flags;
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003769 if (ret) {
Filipe Manana656f30d2014-09-26 12:25:56 +01003770 set_btree_ioerr(p);
Filipe Manana55e3bd22014-09-22 17:41:04 +01003771 end_page_writeback(p);
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003772 if (atomic_sub_and_test(num_pages - i, &eb->io_pages))
3773 end_extent_buffer_writeback(eb);
3774 ret = -EIO;
3775 break;
3776 }
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03003777 offset += PAGE_SIZE;
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003778 update_nr_written(p, wbc, 1);
3779 unlock_page(p);
3780 }
3781
3782 if (unlikely(ret)) {
3783 for (; i < num_pages; i++) {
Chris Masonbbf65cf2014-10-04 09:56:45 -07003784 struct page *p = eb->pages[i];
Liu Bo81465022014-09-23 22:22:33 +08003785 clear_page_dirty_for_io(p);
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003786 unlock_page(p);
3787 }
3788 }
3789
3790 return ret;
3791}
3792
3793int btree_write_cache_pages(struct address_space *mapping,
3794 struct writeback_control *wbc)
3795{
3796 struct extent_io_tree *tree = &BTRFS_I(mapping->host)->io_tree;
3797 struct btrfs_fs_info *fs_info = BTRFS_I(mapping->host)->root->fs_info;
3798 struct extent_buffer *eb, *prev_eb = NULL;
3799 struct extent_page_data epd = {
3800 .bio = NULL,
3801 .tree = tree,
3802 .extent_locked = 0,
3803 .sync_io = wbc->sync_mode == WB_SYNC_ALL,
Josef Bacikde0022b2012-09-25 14:25:58 -04003804 .bio_flags = 0,
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003805 };
3806 int ret = 0;
3807 int done = 0;
3808 int nr_to_write_done = 0;
3809 struct pagevec pvec;
3810 int nr_pages;
3811 pgoff_t index;
3812 pgoff_t end; /* Inclusive */
3813 int scanned = 0;
3814 int tag;
3815
3816 pagevec_init(&pvec, 0);
3817 if (wbc->range_cyclic) {
3818 index = mapping->writeback_index; /* Start from prev offset */
3819 end = -1;
3820 } else {
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03003821 index = wbc->range_start >> PAGE_SHIFT;
3822 end = wbc->range_end >> PAGE_SHIFT;
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003823 scanned = 1;
3824 }
3825 if (wbc->sync_mode == WB_SYNC_ALL)
3826 tag = PAGECACHE_TAG_TOWRITE;
3827 else
3828 tag = PAGECACHE_TAG_DIRTY;
3829retry:
3830 if (wbc->sync_mode == WB_SYNC_ALL)
3831 tag_pages_for_writeback(mapping, index, end);
3832 while (!done && !nr_to_write_done && (index <= end) &&
3833 (nr_pages = pagevec_lookup_tag(&pvec, mapping, &index, tag,
3834 min(end - index, (pgoff_t)PAGEVEC_SIZE-1) + 1))) {
3835 unsigned i;
3836
3837 scanned = 1;
3838 for (i = 0; i < nr_pages; i++) {
3839 struct page *page = pvec.pages[i];
3840
3841 if (!PagePrivate(page))
3842 continue;
3843
3844 if (!wbc->range_cyclic && page->index > end) {
3845 done = 1;
3846 break;
3847 }
3848
Josef Bacikb5bae262012-09-14 13:43:01 -04003849 spin_lock(&mapping->private_lock);
3850 if (!PagePrivate(page)) {
3851 spin_unlock(&mapping->private_lock);
3852 continue;
3853 }
3854
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003855 eb = (struct extent_buffer *)page->private;
Josef Bacikb5bae262012-09-14 13:43:01 -04003856
3857 /*
3858 * Shouldn't happen and normally this would be a BUG_ON
3859 * but no sense in crashing the users box for something
3860 * we can survive anyway.
3861 */
Dulshani Gunawardhanafae7f212013-10-31 10:30:08 +05303862 if (WARN_ON(!eb)) {
Josef Bacikb5bae262012-09-14 13:43:01 -04003863 spin_unlock(&mapping->private_lock);
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003864 continue;
3865 }
3866
Josef Bacikb5bae262012-09-14 13:43:01 -04003867 if (eb == prev_eb) {
3868 spin_unlock(&mapping->private_lock);
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003869 continue;
3870 }
3871
Josef Bacikb5bae262012-09-14 13:43:01 -04003872 ret = atomic_inc_not_zero(&eb->refs);
3873 spin_unlock(&mapping->private_lock);
3874 if (!ret)
3875 continue;
3876
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003877 prev_eb = eb;
3878 ret = lock_extent_buffer_for_io(eb, fs_info, &epd);
3879 if (!ret) {
3880 free_extent_buffer(eb);
3881 continue;
3882 }
3883
3884 ret = write_one_eb(eb, fs_info, wbc, &epd);
3885 if (ret) {
3886 done = 1;
3887 free_extent_buffer(eb);
3888 break;
3889 }
3890 free_extent_buffer(eb);
3891
3892 /*
3893 * the filesystem may choose to bump up nr_to_write.
3894 * We have to make sure to honor the new nr_to_write
3895 * at any time
3896 */
3897 nr_to_write_done = wbc->nr_to_write <= 0;
3898 }
3899 pagevec_release(&pvec);
3900 cond_resched();
3901 }
3902 if (!scanned && !done) {
3903 /*
3904 * We hit the last page and there is more work to be done: wrap
3905 * back to the start of the file
3906 */
3907 scanned = 1;
3908 index = 0;
3909 goto retry;
3910 }
3911 flush_write_bio(&epd);
3912 return ret;
3913}
3914
Chris Masond1310b22008-01-24 16:13:08 -05003915/**
3916 * write_cache_pages - walk the list of dirty pages of the given address space and write all of them.
3917 * @mapping: address space structure to write
3918 * @wbc: subtract the number of written pages from *@wbc->nr_to_write
3919 * @writepage: function called for each page
3920 * @data: data passed to writepage function
3921 *
3922 * If a page is already under I/O, write_cache_pages() skips it, even
3923 * if it's dirty. This is desirable behaviour for memory-cleaning writeback,
3924 * but it is INCORRECT for data-integrity system calls such as fsync(). fsync()
3925 * and msync() need to guarantee that all the data which was dirty at the time
3926 * the call was made get new I/O started against them. If wbc->sync_mode is
3927 * WB_SYNC_ALL then we were called for data integrity and we must wait for
3928 * existing IO to complete.
3929 */
Chris Mason4bef0842008-09-08 11:18:08 -04003930static int extent_write_cache_pages(struct extent_io_tree *tree,
3931 struct address_space *mapping,
3932 struct writeback_control *wbc,
Chris Masond2c3f4f2008-11-19 12:44:22 -05003933 writepage_t writepage, void *data,
3934 void (*flush_fn)(void *))
Chris Masond1310b22008-01-24 16:13:08 -05003935{
Josef Bacik7fd1a3f2012-06-27 17:18:41 -04003936 struct inode *inode = mapping->host;
Chris Masond1310b22008-01-24 16:13:08 -05003937 int ret = 0;
3938 int done = 0;
Chris Masonf85d7d6c2009-09-18 16:03:16 -04003939 int nr_to_write_done = 0;
Chris Masond1310b22008-01-24 16:13:08 -05003940 struct pagevec pvec;
3941 int nr_pages;
3942 pgoff_t index;
3943 pgoff_t end; /* Inclusive */
Liu Boa91326672016-03-07 16:56:21 -08003944 pgoff_t done_index;
3945 int range_whole = 0;
Chris Masond1310b22008-01-24 16:13:08 -05003946 int scanned = 0;
Josef Bacikf7aaa062011-07-15 21:26:38 +00003947 int tag;
Chris Masond1310b22008-01-24 16:13:08 -05003948
Josef Bacik7fd1a3f2012-06-27 17:18:41 -04003949 /*
3950 * We have to hold onto the inode so that ordered extents can do their
3951 * work when the IO finishes. The alternative to this is failing to add
3952 * an ordered extent if the igrab() fails there and that is a huge pain
3953 * to deal with, so instead just hold onto the inode throughout the
3954 * writepages operation. If it fails here we are freeing up the inode
3955 * anyway and we'd rather not waste our time writing out stuff that is
3956 * going to be truncated anyway.
3957 */
3958 if (!igrab(inode))
3959 return 0;
3960
Chris Masond1310b22008-01-24 16:13:08 -05003961 pagevec_init(&pvec, 0);
3962 if (wbc->range_cyclic) {
3963 index = mapping->writeback_index; /* Start from prev offset */
3964 end = -1;
3965 } else {
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03003966 index = wbc->range_start >> PAGE_SHIFT;
3967 end = wbc->range_end >> PAGE_SHIFT;
Liu Boa91326672016-03-07 16:56:21 -08003968 if (wbc->range_start == 0 && wbc->range_end == LLONG_MAX)
3969 range_whole = 1;
Chris Masond1310b22008-01-24 16:13:08 -05003970 scanned = 1;
3971 }
Josef Bacikf7aaa062011-07-15 21:26:38 +00003972 if (wbc->sync_mode == WB_SYNC_ALL)
3973 tag = PAGECACHE_TAG_TOWRITE;
3974 else
3975 tag = PAGECACHE_TAG_DIRTY;
Chris Masond1310b22008-01-24 16:13:08 -05003976retry:
Josef Bacikf7aaa062011-07-15 21:26:38 +00003977 if (wbc->sync_mode == WB_SYNC_ALL)
3978 tag_pages_for_writeback(mapping, index, end);
Liu Boa91326672016-03-07 16:56:21 -08003979 done_index = index;
Chris Masonf85d7d6c2009-09-18 16:03:16 -04003980 while (!done && !nr_to_write_done && (index <= end) &&
Josef Bacikf7aaa062011-07-15 21:26:38 +00003981 (nr_pages = pagevec_lookup_tag(&pvec, mapping, &index, tag,
3982 min(end - index, (pgoff_t)PAGEVEC_SIZE-1) + 1))) {
Chris Masond1310b22008-01-24 16:13:08 -05003983 unsigned i;
3984
3985 scanned = 1;
3986 for (i = 0; i < nr_pages; i++) {
3987 struct page *page = pvec.pages[i];
3988
Liu Boa91326672016-03-07 16:56:21 -08003989 done_index = page->index;
Chris Masond1310b22008-01-24 16:13:08 -05003990 /*
3991 * At this point we hold neither mapping->tree_lock nor
3992 * lock on the page itself: the page may be truncated or
3993 * invalidated (changing page->mapping to NULL), or even
3994 * swizzled back from swapper_space to tmpfs file
3995 * mapping
3996 */
Josef Bacikc8f2f242013-02-11 11:33:00 -05003997 if (!trylock_page(page)) {
3998 flush_fn(data);
3999 lock_page(page);
Chris Mason01d658f2011-11-01 10:08:06 -04004000 }
Chris Masond1310b22008-01-24 16:13:08 -05004001
4002 if (unlikely(page->mapping != mapping)) {
4003 unlock_page(page);
4004 continue;
4005 }
4006
4007 if (!wbc->range_cyclic && page->index > end) {
4008 done = 1;
4009 unlock_page(page);
4010 continue;
4011 }
4012
Chris Masond2c3f4f2008-11-19 12:44:22 -05004013 if (wbc->sync_mode != WB_SYNC_NONE) {
Chris Mason0e6bd952008-11-20 10:46:35 -05004014 if (PageWriteback(page))
4015 flush_fn(data);
Chris Masond1310b22008-01-24 16:13:08 -05004016 wait_on_page_writeback(page);
Chris Masond2c3f4f2008-11-19 12:44:22 -05004017 }
Chris Masond1310b22008-01-24 16:13:08 -05004018
4019 if (PageWriteback(page) ||
4020 !clear_page_dirty_for_io(page)) {
4021 unlock_page(page);
4022 continue;
4023 }
4024
4025 ret = (*writepage)(page, wbc, data);
4026
4027 if (unlikely(ret == AOP_WRITEPAGE_ACTIVATE)) {
4028 unlock_page(page);
4029 ret = 0;
4030 }
Liu Boa91326672016-03-07 16:56:21 -08004031 if (ret < 0) {
4032 /*
4033 * done_index is set past this page,
4034 * so media errors will not choke
4035 * background writeout for the entire
4036 * file. This has consequences for
4037 * range_cyclic semantics (ie. it may
4038 * not be suitable for data integrity
4039 * writeout).
4040 */
4041 done_index = page->index + 1;
4042 done = 1;
4043 break;
4044 }
Chris Masonf85d7d6c2009-09-18 16:03:16 -04004045
4046 /*
4047 * the filesystem may choose to bump up nr_to_write.
4048 * We have to make sure to honor the new nr_to_write
4049 * at any time
4050 */
4051 nr_to_write_done = wbc->nr_to_write <= 0;
Chris Masond1310b22008-01-24 16:13:08 -05004052 }
4053 pagevec_release(&pvec);
4054 cond_resched();
4055 }
Liu Bo894b36e2016-03-07 16:56:22 -08004056 if (!scanned && !done) {
Chris Masond1310b22008-01-24 16:13:08 -05004057 /*
4058 * We hit the last page and there is more work to be done: wrap
4059 * back to the start of the file
4060 */
4061 scanned = 1;
4062 index = 0;
4063 goto retry;
4064 }
Liu Boa91326672016-03-07 16:56:21 -08004065
4066 if (wbc->range_cyclic || (wbc->nr_to_write > 0 && range_whole))
4067 mapping->writeback_index = done_index;
4068
Josef Bacik7fd1a3f2012-06-27 17:18:41 -04004069 btrfs_add_delayed_iput(inode);
Liu Bo894b36e2016-03-07 16:56:22 -08004070 return ret;
Chris Masond1310b22008-01-24 16:13:08 -05004071}
Chris Masond1310b22008-01-24 16:13:08 -05004072
Chris Masonffbd5172009-04-20 15:50:09 -04004073static void flush_epd_write_bio(struct extent_page_data *epd)
4074{
4075 if (epd->bio) {
Jeff Mahoney355808c2011-10-03 23:23:14 -04004076 int ret;
4077
Mike Christie1f7ad752016-06-05 14:31:51 -05004078 bio_set_op_attrs(epd->bio, REQ_OP_WRITE,
4079 epd->sync_io ? WRITE_SYNC : 0);
Jeff Mahoney355808c2011-10-03 23:23:14 -04004080
Mike Christie1f7ad752016-06-05 14:31:51 -05004081 ret = submit_one_bio(epd->bio, 0, epd->bio_flags);
Jeff Mahoney79787ea2012-03-12 16:03:00 +01004082 BUG_ON(ret < 0); /* -ENOMEM */
Chris Masonffbd5172009-04-20 15:50:09 -04004083 epd->bio = NULL;
4084 }
4085}
4086
Chris Masond2c3f4f2008-11-19 12:44:22 -05004087static noinline void flush_write_bio(void *data)
4088{
4089 struct extent_page_data *epd = data;
Chris Masonffbd5172009-04-20 15:50:09 -04004090 flush_epd_write_bio(epd);
Chris Masond2c3f4f2008-11-19 12:44:22 -05004091}
4092
Chris Masond1310b22008-01-24 16:13:08 -05004093int extent_write_full_page(struct extent_io_tree *tree, struct page *page,
4094 get_extent_t *get_extent,
4095 struct writeback_control *wbc)
4096{
4097 int ret;
Chris Masond1310b22008-01-24 16:13:08 -05004098 struct extent_page_data epd = {
4099 .bio = NULL,
4100 .tree = tree,
4101 .get_extent = get_extent,
Chris Mason771ed682008-11-06 22:02:51 -05004102 .extent_locked = 0,
Chris Masonffbd5172009-04-20 15:50:09 -04004103 .sync_io = wbc->sync_mode == WB_SYNC_ALL,
Josef Bacikde0022b2012-09-25 14:25:58 -04004104 .bio_flags = 0,
Chris Masond1310b22008-01-24 16:13:08 -05004105 };
Chris Masond1310b22008-01-24 16:13:08 -05004106
Chris Masond1310b22008-01-24 16:13:08 -05004107 ret = __extent_writepage(page, wbc, &epd);
4108
Chris Masonffbd5172009-04-20 15:50:09 -04004109 flush_epd_write_bio(&epd);
Chris Masond1310b22008-01-24 16:13:08 -05004110 return ret;
4111}
Chris Masond1310b22008-01-24 16:13:08 -05004112
Chris Mason771ed682008-11-06 22:02:51 -05004113int extent_write_locked_range(struct extent_io_tree *tree, struct inode *inode,
4114 u64 start, u64 end, get_extent_t *get_extent,
4115 int mode)
4116{
4117 int ret = 0;
4118 struct address_space *mapping = inode->i_mapping;
4119 struct page *page;
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03004120 unsigned long nr_pages = (end - start + PAGE_SIZE) >>
4121 PAGE_SHIFT;
Chris Mason771ed682008-11-06 22:02:51 -05004122
4123 struct extent_page_data epd = {
4124 .bio = NULL,
4125 .tree = tree,
4126 .get_extent = get_extent,
4127 .extent_locked = 1,
Chris Masonffbd5172009-04-20 15:50:09 -04004128 .sync_io = mode == WB_SYNC_ALL,
Josef Bacikde0022b2012-09-25 14:25:58 -04004129 .bio_flags = 0,
Chris Mason771ed682008-11-06 22:02:51 -05004130 };
4131 struct writeback_control wbc_writepages = {
Chris Mason771ed682008-11-06 22:02:51 -05004132 .sync_mode = mode,
Chris Mason771ed682008-11-06 22:02:51 -05004133 .nr_to_write = nr_pages * 2,
4134 .range_start = start,
4135 .range_end = end + 1,
4136 };
4137
Chris Masond3977122009-01-05 21:25:51 -05004138 while (start <= end) {
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03004139 page = find_get_page(mapping, start >> PAGE_SHIFT);
Chris Mason771ed682008-11-06 22:02:51 -05004140 if (clear_page_dirty_for_io(page))
4141 ret = __extent_writepage(page, &wbc_writepages, &epd);
4142 else {
4143 if (tree->ops && tree->ops->writepage_end_io_hook)
4144 tree->ops->writepage_end_io_hook(page, start,
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03004145 start + PAGE_SIZE - 1,
Chris Mason771ed682008-11-06 22:02:51 -05004146 NULL, 1);
4147 unlock_page(page);
4148 }
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03004149 put_page(page);
4150 start += PAGE_SIZE;
Chris Mason771ed682008-11-06 22:02:51 -05004151 }
4152
Chris Masonffbd5172009-04-20 15:50:09 -04004153 flush_epd_write_bio(&epd);
Chris Mason771ed682008-11-06 22:02:51 -05004154 return ret;
4155}
Chris Masond1310b22008-01-24 16:13:08 -05004156
4157int extent_writepages(struct extent_io_tree *tree,
4158 struct address_space *mapping,
4159 get_extent_t *get_extent,
4160 struct writeback_control *wbc)
4161{
4162 int ret = 0;
4163 struct extent_page_data epd = {
4164 .bio = NULL,
4165 .tree = tree,
4166 .get_extent = get_extent,
Chris Mason771ed682008-11-06 22:02:51 -05004167 .extent_locked = 0,
Chris Masonffbd5172009-04-20 15:50:09 -04004168 .sync_io = wbc->sync_mode == WB_SYNC_ALL,
Josef Bacikde0022b2012-09-25 14:25:58 -04004169 .bio_flags = 0,
Chris Masond1310b22008-01-24 16:13:08 -05004170 };
4171
Chris Mason4bef0842008-09-08 11:18:08 -04004172 ret = extent_write_cache_pages(tree, mapping, wbc,
Chris Masond2c3f4f2008-11-19 12:44:22 -05004173 __extent_writepage, &epd,
4174 flush_write_bio);
Chris Masonffbd5172009-04-20 15:50:09 -04004175 flush_epd_write_bio(&epd);
Chris Masond1310b22008-01-24 16:13:08 -05004176 return ret;
4177}
Chris Masond1310b22008-01-24 16:13:08 -05004178
4179int extent_readpages(struct extent_io_tree *tree,
4180 struct address_space *mapping,
4181 struct list_head *pages, unsigned nr_pages,
4182 get_extent_t get_extent)
4183{
4184 struct bio *bio = NULL;
4185 unsigned page_idx;
Chris Masonc8b97812008-10-29 14:49:59 -04004186 unsigned long bio_flags = 0;
Liu Bo67c96842012-07-20 21:43:09 -06004187 struct page *pagepool[16];
4188 struct page *page;
Miao Xie125bac012013-07-25 19:22:37 +08004189 struct extent_map *em_cached = NULL;
Liu Bo67c96842012-07-20 21:43:09 -06004190 int nr = 0;
Filipe Manana808f80b2015-09-28 09:56:26 +01004191 u64 prev_em_start = (u64)-1;
Chris Masond1310b22008-01-24 16:13:08 -05004192
Chris Masond1310b22008-01-24 16:13:08 -05004193 for (page_idx = 0; page_idx < nr_pages; page_idx++) {
Liu Bo67c96842012-07-20 21:43:09 -06004194 page = list_entry(pages->prev, struct page, lru);
Chris Masond1310b22008-01-24 16:13:08 -05004195
4196 prefetchw(&page->flags);
4197 list_del(&page->lru);
Liu Bo67c96842012-07-20 21:43:09 -06004198 if (add_to_page_cache_lru(page, mapping,
Michal Hocko8a5c7432016-07-26 15:24:53 -07004199 page->index,
4200 readahead_gfp_mask(mapping))) {
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03004201 put_page(page);
Liu Bo67c96842012-07-20 21:43:09 -06004202 continue;
Chris Masond1310b22008-01-24 16:13:08 -05004203 }
Liu Bo67c96842012-07-20 21:43:09 -06004204
4205 pagepool[nr++] = page;
4206 if (nr < ARRAY_SIZE(pagepool))
4207 continue;
Miao Xie125bac012013-07-25 19:22:37 +08004208 __extent_readpages(tree, pagepool, nr, get_extent, &em_cached,
Mike Christie1f7ad752016-06-05 14:31:51 -05004209 &bio, 0, &bio_flags, &prev_em_start);
Liu Bo67c96842012-07-20 21:43:09 -06004210 nr = 0;
Chris Masond1310b22008-01-24 16:13:08 -05004211 }
Miao Xie99740902013-07-25 19:22:36 +08004212 if (nr)
Miao Xie125bac012013-07-25 19:22:37 +08004213 __extent_readpages(tree, pagepool, nr, get_extent, &em_cached,
Mike Christie1f7ad752016-06-05 14:31:51 -05004214 &bio, 0, &bio_flags, &prev_em_start);
Liu Bo67c96842012-07-20 21:43:09 -06004215
Miao Xie125bac012013-07-25 19:22:37 +08004216 if (em_cached)
4217 free_extent_map(em_cached);
4218
Chris Masond1310b22008-01-24 16:13:08 -05004219 BUG_ON(!list_empty(pages));
4220 if (bio)
Mike Christie1f7ad752016-06-05 14:31:51 -05004221 return submit_one_bio(bio, 0, bio_flags);
Chris Masond1310b22008-01-24 16:13:08 -05004222 return 0;
4223}
Chris Masond1310b22008-01-24 16:13:08 -05004224
4225/*
4226 * basic invalidatepage code, this waits on any locked or writeback
4227 * ranges corresponding to the page, and then deletes any extent state
4228 * records from the tree
4229 */
4230int extent_invalidatepage(struct extent_io_tree *tree,
4231 struct page *page, unsigned long offset)
4232{
Josef Bacik2ac55d42010-02-03 19:33:23 +00004233 struct extent_state *cached_state = NULL;
Miao Xie4eee4fa2012-12-21 09:17:45 +00004234 u64 start = page_offset(page);
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03004235 u64 end = start + PAGE_SIZE - 1;
Chris Masond1310b22008-01-24 16:13:08 -05004236 size_t blocksize = page->mapping->host->i_sb->s_blocksize;
4237
Qu Wenruofda28322013-02-26 08:10:22 +00004238 start += ALIGN(offset, blocksize);
Chris Masond1310b22008-01-24 16:13:08 -05004239 if (start > end)
4240 return 0;
4241
David Sterbaff13db42015-12-03 14:30:40 +01004242 lock_extent_bits(tree, start, end, &cached_state);
Chris Mason1edbb732009-09-02 13:24:36 -04004243 wait_on_page_writeback(page);
Chris Masond1310b22008-01-24 16:13:08 -05004244 clear_extent_bit(tree, start, end,
Josef Bacik32c00af2009-10-08 13:34:05 -04004245 EXTENT_LOCKED | EXTENT_DIRTY | EXTENT_DELALLOC |
4246 EXTENT_DO_ACCOUNTING,
Josef Bacik2ac55d42010-02-03 19:33:23 +00004247 1, 1, &cached_state, GFP_NOFS);
Chris Masond1310b22008-01-24 16:13:08 -05004248 return 0;
4249}
Chris Masond1310b22008-01-24 16:13:08 -05004250
4251/*
Chris Mason7b13b7b2008-04-18 10:29:50 -04004252 * a helper for releasepage, this tests for areas of the page that
4253 * are locked or under IO and drops the related state bits if it is safe
4254 * to drop the page.
4255 */
Eric Sandeen48a3b632013-04-25 20:41:01 +00004256static int try_release_extent_state(struct extent_map_tree *map,
4257 struct extent_io_tree *tree,
4258 struct page *page, gfp_t mask)
Chris Mason7b13b7b2008-04-18 10:29:50 -04004259{
Miao Xie4eee4fa2012-12-21 09:17:45 +00004260 u64 start = page_offset(page);
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03004261 u64 end = start + PAGE_SIZE - 1;
Chris Mason7b13b7b2008-04-18 10:29:50 -04004262 int ret = 1;
4263
Chris Mason211f90e2008-07-18 11:56:15 -04004264 if (test_range_bit(tree, start, end,
Chris Mason8b62b722009-09-02 16:53:46 -04004265 EXTENT_IOBITS, 0, NULL))
Chris Mason7b13b7b2008-04-18 10:29:50 -04004266 ret = 0;
4267 else {
4268 if ((mask & GFP_NOFS) == GFP_NOFS)
4269 mask = GFP_NOFS;
Chris Mason11ef1602009-09-23 20:28:46 -04004270 /*
4271 * at this point we can safely clear everything except the
4272 * locked bit and the nodatasum bit
4273 */
Chris Masone3f24cc2011-02-14 12:52:08 -05004274 ret = clear_extent_bit(tree, start, end,
Chris Mason11ef1602009-09-23 20:28:46 -04004275 ~(EXTENT_LOCKED | EXTENT_NODATASUM),
4276 0, 0, NULL, mask);
Chris Masone3f24cc2011-02-14 12:52:08 -05004277
4278 /* if clear_extent_bit failed for enomem reasons,
4279 * we can't allow the release to continue.
4280 */
4281 if (ret < 0)
4282 ret = 0;
4283 else
4284 ret = 1;
Chris Mason7b13b7b2008-04-18 10:29:50 -04004285 }
4286 return ret;
4287}
Chris Mason7b13b7b2008-04-18 10:29:50 -04004288
4289/*
Chris Masond1310b22008-01-24 16:13:08 -05004290 * a helper for releasepage. As long as there are no locked extents
4291 * in the range corresponding to the page, both state records and extent
4292 * map records are removed
4293 */
4294int try_release_extent_mapping(struct extent_map_tree *map,
Chris Mason70dec802008-01-29 09:59:12 -05004295 struct extent_io_tree *tree, struct page *page,
4296 gfp_t mask)
Chris Masond1310b22008-01-24 16:13:08 -05004297{
4298 struct extent_map *em;
Miao Xie4eee4fa2012-12-21 09:17:45 +00004299 u64 start = page_offset(page);
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03004300 u64 end = start + PAGE_SIZE - 1;
Filipe Mananab2486a82018-07-12 01:36:43 +01004301 struct btrfs_inode *btrfs_inode = BTRFS_I(page->mapping->host);
Chris Mason7b13b7b2008-04-18 10:29:50 -04004302
Mel Gormand0164ad2015-11-06 16:28:21 -08004303 if (gfpflags_allow_blocking(mask) &&
Byongho Leeee221842015-12-15 01:42:10 +09004304 page->mapping->host->i_size > SZ_16M) {
Yan39b56372008-02-15 10:40:50 -05004305 u64 len;
Chris Mason70dec802008-01-29 09:59:12 -05004306 while (start <= end) {
Yan39b56372008-02-15 10:40:50 -05004307 len = end - start + 1;
Chris Mason890871b2009-09-02 16:24:52 -04004308 write_lock(&map->lock);
Yan39b56372008-02-15 10:40:50 -05004309 em = lookup_extent_mapping(map, start, len);
Tsutomu Itoh285190d2012-02-16 16:23:58 +09004310 if (!em) {
Chris Mason890871b2009-09-02 16:24:52 -04004311 write_unlock(&map->lock);
Chris Mason70dec802008-01-29 09:59:12 -05004312 break;
4313 }
Chris Mason7f3c74f2008-07-18 12:01:11 -04004314 if (test_bit(EXTENT_FLAG_PINNED, &em->flags) ||
4315 em->start != start) {
Chris Mason890871b2009-09-02 16:24:52 -04004316 write_unlock(&map->lock);
Chris Mason70dec802008-01-29 09:59:12 -05004317 free_extent_map(em);
4318 break;
4319 }
4320 if (!test_range_bit(tree, em->start,
4321 extent_map_end(em) - 1,
Chris Mason8b62b722009-09-02 16:53:46 -04004322 EXTENT_LOCKED | EXTENT_WRITEBACK,
Chris Mason9655d292009-09-02 15:22:30 -04004323 0, NULL)) {
Filipe Mananab2486a82018-07-12 01:36:43 +01004324 set_bit(BTRFS_INODE_NEEDS_FULL_SYNC,
4325 &btrfs_inode->runtime_flags);
Chris Mason70dec802008-01-29 09:59:12 -05004326 remove_extent_mapping(map, em);
4327 /* once for the rb tree */
4328 free_extent_map(em);
4329 }
4330 start = extent_map_end(em);
Chris Mason890871b2009-09-02 16:24:52 -04004331 write_unlock(&map->lock);
Chris Mason70dec802008-01-29 09:59:12 -05004332
4333 /* once for us */
Chris Masond1310b22008-01-24 16:13:08 -05004334 free_extent_map(em);
4335 }
Chris Masond1310b22008-01-24 16:13:08 -05004336 }
Chris Mason7b13b7b2008-04-18 10:29:50 -04004337 return try_release_extent_state(map, tree, page, mask);
Chris Masond1310b22008-01-24 16:13:08 -05004338}
Chris Masond1310b22008-01-24 16:13:08 -05004339
Chris Masonec29ed52011-02-23 16:23:20 -05004340/*
4341 * helper function for fiemap, which doesn't want to see any holes.
4342 * This maps until we find something past 'last'
4343 */
4344static struct extent_map *get_extent_skip_holes(struct inode *inode,
4345 u64 offset,
4346 u64 last,
4347 get_extent_t *get_extent)
4348{
4349 u64 sectorsize = BTRFS_I(inode)->root->sectorsize;
4350 struct extent_map *em;
4351 u64 len;
4352
4353 if (offset >= last)
4354 return NULL;
4355
Dulshani Gunawardhana67871252013-10-31 10:33:04 +05304356 while (1) {
Chris Masonec29ed52011-02-23 16:23:20 -05004357 len = last - offset;
4358 if (len == 0)
4359 break;
Qu Wenruofda28322013-02-26 08:10:22 +00004360 len = ALIGN(len, sectorsize);
Chris Masonec29ed52011-02-23 16:23:20 -05004361 em = get_extent(inode, NULL, 0, offset, len, 0);
David Sterbac7040052011-04-19 18:00:01 +02004362 if (IS_ERR_OR_NULL(em))
Chris Masonec29ed52011-02-23 16:23:20 -05004363 return em;
4364
4365 /* if this isn't a hole return it */
4366 if (!test_bit(EXTENT_FLAG_VACANCY, &em->flags) &&
4367 em->block_start != EXTENT_MAP_HOLE) {
4368 return em;
4369 }
4370
4371 /* this is a hole, advance to the next extent */
4372 offset = extent_map_end(em);
4373 free_extent_map(em);
4374 if (offset >= last)
4375 break;
4376 }
4377 return NULL;
4378}
4379
Qu Wenruo4b1cfa22017-04-07 10:43:15 +08004380/*
4381 * To cache previous fiemap extent
4382 *
4383 * Will be used for merging fiemap extent
4384 */
4385struct fiemap_cache {
4386 u64 offset;
4387 u64 phys;
4388 u64 len;
4389 u32 flags;
4390 bool cached;
4391};
4392
4393/*
4394 * Helper to submit fiemap extent.
4395 *
4396 * Will try to merge current fiemap extent specified by @offset, @phys,
4397 * @len and @flags with cached one.
4398 * And only when we fails to merge, cached one will be submitted as
4399 * fiemap extent.
4400 *
4401 * Return value is the same as fiemap_fill_next_extent().
4402 */
4403static int emit_fiemap_extent(struct fiemap_extent_info *fieinfo,
4404 struct fiemap_cache *cache,
4405 u64 offset, u64 phys, u64 len, u32 flags)
4406{
4407 int ret = 0;
4408
4409 if (!cache->cached)
4410 goto assign;
4411
4412 /*
4413 * Sanity check, extent_fiemap() should have ensured that new
4414 * fiemap extent won't overlap with cahced one.
4415 * Not recoverable.
4416 *
4417 * NOTE: Physical address can overlap, due to compression
4418 */
4419 if (cache->offset + cache->len > offset) {
4420 WARN_ON(1);
4421 return -EINVAL;
4422 }
4423
4424 /*
4425 * Only merges fiemap extents if
4426 * 1) Their logical addresses are continuous
4427 *
4428 * 2) Their physical addresses are continuous
4429 * So truly compressed (physical size smaller than logical size)
4430 * extents won't get merged with each other
4431 *
4432 * 3) Share same flags except FIEMAP_EXTENT_LAST
4433 * So regular extent won't get merged with prealloc extent
4434 */
4435 if (cache->offset + cache->len == offset &&
4436 cache->phys + cache->len == phys &&
4437 (cache->flags & ~FIEMAP_EXTENT_LAST) ==
4438 (flags & ~FIEMAP_EXTENT_LAST)) {
4439 cache->len += len;
4440 cache->flags |= flags;
4441 goto try_submit_last;
4442 }
4443
4444 /* Not mergeable, need to submit cached one */
4445 ret = fiemap_fill_next_extent(fieinfo, cache->offset, cache->phys,
4446 cache->len, cache->flags);
4447 cache->cached = false;
4448 if (ret)
4449 return ret;
4450assign:
4451 cache->cached = true;
4452 cache->offset = offset;
4453 cache->phys = phys;
4454 cache->len = len;
4455 cache->flags = flags;
4456try_submit_last:
4457 if (cache->flags & FIEMAP_EXTENT_LAST) {
4458 ret = fiemap_fill_next_extent(fieinfo, cache->offset,
4459 cache->phys, cache->len, cache->flags);
4460 cache->cached = false;
4461 }
4462 return ret;
4463}
4464
4465/*
Qu Wenruode5f88f2017-06-22 10:01:21 +08004466 * Emit last fiemap cache
Qu Wenruo4b1cfa22017-04-07 10:43:15 +08004467 *
Qu Wenruode5f88f2017-06-22 10:01:21 +08004468 * The last fiemap cache may still be cached in the following case:
4469 * 0 4k 8k
4470 * |<- Fiemap range ->|
4471 * |<------------ First extent ----------->|
4472 *
4473 * In this case, the first extent range will be cached but not emitted.
4474 * So we must emit it before ending extent_fiemap().
Qu Wenruo4b1cfa22017-04-07 10:43:15 +08004475 */
Qu Wenruode5f88f2017-06-22 10:01:21 +08004476static int emit_last_fiemap_cache(struct btrfs_fs_info *fs_info,
4477 struct fiemap_extent_info *fieinfo,
4478 struct fiemap_cache *cache)
Qu Wenruo4b1cfa22017-04-07 10:43:15 +08004479{
4480 int ret;
4481
4482 if (!cache->cached)
4483 return 0;
4484
Qu Wenruo4b1cfa22017-04-07 10:43:15 +08004485 ret = fiemap_fill_next_extent(fieinfo, cache->offset, cache->phys,
4486 cache->len, cache->flags);
4487 cache->cached = false;
4488 if (ret > 0)
4489 ret = 0;
4490 return ret;
4491}
4492
Yehuda Sadeh1506fcc2009-01-21 14:39:14 -05004493int extent_fiemap(struct inode *inode, struct fiemap_extent_info *fieinfo,
4494 __u64 start, __u64 len, get_extent_t *get_extent)
4495{
Josef Bacik975f84f2010-11-23 19:36:57 +00004496 int ret = 0;
Yehuda Sadeh1506fcc2009-01-21 14:39:14 -05004497 u64 off = start;
4498 u64 max = start + len;
4499 u32 flags = 0;
Josef Bacik975f84f2010-11-23 19:36:57 +00004500 u32 found_type;
4501 u64 last;
Chris Masonec29ed52011-02-23 16:23:20 -05004502 u64 last_for_get_extent = 0;
Yehuda Sadeh1506fcc2009-01-21 14:39:14 -05004503 u64 disko = 0;
Chris Masonec29ed52011-02-23 16:23:20 -05004504 u64 isize = i_size_read(inode);
Josef Bacik975f84f2010-11-23 19:36:57 +00004505 struct btrfs_key found_key;
Yehuda Sadeh1506fcc2009-01-21 14:39:14 -05004506 struct extent_map *em = NULL;
Josef Bacik2ac55d42010-02-03 19:33:23 +00004507 struct extent_state *cached_state = NULL;
Josef Bacik975f84f2010-11-23 19:36:57 +00004508 struct btrfs_path *path;
Josef Bacikdc046b12014-09-10 16:20:45 -04004509 struct btrfs_root *root = BTRFS_I(inode)->root;
Qu Wenruo4b1cfa22017-04-07 10:43:15 +08004510 struct fiemap_cache cache = { 0 };
Yehuda Sadeh1506fcc2009-01-21 14:39:14 -05004511 int end = 0;
Chris Masonec29ed52011-02-23 16:23:20 -05004512 u64 em_start = 0;
4513 u64 em_len = 0;
4514 u64 em_end = 0;
Yehuda Sadeh1506fcc2009-01-21 14:39:14 -05004515
4516 if (len == 0)
4517 return -EINVAL;
4518
Josef Bacik975f84f2010-11-23 19:36:57 +00004519 path = btrfs_alloc_path();
4520 if (!path)
4521 return -ENOMEM;
4522 path->leave_spinning = 1;
4523
Qu Wenruo2c919432014-07-18 09:55:43 +08004524 start = round_down(start, BTRFS_I(inode)->root->sectorsize);
4525 len = round_up(max, BTRFS_I(inode)->root->sectorsize) - start;
Josef Bacik4d479cf2011-11-17 11:34:31 -05004526
Chris Masonec29ed52011-02-23 16:23:20 -05004527 /*
4528 * lookup the last file extent. We're not using i_size here
4529 * because there might be preallocation past i_size
4530 */
Josef Bacikdc046b12014-09-10 16:20:45 -04004531 ret = btrfs_lookup_file_extent(NULL, root, path, btrfs_ino(inode), -1,
4532 0);
Josef Bacik975f84f2010-11-23 19:36:57 +00004533 if (ret < 0) {
4534 btrfs_free_path(path);
4535 return ret;
Liu Bo2d324f52016-05-17 17:21:48 -07004536 } else {
4537 WARN_ON(!ret);
4538 if (ret == 1)
4539 ret = 0;
Josef Bacik975f84f2010-11-23 19:36:57 +00004540 }
Liu Bo2d324f52016-05-17 17:21:48 -07004541
Josef Bacik975f84f2010-11-23 19:36:57 +00004542 path->slots[0]--;
Josef Bacik975f84f2010-11-23 19:36:57 +00004543 btrfs_item_key_to_cpu(path->nodes[0], &found_key, path->slots[0]);
David Sterba962a2982014-06-04 18:41:45 +02004544 found_type = found_key.type;
Josef Bacik975f84f2010-11-23 19:36:57 +00004545
Chris Masonec29ed52011-02-23 16:23:20 -05004546 /* No extents, but there might be delalloc bits */
Li Zefan33345d012011-04-20 10:31:50 +08004547 if (found_key.objectid != btrfs_ino(inode) ||
Josef Bacik975f84f2010-11-23 19:36:57 +00004548 found_type != BTRFS_EXTENT_DATA_KEY) {
Chris Masonec29ed52011-02-23 16:23:20 -05004549 /* have to trust i_size as the end */
4550 last = (u64)-1;
4551 last_for_get_extent = isize;
4552 } else {
4553 /*
4554 * remember the start of the last extent. There are a
4555 * bunch of different factors that go into the length of the
4556 * extent, so its much less complex to remember where it started
4557 */
4558 last = found_key.offset;
4559 last_for_get_extent = last + 1;
Josef Bacik975f84f2010-11-23 19:36:57 +00004560 }
Liu Bofe09e162013-09-22 12:54:23 +08004561 btrfs_release_path(path);
Josef Bacik975f84f2010-11-23 19:36:57 +00004562
Chris Masonec29ed52011-02-23 16:23:20 -05004563 /*
4564 * we might have some extents allocated but more delalloc past those
4565 * extents. so, we trust isize unless the start of the last extent is
4566 * beyond isize
4567 */
4568 if (last < isize) {
4569 last = (u64)-1;
4570 last_for_get_extent = isize;
4571 }
4572
David Sterbaff13db42015-12-03 14:30:40 +01004573 lock_extent_bits(&BTRFS_I(inode)->io_tree, start, start + len - 1,
Jeff Mahoneyd0082372012-03-01 14:57:19 +01004574 &cached_state);
Chris Masonec29ed52011-02-23 16:23:20 -05004575
Josef Bacik4d479cf2011-11-17 11:34:31 -05004576 em = get_extent_skip_holes(inode, start, last_for_get_extent,
Chris Masonec29ed52011-02-23 16:23:20 -05004577 get_extent);
Yehuda Sadeh1506fcc2009-01-21 14:39:14 -05004578 if (!em)
4579 goto out;
4580 if (IS_ERR(em)) {
4581 ret = PTR_ERR(em);
4582 goto out;
4583 }
Josef Bacik975f84f2010-11-23 19:36:57 +00004584
Yehuda Sadeh1506fcc2009-01-21 14:39:14 -05004585 while (!end) {
Josef Bacikb76bb702013-07-05 13:52:51 -04004586 u64 offset_in_extent = 0;
Yehuda Sadeh1506fcc2009-01-21 14:39:14 -05004587
Chris Masonea8efc72011-03-08 11:54:40 -05004588 /* break if the extent we found is outside the range */
4589 if (em->start >= max || extent_map_end(em) < off)
4590 break;
4591
4592 /*
4593 * get_extent may return an extent that starts before our
4594 * requested range. We have to make sure the ranges
4595 * we return to fiemap always move forward and don't
4596 * overlap, so adjust the offsets here
4597 */
4598 em_start = max(em->start, off);
4599
4600 /*
4601 * record the offset from the start of the extent
Josef Bacikb76bb702013-07-05 13:52:51 -04004602 * for adjusting the disk offset below. Only do this if the
4603 * extent isn't compressed since our in ram offset may be past
4604 * what we have actually allocated on disk.
Chris Masonea8efc72011-03-08 11:54:40 -05004605 */
Josef Bacikb76bb702013-07-05 13:52:51 -04004606 if (!test_bit(EXTENT_FLAG_COMPRESSED, &em->flags))
4607 offset_in_extent = em_start - em->start;
Chris Masonec29ed52011-02-23 16:23:20 -05004608 em_end = extent_map_end(em);
Chris Masonea8efc72011-03-08 11:54:40 -05004609 em_len = em_end - em_start;
Yehuda Sadeh1506fcc2009-01-21 14:39:14 -05004610 disko = 0;
4611 flags = 0;
4612
Chris Masonea8efc72011-03-08 11:54:40 -05004613 /*
4614 * bump off for our next call to get_extent
4615 */
4616 off = extent_map_end(em);
4617 if (off >= max)
4618 end = 1;
4619
Heiko Carstens93dbfad2009-04-03 10:33:45 -04004620 if (em->block_start == EXTENT_MAP_LAST_BYTE) {
Yehuda Sadeh1506fcc2009-01-21 14:39:14 -05004621 end = 1;
4622 flags |= FIEMAP_EXTENT_LAST;
Heiko Carstens93dbfad2009-04-03 10:33:45 -04004623 } else if (em->block_start == EXTENT_MAP_INLINE) {
Yehuda Sadeh1506fcc2009-01-21 14:39:14 -05004624 flags |= (FIEMAP_EXTENT_DATA_INLINE |
4625 FIEMAP_EXTENT_NOT_ALIGNED);
Heiko Carstens93dbfad2009-04-03 10:33:45 -04004626 } else if (em->block_start == EXTENT_MAP_DELALLOC) {
Yehuda Sadeh1506fcc2009-01-21 14:39:14 -05004627 flags |= (FIEMAP_EXTENT_DELALLOC |
4628 FIEMAP_EXTENT_UNKNOWN);
Josef Bacikdc046b12014-09-10 16:20:45 -04004629 } else if (fieinfo->fi_extents_max) {
Lu Fengqiafce7722016-06-13 09:36:46 +08004630 struct btrfs_trans_handle *trans;
4631
Josef Bacikdc046b12014-09-10 16:20:45 -04004632 u64 bytenr = em->block_start -
4633 (em->start - em->orig_start);
Liu Bofe09e162013-09-22 12:54:23 +08004634
Chris Masonea8efc72011-03-08 11:54:40 -05004635 disko = em->block_start + offset_in_extent;
Liu Bofe09e162013-09-22 12:54:23 +08004636
4637 /*
Lu Fengqiafce7722016-06-13 09:36:46 +08004638 * We need a trans handle to get delayed refs
4639 */
4640 trans = btrfs_join_transaction(root);
4641 /*
4642 * It's OK if we can't start a trans we can still check
4643 * from commit_root
4644 */
4645 if (IS_ERR(trans))
4646 trans = NULL;
4647
4648 /*
Liu Bofe09e162013-09-22 12:54:23 +08004649 * As btrfs supports shared space, this information
4650 * can be exported to userspace tools via
Josef Bacikdc046b12014-09-10 16:20:45 -04004651 * flag FIEMAP_EXTENT_SHARED. If fi_extents_max == 0
4652 * then we're just getting a count and we can skip the
4653 * lookup stuff.
Liu Bofe09e162013-09-22 12:54:23 +08004654 */
Lu Fengqiafce7722016-06-13 09:36:46 +08004655 ret = btrfs_check_shared(trans, root->fs_info,
Josef Bacikdc046b12014-09-10 16:20:45 -04004656 root->objectid,
4657 btrfs_ino(inode), bytenr);
Lu Fengqiafce7722016-06-13 09:36:46 +08004658 if (trans)
4659 btrfs_end_transaction(trans, root);
Josef Bacikdc046b12014-09-10 16:20:45 -04004660 if (ret < 0)
Liu Bofe09e162013-09-22 12:54:23 +08004661 goto out_free;
Josef Bacikdc046b12014-09-10 16:20:45 -04004662 if (ret)
Liu Bofe09e162013-09-22 12:54:23 +08004663 flags |= FIEMAP_EXTENT_SHARED;
Josef Bacikdc046b12014-09-10 16:20:45 -04004664 ret = 0;
Yehuda Sadeh1506fcc2009-01-21 14:39:14 -05004665 }
4666 if (test_bit(EXTENT_FLAG_COMPRESSED, &em->flags))
4667 flags |= FIEMAP_EXTENT_ENCODED;
Josef Bacik0d2b2372015-05-19 10:44:04 -04004668 if (test_bit(EXTENT_FLAG_PREALLOC, &em->flags))
4669 flags |= FIEMAP_EXTENT_UNWRITTEN;
Yehuda Sadeh1506fcc2009-01-21 14:39:14 -05004670
Yehuda Sadeh1506fcc2009-01-21 14:39:14 -05004671 free_extent_map(em);
4672 em = NULL;
Chris Masonec29ed52011-02-23 16:23:20 -05004673 if ((em_start >= last) || em_len == (u64)-1 ||
4674 (last == (u64)-1 && isize <= em_end)) {
Yehuda Sadeh1506fcc2009-01-21 14:39:14 -05004675 flags |= FIEMAP_EXTENT_LAST;
4676 end = 1;
4677 }
4678
Chris Masonec29ed52011-02-23 16:23:20 -05004679 /* now scan forward to see if this is really the last extent. */
4680 em = get_extent_skip_holes(inode, off, last_for_get_extent,
4681 get_extent);
4682 if (IS_ERR(em)) {
4683 ret = PTR_ERR(em);
4684 goto out;
4685 }
4686 if (!em) {
Josef Bacik975f84f2010-11-23 19:36:57 +00004687 flags |= FIEMAP_EXTENT_LAST;
4688 end = 1;
4689 }
Qu Wenruo4b1cfa22017-04-07 10:43:15 +08004690 ret = emit_fiemap_extent(fieinfo, &cache, em_start, disko,
4691 em_len, flags);
Chengyu Song26e726a2015-03-24 18:12:56 -04004692 if (ret) {
4693 if (ret == 1)
4694 ret = 0;
Chris Masonec29ed52011-02-23 16:23:20 -05004695 goto out_free;
Chengyu Song26e726a2015-03-24 18:12:56 -04004696 }
Yehuda Sadeh1506fcc2009-01-21 14:39:14 -05004697 }
4698out_free:
Qu Wenruo4b1cfa22017-04-07 10:43:15 +08004699 if (!ret)
Qu Wenruode5f88f2017-06-22 10:01:21 +08004700 ret = emit_last_fiemap_cache(root->fs_info, fieinfo, &cache);
Yehuda Sadeh1506fcc2009-01-21 14:39:14 -05004701 free_extent_map(em);
4702out:
Liu Bofe09e162013-09-22 12:54:23 +08004703 btrfs_free_path(path);
Liu Boa52f4cd2013-05-01 16:23:41 +00004704 unlock_extent_cached(&BTRFS_I(inode)->io_tree, start, start + len - 1,
Josef Bacik2ac55d42010-02-03 19:33:23 +00004705 &cached_state, GFP_NOFS);
Yehuda Sadeh1506fcc2009-01-21 14:39:14 -05004706 return ret;
4707}
4708
Chris Mason727011e2010-08-06 13:21:20 -04004709static void __free_extent_buffer(struct extent_buffer *eb)
4710{
Eric Sandeen6d49ba12013-04-22 16:12:31 +00004711 btrfs_leak_debug_del(&eb->leak_list);
Chris Mason727011e2010-08-06 13:21:20 -04004712 kmem_cache_free(extent_buffer_cache, eb);
4713}
4714
Josef Bacika26e8c92014-03-28 17:07:27 -04004715int extent_buffer_under_io(struct extent_buffer *eb)
Chris Masond1310b22008-01-24 16:13:08 -05004716{
Josef Bacik0b32f4b2012-03-13 09:38:00 -04004717 return (atomic_read(&eb->io_pages) ||
4718 test_bit(EXTENT_BUFFER_WRITEBACK, &eb->bflags) ||
4719 test_bit(EXTENT_BUFFER_DIRTY, &eb->bflags));
Chris Masond1310b22008-01-24 16:13:08 -05004720}
4721
Miao Xie897ca6e2010-10-26 20:57:29 -04004722/*
4723 * Helper for releasing extent buffer page.
4724 */
David Sterbaa50924e2014-07-31 00:51:36 +02004725static void btrfs_release_extent_buffer_page(struct extent_buffer *eb)
Miao Xie897ca6e2010-10-26 20:57:29 -04004726{
4727 unsigned long index;
4728 struct page *page;
Jan Schmidt815a51c2012-05-16 17:00:02 +02004729 int mapped = !test_bit(EXTENT_BUFFER_DUMMY, &eb->bflags);
Miao Xie897ca6e2010-10-26 20:57:29 -04004730
Josef Bacik0b32f4b2012-03-13 09:38:00 -04004731 BUG_ON(extent_buffer_under_io(eb));
Miao Xie897ca6e2010-10-26 20:57:29 -04004732
David Sterbaa50924e2014-07-31 00:51:36 +02004733 index = num_extent_pages(eb->start, eb->len);
4734 if (index == 0)
Miao Xie897ca6e2010-10-26 20:57:29 -04004735 return;
4736
4737 do {
4738 index--;
David Sterbafb85fc92014-07-31 01:03:53 +02004739 page = eb->pages[index];
Forrest Liu5d2361d2015-02-09 17:31:45 +08004740 if (!page)
4741 continue;
4742 if (mapped)
Josef Bacik4f2de97a2012-03-07 16:20:05 -05004743 spin_lock(&page->mapping->private_lock);
Forrest Liu5d2361d2015-02-09 17:31:45 +08004744 /*
4745 * We do this since we'll remove the pages after we've
4746 * removed the eb from the radix tree, so we could race
4747 * and have this page now attached to the new eb. So
4748 * only clear page_private if it's still connected to
4749 * this eb.
4750 */
4751 if (PagePrivate(page) &&
4752 page->private == (unsigned long)eb) {
4753 BUG_ON(test_bit(EXTENT_BUFFER_DIRTY, &eb->bflags));
4754 BUG_ON(PageDirty(page));
4755 BUG_ON(PageWriteback(page));
Josef Bacik4f2de97a2012-03-07 16:20:05 -05004756 /*
Forrest Liu5d2361d2015-02-09 17:31:45 +08004757 * We need to make sure we haven't be attached
4758 * to a new eb.
Josef Bacik4f2de97a2012-03-07 16:20:05 -05004759 */
Forrest Liu5d2361d2015-02-09 17:31:45 +08004760 ClearPagePrivate(page);
4761 set_page_private(page, 0);
4762 /* One for the page private */
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03004763 put_page(page);
Josef Bacik4f2de97a2012-03-07 16:20:05 -05004764 }
Forrest Liu5d2361d2015-02-09 17:31:45 +08004765
4766 if (mapped)
4767 spin_unlock(&page->mapping->private_lock);
4768
Nicholas D Steeves01327612016-05-19 21:18:45 -04004769 /* One for when we allocated the page */
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03004770 put_page(page);
David Sterbaa50924e2014-07-31 00:51:36 +02004771 } while (index != 0);
Miao Xie897ca6e2010-10-26 20:57:29 -04004772}
4773
4774/*
4775 * Helper for releasing the extent buffer.
4776 */
4777static inline void btrfs_release_extent_buffer(struct extent_buffer *eb)
4778{
David Sterbaa50924e2014-07-31 00:51:36 +02004779 btrfs_release_extent_buffer_page(eb);
Miao Xie897ca6e2010-10-26 20:57:29 -04004780 __free_extent_buffer(eb);
4781}
4782
Josef Bacikf28491e2013-12-16 13:24:27 -05004783static struct extent_buffer *
4784__alloc_extent_buffer(struct btrfs_fs_info *fs_info, u64 start,
David Sterba23d79d82014-06-15 02:55:29 +02004785 unsigned long len)
Josef Bacikdb7f3432013-08-07 14:54:37 -04004786{
4787 struct extent_buffer *eb = NULL;
4788
Michal Hockod1b5c562015-08-19 14:17:40 +02004789 eb = kmem_cache_zalloc(extent_buffer_cache, GFP_NOFS|__GFP_NOFAIL);
Josef Bacikdb7f3432013-08-07 14:54:37 -04004790 eb->start = start;
4791 eb->len = len;
Josef Bacikf28491e2013-12-16 13:24:27 -05004792 eb->fs_info = fs_info;
Josef Bacikdb7f3432013-08-07 14:54:37 -04004793 eb->bflags = 0;
4794 rwlock_init(&eb->lock);
4795 atomic_set(&eb->write_locks, 0);
4796 atomic_set(&eb->read_locks, 0);
4797 atomic_set(&eb->blocking_readers, 0);
4798 atomic_set(&eb->blocking_writers, 0);
4799 atomic_set(&eb->spinning_readers, 0);
4800 atomic_set(&eb->spinning_writers, 0);
4801 eb->lock_nested = 0;
4802 init_waitqueue_head(&eb->write_lock_wq);
4803 init_waitqueue_head(&eb->read_lock_wq);
4804
4805 btrfs_leak_debug_add(&eb->leak_list, &buffers);
4806
4807 spin_lock_init(&eb->refs_lock);
4808 atomic_set(&eb->refs, 1);
4809 atomic_set(&eb->io_pages, 0);
4810
4811 /*
4812 * Sanity checks, currently the maximum is 64k covered by 16x 4k pages
4813 */
4814 BUILD_BUG_ON(BTRFS_MAX_METADATA_BLOCKSIZE
4815 > MAX_INLINE_EXTENT_BUFFER_SIZE);
4816 BUG_ON(len > MAX_INLINE_EXTENT_BUFFER_SIZE);
4817
4818 return eb;
4819}
4820
4821struct extent_buffer *btrfs_clone_extent_buffer(struct extent_buffer *src)
4822{
4823 unsigned long i;
4824 struct page *p;
4825 struct extent_buffer *new;
4826 unsigned long num_pages = num_extent_pages(src->start, src->len);
4827
David Sterba3f556f72014-06-15 03:20:26 +02004828 new = __alloc_extent_buffer(src->fs_info, src->start, src->len);
Josef Bacikdb7f3432013-08-07 14:54:37 -04004829 if (new == NULL)
4830 return NULL;
4831
4832 for (i = 0; i < num_pages; i++) {
Josef Bacik9ec72672013-08-07 16:57:23 -04004833 p = alloc_page(GFP_NOFS);
Josef Bacikdb7f3432013-08-07 14:54:37 -04004834 if (!p) {
4835 btrfs_release_extent_buffer(new);
4836 return NULL;
4837 }
4838 attach_extent_buffer_page(new, p);
4839 WARN_ON(PageDirty(p));
4840 SetPageUptodate(p);
4841 new->pages[i] = p;
4842 }
4843
4844 copy_extent_buffer(new, src, 0, 0, src->len);
4845 set_bit(EXTENT_BUFFER_UPTODATE, &new->bflags);
4846 set_bit(EXTENT_BUFFER_DUMMY, &new->bflags);
4847
4848 return new;
4849}
4850
Omar Sandoval0f331222015-09-29 20:50:31 -07004851struct extent_buffer *__alloc_dummy_extent_buffer(struct btrfs_fs_info *fs_info,
4852 u64 start, unsigned long len)
Josef Bacikdb7f3432013-08-07 14:54:37 -04004853{
4854 struct extent_buffer *eb;
David Sterba3f556f72014-06-15 03:20:26 +02004855 unsigned long num_pages;
Josef Bacikdb7f3432013-08-07 14:54:37 -04004856 unsigned long i;
4857
Omar Sandoval0f331222015-09-29 20:50:31 -07004858 num_pages = num_extent_pages(start, len);
David Sterba3f556f72014-06-15 03:20:26 +02004859
4860 eb = __alloc_extent_buffer(fs_info, start, len);
Josef Bacikdb7f3432013-08-07 14:54:37 -04004861 if (!eb)
4862 return NULL;
4863
4864 for (i = 0; i < num_pages; i++) {
Josef Bacik9ec72672013-08-07 16:57:23 -04004865 eb->pages[i] = alloc_page(GFP_NOFS);
Josef Bacikdb7f3432013-08-07 14:54:37 -04004866 if (!eb->pages[i])
4867 goto err;
4868 }
4869 set_extent_buffer_uptodate(eb);
4870 btrfs_set_header_nritems(eb, 0);
4871 set_bit(EXTENT_BUFFER_DUMMY, &eb->bflags);
4872
4873 return eb;
4874err:
4875 for (; i > 0; i--)
4876 __free_page(eb->pages[i - 1]);
4877 __free_extent_buffer(eb);
4878 return NULL;
4879}
4880
Omar Sandoval0f331222015-09-29 20:50:31 -07004881struct extent_buffer *alloc_dummy_extent_buffer(struct btrfs_fs_info *fs_info,
Feifei Xub9ef22d2016-06-01 19:18:25 +08004882 u64 start, u32 nodesize)
Omar Sandoval0f331222015-09-29 20:50:31 -07004883{
4884 unsigned long len;
4885
4886 if (!fs_info) {
4887 /*
4888 * Called only from tests that don't always have a fs_info
Feifei Xub9ef22d2016-06-01 19:18:25 +08004889 * available
Omar Sandoval0f331222015-09-29 20:50:31 -07004890 */
Feifei Xub9ef22d2016-06-01 19:18:25 +08004891 len = nodesize;
Omar Sandoval0f331222015-09-29 20:50:31 -07004892 } else {
4893 len = fs_info->tree_root->nodesize;
4894 }
4895
4896 return __alloc_dummy_extent_buffer(fs_info, start, len);
4897}
4898
Josef Bacik0b32f4b2012-03-13 09:38:00 -04004899static void check_buffer_tree_ref(struct extent_buffer *eb)
4900{
Chris Mason242e18c2013-01-29 17:49:37 -05004901 int refs;
Josef Bacik0b32f4b2012-03-13 09:38:00 -04004902 /* the ref bit is tricky. We have to make sure it is set
4903 * if we have the buffer dirty. Otherwise the
4904 * code to free a buffer can end up dropping a dirty
4905 * page
4906 *
4907 * Once the ref bit is set, it won't go away while the
4908 * buffer is dirty or in writeback, and it also won't
4909 * go away while we have the reference count on the
4910 * eb bumped.
4911 *
4912 * We can't just set the ref bit without bumping the
4913 * ref on the eb because free_extent_buffer might
4914 * see the ref bit and try to clear it. If this happens
4915 * free_extent_buffer might end up dropping our original
4916 * ref by mistake and freeing the page before we are able
4917 * to add one more ref.
4918 *
4919 * So bump the ref count first, then set the bit. If someone
4920 * beat us to it, drop the ref we added.
4921 */
Chris Mason242e18c2013-01-29 17:49:37 -05004922 refs = atomic_read(&eb->refs);
4923 if (refs >= 2 && test_bit(EXTENT_BUFFER_TREE_REF, &eb->bflags))
4924 return;
4925
Josef Bacik594831c2012-07-20 16:11:08 -04004926 spin_lock(&eb->refs_lock);
4927 if (!test_and_set_bit(EXTENT_BUFFER_TREE_REF, &eb->bflags))
Josef Bacik0b32f4b2012-03-13 09:38:00 -04004928 atomic_inc(&eb->refs);
Josef Bacik594831c2012-07-20 16:11:08 -04004929 spin_unlock(&eb->refs_lock);
Josef Bacik0b32f4b2012-03-13 09:38:00 -04004930}
4931
Mel Gorman2457aec2014-06-04 16:10:31 -07004932static void mark_extent_buffer_accessed(struct extent_buffer *eb,
4933 struct page *accessed)
Josef Bacik5df42352012-03-15 18:24:42 -04004934{
4935 unsigned long num_pages, i;
4936
Josef Bacik0b32f4b2012-03-13 09:38:00 -04004937 check_buffer_tree_ref(eb);
4938
Josef Bacik5df42352012-03-15 18:24:42 -04004939 num_pages = num_extent_pages(eb->start, eb->len);
4940 for (i = 0; i < num_pages; i++) {
David Sterbafb85fc92014-07-31 01:03:53 +02004941 struct page *p = eb->pages[i];
4942
Mel Gorman2457aec2014-06-04 16:10:31 -07004943 if (p != accessed)
4944 mark_page_accessed(p);
Josef Bacik5df42352012-03-15 18:24:42 -04004945 }
4946}
4947
Josef Bacikf28491e2013-12-16 13:24:27 -05004948struct extent_buffer *find_extent_buffer(struct btrfs_fs_info *fs_info,
4949 u64 start)
Chandra Seetharaman452c75c2013-10-07 10:45:25 -05004950{
4951 struct extent_buffer *eb;
4952
4953 rcu_read_lock();
Josef Bacikf28491e2013-12-16 13:24:27 -05004954 eb = radix_tree_lookup(&fs_info->buffer_radix,
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03004955 start >> PAGE_SHIFT);
Chandra Seetharaman452c75c2013-10-07 10:45:25 -05004956 if (eb && atomic_inc_not_zero(&eb->refs)) {
4957 rcu_read_unlock();
Filipe Manana062c19e2015-04-23 11:28:48 +01004958 /*
4959 * Lock our eb's refs_lock to avoid races with
4960 * free_extent_buffer. When we get our eb it might be flagged
4961 * with EXTENT_BUFFER_STALE and another task running
4962 * free_extent_buffer might have seen that flag set,
4963 * eb->refs == 2, that the buffer isn't under IO (dirty and
4964 * writeback flags not set) and it's still in the tree (flag
4965 * EXTENT_BUFFER_TREE_REF set), therefore being in the process
4966 * of decrementing the extent buffer's reference count twice.
4967 * So here we could race and increment the eb's reference count,
4968 * clear its stale flag, mark it as dirty and drop our reference
4969 * before the other task finishes executing free_extent_buffer,
4970 * which would later result in an attempt to free an extent
4971 * buffer that is dirty.
4972 */
4973 if (test_bit(EXTENT_BUFFER_STALE, &eb->bflags)) {
4974 spin_lock(&eb->refs_lock);
4975 spin_unlock(&eb->refs_lock);
4976 }
Mel Gorman2457aec2014-06-04 16:10:31 -07004977 mark_extent_buffer_accessed(eb, NULL);
Chandra Seetharaman452c75c2013-10-07 10:45:25 -05004978 return eb;
4979 }
4980 rcu_read_unlock();
4981
4982 return NULL;
4983}
4984
Josef Bacikfaa2dbf2014-05-07 17:06:09 -04004985#ifdef CONFIG_BTRFS_FS_RUN_SANITY_TESTS
4986struct extent_buffer *alloc_test_extent_buffer(struct btrfs_fs_info *fs_info,
Feifei Xub9ef22d2016-06-01 19:18:25 +08004987 u64 start, u32 nodesize)
Josef Bacikfaa2dbf2014-05-07 17:06:09 -04004988{
4989 struct extent_buffer *eb, *exists = NULL;
4990 int ret;
4991
4992 eb = find_extent_buffer(fs_info, start);
4993 if (eb)
4994 return eb;
Feifei Xub9ef22d2016-06-01 19:18:25 +08004995 eb = alloc_dummy_extent_buffer(fs_info, start, nodesize);
Josef Bacikfaa2dbf2014-05-07 17:06:09 -04004996 if (!eb)
4997 return NULL;
4998 eb->fs_info = fs_info;
4999again:
David Sterbae1860a72016-05-09 14:11:38 +02005000 ret = radix_tree_preload(GFP_NOFS);
Josef Bacikfaa2dbf2014-05-07 17:06:09 -04005001 if (ret)
5002 goto free_eb;
5003 spin_lock(&fs_info->buffer_lock);
5004 ret = radix_tree_insert(&fs_info->buffer_radix,
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03005005 start >> PAGE_SHIFT, eb);
Josef Bacikfaa2dbf2014-05-07 17:06:09 -04005006 spin_unlock(&fs_info->buffer_lock);
5007 radix_tree_preload_end();
5008 if (ret == -EEXIST) {
5009 exists = find_extent_buffer(fs_info, start);
5010 if (exists)
5011 goto free_eb;
5012 else
5013 goto again;
5014 }
5015 check_buffer_tree_ref(eb);
5016 set_bit(EXTENT_BUFFER_IN_TREE, &eb->bflags);
5017
5018 /*
5019 * We will free dummy extent buffer's if they come into
5020 * free_extent_buffer with a ref count of 2, but if we are using this we
5021 * want the buffers to stay in memory until we're done with them, so
5022 * bump the ref count again.
5023 */
5024 atomic_inc(&eb->refs);
5025 return eb;
5026free_eb:
5027 btrfs_release_extent_buffer(eb);
5028 return exists;
5029}
5030#endif
5031
Josef Bacikf28491e2013-12-16 13:24:27 -05005032struct extent_buffer *alloc_extent_buffer(struct btrfs_fs_info *fs_info,
David Sterbace3e6982014-06-15 03:00:04 +02005033 u64 start)
Chris Masond1310b22008-01-24 16:13:08 -05005034{
David Sterbace3e6982014-06-15 03:00:04 +02005035 unsigned long len = fs_info->tree_root->nodesize;
Chris Masond1310b22008-01-24 16:13:08 -05005036 unsigned long num_pages = num_extent_pages(start, len);
5037 unsigned long i;
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03005038 unsigned long index = start >> PAGE_SHIFT;
Chris Masond1310b22008-01-24 16:13:08 -05005039 struct extent_buffer *eb;
Chris Mason6af118ce2008-07-22 11:18:07 -04005040 struct extent_buffer *exists = NULL;
Chris Masond1310b22008-01-24 16:13:08 -05005041 struct page *p;
Josef Bacikf28491e2013-12-16 13:24:27 -05005042 struct address_space *mapping = fs_info->btree_inode->i_mapping;
Chris Masond1310b22008-01-24 16:13:08 -05005043 int uptodate = 1;
Miao Xie19fe0a82010-10-26 20:57:29 -04005044 int ret;
Chris Masond1310b22008-01-24 16:13:08 -05005045
Liu Boc871b0f2016-06-06 12:01:23 -07005046 if (!IS_ALIGNED(start, fs_info->tree_root->sectorsize)) {
5047 btrfs_err(fs_info, "bad tree block start %llu", start);
5048 return ERR_PTR(-EINVAL);
5049 }
5050
Josef Bacikf28491e2013-12-16 13:24:27 -05005051 eb = find_extent_buffer(fs_info, start);
Chandra Seetharaman452c75c2013-10-07 10:45:25 -05005052 if (eb)
Chris Mason6af118ce2008-07-22 11:18:07 -04005053 return eb;
Chris Mason6af118ce2008-07-22 11:18:07 -04005054
David Sterba23d79d82014-06-15 02:55:29 +02005055 eb = __alloc_extent_buffer(fs_info, start, len);
Peter2b114d12008-04-01 11:21:40 -04005056 if (!eb)
Liu Boc871b0f2016-06-06 12:01:23 -07005057 return ERR_PTR(-ENOMEM);
Chris Masond1310b22008-01-24 16:13:08 -05005058
Chris Mason727011e2010-08-06 13:21:20 -04005059 for (i = 0; i < num_pages; i++, index++) {
Michal Hockod1b5c562015-08-19 14:17:40 +02005060 p = find_or_create_page(mapping, index, GFP_NOFS|__GFP_NOFAIL);
Liu Boc871b0f2016-06-06 12:01:23 -07005061 if (!p) {
5062 exists = ERR_PTR(-ENOMEM);
Chris Mason6af118ce2008-07-22 11:18:07 -04005063 goto free_eb;
Liu Boc871b0f2016-06-06 12:01:23 -07005064 }
Josef Bacik4f2de97a2012-03-07 16:20:05 -05005065
5066 spin_lock(&mapping->private_lock);
5067 if (PagePrivate(p)) {
5068 /*
5069 * We could have already allocated an eb for this page
5070 * and attached one so lets see if we can get a ref on
5071 * the existing eb, and if we can we know it's good and
5072 * we can just return that one, else we know we can just
5073 * overwrite page->private.
5074 */
5075 exists = (struct extent_buffer *)p->private;
5076 if (atomic_inc_not_zero(&exists->refs)) {
5077 spin_unlock(&mapping->private_lock);
5078 unlock_page(p);
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03005079 put_page(p);
Mel Gorman2457aec2014-06-04 16:10:31 -07005080 mark_extent_buffer_accessed(exists, p);
Josef Bacik4f2de97a2012-03-07 16:20:05 -05005081 goto free_eb;
5082 }
Omar Sandoval5ca64f42015-02-24 02:47:05 -08005083 exists = NULL;
Josef Bacik4f2de97a2012-03-07 16:20:05 -05005084
Josef Bacik0b32f4b2012-03-13 09:38:00 -04005085 /*
Josef Bacik4f2de97a2012-03-07 16:20:05 -05005086 * Do this so attach doesn't complain and we need to
5087 * drop the ref the old guy had.
5088 */
5089 ClearPagePrivate(p);
Josef Bacik0b32f4b2012-03-13 09:38:00 -04005090 WARN_ON(PageDirty(p));
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03005091 put_page(p);
Chris Masond1310b22008-01-24 16:13:08 -05005092 }
Josef Bacik4f2de97a2012-03-07 16:20:05 -05005093 attach_extent_buffer_page(eb, p);
5094 spin_unlock(&mapping->private_lock);
Josef Bacik0b32f4b2012-03-13 09:38:00 -04005095 WARN_ON(PageDirty(p));
Chris Mason727011e2010-08-06 13:21:20 -04005096 eb->pages[i] = p;
Chris Masond1310b22008-01-24 16:13:08 -05005097 if (!PageUptodate(p))
5098 uptodate = 0;
Chris Masoneb14ab82011-02-10 12:35:00 -05005099
5100 /*
5101 * see below about how we avoid a nasty race with release page
5102 * and why we unlock later
5103 */
Chris Masond1310b22008-01-24 16:13:08 -05005104 }
5105 if (uptodate)
Chris Masonb4ce94d2009-02-04 09:25:08 -05005106 set_bit(EXTENT_BUFFER_UPTODATE, &eb->bflags);
Josef Bacik115391d2012-03-09 09:51:43 -05005107again:
David Sterbae1860a72016-05-09 14:11:38 +02005108 ret = radix_tree_preload(GFP_NOFS);
Liu Boc871b0f2016-06-06 12:01:23 -07005109 if (ret) {
5110 exists = ERR_PTR(ret);
Miao Xie19fe0a82010-10-26 20:57:29 -04005111 goto free_eb;
Liu Boc871b0f2016-06-06 12:01:23 -07005112 }
Miao Xie19fe0a82010-10-26 20:57:29 -04005113
Josef Bacikf28491e2013-12-16 13:24:27 -05005114 spin_lock(&fs_info->buffer_lock);
5115 ret = radix_tree_insert(&fs_info->buffer_radix,
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03005116 start >> PAGE_SHIFT, eb);
Josef Bacikf28491e2013-12-16 13:24:27 -05005117 spin_unlock(&fs_info->buffer_lock);
Chandra Seetharaman452c75c2013-10-07 10:45:25 -05005118 radix_tree_preload_end();
Miao Xie19fe0a82010-10-26 20:57:29 -04005119 if (ret == -EEXIST) {
Josef Bacikf28491e2013-12-16 13:24:27 -05005120 exists = find_extent_buffer(fs_info, start);
Chandra Seetharaman452c75c2013-10-07 10:45:25 -05005121 if (exists)
5122 goto free_eb;
5123 else
Josef Bacik115391d2012-03-09 09:51:43 -05005124 goto again;
Chris Mason6af118ce2008-07-22 11:18:07 -04005125 }
Chris Mason6af118ce2008-07-22 11:18:07 -04005126 /* add one reference for the tree */
Josef Bacik0b32f4b2012-03-13 09:38:00 -04005127 check_buffer_tree_ref(eb);
Josef Bacik34b41ac2013-12-13 10:41:51 -05005128 set_bit(EXTENT_BUFFER_IN_TREE, &eb->bflags);
Chris Masoneb14ab82011-02-10 12:35:00 -05005129
5130 /*
5131 * there is a race where release page may have
5132 * tried to find this extent buffer in the radix
5133 * but failed. It will tell the VM it is safe to
5134 * reclaim the, and it will clear the page private bit.
5135 * We must make sure to set the page private bit properly
5136 * after the extent buffer is in the radix tree so
5137 * it doesn't get lost
5138 */
Chris Mason727011e2010-08-06 13:21:20 -04005139 SetPageChecked(eb->pages[0]);
5140 for (i = 1; i < num_pages; i++) {
David Sterbafb85fc92014-07-31 01:03:53 +02005141 p = eb->pages[i];
Chris Mason727011e2010-08-06 13:21:20 -04005142 ClearPageChecked(p);
5143 unlock_page(p);
5144 }
5145 unlock_page(eb->pages[0]);
Chris Masond1310b22008-01-24 16:13:08 -05005146 return eb;
5147
Chris Mason6af118ce2008-07-22 11:18:07 -04005148free_eb:
Omar Sandoval5ca64f42015-02-24 02:47:05 -08005149 WARN_ON(!atomic_dec_and_test(&eb->refs));
Chris Mason727011e2010-08-06 13:21:20 -04005150 for (i = 0; i < num_pages; i++) {
5151 if (eb->pages[i])
5152 unlock_page(eb->pages[i]);
5153 }
Chris Masoneb14ab82011-02-10 12:35:00 -05005154
Miao Xie897ca6e2010-10-26 20:57:29 -04005155 btrfs_release_extent_buffer(eb);
Chris Mason6af118ce2008-07-22 11:18:07 -04005156 return exists;
Chris Masond1310b22008-01-24 16:13:08 -05005157}
Chris Masond1310b22008-01-24 16:13:08 -05005158
Josef Bacik3083ee22012-03-09 16:01:49 -05005159static inline void btrfs_release_extent_buffer_rcu(struct rcu_head *head)
5160{
5161 struct extent_buffer *eb =
5162 container_of(head, struct extent_buffer, rcu_head);
5163
5164 __free_extent_buffer(eb);
5165}
5166
Josef Bacik3083ee22012-03-09 16:01:49 -05005167/* Expects to have eb->eb_lock already held */
David Sterbaf7a52a42013-04-26 14:56:29 +00005168static int release_extent_buffer(struct extent_buffer *eb)
Josef Bacik3083ee22012-03-09 16:01:49 -05005169{
5170 WARN_ON(atomic_read(&eb->refs) == 0);
5171 if (atomic_dec_and_test(&eb->refs)) {
Josef Bacik34b41ac2013-12-13 10:41:51 -05005172 if (test_and_clear_bit(EXTENT_BUFFER_IN_TREE, &eb->bflags)) {
Josef Bacikf28491e2013-12-16 13:24:27 -05005173 struct btrfs_fs_info *fs_info = eb->fs_info;
Josef Bacik3083ee22012-03-09 16:01:49 -05005174
Jan Schmidt815a51c2012-05-16 17:00:02 +02005175 spin_unlock(&eb->refs_lock);
Josef Bacik3083ee22012-03-09 16:01:49 -05005176
Josef Bacikf28491e2013-12-16 13:24:27 -05005177 spin_lock(&fs_info->buffer_lock);
5178 radix_tree_delete(&fs_info->buffer_radix,
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03005179 eb->start >> PAGE_SHIFT);
Josef Bacikf28491e2013-12-16 13:24:27 -05005180 spin_unlock(&fs_info->buffer_lock);
Josef Bacik34b41ac2013-12-13 10:41:51 -05005181 } else {
5182 spin_unlock(&eb->refs_lock);
Jan Schmidt815a51c2012-05-16 17:00:02 +02005183 }
Josef Bacik3083ee22012-03-09 16:01:49 -05005184
5185 /* Should be safe to release our pages at this point */
David Sterbaa50924e2014-07-31 00:51:36 +02005186 btrfs_release_extent_buffer_page(eb);
Josef Bacikbcb7e442015-03-16 17:38:02 -04005187#ifdef CONFIG_BTRFS_FS_RUN_SANITY_TESTS
5188 if (unlikely(test_bit(EXTENT_BUFFER_DUMMY, &eb->bflags))) {
5189 __free_extent_buffer(eb);
5190 return 1;
5191 }
5192#endif
Josef Bacik3083ee22012-03-09 16:01:49 -05005193 call_rcu(&eb->rcu_head, btrfs_release_extent_buffer_rcu);
Josef Bacike64860a2012-07-20 16:05:36 -04005194 return 1;
Josef Bacik3083ee22012-03-09 16:01:49 -05005195 }
5196 spin_unlock(&eb->refs_lock);
Josef Bacike64860a2012-07-20 16:05:36 -04005197
5198 return 0;
Josef Bacik3083ee22012-03-09 16:01:49 -05005199}
5200
Chris Masond1310b22008-01-24 16:13:08 -05005201void free_extent_buffer(struct extent_buffer *eb)
5202{
Chris Mason242e18c2013-01-29 17:49:37 -05005203 int refs;
5204 int old;
Chris Masond1310b22008-01-24 16:13:08 -05005205 if (!eb)
5206 return;
5207
Chris Mason242e18c2013-01-29 17:49:37 -05005208 while (1) {
5209 refs = atomic_read(&eb->refs);
5210 if (refs <= 3)
5211 break;
5212 old = atomic_cmpxchg(&eb->refs, refs, refs - 1);
5213 if (old == refs)
5214 return;
5215 }
5216
Josef Bacik3083ee22012-03-09 16:01:49 -05005217 spin_lock(&eb->refs_lock);
5218 if (atomic_read(&eb->refs) == 2 &&
Jan Schmidt815a51c2012-05-16 17:00:02 +02005219 test_bit(EXTENT_BUFFER_DUMMY, &eb->bflags))
5220 atomic_dec(&eb->refs);
5221
5222 if (atomic_read(&eb->refs) == 2 &&
Josef Bacik3083ee22012-03-09 16:01:49 -05005223 test_bit(EXTENT_BUFFER_STALE, &eb->bflags) &&
Josef Bacik0b32f4b2012-03-13 09:38:00 -04005224 !extent_buffer_under_io(eb) &&
Josef Bacik3083ee22012-03-09 16:01:49 -05005225 test_and_clear_bit(EXTENT_BUFFER_TREE_REF, &eb->bflags))
5226 atomic_dec(&eb->refs);
Chris Masond1310b22008-01-24 16:13:08 -05005227
Josef Bacik3083ee22012-03-09 16:01:49 -05005228 /*
5229 * I know this is terrible, but it's temporary until we stop tracking
5230 * the uptodate bits and such for the extent buffers.
5231 */
David Sterbaf7a52a42013-04-26 14:56:29 +00005232 release_extent_buffer(eb);
Chris Masond1310b22008-01-24 16:13:08 -05005233}
Chris Masond1310b22008-01-24 16:13:08 -05005234
Josef Bacik3083ee22012-03-09 16:01:49 -05005235void free_extent_buffer_stale(struct extent_buffer *eb)
5236{
5237 if (!eb)
Chris Masond1310b22008-01-24 16:13:08 -05005238 return;
5239
Josef Bacik3083ee22012-03-09 16:01:49 -05005240 spin_lock(&eb->refs_lock);
5241 set_bit(EXTENT_BUFFER_STALE, &eb->bflags);
5242
Josef Bacik0b32f4b2012-03-13 09:38:00 -04005243 if (atomic_read(&eb->refs) == 2 && !extent_buffer_under_io(eb) &&
Josef Bacik3083ee22012-03-09 16:01:49 -05005244 test_and_clear_bit(EXTENT_BUFFER_TREE_REF, &eb->bflags))
5245 atomic_dec(&eb->refs);
David Sterbaf7a52a42013-04-26 14:56:29 +00005246 release_extent_buffer(eb);
Chris Masond1310b22008-01-24 16:13:08 -05005247}
5248
Chris Mason1d4284b2012-03-28 20:31:37 -04005249void clear_extent_buffer_dirty(struct extent_buffer *eb)
Chris Masond1310b22008-01-24 16:13:08 -05005250{
Chris Masond1310b22008-01-24 16:13:08 -05005251 unsigned long i;
5252 unsigned long num_pages;
5253 struct page *page;
5254
Chris Masond1310b22008-01-24 16:13:08 -05005255 num_pages = num_extent_pages(eb->start, eb->len);
5256
5257 for (i = 0; i < num_pages; i++) {
David Sterbafb85fc92014-07-31 01:03:53 +02005258 page = eb->pages[i];
Chris Masonb9473432009-03-13 11:00:37 -04005259 if (!PageDirty(page))
Chris Masond2c3f4f2008-11-19 12:44:22 -05005260 continue;
5261
Chris Masona61e6f22008-07-22 11:18:08 -04005262 lock_page(page);
Chris Masoneb14ab82011-02-10 12:35:00 -05005263 WARN_ON(!PagePrivate(page));
5264
Chris Masond1310b22008-01-24 16:13:08 -05005265 clear_page_dirty_for_io(page);
Sven Wegener0ee0fda2008-07-30 16:54:26 -04005266 spin_lock_irq(&page->mapping->tree_lock);
Chris Masond1310b22008-01-24 16:13:08 -05005267 if (!PageDirty(page)) {
5268 radix_tree_tag_clear(&page->mapping->page_tree,
5269 page_index(page),
5270 PAGECACHE_TAG_DIRTY);
5271 }
Sven Wegener0ee0fda2008-07-30 16:54:26 -04005272 spin_unlock_irq(&page->mapping->tree_lock);
Chris Masonbf0da8c2011-11-04 12:29:37 -04005273 ClearPageError(page);
Chris Masona61e6f22008-07-22 11:18:08 -04005274 unlock_page(page);
Chris Masond1310b22008-01-24 16:13:08 -05005275 }
Josef Bacik0b32f4b2012-03-13 09:38:00 -04005276 WARN_ON(atomic_read(&eb->refs) == 0);
Chris Masond1310b22008-01-24 16:13:08 -05005277}
Chris Masond1310b22008-01-24 16:13:08 -05005278
Josef Bacik0b32f4b2012-03-13 09:38:00 -04005279int set_extent_buffer_dirty(struct extent_buffer *eb)
Chris Masond1310b22008-01-24 16:13:08 -05005280{
5281 unsigned long i;
5282 unsigned long num_pages;
Chris Masonb9473432009-03-13 11:00:37 -04005283 int was_dirty = 0;
Chris Masond1310b22008-01-24 16:13:08 -05005284
Josef Bacik0b32f4b2012-03-13 09:38:00 -04005285 check_buffer_tree_ref(eb);
5286
Chris Masonb9473432009-03-13 11:00:37 -04005287 was_dirty = test_and_set_bit(EXTENT_BUFFER_DIRTY, &eb->bflags);
Josef Bacik0b32f4b2012-03-13 09:38:00 -04005288
Chris Masond1310b22008-01-24 16:13:08 -05005289 num_pages = num_extent_pages(eb->start, eb->len);
Josef Bacik3083ee22012-03-09 16:01:49 -05005290 WARN_ON(atomic_read(&eb->refs) == 0);
Josef Bacik0b32f4b2012-03-13 09:38:00 -04005291 WARN_ON(!test_bit(EXTENT_BUFFER_TREE_REF, &eb->bflags));
5292
Chris Masonb9473432009-03-13 11:00:37 -04005293 for (i = 0; i < num_pages; i++)
David Sterbafb85fc92014-07-31 01:03:53 +02005294 set_page_dirty(eb->pages[i]);
Chris Masonb9473432009-03-13 11:00:37 -04005295 return was_dirty;
Chris Masond1310b22008-01-24 16:13:08 -05005296}
Chris Masond1310b22008-01-24 16:13:08 -05005297
David Sterba69ba3922015-12-03 13:08:59 +01005298void clear_extent_buffer_uptodate(struct extent_buffer *eb)
Chris Mason1259ab72008-05-12 13:39:03 -04005299{
5300 unsigned long i;
5301 struct page *page;
5302 unsigned long num_pages;
5303
Chris Masonb4ce94d2009-02-04 09:25:08 -05005304 clear_bit(EXTENT_BUFFER_UPTODATE, &eb->bflags);
Josef Bacik0b32f4b2012-03-13 09:38:00 -04005305 num_pages = num_extent_pages(eb->start, eb->len);
Chris Mason1259ab72008-05-12 13:39:03 -04005306 for (i = 0; i < num_pages; i++) {
David Sterbafb85fc92014-07-31 01:03:53 +02005307 page = eb->pages[i];
Chris Mason33958dc2008-07-30 10:29:12 -04005308 if (page)
5309 ClearPageUptodate(page);
Chris Mason1259ab72008-05-12 13:39:03 -04005310 }
Chris Mason1259ab72008-05-12 13:39:03 -04005311}
5312
David Sterba09c25a82015-12-03 13:08:59 +01005313void set_extent_buffer_uptodate(struct extent_buffer *eb)
Chris Masond1310b22008-01-24 16:13:08 -05005314{
5315 unsigned long i;
5316 struct page *page;
5317 unsigned long num_pages;
5318
Josef Bacik0b32f4b2012-03-13 09:38:00 -04005319 set_bit(EXTENT_BUFFER_UPTODATE, &eb->bflags);
Chris Masond1310b22008-01-24 16:13:08 -05005320 num_pages = num_extent_pages(eb->start, eb->len);
Chris Masond1310b22008-01-24 16:13:08 -05005321 for (i = 0; i < num_pages; i++) {
David Sterbafb85fc92014-07-31 01:03:53 +02005322 page = eb->pages[i];
Chris Masond1310b22008-01-24 16:13:08 -05005323 SetPageUptodate(page);
5324 }
Chris Masond1310b22008-01-24 16:13:08 -05005325}
Chris Masond1310b22008-01-24 16:13:08 -05005326
Josef Bacik0b32f4b2012-03-13 09:38:00 -04005327int extent_buffer_uptodate(struct extent_buffer *eb)
Chris Masond1310b22008-01-24 16:13:08 -05005328{
Josef Bacik0b32f4b2012-03-13 09:38:00 -04005329 return test_bit(EXTENT_BUFFER_UPTODATE, &eb->bflags);
Chris Masond1310b22008-01-24 16:13:08 -05005330}
Chris Masond1310b22008-01-24 16:13:08 -05005331
5332int read_extent_buffer_pages(struct extent_io_tree *tree,
Josef Bacik8436ea912016-09-02 15:40:03 -04005333 struct extent_buffer *eb, int wait,
Chris Masonf1885912008-04-09 16:28:12 -04005334 get_extent_t *get_extent, int mirror_num)
Chris Masond1310b22008-01-24 16:13:08 -05005335{
5336 unsigned long i;
Chris Masond1310b22008-01-24 16:13:08 -05005337 struct page *page;
5338 int err;
5339 int ret = 0;
Chris Masonce9adaa2008-04-09 16:28:12 -04005340 int locked_pages = 0;
5341 int all_uptodate = 1;
Chris Masond1310b22008-01-24 16:13:08 -05005342 unsigned long num_pages;
Chris Mason727011e2010-08-06 13:21:20 -04005343 unsigned long num_reads = 0;
Chris Masona86c12c2008-02-07 10:50:54 -05005344 struct bio *bio = NULL;
Chris Masonc8b97812008-10-29 14:49:59 -04005345 unsigned long bio_flags = 0;
Chris Masona86c12c2008-02-07 10:50:54 -05005346
Chris Masonb4ce94d2009-02-04 09:25:08 -05005347 if (test_bit(EXTENT_BUFFER_UPTODATE, &eb->bflags))
Chris Masond1310b22008-01-24 16:13:08 -05005348 return 0;
5349
Chris Masond1310b22008-01-24 16:13:08 -05005350 num_pages = num_extent_pages(eb->start, eb->len);
Josef Bacik8436ea912016-09-02 15:40:03 -04005351 for (i = 0; i < num_pages; i++) {
David Sterbafb85fc92014-07-31 01:03:53 +02005352 page = eb->pages[i];
Arne Jansenbb82ab82011-06-10 14:06:53 +02005353 if (wait == WAIT_NONE) {
David Woodhouse2db04962008-08-07 11:19:43 -04005354 if (!trylock_page(page))
Chris Masonce9adaa2008-04-09 16:28:12 -04005355 goto unlock_exit;
Chris Masond1310b22008-01-24 16:13:08 -05005356 } else {
5357 lock_page(page);
5358 }
Chris Masonce9adaa2008-04-09 16:28:12 -04005359 locked_pages++;
Liu Bo2571e732016-08-03 12:33:01 -07005360 }
5361 /*
5362 * We need to firstly lock all pages to make sure that
5363 * the uptodate bit of our pages won't be affected by
5364 * clear_extent_buffer_uptodate().
5365 */
Josef Bacik8436ea912016-09-02 15:40:03 -04005366 for (i = 0; i < num_pages; i++) {
Liu Bo2571e732016-08-03 12:33:01 -07005367 page = eb->pages[i];
Chris Mason727011e2010-08-06 13:21:20 -04005368 if (!PageUptodate(page)) {
5369 num_reads++;
Chris Masonce9adaa2008-04-09 16:28:12 -04005370 all_uptodate = 0;
Chris Mason727011e2010-08-06 13:21:20 -04005371 }
Chris Masonce9adaa2008-04-09 16:28:12 -04005372 }
Liu Bo2571e732016-08-03 12:33:01 -07005373
Chris Masonce9adaa2008-04-09 16:28:12 -04005374 if (all_uptodate) {
Josef Bacik8436ea912016-09-02 15:40:03 -04005375 set_bit(EXTENT_BUFFER_UPTODATE, &eb->bflags);
Chris Masonce9adaa2008-04-09 16:28:12 -04005376 goto unlock_exit;
5377 }
5378
Filipe Manana656f30d2014-09-26 12:25:56 +01005379 clear_bit(EXTENT_BUFFER_READ_ERR, &eb->bflags);
Josef Bacik5cf1ab52012-04-16 09:42:26 -04005380 eb->read_mirror = 0;
Josef Bacik0b32f4b2012-03-13 09:38:00 -04005381 atomic_set(&eb->io_pages, num_reads);
Josef Bacik8436ea912016-09-02 15:40:03 -04005382 for (i = 0; i < num_pages; i++) {
David Sterbafb85fc92014-07-31 01:03:53 +02005383 page = eb->pages[i];
Liu Bobaf863b2016-07-11 10:39:07 -07005384
Chris Masonce9adaa2008-04-09 16:28:12 -04005385 if (!PageUptodate(page)) {
Liu Bobaf863b2016-07-11 10:39:07 -07005386 if (ret) {
5387 atomic_dec(&eb->io_pages);
5388 unlock_page(page);
5389 continue;
5390 }
5391
Chris Masonf1885912008-04-09 16:28:12 -04005392 ClearPageError(page);
Chris Masona86c12c2008-02-07 10:50:54 -05005393 err = __extent_read_full_page(tree, page,
Chris Masonf1885912008-04-09 16:28:12 -04005394 get_extent, &bio,
Josef Bacikd4c7ca82013-04-19 19:49:09 -04005395 mirror_num, &bio_flags,
Mike Christie1f7ad752016-06-05 14:31:51 -05005396 REQ_META);
Liu Bobaf863b2016-07-11 10:39:07 -07005397 if (err) {
Chris Masond1310b22008-01-24 16:13:08 -05005398 ret = err;
Liu Bobaf863b2016-07-11 10:39:07 -07005399 /*
5400 * We use &bio in above __extent_read_full_page,
5401 * so we ensure that if it returns error, the
5402 * current page fails to add itself to bio and
5403 * it's been unlocked.
5404 *
5405 * We must dec io_pages by ourselves.
5406 */
5407 atomic_dec(&eb->io_pages);
5408 }
Chris Masond1310b22008-01-24 16:13:08 -05005409 } else {
5410 unlock_page(page);
5411 }
5412 }
5413
Jeff Mahoney355808c2011-10-03 23:23:14 -04005414 if (bio) {
Mike Christie1f7ad752016-06-05 14:31:51 -05005415 err = submit_one_bio(bio, mirror_num, bio_flags);
Jeff Mahoney79787ea2012-03-12 16:03:00 +01005416 if (err)
5417 return err;
Jeff Mahoney355808c2011-10-03 23:23:14 -04005418 }
Chris Masona86c12c2008-02-07 10:50:54 -05005419
Arne Jansenbb82ab82011-06-10 14:06:53 +02005420 if (ret || wait != WAIT_COMPLETE)
Chris Masond1310b22008-01-24 16:13:08 -05005421 return ret;
Chris Masond3977122009-01-05 21:25:51 -05005422
Josef Bacik8436ea912016-09-02 15:40:03 -04005423 for (i = 0; i < num_pages; i++) {
David Sterbafb85fc92014-07-31 01:03:53 +02005424 page = eb->pages[i];
Chris Masond1310b22008-01-24 16:13:08 -05005425 wait_on_page_locked(page);
Chris Masond3977122009-01-05 21:25:51 -05005426 if (!PageUptodate(page))
Chris Masond1310b22008-01-24 16:13:08 -05005427 ret = -EIO;
Chris Masond1310b22008-01-24 16:13:08 -05005428 }
Chris Masond3977122009-01-05 21:25:51 -05005429
Chris Masond1310b22008-01-24 16:13:08 -05005430 return ret;
Chris Masonce9adaa2008-04-09 16:28:12 -04005431
5432unlock_exit:
Chris Masond3977122009-01-05 21:25:51 -05005433 while (locked_pages > 0) {
Chris Masonce9adaa2008-04-09 16:28:12 -04005434 locked_pages--;
Josef Bacik8436ea912016-09-02 15:40:03 -04005435 page = eb->pages[locked_pages];
5436 unlock_page(page);
Chris Masonce9adaa2008-04-09 16:28:12 -04005437 }
5438 return ret;
Chris Masond1310b22008-01-24 16:13:08 -05005439}
Chris Masond1310b22008-01-24 16:13:08 -05005440
Jeff Mahoney20b304152017-06-28 21:56:53 -06005441void read_extent_buffer(const struct extent_buffer *eb, void *dstv,
5442 unsigned long start, unsigned long len)
Chris Masond1310b22008-01-24 16:13:08 -05005443{
5444 size_t cur;
5445 size_t offset;
5446 struct page *page;
5447 char *kaddr;
5448 char *dst = (char *)dstv;
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03005449 size_t start_offset = eb->start & ((u64)PAGE_SIZE - 1);
5450 unsigned long i = (start_offset + start) >> PAGE_SHIFT;
Chris Masond1310b22008-01-24 16:13:08 -05005451
5452 WARN_ON(start > eb->len);
5453 WARN_ON(start + len > eb->start + eb->len);
5454
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03005455 offset = (start_offset + start) & (PAGE_SIZE - 1);
Chris Masond1310b22008-01-24 16:13:08 -05005456
Chris Masond3977122009-01-05 21:25:51 -05005457 while (len > 0) {
David Sterbafb85fc92014-07-31 01:03:53 +02005458 page = eb->pages[i];
Chris Masond1310b22008-01-24 16:13:08 -05005459
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03005460 cur = min(len, (PAGE_SIZE - offset));
Chris Masona6591712011-07-19 12:04:14 -04005461 kaddr = page_address(page);
Chris Masond1310b22008-01-24 16:13:08 -05005462 memcpy(dst, kaddr + offset, cur);
Chris Masond1310b22008-01-24 16:13:08 -05005463
5464 dst += cur;
5465 len -= cur;
5466 offset = 0;
5467 i++;
5468 }
5469}
Chris Masond1310b22008-01-24 16:13:08 -05005470
Jeff Mahoney20b304152017-06-28 21:56:53 -06005471int read_extent_buffer_to_user(const struct extent_buffer *eb,
5472 void __user *dstv,
5473 unsigned long start, unsigned long len)
Gerhard Heift550ac1d2014-01-30 16:24:01 +01005474{
5475 size_t cur;
5476 size_t offset;
5477 struct page *page;
5478 char *kaddr;
5479 char __user *dst = (char __user *)dstv;
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03005480 size_t start_offset = eb->start & ((u64)PAGE_SIZE - 1);
5481 unsigned long i = (start_offset + start) >> PAGE_SHIFT;
Gerhard Heift550ac1d2014-01-30 16:24:01 +01005482 int ret = 0;
5483
5484 WARN_ON(start > eb->len);
5485 WARN_ON(start + len > eb->start + eb->len);
5486
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03005487 offset = (start_offset + start) & (PAGE_SIZE - 1);
Gerhard Heift550ac1d2014-01-30 16:24:01 +01005488
5489 while (len > 0) {
David Sterbafb85fc92014-07-31 01:03:53 +02005490 page = eb->pages[i];
Gerhard Heift550ac1d2014-01-30 16:24:01 +01005491
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03005492 cur = min(len, (PAGE_SIZE - offset));
Gerhard Heift550ac1d2014-01-30 16:24:01 +01005493 kaddr = page_address(page);
5494 if (copy_to_user(dst, kaddr + offset, cur)) {
5495 ret = -EFAULT;
5496 break;
5497 }
5498
5499 dst += cur;
5500 len -= cur;
5501 offset = 0;
5502 i++;
5503 }
5504
5505 return ret;
5506}
5507
Liu Bo415b35a2016-06-17 19:16:21 -07005508/*
5509 * return 0 if the item is found within a page.
5510 * return 1 if the item spans two pages.
5511 * return -EINVAL otherwise.
5512 */
Jeff Mahoney20b304152017-06-28 21:56:53 -06005513int map_private_extent_buffer(const struct extent_buffer *eb,
5514 unsigned long start, unsigned long min_len,
5515 char **map, unsigned long *map_start,
5516 unsigned long *map_len)
Chris Masond1310b22008-01-24 16:13:08 -05005517{
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03005518 size_t offset = start & (PAGE_SIZE - 1);
Chris Masond1310b22008-01-24 16:13:08 -05005519 char *kaddr;
5520 struct page *p;
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03005521 size_t start_offset = eb->start & ((u64)PAGE_SIZE - 1);
5522 unsigned long i = (start_offset + start) >> PAGE_SHIFT;
Chris Masond1310b22008-01-24 16:13:08 -05005523 unsigned long end_i = (start_offset + start + min_len - 1) >>
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03005524 PAGE_SHIFT;
Chris Masond1310b22008-01-24 16:13:08 -05005525
5526 if (i != end_i)
Liu Bo415b35a2016-06-17 19:16:21 -07005527 return 1;
Chris Masond1310b22008-01-24 16:13:08 -05005528
5529 if (i == 0) {
5530 offset = start_offset;
5531 *map_start = 0;
5532 } else {
5533 offset = 0;
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03005534 *map_start = ((u64)i << PAGE_SHIFT) - start_offset;
Chris Masond1310b22008-01-24 16:13:08 -05005535 }
Chris Masond3977122009-01-05 21:25:51 -05005536
Chris Masond1310b22008-01-24 16:13:08 -05005537 if (start + min_len > eb->len) {
Jeff Mahoney5d163e02016-09-20 10:05:00 -04005538 WARN(1, KERN_ERR "btrfs bad mapping eb start %llu len %lu, wanted %lu %lu\n",
Geert Uytterhoevenc1c9ff72013-08-20 13:20:07 +02005539 eb->start, eb->len, start, min_len);
Josef Bacik850265332011-03-15 14:52:12 -04005540 return -EINVAL;
Chris Masond1310b22008-01-24 16:13:08 -05005541 }
5542
David Sterbafb85fc92014-07-31 01:03:53 +02005543 p = eb->pages[i];
Chris Masona6591712011-07-19 12:04:14 -04005544 kaddr = page_address(p);
Chris Masond1310b22008-01-24 16:13:08 -05005545 *map = kaddr + offset;
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03005546 *map_len = PAGE_SIZE - offset;
Chris Masond1310b22008-01-24 16:13:08 -05005547 return 0;
5548}
Chris Masond1310b22008-01-24 16:13:08 -05005549
Jeff Mahoney20b304152017-06-28 21:56:53 -06005550int memcmp_extent_buffer(const struct extent_buffer *eb, const void *ptrv,
5551 unsigned long start, unsigned long len)
Chris Masond1310b22008-01-24 16:13:08 -05005552{
5553 size_t cur;
5554 size_t offset;
5555 struct page *page;
5556 char *kaddr;
5557 char *ptr = (char *)ptrv;
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03005558 size_t start_offset = eb->start & ((u64)PAGE_SIZE - 1);
5559 unsigned long i = (start_offset + start) >> PAGE_SHIFT;
Chris Masond1310b22008-01-24 16:13:08 -05005560 int ret = 0;
5561
5562 WARN_ON(start > eb->len);
5563 WARN_ON(start + len > eb->start + eb->len);
5564
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03005565 offset = (start_offset + start) & (PAGE_SIZE - 1);
Chris Masond1310b22008-01-24 16:13:08 -05005566
Chris Masond3977122009-01-05 21:25:51 -05005567 while (len > 0) {
David Sterbafb85fc92014-07-31 01:03:53 +02005568 page = eb->pages[i];
Chris Masond1310b22008-01-24 16:13:08 -05005569
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03005570 cur = min(len, (PAGE_SIZE - offset));
Chris Masond1310b22008-01-24 16:13:08 -05005571
Chris Masona6591712011-07-19 12:04:14 -04005572 kaddr = page_address(page);
Chris Masond1310b22008-01-24 16:13:08 -05005573 ret = memcmp(ptr, kaddr + offset, cur);
Chris Masond1310b22008-01-24 16:13:08 -05005574 if (ret)
5575 break;
5576
5577 ptr += cur;
5578 len -= cur;
5579 offset = 0;
5580 i++;
5581 }
5582 return ret;
5583}
Chris Masond1310b22008-01-24 16:13:08 -05005584
5585void write_extent_buffer(struct extent_buffer *eb, const void *srcv,
5586 unsigned long start, unsigned long len)
5587{
5588 size_t cur;
5589 size_t offset;
5590 struct page *page;
5591 char *kaddr;
5592 char *src = (char *)srcv;
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03005593 size_t start_offset = eb->start & ((u64)PAGE_SIZE - 1);
5594 unsigned long i = (start_offset + start) >> PAGE_SHIFT;
Chris Masond1310b22008-01-24 16:13:08 -05005595
5596 WARN_ON(start > eb->len);
5597 WARN_ON(start + len > eb->start + eb->len);
5598
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03005599 offset = (start_offset + start) & (PAGE_SIZE - 1);
Chris Masond1310b22008-01-24 16:13:08 -05005600
Chris Masond3977122009-01-05 21:25:51 -05005601 while (len > 0) {
David Sterbafb85fc92014-07-31 01:03:53 +02005602 page = eb->pages[i];
Chris Masond1310b22008-01-24 16:13:08 -05005603 WARN_ON(!PageUptodate(page));
5604
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03005605 cur = min(len, PAGE_SIZE - offset);
Chris Masona6591712011-07-19 12:04:14 -04005606 kaddr = page_address(page);
Chris Masond1310b22008-01-24 16:13:08 -05005607 memcpy(kaddr + offset, src, cur);
Chris Masond1310b22008-01-24 16:13:08 -05005608
5609 src += cur;
5610 len -= cur;
5611 offset = 0;
5612 i++;
5613 }
5614}
Chris Masond1310b22008-01-24 16:13:08 -05005615
5616void memset_extent_buffer(struct extent_buffer *eb, char c,
5617 unsigned long start, unsigned long len)
5618{
5619 size_t cur;
5620 size_t offset;
5621 struct page *page;
5622 char *kaddr;
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03005623 size_t start_offset = eb->start & ((u64)PAGE_SIZE - 1);
5624 unsigned long i = (start_offset + start) >> PAGE_SHIFT;
Chris Masond1310b22008-01-24 16:13:08 -05005625
5626 WARN_ON(start > eb->len);
5627 WARN_ON(start + len > eb->start + eb->len);
5628
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03005629 offset = (start_offset + start) & (PAGE_SIZE - 1);
Chris Masond1310b22008-01-24 16:13:08 -05005630
Chris Masond3977122009-01-05 21:25:51 -05005631 while (len > 0) {
David Sterbafb85fc92014-07-31 01:03:53 +02005632 page = eb->pages[i];
Chris Masond1310b22008-01-24 16:13:08 -05005633 WARN_ON(!PageUptodate(page));
5634
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03005635 cur = min(len, PAGE_SIZE - offset);
Chris Masona6591712011-07-19 12:04:14 -04005636 kaddr = page_address(page);
Chris Masond1310b22008-01-24 16:13:08 -05005637 memset(kaddr + offset, c, cur);
Chris Masond1310b22008-01-24 16:13:08 -05005638
5639 len -= cur;
5640 offset = 0;
5641 i++;
5642 }
5643}
Chris Masond1310b22008-01-24 16:13:08 -05005644
5645void copy_extent_buffer(struct extent_buffer *dst, struct extent_buffer *src,
5646 unsigned long dst_offset, unsigned long src_offset,
5647 unsigned long len)
5648{
5649 u64 dst_len = dst->len;
5650 size_t cur;
5651 size_t offset;
5652 struct page *page;
5653 char *kaddr;
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03005654 size_t start_offset = dst->start & ((u64)PAGE_SIZE - 1);
5655 unsigned long i = (start_offset + dst_offset) >> PAGE_SHIFT;
Chris Masond1310b22008-01-24 16:13:08 -05005656
5657 WARN_ON(src->len != dst_len);
5658
5659 offset = (start_offset + dst_offset) &
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03005660 (PAGE_SIZE - 1);
Chris Masond1310b22008-01-24 16:13:08 -05005661
Chris Masond3977122009-01-05 21:25:51 -05005662 while (len > 0) {
David Sterbafb85fc92014-07-31 01:03:53 +02005663 page = dst->pages[i];
Chris Masond1310b22008-01-24 16:13:08 -05005664 WARN_ON(!PageUptodate(page));
5665
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03005666 cur = min(len, (unsigned long)(PAGE_SIZE - offset));
Chris Masond1310b22008-01-24 16:13:08 -05005667
Chris Masona6591712011-07-19 12:04:14 -04005668 kaddr = page_address(page);
Chris Masond1310b22008-01-24 16:13:08 -05005669 read_extent_buffer(src, kaddr + offset, src_offset, cur);
Chris Masond1310b22008-01-24 16:13:08 -05005670
5671 src_offset += cur;
5672 len -= cur;
5673 offset = 0;
5674 i++;
5675 }
5676}
Chris Masond1310b22008-01-24 16:13:08 -05005677
Omar Sandoval2fe1d552016-09-22 17:24:20 -07005678void le_bitmap_set(u8 *map, unsigned int start, int len)
5679{
5680 u8 *p = map + BIT_BYTE(start);
5681 const unsigned int size = start + len;
5682 int bits_to_set = BITS_PER_BYTE - (start % BITS_PER_BYTE);
5683 u8 mask_to_set = BITMAP_FIRST_BYTE_MASK(start);
5684
5685 while (len - bits_to_set >= 0) {
5686 *p |= mask_to_set;
5687 len -= bits_to_set;
5688 bits_to_set = BITS_PER_BYTE;
Dan Carpenter9c894692016-10-12 11:33:21 +03005689 mask_to_set = ~0;
Omar Sandoval2fe1d552016-09-22 17:24:20 -07005690 p++;
5691 }
5692 if (len) {
5693 mask_to_set &= BITMAP_LAST_BYTE_MASK(size);
5694 *p |= mask_to_set;
5695 }
5696}
5697
5698void le_bitmap_clear(u8 *map, unsigned int start, int len)
5699{
5700 u8 *p = map + BIT_BYTE(start);
5701 const unsigned int size = start + len;
5702 int bits_to_clear = BITS_PER_BYTE - (start % BITS_PER_BYTE);
5703 u8 mask_to_clear = BITMAP_FIRST_BYTE_MASK(start);
5704
5705 while (len - bits_to_clear >= 0) {
5706 *p &= ~mask_to_clear;
5707 len -= bits_to_clear;
5708 bits_to_clear = BITS_PER_BYTE;
Dan Carpenter9c894692016-10-12 11:33:21 +03005709 mask_to_clear = ~0;
Omar Sandoval2fe1d552016-09-22 17:24:20 -07005710 p++;
5711 }
5712 if (len) {
5713 mask_to_clear &= BITMAP_LAST_BYTE_MASK(size);
5714 *p &= ~mask_to_clear;
5715 }
5716}
Omar Sandoval3e1e8bb2015-09-29 20:50:30 -07005717
5718/*
5719 * eb_bitmap_offset() - calculate the page and offset of the byte containing the
5720 * given bit number
5721 * @eb: the extent buffer
5722 * @start: offset of the bitmap item in the extent buffer
5723 * @nr: bit number
5724 * @page_index: return index of the page in the extent buffer that contains the
5725 * given bit number
5726 * @page_offset: return offset into the page given by page_index
5727 *
5728 * This helper hides the ugliness of finding the byte in an extent buffer which
5729 * contains a given bit.
5730 */
5731static inline void eb_bitmap_offset(struct extent_buffer *eb,
5732 unsigned long start, unsigned long nr,
5733 unsigned long *page_index,
5734 size_t *page_offset)
5735{
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03005736 size_t start_offset = eb->start & ((u64)PAGE_SIZE - 1);
Omar Sandoval3e1e8bb2015-09-29 20:50:30 -07005737 size_t byte_offset = BIT_BYTE(nr);
5738 size_t offset;
5739
5740 /*
5741 * The byte we want is the offset of the extent buffer + the offset of
5742 * the bitmap item in the extent buffer + the offset of the byte in the
5743 * bitmap item.
5744 */
5745 offset = start_offset + start + byte_offset;
5746
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03005747 *page_index = offset >> PAGE_SHIFT;
5748 *page_offset = offset & (PAGE_SIZE - 1);
Omar Sandoval3e1e8bb2015-09-29 20:50:30 -07005749}
5750
5751/**
5752 * extent_buffer_test_bit - determine whether a bit in a bitmap item is set
5753 * @eb: the extent buffer
5754 * @start: offset of the bitmap item in the extent buffer
5755 * @nr: bit number to test
5756 */
5757int extent_buffer_test_bit(struct extent_buffer *eb, unsigned long start,
5758 unsigned long nr)
5759{
Omar Sandoval2fe1d552016-09-22 17:24:20 -07005760 u8 *kaddr;
Omar Sandoval3e1e8bb2015-09-29 20:50:30 -07005761 struct page *page;
5762 unsigned long i;
5763 size_t offset;
5764
5765 eb_bitmap_offset(eb, start, nr, &i, &offset);
5766 page = eb->pages[i];
5767 WARN_ON(!PageUptodate(page));
5768 kaddr = page_address(page);
5769 return 1U & (kaddr[offset] >> (nr & (BITS_PER_BYTE - 1)));
5770}
5771
5772/**
5773 * extent_buffer_bitmap_set - set an area of a bitmap
5774 * @eb: the extent buffer
5775 * @start: offset of the bitmap item in the extent buffer
5776 * @pos: bit number of the first bit
5777 * @len: number of bits to set
5778 */
5779void extent_buffer_bitmap_set(struct extent_buffer *eb, unsigned long start,
5780 unsigned long pos, unsigned long len)
5781{
Omar Sandoval2fe1d552016-09-22 17:24:20 -07005782 u8 *kaddr;
Omar Sandoval3e1e8bb2015-09-29 20:50:30 -07005783 struct page *page;
5784 unsigned long i;
5785 size_t offset;
5786 const unsigned int size = pos + len;
5787 int bits_to_set = BITS_PER_BYTE - (pos % BITS_PER_BYTE);
Omar Sandoval2fe1d552016-09-22 17:24:20 -07005788 u8 mask_to_set = BITMAP_FIRST_BYTE_MASK(pos);
Omar Sandoval3e1e8bb2015-09-29 20:50:30 -07005789
5790 eb_bitmap_offset(eb, start, pos, &i, &offset);
5791 page = eb->pages[i];
5792 WARN_ON(!PageUptodate(page));
5793 kaddr = page_address(page);
5794
5795 while (len >= bits_to_set) {
5796 kaddr[offset] |= mask_to_set;
5797 len -= bits_to_set;
5798 bits_to_set = BITS_PER_BYTE;
Dan Carpenter9c894692016-10-12 11:33:21 +03005799 mask_to_set = ~0;
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03005800 if (++offset >= PAGE_SIZE && len > 0) {
Omar Sandoval3e1e8bb2015-09-29 20:50:30 -07005801 offset = 0;
5802 page = eb->pages[++i];
5803 WARN_ON(!PageUptodate(page));
5804 kaddr = page_address(page);
5805 }
5806 }
5807 if (len) {
5808 mask_to_set &= BITMAP_LAST_BYTE_MASK(size);
5809 kaddr[offset] |= mask_to_set;
5810 }
5811}
5812
5813
5814/**
5815 * extent_buffer_bitmap_clear - clear an area of a bitmap
5816 * @eb: the extent buffer
5817 * @start: offset of the bitmap item in the extent buffer
5818 * @pos: bit number of the first bit
5819 * @len: number of bits to clear
5820 */
5821void extent_buffer_bitmap_clear(struct extent_buffer *eb, unsigned long start,
5822 unsigned long pos, unsigned long len)
5823{
Omar Sandoval2fe1d552016-09-22 17:24:20 -07005824 u8 *kaddr;
Omar Sandoval3e1e8bb2015-09-29 20:50:30 -07005825 struct page *page;
5826 unsigned long i;
5827 size_t offset;
5828 const unsigned int size = pos + len;
5829 int bits_to_clear = BITS_PER_BYTE - (pos % BITS_PER_BYTE);
Omar Sandoval2fe1d552016-09-22 17:24:20 -07005830 u8 mask_to_clear = BITMAP_FIRST_BYTE_MASK(pos);
Omar Sandoval3e1e8bb2015-09-29 20:50:30 -07005831
5832 eb_bitmap_offset(eb, start, pos, &i, &offset);
5833 page = eb->pages[i];
5834 WARN_ON(!PageUptodate(page));
5835 kaddr = page_address(page);
5836
5837 while (len >= bits_to_clear) {
5838 kaddr[offset] &= ~mask_to_clear;
5839 len -= bits_to_clear;
5840 bits_to_clear = BITS_PER_BYTE;
Dan Carpenter9c894692016-10-12 11:33:21 +03005841 mask_to_clear = ~0;
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03005842 if (++offset >= PAGE_SIZE && len > 0) {
Omar Sandoval3e1e8bb2015-09-29 20:50:30 -07005843 offset = 0;
5844 page = eb->pages[++i];
5845 WARN_ON(!PageUptodate(page));
5846 kaddr = page_address(page);
5847 }
5848 }
5849 if (len) {
5850 mask_to_clear &= BITMAP_LAST_BYTE_MASK(size);
5851 kaddr[offset] &= ~mask_to_clear;
5852 }
5853}
5854
Sergei Trofimovich33872062011-04-11 21:52:52 +00005855static inline bool areas_overlap(unsigned long src, unsigned long dst, unsigned long len)
5856{
5857 unsigned long distance = (src > dst) ? src - dst : dst - src;
5858 return distance < len;
5859}
5860
Chris Masond1310b22008-01-24 16:13:08 -05005861static void copy_pages(struct page *dst_page, struct page *src_page,
5862 unsigned long dst_off, unsigned long src_off,
5863 unsigned long len)
5864{
Chris Masona6591712011-07-19 12:04:14 -04005865 char *dst_kaddr = page_address(dst_page);
Chris Masond1310b22008-01-24 16:13:08 -05005866 char *src_kaddr;
Chris Mason727011e2010-08-06 13:21:20 -04005867 int must_memmove = 0;
Chris Masond1310b22008-01-24 16:13:08 -05005868
Sergei Trofimovich33872062011-04-11 21:52:52 +00005869 if (dst_page != src_page) {
Chris Masona6591712011-07-19 12:04:14 -04005870 src_kaddr = page_address(src_page);
Sergei Trofimovich33872062011-04-11 21:52:52 +00005871 } else {
Chris Masond1310b22008-01-24 16:13:08 -05005872 src_kaddr = dst_kaddr;
Chris Mason727011e2010-08-06 13:21:20 -04005873 if (areas_overlap(src_off, dst_off, len))
5874 must_memmove = 1;
Sergei Trofimovich33872062011-04-11 21:52:52 +00005875 }
Chris Masond1310b22008-01-24 16:13:08 -05005876
Chris Mason727011e2010-08-06 13:21:20 -04005877 if (must_memmove)
5878 memmove(dst_kaddr + dst_off, src_kaddr + src_off, len);
5879 else
5880 memcpy(dst_kaddr + dst_off, src_kaddr + src_off, len);
Chris Masond1310b22008-01-24 16:13:08 -05005881}
5882
5883void memcpy_extent_buffer(struct extent_buffer *dst, unsigned long dst_offset,
5884 unsigned long src_offset, unsigned long len)
5885{
5886 size_t cur;
5887 size_t dst_off_in_page;
5888 size_t src_off_in_page;
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03005889 size_t start_offset = dst->start & ((u64)PAGE_SIZE - 1);
Chris Masond1310b22008-01-24 16:13:08 -05005890 unsigned long dst_i;
5891 unsigned long src_i;
5892
5893 if (src_offset + len > dst->len) {
David Sterbaf14d1042015-10-08 11:37:06 +02005894 btrfs_err(dst->fs_info,
Jeff Mahoney5d163e02016-09-20 10:05:00 -04005895 "memmove bogus src_offset %lu move len %lu dst len %lu",
5896 src_offset, len, dst->len);
Chris Masond1310b22008-01-24 16:13:08 -05005897 BUG_ON(1);
5898 }
5899 if (dst_offset + len > dst->len) {
David Sterbaf14d1042015-10-08 11:37:06 +02005900 btrfs_err(dst->fs_info,
Jeff Mahoney5d163e02016-09-20 10:05:00 -04005901 "memmove bogus dst_offset %lu move len %lu dst len %lu",
5902 dst_offset, len, dst->len);
Chris Masond1310b22008-01-24 16:13:08 -05005903 BUG_ON(1);
5904 }
5905
Chris Masond3977122009-01-05 21:25:51 -05005906 while (len > 0) {
Chris Masond1310b22008-01-24 16:13:08 -05005907 dst_off_in_page = (start_offset + dst_offset) &
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03005908 (PAGE_SIZE - 1);
Chris Masond1310b22008-01-24 16:13:08 -05005909 src_off_in_page = (start_offset + src_offset) &
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03005910 (PAGE_SIZE - 1);
Chris Masond1310b22008-01-24 16:13:08 -05005911
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03005912 dst_i = (start_offset + dst_offset) >> PAGE_SHIFT;
5913 src_i = (start_offset + src_offset) >> PAGE_SHIFT;
Chris Masond1310b22008-01-24 16:13:08 -05005914
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03005915 cur = min(len, (unsigned long)(PAGE_SIZE -
Chris Masond1310b22008-01-24 16:13:08 -05005916 src_off_in_page));
5917 cur = min_t(unsigned long, cur,
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03005918 (unsigned long)(PAGE_SIZE - dst_off_in_page));
Chris Masond1310b22008-01-24 16:13:08 -05005919
David Sterbafb85fc92014-07-31 01:03:53 +02005920 copy_pages(dst->pages[dst_i], dst->pages[src_i],
Chris Masond1310b22008-01-24 16:13:08 -05005921 dst_off_in_page, src_off_in_page, cur);
5922
5923 src_offset += cur;
5924 dst_offset += cur;
5925 len -= cur;
5926 }
5927}
Chris Masond1310b22008-01-24 16:13:08 -05005928
5929void memmove_extent_buffer(struct extent_buffer *dst, unsigned long dst_offset,
5930 unsigned long src_offset, unsigned long len)
5931{
5932 size_t cur;
5933 size_t dst_off_in_page;
5934 size_t src_off_in_page;
5935 unsigned long dst_end = dst_offset + len - 1;
5936 unsigned long src_end = src_offset + len - 1;
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03005937 size_t start_offset = dst->start & ((u64)PAGE_SIZE - 1);
Chris Masond1310b22008-01-24 16:13:08 -05005938 unsigned long dst_i;
5939 unsigned long src_i;
5940
5941 if (src_offset + len > dst->len) {
Jeff Mahoney5d163e02016-09-20 10:05:00 -04005942 btrfs_err(dst->fs_info,
5943 "memmove bogus src_offset %lu move len %lu len %lu",
5944 src_offset, len, dst->len);
Chris Masond1310b22008-01-24 16:13:08 -05005945 BUG_ON(1);
5946 }
5947 if (dst_offset + len > dst->len) {
Jeff Mahoney5d163e02016-09-20 10:05:00 -04005948 btrfs_err(dst->fs_info,
5949 "memmove bogus dst_offset %lu move len %lu len %lu",
5950 dst_offset, len, dst->len);
Chris Masond1310b22008-01-24 16:13:08 -05005951 BUG_ON(1);
5952 }
Chris Mason727011e2010-08-06 13:21:20 -04005953 if (dst_offset < src_offset) {
Chris Masond1310b22008-01-24 16:13:08 -05005954 memcpy_extent_buffer(dst, dst_offset, src_offset, len);
5955 return;
5956 }
Chris Masond3977122009-01-05 21:25:51 -05005957 while (len > 0) {
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03005958 dst_i = (start_offset + dst_end) >> PAGE_SHIFT;
5959 src_i = (start_offset + src_end) >> PAGE_SHIFT;
Chris Masond1310b22008-01-24 16:13:08 -05005960
5961 dst_off_in_page = (start_offset + dst_end) &
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03005962 (PAGE_SIZE - 1);
Chris Masond1310b22008-01-24 16:13:08 -05005963 src_off_in_page = (start_offset + src_end) &
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03005964 (PAGE_SIZE - 1);
Chris Masond1310b22008-01-24 16:13:08 -05005965
5966 cur = min_t(unsigned long, len, src_off_in_page + 1);
5967 cur = min(cur, dst_off_in_page + 1);
David Sterbafb85fc92014-07-31 01:03:53 +02005968 copy_pages(dst->pages[dst_i], dst->pages[src_i],
Chris Masond1310b22008-01-24 16:13:08 -05005969 dst_off_in_page - cur + 1,
5970 src_off_in_page - cur + 1, cur);
5971
5972 dst_end -= cur;
5973 src_end -= cur;
5974 len -= cur;
5975 }
5976}
Chris Mason6af118ce2008-07-22 11:18:07 -04005977
David Sterbaf7a52a42013-04-26 14:56:29 +00005978int try_release_extent_buffer(struct page *page)
Miao Xie19fe0a82010-10-26 20:57:29 -04005979{
Chris Mason6af118ce2008-07-22 11:18:07 -04005980 struct extent_buffer *eb;
Miao Xie897ca6e2010-10-26 20:57:29 -04005981
Miao Xie19fe0a82010-10-26 20:57:29 -04005982 /*
Nicholas D Steeves01327612016-05-19 21:18:45 -04005983 * We need to make sure nobody is attaching this page to an eb right
Josef Bacik3083ee22012-03-09 16:01:49 -05005984 * now.
Miao Xie19fe0a82010-10-26 20:57:29 -04005985 */
Josef Bacik3083ee22012-03-09 16:01:49 -05005986 spin_lock(&page->mapping->private_lock);
5987 if (!PagePrivate(page)) {
5988 spin_unlock(&page->mapping->private_lock);
5989 return 1;
Miao Xie19fe0a82010-10-26 20:57:29 -04005990 }
5991
Josef Bacik3083ee22012-03-09 16:01:49 -05005992 eb = (struct extent_buffer *)page->private;
5993 BUG_ON(!eb);
Miao Xie19fe0a82010-10-26 20:57:29 -04005994
Josef Bacik0b32f4b2012-03-13 09:38:00 -04005995 /*
Josef Bacik3083ee22012-03-09 16:01:49 -05005996 * This is a little awful but should be ok, we need to make sure that
5997 * the eb doesn't disappear out from under us while we're looking at
5998 * this page.
5999 */
6000 spin_lock(&eb->refs_lock);
Josef Bacik0b32f4b2012-03-13 09:38:00 -04006001 if (atomic_read(&eb->refs) != 1 || extent_buffer_under_io(eb)) {
Josef Bacik3083ee22012-03-09 16:01:49 -05006002 spin_unlock(&eb->refs_lock);
6003 spin_unlock(&page->mapping->private_lock);
6004 return 0;
6005 }
6006 spin_unlock(&page->mapping->private_lock);
6007
Josef Bacik3083ee22012-03-09 16:01:49 -05006008 /*
6009 * If tree ref isn't set then we know the ref on this eb is a real ref,
6010 * so just return, this page will likely be freed soon anyway.
6011 */
6012 if (!test_and_clear_bit(EXTENT_BUFFER_TREE_REF, &eb->bflags)) {
6013 spin_unlock(&eb->refs_lock);
6014 return 0;
6015 }
Josef Bacik3083ee22012-03-09 16:01:49 -05006016
David Sterbaf7a52a42013-04-26 14:56:29 +00006017 return release_extent_buffer(eb);
Chris Mason6af118ce2008-07-22 11:18:07 -04006018}