blob: c9018a05036e943a52ad91d81019bb4b934b6b9a [file] [log] [blame]
Chris Masond1310b22008-01-24 16:13:08 -05001#include <linux/bitops.h>
2#include <linux/slab.h>
3#include <linux/bio.h>
4#include <linux/mm.h>
Chris Masond1310b22008-01-24 16:13:08 -05005#include <linux/pagemap.h>
6#include <linux/page-flags.h>
7#include <linux/module.h>
8#include <linux/spinlock.h>
9#include <linux/blkdev.h>
10#include <linux/swap.h>
Chris Masond1310b22008-01-24 16:13:08 -050011#include <linux/writeback.h>
12#include <linux/pagevec.h>
Linus Torvalds268bb0c2011-05-20 12:50:29 -070013#include <linux/prefetch.h>
Dan Magenheimer90a887c2011-05-26 10:01:56 -060014#include <linux/cleancache.h>
Chris Masond1310b22008-01-24 16:13:08 -050015#include "extent_io.h"
16#include "extent_map.h"
David Woodhouse2db04962008-08-07 11:19:43 -040017#include "compat.h"
David Woodhouse902b22f2008-08-20 08:51:49 -040018#include "ctree.h"
19#include "btrfs_inode.h"
Jan Schmidt4a54c8c2011-07-22 15:41:52 +020020#include "volumes.h"
Stefan Behrens21adbd52011-11-09 13:44:05 +010021#include "check-integrity.h"
Josef Bacik0b32f4b2012-03-13 09:38:00 -040022#include "locking.h"
Chris Masond1310b22008-01-24 16:13:08 -050023
Chris Masond1310b22008-01-24 16:13:08 -050024static struct kmem_cache *extent_state_cache;
25static struct kmem_cache *extent_buffer_cache;
26
27static LIST_HEAD(buffers);
28static LIST_HEAD(states);
Chris Mason4bef0842008-09-08 11:18:08 -040029
Chris Masonb47eda82008-11-10 12:34:40 -050030#define LEAK_DEBUG 0
Chris Mason39351272009-02-04 09:24:05 -050031#if LEAK_DEBUG
Chris Masond3977122009-01-05 21:25:51 -050032static DEFINE_SPINLOCK(leak_lock);
Chris Mason4bef0842008-09-08 11:18:08 -040033#endif
Chris Masond1310b22008-01-24 16:13:08 -050034
Chris Masond1310b22008-01-24 16:13:08 -050035#define BUFFER_LRU_MAX 64
36
37struct tree_entry {
38 u64 start;
39 u64 end;
Chris Masond1310b22008-01-24 16:13:08 -050040 struct rb_node rb_node;
41};
42
43struct extent_page_data {
44 struct bio *bio;
45 struct extent_io_tree *tree;
46 get_extent_t *get_extent;
Chris Mason771ed682008-11-06 22:02:51 -050047
48 /* tells writepage not to lock the state bits for this range
49 * it still does the unlocking
50 */
Chris Masonffbd5172009-04-20 15:50:09 -040051 unsigned int extent_locked:1;
52
53 /* tells the submit_bio code to use a WRITE_SYNC */
54 unsigned int sync_io:1;
Chris Masond1310b22008-01-24 16:13:08 -050055};
56
Josef Bacik0b32f4b2012-03-13 09:38:00 -040057static noinline void flush_write_bio(void *data);
Jeff Mahoneyc2d904e2011-10-03 23:22:32 -040058static inline struct btrfs_fs_info *
59tree_fs_info(struct extent_io_tree *tree)
60{
61 return btrfs_sb(tree->mapping->host->i_sb);
62}
Josef Bacik0b32f4b2012-03-13 09:38:00 -040063
Chris Masond1310b22008-01-24 16:13:08 -050064int __init extent_io_init(void)
65{
Christoph Hellwig9601e3f2009-04-13 15:33:09 +020066 extent_state_cache = kmem_cache_create("extent_state",
67 sizeof(struct extent_state), 0,
68 SLAB_RECLAIM_ACCOUNT | SLAB_MEM_SPREAD, NULL);
Chris Masond1310b22008-01-24 16:13:08 -050069 if (!extent_state_cache)
70 return -ENOMEM;
71
Christoph Hellwig9601e3f2009-04-13 15:33:09 +020072 extent_buffer_cache = kmem_cache_create("extent_buffers",
73 sizeof(struct extent_buffer), 0,
74 SLAB_RECLAIM_ACCOUNT | SLAB_MEM_SPREAD, NULL);
Chris Masond1310b22008-01-24 16:13:08 -050075 if (!extent_buffer_cache)
76 goto free_state_cache;
77 return 0;
78
79free_state_cache:
80 kmem_cache_destroy(extent_state_cache);
81 return -ENOMEM;
82}
83
84void extent_io_exit(void)
85{
86 struct extent_state *state;
Chris Mason2d2ae542008-03-26 16:24:23 -040087 struct extent_buffer *eb;
Chris Masond1310b22008-01-24 16:13:08 -050088
89 while (!list_empty(&states)) {
Chris Mason2d2ae542008-03-26 16:24:23 -040090 state = list_entry(states.next, struct extent_state, leak_list);
Chris Masond3977122009-01-05 21:25:51 -050091 printk(KERN_ERR "btrfs state leak: start %llu end %llu "
92 "state %lu in tree %p refs %d\n",
93 (unsigned long long)state->start,
94 (unsigned long long)state->end,
95 state->state, state->tree, atomic_read(&state->refs));
Chris Mason2d2ae542008-03-26 16:24:23 -040096 list_del(&state->leak_list);
Chris Masond1310b22008-01-24 16:13:08 -050097 kmem_cache_free(extent_state_cache, state);
98
99 }
100
Chris Mason2d2ae542008-03-26 16:24:23 -0400101 while (!list_empty(&buffers)) {
102 eb = list_entry(buffers.next, struct extent_buffer, leak_list);
Chris Masond3977122009-01-05 21:25:51 -0500103 printk(KERN_ERR "btrfs buffer leak start %llu len %lu "
104 "refs %d\n", (unsigned long long)eb->start,
105 eb->len, atomic_read(&eb->refs));
Chris Mason2d2ae542008-03-26 16:24:23 -0400106 list_del(&eb->leak_list);
107 kmem_cache_free(extent_buffer_cache, eb);
108 }
Chris Masond1310b22008-01-24 16:13:08 -0500109 if (extent_state_cache)
110 kmem_cache_destroy(extent_state_cache);
111 if (extent_buffer_cache)
112 kmem_cache_destroy(extent_buffer_cache);
113}
114
115void extent_io_tree_init(struct extent_io_tree *tree,
David Sterbaf993c882011-04-20 23:35:57 +0200116 struct address_space *mapping)
Chris Masond1310b22008-01-24 16:13:08 -0500117{
Eric Paris6bef4d32010-02-23 19:43:04 +0000118 tree->state = RB_ROOT;
Miao Xie19fe0a82010-10-26 20:57:29 -0400119 INIT_RADIX_TREE(&tree->buffer, GFP_ATOMIC);
Chris Masond1310b22008-01-24 16:13:08 -0500120 tree->ops = NULL;
121 tree->dirty_bytes = 0;
Chris Mason70dec802008-01-29 09:59:12 -0500122 spin_lock_init(&tree->lock);
Chris Mason6af118c2008-07-22 11:18:07 -0400123 spin_lock_init(&tree->buffer_lock);
Chris Masond1310b22008-01-24 16:13:08 -0500124 tree->mapping = mapping;
Chris Masond1310b22008-01-24 16:13:08 -0500125}
Chris Masond1310b22008-01-24 16:13:08 -0500126
Christoph Hellwigb2950862008-12-02 09:54:17 -0500127static struct extent_state *alloc_extent_state(gfp_t mask)
Chris Masond1310b22008-01-24 16:13:08 -0500128{
129 struct extent_state *state;
Chris Mason39351272009-02-04 09:24:05 -0500130#if LEAK_DEBUG
Chris Mason2d2ae542008-03-26 16:24:23 -0400131 unsigned long flags;
Chris Mason4bef0842008-09-08 11:18:08 -0400132#endif
Chris Masond1310b22008-01-24 16:13:08 -0500133
134 state = kmem_cache_alloc(extent_state_cache, mask);
Peter2b114d12008-04-01 11:21:40 -0400135 if (!state)
Chris Masond1310b22008-01-24 16:13:08 -0500136 return state;
137 state->state = 0;
Chris Masond1310b22008-01-24 16:13:08 -0500138 state->private = 0;
Chris Mason70dec802008-01-29 09:59:12 -0500139 state->tree = NULL;
Chris Mason39351272009-02-04 09:24:05 -0500140#if LEAK_DEBUG
Chris Mason2d2ae542008-03-26 16:24:23 -0400141 spin_lock_irqsave(&leak_lock, flags);
142 list_add(&state->leak_list, &states);
143 spin_unlock_irqrestore(&leak_lock, flags);
Chris Mason4bef0842008-09-08 11:18:08 -0400144#endif
Chris Masond1310b22008-01-24 16:13:08 -0500145 atomic_set(&state->refs, 1);
146 init_waitqueue_head(&state->wq);
Jeff Mahoney143bede2012-03-01 14:56:26 +0100147 trace_alloc_extent_state(state, mask, _RET_IP_);
Chris Masond1310b22008-01-24 16:13:08 -0500148 return state;
149}
Chris Masond1310b22008-01-24 16:13:08 -0500150
Chris Mason4845e442010-05-25 20:56:50 -0400151void free_extent_state(struct extent_state *state)
Chris Masond1310b22008-01-24 16:13:08 -0500152{
Chris Masond1310b22008-01-24 16:13:08 -0500153 if (!state)
154 return;
155 if (atomic_dec_and_test(&state->refs)) {
Chris Mason39351272009-02-04 09:24:05 -0500156#if LEAK_DEBUG
Chris Mason2d2ae542008-03-26 16:24:23 -0400157 unsigned long flags;
Chris Mason4bef0842008-09-08 11:18:08 -0400158#endif
Chris Mason70dec802008-01-29 09:59:12 -0500159 WARN_ON(state->tree);
Chris Mason39351272009-02-04 09:24:05 -0500160#if LEAK_DEBUG
Chris Mason2d2ae542008-03-26 16:24:23 -0400161 spin_lock_irqsave(&leak_lock, flags);
162 list_del(&state->leak_list);
163 spin_unlock_irqrestore(&leak_lock, flags);
Chris Mason4bef0842008-09-08 11:18:08 -0400164#endif
Jeff Mahoney143bede2012-03-01 14:56:26 +0100165 trace_free_extent_state(state, _RET_IP_);
Chris Masond1310b22008-01-24 16:13:08 -0500166 kmem_cache_free(extent_state_cache, state);
167 }
168}
Chris Masond1310b22008-01-24 16:13:08 -0500169
170static struct rb_node *tree_insert(struct rb_root *root, u64 offset,
171 struct rb_node *node)
172{
Chris Masond3977122009-01-05 21:25:51 -0500173 struct rb_node **p = &root->rb_node;
174 struct rb_node *parent = NULL;
Chris Masond1310b22008-01-24 16:13:08 -0500175 struct tree_entry *entry;
176
Chris Masond3977122009-01-05 21:25:51 -0500177 while (*p) {
Chris Masond1310b22008-01-24 16:13:08 -0500178 parent = *p;
179 entry = rb_entry(parent, struct tree_entry, rb_node);
180
181 if (offset < entry->start)
182 p = &(*p)->rb_left;
183 else if (offset > entry->end)
184 p = &(*p)->rb_right;
185 else
186 return parent;
187 }
188
189 entry = rb_entry(node, struct tree_entry, rb_node);
Chris Masond1310b22008-01-24 16:13:08 -0500190 rb_link_node(node, parent, p);
191 rb_insert_color(node, root);
192 return NULL;
193}
194
Chris Mason80ea96b2008-02-01 14:51:59 -0500195static struct rb_node *__etree_search(struct extent_io_tree *tree, u64 offset,
Chris Masond1310b22008-01-24 16:13:08 -0500196 struct rb_node **prev_ret,
197 struct rb_node **next_ret)
198{
Chris Mason80ea96b2008-02-01 14:51:59 -0500199 struct rb_root *root = &tree->state;
Chris Masond3977122009-01-05 21:25:51 -0500200 struct rb_node *n = root->rb_node;
Chris Masond1310b22008-01-24 16:13:08 -0500201 struct rb_node *prev = NULL;
202 struct rb_node *orig_prev = NULL;
203 struct tree_entry *entry;
204 struct tree_entry *prev_entry = NULL;
205
Chris Masond3977122009-01-05 21:25:51 -0500206 while (n) {
Chris Masond1310b22008-01-24 16:13:08 -0500207 entry = rb_entry(n, struct tree_entry, rb_node);
208 prev = n;
209 prev_entry = entry;
210
211 if (offset < entry->start)
212 n = n->rb_left;
213 else if (offset > entry->end)
214 n = n->rb_right;
Chris Masond3977122009-01-05 21:25:51 -0500215 else
Chris Masond1310b22008-01-24 16:13:08 -0500216 return n;
217 }
218
219 if (prev_ret) {
220 orig_prev = prev;
Chris Masond3977122009-01-05 21:25:51 -0500221 while (prev && offset > prev_entry->end) {
Chris Masond1310b22008-01-24 16:13:08 -0500222 prev = rb_next(prev);
223 prev_entry = rb_entry(prev, struct tree_entry, rb_node);
224 }
225 *prev_ret = prev;
226 prev = orig_prev;
227 }
228
229 if (next_ret) {
230 prev_entry = rb_entry(prev, struct tree_entry, rb_node);
Chris Masond3977122009-01-05 21:25:51 -0500231 while (prev && offset < prev_entry->start) {
Chris Masond1310b22008-01-24 16:13:08 -0500232 prev = rb_prev(prev);
233 prev_entry = rb_entry(prev, struct tree_entry, rb_node);
234 }
235 *next_ret = prev;
236 }
237 return NULL;
238}
239
Chris Mason80ea96b2008-02-01 14:51:59 -0500240static inline struct rb_node *tree_search(struct extent_io_tree *tree,
241 u64 offset)
Chris Masond1310b22008-01-24 16:13:08 -0500242{
Chris Mason70dec802008-01-29 09:59:12 -0500243 struct rb_node *prev = NULL;
Chris Masond1310b22008-01-24 16:13:08 -0500244 struct rb_node *ret;
Chris Mason70dec802008-01-29 09:59:12 -0500245
Chris Mason80ea96b2008-02-01 14:51:59 -0500246 ret = __etree_search(tree, offset, &prev, NULL);
Chris Masond3977122009-01-05 21:25:51 -0500247 if (!ret)
Chris Masond1310b22008-01-24 16:13:08 -0500248 return prev;
249 return ret;
250}
251
Josef Bacik9ed74f22009-09-11 16:12:44 -0400252static void merge_cb(struct extent_io_tree *tree, struct extent_state *new,
253 struct extent_state *other)
254{
255 if (tree->ops && tree->ops->merge_extent_hook)
256 tree->ops->merge_extent_hook(tree->mapping->host, new,
257 other);
258}
259
Chris Masond1310b22008-01-24 16:13:08 -0500260/*
261 * utility function to look for merge candidates inside a given range.
262 * Any extents with matching state are merged together into a single
263 * extent in the tree. Extents with EXTENT_IO in their state field
264 * are not merged because the end_io handlers need to be able to do
265 * operations on them without sleeping (or doing allocations/splits).
266 *
267 * This should be called with the tree lock held.
268 */
Jeff Mahoney1bf85042011-07-21 16:56:09 +0000269static void merge_state(struct extent_io_tree *tree,
270 struct extent_state *state)
Chris Masond1310b22008-01-24 16:13:08 -0500271{
272 struct extent_state *other;
273 struct rb_node *other_node;
274
Zheng Yan5b21f2e2008-09-26 10:05:38 -0400275 if (state->state & (EXTENT_IOBITS | EXTENT_BOUNDARY))
Jeff Mahoney1bf85042011-07-21 16:56:09 +0000276 return;
Chris Masond1310b22008-01-24 16:13:08 -0500277
278 other_node = rb_prev(&state->rb_node);
279 if (other_node) {
280 other = rb_entry(other_node, struct extent_state, rb_node);
281 if (other->end == state->start - 1 &&
282 other->state == state->state) {
Josef Bacik9ed74f22009-09-11 16:12:44 -0400283 merge_cb(tree, state, other);
Chris Masond1310b22008-01-24 16:13:08 -0500284 state->start = other->start;
Chris Mason70dec802008-01-29 09:59:12 -0500285 other->tree = NULL;
Chris Masond1310b22008-01-24 16:13:08 -0500286 rb_erase(&other->rb_node, &tree->state);
287 free_extent_state(other);
288 }
289 }
290 other_node = rb_next(&state->rb_node);
291 if (other_node) {
292 other = rb_entry(other_node, struct extent_state, rb_node);
293 if (other->start == state->end + 1 &&
294 other->state == state->state) {
Josef Bacik9ed74f22009-09-11 16:12:44 -0400295 merge_cb(tree, state, other);
Josef Bacikdf98b6e2011-06-20 14:53:48 -0400296 state->end = other->end;
297 other->tree = NULL;
298 rb_erase(&other->rb_node, &tree->state);
299 free_extent_state(other);
Chris Masond1310b22008-01-24 16:13:08 -0500300 }
301 }
Chris Masond1310b22008-01-24 16:13:08 -0500302}
303
Jeff Mahoney1bf85042011-07-21 16:56:09 +0000304static void set_state_cb(struct extent_io_tree *tree,
Yan, Zheng0ca1f7c2010-05-16 10:48:47 -0400305 struct extent_state *state, int *bits)
Chris Mason291d6732008-01-29 15:55:23 -0500306{
Jeff Mahoney1bf85042011-07-21 16:56:09 +0000307 if (tree->ops && tree->ops->set_bit_hook)
308 tree->ops->set_bit_hook(tree->mapping->host, state, bits);
Chris Mason291d6732008-01-29 15:55:23 -0500309}
310
311static void clear_state_cb(struct extent_io_tree *tree,
Yan, Zheng0ca1f7c2010-05-16 10:48:47 -0400312 struct extent_state *state, int *bits)
Chris Mason291d6732008-01-29 15:55:23 -0500313{
Josef Bacik9ed74f22009-09-11 16:12:44 -0400314 if (tree->ops && tree->ops->clear_bit_hook)
315 tree->ops->clear_bit_hook(tree->mapping->host, state, bits);
Chris Mason291d6732008-01-29 15:55:23 -0500316}
317
Xiao Guangrong3150b692011-07-14 03:19:08 +0000318static void set_state_bits(struct extent_io_tree *tree,
319 struct extent_state *state, int *bits);
320
Chris Masond1310b22008-01-24 16:13:08 -0500321/*
322 * insert an extent_state struct into the tree. 'bits' are set on the
323 * struct before it is inserted.
324 *
325 * This may return -EEXIST if the extent is already there, in which case the
326 * state struct is freed.
327 *
328 * The tree lock is not taken internally. This is a utility function and
329 * probably isn't what you want to call (see set/clear_extent_bit).
330 */
331static int insert_state(struct extent_io_tree *tree,
332 struct extent_state *state, u64 start, u64 end,
Yan, Zheng0ca1f7c2010-05-16 10:48:47 -0400333 int *bits)
Chris Masond1310b22008-01-24 16:13:08 -0500334{
335 struct rb_node *node;
336
337 if (end < start) {
Chris Masond3977122009-01-05 21:25:51 -0500338 printk(KERN_ERR "btrfs end < start %llu %llu\n",
339 (unsigned long long)end,
340 (unsigned long long)start);
Chris Masond1310b22008-01-24 16:13:08 -0500341 WARN_ON(1);
342 }
Chris Masond1310b22008-01-24 16:13:08 -0500343 state->start = start;
344 state->end = end;
Josef Bacik9ed74f22009-09-11 16:12:44 -0400345
Xiao Guangrong3150b692011-07-14 03:19:08 +0000346 set_state_bits(tree, state, bits);
347
Chris Masond1310b22008-01-24 16:13:08 -0500348 node = tree_insert(&tree->state, end, &state->rb_node);
349 if (node) {
350 struct extent_state *found;
351 found = rb_entry(node, struct extent_state, rb_node);
Chris Masond3977122009-01-05 21:25:51 -0500352 printk(KERN_ERR "btrfs found node %llu %llu on insert of "
353 "%llu %llu\n", (unsigned long long)found->start,
354 (unsigned long long)found->end,
355 (unsigned long long)start, (unsigned long long)end);
Chris Masond1310b22008-01-24 16:13:08 -0500356 return -EEXIST;
357 }
Chris Mason70dec802008-01-29 09:59:12 -0500358 state->tree = tree;
Chris Masond1310b22008-01-24 16:13:08 -0500359 merge_state(tree, state);
360 return 0;
361}
362
Jeff Mahoney1bf85042011-07-21 16:56:09 +0000363static void split_cb(struct extent_io_tree *tree, struct extent_state *orig,
Josef Bacik9ed74f22009-09-11 16:12:44 -0400364 u64 split)
365{
366 if (tree->ops && tree->ops->split_extent_hook)
Jeff Mahoney1bf85042011-07-21 16:56:09 +0000367 tree->ops->split_extent_hook(tree->mapping->host, orig, split);
Josef Bacik9ed74f22009-09-11 16:12:44 -0400368}
369
Chris Masond1310b22008-01-24 16:13:08 -0500370/*
371 * split a given extent state struct in two, inserting the preallocated
372 * struct 'prealloc' as the newly created second half. 'split' indicates an
373 * offset inside 'orig' where it should be split.
374 *
375 * Before calling,
376 * the tree has 'orig' at [orig->start, orig->end]. After calling, there
377 * are two extent state structs in the tree:
378 * prealloc: [orig->start, split - 1]
379 * orig: [ split, orig->end ]
380 *
381 * The tree locks are not taken by this function. They need to be held
382 * by the caller.
383 */
384static int split_state(struct extent_io_tree *tree, struct extent_state *orig,
385 struct extent_state *prealloc, u64 split)
386{
387 struct rb_node *node;
Josef Bacik9ed74f22009-09-11 16:12:44 -0400388
389 split_cb(tree, orig, split);
390
Chris Masond1310b22008-01-24 16:13:08 -0500391 prealloc->start = orig->start;
392 prealloc->end = split - 1;
393 prealloc->state = orig->state;
394 orig->start = split;
395
396 node = tree_insert(&tree->state, prealloc->end, &prealloc->rb_node);
397 if (node) {
Chris Masond1310b22008-01-24 16:13:08 -0500398 free_extent_state(prealloc);
399 return -EEXIST;
400 }
Chris Mason70dec802008-01-29 09:59:12 -0500401 prealloc->tree = tree;
Chris Masond1310b22008-01-24 16:13:08 -0500402 return 0;
403}
404
Li Zefancdc6a392012-03-12 16:39:48 +0800405static struct extent_state *next_state(struct extent_state *state)
406{
407 struct rb_node *next = rb_next(&state->rb_node);
408 if (next)
409 return rb_entry(next, struct extent_state, rb_node);
410 else
411 return NULL;
412}
413
Chris Masond1310b22008-01-24 16:13:08 -0500414/*
415 * utility function to clear some bits in an extent state struct.
Li Zefan8e52acf2012-03-12 16:39:28 +0800416 * it will optionally wake up any one waiting on this state (wake == 1)
Chris Masond1310b22008-01-24 16:13:08 -0500417 *
418 * If no bits are set on the state struct after clearing things, the
419 * struct is freed and removed from the tree
420 */
Li Zefancdc6a392012-03-12 16:39:48 +0800421static struct extent_state *clear_state_bit(struct extent_io_tree *tree,
422 struct extent_state *state,
423 int *bits, int wake)
Chris Masond1310b22008-01-24 16:13:08 -0500424{
Li Zefancdc6a392012-03-12 16:39:48 +0800425 struct extent_state *next;
Yan, Zheng0ca1f7c2010-05-16 10:48:47 -0400426 int bits_to_clear = *bits & ~EXTENT_CTLBITS;
Chris Masond1310b22008-01-24 16:13:08 -0500427
Yan, Zheng0ca1f7c2010-05-16 10:48:47 -0400428 if ((bits_to_clear & EXTENT_DIRTY) && (state->state & EXTENT_DIRTY)) {
Chris Masond1310b22008-01-24 16:13:08 -0500429 u64 range = state->end - state->start + 1;
430 WARN_ON(range > tree->dirty_bytes);
431 tree->dirty_bytes -= range;
432 }
Chris Mason291d6732008-01-29 15:55:23 -0500433 clear_state_cb(tree, state, bits);
Josef Bacik32c00af2009-10-08 13:34:05 -0400434 state->state &= ~bits_to_clear;
Chris Masond1310b22008-01-24 16:13:08 -0500435 if (wake)
436 wake_up(&state->wq);
Yan, Zheng0ca1f7c2010-05-16 10:48:47 -0400437 if (state->state == 0) {
Li Zefancdc6a392012-03-12 16:39:48 +0800438 next = next_state(state);
Chris Mason70dec802008-01-29 09:59:12 -0500439 if (state->tree) {
Chris Masond1310b22008-01-24 16:13:08 -0500440 rb_erase(&state->rb_node, &tree->state);
Chris Mason70dec802008-01-29 09:59:12 -0500441 state->tree = NULL;
Chris Masond1310b22008-01-24 16:13:08 -0500442 free_extent_state(state);
443 } else {
444 WARN_ON(1);
445 }
446 } else {
447 merge_state(tree, state);
Li Zefancdc6a392012-03-12 16:39:48 +0800448 next = next_state(state);
Chris Masond1310b22008-01-24 16:13:08 -0500449 }
Li Zefancdc6a392012-03-12 16:39:48 +0800450 return next;
Chris Masond1310b22008-01-24 16:13:08 -0500451}
452
Xiao Guangrong82337672011-04-20 06:44:57 +0000453static struct extent_state *
454alloc_extent_state_atomic(struct extent_state *prealloc)
455{
456 if (!prealloc)
457 prealloc = alloc_extent_state(GFP_ATOMIC);
458
459 return prealloc;
460}
461
Jeff Mahoneyc2d904e2011-10-03 23:22:32 -0400462void extent_io_tree_panic(struct extent_io_tree *tree, int err)
463{
464 btrfs_panic(tree_fs_info(tree), err, "Locking error: "
465 "Extent tree was modified by another "
466 "thread while locked.");
467}
468
Chris Masond1310b22008-01-24 16:13:08 -0500469/*
470 * clear some bits on a range in the tree. This may require splitting
471 * or inserting elements in the tree, so the gfp mask is used to
472 * indicate which allocations or sleeping are allowed.
473 *
474 * pass 'wake' == 1 to kick any sleepers, and 'delete' == 1 to remove
475 * the given range from the tree regardless of state (ie for truncate).
476 *
477 * the range [start, end] is inclusive.
478 *
Jeff Mahoney6763af82012-03-01 14:56:29 +0100479 * This takes the tree lock, and returns 0 on success and < 0 on error.
Chris Masond1310b22008-01-24 16:13:08 -0500480 */
481int clear_extent_bit(struct extent_io_tree *tree, u64 start, u64 end,
Chris Mason2c64c532009-09-02 15:04:12 -0400482 int bits, int wake, int delete,
483 struct extent_state **cached_state,
484 gfp_t mask)
Chris Masond1310b22008-01-24 16:13:08 -0500485{
486 struct extent_state *state;
Chris Mason2c64c532009-09-02 15:04:12 -0400487 struct extent_state *cached;
Chris Masond1310b22008-01-24 16:13:08 -0500488 struct extent_state *prealloc = NULL;
489 struct rb_node *node;
Yan Zheng5c939df2009-05-27 09:16:03 -0400490 u64 last_end;
Chris Masond1310b22008-01-24 16:13:08 -0500491 int err;
Josef Bacik2ac55d42010-02-03 19:33:23 +0000492 int clear = 0;
Chris Masond1310b22008-01-24 16:13:08 -0500493
Yan, Zheng0ca1f7c2010-05-16 10:48:47 -0400494 if (delete)
495 bits |= ~EXTENT_CTLBITS;
496 bits |= EXTENT_FIRST_DELALLOC;
497
Josef Bacik2ac55d42010-02-03 19:33:23 +0000498 if (bits & (EXTENT_IOBITS | EXTENT_BOUNDARY))
499 clear = 1;
Chris Masond1310b22008-01-24 16:13:08 -0500500again:
501 if (!prealloc && (mask & __GFP_WAIT)) {
502 prealloc = alloc_extent_state(mask);
503 if (!prealloc)
504 return -ENOMEM;
505 }
506
Chris Masoncad321a2008-12-17 14:51:42 -0500507 spin_lock(&tree->lock);
Chris Mason2c64c532009-09-02 15:04:12 -0400508 if (cached_state) {
509 cached = *cached_state;
Josef Bacik2ac55d42010-02-03 19:33:23 +0000510
511 if (clear) {
512 *cached_state = NULL;
513 cached_state = NULL;
514 }
515
Josef Bacikdf98b6e2011-06-20 14:53:48 -0400516 if (cached && cached->tree && cached->start <= start &&
517 cached->end > start) {
Josef Bacik2ac55d42010-02-03 19:33:23 +0000518 if (clear)
519 atomic_dec(&cached->refs);
Chris Mason2c64c532009-09-02 15:04:12 -0400520 state = cached;
Chris Mason42daec22009-09-23 19:51:09 -0400521 goto hit_next;
Chris Mason2c64c532009-09-02 15:04:12 -0400522 }
Josef Bacik2ac55d42010-02-03 19:33:23 +0000523 if (clear)
524 free_extent_state(cached);
Chris Mason2c64c532009-09-02 15:04:12 -0400525 }
Chris Masond1310b22008-01-24 16:13:08 -0500526 /*
527 * this search will find the extents that end after
528 * our range starts
529 */
Chris Mason80ea96b2008-02-01 14:51:59 -0500530 node = tree_search(tree, start);
Chris Masond1310b22008-01-24 16:13:08 -0500531 if (!node)
532 goto out;
533 state = rb_entry(node, struct extent_state, rb_node);
Chris Mason2c64c532009-09-02 15:04:12 -0400534hit_next:
Chris Masond1310b22008-01-24 16:13:08 -0500535 if (state->start > end)
536 goto out;
537 WARN_ON(state->end < start);
Yan Zheng5c939df2009-05-27 09:16:03 -0400538 last_end = state->end;
Chris Masond1310b22008-01-24 16:13:08 -0500539
Liu Bo04493142012-02-16 18:34:37 +0800540 /* the state doesn't have the wanted bits, go ahead */
Li Zefancdc6a392012-03-12 16:39:48 +0800541 if (!(state->state & bits)) {
542 state = next_state(state);
Liu Bo04493142012-02-16 18:34:37 +0800543 goto next;
Li Zefancdc6a392012-03-12 16:39:48 +0800544 }
Liu Bo04493142012-02-16 18:34:37 +0800545
Chris Masond1310b22008-01-24 16:13:08 -0500546 /*
547 * | ---- desired range ---- |
548 * | state | or
549 * | ------------- state -------------- |
550 *
551 * We need to split the extent we found, and may flip
552 * bits on second half.
553 *
554 * If the extent we found extends past our range, we
555 * just split and search again. It'll get split again
556 * the next time though.
557 *
558 * If the extent we found is inside our range, we clear
559 * the desired bit on it.
560 */
561
562 if (state->start < start) {
Xiao Guangrong82337672011-04-20 06:44:57 +0000563 prealloc = alloc_extent_state_atomic(prealloc);
564 BUG_ON(!prealloc);
Chris Masond1310b22008-01-24 16:13:08 -0500565 err = split_state(tree, state, prealloc, start);
Jeff Mahoneyc2d904e2011-10-03 23:22:32 -0400566 if (err)
567 extent_io_tree_panic(tree, err);
568
Chris Masond1310b22008-01-24 16:13:08 -0500569 prealloc = NULL;
570 if (err)
571 goto out;
572 if (state->end <= end) {
Jeff Mahoney6763af82012-03-01 14:56:29 +0100573 clear_state_bit(tree, state, &bits, wake);
Yan Zheng5c939df2009-05-27 09:16:03 -0400574 if (last_end == (u64)-1)
575 goto out;
576 start = last_end + 1;
Chris Masond1310b22008-01-24 16:13:08 -0500577 }
578 goto search_again;
579 }
580 /*
581 * | ---- desired range ---- |
582 * | state |
583 * We need to split the extent, and clear the bit
584 * on the first half
585 */
586 if (state->start <= end && state->end > end) {
Xiao Guangrong82337672011-04-20 06:44:57 +0000587 prealloc = alloc_extent_state_atomic(prealloc);
588 BUG_ON(!prealloc);
Chris Masond1310b22008-01-24 16:13:08 -0500589 err = split_state(tree, state, prealloc, end + 1);
Jeff Mahoneyc2d904e2011-10-03 23:22:32 -0400590 if (err)
591 extent_io_tree_panic(tree, err);
592
Chris Masond1310b22008-01-24 16:13:08 -0500593 if (wake)
594 wake_up(&state->wq);
Chris Mason42daec22009-09-23 19:51:09 -0400595
Jeff Mahoney6763af82012-03-01 14:56:29 +0100596 clear_state_bit(tree, prealloc, &bits, wake);
Josef Bacik9ed74f22009-09-11 16:12:44 -0400597
Chris Masond1310b22008-01-24 16:13:08 -0500598 prealloc = NULL;
599 goto out;
600 }
Chris Mason42daec22009-09-23 19:51:09 -0400601
Li Zefancdc6a392012-03-12 16:39:48 +0800602 state = clear_state_bit(tree, state, &bits, wake);
Liu Bo04493142012-02-16 18:34:37 +0800603next:
Yan Zheng5c939df2009-05-27 09:16:03 -0400604 if (last_end == (u64)-1)
605 goto out;
606 start = last_end + 1;
Li Zefancdc6a392012-03-12 16:39:48 +0800607 if (start <= end && state && !need_resched())
Liu Bo692e5752012-02-16 18:34:36 +0800608 goto hit_next;
Chris Masond1310b22008-01-24 16:13:08 -0500609 goto search_again;
610
611out:
Chris Masoncad321a2008-12-17 14:51:42 -0500612 spin_unlock(&tree->lock);
Chris Masond1310b22008-01-24 16:13:08 -0500613 if (prealloc)
614 free_extent_state(prealloc);
615
Jeff Mahoney6763af82012-03-01 14:56:29 +0100616 return 0;
Chris Masond1310b22008-01-24 16:13:08 -0500617
618search_again:
619 if (start > end)
620 goto out;
Chris Masoncad321a2008-12-17 14:51:42 -0500621 spin_unlock(&tree->lock);
Chris Masond1310b22008-01-24 16:13:08 -0500622 if (mask & __GFP_WAIT)
623 cond_resched();
624 goto again;
625}
Chris Masond1310b22008-01-24 16:13:08 -0500626
Jeff Mahoney143bede2012-03-01 14:56:26 +0100627static void wait_on_state(struct extent_io_tree *tree,
628 struct extent_state *state)
Christoph Hellwig641f5212008-12-02 06:36:10 -0500629 __releases(tree->lock)
630 __acquires(tree->lock)
Chris Masond1310b22008-01-24 16:13:08 -0500631{
632 DEFINE_WAIT(wait);
633 prepare_to_wait(&state->wq, &wait, TASK_UNINTERRUPTIBLE);
Chris Masoncad321a2008-12-17 14:51:42 -0500634 spin_unlock(&tree->lock);
Chris Masond1310b22008-01-24 16:13:08 -0500635 schedule();
Chris Masoncad321a2008-12-17 14:51:42 -0500636 spin_lock(&tree->lock);
Chris Masond1310b22008-01-24 16:13:08 -0500637 finish_wait(&state->wq, &wait);
Chris Masond1310b22008-01-24 16:13:08 -0500638}
639
640/*
641 * waits for one or more bits to clear on a range in the state tree.
642 * The range [start, end] is inclusive.
643 * The tree lock is taken by this function
644 */
Jeff Mahoney143bede2012-03-01 14:56:26 +0100645void wait_extent_bit(struct extent_io_tree *tree, u64 start, u64 end, int bits)
Chris Masond1310b22008-01-24 16:13:08 -0500646{
647 struct extent_state *state;
648 struct rb_node *node;
649
Chris Masoncad321a2008-12-17 14:51:42 -0500650 spin_lock(&tree->lock);
Chris Masond1310b22008-01-24 16:13:08 -0500651again:
652 while (1) {
653 /*
654 * this search will find all the extents that end after
655 * our range starts
656 */
Chris Mason80ea96b2008-02-01 14:51:59 -0500657 node = tree_search(tree, start);
Chris Masond1310b22008-01-24 16:13:08 -0500658 if (!node)
659 break;
660
661 state = rb_entry(node, struct extent_state, rb_node);
662
663 if (state->start > end)
664 goto out;
665
666 if (state->state & bits) {
667 start = state->start;
668 atomic_inc(&state->refs);
669 wait_on_state(tree, state);
670 free_extent_state(state);
671 goto again;
672 }
673 start = state->end + 1;
674
675 if (start > end)
676 break;
677
Xiao Guangrongded91f02011-07-14 03:19:27 +0000678 cond_resched_lock(&tree->lock);
Chris Masond1310b22008-01-24 16:13:08 -0500679 }
680out:
Chris Masoncad321a2008-12-17 14:51:42 -0500681 spin_unlock(&tree->lock);
Chris Masond1310b22008-01-24 16:13:08 -0500682}
Chris Masond1310b22008-01-24 16:13:08 -0500683
Jeff Mahoney1bf85042011-07-21 16:56:09 +0000684static void set_state_bits(struct extent_io_tree *tree,
Chris Masond1310b22008-01-24 16:13:08 -0500685 struct extent_state *state,
Yan, Zheng0ca1f7c2010-05-16 10:48:47 -0400686 int *bits)
Chris Masond1310b22008-01-24 16:13:08 -0500687{
Yan, Zheng0ca1f7c2010-05-16 10:48:47 -0400688 int bits_to_set = *bits & ~EXTENT_CTLBITS;
Josef Bacik9ed74f22009-09-11 16:12:44 -0400689
Jeff Mahoney1bf85042011-07-21 16:56:09 +0000690 set_state_cb(tree, state, bits);
Yan, Zheng0ca1f7c2010-05-16 10:48:47 -0400691 if ((bits_to_set & EXTENT_DIRTY) && !(state->state & EXTENT_DIRTY)) {
Chris Masond1310b22008-01-24 16:13:08 -0500692 u64 range = state->end - state->start + 1;
693 tree->dirty_bytes += range;
694 }
Yan, Zheng0ca1f7c2010-05-16 10:48:47 -0400695 state->state |= bits_to_set;
Chris Masond1310b22008-01-24 16:13:08 -0500696}
697
Chris Mason2c64c532009-09-02 15:04:12 -0400698static void cache_state(struct extent_state *state,
699 struct extent_state **cached_ptr)
700{
701 if (cached_ptr && !(*cached_ptr)) {
702 if (state->state & (EXTENT_IOBITS | EXTENT_BOUNDARY)) {
703 *cached_ptr = state;
704 atomic_inc(&state->refs);
705 }
706 }
707}
708
Arne Jansen507903b2011-04-06 10:02:20 +0000709static void uncache_state(struct extent_state **cached_ptr)
710{
711 if (cached_ptr && (*cached_ptr)) {
712 struct extent_state *state = *cached_ptr;
Chris Mason109b36a2011-04-12 13:57:39 -0400713 *cached_ptr = NULL;
714 free_extent_state(state);
Arne Jansen507903b2011-04-06 10:02:20 +0000715 }
716}
717
Chris Masond1310b22008-01-24 16:13:08 -0500718/*
Chris Mason1edbb732009-09-02 13:24:36 -0400719 * set some bits on a range in the tree. This may require allocations or
720 * sleeping, so the gfp mask is used to indicate what is allowed.
Chris Masond1310b22008-01-24 16:13:08 -0500721 *
Chris Mason1edbb732009-09-02 13:24:36 -0400722 * If any of the exclusive bits are set, this will fail with -EEXIST if some
723 * part of the range already has the desired bits set. The start of the
724 * existing range is returned in failed_start in this case.
Chris Masond1310b22008-01-24 16:13:08 -0500725 *
Chris Mason1edbb732009-09-02 13:24:36 -0400726 * [start, end] is inclusive This takes the tree lock.
Chris Masond1310b22008-01-24 16:13:08 -0500727 */
Chris Mason1edbb732009-09-02 13:24:36 -0400728
Jeff Mahoney3fbe5c02012-03-01 14:57:19 +0100729static int __must_check
730__set_extent_bit(struct extent_io_tree *tree, u64 start, u64 end,
731 int bits, int exclusive_bits, u64 *failed_start,
732 struct extent_state **cached_state, gfp_t mask)
Chris Masond1310b22008-01-24 16:13:08 -0500733{
734 struct extent_state *state;
735 struct extent_state *prealloc = NULL;
736 struct rb_node *node;
Chris Masond1310b22008-01-24 16:13:08 -0500737 int err = 0;
Chris Masond1310b22008-01-24 16:13:08 -0500738 u64 last_start;
739 u64 last_end;
Chris Mason42daec22009-09-23 19:51:09 -0400740
Yan, Zheng0ca1f7c2010-05-16 10:48:47 -0400741 bits |= EXTENT_FIRST_DELALLOC;
Chris Masond1310b22008-01-24 16:13:08 -0500742again:
743 if (!prealloc && (mask & __GFP_WAIT)) {
744 prealloc = alloc_extent_state(mask);
Xiao Guangrong82337672011-04-20 06:44:57 +0000745 BUG_ON(!prealloc);
Chris Masond1310b22008-01-24 16:13:08 -0500746 }
747
Chris Masoncad321a2008-12-17 14:51:42 -0500748 spin_lock(&tree->lock);
Chris Mason9655d292009-09-02 15:22:30 -0400749 if (cached_state && *cached_state) {
750 state = *cached_state;
Josef Bacikdf98b6e2011-06-20 14:53:48 -0400751 if (state->start <= start && state->end > start &&
752 state->tree) {
Chris Mason9655d292009-09-02 15:22:30 -0400753 node = &state->rb_node;
754 goto hit_next;
755 }
756 }
Chris Masond1310b22008-01-24 16:13:08 -0500757 /*
758 * this search will find all the extents that end after
759 * our range starts.
760 */
Chris Mason80ea96b2008-02-01 14:51:59 -0500761 node = tree_search(tree, start);
Chris Masond1310b22008-01-24 16:13:08 -0500762 if (!node) {
Xiao Guangrong82337672011-04-20 06:44:57 +0000763 prealloc = alloc_extent_state_atomic(prealloc);
764 BUG_ON(!prealloc);
Yan, Zheng0ca1f7c2010-05-16 10:48:47 -0400765 err = insert_state(tree, prealloc, start, end, &bits);
Jeff Mahoneyc2d904e2011-10-03 23:22:32 -0400766 if (err)
767 extent_io_tree_panic(tree, err);
768
Chris Masond1310b22008-01-24 16:13:08 -0500769 prealloc = NULL;
Chris Masond1310b22008-01-24 16:13:08 -0500770 goto out;
771 }
Chris Masond1310b22008-01-24 16:13:08 -0500772 state = rb_entry(node, struct extent_state, rb_node);
Chris Mason40431d62009-08-05 12:57:59 -0400773hit_next:
Chris Masond1310b22008-01-24 16:13:08 -0500774 last_start = state->start;
775 last_end = state->end;
776
777 /*
778 * | ---- desired range ---- |
779 * | state |
780 *
781 * Just lock what we found and keep going
782 */
783 if (state->start == start && state->end <= end) {
Chris Mason40431d62009-08-05 12:57:59 -0400784 struct rb_node *next_node;
Chris Mason1edbb732009-09-02 13:24:36 -0400785 if (state->state & exclusive_bits) {
Chris Masond1310b22008-01-24 16:13:08 -0500786 *failed_start = state->start;
787 err = -EEXIST;
788 goto out;
789 }
Chris Mason42daec22009-09-23 19:51:09 -0400790
Jeff Mahoney1bf85042011-07-21 16:56:09 +0000791 set_state_bits(tree, state, &bits);
Josef Bacik9ed74f22009-09-11 16:12:44 -0400792
Chris Mason2c64c532009-09-02 15:04:12 -0400793 cache_state(state, cached_state);
Chris Masond1310b22008-01-24 16:13:08 -0500794 merge_state(tree, state);
Yan Zheng5c939df2009-05-27 09:16:03 -0400795 if (last_end == (u64)-1)
796 goto out;
Chris Mason40431d62009-08-05 12:57:59 -0400797
Yan Zheng5c939df2009-05-27 09:16:03 -0400798 start = last_end + 1;
Josef Bacikdf98b6e2011-06-20 14:53:48 -0400799 next_node = rb_next(&state->rb_node);
Xiao Guangrongc7f895a2011-04-20 06:45:49 +0000800 if (next_node && start < end && prealloc && !need_resched()) {
801 state = rb_entry(next_node, struct extent_state,
802 rb_node);
803 if (state->start == start)
804 goto hit_next;
Chris Mason40431d62009-08-05 12:57:59 -0400805 }
Chris Masond1310b22008-01-24 16:13:08 -0500806 goto search_again;
807 }
808
809 /*
810 * | ---- desired range ---- |
811 * | state |
812 * or
813 * | ------------- state -------------- |
814 *
815 * We need to split the extent we found, and may flip bits on
816 * second half.
817 *
818 * If the extent we found extends past our
819 * range, we just split and search again. It'll get split
820 * again the next time though.
821 *
822 * If the extent we found is inside our range, we set the
823 * desired bit on it.
824 */
825 if (state->start < start) {
Chris Mason1edbb732009-09-02 13:24:36 -0400826 if (state->state & exclusive_bits) {
Chris Masond1310b22008-01-24 16:13:08 -0500827 *failed_start = start;
828 err = -EEXIST;
829 goto out;
830 }
Xiao Guangrong82337672011-04-20 06:44:57 +0000831
832 prealloc = alloc_extent_state_atomic(prealloc);
833 BUG_ON(!prealloc);
Chris Masond1310b22008-01-24 16:13:08 -0500834 err = split_state(tree, state, prealloc, start);
Jeff Mahoneyc2d904e2011-10-03 23:22:32 -0400835 if (err)
836 extent_io_tree_panic(tree, err);
837
Chris Masond1310b22008-01-24 16:13:08 -0500838 prealloc = NULL;
839 if (err)
840 goto out;
841 if (state->end <= end) {
Jeff Mahoney1bf85042011-07-21 16:56:09 +0000842 set_state_bits(tree, state, &bits);
Chris Mason2c64c532009-09-02 15:04:12 -0400843 cache_state(state, cached_state);
Chris Masond1310b22008-01-24 16:13:08 -0500844 merge_state(tree, state);
Yan Zheng5c939df2009-05-27 09:16:03 -0400845 if (last_end == (u64)-1)
846 goto out;
847 start = last_end + 1;
Chris Masond1310b22008-01-24 16:13:08 -0500848 }
849 goto search_again;
850 }
851 /*
852 * | ---- desired range ---- |
853 * | state | or | state |
854 *
855 * There's a hole, we need to insert something in it and
856 * ignore the extent we found.
857 */
858 if (state->start > start) {
859 u64 this_end;
860 if (end < last_start)
861 this_end = end;
862 else
Chris Masond3977122009-01-05 21:25:51 -0500863 this_end = last_start - 1;
Xiao Guangrong82337672011-04-20 06:44:57 +0000864
865 prealloc = alloc_extent_state_atomic(prealloc);
866 BUG_ON(!prealloc);
Xiao Guangrongc7f895a2011-04-20 06:45:49 +0000867
868 /*
869 * Avoid to free 'prealloc' if it can be merged with
870 * the later extent.
871 */
Chris Masond1310b22008-01-24 16:13:08 -0500872 err = insert_state(tree, prealloc, start, this_end,
Yan, Zheng0ca1f7c2010-05-16 10:48:47 -0400873 &bits);
Jeff Mahoneyc2d904e2011-10-03 23:22:32 -0400874 if (err)
875 extent_io_tree_panic(tree, err);
876
Chris Mason2c64c532009-09-02 15:04:12 -0400877 cache_state(prealloc, cached_state);
Chris Masond1310b22008-01-24 16:13:08 -0500878 prealloc = NULL;
Chris Masond1310b22008-01-24 16:13:08 -0500879 start = this_end + 1;
880 goto search_again;
881 }
882 /*
883 * | ---- desired range ---- |
884 * | state |
885 * We need to split the extent, and set the bit
886 * on the first half
887 */
888 if (state->start <= end && state->end > end) {
Chris Mason1edbb732009-09-02 13:24:36 -0400889 if (state->state & exclusive_bits) {
Chris Masond1310b22008-01-24 16:13:08 -0500890 *failed_start = start;
891 err = -EEXIST;
892 goto out;
893 }
Xiao Guangrong82337672011-04-20 06:44:57 +0000894
895 prealloc = alloc_extent_state_atomic(prealloc);
896 BUG_ON(!prealloc);
Chris Masond1310b22008-01-24 16:13:08 -0500897 err = split_state(tree, state, prealloc, end + 1);
Jeff Mahoneyc2d904e2011-10-03 23:22:32 -0400898 if (err)
899 extent_io_tree_panic(tree, err);
Chris Masond1310b22008-01-24 16:13:08 -0500900
Jeff Mahoney1bf85042011-07-21 16:56:09 +0000901 set_state_bits(tree, prealloc, &bits);
Chris Mason2c64c532009-09-02 15:04:12 -0400902 cache_state(prealloc, cached_state);
Chris Masond1310b22008-01-24 16:13:08 -0500903 merge_state(tree, prealloc);
904 prealloc = NULL;
905 goto out;
906 }
907
908 goto search_again;
909
910out:
Chris Masoncad321a2008-12-17 14:51:42 -0500911 spin_unlock(&tree->lock);
Chris Masond1310b22008-01-24 16:13:08 -0500912 if (prealloc)
913 free_extent_state(prealloc);
914
915 return err;
916
917search_again:
918 if (start > end)
919 goto out;
Chris Masoncad321a2008-12-17 14:51:42 -0500920 spin_unlock(&tree->lock);
Chris Masond1310b22008-01-24 16:13:08 -0500921 if (mask & __GFP_WAIT)
922 cond_resched();
923 goto again;
924}
Chris Masond1310b22008-01-24 16:13:08 -0500925
Jeff Mahoney3fbe5c02012-03-01 14:57:19 +0100926int set_extent_bit(struct extent_io_tree *tree, u64 start, u64 end, int bits,
927 u64 *failed_start, struct extent_state **cached_state,
928 gfp_t mask)
929{
930 return __set_extent_bit(tree, start, end, bits, 0, failed_start,
931 cached_state, mask);
932}
933
934
Josef Bacik462d6fa2011-09-26 13:56:12 -0400935/**
936 * convert_extent - convert all bits in a given range from one bit to another
937 * @tree: the io tree to search
938 * @start: the start offset in bytes
939 * @end: the end offset in bytes (inclusive)
940 * @bits: the bits to set in this range
941 * @clear_bits: the bits to clear in this range
942 * @mask: the allocation mask
943 *
944 * This will go through and set bits for the given range. If any states exist
945 * already in this range they are set with the given bit and cleared of the
946 * clear_bits. This is only meant to be used by things that are mergeable, ie
947 * converting from say DELALLOC to DIRTY. This is not meant to be used with
948 * boundary bits like LOCK.
949 */
950int convert_extent_bit(struct extent_io_tree *tree, u64 start, u64 end,
951 int bits, int clear_bits, gfp_t mask)
952{
953 struct extent_state *state;
954 struct extent_state *prealloc = NULL;
955 struct rb_node *node;
956 int err = 0;
957 u64 last_start;
958 u64 last_end;
959
960again:
961 if (!prealloc && (mask & __GFP_WAIT)) {
962 prealloc = alloc_extent_state(mask);
963 if (!prealloc)
964 return -ENOMEM;
965 }
966
967 spin_lock(&tree->lock);
968 /*
969 * this search will find all the extents that end after
970 * our range starts.
971 */
972 node = tree_search(tree, start);
973 if (!node) {
974 prealloc = alloc_extent_state_atomic(prealloc);
Liu Bo1cf4ffd2011-12-07 20:08:40 -0500975 if (!prealloc) {
976 err = -ENOMEM;
977 goto out;
978 }
Josef Bacik462d6fa2011-09-26 13:56:12 -0400979 err = insert_state(tree, prealloc, start, end, &bits);
980 prealloc = NULL;
Jeff Mahoneyc2d904e2011-10-03 23:22:32 -0400981 if (err)
982 extent_io_tree_panic(tree, err);
Josef Bacik462d6fa2011-09-26 13:56:12 -0400983 goto out;
984 }
985 state = rb_entry(node, struct extent_state, rb_node);
986hit_next:
987 last_start = state->start;
988 last_end = state->end;
989
990 /*
991 * | ---- desired range ---- |
992 * | state |
993 *
994 * Just lock what we found and keep going
995 */
996 if (state->start == start && state->end <= end) {
997 struct rb_node *next_node;
998
999 set_state_bits(tree, state, &bits);
1000 clear_state_bit(tree, state, &clear_bits, 0);
Josef Bacik462d6fa2011-09-26 13:56:12 -04001001 if (last_end == (u64)-1)
1002 goto out;
1003
1004 start = last_end + 1;
1005 next_node = rb_next(&state->rb_node);
1006 if (next_node && start < end && prealloc && !need_resched()) {
1007 state = rb_entry(next_node, struct extent_state,
1008 rb_node);
1009 if (state->start == start)
1010 goto hit_next;
1011 }
1012 goto search_again;
1013 }
1014
1015 /*
1016 * | ---- desired range ---- |
1017 * | state |
1018 * or
1019 * | ------------- state -------------- |
1020 *
1021 * We need to split the extent we found, and may flip bits on
1022 * second half.
1023 *
1024 * If the extent we found extends past our
1025 * range, we just split and search again. It'll get split
1026 * again the next time though.
1027 *
1028 * If the extent we found is inside our range, we set the
1029 * desired bit on it.
1030 */
1031 if (state->start < start) {
1032 prealloc = alloc_extent_state_atomic(prealloc);
Liu Bo1cf4ffd2011-12-07 20:08:40 -05001033 if (!prealloc) {
1034 err = -ENOMEM;
1035 goto out;
1036 }
Josef Bacik462d6fa2011-09-26 13:56:12 -04001037 err = split_state(tree, state, prealloc, start);
Jeff Mahoneyc2d904e2011-10-03 23:22:32 -04001038 if (err)
1039 extent_io_tree_panic(tree, err);
Josef Bacik462d6fa2011-09-26 13:56:12 -04001040 prealloc = NULL;
1041 if (err)
1042 goto out;
1043 if (state->end <= end) {
1044 set_state_bits(tree, state, &bits);
1045 clear_state_bit(tree, state, &clear_bits, 0);
Josef Bacik462d6fa2011-09-26 13:56:12 -04001046 if (last_end == (u64)-1)
1047 goto out;
1048 start = last_end + 1;
1049 }
1050 goto search_again;
1051 }
1052 /*
1053 * | ---- desired range ---- |
1054 * | state | or | state |
1055 *
1056 * There's a hole, we need to insert something in it and
1057 * ignore the extent we found.
1058 */
1059 if (state->start > start) {
1060 u64 this_end;
1061 if (end < last_start)
1062 this_end = end;
1063 else
1064 this_end = last_start - 1;
1065
1066 prealloc = alloc_extent_state_atomic(prealloc);
Liu Bo1cf4ffd2011-12-07 20:08:40 -05001067 if (!prealloc) {
1068 err = -ENOMEM;
1069 goto out;
1070 }
Josef Bacik462d6fa2011-09-26 13:56:12 -04001071
1072 /*
1073 * Avoid to free 'prealloc' if it can be merged with
1074 * the later extent.
1075 */
1076 err = insert_state(tree, prealloc, start, this_end,
1077 &bits);
Jeff Mahoneyc2d904e2011-10-03 23:22:32 -04001078 if (err)
1079 extent_io_tree_panic(tree, err);
Josef Bacik462d6fa2011-09-26 13:56:12 -04001080 prealloc = NULL;
1081 start = this_end + 1;
1082 goto search_again;
1083 }
1084 /*
1085 * | ---- desired range ---- |
1086 * | state |
1087 * We need to split the extent, and set the bit
1088 * on the first half
1089 */
1090 if (state->start <= end && state->end > end) {
1091 prealloc = alloc_extent_state_atomic(prealloc);
Liu Bo1cf4ffd2011-12-07 20:08:40 -05001092 if (!prealloc) {
1093 err = -ENOMEM;
1094 goto out;
1095 }
Josef Bacik462d6fa2011-09-26 13:56:12 -04001096
1097 err = split_state(tree, state, prealloc, end + 1);
Jeff Mahoneyc2d904e2011-10-03 23:22:32 -04001098 if (err)
1099 extent_io_tree_panic(tree, err);
Josef Bacik462d6fa2011-09-26 13:56:12 -04001100
1101 set_state_bits(tree, prealloc, &bits);
1102 clear_state_bit(tree, prealloc, &clear_bits, 0);
Josef Bacik462d6fa2011-09-26 13:56:12 -04001103 prealloc = NULL;
1104 goto out;
1105 }
1106
1107 goto search_again;
1108
1109out:
1110 spin_unlock(&tree->lock);
1111 if (prealloc)
1112 free_extent_state(prealloc);
1113
1114 return err;
1115
1116search_again:
1117 if (start > end)
1118 goto out;
1119 spin_unlock(&tree->lock);
1120 if (mask & __GFP_WAIT)
1121 cond_resched();
1122 goto again;
1123}
1124
Chris Masond1310b22008-01-24 16:13:08 -05001125/* wrappers around set/clear extent bit */
1126int set_extent_dirty(struct extent_io_tree *tree, u64 start, u64 end,
1127 gfp_t mask)
1128{
Jeff Mahoney3fbe5c02012-03-01 14:57:19 +01001129 return set_extent_bit(tree, start, end, EXTENT_DIRTY, NULL,
Chris Mason2c64c532009-09-02 15:04:12 -04001130 NULL, mask);
Chris Masond1310b22008-01-24 16:13:08 -05001131}
Chris Masond1310b22008-01-24 16:13:08 -05001132
1133int set_extent_bits(struct extent_io_tree *tree, u64 start, u64 end,
1134 int bits, gfp_t mask)
1135{
Jeff Mahoney3fbe5c02012-03-01 14:57:19 +01001136 return set_extent_bit(tree, start, end, bits, NULL,
Chris Mason2c64c532009-09-02 15:04:12 -04001137 NULL, mask);
Chris Masond1310b22008-01-24 16:13:08 -05001138}
Chris Masond1310b22008-01-24 16:13:08 -05001139
1140int clear_extent_bits(struct extent_io_tree *tree, u64 start, u64 end,
1141 int bits, gfp_t mask)
1142{
Chris Mason2c64c532009-09-02 15:04:12 -04001143 return clear_extent_bit(tree, start, end, bits, 0, 0, NULL, mask);
Chris Masond1310b22008-01-24 16:13:08 -05001144}
Chris Masond1310b22008-01-24 16:13:08 -05001145
1146int set_extent_delalloc(struct extent_io_tree *tree, u64 start, u64 end,
Josef Bacik2ac55d42010-02-03 19:33:23 +00001147 struct extent_state **cached_state, gfp_t mask)
Chris Masond1310b22008-01-24 16:13:08 -05001148{
1149 return set_extent_bit(tree, start, end,
Liu Bofee187d2011-09-29 15:55:28 +08001150 EXTENT_DELALLOC | EXTENT_UPTODATE,
Jeff Mahoney3fbe5c02012-03-01 14:57:19 +01001151 NULL, cached_state, mask);
Chris Masond1310b22008-01-24 16:13:08 -05001152}
Chris Masond1310b22008-01-24 16:13:08 -05001153
1154int clear_extent_dirty(struct extent_io_tree *tree, u64 start, u64 end,
1155 gfp_t mask)
1156{
1157 return clear_extent_bit(tree, start, end,
Josef Bacik32c00af2009-10-08 13:34:05 -04001158 EXTENT_DIRTY | EXTENT_DELALLOC |
Yan, Zheng0ca1f7c2010-05-16 10:48:47 -04001159 EXTENT_DO_ACCOUNTING, 0, 0, NULL, mask);
Chris Masond1310b22008-01-24 16:13:08 -05001160}
Chris Masond1310b22008-01-24 16:13:08 -05001161
1162int set_extent_new(struct extent_io_tree *tree, u64 start, u64 end,
1163 gfp_t mask)
1164{
Jeff Mahoney3fbe5c02012-03-01 14:57:19 +01001165 return set_extent_bit(tree, start, end, EXTENT_NEW, NULL,
Chris Mason2c64c532009-09-02 15:04:12 -04001166 NULL, mask);
Chris Masond1310b22008-01-24 16:13:08 -05001167}
Chris Masond1310b22008-01-24 16:13:08 -05001168
Chris Masond1310b22008-01-24 16:13:08 -05001169int set_extent_uptodate(struct extent_io_tree *tree, u64 start, u64 end,
Arne Jansen507903b2011-04-06 10:02:20 +00001170 struct extent_state **cached_state, gfp_t mask)
Chris Masond1310b22008-01-24 16:13:08 -05001171{
Arne Jansen507903b2011-04-06 10:02:20 +00001172 return set_extent_bit(tree, start, end, EXTENT_UPTODATE, 0,
Jeff Mahoney3fbe5c02012-03-01 14:57:19 +01001173 cached_state, mask);
Chris Masond1310b22008-01-24 16:13:08 -05001174}
Chris Masond1310b22008-01-24 16:13:08 -05001175
Chris Masond3977122009-01-05 21:25:51 -05001176static int clear_extent_uptodate(struct extent_io_tree *tree, u64 start,
Josef Bacik2ac55d42010-02-03 19:33:23 +00001177 u64 end, struct extent_state **cached_state,
1178 gfp_t mask)
Chris Masond1310b22008-01-24 16:13:08 -05001179{
Chris Mason2c64c532009-09-02 15:04:12 -04001180 return clear_extent_bit(tree, start, end, EXTENT_UPTODATE, 0, 0,
Josef Bacik2ac55d42010-02-03 19:33:23 +00001181 cached_state, mask);
Chris Masond1310b22008-01-24 16:13:08 -05001182}
Chris Masond1310b22008-01-24 16:13:08 -05001183
Chris Masond352ac62008-09-29 15:18:18 -04001184/*
1185 * either insert or lock state struct between start and end use mask to tell
1186 * us if waiting is desired.
1187 */
Chris Mason1edbb732009-09-02 13:24:36 -04001188int lock_extent_bits(struct extent_io_tree *tree, u64 start, u64 end,
Jeff Mahoneyd0082372012-03-01 14:57:19 +01001189 int bits, struct extent_state **cached_state)
Chris Masond1310b22008-01-24 16:13:08 -05001190{
1191 int err;
1192 u64 failed_start;
1193 while (1) {
Jeff Mahoney3fbe5c02012-03-01 14:57:19 +01001194 err = __set_extent_bit(tree, start, end, EXTENT_LOCKED | bits,
1195 EXTENT_LOCKED, &failed_start,
1196 cached_state, GFP_NOFS);
Jeff Mahoneyd0082372012-03-01 14:57:19 +01001197 if (err == -EEXIST) {
Chris Masond1310b22008-01-24 16:13:08 -05001198 wait_extent_bit(tree, failed_start, end, EXTENT_LOCKED);
1199 start = failed_start;
Jeff Mahoneyd0082372012-03-01 14:57:19 +01001200 } else
Chris Masond1310b22008-01-24 16:13:08 -05001201 break;
Chris Masond1310b22008-01-24 16:13:08 -05001202 WARN_ON(start > end);
1203 }
1204 return err;
1205}
Chris Masond1310b22008-01-24 16:13:08 -05001206
Jeff Mahoneyd0082372012-03-01 14:57:19 +01001207int lock_extent(struct extent_io_tree *tree, u64 start, u64 end)
Chris Mason1edbb732009-09-02 13:24:36 -04001208{
Jeff Mahoneyd0082372012-03-01 14:57:19 +01001209 return lock_extent_bits(tree, start, end, 0, NULL);
Chris Mason1edbb732009-09-02 13:24:36 -04001210}
1211
Jeff Mahoneyd0082372012-03-01 14:57:19 +01001212int try_lock_extent(struct extent_io_tree *tree, u64 start, u64 end)
Josef Bacik25179202008-10-29 14:49:05 -04001213{
1214 int err;
1215 u64 failed_start;
1216
Jeff Mahoney3fbe5c02012-03-01 14:57:19 +01001217 err = __set_extent_bit(tree, start, end, EXTENT_LOCKED, EXTENT_LOCKED,
1218 &failed_start, NULL, GFP_NOFS);
Yan Zheng66435582008-10-30 14:19:50 -04001219 if (err == -EEXIST) {
1220 if (failed_start > start)
1221 clear_extent_bit(tree, start, failed_start - 1,
Jeff Mahoneyd0082372012-03-01 14:57:19 +01001222 EXTENT_LOCKED, 1, 0, NULL, GFP_NOFS);
Josef Bacik25179202008-10-29 14:49:05 -04001223 return 0;
Yan Zheng66435582008-10-30 14:19:50 -04001224 }
Josef Bacik25179202008-10-29 14:49:05 -04001225 return 1;
1226}
Josef Bacik25179202008-10-29 14:49:05 -04001227
Chris Mason2c64c532009-09-02 15:04:12 -04001228int unlock_extent_cached(struct extent_io_tree *tree, u64 start, u64 end,
1229 struct extent_state **cached, gfp_t mask)
1230{
1231 return clear_extent_bit(tree, start, end, EXTENT_LOCKED, 1, 0, cached,
1232 mask);
1233}
1234
Jeff Mahoneyd0082372012-03-01 14:57:19 +01001235int unlock_extent(struct extent_io_tree *tree, u64 start, u64 end)
Chris Masond1310b22008-01-24 16:13:08 -05001236{
Chris Mason2c64c532009-09-02 15:04:12 -04001237 return clear_extent_bit(tree, start, end, EXTENT_LOCKED, 1, 0, NULL,
Jeff Mahoneyd0082372012-03-01 14:57:19 +01001238 GFP_NOFS);
Chris Masond1310b22008-01-24 16:13:08 -05001239}
Chris Masond1310b22008-01-24 16:13:08 -05001240
1241/*
Chris Masond1310b22008-01-24 16:13:08 -05001242 * helper function to set both pages and extents in the tree writeback
1243 */
Christoph Hellwigb2950862008-12-02 09:54:17 -05001244static int set_range_writeback(struct extent_io_tree *tree, u64 start, u64 end)
Chris Masond1310b22008-01-24 16:13:08 -05001245{
1246 unsigned long index = start >> PAGE_CACHE_SHIFT;
1247 unsigned long end_index = end >> PAGE_CACHE_SHIFT;
1248 struct page *page;
1249
1250 while (index <= end_index) {
1251 page = find_get_page(tree->mapping, index);
Jeff Mahoney79787ea2012-03-12 16:03:00 +01001252 BUG_ON(!page); /* Pages should be in the extent_io_tree */
Chris Masond1310b22008-01-24 16:13:08 -05001253 set_page_writeback(page);
1254 page_cache_release(page);
1255 index++;
1256 }
Chris Masond1310b22008-01-24 16:13:08 -05001257 return 0;
1258}
Chris Masond1310b22008-01-24 16:13:08 -05001259
Chris Masond352ac62008-09-29 15:18:18 -04001260/* find the first state struct with 'bits' set after 'start', and
1261 * return it. tree->lock must be held. NULL will returned if
1262 * nothing was found after 'start'
1263 */
Chris Masond7fc6402008-02-18 12:12:38 -05001264struct extent_state *find_first_extent_bit_state(struct extent_io_tree *tree,
1265 u64 start, int bits)
1266{
1267 struct rb_node *node;
1268 struct extent_state *state;
1269
1270 /*
1271 * this search will find all the extents that end after
1272 * our range starts.
1273 */
1274 node = tree_search(tree, start);
Chris Masond3977122009-01-05 21:25:51 -05001275 if (!node)
Chris Masond7fc6402008-02-18 12:12:38 -05001276 goto out;
Chris Masond7fc6402008-02-18 12:12:38 -05001277
Chris Masond3977122009-01-05 21:25:51 -05001278 while (1) {
Chris Masond7fc6402008-02-18 12:12:38 -05001279 state = rb_entry(node, struct extent_state, rb_node);
Chris Masond3977122009-01-05 21:25:51 -05001280 if (state->end >= start && (state->state & bits))
Chris Masond7fc6402008-02-18 12:12:38 -05001281 return state;
Chris Masond3977122009-01-05 21:25:51 -05001282
Chris Masond7fc6402008-02-18 12:12:38 -05001283 node = rb_next(node);
1284 if (!node)
1285 break;
1286 }
1287out:
1288 return NULL;
1289}
Chris Masond7fc6402008-02-18 12:12:38 -05001290
Chris Masond352ac62008-09-29 15:18:18 -04001291/*
Xiao Guangrong69261c42011-07-14 03:19:45 +00001292 * find the first offset in the io tree with 'bits' set. zero is
1293 * returned if we find something, and *start_ret and *end_ret are
1294 * set to reflect the state struct that was found.
1295 *
1296 * If nothing was found, 1 is returned, < 0 on error
1297 */
1298int find_first_extent_bit(struct extent_io_tree *tree, u64 start,
1299 u64 *start_ret, u64 *end_ret, int bits)
1300{
1301 struct extent_state *state;
1302 int ret = 1;
1303
1304 spin_lock(&tree->lock);
1305 state = find_first_extent_bit_state(tree, start, bits);
1306 if (state) {
1307 *start_ret = state->start;
1308 *end_ret = state->end;
1309 ret = 0;
1310 }
1311 spin_unlock(&tree->lock);
1312 return ret;
1313}
1314
1315/*
Chris Masond352ac62008-09-29 15:18:18 -04001316 * find a contiguous range of bytes in the file marked as delalloc, not
1317 * more than 'max_bytes'. start and end are used to return the range,
1318 *
1319 * 1 is returned if we find something, 0 if nothing was in the tree
1320 */
Chris Masonc8b97812008-10-29 14:49:59 -04001321static noinline u64 find_delalloc_range(struct extent_io_tree *tree,
Josef Bacikc2a128d2010-02-02 21:19:11 +00001322 u64 *start, u64 *end, u64 max_bytes,
1323 struct extent_state **cached_state)
Chris Masond1310b22008-01-24 16:13:08 -05001324{
1325 struct rb_node *node;
1326 struct extent_state *state;
1327 u64 cur_start = *start;
1328 u64 found = 0;
1329 u64 total_bytes = 0;
1330
Chris Masoncad321a2008-12-17 14:51:42 -05001331 spin_lock(&tree->lock);
Chris Masonc8b97812008-10-29 14:49:59 -04001332
Chris Masond1310b22008-01-24 16:13:08 -05001333 /*
1334 * this search will find all the extents that end after
1335 * our range starts.
1336 */
Chris Mason80ea96b2008-02-01 14:51:59 -05001337 node = tree_search(tree, cur_start);
Peter2b114d12008-04-01 11:21:40 -04001338 if (!node) {
Chris Mason3b951512008-04-17 11:29:12 -04001339 if (!found)
1340 *end = (u64)-1;
Chris Masond1310b22008-01-24 16:13:08 -05001341 goto out;
1342 }
1343
Chris Masond3977122009-01-05 21:25:51 -05001344 while (1) {
Chris Masond1310b22008-01-24 16:13:08 -05001345 state = rb_entry(node, struct extent_state, rb_node);
Zheng Yan5b21f2e2008-09-26 10:05:38 -04001346 if (found && (state->start != cur_start ||
1347 (state->state & EXTENT_BOUNDARY))) {
Chris Masond1310b22008-01-24 16:13:08 -05001348 goto out;
1349 }
1350 if (!(state->state & EXTENT_DELALLOC)) {
1351 if (!found)
1352 *end = state->end;
1353 goto out;
1354 }
Josef Bacikc2a128d2010-02-02 21:19:11 +00001355 if (!found) {
Chris Masond1310b22008-01-24 16:13:08 -05001356 *start = state->start;
Josef Bacikc2a128d2010-02-02 21:19:11 +00001357 *cached_state = state;
1358 atomic_inc(&state->refs);
1359 }
Chris Masond1310b22008-01-24 16:13:08 -05001360 found++;
1361 *end = state->end;
1362 cur_start = state->end + 1;
1363 node = rb_next(node);
1364 if (!node)
1365 break;
1366 total_bytes += state->end - state->start + 1;
1367 if (total_bytes >= max_bytes)
1368 break;
1369 }
1370out:
Chris Masoncad321a2008-12-17 14:51:42 -05001371 spin_unlock(&tree->lock);
Chris Masond1310b22008-01-24 16:13:08 -05001372 return found;
1373}
1374
Jeff Mahoney143bede2012-03-01 14:56:26 +01001375static noinline void __unlock_for_delalloc(struct inode *inode,
1376 struct page *locked_page,
1377 u64 start, u64 end)
Chris Masonc8b97812008-10-29 14:49:59 -04001378{
1379 int ret;
1380 struct page *pages[16];
1381 unsigned long index = start >> PAGE_CACHE_SHIFT;
1382 unsigned long end_index = end >> PAGE_CACHE_SHIFT;
1383 unsigned long nr_pages = end_index - index + 1;
1384 int i;
1385
1386 if (index == locked_page->index && end_index == index)
Jeff Mahoney143bede2012-03-01 14:56:26 +01001387 return;
Chris Masonc8b97812008-10-29 14:49:59 -04001388
Chris Masond3977122009-01-05 21:25:51 -05001389 while (nr_pages > 0) {
Chris Masonc8b97812008-10-29 14:49:59 -04001390 ret = find_get_pages_contig(inode->i_mapping, index,
Chris Mason5b050f02008-11-11 09:34:41 -05001391 min_t(unsigned long, nr_pages,
1392 ARRAY_SIZE(pages)), pages);
Chris Masonc8b97812008-10-29 14:49:59 -04001393 for (i = 0; i < ret; i++) {
1394 if (pages[i] != locked_page)
1395 unlock_page(pages[i]);
1396 page_cache_release(pages[i]);
1397 }
1398 nr_pages -= ret;
1399 index += ret;
1400 cond_resched();
1401 }
Chris Masonc8b97812008-10-29 14:49:59 -04001402}
1403
1404static noinline int lock_delalloc_pages(struct inode *inode,
1405 struct page *locked_page,
1406 u64 delalloc_start,
1407 u64 delalloc_end)
1408{
1409 unsigned long index = delalloc_start >> PAGE_CACHE_SHIFT;
1410 unsigned long start_index = index;
1411 unsigned long end_index = delalloc_end >> PAGE_CACHE_SHIFT;
1412 unsigned long pages_locked = 0;
1413 struct page *pages[16];
1414 unsigned long nrpages;
1415 int ret;
1416 int i;
1417
1418 /* the caller is responsible for locking the start index */
1419 if (index == locked_page->index && index == end_index)
1420 return 0;
1421
1422 /* skip the page at the start index */
1423 nrpages = end_index - index + 1;
Chris Masond3977122009-01-05 21:25:51 -05001424 while (nrpages > 0) {
Chris Masonc8b97812008-10-29 14:49:59 -04001425 ret = find_get_pages_contig(inode->i_mapping, index,
Chris Mason5b050f02008-11-11 09:34:41 -05001426 min_t(unsigned long,
1427 nrpages, ARRAY_SIZE(pages)), pages);
Chris Masonc8b97812008-10-29 14:49:59 -04001428 if (ret == 0) {
1429 ret = -EAGAIN;
1430 goto done;
1431 }
1432 /* now we have an array of pages, lock them all */
1433 for (i = 0; i < ret; i++) {
1434 /*
1435 * the caller is taking responsibility for
1436 * locked_page
1437 */
Chris Mason771ed682008-11-06 22:02:51 -05001438 if (pages[i] != locked_page) {
Chris Masonc8b97812008-10-29 14:49:59 -04001439 lock_page(pages[i]);
Chris Masonf2b1c412008-11-10 07:31:30 -05001440 if (!PageDirty(pages[i]) ||
1441 pages[i]->mapping != inode->i_mapping) {
Chris Mason771ed682008-11-06 22:02:51 -05001442 ret = -EAGAIN;
1443 unlock_page(pages[i]);
1444 page_cache_release(pages[i]);
1445 goto done;
1446 }
1447 }
Chris Masonc8b97812008-10-29 14:49:59 -04001448 page_cache_release(pages[i]);
Chris Mason771ed682008-11-06 22:02:51 -05001449 pages_locked++;
Chris Masonc8b97812008-10-29 14:49:59 -04001450 }
Chris Masonc8b97812008-10-29 14:49:59 -04001451 nrpages -= ret;
1452 index += ret;
1453 cond_resched();
1454 }
1455 ret = 0;
1456done:
1457 if (ret && pages_locked) {
1458 __unlock_for_delalloc(inode, locked_page,
1459 delalloc_start,
1460 ((u64)(start_index + pages_locked - 1)) <<
1461 PAGE_CACHE_SHIFT);
1462 }
1463 return ret;
1464}
1465
1466/*
1467 * find a contiguous range of bytes in the file marked as delalloc, not
1468 * more than 'max_bytes'. start and end are used to return the range,
1469 *
1470 * 1 is returned if we find something, 0 if nothing was in the tree
1471 */
1472static noinline u64 find_lock_delalloc_range(struct inode *inode,
1473 struct extent_io_tree *tree,
1474 struct page *locked_page,
1475 u64 *start, u64 *end,
1476 u64 max_bytes)
1477{
1478 u64 delalloc_start;
1479 u64 delalloc_end;
1480 u64 found;
Chris Mason9655d292009-09-02 15:22:30 -04001481 struct extent_state *cached_state = NULL;
Chris Masonc8b97812008-10-29 14:49:59 -04001482 int ret;
1483 int loops = 0;
1484
1485again:
1486 /* step one, find a bunch of delalloc bytes starting at start */
1487 delalloc_start = *start;
1488 delalloc_end = 0;
1489 found = find_delalloc_range(tree, &delalloc_start, &delalloc_end,
Josef Bacikc2a128d2010-02-02 21:19:11 +00001490 max_bytes, &cached_state);
Chris Mason70b99e62008-10-31 12:46:39 -04001491 if (!found || delalloc_end <= *start) {
Chris Masonc8b97812008-10-29 14:49:59 -04001492 *start = delalloc_start;
1493 *end = delalloc_end;
Josef Bacikc2a128d2010-02-02 21:19:11 +00001494 free_extent_state(cached_state);
Chris Masonc8b97812008-10-29 14:49:59 -04001495 return found;
1496 }
1497
1498 /*
Chris Mason70b99e62008-10-31 12:46:39 -04001499 * start comes from the offset of locked_page. We have to lock
1500 * pages in order, so we can't process delalloc bytes before
1501 * locked_page
1502 */
Chris Masond3977122009-01-05 21:25:51 -05001503 if (delalloc_start < *start)
Chris Mason70b99e62008-10-31 12:46:39 -04001504 delalloc_start = *start;
Chris Mason70b99e62008-10-31 12:46:39 -04001505
1506 /*
Chris Masonc8b97812008-10-29 14:49:59 -04001507 * make sure to limit the number of pages we try to lock down
1508 * if we're looping.
1509 */
Chris Masond3977122009-01-05 21:25:51 -05001510 if (delalloc_end + 1 - delalloc_start > max_bytes && loops)
Chris Mason771ed682008-11-06 22:02:51 -05001511 delalloc_end = delalloc_start + PAGE_CACHE_SIZE - 1;
Chris Masond3977122009-01-05 21:25:51 -05001512
Chris Masonc8b97812008-10-29 14:49:59 -04001513 /* step two, lock all the pages after the page that has start */
1514 ret = lock_delalloc_pages(inode, locked_page,
1515 delalloc_start, delalloc_end);
1516 if (ret == -EAGAIN) {
1517 /* some of the pages are gone, lets avoid looping by
1518 * shortening the size of the delalloc range we're searching
1519 */
Chris Mason9655d292009-09-02 15:22:30 -04001520 free_extent_state(cached_state);
Chris Masonc8b97812008-10-29 14:49:59 -04001521 if (!loops) {
1522 unsigned long offset = (*start) & (PAGE_CACHE_SIZE - 1);
1523 max_bytes = PAGE_CACHE_SIZE - offset;
1524 loops = 1;
1525 goto again;
1526 } else {
1527 found = 0;
1528 goto out_failed;
1529 }
1530 }
Jeff Mahoney79787ea2012-03-12 16:03:00 +01001531 BUG_ON(ret); /* Only valid values are 0 and -EAGAIN */
Chris Masonc8b97812008-10-29 14:49:59 -04001532
1533 /* step three, lock the state bits for the whole range */
Jeff Mahoneyd0082372012-03-01 14:57:19 +01001534 lock_extent_bits(tree, delalloc_start, delalloc_end, 0, &cached_state);
Chris Masonc8b97812008-10-29 14:49:59 -04001535
1536 /* then test to make sure it is all still delalloc */
1537 ret = test_range_bit(tree, delalloc_start, delalloc_end,
Chris Mason9655d292009-09-02 15:22:30 -04001538 EXTENT_DELALLOC, 1, cached_state);
Chris Masonc8b97812008-10-29 14:49:59 -04001539 if (!ret) {
Chris Mason9655d292009-09-02 15:22:30 -04001540 unlock_extent_cached(tree, delalloc_start, delalloc_end,
1541 &cached_state, GFP_NOFS);
Chris Masonc8b97812008-10-29 14:49:59 -04001542 __unlock_for_delalloc(inode, locked_page,
1543 delalloc_start, delalloc_end);
1544 cond_resched();
1545 goto again;
1546 }
Chris Mason9655d292009-09-02 15:22:30 -04001547 free_extent_state(cached_state);
Chris Masonc8b97812008-10-29 14:49:59 -04001548 *start = delalloc_start;
1549 *end = delalloc_end;
1550out_failed:
1551 return found;
1552}
1553
1554int extent_clear_unlock_delalloc(struct inode *inode,
1555 struct extent_io_tree *tree,
1556 u64 start, u64 end, struct page *locked_page,
Chris Masona791e352009-10-08 11:27:10 -04001557 unsigned long op)
Chris Masonc8b97812008-10-29 14:49:59 -04001558{
1559 int ret;
1560 struct page *pages[16];
1561 unsigned long index = start >> PAGE_CACHE_SHIFT;
1562 unsigned long end_index = end >> PAGE_CACHE_SHIFT;
1563 unsigned long nr_pages = end_index - index + 1;
1564 int i;
Chris Mason771ed682008-11-06 22:02:51 -05001565 int clear_bits = 0;
Chris Masonc8b97812008-10-29 14:49:59 -04001566
Chris Masona791e352009-10-08 11:27:10 -04001567 if (op & EXTENT_CLEAR_UNLOCK)
Chris Mason771ed682008-11-06 22:02:51 -05001568 clear_bits |= EXTENT_LOCKED;
Chris Masona791e352009-10-08 11:27:10 -04001569 if (op & EXTENT_CLEAR_DIRTY)
Chris Masonc8b97812008-10-29 14:49:59 -04001570 clear_bits |= EXTENT_DIRTY;
1571
Chris Masona791e352009-10-08 11:27:10 -04001572 if (op & EXTENT_CLEAR_DELALLOC)
Chris Mason771ed682008-11-06 22:02:51 -05001573 clear_bits |= EXTENT_DELALLOC;
1574
Chris Mason2c64c532009-09-02 15:04:12 -04001575 clear_extent_bit(tree, start, end, clear_bits, 1, 0, NULL, GFP_NOFS);
Josef Bacik32c00af2009-10-08 13:34:05 -04001576 if (!(op & (EXTENT_CLEAR_UNLOCK_PAGE | EXTENT_CLEAR_DIRTY |
1577 EXTENT_SET_WRITEBACK | EXTENT_END_WRITEBACK |
1578 EXTENT_SET_PRIVATE2)))
Chris Mason771ed682008-11-06 22:02:51 -05001579 return 0;
Chris Masonc8b97812008-10-29 14:49:59 -04001580
Chris Masond3977122009-01-05 21:25:51 -05001581 while (nr_pages > 0) {
Chris Masonc8b97812008-10-29 14:49:59 -04001582 ret = find_get_pages_contig(inode->i_mapping, index,
Chris Mason5b050f02008-11-11 09:34:41 -05001583 min_t(unsigned long,
1584 nr_pages, ARRAY_SIZE(pages)), pages);
Chris Masonc8b97812008-10-29 14:49:59 -04001585 for (i = 0; i < ret; i++) {
Chris Mason8b62b722009-09-02 16:53:46 -04001586
Chris Masona791e352009-10-08 11:27:10 -04001587 if (op & EXTENT_SET_PRIVATE2)
Chris Mason8b62b722009-09-02 16:53:46 -04001588 SetPagePrivate2(pages[i]);
1589
Chris Masonc8b97812008-10-29 14:49:59 -04001590 if (pages[i] == locked_page) {
1591 page_cache_release(pages[i]);
1592 continue;
1593 }
Chris Masona791e352009-10-08 11:27:10 -04001594 if (op & EXTENT_CLEAR_DIRTY)
Chris Masonc8b97812008-10-29 14:49:59 -04001595 clear_page_dirty_for_io(pages[i]);
Chris Masona791e352009-10-08 11:27:10 -04001596 if (op & EXTENT_SET_WRITEBACK)
Chris Masonc8b97812008-10-29 14:49:59 -04001597 set_page_writeback(pages[i]);
Chris Masona791e352009-10-08 11:27:10 -04001598 if (op & EXTENT_END_WRITEBACK)
Chris Masonc8b97812008-10-29 14:49:59 -04001599 end_page_writeback(pages[i]);
Chris Masona791e352009-10-08 11:27:10 -04001600 if (op & EXTENT_CLEAR_UNLOCK_PAGE)
Chris Mason771ed682008-11-06 22:02:51 -05001601 unlock_page(pages[i]);
Chris Masonc8b97812008-10-29 14:49:59 -04001602 page_cache_release(pages[i]);
1603 }
1604 nr_pages -= ret;
1605 index += ret;
1606 cond_resched();
1607 }
1608 return 0;
1609}
Chris Masonc8b97812008-10-29 14:49:59 -04001610
Chris Masond352ac62008-09-29 15:18:18 -04001611/*
1612 * count the number of bytes in the tree that have a given bit(s)
1613 * set. This can be fairly slow, except for EXTENT_DIRTY which is
1614 * cached. The total number found is returned.
1615 */
Chris Masond1310b22008-01-24 16:13:08 -05001616u64 count_range_bits(struct extent_io_tree *tree,
1617 u64 *start, u64 search_end, u64 max_bytes,
Chris Masonec29ed52011-02-23 16:23:20 -05001618 unsigned long bits, int contig)
Chris Masond1310b22008-01-24 16:13:08 -05001619{
1620 struct rb_node *node;
1621 struct extent_state *state;
1622 u64 cur_start = *start;
1623 u64 total_bytes = 0;
Chris Masonec29ed52011-02-23 16:23:20 -05001624 u64 last = 0;
Chris Masond1310b22008-01-24 16:13:08 -05001625 int found = 0;
1626
1627 if (search_end <= cur_start) {
Chris Masond1310b22008-01-24 16:13:08 -05001628 WARN_ON(1);
1629 return 0;
1630 }
1631
Chris Masoncad321a2008-12-17 14:51:42 -05001632 spin_lock(&tree->lock);
Chris Masond1310b22008-01-24 16:13:08 -05001633 if (cur_start == 0 && bits == EXTENT_DIRTY) {
1634 total_bytes = tree->dirty_bytes;
1635 goto out;
1636 }
1637 /*
1638 * this search will find all the extents that end after
1639 * our range starts.
1640 */
Chris Mason80ea96b2008-02-01 14:51:59 -05001641 node = tree_search(tree, cur_start);
Chris Masond3977122009-01-05 21:25:51 -05001642 if (!node)
Chris Masond1310b22008-01-24 16:13:08 -05001643 goto out;
Chris Masond1310b22008-01-24 16:13:08 -05001644
Chris Masond3977122009-01-05 21:25:51 -05001645 while (1) {
Chris Masond1310b22008-01-24 16:13:08 -05001646 state = rb_entry(node, struct extent_state, rb_node);
1647 if (state->start > search_end)
1648 break;
Chris Masonec29ed52011-02-23 16:23:20 -05001649 if (contig && found && state->start > last + 1)
1650 break;
1651 if (state->end >= cur_start && (state->state & bits) == bits) {
Chris Masond1310b22008-01-24 16:13:08 -05001652 total_bytes += min(search_end, state->end) + 1 -
1653 max(cur_start, state->start);
1654 if (total_bytes >= max_bytes)
1655 break;
1656 if (!found) {
Josef Bacikaf60bed2011-05-04 11:11:17 -04001657 *start = max(cur_start, state->start);
Chris Masond1310b22008-01-24 16:13:08 -05001658 found = 1;
1659 }
Chris Masonec29ed52011-02-23 16:23:20 -05001660 last = state->end;
1661 } else if (contig && found) {
1662 break;
Chris Masond1310b22008-01-24 16:13:08 -05001663 }
1664 node = rb_next(node);
1665 if (!node)
1666 break;
1667 }
1668out:
Chris Masoncad321a2008-12-17 14:51:42 -05001669 spin_unlock(&tree->lock);
Chris Masond1310b22008-01-24 16:13:08 -05001670 return total_bytes;
1671}
Christoph Hellwigb2950862008-12-02 09:54:17 -05001672
Chris Masond352ac62008-09-29 15:18:18 -04001673/*
1674 * set the private field for a given byte offset in the tree. If there isn't
1675 * an extent_state there already, this does nothing.
1676 */
Chris Masond1310b22008-01-24 16:13:08 -05001677int set_state_private(struct extent_io_tree *tree, u64 start, u64 private)
1678{
1679 struct rb_node *node;
1680 struct extent_state *state;
1681 int ret = 0;
1682
Chris Masoncad321a2008-12-17 14:51:42 -05001683 spin_lock(&tree->lock);
Chris Masond1310b22008-01-24 16:13:08 -05001684 /*
1685 * this search will find all the extents that end after
1686 * our range starts.
1687 */
Chris Mason80ea96b2008-02-01 14:51:59 -05001688 node = tree_search(tree, start);
Peter2b114d12008-04-01 11:21:40 -04001689 if (!node) {
Chris Masond1310b22008-01-24 16:13:08 -05001690 ret = -ENOENT;
1691 goto out;
1692 }
1693 state = rb_entry(node, struct extent_state, rb_node);
1694 if (state->start != start) {
1695 ret = -ENOENT;
1696 goto out;
1697 }
1698 state->private = private;
1699out:
Chris Masoncad321a2008-12-17 14:51:42 -05001700 spin_unlock(&tree->lock);
Chris Masond1310b22008-01-24 16:13:08 -05001701 return ret;
1702}
1703
1704int get_state_private(struct extent_io_tree *tree, u64 start, u64 *private)
1705{
1706 struct rb_node *node;
1707 struct extent_state *state;
1708 int ret = 0;
1709
Chris Masoncad321a2008-12-17 14:51:42 -05001710 spin_lock(&tree->lock);
Chris Masond1310b22008-01-24 16:13:08 -05001711 /*
1712 * this search will find all the extents that end after
1713 * our range starts.
1714 */
Chris Mason80ea96b2008-02-01 14:51:59 -05001715 node = tree_search(tree, start);
Peter2b114d12008-04-01 11:21:40 -04001716 if (!node) {
Chris Masond1310b22008-01-24 16:13:08 -05001717 ret = -ENOENT;
1718 goto out;
1719 }
1720 state = rb_entry(node, struct extent_state, rb_node);
1721 if (state->start != start) {
1722 ret = -ENOENT;
1723 goto out;
1724 }
1725 *private = state->private;
1726out:
Chris Masoncad321a2008-12-17 14:51:42 -05001727 spin_unlock(&tree->lock);
Chris Masond1310b22008-01-24 16:13:08 -05001728 return ret;
1729}
1730
1731/*
1732 * searches a range in the state tree for a given mask.
Chris Mason70dec802008-01-29 09:59:12 -05001733 * If 'filled' == 1, this returns 1 only if every extent in the tree
Chris Masond1310b22008-01-24 16:13:08 -05001734 * has the bits set. Otherwise, 1 is returned if any bit in the
1735 * range is found set.
1736 */
1737int test_range_bit(struct extent_io_tree *tree, u64 start, u64 end,
Chris Mason9655d292009-09-02 15:22:30 -04001738 int bits, int filled, struct extent_state *cached)
Chris Masond1310b22008-01-24 16:13:08 -05001739{
1740 struct extent_state *state = NULL;
1741 struct rb_node *node;
1742 int bitset = 0;
Chris Masond1310b22008-01-24 16:13:08 -05001743
Chris Masoncad321a2008-12-17 14:51:42 -05001744 spin_lock(&tree->lock);
Josef Bacikdf98b6e2011-06-20 14:53:48 -04001745 if (cached && cached->tree && cached->start <= start &&
1746 cached->end > start)
Chris Mason9655d292009-09-02 15:22:30 -04001747 node = &cached->rb_node;
1748 else
1749 node = tree_search(tree, start);
Chris Masond1310b22008-01-24 16:13:08 -05001750 while (node && start <= end) {
1751 state = rb_entry(node, struct extent_state, rb_node);
1752
1753 if (filled && state->start > start) {
1754 bitset = 0;
1755 break;
1756 }
1757
1758 if (state->start > end)
1759 break;
1760
1761 if (state->state & bits) {
1762 bitset = 1;
1763 if (!filled)
1764 break;
1765 } else if (filled) {
1766 bitset = 0;
1767 break;
1768 }
Chris Mason46562ce2009-09-23 20:23:16 -04001769
1770 if (state->end == (u64)-1)
1771 break;
1772
Chris Masond1310b22008-01-24 16:13:08 -05001773 start = state->end + 1;
1774 if (start > end)
1775 break;
1776 node = rb_next(node);
1777 if (!node) {
1778 if (filled)
1779 bitset = 0;
1780 break;
1781 }
1782 }
Chris Masoncad321a2008-12-17 14:51:42 -05001783 spin_unlock(&tree->lock);
Chris Masond1310b22008-01-24 16:13:08 -05001784 return bitset;
1785}
Chris Masond1310b22008-01-24 16:13:08 -05001786
1787/*
1788 * helper function to set a given page up to date if all the
1789 * extents in the tree for that page are up to date
1790 */
Jeff Mahoney143bede2012-03-01 14:56:26 +01001791static void check_page_uptodate(struct extent_io_tree *tree, struct page *page)
Chris Masond1310b22008-01-24 16:13:08 -05001792{
1793 u64 start = (u64)page->index << PAGE_CACHE_SHIFT;
1794 u64 end = start + PAGE_CACHE_SIZE - 1;
Chris Mason9655d292009-09-02 15:22:30 -04001795 if (test_range_bit(tree, start, end, EXTENT_UPTODATE, 1, NULL))
Chris Masond1310b22008-01-24 16:13:08 -05001796 SetPageUptodate(page);
Chris Masond1310b22008-01-24 16:13:08 -05001797}
1798
1799/*
1800 * helper function to unlock a page if all the extents in the tree
1801 * for that page are unlocked
1802 */
Jeff Mahoney143bede2012-03-01 14:56:26 +01001803static void check_page_locked(struct extent_io_tree *tree, struct page *page)
Chris Masond1310b22008-01-24 16:13:08 -05001804{
1805 u64 start = (u64)page->index << PAGE_CACHE_SHIFT;
1806 u64 end = start + PAGE_CACHE_SIZE - 1;
Chris Mason9655d292009-09-02 15:22:30 -04001807 if (!test_range_bit(tree, start, end, EXTENT_LOCKED, 0, NULL))
Chris Masond1310b22008-01-24 16:13:08 -05001808 unlock_page(page);
Chris Masond1310b22008-01-24 16:13:08 -05001809}
1810
1811/*
1812 * helper function to end page writeback if all the extents
1813 * in the tree for that page are done with writeback
1814 */
Jeff Mahoney143bede2012-03-01 14:56:26 +01001815static void check_page_writeback(struct extent_io_tree *tree,
1816 struct page *page)
Chris Masond1310b22008-01-24 16:13:08 -05001817{
Chris Mason1edbb732009-09-02 13:24:36 -04001818 end_page_writeback(page);
Chris Masond1310b22008-01-24 16:13:08 -05001819}
1820
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02001821/*
1822 * When IO fails, either with EIO or csum verification fails, we
1823 * try other mirrors that might have a good copy of the data. This
1824 * io_failure_record is used to record state as we go through all the
1825 * mirrors. If another mirror has good data, the page is set up to date
1826 * and things continue. If a good mirror can't be found, the original
1827 * bio end_io callback is called to indicate things have failed.
1828 */
1829struct io_failure_record {
1830 struct page *page;
1831 u64 start;
1832 u64 len;
1833 u64 logical;
1834 unsigned long bio_flags;
1835 int this_mirror;
1836 int failed_mirror;
1837 int in_validation;
1838};
1839
1840static int free_io_failure(struct inode *inode, struct io_failure_record *rec,
1841 int did_repair)
1842{
1843 int ret;
1844 int err = 0;
1845 struct extent_io_tree *failure_tree = &BTRFS_I(inode)->io_failure_tree;
1846
1847 set_state_private(failure_tree, rec->start, 0);
1848 ret = clear_extent_bits(failure_tree, rec->start,
1849 rec->start + rec->len - 1,
1850 EXTENT_LOCKED | EXTENT_DIRTY, GFP_NOFS);
1851 if (ret)
1852 err = ret;
1853
1854 if (did_repair) {
1855 ret = clear_extent_bits(&BTRFS_I(inode)->io_tree, rec->start,
1856 rec->start + rec->len - 1,
1857 EXTENT_DAMAGED, GFP_NOFS);
1858 if (ret && !err)
1859 err = ret;
1860 }
1861
1862 kfree(rec);
1863 return err;
1864}
1865
1866static void repair_io_failure_callback(struct bio *bio, int err)
1867{
1868 complete(bio->bi_private);
1869}
1870
1871/*
1872 * this bypasses the standard btrfs submit functions deliberately, as
1873 * the standard behavior is to write all copies in a raid setup. here we only
1874 * want to write the one bad copy. so we do the mapping for ourselves and issue
1875 * submit_bio directly.
1876 * to avoid any synchonization issues, wait for the data after writing, which
1877 * actually prevents the read that triggered the error from finishing.
1878 * currently, there can be no more than two copies of every data bit. thus,
1879 * exactly one rewrite is required.
1880 */
1881int repair_io_failure(struct btrfs_mapping_tree *map_tree, u64 start,
1882 u64 length, u64 logical, struct page *page,
1883 int mirror_num)
1884{
1885 struct bio *bio;
1886 struct btrfs_device *dev;
1887 DECLARE_COMPLETION_ONSTACK(compl);
1888 u64 map_length = 0;
1889 u64 sector;
1890 struct btrfs_bio *bbio = NULL;
1891 int ret;
1892
1893 BUG_ON(!mirror_num);
1894
1895 bio = bio_alloc(GFP_NOFS, 1);
1896 if (!bio)
1897 return -EIO;
1898 bio->bi_private = &compl;
1899 bio->bi_end_io = repair_io_failure_callback;
1900 bio->bi_size = 0;
1901 map_length = length;
1902
1903 ret = btrfs_map_block(map_tree, WRITE, logical,
1904 &map_length, &bbio, mirror_num);
1905 if (ret) {
1906 bio_put(bio);
1907 return -EIO;
1908 }
1909 BUG_ON(mirror_num != bbio->mirror_num);
1910 sector = bbio->stripes[mirror_num-1].physical >> 9;
1911 bio->bi_sector = sector;
1912 dev = bbio->stripes[mirror_num-1].dev;
1913 kfree(bbio);
1914 if (!dev || !dev->bdev || !dev->writeable) {
1915 bio_put(bio);
1916 return -EIO;
1917 }
1918 bio->bi_bdev = dev->bdev;
1919 bio_add_page(bio, page, length, start-page_offset(page));
Stefan Behrens21adbd52011-11-09 13:44:05 +01001920 btrfsic_submit_bio(WRITE_SYNC, bio);
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02001921 wait_for_completion(&compl);
1922
1923 if (!test_bit(BIO_UPTODATE, &bio->bi_flags)) {
1924 /* try to remap that extent elsewhere? */
1925 bio_put(bio);
1926 return -EIO;
1927 }
1928
1929 printk(KERN_INFO "btrfs read error corrected: ino %lu off %llu (dev %s "
1930 "sector %llu)\n", page->mapping->host->i_ino, start,
1931 dev->name, sector);
1932
1933 bio_put(bio);
1934 return 0;
1935}
1936
Josef Bacikea466792012-03-26 21:57:36 -04001937int repair_eb_io_failure(struct btrfs_root *root, struct extent_buffer *eb,
1938 int mirror_num)
1939{
1940 struct btrfs_mapping_tree *map_tree = &root->fs_info->mapping_tree;
1941 u64 start = eb->start;
1942 unsigned long i, num_pages = num_extent_pages(eb->start, eb->len);
Chris Masond95603b2012-04-12 15:55:15 -04001943 int ret = 0;
Josef Bacikea466792012-03-26 21:57:36 -04001944
1945 for (i = 0; i < num_pages; i++) {
1946 struct page *p = extent_buffer_page(eb, i);
1947 ret = repair_io_failure(map_tree, start, PAGE_CACHE_SIZE,
1948 start, p, mirror_num);
1949 if (ret)
1950 break;
1951 start += PAGE_CACHE_SIZE;
1952 }
1953
1954 return ret;
1955}
1956
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02001957/*
1958 * each time an IO finishes, we do a fast check in the IO failure tree
1959 * to see if we need to process or clean up an io_failure_record
1960 */
1961static int clean_io_failure(u64 start, struct page *page)
1962{
1963 u64 private;
1964 u64 private_failure;
1965 struct io_failure_record *failrec;
1966 struct btrfs_mapping_tree *map_tree;
1967 struct extent_state *state;
1968 int num_copies;
1969 int did_repair = 0;
1970 int ret;
1971 struct inode *inode = page->mapping->host;
1972
1973 private = 0;
1974 ret = count_range_bits(&BTRFS_I(inode)->io_failure_tree, &private,
1975 (u64)-1, 1, EXTENT_DIRTY, 0);
1976 if (!ret)
1977 return 0;
1978
1979 ret = get_state_private(&BTRFS_I(inode)->io_failure_tree, start,
1980 &private_failure);
1981 if (ret)
1982 return 0;
1983
1984 failrec = (struct io_failure_record *)(unsigned long) private_failure;
1985 BUG_ON(!failrec->this_mirror);
1986
1987 if (failrec->in_validation) {
1988 /* there was no real error, just free the record */
1989 pr_debug("clean_io_failure: freeing dummy error at %llu\n",
1990 failrec->start);
1991 did_repair = 1;
1992 goto out;
1993 }
1994
1995 spin_lock(&BTRFS_I(inode)->io_tree.lock);
1996 state = find_first_extent_bit_state(&BTRFS_I(inode)->io_tree,
1997 failrec->start,
1998 EXTENT_LOCKED);
1999 spin_unlock(&BTRFS_I(inode)->io_tree.lock);
2000
2001 if (state && state->start == failrec->start) {
2002 map_tree = &BTRFS_I(inode)->root->fs_info->mapping_tree;
2003 num_copies = btrfs_num_copies(map_tree, failrec->logical,
2004 failrec->len);
2005 if (num_copies > 1) {
2006 ret = repair_io_failure(map_tree, start, failrec->len,
2007 failrec->logical, page,
2008 failrec->failed_mirror);
2009 did_repair = !ret;
2010 }
2011 }
2012
2013out:
2014 if (!ret)
2015 ret = free_io_failure(inode, failrec, did_repair);
2016
2017 return ret;
2018}
2019
2020/*
2021 * this is a generic handler for readpage errors (default
2022 * readpage_io_failed_hook). if other copies exist, read those and write back
2023 * good data to the failed position. does not investigate in remapping the
2024 * failed extent elsewhere, hoping the device will be smart enough to do this as
2025 * needed
2026 */
2027
2028static int bio_readpage_error(struct bio *failed_bio, struct page *page,
2029 u64 start, u64 end, int failed_mirror,
2030 struct extent_state *state)
2031{
2032 struct io_failure_record *failrec = NULL;
2033 u64 private;
2034 struct extent_map *em;
2035 struct inode *inode = page->mapping->host;
2036 struct extent_io_tree *failure_tree = &BTRFS_I(inode)->io_failure_tree;
2037 struct extent_io_tree *tree = &BTRFS_I(inode)->io_tree;
2038 struct extent_map_tree *em_tree = &BTRFS_I(inode)->extent_tree;
2039 struct bio *bio;
2040 int num_copies;
2041 int ret;
2042 int read_mode;
2043 u64 logical;
2044
2045 BUG_ON(failed_bio->bi_rw & REQ_WRITE);
2046
2047 ret = get_state_private(failure_tree, start, &private);
2048 if (ret) {
2049 failrec = kzalloc(sizeof(*failrec), GFP_NOFS);
2050 if (!failrec)
2051 return -ENOMEM;
2052 failrec->start = start;
2053 failrec->len = end - start + 1;
2054 failrec->this_mirror = 0;
2055 failrec->bio_flags = 0;
2056 failrec->in_validation = 0;
2057
2058 read_lock(&em_tree->lock);
2059 em = lookup_extent_mapping(em_tree, start, failrec->len);
2060 if (!em) {
2061 read_unlock(&em_tree->lock);
2062 kfree(failrec);
2063 return -EIO;
2064 }
2065
2066 if (em->start > start || em->start + em->len < start) {
2067 free_extent_map(em);
2068 em = NULL;
2069 }
2070 read_unlock(&em_tree->lock);
2071
2072 if (!em || IS_ERR(em)) {
2073 kfree(failrec);
2074 return -EIO;
2075 }
2076 logical = start - em->start;
2077 logical = em->block_start + logical;
2078 if (test_bit(EXTENT_FLAG_COMPRESSED, &em->flags)) {
2079 logical = em->block_start;
2080 failrec->bio_flags = EXTENT_BIO_COMPRESSED;
2081 extent_set_compress_type(&failrec->bio_flags,
2082 em->compress_type);
2083 }
2084 pr_debug("bio_readpage_error: (new) logical=%llu, start=%llu, "
2085 "len=%llu\n", logical, start, failrec->len);
2086 failrec->logical = logical;
2087 free_extent_map(em);
2088
2089 /* set the bits in the private failure tree */
2090 ret = set_extent_bits(failure_tree, start, end,
2091 EXTENT_LOCKED | EXTENT_DIRTY, GFP_NOFS);
2092 if (ret >= 0)
2093 ret = set_state_private(failure_tree, start,
2094 (u64)(unsigned long)failrec);
2095 /* set the bits in the inode's tree */
2096 if (ret >= 0)
2097 ret = set_extent_bits(tree, start, end, EXTENT_DAMAGED,
2098 GFP_NOFS);
2099 if (ret < 0) {
2100 kfree(failrec);
2101 return ret;
2102 }
2103 } else {
2104 failrec = (struct io_failure_record *)(unsigned long)private;
2105 pr_debug("bio_readpage_error: (found) logical=%llu, "
2106 "start=%llu, len=%llu, validation=%d\n",
2107 failrec->logical, failrec->start, failrec->len,
2108 failrec->in_validation);
2109 /*
2110 * when data can be on disk more than twice, add to failrec here
2111 * (e.g. with a list for failed_mirror) to make
2112 * clean_io_failure() clean all those errors at once.
2113 */
2114 }
2115 num_copies = btrfs_num_copies(
2116 &BTRFS_I(inode)->root->fs_info->mapping_tree,
2117 failrec->logical, failrec->len);
2118 if (num_copies == 1) {
2119 /*
2120 * we only have a single copy of the data, so don't bother with
2121 * all the retry and error correction code that follows. no
2122 * matter what the error is, it is very likely to persist.
2123 */
2124 pr_debug("bio_readpage_error: cannot repair, num_copies == 1. "
2125 "state=%p, num_copies=%d, next_mirror %d, "
2126 "failed_mirror %d\n", state, num_copies,
2127 failrec->this_mirror, failed_mirror);
2128 free_io_failure(inode, failrec, 0);
2129 return -EIO;
2130 }
2131
2132 if (!state) {
2133 spin_lock(&tree->lock);
2134 state = find_first_extent_bit_state(tree, failrec->start,
2135 EXTENT_LOCKED);
2136 if (state && state->start != failrec->start)
2137 state = NULL;
2138 spin_unlock(&tree->lock);
2139 }
2140
2141 /*
2142 * there are two premises:
2143 * a) deliver good data to the caller
2144 * b) correct the bad sectors on disk
2145 */
2146 if (failed_bio->bi_vcnt > 1) {
2147 /*
2148 * to fulfill b), we need to know the exact failing sectors, as
2149 * we don't want to rewrite any more than the failed ones. thus,
2150 * we need separate read requests for the failed bio
2151 *
2152 * if the following BUG_ON triggers, our validation request got
2153 * merged. we need separate requests for our algorithm to work.
2154 */
2155 BUG_ON(failrec->in_validation);
2156 failrec->in_validation = 1;
2157 failrec->this_mirror = failed_mirror;
2158 read_mode = READ_SYNC | REQ_FAILFAST_DEV;
2159 } else {
2160 /*
2161 * we're ready to fulfill a) and b) alongside. get a good copy
2162 * of the failed sector and if we succeed, we have setup
2163 * everything for repair_io_failure to do the rest for us.
2164 */
2165 if (failrec->in_validation) {
2166 BUG_ON(failrec->this_mirror != failed_mirror);
2167 failrec->in_validation = 0;
2168 failrec->this_mirror = 0;
2169 }
2170 failrec->failed_mirror = failed_mirror;
2171 failrec->this_mirror++;
2172 if (failrec->this_mirror == failed_mirror)
2173 failrec->this_mirror++;
2174 read_mode = READ_SYNC;
2175 }
2176
2177 if (!state || failrec->this_mirror > num_copies) {
2178 pr_debug("bio_readpage_error: (fail) state=%p, num_copies=%d, "
2179 "next_mirror %d, failed_mirror %d\n", state,
2180 num_copies, failrec->this_mirror, failed_mirror);
2181 free_io_failure(inode, failrec, 0);
2182 return -EIO;
2183 }
2184
2185 bio = bio_alloc(GFP_NOFS, 1);
Tsutomu Itohe627ee72012-04-12 16:03:56 -04002186 if (!bio) {
2187 free_io_failure(inode, failrec, 0);
2188 return -EIO;
2189 }
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002190 bio->bi_private = state;
2191 bio->bi_end_io = failed_bio->bi_end_io;
2192 bio->bi_sector = failrec->logical >> 9;
2193 bio->bi_bdev = BTRFS_I(inode)->root->fs_info->fs_devices->latest_bdev;
2194 bio->bi_size = 0;
2195
2196 bio_add_page(bio, page, failrec->len, start - page_offset(page));
2197
2198 pr_debug("bio_readpage_error: submitting new read[%#x] to "
2199 "this_mirror=%d, num_copies=%d, in_validation=%d\n", read_mode,
2200 failrec->this_mirror, num_copies, failrec->in_validation);
2201
Tsutomu Itoh013bd4c2012-02-16 10:11:40 +09002202 ret = tree->ops->submit_bio_hook(inode, read_mode, bio,
2203 failrec->this_mirror,
2204 failrec->bio_flags, 0);
2205 return ret;
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002206}
2207
Chris Masond1310b22008-01-24 16:13:08 -05002208/* lots and lots of room for performance fixes in the end_bio funcs */
2209
Jeff Mahoney87826df2012-02-15 16:23:57 +01002210int end_extent_writepage(struct page *page, int err, u64 start, u64 end)
2211{
2212 int uptodate = (err == 0);
2213 struct extent_io_tree *tree;
2214 int ret;
2215
2216 tree = &BTRFS_I(page->mapping->host)->io_tree;
2217
2218 if (tree->ops && tree->ops->writepage_end_io_hook) {
2219 ret = tree->ops->writepage_end_io_hook(page, start,
2220 end, NULL, uptodate);
2221 if (ret)
2222 uptodate = 0;
2223 }
2224
2225 if (!uptodate && tree->ops &&
2226 tree->ops->writepage_io_failed_hook) {
2227 ret = tree->ops->writepage_io_failed_hook(NULL, page,
2228 start, end, NULL);
2229 /* Writeback already completed */
2230 if (ret == 0)
2231 return 1;
2232 }
2233
2234 if (!uptodate) {
2235 clear_extent_uptodate(tree, start, end, NULL, GFP_NOFS);
2236 ClearPageUptodate(page);
2237 SetPageError(page);
2238 }
2239 return 0;
2240}
2241
Chris Masond1310b22008-01-24 16:13:08 -05002242/*
2243 * after a writepage IO is done, we need to:
2244 * clear the uptodate bits on error
2245 * clear the writeback bits in the extent tree for this IO
2246 * end_page_writeback if the page has no more pending IO
2247 *
2248 * Scheduling is not allowed, so the extent state tree is expected
2249 * to have one and only one object corresponding to this IO.
2250 */
Chris Masond1310b22008-01-24 16:13:08 -05002251static void end_bio_extent_writepage(struct bio *bio, int err)
Chris Masond1310b22008-01-24 16:13:08 -05002252{
Chris Masond1310b22008-01-24 16:13:08 -05002253 struct bio_vec *bvec = bio->bi_io_vec + bio->bi_vcnt - 1;
David Woodhouse902b22f2008-08-20 08:51:49 -04002254 struct extent_io_tree *tree;
Chris Masond1310b22008-01-24 16:13:08 -05002255 u64 start;
2256 u64 end;
2257 int whole_page;
2258
Chris Masond1310b22008-01-24 16:13:08 -05002259 do {
2260 struct page *page = bvec->bv_page;
David Woodhouse902b22f2008-08-20 08:51:49 -04002261 tree = &BTRFS_I(page->mapping->host)->io_tree;
2262
Chris Masond1310b22008-01-24 16:13:08 -05002263 start = ((u64)page->index << PAGE_CACHE_SHIFT) +
2264 bvec->bv_offset;
2265 end = start + bvec->bv_len - 1;
2266
2267 if (bvec->bv_offset == 0 && bvec->bv_len == PAGE_CACHE_SIZE)
2268 whole_page = 1;
2269 else
2270 whole_page = 0;
2271
2272 if (--bvec >= bio->bi_io_vec)
2273 prefetchw(&bvec->bv_page->flags);
Chris Mason1259ab72008-05-12 13:39:03 -04002274
Jeff Mahoney87826df2012-02-15 16:23:57 +01002275 if (end_extent_writepage(page, err, start, end))
2276 continue;
Chris Mason70dec802008-01-29 09:59:12 -05002277
Chris Masond1310b22008-01-24 16:13:08 -05002278 if (whole_page)
2279 end_page_writeback(page);
2280 else
2281 check_page_writeback(tree, page);
Chris Masond1310b22008-01-24 16:13:08 -05002282 } while (bvec >= bio->bi_io_vec);
Chris Mason2b1f55b2008-09-24 11:48:04 -04002283
Chris Masond1310b22008-01-24 16:13:08 -05002284 bio_put(bio);
Chris Masond1310b22008-01-24 16:13:08 -05002285}
2286
2287/*
2288 * after a readpage IO is done, we need to:
2289 * clear the uptodate bits on error
2290 * set the uptodate bits if things worked
2291 * set the page up to date if all extents in the tree are uptodate
2292 * clear the lock bit in the extent tree
2293 * unlock the page if there are no other extents locked for it
2294 *
2295 * Scheduling is not allowed, so the extent state tree is expected
2296 * to have one and only one object corresponding to this IO.
2297 */
Chris Masond1310b22008-01-24 16:13:08 -05002298static void end_bio_extent_readpage(struct bio *bio, int err)
Chris Masond1310b22008-01-24 16:13:08 -05002299{
2300 int uptodate = test_bit(BIO_UPTODATE, &bio->bi_flags);
Chris Mason4125bf72010-02-03 18:18:45 +00002301 struct bio_vec *bvec_end = bio->bi_io_vec + bio->bi_vcnt - 1;
2302 struct bio_vec *bvec = bio->bi_io_vec;
David Woodhouse902b22f2008-08-20 08:51:49 -04002303 struct extent_io_tree *tree;
Chris Masond1310b22008-01-24 16:13:08 -05002304 u64 start;
2305 u64 end;
2306 int whole_page;
Josef Bacik5cf1ab52012-04-16 09:42:26 -04002307 int mirror;
Chris Masond1310b22008-01-24 16:13:08 -05002308 int ret;
2309
Chris Masond20f7042008-12-08 16:58:54 -05002310 if (err)
2311 uptodate = 0;
2312
Chris Masond1310b22008-01-24 16:13:08 -05002313 do {
2314 struct page *page = bvec->bv_page;
Arne Jansen507903b2011-04-06 10:02:20 +00002315 struct extent_state *cached = NULL;
2316 struct extent_state *state;
2317
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002318 pr_debug("end_bio_extent_readpage: bi_vcnt=%d, idx=%d, err=%d, "
2319 "mirror=%ld\n", bio->bi_vcnt, bio->bi_idx, err,
2320 (long int)bio->bi_bdev);
David Woodhouse902b22f2008-08-20 08:51:49 -04002321 tree = &BTRFS_I(page->mapping->host)->io_tree;
2322
Chris Masond1310b22008-01-24 16:13:08 -05002323 start = ((u64)page->index << PAGE_CACHE_SHIFT) +
2324 bvec->bv_offset;
2325 end = start + bvec->bv_len - 1;
2326
2327 if (bvec->bv_offset == 0 && bvec->bv_len == PAGE_CACHE_SIZE)
2328 whole_page = 1;
2329 else
2330 whole_page = 0;
2331
Chris Mason4125bf72010-02-03 18:18:45 +00002332 if (++bvec <= bvec_end)
Chris Masond1310b22008-01-24 16:13:08 -05002333 prefetchw(&bvec->bv_page->flags);
2334
Arne Jansen507903b2011-04-06 10:02:20 +00002335 spin_lock(&tree->lock);
Chris Mason0d399202011-04-16 06:55:39 -04002336 state = find_first_extent_bit_state(tree, start, EXTENT_LOCKED);
Chris Mason109b36a2011-04-12 13:57:39 -04002337 if (state && state->start == start) {
Arne Jansen507903b2011-04-06 10:02:20 +00002338 /*
2339 * take a reference on the state, unlock will drop
2340 * the ref
2341 */
2342 cache_state(state, &cached);
2343 }
2344 spin_unlock(&tree->lock);
2345
Josef Bacik5cf1ab52012-04-16 09:42:26 -04002346 mirror = (int)(unsigned long)bio->bi_bdev;
Chris Masond1310b22008-01-24 16:13:08 -05002347 if (uptodate && tree->ops && tree->ops->readpage_end_io_hook) {
Chris Mason70dec802008-01-29 09:59:12 -05002348 ret = tree->ops->readpage_end_io_hook(page, start, end,
Josef Bacik5cf1ab52012-04-16 09:42:26 -04002349 state, mirror);
Chris Masond1310b22008-01-24 16:13:08 -05002350 if (ret)
2351 uptodate = 0;
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002352 else
2353 clean_io_failure(start, page);
Chris Masond1310b22008-01-24 16:13:08 -05002354 }
Josef Bacikea466792012-03-26 21:57:36 -04002355
Josef Bacikea466792012-03-26 21:57:36 -04002356 if (!uptodate && tree->ops && tree->ops->readpage_io_failed_hook) {
Josef Bacik5cf1ab52012-04-16 09:42:26 -04002357 ret = tree->ops->readpage_io_failed_hook(page, mirror);
Josef Bacikea466792012-03-26 21:57:36 -04002358 if (!ret && !err &&
2359 test_bit(BIO_UPTODATE, &bio->bi_flags))
2360 uptodate = 1;
2361 } else if (!uptodate) {
Jan Schmidtf4a8e652011-12-01 09:30:36 -05002362 /*
2363 * The generic bio_readpage_error handles errors the
2364 * following way: If possible, new read requests are
2365 * created and submitted and will end up in
2366 * end_bio_extent_readpage as well (if we're lucky, not
2367 * in the !uptodate case). In that case it returns 0 and
2368 * we just go on with the next page in our bio. If it
2369 * can't handle the error it will return -EIO and we
2370 * remain responsible for that page.
2371 */
Josef Bacik5cf1ab52012-04-16 09:42:26 -04002372 ret = bio_readpage_error(bio, page, start, end, mirror, NULL);
Chris Mason7e383262008-04-09 16:28:12 -04002373 if (ret == 0) {
Chris Mason3b951512008-04-17 11:29:12 -04002374 uptodate =
2375 test_bit(BIO_UPTODATE, &bio->bi_flags);
Chris Masond20f7042008-12-08 16:58:54 -05002376 if (err)
2377 uptodate = 0;
Arne Jansen507903b2011-04-06 10:02:20 +00002378 uncache_state(&cached);
Chris Mason7e383262008-04-09 16:28:12 -04002379 continue;
2380 }
2381 }
Chris Mason70dec802008-01-29 09:59:12 -05002382
Josef Bacik0b32f4b2012-03-13 09:38:00 -04002383 if (uptodate && tree->track_uptodate) {
Arne Jansen507903b2011-04-06 10:02:20 +00002384 set_extent_uptodate(tree, start, end, &cached,
David Woodhouse902b22f2008-08-20 08:51:49 -04002385 GFP_ATOMIC);
Chris Mason771ed682008-11-06 22:02:51 -05002386 }
Arne Jansen507903b2011-04-06 10:02:20 +00002387 unlock_extent_cached(tree, start, end, &cached, GFP_ATOMIC);
Chris Masond1310b22008-01-24 16:13:08 -05002388
Chris Mason70dec802008-01-29 09:59:12 -05002389 if (whole_page) {
2390 if (uptodate) {
2391 SetPageUptodate(page);
2392 } else {
2393 ClearPageUptodate(page);
2394 SetPageError(page);
2395 }
Chris Masond1310b22008-01-24 16:13:08 -05002396 unlock_page(page);
Chris Mason70dec802008-01-29 09:59:12 -05002397 } else {
2398 if (uptodate) {
2399 check_page_uptodate(tree, page);
2400 } else {
2401 ClearPageUptodate(page);
2402 SetPageError(page);
2403 }
Chris Masond1310b22008-01-24 16:13:08 -05002404 check_page_locked(tree, page);
Chris Mason70dec802008-01-29 09:59:12 -05002405 }
Chris Mason4125bf72010-02-03 18:18:45 +00002406 } while (bvec <= bvec_end);
Chris Masond1310b22008-01-24 16:13:08 -05002407
2408 bio_put(bio);
Chris Masond1310b22008-01-24 16:13:08 -05002409}
2410
Miao Xie88f794e2010-11-22 03:02:55 +00002411struct bio *
2412btrfs_bio_alloc(struct block_device *bdev, u64 first_sector, int nr_vecs,
2413 gfp_t gfp_flags)
Chris Masond1310b22008-01-24 16:13:08 -05002414{
2415 struct bio *bio;
2416
2417 bio = bio_alloc(gfp_flags, nr_vecs);
2418
2419 if (bio == NULL && (current->flags & PF_MEMALLOC)) {
2420 while (!bio && (nr_vecs /= 2))
2421 bio = bio_alloc(gfp_flags, nr_vecs);
2422 }
2423
2424 if (bio) {
Chris Masone1c4b742008-04-22 13:26:46 -04002425 bio->bi_size = 0;
Chris Masond1310b22008-01-24 16:13:08 -05002426 bio->bi_bdev = bdev;
2427 bio->bi_sector = first_sector;
2428 }
2429 return bio;
2430}
2431
Jeff Mahoney79787ea2012-03-12 16:03:00 +01002432/*
2433 * Since writes are async, they will only return -ENOMEM.
2434 * Reads can return the full range of I/O error conditions.
2435 */
Jeff Mahoney355808c2011-10-03 23:23:14 -04002436static int __must_check submit_one_bio(int rw, struct bio *bio,
2437 int mirror_num, unsigned long bio_flags)
Chris Masond1310b22008-01-24 16:13:08 -05002438{
Chris Masond1310b22008-01-24 16:13:08 -05002439 int ret = 0;
Chris Mason70dec802008-01-29 09:59:12 -05002440 struct bio_vec *bvec = bio->bi_io_vec + bio->bi_vcnt - 1;
2441 struct page *page = bvec->bv_page;
2442 struct extent_io_tree *tree = bio->bi_private;
Chris Mason70dec802008-01-29 09:59:12 -05002443 u64 start;
Chris Mason70dec802008-01-29 09:59:12 -05002444
2445 start = ((u64)page->index << PAGE_CACHE_SHIFT) + bvec->bv_offset;
Chris Mason70dec802008-01-29 09:59:12 -05002446
David Woodhouse902b22f2008-08-20 08:51:49 -04002447 bio->bi_private = NULL;
Chris Masond1310b22008-01-24 16:13:08 -05002448
2449 bio_get(bio);
2450
Chris Mason065631f2008-02-20 12:07:25 -05002451 if (tree->ops && tree->ops->submit_bio_hook)
liubo6b82ce82011-01-26 06:21:39 +00002452 ret = tree->ops->submit_bio_hook(page->mapping->host, rw, bio,
Chris Masoneaf25d92010-05-25 09:48:28 -04002453 mirror_num, bio_flags, start);
Chris Mason0b86a832008-03-24 15:01:56 -04002454 else
Stefan Behrens21adbd52011-11-09 13:44:05 +01002455 btrfsic_submit_bio(rw, bio);
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002456
Chris Masond1310b22008-01-24 16:13:08 -05002457 if (bio_flagged(bio, BIO_EOPNOTSUPP))
2458 ret = -EOPNOTSUPP;
2459 bio_put(bio);
2460 return ret;
2461}
2462
Jeff Mahoney3444a972011-10-03 23:23:13 -04002463static int merge_bio(struct extent_io_tree *tree, struct page *page,
2464 unsigned long offset, size_t size, struct bio *bio,
2465 unsigned long bio_flags)
2466{
2467 int ret = 0;
2468 if (tree->ops && tree->ops->merge_bio_hook)
2469 ret = tree->ops->merge_bio_hook(page, offset, size, bio,
2470 bio_flags);
2471 BUG_ON(ret < 0);
2472 return ret;
2473
2474}
2475
Chris Masond1310b22008-01-24 16:13:08 -05002476static int submit_extent_page(int rw, struct extent_io_tree *tree,
2477 struct page *page, sector_t sector,
2478 size_t size, unsigned long offset,
2479 struct block_device *bdev,
2480 struct bio **bio_ret,
2481 unsigned long max_pages,
Chris Masonf1885912008-04-09 16:28:12 -04002482 bio_end_io_t end_io_func,
Chris Masonc8b97812008-10-29 14:49:59 -04002483 int mirror_num,
2484 unsigned long prev_bio_flags,
2485 unsigned long bio_flags)
Chris Masond1310b22008-01-24 16:13:08 -05002486{
2487 int ret = 0;
2488 struct bio *bio;
2489 int nr;
Chris Masonc8b97812008-10-29 14:49:59 -04002490 int contig = 0;
2491 int this_compressed = bio_flags & EXTENT_BIO_COMPRESSED;
2492 int old_compressed = prev_bio_flags & EXTENT_BIO_COMPRESSED;
Chris Mason5b050f02008-11-11 09:34:41 -05002493 size_t page_size = min_t(size_t, size, PAGE_CACHE_SIZE);
Chris Masond1310b22008-01-24 16:13:08 -05002494
2495 if (bio_ret && *bio_ret) {
2496 bio = *bio_ret;
Chris Masonc8b97812008-10-29 14:49:59 -04002497 if (old_compressed)
2498 contig = bio->bi_sector == sector;
2499 else
2500 contig = bio->bi_sector + (bio->bi_size >> 9) ==
2501 sector;
2502
2503 if (prev_bio_flags != bio_flags || !contig ||
Jeff Mahoney3444a972011-10-03 23:23:13 -04002504 merge_bio(tree, page, offset, page_size, bio, bio_flags) ||
Chris Masonc8b97812008-10-29 14:49:59 -04002505 bio_add_page(bio, page, page_size, offset) < page_size) {
2506 ret = submit_one_bio(rw, bio, mirror_num,
2507 prev_bio_flags);
Jeff Mahoney79787ea2012-03-12 16:03:00 +01002508 if (ret < 0)
2509 return ret;
Chris Masond1310b22008-01-24 16:13:08 -05002510 bio = NULL;
2511 } else {
2512 return 0;
2513 }
2514 }
Chris Masonc8b97812008-10-29 14:49:59 -04002515 if (this_compressed)
2516 nr = BIO_MAX_PAGES;
2517 else
2518 nr = bio_get_nr_vecs(bdev);
2519
Miao Xie88f794e2010-11-22 03:02:55 +00002520 bio = btrfs_bio_alloc(bdev, sector, nr, GFP_NOFS | __GFP_HIGH);
Tsutomu Itoh5df67082011-02-01 09:17:35 +00002521 if (!bio)
2522 return -ENOMEM;
Chris Mason70dec802008-01-29 09:59:12 -05002523
Chris Masonc8b97812008-10-29 14:49:59 -04002524 bio_add_page(bio, page, page_size, offset);
Chris Masond1310b22008-01-24 16:13:08 -05002525 bio->bi_end_io = end_io_func;
2526 bio->bi_private = tree;
Chris Mason70dec802008-01-29 09:59:12 -05002527
Chris Masond3977122009-01-05 21:25:51 -05002528 if (bio_ret)
Chris Masond1310b22008-01-24 16:13:08 -05002529 *bio_ret = bio;
Chris Masond3977122009-01-05 21:25:51 -05002530 else
Chris Masonc8b97812008-10-29 14:49:59 -04002531 ret = submit_one_bio(rw, bio, mirror_num, bio_flags);
Chris Masond1310b22008-01-24 16:13:08 -05002532
2533 return ret;
2534}
2535
Josef Bacik4f2de97a2012-03-07 16:20:05 -05002536void attach_extent_buffer_page(struct extent_buffer *eb, struct page *page)
2537{
2538 if (!PagePrivate(page)) {
2539 SetPagePrivate(page);
2540 page_cache_get(page);
2541 set_page_private(page, (unsigned long)eb);
2542 } else {
2543 WARN_ON(page->private != (unsigned long)eb);
2544 }
2545}
2546
Chris Masond1310b22008-01-24 16:13:08 -05002547void set_page_extent_mapped(struct page *page)
2548{
2549 if (!PagePrivate(page)) {
2550 SetPagePrivate(page);
Chris Masond1310b22008-01-24 16:13:08 -05002551 page_cache_get(page);
Chris Mason6af118c2008-07-22 11:18:07 -04002552 set_page_private(page, EXTENT_PAGE_PRIVATE);
Chris Masond1310b22008-01-24 16:13:08 -05002553 }
2554}
2555
Chris Masond1310b22008-01-24 16:13:08 -05002556/*
2557 * basic readpage implementation. Locked extent state structs are inserted
2558 * into the tree that are removed when the IO is done (by the end_io
2559 * handlers)
Jeff Mahoney79787ea2012-03-12 16:03:00 +01002560 * XXX JDM: This needs looking at to ensure proper page locking
Chris Masond1310b22008-01-24 16:13:08 -05002561 */
2562static int __extent_read_full_page(struct extent_io_tree *tree,
2563 struct page *page,
2564 get_extent_t *get_extent,
Chris Masonc8b97812008-10-29 14:49:59 -04002565 struct bio **bio, int mirror_num,
2566 unsigned long *bio_flags)
Chris Masond1310b22008-01-24 16:13:08 -05002567{
2568 struct inode *inode = page->mapping->host;
2569 u64 start = (u64)page->index << PAGE_CACHE_SHIFT;
2570 u64 page_end = start + PAGE_CACHE_SIZE - 1;
2571 u64 end;
2572 u64 cur = start;
2573 u64 extent_offset;
2574 u64 last_byte = i_size_read(inode);
2575 u64 block_start;
2576 u64 cur_end;
2577 sector_t sector;
2578 struct extent_map *em;
2579 struct block_device *bdev;
Josef Bacik11c65dc2010-05-23 11:07:21 -04002580 struct btrfs_ordered_extent *ordered;
Chris Masond1310b22008-01-24 16:13:08 -05002581 int ret;
2582 int nr = 0;
David Sterba306e16c2011-04-19 14:29:38 +02002583 size_t pg_offset = 0;
Chris Masond1310b22008-01-24 16:13:08 -05002584 size_t iosize;
Chris Masonc8b97812008-10-29 14:49:59 -04002585 size_t disk_io_size;
Chris Masond1310b22008-01-24 16:13:08 -05002586 size_t blocksize = inode->i_sb->s_blocksize;
Chris Masonc8b97812008-10-29 14:49:59 -04002587 unsigned long this_bio_flag = 0;
Chris Masond1310b22008-01-24 16:13:08 -05002588
2589 set_page_extent_mapped(page);
2590
Dan Magenheimer90a887c2011-05-26 10:01:56 -06002591 if (!PageUptodate(page)) {
2592 if (cleancache_get_page(page) == 0) {
2593 BUG_ON(blocksize != PAGE_SIZE);
2594 goto out;
2595 }
2596 }
2597
Chris Masond1310b22008-01-24 16:13:08 -05002598 end = page_end;
Josef Bacik11c65dc2010-05-23 11:07:21 -04002599 while (1) {
Jeff Mahoneyd0082372012-03-01 14:57:19 +01002600 lock_extent(tree, start, end);
Josef Bacik11c65dc2010-05-23 11:07:21 -04002601 ordered = btrfs_lookup_ordered_extent(inode, start);
2602 if (!ordered)
2603 break;
Jeff Mahoneyd0082372012-03-01 14:57:19 +01002604 unlock_extent(tree, start, end);
Josef Bacik11c65dc2010-05-23 11:07:21 -04002605 btrfs_start_ordered_extent(inode, ordered, 1);
2606 btrfs_put_ordered_extent(ordered);
2607 }
Chris Masond1310b22008-01-24 16:13:08 -05002608
Chris Masonc8b97812008-10-29 14:49:59 -04002609 if (page->index == last_byte >> PAGE_CACHE_SHIFT) {
2610 char *userpage;
2611 size_t zero_offset = last_byte & (PAGE_CACHE_SIZE - 1);
2612
2613 if (zero_offset) {
2614 iosize = PAGE_CACHE_SIZE - zero_offset;
Cong Wang7ac687d2011-11-25 23:14:28 +08002615 userpage = kmap_atomic(page);
Chris Masonc8b97812008-10-29 14:49:59 -04002616 memset(userpage + zero_offset, 0, iosize);
2617 flush_dcache_page(page);
Cong Wang7ac687d2011-11-25 23:14:28 +08002618 kunmap_atomic(userpage);
Chris Masonc8b97812008-10-29 14:49:59 -04002619 }
2620 }
Chris Masond1310b22008-01-24 16:13:08 -05002621 while (cur <= end) {
2622 if (cur >= last_byte) {
2623 char *userpage;
Arne Jansen507903b2011-04-06 10:02:20 +00002624 struct extent_state *cached = NULL;
2625
David Sterba306e16c2011-04-19 14:29:38 +02002626 iosize = PAGE_CACHE_SIZE - pg_offset;
Cong Wang7ac687d2011-11-25 23:14:28 +08002627 userpage = kmap_atomic(page);
David Sterba306e16c2011-04-19 14:29:38 +02002628 memset(userpage + pg_offset, 0, iosize);
Chris Masond1310b22008-01-24 16:13:08 -05002629 flush_dcache_page(page);
Cong Wang7ac687d2011-11-25 23:14:28 +08002630 kunmap_atomic(userpage);
Chris Masond1310b22008-01-24 16:13:08 -05002631 set_extent_uptodate(tree, cur, cur + iosize - 1,
Arne Jansen507903b2011-04-06 10:02:20 +00002632 &cached, GFP_NOFS);
2633 unlock_extent_cached(tree, cur, cur + iosize - 1,
2634 &cached, GFP_NOFS);
Chris Masond1310b22008-01-24 16:13:08 -05002635 break;
2636 }
David Sterba306e16c2011-04-19 14:29:38 +02002637 em = get_extent(inode, page, pg_offset, cur,
Chris Masond1310b22008-01-24 16:13:08 -05002638 end - cur + 1, 0);
David Sterbac7040052011-04-19 18:00:01 +02002639 if (IS_ERR_OR_NULL(em)) {
Chris Masond1310b22008-01-24 16:13:08 -05002640 SetPageError(page);
Jeff Mahoneyd0082372012-03-01 14:57:19 +01002641 unlock_extent(tree, cur, end);
Chris Masond1310b22008-01-24 16:13:08 -05002642 break;
2643 }
Chris Masond1310b22008-01-24 16:13:08 -05002644 extent_offset = cur - em->start;
2645 BUG_ON(extent_map_end(em) <= cur);
2646 BUG_ON(end < cur);
2647
Li Zefan261507a02010-12-17 14:21:50 +08002648 if (test_bit(EXTENT_FLAG_COMPRESSED, &em->flags)) {
Chris Masonc8b97812008-10-29 14:49:59 -04002649 this_bio_flag = EXTENT_BIO_COMPRESSED;
Li Zefan261507a02010-12-17 14:21:50 +08002650 extent_set_compress_type(&this_bio_flag,
2651 em->compress_type);
2652 }
Chris Masonc8b97812008-10-29 14:49:59 -04002653
Chris Masond1310b22008-01-24 16:13:08 -05002654 iosize = min(extent_map_end(em) - cur, end - cur + 1);
2655 cur_end = min(extent_map_end(em) - 1, end);
2656 iosize = (iosize + blocksize - 1) & ~((u64)blocksize - 1);
Chris Masonc8b97812008-10-29 14:49:59 -04002657 if (this_bio_flag & EXTENT_BIO_COMPRESSED) {
2658 disk_io_size = em->block_len;
2659 sector = em->block_start >> 9;
2660 } else {
2661 sector = (em->block_start + extent_offset) >> 9;
2662 disk_io_size = iosize;
2663 }
Chris Masond1310b22008-01-24 16:13:08 -05002664 bdev = em->bdev;
2665 block_start = em->block_start;
Yan Zhengd899e052008-10-30 14:25:28 -04002666 if (test_bit(EXTENT_FLAG_PREALLOC, &em->flags))
2667 block_start = EXTENT_MAP_HOLE;
Chris Masond1310b22008-01-24 16:13:08 -05002668 free_extent_map(em);
2669 em = NULL;
2670
2671 /* we've found a hole, just zero and go on */
2672 if (block_start == EXTENT_MAP_HOLE) {
2673 char *userpage;
Arne Jansen507903b2011-04-06 10:02:20 +00002674 struct extent_state *cached = NULL;
2675
Cong Wang7ac687d2011-11-25 23:14:28 +08002676 userpage = kmap_atomic(page);
David Sterba306e16c2011-04-19 14:29:38 +02002677 memset(userpage + pg_offset, 0, iosize);
Chris Masond1310b22008-01-24 16:13:08 -05002678 flush_dcache_page(page);
Cong Wang7ac687d2011-11-25 23:14:28 +08002679 kunmap_atomic(userpage);
Chris Masond1310b22008-01-24 16:13:08 -05002680
2681 set_extent_uptodate(tree, cur, cur + iosize - 1,
Arne Jansen507903b2011-04-06 10:02:20 +00002682 &cached, GFP_NOFS);
2683 unlock_extent_cached(tree, cur, cur + iosize - 1,
2684 &cached, GFP_NOFS);
Chris Masond1310b22008-01-24 16:13:08 -05002685 cur = cur + iosize;
David Sterba306e16c2011-04-19 14:29:38 +02002686 pg_offset += iosize;
Chris Masond1310b22008-01-24 16:13:08 -05002687 continue;
2688 }
2689 /* the get_extent function already copied into the page */
Chris Mason9655d292009-09-02 15:22:30 -04002690 if (test_range_bit(tree, cur, cur_end,
2691 EXTENT_UPTODATE, 1, NULL)) {
Chris Masona1b32a52008-09-05 16:09:51 -04002692 check_page_uptodate(tree, page);
Jeff Mahoneyd0082372012-03-01 14:57:19 +01002693 unlock_extent(tree, cur, cur + iosize - 1);
Chris Masond1310b22008-01-24 16:13:08 -05002694 cur = cur + iosize;
David Sterba306e16c2011-04-19 14:29:38 +02002695 pg_offset += iosize;
Chris Masond1310b22008-01-24 16:13:08 -05002696 continue;
2697 }
Chris Mason70dec802008-01-29 09:59:12 -05002698 /* we have an inline extent but it didn't get marked up
2699 * to date. Error out
2700 */
2701 if (block_start == EXTENT_MAP_INLINE) {
2702 SetPageError(page);
Jeff Mahoneyd0082372012-03-01 14:57:19 +01002703 unlock_extent(tree, cur, cur + iosize - 1);
Chris Mason70dec802008-01-29 09:59:12 -05002704 cur = cur + iosize;
David Sterba306e16c2011-04-19 14:29:38 +02002705 pg_offset += iosize;
Chris Mason70dec802008-01-29 09:59:12 -05002706 continue;
2707 }
Chris Masond1310b22008-01-24 16:13:08 -05002708
2709 ret = 0;
2710 if (tree->ops && tree->ops->readpage_io_hook) {
2711 ret = tree->ops->readpage_io_hook(page, cur,
2712 cur + iosize - 1);
2713 }
2714 if (!ret) {
Chris Mason89642222008-07-24 09:41:53 -04002715 unsigned long pnr = (last_byte >> PAGE_CACHE_SHIFT) + 1;
2716 pnr -= page->index;
Chris Masond1310b22008-01-24 16:13:08 -05002717 ret = submit_extent_page(READ, tree, page,
David Sterba306e16c2011-04-19 14:29:38 +02002718 sector, disk_io_size, pg_offset,
Chris Mason89642222008-07-24 09:41:53 -04002719 bdev, bio, pnr,
Chris Masonc8b97812008-10-29 14:49:59 -04002720 end_bio_extent_readpage, mirror_num,
2721 *bio_flags,
2722 this_bio_flag);
Jeff Mahoney79787ea2012-03-12 16:03:00 +01002723 BUG_ON(ret == -ENOMEM);
Chris Mason89642222008-07-24 09:41:53 -04002724 nr++;
Chris Masonc8b97812008-10-29 14:49:59 -04002725 *bio_flags = this_bio_flag;
Chris Masond1310b22008-01-24 16:13:08 -05002726 }
2727 if (ret)
2728 SetPageError(page);
2729 cur = cur + iosize;
David Sterba306e16c2011-04-19 14:29:38 +02002730 pg_offset += iosize;
Chris Masond1310b22008-01-24 16:13:08 -05002731 }
Dan Magenheimer90a887c2011-05-26 10:01:56 -06002732out:
Chris Masond1310b22008-01-24 16:13:08 -05002733 if (!nr) {
2734 if (!PageError(page))
2735 SetPageUptodate(page);
2736 unlock_page(page);
2737 }
2738 return 0;
2739}
2740
2741int extent_read_full_page(struct extent_io_tree *tree, struct page *page,
Jan Schmidt8ddc7d92011-06-13 20:02:58 +02002742 get_extent_t *get_extent, int mirror_num)
Chris Masond1310b22008-01-24 16:13:08 -05002743{
2744 struct bio *bio = NULL;
Chris Masonc8b97812008-10-29 14:49:59 -04002745 unsigned long bio_flags = 0;
Chris Masond1310b22008-01-24 16:13:08 -05002746 int ret;
2747
Jan Schmidt8ddc7d92011-06-13 20:02:58 +02002748 ret = __extent_read_full_page(tree, page, get_extent, &bio, mirror_num,
Chris Masonc8b97812008-10-29 14:49:59 -04002749 &bio_flags);
Chris Masond1310b22008-01-24 16:13:08 -05002750 if (bio)
Jan Schmidt8ddc7d92011-06-13 20:02:58 +02002751 ret = submit_one_bio(READ, bio, mirror_num, bio_flags);
Chris Masond1310b22008-01-24 16:13:08 -05002752 return ret;
2753}
Chris Masond1310b22008-01-24 16:13:08 -05002754
Chris Mason11c83492009-04-20 15:50:09 -04002755static noinline void update_nr_written(struct page *page,
2756 struct writeback_control *wbc,
2757 unsigned long nr_written)
2758{
2759 wbc->nr_to_write -= nr_written;
2760 if (wbc->range_cyclic || (wbc->nr_to_write > 0 &&
2761 wbc->range_start == 0 && wbc->range_end == LLONG_MAX))
2762 page->mapping->writeback_index = page->index + nr_written;
2763}
2764
Chris Masond1310b22008-01-24 16:13:08 -05002765/*
2766 * the writepage semantics are similar to regular writepage. extent
2767 * records are inserted to lock ranges in the tree, and as dirty areas
2768 * are found, they are marked writeback. Then the lock bits are removed
2769 * and the end_io handler clears the writeback ranges
2770 */
2771static int __extent_writepage(struct page *page, struct writeback_control *wbc,
2772 void *data)
2773{
2774 struct inode *inode = page->mapping->host;
2775 struct extent_page_data *epd = data;
2776 struct extent_io_tree *tree = epd->tree;
2777 u64 start = (u64)page->index << PAGE_CACHE_SHIFT;
2778 u64 delalloc_start;
2779 u64 page_end = start + PAGE_CACHE_SIZE - 1;
2780 u64 end;
2781 u64 cur = start;
2782 u64 extent_offset;
2783 u64 last_byte = i_size_read(inode);
2784 u64 block_start;
2785 u64 iosize;
2786 sector_t sector;
Chris Mason2c64c532009-09-02 15:04:12 -04002787 struct extent_state *cached_state = NULL;
Chris Masond1310b22008-01-24 16:13:08 -05002788 struct extent_map *em;
2789 struct block_device *bdev;
2790 int ret;
2791 int nr = 0;
Chris Mason7f3c74f2008-07-18 12:01:11 -04002792 size_t pg_offset = 0;
Chris Masond1310b22008-01-24 16:13:08 -05002793 size_t blocksize;
2794 loff_t i_size = i_size_read(inode);
2795 unsigned long end_index = i_size >> PAGE_CACHE_SHIFT;
2796 u64 nr_delalloc;
2797 u64 delalloc_end;
Chris Masonc8b97812008-10-29 14:49:59 -04002798 int page_started;
2799 int compressed;
Chris Masonffbd5172009-04-20 15:50:09 -04002800 int write_flags;
Chris Mason771ed682008-11-06 22:02:51 -05002801 unsigned long nr_written = 0;
Josef Bacik9e487102011-08-01 12:08:18 -04002802 bool fill_delalloc = true;
Chris Masond1310b22008-01-24 16:13:08 -05002803
Chris Masonffbd5172009-04-20 15:50:09 -04002804 if (wbc->sync_mode == WB_SYNC_ALL)
Jens Axboe721a9602011-03-09 11:56:30 +01002805 write_flags = WRITE_SYNC;
Chris Masonffbd5172009-04-20 15:50:09 -04002806 else
2807 write_flags = WRITE;
2808
liubo1abe9b82011-03-24 11:18:59 +00002809 trace___extent_writepage(page, inode, wbc);
2810
Chris Masond1310b22008-01-24 16:13:08 -05002811 WARN_ON(!PageLocked(page));
Chris Masonbf0da8c2011-11-04 12:29:37 -04002812
2813 ClearPageError(page);
2814
Chris Mason7f3c74f2008-07-18 12:01:11 -04002815 pg_offset = i_size & (PAGE_CACHE_SIZE - 1);
Chris Mason211c17f2008-05-15 09:13:45 -04002816 if (page->index > end_index ||
Chris Mason7f3c74f2008-07-18 12:01:11 -04002817 (page->index == end_index && !pg_offset)) {
Chris Mason39be25c2008-11-10 11:50:50 -05002818 page->mapping->a_ops->invalidatepage(page, 0);
Chris Masond1310b22008-01-24 16:13:08 -05002819 unlock_page(page);
2820 return 0;
2821 }
2822
2823 if (page->index == end_index) {
2824 char *userpage;
2825
Cong Wang7ac687d2011-11-25 23:14:28 +08002826 userpage = kmap_atomic(page);
Chris Mason7f3c74f2008-07-18 12:01:11 -04002827 memset(userpage + pg_offset, 0,
2828 PAGE_CACHE_SIZE - pg_offset);
Cong Wang7ac687d2011-11-25 23:14:28 +08002829 kunmap_atomic(userpage);
Chris Mason211c17f2008-05-15 09:13:45 -04002830 flush_dcache_page(page);
Chris Masond1310b22008-01-24 16:13:08 -05002831 }
Chris Mason7f3c74f2008-07-18 12:01:11 -04002832 pg_offset = 0;
Chris Masond1310b22008-01-24 16:13:08 -05002833
2834 set_page_extent_mapped(page);
2835
Josef Bacik9e487102011-08-01 12:08:18 -04002836 if (!tree->ops || !tree->ops->fill_delalloc)
2837 fill_delalloc = false;
2838
Chris Masond1310b22008-01-24 16:13:08 -05002839 delalloc_start = start;
2840 delalloc_end = 0;
Chris Masonc8b97812008-10-29 14:49:59 -04002841 page_started = 0;
Josef Bacik9e487102011-08-01 12:08:18 -04002842 if (!epd->extent_locked && fill_delalloc) {
Chris Masonf85d7d6c2009-09-18 16:03:16 -04002843 u64 delalloc_to_write = 0;
Chris Mason11c83492009-04-20 15:50:09 -04002844 /*
2845 * make sure the wbc mapping index is at least updated
2846 * to this page.
2847 */
2848 update_nr_written(page, wbc, 0);
2849
Chris Masond3977122009-01-05 21:25:51 -05002850 while (delalloc_end < page_end) {
Chris Mason771ed682008-11-06 22:02:51 -05002851 nr_delalloc = find_lock_delalloc_range(inode, tree,
Chris Masonc8b97812008-10-29 14:49:59 -04002852 page,
2853 &delalloc_start,
Chris Masond1310b22008-01-24 16:13:08 -05002854 &delalloc_end,
2855 128 * 1024 * 1024);
Chris Mason771ed682008-11-06 22:02:51 -05002856 if (nr_delalloc == 0) {
2857 delalloc_start = delalloc_end + 1;
2858 continue;
2859 }
Tsutomu Itoh013bd4c2012-02-16 10:11:40 +09002860 ret = tree->ops->fill_delalloc(inode, page,
2861 delalloc_start,
2862 delalloc_end,
2863 &page_started,
2864 &nr_written);
Jeff Mahoney79787ea2012-03-12 16:03:00 +01002865 /* File system has been set read-only */
2866 if (ret) {
2867 SetPageError(page);
2868 goto done;
2869 }
Chris Masonf85d7d6c2009-09-18 16:03:16 -04002870 /*
2871 * delalloc_end is already one less than the total
2872 * length, so we don't subtract one from
2873 * PAGE_CACHE_SIZE
2874 */
2875 delalloc_to_write += (delalloc_end - delalloc_start +
2876 PAGE_CACHE_SIZE) >>
2877 PAGE_CACHE_SHIFT;
Chris Masond1310b22008-01-24 16:13:08 -05002878 delalloc_start = delalloc_end + 1;
Chris Masond1310b22008-01-24 16:13:08 -05002879 }
Chris Masonf85d7d6c2009-09-18 16:03:16 -04002880 if (wbc->nr_to_write < delalloc_to_write) {
2881 int thresh = 8192;
2882
2883 if (delalloc_to_write < thresh * 2)
2884 thresh = delalloc_to_write;
2885 wbc->nr_to_write = min_t(u64, delalloc_to_write,
2886 thresh);
2887 }
Chris Masonc8b97812008-10-29 14:49:59 -04002888
Chris Mason771ed682008-11-06 22:02:51 -05002889 /* did the fill delalloc function already unlock and start
2890 * the IO?
2891 */
2892 if (page_started) {
2893 ret = 0;
Chris Mason11c83492009-04-20 15:50:09 -04002894 /*
2895 * we've unlocked the page, so we can't update
2896 * the mapping's writeback index, just update
2897 * nr_to_write.
2898 */
2899 wbc->nr_to_write -= nr_written;
2900 goto done_unlocked;
Chris Mason771ed682008-11-06 22:02:51 -05002901 }
Chris Masonc8b97812008-10-29 14:49:59 -04002902 }
Chris Mason247e7432008-07-17 12:53:51 -04002903 if (tree->ops && tree->ops->writepage_start_hook) {
Chris Masonc8b97812008-10-29 14:49:59 -04002904 ret = tree->ops->writepage_start_hook(page, start,
2905 page_end);
Jeff Mahoney87826df2012-02-15 16:23:57 +01002906 if (ret) {
2907 /* Fixup worker will requeue */
2908 if (ret == -EBUSY)
2909 wbc->pages_skipped++;
2910 else
2911 redirty_page_for_writepage(wbc, page);
Chris Mason11c83492009-04-20 15:50:09 -04002912 update_nr_written(page, wbc, nr_written);
Chris Mason247e7432008-07-17 12:53:51 -04002913 unlock_page(page);
Chris Mason771ed682008-11-06 22:02:51 -05002914 ret = 0;
Chris Mason11c83492009-04-20 15:50:09 -04002915 goto done_unlocked;
Chris Mason247e7432008-07-17 12:53:51 -04002916 }
2917 }
2918
Chris Mason11c83492009-04-20 15:50:09 -04002919 /*
2920 * we don't want to touch the inode after unlocking the page,
2921 * so we update the mapping writeback index now
2922 */
2923 update_nr_written(page, wbc, nr_written + 1);
Chris Mason771ed682008-11-06 22:02:51 -05002924
Chris Masond1310b22008-01-24 16:13:08 -05002925 end = page_end;
Chris Masond1310b22008-01-24 16:13:08 -05002926 if (last_byte <= start) {
Chris Masone6dcd2d2008-07-17 12:53:50 -04002927 if (tree->ops && tree->ops->writepage_end_io_hook)
2928 tree->ops->writepage_end_io_hook(page, start,
2929 page_end, NULL, 1);
Chris Masond1310b22008-01-24 16:13:08 -05002930 goto done;
2931 }
2932
Chris Masond1310b22008-01-24 16:13:08 -05002933 blocksize = inode->i_sb->s_blocksize;
2934
2935 while (cur <= end) {
2936 if (cur >= last_byte) {
Chris Masone6dcd2d2008-07-17 12:53:50 -04002937 if (tree->ops && tree->ops->writepage_end_io_hook)
2938 tree->ops->writepage_end_io_hook(page, cur,
2939 page_end, NULL, 1);
Chris Masond1310b22008-01-24 16:13:08 -05002940 break;
2941 }
Chris Mason7f3c74f2008-07-18 12:01:11 -04002942 em = epd->get_extent(inode, page, pg_offset, cur,
Chris Masond1310b22008-01-24 16:13:08 -05002943 end - cur + 1, 1);
David Sterbac7040052011-04-19 18:00:01 +02002944 if (IS_ERR_OR_NULL(em)) {
Chris Masond1310b22008-01-24 16:13:08 -05002945 SetPageError(page);
2946 break;
2947 }
2948
2949 extent_offset = cur - em->start;
2950 BUG_ON(extent_map_end(em) <= cur);
2951 BUG_ON(end < cur);
2952 iosize = min(extent_map_end(em) - cur, end - cur + 1);
2953 iosize = (iosize + blocksize - 1) & ~((u64)blocksize - 1);
2954 sector = (em->block_start + extent_offset) >> 9;
2955 bdev = em->bdev;
2956 block_start = em->block_start;
Chris Masonc8b97812008-10-29 14:49:59 -04002957 compressed = test_bit(EXTENT_FLAG_COMPRESSED, &em->flags);
Chris Masond1310b22008-01-24 16:13:08 -05002958 free_extent_map(em);
2959 em = NULL;
2960
Chris Masonc8b97812008-10-29 14:49:59 -04002961 /*
2962 * compressed and inline extents are written through other
2963 * paths in the FS
2964 */
2965 if (compressed || block_start == EXTENT_MAP_HOLE ||
Chris Masond1310b22008-01-24 16:13:08 -05002966 block_start == EXTENT_MAP_INLINE) {
Chris Masonc8b97812008-10-29 14:49:59 -04002967 /*
2968 * end_io notification does not happen here for
2969 * compressed extents
2970 */
2971 if (!compressed && tree->ops &&
2972 tree->ops->writepage_end_io_hook)
Chris Masone6dcd2d2008-07-17 12:53:50 -04002973 tree->ops->writepage_end_io_hook(page, cur,
2974 cur + iosize - 1,
2975 NULL, 1);
Chris Masonc8b97812008-10-29 14:49:59 -04002976 else if (compressed) {
2977 /* we don't want to end_page_writeback on
2978 * a compressed extent. this happens
2979 * elsewhere
2980 */
2981 nr++;
2982 }
2983
2984 cur += iosize;
Chris Mason7f3c74f2008-07-18 12:01:11 -04002985 pg_offset += iosize;
Chris Masond1310b22008-01-24 16:13:08 -05002986 continue;
2987 }
Chris Masond1310b22008-01-24 16:13:08 -05002988 /* leave this out until we have a page_mkwrite call */
2989 if (0 && !test_range_bit(tree, cur, cur + iosize - 1,
Chris Mason9655d292009-09-02 15:22:30 -04002990 EXTENT_DIRTY, 0, NULL)) {
Chris Masond1310b22008-01-24 16:13:08 -05002991 cur = cur + iosize;
Chris Mason7f3c74f2008-07-18 12:01:11 -04002992 pg_offset += iosize;
Chris Masond1310b22008-01-24 16:13:08 -05002993 continue;
2994 }
Chris Masonc8b97812008-10-29 14:49:59 -04002995
Chris Masond1310b22008-01-24 16:13:08 -05002996 if (tree->ops && tree->ops->writepage_io_hook) {
2997 ret = tree->ops->writepage_io_hook(page, cur,
2998 cur + iosize - 1);
2999 } else {
3000 ret = 0;
3001 }
Chris Mason1259ab72008-05-12 13:39:03 -04003002 if (ret) {
Chris Masond1310b22008-01-24 16:13:08 -05003003 SetPageError(page);
Chris Mason1259ab72008-05-12 13:39:03 -04003004 } else {
Chris Masond1310b22008-01-24 16:13:08 -05003005 unsigned long max_nr = end_index + 1;
Chris Mason7f3c74f2008-07-18 12:01:11 -04003006
Chris Masond1310b22008-01-24 16:13:08 -05003007 set_range_writeback(tree, cur, cur + iosize - 1);
3008 if (!PageWriteback(page)) {
Chris Masond3977122009-01-05 21:25:51 -05003009 printk(KERN_ERR "btrfs warning page %lu not "
3010 "writeback, cur %llu end %llu\n",
3011 page->index, (unsigned long long)cur,
Chris Masond1310b22008-01-24 16:13:08 -05003012 (unsigned long long)end);
3013 }
3014
Chris Masonffbd5172009-04-20 15:50:09 -04003015 ret = submit_extent_page(write_flags, tree, page,
3016 sector, iosize, pg_offset,
3017 bdev, &epd->bio, max_nr,
Chris Masonc8b97812008-10-29 14:49:59 -04003018 end_bio_extent_writepage,
3019 0, 0, 0);
Chris Masond1310b22008-01-24 16:13:08 -05003020 if (ret)
3021 SetPageError(page);
3022 }
3023 cur = cur + iosize;
Chris Mason7f3c74f2008-07-18 12:01:11 -04003024 pg_offset += iosize;
Chris Masond1310b22008-01-24 16:13:08 -05003025 nr++;
3026 }
3027done:
3028 if (nr == 0) {
3029 /* make sure the mapping tag for page dirty gets cleared */
3030 set_page_writeback(page);
3031 end_page_writeback(page);
3032 }
Chris Masond1310b22008-01-24 16:13:08 -05003033 unlock_page(page);
Chris Mason771ed682008-11-06 22:02:51 -05003034
Chris Mason11c83492009-04-20 15:50:09 -04003035done_unlocked:
3036
Chris Mason2c64c532009-09-02 15:04:12 -04003037 /* drop our reference on any cached states */
3038 free_extent_state(cached_state);
Chris Masond1310b22008-01-24 16:13:08 -05003039 return 0;
3040}
3041
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003042static int eb_wait(void *word)
3043{
3044 io_schedule();
3045 return 0;
3046}
3047
3048static void wait_on_extent_buffer_writeback(struct extent_buffer *eb)
3049{
3050 wait_on_bit(&eb->bflags, EXTENT_BUFFER_WRITEBACK, eb_wait,
3051 TASK_UNINTERRUPTIBLE);
3052}
3053
3054static int lock_extent_buffer_for_io(struct extent_buffer *eb,
3055 struct btrfs_fs_info *fs_info,
3056 struct extent_page_data *epd)
3057{
3058 unsigned long i, num_pages;
3059 int flush = 0;
3060 int ret = 0;
3061
3062 if (!btrfs_try_tree_write_lock(eb)) {
3063 flush = 1;
3064 flush_write_bio(epd);
3065 btrfs_tree_lock(eb);
3066 }
3067
3068 if (test_bit(EXTENT_BUFFER_WRITEBACK, &eb->bflags)) {
3069 btrfs_tree_unlock(eb);
3070 if (!epd->sync_io)
3071 return 0;
3072 if (!flush) {
3073 flush_write_bio(epd);
3074 flush = 1;
3075 }
Chris Masona098d8e2012-03-21 12:09:56 -04003076 while (1) {
3077 wait_on_extent_buffer_writeback(eb);
3078 btrfs_tree_lock(eb);
3079 if (!test_bit(EXTENT_BUFFER_WRITEBACK, &eb->bflags))
3080 break;
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003081 btrfs_tree_unlock(eb);
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003082 }
3083 }
3084
3085 if (test_and_clear_bit(EXTENT_BUFFER_DIRTY, &eb->bflags)) {
3086 set_bit(EXTENT_BUFFER_WRITEBACK, &eb->bflags);
3087 btrfs_set_header_flag(eb, BTRFS_HEADER_FLAG_WRITTEN);
3088 spin_lock(&fs_info->delalloc_lock);
3089 if (fs_info->dirty_metadata_bytes >= eb->len)
3090 fs_info->dirty_metadata_bytes -= eb->len;
3091 else
3092 WARN_ON(1);
3093 spin_unlock(&fs_info->delalloc_lock);
3094 ret = 1;
3095 }
3096
3097 btrfs_tree_unlock(eb);
3098
3099 if (!ret)
3100 return ret;
3101
3102 num_pages = num_extent_pages(eb->start, eb->len);
3103 for (i = 0; i < num_pages; i++) {
3104 struct page *p = extent_buffer_page(eb, i);
3105
3106 if (!trylock_page(p)) {
3107 if (!flush) {
3108 flush_write_bio(epd);
3109 flush = 1;
3110 }
3111 lock_page(p);
3112 }
3113 }
3114
3115 return ret;
3116}
3117
3118static void end_extent_buffer_writeback(struct extent_buffer *eb)
3119{
3120 clear_bit(EXTENT_BUFFER_WRITEBACK, &eb->bflags);
3121 smp_mb__after_clear_bit();
3122 wake_up_bit(&eb->bflags, EXTENT_BUFFER_WRITEBACK);
3123}
3124
3125static void end_bio_extent_buffer_writepage(struct bio *bio, int err)
3126{
3127 int uptodate = err == 0;
3128 struct bio_vec *bvec = bio->bi_io_vec + bio->bi_vcnt - 1;
3129 struct extent_buffer *eb;
3130 int done;
3131
3132 do {
3133 struct page *page = bvec->bv_page;
3134
3135 bvec--;
3136 eb = (struct extent_buffer *)page->private;
3137 BUG_ON(!eb);
3138 done = atomic_dec_and_test(&eb->io_pages);
3139
3140 if (!uptodate || test_bit(EXTENT_BUFFER_IOERR, &eb->bflags)) {
3141 set_bit(EXTENT_BUFFER_IOERR, &eb->bflags);
3142 ClearPageUptodate(page);
3143 SetPageError(page);
3144 }
3145
3146 end_page_writeback(page);
3147
3148 if (!done)
3149 continue;
3150
3151 end_extent_buffer_writeback(eb);
3152 } while (bvec >= bio->bi_io_vec);
3153
3154 bio_put(bio);
3155
3156}
3157
3158static int write_one_eb(struct extent_buffer *eb,
3159 struct btrfs_fs_info *fs_info,
3160 struct writeback_control *wbc,
3161 struct extent_page_data *epd)
3162{
3163 struct block_device *bdev = fs_info->fs_devices->latest_bdev;
3164 u64 offset = eb->start;
3165 unsigned long i, num_pages;
3166 int rw = (epd->sync_io ? WRITE_SYNC : WRITE);
3167 int ret;
3168
3169 clear_bit(EXTENT_BUFFER_IOERR, &eb->bflags);
3170 num_pages = num_extent_pages(eb->start, eb->len);
3171 atomic_set(&eb->io_pages, num_pages);
3172 for (i = 0; i < num_pages; i++) {
3173 struct page *p = extent_buffer_page(eb, i);
3174
3175 clear_page_dirty_for_io(p);
3176 set_page_writeback(p);
3177 ret = submit_extent_page(rw, eb->tree, p, offset >> 9,
3178 PAGE_CACHE_SIZE, 0, bdev, &epd->bio,
3179 -1, end_bio_extent_buffer_writepage,
3180 0, 0, 0);
3181 if (ret) {
3182 set_bit(EXTENT_BUFFER_IOERR, &eb->bflags);
3183 SetPageError(p);
3184 if (atomic_sub_and_test(num_pages - i, &eb->io_pages))
3185 end_extent_buffer_writeback(eb);
3186 ret = -EIO;
3187 break;
3188 }
3189 offset += PAGE_CACHE_SIZE;
3190 update_nr_written(p, wbc, 1);
3191 unlock_page(p);
3192 }
3193
3194 if (unlikely(ret)) {
3195 for (; i < num_pages; i++) {
3196 struct page *p = extent_buffer_page(eb, i);
3197 unlock_page(p);
3198 }
3199 }
3200
3201 return ret;
3202}
3203
3204int btree_write_cache_pages(struct address_space *mapping,
3205 struct writeback_control *wbc)
3206{
3207 struct extent_io_tree *tree = &BTRFS_I(mapping->host)->io_tree;
3208 struct btrfs_fs_info *fs_info = BTRFS_I(mapping->host)->root->fs_info;
3209 struct extent_buffer *eb, *prev_eb = NULL;
3210 struct extent_page_data epd = {
3211 .bio = NULL,
3212 .tree = tree,
3213 .extent_locked = 0,
3214 .sync_io = wbc->sync_mode == WB_SYNC_ALL,
3215 };
3216 int ret = 0;
3217 int done = 0;
3218 int nr_to_write_done = 0;
3219 struct pagevec pvec;
3220 int nr_pages;
3221 pgoff_t index;
3222 pgoff_t end; /* Inclusive */
3223 int scanned = 0;
3224 int tag;
3225
3226 pagevec_init(&pvec, 0);
3227 if (wbc->range_cyclic) {
3228 index = mapping->writeback_index; /* Start from prev offset */
3229 end = -1;
3230 } else {
3231 index = wbc->range_start >> PAGE_CACHE_SHIFT;
3232 end = wbc->range_end >> PAGE_CACHE_SHIFT;
3233 scanned = 1;
3234 }
3235 if (wbc->sync_mode == WB_SYNC_ALL)
3236 tag = PAGECACHE_TAG_TOWRITE;
3237 else
3238 tag = PAGECACHE_TAG_DIRTY;
3239retry:
3240 if (wbc->sync_mode == WB_SYNC_ALL)
3241 tag_pages_for_writeback(mapping, index, end);
3242 while (!done && !nr_to_write_done && (index <= end) &&
3243 (nr_pages = pagevec_lookup_tag(&pvec, mapping, &index, tag,
3244 min(end - index, (pgoff_t)PAGEVEC_SIZE-1) + 1))) {
3245 unsigned i;
3246
3247 scanned = 1;
3248 for (i = 0; i < nr_pages; i++) {
3249 struct page *page = pvec.pages[i];
3250
3251 if (!PagePrivate(page))
3252 continue;
3253
3254 if (!wbc->range_cyclic && page->index > end) {
3255 done = 1;
3256 break;
3257 }
3258
3259 eb = (struct extent_buffer *)page->private;
3260 if (!eb) {
3261 WARN_ON(1);
3262 continue;
3263 }
3264
3265 if (eb == prev_eb)
3266 continue;
3267
3268 if (!atomic_inc_not_zero(&eb->refs)) {
3269 WARN_ON(1);
3270 continue;
3271 }
3272
3273 prev_eb = eb;
3274 ret = lock_extent_buffer_for_io(eb, fs_info, &epd);
3275 if (!ret) {
3276 free_extent_buffer(eb);
3277 continue;
3278 }
3279
3280 ret = write_one_eb(eb, fs_info, wbc, &epd);
3281 if (ret) {
3282 done = 1;
3283 free_extent_buffer(eb);
3284 break;
3285 }
3286 free_extent_buffer(eb);
3287
3288 /*
3289 * the filesystem may choose to bump up nr_to_write.
3290 * We have to make sure to honor the new nr_to_write
3291 * at any time
3292 */
3293 nr_to_write_done = wbc->nr_to_write <= 0;
3294 }
3295 pagevec_release(&pvec);
3296 cond_resched();
3297 }
3298 if (!scanned && !done) {
3299 /*
3300 * We hit the last page and there is more work to be done: wrap
3301 * back to the start of the file
3302 */
3303 scanned = 1;
3304 index = 0;
3305 goto retry;
3306 }
3307 flush_write_bio(&epd);
3308 return ret;
3309}
3310
Chris Masond1310b22008-01-24 16:13:08 -05003311/**
Chris Mason4bef0842008-09-08 11:18:08 -04003312 * write_cache_pages - walk the list of dirty pages of the given address space and write all of them.
Chris Masond1310b22008-01-24 16:13:08 -05003313 * @mapping: address space structure to write
3314 * @wbc: subtract the number of written pages from *@wbc->nr_to_write
3315 * @writepage: function called for each page
3316 * @data: data passed to writepage function
3317 *
3318 * If a page is already under I/O, write_cache_pages() skips it, even
3319 * if it's dirty. This is desirable behaviour for memory-cleaning writeback,
3320 * but it is INCORRECT for data-integrity system calls such as fsync(). fsync()
3321 * and msync() need to guarantee that all the data which was dirty at the time
3322 * the call was made get new I/O started against them. If wbc->sync_mode is
3323 * WB_SYNC_ALL then we were called for data integrity and we must wait for
3324 * existing IO to complete.
3325 */
Christoph Hellwigb2950862008-12-02 09:54:17 -05003326static int extent_write_cache_pages(struct extent_io_tree *tree,
Chris Mason4bef0842008-09-08 11:18:08 -04003327 struct address_space *mapping,
3328 struct writeback_control *wbc,
Chris Masond2c3f4f2008-11-19 12:44:22 -05003329 writepage_t writepage, void *data,
3330 void (*flush_fn)(void *))
Chris Masond1310b22008-01-24 16:13:08 -05003331{
Chris Masond1310b22008-01-24 16:13:08 -05003332 int ret = 0;
3333 int done = 0;
Chris Masonf85d7d6c2009-09-18 16:03:16 -04003334 int nr_to_write_done = 0;
Chris Masond1310b22008-01-24 16:13:08 -05003335 struct pagevec pvec;
3336 int nr_pages;
3337 pgoff_t index;
3338 pgoff_t end; /* Inclusive */
3339 int scanned = 0;
Josef Bacikf7aaa062011-07-15 21:26:38 +00003340 int tag;
Chris Masond1310b22008-01-24 16:13:08 -05003341
Chris Masond1310b22008-01-24 16:13:08 -05003342 pagevec_init(&pvec, 0);
3343 if (wbc->range_cyclic) {
3344 index = mapping->writeback_index; /* Start from prev offset */
3345 end = -1;
3346 } else {
3347 index = wbc->range_start >> PAGE_CACHE_SHIFT;
3348 end = wbc->range_end >> PAGE_CACHE_SHIFT;
Chris Masond1310b22008-01-24 16:13:08 -05003349 scanned = 1;
3350 }
Josef Bacikf7aaa062011-07-15 21:26:38 +00003351 if (wbc->sync_mode == WB_SYNC_ALL)
3352 tag = PAGECACHE_TAG_TOWRITE;
3353 else
3354 tag = PAGECACHE_TAG_DIRTY;
Chris Masond1310b22008-01-24 16:13:08 -05003355retry:
Josef Bacikf7aaa062011-07-15 21:26:38 +00003356 if (wbc->sync_mode == WB_SYNC_ALL)
3357 tag_pages_for_writeback(mapping, index, end);
Chris Masonf85d7d6c2009-09-18 16:03:16 -04003358 while (!done && !nr_to_write_done && (index <= end) &&
Josef Bacikf7aaa062011-07-15 21:26:38 +00003359 (nr_pages = pagevec_lookup_tag(&pvec, mapping, &index, tag,
3360 min(end - index, (pgoff_t)PAGEVEC_SIZE-1) + 1))) {
Chris Masond1310b22008-01-24 16:13:08 -05003361 unsigned i;
3362
3363 scanned = 1;
3364 for (i = 0; i < nr_pages; i++) {
3365 struct page *page = pvec.pages[i];
3366
3367 /*
3368 * At this point we hold neither mapping->tree_lock nor
3369 * lock on the page itself: the page may be truncated or
3370 * invalidated (changing page->mapping to NULL), or even
3371 * swizzled back from swapper_space to tmpfs file
3372 * mapping
3373 */
Chris Mason01d658f2011-11-01 10:08:06 -04003374 if (tree->ops &&
3375 tree->ops->write_cache_pages_lock_hook) {
3376 tree->ops->write_cache_pages_lock_hook(page,
3377 data, flush_fn);
3378 } else {
3379 if (!trylock_page(page)) {
3380 flush_fn(data);
3381 lock_page(page);
3382 }
3383 }
Chris Masond1310b22008-01-24 16:13:08 -05003384
3385 if (unlikely(page->mapping != mapping)) {
3386 unlock_page(page);
3387 continue;
3388 }
3389
3390 if (!wbc->range_cyclic && page->index > end) {
3391 done = 1;
3392 unlock_page(page);
3393 continue;
3394 }
3395
Chris Masond2c3f4f2008-11-19 12:44:22 -05003396 if (wbc->sync_mode != WB_SYNC_NONE) {
Chris Mason0e6bd952008-11-20 10:46:35 -05003397 if (PageWriteback(page))
3398 flush_fn(data);
Chris Masond1310b22008-01-24 16:13:08 -05003399 wait_on_page_writeback(page);
Chris Masond2c3f4f2008-11-19 12:44:22 -05003400 }
Chris Masond1310b22008-01-24 16:13:08 -05003401
3402 if (PageWriteback(page) ||
3403 !clear_page_dirty_for_io(page)) {
3404 unlock_page(page);
3405 continue;
3406 }
3407
3408 ret = (*writepage)(page, wbc, data);
3409
3410 if (unlikely(ret == AOP_WRITEPAGE_ACTIVATE)) {
3411 unlock_page(page);
3412 ret = 0;
3413 }
Chris Masonf85d7d6c2009-09-18 16:03:16 -04003414 if (ret)
Chris Masond1310b22008-01-24 16:13:08 -05003415 done = 1;
Chris Masonf85d7d6c2009-09-18 16:03:16 -04003416
3417 /*
3418 * the filesystem may choose to bump up nr_to_write.
3419 * We have to make sure to honor the new nr_to_write
3420 * at any time
3421 */
3422 nr_to_write_done = wbc->nr_to_write <= 0;
Chris Masond1310b22008-01-24 16:13:08 -05003423 }
3424 pagevec_release(&pvec);
3425 cond_resched();
3426 }
3427 if (!scanned && !done) {
3428 /*
3429 * We hit the last page and there is more work to be done: wrap
3430 * back to the start of the file
3431 */
3432 scanned = 1;
3433 index = 0;
3434 goto retry;
3435 }
Chris Masond1310b22008-01-24 16:13:08 -05003436 return ret;
3437}
Chris Masond1310b22008-01-24 16:13:08 -05003438
Chris Masonffbd5172009-04-20 15:50:09 -04003439static void flush_epd_write_bio(struct extent_page_data *epd)
3440{
3441 if (epd->bio) {
Jeff Mahoney355808c2011-10-03 23:23:14 -04003442 int rw = WRITE;
3443 int ret;
3444
Chris Masonffbd5172009-04-20 15:50:09 -04003445 if (epd->sync_io)
Jeff Mahoney355808c2011-10-03 23:23:14 -04003446 rw = WRITE_SYNC;
3447
3448 ret = submit_one_bio(rw, epd->bio, 0, 0);
Jeff Mahoney79787ea2012-03-12 16:03:00 +01003449 BUG_ON(ret < 0); /* -ENOMEM */
Chris Masonffbd5172009-04-20 15:50:09 -04003450 epd->bio = NULL;
3451 }
3452}
3453
Chris Masond2c3f4f2008-11-19 12:44:22 -05003454static noinline void flush_write_bio(void *data)
3455{
3456 struct extent_page_data *epd = data;
Chris Masonffbd5172009-04-20 15:50:09 -04003457 flush_epd_write_bio(epd);
Chris Masond2c3f4f2008-11-19 12:44:22 -05003458}
3459
Chris Masond1310b22008-01-24 16:13:08 -05003460int extent_write_full_page(struct extent_io_tree *tree, struct page *page,
3461 get_extent_t *get_extent,
3462 struct writeback_control *wbc)
3463{
3464 int ret;
Chris Masond1310b22008-01-24 16:13:08 -05003465 struct extent_page_data epd = {
3466 .bio = NULL,
3467 .tree = tree,
3468 .get_extent = get_extent,
Chris Mason771ed682008-11-06 22:02:51 -05003469 .extent_locked = 0,
Chris Masonffbd5172009-04-20 15:50:09 -04003470 .sync_io = wbc->sync_mode == WB_SYNC_ALL,
Chris Masond1310b22008-01-24 16:13:08 -05003471 };
Chris Masond1310b22008-01-24 16:13:08 -05003472
Chris Masond1310b22008-01-24 16:13:08 -05003473 ret = __extent_writepage(page, wbc, &epd);
3474
Chris Masonffbd5172009-04-20 15:50:09 -04003475 flush_epd_write_bio(&epd);
Chris Masond1310b22008-01-24 16:13:08 -05003476 return ret;
3477}
Chris Masond1310b22008-01-24 16:13:08 -05003478
Chris Mason771ed682008-11-06 22:02:51 -05003479int extent_write_locked_range(struct extent_io_tree *tree, struct inode *inode,
3480 u64 start, u64 end, get_extent_t *get_extent,
3481 int mode)
3482{
3483 int ret = 0;
3484 struct address_space *mapping = inode->i_mapping;
3485 struct page *page;
3486 unsigned long nr_pages = (end - start + PAGE_CACHE_SIZE) >>
3487 PAGE_CACHE_SHIFT;
3488
3489 struct extent_page_data epd = {
3490 .bio = NULL,
3491 .tree = tree,
3492 .get_extent = get_extent,
3493 .extent_locked = 1,
Chris Masonffbd5172009-04-20 15:50:09 -04003494 .sync_io = mode == WB_SYNC_ALL,
Chris Mason771ed682008-11-06 22:02:51 -05003495 };
3496 struct writeback_control wbc_writepages = {
Chris Mason771ed682008-11-06 22:02:51 -05003497 .sync_mode = mode,
Chris Mason771ed682008-11-06 22:02:51 -05003498 .nr_to_write = nr_pages * 2,
3499 .range_start = start,
3500 .range_end = end + 1,
3501 };
3502
Chris Masond3977122009-01-05 21:25:51 -05003503 while (start <= end) {
Chris Mason771ed682008-11-06 22:02:51 -05003504 page = find_get_page(mapping, start >> PAGE_CACHE_SHIFT);
3505 if (clear_page_dirty_for_io(page))
3506 ret = __extent_writepage(page, &wbc_writepages, &epd);
3507 else {
3508 if (tree->ops && tree->ops->writepage_end_io_hook)
3509 tree->ops->writepage_end_io_hook(page, start,
3510 start + PAGE_CACHE_SIZE - 1,
3511 NULL, 1);
3512 unlock_page(page);
3513 }
3514 page_cache_release(page);
3515 start += PAGE_CACHE_SIZE;
3516 }
3517
Chris Masonffbd5172009-04-20 15:50:09 -04003518 flush_epd_write_bio(&epd);
Chris Mason771ed682008-11-06 22:02:51 -05003519 return ret;
3520}
Chris Masond1310b22008-01-24 16:13:08 -05003521
3522int extent_writepages(struct extent_io_tree *tree,
3523 struct address_space *mapping,
3524 get_extent_t *get_extent,
3525 struct writeback_control *wbc)
3526{
3527 int ret = 0;
3528 struct extent_page_data epd = {
3529 .bio = NULL,
3530 .tree = tree,
3531 .get_extent = get_extent,
Chris Mason771ed682008-11-06 22:02:51 -05003532 .extent_locked = 0,
Chris Masonffbd5172009-04-20 15:50:09 -04003533 .sync_io = wbc->sync_mode == WB_SYNC_ALL,
Chris Masond1310b22008-01-24 16:13:08 -05003534 };
3535
Chris Mason4bef0842008-09-08 11:18:08 -04003536 ret = extent_write_cache_pages(tree, mapping, wbc,
Chris Masond2c3f4f2008-11-19 12:44:22 -05003537 __extent_writepage, &epd,
3538 flush_write_bio);
Chris Masonffbd5172009-04-20 15:50:09 -04003539 flush_epd_write_bio(&epd);
Chris Masond1310b22008-01-24 16:13:08 -05003540 return ret;
3541}
Chris Masond1310b22008-01-24 16:13:08 -05003542
3543int extent_readpages(struct extent_io_tree *tree,
3544 struct address_space *mapping,
3545 struct list_head *pages, unsigned nr_pages,
3546 get_extent_t get_extent)
3547{
3548 struct bio *bio = NULL;
3549 unsigned page_idx;
Chris Masonc8b97812008-10-29 14:49:59 -04003550 unsigned long bio_flags = 0;
Chris Masond1310b22008-01-24 16:13:08 -05003551
Chris Masond1310b22008-01-24 16:13:08 -05003552 for (page_idx = 0; page_idx < nr_pages; page_idx++) {
3553 struct page *page = list_entry(pages->prev, struct page, lru);
3554
3555 prefetchw(&page->flags);
3556 list_del(&page->lru);
Nick Piggin28ecb602010-03-17 13:31:04 +00003557 if (!add_to_page_cache_lru(page, mapping,
Itaru Kitayama43e817a2011-04-25 19:43:51 -04003558 page->index, GFP_NOFS)) {
Chris Masonf1885912008-04-09 16:28:12 -04003559 __extent_read_full_page(tree, page, get_extent,
Chris Masonc8b97812008-10-29 14:49:59 -04003560 &bio, 0, &bio_flags);
Chris Masond1310b22008-01-24 16:13:08 -05003561 }
3562 page_cache_release(page);
3563 }
Chris Masond1310b22008-01-24 16:13:08 -05003564 BUG_ON(!list_empty(pages));
3565 if (bio)
Jeff Mahoney79787ea2012-03-12 16:03:00 +01003566 return submit_one_bio(READ, bio, 0, bio_flags);
Chris Masond1310b22008-01-24 16:13:08 -05003567 return 0;
3568}
Chris Masond1310b22008-01-24 16:13:08 -05003569
3570/*
3571 * basic invalidatepage code, this waits on any locked or writeback
3572 * ranges corresponding to the page, and then deletes any extent state
3573 * records from the tree
3574 */
3575int extent_invalidatepage(struct extent_io_tree *tree,
3576 struct page *page, unsigned long offset)
3577{
Josef Bacik2ac55d42010-02-03 19:33:23 +00003578 struct extent_state *cached_state = NULL;
Chris Masond1310b22008-01-24 16:13:08 -05003579 u64 start = ((u64)page->index << PAGE_CACHE_SHIFT);
3580 u64 end = start + PAGE_CACHE_SIZE - 1;
3581 size_t blocksize = page->mapping->host->i_sb->s_blocksize;
3582
Chris Masond3977122009-01-05 21:25:51 -05003583 start += (offset + blocksize - 1) & ~(blocksize - 1);
Chris Masond1310b22008-01-24 16:13:08 -05003584 if (start > end)
3585 return 0;
3586
Jeff Mahoneyd0082372012-03-01 14:57:19 +01003587 lock_extent_bits(tree, start, end, 0, &cached_state);
Chris Mason1edbb732009-09-02 13:24:36 -04003588 wait_on_page_writeback(page);
Chris Masond1310b22008-01-24 16:13:08 -05003589 clear_extent_bit(tree, start, end,
Josef Bacik32c00af2009-10-08 13:34:05 -04003590 EXTENT_LOCKED | EXTENT_DIRTY | EXTENT_DELALLOC |
3591 EXTENT_DO_ACCOUNTING,
Josef Bacik2ac55d42010-02-03 19:33:23 +00003592 1, 1, &cached_state, GFP_NOFS);
Chris Masond1310b22008-01-24 16:13:08 -05003593 return 0;
3594}
Chris Masond1310b22008-01-24 16:13:08 -05003595
3596/*
Chris Mason7b13b7b2008-04-18 10:29:50 -04003597 * a helper for releasepage, this tests for areas of the page that
3598 * are locked or under IO and drops the related state bits if it is safe
3599 * to drop the page.
3600 */
3601int try_release_extent_state(struct extent_map_tree *map,
3602 struct extent_io_tree *tree, struct page *page,
3603 gfp_t mask)
3604{
3605 u64 start = (u64)page->index << PAGE_CACHE_SHIFT;
3606 u64 end = start + PAGE_CACHE_SIZE - 1;
3607 int ret = 1;
3608
Chris Mason211f90e2008-07-18 11:56:15 -04003609 if (test_range_bit(tree, start, end,
Chris Mason8b62b722009-09-02 16:53:46 -04003610 EXTENT_IOBITS, 0, NULL))
Chris Mason7b13b7b2008-04-18 10:29:50 -04003611 ret = 0;
3612 else {
3613 if ((mask & GFP_NOFS) == GFP_NOFS)
3614 mask = GFP_NOFS;
Chris Mason11ef1602009-09-23 20:28:46 -04003615 /*
3616 * at this point we can safely clear everything except the
3617 * locked bit and the nodatasum bit
3618 */
Chris Masone3f24cc2011-02-14 12:52:08 -05003619 ret = clear_extent_bit(tree, start, end,
Chris Mason11ef1602009-09-23 20:28:46 -04003620 ~(EXTENT_LOCKED | EXTENT_NODATASUM),
3621 0, 0, NULL, mask);
Chris Masone3f24cc2011-02-14 12:52:08 -05003622
3623 /* if clear_extent_bit failed for enomem reasons,
3624 * we can't allow the release to continue.
3625 */
3626 if (ret < 0)
3627 ret = 0;
3628 else
3629 ret = 1;
Chris Mason7b13b7b2008-04-18 10:29:50 -04003630 }
3631 return ret;
3632}
Chris Mason7b13b7b2008-04-18 10:29:50 -04003633
3634/*
Chris Masond1310b22008-01-24 16:13:08 -05003635 * a helper for releasepage. As long as there are no locked extents
3636 * in the range corresponding to the page, both state records and extent
3637 * map records are removed
3638 */
3639int try_release_extent_mapping(struct extent_map_tree *map,
Chris Mason70dec802008-01-29 09:59:12 -05003640 struct extent_io_tree *tree, struct page *page,
3641 gfp_t mask)
Chris Masond1310b22008-01-24 16:13:08 -05003642{
3643 struct extent_map *em;
3644 u64 start = (u64)page->index << PAGE_CACHE_SHIFT;
3645 u64 end = start + PAGE_CACHE_SIZE - 1;
Chris Mason7b13b7b2008-04-18 10:29:50 -04003646
Chris Mason70dec802008-01-29 09:59:12 -05003647 if ((mask & __GFP_WAIT) &&
3648 page->mapping->host->i_size > 16 * 1024 * 1024) {
Yan39b56372008-02-15 10:40:50 -05003649 u64 len;
Chris Mason70dec802008-01-29 09:59:12 -05003650 while (start <= end) {
Yan39b56372008-02-15 10:40:50 -05003651 len = end - start + 1;
Chris Mason890871b2009-09-02 16:24:52 -04003652 write_lock(&map->lock);
Yan39b56372008-02-15 10:40:50 -05003653 em = lookup_extent_mapping(map, start, len);
Tsutomu Itoh285190d2012-02-16 16:23:58 +09003654 if (!em) {
Chris Mason890871b2009-09-02 16:24:52 -04003655 write_unlock(&map->lock);
Chris Mason70dec802008-01-29 09:59:12 -05003656 break;
3657 }
Chris Mason7f3c74f2008-07-18 12:01:11 -04003658 if (test_bit(EXTENT_FLAG_PINNED, &em->flags) ||
3659 em->start != start) {
Chris Mason890871b2009-09-02 16:24:52 -04003660 write_unlock(&map->lock);
Chris Mason70dec802008-01-29 09:59:12 -05003661 free_extent_map(em);
3662 break;
3663 }
3664 if (!test_range_bit(tree, em->start,
3665 extent_map_end(em) - 1,
Chris Mason8b62b722009-09-02 16:53:46 -04003666 EXTENT_LOCKED | EXTENT_WRITEBACK,
Chris Mason9655d292009-09-02 15:22:30 -04003667 0, NULL)) {
Chris Mason70dec802008-01-29 09:59:12 -05003668 remove_extent_mapping(map, em);
3669 /* once for the rb tree */
3670 free_extent_map(em);
3671 }
3672 start = extent_map_end(em);
Chris Mason890871b2009-09-02 16:24:52 -04003673 write_unlock(&map->lock);
Chris Mason70dec802008-01-29 09:59:12 -05003674
3675 /* once for us */
Chris Masond1310b22008-01-24 16:13:08 -05003676 free_extent_map(em);
3677 }
Chris Masond1310b22008-01-24 16:13:08 -05003678 }
Chris Mason7b13b7b2008-04-18 10:29:50 -04003679 return try_release_extent_state(map, tree, page, mask);
Chris Masond1310b22008-01-24 16:13:08 -05003680}
Chris Masond1310b22008-01-24 16:13:08 -05003681
Chris Masonec29ed52011-02-23 16:23:20 -05003682/*
3683 * helper function for fiemap, which doesn't want to see any holes.
3684 * This maps until we find something past 'last'
3685 */
3686static struct extent_map *get_extent_skip_holes(struct inode *inode,
3687 u64 offset,
3688 u64 last,
3689 get_extent_t *get_extent)
3690{
3691 u64 sectorsize = BTRFS_I(inode)->root->sectorsize;
3692 struct extent_map *em;
3693 u64 len;
3694
3695 if (offset >= last)
3696 return NULL;
3697
3698 while(1) {
3699 len = last - offset;
3700 if (len == 0)
3701 break;
3702 len = (len + sectorsize - 1) & ~(sectorsize - 1);
3703 em = get_extent(inode, NULL, 0, offset, len, 0);
David Sterbac7040052011-04-19 18:00:01 +02003704 if (IS_ERR_OR_NULL(em))
Chris Masonec29ed52011-02-23 16:23:20 -05003705 return em;
3706
3707 /* if this isn't a hole return it */
3708 if (!test_bit(EXTENT_FLAG_VACANCY, &em->flags) &&
3709 em->block_start != EXTENT_MAP_HOLE) {
3710 return em;
3711 }
3712
3713 /* this is a hole, advance to the next extent */
3714 offset = extent_map_end(em);
3715 free_extent_map(em);
3716 if (offset >= last)
3717 break;
3718 }
3719 return NULL;
3720}
3721
Yehuda Sadeh1506fcc2009-01-21 14:39:14 -05003722int extent_fiemap(struct inode *inode, struct fiemap_extent_info *fieinfo,
3723 __u64 start, __u64 len, get_extent_t *get_extent)
3724{
Josef Bacik975f84f2010-11-23 19:36:57 +00003725 int ret = 0;
Yehuda Sadeh1506fcc2009-01-21 14:39:14 -05003726 u64 off = start;
3727 u64 max = start + len;
3728 u32 flags = 0;
Josef Bacik975f84f2010-11-23 19:36:57 +00003729 u32 found_type;
3730 u64 last;
Chris Masonec29ed52011-02-23 16:23:20 -05003731 u64 last_for_get_extent = 0;
Yehuda Sadeh1506fcc2009-01-21 14:39:14 -05003732 u64 disko = 0;
Chris Masonec29ed52011-02-23 16:23:20 -05003733 u64 isize = i_size_read(inode);
Josef Bacik975f84f2010-11-23 19:36:57 +00003734 struct btrfs_key found_key;
Yehuda Sadeh1506fcc2009-01-21 14:39:14 -05003735 struct extent_map *em = NULL;
Josef Bacik2ac55d42010-02-03 19:33:23 +00003736 struct extent_state *cached_state = NULL;
Josef Bacik975f84f2010-11-23 19:36:57 +00003737 struct btrfs_path *path;
3738 struct btrfs_file_extent_item *item;
Yehuda Sadeh1506fcc2009-01-21 14:39:14 -05003739 int end = 0;
Chris Masonec29ed52011-02-23 16:23:20 -05003740 u64 em_start = 0;
3741 u64 em_len = 0;
3742 u64 em_end = 0;
Yehuda Sadeh1506fcc2009-01-21 14:39:14 -05003743 unsigned long emflags;
Yehuda Sadeh1506fcc2009-01-21 14:39:14 -05003744
3745 if (len == 0)
3746 return -EINVAL;
3747
Josef Bacik975f84f2010-11-23 19:36:57 +00003748 path = btrfs_alloc_path();
3749 if (!path)
3750 return -ENOMEM;
3751 path->leave_spinning = 1;
3752
Josef Bacik4d479cf2011-11-17 11:34:31 -05003753 start = ALIGN(start, BTRFS_I(inode)->root->sectorsize);
3754 len = ALIGN(len, BTRFS_I(inode)->root->sectorsize);
3755
Chris Masonec29ed52011-02-23 16:23:20 -05003756 /*
3757 * lookup the last file extent. We're not using i_size here
3758 * because there might be preallocation past i_size
3759 */
Josef Bacik975f84f2010-11-23 19:36:57 +00003760 ret = btrfs_lookup_file_extent(NULL, BTRFS_I(inode)->root,
Li Zefan33345d012011-04-20 10:31:50 +08003761 path, btrfs_ino(inode), -1, 0);
Josef Bacik975f84f2010-11-23 19:36:57 +00003762 if (ret < 0) {
3763 btrfs_free_path(path);
3764 return ret;
3765 }
3766 WARN_ON(!ret);
3767 path->slots[0]--;
3768 item = btrfs_item_ptr(path->nodes[0], path->slots[0],
3769 struct btrfs_file_extent_item);
3770 btrfs_item_key_to_cpu(path->nodes[0], &found_key, path->slots[0]);
3771 found_type = btrfs_key_type(&found_key);
3772
Chris Masonec29ed52011-02-23 16:23:20 -05003773 /* No extents, but there might be delalloc bits */
Li Zefan33345d012011-04-20 10:31:50 +08003774 if (found_key.objectid != btrfs_ino(inode) ||
Josef Bacik975f84f2010-11-23 19:36:57 +00003775 found_type != BTRFS_EXTENT_DATA_KEY) {
Chris Masonec29ed52011-02-23 16:23:20 -05003776 /* have to trust i_size as the end */
3777 last = (u64)-1;
3778 last_for_get_extent = isize;
3779 } else {
3780 /*
3781 * remember the start of the last extent. There are a
3782 * bunch of different factors that go into the length of the
3783 * extent, so its much less complex to remember where it started
3784 */
3785 last = found_key.offset;
3786 last_for_get_extent = last + 1;
Josef Bacik975f84f2010-11-23 19:36:57 +00003787 }
Josef Bacik975f84f2010-11-23 19:36:57 +00003788 btrfs_free_path(path);
3789
Chris Masonec29ed52011-02-23 16:23:20 -05003790 /*
3791 * we might have some extents allocated but more delalloc past those
3792 * extents. so, we trust isize unless the start of the last extent is
3793 * beyond isize
3794 */
3795 if (last < isize) {
3796 last = (u64)-1;
3797 last_for_get_extent = isize;
3798 }
3799
Josef Bacik2ac55d42010-02-03 19:33:23 +00003800 lock_extent_bits(&BTRFS_I(inode)->io_tree, start, start + len, 0,
Jeff Mahoneyd0082372012-03-01 14:57:19 +01003801 &cached_state);
Chris Masonec29ed52011-02-23 16:23:20 -05003802
Josef Bacik4d479cf2011-11-17 11:34:31 -05003803 em = get_extent_skip_holes(inode, start, last_for_get_extent,
Chris Masonec29ed52011-02-23 16:23:20 -05003804 get_extent);
Yehuda Sadeh1506fcc2009-01-21 14:39:14 -05003805 if (!em)
3806 goto out;
3807 if (IS_ERR(em)) {
3808 ret = PTR_ERR(em);
3809 goto out;
3810 }
Josef Bacik975f84f2010-11-23 19:36:57 +00003811
Yehuda Sadeh1506fcc2009-01-21 14:39:14 -05003812 while (!end) {
Chris Masonea8efc72011-03-08 11:54:40 -05003813 u64 offset_in_extent;
Yehuda Sadeh1506fcc2009-01-21 14:39:14 -05003814
Chris Masonea8efc72011-03-08 11:54:40 -05003815 /* break if the extent we found is outside the range */
3816 if (em->start >= max || extent_map_end(em) < off)
3817 break;
3818
3819 /*
3820 * get_extent may return an extent that starts before our
3821 * requested range. We have to make sure the ranges
3822 * we return to fiemap always move forward and don't
3823 * overlap, so adjust the offsets here
3824 */
3825 em_start = max(em->start, off);
3826
3827 /*
3828 * record the offset from the start of the extent
3829 * for adjusting the disk offset below
3830 */
3831 offset_in_extent = em_start - em->start;
Chris Masonec29ed52011-02-23 16:23:20 -05003832 em_end = extent_map_end(em);
Chris Masonea8efc72011-03-08 11:54:40 -05003833 em_len = em_end - em_start;
Chris Masonec29ed52011-02-23 16:23:20 -05003834 emflags = em->flags;
Yehuda Sadeh1506fcc2009-01-21 14:39:14 -05003835 disko = 0;
3836 flags = 0;
3837
Chris Masonea8efc72011-03-08 11:54:40 -05003838 /*
3839 * bump off for our next call to get_extent
3840 */
3841 off = extent_map_end(em);
3842 if (off >= max)
3843 end = 1;
3844
Heiko Carstens93dbfad2009-04-03 10:33:45 -04003845 if (em->block_start == EXTENT_MAP_LAST_BYTE) {
Yehuda Sadeh1506fcc2009-01-21 14:39:14 -05003846 end = 1;
3847 flags |= FIEMAP_EXTENT_LAST;
Heiko Carstens93dbfad2009-04-03 10:33:45 -04003848 } else if (em->block_start == EXTENT_MAP_INLINE) {
Yehuda Sadeh1506fcc2009-01-21 14:39:14 -05003849 flags |= (FIEMAP_EXTENT_DATA_INLINE |
3850 FIEMAP_EXTENT_NOT_ALIGNED);
Heiko Carstens93dbfad2009-04-03 10:33:45 -04003851 } else if (em->block_start == EXTENT_MAP_DELALLOC) {
Yehuda Sadeh1506fcc2009-01-21 14:39:14 -05003852 flags |= (FIEMAP_EXTENT_DELALLOC |
3853 FIEMAP_EXTENT_UNKNOWN);
Heiko Carstens93dbfad2009-04-03 10:33:45 -04003854 } else {
Chris Masonea8efc72011-03-08 11:54:40 -05003855 disko = em->block_start + offset_in_extent;
Yehuda Sadeh1506fcc2009-01-21 14:39:14 -05003856 }
3857 if (test_bit(EXTENT_FLAG_COMPRESSED, &em->flags))
3858 flags |= FIEMAP_EXTENT_ENCODED;
3859
Yehuda Sadeh1506fcc2009-01-21 14:39:14 -05003860 free_extent_map(em);
3861 em = NULL;
Chris Masonec29ed52011-02-23 16:23:20 -05003862 if ((em_start >= last) || em_len == (u64)-1 ||
3863 (last == (u64)-1 && isize <= em_end)) {
Yehuda Sadeh1506fcc2009-01-21 14:39:14 -05003864 flags |= FIEMAP_EXTENT_LAST;
3865 end = 1;
3866 }
3867
Chris Masonec29ed52011-02-23 16:23:20 -05003868 /* now scan forward to see if this is really the last extent. */
3869 em = get_extent_skip_holes(inode, off, last_for_get_extent,
3870 get_extent);
3871 if (IS_ERR(em)) {
3872 ret = PTR_ERR(em);
3873 goto out;
3874 }
3875 if (!em) {
Josef Bacik975f84f2010-11-23 19:36:57 +00003876 flags |= FIEMAP_EXTENT_LAST;
3877 end = 1;
3878 }
Chris Masonec29ed52011-02-23 16:23:20 -05003879 ret = fiemap_fill_next_extent(fieinfo, em_start, disko,
3880 em_len, flags);
3881 if (ret)
3882 goto out_free;
Yehuda Sadeh1506fcc2009-01-21 14:39:14 -05003883 }
3884out_free:
3885 free_extent_map(em);
3886out:
Josef Bacik2ac55d42010-02-03 19:33:23 +00003887 unlock_extent_cached(&BTRFS_I(inode)->io_tree, start, start + len,
3888 &cached_state, GFP_NOFS);
Yehuda Sadeh1506fcc2009-01-21 14:39:14 -05003889 return ret;
3890}
3891
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02003892inline struct page *extent_buffer_page(struct extent_buffer *eb,
Chris Masond1310b22008-01-24 16:13:08 -05003893 unsigned long i)
3894{
Chris Mason727011e2010-08-06 13:21:20 -04003895 return eb->pages[i];
Chris Masond1310b22008-01-24 16:13:08 -05003896}
3897
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02003898inline unsigned long num_extent_pages(u64 start, u64 len)
Chris Masonce9adaa2008-04-09 16:28:12 -04003899{
Chris Mason6af118c2008-07-22 11:18:07 -04003900 return ((start + len + PAGE_CACHE_SIZE - 1) >> PAGE_CACHE_SHIFT) -
3901 (start >> PAGE_CACHE_SHIFT);
Chris Mason728131d2008-04-09 16:28:12 -04003902}
3903
Chris Mason727011e2010-08-06 13:21:20 -04003904static void __free_extent_buffer(struct extent_buffer *eb)
3905{
3906#if LEAK_DEBUG
3907 unsigned long flags;
3908 spin_lock_irqsave(&leak_lock, flags);
3909 list_del(&eb->leak_list);
3910 spin_unlock_irqrestore(&leak_lock, flags);
3911#endif
3912 if (eb->pages && eb->pages != eb->inline_pages)
3913 kfree(eb->pages);
3914 kmem_cache_free(extent_buffer_cache, eb);
3915}
3916
Chris Masond1310b22008-01-24 16:13:08 -05003917static struct extent_buffer *__alloc_extent_buffer(struct extent_io_tree *tree,
3918 u64 start,
3919 unsigned long len,
3920 gfp_t mask)
3921{
3922 struct extent_buffer *eb = NULL;
Chris Mason39351272009-02-04 09:24:05 -05003923#if LEAK_DEBUG
Chris Mason2d2ae542008-03-26 16:24:23 -04003924 unsigned long flags;
Chris Mason4bef0842008-09-08 11:18:08 -04003925#endif
Chris Masond1310b22008-01-24 16:13:08 -05003926
Chris Masond1310b22008-01-24 16:13:08 -05003927 eb = kmem_cache_zalloc(extent_buffer_cache, mask);
Tsutomu Itoh91ca3382011-01-05 02:32:22 +00003928 if (eb == NULL)
3929 return NULL;
Chris Masond1310b22008-01-24 16:13:08 -05003930 eb->start = start;
3931 eb->len = len;
Josef Bacik4f2de97a2012-03-07 16:20:05 -05003932 eb->tree = tree;
Chris Masonbd681512011-07-16 15:23:14 -04003933 rwlock_init(&eb->lock);
3934 atomic_set(&eb->write_locks, 0);
3935 atomic_set(&eb->read_locks, 0);
3936 atomic_set(&eb->blocking_readers, 0);
3937 atomic_set(&eb->blocking_writers, 0);
3938 atomic_set(&eb->spinning_readers, 0);
3939 atomic_set(&eb->spinning_writers, 0);
Arne Jansen5b25f702011-09-13 10:55:48 +02003940 eb->lock_nested = 0;
Chris Masonbd681512011-07-16 15:23:14 -04003941 init_waitqueue_head(&eb->write_lock_wq);
3942 init_waitqueue_head(&eb->read_lock_wq);
Chris Masonb4ce94d2009-02-04 09:25:08 -05003943
Chris Mason39351272009-02-04 09:24:05 -05003944#if LEAK_DEBUG
Chris Mason2d2ae542008-03-26 16:24:23 -04003945 spin_lock_irqsave(&leak_lock, flags);
3946 list_add(&eb->leak_list, &buffers);
3947 spin_unlock_irqrestore(&leak_lock, flags);
Chris Mason4bef0842008-09-08 11:18:08 -04003948#endif
Josef Bacik3083ee22012-03-09 16:01:49 -05003949 spin_lock_init(&eb->refs_lock);
Chris Masond1310b22008-01-24 16:13:08 -05003950 atomic_set(&eb->refs, 1);
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003951 atomic_set(&eb->io_pages, 0);
Chris Mason727011e2010-08-06 13:21:20 -04003952
3953 if (len > MAX_INLINE_EXTENT_BUFFER_SIZE) {
3954 struct page **pages;
3955 int num_pages = (len + PAGE_CACHE_SIZE - 1) >>
3956 PAGE_CACHE_SHIFT;
3957 pages = kzalloc(num_pages, mask);
3958 if (!pages) {
3959 __free_extent_buffer(eb);
3960 return NULL;
3961 }
3962 eb->pages = pages;
3963 } else {
3964 eb->pages = eb->inline_pages;
3965 }
Chris Masond1310b22008-01-24 16:13:08 -05003966
3967 return eb;
3968}
3969
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003970static int extent_buffer_under_io(struct extent_buffer *eb)
Chris Masond1310b22008-01-24 16:13:08 -05003971{
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003972 return (atomic_read(&eb->io_pages) ||
3973 test_bit(EXTENT_BUFFER_WRITEBACK, &eb->bflags) ||
3974 test_bit(EXTENT_BUFFER_DIRTY, &eb->bflags));
Chris Masond1310b22008-01-24 16:13:08 -05003975}
3976
Miao Xie897ca6e2010-10-26 20:57:29 -04003977/*
3978 * Helper for releasing extent buffer page.
3979 */
3980static void btrfs_release_extent_buffer_page(struct extent_buffer *eb,
3981 unsigned long start_idx)
3982{
3983 unsigned long index;
3984 struct page *page;
3985
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003986 BUG_ON(extent_buffer_under_io(eb));
Miao Xie897ca6e2010-10-26 20:57:29 -04003987
3988 index = num_extent_pages(eb->start, eb->len);
3989 if (start_idx >= index)
3990 return;
3991
3992 do {
3993 index--;
3994 page = extent_buffer_page(eb, index);
Josef Bacik4f2de97a2012-03-07 16:20:05 -05003995 if (page) {
3996 spin_lock(&page->mapping->private_lock);
3997 /*
3998 * We do this since we'll remove the pages after we've
3999 * removed the eb from the radix tree, so we could race
4000 * and have this page now attached to the new eb. So
4001 * only clear page_private if it's still connected to
4002 * this eb.
4003 */
4004 if (PagePrivate(page) &&
4005 page->private == (unsigned long)eb) {
Josef Bacik0b32f4b2012-03-13 09:38:00 -04004006 BUG_ON(test_bit(EXTENT_BUFFER_DIRTY, &eb->bflags));
Josef Bacik3083ee22012-03-09 16:01:49 -05004007 BUG_ON(PageDirty(page));
4008 BUG_ON(PageWriteback(page));
Josef Bacik4f2de97a2012-03-07 16:20:05 -05004009 /*
4010 * We need to make sure we haven't be attached
4011 * to a new eb.
4012 */
4013 ClearPagePrivate(page);
4014 set_page_private(page, 0);
4015 /* One for the page private */
4016 page_cache_release(page);
4017 }
4018 spin_unlock(&page->mapping->private_lock);
4019
4020 /* One for when we alloced the page */
Miao Xie897ca6e2010-10-26 20:57:29 -04004021 page_cache_release(page);
Josef Bacik4f2de97a2012-03-07 16:20:05 -05004022 }
Miao Xie897ca6e2010-10-26 20:57:29 -04004023 } while (index != start_idx);
4024}
4025
4026/*
4027 * Helper for releasing the extent buffer.
4028 */
4029static inline void btrfs_release_extent_buffer(struct extent_buffer *eb)
4030{
4031 btrfs_release_extent_buffer_page(eb, 0);
4032 __free_extent_buffer(eb);
4033}
4034
Josef Bacik0b32f4b2012-03-13 09:38:00 -04004035static void check_buffer_tree_ref(struct extent_buffer *eb)
4036{
4037 /* the ref bit is tricky. We have to make sure it is set
4038 * if we have the buffer dirty. Otherwise the
4039 * code to free a buffer can end up dropping a dirty
4040 * page
4041 *
4042 * Once the ref bit is set, it won't go away while the
4043 * buffer is dirty or in writeback, and it also won't
4044 * go away while we have the reference count on the
4045 * eb bumped.
4046 *
4047 * We can't just set the ref bit without bumping the
4048 * ref on the eb because free_extent_buffer might
4049 * see the ref bit and try to clear it. If this happens
4050 * free_extent_buffer might end up dropping our original
4051 * ref by mistake and freeing the page before we are able
4052 * to add one more ref.
4053 *
4054 * So bump the ref count first, then set the bit. If someone
4055 * beat us to it, drop the ref we added.
4056 */
4057 if (!test_bit(EXTENT_BUFFER_TREE_REF, &eb->bflags)) {
4058 atomic_inc(&eb->refs);
4059 if (test_and_set_bit(EXTENT_BUFFER_TREE_REF, &eb->bflags))
4060 atomic_dec(&eb->refs);
4061 }
4062}
4063
Josef Bacik5df42352012-03-15 18:24:42 -04004064static void mark_extent_buffer_accessed(struct extent_buffer *eb)
4065{
4066 unsigned long num_pages, i;
4067
Josef Bacik0b32f4b2012-03-13 09:38:00 -04004068 check_buffer_tree_ref(eb);
4069
Josef Bacik5df42352012-03-15 18:24:42 -04004070 num_pages = num_extent_pages(eb->start, eb->len);
4071 for (i = 0; i < num_pages; i++) {
4072 struct page *p = extent_buffer_page(eb, i);
4073 mark_page_accessed(p);
4074 }
4075}
4076
Chris Masond1310b22008-01-24 16:13:08 -05004077struct extent_buffer *alloc_extent_buffer(struct extent_io_tree *tree,
Chris Mason727011e2010-08-06 13:21:20 -04004078 u64 start, unsigned long len)
Chris Masond1310b22008-01-24 16:13:08 -05004079{
4080 unsigned long num_pages = num_extent_pages(start, len);
4081 unsigned long i;
4082 unsigned long index = start >> PAGE_CACHE_SHIFT;
4083 struct extent_buffer *eb;
Chris Mason6af118c2008-07-22 11:18:07 -04004084 struct extent_buffer *exists = NULL;
Chris Masond1310b22008-01-24 16:13:08 -05004085 struct page *p;
4086 struct address_space *mapping = tree->mapping;
4087 int uptodate = 1;
Miao Xie19fe0a82010-10-26 20:57:29 -04004088 int ret;
Chris Masond1310b22008-01-24 16:13:08 -05004089
Miao Xie19fe0a82010-10-26 20:57:29 -04004090 rcu_read_lock();
4091 eb = radix_tree_lookup(&tree->buffer, start >> PAGE_CACHE_SHIFT);
4092 if (eb && atomic_inc_not_zero(&eb->refs)) {
4093 rcu_read_unlock();
Josef Bacik5df42352012-03-15 18:24:42 -04004094 mark_extent_buffer_accessed(eb);
Chris Mason6af118c2008-07-22 11:18:07 -04004095 return eb;
4096 }
Miao Xie19fe0a82010-10-26 20:57:29 -04004097 rcu_read_unlock();
Chris Mason6af118c2008-07-22 11:18:07 -04004098
David Sterbaba144192011-04-21 01:12:06 +02004099 eb = __alloc_extent_buffer(tree, start, len, GFP_NOFS);
Peter2b114d12008-04-01 11:21:40 -04004100 if (!eb)
Chris Masond1310b22008-01-24 16:13:08 -05004101 return NULL;
4102
Chris Mason727011e2010-08-06 13:21:20 -04004103 for (i = 0; i < num_pages; i++, index++) {
Chris Masona6591712011-07-19 12:04:14 -04004104 p = find_or_create_page(mapping, index, GFP_NOFS);
Chris Masond1310b22008-01-24 16:13:08 -05004105 if (!p) {
4106 WARN_ON(1);
Chris Mason6af118c2008-07-22 11:18:07 -04004107 goto free_eb;
Chris Masond1310b22008-01-24 16:13:08 -05004108 }
Josef Bacik4f2de97a2012-03-07 16:20:05 -05004109
4110 spin_lock(&mapping->private_lock);
4111 if (PagePrivate(p)) {
4112 /*
4113 * We could have already allocated an eb for this page
4114 * and attached one so lets see if we can get a ref on
4115 * the existing eb, and if we can we know it's good and
4116 * we can just return that one, else we know we can just
4117 * overwrite page->private.
4118 */
4119 exists = (struct extent_buffer *)p->private;
4120 if (atomic_inc_not_zero(&exists->refs)) {
4121 spin_unlock(&mapping->private_lock);
4122 unlock_page(p);
Josef Bacik17de39a2012-05-04 15:16:06 -04004123 page_cache_release(p);
Josef Bacik5df42352012-03-15 18:24:42 -04004124 mark_extent_buffer_accessed(exists);
Josef Bacik4f2de97a2012-03-07 16:20:05 -05004125 goto free_eb;
4126 }
4127
Josef Bacik0b32f4b2012-03-13 09:38:00 -04004128 /*
Josef Bacik4f2de97a2012-03-07 16:20:05 -05004129 * Do this so attach doesn't complain and we need to
4130 * drop the ref the old guy had.
4131 */
4132 ClearPagePrivate(p);
Josef Bacik0b32f4b2012-03-13 09:38:00 -04004133 WARN_ON(PageDirty(p));
Josef Bacik4f2de97a2012-03-07 16:20:05 -05004134 page_cache_release(p);
Chris Masond1310b22008-01-24 16:13:08 -05004135 }
Josef Bacik4f2de97a2012-03-07 16:20:05 -05004136 attach_extent_buffer_page(eb, p);
4137 spin_unlock(&mapping->private_lock);
Josef Bacik0b32f4b2012-03-13 09:38:00 -04004138 WARN_ON(PageDirty(p));
Chris Masond1310b22008-01-24 16:13:08 -05004139 mark_page_accessed(p);
Chris Mason727011e2010-08-06 13:21:20 -04004140 eb->pages[i] = p;
Chris Masond1310b22008-01-24 16:13:08 -05004141 if (!PageUptodate(p))
4142 uptodate = 0;
Chris Masoneb14ab82011-02-10 12:35:00 -05004143
4144 /*
4145 * see below about how we avoid a nasty race with release page
4146 * and why we unlock later
4147 */
Chris Masond1310b22008-01-24 16:13:08 -05004148 }
4149 if (uptodate)
Chris Masonb4ce94d2009-02-04 09:25:08 -05004150 set_bit(EXTENT_BUFFER_UPTODATE, &eb->bflags);
Josef Bacik115391d2012-03-09 09:51:43 -05004151again:
Miao Xie19fe0a82010-10-26 20:57:29 -04004152 ret = radix_tree_preload(GFP_NOFS & ~__GFP_HIGHMEM);
4153 if (ret)
4154 goto free_eb;
4155
Chris Mason6af118c2008-07-22 11:18:07 -04004156 spin_lock(&tree->buffer_lock);
Miao Xie19fe0a82010-10-26 20:57:29 -04004157 ret = radix_tree_insert(&tree->buffer, start >> PAGE_CACHE_SHIFT, eb);
4158 if (ret == -EEXIST) {
4159 exists = radix_tree_lookup(&tree->buffer,
4160 start >> PAGE_CACHE_SHIFT);
Josef Bacik115391d2012-03-09 09:51:43 -05004161 if (!atomic_inc_not_zero(&exists->refs)) {
4162 spin_unlock(&tree->buffer_lock);
4163 radix_tree_preload_end();
Josef Bacik115391d2012-03-09 09:51:43 -05004164 exists = NULL;
4165 goto again;
4166 }
Chris Mason6af118c2008-07-22 11:18:07 -04004167 spin_unlock(&tree->buffer_lock);
Miao Xie19fe0a82010-10-26 20:57:29 -04004168 radix_tree_preload_end();
Josef Bacik5df42352012-03-15 18:24:42 -04004169 mark_extent_buffer_accessed(exists);
Chris Mason6af118c2008-07-22 11:18:07 -04004170 goto free_eb;
4171 }
Chris Mason6af118c2008-07-22 11:18:07 -04004172 /* add one reference for the tree */
Josef Bacik3083ee22012-03-09 16:01:49 -05004173 spin_lock(&eb->refs_lock);
Josef Bacik0b32f4b2012-03-13 09:38:00 -04004174 check_buffer_tree_ref(eb);
Josef Bacik3083ee22012-03-09 16:01:49 -05004175 spin_unlock(&eb->refs_lock);
Yan, Zhengf044ba72010-02-04 08:46:56 +00004176 spin_unlock(&tree->buffer_lock);
Miao Xie19fe0a82010-10-26 20:57:29 -04004177 radix_tree_preload_end();
Chris Masoneb14ab82011-02-10 12:35:00 -05004178
4179 /*
4180 * there is a race where release page may have
4181 * tried to find this extent buffer in the radix
4182 * but failed. It will tell the VM it is safe to
4183 * reclaim the, and it will clear the page private bit.
4184 * We must make sure to set the page private bit properly
4185 * after the extent buffer is in the radix tree so
4186 * it doesn't get lost
4187 */
Chris Mason727011e2010-08-06 13:21:20 -04004188 SetPageChecked(eb->pages[0]);
4189 for (i = 1; i < num_pages; i++) {
4190 p = extent_buffer_page(eb, i);
Chris Mason727011e2010-08-06 13:21:20 -04004191 ClearPageChecked(p);
4192 unlock_page(p);
4193 }
4194 unlock_page(eb->pages[0]);
Chris Masond1310b22008-01-24 16:13:08 -05004195 return eb;
4196
Chris Mason6af118c2008-07-22 11:18:07 -04004197free_eb:
Chris Mason727011e2010-08-06 13:21:20 -04004198 for (i = 0; i < num_pages; i++) {
4199 if (eb->pages[i])
4200 unlock_page(eb->pages[i]);
4201 }
Chris Masoneb14ab82011-02-10 12:35:00 -05004202
Josef Bacik17de39a2012-05-04 15:16:06 -04004203 WARN_ON(!atomic_dec_and_test(&eb->refs));
Miao Xie897ca6e2010-10-26 20:57:29 -04004204 btrfs_release_extent_buffer(eb);
Chris Mason6af118c2008-07-22 11:18:07 -04004205 return exists;
Chris Masond1310b22008-01-24 16:13:08 -05004206}
Chris Masond1310b22008-01-24 16:13:08 -05004207
4208struct extent_buffer *find_extent_buffer(struct extent_io_tree *tree,
David Sterbaf09d1f62011-04-21 01:08:01 +02004209 u64 start, unsigned long len)
Chris Masond1310b22008-01-24 16:13:08 -05004210{
Chris Masond1310b22008-01-24 16:13:08 -05004211 struct extent_buffer *eb;
Chris Masond1310b22008-01-24 16:13:08 -05004212
Miao Xie19fe0a82010-10-26 20:57:29 -04004213 rcu_read_lock();
4214 eb = radix_tree_lookup(&tree->buffer, start >> PAGE_CACHE_SHIFT);
4215 if (eb && atomic_inc_not_zero(&eb->refs)) {
4216 rcu_read_unlock();
Josef Bacik5df42352012-03-15 18:24:42 -04004217 mark_extent_buffer_accessed(eb);
Miao Xie19fe0a82010-10-26 20:57:29 -04004218 return eb;
4219 }
4220 rcu_read_unlock();
Josef Bacik0f9dd462008-09-23 13:14:11 -04004221
Miao Xie19fe0a82010-10-26 20:57:29 -04004222 return NULL;
Chris Masond1310b22008-01-24 16:13:08 -05004223}
Chris Masond1310b22008-01-24 16:13:08 -05004224
Josef Bacik3083ee22012-03-09 16:01:49 -05004225static inline void btrfs_release_extent_buffer_rcu(struct rcu_head *head)
4226{
4227 struct extent_buffer *eb =
4228 container_of(head, struct extent_buffer, rcu_head);
4229
4230 __free_extent_buffer(eb);
4231}
4232
Josef Bacik3083ee22012-03-09 16:01:49 -05004233/* Expects to have eb->eb_lock already held */
4234static void release_extent_buffer(struct extent_buffer *eb, gfp_t mask)
4235{
4236 WARN_ON(atomic_read(&eb->refs) == 0);
4237 if (atomic_dec_and_test(&eb->refs)) {
4238 struct extent_io_tree *tree = eb->tree;
Josef Bacik3083ee22012-03-09 16:01:49 -05004239
4240 spin_unlock(&eb->refs_lock);
4241
Josef Bacik3083ee22012-03-09 16:01:49 -05004242 spin_lock(&tree->buffer_lock);
4243 radix_tree_delete(&tree->buffer,
4244 eb->start >> PAGE_CACHE_SHIFT);
4245 spin_unlock(&tree->buffer_lock);
4246
4247 /* Should be safe to release our pages at this point */
4248 btrfs_release_extent_buffer_page(eb, 0);
4249
4250 call_rcu(&eb->rcu_head, btrfs_release_extent_buffer_rcu);
4251 return;
4252 }
4253 spin_unlock(&eb->refs_lock);
4254}
4255
Chris Masond1310b22008-01-24 16:13:08 -05004256void free_extent_buffer(struct extent_buffer *eb)
4257{
Chris Masond1310b22008-01-24 16:13:08 -05004258 if (!eb)
4259 return;
4260
Josef Bacik3083ee22012-03-09 16:01:49 -05004261 spin_lock(&eb->refs_lock);
4262 if (atomic_read(&eb->refs) == 2 &&
4263 test_bit(EXTENT_BUFFER_STALE, &eb->bflags) &&
Josef Bacik0b32f4b2012-03-13 09:38:00 -04004264 !extent_buffer_under_io(eb) &&
Josef Bacik3083ee22012-03-09 16:01:49 -05004265 test_and_clear_bit(EXTENT_BUFFER_TREE_REF, &eb->bflags))
4266 atomic_dec(&eb->refs);
Chris Masond1310b22008-01-24 16:13:08 -05004267
Josef Bacik3083ee22012-03-09 16:01:49 -05004268 /*
4269 * I know this is terrible, but it's temporary until we stop tracking
4270 * the uptodate bits and such for the extent buffers.
4271 */
4272 release_extent_buffer(eb, GFP_ATOMIC);
Chris Masond1310b22008-01-24 16:13:08 -05004273}
Chris Masond1310b22008-01-24 16:13:08 -05004274
Josef Bacik3083ee22012-03-09 16:01:49 -05004275void free_extent_buffer_stale(struct extent_buffer *eb)
4276{
4277 if (!eb)
Chris Masond1310b22008-01-24 16:13:08 -05004278 return;
4279
Josef Bacik3083ee22012-03-09 16:01:49 -05004280 spin_lock(&eb->refs_lock);
4281 set_bit(EXTENT_BUFFER_STALE, &eb->bflags);
4282
Josef Bacik0b32f4b2012-03-13 09:38:00 -04004283 if (atomic_read(&eb->refs) == 2 && !extent_buffer_under_io(eb) &&
Josef Bacik3083ee22012-03-09 16:01:49 -05004284 test_and_clear_bit(EXTENT_BUFFER_TREE_REF, &eb->bflags))
4285 atomic_dec(&eb->refs);
4286 release_extent_buffer(eb, GFP_NOFS);
Chris Masond1310b22008-01-24 16:13:08 -05004287}
4288
Chris Mason1d4284b2012-03-28 20:31:37 -04004289void clear_extent_buffer_dirty(struct extent_buffer *eb)
Chris Masond1310b22008-01-24 16:13:08 -05004290{
Chris Masond1310b22008-01-24 16:13:08 -05004291 unsigned long i;
4292 unsigned long num_pages;
4293 struct page *page;
4294
Chris Masond1310b22008-01-24 16:13:08 -05004295 num_pages = num_extent_pages(eb->start, eb->len);
4296
4297 for (i = 0; i < num_pages; i++) {
4298 page = extent_buffer_page(eb, i);
Chris Masonb9473432009-03-13 11:00:37 -04004299 if (!PageDirty(page))
Chris Masond2c3f4f2008-11-19 12:44:22 -05004300 continue;
4301
Chris Masona61e6f22008-07-22 11:18:08 -04004302 lock_page(page);
Chris Masoneb14ab82011-02-10 12:35:00 -05004303 WARN_ON(!PagePrivate(page));
4304
Chris Masond1310b22008-01-24 16:13:08 -05004305 clear_page_dirty_for_io(page);
Sven Wegener0ee0fda2008-07-30 16:54:26 -04004306 spin_lock_irq(&page->mapping->tree_lock);
Chris Masond1310b22008-01-24 16:13:08 -05004307 if (!PageDirty(page)) {
4308 radix_tree_tag_clear(&page->mapping->page_tree,
4309 page_index(page),
4310 PAGECACHE_TAG_DIRTY);
4311 }
Sven Wegener0ee0fda2008-07-30 16:54:26 -04004312 spin_unlock_irq(&page->mapping->tree_lock);
Chris Masonbf0da8c2011-11-04 12:29:37 -04004313 ClearPageError(page);
Chris Masona61e6f22008-07-22 11:18:08 -04004314 unlock_page(page);
Chris Masond1310b22008-01-24 16:13:08 -05004315 }
Josef Bacik0b32f4b2012-03-13 09:38:00 -04004316 WARN_ON(atomic_read(&eb->refs) == 0);
Chris Masond1310b22008-01-24 16:13:08 -05004317}
Chris Masond1310b22008-01-24 16:13:08 -05004318
Josef Bacik0b32f4b2012-03-13 09:38:00 -04004319int set_extent_buffer_dirty(struct extent_buffer *eb)
Chris Masond1310b22008-01-24 16:13:08 -05004320{
4321 unsigned long i;
4322 unsigned long num_pages;
Chris Masonb9473432009-03-13 11:00:37 -04004323 int was_dirty = 0;
Chris Masond1310b22008-01-24 16:13:08 -05004324
Josef Bacik0b32f4b2012-03-13 09:38:00 -04004325 check_buffer_tree_ref(eb);
4326
Chris Masonb9473432009-03-13 11:00:37 -04004327 was_dirty = test_and_set_bit(EXTENT_BUFFER_DIRTY, &eb->bflags);
Josef Bacik0b32f4b2012-03-13 09:38:00 -04004328
Chris Masond1310b22008-01-24 16:13:08 -05004329 num_pages = num_extent_pages(eb->start, eb->len);
Josef Bacik3083ee22012-03-09 16:01:49 -05004330 WARN_ON(atomic_read(&eb->refs) == 0);
Josef Bacik0b32f4b2012-03-13 09:38:00 -04004331 WARN_ON(!test_bit(EXTENT_BUFFER_TREE_REF, &eb->bflags));
4332
Chris Masonb9473432009-03-13 11:00:37 -04004333 for (i = 0; i < num_pages; i++)
Josef Bacik0b32f4b2012-03-13 09:38:00 -04004334 set_page_dirty(extent_buffer_page(eb, i));
Chris Masonb9473432009-03-13 11:00:37 -04004335 return was_dirty;
Chris Masond1310b22008-01-24 16:13:08 -05004336}
Chris Masond1310b22008-01-24 16:13:08 -05004337
Josef Bacik0b32f4b2012-03-13 09:38:00 -04004338static int range_straddles_pages(u64 start, u64 len)
Chris Mason19b6caf2011-07-25 06:50:50 -04004339{
4340 if (len < PAGE_CACHE_SIZE)
4341 return 1;
4342 if (start & (PAGE_CACHE_SIZE - 1))
4343 return 1;
4344 if ((start + len) & (PAGE_CACHE_SIZE - 1))
4345 return 1;
4346 return 0;
4347}
4348
Josef Bacik0b32f4b2012-03-13 09:38:00 -04004349int clear_extent_buffer_uptodate(struct extent_buffer *eb)
Chris Mason1259ab72008-05-12 13:39:03 -04004350{
4351 unsigned long i;
4352 struct page *page;
4353 unsigned long num_pages;
4354
Chris Masonb4ce94d2009-02-04 09:25:08 -05004355 clear_bit(EXTENT_BUFFER_UPTODATE, &eb->bflags);
Josef Bacik0b32f4b2012-03-13 09:38:00 -04004356 num_pages = num_extent_pages(eb->start, eb->len);
Chris Mason1259ab72008-05-12 13:39:03 -04004357 for (i = 0; i < num_pages; i++) {
4358 page = extent_buffer_page(eb, i);
Chris Mason33958dc2008-07-30 10:29:12 -04004359 if (page)
4360 ClearPageUptodate(page);
Chris Mason1259ab72008-05-12 13:39:03 -04004361 }
4362 return 0;
4363}
4364
Josef Bacik0b32f4b2012-03-13 09:38:00 -04004365int set_extent_buffer_uptodate(struct extent_buffer *eb)
Chris Masond1310b22008-01-24 16:13:08 -05004366{
4367 unsigned long i;
4368 struct page *page;
4369 unsigned long num_pages;
4370
Josef Bacik0b32f4b2012-03-13 09:38:00 -04004371 set_bit(EXTENT_BUFFER_UPTODATE, &eb->bflags);
Chris Masond1310b22008-01-24 16:13:08 -05004372 num_pages = num_extent_pages(eb->start, eb->len);
Chris Masond1310b22008-01-24 16:13:08 -05004373 for (i = 0; i < num_pages; i++) {
4374 page = extent_buffer_page(eb, i);
Chris Masond1310b22008-01-24 16:13:08 -05004375 SetPageUptodate(page);
4376 }
4377 return 0;
4378}
Chris Masond1310b22008-01-24 16:13:08 -05004379
Chris Masonce9adaa2008-04-09 16:28:12 -04004380int extent_range_uptodate(struct extent_io_tree *tree,
4381 u64 start, u64 end)
4382{
4383 struct page *page;
4384 int ret;
4385 int pg_uptodate = 1;
4386 int uptodate;
4387 unsigned long index;
4388
Josef Bacik0b32f4b2012-03-13 09:38:00 -04004389 if (range_straddles_pages(start, end - start + 1)) {
Chris Mason19b6caf2011-07-25 06:50:50 -04004390 ret = test_range_bit(tree, start, end,
4391 EXTENT_UPTODATE, 1, NULL);
4392 if (ret)
4393 return 1;
4394 }
Chris Masond3977122009-01-05 21:25:51 -05004395 while (start <= end) {
Chris Masonce9adaa2008-04-09 16:28:12 -04004396 index = start >> PAGE_CACHE_SHIFT;
4397 page = find_get_page(tree->mapping, index);
Mitch Harder8bedd512012-01-26 15:01:11 -05004398 if (!page)
4399 return 1;
Chris Masonce9adaa2008-04-09 16:28:12 -04004400 uptodate = PageUptodate(page);
4401 page_cache_release(page);
4402 if (!uptodate) {
4403 pg_uptodate = 0;
4404 break;
4405 }
4406 start += PAGE_CACHE_SIZE;
4407 }
4408 return pg_uptodate;
4409}
4410
Josef Bacik0b32f4b2012-03-13 09:38:00 -04004411int extent_buffer_uptodate(struct extent_buffer *eb)
Chris Masond1310b22008-01-24 16:13:08 -05004412{
Josef Bacik0b32f4b2012-03-13 09:38:00 -04004413 return test_bit(EXTENT_BUFFER_UPTODATE, &eb->bflags);
Chris Masond1310b22008-01-24 16:13:08 -05004414}
Chris Masond1310b22008-01-24 16:13:08 -05004415
4416int read_extent_buffer_pages(struct extent_io_tree *tree,
Arne Jansenbb82ab82011-06-10 14:06:53 +02004417 struct extent_buffer *eb, u64 start, int wait,
Chris Masonf1885912008-04-09 16:28:12 -04004418 get_extent_t *get_extent, int mirror_num)
Chris Masond1310b22008-01-24 16:13:08 -05004419{
4420 unsigned long i;
4421 unsigned long start_i;
4422 struct page *page;
4423 int err;
4424 int ret = 0;
Chris Masonce9adaa2008-04-09 16:28:12 -04004425 int locked_pages = 0;
4426 int all_uptodate = 1;
Chris Masond1310b22008-01-24 16:13:08 -05004427 unsigned long num_pages;
Chris Mason727011e2010-08-06 13:21:20 -04004428 unsigned long num_reads = 0;
Chris Masona86c12c2008-02-07 10:50:54 -05004429 struct bio *bio = NULL;
Chris Masonc8b97812008-10-29 14:49:59 -04004430 unsigned long bio_flags = 0;
Chris Masona86c12c2008-02-07 10:50:54 -05004431
Chris Masonb4ce94d2009-02-04 09:25:08 -05004432 if (test_bit(EXTENT_BUFFER_UPTODATE, &eb->bflags))
Chris Masond1310b22008-01-24 16:13:08 -05004433 return 0;
4434
Chris Masond1310b22008-01-24 16:13:08 -05004435 if (start) {
4436 WARN_ON(start < eb->start);
4437 start_i = (start >> PAGE_CACHE_SHIFT) -
4438 (eb->start >> PAGE_CACHE_SHIFT);
4439 } else {
4440 start_i = 0;
4441 }
4442
4443 num_pages = num_extent_pages(eb->start, eb->len);
4444 for (i = start_i; i < num_pages; i++) {
4445 page = extent_buffer_page(eb, i);
Arne Jansenbb82ab82011-06-10 14:06:53 +02004446 if (wait == WAIT_NONE) {
David Woodhouse2db04962008-08-07 11:19:43 -04004447 if (!trylock_page(page))
Chris Masonce9adaa2008-04-09 16:28:12 -04004448 goto unlock_exit;
Chris Masond1310b22008-01-24 16:13:08 -05004449 } else {
4450 lock_page(page);
4451 }
Chris Masonce9adaa2008-04-09 16:28:12 -04004452 locked_pages++;
Chris Mason727011e2010-08-06 13:21:20 -04004453 if (!PageUptodate(page)) {
4454 num_reads++;
Chris Masonce9adaa2008-04-09 16:28:12 -04004455 all_uptodate = 0;
Chris Mason727011e2010-08-06 13:21:20 -04004456 }
Chris Masonce9adaa2008-04-09 16:28:12 -04004457 }
4458 if (all_uptodate) {
4459 if (start_i == 0)
Chris Masonb4ce94d2009-02-04 09:25:08 -05004460 set_bit(EXTENT_BUFFER_UPTODATE, &eb->bflags);
Chris Masonce9adaa2008-04-09 16:28:12 -04004461 goto unlock_exit;
4462 }
4463
Josef Bacikea466792012-03-26 21:57:36 -04004464 clear_bit(EXTENT_BUFFER_IOERR, &eb->bflags);
Josef Bacik5cf1ab52012-04-16 09:42:26 -04004465 eb->read_mirror = 0;
Josef Bacik0b32f4b2012-03-13 09:38:00 -04004466 atomic_set(&eb->io_pages, num_reads);
Chris Masonce9adaa2008-04-09 16:28:12 -04004467 for (i = start_i; i < num_pages; i++) {
4468 page = extent_buffer_page(eb, i);
Chris Masonce9adaa2008-04-09 16:28:12 -04004469 if (!PageUptodate(page)) {
Chris Masonf1885912008-04-09 16:28:12 -04004470 ClearPageError(page);
Chris Masona86c12c2008-02-07 10:50:54 -05004471 err = __extent_read_full_page(tree, page,
Chris Masonf1885912008-04-09 16:28:12 -04004472 get_extent, &bio,
Chris Masonc8b97812008-10-29 14:49:59 -04004473 mirror_num, &bio_flags);
Chris Masond3977122009-01-05 21:25:51 -05004474 if (err)
Chris Masond1310b22008-01-24 16:13:08 -05004475 ret = err;
Chris Masond1310b22008-01-24 16:13:08 -05004476 } else {
4477 unlock_page(page);
4478 }
4479 }
4480
Jeff Mahoney355808c2011-10-03 23:23:14 -04004481 if (bio) {
4482 err = submit_one_bio(READ, bio, mirror_num, bio_flags);
Jeff Mahoney79787ea2012-03-12 16:03:00 +01004483 if (err)
4484 return err;
Jeff Mahoney355808c2011-10-03 23:23:14 -04004485 }
Chris Masona86c12c2008-02-07 10:50:54 -05004486
Arne Jansenbb82ab82011-06-10 14:06:53 +02004487 if (ret || wait != WAIT_COMPLETE)
Chris Masond1310b22008-01-24 16:13:08 -05004488 return ret;
Chris Masond3977122009-01-05 21:25:51 -05004489
Chris Masond1310b22008-01-24 16:13:08 -05004490 for (i = start_i; i < num_pages; i++) {
4491 page = extent_buffer_page(eb, i);
4492 wait_on_page_locked(page);
Chris Masond3977122009-01-05 21:25:51 -05004493 if (!PageUptodate(page))
Chris Masond1310b22008-01-24 16:13:08 -05004494 ret = -EIO;
Chris Masond1310b22008-01-24 16:13:08 -05004495 }
Chris Masond3977122009-01-05 21:25:51 -05004496
Chris Masond1310b22008-01-24 16:13:08 -05004497 return ret;
Chris Masonce9adaa2008-04-09 16:28:12 -04004498
4499unlock_exit:
4500 i = start_i;
Chris Masond3977122009-01-05 21:25:51 -05004501 while (locked_pages > 0) {
Chris Masonce9adaa2008-04-09 16:28:12 -04004502 page = extent_buffer_page(eb, i);
4503 i++;
4504 unlock_page(page);
4505 locked_pages--;
4506 }
4507 return ret;
Chris Masond1310b22008-01-24 16:13:08 -05004508}
Chris Masond1310b22008-01-24 16:13:08 -05004509
4510void read_extent_buffer(struct extent_buffer *eb, void *dstv,
4511 unsigned long start,
4512 unsigned long len)
4513{
4514 size_t cur;
4515 size_t offset;
4516 struct page *page;
4517 char *kaddr;
4518 char *dst = (char *)dstv;
4519 size_t start_offset = eb->start & ((u64)PAGE_CACHE_SIZE - 1);
4520 unsigned long i = (start_offset + start) >> PAGE_CACHE_SHIFT;
Chris Masond1310b22008-01-24 16:13:08 -05004521
4522 WARN_ON(start > eb->len);
4523 WARN_ON(start + len > eb->start + eb->len);
4524
4525 offset = (start_offset + start) & ((unsigned long)PAGE_CACHE_SIZE - 1);
4526
Chris Masond3977122009-01-05 21:25:51 -05004527 while (len > 0) {
Chris Masond1310b22008-01-24 16:13:08 -05004528 page = extent_buffer_page(eb, i);
Chris Masond1310b22008-01-24 16:13:08 -05004529
4530 cur = min(len, (PAGE_CACHE_SIZE - offset));
Chris Masona6591712011-07-19 12:04:14 -04004531 kaddr = page_address(page);
Chris Masond1310b22008-01-24 16:13:08 -05004532 memcpy(dst, kaddr + offset, cur);
Chris Masond1310b22008-01-24 16:13:08 -05004533
4534 dst += cur;
4535 len -= cur;
4536 offset = 0;
4537 i++;
4538 }
4539}
Chris Masond1310b22008-01-24 16:13:08 -05004540
4541int map_private_extent_buffer(struct extent_buffer *eb, unsigned long start,
Chris Masona6591712011-07-19 12:04:14 -04004542 unsigned long min_len, char **map,
Chris Masond1310b22008-01-24 16:13:08 -05004543 unsigned long *map_start,
Chris Masona6591712011-07-19 12:04:14 -04004544 unsigned long *map_len)
Chris Masond1310b22008-01-24 16:13:08 -05004545{
4546 size_t offset = start & (PAGE_CACHE_SIZE - 1);
4547 char *kaddr;
4548 struct page *p;
4549 size_t start_offset = eb->start & ((u64)PAGE_CACHE_SIZE - 1);
4550 unsigned long i = (start_offset + start) >> PAGE_CACHE_SHIFT;
4551 unsigned long end_i = (start_offset + start + min_len - 1) >>
4552 PAGE_CACHE_SHIFT;
4553
4554 if (i != end_i)
4555 return -EINVAL;
4556
4557 if (i == 0) {
4558 offset = start_offset;
4559 *map_start = 0;
4560 } else {
4561 offset = 0;
4562 *map_start = ((u64)i << PAGE_CACHE_SHIFT) - start_offset;
4563 }
Chris Masond3977122009-01-05 21:25:51 -05004564
Chris Masond1310b22008-01-24 16:13:08 -05004565 if (start + min_len > eb->len) {
Chris Masond3977122009-01-05 21:25:51 -05004566 printk(KERN_ERR "btrfs bad mapping eb start %llu len %lu, "
4567 "wanted %lu %lu\n", (unsigned long long)eb->start,
4568 eb->len, start, min_len);
Chris Masond1310b22008-01-24 16:13:08 -05004569 WARN_ON(1);
Josef Bacik850265332011-03-15 14:52:12 -04004570 return -EINVAL;
Chris Masond1310b22008-01-24 16:13:08 -05004571 }
4572
4573 p = extent_buffer_page(eb, i);
Chris Masona6591712011-07-19 12:04:14 -04004574 kaddr = page_address(p);
Chris Masond1310b22008-01-24 16:13:08 -05004575 *map = kaddr + offset;
4576 *map_len = PAGE_CACHE_SIZE - offset;
4577 return 0;
4578}
Chris Masond1310b22008-01-24 16:13:08 -05004579
Chris Masond1310b22008-01-24 16:13:08 -05004580int memcmp_extent_buffer(struct extent_buffer *eb, const void *ptrv,
4581 unsigned long start,
4582 unsigned long len)
4583{
4584 size_t cur;
4585 size_t offset;
4586 struct page *page;
4587 char *kaddr;
4588 char *ptr = (char *)ptrv;
4589 size_t start_offset = eb->start & ((u64)PAGE_CACHE_SIZE - 1);
4590 unsigned long i = (start_offset + start) >> PAGE_CACHE_SHIFT;
4591 int ret = 0;
4592
4593 WARN_ON(start > eb->len);
4594 WARN_ON(start + len > eb->start + eb->len);
4595
4596 offset = (start_offset + start) & ((unsigned long)PAGE_CACHE_SIZE - 1);
4597
Chris Masond3977122009-01-05 21:25:51 -05004598 while (len > 0) {
Chris Masond1310b22008-01-24 16:13:08 -05004599 page = extent_buffer_page(eb, i);
Chris Masond1310b22008-01-24 16:13:08 -05004600
4601 cur = min(len, (PAGE_CACHE_SIZE - offset));
4602
Chris Masona6591712011-07-19 12:04:14 -04004603 kaddr = page_address(page);
Chris Masond1310b22008-01-24 16:13:08 -05004604 ret = memcmp(ptr, kaddr + offset, cur);
Chris Masond1310b22008-01-24 16:13:08 -05004605 if (ret)
4606 break;
4607
4608 ptr += cur;
4609 len -= cur;
4610 offset = 0;
4611 i++;
4612 }
4613 return ret;
4614}
Chris Masond1310b22008-01-24 16:13:08 -05004615
4616void write_extent_buffer(struct extent_buffer *eb, const void *srcv,
4617 unsigned long start, unsigned long len)
4618{
4619 size_t cur;
4620 size_t offset;
4621 struct page *page;
4622 char *kaddr;
4623 char *src = (char *)srcv;
4624 size_t start_offset = eb->start & ((u64)PAGE_CACHE_SIZE - 1);
4625 unsigned long i = (start_offset + start) >> PAGE_CACHE_SHIFT;
4626
4627 WARN_ON(start > eb->len);
4628 WARN_ON(start + len > eb->start + eb->len);
4629
4630 offset = (start_offset + start) & ((unsigned long)PAGE_CACHE_SIZE - 1);
4631
Chris Masond3977122009-01-05 21:25:51 -05004632 while (len > 0) {
Chris Masond1310b22008-01-24 16:13:08 -05004633 page = extent_buffer_page(eb, i);
4634 WARN_ON(!PageUptodate(page));
4635
4636 cur = min(len, PAGE_CACHE_SIZE - offset);
Chris Masona6591712011-07-19 12:04:14 -04004637 kaddr = page_address(page);
Chris Masond1310b22008-01-24 16:13:08 -05004638 memcpy(kaddr + offset, src, cur);
Chris Masond1310b22008-01-24 16:13:08 -05004639
4640 src += cur;
4641 len -= cur;
4642 offset = 0;
4643 i++;
4644 }
4645}
Chris Masond1310b22008-01-24 16:13:08 -05004646
4647void memset_extent_buffer(struct extent_buffer *eb, char c,
4648 unsigned long start, unsigned long len)
4649{
4650 size_t cur;
4651 size_t offset;
4652 struct page *page;
4653 char *kaddr;
4654 size_t start_offset = eb->start & ((u64)PAGE_CACHE_SIZE - 1);
4655 unsigned long i = (start_offset + start) >> PAGE_CACHE_SHIFT;
4656
4657 WARN_ON(start > eb->len);
4658 WARN_ON(start + len > eb->start + eb->len);
4659
4660 offset = (start_offset + start) & ((unsigned long)PAGE_CACHE_SIZE - 1);
4661
Chris Masond3977122009-01-05 21:25:51 -05004662 while (len > 0) {
Chris Masond1310b22008-01-24 16:13:08 -05004663 page = extent_buffer_page(eb, i);
4664 WARN_ON(!PageUptodate(page));
4665
4666 cur = min(len, PAGE_CACHE_SIZE - offset);
Chris Masona6591712011-07-19 12:04:14 -04004667 kaddr = page_address(page);
Chris Masond1310b22008-01-24 16:13:08 -05004668 memset(kaddr + offset, c, cur);
Chris Masond1310b22008-01-24 16:13:08 -05004669
4670 len -= cur;
4671 offset = 0;
4672 i++;
4673 }
4674}
Chris Masond1310b22008-01-24 16:13:08 -05004675
4676void copy_extent_buffer(struct extent_buffer *dst, struct extent_buffer *src,
4677 unsigned long dst_offset, unsigned long src_offset,
4678 unsigned long len)
4679{
4680 u64 dst_len = dst->len;
4681 size_t cur;
4682 size_t offset;
4683 struct page *page;
4684 char *kaddr;
4685 size_t start_offset = dst->start & ((u64)PAGE_CACHE_SIZE - 1);
4686 unsigned long i = (start_offset + dst_offset) >> PAGE_CACHE_SHIFT;
4687
4688 WARN_ON(src->len != dst_len);
4689
4690 offset = (start_offset + dst_offset) &
4691 ((unsigned long)PAGE_CACHE_SIZE - 1);
4692
Chris Masond3977122009-01-05 21:25:51 -05004693 while (len > 0) {
Chris Masond1310b22008-01-24 16:13:08 -05004694 page = extent_buffer_page(dst, i);
4695 WARN_ON(!PageUptodate(page));
4696
4697 cur = min(len, (unsigned long)(PAGE_CACHE_SIZE - offset));
4698
Chris Masona6591712011-07-19 12:04:14 -04004699 kaddr = page_address(page);
Chris Masond1310b22008-01-24 16:13:08 -05004700 read_extent_buffer(src, kaddr + offset, src_offset, cur);
Chris Masond1310b22008-01-24 16:13:08 -05004701
4702 src_offset += cur;
4703 len -= cur;
4704 offset = 0;
4705 i++;
4706 }
4707}
Chris Masond1310b22008-01-24 16:13:08 -05004708
4709static void move_pages(struct page *dst_page, struct page *src_page,
4710 unsigned long dst_off, unsigned long src_off,
4711 unsigned long len)
4712{
Chris Masona6591712011-07-19 12:04:14 -04004713 char *dst_kaddr = page_address(dst_page);
Chris Masond1310b22008-01-24 16:13:08 -05004714 if (dst_page == src_page) {
4715 memmove(dst_kaddr + dst_off, dst_kaddr + src_off, len);
4716 } else {
Chris Masona6591712011-07-19 12:04:14 -04004717 char *src_kaddr = page_address(src_page);
Chris Masond1310b22008-01-24 16:13:08 -05004718 char *p = dst_kaddr + dst_off + len;
4719 char *s = src_kaddr + src_off + len;
4720
4721 while (len--)
4722 *--p = *--s;
Chris Masond1310b22008-01-24 16:13:08 -05004723 }
Chris Masond1310b22008-01-24 16:13:08 -05004724}
4725
Sergei Trofimovich33872062011-04-11 21:52:52 +00004726static inline bool areas_overlap(unsigned long src, unsigned long dst, unsigned long len)
4727{
4728 unsigned long distance = (src > dst) ? src - dst : dst - src;
4729 return distance < len;
4730}
4731
Chris Masond1310b22008-01-24 16:13:08 -05004732static void copy_pages(struct page *dst_page, struct page *src_page,
4733 unsigned long dst_off, unsigned long src_off,
4734 unsigned long len)
4735{
Chris Masona6591712011-07-19 12:04:14 -04004736 char *dst_kaddr = page_address(dst_page);
Chris Masond1310b22008-01-24 16:13:08 -05004737 char *src_kaddr;
Chris Mason727011e2010-08-06 13:21:20 -04004738 int must_memmove = 0;
Chris Masond1310b22008-01-24 16:13:08 -05004739
Sergei Trofimovich33872062011-04-11 21:52:52 +00004740 if (dst_page != src_page) {
Chris Masona6591712011-07-19 12:04:14 -04004741 src_kaddr = page_address(src_page);
Sergei Trofimovich33872062011-04-11 21:52:52 +00004742 } else {
Chris Masond1310b22008-01-24 16:13:08 -05004743 src_kaddr = dst_kaddr;
Chris Mason727011e2010-08-06 13:21:20 -04004744 if (areas_overlap(src_off, dst_off, len))
4745 must_memmove = 1;
Sergei Trofimovich33872062011-04-11 21:52:52 +00004746 }
Chris Masond1310b22008-01-24 16:13:08 -05004747
Chris Mason727011e2010-08-06 13:21:20 -04004748 if (must_memmove)
4749 memmove(dst_kaddr + dst_off, src_kaddr + src_off, len);
4750 else
4751 memcpy(dst_kaddr + dst_off, src_kaddr + src_off, len);
Chris Masond1310b22008-01-24 16:13:08 -05004752}
4753
4754void memcpy_extent_buffer(struct extent_buffer *dst, unsigned long dst_offset,
4755 unsigned long src_offset, unsigned long len)
4756{
4757 size_t cur;
4758 size_t dst_off_in_page;
4759 size_t src_off_in_page;
4760 size_t start_offset = dst->start & ((u64)PAGE_CACHE_SIZE - 1);
4761 unsigned long dst_i;
4762 unsigned long src_i;
4763
4764 if (src_offset + len > dst->len) {
Chris Masond3977122009-01-05 21:25:51 -05004765 printk(KERN_ERR "btrfs memmove bogus src_offset %lu move "
4766 "len %lu dst len %lu\n", src_offset, len, dst->len);
Chris Masond1310b22008-01-24 16:13:08 -05004767 BUG_ON(1);
4768 }
4769 if (dst_offset + len > dst->len) {
Chris Masond3977122009-01-05 21:25:51 -05004770 printk(KERN_ERR "btrfs memmove bogus dst_offset %lu move "
4771 "len %lu dst len %lu\n", dst_offset, len, dst->len);
Chris Masond1310b22008-01-24 16:13:08 -05004772 BUG_ON(1);
4773 }
4774
Chris Masond3977122009-01-05 21:25:51 -05004775 while (len > 0) {
Chris Masond1310b22008-01-24 16:13:08 -05004776 dst_off_in_page = (start_offset + dst_offset) &
4777 ((unsigned long)PAGE_CACHE_SIZE - 1);
4778 src_off_in_page = (start_offset + src_offset) &
4779 ((unsigned long)PAGE_CACHE_SIZE - 1);
4780
4781 dst_i = (start_offset + dst_offset) >> PAGE_CACHE_SHIFT;
4782 src_i = (start_offset + src_offset) >> PAGE_CACHE_SHIFT;
4783
4784 cur = min(len, (unsigned long)(PAGE_CACHE_SIZE -
4785 src_off_in_page));
4786 cur = min_t(unsigned long, cur,
4787 (unsigned long)(PAGE_CACHE_SIZE - dst_off_in_page));
4788
4789 copy_pages(extent_buffer_page(dst, dst_i),
4790 extent_buffer_page(dst, src_i),
4791 dst_off_in_page, src_off_in_page, cur);
4792
4793 src_offset += cur;
4794 dst_offset += cur;
4795 len -= cur;
4796 }
4797}
Chris Masond1310b22008-01-24 16:13:08 -05004798
4799void memmove_extent_buffer(struct extent_buffer *dst, unsigned long dst_offset,
4800 unsigned long src_offset, unsigned long len)
4801{
4802 size_t cur;
4803 size_t dst_off_in_page;
4804 size_t src_off_in_page;
4805 unsigned long dst_end = dst_offset + len - 1;
4806 unsigned long src_end = src_offset + len - 1;
4807 size_t start_offset = dst->start & ((u64)PAGE_CACHE_SIZE - 1);
4808 unsigned long dst_i;
4809 unsigned long src_i;
4810
4811 if (src_offset + len > dst->len) {
Chris Masond3977122009-01-05 21:25:51 -05004812 printk(KERN_ERR "btrfs memmove bogus src_offset %lu move "
4813 "len %lu len %lu\n", src_offset, len, dst->len);
Chris Masond1310b22008-01-24 16:13:08 -05004814 BUG_ON(1);
4815 }
4816 if (dst_offset + len > dst->len) {
Chris Masond3977122009-01-05 21:25:51 -05004817 printk(KERN_ERR "btrfs memmove bogus dst_offset %lu move "
4818 "len %lu len %lu\n", dst_offset, len, dst->len);
Chris Masond1310b22008-01-24 16:13:08 -05004819 BUG_ON(1);
4820 }
Chris Mason727011e2010-08-06 13:21:20 -04004821 if (dst_offset < src_offset) {
Chris Masond1310b22008-01-24 16:13:08 -05004822 memcpy_extent_buffer(dst, dst_offset, src_offset, len);
4823 return;
4824 }
Chris Masond3977122009-01-05 21:25:51 -05004825 while (len > 0) {
Chris Masond1310b22008-01-24 16:13:08 -05004826 dst_i = (start_offset + dst_end) >> PAGE_CACHE_SHIFT;
4827 src_i = (start_offset + src_end) >> PAGE_CACHE_SHIFT;
4828
4829 dst_off_in_page = (start_offset + dst_end) &
4830 ((unsigned long)PAGE_CACHE_SIZE - 1);
4831 src_off_in_page = (start_offset + src_end) &
4832 ((unsigned long)PAGE_CACHE_SIZE - 1);
4833
4834 cur = min_t(unsigned long, len, src_off_in_page + 1);
4835 cur = min(cur, dst_off_in_page + 1);
4836 move_pages(extent_buffer_page(dst, dst_i),
4837 extent_buffer_page(dst, src_i),
4838 dst_off_in_page - cur + 1,
4839 src_off_in_page - cur + 1, cur);
4840
4841 dst_end -= cur;
4842 src_end -= cur;
4843 len -= cur;
4844 }
4845}
Chris Mason6af118c2008-07-22 11:18:07 -04004846
Josef Bacik3083ee22012-03-09 16:01:49 -05004847int try_release_extent_buffer(struct page *page, gfp_t mask)
Miao Xie19fe0a82010-10-26 20:57:29 -04004848{
Chris Mason6af118c2008-07-22 11:18:07 -04004849 struct extent_buffer *eb;
Miao Xie897ca6e2010-10-26 20:57:29 -04004850
Miao Xie19fe0a82010-10-26 20:57:29 -04004851 /*
Josef Bacik3083ee22012-03-09 16:01:49 -05004852 * We need to make sure noboody is attaching this page to an eb right
4853 * now.
Miao Xie19fe0a82010-10-26 20:57:29 -04004854 */
Josef Bacik3083ee22012-03-09 16:01:49 -05004855 spin_lock(&page->mapping->private_lock);
4856 if (!PagePrivate(page)) {
4857 spin_unlock(&page->mapping->private_lock);
4858 return 1;
Miao Xie19fe0a82010-10-26 20:57:29 -04004859 }
4860
Josef Bacik3083ee22012-03-09 16:01:49 -05004861 eb = (struct extent_buffer *)page->private;
4862 BUG_ON(!eb);
Miao Xie19fe0a82010-10-26 20:57:29 -04004863
Josef Bacik0b32f4b2012-03-13 09:38:00 -04004864 /*
Josef Bacik3083ee22012-03-09 16:01:49 -05004865 * This is a little awful but should be ok, we need to make sure that
4866 * the eb doesn't disappear out from under us while we're looking at
4867 * this page.
4868 */
4869 spin_lock(&eb->refs_lock);
Josef Bacik0b32f4b2012-03-13 09:38:00 -04004870 if (atomic_read(&eb->refs) != 1 || extent_buffer_under_io(eb)) {
Josef Bacik3083ee22012-03-09 16:01:49 -05004871 spin_unlock(&eb->refs_lock);
4872 spin_unlock(&page->mapping->private_lock);
4873 return 0;
4874 }
4875 spin_unlock(&page->mapping->private_lock);
4876
4877 if ((mask & GFP_NOFS) == GFP_NOFS)
4878 mask = GFP_NOFS;
4879
4880 /*
4881 * If tree ref isn't set then we know the ref on this eb is a real ref,
4882 * so just return, this page will likely be freed soon anyway.
4883 */
4884 if (!test_and_clear_bit(EXTENT_BUFFER_TREE_REF, &eb->bflags)) {
4885 spin_unlock(&eb->refs_lock);
4886 return 0;
4887 }
4888 release_extent_buffer(eb, mask);
4889
4890 return 1;
Chris Mason6af118c2008-07-22 11:18:07 -04004891}