blob: 8d904dd7ea9f6175012d8abdcf810a7d8d200ae7 [file] [log] [blame]
Chris Masond1310b22008-01-24 16:13:08 -05001#include <linux/bitops.h>
2#include <linux/slab.h>
3#include <linux/bio.h>
4#include <linux/mm.h>
Chris Masond1310b22008-01-24 16:13:08 -05005#include <linux/pagemap.h>
6#include <linux/page-flags.h>
7#include <linux/module.h>
8#include <linux/spinlock.h>
9#include <linux/blkdev.h>
10#include <linux/swap.h>
Chris Masond1310b22008-01-24 16:13:08 -050011#include <linux/writeback.h>
12#include <linux/pagevec.h>
Linus Torvalds268bb0c2011-05-20 12:50:29 -070013#include <linux/prefetch.h>
Dan Magenheimer90a887c2011-05-26 10:01:56 -060014#include <linux/cleancache.h>
Chris Masond1310b22008-01-24 16:13:08 -050015#include "extent_io.h"
16#include "extent_map.h"
David Woodhouse2db04962008-08-07 11:19:43 -040017#include "compat.h"
David Woodhouse902b22f2008-08-20 08:51:49 -040018#include "ctree.h"
19#include "btrfs_inode.h"
Jan Schmidt4a54c8c2011-07-22 15:41:52 +020020#include "volumes.h"
Stefan Behrens21adbd52011-11-09 13:44:05 +010021#include "check-integrity.h"
Josef Bacik0b32f4b2012-03-13 09:38:00 -040022#include "locking.h"
Chris Masond1310b22008-01-24 16:13:08 -050023
Chris Masond1310b22008-01-24 16:13:08 -050024static struct kmem_cache *extent_state_cache;
25static struct kmem_cache *extent_buffer_cache;
26
27static LIST_HEAD(buffers);
28static LIST_HEAD(states);
Chris Mason4bef0842008-09-08 11:18:08 -040029
Chris Masonb47eda82008-11-10 12:34:40 -050030#define LEAK_DEBUG 0
Chris Mason39351272009-02-04 09:24:05 -050031#if LEAK_DEBUG
Chris Masond3977122009-01-05 21:25:51 -050032static DEFINE_SPINLOCK(leak_lock);
Chris Mason4bef0842008-09-08 11:18:08 -040033#endif
Chris Masond1310b22008-01-24 16:13:08 -050034
Chris Masond1310b22008-01-24 16:13:08 -050035#define BUFFER_LRU_MAX 64
36
37struct tree_entry {
38 u64 start;
39 u64 end;
Chris Masond1310b22008-01-24 16:13:08 -050040 struct rb_node rb_node;
41};
42
43struct extent_page_data {
44 struct bio *bio;
45 struct extent_io_tree *tree;
46 get_extent_t *get_extent;
Chris Mason771ed682008-11-06 22:02:51 -050047
48 /* tells writepage not to lock the state bits for this range
49 * it still does the unlocking
50 */
Chris Masonffbd5172009-04-20 15:50:09 -040051 unsigned int extent_locked:1;
52
53 /* tells the submit_bio code to use a WRITE_SYNC */
54 unsigned int sync_io:1;
Chris Masond1310b22008-01-24 16:13:08 -050055};
56
Josef Bacik0b32f4b2012-03-13 09:38:00 -040057static noinline void flush_write_bio(void *data);
Jeff Mahoneyc2d904e2011-10-03 23:22:32 -040058static inline struct btrfs_fs_info *
59tree_fs_info(struct extent_io_tree *tree)
60{
61 return btrfs_sb(tree->mapping->host->i_sb);
62}
Josef Bacik0b32f4b2012-03-13 09:38:00 -040063
Chris Masond1310b22008-01-24 16:13:08 -050064int __init extent_io_init(void)
65{
Christoph Hellwig9601e3f2009-04-13 15:33:09 +020066 extent_state_cache = kmem_cache_create("extent_state",
67 sizeof(struct extent_state), 0,
68 SLAB_RECLAIM_ACCOUNT | SLAB_MEM_SPREAD, NULL);
Chris Masond1310b22008-01-24 16:13:08 -050069 if (!extent_state_cache)
70 return -ENOMEM;
71
Christoph Hellwig9601e3f2009-04-13 15:33:09 +020072 extent_buffer_cache = kmem_cache_create("extent_buffers",
73 sizeof(struct extent_buffer), 0,
74 SLAB_RECLAIM_ACCOUNT | SLAB_MEM_SPREAD, NULL);
Chris Masond1310b22008-01-24 16:13:08 -050075 if (!extent_buffer_cache)
76 goto free_state_cache;
77 return 0;
78
79free_state_cache:
80 kmem_cache_destroy(extent_state_cache);
81 return -ENOMEM;
82}
83
84void extent_io_exit(void)
85{
86 struct extent_state *state;
Chris Mason2d2ae542008-03-26 16:24:23 -040087 struct extent_buffer *eb;
Chris Masond1310b22008-01-24 16:13:08 -050088
89 while (!list_empty(&states)) {
Chris Mason2d2ae542008-03-26 16:24:23 -040090 state = list_entry(states.next, struct extent_state, leak_list);
Chris Masond3977122009-01-05 21:25:51 -050091 printk(KERN_ERR "btrfs state leak: start %llu end %llu "
92 "state %lu in tree %p refs %d\n",
93 (unsigned long long)state->start,
94 (unsigned long long)state->end,
95 state->state, state->tree, atomic_read(&state->refs));
Chris Mason2d2ae542008-03-26 16:24:23 -040096 list_del(&state->leak_list);
Chris Masond1310b22008-01-24 16:13:08 -050097 kmem_cache_free(extent_state_cache, state);
98
99 }
100
Chris Mason2d2ae542008-03-26 16:24:23 -0400101 while (!list_empty(&buffers)) {
102 eb = list_entry(buffers.next, struct extent_buffer, leak_list);
Chris Masond3977122009-01-05 21:25:51 -0500103 printk(KERN_ERR "btrfs buffer leak start %llu len %lu "
104 "refs %d\n", (unsigned long long)eb->start,
105 eb->len, atomic_read(&eb->refs));
Chris Mason2d2ae542008-03-26 16:24:23 -0400106 list_del(&eb->leak_list);
107 kmem_cache_free(extent_buffer_cache, eb);
108 }
Chris Masond1310b22008-01-24 16:13:08 -0500109 if (extent_state_cache)
110 kmem_cache_destroy(extent_state_cache);
111 if (extent_buffer_cache)
112 kmem_cache_destroy(extent_buffer_cache);
113}
114
115void extent_io_tree_init(struct extent_io_tree *tree,
David Sterbaf993c882011-04-20 23:35:57 +0200116 struct address_space *mapping)
Chris Masond1310b22008-01-24 16:13:08 -0500117{
Eric Paris6bef4d32010-02-23 19:43:04 +0000118 tree->state = RB_ROOT;
Miao Xie19fe0a82010-10-26 20:57:29 -0400119 INIT_RADIX_TREE(&tree->buffer, GFP_ATOMIC);
Chris Masond1310b22008-01-24 16:13:08 -0500120 tree->ops = NULL;
121 tree->dirty_bytes = 0;
Chris Mason70dec802008-01-29 09:59:12 -0500122 spin_lock_init(&tree->lock);
Chris Mason6af118ce2008-07-22 11:18:07 -0400123 spin_lock_init(&tree->buffer_lock);
Chris Masond1310b22008-01-24 16:13:08 -0500124 tree->mapping = mapping;
Chris Masond1310b22008-01-24 16:13:08 -0500125}
Chris Masond1310b22008-01-24 16:13:08 -0500126
Christoph Hellwigb2950862008-12-02 09:54:17 -0500127static struct extent_state *alloc_extent_state(gfp_t mask)
Chris Masond1310b22008-01-24 16:13:08 -0500128{
129 struct extent_state *state;
Chris Mason39351272009-02-04 09:24:05 -0500130#if LEAK_DEBUG
Chris Mason2d2ae542008-03-26 16:24:23 -0400131 unsigned long flags;
Chris Mason4bef0842008-09-08 11:18:08 -0400132#endif
Chris Masond1310b22008-01-24 16:13:08 -0500133
134 state = kmem_cache_alloc(extent_state_cache, mask);
Peter2b114d12008-04-01 11:21:40 -0400135 if (!state)
Chris Masond1310b22008-01-24 16:13:08 -0500136 return state;
137 state->state = 0;
Chris Masond1310b22008-01-24 16:13:08 -0500138 state->private = 0;
Chris Mason70dec802008-01-29 09:59:12 -0500139 state->tree = NULL;
Chris Mason39351272009-02-04 09:24:05 -0500140#if LEAK_DEBUG
Chris Mason2d2ae542008-03-26 16:24:23 -0400141 spin_lock_irqsave(&leak_lock, flags);
142 list_add(&state->leak_list, &states);
143 spin_unlock_irqrestore(&leak_lock, flags);
Chris Mason4bef0842008-09-08 11:18:08 -0400144#endif
Chris Masond1310b22008-01-24 16:13:08 -0500145 atomic_set(&state->refs, 1);
146 init_waitqueue_head(&state->wq);
Jeff Mahoney143bede2012-03-01 14:56:26 +0100147 trace_alloc_extent_state(state, mask, _RET_IP_);
Chris Masond1310b22008-01-24 16:13:08 -0500148 return state;
149}
Chris Masond1310b22008-01-24 16:13:08 -0500150
Chris Mason4845e442010-05-25 20:56:50 -0400151void free_extent_state(struct extent_state *state)
Chris Masond1310b22008-01-24 16:13:08 -0500152{
Chris Masond1310b22008-01-24 16:13:08 -0500153 if (!state)
154 return;
155 if (atomic_dec_and_test(&state->refs)) {
Chris Mason39351272009-02-04 09:24:05 -0500156#if LEAK_DEBUG
Chris Mason2d2ae542008-03-26 16:24:23 -0400157 unsigned long flags;
Chris Mason4bef0842008-09-08 11:18:08 -0400158#endif
Chris Mason70dec802008-01-29 09:59:12 -0500159 WARN_ON(state->tree);
Chris Mason39351272009-02-04 09:24:05 -0500160#if LEAK_DEBUG
Chris Mason2d2ae542008-03-26 16:24:23 -0400161 spin_lock_irqsave(&leak_lock, flags);
162 list_del(&state->leak_list);
163 spin_unlock_irqrestore(&leak_lock, flags);
Chris Mason4bef0842008-09-08 11:18:08 -0400164#endif
Jeff Mahoney143bede2012-03-01 14:56:26 +0100165 trace_free_extent_state(state, _RET_IP_);
Chris Masond1310b22008-01-24 16:13:08 -0500166 kmem_cache_free(extent_state_cache, state);
167 }
168}
Chris Masond1310b22008-01-24 16:13:08 -0500169
170static struct rb_node *tree_insert(struct rb_root *root, u64 offset,
171 struct rb_node *node)
172{
Chris Masond3977122009-01-05 21:25:51 -0500173 struct rb_node **p = &root->rb_node;
174 struct rb_node *parent = NULL;
Chris Masond1310b22008-01-24 16:13:08 -0500175 struct tree_entry *entry;
176
Chris Masond3977122009-01-05 21:25:51 -0500177 while (*p) {
Chris Masond1310b22008-01-24 16:13:08 -0500178 parent = *p;
179 entry = rb_entry(parent, struct tree_entry, rb_node);
180
181 if (offset < entry->start)
182 p = &(*p)->rb_left;
183 else if (offset > entry->end)
184 p = &(*p)->rb_right;
185 else
186 return parent;
187 }
188
189 entry = rb_entry(node, struct tree_entry, rb_node);
Chris Masond1310b22008-01-24 16:13:08 -0500190 rb_link_node(node, parent, p);
191 rb_insert_color(node, root);
192 return NULL;
193}
194
Chris Mason80ea96b2008-02-01 14:51:59 -0500195static struct rb_node *__etree_search(struct extent_io_tree *tree, u64 offset,
Chris Masond1310b22008-01-24 16:13:08 -0500196 struct rb_node **prev_ret,
197 struct rb_node **next_ret)
198{
Chris Mason80ea96b2008-02-01 14:51:59 -0500199 struct rb_root *root = &tree->state;
Chris Masond3977122009-01-05 21:25:51 -0500200 struct rb_node *n = root->rb_node;
Chris Masond1310b22008-01-24 16:13:08 -0500201 struct rb_node *prev = NULL;
202 struct rb_node *orig_prev = NULL;
203 struct tree_entry *entry;
204 struct tree_entry *prev_entry = NULL;
205
Chris Masond3977122009-01-05 21:25:51 -0500206 while (n) {
Chris Masond1310b22008-01-24 16:13:08 -0500207 entry = rb_entry(n, struct tree_entry, rb_node);
208 prev = n;
209 prev_entry = entry;
210
211 if (offset < entry->start)
212 n = n->rb_left;
213 else if (offset > entry->end)
214 n = n->rb_right;
Chris Masond3977122009-01-05 21:25:51 -0500215 else
Chris Masond1310b22008-01-24 16:13:08 -0500216 return n;
217 }
218
219 if (prev_ret) {
220 orig_prev = prev;
Chris Masond3977122009-01-05 21:25:51 -0500221 while (prev && offset > prev_entry->end) {
Chris Masond1310b22008-01-24 16:13:08 -0500222 prev = rb_next(prev);
223 prev_entry = rb_entry(prev, struct tree_entry, rb_node);
224 }
225 *prev_ret = prev;
226 prev = orig_prev;
227 }
228
229 if (next_ret) {
230 prev_entry = rb_entry(prev, struct tree_entry, rb_node);
Chris Masond3977122009-01-05 21:25:51 -0500231 while (prev && offset < prev_entry->start) {
Chris Masond1310b22008-01-24 16:13:08 -0500232 prev = rb_prev(prev);
233 prev_entry = rb_entry(prev, struct tree_entry, rb_node);
234 }
235 *next_ret = prev;
236 }
237 return NULL;
238}
239
Chris Mason80ea96b2008-02-01 14:51:59 -0500240static inline struct rb_node *tree_search(struct extent_io_tree *tree,
241 u64 offset)
Chris Masond1310b22008-01-24 16:13:08 -0500242{
Chris Mason70dec802008-01-29 09:59:12 -0500243 struct rb_node *prev = NULL;
Chris Masond1310b22008-01-24 16:13:08 -0500244 struct rb_node *ret;
Chris Mason70dec802008-01-29 09:59:12 -0500245
Chris Mason80ea96b2008-02-01 14:51:59 -0500246 ret = __etree_search(tree, offset, &prev, NULL);
Chris Masond3977122009-01-05 21:25:51 -0500247 if (!ret)
Chris Masond1310b22008-01-24 16:13:08 -0500248 return prev;
249 return ret;
250}
251
Josef Bacik9ed74f22009-09-11 16:12:44 -0400252static void merge_cb(struct extent_io_tree *tree, struct extent_state *new,
253 struct extent_state *other)
254{
255 if (tree->ops && tree->ops->merge_extent_hook)
256 tree->ops->merge_extent_hook(tree->mapping->host, new,
257 other);
258}
259
Chris Masond1310b22008-01-24 16:13:08 -0500260/*
261 * utility function to look for merge candidates inside a given range.
262 * Any extents with matching state are merged together into a single
263 * extent in the tree. Extents with EXTENT_IO in their state field
264 * are not merged because the end_io handlers need to be able to do
265 * operations on them without sleeping (or doing allocations/splits).
266 *
267 * This should be called with the tree lock held.
268 */
Jeff Mahoney1bf85042011-07-21 16:56:09 +0000269static void merge_state(struct extent_io_tree *tree,
270 struct extent_state *state)
Chris Masond1310b22008-01-24 16:13:08 -0500271{
272 struct extent_state *other;
273 struct rb_node *other_node;
274
Zheng Yan5b21f2e2008-09-26 10:05:38 -0400275 if (state->state & (EXTENT_IOBITS | EXTENT_BOUNDARY))
Jeff Mahoney1bf85042011-07-21 16:56:09 +0000276 return;
Chris Masond1310b22008-01-24 16:13:08 -0500277
278 other_node = rb_prev(&state->rb_node);
279 if (other_node) {
280 other = rb_entry(other_node, struct extent_state, rb_node);
281 if (other->end == state->start - 1 &&
282 other->state == state->state) {
Josef Bacik9ed74f22009-09-11 16:12:44 -0400283 merge_cb(tree, state, other);
Chris Masond1310b22008-01-24 16:13:08 -0500284 state->start = other->start;
Chris Mason70dec802008-01-29 09:59:12 -0500285 other->tree = NULL;
Chris Masond1310b22008-01-24 16:13:08 -0500286 rb_erase(&other->rb_node, &tree->state);
287 free_extent_state(other);
288 }
289 }
290 other_node = rb_next(&state->rb_node);
291 if (other_node) {
292 other = rb_entry(other_node, struct extent_state, rb_node);
293 if (other->start == state->end + 1 &&
294 other->state == state->state) {
Josef Bacik9ed74f22009-09-11 16:12:44 -0400295 merge_cb(tree, state, other);
Josef Bacikdf98b6e2011-06-20 14:53:48 -0400296 state->end = other->end;
297 other->tree = NULL;
298 rb_erase(&other->rb_node, &tree->state);
299 free_extent_state(other);
Chris Masond1310b22008-01-24 16:13:08 -0500300 }
301 }
Chris Masond1310b22008-01-24 16:13:08 -0500302}
303
Jeff Mahoney1bf85042011-07-21 16:56:09 +0000304static void set_state_cb(struct extent_io_tree *tree,
Yan, Zheng0ca1f7c2010-05-16 10:48:47 -0400305 struct extent_state *state, int *bits)
Chris Mason291d6732008-01-29 15:55:23 -0500306{
Jeff Mahoney1bf85042011-07-21 16:56:09 +0000307 if (tree->ops && tree->ops->set_bit_hook)
308 tree->ops->set_bit_hook(tree->mapping->host, state, bits);
Chris Mason291d6732008-01-29 15:55:23 -0500309}
310
311static void clear_state_cb(struct extent_io_tree *tree,
Yan, Zheng0ca1f7c2010-05-16 10:48:47 -0400312 struct extent_state *state, int *bits)
Chris Mason291d6732008-01-29 15:55:23 -0500313{
Josef Bacik9ed74f22009-09-11 16:12:44 -0400314 if (tree->ops && tree->ops->clear_bit_hook)
315 tree->ops->clear_bit_hook(tree->mapping->host, state, bits);
Chris Mason291d6732008-01-29 15:55:23 -0500316}
317
Xiao Guangrong3150b692011-07-14 03:19:08 +0000318static void set_state_bits(struct extent_io_tree *tree,
319 struct extent_state *state, int *bits);
320
Chris Masond1310b22008-01-24 16:13:08 -0500321/*
322 * insert an extent_state struct into the tree. 'bits' are set on the
323 * struct before it is inserted.
324 *
325 * This may return -EEXIST if the extent is already there, in which case the
326 * state struct is freed.
327 *
328 * The tree lock is not taken internally. This is a utility function and
329 * probably isn't what you want to call (see set/clear_extent_bit).
330 */
331static int insert_state(struct extent_io_tree *tree,
332 struct extent_state *state, u64 start, u64 end,
Yan, Zheng0ca1f7c2010-05-16 10:48:47 -0400333 int *bits)
Chris Masond1310b22008-01-24 16:13:08 -0500334{
335 struct rb_node *node;
336
337 if (end < start) {
Chris Masond3977122009-01-05 21:25:51 -0500338 printk(KERN_ERR "btrfs end < start %llu %llu\n",
339 (unsigned long long)end,
340 (unsigned long long)start);
Chris Masond1310b22008-01-24 16:13:08 -0500341 WARN_ON(1);
342 }
Chris Masond1310b22008-01-24 16:13:08 -0500343 state->start = start;
344 state->end = end;
Josef Bacik9ed74f22009-09-11 16:12:44 -0400345
Xiao Guangrong3150b692011-07-14 03:19:08 +0000346 set_state_bits(tree, state, bits);
347
Chris Masond1310b22008-01-24 16:13:08 -0500348 node = tree_insert(&tree->state, end, &state->rb_node);
349 if (node) {
350 struct extent_state *found;
351 found = rb_entry(node, struct extent_state, rb_node);
Chris Masond3977122009-01-05 21:25:51 -0500352 printk(KERN_ERR "btrfs found node %llu %llu on insert of "
353 "%llu %llu\n", (unsigned long long)found->start,
354 (unsigned long long)found->end,
355 (unsigned long long)start, (unsigned long long)end);
Chris Masond1310b22008-01-24 16:13:08 -0500356 return -EEXIST;
357 }
Chris Mason70dec802008-01-29 09:59:12 -0500358 state->tree = tree;
Chris Masond1310b22008-01-24 16:13:08 -0500359 merge_state(tree, state);
360 return 0;
361}
362
Jeff Mahoney1bf85042011-07-21 16:56:09 +0000363static void split_cb(struct extent_io_tree *tree, struct extent_state *orig,
Josef Bacik9ed74f22009-09-11 16:12:44 -0400364 u64 split)
365{
366 if (tree->ops && tree->ops->split_extent_hook)
Jeff Mahoney1bf85042011-07-21 16:56:09 +0000367 tree->ops->split_extent_hook(tree->mapping->host, orig, split);
Josef Bacik9ed74f22009-09-11 16:12:44 -0400368}
369
Chris Masond1310b22008-01-24 16:13:08 -0500370/*
371 * split a given extent state struct in two, inserting the preallocated
372 * struct 'prealloc' as the newly created second half. 'split' indicates an
373 * offset inside 'orig' where it should be split.
374 *
375 * Before calling,
376 * the tree has 'orig' at [orig->start, orig->end]. After calling, there
377 * are two extent state structs in the tree:
378 * prealloc: [orig->start, split - 1]
379 * orig: [ split, orig->end ]
380 *
381 * The tree locks are not taken by this function. They need to be held
382 * by the caller.
383 */
384static int split_state(struct extent_io_tree *tree, struct extent_state *orig,
385 struct extent_state *prealloc, u64 split)
386{
387 struct rb_node *node;
Josef Bacik9ed74f22009-09-11 16:12:44 -0400388
389 split_cb(tree, orig, split);
390
Chris Masond1310b22008-01-24 16:13:08 -0500391 prealloc->start = orig->start;
392 prealloc->end = split - 1;
393 prealloc->state = orig->state;
394 orig->start = split;
395
396 node = tree_insert(&tree->state, prealloc->end, &prealloc->rb_node);
397 if (node) {
Chris Masond1310b22008-01-24 16:13:08 -0500398 free_extent_state(prealloc);
399 return -EEXIST;
400 }
Chris Mason70dec802008-01-29 09:59:12 -0500401 prealloc->tree = tree;
Chris Masond1310b22008-01-24 16:13:08 -0500402 return 0;
403}
404
405/*
406 * utility function to clear some bits in an extent state struct.
407 * it will optionally wake up any one waiting on this state (wake == 1), or
408 * forcibly remove the state from the tree (delete == 1).
409 *
410 * If no bits are set on the state struct after clearing things, the
411 * struct is freed and removed from the tree
412 */
413static int clear_state_bit(struct extent_io_tree *tree,
Yan, Zheng0ca1f7c2010-05-16 10:48:47 -0400414 struct extent_state *state,
415 int *bits, int wake)
Chris Masond1310b22008-01-24 16:13:08 -0500416{
Yan, Zheng0ca1f7c2010-05-16 10:48:47 -0400417 int bits_to_clear = *bits & ~EXTENT_CTLBITS;
Josef Bacik32c00af2009-10-08 13:34:05 -0400418 int ret = state->state & bits_to_clear;
Chris Masond1310b22008-01-24 16:13:08 -0500419
Yan, Zheng0ca1f7c2010-05-16 10:48:47 -0400420 if ((bits_to_clear & EXTENT_DIRTY) && (state->state & EXTENT_DIRTY)) {
Chris Masond1310b22008-01-24 16:13:08 -0500421 u64 range = state->end - state->start + 1;
422 WARN_ON(range > tree->dirty_bytes);
423 tree->dirty_bytes -= range;
424 }
Chris Mason291d6732008-01-29 15:55:23 -0500425 clear_state_cb(tree, state, bits);
Josef Bacik32c00af2009-10-08 13:34:05 -0400426 state->state &= ~bits_to_clear;
Chris Masond1310b22008-01-24 16:13:08 -0500427 if (wake)
428 wake_up(&state->wq);
Yan, Zheng0ca1f7c2010-05-16 10:48:47 -0400429 if (state->state == 0) {
Chris Mason70dec802008-01-29 09:59:12 -0500430 if (state->tree) {
Chris Masond1310b22008-01-24 16:13:08 -0500431 rb_erase(&state->rb_node, &tree->state);
Chris Mason70dec802008-01-29 09:59:12 -0500432 state->tree = NULL;
Chris Masond1310b22008-01-24 16:13:08 -0500433 free_extent_state(state);
434 } else {
435 WARN_ON(1);
436 }
437 } else {
438 merge_state(tree, state);
439 }
440 return ret;
441}
442
Xiao Guangrong82337672011-04-20 06:44:57 +0000443static struct extent_state *
444alloc_extent_state_atomic(struct extent_state *prealloc)
445{
446 if (!prealloc)
447 prealloc = alloc_extent_state(GFP_ATOMIC);
448
449 return prealloc;
450}
451
Jeff Mahoneyc2d904e2011-10-03 23:22:32 -0400452void extent_io_tree_panic(struct extent_io_tree *tree, int err)
453{
454 btrfs_panic(tree_fs_info(tree), err, "Locking error: "
455 "Extent tree was modified by another "
456 "thread while locked.");
457}
458
Chris Masond1310b22008-01-24 16:13:08 -0500459/*
460 * clear some bits on a range in the tree. This may require splitting
461 * or inserting elements in the tree, so the gfp mask is used to
462 * indicate which allocations or sleeping are allowed.
463 *
464 * pass 'wake' == 1 to kick any sleepers, and 'delete' == 1 to remove
465 * the given range from the tree regardless of state (ie for truncate).
466 *
467 * the range [start, end] is inclusive.
468 *
Jeff Mahoney6763af82012-03-01 14:56:29 +0100469 * This takes the tree lock, and returns 0 on success and < 0 on error.
Chris Masond1310b22008-01-24 16:13:08 -0500470 */
471int clear_extent_bit(struct extent_io_tree *tree, u64 start, u64 end,
Chris Mason2c64c532009-09-02 15:04:12 -0400472 int bits, int wake, int delete,
473 struct extent_state **cached_state,
474 gfp_t mask)
Chris Masond1310b22008-01-24 16:13:08 -0500475{
476 struct extent_state *state;
Chris Mason2c64c532009-09-02 15:04:12 -0400477 struct extent_state *cached;
Chris Masond1310b22008-01-24 16:13:08 -0500478 struct extent_state *prealloc = NULL;
Chris Mason2c64c532009-09-02 15:04:12 -0400479 struct rb_node *next_node;
Chris Masond1310b22008-01-24 16:13:08 -0500480 struct rb_node *node;
Yan Zheng5c939df2009-05-27 09:16:03 -0400481 u64 last_end;
Chris Masond1310b22008-01-24 16:13:08 -0500482 int err;
Josef Bacik2ac55d42010-02-03 19:33:23 +0000483 int clear = 0;
Chris Masond1310b22008-01-24 16:13:08 -0500484
Yan, Zheng0ca1f7c2010-05-16 10:48:47 -0400485 if (delete)
486 bits |= ~EXTENT_CTLBITS;
487 bits |= EXTENT_FIRST_DELALLOC;
488
Josef Bacik2ac55d42010-02-03 19:33:23 +0000489 if (bits & (EXTENT_IOBITS | EXTENT_BOUNDARY))
490 clear = 1;
Chris Masond1310b22008-01-24 16:13:08 -0500491again:
492 if (!prealloc && (mask & __GFP_WAIT)) {
493 prealloc = alloc_extent_state(mask);
494 if (!prealloc)
495 return -ENOMEM;
496 }
497
Chris Masoncad321a2008-12-17 14:51:42 -0500498 spin_lock(&tree->lock);
Chris Mason2c64c532009-09-02 15:04:12 -0400499 if (cached_state) {
500 cached = *cached_state;
Josef Bacik2ac55d42010-02-03 19:33:23 +0000501
502 if (clear) {
503 *cached_state = NULL;
504 cached_state = NULL;
505 }
506
Josef Bacikdf98b6e2011-06-20 14:53:48 -0400507 if (cached && cached->tree && cached->start <= start &&
508 cached->end > start) {
Josef Bacik2ac55d42010-02-03 19:33:23 +0000509 if (clear)
510 atomic_dec(&cached->refs);
Chris Mason2c64c532009-09-02 15:04:12 -0400511 state = cached;
Chris Mason42daec22009-09-23 19:51:09 -0400512 goto hit_next;
Chris Mason2c64c532009-09-02 15:04:12 -0400513 }
Josef Bacik2ac55d42010-02-03 19:33:23 +0000514 if (clear)
515 free_extent_state(cached);
Chris Mason2c64c532009-09-02 15:04:12 -0400516 }
Chris Masond1310b22008-01-24 16:13:08 -0500517 /*
518 * this search will find the extents that end after
519 * our range starts
520 */
Chris Mason80ea96b2008-02-01 14:51:59 -0500521 node = tree_search(tree, start);
Chris Masond1310b22008-01-24 16:13:08 -0500522 if (!node)
523 goto out;
524 state = rb_entry(node, struct extent_state, rb_node);
Chris Mason2c64c532009-09-02 15:04:12 -0400525hit_next:
Chris Masond1310b22008-01-24 16:13:08 -0500526 if (state->start > end)
527 goto out;
528 WARN_ON(state->end < start);
Yan Zheng5c939df2009-05-27 09:16:03 -0400529 last_end = state->end;
Chris Masond1310b22008-01-24 16:13:08 -0500530
Liu Bo04493142012-02-16 18:34:37 +0800531 if (state->end < end && !need_resched())
532 next_node = rb_next(&state->rb_node);
533 else
534 next_node = NULL;
535
536 /* the state doesn't have the wanted bits, go ahead */
537 if (!(state->state & bits))
538 goto next;
539
Chris Masond1310b22008-01-24 16:13:08 -0500540 /*
541 * | ---- desired range ---- |
542 * | state | or
543 * | ------------- state -------------- |
544 *
545 * We need to split the extent we found, and may flip
546 * bits on second half.
547 *
548 * If the extent we found extends past our range, we
549 * just split and search again. It'll get split again
550 * the next time though.
551 *
552 * If the extent we found is inside our range, we clear
553 * the desired bit on it.
554 */
555
556 if (state->start < start) {
Xiao Guangrong82337672011-04-20 06:44:57 +0000557 prealloc = alloc_extent_state_atomic(prealloc);
558 BUG_ON(!prealloc);
Chris Masond1310b22008-01-24 16:13:08 -0500559 err = split_state(tree, state, prealloc, start);
Jeff Mahoneyc2d904e2011-10-03 23:22:32 -0400560 if (err)
561 extent_io_tree_panic(tree, err);
562
Chris Masond1310b22008-01-24 16:13:08 -0500563 prealloc = NULL;
564 if (err)
565 goto out;
566 if (state->end <= end) {
Jeff Mahoney6763af82012-03-01 14:56:29 +0100567 clear_state_bit(tree, state, &bits, wake);
Yan Zheng5c939df2009-05-27 09:16:03 -0400568 if (last_end == (u64)-1)
569 goto out;
570 start = last_end + 1;
Chris Masond1310b22008-01-24 16:13:08 -0500571 }
572 goto search_again;
573 }
574 /*
575 * | ---- desired range ---- |
576 * | state |
577 * We need to split the extent, and clear the bit
578 * on the first half
579 */
580 if (state->start <= end && state->end > end) {
Xiao Guangrong82337672011-04-20 06:44:57 +0000581 prealloc = alloc_extent_state_atomic(prealloc);
582 BUG_ON(!prealloc);
Chris Masond1310b22008-01-24 16:13:08 -0500583 err = split_state(tree, state, prealloc, end + 1);
Jeff Mahoneyc2d904e2011-10-03 23:22:32 -0400584 if (err)
585 extent_io_tree_panic(tree, err);
586
Chris Masond1310b22008-01-24 16:13:08 -0500587 if (wake)
588 wake_up(&state->wq);
Chris Mason42daec22009-09-23 19:51:09 -0400589
Jeff Mahoney6763af82012-03-01 14:56:29 +0100590 clear_state_bit(tree, prealloc, &bits, wake);
Josef Bacik9ed74f22009-09-11 16:12:44 -0400591
Chris Masond1310b22008-01-24 16:13:08 -0500592 prealloc = NULL;
593 goto out;
594 }
Chris Mason42daec22009-09-23 19:51:09 -0400595
Jeff Mahoney6763af82012-03-01 14:56:29 +0100596 clear_state_bit(tree, state, &bits, wake);
Liu Bo04493142012-02-16 18:34:37 +0800597next:
Yan Zheng5c939df2009-05-27 09:16:03 -0400598 if (last_end == (u64)-1)
599 goto out;
600 start = last_end + 1;
Chris Mason2c64c532009-09-02 15:04:12 -0400601 if (start <= end && next_node) {
602 state = rb_entry(next_node, struct extent_state,
603 rb_node);
Liu Bo692e5752012-02-16 18:34:36 +0800604 goto hit_next;
Chris Mason2c64c532009-09-02 15:04:12 -0400605 }
Chris Masond1310b22008-01-24 16:13:08 -0500606 goto search_again;
607
608out:
Chris Masoncad321a2008-12-17 14:51:42 -0500609 spin_unlock(&tree->lock);
Chris Masond1310b22008-01-24 16:13:08 -0500610 if (prealloc)
611 free_extent_state(prealloc);
612
Jeff Mahoney6763af82012-03-01 14:56:29 +0100613 return 0;
Chris Masond1310b22008-01-24 16:13:08 -0500614
615search_again:
616 if (start > end)
617 goto out;
Chris Masoncad321a2008-12-17 14:51:42 -0500618 spin_unlock(&tree->lock);
Chris Masond1310b22008-01-24 16:13:08 -0500619 if (mask & __GFP_WAIT)
620 cond_resched();
621 goto again;
622}
Chris Masond1310b22008-01-24 16:13:08 -0500623
Jeff Mahoney143bede2012-03-01 14:56:26 +0100624static void wait_on_state(struct extent_io_tree *tree,
625 struct extent_state *state)
Christoph Hellwig641f5212008-12-02 06:36:10 -0500626 __releases(tree->lock)
627 __acquires(tree->lock)
Chris Masond1310b22008-01-24 16:13:08 -0500628{
629 DEFINE_WAIT(wait);
630 prepare_to_wait(&state->wq, &wait, TASK_UNINTERRUPTIBLE);
Chris Masoncad321a2008-12-17 14:51:42 -0500631 spin_unlock(&tree->lock);
Chris Masond1310b22008-01-24 16:13:08 -0500632 schedule();
Chris Masoncad321a2008-12-17 14:51:42 -0500633 spin_lock(&tree->lock);
Chris Masond1310b22008-01-24 16:13:08 -0500634 finish_wait(&state->wq, &wait);
Chris Masond1310b22008-01-24 16:13:08 -0500635}
636
637/*
638 * waits for one or more bits to clear on a range in the state tree.
639 * The range [start, end] is inclusive.
640 * The tree lock is taken by this function
641 */
Jeff Mahoney143bede2012-03-01 14:56:26 +0100642void wait_extent_bit(struct extent_io_tree *tree, u64 start, u64 end, int bits)
Chris Masond1310b22008-01-24 16:13:08 -0500643{
644 struct extent_state *state;
645 struct rb_node *node;
646
Chris Masoncad321a2008-12-17 14:51:42 -0500647 spin_lock(&tree->lock);
Chris Masond1310b22008-01-24 16:13:08 -0500648again:
649 while (1) {
650 /*
651 * this search will find all the extents that end after
652 * our range starts
653 */
Chris Mason80ea96b2008-02-01 14:51:59 -0500654 node = tree_search(tree, start);
Chris Masond1310b22008-01-24 16:13:08 -0500655 if (!node)
656 break;
657
658 state = rb_entry(node, struct extent_state, rb_node);
659
660 if (state->start > end)
661 goto out;
662
663 if (state->state & bits) {
664 start = state->start;
665 atomic_inc(&state->refs);
666 wait_on_state(tree, state);
667 free_extent_state(state);
668 goto again;
669 }
670 start = state->end + 1;
671
672 if (start > end)
673 break;
674
Xiao Guangrongded91f02011-07-14 03:19:27 +0000675 cond_resched_lock(&tree->lock);
Chris Masond1310b22008-01-24 16:13:08 -0500676 }
677out:
Chris Masoncad321a2008-12-17 14:51:42 -0500678 spin_unlock(&tree->lock);
Chris Masond1310b22008-01-24 16:13:08 -0500679}
Chris Masond1310b22008-01-24 16:13:08 -0500680
Jeff Mahoney1bf85042011-07-21 16:56:09 +0000681static void set_state_bits(struct extent_io_tree *tree,
Chris Masond1310b22008-01-24 16:13:08 -0500682 struct extent_state *state,
Yan, Zheng0ca1f7c2010-05-16 10:48:47 -0400683 int *bits)
Chris Masond1310b22008-01-24 16:13:08 -0500684{
Yan, Zheng0ca1f7c2010-05-16 10:48:47 -0400685 int bits_to_set = *bits & ~EXTENT_CTLBITS;
Josef Bacik9ed74f22009-09-11 16:12:44 -0400686
Jeff Mahoney1bf85042011-07-21 16:56:09 +0000687 set_state_cb(tree, state, bits);
Yan, Zheng0ca1f7c2010-05-16 10:48:47 -0400688 if ((bits_to_set & EXTENT_DIRTY) && !(state->state & EXTENT_DIRTY)) {
Chris Masond1310b22008-01-24 16:13:08 -0500689 u64 range = state->end - state->start + 1;
690 tree->dirty_bytes += range;
691 }
Yan, Zheng0ca1f7c2010-05-16 10:48:47 -0400692 state->state |= bits_to_set;
Chris Masond1310b22008-01-24 16:13:08 -0500693}
694
Chris Mason2c64c532009-09-02 15:04:12 -0400695static void cache_state(struct extent_state *state,
696 struct extent_state **cached_ptr)
697{
698 if (cached_ptr && !(*cached_ptr)) {
699 if (state->state & (EXTENT_IOBITS | EXTENT_BOUNDARY)) {
700 *cached_ptr = state;
701 atomic_inc(&state->refs);
702 }
703 }
704}
705
Arne Jansen507903b2011-04-06 10:02:20 +0000706static void uncache_state(struct extent_state **cached_ptr)
707{
708 if (cached_ptr && (*cached_ptr)) {
709 struct extent_state *state = *cached_ptr;
Chris Mason109b36a2011-04-12 13:57:39 -0400710 *cached_ptr = NULL;
711 free_extent_state(state);
Arne Jansen507903b2011-04-06 10:02:20 +0000712 }
713}
714
Chris Masond1310b22008-01-24 16:13:08 -0500715/*
Chris Mason1edbb732009-09-02 13:24:36 -0400716 * set some bits on a range in the tree. This may require allocations or
717 * sleeping, so the gfp mask is used to indicate what is allowed.
Chris Masond1310b22008-01-24 16:13:08 -0500718 *
Chris Mason1edbb732009-09-02 13:24:36 -0400719 * If any of the exclusive bits are set, this will fail with -EEXIST if some
720 * part of the range already has the desired bits set. The start of the
721 * existing range is returned in failed_start in this case.
Chris Masond1310b22008-01-24 16:13:08 -0500722 *
Chris Mason1edbb732009-09-02 13:24:36 -0400723 * [start, end] is inclusive This takes the tree lock.
Chris Masond1310b22008-01-24 16:13:08 -0500724 */
Chris Mason1edbb732009-09-02 13:24:36 -0400725
Jeff Mahoney3fbe5c02012-03-01 14:57:19 +0100726static int __must_check
727__set_extent_bit(struct extent_io_tree *tree, u64 start, u64 end,
728 int bits, int exclusive_bits, u64 *failed_start,
729 struct extent_state **cached_state, gfp_t mask)
Chris Masond1310b22008-01-24 16:13:08 -0500730{
731 struct extent_state *state;
732 struct extent_state *prealloc = NULL;
733 struct rb_node *node;
Chris Masond1310b22008-01-24 16:13:08 -0500734 int err = 0;
Chris Masond1310b22008-01-24 16:13:08 -0500735 u64 last_start;
736 u64 last_end;
Chris Mason42daec22009-09-23 19:51:09 -0400737
Yan, Zheng0ca1f7c2010-05-16 10:48:47 -0400738 bits |= EXTENT_FIRST_DELALLOC;
Chris Masond1310b22008-01-24 16:13:08 -0500739again:
740 if (!prealloc && (mask & __GFP_WAIT)) {
741 prealloc = alloc_extent_state(mask);
Xiao Guangrong82337672011-04-20 06:44:57 +0000742 BUG_ON(!prealloc);
Chris Masond1310b22008-01-24 16:13:08 -0500743 }
744
Chris Masoncad321a2008-12-17 14:51:42 -0500745 spin_lock(&tree->lock);
Chris Mason9655d292009-09-02 15:22:30 -0400746 if (cached_state && *cached_state) {
747 state = *cached_state;
Josef Bacikdf98b6e2011-06-20 14:53:48 -0400748 if (state->start <= start && state->end > start &&
749 state->tree) {
Chris Mason9655d292009-09-02 15:22:30 -0400750 node = &state->rb_node;
751 goto hit_next;
752 }
753 }
Chris Masond1310b22008-01-24 16:13:08 -0500754 /*
755 * this search will find all the extents that end after
756 * our range starts.
757 */
Chris Mason80ea96b2008-02-01 14:51:59 -0500758 node = tree_search(tree, start);
Chris Masond1310b22008-01-24 16:13:08 -0500759 if (!node) {
Xiao Guangrong82337672011-04-20 06:44:57 +0000760 prealloc = alloc_extent_state_atomic(prealloc);
761 BUG_ON(!prealloc);
Yan, Zheng0ca1f7c2010-05-16 10:48:47 -0400762 err = insert_state(tree, prealloc, start, end, &bits);
Jeff Mahoneyc2d904e2011-10-03 23:22:32 -0400763 if (err)
764 extent_io_tree_panic(tree, err);
765
Chris Masond1310b22008-01-24 16:13:08 -0500766 prealloc = NULL;
Chris Masond1310b22008-01-24 16:13:08 -0500767 goto out;
768 }
Chris Masond1310b22008-01-24 16:13:08 -0500769 state = rb_entry(node, struct extent_state, rb_node);
Chris Mason40431d62009-08-05 12:57:59 -0400770hit_next:
Chris Masond1310b22008-01-24 16:13:08 -0500771 last_start = state->start;
772 last_end = state->end;
773
774 /*
775 * | ---- desired range ---- |
776 * | state |
777 *
778 * Just lock what we found and keep going
779 */
780 if (state->start == start && state->end <= end) {
Chris Mason40431d62009-08-05 12:57:59 -0400781 struct rb_node *next_node;
Chris Mason1edbb732009-09-02 13:24:36 -0400782 if (state->state & exclusive_bits) {
Chris Masond1310b22008-01-24 16:13:08 -0500783 *failed_start = state->start;
784 err = -EEXIST;
785 goto out;
786 }
Chris Mason42daec22009-09-23 19:51:09 -0400787
Jeff Mahoney1bf85042011-07-21 16:56:09 +0000788 set_state_bits(tree, state, &bits);
Josef Bacik9ed74f22009-09-11 16:12:44 -0400789
Chris Mason2c64c532009-09-02 15:04:12 -0400790 cache_state(state, cached_state);
Chris Masond1310b22008-01-24 16:13:08 -0500791 merge_state(tree, state);
Yan Zheng5c939df2009-05-27 09:16:03 -0400792 if (last_end == (u64)-1)
793 goto out;
Chris Mason40431d62009-08-05 12:57:59 -0400794
Yan Zheng5c939df2009-05-27 09:16:03 -0400795 start = last_end + 1;
Josef Bacikdf98b6e2011-06-20 14:53:48 -0400796 next_node = rb_next(&state->rb_node);
Xiao Guangrongc7f895a2011-04-20 06:45:49 +0000797 if (next_node && start < end && prealloc && !need_resched()) {
798 state = rb_entry(next_node, struct extent_state,
799 rb_node);
800 if (state->start == start)
801 goto hit_next;
Chris Mason40431d62009-08-05 12:57:59 -0400802 }
Chris Masond1310b22008-01-24 16:13:08 -0500803 goto search_again;
804 }
805
806 /*
807 * | ---- desired range ---- |
808 * | state |
809 * or
810 * | ------------- state -------------- |
811 *
812 * We need to split the extent we found, and may flip bits on
813 * second half.
814 *
815 * If the extent we found extends past our
816 * range, we just split and search again. It'll get split
817 * again the next time though.
818 *
819 * If the extent we found is inside our range, we set the
820 * desired bit on it.
821 */
822 if (state->start < start) {
Chris Mason1edbb732009-09-02 13:24:36 -0400823 if (state->state & exclusive_bits) {
Chris Masond1310b22008-01-24 16:13:08 -0500824 *failed_start = start;
825 err = -EEXIST;
826 goto out;
827 }
Xiao Guangrong82337672011-04-20 06:44:57 +0000828
829 prealloc = alloc_extent_state_atomic(prealloc);
830 BUG_ON(!prealloc);
Chris Masond1310b22008-01-24 16:13:08 -0500831 err = split_state(tree, state, prealloc, start);
Jeff Mahoneyc2d904e2011-10-03 23:22:32 -0400832 if (err)
833 extent_io_tree_panic(tree, err);
834
Chris Masond1310b22008-01-24 16:13:08 -0500835 prealloc = NULL;
836 if (err)
837 goto out;
838 if (state->end <= end) {
Jeff Mahoney1bf85042011-07-21 16:56:09 +0000839 set_state_bits(tree, state, &bits);
Chris Mason2c64c532009-09-02 15:04:12 -0400840 cache_state(state, cached_state);
Chris Masond1310b22008-01-24 16:13:08 -0500841 merge_state(tree, state);
Yan Zheng5c939df2009-05-27 09:16:03 -0400842 if (last_end == (u64)-1)
843 goto out;
844 start = last_end + 1;
Chris Masond1310b22008-01-24 16:13:08 -0500845 }
846 goto search_again;
847 }
848 /*
849 * | ---- desired range ---- |
850 * | state | or | state |
851 *
852 * There's a hole, we need to insert something in it and
853 * ignore the extent we found.
854 */
855 if (state->start > start) {
856 u64 this_end;
857 if (end < last_start)
858 this_end = end;
859 else
Chris Masond3977122009-01-05 21:25:51 -0500860 this_end = last_start - 1;
Xiao Guangrong82337672011-04-20 06:44:57 +0000861
862 prealloc = alloc_extent_state_atomic(prealloc);
863 BUG_ON(!prealloc);
Xiao Guangrongc7f895a2011-04-20 06:45:49 +0000864
865 /*
866 * Avoid to free 'prealloc' if it can be merged with
867 * the later extent.
868 */
Chris Masond1310b22008-01-24 16:13:08 -0500869 err = insert_state(tree, prealloc, start, this_end,
Yan, Zheng0ca1f7c2010-05-16 10:48:47 -0400870 &bits);
Jeff Mahoneyc2d904e2011-10-03 23:22:32 -0400871 if (err)
872 extent_io_tree_panic(tree, err);
873
Chris Mason2c64c532009-09-02 15:04:12 -0400874 cache_state(prealloc, cached_state);
Chris Masond1310b22008-01-24 16:13:08 -0500875 prealloc = NULL;
Chris Masond1310b22008-01-24 16:13:08 -0500876 start = this_end + 1;
877 goto search_again;
878 }
879 /*
880 * | ---- desired range ---- |
881 * | state |
882 * We need to split the extent, and set the bit
883 * on the first half
884 */
885 if (state->start <= end && state->end > end) {
Chris Mason1edbb732009-09-02 13:24:36 -0400886 if (state->state & exclusive_bits) {
Chris Masond1310b22008-01-24 16:13:08 -0500887 *failed_start = start;
888 err = -EEXIST;
889 goto out;
890 }
Xiao Guangrong82337672011-04-20 06:44:57 +0000891
892 prealloc = alloc_extent_state_atomic(prealloc);
893 BUG_ON(!prealloc);
Chris Masond1310b22008-01-24 16:13:08 -0500894 err = split_state(tree, state, prealloc, end + 1);
Jeff Mahoneyc2d904e2011-10-03 23:22:32 -0400895 if (err)
896 extent_io_tree_panic(tree, err);
Chris Masond1310b22008-01-24 16:13:08 -0500897
Jeff Mahoney1bf85042011-07-21 16:56:09 +0000898 set_state_bits(tree, prealloc, &bits);
Chris Mason2c64c532009-09-02 15:04:12 -0400899 cache_state(prealloc, cached_state);
Chris Masond1310b22008-01-24 16:13:08 -0500900 merge_state(tree, prealloc);
901 prealloc = NULL;
902 goto out;
903 }
904
905 goto search_again;
906
907out:
Chris Masoncad321a2008-12-17 14:51:42 -0500908 spin_unlock(&tree->lock);
Chris Masond1310b22008-01-24 16:13:08 -0500909 if (prealloc)
910 free_extent_state(prealloc);
911
912 return err;
913
914search_again:
915 if (start > end)
916 goto out;
Chris Masoncad321a2008-12-17 14:51:42 -0500917 spin_unlock(&tree->lock);
Chris Masond1310b22008-01-24 16:13:08 -0500918 if (mask & __GFP_WAIT)
919 cond_resched();
920 goto again;
921}
Chris Masond1310b22008-01-24 16:13:08 -0500922
Jeff Mahoney3fbe5c02012-03-01 14:57:19 +0100923int set_extent_bit(struct extent_io_tree *tree, u64 start, u64 end, int bits,
924 u64 *failed_start, struct extent_state **cached_state,
925 gfp_t mask)
926{
927 return __set_extent_bit(tree, start, end, bits, 0, failed_start,
928 cached_state, mask);
929}
930
931
Josef Bacik462d6fa2011-09-26 13:56:12 -0400932/**
933 * convert_extent - convert all bits in a given range from one bit to another
934 * @tree: the io tree to search
935 * @start: the start offset in bytes
936 * @end: the end offset in bytes (inclusive)
937 * @bits: the bits to set in this range
938 * @clear_bits: the bits to clear in this range
939 * @mask: the allocation mask
940 *
941 * This will go through and set bits for the given range. If any states exist
942 * already in this range they are set with the given bit and cleared of the
943 * clear_bits. This is only meant to be used by things that are mergeable, ie
944 * converting from say DELALLOC to DIRTY. This is not meant to be used with
945 * boundary bits like LOCK.
946 */
947int convert_extent_bit(struct extent_io_tree *tree, u64 start, u64 end,
948 int bits, int clear_bits, gfp_t mask)
949{
950 struct extent_state *state;
951 struct extent_state *prealloc = NULL;
952 struct rb_node *node;
953 int err = 0;
954 u64 last_start;
955 u64 last_end;
956
957again:
958 if (!prealloc && (mask & __GFP_WAIT)) {
959 prealloc = alloc_extent_state(mask);
960 if (!prealloc)
961 return -ENOMEM;
962 }
963
964 spin_lock(&tree->lock);
965 /*
966 * this search will find all the extents that end after
967 * our range starts.
968 */
969 node = tree_search(tree, start);
970 if (!node) {
971 prealloc = alloc_extent_state_atomic(prealloc);
Liu Bo1cf4ffd2011-12-07 20:08:40 -0500972 if (!prealloc) {
973 err = -ENOMEM;
974 goto out;
975 }
Josef Bacik462d6fa2011-09-26 13:56:12 -0400976 err = insert_state(tree, prealloc, start, end, &bits);
977 prealloc = NULL;
Jeff Mahoneyc2d904e2011-10-03 23:22:32 -0400978 if (err)
979 extent_io_tree_panic(tree, err);
Josef Bacik462d6fa2011-09-26 13:56:12 -0400980 goto out;
981 }
982 state = rb_entry(node, struct extent_state, rb_node);
983hit_next:
984 last_start = state->start;
985 last_end = state->end;
986
987 /*
988 * | ---- desired range ---- |
989 * | state |
990 *
991 * Just lock what we found and keep going
992 */
993 if (state->start == start && state->end <= end) {
994 struct rb_node *next_node;
995
996 set_state_bits(tree, state, &bits);
997 clear_state_bit(tree, state, &clear_bits, 0);
Josef Bacik462d6fa2011-09-26 13:56:12 -0400998 if (last_end == (u64)-1)
999 goto out;
1000
1001 start = last_end + 1;
1002 next_node = rb_next(&state->rb_node);
1003 if (next_node && start < end && prealloc && !need_resched()) {
1004 state = rb_entry(next_node, struct extent_state,
1005 rb_node);
1006 if (state->start == start)
1007 goto hit_next;
1008 }
1009 goto search_again;
1010 }
1011
1012 /*
1013 * | ---- desired range ---- |
1014 * | state |
1015 * or
1016 * | ------------- state -------------- |
1017 *
1018 * We need to split the extent we found, and may flip bits on
1019 * second half.
1020 *
1021 * If the extent we found extends past our
1022 * range, we just split and search again. It'll get split
1023 * again the next time though.
1024 *
1025 * If the extent we found is inside our range, we set the
1026 * desired bit on it.
1027 */
1028 if (state->start < start) {
1029 prealloc = alloc_extent_state_atomic(prealloc);
Liu Bo1cf4ffd2011-12-07 20:08:40 -05001030 if (!prealloc) {
1031 err = -ENOMEM;
1032 goto out;
1033 }
Josef Bacik462d6fa2011-09-26 13:56:12 -04001034 err = split_state(tree, state, prealloc, start);
Jeff Mahoneyc2d904e2011-10-03 23:22:32 -04001035 if (err)
1036 extent_io_tree_panic(tree, err);
Josef Bacik462d6fa2011-09-26 13:56:12 -04001037 prealloc = NULL;
1038 if (err)
1039 goto out;
1040 if (state->end <= end) {
1041 set_state_bits(tree, state, &bits);
1042 clear_state_bit(tree, state, &clear_bits, 0);
Josef Bacik462d6fa2011-09-26 13:56:12 -04001043 if (last_end == (u64)-1)
1044 goto out;
1045 start = last_end + 1;
1046 }
1047 goto search_again;
1048 }
1049 /*
1050 * | ---- desired range ---- |
1051 * | state | or | state |
1052 *
1053 * There's a hole, we need to insert something in it and
1054 * ignore the extent we found.
1055 */
1056 if (state->start > start) {
1057 u64 this_end;
1058 if (end < last_start)
1059 this_end = end;
1060 else
1061 this_end = last_start - 1;
1062
1063 prealloc = alloc_extent_state_atomic(prealloc);
Liu Bo1cf4ffd2011-12-07 20:08:40 -05001064 if (!prealloc) {
1065 err = -ENOMEM;
1066 goto out;
1067 }
Josef Bacik462d6fa2011-09-26 13:56:12 -04001068
1069 /*
1070 * Avoid to free 'prealloc' if it can be merged with
1071 * the later extent.
1072 */
1073 err = insert_state(tree, prealloc, start, this_end,
1074 &bits);
Jeff Mahoneyc2d904e2011-10-03 23:22:32 -04001075 if (err)
1076 extent_io_tree_panic(tree, err);
Josef Bacik462d6fa2011-09-26 13:56:12 -04001077 prealloc = NULL;
1078 start = this_end + 1;
1079 goto search_again;
1080 }
1081 /*
1082 * | ---- desired range ---- |
1083 * | state |
1084 * We need to split the extent, and set the bit
1085 * on the first half
1086 */
1087 if (state->start <= end && state->end > end) {
1088 prealloc = alloc_extent_state_atomic(prealloc);
Liu Bo1cf4ffd2011-12-07 20:08:40 -05001089 if (!prealloc) {
1090 err = -ENOMEM;
1091 goto out;
1092 }
Josef Bacik462d6fa2011-09-26 13:56:12 -04001093
1094 err = split_state(tree, state, prealloc, end + 1);
Jeff Mahoneyc2d904e2011-10-03 23:22:32 -04001095 if (err)
1096 extent_io_tree_panic(tree, err);
Josef Bacik462d6fa2011-09-26 13:56:12 -04001097
1098 set_state_bits(tree, prealloc, &bits);
1099 clear_state_bit(tree, prealloc, &clear_bits, 0);
Josef Bacik462d6fa2011-09-26 13:56:12 -04001100 prealloc = NULL;
1101 goto out;
1102 }
1103
1104 goto search_again;
1105
1106out:
1107 spin_unlock(&tree->lock);
1108 if (prealloc)
1109 free_extent_state(prealloc);
1110
1111 return err;
1112
1113search_again:
1114 if (start > end)
1115 goto out;
1116 spin_unlock(&tree->lock);
1117 if (mask & __GFP_WAIT)
1118 cond_resched();
1119 goto again;
1120}
1121
Chris Masond1310b22008-01-24 16:13:08 -05001122/* wrappers around set/clear extent bit */
1123int set_extent_dirty(struct extent_io_tree *tree, u64 start, u64 end,
1124 gfp_t mask)
1125{
Jeff Mahoney3fbe5c02012-03-01 14:57:19 +01001126 return set_extent_bit(tree, start, end, EXTENT_DIRTY, NULL,
Chris Mason2c64c532009-09-02 15:04:12 -04001127 NULL, mask);
Chris Masond1310b22008-01-24 16:13:08 -05001128}
Chris Masond1310b22008-01-24 16:13:08 -05001129
1130int set_extent_bits(struct extent_io_tree *tree, u64 start, u64 end,
1131 int bits, gfp_t mask)
1132{
Jeff Mahoney3fbe5c02012-03-01 14:57:19 +01001133 return set_extent_bit(tree, start, end, bits, NULL,
Chris Mason2c64c532009-09-02 15:04:12 -04001134 NULL, mask);
Chris Masond1310b22008-01-24 16:13:08 -05001135}
Chris Masond1310b22008-01-24 16:13:08 -05001136
1137int clear_extent_bits(struct extent_io_tree *tree, u64 start, u64 end,
1138 int bits, gfp_t mask)
1139{
Chris Mason2c64c532009-09-02 15:04:12 -04001140 return clear_extent_bit(tree, start, end, bits, 0, 0, NULL, mask);
Chris Masond1310b22008-01-24 16:13:08 -05001141}
Chris Masond1310b22008-01-24 16:13:08 -05001142
1143int set_extent_delalloc(struct extent_io_tree *tree, u64 start, u64 end,
Josef Bacik2ac55d42010-02-03 19:33:23 +00001144 struct extent_state **cached_state, gfp_t mask)
Chris Masond1310b22008-01-24 16:13:08 -05001145{
1146 return set_extent_bit(tree, start, end,
Liu Bofee187d2011-09-29 15:55:28 +08001147 EXTENT_DELALLOC | EXTENT_UPTODATE,
Jeff Mahoney3fbe5c02012-03-01 14:57:19 +01001148 NULL, cached_state, mask);
Chris Masond1310b22008-01-24 16:13:08 -05001149}
Chris Masond1310b22008-01-24 16:13:08 -05001150
1151int clear_extent_dirty(struct extent_io_tree *tree, u64 start, u64 end,
1152 gfp_t mask)
1153{
1154 return clear_extent_bit(tree, start, end,
Josef Bacik32c00af2009-10-08 13:34:05 -04001155 EXTENT_DIRTY | EXTENT_DELALLOC |
Yan, Zheng0ca1f7c2010-05-16 10:48:47 -04001156 EXTENT_DO_ACCOUNTING, 0, 0, NULL, mask);
Chris Masond1310b22008-01-24 16:13:08 -05001157}
Chris Masond1310b22008-01-24 16:13:08 -05001158
1159int set_extent_new(struct extent_io_tree *tree, u64 start, u64 end,
1160 gfp_t mask)
1161{
Jeff Mahoney3fbe5c02012-03-01 14:57:19 +01001162 return set_extent_bit(tree, start, end, EXTENT_NEW, NULL,
Chris Mason2c64c532009-09-02 15:04:12 -04001163 NULL, mask);
Chris Masond1310b22008-01-24 16:13:08 -05001164}
Chris Masond1310b22008-01-24 16:13:08 -05001165
Chris Masond1310b22008-01-24 16:13:08 -05001166int set_extent_uptodate(struct extent_io_tree *tree, u64 start, u64 end,
Arne Jansen507903b2011-04-06 10:02:20 +00001167 struct extent_state **cached_state, gfp_t mask)
Chris Masond1310b22008-01-24 16:13:08 -05001168{
Arne Jansen507903b2011-04-06 10:02:20 +00001169 return set_extent_bit(tree, start, end, EXTENT_UPTODATE, 0,
Jeff Mahoney3fbe5c02012-03-01 14:57:19 +01001170 cached_state, mask);
Chris Masond1310b22008-01-24 16:13:08 -05001171}
Chris Masond1310b22008-01-24 16:13:08 -05001172
Chris Masond3977122009-01-05 21:25:51 -05001173static int clear_extent_uptodate(struct extent_io_tree *tree, u64 start,
Josef Bacik2ac55d42010-02-03 19:33:23 +00001174 u64 end, struct extent_state **cached_state,
1175 gfp_t mask)
Chris Masond1310b22008-01-24 16:13:08 -05001176{
Chris Mason2c64c532009-09-02 15:04:12 -04001177 return clear_extent_bit(tree, start, end, EXTENT_UPTODATE, 0, 0,
Josef Bacik2ac55d42010-02-03 19:33:23 +00001178 cached_state, mask);
Chris Masond1310b22008-01-24 16:13:08 -05001179}
Chris Masond1310b22008-01-24 16:13:08 -05001180
Chris Masond352ac62008-09-29 15:18:18 -04001181/*
1182 * either insert or lock state struct between start and end use mask to tell
1183 * us if waiting is desired.
1184 */
Chris Mason1edbb732009-09-02 13:24:36 -04001185int lock_extent_bits(struct extent_io_tree *tree, u64 start, u64 end,
Jeff Mahoneyd0082372012-03-01 14:57:19 +01001186 int bits, struct extent_state **cached_state)
Chris Masond1310b22008-01-24 16:13:08 -05001187{
1188 int err;
1189 u64 failed_start;
1190 while (1) {
Jeff Mahoney3fbe5c02012-03-01 14:57:19 +01001191 err = __set_extent_bit(tree, start, end, EXTENT_LOCKED | bits,
1192 EXTENT_LOCKED, &failed_start,
1193 cached_state, GFP_NOFS);
Jeff Mahoneyd0082372012-03-01 14:57:19 +01001194 if (err == -EEXIST) {
Chris Masond1310b22008-01-24 16:13:08 -05001195 wait_extent_bit(tree, failed_start, end, EXTENT_LOCKED);
1196 start = failed_start;
Jeff Mahoneyd0082372012-03-01 14:57:19 +01001197 } else
Chris Masond1310b22008-01-24 16:13:08 -05001198 break;
Chris Masond1310b22008-01-24 16:13:08 -05001199 WARN_ON(start > end);
1200 }
1201 return err;
1202}
Chris Masond1310b22008-01-24 16:13:08 -05001203
Jeff Mahoneyd0082372012-03-01 14:57:19 +01001204int lock_extent(struct extent_io_tree *tree, u64 start, u64 end)
Chris Mason1edbb732009-09-02 13:24:36 -04001205{
Jeff Mahoneyd0082372012-03-01 14:57:19 +01001206 return lock_extent_bits(tree, start, end, 0, NULL);
Chris Mason1edbb732009-09-02 13:24:36 -04001207}
1208
Jeff Mahoneyd0082372012-03-01 14:57:19 +01001209int try_lock_extent(struct extent_io_tree *tree, u64 start, u64 end)
Josef Bacik25179202008-10-29 14:49:05 -04001210{
1211 int err;
1212 u64 failed_start;
1213
Jeff Mahoney3fbe5c02012-03-01 14:57:19 +01001214 err = __set_extent_bit(tree, start, end, EXTENT_LOCKED, EXTENT_LOCKED,
1215 &failed_start, NULL, GFP_NOFS);
Yan Zheng66435582008-10-30 14:19:50 -04001216 if (err == -EEXIST) {
1217 if (failed_start > start)
1218 clear_extent_bit(tree, start, failed_start - 1,
Jeff Mahoneyd0082372012-03-01 14:57:19 +01001219 EXTENT_LOCKED, 1, 0, NULL, GFP_NOFS);
Josef Bacik25179202008-10-29 14:49:05 -04001220 return 0;
Yan Zheng66435582008-10-30 14:19:50 -04001221 }
Josef Bacik25179202008-10-29 14:49:05 -04001222 return 1;
1223}
Josef Bacik25179202008-10-29 14:49:05 -04001224
Chris Mason2c64c532009-09-02 15:04:12 -04001225int unlock_extent_cached(struct extent_io_tree *tree, u64 start, u64 end,
1226 struct extent_state **cached, gfp_t mask)
1227{
1228 return clear_extent_bit(tree, start, end, EXTENT_LOCKED, 1, 0, cached,
1229 mask);
1230}
1231
Jeff Mahoneyd0082372012-03-01 14:57:19 +01001232int unlock_extent(struct extent_io_tree *tree, u64 start, u64 end)
Chris Masond1310b22008-01-24 16:13:08 -05001233{
Chris Mason2c64c532009-09-02 15:04:12 -04001234 return clear_extent_bit(tree, start, end, EXTENT_LOCKED, 1, 0, NULL,
Jeff Mahoneyd0082372012-03-01 14:57:19 +01001235 GFP_NOFS);
Chris Masond1310b22008-01-24 16:13:08 -05001236}
Chris Masond1310b22008-01-24 16:13:08 -05001237
1238/*
Chris Masond1310b22008-01-24 16:13:08 -05001239 * helper function to set both pages and extents in the tree writeback
1240 */
Christoph Hellwigb2950862008-12-02 09:54:17 -05001241static int set_range_writeback(struct extent_io_tree *tree, u64 start, u64 end)
Chris Masond1310b22008-01-24 16:13:08 -05001242{
1243 unsigned long index = start >> PAGE_CACHE_SHIFT;
1244 unsigned long end_index = end >> PAGE_CACHE_SHIFT;
1245 struct page *page;
1246
1247 while (index <= end_index) {
1248 page = find_get_page(tree->mapping, index);
Jeff Mahoney79787ea2012-03-12 16:03:00 +01001249 BUG_ON(!page); /* Pages should be in the extent_io_tree */
Chris Masond1310b22008-01-24 16:13:08 -05001250 set_page_writeback(page);
1251 page_cache_release(page);
1252 index++;
1253 }
Chris Masond1310b22008-01-24 16:13:08 -05001254 return 0;
1255}
Chris Masond1310b22008-01-24 16:13:08 -05001256
Chris Masond352ac62008-09-29 15:18:18 -04001257/* find the first state struct with 'bits' set after 'start', and
1258 * return it. tree->lock must be held. NULL will returned if
1259 * nothing was found after 'start'
1260 */
Chris Masond7fc6402008-02-18 12:12:38 -05001261struct extent_state *find_first_extent_bit_state(struct extent_io_tree *tree,
1262 u64 start, int bits)
1263{
1264 struct rb_node *node;
1265 struct extent_state *state;
1266
1267 /*
1268 * this search will find all the extents that end after
1269 * our range starts.
1270 */
1271 node = tree_search(tree, start);
Chris Masond3977122009-01-05 21:25:51 -05001272 if (!node)
Chris Masond7fc6402008-02-18 12:12:38 -05001273 goto out;
Chris Masond7fc6402008-02-18 12:12:38 -05001274
Chris Masond3977122009-01-05 21:25:51 -05001275 while (1) {
Chris Masond7fc6402008-02-18 12:12:38 -05001276 state = rb_entry(node, struct extent_state, rb_node);
Chris Masond3977122009-01-05 21:25:51 -05001277 if (state->end >= start && (state->state & bits))
Chris Masond7fc6402008-02-18 12:12:38 -05001278 return state;
Chris Masond3977122009-01-05 21:25:51 -05001279
Chris Masond7fc6402008-02-18 12:12:38 -05001280 node = rb_next(node);
1281 if (!node)
1282 break;
1283 }
1284out:
1285 return NULL;
1286}
Chris Masond7fc6402008-02-18 12:12:38 -05001287
Chris Masond352ac62008-09-29 15:18:18 -04001288/*
Xiao Guangrong69261c42011-07-14 03:19:45 +00001289 * find the first offset in the io tree with 'bits' set. zero is
1290 * returned if we find something, and *start_ret and *end_ret are
1291 * set to reflect the state struct that was found.
1292 *
1293 * If nothing was found, 1 is returned, < 0 on error
1294 */
1295int find_first_extent_bit(struct extent_io_tree *tree, u64 start,
1296 u64 *start_ret, u64 *end_ret, int bits)
1297{
1298 struct extent_state *state;
1299 int ret = 1;
1300
1301 spin_lock(&tree->lock);
1302 state = find_first_extent_bit_state(tree, start, bits);
1303 if (state) {
1304 *start_ret = state->start;
1305 *end_ret = state->end;
1306 ret = 0;
1307 }
1308 spin_unlock(&tree->lock);
1309 return ret;
1310}
1311
1312/*
Chris Masond352ac62008-09-29 15:18:18 -04001313 * find a contiguous range of bytes in the file marked as delalloc, not
1314 * more than 'max_bytes'. start and end are used to return the range,
1315 *
1316 * 1 is returned if we find something, 0 if nothing was in the tree
1317 */
Chris Masonc8b97812008-10-29 14:49:59 -04001318static noinline u64 find_delalloc_range(struct extent_io_tree *tree,
Josef Bacikc2a128d2010-02-02 21:19:11 +00001319 u64 *start, u64 *end, u64 max_bytes,
1320 struct extent_state **cached_state)
Chris Masond1310b22008-01-24 16:13:08 -05001321{
1322 struct rb_node *node;
1323 struct extent_state *state;
1324 u64 cur_start = *start;
1325 u64 found = 0;
1326 u64 total_bytes = 0;
1327
Chris Masoncad321a2008-12-17 14:51:42 -05001328 spin_lock(&tree->lock);
Chris Masonc8b97812008-10-29 14:49:59 -04001329
Chris Masond1310b22008-01-24 16:13:08 -05001330 /*
1331 * this search will find all the extents that end after
1332 * our range starts.
1333 */
Chris Mason80ea96b2008-02-01 14:51:59 -05001334 node = tree_search(tree, cur_start);
Peter2b114d12008-04-01 11:21:40 -04001335 if (!node) {
Chris Mason3b951512008-04-17 11:29:12 -04001336 if (!found)
1337 *end = (u64)-1;
Chris Masond1310b22008-01-24 16:13:08 -05001338 goto out;
1339 }
1340
Chris Masond3977122009-01-05 21:25:51 -05001341 while (1) {
Chris Masond1310b22008-01-24 16:13:08 -05001342 state = rb_entry(node, struct extent_state, rb_node);
Zheng Yan5b21f2e2008-09-26 10:05:38 -04001343 if (found && (state->start != cur_start ||
1344 (state->state & EXTENT_BOUNDARY))) {
Chris Masond1310b22008-01-24 16:13:08 -05001345 goto out;
1346 }
1347 if (!(state->state & EXTENT_DELALLOC)) {
1348 if (!found)
1349 *end = state->end;
1350 goto out;
1351 }
Josef Bacikc2a128d2010-02-02 21:19:11 +00001352 if (!found) {
Chris Masond1310b22008-01-24 16:13:08 -05001353 *start = state->start;
Josef Bacikc2a128d2010-02-02 21:19:11 +00001354 *cached_state = state;
1355 atomic_inc(&state->refs);
1356 }
Chris Masond1310b22008-01-24 16:13:08 -05001357 found++;
1358 *end = state->end;
1359 cur_start = state->end + 1;
1360 node = rb_next(node);
1361 if (!node)
1362 break;
1363 total_bytes += state->end - state->start + 1;
1364 if (total_bytes >= max_bytes)
1365 break;
1366 }
1367out:
Chris Masoncad321a2008-12-17 14:51:42 -05001368 spin_unlock(&tree->lock);
Chris Masond1310b22008-01-24 16:13:08 -05001369 return found;
1370}
1371
Jeff Mahoney143bede2012-03-01 14:56:26 +01001372static noinline void __unlock_for_delalloc(struct inode *inode,
1373 struct page *locked_page,
1374 u64 start, u64 end)
Chris Masonc8b97812008-10-29 14:49:59 -04001375{
1376 int ret;
1377 struct page *pages[16];
1378 unsigned long index = start >> PAGE_CACHE_SHIFT;
1379 unsigned long end_index = end >> PAGE_CACHE_SHIFT;
1380 unsigned long nr_pages = end_index - index + 1;
1381 int i;
1382
1383 if (index == locked_page->index && end_index == index)
Jeff Mahoney143bede2012-03-01 14:56:26 +01001384 return;
Chris Masonc8b97812008-10-29 14:49:59 -04001385
Chris Masond3977122009-01-05 21:25:51 -05001386 while (nr_pages > 0) {
Chris Masonc8b97812008-10-29 14:49:59 -04001387 ret = find_get_pages_contig(inode->i_mapping, index,
Chris Mason5b050f02008-11-11 09:34:41 -05001388 min_t(unsigned long, nr_pages,
1389 ARRAY_SIZE(pages)), pages);
Chris Masonc8b97812008-10-29 14:49:59 -04001390 for (i = 0; i < ret; i++) {
1391 if (pages[i] != locked_page)
1392 unlock_page(pages[i]);
1393 page_cache_release(pages[i]);
1394 }
1395 nr_pages -= ret;
1396 index += ret;
1397 cond_resched();
1398 }
Chris Masonc8b97812008-10-29 14:49:59 -04001399}
1400
1401static noinline int lock_delalloc_pages(struct inode *inode,
1402 struct page *locked_page,
1403 u64 delalloc_start,
1404 u64 delalloc_end)
1405{
1406 unsigned long index = delalloc_start >> PAGE_CACHE_SHIFT;
1407 unsigned long start_index = index;
1408 unsigned long end_index = delalloc_end >> PAGE_CACHE_SHIFT;
1409 unsigned long pages_locked = 0;
1410 struct page *pages[16];
1411 unsigned long nrpages;
1412 int ret;
1413 int i;
1414
1415 /* the caller is responsible for locking the start index */
1416 if (index == locked_page->index && index == end_index)
1417 return 0;
1418
1419 /* skip the page at the start index */
1420 nrpages = end_index - index + 1;
Chris Masond3977122009-01-05 21:25:51 -05001421 while (nrpages > 0) {
Chris Masonc8b97812008-10-29 14:49:59 -04001422 ret = find_get_pages_contig(inode->i_mapping, index,
Chris Mason5b050f02008-11-11 09:34:41 -05001423 min_t(unsigned long,
1424 nrpages, ARRAY_SIZE(pages)), pages);
Chris Masonc8b97812008-10-29 14:49:59 -04001425 if (ret == 0) {
1426 ret = -EAGAIN;
1427 goto done;
1428 }
1429 /* now we have an array of pages, lock them all */
1430 for (i = 0; i < ret; i++) {
1431 /*
1432 * the caller is taking responsibility for
1433 * locked_page
1434 */
Chris Mason771ed682008-11-06 22:02:51 -05001435 if (pages[i] != locked_page) {
Chris Masonc8b97812008-10-29 14:49:59 -04001436 lock_page(pages[i]);
Chris Masonf2b1c412008-11-10 07:31:30 -05001437 if (!PageDirty(pages[i]) ||
1438 pages[i]->mapping != inode->i_mapping) {
Chris Mason771ed682008-11-06 22:02:51 -05001439 ret = -EAGAIN;
1440 unlock_page(pages[i]);
1441 page_cache_release(pages[i]);
1442 goto done;
1443 }
1444 }
Chris Masonc8b97812008-10-29 14:49:59 -04001445 page_cache_release(pages[i]);
Chris Mason771ed682008-11-06 22:02:51 -05001446 pages_locked++;
Chris Masonc8b97812008-10-29 14:49:59 -04001447 }
Chris Masonc8b97812008-10-29 14:49:59 -04001448 nrpages -= ret;
1449 index += ret;
1450 cond_resched();
1451 }
1452 ret = 0;
1453done:
1454 if (ret && pages_locked) {
1455 __unlock_for_delalloc(inode, locked_page,
1456 delalloc_start,
1457 ((u64)(start_index + pages_locked - 1)) <<
1458 PAGE_CACHE_SHIFT);
1459 }
1460 return ret;
1461}
1462
1463/*
1464 * find a contiguous range of bytes in the file marked as delalloc, not
1465 * more than 'max_bytes'. start and end are used to return the range,
1466 *
1467 * 1 is returned if we find something, 0 if nothing was in the tree
1468 */
1469static noinline u64 find_lock_delalloc_range(struct inode *inode,
1470 struct extent_io_tree *tree,
1471 struct page *locked_page,
1472 u64 *start, u64 *end,
1473 u64 max_bytes)
1474{
1475 u64 delalloc_start;
1476 u64 delalloc_end;
1477 u64 found;
Chris Mason9655d292009-09-02 15:22:30 -04001478 struct extent_state *cached_state = NULL;
Chris Masonc8b97812008-10-29 14:49:59 -04001479 int ret;
1480 int loops = 0;
1481
1482again:
1483 /* step one, find a bunch of delalloc bytes starting at start */
1484 delalloc_start = *start;
1485 delalloc_end = 0;
1486 found = find_delalloc_range(tree, &delalloc_start, &delalloc_end,
Josef Bacikc2a128d2010-02-02 21:19:11 +00001487 max_bytes, &cached_state);
Chris Mason70b99e62008-10-31 12:46:39 -04001488 if (!found || delalloc_end <= *start) {
Chris Masonc8b97812008-10-29 14:49:59 -04001489 *start = delalloc_start;
1490 *end = delalloc_end;
Josef Bacikc2a128d2010-02-02 21:19:11 +00001491 free_extent_state(cached_state);
Chris Masonc8b97812008-10-29 14:49:59 -04001492 return found;
1493 }
1494
1495 /*
Chris Mason70b99e62008-10-31 12:46:39 -04001496 * start comes from the offset of locked_page. We have to lock
1497 * pages in order, so we can't process delalloc bytes before
1498 * locked_page
1499 */
Chris Masond3977122009-01-05 21:25:51 -05001500 if (delalloc_start < *start)
Chris Mason70b99e62008-10-31 12:46:39 -04001501 delalloc_start = *start;
Chris Mason70b99e62008-10-31 12:46:39 -04001502
1503 /*
Chris Masonc8b97812008-10-29 14:49:59 -04001504 * make sure to limit the number of pages we try to lock down
1505 * if we're looping.
1506 */
Chris Masond3977122009-01-05 21:25:51 -05001507 if (delalloc_end + 1 - delalloc_start > max_bytes && loops)
Chris Mason771ed682008-11-06 22:02:51 -05001508 delalloc_end = delalloc_start + PAGE_CACHE_SIZE - 1;
Chris Masond3977122009-01-05 21:25:51 -05001509
Chris Masonc8b97812008-10-29 14:49:59 -04001510 /* step two, lock all the pages after the page that has start */
1511 ret = lock_delalloc_pages(inode, locked_page,
1512 delalloc_start, delalloc_end);
1513 if (ret == -EAGAIN) {
1514 /* some of the pages are gone, lets avoid looping by
1515 * shortening the size of the delalloc range we're searching
1516 */
Chris Mason9655d292009-09-02 15:22:30 -04001517 free_extent_state(cached_state);
Chris Masonc8b97812008-10-29 14:49:59 -04001518 if (!loops) {
1519 unsigned long offset = (*start) & (PAGE_CACHE_SIZE - 1);
1520 max_bytes = PAGE_CACHE_SIZE - offset;
1521 loops = 1;
1522 goto again;
1523 } else {
1524 found = 0;
1525 goto out_failed;
1526 }
1527 }
Jeff Mahoney79787ea2012-03-12 16:03:00 +01001528 BUG_ON(ret); /* Only valid values are 0 and -EAGAIN */
Chris Masonc8b97812008-10-29 14:49:59 -04001529
1530 /* step three, lock the state bits for the whole range */
Jeff Mahoneyd0082372012-03-01 14:57:19 +01001531 lock_extent_bits(tree, delalloc_start, delalloc_end, 0, &cached_state);
Chris Masonc8b97812008-10-29 14:49:59 -04001532
1533 /* then test to make sure it is all still delalloc */
1534 ret = test_range_bit(tree, delalloc_start, delalloc_end,
Chris Mason9655d292009-09-02 15:22:30 -04001535 EXTENT_DELALLOC, 1, cached_state);
Chris Masonc8b97812008-10-29 14:49:59 -04001536 if (!ret) {
Chris Mason9655d292009-09-02 15:22:30 -04001537 unlock_extent_cached(tree, delalloc_start, delalloc_end,
1538 &cached_state, GFP_NOFS);
Chris Masonc8b97812008-10-29 14:49:59 -04001539 __unlock_for_delalloc(inode, locked_page,
1540 delalloc_start, delalloc_end);
1541 cond_resched();
1542 goto again;
1543 }
Chris Mason9655d292009-09-02 15:22:30 -04001544 free_extent_state(cached_state);
Chris Masonc8b97812008-10-29 14:49:59 -04001545 *start = delalloc_start;
1546 *end = delalloc_end;
1547out_failed:
1548 return found;
1549}
1550
1551int extent_clear_unlock_delalloc(struct inode *inode,
1552 struct extent_io_tree *tree,
1553 u64 start, u64 end, struct page *locked_page,
Chris Masona791e352009-10-08 11:27:10 -04001554 unsigned long op)
Chris Masonc8b97812008-10-29 14:49:59 -04001555{
1556 int ret;
1557 struct page *pages[16];
1558 unsigned long index = start >> PAGE_CACHE_SHIFT;
1559 unsigned long end_index = end >> PAGE_CACHE_SHIFT;
1560 unsigned long nr_pages = end_index - index + 1;
1561 int i;
Chris Mason771ed682008-11-06 22:02:51 -05001562 int clear_bits = 0;
Chris Masonc8b97812008-10-29 14:49:59 -04001563
Chris Masona791e352009-10-08 11:27:10 -04001564 if (op & EXTENT_CLEAR_UNLOCK)
Chris Mason771ed682008-11-06 22:02:51 -05001565 clear_bits |= EXTENT_LOCKED;
Chris Masona791e352009-10-08 11:27:10 -04001566 if (op & EXTENT_CLEAR_DIRTY)
Chris Masonc8b97812008-10-29 14:49:59 -04001567 clear_bits |= EXTENT_DIRTY;
1568
Chris Masona791e352009-10-08 11:27:10 -04001569 if (op & EXTENT_CLEAR_DELALLOC)
Chris Mason771ed682008-11-06 22:02:51 -05001570 clear_bits |= EXTENT_DELALLOC;
1571
Chris Mason2c64c532009-09-02 15:04:12 -04001572 clear_extent_bit(tree, start, end, clear_bits, 1, 0, NULL, GFP_NOFS);
Josef Bacik32c00af2009-10-08 13:34:05 -04001573 if (!(op & (EXTENT_CLEAR_UNLOCK_PAGE | EXTENT_CLEAR_DIRTY |
1574 EXTENT_SET_WRITEBACK | EXTENT_END_WRITEBACK |
1575 EXTENT_SET_PRIVATE2)))
Chris Mason771ed682008-11-06 22:02:51 -05001576 return 0;
Chris Masonc8b97812008-10-29 14:49:59 -04001577
Chris Masond3977122009-01-05 21:25:51 -05001578 while (nr_pages > 0) {
Chris Masonc8b97812008-10-29 14:49:59 -04001579 ret = find_get_pages_contig(inode->i_mapping, index,
Chris Mason5b050f02008-11-11 09:34:41 -05001580 min_t(unsigned long,
1581 nr_pages, ARRAY_SIZE(pages)), pages);
Chris Masonc8b97812008-10-29 14:49:59 -04001582 for (i = 0; i < ret; i++) {
Chris Mason8b62b722009-09-02 16:53:46 -04001583
Chris Masona791e352009-10-08 11:27:10 -04001584 if (op & EXTENT_SET_PRIVATE2)
Chris Mason8b62b722009-09-02 16:53:46 -04001585 SetPagePrivate2(pages[i]);
1586
Chris Masonc8b97812008-10-29 14:49:59 -04001587 if (pages[i] == locked_page) {
1588 page_cache_release(pages[i]);
1589 continue;
1590 }
Chris Masona791e352009-10-08 11:27:10 -04001591 if (op & EXTENT_CLEAR_DIRTY)
Chris Masonc8b97812008-10-29 14:49:59 -04001592 clear_page_dirty_for_io(pages[i]);
Chris Masona791e352009-10-08 11:27:10 -04001593 if (op & EXTENT_SET_WRITEBACK)
Chris Masonc8b97812008-10-29 14:49:59 -04001594 set_page_writeback(pages[i]);
Chris Masona791e352009-10-08 11:27:10 -04001595 if (op & EXTENT_END_WRITEBACK)
Chris Masonc8b97812008-10-29 14:49:59 -04001596 end_page_writeback(pages[i]);
Chris Masona791e352009-10-08 11:27:10 -04001597 if (op & EXTENT_CLEAR_UNLOCK_PAGE)
Chris Mason771ed682008-11-06 22:02:51 -05001598 unlock_page(pages[i]);
Chris Masonc8b97812008-10-29 14:49:59 -04001599 page_cache_release(pages[i]);
1600 }
1601 nr_pages -= ret;
1602 index += ret;
1603 cond_resched();
1604 }
1605 return 0;
1606}
Chris Masonc8b97812008-10-29 14:49:59 -04001607
Chris Masond352ac62008-09-29 15:18:18 -04001608/*
1609 * count the number of bytes in the tree that have a given bit(s)
1610 * set. This can be fairly slow, except for EXTENT_DIRTY which is
1611 * cached. The total number found is returned.
1612 */
Chris Masond1310b22008-01-24 16:13:08 -05001613u64 count_range_bits(struct extent_io_tree *tree,
1614 u64 *start, u64 search_end, u64 max_bytes,
Chris Masonec29ed52011-02-23 16:23:20 -05001615 unsigned long bits, int contig)
Chris Masond1310b22008-01-24 16:13:08 -05001616{
1617 struct rb_node *node;
1618 struct extent_state *state;
1619 u64 cur_start = *start;
1620 u64 total_bytes = 0;
Chris Masonec29ed52011-02-23 16:23:20 -05001621 u64 last = 0;
Chris Masond1310b22008-01-24 16:13:08 -05001622 int found = 0;
1623
1624 if (search_end <= cur_start) {
Chris Masond1310b22008-01-24 16:13:08 -05001625 WARN_ON(1);
1626 return 0;
1627 }
1628
Chris Masoncad321a2008-12-17 14:51:42 -05001629 spin_lock(&tree->lock);
Chris Masond1310b22008-01-24 16:13:08 -05001630 if (cur_start == 0 && bits == EXTENT_DIRTY) {
1631 total_bytes = tree->dirty_bytes;
1632 goto out;
1633 }
1634 /*
1635 * this search will find all the extents that end after
1636 * our range starts.
1637 */
Chris Mason80ea96b2008-02-01 14:51:59 -05001638 node = tree_search(tree, cur_start);
Chris Masond3977122009-01-05 21:25:51 -05001639 if (!node)
Chris Masond1310b22008-01-24 16:13:08 -05001640 goto out;
Chris Masond1310b22008-01-24 16:13:08 -05001641
Chris Masond3977122009-01-05 21:25:51 -05001642 while (1) {
Chris Masond1310b22008-01-24 16:13:08 -05001643 state = rb_entry(node, struct extent_state, rb_node);
1644 if (state->start > search_end)
1645 break;
Chris Masonec29ed52011-02-23 16:23:20 -05001646 if (contig && found && state->start > last + 1)
1647 break;
1648 if (state->end >= cur_start && (state->state & bits) == bits) {
Chris Masond1310b22008-01-24 16:13:08 -05001649 total_bytes += min(search_end, state->end) + 1 -
1650 max(cur_start, state->start);
1651 if (total_bytes >= max_bytes)
1652 break;
1653 if (!found) {
Josef Bacikaf60bed2011-05-04 11:11:17 -04001654 *start = max(cur_start, state->start);
Chris Masond1310b22008-01-24 16:13:08 -05001655 found = 1;
1656 }
Chris Masonec29ed52011-02-23 16:23:20 -05001657 last = state->end;
1658 } else if (contig && found) {
1659 break;
Chris Masond1310b22008-01-24 16:13:08 -05001660 }
1661 node = rb_next(node);
1662 if (!node)
1663 break;
1664 }
1665out:
Chris Masoncad321a2008-12-17 14:51:42 -05001666 spin_unlock(&tree->lock);
Chris Masond1310b22008-01-24 16:13:08 -05001667 return total_bytes;
1668}
Christoph Hellwigb2950862008-12-02 09:54:17 -05001669
Chris Masond352ac62008-09-29 15:18:18 -04001670/*
1671 * set the private field for a given byte offset in the tree. If there isn't
1672 * an extent_state there already, this does nothing.
1673 */
Chris Masond1310b22008-01-24 16:13:08 -05001674int set_state_private(struct extent_io_tree *tree, u64 start, u64 private)
1675{
1676 struct rb_node *node;
1677 struct extent_state *state;
1678 int ret = 0;
1679
Chris Masoncad321a2008-12-17 14:51:42 -05001680 spin_lock(&tree->lock);
Chris Masond1310b22008-01-24 16:13:08 -05001681 /*
1682 * this search will find all the extents that end after
1683 * our range starts.
1684 */
Chris Mason80ea96b2008-02-01 14:51:59 -05001685 node = tree_search(tree, start);
Peter2b114d12008-04-01 11:21:40 -04001686 if (!node) {
Chris Masond1310b22008-01-24 16:13:08 -05001687 ret = -ENOENT;
1688 goto out;
1689 }
1690 state = rb_entry(node, struct extent_state, rb_node);
1691 if (state->start != start) {
1692 ret = -ENOENT;
1693 goto out;
1694 }
1695 state->private = private;
1696out:
Chris Masoncad321a2008-12-17 14:51:42 -05001697 spin_unlock(&tree->lock);
Chris Masond1310b22008-01-24 16:13:08 -05001698 return ret;
1699}
1700
1701int get_state_private(struct extent_io_tree *tree, u64 start, u64 *private)
1702{
1703 struct rb_node *node;
1704 struct extent_state *state;
1705 int ret = 0;
1706
Chris Masoncad321a2008-12-17 14:51:42 -05001707 spin_lock(&tree->lock);
Chris Masond1310b22008-01-24 16:13:08 -05001708 /*
1709 * this search will find all the extents that end after
1710 * our range starts.
1711 */
Chris Mason80ea96b2008-02-01 14:51:59 -05001712 node = tree_search(tree, start);
Peter2b114d12008-04-01 11:21:40 -04001713 if (!node) {
Chris Masond1310b22008-01-24 16:13:08 -05001714 ret = -ENOENT;
1715 goto out;
1716 }
1717 state = rb_entry(node, struct extent_state, rb_node);
1718 if (state->start != start) {
1719 ret = -ENOENT;
1720 goto out;
1721 }
1722 *private = state->private;
1723out:
Chris Masoncad321a2008-12-17 14:51:42 -05001724 spin_unlock(&tree->lock);
Chris Masond1310b22008-01-24 16:13:08 -05001725 return ret;
1726}
1727
1728/*
1729 * searches a range in the state tree for a given mask.
Chris Mason70dec802008-01-29 09:59:12 -05001730 * If 'filled' == 1, this returns 1 only if every extent in the tree
Chris Masond1310b22008-01-24 16:13:08 -05001731 * has the bits set. Otherwise, 1 is returned if any bit in the
1732 * range is found set.
1733 */
1734int test_range_bit(struct extent_io_tree *tree, u64 start, u64 end,
Chris Mason9655d292009-09-02 15:22:30 -04001735 int bits, int filled, struct extent_state *cached)
Chris Masond1310b22008-01-24 16:13:08 -05001736{
1737 struct extent_state *state = NULL;
1738 struct rb_node *node;
1739 int bitset = 0;
Chris Masond1310b22008-01-24 16:13:08 -05001740
Chris Masoncad321a2008-12-17 14:51:42 -05001741 spin_lock(&tree->lock);
Josef Bacikdf98b6e2011-06-20 14:53:48 -04001742 if (cached && cached->tree && cached->start <= start &&
1743 cached->end > start)
Chris Mason9655d292009-09-02 15:22:30 -04001744 node = &cached->rb_node;
1745 else
1746 node = tree_search(tree, start);
Chris Masond1310b22008-01-24 16:13:08 -05001747 while (node && start <= end) {
1748 state = rb_entry(node, struct extent_state, rb_node);
1749
1750 if (filled && state->start > start) {
1751 bitset = 0;
1752 break;
1753 }
1754
1755 if (state->start > end)
1756 break;
1757
1758 if (state->state & bits) {
1759 bitset = 1;
1760 if (!filled)
1761 break;
1762 } else if (filled) {
1763 bitset = 0;
1764 break;
1765 }
Chris Mason46562cec2009-09-23 20:23:16 -04001766
1767 if (state->end == (u64)-1)
1768 break;
1769
Chris Masond1310b22008-01-24 16:13:08 -05001770 start = state->end + 1;
1771 if (start > end)
1772 break;
1773 node = rb_next(node);
1774 if (!node) {
1775 if (filled)
1776 bitset = 0;
1777 break;
1778 }
1779 }
Chris Masoncad321a2008-12-17 14:51:42 -05001780 spin_unlock(&tree->lock);
Chris Masond1310b22008-01-24 16:13:08 -05001781 return bitset;
1782}
Chris Masond1310b22008-01-24 16:13:08 -05001783
1784/*
1785 * helper function to set a given page up to date if all the
1786 * extents in the tree for that page are up to date
1787 */
Jeff Mahoney143bede2012-03-01 14:56:26 +01001788static void check_page_uptodate(struct extent_io_tree *tree, struct page *page)
Chris Masond1310b22008-01-24 16:13:08 -05001789{
1790 u64 start = (u64)page->index << PAGE_CACHE_SHIFT;
1791 u64 end = start + PAGE_CACHE_SIZE - 1;
Chris Mason9655d292009-09-02 15:22:30 -04001792 if (test_range_bit(tree, start, end, EXTENT_UPTODATE, 1, NULL))
Chris Masond1310b22008-01-24 16:13:08 -05001793 SetPageUptodate(page);
Chris Masond1310b22008-01-24 16:13:08 -05001794}
1795
1796/*
1797 * helper function to unlock a page if all the extents in the tree
1798 * for that page are unlocked
1799 */
Jeff Mahoney143bede2012-03-01 14:56:26 +01001800static void check_page_locked(struct extent_io_tree *tree, struct page *page)
Chris Masond1310b22008-01-24 16:13:08 -05001801{
1802 u64 start = (u64)page->index << PAGE_CACHE_SHIFT;
1803 u64 end = start + PAGE_CACHE_SIZE - 1;
Chris Mason9655d292009-09-02 15:22:30 -04001804 if (!test_range_bit(tree, start, end, EXTENT_LOCKED, 0, NULL))
Chris Masond1310b22008-01-24 16:13:08 -05001805 unlock_page(page);
Chris Masond1310b22008-01-24 16:13:08 -05001806}
1807
1808/*
1809 * helper function to end page writeback if all the extents
1810 * in the tree for that page are done with writeback
1811 */
Jeff Mahoney143bede2012-03-01 14:56:26 +01001812static void check_page_writeback(struct extent_io_tree *tree,
1813 struct page *page)
Chris Masond1310b22008-01-24 16:13:08 -05001814{
Chris Mason1edbb732009-09-02 13:24:36 -04001815 end_page_writeback(page);
Chris Masond1310b22008-01-24 16:13:08 -05001816}
1817
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02001818/*
1819 * When IO fails, either with EIO or csum verification fails, we
1820 * try other mirrors that might have a good copy of the data. This
1821 * io_failure_record is used to record state as we go through all the
1822 * mirrors. If another mirror has good data, the page is set up to date
1823 * and things continue. If a good mirror can't be found, the original
1824 * bio end_io callback is called to indicate things have failed.
1825 */
1826struct io_failure_record {
1827 struct page *page;
1828 u64 start;
1829 u64 len;
1830 u64 logical;
1831 unsigned long bio_flags;
1832 int this_mirror;
1833 int failed_mirror;
1834 int in_validation;
1835};
1836
1837static int free_io_failure(struct inode *inode, struct io_failure_record *rec,
1838 int did_repair)
1839{
1840 int ret;
1841 int err = 0;
1842 struct extent_io_tree *failure_tree = &BTRFS_I(inode)->io_failure_tree;
1843
1844 set_state_private(failure_tree, rec->start, 0);
1845 ret = clear_extent_bits(failure_tree, rec->start,
1846 rec->start + rec->len - 1,
1847 EXTENT_LOCKED | EXTENT_DIRTY, GFP_NOFS);
1848 if (ret)
1849 err = ret;
1850
1851 if (did_repair) {
1852 ret = clear_extent_bits(&BTRFS_I(inode)->io_tree, rec->start,
1853 rec->start + rec->len - 1,
1854 EXTENT_DAMAGED, GFP_NOFS);
1855 if (ret && !err)
1856 err = ret;
1857 }
1858
1859 kfree(rec);
1860 return err;
1861}
1862
1863static void repair_io_failure_callback(struct bio *bio, int err)
1864{
1865 complete(bio->bi_private);
1866}
1867
1868/*
1869 * this bypasses the standard btrfs submit functions deliberately, as
1870 * the standard behavior is to write all copies in a raid setup. here we only
1871 * want to write the one bad copy. so we do the mapping for ourselves and issue
1872 * submit_bio directly.
1873 * to avoid any synchonization issues, wait for the data after writing, which
1874 * actually prevents the read that triggered the error from finishing.
1875 * currently, there can be no more than two copies of every data bit. thus,
1876 * exactly one rewrite is required.
1877 */
1878int repair_io_failure(struct btrfs_mapping_tree *map_tree, u64 start,
1879 u64 length, u64 logical, struct page *page,
1880 int mirror_num)
1881{
1882 struct bio *bio;
1883 struct btrfs_device *dev;
1884 DECLARE_COMPLETION_ONSTACK(compl);
1885 u64 map_length = 0;
1886 u64 sector;
1887 struct btrfs_bio *bbio = NULL;
1888 int ret;
1889
1890 BUG_ON(!mirror_num);
1891
1892 bio = bio_alloc(GFP_NOFS, 1);
1893 if (!bio)
1894 return -EIO;
1895 bio->bi_private = &compl;
1896 bio->bi_end_io = repair_io_failure_callback;
1897 bio->bi_size = 0;
1898 map_length = length;
1899
1900 ret = btrfs_map_block(map_tree, WRITE, logical,
1901 &map_length, &bbio, mirror_num);
1902 if (ret) {
1903 bio_put(bio);
1904 return -EIO;
1905 }
1906 BUG_ON(mirror_num != bbio->mirror_num);
1907 sector = bbio->stripes[mirror_num-1].physical >> 9;
1908 bio->bi_sector = sector;
1909 dev = bbio->stripes[mirror_num-1].dev;
1910 kfree(bbio);
1911 if (!dev || !dev->bdev || !dev->writeable) {
1912 bio_put(bio);
1913 return -EIO;
1914 }
1915 bio->bi_bdev = dev->bdev;
1916 bio_add_page(bio, page, length, start-page_offset(page));
Stefan Behrens21adbd52011-11-09 13:44:05 +01001917 btrfsic_submit_bio(WRITE_SYNC, bio);
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02001918 wait_for_completion(&compl);
1919
1920 if (!test_bit(BIO_UPTODATE, &bio->bi_flags)) {
1921 /* try to remap that extent elsewhere? */
1922 bio_put(bio);
1923 return -EIO;
1924 }
1925
1926 printk(KERN_INFO "btrfs read error corrected: ino %lu off %llu (dev %s "
1927 "sector %llu)\n", page->mapping->host->i_ino, start,
1928 dev->name, sector);
1929
1930 bio_put(bio);
1931 return 0;
1932}
1933
Josef Bacikea466792012-03-26 21:57:36 -04001934int repair_eb_io_failure(struct btrfs_root *root, struct extent_buffer *eb,
1935 int mirror_num)
1936{
1937 struct btrfs_mapping_tree *map_tree = &root->fs_info->mapping_tree;
1938 u64 start = eb->start;
1939 unsigned long i, num_pages = num_extent_pages(eb->start, eb->len);
1940 int ret;
1941
1942 for (i = 0; i < num_pages; i++) {
1943 struct page *p = extent_buffer_page(eb, i);
1944 ret = repair_io_failure(map_tree, start, PAGE_CACHE_SIZE,
1945 start, p, mirror_num);
1946 if (ret)
1947 break;
1948 start += PAGE_CACHE_SIZE;
1949 }
1950
1951 return ret;
1952}
1953
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02001954/*
1955 * each time an IO finishes, we do a fast check in the IO failure tree
1956 * to see if we need to process or clean up an io_failure_record
1957 */
1958static int clean_io_failure(u64 start, struct page *page)
1959{
1960 u64 private;
1961 u64 private_failure;
1962 struct io_failure_record *failrec;
1963 struct btrfs_mapping_tree *map_tree;
1964 struct extent_state *state;
1965 int num_copies;
1966 int did_repair = 0;
1967 int ret;
1968 struct inode *inode = page->mapping->host;
1969
1970 private = 0;
1971 ret = count_range_bits(&BTRFS_I(inode)->io_failure_tree, &private,
1972 (u64)-1, 1, EXTENT_DIRTY, 0);
1973 if (!ret)
1974 return 0;
1975
1976 ret = get_state_private(&BTRFS_I(inode)->io_failure_tree, start,
1977 &private_failure);
1978 if (ret)
1979 return 0;
1980
1981 failrec = (struct io_failure_record *)(unsigned long) private_failure;
1982 BUG_ON(!failrec->this_mirror);
1983
1984 if (failrec->in_validation) {
1985 /* there was no real error, just free the record */
1986 pr_debug("clean_io_failure: freeing dummy error at %llu\n",
1987 failrec->start);
1988 did_repair = 1;
1989 goto out;
1990 }
1991
1992 spin_lock(&BTRFS_I(inode)->io_tree.lock);
1993 state = find_first_extent_bit_state(&BTRFS_I(inode)->io_tree,
1994 failrec->start,
1995 EXTENT_LOCKED);
1996 spin_unlock(&BTRFS_I(inode)->io_tree.lock);
1997
1998 if (state && state->start == failrec->start) {
1999 map_tree = &BTRFS_I(inode)->root->fs_info->mapping_tree;
2000 num_copies = btrfs_num_copies(map_tree, failrec->logical,
2001 failrec->len);
2002 if (num_copies > 1) {
2003 ret = repair_io_failure(map_tree, start, failrec->len,
2004 failrec->logical, page,
2005 failrec->failed_mirror);
2006 did_repair = !ret;
2007 }
2008 }
2009
2010out:
2011 if (!ret)
2012 ret = free_io_failure(inode, failrec, did_repair);
2013
2014 return ret;
2015}
2016
2017/*
2018 * this is a generic handler for readpage errors (default
2019 * readpage_io_failed_hook). if other copies exist, read those and write back
2020 * good data to the failed position. does not investigate in remapping the
2021 * failed extent elsewhere, hoping the device will be smart enough to do this as
2022 * needed
2023 */
2024
2025static int bio_readpage_error(struct bio *failed_bio, struct page *page,
2026 u64 start, u64 end, int failed_mirror,
2027 struct extent_state *state)
2028{
2029 struct io_failure_record *failrec = NULL;
2030 u64 private;
2031 struct extent_map *em;
2032 struct inode *inode = page->mapping->host;
2033 struct extent_io_tree *failure_tree = &BTRFS_I(inode)->io_failure_tree;
2034 struct extent_io_tree *tree = &BTRFS_I(inode)->io_tree;
2035 struct extent_map_tree *em_tree = &BTRFS_I(inode)->extent_tree;
2036 struct bio *bio;
2037 int num_copies;
2038 int ret;
2039 int read_mode;
2040 u64 logical;
2041
2042 BUG_ON(failed_bio->bi_rw & REQ_WRITE);
2043
2044 ret = get_state_private(failure_tree, start, &private);
2045 if (ret) {
2046 failrec = kzalloc(sizeof(*failrec), GFP_NOFS);
2047 if (!failrec)
2048 return -ENOMEM;
2049 failrec->start = start;
2050 failrec->len = end - start + 1;
2051 failrec->this_mirror = 0;
2052 failrec->bio_flags = 0;
2053 failrec->in_validation = 0;
2054
2055 read_lock(&em_tree->lock);
2056 em = lookup_extent_mapping(em_tree, start, failrec->len);
2057 if (!em) {
2058 read_unlock(&em_tree->lock);
2059 kfree(failrec);
2060 return -EIO;
2061 }
2062
2063 if (em->start > start || em->start + em->len < start) {
2064 free_extent_map(em);
2065 em = NULL;
2066 }
2067 read_unlock(&em_tree->lock);
2068
2069 if (!em || IS_ERR(em)) {
2070 kfree(failrec);
2071 return -EIO;
2072 }
2073 logical = start - em->start;
2074 logical = em->block_start + logical;
2075 if (test_bit(EXTENT_FLAG_COMPRESSED, &em->flags)) {
2076 logical = em->block_start;
2077 failrec->bio_flags = EXTENT_BIO_COMPRESSED;
2078 extent_set_compress_type(&failrec->bio_flags,
2079 em->compress_type);
2080 }
2081 pr_debug("bio_readpage_error: (new) logical=%llu, start=%llu, "
2082 "len=%llu\n", logical, start, failrec->len);
2083 failrec->logical = logical;
2084 free_extent_map(em);
2085
2086 /* set the bits in the private failure tree */
2087 ret = set_extent_bits(failure_tree, start, end,
2088 EXTENT_LOCKED | EXTENT_DIRTY, GFP_NOFS);
2089 if (ret >= 0)
2090 ret = set_state_private(failure_tree, start,
2091 (u64)(unsigned long)failrec);
2092 /* set the bits in the inode's tree */
2093 if (ret >= 0)
2094 ret = set_extent_bits(tree, start, end, EXTENT_DAMAGED,
2095 GFP_NOFS);
2096 if (ret < 0) {
2097 kfree(failrec);
2098 return ret;
2099 }
2100 } else {
2101 failrec = (struct io_failure_record *)(unsigned long)private;
2102 pr_debug("bio_readpage_error: (found) logical=%llu, "
2103 "start=%llu, len=%llu, validation=%d\n",
2104 failrec->logical, failrec->start, failrec->len,
2105 failrec->in_validation);
2106 /*
2107 * when data can be on disk more than twice, add to failrec here
2108 * (e.g. with a list for failed_mirror) to make
2109 * clean_io_failure() clean all those errors at once.
2110 */
2111 }
2112 num_copies = btrfs_num_copies(
2113 &BTRFS_I(inode)->root->fs_info->mapping_tree,
2114 failrec->logical, failrec->len);
2115 if (num_copies == 1) {
2116 /*
2117 * we only have a single copy of the data, so don't bother with
2118 * all the retry and error correction code that follows. no
2119 * matter what the error is, it is very likely to persist.
2120 */
2121 pr_debug("bio_readpage_error: cannot repair, num_copies == 1. "
2122 "state=%p, num_copies=%d, next_mirror %d, "
2123 "failed_mirror %d\n", state, num_copies,
2124 failrec->this_mirror, failed_mirror);
2125 free_io_failure(inode, failrec, 0);
2126 return -EIO;
2127 }
2128
2129 if (!state) {
2130 spin_lock(&tree->lock);
2131 state = find_first_extent_bit_state(tree, failrec->start,
2132 EXTENT_LOCKED);
2133 if (state && state->start != failrec->start)
2134 state = NULL;
2135 spin_unlock(&tree->lock);
2136 }
2137
2138 /*
2139 * there are two premises:
2140 * a) deliver good data to the caller
2141 * b) correct the bad sectors on disk
2142 */
2143 if (failed_bio->bi_vcnt > 1) {
2144 /*
2145 * to fulfill b), we need to know the exact failing sectors, as
2146 * we don't want to rewrite any more than the failed ones. thus,
2147 * we need separate read requests for the failed bio
2148 *
2149 * if the following BUG_ON triggers, our validation request got
2150 * merged. we need separate requests for our algorithm to work.
2151 */
2152 BUG_ON(failrec->in_validation);
2153 failrec->in_validation = 1;
2154 failrec->this_mirror = failed_mirror;
2155 read_mode = READ_SYNC | REQ_FAILFAST_DEV;
2156 } else {
2157 /*
2158 * we're ready to fulfill a) and b) alongside. get a good copy
2159 * of the failed sector and if we succeed, we have setup
2160 * everything for repair_io_failure to do the rest for us.
2161 */
2162 if (failrec->in_validation) {
2163 BUG_ON(failrec->this_mirror != failed_mirror);
2164 failrec->in_validation = 0;
2165 failrec->this_mirror = 0;
2166 }
2167 failrec->failed_mirror = failed_mirror;
2168 failrec->this_mirror++;
2169 if (failrec->this_mirror == failed_mirror)
2170 failrec->this_mirror++;
2171 read_mode = READ_SYNC;
2172 }
2173
2174 if (!state || failrec->this_mirror > num_copies) {
2175 pr_debug("bio_readpage_error: (fail) state=%p, num_copies=%d, "
2176 "next_mirror %d, failed_mirror %d\n", state,
2177 num_copies, failrec->this_mirror, failed_mirror);
2178 free_io_failure(inode, failrec, 0);
2179 return -EIO;
2180 }
2181
2182 bio = bio_alloc(GFP_NOFS, 1);
2183 bio->bi_private = state;
2184 bio->bi_end_io = failed_bio->bi_end_io;
2185 bio->bi_sector = failrec->logical >> 9;
2186 bio->bi_bdev = BTRFS_I(inode)->root->fs_info->fs_devices->latest_bdev;
2187 bio->bi_size = 0;
2188
2189 bio_add_page(bio, page, failrec->len, start - page_offset(page));
2190
2191 pr_debug("bio_readpage_error: submitting new read[%#x] to "
2192 "this_mirror=%d, num_copies=%d, in_validation=%d\n", read_mode,
2193 failrec->this_mirror, num_copies, failrec->in_validation);
2194
Tsutomu Itoh013bd4c2012-02-16 10:11:40 +09002195 ret = tree->ops->submit_bio_hook(inode, read_mode, bio,
2196 failrec->this_mirror,
2197 failrec->bio_flags, 0);
2198 return ret;
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002199}
2200
Chris Masond1310b22008-01-24 16:13:08 -05002201/* lots and lots of room for performance fixes in the end_bio funcs */
2202
Jeff Mahoney87826df2012-02-15 16:23:57 +01002203int end_extent_writepage(struct page *page, int err, u64 start, u64 end)
2204{
2205 int uptodate = (err == 0);
2206 struct extent_io_tree *tree;
2207 int ret;
2208
2209 tree = &BTRFS_I(page->mapping->host)->io_tree;
2210
2211 if (tree->ops && tree->ops->writepage_end_io_hook) {
2212 ret = tree->ops->writepage_end_io_hook(page, start,
2213 end, NULL, uptodate);
2214 if (ret)
2215 uptodate = 0;
2216 }
2217
2218 if (!uptodate && tree->ops &&
2219 tree->ops->writepage_io_failed_hook) {
2220 ret = tree->ops->writepage_io_failed_hook(NULL, page,
2221 start, end, NULL);
2222 /* Writeback already completed */
2223 if (ret == 0)
2224 return 1;
2225 }
2226
2227 if (!uptodate) {
2228 clear_extent_uptodate(tree, start, end, NULL, GFP_NOFS);
2229 ClearPageUptodate(page);
2230 SetPageError(page);
2231 }
2232 return 0;
2233}
2234
Chris Masond1310b22008-01-24 16:13:08 -05002235/*
2236 * after a writepage IO is done, we need to:
2237 * clear the uptodate bits on error
2238 * clear the writeback bits in the extent tree for this IO
2239 * end_page_writeback if the page has no more pending IO
2240 *
2241 * Scheduling is not allowed, so the extent state tree is expected
2242 * to have one and only one object corresponding to this IO.
2243 */
Chris Masond1310b22008-01-24 16:13:08 -05002244static void end_bio_extent_writepage(struct bio *bio, int err)
Chris Masond1310b22008-01-24 16:13:08 -05002245{
Chris Masond1310b22008-01-24 16:13:08 -05002246 struct bio_vec *bvec = bio->bi_io_vec + bio->bi_vcnt - 1;
David Woodhouse902b22f2008-08-20 08:51:49 -04002247 struct extent_io_tree *tree;
Chris Masond1310b22008-01-24 16:13:08 -05002248 u64 start;
2249 u64 end;
2250 int whole_page;
2251
Chris Masond1310b22008-01-24 16:13:08 -05002252 do {
2253 struct page *page = bvec->bv_page;
David Woodhouse902b22f2008-08-20 08:51:49 -04002254 tree = &BTRFS_I(page->mapping->host)->io_tree;
2255
Chris Masond1310b22008-01-24 16:13:08 -05002256 start = ((u64)page->index << PAGE_CACHE_SHIFT) +
2257 bvec->bv_offset;
2258 end = start + bvec->bv_len - 1;
2259
2260 if (bvec->bv_offset == 0 && bvec->bv_len == PAGE_CACHE_SIZE)
2261 whole_page = 1;
2262 else
2263 whole_page = 0;
2264
2265 if (--bvec >= bio->bi_io_vec)
2266 prefetchw(&bvec->bv_page->flags);
Chris Mason1259ab72008-05-12 13:39:03 -04002267
Jeff Mahoney87826df2012-02-15 16:23:57 +01002268 if (end_extent_writepage(page, err, start, end))
2269 continue;
Chris Mason70dec802008-01-29 09:59:12 -05002270
Chris Masond1310b22008-01-24 16:13:08 -05002271 if (whole_page)
2272 end_page_writeback(page);
2273 else
2274 check_page_writeback(tree, page);
Chris Masond1310b22008-01-24 16:13:08 -05002275 } while (bvec >= bio->bi_io_vec);
Chris Mason2b1f55b2008-09-24 11:48:04 -04002276
Chris Masond1310b22008-01-24 16:13:08 -05002277 bio_put(bio);
Chris Masond1310b22008-01-24 16:13:08 -05002278}
2279
2280/*
2281 * after a readpage IO is done, we need to:
2282 * clear the uptodate bits on error
2283 * set the uptodate bits if things worked
2284 * set the page up to date if all extents in the tree are uptodate
2285 * clear the lock bit in the extent tree
2286 * unlock the page if there are no other extents locked for it
2287 *
2288 * Scheduling is not allowed, so the extent state tree is expected
2289 * to have one and only one object corresponding to this IO.
2290 */
Chris Masond1310b22008-01-24 16:13:08 -05002291static void end_bio_extent_readpage(struct bio *bio, int err)
Chris Masond1310b22008-01-24 16:13:08 -05002292{
2293 int uptodate = test_bit(BIO_UPTODATE, &bio->bi_flags);
Chris Mason4125bf72010-02-03 18:18:45 +00002294 struct bio_vec *bvec_end = bio->bi_io_vec + bio->bi_vcnt - 1;
2295 struct bio_vec *bvec = bio->bi_io_vec;
David Woodhouse902b22f2008-08-20 08:51:49 -04002296 struct extent_io_tree *tree;
Chris Masond1310b22008-01-24 16:13:08 -05002297 u64 start;
2298 u64 end;
2299 int whole_page;
Josef Bacikea466792012-03-26 21:57:36 -04002300 int failed_mirror;
Chris Masond1310b22008-01-24 16:13:08 -05002301 int ret;
2302
Chris Masond20f7042008-12-08 16:58:54 -05002303 if (err)
2304 uptodate = 0;
2305
Chris Masond1310b22008-01-24 16:13:08 -05002306 do {
2307 struct page *page = bvec->bv_page;
Arne Jansen507903b2011-04-06 10:02:20 +00002308 struct extent_state *cached = NULL;
2309 struct extent_state *state;
2310
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002311 pr_debug("end_bio_extent_readpage: bi_vcnt=%d, idx=%d, err=%d, "
2312 "mirror=%ld\n", bio->bi_vcnt, bio->bi_idx, err,
2313 (long int)bio->bi_bdev);
David Woodhouse902b22f2008-08-20 08:51:49 -04002314 tree = &BTRFS_I(page->mapping->host)->io_tree;
2315
Chris Masond1310b22008-01-24 16:13:08 -05002316 start = ((u64)page->index << PAGE_CACHE_SHIFT) +
2317 bvec->bv_offset;
2318 end = start + bvec->bv_len - 1;
2319
2320 if (bvec->bv_offset == 0 && bvec->bv_len == PAGE_CACHE_SIZE)
2321 whole_page = 1;
2322 else
2323 whole_page = 0;
2324
Chris Mason4125bf72010-02-03 18:18:45 +00002325 if (++bvec <= bvec_end)
Chris Masond1310b22008-01-24 16:13:08 -05002326 prefetchw(&bvec->bv_page->flags);
2327
Arne Jansen507903b2011-04-06 10:02:20 +00002328 spin_lock(&tree->lock);
Chris Mason0d399202011-04-16 06:55:39 -04002329 state = find_first_extent_bit_state(tree, start, EXTENT_LOCKED);
Chris Mason109b36a2011-04-12 13:57:39 -04002330 if (state && state->start == start) {
Arne Jansen507903b2011-04-06 10:02:20 +00002331 /*
2332 * take a reference on the state, unlock will drop
2333 * the ref
2334 */
2335 cache_state(state, &cached);
2336 }
2337 spin_unlock(&tree->lock);
2338
Chris Masond1310b22008-01-24 16:13:08 -05002339 if (uptodate && tree->ops && tree->ops->readpage_end_io_hook) {
Chris Mason70dec802008-01-29 09:59:12 -05002340 ret = tree->ops->readpage_end_io_hook(page, start, end,
Arne Jansen507903b2011-04-06 10:02:20 +00002341 state);
Chris Masond1310b22008-01-24 16:13:08 -05002342 if (ret)
2343 uptodate = 0;
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002344 else
2345 clean_io_failure(start, page);
Chris Masond1310b22008-01-24 16:13:08 -05002346 }
Josef Bacikea466792012-03-26 21:57:36 -04002347
2348 if (!uptodate)
Jan Schmidt32240a92011-11-20 07:33:38 -05002349 failed_mirror = (int)(unsigned long)bio->bi_bdev;
Josef Bacikea466792012-03-26 21:57:36 -04002350
2351 if (!uptodate && tree->ops && tree->ops->readpage_io_failed_hook) {
2352 ret = tree->ops->readpage_io_failed_hook(page, failed_mirror);
2353 if (!ret && !err &&
2354 test_bit(BIO_UPTODATE, &bio->bi_flags))
2355 uptodate = 1;
2356 } else if (!uptodate) {
Jan Schmidtf4a8e652011-12-01 09:30:36 -05002357 /*
2358 * The generic bio_readpage_error handles errors the
2359 * following way: If possible, new read requests are
2360 * created and submitted and will end up in
2361 * end_bio_extent_readpage as well (if we're lucky, not
2362 * in the !uptodate case). In that case it returns 0 and
2363 * we just go on with the next page in our bio. If it
2364 * can't handle the error it will return -EIO and we
2365 * remain responsible for that page.
2366 */
2367 ret = bio_readpage_error(bio, page, start, end,
2368 failed_mirror, NULL);
Chris Mason7e383262008-04-09 16:28:12 -04002369 if (ret == 0) {
Chris Mason3b951512008-04-17 11:29:12 -04002370 uptodate =
2371 test_bit(BIO_UPTODATE, &bio->bi_flags);
Chris Masond20f7042008-12-08 16:58:54 -05002372 if (err)
2373 uptodate = 0;
Arne Jansen507903b2011-04-06 10:02:20 +00002374 uncache_state(&cached);
Chris Mason7e383262008-04-09 16:28:12 -04002375 continue;
2376 }
2377 }
Chris Mason70dec802008-01-29 09:59:12 -05002378
Josef Bacik0b32f4b2012-03-13 09:38:00 -04002379 if (uptodate && tree->track_uptodate) {
Arne Jansen507903b2011-04-06 10:02:20 +00002380 set_extent_uptodate(tree, start, end, &cached,
David Woodhouse902b22f2008-08-20 08:51:49 -04002381 GFP_ATOMIC);
Chris Mason771ed682008-11-06 22:02:51 -05002382 }
Arne Jansen507903b2011-04-06 10:02:20 +00002383 unlock_extent_cached(tree, start, end, &cached, GFP_ATOMIC);
Chris Masond1310b22008-01-24 16:13:08 -05002384
Chris Mason70dec802008-01-29 09:59:12 -05002385 if (whole_page) {
2386 if (uptodate) {
2387 SetPageUptodate(page);
2388 } else {
2389 ClearPageUptodate(page);
2390 SetPageError(page);
2391 }
Chris Masond1310b22008-01-24 16:13:08 -05002392 unlock_page(page);
Chris Mason70dec802008-01-29 09:59:12 -05002393 } else {
2394 if (uptodate) {
2395 check_page_uptodate(tree, page);
2396 } else {
2397 ClearPageUptodate(page);
2398 SetPageError(page);
2399 }
Chris Masond1310b22008-01-24 16:13:08 -05002400 check_page_locked(tree, page);
Chris Mason70dec802008-01-29 09:59:12 -05002401 }
Chris Mason4125bf72010-02-03 18:18:45 +00002402 } while (bvec <= bvec_end);
Chris Masond1310b22008-01-24 16:13:08 -05002403
2404 bio_put(bio);
Chris Masond1310b22008-01-24 16:13:08 -05002405}
2406
Miao Xie88f794e2010-11-22 03:02:55 +00002407struct bio *
2408btrfs_bio_alloc(struct block_device *bdev, u64 first_sector, int nr_vecs,
2409 gfp_t gfp_flags)
Chris Masond1310b22008-01-24 16:13:08 -05002410{
2411 struct bio *bio;
2412
2413 bio = bio_alloc(gfp_flags, nr_vecs);
2414
2415 if (bio == NULL && (current->flags & PF_MEMALLOC)) {
2416 while (!bio && (nr_vecs /= 2))
2417 bio = bio_alloc(gfp_flags, nr_vecs);
2418 }
2419
2420 if (bio) {
Chris Masone1c4b742008-04-22 13:26:46 -04002421 bio->bi_size = 0;
Chris Masond1310b22008-01-24 16:13:08 -05002422 bio->bi_bdev = bdev;
2423 bio->bi_sector = first_sector;
2424 }
2425 return bio;
2426}
2427
Jeff Mahoney79787ea2012-03-12 16:03:00 +01002428/*
2429 * Since writes are async, they will only return -ENOMEM.
2430 * Reads can return the full range of I/O error conditions.
2431 */
Jeff Mahoney355808c2011-10-03 23:23:14 -04002432static int __must_check submit_one_bio(int rw, struct bio *bio,
2433 int mirror_num, unsigned long bio_flags)
Chris Masond1310b22008-01-24 16:13:08 -05002434{
Chris Masond1310b22008-01-24 16:13:08 -05002435 int ret = 0;
Chris Mason70dec802008-01-29 09:59:12 -05002436 struct bio_vec *bvec = bio->bi_io_vec + bio->bi_vcnt - 1;
2437 struct page *page = bvec->bv_page;
2438 struct extent_io_tree *tree = bio->bi_private;
Chris Mason70dec802008-01-29 09:59:12 -05002439 u64 start;
Chris Mason70dec802008-01-29 09:59:12 -05002440
2441 start = ((u64)page->index << PAGE_CACHE_SHIFT) + bvec->bv_offset;
Chris Mason70dec802008-01-29 09:59:12 -05002442
David Woodhouse902b22f2008-08-20 08:51:49 -04002443 bio->bi_private = NULL;
Chris Masond1310b22008-01-24 16:13:08 -05002444
2445 bio_get(bio);
2446
Chris Mason065631f2008-02-20 12:07:25 -05002447 if (tree->ops && tree->ops->submit_bio_hook)
liubo6b82ce82011-01-26 06:21:39 +00002448 ret = tree->ops->submit_bio_hook(page->mapping->host, rw, bio,
Chris Masoneaf25d92010-05-25 09:48:28 -04002449 mirror_num, bio_flags, start);
Chris Mason0b86a832008-03-24 15:01:56 -04002450 else
Stefan Behrens21adbd52011-11-09 13:44:05 +01002451 btrfsic_submit_bio(rw, bio);
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02002452
Chris Masond1310b22008-01-24 16:13:08 -05002453 if (bio_flagged(bio, BIO_EOPNOTSUPP))
2454 ret = -EOPNOTSUPP;
2455 bio_put(bio);
2456 return ret;
2457}
2458
Jeff Mahoney3444a972011-10-03 23:23:13 -04002459static int merge_bio(struct extent_io_tree *tree, struct page *page,
2460 unsigned long offset, size_t size, struct bio *bio,
2461 unsigned long bio_flags)
2462{
2463 int ret = 0;
2464 if (tree->ops && tree->ops->merge_bio_hook)
2465 ret = tree->ops->merge_bio_hook(page, offset, size, bio,
2466 bio_flags);
2467 BUG_ON(ret < 0);
2468 return ret;
2469
2470}
2471
Chris Masond1310b22008-01-24 16:13:08 -05002472static int submit_extent_page(int rw, struct extent_io_tree *tree,
2473 struct page *page, sector_t sector,
2474 size_t size, unsigned long offset,
2475 struct block_device *bdev,
2476 struct bio **bio_ret,
2477 unsigned long max_pages,
Chris Masonf1885912008-04-09 16:28:12 -04002478 bio_end_io_t end_io_func,
Chris Masonc8b97812008-10-29 14:49:59 -04002479 int mirror_num,
2480 unsigned long prev_bio_flags,
2481 unsigned long bio_flags)
Chris Masond1310b22008-01-24 16:13:08 -05002482{
2483 int ret = 0;
2484 struct bio *bio;
2485 int nr;
Chris Masonc8b97812008-10-29 14:49:59 -04002486 int contig = 0;
2487 int this_compressed = bio_flags & EXTENT_BIO_COMPRESSED;
2488 int old_compressed = prev_bio_flags & EXTENT_BIO_COMPRESSED;
Chris Mason5b050f02008-11-11 09:34:41 -05002489 size_t page_size = min_t(size_t, size, PAGE_CACHE_SIZE);
Chris Masond1310b22008-01-24 16:13:08 -05002490
2491 if (bio_ret && *bio_ret) {
2492 bio = *bio_ret;
Chris Masonc8b97812008-10-29 14:49:59 -04002493 if (old_compressed)
2494 contig = bio->bi_sector == sector;
2495 else
2496 contig = bio->bi_sector + (bio->bi_size >> 9) ==
2497 sector;
2498
2499 if (prev_bio_flags != bio_flags || !contig ||
Jeff Mahoney3444a972011-10-03 23:23:13 -04002500 merge_bio(tree, page, offset, page_size, bio, bio_flags) ||
Chris Masonc8b97812008-10-29 14:49:59 -04002501 bio_add_page(bio, page, page_size, offset) < page_size) {
2502 ret = submit_one_bio(rw, bio, mirror_num,
2503 prev_bio_flags);
Jeff Mahoney79787ea2012-03-12 16:03:00 +01002504 if (ret < 0)
2505 return ret;
Chris Masond1310b22008-01-24 16:13:08 -05002506 bio = NULL;
2507 } else {
2508 return 0;
2509 }
2510 }
Chris Masonc8b97812008-10-29 14:49:59 -04002511 if (this_compressed)
2512 nr = BIO_MAX_PAGES;
2513 else
2514 nr = bio_get_nr_vecs(bdev);
2515
Miao Xie88f794e2010-11-22 03:02:55 +00002516 bio = btrfs_bio_alloc(bdev, sector, nr, GFP_NOFS | __GFP_HIGH);
Tsutomu Itoh5df67082011-02-01 09:17:35 +00002517 if (!bio)
2518 return -ENOMEM;
Chris Mason70dec802008-01-29 09:59:12 -05002519
Chris Masonc8b97812008-10-29 14:49:59 -04002520 bio_add_page(bio, page, page_size, offset);
Chris Masond1310b22008-01-24 16:13:08 -05002521 bio->bi_end_io = end_io_func;
2522 bio->bi_private = tree;
Chris Mason70dec802008-01-29 09:59:12 -05002523
Chris Masond3977122009-01-05 21:25:51 -05002524 if (bio_ret)
Chris Masond1310b22008-01-24 16:13:08 -05002525 *bio_ret = bio;
Chris Masond3977122009-01-05 21:25:51 -05002526 else
Chris Masonc8b97812008-10-29 14:49:59 -04002527 ret = submit_one_bio(rw, bio, mirror_num, bio_flags);
Chris Masond1310b22008-01-24 16:13:08 -05002528
2529 return ret;
2530}
2531
Josef Bacik4f2de97a2012-03-07 16:20:05 -05002532void attach_extent_buffer_page(struct extent_buffer *eb, struct page *page)
2533{
2534 if (!PagePrivate(page)) {
2535 SetPagePrivate(page);
2536 page_cache_get(page);
2537 set_page_private(page, (unsigned long)eb);
2538 } else {
2539 WARN_ON(page->private != (unsigned long)eb);
2540 }
2541}
2542
Chris Masond1310b22008-01-24 16:13:08 -05002543void set_page_extent_mapped(struct page *page)
2544{
2545 if (!PagePrivate(page)) {
2546 SetPagePrivate(page);
Chris Masond1310b22008-01-24 16:13:08 -05002547 page_cache_get(page);
Chris Mason6af118ce2008-07-22 11:18:07 -04002548 set_page_private(page, EXTENT_PAGE_PRIVATE);
Chris Masond1310b22008-01-24 16:13:08 -05002549 }
2550}
2551
Chris Masond1310b22008-01-24 16:13:08 -05002552/*
2553 * basic readpage implementation. Locked extent state structs are inserted
2554 * into the tree that are removed when the IO is done (by the end_io
2555 * handlers)
Jeff Mahoney79787ea2012-03-12 16:03:00 +01002556 * XXX JDM: This needs looking at to ensure proper page locking
Chris Masond1310b22008-01-24 16:13:08 -05002557 */
2558static int __extent_read_full_page(struct extent_io_tree *tree,
2559 struct page *page,
2560 get_extent_t *get_extent,
Chris Masonc8b97812008-10-29 14:49:59 -04002561 struct bio **bio, int mirror_num,
2562 unsigned long *bio_flags)
Chris Masond1310b22008-01-24 16:13:08 -05002563{
2564 struct inode *inode = page->mapping->host;
2565 u64 start = (u64)page->index << PAGE_CACHE_SHIFT;
2566 u64 page_end = start + PAGE_CACHE_SIZE - 1;
2567 u64 end;
2568 u64 cur = start;
2569 u64 extent_offset;
2570 u64 last_byte = i_size_read(inode);
2571 u64 block_start;
2572 u64 cur_end;
2573 sector_t sector;
2574 struct extent_map *em;
2575 struct block_device *bdev;
Josef Bacik11c65dc2010-05-23 11:07:21 -04002576 struct btrfs_ordered_extent *ordered;
Chris Masond1310b22008-01-24 16:13:08 -05002577 int ret;
2578 int nr = 0;
David Sterba306e16c2011-04-19 14:29:38 +02002579 size_t pg_offset = 0;
Chris Masond1310b22008-01-24 16:13:08 -05002580 size_t iosize;
Chris Masonc8b97812008-10-29 14:49:59 -04002581 size_t disk_io_size;
Chris Masond1310b22008-01-24 16:13:08 -05002582 size_t blocksize = inode->i_sb->s_blocksize;
Chris Masonc8b97812008-10-29 14:49:59 -04002583 unsigned long this_bio_flag = 0;
Chris Masond1310b22008-01-24 16:13:08 -05002584
2585 set_page_extent_mapped(page);
2586
Dan Magenheimer90a887c2011-05-26 10:01:56 -06002587 if (!PageUptodate(page)) {
2588 if (cleancache_get_page(page) == 0) {
2589 BUG_ON(blocksize != PAGE_SIZE);
2590 goto out;
2591 }
2592 }
2593
Chris Masond1310b22008-01-24 16:13:08 -05002594 end = page_end;
Josef Bacik11c65dc2010-05-23 11:07:21 -04002595 while (1) {
Jeff Mahoneyd0082372012-03-01 14:57:19 +01002596 lock_extent(tree, start, end);
Josef Bacik11c65dc2010-05-23 11:07:21 -04002597 ordered = btrfs_lookup_ordered_extent(inode, start);
2598 if (!ordered)
2599 break;
Jeff Mahoneyd0082372012-03-01 14:57:19 +01002600 unlock_extent(tree, start, end);
Josef Bacik11c65dc2010-05-23 11:07:21 -04002601 btrfs_start_ordered_extent(inode, ordered, 1);
2602 btrfs_put_ordered_extent(ordered);
2603 }
Chris Masond1310b22008-01-24 16:13:08 -05002604
Chris Masonc8b97812008-10-29 14:49:59 -04002605 if (page->index == last_byte >> PAGE_CACHE_SHIFT) {
2606 char *userpage;
2607 size_t zero_offset = last_byte & (PAGE_CACHE_SIZE - 1);
2608
2609 if (zero_offset) {
2610 iosize = PAGE_CACHE_SIZE - zero_offset;
Cong Wang7ac687d2011-11-25 23:14:28 +08002611 userpage = kmap_atomic(page);
Chris Masonc8b97812008-10-29 14:49:59 -04002612 memset(userpage + zero_offset, 0, iosize);
2613 flush_dcache_page(page);
Cong Wang7ac687d2011-11-25 23:14:28 +08002614 kunmap_atomic(userpage);
Chris Masonc8b97812008-10-29 14:49:59 -04002615 }
2616 }
Chris Masond1310b22008-01-24 16:13:08 -05002617 while (cur <= end) {
2618 if (cur >= last_byte) {
2619 char *userpage;
Arne Jansen507903b2011-04-06 10:02:20 +00002620 struct extent_state *cached = NULL;
2621
David Sterba306e16c2011-04-19 14:29:38 +02002622 iosize = PAGE_CACHE_SIZE - pg_offset;
Cong Wang7ac687d2011-11-25 23:14:28 +08002623 userpage = kmap_atomic(page);
David Sterba306e16c2011-04-19 14:29:38 +02002624 memset(userpage + pg_offset, 0, iosize);
Chris Masond1310b22008-01-24 16:13:08 -05002625 flush_dcache_page(page);
Cong Wang7ac687d2011-11-25 23:14:28 +08002626 kunmap_atomic(userpage);
Chris Masond1310b22008-01-24 16:13:08 -05002627 set_extent_uptodate(tree, cur, cur + iosize - 1,
Arne Jansen507903b2011-04-06 10:02:20 +00002628 &cached, GFP_NOFS);
2629 unlock_extent_cached(tree, cur, cur + iosize - 1,
2630 &cached, GFP_NOFS);
Chris Masond1310b22008-01-24 16:13:08 -05002631 break;
2632 }
David Sterba306e16c2011-04-19 14:29:38 +02002633 em = get_extent(inode, page, pg_offset, cur,
Chris Masond1310b22008-01-24 16:13:08 -05002634 end - cur + 1, 0);
David Sterbac7040052011-04-19 18:00:01 +02002635 if (IS_ERR_OR_NULL(em)) {
Chris Masond1310b22008-01-24 16:13:08 -05002636 SetPageError(page);
Jeff Mahoneyd0082372012-03-01 14:57:19 +01002637 unlock_extent(tree, cur, end);
Chris Masond1310b22008-01-24 16:13:08 -05002638 break;
2639 }
Chris Masond1310b22008-01-24 16:13:08 -05002640 extent_offset = cur - em->start;
2641 BUG_ON(extent_map_end(em) <= cur);
2642 BUG_ON(end < cur);
2643
Li Zefan261507a02010-12-17 14:21:50 +08002644 if (test_bit(EXTENT_FLAG_COMPRESSED, &em->flags)) {
Chris Masonc8b97812008-10-29 14:49:59 -04002645 this_bio_flag = EXTENT_BIO_COMPRESSED;
Li Zefan261507a02010-12-17 14:21:50 +08002646 extent_set_compress_type(&this_bio_flag,
2647 em->compress_type);
2648 }
Chris Masonc8b97812008-10-29 14:49:59 -04002649
Chris Masond1310b22008-01-24 16:13:08 -05002650 iosize = min(extent_map_end(em) - cur, end - cur + 1);
2651 cur_end = min(extent_map_end(em) - 1, end);
2652 iosize = (iosize + blocksize - 1) & ~((u64)blocksize - 1);
Chris Masonc8b97812008-10-29 14:49:59 -04002653 if (this_bio_flag & EXTENT_BIO_COMPRESSED) {
2654 disk_io_size = em->block_len;
2655 sector = em->block_start >> 9;
2656 } else {
2657 sector = (em->block_start + extent_offset) >> 9;
2658 disk_io_size = iosize;
2659 }
Chris Masond1310b22008-01-24 16:13:08 -05002660 bdev = em->bdev;
2661 block_start = em->block_start;
Yan Zhengd899e052008-10-30 14:25:28 -04002662 if (test_bit(EXTENT_FLAG_PREALLOC, &em->flags))
2663 block_start = EXTENT_MAP_HOLE;
Chris Masond1310b22008-01-24 16:13:08 -05002664 free_extent_map(em);
2665 em = NULL;
2666
2667 /* we've found a hole, just zero and go on */
2668 if (block_start == EXTENT_MAP_HOLE) {
2669 char *userpage;
Arne Jansen507903b2011-04-06 10:02:20 +00002670 struct extent_state *cached = NULL;
2671
Cong Wang7ac687d2011-11-25 23:14:28 +08002672 userpage = kmap_atomic(page);
David Sterba306e16c2011-04-19 14:29:38 +02002673 memset(userpage + pg_offset, 0, iosize);
Chris Masond1310b22008-01-24 16:13:08 -05002674 flush_dcache_page(page);
Cong Wang7ac687d2011-11-25 23:14:28 +08002675 kunmap_atomic(userpage);
Chris Masond1310b22008-01-24 16:13:08 -05002676
2677 set_extent_uptodate(tree, cur, cur + iosize - 1,
Arne Jansen507903b2011-04-06 10:02:20 +00002678 &cached, GFP_NOFS);
2679 unlock_extent_cached(tree, cur, cur + iosize - 1,
2680 &cached, GFP_NOFS);
Chris Masond1310b22008-01-24 16:13:08 -05002681 cur = cur + iosize;
David Sterba306e16c2011-04-19 14:29:38 +02002682 pg_offset += iosize;
Chris Masond1310b22008-01-24 16:13:08 -05002683 continue;
2684 }
2685 /* the get_extent function already copied into the page */
Chris Mason9655d292009-09-02 15:22:30 -04002686 if (test_range_bit(tree, cur, cur_end,
2687 EXTENT_UPTODATE, 1, NULL)) {
Chris Masona1b32a52008-09-05 16:09:51 -04002688 check_page_uptodate(tree, page);
Jeff Mahoneyd0082372012-03-01 14:57:19 +01002689 unlock_extent(tree, cur, cur + iosize - 1);
Chris Masond1310b22008-01-24 16:13:08 -05002690 cur = cur + iosize;
David Sterba306e16c2011-04-19 14:29:38 +02002691 pg_offset += iosize;
Chris Masond1310b22008-01-24 16:13:08 -05002692 continue;
2693 }
Chris Mason70dec802008-01-29 09:59:12 -05002694 /* we have an inline extent but it didn't get marked up
2695 * to date. Error out
2696 */
2697 if (block_start == EXTENT_MAP_INLINE) {
2698 SetPageError(page);
Jeff Mahoneyd0082372012-03-01 14:57:19 +01002699 unlock_extent(tree, cur, cur + iosize - 1);
Chris Mason70dec802008-01-29 09:59:12 -05002700 cur = cur + iosize;
David Sterba306e16c2011-04-19 14:29:38 +02002701 pg_offset += iosize;
Chris Mason70dec802008-01-29 09:59:12 -05002702 continue;
2703 }
Chris Masond1310b22008-01-24 16:13:08 -05002704
2705 ret = 0;
2706 if (tree->ops && tree->ops->readpage_io_hook) {
2707 ret = tree->ops->readpage_io_hook(page, cur,
2708 cur + iosize - 1);
2709 }
2710 if (!ret) {
Chris Mason89642222008-07-24 09:41:53 -04002711 unsigned long pnr = (last_byte >> PAGE_CACHE_SHIFT) + 1;
2712 pnr -= page->index;
Chris Masond1310b22008-01-24 16:13:08 -05002713 ret = submit_extent_page(READ, tree, page,
David Sterba306e16c2011-04-19 14:29:38 +02002714 sector, disk_io_size, pg_offset,
Chris Mason89642222008-07-24 09:41:53 -04002715 bdev, bio, pnr,
Chris Masonc8b97812008-10-29 14:49:59 -04002716 end_bio_extent_readpage, mirror_num,
2717 *bio_flags,
2718 this_bio_flag);
Jeff Mahoney79787ea2012-03-12 16:03:00 +01002719 BUG_ON(ret == -ENOMEM);
Chris Mason89642222008-07-24 09:41:53 -04002720 nr++;
Chris Masonc8b97812008-10-29 14:49:59 -04002721 *bio_flags = this_bio_flag;
Chris Masond1310b22008-01-24 16:13:08 -05002722 }
2723 if (ret)
2724 SetPageError(page);
2725 cur = cur + iosize;
David Sterba306e16c2011-04-19 14:29:38 +02002726 pg_offset += iosize;
Chris Masond1310b22008-01-24 16:13:08 -05002727 }
Dan Magenheimer90a887c2011-05-26 10:01:56 -06002728out:
Chris Masond1310b22008-01-24 16:13:08 -05002729 if (!nr) {
2730 if (!PageError(page))
2731 SetPageUptodate(page);
2732 unlock_page(page);
2733 }
2734 return 0;
2735}
2736
2737int extent_read_full_page(struct extent_io_tree *tree, struct page *page,
Jan Schmidt8ddc7d92011-06-13 20:02:58 +02002738 get_extent_t *get_extent, int mirror_num)
Chris Masond1310b22008-01-24 16:13:08 -05002739{
2740 struct bio *bio = NULL;
Chris Masonc8b97812008-10-29 14:49:59 -04002741 unsigned long bio_flags = 0;
Chris Masond1310b22008-01-24 16:13:08 -05002742 int ret;
2743
Jan Schmidt8ddc7d92011-06-13 20:02:58 +02002744 ret = __extent_read_full_page(tree, page, get_extent, &bio, mirror_num,
Chris Masonc8b97812008-10-29 14:49:59 -04002745 &bio_flags);
Chris Masond1310b22008-01-24 16:13:08 -05002746 if (bio)
Jan Schmidt8ddc7d92011-06-13 20:02:58 +02002747 ret = submit_one_bio(READ, bio, mirror_num, bio_flags);
Chris Masond1310b22008-01-24 16:13:08 -05002748 return ret;
2749}
Chris Masond1310b22008-01-24 16:13:08 -05002750
Chris Mason11c83492009-04-20 15:50:09 -04002751static noinline void update_nr_written(struct page *page,
2752 struct writeback_control *wbc,
2753 unsigned long nr_written)
2754{
2755 wbc->nr_to_write -= nr_written;
2756 if (wbc->range_cyclic || (wbc->nr_to_write > 0 &&
2757 wbc->range_start == 0 && wbc->range_end == LLONG_MAX))
2758 page->mapping->writeback_index = page->index + nr_written;
2759}
2760
Chris Masond1310b22008-01-24 16:13:08 -05002761/*
2762 * the writepage semantics are similar to regular writepage. extent
2763 * records are inserted to lock ranges in the tree, and as dirty areas
2764 * are found, they are marked writeback. Then the lock bits are removed
2765 * and the end_io handler clears the writeback ranges
2766 */
2767static int __extent_writepage(struct page *page, struct writeback_control *wbc,
2768 void *data)
2769{
2770 struct inode *inode = page->mapping->host;
2771 struct extent_page_data *epd = data;
2772 struct extent_io_tree *tree = epd->tree;
2773 u64 start = (u64)page->index << PAGE_CACHE_SHIFT;
2774 u64 delalloc_start;
2775 u64 page_end = start + PAGE_CACHE_SIZE - 1;
2776 u64 end;
2777 u64 cur = start;
2778 u64 extent_offset;
2779 u64 last_byte = i_size_read(inode);
2780 u64 block_start;
2781 u64 iosize;
2782 sector_t sector;
Chris Mason2c64c532009-09-02 15:04:12 -04002783 struct extent_state *cached_state = NULL;
Chris Masond1310b22008-01-24 16:13:08 -05002784 struct extent_map *em;
2785 struct block_device *bdev;
2786 int ret;
2787 int nr = 0;
Chris Mason7f3c74f2008-07-18 12:01:11 -04002788 size_t pg_offset = 0;
Chris Masond1310b22008-01-24 16:13:08 -05002789 size_t blocksize;
2790 loff_t i_size = i_size_read(inode);
2791 unsigned long end_index = i_size >> PAGE_CACHE_SHIFT;
2792 u64 nr_delalloc;
2793 u64 delalloc_end;
Chris Masonc8b97812008-10-29 14:49:59 -04002794 int page_started;
2795 int compressed;
Chris Masonffbd5172009-04-20 15:50:09 -04002796 int write_flags;
Chris Mason771ed682008-11-06 22:02:51 -05002797 unsigned long nr_written = 0;
Josef Bacik9e487102011-08-01 12:08:18 -04002798 bool fill_delalloc = true;
Chris Masond1310b22008-01-24 16:13:08 -05002799
Chris Masonffbd5172009-04-20 15:50:09 -04002800 if (wbc->sync_mode == WB_SYNC_ALL)
Jens Axboe721a9602011-03-09 11:56:30 +01002801 write_flags = WRITE_SYNC;
Chris Masonffbd5172009-04-20 15:50:09 -04002802 else
2803 write_flags = WRITE;
2804
liubo1abe9b82011-03-24 11:18:59 +00002805 trace___extent_writepage(page, inode, wbc);
2806
Chris Masond1310b22008-01-24 16:13:08 -05002807 WARN_ON(!PageLocked(page));
Chris Masonbf0da8c2011-11-04 12:29:37 -04002808
2809 ClearPageError(page);
2810
Chris Mason7f3c74f2008-07-18 12:01:11 -04002811 pg_offset = i_size & (PAGE_CACHE_SIZE - 1);
Chris Mason211c17f2008-05-15 09:13:45 -04002812 if (page->index > end_index ||
Chris Mason7f3c74f2008-07-18 12:01:11 -04002813 (page->index == end_index && !pg_offset)) {
Chris Mason39be25c2008-11-10 11:50:50 -05002814 page->mapping->a_ops->invalidatepage(page, 0);
Chris Masond1310b22008-01-24 16:13:08 -05002815 unlock_page(page);
2816 return 0;
2817 }
2818
2819 if (page->index == end_index) {
2820 char *userpage;
2821
Cong Wang7ac687d2011-11-25 23:14:28 +08002822 userpage = kmap_atomic(page);
Chris Mason7f3c74f2008-07-18 12:01:11 -04002823 memset(userpage + pg_offset, 0,
2824 PAGE_CACHE_SIZE - pg_offset);
Cong Wang7ac687d2011-11-25 23:14:28 +08002825 kunmap_atomic(userpage);
Chris Mason211c17f2008-05-15 09:13:45 -04002826 flush_dcache_page(page);
Chris Masond1310b22008-01-24 16:13:08 -05002827 }
Chris Mason7f3c74f2008-07-18 12:01:11 -04002828 pg_offset = 0;
Chris Masond1310b22008-01-24 16:13:08 -05002829
2830 set_page_extent_mapped(page);
2831
Josef Bacik9e487102011-08-01 12:08:18 -04002832 if (!tree->ops || !tree->ops->fill_delalloc)
2833 fill_delalloc = false;
2834
Chris Masond1310b22008-01-24 16:13:08 -05002835 delalloc_start = start;
2836 delalloc_end = 0;
Chris Masonc8b97812008-10-29 14:49:59 -04002837 page_started = 0;
Josef Bacik9e487102011-08-01 12:08:18 -04002838 if (!epd->extent_locked && fill_delalloc) {
Chris Masonf85d7d6c2009-09-18 16:03:16 -04002839 u64 delalloc_to_write = 0;
Chris Mason11c83492009-04-20 15:50:09 -04002840 /*
2841 * make sure the wbc mapping index is at least updated
2842 * to this page.
2843 */
2844 update_nr_written(page, wbc, 0);
2845
Chris Masond3977122009-01-05 21:25:51 -05002846 while (delalloc_end < page_end) {
Chris Mason771ed682008-11-06 22:02:51 -05002847 nr_delalloc = find_lock_delalloc_range(inode, tree,
Chris Masonc8b97812008-10-29 14:49:59 -04002848 page,
2849 &delalloc_start,
Chris Masond1310b22008-01-24 16:13:08 -05002850 &delalloc_end,
2851 128 * 1024 * 1024);
Chris Mason771ed682008-11-06 22:02:51 -05002852 if (nr_delalloc == 0) {
2853 delalloc_start = delalloc_end + 1;
2854 continue;
2855 }
Tsutomu Itoh013bd4c2012-02-16 10:11:40 +09002856 ret = tree->ops->fill_delalloc(inode, page,
2857 delalloc_start,
2858 delalloc_end,
2859 &page_started,
2860 &nr_written);
Jeff Mahoney79787ea2012-03-12 16:03:00 +01002861 /* File system has been set read-only */
2862 if (ret) {
2863 SetPageError(page);
2864 goto done;
2865 }
Chris Masonf85d7d6c2009-09-18 16:03:16 -04002866 /*
2867 * delalloc_end is already one less than the total
2868 * length, so we don't subtract one from
2869 * PAGE_CACHE_SIZE
2870 */
2871 delalloc_to_write += (delalloc_end - delalloc_start +
2872 PAGE_CACHE_SIZE) >>
2873 PAGE_CACHE_SHIFT;
Chris Masond1310b22008-01-24 16:13:08 -05002874 delalloc_start = delalloc_end + 1;
Chris Masond1310b22008-01-24 16:13:08 -05002875 }
Chris Masonf85d7d6c2009-09-18 16:03:16 -04002876 if (wbc->nr_to_write < delalloc_to_write) {
2877 int thresh = 8192;
2878
2879 if (delalloc_to_write < thresh * 2)
2880 thresh = delalloc_to_write;
2881 wbc->nr_to_write = min_t(u64, delalloc_to_write,
2882 thresh);
2883 }
Chris Masonc8b97812008-10-29 14:49:59 -04002884
Chris Mason771ed682008-11-06 22:02:51 -05002885 /* did the fill delalloc function already unlock and start
2886 * the IO?
2887 */
2888 if (page_started) {
2889 ret = 0;
Chris Mason11c83492009-04-20 15:50:09 -04002890 /*
2891 * we've unlocked the page, so we can't update
2892 * the mapping's writeback index, just update
2893 * nr_to_write.
2894 */
2895 wbc->nr_to_write -= nr_written;
2896 goto done_unlocked;
Chris Mason771ed682008-11-06 22:02:51 -05002897 }
Chris Masonc8b97812008-10-29 14:49:59 -04002898 }
Chris Mason247e7432008-07-17 12:53:51 -04002899 if (tree->ops && tree->ops->writepage_start_hook) {
Chris Masonc8b97812008-10-29 14:49:59 -04002900 ret = tree->ops->writepage_start_hook(page, start,
2901 page_end);
Jeff Mahoney87826df2012-02-15 16:23:57 +01002902 if (ret) {
2903 /* Fixup worker will requeue */
2904 if (ret == -EBUSY)
2905 wbc->pages_skipped++;
2906 else
2907 redirty_page_for_writepage(wbc, page);
Chris Mason11c83492009-04-20 15:50:09 -04002908 update_nr_written(page, wbc, nr_written);
Chris Mason247e7432008-07-17 12:53:51 -04002909 unlock_page(page);
Chris Mason771ed682008-11-06 22:02:51 -05002910 ret = 0;
Chris Mason11c83492009-04-20 15:50:09 -04002911 goto done_unlocked;
Chris Mason247e7432008-07-17 12:53:51 -04002912 }
2913 }
2914
Chris Mason11c83492009-04-20 15:50:09 -04002915 /*
2916 * we don't want to touch the inode after unlocking the page,
2917 * so we update the mapping writeback index now
2918 */
2919 update_nr_written(page, wbc, nr_written + 1);
Chris Mason771ed682008-11-06 22:02:51 -05002920
Chris Masond1310b22008-01-24 16:13:08 -05002921 end = page_end;
Chris Masond1310b22008-01-24 16:13:08 -05002922 if (last_byte <= start) {
Chris Masone6dcd2d2008-07-17 12:53:50 -04002923 if (tree->ops && tree->ops->writepage_end_io_hook)
2924 tree->ops->writepage_end_io_hook(page, start,
2925 page_end, NULL, 1);
Chris Masond1310b22008-01-24 16:13:08 -05002926 goto done;
2927 }
2928
Chris Masond1310b22008-01-24 16:13:08 -05002929 blocksize = inode->i_sb->s_blocksize;
2930
2931 while (cur <= end) {
2932 if (cur >= last_byte) {
Chris Masone6dcd2d2008-07-17 12:53:50 -04002933 if (tree->ops && tree->ops->writepage_end_io_hook)
2934 tree->ops->writepage_end_io_hook(page, cur,
2935 page_end, NULL, 1);
Chris Masond1310b22008-01-24 16:13:08 -05002936 break;
2937 }
Chris Mason7f3c74f2008-07-18 12:01:11 -04002938 em = epd->get_extent(inode, page, pg_offset, cur,
Chris Masond1310b22008-01-24 16:13:08 -05002939 end - cur + 1, 1);
David Sterbac7040052011-04-19 18:00:01 +02002940 if (IS_ERR_OR_NULL(em)) {
Chris Masond1310b22008-01-24 16:13:08 -05002941 SetPageError(page);
2942 break;
2943 }
2944
2945 extent_offset = cur - em->start;
2946 BUG_ON(extent_map_end(em) <= cur);
2947 BUG_ON(end < cur);
2948 iosize = min(extent_map_end(em) - cur, end - cur + 1);
2949 iosize = (iosize + blocksize - 1) & ~((u64)blocksize - 1);
2950 sector = (em->block_start + extent_offset) >> 9;
2951 bdev = em->bdev;
2952 block_start = em->block_start;
Chris Masonc8b97812008-10-29 14:49:59 -04002953 compressed = test_bit(EXTENT_FLAG_COMPRESSED, &em->flags);
Chris Masond1310b22008-01-24 16:13:08 -05002954 free_extent_map(em);
2955 em = NULL;
2956
Chris Masonc8b97812008-10-29 14:49:59 -04002957 /*
2958 * compressed and inline extents are written through other
2959 * paths in the FS
2960 */
2961 if (compressed || block_start == EXTENT_MAP_HOLE ||
Chris Masond1310b22008-01-24 16:13:08 -05002962 block_start == EXTENT_MAP_INLINE) {
Chris Masonc8b97812008-10-29 14:49:59 -04002963 /*
2964 * end_io notification does not happen here for
2965 * compressed extents
2966 */
2967 if (!compressed && tree->ops &&
2968 tree->ops->writepage_end_io_hook)
Chris Masone6dcd2d2008-07-17 12:53:50 -04002969 tree->ops->writepage_end_io_hook(page, cur,
2970 cur + iosize - 1,
2971 NULL, 1);
Chris Masonc8b97812008-10-29 14:49:59 -04002972 else if (compressed) {
2973 /* we don't want to end_page_writeback on
2974 * a compressed extent. this happens
2975 * elsewhere
2976 */
2977 nr++;
2978 }
2979
2980 cur += iosize;
Chris Mason7f3c74f2008-07-18 12:01:11 -04002981 pg_offset += iosize;
Chris Masond1310b22008-01-24 16:13:08 -05002982 continue;
2983 }
Chris Masond1310b22008-01-24 16:13:08 -05002984 /* leave this out until we have a page_mkwrite call */
2985 if (0 && !test_range_bit(tree, cur, cur + iosize - 1,
Chris Mason9655d292009-09-02 15:22:30 -04002986 EXTENT_DIRTY, 0, NULL)) {
Chris Masond1310b22008-01-24 16:13:08 -05002987 cur = cur + iosize;
Chris Mason7f3c74f2008-07-18 12:01:11 -04002988 pg_offset += iosize;
Chris Masond1310b22008-01-24 16:13:08 -05002989 continue;
2990 }
Chris Masonc8b97812008-10-29 14:49:59 -04002991
Chris Masond1310b22008-01-24 16:13:08 -05002992 if (tree->ops && tree->ops->writepage_io_hook) {
2993 ret = tree->ops->writepage_io_hook(page, cur,
2994 cur + iosize - 1);
2995 } else {
2996 ret = 0;
2997 }
Chris Mason1259ab72008-05-12 13:39:03 -04002998 if (ret) {
Chris Masond1310b22008-01-24 16:13:08 -05002999 SetPageError(page);
Chris Mason1259ab72008-05-12 13:39:03 -04003000 } else {
Chris Masond1310b22008-01-24 16:13:08 -05003001 unsigned long max_nr = end_index + 1;
Chris Mason7f3c74f2008-07-18 12:01:11 -04003002
Chris Masond1310b22008-01-24 16:13:08 -05003003 set_range_writeback(tree, cur, cur + iosize - 1);
3004 if (!PageWriteback(page)) {
Chris Masond3977122009-01-05 21:25:51 -05003005 printk(KERN_ERR "btrfs warning page %lu not "
3006 "writeback, cur %llu end %llu\n",
3007 page->index, (unsigned long long)cur,
Chris Masond1310b22008-01-24 16:13:08 -05003008 (unsigned long long)end);
3009 }
3010
Chris Masonffbd5172009-04-20 15:50:09 -04003011 ret = submit_extent_page(write_flags, tree, page,
3012 sector, iosize, pg_offset,
3013 bdev, &epd->bio, max_nr,
Chris Masonc8b97812008-10-29 14:49:59 -04003014 end_bio_extent_writepage,
3015 0, 0, 0);
Chris Masond1310b22008-01-24 16:13:08 -05003016 if (ret)
3017 SetPageError(page);
3018 }
3019 cur = cur + iosize;
Chris Mason7f3c74f2008-07-18 12:01:11 -04003020 pg_offset += iosize;
Chris Masond1310b22008-01-24 16:13:08 -05003021 nr++;
3022 }
3023done:
3024 if (nr == 0) {
3025 /* make sure the mapping tag for page dirty gets cleared */
3026 set_page_writeback(page);
3027 end_page_writeback(page);
3028 }
Chris Masond1310b22008-01-24 16:13:08 -05003029 unlock_page(page);
Chris Mason771ed682008-11-06 22:02:51 -05003030
Chris Mason11c83492009-04-20 15:50:09 -04003031done_unlocked:
3032
Chris Mason2c64c532009-09-02 15:04:12 -04003033 /* drop our reference on any cached states */
3034 free_extent_state(cached_state);
Chris Masond1310b22008-01-24 16:13:08 -05003035 return 0;
3036}
3037
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003038static int eb_wait(void *word)
3039{
3040 io_schedule();
3041 return 0;
3042}
3043
3044static void wait_on_extent_buffer_writeback(struct extent_buffer *eb)
3045{
3046 wait_on_bit(&eb->bflags, EXTENT_BUFFER_WRITEBACK, eb_wait,
3047 TASK_UNINTERRUPTIBLE);
3048}
3049
3050static int lock_extent_buffer_for_io(struct extent_buffer *eb,
3051 struct btrfs_fs_info *fs_info,
3052 struct extent_page_data *epd)
3053{
3054 unsigned long i, num_pages;
3055 int flush = 0;
3056 int ret = 0;
3057
3058 if (!btrfs_try_tree_write_lock(eb)) {
3059 flush = 1;
3060 flush_write_bio(epd);
3061 btrfs_tree_lock(eb);
3062 }
3063
3064 if (test_bit(EXTENT_BUFFER_WRITEBACK, &eb->bflags)) {
3065 btrfs_tree_unlock(eb);
3066 if (!epd->sync_io)
3067 return 0;
3068 if (!flush) {
3069 flush_write_bio(epd);
3070 flush = 1;
3071 }
Chris Masona098d8e2012-03-21 12:09:56 -04003072 while (1) {
3073 wait_on_extent_buffer_writeback(eb);
3074 btrfs_tree_lock(eb);
3075 if (!test_bit(EXTENT_BUFFER_WRITEBACK, &eb->bflags))
3076 break;
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003077 btrfs_tree_unlock(eb);
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003078 }
3079 }
3080
3081 if (test_and_clear_bit(EXTENT_BUFFER_DIRTY, &eb->bflags)) {
3082 set_bit(EXTENT_BUFFER_WRITEBACK, &eb->bflags);
3083 btrfs_set_header_flag(eb, BTRFS_HEADER_FLAG_WRITTEN);
3084 spin_lock(&fs_info->delalloc_lock);
3085 if (fs_info->dirty_metadata_bytes >= eb->len)
3086 fs_info->dirty_metadata_bytes -= eb->len;
3087 else
3088 WARN_ON(1);
3089 spin_unlock(&fs_info->delalloc_lock);
3090 ret = 1;
3091 }
3092
3093 btrfs_tree_unlock(eb);
3094
3095 if (!ret)
3096 return ret;
3097
3098 num_pages = num_extent_pages(eb->start, eb->len);
3099 for (i = 0; i < num_pages; i++) {
3100 struct page *p = extent_buffer_page(eb, i);
3101
3102 if (!trylock_page(p)) {
3103 if (!flush) {
3104 flush_write_bio(epd);
3105 flush = 1;
3106 }
3107 lock_page(p);
3108 }
3109 }
3110
3111 return ret;
3112}
3113
3114static void end_extent_buffer_writeback(struct extent_buffer *eb)
3115{
3116 clear_bit(EXTENT_BUFFER_WRITEBACK, &eb->bflags);
3117 smp_mb__after_clear_bit();
3118 wake_up_bit(&eb->bflags, EXTENT_BUFFER_WRITEBACK);
3119}
3120
3121static void end_bio_extent_buffer_writepage(struct bio *bio, int err)
3122{
3123 int uptodate = err == 0;
3124 struct bio_vec *bvec = bio->bi_io_vec + bio->bi_vcnt - 1;
3125 struct extent_buffer *eb;
3126 int done;
3127
3128 do {
3129 struct page *page = bvec->bv_page;
3130
3131 bvec--;
3132 eb = (struct extent_buffer *)page->private;
3133 BUG_ON(!eb);
3134 done = atomic_dec_and_test(&eb->io_pages);
3135
3136 if (!uptodate || test_bit(EXTENT_BUFFER_IOERR, &eb->bflags)) {
3137 set_bit(EXTENT_BUFFER_IOERR, &eb->bflags);
3138 ClearPageUptodate(page);
3139 SetPageError(page);
3140 }
3141
3142 end_page_writeback(page);
3143
3144 if (!done)
3145 continue;
3146
3147 end_extent_buffer_writeback(eb);
3148 } while (bvec >= bio->bi_io_vec);
3149
3150 bio_put(bio);
3151
3152}
3153
3154static int write_one_eb(struct extent_buffer *eb,
3155 struct btrfs_fs_info *fs_info,
3156 struct writeback_control *wbc,
3157 struct extent_page_data *epd)
3158{
3159 struct block_device *bdev = fs_info->fs_devices->latest_bdev;
3160 u64 offset = eb->start;
3161 unsigned long i, num_pages;
3162 int rw = (epd->sync_io ? WRITE_SYNC : WRITE);
3163 int ret;
3164
3165 clear_bit(EXTENT_BUFFER_IOERR, &eb->bflags);
3166 num_pages = num_extent_pages(eb->start, eb->len);
3167 atomic_set(&eb->io_pages, num_pages);
3168 for (i = 0; i < num_pages; i++) {
3169 struct page *p = extent_buffer_page(eb, i);
3170
3171 clear_page_dirty_for_io(p);
3172 set_page_writeback(p);
3173 ret = submit_extent_page(rw, eb->tree, p, offset >> 9,
3174 PAGE_CACHE_SIZE, 0, bdev, &epd->bio,
3175 -1, end_bio_extent_buffer_writepage,
3176 0, 0, 0);
3177 if (ret) {
3178 set_bit(EXTENT_BUFFER_IOERR, &eb->bflags);
3179 SetPageError(p);
3180 if (atomic_sub_and_test(num_pages - i, &eb->io_pages))
3181 end_extent_buffer_writeback(eb);
3182 ret = -EIO;
3183 break;
3184 }
3185 offset += PAGE_CACHE_SIZE;
3186 update_nr_written(p, wbc, 1);
3187 unlock_page(p);
3188 }
3189
3190 if (unlikely(ret)) {
3191 for (; i < num_pages; i++) {
3192 struct page *p = extent_buffer_page(eb, i);
3193 unlock_page(p);
3194 }
3195 }
3196
3197 return ret;
3198}
3199
3200int btree_write_cache_pages(struct address_space *mapping,
3201 struct writeback_control *wbc)
3202{
3203 struct extent_io_tree *tree = &BTRFS_I(mapping->host)->io_tree;
3204 struct btrfs_fs_info *fs_info = BTRFS_I(mapping->host)->root->fs_info;
3205 struct extent_buffer *eb, *prev_eb = NULL;
3206 struct extent_page_data epd = {
3207 .bio = NULL,
3208 .tree = tree,
3209 .extent_locked = 0,
3210 .sync_io = wbc->sync_mode == WB_SYNC_ALL,
3211 };
3212 int ret = 0;
3213 int done = 0;
3214 int nr_to_write_done = 0;
3215 struct pagevec pvec;
3216 int nr_pages;
3217 pgoff_t index;
3218 pgoff_t end; /* Inclusive */
3219 int scanned = 0;
3220 int tag;
3221
3222 pagevec_init(&pvec, 0);
3223 if (wbc->range_cyclic) {
3224 index = mapping->writeback_index; /* Start from prev offset */
3225 end = -1;
3226 } else {
3227 index = wbc->range_start >> PAGE_CACHE_SHIFT;
3228 end = wbc->range_end >> PAGE_CACHE_SHIFT;
3229 scanned = 1;
3230 }
3231 if (wbc->sync_mode == WB_SYNC_ALL)
3232 tag = PAGECACHE_TAG_TOWRITE;
3233 else
3234 tag = PAGECACHE_TAG_DIRTY;
3235retry:
3236 if (wbc->sync_mode == WB_SYNC_ALL)
3237 tag_pages_for_writeback(mapping, index, end);
3238 while (!done && !nr_to_write_done && (index <= end) &&
3239 (nr_pages = pagevec_lookup_tag(&pvec, mapping, &index, tag,
3240 min(end - index, (pgoff_t)PAGEVEC_SIZE-1) + 1))) {
3241 unsigned i;
3242
3243 scanned = 1;
3244 for (i = 0; i < nr_pages; i++) {
3245 struct page *page = pvec.pages[i];
3246
3247 if (!PagePrivate(page))
3248 continue;
3249
3250 if (!wbc->range_cyclic && page->index > end) {
3251 done = 1;
3252 break;
3253 }
3254
3255 eb = (struct extent_buffer *)page->private;
3256 if (!eb) {
3257 WARN_ON(1);
3258 continue;
3259 }
3260
3261 if (eb == prev_eb)
3262 continue;
3263
3264 if (!atomic_inc_not_zero(&eb->refs)) {
3265 WARN_ON(1);
3266 continue;
3267 }
3268
3269 prev_eb = eb;
3270 ret = lock_extent_buffer_for_io(eb, fs_info, &epd);
3271 if (!ret) {
3272 free_extent_buffer(eb);
3273 continue;
3274 }
3275
3276 ret = write_one_eb(eb, fs_info, wbc, &epd);
3277 if (ret) {
3278 done = 1;
3279 free_extent_buffer(eb);
3280 break;
3281 }
3282 free_extent_buffer(eb);
3283
3284 /*
3285 * the filesystem may choose to bump up nr_to_write.
3286 * We have to make sure to honor the new nr_to_write
3287 * at any time
3288 */
3289 nr_to_write_done = wbc->nr_to_write <= 0;
3290 }
3291 pagevec_release(&pvec);
3292 cond_resched();
3293 }
3294 if (!scanned && !done) {
3295 /*
3296 * We hit the last page and there is more work to be done: wrap
3297 * back to the start of the file
3298 */
3299 scanned = 1;
3300 index = 0;
3301 goto retry;
3302 }
3303 flush_write_bio(&epd);
3304 return ret;
3305}
3306
Chris Masond1310b22008-01-24 16:13:08 -05003307/**
Chris Mason4bef0842008-09-08 11:18:08 -04003308 * write_cache_pages - walk the list of dirty pages of the given address space and write all of them.
Chris Masond1310b22008-01-24 16:13:08 -05003309 * @mapping: address space structure to write
3310 * @wbc: subtract the number of written pages from *@wbc->nr_to_write
3311 * @writepage: function called for each page
3312 * @data: data passed to writepage function
3313 *
3314 * If a page is already under I/O, write_cache_pages() skips it, even
3315 * if it's dirty. This is desirable behaviour for memory-cleaning writeback,
3316 * but it is INCORRECT for data-integrity system calls such as fsync(). fsync()
3317 * and msync() need to guarantee that all the data which was dirty at the time
3318 * the call was made get new I/O started against them. If wbc->sync_mode is
3319 * WB_SYNC_ALL then we were called for data integrity and we must wait for
3320 * existing IO to complete.
3321 */
Christoph Hellwigb2950862008-12-02 09:54:17 -05003322static int extent_write_cache_pages(struct extent_io_tree *tree,
Chris Mason4bef0842008-09-08 11:18:08 -04003323 struct address_space *mapping,
3324 struct writeback_control *wbc,
Chris Masond2c3f4f2008-11-19 12:44:22 -05003325 writepage_t writepage, void *data,
3326 void (*flush_fn)(void *))
Chris Masond1310b22008-01-24 16:13:08 -05003327{
Chris Masond1310b22008-01-24 16:13:08 -05003328 int ret = 0;
3329 int done = 0;
Chris Masonf85d7d6c2009-09-18 16:03:16 -04003330 int nr_to_write_done = 0;
Chris Masond1310b22008-01-24 16:13:08 -05003331 struct pagevec pvec;
3332 int nr_pages;
3333 pgoff_t index;
3334 pgoff_t end; /* Inclusive */
3335 int scanned = 0;
Josef Bacikf7aaa062011-07-15 21:26:38 +00003336 int tag;
Chris Masond1310b22008-01-24 16:13:08 -05003337
Chris Masond1310b22008-01-24 16:13:08 -05003338 pagevec_init(&pvec, 0);
3339 if (wbc->range_cyclic) {
3340 index = mapping->writeback_index; /* Start from prev offset */
3341 end = -1;
3342 } else {
3343 index = wbc->range_start >> PAGE_CACHE_SHIFT;
3344 end = wbc->range_end >> PAGE_CACHE_SHIFT;
Chris Masond1310b22008-01-24 16:13:08 -05003345 scanned = 1;
3346 }
Josef Bacikf7aaa062011-07-15 21:26:38 +00003347 if (wbc->sync_mode == WB_SYNC_ALL)
3348 tag = PAGECACHE_TAG_TOWRITE;
3349 else
3350 tag = PAGECACHE_TAG_DIRTY;
Chris Masond1310b22008-01-24 16:13:08 -05003351retry:
Josef Bacikf7aaa062011-07-15 21:26:38 +00003352 if (wbc->sync_mode == WB_SYNC_ALL)
3353 tag_pages_for_writeback(mapping, index, end);
Chris Masonf85d7d6c2009-09-18 16:03:16 -04003354 while (!done && !nr_to_write_done && (index <= end) &&
Josef Bacikf7aaa062011-07-15 21:26:38 +00003355 (nr_pages = pagevec_lookup_tag(&pvec, mapping, &index, tag,
3356 min(end - index, (pgoff_t)PAGEVEC_SIZE-1) + 1))) {
Chris Masond1310b22008-01-24 16:13:08 -05003357 unsigned i;
3358
3359 scanned = 1;
3360 for (i = 0; i < nr_pages; i++) {
3361 struct page *page = pvec.pages[i];
3362
3363 /*
3364 * At this point we hold neither mapping->tree_lock nor
3365 * lock on the page itself: the page may be truncated or
3366 * invalidated (changing page->mapping to NULL), or even
3367 * swizzled back from swapper_space to tmpfs file
3368 * mapping
3369 */
Chris Mason01d658f2011-11-01 10:08:06 -04003370 if (tree->ops &&
3371 tree->ops->write_cache_pages_lock_hook) {
3372 tree->ops->write_cache_pages_lock_hook(page,
3373 data, flush_fn);
3374 } else {
3375 if (!trylock_page(page)) {
3376 flush_fn(data);
3377 lock_page(page);
3378 }
3379 }
Chris Masond1310b22008-01-24 16:13:08 -05003380
3381 if (unlikely(page->mapping != mapping)) {
3382 unlock_page(page);
3383 continue;
3384 }
3385
3386 if (!wbc->range_cyclic && page->index > end) {
3387 done = 1;
3388 unlock_page(page);
3389 continue;
3390 }
3391
Chris Masond2c3f4f2008-11-19 12:44:22 -05003392 if (wbc->sync_mode != WB_SYNC_NONE) {
Chris Mason0e6bd952008-11-20 10:46:35 -05003393 if (PageWriteback(page))
3394 flush_fn(data);
Chris Masond1310b22008-01-24 16:13:08 -05003395 wait_on_page_writeback(page);
Chris Masond2c3f4f2008-11-19 12:44:22 -05003396 }
Chris Masond1310b22008-01-24 16:13:08 -05003397
3398 if (PageWriteback(page) ||
3399 !clear_page_dirty_for_io(page)) {
3400 unlock_page(page);
3401 continue;
3402 }
3403
3404 ret = (*writepage)(page, wbc, data);
3405
3406 if (unlikely(ret == AOP_WRITEPAGE_ACTIVATE)) {
3407 unlock_page(page);
3408 ret = 0;
3409 }
Chris Masonf85d7d6c2009-09-18 16:03:16 -04003410 if (ret)
Chris Masond1310b22008-01-24 16:13:08 -05003411 done = 1;
Chris Masonf85d7d6c2009-09-18 16:03:16 -04003412
3413 /*
3414 * the filesystem may choose to bump up nr_to_write.
3415 * We have to make sure to honor the new nr_to_write
3416 * at any time
3417 */
3418 nr_to_write_done = wbc->nr_to_write <= 0;
Chris Masond1310b22008-01-24 16:13:08 -05003419 }
3420 pagevec_release(&pvec);
3421 cond_resched();
3422 }
3423 if (!scanned && !done) {
3424 /*
3425 * We hit the last page and there is more work to be done: wrap
3426 * back to the start of the file
3427 */
3428 scanned = 1;
3429 index = 0;
3430 goto retry;
3431 }
Chris Masond1310b22008-01-24 16:13:08 -05003432 return ret;
3433}
Chris Masond1310b22008-01-24 16:13:08 -05003434
Chris Masonffbd5172009-04-20 15:50:09 -04003435static void flush_epd_write_bio(struct extent_page_data *epd)
3436{
3437 if (epd->bio) {
Jeff Mahoney355808c2011-10-03 23:23:14 -04003438 int rw = WRITE;
3439 int ret;
3440
Chris Masonffbd5172009-04-20 15:50:09 -04003441 if (epd->sync_io)
Jeff Mahoney355808c2011-10-03 23:23:14 -04003442 rw = WRITE_SYNC;
3443
3444 ret = submit_one_bio(rw, epd->bio, 0, 0);
Jeff Mahoney79787ea2012-03-12 16:03:00 +01003445 BUG_ON(ret < 0); /* -ENOMEM */
Chris Masonffbd5172009-04-20 15:50:09 -04003446 epd->bio = NULL;
3447 }
3448}
3449
Chris Masond2c3f4f2008-11-19 12:44:22 -05003450static noinline void flush_write_bio(void *data)
3451{
3452 struct extent_page_data *epd = data;
Chris Masonffbd5172009-04-20 15:50:09 -04003453 flush_epd_write_bio(epd);
Chris Masond2c3f4f2008-11-19 12:44:22 -05003454}
3455
Chris Masond1310b22008-01-24 16:13:08 -05003456int extent_write_full_page(struct extent_io_tree *tree, struct page *page,
3457 get_extent_t *get_extent,
3458 struct writeback_control *wbc)
3459{
3460 int ret;
Chris Masond1310b22008-01-24 16:13:08 -05003461 struct extent_page_data epd = {
3462 .bio = NULL,
3463 .tree = tree,
3464 .get_extent = get_extent,
Chris Mason771ed682008-11-06 22:02:51 -05003465 .extent_locked = 0,
Chris Masonffbd5172009-04-20 15:50:09 -04003466 .sync_io = wbc->sync_mode == WB_SYNC_ALL,
Chris Masond1310b22008-01-24 16:13:08 -05003467 };
Chris Masond1310b22008-01-24 16:13:08 -05003468
Chris Masond1310b22008-01-24 16:13:08 -05003469 ret = __extent_writepage(page, wbc, &epd);
3470
Chris Masonffbd5172009-04-20 15:50:09 -04003471 flush_epd_write_bio(&epd);
Chris Masond1310b22008-01-24 16:13:08 -05003472 return ret;
3473}
Chris Masond1310b22008-01-24 16:13:08 -05003474
Chris Mason771ed682008-11-06 22:02:51 -05003475int extent_write_locked_range(struct extent_io_tree *tree, struct inode *inode,
3476 u64 start, u64 end, get_extent_t *get_extent,
3477 int mode)
3478{
3479 int ret = 0;
3480 struct address_space *mapping = inode->i_mapping;
3481 struct page *page;
3482 unsigned long nr_pages = (end - start + PAGE_CACHE_SIZE) >>
3483 PAGE_CACHE_SHIFT;
3484
3485 struct extent_page_data epd = {
3486 .bio = NULL,
3487 .tree = tree,
3488 .get_extent = get_extent,
3489 .extent_locked = 1,
Chris Masonffbd5172009-04-20 15:50:09 -04003490 .sync_io = mode == WB_SYNC_ALL,
Chris Mason771ed682008-11-06 22:02:51 -05003491 };
3492 struct writeback_control wbc_writepages = {
Chris Mason771ed682008-11-06 22:02:51 -05003493 .sync_mode = mode,
Chris Mason771ed682008-11-06 22:02:51 -05003494 .nr_to_write = nr_pages * 2,
3495 .range_start = start,
3496 .range_end = end + 1,
3497 };
3498
Chris Masond3977122009-01-05 21:25:51 -05003499 while (start <= end) {
Chris Mason771ed682008-11-06 22:02:51 -05003500 page = find_get_page(mapping, start >> PAGE_CACHE_SHIFT);
3501 if (clear_page_dirty_for_io(page))
3502 ret = __extent_writepage(page, &wbc_writepages, &epd);
3503 else {
3504 if (tree->ops && tree->ops->writepage_end_io_hook)
3505 tree->ops->writepage_end_io_hook(page, start,
3506 start + PAGE_CACHE_SIZE - 1,
3507 NULL, 1);
3508 unlock_page(page);
3509 }
3510 page_cache_release(page);
3511 start += PAGE_CACHE_SIZE;
3512 }
3513
Chris Masonffbd5172009-04-20 15:50:09 -04003514 flush_epd_write_bio(&epd);
Chris Mason771ed682008-11-06 22:02:51 -05003515 return ret;
3516}
Chris Masond1310b22008-01-24 16:13:08 -05003517
3518int extent_writepages(struct extent_io_tree *tree,
3519 struct address_space *mapping,
3520 get_extent_t *get_extent,
3521 struct writeback_control *wbc)
3522{
3523 int ret = 0;
3524 struct extent_page_data epd = {
3525 .bio = NULL,
3526 .tree = tree,
3527 .get_extent = get_extent,
Chris Mason771ed682008-11-06 22:02:51 -05003528 .extent_locked = 0,
Chris Masonffbd5172009-04-20 15:50:09 -04003529 .sync_io = wbc->sync_mode == WB_SYNC_ALL,
Chris Masond1310b22008-01-24 16:13:08 -05003530 };
3531
Chris Mason4bef0842008-09-08 11:18:08 -04003532 ret = extent_write_cache_pages(tree, mapping, wbc,
Chris Masond2c3f4f2008-11-19 12:44:22 -05003533 __extent_writepage, &epd,
3534 flush_write_bio);
Chris Masonffbd5172009-04-20 15:50:09 -04003535 flush_epd_write_bio(&epd);
Chris Masond1310b22008-01-24 16:13:08 -05003536 return ret;
3537}
Chris Masond1310b22008-01-24 16:13:08 -05003538
3539int extent_readpages(struct extent_io_tree *tree,
3540 struct address_space *mapping,
3541 struct list_head *pages, unsigned nr_pages,
3542 get_extent_t get_extent)
3543{
3544 struct bio *bio = NULL;
3545 unsigned page_idx;
Chris Masonc8b97812008-10-29 14:49:59 -04003546 unsigned long bio_flags = 0;
Chris Masond1310b22008-01-24 16:13:08 -05003547
Chris Masond1310b22008-01-24 16:13:08 -05003548 for (page_idx = 0; page_idx < nr_pages; page_idx++) {
3549 struct page *page = list_entry(pages->prev, struct page, lru);
3550
3551 prefetchw(&page->flags);
3552 list_del(&page->lru);
Nick Piggin28ecb6092010-03-17 13:31:04 +00003553 if (!add_to_page_cache_lru(page, mapping,
Itaru Kitayama43e817a2011-04-25 19:43:51 -04003554 page->index, GFP_NOFS)) {
Chris Masonf1885912008-04-09 16:28:12 -04003555 __extent_read_full_page(tree, page, get_extent,
Chris Masonc8b97812008-10-29 14:49:59 -04003556 &bio, 0, &bio_flags);
Chris Masond1310b22008-01-24 16:13:08 -05003557 }
3558 page_cache_release(page);
3559 }
Chris Masond1310b22008-01-24 16:13:08 -05003560 BUG_ON(!list_empty(pages));
3561 if (bio)
Jeff Mahoney79787ea2012-03-12 16:03:00 +01003562 return submit_one_bio(READ, bio, 0, bio_flags);
Chris Masond1310b22008-01-24 16:13:08 -05003563 return 0;
3564}
Chris Masond1310b22008-01-24 16:13:08 -05003565
3566/*
3567 * basic invalidatepage code, this waits on any locked or writeback
3568 * ranges corresponding to the page, and then deletes any extent state
3569 * records from the tree
3570 */
3571int extent_invalidatepage(struct extent_io_tree *tree,
3572 struct page *page, unsigned long offset)
3573{
Josef Bacik2ac55d42010-02-03 19:33:23 +00003574 struct extent_state *cached_state = NULL;
Chris Masond1310b22008-01-24 16:13:08 -05003575 u64 start = ((u64)page->index << PAGE_CACHE_SHIFT);
3576 u64 end = start + PAGE_CACHE_SIZE - 1;
3577 size_t blocksize = page->mapping->host->i_sb->s_blocksize;
3578
Chris Masond3977122009-01-05 21:25:51 -05003579 start += (offset + blocksize - 1) & ~(blocksize - 1);
Chris Masond1310b22008-01-24 16:13:08 -05003580 if (start > end)
3581 return 0;
3582
Jeff Mahoneyd0082372012-03-01 14:57:19 +01003583 lock_extent_bits(tree, start, end, 0, &cached_state);
Chris Mason1edbb732009-09-02 13:24:36 -04003584 wait_on_page_writeback(page);
Chris Masond1310b22008-01-24 16:13:08 -05003585 clear_extent_bit(tree, start, end,
Josef Bacik32c00af2009-10-08 13:34:05 -04003586 EXTENT_LOCKED | EXTENT_DIRTY | EXTENT_DELALLOC |
3587 EXTENT_DO_ACCOUNTING,
Josef Bacik2ac55d42010-02-03 19:33:23 +00003588 1, 1, &cached_state, GFP_NOFS);
Chris Masond1310b22008-01-24 16:13:08 -05003589 return 0;
3590}
Chris Masond1310b22008-01-24 16:13:08 -05003591
3592/*
Chris Mason7b13b7b2008-04-18 10:29:50 -04003593 * a helper for releasepage, this tests for areas of the page that
3594 * are locked or under IO and drops the related state bits if it is safe
3595 * to drop the page.
3596 */
3597int try_release_extent_state(struct extent_map_tree *map,
3598 struct extent_io_tree *tree, struct page *page,
3599 gfp_t mask)
3600{
3601 u64 start = (u64)page->index << PAGE_CACHE_SHIFT;
3602 u64 end = start + PAGE_CACHE_SIZE - 1;
3603 int ret = 1;
3604
Chris Mason211f90e2008-07-18 11:56:15 -04003605 if (test_range_bit(tree, start, end,
Chris Mason8b62b722009-09-02 16:53:46 -04003606 EXTENT_IOBITS, 0, NULL))
Chris Mason7b13b7b2008-04-18 10:29:50 -04003607 ret = 0;
3608 else {
3609 if ((mask & GFP_NOFS) == GFP_NOFS)
3610 mask = GFP_NOFS;
Chris Mason11ef1602009-09-23 20:28:46 -04003611 /*
3612 * at this point we can safely clear everything except the
3613 * locked bit and the nodatasum bit
3614 */
Chris Masone3f24cc2011-02-14 12:52:08 -05003615 ret = clear_extent_bit(tree, start, end,
Chris Mason11ef1602009-09-23 20:28:46 -04003616 ~(EXTENT_LOCKED | EXTENT_NODATASUM),
3617 0, 0, NULL, mask);
Chris Masone3f24cc2011-02-14 12:52:08 -05003618
3619 /* if clear_extent_bit failed for enomem reasons,
3620 * we can't allow the release to continue.
3621 */
3622 if (ret < 0)
3623 ret = 0;
3624 else
3625 ret = 1;
Chris Mason7b13b7b2008-04-18 10:29:50 -04003626 }
3627 return ret;
3628}
Chris Mason7b13b7b2008-04-18 10:29:50 -04003629
3630/*
Chris Masond1310b22008-01-24 16:13:08 -05003631 * a helper for releasepage. As long as there are no locked extents
3632 * in the range corresponding to the page, both state records and extent
3633 * map records are removed
3634 */
3635int try_release_extent_mapping(struct extent_map_tree *map,
Chris Mason70dec802008-01-29 09:59:12 -05003636 struct extent_io_tree *tree, struct page *page,
3637 gfp_t mask)
Chris Masond1310b22008-01-24 16:13:08 -05003638{
3639 struct extent_map *em;
3640 u64 start = (u64)page->index << PAGE_CACHE_SHIFT;
3641 u64 end = start + PAGE_CACHE_SIZE - 1;
Chris Mason7b13b7b2008-04-18 10:29:50 -04003642
Chris Mason70dec802008-01-29 09:59:12 -05003643 if ((mask & __GFP_WAIT) &&
3644 page->mapping->host->i_size > 16 * 1024 * 1024) {
Yan39b56372008-02-15 10:40:50 -05003645 u64 len;
Chris Mason70dec802008-01-29 09:59:12 -05003646 while (start <= end) {
Yan39b56372008-02-15 10:40:50 -05003647 len = end - start + 1;
Chris Mason890871b2009-09-02 16:24:52 -04003648 write_lock(&map->lock);
Yan39b56372008-02-15 10:40:50 -05003649 em = lookup_extent_mapping(map, start, len);
Tsutomu Itoh285190d2012-02-16 16:23:58 +09003650 if (!em) {
Chris Mason890871b2009-09-02 16:24:52 -04003651 write_unlock(&map->lock);
Chris Mason70dec802008-01-29 09:59:12 -05003652 break;
3653 }
Chris Mason7f3c74f2008-07-18 12:01:11 -04003654 if (test_bit(EXTENT_FLAG_PINNED, &em->flags) ||
3655 em->start != start) {
Chris Mason890871b2009-09-02 16:24:52 -04003656 write_unlock(&map->lock);
Chris Mason70dec802008-01-29 09:59:12 -05003657 free_extent_map(em);
3658 break;
3659 }
3660 if (!test_range_bit(tree, em->start,
3661 extent_map_end(em) - 1,
Chris Mason8b62b722009-09-02 16:53:46 -04003662 EXTENT_LOCKED | EXTENT_WRITEBACK,
Chris Mason9655d292009-09-02 15:22:30 -04003663 0, NULL)) {
Chris Mason70dec802008-01-29 09:59:12 -05003664 remove_extent_mapping(map, em);
3665 /* once for the rb tree */
3666 free_extent_map(em);
3667 }
3668 start = extent_map_end(em);
Chris Mason890871b2009-09-02 16:24:52 -04003669 write_unlock(&map->lock);
Chris Mason70dec802008-01-29 09:59:12 -05003670
3671 /* once for us */
Chris Masond1310b22008-01-24 16:13:08 -05003672 free_extent_map(em);
3673 }
Chris Masond1310b22008-01-24 16:13:08 -05003674 }
Chris Mason7b13b7b2008-04-18 10:29:50 -04003675 return try_release_extent_state(map, tree, page, mask);
Chris Masond1310b22008-01-24 16:13:08 -05003676}
Chris Masond1310b22008-01-24 16:13:08 -05003677
Chris Masonec29ed52011-02-23 16:23:20 -05003678/*
3679 * helper function for fiemap, which doesn't want to see any holes.
3680 * This maps until we find something past 'last'
3681 */
3682static struct extent_map *get_extent_skip_holes(struct inode *inode,
3683 u64 offset,
3684 u64 last,
3685 get_extent_t *get_extent)
3686{
3687 u64 sectorsize = BTRFS_I(inode)->root->sectorsize;
3688 struct extent_map *em;
3689 u64 len;
3690
3691 if (offset >= last)
3692 return NULL;
3693
3694 while(1) {
3695 len = last - offset;
3696 if (len == 0)
3697 break;
3698 len = (len + sectorsize - 1) & ~(sectorsize - 1);
3699 em = get_extent(inode, NULL, 0, offset, len, 0);
David Sterbac7040052011-04-19 18:00:01 +02003700 if (IS_ERR_OR_NULL(em))
Chris Masonec29ed52011-02-23 16:23:20 -05003701 return em;
3702
3703 /* if this isn't a hole return it */
3704 if (!test_bit(EXTENT_FLAG_VACANCY, &em->flags) &&
3705 em->block_start != EXTENT_MAP_HOLE) {
3706 return em;
3707 }
3708
3709 /* this is a hole, advance to the next extent */
3710 offset = extent_map_end(em);
3711 free_extent_map(em);
3712 if (offset >= last)
3713 break;
3714 }
3715 return NULL;
3716}
3717
Yehuda Sadeh1506fcc2009-01-21 14:39:14 -05003718int extent_fiemap(struct inode *inode, struct fiemap_extent_info *fieinfo,
3719 __u64 start, __u64 len, get_extent_t *get_extent)
3720{
Josef Bacik975f84f2010-11-23 19:36:57 +00003721 int ret = 0;
Yehuda Sadeh1506fcc2009-01-21 14:39:14 -05003722 u64 off = start;
3723 u64 max = start + len;
3724 u32 flags = 0;
Josef Bacik975f84f2010-11-23 19:36:57 +00003725 u32 found_type;
3726 u64 last;
Chris Masonec29ed52011-02-23 16:23:20 -05003727 u64 last_for_get_extent = 0;
Yehuda Sadeh1506fcc2009-01-21 14:39:14 -05003728 u64 disko = 0;
Chris Masonec29ed52011-02-23 16:23:20 -05003729 u64 isize = i_size_read(inode);
Josef Bacik975f84f2010-11-23 19:36:57 +00003730 struct btrfs_key found_key;
Yehuda Sadeh1506fcc2009-01-21 14:39:14 -05003731 struct extent_map *em = NULL;
Josef Bacik2ac55d42010-02-03 19:33:23 +00003732 struct extent_state *cached_state = NULL;
Josef Bacik975f84f2010-11-23 19:36:57 +00003733 struct btrfs_path *path;
3734 struct btrfs_file_extent_item *item;
Yehuda Sadeh1506fcc2009-01-21 14:39:14 -05003735 int end = 0;
Chris Masonec29ed52011-02-23 16:23:20 -05003736 u64 em_start = 0;
3737 u64 em_len = 0;
3738 u64 em_end = 0;
Yehuda Sadeh1506fcc2009-01-21 14:39:14 -05003739 unsigned long emflags;
Yehuda Sadeh1506fcc2009-01-21 14:39:14 -05003740
3741 if (len == 0)
3742 return -EINVAL;
3743
Josef Bacik975f84f2010-11-23 19:36:57 +00003744 path = btrfs_alloc_path();
3745 if (!path)
3746 return -ENOMEM;
3747 path->leave_spinning = 1;
3748
Josef Bacik4d479cf2011-11-17 11:34:31 -05003749 start = ALIGN(start, BTRFS_I(inode)->root->sectorsize);
3750 len = ALIGN(len, BTRFS_I(inode)->root->sectorsize);
3751
Chris Masonec29ed52011-02-23 16:23:20 -05003752 /*
3753 * lookup the last file extent. We're not using i_size here
3754 * because there might be preallocation past i_size
3755 */
Josef Bacik975f84f2010-11-23 19:36:57 +00003756 ret = btrfs_lookup_file_extent(NULL, BTRFS_I(inode)->root,
Li Zefan33345d012011-04-20 10:31:50 +08003757 path, btrfs_ino(inode), -1, 0);
Josef Bacik975f84f2010-11-23 19:36:57 +00003758 if (ret < 0) {
3759 btrfs_free_path(path);
3760 return ret;
3761 }
3762 WARN_ON(!ret);
3763 path->slots[0]--;
3764 item = btrfs_item_ptr(path->nodes[0], path->slots[0],
3765 struct btrfs_file_extent_item);
3766 btrfs_item_key_to_cpu(path->nodes[0], &found_key, path->slots[0]);
3767 found_type = btrfs_key_type(&found_key);
3768
Chris Masonec29ed52011-02-23 16:23:20 -05003769 /* No extents, but there might be delalloc bits */
Li Zefan33345d012011-04-20 10:31:50 +08003770 if (found_key.objectid != btrfs_ino(inode) ||
Josef Bacik975f84f2010-11-23 19:36:57 +00003771 found_type != BTRFS_EXTENT_DATA_KEY) {
Chris Masonec29ed52011-02-23 16:23:20 -05003772 /* have to trust i_size as the end */
3773 last = (u64)-1;
3774 last_for_get_extent = isize;
3775 } else {
3776 /*
3777 * remember the start of the last extent. There are a
3778 * bunch of different factors that go into the length of the
3779 * extent, so its much less complex to remember where it started
3780 */
3781 last = found_key.offset;
3782 last_for_get_extent = last + 1;
Josef Bacik975f84f2010-11-23 19:36:57 +00003783 }
Josef Bacik975f84f2010-11-23 19:36:57 +00003784 btrfs_free_path(path);
3785
Chris Masonec29ed52011-02-23 16:23:20 -05003786 /*
3787 * we might have some extents allocated but more delalloc past those
3788 * extents. so, we trust isize unless the start of the last extent is
3789 * beyond isize
3790 */
3791 if (last < isize) {
3792 last = (u64)-1;
3793 last_for_get_extent = isize;
3794 }
3795
Josef Bacik2ac55d42010-02-03 19:33:23 +00003796 lock_extent_bits(&BTRFS_I(inode)->io_tree, start, start + len, 0,
Jeff Mahoneyd0082372012-03-01 14:57:19 +01003797 &cached_state);
Chris Masonec29ed52011-02-23 16:23:20 -05003798
Josef Bacik4d479cf2011-11-17 11:34:31 -05003799 em = get_extent_skip_holes(inode, start, last_for_get_extent,
Chris Masonec29ed52011-02-23 16:23:20 -05003800 get_extent);
Yehuda Sadeh1506fcc2009-01-21 14:39:14 -05003801 if (!em)
3802 goto out;
3803 if (IS_ERR(em)) {
3804 ret = PTR_ERR(em);
3805 goto out;
3806 }
Josef Bacik975f84f2010-11-23 19:36:57 +00003807
Yehuda Sadeh1506fcc2009-01-21 14:39:14 -05003808 while (!end) {
Chris Masonea8efc72011-03-08 11:54:40 -05003809 u64 offset_in_extent;
Yehuda Sadeh1506fcc2009-01-21 14:39:14 -05003810
Chris Masonea8efc72011-03-08 11:54:40 -05003811 /* break if the extent we found is outside the range */
3812 if (em->start >= max || extent_map_end(em) < off)
3813 break;
3814
3815 /*
3816 * get_extent may return an extent that starts before our
3817 * requested range. We have to make sure the ranges
3818 * we return to fiemap always move forward and don't
3819 * overlap, so adjust the offsets here
3820 */
3821 em_start = max(em->start, off);
3822
3823 /*
3824 * record the offset from the start of the extent
3825 * for adjusting the disk offset below
3826 */
3827 offset_in_extent = em_start - em->start;
Chris Masonec29ed52011-02-23 16:23:20 -05003828 em_end = extent_map_end(em);
Chris Masonea8efc72011-03-08 11:54:40 -05003829 em_len = em_end - em_start;
Chris Masonec29ed52011-02-23 16:23:20 -05003830 emflags = em->flags;
Yehuda Sadeh1506fcc2009-01-21 14:39:14 -05003831 disko = 0;
3832 flags = 0;
3833
Chris Masonea8efc72011-03-08 11:54:40 -05003834 /*
3835 * bump off for our next call to get_extent
3836 */
3837 off = extent_map_end(em);
3838 if (off >= max)
3839 end = 1;
3840
Heiko Carstens93dbfad2009-04-03 10:33:45 -04003841 if (em->block_start == EXTENT_MAP_LAST_BYTE) {
Yehuda Sadeh1506fcc2009-01-21 14:39:14 -05003842 end = 1;
3843 flags |= FIEMAP_EXTENT_LAST;
Heiko Carstens93dbfad2009-04-03 10:33:45 -04003844 } else if (em->block_start == EXTENT_MAP_INLINE) {
Yehuda Sadeh1506fcc2009-01-21 14:39:14 -05003845 flags |= (FIEMAP_EXTENT_DATA_INLINE |
3846 FIEMAP_EXTENT_NOT_ALIGNED);
Heiko Carstens93dbfad2009-04-03 10:33:45 -04003847 } else if (em->block_start == EXTENT_MAP_DELALLOC) {
Yehuda Sadeh1506fcc2009-01-21 14:39:14 -05003848 flags |= (FIEMAP_EXTENT_DELALLOC |
3849 FIEMAP_EXTENT_UNKNOWN);
Heiko Carstens93dbfad2009-04-03 10:33:45 -04003850 } else {
Chris Masonea8efc72011-03-08 11:54:40 -05003851 disko = em->block_start + offset_in_extent;
Yehuda Sadeh1506fcc2009-01-21 14:39:14 -05003852 }
3853 if (test_bit(EXTENT_FLAG_COMPRESSED, &em->flags))
3854 flags |= FIEMAP_EXTENT_ENCODED;
3855
Yehuda Sadeh1506fcc2009-01-21 14:39:14 -05003856 free_extent_map(em);
3857 em = NULL;
Chris Masonec29ed52011-02-23 16:23:20 -05003858 if ((em_start >= last) || em_len == (u64)-1 ||
3859 (last == (u64)-1 && isize <= em_end)) {
Yehuda Sadeh1506fcc2009-01-21 14:39:14 -05003860 flags |= FIEMAP_EXTENT_LAST;
3861 end = 1;
3862 }
3863
Chris Masonec29ed52011-02-23 16:23:20 -05003864 /* now scan forward to see if this is really the last extent. */
3865 em = get_extent_skip_holes(inode, off, last_for_get_extent,
3866 get_extent);
3867 if (IS_ERR(em)) {
3868 ret = PTR_ERR(em);
3869 goto out;
3870 }
3871 if (!em) {
Josef Bacik975f84f2010-11-23 19:36:57 +00003872 flags |= FIEMAP_EXTENT_LAST;
3873 end = 1;
3874 }
Chris Masonec29ed52011-02-23 16:23:20 -05003875 ret = fiemap_fill_next_extent(fieinfo, em_start, disko,
3876 em_len, flags);
3877 if (ret)
3878 goto out_free;
Yehuda Sadeh1506fcc2009-01-21 14:39:14 -05003879 }
3880out_free:
3881 free_extent_map(em);
3882out:
Josef Bacik2ac55d42010-02-03 19:33:23 +00003883 unlock_extent_cached(&BTRFS_I(inode)->io_tree, start, start + len,
3884 &cached_state, GFP_NOFS);
Yehuda Sadeh1506fcc2009-01-21 14:39:14 -05003885 return ret;
3886}
3887
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02003888inline struct page *extent_buffer_page(struct extent_buffer *eb,
Chris Masond1310b22008-01-24 16:13:08 -05003889 unsigned long i)
3890{
Chris Mason727011e2010-08-06 13:21:20 -04003891 return eb->pages[i];
Chris Masond1310b22008-01-24 16:13:08 -05003892}
3893
Jan Schmidt4a54c8c2011-07-22 15:41:52 +02003894inline unsigned long num_extent_pages(u64 start, u64 len)
Chris Masonce9adaa2008-04-09 16:28:12 -04003895{
Chris Mason6af118ce2008-07-22 11:18:07 -04003896 return ((start + len + PAGE_CACHE_SIZE - 1) >> PAGE_CACHE_SHIFT) -
3897 (start >> PAGE_CACHE_SHIFT);
Chris Mason728131d2008-04-09 16:28:12 -04003898}
3899
Chris Mason727011e2010-08-06 13:21:20 -04003900static void __free_extent_buffer(struct extent_buffer *eb)
3901{
3902#if LEAK_DEBUG
3903 unsigned long flags;
3904 spin_lock_irqsave(&leak_lock, flags);
3905 list_del(&eb->leak_list);
3906 spin_unlock_irqrestore(&leak_lock, flags);
3907#endif
3908 if (eb->pages && eb->pages != eb->inline_pages)
3909 kfree(eb->pages);
3910 kmem_cache_free(extent_buffer_cache, eb);
3911}
3912
Chris Masond1310b22008-01-24 16:13:08 -05003913static struct extent_buffer *__alloc_extent_buffer(struct extent_io_tree *tree,
3914 u64 start,
3915 unsigned long len,
3916 gfp_t mask)
3917{
3918 struct extent_buffer *eb = NULL;
Chris Mason39351272009-02-04 09:24:05 -05003919#if LEAK_DEBUG
Chris Mason2d2ae542008-03-26 16:24:23 -04003920 unsigned long flags;
Chris Mason4bef0842008-09-08 11:18:08 -04003921#endif
Chris Masond1310b22008-01-24 16:13:08 -05003922
Chris Masond1310b22008-01-24 16:13:08 -05003923 eb = kmem_cache_zalloc(extent_buffer_cache, mask);
Tsutomu Itoh91ca3382011-01-05 02:32:22 +00003924 if (eb == NULL)
3925 return NULL;
Chris Masond1310b22008-01-24 16:13:08 -05003926 eb->start = start;
3927 eb->len = len;
Josef Bacik4f2de97a2012-03-07 16:20:05 -05003928 eb->tree = tree;
Chris Masonbd681512011-07-16 15:23:14 -04003929 rwlock_init(&eb->lock);
3930 atomic_set(&eb->write_locks, 0);
3931 atomic_set(&eb->read_locks, 0);
3932 atomic_set(&eb->blocking_readers, 0);
3933 atomic_set(&eb->blocking_writers, 0);
3934 atomic_set(&eb->spinning_readers, 0);
3935 atomic_set(&eb->spinning_writers, 0);
Arne Jansen5b25f702011-09-13 10:55:48 +02003936 eb->lock_nested = 0;
Chris Masonbd681512011-07-16 15:23:14 -04003937 init_waitqueue_head(&eb->write_lock_wq);
3938 init_waitqueue_head(&eb->read_lock_wq);
Chris Masonb4ce94d2009-02-04 09:25:08 -05003939
Chris Mason39351272009-02-04 09:24:05 -05003940#if LEAK_DEBUG
Chris Mason2d2ae542008-03-26 16:24:23 -04003941 spin_lock_irqsave(&leak_lock, flags);
3942 list_add(&eb->leak_list, &buffers);
3943 spin_unlock_irqrestore(&leak_lock, flags);
Chris Mason4bef0842008-09-08 11:18:08 -04003944#endif
Josef Bacik3083ee22012-03-09 16:01:49 -05003945 spin_lock_init(&eb->refs_lock);
Chris Masond1310b22008-01-24 16:13:08 -05003946 atomic_set(&eb->refs, 1);
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003947 atomic_set(&eb->io_pages, 0);
Chris Mason727011e2010-08-06 13:21:20 -04003948
3949 if (len > MAX_INLINE_EXTENT_BUFFER_SIZE) {
3950 struct page **pages;
3951 int num_pages = (len + PAGE_CACHE_SIZE - 1) >>
3952 PAGE_CACHE_SHIFT;
3953 pages = kzalloc(num_pages, mask);
3954 if (!pages) {
3955 __free_extent_buffer(eb);
3956 return NULL;
3957 }
3958 eb->pages = pages;
3959 } else {
3960 eb->pages = eb->inline_pages;
3961 }
Chris Masond1310b22008-01-24 16:13:08 -05003962
3963 return eb;
3964}
3965
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003966static int extent_buffer_under_io(struct extent_buffer *eb)
Chris Masond1310b22008-01-24 16:13:08 -05003967{
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003968 return (atomic_read(&eb->io_pages) ||
3969 test_bit(EXTENT_BUFFER_WRITEBACK, &eb->bflags) ||
3970 test_bit(EXTENT_BUFFER_DIRTY, &eb->bflags));
Chris Masond1310b22008-01-24 16:13:08 -05003971}
3972
Miao Xie897ca6e2010-10-26 20:57:29 -04003973/*
3974 * Helper for releasing extent buffer page.
3975 */
3976static void btrfs_release_extent_buffer_page(struct extent_buffer *eb,
3977 unsigned long start_idx)
3978{
3979 unsigned long index;
3980 struct page *page;
3981
Josef Bacik0b32f4b2012-03-13 09:38:00 -04003982 BUG_ON(extent_buffer_under_io(eb));
Miao Xie897ca6e2010-10-26 20:57:29 -04003983
3984 index = num_extent_pages(eb->start, eb->len);
3985 if (start_idx >= index)
3986 return;
3987
3988 do {
3989 index--;
3990 page = extent_buffer_page(eb, index);
Josef Bacik4f2de97a2012-03-07 16:20:05 -05003991 if (page) {
3992 spin_lock(&page->mapping->private_lock);
3993 /*
3994 * We do this since we'll remove the pages after we've
3995 * removed the eb from the radix tree, so we could race
3996 * and have this page now attached to the new eb. So
3997 * only clear page_private if it's still connected to
3998 * this eb.
3999 */
4000 if (PagePrivate(page) &&
4001 page->private == (unsigned long)eb) {
Josef Bacik0b32f4b2012-03-13 09:38:00 -04004002 BUG_ON(test_bit(EXTENT_BUFFER_DIRTY, &eb->bflags));
Josef Bacik3083ee22012-03-09 16:01:49 -05004003 BUG_ON(PageDirty(page));
4004 BUG_ON(PageWriteback(page));
Josef Bacik4f2de97a2012-03-07 16:20:05 -05004005 /*
4006 * We need to make sure we haven't be attached
4007 * to a new eb.
4008 */
4009 ClearPagePrivate(page);
4010 set_page_private(page, 0);
4011 /* One for the page private */
4012 page_cache_release(page);
4013 }
4014 spin_unlock(&page->mapping->private_lock);
4015
4016 /* One for when we alloced the page */
Miao Xie897ca6e2010-10-26 20:57:29 -04004017 page_cache_release(page);
Josef Bacik4f2de97a2012-03-07 16:20:05 -05004018 }
Miao Xie897ca6e2010-10-26 20:57:29 -04004019 } while (index != start_idx);
4020}
4021
4022/*
4023 * Helper for releasing the extent buffer.
4024 */
4025static inline void btrfs_release_extent_buffer(struct extent_buffer *eb)
4026{
4027 btrfs_release_extent_buffer_page(eb, 0);
4028 __free_extent_buffer(eb);
4029}
4030
Josef Bacik0b32f4b2012-03-13 09:38:00 -04004031static void check_buffer_tree_ref(struct extent_buffer *eb)
4032{
4033 /* the ref bit is tricky. We have to make sure it is set
4034 * if we have the buffer dirty. Otherwise the
4035 * code to free a buffer can end up dropping a dirty
4036 * page
4037 *
4038 * Once the ref bit is set, it won't go away while the
4039 * buffer is dirty or in writeback, and it also won't
4040 * go away while we have the reference count on the
4041 * eb bumped.
4042 *
4043 * We can't just set the ref bit without bumping the
4044 * ref on the eb because free_extent_buffer might
4045 * see the ref bit and try to clear it. If this happens
4046 * free_extent_buffer might end up dropping our original
4047 * ref by mistake and freeing the page before we are able
4048 * to add one more ref.
4049 *
4050 * So bump the ref count first, then set the bit. If someone
4051 * beat us to it, drop the ref we added.
4052 */
4053 if (!test_bit(EXTENT_BUFFER_TREE_REF, &eb->bflags)) {
4054 atomic_inc(&eb->refs);
4055 if (test_and_set_bit(EXTENT_BUFFER_TREE_REF, &eb->bflags))
4056 atomic_dec(&eb->refs);
4057 }
4058}
4059
Josef Bacik5df42352012-03-15 18:24:42 -04004060static void mark_extent_buffer_accessed(struct extent_buffer *eb)
4061{
4062 unsigned long num_pages, i;
4063
Josef Bacik0b32f4b2012-03-13 09:38:00 -04004064 check_buffer_tree_ref(eb);
4065
Josef Bacik5df42352012-03-15 18:24:42 -04004066 num_pages = num_extent_pages(eb->start, eb->len);
4067 for (i = 0; i < num_pages; i++) {
4068 struct page *p = extent_buffer_page(eb, i);
4069 mark_page_accessed(p);
4070 }
4071}
4072
Chris Masond1310b22008-01-24 16:13:08 -05004073struct extent_buffer *alloc_extent_buffer(struct extent_io_tree *tree,
Chris Mason727011e2010-08-06 13:21:20 -04004074 u64 start, unsigned long len)
Chris Masond1310b22008-01-24 16:13:08 -05004075{
4076 unsigned long num_pages = num_extent_pages(start, len);
4077 unsigned long i;
4078 unsigned long index = start >> PAGE_CACHE_SHIFT;
4079 struct extent_buffer *eb;
Chris Mason6af118ce2008-07-22 11:18:07 -04004080 struct extent_buffer *exists = NULL;
Chris Masond1310b22008-01-24 16:13:08 -05004081 struct page *p;
4082 struct address_space *mapping = tree->mapping;
4083 int uptodate = 1;
Miao Xie19fe0a82010-10-26 20:57:29 -04004084 int ret;
Chris Masond1310b22008-01-24 16:13:08 -05004085
Miao Xie19fe0a82010-10-26 20:57:29 -04004086 rcu_read_lock();
4087 eb = radix_tree_lookup(&tree->buffer, start >> PAGE_CACHE_SHIFT);
4088 if (eb && atomic_inc_not_zero(&eb->refs)) {
4089 rcu_read_unlock();
Josef Bacik5df42352012-03-15 18:24:42 -04004090 mark_extent_buffer_accessed(eb);
Chris Mason6af118ce2008-07-22 11:18:07 -04004091 return eb;
4092 }
Miao Xie19fe0a82010-10-26 20:57:29 -04004093 rcu_read_unlock();
Chris Mason6af118ce2008-07-22 11:18:07 -04004094
David Sterbaba144192011-04-21 01:12:06 +02004095 eb = __alloc_extent_buffer(tree, start, len, GFP_NOFS);
Peter2b114d12008-04-01 11:21:40 -04004096 if (!eb)
Chris Masond1310b22008-01-24 16:13:08 -05004097 return NULL;
4098
Chris Mason727011e2010-08-06 13:21:20 -04004099 for (i = 0; i < num_pages; i++, index++) {
Chris Masona6591712011-07-19 12:04:14 -04004100 p = find_or_create_page(mapping, index, GFP_NOFS);
Chris Masond1310b22008-01-24 16:13:08 -05004101 if (!p) {
4102 WARN_ON(1);
Chris Mason6af118ce2008-07-22 11:18:07 -04004103 goto free_eb;
Chris Masond1310b22008-01-24 16:13:08 -05004104 }
Josef Bacik4f2de97a2012-03-07 16:20:05 -05004105
4106 spin_lock(&mapping->private_lock);
4107 if (PagePrivate(p)) {
4108 /*
4109 * We could have already allocated an eb for this page
4110 * and attached one so lets see if we can get a ref on
4111 * the existing eb, and if we can we know it's good and
4112 * we can just return that one, else we know we can just
4113 * overwrite page->private.
4114 */
4115 exists = (struct extent_buffer *)p->private;
4116 if (atomic_inc_not_zero(&exists->refs)) {
4117 spin_unlock(&mapping->private_lock);
4118 unlock_page(p);
Josef Bacik5df42352012-03-15 18:24:42 -04004119 mark_extent_buffer_accessed(exists);
Josef Bacik4f2de97a2012-03-07 16:20:05 -05004120 goto free_eb;
4121 }
4122
Josef Bacik0b32f4b2012-03-13 09:38:00 -04004123 /*
Josef Bacik4f2de97a2012-03-07 16:20:05 -05004124 * Do this so attach doesn't complain and we need to
4125 * drop the ref the old guy had.
4126 */
4127 ClearPagePrivate(p);
Josef Bacik0b32f4b2012-03-13 09:38:00 -04004128 WARN_ON(PageDirty(p));
Josef Bacik4f2de97a2012-03-07 16:20:05 -05004129 page_cache_release(p);
Chris Masond1310b22008-01-24 16:13:08 -05004130 }
Josef Bacik4f2de97a2012-03-07 16:20:05 -05004131 attach_extent_buffer_page(eb, p);
4132 spin_unlock(&mapping->private_lock);
Josef Bacik0b32f4b2012-03-13 09:38:00 -04004133 WARN_ON(PageDirty(p));
Chris Masond1310b22008-01-24 16:13:08 -05004134 mark_page_accessed(p);
Chris Mason727011e2010-08-06 13:21:20 -04004135 eb->pages[i] = p;
Chris Masond1310b22008-01-24 16:13:08 -05004136 if (!PageUptodate(p))
4137 uptodate = 0;
Chris Masoneb14ab82011-02-10 12:35:00 -05004138
4139 /*
4140 * see below about how we avoid a nasty race with release page
4141 * and why we unlock later
4142 */
Chris Masond1310b22008-01-24 16:13:08 -05004143 }
4144 if (uptodate)
Chris Masonb4ce94d2009-02-04 09:25:08 -05004145 set_bit(EXTENT_BUFFER_UPTODATE, &eb->bflags);
Josef Bacik115391d2012-03-09 09:51:43 -05004146again:
Miao Xie19fe0a82010-10-26 20:57:29 -04004147 ret = radix_tree_preload(GFP_NOFS & ~__GFP_HIGHMEM);
4148 if (ret)
4149 goto free_eb;
4150
Chris Mason6af118ce2008-07-22 11:18:07 -04004151 spin_lock(&tree->buffer_lock);
Miao Xie19fe0a82010-10-26 20:57:29 -04004152 ret = radix_tree_insert(&tree->buffer, start >> PAGE_CACHE_SHIFT, eb);
4153 if (ret == -EEXIST) {
4154 exists = radix_tree_lookup(&tree->buffer,
4155 start >> PAGE_CACHE_SHIFT);
Josef Bacik115391d2012-03-09 09:51:43 -05004156 if (!atomic_inc_not_zero(&exists->refs)) {
4157 spin_unlock(&tree->buffer_lock);
4158 radix_tree_preload_end();
Josef Bacik115391d2012-03-09 09:51:43 -05004159 exists = NULL;
4160 goto again;
4161 }
Chris Mason6af118ce2008-07-22 11:18:07 -04004162 spin_unlock(&tree->buffer_lock);
Miao Xie19fe0a82010-10-26 20:57:29 -04004163 radix_tree_preload_end();
Josef Bacik5df42352012-03-15 18:24:42 -04004164 mark_extent_buffer_accessed(exists);
Chris Mason6af118ce2008-07-22 11:18:07 -04004165 goto free_eb;
4166 }
Chris Mason6af118ce2008-07-22 11:18:07 -04004167 /* add one reference for the tree */
Josef Bacik3083ee22012-03-09 16:01:49 -05004168 spin_lock(&eb->refs_lock);
Josef Bacik0b32f4b2012-03-13 09:38:00 -04004169 check_buffer_tree_ref(eb);
Josef Bacik3083ee22012-03-09 16:01:49 -05004170 spin_unlock(&eb->refs_lock);
Yan, Zhengf044ba72010-02-04 08:46:56 +00004171 spin_unlock(&tree->buffer_lock);
Miao Xie19fe0a82010-10-26 20:57:29 -04004172 radix_tree_preload_end();
Chris Masoneb14ab82011-02-10 12:35:00 -05004173
4174 /*
4175 * there is a race where release page may have
4176 * tried to find this extent buffer in the radix
4177 * but failed. It will tell the VM it is safe to
4178 * reclaim the, and it will clear the page private bit.
4179 * We must make sure to set the page private bit properly
4180 * after the extent buffer is in the radix tree so
4181 * it doesn't get lost
4182 */
Chris Mason727011e2010-08-06 13:21:20 -04004183 SetPageChecked(eb->pages[0]);
4184 for (i = 1; i < num_pages; i++) {
4185 p = extent_buffer_page(eb, i);
Chris Mason727011e2010-08-06 13:21:20 -04004186 ClearPageChecked(p);
4187 unlock_page(p);
4188 }
4189 unlock_page(eb->pages[0]);
Chris Masond1310b22008-01-24 16:13:08 -05004190 return eb;
4191
Chris Mason6af118ce2008-07-22 11:18:07 -04004192free_eb:
Chris Mason727011e2010-08-06 13:21:20 -04004193 for (i = 0; i < num_pages; i++) {
4194 if (eb->pages[i])
4195 unlock_page(eb->pages[i]);
4196 }
Chris Masoneb14ab82011-02-10 12:35:00 -05004197
Chris Masond1310b22008-01-24 16:13:08 -05004198 if (!atomic_dec_and_test(&eb->refs))
Chris Mason6af118ce2008-07-22 11:18:07 -04004199 return exists;
Miao Xie897ca6e2010-10-26 20:57:29 -04004200 btrfs_release_extent_buffer(eb);
Chris Mason6af118ce2008-07-22 11:18:07 -04004201 return exists;
Chris Masond1310b22008-01-24 16:13:08 -05004202}
Chris Masond1310b22008-01-24 16:13:08 -05004203
4204struct extent_buffer *find_extent_buffer(struct extent_io_tree *tree,
David Sterbaf09d1f62011-04-21 01:08:01 +02004205 u64 start, unsigned long len)
Chris Masond1310b22008-01-24 16:13:08 -05004206{
Chris Masond1310b22008-01-24 16:13:08 -05004207 struct extent_buffer *eb;
Chris Masond1310b22008-01-24 16:13:08 -05004208
Miao Xie19fe0a82010-10-26 20:57:29 -04004209 rcu_read_lock();
4210 eb = radix_tree_lookup(&tree->buffer, start >> PAGE_CACHE_SHIFT);
4211 if (eb && atomic_inc_not_zero(&eb->refs)) {
4212 rcu_read_unlock();
Josef Bacik5df42352012-03-15 18:24:42 -04004213 mark_extent_buffer_accessed(eb);
Miao Xie19fe0a82010-10-26 20:57:29 -04004214 return eb;
4215 }
4216 rcu_read_unlock();
Josef Bacik0f9dd462008-09-23 13:14:11 -04004217
Miao Xie19fe0a82010-10-26 20:57:29 -04004218 return NULL;
Chris Masond1310b22008-01-24 16:13:08 -05004219}
Chris Masond1310b22008-01-24 16:13:08 -05004220
Josef Bacik3083ee22012-03-09 16:01:49 -05004221static inline void btrfs_release_extent_buffer_rcu(struct rcu_head *head)
4222{
4223 struct extent_buffer *eb =
4224 container_of(head, struct extent_buffer, rcu_head);
4225
4226 __free_extent_buffer(eb);
4227}
4228
Josef Bacik3083ee22012-03-09 16:01:49 -05004229/* Expects to have eb->eb_lock already held */
4230static void release_extent_buffer(struct extent_buffer *eb, gfp_t mask)
4231{
4232 WARN_ON(atomic_read(&eb->refs) == 0);
4233 if (atomic_dec_and_test(&eb->refs)) {
4234 struct extent_io_tree *tree = eb->tree;
Josef Bacik3083ee22012-03-09 16:01:49 -05004235
4236 spin_unlock(&eb->refs_lock);
4237
Josef Bacik3083ee22012-03-09 16:01:49 -05004238 spin_lock(&tree->buffer_lock);
4239 radix_tree_delete(&tree->buffer,
4240 eb->start >> PAGE_CACHE_SHIFT);
4241 spin_unlock(&tree->buffer_lock);
4242
4243 /* Should be safe to release our pages at this point */
4244 btrfs_release_extent_buffer_page(eb, 0);
4245
4246 call_rcu(&eb->rcu_head, btrfs_release_extent_buffer_rcu);
4247 return;
4248 }
4249 spin_unlock(&eb->refs_lock);
4250}
4251
Chris Masond1310b22008-01-24 16:13:08 -05004252void free_extent_buffer(struct extent_buffer *eb)
4253{
Chris Masond1310b22008-01-24 16:13:08 -05004254 if (!eb)
4255 return;
4256
Josef Bacik3083ee22012-03-09 16:01:49 -05004257 spin_lock(&eb->refs_lock);
4258 if (atomic_read(&eb->refs) == 2 &&
4259 test_bit(EXTENT_BUFFER_STALE, &eb->bflags) &&
Josef Bacik0b32f4b2012-03-13 09:38:00 -04004260 !extent_buffer_under_io(eb) &&
Josef Bacik3083ee22012-03-09 16:01:49 -05004261 test_and_clear_bit(EXTENT_BUFFER_TREE_REF, &eb->bflags))
4262 atomic_dec(&eb->refs);
Chris Masond1310b22008-01-24 16:13:08 -05004263
Josef Bacik3083ee22012-03-09 16:01:49 -05004264 /*
4265 * I know this is terrible, but it's temporary until we stop tracking
4266 * the uptodate bits and such for the extent buffers.
4267 */
4268 release_extent_buffer(eb, GFP_ATOMIC);
Chris Masond1310b22008-01-24 16:13:08 -05004269}
Chris Masond1310b22008-01-24 16:13:08 -05004270
Josef Bacik3083ee22012-03-09 16:01:49 -05004271void free_extent_buffer_stale(struct extent_buffer *eb)
4272{
4273 if (!eb)
Chris Masond1310b22008-01-24 16:13:08 -05004274 return;
4275
Josef Bacik3083ee22012-03-09 16:01:49 -05004276 spin_lock(&eb->refs_lock);
4277 set_bit(EXTENT_BUFFER_STALE, &eb->bflags);
4278
Josef Bacik0b32f4b2012-03-13 09:38:00 -04004279 if (atomic_read(&eb->refs) == 2 && !extent_buffer_under_io(eb) &&
Josef Bacik3083ee22012-03-09 16:01:49 -05004280 test_and_clear_bit(EXTENT_BUFFER_TREE_REF, &eb->bflags))
4281 atomic_dec(&eb->refs);
4282 release_extent_buffer(eb, GFP_NOFS);
Chris Masond1310b22008-01-24 16:13:08 -05004283}
4284
Chris Mason1d4284b2012-03-28 20:31:37 -04004285void clear_extent_buffer_dirty(struct extent_buffer *eb)
Chris Masond1310b22008-01-24 16:13:08 -05004286{
Chris Masond1310b22008-01-24 16:13:08 -05004287 unsigned long i;
4288 unsigned long num_pages;
4289 struct page *page;
4290
Chris Masond1310b22008-01-24 16:13:08 -05004291 num_pages = num_extent_pages(eb->start, eb->len);
4292
4293 for (i = 0; i < num_pages; i++) {
4294 page = extent_buffer_page(eb, i);
Chris Masonb9473432009-03-13 11:00:37 -04004295 if (!PageDirty(page))
Chris Masond2c3f4f2008-11-19 12:44:22 -05004296 continue;
4297
Chris Masona61e6f22008-07-22 11:18:08 -04004298 lock_page(page);
Chris Masoneb14ab82011-02-10 12:35:00 -05004299 WARN_ON(!PagePrivate(page));
4300
Chris Masond1310b22008-01-24 16:13:08 -05004301 clear_page_dirty_for_io(page);
Sven Wegener0ee0fda2008-07-30 16:54:26 -04004302 spin_lock_irq(&page->mapping->tree_lock);
Chris Masond1310b22008-01-24 16:13:08 -05004303 if (!PageDirty(page)) {
4304 radix_tree_tag_clear(&page->mapping->page_tree,
4305 page_index(page),
4306 PAGECACHE_TAG_DIRTY);
4307 }
Sven Wegener0ee0fda2008-07-30 16:54:26 -04004308 spin_unlock_irq(&page->mapping->tree_lock);
Chris Masonbf0da8c2011-11-04 12:29:37 -04004309 ClearPageError(page);
Chris Masona61e6f22008-07-22 11:18:08 -04004310 unlock_page(page);
Chris Masond1310b22008-01-24 16:13:08 -05004311 }
Josef Bacik0b32f4b2012-03-13 09:38:00 -04004312 WARN_ON(atomic_read(&eb->refs) == 0);
Chris Masond1310b22008-01-24 16:13:08 -05004313}
Chris Masond1310b22008-01-24 16:13:08 -05004314
Josef Bacik0b32f4b2012-03-13 09:38:00 -04004315int set_extent_buffer_dirty(struct extent_buffer *eb)
Chris Masond1310b22008-01-24 16:13:08 -05004316{
4317 unsigned long i;
4318 unsigned long num_pages;
Chris Masonb9473432009-03-13 11:00:37 -04004319 int was_dirty = 0;
Chris Masond1310b22008-01-24 16:13:08 -05004320
Josef Bacik0b32f4b2012-03-13 09:38:00 -04004321 check_buffer_tree_ref(eb);
4322
Chris Masonb9473432009-03-13 11:00:37 -04004323 was_dirty = test_and_set_bit(EXTENT_BUFFER_DIRTY, &eb->bflags);
Josef Bacik0b32f4b2012-03-13 09:38:00 -04004324
Chris Masond1310b22008-01-24 16:13:08 -05004325 num_pages = num_extent_pages(eb->start, eb->len);
Josef Bacik3083ee22012-03-09 16:01:49 -05004326 WARN_ON(atomic_read(&eb->refs) == 0);
Josef Bacik0b32f4b2012-03-13 09:38:00 -04004327 WARN_ON(!test_bit(EXTENT_BUFFER_TREE_REF, &eb->bflags));
4328
Chris Masonb9473432009-03-13 11:00:37 -04004329 for (i = 0; i < num_pages; i++)
Josef Bacik0b32f4b2012-03-13 09:38:00 -04004330 set_page_dirty(extent_buffer_page(eb, i));
Chris Masonb9473432009-03-13 11:00:37 -04004331 return was_dirty;
Chris Masond1310b22008-01-24 16:13:08 -05004332}
Chris Masond1310b22008-01-24 16:13:08 -05004333
Josef Bacik0b32f4b2012-03-13 09:38:00 -04004334static int range_straddles_pages(u64 start, u64 len)
Chris Mason19b6caf2011-07-25 06:50:50 -04004335{
4336 if (len < PAGE_CACHE_SIZE)
4337 return 1;
4338 if (start & (PAGE_CACHE_SIZE - 1))
4339 return 1;
4340 if ((start + len) & (PAGE_CACHE_SIZE - 1))
4341 return 1;
4342 return 0;
4343}
4344
Josef Bacik0b32f4b2012-03-13 09:38:00 -04004345int clear_extent_buffer_uptodate(struct extent_buffer *eb)
Chris Mason1259ab72008-05-12 13:39:03 -04004346{
4347 unsigned long i;
4348 struct page *page;
4349 unsigned long num_pages;
4350
Chris Masonb4ce94d2009-02-04 09:25:08 -05004351 clear_bit(EXTENT_BUFFER_UPTODATE, &eb->bflags);
Josef Bacik0b32f4b2012-03-13 09:38:00 -04004352 num_pages = num_extent_pages(eb->start, eb->len);
Chris Mason1259ab72008-05-12 13:39:03 -04004353 for (i = 0; i < num_pages; i++) {
4354 page = extent_buffer_page(eb, i);
Chris Mason33958dc2008-07-30 10:29:12 -04004355 if (page)
4356 ClearPageUptodate(page);
Chris Mason1259ab72008-05-12 13:39:03 -04004357 }
4358 return 0;
4359}
4360
Josef Bacik0b32f4b2012-03-13 09:38:00 -04004361int set_extent_buffer_uptodate(struct extent_buffer *eb)
Chris Masond1310b22008-01-24 16:13:08 -05004362{
4363 unsigned long i;
4364 struct page *page;
4365 unsigned long num_pages;
4366
Josef Bacik0b32f4b2012-03-13 09:38:00 -04004367 set_bit(EXTENT_BUFFER_UPTODATE, &eb->bflags);
Chris Masond1310b22008-01-24 16:13:08 -05004368 num_pages = num_extent_pages(eb->start, eb->len);
Chris Masond1310b22008-01-24 16:13:08 -05004369 for (i = 0; i < num_pages; i++) {
4370 page = extent_buffer_page(eb, i);
Chris Masond1310b22008-01-24 16:13:08 -05004371 SetPageUptodate(page);
4372 }
4373 return 0;
4374}
Chris Masond1310b22008-01-24 16:13:08 -05004375
Chris Masonce9adaa2008-04-09 16:28:12 -04004376int extent_range_uptodate(struct extent_io_tree *tree,
4377 u64 start, u64 end)
4378{
4379 struct page *page;
4380 int ret;
4381 int pg_uptodate = 1;
4382 int uptodate;
4383 unsigned long index;
4384
Josef Bacik0b32f4b2012-03-13 09:38:00 -04004385 if (range_straddles_pages(start, end - start + 1)) {
Chris Mason19b6caf2011-07-25 06:50:50 -04004386 ret = test_range_bit(tree, start, end,
4387 EXTENT_UPTODATE, 1, NULL);
4388 if (ret)
4389 return 1;
4390 }
Chris Masond3977122009-01-05 21:25:51 -05004391 while (start <= end) {
Chris Masonce9adaa2008-04-09 16:28:12 -04004392 index = start >> PAGE_CACHE_SHIFT;
4393 page = find_get_page(tree->mapping, index);
Mitch Harder8bedd512012-01-26 15:01:11 -05004394 if (!page)
4395 return 1;
Chris Masonce9adaa2008-04-09 16:28:12 -04004396 uptodate = PageUptodate(page);
4397 page_cache_release(page);
4398 if (!uptodate) {
4399 pg_uptodate = 0;
4400 break;
4401 }
4402 start += PAGE_CACHE_SIZE;
4403 }
4404 return pg_uptodate;
4405}
4406
Josef Bacik0b32f4b2012-03-13 09:38:00 -04004407int extent_buffer_uptodate(struct extent_buffer *eb)
Chris Masond1310b22008-01-24 16:13:08 -05004408{
Josef Bacik0b32f4b2012-03-13 09:38:00 -04004409 return test_bit(EXTENT_BUFFER_UPTODATE, &eb->bflags);
Chris Masond1310b22008-01-24 16:13:08 -05004410}
Chris Masond1310b22008-01-24 16:13:08 -05004411
4412int read_extent_buffer_pages(struct extent_io_tree *tree,
Arne Jansenbb82ab82011-06-10 14:06:53 +02004413 struct extent_buffer *eb, u64 start, int wait,
Chris Masonf1885912008-04-09 16:28:12 -04004414 get_extent_t *get_extent, int mirror_num)
Chris Masond1310b22008-01-24 16:13:08 -05004415{
4416 unsigned long i;
4417 unsigned long start_i;
4418 struct page *page;
4419 int err;
4420 int ret = 0;
Chris Masonce9adaa2008-04-09 16:28:12 -04004421 int locked_pages = 0;
4422 int all_uptodate = 1;
Chris Masond1310b22008-01-24 16:13:08 -05004423 unsigned long num_pages;
Chris Mason727011e2010-08-06 13:21:20 -04004424 unsigned long num_reads = 0;
Chris Masona86c12c2008-02-07 10:50:54 -05004425 struct bio *bio = NULL;
Chris Masonc8b97812008-10-29 14:49:59 -04004426 unsigned long bio_flags = 0;
Chris Masona86c12c2008-02-07 10:50:54 -05004427
Chris Masonb4ce94d2009-02-04 09:25:08 -05004428 if (test_bit(EXTENT_BUFFER_UPTODATE, &eb->bflags))
Chris Masond1310b22008-01-24 16:13:08 -05004429 return 0;
4430
Chris Masond1310b22008-01-24 16:13:08 -05004431 if (start) {
4432 WARN_ON(start < eb->start);
4433 start_i = (start >> PAGE_CACHE_SHIFT) -
4434 (eb->start >> PAGE_CACHE_SHIFT);
4435 } else {
4436 start_i = 0;
4437 }
4438
4439 num_pages = num_extent_pages(eb->start, eb->len);
4440 for (i = start_i; i < num_pages; i++) {
4441 page = extent_buffer_page(eb, i);
Arne Jansenbb82ab82011-06-10 14:06:53 +02004442 if (wait == WAIT_NONE) {
David Woodhouse2db04962008-08-07 11:19:43 -04004443 if (!trylock_page(page))
Chris Masonce9adaa2008-04-09 16:28:12 -04004444 goto unlock_exit;
Chris Masond1310b22008-01-24 16:13:08 -05004445 } else {
4446 lock_page(page);
4447 }
Chris Masonce9adaa2008-04-09 16:28:12 -04004448 locked_pages++;
Chris Mason727011e2010-08-06 13:21:20 -04004449 if (!PageUptodate(page)) {
4450 num_reads++;
Chris Masonce9adaa2008-04-09 16:28:12 -04004451 all_uptodate = 0;
Chris Mason727011e2010-08-06 13:21:20 -04004452 }
Chris Masonce9adaa2008-04-09 16:28:12 -04004453 }
4454 if (all_uptodate) {
4455 if (start_i == 0)
Chris Masonb4ce94d2009-02-04 09:25:08 -05004456 set_bit(EXTENT_BUFFER_UPTODATE, &eb->bflags);
Chris Masonce9adaa2008-04-09 16:28:12 -04004457 goto unlock_exit;
4458 }
4459
Josef Bacikea466792012-03-26 21:57:36 -04004460 clear_bit(EXTENT_BUFFER_IOERR, &eb->bflags);
4461 eb->failed_mirror = 0;
Josef Bacik0b32f4b2012-03-13 09:38:00 -04004462 atomic_set(&eb->io_pages, num_reads);
Chris Masonce9adaa2008-04-09 16:28:12 -04004463 for (i = start_i; i < num_pages; i++) {
4464 page = extent_buffer_page(eb, i);
Chris Masonce9adaa2008-04-09 16:28:12 -04004465 if (!PageUptodate(page)) {
Chris Masonf1885912008-04-09 16:28:12 -04004466 ClearPageError(page);
Chris Masona86c12c2008-02-07 10:50:54 -05004467 err = __extent_read_full_page(tree, page,
Chris Masonf1885912008-04-09 16:28:12 -04004468 get_extent, &bio,
Chris Masonc8b97812008-10-29 14:49:59 -04004469 mirror_num, &bio_flags);
Chris Masond3977122009-01-05 21:25:51 -05004470 if (err)
Chris Masond1310b22008-01-24 16:13:08 -05004471 ret = err;
Chris Masond1310b22008-01-24 16:13:08 -05004472 } else {
4473 unlock_page(page);
4474 }
4475 }
4476
Jeff Mahoney355808c2011-10-03 23:23:14 -04004477 if (bio) {
4478 err = submit_one_bio(READ, bio, mirror_num, bio_flags);
Jeff Mahoney79787ea2012-03-12 16:03:00 +01004479 if (err)
4480 return err;
Jeff Mahoney355808c2011-10-03 23:23:14 -04004481 }
Chris Masona86c12c2008-02-07 10:50:54 -05004482
Arne Jansenbb82ab82011-06-10 14:06:53 +02004483 if (ret || wait != WAIT_COMPLETE)
Chris Masond1310b22008-01-24 16:13:08 -05004484 return ret;
Chris Masond3977122009-01-05 21:25:51 -05004485
Chris Masond1310b22008-01-24 16:13:08 -05004486 for (i = start_i; i < num_pages; i++) {
4487 page = extent_buffer_page(eb, i);
4488 wait_on_page_locked(page);
Chris Masond3977122009-01-05 21:25:51 -05004489 if (!PageUptodate(page))
Chris Masond1310b22008-01-24 16:13:08 -05004490 ret = -EIO;
Chris Masond1310b22008-01-24 16:13:08 -05004491 }
Chris Masond3977122009-01-05 21:25:51 -05004492
Chris Masond1310b22008-01-24 16:13:08 -05004493 return ret;
Chris Masonce9adaa2008-04-09 16:28:12 -04004494
4495unlock_exit:
4496 i = start_i;
Chris Masond3977122009-01-05 21:25:51 -05004497 while (locked_pages > 0) {
Chris Masonce9adaa2008-04-09 16:28:12 -04004498 page = extent_buffer_page(eb, i);
4499 i++;
4500 unlock_page(page);
4501 locked_pages--;
4502 }
4503 return ret;
Chris Masond1310b22008-01-24 16:13:08 -05004504}
Chris Masond1310b22008-01-24 16:13:08 -05004505
4506void read_extent_buffer(struct extent_buffer *eb, void *dstv,
4507 unsigned long start,
4508 unsigned long len)
4509{
4510 size_t cur;
4511 size_t offset;
4512 struct page *page;
4513 char *kaddr;
4514 char *dst = (char *)dstv;
4515 size_t start_offset = eb->start & ((u64)PAGE_CACHE_SIZE - 1);
4516 unsigned long i = (start_offset + start) >> PAGE_CACHE_SHIFT;
Chris Masond1310b22008-01-24 16:13:08 -05004517
4518 WARN_ON(start > eb->len);
4519 WARN_ON(start + len > eb->start + eb->len);
4520
4521 offset = (start_offset + start) & ((unsigned long)PAGE_CACHE_SIZE - 1);
4522
Chris Masond3977122009-01-05 21:25:51 -05004523 while (len > 0) {
Chris Masond1310b22008-01-24 16:13:08 -05004524 page = extent_buffer_page(eb, i);
Chris Masond1310b22008-01-24 16:13:08 -05004525
4526 cur = min(len, (PAGE_CACHE_SIZE - offset));
Chris Masona6591712011-07-19 12:04:14 -04004527 kaddr = page_address(page);
Chris Masond1310b22008-01-24 16:13:08 -05004528 memcpy(dst, kaddr + offset, cur);
Chris Masond1310b22008-01-24 16:13:08 -05004529
4530 dst += cur;
4531 len -= cur;
4532 offset = 0;
4533 i++;
4534 }
4535}
Chris Masond1310b22008-01-24 16:13:08 -05004536
4537int map_private_extent_buffer(struct extent_buffer *eb, unsigned long start,
Chris Masona6591712011-07-19 12:04:14 -04004538 unsigned long min_len, char **map,
Chris Masond1310b22008-01-24 16:13:08 -05004539 unsigned long *map_start,
Chris Masona6591712011-07-19 12:04:14 -04004540 unsigned long *map_len)
Chris Masond1310b22008-01-24 16:13:08 -05004541{
4542 size_t offset = start & (PAGE_CACHE_SIZE - 1);
4543 char *kaddr;
4544 struct page *p;
4545 size_t start_offset = eb->start & ((u64)PAGE_CACHE_SIZE - 1);
4546 unsigned long i = (start_offset + start) >> PAGE_CACHE_SHIFT;
4547 unsigned long end_i = (start_offset + start + min_len - 1) >>
4548 PAGE_CACHE_SHIFT;
4549
4550 if (i != end_i)
4551 return -EINVAL;
4552
4553 if (i == 0) {
4554 offset = start_offset;
4555 *map_start = 0;
4556 } else {
4557 offset = 0;
4558 *map_start = ((u64)i << PAGE_CACHE_SHIFT) - start_offset;
4559 }
Chris Masond3977122009-01-05 21:25:51 -05004560
Chris Masond1310b22008-01-24 16:13:08 -05004561 if (start + min_len > eb->len) {
Chris Masond3977122009-01-05 21:25:51 -05004562 printk(KERN_ERR "btrfs bad mapping eb start %llu len %lu, "
4563 "wanted %lu %lu\n", (unsigned long long)eb->start,
4564 eb->len, start, min_len);
Chris Masond1310b22008-01-24 16:13:08 -05004565 WARN_ON(1);
Josef Bacik850265332011-03-15 14:52:12 -04004566 return -EINVAL;
Chris Masond1310b22008-01-24 16:13:08 -05004567 }
4568
4569 p = extent_buffer_page(eb, i);
Chris Masona6591712011-07-19 12:04:14 -04004570 kaddr = page_address(p);
Chris Masond1310b22008-01-24 16:13:08 -05004571 *map = kaddr + offset;
4572 *map_len = PAGE_CACHE_SIZE - offset;
4573 return 0;
4574}
Chris Masond1310b22008-01-24 16:13:08 -05004575
Chris Masond1310b22008-01-24 16:13:08 -05004576int memcmp_extent_buffer(struct extent_buffer *eb, const void *ptrv,
4577 unsigned long start,
4578 unsigned long len)
4579{
4580 size_t cur;
4581 size_t offset;
4582 struct page *page;
4583 char *kaddr;
4584 char *ptr = (char *)ptrv;
4585 size_t start_offset = eb->start & ((u64)PAGE_CACHE_SIZE - 1);
4586 unsigned long i = (start_offset + start) >> PAGE_CACHE_SHIFT;
4587 int ret = 0;
4588
4589 WARN_ON(start > eb->len);
4590 WARN_ON(start + len > eb->start + eb->len);
4591
4592 offset = (start_offset + start) & ((unsigned long)PAGE_CACHE_SIZE - 1);
4593
Chris Masond3977122009-01-05 21:25:51 -05004594 while (len > 0) {
Chris Masond1310b22008-01-24 16:13:08 -05004595 page = extent_buffer_page(eb, i);
Chris Masond1310b22008-01-24 16:13:08 -05004596
4597 cur = min(len, (PAGE_CACHE_SIZE - offset));
4598
Chris Masona6591712011-07-19 12:04:14 -04004599 kaddr = page_address(page);
Chris Masond1310b22008-01-24 16:13:08 -05004600 ret = memcmp(ptr, kaddr + offset, cur);
Chris Masond1310b22008-01-24 16:13:08 -05004601 if (ret)
4602 break;
4603
4604 ptr += cur;
4605 len -= cur;
4606 offset = 0;
4607 i++;
4608 }
4609 return ret;
4610}
Chris Masond1310b22008-01-24 16:13:08 -05004611
4612void write_extent_buffer(struct extent_buffer *eb, const void *srcv,
4613 unsigned long start, unsigned long len)
4614{
4615 size_t cur;
4616 size_t offset;
4617 struct page *page;
4618 char *kaddr;
4619 char *src = (char *)srcv;
4620 size_t start_offset = eb->start & ((u64)PAGE_CACHE_SIZE - 1);
4621 unsigned long i = (start_offset + start) >> PAGE_CACHE_SHIFT;
4622
4623 WARN_ON(start > eb->len);
4624 WARN_ON(start + len > eb->start + eb->len);
4625
4626 offset = (start_offset + start) & ((unsigned long)PAGE_CACHE_SIZE - 1);
4627
Chris Masond3977122009-01-05 21:25:51 -05004628 while (len > 0) {
Chris Masond1310b22008-01-24 16:13:08 -05004629 page = extent_buffer_page(eb, i);
4630 WARN_ON(!PageUptodate(page));
4631
4632 cur = min(len, PAGE_CACHE_SIZE - offset);
Chris Masona6591712011-07-19 12:04:14 -04004633 kaddr = page_address(page);
Chris Masond1310b22008-01-24 16:13:08 -05004634 memcpy(kaddr + offset, src, cur);
Chris Masond1310b22008-01-24 16:13:08 -05004635
4636 src += cur;
4637 len -= cur;
4638 offset = 0;
4639 i++;
4640 }
4641}
Chris Masond1310b22008-01-24 16:13:08 -05004642
4643void memset_extent_buffer(struct extent_buffer *eb, char c,
4644 unsigned long start, unsigned long len)
4645{
4646 size_t cur;
4647 size_t offset;
4648 struct page *page;
4649 char *kaddr;
4650 size_t start_offset = eb->start & ((u64)PAGE_CACHE_SIZE - 1);
4651 unsigned long i = (start_offset + start) >> PAGE_CACHE_SHIFT;
4652
4653 WARN_ON(start > eb->len);
4654 WARN_ON(start + len > eb->start + eb->len);
4655
4656 offset = (start_offset + start) & ((unsigned long)PAGE_CACHE_SIZE - 1);
4657
Chris Masond3977122009-01-05 21:25:51 -05004658 while (len > 0) {
Chris Masond1310b22008-01-24 16:13:08 -05004659 page = extent_buffer_page(eb, i);
4660 WARN_ON(!PageUptodate(page));
4661
4662 cur = min(len, PAGE_CACHE_SIZE - offset);
Chris Masona6591712011-07-19 12:04:14 -04004663 kaddr = page_address(page);
Chris Masond1310b22008-01-24 16:13:08 -05004664 memset(kaddr + offset, c, cur);
Chris Masond1310b22008-01-24 16:13:08 -05004665
4666 len -= cur;
4667 offset = 0;
4668 i++;
4669 }
4670}
Chris Masond1310b22008-01-24 16:13:08 -05004671
4672void copy_extent_buffer(struct extent_buffer *dst, struct extent_buffer *src,
4673 unsigned long dst_offset, unsigned long src_offset,
4674 unsigned long len)
4675{
4676 u64 dst_len = dst->len;
4677 size_t cur;
4678 size_t offset;
4679 struct page *page;
4680 char *kaddr;
4681 size_t start_offset = dst->start & ((u64)PAGE_CACHE_SIZE - 1);
4682 unsigned long i = (start_offset + dst_offset) >> PAGE_CACHE_SHIFT;
4683
4684 WARN_ON(src->len != dst_len);
4685
4686 offset = (start_offset + dst_offset) &
4687 ((unsigned long)PAGE_CACHE_SIZE - 1);
4688
Chris Masond3977122009-01-05 21:25:51 -05004689 while (len > 0) {
Chris Masond1310b22008-01-24 16:13:08 -05004690 page = extent_buffer_page(dst, i);
4691 WARN_ON(!PageUptodate(page));
4692
4693 cur = min(len, (unsigned long)(PAGE_CACHE_SIZE - offset));
4694
Chris Masona6591712011-07-19 12:04:14 -04004695 kaddr = page_address(page);
Chris Masond1310b22008-01-24 16:13:08 -05004696 read_extent_buffer(src, kaddr + offset, src_offset, cur);
Chris Masond1310b22008-01-24 16:13:08 -05004697
4698 src_offset += cur;
4699 len -= cur;
4700 offset = 0;
4701 i++;
4702 }
4703}
Chris Masond1310b22008-01-24 16:13:08 -05004704
4705static void move_pages(struct page *dst_page, struct page *src_page,
4706 unsigned long dst_off, unsigned long src_off,
4707 unsigned long len)
4708{
Chris Masona6591712011-07-19 12:04:14 -04004709 char *dst_kaddr = page_address(dst_page);
Chris Masond1310b22008-01-24 16:13:08 -05004710 if (dst_page == src_page) {
4711 memmove(dst_kaddr + dst_off, dst_kaddr + src_off, len);
4712 } else {
Chris Masona6591712011-07-19 12:04:14 -04004713 char *src_kaddr = page_address(src_page);
Chris Masond1310b22008-01-24 16:13:08 -05004714 char *p = dst_kaddr + dst_off + len;
4715 char *s = src_kaddr + src_off + len;
4716
4717 while (len--)
4718 *--p = *--s;
Chris Masond1310b22008-01-24 16:13:08 -05004719 }
Chris Masond1310b22008-01-24 16:13:08 -05004720}
4721
Sergei Trofimovich33872062011-04-11 21:52:52 +00004722static inline bool areas_overlap(unsigned long src, unsigned long dst, unsigned long len)
4723{
4724 unsigned long distance = (src > dst) ? src - dst : dst - src;
4725 return distance < len;
4726}
4727
Chris Masond1310b22008-01-24 16:13:08 -05004728static void copy_pages(struct page *dst_page, struct page *src_page,
4729 unsigned long dst_off, unsigned long src_off,
4730 unsigned long len)
4731{
Chris Masona6591712011-07-19 12:04:14 -04004732 char *dst_kaddr = page_address(dst_page);
Chris Masond1310b22008-01-24 16:13:08 -05004733 char *src_kaddr;
Chris Mason727011e2010-08-06 13:21:20 -04004734 int must_memmove = 0;
Chris Masond1310b22008-01-24 16:13:08 -05004735
Sergei Trofimovich33872062011-04-11 21:52:52 +00004736 if (dst_page != src_page) {
Chris Masona6591712011-07-19 12:04:14 -04004737 src_kaddr = page_address(src_page);
Sergei Trofimovich33872062011-04-11 21:52:52 +00004738 } else {
Chris Masond1310b22008-01-24 16:13:08 -05004739 src_kaddr = dst_kaddr;
Chris Mason727011e2010-08-06 13:21:20 -04004740 if (areas_overlap(src_off, dst_off, len))
4741 must_memmove = 1;
Sergei Trofimovich33872062011-04-11 21:52:52 +00004742 }
Chris Masond1310b22008-01-24 16:13:08 -05004743
Chris Mason727011e2010-08-06 13:21:20 -04004744 if (must_memmove)
4745 memmove(dst_kaddr + dst_off, src_kaddr + src_off, len);
4746 else
4747 memcpy(dst_kaddr + dst_off, src_kaddr + src_off, len);
Chris Masond1310b22008-01-24 16:13:08 -05004748}
4749
4750void memcpy_extent_buffer(struct extent_buffer *dst, unsigned long dst_offset,
4751 unsigned long src_offset, unsigned long len)
4752{
4753 size_t cur;
4754 size_t dst_off_in_page;
4755 size_t src_off_in_page;
4756 size_t start_offset = dst->start & ((u64)PAGE_CACHE_SIZE - 1);
4757 unsigned long dst_i;
4758 unsigned long src_i;
4759
4760 if (src_offset + len > dst->len) {
Chris Masond3977122009-01-05 21:25:51 -05004761 printk(KERN_ERR "btrfs memmove bogus src_offset %lu move "
4762 "len %lu dst len %lu\n", src_offset, len, dst->len);
Chris Masond1310b22008-01-24 16:13:08 -05004763 BUG_ON(1);
4764 }
4765 if (dst_offset + len > dst->len) {
Chris Masond3977122009-01-05 21:25:51 -05004766 printk(KERN_ERR "btrfs memmove bogus dst_offset %lu move "
4767 "len %lu dst len %lu\n", dst_offset, len, dst->len);
Chris Masond1310b22008-01-24 16:13:08 -05004768 BUG_ON(1);
4769 }
4770
Chris Masond3977122009-01-05 21:25:51 -05004771 while (len > 0) {
Chris Masond1310b22008-01-24 16:13:08 -05004772 dst_off_in_page = (start_offset + dst_offset) &
4773 ((unsigned long)PAGE_CACHE_SIZE - 1);
4774 src_off_in_page = (start_offset + src_offset) &
4775 ((unsigned long)PAGE_CACHE_SIZE - 1);
4776
4777 dst_i = (start_offset + dst_offset) >> PAGE_CACHE_SHIFT;
4778 src_i = (start_offset + src_offset) >> PAGE_CACHE_SHIFT;
4779
4780 cur = min(len, (unsigned long)(PAGE_CACHE_SIZE -
4781 src_off_in_page));
4782 cur = min_t(unsigned long, cur,
4783 (unsigned long)(PAGE_CACHE_SIZE - dst_off_in_page));
4784
4785 copy_pages(extent_buffer_page(dst, dst_i),
4786 extent_buffer_page(dst, src_i),
4787 dst_off_in_page, src_off_in_page, cur);
4788
4789 src_offset += cur;
4790 dst_offset += cur;
4791 len -= cur;
4792 }
4793}
Chris Masond1310b22008-01-24 16:13:08 -05004794
4795void memmove_extent_buffer(struct extent_buffer *dst, unsigned long dst_offset,
4796 unsigned long src_offset, unsigned long len)
4797{
4798 size_t cur;
4799 size_t dst_off_in_page;
4800 size_t src_off_in_page;
4801 unsigned long dst_end = dst_offset + len - 1;
4802 unsigned long src_end = src_offset + len - 1;
4803 size_t start_offset = dst->start & ((u64)PAGE_CACHE_SIZE - 1);
4804 unsigned long dst_i;
4805 unsigned long src_i;
4806
4807 if (src_offset + len > dst->len) {
Chris Masond3977122009-01-05 21:25:51 -05004808 printk(KERN_ERR "btrfs memmove bogus src_offset %lu move "
4809 "len %lu len %lu\n", src_offset, len, dst->len);
Chris Masond1310b22008-01-24 16:13:08 -05004810 BUG_ON(1);
4811 }
4812 if (dst_offset + len > dst->len) {
Chris Masond3977122009-01-05 21:25:51 -05004813 printk(KERN_ERR "btrfs memmove bogus dst_offset %lu move "
4814 "len %lu len %lu\n", dst_offset, len, dst->len);
Chris Masond1310b22008-01-24 16:13:08 -05004815 BUG_ON(1);
4816 }
Chris Mason727011e2010-08-06 13:21:20 -04004817 if (dst_offset < src_offset) {
Chris Masond1310b22008-01-24 16:13:08 -05004818 memcpy_extent_buffer(dst, dst_offset, src_offset, len);
4819 return;
4820 }
Chris Masond3977122009-01-05 21:25:51 -05004821 while (len > 0) {
Chris Masond1310b22008-01-24 16:13:08 -05004822 dst_i = (start_offset + dst_end) >> PAGE_CACHE_SHIFT;
4823 src_i = (start_offset + src_end) >> PAGE_CACHE_SHIFT;
4824
4825 dst_off_in_page = (start_offset + dst_end) &
4826 ((unsigned long)PAGE_CACHE_SIZE - 1);
4827 src_off_in_page = (start_offset + src_end) &
4828 ((unsigned long)PAGE_CACHE_SIZE - 1);
4829
4830 cur = min_t(unsigned long, len, src_off_in_page + 1);
4831 cur = min(cur, dst_off_in_page + 1);
4832 move_pages(extent_buffer_page(dst, dst_i),
4833 extent_buffer_page(dst, src_i),
4834 dst_off_in_page - cur + 1,
4835 src_off_in_page - cur + 1, cur);
4836
4837 dst_end -= cur;
4838 src_end -= cur;
4839 len -= cur;
4840 }
4841}
Chris Mason6af118ce2008-07-22 11:18:07 -04004842
Josef Bacik3083ee22012-03-09 16:01:49 -05004843int try_release_extent_buffer(struct page *page, gfp_t mask)
Miao Xie19fe0a82010-10-26 20:57:29 -04004844{
Chris Mason6af118ce2008-07-22 11:18:07 -04004845 struct extent_buffer *eb;
Miao Xie897ca6e2010-10-26 20:57:29 -04004846
Miao Xie19fe0a82010-10-26 20:57:29 -04004847 /*
Josef Bacik3083ee22012-03-09 16:01:49 -05004848 * We need to make sure noboody is attaching this page to an eb right
4849 * now.
Miao Xie19fe0a82010-10-26 20:57:29 -04004850 */
Josef Bacik3083ee22012-03-09 16:01:49 -05004851 spin_lock(&page->mapping->private_lock);
4852 if (!PagePrivate(page)) {
4853 spin_unlock(&page->mapping->private_lock);
4854 return 1;
Miao Xie19fe0a82010-10-26 20:57:29 -04004855 }
4856
Josef Bacik3083ee22012-03-09 16:01:49 -05004857 eb = (struct extent_buffer *)page->private;
4858 BUG_ON(!eb);
Miao Xie19fe0a82010-10-26 20:57:29 -04004859
Josef Bacik0b32f4b2012-03-13 09:38:00 -04004860 /*
Josef Bacik3083ee22012-03-09 16:01:49 -05004861 * This is a little awful but should be ok, we need to make sure that
4862 * the eb doesn't disappear out from under us while we're looking at
4863 * this page.
4864 */
4865 spin_lock(&eb->refs_lock);
Josef Bacik0b32f4b2012-03-13 09:38:00 -04004866 if (atomic_read(&eb->refs) != 1 || extent_buffer_under_io(eb)) {
Josef Bacik3083ee22012-03-09 16:01:49 -05004867 spin_unlock(&eb->refs_lock);
4868 spin_unlock(&page->mapping->private_lock);
4869 return 0;
4870 }
4871 spin_unlock(&page->mapping->private_lock);
4872
4873 if ((mask & GFP_NOFS) == GFP_NOFS)
4874 mask = GFP_NOFS;
4875
4876 /*
4877 * If tree ref isn't set then we know the ref on this eb is a real ref,
4878 * so just return, this page will likely be freed soon anyway.
4879 */
4880 if (!test_and_clear_bit(EXTENT_BUFFER_TREE_REF, &eb->bflags)) {
4881 spin_unlock(&eb->refs_lock);
4882 return 0;
4883 }
4884 release_extent_buffer(eb, mask);
4885
4886 return 1;
Chris Mason6af118ce2008-07-22 11:18:07 -04004887}