blob: b181a94a717058217e25c73499438176f703097a [file] [log] [blame]
Chris Masond1310b22008-01-24 16:13:08 -05001#include <linux/bitops.h>
2#include <linux/slab.h>
3#include <linux/bio.h>
4#include <linux/mm.h>
Chris Masond1310b22008-01-24 16:13:08 -05005#include <linux/pagemap.h>
6#include <linux/page-flags.h>
7#include <linux/module.h>
8#include <linux/spinlock.h>
9#include <linux/blkdev.h>
10#include <linux/swap.h>
Chris Masond1310b22008-01-24 16:13:08 -050011#include <linux/writeback.h>
12#include <linux/pagevec.h>
13#include "extent_io.h"
14#include "extent_map.h"
David Woodhouse2db04962008-08-07 11:19:43 -040015#include "compat.h"
David Woodhouse902b22f2008-08-20 08:51:49 -040016#include "ctree.h"
17#include "btrfs_inode.h"
Chris Masond1310b22008-01-24 16:13:08 -050018
Chris Masond1310b22008-01-24 16:13:08 -050019static struct kmem_cache *extent_state_cache;
20static struct kmem_cache *extent_buffer_cache;
21
22static LIST_HEAD(buffers);
23static LIST_HEAD(states);
Chris Mason4bef0842008-09-08 11:18:08 -040024
Chris Masonb47eda82008-11-10 12:34:40 -050025#define LEAK_DEBUG 0
Chris Mason39351272009-02-04 09:24:05 -050026#if LEAK_DEBUG
Chris Masond3977122009-01-05 21:25:51 -050027static DEFINE_SPINLOCK(leak_lock);
Chris Mason4bef0842008-09-08 11:18:08 -040028#endif
Chris Masond1310b22008-01-24 16:13:08 -050029
Chris Masond1310b22008-01-24 16:13:08 -050030#define BUFFER_LRU_MAX 64
31
32struct tree_entry {
33 u64 start;
34 u64 end;
Chris Masond1310b22008-01-24 16:13:08 -050035 struct rb_node rb_node;
36};
37
38struct extent_page_data {
39 struct bio *bio;
40 struct extent_io_tree *tree;
41 get_extent_t *get_extent;
Chris Mason771ed682008-11-06 22:02:51 -050042
43 /* tells writepage not to lock the state bits for this range
44 * it still does the unlocking
45 */
Chris Masonffbd5172009-04-20 15:50:09 -040046 unsigned int extent_locked:1;
47
48 /* tells the submit_bio code to use a WRITE_SYNC */
49 unsigned int sync_io:1;
Chris Masond1310b22008-01-24 16:13:08 -050050};
51
52int __init extent_io_init(void)
53{
Christoph Hellwig9601e3f2009-04-13 15:33:09 +020054 extent_state_cache = kmem_cache_create("extent_state",
55 sizeof(struct extent_state), 0,
56 SLAB_RECLAIM_ACCOUNT | SLAB_MEM_SPREAD, NULL);
Chris Masond1310b22008-01-24 16:13:08 -050057 if (!extent_state_cache)
58 return -ENOMEM;
59
Christoph Hellwig9601e3f2009-04-13 15:33:09 +020060 extent_buffer_cache = kmem_cache_create("extent_buffers",
61 sizeof(struct extent_buffer), 0,
62 SLAB_RECLAIM_ACCOUNT | SLAB_MEM_SPREAD, NULL);
Chris Masond1310b22008-01-24 16:13:08 -050063 if (!extent_buffer_cache)
64 goto free_state_cache;
65 return 0;
66
67free_state_cache:
68 kmem_cache_destroy(extent_state_cache);
69 return -ENOMEM;
70}
71
72void extent_io_exit(void)
73{
74 struct extent_state *state;
Chris Mason2d2ae542008-03-26 16:24:23 -040075 struct extent_buffer *eb;
Chris Masond1310b22008-01-24 16:13:08 -050076
77 while (!list_empty(&states)) {
Chris Mason2d2ae542008-03-26 16:24:23 -040078 state = list_entry(states.next, struct extent_state, leak_list);
Chris Masond3977122009-01-05 21:25:51 -050079 printk(KERN_ERR "btrfs state leak: start %llu end %llu "
80 "state %lu in tree %p refs %d\n",
81 (unsigned long long)state->start,
82 (unsigned long long)state->end,
83 state->state, state->tree, atomic_read(&state->refs));
Chris Mason2d2ae542008-03-26 16:24:23 -040084 list_del(&state->leak_list);
Chris Masond1310b22008-01-24 16:13:08 -050085 kmem_cache_free(extent_state_cache, state);
86
87 }
88
Chris Mason2d2ae542008-03-26 16:24:23 -040089 while (!list_empty(&buffers)) {
90 eb = list_entry(buffers.next, struct extent_buffer, leak_list);
Chris Masond3977122009-01-05 21:25:51 -050091 printk(KERN_ERR "btrfs buffer leak start %llu len %lu "
92 "refs %d\n", (unsigned long long)eb->start,
93 eb->len, atomic_read(&eb->refs));
Chris Mason2d2ae542008-03-26 16:24:23 -040094 list_del(&eb->leak_list);
95 kmem_cache_free(extent_buffer_cache, eb);
96 }
Chris Masond1310b22008-01-24 16:13:08 -050097 if (extent_state_cache)
98 kmem_cache_destroy(extent_state_cache);
99 if (extent_buffer_cache)
100 kmem_cache_destroy(extent_buffer_cache);
101}
102
103void extent_io_tree_init(struct extent_io_tree *tree,
David Sterbaf993c882011-04-20 23:35:57 +0200104 struct address_space *mapping)
Chris Masond1310b22008-01-24 16:13:08 -0500105{
Eric Paris6bef4d32010-02-23 19:43:04 +0000106 tree->state = RB_ROOT;
Miao Xie19fe0a82010-10-26 20:57:29 -0400107 INIT_RADIX_TREE(&tree->buffer, GFP_ATOMIC);
Chris Masond1310b22008-01-24 16:13:08 -0500108 tree->ops = NULL;
109 tree->dirty_bytes = 0;
Chris Mason70dec802008-01-29 09:59:12 -0500110 spin_lock_init(&tree->lock);
Chris Mason6af118ce2008-07-22 11:18:07 -0400111 spin_lock_init(&tree->buffer_lock);
Chris Masond1310b22008-01-24 16:13:08 -0500112 tree->mapping = mapping;
Chris Masond1310b22008-01-24 16:13:08 -0500113}
Chris Masond1310b22008-01-24 16:13:08 -0500114
Christoph Hellwigb2950862008-12-02 09:54:17 -0500115static struct extent_state *alloc_extent_state(gfp_t mask)
Chris Masond1310b22008-01-24 16:13:08 -0500116{
117 struct extent_state *state;
Chris Mason39351272009-02-04 09:24:05 -0500118#if LEAK_DEBUG
Chris Mason2d2ae542008-03-26 16:24:23 -0400119 unsigned long flags;
Chris Mason4bef0842008-09-08 11:18:08 -0400120#endif
Chris Masond1310b22008-01-24 16:13:08 -0500121
122 state = kmem_cache_alloc(extent_state_cache, mask);
Peter2b114d12008-04-01 11:21:40 -0400123 if (!state)
Chris Masond1310b22008-01-24 16:13:08 -0500124 return state;
125 state->state = 0;
Chris Masond1310b22008-01-24 16:13:08 -0500126 state->private = 0;
Chris Mason70dec802008-01-29 09:59:12 -0500127 state->tree = NULL;
Chris Mason39351272009-02-04 09:24:05 -0500128#if LEAK_DEBUG
Chris Mason2d2ae542008-03-26 16:24:23 -0400129 spin_lock_irqsave(&leak_lock, flags);
130 list_add(&state->leak_list, &states);
131 spin_unlock_irqrestore(&leak_lock, flags);
Chris Mason4bef0842008-09-08 11:18:08 -0400132#endif
Chris Masond1310b22008-01-24 16:13:08 -0500133 atomic_set(&state->refs, 1);
134 init_waitqueue_head(&state->wq);
135 return state;
136}
Chris Masond1310b22008-01-24 16:13:08 -0500137
Chris Mason4845e442010-05-25 20:56:50 -0400138void free_extent_state(struct extent_state *state)
Chris Masond1310b22008-01-24 16:13:08 -0500139{
Chris Masond1310b22008-01-24 16:13:08 -0500140 if (!state)
141 return;
142 if (atomic_dec_and_test(&state->refs)) {
Chris Mason39351272009-02-04 09:24:05 -0500143#if LEAK_DEBUG
Chris Mason2d2ae542008-03-26 16:24:23 -0400144 unsigned long flags;
Chris Mason4bef0842008-09-08 11:18:08 -0400145#endif
Chris Mason70dec802008-01-29 09:59:12 -0500146 WARN_ON(state->tree);
Chris Mason39351272009-02-04 09:24:05 -0500147#if LEAK_DEBUG
Chris Mason2d2ae542008-03-26 16:24:23 -0400148 spin_lock_irqsave(&leak_lock, flags);
149 list_del(&state->leak_list);
150 spin_unlock_irqrestore(&leak_lock, flags);
Chris Mason4bef0842008-09-08 11:18:08 -0400151#endif
Chris Masond1310b22008-01-24 16:13:08 -0500152 kmem_cache_free(extent_state_cache, state);
153 }
154}
Chris Masond1310b22008-01-24 16:13:08 -0500155
156static struct rb_node *tree_insert(struct rb_root *root, u64 offset,
157 struct rb_node *node)
158{
Chris Masond3977122009-01-05 21:25:51 -0500159 struct rb_node **p = &root->rb_node;
160 struct rb_node *parent = NULL;
Chris Masond1310b22008-01-24 16:13:08 -0500161 struct tree_entry *entry;
162
Chris Masond3977122009-01-05 21:25:51 -0500163 while (*p) {
Chris Masond1310b22008-01-24 16:13:08 -0500164 parent = *p;
165 entry = rb_entry(parent, struct tree_entry, rb_node);
166
167 if (offset < entry->start)
168 p = &(*p)->rb_left;
169 else if (offset > entry->end)
170 p = &(*p)->rb_right;
171 else
172 return parent;
173 }
174
175 entry = rb_entry(node, struct tree_entry, rb_node);
Chris Masond1310b22008-01-24 16:13:08 -0500176 rb_link_node(node, parent, p);
177 rb_insert_color(node, root);
178 return NULL;
179}
180
Chris Mason80ea96b2008-02-01 14:51:59 -0500181static struct rb_node *__etree_search(struct extent_io_tree *tree, u64 offset,
Chris Masond1310b22008-01-24 16:13:08 -0500182 struct rb_node **prev_ret,
183 struct rb_node **next_ret)
184{
Chris Mason80ea96b2008-02-01 14:51:59 -0500185 struct rb_root *root = &tree->state;
Chris Masond3977122009-01-05 21:25:51 -0500186 struct rb_node *n = root->rb_node;
Chris Masond1310b22008-01-24 16:13:08 -0500187 struct rb_node *prev = NULL;
188 struct rb_node *orig_prev = NULL;
189 struct tree_entry *entry;
190 struct tree_entry *prev_entry = NULL;
191
Chris Masond3977122009-01-05 21:25:51 -0500192 while (n) {
Chris Masond1310b22008-01-24 16:13:08 -0500193 entry = rb_entry(n, struct tree_entry, rb_node);
194 prev = n;
195 prev_entry = entry;
196
197 if (offset < entry->start)
198 n = n->rb_left;
199 else if (offset > entry->end)
200 n = n->rb_right;
Chris Masond3977122009-01-05 21:25:51 -0500201 else
Chris Masond1310b22008-01-24 16:13:08 -0500202 return n;
203 }
204
205 if (prev_ret) {
206 orig_prev = prev;
Chris Masond3977122009-01-05 21:25:51 -0500207 while (prev && offset > prev_entry->end) {
Chris Masond1310b22008-01-24 16:13:08 -0500208 prev = rb_next(prev);
209 prev_entry = rb_entry(prev, struct tree_entry, rb_node);
210 }
211 *prev_ret = prev;
212 prev = orig_prev;
213 }
214
215 if (next_ret) {
216 prev_entry = rb_entry(prev, struct tree_entry, rb_node);
Chris Masond3977122009-01-05 21:25:51 -0500217 while (prev && offset < prev_entry->start) {
Chris Masond1310b22008-01-24 16:13:08 -0500218 prev = rb_prev(prev);
219 prev_entry = rb_entry(prev, struct tree_entry, rb_node);
220 }
221 *next_ret = prev;
222 }
223 return NULL;
224}
225
Chris Mason80ea96b2008-02-01 14:51:59 -0500226static inline struct rb_node *tree_search(struct extent_io_tree *tree,
227 u64 offset)
Chris Masond1310b22008-01-24 16:13:08 -0500228{
Chris Mason70dec802008-01-29 09:59:12 -0500229 struct rb_node *prev = NULL;
Chris Masond1310b22008-01-24 16:13:08 -0500230 struct rb_node *ret;
Chris Mason70dec802008-01-29 09:59:12 -0500231
Chris Mason80ea96b2008-02-01 14:51:59 -0500232 ret = __etree_search(tree, offset, &prev, NULL);
Chris Masond3977122009-01-05 21:25:51 -0500233 if (!ret)
Chris Masond1310b22008-01-24 16:13:08 -0500234 return prev;
235 return ret;
236}
237
Josef Bacik9ed74f22009-09-11 16:12:44 -0400238static void merge_cb(struct extent_io_tree *tree, struct extent_state *new,
239 struct extent_state *other)
240{
241 if (tree->ops && tree->ops->merge_extent_hook)
242 tree->ops->merge_extent_hook(tree->mapping->host, new,
243 other);
244}
245
Chris Masond1310b22008-01-24 16:13:08 -0500246/*
247 * utility function to look for merge candidates inside a given range.
248 * Any extents with matching state are merged together into a single
249 * extent in the tree. Extents with EXTENT_IO in their state field
250 * are not merged because the end_io handlers need to be able to do
251 * operations on them without sleeping (or doing allocations/splits).
252 *
253 * This should be called with the tree lock held.
254 */
255static int merge_state(struct extent_io_tree *tree,
256 struct extent_state *state)
257{
258 struct extent_state *other;
259 struct rb_node *other_node;
260
Zheng Yan5b21f2e2008-09-26 10:05:38 -0400261 if (state->state & (EXTENT_IOBITS | EXTENT_BOUNDARY))
Chris Masond1310b22008-01-24 16:13:08 -0500262 return 0;
263
264 other_node = rb_prev(&state->rb_node);
265 if (other_node) {
266 other = rb_entry(other_node, struct extent_state, rb_node);
267 if (other->end == state->start - 1 &&
268 other->state == state->state) {
Josef Bacik9ed74f22009-09-11 16:12:44 -0400269 merge_cb(tree, state, other);
Chris Masond1310b22008-01-24 16:13:08 -0500270 state->start = other->start;
Chris Mason70dec802008-01-29 09:59:12 -0500271 other->tree = NULL;
Chris Masond1310b22008-01-24 16:13:08 -0500272 rb_erase(&other->rb_node, &tree->state);
273 free_extent_state(other);
274 }
275 }
276 other_node = rb_next(&state->rb_node);
277 if (other_node) {
278 other = rb_entry(other_node, struct extent_state, rb_node);
279 if (other->start == state->end + 1 &&
280 other->state == state->state) {
Josef Bacik9ed74f22009-09-11 16:12:44 -0400281 merge_cb(tree, state, other);
Chris Masond1310b22008-01-24 16:13:08 -0500282 other->start = state->start;
Chris Mason70dec802008-01-29 09:59:12 -0500283 state->tree = NULL;
Chris Masond1310b22008-01-24 16:13:08 -0500284 rb_erase(&state->rb_node, &tree->state);
285 free_extent_state(state);
Josef Bacik9ed74f22009-09-11 16:12:44 -0400286 state = NULL;
Chris Masond1310b22008-01-24 16:13:08 -0500287 }
288 }
Josef Bacik9ed74f22009-09-11 16:12:44 -0400289
Chris Masond1310b22008-01-24 16:13:08 -0500290 return 0;
291}
292
Josef Bacik9ed74f22009-09-11 16:12:44 -0400293static int set_state_cb(struct extent_io_tree *tree,
Yan, Zheng0ca1f7c2010-05-16 10:48:47 -0400294 struct extent_state *state, int *bits)
Chris Mason291d6732008-01-29 15:55:23 -0500295{
296 if (tree->ops && tree->ops->set_bit_hook) {
Josef Bacik9ed74f22009-09-11 16:12:44 -0400297 return tree->ops->set_bit_hook(tree->mapping->host,
Yan, Zheng0ca1f7c2010-05-16 10:48:47 -0400298 state, bits);
Chris Mason291d6732008-01-29 15:55:23 -0500299 }
Josef Bacik9ed74f22009-09-11 16:12:44 -0400300
301 return 0;
Chris Mason291d6732008-01-29 15:55:23 -0500302}
303
304static void clear_state_cb(struct extent_io_tree *tree,
Yan, Zheng0ca1f7c2010-05-16 10:48:47 -0400305 struct extent_state *state, int *bits)
Chris Mason291d6732008-01-29 15:55:23 -0500306{
Josef Bacik9ed74f22009-09-11 16:12:44 -0400307 if (tree->ops && tree->ops->clear_bit_hook)
308 tree->ops->clear_bit_hook(tree->mapping->host, state, bits);
Chris Mason291d6732008-01-29 15:55:23 -0500309}
310
Chris Masond1310b22008-01-24 16:13:08 -0500311/*
312 * insert an extent_state struct into the tree. 'bits' are set on the
313 * struct before it is inserted.
314 *
315 * This may return -EEXIST if the extent is already there, in which case the
316 * state struct is freed.
317 *
318 * The tree lock is not taken internally. This is a utility function and
319 * probably isn't what you want to call (see set/clear_extent_bit).
320 */
321static int insert_state(struct extent_io_tree *tree,
322 struct extent_state *state, u64 start, u64 end,
Yan, Zheng0ca1f7c2010-05-16 10:48:47 -0400323 int *bits)
Chris Masond1310b22008-01-24 16:13:08 -0500324{
325 struct rb_node *node;
Yan, Zheng0ca1f7c2010-05-16 10:48:47 -0400326 int bits_to_set = *bits & ~EXTENT_CTLBITS;
Josef Bacik9ed74f22009-09-11 16:12:44 -0400327 int ret;
Chris Masond1310b22008-01-24 16:13:08 -0500328
329 if (end < start) {
Chris Masond3977122009-01-05 21:25:51 -0500330 printk(KERN_ERR "btrfs end < start %llu %llu\n",
331 (unsigned long long)end,
332 (unsigned long long)start);
Chris Masond1310b22008-01-24 16:13:08 -0500333 WARN_ON(1);
334 }
Chris Masond1310b22008-01-24 16:13:08 -0500335 state->start = start;
336 state->end = end;
Josef Bacik9ed74f22009-09-11 16:12:44 -0400337 ret = set_state_cb(tree, state, bits);
338 if (ret)
339 return ret;
340
Yan, Zheng0ca1f7c2010-05-16 10:48:47 -0400341 if (bits_to_set & EXTENT_DIRTY)
Josef Bacik9ed74f22009-09-11 16:12:44 -0400342 tree->dirty_bytes += end - start + 1;
Yan, Zheng0ca1f7c2010-05-16 10:48:47 -0400343 state->state |= bits_to_set;
Chris Masond1310b22008-01-24 16:13:08 -0500344 node = tree_insert(&tree->state, end, &state->rb_node);
345 if (node) {
346 struct extent_state *found;
347 found = rb_entry(node, struct extent_state, rb_node);
Chris Masond3977122009-01-05 21:25:51 -0500348 printk(KERN_ERR "btrfs found node %llu %llu on insert of "
349 "%llu %llu\n", (unsigned long long)found->start,
350 (unsigned long long)found->end,
351 (unsigned long long)start, (unsigned long long)end);
Chris Masond1310b22008-01-24 16:13:08 -0500352 free_extent_state(state);
353 return -EEXIST;
354 }
Chris Mason70dec802008-01-29 09:59:12 -0500355 state->tree = tree;
Chris Masond1310b22008-01-24 16:13:08 -0500356 merge_state(tree, state);
357 return 0;
358}
359
Josef Bacik9ed74f22009-09-11 16:12:44 -0400360static int split_cb(struct extent_io_tree *tree, struct extent_state *orig,
361 u64 split)
362{
363 if (tree->ops && tree->ops->split_extent_hook)
364 return tree->ops->split_extent_hook(tree->mapping->host,
365 orig, split);
366 return 0;
367}
368
Chris Masond1310b22008-01-24 16:13:08 -0500369/*
370 * split a given extent state struct in two, inserting the preallocated
371 * struct 'prealloc' as the newly created second half. 'split' indicates an
372 * offset inside 'orig' where it should be split.
373 *
374 * Before calling,
375 * the tree has 'orig' at [orig->start, orig->end]. After calling, there
376 * are two extent state structs in the tree:
377 * prealloc: [orig->start, split - 1]
378 * orig: [ split, orig->end ]
379 *
380 * The tree locks are not taken by this function. They need to be held
381 * by the caller.
382 */
383static int split_state(struct extent_io_tree *tree, struct extent_state *orig,
384 struct extent_state *prealloc, u64 split)
385{
386 struct rb_node *node;
Josef Bacik9ed74f22009-09-11 16:12:44 -0400387
388 split_cb(tree, orig, split);
389
Chris Masond1310b22008-01-24 16:13:08 -0500390 prealloc->start = orig->start;
391 prealloc->end = split - 1;
392 prealloc->state = orig->state;
393 orig->start = split;
394
395 node = tree_insert(&tree->state, prealloc->end, &prealloc->rb_node);
396 if (node) {
Chris Masond1310b22008-01-24 16:13:08 -0500397 free_extent_state(prealloc);
398 return -EEXIST;
399 }
Chris Mason70dec802008-01-29 09:59:12 -0500400 prealloc->tree = tree;
Chris Masond1310b22008-01-24 16:13:08 -0500401 return 0;
402}
403
404/*
405 * utility function to clear some bits in an extent state struct.
406 * it will optionally wake up any one waiting on this state (wake == 1), or
407 * forcibly remove the state from the tree (delete == 1).
408 *
409 * If no bits are set on the state struct after clearing things, the
410 * struct is freed and removed from the tree
411 */
412static int clear_state_bit(struct extent_io_tree *tree,
Yan, Zheng0ca1f7c2010-05-16 10:48:47 -0400413 struct extent_state *state,
414 int *bits, int wake)
Chris Masond1310b22008-01-24 16:13:08 -0500415{
Yan, Zheng0ca1f7c2010-05-16 10:48:47 -0400416 int bits_to_clear = *bits & ~EXTENT_CTLBITS;
Josef Bacik32c00af2009-10-08 13:34:05 -0400417 int ret = state->state & bits_to_clear;
Chris Masond1310b22008-01-24 16:13:08 -0500418
Yan, Zheng0ca1f7c2010-05-16 10:48:47 -0400419 if ((bits_to_clear & EXTENT_DIRTY) && (state->state & EXTENT_DIRTY)) {
Chris Masond1310b22008-01-24 16:13:08 -0500420 u64 range = state->end - state->start + 1;
421 WARN_ON(range > tree->dirty_bytes);
422 tree->dirty_bytes -= range;
423 }
Chris Mason291d6732008-01-29 15:55:23 -0500424 clear_state_cb(tree, state, bits);
Josef Bacik32c00af2009-10-08 13:34:05 -0400425 state->state &= ~bits_to_clear;
Chris Masond1310b22008-01-24 16:13:08 -0500426 if (wake)
427 wake_up(&state->wq);
Yan, Zheng0ca1f7c2010-05-16 10:48:47 -0400428 if (state->state == 0) {
Chris Mason70dec802008-01-29 09:59:12 -0500429 if (state->tree) {
Chris Masond1310b22008-01-24 16:13:08 -0500430 rb_erase(&state->rb_node, &tree->state);
Chris Mason70dec802008-01-29 09:59:12 -0500431 state->tree = NULL;
Chris Masond1310b22008-01-24 16:13:08 -0500432 free_extent_state(state);
433 } else {
434 WARN_ON(1);
435 }
436 } else {
437 merge_state(tree, state);
438 }
439 return ret;
440}
441
Xiao Guangrong82337672011-04-20 06:44:57 +0000442static struct extent_state *
443alloc_extent_state_atomic(struct extent_state *prealloc)
444{
445 if (!prealloc)
446 prealloc = alloc_extent_state(GFP_ATOMIC);
447
448 return prealloc;
449}
450
Chris Masond1310b22008-01-24 16:13:08 -0500451/*
452 * clear some bits on a range in the tree. This may require splitting
453 * or inserting elements in the tree, so the gfp mask is used to
454 * indicate which allocations or sleeping are allowed.
455 *
456 * pass 'wake' == 1 to kick any sleepers, and 'delete' == 1 to remove
457 * the given range from the tree regardless of state (ie for truncate).
458 *
459 * the range [start, end] is inclusive.
460 *
461 * This takes the tree lock, and returns < 0 on error, > 0 if any of the
462 * bits were already set, or zero if none of the bits were already set.
463 */
464int clear_extent_bit(struct extent_io_tree *tree, u64 start, u64 end,
Chris Mason2c64c532009-09-02 15:04:12 -0400465 int bits, int wake, int delete,
466 struct extent_state **cached_state,
467 gfp_t mask)
Chris Masond1310b22008-01-24 16:13:08 -0500468{
469 struct extent_state *state;
Chris Mason2c64c532009-09-02 15:04:12 -0400470 struct extent_state *cached;
Chris Masond1310b22008-01-24 16:13:08 -0500471 struct extent_state *prealloc = NULL;
Chris Mason2c64c532009-09-02 15:04:12 -0400472 struct rb_node *next_node;
Chris Masond1310b22008-01-24 16:13:08 -0500473 struct rb_node *node;
Yan Zheng5c939df2009-05-27 09:16:03 -0400474 u64 last_end;
Chris Masond1310b22008-01-24 16:13:08 -0500475 int err;
476 int set = 0;
Josef Bacik2ac55d42010-02-03 19:33:23 +0000477 int clear = 0;
Chris Masond1310b22008-01-24 16:13:08 -0500478
Yan, Zheng0ca1f7c2010-05-16 10:48:47 -0400479 if (delete)
480 bits |= ~EXTENT_CTLBITS;
481 bits |= EXTENT_FIRST_DELALLOC;
482
Josef Bacik2ac55d42010-02-03 19:33:23 +0000483 if (bits & (EXTENT_IOBITS | EXTENT_BOUNDARY))
484 clear = 1;
Chris Masond1310b22008-01-24 16:13:08 -0500485again:
486 if (!prealloc && (mask & __GFP_WAIT)) {
487 prealloc = alloc_extent_state(mask);
Chris Masonc309df02011-05-26 17:43:59 -0400488 if (!prealloc)
489 return -ENOMEM;
Chris Masond1310b22008-01-24 16:13:08 -0500490 }
491
Chris Masoncad321a2008-12-17 14:51:42 -0500492 spin_lock(&tree->lock);
Chris Mason2c64c532009-09-02 15:04:12 -0400493 if (cached_state) {
494 cached = *cached_state;
Josef Bacik2ac55d42010-02-03 19:33:23 +0000495
496 if (clear) {
497 *cached_state = NULL;
498 cached_state = NULL;
499 }
500
Chris Mason42daec22009-09-23 19:51:09 -0400501 if (cached && cached->tree && cached->start == start) {
Josef Bacik2ac55d42010-02-03 19:33:23 +0000502 if (clear)
503 atomic_dec(&cached->refs);
Chris Mason2c64c532009-09-02 15:04:12 -0400504 state = cached;
Chris Mason42daec22009-09-23 19:51:09 -0400505 goto hit_next;
Chris Mason2c64c532009-09-02 15:04:12 -0400506 }
Josef Bacik2ac55d42010-02-03 19:33:23 +0000507 if (clear)
508 free_extent_state(cached);
Chris Mason2c64c532009-09-02 15:04:12 -0400509 }
Chris Masond1310b22008-01-24 16:13:08 -0500510 /*
511 * this search will find the extents that end after
512 * our range starts
513 */
Chris Mason80ea96b2008-02-01 14:51:59 -0500514 node = tree_search(tree, start);
Chris Masond1310b22008-01-24 16:13:08 -0500515 if (!node)
516 goto out;
517 state = rb_entry(node, struct extent_state, rb_node);
Chris Mason2c64c532009-09-02 15:04:12 -0400518hit_next:
Chris Masond1310b22008-01-24 16:13:08 -0500519 if (state->start > end)
520 goto out;
521 WARN_ON(state->end < start);
Yan Zheng5c939df2009-05-27 09:16:03 -0400522 last_end = state->end;
Chris Masond1310b22008-01-24 16:13:08 -0500523
524 /*
525 * | ---- desired range ---- |
526 * | state | or
527 * | ------------- state -------------- |
528 *
529 * We need to split the extent we found, and may flip
530 * bits on second half.
531 *
532 * If the extent we found extends past our range, we
533 * just split and search again. It'll get split again
534 * the next time though.
535 *
536 * If the extent we found is inside our range, we clear
537 * the desired bit on it.
538 */
539
540 if (state->start < start) {
Xiao Guangrong82337672011-04-20 06:44:57 +0000541 prealloc = alloc_extent_state_atomic(prealloc);
542 BUG_ON(!prealloc);
Chris Masond1310b22008-01-24 16:13:08 -0500543 err = split_state(tree, state, prealloc, start);
544 BUG_ON(err == -EEXIST);
545 prealloc = NULL;
546 if (err)
547 goto out;
548 if (state->end <= end) {
Yan, Zheng0ca1f7c2010-05-16 10:48:47 -0400549 set |= clear_state_bit(tree, state, &bits, wake);
Yan Zheng5c939df2009-05-27 09:16:03 -0400550 if (last_end == (u64)-1)
551 goto out;
552 start = last_end + 1;
Chris Masond1310b22008-01-24 16:13:08 -0500553 }
554 goto search_again;
555 }
556 /*
557 * | ---- desired range ---- |
558 * | state |
559 * We need to split the extent, and clear the bit
560 * on the first half
561 */
562 if (state->start <= end && state->end > end) {
Xiao Guangrong82337672011-04-20 06:44:57 +0000563 prealloc = alloc_extent_state_atomic(prealloc);
564 BUG_ON(!prealloc);
Chris Masond1310b22008-01-24 16:13:08 -0500565 err = split_state(tree, state, prealloc, end + 1);
566 BUG_ON(err == -EEXIST);
Chris Masond1310b22008-01-24 16:13:08 -0500567 if (wake)
568 wake_up(&state->wq);
Chris Mason42daec22009-09-23 19:51:09 -0400569
Yan, Zheng0ca1f7c2010-05-16 10:48:47 -0400570 set |= clear_state_bit(tree, prealloc, &bits, wake);
Josef Bacik9ed74f22009-09-11 16:12:44 -0400571
Chris Masond1310b22008-01-24 16:13:08 -0500572 prealloc = NULL;
573 goto out;
574 }
Chris Mason42daec22009-09-23 19:51:09 -0400575
Chris Mason2c64c532009-09-02 15:04:12 -0400576 if (state->end < end && prealloc && !need_resched())
577 next_node = rb_next(&state->rb_node);
578 else
579 next_node = NULL;
Chris Mason42daec22009-09-23 19:51:09 -0400580
Yan, Zheng0ca1f7c2010-05-16 10:48:47 -0400581 set |= clear_state_bit(tree, state, &bits, wake);
Yan Zheng5c939df2009-05-27 09:16:03 -0400582 if (last_end == (u64)-1)
583 goto out;
584 start = last_end + 1;
Chris Mason2c64c532009-09-02 15:04:12 -0400585 if (start <= end && next_node) {
586 state = rb_entry(next_node, struct extent_state,
587 rb_node);
588 if (state->start == start)
589 goto hit_next;
590 }
Chris Masond1310b22008-01-24 16:13:08 -0500591 goto search_again;
592
593out:
Chris Masoncad321a2008-12-17 14:51:42 -0500594 spin_unlock(&tree->lock);
Chris Masond1310b22008-01-24 16:13:08 -0500595 if (prealloc)
596 free_extent_state(prealloc);
597
598 return set;
599
600search_again:
601 if (start > end)
602 goto out;
Chris Masoncad321a2008-12-17 14:51:42 -0500603 spin_unlock(&tree->lock);
Chris Masond1310b22008-01-24 16:13:08 -0500604 if (mask & __GFP_WAIT)
605 cond_resched();
606 goto again;
607}
Chris Masond1310b22008-01-24 16:13:08 -0500608
609static int wait_on_state(struct extent_io_tree *tree,
610 struct extent_state *state)
Christoph Hellwig641f5212008-12-02 06:36:10 -0500611 __releases(tree->lock)
612 __acquires(tree->lock)
Chris Masond1310b22008-01-24 16:13:08 -0500613{
614 DEFINE_WAIT(wait);
615 prepare_to_wait(&state->wq, &wait, TASK_UNINTERRUPTIBLE);
Chris Masoncad321a2008-12-17 14:51:42 -0500616 spin_unlock(&tree->lock);
Chris Masond1310b22008-01-24 16:13:08 -0500617 schedule();
Chris Masoncad321a2008-12-17 14:51:42 -0500618 spin_lock(&tree->lock);
Chris Masond1310b22008-01-24 16:13:08 -0500619 finish_wait(&state->wq, &wait);
620 return 0;
621}
622
623/*
624 * waits for one or more bits to clear on a range in the state tree.
625 * The range [start, end] is inclusive.
626 * The tree lock is taken by this function
627 */
628int wait_extent_bit(struct extent_io_tree *tree, u64 start, u64 end, int bits)
629{
630 struct extent_state *state;
631 struct rb_node *node;
632
Chris Masoncad321a2008-12-17 14:51:42 -0500633 spin_lock(&tree->lock);
Chris Masond1310b22008-01-24 16:13:08 -0500634again:
635 while (1) {
636 /*
637 * this search will find all the extents that end after
638 * our range starts
639 */
Chris Mason80ea96b2008-02-01 14:51:59 -0500640 node = tree_search(tree, start);
Chris Masond1310b22008-01-24 16:13:08 -0500641 if (!node)
642 break;
643
644 state = rb_entry(node, struct extent_state, rb_node);
645
646 if (state->start > end)
647 goto out;
648
649 if (state->state & bits) {
650 start = state->start;
651 atomic_inc(&state->refs);
652 wait_on_state(tree, state);
653 free_extent_state(state);
654 goto again;
655 }
656 start = state->end + 1;
657
658 if (start > end)
659 break;
660
661 if (need_resched()) {
Chris Masoncad321a2008-12-17 14:51:42 -0500662 spin_unlock(&tree->lock);
Chris Masond1310b22008-01-24 16:13:08 -0500663 cond_resched();
Chris Masoncad321a2008-12-17 14:51:42 -0500664 spin_lock(&tree->lock);
Chris Masond1310b22008-01-24 16:13:08 -0500665 }
666 }
667out:
Chris Masoncad321a2008-12-17 14:51:42 -0500668 spin_unlock(&tree->lock);
Chris Masond1310b22008-01-24 16:13:08 -0500669 return 0;
670}
Chris Masond1310b22008-01-24 16:13:08 -0500671
Josef Bacik9ed74f22009-09-11 16:12:44 -0400672static int set_state_bits(struct extent_io_tree *tree,
Chris Masond1310b22008-01-24 16:13:08 -0500673 struct extent_state *state,
Yan, Zheng0ca1f7c2010-05-16 10:48:47 -0400674 int *bits)
Chris Masond1310b22008-01-24 16:13:08 -0500675{
Josef Bacik9ed74f22009-09-11 16:12:44 -0400676 int ret;
Yan, Zheng0ca1f7c2010-05-16 10:48:47 -0400677 int bits_to_set = *bits & ~EXTENT_CTLBITS;
Josef Bacik9ed74f22009-09-11 16:12:44 -0400678
679 ret = set_state_cb(tree, state, bits);
680 if (ret)
681 return ret;
Yan, Zheng0ca1f7c2010-05-16 10:48:47 -0400682 if ((bits_to_set & EXTENT_DIRTY) && !(state->state & EXTENT_DIRTY)) {
Chris Masond1310b22008-01-24 16:13:08 -0500683 u64 range = state->end - state->start + 1;
684 tree->dirty_bytes += range;
685 }
Yan, Zheng0ca1f7c2010-05-16 10:48:47 -0400686 state->state |= bits_to_set;
Josef Bacik9ed74f22009-09-11 16:12:44 -0400687
688 return 0;
Chris Masond1310b22008-01-24 16:13:08 -0500689}
690
Chris Mason2c64c532009-09-02 15:04:12 -0400691static void cache_state(struct extent_state *state,
692 struct extent_state **cached_ptr)
693{
694 if (cached_ptr && !(*cached_ptr)) {
695 if (state->state & (EXTENT_IOBITS | EXTENT_BOUNDARY)) {
696 *cached_ptr = state;
697 atomic_inc(&state->refs);
698 }
699 }
700}
701
Arne Jansen507903b2011-04-06 10:02:20 +0000702static void uncache_state(struct extent_state **cached_ptr)
703{
704 if (cached_ptr && (*cached_ptr)) {
705 struct extent_state *state = *cached_ptr;
Chris Mason109b36a2011-04-12 13:57:39 -0400706 *cached_ptr = NULL;
707 free_extent_state(state);
Arne Jansen507903b2011-04-06 10:02:20 +0000708 }
709}
710
Chris Masond1310b22008-01-24 16:13:08 -0500711/*
Chris Mason1edbb732009-09-02 13:24:36 -0400712 * set some bits on a range in the tree. This may require allocations or
713 * sleeping, so the gfp mask is used to indicate what is allowed.
Chris Masond1310b22008-01-24 16:13:08 -0500714 *
Chris Mason1edbb732009-09-02 13:24:36 -0400715 * If any of the exclusive bits are set, this will fail with -EEXIST if some
716 * part of the range already has the desired bits set. The start of the
717 * existing range is returned in failed_start in this case.
Chris Masond1310b22008-01-24 16:13:08 -0500718 *
Chris Mason1edbb732009-09-02 13:24:36 -0400719 * [start, end] is inclusive This takes the tree lock.
Chris Masond1310b22008-01-24 16:13:08 -0500720 */
Chris Mason1edbb732009-09-02 13:24:36 -0400721
Chris Mason4845e442010-05-25 20:56:50 -0400722int set_extent_bit(struct extent_io_tree *tree, u64 start, u64 end,
723 int bits, int exclusive_bits, u64 *failed_start,
724 struct extent_state **cached_state, gfp_t mask)
Chris Masond1310b22008-01-24 16:13:08 -0500725{
726 struct extent_state *state;
727 struct extent_state *prealloc = NULL;
728 struct rb_node *node;
Chris Masond1310b22008-01-24 16:13:08 -0500729 int err = 0;
Chris Masond1310b22008-01-24 16:13:08 -0500730 u64 last_start;
731 u64 last_end;
Chris Mason42daec22009-09-23 19:51:09 -0400732
Yan, Zheng0ca1f7c2010-05-16 10:48:47 -0400733 bits |= EXTENT_FIRST_DELALLOC;
Chris Masond1310b22008-01-24 16:13:08 -0500734again:
735 if (!prealloc && (mask & __GFP_WAIT)) {
736 prealloc = alloc_extent_state(mask);
Xiao Guangrong82337672011-04-20 06:44:57 +0000737 BUG_ON(!prealloc);
Chris Masond1310b22008-01-24 16:13:08 -0500738 }
739
Chris Masoncad321a2008-12-17 14:51:42 -0500740 spin_lock(&tree->lock);
Chris Mason9655d292009-09-02 15:22:30 -0400741 if (cached_state && *cached_state) {
742 state = *cached_state;
743 if (state->start == start && state->tree) {
744 node = &state->rb_node;
745 goto hit_next;
746 }
747 }
Chris Masond1310b22008-01-24 16:13:08 -0500748 /*
749 * this search will find all the extents that end after
750 * our range starts.
751 */
Chris Mason80ea96b2008-02-01 14:51:59 -0500752 node = tree_search(tree, start);
Chris Masond1310b22008-01-24 16:13:08 -0500753 if (!node) {
Xiao Guangrong82337672011-04-20 06:44:57 +0000754 prealloc = alloc_extent_state_atomic(prealloc);
755 BUG_ON(!prealloc);
Yan, Zheng0ca1f7c2010-05-16 10:48:47 -0400756 err = insert_state(tree, prealloc, start, end, &bits);
Chris Masond1310b22008-01-24 16:13:08 -0500757 prealloc = NULL;
758 BUG_ON(err == -EEXIST);
759 goto out;
760 }
Chris Masond1310b22008-01-24 16:13:08 -0500761 state = rb_entry(node, struct extent_state, rb_node);
Chris Mason40431d62009-08-05 12:57:59 -0400762hit_next:
Chris Masond1310b22008-01-24 16:13:08 -0500763 last_start = state->start;
764 last_end = state->end;
765
766 /*
767 * | ---- desired range ---- |
768 * | state |
769 *
770 * Just lock what we found and keep going
771 */
772 if (state->start == start && state->end <= end) {
Chris Mason40431d62009-08-05 12:57:59 -0400773 struct rb_node *next_node;
Chris Mason1edbb732009-09-02 13:24:36 -0400774 if (state->state & exclusive_bits) {
Chris Masond1310b22008-01-24 16:13:08 -0500775 *failed_start = state->start;
776 err = -EEXIST;
777 goto out;
778 }
Chris Mason42daec22009-09-23 19:51:09 -0400779
Yan, Zheng0ca1f7c2010-05-16 10:48:47 -0400780 err = set_state_bits(tree, state, &bits);
Josef Bacik9ed74f22009-09-11 16:12:44 -0400781 if (err)
782 goto out;
783
Xiao Guangrongc7f895a2011-04-20 06:45:49 +0000784 next_node = rb_next(node);
Chris Mason2c64c532009-09-02 15:04:12 -0400785 cache_state(state, cached_state);
Chris Masond1310b22008-01-24 16:13:08 -0500786 merge_state(tree, state);
Yan Zheng5c939df2009-05-27 09:16:03 -0400787 if (last_end == (u64)-1)
788 goto out;
Chris Mason40431d62009-08-05 12:57:59 -0400789
Yan Zheng5c939df2009-05-27 09:16:03 -0400790 start = last_end + 1;
Xiao Guangrongc7f895a2011-04-20 06:45:49 +0000791 if (next_node && start < end && prealloc && !need_resched()) {
792 state = rb_entry(next_node, struct extent_state,
793 rb_node);
794 if (state->start == start)
795 goto hit_next;
Chris Mason40431d62009-08-05 12:57:59 -0400796 }
Chris Masond1310b22008-01-24 16:13:08 -0500797 goto search_again;
798 }
799
800 /*
801 * | ---- desired range ---- |
802 * | state |
803 * or
804 * | ------------- state -------------- |
805 *
806 * We need to split the extent we found, and may flip bits on
807 * second half.
808 *
809 * If the extent we found extends past our
810 * range, we just split and search again. It'll get split
811 * again the next time though.
812 *
813 * If the extent we found is inside our range, we set the
814 * desired bit on it.
815 */
816 if (state->start < start) {
Chris Mason1edbb732009-09-02 13:24:36 -0400817 if (state->state & exclusive_bits) {
Chris Masond1310b22008-01-24 16:13:08 -0500818 *failed_start = start;
819 err = -EEXIST;
820 goto out;
821 }
Xiao Guangrong82337672011-04-20 06:44:57 +0000822
823 prealloc = alloc_extent_state_atomic(prealloc);
824 BUG_ON(!prealloc);
Chris Masond1310b22008-01-24 16:13:08 -0500825 err = split_state(tree, state, prealloc, start);
826 BUG_ON(err == -EEXIST);
827 prealloc = NULL;
828 if (err)
829 goto out;
830 if (state->end <= end) {
Yan, Zheng0ca1f7c2010-05-16 10:48:47 -0400831 err = set_state_bits(tree, state, &bits);
Josef Bacik9ed74f22009-09-11 16:12:44 -0400832 if (err)
833 goto out;
Chris Mason2c64c532009-09-02 15:04:12 -0400834 cache_state(state, cached_state);
Chris Masond1310b22008-01-24 16:13:08 -0500835 merge_state(tree, state);
Yan Zheng5c939df2009-05-27 09:16:03 -0400836 if (last_end == (u64)-1)
837 goto out;
838 start = last_end + 1;
Chris Masond1310b22008-01-24 16:13:08 -0500839 }
840 goto search_again;
841 }
842 /*
843 * | ---- desired range ---- |
844 * | state | or | state |
845 *
846 * There's a hole, we need to insert something in it and
847 * ignore the extent we found.
848 */
849 if (state->start > start) {
850 u64 this_end;
851 if (end < last_start)
852 this_end = end;
853 else
Chris Masond3977122009-01-05 21:25:51 -0500854 this_end = last_start - 1;
Xiao Guangrong82337672011-04-20 06:44:57 +0000855
856 prealloc = alloc_extent_state_atomic(prealloc);
857 BUG_ON(!prealloc);
Xiao Guangrongc7f895a2011-04-20 06:45:49 +0000858
859 /*
860 * Avoid to free 'prealloc' if it can be merged with
861 * the later extent.
862 */
863 atomic_inc(&prealloc->refs);
Chris Masond1310b22008-01-24 16:13:08 -0500864 err = insert_state(tree, prealloc, start, this_end,
Yan, Zheng0ca1f7c2010-05-16 10:48:47 -0400865 &bits);
Josef Bacik9ed74f22009-09-11 16:12:44 -0400866 BUG_ON(err == -EEXIST);
867 if (err) {
Xiao Guangrongc7f895a2011-04-20 06:45:49 +0000868 free_extent_state(prealloc);
Josef Bacik9ed74f22009-09-11 16:12:44 -0400869 prealloc = NULL;
870 goto out;
871 }
Chris Mason2c64c532009-09-02 15:04:12 -0400872 cache_state(prealloc, cached_state);
Xiao Guangrongc7f895a2011-04-20 06:45:49 +0000873 free_extent_state(prealloc);
Chris Masond1310b22008-01-24 16:13:08 -0500874 prealloc = NULL;
Chris Masond1310b22008-01-24 16:13:08 -0500875 start = this_end + 1;
876 goto search_again;
877 }
878 /*
879 * | ---- desired range ---- |
880 * | state |
881 * We need to split the extent, and set the bit
882 * on the first half
883 */
884 if (state->start <= end && state->end > end) {
Chris Mason1edbb732009-09-02 13:24:36 -0400885 if (state->state & exclusive_bits) {
Chris Masond1310b22008-01-24 16:13:08 -0500886 *failed_start = start;
887 err = -EEXIST;
888 goto out;
889 }
Xiao Guangrong82337672011-04-20 06:44:57 +0000890
891 prealloc = alloc_extent_state_atomic(prealloc);
892 BUG_ON(!prealloc);
Chris Masond1310b22008-01-24 16:13:08 -0500893 err = split_state(tree, state, prealloc, end + 1);
894 BUG_ON(err == -EEXIST);
895
Yan, Zheng0ca1f7c2010-05-16 10:48:47 -0400896 err = set_state_bits(tree, prealloc, &bits);
Josef Bacik9ed74f22009-09-11 16:12:44 -0400897 if (err) {
898 prealloc = NULL;
899 goto out;
900 }
Chris Mason2c64c532009-09-02 15:04:12 -0400901 cache_state(prealloc, cached_state);
Chris Masond1310b22008-01-24 16:13:08 -0500902 merge_state(tree, prealloc);
903 prealloc = NULL;
904 goto out;
905 }
906
907 goto search_again;
908
909out:
Chris Masoncad321a2008-12-17 14:51:42 -0500910 spin_unlock(&tree->lock);
Chris Masond1310b22008-01-24 16:13:08 -0500911 if (prealloc)
912 free_extent_state(prealloc);
913
914 return err;
915
916search_again:
917 if (start > end)
918 goto out;
Chris Masoncad321a2008-12-17 14:51:42 -0500919 spin_unlock(&tree->lock);
Chris Masond1310b22008-01-24 16:13:08 -0500920 if (mask & __GFP_WAIT)
921 cond_resched();
922 goto again;
923}
Chris Masond1310b22008-01-24 16:13:08 -0500924
925/* wrappers around set/clear extent bit */
926int set_extent_dirty(struct extent_io_tree *tree, u64 start, u64 end,
927 gfp_t mask)
928{
929 return set_extent_bit(tree, start, end, EXTENT_DIRTY, 0, NULL,
Chris Mason2c64c532009-09-02 15:04:12 -0400930 NULL, mask);
Chris Masond1310b22008-01-24 16:13:08 -0500931}
Chris Masond1310b22008-01-24 16:13:08 -0500932
933int set_extent_bits(struct extent_io_tree *tree, u64 start, u64 end,
934 int bits, gfp_t mask)
935{
936 return set_extent_bit(tree, start, end, bits, 0, NULL,
Chris Mason2c64c532009-09-02 15:04:12 -0400937 NULL, mask);
Chris Masond1310b22008-01-24 16:13:08 -0500938}
Chris Masond1310b22008-01-24 16:13:08 -0500939
940int clear_extent_bits(struct extent_io_tree *tree, u64 start, u64 end,
941 int bits, gfp_t mask)
942{
Chris Mason2c64c532009-09-02 15:04:12 -0400943 return clear_extent_bit(tree, start, end, bits, 0, 0, NULL, mask);
Chris Masond1310b22008-01-24 16:13:08 -0500944}
Chris Masond1310b22008-01-24 16:13:08 -0500945
946int set_extent_delalloc(struct extent_io_tree *tree, u64 start, u64 end,
Josef Bacik2ac55d42010-02-03 19:33:23 +0000947 struct extent_state **cached_state, gfp_t mask)
Chris Masond1310b22008-01-24 16:13:08 -0500948{
949 return set_extent_bit(tree, start, end,
Chris Mason40431d62009-08-05 12:57:59 -0400950 EXTENT_DELALLOC | EXTENT_DIRTY | EXTENT_UPTODATE,
Josef Bacik2ac55d42010-02-03 19:33:23 +0000951 0, NULL, cached_state, mask);
Chris Masond1310b22008-01-24 16:13:08 -0500952}
Chris Masond1310b22008-01-24 16:13:08 -0500953
954int clear_extent_dirty(struct extent_io_tree *tree, u64 start, u64 end,
955 gfp_t mask)
956{
957 return clear_extent_bit(tree, start, end,
Josef Bacik32c00af2009-10-08 13:34:05 -0400958 EXTENT_DIRTY | EXTENT_DELALLOC |
Yan, Zheng0ca1f7c2010-05-16 10:48:47 -0400959 EXTENT_DO_ACCOUNTING, 0, 0, NULL, mask);
Chris Masond1310b22008-01-24 16:13:08 -0500960}
Chris Masond1310b22008-01-24 16:13:08 -0500961
962int set_extent_new(struct extent_io_tree *tree, u64 start, u64 end,
963 gfp_t mask)
964{
965 return set_extent_bit(tree, start, end, EXTENT_NEW, 0, NULL,
Chris Mason2c64c532009-09-02 15:04:12 -0400966 NULL, mask);
Chris Masond1310b22008-01-24 16:13:08 -0500967}
Chris Masond1310b22008-01-24 16:13:08 -0500968
Chris Masond1310b22008-01-24 16:13:08 -0500969int set_extent_uptodate(struct extent_io_tree *tree, u64 start, u64 end,
Arne Jansen507903b2011-04-06 10:02:20 +0000970 struct extent_state **cached_state, gfp_t mask)
Chris Masond1310b22008-01-24 16:13:08 -0500971{
Arne Jansen507903b2011-04-06 10:02:20 +0000972 return set_extent_bit(tree, start, end, EXTENT_UPTODATE, 0,
973 NULL, cached_state, mask);
Chris Masond1310b22008-01-24 16:13:08 -0500974}
Chris Masond1310b22008-01-24 16:13:08 -0500975
Chris Masond3977122009-01-05 21:25:51 -0500976static int clear_extent_uptodate(struct extent_io_tree *tree, u64 start,
Josef Bacik2ac55d42010-02-03 19:33:23 +0000977 u64 end, struct extent_state **cached_state,
978 gfp_t mask)
Chris Masond1310b22008-01-24 16:13:08 -0500979{
Chris Mason2c64c532009-09-02 15:04:12 -0400980 return clear_extent_bit(tree, start, end, EXTENT_UPTODATE, 0, 0,
Josef Bacik2ac55d42010-02-03 19:33:23 +0000981 cached_state, mask);
Chris Masond1310b22008-01-24 16:13:08 -0500982}
Chris Masond1310b22008-01-24 16:13:08 -0500983
Chris Masond352ac62008-09-29 15:18:18 -0400984/*
985 * either insert or lock state struct between start and end use mask to tell
986 * us if waiting is desired.
987 */
Chris Mason1edbb732009-09-02 13:24:36 -0400988int lock_extent_bits(struct extent_io_tree *tree, u64 start, u64 end,
Chris Mason2c64c532009-09-02 15:04:12 -0400989 int bits, struct extent_state **cached_state, gfp_t mask)
Chris Masond1310b22008-01-24 16:13:08 -0500990{
991 int err;
992 u64 failed_start;
993 while (1) {
Chris Mason1edbb732009-09-02 13:24:36 -0400994 err = set_extent_bit(tree, start, end, EXTENT_LOCKED | bits,
Chris Mason2c64c532009-09-02 15:04:12 -0400995 EXTENT_LOCKED, &failed_start,
996 cached_state, mask);
Chris Masond1310b22008-01-24 16:13:08 -0500997 if (err == -EEXIST && (mask & __GFP_WAIT)) {
998 wait_extent_bit(tree, failed_start, end, EXTENT_LOCKED);
999 start = failed_start;
1000 } else {
1001 break;
1002 }
1003 WARN_ON(start > end);
1004 }
1005 return err;
1006}
Chris Masond1310b22008-01-24 16:13:08 -05001007
Chris Mason1edbb732009-09-02 13:24:36 -04001008int lock_extent(struct extent_io_tree *tree, u64 start, u64 end, gfp_t mask)
1009{
Chris Mason2c64c532009-09-02 15:04:12 -04001010 return lock_extent_bits(tree, start, end, 0, NULL, mask);
Chris Mason1edbb732009-09-02 13:24:36 -04001011}
1012
Josef Bacik25179202008-10-29 14:49:05 -04001013int try_lock_extent(struct extent_io_tree *tree, u64 start, u64 end,
1014 gfp_t mask)
1015{
1016 int err;
1017 u64 failed_start;
1018
Chris Mason2c64c532009-09-02 15:04:12 -04001019 err = set_extent_bit(tree, start, end, EXTENT_LOCKED, EXTENT_LOCKED,
1020 &failed_start, NULL, mask);
Yan Zheng66435582008-10-30 14:19:50 -04001021 if (err == -EEXIST) {
1022 if (failed_start > start)
1023 clear_extent_bit(tree, start, failed_start - 1,
Chris Mason2c64c532009-09-02 15:04:12 -04001024 EXTENT_LOCKED, 1, 0, NULL, mask);
Josef Bacik25179202008-10-29 14:49:05 -04001025 return 0;
Yan Zheng66435582008-10-30 14:19:50 -04001026 }
Josef Bacik25179202008-10-29 14:49:05 -04001027 return 1;
1028}
Josef Bacik25179202008-10-29 14:49:05 -04001029
Chris Mason2c64c532009-09-02 15:04:12 -04001030int unlock_extent_cached(struct extent_io_tree *tree, u64 start, u64 end,
1031 struct extent_state **cached, gfp_t mask)
1032{
1033 return clear_extent_bit(tree, start, end, EXTENT_LOCKED, 1, 0, cached,
1034 mask);
1035}
1036
Arne Jansen507903b2011-04-06 10:02:20 +00001037int unlock_extent(struct extent_io_tree *tree, u64 start, u64 end, gfp_t mask)
Chris Masond1310b22008-01-24 16:13:08 -05001038{
Chris Mason2c64c532009-09-02 15:04:12 -04001039 return clear_extent_bit(tree, start, end, EXTENT_LOCKED, 1, 0, NULL,
1040 mask);
Chris Masond1310b22008-01-24 16:13:08 -05001041}
Chris Masond1310b22008-01-24 16:13:08 -05001042
1043/*
Chris Masond1310b22008-01-24 16:13:08 -05001044 * helper function to set both pages and extents in the tree writeback
1045 */
Christoph Hellwigb2950862008-12-02 09:54:17 -05001046static int set_range_writeback(struct extent_io_tree *tree, u64 start, u64 end)
Chris Masond1310b22008-01-24 16:13:08 -05001047{
1048 unsigned long index = start >> PAGE_CACHE_SHIFT;
1049 unsigned long end_index = end >> PAGE_CACHE_SHIFT;
1050 struct page *page;
1051
1052 while (index <= end_index) {
1053 page = find_get_page(tree->mapping, index);
1054 BUG_ON(!page);
1055 set_page_writeback(page);
1056 page_cache_release(page);
1057 index++;
1058 }
Chris Masond1310b22008-01-24 16:13:08 -05001059 return 0;
1060}
Chris Masond1310b22008-01-24 16:13:08 -05001061
Chris Masond352ac62008-09-29 15:18:18 -04001062/*
1063 * find the first offset in the io tree with 'bits' set. zero is
1064 * returned if we find something, and *start_ret and *end_ret are
1065 * set to reflect the state struct that was found.
1066 *
1067 * If nothing was found, 1 is returned, < 0 on error
1068 */
Chris Masond1310b22008-01-24 16:13:08 -05001069int find_first_extent_bit(struct extent_io_tree *tree, u64 start,
1070 u64 *start_ret, u64 *end_ret, int bits)
1071{
1072 struct rb_node *node;
1073 struct extent_state *state;
1074 int ret = 1;
1075
Chris Masoncad321a2008-12-17 14:51:42 -05001076 spin_lock(&tree->lock);
Chris Masond1310b22008-01-24 16:13:08 -05001077 /*
1078 * this search will find all the extents that end after
1079 * our range starts.
1080 */
Chris Mason80ea96b2008-02-01 14:51:59 -05001081 node = tree_search(tree, start);
Chris Masond3977122009-01-05 21:25:51 -05001082 if (!node)
Chris Masond1310b22008-01-24 16:13:08 -05001083 goto out;
Chris Masond1310b22008-01-24 16:13:08 -05001084
Chris Masond3977122009-01-05 21:25:51 -05001085 while (1) {
Chris Masond1310b22008-01-24 16:13:08 -05001086 state = rb_entry(node, struct extent_state, rb_node);
1087 if (state->end >= start && (state->state & bits)) {
1088 *start_ret = state->start;
1089 *end_ret = state->end;
1090 ret = 0;
1091 break;
1092 }
1093 node = rb_next(node);
1094 if (!node)
1095 break;
1096 }
1097out:
Chris Masoncad321a2008-12-17 14:51:42 -05001098 spin_unlock(&tree->lock);
Chris Masond1310b22008-01-24 16:13:08 -05001099 return ret;
1100}
Chris Masond1310b22008-01-24 16:13:08 -05001101
Chris Masond352ac62008-09-29 15:18:18 -04001102/* find the first state struct with 'bits' set after 'start', and
1103 * return it. tree->lock must be held. NULL will returned if
1104 * nothing was found after 'start'
1105 */
Chris Masond7fc6402008-02-18 12:12:38 -05001106struct extent_state *find_first_extent_bit_state(struct extent_io_tree *tree,
1107 u64 start, int bits)
1108{
1109 struct rb_node *node;
1110 struct extent_state *state;
1111
1112 /*
1113 * this search will find all the extents that end after
1114 * our range starts.
1115 */
1116 node = tree_search(tree, start);
Chris Masond3977122009-01-05 21:25:51 -05001117 if (!node)
Chris Masond7fc6402008-02-18 12:12:38 -05001118 goto out;
Chris Masond7fc6402008-02-18 12:12:38 -05001119
Chris Masond3977122009-01-05 21:25:51 -05001120 while (1) {
Chris Masond7fc6402008-02-18 12:12:38 -05001121 state = rb_entry(node, struct extent_state, rb_node);
Chris Masond3977122009-01-05 21:25:51 -05001122 if (state->end >= start && (state->state & bits))
Chris Masond7fc6402008-02-18 12:12:38 -05001123 return state;
Chris Masond3977122009-01-05 21:25:51 -05001124
Chris Masond7fc6402008-02-18 12:12:38 -05001125 node = rb_next(node);
1126 if (!node)
1127 break;
1128 }
1129out:
1130 return NULL;
1131}
Chris Masond7fc6402008-02-18 12:12:38 -05001132
Chris Masond352ac62008-09-29 15:18:18 -04001133/*
1134 * find a contiguous range of bytes in the file marked as delalloc, not
1135 * more than 'max_bytes'. start and end are used to return the range,
1136 *
1137 * 1 is returned if we find something, 0 if nothing was in the tree
1138 */
Chris Masonc8b97812008-10-29 14:49:59 -04001139static noinline u64 find_delalloc_range(struct extent_io_tree *tree,
Josef Bacikc2a128d2010-02-02 21:19:11 +00001140 u64 *start, u64 *end, u64 max_bytes,
1141 struct extent_state **cached_state)
Chris Masond1310b22008-01-24 16:13:08 -05001142{
1143 struct rb_node *node;
1144 struct extent_state *state;
1145 u64 cur_start = *start;
1146 u64 found = 0;
1147 u64 total_bytes = 0;
1148
Chris Masoncad321a2008-12-17 14:51:42 -05001149 spin_lock(&tree->lock);
Chris Masonc8b97812008-10-29 14:49:59 -04001150
Chris Masond1310b22008-01-24 16:13:08 -05001151 /*
1152 * this search will find all the extents that end after
1153 * our range starts.
1154 */
Chris Mason80ea96b2008-02-01 14:51:59 -05001155 node = tree_search(tree, cur_start);
Peter2b114d12008-04-01 11:21:40 -04001156 if (!node) {
Chris Mason3b951512008-04-17 11:29:12 -04001157 if (!found)
1158 *end = (u64)-1;
Chris Masond1310b22008-01-24 16:13:08 -05001159 goto out;
1160 }
1161
Chris Masond3977122009-01-05 21:25:51 -05001162 while (1) {
Chris Masond1310b22008-01-24 16:13:08 -05001163 state = rb_entry(node, struct extent_state, rb_node);
Zheng Yan5b21f2e2008-09-26 10:05:38 -04001164 if (found && (state->start != cur_start ||
1165 (state->state & EXTENT_BOUNDARY))) {
Chris Masond1310b22008-01-24 16:13:08 -05001166 goto out;
1167 }
1168 if (!(state->state & EXTENT_DELALLOC)) {
1169 if (!found)
1170 *end = state->end;
1171 goto out;
1172 }
Josef Bacikc2a128d2010-02-02 21:19:11 +00001173 if (!found) {
Chris Masond1310b22008-01-24 16:13:08 -05001174 *start = state->start;
Josef Bacikc2a128d2010-02-02 21:19:11 +00001175 *cached_state = state;
1176 atomic_inc(&state->refs);
1177 }
Chris Masond1310b22008-01-24 16:13:08 -05001178 found++;
1179 *end = state->end;
1180 cur_start = state->end + 1;
1181 node = rb_next(node);
1182 if (!node)
1183 break;
1184 total_bytes += state->end - state->start + 1;
1185 if (total_bytes >= max_bytes)
1186 break;
1187 }
1188out:
Chris Masoncad321a2008-12-17 14:51:42 -05001189 spin_unlock(&tree->lock);
Chris Masond1310b22008-01-24 16:13:08 -05001190 return found;
1191}
1192
Chris Masonc8b97812008-10-29 14:49:59 -04001193static noinline int __unlock_for_delalloc(struct inode *inode,
1194 struct page *locked_page,
1195 u64 start, u64 end)
1196{
1197 int ret;
1198 struct page *pages[16];
1199 unsigned long index = start >> PAGE_CACHE_SHIFT;
1200 unsigned long end_index = end >> PAGE_CACHE_SHIFT;
1201 unsigned long nr_pages = end_index - index + 1;
1202 int i;
1203
1204 if (index == locked_page->index && end_index == index)
1205 return 0;
1206
Chris Masond3977122009-01-05 21:25:51 -05001207 while (nr_pages > 0) {
Chris Masonc8b97812008-10-29 14:49:59 -04001208 ret = find_get_pages_contig(inode->i_mapping, index,
Chris Mason5b050f02008-11-11 09:34:41 -05001209 min_t(unsigned long, nr_pages,
1210 ARRAY_SIZE(pages)), pages);
Chris Masonc8b97812008-10-29 14:49:59 -04001211 for (i = 0; i < ret; i++) {
1212 if (pages[i] != locked_page)
1213 unlock_page(pages[i]);
1214 page_cache_release(pages[i]);
1215 }
1216 nr_pages -= ret;
1217 index += ret;
1218 cond_resched();
1219 }
1220 return 0;
1221}
1222
1223static noinline int lock_delalloc_pages(struct inode *inode,
1224 struct page *locked_page,
1225 u64 delalloc_start,
1226 u64 delalloc_end)
1227{
1228 unsigned long index = delalloc_start >> PAGE_CACHE_SHIFT;
1229 unsigned long start_index = index;
1230 unsigned long end_index = delalloc_end >> PAGE_CACHE_SHIFT;
1231 unsigned long pages_locked = 0;
1232 struct page *pages[16];
1233 unsigned long nrpages;
1234 int ret;
1235 int i;
1236
1237 /* the caller is responsible for locking the start index */
1238 if (index == locked_page->index && index == end_index)
1239 return 0;
1240
1241 /* skip the page at the start index */
1242 nrpages = end_index - index + 1;
Chris Masond3977122009-01-05 21:25:51 -05001243 while (nrpages > 0) {
Chris Masonc8b97812008-10-29 14:49:59 -04001244 ret = find_get_pages_contig(inode->i_mapping, index,
Chris Mason5b050f02008-11-11 09:34:41 -05001245 min_t(unsigned long,
1246 nrpages, ARRAY_SIZE(pages)), pages);
Chris Masonc8b97812008-10-29 14:49:59 -04001247 if (ret == 0) {
1248 ret = -EAGAIN;
1249 goto done;
1250 }
1251 /* now we have an array of pages, lock them all */
1252 for (i = 0; i < ret; i++) {
1253 /*
1254 * the caller is taking responsibility for
1255 * locked_page
1256 */
Chris Mason771ed682008-11-06 22:02:51 -05001257 if (pages[i] != locked_page) {
Chris Masonc8b97812008-10-29 14:49:59 -04001258 lock_page(pages[i]);
Chris Masonf2b1c412008-11-10 07:31:30 -05001259 if (!PageDirty(pages[i]) ||
1260 pages[i]->mapping != inode->i_mapping) {
Chris Mason771ed682008-11-06 22:02:51 -05001261 ret = -EAGAIN;
1262 unlock_page(pages[i]);
1263 page_cache_release(pages[i]);
1264 goto done;
1265 }
1266 }
Chris Masonc8b97812008-10-29 14:49:59 -04001267 page_cache_release(pages[i]);
Chris Mason771ed682008-11-06 22:02:51 -05001268 pages_locked++;
Chris Masonc8b97812008-10-29 14:49:59 -04001269 }
Chris Masonc8b97812008-10-29 14:49:59 -04001270 nrpages -= ret;
1271 index += ret;
1272 cond_resched();
1273 }
1274 ret = 0;
1275done:
1276 if (ret && pages_locked) {
1277 __unlock_for_delalloc(inode, locked_page,
1278 delalloc_start,
1279 ((u64)(start_index + pages_locked - 1)) <<
1280 PAGE_CACHE_SHIFT);
1281 }
1282 return ret;
1283}
1284
1285/*
1286 * find a contiguous range of bytes in the file marked as delalloc, not
1287 * more than 'max_bytes'. start and end are used to return the range,
1288 *
1289 * 1 is returned if we find something, 0 if nothing was in the tree
1290 */
1291static noinline u64 find_lock_delalloc_range(struct inode *inode,
1292 struct extent_io_tree *tree,
1293 struct page *locked_page,
1294 u64 *start, u64 *end,
1295 u64 max_bytes)
1296{
1297 u64 delalloc_start;
1298 u64 delalloc_end;
1299 u64 found;
Chris Mason9655d292009-09-02 15:22:30 -04001300 struct extent_state *cached_state = NULL;
Chris Masonc8b97812008-10-29 14:49:59 -04001301 int ret;
1302 int loops = 0;
1303
1304again:
1305 /* step one, find a bunch of delalloc bytes starting at start */
1306 delalloc_start = *start;
1307 delalloc_end = 0;
1308 found = find_delalloc_range(tree, &delalloc_start, &delalloc_end,
Josef Bacikc2a128d2010-02-02 21:19:11 +00001309 max_bytes, &cached_state);
Chris Mason70b99e62008-10-31 12:46:39 -04001310 if (!found || delalloc_end <= *start) {
Chris Masonc8b97812008-10-29 14:49:59 -04001311 *start = delalloc_start;
1312 *end = delalloc_end;
Josef Bacikc2a128d2010-02-02 21:19:11 +00001313 free_extent_state(cached_state);
Chris Masonc8b97812008-10-29 14:49:59 -04001314 return found;
1315 }
1316
1317 /*
Chris Mason70b99e62008-10-31 12:46:39 -04001318 * start comes from the offset of locked_page. We have to lock
1319 * pages in order, so we can't process delalloc bytes before
1320 * locked_page
1321 */
Chris Masond3977122009-01-05 21:25:51 -05001322 if (delalloc_start < *start)
Chris Mason70b99e62008-10-31 12:46:39 -04001323 delalloc_start = *start;
Chris Mason70b99e62008-10-31 12:46:39 -04001324
1325 /*
Chris Masonc8b97812008-10-29 14:49:59 -04001326 * make sure to limit the number of pages we try to lock down
1327 * if we're looping.
1328 */
Chris Masond3977122009-01-05 21:25:51 -05001329 if (delalloc_end + 1 - delalloc_start > max_bytes && loops)
Chris Mason771ed682008-11-06 22:02:51 -05001330 delalloc_end = delalloc_start + PAGE_CACHE_SIZE - 1;
Chris Masond3977122009-01-05 21:25:51 -05001331
Chris Masonc8b97812008-10-29 14:49:59 -04001332 /* step two, lock all the pages after the page that has start */
1333 ret = lock_delalloc_pages(inode, locked_page,
1334 delalloc_start, delalloc_end);
1335 if (ret == -EAGAIN) {
1336 /* some of the pages are gone, lets avoid looping by
1337 * shortening the size of the delalloc range we're searching
1338 */
Chris Mason9655d292009-09-02 15:22:30 -04001339 free_extent_state(cached_state);
Chris Masonc8b97812008-10-29 14:49:59 -04001340 if (!loops) {
1341 unsigned long offset = (*start) & (PAGE_CACHE_SIZE - 1);
1342 max_bytes = PAGE_CACHE_SIZE - offset;
1343 loops = 1;
1344 goto again;
1345 } else {
1346 found = 0;
1347 goto out_failed;
1348 }
1349 }
1350 BUG_ON(ret);
1351
1352 /* step three, lock the state bits for the whole range */
Chris Mason9655d292009-09-02 15:22:30 -04001353 lock_extent_bits(tree, delalloc_start, delalloc_end,
1354 0, &cached_state, GFP_NOFS);
Chris Masonc8b97812008-10-29 14:49:59 -04001355
1356 /* then test to make sure it is all still delalloc */
1357 ret = test_range_bit(tree, delalloc_start, delalloc_end,
Chris Mason9655d292009-09-02 15:22:30 -04001358 EXTENT_DELALLOC, 1, cached_state);
Chris Masonc8b97812008-10-29 14:49:59 -04001359 if (!ret) {
Chris Mason9655d292009-09-02 15:22:30 -04001360 unlock_extent_cached(tree, delalloc_start, delalloc_end,
1361 &cached_state, GFP_NOFS);
Chris Masonc8b97812008-10-29 14:49:59 -04001362 __unlock_for_delalloc(inode, locked_page,
1363 delalloc_start, delalloc_end);
1364 cond_resched();
1365 goto again;
1366 }
Chris Mason9655d292009-09-02 15:22:30 -04001367 free_extent_state(cached_state);
Chris Masonc8b97812008-10-29 14:49:59 -04001368 *start = delalloc_start;
1369 *end = delalloc_end;
1370out_failed:
1371 return found;
1372}
1373
1374int extent_clear_unlock_delalloc(struct inode *inode,
1375 struct extent_io_tree *tree,
1376 u64 start, u64 end, struct page *locked_page,
Chris Masona791e352009-10-08 11:27:10 -04001377 unsigned long op)
Chris Masonc8b97812008-10-29 14:49:59 -04001378{
1379 int ret;
1380 struct page *pages[16];
1381 unsigned long index = start >> PAGE_CACHE_SHIFT;
1382 unsigned long end_index = end >> PAGE_CACHE_SHIFT;
1383 unsigned long nr_pages = end_index - index + 1;
1384 int i;
Chris Mason771ed682008-11-06 22:02:51 -05001385 int clear_bits = 0;
Chris Masonc8b97812008-10-29 14:49:59 -04001386
Chris Masona791e352009-10-08 11:27:10 -04001387 if (op & EXTENT_CLEAR_UNLOCK)
Chris Mason771ed682008-11-06 22:02:51 -05001388 clear_bits |= EXTENT_LOCKED;
Chris Masona791e352009-10-08 11:27:10 -04001389 if (op & EXTENT_CLEAR_DIRTY)
Chris Masonc8b97812008-10-29 14:49:59 -04001390 clear_bits |= EXTENT_DIRTY;
1391
Chris Masona791e352009-10-08 11:27:10 -04001392 if (op & EXTENT_CLEAR_DELALLOC)
Chris Mason771ed682008-11-06 22:02:51 -05001393 clear_bits |= EXTENT_DELALLOC;
1394
Chris Mason2c64c532009-09-02 15:04:12 -04001395 clear_extent_bit(tree, start, end, clear_bits, 1, 0, NULL, GFP_NOFS);
Josef Bacik32c00af2009-10-08 13:34:05 -04001396 if (!(op & (EXTENT_CLEAR_UNLOCK_PAGE | EXTENT_CLEAR_DIRTY |
1397 EXTENT_SET_WRITEBACK | EXTENT_END_WRITEBACK |
1398 EXTENT_SET_PRIVATE2)))
Chris Mason771ed682008-11-06 22:02:51 -05001399 return 0;
Chris Masonc8b97812008-10-29 14:49:59 -04001400
Chris Masond3977122009-01-05 21:25:51 -05001401 while (nr_pages > 0) {
Chris Masonc8b97812008-10-29 14:49:59 -04001402 ret = find_get_pages_contig(inode->i_mapping, index,
Chris Mason5b050f02008-11-11 09:34:41 -05001403 min_t(unsigned long,
1404 nr_pages, ARRAY_SIZE(pages)), pages);
Chris Masonc8b97812008-10-29 14:49:59 -04001405 for (i = 0; i < ret; i++) {
Chris Mason8b62b722009-09-02 16:53:46 -04001406
Chris Masona791e352009-10-08 11:27:10 -04001407 if (op & EXTENT_SET_PRIVATE2)
Chris Mason8b62b722009-09-02 16:53:46 -04001408 SetPagePrivate2(pages[i]);
1409
Chris Masonc8b97812008-10-29 14:49:59 -04001410 if (pages[i] == locked_page) {
1411 page_cache_release(pages[i]);
1412 continue;
1413 }
Chris Masona791e352009-10-08 11:27:10 -04001414 if (op & EXTENT_CLEAR_DIRTY)
Chris Masonc8b97812008-10-29 14:49:59 -04001415 clear_page_dirty_for_io(pages[i]);
Chris Masona791e352009-10-08 11:27:10 -04001416 if (op & EXTENT_SET_WRITEBACK)
Chris Masonc8b97812008-10-29 14:49:59 -04001417 set_page_writeback(pages[i]);
Chris Masona791e352009-10-08 11:27:10 -04001418 if (op & EXTENT_END_WRITEBACK)
Chris Masonc8b97812008-10-29 14:49:59 -04001419 end_page_writeback(pages[i]);
Chris Masona791e352009-10-08 11:27:10 -04001420 if (op & EXTENT_CLEAR_UNLOCK_PAGE)
Chris Mason771ed682008-11-06 22:02:51 -05001421 unlock_page(pages[i]);
Chris Masonc8b97812008-10-29 14:49:59 -04001422 page_cache_release(pages[i]);
1423 }
1424 nr_pages -= ret;
1425 index += ret;
1426 cond_resched();
1427 }
1428 return 0;
1429}
Chris Masonc8b97812008-10-29 14:49:59 -04001430
Chris Masond352ac62008-09-29 15:18:18 -04001431/*
1432 * count the number of bytes in the tree that have a given bit(s)
1433 * set. This can be fairly slow, except for EXTENT_DIRTY which is
1434 * cached. The total number found is returned.
1435 */
Chris Masond1310b22008-01-24 16:13:08 -05001436u64 count_range_bits(struct extent_io_tree *tree,
1437 u64 *start, u64 search_end, u64 max_bytes,
Chris Masonec29ed52011-02-23 16:23:20 -05001438 unsigned long bits, int contig)
Chris Masond1310b22008-01-24 16:13:08 -05001439{
1440 struct rb_node *node;
1441 struct extent_state *state;
1442 u64 cur_start = *start;
1443 u64 total_bytes = 0;
Chris Masonec29ed52011-02-23 16:23:20 -05001444 u64 last = 0;
Chris Masond1310b22008-01-24 16:13:08 -05001445 int found = 0;
1446
1447 if (search_end <= cur_start) {
Chris Masond1310b22008-01-24 16:13:08 -05001448 WARN_ON(1);
1449 return 0;
1450 }
1451
Chris Masoncad321a2008-12-17 14:51:42 -05001452 spin_lock(&tree->lock);
Chris Masond1310b22008-01-24 16:13:08 -05001453 if (cur_start == 0 && bits == EXTENT_DIRTY) {
1454 total_bytes = tree->dirty_bytes;
1455 goto out;
1456 }
1457 /*
1458 * this search will find all the extents that end after
1459 * our range starts.
1460 */
Chris Mason80ea96b2008-02-01 14:51:59 -05001461 node = tree_search(tree, cur_start);
Chris Masond3977122009-01-05 21:25:51 -05001462 if (!node)
Chris Masond1310b22008-01-24 16:13:08 -05001463 goto out;
Chris Masond1310b22008-01-24 16:13:08 -05001464
Chris Masond3977122009-01-05 21:25:51 -05001465 while (1) {
Chris Masond1310b22008-01-24 16:13:08 -05001466 state = rb_entry(node, struct extent_state, rb_node);
1467 if (state->start > search_end)
1468 break;
Chris Masonec29ed52011-02-23 16:23:20 -05001469 if (contig && found && state->start > last + 1)
1470 break;
1471 if (state->end >= cur_start && (state->state & bits) == bits) {
Chris Masond1310b22008-01-24 16:13:08 -05001472 total_bytes += min(search_end, state->end) + 1 -
1473 max(cur_start, state->start);
1474 if (total_bytes >= max_bytes)
1475 break;
1476 if (!found) {
Josef Bacikaf60bed2011-05-04 11:11:17 -04001477 *start = max(cur_start, state->start);
Chris Masond1310b22008-01-24 16:13:08 -05001478 found = 1;
1479 }
Chris Masonec29ed52011-02-23 16:23:20 -05001480 last = state->end;
1481 } else if (contig && found) {
1482 break;
Chris Masond1310b22008-01-24 16:13:08 -05001483 }
1484 node = rb_next(node);
1485 if (!node)
1486 break;
1487 }
1488out:
Chris Masoncad321a2008-12-17 14:51:42 -05001489 spin_unlock(&tree->lock);
Chris Masond1310b22008-01-24 16:13:08 -05001490 return total_bytes;
1491}
Christoph Hellwigb2950862008-12-02 09:54:17 -05001492
Chris Masond352ac62008-09-29 15:18:18 -04001493/*
1494 * set the private field for a given byte offset in the tree. If there isn't
1495 * an extent_state there already, this does nothing.
1496 */
Chris Masond1310b22008-01-24 16:13:08 -05001497int set_state_private(struct extent_io_tree *tree, u64 start, u64 private)
1498{
1499 struct rb_node *node;
1500 struct extent_state *state;
1501 int ret = 0;
1502
Chris Masoncad321a2008-12-17 14:51:42 -05001503 spin_lock(&tree->lock);
Chris Masond1310b22008-01-24 16:13:08 -05001504 /*
1505 * this search will find all the extents that end after
1506 * our range starts.
1507 */
Chris Mason80ea96b2008-02-01 14:51:59 -05001508 node = tree_search(tree, start);
Peter2b114d12008-04-01 11:21:40 -04001509 if (!node) {
Chris Masond1310b22008-01-24 16:13:08 -05001510 ret = -ENOENT;
1511 goto out;
1512 }
1513 state = rb_entry(node, struct extent_state, rb_node);
1514 if (state->start != start) {
1515 ret = -ENOENT;
1516 goto out;
1517 }
1518 state->private = private;
1519out:
Chris Masoncad321a2008-12-17 14:51:42 -05001520 spin_unlock(&tree->lock);
Chris Masond1310b22008-01-24 16:13:08 -05001521 return ret;
1522}
1523
1524int get_state_private(struct extent_io_tree *tree, u64 start, u64 *private)
1525{
1526 struct rb_node *node;
1527 struct extent_state *state;
1528 int ret = 0;
1529
Chris Masoncad321a2008-12-17 14:51:42 -05001530 spin_lock(&tree->lock);
Chris Masond1310b22008-01-24 16:13:08 -05001531 /*
1532 * this search will find all the extents that end after
1533 * our range starts.
1534 */
Chris Mason80ea96b2008-02-01 14:51:59 -05001535 node = tree_search(tree, start);
Peter2b114d12008-04-01 11:21:40 -04001536 if (!node) {
Chris Masond1310b22008-01-24 16:13:08 -05001537 ret = -ENOENT;
1538 goto out;
1539 }
1540 state = rb_entry(node, struct extent_state, rb_node);
1541 if (state->start != start) {
1542 ret = -ENOENT;
1543 goto out;
1544 }
1545 *private = state->private;
1546out:
Chris Masoncad321a2008-12-17 14:51:42 -05001547 spin_unlock(&tree->lock);
Chris Masond1310b22008-01-24 16:13:08 -05001548 return ret;
1549}
1550
1551/*
1552 * searches a range in the state tree for a given mask.
Chris Mason70dec802008-01-29 09:59:12 -05001553 * If 'filled' == 1, this returns 1 only if every extent in the tree
Chris Masond1310b22008-01-24 16:13:08 -05001554 * has the bits set. Otherwise, 1 is returned if any bit in the
1555 * range is found set.
1556 */
1557int test_range_bit(struct extent_io_tree *tree, u64 start, u64 end,
Chris Mason9655d292009-09-02 15:22:30 -04001558 int bits, int filled, struct extent_state *cached)
Chris Masond1310b22008-01-24 16:13:08 -05001559{
1560 struct extent_state *state = NULL;
1561 struct rb_node *node;
1562 int bitset = 0;
Chris Masond1310b22008-01-24 16:13:08 -05001563
Chris Masoncad321a2008-12-17 14:51:42 -05001564 spin_lock(&tree->lock);
Chris Mason9655d292009-09-02 15:22:30 -04001565 if (cached && cached->tree && cached->start == start)
1566 node = &cached->rb_node;
1567 else
1568 node = tree_search(tree, start);
Chris Masond1310b22008-01-24 16:13:08 -05001569 while (node && start <= end) {
1570 state = rb_entry(node, struct extent_state, rb_node);
1571
1572 if (filled && state->start > start) {
1573 bitset = 0;
1574 break;
1575 }
1576
1577 if (state->start > end)
1578 break;
1579
1580 if (state->state & bits) {
1581 bitset = 1;
1582 if (!filled)
1583 break;
1584 } else if (filled) {
1585 bitset = 0;
1586 break;
1587 }
Chris Mason46562cec2009-09-23 20:23:16 -04001588
1589 if (state->end == (u64)-1)
1590 break;
1591
Chris Masond1310b22008-01-24 16:13:08 -05001592 start = state->end + 1;
1593 if (start > end)
1594 break;
1595 node = rb_next(node);
1596 if (!node) {
1597 if (filled)
1598 bitset = 0;
1599 break;
1600 }
1601 }
Chris Masoncad321a2008-12-17 14:51:42 -05001602 spin_unlock(&tree->lock);
Chris Masond1310b22008-01-24 16:13:08 -05001603 return bitset;
1604}
Chris Masond1310b22008-01-24 16:13:08 -05001605
1606/*
1607 * helper function to set a given page up to date if all the
1608 * extents in the tree for that page are up to date
1609 */
1610static int check_page_uptodate(struct extent_io_tree *tree,
1611 struct page *page)
1612{
1613 u64 start = (u64)page->index << PAGE_CACHE_SHIFT;
1614 u64 end = start + PAGE_CACHE_SIZE - 1;
Chris Mason9655d292009-09-02 15:22:30 -04001615 if (test_range_bit(tree, start, end, EXTENT_UPTODATE, 1, NULL))
Chris Masond1310b22008-01-24 16:13:08 -05001616 SetPageUptodate(page);
1617 return 0;
1618}
1619
1620/*
1621 * helper function to unlock a page if all the extents in the tree
1622 * for that page are unlocked
1623 */
1624static int check_page_locked(struct extent_io_tree *tree,
1625 struct page *page)
1626{
1627 u64 start = (u64)page->index << PAGE_CACHE_SHIFT;
1628 u64 end = start + PAGE_CACHE_SIZE - 1;
Chris Mason9655d292009-09-02 15:22:30 -04001629 if (!test_range_bit(tree, start, end, EXTENT_LOCKED, 0, NULL))
Chris Masond1310b22008-01-24 16:13:08 -05001630 unlock_page(page);
1631 return 0;
1632}
1633
1634/*
1635 * helper function to end page writeback if all the extents
1636 * in the tree for that page are done with writeback
1637 */
1638static int check_page_writeback(struct extent_io_tree *tree,
1639 struct page *page)
1640{
Chris Mason1edbb732009-09-02 13:24:36 -04001641 end_page_writeback(page);
Chris Masond1310b22008-01-24 16:13:08 -05001642 return 0;
1643}
1644
1645/* lots and lots of room for performance fixes in the end_bio funcs */
1646
1647/*
1648 * after a writepage IO is done, we need to:
1649 * clear the uptodate bits on error
1650 * clear the writeback bits in the extent tree for this IO
1651 * end_page_writeback if the page has no more pending IO
1652 *
1653 * Scheduling is not allowed, so the extent state tree is expected
1654 * to have one and only one object corresponding to this IO.
1655 */
Chris Masond1310b22008-01-24 16:13:08 -05001656static void end_bio_extent_writepage(struct bio *bio, int err)
Chris Masond1310b22008-01-24 16:13:08 -05001657{
Chris Mason1259ab72008-05-12 13:39:03 -04001658 int uptodate = err == 0;
Chris Masond1310b22008-01-24 16:13:08 -05001659 struct bio_vec *bvec = bio->bi_io_vec + bio->bi_vcnt - 1;
David Woodhouse902b22f2008-08-20 08:51:49 -04001660 struct extent_io_tree *tree;
Chris Masond1310b22008-01-24 16:13:08 -05001661 u64 start;
1662 u64 end;
1663 int whole_page;
Chris Mason1259ab72008-05-12 13:39:03 -04001664 int ret;
Chris Masond1310b22008-01-24 16:13:08 -05001665
Chris Masond1310b22008-01-24 16:13:08 -05001666 do {
1667 struct page *page = bvec->bv_page;
David Woodhouse902b22f2008-08-20 08:51:49 -04001668 tree = &BTRFS_I(page->mapping->host)->io_tree;
1669
Chris Masond1310b22008-01-24 16:13:08 -05001670 start = ((u64)page->index << PAGE_CACHE_SHIFT) +
1671 bvec->bv_offset;
1672 end = start + bvec->bv_len - 1;
1673
1674 if (bvec->bv_offset == 0 && bvec->bv_len == PAGE_CACHE_SIZE)
1675 whole_page = 1;
1676 else
1677 whole_page = 0;
1678
1679 if (--bvec >= bio->bi_io_vec)
1680 prefetchw(&bvec->bv_page->flags);
Chris Mason1259ab72008-05-12 13:39:03 -04001681 if (tree->ops && tree->ops->writepage_end_io_hook) {
1682 ret = tree->ops->writepage_end_io_hook(page, start,
David Woodhouse902b22f2008-08-20 08:51:49 -04001683 end, NULL, uptodate);
Chris Mason1259ab72008-05-12 13:39:03 -04001684 if (ret)
1685 uptodate = 0;
1686 }
1687
1688 if (!uptodate && tree->ops &&
1689 tree->ops->writepage_io_failed_hook) {
1690 ret = tree->ops->writepage_io_failed_hook(bio, page,
David Woodhouse902b22f2008-08-20 08:51:49 -04001691 start, end, NULL);
Chris Mason1259ab72008-05-12 13:39:03 -04001692 if (ret == 0) {
Chris Mason1259ab72008-05-12 13:39:03 -04001693 uptodate = (err == 0);
1694 continue;
1695 }
1696 }
1697
Chris Masond1310b22008-01-24 16:13:08 -05001698 if (!uptodate) {
Josef Bacik2ac55d42010-02-03 19:33:23 +00001699 clear_extent_uptodate(tree, start, end, NULL, GFP_NOFS);
Chris Masond1310b22008-01-24 16:13:08 -05001700 ClearPageUptodate(page);
1701 SetPageError(page);
1702 }
Chris Mason70dec802008-01-29 09:59:12 -05001703
Chris Masond1310b22008-01-24 16:13:08 -05001704 if (whole_page)
1705 end_page_writeback(page);
1706 else
1707 check_page_writeback(tree, page);
Chris Masond1310b22008-01-24 16:13:08 -05001708 } while (bvec >= bio->bi_io_vec);
Chris Mason2b1f55b2008-09-24 11:48:04 -04001709
Chris Masond1310b22008-01-24 16:13:08 -05001710 bio_put(bio);
Chris Masond1310b22008-01-24 16:13:08 -05001711}
1712
1713/*
1714 * after a readpage IO is done, we need to:
1715 * clear the uptodate bits on error
1716 * set the uptodate bits if things worked
1717 * set the page up to date if all extents in the tree are uptodate
1718 * clear the lock bit in the extent tree
1719 * unlock the page if there are no other extents locked for it
1720 *
1721 * Scheduling is not allowed, so the extent state tree is expected
1722 * to have one and only one object corresponding to this IO.
1723 */
Chris Masond1310b22008-01-24 16:13:08 -05001724static void end_bio_extent_readpage(struct bio *bio, int err)
Chris Masond1310b22008-01-24 16:13:08 -05001725{
1726 int uptodate = test_bit(BIO_UPTODATE, &bio->bi_flags);
Chris Mason4125bf72010-02-03 18:18:45 +00001727 struct bio_vec *bvec_end = bio->bi_io_vec + bio->bi_vcnt - 1;
1728 struct bio_vec *bvec = bio->bi_io_vec;
David Woodhouse902b22f2008-08-20 08:51:49 -04001729 struct extent_io_tree *tree;
Chris Masond1310b22008-01-24 16:13:08 -05001730 u64 start;
1731 u64 end;
1732 int whole_page;
1733 int ret;
1734
Chris Masond20f7042008-12-08 16:58:54 -05001735 if (err)
1736 uptodate = 0;
1737
Chris Masond1310b22008-01-24 16:13:08 -05001738 do {
1739 struct page *page = bvec->bv_page;
Arne Jansen507903b2011-04-06 10:02:20 +00001740 struct extent_state *cached = NULL;
1741 struct extent_state *state;
1742
David Woodhouse902b22f2008-08-20 08:51:49 -04001743 tree = &BTRFS_I(page->mapping->host)->io_tree;
1744
Chris Masond1310b22008-01-24 16:13:08 -05001745 start = ((u64)page->index << PAGE_CACHE_SHIFT) +
1746 bvec->bv_offset;
1747 end = start + bvec->bv_len - 1;
1748
1749 if (bvec->bv_offset == 0 && bvec->bv_len == PAGE_CACHE_SIZE)
1750 whole_page = 1;
1751 else
1752 whole_page = 0;
1753
Chris Mason4125bf72010-02-03 18:18:45 +00001754 if (++bvec <= bvec_end)
Chris Masond1310b22008-01-24 16:13:08 -05001755 prefetchw(&bvec->bv_page->flags);
1756
Arne Jansen507903b2011-04-06 10:02:20 +00001757 spin_lock(&tree->lock);
Chris Mason0d399202011-04-16 06:55:39 -04001758 state = find_first_extent_bit_state(tree, start, EXTENT_LOCKED);
Chris Mason109b36a2011-04-12 13:57:39 -04001759 if (state && state->start == start) {
Arne Jansen507903b2011-04-06 10:02:20 +00001760 /*
1761 * take a reference on the state, unlock will drop
1762 * the ref
1763 */
1764 cache_state(state, &cached);
1765 }
1766 spin_unlock(&tree->lock);
1767
Chris Masond1310b22008-01-24 16:13:08 -05001768 if (uptodate && tree->ops && tree->ops->readpage_end_io_hook) {
Chris Mason70dec802008-01-29 09:59:12 -05001769 ret = tree->ops->readpage_end_io_hook(page, start, end,
Arne Jansen507903b2011-04-06 10:02:20 +00001770 state);
Chris Masond1310b22008-01-24 16:13:08 -05001771 if (ret)
1772 uptodate = 0;
1773 }
Chris Mason7e383262008-04-09 16:28:12 -04001774 if (!uptodate && tree->ops &&
1775 tree->ops->readpage_io_failed_hook) {
1776 ret = tree->ops->readpage_io_failed_hook(bio, page,
David Woodhouse902b22f2008-08-20 08:51:49 -04001777 start, end, NULL);
Chris Mason7e383262008-04-09 16:28:12 -04001778 if (ret == 0) {
Chris Mason3b951512008-04-17 11:29:12 -04001779 uptodate =
1780 test_bit(BIO_UPTODATE, &bio->bi_flags);
Chris Masond20f7042008-12-08 16:58:54 -05001781 if (err)
1782 uptodate = 0;
Arne Jansen507903b2011-04-06 10:02:20 +00001783 uncache_state(&cached);
Chris Mason7e383262008-04-09 16:28:12 -04001784 continue;
1785 }
1786 }
Chris Mason70dec802008-01-29 09:59:12 -05001787
Chris Mason771ed682008-11-06 22:02:51 -05001788 if (uptodate) {
Arne Jansen507903b2011-04-06 10:02:20 +00001789 set_extent_uptodate(tree, start, end, &cached,
David Woodhouse902b22f2008-08-20 08:51:49 -04001790 GFP_ATOMIC);
Chris Mason771ed682008-11-06 22:02:51 -05001791 }
Arne Jansen507903b2011-04-06 10:02:20 +00001792 unlock_extent_cached(tree, start, end, &cached, GFP_ATOMIC);
Chris Masond1310b22008-01-24 16:13:08 -05001793
Chris Mason70dec802008-01-29 09:59:12 -05001794 if (whole_page) {
1795 if (uptodate) {
1796 SetPageUptodate(page);
1797 } else {
1798 ClearPageUptodate(page);
1799 SetPageError(page);
1800 }
Chris Masond1310b22008-01-24 16:13:08 -05001801 unlock_page(page);
Chris Mason70dec802008-01-29 09:59:12 -05001802 } else {
1803 if (uptodate) {
1804 check_page_uptodate(tree, page);
1805 } else {
1806 ClearPageUptodate(page);
1807 SetPageError(page);
1808 }
Chris Masond1310b22008-01-24 16:13:08 -05001809 check_page_locked(tree, page);
Chris Mason70dec802008-01-29 09:59:12 -05001810 }
Chris Mason4125bf72010-02-03 18:18:45 +00001811 } while (bvec <= bvec_end);
Chris Masond1310b22008-01-24 16:13:08 -05001812
1813 bio_put(bio);
Chris Masond1310b22008-01-24 16:13:08 -05001814}
1815
Miao Xie88f794e2010-11-22 03:02:55 +00001816struct bio *
1817btrfs_bio_alloc(struct block_device *bdev, u64 first_sector, int nr_vecs,
1818 gfp_t gfp_flags)
Chris Masond1310b22008-01-24 16:13:08 -05001819{
1820 struct bio *bio;
1821
1822 bio = bio_alloc(gfp_flags, nr_vecs);
1823
1824 if (bio == NULL && (current->flags & PF_MEMALLOC)) {
1825 while (!bio && (nr_vecs /= 2))
1826 bio = bio_alloc(gfp_flags, nr_vecs);
1827 }
1828
1829 if (bio) {
Chris Masone1c4b742008-04-22 13:26:46 -04001830 bio->bi_size = 0;
Chris Masond1310b22008-01-24 16:13:08 -05001831 bio->bi_bdev = bdev;
1832 bio->bi_sector = first_sector;
1833 }
1834 return bio;
1835}
1836
Chris Masonc8b97812008-10-29 14:49:59 -04001837static int submit_one_bio(int rw, struct bio *bio, int mirror_num,
1838 unsigned long bio_flags)
Chris Masond1310b22008-01-24 16:13:08 -05001839{
Chris Masond1310b22008-01-24 16:13:08 -05001840 int ret = 0;
Chris Mason70dec802008-01-29 09:59:12 -05001841 struct bio_vec *bvec = bio->bi_io_vec + bio->bi_vcnt - 1;
1842 struct page *page = bvec->bv_page;
1843 struct extent_io_tree *tree = bio->bi_private;
Chris Mason70dec802008-01-29 09:59:12 -05001844 u64 start;
Chris Mason70dec802008-01-29 09:59:12 -05001845
1846 start = ((u64)page->index << PAGE_CACHE_SHIFT) + bvec->bv_offset;
Chris Mason70dec802008-01-29 09:59:12 -05001847
David Woodhouse902b22f2008-08-20 08:51:49 -04001848 bio->bi_private = NULL;
Chris Masond1310b22008-01-24 16:13:08 -05001849
1850 bio_get(bio);
1851
Chris Mason065631f2008-02-20 12:07:25 -05001852 if (tree->ops && tree->ops->submit_bio_hook)
liubo6b82ce82011-01-26 06:21:39 +00001853 ret = tree->ops->submit_bio_hook(page->mapping->host, rw, bio,
Chris Masoneaf25d92010-05-25 09:48:28 -04001854 mirror_num, bio_flags, start);
Chris Mason0b86a832008-03-24 15:01:56 -04001855 else
1856 submit_bio(rw, bio);
Chris Masond1310b22008-01-24 16:13:08 -05001857 if (bio_flagged(bio, BIO_EOPNOTSUPP))
1858 ret = -EOPNOTSUPP;
1859 bio_put(bio);
1860 return ret;
1861}
1862
1863static int submit_extent_page(int rw, struct extent_io_tree *tree,
1864 struct page *page, sector_t sector,
1865 size_t size, unsigned long offset,
1866 struct block_device *bdev,
1867 struct bio **bio_ret,
1868 unsigned long max_pages,
Chris Masonf1885912008-04-09 16:28:12 -04001869 bio_end_io_t end_io_func,
Chris Masonc8b97812008-10-29 14:49:59 -04001870 int mirror_num,
1871 unsigned long prev_bio_flags,
1872 unsigned long bio_flags)
Chris Masond1310b22008-01-24 16:13:08 -05001873{
1874 int ret = 0;
1875 struct bio *bio;
1876 int nr;
Chris Masonc8b97812008-10-29 14:49:59 -04001877 int contig = 0;
1878 int this_compressed = bio_flags & EXTENT_BIO_COMPRESSED;
1879 int old_compressed = prev_bio_flags & EXTENT_BIO_COMPRESSED;
Chris Mason5b050f02008-11-11 09:34:41 -05001880 size_t page_size = min_t(size_t, size, PAGE_CACHE_SIZE);
Chris Masond1310b22008-01-24 16:13:08 -05001881
1882 if (bio_ret && *bio_ret) {
1883 bio = *bio_ret;
Chris Masonc8b97812008-10-29 14:49:59 -04001884 if (old_compressed)
1885 contig = bio->bi_sector == sector;
1886 else
1887 contig = bio->bi_sector + (bio->bi_size >> 9) ==
1888 sector;
1889
1890 if (prev_bio_flags != bio_flags || !contig ||
Chris Mason239b14b2008-03-24 15:02:07 -04001891 (tree->ops && tree->ops->merge_bio_hook &&
Chris Masonc8b97812008-10-29 14:49:59 -04001892 tree->ops->merge_bio_hook(page, offset, page_size, bio,
1893 bio_flags)) ||
1894 bio_add_page(bio, page, page_size, offset) < page_size) {
1895 ret = submit_one_bio(rw, bio, mirror_num,
1896 prev_bio_flags);
Chris Masond1310b22008-01-24 16:13:08 -05001897 bio = NULL;
1898 } else {
1899 return 0;
1900 }
1901 }
Chris Masonc8b97812008-10-29 14:49:59 -04001902 if (this_compressed)
1903 nr = BIO_MAX_PAGES;
1904 else
1905 nr = bio_get_nr_vecs(bdev);
1906
Miao Xie88f794e2010-11-22 03:02:55 +00001907 bio = btrfs_bio_alloc(bdev, sector, nr, GFP_NOFS | __GFP_HIGH);
Tsutomu Itoh5df67082011-02-01 09:17:35 +00001908 if (!bio)
1909 return -ENOMEM;
Chris Mason70dec802008-01-29 09:59:12 -05001910
Chris Masonc8b97812008-10-29 14:49:59 -04001911 bio_add_page(bio, page, page_size, offset);
Chris Masond1310b22008-01-24 16:13:08 -05001912 bio->bi_end_io = end_io_func;
1913 bio->bi_private = tree;
Chris Mason70dec802008-01-29 09:59:12 -05001914
Chris Masond3977122009-01-05 21:25:51 -05001915 if (bio_ret)
Chris Masond1310b22008-01-24 16:13:08 -05001916 *bio_ret = bio;
Chris Masond3977122009-01-05 21:25:51 -05001917 else
Chris Masonc8b97812008-10-29 14:49:59 -04001918 ret = submit_one_bio(rw, bio, mirror_num, bio_flags);
Chris Masond1310b22008-01-24 16:13:08 -05001919
1920 return ret;
1921}
1922
1923void set_page_extent_mapped(struct page *page)
1924{
1925 if (!PagePrivate(page)) {
1926 SetPagePrivate(page);
Chris Masond1310b22008-01-24 16:13:08 -05001927 page_cache_get(page);
Chris Mason6af118ce2008-07-22 11:18:07 -04001928 set_page_private(page, EXTENT_PAGE_PRIVATE);
Chris Masond1310b22008-01-24 16:13:08 -05001929 }
1930}
1931
Christoph Hellwigb2950862008-12-02 09:54:17 -05001932static void set_page_extent_head(struct page *page, unsigned long len)
Chris Masond1310b22008-01-24 16:13:08 -05001933{
Chris Masoneb14ab82011-02-10 12:35:00 -05001934 WARN_ON(!PagePrivate(page));
Chris Masond1310b22008-01-24 16:13:08 -05001935 set_page_private(page, EXTENT_PAGE_PRIVATE_FIRST_PAGE | len << 2);
1936}
1937
1938/*
1939 * basic readpage implementation. Locked extent state structs are inserted
1940 * into the tree that are removed when the IO is done (by the end_io
1941 * handlers)
1942 */
1943static int __extent_read_full_page(struct extent_io_tree *tree,
1944 struct page *page,
1945 get_extent_t *get_extent,
Chris Masonc8b97812008-10-29 14:49:59 -04001946 struct bio **bio, int mirror_num,
1947 unsigned long *bio_flags)
Chris Masond1310b22008-01-24 16:13:08 -05001948{
1949 struct inode *inode = page->mapping->host;
1950 u64 start = (u64)page->index << PAGE_CACHE_SHIFT;
1951 u64 page_end = start + PAGE_CACHE_SIZE - 1;
1952 u64 end;
1953 u64 cur = start;
1954 u64 extent_offset;
1955 u64 last_byte = i_size_read(inode);
1956 u64 block_start;
1957 u64 cur_end;
1958 sector_t sector;
1959 struct extent_map *em;
1960 struct block_device *bdev;
Josef Bacik11c65dc2010-05-23 11:07:21 -04001961 struct btrfs_ordered_extent *ordered;
Chris Masond1310b22008-01-24 16:13:08 -05001962 int ret;
1963 int nr = 0;
David Sterba306e16c2011-04-19 14:29:38 +02001964 size_t pg_offset = 0;
Chris Masond1310b22008-01-24 16:13:08 -05001965 size_t iosize;
Chris Masonc8b97812008-10-29 14:49:59 -04001966 size_t disk_io_size;
Chris Masond1310b22008-01-24 16:13:08 -05001967 size_t blocksize = inode->i_sb->s_blocksize;
Chris Masonc8b97812008-10-29 14:49:59 -04001968 unsigned long this_bio_flag = 0;
Chris Masond1310b22008-01-24 16:13:08 -05001969
1970 set_page_extent_mapped(page);
1971
1972 end = page_end;
Josef Bacik11c65dc2010-05-23 11:07:21 -04001973 while (1) {
1974 lock_extent(tree, start, end, GFP_NOFS);
1975 ordered = btrfs_lookup_ordered_extent(inode, start);
1976 if (!ordered)
1977 break;
1978 unlock_extent(tree, start, end, GFP_NOFS);
1979 btrfs_start_ordered_extent(inode, ordered, 1);
1980 btrfs_put_ordered_extent(ordered);
1981 }
Chris Masond1310b22008-01-24 16:13:08 -05001982
Chris Masonc8b97812008-10-29 14:49:59 -04001983 if (page->index == last_byte >> PAGE_CACHE_SHIFT) {
1984 char *userpage;
1985 size_t zero_offset = last_byte & (PAGE_CACHE_SIZE - 1);
1986
1987 if (zero_offset) {
1988 iosize = PAGE_CACHE_SIZE - zero_offset;
1989 userpage = kmap_atomic(page, KM_USER0);
1990 memset(userpage + zero_offset, 0, iosize);
1991 flush_dcache_page(page);
1992 kunmap_atomic(userpage, KM_USER0);
1993 }
1994 }
Chris Masond1310b22008-01-24 16:13:08 -05001995 while (cur <= end) {
1996 if (cur >= last_byte) {
1997 char *userpage;
Arne Jansen507903b2011-04-06 10:02:20 +00001998 struct extent_state *cached = NULL;
1999
David Sterba306e16c2011-04-19 14:29:38 +02002000 iosize = PAGE_CACHE_SIZE - pg_offset;
Chris Masond1310b22008-01-24 16:13:08 -05002001 userpage = kmap_atomic(page, KM_USER0);
David Sterba306e16c2011-04-19 14:29:38 +02002002 memset(userpage + pg_offset, 0, iosize);
Chris Masond1310b22008-01-24 16:13:08 -05002003 flush_dcache_page(page);
2004 kunmap_atomic(userpage, KM_USER0);
2005 set_extent_uptodate(tree, cur, cur + iosize - 1,
Arne Jansen507903b2011-04-06 10:02:20 +00002006 &cached, GFP_NOFS);
2007 unlock_extent_cached(tree, cur, cur + iosize - 1,
2008 &cached, GFP_NOFS);
Chris Masond1310b22008-01-24 16:13:08 -05002009 break;
2010 }
David Sterba306e16c2011-04-19 14:29:38 +02002011 em = get_extent(inode, page, pg_offset, cur,
Chris Masond1310b22008-01-24 16:13:08 -05002012 end - cur + 1, 0);
David Sterbac7040052011-04-19 18:00:01 +02002013 if (IS_ERR_OR_NULL(em)) {
Chris Masond1310b22008-01-24 16:13:08 -05002014 SetPageError(page);
2015 unlock_extent(tree, cur, end, GFP_NOFS);
2016 break;
2017 }
Chris Masond1310b22008-01-24 16:13:08 -05002018 extent_offset = cur - em->start;
2019 BUG_ON(extent_map_end(em) <= cur);
2020 BUG_ON(end < cur);
2021
Li Zefan261507a02010-12-17 14:21:50 +08002022 if (test_bit(EXTENT_FLAG_COMPRESSED, &em->flags)) {
Chris Masonc8b97812008-10-29 14:49:59 -04002023 this_bio_flag = EXTENT_BIO_COMPRESSED;
Li Zefan261507a02010-12-17 14:21:50 +08002024 extent_set_compress_type(&this_bio_flag,
2025 em->compress_type);
2026 }
Chris Masonc8b97812008-10-29 14:49:59 -04002027
Chris Masond1310b22008-01-24 16:13:08 -05002028 iosize = min(extent_map_end(em) - cur, end - cur + 1);
2029 cur_end = min(extent_map_end(em) - 1, end);
2030 iosize = (iosize + blocksize - 1) & ~((u64)blocksize - 1);
Chris Masonc8b97812008-10-29 14:49:59 -04002031 if (this_bio_flag & EXTENT_BIO_COMPRESSED) {
2032 disk_io_size = em->block_len;
2033 sector = em->block_start >> 9;
2034 } else {
2035 sector = (em->block_start + extent_offset) >> 9;
2036 disk_io_size = iosize;
2037 }
Chris Masond1310b22008-01-24 16:13:08 -05002038 bdev = em->bdev;
2039 block_start = em->block_start;
Yan Zhengd899e052008-10-30 14:25:28 -04002040 if (test_bit(EXTENT_FLAG_PREALLOC, &em->flags))
2041 block_start = EXTENT_MAP_HOLE;
Chris Masond1310b22008-01-24 16:13:08 -05002042 free_extent_map(em);
2043 em = NULL;
2044
2045 /* we've found a hole, just zero and go on */
2046 if (block_start == EXTENT_MAP_HOLE) {
2047 char *userpage;
Arne Jansen507903b2011-04-06 10:02:20 +00002048 struct extent_state *cached = NULL;
2049
Chris Masond1310b22008-01-24 16:13:08 -05002050 userpage = kmap_atomic(page, KM_USER0);
David Sterba306e16c2011-04-19 14:29:38 +02002051 memset(userpage + pg_offset, 0, iosize);
Chris Masond1310b22008-01-24 16:13:08 -05002052 flush_dcache_page(page);
2053 kunmap_atomic(userpage, KM_USER0);
2054
2055 set_extent_uptodate(tree, cur, cur + iosize - 1,
Arne Jansen507903b2011-04-06 10:02:20 +00002056 &cached, GFP_NOFS);
2057 unlock_extent_cached(tree, cur, cur + iosize - 1,
2058 &cached, GFP_NOFS);
Chris Masond1310b22008-01-24 16:13:08 -05002059 cur = cur + iosize;
David Sterba306e16c2011-04-19 14:29:38 +02002060 pg_offset += iosize;
Chris Masond1310b22008-01-24 16:13:08 -05002061 continue;
2062 }
2063 /* the get_extent function already copied into the page */
Chris Mason9655d292009-09-02 15:22:30 -04002064 if (test_range_bit(tree, cur, cur_end,
2065 EXTENT_UPTODATE, 1, NULL)) {
Chris Masona1b32a52008-09-05 16:09:51 -04002066 check_page_uptodate(tree, page);
Chris Masond1310b22008-01-24 16:13:08 -05002067 unlock_extent(tree, cur, cur + iosize - 1, GFP_NOFS);
2068 cur = cur + iosize;
David Sterba306e16c2011-04-19 14:29:38 +02002069 pg_offset += iosize;
Chris Masond1310b22008-01-24 16:13:08 -05002070 continue;
2071 }
Chris Mason70dec802008-01-29 09:59:12 -05002072 /* we have an inline extent but it didn't get marked up
2073 * to date. Error out
2074 */
2075 if (block_start == EXTENT_MAP_INLINE) {
2076 SetPageError(page);
2077 unlock_extent(tree, cur, cur + iosize - 1, GFP_NOFS);
2078 cur = cur + iosize;
David Sterba306e16c2011-04-19 14:29:38 +02002079 pg_offset += iosize;
Chris Mason70dec802008-01-29 09:59:12 -05002080 continue;
2081 }
Chris Masond1310b22008-01-24 16:13:08 -05002082
2083 ret = 0;
2084 if (tree->ops && tree->ops->readpage_io_hook) {
2085 ret = tree->ops->readpage_io_hook(page, cur,
2086 cur + iosize - 1);
2087 }
2088 if (!ret) {
Chris Mason89642222008-07-24 09:41:53 -04002089 unsigned long pnr = (last_byte >> PAGE_CACHE_SHIFT) + 1;
2090 pnr -= page->index;
Chris Masond1310b22008-01-24 16:13:08 -05002091 ret = submit_extent_page(READ, tree, page,
David Sterba306e16c2011-04-19 14:29:38 +02002092 sector, disk_io_size, pg_offset,
Chris Mason89642222008-07-24 09:41:53 -04002093 bdev, bio, pnr,
Chris Masonc8b97812008-10-29 14:49:59 -04002094 end_bio_extent_readpage, mirror_num,
2095 *bio_flags,
2096 this_bio_flag);
Chris Mason89642222008-07-24 09:41:53 -04002097 nr++;
Chris Masonc8b97812008-10-29 14:49:59 -04002098 *bio_flags = this_bio_flag;
Chris Masond1310b22008-01-24 16:13:08 -05002099 }
2100 if (ret)
2101 SetPageError(page);
2102 cur = cur + iosize;
David Sterba306e16c2011-04-19 14:29:38 +02002103 pg_offset += iosize;
Chris Masond1310b22008-01-24 16:13:08 -05002104 }
2105 if (!nr) {
2106 if (!PageError(page))
2107 SetPageUptodate(page);
2108 unlock_page(page);
2109 }
2110 return 0;
2111}
2112
2113int extent_read_full_page(struct extent_io_tree *tree, struct page *page,
2114 get_extent_t *get_extent)
2115{
2116 struct bio *bio = NULL;
Chris Masonc8b97812008-10-29 14:49:59 -04002117 unsigned long bio_flags = 0;
Chris Masond1310b22008-01-24 16:13:08 -05002118 int ret;
2119
Chris Masonc8b97812008-10-29 14:49:59 -04002120 ret = __extent_read_full_page(tree, page, get_extent, &bio, 0,
2121 &bio_flags);
Chris Masond1310b22008-01-24 16:13:08 -05002122 if (bio)
liubo6b82ce82011-01-26 06:21:39 +00002123 ret = submit_one_bio(READ, bio, 0, bio_flags);
Chris Masond1310b22008-01-24 16:13:08 -05002124 return ret;
2125}
Chris Masond1310b22008-01-24 16:13:08 -05002126
Chris Mason11c83492009-04-20 15:50:09 -04002127static noinline void update_nr_written(struct page *page,
2128 struct writeback_control *wbc,
2129 unsigned long nr_written)
2130{
2131 wbc->nr_to_write -= nr_written;
2132 if (wbc->range_cyclic || (wbc->nr_to_write > 0 &&
2133 wbc->range_start == 0 && wbc->range_end == LLONG_MAX))
2134 page->mapping->writeback_index = page->index + nr_written;
2135}
2136
Chris Masond1310b22008-01-24 16:13:08 -05002137/*
2138 * the writepage semantics are similar to regular writepage. extent
2139 * records are inserted to lock ranges in the tree, and as dirty areas
2140 * are found, they are marked writeback. Then the lock bits are removed
2141 * and the end_io handler clears the writeback ranges
2142 */
2143static int __extent_writepage(struct page *page, struct writeback_control *wbc,
2144 void *data)
2145{
2146 struct inode *inode = page->mapping->host;
2147 struct extent_page_data *epd = data;
2148 struct extent_io_tree *tree = epd->tree;
2149 u64 start = (u64)page->index << PAGE_CACHE_SHIFT;
2150 u64 delalloc_start;
2151 u64 page_end = start + PAGE_CACHE_SIZE - 1;
2152 u64 end;
2153 u64 cur = start;
2154 u64 extent_offset;
2155 u64 last_byte = i_size_read(inode);
2156 u64 block_start;
2157 u64 iosize;
2158 sector_t sector;
Chris Mason2c64c532009-09-02 15:04:12 -04002159 struct extent_state *cached_state = NULL;
Chris Masond1310b22008-01-24 16:13:08 -05002160 struct extent_map *em;
2161 struct block_device *bdev;
2162 int ret;
2163 int nr = 0;
Chris Mason7f3c74f2008-07-18 12:01:11 -04002164 size_t pg_offset = 0;
Chris Masond1310b22008-01-24 16:13:08 -05002165 size_t blocksize;
2166 loff_t i_size = i_size_read(inode);
2167 unsigned long end_index = i_size >> PAGE_CACHE_SHIFT;
2168 u64 nr_delalloc;
2169 u64 delalloc_end;
Chris Masonc8b97812008-10-29 14:49:59 -04002170 int page_started;
2171 int compressed;
Chris Masonffbd5172009-04-20 15:50:09 -04002172 int write_flags;
Chris Mason771ed682008-11-06 22:02:51 -05002173 unsigned long nr_written = 0;
Chris Masond1310b22008-01-24 16:13:08 -05002174
Chris Masonffbd5172009-04-20 15:50:09 -04002175 if (wbc->sync_mode == WB_SYNC_ALL)
Jens Axboe721a9602011-03-09 11:56:30 +01002176 write_flags = WRITE_SYNC;
Chris Masonffbd5172009-04-20 15:50:09 -04002177 else
2178 write_flags = WRITE;
2179
liubo1abe9b82011-03-24 11:18:59 +00002180 trace___extent_writepage(page, inode, wbc);
2181
Chris Masond1310b22008-01-24 16:13:08 -05002182 WARN_ON(!PageLocked(page));
Chris Mason7f3c74f2008-07-18 12:01:11 -04002183 pg_offset = i_size & (PAGE_CACHE_SIZE - 1);
Chris Mason211c17f2008-05-15 09:13:45 -04002184 if (page->index > end_index ||
Chris Mason7f3c74f2008-07-18 12:01:11 -04002185 (page->index == end_index && !pg_offset)) {
Chris Mason39be25c2008-11-10 11:50:50 -05002186 page->mapping->a_ops->invalidatepage(page, 0);
Chris Masond1310b22008-01-24 16:13:08 -05002187 unlock_page(page);
2188 return 0;
2189 }
2190
2191 if (page->index == end_index) {
2192 char *userpage;
2193
Chris Masond1310b22008-01-24 16:13:08 -05002194 userpage = kmap_atomic(page, KM_USER0);
Chris Mason7f3c74f2008-07-18 12:01:11 -04002195 memset(userpage + pg_offset, 0,
2196 PAGE_CACHE_SIZE - pg_offset);
Chris Masond1310b22008-01-24 16:13:08 -05002197 kunmap_atomic(userpage, KM_USER0);
Chris Mason211c17f2008-05-15 09:13:45 -04002198 flush_dcache_page(page);
Chris Masond1310b22008-01-24 16:13:08 -05002199 }
Chris Mason7f3c74f2008-07-18 12:01:11 -04002200 pg_offset = 0;
Chris Masond1310b22008-01-24 16:13:08 -05002201
2202 set_page_extent_mapped(page);
2203
2204 delalloc_start = start;
2205 delalloc_end = 0;
Chris Masonc8b97812008-10-29 14:49:59 -04002206 page_started = 0;
Chris Mason771ed682008-11-06 22:02:51 -05002207 if (!epd->extent_locked) {
Chris Masonf85d7d6c2009-09-18 16:03:16 -04002208 u64 delalloc_to_write = 0;
Chris Mason11c83492009-04-20 15:50:09 -04002209 /*
2210 * make sure the wbc mapping index is at least updated
2211 * to this page.
2212 */
2213 update_nr_written(page, wbc, 0);
2214
Chris Masond3977122009-01-05 21:25:51 -05002215 while (delalloc_end < page_end) {
Chris Mason771ed682008-11-06 22:02:51 -05002216 nr_delalloc = find_lock_delalloc_range(inode, tree,
Chris Masonc8b97812008-10-29 14:49:59 -04002217 page,
2218 &delalloc_start,
Chris Masond1310b22008-01-24 16:13:08 -05002219 &delalloc_end,
2220 128 * 1024 * 1024);
Chris Mason771ed682008-11-06 22:02:51 -05002221 if (nr_delalloc == 0) {
2222 delalloc_start = delalloc_end + 1;
2223 continue;
2224 }
2225 tree->ops->fill_delalloc(inode, page, delalloc_start,
2226 delalloc_end, &page_started,
2227 &nr_written);
Chris Masonf85d7d6c2009-09-18 16:03:16 -04002228 /*
2229 * delalloc_end is already one less than the total
2230 * length, so we don't subtract one from
2231 * PAGE_CACHE_SIZE
2232 */
2233 delalloc_to_write += (delalloc_end - delalloc_start +
2234 PAGE_CACHE_SIZE) >>
2235 PAGE_CACHE_SHIFT;
Chris Masond1310b22008-01-24 16:13:08 -05002236 delalloc_start = delalloc_end + 1;
Chris Masond1310b22008-01-24 16:13:08 -05002237 }
Chris Masonf85d7d6c2009-09-18 16:03:16 -04002238 if (wbc->nr_to_write < delalloc_to_write) {
2239 int thresh = 8192;
2240
2241 if (delalloc_to_write < thresh * 2)
2242 thresh = delalloc_to_write;
2243 wbc->nr_to_write = min_t(u64, delalloc_to_write,
2244 thresh);
2245 }
Chris Masonc8b97812008-10-29 14:49:59 -04002246
Chris Mason771ed682008-11-06 22:02:51 -05002247 /* did the fill delalloc function already unlock and start
2248 * the IO?
2249 */
2250 if (page_started) {
2251 ret = 0;
Chris Mason11c83492009-04-20 15:50:09 -04002252 /*
2253 * we've unlocked the page, so we can't update
2254 * the mapping's writeback index, just update
2255 * nr_to_write.
2256 */
2257 wbc->nr_to_write -= nr_written;
2258 goto done_unlocked;
Chris Mason771ed682008-11-06 22:02:51 -05002259 }
Chris Masonc8b97812008-10-29 14:49:59 -04002260 }
Chris Mason247e7432008-07-17 12:53:51 -04002261 if (tree->ops && tree->ops->writepage_start_hook) {
Chris Masonc8b97812008-10-29 14:49:59 -04002262 ret = tree->ops->writepage_start_hook(page, start,
2263 page_end);
Chris Mason247e7432008-07-17 12:53:51 -04002264 if (ret == -EAGAIN) {
Chris Mason247e7432008-07-17 12:53:51 -04002265 redirty_page_for_writepage(wbc, page);
Chris Mason11c83492009-04-20 15:50:09 -04002266 update_nr_written(page, wbc, nr_written);
Chris Mason247e7432008-07-17 12:53:51 -04002267 unlock_page(page);
Chris Mason771ed682008-11-06 22:02:51 -05002268 ret = 0;
Chris Mason11c83492009-04-20 15:50:09 -04002269 goto done_unlocked;
Chris Mason247e7432008-07-17 12:53:51 -04002270 }
2271 }
2272
Chris Mason11c83492009-04-20 15:50:09 -04002273 /*
2274 * we don't want to touch the inode after unlocking the page,
2275 * so we update the mapping writeback index now
2276 */
2277 update_nr_written(page, wbc, nr_written + 1);
Chris Mason771ed682008-11-06 22:02:51 -05002278
Chris Masond1310b22008-01-24 16:13:08 -05002279 end = page_end;
Chris Masond1310b22008-01-24 16:13:08 -05002280 if (last_byte <= start) {
Chris Masone6dcd2d2008-07-17 12:53:50 -04002281 if (tree->ops && tree->ops->writepage_end_io_hook)
2282 tree->ops->writepage_end_io_hook(page, start,
2283 page_end, NULL, 1);
Chris Masond1310b22008-01-24 16:13:08 -05002284 goto done;
2285 }
2286
Chris Masond1310b22008-01-24 16:13:08 -05002287 blocksize = inode->i_sb->s_blocksize;
2288
2289 while (cur <= end) {
2290 if (cur >= last_byte) {
Chris Masone6dcd2d2008-07-17 12:53:50 -04002291 if (tree->ops && tree->ops->writepage_end_io_hook)
2292 tree->ops->writepage_end_io_hook(page, cur,
2293 page_end, NULL, 1);
Chris Masond1310b22008-01-24 16:13:08 -05002294 break;
2295 }
Chris Mason7f3c74f2008-07-18 12:01:11 -04002296 em = epd->get_extent(inode, page, pg_offset, cur,
Chris Masond1310b22008-01-24 16:13:08 -05002297 end - cur + 1, 1);
David Sterbac7040052011-04-19 18:00:01 +02002298 if (IS_ERR_OR_NULL(em)) {
Chris Masond1310b22008-01-24 16:13:08 -05002299 SetPageError(page);
2300 break;
2301 }
2302
2303 extent_offset = cur - em->start;
2304 BUG_ON(extent_map_end(em) <= cur);
2305 BUG_ON(end < cur);
2306 iosize = min(extent_map_end(em) - cur, end - cur + 1);
2307 iosize = (iosize + blocksize - 1) & ~((u64)blocksize - 1);
2308 sector = (em->block_start + extent_offset) >> 9;
2309 bdev = em->bdev;
2310 block_start = em->block_start;
Chris Masonc8b97812008-10-29 14:49:59 -04002311 compressed = test_bit(EXTENT_FLAG_COMPRESSED, &em->flags);
Chris Masond1310b22008-01-24 16:13:08 -05002312 free_extent_map(em);
2313 em = NULL;
2314
Chris Masonc8b97812008-10-29 14:49:59 -04002315 /*
2316 * compressed and inline extents are written through other
2317 * paths in the FS
2318 */
2319 if (compressed || block_start == EXTENT_MAP_HOLE ||
Chris Masond1310b22008-01-24 16:13:08 -05002320 block_start == EXTENT_MAP_INLINE) {
Chris Masonc8b97812008-10-29 14:49:59 -04002321 /*
2322 * end_io notification does not happen here for
2323 * compressed extents
2324 */
2325 if (!compressed && tree->ops &&
2326 tree->ops->writepage_end_io_hook)
Chris Masone6dcd2d2008-07-17 12:53:50 -04002327 tree->ops->writepage_end_io_hook(page, cur,
2328 cur + iosize - 1,
2329 NULL, 1);
Chris Masonc8b97812008-10-29 14:49:59 -04002330 else if (compressed) {
2331 /* we don't want to end_page_writeback on
2332 * a compressed extent. this happens
2333 * elsewhere
2334 */
2335 nr++;
2336 }
2337
2338 cur += iosize;
Chris Mason7f3c74f2008-07-18 12:01:11 -04002339 pg_offset += iosize;
Chris Masond1310b22008-01-24 16:13:08 -05002340 continue;
2341 }
Chris Masond1310b22008-01-24 16:13:08 -05002342 /* leave this out until we have a page_mkwrite call */
2343 if (0 && !test_range_bit(tree, cur, cur + iosize - 1,
Chris Mason9655d292009-09-02 15:22:30 -04002344 EXTENT_DIRTY, 0, NULL)) {
Chris Masond1310b22008-01-24 16:13:08 -05002345 cur = cur + iosize;
Chris Mason7f3c74f2008-07-18 12:01:11 -04002346 pg_offset += iosize;
Chris Masond1310b22008-01-24 16:13:08 -05002347 continue;
2348 }
Chris Masonc8b97812008-10-29 14:49:59 -04002349
Chris Masond1310b22008-01-24 16:13:08 -05002350 if (tree->ops && tree->ops->writepage_io_hook) {
2351 ret = tree->ops->writepage_io_hook(page, cur,
2352 cur + iosize - 1);
2353 } else {
2354 ret = 0;
2355 }
Chris Mason1259ab72008-05-12 13:39:03 -04002356 if (ret) {
Chris Masond1310b22008-01-24 16:13:08 -05002357 SetPageError(page);
Chris Mason1259ab72008-05-12 13:39:03 -04002358 } else {
Chris Masond1310b22008-01-24 16:13:08 -05002359 unsigned long max_nr = end_index + 1;
Chris Mason7f3c74f2008-07-18 12:01:11 -04002360
Chris Masond1310b22008-01-24 16:13:08 -05002361 set_range_writeback(tree, cur, cur + iosize - 1);
2362 if (!PageWriteback(page)) {
Chris Masond3977122009-01-05 21:25:51 -05002363 printk(KERN_ERR "btrfs warning page %lu not "
2364 "writeback, cur %llu end %llu\n",
2365 page->index, (unsigned long long)cur,
Chris Masond1310b22008-01-24 16:13:08 -05002366 (unsigned long long)end);
2367 }
2368
Chris Masonffbd5172009-04-20 15:50:09 -04002369 ret = submit_extent_page(write_flags, tree, page,
2370 sector, iosize, pg_offset,
2371 bdev, &epd->bio, max_nr,
Chris Masonc8b97812008-10-29 14:49:59 -04002372 end_bio_extent_writepage,
2373 0, 0, 0);
Chris Masond1310b22008-01-24 16:13:08 -05002374 if (ret)
2375 SetPageError(page);
2376 }
2377 cur = cur + iosize;
Chris Mason7f3c74f2008-07-18 12:01:11 -04002378 pg_offset += iosize;
Chris Masond1310b22008-01-24 16:13:08 -05002379 nr++;
2380 }
2381done:
2382 if (nr == 0) {
2383 /* make sure the mapping tag for page dirty gets cleared */
2384 set_page_writeback(page);
2385 end_page_writeback(page);
2386 }
Chris Masond1310b22008-01-24 16:13:08 -05002387 unlock_page(page);
Chris Mason771ed682008-11-06 22:02:51 -05002388
Chris Mason11c83492009-04-20 15:50:09 -04002389done_unlocked:
2390
Chris Mason2c64c532009-09-02 15:04:12 -04002391 /* drop our reference on any cached states */
2392 free_extent_state(cached_state);
Chris Masond1310b22008-01-24 16:13:08 -05002393 return 0;
2394}
2395
Chris Masond1310b22008-01-24 16:13:08 -05002396/**
Chris Mason4bef0842008-09-08 11:18:08 -04002397 * write_cache_pages - walk the list of dirty pages of the given address space and write all of them.
Chris Masond1310b22008-01-24 16:13:08 -05002398 * @mapping: address space structure to write
2399 * @wbc: subtract the number of written pages from *@wbc->nr_to_write
2400 * @writepage: function called for each page
2401 * @data: data passed to writepage function
2402 *
2403 * If a page is already under I/O, write_cache_pages() skips it, even
2404 * if it's dirty. This is desirable behaviour for memory-cleaning writeback,
2405 * but it is INCORRECT for data-integrity system calls such as fsync(). fsync()
2406 * and msync() need to guarantee that all the data which was dirty at the time
2407 * the call was made get new I/O started against them. If wbc->sync_mode is
2408 * WB_SYNC_ALL then we were called for data integrity and we must wait for
2409 * existing IO to complete.
2410 */
Christoph Hellwigb2950862008-12-02 09:54:17 -05002411static int extent_write_cache_pages(struct extent_io_tree *tree,
Chris Mason4bef0842008-09-08 11:18:08 -04002412 struct address_space *mapping,
2413 struct writeback_control *wbc,
Chris Masond2c3f4f2008-11-19 12:44:22 -05002414 writepage_t writepage, void *data,
2415 void (*flush_fn)(void *))
Chris Masond1310b22008-01-24 16:13:08 -05002416{
Chris Masond1310b22008-01-24 16:13:08 -05002417 int ret = 0;
2418 int done = 0;
Chris Masonf85d7d6c2009-09-18 16:03:16 -04002419 int nr_to_write_done = 0;
Chris Masond1310b22008-01-24 16:13:08 -05002420 struct pagevec pvec;
2421 int nr_pages;
2422 pgoff_t index;
2423 pgoff_t end; /* Inclusive */
2424 int scanned = 0;
Chris Masond1310b22008-01-24 16:13:08 -05002425
Chris Masond1310b22008-01-24 16:13:08 -05002426 pagevec_init(&pvec, 0);
2427 if (wbc->range_cyclic) {
2428 index = mapping->writeback_index; /* Start from prev offset */
2429 end = -1;
2430 } else {
2431 index = wbc->range_start >> PAGE_CACHE_SHIFT;
2432 end = wbc->range_end >> PAGE_CACHE_SHIFT;
Chris Masond1310b22008-01-24 16:13:08 -05002433 scanned = 1;
2434 }
2435retry:
Chris Masonf85d7d6c2009-09-18 16:03:16 -04002436 while (!done && !nr_to_write_done && (index <= end) &&
Chris Masond1310b22008-01-24 16:13:08 -05002437 (nr_pages = pagevec_lookup_tag(&pvec, mapping, &index,
Chris Masond3977122009-01-05 21:25:51 -05002438 PAGECACHE_TAG_DIRTY, min(end - index,
2439 (pgoff_t)PAGEVEC_SIZE-1) + 1))) {
Chris Masond1310b22008-01-24 16:13:08 -05002440 unsigned i;
2441
2442 scanned = 1;
2443 for (i = 0; i < nr_pages; i++) {
2444 struct page *page = pvec.pages[i];
2445
2446 /*
2447 * At this point we hold neither mapping->tree_lock nor
2448 * lock on the page itself: the page may be truncated or
2449 * invalidated (changing page->mapping to NULL), or even
2450 * swizzled back from swapper_space to tmpfs file
2451 * mapping
2452 */
Chris Mason4bef0842008-09-08 11:18:08 -04002453 if (tree->ops && tree->ops->write_cache_pages_lock_hook)
2454 tree->ops->write_cache_pages_lock_hook(page);
2455 else
2456 lock_page(page);
Chris Masond1310b22008-01-24 16:13:08 -05002457
2458 if (unlikely(page->mapping != mapping)) {
2459 unlock_page(page);
2460 continue;
2461 }
2462
2463 if (!wbc->range_cyclic && page->index > end) {
2464 done = 1;
2465 unlock_page(page);
2466 continue;
2467 }
2468
Chris Masond2c3f4f2008-11-19 12:44:22 -05002469 if (wbc->sync_mode != WB_SYNC_NONE) {
Chris Mason0e6bd952008-11-20 10:46:35 -05002470 if (PageWriteback(page))
2471 flush_fn(data);
Chris Masond1310b22008-01-24 16:13:08 -05002472 wait_on_page_writeback(page);
Chris Masond2c3f4f2008-11-19 12:44:22 -05002473 }
Chris Masond1310b22008-01-24 16:13:08 -05002474
2475 if (PageWriteback(page) ||
2476 !clear_page_dirty_for_io(page)) {
2477 unlock_page(page);
2478 continue;
2479 }
2480
2481 ret = (*writepage)(page, wbc, data);
2482
2483 if (unlikely(ret == AOP_WRITEPAGE_ACTIVATE)) {
2484 unlock_page(page);
2485 ret = 0;
2486 }
Chris Masonf85d7d6c2009-09-18 16:03:16 -04002487 if (ret)
Chris Masond1310b22008-01-24 16:13:08 -05002488 done = 1;
Chris Masonf85d7d6c2009-09-18 16:03:16 -04002489
2490 /*
2491 * the filesystem may choose to bump up nr_to_write.
2492 * We have to make sure to honor the new nr_to_write
2493 * at any time
2494 */
2495 nr_to_write_done = wbc->nr_to_write <= 0;
Chris Masond1310b22008-01-24 16:13:08 -05002496 }
2497 pagevec_release(&pvec);
2498 cond_resched();
2499 }
2500 if (!scanned && !done) {
2501 /*
2502 * We hit the last page and there is more work to be done: wrap
2503 * back to the start of the file
2504 */
2505 scanned = 1;
2506 index = 0;
2507 goto retry;
2508 }
Chris Masond1310b22008-01-24 16:13:08 -05002509 return ret;
2510}
Chris Masond1310b22008-01-24 16:13:08 -05002511
Chris Masonffbd5172009-04-20 15:50:09 -04002512static void flush_epd_write_bio(struct extent_page_data *epd)
2513{
2514 if (epd->bio) {
2515 if (epd->sync_io)
2516 submit_one_bio(WRITE_SYNC, epd->bio, 0, 0);
2517 else
2518 submit_one_bio(WRITE, epd->bio, 0, 0);
2519 epd->bio = NULL;
2520 }
2521}
2522
Chris Masond2c3f4f2008-11-19 12:44:22 -05002523static noinline void flush_write_bio(void *data)
2524{
2525 struct extent_page_data *epd = data;
Chris Masonffbd5172009-04-20 15:50:09 -04002526 flush_epd_write_bio(epd);
Chris Masond2c3f4f2008-11-19 12:44:22 -05002527}
2528
Chris Masond1310b22008-01-24 16:13:08 -05002529int extent_write_full_page(struct extent_io_tree *tree, struct page *page,
2530 get_extent_t *get_extent,
2531 struct writeback_control *wbc)
2532{
2533 int ret;
2534 struct address_space *mapping = page->mapping;
2535 struct extent_page_data epd = {
2536 .bio = NULL,
2537 .tree = tree,
2538 .get_extent = get_extent,
Chris Mason771ed682008-11-06 22:02:51 -05002539 .extent_locked = 0,
Chris Masonffbd5172009-04-20 15:50:09 -04002540 .sync_io = wbc->sync_mode == WB_SYNC_ALL,
Chris Masond1310b22008-01-24 16:13:08 -05002541 };
2542 struct writeback_control wbc_writepages = {
Chris Masond313d7a2009-04-20 15:50:09 -04002543 .sync_mode = wbc->sync_mode,
Chris Masond1310b22008-01-24 16:13:08 -05002544 .older_than_this = NULL,
2545 .nr_to_write = 64,
2546 .range_start = page_offset(page) + PAGE_CACHE_SIZE,
2547 .range_end = (loff_t)-1,
2548 };
2549
Chris Masond1310b22008-01-24 16:13:08 -05002550 ret = __extent_writepage(page, wbc, &epd);
2551
Chris Mason4bef0842008-09-08 11:18:08 -04002552 extent_write_cache_pages(tree, mapping, &wbc_writepages,
Chris Masond2c3f4f2008-11-19 12:44:22 -05002553 __extent_writepage, &epd, flush_write_bio);
Chris Masonffbd5172009-04-20 15:50:09 -04002554 flush_epd_write_bio(&epd);
Chris Masond1310b22008-01-24 16:13:08 -05002555 return ret;
2556}
Chris Masond1310b22008-01-24 16:13:08 -05002557
Chris Mason771ed682008-11-06 22:02:51 -05002558int extent_write_locked_range(struct extent_io_tree *tree, struct inode *inode,
2559 u64 start, u64 end, get_extent_t *get_extent,
2560 int mode)
2561{
2562 int ret = 0;
2563 struct address_space *mapping = inode->i_mapping;
2564 struct page *page;
2565 unsigned long nr_pages = (end - start + PAGE_CACHE_SIZE) >>
2566 PAGE_CACHE_SHIFT;
2567
2568 struct extent_page_data epd = {
2569 .bio = NULL,
2570 .tree = tree,
2571 .get_extent = get_extent,
2572 .extent_locked = 1,
Chris Masonffbd5172009-04-20 15:50:09 -04002573 .sync_io = mode == WB_SYNC_ALL,
Chris Mason771ed682008-11-06 22:02:51 -05002574 };
2575 struct writeback_control wbc_writepages = {
Chris Mason771ed682008-11-06 22:02:51 -05002576 .sync_mode = mode,
2577 .older_than_this = NULL,
2578 .nr_to_write = nr_pages * 2,
2579 .range_start = start,
2580 .range_end = end + 1,
2581 };
2582
Chris Masond3977122009-01-05 21:25:51 -05002583 while (start <= end) {
Chris Mason771ed682008-11-06 22:02:51 -05002584 page = find_get_page(mapping, start >> PAGE_CACHE_SHIFT);
2585 if (clear_page_dirty_for_io(page))
2586 ret = __extent_writepage(page, &wbc_writepages, &epd);
2587 else {
2588 if (tree->ops && tree->ops->writepage_end_io_hook)
2589 tree->ops->writepage_end_io_hook(page, start,
2590 start + PAGE_CACHE_SIZE - 1,
2591 NULL, 1);
2592 unlock_page(page);
2593 }
2594 page_cache_release(page);
2595 start += PAGE_CACHE_SIZE;
2596 }
2597
Chris Masonffbd5172009-04-20 15:50:09 -04002598 flush_epd_write_bio(&epd);
Chris Mason771ed682008-11-06 22:02:51 -05002599 return ret;
2600}
Chris Masond1310b22008-01-24 16:13:08 -05002601
2602int extent_writepages(struct extent_io_tree *tree,
2603 struct address_space *mapping,
2604 get_extent_t *get_extent,
2605 struct writeback_control *wbc)
2606{
2607 int ret = 0;
2608 struct extent_page_data epd = {
2609 .bio = NULL,
2610 .tree = tree,
2611 .get_extent = get_extent,
Chris Mason771ed682008-11-06 22:02:51 -05002612 .extent_locked = 0,
Chris Masonffbd5172009-04-20 15:50:09 -04002613 .sync_io = wbc->sync_mode == WB_SYNC_ALL,
Chris Masond1310b22008-01-24 16:13:08 -05002614 };
2615
Chris Mason4bef0842008-09-08 11:18:08 -04002616 ret = extent_write_cache_pages(tree, mapping, wbc,
Chris Masond2c3f4f2008-11-19 12:44:22 -05002617 __extent_writepage, &epd,
2618 flush_write_bio);
Chris Masonffbd5172009-04-20 15:50:09 -04002619 flush_epd_write_bio(&epd);
Chris Masond1310b22008-01-24 16:13:08 -05002620 return ret;
2621}
Chris Masond1310b22008-01-24 16:13:08 -05002622
2623int extent_readpages(struct extent_io_tree *tree,
2624 struct address_space *mapping,
2625 struct list_head *pages, unsigned nr_pages,
2626 get_extent_t get_extent)
2627{
2628 struct bio *bio = NULL;
2629 unsigned page_idx;
Chris Masonc8b97812008-10-29 14:49:59 -04002630 unsigned long bio_flags = 0;
Chris Masond1310b22008-01-24 16:13:08 -05002631
Chris Masond1310b22008-01-24 16:13:08 -05002632 for (page_idx = 0; page_idx < nr_pages; page_idx++) {
2633 struct page *page = list_entry(pages->prev, struct page, lru);
2634
2635 prefetchw(&page->flags);
2636 list_del(&page->lru);
Nick Piggin28ecb6092010-03-17 13:31:04 +00002637 if (!add_to_page_cache_lru(page, mapping,
Itaru Kitayama43e817a2011-04-25 19:43:51 -04002638 page->index, GFP_NOFS)) {
Chris Masonf1885912008-04-09 16:28:12 -04002639 __extent_read_full_page(tree, page, get_extent,
Chris Masonc8b97812008-10-29 14:49:59 -04002640 &bio, 0, &bio_flags);
Chris Masond1310b22008-01-24 16:13:08 -05002641 }
2642 page_cache_release(page);
2643 }
Chris Masond1310b22008-01-24 16:13:08 -05002644 BUG_ON(!list_empty(pages));
2645 if (bio)
Chris Masonc8b97812008-10-29 14:49:59 -04002646 submit_one_bio(READ, bio, 0, bio_flags);
Chris Masond1310b22008-01-24 16:13:08 -05002647 return 0;
2648}
Chris Masond1310b22008-01-24 16:13:08 -05002649
2650/*
2651 * basic invalidatepage code, this waits on any locked or writeback
2652 * ranges corresponding to the page, and then deletes any extent state
2653 * records from the tree
2654 */
2655int extent_invalidatepage(struct extent_io_tree *tree,
2656 struct page *page, unsigned long offset)
2657{
Josef Bacik2ac55d42010-02-03 19:33:23 +00002658 struct extent_state *cached_state = NULL;
Chris Masond1310b22008-01-24 16:13:08 -05002659 u64 start = ((u64)page->index << PAGE_CACHE_SHIFT);
2660 u64 end = start + PAGE_CACHE_SIZE - 1;
2661 size_t blocksize = page->mapping->host->i_sb->s_blocksize;
2662
Chris Masond3977122009-01-05 21:25:51 -05002663 start += (offset + blocksize - 1) & ~(blocksize - 1);
Chris Masond1310b22008-01-24 16:13:08 -05002664 if (start > end)
2665 return 0;
2666
Josef Bacik2ac55d42010-02-03 19:33:23 +00002667 lock_extent_bits(tree, start, end, 0, &cached_state, GFP_NOFS);
Chris Mason1edbb732009-09-02 13:24:36 -04002668 wait_on_page_writeback(page);
Chris Masond1310b22008-01-24 16:13:08 -05002669 clear_extent_bit(tree, start, end,
Josef Bacik32c00af2009-10-08 13:34:05 -04002670 EXTENT_LOCKED | EXTENT_DIRTY | EXTENT_DELALLOC |
2671 EXTENT_DO_ACCOUNTING,
Josef Bacik2ac55d42010-02-03 19:33:23 +00002672 1, 1, &cached_state, GFP_NOFS);
Chris Masond1310b22008-01-24 16:13:08 -05002673 return 0;
2674}
Chris Masond1310b22008-01-24 16:13:08 -05002675
2676/*
Chris Mason7b13b7b2008-04-18 10:29:50 -04002677 * a helper for releasepage, this tests for areas of the page that
2678 * are locked or under IO and drops the related state bits if it is safe
2679 * to drop the page.
2680 */
2681int try_release_extent_state(struct extent_map_tree *map,
2682 struct extent_io_tree *tree, struct page *page,
2683 gfp_t mask)
2684{
2685 u64 start = (u64)page->index << PAGE_CACHE_SHIFT;
2686 u64 end = start + PAGE_CACHE_SIZE - 1;
2687 int ret = 1;
2688
Chris Mason211f90e2008-07-18 11:56:15 -04002689 if (test_range_bit(tree, start, end,
Chris Mason8b62b722009-09-02 16:53:46 -04002690 EXTENT_IOBITS, 0, NULL))
Chris Mason7b13b7b2008-04-18 10:29:50 -04002691 ret = 0;
2692 else {
2693 if ((mask & GFP_NOFS) == GFP_NOFS)
2694 mask = GFP_NOFS;
Chris Mason11ef1602009-09-23 20:28:46 -04002695 /*
2696 * at this point we can safely clear everything except the
2697 * locked bit and the nodatasum bit
2698 */
Chris Masone3f24cc2011-02-14 12:52:08 -05002699 ret = clear_extent_bit(tree, start, end,
Chris Mason11ef1602009-09-23 20:28:46 -04002700 ~(EXTENT_LOCKED | EXTENT_NODATASUM),
2701 0, 0, NULL, mask);
Chris Masone3f24cc2011-02-14 12:52:08 -05002702
2703 /* if clear_extent_bit failed for enomem reasons,
2704 * we can't allow the release to continue.
2705 */
2706 if (ret < 0)
2707 ret = 0;
2708 else
2709 ret = 1;
Chris Mason7b13b7b2008-04-18 10:29:50 -04002710 }
2711 return ret;
2712}
Chris Mason7b13b7b2008-04-18 10:29:50 -04002713
2714/*
Chris Masond1310b22008-01-24 16:13:08 -05002715 * a helper for releasepage. As long as there are no locked extents
2716 * in the range corresponding to the page, both state records and extent
2717 * map records are removed
2718 */
2719int try_release_extent_mapping(struct extent_map_tree *map,
Chris Mason70dec802008-01-29 09:59:12 -05002720 struct extent_io_tree *tree, struct page *page,
2721 gfp_t mask)
Chris Masond1310b22008-01-24 16:13:08 -05002722{
2723 struct extent_map *em;
2724 u64 start = (u64)page->index << PAGE_CACHE_SHIFT;
2725 u64 end = start + PAGE_CACHE_SIZE - 1;
Chris Mason7b13b7b2008-04-18 10:29:50 -04002726
Chris Mason70dec802008-01-29 09:59:12 -05002727 if ((mask & __GFP_WAIT) &&
2728 page->mapping->host->i_size > 16 * 1024 * 1024) {
Yan39b56372008-02-15 10:40:50 -05002729 u64 len;
Chris Mason70dec802008-01-29 09:59:12 -05002730 while (start <= end) {
Yan39b56372008-02-15 10:40:50 -05002731 len = end - start + 1;
Chris Mason890871b2009-09-02 16:24:52 -04002732 write_lock(&map->lock);
Yan39b56372008-02-15 10:40:50 -05002733 em = lookup_extent_mapping(map, start, len);
David Sterbac7040052011-04-19 18:00:01 +02002734 if (IS_ERR_OR_NULL(em)) {
Chris Mason890871b2009-09-02 16:24:52 -04002735 write_unlock(&map->lock);
Chris Mason70dec802008-01-29 09:59:12 -05002736 break;
2737 }
Chris Mason7f3c74f2008-07-18 12:01:11 -04002738 if (test_bit(EXTENT_FLAG_PINNED, &em->flags) ||
2739 em->start != start) {
Chris Mason890871b2009-09-02 16:24:52 -04002740 write_unlock(&map->lock);
Chris Mason70dec802008-01-29 09:59:12 -05002741 free_extent_map(em);
2742 break;
2743 }
2744 if (!test_range_bit(tree, em->start,
2745 extent_map_end(em) - 1,
Chris Mason8b62b722009-09-02 16:53:46 -04002746 EXTENT_LOCKED | EXTENT_WRITEBACK,
Chris Mason9655d292009-09-02 15:22:30 -04002747 0, NULL)) {
Chris Mason70dec802008-01-29 09:59:12 -05002748 remove_extent_mapping(map, em);
2749 /* once for the rb tree */
2750 free_extent_map(em);
2751 }
2752 start = extent_map_end(em);
Chris Mason890871b2009-09-02 16:24:52 -04002753 write_unlock(&map->lock);
Chris Mason70dec802008-01-29 09:59:12 -05002754
2755 /* once for us */
Chris Masond1310b22008-01-24 16:13:08 -05002756 free_extent_map(em);
2757 }
Chris Masond1310b22008-01-24 16:13:08 -05002758 }
Chris Mason7b13b7b2008-04-18 10:29:50 -04002759 return try_release_extent_state(map, tree, page, mask);
Chris Masond1310b22008-01-24 16:13:08 -05002760}
Chris Masond1310b22008-01-24 16:13:08 -05002761
Chris Masonec29ed52011-02-23 16:23:20 -05002762/*
2763 * helper function for fiemap, which doesn't want to see any holes.
2764 * This maps until we find something past 'last'
2765 */
2766static struct extent_map *get_extent_skip_holes(struct inode *inode,
2767 u64 offset,
2768 u64 last,
2769 get_extent_t *get_extent)
2770{
2771 u64 sectorsize = BTRFS_I(inode)->root->sectorsize;
2772 struct extent_map *em;
2773 u64 len;
2774
2775 if (offset >= last)
2776 return NULL;
2777
2778 while(1) {
2779 len = last - offset;
2780 if (len == 0)
2781 break;
2782 len = (len + sectorsize - 1) & ~(sectorsize - 1);
2783 em = get_extent(inode, NULL, 0, offset, len, 0);
David Sterbac7040052011-04-19 18:00:01 +02002784 if (IS_ERR_OR_NULL(em))
Chris Masonec29ed52011-02-23 16:23:20 -05002785 return em;
2786
2787 /* if this isn't a hole return it */
2788 if (!test_bit(EXTENT_FLAG_VACANCY, &em->flags) &&
2789 em->block_start != EXTENT_MAP_HOLE) {
2790 return em;
2791 }
2792
2793 /* this is a hole, advance to the next extent */
2794 offset = extent_map_end(em);
2795 free_extent_map(em);
2796 if (offset >= last)
2797 break;
2798 }
2799 return NULL;
2800}
2801
Yehuda Sadeh1506fcc2009-01-21 14:39:14 -05002802int extent_fiemap(struct inode *inode, struct fiemap_extent_info *fieinfo,
2803 __u64 start, __u64 len, get_extent_t *get_extent)
2804{
Josef Bacik975f84f2010-11-23 19:36:57 +00002805 int ret = 0;
Yehuda Sadeh1506fcc2009-01-21 14:39:14 -05002806 u64 off = start;
2807 u64 max = start + len;
2808 u32 flags = 0;
Josef Bacik975f84f2010-11-23 19:36:57 +00002809 u32 found_type;
2810 u64 last;
Chris Masonec29ed52011-02-23 16:23:20 -05002811 u64 last_for_get_extent = 0;
Yehuda Sadeh1506fcc2009-01-21 14:39:14 -05002812 u64 disko = 0;
Chris Masonec29ed52011-02-23 16:23:20 -05002813 u64 isize = i_size_read(inode);
Josef Bacik975f84f2010-11-23 19:36:57 +00002814 struct btrfs_key found_key;
Yehuda Sadeh1506fcc2009-01-21 14:39:14 -05002815 struct extent_map *em = NULL;
Josef Bacik2ac55d42010-02-03 19:33:23 +00002816 struct extent_state *cached_state = NULL;
Josef Bacik975f84f2010-11-23 19:36:57 +00002817 struct btrfs_path *path;
2818 struct btrfs_file_extent_item *item;
Yehuda Sadeh1506fcc2009-01-21 14:39:14 -05002819 int end = 0;
Chris Masonec29ed52011-02-23 16:23:20 -05002820 u64 em_start = 0;
2821 u64 em_len = 0;
2822 u64 em_end = 0;
Yehuda Sadeh1506fcc2009-01-21 14:39:14 -05002823 unsigned long emflags;
Yehuda Sadeh1506fcc2009-01-21 14:39:14 -05002824
2825 if (len == 0)
2826 return -EINVAL;
2827
Josef Bacik975f84f2010-11-23 19:36:57 +00002828 path = btrfs_alloc_path();
2829 if (!path)
2830 return -ENOMEM;
2831 path->leave_spinning = 1;
2832
Chris Masonec29ed52011-02-23 16:23:20 -05002833 /*
2834 * lookup the last file extent. We're not using i_size here
2835 * because there might be preallocation past i_size
2836 */
Josef Bacik975f84f2010-11-23 19:36:57 +00002837 ret = btrfs_lookup_file_extent(NULL, BTRFS_I(inode)->root,
Li Zefan33345d012011-04-20 10:31:50 +08002838 path, btrfs_ino(inode), -1, 0);
Josef Bacik975f84f2010-11-23 19:36:57 +00002839 if (ret < 0) {
2840 btrfs_free_path(path);
2841 return ret;
2842 }
2843 WARN_ON(!ret);
2844 path->slots[0]--;
2845 item = btrfs_item_ptr(path->nodes[0], path->slots[0],
2846 struct btrfs_file_extent_item);
2847 btrfs_item_key_to_cpu(path->nodes[0], &found_key, path->slots[0]);
2848 found_type = btrfs_key_type(&found_key);
2849
Chris Masonec29ed52011-02-23 16:23:20 -05002850 /* No extents, but there might be delalloc bits */
Li Zefan33345d012011-04-20 10:31:50 +08002851 if (found_key.objectid != btrfs_ino(inode) ||
Josef Bacik975f84f2010-11-23 19:36:57 +00002852 found_type != BTRFS_EXTENT_DATA_KEY) {
Chris Masonec29ed52011-02-23 16:23:20 -05002853 /* have to trust i_size as the end */
2854 last = (u64)-1;
2855 last_for_get_extent = isize;
2856 } else {
2857 /*
2858 * remember the start of the last extent. There are a
2859 * bunch of different factors that go into the length of the
2860 * extent, so its much less complex to remember where it started
2861 */
2862 last = found_key.offset;
2863 last_for_get_extent = last + 1;
Josef Bacik975f84f2010-11-23 19:36:57 +00002864 }
Josef Bacik975f84f2010-11-23 19:36:57 +00002865 btrfs_free_path(path);
2866
Chris Masonec29ed52011-02-23 16:23:20 -05002867 /*
2868 * we might have some extents allocated but more delalloc past those
2869 * extents. so, we trust isize unless the start of the last extent is
2870 * beyond isize
2871 */
2872 if (last < isize) {
2873 last = (u64)-1;
2874 last_for_get_extent = isize;
2875 }
2876
Josef Bacik2ac55d42010-02-03 19:33:23 +00002877 lock_extent_bits(&BTRFS_I(inode)->io_tree, start, start + len, 0,
2878 &cached_state, GFP_NOFS);
Chris Masonec29ed52011-02-23 16:23:20 -05002879
2880 em = get_extent_skip_holes(inode, off, last_for_get_extent,
2881 get_extent);
Yehuda Sadeh1506fcc2009-01-21 14:39:14 -05002882 if (!em)
2883 goto out;
2884 if (IS_ERR(em)) {
2885 ret = PTR_ERR(em);
2886 goto out;
2887 }
Josef Bacik975f84f2010-11-23 19:36:57 +00002888
Yehuda Sadeh1506fcc2009-01-21 14:39:14 -05002889 while (!end) {
Chris Masonea8efc72011-03-08 11:54:40 -05002890 u64 offset_in_extent;
Yehuda Sadeh1506fcc2009-01-21 14:39:14 -05002891
Chris Masonea8efc72011-03-08 11:54:40 -05002892 /* break if the extent we found is outside the range */
2893 if (em->start >= max || extent_map_end(em) < off)
2894 break;
2895
2896 /*
2897 * get_extent may return an extent that starts before our
2898 * requested range. We have to make sure the ranges
2899 * we return to fiemap always move forward and don't
2900 * overlap, so adjust the offsets here
2901 */
2902 em_start = max(em->start, off);
2903
2904 /*
2905 * record the offset from the start of the extent
2906 * for adjusting the disk offset below
2907 */
2908 offset_in_extent = em_start - em->start;
Chris Masonec29ed52011-02-23 16:23:20 -05002909 em_end = extent_map_end(em);
Chris Masonea8efc72011-03-08 11:54:40 -05002910 em_len = em_end - em_start;
Chris Masonec29ed52011-02-23 16:23:20 -05002911 emflags = em->flags;
Yehuda Sadeh1506fcc2009-01-21 14:39:14 -05002912 disko = 0;
2913 flags = 0;
2914
Chris Masonea8efc72011-03-08 11:54:40 -05002915 /*
2916 * bump off for our next call to get_extent
2917 */
2918 off = extent_map_end(em);
2919 if (off >= max)
2920 end = 1;
2921
Heiko Carstens93dbfad2009-04-03 10:33:45 -04002922 if (em->block_start == EXTENT_MAP_LAST_BYTE) {
Yehuda Sadeh1506fcc2009-01-21 14:39:14 -05002923 end = 1;
2924 flags |= FIEMAP_EXTENT_LAST;
Heiko Carstens93dbfad2009-04-03 10:33:45 -04002925 } else if (em->block_start == EXTENT_MAP_INLINE) {
Yehuda Sadeh1506fcc2009-01-21 14:39:14 -05002926 flags |= (FIEMAP_EXTENT_DATA_INLINE |
2927 FIEMAP_EXTENT_NOT_ALIGNED);
Heiko Carstens93dbfad2009-04-03 10:33:45 -04002928 } else if (em->block_start == EXTENT_MAP_DELALLOC) {
Yehuda Sadeh1506fcc2009-01-21 14:39:14 -05002929 flags |= (FIEMAP_EXTENT_DELALLOC |
2930 FIEMAP_EXTENT_UNKNOWN);
Heiko Carstens93dbfad2009-04-03 10:33:45 -04002931 } else {
Chris Masonea8efc72011-03-08 11:54:40 -05002932 disko = em->block_start + offset_in_extent;
Yehuda Sadeh1506fcc2009-01-21 14:39:14 -05002933 }
2934 if (test_bit(EXTENT_FLAG_COMPRESSED, &em->flags))
2935 flags |= FIEMAP_EXTENT_ENCODED;
2936
Yehuda Sadeh1506fcc2009-01-21 14:39:14 -05002937 free_extent_map(em);
2938 em = NULL;
Chris Masonec29ed52011-02-23 16:23:20 -05002939 if ((em_start >= last) || em_len == (u64)-1 ||
2940 (last == (u64)-1 && isize <= em_end)) {
Yehuda Sadeh1506fcc2009-01-21 14:39:14 -05002941 flags |= FIEMAP_EXTENT_LAST;
2942 end = 1;
2943 }
2944
Chris Masonec29ed52011-02-23 16:23:20 -05002945 /* now scan forward to see if this is really the last extent. */
2946 em = get_extent_skip_holes(inode, off, last_for_get_extent,
2947 get_extent);
2948 if (IS_ERR(em)) {
2949 ret = PTR_ERR(em);
2950 goto out;
2951 }
2952 if (!em) {
Josef Bacik975f84f2010-11-23 19:36:57 +00002953 flags |= FIEMAP_EXTENT_LAST;
2954 end = 1;
2955 }
Chris Masonec29ed52011-02-23 16:23:20 -05002956 ret = fiemap_fill_next_extent(fieinfo, em_start, disko,
2957 em_len, flags);
2958 if (ret)
2959 goto out_free;
Yehuda Sadeh1506fcc2009-01-21 14:39:14 -05002960 }
2961out_free:
2962 free_extent_map(em);
2963out:
Josef Bacik2ac55d42010-02-03 19:33:23 +00002964 unlock_extent_cached(&BTRFS_I(inode)->io_tree, start, start + len,
2965 &cached_state, GFP_NOFS);
Yehuda Sadeh1506fcc2009-01-21 14:39:14 -05002966 return ret;
2967}
2968
Chris Masond1310b22008-01-24 16:13:08 -05002969static inline struct page *extent_buffer_page(struct extent_buffer *eb,
2970 unsigned long i)
2971{
2972 struct page *p;
2973 struct address_space *mapping;
2974
2975 if (i == 0)
2976 return eb->first_page;
2977 i += eb->start >> PAGE_CACHE_SHIFT;
2978 mapping = eb->first_page->mapping;
Chris Mason33958dc2008-07-30 10:29:12 -04002979 if (!mapping)
2980 return NULL;
Sven Wegener0ee0fda2008-07-30 16:54:26 -04002981
2982 /*
2983 * extent_buffer_page is only called after pinning the page
2984 * by increasing the reference count. So we know the page must
2985 * be in the radix tree.
2986 */
Sven Wegener0ee0fda2008-07-30 16:54:26 -04002987 rcu_read_lock();
Chris Masond1310b22008-01-24 16:13:08 -05002988 p = radix_tree_lookup(&mapping->page_tree, i);
Sven Wegener0ee0fda2008-07-30 16:54:26 -04002989 rcu_read_unlock();
Chris Mason2b1f55b2008-09-24 11:48:04 -04002990
Chris Masond1310b22008-01-24 16:13:08 -05002991 return p;
2992}
2993
Chris Mason6af118ce2008-07-22 11:18:07 -04002994static inline unsigned long num_extent_pages(u64 start, u64 len)
Chris Masonce9adaa2008-04-09 16:28:12 -04002995{
Chris Mason6af118ce2008-07-22 11:18:07 -04002996 return ((start + len + PAGE_CACHE_SIZE - 1) >> PAGE_CACHE_SHIFT) -
2997 (start >> PAGE_CACHE_SHIFT);
Chris Mason728131d2008-04-09 16:28:12 -04002998}
2999
Chris Masond1310b22008-01-24 16:13:08 -05003000static struct extent_buffer *__alloc_extent_buffer(struct extent_io_tree *tree,
3001 u64 start,
3002 unsigned long len,
3003 gfp_t mask)
3004{
3005 struct extent_buffer *eb = NULL;
Chris Mason39351272009-02-04 09:24:05 -05003006#if LEAK_DEBUG
Chris Mason2d2ae542008-03-26 16:24:23 -04003007 unsigned long flags;
Chris Mason4bef0842008-09-08 11:18:08 -04003008#endif
Chris Masond1310b22008-01-24 16:13:08 -05003009
Chris Masond1310b22008-01-24 16:13:08 -05003010 eb = kmem_cache_zalloc(extent_buffer_cache, mask);
Tsutomu Itoh91ca3382011-01-05 02:32:22 +00003011 if (eb == NULL)
3012 return NULL;
Chris Masond1310b22008-01-24 16:13:08 -05003013 eb->start = start;
3014 eb->len = len;
Chris Masonb4ce94d2009-02-04 09:25:08 -05003015 spin_lock_init(&eb->lock);
3016 init_waitqueue_head(&eb->lock_wq);
3017
Chris Mason39351272009-02-04 09:24:05 -05003018#if LEAK_DEBUG
Chris Mason2d2ae542008-03-26 16:24:23 -04003019 spin_lock_irqsave(&leak_lock, flags);
3020 list_add(&eb->leak_list, &buffers);
3021 spin_unlock_irqrestore(&leak_lock, flags);
Chris Mason4bef0842008-09-08 11:18:08 -04003022#endif
Chris Masond1310b22008-01-24 16:13:08 -05003023 atomic_set(&eb->refs, 1);
3024
3025 return eb;
3026}
3027
3028static void __free_extent_buffer(struct extent_buffer *eb)
3029{
Chris Mason39351272009-02-04 09:24:05 -05003030#if LEAK_DEBUG
Chris Mason2d2ae542008-03-26 16:24:23 -04003031 unsigned long flags;
3032 spin_lock_irqsave(&leak_lock, flags);
3033 list_del(&eb->leak_list);
3034 spin_unlock_irqrestore(&leak_lock, flags);
Chris Mason4bef0842008-09-08 11:18:08 -04003035#endif
Chris Masond1310b22008-01-24 16:13:08 -05003036 kmem_cache_free(extent_buffer_cache, eb);
3037}
3038
Miao Xie897ca6e2010-10-26 20:57:29 -04003039/*
3040 * Helper for releasing extent buffer page.
3041 */
3042static void btrfs_release_extent_buffer_page(struct extent_buffer *eb,
3043 unsigned long start_idx)
3044{
3045 unsigned long index;
3046 struct page *page;
3047
3048 if (!eb->first_page)
3049 return;
3050
3051 index = num_extent_pages(eb->start, eb->len);
3052 if (start_idx >= index)
3053 return;
3054
3055 do {
3056 index--;
3057 page = extent_buffer_page(eb, index);
3058 if (page)
3059 page_cache_release(page);
3060 } while (index != start_idx);
3061}
3062
3063/*
3064 * Helper for releasing the extent buffer.
3065 */
3066static inline void btrfs_release_extent_buffer(struct extent_buffer *eb)
3067{
3068 btrfs_release_extent_buffer_page(eb, 0);
3069 __free_extent_buffer(eb);
3070}
3071
Chris Masond1310b22008-01-24 16:13:08 -05003072struct extent_buffer *alloc_extent_buffer(struct extent_io_tree *tree,
3073 u64 start, unsigned long len,
David Sterbaba144192011-04-21 01:12:06 +02003074 struct page *page0)
Chris Masond1310b22008-01-24 16:13:08 -05003075{
3076 unsigned long num_pages = num_extent_pages(start, len);
3077 unsigned long i;
3078 unsigned long index = start >> PAGE_CACHE_SHIFT;
3079 struct extent_buffer *eb;
Chris Mason6af118ce2008-07-22 11:18:07 -04003080 struct extent_buffer *exists = NULL;
Chris Masond1310b22008-01-24 16:13:08 -05003081 struct page *p;
3082 struct address_space *mapping = tree->mapping;
3083 int uptodate = 1;
Miao Xie19fe0a82010-10-26 20:57:29 -04003084 int ret;
Chris Masond1310b22008-01-24 16:13:08 -05003085
Miao Xie19fe0a82010-10-26 20:57:29 -04003086 rcu_read_lock();
3087 eb = radix_tree_lookup(&tree->buffer, start >> PAGE_CACHE_SHIFT);
3088 if (eb && atomic_inc_not_zero(&eb->refs)) {
3089 rcu_read_unlock();
Josef Bacik0f9dd462008-09-23 13:14:11 -04003090 mark_page_accessed(eb->first_page);
Chris Mason6af118ce2008-07-22 11:18:07 -04003091 return eb;
3092 }
Miao Xie19fe0a82010-10-26 20:57:29 -04003093 rcu_read_unlock();
Chris Mason6af118ce2008-07-22 11:18:07 -04003094
David Sterbaba144192011-04-21 01:12:06 +02003095 eb = __alloc_extent_buffer(tree, start, len, GFP_NOFS);
Peter2b114d12008-04-01 11:21:40 -04003096 if (!eb)
Chris Masond1310b22008-01-24 16:13:08 -05003097 return NULL;
3098
Chris Masond1310b22008-01-24 16:13:08 -05003099 if (page0) {
3100 eb->first_page = page0;
3101 i = 1;
3102 index++;
3103 page_cache_get(page0);
3104 mark_page_accessed(page0);
3105 set_page_extent_mapped(page0);
Chris Masond1310b22008-01-24 16:13:08 -05003106 set_page_extent_head(page0, len);
Chris Masonf1885912008-04-09 16:28:12 -04003107 uptodate = PageUptodate(page0);
Chris Masond1310b22008-01-24 16:13:08 -05003108 } else {
3109 i = 0;
3110 }
3111 for (; i < num_pages; i++, index++) {
David Sterbaba144192011-04-21 01:12:06 +02003112 p = find_or_create_page(mapping, index, GFP_NOFS | __GFP_HIGHMEM);
Chris Masond1310b22008-01-24 16:13:08 -05003113 if (!p) {
3114 WARN_ON(1);
Chris Mason6af118ce2008-07-22 11:18:07 -04003115 goto free_eb;
Chris Masond1310b22008-01-24 16:13:08 -05003116 }
3117 set_page_extent_mapped(p);
3118 mark_page_accessed(p);
3119 if (i == 0) {
3120 eb->first_page = p;
3121 set_page_extent_head(p, len);
3122 } else {
3123 set_page_private(p, EXTENT_PAGE_PRIVATE);
3124 }
3125 if (!PageUptodate(p))
3126 uptodate = 0;
Chris Masoneb14ab82011-02-10 12:35:00 -05003127
3128 /*
3129 * see below about how we avoid a nasty race with release page
3130 * and why we unlock later
3131 */
3132 if (i != 0)
3133 unlock_page(p);
Chris Masond1310b22008-01-24 16:13:08 -05003134 }
3135 if (uptodate)
Chris Masonb4ce94d2009-02-04 09:25:08 -05003136 set_bit(EXTENT_BUFFER_UPTODATE, &eb->bflags);
Chris Masond1310b22008-01-24 16:13:08 -05003137
Miao Xie19fe0a82010-10-26 20:57:29 -04003138 ret = radix_tree_preload(GFP_NOFS & ~__GFP_HIGHMEM);
3139 if (ret)
3140 goto free_eb;
3141
Chris Mason6af118ce2008-07-22 11:18:07 -04003142 spin_lock(&tree->buffer_lock);
Miao Xie19fe0a82010-10-26 20:57:29 -04003143 ret = radix_tree_insert(&tree->buffer, start >> PAGE_CACHE_SHIFT, eb);
3144 if (ret == -EEXIST) {
3145 exists = radix_tree_lookup(&tree->buffer,
3146 start >> PAGE_CACHE_SHIFT);
Chris Mason6af118ce2008-07-22 11:18:07 -04003147 /* add one reference for the caller */
3148 atomic_inc(&exists->refs);
3149 spin_unlock(&tree->buffer_lock);
Miao Xie19fe0a82010-10-26 20:57:29 -04003150 radix_tree_preload_end();
Chris Mason6af118ce2008-07-22 11:18:07 -04003151 goto free_eb;
3152 }
Chris Mason6af118ce2008-07-22 11:18:07 -04003153 /* add one reference for the tree */
3154 atomic_inc(&eb->refs);
Yan, Zhengf044ba72010-02-04 08:46:56 +00003155 spin_unlock(&tree->buffer_lock);
Miao Xie19fe0a82010-10-26 20:57:29 -04003156 radix_tree_preload_end();
Chris Masoneb14ab82011-02-10 12:35:00 -05003157
3158 /*
3159 * there is a race where release page may have
3160 * tried to find this extent buffer in the radix
3161 * but failed. It will tell the VM it is safe to
3162 * reclaim the, and it will clear the page private bit.
3163 * We must make sure to set the page private bit properly
3164 * after the extent buffer is in the radix tree so
3165 * it doesn't get lost
3166 */
3167 set_page_extent_mapped(eb->first_page);
3168 set_page_extent_head(eb->first_page, eb->len);
3169 if (!page0)
3170 unlock_page(eb->first_page);
Chris Masond1310b22008-01-24 16:13:08 -05003171 return eb;
3172
Chris Mason6af118ce2008-07-22 11:18:07 -04003173free_eb:
Chris Masoneb14ab82011-02-10 12:35:00 -05003174 if (eb->first_page && !page0)
3175 unlock_page(eb->first_page);
3176
Chris Masond1310b22008-01-24 16:13:08 -05003177 if (!atomic_dec_and_test(&eb->refs))
Chris Mason6af118ce2008-07-22 11:18:07 -04003178 return exists;
Miao Xie897ca6e2010-10-26 20:57:29 -04003179 btrfs_release_extent_buffer(eb);
Chris Mason6af118ce2008-07-22 11:18:07 -04003180 return exists;
Chris Masond1310b22008-01-24 16:13:08 -05003181}
Chris Masond1310b22008-01-24 16:13:08 -05003182
3183struct extent_buffer *find_extent_buffer(struct extent_io_tree *tree,
David Sterbaf09d1f62011-04-21 01:08:01 +02003184 u64 start, unsigned long len)
Chris Masond1310b22008-01-24 16:13:08 -05003185{
Chris Masond1310b22008-01-24 16:13:08 -05003186 struct extent_buffer *eb;
Chris Masond1310b22008-01-24 16:13:08 -05003187
Miao Xie19fe0a82010-10-26 20:57:29 -04003188 rcu_read_lock();
3189 eb = radix_tree_lookup(&tree->buffer, start >> PAGE_CACHE_SHIFT);
3190 if (eb && atomic_inc_not_zero(&eb->refs)) {
3191 rcu_read_unlock();
Josef Bacik0f9dd462008-09-23 13:14:11 -04003192 mark_page_accessed(eb->first_page);
Miao Xie19fe0a82010-10-26 20:57:29 -04003193 return eb;
3194 }
3195 rcu_read_unlock();
Josef Bacik0f9dd462008-09-23 13:14:11 -04003196
Miao Xie19fe0a82010-10-26 20:57:29 -04003197 return NULL;
Chris Masond1310b22008-01-24 16:13:08 -05003198}
Chris Masond1310b22008-01-24 16:13:08 -05003199
3200void free_extent_buffer(struct extent_buffer *eb)
3201{
Chris Masond1310b22008-01-24 16:13:08 -05003202 if (!eb)
3203 return;
3204
3205 if (!atomic_dec_and_test(&eb->refs))
3206 return;
3207
Chris Mason6af118ce2008-07-22 11:18:07 -04003208 WARN_ON(1);
Chris Masond1310b22008-01-24 16:13:08 -05003209}
Chris Masond1310b22008-01-24 16:13:08 -05003210
3211int clear_extent_buffer_dirty(struct extent_io_tree *tree,
3212 struct extent_buffer *eb)
3213{
Chris Masond1310b22008-01-24 16:13:08 -05003214 unsigned long i;
3215 unsigned long num_pages;
3216 struct page *page;
3217
Chris Masond1310b22008-01-24 16:13:08 -05003218 num_pages = num_extent_pages(eb->start, eb->len);
3219
3220 for (i = 0; i < num_pages; i++) {
3221 page = extent_buffer_page(eb, i);
Chris Masonb9473432009-03-13 11:00:37 -04003222 if (!PageDirty(page))
Chris Masond2c3f4f2008-11-19 12:44:22 -05003223 continue;
3224
Chris Masona61e6f22008-07-22 11:18:08 -04003225 lock_page(page);
Chris Masoneb14ab82011-02-10 12:35:00 -05003226 WARN_ON(!PagePrivate(page));
3227
3228 set_page_extent_mapped(page);
Chris Masond1310b22008-01-24 16:13:08 -05003229 if (i == 0)
3230 set_page_extent_head(page, eb->len);
Chris Masond1310b22008-01-24 16:13:08 -05003231
Chris Masond1310b22008-01-24 16:13:08 -05003232 clear_page_dirty_for_io(page);
Sven Wegener0ee0fda2008-07-30 16:54:26 -04003233 spin_lock_irq(&page->mapping->tree_lock);
Chris Masond1310b22008-01-24 16:13:08 -05003234 if (!PageDirty(page)) {
3235 radix_tree_tag_clear(&page->mapping->page_tree,
3236 page_index(page),
3237 PAGECACHE_TAG_DIRTY);
3238 }
Sven Wegener0ee0fda2008-07-30 16:54:26 -04003239 spin_unlock_irq(&page->mapping->tree_lock);
Chris Masona61e6f22008-07-22 11:18:08 -04003240 unlock_page(page);
Chris Masond1310b22008-01-24 16:13:08 -05003241 }
3242 return 0;
3243}
Chris Masond1310b22008-01-24 16:13:08 -05003244
Chris Masond1310b22008-01-24 16:13:08 -05003245int set_extent_buffer_dirty(struct extent_io_tree *tree,
3246 struct extent_buffer *eb)
3247{
3248 unsigned long i;
3249 unsigned long num_pages;
Chris Masonb9473432009-03-13 11:00:37 -04003250 int was_dirty = 0;
Chris Masond1310b22008-01-24 16:13:08 -05003251
Chris Masonb9473432009-03-13 11:00:37 -04003252 was_dirty = test_and_set_bit(EXTENT_BUFFER_DIRTY, &eb->bflags);
Chris Masond1310b22008-01-24 16:13:08 -05003253 num_pages = num_extent_pages(eb->start, eb->len);
Chris Masonb9473432009-03-13 11:00:37 -04003254 for (i = 0; i < num_pages; i++)
Chris Masond1310b22008-01-24 16:13:08 -05003255 __set_page_dirty_nobuffers(extent_buffer_page(eb, i));
Chris Masonb9473432009-03-13 11:00:37 -04003256 return was_dirty;
Chris Masond1310b22008-01-24 16:13:08 -05003257}
Chris Masond1310b22008-01-24 16:13:08 -05003258
Chris Mason1259ab72008-05-12 13:39:03 -04003259int clear_extent_buffer_uptodate(struct extent_io_tree *tree,
Josef Bacik2ac55d42010-02-03 19:33:23 +00003260 struct extent_buffer *eb,
3261 struct extent_state **cached_state)
Chris Mason1259ab72008-05-12 13:39:03 -04003262{
3263 unsigned long i;
3264 struct page *page;
3265 unsigned long num_pages;
3266
3267 num_pages = num_extent_pages(eb->start, eb->len);
Chris Masonb4ce94d2009-02-04 09:25:08 -05003268 clear_bit(EXTENT_BUFFER_UPTODATE, &eb->bflags);
Chris Mason1259ab72008-05-12 13:39:03 -04003269
3270 clear_extent_uptodate(tree, eb->start, eb->start + eb->len - 1,
Josef Bacik2ac55d42010-02-03 19:33:23 +00003271 cached_state, GFP_NOFS);
Chris Mason1259ab72008-05-12 13:39:03 -04003272 for (i = 0; i < num_pages; i++) {
3273 page = extent_buffer_page(eb, i);
Chris Mason33958dc2008-07-30 10:29:12 -04003274 if (page)
3275 ClearPageUptodate(page);
Chris Mason1259ab72008-05-12 13:39:03 -04003276 }
3277 return 0;
3278}
3279
Chris Masond1310b22008-01-24 16:13:08 -05003280int set_extent_buffer_uptodate(struct extent_io_tree *tree,
3281 struct extent_buffer *eb)
3282{
3283 unsigned long i;
3284 struct page *page;
3285 unsigned long num_pages;
3286
3287 num_pages = num_extent_pages(eb->start, eb->len);
3288
3289 set_extent_uptodate(tree, eb->start, eb->start + eb->len - 1,
Arne Jansen507903b2011-04-06 10:02:20 +00003290 NULL, GFP_NOFS);
Chris Masond1310b22008-01-24 16:13:08 -05003291 for (i = 0; i < num_pages; i++) {
3292 page = extent_buffer_page(eb, i);
3293 if ((i == 0 && (eb->start & (PAGE_CACHE_SIZE - 1))) ||
3294 ((i == num_pages - 1) &&
3295 ((eb->start + eb->len) & (PAGE_CACHE_SIZE - 1)))) {
3296 check_page_uptodate(tree, page);
3297 continue;
3298 }
3299 SetPageUptodate(page);
3300 }
3301 return 0;
3302}
Chris Masond1310b22008-01-24 16:13:08 -05003303
Chris Masonce9adaa2008-04-09 16:28:12 -04003304int extent_range_uptodate(struct extent_io_tree *tree,
3305 u64 start, u64 end)
3306{
3307 struct page *page;
3308 int ret;
3309 int pg_uptodate = 1;
3310 int uptodate;
3311 unsigned long index;
3312
Chris Mason9655d292009-09-02 15:22:30 -04003313 ret = test_range_bit(tree, start, end, EXTENT_UPTODATE, 1, NULL);
Chris Masonce9adaa2008-04-09 16:28:12 -04003314 if (ret)
3315 return 1;
Chris Masond3977122009-01-05 21:25:51 -05003316 while (start <= end) {
Chris Masonce9adaa2008-04-09 16:28:12 -04003317 index = start >> PAGE_CACHE_SHIFT;
3318 page = find_get_page(tree->mapping, index);
3319 uptodate = PageUptodate(page);
3320 page_cache_release(page);
3321 if (!uptodate) {
3322 pg_uptodate = 0;
3323 break;
3324 }
3325 start += PAGE_CACHE_SIZE;
3326 }
3327 return pg_uptodate;
3328}
3329
Chris Masond1310b22008-01-24 16:13:08 -05003330int extent_buffer_uptodate(struct extent_io_tree *tree,
Josef Bacik2ac55d42010-02-03 19:33:23 +00003331 struct extent_buffer *eb,
3332 struct extent_state *cached_state)
Chris Masond1310b22008-01-24 16:13:08 -05003333{
Chris Mason728131d2008-04-09 16:28:12 -04003334 int ret = 0;
Chris Masonce9adaa2008-04-09 16:28:12 -04003335 unsigned long num_pages;
3336 unsigned long i;
Chris Mason728131d2008-04-09 16:28:12 -04003337 struct page *page;
3338 int pg_uptodate = 1;
3339
Chris Masonb4ce94d2009-02-04 09:25:08 -05003340 if (test_bit(EXTENT_BUFFER_UPTODATE, &eb->bflags))
Chris Mason42352982008-04-28 16:40:52 -04003341 return 1;
Chris Mason728131d2008-04-09 16:28:12 -04003342
Chris Mason42352982008-04-28 16:40:52 -04003343 ret = test_range_bit(tree, eb->start, eb->start + eb->len - 1,
Josef Bacik2ac55d42010-02-03 19:33:23 +00003344 EXTENT_UPTODATE, 1, cached_state);
Chris Mason42352982008-04-28 16:40:52 -04003345 if (ret)
3346 return ret;
Chris Mason728131d2008-04-09 16:28:12 -04003347
3348 num_pages = num_extent_pages(eb->start, eb->len);
3349 for (i = 0; i < num_pages; i++) {
3350 page = extent_buffer_page(eb, i);
3351 if (!PageUptodate(page)) {
3352 pg_uptodate = 0;
3353 break;
3354 }
3355 }
Chris Mason42352982008-04-28 16:40:52 -04003356 return pg_uptodate;
Chris Masond1310b22008-01-24 16:13:08 -05003357}
Chris Masond1310b22008-01-24 16:13:08 -05003358
3359int read_extent_buffer_pages(struct extent_io_tree *tree,
3360 struct extent_buffer *eb,
Chris Masona86c12c2008-02-07 10:50:54 -05003361 u64 start, int wait,
Chris Masonf1885912008-04-09 16:28:12 -04003362 get_extent_t *get_extent, int mirror_num)
Chris Masond1310b22008-01-24 16:13:08 -05003363{
3364 unsigned long i;
3365 unsigned long start_i;
3366 struct page *page;
3367 int err;
3368 int ret = 0;
Chris Masonce9adaa2008-04-09 16:28:12 -04003369 int locked_pages = 0;
3370 int all_uptodate = 1;
3371 int inc_all_pages = 0;
Chris Masond1310b22008-01-24 16:13:08 -05003372 unsigned long num_pages;
Chris Masona86c12c2008-02-07 10:50:54 -05003373 struct bio *bio = NULL;
Chris Masonc8b97812008-10-29 14:49:59 -04003374 unsigned long bio_flags = 0;
Chris Masona86c12c2008-02-07 10:50:54 -05003375
Chris Masonb4ce94d2009-02-04 09:25:08 -05003376 if (test_bit(EXTENT_BUFFER_UPTODATE, &eb->bflags))
Chris Masond1310b22008-01-24 16:13:08 -05003377 return 0;
3378
Chris Masonce9adaa2008-04-09 16:28:12 -04003379 if (test_range_bit(tree, eb->start, eb->start + eb->len - 1,
Chris Mason9655d292009-09-02 15:22:30 -04003380 EXTENT_UPTODATE, 1, NULL)) {
Chris Masond1310b22008-01-24 16:13:08 -05003381 return 0;
3382 }
3383
3384 if (start) {
3385 WARN_ON(start < eb->start);
3386 start_i = (start >> PAGE_CACHE_SHIFT) -
3387 (eb->start >> PAGE_CACHE_SHIFT);
3388 } else {
3389 start_i = 0;
3390 }
3391
3392 num_pages = num_extent_pages(eb->start, eb->len);
3393 for (i = start_i; i < num_pages; i++) {
3394 page = extent_buffer_page(eb, i);
Chris Masond1310b22008-01-24 16:13:08 -05003395 if (!wait) {
David Woodhouse2db04962008-08-07 11:19:43 -04003396 if (!trylock_page(page))
Chris Masonce9adaa2008-04-09 16:28:12 -04003397 goto unlock_exit;
Chris Masond1310b22008-01-24 16:13:08 -05003398 } else {
3399 lock_page(page);
3400 }
Chris Masonce9adaa2008-04-09 16:28:12 -04003401 locked_pages++;
Chris Masond3977122009-01-05 21:25:51 -05003402 if (!PageUptodate(page))
Chris Masonce9adaa2008-04-09 16:28:12 -04003403 all_uptodate = 0;
Chris Masonce9adaa2008-04-09 16:28:12 -04003404 }
3405 if (all_uptodate) {
3406 if (start_i == 0)
Chris Masonb4ce94d2009-02-04 09:25:08 -05003407 set_bit(EXTENT_BUFFER_UPTODATE, &eb->bflags);
Chris Masonce9adaa2008-04-09 16:28:12 -04003408 goto unlock_exit;
3409 }
3410
3411 for (i = start_i; i < num_pages; i++) {
3412 page = extent_buffer_page(eb, i);
Chris Masoneb14ab82011-02-10 12:35:00 -05003413
3414 WARN_ON(!PagePrivate(page));
3415
3416 set_page_extent_mapped(page);
3417 if (i == 0)
3418 set_page_extent_head(page, eb->len);
3419
Chris Masonce9adaa2008-04-09 16:28:12 -04003420 if (inc_all_pages)
3421 page_cache_get(page);
3422 if (!PageUptodate(page)) {
3423 if (start_i == 0)
3424 inc_all_pages = 1;
Chris Masonf1885912008-04-09 16:28:12 -04003425 ClearPageError(page);
Chris Masona86c12c2008-02-07 10:50:54 -05003426 err = __extent_read_full_page(tree, page,
Chris Masonf1885912008-04-09 16:28:12 -04003427 get_extent, &bio,
Chris Masonc8b97812008-10-29 14:49:59 -04003428 mirror_num, &bio_flags);
Chris Masond3977122009-01-05 21:25:51 -05003429 if (err)
Chris Masond1310b22008-01-24 16:13:08 -05003430 ret = err;
Chris Masond1310b22008-01-24 16:13:08 -05003431 } else {
3432 unlock_page(page);
3433 }
3434 }
3435
Chris Masona86c12c2008-02-07 10:50:54 -05003436 if (bio)
Chris Masonc8b97812008-10-29 14:49:59 -04003437 submit_one_bio(READ, bio, mirror_num, bio_flags);
Chris Masona86c12c2008-02-07 10:50:54 -05003438
Chris Masond3977122009-01-05 21:25:51 -05003439 if (ret || !wait)
Chris Masond1310b22008-01-24 16:13:08 -05003440 return ret;
Chris Masond3977122009-01-05 21:25:51 -05003441
Chris Masond1310b22008-01-24 16:13:08 -05003442 for (i = start_i; i < num_pages; i++) {
3443 page = extent_buffer_page(eb, i);
3444 wait_on_page_locked(page);
Chris Masond3977122009-01-05 21:25:51 -05003445 if (!PageUptodate(page))
Chris Masond1310b22008-01-24 16:13:08 -05003446 ret = -EIO;
Chris Masond1310b22008-01-24 16:13:08 -05003447 }
Chris Masond3977122009-01-05 21:25:51 -05003448
Chris Masond1310b22008-01-24 16:13:08 -05003449 if (!ret)
Chris Masonb4ce94d2009-02-04 09:25:08 -05003450 set_bit(EXTENT_BUFFER_UPTODATE, &eb->bflags);
Chris Masond1310b22008-01-24 16:13:08 -05003451 return ret;
Chris Masonce9adaa2008-04-09 16:28:12 -04003452
3453unlock_exit:
3454 i = start_i;
Chris Masond3977122009-01-05 21:25:51 -05003455 while (locked_pages > 0) {
Chris Masonce9adaa2008-04-09 16:28:12 -04003456 page = extent_buffer_page(eb, i);
3457 i++;
3458 unlock_page(page);
3459 locked_pages--;
3460 }
3461 return ret;
Chris Masond1310b22008-01-24 16:13:08 -05003462}
Chris Masond1310b22008-01-24 16:13:08 -05003463
3464void read_extent_buffer(struct extent_buffer *eb, void *dstv,
3465 unsigned long start,
3466 unsigned long len)
3467{
3468 size_t cur;
3469 size_t offset;
3470 struct page *page;
3471 char *kaddr;
3472 char *dst = (char *)dstv;
3473 size_t start_offset = eb->start & ((u64)PAGE_CACHE_SIZE - 1);
3474 unsigned long i = (start_offset + start) >> PAGE_CACHE_SHIFT;
Chris Masond1310b22008-01-24 16:13:08 -05003475
3476 WARN_ON(start > eb->len);
3477 WARN_ON(start + len > eb->start + eb->len);
3478
3479 offset = (start_offset + start) & ((unsigned long)PAGE_CACHE_SIZE - 1);
3480
Chris Masond3977122009-01-05 21:25:51 -05003481 while (len > 0) {
Chris Masond1310b22008-01-24 16:13:08 -05003482 page = extent_buffer_page(eb, i);
Chris Masond1310b22008-01-24 16:13:08 -05003483
3484 cur = min(len, (PAGE_CACHE_SIZE - offset));
3485 kaddr = kmap_atomic(page, KM_USER1);
3486 memcpy(dst, kaddr + offset, cur);
3487 kunmap_atomic(kaddr, KM_USER1);
3488
3489 dst += cur;
3490 len -= cur;
3491 offset = 0;
3492 i++;
3493 }
3494}
Chris Masond1310b22008-01-24 16:13:08 -05003495
3496int map_private_extent_buffer(struct extent_buffer *eb, unsigned long start,
3497 unsigned long min_len, char **token, char **map,
3498 unsigned long *map_start,
3499 unsigned long *map_len, int km)
3500{
3501 size_t offset = start & (PAGE_CACHE_SIZE - 1);
3502 char *kaddr;
3503 struct page *p;
3504 size_t start_offset = eb->start & ((u64)PAGE_CACHE_SIZE - 1);
3505 unsigned long i = (start_offset + start) >> PAGE_CACHE_SHIFT;
3506 unsigned long end_i = (start_offset + start + min_len - 1) >>
3507 PAGE_CACHE_SHIFT;
3508
3509 if (i != end_i)
3510 return -EINVAL;
3511
3512 if (i == 0) {
3513 offset = start_offset;
3514 *map_start = 0;
3515 } else {
3516 offset = 0;
3517 *map_start = ((u64)i << PAGE_CACHE_SHIFT) - start_offset;
3518 }
Chris Masond3977122009-01-05 21:25:51 -05003519
Chris Masond1310b22008-01-24 16:13:08 -05003520 if (start + min_len > eb->len) {
Chris Masond3977122009-01-05 21:25:51 -05003521 printk(KERN_ERR "btrfs bad mapping eb start %llu len %lu, "
3522 "wanted %lu %lu\n", (unsigned long long)eb->start,
3523 eb->len, start, min_len);
Chris Masond1310b22008-01-24 16:13:08 -05003524 WARN_ON(1);
Josef Bacik850265332011-03-15 14:52:12 -04003525 return -EINVAL;
Chris Masond1310b22008-01-24 16:13:08 -05003526 }
3527
3528 p = extent_buffer_page(eb, i);
Chris Masond1310b22008-01-24 16:13:08 -05003529 kaddr = kmap_atomic(p, km);
3530 *token = kaddr;
3531 *map = kaddr + offset;
3532 *map_len = PAGE_CACHE_SIZE - offset;
3533 return 0;
3534}
Chris Masond1310b22008-01-24 16:13:08 -05003535
3536int map_extent_buffer(struct extent_buffer *eb, unsigned long start,
3537 unsigned long min_len,
3538 char **token, char **map,
3539 unsigned long *map_start,
3540 unsigned long *map_len, int km)
3541{
3542 int err;
3543 int save = 0;
3544 if (eb->map_token) {
3545 unmap_extent_buffer(eb, eb->map_token, km);
3546 eb->map_token = NULL;
3547 save = 1;
3548 }
3549 err = map_private_extent_buffer(eb, start, min_len, token, map,
3550 map_start, map_len, km);
3551 if (!err && save) {
3552 eb->map_token = *token;
3553 eb->kaddr = *map;
3554 eb->map_start = *map_start;
3555 eb->map_len = *map_len;
3556 }
3557 return err;
3558}
Chris Masond1310b22008-01-24 16:13:08 -05003559
3560void unmap_extent_buffer(struct extent_buffer *eb, char *token, int km)
3561{
3562 kunmap_atomic(token, km);
3563}
Chris Masond1310b22008-01-24 16:13:08 -05003564
3565int memcmp_extent_buffer(struct extent_buffer *eb, const void *ptrv,
3566 unsigned long start,
3567 unsigned long len)
3568{
3569 size_t cur;
3570 size_t offset;
3571 struct page *page;
3572 char *kaddr;
3573 char *ptr = (char *)ptrv;
3574 size_t start_offset = eb->start & ((u64)PAGE_CACHE_SIZE - 1);
3575 unsigned long i = (start_offset + start) >> PAGE_CACHE_SHIFT;
3576 int ret = 0;
3577
3578 WARN_ON(start > eb->len);
3579 WARN_ON(start + len > eb->start + eb->len);
3580
3581 offset = (start_offset + start) & ((unsigned long)PAGE_CACHE_SIZE - 1);
3582
Chris Masond3977122009-01-05 21:25:51 -05003583 while (len > 0) {
Chris Masond1310b22008-01-24 16:13:08 -05003584 page = extent_buffer_page(eb, i);
Chris Masond1310b22008-01-24 16:13:08 -05003585
3586 cur = min(len, (PAGE_CACHE_SIZE - offset));
3587
3588 kaddr = kmap_atomic(page, KM_USER0);
3589 ret = memcmp(ptr, kaddr + offset, cur);
3590 kunmap_atomic(kaddr, KM_USER0);
3591 if (ret)
3592 break;
3593
3594 ptr += cur;
3595 len -= cur;
3596 offset = 0;
3597 i++;
3598 }
3599 return ret;
3600}
Chris Masond1310b22008-01-24 16:13:08 -05003601
3602void write_extent_buffer(struct extent_buffer *eb, const void *srcv,
3603 unsigned long start, unsigned long len)
3604{
3605 size_t cur;
3606 size_t offset;
3607 struct page *page;
3608 char *kaddr;
3609 char *src = (char *)srcv;
3610 size_t start_offset = eb->start & ((u64)PAGE_CACHE_SIZE - 1);
3611 unsigned long i = (start_offset + start) >> PAGE_CACHE_SHIFT;
3612
3613 WARN_ON(start > eb->len);
3614 WARN_ON(start + len > eb->start + eb->len);
3615
3616 offset = (start_offset + start) & ((unsigned long)PAGE_CACHE_SIZE - 1);
3617
Chris Masond3977122009-01-05 21:25:51 -05003618 while (len > 0) {
Chris Masond1310b22008-01-24 16:13:08 -05003619 page = extent_buffer_page(eb, i);
3620 WARN_ON(!PageUptodate(page));
3621
3622 cur = min(len, PAGE_CACHE_SIZE - offset);
3623 kaddr = kmap_atomic(page, KM_USER1);
3624 memcpy(kaddr + offset, src, cur);
3625 kunmap_atomic(kaddr, KM_USER1);
3626
3627 src += cur;
3628 len -= cur;
3629 offset = 0;
3630 i++;
3631 }
3632}
Chris Masond1310b22008-01-24 16:13:08 -05003633
3634void memset_extent_buffer(struct extent_buffer *eb, char c,
3635 unsigned long start, unsigned long len)
3636{
3637 size_t cur;
3638 size_t offset;
3639 struct page *page;
3640 char *kaddr;
3641 size_t start_offset = eb->start & ((u64)PAGE_CACHE_SIZE - 1);
3642 unsigned long i = (start_offset + start) >> PAGE_CACHE_SHIFT;
3643
3644 WARN_ON(start > eb->len);
3645 WARN_ON(start + len > eb->start + eb->len);
3646
3647 offset = (start_offset + start) & ((unsigned long)PAGE_CACHE_SIZE - 1);
3648
Chris Masond3977122009-01-05 21:25:51 -05003649 while (len > 0) {
Chris Masond1310b22008-01-24 16:13:08 -05003650 page = extent_buffer_page(eb, i);
3651 WARN_ON(!PageUptodate(page));
3652
3653 cur = min(len, PAGE_CACHE_SIZE - offset);
3654 kaddr = kmap_atomic(page, KM_USER0);
3655 memset(kaddr + offset, c, cur);
3656 kunmap_atomic(kaddr, KM_USER0);
3657
3658 len -= cur;
3659 offset = 0;
3660 i++;
3661 }
3662}
Chris Masond1310b22008-01-24 16:13:08 -05003663
3664void copy_extent_buffer(struct extent_buffer *dst, struct extent_buffer *src,
3665 unsigned long dst_offset, unsigned long src_offset,
3666 unsigned long len)
3667{
3668 u64 dst_len = dst->len;
3669 size_t cur;
3670 size_t offset;
3671 struct page *page;
3672 char *kaddr;
3673 size_t start_offset = dst->start & ((u64)PAGE_CACHE_SIZE - 1);
3674 unsigned long i = (start_offset + dst_offset) >> PAGE_CACHE_SHIFT;
3675
3676 WARN_ON(src->len != dst_len);
3677
3678 offset = (start_offset + dst_offset) &
3679 ((unsigned long)PAGE_CACHE_SIZE - 1);
3680
Chris Masond3977122009-01-05 21:25:51 -05003681 while (len > 0) {
Chris Masond1310b22008-01-24 16:13:08 -05003682 page = extent_buffer_page(dst, i);
3683 WARN_ON(!PageUptodate(page));
3684
3685 cur = min(len, (unsigned long)(PAGE_CACHE_SIZE - offset));
3686
3687 kaddr = kmap_atomic(page, KM_USER0);
3688 read_extent_buffer(src, kaddr + offset, src_offset, cur);
3689 kunmap_atomic(kaddr, KM_USER0);
3690
3691 src_offset += cur;
3692 len -= cur;
3693 offset = 0;
3694 i++;
3695 }
3696}
Chris Masond1310b22008-01-24 16:13:08 -05003697
3698static void move_pages(struct page *dst_page, struct page *src_page,
3699 unsigned long dst_off, unsigned long src_off,
3700 unsigned long len)
3701{
3702 char *dst_kaddr = kmap_atomic(dst_page, KM_USER0);
3703 if (dst_page == src_page) {
3704 memmove(dst_kaddr + dst_off, dst_kaddr + src_off, len);
3705 } else {
3706 char *src_kaddr = kmap_atomic(src_page, KM_USER1);
3707 char *p = dst_kaddr + dst_off + len;
3708 char *s = src_kaddr + src_off + len;
3709
3710 while (len--)
3711 *--p = *--s;
3712
3713 kunmap_atomic(src_kaddr, KM_USER1);
3714 }
3715 kunmap_atomic(dst_kaddr, KM_USER0);
3716}
3717
Sergei Trofimovich33872062011-04-11 21:52:52 +00003718static inline bool areas_overlap(unsigned long src, unsigned long dst, unsigned long len)
3719{
3720 unsigned long distance = (src > dst) ? src - dst : dst - src;
3721 return distance < len;
3722}
3723
Chris Masond1310b22008-01-24 16:13:08 -05003724static void copy_pages(struct page *dst_page, struct page *src_page,
3725 unsigned long dst_off, unsigned long src_off,
3726 unsigned long len)
3727{
3728 char *dst_kaddr = kmap_atomic(dst_page, KM_USER0);
3729 char *src_kaddr;
3730
Sergei Trofimovich33872062011-04-11 21:52:52 +00003731 if (dst_page != src_page) {
Chris Masond1310b22008-01-24 16:13:08 -05003732 src_kaddr = kmap_atomic(src_page, KM_USER1);
Sergei Trofimovich33872062011-04-11 21:52:52 +00003733 } else {
Chris Masond1310b22008-01-24 16:13:08 -05003734 src_kaddr = dst_kaddr;
Sergei Trofimovich33872062011-04-11 21:52:52 +00003735 BUG_ON(areas_overlap(src_off, dst_off, len));
3736 }
Chris Masond1310b22008-01-24 16:13:08 -05003737
3738 memcpy(dst_kaddr + dst_off, src_kaddr + src_off, len);
3739 kunmap_atomic(dst_kaddr, KM_USER0);
3740 if (dst_page != src_page)
3741 kunmap_atomic(src_kaddr, KM_USER1);
3742}
3743
3744void memcpy_extent_buffer(struct extent_buffer *dst, unsigned long dst_offset,
3745 unsigned long src_offset, unsigned long len)
3746{
3747 size_t cur;
3748 size_t dst_off_in_page;
3749 size_t src_off_in_page;
3750 size_t start_offset = dst->start & ((u64)PAGE_CACHE_SIZE - 1);
3751 unsigned long dst_i;
3752 unsigned long src_i;
3753
3754 if (src_offset + len > dst->len) {
Chris Masond3977122009-01-05 21:25:51 -05003755 printk(KERN_ERR "btrfs memmove bogus src_offset %lu move "
3756 "len %lu dst len %lu\n", src_offset, len, dst->len);
Chris Masond1310b22008-01-24 16:13:08 -05003757 BUG_ON(1);
3758 }
3759 if (dst_offset + len > dst->len) {
Chris Masond3977122009-01-05 21:25:51 -05003760 printk(KERN_ERR "btrfs memmove bogus dst_offset %lu move "
3761 "len %lu dst len %lu\n", dst_offset, len, dst->len);
Chris Masond1310b22008-01-24 16:13:08 -05003762 BUG_ON(1);
3763 }
3764
Chris Masond3977122009-01-05 21:25:51 -05003765 while (len > 0) {
Chris Masond1310b22008-01-24 16:13:08 -05003766 dst_off_in_page = (start_offset + dst_offset) &
3767 ((unsigned long)PAGE_CACHE_SIZE - 1);
3768 src_off_in_page = (start_offset + src_offset) &
3769 ((unsigned long)PAGE_CACHE_SIZE - 1);
3770
3771 dst_i = (start_offset + dst_offset) >> PAGE_CACHE_SHIFT;
3772 src_i = (start_offset + src_offset) >> PAGE_CACHE_SHIFT;
3773
3774 cur = min(len, (unsigned long)(PAGE_CACHE_SIZE -
3775 src_off_in_page));
3776 cur = min_t(unsigned long, cur,
3777 (unsigned long)(PAGE_CACHE_SIZE - dst_off_in_page));
3778
3779 copy_pages(extent_buffer_page(dst, dst_i),
3780 extent_buffer_page(dst, src_i),
3781 dst_off_in_page, src_off_in_page, cur);
3782
3783 src_offset += cur;
3784 dst_offset += cur;
3785 len -= cur;
3786 }
3787}
Chris Masond1310b22008-01-24 16:13:08 -05003788
3789void memmove_extent_buffer(struct extent_buffer *dst, unsigned long dst_offset,
3790 unsigned long src_offset, unsigned long len)
3791{
3792 size_t cur;
3793 size_t dst_off_in_page;
3794 size_t src_off_in_page;
3795 unsigned long dst_end = dst_offset + len - 1;
3796 unsigned long src_end = src_offset + len - 1;
3797 size_t start_offset = dst->start & ((u64)PAGE_CACHE_SIZE - 1);
3798 unsigned long dst_i;
3799 unsigned long src_i;
3800
3801 if (src_offset + len > dst->len) {
Chris Masond3977122009-01-05 21:25:51 -05003802 printk(KERN_ERR "btrfs memmove bogus src_offset %lu move "
3803 "len %lu len %lu\n", src_offset, len, dst->len);
Chris Masond1310b22008-01-24 16:13:08 -05003804 BUG_ON(1);
3805 }
3806 if (dst_offset + len > dst->len) {
Chris Masond3977122009-01-05 21:25:51 -05003807 printk(KERN_ERR "btrfs memmove bogus dst_offset %lu move "
3808 "len %lu len %lu\n", dst_offset, len, dst->len);
Chris Masond1310b22008-01-24 16:13:08 -05003809 BUG_ON(1);
3810 }
Sergei Trofimovich33872062011-04-11 21:52:52 +00003811 if (!areas_overlap(src_offset, dst_offset, len)) {
Chris Masond1310b22008-01-24 16:13:08 -05003812 memcpy_extent_buffer(dst, dst_offset, src_offset, len);
3813 return;
3814 }
Chris Masond3977122009-01-05 21:25:51 -05003815 while (len > 0) {
Chris Masond1310b22008-01-24 16:13:08 -05003816 dst_i = (start_offset + dst_end) >> PAGE_CACHE_SHIFT;
3817 src_i = (start_offset + src_end) >> PAGE_CACHE_SHIFT;
3818
3819 dst_off_in_page = (start_offset + dst_end) &
3820 ((unsigned long)PAGE_CACHE_SIZE - 1);
3821 src_off_in_page = (start_offset + src_end) &
3822 ((unsigned long)PAGE_CACHE_SIZE - 1);
3823
3824 cur = min_t(unsigned long, len, src_off_in_page + 1);
3825 cur = min(cur, dst_off_in_page + 1);
3826 move_pages(extent_buffer_page(dst, dst_i),
3827 extent_buffer_page(dst, src_i),
3828 dst_off_in_page - cur + 1,
3829 src_off_in_page - cur + 1, cur);
3830
3831 dst_end -= cur;
3832 src_end -= cur;
3833 len -= cur;
3834 }
3835}
Chris Mason6af118ce2008-07-22 11:18:07 -04003836
Miao Xie19fe0a82010-10-26 20:57:29 -04003837static inline void btrfs_release_extent_buffer_rcu(struct rcu_head *head)
3838{
3839 struct extent_buffer *eb =
3840 container_of(head, struct extent_buffer, rcu_head);
3841
3842 btrfs_release_extent_buffer(eb);
3843}
3844
Chris Mason6af118ce2008-07-22 11:18:07 -04003845int try_release_extent_buffer(struct extent_io_tree *tree, struct page *page)
3846{
3847 u64 start = page_offset(page);
3848 struct extent_buffer *eb;
3849 int ret = 1;
Chris Mason6af118ce2008-07-22 11:18:07 -04003850
3851 spin_lock(&tree->buffer_lock);
Miao Xie19fe0a82010-10-26 20:57:29 -04003852 eb = radix_tree_lookup(&tree->buffer, start >> PAGE_CACHE_SHIFT);
Chris Mason45f49bc2010-11-21 22:27:44 -05003853 if (!eb) {
3854 spin_unlock(&tree->buffer_lock);
3855 return ret;
3856 }
Chris Mason6af118ce2008-07-22 11:18:07 -04003857
Chris Masonb9473432009-03-13 11:00:37 -04003858 if (test_bit(EXTENT_BUFFER_DIRTY, &eb->bflags)) {
3859 ret = 0;
3860 goto out;
3861 }
Miao Xie897ca6e2010-10-26 20:57:29 -04003862
Miao Xie19fe0a82010-10-26 20:57:29 -04003863 /*
3864 * set @eb->refs to 0 if it is already 1, and then release the @eb.
3865 * Or go back.
3866 */
3867 if (atomic_cmpxchg(&eb->refs, 1, 0) != 1) {
3868 ret = 0;
3869 goto out;
3870 }
3871
3872 radix_tree_delete(&tree->buffer, start >> PAGE_CACHE_SHIFT);
Chris Mason6af118ce2008-07-22 11:18:07 -04003873out:
3874 spin_unlock(&tree->buffer_lock);
Miao Xie19fe0a82010-10-26 20:57:29 -04003875
3876 /* at this point we can safely release the extent buffer */
3877 if (atomic_read(&eb->refs) == 0)
3878 call_rcu(&eb->rcu_head, btrfs_release_extent_buffer_rcu);
Chris Mason6af118ce2008-07-22 11:18:07 -04003879 return ret;
3880}