blob: 44ee5d2e52a41935828ac954ebeefe8ce17568e6 [file] [log] [blame]
Chris Mason6cbd5572007-06-12 09:07:21 -04001/*
Chris Masond352ac62008-09-29 15:18:18 -04002 * Copyright (C) 2007,2008 Oracle. All rights reserved.
Chris Mason6cbd5572007-06-12 09:07:21 -04003 *
4 * This program is free software; you can redistribute it and/or
5 * modify it under the terms of the GNU General Public
6 * License v2 as published by the Free Software Foundation.
7 *
8 * This program is distributed in the hope that it will be useful,
9 * but WITHOUT ANY WARRANTY; without even the implied warranty of
10 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
11 * General Public License for more details.
12 *
13 * You should have received a copy of the GNU General Public
14 * License along with this program; if not, write to the
15 * Free Software Foundation, Inc., 59 Temple Place - Suite 330,
16 * Boston, MA 021110-1307, USA.
17 */
18
Chris Masona6b6e752007-10-15 16:22:39 -040019#include <linux/sched.h>
Tejun Heo5a0e3ad2010-03-24 17:04:11 +090020#include <linux/slab.h>
Jan Schmidtbd989ba2012-05-16 17:18:50 +020021#include <linux/rbtree.h>
Chris Masoneb60cea2007-02-02 09:18:22 -050022#include "ctree.h"
23#include "disk-io.h"
Chris Mason7f5c1512007-03-23 15:56:19 -040024#include "transaction.h"
Chris Mason5f39d392007-10-15 16:14:19 -040025#include "print-tree.h"
Chris Mason925baed2008-06-25 16:01:30 -040026#include "locking.h"
Chris Mason9a8dd152007-02-23 08:38:36 -050027
Chris Masone089f052007-03-16 16:20:31 -040028static int split_node(struct btrfs_trans_handle *trans, struct btrfs_root
29 *root, struct btrfs_path *path, int level);
30static int split_leaf(struct btrfs_trans_handle *trans, struct btrfs_root
Chris Masond4dbff92007-04-04 14:08:15 -040031 *root, struct btrfs_key *ins_key,
Chris Masoncc0c5532007-10-25 15:42:57 -040032 struct btrfs_path *path, int data_size, int extend);
Chris Mason5f39d392007-10-15 16:14:19 -040033static int push_node_left(struct btrfs_trans_handle *trans,
34 struct btrfs_root *root, struct extent_buffer *dst,
Chris Mason971a1f62008-04-24 10:54:32 -040035 struct extent_buffer *src, int empty);
Chris Mason5f39d392007-10-15 16:14:19 -040036static int balance_node_right(struct btrfs_trans_handle *trans,
37 struct btrfs_root *root,
38 struct extent_buffer *dst_buf,
39 struct extent_buffer *src_buf);
Tsutomu Itohafe5fea2013-04-16 05:18:22 +000040static void del_ptr(struct btrfs_root *root, struct btrfs_path *path,
41 int level, int slot);
Filipe David Borba Manana5de865e2013-12-20 15:17:46 +000042static int tree_mod_log_free_eb(struct btrfs_fs_info *fs_info,
Jan Schmidtf2304752012-05-26 11:43:17 +020043 struct extent_buffer *eb);
Chris Masond97e63b2007-02-20 16:40:44 -050044
Chris Mason2c90e5d2007-04-02 10:50:19 -040045struct btrfs_path *btrfs_alloc_path(void)
46{
Chris Masondf24a2b2007-04-04 09:36:31 -040047 struct btrfs_path *path;
Jeff Mahoneye00f7302009-02-12 14:11:25 -050048 path = kmem_cache_zalloc(btrfs_path_cachep, GFP_NOFS);
Chris Masondf24a2b2007-04-04 09:36:31 -040049 return path;
Chris Mason2c90e5d2007-04-02 10:50:19 -040050}
51
Chris Masonb4ce94d2009-02-04 09:25:08 -050052/*
53 * set all locked nodes in the path to blocking locks. This should
54 * be done before scheduling
55 */
56noinline void btrfs_set_path_blocking(struct btrfs_path *p)
57{
58 int i;
59 for (i = 0; i < BTRFS_MAX_LEVEL; i++) {
Chris Masonbd681512011-07-16 15:23:14 -040060 if (!p->nodes[i] || !p->locks[i])
61 continue;
62 btrfs_set_lock_blocking_rw(p->nodes[i], p->locks[i]);
63 if (p->locks[i] == BTRFS_READ_LOCK)
64 p->locks[i] = BTRFS_READ_LOCK_BLOCKING;
65 else if (p->locks[i] == BTRFS_WRITE_LOCK)
66 p->locks[i] = BTRFS_WRITE_LOCK_BLOCKING;
Chris Masonb4ce94d2009-02-04 09:25:08 -050067 }
68}
69
70/*
71 * reset all the locked nodes in the patch to spinning locks.
Chris Mason4008c042009-02-12 14:09:45 -050072 *
73 * held is used to keep lockdep happy, when lockdep is enabled
74 * we set held to a blocking lock before we go around and
75 * retake all the spinlocks in the path. You can safely use NULL
76 * for held
Chris Masonb4ce94d2009-02-04 09:25:08 -050077 */
Chris Mason4008c042009-02-12 14:09:45 -050078noinline void btrfs_clear_path_blocking(struct btrfs_path *p,
Chris Masonbd681512011-07-16 15:23:14 -040079 struct extent_buffer *held, int held_rw)
Chris Masonb4ce94d2009-02-04 09:25:08 -050080{
81 int i;
Chris Mason4008c042009-02-12 14:09:45 -050082
83#ifdef CONFIG_DEBUG_LOCK_ALLOC
84 /* lockdep really cares that we take all of these spinlocks
85 * in the right order. If any of the locks in the path are not
86 * currently blocking, it is going to complain. So, make really
87 * really sure by forcing the path to blocking before we clear
88 * the path blocking.
89 */
Chris Masonbd681512011-07-16 15:23:14 -040090 if (held) {
91 btrfs_set_lock_blocking_rw(held, held_rw);
92 if (held_rw == BTRFS_WRITE_LOCK)
93 held_rw = BTRFS_WRITE_LOCK_BLOCKING;
94 else if (held_rw == BTRFS_READ_LOCK)
95 held_rw = BTRFS_READ_LOCK_BLOCKING;
96 }
Chris Mason4008c042009-02-12 14:09:45 -050097 btrfs_set_path_blocking(p);
98#endif
99
100 for (i = BTRFS_MAX_LEVEL - 1; i >= 0; i--) {
Chris Masonbd681512011-07-16 15:23:14 -0400101 if (p->nodes[i] && p->locks[i]) {
102 btrfs_clear_lock_blocking_rw(p->nodes[i], p->locks[i]);
103 if (p->locks[i] == BTRFS_WRITE_LOCK_BLOCKING)
104 p->locks[i] = BTRFS_WRITE_LOCK;
105 else if (p->locks[i] == BTRFS_READ_LOCK_BLOCKING)
106 p->locks[i] = BTRFS_READ_LOCK;
107 }
Chris Masonb4ce94d2009-02-04 09:25:08 -0500108 }
Chris Mason4008c042009-02-12 14:09:45 -0500109
110#ifdef CONFIG_DEBUG_LOCK_ALLOC
111 if (held)
Chris Masonbd681512011-07-16 15:23:14 -0400112 btrfs_clear_lock_blocking_rw(held, held_rw);
Chris Mason4008c042009-02-12 14:09:45 -0500113#endif
Chris Masonb4ce94d2009-02-04 09:25:08 -0500114}
115
Chris Masond352ac62008-09-29 15:18:18 -0400116/* this also releases the path */
Chris Mason2c90e5d2007-04-02 10:50:19 -0400117void btrfs_free_path(struct btrfs_path *p)
118{
Jesper Juhlff175d52010-12-25 21:22:30 +0000119 if (!p)
120 return;
David Sterbab3b4aa72011-04-21 01:20:15 +0200121 btrfs_release_path(p);
Chris Mason2c90e5d2007-04-02 10:50:19 -0400122 kmem_cache_free(btrfs_path_cachep, p);
123}
124
Chris Masond352ac62008-09-29 15:18:18 -0400125/*
126 * path release drops references on the extent buffers in the path
127 * and it drops any locks held by this path
128 *
129 * It is safe to call this on paths that no locks or extent buffers held.
130 */
David Sterbab3b4aa72011-04-21 01:20:15 +0200131noinline void btrfs_release_path(struct btrfs_path *p)
Chris Masoneb60cea2007-02-02 09:18:22 -0500132{
133 int i;
Chris Masona2135012008-06-25 16:01:30 -0400134
Chris Mason234b63a2007-03-13 10:46:10 -0400135 for (i = 0; i < BTRFS_MAX_LEVEL; i++) {
Chris Mason3f157a22008-06-25 16:01:31 -0400136 p->slots[i] = 0;
Chris Masoneb60cea2007-02-02 09:18:22 -0500137 if (!p->nodes[i])
Chris Mason925baed2008-06-25 16:01:30 -0400138 continue;
139 if (p->locks[i]) {
Chris Masonbd681512011-07-16 15:23:14 -0400140 btrfs_tree_unlock_rw(p->nodes[i], p->locks[i]);
Chris Mason925baed2008-06-25 16:01:30 -0400141 p->locks[i] = 0;
142 }
Chris Mason5f39d392007-10-15 16:14:19 -0400143 free_extent_buffer(p->nodes[i]);
Chris Mason3f157a22008-06-25 16:01:31 -0400144 p->nodes[i] = NULL;
Chris Masoneb60cea2007-02-02 09:18:22 -0500145 }
146}
147
Chris Masond352ac62008-09-29 15:18:18 -0400148/*
149 * safely gets a reference on the root node of a tree. A lock
150 * is not taken, so a concurrent writer may put a different node
151 * at the root of the tree. See btrfs_lock_root_node for the
152 * looping required.
153 *
154 * The extent buffer returned by this has a reference taken, so
155 * it won't disappear. It may stop being the root of the tree
156 * at any time because there are no locks held.
157 */
Chris Mason925baed2008-06-25 16:01:30 -0400158struct extent_buffer *btrfs_root_node(struct btrfs_root *root)
159{
160 struct extent_buffer *eb;
Chris Mason240f62c2011-03-23 14:54:42 -0400161
Josef Bacik3083ee22012-03-09 16:01:49 -0500162 while (1) {
163 rcu_read_lock();
164 eb = rcu_dereference(root->node);
165
166 /*
167 * RCU really hurts here, we could free up the root node because
168 * it was cow'ed but we may not get the new root node yet so do
169 * the inc_not_zero dance and if it doesn't work then
170 * synchronize_rcu and try again.
171 */
172 if (atomic_inc_not_zero(&eb->refs)) {
173 rcu_read_unlock();
174 break;
175 }
176 rcu_read_unlock();
177 synchronize_rcu();
178 }
Chris Mason925baed2008-06-25 16:01:30 -0400179 return eb;
180}
181
Chris Masond352ac62008-09-29 15:18:18 -0400182/* loop around taking references on and locking the root node of the
183 * tree until you end up with a lock on the root. A locked buffer
184 * is returned, with a reference held.
185 */
Chris Mason925baed2008-06-25 16:01:30 -0400186struct extent_buffer *btrfs_lock_root_node(struct btrfs_root *root)
187{
188 struct extent_buffer *eb;
189
Chris Masond3977122009-01-05 21:25:51 -0500190 while (1) {
Chris Mason925baed2008-06-25 16:01:30 -0400191 eb = btrfs_root_node(root);
192 btrfs_tree_lock(eb);
Chris Mason240f62c2011-03-23 14:54:42 -0400193 if (eb == root->node)
Chris Mason925baed2008-06-25 16:01:30 -0400194 break;
Chris Mason925baed2008-06-25 16:01:30 -0400195 btrfs_tree_unlock(eb);
196 free_extent_buffer(eb);
197 }
198 return eb;
199}
200
Chris Masonbd681512011-07-16 15:23:14 -0400201/* loop around taking references on and locking the root node of the
202 * tree until you end up with a lock on the root. A locked buffer
203 * is returned, with a reference held.
204 */
Eric Sandeen48a3b632013-04-25 20:41:01 +0000205static struct extent_buffer *btrfs_read_lock_root_node(struct btrfs_root *root)
Chris Masonbd681512011-07-16 15:23:14 -0400206{
207 struct extent_buffer *eb;
208
209 while (1) {
210 eb = btrfs_root_node(root);
211 btrfs_tree_read_lock(eb);
212 if (eb == root->node)
213 break;
214 btrfs_tree_read_unlock(eb);
215 free_extent_buffer(eb);
216 }
217 return eb;
218}
219
Chris Masond352ac62008-09-29 15:18:18 -0400220/* cowonly root (everything not a reference counted cow subvolume), just get
221 * put onto a simple dirty list. transaction.c walks this to make sure they
222 * get properly updated on disk.
223 */
Chris Mason0b86a832008-03-24 15:01:56 -0400224static void add_root_to_dirty_list(struct btrfs_root *root)
225{
Chris Masone5846fc2012-05-03 12:08:48 -0400226 spin_lock(&root->fs_info->trans_lock);
Miao Xie27cdeb72014-04-02 19:51:05 +0800227 if (test_bit(BTRFS_ROOT_TRACK_DIRTY, &root->state) &&
228 list_empty(&root->dirty_list)) {
Chris Mason0b86a832008-03-24 15:01:56 -0400229 list_add(&root->dirty_list,
230 &root->fs_info->dirty_cowonly_roots);
231 }
Chris Masone5846fc2012-05-03 12:08:48 -0400232 spin_unlock(&root->fs_info->trans_lock);
Chris Mason0b86a832008-03-24 15:01:56 -0400233}
234
Chris Masond352ac62008-09-29 15:18:18 -0400235/*
236 * used by snapshot creation to make a copy of a root for a tree with
237 * a given objectid. The buffer with the new root node is returned in
238 * cow_ret, and this func returns zero on success or a negative error code.
239 */
Chris Masonbe20aa92007-12-17 20:14:01 -0500240int btrfs_copy_root(struct btrfs_trans_handle *trans,
241 struct btrfs_root *root,
242 struct extent_buffer *buf,
243 struct extent_buffer **cow_ret, u64 new_root_objectid)
244{
245 struct extent_buffer *cow;
Chris Masonbe20aa92007-12-17 20:14:01 -0500246 int ret = 0;
247 int level;
Yan Zheng5d4f98a2009-06-10 10:45:14 -0400248 struct btrfs_disk_key disk_key;
Chris Masonbe20aa92007-12-17 20:14:01 -0500249
Miao Xie27cdeb72014-04-02 19:51:05 +0800250 WARN_ON(test_bit(BTRFS_ROOT_REF_COWS, &root->state) &&
251 trans->transid != root->fs_info->running_transaction->transid);
252 WARN_ON(test_bit(BTRFS_ROOT_REF_COWS, &root->state) &&
253 trans->transid != root->last_trans);
Chris Masonbe20aa92007-12-17 20:14:01 -0500254
255 level = btrfs_header_level(buf);
Yan Zheng5d4f98a2009-06-10 10:45:14 -0400256 if (level == 0)
257 btrfs_item_key(buf, &disk_key, 0);
258 else
259 btrfs_node_key(buf, &disk_key, 0);
Zheng Yan31840ae2008-09-23 13:14:14 -0400260
Yan Zheng5d4f98a2009-06-10 10:45:14 -0400261 cow = btrfs_alloc_free_block(trans, root, buf->len, 0,
262 new_root_objectid, &disk_key, level,
Jan Schmidt5581a512012-05-16 17:04:52 +0200263 buf->start, 0);
Yan Zheng5d4f98a2009-06-10 10:45:14 -0400264 if (IS_ERR(cow))
Chris Masonbe20aa92007-12-17 20:14:01 -0500265 return PTR_ERR(cow);
266
267 copy_extent_buffer(cow, buf, 0, 0, cow->len);
268 btrfs_set_header_bytenr(cow, cow->start);
269 btrfs_set_header_generation(cow, trans->transid);
Yan Zheng5d4f98a2009-06-10 10:45:14 -0400270 btrfs_set_header_backref_rev(cow, BTRFS_MIXED_BACKREF_REV);
271 btrfs_clear_header_flag(cow, BTRFS_HEADER_FLAG_WRITTEN |
272 BTRFS_HEADER_FLAG_RELOC);
273 if (new_root_objectid == BTRFS_TREE_RELOC_OBJECTID)
274 btrfs_set_header_flag(cow, BTRFS_HEADER_FLAG_RELOC);
275 else
276 btrfs_set_header_owner(cow, new_root_objectid);
Chris Masonbe20aa92007-12-17 20:14:01 -0500277
Ross Kirk0a4e5582013-09-24 10:12:38 +0100278 write_extent_buffer(cow, root->fs_info->fsid, btrfs_header_fsid(),
Yan Zheng2b820322008-11-17 21:11:30 -0500279 BTRFS_FSID_SIZE);
280
Chris Masonbe20aa92007-12-17 20:14:01 -0500281 WARN_ON(btrfs_header_generation(buf) > trans->transid);
Yan Zheng5d4f98a2009-06-10 10:45:14 -0400282 if (new_root_objectid == BTRFS_TREE_RELOC_OBJECTID)
Josef Bacike339a6b2014-07-02 10:54:25 -0700283 ret = btrfs_inc_ref(trans, root, cow, 1);
Yan Zheng5d4f98a2009-06-10 10:45:14 -0400284 else
Josef Bacike339a6b2014-07-02 10:54:25 -0700285 ret = btrfs_inc_ref(trans, root, cow, 0);
Chris Mason4aec2b52007-12-18 16:25:45 -0500286
Chris Masonbe20aa92007-12-17 20:14:01 -0500287 if (ret)
288 return ret;
289
290 btrfs_mark_buffer_dirty(cow);
291 *cow_ret = cow;
292 return 0;
293}
294
Jan Schmidtbd989ba2012-05-16 17:18:50 +0200295enum mod_log_op {
296 MOD_LOG_KEY_REPLACE,
297 MOD_LOG_KEY_ADD,
298 MOD_LOG_KEY_REMOVE,
299 MOD_LOG_KEY_REMOVE_WHILE_FREEING,
300 MOD_LOG_KEY_REMOVE_WHILE_MOVING,
301 MOD_LOG_MOVE_KEYS,
302 MOD_LOG_ROOT_REPLACE,
303};
304
305struct tree_mod_move {
306 int dst_slot;
307 int nr_items;
308};
309
310struct tree_mod_root {
311 u64 logical;
312 u8 level;
313};
314
315struct tree_mod_elem {
316 struct rb_node node;
317 u64 index; /* shifted logical */
Jan Schmidt097b8a72012-06-21 11:08:04 +0200318 u64 seq;
Jan Schmidtbd989ba2012-05-16 17:18:50 +0200319 enum mod_log_op op;
320
321 /* this is used for MOD_LOG_KEY_* and MOD_LOG_MOVE_KEYS operations */
322 int slot;
323
324 /* this is used for MOD_LOG_KEY* and MOD_LOG_ROOT_REPLACE */
325 u64 generation;
326
327 /* those are used for op == MOD_LOG_KEY_{REPLACE,REMOVE} */
328 struct btrfs_disk_key key;
329 u64 blockptr;
330
331 /* this is used for op == MOD_LOG_MOVE_KEYS */
332 struct tree_mod_move move;
333
334 /* this is used for op == MOD_LOG_ROOT_REPLACE */
335 struct tree_mod_root old_root;
336};
337
Jan Schmidt097b8a72012-06-21 11:08:04 +0200338static inline void tree_mod_log_read_lock(struct btrfs_fs_info *fs_info)
Jan Schmidtbd989ba2012-05-16 17:18:50 +0200339{
Jan Schmidt097b8a72012-06-21 11:08:04 +0200340 read_lock(&fs_info->tree_mod_log_lock);
Jan Schmidtbd989ba2012-05-16 17:18:50 +0200341}
342
Jan Schmidt097b8a72012-06-21 11:08:04 +0200343static inline void tree_mod_log_read_unlock(struct btrfs_fs_info *fs_info)
Jan Schmidtbd989ba2012-05-16 17:18:50 +0200344{
Jan Schmidt097b8a72012-06-21 11:08:04 +0200345 read_unlock(&fs_info->tree_mod_log_lock);
346}
347
348static inline void tree_mod_log_write_lock(struct btrfs_fs_info *fs_info)
349{
350 write_lock(&fs_info->tree_mod_log_lock);
351}
352
353static inline void tree_mod_log_write_unlock(struct btrfs_fs_info *fs_info)
354{
355 write_unlock(&fs_info->tree_mod_log_lock);
356}
357
358/*
Josef Bacikfcebe452014-05-13 17:30:47 -0700359 * Pull a new tree mod seq number for our operation.
Jan Schmidtfc36ed7e2013-04-24 16:57:33 +0000360 */
Josef Bacikfcebe452014-05-13 17:30:47 -0700361static inline u64 btrfs_inc_tree_mod_seq(struct btrfs_fs_info *fs_info)
Jan Schmidtfc36ed7e2013-04-24 16:57:33 +0000362{
363 return atomic64_inc_return(&fs_info->tree_mod_seq);
364}
365
366/*
Jan Schmidt097b8a72012-06-21 11:08:04 +0200367 * This adds a new blocker to the tree mod log's blocker list if the @elem
368 * passed does not already have a sequence number set. So when a caller expects
369 * to record tree modifications, it should ensure to set elem->seq to zero
370 * before calling btrfs_get_tree_mod_seq.
371 * Returns a fresh, unused tree log modification sequence number, even if no new
372 * blocker was added.
373 */
374u64 btrfs_get_tree_mod_seq(struct btrfs_fs_info *fs_info,
375 struct seq_list *elem)
376{
Jan Schmidt097b8a72012-06-21 11:08:04 +0200377 tree_mod_log_write_lock(fs_info);
Jan Schmidtbd989ba2012-05-16 17:18:50 +0200378 spin_lock(&fs_info->tree_mod_seq_lock);
Jan Schmidt097b8a72012-06-21 11:08:04 +0200379 if (!elem->seq) {
Josef Bacikfcebe452014-05-13 17:30:47 -0700380 elem->seq = btrfs_inc_tree_mod_seq(fs_info);
Jan Schmidt097b8a72012-06-21 11:08:04 +0200381 list_add_tail(&elem->list, &fs_info->tree_mod_seq_list);
382 }
Jan Schmidtbd989ba2012-05-16 17:18:50 +0200383 spin_unlock(&fs_info->tree_mod_seq_lock);
Jan Schmidt097b8a72012-06-21 11:08:04 +0200384 tree_mod_log_write_unlock(fs_info);
385
Josef Bacikfcebe452014-05-13 17:30:47 -0700386 return elem->seq;
Jan Schmidtbd989ba2012-05-16 17:18:50 +0200387}
388
389void btrfs_put_tree_mod_seq(struct btrfs_fs_info *fs_info,
390 struct seq_list *elem)
391{
392 struct rb_root *tm_root;
393 struct rb_node *node;
394 struct rb_node *next;
395 struct seq_list *cur_elem;
396 struct tree_mod_elem *tm;
397 u64 min_seq = (u64)-1;
398 u64 seq_putting = elem->seq;
399
400 if (!seq_putting)
401 return;
402
Jan Schmidtbd989ba2012-05-16 17:18:50 +0200403 spin_lock(&fs_info->tree_mod_seq_lock);
404 list_del(&elem->list);
Jan Schmidt097b8a72012-06-21 11:08:04 +0200405 elem->seq = 0;
Jan Schmidtbd989ba2012-05-16 17:18:50 +0200406
407 list_for_each_entry(cur_elem, &fs_info->tree_mod_seq_list, list) {
Jan Schmidt097b8a72012-06-21 11:08:04 +0200408 if (cur_elem->seq < min_seq) {
Jan Schmidtbd989ba2012-05-16 17:18:50 +0200409 if (seq_putting > cur_elem->seq) {
410 /*
411 * blocker with lower sequence number exists, we
412 * cannot remove anything from the log
413 */
Jan Schmidt097b8a72012-06-21 11:08:04 +0200414 spin_unlock(&fs_info->tree_mod_seq_lock);
415 return;
Jan Schmidtbd989ba2012-05-16 17:18:50 +0200416 }
417 min_seq = cur_elem->seq;
418 }
419 }
Jan Schmidt097b8a72012-06-21 11:08:04 +0200420 spin_unlock(&fs_info->tree_mod_seq_lock);
421
422 /*
Jan Schmidtbd989ba2012-05-16 17:18:50 +0200423 * anything that's lower than the lowest existing (read: blocked)
424 * sequence number can be removed from the tree.
425 */
Jan Schmidt097b8a72012-06-21 11:08:04 +0200426 tree_mod_log_write_lock(fs_info);
Jan Schmidtbd989ba2012-05-16 17:18:50 +0200427 tm_root = &fs_info->tree_mod_log;
428 for (node = rb_first(tm_root); node; node = next) {
429 next = rb_next(node);
430 tm = container_of(node, struct tree_mod_elem, node);
Jan Schmidt097b8a72012-06-21 11:08:04 +0200431 if (tm->seq > min_seq)
Jan Schmidtbd989ba2012-05-16 17:18:50 +0200432 continue;
433 rb_erase(node, tm_root);
Jan Schmidtbd989ba2012-05-16 17:18:50 +0200434 kfree(tm);
435 }
Jan Schmidt097b8a72012-06-21 11:08:04 +0200436 tree_mod_log_write_unlock(fs_info);
Jan Schmidtbd989ba2012-05-16 17:18:50 +0200437}
438
439/*
440 * key order of the log:
441 * index -> sequence
442 *
443 * the index is the shifted logical of the *new* root node for root replace
444 * operations, or the shifted logical of the affected block for all other
445 * operations.
Filipe David Borba Manana5de865e2013-12-20 15:17:46 +0000446 *
447 * Note: must be called with write lock (tree_mod_log_write_lock).
Jan Schmidtbd989ba2012-05-16 17:18:50 +0200448 */
449static noinline int
450__tree_mod_log_insert(struct btrfs_fs_info *fs_info, struct tree_mod_elem *tm)
451{
452 struct rb_root *tm_root;
453 struct rb_node **new;
454 struct rb_node *parent = NULL;
455 struct tree_mod_elem *cur;
Jan Schmidtbd989ba2012-05-16 17:18:50 +0200456
Josef Bacikc8cc6342013-07-01 16:18:19 -0400457 BUG_ON(!tm);
458
Josef Bacikfcebe452014-05-13 17:30:47 -0700459 tm->seq = btrfs_inc_tree_mod_seq(fs_info);
Jan Schmidtbd989ba2012-05-16 17:18:50 +0200460
Jan Schmidtbd989ba2012-05-16 17:18:50 +0200461 tm_root = &fs_info->tree_mod_log;
462 new = &tm_root->rb_node;
463 while (*new) {
464 cur = container_of(*new, struct tree_mod_elem, node);
465 parent = *new;
466 if (cur->index < tm->index)
467 new = &((*new)->rb_left);
468 else if (cur->index > tm->index)
469 new = &((*new)->rb_right);
Jan Schmidt097b8a72012-06-21 11:08:04 +0200470 else if (cur->seq < tm->seq)
Jan Schmidtbd989ba2012-05-16 17:18:50 +0200471 new = &((*new)->rb_left);
Jan Schmidt097b8a72012-06-21 11:08:04 +0200472 else if (cur->seq > tm->seq)
Jan Schmidtbd989ba2012-05-16 17:18:50 +0200473 new = &((*new)->rb_right);
Filipe David Borba Manana5de865e2013-12-20 15:17:46 +0000474 else
475 return -EEXIST;
Jan Schmidtbd989ba2012-05-16 17:18:50 +0200476 }
477
478 rb_link_node(&tm->node, parent, new);
479 rb_insert_color(&tm->node, tm_root);
Filipe David Borba Manana5de865e2013-12-20 15:17:46 +0000480 return 0;
Jan Schmidtbd989ba2012-05-16 17:18:50 +0200481}
482
Jan Schmidt097b8a72012-06-21 11:08:04 +0200483/*
484 * Determines if logging can be omitted. Returns 1 if it can. Otherwise, it
485 * returns zero with the tree_mod_log_lock acquired. The caller must hold
486 * this until all tree mod log insertions are recorded in the rb tree and then
487 * call tree_mod_log_write_unlock() to release.
488 */
Jan Schmidte9b7fd42012-05-31 14:59:09 +0200489static inline int tree_mod_dont_log(struct btrfs_fs_info *fs_info,
490 struct extent_buffer *eb) {
491 smp_mb();
492 if (list_empty(&(fs_info)->tree_mod_seq_list))
493 return 1;
Jan Schmidt097b8a72012-06-21 11:08:04 +0200494 if (eb && btrfs_header_level(eb) == 0)
Jan Schmidte9b7fd42012-05-31 14:59:09 +0200495 return 1;
Filipe David Borba Manana5de865e2013-12-20 15:17:46 +0000496
497 tree_mod_log_write_lock(fs_info);
498 if (list_empty(&(fs_info)->tree_mod_seq_list)) {
499 tree_mod_log_write_unlock(fs_info);
500 return 1;
501 }
502
Jan Schmidte9b7fd42012-05-31 14:59:09 +0200503 return 0;
504}
505
Filipe David Borba Manana5de865e2013-12-20 15:17:46 +0000506/* Similar to tree_mod_dont_log, but doesn't acquire any locks. */
507static inline int tree_mod_need_log(const struct btrfs_fs_info *fs_info,
508 struct extent_buffer *eb)
509{
510 smp_mb();
511 if (list_empty(&(fs_info)->tree_mod_seq_list))
512 return 0;
513 if (eb && btrfs_header_level(eb) == 0)
514 return 0;
515
516 return 1;
517}
518
519static struct tree_mod_elem *
520alloc_tree_mod_elem(struct extent_buffer *eb, int slot,
521 enum mod_log_op op, gfp_t flags)
Jan Schmidtbd989ba2012-05-16 17:18:50 +0200522{
Jan Schmidt097b8a72012-06-21 11:08:04 +0200523 struct tree_mod_elem *tm;
Jan Schmidtbd989ba2012-05-16 17:18:50 +0200524
Josef Bacikc8cc6342013-07-01 16:18:19 -0400525 tm = kzalloc(sizeof(*tm), flags);
526 if (!tm)
Filipe David Borba Manana5de865e2013-12-20 15:17:46 +0000527 return NULL;
Jan Schmidtbd989ba2012-05-16 17:18:50 +0200528
529 tm->index = eb->start >> PAGE_CACHE_SHIFT;
530 if (op != MOD_LOG_KEY_ADD) {
531 btrfs_node_key(eb, &tm->key, slot);
532 tm->blockptr = btrfs_node_blockptr(eb, slot);
533 }
534 tm->op = op;
535 tm->slot = slot;
536 tm->generation = btrfs_node_ptr_generation(eb, slot);
Filipe David Borba Manana5de865e2013-12-20 15:17:46 +0000537 RB_CLEAR_NODE(&tm->node);
Jan Schmidtbd989ba2012-05-16 17:18:50 +0200538
Filipe David Borba Manana5de865e2013-12-20 15:17:46 +0000539 return tm;
Jan Schmidt097b8a72012-06-21 11:08:04 +0200540}
541
542static noinline int
Josef Bacikc8cc6342013-07-01 16:18:19 -0400543tree_mod_log_insert_key(struct btrfs_fs_info *fs_info,
544 struct extent_buffer *eb, int slot,
545 enum mod_log_op op, gfp_t flags)
Jan Schmidt097b8a72012-06-21 11:08:04 +0200546{
Filipe David Borba Manana5de865e2013-12-20 15:17:46 +0000547 struct tree_mod_elem *tm;
548 int ret;
549
550 if (!tree_mod_need_log(fs_info, eb))
Jan Schmidt097b8a72012-06-21 11:08:04 +0200551 return 0;
552
Filipe David Borba Manana5de865e2013-12-20 15:17:46 +0000553 tm = alloc_tree_mod_elem(eb, slot, op, flags);
554 if (!tm)
555 return -ENOMEM;
556
557 if (tree_mod_dont_log(fs_info, eb)) {
558 kfree(tm);
559 return 0;
560 }
561
562 ret = __tree_mod_log_insert(fs_info, tm);
563 tree_mod_log_write_unlock(fs_info);
564 if (ret)
565 kfree(tm);
566
567 return ret;
Jan Schmidt097b8a72012-06-21 11:08:04 +0200568}
569
570static noinline int
Jan Schmidtbd989ba2012-05-16 17:18:50 +0200571tree_mod_log_insert_move(struct btrfs_fs_info *fs_info,
572 struct extent_buffer *eb, int dst_slot, int src_slot,
573 int nr_items, gfp_t flags)
574{
Filipe David Borba Manana5de865e2013-12-20 15:17:46 +0000575 struct tree_mod_elem *tm = NULL;
576 struct tree_mod_elem **tm_list = NULL;
577 int ret = 0;
Jan Schmidtbd989ba2012-05-16 17:18:50 +0200578 int i;
Filipe David Borba Manana5de865e2013-12-20 15:17:46 +0000579 int locked = 0;
Jan Schmidtbd989ba2012-05-16 17:18:50 +0200580
Filipe David Borba Manana5de865e2013-12-20 15:17:46 +0000581 if (!tree_mod_need_log(fs_info, eb))
Jan Schmidtf3956942012-05-31 15:02:32 +0200582 return 0;
Jan Schmidtbd989ba2012-05-16 17:18:50 +0200583
Filipe David Borba Manana5de865e2013-12-20 15:17:46 +0000584 tm_list = kzalloc(nr_items * sizeof(struct tree_mod_elem *), flags);
585 if (!tm_list)
586 return -ENOMEM;
Jan Schmidtbd989ba2012-05-16 17:18:50 +0200587
Josef Bacikc8cc6342013-07-01 16:18:19 -0400588 tm = kzalloc(sizeof(*tm), flags);
Filipe David Borba Manana5de865e2013-12-20 15:17:46 +0000589 if (!tm) {
590 ret = -ENOMEM;
591 goto free_tms;
592 }
Jan Schmidtf3956942012-05-31 15:02:32 +0200593
Jan Schmidtbd989ba2012-05-16 17:18:50 +0200594 tm->index = eb->start >> PAGE_CACHE_SHIFT;
595 tm->slot = src_slot;
596 tm->move.dst_slot = dst_slot;
597 tm->move.nr_items = nr_items;
598 tm->op = MOD_LOG_MOVE_KEYS;
599
Filipe David Borba Manana5de865e2013-12-20 15:17:46 +0000600 for (i = 0; i + dst_slot < src_slot && i < nr_items; i++) {
601 tm_list[i] = alloc_tree_mod_elem(eb, i + dst_slot,
602 MOD_LOG_KEY_REMOVE_WHILE_MOVING, flags);
603 if (!tm_list[i]) {
604 ret = -ENOMEM;
605 goto free_tms;
606 }
607 }
608
609 if (tree_mod_dont_log(fs_info, eb))
610 goto free_tms;
611 locked = 1;
612
613 /*
614 * When we override something during the move, we log these removals.
615 * This can only happen when we move towards the beginning of the
616 * buffer, i.e. dst_slot < src_slot.
617 */
618 for (i = 0; i + dst_slot < src_slot && i < nr_items; i++) {
619 ret = __tree_mod_log_insert(fs_info, tm_list[i]);
620 if (ret)
621 goto free_tms;
622 }
623
624 ret = __tree_mod_log_insert(fs_info, tm);
625 if (ret)
626 goto free_tms;
627 tree_mod_log_write_unlock(fs_info);
628 kfree(tm_list);
629
630 return 0;
631free_tms:
632 for (i = 0; i < nr_items; i++) {
633 if (tm_list[i] && !RB_EMPTY_NODE(&tm_list[i]->node))
634 rb_erase(&tm_list[i]->node, &fs_info->tree_mod_log);
635 kfree(tm_list[i]);
636 }
637 if (locked)
638 tree_mod_log_write_unlock(fs_info);
639 kfree(tm_list);
640 kfree(tm);
641
642 return ret;
Jan Schmidtbd989ba2012-05-16 17:18:50 +0200643}
644
Filipe David Borba Manana5de865e2013-12-20 15:17:46 +0000645static inline int
646__tree_mod_log_free_eb(struct btrfs_fs_info *fs_info,
647 struct tree_mod_elem **tm_list,
648 int nritems)
Jan Schmidt097b8a72012-06-21 11:08:04 +0200649{
Filipe David Borba Manana5de865e2013-12-20 15:17:46 +0000650 int i, j;
Jan Schmidt097b8a72012-06-21 11:08:04 +0200651 int ret;
652
Jan Schmidt097b8a72012-06-21 11:08:04 +0200653 for (i = nritems - 1; i >= 0; i--) {
Filipe David Borba Manana5de865e2013-12-20 15:17:46 +0000654 ret = __tree_mod_log_insert(fs_info, tm_list[i]);
655 if (ret) {
656 for (j = nritems - 1; j > i; j--)
657 rb_erase(&tm_list[j]->node,
658 &fs_info->tree_mod_log);
659 return ret;
660 }
Jan Schmidt097b8a72012-06-21 11:08:04 +0200661 }
Filipe David Borba Manana5de865e2013-12-20 15:17:46 +0000662
663 return 0;
Jan Schmidt097b8a72012-06-21 11:08:04 +0200664}
665
Jan Schmidtbd989ba2012-05-16 17:18:50 +0200666static noinline int
667tree_mod_log_insert_root(struct btrfs_fs_info *fs_info,
668 struct extent_buffer *old_root,
Jan Schmidt90f8d622013-04-13 13:19:53 +0000669 struct extent_buffer *new_root, gfp_t flags,
670 int log_removal)
Jan Schmidtbd989ba2012-05-16 17:18:50 +0200671{
Filipe David Borba Manana5de865e2013-12-20 15:17:46 +0000672 struct tree_mod_elem *tm = NULL;
673 struct tree_mod_elem **tm_list = NULL;
674 int nritems = 0;
675 int ret = 0;
676 int i;
Jan Schmidtbd989ba2012-05-16 17:18:50 +0200677
Filipe David Borba Manana5de865e2013-12-20 15:17:46 +0000678 if (!tree_mod_need_log(fs_info, NULL))
Jan Schmidt097b8a72012-06-21 11:08:04 +0200679 return 0;
680
Filipe David Borba Manana5de865e2013-12-20 15:17:46 +0000681 if (log_removal && btrfs_header_level(old_root) > 0) {
682 nritems = btrfs_header_nritems(old_root);
683 tm_list = kzalloc(nritems * sizeof(struct tree_mod_elem *),
684 flags);
685 if (!tm_list) {
686 ret = -ENOMEM;
687 goto free_tms;
688 }
689 for (i = 0; i < nritems; i++) {
690 tm_list[i] = alloc_tree_mod_elem(old_root, i,
691 MOD_LOG_KEY_REMOVE_WHILE_FREEING, flags);
692 if (!tm_list[i]) {
693 ret = -ENOMEM;
694 goto free_tms;
695 }
696 }
697 }
Jan Schmidtd9abbf12013-03-20 13:49:48 +0000698
Josef Bacikc8cc6342013-07-01 16:18:19 -0400699 tm = kzalloc(sizeof(*tm), flags);
Filipe David Borba Manana5de865e2013-12-20 15:17:46 +0000700 if (!tm) {
701 ret = -ENOMEM;
702 goto free_tms;
703 }
Jan Schmidtbd989ba2012-05-16 17:18:50 +0200704
705 tm->index = new_root->start >> PAGE_CACHE_SHIFT;
706 tm->old_root.logical = old_root->start;
707 tm->old_root.level = btrfs_header_level(old_root);
708 tm->generation = btrfs_header_generation(old_root);
709 tm->op = MOD_LOG_ROOT_REPLACE;
710
Filipe David Borba Manana5de865e2013-12-20 15:17:46 +0000711 if (tree_mod_dont_log(fs_info, NULL))
712 goto free_tms;
713
714 if (tm_list)
715 ret = __tree_mod_log_free_eb(fs_info, tm_list, nritems);
716 if (!ret)
717 ret = __tree_mod_log_insert(fs_info, tm);
718
719 tree_mod_log_write_unlock(fs_info);
720 if (ret)
721 goto free_tms;
722 kfree(tm_list);
723
724 return ret;
725
726free_tms:
727 if (tm_list) {
728 for (i = 0; i < nritems; i++)
729 kfree(tm_list[i]);
730 kfree(tm_list);
731 }
732 kfree(tm);
733
734 return ret;
Jan Schmidtbd989ba2012-05-16 17:18:50 +0200735}
736
737static struct tree_mod_elem *
738__tree_mod_log_search(struct btrfs_fs_info *fs_info, u64 start, u64 min_seq,
739 int smallest)
740{
741 struct rb_root *tm_root;
742 struct rb_node *node;
743 struct tree_mod_elem *cur = NULL;
744 struct tree_mod_elem *found = NULL;
745 u64 index = start >> PAGE_CACHE_SHIFT;
746
Jan Schmidt097b8a72012-06-21 11:08:04 +0200747 tree_mod_log_read_lock(fs_info);
Jan Schmidtbd989ba2012-05-16 17:18:50 +0200748 tm_root = &fs_info->tree_mod_log;
749 node = tm_root->rb_node;
750 while (node) {
751 cur = container_of(node, struct tree_mod_elem, node);
752 if (cur->index < index) {
753 node = node->rb_left;
754 } else if (cur->index > index) {
755 node = node->rb_right;
Jan Schmidt097b8a72012-06-21 11:08:04 +0200756 } else if (cur->seq < min_seq) {
Jan Schmidtbd989ba2012-05-16 17:18:50 +0200757 node = node->rb_left;
758 } else if (!smallest) {
759 /* we want the node with the highest seq */
760 if (found)
Jan Schmidt097b8a72012-06-21 11:08:04 +0200761 BUG_ON(found->seq > cur->seq);
Jan Schmidtbd989ba2012-05-16 17:18:50 +0200762 found = cur;
763 node = node->rb_left;
Jan Schmidt097b8a72012-06-21 11:08:04 +0200764 } else if (cur->seq > min_seq) {
Jan Schmidtbd989ba2012-05-16 17:18:50 +0200765 /* we want the node with the smallest seq */
766 if (found)
Jan Schmidt097b8a72012-06-21 11:08:04 +0200767 BUG_ON(found->seq < cur->seq);
Jan Schmidtbd989ba2012-05-16 17:18:50 +0200768 found = cur;
769 node = node->rb_right;
770 } else {
771 found = cur;
772 break;
773 }
774 }
Jan Schmidt097b8a72012-06-21 11:08:04 +0200775 tree_mod_log_read_unlock(fs_info);
Jan Schmidtbd989ba2012-05-16 17:18:50 +0200776
777 return found;
778}
779
780/*
781 * this returns the element from the log with the smallest time sequence
782 * value that's in the log (the oldest log item). any element with a time
783 * sequence lower than min_seq will be ignored.
784 */
785static struct tree_mod_elem *
786tree_mod_log_search_oldest(struct btrfs_fs_info *fs_info, u64 start,
787 u64 min_seq)
788{
789 return __tree_mod_log_search(fs_info, start, min_seq, 1);
790}
791
792/*
793 * this returns the element from the log with the largest time sequence
794 * value that's in the log (the most recent log item). any element with
795 * a time sequence lower than min_seq will be ignored.
796 */
797static struct tree_mod_elem *
798tree_mod_log_search(struct btrfs_fs_info *fs_info, u64 start, u64 min_seq)
799{
800 return __tree_mod_log_search(fs_info, start, min_seq, 0);
801}
802
Filipe David Borba Manana5de865e2013-12-20 15:17:46 +0000803static noinline int
Jan Schmidtbd989ba2012-05-16 17:18:50 +0200804tree_mod_log_eb_copy(struct btrfs_fs_info *fs_info, struct extent_buffer *dst,
805 struct extent_buffer *src, unsigned long dst_offset,
Jan Schmidt90f8d622013-04-13 13:19:53 +0000806 unsigned long src_offset, int nr_items)
Jan Schmidtbd989ba2012-05-16 17:18:50 +0200807{
Filipe David Borba Manana5de865e2013-12-20 15:17:46 +0000808 int ret = 0;
809 struct tree_mod_elem **tm_list = NULL;
810 struct tree_mod_elem **tm_list_add, **tm_list_rem;
Jan Schmidtbd989ba2012-05-16 17:18:50 +0200811 int i;
Filipe David Borba Manana5de865e2013-12-20 15:17:46 +0000812 int locked = 0;
Jan Schmidtbd989ba2012-05-16 17:18:50 +0200813
Filipe David Borba Manana5de865e2013-12-20 15:17:46 +0000814 if (!tree_mod_need_log(fs_info, NULL))
815 return 0;
Jan Schmidtbd989ba2012-05-16 17:18:50 +0200816
Josef Bacikc8cc6342013-07-01 16:18:19 -0400817 if (btrfs_header_level(dst) == 0 && btrfs_header_level(src) == 0)
Filipe David Borba Manana5de865e2013-12-20 15:17:46 +0000818 return 0;
819
820 tm_list = kzalloc(nr_items * 2 * sizeof(struct tree_mod_elem *),
821 GFP_NOFS);
822 if (!tm_list)
823 return -ENOMEM;
824
825 tm_list_add = tm_list;
826 tm_list_rem = tm_list + nr_items;
827 for (i = 0; i < nr_items; i++) {
828 tm_list_rem[i] = alloc_tree_mod_elem(src, i + src_offset,
829 MOD_LOG_KEY_REMOVE, GFP_NOFS);
830 if (!tm_list_rem[i]) {
831 ret = -ENOMEM;
832 goto free_tms;
833 }
834
835 tm_list_add[i] = alloc_tree_mod_elem(dst, i + dst_offset,
836 MOD_LOG_KEY_ADD, GFP_NOFS);
837 if (!tm_list_add[i]) {
838 ret = -ENOMEM;
839 goto free_tms;
840 }
841 }
842
843 if (tree_mod_dont_log(fs_info, NULL))
844 goto free_tms;
845 locked = 1;
Jan Schmidtbd989ba2012-05-16 17:18:50 +0200846
Jan Schmidtbd989ba2012-05-16 17:18:50 +0200847 for (i = 0; i < nr_items; i++) {
Filipe David Borba Manana5de865e2013-12-20 15:17:46 +0000848 ret = __tree_mod_log_insert(fs_info, tm_list_rem[i]);
849 if (ret)
850 goto free_tms;
851 ret = __tree_mod_log_insert(fs_info, tm_list_add[i]);
852 if (ret)
853 goto free_tms;
Jan Schmidtbd989ba2012-05-16 17:18:50 +0200854 }
Filipe David Borba Manana5de865e2013-12-20 15:17:46 +0000855
856 tree_mod_log_write_unlock(fs_info);
857 kfree(tm_list);
858
859 return 0;
860
861free_tms:
862 for (i = 0; i < nr_items * 2; i++) {
863 if (tm_list[i] && !RB_EMPTY_NODE(&tm_list[i]->node))
864 rb_erase(&tm_list[i]->node, &fs_info->tree_mod_log);
865 kfree(tm_list[i]);
866 }
867 if (locked)
868 tree_mod_log_write_unlock(fs_info);
869 kfree(tm_list);
870
871 return ret;
Jan Schmidtbd989ba2012-05-16 17:18:50 +0200872}
873
874static inline void
875tree_mod_log_eb_move(struct btrfs_fs_info *fs_info, struct extent_buffer *dst,
876 int dst_offset, int src_offset, int nr_items)
877{
878 int ret;
879 ret = tree_mod_log_insert_move(fs_info, dst, dst_offset, src_offset,
880 nr_items, GFP_NOFS);
881 BUG_ON(ret < 0);
882}
883
Jan Schmidt097b8a72012-06-21 11:08:04 +0200884static noinline void
Jan Schmidtbd989ba2012-05-16 17:18:50 +0200885tree_mod_log_set_node_key(struct btrfs_fs_info *fs_info,
Liu Bo32adf092012-10-19 12:52:15 +0000886 struct extent_buffer *eb, int slot, int atomic)
Jan Schmidtbd989ba2012-05-16 17:18:50 +0200887{
888 int ret;
889
Filipe David Borba Manana78357762013-12-12 19:19:52 +0000890 ret = tree_mod_log_insert_key(fs_info, eb, slot,
Josef Bacikc8cc6342013-07-01 16:18:19 -0400891 MOD_LOG_KEY_REPLACE,
892 atomic ? GFP_ATOMIC : GFP_NOFS);
Jan Schmidtbd989ba2012-05-16 17:18:50 +0200893 BUG_ON(ret < 0);
894}
895
Filipe David Borba Manana5de865e2013-12-20 15:17:46 +0000896static noinline int
Jan Schmidt097b8a72012-06-21 11:08:04 +0200897tree_mod_log_free_eb(struct btrfs_fs_info *fs_info, struct extent_buffer *eb)
Jan Schmidtbd989ba2012-05-16 17:18:50 +0200898{
Filipe David Borba Manana5de865e2013-12-20 15:17:46 +0000899 struct tree_mod_elem **tm_list = NULL;
900 int nritems = 0;
901 int i;
902 int ret = 0;
903
904 if (btrfs_header_level(eb) == 0)
905 return 0;
906
907 if (!tree_mod_need_log(fs_info, NULL))
908 return 0;
909
910 nritems = btrfs_header_nritems(eb);
911 tm_list = kzalloc(nritems * sizeof(struct tree_mod_elem *),
912 GFP_NOFS);
913 if (!tm_list)
914 return -ENOMEM;
915
916 for (i = 0; i < nritems; i++) {
917 tm_list[i] = alloc_tree_mod_elem(eb, i,
918 MOD_LOG_KEY_REMOVE_WHILE_FREEING, GFP_NOFS);
919 if (!tm_list[i]) {
920 ret = -ENOMEM;
921 goto free_tms;
922 }
923 }
924
Jan Schmidte9b7fd42012-05-31 14:59:09 +0200925 if (tree_mod_dont_log(fs_info, eb))
Filipe David Borba Manana5de865e2013-12-20 15:17:46 +0000926 goto free_tms;
927
928 ret = __tree_mod_log_free_eb(fs_info, tm_list, nritems);
929 tree_mod_log_write_unlock(fs_info);
930 if (ret)
931 goto free_tms;
932 kfree(tm_list);
933
934 return 0;
935
936free_tms:
937 for (i = 0; i < nritems; i++)
938 kfree(tm_list[i]);
939 kfree(tm_list);
940
941 return ret;
Jan Schmidtbd989ba2012-05-16 17:18:50 +0200942}
943
Jan Schmidt097b8a72012-06-21 11:08:04 +0200944static noinline void
Jan Schmidtbd989ba2012-05-16 17:18:50 +0200945tree_mod_log_set_root_pointer(struct btrfs_root *root,
Jan Schmidt90f8d622013-04-13 13:19:53 +0000946 struct extent_buffer *new_root_node,
947 int log_removal)
Jan Schmidtbd989ba2012-05-16 17:18:50 +0200948{
949 int ret;
Jan Schmidtbd989ba2012-05-16 17:18:50 +0200950 ret = tree_mod_log_insert_root(root->fs_info, root->node,
Jan Schmidt90f8d622013-04-13 13:19:53 +0000951 new_root_node, GFP_NOFS, log_removal);
Jan Schmidtbd989ba2012-05-16 17:18:50 +0200952 BUG_ON(ret < 0);
953}
954
Chris Masond352ac62008-09-29 15:18:18 -0400955/*
Yan Zheng5d4f98a2009-06-10 10:45:14 -0400956 * check if the tree block can be shared by multiple trees
957 */
958int btrfs_block_can_be_shared(struct btrfs_root *root,
959 struct extent_buffer *buf)
960{
961 /*
962 * Tree blocks not in refernece counted trees and tree roots
963 * are never shared. If a block was allocated after the last
964 * snapshot and the block was not allocated by tree relocation,
965 * we know the block is not shared.
966 */
Miao Xie27cdeb72014-04-02 19:51:05 +0800967 if (test_bit(BTRFS_ROOT_REF_COWS, &root->state) &&
Yan Zheng5d4f98a2009-06-10 10:45:14 -0400968 buf != root->node && buf != root->commit_root &&
969 (btrfs_header_generation(buf) <=
970 btrfs_root_last_snapshot(&root->root_item) ||
971 btrfs_header_flag(buf, BTRFS_HEADER_FLAG_RELOC)))
972 return 1;
973#ifdef BTRFS_COMPAT_EXTENT_TREE_V0
Miao Xie27cdeb72014-04-02 19:51:05 +0800974 if (test_bit(BTRFS_ROOT_REF_COWS, &root->state) &&
Yan Zheng5d4f98a2009-06-10 10:45:14 -0400975 btrfs_header_backref_rev(buf) < BTRFS_MIXED_BACKREF_REV)
976 return 1;
977#endif
978 return 0;
979}
980
981static noinline int update_ref_for_cow(struct btrfs_trans_handle *trans,
982 struct btrfs_root *root,
983 struct extent_buffer *buf,
Yan, Zhengf0486c62010-05-16 10:46:25 -0400984 struct extent_buffer *cow,
985 int *last_ref)
Yan Zheng5d4f98a2009-06-10 10:45:14 -0400986{
987 u64 refs;
988 u64 owner;
989 u64 flags;
990 u64 new_flags = 0;
991 int ret;
992
993 /*
994 * Backrefs update rules:
995 *
996 * Always use full backrefs for extent pointers in tree block
997 * allocated by tree relocation.
998 *
999 * If a shared tree block is no longer referenced by its owner
1000 * tree (btrfs_header_owner(buf) == root->root_key.objectid),
1001 * use full backrefs for extent pointers in tree block.
1002 *
1003 * If a tree block is been relocating
1004 * (root->root_key.objectid == BTRFS_TREE_RELOC_OBJECTID),
1005 * use full backrefs for extent pointers in tree block.
1006 * The reason for this is some operations (such as drop tree)
1007 * are only allowed for blocks use full backrefs.
1008 */
1009
1010 if (btrfs_block_can_be_shared(root, buf)) {
1011 ret = btrfs_lookup_extent_info(trans, root, buf->start,
Josef Bacik3173a182013-03-07 14:22:04 -05001012 btrfs_header_level(buf), 1,
1013 &refs, &flags);
Mark Fashehbe1a5562011-08-08 13:20:18 -07001014 if (ret)
1015 return ret;
Mark Fashehe5df9572011-08-29 14:17:04 -07001016 if (refs == 0) {
1017 ret = -EROFS;
1018 btrfs_std_error(root->fs_info, ret);
1019 return ret;
1020 }
Yan Zheng5d4f98a2009-06-10 10:45:14 -04001021 } else {
1022 refs = 1;
1023 if (root->root_key.objectid == BTRFS_TREE_RELOC_OBJECTID ||
1024 btrfs_header_backref_rev(buf) < BTRFS_MIXED_BACKREF_REV)
1025 flags = BTRFS_BLOCK_FLAG_FULL_BACKREF;
1026 else
1027 flags = 0;
1028 }
1029
1030 owner = btrfs_header_owner(buf);
1031 BUG_ON(owner == BTRFS_TREE_RELOC_OBJECTID &&
1032 !(flags & BTRFS_BLOCK_FLAG_FULL_BACKREF));
1033
1034 if (refs > 1) {
1035 if ((owner == root->root_key.objectid ||
1036 root->root_key.objectid == BTRFS_TREE_RELOC_OBJECTID) &&
1037 !(flags & BTRFS_BLOCK_FLAG_FULL_BACKREF)) {
Josef Bacike339a6b2014-07-02 10:54:25 -07001038 ret = btrfs_inc_ref(trans, root, buf, 1);
Jeff Mahoney79787ea2012-03-12 16:03:00 +01001039 BUG_ON(ret); /* -ENOMEM */
Yan Zheng5d4f98a2009-06-10 10:45:14 -04001040
1041 if (root->root_key.objectid ==
1042 BTRFS_TREE_RELOC_OBJECTID) {
Josef Bacike339a6b2014-07-02 10:54:25 -07001043 ret = btrfs_dec_ref(trans, root, buf, 0);
Jeff Mahoney79787ea2012-03-12 16:03:00 +01001044 BUG_ON(ret); /* -ENOMEM */
Josef Bacike339a6b2014-07-02 10:54:25 -07001045 ret = btrfs_inc_ref(trans, root, cow, 1);
Jeff Mahoney79787ea2012-03-12 16:03:00 +01001046 BUG_ON(ret); /* -ENOMEM */
Yan Zheng5d4f98a2009-06-10 10:45:14 -04001047 }
1048 new_flags |= BTRFS_BLOCK_FLAG_FULL_BACKREF;
1049 } else {
1050
1051 if (root->root_key.objectid ==
1052 BTRFS_TREE_RELOC_OBJECTID)
Josef Bacike339a6b2014-07-02 10:54:25 -07001053 ret = btrfs_inc_ref(trans, root, cow, 1);
Yan Zheng5d4f98a2009-06-10 10:45:14 -04001054 else
Josef Bacike339a6b2014-07-02 10:54:25 -07001055 ret = btrfs_inc_ref(trans, root, cow, 0);
Jeff Mahoney79787ea2012-03-12 16:03:00 +01001056 BUG_ON(ret); /* -ENOMEM */
Yan Zheng5d4f98a2009-06-10 10:45:14 -04001057 }
1058 if (new_flags != 0) {
Josef Bacikb1c79e02013-05-09 13:49:30 -04001059 int level = btrfs_header_level(buf);
1060
Yan Zheng5d4f98a2009-06-10 10:45:14 -04001061 ret = btrfs_set_disk_extent_flags(trans, root,
1062 buf->start,
1063 buf->len,
Josef Bacikb1c79e02013-05-09 13:49:30 -04001064 new_flags, level, 0);
Mark Fashehbe1a5562011-08-08 13:20:18 -07001065 if (ret)
1066 return ret;
Yan Zheng5d4f98a2009-06-10 10:45:14 -04001067 }
1068 } else {
1069 if (flags & BTRFS_BLOCK_FLAG_FULL_BACKREF) {
1070 if (root->root_key.objectid ==
1071 BTRFS_TREE_RELOC_OBJECTID)
Josef Bacike339a6b2014-07-02 10:54:25 -07001072 ret = btrfs_inc_ref(trans, root, cow, 1);
Yan Zheng5d4f98a2009-06-10 10:45:14 -04001073 else
Josef Bacike339a6b2014-07-02 10:54:25 -07001074 ret = btrfs_inc_ref(trans, root, cow, 0);
Jeff Mahoney79787ea2012-03-12 16:03:00 +01001075 BUG_ON(ret); /* -ENOMEM */
Josef Bacike339a6b2014-07-02 10:54:25 -07001076 ret = btrfs_dec_ref(trans, root, buf, 1);
Jeff Mahoney79787ea2012-03-12 16:03:00 +01001077 BUG_ON(ret); /* -ENOMEM */
Yan Zheng5d4f98a2009-06-10 10:45:14 -04001078 }
1079 clean_tree_block(trans, root, buf);
Yan, Zhengf0486c62010-05-16 10:46:25 -04001080 *last_ref = 1;
Yan Zheng5d4f98a2009-06-10 10:45:14 -04001081 }
1082 return 0;
1083}
1084
1085/*
Chris Masond3977122009-01-05 21:25:51 -05001086 * does the dirty work in cow of a single block. The parent block (if
1087 * supplied) is updated to point to the new cow copy. The new buffer is marked
1088 * dirty and returned locked. If you modify the block it needs to be marked
1089 * dirty again.
Chris Masond352ac62008-09-29 15:18:18 -04001090 *
1091 * search_start -- an allocation hint for the new block
1092 *
Chris Masond3977122009-01-05 21:25:51 -05001093 * empty_size -- a hint that you plan on doing more cow. This is the size in
1094 * bytes the allocator should try to find free next to the block it returns.
1095 * This is just a hint and may be ignored by the allocator.
Chris Masond352ac62008-09-29 15:18:18 -04001096 */
Chris Masond3977122009-01-05 21:25:51 -05001097static noinline int __btrfs_cow_block(struct btrfs_trans_handle *trans,
Chris Mason5f39d392007-10-15 16:14:19 -04001098 struct btrfs_root *root,
1099 struct extent_buffer *buf,
1100 struct extent_buffer *parent, int parent_slot,
1101 struct extent_buffer **cow_ret,
Chris Mason9fa8cfe2009-03-13 10:24:59 -04001102 u64 search_start, u64 empty_size)
Chris Mason6702ed42007-08-07 16:15:09 -04001103{
Yan Zheng5d4f98a2009-06-10 10:45:14 -04001104 struct btrfs_disk_key disk_key;
Chris Mason5f39d392007-10-15 16:14:19 -04001105 struct extent_buffer *cow;
Mark Fashehbe1a5562011-08-08 13:20:18 -07001106 int level, ret;
Yan, Zhengf0486c62010-05-16 10:46:25 -04001107 int last_ref = 0;
Chris Mason925baed2008-06-25 16:01:30 -04001108 int unlock_orig = 0;
Yan Zheng5d4f98a2009-06-10 10:45:14 -04001109 u64 parent_start;
Chris Mason6702ed42007-08-07 16:15:09 -04001110
Chris Mason925baed2008-06-25 16:01:30 -04001111 if (*cow_ret == buf)
1112 unlock_orig = 1;
1113
Chris Masonb9447ef2009-03-09 11:45:38 -04001114 btrfs_assert_tree_locked(buf);
Chris Mason925baed2008-06-25 16:01:30 -04001115
Miao Xie27cdeb72014-04-02 19:51:05 +08001116 WARN_ON(test_bit(BTRFS_ROOT_REF_COWS, &root->state) &&
1117 trans->transid != root->fs_info->running_transaction->transid);
1118 WARN_ON(test_bit(BTRFS_ROOT_REF_COWS, &root->state) &&
1119 trans->transid != root->last_trans);
Chris Mason5f39d392007-10-15 16:14:19 -04001120
Chris Mason7bb86312007-12-11 09:25:06 -05001121 level = btrfs_header_level(buf);
Zheng Yan31840ae2008-09-23 13:14:14 -04001122
Yan Zheng5d4f98a2009-06-10 10:45:14 -04001123 if (level == 0)
1124 btrfs_item_key(buf, &disk_key, 0);
1125 else
1126 btrfs_node_key(buf, &disk_key, 0);
1127
1128 if (root->root_key.objectid == BTRFS_TREE_RELOC_OBJECTID) {
1129 if (parent)
1130 parent_start = parent->start;
1131 else
1132 parent_start = 0;
1133 } else
1134 parent_start = 0;
1135
1136 cow = btrfs_alloc_free_block(trans, root, buf->len, parent_start,
1137 root->root_key.objectid, &disk_key,
Jan Schmidt5581a512012-05-16 17:04:52 +02001138 level, search_start, empty_size);
Chris Mason6702ed42007-08-07 16:15:09 -04001139 if (IS_ERR(cow))
1140 return PTR_ERR(cow);
1141
Chris Masonb4ce94d2009-02-04 09:25:08 -05001142 /* cow is set to blocking by btrfs_init_new_buffer */
1143
Chris Mason5f39d392007-10-15 16:14:19 -04001144 copy_extent_buffer(cow, buf, 0, 0, cow->len);
Chris Masondb945352007-10-15 16:15:53 -04001145 btrfs_set_header_bytenr(cow, cow->start);
Chris Mason5f39d392007-10-15 16:14:19 -04001146 btrfs_set_header_generation(cow, trans->transid);
Yan Zheng5d4f98a2009-06-10 10:45:14 -04001147 btrfs_set_header_backref_rev(cow, BTRFS_MIXED_BACKREF_REV);
1148 btrfs_clear_header_flag(cow, BTRFS_HEADER_FLAG_WRITTEN |
1149 BTRFS_HEADER_FLAG_RELOC);
1150 if (root->root_key.objectid == BTRFS_TREE_RELOC_OBJECTID)
1151 btrfs_set_header_flag(cow, BTRFS_HEADER_FLAG_RELOC);
1152 else
1153 btrfs_set_header_owner(cow, root->root_key.objectid);
Chris Mason6702ed42007-08-07 16:15:09 -04001154
Ross Kirk0a4e5582013-09-24 10:12:38 +01001155 write_extent_buffer(cow, root->fs_info->fsid, btrfs_header_fsid(),
Yan Zheng2b820322008-11-17 21:11:30 -05001156 BTRFS_FSID_SIZE);
1157
Mark Fashehbe1a5562011-08-08 13:20:18 -07001158 ret = update_ref_for_cow(trans, root, buf, cow, &last_ref);
Mark Fashehb68dc2a2011-08-29 14:30:39 -07001159 if (ret) {
Jeff Mahoney79787ea2012-03-12 16:03:00 +01001160 btrfs_abort_transaction(trans, root, ret);
Mark Fashehb68dc2a2011-08-29 14:30:39 -07001161 return ret;
1162 }
Zheng Yan1a40e232008-09-26 10:09:34 -04001163
Miao Xie27cdeb72014-04-02 19:51:05 +08001164 if (test_bit(BTRFS_ROOT_REF_COWS, &root->state)) {
Josef Bacik83d4cfd2013-08-30 15:09:51 -04001165 ret = btrfs_reloc_cow_block(trans, root, buf, cow);
1166 if (ret)
1167 return ret;
1168 }
Yan, Zheng3fd0a552010-05-16 10:49:59 -04001169
Chris Mason6702ed42007-08-07 16:15:09 -04001170 if (buf == root->node) {
Chris Mason925baed2008-06-25 16:01:30 -04001171 WARN_ON(parent && parent != buf);
Yan Zheng5d4f98a2009-06-10 10:45:14 -04001172 if (root->root_key.objectid == BTRFS_TREE_RELOC_OBJECTID ||
1173 btrfs_header_backref_rev(buf) < BTRFS_MIXED_BACKREF_REV)
1174 parent_start = buf->start;
1175 else
1176 parent_start = 0;
Chris Mason925baed2008-06-25 16:01:30 -04001177
Chris Mason5f39d392007-10-15 16:14:19 -04001178 extent_buffer_get(cow);
Jan Schmidt90f8d622013-04-13 13:19:53 +00001179 tree_mod_log_set_root_pointer(root, cow, 1);
Chris Mason240f62c2011-03-23 14:54:42 -04001180 rcu_assign_pointer(root->node, cow);
Chris Mason925baed2008-06-25 16:01:30 -04001181
Yan, Zhengf0486c62010-05-16 10:46:25 -04001182 btrfs_free_tree_block(trans, root, buf, parent_start,
Jan Schmidt5581a512012-05-16 17:04:52 +02001183 last_ref);
Chris Mason5f39d392007-10-15 16:14:19 -04001184 free_extent_buffer(buf);
Chris Mason0b86a832008-03-24 15:01:56 -04001185 add_root_to_dirty_list(root);
Chris Mason6702ed42007-08-07 16:15:09 -04001186 } else {
Yan Zheng5d4f98a2009-06-10 10:45:14 -04001187 if (root->root_key.objectid == BTRFS_TREE_RELOC_OBJECTID)
1188 parent_start = parent->start;
1189 else
1190 parent_start = 0;
1191
1192 WARN_ON(trans->transid != btrfs_header_generation(parent));
Jan Schmidtf2304752012-05-26 11:43:17 +02001193 tree_mod_log_insert_key(root->fs_info, parent, parent_slot,
Josef Bacikc8cc6342013-07-01 16:18:19 -04001194 MOD_LOG_KEY_REPLACE, GFP_NOFS);
Chris Mason5f39d392007-10-15 16:14:19 -04001195 btrfs_set_node_blockptr(parent, parent_slot,
Chris Masondb945352007-10-15 16:15:53 -04001196 cow->start);
Chris Mason74493f72007-12-11 09:25:06 -05001197 btrfs_set_node_ptr_generation(parent, parent_slot,
1198 trans->transid);
Chris Mason6702ed42007-08-07 16:15:09 -04001199 btrfs_mark_buffer_dirty(parent);
Filipe David Borba Manana5de865e2013-12-20 15:17:46 +00001200 if (last_ref) {
1201 ret = tree_mod_log_free_eb(root->fs_info, buf);
1202 if (ret) {
1203 btrfs_abort_transaction(trans, root, ret);
1204 return ret;
1205 }
1206 }
Yan, Zhengf0486c62010-05-16 10:46:25 -04001207 btrfs_free_tree_block(trans, root, buf, parent_start,
Jan Schmidt5581a512012-05-16 17:04:52 +02001208 last_ref);
Chris Mason6702ed42007-08-07 16:15:09 -04001209 }
Chris Mason925baed2008-06-25 16:01:30 -04001210 if (unlock_orig)
1211 btrfs_tree_unlock(buf);
Josef Bacik3083ee22012-03-09 16:01:49 -05001212 free_extent_buffer_stale(buf);
Chris Mason6702ed42007-08-07 16:15:09 -04001213 btrfs_mark_buffer_dirty(cow);
1214 *cow_ret = cow;
1215 return 0;
1216}
1217
Jan Schmidt5d9e75c42012-05-16 18:25:47 +02001218/*
1219 * returns the logical address of the oldest predecessor of the given root.
1220 * entries older than time_seq are ignored.
1221 */
1222static struct tree_mod_elem *
1223__tree_mod_log_oldest_root(struct btrfs_fs_info *fs_info,
Jan Schmidt30b04632013-04-13 13:19:54 +00001224 struct extent_buffer *eb_root, u64 time_seq)
Jan Schmidt5d9e75c42012-05-16 18:25:47 +02001225{
1226 struct tree_mod_elem *tm;
1227 struct tree_mod_elem *found = NULL;
Jan Schmidt30b04632013-04-13 13:19:54 +00001228 u64 root_logical = eb_root->start;
Jan Schmidt5d9e75c42012-05-16 18:25:47 +02001229 int looped = 0;
1230
1231 if (!time_seq)
Stefan Behrens35a36212013-08-14 18:12:25 +02001232 return NULL;
Jan Schmidt5d9e75c42012-05-16 18:25:47 +02001233
1234 /*
1235 * the very last operation that's logged for a root is the replacement
1236 * operation (if it is replaced at all). this has the index of the *new*
1237 * root, making it the very first operation that's logged for this root.
1238 */
1239 while (1) {
1240 tm = tree_mod_log_search_oldest(fs_info, root_logical,
1241 time_seq);
1242 if (!looped && !tm)
Stefan Behrens35a36212013-08-14 18:12:25 +02001243 return NULL;
Jan Schmidt5d9e75c42012-05-16 18:25:47 +02001244 /*
Jan Schmidt28da9fb2012-06-21 10:59:13 +02001245 * if there are no tree operation for the oldest root, we simply
1246 * return it. this should only happen if that (old) root is at
1247 * level 0.
Jan Schmidt5d9e75c42012-05-16 18:25:47 +02001248 */
Jan Schmidt28da9fb2012-06-21 10:59:13 +02001249 if (!tm)
1250 break;
Jan Schmidt5d9e75c42012-05-16 18:25:47 +02001251
Jan Schmidt28da9fb2012-06-21 10:59:13 +02001252 /*
1253 * if there's an operation that's not a root replacement, we
1254 * found the oldest version of our root. normally, we'll find a
1255 * MOD_LOG_KEY_REMOVE_WHILE_FREEING operation here.
1256 */
Jan Schmidt5d9e75c42012-05-16 18:25:47 +02001257 if (tm->op != MOD_LOG_ROOT_REPLACE)
1258 break;
1259
1260 found = tm;
1261 root_logical = tm->old_root.logical;
Jan Schmidt5d9e75c42012-05-16 18:25:47 +02001262 looped = 1;
1263 }
1264
Jan Schmidta95236d2012-06-05 16:41:24 +02001265 /* if there's no old root to return, return what we found instead */
1266 if (!found)
1267 found = tm;
1268
Jan Schmidt5d9e75c42012-05-16 18:25:47 +02001269 return found;
1270}
1271
1272/*
1273 * tm is a pointer to the first operation to rewind within eb. then, all
1274 * previous operations will be rewinded (until we reach something older than
1275 * time_seq).
1276 */
1277static void
Josef Bacikf1ca7e982013-06-29 23:15:19 -04001278__tree_mod_log_rewind(struct btrfs_fs_info *fs_info, struct extent_buffer *eb,
1279 u64 time_seq, struct tree_mod_elem *first_tm)
Jan Schmidt5d9e75c42012-05-16 18:25:47 +02001280{
1281 u32 n;
1282 struct rb_node *next;
1283 struct tree_mod_elem *tm = first_tm;
1284 unsigned long o_dst;
1285 unsigned long o_src;
1286 unsigned long p_size = sizeof(struct btrfs_key_ptr);
1287
1288 n = btrfs_header_nritems(eb);
Josef Bacikf1ca7e982013-06-29 23:15:19 -04001289 tree_mod_log_read_lock(fs_info);
Jan Schmidt097b8a72012-06-21 11:08:04 +02001290 while (tm && tm->seq >= time_seq) {
Jan Schmidt5d9e75c42012-05-16 18:25:47 +02001291 /*
1292 * all the operations are recorded with the operator used for
1293 * the modification. as we're going backwards, we do the
1294 * opposite of each operation here.
1295 */
1296 switch (tm->op) {
1297 case MOD_LOG_KEY_REMOVE_WHILE_FREEING:
1298 BUG_ON(tm->slot < n);
Eric Sandeen1c697d42013-01-31 00:54:56 +00001299 /* Fallthrough */
Liu Bo95c80bb2012-10-19 09:50:52 +00001300 case MOD_LOG_KEY_REMOVE_WHILE_MOVING:
Chris Mason4c3e6962012-12-18 15:43:18 -05001301 case MOD_LOG_KEY_REMOVE:
Jan Schmidt5d9e75c42012-05-16 18:25:47 +02001302 btrfs_set_node_key(eb, &tm->key, tm->slot);
1303 btrfs_set_node_blockptr(eb, tm->slot, tm->blockptr);
1304 btrfs_set_node_ptr_generation(eb, tm->slot,
1305 tm->generation);
Chris Mason4c3e6962012-12-18 15:43:18 -05001306 n++;
Jan Schmidt5d9e75c42012-05-16 18:25:47 +02001307 break;
1308 case MOD_LOG_KEY_REPLACE:
1309 BUG_ON(tm->slot >= n);
1310 btrfs_set_node_key(eb, &tm->key, tm->slot);
1311 btrfs_set_node_blockptr(eb, tm->slot, tm->blockptr);
1312 btrfs_set_node_ptr_generation(eb, tm->slot,
1313 tm->generation);
1314 break;
1315 case MOD_LOG_KEY_ADD:
Jan Schmidt19956c72012-06-22 14:52:13 +02001316 /* if a move operation is needed it's in the log */
Jan Schmidt5d9e75c42012-05-16 18:25:47 +02001317 n--;
1318 break;
1319 case MOD_LOG_MOVE_KEYS:
Jan Schmidtc3193102012-05-31 19:24:36 +02001320 o_dst = btrfs_node_key_ptr_offset(tm->slot);
1321 o_src = btrfs_node_key_ptr_offset(tm->move.dst_slot);
1322 memmove_extent_buffer(eb, o_dst, o_src,
Jan Schmidt5d9e75c42012-05-16 18:25:47 +02001323 tm->move.nr_items * p_size);
1324 break;
1325 case MOD_LOG_ROOT_REPLACE:
1326 /*
1327 * this operation is special. for roots, this must be
1328 * handled explicitly before rewinding.
1329 * for non-roots, this operation may exist if the node
1330 * was a root: root A -> child B; then A gets empty and
1331 * B is promoted to the new root. in the mod log, we'll
1332 * have a root-replace operation for B, a tree block
1333 * that is no root. we simply ignore that operation.
1334 */
1335 break;
1336 }
1337 next = rb_next(&tm->node);
1338 if (!next)
1339 break;
1340 tm = container_of(next, struct tree_mod_elem, node);
1341 if (tm->index != first_tm->index)
1342 break;
1343 }
Josef Bacikf1ca7e982013-06-29 23:15:19 -04001344 tree_mod_log_read_unlock(fs_info);
Jan Schmidt5d9e75c42012-05-16 18:25:47 +02001345 btrfs_set_header_nritems(eb, n);
1346}
1347
Jan Schmidt47fb0912013-04-13 13:19:55 +00001348/*
1349 * Called with eb read locked. If the buffer cannot be rewinded, the same buffer
1350 * is returned. If rewind operations happen, a fresh buffer is returned. The
1351 * returned buffer is always read-locked. If the returned buffer is not the
1352 * input buffer, the lock on the input buffer is released and the input buffer
1353 * is freed (its refcount is decremented).
1354 */
Jan Schmidt5d9e75c42012-05-16 18:25:47 +02001355static struct extent_buffer *
Josef Bacik9ec72672013-08-07 16:57:23 -04001356tree_mod_log_rewind(struct btrfs_fs_info *fs_info, struct btrfs_path *path,
1357 struct extent_buffer *eb, u64 time_seq)
Jan Schmidt5d9e75c42012-05-16 18:25:47 +02001358{
1359 struct extent_buffer *eb_rewin;
1360 struct tree_mod_elem *tm;
1361
1362 if (!time_seq)
1363 return eb;
1364
1365 if (btrfs_header_level(eb) == 0)
1366 return eb;
1367
1368 tm = tree_mod_log_search(fs_info, eb->start, time_seq);
1369 if (!tm)
1370 return eb;
1371
Josef Bacik9ec72672013-08-07 16:57:23 -04001372 btrfs_set_path_blocking(path);
1373 btrfs_set_lock_blocking_rw(eb, BTRFS_READ_LOCK);
1374
Jan Schmidt5d9e75c42012-05-16 18:25:47 +02001375 if (tm->op == MOD_LOG_KEY_REMOVE_WHILE_FREEING) {
1376 BUG_ON(tm->slot != 0);
1377 eb_rewin = alloc_dummy_extent_buffer(eb->start,
1378 fs_info->tree_root->nodesize);
Josef Bacikdb7f3432013-08-07 14:54:37 -04001379 if (!eb_rewin) {
Josef Bacik9ec72672013-08-07 16:57:23 -04001380 btrfs_tree_read_unlock_blocking(eb);
Josef Bacikdb7f3432013-08-07 14:54:37 -04001381 free_extent_buffer(eb);
1382 return NULL;
1383 }
Jan Schmidt5d9e75c42012-05-16 18:25:47 +02001384 btrfs_set_header_bytenr(eb_rewin, eb->start);
1385 btrfs_set_header_backref_rev(eb_rewin,
1386 btrfs_header_backref_rev(eb));
1387 btrfs_set_header_owner(eb_rewin, btrfs_header_owner(eb));
Jan Schmidtc3193102012-05-31 19:24:36 +02001388 btrfs_set_header_level(eb_rewin, btrfs_header_level(eb));
Jan Schmidt5d9e75c42012-05-16 18:25:47 +02001389 } else {
1390 eb_rewin = btrfs_clone_extent_buffer(eb);
Josef Bacikdb7f3432013-08-07 14:54:37 -04001391 if (!eb_rewin) {
Josef Bacik9ec72672013-08-07 16:57:23 -04001392 btrfs_tree_read_unlock_blocking(eb);
Josef Bacikdb7f3432013-08-07 14:54:37 -04001393 free_extent_buffer(eb);
1394 return NULL;
1395 }
Jan Schmidt5d9e75c42012-05-16 18:25:47 +02001396 }
1397
Josef Bacik9ec72672013-08-07 16:57:23 -04001398 btrfs_clear_path_blocking(path, NULL, BTRFS_READ_LOCK);
1399 btrfs_tree_read_unlock_blocking(eb);
Jan Schmidt5d9e75c42012-05-16 18:25:47 +02001400 free_extent_buffer(eb);
1401
Jan Schmidt47fb0912013-04-13 13:19:55 +00001402 extent_buffer_get(eb_rewin);
1403 btrfs_tree_read_lock(eb_rewin);
Josef Bacikf1ca7e982013-06-29 23:15:19 -04001404 __tree_mod_log_rewind(fs_info, eb_rewin, time_seq, tm);
Jan Schmidt57911b82012-10-19 09:22:03 +02001405 WARN_ON(btrfs_header_nritems(eb_rewin) >
Arne Jansen2a745b12013-02-13 04:20:01 -07001406 BTRFS_NODEPTRS_PER_BLOCK(fs_info->tree_root));
Jan Schmidt5d9e75c42012-05-16 18:25:47 +02001407
1408 return eb_rewin;
1409}
1410
Jan Schmidt8ba97a12012-06-04 16:54:57 +02001411/*
1412 * get_old_root() rewinds the state of @root's root node to the given @time_seq
1413 * value. If there are no changes, the current root->root_node is returned. If
1414 * anything changed in between, there's a fresh buffer allocated on which the
1415 * rewind operations are done. In any case, the returned buffer is read locked.
1416 * Returns NULL on error (with no locks held).
1417 */
Jan Schmidt5d9e75c42012-05-16 18:25:47 +02001418static inline struct extent_buffer *
1419get_old_root(struct btrfs_root *root, u64 time_seq)
1420{
1421 struct tree_mod_elem *tm;
Jan Schmidt30b04632013-04-13 13:19:54 +00001422 struct extent_buffer *eb = NULL;
1423 struct extent_buffer *eb_root;
Liu Bo7bfdcf72012-10-25 07:30:19 -06001424 struct extent_buffer *old;
Jan Schmidta95236d2012-06-05 16:41:24 +02001425 struct tree_mod_root *old_root = NULL;
Chris Mason4325edd2012-06-15 20:02:02 -04001426 u64 old_generation = 0;
Jan Schmidta95236d2012-06-05 16:41:24 +02001427 u64 logical;
Jan Schmidt834328a2012-10-23 11:27:33 +02001428 u32 blocksize;
Jan Schmidt5d9e75c42012-05-16 18:25:47 +02001429
Jan Schmidt30b04632013-04-13 13:19:54 +00001430 eb_root = btrfs_read_lock_root_node(root);
1431 tm = __tree_mod_log_oldest_root(root->fs_info, eb_root, time_seq);
Jan Schmidt5d9e75c42012-05-16 18:25:47 +02001432 if (!tm)
Jan Schmidt30b04632013-04-13 13:19:54 +00001433 return eb_root;
Jan Schmidt5d9e75c42012-05-16 18:25:47 +02001434
Jan Schmidta95236d2012-06-05 16:41:24 +02001435 if (tm->op == MOD_LOG_ROOT_REPLACE) {
1436 old_root = &tm->old_root;
1437 old_generation = tm->generation;
1438 logical = old_root->logical;
1439 } else {
Jan Schmidt30b04632013-04-13 13:19:54 +00001440 logical = eb_root->start;
Jan Schmidta95236d2012-06-05 16:41:24 +02001441 }
Jan Schmidt5d9e75c42012-05-16 18:25:47 +02001442
Jan Schmidta95236d2012-06-05 16:41:24 +02001443 tm = tree_mod_log_search(root->fs_info, logical, time_seq);
Jan Schmidt834328a2012-10-23 11:27:33 +02001444 if (old_root && tm && tm->op != MOD_LOG_KEY_REMOVE_WHILE_FREEING) {
Jan Schmidt30b04632013-04-13 13:19:54 +00001445 btrfs_tree_read_unlock(eb_root);
1446 free_extent_buffer(eb_root);
Jan Schmidt834328a2012-10-23 11:27:33 +02001447 blocksize = btrfs_level_size(root, old_root->level);
Liu Bo7bfdcf72012-10-25 07:30:19 -06001448 old = read_tree_block(root, logical, blocksize, 0);
Dulshani Gunawardhanafae7f212013-10-31 10:30:08 +05301449 if (WARN_ON(!old || !extent_buffer_uptodate(old))) {
Josef Bacik416bc652013-04-23 14:17:42 -04001450 free_extent_buffer(old);
Frank Holtonefe120a2013-12-20 11:37:06 -05001451 btrfs_warn(root->fs_info,
1452 "failed to read tree block %llu from get_old_root", logical);
Jan Schmidt834328a2012-10-23 11:27:33 +02001453 } else {
Liu Bo7bfdcf72012-10-25 07:30:19 -06001454 eb = btrfs_clone_extent_buffer(old);
1455 free_extent_buffer(old);
Jan Schmidt834328a2012-10-23 11:27:33 +02001456 }
1457 } else if (old_root) {
Jan Schmidt30b04632013-04-13 13:19:54 +00001458 btrfs_tree_read_unlock(eb_root);
1459 free_extent_buffer(eb_root);
Jan Schmidt28da9fb2012-06-21 10:59:13 +02001460 eb = alloc_dummy_extent_buffer(logical, root->nodesize);
Jan Schmidt834328a2012-10-23 11:27:33 +02001461 } else {
Josef Bacik9ec72672013-08-07 16:57:23 -04001462 btrfs_set_lock_blocking_rw(eb_root, BTRFS_READ_LOCK);
Jan Schmidt30b04632013-04-13 13:19:54 +00001463 eb = btrfs_clone_extent_buffer(eb_root);
Josef Bacik9ec72672013-08-07 16:57:23 -04001464 btrfs_tree_read_unlock_blocking(eb_root);
Jan Schmidt30b04632013-04-13 13:19:54 +00001465 free_extent_buffer(eb_root);
Jan Schmidt834328a2012-10-23 11:27:33 +02001466 }
1467
Jan Schmidt8ba97a12012-06-04 16:54:57 +02001468 if (!eb)
1469 return NULL;
Jan Schmidtd6381082012-10-23 14:21:05 +02001470 extent_buffer_get(eb);
Jan Schmidt8ba97a12012-06-04 16:54:57 +02001471 btrfs_tree_read_lock(eb);
Jan Schmidta95236d2012-06-05 16:41:24 +02001472 if (old_root) {
Jan Schmidt5d9e75c42012-05-16 18:25:47 +02001473 btrfs_set_header_bytenr(eb, eb->start);
1474 btrfs_set_header_backref_rev(eb, BTRFS_MIXED_BACKREF_REV);
Jan Schmidt30b04632013-04-13 13:19:54 +00001475 btrfs_set_header_owner(eb, btrfs_header_owner(eb_root));
Jan Schmidta95236d2012-06-05 16:41:24 +02001476 btrfs_set_header_level(eb, old_root->level);
1477 btrfs_set_header_generation(eb, old_generation);
Jan Schmidt5d9e75c42012-05-16 18:25:47 +02001478 }
Jan Schmidt28da9fb2012-06-21 10:59:13 +02001479 if (tm)
Josef Bacikf1ca7e982013-06-29 23:15:19 -04001480 __tree_mod_log_rewind(root->fs_info, eb, time_seq, tm);
Jan Schmidt28da9fb2012-06-21 10:59:13 +02001481 else
1482 WARN_ON(btrfs_header_level(eb) != 0);
Jan Schmidt57911b82012-10-19 09:22:03 +02001483 WARN_ON(btrfs_header_nritems(eb) > BTRFS_NODEPTRS_PER_BLOCK(root));
Jan Schmidt5d9e75c42012-05-16 18:25:47 +02001484
1485 return eb;
1486}
1487
Jan Schmidt5b6602e2012-10-23 11:28:27 +02001488int btrfs_old_root_level(struct btrfs_root *root, u64 time_seq)
1489{
1490 struct tree_mod_elem *tm;
1491 int level;
Jan Schmidt30b04632013-04-13 13:19:54 +00001492 struct extent_buffer *eb_root = btrfs_root_node(root);
Jan Schmidt5b6602e2012-10-23 11:28:27 +02001493
Jan Schmidt30b04632013-04-13 13:19:54 +00001494 tm = __tree_mod_log_oldest_root(root->fs_info, eb_root, time_seq);
Jan Schmidt5b6602e2012-10-23 11:28:27 +02001495 if (tm && tm->op == MOD_LOG_ROOT_REPLACE) {
1496 level = tm->old_root.level;
1497 } else {
Jan Schmidt30b04632013-04-13 13:19:54 +00001498 level = btrfs_header_level(eb_root);
Jan Schmidt5b6602e2012-10-23 11:28:27 +02001499 }
Jan Schmidt30b04632013-04-13 13:19:54 +00001500 free_extent_buffer(eb_root);
Jan Schmidt5b6602e2012-10-23 11:28:27 +02001501
1502 return level;
1503}
1504
Yan Zheng5d4f98a2009-06-10 10:45:14 -04001505static inline int should_cow_block(struct btrfs_trans_handle *trans,
1506 struct btrfs_root *root,
1507 struct extent_buffer *buf)
1508{
Josef Bacikfaa2dbf2014-05-07 17:06:09 -04001509#ifdef CONFIG_BTRFS_FS_RUN_SANITY_TESTS
1510 if (unlikely(test_bit(BTRFS_ROOT_DUMMY_ROOT, &root->state)))
1511 return 0;
1512#endif
Liu Bof1ebcc72011-11-14 20:48:06 -05001513 /* ensure we can see the force_cow */
1514 smp_rmb();
1515
1516 /*
1517 * We do not need to cow a block if
1518 * 1) this block is not created or changed in this transaction;
1519 * 2) this block does not belong to TREE_RELOC tree;
1520 * 3) the root is not forced COW.
1521 *
1522 * What is forced COW:
1523 * when we create snapshot during commiting the transaction,
1524 * after we've finished coping src root, we must COW the shared
1525 * block to ensure the metadata consistency.
1526 */
Yan Zheng5d4f98a2009-06-10 10:45:14 -04001527 if (btrfs_header_generation(buf) == trans->transid &&
1528 !btrfs_header_flag(buf, BTRFS_HEADER_FLAG_WRITTEN) &&
1529 !(root->root_key.objectid != BTRFS_TREE_RELOC_OBJECTID &&
Liu Bof1ebcc72011-11-14 20:48:06 -05001530 btrfs_header_flag(buf, BTRFS_HEADER_FLAG_RELOC)) &&
Miao Xie27cdeb72014-04-02 19:51:05 +08001531 !test_bit(BTRFS_ROOT_FORCE_COW, &root->state))
Yan Zheng5d4f98a2009-06-10 10:45:14 -04001532 return 0;
1533 return 1;
1534}
1535
Chris Masond352ac62008-09-29 15:18:18 -04001536/*
1537 * cows a single block, see __btrfs_cow_block for the real work.
1538 * This version of it has extra checks so that a block isn't cow'd more than
1539 * once per transaction, as long as it hasn't been written yet
1540 */
Chris Masond3977122009-01-05 21:25:51 -05001541noinline int btrfs_cow_block(struct btrfs_trans_handle *trans,
Chris Mason5f39d392007-10-15 16:14:19 -04001542 struct btrfs_root *root, struct extent_buffer *buf,
1543 struct extent_buffer *parent, int parent_slot,
Chris Mason9fa8cfe2009-03-13 10:24:59 -04001544 struct extent_buffer **cow_ret)
Chris Mason02217ed2007-03-02 16:08:05 -05001545{
Chris Mason6702ed42007-08-07 16:15:09 -04001546 u64 search_start;
Chris Masonf510cfe2007-10-15 16:14:48 -04001547 int ret;
Chris Masondc17ff82008-01-08 15:46:30 -05001548
Julia Lawall31b1a2b2012-11-03 10:58:34 +00001549 if (trans->transaction != root->fs_info->running_transaction)
1550 WARN(1, KERN_CRIT "trans %llu running %llu\n",
Geert Uytterhoevenc1c9ff72013-08-20 13:20:07 +02001551 trans->transid,
Chris Masonccd467d2007-06-28 15:57:36 -04001552 root->fs_info->running_transaction->transid);
Julia Lawall31b1a2b2012-11-03 10:58:34 +00001553
1554 if (trans->transid != root->fs_info->generation)
1555 WARN(1, KERN_CRIT "trans %llu running %llu\n",
Geert Uytterhoevenc1c9ff72013-08-20 13:20:07 +02001556 trans->transid, root->fs_info->generation);
Chris Masondc17ff82008-01-08 15:46:30 -05001557
Yan Zheng5d4f98a2009-06-10 10:45:14 -04001558 if (!should_cow_block(trans, root, buf)) {
Chris Mason02217ed2007-03-02 16:08:05 -05001559 *cow_ret = buf;
1560 return 0;
1561 }
Chris Masonc4876852009-02-04 09:24:25 -05001562
Chris Mason0b86a832008-03-24 15:01:56 -04001563 search_start = buf->start & ~((u64)(1024 * 1024 * 1024) - 1);
Chris Masonb4ce94d2009-02-04 09:25:08 -05001564
1565 if (parent)
1566 btrfs_set_lock_blocking(parent);
1567 btrfs_set_lock_blocking(buf);
1568
Chris Masonf510cfe2007-10-15 16:14:48 -04001569 ret = __btrfs_cow_block(trans, root, buf, parent,
Chris Mason9fa8cfe2009-03-13 10:24:59 -04001570 parent_slot, cow_ret, search_start, 0);
liubo1abe9b82011-03-24 11:18:59 +00001571
1572 trace_btrfs_cow_block(root, buf, *cow_ret);
1573
Chris Masonf510cfe2007-10-15 16:14:48 -04001574 return ret;
Chris Mason6702ed42007-08-07 16:15:09 -04001575}
1576
Chris Masond352ac62008-09-29 15:18:18 -04001577/*
1578 * helper function for defrag to decide if two blocks pointed to by a
1579 * node are actually close by
1580 */
Chris Mason6b800532007-10-15 16:17:34 -04001581static int close_blocks(u64 blocknr, u64 other, u32 blocksize)
Chris Mason6702ed42007-08-07 16:15:09 -04001582{
Chris Mason6b800532007-10-15 16:17:34 -04001583 if (blocknr < other && other - (blocknr + blocksize) < 32768)
Chris Mason6702ed42007-08-07 16:15:09 -04001584 return 1;
Chris Mason6b800532007-10-15 16:17:34 -04001585 if (blocknr > other && blocknr - (other + blocksize) < 32768)
Chris Mason6702ed42007-08-07 16:15:09 -04001586 return 1;
Chris Mason02217ed2007-03-02 16:08:05 -05001587 return 0;
1588}
1589
Chris Mason081e9572007-11-06 10:26:24 -05001590/*
1591 * compare two keys in a memcmp fashion
1592 */
1593static int comp_keys(struct btrfs_disk_key *disk, struct btrfs_key *k2)
1594{
1595 struct btrfs_key k1;
1596
1597 btrfs_disk_key_to_cpu(&k1, disk);
1598
Diego Calleja20736ab2009-07-24 11:06:52 -04001599 return btrfs_comp_cpu_keys(&k1, k2);
Chris Mason081e9572007-11-06 10:26:24 -05001600}
1601
Josef Bacikf3465ca2008-11-12 14:19:50 -05001602/*
1603 * same as comp_keys only with two btrfs_key's
1604 */
Yan Zheng5d4f98a2009-06-10 10:45:14 -04001605int btrfs_comp_cpu_keys(struct btrfs_key *k1, struct btrfs_key *k2)
Josef Bacikf3465ca2008-11-12 14:19:50 -05001606{
1607 if (k1->objectid > k2->objectid)
1608 return 1;
1609 if (k1->objectid < k2->objectid)
1610 return -1;
1611 if (k1->type > k2->type)
1612 return 1;
1613 if (k1->type < k2->type)
1614 return -1;
1615 if (k1->offset > k2->offset)
1616 return 1;
1617 if (k1->offset < k2->offset)
1618 return -1;
1619 return 0;
1620}
Chris Mason081e9572007-11-06 10:26:24 -05001621
Chris Masond352ac62008-09-29 15:18:18 -04001622/*
1623 * this is used by the defrag code to go through all the
1624 * leaves pointed to by a node and reallocate them so that
1625 * disk order is close to key order
1626 */
Chris Mason6702ed42007-08-07 16:15:09 -04001627int btrfs_realloc_node(struct btrfs_trans_handle *trans,
Chris Mason5f39d392007-10-15 16:14:19 -04001628 struct btrfs_root *root, struct extent_buffer *parent,
Eric Sandeende78b512013-01-31 18:21:12 +00001629 int start_slot, u64 *last_ret,
Chris Masona6b6e752007-10-15 16:22:39 -04001630 struct btrfs_key *progress)
Chris Mason6702ed42007-08-07 16:15:09 -04001631{
Chris Mason6b800532007-10-15 16:17:34 -04001632 struct extent_buffer *cur;
Chris Mason6702ed42007-08-07 16:15:09 -04001633 u64 blocknr;
Chris Masonca7a79a2008-05-12 12:59:19 -04001634 u64 gen;
Chris Masone9d0b132007-08-10 14:06:19 -04001635 u64 search_start = *last_ret;
1636 u64 last_block = 0;
Chris Mason6702ed42007-08-07 16:15:09 -04001637 u64 other;
1638 u32 parent_nritems;
Chris Mason6702ed42007-08-07 16:15:09 -04001639 int end_slot;
1640 int i;
1641 int err = 0;
Chris Masonf2183bd2007-08-10 14:42:37 -04001642 int parent_level;
Chris Mason6b800532007-10-15 16:17:34 -04001643 int uptodate;
1644 u32 blocksize;
Chris Mason081e9572007-11-06 10:26:24 -05001645 int progress_passed = 0;
1646 struct btrfs_disk_key disk_key;
Chris Mason6702ed42007-08-07 16:15:09 -04001647
Chris Mason5708b952007-10-25 15:43:18 -04001648 parent_level = btrfs_header_level(parent);
Chris Mason5708b952007-10-25 15:43:18 -04001649
Julia Lawall6c1500f2012-11-03 20:30:18 +00001650 WARN_ON(trans->transaction != root->fs_info->running_transaction);
1651 WARN_ON(trans->transid != root->fs_info->generation);
Chris Mason86479a02007-09-10 19:58:16 -04001652
Chris Mason6b800532007-10-15 16:17:34 -04001653 parent_nritems = btrfs_header_nritems(parent);
Chris Mason6b800532007-10-15 16:17:34 -04001654 blocksize = btrfs_level_size(root, parent_level - 1);
Chris Mason6702ed42007-08-07 16:15:09 -04001655 end_slot = parent_nritems;
1656
1657 if (parent_nritems == 1)
1658 return 0;
1659
Chris Masonb4ce94d2009-02-04 09:25:08 -05001660 btrfs_set_lock_blocking(parent);
1661
Chris Mason6702ed42007-08-07 16:15:09 -04001662 for (i = start_slot; i < end_slot; i++) {
1663 int close = 1;
Chris Masona6b6e752007-10-15 16:22:39 -04001664
Chris Mason081e9572007-11-06 10:26:24 -05001665 btrfs_node_key(parent, &disk_key, i);
1666 if (!progress_passed && comp_keys(&disk_key, progress) < 0)
1667 continue;
1668
1669 progress_passed = 1;
Chris Mason6b800532007-10-15 16:17:34 -04001670 blocknr = btrfs_node_blockptr(parent, i);
Chris Masonca7a79a2008-05-12 12:59:19 -04001671 gen = btrfs_node_ptr_generation(parent, i);
Chris Masone9d0b132007-08-10 14:06:19 -04001672 if (last_block == 0)
1673 last_block = blocknr;
Chris Mason5708b952007-10-25 15:43:18 -04001674
Chris Mason6702ed42007-08-07 16:15:09 -04001675 if (i > 0) {
Chris Mason6b800532007-10-15 16:17:34 -04001676 other = btrfs_node_blockptr(parent, i - 1);
1677 close = close_blocks(blocknr, other, blocksize);
Chris Mason6702ed42007-08-07 16:15:09 -04001678 }
Chris Mason0ef3e662008-05-24 14:04:53 -04001679 if (!close && i < end_slot - 2) {
Chris Mason6b800532007-10-15 16:17:34 -04001680 other = btrfs_node_blockptr(parent, i + 1);
1681 close = close_blocks(blocknr, other, blocksize);
Chris Mason6702ed42007-08-07 16:15:09 -04001682 }
Chris Masone9d0b132007-08-10 14:06:19 -04001683 if (close) {
1684 last_block = blocknr;
Chris Mason6702ed42007-08-07 16:15:09 -04001685 continue;
Chris Masone9d0b132007-08-10 14:06:19 -04001686 }
Chris Mason6702ed42007-08-07 16:15:09 -04001687
Chris Mason6b800532007-10-15 16:17:34 -04001688 cur = btrfs_find_tree_block(root, blocknr, blocksize);
1689 if (cur)
Chris Masonb9fab912012-05-06 07:23:47 -04001690 uptodate = btrfs_buffer_uptodate(cur, gen, 0);
Chris Mason6b800532007-10-15 16:17:34 -04001691 else
1692 uptodate = 0;
Chris Mason5708b952007-10-25 15:43:18 -04001693 if (!cur || !uptodate) {
Chris Mason6b800532007-10-15 16:17:34 -04001694 if (!cur) {
1695 cur = read_tree_block(root, blocknr,
Chris Masonca7a79a2008-05-12 12:59:19 -04001696 blocksize, gen);
Josef Bacik416bc652013-04-23 14:17:42 -04001697 if (!cur || !extent_buffer_uptodate(cur)) {
1698 free_extent_buffer(cur);
Tsutomu Itoh97d9a8a2011-03-24 06:33:21 +00001699 return -EIO;
Josef Bacik416bc652013-04-23 14:17:42 -04001700 }
Chris Mason6b800532007-10-15 16:17:34 -04001701 } else if (!uptodate) {
Tsutomu Itoh018642a2012-05-29 18:10:13 +09001702 err = btrfs_read_buffer(cur, gen);
1703 if (err) {
1704 free_extent_buffer(cur);
1705 return err;
1706 }
Chris Masonf2183bd2007-08-10 14:42:37 -04001707 }
Chris Mason6702ed42007-08-07 16:15:09 -04001708 }
Chris Masone9d0b132007-08-10 14:06:19 -04001709 if (search_start == 0)
Chris Mason6b800532007-10-15 16:17:34 -04001710 search_start = last_block;
Chris Masone9d0b132007-08-10 14:06:19 -04001711
Chris Masone7a84562008-06-25 16:01:31 -04001712 btrfs_tree_lock(cur);
Chris Masonb4ce94d2009-02-04 09:25:08 -05001713 btrfs_set_lock_blocking(cur);
Chris Mason6b800532007-10-15 16:17:34 -04001714 err = __btrfs_cow_block(trans, root, cur, parent, i,
Chris Masone7a84562008-06-25 16:01:31 -04001715 &cur, search_start,
Chris Mason6b800532007-10-15 16:17:34 -04001716 min(16 * blocksize,
Chris Mason9fa8cfe2009-03-13 10:24:59 -04001717 (end_slot - i) * blocksize));
Yan252c38f2007-08-29 09:11:44 -04001718 if (err) {
Chris Masone7a84562008-06-25 16:01:31 -04001719 btrfs_tree_unlock(cur);
Chris Mason6b800532007-10-15 16:17:34 -04001720 free_extent_buffer(cur);
Chris Mason6702ed42007-08-07 16:15:09 -04001721 break;
Yan252c38f2007-08-29 09:11:44 -04001722 }
Chris Masone7a84562008-06-25 16:01:31 -04001723 search_start = cur->start;
1724 last_block = cur->start;
Chris Masonf2183bd2007-08-10 14:42:37 -04001725 *last_ret = search_start;
Chris Masone7a84562008-06-25 16:01:31 -04001726 btrfs_tree_unlock(cur);
1727 free_extent_buffer(cur);
Chris Mason6702ed42007-08-07 16:15:09 -04001728 }
1729 return err;
1730}
1731
Chris Mason74123bd2007-02-02 11:05:29 -05001732/*
1733 * The leaf data grows from end-to-front in the node.
1734 * this returns the address of the start of the last item,
1735 * which is the stop of the leaf data stack
1736 */
Chris Mason123abc82007-03-14 14:14:43 -04001737static inline unsigned int leaf_data_end(struct btrfs_root *root,
Chris Mason5f39d392007-10-15 16:14:19 -04001738 struct extent_buffer *leaf)
Chris Masonbe0e5c02007-01-26 15:51:26 -05001739{
Chris Mason5f39d392007-10-15 16:14:19 -04001740 u32 nr = btrfs_header_nritems(leaf);
Chris Masonbe0e5c02007-01-26 15:51:26 -05001741 if (nr == 0)
Chris Mason123abc82007-03-14 14:14:43 -04001742 return BTRFS_LEAF_DATA_SIZE(root);
Chris Mason5f39d392007-10-15 16:14:19 -04001743 return btrfs_item_offset_nr(leaf, nr - 1);
Chris Masonbe0e5c02007-01-26 15:51:26 -05001744}
1745
Chris Masonaa5d6be2007-02-28 16:35:06 -05001746
Chris Mason74123bd2007-02-02 11:05:29 -05001747/*
Chris Mason5f39d392007-10-15 16:14:19 -04001748 * search for key in the extent_buffer. The items start at offset p,
1749 * and they are item_size apart. There are 'max' items in p.
1750 *
Chris Mason74123bd2007-02-02 11:05:29 -05001751 * the slot in the array is returned via slot, and it points to
1752 * the place where you would insert key if it is not found in
1753 * the array.
1754 *
1755 * slot may point to max if the key is bigger than all of the keys
1756 */
Chris Masone02119d2008-09-05 16:13:11 -04001757static noinline int generic_bin_search(struct extent_buffer *eb,
1758 unsigned long p,
1759 int item_size, struct btrfs_key *key,
1760 int max, int *slot)
Chris Masonbe0e5c02007-01-26 15:51:26 -05001761{
1762 int low = 0;
1763 int high = max;
1764 int mid;
1765 int ret;
Chris Mason479965d2007-10-15 16:14:27 -04001766 struct btrfs_disk_key *tmp = NULL;
Chris Mason5f39d392007-10-15 16:14:19 -04001767 struct btrfs_disk_key unaligned;
1768 unsigned long offset;
Chris Mason5f39d392007-10-15 16:14:19 -04001769 char *kaddr = NULL;
1770 unsigned long map_start = 0;
1771 unsigned long map_len = 0;
Chris Mason479965d2007-10-15 16:14:27 -04001772 int err;
Chris Masonbe0e5c02007-01-26 15:51:26 -05001773
Chris Masond3977122009-01-05 21:25:51 -05001774 while (low < high) {
Chris Masonbe0e5c02007-01-26 15:51:26 -05001775 mid = (low + high) / 2;
Chris Mason5f39d392007-10-15 16:14:19 -04001776 offset = p + mid * item_size;
1777
Chris Masona6591712011-07-19 12:04:14 -04001778 if (!kaddr || offset < map_start ||
Chris Mason5f39d392007-10-15 16:14:19 -04001779 (offset + sizeof(struct btrfs_disk_key)) >
1780 map_start + map_len) {
Chris Mason934d3752008-12-08 16:43:10 -05001781
1782 err = map_private_extent_buffer(eb, offset,
Chris Mason479965d2007-10-15 16:14:27 -04001783 sizeof(struct btrfs_disk_key),
Chris Masona6591712011-07-19 12:04:14 -04001784 &kaddr, &map_start, &map_len);
Chris Mason5f39d392007-10-15 16:14:19 -04001785
Chris Mason479965d2007-10-15 16:14:27 -04001786 if (!err) {
1787 tmp = (struct btrfs_disk_key *)(kaddr + offset -
1788 map_start);
1789 } else {
1790 read_extent_buffer(eb, &unaligned,
1791 offset, sizeof(unaligned));
1792 tmp = &unaligned;
1793 }
1794
Chris Mason5f39d392007-10-15 16:14:19 -04001795 } else {
1796 tmp = (struct btrfs_disk_key *)(kaddr + offset -
1797 map_start);
1798 }
Chris Masonbe0e5c02007-01-26 15:51:26 -05001799 ret = comp_keys(tmp, key);
1800
1801 if (ret < 0)
1802 low = mid + 1;
1803 else if (ret > 0)
1804 high = mid;
1805 else {
1806 *slot = mid;
1807 return 0;
1808 }
1809 }
1810 *slot = low;
1811 return 1;
1812}
1813
Chris Mason97571fd2007-02-24 13:39:08 -05001814/*
1815 * simple bin_search frontend that does the right thing for
1816 * leaves vs nodes
1817 */
Chris Mason5f39d392007-10-15 16:14:19 -04001818static int bin_search(struct extent_buffer *eb, struct btrfs_key *key,
1819 int level, int *slot)
Chris Masonbe0e5c02007-01-26 15:51:26 -05001820{
Wang Sheng-Huif7757382012-03-30 15:14:27 +08001821 if (level == 0)
Chris Mason5f39d392007-10-15 16:14:19 -04001822 return generic_bin_search(eb,
1823 offsetof(struct btrfs_leaf, items),
Chris Mason0783fcf2007-03-12 20:12:07 -04001824 sizeof(struct btrfs_item),
Chris Mason5f39d392007-10-15 16:14:19 -04001825 key, btrfs_header_nritems(eb),
Chris Mason7518a232007-03-12 12:01:18 -04001826 slot);
Wang Sheng-Huif7757382012-03-30 15:14:27 +08001827 else
Chris Mason5f39d392007-10-15 16:14:19 -04001828 return generic_bin_search(eb,
1829 offsetof(struct btrfs_node, ptrs),
Chris Mason123abc82007-03-14 14:14:43 -04001830 sizeof(struct btrfs_key_ptr),
Chris Mason5f39d392007-10-15 16:14:19 -04001831 key, btrfs_header_nritems(eb),
Chris Mason7518a232007-03-12 12:01:18 -04001832 slot);
Chris Masonbe0e5c02007-01-26 15:51:26 -05001833}
1834
Yan Zheng5d4f98a2009-06-10 10:45:14 -04001835int btrfs_bin_search(struct extent_buffer *eb, struct btrfs_key *key,
1836 int level, int *slot)
1837{
1838 return bin_search(eb, key, level, slot);
1839}
1840
Yan, Zhengf0486c62010-05-16 10:46:25 -04001841static void root_add_used(struct btrfs_root *root, u32 size)
1842{
1843 spin_lock(&root->accounting_lock);
1844 btrfs_set_root_used(&root->root_item,
1845 btrfs_root_used(&root->root_item) + size);
1846 spin_unlock(&root->accounting_lock);
1847}
1848
1849static void root_sub_used(struct btrfs_root *root, u32 size)
1850{
1851 spin_lock(&root->accounting_lock);
1852 btrfs_set_root_used(&root->root_item,
1853 btrfs_root_used(&root->root_item) - size);
1854 spin_unlock(&root->accounting_lock);
1855}
1856
Chris Masond352ac62008-09-29 15:18:18 -04001857/* given a node and slot number, this reads the blocks it points to. The
1858 * extent buffer is returned with a reference taken (but unlocked).
1859 * NULL is returned on error.
1860 */
Chris Masone02119d2008-09-05 16:13:11 -04001861static noinline struct extent_buffer *read_node_slot(struct btrfs_root *root,
Chris Mason5f39d392007-10-15 16:14:19 -04001862 struct extent_buffer *parent, int slot)
Chris Masonbb803952007-03-01 12:04:21 -05001863{
Chris Masonca7a79a2008-05-12 12:59:19 -04001864 int level = btrfs_header_level(parent);
Josef Bacik416bc652013-04-23 14:17:42 -04001865 struct extent_buffer *eb;
1866
Chris Masonbb803952007-03-01 12:04:21 -05001867 if (slot < 0)
1868 return NULL;
Chris Mason5f39d392007-10-15 16:14:19 -04001869 if (slot >= btrfs_header_nritems(parent))
Chris Masonbb803952007-03-01 12:04:21 -05001870 return NULL;
Chris Masonca7a79a2008-05-12 12:59:19 -04001871
1872 BUG_ON(level == 0);
1873
Josef Bacik416bc652013-04-23 14:17:42 -04001874 eb = read_tree_block(root, btrfs_node_blockptr(parent, slot),
1875 btrfs_level_size(root, level - 1),
1876 btrfs_node_ptr_generation(parent, slot));
1877 if (eb && !extent_buffer_uptodate(eb)) {
1878 free_extent_buffer(eb);
1879 eb = NULL;
1880 }
1881
1882 return eb;
Chris Masonbb803952007-03-01 12:04:21 -05001883}
1884
Chris Masond352ac62008-09-29 15:18:18 -04001885/*
1886 * node level balancing, used to make sure nodes are in proper order for
1887 * item deletion. We balance from the top down, so we have to make sure
1888 * that a deletion won't leave an node completely empty later on.
1889 */
Chris Masone02119d2008-09-05 16:13:11 -04001890static noinline int balance_level(struct btrfs_trans_handle *trans,
Chris Mason98ed5172008-01-03 10:01:48 -05001891 struct btrfs_root *root,
1892 struct btrfs_path *path, int level)
Chris Masonbb803952007-03-01 12:04:21 -05001893{
Chris Mason5f39d392007-10-15 16:14:19 -04001894 struct extent_buffer *right = NULL;
1895 struct extent_buffer *mid;
1896 struct extent_buffer *left = NULL;
1897 struct extent_buffer *parent = NULL;
Chris Masonbb803952007-03-01 12:04:21 -05001898 int ret = 0;
1899 int wret;
1900 int pslot;
Chris Masonbb803952007-03-01 12:04:21 -05001901 int orig_slot = path->slots[level];
Chris Mason79f95c82007-03-01 15:16:26 -05001902 u64 orig_ptr;
Chris Masonbb803952007-03-01 12:04:21 -05001903
1904 if (level == 0)
1905 return 0;
1906
Chris Mason5f39d392007-10-15 16:14:19 -04001907 mid = path->nodes[level];
Chris Masonb4ce94d2009-02-04 09:25:08 -05001908
Chris Masonbd681512011-07-16 15:23:14 -04001909 WARN_ON(path->locks[level] != BTRFS_WRITE_LOCK &&
1910 path->locks[level] != BTRFS_WRITE_LOCK_BLOCKING);
Chris Mason7bb86312007-12-11 09:25:06 -05001911 WARN_ON(btrfs_header_generation(mid) != trans->transid);
1912
Chris Mason1d4f8a02007-03-13 09:28:32 -04001913 orig_ptr = btrfs_node_blockptr(mid, orig_slot);
Chris Mason79f95c82007-03-01 15:16:26 -05001914
Li Zefana05a9bb2011-09-06 16:55:34 +08001915 if (level < BTRFS_MAX_LEVEL - 1) {
Chris Mason5f39d392007-10-15 16:14:19 -04001916 parent = path->nodes[level + 1];
Li Zefana05a9bb2011-09-06 16:55:34 +08001917 pslot = path->slots[level + 1];
1918 }
Chris Masonbb803952007-03-01 12:04:21 -05001919
Chris Mason40689472007-03-17 14:29:23 -04001920 /*
1921 * deal with the case where there is only one pointer in the root
1922 * by promoting the node below to a root
1923 */
Chris Mason5f39d392007-10-15 16:14:19 -04001924 if (!parent) {
1925 struct extent_buffer *child;
Chris Masonbb803952007-03-01 12:04:21 -05001926
Chris Mason5f39d392007-10-15 16:14:19 -04001927 if (btrfs_header_nritems(mid) != 1)
Chris Masonbb803952007-03-01 12:04:21 -05001928 return 0;
1929
1930 /* promote the child to a root */
Chris Mason5f39d392007-10-15 16:14:19 -04001931 child = read_node_slot(root, mid, 0);
Mark Fasheh305a26a2011-09-01 11:27:57 -07001932 if (!child) {
1933 ret = -EROFS;
1934 btrfs_std_error(root->fs_info, ret);
1935 goto enospc;
1936 }
1937
Chris Mason925baed2008-06-25 16:01:30 -04001938 btrfs_tree_lock(child);
Chris Masonb4ce94d2009-02-04 09:25:08 -05001939 btrfs_set_lock_blocking(child);
Chris Mason9fa8cfe2009-03-13 10:24:59 -04001940 ret = btrfs_cow_block(trans, root, child, mid, 0, &child);
Yan, Zhengf0486c62010-05-16 10:46:25 -04001941 if (ret) {
1942 btrfs_tree_unlock(child);
1943 free_extent_buffer(child);
1944 goto enospc;
1945 }
Yan2f375ab2008-02-01 14:58:07 -05001946
Jan Schmidt90f8d622013-04-13 13:19:53 +00001947 tree_mod_log_set_root_pointer(root, child, 1);
Chris Mason240f62c2011-03-23 14:54:42 -04001948 rcu_assign_pointer(root->node, child);
Chris Mason925baed2008-06-25 16:01:30 -04001949
Chris Mason0b86a832008-03-24 15:01:56 -04001950 add_root_to_dirty_list(root);
Chris Mason925baed2008-06-25 16:01:30 -04001951 btrfs_tree_unlock(child);
Chris Masonb4ce94d2009-02-04 09:25:08 -05001952
Chris Mason925baed2008-06-25 16:01:30 -04001953 path->locks[level] = 0;
Chris Masonbb803952007-03-01 12:04:21 -05001954 path->nodes[level] = NULL;
Chris Mason5f39d392007-10-15 16:14:19 -04001955 clean_tree_block(trans, root, mid);
Chris Mason925baed2008-06-25 16:01:30 -04001956 btrfs_tree_unlock(mid);
Chris Masonbb803952007-03-01 12:04:21 -05001957 /* once for the path */
Chris Mason5f39d392007-10-15 16:14:19 -04001958 free_extent_buffer(mid);
Yan, Zhengf0486c62010-05-16 10:46:25 -04001959
1960 root_sub_used(root, mid->len);
Jan Schmidt5581a512012-05-16 17:04:52 +02001961 btrfs_free_tree_block(trans, root, mid, 0, 1);
Chris Masonbb803952007-03-01 12:04:21 -05001962 /* once for the root ptr */
Josef Bacik3083ee22012-03-09 16:01:49 -05001963 free_extent_buffer_stale(mid);
Yan, Zhengf0486c62010-05-16 10:46:25 -04001964 return 0;
Chris Masonbb803952007-03-01 12:04:21 -05001965 }
Chris Mason5f39d392007-10-15 16:14:19 -04001966 if (btrfs_header_nritems(mid) >
Chris Mason123abc82007-03-14 14:14:43 -04001967 BTRFS_NODEPTRS_PER_BLOCK(root) / 4)
Chris Masonbb803952007-03-01 12:04:21 -05001968 return 0;
1969
Chris Mason5f39d392007-10-15 16:14:19 -04001970 left = read_node_slot(root, parent, pslot - 1);
1971 if (left) {
Chris Mason925baed2008-06-25 16:01:30 -04001972 btrfs_tree_lock(left);
Chris Masonb4ce94d2009-02-04 09:25:08 -05001973 btrfs_set_lock_blocking(left);
Chris Mason5f39d392007-10-15 16:14:19 -04001974 wret = btrfs_cow_block(trans, root, left,
Chris Mason9fa8cfe2009-03-13 10:24:59 -04001975 parent, pslot - 1, &left);
Chris Mason54aa1f42007-06-22 14:16:25 -04001976 if (wret) {
1977 ret = wret;
1978 goto enospc;
1979 }
Chris Mason2cc58cf2007-08-27 16:49:44 -04001980 }
Chris Mason5f39d392007-10-15 16:14:19 -04001981 right = read_node_slot(root, parent, pslot + 1);
1982 if (right) {
Chris Mason925baed2008-06-25 16:01:30 -04001983 btrfs_tree_lock(right);
Chris Masonb4ce94d2009-02-04 09:25:08 -05001984 btrfs_set_lock_blocking(right);
Chris Mason5f39d392007-10-15 16:14:19 -04001985 wret = btrfs_cow_block(trans, root, right,
Chris Mason9fa8cfe2009-03-13 10:24:59 -04001986 parent, pslot + 1, &right);
Chris Mason2cc58cf2007-08-27 16:49:44 -04001987 if (wret) {
1988 ret = wret;
1989 goto enospc;
1990 }
1991 }
1992
1993 /* first, try to make some room in the middle buffer */
Chris Mason5f39d392007-10-15 16:14:19 -04001994 if (left) {
1995 orig_slot += btrfs_header_nritems(left);
Chris Masonbce4eae2008-04-24 14:42:46 -04001996 wret = push_node_left(trans, root, left, mid, 1);
Chris Mason79f95c82007-03-01 15:16:26 -05001997 if (wret < 0)
1998 ret = wret;
Chris Masonbb803952007-03-01 12:04:21 -05001999 }
Chris Mason79f95c82007-03-01 15:16:26 -05002000
2001 /*
2002 * then try to empty the right most buffer into the middle
2003 */
Chris Mason5f39d392007-10-15 16:14:19 -04002004 if (right) {
Chris Mason971a1f62008-04-24 10:54:32 -04002005 wret = push_node_left(trans, root, mid, right, 1);
Chris Mason54aa1f42007-06-22 14:16:25 -04002006 if (wret < 0 && wret != -ENOSPC)
Chris Mason79f95c82007-03-01 15:16:26 -05002007 ret = wret;
Chris Mason5f39d392007-10-15 16:14:19 -04002008 if (btrfs_header_nritems(right) == 0) {
Chris Mason5f39d392007-10-15 16:14:19 -04002009 clean_tree_block(trans, root, right);
Chris Mason925baed2008-06-25 16:01:30 -04002010 btrfs_tree_unlock(right);
Tsutomu Itohafe5fea2013-04-16 05:18:22 +00002011 del_ptr(root, path, level + 1, pslot + 1);
Yan, Zhengf0486c62010-05-16 10:46:25 -04002012 root_sub_used(root, right->len);
Jan Schmidt5581a512012-05-16 17:04:52 +02002013 btrfs_free_tree_block(trans, root, right, 0, 1);
Josef Bacik3083ee22012-03-09 16:01:49 -05002014 free_extent_buffer_stale(right);
Yan, Zhengf0486c62010-05-16 10:46:25 -04002015 right = NULL;
Chris Masonbb803952007-03-01 12:04:21 -05002016 } else {
Chris Mason5f39d392007-10-15 16:14:19 -04002017 struct btrfs_disk_key right_key;
2018 btrfs_node_key(right, &right_key, 0);
Jan Schmidtf2304752012-05-26 11:43:17 +02002019 tree_mod_log_set_node_key(root->fs_info, parent,
Liu Bo32adf092012-10-19 12:52:15 +00002020 pslot + 1, 0);
Chris Mason5f39d392007-10-15 16:14:19 -04002021 btrfs_set_node_key(parent, &right_key, pslot + 1);
2022 btrfs_mark_buffer_dirty(parent);
Chris Masonbb803952007-03-01 12:04:21 -05002023 }
2024 }
Chris Mason5f39d392007-10-15 16:14:19 -04002025 if (btrfs_header_nritems(mid) == 1) {
Chris Mason79f95c82007-03-01 15:16:26 -05002026 /*
2027 * we're not allowed to leave a node with one item in the
2028 * tree during a delete. A deletion from lower in the tree
2029 * could try to delete the only pointer in this node.
2030 * So, pull some keys from the left.
2031 * There has to be a left pointer at this point because
2032 * otherwise we would have pulled some pointers from the
2033 * right
2034 */
Mark Fasheh305a26a2011-09-01 11:27:57 -07002035 if (!left) {
2036 ret = -EROFS;
2037 btrfs_std_error(root->fs_info, ret);
2038 goto enospc;
2039 }
Chris Mason5f39d392007-10-15 16:14:19 -04002040 wret = balance_node_right(trans, root, mid, left);
Chris Mason54aa1f42007-06-22 14:16:25 -04002041 if (wret < 0) {
Chris Mason79f95c82007-03-01 15:16:26 -05002042 ret = wret;
Chris Mason54aa1f42007-06-22 14:16:25 -04002043 goto enospc;
2044 }
Chris Masonbce4eae2008-04-24 14:42:46 -04002045 if (wret == 1) {
2046 wret = push_node_left(trans, root, left, mid, 1);
2047 if (wret < 0)
2048 ret = wret;
2049 }
Chris Mason79f95c82007-03-01 15:16:26 -05002050 BUG_ON(wret == 1);
2051 }
Chris Mason5f39d392007-10-15 16:14:19 -04002052 if (btrfs_header_nritems(mid) == 0) {
Chris Mason5f39d392007-10-15 16:14:19 -04002053 clean_tree_block(trans, root, mid);
Chris Mason925baed2008-06-25 16:01:30 -04002054 btrfs_tree_unlock(mid);
Tsutomu Itohafe5fea2013-04-16 05:18:22 +00002055 del_ptr(root, path, level + 1, pslot);
Yan, Zhengf0486c62010-05-16 10:46:25 -04002056 root_sub_used(root, mid->len);
Jan Schmidt5581a512012-05-16 17:04:52 +02002057 btrfs_free_tree_block(trans, root, mid, 0, 1);
Josef Bacik3083ee22012-03-09 16:01:49 -05002058 free_extent_buffer_stale(mid);
Yan, Zhengf0486c62010-05-16 10:46:25 -04002059 mid = NULL;
Chris Mason79f95c82007-03-01 15:16:26 -05002060 } else {
2061 /* update the parent key to reflect our changes */
Chris Mason5f39d392007-10-15 16:14:19 -04002062 struct btrfs_disk_key mid_key;
2063 btrfs_node_key(mid, &mid_key, 0);
Liu Bo32adf092012-10-19 12:52:15 +00002064 tree_mod_log_set_node_key(root->fs_info, parent,
Jan Schmidtf2304752012-05-26 11:43:17 +02002065 pslot, 0);
Chris Mason5f39d392007-10-15 16:14:19 -04002066 btrfs_set_node_key(parent, &mid_key, pslot);
2067 btrfs_mark_buffer_dirty(parent);
Chris Mason79f95c82007-03-01 15:16:26 -05002068 }
Chris Masonbb803952007-03-01 12:04:21 -05002069
Chris Mason79f95c82007-03-01 15:16:26 -05002070 /* update the path */
Chris Mason5f39d392007-10-15 16:14:19 -04002071 if (left) {
2072 if (btrfs_header_nritems(left) > orig_slot) {
2073 extent_buffer_get(left);
Chris Mason925baed2008-06-25 16:01:30 -04002074 /* left was locked after cow */
Chris Mason5f39d392007-10-15 16:14:19 -04002075 path->nodes[level] = left;
Chris Masonbb803952007-03-01 12:04:21 -05002076 path->slots[level + 1] -= 1;
2077 path->slots[level] = orig_slot;
Chris Mason925baed2008-06-25 16:01:30 -04002078 if (mid) {
2079 btrfs_tree_unlock(mid);
Chris Mason5f39d392007-10-15 16:14:19 -04002080 free_extent_buffer(mid);
Chris Mason925baed2008-06-25 16:01:30 -04002081 }
Chris Masonbb803952007-03-01 12:04:21 -05002082 } else {
Chris Mason5f39d392007-10-15 16:14:19 -04002083 orig_slot -= btrfs_header_nritems(left);
Chris Masonbb803952007-03-01 12:04:21 -05002084 path->slots[level] = orig_slot;
2085 }
2086 }
Chris Mason79f95c82007-03-01 15:16:26 -05002087 /* double check we haven't messed things up */
Chris Masone20d96d2007-03-22 12:13:20 -04002088 if (orig_ptr !=
Chris Mason5f39d392007-10-15 16:14:19 -04002089 btrfs_node_blockptr(path->nodes[level], path->slots[level]))
Chris Mason79f95c82007-03-01 15:16:26 -05002090 BUG();
Chris Mason54aa1f42007-06-22 14:16:25 -04002091enospc:
Chris Mason925baed2008-06-25 16:01:30 -04002092 if (right) {
2093 btrfs_tree_unlock(right);
Chris Mason5f39d392007-10-15 16:14:19 -04002094 free_extent_buffer(right);
Chris Mason925baed2008-06-25 16:01:30 -04002095 }
2096 if (left) {
2097 if (path->nodes[level] != left)
2098 btrfs_tree_unlock(left);
Chris Mason5f39d392007-10-15 16:14:19 -04002099 free_extent_buffer(left);
Chris Mason925baed2008-06-25 16:01:30 -04002100 }
Chris Masonbb803952007-03-01 12:04:21 -05002101 return ret;
2102}
2103
Chris Masond352ac62008-09-29 15:18:18 -04002104/* Node balancing for insertion. Here we only split or push nodes around
2105 * when they are completely full. This is also done top down, so we
2106 * have to be pessimistic.
2107 */
Chris Masond3977122009-01-05 21:25:51 -05002108static noinline int push_nodes_for_insert(struct btrfs_trans_handle *trans,
Chris Mason98ed5172008-01-03 10:01:48 -05002109 struct btrfs_root *root,
2110 struct btrfs_path *path, int level)
Chris Masone66f7092007-04-20 13:16:02 -04002111{
Chris Mason5f39d392007-10-15 16:14:19 -04002112 struct extent_buffer *right = NULL;
2113 struct extent_buffer *mid;
2114 struct extent_buffer *left = NULL;
2115 struct extent_buffer *parent = NULL;
Chris Masone66f7092007-04-20 13:16:02 -04002116 int ret = 0;
2117 int wret;
2118 int pslot;
2119 int orig_slot = path->slots[level];
Chris Masone66f7092007-04-20 13:16:02 -04002120
2121 if (level == 0)
2122 return 1;
2123
Chris Mason5f39d392007-10-15 16:14:19 -04002124 mid = path->nodes[level];
Chris Mason7bb86312007-12-11 09:25:06 -05002125 WARN_ON(btrfs_header_generation(mid) != trans->transid);
Chris Masone66f7092007-04-20 13:16:02 -04002126
Li Zefana05a9bb2011-09-06 16:55:34 +08002127 if (level < BTRFS_MAX_LEVEL - 1) {
Chris Mason5f39d392007-10-15 16:14:19 -04002128 parent = path->nodes[level + 1];
Li Zefana05a9bb2011-09-06 16:55:34 +08002129 pslot = path->slots[level + 1];
2130 }
Chris Masone66f7092007-04-20 13:16:02 -04002131
Chris Mason5f39d392007-10-15 16:14:19 -04002132 if (!parent)
Chris Masone66f7092007-04-20 13:16:02 -04002133 return 1;
Chris Masone66f7092007-04-20 13:16:02 -04002134
Chris Mason5f39d392007-10-15 16:14:19 -04002135 left = read_node_slot(root, parent, pslot - 1);
Chris Masone66f7092007-04-20 13:16:02 -04002136
2137 /* first, try to make some room in the middle buffer */
Chris Mason5f39d392007-10-15 16:14:19 -04002138 if (left) {
Chris Masone66f7092007-04-20 13:16:02 -04002139 u32 left_nr;
Chris Mason925baed2008-06-25 16:01:30 -04002140
2141 btrfs_tree_lock(left);
Chris Masonb4ce94d2009-02-04 09:25:08 -05002142 btrfs_set_lock_blocking(left);
2143
Chris Mason5f39d392007-10-15 16:14:19 -04002144 left_nr = btrfs_header_nritems(left);
Chris Mason33ade1f2007-04-20 13:48:57 -04002145 if (left_nr >= BTRFS_NODEPTRS_PER_BLOCK(root) - 1) {
2146 wret = 1;
2147 } else {
Chris Mason5f39d392007-10-15 16:14:19 -04002148 ret = btrfs_cow_block(trans, root, left, parent,
Chris Mason9fa8cfe2009-03-13 10:24:59 -04002149 pslot - 1, &left);
Chris Mason54aa1f42007-06-22 14:16:25 -04002150 if (ret)
2151 wret = 1;
2152 else {
Chris Mason54aa1f42007-06-22 14:16:25 -04002153 wret = push_node_left(trans, root,
Chris Mason971a1f62008-04-24 10:54:32 -04002154 left, mid, 0);
Chris Mason54aa1f42007-06-22 14:16:25 -04002155 }
Chris Mason33ade1f2007-04-20 13:48:57 -04002156 }
Chris Masone66f7092007-04-20 13:16:02 -04002157 if (wret < 0)
2158 ret = wret;
2159 if (wret == 0) {
Chris Mason5f39d392007-10-15 16:14:19 -04002160 struct btrfs_disk_key disk_key;
Chris Masone66f7092007-04-20 13:16:02 -04002161 orig_slot += left_nr;
Chris Mason5f39d392007-10-15 16:14:19 -04002162 btrfs_node_key(mid, &disk_key, 0);
Jan Schmidtf2304752012-05-26 11:43:17 +02002163 tree_mod_log_set_node_key(root->fs_info, parent,
Liu Bo32adf092012-10-19 12:52:15 +00002164 pslot, 0);
Chris Mason5f39d392007-10-15 16:14:19 -04002165 btrfs_set_node_key(parent, &disk_key, pslot);
2166 btrfs_mark_buffer_dirty(parent);
2167 if (btrfs_header_nritems(left) > orig_slot) {
2168 path->nodes[level] = left;
Chris Masone66f7092007-04-20 13:16:02 -04002169 path->slots[level + 1] -= 1;
2170 path->slots[level] = orig_slot;
Chris Mason925baed2008-06-25 16:01:30 -04002171 btrfs_tree_unlock(mid);
Chris Mason5f39d392007-10-15 16:14:19 -04002172 free_extent_buffer(mid);
Chris Masone66f7092007-04-20 13:16:02 -04002173 } else {
2174 orig_slot -=
Chris Mason5f39d392007-10-15 16:14:19 -04002175 btrfs_header_nritems(left);
Chris Masone66f7092007-04-20 13:16:02 -04002176 path->slots[level] = orig_slot;
Chris Mason925baed2008-06-25 16:01:30 -04002177 btrfs_tree_unlock(left);
Chris Mason5f39d392007-10-15 16:14:19 -04002178 free_extent_buffer(left);
Chris Masone66f7092007-04-20 13:16:02 -04002179 }
Chris Masone66f7092007-04-20 13:16:02 -04002180 return 0;
2181 }
Chris Mason925baed2008-06-25 16:01:30 -04002182 btrfs_tree_unlock(left);
Chris Mason5f39d392007-10-15 16:14:19 -04002183 free_extent_buffer(left);
Chris Masone66f7092007-04-20 13:16:02 -04002184 }
Chris Mason925baed2008-06-25 16:01:30 -04002185 right = read_node_slot(root, parent, pslot + 1);
Chris Masone66f7092007-04-20 13:16:02 -04002186
2187 /*
2188 * then try to empty the right most buffer into the middle
2189 */
Chris Mason5f39d392007-10-15 16:14:19 -04002190 if (right) {
Chris Mason33ade1f2007-04-20 13:48:57 -04002191 u32 right_nr;
Chris Masonb4ce94d2009-02-04 09:25:08 -05002192
Chris Mason925baed2008-06-25 16:01:30 -04002193 btrfs_tree_lock(right);
Chris Masonb4ce94d2009-02-04 09:25:08 -05002194 btrfs_set_lock_blocking(right);
2195
Chris Mason5f39d392007-10-15 16:14:19 -04002196 right_nr = btrfs_header_nritems(right);
Chris Mason33ade1f2007-04-20 13:48:57 -04002197 if (right_nr >= BTRFS_NODEPTRS_PER_BLOCK(root) - 1) {
2198 wret = 1;
2199 } else {
Chris Mason5f39d392007-10-15 16:14:19 -04002200 ret = btrfs_cow_block(trans, root, right,
2201 parent, pslot + 1,
Chris Mason9fa8cfe2009-03-13 10:24:59 -04002202 &right);
Chris Mason54aa1f42007-06-22 14:16:25 -04002203 if (ret)
2204 wret = 1;
2205 else {
Chris Mason54aa1f42007-06-22 14:16:25 -04002206 wret = balance_node_right(trans, root,
Chris Mason5f39d392007-10-15 16:14:19 -04002207 right, mid);
Chris Mason54aa1f42007-06-22 14:16:25 -04002208 }
Chris Mason33ade1f2007-04-20 13:48:57 -04002209 }
Chris Masone66f7092007-04-20 13:16:02 -04002210 if (wret < 0)
2211 ret = wret;
2212 if (wret == 0) {
Chris Mason5f39d392007-10-15 16:14:19 -04002213 struct btrfs_disk_key disk_key;
2214
2215 btrfs_node_key(right, &disk_key, 0);
Jan Schmidtf2304752012-05-26 11:43:17 +02002216 tree_mod_log_set_node_key(root->fs_info, parent,
Liu Bo32adf092012-10-19 12:52:15 +00002217 pslot + 1, 0);
Chris Mason5f39d392007-10-15 16:14:19 -04002218 btrfs_set_node_key(parent, &disk_key, pslot + 1);
2219 btrfs_mark_buffer_dirty(parent);
2220
2221 if (btrfs_header_nritems(mid) <= orig_slot) {
2222 path->nodes[level] = right;
Chris Masone66f7092007-04-20 13:16:02 -04002223 path->slots[level + 1] += 1;
2224 path->slots[level] = orig_slot -
Chris Mason5f39d392007-10-15 16:14:19 -04002225 btrfs_header_nritems(mid);
Chris Mason925baed2008-06-25 16:01:30 -04002226 btrfs_tree_unlock(mid);
Chris Mason5f39d392007-10-15 16:14:19 -04002227 free_extent_buffer(mid);
Chris Masone66f7092007-04-20 13:16:02 -04002228 } else {
Chris Mason925baed2008-06-25 16:01:30 -04002229 btrfs_tree_unlock(right);
Chris Mason5f39d392007-10-15 16:14:19 -04002230 free_extent_buffer(right);
Chris Masone66f7092007-04-20 13:16:02 -04002231 }
Chris Masone66f7092007-04-20 13:16:02 -04002232 return 0;
2233 }
Chris Mason925baed2008-06-25 16:01:30 -04002234 btrfs_tree_unlock(right);
Chris Mason5f39d392007-10-15 16:14:19 -04002235 free_extent_buffer(right);
Chris Masone66f7092007-04-20 13:16:02 -04002236 }
Chris Masone66f7092007-04-20 13:16:02 -04002237 return 1;
2238}
2239
Chris Mason74123bd2007-02-02 11:05:29 -05002240/*
Chris Masond352ac62008-09-29 15:18:18 -04002241 * readahead one full node of leaves, finding things that are close
2242 * to the block in 'slot', and triggering ra on them.
Chris Mason3c69fae2007-08-07 15:52:22 -04002243 */
Chris Masonc8c42862009-04-03 10:14:18 -04002244static void reada_for_search(struct btrfs_root *root,
2245 struct btrfs_path *path,
2246 int level, int slot, u64 objectid)
Chris Mason3c69fae2007-08-07 15:52:22 -04002247{
Chris Mason5f39d392007-10-15 16:14:19 -04002248 struct extent_buffer *node;
Chris Mason01f46652007-12-21 16:24:26 -05002249 struct btrfs_disk_key disk_key;
Chris Mason3c69fae2007-08-07 15:52:22 -04002250 u32 nritems;
Chris Mason3c69fae2007-08-07 15:52:22 -04002251 u64 search;
Chris Masona7175312009-01-22 09:23:10 -05002252 u64 target;
Chris Mason6b800532007-10-15 16:17:34 -04002253 u64 nread = 0;
Josef Bacikcb25c2e2011-05-11 12:17:34 -04002254 u64 gen;
Chris Mason3c69fae2007-08-07 15:52:22 -04002255 int direction = path->reada;
Chris Mason5f39d392007-10-15 16:14:19 -04002256 struct extent_buffer *eb;
Chris Mason6b800532007-10-15 16:17:34 -04002257 u32 nr;
2258 u32 blocksize;
2259 u32 nscan = 0;
Chris Masondb945352007-10-15 16:15:53 -04002260
Chris Masona6b6e752007-10-15 16:22:39 -04002261 if (level != 1)
Chris Mason3c69fae2007-08-07 15:52:22 -04002262 return;
2263
Chris Mason6702ed42007-08-07 16:15:09 -04002264 if (!path->nodes[level])
2265 return;
2266
Chris Mason5f39d392007-10-15 16:14:19 -04002267 node = path->nodes[level];
Chris Mason925baed2008-06-25 16:01:30 -04002268
Chris Mason3c69fae2007-08-07 15:52:22 -04002269 search = btrfs_node_blockptr(node, slot);
Chris Mason6b800532007-10-15 16:17:34 -04002270 blocksize = btrfs_level_size(root, level - 1);
2271 eb = btrfs_find_tree_block(root, search, blocksize);
Chris Mason5f39d392007-10-15 16:14:19 -04002272 if (eb) {
2273 free_extent_buffer(eb);
Chris Mason3c69fae2007-08-07 15:52:22 -04002274 return;
2275 }
2276
Chris Masona7175312009-01-22 09:23:10 -05002277 target = search;
Chris Mason6b800532007-10-15 16:17:34 -04002278
Chris Mason5f39d392007-10-15 16:14:19 -04002279 nritems = btrfs_header_nritems(node);
Chris Mason6b800532007-10-15 16:17:34 -04002280 nr = slot;
Josef Bacik25b8b932011-06-08 14:36:54 -04002281
Chris Masond3977122009-01-05 21:25:51 -05002282 while (1) {
Chris Mason6b800532007-10-15 16:17:34 -04002283 if (direction < 0) {
2284 if (nr == 0)
2285 break;
2286 nr--;
2287 } else if (direction > 0) {
2288 nr++;
2289 if (nr >= nritems)
2290 break;
Chris Mason3c69fae2007-08-07 15:52:22 -04002291 }
Chris Mason01f46652007-12-21 16:24:26 -05002292 if (path->reada < 0 && objectid) {
2293 btrfs_node_key(node, &disk_key, nr);
2294 if (btrfs_disk_key_objectid(&disk_key) != objectid)
2295 break;
2296 }
Chris Mason6b800532007-10-15 16:17:34 -04002297 search = btrfs_node_blockptr(node, nr);
Chris Masona7175312009-01-22 09:23:10 -05002298 if ((search <= target && target - search <= 65536) ||
2299 (search > target && search - target <= 65536)) {
Josef Bacikcb25c2e2011-05-11 12:17:34 -04002300 gen = btrfs_node_ptr_generation(node, nr);
Josef Bacikcb25c2e2011-05-11 12:17:34 -04002301 readahead_tree_block(root, search, blocksize, gen);
Chris Mason6b800532007-10-15 16:17:34 -04002302 nread += blocksize;
2303 }
2304 nscan++;
Chris Masona7175312009-01-22 09:23:10 -05002305 if ((nread > 65536 || nscan > 32))
Chris Mason6b800532007-10-15 16:17:34 -04002306 break;
Chris Mason3c69fae2007-08-07 15:52:22 -04002307 }
2308}
Chris Mason925baed2008-06-25 16:01:30 -04002309
Josef Bacik0b088512013-06-17 14:23:02 -04002310static noinline void reada_for_balance(struct btrfs_root *root,
2311 struct btrfs_path *path, int level)
Chris Masonb4ce94d2009-02-04 09:25:08 -05002312{
2313 int slot;
2314 int nritems;
2315 struct extent_buffer *parent;
2316 struct extent_buffer *eb;
2317 u64 gen;
2318 u64 block1 = 0;
2319 u64 block2 = 0;
Chris Masonb4ce94d2009-02-04 09:25:08 -05002320 int blocksize;
2321
Chris Mason8c594ea2009-04-20 15:50:10 -04002322 parent = path->nodes[level + 1];
Chris Masonb4ce94d2009-02-04 09:25:08 -05002323 if (!parent)
Josef Bacik0b088512013-06-17 14:23:02 -04002324 return;
Chris Masonb4ce94d2009-02-04 09:25:08 -05002325
2326 nritems = btrfs_header_nritems(parent);
Chris Mason8c594ea2009-04-20 15:50:10 -04002327 slot = path->slots[level + 1];
Chris Masonb4ce94d2009-02-04 09:25:08 -05002328 blocksize = btrfs_level_size(root, level);
2329
2330 if (slot > 0) {
2331 block1 = btrfs_node_blockptr(parent, slot - 1);
2332 gen = btrfs_node_ptr_generation(parent, slot - 1);
2333 eb = btrfs_find_tree_block(root, block1, blocksize);
Chris Masonb9fab912012-05-06 07:23:47 -04002334 /*
2335 * if we get -eagain from btrfs_buffer_uptodate, we
2336 * don't want to return eagain here. That will loop
2337 * forever
2338 */
2339 if (eb && btrfs_buffer_uptodate(eb, gen, 1) != 0)
Chris Masonb4ce94d2009-02-04 09:25:08 -05002340 block1 = 0;
2341 free_extent_buffer(eb);
2342 }
Chris Mason8c594ea2009-04-20 15:50:10 -04002343 if (slot + 1 < nritems) {
Chris Masonb4ce94d2009-02-04 09:25:08 -05002344 block2 = btrfs_node_blockptr(parent, slot + 1);
2345 gen = btrfs_node_ptr_generation(parent, slot + 1);
2346 eb = btrfs_find_tree_block(root, block2, blocksize);
Chris Masonb9fab912012-05-06 07:23:47 -04002347 if (eb && btrfs_buffer_uptodate(eb, gen, 1) != 0)
Chris Masonb4ce94d2009-02-04 09:25:08 -05002348 block2 = 0;
2349 free_extent_buffer(eb);
2350 }
Chris Mason8c594ea2009-04-20 15:50:10 -04002351
Josef Bacik0b088512013-06-17 14:23:02 -04002352 if (block1)
2353 readahead_tree_block(root, block1, blocksize, 0);
2354 if (block2)
2355 readahead_tree_block(root, block2, blocksize, 0);
Chris Masonb4ce94d2009-02-04 09:25:08 -05002356}
2357
2358
2359/*
Chris Masond3977122009-01-05 21:25:51 -05002360 * when we walk down the tree, it is usually safe to unlock the higher layers
2361 * in the tree. The exceptions are when our path goes through slot 0, because
2362 * operations on the tree might require changing key pointers higher up in the
2363 * tree.
Chris Masond352ac62008-09-29 15:18:18 -04002364 *
Chris Masond3977122009-01-05 21:25:51 -05002365 * callers might also have set path->keep_locks, which tells this code to keep
2366 * the lock if the path points to the last slot in the block. This is part of
2367 * walking through the tree, and selecting the next slot in the higher block.
Chris Masond352ac62008-09-29 15:18:18 -04002368 *
Chris Masond3977122009-01-05 21:25:51 -05002369 * lowest_unlock sets the lowest level in the tree we're allowed to unlock. so
2370 * if lowest_unlock is 1, level 0 won't be unlocked
Chris Masond352ac62008-09-29 15:18:18 -04002371 */
Chris Masone02119d2008-09-05 16:13:11 -04002372static noinline void unlock_up(struct btrfs_path *path, int level,
Chris Masonf7c79f32012-03-19 15:54:38 -04002373 int lowest_unlock, int min_write_lock_level,
2374 int *write_lock_level)
Chris Mason925baed2008-06-25 16:01:30 -04002375{
2376 int i;
2377 int skip_level = level;
Chris Mason051e1b92008-06-25 16:01:30 -04002378 int no_skips = 0;
Chris Mason925baed2008-06-25 16:01:30 -04002379 struct extent_buffer *t;
2380
2381 for (i = level; i < BTRFS_MAX_LEVEL; i++) {
2382 if (!path->nodes[i])
2383 break;
2384 if (!path->locks[i])
2385 break;
Chris Mason051e1b92008-06-25 16:01:30 -04002386 if (!no_skips && path->slots[i] == 0) {
Chris Mason925baed2008-06-25 16:01:30 -04002387 skip_level = i + 1;
2388 continue;
2389 }
Chris Mason051e1b92008-06-25 16:01:30 -04002390 if (!no_skips && path->keep_locks) {
Chris Mason925baed2008-06-25 16:01:30 -04002391 u32 nritems;
2392 t = path->nodes[i];
2393 nritems = btrfs_header_nritems(t);
Chris Mason051e1b92008-06-25 16:01:30 -04002394 if (nritems < 1 || path->slots[i] >= nritems - 1) {
Chris Mason925baed2008-06-25 16:01:30 -04002395 skip_level = i + 1;
2396 continue;
2397 }
2398 }
Chris Mason051e1b92008-06-25 16:01:30 -04002399 if (skip_level < i && i >= lowest_unlock)
2400 no_skips = 1;
2401
Chris Mason925baed2008-06-25 16:01:30 -04002402 t = path->nodes[i];
2403 if (i >= lowest_unlock && i > skip_level && path->locks[i]) {
Chris Masonbd681512011-07-16 15:23:14 -04002404 btrfs_tree_unlock_rw(t, path->locks[i]);
Chris Mason925baed2008-06-25 16:01:30 -04002405 path->locks[i] = 0;
Chris Masonf7c79f32012-03-19 15:54:38 -04002406 if (write_lock_level &&
2407 i > min_write_lock_level &&
2408 i <= *write_lock_level) {
2409 *write_lock_level = i - 1;
2410 }
Chris Mason925baed2008-06-25 16:01:30 -04002411 }
2412 }
2413}
2414
Chris Mason3c69fae2007-08-07 15:52:22 -04002415/*
Chris Masonb4ce94d2009-02-04 09:25:08 -05002416 * This releases any locks held in the path starting at level and
2417 * going all the way up to the root.
2418 *
2419 * btrfs_search_slot will keep the lock held on higher nodes in a few
2420 * corner cases, such as COW of the block at slot zero in the node. This
2421 * ignores those rules, and it should only be called when there are no
2422 * more updates to be done higher up in the tree.
2423 */
2424noinline void btrfs_unlock_up_safe(struct btrfs_path *path, int level)
2425{
2426 int i;
2427
Josef Bacik09a2a8f92013-04-05 16:51:15 -04002428 if (path->keep_locks)
Chris Masonb4ce94d2009-02-04 09:25:08 -05002429 return;
2430
2431 for (i = level; i < BTRFS_MAX_LEVEL; i++) {
2432 if (!path->nodes[i])
Chris Mason12f4dac2009-02-04 09:31:42 -05002433 continue;
Chris Masonb4ce94d2009-02-04 09:25:08 -05002434 if (!path->locks[i])
Chris Mason12f4dac2009-02-04 09:31:42 -05002435 continue;
Chris Masonbd681512011-07-16 15:23:14 -04002436 btrfs_tree_unlock_rw(path->nodes[i], path->locks[i]);
Chris Masonb4ce94d2009-02-04 09:25:08 -05002437 path->locks[i] = 0;
2438 }
2439}
2440
2441/*
Chris Masonc8c42862009-04-03 10:14:18 -04002442 * helper function for btrfs_search_slot. The goal is to find a block
2443 * in cache without setting the path to blocking. If we find the block
2444 * we return zero and the path is unchanged.
2445 *
2446 * If we can't find the block, we set the path blocking and do some
2447 * reada. -EAGAIN is returned and the search must be repeated.
2448 */
2449static int
2450read_block_for_search(struct btrfs_trans_handle *trans,
2451 struct btrfs_root *root, struct btrfs_path *p,
2452 struct extent_buffer **eb_ret, int level, int slot,
Jan Schmidt5d9e75c42012-05-16 18:25:47 +02002453 struct btrfs_key *key, u64 time_seq)
Chris Masonc8c42862009-04-03 10:14:18 -04002454{
2455 u64 blocknr;
2456 u64 gen;
2457 u32 blocksize;
2458 struct extent_buffer *b = *eb_ret;
2459 struct extent_buffer *tmp;
Chris Mason76a05b32009-05-14 13:24:30 -04002460 int ret;
Chris Masonc8c42862009-04-03 10:14:18 -04002461
2462 blocknr = btrfs_node_blockptr(b, slot);
2463 gen = btrfs_node_ptr_generation(b, slot);
2464 blocksize = btrfs_level_size(root, level - 1);
2465
2466 tmp = btrfs_find_tree_block(root, blocknr, blocksize);
Chris Masoncb449212010-10-24 11:01:27 -04002467 if (tmp) {
Chris Masonb9fab912012-05-06 07:23:47 -04002468 /* first we do an atomic uptodate check */
Josef Bacikbdf7c002013-06-17 13:44:48 -04002469 if (btrfs_buffer_uptodate(tmp, gen, 1) > 0) {
2470 *eb_ret = tmp;
2471 return 0;
Chris Masoncb449212010-10-24 11:01:27 -04002472 }
Josef Bacikbdf7c002013-06-17 13:44:48 -04002473
2474 /* the pages were up to date, but we failed
2475 * the generation number check. Do a full
2476 * read for the generation number that is correct.
2477 * We must do this without dropping locks so
2478 * we can trust our generation number
2479 */
2480 btrfs_set_path_blocking(p);
2481
2482 /* now we're allowed to do a blocking uptodate check */
2483 ret = btrfs_read_buffer(tmp, gen);
2484 if (!ret) {
2485 *eb_ret = tmp;
2486 return 0;
2487 }
2488 free_extent_buffer(tmp);
2489 btrfs_release_path(p);
2490 return -EIO;
Chris Masonc8c42862009-04-03 10:14:18 -04002491 }
2492
2493 /*
2494 * reduce lock contention at high levels
2495 * of the btree by dropping locks before
Chris Mason76a05b32009-05-14 13:24:30 -04002496 * we read. Don't release the lock on the current
2497 * level because we need to walk this node to figure
2498 * out which blocks to read.
Chris Masonc8c42862009-04-03 10:14:18 -04002499 */
Chris Mason8c594ea2009-04-20 15:50:10 -04002500 btrfs_unlock_up_safe(p, level + 1);
2501 btrfs_set_path_blocking(p);
2502
Chris Masoncb449212010-10-24 11:01:27 -04002503 free_extent_buffer(tmp);
Chris Masonc8c42862009-04-03 10:14:18 -04002504 if (p->reada)
2505 reada_for_search(root, p, level, slot, key->objectid);
2506
David Sterbab3b4aa72011-04-21 01:20:15 +02002507 btrfs_release_path(p);
Chris Mason76a05b32009-05-14 13:24:30 -04002508
2509 ret = -EAGAIN;
Yan, Zheng5bdd3532010-05-26 11:20:30 -04002510 tmp = read_tree_block(root, blocknr, blocksize, 0);
Chris Mason76a05b32009-05-14 13:24:30 -04002511 if (tmp) {
2512 /*
2513 * If the read above didn't mark this buffer up to date,
2514 * it will never end up being up to date. Set ret to EIO now
2515 * and give up so that our caller doesn't loop forever
2516 * on our EAGAINs.
2517 */
Chris Masonb9fab912012-05-06 07:23:47 -04002518 if (!btrfs_buffer_uptodate(tmp, 0, 0))
Chris Mason76a05b32009-05-14 13:24:30 -04002519 ret = -EIO;
Chris Masonc8c42862009-04-03 10:14:18 -04002520 free_extent_buffer(tmp);
Chris Mason76a05b32009-05-14 13:24:30 -04002521 }
2522 return ret;
Chris Masonc8c42862009-04-03 10:14:18 -04002523}
2524
2525/*
2526 * helper function for btrfs_search_slot. This does all of the checks
2527 * for node-level blocks and does any balancing required based on
2528 * the ins_len.
2529 *
2530 * If no extra work was required, zero is returned. If we had to
2531 * drop the path, -EAGAIN is returned and btrfs_search_slot must
2532 * start over
2533 */
2534static int
2535setup_nodes_for_search(struct btrfs_trans_handle *trans,
2536 struct btrfs_root *root, struct btrfs_path *p,
Chris Masonbd681512011-07-16 15:23:14 -04002537 struct extent_buffer *b, int level, int ins_len,
2538 int *write_lock_level)
Chris Masonc8c42862009-04-03 10:14:18 -04002539{
2540 int ret;
2541 if ((p->search_for_split || ins_len > 0) && btrfs_header_nritems(b) >=
2542 BTRFS_NODEPTRS_PER_BLOCK(root) - 3) {
2543 int sret;
2544
Chris Masonbd681512011-07-16 15:23:14 -04002545 if (*write_lock_level < level + 1) {
2546 *write_lock_level = level + 1;
2547 btrfs_release_path(p);
2548 goto again;
2549 }
2550
Chris Masonc8c42862009-04-03 10:14:18 -04002551 btrfs_set_path_blocking(p);
Josef Bacik0b088512013-06-17 14:23:02 -04002552 reada_for_balance(root, p, level);
Chris Masonc8c42862009-04-03 10:14:18 -04002553 sret = split_node(trans, root, p, level);
Chris Masonbd681512011-07-16 15:23:14 -04002554 btrfs_clear_path_blocking(p, NULL, 0);
Chris Masonc8c42862009-04-03 10:14:18 -04002555
2556 BUG_ON(sret > 0);
2557 if (sret) {
2558 ret = sret;
2559 goto done;
2560 }
2561 b = p->nodes[level];
2562 } else if (ins_len < 0 && btrfs_header_nritems(b) <
Chris Masoncfbb9302009-05-18 10:41:58 -04002563 BTRFS_NODEPTRS_PER_BLOCK(root) / 2) {
Chris Masonc8c42862009-04-03 10:14:18 -04002564 int sret;
2565
Chris Masonbd681512011-07-16 15:23:14 -04002566 if (*write_lock_level < level + 1) {
2567 *write_lock_level = level + 1;
2568 btrfs_release_path(p);
2569 goto again;
2570 }
2571
Chris Masonc8c42862009-04-03 10:14:18 -04002572 btrfs_set_path_blocking(p);
Josef Bacik0b088512013-06-17 14:23:02 -04002573 reada_for_balance(root, p, level);
Chris Masonc8c42862009-04-03 10:14:18 -04002574 sret = balance_level(trans, root, p, level);
Chris Masonbd681512011-07-16 15:23:14 -04002575 btrfs_clear_path_blocking(p, NULL, 0);
Chris Masonc8c42862009-04-03 10:14:18 -04002576
2577 if (sret) {
2578 ret = sret;
2579 goto done;
2580 }
2581 b = p->nodes[level];
2582 if (!b) {
David Sterbab3b4aa72011-04-21 01:20:15 +02002583 btrfs_release_path(p);
Chris Masonc8c42862009-04-03 10:14:18 -04002584 goto again;
2585 }
2586 BUG_ON(btrfs_header_nritems(b) == 1);
2587 }
2588 return 0;
2589
2590again:
2591 ret = -EAGAIN;
2592done:
2593 return ret;
2594}
2595
Filipe David Borba Mananad7396f02013-08-30 15:46:43 +01002596static void key_search_validate(struct extent_buffer *b,
2597 struct btrfs_key *key,
2598 int level)
2599{
2600#ifdef CONFIG_BTRFS_ASSERT
2601 struct btrfs_disk_key disk_key;
2602
2603 btrfs_cpu_key_to_disk(&disk_key, key);
2604
2605 if (level == 0)
2606 ASSERT(!memcmp_extent_buffer(b, &disk_key,
2607 offsetof(struct btrfs_leaf, items[0].key),
2608 sizeof(disk_key)));
2609 else
2610 ASSERT(!memcmp_extent_buffer(b, &disk_key,
2611 offsetof(struct btrfs_node, ptrs[0].key),
2612 sizeof(disk_key)));
2613#endif
2614}
2615
2616static int key_search(struct extent_buffer *b, struct btrfs_key *key,
2617 int level, int *prev_cmp, int *slot)
2618{
2619 if (*prev_cmp != 0) {
2620 *prev_cmp = bin_search(b, key, level, slot);
2621 return *prev_cmp;
2622 }
2623
2624 key_search_validate(b, key, level);
2625 *slot = 0;
2626
2627 return 0;
2628}
2629
Kelley Nielsen3f870c22013-11-04 19:37:39 -08002630int btrfs_find_item(struct btrfs_root *fs_root, struct btrfs_path *found_path,
Kelley Nielsene33d5c32013-11-04 19:33:33 -08002631 u64 iobjectid, u64 ioff, u8 key_type,
2632 struct btrfs_key *found_key)
2633{
2634 int ret;
2635 struct btrfs_key key;
2636 struct extent_buffer *eb;
Kelley Nielsen3f870c22013-11-04 19:37:39 -08002637 struct btrfs_path *path;
Kelley Nielsene33d5c32013-11-04 19:33:33 -08002638
2639 key.type = key_type;
2640 key.objectid = iobjectid;
2641 key.offset = ioff;
2642
Kelley Nielsen3f870c22013-11-04 19:37:39 -08002643 if (found_path == NULL) {
2644 path = btrfs_alloc_path();
2645 if (!path)
2646 return -ENOMEM;
2647 } else
2648 path = found_path;
2649
Kelley Nielsene33d5c32013-11-04 19:33:33 -08002650 ret = btrfs_search_slot(NULL, fs_root, &key, path, 0, 0);
Kelley Nielsen3f870c22013-11-04 19:37:39 -08002651 if ((ret < 0) || (found_key == NULL)) {
2652 if (path != found_path)
2653 btrfs_free_path(path);
Kelley Nielsene33d5c32013-11-04 19:33:33 -08002654 return ret;
Kelley Nielsen3f870c22013-11-04 19:37:39 -08002655 }
Kelley Nielsene33d5c32013-11-04 19:33:33 -08002656
2657 eb = path->nodes[0];
2658 if (ret && path->slots[0] >= btrfs_header_nritems(eb)) {
2659 ret = btrfs_next_leaf(fs_root, path);
2660 if (ret)
2661 return ret;
2662 eb = path->nodes[0];
2663 }
2664
2665 btrfs_item_key_to_cpu(eb, found_key, path->slots[0]);
2666 if (found_key->type != key.type ||
2667 found_key->objectid != key.objectid)
2668 return 1;
2669
2670 return 0;
2671}
2672
Chris Masonc8c42862009-04-03 10:14:18 -04002673/*
Chris Mason74123bd2007-02-02 11:05:29 -05002674 * look for key in the tree. path is filled in with nodes along the way
2675 * if key is found, we return zero and you can find the item in the leaf
2676 * level of the path (level 0)
2677 *
2678 * If the key isn't found, the path points to the slot where it should
Chris Masonaa5d6be2007-02-28 16:35:06 -05002679 * be inserted, and 1 is returned. If there are other errors during the
2680 * search a negative error number is returned.
Chris Mason97571fd2007-02-24 13:39:08 -05002681 *
2682 * if ins_len > 0, nodes and leaves will be split as we walk down the
2683 * tree. if ins_len < 0, nodes will be merged as we walk down the tree (if
2684 * possible)
Chris Mason74123bd2007-02-02 11:05:29 -05002685 */
Chris Masone089f052007-03-16 16:20:31 -04002686int btrfs_search_slot(struct btrfs_trans_handle *trans, struct btrfs_root
2687 *root, struct btrfs_key *key, struct btrfs_path *p, int
2688 ins_len, int cow)
Chris Masonbe0e5c02007-01-26 15:51:26 -05002689{
Chris Mason5f39d392007-10-15 16:14:19 -04002690 struct extent_buffer *b;
Chris Masonbe0e5c02007-01-26 15:51:26 -05002691 int slot;
2692 int ret;
Yan Zheng33c66f42009-07-22 09:59:00 -04002693 int err;
Chris Masonbe0e5c02007-01-26 15:51:26 -05002694 int level;
Chris Mason925baed2008-06-25 16:01:30 -04002695 int lowest_unlock = 1;
Chris Masonbd681512011-07-16 15:23:14 -04002696 int root_lock;
2697 /* everything at write_lock_level or lower must be write locked */
2698 int write_lock_level = 0;
Chris Mason9f3a7422007-08-07 15:52:19 -04002699 u8 lowest_level = 0;
Chris Masonf7c79f32012-03-19 15:54:38 -04002700 int min_write_lock_level;
Filipe David Borba Mananad7396f02013-08-30 15:46:43 +01002701 int prev_cmp;
Chris Mason9f3a7422007-08-07 15:52:19 -04002702
Chris Mason6702ed42007-08-07 16:15:09 -04002703 lowest_level = p->lowest_level;
Chris Mason323ac952008-10-01 19:05:46 -04002704 WARN_ON(lowest_level && ins_len > 0);
Chris Mason22b0ebd2007-03-30 08:47:31 -04002705 WARN_ON(p->nodes[0] != NULL);
Filipe David Borba Mananaeb653de2013-12-23 11:53:02 +00002706 BUG_ON(!cow && ins_len);
Josef Bacik25179202008-10-29 14:49:05 -04002707
Chris Masonbd681512011-07-16 15:23:14 -04002708 if (ins_len < 0) {
Chris Mason925baed2008-06-25 16:01:30 -04002709 lowest_unlock = 2;
Chris Mason65b51a02008-08-01 15:11:20 -04002710
Chris Masonbd681512011-07-16 15:23:14 -04002711 /* when we are removing items, we might have to go up to level
2712 * two as we update tree pointers Make sure we keep write
2713 * for those levels as well
2714 */
2715 write_lock_level = 2;
2716 } else if (ins_len > 0) {
2717 /*
2718 * for inserting items, make sure we have a write lock on
2719 * level 1 so we can update keys
2720 */
2721 write_lock_level = 1;
2722 }
2723
2724 if (!cow)
2725 write_lock_level = -1;
2726
Josef Bacik09a2a8f92013-04-05 16:51:15 -04002727 if (cow && (p->keep_locks || p->lowest_level))
Chris Masonbd681512011-07-16 15:23:14 -04002728 write_lock_level = BTRFS_MAX_LEVEL;
2729
Chris Masonf7c79f32012-03-19 15:54:38 -04002730 min_write_lock_level = write_lock_level;
2731
Chris Masonbb803952007-03-01 12:04:21 -05002732again:
Filipe David Borba Mananad7396f02013-08-30 15:46:43 +01002733 prev_cmp = -1;
Chris Masonbd681512011-07-16 15:23:14 -04002734 /*
2735 * we try very hard to do read locks on the root
2736 */
2737 root_lock = BTRFS_READ_LOCK;
2738 level = 0;
Yan Zheng5d4f98a2009-06-10 10:45:14 -04002739 if (p->search_commit_root) {
Chris Masonbd681512011-07-16 15:23:14 -04002740 /*
2741 * the commit roots are read only
2742 * so we always do read locks
2743 */
Josef Bacik3f8a18c2014-03-28 17:16:01 -04002744 if (p->need_commit_sem)
2745 down_read(&root->fs_info->commit_root_sem);
Yan Zheng5d4f98a2009-06-10 10:45:14 -04002746 b = root->commit_root;
2747 extent_buffer_get(b);
Chris Masonbd681512011-07-16 15:23:14 -04002748 level = btrfs_header_level(b);
Josef Bacik3f8a18c2014-03-28 17:16:01 -04002749 if (p->need_commit_sem)
2750 up_read(&root->fs_info->commit_root_sem);
Yan Zheng5d4f98a2009-06-10 10:45:14 -04002751 if (!p->skip_locking)
Chris Masonbd681512011-07-16 15:23:14 -04002752 btrfs_tree_read_lock(b);
Yan Zheng5d4f98a2009-06-10 10:45:14 -04002753 } else {
Chris Masonbd681512011-07-16 15:23:14 -04002754 if (p->skip_locking) {
Yan Zheng5d4f98a2009-06-10 10:45:14 -04002755 b = btrfs_root_node(root);
Chris Masonbd681512011-07-16 15:23:14 -04002756 level = btrfs_header_level(b);
2757 } else {
2758 /* we don't know the level of the root node
2759 * until we actually have it read locked
2760 */
2761 b = btrfs_read_lock_root_node(root);
2762 level = btrfs_header_level(b);
2763 if (level <= write_lock_level) {
2764 /* whoops, must trade for write lock */
2765 btrfs_tree_read_unlock(b);
2766 free_extent_buffer(b);
2767 b = btrfs_lock_root_node(root);
2768 root_lock = BTRFS_WRITE_LOCK;
2769
2770 /* the level might have changed, check again */
2771 level = btrfs_header_level(b);
2772 }
2773 }
Yan Zheng5d4f98a2009-06-10 10:45:14 -04002774 }
Chris Masonbd681512011-07-16 15:23:14 -04002775 p->nodes[level] = b;
2776 if (!p->skip_locking)
2777 p->locks[level] = root_lock;
Chris Mason925baed2008-06-25 16:01:30 -04002778
Chris Masoneb60cea2007-02-02 09:18:22 -05002779 while (b) {
Chris Mason5f39d392007-10-15 16:14:19 -04002780 level = btrfs_header_level(b);
Chris Mason65b51a02008-08-01 15:11:20 -04002781
2782 /*
2783 * setup the path here so we can release it under lock
2784 * contention with the cow code
2785 */
Chris Mason02217ed2007-03-02 16:08:05 -05002786 if (cow) {
Chris Masonc8c42862009-04-03 10:14:18 -04002787 /*
2788 * if we don't really need to cow this block
2789 * then we don't want to set the path blocking,
2790 * so we test it here
2791 */
Yan Zheng5d4f98a2009-06-10 10:45:14 -04002792 if (!should_cow_block(trans, root, b))
Chris Mason65b51a02008-08-01 15:11:20 -04002793 goto cow_done;
Yan Zheng5d4f98a2009-06-10 10:45:14 -04002794
Chris Masonb4ce94d2009-02-04 09:25:08 -05002795 btrfs_set_path_blocking(p);
2796
Chris Masonbd681512011-07-16 15:23:14 -04002797 /*
2798 * must have write locks on this node and the
2799 * parent
2800 */
Josef Bacik5124e002012-11-07 13:44:13 -05002801 if (level > write_lock_level ||
2802 (level + 1 > write_lock_level &&
2803 level + 1 < BTRFS_MAX_LEVEL &&
2804 p->nodes[level + 1])) {
Chris Masonbd681512011-07-16 15:23:14 -04002805 write_lock_level = level + 1;
2806 btrfs_release_path(p);
2807 goto again;
2808 }
2809
Yan Zheng33c66f42009-07-22 09:59:00 -04002810 err = btrfs_cow_block(trans, root, b,
2811 p->nodes[level + 1],
2812 p->slots[level + 1], &b);
2813 if (err) {
Yan Zheng33c66f42009-07-22 09:59:00 -04002814 ret = err;
Chris Mason65b51a02008-08-01 15:11:20 -04002815 goto done;
Chris Mason54aa1f42007-06-22 14:16:25 -04002816 }
Chris Mason02217ed2007-03-02 16:08:05 -05002817 }
Chris Mason65b51a02008-08-01 15:11:20 -04002818cow_done:
Chris Masoneb60cea2007-02-02 09:18:22 -05002819 p->nodes[level] = b;
Chris Masonbd681512011-07-16 15:23:14 -04002820 btrfs_clear_path_blocking(p, NULL, 0);
Chris Masonb4ce94d2009-02-04 09:25:08 -05002821
2822 /*
2823 * we have a lock on b and as long as we aren't changing
2824 * the tree, there is no way to for the items in b to change.
2825 * It is safe to drop the lock on our parent before we
2826 * go through the expensive btree search on b.
2827 *
Filipe David Borba Mananaeb653de2013-12-23 11:53:02 +00002828 * If we're inserting or deleting (ins_len != 0), then we might
2829 * be changing slot zero, which may require changing the parent.
2830 * So, we can't drop the lock until after we know which slot
2831 * we're operating on.
Chris Masonb4ce94d2009-02-04 09:25:08 -05002832 */
Filipe David Borba Mananaeb653de2013-12-23 11:53:02 +00002833 if (!ins_len && !p->keep_locks) {
2834 int u = level + 1;
2835
2836 if (u < BTRFS_MAX_LEVEL && p->locks[u]) {
2837 btrfs_tree_unlock_rw(p->nodes[u], p->locks[u]);
2838 p->locks[u] = 0;
2839 }
2840 }
Chris Masonb4ce94d2009-02-04 09:25:08 -05002841
Filipe David Borba Mananad7396f02013-08-30 15:46:43 +01002842 ret = key_search(b, key, level, &prev_cmp, &slot);
Chris Masonb4ce94d2009-02-04 09:25:08 -05002843
Chris Mason5f39d392007-10-15 16:14:19 -04002844 if (level != 0) {
Yan Zheng33c66f42009-07-22 09:59:00 -04002845 int dec = 0;
2846 if (ret && slot > 0) {
2847 dec = 1;
Chris Masonbe0e5c02007-01-26 15:51:26 -05002848 slot -= 1;
Yan Zheng33c66f42009-07-22 09:59:00 -04002849 }
Chris Masonbe0e5c02007-01-26 15:51:26 -05002850 p->slots[level] = slot;
Yan Zheng33c66f42009-07-22 09:59:00 -04002851 err = setup_nodes_for_search(trans, root, p, b, level,
Chris Masonbd681512011-07-16 15:23:14 -04002852 ins_len, &write_lock_level);
Yan Zheng33c66f42009-07-22 09:59:00 -04002853 if (err == -EAGAIN)
Chris Masonc8c42862009-04-03 10:14:18 -04002854 goto again;
Yan Zheng33c66f42009-07-22 09:59:00 -04002855 if (err) {
2856 ret = err;
Chris Masonc8c42862009-04-03 10:14:18 -04002857 goto done;
Yan Zheng33c66f42009-07-22 09:59:00 -04002858 }
Chris Masonc8c42862009-04-03 10:14:18 -04002859 b = p->nodes[level];
2860 slot = p->slots[level];
Chris Masonb4ce94d2009-02-04 09:25:08 -05002861
Chris Masonbd681512011-07-16 15:23:14 -04002862 /*
2863 * slot 0 is special, if we change the key
2864 * we have to update the parent pointer
2865 * which means we must have a write lock
2866 * on the parent
2867 */
Filipe David Borba Mananaeb653de2013-12-23 11:53:02 +00002868 if (slot == 0 && ins_len &&
Chris Masonbd681512011-07-16 15:23:14 -04002869 write_lock_level < level + 1) {
2870 write_lock_level = level + 1;
2871 btrfs_release_path(p);
2872 goto again;
2873 }
2874
Chris Masonf7c79f32012-03-19 15:54:38 -04002875 unlock_up(p, level, lowest_unlock,
2876 min_write_lock_level, &write_lock_level);
Chris Masonf9efa9c2008-06-25 16:14:04 -04002877
Chris Mason925baed2008-06-25 16:01:30 -04002878 if (level == lowest_level) {
Yan Zheng33c66f42009-07-22 09:59:00 -04002879 if (dec)
2880 p->slots[level]++;
Zheng Yan5b21f2e2008-09-26 10:05:38 -04002881 goto done;
Chris Mason925baed2008-06-25 16:01:30 -04002882 }
Chris Masonca7a79a2008-05-12 12:59:19 -04002883
Yan Zheng33c66f42009-07-22 09:59:00 -04002884 err = read_block_for_search(trans, root, p,
Jan Schmidt5d9e75c42012-05-16 18:25:47 +02002885 &b, level, slot, key, 0);
Yan Zheng33c66f42009-07-22 09:59:00 -04002886 if (err == -EAGAIN)
Chris Masonc8c42862009-04-03 10:14:18 -04002887 goto again;
Yan Zheng33c66f42009-07-22 09:59:00 -04002888 if (err) {
2889 ret = err;
Chris Mason76a05b32009-05-14 13:24:30 -04002890 goto done;
Yan Zheng33c66f42009-07-22 09:59:00 -04002891 }
Chris Mason76a05b32009-05-14 13:24:30 -04002892
Chris Masonb4ce94d2009-02-04 09:25:08 -05002893 if (!p->skip_locking) {
Chris Masonbd681512011-07-16 15:23:14 -04002894 level = btrfs_header_level(b);
2895 if (level <= write_lock_level) {
2896 err = btrfs_try_tree_write_lock(b);
2897 if (!err) {
2898 btrfs_set_path_blocking(p);
2899 btrfs_tree_lock(b);
2900 btrfs_clear_path_blocking(p, b,
2901 BTRFS_WRITE_LOCK);
2902 }
2903 p->locks[level] = BTRFS_WRITE_LOCK;
2904 } else {
2905 err = btrfs_try_tree_read_lock(b);
2906 if (!err) {
2907 btrfs_set_path_blocking(p);
2908 btrfs_tree_read_lock(b);
2909 btrfs_clear_path_blocking(p, b,
2910 BTRFS_READ_LOCK);
2911 }
2912 p->locks[level] = BTRFS_READ_LOCK;
Chris Masonb4ce94d2009-02-04 09:25:08 -05002913 }
Chris Masonbd681512011-07-16 15:23:14 -04002914 p->nodes[level] = b;
Chris Masonb4ce94d2009-02-04 09:25:08 -05002915 }
Chris Masonbe0e5c02007-01-26 15:51:26 -05002916 } else {
2917 p->slots[level] = slot;
Yan Zheng87b29b22008-12-17 10:21:48 -05002918 if (ins_len > 0 &&
2919 btrfs_leaf_free_space(root, b) < ins_len) {
Chris Masonbd681512011-07-16 15:23:14 -04002920 if (write_lock_level < 1) {
2921 write_lock_level = 1;
2922 btrfs_release_path(p);
2923 goto again;
2924 }
2925
Chris Masonb4ce94d2009-02-04 09:25:08 -05002926 btrfs_set_path_blocking(p);
Yan Zheng33c66f42009-07-22 09:59:00 -04002927 err = split_leaf(trans, root, key,
2928 p, ins_len, ret == 0);
Chris Masonbd681512011-07-16 15:23:14 -04002929 btrfs_clear_path_blocking(p, NULL, 0);
Chris Masonb4ce94d2009-02-04 09:25:08 -05002930
Yan Zheng33c66f42009-07-22 09:59:00 -04002931 BUG_ON(err > 0);
2932 if (err) {
2933 ret = err;
Chris Mason65b51a02008-08-01 15:11:20 -04002934 goto done;
2935 }
Chris Mason5c680ed2007-02-22 11:39:13 -05002936 }
Chris Mason459931e2008-12-10 09:10:46 -05002937 if (!p->search_for_split)
Chris Masonf7c79f32012-03-19 15:54:38 -04002938 unlock_up(p, level, lowest_unlock,
2939 min_write_lock_level, &write_lock_level);
Chris Mason65b51a02008-08-01 15:11:20 -04002940 goto done;
Chris Masonbe0e5c02007-01-26 15:51:26 -05002941 }
2942 }
Chris Mason65b51a02008-08-01 15:11:20 -04002943 ret = 1;
2944done:
Chris Masonb4ce94d2009-02-04 09:25:08 -05002945 /*
2946 * we don't really know what they plan on doing with the path
2947 * from here on, so for now just mark it as blocking
2948 */
Chris Masonb9473432009-03-13 11:00:37 -04002949 if (!p->leave_spinning)
2950 btrfs_set_path_blocking(p);
Chris Mason76a05b32009-05-14 13:24:30 -04002951 if (ret < 0)
David Sterbab3b4aa72011-04-21 01:20:15 +02002952 btrfs_release_path(p);
Chris Mason65b51a02008-08-01 15:11:20 -04002953 return ret;
Chris Masonbe0e5c02007-01-26 15:51:26 -05002954}
2955
Chris Mason74123bd2007-02-02 11:05:29 -05002956/*
Jan Schmidt5d9e75c42012-05-16 18:25:47 +02002957 * Like btrfs_search_slot, this looks for a key in the given tree. It uses the
2958 * current state of the tree together with the operations recorded in the tree
2959 * modification log to search for the key in a previous version of this tree, as
2960 * denoted by the time_seq parameter.
2961 *
2962 * Naturally, there is no support for insert, delete or cow operations.
2963 *
2964 * The resulting path and return value will be set up as if we called
2965 * btrfs_search_slot at that point in time with ins_len and cow both set to 0.
2966 */
2967int btrfs_search_old_slot(struct btrfs_root *root, struct btrfs_key *key,
2968 struct btrfs_path *p, u64 time_seq)
2969{
2970 struct extent_buffer *b;
2971 int slot;
2972 int ret;
2973 int err;
2974 int level;
2975 int lowest_unlock = 1;
2976 u8 lowest_level = 0;
Josef Bacikd4b40872013-09-24 14:09:34 -04002977 int prev_cmp = -1;
Jan Schmidt5d9e75c42012-05-16 18:25:47 +02002978
2979 lowest_level = p->lowest_level;
2980 WARN_ON(p->nodes[0] != NULL);
2981
2982 if (p->search_commit_root) {
2983 BUG_ON(time_seq);
2984 return btrfs_search_slot(NULL, root, key, p, 0, 0);
2985 }
2986
2987again:
Jan Schmidt5d9e75c42012-05-16 18:25:47 +02002988 b = get_old_root(root, time_seq);
Jan Schmidt5d9e75c42012-05-16 18:25:47 +02002989 level = btrfs_header_level(b);
Jan Schmidt5d9e75c42012-05-16 18:25:47 +02002990 p->locks[level] = BTRFS_READ_LOCK;
2991
2992 while (b) {
2993 level = btrfs_header_level(b);
2994 p->nodes[level] = b;
2995 btrfs_clear_path_blocking(p, NULL, 0);
2996
2997 /*
2998 * we have a lock on b and as long as we aren't changing
2999 * the tree, there is no way to for the items in b to change.
3000 * It is safe to drop the lock on our parent before we
3001 * go through the expensive btree search on b.
3002 */
3003 btrfs_unlock_up_safe(p, level + 1);
3004
Josef Bacikd4b40872013-09-24 14:09:34 -04003005 /*
3006 * Since we can unwind eb's we want to do a real search every
3007 * time.
3008 */
3009 prev_cmp = -1;
Filipe David Borba Mananad7396f02013-08-30 15:46:43 +01003010 ret = key_search(b, key, level, &prev_cmp, &slot);
Jan Schmidt5d9e75c42012-05-16 18:25:47 +02003011
3012 if (level != 0) {
3013 int dec = 0;
3014 if (ret && slot > 0) {
3015 dec = 1;
3016 slot -= 1;
3017 }
3018 p->slots[level] = slot;
3019 unlock_up(p, level, lowest_unlock, 0, NULL);
3020
3021 if (level == lowest_level) {
3022 if (dec)
3023 p->slots[level]++;
3024 goto done;
3025 }
3026
3027 err = read_block_for_search(NULL, root, p, &b, level,
3028 slot, key, time_seq);
3029 if (err == -EAGAIN)
3030 goto again;
3031 if (err) {
3032 ret = err;
3033 goto done;
3034 }
3035
3036 level = btrfs_header_level(b);
3037 err = btrfs_try_tree_read_lock(b);
3038 if (!err) {
3039 btrfs_set_path_blocking(p);
3040 btrfs_tree_read_lock(b);
3041 btrfs_clear_path_blocking(p, b,
3042 BTRFS_READ_LOCK);
3043 }
Josef Bacik9ec72672013-08-07 16:57:23 -04003044 b = tree_mod_log_rewind(root->fs_info, p, b, time_seq);
Josef Bacikdb7f3432013-08-07 14:54:37 -04003045 if (!b) {
3046 ret = -ENOMEM;
3047 goto done;
3048 }
Jan Schmidt5d9e75c42012-05-16 18:25:47 +02003049 p->locks[level] = BTRFS_READ_LOCK;
3050 p->nodes[level] = b;
Jan Schmidt5d9e75c42012-05-16 18:25:47 +02003051 } else {
3052 p->slots[level] = slot;
3053 unlock_up(p, level, lowest_unlock, 0, NULL);
3054 goto done;
3055 }
3056 }
3057 ret = 1;
3058done:
3059 if (!p->leave_spinning)
3060 btrfs_set_path_blocking(p);
3061 if (ret < 0)
3062 btrfs_release_path(p);
3063
3064 return ret;
3065}
3066
3067/*
Arne Jansen2f38b3e2011-09-13 11:18:10 +02003068 * helper to use instead of search slot if no exact match is needed but
3069 * instead the next or previous item should be returned.
3070 * When find_higher is true, the next higher item is returned, the next lower
3071 * otherwise.
3072 * When return_any and find_higher are both true, and no higher item is found,
3073 * return the next lower instead.
3074 * When return_any is true and find_higher is false, and no lower item is found,
3075 * return the next higher instead.
3076 * It returns 0 if any item is found, 1 if none is found (tree empty), and
3077 * < 0 on error
3078 */
3079int btrfs_search_slot_for_read(struct btrfs_root *root,
3080 struct btrfs_key *key, struct btrfs_path *p,
3081 int find_higher, int return_any)
3082{
3083 int ret;
3084 struct extent_buffer *leaf;
3085
3086again:
3087 ret = btrfs_search_slot(NULL, root, key, p, 0, 0);
3088 if (ret <= 0)
3089 return ret;
3090 /*
3091 * a return value of 1 means the path is at the position where the
3092 * item should be inserted. Normally this is the next bigger item,
3093 * but in case the previous item is the last in a leaf, path points
3094 * to the first free slot in the previous leaf, i.e. at an invalid
3095 * item.
3096 */
3097 leaf = p->nodes[0];
3098
3099 if (find_higher) {
3100 if (p->slots[0] >= btrfs_header_nritems(leaf)) {
3101 ret = btrfs_next_leaf(root, p);
3102 if (ret <= 0)
3103 return ret;
3104 if (!return_any)
3105 return 1;
3106 /*
3107 * no higher item found, return the next
3108 * lower instead
3109 */
3110 return_any = 0;
3111 find_higher = 0;
3112 btrfs_release_path(p);
3113 goto again;
3114 }
3115 } else {
Arne Jansene6793762011-09-13 11:18:10 +02003116 if (p->slots[0] == 0) {
3117 ret = btrfs_prev_leaf(root, p);
3118 if (ret < 0)
3119 return ret;
3120 if (!ret) {
Filipe David Borba Manana23c6bf62014-01-11 21:28:54 +00003121 leaf = p->nodes[0];
3122 if (p->slots[0] == btrfs_header_nritems(leaf))
3123 p->slots[0]--;
Arne Jansene6793762011-09-13 11:18:10 +02003124 return 0;
Arne Jansen2f38b3e2011-09-13 11:18:10 +02003125 }
Arne Jansene6793762011-09-13 11:18:10 +02003126 if (!return_any)
3127 return 1;
3128 /*
3129 * no lower item found, return the next
3130 * higher instead
3131 */
3132 return_any = 0;
3133 find_higher = 1;
3134 btrfs_release_path(p);
3135 goto again;
3136 } else {
Arne Jansen2f38b3e2011-09-13 11:18:10 +02003137 --p->slots[0];
3138 }
3139 }
3140 return 0;
3141}
3142
3143/*
Chris Mason74123bd2007-02-02 11:05:29 -05003144 * adjust the pointers going up the tree, starting at level
3145 * making sure the right key of each node is points to 'key'.
3146 * This is used after shifting pointers to the left, so it stops
3147 * fixing up pointers when a given leaf/node is not in slot 0 of the
3148 * higher levels
Chris Masonaa5d6be2007-02-28 16:35:06 -05003149 *
Chris Mason74123bd2007-02-02 11:05:29 -05003150 */
Tsutomu Itohd6a0a122013-04-16 05:18:02 +00003151static void fixup_low_keys(struct btrfs_root *root, struct btrfs_path *path,
Jeff Mahoney143bede2012-03-01 14:56:26 +01003152 struct btrfs_disk_key *key, int level)
Chris Masonbe0e5c02007-01-26 15:51:26 -05003153{
3154 int i;
Chris Mason5f39d392007-10-15 16:14:19 -04003155 struct extent_buffer *t;
3156
Chris Mason234b63a2007-03-13 10:46:10 -04003157 for (i = level; i < BTRFS_MAX_LEVEL; i++) {
Chris Masonbe0e5c02007-01-26 15:51:26 -05003158 int tslot = path->slots[i];
Chris Masoneb60cea2007-02-02 09:18:22 -05003159 if (!path->nodes[i])
Chris Masonbe0e5c02007-01-26 15:51:26 -05003160 break;
Chris Mason5f39d392007-10-15 16:14:19 -04003161 t = path->nodes[i];
Liu Bo32adf092012-10-19 12:52:15 +00003162 tree_mod_log_set_node_key(root->fs_info, t, tslot, 1);
Chris Mason5f39d392007-10-15 16:14:19 -04003163 btrfs_set_node_key(t, key, tslot);
Chris Masond6025572007-03-30 14:27:56 -04003164 btrfs_mark_buffer_dirty(path->nodes[i]);
Chris Masonbe0e5c02007-01-26 15:51:26 -05003165 if (tslot != 0)
3166 break;
3167 }
3168}
3169
Chris Mason74123bd2007-02-02 11:05:29 -05003170/*
Zheng Yan31840ae2008-09-23 13:14:14 -04003171 * update item key.
3172 *
3173 * This function isn't completely safe. It's the caller's responsibility
3174 * that the new key won't break the order
3175 */
Tsutomu Itohafe5fea2013-04-16 05:18:22 +00003176void btrfs_set_item_key_safe(struct btrfs_root *root, struct btrfs_path *path,
Jeff Mahoney143bede2012-03-01 14:56:26 +01003177 struct btrfs_key *new_key)
Zheng Yan31840ae2008-09-23 13:14:14 -04003178{
3179 struct btrfs_disk_key disk_key;
3180 struct extent_buffer *eb;
3181 int slot;
3182
3183 eb = path->nodes[0];
3184 slot = path->slots[0];
3185 if (slot > 0) {
3186 btrfs_item_key(eb, &disk_key, slot - 1);
Jeff Mahoney143bede2012-03-01 14:56:26 +01003187 BUG_ON(comp_keys(&disk_key, new_key) >= 0);
Zheng Yan31840ae2008-09-23 13:14:14 -04003188 }
3189 if (slot < btrfs_header_nritems(eb) - 1) {
3190 btrfs_item_key(eb, &disk_key, slot + 1);
Jeff Mahoney143bede2012-03-01 14:56:26 +01003191 BUG_ON(comp_keys(&disk_key, new_key) <= 0);
Zheng Yan31840ae2008-09-23 13:14:14 -04003192 }
3193
3194 btrfs_cpu_key_to_disk(&disk_key, new_key);
3195 btrfs_set_item_key(eb, &disk_key, slot);
3196 btrfs_mark_buffer_dirty(eb);
3197 if (slot == 0)
Tsutomu Itohd6a0a122013-04-16 05:18:02 +00003198 fixup_low_keys(root, path, &disk_key, 1);
Zheng Yan31840ae2008-09-23 13:14:14 -04003199}
3200
3201/*
Chris Mason74123bd2007-02-02 11:05:29 -05003202 * try to push data from one node into the next node left in the
Chris Mason79f95c82007-03-01 15:16:26 -05003203 * tree.
Chris Masonaa5d6be2007-02-28 16:35:06 -05003204 *
3205 * returns 0 if some ptrs were pushed left, < 0 if there was some horrible
3206 * error, and > 0 if there was no room in the left hand block.
Chris Mason74123bd2007-02-02 11:05:29 -05003207 */
Chris Mason98ed5172008-01-03 10:01:48 -05003208static int push_node_left(struct btrfs_trans_handle *trans,
3209 struct btrfs_root *root, struct extent_buffer *dst,
Chris Mason971a1f62008-04-24 10:54:32 -04003210 struct extent_buffer *src, int empty)
Chris Masonbe0e5c02007-01-26 15:51:26 -05003211{
Chris Masonbe0e5c02007-01-26 15:51:26 -05003212 int push_items = 0;
Chris Masonbb803952007-03-01 12:04:21 -05003213 int src_nritems;
3214 int dst_nritems;
Chris Masonaa5d6be2007-02-28 16:35:06 -05003215 int ret = 0;
Chris Masonbe0e5c02007-01-26 15:51:26 -05003216
Chris Mason5f39d392007-10-15 16:14:19 -04003217 src_nritems = btrfs_header_nritems(src);
3218 dst_nritems = btrfs_header_nritems(dst);
Chris Mason123abc82007-03-14 14:14:43 -04003219 push_items = BTRFS_NODEPTRS_PER_BLOCK(root) - dst_nritems;
Chris Mason7bb86312007-12-11 09:25:06 -05003220 WARN_ON(btrfs_header_generation(src) != trans->transid);
3221 WARN_ON(btrfs_header_generation(dst) != trans->transid);
Chris Mason54aa1f42007-06-22 14:16:25 -04003222
Chris Masonbce4eae2008-04-24 14:42:46 -04003223 if (!empty && src_nritems <= 8)
Chris Mason971a1f62008-04-24 10:54:32 -04003224 return 1;
3225
Chris Masond3977122009-01-05 21:25:51 -05003226 if (push_items <= 0)
Chris Masonbe0e5c02007-01-26 15:51:26 -05003227 return 1;
3228
Chris Masonbce4eae2008-04-24 14:42:46 -04003229 if (empty) {
Chris Mason971a1f62008-04-24 10:54:32 -04003230 push_items = min(src_nritems, push_items);
Chris Masonbce4eae2008-04-24 14:42:46 -04003231 if (push_items < src_nritems) {
3232 /* leave at least 8 pointers in the node if
3233 * we aren't going to empty it
3234 */
3235 if (src_nritems - push_items < 8) {
3236 if (push_items <= 8)
3237 return 1;
3238 push_items -= 8;
3239 }
3240 }
3241 } else
3242 push_items = min(src_nritems - 8, push_items);
Chris Mason79f95c82007-03-01 15:16:26 -05003243
Filipe David Borba Manana5de865e2013-12-20 15:17:46 +00003244 ret = tree_mod_log_eb_copy(root->fs_info, dst, src, dst_nritems, 0,
3245 push_items);
3246 if (ret) {
3247 btrfs_abort_transaction(trans, root, ret);
3248 return ret;
3249 }
Chris Mason5f39d392007-10-15 16:14:19 -04003250 copy_extent_buffer(dst, src,
3251 btrfs_node_key_ptr_offset(dst_nritems),
3252 btrfs_node_key_ptr_offset(0),
Chris Masond3977122009-01-05 21:25:51 -05003253 push_items * sizeof(struct btrfs_key_ptr));
Chris Mason5f39d392007-10-15 16:14:19 -04003254
Chris Masonbb803952007-03-01 12:04:21 -05003255 if (push_items < src_nritems) {
Jan Schmidt57911b82012-10-19 09:22:03 +02003256 /*
3257 * don't call tree_mod_log_eb_move here, key removal was already
3258 * fully logged by tree_mod_log_eb_copy above.
3259 */
Chris Mason5f39d392007-10-15 16:14:19 -04003260 memmove_extent_buffer(src, btrfs_node_key_ptr_offset(0),
3261 btrfs_node_key_ptr_offset(push_items),
3262 (src_nritems - push_items) *
3263 sizeof(struct btrfs_key_ptr));
Chris Masonbb803952007-03-01 12:04:21 -05003264 }
Chris Mason5f39d392007-10-15 16:14:19 -04003265 btrfs_set_header_nritems(src, src_nritems - push_items);
3266 btrfs_set_header_nritems(dst, dst_nritems + push_items);
3267 btrfs_mark_buffer_dirty(src);
3268 btrfs_mark_buffer_dirty(dst);
Zheng Yan31840ae2008-09-23 13:14:14 -04003269
Chris Masonbb803952007-03-01 12:04:21 -05003270 return ret;
Chris Masonbe0e5c02007-01-26 15:51:26 -05003271}
3272
Chris Mason97571fd2007-02-24 13:39:08 -05003273/*
Chris Mason79f95c82007-03-01 15:16:26 -05003274 * try to push data from one node into the next node right in the
3275 * tree.
3276 *
3277 * returns 0 if some ptrs were pushed, < 0 if there was some horrible
3278 * error, and > 0 if there was no room in the right hand block.
3279 *
3280 * this will only push up to 1/2 the contents of the left node over
3281 */
Chris Mason5f39d392007-10-15 16:14:19 -04003282static int balance_node_right(struct btrfs_trans_handle *trans,
3283 struct btrfs_root *root,
3284 struct extent_buffer *dst,
3285 struct extent_buffer *src)
Chris Mason79f95c82007-03-01 15:16:26 -05003286{
Chris Mason79f95c82007-03-01 15:16:26 -05003287 int push_items = 0;
3288 int max_push;
3289 int src_nritems;
3290 int dst_nritems;
3291 int ret = 0;
Chris Mason79f95c82007-03-01 15:16:26 -05003292
Chris Mason7bb86312007-12-11 09:25:06 -05003293 WARN_ON(btrfs_header_generation(src) != trans->transid);
3294 WARN_ON(btrfs_header_generation(dst) != trans->transid);
3295
Chris Mason5f39d392007-10-15 16:14:19 -04003296 src_nritems = btrfs_header_nritems(src);
3297 dst_nritems = btrfs_header_nritems(dst);
Chris Mason123abc82007-03-14 14:14:43 -04003298 push_items = BTRFS_NODEPTRS_PER_BLOCK(root) - dst_nritems;
Chris Masond3977122009-01-05 21:25:51 -05003299 if (push_items <= 0)
Chris Mason79f95c82007-03-01 15:16:26 -05003300 return 1;
Chris Masonbce4eae2008-04-24 14:42:46 -04003301
Chris Masond3977122009-01-05 21:25:51 -05003302 if (src_nritems < 4)
Chris Masonbce4eae2008-04-24 14:42:46 -04003303 return 1;
Chris Mason79f95c82007-03-01 15:16:26 -05003304
3305 max_push = src_nritems / 2 + 1;
3306 /* don't try to empty the node */
Chris Masond3977122009-01-05 21:25:51 -05003307 if (max_push >= src_nritems)
Chris Mason79f95c82007-03-01 15:16:26 -05003308 return 1;
Yan252c38f2007-08-29 09:11:44 -04003309
Chris Mason79f95c82007-03-01 15:16:26 -05003310 if (max_push < push_items)
3311 push_items = max_push;
3312
Jan Schmidtf2304752012-05-26 11:43:17 +02003313 tree_mod_log_eb_move(root->fs_info, dst, push_items, 0, dst_nritems);
Chris Mason5f39d392007-10-15 16:14:19 -04003314 memmove_extent_buffer(dst, btrfs_node_key_ptr_offset(push_items),
3315 btrfs_node_key_ptr_offset(0),
3316 (dst_nritems) *
3317 sizeof(struct btrfs_key_ptr));
Chris Masond6025572007-03-30 14:27:56 -04003318
Filipe David Borba Manana5de865e2013-12-20 15:17:46 +00003319 ret = tree_mod_log_eb_copy(root->fs_info, dst, src, 0,
3320 src_nritems - push_items, push_items);
3321 if (ret) {
3322 btrfs_abort_transaction(trans, root, ret);
3323 return ret;
3324 }
Chris Mason5f39d392007-10-15 16:14:19 -04003325 copy_extent_buffer(dst, src,
3326 btrfs_node_key_ptr_offset(0),
3327 btrfs_node_key_ptr_offset(src_nritems - push_items),
Chris Masond3977122009-01-05 21:25:51 -05003328 push_items * sizeof(struct btrfs_key_ptr));
Chris Mason79f95c82007-03-01 15:16:26 -05003329
Chris Mason5f39d392007-10-15 16:14:19 -04003330 btrfs_set_header_nritems(src, src_nritems - push_items);
3331 btrfs_set_header_nritems(dst, dst_nritems + push_items);
Chris Mason79f95c82007-03-01 15:16:26 -05003332
Chris Mason5f39d392007-10-15 16:14:19 -04003333 btrfs_mark_buffer_dirty(src);
3334 btrfs_mark_buffer_dirty(dst);
Zheng Yan31840ae2008-09-23 13:14:14 -04003335
Chris Mason79f95c82007-03-01 15:16:26 -05003336 return ret;
3337}
3338
3339/*
Chris Mason97571fd2007-02-24 13:39:08 -05003340 * helper function to insert a new root level in the tree.
3341 * A new node is allocated, and a single item is inserted to
3342 * point to the existing root
Chris Masonaa5d6be2007-02-28 16:35:06 -05003343 *
3344 * returns zero on success or < 0 on failure.
Chris Mason97571fd2007-02-24 13:39:08 -05003345 */
Chris Masond3977122009-01-05 21:25:51 -05003346static noinline int insert_new_root(struct btrfs_trans_handle *trans,
Chris Mason5f39d392007-10-15 16:14:19 -04003347 struct btrfs_root *root,
Liu Bofdd99c72013-05-22 12:06:51 +00003348 struct btrfs_path *path, int level)
Chris Mason5c680ed2007-02-22 11:39:13 -05003349{
Chris Mason7bb86312007-12-11 09:25:06 -05003350 u64 lower_gen;
Chris Mason5f39d392007-10-15 16:14:19 -04003351 struct extent_buffer *lower;
3352 struct extent_buffer *c;
Chris Mason925baed2008-06-25 16:01:30 -04003353 struct extent_buffer *old;
Chris Mason5f39d392007-10-15 16:14:19 -04003354 struct btrfs_disk_key lower_key;
Chris Mason5c680ed2007-02-22 11:39:13 -05003355
3356 BUG_ON(path->nodes[level]);
3357 BUG_ON(path->nodes[level-1] != root->node);
3358
Chris Mason7bb86312007-12-11 09:25:06 -05003359 lower = path->nodes[level-1];
3360 if (level == 1)
3361 btrfs_item_key(lower, &lower_key, 0);
3362 else
3363 btrfs_node_key(lower, &lower_key, 0);
3364
Zheng Yan31840ae2008-09-23 13:14:14 -04003365 c = btrfs_alloc_free_block(trans, root, root->nodesize, 0,
Yan Zheng5d4f98a2009-06-10 10:45:14 -04003366 root->root_key.objectid, &lower_key,
Jan Schmidt5581a512012-05-16 17:04:52 +02003367 level, root->node->start, 0);
Chris Mason5f39d392007-10-15 16:14:19 -04003368 if (IS_ERR(c))
3369 return PTR_ERR(c);
Chris Mason925baed2008-06-25 16:01:30 -04003370
Yan, Zhengf0486c62010-05-16 10:46:25 -04003371 root_add_used(root, root->nodesize);
3372
Yan Zheng5d4f98a2009-06-10 10:45:14 -04003373 memset_extent_buffer(c, 0, 0, sizeof(struct btrfs_header));
Chris Mason5f39d392007-10-15 16:14:19 -04003374 btrfs_set_header_nritems(c, 1);
3375 btrfs_set_header_level(c, level);
Chris Masondb945352007-10-15 16:15:53 -04003376 btrfs_set_header_bytenr(c, c->start);
Chris Mason5f39d392007-10-15 16:14:19 -04003377 btrfs_set_header_generation(c, trans->transid);
Yan Zheng5d4f98a2009-06-10 10:45:14 -04003378 btrfs_set_header_backref_rev(c, BTRFS_MIXED_BACKREF_REV);
Chris Mason5f39d392007-10-15 16:14:19 -04003379 btrfs_set_header_owner(c, root->root_key.objectid);
Chris Masond5719762007-03-23 10:01:08 -04003380
Ross Kirk0a4e5582013-09-24 10:12:38 +01003381 write_extent_buffer(c, root->fs_info->fsid, btrfs_header_fsid(),
Chris Mason5f39d392007-10-15 16:14:19 -04003382 BTRFS_FSID_SIZE);
Chris Masone17cade2008-04-15 15:41:47 -04003383
3384 write_extent_buffer(c, root->fs_info->chunk_tree_uuid,
Geert Uytterhoevenb308bc22013-08-20 13:20:15 +02003385 btrfs_header_chunk_tree_uuid(c), BTRFS_UUID_SIZE);
Chris Masone17cade2008-04-15 15:41:47 -04003386
Chris Mason5f39d392007-10-15 16:14:19 -04003387 btrfs_set_node_key(c, &lower_key, 0);
Chris Masondb945352007-10-15 16:15:53 -04003388 btrfs_set_node_blockptr(c, 0, lower->start);
Chris Mason7bb86312007-12-11 09:25:06 -05003389 lower_gen = btrfs_header_generation(lower);
Zheng Yan31840ae2008-09-23 13:14:14 -04003390 WARN_ON(lower_gen != trans->transid);
Chris Mason7bb86312007-12-11 09:25:06 -05003391
3392 btrfs_set_node_ptr_generation(c, 0, lower_gen);
Chris Mason5f39d392007-10-15 16:14:19 -04003393
3394 btrfs_mark_buffer_dirty(c);
Chris Masond5719762007-03-23 10:01:08 -04003395
Chris Mason925baed2008-06-25 16:01:30 -04003396 old = root->node;
Liu Bofdd99c72013-05-22 12:06:51 +00003397 tree_mod_log_set_root_pointer(root, c, 0);
Chris Mason240f62c2011-03-23 14:54:42 -04003398 rcu_assign_pointer(root->node, c);
Chris Mason925baed2008-06-25 16:01:30 -04003399
3400 /* the super has an extra ref to root->node */
3401 free_extent_buffer(old);
3402
Chris Mason0b86a832008-03-24 15:01:56 -04003403 add_root_to_dirty_list(root);
Chris Mason5f39d392007-10-15 16:14:19 -04003404 extent_buffer_get(c);
3405 path->nodes[level] = c;
Chris Masonbd681512011-07-16 15:23:14 -04003406 path->locks[level] = BTRFS_WRITE_LOCK;
Chris Mason5c680ed2007-02-22 11:39:13 -05003407 path->slots[level] = 0;
3408 return 0;
3409}
3410
Chris Mason74123bd2007-02-02 11:05:29 -05003411/*
3412 * worker function to insert a single pointer in a node.
3413 * the node should have enough room for the pointer already
Chris Mason97571fd2007-02-24 13:39:08 -05003414 *
Chris Mason74123bd2007-02-02 11:05:29 -05003415 * slot and level indicate where you want the key to go, and
3416 * blocknr is the block the key points to.
3417 */
Jeff Mahoney143bede2012-03-01 14:56:26 +01003418static void insert_ptr(struct btrfs_trans_handle *trans,
3419 struct btrfs_root *root, struct btrfs_path *path,
3420 struct btrfs_disk_key *key, u64 bytenr,
Jan Schmidtc3e06962012-06-21 11:01:06 +02003421 int slot, int level)
Chris Mason74123bd2007-02-02 11:05:29 -05003422{
Chris Mason5f39d392007-10-15 16:14:19 -04003423 struct extent_buffer *lower;
Chris Mason74123bd2007-02-02 11:05:29 -05003424 int nritems;
Jan Schmidtf3ea38d2012-05-26 11:45:21 +02003425 int ret;
Chris Mason5c680ed2007-02-22 11:39:13 -05003426
3427 BUG_ON(!path->nodes[level]);
Yan, Zhengf0486c62010-05-16 10:46:25 -04003428 btrfs_assert_tree_locked(path->nodes[level]);
Chris Mason5f39d392007-10-15 16:14:19 -04003429 lower = path->nodes[level];
3430 nritems = btrfs_header_nritems(lower);
Stoyan Gaydarovc2934982009-04-02 17:05:11 -04003431 BUG_ON(slot > nritems);
Jeff Mahoney143bede2012-03-01 14:56:26 +01003432 BUG_ON(nritems == BTRFS_NODEPTRS_PER_BLOCK(root));
Chris Mason74123bd2007-02-02 11:05:29 -05003433 if (slot != nritems) {
Jan Schmidtc3e06962012-06-21 11:01:06 +02003434 if (level)
Jan Schmidtf3ea38d2012-05-26 11:45:21 +02003435 tree_mod_log_eb_move(root->fs_info, lower, slot + 1,
3436 slot, nritems - slot);
Chris Mason5f39d392007-10-15 16:14:19 -04003437 memmove_extent_buffer(lower,
3438 btrfs_node_key_ptr_offset(slot + 1),
3439 btrfs_node_key_ptr_offset(slot),
Chris Masond6025572007-03-30 14:27:56 -04003440 (nritems - slot) * sizeof(struct btrfs_key_ptr));
Chris Mason74123bd2007-02-02 11:05:29 -05003441 }
Jan Schmidtc3e06962012-06-21 11:01:06 +02003442 if (level) {
Jan Schmidtf3ea38d2012-05-26 11:45:21 +02003443 ret = tree_mod_log_insert_key(root->fs_info, lower, slot,
Josef Bacikc8cc6342013-07-01 16:18:19 -04003444 MOD_LOG_KEY_ADD, GFP_NOFS);
Jan Schmidtf3ea38d2012-05-26 11:45:21 +02003445 BUG_ON(ret < 0);
3446 }
Chris Mason5f39d392007-10-15 16:14:19 -04003447 btrfs_set_node_key(lower, key, slot);
Chris Masondb945352007-10-15 16:15:53 -04003448 btrfs_set_node_blockptr(lower, slot, bytenr);
Chris Mason74493f72007-12-11 09:25:06 -05003449 WARN_ON(trans->transid == 0);
3450 btrfs_set_node_ptr_generation(lower, slot, trans->transid);
Chris Mason5f39d392007-10-15 16:14:19 -04003451 btrfs_set_header_nritems(lower, nritems + 1);
3452 btrfs_mark_buffer_dirty(lower);
Chris Mason74123bd2007-02-02 11:05:29 -05003453}
3454
Chris Mason97571fd2007-02-24 13:39:08 -05003455/*
3456 * split the node at the specified level in path in two.
3457 * The path is corrected to point to the appropriate node after the split
3458 *
3459 * Before splitting this tries to make some room in the node by pushing
3460 * left and right, if either one works, it returns right away.
Chris Masonaa5d6be2007-02-28 16:35:06 -05003461 *
3462 * returns 0 on success and < 0 on failure
Chris Mason97571fd2007-02-24 13:39:08 -05003463 */
Chris Masone02119d2008-09-05 16:13:11 -04003464static noinline int split_node(struct btrfs_trans_handle *trans,
3465 struct btrfs_root *root,
3466 struct btrfs_path *path, int level)
Chris Masonbe0e5c02007-01-26 15:51:26 -05003467{
Chris Mason5f39d392007-10-15 16:14:19 -04003468 struct extent_buffer *c;
3469 struct extent_buffer *split;
3470 struct btrfs_disk_key disk_key;
Chris Masonbe0e5c02007-01-26 15:51:26 -05003471 int mid;
Chris Mason5c680ed2007-02-22 11:39:13 -05003472 int ret;
Chris Mason7518a232007-03-12 12:01:18 -04003473 u32 c_nritems;
Chris Masonbe0e5c02007-01-26 15:51:26 -05003474
Chris Mason5f39d392007-10-15 16:14:19 -04003475 c = path->nodes[level];
Chris Mason7bb86312007-12-11 09:25:06 -05003476 WARN_ON(btrfs_header_generation(c) != trans->transid);
Chris Mason5f39d392007-10-15 16:14:19 -04003477 if (c == root->node) {
Jan Schmidtd9abbf12013-03-20 13:49:48 +00003478 /*
Jan Schmidt90f8d622013-04-13 13:19:53 +00003479 * trying to split the root, lets make a new one
3480 *
Liu Bofdd99c72013-05-22 12:06:51 +00003481 * tree mod log: We don't log_removal old root in
Jan Schmidt90f8d622013-04-13 13:19:53 +00003482 * insert_new_root, because that root buffer will be kept as a
3483 * normal node. We are going to log removal of half of the
3484 * elements below with tree_mod_log_eb_copy. We're holding a
3485 * tree lock on the buffer, which is why we cannot race with
3486 * other tree_mod_log users.
Jan Schmidtd9abbf12013-03-20 13:49:48 +00003487 */
Liu Bofdd99c72013-05-22 12:06:51 +00003488 ret = insert_new_root(trans, root, path, level + 1);
Chris Mason5c680ed2007-02-22 11:39:13 -05003489 if (ret)
3490 return ret;
Chris Masonb3612422009-05-13 19:12:15 -04003491 } else {
Chris Masone66f7092007-04-20 13:16:02 -04003492 ret = push_nodes_for_insert(trans, root, path, level);
Chris Mason5f39d392007-10-15 16:14:19 -04003493 c = path->nodes[level];
3494 if (!ret && btrfs_header_nritems(c) <
Chris Masonc448acf2008-04-24 09:34:34 -04003495 BTRFS_NODEPTRS_PER_BLOCK(root) - 3)
Chris Masone66f7092007-04-20 13:16:02 -04003496 return 0;
Chris Mason54aa1f42007-06-22 14:16:25 -04003497 if (ret < 0)
3498 return ret;
Chris Masonbe0e5c02007-01-26 15:51:26 -05003499 }
Chris Masone66f7092007-04-20 13:16:02 -04003500
Chris Mason5f39d392007-10-15 16:14:19 -04003501 c_nritems = btrfs_header_nritems(c);
Yan Zheng5d4f98a2009-06-10 10:45:14 -04003502 mid = (c_nritems + 1) / 2;
3503 btrfs_node_key(c, &disk_key, mid);
Chris Mason7bb86312007-12-11 09:25:06 -05003504
Yan Zheng5d4f98a2009-06-10 10:45:14 -04003505 split = btrfs_alloc_free_block(trans, root, root->nodesize, 0,
Zheng Yan31840ae2008-09-23 13:14:14 -04003506 root->root_key.objectid,
Jan Schmidt5581a512012-05-16 17:04:52 +02003507 &disk_key, level, c->start, 0);
Chris Mason5f39d392007-10-15 16:14:19 -04003508 if (IS_ERR(split))
3509 return PTR_ERR(split);
Chris Mason54aa1f42007-06-22 14:16:25 -04003510
Yan, Zhengf0486c62010-05-16 10:46:25 -04003511 root_add_used(root, root->nodesize);
3512
Yan Zheng5d4f98a2009-06-10 10:45:14 -04003513 memset_extent_buffer(split, 0, 0, sizeof(struct btrfs_header));
Chris Mason5f39d392007-10-15 16:14:19 -04003514 btrfs_set_header_level(split, btrfs_header_level(c));
Chris Masondb945352007-10-15 16:15:53 -04003515 btrfs_set_header_bytenr(split, split->start);
Chris Mason5f39d392007-10-15 16:14:19 -04003516 btrfs_set_header_generation(split, trans->transid);
Yan Zheng5d4f98a2009-06-10 10:45:14 -04003517 btrfs_set_header_backref_rev(split, BTRFS_MIXED_BACKREF_REV);
Chris Mason5f39d392007-10-15 16:14:19 -04003518 btrfs_set_header_owner(split, root->root_key.objectid);
3519 write_extent_buffer(split, root->fs_info->fsid,
Ross Kirk0a4e5582013-09-24 10:12:38 +01003520 btrfs_header_fsid(), BTRFS_FSID_SIZE);
Chris Masone17cade2008-04-15 15:41:47 -04003521 write_extent_buffer(split, root->fs_info->chunk_tree_uuid,
Geert Uytterhoevenb308bc22013-08-20 13:20:15 +02003522 btrfs_header_chunk_tree_uuid(split),
Chris Masone17cade2008-04-15 15:41:47 -04003523 BTRFS_UUID_SIZE);
Chris Mason5f39d392007-10-15 16:14:19 -04003524
Filipe David Borba Manana5de865e2013-12-20 15:17:46 +00003525 ret = tree_mod_log_eb_copy(root->fs_info, split, c, 0,
3526 mid, c_nritems - mid);
3527 if (ret) {
3528 btrfs_abort_transaction(trans, root, ret);
3529 return ret;
3530 }
Chris Mason5f39d392007-10-15 16:14:19 -04003531 copy_extent_buffer(split, c,
3532 btrfs_node_key_ptr_offset(0),
3533 btrfs_node_key_ptr_offset(mid),
3534 (c_nritems - mid) * sizeof(struct btrfs_key_ptr));
3535 btrfs_set_header_nritems(split, c_nritems - mid);
3536 btrfs_set_header_nritems(c, mid);
Chris Masonaa5d6be2007-02-28 16:35:06 -05003537 ret = 0;
3538
Chris Mason5f39d392007-10-15 16:14:19 -04003539 btrfs_mark_buffer_dirty(c);
3540 btrfs_mark_buffer_dirty(split);
3541
Jeff Mahoney143bede2012-03-01 14:56:26 +01003542 insert_ptr(trans, root, path, &disk_key, split->start,
Jan Schmidtc3e06962012-06-21 11:01:06 +02003543 path->slots[level + 1] + 1, level + 1);
Chris Masonaa5d6be2007-02-28 16:35:06 -05003544
Chris Mason5de08d72007-02-24 06:24:44 -05003545 if (path->slots[level] >= mid) {
Chris Mason5c680ed2007-02-22 11:39:13 -05003546 path->slots[level] -= mid;
Chris Mason925baed2008-06-25 16:01:30 -04003547 btrfs_tree_unlock(c);
Chris Mason5f39d392007-10-15 16:14:19 -04003548 free_extent_buffer(c);
3549 path->nodes[level] = split;
Chris Mason5c680ed2007-02-22 11:39:13 -05003550 path->slots[level + 1] += 1;
3551 } else {
Chris Mason925baed2008-06-25 16:01:30 -04003552 btrfs_tree_unlock(split);
Chris Mason5f39d392007-10-15 16:14:19 -04003553 free_extent_buffer(split);
Chris Masonbe0e5c02007-01-26 15:51:26 -05003554 }
Chris Masonaa5d6be2007-02-28 16:35:06 -05003555 return ret;
Chris Masonbe0e5c02007-01-26 15:51:26 -05003556}
3557
Chris Mason74123bd2007-02-02 11:05:29 -05003558/*
3559 * how many bytes are required to store the items in a leaf. start
3560 * and nr indicate which items in the leaf to check. This totals up the
3561 * space used both by the item structs and the item data
3562 */
Chris Mason5f39d392007-10-15 16:14:19 -04003563static int leaf_space_used(struct extent_buffer *l, int start, int nr)
Chris Masonbe0e5c02007-01-26 15:51:26 -05003564{
Josef Bacik41be1f32012-10-15 13:43:18 -04003565 struct btrfs_item *start_item;
3566 struct btrfs_item *end_item;
3567 struct btrfs_map_token token;
Chris Masonbe0e5c02007-01-26 15:51:26 -05003568 int data_len;
Chris Mason5f39d392007-10-15 16:14:19 -04003569 int nritems = btrfs_header_nritems(l);
Chris Masond4dbff92007-04-04 14:08:15 -04003570 int end = min(nritems, start + nr) - 1;
Chris Masonbe0e5c02007-01-26 15:51:26 -05003571
3572 if (!nr)
3573 return 0;
Josef Bacik41be1f32012-10-15 13:43:18 -04003574 btrfs_init_map_token(&token);
Ross Kirkdd3cc162013-09-16 15:58:09 +01003575 start_item = btrfs_item_nr(start);
3576 end_item = btrfs_item_nr(end);
Josef Bacik41be1f32012-10-15 13:43:18 -04003577 data_len = btrfs_token_item_offset(l, start_item, &token) +
3578 btrfs_token_item_size(l, start_item, &token);
3579 data_len = data_len - btrfs_token_item_offset(l, end_item, &token);
Chris Mason0783fcf2007-03-12 20:12:07 -04003580 data_len += sizeof(struct btrfs_item) * nr;
Chris Masond4dbff92007-04-04 14:08:15 -04003581 WARN_ON(data_len < 0);
Chris Masonbe0e5c02007-01-26 15:51:26 -05003582 return data_len;
3583}
3584
Chris Mason74123bd2007-02-02 11:05:29 -05003585/*
Chris Masond4dbff92007-04-04 14:08:15 -04003586 * The space between the end of the leaf items and
3587 * the start of the leaf data. IOW, how much room
3588 * the leaf has left for both items and data
3589 */
Chris Masond3977122009-01-05 21:25:51 -05003590noinline int btrfs_leaf_free_space(struct btrfs_root *root,
Chris Masone02119d2008-09-05 16:13:11 -04003591 struct extent_buffer *leaf)
Chris Masond4dbff92007-04-04 14:08:15 -04003592{
Chris Mason5f39d392007-10-15 16:14:19 -04003593 int nritems = btrfs_header_nritems(leaf);
3594 int ret;
3595 ret = BTRFS_LEAF_DATA_SIZE(root) - leaf_space_used(leaf, 0, nritems);
3596 if (ret < 0) {
Frank Holtonefe120a2013-12-20 11:37:06 -05003597 btrfs_crit(root->fs_info,
3598 "leaf free space ret %d, leaf data size %lu, used %d nritems %d",
Jens Axboeae2f5412007-10-19 09:22:59 -04003599 ret, (unsigned long) BTRFS_LEAF_DATA_SIZE(root),
Chris Mason5f39d392007-10-15 16:14:19 -04003600 leaf_space_used(leaf, 0, nritems), nritems);
3601 }
3602 return ret;
Chris Masond4dbff92007-04-04 14:08:15 -04003603}
3604
Chris Mason99d8f832010-07-07 10:51:48 -04003605/*
3606 * min slot controls the lowest index we're willing to push to the
3607 * right. We'll push up to and including min_slot, but no lower
3608 */
Chris Mason44871b12009-03-13 10:04:31 -04003609static noinline int __push_leaf_right(struct btrfs_trans_handle *trans,
3610 struct btrfs_root *root,
3611 struct btrfs_path *path,
3612 int data_size, int empty,
3613 struct extent_buffer *right,
Chris Mason99d8f832010-07-07 10:51:48 -04003614 int free_space, u32 left_nritems,
3615 u32 min_slot)
Chris Mason00ec4c52007-02-24 12:47:20 -05003616{
Chris Mason5f39d392007-10-15 16:14:19 -04003617 struct extent_buffer *left = path->nodes[0];
Chris Mason44871b12009-03-13 10:04:31 -04003618 struct extent_buffer *upper = path->nodes[1];
Chris Masoncfed81a2012-03-03 07:40:03 -05003619 struct btrfs_map_token token;
Chris Mason5f39d392007-10-15 16:14:19 -04003620 struct btrfs_disk_key disk_key;
Chris Mason00ec4c52007-02-24 12:47:20 -05003621 int slot;
Chris Mason34a38212007-11-07 13:31:03 -05003622 u32 i;
Chris Mason00ec4c52007-02-24 12:47:20 -05003623 int push_space = 0;
3624 int push_items = 0;
Chris Mason0783fcf2007-03-12 20:12:07 -04003625 struct btrfs_item *item;
Chris Mason34a38212007-11-07 13:31:03 -05003626 u32 nr;
Chris Mason7518a232007-03-12 12:01:18 -04003627 u32 right_nritems;
Chris Mason5f39d392007-10-15 16:14:19 -04003628 u32 data_end;
Chris Masondb945352007-10-15 16:15:53 -04003629 u32 this_item_size;
Chris Mason00ec4c52007-02-24 12:47:20 -05003630
Chris Masoncfed81a2012-03-03 07:40:03 -05003631 btrfs_init_map_token(&token);
3632
Chris Mason34a38212007-11-07 13:31:03 -05003633 if (empty)
3634 nr = 0;
3635 else
Chris Mason99d8f832010-07-07 10:51:48 -04003636 nr = max_t(u32, 1, min_slot);
Chris Mason34a38212007-11-07 13:31:03 -05003637
Zheng Yan31840ae2008-09-23 13:14:14 -04003638 if (path->slots[0] >= left_nritems)
Yan Zheng87b29b22008-12-17 10:21:48 -05003639 push_space += data_size;
Zheng Yan31840ae2008-09-23 13:14:14 -04003640
Chris Mason44871b12009-03-13 10:04:31 -04003641 slot = path->slots[1];
Chris Mason34a38212007-11-07 13:31:03 -05003642 i = left_nritems - 1;
3643 while (i >= nr) {
Ross Kirkdd3cc162013-09-16 15:58:09 +01003644 item = btrfs_item_nr(i);
Chris Masondb945352007-10-15 16:15:53 -04003645
Zheng Yan31840ae2008-09-23 13:14:14 -04003646 if (!empty && push_items > 0) {
3647 if (path->slots[0] > i)
3648 break;
3649 if (path->slots[0] == i) {
3650 int space = btrfs_leaf_free_space(root, left);
3651 if (space + push_space * 2 > free_space)
3652 break;
3653 }
3654 }
3655
Chris Mason00ec4c52007-02-24 12:47:20 -05003656 if (path->slots[0] == i)
Yan Zheng87b29b22008-12-17 10:21:48 -05003657 push_space += data_size;
Chris Masondb945352007-10-15 16:15:53 -04003658
Chris Masondb945352007-10-15 16:15:53 -04003659 this_item_size = btrfs_item_size(left, item);
3660 if (this_item_size + sizeof(*item) + push_space > free_space)
Chris Mason00ec4c52007-02-24 12:47:20 -05003661 break;
Zheng Yan31840ae2008-09-23 13:14:14 -04003662
Chris Mason00ec4c52007-02-24 12:47:20 -05003663 push_items++;
Chris Masondb945352007-10-15 16:15:53 -04003664 push_space += this_item_size + sizeof(*item);
Chris Mason34a38212007-11-07 13:31:03 -05003665 if (i == 0)
3666 break;
3667 i--;
Chris Masondb945352007-10-15 16:15:53 -04003668 }
Chris Mason5f39d392007-10-15 16:14:19 -04003669
Chris Mason925baed2008-06-25 16:01:30 -04003670 if (push_items == 0)
3671 goto out_unlock;
Chris Mason5f39d392007-10-15 16:14:19 -04003672
Julia Lawall6c1500f2012-11-03 20:30:18 +00003673 WARN_ON(!empty && push_items == left_nritems);
Chris Mason5f39d392007-10-15 16:14:19 -04003674
Chris Mason00ec4c52007-02-24 12:47:20 -05003675 /* push left to right */
Chris Mason5f39d392007-10-15 16:14:19 -04003676 right_nritems = btrfs_header_nritems(right);
Chris Mason34a38212007-11-07 13:31:03 -05003677
Chris Mason5f39d392007-10-15 16:14:19 -04003678 push_space = btrfs_item_end_nr(left, left_nritems - push_items);
Chris Mason123abc82007-03-14 14:14:43 -04003679 push_space -= leaf_data_end(root, left);
Chris Mason5f39d392007-10-15 16:14:19 -04003680
Chris Mason00ec4c52007-02-24 12:47:20 -05003681 /* make room in the right data area */
Chris Mason5f39d392007-10-15 16:14:19 -04003682 data_end = leaf_data_end(root, right);
3683 memmove_extent_buffer(right,
3684 btrfs_leaf_data(right) + data_end - push_space,
3685 btrfs_leaf_data(right) + data_end,
3686 BTRFS_LEAF_DATA_SIZE(root) - data_end);
3687
Chris Mason00ec4c52007-02-24 12:47:20 -05003688 /* copy from the left data area */
Chris Mason5f39d392007-10-15 16:14:19 -04003689 copy_extent_buffer(right, left, btrfs_leaf_data(right) +
Chris Masond6025572007-03-30 14:27:56 -04003690 BTRFS_LEAF_DATA_SIZE(root) - push_space,
3691 btrfs_leaf_data(left) + leaf_data_end(root, left),
3692 push_space);
Chris Mason5f39d392007-10-15 16:14:19 -04003693
3694 memmove_extent_buffer(right, btrfs_item_nr_offset(push_items),
3695 btrfs_item_nr_offset(0),
3696 right_nritems * sizeof(struct btrfs_item));
3697
Chris Mason00ec4c52007-02-24 12:47:20 -05003698 /* copy the items from left to right */
Chris Mason5f39d392007-10-15 16:14:19 -04003699 copy_extent_buffer(right, left, btrfs_item_nr_offset(0),
3700 btrfs_item_nr_offset(left_nritems - push_items),
3701 push_items * sizeof(struct btrfs_item));
Chris Mason00ec4c52007-02-24 12:47:20 -05003702
3703 /* update the item pointers */
Chris Mason7518a232007-03-12 12:01:18 -04003704 right_nritems += push_items;
Chris Mason5f39d392007-10-15 16:14:19 -04003705 btrfs_set_header_nritems(right, right_nritems);
Chris Mason123abc82007-03-14 14:14:43 -04003706 push_space = BTRFS_LEAF_DATA_SIZE(root);
Chris Mason7518a232007-03-12 12:01:18 -04003707 for (i = 0; i < right_nritems; i++) {
Ross Kirkdd3cc162013-09-16 15:58:09 +01003708 item = btrfs_item_nr(i);
Chris Masoncfed81a2012-03-03 07:40:03 -05003709 push_space -= btrfs_token_item_size(right, item, &token);
3710 btrfs_set_token_item_offset(right, item, push_space, &token);
Chris Masondb945352007-10-15 16:15:53 -04003711 }
3712
Chris Mason7518a232007-03-12 12:01:18 -04003713 left_nritems -= push_items;
Chris Mason5f39d392007-10-15 16:14:19 -04003714 btrfs_set_header_nritems(left, left_nritems);
Chris Mason00ec4c52007-02-24 12:47:20 -05003715
Chris Mason34a38212007-11-07 13:31:03 -05003716 if (left_nritems)
3717 btrfs_mark_buffer_dirty(left);
Yan, Zhengf0486c62010-05-16 10:46:25 -04003718 else
3719 clean_tree_block(trans, root, left);
3720
Chris Mason5f39d392007-10-15 16:14:19 -04003721 btrfs_mark_buffer_dirty(right);
Chris Masona429e512007-04-18 16:15:28 -04003722
Chris Mason5f39d392007-10-15 16:14:19 -04003723 btrfs_item_key(right, &disk_key, 0);
3724 btrfs_set_node_key(upper, &disk_key, slot + 1);
Chris Masond6025572007-03-30 14:27:56 -04003725 btrfs_mark_buffer_dirty(upper);
Chris Mason02217ed2007-03-02 16:08:05 -05003726
Chris Mason00ec4c52007-02-24 12:47:20 -05003727 /* then fixup the leaf pointer in the path */
Chris Mason7518a232007-03-12 12:01:18 -04003728 if (path->slots[0] >= left_nritems) {
3729 path->slots[0] -= left_nritems;
Chris Mason925baed2008-06-25 16:01:30 -04003730 if (btrfs_header_nritems(path->nodes[0]) == 0)
3731 clean_tree_block(trans, root, path->nodes[0]);
3732 btrfs_tree_unlock(path->nodes[0]);
Chris Mason5f39d392007-10-15 16:14:19 -04003733 free_extent_buffer(path->nodes[0]);
3734 path->nodes[0] = right;
Chris Mason00ec4c52007-02-24 12:47:20 -05003735 path->slots[1] += 1;
3736 } else {
Chris Mason925baed2008-06-25 16:01:30 -04003737 btrfs_tree_unlock(right);
Chris Mason5f39d392007-10-15 16:14:19 -04003738 free_extent_buffer(right);
Chris Mason00ec4c52007-02-24 12:47:20 -05003739 }
3740 return 0;
Chris Mason925baed2008-06-25 16:01:30 -04003741
3742out_unlock:
3743 btrfs_tree_unlock(right);
3744 free_extent_buffer(right);
3745 return 1;
Chris Mason00ec4c52007-02-24 12:47:20 -05003746}
Chris Mason925baed2008-06-25 16:01:30 -04003747
Chris Mason00ec4c52007-02-24 12:47:20 -05003748/*
Chris Mason44871b12009-03-13 10:04:31 -04003749 * push some data in the path leaf to the right, trying to free up at
3750 * least data_size bytes. returns zero if the push worked, nonzero otherwise
3751 *
3752 * returns 1 if the push failed because the other node didn't have enough
3753 * room, 0 if everything worked out and < 0 if there were major errors.
Chris Mason99d8f832010-07-07 10:51:48 -04003754 *
3755 * this will push starting from min_slot to the end of the leaf. It won't
3756 * push any slot lower than min_slot
Chris Mason44871b12009-03-13 10:04:31 -04003757 */
3758static int push_leaf_right(struct btrfs_trans_handle *trans, struct btrfs_root
Chris Mason99d8f832010-07-07 10:51:48 -04003759 *root, struct btrfs_path *path,
3760 int min_data_size, int data_size,
3761 int empty, u32 min_slot)
Chris Mason44871b12009-03-13 10:04:31 -04003762{
3763 struct extent_buffer *left = path->nodes[0];
3764 struct extent_buffer *right;
3765 struct extent_buffer *upper;
3766 int slot;
3767 int free_space;
3768 u32 left_nritems;
3769 int ret;
3770
3771 if (!path->nodes[1])
3772 return 1;
3773
3774 slot = path->slots[1];
3775 upper = path->nodes[1];
3776 if (slot >= btrfs_header_nritems(upper) - 1)
3777 return 1;
3778
3779 btrfs_assert_tree_locked(path->nodes[1]);
3780
3781 right = read_node_slot(root, upper, slot + 1);
Tsutomu Itoh91ca3382011-01-05 02:32:22 +00003782 if (right == NULL)
3783 return 1;
3784
Chris Mason44871b12009-03-13 10:04:31 -04003785 btrfs_tree_lock(right);
3786 btrfs_set_lock_blocking(right);
3787
3788 free_space = btrfs_leaf_free_space(root, right);
3789 if (free_space < data_size)
3790 goto out_unlock;
3791
3792 /* cow and double check */
3793 ret = btrfs_cow_block(trans, root, right, upper,
3794 slot + 1, &right);
3795 if (ret)
3796 goto out_unlock;
3797
3798 free_space = btrfs_leaf_free_space(root, right);
3799 if (free_space < data_size)
3800 goto out_unlock;
3801
3802 left_nritems = btrfs_header_nritems(left);
3803 if (left_nritems == 0)
3804 goto out_unlock;
3805
Filipe David Borba Manana2ef1fed2013-12-04 22:17:39 +00003806 if (path->slots[0] == left_nritems && !empty) {
3807 /* Key greater than all keys in the leaf, right neighbor has
3808 * enough room for it and we're not emptying our leaf to delete
3809 * it, therefore use right neighbor to insert the new item and
3810 * no need to touch/dirty our left leaft. */
3811 btrfs_tree_unlock(left);
3812 free_extent_buffer(left);
3813 path->nodes[0] = right;
3814 path->slots[0] = 0;
3815 path->slots[1]++;
3816 return 0;
3817 }
3818
Chris Mason99d8f832010-07-07 10:51:48 -04003819 return __push_leaf_right(trans, root, path, min_data_size, empty,
3820 right, free_space, left_nritems, min_slot);
Chris Mason44871b12009-03-13 10:04:31 -04003821out_unlock:
3822 btrfs_tree_unlock(right);
3823 free_extent_buffer(right);
3824 return 1;
3825}
3826
3827/*
Chris Mason74123bd2007-02-02 11:05:29 -05003828 * push some data in the path leaf to the left, trying to free up at
3829 * least data_size bytes. returns zero if the push worked, nonzero otherwise
Chris Mason99d8f832010-07-07 10:51:48 -04003830 *
3831 * max_slot can put a limit on how far into the leaf we'll push items. The
3832 * item at 'max_slot' won't be touched. Use (u32)-1 to make us do all the
3833 * items
Chris Mason74123bd2007-02-02 11:05:29 -05003834 */
Chris Mason44871b12009-03-13 10:04:31 -04003835static noinline int __push_leaf_left(struct btrfs_trans_handle *trans,
3836 struct btrfs_root *root,
3837 struct btrfs_path *path, int data_size,
3838 int empty, struct extent_buffer *left,
Chris Mason99d8f832010-07-07 10:51:48 -04003839 int free_space, u32 right_nritems,
3840 u32 max_slot)
Chris Masonbe0e5c02007-01-26 15:51:26 -05003841{
Chris Mason5f39d392007-10-15 16:14:19 -04003842 struct btrfs_disk_key disk_key;
3843 struct extent_buffer *right = path->nodes[0];
Chris Masonbe0e5c02007-01-26 15:51:26 -05003844 int i;
Chris Masonbe0e5c02007-01-26 15:51:26 -05003845 int push_space = 0;
3846 int push_items = 0;
Chris Mason0783fcf2007-03-12 20:12:07 -04003847 struct btrfs_item *item;
Chris Mason7518a232007-03-12 12:01:18 -04003848 u32 old_left_nritems;
Chris Mason34a38212007-11-07 13:31:03 -05003849 u32 nr;
Chris Masonaa5d6be2007-02-28 16:35:06 -05003850 int ret = 0;
Chris Masondb945352007-10-15 16:15:53 -04003851 u32 this_item_size;
3852 u32 old_left_item_size;
Chris Masoncfed81a2012-03-03 07:40:03 -05003853 struct btrfs_map_token token;
3854
3855 btrfs_init_map_token(&token);
Chris Masonbe0e5c02007-01-26 15:51:26 -05003856
Chris Mason34a38212007-11-07 13:31:03 -05003857 if (empty)
Chris Mason99d8f832010-07-07 10:51:48 -04003858 nr = min(right_nritems, max_slot);
Chris Mason34a38212007-11-07 13:31:03 -05003859 else
Chris Mason99d8f832010-07-07 10:51:48 -04003860 nr = min(right_nritems - 1, max_slot);
Chris Mason34a38212007-11-07 13:31:03 -05003861
3862 for (i = 0; i < nr; i++) {
Ross Kirkdd3cc162013-09-16 15:58:09 +01003863 item = btrfs_item_nr(i);
Chris Masondb945352007-10-15 16:15:53 -04003864
Zheng Yan31840ae2008-09-23 13:14:14 -04003865 if (!empty && push_items > 0) {
3866 if (path->slots[0] < i)
3867 break;
3868 if (path->slots[0] == i) {
3869 int space = btrfs_leaf_free_space(root, right);
3870 if (space + push_space * 2 > free_space)
3871 break;
3872 }
3873 }
3874
Chris Masonbe0e5c02007-01-26 15:51:26 -05003875 if (path->slots[0] == i)
Yan Zheng87b29b22008-12-17 10:21:48 -05003876 push_space += data_size;
Chris Masondb945352007-10-15 16:15:53 -04003877
3878 this_item_size = btrfs_item_size(right, item);
3879 if (this_item_size + sizeof(*item) + push_space > free_space)
Chris Masonbe0e5c02007-01-26 15:51:26 -05003880 break;
Chris Masondb945352007-10-15 16:15:53 -04003881
Chris Masonbe0e5c02007-01-26 15:51:26 -05003882 push_items++;
Chris Masondb945352007-10-15 16:15:53 -04003883 push_space += this_item_size + sizeof(*item);
Chris Masonbe0e5c02007-01-26 15:51:26 -05003884 }
Chris Masondb945352007-10-15 16:15:53 -04003885
Chris Masonbe0e5c02007-01-26 15:51:26 -05003886 if (push_items == 0) {
Chris Mason925baed2008-06-25 16:01:30 -04003887 ret = 1;
3888 goto out;
Chris Masonbe0e5c02007-01-26 15:51:26 -05003889 }
Dulshani Gunawardhanafae7f212013-10-31 10:30:08 +05303890 WARN_ON(!empty && push_items == btrfs_header_nritems(right));
Chris Mason5f39d392007-10-15 16:14:19 -04003891
Chris Masonbe0e5c02007-01-26 15:51:26 -05003892 /* push data from right to left */
Chris Mason5f39d392007-10-15 16:14:19 -04003893 copy_extent_buffer(left, right,
3894 btrfs_item_nr_offset(btrfs_header_nritems(left)),
3895 btrfs_item_nr_offset(0),
3896 push_items * sizeof(struct btrfs_item));
3897
Chris Mason123abc82007-03-14 14:14:43 -04003898 push_space = BTRFS_LEAF_DATA_SIZE(root) -
Chris Masond3977122009-01-05 21:25:51 -05003899 btrfs_item_offset_nr(right, push_items - 1);
Chris Mason5f39d392007-10-15 16:14:19 -04003900
3901 copy_extent_buffer(left, right, btrfs_leaf_data(left) +
Chris Masond6025572007-03-30 14:27:56 -04003902 leaf_data_end(root, left) - push_space,
3903 btrfs_leaf_data(right) +
Chris Mason5f39d392007-10-15 16:14:19 -04003904 btrfs_item_offset_nr(right, push_items - 1),
Chris Masond6025572007-03-30 14:27:56 -04003905 push_space);
Chris Mason5f39d392007-10-15 16:14:19 -04003906 old_left_nritems = btrfs_header_nritems(left);
Yan Zheng87b29b22008-12-17 10:21:48 -05003907 BUG_ON(old_left_nritems <= 0);
Chris Masoneb60cea2007-02-02 09:18:22 -05003908
Chris Masondb945352007-10-15 16:15:53 -04003909 old_left_item_size = btrfs_item_offset_nr(left, old_left_nritems - 1);
Chris Mason0783fcf2007-03-12 20:12:07 -04003910 for (i = old_left_nritems; i < old_left_nritems + push_items; i++) {
Chris Mason5f39d392007-10-15 16:14:19 -04003911 u32 ioff;
Chris Masondb945352007-10-15 16:15:53 -04003912
Ross Kirkdd3cc162013-09-16 15:58:09 +01003913 item = btrfs_item_nr(i);
Chris Masondb945352007-10-15 16:15:53 -04003914
Chris Masoncfed81a2012-03-03 07:40:03 -05003915 ioff = btrfs_token_item_offset(left, item, &token);
3916 btrfs_set_token_item_offset(left, item,
3917 ioff - (BTRFS_LEAF_DATA_SIZE(root) - old_left_item_size),
3918 &token);
Chris Masonbe0e5c02007-01-26 15:51:26 -05003919 }
Chris Mason5f39d392007-10-15 16:14:19 -04003920 btrfs_set_header_nritems(left, old_left_nritems + push_items);
Chris Masonbe0e5c02007-01-26 15:51:26 -05003921
3922 /* fixup right node */
Julia Lawall31b1a2b2012-11-03 10:58:34 +00003923 if (push_items > right_nritems)
3924 WARN(1, KERN_CRIT "push items %d nr %u\n", push_items,
Chris Masond3977122009-01-05 21:25:51 -05003925 right_nritems);
Chris Mason5f39d392007-10-15 16:14:19 -04003926
Chris Mason34a38212007-11-07 13:31:03 -05003927 if (push_items < right_nritems) {
3928 push_space = btrfs_item_offset_nr(right, push_items - 1) -
3929 leaf_data_end(root, right);
3930 memmove_extent_buffer(right, btrfs_leaf_data(right) +
3931 BTRFS_LEAF_DATA_SIZE(root) - push_space,
3932 btrfs_leaf_data(right) +
3933 leaf_data_end(root, right), push_space);
3934
3935 memmove_extent_buffer(right, btrfs_item_nr_offset(0),
Chris Mason5f39d392007-10-15 16:14:19 -04003936 btrfs_item_nr_offset(push_items),
3937 (btrfs_header_nritems(right) - push_items) *
3938 sizeof(struct btrfs_item));
Chris Mason34a38212007-11-07 13:31:03 -05003939 }
Yaneef1c492007-11-26 10:58:13 -05003940 right_nritems -= push_items;
3941 btrfs_set_header_nritems(right, right_nritems);
Chris Mason123abc82007-03-14 14:14:43 -04003942 push_space = BTRFS_LEAF_DATA_SIZE(root);
Chris Mason5f39d392007-10-15 16:14:19 -04003943 for (i = 0; i < right_nritems; i++) {
Ross Kirkdd3cc162013-09-16 15:58:09 +01003944 item = btrfs_item_nr(i);
Chris Masondb945352007-10-15 16:15:53 -04003945
Chris Masoncfed81a2012-03-03 07:40:03 -05003946 push_space = push_space - btrfs_token_item_size(right,
3947 item, &token);
3948 btrfs_set_token_item_offset(right, item, push_space, &token);
Chris Masondb945352007-10-15 16:15:53 -04003949 }
Chris Masoneb60cea2007-02-02 09:18:22 -05003950
Chris Mason5f39d392007-10-15 16:14:19 -04003951 btrfs_mark_buffer_dirty(left);
Chris Mason34a38212007-11-07 13:31:03 -05003952 if (right_nritems)
3953 btrfs_mark_buffer_dirty(right);
Yan, Zhengf0486c62010-05-16 10:46:25 -04003954 else
3955 clean_tree_block(trans, root, right);
Chris Mason098f59c2007-05-11 11:33:21 -04003956
Chris Mason5f39d392007-10-15 16:14:19 -04003957 btrfs_item_key(right, &disk_key, 0);
Tsutomu Itohd6a0a122013-04-16 05:18:02 +00003958 fixup_low_keys(root, path, &disk_key, 1);
Chris Masonbe0e5c02007-01-26 15:51:26 -05003959
3960 /* then fixup the leaf pointer in the path */
3961 if (path->slots[0] < push_items) {
3962 path->slots[0] += old_left_nritems;
Chris Mason925baed2008-06-25 16:01:30 -04003963 btrfs_tree_unlock(path->nodes[0]);
Chris Mason5f39d392007-10-15 16:14:19 -04003964 free_extent_buffer(path->nodes[0]);
3965 path->nodes[0] = left;
Chris Masonbe0e5c02007-01-26 15:51:26 -05003966 path->slots[1] -= 1;
3967 } else {
Chris Mason925baed2008-06-25 16:01:30 -04003968 btrfs_tree_unlock(left);
Chris Mason5f39d392007-10-15 16:14:19 -04003969 free_extent_buffer(left);
Chris Masonbe0e5c02007-01-26 15:51:26 -05003970 path->slots[0] -= push_items;
3971 }
Chris Masoneb60cea2007-02-02 09:18:22 -05003972 BUG_ON(path->slots[0] < 0);
Chris Masonaa5d6be2007-02-28 16:35:06 -05003973 return ret;
Chris Mason925baed2008-06-25 16:01:30 -04003974out:
3975 btrfs_tree_unlock(left);
3976 free_extent_buffer(left);
3977 return ret;
Chris Masonbe0e5c02007-01-26 15:51:26 -05003978}
3979
Chris Mason74123bd2007-02-02 11:05:29 -05003980/*
Chris Mason44871b12009-03-13 10:04:31 -04003981 * push some data in the path leaf to the left, trying to free up at
3982 * least data_size bytes. returns zero if the push worked, nonzero otherwise
Chris Mason99d8f832010-07-07 10:51:48 -04003983 *
3984 * max_slot can put a limit on how far into the leaf we'll push items. The
3985 * item at 'max_slot' won't be touched. Use (u32)-1 to make us push all the
3986 * items
Chris Mason44871b12009-03-13 10:04:31 -04003987 */
3988static int push_leaf_left(struct btrfs_trans_handle *trans, struct btrfs_root
Chris Mason99d8f832010-07-07 10:51:48 -04003989 *root, struct btrfs_path *path, int min_data_size,
3990 int data_size, int empty, u32 max_slot)
Chris Mason44871b12009-03-13 10:04:31 -04003991{
3992 struct extent_buffer *right = path->nodes[0];
3993 struct extent_buffer *left;
3994 int slot;
3995 int free_space;
3996 u32 right_nritems;
3997 int ret = 0;
3998
3999 slot = path->slots[1];
4000 if (slot == 0)
4001 return 1;
4002 if (!path->nodes[1])
4003 return 1;
4004
4005 right_nritems = btrfs_header_nritems(right);
4006 if (right_nritems == 0)
4007 return 1;
4008
4009 btrfs_assert_tree_locked(path->nodes[1]);
4010
4011 left = read_node_slot(root, path->nodes[1], slot - 1);
Tsutomu Itoh91ca3382011-01-05 02:32:22 +00004012 if (left == NULL)
4013 return 1;
4014
Chris Mason44871b12009-03-13 10:04:31 -04004015 btrfs_tree_lock(left);
4016 btrfs_set_lock_blocking(left);
4017
4018 free_space = btrfs_leaf_free_space(root, left);
4019 if (free_space < data_size) {
4020 ret = 1;
4021 goto out;
4022 }
4023
4024 /* cow and double check */
4025 ret = btrfs_cow_block(trans, root, left,
4026 path->nodes[1], slot - 1, &left);
4027 if (ret) {
4028 /* we hit -ENOSPC, but it isn't fatal here */
Jeff Mahoney79787ea2012-03-12 16:03:00 +01004029 if (ret == -ENOSPC)
4030 ret = 1;
Chris Mason44871b12009-03-13 10:04:31 -04004031 goto out;
4032 }
4033
4034 free_space = btrfs_leaf_free_space(root, left);
4035 if (free_space < data_size) {
4036 ret = 1;
4037 goto out;
4038 }
4039
Chris Mason99d8f832010-07-07 10:51:48 -04004040 return __push_leaf_left(trans, root, path, min_data_size,
4041 empty, left, free_space, right_nritems,
4042 max_slot);
Chris Mason44871b12009-03-13 10:04:31 -04004043out:
4044 btrfs_tree_unlock(left);
4045 free_extent_buffer(left);
4046 return ret;
4047}
4048
4049/*
Chris Mason74123bd2007-02-02 11:05:29 -05004050 * split the path's leaf in two, making sure there is at least data_size
4051 * available for the resulting leaf level of the path.
4052 */
Jeff Mahoney143bede2012-03-01 14:56:26 +01004053static noinline void copy_for_split(struct btrfs_trans_handle *trans,
4054 struct btrfs_root *root,
4055 struct btrfs_path *path,
4056 struct extent_buffer *l,
4057 struct extent_buffer *right,
4058 int slot, int mid, int nritems)
Chris Masonbe0e5c02007-01-26 15:51:26 -05004059{
Chris Masonbe0e5c02007-01-26 15:51:26 -05004060 int data_copy_size;
4061 int rt_data_off;
4062 int i;
Chris Masond4dbff92007-04-04 14:08:15 -04004063 struct btrfs_disk_key disk_key;
Chris Masoncfed81a2012-03-03 07:40:03 -05004064 struct btrfs_map_token token;
4065
4066 btrfs_init_map_token(&token);
Chris Masonbe0e5c02007-01-26 15:51:26 -05004067
Chris Mason5f39d392007-10-15 16:14:19 -04004068 nritems = nritems - mid;
4069 btrfs_set_header_nritems(right, nritems);
4070 data_copy_size = btrfs_item_end_nr(l, mid) - leaf_data_end(root, l);
4071
4072 copy_extent_buffer(right, l, btrfs_item_nr_offset(0),
4073 btrfs_item_nr_offset(mid),
4074 nritems * sizeof(struct btrfs_item));
4075
4076 copy_extent_buffer(right, l,
Chris Masond6025572007-03-30 14:27:56 -04004077 btrfs_leaf_data(right) + BTRFS_LEAF_DATA_SIZE(root) -
4078 data_copy_size, btrfs_leaf_data(l) +
4079 leaf_data_end(root, l), data_copy_size);
Chris Mason74123bd2007-02-02 11:05:29 -05004080
Chris Mason5f39d392007-10-15 16:14:19 -04004081 rt_data_off = BTRFS_LEAF_DATA_SIZE(root) -
4082 btrfs_item_end_nr(l, mid);
4083
4084 for (i = 0; i < nritems; i++) {
Ross Kirkdd3cc162013-09-16 15:58:09 +01004085 struct btrfs_item *item = btrfs_item_nr(i);
Chris Masondb945352007-10-15 16:15:53 -04004086 u32 ioff;
4087
Chris Masoncfed81a2012-03-03 07:40:03 -05004088 ioff = btrfs_token_item_offset(right, item, &token);
4089 btrfs_set_token_item_offset(right, item,
4090 ioff + rt_data_off, &token);
Chris Mason0783fcf2007-03-12 20:12:07 -04004091 }
Chris Mason74123bd2007-02-02 11:05:29 -05004092
Chris Mason5f39d392007-10-15 16:14:19 -04004093 btrfs_set_header_nritems(l, mid);
Chris Mason5f39d392007-10-15 16:14:19 -04004094 btrfs_item_key(right, &disk_key, 0);
Jeff Mahoney143bede2012-03-01 14:56:26 +01004095 insert_ptr(trans, root, path, &disk_key, right->start,
Jan Schmidtc3e06962012-06-21 11:01:06 +02004096 path->slots[1] + 1, 1);
Chris Mason5f39d392007-10-15 16:14:19 -04004097
4098 btrfs_mark_buffer_dirty(right);
4099 btrfs_mark_buffer_dirty(l);
Chris Masoneb60cea2007-02-02 09:18:22 -05004100 BUG_ON(path->slots[0] != slot);
Chris Mason5f39d392007-10-15 16:14:19 -04004101
Chris Masonbe0e5c02007-01-26 15:51:26 -05004102 if (mid <= slot) {
Chris Mason925baed2008-06-25 16:01:30 -04004103 btrfs_tree_unlock(path->nodes[0]);
Chris Mason5f39d392007-10-15 16:14:19 -04004104 free_extent_buffer(path->nodes[0]);
4105 path->nodes[0] = right;
Chris Masonbe0e5c02007-01-26 15:51:26 -05004106 path->slots[0] -= mid;
4107 path->slots[1] += 1;
Chris Mason925baed2008-06-25 16:01:30 -04004108 } else {
4109 btrfs_tree_unlock(right);
Chris Mason5f39d392007-10-15 16:14:19 -04004110 free_extent_buffer(right);
Chris Mason925baed2008-06-25 16:01:30 -04004111 }
Chris Mason5f39d392007-10-15 16:14:19 -04004112
Chris Masoneb60cea2007-02-02 09:18:22 -05004113 BUG_ON(path->slots[0] < 0);
Chris Mason44871b12009-03-13 10:04:31 -04004114}
4115
4116/*
Chris Mason99d8f832010-07-07 10:51:48 -04004117 * double splits happen when we need to insert a big item in the middle
4118 * of a leaf. A double split can leave us with 3 mostly empty leaves:
4119 * leaf: [ slots 0 - N] [ our target ] [ N + 1 - total in leaf ]
4120 * A B C
4121 *
4122 * We avoid this by trying to push the items on either side of our target
4123 * into the adjacent leaves. If all goes well we can avoid the double split
4124 * completely.
4125 */
4126static noinline int push_for_double_split(struct btrfs_trans_handle *trans,
4127 struct btrfs_root *root,
4128 struct btrfs_path *path,
4129 int data_size)
4130{
4131 int ret;
4132 int progress = 0;
4133 int slot;
4134 u32 nritems;
Filipe David Borba Manana5a4267c2013-11-25 03:20:46 +00004135 int space_needed = data_size;
Chris Mason99d8f832010-07-07 10:51:48 -04004136
4137 slot = path->slots[0];
Filipe David Borba Manana5a4267c2013-11-25 03:20:46 +00004138 if (slot < btrfs_header_nritems(path->nodes[0]))
4139 space_needed -= btrfs_leaf_free_space(root, path->nodes[0]);
Chris Mason99d8f832010-07-07 10:51:48 -04004140
4141 /*
4142 * try to push all the items after our slot into the
4143 * right leaf
4144 */
Filipe David Borba Manana5a4267c2013-11-25 03:20:46 +00004145 ret = push_leaf_right(trans, root, path, 1, space_needed, 0, slot);
Chris Mason99d8f832010-07-07 10:51:48 -04004146 if (ret < 0)
4147 return ret;
4148
4149 if (ret == 0)
4150 progress++;
4151
4152 nritems = btrfs_header_nritems(path->nodes[0]);
4153 /*
4154 * our goal is to get our slot at the start or end of a leaf. If
4155 * we've done so we're done
4156 */
4157 if (path->slots[0] == 0 || path->slots[0] == nritems)
4158 return 0;
4159
4160 if (btrfs_leaf_free_space(root, path->nodes[0]) >= data_size)
4161 return 0;
4162
4163 /* try to push all the items before our slot into the next leaf */
4164 slot = path->slots[0];
Filipe David Borba Manana5a4267c2013-11-25 03:20:46 +00004165 ret = push_leaf_left(trans, root, path, 1, space_needed, 0, slot);
Chris Mason99d8f832010-07-07 10:51:48 -04004166 if (ret < 0)
4167 return ret;
4168
4169 if (ret == 0)
4170 progress++;
4171
4172 if (progress)
4173 return 0;
4174 return 1;
4175}
4176
4177/*
Chris Mason44871b12009-03-13 10:04:31 -04004178 * split the path's leaf in two, making sure there is at least data_size
4179 * available for the resulting leaf level of the path.
4180 *
4181 * returns 0 if all went well and < 0 on failure.
4182 */
4183static noinline int split_leaf(struct btrfs_trans_handle *trans,
4184 struct btrfs_root *root,
4185 struct btrfs_key *ins_key,
4186 struct btrfs_path *path, int data_size,
4187 int extend)
4188{
Yan Zheng5d4f98a2009-06-10 10:45:14 -04004189 struct btrfs_disk_key disk_key;
Chris Mason44871b12009-03-13 10:04:31 -04004190 struct extent_buffer *l;
4191 u32 nritems;
4192 int mid;
4193 int slot;
4194 struct extent_buffer *right;
4195 int ret = 0;
4196 int wret;
Yan Zheng5d4f98a2009-06-10 10:45:14 -04004197 int split;
Chris Mason44871b12009-03-13 10:04:31 -04004198 int num_doubles = 0;
Chris Mason99d8f832010-07-07 10:51:48 -04004199 int tried_avoid_double = 0;
Chris Mason44871b12009-03-13 10:04:31 -04004200
Yan, Zhenga5719522009-09-24 09:17:31 -04004201 l = path->nodes[0];
4202 slot = path->slots[0];
4203 if (extend && data_size + btrfs_item_size_nr(l, slot) +
4204 sizeof(struct btrfs_item) > BTRFS_LEAF_DATA_SIZE(root))
4205 return -EOVERFLOW;
4206
Chris Mason44871b12009-03-13 10:04:31 -04004207 /* first try to make some room by pushing left and right */
Liu Bo33157e02013-05-22 12:07:06 +00004208 if (data_size && path->nodes[1]) {
Filipe David Borba Manana5a4267c2013-11-25 03:20:46 +00004209 int space_needed = data_size;
4210
4211 if (slot < btrfs_header_nritems(l))
4212 space_needed -= btrfs_leaf_free_space(root, l);
4213
4214 wret = push_leaf_right(trans, root, path, space_needed,
4215 space_needed, 0, 0);
Chris Mason44871b12009-03-13 10:04:31 -04004216 if (wret < 0)
4217 return wret;
4218 if (wret) {
Filipe David Borba Manana5a4267c2013-11-25 03:20:46 +00004219 wret = push_leaf_left(trans, root, path, space_needed,
4220 space_needed, 0, (u32)-1);
Chris Mason44871b12009-03-13 10:04:31 -04004221 if (wret < 0)
4222 return wret;
4223 }
4224 l = path->nodes[0];
4225
4226 /* did the pushes work? */
4227 if (btrfs_leaf_free_space(root, l) >= data_size)
4228 return 0;
4229 }
4230
4231 if (!path->nodes[1]) {
Liu Bofdd99c72013-05-22 12:06:51 +00004232 ret = insert_new_root(trans, root, path, 1);
Chris Mason44871b12009-03-13 10:04:31 -04004233 if (ret)
4234 return ret;
4235 }
4236again:
Yan Zheng5d4f98a2009-06-10 10:45:14 -04004237 split = 1;
Chris Mason44871b12009-03-13 10:04:31 -04004238 l = path->nodes[0];
4239 slot = path->slots[0];
4240 nritems = btrfs_header_nritems(l);
4241 mid = (nritems + 1) / 2;
4242
Yan Zheng5d4f98a2009-06-10 10:45:14 -04004243 if (mid <= slot) {
4244 if (nritems == 1 ||
4245 leaf_space_used(l, mid, nritems - mid) + data_size >
4246 BTRFS_LEAF_DATA_SIZE(root)) {
4247 if (slot >= nritems) {
4248 split = 0;
4249 } else {
4250 mid = slot;
4251 if (mid != nritems &&
4252 leaf_space_used(l, mid, nritems - mid) +
4253 data_size > BTRFS_LEAF_DATA_SIZE(root)) {
Chris Mason99d8f832010-07-07 10:51:48 -04004254 if (data_size && !tried_avoid_double)
4255 goto push_for_double;
Yan Zheng5d4f98a2009-06-10 10:45:14 -04004256 split = 2;
4257 }
4258 }
4259 }
4260 } else {
4261 if (leaf_space_used(l, 0, mid) + data_size >
4262 BTRFS_LEAF_DATA_SIZE(root)) {
4263 if (!extend && data_size && slot == 0) {
4264 split = 0;
4265 } else if ((extend || !data_size) && slot == 0) {
4266 mid = 1;
4267 } else {
4268 mid = slot;
4269 if (mid != nritems &&
4270 leaf_space_used(l, mid, nritems - mid) +
4271 data_size > BTRFS_LEAF_DATA_SIZE(root)) {
Chris Mason99d8f832010-07-07 10:51:48 -04004272 if (data_size && !tried_avoid_double)
4273 goto push_for_double;
Dulshani Gunawardhana67871252013-10-31 10:33:04 +05304274 split = 2;
Yan Zheng5d4f98a2009-06-10 10:45:14 -04004275 }
4276 }
4277 }
4278 }
4279
4280 if (split == 0)
4281 btrfs_cpu_key_to_disk(&disk_key, ins_key);
4282 else
4283 btrfs_item_key(l, &disk_key, mid);
4284
4285 right = btrfs_alloc_free_block(trans, root, root->leafsize, 0,
Chris Mason44871b12009-03-13 10:04:31 -04004286 root->root_key.objectid,
Jan Schmidt5581a512012-05-16 17:04:52 +02004287 &disk_key, 0, l->start, 0);
Yan, Zhengf0486c62010-05-16 10:46:25 -04004288 if (IS_ERR(right))
Chris Mason44871b12009-03-13 10:04:31 -04004289 return PTR_ERR(right);
Yan, Zhengf0486c62010-05-16 10:46:25 -04004290
4291 root_add_used(root, root->leafsize);
Chris Mason44871b12009-03-13 10:04:31 -04004292
4293 memset_extent_buffer(right, 0, 0, sizeof(struct btrfs_header));
4294 btrfs_set_header_bytenr(right, right->start);
4295 btrfs_set_header_generation(right, trans->transid);
Yan Zheng5d4f98a2009-06-10 10:45:14 -04004296 btrfs_set_header_backref_rev(right, BTRFS_MIXED_BACKREF_REV);
Chris Mason44871b12009-03-13 10:04:31 -04004297 btrfs_set_header_owner(right, root->root_key.objectid);
4298 btrfs_set_header_level(right, 0);
4299 write_extent_buffer(right, root->fs_info->fsid,
Ross Kirk0a4e5582013-09-24 10:12:38 +01004300 btrfs_header_fsid(), BTRFS_FSID_SIZE);
Chris Mason44871b12009-03-13 10:04:31 -04004301
4302 write_extent_buffer(right, root->fs_info->chunk_tree_uuid,
Geert Uytterhoevenb308bc22013-08-20 13:20:15 +02004303 btrfs_header_chunk_tree_uuid(right),
Chris Mason44871b12009-03-13 10:04:31 -04004304 BTRFS_UUID_SIZE);
4305
Yan Zheng5d4f98a2009-06-10 10:45:14 -04004306 if (split == 0) {
4307 if (mid <= slot) {
4308 btrfs_set_header_nritems(right, 0);
Jeff Mahoney143bede2012-03-01 14:56:26 +01004309 insert_ptr(trans, root, path, &disk_key, right->start,
Jan Schmidtc3e06962012-06-21 11:01:06 +02004310 path->slots[1] + 1, 1);
Yan Zheng5d4f98a2009-06-10 10:45:14 -04004311 btrfs_tree_unlock(path->nodes[0]);
4312 free_extent_buffer(path->nodes[0]);
4313 path->nodes[0] = right;
4314 path->slots[0] = 0;
4315 path->slots[1] += 1;
4316 } else {
4317 btrfs_set_header_nritems(right, 0);
Jeff Mahoney143bede2012-03-01 14:56:26 +01004318 insert_ptr(trans, root, path, &disk_key, right->start,
Jan Schmidtc3e06962012-06-21 11:01:06 +02004319 path->slots[1], 1);
Yan Zheng5d4f98a2009-06-10 10:45:14 -04004320 btrfs_tree_unlock(path->nodes[0]);
4321 free_extent_buffer(path->nodes[0]);
4322 path->nodes[0] = right;
4323 path->slots[0] = 0;
Jeff Mahoney143bede2012-03-01 14:56:26 +01004324 if (path->slots[1] == 0)
Tsutomu Itohd6a0a122013-04-16 05:18:02 +00004325 fixup_low_keys(root, path, &disk_key, 1);
Chris Mason44871b12009-03-13 10:04:31 -04004326 }
Yan Zheng5d4f98a2009-06-10 10:45:14 -04004327 btrfs_mark_buffer_dirty(right);
4328 return ret;
Chris Mason44871b12009-03-13 10:04:31 -04004329 }
4330
Jeff Mahoney143bede2012-03-01 14:56:26 +01004331 copy_for_split(trans, root, path, l, right, slot, mid, nritems);
Chris Mason44871b12009-03-13 10:04:31 -04004332
Yan Zheng5d4f98a2009-06-10 10:45:14 -04004333 if (split == 2) {
Chris Masoncc0c5532007-10-25 15:42:57 -04004334 BUG_ON(num_doubles != 0);
4335 num_doubles++;
4336 goto again;
Chris Mason3326d1b2007-10-15 16:18:25 -04004337 }
Chris Mason44871b12009-03-13 10:04:31 -04004338
Jeff Mahoney143bede2012-03-01 14:56:26 +01004339 return 0;
Chris Mason99d8f832010-07-07 10:51:48 -04004340
4341push_for_double:
4342 push_for_double_split(trans, root, path, data_size);
4343 tried_avoid_double = 1;
4344 if (btrfs_leaf_free_space(root, path->nodes[0]) >= data_size)
4345 return 0;
4346 goto again;
Chris Masonbe0e5c02007-01-26 15:51:26 -05004347}
4348
Yan, Zhengad48fd752009-11-12 09:33:58 +00004349static noinline int setup_leaf_for_split(struct btrfs_trans_handle *trans,
4350 struct btrfs_root *root,
4351 struct btrfs_path *path, int ins_len)
Chris Mason459931e2008-12-10 09:10:46 -05004352{
Yan, Zhengad48fd752009-11-12 09:33:58 +00004353 struct btrfs_key key;
Chris Mason459931e2008-12-10 09:10:46 -05004354 struct extent_buffer *leaf;
Yan, Zhengad48fd752009-11-12 09:33:58 +00004355 struct btrfs_file_extent_item *fi;
4356 u64 extent_len = 0;
4357 u32 item_size;
4358 int ret;
Chris Mason459931e2008-12-10 09:10:46 -05004359
4360 leaf = path->nodes[0];
Yan, Zhengad48fd752009-11-12 09:33:58 +00004361 btrfs_item_key_to_cpu(leaf, &key, path->slots[0]);
4362
4363 BUG_ON(key.type != BTRFS_EXTENT_DATA_KEY &&
4364 key.type != BTRFS_EXTENT_CSUM_KEY);
4365
4366 if (btrfs_leaf_free_space(root, leaf) >= ins_len)
4367 return 0;
Chris Mason459931e2008-12-10 09:10:46 -05004368
4369 item_size = btrfs_item_size_nr(leaf, path->slots[0]);
Yan, Zhengad48fd752009-11-12 09:33:58 +00004370 if (key.type == BTRFS_EXTENT_DATA_KEY) {
4371 fi = btrfs_item_ptr(leaf, path->slots[0],
4372 struct btrfs_file_extent_item);
4373 extent_len = btrfs_file_extent_num_bytes(leaf, fi);
4374 }
David Sterbab3b4aa72011-04-21 01:20:15 +02004375 btrfs_release_path(path);
Chris Mason459931e2008-12-10 09:10:46 -05004376
Chris Mason459931e2008-12-10 09:10:46 -05004377 path->keep_locks = 1;
Yan, Zhengad48fd752009-11-12 09:33:58 +00004378 path->search_for_split = 1;
4379 ret = btrfs_search_slot(trans, root, &key, path, 0, 1);
Chris Mason459931e2008-12-10 09:10:46 -05004380 path->search_for_split = 0;
Yan, Zhengad48fd752009-11-12 09:33:58 +00004381 if (ret < 0)
4382 goto err;
Chris Mason459931e2008-12-10 09:10:46 -05004383
Yan, Zhengad48fd752009-11-12 09:33:58 +00004384 ret = -EAGAIN;
4385 leaf = path->nodes[0];
Chris Mason459931e2008-12-10 09:10:46 -05004386 /* if our item isn't there or got smaller, return now */
Yan, Zhengad48fd752009-11-12 09:33:58 +00004387 if (ret > 0 || item_size != btrfs_item_size_nr(leaf, path->slots[0]))
4388 goto err;
4389
Chris Mason109f6ae2010-04-02 09:20:18 -04004390 /* the leaf has changed, it now has room. return now */
4391 if (btrfs_leaf_free_space(root, path->nodes[0]) >= ins_len)
4392 goto err;
4393
Yan, Zhengad48fd752009-11-12 09:33:58 +00004394 if (key.type == BTRFS_EXTENT_DATA_KEY) {
4395 fi = btrfs_item_ptr(leaf, path->slots[0],
4396 struct btrfs_file_extent_item);
4397 if (extent_len != btrfs_file_extent_num_bytes(leaf, fi))
4398 goto err;
Chris Mason459931e2008-12-10 09:10:46 -05004399 }
4400
Chris Masonb9473432009-03-13 11:00:37 -04004401 btrfs_set_path_blocking(path);
Yan, Zhengad48fd752009-11-12 09:33:58 +00004402 ret = split_leaf(trans, root, &key, path, ins_len, 1);
Yan, Zhengf0486c62010-05-16 10:46:25 -04004403 if (ret)
4404 goto err;
Chris Mason459931e2008-12-10 09:10:46 -05004405
Yan, Zhengad48fd752009-11-12 09:33:58 +00004406 path->keep_locks = 0;
Chris Masonb9473432009-03-13 11:00:37 -04004407 btrfs_unlock_up_safe(path, 1);
Yan, Zhengad48fd752009-11-12 09:33:58 +00004408 return 0;
4409err:
4410 path->keep_locks = 0;
4411 return ret;
4412}
4413
4414static noinline int split_item(struct btrfs_trans_handle *trans,
4415 struct btrfs_root *root,
4416 struct btrfs_path *path,
4417 struct btrfs_key *new_key,
4418 unsigned long split_offset)
4419{
4420 struct extent_buffer *leaf;
4421 struct btrfs_item *item;
4422 struct btrfs_item *new_item;
4423 int slot;
4424 char *buf;
4425 u32 nritems;
4426 u32 item_size;
4427 u32 orig_offset;
4428 struct btrfs_disk_key disk_key;
4429
Chris Masonb9473432009-03-13 11:00:37 -04004430 leaf = path->nodes[0];
4431 BUG_ON(btrfs_leaf_free_space(root, leaf) < sizeof(struct btrfs_item));
4432
Chris Masonb4ce94d2009-02-04 09:25:08 -05004433 btrfs_set_path_blocking(path);
4434
Ross Kirkdd3cc162013-09-16 15:58:09 +01004435 item = btrfs_item_nr(path->slots[0]);
Chris Mason459931e2008-12-10 09:10:46 -05004436 orig_offset = btrfs_item_offset(leaf, item);
4437 item_size = btrfs_item_size(leaf, item);
4438
Chris Mason459931e2008-12-10 09:10:46 -05004439 buf = kmalloc(item_size, GFP_NOFS);
Yan, Zhengad48fd752009-11-12 09:33:58 +00004440 if (!buf)
4441 return -ENOMEM;
4442
Chris Mason459931e2008-12-10 09:10:46 -05004443 read_extent_buffer(leaf, buf, btrfs_item_ptr_offset(leaf,
4444 path->slots[0]), item_size);
Yan, Zhengad48fd752009-11-12 09:33:58 +00004445
Chris Mason459931e2008-12-10 09:10:46 -05004446 slot = path->slots[0] + 1;
Chris Mason459931e2008-12-10 09:10:46 -05004447 nritems = btrfs_header_nritems(leaf);
Chris Mason459931e2008-12-10 09:10:46 -05004448 if (slot != nritems) {
4449 /* shift the items */
4450 memmove_extent_buffer(leaf, btrfs_item_nr_offset(slot + 1),
Yan, Zhengad48fd752009-11-12 09:33:58 +00004451 btrfs_item_nr_offset(slot),
4452 (nritems - slot) * sizeof(struct btrfs_item));
Chris Mason459931e2008-12-10 09:10:46 -05004453 }
4454
4455 btrfs_cpu_key_to_disk(&disk_key, new_key);
4456 btrfs_set_item_key(leaf, &disk_key, slot);
4457
Ross Kirkdd3cc162013-09-16 15:58:09 +01004458 new_item = btrfs_item_nr(slot);
Chris Mason459931e2008-12-10 09:10:46 -05004459
4460 btrfs_set_item_offset(leaf, new_item, orig_offset);
4461 btrfs_set_item_size(leaf, new_item, item_size - split_offset);
4462
4463 btrfs_set_item_offset(leaf, item,
4464 orig_offset + item_size - split_offset);
4465 btrfs_set_item_size(leaf, item, split_offset);
4466
4467 btrfs_set_header_nritems(leaf, nritems + 1);
4468
4469 /* write the data for the start of the original item */
4470 write_extent_buffer(leaf, buf,
4471 btrfs_item_ptr_offset(leaf, path->slots[0]),
4472 split_offset);
4473
4474 /* write the data for the new item */
4475 write_extent_buffer(leaf, buf + split_offset,
4476 btrfs_item_ptr_offset(leaf, slot),
4477 item_size - split_offset);
4478 btrfs_mark_buffer_dirty(leaf);
4479
Yan, Zhengad48fd752009-11-12 09:33:58 +00004480 BUG_ON(btrfs_leaf_free_space(root, leaf) < 0);
Chris Mason459931e2008-12-10 09:10:46 -05004481 kfree(buf);
Yan, Zhengad48fd752009-11-12 09:33:58 +00004482 return 0;
4483}
4484
4485/*
4486 * This function splits a single item into two items,
4487 * giving 'new_key' to the new item and splitting the
4488 * old one at split_offset (from the start of the item).
4489 *
4490 * The path may be released by this operation. After
4491 * the split, the path is pointing to the old item. The
4492 * new item is going to be in the same node as the old one.
4493 *
4494 * Note, the item being split must be smaller enough to live alone on
4495 * a tree block with room for one extra struct btrfs_item
4496 *
4497 * This allows us to split the item in place, keeping a lock on the
4498 * leaf the entire time.
4499 */
4500int btrfs_split_item(struct btrfs_trans_handle *trans,
4501 struct btrfs_root *root,
4502 struct btrfs_path *path,
4503 struct btrfs_key *new_key,
4504 unsigned long split_offset)
4505{
4506 int ret;
4507 ret = setup_leaf_for_split(trans, root, path,
4508 sizeof(struct btrfs_item));
4509 if (ret)
4510 return ret;
4511
4512 ret = split_item(trans, root, path, new_key, split_offset);
Chris Mason459931e2008-12-10 09:10:46 -05004513 return ret;
4514}
4515
4516/*
Yan, Zhengad48fd752009-11-12 09:33:58 +00004517 * This function duplicate a item, giving 'new_key' to the new item.
4518 * It guarantees both items live in the same tree leaf and the new item
4519 * is contiguous with the original item.
4520 *
4521 * This allows us to split file extent in place, keeping a lock on the
4522 * leaf the entire time.
4523 */
4524int btrfs_duplicate_item(struct btrfs_trans_handle *trans,
4525 struct btrfs_root *root,
4526 struct btrfs_path *path,
4527 struct btrfs_key *new_key)
4528{
4529 struct extent_buffer *leaf;
4530 int ret;
4531 u32 item_size;
4532
4533 leaf = path->nodes[0];
4534 item_size = btrfs_item_size_nr(leaf, path->slots[0]);
4535 ret = setup_leaf_for_split(trans, root, path,
4536 item_size + sizeof(struct btrfs_item));
4537 if (ret)
4538 return ret;
4539
4540 path->slots[0]++;
Tsutomu Itohafe5fea2013-04-16 05:18:22 +00004541 setup_items_for_insert(root, path, new_key, &item_size,
Jeff Mahoney143bede2012-03-01 14:56:26 +01004542 item_size, item_size +
4543 sizeof(struct btrfs_item), 1);
Yan, Zhengad48fd752009-11-12 09:33:58 +00004544 leaf = path->nodes[0];
4545 memcpy_extent_buffer(leaf,
4546 btrfs_item_ptr_offset(leaf, path->slots[0]),
4547 btrfs_item_ptr_offset(leaf, path->slots[0] - 1),
4548 item_size);
4549 return 0;
4550}
4551
4552/*
Chris Masond352ac62008-09-29 15:18:18 -04004553 * make the item pointed to by the path smaller. new_size indicates
4554 * how small to make it, and from_end tells us if we just chop bytes
4555 * off the end of the item or if we shift the item to chop bytes off
4556 * the front.
4557 */
Tsutomu Itohafe5fea2013-04-16 05:18:22 +00004558void btrfs_truncate_item(struct btrfs_root *root, struct btrfs_path *path,
Jeff Mahoney143bede2012-03-01 14:56:26 +01004559 u32 new_size, int from_end)
Chris Masonb18c6682007-04-17 13:26:50 -04004560{
Chris Masonb18c6682007-04-17 13:26:50 -04004561 int slot;
Chris Mason5f39d392007-10-15 16:14:19 -04004562 struct extent_buffer *leaf;
4563 struct btrfs_item *item;
Chris Masonb18c6682007-04-17 13:26:50 -04004564 u32 nritems;
4565 unsigned int data_end;
4566 unsigned int old_data_start;
4567 unsigned int old_size;
4568 unsigned int size_diff;
4569 int i;
Chris Masoncfed81a2012-03-03 07:40:03 -05004570 struct btrfs_map_token token;
4571
4572 btrfs_init_map_token(&token);
Chris Masonb18c6682007-04-17 13:26:50 -04004573
Chris Mason5f39d392007-10-15 16:14:19 -04004574 leaf = path->nodes[0];
Chris Mason179e29e2007-11-01 11:28:41 -04004575 slot = path->slots[0];
4576
4577 old_size = btrfs_item_size_nr(leaf, slot);
4578 if (old_size == new_size)
Jeff Mahoney143bede2012-03-01 14:56:26 +01004579 return;
Chris Masonb18c6682007-04-17 13:26:50 -04004580
Chris Mason5f39d392007-10-15 16:14:19 -04004581 nritems = btrfs_header_nritems(leaf);
Chris Masonb18c6682007-04-17 13:26:50 -04004582 data_end = leaf_data_end(root, leaf);
4583
Chris Mason5f39d392007-10-15 16:14:19 -04004584 old_data_start = btrfs_item_offset_nr(leaf, slot);
Chris Mason179e29e2007-11-01 11:28:41 -04004585
Chris Masonb18c6682007-04-17 13:26:50 -04004586 size_diff = old_size - new_size;
4587
4588 BUG_ON(slot < 0);
4589 BUG_ON(slot >= nritems);
4590
4591 /*
4592 * item0..itemN ... dataN.offset..dataN.size .. data0.size
4593 */
4594 /* first correct the data pointers */
4595 for (i = slot; i < nritems; i++) {
Chris Mason5f39d392007-10-15 16:14:19 -04004596 u32 ioff;
Ross Kirkdd3cc162013-09-16 15:58:09 +01004597 item = btrfs_item_nr(i);
Chris Masondb945352007-10-15 16:15:53 -04004598
Chris Masoncfed81a2012-03-03 07:40:03 -05004599 ioff = btrfs_token_item_offset(leaf, item, &token);
4600 btrfs_set_token_item_offset(leaf, item,
4601 ioff + size_diff, &token);
Chris Masonb18c6682007-04-17 13:26:50 -04004602 }
Chris Masondb945352007-10-15 16:15:53 -04004603
Chris Masonb18c6682007-04-17 13:26:50 -04004604 /* shift the data */
Chris Mason179e29e2007-11-01 11:28:41 -04004605 if (from_end) {
4606 memmove_extent_buffer(leaf, btrfs_leaf_data(leaf) +
4607 data_end + size_diff, btrfs_leaf_data(leaf) +
4608 data_end, old_data_start + new_size - data_end);
4609 } else {
4610 struct btrfs_disk_key disk_key;
4611 u64 offset;
4612
4613 btrfs_item_key(leaf, &disk_key, slot);
4614
4615 if (btrfs_disk_key_type(&disk_key) == BTRFS_EXTENT_DATA_KEY) {
4616 unsigned long ptr;
4617 struct btrfs_file_extent_item *fi;
4618
4619 fi = btrfs_item_ptr(leaf, slot,
4620 struct btrfs_file_extent_item);
4621 fi = (struct btrfs_file_extent_item *)(
4622 (unsigned long)fi - size_diff);
4623
4624 if (btrfs_file_extent_type(leaf, fi) ==
4625 BTRFS_FILE_EXTENT_INLINE) {
4626 ptr = btrfs_item_ptr_offset(leaf, slot);
4627 memmove_extent_buffer(leaf, ptr,
Chris Masond3977122009-01-05 21:25:51 -05004628 (unsigned long)fi,
4629 offsetof(struct btrfs_file_extent_item,
Chris Mason179e29e2007-11-01 11:28:41 -04004630 disk_bytenr));
4631 }
4632 }
4633
4634 memmove_extent_buffer(leaf, btrfs_leaf_data(leaf) +
4635 data_end + size_diff, btrfs_leaf_data(leaf) +
4636 data_end, old_data_start - data_end);
4637
4638 offset = btrfs_disk_key_offset(&disk_key);
4639 btrfs_set_disk_key_offset(&disk_key, offset + size_diff);
4640 btrfs_set_item_key(leaf, &disk_key, slot);
4641 if (slot == 0)
Tsutomu Itohd6a0a122013-04-16 05:18:02 +00004642 fixup_low_keys(root, path, &disk_key, 1);
Chris Mason179e29e2007-11-01 11:28:41 -04004643 }
Chris Mason5f39d392007-10-15 16:14:19 -04004644
Ross Kirkdd3cc162013-09-16 15:58:09 +01004645 item = btrfs_item_nr(slot);
Chris Mason5f39d392007-10-15 16:14:19 -04004646 btrfs_set_item_size(leaf, item, new_size);
4647 btrfs_mark_buffer_dirty(leaf);
Chris Masonb18c6682007-04-17 13:26:50 -04004648
Chris Mason5f39d392007-10-15 16:14:19 -04004649 if (btrfs_leaf_free_space(root, leaf) < 0) {
4650 btrfs_print_leaf(root, leaf);
Chris Masonb18c6682007-04-17 13:26:50 -04004651 BUG();
Chris Mason5f39d392007-10-15 16:14:19 -04004652 }
Chris Masonb18c6682007-04-17 13:26:50 -04004653}
4654
Chris Masond352ac62008-09-29 15:18:18 -04004655/*
Stefan Behrens8f69dbd2013-05-07 10:23:30 +00004656 * make the item pointed to by the path bigger, data_size is the added size.
Chris Masond352ac62008-09-29 15:18:18 -04004657 */
Tsutomu Itoh4b90c682013-04-16 05:18:49 +00004658void btrfs_extend_item(struct btrfs_root *root, struct btrfs_path *path,
Jeff Mahoney143bede2012-03-01 14:56:26 +01004659 u32 data_size)
Chris Mason6567e832007-04-16 09:22:45 -04004660{
Chris Mason6567e832007-04-16 09:22:45 -04004661 int slot;
Chris Mason5f39d392007-10-15 16:14:19 -04004662 struct extent_buffer *leaf;
4663 struct btrfs_item *item;
Chris Mason6567e832007-04-16 09:22:45 -04004664 u32 nritems;
4665 unsigned int data_end;
4666 unsigned int old_data;
4667 unsigned int old_size;
4668 int i;
Chris Masoncfed81a2012-03-03 07:40:03 -05004669 struct btrfs_map_token token;
4670
4671 btrfs_init_map_token(&token);
Chris Mason6567e832007-04-16 09:22:45 -04004672
Chris Mason5f39d392007-10-15 16:14:19 -04004673 leaf = path->nodes[0];
Chris Mason6567e832007-04-16 09:22:45 -04004674
Chris Mason5f39d392007-10-15 16:14:19 -04004675 nritems = btrfs_header_nritems(leaf);
Chris Mason6567e832007-04-16 09:22:45 -04004676 data_end = leaf_data_end(root, leaf);
4677
Chris Mason5f39d392007-10-15 16:14:19 -04004678 if (btrfs_leaf_free_space(root, leaf) < data_size) {
4679 btrfs_print_leaf(root, leaf);
Chris Mason6567e832007-04-16 09:22:45 -04004680 BUG();
Chris Mason5f39d392007-10-15 16:14:19 -04004681 }
Chris Mason6567e832007-04-16 09:22:45 -04004682 slot = path->slots[0];
Chris Mason5f39d392007-10-15 16:14:19 -04004683 old_data = btrfs_item_end_nr(leaf, slot);
Chris Mason6567e832007-04-16 09:22:45 -04004684
4685 BUG_ON(slot < 0);
Chris Mason3326d1b2007-10-15 16:18:25 -04004686 if (slot >= nritems) {
4687 btrfs_print_leaf(root, leaf);
Frank Holtonefe120a2013-12-20 11:37:06 -05004688 btrfs_crit(root->fs_info, "slot %d too large, nritems %d",
Chris Masond3977122009-01-05 21:25:51 -05004689 slot, nritems);
Chris Mason3326d1b2007-10-15 16:18:25 -04004690 BUG_ON(1);
4691 }
Chris Mason6567e832007-04-16 09:22:45 -04004692
4693 /*
4694 * item0..itemN ... dataN.offset..dataN.size .. data0.size
4695 */
4696 /* first correct the data pointers */
4697 for (i = slot; i < nritems; i++) {
Chris Mason5f39d392007-10-15 16:14:19 -04004698 u32 ioff;
Ross Kirkdd3cc162013-09-16 15:58:09 +01004699 item = btrfs_item_nr(i);
Chris Masondb945352007-10-15 16:15:53 -04004700
Chris Masoncfed81a2012-03-03 07:40:03 -05004701 ioff = btrfs_token_item_offset(leaf, item, &token);
4702 btrfs_set_token_item_offset(leaf, item,
4703 ioff - data_size, &token);
Chris Mason6567e832007-04-16 09:22:45 -04004704 }
Chris Mason5f39d392007-10-15 16:14:19 -04004705
Chris Mason6567e832007-04-16 09:22:45 -04004706 /* shift the data */
Chris Mason5f39d392007-10-15 16:14:19 -04004707 memmove_extent_buffer(leaf, btrfs_leaf_data(leaf) +
Chris Mason6567e832007-04-16 09:22:45 -04004708 data_end - data_size, btrfs_leaf_data(leaf) +
4709 data_end, old_data - data_end);
Chris Mason5f39d392007-10-15 16:14:19 -04004710
Chris Mason6567e832007-04-16 09:22:45 -04004711 data_end = old_data;
Chris Mason5f39d392007-10-15 16:14:19 -04004712 old_size = btrfs_item_size_nr(leaf, slot);
Ross Kirkdd3cc162013-09-16 15:58:09 +01004713 item = btrfs_item_nr(slot);
Chris Mason5f39d392007-10-15 16:14:19 -04004714 btrfs_set_item_size(leaf, item, old_size + data_size);
4715 btrfs_mark_buffer_dirty(leaf);
Chris Mason6567e832007-04-16 09:22:45 -04004716
Chris Mason5f39d392007-10-15 16:14:19 -04004717 if (btrfs_leaf_free_space(root, leaf) < 0) {
4718 btrfs_print_leaf(root, leaf);
Chris Mason6567e832007-04-16 09:22:45 -04004719 BUG();
Chris Mason5f39d392007-10-15 16:14:19 -04004720 }
Chris Mason6567e832007-04-16 09:22:45 -04004721}
4722
Chris Mason74123bd2007-02-02 11:05:29 -05004723/*
Chris Mason44871b12009-03-13 10:04:31 -04004724 * this is a helper for btrfs_insert_empty_items, the main goal here is
4725 * to save stack depth by doing the bulk of the work in a function
4726 * that doesn't call btrfs_search_slot
Chris Mason74123bd2007-02-02 11:05:29 -05004727 */
Tsutomu Itohafe5fea2013-04-16 05:18:22 +00004728void setup_items_for_insert(struct btrfs_root *root, struct btrfs_path *path,
Jeff Mahoney143bede2012-03-01 14:56:26 +01004729 struct btrfs_key *cpu_key, u32 *data_size,
4730 u32 total_data, u32 total_size, int nr)
Chris Masonbe0e5c02007-01-26 15:51:26 -05004731{
Chris Mason5f39d392007-10-15 16:14:19 -04004732 struct btrfs_item *item;
Chris Mason9c583092008-01-29 15:15:18 -05004733 int i;
Chris Mason7518a232007-03-12 12:01:18 -04004734 u32 nritems;
Chris Masonbe0e5c02007-01-26 15:51:26 -05004735 unsigned int data_end;
Chris Masone2fa7222007-03-12 16:22:34 -04004736 struct btrfs_disk_key disk_key;
Chris Mason44871b12009-03-13 10:04:31 -04004737 struct extent_buffer *leaf;
4738 int slot;
Chris Masoncfed81a2012-03-03 07:40:03 -05004739 struct btrfs_map_token token;
4740
4741 btrfs_init_map_token(&token);
Chris Masone2fa7222007-03-12 16:22:34 -04004742
Chris Mason5f39d392007-10-15 16:14:19 -04004743 leaf = path->nodes[0];
Chris Mason44871b12009-03-13 10:04:31 -04004744 slot = path->slots[0];
Chris Mason74123bd2007-02-02 11:05:29 -05004745
Chris Mason5f39d392007-10-15 16:14:19 -04004746 nritems = btrfs_header_nritems(leaf);
Chris Mason123abc82007-03-14 14:14:43 -04004747 data_end = leaf_data_end(root, leaf);
Chris Masoneb60cea2007-02-02 09:18:22 -05004748
Chris Masonf25956c2008-09-12 15:32:53 -04004749 if (btrfs_leaf_free_space(root, leaf) < total_size) {
Chris Mason3326d1b2007-10-15 16:18:25 -04004750 btrfs_print_leaf(root, leaf);
Frank Holtonefe120a2013-12-20 11:37:06 -05004751 btrfs_crit(root->fs_info, "not enough freespace need %u have %d",
Chris Mason9c583092008-01-29 15:15:18 -05004752 total_size, btrfs_leaf_free_space(root, leaf));
Chris Masonbe0e5c02007-01-26 15:51:26 -05004753 BUG();
Chris Masond4dbff92007-04-04 14:08:15 -04004754 }
Chris Mason5f39d392007-10-15 16:14:19 -04004755
Chris Masonbe0e5c02007-01-26 15:51:26 -05004756 if (slot != nritems) {
Chris Mason5f39d392007-10-15 16:14:19 -04004757 unsigned int old_data = btrfs_item_end_nr(leaf, slot);
Chris Masonbe0e5c02007-01-26 15:51:26 -05004758
Chris Mason5f39d392007-10-15 16:14:19 -04004759 if (old_data < data_end) {
4760 btrfs_print_leaf(root, leaf);
Frank Holtonefe120a2013-12-20 11:37:06 -05004761 btrfs_crit(root->fs_info, "slot %d old_data %d data_end %d",
Chris Mason5f39d392007-10-15 16:14:19 -04004762 slot, old_data, data_end);
4763 BUG_ON(1);
4764 }
Chris Masonbe0e5c02007-01-26 15:51:26 -05004765 /*
4766 * item0..itemN ... dataN.offset..dataN.size .. data0.size
4767 */
4768 /* first correct the data pointers */
Chris Mason0783fcf2007-03-12 20:12:07 -04004769 for (i = slot; i < nritems; i++) {
Chris Mason5f39d392007-10-15 16:14:19 -04004770 u32 ioff;
Chris Masondb945352007-10-15 16:15:53 -04004771
Ross Kirkdd3cc162013-09-16 15:58:09 +01004772 item = btrfs_item_nr( i);
Chris Masoncfed81a2012-03-03 07:40:03 -05004773 ioff = btrfs_token_item_offset(leaf, item, &token);
4774 btrfs_set_token_item_offset(leaf, item,
4775 ioff - total_data, &token);
Chris Mason0783fcf2007-03-12 20:12:07 -04004776 }
Chris Masonbe0e5c02007-01-26 15:51:26 -05004777 /* shift the items */
Chris Mason9c583092008-01-29 15:15:18 -05004778 memmove_extent_buffer(leaf, btrfs_item_nr_offset(slot + nr),
Chris Mason5f39d392007-10-15 16:14:19 -04004779 btrfs_item_nr_offset(slot),
Chris Masond6025572007-03-30 14:27:56 -04004780 (nritems - slot) * sizeof(struct btrfs_item));
Chris Masonbe0e5c02007-01-26 15:51:26 -05004781
4782 /* shift the data */
Chris Mason5f39d392007-10-15 16:14:19 -04004783 memmove_extent_buffer(leaf, btrfs_leaf_data(leaf) +
Chris Mason9c583092008-01-29 15:15:18 -05004784 data_end - total_data, btrfs_leaf_data(leaf) +
Chris Masond6025572007-03-30 14:27:56 -04004785 data_end, old_data - data_end);
Chris Masonbe0e5c02007-01-26 15:51:26 -05004786 data_end = old_data;
4787 }
Chris Mason5f39d392007-10-15 16:14:19 -04004788
Chris Mason62e27492007-03-15 12:56:47 -04004789 /* setup the item for the new data */
Chris Mason9c583092008-01-29 15:15:18 -05004790 for (i = 0; i < nr; i++) {
4791 btrfs_cpu_key_to_disk(&disk_key, cpu_key + i);
4792 btrfs_set_item_key(leaf, &disk_key, slot + i);
Ross Kirkdd3cc162013-09-16 15:58:09 +01004793 item = btrfs_item_nr(slot + i);
Chris Masoncfed81a2012-03-03 07:40:03 -05004794 btrfs_set_token_item_offset(leaf, item,
4795 data_end - data_size[i], &token);
Chris Mason9c583092008-01-29 15:15:18 -05004796 data_end -= data_size[i];
Chris Masoncfed81a2012-03-03 07:40:03 -05004797 btrfs_set_token_item_size(leaf, item, data_size[i], &token);
Chris Mason9c583092008-01-29 15:15:18 -05004798 }
Chris Mason44871b12009-03-13 10:04:31 -04004799
Chris Mason9c583092008-01-29 15:15:18 -05004800 btrfs_set_header_nritems(leaf, nritems + nr);
Chris Masonaa5d6be2007-02-28 16:35:06 -05004801
Chris Mason5a01a2e2008-01-30 11:43:54 -05004802 if (slot == 0) {
4803 btrfs_cpu_key_to_disk(&disk_key, cpu_key);
Tsutomu Itohd6a0a122013-04-16 05:18:02 +00004804 fixup_low_keys(root, path, &disk_key, 1);
Chris Mason5a01a2e2008-01-30 11:43:54 -05004805 }
Chris Masonb9473432009-03-13 11:00:37 -04004806 btrfs_unlock_up_safe(path, 1);
4807 btrfs_mark_buffer_dirty(leaf);
Chris Masonaa5d6be2007-02-28 16:35:06 -05004808
Chris Mason5f39d392007-10-15 16:14:19 -04004809 if (btrfs_leaf_free_space(root, leaf) < 0) {
4810 btrfs_print_leaf(root, leaf);
Chris Masonbe0e5c02007-01-26 15:51:26 -05004811 BUG();
Chris Mason5f39d392007-10-15 16:14:19 -04004812 }
Chris Mason44871b12009-03-13 10:04:31 -04004813}
4814
4815/*
4816 * Given a key and some data, insert items into the tree.
4817 * This does all the path init required, making room in the tree if needed.
4818 */
4819int btrfs_insert_empty_items(struct btrfs_trans_handle *trans,
4820 struct btrfs_root *root,
4821 struct btrfs_path *path,
4822 struct btrfs_key *cpu_key, u32 *data_size,
4823 int nr)
4824{
Chris Mason44871b12009-03-13 10:04:31 -04004825 int ret = 0;
4826 int slot;
4827 int i;
4828 u32 total_size = 0;
4829 u32 total_data = 0;
4830
4831 for (i = 0; i < nr; i++)
4832 total_data += data_size[i];
4833
4834 total_size = total_data + (nr * sizeof(struct btrfs_item));
4835 ret = btrfs_search_slot(trans, root, cpu_key, path, total_size, 1);
4836 if (ret == 0)
4837 return -EEXIST;
4838 if (ret < 0)
Jeff Mahoney143bede2012-03-01 14:56:26 +01004839 return ret;
Chris Mason44871b12009-03-13 10:04:31 -04004840
Chris Mason44871b12009-03-13 10:04:31 -04004841 slot = path->slots[0];
4842 BUG_ON(slot < 0);
4843
Tsutomu Itohafe5fea2013-04-16 05:18:22 +00004844 setup_items_for_insert(root, path, cpu_key, data_size,
Chris Mason44871b12009-03-13 10:04:31 -04004845 total_data, total_size, nr);
Jeff Mahoney143bede2012-03-01 14:56:26 +01004846 return 0;
Chris Mason62e27492007-03-15 12:56:47 -04004847}
4848
4849/*
4850 * Given a key and some data, insert an item into the tree.
4851 * This does all the path init required, making room in the tree if needed.
4852 */
Chris Masone089f052007-03-16 16:20:31 -04004853int btrfs_insert_item(struct btrfs_trans_handle *trans, struct btrfs_root
4854 *root, struct btrfs_key *cpu_key, void *data, u32
4855 data_size)
Chris Mason62e27492007-03-15 12:56:47 -04004856{
4857 int ret = 0;
Chris Mason2c90e5d2007-04-02 10:50:19 -04004858 struct btrfs_path *path;
Chris Mason5f39d392007-10-15 16:14:19 -04004859 struct extent_buffer *leaf;
4860 unsigned long ptr;
Chris Mason62e27492007-03-15 12:56:47 -04004861
Chris Mason2c90e5d2007-04-02 10:50:19 -04004862 path = btrfs_alloc_path();
Tsutomu Itohdb5b4932011-03-23 08:14:16 +00004863 if (!path)
4864 return -ENOMEM;
Chris Mason2c90e5d2007-04-02 10:50:19 -04004865 ret = btrfs_insert_empty_item(trans, root, path, cpu_key, data_size);
Chris Mason62e27492007-03-15 12:56:47 -04004866 if (!ret) {
Chris Mason5f39d392007-10-15 16:14:19 -04004867 leaf = path->nodes[0];
4868 ptr = btrfs_item_ptr_offset(leaf, path->slots[0]);
4869 write_extent_buffer(leaf, data, ptr, data_size);
4870 btrfs_mark_buffer_dirty(leaf);
Chris Mason62e27492007-03-15 12:56:47 -04004871 }
Chris Mason2c90e5d2007-04-02 10:50:19 -04004872 btrfs_free_path(path);
Chris Masonaa5d6be2007-02-28 16:35:06 -05004873 return ret;
Chris Masonbe0e5c02007-01-26 15:51:26 -05004874}
4875
Chris Mason74123bd2007-02-02 11:05:29 -05004876/*
Chris Mason5de08d72007-02-24 06:24:44 -05004877 * delete the pointer from a given node.
Chris Mason74123bd2007-02-02 11:05:29 -05004878 *
Chris Masond352ac62008-09-29 15:18:18 -04004879 * the tree should have been previously balanced so the deletion does not
4880 * empty a node.
Chris Mason74123bd2007-02-02 11:05:29 -05004881 */
Tsutomu Itohafe5fea2013-04-16 05:18:22 +00004882static void del_ptr(struct btrfs_root *root, struct btrfs_path *path,
4883 int level, int slot)
Chris Masonbe0e5c02007-01-26 15:51:26 -05004884{
Chris Mason5f39d392007-10-15 16:14:19 -04004885 struct extent_buffer *parent = path->nodes[level];
Chris Mason7518a232007-03-12 12:01:18 -04004886 u32 nritems;
Jan Schmidtf3ea38d2012-05-26 11:45:21 +02004887 int ret;
Chris Masonbe0e5c02007-01-26 15:51:26 -05004888
Chris Mason5f39d392007-10-15 16:14:19 -04004889 nritems = btrfs_header_nritems(parent);
Chris Masond3977122009-01-05 21:25:51 -05004890 if (slot != nritems - 1) {
Liu Bo0e411ec2012-10-19 09:50:54 +00004891 if (level)
Jan Schmidtf3ea38d2012-05-26 11:45:21 +02004892 tree_mod_log_eb_move(root->fs_info, parent, slot,
4893 slot + 1, nritems - slot - 1);
Chris Mason5f39d392007-10-15 16:14:19 -04004894 memmove_extent_buffer(parent,
4895 btrfs_node_key_ptr_offset(slot),
4896 btrfs_node_key_ptr_offset(slot + 1),
Chris Masond6025572007-03-30 14:27:56 -04004897 sizeof(struct btrfs_key_ptr) *
4898 (nritems - slot - 1));
Chris Mason57ba86c2012-12-18 19:35:32 -05004899 } else if (level) {
4900 ret = tree_mod_log_insert_key(root->fs_info, parent, slot,
Josef Bacikc8cc6342013-07-01 16:18:19 -04004901 MOD_LOG_KEY_REMOVE, GFP_NOFS);
Chris Mason57ba86c2012-12-18 19:35:32 -05004902 BUG_ON(ret < 0);
Chris Masonbb803952007-03-01 12:04:21 -05004903 }
Jan Schmidtf3ea38d2012-05-26 11:45:21 +02004904
Chris Mason7518a232007-03-12 12:01:18 -04004905 nritems--;
Chris Mason5f39d392007-10-15 16:14:19 -04004906 btrfs_set_header_nritems(parent, nritems);
Chris Mason7518a232007-03-12 12:01:18 -04004907 if (nritems == 0 && parent == root->node) {
Chris Mason5f39d392007-10-15 16:14:19 -04004908 BUG_ON(btrfs_header_level(root->node) != 1);
Chris Masonbb803952007-03-01 12:04:21 -05004909 /* just turn the root into a leaf and break */
Chris Mason5f39d392007-10-15 16:14:19 -04004910 btrfs_set_header_level(root->node, 0);
Chris Masonbb803952007-03-01 12:04:21 -05004911 } else if (slot == 0) {
Chris Mason5f39d392007-10-15 16:14:19 -04004912 struct btrfs_disk_key disk_key;
4913
4914 btrfs_node_key(parent, &disk_key, 0);
Tsutomu Itohd6a0a122013-04-16 05:18:02 +00004915 fixup_low_keys(root, path, &disk_key, level + 1);
Chris Masonbe0e5c02007-01-26 15:51:26 -05004916 }
Chris Masond6025572007-03-30 14:27:56 -04004917 btrfs_mark_buffer_dirty(parent);
Chris Masonbe0e5c02007-01-26 15:51:26 -05004918}
4919
Chris Mason74123bd2007-02-02 11:05:29 -05004920/*
Chris Mason323ac952008-10-01 19:05:46 -04004921 * a helper function to delete the leaf pointed to by path->slots[1] and
Yan Zheng5d4f98a2009-06-10 10:45:14 -04004922 * path->nodes[1].
Chris Mason323ac952008-10-01 19:05:46 -04004923 *
4924 * This deletes the pointer in path->nodes[1] and frees the leaf
4925 * block extent. zero is returned if it all worked out, < 0 otherwise.
4926 *
4927 * The path must have already been setup for deleting the leaf, including
4928 * all the proper balancing. path->nodes[1] must be locked.
4929 */
Jeff Mahoney143bede2012-03-01 14:56:26 +01004930static noinline void btrfs_del_leaf(struct btrfs_trans_handle *trans,
4931 struct btrfs_root *root,
4932 struct btrfs_path *path,
4933 struct extent_buffer *leaf)
Chris Mason323ac952008-10-01 19:05:46 -04004934{
Yan Zheng5d4f98a2009-06-10 10:45:14 -04004935 WARN_ON(btrfs_header_generation(leaf) != trans->transid);
Tsutomu Itohafe5fea2013-04-16 05:18:22 +00004936 del_ptr(root, path, 1, path->slots[1]);
Chris Mason323ac952008-10-01 19:05:46 -04004937
Chris Mason4d081c42009-02-04 09:31:28 -05004938 /*
4939 * btrfs_free_extent is expensive, we want to make sure we
4940 * aren't holding any locks when we call it
4941 */
4942 btrfs_unlock_up_safe(path, 0);
4943
Yan, Zhengf0486c62010-05-16 10:46:25 -04004944 root_sub_used(root, leaf->len);
4945
Josef Bacik3083ee22012-03-09 16:01:49 -05004946 extent_buffer_get(leaf);
Jan Schmidt5581a512012-05-16 17:04:52 +02004947 btrfs_free_tree_block(trans, root, leaf, 0, 1);
Josef Bacik3083ee22012-03-09 16:01:49 -05004948 free_extent_buffer_stale(leaf);
Chris Mason323ac952008-10-01 19:05:46 -04004949}
4950/*
Chris Mason74123bd2007-02-02 11:05:29 -05004951 * delete the item at the leaf level in path. If that empties
4952 * the leaf, remove it from the tree
4953 */
Chris Mason85e21ba2008-01-29 15:11:36 -05004954int btrfs_del_items(struct btrfs_trans_handle *trans, struct btrfs_root *root,
4955 struct btrfs_path *path, int slot, int nr)
Chris Masonbe0e5c02007-01-26 15:51:26 -05004956{
Chris Mason5f39d392007-10-15 16:14:19 -04004957 struct extent_buffer *leaf;
4958 struct btrfs_item *item;
Chris Mason85e21ba2008-01-29 15:11:36 -05004959 int last_off;
4960 int dsize = 0;
Chris Masonaa5d6be2007-02-28 16:35:06 -05004961 int ret = 0;
4962 int wret;
Chris Mason85e21ba2008-01-29 15:11:36 -05004963 int i;
Chris Mason7518a232007-03-12 12:01:18 -04004964 u32 nritems;
Chris Masoncfed81a2012-03-03 07:40:03 -05004965 struct btrfs_map_token token;
4966
4967 btrfs_init_map_token(&token);
Chris Masonbe0e5c02007-01-26 15:51:26 -05004968
Chris Mason5f39d392007-10-15 16:14:19 -04004969 leaf = path->nodes[0];
Chris Mason85e21ba2008-01-29 15:11:36 -05004970 last_off = btrfs_item_offset_nr(leaf, slot + nr - 1);
4971
4972 for (i = 0; i < nr; i++)
4973 dsize += btrfs_item_size_nr(leaf, slot + i);
4974
Chris Mason5f39d392007-10-15 16:14:19 -04004975 nritems = btrfs_header_nritems(leaf);
Chris Masonbe0e5c02007-01-26 15:51:26 -05004976
Chris Mason85e21ba2008-01-29 15:11:36 -05004977 if (slot + nr != nritems) {
Chris Mason123abc82007-03-14 14:14:43 -04004978 int data_end = leaf_data_end(root, leaf);
Chris Mason5f39d392007-10-15 16:14:19 -04004979
4980 memmove_extent_buffer(leaf, btrfs_leaf_data(leaf) +
Chris Masond6025572007-03-30 14:27:56 -04004981 data_end + dsize,
4982 btrfs_leaf_data(leaf) + data_end,
Chris Mason85e21ba2008-01-29 15:11:36 -05004983 last_off - data_end);
Chris Mason5f39d392007-10-15 16:14:19 -04004984
Chris Mason85e21ba2008-01-29 15:11:36 -05004985 for (i = slot + nr; i < nritems; i++) {
Chris Mason5f39d392007-10-15 16:14:19 -04004986 u32 ioff;
Chris Masondb945352007-10-15 16:15:53 -04004987
Ross Kirkdd3cc162013-09-16 15:58:09 +01004988 item = btrfs_item_nr(i);
Chris Masoncfed81a2012-03-03 07:40:03 -05004989 ioff = btrfs_token_item_offset(leaf, item, &token);
4990 btrfs_set_token_item_offset(leaf, item,
4991 ioff + dsize, &token);
Chris Mason0783fcf2007-03-12 20:12:07 -04004992 }
Chris Masondb945352007-10-15 16:15:53 -04004993
Chris Mason5f39d392007-10-15 16:14:19 -04004994 memmove_extent_buffer(leaf, btrfs_item_nr_offset(slot),
Chris Mason85e21ba2008-01-29 15:11:36 -05004995 btrfs_item_nr_offset(slot + nr),
Chris Masond6025572007-03-30 14:27:56 -04004996 sizeof(struct btrfs_item) *
Chris Mason85e21ba2008-01-29 15:11:36 -05004997 (nritems - slot - nr));
Chris Masonbe0e5c02007-01-26 15:51:26 -05004998 }
Chris Mason85e21ba2008-01-29 15:11:36 -05004999 btrfs_set_header_nritems(leaf, nritems - nr);
5000 nritems -= nr;
Chris Mason5f39d392007-10-15 16:14:19 -04005001
Chris Mason74123bd2007-02-02 11:05:29 -05005002 /* delete the leaf if we've emptied it */
Chris Mason7518a232007-03-12 12:01:18 -04005003 if (nritems == 0) {
Chris Mason5f39d392007-10-15 16:14:19 -04005004 if (leaf == root->node) {
5005 btrfs_set_header_level(leaf, 0);
Chris Mason9a8dd152007-02-23 08:38:36 -05005006 } else {
Yan, Zhengf0486c62010-05-16 10:46:25 -04005007 btrfs_set_path_blocking(path);
5008 clean_tree_block(trans, root, leaf);
Jeff Mahoney143bede2012-03-01 14:56:26 +01005009 btrfs_del_leaf(trans, root, path, leaf);
Chris Mason9a8dd152007-02-23 08:38:36 -05005010 }
Chris Masonbe0e5c02007-01-26 15:51:26 -05005011 } else {
Chris Mason7518a232007-03-12 12:01:18 -04005012 int used = leaf_space_used(leaf, 0, nritems);
Chris Masonaa5d6be2007-02-28 16:35:06 -05005013 if (slot == 0) {
Chris Mason5f39d392007-10-15 16:14:19 -04005014 struct btrfs_disk_key disk_key;
5015
5016 btrfs_item_key(leaf, &disk_key, 0);
Tsutomu Itohd6a0a122013-04-16 05:18:02 +00005017 fixup_low_keys(root, path, &disk_key, 1);
Chris Masonaa5d6be2007-02-28 16:35:06 -05005018 }
Chris Masonaa5d6be2007-02-28 16:35:06 -05005019
Chris Mason74123bd2007-02-02 11:05:29 -05005020 /* delete the leaf if it is mostly empty */
Yan Zhengd717aa12009-07-24 12:42:46 -04005021 if (used < BTRFS_LEAF_DATA_SIZE(root) / 3) {
Chris Masonbe0e5c02007-01-26 15:51:26 -05005022 /* push_leaf_left fixes the path.
5023 * make sure the path still points to our leaf
5024 * for possible call to del_ptr below
5025 */
Chris Mason4920c9a2007-01-26 16:38:42 -05005026 slot = path->slots[1];
Chris Mason5f39d392007-10-15 16:14:19 -04005027 extent_buffer_get(leaf);
5028
Chris Masonb9473432009-03-13 11:00:37 -04005029 btrfs_set_path_blocking(path);
Chris Mason99d8f832010-07-07 10:51:48 -04005030 wret = push_leaf_left(trans, root, path, 1, 1,
5031 1, (u32)-1);
Chris Mason54aa1f42007-06-22 14:16:25 -04005032 if (wret < 0 && wret != -ENOSPC)
Chris Masonaa5d6be2007-02-28 16:35:06 -05005033 ret = wret;
Chris Mason5f39d392007-10-15 16:14:19 -04005034
5035 if (path->nodes[0] == leaf &&
5036 btrfs_header_nritems(leaf)) {
Chris Mason99d8f832010-07-07 10:51:48 -04005037 wret = push_leaf_right(trans, root, path, 1,
5038 1, 1, 0);
Chris Mason54aa1f42007-06-22 14:16:25 -04005039 if (wret < 0 && wret != -ENOSPC)
Chris Masonaa5d6be2007-02-28 16:35:06 -05005040 ret = wret;
5041 }
Chris Mason5f39d392007-10-15 16:14:19 -04005042
5043 if (btrfs_header_nritems(leaf) == 0) {
Chris Mason323ac952008-10-01 19:05:46 -04005044 path->slots[1] = slot;
Jeff Mahoney143bede2012-03-01 14:56:26 +01005045 btrfs_del_leaf(trans, root, path, leaf);
Chris Mason5f39d392007-10-15 16:14:19 -04005046 free_extent_buffer(leaf);
Jeff Mahoney143bede2012-03-01 14:56:26 +01005047 ret = 0;
Chris Mason5de08d72007-02-24 06:24:44 -05005048 } else {
Chris Mason925baed2008-06-25 16:01:30 -04005049 /* if we're still in the path, make sure
5050 * we're dirty. Otherwise, one of the
5051 * push_leaf functions must have already
5052 * dirtied this buffer
5053 */
5054 if (path->nodes[0] == leaf)
5055 btrfs_mark_buffer_dirty(leaf);
Chris Mason5f39d392007-10-15 16:14:19 -04005056 free_extent_buffer(leaf);
Chris Masonbe0e5c02007-01-26 15:51:26 -05005057 }
Chris Masond5719762007-03-23 10:01:08 -04005058 } else {
Chris Mason5f39d392007-10-15 16:14:19 -04005059 btrfs_mark_buffer_dirty(leaf);
Chris Masonbe0e5c02007-01-26 15:51:26 -05005060 }
5061 }
Chris Masonaa5d6be2007-02-28 16:35:06 -05005062 return ret;
Chris Masonbe0e5c02007-01-26 15:51:26 -05005063}
5064
Chris Mason97571fd2007-02-24 13:39:08 -05005065/*
Chris Mason925baed2008-06-25 16:01:30 -04005066 * search the tree again to find a leaf with lesser keys
Chris Mason7bb86312007-12-11 09:25:06 -05005067 * returns 0 if it found something or 1 if there are no lesser leaves.
5068 * returns < 0 on io errors.
Chris Masond352ac62008-09-29 15:18:18 -04005069 *
5070 * This may release the path, and so you may lose any locks held at the
5071 * time you call it.
Chris Mason7bb86312007-12-11 09:25:06 -05005072 */
Josef Bacik16e75492013-10-22 12:18:51 -04005073int btrfs_prev_leaf(struct btrfs_root *root, struct btrfs_path *path)
Chris Mason7bb86312007-12-11 09:25:06 -05005074{
Chris Mason925baed2008-06-25 16:01:30 -04005075 struct btrfs_key key;
5076 struct btrfs_disk_key found_key;
5077 int ret;
Chris Mason7bb86312007-12-11 09:25:06 -05005078
Chris Mason925baed2008-06-25 16:01:30 -04005079 btrfs_item_key_to_cpu(path->nodes[0], &key, 0);
Chris Mason7bb86312007-12-11 09:25:06 -05005080
Filipe David Borba Mananae8b0d722013-10-15 00:12:27 +01005081 if (key.offset > 0) {
Chris Mason925baed2008-06-25 16:01:30 -04005082 key.offset--;
Filipe David Borba Mananae8b0d722013-10-15 00:12:27 +01005083 } else if (key.type > 0) {
Chris Mason925baed2008-06-25 16:01:30 -04005084 key.type--;
Filipe David Borba Mananae8b0d722013-10-15 00:12:27 +01005085 key.offset = (u64)-1;
5086 } else if (key.objectid > 0) {
Chris Mason925baed2008-06-25 16:01:30 -04005087 key.objectid--;
Filipe David Borba Mananae8b0d722013-10-15 00:12:27 +01005088 key.type = (u8)-1;
5089 key.offset = (u64)-1;
5090 } else {
Chris Mason925baed2008-06-25 16:01:30 -04005091 return 1;
Filipe David Borba Mananae8b0d722013-10-15 00:12:27 +01005092 }
Chris Mason7bb86312007-12-11 09:25:06 -05005093
David Sterbab3b4aa72011-04-21 01:20:15 +02005094 btrfs_release_path(path);
Chris Mason925baed2008-06-25 16:01:30 -04005095 ret = btrfs_search_slot(NULL, root, &key, path, 0, 0);
5096 if (ret < 0)
5097 return ret;
5098 btrfs_item_key(path->nodes[0], &found_key, 0);
5099 ret = comp_keys(&found_key, &key);
Filipe Manana337c6f62014-06-09 13:22:13 +01005100 /*
5101 * We might have had an item with the previous key in the tree right
5102 * before we released our path. And after we released our path, that
5103 * item might have been pushed to the first slot (0) of the leaf we
5104 * were holding due to a tree balance. Alternatively, an item with the
5105 * previous key can exist as the only element of a leaf (big fat item).
5106 * Therefore account for these 2 cases, so that our callers (like
5107 * btrfs_previous_item) don't miss an existing item with a key matching
5108 * the previous key we computed above.
5109 */
5110 if (ret <= 0)
Chris Mason925baed2008-06-25 16:01:30 -04005111 return 0;
5112 return 1;
Chris Mason7bb86312007-12-11 09:25:06 -05005113}
5114
Chris Mason3f157a22008-06-25 16:01:31 -04005115/*
5116 * A helper function to walk down the tree starting at min_key, and looking
Eric Sandeende78b512013-01-31 18:21:12 +00005117 * for nodes or leaves that are have a minimum transaction id.
5118 * This is used by the btree defrag code, and tree logging
Chris Mason3f157a22008-06-25 16:01:31 -04005119 *
5120 * This does not cow, but it does stuff the starting key it finds back
5121 * into min_key, so you can call btrfs_search_slot with cow=1 on the
5122 * key and get a writable path.
5123 *
5124 * This does lock as it descends, and path->keep_locks should be set
5125 * to 1 by the caller.
5126 *
5127 * This honors path->lowest_level to prevent descent past a given level
5128 * of the tree.
5129 *
Chris Masond352ac62008-09-29 15:18:18 -04005130 * min_trans indicates the oldest transaction that you are interested
5131 * in walking through. Any nodes or leaves older than min_trans are
5132 * skipped over (without reading them).
5133 *
Chris Mason3f157a22008-06-25 16:01:31 -04005134 * returns zero if something useful was found, < 0 on error and 1 if there
5135 * was nothing in the tree that matched the search criteria.
5136 */
5137int btrfs_search_forward(struct btrfs_root *root, struct btrfs_key *min_key,
Eric Sandeende78b512013-01-31 18:21:12 +00005138 struct btrfs_path *path,
Chris Mason3f157a22008-06-25 16:01:31 -04005139 u64 min_trans)
5140{
5141 struct extent_buffer *cur;
5142 struct btrfs_key found_key;
5143 int slot;
Yan96524802008-07-24 12:19:49 -04005144 int sret;
Chris Mason3f157a22008-06-25 16:01:31 -04005145 u32 nritems;
5146 int level;
5147 int ret = 1;
5148
Chris Mason934d3752008-12-08 16:43:10 -05005149 WARN_ON(!path->keep_locks);
Chris Mason3f157a22008-06-25 16:01:31 -04005150again:
Chris Masonbd681512011-07-16 15:23:14 -04005151 cur = btrfs_read_lock_root_node(root);
Chris Mason3f157a22008-06-25 16:01:31 -04005152 level = btrfs_header_level(cur);
Chris Masone02119d2008-09-05 16:13:11 -04005153 WARN_ON(path->nodes[level]);
Chris Mason3f157a22008-06-25 16:01:31 -04005154 path->nodes[level] = cur;
Chris Masonbd681512011-07-16 15:23:14 -04005155 path->locks[level] = BTRFS_READ_LOCK;
Chris Mason3f157a22008-06-25 16:01:31 -04005156
5157 if (btrfs_header_generation(cur) < min_trans) {
5158 ret = 1;
5159 goto out;
5160 }
Chris Masond3977122009-01-05 21:25:51 -05005161 while (1) {
Chris Mason3f157a22008-06-25 16:01:31 -04005162 nritems = btrfs_header_nritems(cur);
5163 level = btrfs_header_level(cur);
Yan96524802008-07-24 12:19:49 -04005164 sret = bin_search(cur, min_key, level, &slot);
Chris Mason3f157a22008-06-25 16:01:31 -04005165
Chris Mason323ac952008-10-01 19:05:46 -04005166 /* at the lowest level, we're done, setup the path and exit */
5167 if (level == path->lowest_level) {
Chris Masone02119d2008-09-05 16:13:11 -04005168 if (slot >= nritems)
5169 goto find_next_key;
Chris Mason3f157a22008-06-25 16:01:31 -04005170 ret = 0;
5171 path->slots[level] = slot;
5172 btrfs_item_key_to_cpu(cur, &found_key, slot);
5173 goto out;
5174 }
Yan96524802008-07-24 12:19:49 -04005175 if (sret && slot > 0)
5176 slot--;
Chris Mason3f157a22008-06-25 16:01:31 -04005177 /*
Eric Sandeende78b512013-01-31 18:21:12 +00005178 * check this node pointer against the min_trans parameters.
5179 * If it is too old, old, skip to the next one.
Chris Mason3f157a22008-06-25 16:01:31 -04005180 */
Chris Masond3977122009-01-05 21:25:51 -05005181 while (slot < nritems) {
Chris Mason3f157a22008-06-25 16:01:31 -04005182 u64 gen;
Chris Masone02119d2008-09-05 16:13:11 -04005183
Chris Mason3f157a22008-06-25 16:01:31 -04005184 gen = btrfs_node_ptr_generation(cur, slot);
5185 if (gen < min_trans) {
5186 slot++;
5187 continue;
5188 }
Eric Sandeende78b512013-01-31 18:21:12 +00005189 break;
Chris Mason3f157a22008-06-25 16:01:31 -04005190 }
Chris Masone02119d2008-09-05 16:13:11 -04005191find_next_key:
Chris Mason3f157a22008-06-25 16:01:31 -04005192 /*
5193 * we didn't find a candidate key in this node, walk forward
5194 * and find another one
5195 */
5196 if (slot >= nritems) {
Chris Masone02119d2008-09-05 16:13:11 -04005197 path->slots[level] = slot;
Chris Masonb4ce94d2009-02-04 09:25:08 -05005198 btrfs_set_path_blocking(path);
Chris Masone02119d2008-09-05 16:13:11 -04005199 sret = btrfs_find_next_key(root, path, min_key, level,
Eric Sandeende78b512013-01-31 18:21:12 +00005200 min_trans);
Chris Masone02119d2008-09-05 16:13:11 -04005201 if (sret == 0) {
David Sterbab3b4aa72011-04-21 01:20:15 +02005202 btrfs_release_path(path);
Chris Mason3f157a22008-06-25 16:01:31 -04005203 goto again;
5204 } else {
5205 goto out;
5206 }
5207 }
5208 /* save our key for returning back */
5209 btrfs_node_key_to_cpu(cur, &found_key, slot);
5210 path->slots[level] = slot;
5211 if (level == path->lowest_level) {
5212 ret = 0;
Chris Masonf7c79f32012-03-19 15:54:38 -04005213 unlock_up(path, level, 1, 0, NULL);
Chris Mason3f157a22008-06-25 16:01:31 -04005214 goto out;
5215 }
Chris Masonb4ce94d2009-02-04 09:25:08 -05005216 btrfs_set_path_blocking(path);
Chris Mason3f157a22008-06-25 16:01:31 -04005217 cur = read_node_slot(root, cur, slot);
Jeff Mahoney79787ea2012-03-12 16:03:00 +01005218 BUG_ON(!cur); /* -ENOMEM */
Chris Mason3f157a22008-06-25 16:01:31 -04005219
Chris Masonbd681512011-07-16 15:23:14 -04005220 btrfs_tree_read_lock(cur);
Chris Masonb4ce94d2009-02-04 09:25:08 -05005221
Chris Masonbd681512011-07-16 15:23:14 -04005222 path->locks[level - 1] = BTRFS_READ_LOCK;
Chris Mason3f157a22008-06-25 16:01:31 -04005223 path->nodes[level - 1] = cur;
Chris Masonf7c79f32012-03-19 15:54:38 -04005224 unlock_up(path, level, 1, 0, NULL);
Chris Masonbd681512011-07-16 15:23:14 -04005225 btrfs_clear_path_blocking(path, NULL, 0);
Chris Mason3f157a22008-06-25 16:01:31 -04005226 }
5227out:
5228 if (ret == 0)
5229 memcpy(min_key, &found_key, sizeof(found_key));
Chris Masonb4ce94d2009-02-04 09:25:08 -05005230 btrfs_set_path_blocking(path);
Chris Mason3f157a22008-06-25 16:01:31 -04005231 return ret;
5232}
5233
Alexander Block70698302012-06-05 21:07:48 +02005234static void tree_move_down(struct btrfs_root *root,
5235 struct btrfs_path *path,
5236 int *level, int root_level)
5237{
Chris Mason74dd17f2012-08-07 16:25:13 -04005238 BUG_ON(*level == 0);
Alexander Block70698302012-06-05 21:07:48 +02005239 path->nodes[*level - 1] = read_node_slot(root, path->nodes[*level],
5240 path->slots[*level]);
5241 path->slots[*level - 1] = 0;
5242 (*level)--;
5243}
5244
5245static int tree_move_next_or_upnext(struct btrfs_root *root,
5246 struct btrfs_path *path,
5247 int *level, int root_level)
5248{
5249 int ret = 0;
5250 int nritems;
5251 nritems = btrfs_header_nritems(path->nodes[*level]);
5252
5253 path->slots[*level]++;
5254
Chris Mason74dd17f2012-08-07 16:25:13 -04005255 while (path->slots[*level] >= nritems) {
Alexander Block70698302012-06-05 21:07:48 +02005256 if (*level == root_level)
5257 return -1;
5258
5259 /* move upnext */
5260 path->slots[*level] = 0;
5261 free_extent_buffer(path->nodes[*level]);
5262 path->nodes[*level] = NULL;
5263 (*level)++;
5264 path->slots[*level]++;
5265
5266 nritems = btrfs_header_nritems(path->nodes[*level]);
5267 ret = 1;
5268 }
5269 return ret;
5270}
5271
5272/*
5273 * Returns 1 if it had to move up and next. 0 is returned if it moved only next
5274 * or down.
5275 */
5276static int tree_advance(struct btrfs_root *root,
5277 struct btrfs_path *path,
5278 int *level, int root_level,
5279 int allow_down,
5280 struct btrfs_key *key)
5281{
5282 int ret;
5283
5284 if (*level == 0 || !allow_down) {
5285 ret = tree_move_next_or_upnext(root, path, level, root_level);
5286 } else {
5287 tree_move_down(root, path, level, root_level);
5288 ret = 0;
5289 }
5290 if (ret >= 0) {
5291 if (*level == 0)
5292 btrfs_item_key_to_cpu(path->nodes[*level], key,
5293 path->slots[*level]);
5294 else
5295 btrfs_node_key_to_cpu(path->nodes[*level], key,
5296 path->slots[*level]);
5297 }
5298 return ret;
5299}
5300
5301static int tree_compare_item(struct btrfs_root *left_root,
5302 struct btrfs_path *left_path,
5303 struct btrfs_path *right_path,
5304 char *tmp_buf)
5305{
5306 int cmp;
5307 int len1, len2;
5308 unsigned long off1, off2;
5309
5310 len1 = btrfs_item_size_nr(left_path->nodes[0], left_path->slots[0]);
5311 len2 = btrfs_item_size_nr(right_path->nodes[0], right_path->slots[0]);
5312 if (len1 != len2)
5313 return 1;
5314
5315 off1 = btrfs_item_ptr_offset(left_path->nodes[0], left_path->slots[0]);
5316 off2 = btrfs_item_ptr_offset(right_path->nodes[0],
5317 right_path->slots[0]);
5318
5319 read_extent_buffer(left_path->nodes[0], tmp_buf, off1, len1);
5320
5321 cmp = memcmp_extent_buffer(right_path->nodes[0], tmp_buf, off2, len1);
5322 if (cmp)
5323 return 1;
5324 return 0;
5325}
5326
5327#define ADVANCE 1
5328#define ADVANCE_ONLY_NEXT -1
5329
5330/*
5331 * This function compares two trees and calls the provided callback for
5332 * every changed/new/deleted item it finds.
5333 * If shared tree blocks are encountered, whole subtrees are skipped, making
5334 * the compare pretty fast on snapshotted subvolumes.
5335 *
5336 * This currently works on commit roots only. As commit roots are read only,
5337 * we don't do any locking. The commit roots are protected with transactions.
5338 * Transactions are ended and rejoined when a commit is tried in between.
5339 *
5340 * This function checks for modifications done to the trees while comparing.
5341 * If it detects a change, it aborts immediately.
5342 */
5343int btrfs_compare_trees(struct btrfs_root *left_root,
5344 struct btrfs_root *right_root,
5345 btrfs_changed_cb_t changed_cb, void *ctx)
5346{
5347 int ret;
5348 int cmp;
Alexander Block70698302012-06-05 21:07:48 +02005349 struct btrfs_path *left_path = NULL;
5350 struct btrfs_path *right_path = NULL;
5351 struct btrfs_key left_key;
5352 struct btrfs_key right_key;
5353 char *tmp_buf = NULL;
5354 int left_root_level;
5355 int right_root_level;
5356 int left_level;
5357 int right_level;
5358 int left_end_reached;
5359 int right_end_reached;
5360 int advance_left;
5361 int advance_right;
5362 u64 left_blockptr;
5363 u64 right_blockptr;
Filipe Manana6baa4292014-02-20 21:15:25 +00005364 u64 left_gen;
5365 u64 right_gen;
Alexander Block70698302012-06-05 21:07:48 +02005366
5367 left_path = btrfs_alloc_path();
5368 if (!left_path) {
5369 ret = -ENOMEM;
5370 goto out;
5371 }
5372 right_path = btrfs_alloc_path();
5373 if (!right_path) {
5374 ret = -ENOMEM;
5375 goto out;
5376 }
5377
5378 tmp_buf = kmalloc(left_root->leafsize, GFP_NOFS);
5379 if (!tmp_buf) {
5380 ret = -ENOMEM;
5381 goto out;
5382 }
5383
5384 left_path->search_commit_root = 1;
5385 left_path->skip_locking = 1;
5386 right_path->search_commit_root = 1;
5387 right_path->skip_locking = 1;
5388
Alexander Block70698302012-06-05 21:07:48 +02005389 /*
5390 * Strategy: Go to the first items of both trees. Then do
5391 *
5392 * If both trees are at level 0
5393 * Compare keys of current items
5394 * If left < right treat left item as new, advance left tree
5395 * and repeat
5396 * If left > right treat right item as deleted, advance right tree
5397 * and repeat
5398 * If left == right do deep compare of items, treat as changed if
5399 * needed, advance both trees and repeat
5400 * If both trees are at the same level but not at level 0
5401 * Compare keys of current nodes/leafs
5402 * If left < right advance left tree and repeat
5403 * If left > right advance right tree and repeat
5404 * If left == right compare blockptrs of the next nodes/leafs
5405 * If they match advance both trees but stay at the same level
5406 * and repeat
5407 * If they don't match advance both trees while allowing to go
5408 * deeper and repeat
5409 * If tree levels are different
5410 * Advance the tree that needs it and repeat
5411 *
5412 * Advancing a tree means:
5413 * If we are at level 0, try to go to the next slot. If that's not
5414 * possible, go one level up and repeat. Stop when we found a level
5415 * where we could go to the next slot. We may at this point be on a
5416 * node or a leaf.
5417 *
5418 * If we are not at level 0 and not on shared tree blocks, go one
5419 * level deeper.
5420 *
5421 * If we are not at level 0 and on shared tree blocks, go one slot to
5422 * the right if possible or go up and right.
5423 */
5424
Josef Bacik3f8a18c2014-03-28 17:16:01 -04005425 down_read(&left_root->fs_info->commit_root_sem);
Alexander Block70698302012-06-05 21:07:48 +02005426 left_level = btrfs_header_level(left_root->commit_root);
5427 left_root_level = left_level;
5428 left_path->nodes[left_level] = left_root->commit_root;
5429 extent_buffer_get(left_path->nodes[left_level]);
5430
5431 right_level = btrfs_header_level(right_root->commit_root);
5432 right_root_level = right_level;
5433 right_path->nodes[right_level] = right_root->commit_root;
5434 extent_buffer_get(right_path->nodes[right_level]);
Josef Bacik3f8a18c2014-03-28 17:16:01 -04005435 up_read(&left_root->fs_info->commit_root_sem);
Alexander Block70698302012-06-05 21:07:48 +02005436
5437 if (left_level == 0)
5438 btrfs_item_key_to_cpu(left_path->nodes[left_level],
5439 &left_key, left_path->slots[left_level]);
5440 else
5441 btrfs_node_key_to_cpu(left_path->nodes[left_level],
5442 &left_key, left_path->slots[left_level]);
5443 if (right_level == 0)
5444 btrfs_item_key_to_cpu(right_path->nodes[right_level],
5445 &right_key, right_path->slots[right_level]);
5446 else
5447 btrfs_node_key_to_cpu(right_path->nodes[right_level],
5448 &right_key, right_path->slots[right_level]);
5449
5450 left_end_reached = right_end_reached = 0;
5451 advance_left = advance_right = 0;
5452
5453 while (1) {
Alexander Block70698302012-06-05 21:07:48 +02005454 if (advance_left && !left_end_reached) {
5455 ret = tree_advance(left_root, left_path, &left_level,
5456 left_root_level,
5457 advance_left != ADVANCE_ONLY_NEXT,
5458 &left_key);
5459 if (ret < 0)
5460 left_end_reached = ADVANCE;
5461 advance_left = 0;
5462 }
5463 if (advance_right && !right_end_reached) {
5464 ret = tree_advance(right_root, right_path, &right_level,
5465 right_root_level,
5466 advance_right != ADVANCE_ONLY_NEXT,
5467 &right_key);
5468 if (ret < 0)
5469 right_end_reached = ADVANCE;
5470 advance_right = 0;
5471 }
5472
5473 if (left_end_reached && right_end_reached) {
5474 ret = 0;
5475 goto out;
5476 } else if (left_end_reached) {
5477 if (right_level == 0) {
5478 ret = changed_cb(left_root, right_root,
5479 left_path, right_path,
5480 &right_key,
5481 BTRFS_COMPARE_TREE_DELETED,
5482 ctx);
5483 if (ret < 0)
5484 goto out;
5485 }
5486 advance_right = ADVANCE;
5487 continue;
5488 } else if (right_end_reached) {
5489 if (left_level == 0) {
5490 ret = changed_cb(left_root, right_root,
5491 left_path, right_path,
5492 &left_key,
5493 BTRFS_COMPARE_TREE_NEW,
5494 ctx);
5495 if (ret < 0)
5496 goto out;
5497 }
5498 advance_left = ADVANCE;
5499 continue;
5500 }
5501
5502 if (left_level == 0 && right_level == 0) {
5503 cmp = btrfs_comp_cpu_keys(&left_key, &right_key);
5504 if (cmp < 0) {
5505 ret = changed_cb(left_root, right_root,
5506 left_path, right_path,
5507 &left_key,
5508 BTRFS_COMPARE_TREE_NEW,
5509 ctx);
5510 if (ret < 0)
5511 goto out;
5512 advance_left = ADVANCE;
5513 } else if (cmp > 0) {
5514 ret = changed_cb(left_root, right_root,
5515 left_path, right_path,
5516 &right_key,
5517 BTRFS_COMPARE_TREE_DELETED,
5518 ctx);
5519 if (ret < 0)
5520 goto out;
5521 advance_right = ADVANCE;
5522 } else {
Josef Bacikba5e8f22013-08-16 16:52:55 -04005523 enum btrfs_compare_tree_result cmp;
5524
Chris Mason74dd17f2012-08-07 16:25:13 -04005525 WARN_ON(!extent_buffer_uptodate(left_path->nodes[0]));
Alexander Block70698302012-06-05 21:07:48 +02005526 ret = tree_compare_item(left_root, left_path,
5527 right_path, tmp_buf);
Josef Bacikba5e8f22013-08-16 16:52:55 -04005528 if (ret)
5529 cmp = BTRFS_COMPARE_TREE_CHANGED;
5530 else
5531 cmp = BTRFS_COMPARE_TREE_SAME;
5532 ret = changed_cb(left_root, right_root,
5533 left_path, right_path,
5534 &left_key, cmp, ctx);
5535 if (ret < 0)
5536 goto out;
Alexander Block70698302012-06-05 21:07:48 +02005537 advance_left = ADVANCE;
5538 advance_right = ADVANCE;
5539 }
5540 } else if (left_level == right_level) {
5541 cmp = btrfs_comp_cpu_keys(&left_key, &right_key);
5542 if (cmp < 0) {
5543 advance_left = ADVANCE;
5544 } else if (cmp > 0) {
5545 advance_right = ADVANCE;
5546 } else {
5547 left_blockptr = btrfs_node_blockptr(
5548 left_path->nodes[left_level],
5549 left_path->slots[left_level]);
5550 right_blockptr = btrfs_node_blockptr(
5551 right_path->nodes[right_level],
5552 right_path->slots[right_level]);
Filipe Manana6baa4292014-02-20 21:15:25 +00005553 left_gen = btrfs_node_ptr_generation(
5554 left_path->nodes[left_level],
5555 left_path->slots[left_level]);
5556 right_gen = btrfs_node_ptr_generation(
5557 right_path->nodes[right_level],
5558 right_path->slots[right_level]);
5559 if (left_blockptr == right_blockptr &&
5560 left_gen == right_gen) {
Alexander Block70698302012-06-05 21:07:48 +02005561 /*
5562 * As we're on a shared block, don't
5563 * allow to go deeper.
5564 */
5565 advance_left = ADVANCE_ONLY_NEXT;
5566 advance_right = ADVANCE_ONLY_NEXT;
5567 } else {
5568 advance_left = ADVANCE;
5569 advance_right = ADVANCE;
5570 }
5571 }
5572 } else if (left_level < right_level) {
5573 advance_right = ADVANCE;
5574 } else {
5575 advance_left = ADVANCE;
5576 }
5577 }
5578
5579out:
5580 btrfs_free_path(left_path);
5581 btrfs_free_path(right_path);
5582 kfree(tmp_buf);
Alexander Block70698302012-06-05 21:07:48 +02005583 return ret;
5584}
5585
Chris Mason3f157a22008-06-25 16:01:31 -04005586/*
5587 * this is similar to btrfs_next_leaf, but does not try to preserve
5588 * and fixup the path. It looks for and returns the next key in the
Eric Sandeende78b512013-01-31 18:21:12 +00005589 * tree based on the current path and the min_trans parameters.
Chris Mason3f157a22008-06-25 16:01:31 -04005590 *
5591 * 0 is returned if another key is found, < 0 if there are any errors
5592 * and 1 is returned if there are no higher keys in the tree
5593 *
5594 * path->keep_locks should be set to 1 on the search made before
5595 * calling this function.
5596 */
Chris Masone7a84562008-06-25 16:01:31 -04005597int btrfs_find_next_key(struct btrfs_root *root, struct btrfs_path *path,
Eric Sandeende78b512013-01-31 18:21:12 +00005598 struct btrfs_key *key, int level, u64 min_trans)
Chris Masone7a84562008-06-25 16:01:31 -04005599{
Chris Masone7a84562008-06-25 16:01:31 -04005600 int slot;
5601 struct extent_buffer *c;
5602
Chris Mason934d3752008-12-08 16:43:10 -05005603 WARN_ON(!path->keep_locks);
Chris Masond3977122009-01-05 21:25:51 -05005604 while (level < BTRFS_MAX_LEVEL) {
Chris Masone7a84562008-06-25 16:01:31 -04005605 if (!path->nodes[level])
5606 return 1;
5607
5608 slot = path->slots[level] + 1;
5609 c = path->nodes[level];
Chris Mason3f157a22008-06-25 16:01:31 -04005610next:
Chris Masone7a84562008-06-25 16:01:31 -04005611 if (slot >= btrfs_header_nritems(c)) {
Yan Zheng33c66f42009-07-22 09:59:00 -04005612 int ret;
5613 int orig_lowest;
5614 struct btrfs_key cur_key;
5615 if (level + 1 >= BTRFS_MAX_LEVEL ||
5616 !path->nodes[level + 1])
Chris Masone7a84562008-06-25 16:01:31 -04005617 return 1;
Yan Zheng33c66f42009-07-22 09:59:00 -04005618
5619 if (path->locks[level + 1]) {
5620 level++;
5621 continue;
5622 }
5623
5624 slot = btrfs_header_nritems(c) - 1;
5625 if (level == 0)
5626 btrfs_item_key_to_cpu(c, &cur_key, slot);
5627 else
5628 btrfs_node_key_to_cpu(c, &cur_key, slot);
5629
5630 orig_lowest = path->lowest_level;
David Sterbab3b4aa72011-04-21 01:20:15 +02005631 btrfs_release_path(path);
Yan Zheng33c66f42009-07-22 09:59:00 -04005632 path->lowest_level = level;
5633 ret = btrfs_search_slot(NULL, root, &cur_key, path,
5634 0, 0);
5635 path->lowest_level = orig_lowest;
5636 if (ret < 0)
5637 return ret;
5638
5639 c = path->nodes[level];
5640 slot = path->slots[level];
5641 if (ret == 0)
5642 slot++;
5643 goto next;
Chris Masone7a84562008-06-25 16:01:31 -04005644 }
Yan Zheng33c66f42009-07-22 09:59:00 -04005645
Chris Masone7a84562008-06-25 16:01:31 -04005646 if (level == 0)
5647 btrfs_item_key_to_cpu(c, key, slot);
Chris Mason3f157a22008-06-25 16:01:31 -04005648 else {
Chris Mason3f157a22008-06-25 16:01:31 -04005649 u64 gen = btrfs_node_ptr_generation(c, slot);
5650
Chris Mason3f157a22008-06-25 16:01:31 -04005651 if (gen < min_trans) {
5652 slot++;
5653 goto next;
5654 }
Chris Masone7a84562008-06-25 16:01:31 -04005655 btrfs_node_key_to_cpu(c, key, slot);
Chris Mason3f157a22008-06-25 16:01:31 -04005656 }
Chris Masone7a84562008-06-25 16:01:31 -04005657 return 0;
5658 }
5659 return 1;
5660}
5661
Chris Mason7bb86312007-12-11 09:25:06 -05005662/*
Chris Mason925baed2008-06-25 16:01:30 -04005663 * search the tree again to find a leaf with greater keys
Chris Mason0f70abe2007-02-28 16:46:22 -05005664 * returns 0 if it found something or 1 if there are no greater leaves.
5665 * returns < 0 on io errors.
Chris Mason97571fd2007-02-24 13:39:08 -05005666 */
Chris Mason234b63a2007-03-13 10:46:10 -04005667int btrfs_next_leaf(struct btrfs_root *root, struct btrfs_path *path)
Chris Masond97e63b2007-02-20 16:40:44 -05005668{
Jan Schmidt3d7806e2012-06-11 08:29:29 +02005669 return btrfs_next_old_leaf(root, path, 0);
5670}
5671
5672int btrfs_next_old_leaf(struct btrfs_root *root, struct btrfs_path *path,
5673 u64 time_seq)
5674{
Chris Masond97e63b2007-02-20 16:40:44 -05005675 int slot;
Chris Mason8e73f272009-04-03 10:14:18 -04005676 int level;
Chris Mason5f39d392007-10-15 16:14:19 -04005677 struct extent_buffer *c;
Chris Mason8e73f272009-04-03 10:14:18 -04005678 struct extent_buffer *next;
Chris Mason925baed2008-06-25 16:01:30 -04005679 struct btrfs_key key;
5680 u32 nritems;
5681 int ret;
Chris Mason8e73f272009-04-03 10:14:18 -04005682 int old_spinning = path->leave_spinning;
Chris Masonbd681512011-07-16 15:23:14 -04005683 int next_rw_lock = 0;
Chris Mason925baed2008-06-25 16:01:30 -04005684
5685 nritems = btrfs_header_nritems(path->nodes[0]);
Chris Masond3977122009-01-05 21:25:51 -05005686 if (nritems == 0)
Chris Mason925baed2008-06-25 16:01:30 -04005687 return 1;
Chris Mason925baed2008-06-25 16:01:30 -04005688
Chris Mason8e73f272009-04-03 10:14:18 -04005689 btrfs_item_key_to_cpu(path->nodes[0], &key, nritems - 1);
5690again:
5691 level = 1;
5692 next = NULL;
Chris Masonbd681512011-07-16 15:23:14 -04005693 next_rw_lock = 0;
David Sterbab3b4aa72011-04-21 01:20:15 +02005694 btrfs_release_path(path);
Chris Mason8e73f272009-04-03 10:14:18 -04005695
Chris Masona2135012008-06-25 16:01:30 -04005696 path->keep_locks = 1;
Chris Mason31533fb2011-07-26 16:01:59 -04005697 path->leave_spinning = 1;
Chris Mason8e73f272009-04-03 10:14:18 -04005698
Jan Schmidt3d7806e2012-06-11 08:29:29 +02005699 if (time_seq)
5700 ret = btrfs_search_old_slot(root, &key, path, time_seq);
5701 else
5702 ret = btrfs_search_slot(NULL, root, &key, path, 0, 0);
Chris Mason925baed2008-06-25 16:01:30 -04005703 path->keep_locks = 0;
5704
5705 if (ret < 0)
5706 return ret;
5707
Chris Masona2135012008-06-25 16:01:30 -04005708 nritems = btrfs_header_nritems(path->nodes[0]);
Chris Mason168fd7d2008-06-25 16:01:30 -04005709 /*
5710 * by releasing the path above we dropped all our locks. A balance
5711 * could have added more items next to the key that used to be
5712 * at the very end of the block. So, check again here and
5713 * advance the path if there are now more items available.
5714 */
Chris Masona2135012008-06-25 16:01:30 -04005715 if (nritems > 0 && path->slots[0] < nritems - 1) {
Yan Zhenge457afe2009-07-22 09:59:00 -04005716 if (ret == 0)
5717 path->slots[0]++;
Chris Mason8e73f272009-04-03 10:14:18 -04005718 ret = 0;
Chris Mason925baed2008-06-25 16:01:30 -04005719 goto done;
5720 }
Liu Bo0b43e042014-06-09 11:04:49 +08005721 /*
5722 * So the above check misses one case:
5723 * - after releasing the path above, someone has removed the item that
5724 * used to be at the very end of the block, and balance between leafs
5725 * gets another one with bigger key.offset to replace it.
5726 *
5727 * This one should be returned as well, or we can get leaf corruption
5728 * later(esp. in __btrfs_drop_extents()).
5729 *
5730 * And a bit more explanation about this check,
5731 * with ret > 0, the key isn't found, the path points to the slot
5732 * where it should be inserted, so the path->slots[0] item must be the
5733 * bigger one.
5734 */
5735 if (nritems > 0 && ret > 0 && path->slots[0] == nritems - 1) {
5736 ret = 0;
5737 goto done;
5738 }
Chris Masond97e63b2007-02-20 16:40:44 -05005739
Chris Masond3977122009-01-05 21:25:51 -05005740 while (level < BTRFS_MAX_LEVEL) {
Chris Mason8e73f272009-04-03 10:14:18 -04005741 if (!path->nodes[level]) {
5742 ret = 1;
5743 goto done;
5744 }
Chris Mason5f39d392007-10-15 16:14:19 -04005745
Chris Masond97e63b2007-02-20 16:40:44 -05005746 slot = path->slots[level] + 1;
5747 c = path->nodes[level];
Chris Mason5f39d392007-10-15 16:14:19 -04005748 if (slot >= btrfs_header_nritems(c)) {
Chris Masond97e63b2007-02-20 16:40:44 -05005749 level++;
Chris Mason8e73f272009-04-03 10:14:18 -04005750 if (level == BTRFS_MAX_LEVEL) {
5751 ret = 1;
5752 goto done;
5753 }
Chris Masond97e63b2007-02-20 16:40:44 -05005754 continue;
5755 }
Chris Mason5f39d392007-10-15 16:14:19 -04005756
Chris Mason925baed2008-06-25 16:01:30 -04005757 if (next) {
Chris Masonbd681512011-07-16 15:23:14 -04005758 btrfs_tree_unlock_rw(next, next_rw_lock);
Chris Mason5f39d392007-10-15 16:14:19 -04005759 free_extent_buffer(next);
Chris Mason925baed2008-06-25 16:01:30 -04005760 }
Chris Mason5f39d392007-10-15 16:14:19 -04005761
Chris Mason8e73f272009-04-03 10:14:18 -04005762 next = c;
Chris Masonbd681512011-07-16 15:23:14 -04005763 next_rw_lock = path->locks[level];
Chris Mason8e73f272009-04-03 10:14:18 -04005764 ret = read_block_for_search(NULL, root, path, &next, level,
Jan Schmidt5d9e75c42012-05-16 18:25:47 +02005765 slot, &key, 0);
Chris Mason8e73f272009-04-03 10:14:18 -04005766 if (ret == -EAGAIN)
5767 goto again;
Chris Mason5f39d392007-10-15 16:14:19 -04005768
Chris Mason76a05b32009-05-14 13:24:30 -04005769 if (ret < 0) {
David Sterbab3b4aa72011-04-21 01:20:15 +02005770 btrfs_release_path(path);
Chris Mason76a05b32009-05-14 13:24:30 -04005771 goto done;
5772 }
5773
Chris Mason5cd57b22008-06-25 16:01:30 -04005774 if (!path->skip_locking) {
Chris Masonbd681512011-07-16 15:23:14 -04005775 ret = btrfs_try_tree_read_lock(next);
Jan Schmidtd42244a2012-06-22 14:51:15 +02005776 if (!ret && time_seq) {
5777 /*
5778 * If we don't get the lock, we may be racing
5779 * with push_leaf_left, holding that lock while
5780 * itself waiting for the leaf we've currently
5781 * locked. To solve this situation, we give up
5782 * on our lock and cycle.
5783 */
Jan Schmidtcf538832012-07-04 15:42:48 +02005784 free_extent_buffer(next);
Jan Schmidtd42244a2012-06-22 14:51:15 +02005785 btrfs_release_path(path);
5786 cond_resched();
5787 goto again;
5788 }
Chris Mason8e73f272009-04-03 10:14:18 -04005789 if (!ret) {
5790 btrfs_set_path_blocking(path);
Chris Masonbd681512011-07-16 15:23:14 -04005791 btrfs_tree_read_lock(next);
Chris Mason31533fb2011-07-26 16:01:59 -04005792 btrfs_clear_path_blocking(path, next,
Chris Masonbd681512011-07-16 15:23:14 -04005793 BTRFS_READ_LOCK);
Chris Mason8e73f272009-04-03 10:14:18 -04005794 }
Chris Mason31533fb2011-07-26 16:01:59 -04005795 next_rw_lock = BTRFS_READ_LOCK;
Chris Mason5cd57b22008-06-25 16:01:30 -04005796 }
Chris Masond97e63b2007-02-20 16:40:44 -05005797 break;
5798 }
5799 path->slots[level] = slot;
Chris Masond3977122009-01-05 21:25:51 -05005800 while (1) {
Chris Masond97e63b2007-02-20 16:40:44 -05005801 level--;
5802 c = path->nodes[level];
Chris Mason925baed2008-06-25 16:01:30 -04005803 if (path->locks[level])
Chris Masonbd681512011-07-16 15:23:14 -04005804 btrfs_tree_unlock_rw(c, path->locks[level]);
Chris Mason8e73f272009-04-03 10:14:18 -04005805
Chris Mason5f39d392007-10-15 16:14:19 -04005806 free_extent_buffer(c);
Chris Masond97e63b2007-02-20 16:40:44 -05005807 path->nodes[level] = next;
5808 path->slots[level] = 0;
Chris Masona74a4b92008-06-25 16:01:31 -04005809 if (!path->skip_locking)
Chris Masonbd681512011-07-16 15:23:14 -04005810 path->locks[level] = next_rw_lock;
Chris Masond97e63b2007-02-20 16:40:44 -05005811 if (!level)
5812 break;
Chris Masonb4ce94d2009-02-04 09:25:08 -05005813
Chris Mason8e73f272009-04-03 10:14:18 -04005814 ret = read_block_for_search(NULL, root, path, &next, level,
Jan Schmidt5d9e75c42012-05-16 18:25:47 +02005815 0, &key, 0);
Chris Mason8e73f272009-04-03 10:14:18 -04005816 if (ret == -EAGAIN)
5817 goto again;
5818
Chris Mason76a05b32009-05-14 13:24:30 -04005819 if (ret < 0) {
David Sterbab3b4aa72011-04-21 01:20:15 +02005820 btrfs_release_path(path);
Chris Mason76a05b32009-05-14 13:24:30 -04005821 goto done;
5822 }
5823
Chris Mason5cd57b22008-06-25 16:01:30 -04005824 if (!path->skip_locking) {
Chris Masonbd681512011-07-16 15:23:14 -04005825 ret = btrfs_try_tree_read_lock(next);
Chris Mason8e73f272009-04-03 10:14:18 -04005826 if (!ret) {
5827 btrfs_set_path_blocking(path);
Chris Masonbd681512011-07-16 15:23:14 -04005828 btrfs_tree_read_lock(next);
Chris Mason31533fb2011-07-26 16:01:59 -04005829 btrfs_clear_path_blocking(path, next,
Chris Masonbd681512011-07-16 15:23:14 -04005830 BTRFS_READ_LOCK);
Chris Mason8e73f272009-04-03 10:14:18 -04005831 }
Chris Mason31533fb2011-07-26 16:01:59 -04005832 next_rw_lock = BTRFS_READ_LOCK;
Chris Mason5cd57b22008-06-25 16:01:30 -04005833 }
Chris Masond97e63b2007-02-20 16:40:44 -05005834 }
Chris Mason8e73f272009-04-03 10:14:18 -04005835 ret = 0;
Chris Mason925baed2008-06-25 16:01:30 -04005836done:
Chris Masonf7c79f32012-03-19 15:54:38 -04005837 unlock_up(path, 0, 1, 0, NULL);
Chris Mason8e73f272009-04-03 10:14:18 -04005838 path->leave_spinning = old_spinning;
5839 if (!old_spinning)
5840 btrfs_set_path_blocking(path);
5841
5842 return ret;
Chris Masond97e63b2007-02-20 16:40:44 -05005843}
Chris Mason0b86a832008-03-24 15:01:56 -04005844
Chris Mason3f157a22008-06-25 16:01:31 -04005845/*
5846 * this uses btrfs_prev_leaf to walk backwards in the tree, and keeps
5847 * searching until it gets past min_objectid or finds an item of 'type'
5848 *
5849 * returns 0 if something is found, 1 if nothing was found and < 0 on error
5850 */
Chris Mason0b86a832008-03-24 15:01:56 -04005851int btrfs_previous_item(struct btrfs_root *root,
5852 struct btrfs_path *path, u64 min_objectid,
5853 int type)
5854{
5855 struct btrfs_key found_key;
5856 struct extent_buffer *leaf;
Chris Masone02119d2008-09-05 16:13:11 -04005857 u32 nritems;
Chris Mason0b86a832008-03-24 15:01:56 -04005858 int ret;
5859
Chris Masond3977122009-01-05 21:25:51 -05005860 while (1) {
Chris Mason0b86a832008-03-24 15:01:56 -04005861 if (path->slots[0] == 0) {
Chris Masonb4ce94d2009-02-04 09:25:08 -05005862 btrfs_set_path_blocking(path);
Chris Mason0b86a832008-03-24 15:01:56 -04005863 ret = btrfs_prev_leaf(root, path);
5864 if (ret != 0)
5865 return ret;
5866 } else {
5867 path->slots[0]--;
5868 }
5869 leaf = path->nodes[0];
Chris Masone02119d2008-09-05 16:13:11 -04005870 nritems = btrfs_header_nritems(leaf);
5871 if (nritems == 0)
5872 return 1;
5873 if (path->slots[0] == nritems)
5874 path->slots[0]--;
5875
Chris Mason0b86a832008-03-24 15:01:56 -04005876 btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]);
Chris Masone02119d2008-09-05 16:13:11 -04005877 if (found_key.objectid < min_objectid)
5878 break;
Yan Zheng0a4eefb2009-07-24 11:06:53 -04005879 if (found_key.type == type)
5880 return 0;
Chris Masone02119d2008-09-05 16:13:11 -04005881 if (found_key.objectid == min_objectid &&
5882 found_key.type < type)
5883 break;
Chris Mason0b86a832008-03-24 15:01:56 -04005884 }
5885 return 1;
5886}
Wang Shilongade2e0b2014-01-12 21:38:33 +08005887
5888/*
5889 * search in extent tree to find a previous Metadata/Data extent item with
5890 * min objecitd.
5891 *
5892 * returns 0 if something is found, 1 if nothing was found and < 0 on error
5893 */
5894int btrfs_previous_extent_item(struct btrfs_root *root,
5895 struct btrfs_path *path, u64 min_objectid)
5896{
5897 struct btrfs_key found_key;
5898 struct extent_buffer *leaf;
5899 u32 nritems;
5900 int ret;
5901
5902 while (1) {
5903 if (path->slots[0] == 0) {
5904 btrfs_set_path_blocking(path);
5905 ret = btrfs_prev_leaf(root, path);
5906 if (ret != 0)
5907 return ret;
5908 } else {
5909 path->slots[0]--;
5910 }
5911 leaf = path->nodes[0];
5912 nritems = btrfs_header_nritems(leaf);
5913 if (nritems == 0)
5914 return 1;
5915 if (path->slots[0] == nritems)
5916 path->slots[0]--;
5917
5918 btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]);
5919 if (found_key.objectid < min_objectid)
5920 break;
5921 if (found_key.type == BTRFS_EXTENT_ITEM_KEY ||
5922 found_key.type == BTRFS_METADATA_ITEM_KEY)
5923 return 0;
5924 if (found_key.objectid == min_objectid &&
5925 found_key.type < BTRFS_EXTENT_ITEM_KEY)
5926 break;
5927 }
5928 return 1;
5929}