blob: 6b2ec90892293f9c81e3f3a17663b17eadddee2e [file] [log] [blame]
Chris Mason6cbd5572007-06-12 09:07:21 -04001/*
Chris Masond352ac62008-09-29 15:18:18 -04002 * Copyright (C) 2007,2008 Oracle. All rights reserved.
Chris Mason6cbd5572007-06-12 09:07:21 -04003 *
4 * This program is free software; you can redistribute it and/or
5 * modify it under the terms of the GNU General Public
6 * License v2 as published by the Free Software Foundation.
7 *
8 * This program is distributed in the hope that it will be useful,
9 * but WITHOUT ANY WARRANTY; without even the implied warranty of
10 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
11 * General Public License for more details.
12 *
13 * You should have received a copy of the GNU General Public
14 * License along with this program; if not, write to the
15 * Free Software Foundation, Inc., 59 Temple Place - Suite 330,
16 * Boston, MA 021110-1307, USA.
17 */
18
Chris Masona6b6e752007-10-15 16:22:39 -040019#include <linux/sched.h>
Tejun Heo5a0e3ad2010-03-24 17:04:11 +090020#include <linux/slab.h>
Jan Schmidtbd989ba2012-05-16 17:18:50 +020021#include <linux/rbtree.h>
Chris Masoneb60cea2007-02-02 09:18:22 -050022#include "ctree.h"
23#include "disk-io.h"
Chris Mason7f5c1512007-03-23 15:56:19 -040024#include "transaction.h"
Chris Mason5f39d392007-10-15 16:14:19 -040025#include "print-tree.h"
Chris Mason925baed2008-06-25 16:01:30 -040026#include "locking.h"
Chris Mason9a8dd152007-02-23 08:38:36 -050027
Chris Masone089f052007-03-16 16:20:31 -040028static int split_node(struct btrfs_trans_handle *trans, struct btrfs_root
29 *root, struct btrfs_path *path, int level);
30static int split_leaf(struct btrfs_trans_handle *trans, struct btrfs_root
Chris Masond4dbff92007-04-04 14:08:15 -040031 *root, struct btrfs_key *ins_key,
Chris Masoncc0c5532007-10-25 15:42:57 -040032 struct btrfs_path *path, int data_size, int extend);
Chris Mason5f39d392007-10-15 16:14:19 -040033static int push_node_left(struct btrfs_trans_handle *trans,
34 struct btrfs_root *root, struct extent_buffer *dst,
Chris Mason971a1f62008-04-24 10:54:32 -040035 struct extent_buffer *src, int empty);
Chris Mason5f39d392007-10-15 16:14:19 -040036static int balance_node_right(struct btrfs_trans_handle *trans,
37 struct btrfs_root *root,
38 struct extent_buffer *dst_buf,
39 struct extent_buffer *src_buf);
Tsutomu Itohafe5fea2013-04-16 05:18:22 +000040static void del_ptr(struct btrfs_root *root, struct btrfs_path *path,
41 int level, int slot);
Filipe David Borba Manana5de865e2013-12-20 15:17:46 +000042static int tree_mod_log_free_eb(struct btrfs_fs_info *fs_info,
Jan Schmidtf2304752012-05-26 11:43:17 +020043 struct extent_buffer *eb);
Chris Masond97e63b2007-02-20 16:40:44 -050044
Chris Mason2c90e5d2007-04-02 10:50:19 -040045struct btrfs_path *btrfs_alloc_path(void)
46{
Chris Masondf24a2b2007-04-04 09:36:31 -040047 struct btrfs_path *path;
Jeff Mahoneye00f7302009-02-12 14:11:25 -050048 path = kmem_cache_zalloc(btrfs_path_cachep, GFP_NOFS);
Chris Masondf24a2b2007-04-04 09:36:31 -040049 return path;
Chris Mason2c90e5d2007-04-02 10:50:19 -040050}
51
Chris Masonb4ce94d2009-02-04 09:25:08 -050052/*
53 * set all locked nodes in the path to blocking locks. This should
54 * be done before scheduling
55 */
56noinline void btrfs_set_path_blocking(struct btrfs_path *p)
57{
58 int i;
59 for (i = 0; i < BTRFS_MAX_LEVEL; i++) {
Chris Masonbd681512011-07-16 15:23:14 -040060 if (!p->nodes[i] || !p->locks[i])
61 continue;
62 btrfs_set_lock_blocking_rw(p->nodes[i], p->locks[i]);
63 if (p->locks[i] == BTRFS_READ_LOCK)
64 p->locks[i] = BTRFS_READ_LOCK_BLOCKING;
65 else if (p->locks[i] == BTRFS_WRITE_LOCK)
66 p->locks[i] = BTRFS_WRITE_LOCK_BLOCKING;
Chris Masonb4ce94d2009-02-04 09:25:08 -050067 }
68}
69
70/*
71 * reset all the locked nodes in the patch to spinning locks.
Chris Mason4008c042009-02-12 14:09:45 -050072 *
73 * held is used to keep lockdep happy, when lockdep is enabled
74 * we set held to a blocking lock before we go around and
75 * retake all the spinlocks in the path. You can safely use NULL
76 * for held
Chris Masonb4ce94d2009-02-04 09:25:08 -050077 */
Chris Mason4008c042009-02-12 14:09:45 -050078noinline void btrfs_clear_path_blocking(struct btrfs_path *p,
Chris Masonbd681512011-07-16 15:23:14 -040079 struct extent_buffer *held, int held_rw)
Chris Masonb4ce94d2009-02-04 09:25:08 -050080{
81 int i;
Chris Mason4008c042009-02-12 14:09:45 -050082
Chris Masonbd681512011-07-16 15:23:14 -040083 if (held) {
84 btrfs_set_lock_blocking_rw(held, held_rw);
85 if (held_rw == BTRFS_WRITE_LOCK)
86 held_rw = BTRFS_WRITE_LOCK_BLOCKING;
87 else if (held_rw == BTRFS_READ_LOCK)
88 held_rw = BTRFS_READ_LOCK_BLOCKING;
89 }
Chris Mason4008c042009-02-12 14:09:45 -050090 btrfs_set_path_blocking(p);
Chris Mason4008c042009-02-12 14:09:45 -050091
92 for (i = BTRFS_MAX_LEVEL - 1; i >= 0; i--) {
Chris Masonbd681512011-07-16 15:23:14 -040093 if (p->nodes[i] && p->locks[i]) {
94 btrfs_clear_lock_blocking_rw(p->nodes[i], p->locks[i]);
95 if (p->locks[i] == BTRFS_WRITE_LOCK_BLOCKING)
96 p->locks[i] = BTRFS_WRITE_LOCK;
97 else if (p->locks[i] == BTRFS_READ_LOCK_BLOCKING)
98 p->locks[i] = BTRFS_READ_LOCK;
99 }
Chris Masonb4ce94d2009-02-04 09:25:08 -0500100 }
Chris Mason4008c042009-02-12 14:09:45 -0500101
Chris Mason4008c042009-02-12 14:09:45 -0500102 if (held)
Chris Masonbd681512011-07-16 15:23:14 -0400103 btrfs_clear_lock_blocking_rw(held, held_rw);
Chris Masonb4ce94d2009-02-04 09:25:08 -0500104}
105
Chris Masond352ac62008-09-29 15:18:18 -0400106/* this also releases the path */
Chris Mason2c90e5d2007-04-02 10:50:19 -0400107void btrfs_free_path(struct btrfs_path *p)
108{
Jesper Juhlff175d52010-12-25 21:22:30 +0000109 if (!p)
110 return;
David Sterbab3b4aa72011-04-21 01:20:15 +0200111 btrfs_release_path(p);
Chris Mason2c90e5d2007-04-02 10:50:19 -0400112 kmem_cache_free(btrfs_path_cachep, p);
113}
114
Chris Masond352ac62008-09-29 15:18:18 -0400115/*
116 * path release drops references on the extent buffers in the path
117 * and it drops any locks held by this path
118 *
119 * It is safe to call this on paths that no locks or extent buffers held.
120 */
David Sterbab3b4aa72011-04-21 01:20:15 +0200121noinline void btrfs_release_path(struct btrfs_path *p)
Chris Masoneb60cea2007-02-02 09:18:22 -0500122{
123 int i;
Chris Masona2135012008-06-25 16:01:30 -0400124
Chris Mason234b63a2007-03-13 10:46:10 -0400125 for (i = 0; i < BTRFS_MAX_LEVEL; i++) {
Chris Mason3f157a22008-06-25 16:01:31 -0400126 p->slots[i] = 0;
Chris Masoneb60cea2007-02-02 09:18:22 -0500127 if (!p->nodes[i])
Chris Mason925baed2008-06-25 16:01:30 -0400128 continue;
129 if (p->locks[i]) {
Chris Masonbd681512011-07-16 15:23:14 -0400130 btrfs_tree_unlock_rw(p->nodes[i], p->locks[i]);
Chris Mason925baed2008-06-25 16:01:30 -0400131 p->locks[i] = 0;
132 }
Chris Mason5f39d392007-10-15 16:14:19 -0400133 free_extent_buffer(p->nodes[i]);
Chris Mason3f157a22008-06-25 16:01:31 -0400134 p->nodes[i] = NULL;
Chris Masoneb60cea2007-02-02 09:18:22 -0500135 }
136}
137
Chris Masond352ac62008-09-29 15:18:18 -0400138/*
139 * safely gets a reference on the root node of a tree. A lock
140 * is not taken, so a concurrent writer may put a different node
141 * at the root of the tree. See btrfs_lock_root_node for the
142 * looping required.
143 *
144 * The extent buffer returned by this has a reference taken, so
145 * it won't disappear. It may stop being the root of the tree
146 * at any time because there are no locks held.
147 */
Chris Mason925baed2008-06-25 16:01:30 -0400148struct extent_buffer *btrfs_root_node(struct btrfs_root *root)
149{
150 struct extent_buffer *eb;
Chris Mason240f62c2011-03-23 14:54:42 -0400151
Josef Bacik3083ee22012-03-09 16:01:49 -0500152 while (1) {
153 rcu_read_lock();
154 eb = rcu_dereference(root->node);
155
156 /*
157 * RCU really hurts here, we could free up the root node because
158 * it was cow'ed but we may not get the new root node yet so do
159 * the inc_not_zero dance and if it doesn't work then
160 * synchronize_rcu and try again.
161 */
162 if (atomic_inc_not_zero(&eb->refs)) {
163 rcu_read_unlock();
164 break;
165 }
166 rcu_read_unlock();
167 synchronize_rcu();
168 }
Chris Mason925baed2008-06-25 16:01:30 -0400169 return eb;
170}
171
Chris Masond352ac62008-09-29 15:18:18 -0400172/* loop around taking references on and locking the root node of the
173 * tree until you end up with a lock on the root. A locked buffer
174 * is returned, with a reference held.
175 */
Chris Mason925baed2008-06-25 16:01:30 -0400176struct extent_buffer *btrfs_lock_root_node(struct btrfs_root *root)
177{
178 struct extent_buffer *eb;
179
Chris Masond3977122009-01-05 21:25:51 -0500180 while (1) {
Chris Mason925baed2008-06-25 16:01:30 -0400181 eb = btrfs_root_node(root);
182 btrfs_tree_lock(eb);
Chris Mason240f62c2011-03-23 14:54:42 -0400183 if (eb == root->node)
Chris Mason925baed2008-06-25 16:01:30 -0400184 break;
Chris Mason925baed2008-06-25 16:01:30 -0400185 btrfs_tree_unlock(eb);
186 free_extent_buffer(eb);
187 }
188 return eb;
189}
190
Chris Masonbd681512011-07-16 15:23:14 -0400191/* loop around taking references on and locking the root node of the
192 * tree until you end up with a lock on the root. A locked buffer
193 * is returned, with a reference held.
194 */
Eric Sandeen48a3b632013-04-25 20:41:01 +0000195static struct extent_buffer *btrfs_read_lock_root_node(struct btrfs_root *root)
Chris Masonbd681512011-07-16 15:23:14 -0400196{
197 struct extent_buffer *eb;
198
199 while (1) {
200 eb = btrfs_root_node(root);
201 btrfs_tree_read_lock(eb);
202 if (eb == root->node)
203 break;
204 btrfs_tree_read_unlock(eb);
205 free_extent_buffer(eb);
206 }
207 return eb;
208}
209
Chris Masond352ac62008-09-29 15:18:18 -0400210/* cowonly root (everything not a reference counted cow subvolume), just get
211 * put onto a simple dirty list. transaction.c walks this to make sure they
212 * get properly updated on disk.
213 */
Chris Mason0b86a832008-03-24 15:01:56 -0400214static void add_root_to_dirty_list(struct btrfs_root *root)
215{
Josef Bacike7070be2014-12-16 08:54:43 -0800216 if (test_bit(BTRFS_ROOT_DIRTY, &root->state) ||
217 !test_bit(BTRFS_ROOT_TRACK_DIRTY, &root->state))
218 return;
219
Chris Masone5846fc2012-05-03 12:08:48 -0400220 spin_lock(&root->fs_info->trans_lock);
Josef Bacike7070be2014-12-16 08:54:43 -0800221 if (!test_and_set_bit(BTRFS_ROOT_DIRTY, &root->state)) {
222 /* Want the extent tree to be the last on the list */
223 if (root->objectid == BTRFS_EXTENT_TREE_OBJECTID)
224 list_move_tail(&root->dirty_list,
225 &root->fs_info->dirty_cowonly_roots);
226 else
227 list_move(&root->dirty_list,
228 &root->fs_info->dirty_cowonly_roots);
Chris Mason0b86a832008-03-24 15:01:56 -0400229 }
Chris Masone5846fc2012-05-03 12:08:48 -0400230 spin_unlock(&root->fs_info->trans_lock);
Chris Mason0b86a832008-03-24 15:01:56 -0400231}
232
Chris Masond352ac62008-09-29 15:18:18 -0400233/*
234 * used by snapshot creation to make a copy of a root for a tree with
235 * a given objectid. The buffer with the new root node is returned in
236 * cow_ret, and this func returns zero on success or a negative error code.
237 */
Chris Masonbe20aa92007-12-17 20:14:01 -0500238int btrfs_copy_root(struct btrfs_trans_handle *trans,
239 struct btrfs_root *root,
240 struct extent_buffer *buf,
241 struct extent_buffer **cow_ret, u64 new_root_objectid)
242{
243 struct extent_buffer *cow;
Chris Masonbe20aa92007-12-17 20:14:01 -0500244 int ret = 0;
245 int level;
Yan Zheng5d4f98a2009-06-10 10:45:14 -0400246 struct btrfs_disk_key disk_key;
Chris Masonbe20aa92007-12-17 20:14:01 -0500247
Miao Xie27cdeb72014-04-02 19:51:05 +0800248 WARN_ON(test_bit(BTRFS_ROOT_REF_COWS, &root->state) &&
249 trans->transid != root->fs_info->running_transaction->transid);
250 WARN_ON(test_bit(BTRFS_ROOT_REF_COWS, &root->state) &&
251 trans->transid != root->last_trans);
Chris Masonbe20aa92007-12-17 20:14:01 -0500252
253 level = btrfs_header_level(buf);
Yan Zheng5d4f98a2009-06-10 10:45:14 -0400254 if (level == 0)
255 btrfs_item_key(buf, &disk_key, 0);
256 else
257 btrfs_node_key(buf, &disk_key, 0);
Zheng Yan31840ae2008-09-23 13:14:14 -0400258
David Sterba4d75f8a2014-06-15 01:54:12 +0200259 cow = btrfs_alloc_tree_block(trans, root, 0, new_root_objectid,
260 &disk_key, level, buf->start, 0);
Yan Zheng5d4f98a2009-06-10 10:45:14 -0400261 if (IS_ERR(cow))
Chris Masonbe20aa92007-12-17 20:14:01 -0500262 return PTR_ERR(cow);
263
264 copy_extent_buffer(cow, buf, 0, 0, cow->len);
265 btrfs_set_header_bytenr(cow, cow->start);
266 btrfs_set_header_generation(cow, trans->transid);
Yan Zheng5d4f98a2009-06-10 10:45:14 -0400267 btrfs_set_header_backref_rev(cow, BTRFS_MIXED_BACKREF_REV);
268 btrfs_clear_header_flag(cow, BTRFS_HEADER_FLAG_WRITTEN |
269 BTRFS_HEADER_FLAG_RELOC);
270 if (new_root_objectid == BTRFS_TREE_RELOC_OBJECTID)
271 btrfs_set_header_flag(cow, BTRFS_HEADER_FLAG_RELOC);
272 else
273 btrfs_set_header_owner(cow, new_root_objectid);
Chris Masonbe20aa92007-12-17 20:14:01 -0500274
Ross Kirk0a4e5582013-09-24 10:12:38 +0100275 write_extent_buffer(cow, root->fs_info->fsid, btrfs_header_fsid(),
Yan Zheng2b820322008-11-17 21:11:30 -0500276 BTRFS_FSID_SIZE);
277
Chris Masonbe20aa92007-12-17 20:14:01 -0500278 WARN_ON(btrfs_header_generation(buf) > trans->transid);
Yan Zheng5d4f98a2009-06-10 10:45:14 -0400279 if (new_root_objectid == BTRFS_TREE_RELOC_OBJECTID)
Josef Bacike339a6b2014-07-02 10:54:25 -0700280 ret = btrfs_inc_ref(trans, root, cow, 1);
Yan Zheng5d4f98a2009-06-10 10:45:14 -0400281 else
Josef Bacike339a6b2014-07-02 10:54:25 -0700282 ret = btrfs_inc_ref(trans, root, cow, 0);
Chris Mason4aec2b52007-12-18 16:25:45 -0500283
Chris Masonbe20aa92007-12-17 20:14:01 -0500284 if (ret)
285 return ret;
286
287 btrfs_mark_buffer_dirty(cow);
288 *cow_ret = cow;
289 return 0;
290}
291
Jan Schmidtbd989ba2012-05-16 17:18:50 +0200292enum mod_log_op {
293 MOD_LOG_KEY_REPLACE,
294 MOD_LOG_KEY_ADD,
295 MOD_LOG_KEY_REMOVE,
296 MOD_LOG_KEY_REMOVE_WHILE_FREEING,
297 MOD_LOG_KEY_REMOVE_WHILE_MOVING,
298 MOD_LOG_MOVE_KEYS,
299 MOD_LOG_ROOT_REPLACE,
300};
301
302struct tree_mod_move {
303 int dst_slot;
304 int nr_items;
305};
306
307struct tree_mod_root {
308 u64 logical;
309 u8 level;
310};
311
312struct tree_mod_elem {
313 struct rb_node node;
314 u64 index; /* shifted logical */
Jan Schmidt097b8a72012-06-21 11:08:04 +0200315 u64 seq;
Jan Schmidtbd989ba2012-05-16 17:18:50 +0200316 enum mod_log_op op;
317
318 /* this is used for MOD_LOG_KEY_* and MOD_LOG_MOVE_KEYS operations */
319 int slot;
320
321 /* this is used for MOD_LOG_KEY* and MOD_LOG_ROOT_REPLACE */
322 u64 generation;
323
324 /* those are used for op == MOD_LOG_KEY_{REPLACE,REMOVE} */
325 struct btrfs_disk_key key;
326 u64 blockptr;
327
328 /* this is used for op == MOD_LOG_MOVE_KEYS */
329 struct tree_mod_move move;
330
331 /* this is used for op == MOD_LOG_ROOT_REPLACE */
332 struct tree_mod_root old_root;
333};
334
Jan Schmidt097b8a72012-06-21 11:08:04 +0200335static inline void tree_mod_log_read_lock(struct btrfs_fs_info *fs_info)
Jan Schmidtbd989ba2012-05-16 17:18:50 +0200336{
Jan Schmidt097b8a72012-06-21 11:08:04 +0200337 read_lock(&fs_info->tree_mod_log_lock);
Jan Schmidtbd989ba2012-05-16 17:18:50 +0200338}
339
Jan Schmidt097b8a72012-06-21 11:08:04 +0200340static inline void tree_mod_log_read_unlock(struct btrfs_fs_info *fs_info)
Jan Schmidtbd989ba2012-05-16 17:18:50 +0200341{
Jan Schmidt097b8a72012-06-21 11:08:04 +0200342 read_unlock(&fs_info->tree_mod_log_lock);
343}
344
345static inline void tree_mod_log_write_lock(struct btrfs_fs_info *fs_info)
346{
347 write_lock(&fs_info->tree_mod_log_lock);
348}
349
350static inline void tree_mod_log_write_unlock(struct btrfs_fs_info *fs_info)
351{
352 write_unlock(&fs_info->tree_mod_log_lock);
353}
354
355/*
Josef Bacikfcebe452014-05-13 17:30:47 -0700356 * Pull a new tree mod seq number for our operation.
Jan Schmidtfc36ed7e2013-04-24 16:57:33 +0000357 */
Josef Bacikfcebe452014-05-13 17:30:47 -0700358static inline u64 btrfs_inc_tree_mod_seq(struct btrfs_fs_info *fs_info)
Jan Schmidtfc36ed7e2013-04-24 16:57:33 +0000359{
360 return atomic64_inc_return(&fs_info->tree_mod_seq);
361}
362
363/*
Jan Schmidt097b8a72012-06-21 11:08:04 +0200364 * This adds a new blocker to the tree mod log's blocker list if the @elem
365 * passed does not already have a sequence number set. So when a caller expects
366 * to record tree modifications, it should ensure to set elem->seq to zero
367 * before calling btrfs_get_tree_mod_seq.
368 * Returns a fresh, unused tree log modification sequence number, even if no new
369 * blocker was added.
370 */
371u64 btrfs_get_tree_mod_seq(struct btrfs_fs_info *fs_info,
372 struct seq_list *elem)
373{
Jan Schmidt097b8a72012-06-21 11:08:04 +0200374 tree_mod_log_write_lock(fs_info);
Jan Schmidtbd989ba2012-05-16 17:18:50 +0200375 spin_lock(&fs_info->tree_mod_seq_lock);
Jan Schmidt097b8a72012-06-21 11:08:04 +0200376 if (!elem->seq) {
Josef Bacikfcebe452014-05-13 17:30:47 -0700377 elem->seq = btrfs_inc_tree_mod_seq(fs_info);
Jan Schmidt097b8a72012-06-21 11:08:04 +0200378 list_add_tail(&elem->list, &fs_info->tree_mod_seq_list);
379 }
Jan Schmidtbd989ba2012-05-16 17:18:50 +0200380 spin_unlock(&fs_info->tree_mod_seq_lock);
Jan Schmidt097b8a72012-06-21 11:08:04 +0200381 tree_mod_log_write_unlock(fs_info);
382
Josef Bacikfcebe452014-05-13 17:30:47 -0700383 return elem->seq;
Jan Schmidtbd989ba2012-05-16 17:18:50 +0200384}
385
386void btrfs_put_tree_mod_seq(struct btrfs_fs_info *fs_info,
387 struct seq_list *elem)
388{
389 struct rb_root *tm_root;
390 struct rb_node *node;
391 struct rb_node *next;
392 struct seq_list *cur_elem;
393 struct tree_mod_elem *tm;
394 u64 min_seq = (u64)-1;
395 u64 seq_putting = elem->seq;
396
397 if (!seq_putting)
398 return;
399
Jan Schmidtbd989ba2012-05-16 17:18:50 +0200400 spin_lock(&fs_info->tree_mod_seq_lock);
401 list_del(&elem->list);
Jan Schmidt097b8a72012-06-21 11:08:04 +0200402 elem->seq = 0;
Jan Schmidtbd989ba2012-05-16 17:18:50 +0200403
404 list_for_each_entry(cur_elem, &fs_info->tree_mod_seq_list, list) {
Jan Schmidt097b8a72012-06-21 11:08:04 +0200405 if (cur_elem->seq < min_seq) {
Jan Schmidtbd989ba2012-05-16 17:18:50 +0200406 if (seq_putting > cur_elem->seq) {
407 /*
408 * blocker with lower sequence number exists, we
409 * cannot remove anything from the log
410 */
Jan Schmidt097b8a72012-06-21 11:08:04 +0200411 spin_unlock(&fs_info->tree_mod_seq_lock);
412 return;
Jan Schmidtbd989ba2012-05-16 17:18:50 +0200413 }
414 min_seq = cur_elem->seq;
415 }
416 }
Jan Schmidt097b8a72012-06-21 11:08:04 +0200417 spin_unlock(&fs_info->tree_mod_seq_lock);
418
419 /*
Jan Schmidtbd989ba2012-05-16 17:18:50 +0200420 * anything that's lower than the lowest existing (read: blocked)
421 * sequence number can be removed from the tree.
422 */
Jan Schmidt097b8a72012-06-21 11:08:04 +0200423 tree_mod_log_write_lock(fs_info);
Jan Schmidtbd989ba2012-05-16 17:18:50 +0200424 tm_root = &fs_info->tree_mod_log;
425 for (node = rb_first(tm_root); node; node = next) {
426 next = rb_next(node);
427 tm = container_of(node, struct tree_mod_elem, node);
Jan Schmidt097b8a72012-06-21 11:08:04 +0200428 if (tm->seq > min_seq)
Jan Schmidtbd989ba2012-05-16 17:18:50 +0200429 continue;
430 rb_erase(node, tm_root);
Jan Schmidtbd989ba2012-05-16 17:18:50 +0200431 kfree(tm);
432 }
Jan Schmidt097b8a72012-06-21 11:08:04 +0200433 tree_mod_log_write_unlock(fs_info);
Jan Schmidtbd989ba2012-05-16 17:18:50 +0200434}
435
436/*
437 * key order of the log:
438 * index -> sequence
439 *
440 * the index is the shifted logical of the *new* root node for root replace
441 * operations, or the shifted logical of the affected block for all other
442 * operations.
Filipe David Borba Manana5de865e2013-12-20 15:17:46 +0000443 *
444 * Note: must be called with write lock (tree_mod_log_write_lock).
Jan Schmidtbd989ba2012-05-16 17:18:50 +0200445 */
446static noinline int
447__tree_mod_log_insert(struct btrfs_fs_info *fs_info, struct tree_mod_elem *tm)
448{
449 struct rb_root *tm_root;
450 struct rb_node **new;
451 struct rb_node *parent = NULL;
452 struct tree_mod_elem *cur;
Jan Schmidtbd989ba2012-05-16 17:18:50 +0200453
Josef Bacikc8cc6342013-07-01 16:18:19 -0400454 BUG_ON(!tm);
455
Josef Bacikfcebe452014-05-13 17:30:47 -0700456 tm->seq = btrfs_inc_tree_mod_seq(fs_info);
Jan Schmidtbd989ba2012-05-16 17:18:50 +0200457
Jan Schmidtbd989ba2012-05-16 17:18:50 +0200458 tm_root = &fs_info->tree_mod_log;
459 new = &tm_root->rb_node;
460 while (*new) {
461 cur = container_of(*new, struct tree_mod_elem, node);
462 parent = *new;
463 if (cur->index < tm->index)
464 new = &((*new)->rb_left);
465 else if (cur->index > tm->index)
466 new = &((*new)->rb_right);
Jan Schmidt097b8a72012-06-21 11:08:04 +0200467 else if (cur->seq < tm->seq)
Jan Schmidtbd989ba2012-05-16 17:18:50 +0200468 new = &((*new)->rb_left);
Jan Schmidt097b8a72012-06-21 11:08:04 +0200469 else if (cur->seq > tm->seq)
Jan Schmidtbd989ba2012-05-16 17:18:50 +0200470 new = &((*new)->rb_right);
Filipe David Borba Manana5de865e2013-12-20 15:17:46 +0000471 else
472 return -EEXIST;
Jan Schmidtbd989ba2012-05-16 17:18:50 +0200473 }
474
475 rb_link_node(&tm->node, parent, new);
476 rb_insert_color(&tm->node, tm_root);
Filipe David Borba Manana5de865e2013-12-20 15:17:46 +0000477 return 0;
Jan Schmidtbd989ba2012-05-16 17:18:50 +0200478}
479
Jan Schmidt097b8a72012-06-21 11:08:04 +0200480/*
481 * Determines if logging can be omitted. Returns 1 if it can. Otherwise, it
482 * returns zero with the tree_mod_log_lock acquired. The caller must hold
483 * this until all tree mod log insertions are recorded in the rb tree and then
484 * call tree_mod_log_write_unlock() to release.
485 */
Jan Schmidte9b7fd42012-05-31 14:59:09 +0200486static inline int tree_mod_dont_log(struct btrfs_fs_info *fs_info,
487 struct extent_buffer *eb) {
488 smp_mb();
489 if (list_empty(&(fs_info)->tree_mod_seq_list))
490 return 1;
Jan Schmidt097b8a72012-06-21 11:08:04 +0200491 if (eb && btrfs_header_level(eb) == 0)
Jan Schmidte9b7fd42012-05-31 14:59:09 +0200492 return 1;
Filipe David Borba Manana5de865e2013-12-20 15:17:46 +0000493
494 tree_mod_log_write_lock(fs_info);
495 if (list_empty(&(fs_info)->tree_mod_seq_list)) {
496 tree_mod_log_write_unlock(fs_info);
497 return 1;
498 }
499
Jan Schmidte9b7fd42012-05-31 14:59:09 +0200500 return 0;
501}
502
Filipe David Borba Manana5de865e2013-12-20 15:17:46 +0000503/* Similar to tree_mod_dont_log, but doesn't acquire any locks. */
504static inline int tree_mod_need_log(const struct btrfs_fs_info *fs_info,
505 struct extent_buffer *eb)
506{
507 smp_mb();
508 if (list_empty(&(fs_info)->tree_mod_seq_list))
509 return 0;
510 if (eb && btrfs_header_level(eb) == 0)
511 return 0;
512
513 return 1;
514}
515
516static struct tree_mod_elem *
517alloc_tree_mod_elem(struct extent_buffer *eb, int slot,
518 enum mod_log_op op, gfp_t flags)
Jan Schmidtbd989ba2012-05-16 17:18:50 +0200519{
Jan Schmidt097b8a72012-06-21 11:08:04 +0200520 struct tree_mod_elem *tm;
Jan Schmidtbd989ba2012-05-16 17:18:50 +0200521
Josef Bacikc8cc6342013-07-01 16:18:19 -0400522 tm = kzalloc(sizeof(*tm), flags);
523 if (!tm)
Filipe David Borba Manana5de865e2013-12-20 15:17:46 +0000524 return NULL;
Jan Schmidtbd989ba2012-05-16 17:18:50 +0200525
526 tm->index = eb->start >> PAGE_CACHE_SHIFT;
527 if (op != MOD_LOG_KEY_ADD) {
528 btrfs_node_key(eb, &tm->key, slot);
529 tm->blockptr = btrfs_node_blockptr(eb, slot);
530 }
531 tm->op = op;
532 tm->slot = slot;
533 tm->generation = btrfs_node_ptr_generation(eb, slot);
Filipe David Borba Manana5de865e2013-12-20 15:17:46 +0000534 RB_CLEAR_NODE(&tm->node);
Jan Schmidtbd989ba2012-05-16 17:18:50 +0200535
Filipe David Borba Manana5de865e2013-12-20 15:17:46 +0000536 return tm;
Jan Schmidt097b8a72012-06-21 11:08:04 +0200537}
538
539static noinline int
Josef Bacikc8cc6342013-07-01 16:18:19 -0400540tree_mod_log_insert_key(struct btrfs_fs_info *fs_info,
541 struct extent_buffer *eb, int slot,
542 enum mod_log_op op, gfp_t flags)
Jan Schmidt097b8a72012-06-21 11:08:04 +0200543{
Filipe David Borba Manana5de865e2013-12-20 15:17:46 +0000544 struct tree_mod_elem *tm;
545 int ret;
546
547 if (!tree_mod_need_log(fs_info, eb))
Jan Schmidt097b8a72012-06-21 11:08:04 +0200548 return 0;
549
Filipe David Borba Manana5de865e2013-12-20 15:17:46 +0000550 tm = alloc_tree_mod_elem(eb, slot, op, flags);
551 if (!tm)
552 return -ENOMEM;
553
554 if (tree_mod_dont_log(fs_info, eb)) {
555 kfree(tm);
556 return 0;
557 }
558
559 ret = __tree_mod_log_insert(fs_info, tm);
560 tree_mod_log_write_unlock(fs_info);
561 if (ret)
562 kfree(tm);
563
564 return ret;
Jan Schmidt097b8a72012-06-21 11:08:04 +0200565}
566
567static noinline int
Jan Schmidtbd989ba2012-05-16 17:18:50 +0200568tree_mod_log_insert_move(struct btrfs_fs_info *fs_info,
569 struct extent_buffer *eb, int dst_slot, int src_slot,
570 int nr_items, gfp_t flags)
571{
Filipe David Borba Manana5de865e2013-12-20 15:17:46 +0000572 struct tree_mod_elem *tm = NULL;
573 struct tree_mod_elem **tm_list = NULL;
574 int ret = 0;
Jan Schmidtbd989ba2012-05-16 17:18:50 +0200575 int i;
Filipe David Borba Manana5de865e2013-12-20 15:17:46 +0000576 int locked = 0;
Jan Schmidtbd989ba2012-05-16 17:18:50 +0200577
Filipe David Borba Manana5de865e2013-12-20 15:17:46 +0000578 if (!tree_mod_need_log(fs_info, eb))
Jan Schmidtf3956942012-05-31 15:02:32 +0200579 return 0;
Jan Schmidtbd989ba2012-05-16 17:18:50 +0200580
Filipe David Borba Manana5de865e2013-12-20 15:17:46 +0000581 tm_list = kzalloc(nr_items * sizeof(struct tree_mod_elem *), flags);
582 if (!tm_list)
583 return -ENOMEM;
Jan Schmidtbd989ba2012-05-16 17:18:50 +0200584
Josef Bacikc8cc6342013-07-01 16:18:19 -0400585 tm = kzalloc(sizeof(*tm), flags);
Filipe David Borba Manana5de865e2013-12-20 15:17:46 +0000586 if (!tm) {
587 ret = -ENOMEM;
588 goto free_tms;
589 }
Jan Schmidtf3956942012-05-31 15:02:32 +0200590
Jan Schmidtbd989ba2012-05-16 17:18:50 +0200591 tm->index = eb->start >> PAGE_CACHE_SHIFT;
592 tm->slot = src_slot;
593 tm->move.dst_slot = dst_slot;
594 tm->move.nr_items = nr_items;
595 tm->op = MOD_LOG_MOVE_KEYS;
596
Filipe David Borba Manana5de865e2013-12-20 15:17:46 +0000597 for (i = 0; i + dst_slot < src_slot && i < nr_items; i++) {
598 tm_list[i] = alloc_tree_mod_elem(eb, i + dst_slot,
599 MOD_LOG_KEY_REMOVE_WHILE_MOVING, flags);
600 if (!tm_list[i]) {
601 ret = -ENOMEM;
602 goto free_tms;
603 }
604 }
605
606 if (tree_mod_dont_log(fs_info, eb))
607 goto free_tms;
608 locked = 1;
609
610 /*
611 * When we override something during the move, we log these removals.
612 * This can only happen when we move towards the beginning of the
613 * buffer, i.e. dst_slot < src_slot.
614 */
615 for (i = 0; i + dst_slot < src_slot && i < nr_items; i++) {
616 ret = __tree_mod_log_insert(fs_info, tm_list[i]);
617 if (ret)
618 goto free_tms;
619 }
620
621 ret = __tree_mod_log_insert(fs_info, tm);
622 if (ret)
623 goto free_tms;
624 tree_mod_log_write_unlock(fs_info);
625 kfree(tm_list);
626
627 return 0;
628free_tms:
629 for (i = 0; i < nr_items; i++) {
630 if (tm_list[i] && !RB_EMPTY_NODE(&tm_list[i]->node))
631 rb_erase(&tm_list[i]->node, &fs_info->tree_mod_log);
632 kfree(tm_list[i]);
633 }
634 if (locked)
635 tree_mod_log_write_unlock(fs_info);
636 kfree(tm_list);
637 kfree(tm);
638
639 return ret;
Jan Schmidtbd989ba2012-05-16 17:18:50 +0200640}
641
Filipe David Borba Manana5de865e2013-12-20 15:17:46 +0000642static inline int
643__tree_mod_log_free_eb(struct btrfs_fs_info *fs_info,
644 struct tree_mod_elem **tm_list,
645 int nritems)
Jan Schmidt097b8a72012-06-21 11:08:04 +0200646{
Filipe David Borba Manana5de865e2013-12-20 15:17:46 +0000647 int i, j;
Jan Schmidt097b8a72012-06-21 11:08:04 +0200648 int ret;
649
Jan Schmidt097b8a72012-06-21 11:08:04 +0200650 for (i = nritems - 1; i >= 0; i--) {
Filipe David Borba Manana5de865e2013-12-20 15:17:46 +0000651 ret = __tree_mod_log_insert(fs_info, tm_list[i]);
652 if (ret) {
653 for (j = nritems - 1; j > i; j--)
654 rb_erase(&tm_list[j]->node,
655 &fs_info->tree_mod_log);
656 return ret;
657 }
Jan Schmidt097b8a72012-06-21 11:08:04 +0200658 }
Filipe David Borba Manana5de865e2013-12-20 15:17:46 +0000659
660 return 0;
Jan Schmidt097b8a72012-06-21 11:08:04 +0200661}
662
Jan Schmidtbd989ba2012-05-16 17:18:50 +0200663static noinline int
664tree_mod_log_insert_root(struct btrfs_fs_info *fs_info,
665 struct extent_buffer *old_root,
Jan Schmidt90f8d622013-04-13 13:19:53 +0000666 struct extent_buffer *new_root, gfp_t flags,
667 int log_removal)
Jan Schmidtbd989ba2012-05-16 17:18:50 +0200668{
Filipe David Borba Manana5de865e2013-12-20 15:17:46 +0000669 struct tree_mod_elem *tm = NULL;
670 struct tree_mod_elem **tm_list = NULL;
671 int nritems = 0;
672 int ret = 0;
673 int i;
Jan Schmidtbd989ba2012-05-16 17:18:50 +0200674
Filipe David Borba Manana5de865e2013-12-20 15:17:46 +0000675 if (!tree_mod_need_log(fs_info, NULL))
Jan Schmidt097b8a72012-06-21 11:08:04 +0200676 return 0;
677
Filipe David Borba Manana5de865e2013-12-20 15:17:46 +0000678 if (log_removal && btrfs_header_level(old_root) > 0) {
679 nritems = btrfs_header_nritems(old_root);
680 tm_list = kzalloc(nritems * sizeof(struct tree_mod_elem *),
681 flags);
682 if (!tm_list) {
683 ret = -ENOMEM;
684 goto free_tms;
685 }
686 for (i = 0; i < nritems; i++) {
687 tm_list[i] = alloc_tree_mod_elem(old_root, i,
688 MOD_LOG_KEY_REMOVE_WHILE_FREEING, flags);
689 if (!tm_list[i]) {
690 ret = -ENOMEM;
691 goto free_tms;
692 }
693 }
694 }
Jan Schmidtd9abbf12013-03-20 13:49:48 +0000695
Josef Bacikc8cc6342013-07-01 16:18:19 -0400696 tm = kzalloc(sizeof(*tm), flags);
Filipe David Borba Manana5de865e2013-12-20 15:17:46 +0000697 if (!tm) {
698 ret = -ENOMEM;
699 goto free_tms;
700 }
Jan Schmidtbd989ba2012-05-16 17:18:50 +0200701
702 tm->index = new_root->start >> PAGE_CACHE_SHIFT;
703 tm->old_root.logical = old_root->start;
704 tm->old_root.level = btrfs_header_level(old_root);
705 tm->generation = btrfs_header_generation(old_root);
706 tm->op = MOD_LOG_ROOT_REPLACE;
707
Filipe David Borba Manana5de865e2013-12-20 15:17:46 +0000708 if (tree_mod_dont_log(fs_info, NULL))
709 goto free_tms;
710
711 if (tm_list)
712 ret = __tree_mod_log_free_eb(fs_info, tm_list, nritems);
713 if (!ret)
714 ret = __tree_mod_log_insert(fs_info, tm);
715
716 tree_mod_log_write_unlock(fs_info);
717 if (ret)
718 goto free_tms;
719 kfree(tm_list);
720
721 return ret;
722
723free_tms:
724 if (tm_list) {
725 for (i = 0; i < nritems; i++)
726 kfree(tm_list[i]);
727 kfree(tm_list);
728 }
729 kfree(tm);
730
731 return ret;
Jan Schmidtbd989ba2012-05-16 17:18:50 +0200732}
733
734static struct tree_mod_elem *
735__tree_mod_log_search(struct btrfs_fs_info *fs_info, u64 start, u64 min_seq,
736 int smallest)
737{
738 struct rb_root *tm_root;
739 struct rb_node *node;
740 struct tree_mod_elem *cur = NULL;
741 struct tree_mod_elem *found = NULL;
742 u64 index = start >> PAGE_CACHE_SHIFT;
743
Jan Schmidt097b8a72012-06-21 11:08:04 +0200744 tree_mod_log_read_lock(fs_info);
Jan Schmidtbd989ba2012-05-16 17:18:50 +0200745 tm_root = &fs_info->tree_mod_log;
746 node = tm_root->rb_node;
747 while (node) {
748 cur = container_of(node, struct tree_mod_elem, node);
749 if (cur->index < index) {
750 node = node->rb_left;
751 } else if (cur->index > index) {
752 node = node->rb_right;
Jan Schmidt097b8a72012-06-21 11:08:04 +0200753 } else if (cur->seq < min_seq) {
Jan Schmidtbd989ba2012-05-16 17:18:50 +0200754 node = node->rb_left;
755 } else if (!smallest) {
756 /* we want the node with the highest seq */
757 if (found)
Jan Schmidt097b8a72012-06-21 11:08:04 +0200758 BUG_ON(found->seq > cur->seq);
Jan Schmidtbd989ba2012-05-16 17:18:50 +0200759 found = cur;
760 node = node->rb_left;
Jan Schmidt097b8a72012-06-21 11:08:04 +0200761 } else if (cur->seq > min_seq) {
Jan Schmidtbd989ba2012-05-16 17:18:50 +0200762 /* we want the node with the smallest seq */
763 if (found)
Jan Schmidt097b8a72012-06-21 11:08:04 +0200764 BUG_ON(found->seq < cur->seq);
Jan Schmidtbd989ba2012-05-16 17:18:50 +0200765 found = cur;
766 node = node->rb_right;
767 } else {
768 found = cur;
769 break;
770 }
771 }
Jan Schmidt097b8a72012-06-21 11:08:04 +0200772 tree_mod_log_read_unlock(fs_info);
Jan Schmidtbd989ba2012-05-16 17:18:50 +0200773
774 return found;
775}
776
777/*
778 * this returns the element from the log with the smallest time sequence
779 * value that's in the log (the oldest log item). any element with a time
780 * sequence lower than min_seq will be ignored.
781 */
782static struct tree_mod_elem *
783tree_mod_log_search_oldest(struct btrfs_fs_info *fs_info, u64 start,
784 u64 min_seq)
785{
786 return __tree_mod_log_search(fs_info, start, min_seq, 1);
787}
788
789/*
790 * this returns the element from the log with the largest time sequence
791 * value that's in the log (the most recent log item). any element with
792 * a time sequence lower than min_seq will be ignored.
793 */
794static struct tree_mod_elem *
795tree_mod_log_search(struct btrfs_fs_info *fs_info, u64 start, u64 min_seq)
796{
797 return __tree_mod_log_search(fs_info, start, min_seq, 0);
798}
799
Filipe David Borba Manana5de865e2013-12-20 15:17:46 +0000800static noinline int
Jan Schmidtbd989ba2012-05-16 17:18:50 +0200801tree_mod_log_eb_copy(struct btrfs_fs_info *fs_info, struct extent_buffer *dst,
802 struct extent_buffer *src, unsigned long dst_offset,
Jan Schmidt90f8d622013-04-13 13:19:53 +0000803 unsigned long src_offset, int nr_items)
Jan Schmidtbd989ba2012-05-16 17:18:50 +0200804{
Filipe David Borba Manana5de865e2013-12-20 15:17:46 +0000805 int ret = 0;
806 struct tree_mod_elem **tm_list = NULL;
807 struct tree_mod_elem **tm_list_add, **tm_list_rem;
Jan Schmidtbd989ba2012-05-16 17:18:50 +0200808 int i;
Filipe David Borba Manana5de865e2013-12-20 15:17:46 +0000809 int locked = 0;
Jan Schmidtbd989ba2012-05-16 17:18:50 +0200810
Filipe David Borba Manana5de865e2013-12-20 15:17:46 +0000811 if (!tree_mod_need_log(fs_info, NULL))
812 return 0;
Jan Schmidtbd989ba2012-05-16 17:18:50 +0200813
Josef Bacikc8cc6342013-07-01 16:18:19 -0400814 if (btrfs_header_level(dst) == 0 && btrfs_header_level(src) == 0)
Filipe David Borba Manana5de865e2013-12-20 15:17:46 +0000815 return 0;
816
817 tm_list = kzalloc(nr_items * 2 * sizeof(struct tree_mod_elem *),
818 GFP_NOFS);
819 if (!tm_list)
820 return -ENOMEM;
821
822 tm_list_add = tm_list;
823 tm_list_rem = tm_list + nr_items;
824 for (i = 0; i < nr_items; i++) {
825 tm_list_rem[i] = alloc_tree_mod_elem(src, i + src_offset,
826 MOD_LOG_KEY_REMOVE, GFP_NOFS);
827 if (!tm_list_rem[i]) {
828 ret = -ENOMEM;
829 goto free_tms;
830 }
831
832 tm_list_add[i] = alloc_tree_mod_elem(dst, i + dst_offset,
833 MOD_LOG_KEY_ADD, GFP_NOFS);
834 if (!tm_list_add[i]) {
835 ret = -ENOMEM;
836 goto free_tms;
837 }
838 }
839
840 if (tree_mod_dont_log(fs_info, NULL))
841 goto free_tms;
842 locked = 1;
Jan Schmidtbd989ba2012-05-16 17:18:50 +0200843
Jan Schmidtbd989ba2012-05-16 17:18:50 +0200844 for (i = 0; i < nr_items; i++) {
Filipe David Borba Manana5de865e2013-12-20 15:17:46 +0000845 ret = __tree_mod_log_insert(fs_info, tm_list_rem[i]);
846 if (ret)
847 goto free_tms;
848 ret = __tree_mod_log_insert(fs_info, tm_list_add[i]);
849 if (ret)
850 goto free_tms;
Jan Schmidtbd989ba2012-05-16 17:18:50 +0200851 }
Filipe David Borba Manana5de865e2013-12-20 15:17:46 +0000852
853 tree_mod_log_write_unlock(fs_info);
854 kfree(tm_list);
855
856 return 0;
857
858free_tms:
859 for (i = 0; i < nr_items * 2; i++) {
860 if (tm_list[i] && !RB_EMPTY_NODE(&tm_list[i]->node))
861 rb_erase(&tm_list[i]->node, &fs_info->tree_mod_log);
862 kfree(tm_list[i]);
863 }
864 if (locked)
865 tree_mod_log_write_unlock(fs_info);
866 kfree(tm_list);
867
868 return ret;
Jan Schmidtbd989ba2012-05-16 17:18:50 +0200869}
870
871static inline void
872tree_mod_log_eb_move(struct btrfs_fs_info *fs_info, struct extent_buffer *dst,
873 int dst_offset, int src_offset, int nr_items)
874{
875 int ret;
876 ret = tree_mod_log_insert_move(fs_info, dst, dst_offset, src_offset,
877 nr_items, GFP_NOFS);
878 BUG_ON(ret < 0);
879}
880
Jan Schmidt097b8a72012-06-21 11:08:04 +0200881static noinline void
Jan Schmidtbd989ba2012-05-16 17:18:50 +0200882tree_mod_log_set_node_key(struct btrfs_fs_info *fs_info,
Liu Bo32adf092012-10-19 12:52:15 +0000883 struct extent_buffer *eb, int slot, int atomic)
Jan Schmidtbd989ba2012-05-16 17:18:50 +0200884{
885 int ret;
886
Filipe David Borba Manana78357762013-12-12 19:19:52 +0000887 ret = tree_mod_log_insert_key(fs_info, eb, slot,
Josef Bacikc8cc6342013-07-01 16:18:19 -0400888 MOD_LOG_KEY_REPLACE,
889 atomic ? GFP_ATOMIC : GFP_NOFS);
Jan Schmidtbd989ba2012-05-16 17:18:50 +0200890 BUG_ON(ret < 0);
891}
892
Filipe David Borba Manana5de865e2013-12-20 15:17:46 +0000893static noinline int
Jan Schmidt097b8a72012-06-21 11:08:04 +0200894tree_mod_log_free_eb(struct btrfs_fs_info *fs_info, struct extent_buffer *eb)
Jan Schmidtbd989ba2012-05-16 17:18:50 +0200895{
Filipe David Borba Manana5de865e2013-12-20 15:17:46 +0000896 struct tree_mod_elem **tm_list = NULL;
897 int nritems = 0;
898 int i;
899 int ret = 0;
900
901 if (btrfs_header_level(eb) == 0)
902 return 0;
903
904 if (!tree_mod_need_log(fs_info, NULL))
905 return 0;
906
907 nritems = btrfs_header_nritems(eb);
908 tm_list = kzalloc(nritems * sizeof(struct tree_mod_elem *),
909 GFP_NOFS);
910 if (!tm_list)
911 return -ENOMEM;
912
913 for (i = 0; i < nritems; i++) {
914 tm_list[i] = alloc_tree_mod_elem(eb, i,
915 MOD_LOG_KEY_REMOVE_WHILE_FREEING, GFP_NOFS);
916 if (!tm_list[i]) {
917 ret = -ENOMEM;
918 goto free_tms;
919 }
920 }
921
Jan Schmidte9b7fd42012-05-31 14:59:09 +0200922 if (tree_mod_dont_log(fs_info, eb))
Filipe David Borba Manana5de865e2013-12-20 15:17:46 +0000923 goto free_tms;
924
925 ret = __tree_mod_log_free_eb(fs_info, tm_list, nritems);
926 tree_mod_log_write_unlock(fs_info);
927 if (ret)
928 goto free_tms;
929 kfree(tm_list);
930
931 return 0;
932
933free_tms:
934 for (i = 0; i < nritems; i++)
935 kfree(tm_list[i]);
936 kfree(tm_list);
937
938 return ret;
Jan Schmidtbd989ba2012-05-16 17:18:50 +0200939}
940
Jan Schmidt097b8a72012-06-21 11:08:04 +0200941static noinline void
Jan Schmidtbd989ba2012-05-16 17:18:50 +0200942tree_mod_log_set_root_pointer(struct btrfs_root *root,
Jan Schmidt90f8d622013-04-13 13:19:53 +0000943 struct extent_buffer *new_root_node,
944 int log_removal)
Jan Schmidtbd989ba2012-05-16 17:18:50 +0200945{
946 int ret;
Jan Schmidtbd989ba2012-05-16 17:18:50 +0200947 ret = tree_mod_log_insert_root(root->fs_info, root->node,
Jan Schmidt90f8d622013-04-13 13:19:53 +0000948 new_root_node, GFP_NOFS, log_removal);
Jan Schmidtbd989ba2012-05-16 17:18:50 +0200949 BUG_ON(ret < 0);
950}
951
Chris Masond352ac62008-09-29 15:18:18 -0400952/*
Yan Zheng5d4f98a2009-06-10 10:45:14 -0400953 * check if the tree block can be shared by multiple trees
954 */
955int btrfs_block_can_be_shared(struct btrfs_root *root,
956 struct extent_buffer *buf)
957{
958 /*
959 * Tree blocks not in refernece counted trees and tree roots
960 * are never shared. If a block was allocated after the last
961 * snapshot and the block was not allocated by tree relocation,
962 * we know the block is not shared.
963 */
Miao Xie27cdeb72014-04-02 19:51:05 +0800964 if (test_bit(BTRFS_ROOT_REF_COWS, &root->state) &&
Yan Zheng5d4f98a2009-06-10 10:45:14 -0400965 buf != root->node && buf != root->commit_root &&
966 (btrfs_header_generation(buf) <=
967 btrfs_root_last_snapshot(&root->root_item) ||
968 btrfs_header_flag(buf, BTRFS_HEADER_FLAG_RELOC)))
969 return 1;
970#ifdef BTRFS_COMPAT_EXTENT_TREE_V0
Miao Xie27cdeb72014-04-02 19:51:05 +0800971 if (test_bit(BTRFS_ROOT_REF_COWS, &root->state) &&
Yan Zheng5d4f98a2009-06-10 10:45:14 -0400972 btrfs_header_backref_rev(buf) < BTRFS_MIXED_BACKREF_REV)
973 return 1;
974#endif
975 return 0;
976}
977
978static noinline int update_ref_for_cow(struct btrfs_trans_handle *trans,
979 struct btrfs_root *root,
980 struct extent_buffer *buf,
Yan, Zhengf0486c62010-05-16 10:46:25 -0400981 struct extent_buffer *cow,
982 int *last_ref)
Yan Zheng5d4f98a2009-06-10 10:45:14 -0400983{
984 u64 refs;
985 u64 owner;
986 u64 flags;
987 u64 new_flags = 0;
988 int ret;
989
990 /*
991 * Backrefs update rules:
992 *
993 * Always use full backrefs for extent pointers in tree block
994 * allocated by tree relocation.
995 *
996 * If a shared tree block is no longer referenced by its owner
997 * tree (btrfs_header_owner(buf) == root->root_key.objectid),
998 * use full backrefs for extent pointers in tree block.
999 *
1000 * If a tree block is been relocating
1001 * (root->root_key.objectid == BTRFS_TREE_RELOC_OBJECTID),
1002 * use full backrefs for extent pointers in tree block.
1003 * The reason for this is some operations (such as drop tree)
1004 * are only allowed for blocks use full backrefs.
1005 */
1006
1007 if (btrfs_block_can_be_shared(root, buf)) {
1008 ret = btrfs_lookup_extent_info(trans, root, buf->start,
Josef Bacik3173a182013-03-07 14:22:04 -05001009 btrfs_header_level(buf), 1,
1010 &refs, &flags);
Mark Fashehbe1a5562011-08-08 13:20:18 -07001011 if (ret)
1012 return ret;
Mark Fashehe5df9572011-08-29 14:17:04 -07001013 if (refs == 0) {
1014 ret = -EROFS;
1015 btrfs_std_error(root->fs_info, ret);
1016 return ret;
1017 }
Yan Zheng5d4f98a2009-06-10 10:45:14 -04001018 } else {
1019 refs = 1;
1020 if (root->root_key.objectid == BTRFS_TREE_RELOC_OBJECTID ||
1021 btrfs_header_backref_rev(buf) < BTRFS_MIXED_BACKREF_REV)
1022 flags = BTRFS_BLOCK_FLAG_FULL_BACKREF;
1023 else
1024 flags = 0;
1025 }
1026
1027 owner = btrfs_header_owner(buf);
1028 BUG_ON(owner == BTRFS_TREE_RELOC_OBJECTID &&
1029 !(flags & BTRFS_BLOCK_FLAG_FULL_BACKREF));
1030
1031 if (refs > 1) {
1032 if ((owner == root->root_key.objectid ||
1033 root->root_key.objectid == BTRFS_TREE_RELOC_OBJECTID) &&
1034 !(flags & BTRFS_BLOCK_FLAG_FULL_BACKREF)) {
Josef Bacike339a6b2014-07-02 10:54:25 -07001035 ret = btrfs_inc_ref(trans, root, buf, 1);
Jeff Mahoney79787ea2012-03-12 16:03:00 +01001036 BUG_ON(ret); /* -ENOMEM */
Yan Zheng5d4f98a2009-06-10 10:45:14 -04001037
1038 if (root->root_key.objectid ==
1039 BTRFS_TREE_RELOC_OBJECTID) {
Josef Bacike339a6b2014-07-02 10:54:25 -07001040 ret = btrfs_dec_ref(trans, root, buf, 0);
Jeff Mahoney79787ea2012-03-12 16:03:00 +01001041 BUG_ON(ret); /* -ENOMEM */
Josef Bacike339a6b2014-07-02 10:54:25 -07001042 ret = btrfs_inc_ref(trans, root, cow, 1);
Jeff Mahoney79787ea2012-03-12 16:03:00 +01001043 BUG_ON(ret); /* -ENOMEM */
Yan Zheng5d4f98a2009-06-10 10:45:14 -04001044 }
1045 new_flags |= BTRFS_BLOCK_FLAG_FULL_BACKREF;
1046 } else {
1047
1048 if (root->root_key.objectid ==
1049 BTRFS_TREE_RELOC_OBJECTID)
Josef Bacike339a6b2014-07-02 10:54:25 -07001050 ret = btrfs_inc_ref(trans, root, cow, 1);
Yan Zheng5d4f98a2009-06-10 10:45:14 -04001051 else
Josef Bacike339a6b2014-07-02 10:54:25 -07001052 ret = btrfs_inc_ref(trans, root, cow, 0);
Jeff Mahoney79787ea2012-03-12 16:03:00 +01001053 BUG_ON(ret); /* -ENOMEM */
Yan Zheng5d4f98a2009-06-10 10:45:14 -04001054 }
1055 if (new_flags != 0) {
Josef Bacikb1c79e02013-05-09 13:49:30 -04001056 int level = btrfs_header_level(buf);
1057
Yan Zheng5d4f98a2009-06-10 10:45:14 -04001058 ret = btrfs_set_disk_extent_flags(trans, root,
1059 buf->start,
1060 buf->len,
Josef Bacikb1c79e02013-05-09 13:49:30 -04001061 new_flags, level, 0);
Mark Fashehbe1a5562011-08-08 13:20:18 -07001062 if (ret)
1063 return ret;
Yan Zheng5d4f98a2009-06-10 10:45:14 -04001064 }
1065 } else {
1066 if (flags & BTRFS_BLOCK_FLAG_FULL_BACKREF) {
1067 if (root->root_key.objectid ==
1068 BTRFS_TREE_RELOC_OBJECTID)
Josef Bacike339a6b2014-07-02 10:54:25 -07001069 ret = btrfs_inc_ref(trans, root, cow, 1);
Yan Zheng5d4f98a2009-06-10 10:45:14 -04001070 else
Josef Bacike339a6b2014-07-02 10:54:25 -07001071 ret = btrfs_inc_ref(trans, root, cow, 0);
Jeff Mahoney79787ea2012-03-12 16:03:00 +01001072 BUG_ON(ret); /* -ENOMEM */
Josef Bacike339a6b2014-07-02 10:54:25 -07001073 ret = btrfs_dec_ref(trans, root, buf, 1);
Jeff Mahoney79787ea2012-03-12 16:03:00 +01001074 BUG_ON(ret); /* -ENOMEM */
Yan Zheng5d4f98a2009-06-10 10:45:14 -04001075 }
1076 clean_tree_block(trans, root, buf);
Yan, Zhengf0486c62010-05-16 10:46:25 -04001077 *last_ref = 1;
Yan Zheng5d4f98a2009-06-10 10:45:14 -04001078 }
1079 return 0;
1080}
1081
1082/*
Chris Masond3977122009-01-05 21:25:51 -05001083 * does the dirty work in cow of a single block. The parent block (if
1084 * supplied) is updated to point to the new cow copy. The new buffer is marked
1085 * dirty and returned locked. If you modify the block it needs to be marked
1086 * dirty again.
Chris Masond352ac62008-09-29 15:18:18 -04001087 *
1088 * search_start -- an allocation hint for the new block
1089 *
Chris Masond3977122009-01-05 21:25:51 -05001090 * empty_size -- a hint that you plan on doing more cow. This is the size in
1091 * bytes the allocator should try to find free next to the block it returns.
1092 * This is just a hint and may be ignored by the allocator.
Chris Masond352ac62008-09-29 15:18:18 -04001093 */
Chris Masond3977122009-01-05 21:25:51 -05001094static noinline int __btrfs_cow_block(struct btrfs_trans_handle *trans,
Chris Mason5f39d392007-10-15 16:14:19 -04001095 struct btrfs_root *root,
1096 struct extent_buffer *buf,
1097 struct extent_buffer *parent, int parent_slot,
1098 struct extent_buffer **cow_ret,
Chris Mason9fa8cfe2009-03-13 10:24:59 -04001099 u64 search_start, u64 empty_size)
Chris Mason6702ed42007-08-07 16:15:09 -04001100{
Yan Zheng5d4f98a2009-06-10 10:45:14 -04001101 struct btrfs_disk_key disk_key;
Chris Mason5f39d392007-10-15 16:14:19 -04001102 struct extent_buffer *cow;
Mark Fashehbe1a5562011-08-08 13:20:18 -07001103 int level, ret;
Yan, Zhengf0486c62010-05-16 10:46:25 -04001104 int last_ref = 0;
Chris Mason925baed2008-06-25 16:01:30 -04001105 int unlock_orig = 0;
Yan Zheng5d4f98a2009-06-10 10:45:14 -04001106 u64 parent_start;
Chris Mason6702ed42007-08-07 16:15:09 -04001107
Chris Mason925baed2008-06-25 16:01:30 -04001108 if (*cow_ret == buf)
1109 unlock_orig = 1;
1110
Chris Masonb9447ef2009-03-09 11:45:38 -04001111 btrfs_assert_tree_locked(buf);
Chris Mason925baed2008-06-25 16:01:30 -04001112
Miao Xie27cdeb72014-04-02 19:51:05 +08001113 WARN_ON(test_bit(BTRFS_ROOT_REF_COWS, &root->state) &&
1114 trans->transid != root->fs_info->running_transaction->transid);
1115 WARN_ON(test_bit(BTRFS_ROOT_REF_COWS, &root->state) &&
1116 trans->transid != root->last_trans);
Chris Mason5f39d392007-10-15 16:14:19 -04001117
Chris Mason7bb86312007-12-11 09:25:06 -05001118 level = btrfs_header_level(buf);
Zheng Yan31840ae2008-09-23 13:14:14 -04001119
Yan Zheng5d4f98a2009-06-10 10:45:14 -04001120 if (level == 0)
1121 btrfs_item_key(buf, &disk_key, 0);
1122 else
1123 btrfs_node_key(buf, &disk_key, 0);
1124
1125 if (root->root_key.objectid == BTRFS_TREE_RELOC_OBJECTID) {
1126 if (parent)
1127 parent_start = parent->start;
1128 else
1129 parent_start = 0;
1130 } else
1131 parent_start = 0;
1132
David Sterba4d75f8a2014-06-15 01:54:12 +02001133 cow = btrfs_alloc_tree_block(trans, root, parent_start,
1134 root->root_key.objectid, &disk_key, level,
1135 search_start, empty_size);
Chris Mason6702ed42007-08-07 16:15:09 -04001136 if (IS_ERR(cow))
1137 return PTR_ERR(cow);
1138
Chris Masonb4ce94d2009-02-04 09:25:08 -05001139 /* cow is set to blocking by btrfs_init_new_buffer */
1140
Chris Mason5f39d392007-10-15 16:14:19 -04001141 copy_extent_buffer(cow, buf, 0, 0, cow->len);
Chris Masondb945352007-10-15 16:15:53 -04001142 btrfs_set_header_bytenr(cow, cow->start);
Chris Mason5f39d392007-10-15 16:14:19 -04001143 btrfs_set_header_generation(cow, trans->transid);
Yan Zheng5d4f98a2009-06-10 10:45:14 -04001144 btrfs_set_header_backref_rev(cow, BTRFS_MIXED_BACKREF_REV);
1145 btrfs_clear_header_flag(cow, BTRFS_HEADER_FLAG_WRITTEN |
1146 BTRFS_HEADER_FLAG_RELOC);
1147 if (root->root_key.objectid == BTRFS_TREE_RELOC_OBJECTID)
1148 btrfs_set_header_flag(cow, BTRFS_HEADER_FLAG_RELOC);
1149 else
1150 btrfs_set_header_owner(cow, root->root_key.objectid);
Chris Mason6702ed42007-08-07 16:15:09 -04001151
Ross Kirk0a4e5582013-09-24 10:12:38 +01001152 write_extent_buffer(cow, root->fs_info->fsid, btrfs_header_fsid(),
Yan Zheng2b820322008-11-17 21:11:30 -05001153 BTRFS_FSID_SIZE);
1154
Mark Fashehbe1a5562011-08-08 13:20:18 -07001155 ret = update_ref_for_cow(trans, root, buf, cow, &last_ref);
Mark Fashehb68dc2a2011-08-29 14:30:39 -07001156 if (ret) {
Jeff Mahoney79787ea2012-03-12 16:03:00 +01001157 btrfs_abort_transaction(trans, root, ret);
Mark Fashehb68dc2a2011-08-29 14:30:39 -07001158 return ret;
1159 }
Zheng Yan1a40e232008-09-26 10:09:34 -04001160
Miao Xie27cdeb72014-04-02 19:51:05 +08001161 if (test_bit(BTRFS_ROOT_REF_COWS, &root->state)) {
Josef Bacik83d4cfd2013-08-30 15:09:51 -04001162 ret = btrfs_reloc_cow_block(trans, root, buf, cow);
1163 if (ret)
1164 return ret;
1165 }
Yan, Zheng3fd0a552010-05-16 10:49:59 -04001166
Chris Mason6702ed42007-08-07 16:15:09 -04001167 if (buf == root->node) {
Chris Mason925baed2008-06-25 16:01:30 -04001168 WARN_ON(parent && parent != buf);
Yan Zheng5d4f98a2009-06-10 10:45:14 -04001169 if (root->root_key.objectid == BTRFS_TREE_RELOC_OBJECTID ||
1170 btrfs_header_backref_rev(buf) < BTRFS_MIXED_BACKREF_REV)
1171 parent_start = buf->start;
1172 else
1173 parent_start = 0;
Chris Mason925baed2008-06-25 16:01:30 -04001174
Chris Mason5f39d392007-10-15 16:14:19 -04001175 extent_buffer_get(cow);
Jan Schmidt90f8d622013-04-13 13:19:53 +00001176 tree_mod_log_set_root_pointer(root, cow, 1);
Chris Mason240f62c2011-03-23 14:54:42 -04001177 rcu_assign_pointer(root->node, cow);
Chris Mason925baed2008-06-25 16:01:30 -04001178
Yan, Zhengf0486c62010-05-16 10:46:25 -04001179 btrfs_free_tree_block(trans, root, buf, parent_start,
Jan Schmidt5581a512012-05-16 17:04:52 +02001180 last_ref);
Chris Mason5f39d392007-10-15 16:14:19 -04001181 free_extent_buffer(buf);
Chris Mason0b86a832008-03-24 15:01:56 -04001182 add_root_to_dirty_list(root);
Chris Mason6702ed42007-08-07 16:15:09 -04001183 } else {
Yan Zheng5d4f98a2009-06-10 10:45:14 -04001184 if (root->root_key.objectid == BTRFS_TREE_RELOC_OBJECTID)
1185 parent_start = parent->start;
1186 else
1187 parent_start = 0;
1188
1189 WARN_ON(trans->transid != btrfs_header_generation(parent));
Jan Schmidtf2304752012-05-26 11:43:17 +02001190 tree_mod_log_insert_key(root->fs_info, parent, parent_slot,
Josef Bacikc8cc6342013-07-01 16:18:19 -04001191 MOD_LOG_KEY_REPLACE, GFP_NOFS);
Chris Mason5f39d392007-10-15 16:14:19 -04001192 btrfs_set_node_blockptr(parent, parent_slot,
Chris Masondb945352007-10-15 16:15:53 -04001193 cow->start);
Chris Mason74493f72007-12-11 09:25:06 -05001194 btrfs_set_node_ptr_generation(parent, parent_slot,
1195 trans->transid);
Chris Mason6702ed42007-08-07 16:15:09 -04001196 btrfs_mark_buffer_dirty(parent);
Filipe David Borba Manana5de865e2013-12-20 15:17:46 +00001197 if (last_ref) {
1198 ret = tree_mod_log_free_eb(root->fs_info, buf);
1199 if (ret) {
1200 btrfs_abort_transaction(trans, root, ret);
1201 return ret;
1202 }
1203 }
Yan, Zhengf0486c62010-05-16 10:46:25 -04001204 btrfs_free_tree_block(trans, root, buf, parent_start,
Jan Schmidt5581a512012-05-16 17:04:52 +02001205 last_ref);
Chris Mason6702ed42007-08-07 16:15:09 -04001206 }
Chris Mason925baed2008-06-25 16:01:30 -04001207 if (unlock_orig)
1208 btrfs_tree_unlock(buf);
Josef Bacik3083ee22012-03-09 16:01:49 -05001209 free_extent_buffer_stale(buf);
Chris Mason6702ed42007-08-07 16:15:09 -04001210 btrfs_mark_buffer_dirty(cow);
1211 *cow_ret = cow;
1212 return 0;
1213}
1214
Jan Schmidt5d9e75c42012-05-16 18:25:47 +02001215/*
1216 * returns the logical address of the oldest predecessor of the given root.
1217 * entries older than time_seq are ignored.
1218 */
1219static struct tree_mod_elem *
1220__tree_mod_log_oldest_root(struct btrfs_fs_info *fs_info,
Jan Schmidt30b04632013-04-13 13:19:54 +00001221 struct extent_buffer *eb_root, u64 time_seq)
Jan Schmidt5d9e75c42012-05-16 18:25:47 +02001222{
1223 struct tree_mod_elem *tm;
1224 struct tree_mod_elem *found = NULL;
Jan Schmidt30b04632013-04-13 13:19:54 +00001225 u64 root_logical = eb_root->start;
Jan Schmidt5d9e75c42012-05-16 18:25:47 +02001226 int looped = 0;
1227
1228 if (!time_seq)
Stefan Behrens35a36212013-08-14 18:12:25 +02001229 return NULL;
Jan Schmidt5d9e75c42012-05-16 18:25:47 +02001230
1231 /*
1232 * the very last operation that's logged for a root is the replacement
1233 * operation (if it is replaced at all). this has the index of the *new*
1234 * root, making it the very first operation that's logged for this root.
1235 */
1236 while (1) {
1237 tm = tree_mod_log_search_oldest(fs_info, root_logical,
1238 time_seq);
1239 if (!looped && !tm)
Stefan Behrens35a36212013-08-14 18:12:25 +02001240 return NULL;
Jan Schmidt5d9e75c42012-05-16 18:25:47 +02001241 /*
Jan Schmidt28da9fb2012-06-21 10:59:13 +02001242 * if there are no tree operation for the oldest root, we simply
1243 * return it. this should only happen if that (old) root is at
1244 * level 0.
Jan Schmidt5d9e75c42012-05-16 18:25:47 +02001245 */
Jan Schmidt28da9fb2012-06-21 10:59:13 +02001246 if (!tm)
1247 break;
Jan Schmidt5d9e75c42012-05-16 18:25:47 +02001248
Jan Schmidt28da9fb2012-06-21 10:59:13 +02001249 /*
1250 * if there's an operation that's not a root replacement, we
1251 * found the oldest version of our root. normally, we'll find a
1252 * MOD_LOG_KEY_REMOVE_WHILE_FREEING operation here.
1253 */
Jan Schmidt5d9e75c42012-05-16 18:25:47 +02001254 if (tm->op != MOD_LOG_ROOT_REPLACE)
1255 break;
1256
1257 found = tm;
1258 root_logical = tm->old_root.logical;
Jan Schmidt5d9e75c42012-05-16 18:25:47 +02001259 looped = 1;
1260 }
1261
Jan Schmidta95236d2012-06-05 16:41:24 +02001262 /* if there's no old root to return, return what we found instead */
1263 if (!found)
1264 found = tm;
1265
Jan Schmidt5d9e75c42012-05-16 18:25:47 +02001266 return found;
1267}
1268
1269/*
1270 * tm is a pointer to the first operation to rewind within eb. then, all
1271 * previous operations will be rewinded (until we reach something older than
1272 * time_seq).
1273 */
1274static void
Josef Bacikf1ca7e982013-06-29 23:15:19 -04001275__tree_mod_log_rewind(struct btrfs_fs_info *fs_info, struct extent_buffer *eb,
1276 u64 time_seq, struct tree_mod_elem *first_tm)
Jan Schmidt5d9e75c42012-05-16 18:25:47 +02001277{
1278 u32 n;
1279 struct rb_node *next;
1280 struct tree_mod_elem *tm = first_tm;
1281 unsigned long o_dst;
1282 unsigned long o_src;
1283 unsigned long p_size = sizeof(struct btrfs_key_ptr);
1284
1285 n = btrfs_header_nritems(eb);
Josef Bacikf1ca7e982013-06-29 23:15:19 -04001286 tree_mod_log_read_lock(fs_info);
Jan Schmidt097b8a72012-06-21 11:08:04 +02001287 while (tm && tm->seq >= time_seq) {
Jan Schmidt5d9e75c42012-05-16 18:25:47 +02001288 /*
1289 * all the operations are recorded with the operator used for
1290 * the modification. as we're going backwards, we do the
1291 * opposite of each operation here.
1292 */
1293 switch (tm->op) {
1294 case MOD_LOG_KEY_REMOVE_WHILE_FREEING:
1295 BUG_ON(tm->slot < n);
Eric Sandeen1c697d42013-01-31 00:54:56 +00001296 /* Fallthrough */
Liu Bo95c80bb2012-10-19 09:50:52 +00001297 case MOD_LOG_KEY_REMOVE_WHILE_MOVING:
Chris Mason4c3e6962012-12-18 15:43:18 -05001298 case MOD_LOG_KEY_REMOVE:
Jan Schmidt5d9e75c42012-05-16 18:25:47 +02001299 btrfs_set_node_key(eb, &tm->key, tm->slot);
1300 btrfs_set_node_blockptr(eb, tm->slot, tm->blockptr);
1301 btrfs_set_node_ptr_generation(eb, tm->slot,
1302 tm->generation);
Chris Mason4c3e6962012-12-18 15:43:18 -05001303 n++;
Jan Schmidt5d9e75c42012-05-16 18:25:47 +02001304 break;
1305 case MOD_LOG_KEY_REPLACE:
1306 BUG_ON(tm->slot >= n);
1307 btrfs_set_node_key(eb, &tm->key, tm->slot);
1308 btrfs_set_node_blockptr(eb, tm->slot, tm->blockptr);
1309 btrfs_set_node_ptr_generation(eb, tm->slot,
1310 tm->generation);
1311 break;
1312 case MOD_LOG_KEY_ADD:
Jan Schmidt19956c72012-06-22 14:52:13 +02001313 /* if a move operation is needed it's in the log */
Jan Schmidt5d9e75c42012-05-16 18:25:47 +02001314 n--;
1315 break;
1316 case MOD_LOG_MOVE_KEYS:
Jan Schmidtc3193102012-05-31 19:24:36 +02001317 o_dst = btrfs_node_key_ptr_offset(tm->slot);
1318 o_src = btrfs_node_key_ptr_offset(tm->move.dst_slot);
1319 memmove_extent_buffer(eb, o_dst, o_src,
Jan Schmidt5d9e75c42012-05-16 18:25:47 +02001320 tm->move.nr_items * p_size);
1321 break;
1322 case MOD_LOG_ROOT_REPLACE:
1323 /*
1324 * this operation is special. for roots, this must be
1325 * handled explicitly before rewinding.
1326 * for non-roots, this operation may exist if the node
1327 * was a root: root A -> child B; then A gets empty and
1328 * B is promoted to the new root. in the mod log, we'll
1329 * have a root-replace operation for B, a tree block
1330 * that is no root. we simply ignore that operation.
1331 */
1332 break;
1333 }
1334 next = rb_next(&tm->node);
1335 if (!next)
1336 break;
1337 tm = container_of(next, struct tree_mod_elem, node);
1338 if (tm->index != first_tm->index)
1339 break;
1340 }
Josef Bacikf1ca7e982013-06-29 23:15:19 -04001341 tree_mod_log_read_unlock(fs_info);
Jan Schmidt5d9e75c42012-05-16 18:25:47 +02001342 btrfs_set_header_nritems(eb, n);
1343}
1344
Jan Schmidt47fb0912013-04-13 13:19:55 +00001345/*
1346 * Called with eb read locked. If the buffer cannot be rewinded, the same buffer
1347 * is returned. If rewind operations happen, a fresh buffer is returned. The
1348 * returned buffer is always read-locked. If the returned buffer is not the
1349 * input buffer, the lock on the input buffer is released and the input buffer
1350 * is freed (its refcount is decremented).
1351 */
Jan Schmidt5d9e75c42012-05-16 18:25:47 +02001352static struct extent_buffer *
Josef Bacik9ec72672013-08-07 16:57:23 -04001353tree_mod_log_rewind(struct btrfs_fs_info *fs_info, struct btrfs_path *path,
1354 struct extent_buffer *eb, u64 time_seq)
Jan Schmidt5d9e75c42012-05-16 18:25:47 +02001355{
1356 struct extent_buffer *eb_rewin;
1357 struct tree_mod_elem *tm;
1358
1359 if (!time_seq)
1360 return eb;
1361
1362 if (btrfs_header_level(eb) == 0)
1363 return eb;
1364
1365 tm = tree_mod_log_search(fs_info, eb->start, time_seq);
1366 if (!tm)
1367 return eb;
1368
Josef Bacik9ec72672013-08-07 16:57:23 -04001369 btrfs_set_path_blocking(path);
1370 btrfs_set_lock_blocking_rw(eb, BTRFS_READ_LOCK);
1371
Jan Schmidt5d9e75c42012-05-16 18:25:47 +02001372 if (tm->op == MOD_LOG_KEY_REMOVE_WHILE_FREEING) {
1373 BUG_ON(tm->slot != 0);
David Sterba3f556f72014-06-15 03:20:26 +02001374 eb_rewin = alloc_dummy_extent_buffer(fs_info, eb->start);
Josef Bacikdb7f3432013-08-07 14:54:37 -04001375 if (!eb_rewin) {
Josef Bacik9ec72672013-08-07 16:57:23 -04001376 btrfs_tree_read_unlock_blocking(eb);
Josef Bacikdb7f3432013-08-07 14:54:37 -04001377 free_extent_buffer(eb);
1378 return NULL;
1379 }
Jan Schmidt5d9e75c42012-05-16 18:25:47 +02001380 btrfs_set_header_bytenr(eb_rewin, eb->start);
1381 btrfs_set_header_backref_rev(eb_rewin,
1382 btrfs_header_backref_rev(eb));
1383 btrfs_set_header_owner(eb_rewin, btrfs_header_owner(eb));
Jan Schmidtc3193102012-05-31 19:24:36 +02001384 btrfs_set_header_level(eb_rewin, btrfs_header_level(eb));
Jan Schmidt5d9e75c42012-05-16 18:25:47 +02001385 } else {
1386 eb_rewin = btrfs_clone_extent_buffer(eb);
Josef Bacikdb7f3432013-08-07 14:54:37 -04001387 if (!eb_rewin) {
Josef Bacik9ec72672013-08-07 16:57:23 -04001388 btrfs_tree_read_unlock_blocking(eb);
Josef Bacikdb7f3432013-08-07 14:54:37 -04001389 free_extent_buffer(eb);
1390 return NULL;
1391 }
Jan Schmidt5d9e75c42012-05-16 18:25:47 +02001392 }
1393
Josef Bacik9ec72672013-08-07 16:57:23 -04001394 btrfs_clear_path_blocking(path, NULL, BTRFS_READ_LOCK);
1395 btrfs_tree_read_unlock_blocking(eb);
Jan Schmidt5d9e75c42012-05-16 18:25:47 +02001396 free_extent_buffer(eb);
1397
Jan Schmidt47fb0912013-04-13 13:19:55 +00001398 extent_buffer_get(eb_rewin);
1399 btrfs_tree_read_lock(eb_rewin);
Josef Bacikf1ca7e982013-06-29 23:15:19 -04001400 __tree_mod_log_rewind(fs_info, eb_rewin, time_seq, tm);
Jan Schmidt57911b82012-10-19 09:22:03 +02001401 WARN_ON(btrfs_header_nritems(eb_rewin) >
Arne Jansen2a745b12013-02-13 04:20:01 -07001402 BTRFS_NODEPTRS_PER_BLOCK(fs_info->tree_root));
Jan Schmidt5d9e75c42012-05-16 18:25:47 +02001403
1404 return eb_rewin;
1405}
1406
Jan Schmidt8ba97a12012-06-04 16:54:57 +02001407/*
1408 * get_old_root() rewinds the state of @root's root node to the given @time_seq
1409 * value. If there are no changes, the current root->root_node is returned. If
1410 * anything changed in between, there's a fresh buffer allocated on which the
1411 * rewind operations are done. In any case, the returned buffer is read locked.
1412 * Returns NULL on error (with no locks held).
1413 */
Jan Schmidt5d9e75c42012-05-16 18:25:47 +02001414static inline struct extent_buffer *
1415get_old_root(struct btrfs_root *root, u64 time_seq)
1416{
1417 struct tree_mod_elem *tm;
Jan Schmidt30b04632013-04-13 13:19:54 +00001418 struct extent_buffer *eb = NULL;
1419 struct extent_buffer *eb_root;
Liu Bo7bfdcf72012-10-25 07:30:19 -06001420 struct extent_buffer *old;
Jan Schmidta95236d2012-06-05 16:41:24 +02001421 struct tree_mod_root *old_root = NULL;
Chris Mason4325edd2012-06-15 20:02:02 -04001422 u64 old_generation = 0;
Jan Schmidta95236d2012-06-05 16:41:24 +02001423 u64 logical;
Jan Schmidt5d9e75c42012-05-16 18:25:47 +02001424
Jan Schmidt30b04632013-04-13 13:19:54 +00001425 eb_root = btrfs_read_lock_root_node(root);
1426 tm = __tree_mod_log_oldest_root(root->fs_info, eb_root, time_seq);
Jan Schmidt5d9e75c42012-05-16 18:25:47 +02001427 if (!tm)
Jan Schmidt30b04632013-04-13 13:19:54 +00001428 return eb_root;
Jan Schmidt5d9e75c42012-05-16 18:25:47 +02001429
Jan Schmidta95236d2012-06-05 16:41:24 +02001430 if (tm->op == MOD_LOG_ROOT_REPLACE) {
1431 old_root = &tm->old_root;
1432 old_generation = tm->generation;
1433 logical = old_root->logical;
1434 } else {
Jan Schmidt30b04632013-04-13 13:19:54 +00001435 logical = eb_root->start;
Jan Schmidta95236d2012-06-05 16:41:24 +02001436 }
Jan Schmidt5d9e75c42012-05-16 18:25:47 +02001437
Jan Schmidta95236d2012-06-05 16:41:24 +02001438 tm = tree_mod_log_search(root->fs_info, logical, time_seq);
Jan Schmidt834328a2012-10-23 11:27:33 +02001439 if (old_root && tm && tm->op != MOD_LOG_KEY_REMOVE_WHILE_FREEING) {
Jan Schmidt30b04632013-04-13 13:19:54 +00001440 btrfs_tree_read_unlock(eb_root);
1441 free_extent_buffer(eb_root);
David Sterbace86cd52014-06-15 01:07:32 +02001442 old = read_tree_block(root, logical, 0);
Dulshani Gunawardhanafae7f212013-10-31 10:30:08 +05301443 if (WARN_ON(!old || !extent_buffer_uptodate(old))) {
Josef Bacik416bc652013-04-23 14:17:42 -04001444 free_extent_buffer(old);
Frank Holtonefe120a2013-12-20 11:37:06 -05001445 btrfs_warn(root->fs_info,
1446 "failed to read tree block %llu from get_old_root", logical);
Jan Schmidt834328a2012-10-23 11:27:33 +02001447 } else {
Liu Bo7bfdcf72012-10-25 07:30:19 -06001448 eb = btrfs_clone_extent_buffer(old);
1449 free_extent_buffer(old);
Jan Schmidt834328a2012-10-23 11:27:33 +02001450 }
1451 } else if (old_root) {
Jan Schmidt30b04632013-04-13 13:19:54 +00001452 btrfs_tree_read_unlock(eb_root);
1453 free_extent_buffer(eb_root);
David Sterba3f556f72014-06-15 03:20:26 +02001454 eb = alloc_dummy_extent_buffer(root->fs_info, logical);
Jan Schmidt834328a2012-10-23 11:27:33 +02001455 } else {
Josef Bacik9ec72672013-08-07 16:57:23 -04001456 btrfs_set_lock_blocking_rw(eb_root, BTRFS_READ_LOCK);
Jan Schmidt30b04632013-04-13 13:19:54 +00001457 eb = btrfs_clone_extent_buffer(eb_root);
Josef Bacik9ec72672013-08-07 16:57:23 -04001458 btrfs_tree_read_unlock_blocking(eb_root);
Jan Schmidt30b04632013-04-13 13:19:54 +00001459 free_extent_buffer(eb_root);
Jan Schmidt834328a2012-10-23 11:27:33 +02001460 }
1461
Jan Schmidt8ba97a12012-06-04 16:54:57 +02001462 if (!eb)
1463 return NULL;
Jan Schmidtd6381082012-10-23 14:21:05 +02001464 extent_buffer_get(eb);
Jan Schmidt8ba97a12012-06-04 16:54:57 +02001465 btrfs_tree_read_lock(eb);
Jan Schmidta95236d2012-06-05 16:41:24 +02001466 if (old_root) {
Jan Schmidt5d9e75c42012-05-16 18:25:47 +02001467 btrfs_set_header_bytenr(eb, eb->start);
1468 btrfs_set_header_backref_rev(eb, BTRFS_MIXED_BACKREF_REV);
Jan Schmidt30b04632013-04-13 13:19:54 +00001469 btrfs_set_header_owner(eb, btrfs_header_owner(eb_root));
Jan Schmidta95236d2012-06-05 16:41:24 +02001470 btrfs_set_header_level(eb, old_root->level);
1471 btrfs_set_header_generation(eb, old_generation);
Jan Schmidt5d9e75c42012-05-16 18:25:47 +02001472 }
Jan Schmidt28da9fb2012-06-21 10:59:13 +02001473 if (tm)
Josef Bacikf1ca7e982013-06-29 23:15:19 -04001474 __tree_mod_log_rewind(root->fs_info, eb, time_seq, tm);
Jan Schmidt28da9fb2012-06-21 10:59:13 +02001475 else
1476 WARN_ON(btrfs_header_level(eb) != 0);
Jan Schmidt57911b82012-10-19 09:22:03 +02001477 WARN_ON(btrfs_header_nritems(eb) > BTRFS_NODEPTRS_PER_BLOCK(root));
Jan Schmidt5d9e75c42012-05-16 18:25:47 +02001478
1479 return eb;
1480}
1481
Jan Schmidt5b6602e2012-10-23 11:28:27 +02001482int btrfs_old_root_level(struct btrfs_root *root, u64 time_seq)
1483{
1484 struct tree_mod_elem *tm;
1485 int level;
Jan Schmidt30b04632013-04-13 13:19:54 +00001486 struct extent_buffer *eb_root = btrfs_root_node(root);
Jan Schmidt5b6602e2012-10-23 11:28:27 +02001487
Jan Schmidt30b04632013-04-13 13:19:54 +00001488 tm = __tree_mod_log_oldest_root(root->fs_info, eb_root, time_seq);
Jan Schmidt5b6602e2012-10-23 11:28:27 +02001489 if (tm && tm->op == MOD_LOG_ROOT_REPLACE) {
1490 level = tm->old_root.level;
1491 } else {
Jan Schmidt30b04632013-04-13 13:19:54 +00001492 level = btrfs_header_level(eb_root);
Jan Schmidt5b6602e2012-10-23 11:28:27 +02001493 }
Jan Schmidt30b04632013-04-13 13:19:54 +00001494 free_extent_buffer(eb_root);
Jan Schmidt5b6602e2012-10-23 11:28:27 +02001495
1496 return level;
1497}
1498
Yan Zheng5d4f98a2009-06-10 10:45:14 -04001499static inline int should_cow_block(struct btrfs_trans_handle *trans,
1500 struct btrfs_root *root,
1501 struct extent_buffer *buf)
1502{
David Sterbafccb84c2014-09-29 23:53:21 +02001503 if (btrfs_test_is_dummy_root(root))
Josef Bacikfaa2dbf2014-05-07 17:06:09 -04001504 return 0;
David Sterbafccb84c2014-09-29 23:53:21 +02001505
Liu Bof1ebcc72011-11-14 20:48:06 -05001506 /* ensure we can see the force_cow */
1507 smp_rmb();
1508
1509 /*
1510 * We do not need to cow a block if
1511 * 1) this block is not created or changed in this transaction;
1512 * 2) this block does not belong to TREE_RELOC tree;
1513 * 3) the root is not forced COW.
1514 *
1515 * What is forced COW:
1516 * when we create snapshot during commiting the transaction,
1517 * after we've finished coping src root, we must COW the shared
1518 * block to ensure the metadata consistency.
1519 */
Yan Zheng5d4f98a2009-06-10 10:45:14 -04001520 if (btrfs_header_generation(buf) == trans->transid &&
1521 !btrfs_header_flag(buf, BTRFS_HEADER_FLAG_WRITTEN) &&
1522 !(root->root_key.objectid != BTRFS_TREE_RELOC_OBJECTID &&
Liu Bof1ebcc72011-11-14 20:48:06 -05001523 btrfs_header_flag(buf, BTRFS_HEADER_FLAG_RELOC)) &&
Miao Xie27cdeb72014-04-02 19:51:05 +08001524 !test_bit(BTRFS_ROOT_FORCE_COW, &root->state))
Yan Zheng5d4f98a2009-06-10 10:45:14 -04001525 return 0;
1526 return 1;
1527}
1528
Chris Masond352ac62008-09-29 15:18:18 -04001529/*
1530 * cows a single block, see __btrfs_cow_block for the real work.
1531 * This version of it has extra checks so that a block isn't cow'd more than
1532 * once per transaction, as long as it hasn't been written yet
1533 */
Chris Masond3977122009-01-05 21:25:51 -05001534noinline int btrfs_cow_block(struct btrfs_trans_handle *trans,
Chris Mason5f39d392007-10-15 16:14:19 -04001535 struct btrfs_root *root, struct extent_buffer *buf,
1536 struct extent_buffer *parent, int parent_slot,
Chris Mason9fa8cfe2009-03-13 10:24:59 -04001537 struct extent_buffer **cow_ret)
Chris Mason02217ed2007-03-02 16:08:05 -05001538{
Chris Mason6702ed42007-08-07 16:15:09 -04001539 u64 search_start;
Chris Masonf510cfe2007-10-15 16:14:48 -04001540 int ret;
Chris Masondc17ff82008-01-08 15:46:30 -05001541
Julia Lawall31b1a2b2012-11-03 10:58:34 +00001542 if (trans->transaction != root->fs_info->running_transaction)
1543 WARN(1, KERN_CRIT "trans %llu running %llu\n",
Geert Uytterhoevenc1c9ff72013-08-20 13:20:07 +02001544 trans->transid,
Chris Masonccd467d2007-06-28 15:57:36 -04001545 root->fs_info->running_transaction->transid);
Julia Lawall31b1a2b2012-11-03 10:58:34 +00001546
1547 if (trans->transid != root->fs_info->generation)
1548 WARN(1, KERN_CRIT "trans %llu running %llu\n",
Geert Uytterhoevenc1c9ff72013-08-20 13:20:07 +02001549 trans->transid, root->fs_info->generation);
Chris Masondc17ff82008-01-08 15:46:30 -05001550
Yan Zheng5d4f98a2009-06-10 10:45:14 -04001551 if (!should_cow_block(trans, root, buf)) {
Chris Mason02217ed2007-03-02 16:08:05 -05001552 *cow_ret = buf;
1553 return 0;
1554 }
Chris Masonc4876852009-02-04 09:24:25 -05001555
Chris Mason0b86a832008-03-24 15:01:56 -04001556 search_start = buf->start & ~((u64)(1024 * 1024 * 1024) - 1);
Chris Masonb4ce94d2009-02-04 09:25:08 -05001557
1558 if (parent)
1559 btrfs_set_lock_blocking(parent);
1560 btrfs_set_lock_blocking(buf);
1561
Chris Masonf510cfe2007-10-15 16:14:48 -04001562 ret = __btrfs_cow_block(trans, root, buf, parent,
Chris Mason9fa8cfe2009-03-13 10:24:59 -04001563 parent_slot, cow_ret, search_start, 0);
liubo1abe9b82011-03-24 11:18:59 +00001564
1565 trace_btrfs_cow_block(root, buf, *cow_ret);
1566
Chris Masonf510cfe2007-10-15 16:14:48 -04001567 return ret;
Chris Mason6702ed42007-08-07 16:15:09 -04001568}
1569
Chris Masond352ac62008-09-29 15:18:18 -04001570/*
1571 * helper function for defrag to decide if two blocks pointed to by a
1572 * node are actually close by
1573 */
Chris Mason6b800532007-10-15 16:17:34 -04001574static int close_blocks(u64 blocknr, u64 other, u32 blocksize)
Chris Mason6702ed42007-08-07 16:15:09 -04001575{
Chris Mason6b800532007-10-15 16:17:34 -04001576 if (blocknr < other && other - (blocknr + blocksize) < 32768)
Chris Mason6702ed42007-08-07 16:15:09 -04001577 return 1;
Chris Mason6b800532007-10-15 16:17:34 -04001578 if (blocknr > other && blocknr - (other + blocksize) < 32768)
Chris Mason6702ed42007-08-07 16:15:09 -04001579 return 1;
Chris Mason02217ed2007-03-02 16:08:05 -05001580 return 0;
1581}
1582
Chris Mason081e9572007-11-06 10:26:24 -05001583/*
1584 * compare two keys in a memcmp fashion
1585 */
1586static int comp_keys(struct btrfs_disk_key *disk, struct btrfs_key *k2)
1587{
1588 struct btrfs_key k1;
1589
1590 btrfs_disk_key_to_cpu(&k1, disk);
1591
Diego Calleja20736ab2009-07-24 11:06:52 -04001592 return btrfs_comp_cpu_keys(&k1, k2);
Chris Mason081e9572007-11-06 10:26:24 -05001593}
1594
Josef Bacikf3465ca2008-11-12 14:19:50 -05001595/*
1596 * same as comp_keys only with two btrfs_key's
1597 */
Yan Zheng5d4f98a2009-06-10 10:45:14 -04001598int btrfs_comp_cpu_keys(struct btrfs_key *k1, struct btrfs_key *k2)
Josef Bacikf3465ca2008-11-12 14:19:50 -05001599{
1600 if (k1->objectid > k2->objectid)
1601 return 1;
1602 if (k1->objectid < k2->objectid)
1603 return -1;
1604 if (k1->type > k2->type)
1605 return 1;
1606 if (k1->type < k2->type)
1607 return -1;
1608 if (k1->offset > k2->offset)
1609 return 1;
1610 if (k1->offset < k2->offset)
1611 return -1;
1612 return 0;
1613}
Chris Mason081e9572007-11-06 10:26:24 -05001614
Chris Masond352ac62008-09-29 15:18:18 -04001615/*
1616 * this is used by the defrag code to go through all the
1617 * leaves pointed to by a node and reallocate them so that
1618 * disk order is close to key order
1619 */
Chris Mason6702ed42007-08-07 16:15:09 -04001620int btrfs_realloc_node(struct btrfs_trans_handle *trans,
Chris Mason5f39d392007-10-15 16:14:19 -04001621 struct btrfs_root *root, struct extent_buffer *parent,
Eric Sandeende78b512013-01-31 18:21:12 +00001622 int start_slot, u64 *last_ret,
Chris Masona6b6e752007-10-15 16:22:39 -04001623 struct btrfs_key *progress)
Chris Mason6702ed42007-08-07 16:15:09 -04001624{
Chris Mason6b800532007-10-15 16:17:34 -04001625 struct extent_buffer *cur;
Chris Mason6702ed42007-08-07 16:15:09 -04001626 u64 blocknr;
Chris Masonca7a79a2008-05-12 12:59:19 -04001627 u64 gen;
Chris Masone9d0b132007-08-10 14:06:19 -04001628 u64 search_start = *last_ret;
1629 u64 last_block = 0;
Chris Mason6702ed42007-08-07 16:15:09 -04001630 u64 other;
1631 u32 parent_nritems;
Chris Mason6702ed42007-08-07 16:15:09 -04001632 int end_slot;
1633 int i;
1634 int err = 0;
Chris Masonf2183bd2007-08-10 14:42:37 -04001635 int parent_level;
Chris Mason6b800532007-10-15 16:17:34 -04001636 int uptodate;
1637 u32 blocksize;
Chris Mason081e9572007-11-06 10:26:24 -05001638 int progress_passed = 0;
1639 struct btrfs_disk_key disk_key;
Chris Mason6702ed42007-08-07 16:15:09 -04001640
Chris Mason5708b952007-10-25 15:43:18 -04001641 parent_level = btrfs_header_level(parent);
Chris Mason5708b952007-10-25 15:43:18 -04001642
Julia Lawall6c1500f2012-11-03 20:30:18 +00001643 WARN_ON(trans->transaction != root->fs_info->running_transaction);
1644 WARN_ON(trans->transid != root->fs_info->generation);
Chris Mason86479a02007-09-10 19:58:16 -04001645
Chris Mason6b800532007-10-15 16:17:34 -04001646 parent_nritems = btrfs_header_nritems(parent);
David Sterba707e8a02014-06-04 19:22:26 +02001647 blocksize = root->nodesize;
Chris Mason6702ed42007-08-07 16:15:09 -04001648 end_slot = parent_nritems;
1649
1650 if (parent_nritems == 1)
1651 return 0;
1652
Chris Masonb4ce94d2009-02-04 09:25:08 -05001653 btrfs_set_lock_blocking(parent);
1654
Chris Mason6702ed42007-08-07 16:15:09 -04001655 for (i = start_slot; i < end_slot; i++) {
1656 int close = 1;
Chris Masona6b6e752007-10-15 16:22:39 -04001657
Chris Mason081e9572007-11-06 10:26:24 -05001658 btrfs_node_key(parent, &disk_key, i);
1659 if (!progress_passed && comp_keys(&disk_key, progress) < 0)
1660 continue;
1661
1662 progress_passed = 1;
Chris Mason6b800532007-10-15 16:17:34 -04001663 blocknr = btrfs_node_blockptr(parent, i);
Chris Masonca7a79a2008-05-12 12:59:19 -04001664 gen = btrfs_node_ptr_generation(parent, i);
Chris Masone9d0b132007-08-10 14:06:19 -04001665 if (last_block == 0)
1666 last_block = blocknr;
Chris Mason5708b952007-10-25 15:43:18 -04001667
Chris Mason6702ed42007-08-07 16:15:09 -04001668 if (i > 0) {
Chris Mason6b800532007-10-15 16:17:34 -04001669 other = btrfs_node_blockptr(parent, i - 1);
1670 close = close_blocks(blocknr, other, blocksize);
Chris Mason6702ed42007-08-07 16:15:09 -04001671 }
Chris Mason0ef3e662008-05-24 14:04:53 -04001672 if (!close && i < end_slot - 2) {
Chris Mason6b800532007-10-15 16:17:34 -04001673 other = btrfs_node_blockptr(parent, i + 1);
1674 close = close_blocks(blocknr, other, blocksize);
Chris Mason6702ed42007-08-07 16:15:09 -04001675 }
Chris Masone9d0b132007-08-10 14:06:19 -04001676 if (close) {
1677 last_block = blocknr;
Chris Mason6702ed42007-08-07 16:15:09 -04001678 continue;
Chris Masone9d0b132007-08-10 14:06:19 -04001679 }
Chris Mason6702ed42007-08-07 16:15:09 -04001680
David Sterba0308af42014-06-15 01:43:40 +02001681 cur = btrfs_find_tree_block(root, blocknr);
Chris Mason6b800532007-10-15 16:17:34 -04001682 if (cur)
Chris Masonb9fab912012-05-06 07:23:47 -04001683 uptodate = btrfs_buffer_uptodate(cur, gen, 0);
Chris Mason6b800532007-10-15 16:17:34 -04001684 else
1685 uptodate = 0;
Chris Mason5708b952007-10-25 15:43:18 -04001686 if (!cur || !uptodate) {
Chris Mason6b800532007-10-15 16:17:34 -04001687 if (!cur) {
David Sterbace86cd52014-06-15 01:07:32 +02001688 cur = read_tree_block(root, blocknr, gen);
Josef Bacik416bc652013-04-23 14:17:42 -04001689 if (!cur || !extent_buffer_uptodate(cur)) {
1690 free_extent_buffer(cur);
Tsutomu Itoh97d9a8a2011-03-24 06:33:21 +00001691 return -EIO;
Josef Bacik416bc652013-04-23 14:17:42 -04001692 }
Chris Mason6b800532007-10-15 16:17:34 -04001693 } else if (!uptodate) {
Tsutomu Itoh018642a2012-05-29 18:10:13 +09001694 err = btrfs_read_buffer(cur, gen);
1695 if (err) {
1696 free_extent_buffer(cur);
1697 return err;
1698 }
Chris Masonf2183bd2007-08-10 14:42:37 -04001699 }
Chris Mason6702ed42007-08-07 16:15:09 -04001700 }
Chris Masone9d0b132007-08-10 14:06:19 -04001701 if (search_start == 0)
Chris Mason6b800532007-10-15 16:17:34 -04001702 search_start = last_block;
Chris Masone9d0b132007-08-10 14:06:19 -04001703
Chris Masone7a84562008-06-25 16:01:31 -04001704 btrfs_tree_lock(cur);
Chris Masonb4ce94d2009-02-04 09:25:08 -05001705 btrfs_set_lock_blocking(cur);
Chris Mason6b800532007-10-15 16:17:34 -04001706 err = __btrfs_cow_block(trans, root, cur, parent, i,
Chris Masone7a84562008-06-25 16:01:31 -04001707 &cur, search_start,
Chris Mason6b800532007-10-15 16:17:34 -04001708 min(16 * blocksize,
Chris Mason9fa8cfe2009-03-13 10:24:59 -04001709 (end_slot - i) * blocksize));
Yan252c38f2007-08-29 09:11:44 -04001710 if (err) {
Chris Masone7a84562008-06-25 16:01:31 -04001711 btrfs_tree_unlock(cur);
Chris Mason6b800532007-10-15 16:17:34 -04001712 free_extent_buffer(cur);
Chris Mason6702ed42007-08-07 16:15:09 -04001713 break;
Yan252c38f2007-08-29 09:11:44 -04001714 }
Chris Masone7a84562008-06-25 16:01:31 -04001715 search_start = cur->start;
1716 last_block = cur->start;
Chris Masonf2183bd2007-08-10 14:42:37 -04001717 *last_ret = search_start;
Chris Masone7a84562008-06-25 16:01:31 -04001718 btrfs_tree_unlock(cur);
1719 free_extent_buffer(cur);
Chris Mason6702ed42007-08-07 16:15:09 -04001720 }
1721 return err;
1722}
1723
Chris Mason74123bd2007-02-02 11:05:29 -05001724/*
1725 * The leaf data grows from end-to-front in the node.
1726 * this returns the address of the start of the last item,
1727 * which is the stop of the leaf data stack
1728 */
Chris Mason123abc82007-03-14 14:14:43 -04001729static inline unsigned int leaf_data_end(struct btrfs_root *root,
Chris Mason5f39d392007-10-15 16:14:19 -04001730 struct extent_buffer *leaf)
Chris Masonbe0e5c02007-01-26 15:51:26 -05001731{
Chris Mason5f39d392007-10-15 16:14:19 -04001732 u32 nr = btrfs_header_nritems(leaf);
Chris Masonbe0e5c02007-01-26 15:51:26 -05001733 if (nr == 0)
Chris Mason123abc82007-03-14 14:14:43 -04001734 return BTRFS_LEAF_DATA_SIZE(root);
Chris Mason5f39d392007-10-15 16:14:19 -04001735 return btrfs_item_offset_nr(leaf, nr - 1);
Chris Masonbe0e5c02007-01-26 15:51:26 -05001736}
1737
Chris Masonaa5d6be2007-02-28 16:35:06 -05001738
Chris Mason74123bd2007-02-02 11:05:29 -05001739/*
Chris Mason5f39d392007-10-15 16:14:19 -04001740 * search for key in the extent_buffer. The items start at offset p,
1741 * and they are item_size apart. There are 'max' items in p.
1742 *
Chris Mason74123bd2007-02-02 11:05:29 -05001743 * the slot in the array is returned via slot, and it points to
1744 * the place where you would insert key if it is not found in
1745 * the array.
1746 *
1747 * slot may point to max if the key is bigger than all of the keys
1748 */
Chris Masone02119d2008-09-05 16:13:11 -04001749static noinline int generic_bin_search(struct extent_buffer *eb,
1750 unsigned long p,
1751 int item_size, struct btrfs_key *key,
1752 int max, int *slot)
Chris Masonbe0e5c02007-01-26 15:51:26 -05001753{
1754 int low = 0;
1755 int high = max;
1756 int mid;
1757 int ret;
Chris Mason479965d2007-10-15 16:14:27 -04001758 struct btrfs_disk_key *tmp = NULL;
Chris Mason5f39d392007-10-15 16:14:19 -04001759 struct btrfs_disk_key unaligned;
1760 unsigned long offset;
Chris Mason5f39d392007-10-15 16:14:19 -04001761 char *kaddr = NULL;
1762 unsigned long map_start = 0;
1763 unsigned long map_len = 0;
Chris Mason479965d2007-10-15 16:14:27 -04001764 int err;
Chris Masonbe0e5c02007-01-26 15:51:26 -05001765
Chris Masond3977122009-01-05 21:25:51 -05001766 while (low < high) {
Chris Masonbe0e5c02007-01-26 15:51:26 -05001767 mid = (low + high) / 2;
Chris Mason5f39d392007-10-15 16:14:19 -04001768 offset = p + mid * item_size;
1769
Chris Masona6591712011-07-19 12:04:14 -04001770 if (!kaddr || offset < map_start ||
Chris Mason5f39d392007-10-15 16:14:19 -04001771 (offset + sizeof(struct btrfs_disk_key)) >
1772 map_start + map_len) {
Chris Mason934d3752008-12-08 16:43:10 -05001773
1774 err = map_private_extent_buffer(eb, offset,
Chris Mason479965d2007-10-15 16:14:27 -04001775 sizeof(struct btrfs_disk_key),
Chris Masona6591712011-07-19 12:04:14 -04001776 &kaddr, &map_start, &map_len);
Chris Mason5f39d392007-10-15 16:14:19 -04001777
Chris Mason479965d2007-10-15 16:14:27 -04001778 if (!err) {
1779 tmp = (struct btrfs_disk_key *)(kaddr + offset -
1780 map_start);
1781 } else {
1782 read_extent_buffer(eb, &unaligned,
1783 offset, sizeof(unaligned));
1784 tmp = &unaligned;
1785 }
1786
Chris Mason5f39d392007-10-15 16:14:19 -04001787 } else {
1788 tmp = (struct btrfs_disk_key *)(kaddr + offset -
1789 map_start);
1790 }
Chris Masonbe0e5c02007-01-26 15:51:26 -05001791 ret = comp_keys(tmp, key);
1792
1793 if (ret < 0)
1794 low = mid + 1;
1795 else if (ret > 0)
1796 high = mid;
1797 else {
1798 *slot = mid;
1799 return 0;
1800 }
1801 }
1802 *slot = low;
1803 return 1;
1804}
1805
Chris Mason97571fd2007-02-24 13:39:08 -05001806/*
1807 * simple bin_search frontend that does the right thing for
1808 * leaves vs nodes
1809 */
Chris Mason5f39d392007-10-15 16:14:19 -04001810static int bin_search(struct extent_buffer *eb, struct btrfs_key *key,
1811 int level, int *slot)
Chris Masonbe0e5c02007-01-26 15:51:26 -05001812{
Wang Sheng-Huif7757382012-03-30 15:14:27 +08001813 if (level == 0)
Chris Mason5f39d392007-10-15 16:14:19 -04001814 return generic_bin_search(eb,
1815 offsetof(struct btrfs_leaf, items),
Chris Mason0783fcf2007-03-12 20:12:07 -04001816 sizeof(struct btrfs_item),
Chris Mason5f39d392007-10-15 16:14:19 -04001817 key, btrfs_header_nritems(eb),
Chris Mason7518a232007-03-12 12:01:18 -04001818 slot);
Wang Sheng-Huif7757382012-03-30 15:14:27 +08001819 else
Chris Mason5f39d392007-10-15 16:14:19 -04001820 return generic_bin_search(eb,
1821 offsetof(struct btrfs_node, ptrs),
Chris Mason123abc82007-03-14 14:14:43 -04001822 sizeof(struct btrfs_key_ptr),
Chris Mason5f39d392007-10-15 16:14:19 -04001823 key, btrfs_header_nritems(eb),
Chris Mason7518a232007-03-12 12:01:18 -04001824 slot);
Chris Masonbe0e5c02007-01-26 15:51:26 -05001825}
1826
Yan Zheng5d4f98a2009-06-10 10:45:14 -04001827int btrfs_bin_search(struct extent_buffer *eb, struct btrfs_key *key,
1828 int level, int *slot)
1829{
1830 return bin_search(eb, key, level, slot);
1831}
1832
Yan, Zhengf0486c62010-05-16 10:46:25 -04001833static void root_add_used(struct btrfs_root *root, u32 size)
1834{
1835 spin_lock(&root->accounting_lock);
1836 btrfs_set_root_used(&root->root_item,
1837 btrfs_root_used(&root->root_item) + size);
1838 spin_unlock(&root->accounting_lock);
1839}
1840
1841static void root_sub_used(struct btrfs_root *root, u32 size)
1842{
1843 spin_lock(&root->accounting_lock);
1844 btrfs_set_root_used(&root->root_item,
1845 btrfs_root_used(&root->root_item) - size);
1846 spin_unlock(&root->accounting_lock);
1847}
1848
Chris Masond352ac62008-09-29 15:18:18 -04001849/* given a node and slot number, this reads the blocks it points to. The
1850 * extent buffer is returned with a reference taken (but unlocked).
1851 * NULL is returned on error.
1852 */
Chris Masone02119d2008-09-05 16:13:11 -04001853static noinline struct extent_buffer *read_node_slot(struct btrfs_root *root,
Chris Mason5f39d392007-10-15 16:14:19 -04001854 struct extent_buffer *parent, int slot)
Chris Masonbb803952007-03-01 12:04:21 -05001855{
Chris Masonca7a79a2008-05-12 12:59:19 -04001856 int level = btrfs_header_level(parent);
Josef Bacik416bc652013-04-23 14:17:42 -04001857 struct extent_buffer *eb;
1858
Chris Masonbb803952007-03-01 12:04:21 -05001859 if (slot < 0)
1860 return NULL;
Chris Mason5f39d392007-10-15 16:14:19 -04001861 if (slot >= btrfs_header_nritems(parent))
Chris Masonbb803952007-03-01 12:04:21 -05001862 return NULL;
Chris Masonca7a79a2008-05-12 12:59:19 -04001863
1864 BUG_ON(level == 0);
1865
Josef Bacik416bc652013-04-23 14:17:42 -04001866 eb = read_tree_block(root, btrfs_node_blockptr(parent, slot),
Josef Bacik416bc652013-04-23 14:17:42 -04001867 btrfs_node_ptr_generation(parent, slot));
1868 if (eb && !extent_buffer_uptodate(eb)) {
1869 free_extent_buffer(eb);
1870 eb = NULL;
1871 }
1872
1873 return eb;
Chris Masonbb803952007-03-01 12:04:21 -05001874}
1875
Chris Masond352ac62008-09-29 15:18:18 -04001876/*
1877 * node level balancing, used to make sure nodes are in proper order for
1878 * item deletion. We balance from the top down, so we have to make sure
1879 * that a deletion won't leave an node completely empty later on.
1880 */
Chris Masone02119d2008-09-05 16:13:11 -04001881static noinline int balance_level(struct btrfs_trans_handle *trans,
Chris Mason98ed5172008-01-03 10:01:48 -05001882 struct btrfs_root *root,
1883 struct btrfs_path *path, int level)
Chris Masonbb803952007-03-01 12:04:21 -05001884{
Chris Mason5f39d392007-10-15 16:14:19 -04001885 struct extent_buffer *right = NULL;
1886 struct extent_buffer *mid;
1887 struct extent_buffer *left = NULL;
1888 struct extent_buffer *parent = NULL;
Chris Masonbb803952007-03-01 12:04:21 -05001889 int ret = 0;
1890 int wret;
1891 int pslot;
Chris Masonbb803952007-03-01 12:04:21 -05001892 int orig_slot = path->slots[level];
Chris Mason79f95c82007-03-01 15:16:26 -05001893 u64 orig_ptr;
Chris Masonbb803952007-03-01 12:04:21 -05001894
1895 if (level == 0)
1896 return 0;
1897
Chris Mason5f39d392007-10-15 16:14:19 -04001898 mid = path->nodes[level];
Chris Masonb4ce94d2009-02-04 09:25:08 -05001899
Chris Masonbd681512011-07-16 15:23:14 -04001900 WARN_ON(path->locks[level] != BTRFS_WRITE_LOCK &&
1901 path->locks[level] != BTRFS_WRITE_LOCK_BLOCKING);
Chris Mason7bb86312007-12-11 09:25:06 -05001902 WARN_ON(btrfs_header_generation(mid) != trans->transid);
1903
Chris Mason1d4f8a02007-03-13 09:28:32 -04001904 orig_ptr = btrfs_node_blockptr(mid, orig_slot);
Chris Mason79f95c82007-03-01 15:16:26 -05001905
Li Zefana05a9bb2011-09-06 16:55:34 +08001906 if (level < BTRFS_MAX_LEVEL - 1) {
Chris Mason5f39d392007-10-15 16:14:19 -04001907 parent = path->nodes[level + 1];
Li Zefana05a9bb2011-09-06 16:55:34 +08001908 pslot = path->slots[level + 1];
1909 }
Chris Masonbb803952007-03-01 12:04:21 -05001910
Chris Mason40689472007-03-17 14:29:23 -04001911 /*
1912 * deal with the case where there is only one pointer in the root
1913 * by promoting the node below to a root
1914 */
Chris Mason5f39d392007-10-15 16:14:19 -04001915 if (!parent) {
1916 struct extent_buffer *child;
Chris Masonbb803952007-03-01 12:04:21 -05001917
Chris Mason5f39d392007-10-15 16:14:19 -04001918 if (btrfs_header_nritems(mid) != 1)
Chris Masonbb803952007-03-01 12:04:21 -05001919 return 0;
1920
1921 /* promote the child to a root */
Chris Mason5f39d392007-10-15 16:14:19 -04001922 child = read_node_slot(root, mid, 0);
Mark Fasheh305a26a2011-09-01 11:27:57 -07001923 if (!child) {
1924 ret = -EROFS;
1925 btrfs_std_error(root->fs_info, ret);
1926 goto enospc;
1927 }
1928
Chris Mason925baed2008-06-25 16:01:30 -04001929 btrfs_tree_lock(child);
Chris Masonb4ce94d2009-02-04 09:25:08 -05001930 btrfs_set_lock_blocking(child);
Chris Mason9fa8cfe2009-03-13 10:24:59 -04001931 ret = btrfs_cow_block(trans, root, child, mid, 0, &child);
Yan, Zhengf0486c62010-05-16 10:46:25 -04001932 if (ret) {
1933 btrfs_tree_unlock(child);
1934 free_extent_buffer(child);
1935 goto enospc;
1936 }
Yan2f375ab2008-02-01 14:58:07 -05001937
Jan Schmidt90f8d622013-04-13 13:19:53 +00001938 tree_mod_log_set_root_pointer(root, child, 1);
Chris Mason240f62c2011-03-23 14:54:42 -04001939 rcu_assign_pointer(root->node, child);
Chris Mason925baed2008-06-25 16:01:30 -04001940
Chris Mason0b86a832008-03-24 15:01:56 -04001941 add_root_to_dirty_list(root);
Chris Mason925baed2008-06-25 16:01:30 -04001942 btrfs_tree_unlock(child);
Chris Masonb4ce94d2009-02-04 09:25:08 -05001943
Chris Mason925baed2008-06-25 16:01:30 -04001944 path->locks[level] = 0;
Chris Masonbb803952007-03-01 12:04:21 -05001945 path->nodes[level] = NULL;
Chris Mason5f39d392007-10-15 16:14:19 -04001946 clean_tree_block(trans, root, mid);
Chris Mason925baed2008-06-25 16:01:30 -04001947 btrfs_tree_unlock(mid);
Chris Masonbb803952007-03-01 12:04:21 -05001948 /* once for the path */
Chris Mason5f39d392007-10-15 16:14:19 -04001949 free_extent_buffer(mid);
Yan, Zhengf0486c62010-05-16 10:46:25 -04001950
1951 root_sub_used(root, mid->len);
Jan Schmidt5581a512012-05-16 17:04:52 +02001952 btrfs_free_tree_block(trans, root, mid, 0, 1);
Chris Masonbb803952007-03-01 12:04:21 -05001953 /* once for the root ptr */
Josef Bacik3083ee22012-03-09 16:01:49 -05001954 free_extent_buffer_stale(mid);
Yan, Zhengf0486c62010-05-16 10:46:25 -04001955 return 0;
Chris Masonbb803952007-03-01 12:04:21 -05001956 }
Chris Mason5f39d392007-10-15 16:14:19 -04001957 if (btrfs_header_nritems(mid) >
Chris Mason123abc82007-03-14 14:14:43 -04001958 BTRFS_NODEPTRS_PER_BLOCK(root) / 4)
Chris Masonbb803952007-03-01 12:04:21 -05001959 return 0;
1960
Chris Mason5f39d392007-10-15 16:14:19 -04001961 left = read_node_slot(root, parent, pslot - 1);
1962 if (left) {
Chris Mason925baed2008-06-25 16:01:30 -04001963 btrfs_tree_lock(left);
Chris Masonb4ce94d2009-02-04 09:25:08 -05001964 btrfs_set_lock_blocking(left);
Chris Mason5f39d392007-10-15 16:14:19 -04001965 wret = btrfs_cow_block(trans, root, left,
Chris Mason9fa8cfe2009-03-13 10:24:59 -04001966 parent, pslot - 1, &left);
Chris Mason54aa1f42007-06-22 14:16:25 -04001967 if (wret) {
1968 ret = wret;
1969 goto enospc;
1970 }
Chris Mason2cc58cf2007-08-27 16:49:44 -04001971 }
Chris Mason5f39d392007-10-15 16:14:19 -04001972 right = read_node_slot(root, parent, pslot + 1);
1973 if (right) {
Chris Mason925baed2008-06-25 16:01:30 -04001974 btrfs_tree_lock(right);
Chris Masonb4ce94d2009-02-04 09:25:08 -05001975 btrfs_set_lock_blocking(right);
Chris Mason5f39d392007-10-15 16:14:19 -04001976 wret = btrfs_cow_block(trans, root, right,
Chris Mason9fa8cfe2009-03-13 10:24:59 -04001977 parent, pslot + 1, &right);
Chris Mason2cc58cf2007-08-27 16:49:44 -04001978 if (wret) {
1979 ret = wret;
1980 goto enospc;
1981 }
1982 }
1983
1984 /* first, try to make some room in the middle buffer */
Chris Mason5f39d392007-10-15 16:14:19 -04001985 if (left) {
1986 orig_slot += btrfs_header_nritems(left);
Chris Masonbce4eae2008-04-24 14:42:46 -04001987 wret = push_node_left(trans, root, left, mid, 1);
Chris Mason79f95c82007-03-01 15:16:26 -05001988 if (wret < 0)
1989 ret = wret;
Chris Masonbb803952007-03-01 12:04:21 -05001990 }
Chris Mason79f95c82007-03-01 15:16:26 -05001991
1992 /*
1993 * then try to empty the right most buffer into the middle
1994 */
Chris Mason5f39d392007-10-15 16:14:19 -04001995 if (right) {
Chris Mason971a1f62008-04-24 10:54:32 -04001996 wret = push_node_left(trans, root, mid, right, 1);
Chris Mason54aa1f42007-06-22 14:16:25 -04001997 if (wret < 0 && wret != -ENOSPC)
Chris Mason79f95c82007-03-01 15:16:26 -05001998 ret = wret;
Chris Mason5f39d392007-10-15 16:14:19 -04001999 if (btrfs_header_nritems(right) == 0) {
Chris Mason5f39d392007-10-15 16:14:19 -04002000 clean_tree_block(trans, root, right);
Chris Mason925baed2008-06-25 16:01:30 -04002001 btrfs_tree_unlock(right);
Tsutomu Itohafe5fea2013-04-16 05:18:22 +00002002 del_ptr(root, path, level + 1, pslot + 1);
Yan, Zhengf0486c62010-05-16 10:46:25 -04002003 root_sub_used(root, right->len);
Jan Schmidt5581a512012-05-16 17:04:52 +02002004 btrfs_free_tree_block(trans, root, right, 0, 1);
Josef Bacik3083ee22012-03-09 16:01:49 -05002005 free_extent_buffer_stale(right);
Yan, Zhengf0486c62010-05-16 10:46:25 -04002006 right = NULL;
Chris Masonbb803952007-03-01 12:04:21 -05002007 } else {
Chris Mason5f39d392007-10-15 16:14:19 -04002008 struct btrfs_disk_key right_key;
2009 btrfs_node_key(right, &right_key, 0);
Jan Schmidtf2304752012-05-26 11:43:17 +02002010 tree_mod_log_set_node_key(root->fs_info, parent,
Liu Bo32adf092012-10-19 12:52:15 +00002011 pslot + 1, 0);
Chris Mason5f39d392007-10-15 16:14:19 -04002012 btrfs_set_node_key(parent, &right_key, pslot + 1);
2013 btrfs_mark_buffer_dirty(parent);
Chris Masonbb803952007-03-01 12:04:21 -05002014 }
2015 }
Chris Mason5f39d392007-10-15 16:14:19 -04002016 if (btrfs_header_nritems(mid) == 1) {
Chris Mason79f95c82007-03-01 15:16:26 -05002017 /*
2018 * we're not allowed to leave a node with one item in the
2019 * tree during a delete. A deletion from lower in the tree
2020 * could try to delete the only pointer in this node.
2021 * So, pull some keys from the left.
2022 * There has to be a left pointer at this point because
2023 * otherwise we would have pulled some pointers from the
2024 * right
2025 */
Mark Fasheh305a26a2011-09-01 11:27:57 -07002026 if (!left) {
2027 ret = -EROFS;
2028 btrfs_std_error(root->fs_info, ret);
2029 goto enospc;
2030 }
Chris Mason5f39d392007-10-15 16:14:19 -04002031 wret = balance_node_right(trans, root, mid, left);
Chris Mason54aa1f42007-06-22 14:16:25 -04002032 if (wret < 0) {
Chris Mason79f95c82007-03-01 15:16:26 -05002033 ret = wret;
Chris Mason54aa1f42007-06-22 14:16:25 -04002034 goto enospc;
2035 }
Chris Masonbce4eae2008-04-24 14:42:46 -04002036 if (wret == 1) {
2037 wret = push_node_left(trans, root, left, mid, 1);
2038 if (wret < 0)
2039 ret = wret;
2040 }
Chris Mason79f95c82007-03-01 15:16:26 -05002041 BUG_ON(wret == 1);
2042 }
Chris Mason5f39d392007-10-15 16:14:19 -04002043 if (btrfs_header_nritems(mid) == 0) {
Chris Mason5f39d392007-10-15 16:14:19 -04002044 clean_tree_block(trans, root, mid);
Chris Mason925baed2008-06-25 16:01:30 -04002045 btrfs_tree_unlock(mid);
Tsutomu Itohafe5fea2013-04-16 05:18:22 +00002046 del_ptr(root, path, level + 1, pslot);
Yan, Zhengf0486c62010-05-16 10:46:25 -04002047 root_sub_used(root, mid->len);
Jan Schmidt5581a512012-05-16 17:04:52 +02002048 btrfs_free_tree_block(trans, root, mid, 0, 1);
Josef Bacik3083ee22012-03-09 16:01:49 -05002049 free_extent_buffer_stale(mid);
Yan, Zhengf0486c62010-05-16 10:46:25 -04002050 mid = NULL;
Chris Mason79f95c82007-03-01 15:16:26 -05002051 } else {
2052 /* update the parent key to reflect our changes */
Chris Mason5f39d392007-10-15 16:14:19 -04002053 struct btrfs_disk_key mid_key;
2054 btrfs_node_key(mid, &mid_key, 0);
Liu Bo32adf092012-10-19 12:52:15 +00002055 tree_mod_log_set_node_key(root->fs_info, parent,
Jan Schmidtf2304752012-05-26 11:43:17 +02002056 pslot, 0);
Chris Mason5f39d392007-10-15 16:14:19 -04002057 btrfs_set_node_key(parent, &mid_key, pslot);
2058 btrfs_mark_buffer_dirty(parent);
Chris Mason79f95c82007-03-01 15:16:26 -05002059 }
Chris Masonbb803952007-03-01 12:04:21 -05002060
Chris Mason79f95c82007-03-01 15:16:26 -05002061 /* update the path */
Chris Mason5f39d392007-10-15 16:14:19 -04002062 if (left) {
2063 if (btrfs_header_nritems(left) > orig_slot) {
2064 extent_buffer_get(left);
Chris Mason925baed2008-06-25 16:01:30 -04002065 /* left was locked after cow */
Chris Mason5f39d392007-10-15 16:14:19 -04002066 path->nodes[level] = left;
Chris Masonbb803952007-03-01 12:04:21 -05002067 path->slots[level + 1] -= 1;
2068 path->slots[level] = orig_slot;
Chris Mason925baed2008-06-25 16:01:30 -04002069 if (mid) {
2070 btrfs_tree_unlock(mid);
Chris Mason5f39d392007-10-15 16:14:19 -04002071 free_extent_buffer(mid);
Chris Mason925baed2008-06-25 16:01:30 -04002072 }
Chris Masonbb803952007-03-01 12:04:21 -05002073 } else {
Chris Mason5f39d392007-10-15 16:14:19 -04002074 orig_slot -= btrfs_header_nritems(left);
Chris Masonbb803952007-03-01 12:04:21 -05002075 path->slots[level] = orig_slot;
2076 }
2077 }
Chris Mason79f95c82007-03-01 15:16:26 -05002078 /* double check we haven't messed things up */
Chris Masone20d96d2007-03-22 12:13:20 -04002079 if (orig_ptr !=
Chris Mason5f39d392007-10-15 16:14:19 -04002080 btrfs_node_blockptr(path->nodes[level], path->slots[level]))
Chris Mason79f95c82007-03-01 15:16:26 -05002081 BUG();
Chris Mason54aa1f42007-06-22 14:16:25 -04002082enospc:
Chris Mason925baed2008-06-25 16:01:30 -04002083 if (right) {
2084 btrfs_tree_unlock(right);
Chris Mason5f39d392007-10-15 16:14:19 -04002085 free_extent_buffer(right);
Chris Mason925baed2008-06-25 16:01:30 -04002086 }
2087 if (left) {
2088 if (path->nodes[level] != left)
2089 btrfs_tree_unlock(left);
Chris Mason5f39d392007-10-15 16:14:19 -04002090 free_extent_buffer(left);
Chris Mason925baed2008-06-25 16:01:30 -04002091 }
Chris Masonbb803952007-03-01 12:04:21 -05002092 return ret;
2093}
2094
Chris Masond352ac62008-09-29 15:18:18 -04002095/* Node balancing for insertion. Here we only split or push nodes around
2096 * when they are completely full. This is also done top down, so we
2097 * have to be pessimistic.
2098 */
Chris Masond3977122009-01-05 21:25:51 -05002099static noinline int push_nodes_for_insert(struct btrfs_trans_handle *trans,
Chris Mason98ed5172008-01-03 10:01:48 -05002100 struct btrfs_root *root,
2101 struct btrfs_path *path, int level)
Chris Masone66f7092007-04-20 13:16:02 -04002102{
Chris Mason5f39d392007-10-15 16:14:19 -04002103 struct extent_buffer *right = NULL;
2104 struct extent_buffer *mid;
2105 struct extent_buffer *left = NULL;
2106 struct extent_buffer *parent = NULL;
Chris Masone66f7092007-04-20 13:16:02 -04002107 int ret = 0;
2108 int wret;
2109 int pslot;
2110 int orig_slot = path->slots[level];
Chris Masone66f7092007-04-20 13:16:02 -04002111
2112 if (level == 0)
2113 return 1;
2114
Chris Mason5f39d392007-10-15 16:14:19 -04002115 mid = path->nodes[level];
Chris Mason7bb86312007-12-11 09:25:06 -05002116 WARN_ON(btrfs_header_generation(mid) != trans->transid);
Chris Masone66f7092007-04-20 13:16:02 -04002117
Li Zefana05a9bb2011-09-06 16:55:34 +08002118 if (level < BTRFS_MAX_LEVEL - 1) {
Chris Mason5f39d392007-10-15 16:14:19 -04002119 parent = path->nodes[level + 1];
Li Zefana05a9bb2011-09-06 16:55:34 +08002120 pslot = path->slots[level + 1];
2121 }
Chris Masone66f7092007-04-20 13:16:02 -04002122
Chris Mason5f39d392007-10-15 16:14:19 -04002123 if (!parent)
Chris Masone66f7092007-04-20 13:16:02 -04002124 return 1;
Chris Masone66f7092007-04-20 13:16:02 -04002125
Chris Mason5f39d392007-10-15 16:14:19 -04002126 left = read_node_slot(root, parent, pslot - 1);
Chris Masone66f7092007-04-20 13:16:02 -04002127
2128 /* first, try to make some room in the middle buffer */
Chris Mason5f39d392007-10-15 16:14:19 -04002129 if (left) {
Chris Masone66f7092007-04-20 13:16:02 -04002130 u32 left_nr;
Chris Mason925baed2008-06-25 16:01:30 -04002131
2132 btrfs_tree_lock(left);
Chris Masonb4ce94d2009-02-04 09:25:08 -05002133 btrfs_set_lock_blocking(left);
2134
Chris Mason5f39d392007-10-15 16:14:19 -04002135 left_nr = btrfs_header_nritems(left);
Chris Mason33ade1f2007-04-20 13:48:57 -04002136 if (left_nr >= BTRFS_NODEPTRS_PER_BLOCK(root) - 1) {
2137 wret = 1;
2138 } else {
Chris Mason5f39d392007-10-15 16:14:19 -04002139 ret = btrfs_cow_block(trans, root, left, parent,
Chris Mason9fa8cfe2009-03-13 10:24:59 -04002140 pslot - 1, &left);
Chris Mason54aa1f42007-06-22 14:16:25 -04002141 if (ret)
2142 wret = 1;
2143 else {
Chris Mason54aa1f42007-06-22 14:16:25 -04002144 wret = push_node_left(trans, root,
Chris Mason971a1f62008-04-24 10:54:32 -04002145 left, mid, 0);
Chris Mason54aa1f42007-06-22 14:16:25 -04002146 }
Chris Mason33ade1f2007-04-20 13:48:57 -04002147 }
Chris Masone66f7092007-04-20 13:16:02 -04002148 if (wret < 0)
2149 ret = wret;
2150 if (wret == 0) {
Chris Mason5f39d392007-10-15 16:14:19 -04002151 struct btrfs_disk_key disk_key;
Chris Masone66f7092007-04-20 13:16:02 -04002152 orig_slot += left_nr;
Chris Mason5f39d392007-10-15 16:14:19 -04002153 btrfs_node_key(mid, &disk_key, 0);
Jan Schmidtf2304752012-05-26 11:43:17 +02002154 tree_mod_log_set_node_key(root->fs_info, parent,
Liu Bo32adf092012-10-19 12:52:15 +00002155 pslot, 0);
Chris Mason5f39d392007-10-15 16:14:19 -04002156 btrfs_set_node_key(parent, &disk_key, pslot);
2157 btrfs_mark_buffer_dirty(parent);
2158 if (btrfs_header_nritems(left) > orig_slot) {
2159 path->nodes[level] = left;
Chris Masone66f7092007-04-20 13:16:02 -04002160 path->slots[level + 1] -= 1;
2161 path->slots[level] = orig_slot;
Chris Mason925baed2008-06-25 16:01:30 -04002162 btrfs_tree_unlock(mid);
Chris Mason5f39d392007-10-15 16:14:19 -04002163 free_extent_buffer(mid);
Chris Masone66f7092007-04-20 13:16:02 -04002164 } else {
2165 orig_slot -=
Chris Mason5f39d392007-10-15 16:14:19 -04002166 btrfs_header_nritems(left);
Chris Masone66f7092007-04-20 13:16:02 -04002167 path->slots[level] = orig_slot;
Chris Mason925baed2008-06-25 16:01:30 -04002168 btrfs_tree_unlock(left);
Chris Mason5f39d392007-10-15 16:14:19 -04002169 free_extent_buffer(left);
Chris Masone66f7092007-04-20 13:16:02 -04002170 }
Chris Masone66f7092007-04-20 13:16:02 -04002171 return 0;
2172 }
Chris Mason925baed2008-06-25 16:01:30 -04002173 btrfs_tree_unlock(left);
Chris Mason5f39d392007-10-15 16:14:19 -04002174 free_extent_buffer(left);
Chris Masone66f7092007-04-20 13:16:02 -04002175 }
Chris Mason925baed2008-06-25 16:01:30 -04002176 right = read_node_slot(root, parent, pslot + 1);
Chris Masone66f7092007-04-20 13:16:02 -04002177
2178 /*
2179 * then try to empty the right most buffer into the middle
2180 */
Chris Mason5f39d392007-10-15 16:14:19 -04002181 if (right) {
Chris Mason33ade1f2007-04-20 13:48:57 -04002182 u32 right_nr;
Chris Masonb4ce94d2009-02-04 09:25:08 -05002183
Chris Mason925baed2008-06-25 16:01:30 -04002184 btrfs_tree_lock(right);
Chris Masonb4ce94d2009-02-04 09:25:08 -05002185 btrfs_set_lock_blocking(right);
2186
Chris Mason5f39d392007-10-15 16:14:19 -04002187 right_nr = btrfs_header_nritems(right);
Chris Mason33ade1f2007-04-20 13:48:57 -04002188 if (right_nr >= BTRFS_NODEPTRS_PER_BLOCK(root) - 1) {
2189 wret = 1;
2190 } else {
Chris Mason5f39d392007-10-15 16:14:19 -04002191 ret = btrfs_cow_block(trans, root, right,
2192 parent, pslot + 1,
Chris Mason9fa8cfe2009-03-13 10:24:59 -04002193 &right);
Chris Mason54aa1f42007-06-22 14:16:25 -04002194 if (ret)
2195 wret = 1;
2196 else {
Chris Mason54aa1f42007-06-22 14:16:25 -04002197 wret = balance_node_right(trans, root,
Chris Mason5f39d392007-10-15 16:14:19 -04002198 right, mid);
Chris Mason54aa1f42007-06-22 14:16:25 -04002199 }
Chris Mason33ade1f2007-04-20 13:48:57 -04002200 }
Chris Masone66f7092007-04-20 13:16:02 -04002201 if (wret < 0)
2202 ret = wret;
2203 if (wret == 0) {
Chris Mason5f39d392007-10-15 16:14:19 -04002204 struct btrfs_disk_key disk_key;
2205
2206 btrfs_node_key(right, &disk_key, 0);
Jan Schmidtf2304752012-05-26 11:43:17 +02002207 tree_mod_log_set_node_key(root->fs_info, parent,
Liu Bo32adf092012-10-19 12:52:15 +00002208 pslot + 1, 0);
Chris Mason5f39d392007-10-15 16:14:19 -04002209 btrfs_set_node_key(parent, &disk_key, pslot + 1);
2210 btrfs_mark_buffer_dirty(parent);
2211
2212 if (btrfs_header_nritems(mid) <= orig_slot) {
2213 path->nodes[level] = right;
Chris Masone66f7092007-04-20 13:16:02 -04002214 path->slots[level + 1] += 1;
2215 path->slots[level] = orig_slot -
Chris Mason5f39d392007-10-15 16:14:19 -04002216 btrfs_header_nritems(mid);
Chris Mason925baed2008-06-25 16:01:30 -04002217 btrfs_tree_unlock(mid);
Chris Mason5f39d392007-10-15 16:14:19 -04002218 free_extent_buffer(mid);
Chris Masone66f7092007-04-20 13:16:02 -04002219 } else {
Chris Mason925baed2008-06-25 16:01:30 -04002220 btrfs_tree_unlock(right);
Chris Mason5f39d392007-10-15 16:14:19 -04002221 free_extent_buffer(right);
Chris Masone66f7092007-04-20 13:16:02 -04002222 }
Chris Masone66f7092007-04-20 13:16:02 -04002223 return 0;
2224 }
Chris Mason925baed2008-06-25 16:01:30 -04002225 btrfs_tree_unlock(right);
Chris Mason5f39d392007-10-15 16:14:19 -04002226 free_extent_buffer(right);
Chris Masone66f7092007-04-20 13:16:02 -04002227 }
Chris Masone66f7092007-04-20 13:16:02 -04002228 return 1;
2229}
2230
Chris Mason74123bd2007-02-02 11:05:29 -05002231/*
Chris Masond352ac62008-09-29 15:18:18 -04002232 * readahead one full node of leaves, finding things that are close
2233 * to the block in 'slot', and triggering ra on them.
Chris Mason3c69fae2007-08-07 15:52:22 -04002234 */
Chris Masonc8c42862009-04-03 10:14:18 -04002235static void reada_for_search(struct btrfs_root *root,
2236 struct btrfs_path *path,
2237 int level, int slot, u64 objectid)
Chris Mason3c69fae2007-08-07 15:52:22 -04002238{
Chris Mason5f39d392007-10-15 16:14:19 -04002239 struct extent_buffer *node;
Chris Mason01f46652007-12-21 16:24:26 -05002240 struct btrfs_disk_key disk_key;
Chris Mason3c69fae2007-08-07 15:52:22 -04002241 u32 nritems;
Chris Mason3c69fae2007-08-07 15:52:22 -04002242 u64 search;
Chris Masona7175312009-01-22 09:23:10 -05002243 u64 target;
Chris Mason6b800532007-10-15 16:17:34 -04002244 u64 nread = 0;
Josef Bacikcb25c2e2011-05-11 12:17:34 -04002245 u64 gen;
Chris Mason3c69fae2007-08-07 15:52:22 -04002246 int direction = path->reada;
Chris Mason5f39d392007-10-15 16:14:19 -04002247 struct extent_buffer *eb;
Chris Mason6b800532007-10-15 16:17:34 -04002248 u32 nr;
2249 u32 blocksize;
2250 u32 nscan = 0;
Chris Masondb945352007-10-15 16:15:53 -04002251
Chris Masona6b6e752007-10-15 16:22:39 -04002252 if (level != 1)
Chris Mason3c69fae2007-08-07 15:52:22 -04002253 return;
2254
Chris Mason6702ed42007-08-07 16:15:09 -04002255 if (!path->nodes[level])
2256 return;
2257
Chris Mason5f39d392007-10-15 16:14:19 -04002258 node = path->nodes[level];
Chris Mason925baed2008-06-25 16:01:30 -04002259
Chris Mason3c69fae2007-08-07 15:52:22 -04002260 search = btrfs_node_blockptr(node, slot);
David Sterba707e8a02014-06-04 19:22:26 +02002261 blocksize = root->nodesize;
David Sterba0308af42014-06-15 01:43:40 +02002262 eb = btrfs_find_tree_block(root, search);
Chris Mason5f39d392007-10-15 16:14:19 -04002263 if (eb) {
2264 free_extent_buffer(eb);
Chris Mason3c69fae2007-08-07 15:52:22 -04002265 return;
2266 }
2267
Chris Masona7175312009-01-22 09:23:10 -05002268 target = search;
Chris Mason6b800532007-10-15 16:17:34 -04002269
Chris Mason5f39d392007-10-15 16:14:19 -04002270 nritems = btrfs_header_nritems(node);
Chris Mason6b800532007-10-15 16:17:34 -04002271 nr = slot;
Josef Bacik25b8b932011-06-08 14:36:54 -04002272
Chris Masond3977122009-01-05 21:25:51 -05002273 while (1) {
Chris Mason6b800532007-10-15 16:17:34 -04002274 if (direction < 0) {
2275 if (nr == 0)
2276 break;
2277 nr--;
2278 } else if (direction > 0) {
2279 nr++;
2280 if (nr >= nritems)
2281 break;
Chris Mason3c69fae2007-08-07 15:52:22 -04002282 }
Chris Mason01f46652007-12-21 16:24:26 -05002283 if (path->reada < 0 && objectid) {
2284 btrfs_node_key(node, &disk_key, nr);
2285 if (btrfs_disk_key_objectid(&disk_key) != objectid)
2286 break;
2287 }
Chris Mason6b800532007-10-15 16:17:34 -04002288 search = btrfs_node_blockptr(node, nr);
Chris Masona7175312009-01-22 09:23:10 -05002289 if ((search <= target && target - search <= 65536) ||
2290 (search > target && search - target <= 65536)) {
Josef Bacikcb25c2e2011-05-11 12:17:34 -04002291 gen = btrfs_node_ptr_generation(node, nr);
David Sterbad3e46fe2014-06-15 02:04:19 +02002292 readahead_tree_block(root, search);
Chris Mason6b800532007-10-15 16:17:34 -04002293 nread += blocksize;
2294 }
2295 nscan++;
Chris Masona7175312009-01-22 09:23:10 -05002296 if ((nread > 65536 || nscan > 32))
Chris Mason6b800532007-10-15 16:17:34 -04002297 break;
Chris Mason3c69fae2007-08-07 15:52:22 -04002298 }
2299}
Chris Mason925baed2008-06-25 16:01:30 -04002300
Josef Bacik0b088512013-06-17 14:23:02 -04002301static noinline void reada_for_balance(struct btrfs_root *root,
2302 struct btrfs_path *path, int level)
Chris Masonb4ce94d2009-02-04 09:25:08 -05002303{
2304 int slot;
2305 int nritems;
2306 struct extent_buffer *parent;
2307 struct extent_buffer *eb;
2308 u64 gen;
2309 u64 block1 = 0;
2310 u64 block2 = 0;
Chris Masonb4ce94d2009-02-04 09:25:08 -05002311
Chris Mason8c594ea2009-04-20 15:50:10 -04002312 parent = path->nodes[level + 1];
Chris Masonb4ce94d2009-02-04 09:25:08 -05002313 if (!parent)
Josef Bacik0b088512013-06-17 14:23:02 -04002314 return;
Chris Masonb4ce94d2009-02-04 09:25:08 -05002315
2316 nritems = btrfs_header_nritems(parent);
Chris Mason8c594ea2009-04-20 15:50:10 -04002317 slot = path->slots[level + 1];
Chris Masonb4ce94d2009-02-04 09:25:08 -05002318
2319 if (slot > 0) {
2320 block1 = btrfs_node_blockptr(parent, slot - 1);
2321 gen = btrfs_node_ptr_generation(parent, slot - 1);
David Sterba0308af42014-06-15 01:43:40 +02002322 eb = btrfs_find_tree_block(root, block1);
Chris Masonb9fab912012-05-06 07:23:47 -04002323 /*
2324 * if we get -eagain from btrfs_buffer_uptodate, we
2325 * don't want to return eagain here. That will loop
2326 * forever
2327 */
2328 if (eb && btrfs_buffer_uptodate(eb, gen, 1) != 0)
Chris Masonb4ce94d2009-02-04 09:25:08 -05002329 block1 = 0;
2330 free_extent_buffer(eb);
2331 }
Chris Mason8c594ea2009-04-20 15:50:10 -04002332 if (slot + 1 < nritems) {
Chris Masonb4ce94d2009-02-04 09:25:08 -05002333 block2 = btrfs_node_blockptr(parent, slot + 1);
2334 gen = btrfs_node_ptr_generation(parent, slot + 1);
David Sterba0308af42014-06-15 01:43:40 +02002335 eb = btrfs_find_tree_block(root, block2);
Chris Masonb9fab912012-05-06 07:23:47 -04002336 if (eb && btrfs_buffer_uptodate(eb, gen, 1) != 0)
Chris Masonb4ce94d2009-02-04 09:25:08 -05002337 block2 = 0;
2338 free_extent_buffer(eb);
2339 }
Chris Mason8c594ea2009-04-20 15:50:10 -04002340
Josef Bacik0b088512013-06-17 14:23:02 -04002341 if (block1)
David Sterbad3e46fe2014-06-15 02:04:19 +02002342 readahead_tree_block(root, block1);
Josef Bacik0b088512013-06-17 14:23:02 -04002343 if (block2)
David Sterbad3e46fe2014-06-15 02:04:19 +02002344 readahead_tree_block(root, block2);
Chris Masonb4ce94d2009-02-04 09:25:08 -05002345}
2346
2347
2348/*
Chris Masond3977122009-01-05 21:25:51 -05002349 * when we walk down the tree, it is usually safe to unlock the higher layers
2350 * in the tree. The exceptions are when our path goes through slot 0, because
2351 * operations on the tree might require changing key pointers higher up in the
2352 * tree.
Chris Masond352ac62008-09-29 15:18:18 -04002353 *
Chris Masond3977122009-01-05 21:25:51 -05002354 * callers might also have set path->keep_locks, which tells this code to keep
2355 * the lock if the path points to the last slot in the block. This is part of
2356 * walking through the tree, and selecting the next slot in the higher block.
Chris Masond352ac62008-09-29 15:18:18 -04002357 *
Chris Masond3977122009-01-05 21:25:51 -05002358 * lowest_unlock sets the lowest level in the tree we're allowed to unlock. so
2359 * if lowest_unlock is 1, level 0 won't be unlocked
Chris Masond352ac62008-09-29 15:18:18 -04002360 */
Chris Masone02119d2008-09-05 16:13:11 -04002361static noinline void unlock_up(struct btrfs_path *path, int level,
Chris Masonf7c79f32012-03-19 15:54:38 -04002362 int lowest_unlock, int min_write_lock_level,
2363 int *write_lock_level)
Chris Mason925baed2008-06-25 16:01:30 -04002364{
2365 int i;
2366 int skip_level = level;
Chris Mason051e1b92008-06-25 16:01:30 -04002367 int no_skips = 0;
Chris Mason925baed2008-06-25 16:01:30 -04002368 struct extent_buffer *t;
2369
2370 for (i = level; i < BTRFS_MAX_LEVEL; i++) {
2371 if (!path->nodes[i])
2372 break;
2373 if (!path->locks[i])
2374 break;
Chris Mason051e1b92008-06-25 16:01:30 -04002375 if (!no_skips && path->slots[i] == 0) {
Chris Mason925baed2008-06-25 16:01:30 -04002376 skip_level = i + 1;
2377 continue;
2378 }
Chris Mason051e1b92008-06-25 16:01:30 -04002379 if (!no_skips && path->keep_locks) {
Chris Mason925baed2008-06-25 16:01:30 -04002380 u32 nritems;
2381 t = path->nodes[i];
2382 nritems = btrfs_header_nritems(t);
Chris Mason051e1b92008-06-25 16:01:30 -04002383 if (nritems < 1 || path->slots[i] >= nritems - 1) {
Chris Mason925baed2008-06-25 16:01:30 -04002384 skip_level = i + 1;
2385 continue;
2386 }
2387 }
Chris Mason051e1b92008-06-25 16:01:30 -04002388 if (skip_level < i && i >= lowest_unlock)
2389 no_skips = 1;
2390
Chris Mason925baed2008-06-25 16:01:30 -04002391 t = path->nodes[i];
2392 if (i >= lowest_unlock && i > skip_level && path->locks[i]) {
Chris Masonbd681512011-07-16 15:23:14 -04002393 btrfs_tree_unlock_rw(t, path->locks[i]);
Chris Mason925baed2008-06-25 16:01:30 -04002394 path->locks[i] = 0;
Chris Masonf7c79f32012-03-19 15:54:38 -04002395 if (write_lock_level &&
2396 i > min_write_lock_level &&
2397 i <= *write_lock_level) {
2398 *write_lock_level = i - 1;
2399 }
Chris Mason925baed2008-06-25 16:01:30 -04002400 }
2401 }
2402}
2403
Chris Mason3c69fae2007-08-07 15:52:22 -04002404/*
Chris Masonb4ce94d2009-02-04 09:25:08 -05002405 * This releases any locks held in the path starting at level and
2406 * going all the way up to the root.
2407 *
2408 * btrfs_search_slot will keep the lock held on higher nodes in a few
2409 * corner cases, such as COW of the block at slot zero in the node. This
2410 * ignores those rules, and it should only be called when there are no
2411 * more updates to be done higher up in the tree.
2412 */
2413noinline void btrfs_unlock_up_safe(struct btrfs_path *path, int level)
2414{
2415 int i;
2416
Josef Bacik09a2a8f92013-04-05 16:51:15 -04002417 if (path->keep_locks)
Chris Masonb4ce94d2009-02-04 09:25:08 -05002418 return;
2419
2420 for (i = level; i < BTRFS_MAX_LEVEL; i++) {
2421 if (!path->nodes[i])
Chris Mason12f4dac2009-02-04 09:31:42 -05002422 continue;
Chris Masonb4ce94d2009-02-04 09:25:08 -05002423 if (!path->locks[i])
Chris Mason12f4dac2009-02-04 09:31:42 -05002424 continue;
Chris Masonbd681512011-07-16 15:23:14 -04002425 btrfs_tree_unlock_rw(path->nodes[i], path->locks[i]);
Chris Masonb4ce94d2009-02-04 09:25:08 -05002426 path->locks[i] = 0;
2427 }
2428}
2429
2430/*
Chris Masonc8c42862009-04-03 10:14:18 -04002431 * helper function for btrfs_search_slot. The goal is to find a block
2432 * in cache without setting the path to blocking. If we find the block
2433 * we return zero and the path is unchanged.
2434 *
2435 * If we can't find the block, we set the path blocking and do some
2436 * reada. -EAGAIN is returned and the search must be repeated.
2437 */
2438static int
2439read_block_for_search(struct btrfs_trans_handle *trans,
2440 struct btrfs_root *root, struct btrfs_path *p,
2441 struct extent_buffer **eb_ret, int level, int slot,
Jan Schmidt5d9e75c42012-05-16 18:25:47 +02002442 struct btrfs_key *key, u64 time_seq)
Chris Masonc8c42862009-04-03 10:14:18 -04002443{
2444 u64 blocknr;
2445 u64 gen;
Chris Masonc8c42862009-04-03 10:14:18 -04002446 struct extent_buffer *b = *eb_ret;
2447 struct extent_buffer *tmp;
Chris Mason76a05b32009-05-14 13:24:30 -04002448 int ret;
Chris Masonc8c42862009-04-03 10:14:18 -04002449
2450 blocknr = btrfs_node_blockptr(b, slot);
2451 gen = btrfs_node_ptr_generation(b, slot);
Chris Masonc8c42862009-04-03 10:14:18 -04002452
David Sterba0308af42014-06-15 01:43:40 +02002453 tmp = btrfs_find_tree_block(root, blocknr);
Chris Masoncb449212010-10-24 11:01:27 -04002454 if (tmp) {
Chris Masonb9fab912012-05-06 07:23:47 -04002455 /* first we do an atomic uptodate check */
Josef Bacikbdf7c002013-06-17 13:44:48 -04002456 if (btrfs_buffer_uptodate(tmp, gen, 1) > 0) {
2457 *eb_ret = tmp;
2458 return 0;
Chris Masoncb449212010-10-24 11:01:27 -04002459 }
Josef Bacikbdf7c002013-06-17 13:44:48 -04002460
2461 /* the pages were up to date, but we failed
2462 * the generation number check. Do a full
2463 * read for the generation number that is correct.
2464 * We must do this without dropping locks so
2465 * we can trust our generation number
2466 */
2467 btrfs_set_path_blocking(p);
2468
2469 /* now we're allowed to do a blocking uptodate check */
2470 ret = btrfs_read_buffer(tmp, gen);
2471 if (!ret) {
2472 *eb_ret = tmp;
2473 return 0;
2474 }
2475 free_extent_buffer(tmp);
2476 btrfs_release_path(p);
2477 return -EIO;
Chris Masonc8c42862009-04-03 10:14:18 -04002478 }
2479
2480 /*
2481 * reduce lock contention at high levels
2482 * of the btree by dropping locks before
Chris Mason76a05b32009-05-14 13:24:30 -04002483 * we read. Don't release the lock on the current
2484 * level because we need to walk this node to figure
2485 * out which blocks to read.
Chris Masonc8c42862009-04-03 10:14:18 -04002486 */
Chris Mason8c594ea2009-04-20 15:50:10 -04002487 btrfs_unlock_up_safe(p, level + 1);
2488 btrfs_set_path_blocking(p);
2489
Chris Masoncb449212010-10-24 11:01:27 -04002490 free_extent_buffer(tmp);
Chris Masonc8c42862009-04-03 10:14:18 -04002491 if (p->reada)
2492 reada_for_search(root, p, level, slot, key->objectid);
2493
David Sterbab3b4aa72011-04-21 01:20:15 +02002494 btrfs_release_path(p);
Chris Mason76a05b32009-05-14 13:24:30 -04002495
2496 ret = -EAGAIN;
David Sterbace86cd52014-06-15 01:07:32 +02002497 tmp = read_tree_block(root, blocknr, 0);
Chris Mason76a05b32009-05-14 13:24:30 -04002498 if (tmp) {
2499 /*
2500 * If the read above didn't mark this buffer up to date,
2501 * it will never end up being up to date. Set ret to EIO now
2502 * and give up so that our caller doesn't loop forever
2503 * on our EAGAINs.
2504 */
Chris Masonb9fab912012-05-06 07:23:47 -04002505 if (!btrfs_buffer_uptodate(tmp, 0, 0))
Chris Mason76a05b32009-05-14 13:24:30 -04002506 ret = -EIO;
Chris Masonc8c42862009-04-03 10:14:18 -04002507 free_extent_buffer(tmp);
Chris Mason76a05b32009-05-14 13:24:30 -04002508 }
2509 return ret;
Chris Masonc8c42862009-04-03 10:14:18 -04002510}
2511
2512/*
2513 * helper function for btrfs_search_slot. This does all of the checks
2514 * for node-level blocks and does any balancing required based on
2515 * the ins_len.
2516 *
2517 * If no extra work was required, zero is returned. If we had to
2518 * drop the path, -EAGAIN is returned and btrfs_search_slot must
2519 * start over
2520 */
2521static int
2522setup_nodes_for_search(struct btrfs_trans_handle *trans,
2523 struct btrfs_root *root, struct btrfs_path *p,
Chris Masonbd681512011-07-16 15:23:14 -04002524 struct extent_buffer *b, int level, int ins_len,
2525 int *write_lock_level)
Chris Masonc8c42862009-04-03 10:14:18 -04002526{
2527 int ret;
2528 if ((p->search_for_split || ins_len > 0) && btrfs_header_nritems(b) >=
2529 BTRFS_NODEPTRS_PER_BLOCK(root) - 3) {
2530 int sret;
2531
Chris Masonbd681512011-07-16 15:23:14 -04002532 if (*write_lock_level < level + 1) {
2533 *write_lock_level = level + 1;
2534 btrfs_release_path(p);
2535 goto again;
2536 }
2537
Chris Masonc8c42862009-04-03 10:14:18 -04002538 btrfs_set_path_blocking(p);
Josef Bacik0b088512013-06-17 14:23:02 -04002539 reada_for_balance(root, p, level);
Chris Masonc8c42862009-04-03 10:14:18 -04002540 sret = split_node(trans, root, p, level);
Chris Masonbd681512011-07-16 15:23:14 -04002541 btrfs_clear_path_blocking(p, NULL, 0);
Chris Masonc8c42862009-04-03 10:14:18 -04002542
2543 BUG_ON(sret > 0);
2544 if (sret) {
2545 ret = sret;
2546 goto done;
2547 }
2548 b = p->nodes[level];
2549 } else if (ins_len < 0 && btrfs_header_nritems(b) <
Chris Masoncfbb9302009-05-18 10:41:58 -04002550 BTRFS_NODEPTRS_PER_BLOCK(root) / 2) {
Chris Masonc8c42862009-04-03 10:14:18 -04002551 int sret;
2552
Chris Masonbd681512011-07-16 15:23:14 -04002553 if (*write_lock_level < level + 1) {
2554 *write_lock_level = level + 1;
2555 btrfs_release_path(p);
2556 goto again;
2557 }
2558
Chris Masonc8c42862009-04-03 10:14:18 -04002559 btrfs_set_path_blocking(p);
Josef Bacik0b088512013-06-17 14:23:02 -04002560 reada_for_balance(root, p, level);
Chris Masonc8c42862009-04-03 10:14:18 -04002561 sret = balance_level(trans, root, p, level);
Chris Masonbd681512011-07-16 15:23:14 -04002562 btrfs_clear_path_blocking(p, NULL, 0);
Chris Masonc8c42862009-04-03 10:14:18 -04002563
2564 if (sret) {
2565 ret = sret;
2566 goto done;
2567 }
2568 b = p->nodes[level];
2569 if (!b) {
David Sterbab3b4aa72011-04-21 01:20:15 +02002570 btrfs_release_path(p);
Chris Masonc8c42862009-04-03 10:14:18 -04002571 goto again;
2572 }
2573 BUG_ON(btrfs_header_nritems(b) == 1);
2574 }
2575 return 0;
2576
2577again:
2578 ret = -EAGAIN;
2579done:
2580 return ret;
2581}
2582
Filipe David Borba Mananad7396f02013-08-30 15:46:43 +01002583static void key_search_validate(struct extent_buffer *b,
2584 struct btrfs_key *key,
2585 int level)
2586{
2587#ifdef CONFIG_BTRFS_ASSERT
2588 struct btrfs_disk_key disk_key;
2589
2590 btrfs_cpu_key_to_disk(&disk_key, key);
2591
2592 if (level == 0)
2593 ASSERT(!memcmp_extent_buffer(b, &disk_key,
2594 offsetof(struct btrfs_leaf, items[0].key),
2595 sizeof(disk_key)));
2596 else
2597 ASSERT(!memcmp_extent_buffer(b, &disk_key,
2598 offsetof(struct btrfs_node, ptrs[0].key),
2599 sizeof(disk_key)));
2600#endif
2601}
2602
2603static int key_search(struct extent_buffer *b, struct btrfs_key *key,
2604 int level, int *prev_cmp, int *slot)
2605{
2606 if (*prev_cmp != 0) {
2607 *prev_cmp = bin_search(b, key, level, slot);
2608 return *prev_cmp;
2609 }
2610
2611 key_search_validate(b, key, level);
2612 *slot = 0;
2613
2614 return 0;
2615}
2616
David Sterba381cf652015-01-02 18:45:16 +01002617int btrfs_find_item(struct btrfs_root *fs_root, struct btrfs_path *path,
Kelley Nielsene33d5c32013-11-04 19:33:33 -08002618 u64 iobjectid, u64 ioff, u8 key_type,
2619 struct btrfs_key *found_key)
2620{
2621 int ret;
2622 struct btrfs_key key;
2623 struct extent_buffer *eb;
David Sterba381cf652015-01-02 18:45:16 +01002624
2625 ASSERT(path);
David Sterba1d4c08e2015-01-02 19:36:14 +01002626 ASSERT(found_key);
Kelley Nielsene33d5c32013-11-04 19:33:33 -08002627
2628 key.type = key_type;
2629 key.objectid = iobjectid;
2630 key.offset = ioff;
2631
2632 ret = btrfs_search_slot(NULL, fs_root, &key, path, 0, 0);
David Sterba1d4c08e2015-01-02 19:36:14 +01002633 if (ret < 0)
Kelley Nielsene33d5c32013-11-04 19:33:33 -08002634 return ret;
2635
2636 eb = path->nodes[0];
2637 if (ret && path->slots[0] >= btrfs_header_nritems(eb)) {
2638 ret = btrfs_next_leaf(fs_root, path);
2639 if (ret)
2640 return ret;
2641 eb = path->nodes[0];
2642 }
2643
2644 btrfs_item_key_to_cpu(eb, found_key, path->slots[0]);
2645 if (found_key->type != key.type ||
2646 found_key->objectid != key.objectid)
2647 return 1;
2648
2649 return 0;
2650}
2651
Chris Masonc8c42862009-04-03 10:14:18 -04002652/*
Chris Mason74123bd2007-02-02 11:05:29 -05002653 * look for key in the tree. path is filled in with nodes along the way
2654 * if key is found, we return zero and you can find the item in the leaf
2655 * level of the path (level 0)
2656 *
2657 * If the key isn't found, the path points to the slot where it should
Chris Masonaa5d6be2007-02-28 16:35:06 -05002658 * be inserted, and 1 is returned. If there are other errors during the
2659 * search a negative error number is returned.
Chris Mason97571fd2007-02-24 13:39:08 -05002660 *
2661 * if ins_len > 0, nodes and leaves will be split as we walk down the
2662 * tree. if ins_len < 0, nodes will be merged as we walk down the tree (if
2663 * possible)
Chris Mason74123bd2007-02-02 11:05:29 -05002664 */
Chris Masone089f052007-03-16 16:20:31 -04002665int btrfs_search_slot(struct btrfs_trans_handle *trans, struct btrfs_root
2666 *root, struct btrfs_key *key, struct btrfs_path *p, int
2667 ins_len, int cow)
Chris Masonbe0e5c02007-01-26 15:51:26 -05002668{
Chris Mason5f39d392007-10-15 16:14:19 -04002669 struct extent_buffer *b;
Chris Masonbe0e5c02007-01-26 15:51:26 -05002670 int slot;
2671 int ret;
Yan Zheng33c66f42009-07-22 09:59:00 -04002672 int err;
Chris Masonbe0e5c02007-01-26 15:51:26 -05002673 int level;
Chris Mason925baed2008-06-25 16:01:30 -04002674 int lowest_unlock = 1;
Chris Masonbd681512011-07-16 15:23:14 -04002675 int root_lock;
2676 /* everything at write_lock_level or lower must be write locked */
2677 int write_lock_level = 0;
Chris Mason9f3a7422007-08-07 15:52:19 -04002678 u8 lowest_level = 0;
Chris Masonf7c79f32012-03-19 15:54:38 -04002679 int min_write_lock_level;
Filipe David Borba Mananad7396f02013-08-30 15:46:43 +01002680 int prev_cmp;
Chris Mason9f3a7422007-08-07 15:52:19 -04002681
Chris Mason6702ed42007-08-07 16:15:09 -04002682 lowest_level = p->lowest_level;
Chris Mason323ac952008-10-01 19:05:46 -04002683 WARN_ON(lowest_level && ins_len > 0);
Chris Mason22b0ebd2007-03-30 08:47:31 -04002684 WARN_ON(p->nodes[0] != NULL);
Filipe David Borba Mananaeb653de2013-12-23 11:53:02 +00002685 BUG_ON(!cow && ins_len);
Josef Bacik25179202008-10-29 14:49:05 -04002686
Chris Masonbd681512011-07-16 15:23:14 -04002687 if (ins_len < 0) {
Chris Mason925baed2008-06-25 16:01:30 -04002688 lowest_unlock = 2;
Chris Mason65b51a02008-08-01 15:11:20 -04002689
Chris Masonbd681512011-07-16 15:23:14 -04002690 /* when we are removing items, we might have to go up to level
2691 * two as we update tree pointers Make sure we keep write
2692 * for those levels as well
2693 */
2694 write_lock_level = 2;
2695 } else if (ins_len > 0) {
2696 /*
2697 * for inserting items, make sure we have a write lock on
2698 * level 1 so we can update keys
2699 */
2700 write_lock_level = 1;
2701 }
2702
2703 if (!cow)
2704 write_lock_level = -1;
2705
Josef Bacik09a2a8f92013-04-05 16:51:15 -04002706 if (cow && (p->keep_locks || p->lowest_level))
Chris Masonbd681512011-07-16 15:23:14 -04002707 write_lock_level = BTRFS_MAX_LEVEL;
2708
Chris Masonf7c79f32012-03-19 15:54:38 -04002709 min_write_lock_level = write_lock_level;
2710
Chris Masonbb803952007-03-01 12:04:21 -05002711again:
Filipe David Borba Mananad7396f02013-08-30 15:46:43 +01002712 prev_cmp = -1;
Chris Masonbd681512011-07-16 15:23:14 -04002713 /*
2714 * we try very hard to do read locks on the root
2715 */
2716 root_lock = BTRFS_READ_LOCK;
2717 level = 0;
Yan Zheng5d4f98a2009-06-10 10:45:14 -04002718 if (p->search_commit_root) {
Chris Masonbd681512011-07-16 15:23:14 -04002719 /*
2720 * the commit roots are read only
2721 * so we always do read locks
2722 */
Josef Bacik3f8a18c2014-03-28 17:16:01 -04002723 if (p->need_commit_sem)
2724 down_read(&root->fs_info->commit_root_sem);
Yan Zheng5d4f98a2009-06-10 10:45:14 -04002725 b = root->commit_root;
2726 extent_buffer_get(b);
Chris Masonbd681512011-07-16 15:23:14 -04002727 level = btrfs_header_level(b);
Josef Bacik3f8a18c2014-03-28 17:16:01 -04002728 if (p->need_commit_sem)
2729 up_read(&root->fs_info->commit_root_sem);
Yan Zheng5d4f98a2009-06-10 10:45:14 -04002730 if (!p->skip_locking)
Chris Masonbd681512011-07-16 15:23:14 -04002731 btrfs_tree_read_lock(b);
Yan Zheng5d4f98a2009-06-10 10:45:14 -04002732 } else {
Chris Masonbd681512011-07-16 15:23:14 -04002733 if (p->skip_locking) {
Yan Zheng5d4f98a2009-06-10 10:45:14 -04002734 b = btrfs_root_node(root);
Chris Masonbd681512011-07-16 15:23:14 -04002735 level = btrfs_header_level(b);
2736 } else {
2737 /* we don't know the level of the root node
2738 * until we actually have it read locked
2739 */
2740 b = btrfs_read_lock_root_node(root);
2741 level = btrfs_header_level(b);
2742 if (level <= write_lock_level) {
2743 /* whoops, must trade for write lock */
2744 btrfs_tree_read_unlock(b);
2745 free_extent_buffer(b);
2746 b = btrfs_lock_root_node(root);
2747 root_lock = BTRFS_WRITE_LOCK;
2748
2749 /* the level might have changed, check again */
2750 level = btrfs_header_level(b);
2751 }
2752 }
Yan Zheng5d4f98a2009-06-10 10:45:14 -04002753 }
Chris Masonbd681512011-07-16 15:23:14 -04002754 p->nodes[level] = b;
2755 if (!p->skip_locking)
2756 p->locks[level] = root_lock;
Chris Mason925baed2008-06-25 16:01:30 -04002757
Chris Masoneb60cea2007-02-02 09:18:22 -05002758 while (b) {
Chris Mason5f39d392007-10-15 16:14:19 -04002759 level = btrfs_header_level(b);
Chris Mason65b51a02008-08-01 15:11:20 -04002760
2761 /*
2762 * setup the path here so we can release it under lock
2763 * contention with the cow code
2764 */
Chris Mason02217ed2007-03-02 16:08:05 -05002765 if (cow) {
Chris Masonc8c42862009-04-03 10:14:18 -04002766 /*
2767 * if we don't really need to cow this block
2768 * then we don't want to set the path blocking,
2769 * so we test it here
2770 */
Yan Zheng5d4f98a2009-06-10 10:45:14 -04002771 if (!should_cow_block(trans, root, b))
Chris Mason65b51a02008-08-01 15:11:20 -04002772 goto cow_done;
Yan Zheng5d4f98a2009-06-10 10:45:14 -04002773
Chris Masonbd681512011-07-16 15:23:14 -04002774 /*
2775 * must have write locks on this node and the
2776 * parent
2777 */
Josef Bacik5124e002012-11-07 13:44:13 -05002778 if (level > write_lock_level ||
2779 (level + 1 > write_lock_level &&
2780 level + 1 < BTRFS_MAX_LEVEL &&
2781 p->nodes[level + 1])) {
Chris Masonbd681512011-07-16 15:23:14 -04002782 write_lock_level = level + 1;
2783 btrfs_release_path(p);
2784 goto again;
2785 }
2786
Filipe Manana160f4082014-07-28 19:37:17 +01002787 btrfs_set_path_blocking(p);
Yan Zheng33c66f42009-07-22 09:59:00 -04002788 err = btrfs_cow_block(trans, root, b,
2789 p->nodes[level + 1],
2790 p->slots[level + 1], &b);
2791 if (err) {
Yan Zheng33c66f42009-07-22 09:59:00 -04002792 ret = err;
Chris Mason65b51a02008-08-01 15:11:20 -04002793 goto done;
Chris Mason54aa1f42007-06-22 14:16:25 -04002794 }
Chris Mason02217ed2007-03-02 16:08:05 -05002795 }
Chris Mason65b51a02008-08-01 15:11:20 -04002796cow_done:
Chris Masoneb60cea2007-02-02 09:18:22 -05002797 p->nodes[level] = b;
Chris Masonbd681512011-07-16 15:23:14 -04002798 btrfs_clear_path_blocking(p, NULL, 0);
Chris Masonb4ce94d2009-02-04 09:25:08 -05002799
2800 /*
2801 * we have a lock on b and as long as we aren't changing
2802 * the tree, there is no way to for the items in b to change.
2803 * It is safe to drop the lock on our parent before we
2804 * go through the expensive btree search on b.
2805 *
Filipe David Borba Mananaeb653de2013-12-23 11:53:02 +00002806 * If we're inserting or deleting (ins_len != 0), then we might
2807 * be changing slot zero, which may require changing the parent.
2808 * So, we can't drop the lock until after we know which slot
2809 * we're operating on.
Chris Masonb4ce94d2009-02-04 09:25:08 -05002810 */
Filipe David Borba Mananaeb653de2013-12-23 11:53:02 +00002811 if (!ins_len && !p->keep_locks) {
2812 int u = level + 1;
2813
2814 if (u < BTRFS_MAX_LEVEL && p->locks[u]) {
2815 btrfs_tree_unlock_rw(p->nodes[u], p->locks[u]);
2816 p->locks[u] = 0;
2817 }
2818 }
Chris Masonb4ce94d2009-02-04 09:25:08 -05002819
Filipe David Borba Mananad7396f02013-08-30 15:46:43 +01002820 ret = key_search(b, key, level, &prev_cmp, &slot);
Chris Masonb4ce94d2009-02-04 09:25:08 -05002821
Chris Mason5f39d392007-10-15 16:14:19 -04002822 if (level != 0) {
Yan Zheng33c66f42009-07-22 09:59:00 -04002823 int dec = 0;
2824 if (ret && slot > 0) {
2825 dec = 1;
Chris Masonbe0e5c02007-01-26 15:51:26 -05002826 slot -= 1;
Yan Zheng33c66f42009-07-22 09:59:00 -04002827 }
Chris Masonbe0e5c02007-01-26 15:51:26 -05002828 p->slots[level] = slot;
Yan Zheng33c66f42009-07-22 09:59:00 -04002829 err = setup_nodes_for_search(trans, root, p, b, level,
Chris Masonbd681512011-07-16 15:23:14 -04002830 ins_len, &write_lock_level);
Yan Zheng33c66f42009-07-22 09:59:00 -04002831 if (err == -EAGAIN)
Chris Masonc8c42862009-04-03 10:14:18 -04002832 goto again;
Yan Zheng33c66f42009-07-22 09:59:00 -04002833 if (err) {
2834 ret = err;
Chris Masonc8c42862009-04-03 10:14:18 -04002835 goto done;
Yan Zheng33c66f42009-07-22 09:59:00 -04002836 }
Chris Masonc8c42862009-04-03 10:14:18 -04002837 b = p->nodes[level];
2838 slot = p->slots[level];
Chris Masonb4ce94d2009-02-04 09:25:08 -05002839
Chris Masonbd681512011-07-16 15:23:14 -04002840 /*
2841 * slot 0 is special, if we change the key
2842 * we have to update the parent pointer
2843 * which means we must have a write lock
2844 * on the parent
2845 */
Filipe David Borba Mananaeb653de2013-12-23 11:53:02 +00002846 if (slot == 0 && ins_len &&
Chris Masonbd681512011-07-16 15:23:14 -04002847 write_lock_level < level + 1) {
2848 write_lock_level = level + 1;
2849 btrfs_release_path(p);
2850 goto again;
2851 }
2852
Chris Masonf7c79f32012-03-19 15:54:38 -04002853 unlock_up(p, level, lowest_unlock,
2854 min_write_lock_level, &write_lock_level);
Chris Masonf9efa9c2008-06-25 16:14:04 -04002855
Chris Mason925baed2008-06-25 16:01:30 -04002856 if (level == lowest_level) {
Yan Zheng33c66f42009-07-22 09:59:00 -04002857 if (dec)
2858 p->slots[level]++;
Zheng Yan5b21f2e2008-09-26 10:05:38 -04002859 goto done;
Chris Mason925baed2008-06-25 16:01:30 -04002860 }
Chris Masonca7a79a2008-05-12 12:59:19 -04002861
Yan Zheng33c66f42009-07-22 09:59:00 -04002862 err = read_block_for_search(trans, root, p,
Jan Schmidt5d9e75c42012-05-16 18:25:47 +02002863 &b, level, slot, key, 0);
Yan Zheng33c66f42009-07-22 09:59:00 -04002864 if (err == -EAGAIN)
Chris Masonc8c42862009-04-03 10:14:18 -04002865 goto again;
Yan Zheng33c66f42009-07-22 09:59:00 -04002866 if (err) {
2867 ret = err;
Chris Mason76a05b32009-05-14 13:24:30 -04002868 goto done;
Yan Zheng33c66f42009-07-22 09:59:00 -04002869 }
Chris Mason76a05b32009-05-14 13:24:30 -04002870
Chris Masonb4ce94d2009-02-04 09:25:08 -05002871 if (!p->skip_locking) {
Chris Masonbd681512011-07-16 15:23:14 -04002872 level = btrfs_header_level(b);
2873 if (level <= write_lock_level) {
2874 err = btrfs_try_tree_write_lock(b);
2875 if (!err) {
2876 btrfs_set_path_blocking(p);
2877 btrfs_tree_lock(b);
2878 btrfs_clear_path_blocking(p, b,
2879 BTRFS_WRITE_LOCK);
2880 }
2881 p->locks[level] = BTRFS_WRITE_LOCK;
2882 } else {
Chris Masonf82c4582014-11-19 10:25:09 -08002883 err = btrfs_tree_read_lock_atomic(b);
Chris Masonbd681512011-07-16 15:23:14 -04002884 if (!err) {
2885 btrfs_set_path_blocking(p);
2886 btrfs_tree_read_lock(b);
2887 btrfs_clear_path_blocking(p, b,
2888 BTRFS_READ_LOCK);
2889 }
2890 p->locks[level] = BTRFS_READ_LOCK;
Chris Masonb4ce94d2009-02-04 09:25:08 -05002891 }
Chris Masonbd681512011-07-16 15:23:14 -04002892 p->nodes[level] = b;
Chris Masonb4ce94d2009-02-04 09:25:08 -05002893 }
Chris Masonbe0e5c02007-01-26 15:51:26 -05002894 } else {
2895 p->slots[level] = slot;
Yan Zheng87b29b22008-12-17 10:21:48 -05002896 if (ins_len > 0 &&
2897 btrfs_leaf_free_space(root, b) < ins_len) {
Chris Masonbd681512011-07-16 15:23:14 -04002898 if (write_lock_level < 1) {
2899 write_lock_level = 1;
2900 btrfs_release_path(p);
2901 goto again;
2902 }
2903
Chris Masonb4ce94d2009-02-04 09:25:08 -05002904 btrfs_set_path_blocking(p);
Yan Zheng33c66f42009-07-22 09:59:00 -04002905 err = split_leaf(trans, root, key,
2906 p, ins_len, ret == 0);
Chris Masonbd681512011-07-16 15:23:14 -04002907 btrfs_clear_path_blocking(p, NULL, 0);
Chris Masonb4ce94d2009-02-04 09:25:08 -05002908
Yan Zheng33c66f42009-07-22 09:59:00 -04002909 BUG_ON(err > 0);
2910 if (err) {
2911 ret = err;
Chris Mason65b51a02008-08-01 15:11:20 -04002912 goto done;
2913 }
Chris Mason5c680ed2007-02-22 11:39:13 -05002914 }
Chris Mason459931e2008-12-10 09:10:46 -05002915 if (!p->search_for_split)
Chris Masonf7c79f32012-03-19 15:54:38 -04002916 unlock_up(p, level, lowest_unlock,
2917 min_write_lock_level, &write_lock_level);
Chris Mason65b51a02008-08-01 15:11:20 -04002918 goto done;
Chris Masonbe0e5c02007-01-26 15:51:26 -05002919 }
2920 }
Chris Mason65b51a02008-08-01 15:11:20 -04002921 ret = 1;
2922done:
Chris Masonb4ce94d2009-02-04 09:25:08 -05002923 /*
2924 * we don't really know what they plan on doing with the path
2925 * from here on, so for now just mark it as blocking
2926 */
Chris Masonb9473432009-03-13 11:00:37 -04002927 if (!p->leave_spinning)
2928 btrfs_set_path_blocking(p);
Filipe Manana5f5bc6b2014-11-09 08:38:39 +00002929 if (ret < 0 && !p->skip_release_on_error)
David Sterbab3b4aa72011-04-21 01:20:15 +02002930 btrfs_release_path(p);
Chris Mason65b51a02008-08-01 15:11:20 -04002931 return ret;
Chris Masonbe0e5c02007-01-26 15:51:26 -05002932}
2933
Chris Mason74123bd2007-02-02 11:05:29 -05002934/*
Jan Schmidt5d9e75c42012-05-16 18:25:47 +02002935 * Like btrfs_search_slot, this looks for a key in the given tree. It uses the
2936 * current state of the tree together with the operations recorded in the tree
2937 * modification log to search for the key in a previous version of this tree, as
2938 * denoted by the time_seq parameter.
2939 *
2940 * Naturally, there is no support for insert, delete or cow operations.
2941 *
2942 * The resulting path and return value will be set up as if we called
2943 * btrfs_search_slot at that point in time with ins_len and cow both set to 0.
2944 */
2945int btrfs_search_old_slot(struct btrfs_root *root, struct btrfs_key *key,
2946 struct btrfs_path *p, u64 time_seq)
2947{
2948 struct extent_buffer *b;
2949 int slot;
2950 int ret;
2951 int err;
2952 int level;
2953 int lowest_unlock = 1;
2954 u8 lowest_level = 0;
Josef Bacikd4b40872013-09-24 14:09:34 -04002955 int prev_cmp = -1;
Jan Schmidt5d9e75c42012-05-16 18:25:47 +02002956
2957 lowest_level = p->lowest_level;
2958 WARN_ON(p->nodes[0] != NULL);
2959
2960 if (p->search_commit_root) {
2961 BUG_ON(time_seq);
2962 return btrfs_search_slot(NULL, root, key, p, 0, 0);
2963 }
2964
2965again:
Jan Schmidt5d9e75c42012-05-16 18:25:47 +02002966 b = get_old_root(root, time_seq);
Jan Schmidt5d9e75c42012-05-16 18:25:47 +02002967 level = btrfs_header_level(b);
Jan Schmidt5d9e75c42012-05-16 18:25:47 +02002968 p->locks[level] = BTRFS_READ_LOCK;
2969
2970 while (b) {
2971 level = btrfs_header_level(b);
2972 p->nodes[level] = b;
2973 btrfs_clear_path_blocking(p, NULL, 0);
2974
2975 /*
2976 * we have a lock on b and as long as we aren't changing
2977 * the tree, there is no way to for the items in b to change.
2978 * It is safe to drop the lock on our parent before we
2979 * go through the expensive btree search on b.
2980 */
2981 btrfs_unlock_up_safe(p, level + 1);
2982
Josef Bacikd4b40872013-09-24 14:09:34 -04002983 /*
2984 * Since we can unwind eb's we want to do a real search every
2985 * time.
2986 */
2987 prev_cmp = -1;
Filipe David Borba Mananad7396f02013-08-30 15:46:43 +01002988 ret = key_search(b, key, level, &prev_cmp, &slot);
Jan Schmidt5d9e75c42012-05-16 18:25:47 +02002989
2990 if (level != 0) {
2991 int dec = 0;
2992 if (ret && slot > 0) {
2993 dec = 1;
2994 slot -= 1;
2995 }
2996 p->slots[level] = slot;
2997 unlock_up(p, level, lowest_unlock, 0, NULL);
2998
2999 if (level == lowest_level) {
3000 if (dec)
3001 p->slots[level]++;
3002 goto done;
3003 }
3004
3005 err = read_block_for_search(NULL, root, p, &b, level,
3006 slot, key, time_seq);
3007 if (err == -EAGAIN)
3008 goto again;
3009 if (err) {
3010 ret = err;
3011 goto done;
3012 }
3013
3014 level = btrfs_header_level(b);
Chris Masonf82c4582014-11-19 10:25:09 -08003015 err = btrfs_tree_read_lock_atomic(b);
Jan Schmidt5d9e75c42012-05-16 18:25:47 +02003016 if (!err) {
3017 btrfs_set_path_blocking(p);
3018 btrfs_tree_read_lock(b);
3019 btrfs_clear_path_blocking(p, b,
3020 BTRFS_READ_LOCK);
3021 }
Josef Bacik9ec72672013-08-07 16:57:23 -04003022 b = tree_mod_log_rewind(root->fs_info, p, b, time_seq);
Josef Bacikdb7f3432013-08-07 14:54:37 -04003023 if (!b) {
3024 ret = -ENOMEM;
3025 goto done;
3026 }
Jan Schmidt5d9e75c42012-05-16 18:25:47 +02003027 p->locks[level] = BTRFS_READ_LOCK;
3028 p->nodes[level] = b;
Jan Schmidt5d9e75c42012-05-16 18:25:47 +02003029 } else {
3030 p->slots[level] = slot;
3031 unlock_up(p, level, lowest_unlock, 0, NULL);
3032 goto done;
3033 }
3034 }
3035 ret = 1;
3036done:
3037 if (!p->leave_spinning)
3038 btrfs_set_path_blocking(p);
3039 if (ret < 0)
3040 btrfs_release_path(p);
3041
3042 return ret;
3043}
3044
3045/*
Arne Jansen2f38b3e2011-09-13 11:18:10 +02003046 * helper to use instead of search slot if no exact match is needed but
3047 * instead the next or previous item should be returned.
3048 * When find_higher is true, the next higher item is returned, the next lower
3049 * otherwise.
3050 * When return_any and find_higher are both true, and no higher item is found,
3051 * return the next lower instead.
3052 * When return_any is true and find_higher is false, and no lower item is found,
3053 * return the next higher instead.
3054 * It returns 0 if any item is found, 1 if none is found (tree empty), and
3055 * < 0 on error
3056 */
3057int btrfs_search_slot_for_read(struct btrfs_root *root,
3058 struct btrfs_key *key, struct btrfs_path *p,
3059 int find_higher, int return_any)
3060{
3061 int ret;
3062 struct extent_buffer *leaf;
3063
3064again:
3065 ret = btrfs_search_slot(NULL, root, key, p, 0, 0);
3066 if (ret <= 0)
3067 return ret;
3068 /*
3069 * a return value of 1 means the path is at the position where the
3070 * item should be inserted. Normally this is the next bigger item,
3071 * but in case the previous item is the last in a leaf, path points
3072 * to the first free slot in the previous leaf, i.e. at an invalid
3073 * item.
3074 */
3075 leaf = p->nodes[0];
3076
3077 if (find_higher) {
3078 if (p->slots[0] >= btrfs_header_nritems(leaf)) {
3079 ret = btrfs_next_leaf(root, p);
3080 if (ret <= 0)
3081 return ret;
3082 if (!return_any)
3083 return 1;
3084 /*
3085 * no higher item found, return the next
3086 * lower instead
3087 */
3088 return_any = 0;
3089 find_higher = 0;
3090 btrfs_release_path(p);
3091 goto again;
3092 }
3093 } else {
Arne Jansene6793762011-09-13 11:18:10 +02003094 if (p->slots[0] == 0) {
3095 ret = btrfs_prev_leaf(root, p);
3096 if (ret < 0)
3097 return ret;
3098 if (!ret) {
Filipe David Borba Manana23c6bf62014-01-11 21:28:54 +00003099 leaf = p->nodes[0];
3100 if (p->slots[0] == btrfs_header_nritems(leaf))
3101 p->slots[0]--;
Arne Jansene6793762011-09-13 11:18:10 +02003102 return 0;
Arne Jansen2f38b3e2011-09-13 11:18:10 +02003103 }
Arne Jansene6793762011-09-13 11:18:10 +02003104 if (!return_any)
3105 return 1;
3106 /*
3107 * no lower item found, return the next
3108 * higher instead
3109 */
3110 return_any = 0;
3111 find_higher = 1;
3112 btrfs_release_path(p);
3113 goto again;
3114 } else {
Arne Jansen2f38b3e2011-09-13 11:18:10 +02003115 --p->slots[0];
3116 }
3117 }
3118 return 0;
3119}
3120
3121/*
Chris Mason74123bd2007-02-02 11:05:29 -05003122 * adjust the pointers going up the tree, starting at level
3123 * making sure the right key of each node is points to 'key'.
3124 * This is used after shifting pointers to the left, so it stops
3125 * fixing up pointers when a given leaf/node is not in slot 0 of the
3126 * higher levels
Chris Masonaa5d6be2007-02-28 16:35:06 -05003127 *
Chris Mason74123bd2007-02-02 11:05:29 -05003128 */
Tsutomu Itohd6a0a122013-04-16 05:18:02 +00003129static void fixup_low_keys(struct btrfs_root *root, struct btrfs_path *path,
Jeff Mahoney143bede2012-03-01 14:56:26 +01003130 struct btrfs_disk_key *key, int level)
Chris Masonbe0e5c02007-01-26 15:51:26 -05003131{
3132 int i;
Chris Mason5f39d392007-10-15 16:14:19 -04003133 struct extent_buffer *t;
3134
Chris Mason234b63a2007-03-13 10:46:10 -04003135 for (i = level; i < BTRFS_MAX_LEVEL; i++) {
Chris Masonbe0e5c02007-01-26 15:51:26 -05003136 int tslot = path->slots[i];
Chris Masoneb60cea2007-02-02 09:18:22 -05003137 if (!path->nodes[i])
Chris Masonbe0e5c02007-01-26 15:51:26 -05003138 break;
Chris Mason5f39d392007-10-15 16:14:19 -04003139 t = path->nodes[i];
Liu Bo32adf092012-10-19 12:52:15 +00003140 tree_mod_log_set_node_key(root->fs_info, t, tslot, 1);
Chris Mason5f39d392007-10-15 16:14:19 -04003141 btrfs_set_node_key(t, key, tslot);
Chris Masond6025572007-03-30 14:27:56 -04003142 btrfs_mark_buffer_dirty(path->nodes[i]);
Chris Masonbe0e5c02007-01-26 15:51:26 -05003143 if (tslot != 0)
3144 break;
3145 }
3146}
3147
Chris Mason74123bd2007-02-02 11:05:29 -05003148/*
Zheng Yan31840ae2008-09-23 13:14:14 -04003149 * update item key.
3150 *
3151 * This function isn't completely safe. It's the caller's responsibility
3152 * that the new key won't break the order
3153 */
Tsutomu Itohafe5fea2013-04-16 05:18:22 +00003154void btrfs_set_item_key_safe(struct btrfs_root *root, struct btrfs_path *path,
Jeff Mahoney143bede2012-03-01 14:56:26 +01003155 struct btrfs_key *new_key)
Zheng Yan31840ae2008-09-23 13:14:14 -04003156{
3157 struct btrfs_disk_key disk_key;
3158 struct extent_buffer *eb;
3159 int slot;
3160
3161 eb = path->nodes[0];
3162 slot = path->slots[0];
3163 if (slot > 0) {
3164 btrfs_item_key(eb, &disk_key, slot - 1);
Jeff Mahoney143bede2012-03-01 14:56:26 +01003165 BUG_ON(comp_keys(&disk_key, new_key) >= 0);
Zheng Yan31840ae2008-09-23 13:14:14 -04003166 }
3167 if (slot < btrfs_header_nritems(eb) - 1) {
3168 btrfs_item_key(eb, &disk_key, slot + 1);
Jeff Mahoney143bede2012-03-01 14:56:26 +01003169 BUG_ON(comp_keys(&disk_key, new_key) <= 0);
Zheng Yan31840ae2008-09-23 13:14:14 -04003170 }
3171
3172 btrfs_cpu_key_to_disk(&disk_key, new_key);
3173 btrfs_set_item_key(eb, &disk_key, slot);
3174 btrfs_mark_buffer_dirty(eb);
3175 if (slot == 0)
Tsutomu Itohd6a0a122013-04-16 05:18:02 +00003176 fixup_low_keys(root, path, &disk_key, 1);
Zheng Yan31840ae2008-09-23 13:14:14 -04003177}
3178
3179/*
Chris Mason74123bd2007-02-02 11:05:29 -05003180 * try to push data from one node into the next node left in the
Chris Mason79f95c82007-03-01 15:16:26 -05003181 * tree.
Chris Masonaa5d6be2007-02-28 16:35:06 -05003182 *
3183 * returns 0 if some ptrs were pushed left, < 0 if there was some horrible
3184 * error, and > 0 if there was no room in the left hand block.
Chris Mason74123bd2007-02-02 11:05:29 -05003185 */
Chris Mason98ed5172008-01-03 10:01:48 -05003186static int push_node_left(struct btrfs_trans_handle *trans,
3187 struct btrfs_root *root, struct extent_buffer *dst,
Chris Mason971a1f62008-04-24 10:54:32 -04003188 struct extent_buffer *src, int empty)
Chris Masonbe0e5c02007-01-26 15:51:26 -05003189{
Chris Masonbe0e5c02007-01-26 15:51:26 -05003190 int push_items = 0;
Chris Masonbb803952007-03-01 12:04:21 -05003191 int src_nritems;
3192 int dst_nritems;
Chris Masonaa5d6be2007-02-28 16:35:06 -05003193 int ret = 0;
Chris Masonbe0e5c02007-01-26 15:51:26 -05003194
Chris Mason5f39d392007-10-15 16:14:19 -04003195 src_nritems = btrfs_header_nritems(src);
3196 dst_nritems = btrfs_header_nritems(dst);
Chris Mason123abc82007-03-14 14:14:43 -04003197 push_items = BTRFS_NODEPTRS_PER_BLOCK(root) - dst_nritems;
Chris Mason7bb86312007-12-11 09:25:06 -05003198 WARN_ON(btrfs_header_generation(src) != trans->transid);
3199 WARN_ON(btrfs_header_generation(dst) != trans->transid);
Chris Mason54aa1f42007-06-22 14:16:25 -04003200
Chris Masonbce4eae2008-04-24 14:42:46 -04003201 if (!empty && src_nritems <= 8)
Chris Mason971a1f62008-04-24 10:54:32 -04003202 return 1;
3203
Chris Masond3977122009-01-05 21:25:51 -05003204 if (push_items <= 0)
Chris Masonbe0e5c02007-01-26 15:51:26 -05003205 return 1;
3206
Chris Masonbce4eae2008-04-24 14:42:46 -04003207 if (empty) {
Chris Mason971a1f62008-04-24 10:54:32 -04003208 push_items = min(src_nritems, push_items);
Chris Masonbce4eae2008-04-24 14:42:46 -04003209 if (push_items < src_nritems) {
3210 /* leave at least 8 pointers in the node if
3211 * we aren't going to empty it
3212 */
3213 if (src_nritems - push_items < 8) {
3214 if (push_items <= 8)
3215 return 1;
3216 push_items -= 8;
3217 }
3218 }
3219 } else
3220 push_items = min(src_nritems - 8, push_items);
Chris Mason79f95c82007-03-01 15:16:26 -05003221
Filipe David Borba Manana5de865e2013-12-20 15:17:46 +00003222 ret = tree_mod_log_eb_copy(root->fs_info, dst, src, dst_nritems, 0,
3223 push_items);
3224 if (ret) {
3225 btrfs_abort_transaction(trans, root, ret);
3226 return ret;
3227 }
Chris Mason5f39d392007-10-15 16:14:19 -04003228 copy_extent_buffer(dst, src,
3229 btrfs_node_key_ptr_offset(dst_nritems),
3230 btrfs_node_key_ptr_offset(0),
Chris Masond3977122009-01-05 21:25:51 -05003231 push_items * sizeof(struct btrfs_key_ptr));
Chris Mason5f39d392007-10-15 16:14:19 -04003232
Chris Masonbb803952007-03-01 12:04:21 -05003233 if (push_items < src_nritems) {
Jan Schmidt57911b82012-10-19 09:22:03 +02003234 /*
3235 * don't call tree_mod_log_eb_move here, key removal was already
3236 * fully logged by tree_mod_log_eb_copy above.
3237 */
Chris Mason5f39d392007-10-15 16:14:19 -04003238 memmove_extent_buffer(src, btrfs_node_key_ptr_offset(0),
3239 btrfs_node_key_ptr_offset(push_items),
3240 (src_nritems - push_items) *
3241 sizeof(struct btrfs_key_ptr));
Chris Masonbb803952007-03-01 12:04:21 -05003242 }
Chris Mason5f39d392007-10-15 16:14:19 -04003243 btrfs_set_header_nritems(src, src_nritems - push_items);
3244 btrfs_set_header_nritems(dst, dst_nritems + push_items);
3245 btrfs_mark_buffer_dirty(src);
3246 btrfs_mark_buffer_dirty(dst);
Zheng Yan31840ae2008-09-23 13:14:14 -04003247
Chris Masonbb803952007-03-01 12:04:21 -05003248 return ret;
Chris Masonbe0e5c02007-01-26 15:51:26 -05003249}
3250
Chris Mason97571fd2007-02-24 13:39:08 -05003251/*
Chris Mason79f95c82007-03-01 15:16:26 -05003252 * try to push data from one node into the next node right in the
3253 * tree.
3254 *
3255 * returns 0 if some ptrs were pushed, < 0 if there was some horrible
3256 * error, and > 0 if there was no room in the right hand block.
3257 *
3258 * this will only push up to 1/2 the contents of the left node over
3259 */
Chris Mason5f39d392007-10-15 16:14:19 -04003260static int balance_node_right(struct btrfs_trans_handle *trans,
3261 struct btrfs_root *root,
3262 struct extent_buffer *dst,
3263 struct extent_buffer *src)
Chris Mason79f95c82007-03-01 15:16:26 -05003264{
Chris Mason79f95c82007-03-01 15:16:26 -05003265 int push_items = 0;
3266 int max_push;
3267 int src_nritems;
3268 int dst_nritems;
3269 int ret = 0;
Chris Mason79f95c82007-03-01 15:16:26 -05003270
Chris Mason7bb86312007-12-11 09:25:06 -05003271 WARN_ON(btrfs_header_generation(src) != trans->transid);
3272 WARN_ON(btrfs_header_generation(dst) != trans->transid);
3273
Chris Mason5f39d392007-10-15 16:14:19 -04003274 src_nritems = btrfs_header_nritems(src);
3275 dst_nritems = btrfs_header_nritems(dst);
Chris Mason123abc82007-03-14 14:14:43 -04003276 push_items = BTRFS_NODEPTRS_PER_BLOCK(root) - dst_nritems;
Chris Masond3977122009-01-05 21:25:51 -05003277 if (push_items <= 0)
Chris Mason79f95c82007-03-01 15:16:26 -05003278 return 1;
Chris Masonbce4eae2008-04-24 14:42:46 -04003279
Chris Masond3977122009-01-05 21:25:51 -05003280 if (src_nritems < 4)
Chris Masonbce4eae2008-04-24 14:42:46 -04003281 return 1;
Chris Mason79f95c82007-03-01 15:16:26 -05003282
3283 max_push = src_nritems / 2 + 1;
3284 /* don't try to empty the node */
Chris Masond3977122009-01-05 21:25:51 -05003285 if (max_push >= src_nritems)
Chris Mason79f95c82007-03-01 15:16:26 -05003286 return 1;
Yan252c38f2007-08-29 09:11:44 -04003287
Chris Mason79f95c82007-03-01 15:16:26 -05003288 if (max_push < push_items)
3289 push_items = max_push;
3290
Jan Schmidtf2304752012-05-26 11:43:17 +02003291 tree_mod_log_eb_move(root->fs_info, dst, push_items, 0, dst_nritems);
Chris Mason5f39d392007-10-15 16:14:19 -04003292 memmove_extent_buffer(dst, btrfs_node_key_ptr_offset(push_items),
3293 btrfs_node_key_ptr_offset(0),
3294 (dst_nritems) *
3295 sizeof(struct btrfs_key_ptr));
Chris Masond6025572007-03-30 14:27:56 -04003296
Filipe David Borba Manana5de865e2013-12-20 15:17:46 +00003297 ret = tree_mod_log_eb_copy(root->fs_info, dst, src, 0,
3298 src_nritems - push_items, push_items);
3299 if (ret) {
3300 btrfs_abort_transaction(trans, root, ret);
3301 return ret;
3302 }
Chris Mason5f39d392007-10-15 16:14:19 -04003303 copy_extent_buffer(dst, src,
3304 btrfs_node_key_ptr_offset(0),
3305 btrfs_node_key_ptr_offset(src_nritems - push_items),
Chris Masond3977122009-01-05 21:25:51 -05003306 push_items * sizeof(struct btrfs_key_ptr));
Chris Mason79f95c82007-03-01 15:16:26 -05003307
Chris Mason5f39d392007-10-15 16:14:19 -04003308 btrfs_set_header_nritems(src, src_nritems - push_items);
3309 btrfs_set_header_nritems(dst, dst_nritems + push_items);
Chris Mason79f95c82007-03-01 15:16:26 -05003310
Chris Mason5f39d392007-10-15 16:14:19 -04003311 btrfs_mark_buffer_dirty(src);
3312 btrfs_mark_buffer_dirty(dst);
Zheng Yan31840ae2008-09-23 13:14:14 -04003313
Chris Mason79f95c82007-03-01 15:16:26 -05003314 return ret;
3315}
3316
3317/*
Chris Mason97571fd2007-02-24 13:39:08 -05003318 * helper function to insert a new root level in the tree.
3319 * A new node is allocated, and a single item is inserted to
3320 * point to the existing root
Chris Masonaa5d6be2007-02-28 16:35:06 -05003321 *
3322 * returns zero on success or < 0 on failure.
Chris Mason97571fd2007-02-24 13:39:08 -05003323 */
Chris Masond3977122009-01-05 21:25:51 -05003324static noinline int insert_new_root(struct btrfs_trans_handle *trans,
Chris Mason5f39d392007-10-15 16:14:19 -04003325 struct btrfs_root *root,
Liu Bofdd99c72013-05-22 12:06:51 +00003326 struct btrfs_path *path, int level)
Chris Mason5c680ed2007-02-22 11:39:13 -05003327{
Chris Mason7bb86312007-12-11 09:25:06 -05003328 u64 lower_gen;
Chris Mason5f39d392007-10-15 16:14:19 -04003329 struct extent_buffer *lower;
3330 struct extent_buffer *c;
Chris Mason925baed2008-06-25 16:01:30 -04003331 struct extent_buffer *old;
Chris Mason5f39d392007-10-15 16:14:19 -04003332 struct btrfs_disk_key lower_key;
Chris Mason5c680ed2007-02-22 11:39:13 -05003333
3334 BUG_ON(path->nodes[level]);
3335 BUG_ON(path->nodes[level-1] != root->node);
3336
Chris Mason7bb86312007-12-11 09:25:06 -05003337 lower = path->nodes[level-1];
3338 if (level == 1)
3339 btrfs_item_key(lower, &lower_key, 0);
3340 else
3341 btrfs_node_key(lower, &lower_key, 0);
3342
David Sterba4d75f8a2014-06-15 01:54:12 +02003343 c = btrfs_alloc_tree_block(trans, root, 0, root->root_key.objectid,
3344 &lower_key, level, root->node->start, 0);
Chris Mason5f39d392007-10-15 16:14:19 -04003345 if (IS_ERR(c))
3346 return PTR_ERR(c);
Chris Mason925baed2008-06-25 16:01:30 -04003347
Yan, Zhengf0486c62010-05-16 10:46:25 -04003348 root_add_used(root, root->nodesize);
3349
Yan Zheng5d4f98a2009-06-10 10:45:14 -04003350 memset_extent_buffer(c, 0, 0, sizeof(struct btrfs_header));
Chris Mason5f39d392007-10-15 16:14:19 -04003351 btrfs_set_header_nritems(c, 1);
3352 btrfs_set_header_level(c, level);
Chris Masondb945352007-10-15 16:15:53 -04003353 btrfs_set_header_bytenr(c, c->start);
Chris Mason5f39d392007-10-15 16:14:19 -04003354 btrfs_set_header_generation(c, trans->transid);
Yan Zheng5d4f98a2009-06-10 10:45:14 -04003355 btrfs_set_header_backref_rev(c, BTRFS_MIXED_BACKREF_REV);
Chris Mason5f39d392007-10-15 16:14:19 -04003356 btrfs_set_header_owner(c, root->root_key.objectid);
Chris Masond5719762007-03-23 10:01:08 -04003357
Ross Kirk0a4e5582013-09-24 10:12:38 +01003358 write_extent_buffer(c, root->fs_info->fsid, btrfs_header_fsid(),
Chris Mason5f39d392007-10-15 16:14:19 -04003359 BTRFS_FSID_SIZE);
Chris Masone17cade2008-04-15 15:41:47 -04003360
3361 write_extent_buffer(c, root->fs_info->chunk_tree_uuid,
Geert Uytterhoevenb308bc22013-08-20 13:20:15 +02003362 btrfs_header_chunk_tree_uuid(c), BTRFS_UUID_SIZE);
Chris Masone17cade2008-04-15 15:41:47 -04003363
Chris Mason5f39d392007-10-15 16:14:19 -04003364 btrfs_set_node_key(c, &lower_key, 0);
Chris Masondb945352007-10-15 16:15:53 -04003365 btrfs_set_node_blockptr(c, 0, lower->start);
Chris Mason7bb86312007-12-11 09:25:06 -05003366 lower_gen = btrfs_header_generation(lower);
Zheng Yan31840ae2008-09-23 13:14:14 -04003367 WARN_ON(lower_gen != trans->transid);
Chris Mason7bb86312007-12-11 09:25:06 -05003368
3369 btrfs_set_node_ptr_generation(c, 0, lower_gen);
Chris Mason5f39d392007-10-15 16:14:19 -04003370
3371 btrfs_mark_buffer_dirty(c);
Chris Masond5719762007-03-23 10:01:08 -04003372
Chris Mason925baed2008-06-25 16:01:30 -04003373 old = root->node;
Liu Bofdd99c72013-05-22 12:06:51 +00003374 tree_mod_log_set_root_pointer(root, c, 0);
Chris Mason240f62c2011-03-23 14:54:42 -04003375 rcu_assign_pointer(root->node, c);
Chris Mason925baed2008-06-25 16:01:30 -04003376
3377 /* the super has an extra ref to root->node */
3378 free_extent_buffer(old);
3379
Chris Mason0b86a832008-03-24 15:01:56 -04003380 add_root_to_dirty_list(root);
Chris Mason5f39d392007-10-15 16:14:19 -04003381 extent_buffer_get(c);
3382 path->nodes[level] = c;
Chris Masonbd681512011-07-16 15:23:14 -04003383 path->locks[level] = BTRFS_WRITE_LOCK;
Chris Mason5c680ed2007-02-22 11:39:13 -05003384 path->slots[level] = 0;
3385 return 0;
3386}
3387
Chris Mason74123bd2007-02-02 11:05:29 -05003388/*
3389 * worker function to insert a single pointer in a node.
3390 * the node should have enough room for the pointer already
Chris Mason97571fd2007-02-24 13:39:08 -05003391 *
Chris Mason74123bd2007-02-02 11:05:29 -05003392 * slot and level indicate where you want the key to go, and
3393 * blocknr is the block the key points to.
3394 */
Jeff Mahoney143bede2012-03-01 14:56:26 +01003395static void insert_ptr(struct btrfs_trans_handle *trans,
3396 struct btrfs_root *root, struct btrfs_path *path,
3397 struct btrfs_disk_key *key, u64 bytenr,
Jan Schmidtc3e06962012-06-21 11:01:06 +02003398 int slot, int level)
Chris Mason74123bd2007-02-02 11:05:29 -05003399{
Chris Mason5f39d392007-10-15 16:14:19 -04003400 struct extent_buffer *lower;
Chris Mason74123bd2007-02-02 11:05:29 -05003401 int nritems;
Jan Schmidtf3ea38d2012-05-26 11:45:21 +02003402 int ret;
Chris Mason5c680ed2007-02-22 11:39:13 -05003403
3404 BUG_ON(!path->nodes[level]);
Yan, Zhengf0486c62010-05-16 10:46:25 -04003405 btrfs_assert_tree_locked(path->nodes[level]);
Chris Mason5f39d392007-10-15 16:14:19 -04003406 lower = path->nodes[level];
3407 nritems = btrfs_header_nritems(lower);
Stoyan Gaydarovc2934982009-04-02 17:05:11 -04003408 BUG_ON(slot > nritems);
Jeff Mahoney143bede2012-03-01 14:56:26 +01003409 BUG_ON(nritems == BTRFS_NODEPTRS_PER_BLOCK(root));
Chris Mason74123bd2007-02-02 11:05:29 -05003410 if (slot != nritems) {
Jan Schmidtc3e06962012-06-21 11:01:06 +02003411 if (level)
Jan Schmidtf3ea38d2012-05-26 11:45:21 +02003412 tree_mod_log_eb_move(root->fs_info, lower, slot + 1,
3413 slot, nritems - slot);
Chris Mason5f39d392007-10-15 16:14:19 -04003414 memmove_extent_buffer(lower,
3415 btrfs_node_key_ptr_offset(slot + 1),
3416 btrfs_node_key_ptr_offset(slot),
Chris Masond6025572007-03-30 14:27:56 -04003417 (nritems - slot) * sizeof(struct btrfs_key_ptr));
Chris Mason74123bd2007-02-02 11:05:29 -05003418 }
Jan Schmidtc3e06962012-06-21 11:01:06 +02003419 if (level) {
Jan Schmidtf3ea38d2012-05-26 11:45:21 +02003420 ret = tree_mod_log_insert_key(root->fs_info, lower, slot,
Josef Bacikc8cc6342013-07-01 16:18:19 -04003421 MOD_LOG_KEY_ADD, GFP_NOFS);
Jan Schmidtf3ea38d2012-05-26 11:45:21 +02003422 BUG_ON(ret < 0);
3423 }
Chris Mason5f39d392007-10-15 16:14:19 -04003424 btrfs_set_node_key(lower, key, slot);
Chris Masondb945352007-10-15 16:15:53 -04003425 btrfs_set_node_blockptr(lower, slot, bytenr);
Chris Mason74493f72007-12-11 09:25:06 -05003426 WARN_ON(trans->transid == 0);
3427 btrfs_set_node_ptr_generation(lower, slot, trans->transid);
Chris Mason5f39d392007-10-15 16:14:19 -04003428 btrfs_set_header_nritems(lower, nritems + 1);
3429 btrfs_mark_buffer_dirty(lower);
Chris Mason74123bd2007-02-02 11:05:29 -05003430}
3431
Chris Mason97571fd2007-02-24 13:39:08 -05003432/*
3433 * split the node at the specified level in path in two.
3434 * The path is corrected to point to the appropriate node after the split
3435 *
3436 * Before splitting this tries to make some room in the node by pushing
3437 * left and right, if either one works, it returns right away.
Chris Masonaa5d6be2007-02-28 16:35:06 -05003438 *
3439 * returns 0 on success and < 0 on failure
Chris Mason97571fd2007-02-24 13:39:08 -05003440 */
Chris Masone02119d2008-09-05 16:13:11 -04003441static noinline int split_node(struct btrfs_trans_handle *trans,
3442 struct btrfs_root *root,
3443 struct btrfs_path *path, int level)
Chris Masonbe0e5c02007-01-26 15:51:26 -05003444{
Chris Mason5f39d392007-10-15 16:14:19 -04003445 struct extent_buffer *c;
3446 struct extent_buffer *split;
3447 struct btrfs_disk_key disk_key;
Chris Masonbe0e5c02007-01-26 15:51:26 -05003448 int mid;
Chris Mason5c680ed2007-02-22 11:39:13 -05003449 int ret;
Chris Mason7518a232007-03-12 12:01:18 -04003450 u32 c_nritems;
Chris Masonbe0e5c02007-01-26 15:51:26 -05003451
Chris Mason5f39d392007-10-15 16:14:19 -04003452 c = path->nodes[level];
Chris Mason7bb86312007-12-11 09:25:06 -05003453 WARN_ON(btrfs_header_generation(c) != trans->transid);
Chris Mason5f39d392007-10-15 16:14:19 -04003454 if (c == root->node) {
Jan Schmidtd9abbf12013-03-20 13:49:48 +00003455 /*
Jan Schmidt90f8d622013-04-13 13:19:53 +00003456 * trying to split the root, lets make a new one
3457 *
Liu Bofdd99c72013-05-22 12:06:51 +00003458 * tree mod log: We don't log_removal old root in
Jan Schmidt90f8d622013-04-13 13:19:53 +00003459 * insert_new_root, because that root buffer will be kept as a
3460 * normal node. We are going to log removal of half of the
3461 * elements below with tree_mod_log_eb_copy. We're holding a
3462 * tree lock on the buffer, which is why we cannot race with
3463 * other tree_mod_log users.
Jan Schmidtd9abbf12013-03-20 13:49:48 +00003464 */
Liu Bofdd99c72013-05-22 12:06:51 +00003465 ret = insert_new_root(trans, root, path, level + 1);
Chris Mason5c680ed2007-02-22 11:39:13 -05003466 if (ret)
3467 return ret;
Chris Masonb3612422009-05-13 19:12:15 -04003468 } else {
Chris Masone66f7092007-04-20 13:16:02 -04003469 ret = push_nodes_for_insert(trans, root, path, level);
Chris Mason5f39d392007-10-15 16:14:19 -04003470 c = path->nodes[level];
3471 if (!ret && btrfs_header_nritems(c) <
Chris Masonc448acf2008-04-24 09:34:34 -04003472 BTRFS_NODEPTRS_PER_BLOCK(root) - 3)
Chris Masone66f7092007-04-20 13:16:02 -04003473 return 0;
Chris Mason54aa1f42007-06-22 14:16:25 -04003474 if (ret < 0)
3475 return ret;
Chris Masonbe0e5c02007-01-26 15:51:26 -05003476 }
Chris Masone66f7092007-04-20 13:16:02 -04003477
Chris Mason5f39d392007-10-15 16:14:19 -04003478 c_nritems = btrfs_header_nritems(c);
Yan Zheng5d4f98a2009-06-10 10:45:14 -04003479 mid = (c_nritems + 1) / 2;
3480 btrfs_node_key(c, &disk_key, mid);
Chris Mason7bb86312007-12-11 09:25:06 -05003481
David Sterba4d75f8a2014-06-15 01:54:12 +02003482 split = btrfs_alloc_tree_block(trans, root, 0, root->root_key.objectid,
3483 &disk_key, level, c->start, 0);
Chris Mason5f39d392007-10-15 16:14:19 -04003484 if (IS_ERR(split))
3485 return PTR_ERR(split);
Chris Mason54aa1f42007-06-22 14:16:25 -04003486
Yan, Zhengf0486c62010-05-16 10:46:25 -04003487 root_add_used(root, root->nodesize);
3488
Yan Zheng5d4f98a2009-06-10 10:45:14 -04003489 memset_extent_buffer(split, 0, 0, sizeof(struct btrfs_header));
Chris Mason5f39d392007-10-15 16:14:19 -04003490 btrfs_set_header_level(split, btrfs_header_level(c));
Chris Masondb945352007-10-15 16:15:53 -04003491 btrfs_set_header_bytenr(split, split->start);
Chris Mason5f39d392007-10-15 16:14:19 -04003492 btrfs_set_header_generation(split, trans->transid);
Yan Zheng5d4f98a2009-06-10 10:45:14 -04003493 btrfs_set_header_backref_rev(split, BTRFS_MIXED_BACKREF_REV);
Chris Mason5f39d392007-10-15 16:14:19 -04003494 btrfs_set_header_owner(split, root->root_key.objectid);
3495 write_extent_buffer(split, root->fs_info->fsid,
Ross Kirk0a4e5582013-09-24 10:12:38 +01003496 btrfs_header_fsid(), BTRFS_FSID_SIZE);
Chris Masone17cade2008-04-15 15:41:47 -04003497 write_extent_buffer(split, root->fs_info->chunk_tree_uuid,
Geert Uytterhoevenb308bc22013-08-20 13:20:15 +02003498 btrfs_header_chunk_tree_uuid(split),
Chris Masone17cade2008-04-15 15:41:47 -04003499 BTRFS_UUID_SIZE);
Chris Mason5f39d392007-10-15 16:14:19 -04003500
Filipe David Borba Manana5de865e2013-12-20 15:17:46 +00003501 ret = tree_mod_log_eb_copy(root->fs_info, split, c, 0,
3502 mid, c_nritems - mid);
3503 if (ret) {
3504 btrfs_abort_transaction(trans, root, ret);
3505 return ret;
3506 }
Chris Mason5f39d392007-10-15 16:14:19 -04003507 copy_extent_buffer(split, c,
3508 btrfs_node_key_ptr_offset(0),
3509 btrfs_node_key_ptr_offset(mid),
3510 (c_nritems - mid) * sizeof(struct btrfs_key_ptr));
3511 btrfs_set_header_nritems(split, c_nritems - mid);
3512 btrfs_set_header_nritems(c, mid);
Chris Masonaa5d6be2007-02-28 16:35:06 -05003513 ret = 0;
3514
Chris Mason5f39d392007-10-15 16:14:19 -04003515 btrfs_mark_buffer_dirty(c);
3516 btrfs_mark_buffer_dirty(split);
3517
Jeff Mahoney143bede2012-03-01 14:56:26 +01003518 insert_ptr(trans, root, path, &disk_key, split->start,
Jan Schmidtc3e06962012-06-21 11:01:06 +02003519 path->slots[level + 1] + 1, level + 1);
Chris Masonaa5d6be2007-02-28 16:35:06 -05003520
Chris Mason5de08d72007-02-24 06:24:44 -05003521 if (path->slots[level] >= mid) {
Chris Mason5c680ed2007-02-22 11:39:13 -05003522 path->slots[level] -= mid;
Chris Mason925baed2008-06-25 16:01:30 -04003523 btrfs_tree_unlock(c);
Chris Mason5f39d392007-10-15 16:14:19 -04003524 free_extent_buffer(c);
3525 path->nodes[level] = split;
Chris Mason5c680ed2007-02-22 11:39:13 -05003526 path->slots[level + 1] += 1;
3527 } else {
Chris Mason925baed2008-06-25 16:01:30 -04003528 btrfs_tree_unlock(split);
Chris Mason5f39d392007-10-15 16:14:19 -04003529 free_extent_buffer(split);
Chris Masonbe0e5c02007-01-26 15:51:26 -05003530 }
Chris Masonaa5d6be2007-02-28 16:35:06 -05003531 return ret;
Chris Masonbe0e5c02007-01-26 15:51:26 -05003532}
3533
Chris Mason74123bd2007-02-02 11:05:29 -05003534/*
3535 * how many bytes are required to store the items in a leaf. start
3536 * and nr indicate which items in the leaf to check. This totals up the
3537 * space used both by the item structs and the item data
3538 */
Chris Mason5f39d392007-10-15 16:14:19 -04003539static int leaf_space_used(struct extent_buffer *l, int start, int nr)
Chris Masonbe0e5c02007-01-26 15:51:26 -05003540{
Josef Bacik41be1f32012-10-15 13:43:18 -04003541 struct btrfs_item *start_item;
3542 struct btrfs_item *end_item;
3543 struct btrfs_map_token token;
Chris Masonbe0e5c02007-01-26 15:51:26 -05003544 int data_len;
Chris Mason5f39d392007-10-15 16:14:19 -04003545 int nritems = btrfs_header_nritems(l);
Chris Masond4dbff92007-04-04 14:08:15 -04003546 int end = min(nritems, start + nr) - 1;
Chris Masonbe0e5c02007-01-26 15:51:26 -05003547
3548 if (!nr)
3549 return 0;
Josef Bacik41be1f32012-10-15 13:43:18 -04003550 btrfs_init_map_token(&token);
Ross Kirkdd3cc162013-09-16 15:58:09 +01003551 start_item = btrfs_item_nr(start);
3552 end_item = btrfs_item_nr(end);
Josef Bacik41be1f32012-10-15 13:43:18 -04003553 data_len = btrfs_token_item_offset(l, start_item, &token) +
3554 btrfs_token_item_size(l, start_item, &token);
3555 data_len = data_len - btrfs_token_item_offset(l, end_item, &token);
Chris Mason0783fcf2007-03-12 20:12:07 -04003556 data_len += sizeof(struct btrfs_item) * nr;
Chris Masond4dbff92007-04-04 14:08:15 -04003557 WARN_ON(data_len < 0);
Chris Masonbe0e5c02007-01-26 15:51:26 -05003558 return data_len;
3559}
3560
Chris Mason74123bd2007-02-02 11:05:29 -05003561/*
Chris Masond4dbff92007-04-04 14:08:15 -04003562 * The space between the end of the leaf items and
3563 * the start of the leaf data. IOW, how much room
3564 * the leaf has left for both items and data
3565 */
Chris Masond3977122009-01-05 21:25:51 -05003566noinline int btrfs_leaf_free_space(struct btrfs_root *root,
Chris Masone02119d2008-09-05 16:13:11 -04003567 struct extent_buffer *leaf)
Chris Masond4dbff92007-04-04 14:08:15 -04003568{
Chris Mason5f39d392007-10-15 16:14:19 -04003569 int nritems = btrfs_header_nritems(leaf);
3570 int ret;
3571 ret = BTRFS_LEAF_DATA_SIZE(root) - leaf_space_used(leaf, 0, nritems);
3572 if (ret < 0) {
Frank Holtonefe120a2013-12-20 11:37:06 -05003573 btrfs_crit(root->fs_info,
3574 "leaf free space ret %d, leaf data size %lu, used %d nritems %d",
Jens Axboeae2f5412007-10-19 09:22:59 -04003575 ret, (unsigned long) BTRFS_LEAF_DATA_SIZE(root),
Chris Mason5f39d392007-10-15 16:14:19 -04003576 leaf_space_used(leaf, 0, nritems), nritems);
3577 }
3578 return ret;
Chris Masond4dbff92007-04-04 14:08:15 -04003579}
3580
Chris Mason99d8f832010-07-07 10:51:48 -04003581/*
3582 * min slot controls the lowest index we're willing to push to the
3583 * right. We'll push up to and including min_slot, but no lower
3584 */
Chris Mason44871b12009-03-13 10:04:31 -04003585static noinline int __push_leaf_right(struct btrfs_trans_handle *trans,
3586 struct btrfs_root *root,
3587 struct btrfs_path *path,
3588 int data_size, int empty,
3589 struct extent_buffer *right,
Chris Mason99d8f832010-07-07 10:51:48 -04003590 int free_space, u32 left_nritems,
3591 u32 min_slot)
Chris Mason00ec4c52007-02-24 12:47:20 -05003592{
Chris Mason5f39d392007-10-15 16:14:19 -04003593 struct extent_buffer *left = path->nodes[0];
Chris Mason44871b12009-03-13 10:04:31 -04003594 struct extent_buffer *upper = path->nodes[1];
Chris Masoncfed81a2012-03-03 07:40:03 -05003595 struct btrfs_map_token token;
Chris Mason5f39d392007-10-15 16:14:19 -04003596 struct btrfs_disk_key disk_key;
Chris Mason00ec4c52007-02-24 12:47:20 -05003597 int slot;
Chris Mason34a38212007-11-07 13:31:03 -05003598 u32 i;
Chris Mason00ec4c52007-02-24 12:47:20 -05003599 int push_space = 0;
3600 int push_items = 0;
Chris Mason0783fcf2007-03-12 20:12:07 -04003601 struct btrfs_item *item;
Chris Mason34a38212007-11-07 13:31:03 -05003602 u32 nr;
Chris Mason7518a232007-03-12 12:01:18 -04003603 u32 right_nritems;
Chris Mason5f39d392007-10-15 16:14:19 -04003604 u32 data_end;
Chris Masondb945352007-10-15 16:15:53 -04003605 u32 this_item_size;
Chris Mason00ec4c52007-02-24 12:47:20 -05003606
Chris Masoncfed81a2012-03-03 07:40:03 -05003607 btrfs_init_map_token(&token);
3608
Chris Mason34a38212007-11-07 13:31:03 -05003609 if (empty)
3610 nr = 0;
3611 else
Chris Mason99d8f832010-07-07 10:51:48 -04003612 nr = max_t(u32, 1, min_slot);
Chris Mason34a38212007-11-07 13:31:03 -05003613
Zheng Yan31840ae2008-09-23 13:14:14 -04003614 if (path->slots[0] >= left_nritems)
Yan Zheng87b29b22008-12-17 10:21:48 -05003615 push_space += data_size;
Zheng Yan31840ae2008-09-23 13:14:14 -04003616
Chris Mason44871b12009-03-13 10:04:31 -04003617 slot = path->slots[1];
Chris Mason34a38212007-11-07 13:31:03 -05003618 i = left_nritems - 1;
3619 while (i >= nr) {
Ross Kirkdd3cc162013-09-16 15:58:09 +01003620 item = btrfs_item_nr(i);
Chris Masondb945352007-10-15 16:15:53 -04003621
Zheng Yan31840ae2008-09-23 13:14:14 -04003622 if (!empty && push_items > 0) {
3623 if (path->slots[0] > i)
3624 break;
3625 if (path->slots[0] == i) {
3626 int space = btrfs_leaf_free_space(root, left);
3627 if (space + push_space * 2 > free_space)
3628 break;
3629 }
3630 }
3631
Chris Mason00ec4c52007-02-24 12:47:20 -05003632 if (path->slots[0] == i)
Yan Zheng87b29b22008-12-17 10:21:48 -05003633 push_space += data_size;
Chris Masondb945352007-10-15 16:15:53 -04003634
Chris Masondb945352007-10-15 16:15:53 -04003635 this_item_size = btrfs_item_size(left, item);
3636 if (this_item_size + sizeof(*item) + push_space > free_space)
Chris Mason00ec4c52007-02-24 12:47:20 -05003637 break;
Zheng Yan31840ae2008-09-23 13:14:14 -04003638
Chris Mason00ec4c52007-02-24 12:47:20 -05003639 push_items++;
Chris Masondb945352007-10-15 16:15:53 -04003640 push_space += this_item_size + sizeof(*item);
Chris Mason34a38212007-11-07 13:31:03 -05003641 if (i == 0)
3642 break;
3643 i--;
Chris Masondb945352007-10-15 16:15:53 -04003644 }
Chris Mason5f39d392007-10-15 16:14:19 -04003645
Chris Mason925baed2008-06-25 16:01:30 -04003646 if (push_items == 0)
3647 goto out_unlock;
Chris Mason5f39d392007-10-15 16:14:19 -04003648
Julia Lawall6c1500f2012-11-03 20:30:18 +00003649 WARN_ON(!empty && push_items == left_nritems);
Chris Mason5f39d392007-10-15 16:14:19 -04003650
Chris Mason00ec4c52007-02-24 12:47:20 -05003651 /* push left to right */
Chris Mason5f39d392007-10-15 16:14:19 -04003652 right_nritems = btrfs_header_nritems(right);
Chris Mason34a38212007-11-07 13:31:03 -05003653
Chris Mason5f39d392007-10-15 16:14:19 -04003654 push_space = btrfs_item_end_nr(left, left_nritems - push_items);
Chris Mason123abc82007-03-14 14:14:43 -04003655 push_space -= leaf_data_end(root, left);
Chris Mason5f39d392007-10-15 16:14:19 -04003656
Chris Mason00ec4c52007-02-24 12:47:20 -05003657 /* make room in the right data area */
Chris Mason5f39d392007-10-15 16:14:19 -04003658 data_end = leaf_data_end(root, right);
3659 memmove_extent_buffer(right,
3660 btrfs_leaf_data(right) + data_end - push_space,
3661 btrfs_leaf_data(right) + data_end,
3662 BTRFS_LEAF_DATA_SIZE(root) - data_end);
3663
Chris Mason00ec4c52007-02-24 12:47:20 -05003664 /* copy from the left data area */
Chris Mason5f39d392007-10-15 16:14:19 -04003665 copy_extent_buffer(right, left, btrfs_leaf_data(right) +
Chris Masond6025572007-03-30 14:27:56 -04003666 BTRFS_LEAF_DATA_SIZE(root) - push_space,
3667 btrfs_leaf_data(left) + leaf_data_end(root, left),
3668 push_space);
Chris Mason5f39d392007-10-15 16:14:19 -04003669
3670 memmove_extent_buffer(right, btrfs_item_nr_offset(push_items),
3671 btrfs_item_nr_offset(0),
3672 right_nritems * sizeof(struct btrfs_item));
3673
Chris Mason00ec4c52007-02-24 12:47:20 -05003674 /* copy the items from left to right */
Chris Mason5f39d392007-10-15 16:14:19 -04003675 copy_extent_buffer(right, left, btrfs_item_nr_offset(0),
3676 btrfs_item_nr_offset(left_nritems - push_items),
3677 push_items * sizeof(struct btrfs_item));
Chris Mason00ec4c52007-02-24 12:47:20 -05003678
3679 /* update the item pointers */
Chris Mason7518a232007-03-12 12:01:18 -04003680 right_nritems += push_items;
Chris Mason5f39d392007-10-15 16:14:19 -04003681 btrfs_set_header_nritems(right, right_nritems);
Chris Mason123abc82007-03-14 14:14:43 -04003682 push_space = BTRFS_LEAF_DATA_SIZE(root);
Chris Mason7518a232007-03-12 12:01:18 -04003683 for (i = 0; i < right_nritems; i++) {
Ross Kirkdd3cc162013-09-16 15:58:09 +01003684 item = btrfs_item_nr(i);
Chris Masoncfed81a2012-03-03 07:40:03 -05003685 push_space -= btrfs_token_item_size(right, item, &token);
3686 btrfs_set_token_item_offset(right, item, push_space, &token);
Chris Masondb945352007-10-15 16:15:53 -04003687 }
3688
Chris Mason7518a232007-03-12 12:01:18 -04003689 left_nritems -= push_items;
Chris Mason5f39d392007-10-15 16:14:19 -04003690 btrfs_set_header_nritems(left, left_nritems);
Chris Mason00ec4c52007-02-24 12:47:20 -05003691
Chris Mason34a38212007-11-07 13:31:03 -05003692 if (left_nritems)
3693 btrfs_mark_buffer_dirty(left);
Yan, Zhengf0486c62010-05-16 10:46:25 -04003694 else
3695 clean_tree_block(trans, root, left);
3696
Chris Mason5f39d392007-10-15 16:14:19 -04003697 btrfs_mark_buffer_dirty(right);
Chris Masona429e512007-04-18 16:15:28 -04003698
Chris Mason5f39d392007-10-15 16:14:19 -04003699 btrfs_item_key(right, &disk_key, 0);
3700 btrfs_set_node_key(upper, &disk_key, slot + 1);
Chris Masond6025572007-03-30 14:27:56 -04003701 btrfs_mark_buffer_dirty(upper);
Chris Mason02217ed2007-03-02 16:08:05 -05003702
Chris Mason00ec4c52007-02-24 12:47:20 -05003703 /* then fixup the leaf pointer in the path */
Chris Mason7518a232007-03-12 12:01:18 -04003704 if (path->slots[0] >= left_nritems) {
3705 path->slots[0] -= left_nritems;
Chris Mason925baed2008-06-25 16:01:30 -04003706 if (btrfs_header_nritems(path->nodes[0]) == 0)
3707 clean_tree_block(trans, root, path->nodes[0]);
3708 btrfs_tree_unlock(path->nodes[0]);
Chris Mason5f39d392007-10-15 16:14:19 -04003709 free_extent_buffer(path->nodes[0]);
3710 path->nodes[0] = right;
Chris Mason00ec4c52007-02-24 12:47:20 -05003711 path->slots[1] += 1;
3712 } else {
Chris Mason925baed2008-06-25 16:01:30 -04003713 btrfs_tree_unlock(right);
Chris Mason5f39d392007-10-15 16:14:19 -04003714 free_extent_buffer(right);
Chris Mason00ec4c52007-02-24 12:47:20 -05003715 }
3716 return 0;
Chris Mason925baed2008-06-25 16:01:30 -04003717
3718out_unlock:
3719 btrfs_tree_unlock(right);
3720 free_extent_buffer(right);
3721 return 1;
Chris Mason00ec4c52007-02-24 12:47:20 -05003722}
Chris Mason925baed2008-06-25 16:01:30 -04003723
Chris Mason00ec4c52007-02-24 12:47:20 -05003724/*
Chris Mason44871b12009-03-13 10:04:31 -04003725 * push some data in the path leaf to the right, trying to free up at
3726 * least data_size bytes. returns zero if the push worked, nonzero otherwise
3727 *
3728 * returns 1 if the push failed because the other node didn't have enough
3729 * room, 0 if everything worked out and < 0 if there were major errors.
Chris Mason99d8f832010-07-07 10:51:48 -04003730 *
3731 * this will push starting from min_slot to the end of the leaf. It won't
3732 * push any slot lower than min_slot
Chris Mason44871b12009-03-13 10:04:31 -04003733 */
3734static int push_leaf_right(struct btrfs_trans_handle *trans, struct btrfs_root
Chris Mason99d8f832010-07-07 10:51:48 -04003735 *root, struct btrfs_path *path,
3736 int min_data_size, int data_size,
3737 int empty, u32 min_slot)
Chris Mason44871b12009-03-13 10:04:31 -04003738{
3739 struct extent_buffer *left = path->nodes[0];
3740 struct extent_buffer *right;
3741 struct extent_buffer *upper;
3742 int slot;
3743 int free_space;
3744 u32 left_nritems;
3745 int ret;
3746
3747 if (!path->nodes[1])
3748 return 1;
3749
3750 slot = path->slots[1];
3751 upper = path->nodes[1];
3752 if (slot >= btrfs_header_nritems(upper) - 1)
3753 return 1;
3754
3755 btrfs_assert_tree_locked(path->nodes[1]);
3756
3757 right = read_node_slot(root, upper, slot + 1);
Tsutomu Itoh91ca3382011-01-05 02:32:22 +00003758 if (right == NULL)
3759 return 1;
3760
Chris Mason44871b12009-03-13 10:04:31 -04003761 btrfs_tree_lock(right);
3762 btrfs_set_lock_blocking(right);
3763
3764 free_space = btrfs_leaf_free_space(root, right);
3765 if (free_space < data_size)
3766 goto out_unlock;
3767
3768 /* cow and double check */
3769 ret = btrfs_cow_block(trans, root, right, upper,
3770 slot + 1, &right);
3771 if (ret)
3772 goto out_unlock;
3773
3774 free_space = btrfs_leaf_free_space(root, right);
3775 if (free_space < data_size)
3776 goto out_unlock;
3777
3778 left_nritems = btrfs_header_nritems(left);
3779 if (left_nritems == 0)
3780 goto out_unlock;
3781
Filipe David Borba Manana2ef1fed2013-12-04 22:17:39 +00003782 if (path->slots[0] == left_nritems && !empty) {
3783 /* Key greater than all keys in the leaf, right neighbor has
3784 * enough room for it and we're not emptying our leaf to delete
3785 * it, therefore use right neighbor to insert the new item and
3786 * no need to touch/dirty our left leaft. */
3787 btrfs_tree_unlock(left);
3788 free_extent_buffer(left);
3789 path->nodes[0] = right;
3790 path->slots[0] = 0;
3791 path->slots[1]++;
3792 return 0;
3793 }
3794
Chris Mason99d8f832010-07-07 10:51:48 -04003795 return __push_leaf_right(trans, root, path, min_data_size, empty,
3796 right, free_space, left_nritems, min_slot);
Chris Mason44871b12009-03-13 10:04:31 -04003797out_unlock:
3798 btrfs_tree_unlock(right);
3799 free_extent_buffer(right);
3800 return 1;
3801}
3802
3803/*
Chris Mason74123bd2007-02-02 11:05:29 -05003804 * push some data in the path leaf to the left, trying to free up at
3805 * least data_size bytes. returns zero if the push worked, nonzero otherwise
Chris Mason99d8f832010-07-07 10:51:48 -04003806 *
3807 * max_slot can put a limit on how far into the leaf we'll push items. The
3808 * item at 'max_slot' won't be touched. Use (u32)-1 to make us do all the
3809 * items
Chris Mason74123bd2007-02-02 11:05:29 -05003810 */
Chris Mason44871b12009-03-13 10:04:31 -04003811static noinline int __push_leaf_left(struct btrfs_trans_handle *trans,
3812 struct btrfs_root *root,
3813 struct btrfs_path *path, int data_size,
3814 int empty, struct extent_buffer *left,
Chris Mason99d8f832010-07-07 10:51:48 -04003815 int free_space, u32 right_nritems,
3816 u32 max_slot)
Chris Masonbe0e5c02007-01-26 15:51:26 -05003817{
Chris Mason5f39d392007-10-15 16:14:19 -04003818 struct btrfs_disk_key disk_key;
3819 struct extent_buffer *right = path->nodes[0];
Chris Masonbe0e5c02007-01-26 15:51:26 -05003820 int i;
Chris Masonbe0e5c02007-01-26 15:51:26 -05003821 int push_space = 0;
3822 int push_items = 0;
Chris Mason0783fcf2007-03-12 20:12:07 -04003823 struct btrfs_item *item;
Chris Mason7518a232007-03-12 12:01:18 -04003824 u32 old_left_nritems;
Chris Mason34a38212007-11-07 13:31:03 -05003825 u32 nr;
Chris Masonaa5d6be2007-02-28 16:35:06 -05003826 int ret = 0;
Chris Masondb945352007-10-15 16:15:53 -04003827 u32 this_item_size;
3828 u32 old_left_item_size;
Chris Masoncfed81a2012-03-03 07:40:03 -05003829 struct btrfs_map_token token;
3830
3831 btrfs_init_map_token(&token);
Chris Masonbe0e5c02007-01-26 15:51:26 -05003832
Chris Mason34a38212007-11-07 13:31:03 -05003833 if (empty)
Chris Mason99d8f832010-07-07 10:51:48 -04003834 nr = min(right_nritems, max_slot);
Chris Mason34a38212007-11-07 13:31:03 -05003835 else
Chris Mason99d8f832010-07-07 10:51:48 -04003836 nr = min(right_nritems - 1, max_slot);
Chris Mason34a38212007-11-07 13:31:03 -05003837
3838 for (i = 0; i < nr; i++) {
Ross Kirkdd3cc162013-09-16 15:58:09 +01003839 item = btrfs_item_nr(i);
Chris Masondb945352007-10-15 16:15:53 -04003840
Zheng Yan31840ae2008-09-23 13:14:14 -04003841 if (!empty && push_items > 0) {
3842 if (path->slots[0] < i)
3843 break;
3844 if (path->slots[0] == i) {
3845 int space = btrfs_leaf_free_space(root, right);
3846 if (space + push_space * 2 > free_space)
3847 break;
3848 }
3849 }
3850
Chris Masonbe0e5c02007-01-26 15:51:26 -05003851 if (path->slots[0] == i)
Yan Zheng87b29b22008-12-17 10:21:48 -05003852 push_space += data_size;
Chris Masondb945352007-10-15 16:15:53 -04003853
3854 this_item_size = btrfs_item_size(right, item);
3855 if (this_item_size + sizeof(*item) + push_space > free_space)
Chris Masonbe0e5c02007-01-26 15:51:26 -05003856 break;
Chris Masondb945352007-10-15 16:15:53 -04003857
Chris Masonbe0e5c02007-01-26 15:51:26 -05003858 push_items++;
Chris Masondb945352007-10-15 16:15:53 -04003859 push_space += this_item_size + sizeof(*item);
Chris Masonbe0e5c02007-01-26 15:51:26 -05003860 }
Chris Masondb945352007-10-15 16:15:53 -04003861
Chris Masonbe0e5c02007-01-26 15:51:26 -05003862 if (push_items == 0) {
Chris Mason925baed2008-06-25 16:01:30 -04003863 ret = 1;
3864 goto out;
Chris Masonbe0e5c02007-01-26 15:51:26 -05003865 }
Dulshani Gunawardhanafae7f212013-10-31 10:30:08 +05303866 WARN_ON(!empty && push_items == btrfs_header_nritems(right));
Chris Mason5f39d392007-10-15 16:14:19 -04003867
Chris Masonbe0e5c02007-01-26 15:51:26 -05003868 /* push data from right to left */
Chris Mason5f39d392007-10-15 16:14:19 -04003869 copy_extent_buffer(left, right,
3870 btrfs_item_nr_offset(btrfs_header_nritems(left)),
3871 btrfs_item_nr_offset(0),
3872 push_items * sizeof(struct btrfs_item));
3873
Chris Mason123abc82007-03-14 14:14:43 -04003874 push_space = BTRFS_LEAF_DATA_SIZE(root) -
Chris Masond3977122009-01-05 21:25:51 -05003875 btrfs_item_offset_nr(right, push_items - 1);
Chris Mason5f39d392007-10-15 16:14:19 -04003876
3877 copy_extent_buffer(left, right, btrfs_leaf_data(left) +
Chris Masond6025572007-03-30 14:27:56 -04003878 leaf_data_end(root, left) - push_space,
3879 btrfs_leaf_data(right) +
Chris Mason5f39d392007-10-15 16:14:19 -04003880 btrfs_item_offset_nr(right, push_items - 1),
Chris Masond6025572007-03-30 14:27:56 -04003881 push_space);
Chris Mason5f39d392007-10-15 16:14:19 -04003882 old_left_nritems = btrfs_header_nritems(left);
Yan Zheng87b29b22008-12-17 10:21:48 -05003883 BUG_ON(old_left_nritems <= 0);
Chris Masoneb60cea2007-02-02 09:18:22 -05003884
Chris Masondb945352007-10-15 16:15:53 -04003885 old_left_item_size = btrfs_item_offset_nr(left, old_left_nritems - 1);
Chris Mason0783fcf2007-03-12 20:12:07 -04003886 for (i = old_left_nritems; i < old_left_nritems + push_items; i++) {
Chris Mason5f39d392007-10-15 16:14:19 -04003887 u32 ioff;
Chris Masondb945352007-10-15 16:15:53 -04003888
Ross Kirkdd3cc162013-09-16 15:58:09 +01003889 item = btrfs_item_nr(i);
Chris Masondb945352007-10-15 16:15:53 -04003890
Chris Masoncfed81a2012-03-03 07:40:03 -05003891 ioff = btrfs_token_item_offset(left, item, &token);
3892 btrfs_set_token_item_offset(left, item,
3893 ioff - (BTRFS_LEAF_DATA_SIZE(root) - old_left_item_size),
3894 &token);
Chris Masonbe0e5c02007-01-26 15:51:26 -05003895 }
Chris Mason5f39d392007-10-15 16:14:19 -04003896 btrfs_set_header_nritems(left, old_left_nritems + push_items);
Chris Masonbe0e5c02007-01-26 15:51:26 -05003897
3898 /* fixup right node */
Julia Lawall31b1a2b2012-11-03 10:58:34 +00003899 if (push_items > right_nritems)
3900 WARN(1, KERN_CRIT "push items %d nr %u\n", push_items,
Chris Masond3977122009-01-05 21:25:51 -05003901 right_nritems);
Chris Mason5f39d392007-10-15 16:14:19 -04003902
Chris Mason34a38212007-11-07 13:31:03 -05003903 if (push_items < right_nritems) {
3904 push_space = btrfs_item_offset_nr(right, push_items - 1) -
3905 leaf_data_end(root, right);
3906 memmove_extent_buffer(right, btrfs_leaf_data(right) +
3907 BTRFS_LEAF_DATA_SIZE(root) - push_space,
3908 btrfs_leaf_data(right) +
3909 leaf_data_end(root, right), push_space);
3910
3911 memmove_extent_buffer(right, btrfs_item_nr_offset(0),
Chris Mason5f39d392007-10-15 16:14:19 -04003912 btrfs_item_nr_offset(push_items),
3913 (btrfs_header_nritems(right) - push_items) *
3914 sizeof(struct btrfs_item));
Chris Mason34a38212007-11-07 13:31:03 -05003915 }
Yaneef1c492007-11-26 10:58:13 -05003916 right_nritems -= push_items;
3917 btrfs_set_header_nritems(right, right_nritems);
Chris Mason123abc82007-03-14 14:14:43 -04003918 push_space = BTRFS_LEAF_DATA_SIZE(root);
Chris Mason5f39d392007-10-15 16:14:19 -04003919 for (i = 0; i < right_nritems; i++) {
Ross Kirkdd3cc162013-09-16 15:58:09 +01003920 item = btrfs_item_nr(i);
Chris Masondb945352007-10-15 16:15:53 -04003921
Chris Masoncfed81a2012-03-03 07:40:03 -05003922 push_space = push_space - btrfs_token_item_size(right,
3923 item, &token);
3924 btrfs_set_token_item_offset(right, item, push_space, &token);
Chris Masondb945352007-10-15 16:15:53 -04003925 }
Chris Masoneb60cea2007-02-02 09:18:22 -05003926
Chris Mason5f39d392007-10-15 16:14:19 -04003927 btrfs_mark_buffer_dirty(left);
Chris Mason34a38212007-11-07 13:31:03 -05003928 if (right_nritems)
3929 btrfs_mark_buffer_dirty(right);
Yan, Zhengf0486c62010-05-16 10:46:25 -04003930 else
3931 clean_tree_block(trans, root, right);
Chris Mason098f59c2007-05-11 11:33:21 -04003932
Chris Mason5f39d392007-10-15 16:14:19 -04003933 btrfs_item_key(right, &disk_key, 0);
Tsutomu Itohd6a0a122013-04-16 05:18:02 +00003934 fixup_low_keys(root, path, &disk_key, 1);
Chris Masonbe0e5c02007-01-26 15:51:26 -05003935
3936 /* then fixup the leaf pointer in the path */
3937 if (path->slots[0] < push_items) {
3938 path->slots[0] += old_left_nritems;
Chris Mason925baed2008-06-25 16:01:30 -04003939 btrfs_tree_unlock(path->nodes[0]);
Chris Mason5f39d392007-10-15 16:14:19 -04003940 free_extent_buffer(path->nodes[0]);
3941 path->nodes[0] = left;
Chris Masonbe0e5c02007-01-26 15:51:26 -05003942 path->slots[1] -= 1;
3943 } else {
Chris Mason925baed2008-06-25 16:01:30 -04003944 btrfs_tree_unlock(left);
Chris Mason5f39d392007-10-15 16:14:19 -04003945 free_extent_buffer(left);
Chris Masonbe0e5c02007-01-26 15:51:26 -05003946 path->slots[0] -= push_items;
3947 }
Chris Masoneb60cea2007-02-02 09:18:22 -05003948 BUG_ON(path->slots[0] < 0);
Chris Masonaa5d6be2007-02-28 16:35:06 -05003949 return ret;
Chris Mason925baed2008-06-25 16:01:30 -04003950out:
3951 btrfs_tree_unlock(left);
3952 free_extent_buffer(left);
3953 return ret;
Chris Masonbe0e5c02007-01-26 15:51:26 -05003954}
3955
Chris Mason74123bd2007-02-02 11:05:29 -05003956/*
Chris Mason44871b12009-03-13 10:04:31 -04003957 * push some data in the path leaf to the left, trying to free up at
3958 * least data_size bytes. returns zero if the push worked, nonzero otherwise
Chris Mason99d8f832010-07-07 10:51:48 -04003959 *
3960 * max_slot can put a limit on how far into the leaf we'll push items. The
3961 * item at 'max_slot' won't be touched. Use (u32)-1 to make us push all the
3962 * items
Chris Mason44871b12009-03-13 10:04:31 -04003963 */
3964static int push_leaf_left(struct btrfs_trans_handle *trans, struct btrfs_root
Chris Mason99d8f832010-07-07 10:51:48 -04003965 *root, struct btrfs_path *path, int min_data_size,
3966 int data_size, int empty, u32 max_slot)
Chris Mason44871b12009-03-13 10:04:31 -04003967{
3968 struct extent_buffer *right = path->nodes[0];
3969 struct extent_buffer *left;
3970 int slot;
3971 int free_space;
3972 u32 right_nritems;
3973 int ret = 0;
3974
3975 slot = path->slots[1];
3976 if (slot == 0)
3977 return 1;
3978 if (!path->nodes[1])
3979 return 1;
3980
3981 right_nritems = btrfs_header_nritems(right);
3982 if (right_nritems == 0)
3983 return 1;
3984
3985 btrfs_assert_tree_locked(path->nodes[1]);
3986
3987 left = read_node_slot(root, path->nodes[1], slot - 1);
Tsutomu Itoh91ca3382011-01-05 02:32:22 +00003988 if (left == NULL)
3989 return 1;
3990
Chris Mason44871b12009-03-13 10:04:31 -04003991 btrfs_tree_lock(left);
3992 btrfs_set_lock_blocking(left);
3993
3994 free_space = btrfs_leaf_free_space(root, left);
3995 if (free_space < data_size) {
3996 ret = 1;
3997 goto out;
3998 }
3999
4000 /* cow and double check */
4001 ret = btrfs_cow_block(trans, root, left,
4002 path->nodes[1], slot - 1, &left);
4003 if (ret) {
4004 /* we hit -ENOSPC, but it isn't fatal here */
Jeff Mahoney79787ea2012-03-12 16:03:00 +01004005 if (ret == -ENOSPC)
4006 ret = 1;
Chris Mason44871b12009-03-13 10:04:31 -04004007 goto out;
4008 }
4009
4010 free_space = btrfs_leaf_free_space(root, left);
4011 if (free_space < data_size) {
4012 ret = 1;
4013 goto out;
4014 }
4015
Chris Mason99d8f832010-07-07 10:51:48 -04004016 return __push_leaf_left(trans, root, path, min_data_size,
4017 empty, left, free_space, right_nritems,
4018 max_slot);
Chris Mason44871b12009-03-13 10:04:31 -04004019out:
4020 btrfs_tree_unlock(left);
4021 free_extent_buffer(left);
4022 return ret;
4023}
4024
4025/*
Chris Mason74123bd2007-02-02 11:05:29 -05004026 * split the path's leaf in two, making sure there is at least data_size
4027 * available for the resulting leaf level of the path.
4028 */
Jeff Mahoney143bede2012-03-01 14:56:26 +01004029static noinline void copy_for_split(struct btrfs_trans_handle *trans,
4030 struct btrfs_root *root,
4031 struct btrfs_path *path,
4032 struct extent_buffer *l,
4033 struct extent_buffer *right,
4034 int slot, int mid, int nritems)
Chris Masonbe0e5c02007-01-26 15:51:26 -05004035{
Chris Masonbe0e5c02007-01-26 15:51:26 -05004036 int data_copy_size;
4037 int rt_data_off;
4038 int i;
Chris Masond4dbff92007-04-04 14:08:15 -04004039 struct btrfs_disk_key disk_key;
Chris Masoncfed81a2012-03-03 07:40:03 -05004040 struct btrfs_map_token token;
4041
4042 btrfs_init_map_token(&token);
Chris Masonbe0e5c02007-01-26 15:51:26 -05004043
Chris Mason5f39d392007-10-15 16:14:19 -04004044 nritems = nritems - mid;
4045 btrfs_set_header_nritems(right, nritems);
4046 data_copy_size = btrfs_item_end_nr(l, mid) - leaf_data_end(root, l);
4047
4048 copy_extent_buffer(right, l, btrfs_item_nr_offset(0),
4049 btrfs_item_nr_offset(mid),
4050 nritems * sizeof(struct btrfs_item));
4051
4052 copy_extent_buffer(right, l,
Chris Masond6025572007-03-30 14:27:56 -04004053 btrfs_leaf_data(right) + BTRFS_LEAF_DATA_SIZE(root) -
4054 data_copy_size, btrfs_leaf_data(l) +
4055 leaf_data_end(root, l), data_copy_size);
Chris Mason74123bd2007-02-02 11:05:29 -05004056
Chris Mason5f39d392007-10-15 16:14:19 -04004057 rt_data_off = BTRFS_LEAF_DATA_SIZE(root) -
4058 btrfs_item_end_nr(l, mid);
4059
4060 for (i = 0; i < nritems; i++) {
Ross Kirkdd3cc162013-09-16 15:58:09 +01004061 struct btrfs_item *item = btrfs_item_nr(i);
Chris Masondb945352007-10-15 16:15:53 -04004062 u32 ioff;
4063
Chris Masoncfed81a2012-03-03 07:40:03 -05004064 ioff = btrfs_token_item_offset(right, item, &token);
4065 btrfs_set_token_item_offset(right, item,
4066 ioff + rt_data_off, &token);
Chris Mason0783fcf2007-03-12 20:12:07 -04004067 }
Chris Mason74123bd2007-02-02 11:05:29 -05004068
Chris Mason5f39d392007-10-15 16:14:19 -04004069 btrfs_set_header_nritems(l, mid);
Chris Mason5f39d392007-10-15 16:14:19 -04004070 btrfs_item_key(right, &disk_key, 0);
Jeff Mahoney143bede2012-03-01 14:56:26 +01004071 insert_ptr(trans, root, path, &disk_key, right->start,
Jan Schmidtc3e06962012-06-21 11:01:06 +02004072 path->slots[1] + 1, 1);
Chris Mason5f39d392007-10-15 16:14:19 -04004073
4074 btrfs_mark_buffer_dirty(right);
4075 btrfs_mark_buffer_dirty(l);
Chris Masoneb60cea2007-02-02 09:18:22 -05004076 BUG_ON(path->slots[0] != slot);
Chris Mason5f39d392007-10-15 16:14:19 -04004077
Chris Masonbe0e5c02007-01-26 15:51:26 -05004078 if (mid <= slot) {
Chris Mason925baed2008-06-25 16:01:30 -04004079 btrfs_tree_unlock(path->nodes[0]);
Chris Mason5f39d392007-10-15 16:14:19 -04004080 free_extent_buffer(path->nodes[0]);
4081 path->nodes[0] = right;
Chris Masonbe0e5c02007-01-26 15:51:26 -05004082 path->slots[0] -= mid;
4083 path->slots[1] += 1;
Chris Mason925baed2008-06-25 16:01:30 -04004084 } else {
4085 btrfs_tree_unlock(right);
Chris Mason5f39d392007-10-15 16:14:19 -04004086 free_extent_buffer(right);
Chris Mason925baed2008-06-25 16:01:30 -04004087 }
Chris Mason5f39d392007-10-15 16:14:19 -04004088
Chris Masoneb60cea2007-02-02 09:18:22 -05004089 BUG_ON(path->slots[0] < 0);
Chris Mason44871b12009-03-13 10:04:31 -04004090}
4091
4092/*
Chris Mason99d8f832010-07-07 10:51:48 -04004093 * double splits happen when we need to insert a big item in the middle
4094 * of a leaf. A double split can leave us with 3 mostly empty leaves:
4095 * leaf: [ slots 0 - N] [ our target ] [ N + 1 - total in leaf ]
4096 * A B C
4097 *
4098 * We avoid this by trying to push the items on either side of our target
4099 * into the adjacent leaves. If all goes well we can avoid the double split
4100 * completely.
4101 */
4102static noinline int push_for_double_split(struct btrfs_trans_handle *trans,
4103 struct btrfs_root *root,
4104 struct btrfs_path *path,
4105 int data_size)
4106{
4107 int ret;
4108 int progress = 0;
4109 int slot;
4110 u32 nritems;
Filipe David Borba Manana5a4267c2013-11-25 03:20:46 +00004111 int space_needed = data_size;
Chris Mason99d8f832010-07-07 10:51:48 -04004112
4113 slot = path->slots[0];
Filipe David Borba Manana5a4267c2013-11-25 03:20:46 +00004114 if (slot < btrfs_header_nritems(path->nodes[0]))
4115 space_needed -= btrfs_leaf_free_space(root, path->nodes[0]);
Chris Mason99d8f832010-07-07 10:51:48 -04004116
4117 /*
4118 * try to push all the items after our slot into the
4119 * right leaf
4120 */
Filipe David Borba Manana5a4267c2013-11-25 03:20:46 +00004121 ret = push_leaf_right(trans, root, path, 1, space_needed, 0, slot);
Chris Mason99d8f832010-07-07 10:51:48 -04004122 if (ret < 0)
4123 return ret;
4124
4125 if (ret == 0)
4126 progress++;
4127
4128 nritems = btrfs_header_nritems(path->nodes[0]);
4129 /*
4130 * our goal is to get our slot at the start or end of a leaf. If
4131 * we've done so we're done
4132 */
4133 if (path->slots[0] == 0 || path->slots[0] == nritems)
4134 return 0;
4135
4136 if (btrfs_leaf_free_space(root, path->nodes[0]) >= data_size)
4137 return 0;
4138
4139 /* try to push all the items before our slot into the next leaf */
4140 slot = path->slots[0];
Filipe David Borba Manana5a4267c2013-11-25 03:20:46 +00004141 ret = push_leaf_left(trans, root, path, 1, space_needed, 0, slot);
Chris Mason99d8f832010-07-07 10:51:48 -04004142 if (ret < 0)
4143 return ret;
4144
4145 if (ret == 0)
4146 progress++;
4147
4148 if (progress)
4149 return 0;
4150 return 1;
4151}
4152
4153/*
Chris Mason44871b12009-03-13 10:04:31 -04004154 * split the path's leaf in two, making sure there is at least data_size
4155 * available for the resulting leaf level of the path.
4156 *
4157 * returns 0 if all went well and < 0 on failure.
4158 */
4159static noinline int split_leaf(struct btrfs_trans_handle *trans,
4160 struct btrfs_root *root,
4161 struct btrfs_key *ins_key,
4162 struct btrfs_path *path, int data_size,
4163 int extend)
4164{
Yan Zheng5d4f98a2009-06-10 10:45:14 -04004165 struct btrfs_disk_key disk_key;
Chris Mason44871b12009-03-13 10:04:31 -04004166 struct extent_buffer *l;
4167 u32 nritems;
4168 int mid;
4169 int slot;
4170 struct extent_buffer *right;
4171 int ret = 0;
4172 int wret;
Yan Zheng5d4f98a2009-06-10 10:45:14 -04004173 int split;
Chris Mason44871b12009-03-13 10:04:31 -04004174 int num_doubles = 0;
Chris Mason99d8f832010-07-07 10:51:48 -04004175 int tried_avoid_double = 0;
Chris Mason44871b12009-03-13 10:04:31 -04004176
Yan, Zhenga5719522009-09-24 09:17:31 -04004177 l = path->nodes[0];
4178 slot = path->slots[0];
4179 if (extend && data_size + btrfs_item_size_nr(l, slot) +
4180 sizeof(struct btrfs_item) > BTRFS_LEAF_DATA_SIZE(root))
4181 return -EOVERFLOW;
4182
Chris Mason44871b12009-03-13 10:04:31 -04004183 /* first try to make some room by pushing left and right */
Liu Bo33157e02013-05-22 12:07:06 +00004184 if (data_size && path->nodes[1]) {
Filipe David Borba Manana5a4267c2013-11-25 03:20:46 +00004185 int space_needed = data_size;
4186
4187 if (slot < btrfs_header_nritems(l))
4188 space_needed -= btrfs_leaf_free_space(root, l);
4189
4190 wret = push_leaf_right(trans, root, path, space_needed,
4191 space_needed, 0, 0);
Chris Mason44871b12009-03-13 10:04:31 -04004192 if (wret < 0)
4193 return wret;
4194 if (wret) {
Filipe David Borba Manana5a4267c2013-11-25 03:20:46 +00004195 wret = push_leaf_left(trans, root, path, space_needed,
4196 space_needed, 0, (u32)-1);
Chris Mason44871b12009-03-13 10:04:31 -04004197 if (wret < 0)
4198 return wret;
4199 }
4200 l = path->nodes[0];
4201
4202 /* did the pushes work? */
4203 if (btrfs_leaf_free_space(root, l) >= data_size)
4204 return 0;
4205 }
4206
4207 if (!path->nodes[1]) {
Liu Bofdd99c72013-05-22 12:06:51 +00004208 ret = insert_new_root(trans, root, path, 1);
Chris Mason44871b12009-03-13 10:04:31 -04004209 if (ret)
4210 return ret;
4211 }
4212again:
Yan Zheng5d4f98a2009-06-10 10:45:14 -04004213 split = 1;
Chris Mason44871b12009-03-13 10:04:31 -04004214 l = path->nodes[0];
4215 slot = path->slots[0];
4216 nritems = btrfs_header_nritems(l);
4217 mid = (nritems + 1) / 2;
4218
Yan Zheng5d4f98a2009-06-10 10:45:14 -04004219 if (mid <= slot) {
4220 if (nritems == 1 ||
4221 leaf_space_used(l, mid, nritems - mid) + data_size >
4222 BTRFS_LEAF_DATA_SIZE(root)) {
4223 if (slot >= nritems) {
4224 split = 0;
4225 } else {
4226 mid = slot;
4227 if (mid != nritems &&
4228 leaf_space_used(l, mid, nritems - mid) +
4229 data_size > BTRFS_LEAF_DATA_SIZE(root)) {
Chris Mason99d8f832010-07-07 10:51:48 -04004230 if (data_size && !tried_avoid_double)
4231 goto push_for_double;
Yan Zheng5d4f98a2009-06-10 10:45:14 -04004232 split = 2;
4233 }
4234 }
4235 }
4236 } else {
4237 if (leaf_space_used(l, 0, mid) + data_size >
4238 BTRFS_LEAF_DATA_SIZE(root)) {
4239 if (!extend && data_size && slot == 0) {
4240 split = 0;
4241 } else if ((extend || !data_size) && slot == 0) {
4242 mid = 1;
4243 } else {
4244 mid = slot;
4245 if (mid != nritems &&
4246 leaf_space_used(l, mid, nritems - mid) +
4247 data_size > BTRFS_LEAF_DATA_SIZE(root)) {
Chris Mason99d8f832010-07-07 10:51:48 -04004248 if (data_size && !tried_avoid_double)
4249 goto push_for_double;
Dulshani Gunawardhana67871252013-10-31 10:33:04 +05304250 split = 2;
Yan Zheng5d4f98a2009-06-10 10:45:14 -04004251 }
4252 }
4253 }
4254 }
4255
4256 if (split == 0)
4257 btrfs_cpu_key_to_disk(&disk_key, ins_key);
4258 else
4259 btrfs_item_key(l, &disk_key, mid);
4260
David Sterba4d75f8a2014-06-15 01:54:12 +02004261 right = btrfs_alloc_tree_block(trans, root, 0, root->root_key.objectid,
4262 &disk_key, 0, l->start, 0);
Yan, Zhengf0486c62010-05-16 10:46:25 -04004263 if (IS_ERR(right))
Chris Mason44871b12009-03-13 10:04:31 -04004264 return PTR_ERR(right);
Yan, Zhengf0486c62010-05-16 10:46:25 -04004265
David Sterba707e8a02014-06-04 19:22:26 +02004266 root_add_used(root, root->nodesize);
Chris Mason44871b12009-03-13 10:04:31 -04004267
4268 memset_extent_buffer(right, 0, 0, sizeof(struct btrfs_header));
4269 btrfs_set_header_bytenr(right, right->start);
4270 btrfs_set_header_generation(right, trans->transid);
Yan Zheng5d4f98a2009-06-10 10:45:14 -04004271 btrfs_set_header_backref_rev(right, BTRFS_MIXED_BACKREF_REV);
Chris Mason44871b12009-03-13 10:04:31 -04004272 btrfs_set_header_owner(right, root->root_key.objectid);
4273 btrfs_set_header_level(right, 0);
4274 write_extent_buffer(right, root->fs_info->fsid,
Ross Kirk0a4e5582013-09-24 10:12:38 +01004275 btrfs_header_fsid(), BTRFS_FSID_SIZE);
Chris Mason44871b12009-03-13 10:04:31 -04004276
4277 write_extent_buffer(right, root->fs_info->chunk_tree_uuid,
Geert Uytterhoevenb308bc22013-08-20 13:20:15 +02004278 btrfs_header_chunk_tree_uuid(right),
Chris Mason44871b12009-03-13 10:04:31 -04004279 BTRFS_UUID_SIZE);
4280
Yan Zheng5d4f98a2009-06-10 10:45:14 -04004281 if (split == 0) {
4282 if (mid <= slot) {
4283 btrfs_set_header_nritems(right, 0);
Jeff Mahoney143bede2012-03-01 14:56:26 +01004284 insert_ptr(trans, root, path, &disk_key, right->start,
Jan Schmidtc3e06962012-06-21 11:01:06 +02004285 path->slots[1] + 1, 1);
Yan Zheng5d4f98a2009-06-10 10:45:14 -04004286 btrfs_tree_unlock(path->nodes[0]);
4287 free_extent_buffer(path->nodes[0]);
4288 path->nodes[0] = right;
4289 path->slots[0] = 0;
4290 path->slots[1] += 1;
4291 } else {
4292 btrfs_set_header_nritems(right, 0);
Jeff Mahoney143bede2012-03-01 14:56:26 +01004293 insert_ptr(trans, root, path, &disk_key, right->start,
Jan Schmidtc3e06962012-06-21 11:01:06 +02004294 path->slots[1], 1);
Yan Zheng5d4f98a2009-06-10 10:45:14 -04004295 btrfs_tree_unlock(path->nodes[0]);
4296 free_extent_buffer(path->nodes[0]);
4297 path->nodes[0] = right;
4298 path->slots[0] = 0;
Jeff Mahoney143bede2012-03-01 14:56:26 +01004299 if (path->slots[1] == 0)
Tsutomu Itohd6a0a122013-04-16 05:18:02 +00004300 fixup_low_keys(root, path, &disk_key, 1);
Chris Mason44871b12009-03-13 10:04:31 -04004301 }
Yan Zheng5d4f98a2009-06-10 10:45:14 -04004302 btrfs_mark_buffer_dirty(right);
4303 return ret;
Chris Mason44871b12009-03-13 10:04:31 -04004304 }
4305
Jeff Mahoney143bede2012-03-01 14:56:26 +01004306 copy_for_split(trans, root, path, l, right, slot, mid, nritems);
Chris Mason44871b12009-03-13 10:04:31 -04004307
Yan Zheng5d4f98a2009-06-10 10:45:14 -04004308 if (split == 2) {
Chris Masoncc0c5532007-10-25 15:42:57 -04004309 BUG_ON(num_doubles != 0);
4310 num_doubles++;
4311 goto again;
Chris Mason3326d1b2007-10-15 16:18:25 -04004312 }
Chris Mason44871b12009-03-13 10:04:31 -04004313
Jeff Mahoney143bede2012-03-01 14:56:26 +01004314 return 0;
Chris Mason99d8f832010-07-07 10:51:48 -04004315
4316push_for_double:
4317 push_for_double_split(trans, root, path, data_size);
4318 tried_avoid_double = 1;
4319 if (btrfs_leaf_free_space(root, path->nodes[0]) >= data_size)
4320 return 0;
4321 goto again;
Chris Masonbe0e5c02007-01-26 15:51:26 -05004322}
4323
Yan, Zhengad48fd752009-11-12 09:33:58 +00004324static noinline int setup_leaf_for_split(struct btrfs_trans_handle *trans,
4325 struct btrfs_root *root,
4326 struct btrfs_path *path, int ins_len)
Chris Mason459931e2008-12-10 09:10:46 -05004327{
Yan, Zhengad48fd752009-11-12 09:33:58 +00004328 struct btrfs_key key;
Chris Mason459931e2008-12-10 09:10:46 -05004329 struct extent_buffer *leaf;
Yan, Zhengad48fd752009-11-12 09:33:58 +00004330 struct btrfs_file_extent_item *fi;
4331 u64 extent_len = 0;
4332 u32 item_size;
4333 int ret;
Chris Mason459931e2008-12-10 09:10:46 -05004334
4335 leaf = path->nodes[0];
Yan, Zhengad48fd752009-11-12 09:33:58 +00004336 btrfs_item_key_to_cpu(leaf, &key, path->slots[0]);
4337
4338 BUG_ON(key.type != BTRFS_EXTENT_DATA_KEY &&
4339 key.type != BTRFS_EXTENT_CSUM_KEY);
4340
4341 if (btrfs_leaf_free_space(root, leaf) >= ins_len)
4342 return 0;
Chris Mason459931e2008-12-10 09:10:46 -05004343
4344 item_size = btrfs_item_size_nr(leaf, path->slots[0]);
Yan, Zhengad48fd752009-11-12 09:33:58 +00004345 if (key.type == BTRFS_EXTENT_DATA_KEY) {
4346 fi = btrfs_item_ptr(leaf, path->slots[0],
4347 struct btrfs_file_extent_item);
4348 extent_len = btrfs_file_extent_num_bytes(leaf, fi);
4349 }
David Sterbab3b4aa72011-04-21 01:20:15 +02004350 btrfs_release_path(path);
Chris Mason459931e2008-12-10 09:10:46 -05004351
Chris Mason459931e2008-12-10 09:10:46 -05004352 path->keep_locks = 1;
Yan, Zhengad48fd752009-11-12 09:33:58 +00004353 path->search_for_split = 1;
4354 ret = btrfs_search_slot(trans, root, &key, path, 0, 1);
Chris Mason459931e2008-12-10 09:10:46 -05004355 path->search_for_split = 0;
Yan, Zhengad48fd752009-11-12 09:33:58 +00004356 if (ret < 0)
4357 goto err;
Chris Mason459931e2008-12-10 09:10:46 -05004358
Yan, Zhengad48fd752009-11-12 09:33:58 +00004359 ret = -EAGAIN;
4360 leaf = path->nodes[0];
Chris Mason459931e2008-12-10 09:10:46 -05004361 /* if our item isn't there or got smaller, return now */
Yan, Zhengad48fd752009-11-12 09:33:58 +00004362 if (ret > 0 || item_size != btrfs_item_size_nr(leaf, path->slots[0]))
4363 goto err;
4364
Chris Mason109f6ae2010-04-02 09:20:18 -04004365 /* the leaf has changed, it now has room. return now */
4366 if (btrfs_leaf_free_space(root, path->nodes[0]) >= ins_len)
4367 goto err;
4368
Yan, Zhengad48fd752009-11-12 09:33:58 +00004369 if (key.type == BTRFS_EXTENT_DATA_KEY) {
4370 fi = btrfs_item_ptr(leaf, path->slots[0],
4371 struct btrfs_file_extent_item);
4372 if (extent_len != btrfs_file_extent_num_bytes(leaf, fi))
4373 goto err;
Chris Mason459931e2008-12-10 09:10:46 -05004374 }
4375
Chris Masonb9473432009-03-13 11:00:37 -04004376 btrfs_set_path_blocking(path);
Yan, Zhengad48fd752009-11-12 09:33:58 +00004377 ret = split_leaf(trans, root, &key, path, ins_len, 1);
Yan, Zhengf0486c62010-05-16 10:46:25 -04004378 if (ret)
4379 goto err;
Chris Mason459931e2008-12-10 09:10:46 -05004380
Yan, Zhengad48fd752009-11-12 09:33:58 +00004381 path->keep_locks = 0;
Chris Masonb9473432009-03-13 11:00:37 -04004382 btrfs_unlock_up_safe(path, 1);
Yan, Zhengad48fd752009-11-12 09:33:58 +00004383 return 0;
4384err:
4385 path->keep_locks = 0;
4386 return ret;
4387}
4388
4389static noinline int split_item(struct btrfs_trans_handle *trans,
4390 struct btrfs_root *root,
4391 struct btrfs_path *path,
4392 struct btrfs_key *new_key,
4393 unsigned long split_offset)
4394{
4395 struct extent_buffer *leaf;
4396 struct btrfs_item *item;
4397 struct btrfs_item *new_item;
4398 int slot;
4399 char *buf;
4400 u32 nritems;
4401 u32 item_size;
4402 u32 orig_offset;
4403 struct btrfs_disk_key disk_key;
4404
Chris Masonb9473432009-03-13 11:00:37 -04004405 leaf = path->nodes[0];
4406 BUG_ON(btrfs_leaf_free_space(root, leaf) < sizeof(struct btrfs_item));
4407
Chris Masonb4ce94d2009-02-04 09:25:08 -05004408 btrfs_set_path_blocking(path);
4409
Ross Kirkdd3cc162013-09-16 15:58:09 +01004410 item = btrfs_item_nr(path->slots[0]);
Chris Mason459931e2008-12-10 09:10:46 -05004411 orig_offset = btrfs_item_offset(leaf, item);
4412 item_size = btrfs_item_size(leaf, item);
4413
Chris Mason459931e2008-12-10 09:10:46 -05004414 buf = kmalloc(item_size, GFP_NOFS);
Yan, Zhengad48fd752009-11-12 09:33:58 +00004415 if (!buf)
4416 return -ENOMEM;
4417
Chris Mason459931e2008-12-10 09:10:46 -05004418 read_extent_buffer(leaf, buf, btrfs_item_ptr_offset(leaf,
4419 path->slots[0]), item_size);
Yan, Zhengad48fd752009-11-12 09:33:58 +00004420
Chris Mason459931e2008-12-10 09:10:46 -05004421 slot = path->slots[0] + 1;
Chris Mason459931e2008-12-10 09:10:46 -05004422 nritems = btrfs_header_nritems(leaf);
Chris Mason459931e2008-12-10 09:10:46 -05004423 if (slot != nritems) {
4424 /* shift the items */
4425 memmove_extent_buffer(leaf, btrfs_item_nr_offset(slot + 1),
Yan, Zhengad48fd752009-11-12 09:33:58 +00004426 btrfs_item_nr_offset(slot),
4427 (nritems - slot) * sizeof(struct btrfs_item));
Chris Mason459931e2008-12-10 09:10:46 -05004428 }
4429
4430 btrfs_cpu_key_to_disk(&disk_key, new_key);
4431 btrfs_set_item_key(leaf, &disk_key, slot);
4432
Ross Kirkdd3cc162013-09-16 15:58:09 +01004433 new_item = btrfs_item_nr(slot);
Chris Mason459931e2008-12-10 09:10:46 -05004434
4435 btrfs_set_item_offset(leaf, new_item, orig_offset);
4436 btrfs_set_item_size(leaf, new_item, item_size - split_offset);
4437
4438 btrfs_set_item_offset(leaf, item,
4439 orig_offset + item_size - split_offset);
4440 btrfs_set_item_size(leaf, item, split_offset);
4441
4442 btrfs_set_header_nritems(leaf, nritems + 1);
4443
4444 /* write the data for the start of the original item */
4445 write_extent_buffer(leaf, buf,
4446 btrfs_item_ptr_offset(leaf, path->slots[0]),
4447 split_offset);
4448
4449 /* write the data for the new item */
4450 write_extent_buffer(leaf, buf + split_offset,
4451 btrfs_item_ptr_offset(leaf, slot),
4452 item_size - split_offset);
4453 btrfs_mark_buffer_dirty(leaf);
4454
Yan, Zhengad48fd752009-11-12 09:33:58 +00004455 BUG_ON(btrfs_leaf_free_space(root, leaf) < 0);
Chris Mason459931e2008-12-10 09:10:46 -05004456 kfree(buf);
Yan, Zhengad48fd752009-11-12 09:33:58 +00004457 return 0;
4458}
4459
4460/*
4461 * This function splits a single item into two items,
4462 * giving 'new_key' to the new item and splitting the
4463 * old one at split_offset (from the start of the item).
4464 *
4465 * The path may be released by this operation. After
4466 * the split, the path is pointing to the old item. The
4467 * new item is going to be in the same node as the old one.
4468 *
4469 * Note, the item being split must be smaller enough to live alone on
4470 * a tree block with room for one extra struct btrfs_item
4471 *
4472 * This allows us to split the item in place, keeping a lock on the
4473 * leaf the entire time.
4474 */
4475int btrfs_split_item(struct btrfs_trans_handle *trans,
4476 struct btrfs_root *root,
4477 struct btrfs_path *path,
4478 struct btrfs_key *new_key,
4479 unsigned long split_offset)
4480{
4481 int ret;
4482 ret = setup_leaf_for_split(trans, root, path,
4483 sizeof(struct btrfs_item));
4484 if (ret)
4485 return ret;
4486
4487 ret = split_item(trans, root, path, new_key, split_offset);
Chris Mason459931e2008-12-10 09:10:46 -05004488 return ret;
4489}
4490
4491/*
Yan, Zhengad48fd752009-11-12 09:33:58 +00004492 * This function duplicate a item, giving 'new_key' to the new item.
4493 * It guarantees both items live in the same tree leaf and the new item
4494 * is contiguous with the original item.
4495 *
4496 * This allows us to split file extent in place, keeping a lock on the
4497 * leaf the entire time.
4498 */
4499int btrfs_duplicate_item(struct btrfs_trans_handle *trans,
4500 struct btrfs_root *root,
4501 struct btrfs_path *path,
4502 struct btrfs_key *new_key)
4503{
4504 struct extent_buffer *leaf;
4505 int ret;
4506 u32 item_size;
4507
4508 leaf = path->nodes[0];
4509 item_size = btrfs_item_size_nr(leaf, path->slots[0]);
4510 ret = setup_leaf_for_split(trans, root, path,
4511 item_size + sizeof(struct btrfs_item));
4512 if (ret)
4513 return ret;
4514
4515 path->slots[0]++;
Tsutomu Itohafe5fea2013-04-16 05:18:22 +00004516 setup_items_for_insert(root, path, new_key, &item_size,
Jeff Mahoney143bede2012-03-01 14:56:26 +01004517 item_size, item_size +
4518 sizeof(struct btrfs_item), 1);
Yan, Zhengad48fd752009-11-12 09:33:58 +00004519 leaf = path->nodes[0];
4520 memcpy_extent_buffer(leaf,
4521 btrfs_item_ptr_offset(leaf, path->slots[0]),
4522 btrfs_item_ptr_offset(leaf, path->slots[0] - 1),
4523 item_size);
4524 return 0;
4525}
4526
4527/*
Chris Masond352ac62008-09-29 15:18:18 -04004528 * make the item pointed to by the path smaller. new_size indicates
4529 * how small to make it, and from_end tells us if we just chop bytes
4530 * off the end of the item or if we shift the item to chop bytes off
4531 * the front.
4532 */
Tsutomu Itohafe5fea2013-04-16 05:18:22 +00004533void btrfs_truncate_item(struct btrfs_root *root, struct btrfs_path *path,
Jeff Mahoney143bede2012-03-01 14:56:26 +01004534 u32 new_size, int from_end)
Chris Masonb18c6682007-04-17 13:26:50 -04004535{
Chris Masonb18c6682007-04-17 13:26:50 -04004536 int slot;
Chris Mason5f39d392007-10-15 16:14:19 -04004537 struct extent_buffer *leaf;
4538 struct btrfs_item *item;
Chris Masonb18c6682007-04-17 13:26:50 -04004539 u32 nritems;
4540 unsigned int data_end;
4541 unsigned int old_data_start;
4542 unsigned int old_size;
4543 unsigned int size_diff;
4544 int i;
Chris Masoncfed81a2012-03-03 07:40:03 -05004545 struct btrfs_map_token token;
4546
4547 btrfs_init_map_token(&token);
Chris Masonb18c6682007-04-17 13:26:50 -04004548
Chris Mason5f39d392007-10-15 16:14:19 -04004549 leaf = path->nodes[0];
Chris Mason179e29e2007-11-01 11:28:41 -04004550 slot = path->slots[0];
4551
4552 old_size = btrfs_item_size_nr(leaf, slot);
4553 if (old_size == new_size)
Jeff Mahoney143bede2012-03-01 14:56:26 +01004554 return;
Chris Masonb18c6682007-04-17 13:26:50 -04004555
Chris Mason5f39d392007-10-15 16:14:19 -04004556 nritems = btrfs_header_nritems(leaf);
Chris Masonb18c6682007-04-17 13:26:50 -04004557 data_end = leaf_data_end(root, leaf);
4558
Chris Mason5f39d392007-10-15 16:14:19 -04004559 old_data_start = btrfs_item_offset_nr(leaf, slot);
Chris Mason179e29e2007-11-01 11:28:41 -04004560
Chris Masonb18c6682007-04-17 13:26:50 -04004561 size_diff = old_size - new_size;
4562
4563 BUG_ON(slot < 0);
4564 BUG_ON(slot >= nritems);
4565
4566 /*
4567 * item0..itemN ... dataN.offset..dataN.size .. data0.size
4568 */
4569 /* first correct the data pointers */
4570 for (i = slot; i < nritems; i++) {
Chris Mason5f39d392007-10-15 16:14:19 -04004571 u32 ioff;
Ross Kirkdd3cc162013-09-16 15:58:09 +01004572 item = btrfs_item_nr(i);
Chris Masondb945352007-10-15 16:15:53 -04004573
Chris Masoncfed81a2012-03-03 07:40:03 -05004574 ioff = btrfs_token_item_offset(leaf, item, &token);
4575 btrfs_set_token_item_offset(leaf, item,
4576 ioff + size_diff, &token);
Chris Masonb18c6682007-04-17 13:26:50 -04004577 }
Chris Masondb945352007-10-15 16:15:53 -04004578
Chris Masonb18c6682007-04-17 13:26:50 -04004579 /* shift the data */
Chris Mason179e29e2007-11-01 11:28:41 -04004580 if (from_end) {
4581 memmove_extent_buffer(leaf, btrfs_leaf_data(leaf) +
4582 data_end + size_diff, btrfs_leaf_data(leaf) +
4583 data_end, old_data_start + new_size - data_end);
4584 } else {
4585 struct btrfs_disk_key disk_key;
4586 u64 offset;
4587
4588 btrfs_item_key(leaf, &disk_key, slot);
4589
4590 if (btrfs_disk_key_type(&disk_key) == BTRFS_EXTENT_DATA_KEY) {
4591 unsigned long ptr;
4592 struct btrfs_file_extent_item *fi;
4593
4594 fi = btrfs_item_ptr(leaf, slot,
4595 struct btrfs_file_extent_item);
4596 fi = (struct btrfs_file_extent_item *)(
4597 (unsigned long)fi - size_diff);
4598
4599 if (btrfs_file_extent_type(leaf, fi) ==
4600 BTRFS_FILE_EXTENT_INLINE) {
4601 ptr = btrfs_item_ptr_offset(leaf, slot);
4602 memmove_extent_buffer(leaf, ptr,
Chris Masond3977122009-01-05 21:25:51 -05004603 (unsigned long)fi,
David Sterba7ec20af2014-07-24 17:34:58 +02004604 BTRFS_FILE_EXTENT_INLINE_DATA_START);
Chris Mason179e29e2007-11-01 11:28:41 -04004605 }
4606 }
4607
4608 memmove_extent_buffer(leaf, btrfs_leaf_data(leaf) +
4609 data_end + size_diff, btrfs_leaf_data(leaf) +
4610 data_end, old_data_start - data_end);
4611
4612 offset = btrfs_disk_key_offset(&disk_key);
4613 btrfs_set_disk_key_offset(&disk_key, offset + size_diff);
4614 btrfs_set_item_key(leaf, &disk_key, slot);
4615 if (slot == 0)
Tsutomu Itohd6a0a122013-04-16 05:18:02 +00004616 fixup_low_keys(root, path, &disk_key, 1);
Chris Mason179e29e2007-11-01 11:28:41 -04004617 }
Chris Mason5f39d392007-10-15 16:14:19 -04004618
Ross Kirkdd3cc162013-09-16 15:58:09 +01004619 item = btrfs_item_nr(slot);
Chris Mason5f39d392007-10-15 16:14:19 -04004620 btrfs_set_item_size(leaf, item, new_size);
4621 btrfs_mark_buffer_dirty(leaf);
Chris Masonb18c6682007-04-17 13:26:50 -04004622
Chris Mason5f39d392007-10-15 16:14:19 -04004623 if (btrfs_leaf_free_space(root, leaf) < 0) {
4624 btrfs_print_leaf(root, leaf);
Chris Masonb18c6682007-04-17 13:26:50 -04004625 BUG();
Chris Mason5f39d392007-10-15 16:14:19 -04004626 }
Chris Masonb18c6682007-04-17 13:26:50 -04004627}
4628
Chris Masond352ac62008-09-29 15:18:18 -04004629/*
Stefan Behrens8f69dbd2013-05-07 10:23:30 +00004630 * make the item pointed to by the path bigger, data_size is the added size.
Chris Masond352ac62008-09-29 15:18:18 -04004631 */
Tsutomu Itoh4b90c682013-04-16 05:18:49 +00004632void btrfs_extend_item(struct btrfs_root *root, struct btrfs_path *path,
Jeff Mahoney143bede2012-03-01 14:56:26 +01004633 u32 data_size)
Chris Mason6567e832007-04-16 09:22:45 -04004634{
Chris Mason6567e832007-04-16 09:22:45 -04004635 int slot;
Chris Mason5f39d392007-10-15 16:14:19 -04004636 struct extent_buffer *leaf;
4637 struct btrfs_item *item;
Chris Mason6567e832007-04-16 09:22:45 -04004638 u32 nritems;
4639 unsigned int data_end;
4640 unsigned int old_data;
4641 unsigned int old_size;
4642 int i;
Chris Masoncfed81a2012-03-03 07:40:03 -05004643 struct btrfs_map_token token;
4644
4645 btrfs_init_map_token(&token);
Chris Mason6567e832007-04-16 09:22:45 -04004646
Chris Mason5f39d392007-10-15 16:14:19 -04004647 leaf = path->nodes[0];
Chris Mason6567e832007-04-16 09:22:45 -04004648
Chris Mason5f39d392007-10-15 16:14:19 -04004649 nritems = btrfs_header_nritems(leaf);
Chris Mason6567e832007-04-16 09:22:45 -04004650 data_end = leaf_data_end(root, leaf);
4651
Chris Mason5f39d392007-10-15 16:14:19 -04004652 if (btrfs_leaf_free_space(root, leaf) < data_size) {
4653 btrfs_print_leaf(root, leaf);
Chris Mason6567e832007-04-16 09:22:45 -04004654 BUG();
Chris Mason5f39d392007-10-15 16:14:19 -04004655 }
Chris Mason6567e832007-04-16 09:22:45 -04004656 slot = path->slots[0];
Chris Mason5f39d392007-10-15 16:14:19 -04004657 old_data = btrfs_item_end_nr(leaf, slot);
Chris Mason6567e832007-04-16 09:22:45 -04004658
4659 BUG_ON(slot < 0);
Chris Mason3326d1b2007-10-15 16:18:25 -04004660 if (slot >= nritems) {
4661 btrfs_print_leaf(root, leaf);
Frank Holtonefe120a2013-12-20 11:37:06 -05004662 btrfs_crit(root->fs_info, "slot %d too large, nritems %d",
Chris Masond3977122009-01-05 21:25:51 -05004663 slot, nritems);
Chris Mason3326d1b2007-10-15 16:18:25 -04004664 BUG_ON(1);
4665 }
Chris Mason6567e832007-04-16 09:22:45 -04004666
4667 /*
4668 * item0..itemN ... dataN.offset..dataN.size .. data0.size
4669 */
4670 /* first correct the data pointers */
4671 for (i = slot; i < nritems; i++) {
Chris Mason5f39d392007-10-15 16:14:19 -04004672 u32 ioff;
Ross Kirkdd3cc162013-09-16 15:58:09 +01004673 item = btrfs_item_nr(i);
Chris Masondb945352007-10-15 16:15:53 -04004674
Chris Masoncfed81a2012-03-03 07:40:03 -05004675 ioff = btrfs_token_item_offset(leaf, item, &token);
4676 btrfs_set_token_item_offset(leaf, item,
4677 ioff - data_size, &token);
Chris Mason6567e832007-04-16 09:22:45 -04004678 }
Chris Mason5f39d392007-10-15 16:14:19 -04004679
Chris Mason6567e832007-04-16 09:22:45 -04004680 /* shift the data */
Chris Mason5f39d392007-10-15 16:14:19 -04004681 memmove_extent_buffer(leaf, btrfs_leaf_data(leaf) +
Chris Mason6567e832007-04-16 09:22:45 -04004682 data_end - data_size, btrfs_leaf_data(leaf) +
4683 data_end, old_data - data_end);
Chris Mason5f39d392007-10-15 16:14:19 -04004684
Chris Mason6567e832007-04-16 09:22:45 -04004685 data_end = old_data;
Chris Mason5f39d392007-10-15 16:14:19 -04004686 old_size = btrfs_item_size_nr(leaf, slot);
Ross Kirkdd3cc162013-09-16 15:58:09 +01004687 item = btrfs_item_nr(slot);
Chris Mason5f39d392007-10-15 16:14:19 -04004688 btrfs_set_item_size(leaf, item, old_size + data_size);
4689 btrfs_mark_buffer_dirty(leaf);
Chris Mason6567e832007-04-16 09:22:45 -04004690
Chris Mason5f39d392007-10-15 16:14:19 -04004691 if (btrfs_leaf_free_space(root, leaf) < 0) {
4692 btrfs_print_leaf(root, leaf);
Chris Mason6567e832007-04-16 09:22:45 -04004693 BUG();
Chris Mason5f39d392007-10-15 16:14:19 -04004694 }
Chris Mason6567e832007-04-16 09:22:45 -04004695}
4696
Chris Mason74123bd2007-02-02 11:05:29 -05004697/*
Chris Mason44871b12009-03-13 10:04:31 -04004698 * this is a helper for btrfs_insert_empty_items, the main goal here is
4699 * to save stack depth by doing the bulk of the work in a function
4700 * that doesn't call btrfs_search_slot
Chris Mason74123bd2007-02-02 11:05:29 -05004701 */
Tsutomu Itohafe5fea2013-04-16 05:18:22 +00004702void setup_items_for_insert(struct btrfs_root *root, struct btrfs_path *path,
Jeff Mahoney143bede2012-03-01 14:56:26 +01004703 struct btrfs_key *cpu_key, u32 *data_size,
4704 u32 total_data, u32 total_size, int nr)
Chris Masonbe0e5c02007-01-26 15:51:26 -05004705{
Chris Mason5f39d392007-10-15 16:14:19 -04004706 struct btrfs_item *item;
Chris Mason9c583092008-01-29 15:15:18 -05004707 int i;
Chris Mason7518a232007-03-12 12:01:18 -04004708 u32 nritems;
Chris Masonbe0e5c02007-01-26 15:51:26 -05004709 unsigned int data_end;
Chris Masone2fa7222007-03-12 16:22:34 -04004710 struct btrfs_disk_key disk_key;
Chris Mason44871b12009-03-13 10:04:31 -04004711 struct extent_buffer *leaf;
4712 int slot;
Chris Masoncfed81a2012-03-03 07:40:03 -05004713 struct btrfs_map_token token;
4714
Filipe Manana24cdc842014-07-28 19:34:35 +01004715 if (path->slots[0] == 0) {
4716 btrfs_cpu_key_to_disk(&disk_key, cpu_key);
4717 fixup_low_keys(root, path, &disk_key, 1);
4718 }
4719 btrfs_unlock_up_safe(path, 1);
4720
Chris Masoncfed81a2012-03-03 07:40:03 -05004721 btrfs_init_map_token(&token);
Chris Masone2fa7222007-03-12 16:22:34 -04004722
Chris Mason5f39d392007-10-15 16:14:19 -04004723 leaf = path->nodes[0];
Chris Mason44871b12009-03-13 10:04:31 -04004724 slot = path->slots[0];
Chris Mason74123bd2007-02-02 11:05:29 -05004725
Chris Mason5f39d392007-10-15 16:14:19 -04004726 nritems = btrfs_header_nritems(leaf);
Chris Mason123abc82007-03-14 14:14:43 -04004727 data_end = leaf_data_end(root, leaf);
Chris Masoneb60cea2007-02-02 09:18:22 -05004728
Chris Masonf25956c2008-09-12 15:32:53 -04004729 if (btrfs_leaf_free_space(root, leaf) < total_size) {
Chris Mason3326d1b2007-10-15 16:18:25 -04004730 btrfs_print_leaf(root, leaf);
Frank Holtonefe120a2013-12-20 11:37:06 -05004731 btrfs_crit(root->fs_info, "not enough freespace need %u have %d",
Chris Mason9c583092008-01-29 15:15:18 -05004732 total_size, btrfs_leaf_free_space(root, leaf));
Chris Masonbe0e5c02007-01-26 15:51:26 -05004733 BUG();
Chris Masond4dbff92007-04-04 14:08:15 -04004734 }
Chris Mason5f39d392007-10-15 16:14:19 -04004735
Chris Masonbe0e5c02007-01-26 15:51:26 -05004736 if (slot != nritems) {
Chris Mason5f39d392007-10-15 16:14:19 -04004737 unsigned int old_data = btrfs_item_end_nr(leaf, slot);
Chris Masonbe0e5c02007-01-26 15:51:26 -05004738
Chris Mason5f39d392007-10-15 16:14:19 -04004739 if (old_data < data_end) {
4740 btrfs_print_leaf(root, leaf);
Frank Holtonefe120a2013-12-20 11:37:06 -05004741 btrfs_crit(root->fs_info, "slot %d old_data %d data_end %d",
Chris Mason5f39d392007-10-15 16:14:19 -04004742 slot, old_data, data_end);
4743 BUG_ON(1);
4744 }
Chris Masonbe0e5c02007-01-26 15:51:26 -05004745 /*
4746 * item0..itemN ... dataN.offset..dataN.size .. data0.size
4747 */
4748 /* first correct the data pointers */
Chris Mason0783fcf2007-03-12 20:12:07 -04004749 for (i = slot; i < nritems; i++) {
Chris Mason5f39d392007-10-15 16:14:19 -04004750 u32 ioff;
Chris Masondb945352007-10-15 16:15:53 -04004751
Ross Kirkdd3cc162013-09-16 15:58:09 +01004752 item = btrfs_item_nr( i);
Chris Masoncfed81a2012-03-03 07:40:03 -05004753 ioff = btrfs_token_item_offset(leaf, item, &token);
4754 btrfs_set_token_item_offset(leaf, item,
4755 ioff - total_data, &token);
Chris Mason0783fcf2007-03-12 20:12:07 -04004756 }
Chris Masonbe0e5c02007-01-26 15:51:26 -05004757 /* shift the items */
Chris Mason9c583092008-01-29 15:15:18 -05004758 memmove_extent_buffer(leaf, btrfs_item_nr_offset(slot + nr),
Chris Mason5f39d392007-10-15 16:14:19 -04004759 btrfs_item_nr_offset(slot),
Chris Masond6025572007-03-30 14:27:56 -04004760 (nritems - slot) * sizeof(struct btrfs_item));
Chris Masonbe0e5c02007-01-26 15:51:26 -05004761
4762 /* shift the data */
Chris Mason5f39d392007-10-15 16:14:19 -04004763 memmove_extent_buffer(leaf, btrfs_leaf_data(leaf) +
Chris Mason9c583092008-01-29 15:15:18 -05004764 data_end - total_data, btrfs_leaf_data(leaf) +
Chris Masond6025572007-03-30 14:27:56 -04004765 data_end, old_data - data_end);
Chris Masonbe0e5c02007-01-26 15:51:26 -05004766 data_end = old_data;
4767 }
Chris Mason5f39d392007-10-15 16:14:19 -04004768
Chris Mason62e27492007-03-15 12:56:47 -04004769 /* setup the item for the new data */
Chris Mason9c583092008-01-29 15:15:18 -05004770 for (i = 0; i < nr; i++) {
4771 btrfs_cpu_key_to_disk(&disk_key, cpu_key + i);
4772 btrfs_set_item_key(leaf, &disk_key, slot + i);
Ross Kirkdd3cc162013-09-16 15:58:09 +01004773 item = btrfs_item_nr(slot + i);
Chris Masoncfed81a2012-03-03 07:40:03 -05004774 btrfs_set_token_item_offset(leaf, item,
4775 data_end - data_size[i], &token);
Chris Mason9c583092008-01-29 15:15:18 -05004776 data_end -= data_size[i];
Chris Masoncfed81a2012-03-03 07:40:03 -05004777 btrfs_set_token_item_size(leaf, item, data_size[i], &token);
Chris Mason9c583092008-01-29 15:15:18 -05004778 }
Chris Mason44871b12009-03-13 10:04:31 -04004779
Chris Mason9c583092008-01-29 15:15:18 -05004780 btrfs_set_header_nritems(leaf, nritems + nr);
Chris Masonb9473432009-03-13 11:00:37 -04004781 btrfs_mark_buffer_dirty(leaf);
Chris Masonaa5d6be2007-02-28 16:35:06 -05004782
Chris Mason5f39d392007-10-15 16:14:19 -04004783 if (btrfs_leaf_free_space(root, leaf) < 0) {
4784 btrfs_print_leaf(root, leaf);
Chris Masonbe0e5c02007-01-26 15:51:26 -05004785 BUG();
Chris Mason5f39d392007-10-15 16:14:19 -04004786 }
Chris Mason44871b12009-03-13 10:04:31 -04004787}
4788
4789/*
4790 * Given a key and some data, insert items into the tree.
4791 * This does all the path init required, making room in the tree if needed.
4792 */
4793int btrfs_insert_empty_items(struct btrfs_trans_handle *trans,
4794 struct btrfs_root *root,
4795 struct btrfs_path *path,
4796 struct btrfs_key *cpu_key, u32 *data_size,
4797 int nr)
4798{
Chris Mason44871b12009-03-13 10:04:31 -04004799 int ret = 0;
4800 int slot;
4801 int i;
4802 u32 total_size = 0;
4803 u32 total_data = 0;
4804
4805 for (i = 0; i < nr; i++)
4806 total_data += data_size[i];
4807
4808 total_size = total_data + (nr * sizeof(struct btrfs_item));
4809 ret = btrfs_search_slot(trans, root, cpu_key, path, total_size, 1);
4810 if (ret == 0)
4811 return -EEXIST;
4812 if (ret < 0)
Jeff Mahoney143bede2012-03-01 14:56:26 +01004813 return ret;
Chris Mason44871b12009-03-13 10:04:31 -04004814
Chris Mason44871b12009-03-13 10:04:31 -04004815 slot = path->slots[0];
4816 BUG_ON(slot < 0);
4817
Tsutomu Itohafe5fea2013-04-16 05:18:22 +00004818 setup_items_for_insert(root, path, cpu_key, data_size,
Chris Mason44871b12009-03-13 10:04:31 -04004819 total_data, total_size, nr);
Jeff Mahoney143bede2012-03-01 14:56:26 +01004820 return 0;
Chris Mason62e27492007-03-15 12:56:47 -04004821}
4822
4823/*
4824 * Given a key and some data, insert an item into the tree.
4825 * This does all the path init required, making room in the tree if needed.
4826 */
Chris Masone089f052007-03-16 16:20:31 -04004827int btrfs_insert_item(struct btrfs_trans_handle *trans, struct btrfs_root
4828 *root, struct btrfs_key *cpu_key, void *data, u32
4829 data_size)
Chris Mason62e27492007-03-15 12:56:47 -04004830{
4831 int ret = 0;
Chris Mason2c90e5d2007-04-02 10:50:19 -04004832 struct btrfs_path *path;
Chris Mason5f39d392007-10-15 16:14:19 -04004833 struct extent_buffer *leaf;
4834 unsigned long ptr;
Chris Mason62e27492007-03-15 12:56:47 -04004835
Chris Mason2c90e5d2007-04-02 10:50:19 -04004836 path = btrfs_alloc_path();
Tsutomu Itohdb5b4932011-03-23 08:14:16 +00004837 if (!path)
4838 return -ENOMEM;
Chris Mason2c90e5d2007-04-02 10:50:19 -04004839 ret = btrfs_insert_empty_item(trans, root, path, cpu_key, data_size);
Chris Mason62e27492007-03-15 12:56:47 -04004840 if (!ret) {
Chris Mason5f39d392007-10-15 16:14:19 -04004841 leaf = path->nodes[0];
4842 ptr = btrfs_item_ptr_offset(leaf, path->slots[0]);
4843 write_extent_buffer(leaf, data, ptr, data_size);
4844 btrfs_mark_buffer_dirty(leaf);
Chris Mason62e27492007-03-15 12:56:47 -04004845 }
Chris Mason2c90e5d2007-04-02 10:50:19 -04004846 btrfs_free_path(path);
Chris Masonaa5d6be2007-02-28 16:35:06 -05004847 return ret;
Chris Masonbe0e5c02007-01-26 15:51:26 -05004848}
4849
Chris Mason74123bd2007-02-02 11:05:29 -05004850/*
Chris Mason5de08d72007-02-24 06:24:44 -05004851 * delete the pointer from a given node.
Chris Mason74123bd2007-02-02 11:05:29 -05004852 *
Chris Masond352ac62008-09-29 15:18:18 -04004853 * the tree should have been previously balanced so the deletion does not
4854 * empty a node.
Chris Mason74123bd2007-02-02 11:05:29 -05004855 */
Tsutomu Itohafe5fea2013-04-16 05:18:22 +00004856static void del_ptr(struct btrfs_root *root, struct btrfs_path *path,
4857 int level, int slot)
Chris Masonbe0e5c02007-01-26 15:51:26 -05004858{
Chris Mason5f39d392007-10-15 16:14:19 -04004859 struct extent_buffer *parent = path->nodes[level];
Chris Mason7518a232007-03-12 12:01:18 -04004860 u32 nritems;
Jan Schmidtf3ea38d2012-05-26 11:45:21 +02004861 int ret;
Chris Masonbe0e5c02007-01-26 15:51:26 -05004862
Chris Mason5f39d392007-10-15 16:14:19 -04004863 nritems = btrfs_header_nritems(parent);
Chris Masond3977122009-01-05 21:25:51 -05004864 if (slot != nritems - 1) {
Liu Bo0e411ec2012-10-19 09:50:54 +00004865 if (level)
Jan Schmidtf3ea38d2012-05-26 11:45:21 +02004866 tree_mod_log_eb_move(root->fs_info, parent, slot,
4867 slot + 1, nritems - slot - 1);
Chris Mason5f39d392007-10-15 16:14:19 -04004868 memmove_extent_buffer(parent,
4869 btrfs_node_key_ptr_offset(slot),
4870 btrfs_node_key_ptr_offset(slot + 1),
Chris Masond6025572007-03-30 14:27:56 -04004871 sizeof(struct btrfs_key_ptr) *
4872 (nritems - slot - 1));
Chris Mason57ba86c2012-12-18 19:35:32 -05004873 } else if (level) {
4874 ret = tree_mod_log_insert_key(root->fs_info, parent, slot,
Josef Bacikc8cc6342013-07-01 16:18:19 -04004875 MOD_LOG_KEY_REMOVE, GFP_NOFS);
Chris Mason57ba86c2012-12-18 19:35:32 -05004876 BUG_ON(ret < 0);
Chris Masonbb803952007-03-01 12:04:21 -05004877 }
Jan Schmidtf3ea38d2012-05-26 11:45:21 +02004878
Chris Mason7518a232007-03-12 12:01:18 -04004879 nritems--;
Chris Mason5f39d392007-10-15 16:14:19 -04004880 btrfs_set_header_nritems(parent, nritems);
Chris Mason7518a232007-03-12 12:01:18 -04004881 if (nritems == 0 && parent == root->node) {
Chris Mason5f39d392007-10-15 16:14:19 -04004882 BUG_ON(btrfs_header_level(root->node) != 1);
Chris Masonbb803952007-03-01 12:04:21 -05004883 /* just turn the root into a leaf and break */
Chris Mason5f39d392007-10-15 16:14:19 -04004884 btrfs_set_header_level(root->node, 0);
Chris Masonbb803952007-03-01 12:04:21 -05004885 } else if (slot == 0) {
Chris Mason5f39d392007-10-15 16:14:19 -04004886 struct btrfs_disk_key disk_key;
4887
4888 btrfs_node_key(parent, &disk_key, 0);
Tsutomu Itohd6a0a122013-04-16 05:18:02 +00004889 fixup_low_keys(root, path, &disk_key, level + 1);
Chris Masonbe0e5c02007-01-26 15:51:26 -05004890 }
Chris Masond6025572007-03-30 14:27:56 -04004891 btrfs_mark_buffer_dirty(parent);
Chris Masonbe0e5c02007-01-26 15:51:26 -05004892}
4893
Chris Mason74123bd2007-02-02 11:05:29 -05004894/*
Chris Mason323ac952008-10-01 19:05:46 -04004895 * a helper function to delete the leaf pointed to by path->slots[1] and
Yan Zheng5d4f98a2009-06-10 10:45:14 -04004896 * path->nodes[1].
Chris Mason323ac952008-10-01 19:05:46 -04004897 *
4898 * This deletes the pointer in path->nodes[1] and frees the leaf
4899 * block extent. zero is returned if it all worked out, < 0 otherwise.
4900 *
4901 * The path must have already been setup for deleting the leaf, including
4902 * all the proper balancing. path->nodes[1] must be locked.
4903 */
Jeff Mahoney143bede2012-03-01 14:56:26 +01004904static noinline void btrfs_del_leaf(struct btrfs_trans_handle *trans,
4905 struct btrfs_root *root,
4906 struct btrfs_path *path,
4907 struct extent_buffer *leaf)
Chris Mason323ac952008-10-01 19:05:46 -04004908{
Yan Zheng5d4f98a2009-06-10 10:45:14 -04004909 WARN_ON(btrfs_header_generation(leaf) != trans->transid);
Tsutomu Itohafe5fea2013-04-16 05:18:22 +00004910 del_ptr(root, path, 1, path->slots[1]);
Chris Mason323ac952008-10-01 19:05:46 -04004911
Chris Mason4d081c42009-02-04 09:31:28 -05004912 /*
4913 * btrfs_free_extent is expensive, we want to make sure we
4914 * aren't holding any locks when we call it
4915 */
4916 btrfs_unlock_up_safe(path, 0);
4917
Yan, Zhengf0486c62010-05-16 10:46:25 -04004918 root_sub_used(root, leaf->len);
4919
Josef Bacik3083ee22012-03-09 16:01:49 -05004920 extent_buffer_get(leaf);
Jan Schmidt5581a512012-05-16 17:04:52 +02004921 btrfs_free_tree_block(trans, root, leaf, 0, 1);
Josef Bacik3083ee22012-03-09 16:01:49 -05004922 free_extent_buffer_stale(leaf);
Chris Mason323ac952008-10-01 19:05:46 -04004923}
4924/*
Chris Mason74123bd2007-02-02 11:05:29 -05004925 * delete the item at the leaf level in path. If that empties
4926 * the leaf, remove it from the tree
4927 */
Chris Mason85e21ba2008-01-29 15:11:36 -05004928int btrfs_del_items(struct btrfs_trans_handle *trans, struct btrfs_root *root,
4929 struct btrfs_path *path, int slot, int nr)
Chris Masonbe0e5c02007-01-26 15:51:26 -05004930{
Chris Mason5f39d392007-10-15 16:14:19 -04004931 struct extent_buffer *leaf;
4932 struct btrfs_item *item;
Chris Mason85e21ba2008-01-29 15:11:36 -05004933 int last_off;
4934 int dsize = 0;
Chris Masonaa5d6be2007-02-28 16:35:06 -05004935 int ret = 0;
4936 int wret;
Chris Mason85e21ba2008-01-29 15:11:36 -05004937 int i;
Chris Mason7518a232007-03-12 12:01:18 -04004938 u32 nritems;
Chris Masoncfed81a2012-03-03 07:40:03 -05004939 struct btrfs_map_token token;
4940
4941 btrfs_init_map_token(&token);
Chris Masonbe0e5c02007-01-26 15:51:26 -05004942
Chris Mason5f39d392007-10-15 16:14:19 -04004943 leaf = path->nodes[0];
Chris Mason85e21ba2008-01-29 15:11:36 -05004944 last_off = btrfs_item_offset_nr(leaf, slot + nr - 1);
4945
4946 for (i = 0; i < nr; i++)
4947 dsize += btrfs_item_size_nr(leaf, slot + i);
4948
Chris Mason5f39d392007-10-15 16:14:19 -04004949 nritems = btrfs_header_nritems(leaf);
Chris Masonbe0e5c02007-01-26 15:51:26 -05004950
Chris Mason85e21ba2008-01-29 15:11:36 -05004951 if (slot + nr != nritems) {
Chris Mason123abc82007-03-14 14:14:43 -04004952 int data_end = leaf_data_end(root, leaf);
Chris Mason5f39d392007-10-15 16:14:19 -04004953
4954 memmove_extent_buffer(leaf, btrfs_leaf_data(leaf) +
Chris Masond6025572007-03-30 14:27:56 -04004955 data_end + dsize,
4956 btrfs_leaf_data(leaf) + data_end,
Chris Mason85e21ba2008-01-29 15:11:36 -05004957 last_off - data_end);
Chris Mason5f39d392007-10-15 16:14:19 -04004958
Chris Mason85e21ba2008-01-29 15:11:36 -05004959 for (i = slot + nr; i < nritems; i++) {
Chris Mason5f39d392007-10-15 16:14:19 -04004960 u32 ioff;
Chris Masondb945352007-10-15 16:15:53 -04004961
Ross Kirkdd3cc162013-09-16 15:58:09 +01004962 item = btrfs_item_nr(i);
Chris Masoncfed81a2012-03-03 07:40:03 -05004963 ioff = btrfs_token_item_offset(leaf, item, &token);
4964 btrfs_set_token_item_offset(leaf, item,
4965 ioff + dsize, &token);
Chris Mason0783fcf2007-03-12 20:12:07 -04004966 }
Chris Masondb945352007-10-15 16:15:53 -04004967
Chris Mason5f39d392007-10-15 16:14:19 -04004968 memmove_extent_buffer(leaf, btrfs_item_nr_offset(slot),
Chris Mason85e21ba2008-01-29 15:11:36 -05004969 btrfs_item_nr_offset(slot + nr),
Chris Masond6025572007-03-30 14:27:56 -04004970 sizeof(struct btrfs_item) *
Chris Mason85e21ba2008-01-29 15:11:36 -05004971 (nritems - slot - nr));
Chris Masonbe0e5c02007-01-26 15:51:26 -05004972 }
Chris Mason85e21ba2008-01-29 15:11:36 -05004973 btrfs_set_header_nritems(leaf, nritems - nr);
4974 nritems -= nr;
Chris Mason5f39d392007-10-15 16:14:19 -04004975
Chris Mason74123bd2007-02-02 11:05:29 -05004976 /* delete the leaf if we've emptied it */
Chris Mason7518a232007-03-12 12:01:18 -04004977 if (nritems == 0) {
Chris Mason5f39d392007-10-15 16:14:19 -04004978 if (leaf == root->node) {
4979 btrfs_set_header_level(leaf, 0);
Chris Mason9a8dd152007-02-23 08:38:36 -05004980 } else {
Yan, Zhengf0486c62010-05-16 10:46:25 -04004981 btrfs_set_path_blocking(path);
4982 clean_tree_block(trans, root, leaf);
Jeff Mahoney143bede2012-03-01 14:56:26 +01004983 btrfs_del_leaf(trans, root, path, leaf);
Chris Mason9a8dd152007-02-23 08:38:36 -05004984 }
Chris Masonbe0e5c02007-01-26 15:51:26 -05004985 } else {
Chris Mason7518a232007-03-12 12:01:18 -04004986 int used = leaf_space_used(leaf, 0, nritems);
Chris Masonaa5d6be2007-02-28 16:35:06 -05004987 if (slot == 0) {
Chris Mason5f39d392007-10-15 16:14:19 -04004988 struct btrfs_disk_key disk_key;
4989
4990 btrfs_item_key(leaf, &disk_key, 0);
Tsutomu Itohd6a0a122013-04-16 05:18:02 +00004991 fixup_low_keys(root, path, &disk_key, 1);
Chris Masonaa5d6be2007-02-28 16:35:06 -05004992 }
Chris Masonaa5d6be2007-02-28 16:35:06 -05004993
Chris Mason74123bd2007-02-02 11:05:29 -05004994 /* delete the leaf if it is mostly empty */
Yan Zhengd717aa12009-07-24 12:42:46 -04004995 if (used < BTRFS_LEAF_DATA_SIZE(root) / 3) {
Chris Masonbe0e5c02007-01-26 15:51:26 -05004996 /* push_leaf_left fixes the path.
4997 * make sure the path still points to our leaf
4998 * for possible call to del_ptr below
4999 */
Chris Mason4920c9a2007-01-26 16:38:42 -05005000 slot = path->slots[1];
Chris Mason5f39d392007-10-15 16:14:19 -04005001 extent_buffer_get(leaf);
5002
Chris Masonb9473432009-03-13 11:00:37 -04005003 btrfs_set_path_blocking(path);
Chris Mason99d8f832010-07-07 10:51:48 -04005004 wret = push_leaf_left(trans, root, path, 1, 1,
5005 1, (u32)-1);
Chris Mason54aa1f42007-06-22 14:16:25 -04005006 if (wret < 0 && wret != -ENOSPC)
Chris Masonaa5d6be2007-02-28 16:35:06 -05005007 ret = wret;
Chris Mason5f39d392007-10-15 16:14:19 -04005008
5009 if (path->nodes[0] == leaf &&
5010 btrfs_header_nritems(leaf)) {
Chris Mason99d8f832010-07-07 10:51:48 -04005011 wret = push_leaf_right(trans, root, path, 1,
5012 1, 1, 0);
Chris Mason54aa1f42007-06-22 14:16:25 -04005013 if (wret < 0 && wret != -ENOSPC)
Chris Masonaa5d6be2007-02-28 16:35:06 -05005014 ret = wret;
5015 }
Chris Mason5f39d392007-10-15 16:14:19 -04005016
5017 if (btrfs_header_nritems(leaf) == 0) {
Chris Mason323ac952008-10-01 19:05:46 -04005018 path->slots[1] = slot;
Jeff Mahoney143bede2012-03-01 14:56:26 +01005019 btrfs_del_leaf(trans, root, path, leaf);
Chris Mason5f39d392007-10-15 16:14:19 -04005020 free_extent_buffer(leaf);
Jeff Mahoney143bede2012-03-01 14:56:26 +01005021 ret = 0;
Chris Mason5de08d72007-02-24 06:24:44 -05005022 } else {
Chris Mason925baed2008-06-25 16:01:30 -04005023 /* if we're still in the path, make sure
5024 * we're dirty. Otherwise, one of the
5025 * push_leaf functions must have already
5026 * dirtied this buffer
5027 */
5028 if (path->nodes[0] == leaf)
5029 btrfs_mark_buffer_dirty(leaf);
Chris Mason5f39d392007-10-15 16:14:19 -04005030 free_extent_buffer(leaf);
Chris Masonbe0e5c02007-01-26 15:51:26 -05005031 }
Chris Masond5719762007-03-23 10:01:08 -04005032 } else {
Chris Mason5f39d392007-10-15 16:14:19 -04005033 btrfs_mark_buffer_dirty(leaf);
Chris Masonbe0e5c02007-01-26 15:51:26 -05005034 }
5035 }
Chris Masonaa5d6be2007-02-28 16:35:06 -05005036 return ret;
Chris Masonbe0e5c02007-01-26 15:51:26 -05005037}
5038
Chris Mason97571fd2007-02-24 13:39:08 -05005039/*
Chris Mason925baed2008-06-25 16:01:30 -04005040 * search the tree again to find a leaf with lesser keys
Chris Mason7bb86312007-12-11 09:25:06 -05005041 * returns 0 if it found something or 1 if there are no lesser leaves.
5042 * returns < 0 on io errors.
Chris Masond352ac62008-09-29 15:18:18 -04005043 *
5044 * This may release the path, and so you may lose any locks held at the
5045 * time you call it.
Chris Mason7bb86312007-12-11 09:25:06 -05005046 */
Josef Bacik16e75492013-10-22 12:18:51 -04005047int btrfs_prev_leaf(struct btrfs_root *root, struct btrfs_path *path)
Chris Mason7bb86312007-12-11 09:25:06 -05005048{
Chris Mason925baed2008-06-25 16:01:30 -04005049 struct btrfs_key key;
5050 struct btrfs_disk_key found_key;
5051 int ret;
Chris Mason7bb86312007-12-11 09:25:06 -05005052
Chris Mason925baed2008-06-25 16:01:30 -04005053 btrfs_item_key_to_cpu(path->nodes[0], &key, 0);
Chris Mason7bb86312007-12-11 09:25:06 -05005054
Filipe David Borba Mananae8b0d722013-10-15 00:12:27 +01005055 if (key.offset > 0) {
Chris Mason925baed2008-06-25 16:01:30 -04005056 key.offset--;
Filipe David Borba Mananae8b0d722013-10-15 00:12:27 +01005057 } else if (key.type > 0) {
Chris Mason925baed2008-06-25 16:01:30 -04005058 key.type--;
Filipe David Borba Mananae8b0d722013-10-15 00:12:27 +01005059 key.offset = (u64)-1;
5060 } else if (key.objectid > 0) {
Chris Mason925baed2008-06-25 16:01:30 -04005061 key.objectid--;
Filipe David Borba Mananae8b0d722013-10-15 00:12:27 +01005062 key.type = (u8)-1;
5063 key.offset = (u64)-1;
5064 } else {
Chris Mason925baed2008-06-25 16:01:30 -04005065 return 1;
Filipe David Borba Mananae8b0d722013-10-15 00:12:27 +01005066 }
Chris Mason7bb86312007-12-11 09:25:06 -05005067
David Sterbab3b4aa72011-04-21 01:20:15 +02005068 btrfs_release_path(path);
Chris Mason925baed2008-06-25 16:01:30 -04005069 ret = btrfs_search_slot(NULL, root, &key, path, 0, 0);
5070 if (ret < 0)
5071 return ret;
5072 btrfs_item_key(path->nodes[0], &found_key, 0);
5073 ret = comp_keys(&found_key, &key);
Filipe Manana337c6f62014-06-09 13:22:13 +01005074 /*
5075 * We might have had an item with the previous key in the tree right
5076 * before we released our path. And after we released our path, that
5077 * item might have been pushed to the first slot (0) of the leaf we
5078 * were holding due to a tree balance. Alternatively, an item with the
5079 * previous key can exist as the only element of a leaf (big fat item).
5080 * Therefore account for these 2 cases, so that our callers (like
5081 * btrfs_previous_item) don't miss an existing item with a key matching
5082 * the previous key we computed above.
5083 */
5084 if (ret <= 0)
Chris Mason925baed2008-06-25 16:01:30 -04005085 return 0;
5086 return 1;
Chris Mason7bb86312007-12-11 09:25:06 -05005087}
5088
Chris Mason3f157a22008-06-25 16:01:31 -04005089/*
5090 * A helper function to walk down the tree starting at min_key, and looking
Eric Sandeende78b512013-01-31 18:21:12 +00005091 * for nodes or leaves that are have a minimum transaction id.
5092 * This is used by the btree defrag code, and tree logging
Chris Mason3f157a22008-06-25 16:01:31 -04005093 *
5094 * This does not cow, but it does stuff the starting key it finds back
5095 * into min_key, so you can call btrfs_search_slot with cow=1 on the
5096 * key and get a writable path.
5097 *
5098 * This does lock as it descends, and path->keep_locks should be set
5099 * to 1 by the caller.
5100 *
5101 * This honors path->lowest_level to prevent descent past a given level
5102 * of the tree.
5103 *
Chris Masond352ac62008-09-29 15:18:18 -04005104 * min_trans indicates the oldest transaction that you are interested
5105 * in walking through. Any nodes or leaves older than min_trans are
5106 * skipped over (without reading them).
5107 *
Chris Mason3f157a22008-06-25 16:01:31 -04005108 * returns zero if something useful was found, < 0 on error and 1 if there
5109 * was nothing in the tree that matched the search criteria.
5110 */
5111int btrfs_search_forward(struct btrfs_root *root, struct btrfs_key *min_key,
Eric Sandeende78b512013-01-31 18:21:12 +00005112 struct btrfs_path *path,
Chris Mason3f157a22008-06-25 16:01:31 -04005113 u64 min_trans)
5114{
5115 struct extent_buffer *cur;
5116 struct btrfs_key found_key;
5117 int slot;
Yan96524802008-07-24 12:19:49 -04005118 int sret;
Chris Mason3f157a22008-06-25 16:01:31 -04005119 u32 nritems;
5120 int level;
5121 int ret = 1;
Filipe Mananaf98de9b2014-08-04 19:37:21 +01005122 int keep_locks = path->keep_locks;
Chris Mason3f157a22008-06-25 16:01:31 -04005123
Filipe Mananaf98de9b2014-08-04 19:37:21 +01005124 path->keep_locks = 1;
Chris Mason3f157a22008-06-25 16:01:31 -04005125again:
Chris Masonbd681512011-07-16 15:23:14 -04005126 cur = btrfs_read_lock_root_node(root);
Chris Mason3f157a22008-06-25 16:01:31 -04005127 level = btrfs_header_level(cur);
Chris Masone02119d2008-09-05 16:13:11 -04005128 WARN_ON(path->nodes[level]);
Chris Mason3f157a22008-06-25 16:01:31 -04005129 path->nodes[level] = cur;
Chris Masonbd681512011-07-16 15:23:14 -04005130 path->locks[level] = BTRFS_READ_LOCK;
Chris Mason3f157a22008-06-25 16:01:31 -04005131
5132 if (btrfs_header_generation(cur) < min_trans) {
5133 ret = 1;
5134 goto out;
5135 }
Chris Masond3977122009-01-05 21:25:51 -05005136 while (1) {
Chris Mason3f157a22008-06-25 16:01:31 -04005137 nritems = btrfs_header_nritems(cur);
5138 level = btrfs_header_level(cur);
Yan96524802008-07-24 12:19:49 -04005139 sret = bin_search(cur, min_key, level, &slot);
Chris Mason3f157a22008-06-25 16:01:31 -04005140
Chris Mason323ac952008-10-01 19:05:46 -04005141 /* at the lowest level, we're done, setup the path and exit */
5142 if (level == path->lowest_level) {
Chris Masone02119d2008-09-05 16:13:11 -04005143 if (slot >= nritems)
5144 goto find_next_key;
Chris Mason3f157a22008-06-25 16:01:31 -04005145 ret = 0;
5146 path->slots[level] = slot;
5147 btrfs_item_key_to_cpu(cur, &found_key, slot);
5148 goto out;
5149 }
Yan96524802008-07-24 12:19:49 -04005150 if (sret && slot > 0)
5151 slot--;
Chris Mason3f157a22008-06-25 16:01:31 -04005152 /*
Eric Sandeende78b512013-01-31 18:21:12 +00005153 * check this node pointer against the min_trans parameters.
5154 * If it is too old, old, skip to the next one.
Chris Mason3f157a22008-06-25 16:01:31 -04005155 */
Chris Masond3977122009-01-05 21:25:51 -05005156 while (slot < nritems) {
Chris Mason3f157a22008-06-25 16:01:31 -04005157 u64 gen;
Chris Masone02119d2008-09-05 16:13:11 -04005158
Chris Mason3f157a22008-06-25 16:01:31 -04005159 gen = btrfs_node_ptr_generation(cur, slot);
5160 if (gen < min_trans) {
5161 slot++;
5162 continue;
5163 }
Eric Sandeende78b512013-01-31 18:21:12 +00005164 break;
Chris Mason3f157a22008-06-25 16:01:31 -04005165 }
Chris Masone02119d2008-09-05 16:13:11 -04005166find_next_key:
Chris Mason3f157a22008-06-25 16:01:31 -04005167 /*
5168 * we didn't find a candidate key in this node, walk forward
5169 * and find another one
5170 */
5171 if (slot >= nritems) {
Chris Masone02119d2008-09-05 16:13:11 -04005172 path->slots[level] = slot;
Chris Masonb4ce94d2009-02-04 09:25:08 -05005173 btrfs_set_path_blocking(path);
Chris Masone02119d2008-09-05 16:13:11 -04005174 sret = btrfs_find_next_key(root, path, min_key, level,
Eric Sandeende78b512013-01-31 18:21:12 +00005175 min_trans);
Chris Masone02119d2008-09-05 16:13:11 -04005176 if (sret == 0) {
David Sterbab3b4aa72011-04-21 01:20:15 +02005177 btrfs_release_path(path);
Chris Mason3f157a22008-06-25 16:01:31 -04005178 goto again;
5179 } else {
5180 goto out;
5181 }
5182 }
5183 /* save our key for returning back */
5184 btrfs_node_key_to_cpu(cur, &found_key, slot);
5185 path->slots[level] = slot;
5186 if (level == path->lowest_level) {
5187 ret = 0;
Chris Mason3f157a22008-06-25 16:01:31 -04005188 goto out;
5189 }
Chris Masonb4ce94d2009-02-04 09:25:08 -05005190 btrfs_set_path_blocking(path);
Chris Mason3f157a22008-06-25 16:01:31 -04005191 cur = read_node_slot(root, cur, slot);
Jeff Mahoney79787ea2012-03-12 16:03:00 +01005192 BUG_ON(!cur); /* -ENOMEM */
Chris Mason3f157a22008-06-25 16:01:31 -04005193
Chris Masonbd681512011-07-16 15:23:14 -04005194 btrfs_tree_read_lock(cur);
Chris Masonb4ce94d2009-02-04 09:25:08 -05005195
Chris Masonbd681512011-07-16 15:23:14 -04005196 path->locks[level - 1] = BTRFS_READ_LOCK;
Chris Mason3f157a22008-06-25 16:01:31 -04005197 path->nodes[level - 1] = cur;
Chris Masonf7c79f32012-03-19 15:54:38 -04005198 unlock_up(path, level, 1, 0, NULL);
Chris Masonbd681512011-07-16 15:23:14 -04005199 btrfs_clear_path_blocking(path, NULL, 0);
Chris Mason3f157a22008-06-25 16:01:31 -04005200 }
5201out:
Filipe Mananaf98de9b2014-08-04 19:37:21 +01005202 path->keep_locks = keep_locks;
5203 if (ret == 0) {
5204 btrfs_unlock_up_safe(path, path->lowest_level + 1);
5205 btrfs_set_path_blocking(path);
Chris Mason3f157a22008-06-25 16:01:31 -04005206 memcpy(min_key, &found_key, sizeof(found_key));
Filipe Mananaf98de9b2014-08-04 19:37:21 +01005207 }
Chris Mason3f157a22008-06-25 16:01:31 -04005208 return ret;
5209}
5210
Alexander Block70698302012-06-05 21:07:48 +02005211static void tree_move_down(struct btrfs_root *root,
5212 struct btrfs_path *path,
5213 int *level, int root_level)
5214{
Chris Mason74dd17f2012-08-07 16:25:13 -04005215 BUG_ON(*level == 0);
Alexander Block70698302012-06-05 21:07:48 +02005216 path->nodes[*level - 1] = read_node_slot(root, path->nodes[*level],
5217 path->slots[*level]);
5218 path->slots[*level - 1] = 0;
5219 (*level)--;
5220}
5221
5222static int tree_move_next_or_upnext(struct btrfs_root *root,
5223 struct btrfs_path *path,
5224 int *level, int root_level)
5225{
5226 int ret = 0;
5227 int nritems;
5228 nritems = btrfs_header_nritems(path->nodes[*level]);
5229
5230 path->slots[*level]++;
5231
Chris Mason74dd17f2012-08-07 16:25:13 -04005232 while (path->slots[*level] >= nritems) {
Alexander Block70698302012-06-05 21:07:48 +02005233 if (*level == root_level)
5234 return -1;
5235
5236 /* move upnext */
5237 path->slots[*level] = 0;
5238 free_extent_buffer(path->nodes[*level]);
5239 path->nodes[*level] = NULL;
5240 (*level)++;
5241 path->slots[*level]++;
5242
5243 nritems = btrfs_header_nritems(path->nodes[*level]);
5244 ret = 1;
5245 }
5246 return ret;
5247}
5248
5249/*
5250 * Returns 1 if it had to move up and next. 0 is returned if it moved only next
5251 * or down.
5252 */
5253static int tree_advance(struct btrfs_root *root,
5254 struct btrfs_path *path,
5255 int *level, int root_level,
5256 int allow_down,
5257 struct btrfs_key *key)
5258{
5259 int ret;
5260
5261 if (*level == 0 || !allow_down) {
5262 ret = tree_move_next_or_upnext(root, path, level, root_level);
5263 } else {
5264 tree_move_down(root, path, level, root_level);
5265 ret = 0;
5266 }
5267 if (ret >= 0) {
5268 if (*level == 0)
5269 btrfs_item_key_to_cpu(path->nodes[*level], key,
5270 path->slots[*level]);
5271 else
5272 btrfs_node_key_to_cpu(path->nodes[*level], key,
5273 path->slots[*level]);
5274 }
5275 return ret;
5276}
5277
5278static int tree_compare_item(struct btrfs_root *left_root,
5279 struct btrfs_path *left_path,
5280 struct btrfs_path *right_path,
5281 char *tmp_buf)
5282{
5283 int cmp;
5284 int len1, len2;
5285 unsigned long off1, off2;
5286
5287 len1 = btrfs_item_size_nr(left_path->nodes[0], left_path->slots[0]);
5288 len2 = btrfs_item_size_nr(right_path->nodes[0], right_path->slots[0]);
5289 if (len1 != len2)
5290 return 1;
5291
5292 off1 = btrfs_item_ptr_offset(left_path->nodes[0], left_path->slots[0]);
5293 off2 = btrfs_item_ptr_offset(right_path->nodes[0],
5294 right_path->slots[0]);
5295
5296 read_extent_buffer(left_path->nodes[0], tmp_buf, off1, len1);
5297
5298 cmp = memcmp_extent_buffer(right_path->nodes[0], tmp_buf, off2, len1);
5299 if (cmp)
5300 return 1;
5301 return 0;
5302}
5303
5304#define ADVANCE 1
5305#define ADVANCE_ONLY_NEXT -1
5306
5307/*
5308 * This function compares two trees and calls the provided callback for
5309 * every changed/new/deleted item it finds.
5310 * If shared tree blocks are encountered, whole subtrees are skipped, making
5311 * the compare pretty fast on snapshotted subvolumes.
5312 *
5313 * This currently works on commit roots only. As commit roots are read only,
5314 * we don't do any locking. The commit roots are protected with transactions.
5315 * Transactions are ended and rejoined when a commit is tried in between.
5316 *
5317 * This function checks for modifications done to the trees while comparing.
5318 * If it detects a change, it aborts immediately.
5319 */
5320int btrfs_compare_trees(struct btrfs_root *left_root,
5321 struct btrfs_root *right_root,
5322 btrfs_changed_cb_t changed_cb, void *ctx)
5323{
5324 int ret;
5325 int cmp;
Alexander Block70698302012-06-05 21:07:48 +02005326 struct btrfs_path *left_path = NULL;
5327 struct btrfs_path *right_path = NULL;
5328 struct btrfs_key left_key;
5329 struct btrfs_key right_key;
5330 char *tmp_buf = NULL;
5331 int left_root_level;
5332 int right_root_level;
5333 int left_level;
5334 int right_level;
5335 int left_end_reached;
5336 int right_end_reached;
5337 int advance_left;
5338 int advance_right;
5339 u64 left_blockptr;
5340 u64 right_blockptr;
Filipe Manana6baa4292014-02-20 21:15:25 +00005341 u64 left_gen;
5342 u64 right_gen;
Alexander Block70698302012-06-05 21:07:48 +02005343
5344 left_path = btrfs_alloc_path();
5345 if (!left_path) {
5346 ret = -ENOMEM;
5347 goto out;
5348 }
5349 right_path = btrfs_alloc_path();
5350 if (!right_path) {
5351 ret = -ENOMEM;
5352 goto out;
5353 }
5354
David Sterba707e8a02014-06-04 19:22:26 +02005355 tmp_buf = kmalloc(left_root->nodesize, GFP_NOFS);
Alexander Block70698302012-06-05 21:07:48 +02005356 if (!tmp_buf) {
5357 ret = -ENOMEM;
5358 goto out;
5359 }
5360
5361 left_path->search_commit_root = 1;
5362 left_path->skip_locking = 1;
5363 right_path->search_commit_root = 1;
5364 right_path->skip_locking = 1;
5365
Alexander Block70698302012-06-05 21:07:48 +02005366 /*
5367 * Strategy: Go to the first items of both trees. Then do
5368 *
5369 * If both trees are at level 0
5370 * Compare keys of current items
5371 * If left < right treat left item as new, advance left tree
5372 * and repeat
5373 * If left > right treat right item as deleted, advance right tree
5374 * and repeat
5375 * If left == right do deep compare of items, treat as changed if
5376 * needed, advance both trees and repeat
5377 * If both trees are at the same level but not at level 0
5378 * Compare keys of current nodes/leafs
5379 * If left < right advance left tree and repeat
5380 * If left > right advance right tree and repeat
5381 * If left == right compare blockptrs of the next nodes/leafs
5382 * If they match advance both trees but stay at the same level
5383 * and repeat
5384 * If they don't match advance both trees while allowing to go
5385 * deeper and repeat
5386 * If tree levels are different
5387 * Advance the tree that needs it and repeat
5388 *
5389 * Advancing a tree means:
5390 * If we are at level 0, try to go to the next slot. If that's not
5391 * possible, go one level up and repeat. Stop when we found a level
5392 * where we could go to the next slot. We may at this point be on a
5393 * node or a leaf.
5394 *
5395 * If we are not at level 0 and not on shared tree blocks, go one
5396 * level deeper.
5397 *
5398 * If we are not at level 0 and on shared tree blocks, go one slot to
5399 * the right if possible or go up and right.
5400 */
5401
Josef Bacik3f8a18c2014-03-28 17:16:01 -04005402 down_read(&left_root->fs_info->commit_root_sem);
Alexander Block70698302012-06-05 21:07:48 +02005403 left_level = btrfs_header_level(left_root->commit_root);
5404 left_root_level = left_level;
5405 left_path->nodes[left_level] = left_root->commit_root;
5406 extent_buffer_get(left_path->nodes[left_level]);
5407
5408 right_level = btrfs_header_level(right_root->commit_root);
5409 right_root_level = right_level;
5410 right_path->nodes[right_level] = right_root->commit_root;
5411 extent_buffer_get(right_path->nodes[right_level]);
Josef Bacik3f8a18c2014-03-28 17:16:01 -04005412 up_read(&left_root->fs_info->commit_root_sem);
Alexander Block70698302012-06-05 21:07:48 +02005413
5414 if (left_level == 0)
5415 btrfs_item_key_to_cpu(left_path->nodes[left_level],
5416 &left_key, left_path->slots[left_level]);
5417 else
5418 btrfs_node_key_to_cpu(left_path->nodes[left_level],
5419 &left_key, left_path->slots[left_level]);
5420 if (right_level == 0)
5421 btrfs_item_key_to_cpu(right_path->nodes[right_level],
5422 &right_key, right_path->slots[right_level]);
5423 else
5424 btrfs_node_key_to_cpu(right_path->nodes[right_level],
5425 &right_key, right_path->slots[right_level]);
5426
5427 left_end_reached = right_end_reached = 0;
5428 advance_left = advance_right = 0;
5429
5430 while (1) {
Alexander Block70698302012-06-05 21:07:48 +02005431 if (advance_left && !left_end_reached) {
5432 ret = tree_advance(left_root, left_path, &left_level,
5433 left_root_level,
5434 advance_left != ADVANCE_ONLY_NEXT,
5435 &left_key);
5436 if (ret < 0)
5437 left_end_reached = ADVANCE;
5438 advance_left = 0;
5439 }
5440 if (advance_right && !right_end_reached) {
5441 ret = tree_advance(right_root, right_path, &right_level,
5442 right_root_level,
5443 advance_right != ADVANCE_ONLY_NEXT,
5444 &right_key);
5445 if (ret < 0)
5446 right_end_reached = ADVANCE;
5447 advance_right = 0;
5448 }
5449
5450 if (left_end_reached && right_end_reached) {
5451 ret = 0;
5452 goto out;
5453 } else if (left_end_reached) {
5454 if (right_level == 0) {
5455 ret = changed_cb(left_root, right_root,
5456 left_path, right_path,
5457 &right_key,
5458 BTRFS_COMPARE_TREE_DELETED,
5459 ctx);
5460 if (ret < 0)
5461 goto out;
5462 }
5463 advance_right = ADVANCE;
5464 continue;
5465 } else if (right_end_reached) {
5466 if (left_level == 0) {
5467 ret = changed_cb(left_root, right_root,
5468 left_path, right_path,
5469 &left_key,
5470 BTRFS_COMPARE_TREE_NEW,
5471 ctx);
5472 if (ret < 0)
5473 goto out;
5474 }
5475 advance_left = ADVANCE;
5476 continue;
5477 }
5478
5479 if (left_level == 0 && right_level == 0) {
5480 cmp = btrfs_comp_cpu_keys(&left_key, &right_key);
5481 if (cmp < 0) {
5482 ret = changed_cb(left_root, right_root,
5483 left_path, right_path,
5484 &left_key,
5485 BTRFS_COMPARE_TREE_NEW,
5486 ctx);
5487 if (ret < 0)
5488 goto out;
5489 advance_left = ADVANCE;
5490 } else if (cmp > 0) {
5491 ret = changed_cb(left_root, right_root,
5492 left_path, right_path,
5493 &right_key,
5494 BTRFS_COMPARE_TREE_DELETED,
5495 ctx);
5496 if (ret < 0)
5497 goto out;
5498 advance_right = ADVANCE;
5499 } else {
Fabian Frederickb99d9a62014-09-25 19:35:02 +02005500 enum btrfs_compare_tree_result result;
Josef Bacikba5e8f22013-08-16 16:52:55 -04005501
Chris Mason74dd17f2012-08-07 16:25:13 -04005502 WARN_ON(!extent_buffer_uptodate(left_path->nodes[0]));
Alexander Block70698302012-06-05 21:07:48 +02005503 ret = tree_compare_item(left_root, left_path,
5504 right_path, tmp_buf);
Josef Bacikba5e8f22013-08-16 16:52:55 -04005505 if (ret)
Fabian Frederickb99d9a62014-09-25 19:35:02 +02005506 result = BTRFS_COMPARE_TREE_CHANGED;
Josef Bacikba5e8f22013-08-16 16:52:55 -04005507 else
Fabian Frederickb99d9a62014-09-25 19:35:02 +02005508 result = BTRFS_COMPARE_TREE_SAME;
Josef Bacikba5e8f22013-08-16 16:52:55 -04005509 ret = changed_cb(left_root, right_root,
5510 left_path, right_path,
Fabian Frederickb99d9a62014-09-25 19:35:02 +02005511 &left_key, result, ctx);
Josef Bacikba5e8f22013-08-16 16:52:55 -04005512 if (ret < 0)
5513 goto out;
Alexander Block70698302012-06-05 21:07:48 +02005514 advance_left = ADVANCE;
5515 advance_right = ADVANCE;
5516 }
5517 } else if (left_level == right_level) {
5518 cmp = btrfs_comp_cpu_keys(&left_key, &right_key);
5519 if (cmp < 0) {
5520 advance_left = ADVANCE;
5521 } else if (cmp > 0) {
5522 advance_right = ADVANCE;
5523 } else {
5524 left_blockptr = btrfs_node_blockptr(
5525 left_path->nodes[left_level],
5526 left_path->slots[left_level]);
5527 right_blockptr = btrfs_node_blockptr(
5528 right_path->nodes[right_level],
5529 right_path->slots[right_level]);
Filipe Manana6baa4292014-02-20 21:15:25 +00005530 left_gen = btrfs_node_ptr_generation(
5531 left_path->nodes[left_level],
5532 left_path->slots[left_level]);
5533 right_gen = btrfs_node_ptr_generation(
5534 right_path->nodes[right_level],
5535 right_path->slots[right_level]);
5536 if (left_blockptr == right_blockptr &&
5537 left_gen == right_gen) {
Alexander Block70698302012-06-05 21:07:48 +02005538 /*
5539 * As we're on a shared block, don't
5540 * allow to go deeper.
5541 */
5542 advance_left = ADVANCE_ONLY_NEXT;
5543 advance_right = ADVANCE_ONLY_NEXT;
5544 } else {
5545 advance_left = ADVANCE;
5546 advance_right = ADVANCE;
5547 }
5548 }
5549 } else if (left_level < right_level) {
5550 advance_right = ADVANCE;
5551 } else {
5552 advance_left = ADVANCE;
5553 }
5554 }
5555
5556out:
5557 btrfs_free_path(left_path);
5558 btrfs_free_path(right_path);
5559 kfree(tmp_buf);
Alexander Block70698302012-06-05 21:07:48 +02005560 return ret;
5561}
5562
Chris Mason3f157a22008-06-25 16:01:31 -04005563/*
5564 * this is similar to btrfs_next_leaf, but does not try to preserve
5565 * and fixup the path. It looks for and returns the next key in the
Eric Sandeende78b512013-01-31 18:21:12 +00005566 * tree based on the current path and the min_trans parameters.
Chris Mason3f157a22008-06-25 16:01:31 -04005567 *
5568 * 0 is returned if another key is found, < 0 if there are any errors
5569 * and 1 is returned if there are no higher keys in the tree
5570 *
5571 * path->keep_locks should be set to 1 on the search made before
5572 * calling this function.
5573 */
Chris Masone7a84562008-06-25 16:01:31 -04005574int btrfs_find_next_key(struct btrfs_root *root, struct btrfs_path *path,
Eric Sandeende78b512013-01-31 18:21:12 +00005575 struct btrfs_key *key, int level, u64 min_trans)
Chris Masone7a84562008-06-25 16:01:31 -04005576{
Chris Masone7a84562008-06-25 16:01:31 -04005577 int slot;
5578 struct extent_buffer *c;
5579
Chris Mason934d3752008-12-08 16:43:10 -05005580 WARN_ON(!path->keep_locks);
Chris Masond3977122009-01-05 21:25:51 -05005581 while (level < BTRFS_MAX_LEVEL) {
Chris Masone7a84562008-06-25 16:01:31 -04005582 if (!path->nodes[level])
5583 return 1;
5584
5585 slot = path->slots[level] + 1;
5586 c = path->nodes[level];
Chris Mason3f157a22008-06-25 16:01:31 -04005587next:
Chris Masone7a84562008-06-25 16:01:31 -04005588 if (slot >= btrfs_header_nritems(c)) {
Yan Zheng33c66f42009-07-22 09:59:00 -04005589 int ret;
5590 int orig_lowest;
5591 struct btrfs_key cur_key;
5592 if (level + 1 >= BTRFS_MAX_LEVEL ||
5593 !path->nodes[level + 1])
Chris Masone7a84562008-06-25 16:01:31 -04005594 return 1;
Yan Zheng33c66f42009-07-22 09:59:00 -04005595
5596 if (path->locks[level + 1]) {
5597 level++;
5598 continue;
5599 }
5600
5601 slot = btrfs_header_nritems(c) - 1;
5602 if (level == 0)
5603 btrfs_item_key_to_cpu(c, &cur_key, slot);
5604 else
5605 btrfs_node_key_to_cpu(c, &cur_key, slot);
5606
5607 orig_lowest = path->lowest_level;
David Sterbab3b4aa72011-04-21 01:20:15 +02005608 btrfs_release_path(path);
Yan Zheng33c66f42009-07-22 09:59:00 -04005609 path->lowest_level = level;
5610 ret = btrfs_search_slot(NULL, root, &cur_key, path,
5611 0, 0);
5612 path->lowest_level = orig_lowest;
5613 if (ret < 0)
5614 return ret;
5615
5616 c = path->nodes[level];
5617 slot = path->slots[level];
5618 if (ret == 0)
5619 slot++;
5620 goto next;
Chris Masone7a84562008-06-25 16:01:31 -04005621 }
Yan Zheng33c66f42009-07-22 09:59:00 -04005622
Chris Masone7a84562008-06-25 16:01:31 -04005623 if (level == 0)
5624 btrfs_item_key_to_cpu(c, key, slot);
Chris Mason3f157a22008-06-25 16:01:31 -04005625 else {
Chris Mason3f157a22008-06-25 16:01:31 -04005626 u64 gen = btrfs_node_ptr_generation(c, slot);
5627
Chris Mason3f157a22008-06-25 16:01:31 -04005628 if (gen < min_trans) {
5629 slot++;
5630 goto next;
5631 }
Chris Masone7a84562008-06-25 16:01:31 -04005632 btrfs_node_key_to_cpu(c, key, slot);
Chris Mason3f157a22008-06-25 16:01:31 -04005633 }
Chris Masone7a84562008-06-25 16:01:31 -04005634 return 0;
5635 }
5636 return 1;
5637}
5638
Chris Mason7bb86312007-12-11 09:25:06 -05005639/*
Chris Mason925baed2008-06-25 16:01:30 -04005640 * search the tree again to find a leaf with greater keys
Chris Mason0f70abe2007-02-28 16:46:22 -05005641 * returns 0 if it found something or 1 if there are no greater leaves.
5642 * returns < 0 on io errors.
Chris Mason97571fd2007-02-24 13:39:08 -05005643 */
Chris Mason234b63a2007-03-13 10:46:10 -04005644int btrfs_next_leaf(struct btrfs_root *root, struct btrfs_path *path)
Chris Masond97e63b2007-02-20 16:40:44 -05005645{
Jan Schmidt3d7806e2012-06-11 08:29:29 +02005646 return btrfs_next_old_leaf(root, path, 0);
5647}
5648
5649int btrfs_next_old_leaf(struct btrfs_root *root, struct btrfs_path *path,
5650 u64 time_seq)
5651{
Chris Masond97e63b2007-02-20 16:40:44 -05005652 int slot;
Chris Mason8e73f272009-04-03 10:14:18 -04005653 int level;
Chris Mason5f39d392007-10-15 16:14:19 -04005654 struct extent_buffer *c;
Chris Mason8e73f272009-04-03 10:14:18 -04005655 struct extent_buffer *next;
Chris Mason925baed2008-06-25 16:01:30 -04005656 struct btrfs_key key;
5657 u32 nritems;
5658 int ret;
Chris Mason8e73f272009-04-03 10:14:18 -04005659 int old_spinning = path->leave_spinning;
Chris Masonbd681512011-07-16 15:23:14 -04005660 int next_rw_lock = 0;
Chris Mason925baed2008-06-25 16:01:30 -04005661
5662 nritems = btrfs_header_nritems(path->nodes[0]);
Chris Masond3977122009-01-05 21:25:51 -05005663 if (nritems == 0)
Chris Mason925baed2008-06-25 16:01:30 -04005664 return 1;
Chris Mason925baed2008-06-25 16:01:30 -04005665
Chris Mason8e73f272009-04-03 10:14:18 -04005666 btrfs_item_key_to_cpu(path->nodes[0], &key, nritems - 1);
5667again:
5668 level = 1;
5669 next = NULL;
Chris Masonbd681512011-07-16 15:23:14 -04005670 next_rw_lock = 0;
David Sterbab3b4aa72011-04-21 01:20:15 +02005671 btrfs_release_path(path);
Chris Mason8e73f272009-04-03 10:14:18 -04005672
Chris Masona2135012008-06-25 16:01:30 -04005673 path->keep_locks = 1;
Chris Mason31533fb2011-07-26 16:01:59 -04005674 path->leave_spinning = 1;
Chris Mason8e73f272009-04-03 10:14:18 -04005675
Jan Schmidt3d7806e2012-06-11 08:29:29 +02005676 if (time_seq)
5677 ret = btrfs_search_old_slot(root, &key, path, time_seq);
5678 else
5679 ret = btrfs_search_slot(NULL, root, &key, path, 0, 0);
Chris Mason925baed2008-06-25 16:01:30 -04005680 path->keep_locks = 0;
5681
5682 if (ret < 0)
5683 return ret;
5684
Chris Masona2135012008-06-25 16:01:30 -04005685 nritems = btrfs_header_nritems(path->nodes[0]);
Chris Mason168fd7d2008-06-25 16:01:30 -04005686 /*
5687 * by releasing the path above we dropped all our locks. A balance
5688 * could have added more items next to the key that used to be
5689 * at the very end of the block. So, check again here and
5690 * advance the path if there are now more items available.
5691 */
Chris Masona2135012008-06-25 16:01:30 -04005692 if (nritems > 0 && path->slots[0] < nritems - 1) {
Yan Zhenge457afe2009-07-22 09:59:00 -04005693 if (ret == 0)
5694 path->slots[0]++;
Chris Mason8e73f272009-04-03 10:14:18 -04005695 ret = 0;
Chris Mason925baed2008-06-25 16:01:30 -04005696 goto done;
5697 }
Liu Bo0b43e042014-06-09 11:04:49 +08005698 /*
5699 * So the above check misses one case:
5700 * - after releasing the path above, someone has removed the item that
5701 * used to be at the very end of the block, and balance between leafs
5702 * gets another one with bigger key.offset to replace it.
5703 *
5704 * This one should be returned as well, or we can get leaf corruption
5705 * later(esp. in __btrfs_drop_extents()).
5706 *
5707 * And a bit more explanation about this check,
5708 * with ret > 0, the key isn't found, the path points to the slot
5709 * where it should be inserted, so the path->slots[0] item must be the
5710 * bigger one.
5711 */
5712 if (nritems > 0 && ret > 0 && path->slots[0] == nritems - 1) {
5713 ret = 0;
5714 goto done;
5715 }
Chris Masond97e63b2007-02-20 16:40:44 -05005716
Chris Masond3977122009-01-05 21:25:51 -05005717 while (level < BTRFS_MAX_LEVEL) {
Chris Mason8e73f272009-04-03 10:14:18 -04005718 if (!path->nodes[level]) {
5719 ret = 1;
5720 goto done;
5721 }
Chris Mason5f39d392007-10-15 16:14:19 -04005722
Chris Masond97e63b2007-02-20 16:40:44 -05005723 slot = path->slots[level] + 1;
5724 c = path->nodes[level];
Chris Mason5f39d392007-10-15 16:14:19 -04005725 if (slot >= btrfs_header_nritems(c)) {
Chris Masond97e63b2007-02-20 16:40:44 -05005726 level++;
Chris Mason8e73f272009-04-03 10:14:18 -04005727 if (level == BTRFS_MAX_LEVEL) {
5728 ret = 1;
5729 goto done;
5730 }
Chris Masond97e63b2007-02-20 16:40:44 -05005731 continue;
5732 }
Chris Mason5f39d392007-10-15 16:14:19 -04005733
Chris Mason925baed2008-06-25 16:01:30 -04005734 if (next) {
Chris Masonbd681512011-07-16 15:23:14 -04005735 btrfs_tree_unlock_rw(next, next_rw_lock);
Chris Mason5f39d392007-10-15 16:14:19 -04005736 free_extent_buffer(next);
Chris Mason925baed2008-06-25 16:01:30 -04005737 }
Chris Mason5f39d392007-10-15 16:14:19 -04005738
Chris Mason8e73f272009-04-03 10:14:18 -04005739 next = c;
Chris Masonbd681512011-07-16 15:23:14 -04005740 next_rw_lock = path->locks[level];
Chris Mason8e73f272009-04-03 10:14:18 -04005741 ret = read_block_for_search(NULL, root, path, &next, level,
Jan Schmidt5d9e75c42012-05-16 18:25:47 +02005742 slot, &key, 0);
Chris Mason8e73f272009-04-03 10:14:18 -04005743 if (ret == -EAGAIN)
5744 goto again;
Chris Mason5f39d392007-10-15 16:14:19 -04005745
Chris Mason76a05b32009-05-14 13:24:30 -04005746 if (ret < 0) {
David Sterbab3b4aa72011-04-21 01:20:15 +02005747 btrfs_release_path(path);
Chris Mason76a05b32009-05-14 13:24:30 -04005748 goto done;
5749 }
5750
Chris Mason5cd57b22008-06-25 16:01:30 -04005751 if (!path->skip_locking) {
Chris Masonbd681512011-07-16 15:23:14 -04005752 ret = btrfs_try_tree_read_lock(next);
Jan Schmidtd42244a2012-06-22 14:51:15 +02005753 if (!ret && time_seq) {
5754 /*
5755 * If we don't get the lock, we may be racing
5756 * with push_leaf_left, holding that lock while
5757 * itself waiting for the leaf we've currently
5758 * locked. To solve this situation, we give up
5759 * on our lock and cycle.
5760 */
Jan Schmidtcf538832012-07-04 15:42:48 +02005761 free_extent_buffer(next);
Jan Schmidtd42244a2012-06-22 14:51:15 +02005762 btrfs_release_path(path);
5763 cond_resched();
5764 goto again;
5765 }
Chris Mason8e73f272009-04-03 10:14:18 -04005766 if (!ret) {
5767 btrfs_set_path_blocking(path);
Chris Masonbd681512011-07-16 15:23:14 -04005768 btrfs_tree_read_lock(next);
Chris Mason31533fb2011-07-26 16:01:59 -04005769 btrfs_clear_path_blocking(path, next,
Chris Masonbd681512011-07-16 15:23:14 -04005770 BTRFS_READ_LOCK);
Chris Mason8e73f272009-04-03 10:14:18 -04005771 }
Chris Mason31533fb2011-07-26 16:01:59 -04005772 next_rw_lock = BTRFS_READ_LOCK;
Chris Mason5cd57b22008-06-25 16:01:30 -04005773 }
Chris Masond97e63b2007-02-20 16:40:44 -05005774 break;
5775 }
5776 path->slots[level] = slot;
Chris Masond3977122009-01-05 21:25:51 -05005777 while (1) {
Chris Masond97e63b2007-02-20 16:40:44 -05005778 level--;
5779 c = path->nodes[level];
Chris Mason925baed2008-06-25 16:01:30 -04005780 if (path->locks[level])
Chris Masonbd681512011-07-16 15:23:14 -04005781 btrfs_tree_unlock_rw(c, path->locks[level]);
Chris Mason8e73f272009-04-03 10:14:18 -04005782
Chris Mason5f39d392007-10-15 16:14:19 -04005783 free_extent_buffer(c);
Chris Masond97e63b2007-02-20 16:40:44 -05005784 path->nodes[level] = next;
5785 path->slots[level] = 0;
Chris Masona74a4b92008-06-25 16:01:31 -04005786 if (!path->skip_locking)
Chris Masonbd681512011-07-16 15:23:14 -04005787 path->locks[level] = next_rw_lock;
Chris Masond97e63b2007-02-20 16:40:44 -05005788 if (!level)
5789 break;
Chris Masonb4ce94d2009-02-04 09:25:08 -05005790
Chris Mason8e73f272009-04-03 10:14:18 -04005791 ret = read_block_for_search(NULL, root, path, &next, level,
Jan Schmidt5d9e75c42012-05-16 18:25:47 +02005792 0, &key, 0);
Chris Mason8e73f272009-04-03 10:14:18 -04005793 if (ret == -EAGAIN)
5794 goto again;
5795
Chris Mason76a05b32009-05-14 13:24:30 -04005796 if (ret < 0) {
David Sterbab3b4aa72011-04-21 01:20:15 +02005797 btrfs_release_path(path);
Chris Mason76a05b32009-05-14 13:24:30 -04005798 goto done;
5799 }
5800
Chris Mason5cd57b22008-06-25 16:01:30 -04005801 if (!path->skip_locking) {
Chris Masonbd681512011-07-16 15:23:14 -04005802 ret = btrfs_try_tree_read_lock(next);
Chris Mason8e73f272009-04-03 10:14:18 -04005803 if (!ret) {
5804 btrfs_set_path_blocking(path);
Chris Masonbd681512011-07-16 15:23:14 -04005805 btrfs_tree_read_lock(next);
Chris Mason31533fb2011-07-26 16:01:59 -04005806 btrfs_clear_path_blocking(path, next,
Chris Masonbd681512011-07-16 15:23:14 -04005807 BTRFS_READ_LOCK);
Chris Mason8e73f272009-04-03 10:14:18 -04005808 }
Chris Mason31533fb2011-07-26 16:01:59 -04005809 next_rw_lock = BTRFS_READ_LOCK;
Chris Mason5cd57b22008-06-25 16:01:30 -04005810 }
Chris Masond97e63b2007-02-20 16:40:44 -05005811 }
Chris Mason8e73f272009-04-03 10:14:18 -04005812 ret = 0;
Chris Mason925baed2008-06-25 16:01:30 -04005813done:
Chris Masonf7c79f32012-03-19 15:54:38 -04005814 unlock_up(path, 0, 1, 0, NULL);
Chris Mason8e73f272009-04-03 10:14:18 -04005815 path->leave_spinning = old_spinning;
5816 if (!old_spinning)
5817 btrfs_set_path_blocking(path);
5818
5819 return ret;
Chris Masond97e63b2007-02-20 16:40:44 -05005820}
Chris Mason0b86a832008-03-24 15:01:56 -04005821
Chris Mason3f157a22008-06-25 16:01:31 -04005822/*
5823 * this uses btrfs_prev_leaf to walk backwards in the tree, and keeps
5824 * searching until it gets past min_objectid or finds an item of 'type'
5825 *
5826 * returns 0 if something is found, 1 if nothing was found and < 0 on error
5827 */
Chris Mason0b86a832008-03-24 15:01:56 -04005828int btrfs_previous_item(struct btrfs_root *root,
5829 struct btrfs_path *path, u64 min_objectid,
5830 int type)
5831{
5832 struct btrfs_key found_key;
5833 struct extent_buffer *leaf;
Chris Masone02119d2008-09-05 16:13:11 -04005834 u32 nritems;
Chris Mason0b86a832008-03-24 15:01:56 -04005835 int ret;
5836
Chris Masond3977122009-01-05 21:25:51 -05005837 while (1) {
Chris Mason0b86a832008-03-24 15:01:56 -04005838 if (path->slots[0] == 0) {
Chris Masonb4ce94d2009-02-04 09:25:08 -05005839 btrfs_set_path_blocking(path);
Chris Mason0b86a832008-03-24 15:01:56 -04005840 ret = btrfs_prev_leaf(root, path);
5841 if (ret != 0)
5842 return ret;
5843 } else {
5844 path->slots[0]--;
5845 }
5846 leaf = path->nodes[0];
Chris Masone02119d2008-09-05 16:13:11 -04005847 nritems = btrfs_header_nritems(leaf);
5848 if (nritems == 0)
5849 return 1;
5850 if (path->slots[0] == nritems)
5851 path->slots[0]--;
5852
Chris Mason0b86a832008-03-24 15:01:56 -04005853 btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]);
Chris Masone02119d2008-09-05 16:13:11 -04005854 if (found_key.objectid < min_objectid)
5855 break;
Yan Zheng0a4eefb2009-07-24 11:06:53 -04005856 if (found_key.type == type)
5857 return 0;
Chris Masone02119d2008-09-05 16:13:11 -04005858 if (found_key.objectid == min_objectid &&
5859 found_key.type < type)
5860 break;
Chris Mason0b86a832008-03-24 15:01:56 -04005861 }
5862 return 1;
5863}
Wang Shilongade2e0b2014-01-12 21:38:33 +08005864
5865/*
5866 * search in extent tree to find a previous Metadata/Data extent item with
5867 * min objecitd.
5868 *
5869 * returns 0 if something is found, 1 if nothing was found and < 0 on error
5870 */
5871int btrfs_previous_extent_item(struct btrfs_root *root,
5872 struct btrfs_path *path, u64 min_objectid)
5873{
5874 struct btrfs_key found_key;
5875 struct extent_buffer *leaf;
5876 u32 nritems;
5877 int ret;
5878
5879 while (1) {
5880 if (path->slots[0] == 0) {
5881 btrfs_set_path_blocking(path);
5882 ret = btrfs_prev_leaf(root, path);
5883 if (ret != 0)
5884 return ret;
5885 } else {
5886 path->slots[0]--;
5887 }
5888 leaf = path->nodes[0];
5889 nritems = btrfs_header_nritems(leaf);
5890 if (nritems == 0)
5891 return 1;
5892 if (path->slots[0] == nritems)
5893 path->slots[0]--;
5894
5895 btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]);
5896 if (found_key.objectid < min_objectid)
5897 break;
5898 if (found_key.type == BTRFS_EXTENT_ITEM_KEY ||
5899 found_key.type == BTRFS_METADATA_ITEM_KEY)
5900 return 0;
5901 if (found_key.objectid == min_objectid &&
5902 found_key.type < BTRFS_EXTENT_ITEM_KEY)
5903 break;
5904 }
5905 return 1;
5906}