blob: 5b8e235c4b6d6299183ebba91976f0d53612c16c [file] [log] [blame]
Chris Mason6cbd5572007-06-12 09:07:21 -04001/*
Chris Masond352ac62008-09-29 15:18:18 -04002 * Copyright (C) 2007,2008 Oracle. All rights reserved.
Chris Mason6cbd5572007-06-12 09:07:21 -04003 *
4 * This program is free software; you can redistribute it and/or
5 * modify it under the terms of the GNU General Public
6 * License v2 as published by the Free Software Foundation.
7 *
8 * This program is distributed in the hope that it will be useful,
9 * but WITHOUT ANY WARRANTY; without even the implied warranty of
10 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
11 * General Public License for more details.
12 *
13 * You should have received a copy of the GNU General Public
14 * License along with this program; if not, write to the
15 * Free Software Foundation, Inc., 59 Temple Place - Suite 330,
16 * Boston, MA 021110-1307, USA.
17 */
18
Chris Masona6b6e752007-10-15 16:22:39 -040019#include <linux/sched.h>
Tejun Heo5a0e3ad2010-03-24 17:04:11 +090020#include <linux/slab.h>
Jan Schmidtbd989ba2012-05-16 17:18:50 +020021#include <linux/rbtree.h>
Chris Masoneb60cea2007-02-02 09:18:22 -050022#include "ctree.h"
23#include "disk-io.h"
Chris Mason7f5c1512007-03-23 15:56:19 -040024#include "transaction.h"
Chris Mason5f39d392007-10-15 16:14:19 -040025#include "print-tree.h"
Chris Mason925baed2008-06-25 16:01:30 -040026#include "locking.h"
Chris Mason9a8dd152007-02-23 08:38:36 -050027
Chris Masone089f052007-03-16 16:20:31 -040028static int split_node(struct btrfs_trans_handle *trans, struct btrfs_root
29 *root, struct btrfs_path *path, int level);
30static int split_leaf(struct btrfs_trans_handle *trans, struct btrfs_root
Chris Masond4dbff92007-04-04 14:08:15 -040031 *root, struct btrfs_key *ins_key,
Chris Masoncc0c5532007-10-25 15:42:57 -040032 struct btrfs_path *path, int data_size, int extend);
Chris Mason5f39d392007-10-15 16:14:19 -040033static int push_node_left(struct btrfs_trans_handle *trans,
34 struct btrfs_root *root, struct extent_buffer *dst,
Chris Mason971a1f62008-04-24 10:54:32 -040035 struct extent_buffer *src, int empty);
Chris Mason5f39d392007-10-15 16:14:19 -040036static int balance_node_right(struct btrfs_trans_handle *trans,
37 struct btrfs_root *root,
38 struct extent_buffer *dst_buf,
39 struct extent_buffer *src_buf);
Tsutomu Itohafe5fea2013-04-16 05:18:22 +000040static void del_ptr(struct btrfs_root *root, struct btrfs_path *path,
41 int level, int slot);
Filipe David Borba Manana5de865e2013-12-20 15:17:46 +000042static int tree_mod_log_free_eb(struct btrfs_fs_info *fs_info,
Jan Schmidtf2304752012-05-26 11:43:17 +020043 struct extent_buffer *eb);
Chris Masond97e63b2007-02-20 16:40:44 -050044
Chris Mason2c90e5d2007-04-02 10:50:19 -040045struct btrfs_path *btrfs_alloc_path(void)
46{
Chris Masondf24a2b2007-04-04 09:36:31 -040047 struct btrfs_path *path;
Jeff Mahoneye00f7302009-02-12 14:11:25 -050048 path = kmem_cache_zalloc(btrfs_path_cachep, GFP_NOFS);
Chris Masondf24a2b2007-04-04 09:36:31 -040049 return path;
Chris Mason2c90e5d2007-04-02 10:50:19 -040050}
51
Chris Masonb4ce94d2009-02-04 09:25:08 -050052/*
53 * set all locked nodes in the path to blocking locks. This should
54 * be done before scheduling
55 */
56noinline void btrfs_set_path_blocking(struct btrfs_path *p)
57{
58 int i;
59 for (i = 0; i < BTRFS_MAX_LEVEL; i++) {
Chris Masonbd681512011-07-16 15:23:14 -040060 if (!p->nodes[i] || !p->locks[i])
61 continue;
62 btrfs_set_lock_blocking_rw(p->nodes[i], p->locks[i]);
63 if (p->locks[i] == BTRFS_READ_LOCK)
64 p->locks[i] = BTRFS_READ_LOCK_BLOCKING;
65 else if (p->locks[i] == BTRFS_WRITE_LOCK)
66 p->locks[i] = BTRFS_WRITE_LOCK_BLOCKING;
Chris Masonb4ce94d2009-02-04 09:25:08 -050067 }
68}
69
70/*
71 * reset all the locked nodes in the patch to spinning locks.
Chris Mason4008c042009-02-12 14:09:45 -050072 *
73 * held is used to keep lockdep happy, when lockdep is enabled
74 * we set held to a blocking lock before we go around and
75 * retake all the spinlocks in the path. You can safely use NULL
76 * for held
Chris Masonb4ce94d2009-02-04 09:25:08 -050077 */
Chris Mason4008c042009-02-12 14:09:45 -050078noinline void btrfs_clear_path_blocking(struct btrfs_path *p,
Chris Masonbd681512011-07-16 15:23:14 -040079 struct extent_buffer *held, int held_rw)
Chris Masonb4ce94d2009-02-04 09:25:08 -050080{
81 int i;
Chris Mason4008c042009-02-12 14:09:45 -050082
Chris Masonbd681512011-07-16 15:23:14 -040083 if (held) {
84 btrfs_set_lock_blocking_rw(held, held_rw);
85 if (held_rw == BTRFS_WRITE_LOCK)
86 held_rw = BTRFS_WRITE_LOCK_BLOCKING;
87 else if (held_rw == BTRFS_READ_LOCK)
88 held_rw = BTRFS_READ_LOCK_BLOCKING;
89 }
Chris Mason4008c042009-02-12 14:09:45 -050090 btrfs_set_path_blocking(p);
Chris Mason4008c042009-02-12 14:09:45 -050091
92 for (i = BTRFS_MAX_LEVEL - 1; i >= 0; i--) {
Chris Masonbd681512011-07-16 15:23:14 -040093 if (p->nodes[i] && p->locks[i]) {
94 btrfs_clear_lock_blocking_rw(p->nodes[i], p->locks[i]);
95 if (p->locks[i] == BTRFS_WRITE_LOCK_BLOCKING)
96 p->locks[i] = BTRFS_WRITE_LOCK;
97 else if (p->locks[i] == BTRFS_READ_LOCK_BLOCKING)
98 p->locks[i] = BTRFS_READ_LOCK;
99 }
Chris Masonb4ce94d2009-02-04 09:25:08 -0500100 }
Chris Mason4008c042009-02-12 14:09:45 -0500101
Chris Mason4008c042009-02-12 14:09:45 -0500102 if (held)
Chris Masonbd681512011-07-16 15:23:14 -0400103 btrfs_clear_lock_blocking_rw(held, held_rw);
Chris Masonb4ce94d2009-02-04 09:25:08 -0500104}
105
Chris Masond352ac62008-09-29 15:18:18 -0400106/* this also releases the path */
Chris Mason2c90e5d2007-04-02 10:50:19 -0400107void btrfs_free_path(struct btrfs_path *p)
108{
Jesper Juhlff175d52010-12-25 21:22:30 +0000109 if (!p)
110 return;
David Sterbab3b4aa72011-04-21 01:20:15 +0200111 btrfs_release_path(p);
Chris Mason2c90e5d2007-04-02 10:50:19 -0400112 kmem_cache_free(btrfs_path_cachep, p);
113}
114
Chris Masond352ac62008-09-29 15:18:18 -0400115/*
116 * path release drops references on the extent buffers in the path
117 * and it drops any locks held by this path
118 *
119 * It is safe to call this on paths that no locks or extent buffers held.
120 */
David Sterbab3b4aa72011-04-21 01:20:15 +0200121noinline void btrfs_release_path(struct btrfs_path *p)
Chris Masoneb60cea2007-02-02 09:18:22 -0500122{
123 int i;
Chris Masona2135012008-06-25 16:01:30 -0400124
Chris Mason234b63a2007-03-13 10:46:10 -0400125 for (i = 0; i < BTRFS_MAX_LEVEL; i++) {
Chris Mason3f157a22008-06-25 16:01:31 -0400126 p->slots[i] = 0;
Chris Masoneb60cea2007-02-02 09:18:22 -0500127 if (!p->nodes[i])
Chris Mason925baed2008-06-25 16:01:30 -0400128 continue;
129 if (p->locks[i]) {
Chris Masonbd681512011-07-16 15:23:14 -0400130 btrfs_tree_unlock_rw(p->nodes[i], p->locks[i]);
Chris Mason925baed2008-06-25 16:01:30 -0400131 p->locks[i] = 0;
132 }
Chris Mason5f39d392007-10-15 16:14:19 -0400133 free_extent_buffer(p->nodes[i]);
Chris Mason3f157a22008-06-25 16:01:31 -0400134 p->nodes[i] = NULL;
Chris Masoneb60cea2007-02-02 09:18:22 -0500135 }
136}
137
Chris Masond352ac62008-09-29 15:18:18 -0400138/*
139 * safely gets a reference on the root node of a tree. A lock
140 * is not taken, so a concurrent writer may put a different node
141 * at the root of the tree. See btrfs_lock_root_node for the
142 * looping required.
143 *
144 * The extent buffer returned by this has a reference taken, so
145 * it won't disappear. It may stop being the root of the tree
146 * at any time because there are no locks held.
147 */
Chris Mason925baed2008-06-25 16:01:30 -0400148struct extent_buffer *btrfs_root_node(struct btrfs_root *root)
149{
150 struct extent_buffer *eb;
Chris Mason240f62c2011-03-23 14:54:42 -0400151
Josef Bacik3083ee22012-03-09 16:01:49 -0500152 while (1) {
153 rcu_read_lock();
154 eb = rcu_dereference(root->node);
155
156 /*
157 * RCU really hurts here, we could free up the root node because
158 * it was cow'ed but we may not get the new root node yet so do
159 * the inc_not_zero dance and if it doesn't work then
160 * synchronize_rcu and try again.
161 */
162 if (atomic_inc_not_zero(&eb->refs)) {
163 rcu_read_unlock();
164 break;
165 }
166 rcu_read_unlock();
167 synchronize_rcu();
168 }
Chris Mason925baed2008-06-25 16:01:30 -0400169 return eb;
170}
171
Chris Masond352ac62008-09-29 15:18:18 -0400172/* loop around taking references on and locking the root node of the
173 * tree until you end up with a lock on the root. A locked buffer
174 * is returned, with a reference held.
175 */
Chris Mason925baed2008-06-25 16:01:30 -0400176struct extent_buffer *btrfs_lock_root_node(struct btrfs_root *root)
177{
178 struct extent_buffer *eb;
179
Chris Masond3977122009-01-05 21:25:51 -0500180 while (1) {
Chris Mason925baed2008-06-25 16:01:30 -0400181 eb = btrfs_root_node(root);
182 btrfs_tree_lock(eb);
Chris Mason240f62c2011-03-23 14:54:42 -0400183 if (eb == root->node)
Chris Mason925baed2008-06-25 16:01:30 -0400184 break;
Chris Mason925baed2008-06-25 16:01:30 -0400185 btrfs_tree_unlock(eb);
186 free_extent_buffer(eb);
187 }
188 return eb;
189}
190
Chris Masonbd681512011-07-16 15:23:14 -0400191/* loop around taking references on and locking the root node of the
192 * tree until you end up with a lock on the root. A locked buffer
193 * is returned, with a reference held.
194 */
Eric Sandeen48a3b632013-04-25 20:41:01 +0000195static struct extent_buffer *btrfs_read_lock_root_node(struct btrfs_root *root)
Chris Masonbd681512011-07-16 15:23:14 -0400196{
197 struct extent_buffer *eb;
198
199 while (1) {
200 eb = btrfs_root_node(root);
201 btrfs_tree_read_lock(eb);
202 if (eb == root->node)
203 break;
204 btrfs_tree_read_unlock(eb);
205 free_extent_buffer(eb);
206 }
207 return eb;
208}
209
Chris Masond352ac62008-09-29 15:18:18 -0400210/* cowonly root (everything not a reference counted cow subvolume), just get
211 * put onto a simple dirty list. transaction.c walks this to make sure they
212 * get properly updated on disk.
213 */
Chris Mason0b86a832008-03-24 15:01:56 -0400214static void add_root_to_dirty_list(struct btrfs_root *root)
215{
Josef Bacike7070be2014-12-16 08:54:43 -0800216 if (test_bit(BTRFS_ROOT_DIRTY, &root->state) ||
217 !test_bit(BTRFS_ROOT_TRACK_DIRTY, &root->state))
218 return;
219
Chris Masone5846fc2012-05-03 12:08:48 -0400220 spin_lock(&root->fs_info->trans_lock);
Josef Bacike7070be2014-12-16 08:54:43 -0800221 if (!test_and_set_bit(BTRFS_ROOT_DIRTY, &root->state)) {
222 /* Want the extent tree to be the last on the list */
223 if (root->objectid == BTRFS_EXTENT_TREE_OBJECTID)
224 list_move_tail(&root->dirty_list,
225 &root->fs_info->dirty_cowonly_roots);
226 else
227 list_move(&root->dirty_list,
228 &root->fs_info->dirty_cowonly_roots);
Chris Mason0b86a832008-03-24 15:01:56 -0400229 }
Chris Masone5846fc2012-05-03 12:08:48 -0400230 spin_unlock(&root->fs_info->trans_lock);
Chris Mason0b86a832008-03-24 15:01:56 -0400231}
232
Chris Masond352ac62008-09-29 15:18:18 -0400233/*
234 * used by snapshot creation to make a copy of a root for a tree with
235 * a given objectid. The buffer with the new root node is returned in
236 * cow_ret, and this func returns zero on success or a negative error code.
237 */
Chris Masonbe20aa92007-12-17 20:14:01 -0500238int btrfs_copy_root(struct btrfs_trans_handle *trans,
239 struct btrfs_root *root,
240 struct extent_buffer *buf,
241 struct extent_buffer **cow_ret, u64 new_root_objectid)
242{
243 struct extent_buffer *cow;
Chris Masonbe20aa92007-12-17 20:14:01 -0500244 int ret = 0;
245 int level;
Yan Zheng5d4f98a2009-06-10 10:45:14 -0400246 struct btrfs_disk_key disk_key;
Chris Masonbe20aa92007-12-17 20:14:01 -0500247
Miao Xie27cdeb72014-04-02 19:51:05 +0800248 WARN_ON(test_bit(BTRFS_ROOT_REF_COWS, &root->state) &&
249 trans->transid != root->fs_info->running_transaction->transid);
250 WARN_ON(test_bit(BTRFS_ROOT_REF_COWS, &root->state) &&
251 trans->transid != root->last_trans);
Chris Masonbe20aa92007-12-17 20:14:01 -0500252
253 level = btrfs_header_level(buf);
Yan Zheng5d4f98a2009-06-10 10:45:14 -0400254 if (level == 0)
255 btrfs_item_key(buf, &disk_key, 0);
256 else
257 btrfs_node_key(buf, &disk_key, 0);
Zheng Yan31840ae2008-09-23 13:14:14 -0400258
David Sterba4d75f8a2014-06-15 01:54:12 +0200259 cow = btrfs_alloc_tree_block(trans, root, 0, new_root_objectid,
260 &disk_key, level, buf->start, 0);
Yan Zheng5d4f98a2009-06-10 10:45:14 -0400261 if (IS_ERR(cow))
Chris Masonbe20aa92007-12-17 20:14:01 -0500262 return PTR_ERR(cow);
263
264 copy_extent_buffer(cow, buf, 0, 0, cow->len);
265 btrfs_set_header_bytenr(cow, cow->start);
266 btrfs_set_header_generation(cow, trans->transid);
Yan Zheng5d4f98a2009-06-10 10:45:14 -0400267 btrfs_set_header_backref_rev(cow, BTRFS_MIXED_BACKREF_REV);
268 btrfs_clear_header_flag(cow, BTRFS_HEADER_FLAG_WRITTEN |
269 BTRFS_HEADER_FLAG_RELOC);
270 if (new_root_objectid == BTRFS_TREE_RELOC_OBJECTID)
271 btrfs_set_header_flag(cow, BTRFS_HEADER_FLAG_RELOC);
272 else
273 btrfs_set_header_owner(cow, new_root_objectid);
Chris Masonbe20aa92007-12-17 20:14:01 -0500274
Ross Kirk0a4e5582013-09-24 10:12:38 +0100275 write_extent_buffer(cow, root->fs_info->fsid, btrfs_header_fsid(),
Yan Zheng2b820322008-11-17 21:11:30 -0500276 BTRFS_FSID_SIZE);
277
Chris Masonbe20aa92007-12-17 20:14:01 -0500278 WARN_ON(btrfs_header_generation(buf) > trans->transid);
Yan Zheng5d4f98a2009-06-10 10:45:14 -0400279 if (new_root_objectid == BTRFS_TREE_RELOC_OBJECTID)
Josef Bacike339a6b2014-07-02 10:54:25 -0700280 ret = btrfs_inc_ref(trans, root, cow, 1);
Yan Zheng5d4f98a2009-06-10 10:45:14 -0400281 else
Josef Bacike339a6b2014-07-02 10:54:25 -0700282 ret = btrfs_inc_ref(trans, root, cow, 0);
Chris Mason4aec2b52007-12-18 16:25:45 -0500283
Chris Masonbe20aa92007-12-17 20:14:01 -0500284 if (ret)
285 return ret;
286
287 btrfs_mark_buffer_dirty(cow);
288 *cow_ret = cow;
289 return 0;
290}
291
Jan Schmidtbd989ba2012-05-16 17:18:50 +0200292enum mod_log_op {
293 MOD_LOG_KEY_REPLACE,
294 MOD_LOG_KEY_ADD,
295 MOD_LOG_KEY_REMOVE,
296 MOD_LOG_KEY_REMOVE_WHILE_FREEING,
297 MOD_LOG_KEY_REMOVE_WHILE_MOVING,
298 MOD_LOG_MOVE_KEYS,
299 MOD_LOG_ROOT_REPLACE,
300};
301
302struct tree_mod_move {
303 int dst_slot;
304 int nr_items;
305};
306
307struct tree_mod_root {
308 u64 logical;
309 u8 level;
310};
311
312struct tree_mod_elem {
313 struct rb_node node;
314 u64 index; /* shifted logical */
Jan Schmidt097b8a72012-06-21 11:08:04 +0200315 u64 seq;
Jan Schmidtbd989ba2012-05-16 17:18:50 +0200316 enum mod_log_op op;
317
318 /* this is used for MOD_LOG_KEY_* and MOD_LOG_MOVE_KEYS operations */
319 int slot;
320
321 /* this is used for MOD_LOG_KEY* and MOD_LOG_ROOT_REPLACE */
322 u64 generation;
323
324 /* those are used for op == MOD_LOG_KEY_{REPLACE,REMOVE} */
325 struct btrfs_disk_key key;
326 u64 blockptr;
327
328 /* this is used for op == MOD_LOG_MOVE_KEYS */
329 struct tree_mod_move move;
330
331 /* this is used for op == MOD_LOG_ROOT_REPLACE */
332 struct tree_mod_root old_root;
333};
334
Jan Schmidt097b8a72012-06-21 11:08:04 +0200335static inline void tree_mod_log_read_lock(struct btrfs_fs_info *fs_info)
Jan Schmidtbd989ba2012-05-16 17:18:50 +0200336{
Jan Schmidt097b8a72012-06-21 11:08:04 +0200337 read_lock(&fs_info->tree_mod_log_lock);
Jan Schmidtbd989ba2012-05-16 17:18:50 +0200338}
339
Jan Schmidt097b8a72012-06-21 11:08:04 +0200340static inline void tree_mod_log_read_unlock(struct btrfs_fs_info *fs_info)
Jan Schmidtbd989ba2012-05-16 17:18:50 +0200341{
Jan Schmidt097b8a72012-06-21 11:08:04 +0200342 read_unlock(&fs_info->tree_mod_log_lock);
343}
344
345static inline void tree_mod_log_write_lock(struct btrfs_fs_info *fs_info)
346{
347 write_lock(&fs_info->tree_mod_log_lock);
348}
349
350static inline void tree_mod_log_write_unlock(struct btrfs_fs_info *fs_info)
351{
352 write_unlock(&fs_info->tree_mod_log_lock);
353}
354
355/*
Josef Bacikfcebe452014-05-13 17:30:47 -0700356 * Pull a new tree mod seq number for our operation.
Jan Schmidtfc36ed7e2013-04-24 16:57:33 +0000357 */
Josef Bacikfcebe452014-05-13 17:30:47 -0700358static inline u64 btrfs_inc_tree_mod_seq(struct btrfs_fs_info *fs_info)
Jan Schmidtfc36ed7e2013-04-24 16:57:33 +0000359{
360 return atomic64_inc_return(&fs_info->tree_mod_seq);
361}
362
363/*
Jan Schmidt097b8a72012-06-21 11:08:04 +0200364 * This adds a new blocker to the tree mod log's blocker list if the @elem
365 * passed does not already have a sequence number set. So when a caller expects
366 * to record tree modifications, it should ensure to set elem->seq to zero
367 * before calling btrfs_get_tree_mod_seq.
368 * Returns a fresh, unused tree log modification sequence number, even if no new
369 * blocker was added.
370 */
371u64 btrfs_get_tree_mod_seq(struct btrfs_fs_info *fs_info,
372 struct seq_list *elem)
373{
Jan Schmidt097b8a72012-06-21 11:08:04 +0200374 tree_mod_log_write_lock(fs_info);
Jan Schmidtbd989ba2012-05-16 17:18:50 +0200375 spin_lock(&fs_info->tree_mod_seq_lock);
Jan Schmidt097b8a72012-06-21 11:08:04 +0200376 if (!elem->seq) {
Josef Bacikfcebe452014-05-13 17:30:47 -0700377 elem->seq = btrfs_inc_tree_mod_seq(fs_info);
Jan Schmidt097b8a72012-06-21 11:08:04 +0200378 list_add_tail(&elem->list, &fs_info->tree_mod_seq_list);
379 }
Jan Schmidtbd989ba2012-05-16 17:18:50 +0200380 spin_unlock(&fs_info->tree_mod_seq_lock);
Jan Schmidt097b8a72012-06-21 11:08:04 +0200381 tree_mod_log_write_unlock(fs_info);
382
Josef Bacikfcebe452014-05-13 17:30:47 -0700383 return elem->seq;
Jan Schmidtbd989ba2012-05-16 17:18:50 +0200384}
385
386void btrfs_put_tree_mod_seq(struct btrfs_fs_info *fs_info,
387 struct seq_list *elem)
388{
389 struct rb_root *tm_root;
390 struct rb_node *node;
391 struct rb_node *next;
392 struct seq_list *cur_elem;
393 struct tree_mod_elem *tm;
394 u64 min_seq = (u64)-1;
395 u64 seq_putting = elem->seq;
396
397 if (!seq_putting)
398 return;
399
Jan Schmidtbd989ba2012-05-16 17:18:50 +0200400 spin_lock(&fs_info->tree_mod_seq_lock);
401 list_del(&elem->list);
Jan Schmidt097b8a72012-06-21 11:08:04 +0200402 elem->seq = 0;
Jan Schmidtbd989ba2012-05-16 17:18:50 +0200403
404 list_for_each_entry(cur_elem, &fs_info->tree_mod_seq_list, list) {
Jan Schmidt097b8a72012-06-21 11:08:04 +0200405 if (cur_elem->seq < min_seq) {
Jan Schmidtbd989ba2012-05-16 17:18:50 +0200406 if (seq_putting > cur_elem->seq) {
407 /*
408 * blocker with lower sequence number exists, we
409 * cannot remove anything from the log
410 */
Jan Schmidt097b8a72012-06-21 11:08:04 +0200411 spin_unlock(&fs_info->tree_mod_seq_lock);
412 return;
Jan Schmidtbd989ba2012-05-16 17:18:50 +0200413 }
414 min_seq = cur_elem->seq;
415 }
416 }
Jan Schmidt097b8a72012-06-21 11:08:04 +0200417 spin_unlock(&fs_info->tree_mod_seq_lock);
418
419 /*
Jan Schmidtbd989ba2012-05-16 17:18:50 +0200420 * anything that's lower than the lowest existing (read: blocked)
421 * sequence number can be removed from the tree.
422 */
Jan Schmidt097b8a72012-06-21 11:08:04 +0200423 tree_mod_log_write_lock(fs_info);
Jan Schmidtbd989ba2012-05-16 17:18:50 +0200424 tm_root = &fs_info->tree_mod_log;
425 for (node = rb_first(tm_root); node; node = next) {
426 next = rb_next(node);
427 tm = container_of(node, struct tree_mod_elem, node);
Jan Schmidt097b8a72012-06-21 11:08:04 +0200428 if (tm->seq > min_seq)
Jan Schmidtbd989ba2012-05-16 17:18:50 +0200429 continue;
430 rb_erase(node, tm_root);
Jan Schmidtbd989ba2012-05-16 17:18:50 +0200431 kfree(tm);
432 }
Jan Schmidt097b8a72012-06-21 11:08:04 +0200433 tree_mod_log_write_unlock(fs_info);
Jan Schmidtbd989ba2012-05-16 17:18:50 +0200434}
435
436/*
437 * key order of the log:
438 * index -> sequence
439 *
440 * the index is the shifted logical of the *new* root node for root replace
441 * operations, or the shifted logical of the affected block for all other
442 * operations.
Filipe David Borba Manana5de865e2013-12-20 15:17:46 +0000443 *
444 * Note: must be called with write lock (tree_mod_log_write_lock).
Jan Schmidtbd989ba2012-05-16 17:18:50 +0200445 */
446static noinline int
447__tree_mod_log_insert(struct btrfs_fs_info *fs_info, struct tree_mod_elem *tm)
448{
449 struct rb_root *tm_root;
450 struct rb_node **new;
451 struct rb_node *parent = NULL;
452 struct tree_mod_elem *cur;
Jan Schmidtbd989ba2012-05-16 17:18:50 +0200453
Josef Bacikc8cc6342013-07-01 16:18:19 -0400454 BUG_ON(!tm);
455
Josef Bacikfcebe452014-05-13 17:30:47 -0700456 tm->seq = btrfs_inc_tree_mod_seq(fs_info);
Jan Schmidtbd989ba2012-05-16 17:18:50 +0200457
Jan Schmidtbd989ba2012-05-16 17:18:50 +0200458 tm_root = &fs_info->tree_mod_log;
459 new = &tm_root->rb_node;
460 while (*new) {
461 cur = container_of(*new, struct tree_mod_elem, node);
462 parent = *new;
463 if (cur->index < tm->index)
464 new = &((*new)->rb_left);
465 else if (cur->index > tm->index)
466 new = &((*new)->rb_right);
Jan Schmidt097b8a72012-06-21 11:08:04 +0200467 else if (cur->seq < tm->seq)
Jan Schmidtbd989ba2012-05-16 17:18:50 +0200468 new = &((*new)->rb_left);
Jan Schmidt097b8a72012-06-21 11:08:04 +0200469 else if (cur->seq > tm->seq)
Jan Schmidtbd989ba2012-05-16 17:18:50 +0200470 new = &((*new)->rb_right);
Filipe David Borba Manana5de865e2013-12-20 15:17:46 +0000471 else
472 return -EEXIST;
Jan Schmidtbd989ba2012-05-16 17:18:50 +0200473 }
474
475 rb_link_node(&tm->node, parent, new);
476 rb_insert_color(&tm->node, tm_root);
Filipe David Borba Manana5de865e2013-12-20 15:17:46 +0000477 return 0;
Jan Schmidtbd989ba2012-05-16 17:18:50 +0200478}
479
Jan Schmidt097b8a72012-06-21 11:08:04 +0200480/*
481 * Determines if logging can be omitted. Returns 1 if it can. Otherwise, it
482 * returns zero with the tree_mod_log_lock acquired. The caller must hold
483 * this until all tree mod log insertions are recorded in the rb tree and then
484 * call tree_mod_log_write_unlock() to release.
485 */
Jan Schmidte9b7fd42012-05-31 14:59:09 +0200486static inline int tree_mod_dont_log(struct btrfs_fs_info *fs_info,
487 struct extent_buffer *eb) {
488 smp_mb();
489 if (list_empty(&(fs_info)->tree_mod_seq_list))
490 return 1;
Jan Schmidt097b8a72012-06-21 11:08:04 +0200491 if (eb && btrfs_header_level(eb) == 0)
Jan Schmidte9b7fd42012-05-31 14:59:09 +0200492 return 1;
Filipe David Borba Manana5de865e2013-12-20 15:17:46 +0000493
494 tree_mod_log_write_lock(fs_info);
495 if (list_empty(&(fs_info)->tree_mod_seq_list)) {
496 tree_mod_log_write_unlock(fs_info);
497 return 1;
498 }
499
Jan Schmidte9b7fd42012-05-31 14:59:09 +0200500 return 0;
501}
502
Filipe David Borba Manana5de865e2013-12-20 15:17:46 +0000503/* Similar to tree_mod_dont_log, but doesn't acquire any locks. */
504static inline int tree_mod_need_log(const struct btrfs_fs_info *fs_info,
505 struct extent_buffer *eb)
506{
507 smp_mb();
508 if (list_empty(&(fs_info)->tree_mod_seq_list))
509 return 0;
510 if (eb && btrfs_header_level(eb) == 0)
511 return 0;
512
513 return 1;
514}
515
516static struct tree_mod_elem *
517alloc_tree_mod_elem(struct extent_buffer *eb, int slot,
518 enum mod_log_op op, gfp_t flags)
Jan Schmidtbd989ba2012-05-16 17:18:50 +0200519{
Jan Schmidt097b8a72012-06-21 11:08:04 +0200520 struct tree_mod_elem *tm;
Jan Schmidtbd989ba2012-05-16 17:18:50 +0200521
Josef Bacikc8cc6342013-07-01 16:18:19 -0400522 tm = kzalloc(sizeof(*tm), flags);
523 if (!tm)
Filipe David Borba Manana5de865e2013-12-20 15:17:46 +0000524 return NULL;
Jan Schmidtbd989ba2012-05-16 17:18:50 +0200525
526 tm->index = eb->start >> PAGE_CACHE_SHIFT;
527 if (op != MOD_LOG_KEY_ADD) {
528 btrfs_node_key(eb, &tm->key, slot);
529 tm->blockptr = btrfs_node_blockptr(eb, slot);
530 }
531 tm->op = op;
532 tm->slot = slot;
533 tm->generation = btrfs_node_ptr_generation(eb, slot);
Filipe David Borba Manana5de865e2013-12-20 15:17:46 +0000534 RB_CLEAR_NODE(&tm->node);
Jan Schmidtbd989ba2012-05-16 17:18:50 +0200535
Filipe David Borba Manana5de865e2013-12-20 15:17:46 +0000536 return tm;
Jan Schmidt097b8a72012-06-21 11:08:04 +0200537}
538
539static noinline int
Josef Bacikc8cc6342013-07-01 16:18:19 -0400540tree_mod_log_insert_key(struct btrfs_fs_info *fs_info,
541 struct extent_buffer *eb, int slot,
542 enum mod_log_op op, gfp_t flags)
Jan Schmidt097b8a72012-06-21 11:08:04 +0200543{
Filipe David Borba Manana5de865e2013-12-20 15:17:46 +0000544 struct tree_mod_elem *tm;
545 int ret;
546
547 if (!tree_mod_need_log(fs_info, eb))
Jan Schmidt097b8a72012-06-21 11:08:04 +0200548 return 0;
549
Filipe David Borba Manana5de865e2013-12-20 15:17:46 +0000550 tm = alloc_tree_mod_elem(eb, slot, op, flags);
551 if (!tm)
552 return -ENOMEM;
553
554 if (tree_mod_dont_log(fs_info, eb)) {
555 kfree(tm);
556 return 0;
557 }
558
559 ret = __tree_mod_log_insert(fs_info, tm);
560 tree_mod_log_write_unlock(fs_info);
561 if (ret)
562 kfree(tm);
563
564 return ret;
Jan Schmidt097b8a72012-06-21 11:08:04 +0200565}
566
567static noinline int
Jan Schmidtbd989ba2012-05-16 17:18:50 +0200568tree_mod_log_insert_move(struct btrfs_fs_info *fs_info,
569 struct extent_buffer *eb, int dst_slot, int src_slot,
570 int nr_items, gfp_t flags)
571{
Filipe David Borba Manana5de865e2013-12-20 15:17:46 +0000572 struct tree_mod_elem *tm = NULL;
573 struct tree_mod_elem **tm_list = NULL;
574 int ret = 0;
Jan Schmidtbd989ba2012-05-16 17:18:50 +0200575 int i;
Filipe David Borba Manana5de865e2013-12-20 15:17:46 +0000576 int locked = 0;
Jan Schmidtbd989ba2012-05-16 17:18:50 +0200577
Filipe David Borba Manana5de865e2013-12-20 15:17:46 +0000578 if (!tree_mod_need_log(fs_info, eb))
Jan Schmidtf3956942012-05-31 15:02:32 +0200579 return 0;
Jan Schmidtbd989ba2012-05-16 17:18:50 +0200580
David Sterba31e818f2015-02-20 18:00:26 +0100581 tm_list = kcalloc(nr_items, sizeof(struct tree_mod_elem *), flags);
Filipe David Borba Manana5de865e2013-12-20 15:17:46 +0000582 if (!tm_list)
583 return -ENOMEM;
Jan Schmidtbd989ba2012-05-16 17:18:50 +0200584
Josef Bacikc8cc6342013-07-01 16:18:19 -0400585 tm = kzalloc(sizeof(*tm), flags);
Filipe David Borba Manana5de865e2013-12-20 15:17:46 +0000586 if (!tm) {
587 ret = -ENOMEM;
588 goto free_tms;
589 }
Jan Schmidtf3956942012-05-31 15:02:32 +0200590
Jan Schmidtbd989ba2012-05-16 17:18:50 +0200591 tm->index = eb->start >> PAGE_CACHE_SHIFT;
592 tm->slot = src_slot;
593 tm->move.dst_slot = dst_slot;
594 tm->move.nr_items = nr_items;
595 tm->op = MOD_LOG_MOVE_KEYS;
596
Filipe David Borba Manana5de865e2013-12-20 15:17:46 +0000597 for (i = 0; i + dst_slot < src_slot && i < nr_items; i++) {
598 tm_list[i] = alloc_tree_mod_elem(eb, i + dst_slot,
599 MOD_LOG_KEY_REMOVE_WHILE_MOVING, flags);
600 if (!tm_list[i]) {
601 ret = -ENOMEM;
602 goto free_tms;
603 }
604 }
605
606 if (tree_mod_dont_log(fs_info, eb))
607 goto free_tms;
608 locked = 1;
609
610 /*
611 * When we override something during the move, we log these removals.
612 * This can only happen when we move towards the beginning of the
613 * buffer, i.e. dst_slot < src_slot.
614 */
615 for (i = 0; i + dst_slot < src_slot && i < nr_items; i++) {
616 ret = __tree_mod_log_insert(fs_info, tm_list[i]);
617 if (ret)
618 goto free_tms;
619 }
620
621 ret = __tree_mod_log_insert(fs_info, tm);
622 if (ret)
623 goto free_tms;
624 tree_mod_log_write_unlock(fs_info);
625 kfree(tm_list);
626
627 return 0;
628free_tms:
629 for (i = 0; i < nr_items; i++) {
630 if (tm_list[i] && !RB_EMPTY_NODE(&tm_list[i]->node))
631 rb_erase(&tm_list[i]->node, &fs_info->tree_mod_log);
632 kfree(tm_list[i]);
633 }
634 if (locked)
635 tree_mod_log_write_unlock(fs_info);
636 kfree(tm_list);
637 kfree(tm);
638
639 return ret;
Jan Schmidtbd989ba2012-05-16 17:18:50 +0200640}
641
Filipe David Borba Manana5de865e2013-12-20 15:17:46 +0000642static inline int
643__tree_mod_log_free_eb(struct btrfs_fs_info *fs_info,
644 struct tree_mod_elem **tm_list,
645 int nritems)
Jan Schmidt097b8a72012-06-21 11:08:04 +0200646{
Filipe David Borba Manana5de865e2013-12-20 15:17:46 +0000647 int i, j;
Jan Schmidt097b8a72012-06-21 11:08:04 +0200648 int ret;
649
Jan Schmidt097b8a72012-06-21 11:08:04 +0200650 for (i = nritems - 1; i >= 0; i--) {
Filipe David Borba Manana5de865e2013-12-20 15:17:46 +0000651 ret = __tree_mod_log_insert(fs_info, tm_list[i]);
652 if (ret) {
653 for (j = nritems - 1; j > i; j--)
654 rb_erase(&tm_list[j]->node,
655 &fs_info->tree_mod_log);
656 return ret;
657 }
Jan Schmidt097b8a72012-06-21 11:08:04 +0200658 }
Filipe David Borba Manana5de865e2013-12-20 15:17:46 +0000659
660 return 0;
Jan Schmidt097b8a72012-06-21 11:08:04 +0200661}
662
Jan Schmidtbd989ba2012-05-16 17:18:50 +0200663static noinline int
664tree_mod_log_insert_root(struct btrfs_fs_info *fs_info,
665 struct extent_buffer *old_root,
Jan Schmidt90f8d622013-04-13 13:19:53 +0000666 struct extent_buffer *new_root, gfp_t flags,
667 int log_removal)
Jan Schmidtbd989ba2012-05-16 17:18:50 +0200668{
Filipe David Borba Manana5de865e2013-12-20 15:17:46 +0000669 struct tree_mod_elem *tm = NULL;
670 struct tree_mod_elem **tm_list = NULL;
671 int nritems = 0;
672 int ret = 0;
673 int i;
Jan Schmidtbd989ba2012-05-16 17:18:50 +0200674
Filipe David Borba Manana5de865e2013-12-20 15:17:46 +0000675 if (!tree_mod_need_log(fs_info, NULL))
Jan Schmidt097b8a72012-06-21 11:08:04 +0200676 return 0;
677
Filipe David Borba Manana5de865e2013-12-20 15:17:46 +0000678 if (log_removal && btrfs_header_level(old_root) > 0) {
679 nritems = btrfs_header_nritems(old_root);
David Sterba31e818f2015-02-20 18:00:26 +0100680 tm_list = kcalloc(nritems, sizeof(struct tree_mod_elem *),
Filipe David Borba Manana5de865e2013-12-20 15:17:46 +0000681 flags);
682 if (!tm_list) {
683 ret = -ENOMEM;
684 goto free_tms;
685 }
686 for (i = 0; i < nritems; i++) {
687 tm_list[i] = alloc_tree_mod_elem(old_root, i,
688 MOD_LOG_KEY_REMOVE_WHILE_FREEING, flags);
689 if (!tm_list[i]) {
690 ret = -ENOMEM;
691 goto free_tms;
692 }
693 }
694 }
Jan Schmidtd9abbf12013-03-20 13:49:48 +0000695
Josef Bacikc8cc6342013-07-01 16:18:19 -0400696 tm = kzalloc(sizeof(*tm), flags);
Filipe David Borba Manana5de865e2013-12-20 15:17:46 +0000697 if (!tm) {
698 ret = -ENOMEM;
699 goto free_tms;
700 }
Jan Schmidtbd989ba2012-05-16 17:18:50 +0200701
702 tm->index = new_root->start >> PAGE_CACHE_SHIFT;
703 tm->old_root.logical = old_root->start;
704 tm->old_root.level = btrfs_header_level(old_root);
705 tm->generation = btrfs_header_generation(old_root);
706 tm->op = MOD_LOG_ROOT_REPLACE;
707
Filipe David Borba Manana5de865e2013-12-20 15:17:46 +0000708 if (tree_mod_dont_log(fs_info, NULL))
709 goto free_tms;
710
711 if (tm_list)
712 ret = __tree_mod_log_free_eb(fs_info, tm_list, nritems);
713 if (!ret)
714 ret = __tree_mod_log_insert(fs_info, tm);
715
716 tree_mod_log_write_unlock(fs_info);
717 if (ret)
718 goto free_tms;
719 kfree(tm_list);
720
721 return ret;
722
723free_tms:
724 if (tm_list) {
725 for (i = 0; i < nritems; i++)
726 kfree(tm_list[i]);
727 kfree(tm_list);
728 }
729 kfree(tm);
730
731 return ret;
Jan Schmidtbd989ba2012-05-16 17:18:50 +0200732}
733
734static struct tree_mod_elem *
735__tree_mod_log_search(struct btrfs_fs_info *fs_info, u64 start, u64 min_seq,
736 int smallest)
737{
738 struct rb_root *tm_root;
739 struct rb_node *node;
740 struct tree_mod_elem *cur = NULL;
741 struct tree_mod_elem *found = NULL;
742 u64 index = start >> PAGE_CACHE_SHIFT;
743
Jan Schmidt097b8a72012-06-21 11:08:04 +0200744 tree_mod_log_read_lock(fs_info);
Jan Schmidtbd989ba2012-05-16 17:18:50 +0200745 tm_root = &fs_info->tree_mod_log;
746 node = tm_root->rb_node;
747 while (node) {
748 cur = container_of(node, struct tree_mod_elem, node);
749 if (cur->index < index) {
750 node = node->rb_left;
751 } else if (cur->index > index) {
752 node = node->rb_right;
Jan Schmidt097b8a72012-06-21 11:08:04 +0200753 } else if (cur->seq < min_seq) {
Jan Schmidtbd989ba2012-05-16 17:18:50 +0200754 node = node->rb_left;
755 } else if (!smallest) {
756 /* we want the node with the highest seq */
757 if (found)
Jan Schmidt097b8a72012-06-21 11:08:04 +0200758 BUG_ON(found->seq > cur->seq);
Jan Schmidtbd989ba2012-05-16 17:18:50 +0200759 found = cur;
760 node = node->rb_left;
Jan Schmidt097b8a72012-06-21 11:08:04 +0200761 } else if (cur->seq > min_seq) {
Jan Schmidtbd989ba2012-05-16 17:18:50 +0200762 /* we want the node with the smallest seq */
763 if (found)
Jan Schmidt097b8a72012-06-21 11:08:04 +0200764 BUG_ON(found->seq < cur->seq);
Jan Schmidtbd989ba2012-05-16 17:18:50 +0200765 found = cur;
766 node = node->rb_right;
767 } else {
768 found = cur;
769 break;
770 }
771 }
Jan Schmidt097b8a72012-06-21 11:08:04 +0200772 tree_mod_log_read_unlock(fs_info);
Jan Schmidtbd989ba2012-05-16 17:18:50 +0200773
774 return found;
775}
776
777/*
778 * this returns the element from the log with the smallest time sequence
779 * value that's in the log (the oldest log item). any element with a time
780 * sequence lower than min_seq will be ignored.
781 */
782static struct tree_mod_elem *
783tree_mod_log_search_oldest(struct btrfs_fs_info *fs_info, u64 start,
784 u64 min_seq)
785{
786 return __tree_mod_log_search(fs_info, start, min_seq, 1);
787}
788
789/*
790 * this returns the element from the log with the largest time sequence
791 * value that's in the log (the most recent log item). any element with
792 * a time sequence lower than min_seq will be ignored.
793 */
794static struct tree_mod_elem *
795tree_mod_log_search(struct btrfs_fs_info *fs_info, u64 start, u64 min_seq)
796{
797 return __tree_mod_log_search(fs_info, start, min_seq, 0);
798}
799
Filipe David Borba Manana5de865e2013-12-20 15:17:46 +0000800static noinline int
Jan Schmidtbd989ba2012-05-16 17:18:50 +0200801tree_mod_log_eb_copy(struct btrfs_fs_info *fs_info, struct extent_buffer *dst,
802 struct extent_buffer *src, unsigned long dst_offset,
Jan Schmidt90f8d622013-04-13 13:19:53 +0000803 unsigned long src_offset, int nr_items)
Jan Schmidtbd989ba2012-05-16 17:18:50 +0200804{
Filipe David Borba Manana5de865e2013-12-20 15:17:46 +0000805 int ret = 0;
806 struct tree_mod_elem **tm_list = NULL;
807 struct tree_mod_elem **tm_list_add, **tm_list_rem;
Jan Schmidtbd989ba2012-05-16 17:18:50 +0200808 int i;
Filipe David Borba Manana5de865e2013-12-20 15:17:46 +0000809 int locked = 0;
Jan Schmidtbd989ba2012-05-16 17:18:50 +0200810
Filipe David Borba Manana5de865e2013-12-20 15:17:46 +0000811 if (!tree_mod_need_log(fs_info, NULL))
812 return 0;
Jan Schmidtbd989ba2012-05-16 17:18:50 +0200813
Josef Bacikc8cc6342013-07-01 16:18:19 -0400814 if (btrfs_header_level(dst) == 0 && btrfs_header_level(src) == 0)
Filipe David Borba Manana5de865e2013-12-20 15:17:46 +0000815 return 0;
816
David Sterba31e818f2015-02-20 18:00:26 +0100817 tm_list = kcalloc(nr_items * 2, sizeof(struct tree_mod_elem *),
Filipe David Borba Manana5de865e2013-12-20 15:17:46 +0000818 GFP_NOFS);
819 if (!tm_list)
820 return -ENOMEM;
821
822 tm_list_add = tm_list;
823 tm_list_rem = tm_list + nr_items;
824 for (i = 0; i < nr_items; i++) {
825 tm_list_rem[i] = alloc_tree_mod_elem(src, i + src_offset,
826 MOD_LOG_KEY_REMOVE, GFP_NOFS);
827 if (!tm_list_rem[i]) {
828 ret = -ENOMEM;
829 goto free_tms;
830 }
831
832 tm_list_add[i] = alloc_tree_mod_elem(dst, i + dst_offset,
833 MOD_LOG_KEY_ADD, GFP_NOFS);
834 if (!tm_list_add[i]) {
835 ret = -ENOMEM;
836 goto free_tms;
837 }
838 }
839
840 if (tree_mod_dont_log(fs_info, NULL))
841 goto free_tms;
842 locked = 1;
Jan Schmidtbd989ba2012-05-16 17:18:50 +0200843
Jan Schmidtbd989ba2012-05-16 17:18:50 +0200844 for (i = 0; i < nr_items; i++) {
Filipe David Borba Manana5de865e2013-12-20 15:17:46 +0000845 ret = __tree_mod_log_insert(fs_info, tm_list_rem[i]);
846 if (ret)
847 goto free_tms;
848 ret = __tree_mod_log_insert(fs_info, tm_list_add[i]);
849 if (ret)
850 goto free_tms;
Jan Schmidtbd989ba2012-05-16 17:18:50 +0200851 }
Filipe David Borba Manana5de865e2013-12-20 15:17:46 +0000852
853 tree_mod_log_write_unlock(fs_info);
854 kfree(tm_list);
855
856 return 0;
857
858free_tms:
859 for (i = 0; i < nr_items * 2; i++) {
860 if (tm_list[i] && !RB_EMPTY_NODE(&tm_list[i]->node))
861 rb_erase(&tm_list[i]->node, &fs_info->tree_mod_log);
862 kfree(tm_list[i]);
863 }
864 if (locked)
865 tree_mod_log_write_unlock(fs_info);
866 kfree(tm_list);
867
868 return ret;
Jan Schmidtbd989ba2012-05-16 17:18:50 +0200869}
870
871static inline void
872tree_mod_log_eb_move(struct btrfs_fs_info *fs_info, struct extent_buffer *dst,
873 int dst_offset, int src_offset, int nr_items)
874{
875 int ret;
876 ret = tree_mod_log_insert_move(fs_info, dst, dst_offset, src_offset,
877 nr_items, GFP_NOFS);
878 BUG_ON(ret < 0);
879}
880
Jan Schmidt097b8a72012-06-21 11:08:04 +0200881static noinline void
Jan Schmidtbd989ba2012-05-16 17:18:50 +0200882tree_mod_log_set_node_key(struct btrfs_fs_info *fs_info,
Liu Bo32adf092012-10-19 12:52:15 +0000883 struct extent_buffer *eb, int slot, int atomic)
Jan Schmidtbd989ba2012-05-16 17:18:50 +0200884{
885 int ret;
886
Filipe David Borba Manana78357762013-12-12 19:19:52 +0000887 ret = tree_mod_log_insert_key(fs_info, eb, slot,
Josef Bacikc8cc6342013-07-01 16:18:19 -0400888 MOD_LOG_KEY_REPLACE,
889 atomic ? GFP_ATOMIC : GFP_NOFS);
Jan Schmidtbd989ba2012-05-16 17:18:50 +0200890 BUG_ON(ret < 0);
891}
892
Filipe David Borba Manana5de865e2013-12-20 15:17:46 +0000893static noinline int
Jan Schmidt097b8a72012-06-21 11:08:04 +0200894tree_mod_log_free_eb(struct btrfs_fs_info *fs_info, struct extent_buffer *eb)
Jan Schmidtbd989ba2012-05-16 17:18:50 +0200895{
Filipe David Borba Manana5de865e2013-12-20 15:17:46 +0000896 struct tree_mod_elem **tm_list = NULL;
897 int nritems = 0;
898 int i;
899 int ret = 0;
900
901 if (btrfs_header_level(eb) == 0)
902 return 0;
903
904 if (!tree_mod_need_log(fs_info, NULL))
905 return 0;
906
907 nritems = btrfs_header_nritems(eb);
David Sterba31e818f2015-02-20 18:00:26 +0100908 tm_list = kcalloc(nritems, sizeof(struct tree_mod_elem *), GFP_NOFS);
Filipe David Borba Manana5de865e2013-12-20 15:17:46 +0000909 if (!tm_list)
910 return -ENOMEM;
911
912 for (i = 0; i < nritems; i++) {
913 tm_list[i] = alloc_tree_mod_elem(eb, i,
914 MOD_LOG_KEY_REMOVE_WHILE_FREEING, GFP_NOFS);
915 if (!tm_list[i]) {
916 ret = -ENOMEM;
917 goto free_tms;
918 }
919 }
920
Jan Schmidte9b7fd42012-05-31 14:59:09 +0200921 if (tree_mod_dont_log(fs_info, eb))
Filipe David Borba Manana5de865e2013-12-20 15:17:46 +0000922 goto free_tms;
923
924 ret = __tree_mod_log_free_eb(fs_info, tm_list, nritems);
925 tree_mod_log_write_unlock(fs_info);
926 if (ret)
927 goto free_tms;
928 kfree(tm_list);
929
930 return 0;
931
932free_tms:
933 for (i = 0; i < nritems; i++)
934 kfree(tm_list[i]);
935 kfree(tm_list);
936
937 return ret;
Jan Schmidtbd989ba2012-05-16 17:18:50 +0200938}
939
Jan Schmidt097b8a72012-06-21 11:08:04 +0200940static noinline void
Jan Schmidtbd989ba2012-05-16 17:18:50 +0200941tree_mod_log_set_root_pointer(struct btrfs_root *root,
Jan Schmidt90f8d622013-04-13 13:19:53 +0000942 struct extent_buffer *new_root_node,
943 int log_removal)
Jan Schmidtbd989ba2012-05-16 17:18:50 +0200944{
945 int ret;
Jan Schmidtbd989ba2012-05-16 17:18:50 +0200946 ret = tree_mod_log_insert_root(root->fs_info, root->node,
Jan Schmidt90f8d622013-04-13 13:19:53 +0000947 new_root_node, GFP_NOFS, log_removal);
Jan Schmidtbd989ba2012-05-16 17:18:50 +0200948 BUG_ON(ret < 0);
949}
950
Chris Masond352ac62008-09-29 15:18:18 -0400951/*
Yan Zheng5d4f98a2009-06-10 10:45:14 -0400952 * check if the tree block can be shared by multiple trees
953 */
954int btrfs_block_can_be_shared(struct btrfs_root *root,
955 struct extent_buffer *buf)
956{
957 /*
958 * Tree blocks not in refernece counted trees and tree roots
959 * are never shared. If a block was allocated after the last
960 * snapshot and the block was not allocated by tree relocation,
961 * we know the block is not shared.
962 */
Miao Xie27cdeb72014-04-02 19:51:05 +0800963 if (test_bit(BTRFS_ROOT_REF_COWS, &root->state) &&
Yan Zheng5d4f98a2009-06-10 10:45:14 -0400964 buf != root->node && buf != root->commit_root &&
965 (btrfs_header_generation(buf) <=
966 btrfs_root_last_snapshot(&root->root_item) ||
967 btrfs_header_flag(buf, BTRFS_HEADER_FLAG_RELOC)))
968 return 1;
969#ifdef BTRFS_COMPAT_EXTENT_TREE_V0
Miao Xie27cdeb72014-04-02 19:51:05 +0800970 if (test_bit(BTRFS_ROOT_REF_COWS, &root->state) &&
Yan Zheng5d4f98a2009-06-10 10:45:14 -0400971 btrfs_header_backref_rev(buf) < BTRFS_MIXED_BACKREF_REV)
972 return 1;
973#endif
974 return 0;
975}
976
977static noinline int update_ref_for_cow(struct btrfs_trans_handle *trans,
978 struct btrfs_root *root,
979 struct extent_buffer *buf,
Yan, Zhengf0486c62010-05-16 10:46:25 -0400980 struct extent_buffer *cow,
981 int *last_ref)
Yan Zheng5d4f98a2009-06-10 10:45:14 -0400982{
983 u64 refs;
984 u64 owner;
985 u64 flags;
986 u64 new_flags = 0;
987 int ret;
988
989 /*
990 * Backrefs update rules:
991 *
992 * Always use full backrefs for extent pointers in tree block
993 * allocated by tree relocation.
994 *
995 * If a shared tree block is no longer referenced by its owner
996 * tree (btrfs_header_owner(buf) == root->root_key.objectid),
997 * use full backrefs for extent pointers in tree block.
998 *
999 * If a tree block is been relocating
1000 * (root->root_key.objectid == BTRFS_TREE_RELOC_OBJECTID),
1001 * use full backrefs for extent pointers in tree block.
1002 * The reason for this is some operations (such as drop tree)
1003 * are only allowed for blocks use full backrefs.
1004 */
1005
1006 if (btrfs_block_can_be_shared(root, buf)) {
1007 ret = btrfs_lookup_extent_info(trans, root, buf->start,
Josef Bacik3173a182013-03-07 14:22:04 -05001008 btrfs_header_level(buf), 1,
1009 &refs, &flags);
Mark Fashehbe1a5562011-08-08 13:20:18 -07001010 if (ret)
1011 return ret;
Mark Fashehe5df9572011-08-29 14:17:04 -07001012 if (refs == 0) {
1013 ret = -EROFS;
Anand Jaina4553fe2015-09-25 14:43:01 +08001014 btrfs_std_error(root->fs_info, ret, NULL);
Mark Fashehe5df9572011-08-29 14:17:04 -07001015 return ret;
1016 }
Yan Zheng5d4f98a2009-06-10 10:45:14 -04001017 } else {
1018 refs = 1;
1019 if (root->root_key.objectid == BTRFS_TREE_RELOC_OBJECTID ||
1020 btrfs_header_backref_rev(buf) < BTRFS_MIXED_BACKREF_REV)
1021 flags = BTRFS_BLOCK_FLAG_FULL_BACKREF;
1022 else
1023 flags = 0;
1024 }
1025
1026 owner = btrfs_header_owner(buf);
1027 BUG_ON(owner == BTRFS_TREE_RELOC_OBJECTID &&
1028 !(flags & BTRFS_BLOCK_FLAG_FULL_BACKREF));
1029
1030 if (refs > 1) {
1031 if ((owner == root->root_key.objectid ||
1032 root->root_key.objectid == BTRFS_TREE_RELOC_OBJECTID) &&
1033 !(flags & BTRFS_BLOCK_FLAG_FULL_BACKREF)) {
Josef Bacike339a6b2014-07-02 10:54:25 -07001034 ret = btrfs_inc_ref(trans, root, buf, 1);
Jeff Mahoney79787ea2012-03-12 16:03:00 +01001035 BUG_ON(ret); /* -ENOMEM */
Yan Zheng5d4f98a2009-06-10 10:45:14 -04001036
1037 if (root->root_key.objectid ==
1038 BTRFS_TREE_RELOC_OBJECTID) {
Josef Bacike339a6b2014-07-02 10:54:25 -07001039 ret = btrfs_dec_ref(trans, root, buf, 0);
Jeff Mahoney79787ea2012-03-12 16:03:00 +01001040 BUG_ON(ret); /* -ENOMEM */
Josef Bacike339a6b2014-07-02 10:54:25 -07001041 ret = btrfs_inc_ref(trans, root, cow, 1);
Jeff Mahoney79787ea2012-03-12 16:03:00 +01001042 BUG_ON(ret); /* -ENOMEM */
Yan Zheng5d4f98a2009-06-10 10:45:14 -04001043 }
1044 new_flags |= BTRFS_BLOCK_FLAG_FULL_BACKREF;
1045 } else {
1046
1047 if (root->root_key.objectid ==
1048 BTRFS_TREE_RELOC_OBJECTID)
Josef Bacike339a6b2014-07-02 10:54:25 -07001049 ret = btrfs_inc_ref(trans, root, cow, 1);
Yan Zheng5d4f98a2009-06-10 10:45:14 -04001050 else
Josef Bacike339a6b2014-07-02 10:54:25 -07001051 ret = btrfs_inc_ref(trans, root, cow, 0);
Jeff Mahoney79787ea2012-03-12 16:03:00 +01001052 BUG_ON(ret); /* -ENOMEM */
Yan Zheng5d4f98a2009-06-10 10:45:14 -04001053 }
1054 if (new_flags != 0) {
Josef Bacikb1c79e02013-05-09 13:49:30 -04001055 int level = btrfs_header_level(buf);
1056
Yan Zheng5d4f98a2009-06-10 10:45:14 -04001057 ret = btrfs_set_disk_extent_flags(trans, root,
1058 buf->start,
1059 buf->len,
Josef Bacikb1c79e02013-05-09 13:49:30 -04001060 new_flags, level, 0);
Mark Fashehbe1a5562011-08-08 13:20:18 -07001061 if (ret)
1062 return ret;
Yan Zheng5d4f98a2009-06-10 10:45:14 -04001063 }
1064 } else {
1065 if (flags & BTRFS_BLOCK_FLAG_FULL_BACKREF) {
1066 if (root->root_key.objectid ==
1067 BTRFS_TREE_RELOC_OBJECTID)
Josef Bacike339a6b2014-07-02 10:54:25 -07001068 ret = btrfs_inc_ref(trans, root, cow, 1);
Yan Zheng5d4f98a2009-06-10 10:45:14 -04001069 else
Josef Bacike339a6b2014-07-02 10:54:25 -07001070 ret = btrfs_inc_ref(trans, root, cow, 0);
Jeff Mahoney79787ea2012-03-12 16:03:00 +01001071 BUG_ON(ret); /* -ENOMEM */
Josef Bacike339a6b2014-07-02 10:54:25 -07001072 ret = btrfs_dec_ref(trans, root, buf, 1);
Jeff Mahoney79787ea2012-03-12 16:03:00 +01001073 BUG_ON(ret); /* -ENOMEM */
Yan Zheng5d4f98a2009-06-10 10:45:14 -04001074 }
Daniel Dressler01d58472014-11-21 17:15:07 +09001075 clean_tree_block(trans, root->fs_info, buf);
Yan, Zhengf0486c62010-05-16 10:46:25 -04001076 *last_ref = 1;
Yan Zheng5d4f98a2009-06-10 10:45:14 -04001077 }
1078 return 0;
1079}
1080
1081/*
Chris Masond3977122009-01-05 21:25:51 -05001082 * does the dirty work in cow of a single block. The parent block (if
1083 * supplied) is updated to point to the new cow copy. The new buffer is marked
1084 * dirty and returned locked. If you modify the block it needs to be marked
1085 * dirty again.
Chris Masond352ac62008-09-29 15:18:18 -04001086 *
1087 * search_start -- an allocation hint for the new block
1088 *
Chris Masond3977122009-01-05 21:25:51 -05001089 * empty_size -- a hint that you plan on doing more cow. This is the size in
1090 * bytes the allocator should try to find free next to the block it returns.
1091 * This is just a hint and may be ignored by the allocator.
Chris Masond352ac62008-09-29 15:18:18 -04001092 */
Chris Masond3977122009-01-05 21:25:51 -05001093static noinline int __btrfs_cow_block(struct btrfs_trans_handle *trans,
Chris Mason5f39d392007-10-15 16:14:19 -04001094 struct btrfs_root *root,
1095 struct extent_buffer *buf,
1096 struct extent_buffer *parent, int parent_slot,
1097 struct extent_buffer **cow_ret,
Chris Mason9fa8cfe2009-03-13 10:24:59 -04001098 u64 search_start, u64 empty_size)
Chris Mason6702ed42007-08-07 16:15:09 -04001099{
Yan Zheng5d4f98a2009-06-10 10:45:14 -04001100 struct btrfs_disk_key disk_key;
Chris Mason5f39d392007-10-15 16:14:19 -04001101 struct extent_buffer *cow;
Mark Fashehbe1a5562011-08-08 13:20:18 -07001102 int level, ret;
Yan, Zhengf0486c62010-05-16 10:46:25 -04001103 int last_ref = 0;
Chris Mason925baed2008-06-25 16:01:30 -04001104 int unlock_orig = 0;
Yan Zheng5d4f98a2009-06-10 10:45:14 -04001105 u64 parent_start;
Chris Mason6702ed42007-08-07 16:15:09 -04001106
Chris Mason925baed2008-06-25 16:01:30 -04001107 if (*cow_ret == buf)
1108 unlock_orig = 1;
1109
Chris Masonb9447ef2009-03-09 11:45:38 -04001110 btrfs_assert_tree_locked(buf);
Chris Mason925baed2008-06-25 16:01:30 -04001111
Miao Xie27cdeb72014-04-02 19:51:05 +08001112 WARN_ON(test_bit(BTRFS_ROOT_REF_COWS, &root->state) &&
1113 trans->transid != root->fs_info->running_transaction->transid);
1114 WARN_ON(test_bit(BTRFS_ROOT_REF_COWS, &root->state) &&
1115 trans->transid != root->last_trans);
Chris Mason5f39d392007-10-15 16:14:19 -04001116
Chris Mason7bb86312007-12-11 09:25:06 -05001117 level = btrfs_header_level(buf);
Zheng Yan31840ae2008-09-23 13:14:14 -04001118
Yan Zheng5d4f98a2009-06-10 10:45:14 -04001119 if (level == 0)
1120 btrfs_item_key(buf, &disk_key, 0);
1121 else
1122 btrfs_node_key(buf, &disk_key, 0);
1123
1124 if (root->root_key.objectid == BTRFS_TREE_RELOC_OBJECTID) {
1125 if (parent)
1126 parent_start = parent->start;
1127 else
1128 parent_start = 0;
1129 } else
1130 parent_start = 0;
1131
David Sterba4d75f8a2014-06-15 01:54:12 +02001132 cow = btrfs_alloc_tree_block(trans, root, parent_start,
1133 root->root_key.objectid, &disk_key, level,
1134 search_start, empty_size);
Chris Mason6702ed42007-08-07 16:15:09 -04001135 if (IS_ERR(cow))
1136 return PTR_ERR(cow);
1137
Chris Masonb4ce94d2009-02-04 09:25:08 -05001138 /* cow is set to blocking by btrfs_init_new_buffer */
1139
Chris Mason5f39d392007-10-15 16:14:19 -04001140 copy_extent_buffer(cow, buf, 0, 0, cow->len);
Chris Masondb945352007-10-15 16:15:53 -04001141 btrfs_set_header_bytenr(cow, cow->start);
Chris Mason5f39d392007-10-15 16:14:19 -04001142 btrfs_set_header_generation(cow, trans->transid);
Yan Zheng5d4f98a2009-06-10 10:45:14 -04001143 btrfs_set_header_backref_rev(cow, BTRFS_MIXED_BACKREF_REV);
1144 btrfs_clear_header_flag(cow, BTRFS_HEADER_FLAG_WRITTEN |
1145 BTRFS_HEADER_FLAG_RELOC);
1146 if (root->root_key.objectid == BTRFS_TREE_RELOC_OBJECTID)
1147 btrfs_set_header_flag(cow, BTRFS_HEADER_FLAG_RELOC);
1148 else
1149 btrfs_set_header_owner(cow, root->root_key.objectid);
Chris Mason6702ed42007-08-07 16:15:09 -04001150
Ross Kirk0a4e5582013-09-24 10:12:38 +01001151 write_extent_buffer(cow, root->fs_info->fsid, btrfs_header_fsid(),
Yan Zheng2b820322008-11-17 21:11:30 -05001152 BTRFS_FSID_SIZE);
1153
Mark Fashehbe1a5562011-08-08 13:20:18 -07001154 ret = update_ref_for_cow(trans, root, buf, cow, &last_ref);
Mark Fashehb68dc2a2011-08-29 14:30:39 -07001155 if (ret) {
Jeff Mahoney79787ea2012-03-12 16:03:00 +01001156 btrfs_abort_transaction(trans, root, ret);
Mark Fashehb68dc2a2011-08-29 14:30:39 -07001157 return ret;
1158 }
Zheng Yan1a40e232008-09-26 10:09:34 -04001159
Miao Xie27cdeb72014-04-02 19:51:05 +08001160 if (test_bit(BTRFS_ROOT_REF_COWS, &root->state)) {
Josef Bacik83d4cfd2013-08-30 15:09:51 -04001161 ret = btrfs_reloc_cow_block(trans, root, buf, cow);
Zhaolei93314e32015-08-06 21:56:58 +08001162 if (ret) {
1163 btrfs_abort_transaction(trans, root, ret);
Josef Bacik83d4cfd2013-08-30 15:09:51 -04001164 return ret;
Zhaolei93314e32015-08-06 21:56:58 +08001165 }
Josef Bacik83d4cfd2013-08-30 15:09:51 -04001166 }
Yan, Zheng3fd0a552010-05-16 10:49:59 -04001167
Chris Mason6702ed42007-08-07 16:15:09 -04001168 if (buf == root->node) {
Chris Mason925baed2008-06-25 16:01:30 -04001169 WARN_ON(parent && parent != buf);
Yan Zheng5d4f98a2009-06-10 10:45:14 -04001170 if (root->root_key.objectid == BTRFS_TREE_RELOC_OBJECTID ||
1171 btrfs_header_backref_rev(buf) < BTRFS_MIXED_BACKREF_REV)
1172 parent_start = buf->start;
1173 else
1174 parent_start = 0;
Chris Mason925baed2008-06-25 16:01:30 -04001175
Chris Mason5f39d392007-10-15 16:14:19 -04001176 extent_buffer_get(cow);
Jan Schmidt90f8d622013-04-13 13:19:53 +00001177 tree_mod_log_set_root_pointer(root, cow, 1);
Chris Mason240f62c2011-03-23 14:54:42 -04001178 rcu_assign_pointer(root->node, cow);
Chris Mason925baed2008-06-25 16:01:30 -04001179
Yan, Zhengf0486c62010-05-16 10:46:25 -04001180 btrfs_free_tree_block(trans, root, buf, parent_start,
Jan Schmidt5581a512012-05-16 17:04:52 +02001181 last_ref);
Chris Mason5f39d392007-10-15 16:14:19 -04001182 free_extent_buffer(buf);
Chris Mason0b86a832008-03-24 15:01:56 -04001183 add_root_to_dirty_list(root);
Chris Mason6702ed42007-08-07 16:15:09 -04001184 } else {
Yan Zheng5d4f98a2009-06-10 10:45:14 -04001185 if (root->root_key.objectid == BTRFS_TREE_RELOC_OBJECTID)
1186 parent_start = parent->start;
1187 else
1188 parent_start = 0;
1189
1190 WARN_ON(trans->transid != btrfs_header_generation(parent));
Jan Schmidtf2304752012-05-26 11:43:17 +02001191 tree_mod_log_insert_key(root->fs_info, parent, parent_slot,
Josef Bacikc8cc6342013-07-01 16:18:19 -04001192 MOD_LOG_KEY_REPLACE, GFP_NOFS);
Chris Mason5f39d392007-10-15 16:14:19 -04001193 btrfs_set_node_blockptr(parent, parent_slot,
Chris Masondb945352007-10-15 16:15:53 -04001194 cow->start);
Chris Mason74493f72007-12-11 09:25:06 -05001195 btrfs_set_node_ptr_generation(parent, parent_slot,
1196 trans->transid);
Chris Mason6702ed42007-08-07 16:15:09 -04001197 btrfs_mark_buffer_dirty(parent);
Filipe David Borba Manana5de865e2013-12-20 15:17:46 +00001198 if (last_ref) {
1199 ret = tree_mod_log_free_eb(root->fs_info, buf);
1200 if (ret) {
1201 btrfs_abort_transaction(trans, root, ret);
1202 return ret;
1203 }
1204 }
Yan, Zhengf0486c62010-05-16 10:46:25 -04001205 btrfs_free_tree_block(trans, root, buf, parent_start,
Jan Schmidt5581a512012-05-16 17:04:52 +02001206 last_ref);
Chris Mason6702ed42007-08-07 16:15:09 -04001207 }
Chris Mason925baed2008-06-25 16:01:30 -04001208 if (unlock_orig)
1209 btrfs_tree_unlock(buf);
Josef Bacik3083ee22012-03-09 16:01:49 -05001210 free_extent_buffer_stale(buf);
Chris Mason6702ed42007-08-07 16:15:09 -04001211 btrfs_mark_buffer_dirty(cow);
1212 *cow_ret = cow;
1213 return 0;
1214}
1215
Jan Schmidt5d9e75c42012-05-16 18:25:47 +02001216/*
1217 * returns the logical address of the oldest predecessor of the given root.
1218 * entries older than time_seq are ignored.
1219 */
1220static struct tree_mod_elem *
1221__tree_mod_log_oldest_root(struct btrfs_fs_info *fs_info,
Jan Schmidt30b04632013-04-13 13:19:54 +00001222 struct extent_buffer *eb_root, u64 time_seq)
Jan Schmidt5d9e75c42012-05-16 18:25:47 +02001223{
1224 struct tree_mod_elem *tm;
1225 struct tree_mod_elem *found = NULL;
Jan Schmidt30b04632013-04-13 13:19:54 +00001226 u64 root_logical = eb_root->start;
Jan Schmidt5d9e75c42012-05-16 18:25:47 +02001227 int looped = 0;
1228
1229 if (!time_seq)
Stefan Behrens35a36212013-08-14 18:12:25 +02001230 return NULL;
Jan Schmidt5d9e75c42012-05-16 18:25:47 +02001231
1232 /*
1233 * the very last operation that's logged for a root is the replacement
1234 * operation (if it is replaced at all). this has the index of the *new*
1235 * root, making it the very first operation that's logged for this root.
1236 */
1237 while (1) {
1238 tm = tree_mod_log_search_oldest(fs_info, root_logical,
1239 time_seq);
1240 if (!looped && !tm)
Stefan Behrens35a36212013-08-14 18:12:25 +02001241 return NULL;
Jan Schmidt5d9e75c42012-05-16 18:25:47 +02001242 /*
Jan Schmidt28da9fb2012-06-21 10:59:13 +02001243 * if there are no tree operation for the oldest root, we simply
1244 * return it. this should only happen if that (old) root is at
1245 * level 0.
Jan Schmidt5d9e75c42012-05-16 18:25:47 +02001246 */
Jan Schmidt28da9fb2012-06-21 10:59:13 +02001247 if (!tm)
1248 break;
Jan Schmidt5d9e75c42012-05-16 18:25:47 +02001249
Jan Schmidt28da9fb2012-06-21 10:59:13 +02001250 /*
1251 * if there's an operation that's not a root replacement, we
1252 * found the oldest version of our root. normally, we'll find a
1253 * MOD_LOG_KEY_REMOVE_WHILE_FREEING operation here.
1254 */
Jan Schmidt5d9e75c42012-05-16 18:25:47 +02001255 if (tm->op != MOD_LOG_ROOT_REPLACE)
1256 break;
1257
1258 found = tm;
1259 root_logical = tm->old_root.logical;
Jan Schmidt5d9e75c42012-05-16 18:25:47 +02001260 looped = 1;
1261 }
1262
Jan Schmidta95236d2012-06-05 16:41:24 +02001263 /* if there's no old root to return, return what we found instead */
1264 if (!found)
1265 found = tm;
1266
Jan Schmidt5d9e75c42012-05-16 18:25:47 +02001267 return found;
1268}
1269
1270/*
1271 * tm is a pointer to the first operation to rewind within eb. then, all
1272 * previous operations will be rewinded (until we reach something older than
1273 * time_seq).
1274 */
1275static void
Josef Bacikf1ca7e982013-06-29 23:15:19 -04001276__tree_mod_log_rewind(struct btrfs_fs_info *fs_info, struct extent_buffer *eb,
1277 u64 time_seq, struct tree_mod_elem *first_tm)
Jan Schmidt5d9e75c42012-05-16 18:25:47 +02001278{
1279 u32 n;
1280 struct rb_node *next;
1281 struct tree_mod_elem *tm = first_tm;
1282 unsigned long o_dst;
1283 unsigned long o_src;
1284 unsigned long p_size = sizeof(struct btrfs_key_ptr);
1285
1286 n = btrfs_header_nritems(eb);
Josef Bacikf1ca7e982013-06-29 23:15:19 -04001287 tree_mod_log_read_lock(fs_info);
Jan Schmidt097b8a72012-06-21 11:08:04 +02001288 while (tm && tm->seq >= time_seq) {
Jan Schmidt5d9e75c42012-05-16 18:25:47 +02001289 /*
1290 * all the operations are recorded with the operator used for
1291 * the modification. as we're going backwards, we do the
1292 * opposite of each operation here.
1293 */
1294 switch (tm->op) {
1295 case MOD_LOG_KEY_REMOVE_WHILE_FREEING:
1296 BUG_ON(tm->slot < n);
Eric Sandeen1c697d42013-01-31 00:54:56 +00001297 /* Fallthrough */
Liu Bo95c80bb2012-10-19 09:50:52 +00001298 case MOD_LOG_KEY_REMOVE_WHILE_MOVING:
Chris Mason4c3e6962012-12-18 15:43:18 -05001299 case MOD_LOG_KEY_REMOVE:
Jan Schmidt5d9e75c42012-05-16 18:25:47 +02001300 btrfs_set_node_key(eb, &tm->key, tm->slot);
1301 btrfs_set_node_blockptr(eb, tm->slot, tm->blockptr);
1302 btrfs_set_node_ptr_generation(eb, tm->slot,
1303 tm->generation);
Chris Mason4c3e6962012-12-18 15:43:18 -05001304 n++;
Jan Schmidt5d9e75c42012-05-16 18:25:47 +02001305 break;
1306 case MOD_LOG_KEY_REPLACE:
1307 BUG_ON(tm->slot >= n);
1308 btrfs_set_node_key(eb, &tm->key, tm->slot);
1309 btrfs_set_node_blockptr(eb, tm->slot, tm->blockptr);
1310 btrfs_set_node_ptr_generation(eb, tm->slot,
1311 tm->generation);
1312 break;
1313 case MOD_LOG_KEY_ADD:
Jan Schmidt19956c72012-06-22 14:52:13 +02001314 /* if a move operation is needed it's in the log */
Jan Schmidt5d9e75c42012-05-16 18:25:47 +02001315 n--;
1316 break;
1317 case MOD_LOG_MOVE_KEYS:
Jan Schmidtc3193102012-05-31 19:24:36 +02001318 o_dst = btrfs_node_key_ptr_offset(tm->slot);
1319 o_src = btrfs_node_key_ptr_offset(tm->move.dst_slot);
1320 memmove_extent_buffer(eb, o_dst, o_src,
Jan Schmidt5d9e75c42012-05-16 18:25:47 +02001321 tm->move.nr_items * p_size);
1322 break;
1323 case MOD_LOG_ROOT_REPLACE:
1324 /*
1325 * this operation is special. for roots, this must be
1326 * handled explicitly before rewinding.
1327 * for non-roots, this operation may exist if the node
1328 * was a root: root A -> child B; then A gets empty and
1329 * B is promoted to the new root. in the mod log, we'll
1330 * have a root-replace operation for B, a tree block
1331 * that is no root. we simply ignore that operation.
1332 */
1333 break;
1334 }
1335 next = rb_next(&tm->node);
1336 if (!next)
1337 break;
1338 tm = container_of(next, struct tree_mod_elem, node);
1339 if (tm->index != first_tm->index)
1340 break;
1341 }
Josef Bacikf1ca7e982013-06-29 23:15:19 -04001342 tree_mod_log_read_unlock(fs_info);
Jan Schmidt5d9e75c42012-05-16 18:25:47 +02001343 btrfs_set_header_nritems(eb, n);
1344}
1345
Jan Schmidt47fb0912013-04-13 13:19:55 +00001346/*
1347 * Called with eb read locked. If the buffer cannot be rewinded, the same buffer
1348 * is returned. If rewind operations happen, a fresh buffer is returned. The
1349 * returned buffer is always read-locked. If the returned buffer is not the
1350 * input buffer, the lock on the input buffer is released and the input buffer
1351 * is freed (its refcount is decremented).
1352 */
Jan Schmidt5d9e75c42012-05-16 18:25:47 +02001353static struct extent_buffer *
Josef Bacik9ec72672013-08-07 16:57:23 -04001354tree_mod_log_rewind(struct btrfs_fs_info *fs_info, struct btrfs_path *path,
1355 struct extent_buffer *eb, u64 time_seq)
Jan Schmidt5d9e75c42012-05-16 18:25:47 +02001356{
1357 struct extent_buffer *eb_rewin;
1358 struct tree_mod_elem *tm;
1359
1360 if (!time_seq)
1361 return eb;
1362
1363 if (btrfs_header_level(eb) == 0)
1364 return eb;
1365
1366 tm = tree_mod_log_search(fs_info, eb->start, time_seq);
1367 if (!tm)
1368 return eb;
1369
Josef Bacik9ec72672013-08-07 16:57:23 -04001370 btrfs_set_path_blocking(path);
1371 btrfs_set_lock_blocking_rw(eb, BTRFS_READ_LOCK);
1372
Jan Schmidt5d9e75c42012-05-16 18:25:47 +02001373 if (tm->op == MOD_LOG_KEY_REMOVE_WHILE_FREEING) {
1374 BUG_ON(tm->slot != 0);
David Sterba3f556f72014-06-15 03:20:26 +02001375 eb_rewin = alloc_dummy_extent_buffer(fs_info, eb->start);
Josef Bacikdb7f3432013-08-07 14:54:37 -04001376 if (!eb_rewin) {
Josef Bacik9ec72672013-08-07 16:57:23 -04001377 btrfs_tree_read_unlock_blocking(eb);
Josef Bacikdb7f3432013-08-07 14:54:37 -04001378 free_extent_buffer(eb);
1379 return NULL;
1380 }
Jan Schmidt5d9e75c42012-05-16 18:25:47 +02001381 btrfs_set_header_bytenr(eb_rewin, eb->start);
1382 btrfs_set_header_backref_rev(eb_rewin,
1383 btrfs_header_backref_rev(eb));
1384 btrfs_set_header_owner(eb_rewin, btrfs_header_owner(eb));
Jan Schmidtc3193102012-05-31 19:24:36 +02001385 btrfs_set_header_level(eb_rewin, btrfs_header_level(eb));
Jan Schmidt5d9e75c42012-05-16 18:25:47 +02001386 } else {
1387 eb_rewin = btrfs_clone_extent_buffer(eb);
Josef Bacikdb7f3432013-08-07 14:54:37 -04001388 if (!eb_rewin) {
Josef Bacik9ec72672013-08-07 16:57:23 -04001389 btrfs_tree_read_unlock_blocking(eb);
Josef Bacikdb7f3432013-08-07 14:54:37 -04001390 free_extent_buffer(eb);
1391 return NULL;
1392 }
Jan Schmidt5d9e75c42012-05-16 18:25:47 +02001393 }
1394
Josef Bacik9ec72672013-08-07 16:57:23 -04001395 btrfs_clear_path_blocking(path, NULL, BTRFS_READ_LOCK);
1396 btrfs_tree_read_unlock_blocking(eb);
Jan Schmidt5d9e75c42012-05-16 18:25:47 +02001397 free_extent_buffer(eb);
1398
Jan Schmidt47fb0912013-04-13 13:19:55 +00001399 extent_buffer_get(eb_rewin);
1400 btrfs_tree_read_lock(eb_rewin);
Josef Bacikf1ca7e982013-06-29 23:15:19 -04001401 __tree_mod_log_rewind(fs_info, eb_rewin, time_seq, tm);
Jan Schmidt57911b82012-10-19 09:22:03 +02001402 WARN_ON(btrfs_header_nritems(eb_rewin) >
Arne Jansen2a745b12013-02-13 04:20:01 -07001403 BTRFS_NODEPTRS_PER_BLOCK(fs_info->tree_root));
Jan Schmidt5d9e75c42012-05-16 18:25:47 +02001404
1405 return eb_rewin;
1406}
1407
Jan Schmidt8ba97a12012-06-04 16:54:57 +02001408/*
1409 * get_old_root() rewinds the state of @root's root node to the given @time_seq
1410 * value. If there are no changes, the current root->root_node is returned. If
1411 * anything changed in between, there's a fresh buffer allocated on which the
1412 * rewind operations are done. In any case, the returned buffer is read locked.
1413 * Returns NULL on error (with no locks held).
1414 */
Jan Schmidt5d9e75c42012-05-16 18:25:47 +02001415static inline struct extent_buffer *
1416get_old_root(struct btrfs_root *root, u64 time_seq)
1417{
1418 struct tree_mod_elem *tm;
Jan Schmidt30b04632013-04-13 13:19:54 +00001419 struct extent_buffer *eb = NULL;
1420 struct extent_buffer *eb_root;
Liu Bo7bfdcf72012-10-25 07:30:19 -06001421 struct extent_buffer *old;
Jan Schmidta95236d2012-06-05 16:41:24 +02001422 struct tree_mod_root *old_root = NULL;
Chris Mason4325edd2012-06-15 20:02:02 -04001423 u64 old_generation = 0;
Jan Schmidta95236d2012-06-05 16:41:24 +02001424 u64 logical;
Jan Schmidt5d9e75c42012-05-16 18:25:47 +02001425
Jan Schmidt30b04632013-04-13 13:19:54 +00001426 eb_root = btrfs_read_lock_root_node(root);
1427 tm = __tree_mod_log_oldest_root(root->fs_info, eb_root, time_seq);
Jan Schmidt5d9e75c42012-05-16 18:25:47 +02001428 if (!tm)
Jan Schmidt30b04632013-04-13 13:19:54 +00001429 return eb_root;
Jan Schmidt5d9e75c42012-05-16 18:25:47 +02001430
Jan Schmidta95236d2012-06-05 16:41:24 +02001431 if (tm->op == MOD_LOG_ROOT_REPLACE) {
1432 old_root = &tm->old_root;
1433 old_generation = tm->generation;
1434 logical = old_root->logical;
1435 } else {
Jan Schmidt30b04632013-04-13 13:19:54 +00001436 logical = eb_root->start;
Jan Schmidta95236d2012-06-05 16:41:24 +02001437 }
Jan Schmidt5d9e75c42012-05-16 18:25:47 +02001438
Jan Schmidta95236d2012-06-05 16:41:24 +02001439 tm = tree_mod_log_search(root->fs_info, logical, time_seq);
Jan Schmidt834328a2012-10-23 11:27:33 +02001440 if (old_root && tm && tm->op != MOD_LOG_KEY_REMOVE_WHILE_FREEING) {
Jan Schmidt30b04632013-04-13 13:19:54 +00001441 btrfs_tree_read_unlock(eb_root);
1442 free_extent_buffer(eb_root);
David Sterbace86cd52014-06-15 01:07:32 +02001443 old = read_tree_block(root, logical, 0);
Liu Bo64c043d2015-05-25 17:30:15 +08001444 if (WARN_ON(IS_ERR(old) || !extent_buffer_uptodate(old))) {
1445 if (!IS_ERR(old))
1446 free_extent_buffer(old);
Frank Holtonefe120a2013-12-20 11:37:06 -05001447 btrfs_warn(root->fs_info,
1448 "failed to read tree block %llu from get_old_root", logical);
Jan Schmidt834328a2012-10-23 11:27:33 +02001449 } else {
Liu Bo7bfdcf72012-10-25 07:30:19 -06001450 eb = btrfs_clone_extent_buffer(old);
1451 free_extent_buffer(old);
Jan Schmidt834328a2012-10-23 11:27:33 +02001452 }
1453 } else if (old_root) {
Jan Schmidt30b04632013-04-13 13:19:54 +00001454 btrfs_tree_read_unlock(eb_root);
1455 free_extent_buffer(eb_root);
David Sterba3f556f72014-06-15 03:20:26 +02001456 eb = alloc_dummy_extent_buffer(root->fs_info, logical);
Jan Schmidt834328a2012-10-23 11:27:33 +02001457 } else {
Josef Bacik9ec72672013-08-07 16:57:23 -04001458 btrfs_set_lock_blocking_rw(eb_root, BTRFS_READ_LOCK);
Jan Schmidt30b04632013-04-13 13:19:54 +00001459 eb = btrfs_clone_extent_buffer(eb_root);
Josef Bacik9ec72672013-08-07 16:57:23 -04001460 btrfs_tree_read_unlock_blocking(eb_root);
Jan Schmidt30b04632013-04-13 13:19:54 +00001461 free_extent_buffer(eb_root);
Jan Schmidt834328a2012-10-23 11:27:33 +02001462 }
1463
Jan Schmidt8ba97a12012-06-04 16:54:57 +02001464 if (!eb)
1465 return NULL;
Jan Schmidtd6381082012-10-23 14:21:05 +02001466 extent_buffer_get(eb);
Jan Schmidt8ba97a12012-06-04 16:54:57 +02001467 btrfs_tree_read_lock(eb);
Jan Schmidta95236d2012-06-05 16:41:24 +02001468 if (old_root) {
Jan Schmidt5d9e75c42012-05-16 18:25:47 +02001469 btrfs_set_header_bytenr(eb, eb->start);
1470 btrfs_set_header_backref_rev(eb, BTRFS_MIXED_BACKREF_REV);
Jan Schmidt30b04632013-04-13 13:19:54 +00001471 btrfs_set_header_owner(eb, btrfs_header_owner(eb_root));
Jan Schmidta95236d2012-06-05 16:41:24 +02001472 btrfs_set_header_level(eb, old_root->level);
1473 btrfs_set_header_generation(eb, old_generation);
Jan Schmidt5d9e75c42012-05-16 18:25:47 +02001474 }
Jan Schmidt28da9fb2012-06-21 10:59:13 +02001475 if (tm)
Josef Bacikf1ca7e982013-06-29 23:15:19 -04001476 __tree_mod_log_rewind(root->fs_info, eb, time_seq, tm);
Jan Schmidt28da9fb2012-06-21 10:59:13 +02001477 else
1478 WARN_ON(btrfs_header_level(eb) != 0);
Jan Schmidt57911b82012-10-19 09:22:03 +02001479 WARN_ON(btrfs_header_nritems(eb) > BTRFS_NODEPTRS_PER_BLOCK(root));
Jan Schmidt5d9e75c42012-05-16 18:25:47 +02001480
1481 return eb;
1482}
1483
Jan Schmidt5b6602e2012-10-23 11:28:27 +02001484int btrfs_old_root_level(struct btrfs_root *root, u64 time_seq)
1485{
1486 struct tree_mod_elem *tm;
1487 int level;
Jan Schmidt30b04632013-04-13 13:19:54 +00001488 struct extent_buffer *eb_root = btrfs_root_node(root);
Jan Schmidt5b6602e2012-10-23 11:28:27 +02001489
Jan Schmidt30b04632013-04-13 13:19:54 +00001490 tm = __tree_mod_log_oldest_root(root->fs_info, eb_root, time_seq);
Jan Schmidt5b6602e2012-10-23 11:28:27 +02001491 if (tm && tm->op == MOD_LOG_ROOT_REPLACE) {
1492 level = tm->old_root.level;
1493 } else {
Jan Schmidt30b04632013-04-13 13:19:54 +00001494 level = btrfs_header_level(eb_root);
Jan Schmidt5b6602e2012-10-23 11:28:27 +02001495 }
Jan Schmidt30b04632013-04-13 13:19:54 +00001496 free_extent_buffer(eb_root);
Jan Schmidt5b6602e2012-10-23 11:28:27 +02001497
1498 return level;
1499}
1500
Yan Zheng5d4f98a2009-06-10 10:45:14 -04001501static inline int should_cow_block(struct btrfs_trans_handle *trans,
1502 struct btrfs_root *root,
1503 struct extent_buffer *buf)
1504{
David Sterbafccb84c2014-09-29 23:53:21 +02001505 if (btrfs_test_is_dummy_root(root))
Josef Bacikfaa2dbf2014-05-07 17:06:09 -04001506 return 0;
David Sterbafccb84c2014-09-29 23:53:21 +02001507
Liu Bof1ebcc72011-11-14 20:48:06 -05001508 /* ensure we can see the force_cow */
1509 smp_rmb();
1510
1511 /*
1512 * We do not need to cow a block if
1513 * 1) this block is not created or changed in this transaction;
1514 * 2) this block does not belong to TREE_RELOC tree;
1515 * 3) the root is not forced COW.
1516 *
1517 * What is forced COW:
1518 * when we create snapshot during commiting the transaction,
1519 * after we've finished coping src root, we must COW the shared
1520 * block to ensure the metadata consistency.
1521 */
Yan Zheng5d4f98a2009-06-10 10:45:14 -04001522 if (btrfs_header_generation(buf) == trans->transid &&
1523 !btrfs_header_flag(buf, BTRFS_HEADER_FLAG_WRITTEN) &&
1524 !(root->root_key.objectid != BTRFS_TREE_RELOC_OBJECTID &&
Liu Bof1ebcc72011-11-14 20:48:06 -05001525 btrfs_header_flag(buf, BTRFS_HEADER_FLAG_RELOC)) &&
Miao Xie27cdeb72014-04-02 19:51:05 +08001526 !test_bit(BTRFS_ROOT_FORCE_COW, &root->state))
Yan Zheng5d4f98a2009-06-10 10:45:14 -04001527 return 0;
1528 return 1;
1529}
1530
Chris Masond352ac62008-09-29 15:18:18 -04001531/*
1532 * cows a single block, see __btrfs_cow_block for the real work.
1533 * This version of it has extra checks so that a block isn't cow'd more than
1534 * once per transaction, as long as it hasn't been written yet
1535 */
Chris Masond3977122009-01-05 21:25:51 -05001536noinline int btrfs_cow_block(struct btrfs_trans_handle *trans,
Chris Mason5f39d392007-10-15 16:14:19 -04001537 struct btrfs_root *root, struct extent_buffer *buf,
1538 struct extent_buffer *parent, int parent_slot,
Chris Mason9fa8cfe2009-03-13 10:24:59 -04001539 struct extent_buffer **cow_ret)
Chris Mason02217ed2007-03-02 16:08:05 -05001540{
Chris Mason6702ed42007-08-07 16:15:09 -04001541 u64 search_start;
Chris Masonf510cfe2007-10-15 16:14:48 -04001542 int ret;
Chris Masondc17ff82008-01-08 15:46:30 -05001543
Julia Lawall31b1a2b2012-11-03 10:58:34 +00001544 if (trans->transaction != root->fs_info->running_transaction)
1545 WARN(1, KERN_CRIT "trans %llu running %llu\n",
Geert Uytterhoevenc1c9ff72013-08-20 13:20:07 +02001546 trans->transid,
Chris Masonccd467d2007-06-28 15:57:36 -04001547 root->fs_info->running_transaction->transid);
Julia Lawall31b1a2b2012-11-03 10:58:34 +00001548
1549 if (trans->transid != root->fs_info->generation)
1550 WARN(1, KERN_CRIT "trans %llu running %llu\n",
Geert Uytterhoevenc1c9ff72013-08-20 13:20:07 +02001551 trans->transid, root->fs_info->generation);
Chris Masondc17ff82008-01-08 15:46:30 -05001552
Yan Zheng5d4f98a2009-06-10 10:45:14 -04001553 if (!should_cow_block(trans, root, buf)) {
Chris Mason02217ed2007-03-02 16:08:05 -05001554 *cow_ret = buf;
1555 return 0;
1556 }
Chris Masonc4876852009-02-04 09:24:25 -05001557
Chris Mason0b86a832008-03-24 15:01:56 -04001558 search_start = buf->start & ~((u64)(1024 * 1024 * 1024) - 1);
Chris Masonb4ce94d2009-02-04 09:25:08 -05001559
1560 if (parent)
1561 btrfs_set_lock_blocking(parent);
1562 btrfs_set_lock_blocking(buf);
1563
Chris Masonf510cfe2007-10-15 16:14:48 -04001564 ret = __btrfs_cow_block(trans, root, buf, parent,
Chris Mason9fa8cfe2009-03-13 10:24:59 -04001565 parent_slot, cow_ret, search_start, 0);
liubo1abe9b82011-03-24 11:18:59 +00001566
1567 trace_btrfs_cow_block(root, buf, *cow_ret);
1568
Chris Masonf510cfe2007-10-15 16:14:48 -04001569 return ret;
Chris Mason6702ed42007-08-07 16:15:09 -04001570}
1571
Chris Masond352ac62008-09-29 15:18:18 -04001572/*
1573 * helper function for defrag to decide if two blocks pointed to by a
1574 * node are actually close by
1575 */
Chris Mason6b800532007-10-15 16:17:34 -04001576static int close_blocks(u64 blocknr, u64 other, u32 blocksize)
Chris Mason6702ed42007-08-07 16:15:09 -04001577{
Chris Mason6b800532007-10-15 16:17:34 -04001578 if (blocknr < other && other - (blocknr + blocksize) < 32768)
Chris Mason6702ed42007-08-07 16:15:09 -04001579 return 1;
Chris Mason6b800532007-10-15 16:17:34 -04001580 if (blocknr > other && blocknr - (other + blocksize) < 32768)
Chris Mason6702ed42007-08-07 16:15:09 -04001581 return 1;
Chris Mason02217ed2007-03-02 16:08:05 -05001582 return 0;
1583}
1584
Chris Mason081e9572007-11-06 10:26:24 -05001585/*
1586 * compare two keys in a memcmp fashion
1587 */
1588static int comp_keys(struct btrfs_disk_key *disk, struct btrfs_key *k2)
1589{
1590 struct btrfs_key k1;
1591
1592 btrfs_disk_key_to_cpu(&k1, disk);
1593
Diego Calleja20736ab2009-07-24 11:06:52 -04001594 return btrfs_comp_cpu_keys(&k1, k2);
Chris Mason081e9572007-11-06 10:26:24 -05001595}
1596
Josef Bacikf3465ca2008-11-12 14:19:50 -05001597/*
1598 * same as comp_keys only with two btrfs_key's
1599 */
Yan Zheng5d4f98a2009-06-10 10:45:14 -04001600int btrfs_comp_cpu_keys(struct btrfs_key *k1, struct btrfs_key *k2)
Josef Bacikf3465ca2008-11-12 14:19:50 -05001601{
1602 if (k1->objectid > k2->objectid)
1603 return 1;
1604 if (k1->objectid < k2->objectid)
1605 return -1;
1606 if (k1->type > k2->type)
1607 return 1;
1608 if (k1->type < k2->type)
1609 return -1;
1610 if (k1->offset > k2->offset)
1611 return 1;
1612 if (k1->offset < k2->offset)
1613 return -1;
1614 return 0;
1615}
Chris Mason081e9572007-11-06 10:26:24 -05001616
Chris Masond352ac62008-09-29 15:18:18 -04001617/*
1618 * this is used by the defrag code to go through all the
1619 * leaves pointed to by a node and reallocate them so that
1620 * disk order is close to key order
1621 */
Chris Mason6702ed42007-08-07 16:15:09 -04001622int btrfs_realloc_node(struct btrfs_trans_handle *trans,
Chris Mason5f39d392007-10-15 16:14:19 -04001623 struct btrfs_root *root, struct extent_buffer *parent,
Eric Sandeende78b512013-01-31 18:21:12 +00001624 int start_slot, u64 *last_ret,
Chris Masona6b6e752007-10-15 16:22:39 -04001625 struct btrfs_key *progress)
Chris Mason6702ed42007-08-07 16:15:09 -04001626{
Chris Mason6b800532007-10-15 16:17:34 -04001627 struct extent_buffer *cur;
Chris Mason6702ed42007-08-07 16:15:09 -04001628 u64 blocknr;
Chris Masonca7a79a2008-05-12 12:59:19 -04001629 u64 gen;
Chris Masone9d0b132007-08-10 14:06:19 -04001630 u64 search_start = *last_ret;
1631 u64 last_block = 0;
Chris Mason6702ed42007-08-07 16:15:09 -04001632 u64 other;
1633 u32 parent_nritems;
Chris Mason6702ed42007-08-07 16:15:09 -04001634 int end_slot;
1635 int i;
1636 int err = 0;
Chris Masonf2183bd2007-08-10 14:42:37 -04001637 int parent_level;
Chris Mason6b800532007-10-15 16:17:34 -04001638 int uptodate;
1639 u32 blocksize;
Chris Mason081e9572007-11-06 10:26:24 -05001640 int progress_passed = 0;
1641 struct btrfs_disk_key disk_key;
Chris Mason6702ed42007-08-07 16:15:09 -04001642
Chris Mason5708b952007-10-25 15:43:18 -04001643 parent_level = btrfs_header_level(parent);
Chris Mason5708b952007-10-25 15:43:18 -04001644
Julia Lawall6c1500f2012-11-03 20:30:18 +00001645 WARN_ON(trans->transaction != root->fs_info->running_transaction);
1646 WARN_ON(trans->transid != root->fs_info->generation);
Chris Mason86479a02007-09-10 19:58:16 -04001647
Chris Mason6b800532007-10-15 16:17:34 -04001648 parent_nritems = btrfs_header_nritems(parent);
David Sterba707e8a02014-06-04 19:22:26 +02001649 blocksize = root->nodesize;
Filipe Manana5dfe2be2015-02-23 19:48:52 +00001650 end_slot = parent_nritems - 1;
Chris Mason6702ed42007-08-07 16:15:09 -04001651
Filipe Manana5dfe2be2015-02-23 19:48:52 +00001652 if (parent_nritems <= 1)
Chris Mason6702ed42007-08-07 16:15:09 -04001653 return 0;
1654
Chris Masonb4ce94d2009-02-04 09:25:08 -05001655 btrfs_set_lock_blocking(parent);
1656
Filipe Manana5dfe2be2015-02-23 19:48:52 +00001657 for (i = start_slot; i <= end_slot; i++) {
Chris Mason6702ed42007-08-07 16:15:09 -04001658 int close = 1;
Chris Masona6b6e752007-10-15 16:22:39 -04001659
Chris Mason081e9572007-11-06 10:26:24 -05001660 btrfs_node_key(parent, &disk_key, i);
1661 if (!progress_passed && comp_keys(&disk_key, progress) < 0)
1662 continue;
1663
1664 progress_passed = 1;
Chris Mason6b800532007-10-15 16:17:34 -04001665 blocknr = btrfs_node_blockptr(parent, i);
Chris Masonca7a79a2008-05-12 12:59:19 -04001666 gen = btrfs_node_ptr_generation(parent, i);
Chris Masone9d0b132007-08-10 14:06:19 -04001667 if (last_block == 0)
1668 last_block = blocknr;
Chris Mason5708b952007-10-25 15:43:18 -04001669
Chris Mason6702ed42007-08-07 16:15:09 -04001670 if (i > 0) {
Chris Mason6b800532007-10-15 16:17:34 -04001671 other = btrfs_node_blockptr(parent, i - 1);
1672 close = close_blocks(blocknr, other, blocksize);
Chris Mason6702ed42007-08-07 16:15:09 -04001673 }
Filipe Manana5dfe2be2015-02-23 19:48:52 +00001674 if (!close && i < end_slot) {
Chris Mason6b800532007-10-15 16:17:34 -04001675 other = btrfs_node_blockptr(parent, i + 1);
1676 close = close_blocks(blocknr, other, blocksize);
Chris Mason6702ed42007-08-07 16:15:09 -04001677 }
Chris Masone9d0b132007-08-10 14:06:19 -04001678 if (close) {
1679 last_block = blocknr;
Chris Mason6702ed42007-08-07 16:15:09 -04001680 continue;
Chris Masone9d0b132007-08-10 14:06:19 -04001681 }
Chris Mason6702ed42007-08-07 16:15:09 -04001682
Daniel Dressler01d58472014-11-21 17:15:07 +09001683 cur = btrfs_find_tree_block(root->fs_info, blocknr);
Chris Mason6b800532007-10-15 16:17:34 -04001684 if (cur)
Chris Masonb9fab912012-05-06 07:23:47 -04001685 uptodate = btrfs_buffer_uptodate(cur, gen, 0);
Chris Mason6b800532007-10-15 16:17:34 -04001686 else
1687 uptodate = 0;
Chris Mason5708b952007-10-25 15:43:18 -04001688 if (!cur || !uptodate) {
Chris Mason6b800532007-10-15 16:17:34 -04001689 if (!cur) {
David Sterbace86cd52014-06-15 01:07:32 +02001690 cur = read_tree_block(root, blocknr, gen);
Liu Bo64c043d2015-05-25 17:30:15 +08001691 if (IS_ERR(cur)) {
1692 return PTR_ERR(cur);
1693 } else if (!extent_buffer_uptodate(cur)) {
Josef Bacik416bc652013-04-23 14:17:42 -04001694 free_extent_buffer(cur);
Tsutomu Itoh97d9a8a2011-03-24 06:33:21 +00001695 return -EIO;
Josef Bacik416bc652013-04-23 14:17:42 -04001696 }
Chris Mason6b800532007-10-15 16:17:34 -04001697 } else if (!uptodate) {
Tsutomu Itoh018642a2012-05-29 18:10:13 +09001698 err = btrfs_read_buffer(cur, gen);
1699 if (err) {
1700 free_extent_buffer(cur);
1701 return err;
1702 }
Chris Masonf2183bd2007-08-10 14:42:37 -04001703 }
Chris Mason6702ed42007-08-07 16:15:09 -04001704 }
Chris Masone9d0b132007-08-10 14:06:19 -04001705 if (search_start == 0)
Chris Mason6b800532007-10-15 16:17:34 -04001706 search_start = last_block;
Chris Masone9d0b132007-08-10 14:06:19 -04001707
Chris Masone7a84562008-06-25 16:01:31 -04001708 btrfs_tree_lock(cur);
Chris Masonb4ce94d2009-02-04 09:25:08 -05001709 btrfs_set_lock_blocking(cur);
Chris Mason6b800532007-10-15 16:17:34 -04001710 err = __btrfs_cow_block(trans, root, cur, parent, i,
Chris Masone7a84562008-06-25 16:01:31 -04001711 &cur, search_start,
Chris Mason6b800532007-10-15 16:17:34 -04001712 min(16 * blocksize,
Chris Mason9fa8cfe2009-03-13 10:24:59 -04001713 (end_slot - i) * blocksize));
Yan252c38f2007-08-29 09:11:44 -04001714 if (err) {
Chris Masone7a84562008-06-25 16:01:31 -04001715 btrfs_tree_unlock(cur);
Chris Mason6b800532007-10-15 16:17:34 -04001716 free_extent_buffer(cur);
Chris Mason6702ed42007-08-07 16:15:09 -04001717 break;
Yan252c38f2007-08-29 09:11:44 -04001718 }
Chris Masone7a84562008-06-25 16:01:31 -04001719 search_start = cur->start;
1720 last_block = cur->start;
Chris Masonf2183bd2007-08-10 14:42:37 -04001721 *last_ret = search_start;
Chris Masone7a84562008-06-25 16:01:31 -04001722 btrfs_tree_unlock(cur);
1723 free_extent_buffer(cur);
Chris Mason6702ed42007-08-07 16:15:09 -04001724 }
1725 return err;
1726}
1727
Chris Mason74123bd2007-02-02 11:05:29 -05001728/*
1729 * The leaf data grows from end-to-front in the node.
1730 * this returns the address of the start of the last item,
1731 * which is the stop of the leaf data stack
1732 */
Chris Mason123abc82007-03-14 14:14:43 -04001733static inline unsigned int leaf_data_end(struct btrfs_root *root,
Chris Mason5f39d392007-10-15 16:14:19 -04001734 struct extent_buffer *leaf)
Chris Masonbe0e5c02007-01-26 15:51:26 -05001735{
Chris Mason5f39d392007-10-15 16:14:19 -04001736 u32 nr = btrfs_header_nritems(leaf);
Chris Masonbe0e5c02007-01-26 15:51:26 -05001737 if (nr == 0)
Chris Mason123abc82007-03-14 14:14:43 -04001738 return BTRFS_LEAF_DATA_SIZE(root);
Chris Mason5f39d392007-10-15 16:14:19 -04001739 return btrfs_item_offset_nr(leaf, nr - 1);
Chris Masonbe0e5c02007-01-26 15:51:26 -05001740}
1741
Chris Masonaa5d6be2007-02-28 16:35:06 -05001742
Chris Mason74123bd2007-02-02 11:05:29 -05001743/*
Chris Mason5f39d392007-10-15 16:14:19 -04001744 * search for key in the extent_buffer. The items start at offset p,
1745 * and they are item_size apart. There are 'max' items in p.
1746 *
Chris Mason74123bd2007-02-02 11:05:29 -05001747 * the slot in the array is returned via slot, and it points to
1748 * the place where you would insert key if it is not found in
1749 * the array.
1750 *
1751 * slot may point to max if the key is bigger than all of the keys
1752 */
Chris Masone02119d2008-09-05 16:13:11 -04001753static noinline int generic_bin_search(struct extent_buffer *eb,
1754 unsigned long p,
1755 int item_size, struct btrfs_key *key,
1756 int max, int *slot)
Chris Masonbe0e5c02007-01-26 15:51:26 -05001757{
1758 int low = 0;
1759 int high = max;
1760 int mid;
1761 int ret;
Chris Mason479965d2007-10-15 16:14:27 -04001762 struct btrfs_disk_key *tmp = NULL;
Chris Mason5f39d392007-10-15 16:14:19 -04001763 struct btrfs_disk_key unaligned;
1764 unsigned long offset;
Chris Mason5f39d392007-10-15 16:14:19 -04001765 char *kaddr = NULL;
1766 unsigned long map_start = 0;
1767 unsigned long map_len = 0;
Chris Mason479965d2007-10-15 16:14:27 -04001768 int err;
Chris Masonbe0e5c02007-01-26 15:51:26 -05001769
Chris Masond3977122009-01-05 21:25:51 -05001770 while (low < high) {
Chris Masonbe0e5c02007-01-26 15:51:26 -05001771 mid = (low + high) / 2;
Chris Mason5f39d392007-10-15 16:14:19 -04001772 offset = p + mid * item_size;
1773
Chris Masona6591712011-07-19 12:04:14 -04001774 if (!kaddr || offset < map_start ||
Chris Mason5f39d392007-10-15 16:14:19 -04001775 (offset + sizeof(struct btrfs_disk_key)) >
1776 map_start + map_len) {
Chris Mason934d3752008-12-08 16:43:10 -05001777
1778 err = map_private_extent_buffer(eb, offset,
Chris Mason479965d2007-10-15 16:14:27 -04001779 sizeof(struct btrfs_disk_key),
Chris Masona6591712011-07-19 12:04:14 -04001780 &kaddr, &map_start, &map_len);
Chris Mason5f39d392007-10-15 16:14:19 -04001781
Chris Mason479965d2007-10-15 16:14:27 -04001782 if (!err) {
1783 tmp = (struct btrfs_disk_key *)(kaddr + offset -
1784 map_start);
1785 } else {
1786 read_extent_buffer(eb, &unaligned,
1787 offset, sizeof(unaligned));
1788 tmp = &unaligned;
1789 }
1790
Chris Mason5f39d392007-10-15 16:14:19 -04001791 } else {
1792 tmp = (struct btrfs_disk_key *)(kaddr + offset -
1793 map_start);
1794 }
Chris Masonbe0e5c02007-01-26 15:51:26 -05001795 ret = comp_keys(tmp, key);
1796
1797 if (ret < 0)
1798 low = mid + 1;
1799 else if (ret > 0)
1800 high = mid;
1801 else {
1802 *slot = mid;
1803 return 0;
1804 }
1805 }
1806 *slot = low;
1807 return 1;
1808}
1809
Chris Mason97571fd2007-02-24 13:39:08 -05001810/*
1811 * simple bin_search frontend that does the right thing for
1812 * leaves vs nodes
1813 */
Chris Mason5f39d392007-10-15 16:14:19 -04001814static int bin_search(struct extent_buffer *eb, struct btrfs_key *key,
1815 int level, int *slot)
Chris Masonbe0e5c02007-01-26 15:51:26 -05001816{
Wang Sheng-Huif7757382012-03-30 15:14:27 +08001817 if (level == 0)
Chris Mason5f39d392007-10-15 16:14:19 -04001818 return generic_bin_search(eb,
1819 offsetof(struct btrfs_leaf, items),
Chris Mason0783fcf2007-03-12 20:12:07 -04001820 sizeof(struct btrfs_item),
Chris Mason5f39d392007-10-15 16:14:19 -04001821 key, btrfs_header_nritems(eb),
Chris Mason7518a232007-03-12 12:01:18 -04001822 slot);
Wang Sheng-Huif7757382012-03-30 15:14:27 +08001823 else
Chris Mason5f39d392007-10-15 16:14:19 -04001824 return generic_bin_search(eb,
1825 offsetof(struct btrfs_node, ptrs),
Chris Mason123abc82007-03-14 14:14:43 -04001826 sizeof(struct btrfs_key_ptr),
Chris Mason5f39d392007-10-15 16:14:19 -04001827 key, btrfs_header_nritems(eb),
Chris Mason7518a232007-03-12 12:01:18 -04001828 slot);
Chris Masonbe0e5c02007-01-26 15:51:26 -05001829}
1830
Yan Zheng5d4f98a2009-06-10 10:45:14 -04001831int btrfs_bin_search(struct extent_buffer *eb, struct btrfs_key *key,
1832 int level, int *slot)
1833{
1834 return bin_search(eb, key, level, slot);
1835}
1836
Yan, Zhengf0486c62010-05-16 10:46:25 -04001837static void root_add_used(struct btrfs_root *root, u32 size)
1838{
1839 spin_lock(&root->accounting_lock);
1840 btrfs_set_root_used(&root->root_item,
1841 btrfs_root_used(&root->root_item) + size);
1842 spin_unlock(&root->accounting_lock);
1843}
1844
1845static void root_sub_used(struct btrfs_root *root, u32 size)
1846{
1847 spin_lock(&root->accounting_lock);
1848 btrfs_set_root_used(&root->root_item,
1849 btrfs_root_used(&root->root_item) - size);
1850 spin_unlock(&root->accounting_lock);
1851}
1852
Chris Masond352ac62008-09-29 15:18:18 -04001853/* given a node and slot number, this reads the blocks it points to. The
1854 * extent buffer is returned with a reference taken (but unlocked).
1855 * NULL is returned on error.
1856 */
Chris Masone02119d2008-09-05 16:13:11 -04001857static noinline struct extent_buffer *read_node_slot(struct btrfs_root *root,
Chris Mason5f39d392007-10-15 16:14:19 -04001858 struct extent_buffer *parent, int slot)
Chris Masonbb803952007-03-01 12:04:21 -05001859{
Chris Masonca7a79a2008-05-12 12:59:19 -04001860 int level = btrfs_header_level(parent);
Josef Bacik416bc652013-04-23 14:17:42 -04001861 struct extent_buffer *eb;
1862
Chris Masonbb803952007-03-01 12:04:21 -05001863 if (slot < 0)
1864 return NULL;
Chris Mason5f39d392007-10-15 16:14:19 -04001865 if (slot >= btrfs_header_nritems(parent))
Chris Masonbb803952007-03-01 12:04:21 -05001866 return NULL;
Chris Masonca7a79a2008-05-12 12:59:19 -04001867
1868 BUG_ON(level == 0);
1869
Josef Bacik416bc652013-04-23 14:17:42 -04001870 eb = read_tree_block(root, btrfs_node_blockptr(parent, slot),
Josef Bacik416bc652013-04-23 14:17:42 -04001871 btrfs_node_ptr_generation(parent, slot));
Liu Bo64c043d2015-05-25 17:30:15 +08001872 if (IS_ERR(eb) || !extent_buffer_uptodate(eb)) {
1873 if (!IS_ERR(eb))
1874 free_extent_buffer(eb);
Josef Bacik416bc652013-04-23 14:17:42 -04001875 eb = NULL;
1876 }
1877
1878 return eb;
Chris Masonbb803952007-03-01 12:04:21 -05001879}
1880
Chris Masond352ac62008-09-29 15:18:18 -04001881/*
1882 * node level balancing, used to make sure nodes are in proper order for
1883 * item deletion. We balance from the top down, so we have to make sure
1884 * that a deletion won't leave an node completely empty later on.
1885 */
Chris Masone02119d2008-09-05 16:13:11 -04001886static noinline int balance_level(struct btrfs_trans_handle *trans,
Chris Mason98ed5172008-01-03 10:01:48 -05001887 struct btrfs_root *root,
1888 struct btrfs_path *path, int level)
Chris Masonbb803952007-03-01 12:04:21 -05001889{
Chris Mason5f39d392007-10-15 16:14:19 -04001890 struct extent_buffer *right = NULL;
1891 struct extent_buffer *mid;
1892 struct extent_buffer *left = NULL;
1893 struct extent_buffer *parent = NULL;
Chris Masonbb803952007-03-01 12:04:21 -05001894 int ret = 0;
1895 int wret;
1896 int pslot;
Chris Masonbb803952007-03-01 12:04:21 -05001897 int orig_slot = path->slots[level];
Chris Mason79f95c82007-03-01 15:16:26 -05001898 u64 orig_ptr;
Chris Masonbb803952007-03-01 12:04:21 -05001899
1900 if (level == 0)
1901 return 0;
1902
Chris Mason5f39d392007-10-15 16:14:19 -04001903 mid = path->nodes[level];
Chris Masonb4ce94d2009-02-04 09:25:08 -05001904
Chris Masonbd681512011-07-16 15:23:14 -04001905 WARN_ON(path->locks[level] != BTRFS_WRITE_LOCK &&
1906 path->locks[level] != BTRFS_WRITE_LOCK_BLOCKING);
Chris Mason7bb86312007-12-11 09:25:06 -05001907 WARN_ON(btrfs_header_generation(mid) != trans->transid);
1908
Chris Mason1d4f8a02007-03-13 09:28:32 -04001909 orig_ptr = btrfs_node_blockptr(mid, orig_slot);
Chris Mason79f95c82007-03-01 15:16:26 -05001910
Li Zefana05a9bb2011-09-06 16:55:34 +08001911 if (level < BTRFS_MAX_LEVEL - 1) {
Chris Mason5f39d392007-10-15 16:14:19 -04001912 parent = path->nodes[level + 1];
Li Zefana05a9bb2011-09-06 16:55:34 +08001913 pslot = path->slots[level + 1];
1914 }
Chris Masonbb803952007-03-01 12:04:21 -05001915
Chris Mason40689472007-03-17 14:29:23 -04001916 /*
1917 * deal with the case where there is only one pointer in the root
1918 * by promoting the node below to a root
1919 */
Chris Mason5f39d392007-10-15 16:14:19 -04001920 if (!parent) {
1921 struct extent_buffer *child;
Chris Masonbb803952007-03-01 12:04:21 -05001922
Chris Mason5f39d392007-10-15 16:14:19 -04001923 if (btrfs_header_nritems(mid) != 1)
Chris Masonbb803952007-03-01 12:04:21 -05001924 return 0;
1925
1926 /* promote the child to a root */
Chris Mason5f39d392007-10-15 16:14:19 -04001927 child = read_node_slot(root, mid, 0);
Mark Fasheh305a26a2011-09-01 11:27:57 -07001928 if (!child) {
1929 ret = -EROFS;
Anand Jaina4553fe2015-09-25 14:43:01 +08001930 btrfs_std_error(root->fs_info, ret, NULL);
Mark Fasheh305a26a2011-09-01 11:27:57 -07001931 goto enospc;
1932 }
1933
Chris Mason925baed2008-06-25 16:01:30 -04001934 btrfs_tree_lock(child);
Chris Masonb4ce94d2009-02-04 09:25:08 -05001935 btrfs_set_lock_blocking(child);
Chris Mason9fa8cfe2009-03-13 10:24:59 -04001936 ret = btrfs_cow_block(trans, root, child, mid, 0, &child);
Yan, Zhengf0486c62010-05-16 10:46:25 -04001937 if (ret) {
1938 btrfs_tree_unlock(child);
1939 free_extent_buffer(child);
1940 goto enospc;
1941 }
Yan2f375ab2008-02-01 14:58:07 -05001942
Jan Schmidt90f8d622013-04-13 13:19:53 +00001943 tree_mod_log_set_root_pointer(root, child, 1);
Chris Mason240f62c2011-03-23 14:54:42 -04001944 rcu_assign_pointer(root->node, child);
Chris Mason925baed2008-06-25 16:01:30 -04001945
Chris Mason0b86a832008-03-24 15:01:56 -04001946 add_root_to_dirty_list(root);
Chris Mason925baed2008-06-25 16:01:30 -04001947 btrfs_tree_unlock(child);
Chris Masonb4ce94d2009-02-04 09:25:08 -05001948
Chris Mason925baed2008-06-25 16:01:30 -04001949 path->locks[level] = 0;
Chris Masonbb803952007-03-01 12:04:21 -05001950 path->nodes[level] = NULL;
Daniel Dressler01d58472014-11-21 17:15:07 +09001951 clean_tree_block(trans, root->fs_info, mid);
Chris Mason925baed2008-06-25 16:01:30 -04001952 btrfs_tree_unlock(mid);
Chris Masonbb803952007-03-01 12:04:21 -05001953 /* once for the path */
Chris Mason5f39d392007-10-15 16:14:19 -04001954 free_extent_buffer(mid);
Yan, Zhengf0486c62010-05-16 10:46:25 -04001955
1956 root_sub_used(root, mid->len);
Jan Schmidt5581a512012-05-16 17:04:52 +02001957 btrfs_free_tree_block(trans, root, mid, 0, 1);
Chris Masonbb803952007-03-01 12:04:21 -05001958 /* once for the root ptr */
Josef Bacik3083ee22012-03-09 16:01:49 -05001959 free_extent_buffer_stale(mid);
Yan, Zhengf0486c62010-05-16 10:46:25 -04001960 return 0;
Chris Masonbb803952007-03-01 12:04:21 -05001961 }
Chris Mason5f39d392007-10-15 16:14:19 -04001962 if (btrfs_header_nritems(mid) >
Chris Mason123abc82007-03-14 14:14:43 -04001963 BTRFS_NODEPTRS_PER_BLOCK(root) / 4)
Chris Masonbb803952007-03-01 12:04:21 -05001964 return 0;
1965
Chris Mason5f39d392007-10-15 16:14:19 -04001966 left = read_node_slot(root, parent, pslot - 1);
1967 if (left) {
Chris Mason925baed2008-06-25 16:01:30 -04001968 btrfs_tree_lock(left);
Chris Masonb4ce94d2009-02-04 09:25:08 -05001969 btrfs_set_lock_blocking(left);
Chris Mason5f39d392007-10-15 16:14:19 -04001970 wret = btrfs_cow_block(trans, root, left,
Chris Mason9fa8cfe2009-03-13 10:24:59 -04001971 parent, pslot - 1, &left);
Chris Mason54aa1f42007-06-22 14:16:25 -04001972 if (wret) {
1973 ret = wret;
1974 goto enospc;
1975 }
Chris Mason2cc58cf2007-08-27 16:49:44 -04001976 }
Chris Mason5f39d392007-10-15 16:14:19 -04001977 right = read_node_slot(root, parent, pslot + 1);
1978 if (right) {
Chris Mason925baed2008-06-25 16:01:30 -04001979 btrfs_tree_lock(right);
Chris Masonb4ce94d2009-02-04 09:25:08 -05001980 btrfs_set_lock_blocking(right);
Chris Mason5f39d392007-10-15 16:14:19 -04001981 wret = btrfs_cow_block(trans, root, right,
Chris Mason9fa8cfe2009-03-13 10:24:59 -04001982 parent, pslot + 1, &right);
Chris Mason2cc58cf2007-08-27 16:49:44 -04001983 if (wret) {
1984 ret = wret;
1985 goto enospc;
1986 }
1987 }
1988
1989 /* first, try to make some room in the middle buffer */
Chris Mason5f39d392007-10-15 16:14:19 -04001990 if (left) {
1991 orig_slot += btrfs_header_nritems(left);
Chris Masonbce4eae2008-04-24 14:42:46 -04001992 wret = push_node_left(trans, root, left, mid, 1);
Chris Mason79f95c82007-03-01 15:16:26 -05001993 if (wret < 0)
1994 ret = wret;
Chris Masonbb803952007-03-01 12:04:21 -05001995 }
Chris Mason79f95c82007-03-01 15:16:26 -05001996
1997 /*
1998 * then try to empty the right most buffer into the middle
1999 */
Chris Mason5f39d392007-10-15 16:14:19 -04002000 if (right) {
Chris Mason971a1f62008-04-24 10:54:32 -04002001 wret = push_node_left(trans, root, mid, right, 1);
Chris Mason54aa1f42007-06-22 14:16:25 -04002002 if (wret < 0 && wret != -ENOSPC)
Chris Mason79f95c82007-03-01 15:16:26 -05002003 ret = wret;
Chris Mason5f39d392007-10-15 16:14:19 -04002004 if (btrfs_header_nritems(right) == 0) {
Daniel Dressler01d58472014-11-21 17:15:07 +09002005 clean_tree_block(trans, root->fs_info, right);
Chris Mason925baed2008-06-25 16:01:30 -04002006 btrfs_tree_unlock(right);
Tsutomu Itohafe5fea2013-04-16 05:18:22 +00002007 del_ptr(root, path, level + 1, pslot + 1);
Yan, Zhengf0486c62010-05-16 10:46:25 -04002008 root_sub_used(root, right->len);
Jan Schmidt5581a512012-05-16 17:04:52 +02002009 btrfs_free_tree_block(trans, root, right, 0, 1);
Josef Bacik3083ee22012-03-09 16:01:49 -05002010 free_extent_buffer_stale(right);
Yan, Zhengf0486c62010-05-16 10:46:25 -04002011 right = NULL;
Chris Masonbb803952007-03-01 12:04:21 -05002012 } else {
Chris Mason5f39d392007-10-15 16:14:19 -04002013 struct btrfs_disk_key right_key;
2014 btrfs_node_key(right, &right_key, 0);
Jan Schmidtf2304752012-05-26 11:43:17 +02002015 tree_mod_log_set_node_key(root->fs_info, parent,
Liu Bo32adf092012-10-19 12:52:15 +00002016 pslot + 1, 0);
Chris Mason5f39d392007-10-15 16:14:19 -04002017 btrfs_set_node_key(parent, &right_key, pslot + 1);
2018 btrfs_mark_buffer_dirty(parent);
Chris Masonbb803952007-03-01 12:04:21 -05002019 }
2020 }
Chris Mason5f39d392007-10-15 16:14:19 -04002021 if (btrfs_header_nritems(mid) == 1) {
Chris Mason79f95c82007-03-01 15:16:26 -05002022 /*
2023 * we're not allowed to leave a node with one item in the
2024 * tree during a delete. A deletion from lower in the tree
2025 * could try to delete the only pointer in this node.
2026 * So, pull some keys from the left.
2027 * There has to be a left pointer at this point because
2028 * otherwise we would have pulled some pointers from the
2029 * right
2030 */
Mark Fasheh305a26a2011-09-01 11:27:57 -07002031 if (!left) {
2032 ret = -EROFS;
Anand Jaina4553fe2015-09-25 14:43:01 +08002033 btrfs_std_error(root->fs_info, ret, NULL);
Mark Fasheh305a26a2011-09-01 11:27:57 -07002034 goto enospc;
2035 }
Chris Mason5f39d392007-10-15 16:14:19 -04002036 wret = balance_node_right(trans, root, mid, left);
Chris Mason54aa1f42007-06-22 14:16:25 -04002037 if (wret < 0) {
Chris Mason79f95c82007-03-01 15:16:26 -05002038 ret = wret;
Chris Mason54aa1f42007-06-22 14:16:25 -04002039 goto enospc;
2040 }
Chris Masonbce4eae2008-04-24 14:42:46 -04002041 if (wret == 1) {
2042 wret = push_node_left(trans, root, left, mid, 1);
2043 if (wret < 0)
2044 ret = wret;
2045 }
Chris Mason79f95c82007-03-01 15:16:26 -05002046 BUG_ON(wret == 1);
2047 }
Chris Mason5f39d392007-10-15 16:14:19 -04002048 if (btrfs_header_nritems(mid) == 0) {
Daniel Dressler01d58472014-11-21 17:15:07 +09002049 clean_tree_block(trans, root->fs_info, mid);
Chris Mason925baed2008-06-25 16:01:30 -04002050 btrfs_tree_unlock(mid);
Tsutomu Itohafe5fea2013-04-16 05:18:22 +00002051 del_ptr(root, path, level + 1, pslot);
Yan, Zhengf0486c62010-05-16 10:46:25 -04002052 root_sub_used(root, mid->len);
Jan Schmidt5581a512012-05-16 17:04:52 +02002053 btrfs_free_tree_block(trans, root, mid, 0, 1);
Josef Bacik3083ee22012-03-09 16:01:49 -05002054 free_extent_buffer_stale(mid);
Yan, Zhengf0486c62010-05-16 10:46:25 -04002055 mid = NULL;
Chris Mason79f95c82007-03-01 15:16:26 -05002056 } else {
2057 /* update the parent key to reflect our changes */
Chris Mason5f39d392007-10-15 16:14:19 -04002058 struct btrfs_disk_key mid_key;
2059 btrfs_node_key(mid, &mid_key, 0);
Liu Bo32adf092012-10-19 12:52:15 +00002060 tree_mod_log_set_node_key(root->fs_info, parent,
Jan Schmidtf2304752012-05-26 11:43:17 +02002061 pslot, 0);
Chris Mason5f39d392007-10-15 16:14:19 -04002062 btrfs_set_node_key(parent, &mid_key, pslot);
2063 btrfs_mark_buffer_dirty(parent);
Chris Mason79f95c82007-03-01 15:16:26 -05002064 }
Chris Masonbb803952007-03-01 12:04:21 -05002065
Chris Mason79f95c82007-03-01 15:16:26 -05002066 /* update the path */
Chris Mason5f39d392007-10-15 16:14:19 -04002067 if (left) {
2068 if (btrfs_header_nritems(left) > orig_slot) {
2069 extent_buffer_get(left);
Chris Mason925baed2008-06-25 16:01:30 -04002070 /* left was locked after cow */
Chris Mason5f39d392007-10-15 16:14:19 -04002071 path->nodes[level] = left;
Chris Masonbb803952007-03-01 12:04:21 -05002072 path->slots[level + 1] -= 1;
2073 path->slots[level] = orig_slot;
Chris Mason925baed2008-06-25 16:01:30 -04002074 if (mid) {
2075 btrfs_tree_unlock(mid);
Chris Mason5f39d392007-10-15 16:14:19 -04002076 free_extent_buffer(mid);
Chris Mason925baed2008-06-25 16:01:30 -04002077 }
Chris Masonbb803952007-03-01 12:04:21 -05002078 } else {
Chris Mason5f39d392007-10-15 16:14:19 -04002079 orig_slot -= btrfs_header_nritems(left);
Chris Masonbb803952007-03-01 12:04:21 -05002080 path->slots[level] = orig_slot;
2081 }
2082 }
Chris Mason79f95c82007-03-01 15:16:26 -05002083 /* double check we haven't messed things up */
Chris Masone20d96d2007-03-22 12:13:20 -04002084 if (orig_ptr !=
Chris Mason5f39d392007-10-15 16:14:19 -04002085 btrfs_node_blockptr(path->nodes[level], path->slots[level]))
Chris Mason79f95c82007-03-01 15:16:26 -05002086 BUG();
Chris Mason54aa1f42007-06-22 14:16:25 -04002087enospc:
Chris Mason925baed2008-06-25 16:01:30 -04002088 if (right) {
2089 btrfs_tree_unlock(right);
Chris Mason5f39d392007-10-15 16:14:19 -04002090 free_extent_buffer(right);
Chris Mason925baed2008-06-25 16:01:30 -04002091 }
2092 if (left) {
2093 if (path->nodes[level] != left)
2094 btrfs_tree_unlock(left);
Chris Mason5f39d392007-10-15 16:14:19 -04002095 free_extent_buffer(left);
Chris Mason925baed2008-06-25 16:01:30 -04002096 }
Chris Masonbb803952007-03-01 12:04:21 -05002097 return ret;
2098}
2099
Chris Masond352ac62008-09-29 15:18:18 -04002100/* Node balancing for insertion. Here we only split or push nodes around
2101 * when they are completely full. This is also done top down, so we
2102 * have to be pessimistic.
2103 */
Chris Masond3977122009-01-05 21:25:51 -05002104static noinline int push_nodes_for_insert(struct btrfs_trans_handle *trans,
Chris Mason98ed5172008-01-03 10:01:48 -05002105 struct btrfs_root *root,
2106 struct btrfs_path *path, int level)
Chris Masone66f7092007-04-20 13:16:02 -04002107{
Chris Mason5f39d392007-10-15 16:14:19 -04002108 struct extent_buffer *right = NULL;
2109 struct extent_buffer *mid;
2110 struct extent_buffer *left = NULL;
2111 struct extent_buffer *parent = NULL;
Chris Masone66f7092007-04-20 13:16:02 -04002112 int ret = 0;
2113 int wret;
2114 int pslot;
2115 int orig_slot = path->slots[level];
Chris Masone66f7092007-04-20 13:16:02 -04002116
2117 if (level == 0)
2118 return 1;
2119
Chris Mason5f39d392007-10-15 16:14:19 -04002120 mid = path->nodes[level];
Chris Mason7bb86312007-12-11 09:25:06 -05002121 WARN_ON(btrfs_header_generation(mid) != trans->transid);
Chris Masone66f7092007-04-20 13:16:02 -04002122
Li Zefana05a9bb2011-09-06 16:55:34 +08002123 if (level < BTRFS_MAX_LEVEL - 1) {
Chris Mason5f39d392007-10-15 16:14:19 -04002124 parent = path->nodes[level + 1];
Li Zefana05a9bb2011-09-06 16:55:34 +08002125 pslot = path->slots[level + 1];
2126 }
Chris Masone66f7092007-04-20 13:16:02 -04002127
Chris Mason5f39d392007-10-15 16:14:19 -04002128 if (!parent)
Chris Masone66f7092007-04-20 13:16:02 -04002129 return 1;
Chris Masone66f7092007-04-20 13:16:02 -04002130
Chris Mason5f39d392007-10-15 16:14:19 -04002131 left = read_node_slot(root, parent, pslot - 1);
Chris Masone66f7092007-04-20 13:16:02 -04002132
2133 /* first, try to make some room in the middle buffer */
Chris Mason5f39d392007-10-15 16:14:19 -04002134 if (left) {
Chris Masone66f7092007-04-20 13:16:02 -04002135 u32 left_nr;
Chris Mason925baed2008-06-25 16:01:30 -04002136
2137 btrfs_tree_lock(left);
Chris Masonb4ce94d2009-02-04 09:25:08 -05002138 btrfs_set_lock_blocking(left);
2139
Chris Mason5f39d392007-10-15 16:14:19 -04002140 left_nr = btrfs_header_nritems(left);
Chris Mason33ade1f2007-04-20 13:48:57 -04002141 if (left_nr >= BTRFS_NODEPTRS_PER_BLOCK(root) - 1) {
2142 wret = 1;
2143 } else {
Chris Mason5f39d392007-10-15 16:14:19 -04002144 ret = btrfs_cow_block(trans, root, left, parent,
Chris Mason9fa8cfe2009-03-13 10:24:59 -04002145 pslot - 1, &left);
Chris Mason54aa1f42007-06-22 14:16:25 -04002146 if (ret)
2147 wret = 1;
2148 else {
Chris Mason54aa1f42007-06-22 14:16:25 -04002149 wret = push_node_left(trans, root,
Chris Mason971a1f62008-04-24 10:54:32 -04002150 left, mid, 0);
Chris Mason54aa1f42007-06-22 14:16:25 -04002151 }
Chris Mason33ade1f2007-04-20 13:48:57 -04002152 }
Chris Masone66f7092007-04-20 13:16:02 -04002153 if (wret < 0)
2154 ret = wret;
2155 if (wret == 0) {
Chris Mason5f39d392007-10-15 16:14:19 -04002156 struct btrfs_disk_key disk_key;
Chris Masone66f7092007-04-20 13:16:02 -04002157 orig_slot += left_nr;
Chris Mason5f39d392007-10-15 16:14:19 -04002158 btrfs_node_key(mid, &disk_key, 0);
Jan Schmidtf2304752012-05-26 11:43:17 +02002159 tree_mod_log_set_node_key(root->fs_info, parent,
Liu Bo32adf092012-10-19 12:52:15 +00002160 pslot, 0);
Chris Mason5f39d392007-10-15 16:14:19 -04002161 btrfs_set_node_key(parent, &disk_key, pslot);
2162 btrfs_mark_buffer_dirty(parent);
2163 if (btrfs_header_nritems(left) > orig_slot) {
2164 path->nodes[level] = left;
Chris Masone66f7092007-04-20 13:16:02 -04002165 path->slots[level + 1] -= 1;
2166 path->slots[level] = orig_slot;
Chris Mason925baed2008-06-25 16:01:30 -04002167 btrfs_tree_unlock(mid);
Chris Mason5f39d392007-10-15 16:14:19 -04002168 free_extent_buffer(mid);
Chris Masone66f7092007-04-20 13:16:02 -04002169 } else {
2170 orig_slot -=
Chris Mason5f39d392007-10-15 16:14:19 -04002171 btrfs_header_nritems(left);
Chris Masone66f7092007-04-20 13:16:02 -04002172 path->slots[level] = orig_slot;
Chris Mason925baed2008-06-25 16:01:30 -04002173 btrfs_tree_unlock(left);
Chris Mason5f39d392007-10-15 16:14:19 -04002174 free_extent_buffer(left);
Chris Masone66f7092007-04-20 13:16:02 -04002175 }
Chris Masone66f7092007-04-20 13:16:02 -04002176 return 0;
2177 }
Chris Mason925baed2008-06-25 16:01:30 -04002178 btrfs_tree_unlock(left);
Chris Mason5f39d392007-10-15 16:14:19 -04002179 free_extent_buffer(left);
Chris Masone66f7092007-04-20 13:16:02 -04002180 }
Chris Mason925baed2008-06-25 16:01:30 -04002181 right = read_node_slot(root, parent, pslot + 1);
Chris Masone66f7092007-04-20 13:16:02 -04002182
2183 /*
2184 * then try to empty the right most buffer into the middle
2185 */
Chris Mason5f39d392007-10-15 16:14:19 -04002186 if (right) {
Chris Mason33ade1f2007-04-20 13:48:57 -04002187 u32 right_nr;
Chris Masonb4ce94d2009-02-04 09:25:08 -05002188
Chris Mason925baed2008-06-25 16:01:30 -04002189 btrfs_tree_lock(right);
Chris Masonb4ce94d2009-02-04 09:25:08 -05002190 btrfs_set_lock_blocking(right);
2191
Chris Mason5f39d392007-10-15 16:14:19 -04002192 right_nr = btrfs_header_nritems(right);
Chris Mason33ade1f2007-04-20 13:48:57 -04002193 if (right_nr >= BTRFS_NODEPTRS_PER_BLOCK(root) - 1) {
2194 wret = 1;
2195 } else {
Chris Mason5f39d392007-10-15 16:14:19 -04002196 ret = btrfs_cow_block(trans, root, right,
2197 parent, pslot + 1,
Chris Mason9fa8cfe2009-03-13 10:24:59 -04002198 &right);
Chris Mason54aa1f42007-06-22 14:16:25 -04002199 if (ret)
2200 wret = 1;
2201 else {
Chris Mason54aa1f42007-06-22 14:16:25 -04002202 wret = balance_node_right(trans, root,
Chris Mason5f39d392007-10-15 16:14:19 -04002203 right, mid);
Chris Mason54aa1f42007-06-22 14:16:25 -04002204 }
Chris Mason33ade1f2007-04-20 13:48:57 -04002205 }
Chris Masone66f7092007-04-20 13:16:02 -04002206 if (wret < 0)
2207 ret = wret;
2208 if (wret == 0) {
Chris Mason5f39d392007-10-15 16:14:19 -04002209 struct btrfs_disk_key disk_key;
2210
2211 btrfs_node_key(right, &disk_key, 0);
Jan Schmidtf2304752012-05-26 11:43:17 +02002212 tree_mod_log_set_node_key(root->fs_info, parent,
Liu Bo32adf092012-10-19 12:52:15 +00002213 pslot + 1, 0);
Chris Mason5f39d392007-10-15 16:14:19 -04002214 btrfs_set_node_key(parent, &disk_key, pslot + 1);
2215 btrfs_mark_buffer_dirty(parent);
2216
2217 if (btrfs_header_nritems(mid) <= orig_slot) {
2218 path->nodes[level] = right;
Chris Masone66f7092007-04-20 13:16:02 -04002219 path->slots[level + 1] += 1;
2220 path->slots[level] = orig_slot -
Chris Mason5f39d392007-10-15 16:14:19 -04002221 btrfs_header_nritems(mid);
Chris Mason925baed2008-06-25 16:01:30 -04002222 btrfs_tree_unlock(mid);
Chris Mason5f39d392007-10-15 16:14:19 -04002223 free_extent_buffer(mid);
Chris Masone66f7092007-04-20 13:16:02 -04002224 } else {
Chris Mason925baed2008-06-25 16:01:30 -04002225 btrfs_tree_unlock(right);
Chris Mason5f39d392007-10-15 16:14:19 -04002226 free_extent_buffer(right);
Chris Masone66f7092007-04-20 13:16:02 -04002227 }
Chris Masone66f7092007-04-20 13:16:02 -04002228 return 0;
2229 }
Chris Mason925baed2008-06-25 16:01:30 -04002230 btrfs_tree_unlock(right);
Chris Mason5f39d392007-10-15 16:14:19 -04002231 free_extent_buffer(right);
Chris Masone66f7092007-04-20 13:16:02 -04002232 }
Chris Masone66f7092007-04-20 13:16:02 -04002233 return 1;
2234}
2235
Chris Mason74123bd2007-02-02 11:05:29 -05002236/*
Chris Masond352ac62008-09-29 15:18:18 -04002237 * readahead one full node of leaves, finding things that are close
2238 * to the block in 'slot', and triggering ra on them.
Chris Mason3c69fae2007-08-07 15:52:22 -04002239 */
Chris Masonc8c42862009-04-03 10:14:18 -04002240static void reada_for_search(struct btrfs_root *root,
2241 struct btrfs_path *path,
2242 int level, int slot, u64 objectid)
Chris Mason3c69fae2007-08-07 15:52:22 -04002243{
Chris Mason5f39d392007-10-15 16:14:19 -04002244 struct extent_buffer *node;
Chris Mason01f46652007-12-21 16:24:26 -05002245 struct btrfs_disk_key disk_key;
Chris Mason3c69fae2007-08-07 15:52:22 -04002246 u32 nritems;
Chris Mason3c69fae2007-08-07 15:52:22 -04002247 u64 search;
Chris Masona7175312009-01-22 09:23:10 -05002248 u64 target;
Chris Mason6b800532007-10-15 16:17:34 -04002249 u64 nread = 0;
Josef Bacikcb25c2e2011-05-11 12:17:34 -04002250 u64 gen;
Chris Mason3c69fae2007-08-07 15:52:22 -04002251 int direction = path->reada;
Chris Mason5f39d392007-10-15 16:14:19 -04002252 struct extent_buffer *eb;
Chris Mason6b800532007-10-15 16:17:34 -04002253 u32 nr;
2254 u32 blocksize;
2255 u32 nscan = 0;
Chris Masondb945352007-10-15 16:15:53 -04002256
Chris Masona6b6e752007-10-15 16:22:39 -04002257 if (level != 1)
Chris Mason3c69fae2007-08-07 15:52:22 -04002258 return;
2259
Chris Mason6702ed42007-08-07 16:15:09 -04002260 if (!path->nodes[level])
2261 return;
2262
Chris Mason5f39d392007-10-15 16:14:19 -04002263 node = path->nodes[level];
Chris Mason925baed2008-06-25 16:01:30 -04002264
Chris Mason3c69fae2007-08-07 15:52:22 -04002265 search = btrfs_node_blockptr(node, slot);
David Sterba707e8a02014-06-04 19:22:26 +02002266 blocksize = root->nodesize;
Daniel Dressler01d58472014-11-21 17:15:07 +09002267 eb = btrfs_find_tree_block(root->fs_info, search);
Chris Mason5f39d392007-10-15 16:14:19 -04002268 if (eb) {
2269 free_extent_buffer(eb);
Chris Mason3c69fae2007-08-07 15:52:22 -04002270 return;
2271 }
2272
Chris Masona7175312009-01-22 09:23:10 -05002273 target = search;
Chris Mason6b800532007-10-15 16:17:34 -04002274
Chris Mason5f39d392007-10-15 16:14:19 -04002275 nritems = btrfs_header_nritems(node);
Chris Mason6b800532007-10-15 16:17:34 -04002276 nr = slot;
Josef Bacik25b8b932011-06-08 14:36:54 -04002277
Chris Masond3977122009-01-05 21:25:51 -05002278 while (1) {
Chris Mason6b800532007-10-15 16:17:34 -04002279 if (direction < 0) {
2280 if (nr == 0)
2281 break;
2282 nr--;
2283 } else if (direction > 0) {
2284 nr++;
2285 if (nr >= nritems)
2286 break;
Chris Mason3c69fae2007-08-07 15:52:22 -04002287 }
Chris Mason01f46652007-12-21 16:24:26 -05002288 if (path->reada < 0 && objectid) {
2289 btrfs_node_key(node, &disk_key, nr);
2290 if (btrfs_disk_key_objectid(&disk_key) != objectid)
2291 break;
2292 }
Chris Mason6b800532007-10-15 16:17:34 -04002293 search = btrfs_node_blockptr(node, nr);
Chris Masona7175312009-01-22 09:23:10 -05002294 if ((search <= target && target - search <= 65536) ||
2295 (search > target && search - target <= 65536)) {
Josef Bacikcb25c2e2011-05-11 12:17:34 -04002296 gen = btrfs_node_ptr_generation(node, nr);
David Sterbad3e46fe2014-06-15 02:04:19 +02002297 readahead_tree_block(root, search);
Chris Mason6b800532007-10-15 16:17:34 -04002298 nread += blocksize;
2299 }
2300 nscan++;
Chris Masona7175312009-01-22 09:23:10 -05002301 if ((nread > 65536 || nscan > 32))
Chris Mason6b800532007-10-15 16:17:34 -04002302 break;
Chris Mason3c69fae2007-08-07 15:52:22 -04002303 }
2304}
Chris Mason925baed2008-06-25 16:01:30 -04002305
Josef Bacik0b088512013-06-17 14:23:02 -04002306static noinline void reada_for_balance(struct btrfs_root *root,
2307 struct btrfs_path *path, int level)
Chris Masonb4ce94d2009-02-04 09:25:08 -05002308{
2309 int slot;
2310 int nritems;
2311 struct extent_buffer *parent;
2312 struct extent_buffer *eb;
2313 u64 gen;
2314 u64 block1 = 0;
2315 u64 block2 = 0;
Chris Masonb4ce94d2009-02-04 09:25:08 -05002316
Chris Mason8c594ea2009-04-20 15:50:10 -04002317 parent = path->nodes[level + 1];
Chris Masonb4ce94d2009-02-04 09:25:08 -05002318 if (!parent)
Josef Bacik0b088512013-06-17 14:23:02 -04002319 return;
Chris Masonb4ce94d2009-02-04 09:25:08 -05002320
2321 nritems = btrfs_header_nritems(parent);
Chris Mason8c594ea2009-04-20 15:50:10 -04002322 slot = path->slots[level + 1];
Chris Masonb4ce94d2009-02-04 09:25:08 -05002323
2324 if (slot > 0) {
2325 block1 = btrfs_node_blockptr(parent, slot - 1);
2326 gen = btrfs_node_ptr_generation(parent, slot - 1);
Daniel Dressler01d58472014-11-21 17:15:07 +09002327 eb = btrfs_find_tree_block(root->fs_info, block1);
Chris Masonb9fab912012-05-06 07:23:47 -04002328 /*
2329 * if we get -eagain from btrfs_buffer_uptodate, we
2330 * don't want to return eagain here. That will loop
2331 * forever
2332 */
2333 if (eb && btrfs_buffer_uptodate(eb, gen, 1) != 0)
Chris Masonb4ce94d2009-02-04 09:25:08 -05002334 block1 = 0;
2335 free_extent_buffer(eb);
2336 }
Chris Mason8c594ea2009-04-20 15:50:10 -04002337 if (slot + 1 < nritems) {
Chris Masonb4ce94d2009-02-04 09:25:08 -05002338 block2 = btrfs_node_blockptr(parent, slot + 1);
2339 gen = btrfs_node_ptr_generation(parent, slot + 1);
Daniel Dressler01d58472014-11-21 17:15:07 +09002340 eb = btrfs_find_tree_block(root->fs_info, block2);
Chris Masonb9fab912012-05-06 07:23:47 -04002341 if (eb && btrfs_buffer_uptodate(eb, gen, 1) != 0)
Chris Masonb4ce94d2009-02-04 09:25:08 -05002342 block2 = 0;
2343 free_extent_buffer(eb);
2344 }
Chris Mason8c594ea2009-04-20 15:50:10 -04002345
Josef Bacik0b088512013-06-17 14:23:02 -04002346 if (block1)
David Sterbad3e46fe2014-06-15 02:04:19 +02002347 readahead_tree_block(root, block1);
Josef Bacik0b088512013-06-17 14:23:02 -04002348 if (block2)
David Sterbad3e46fe2014-06-15 02:04:19 +02002349 readahead_tree_block(root, block2);
Chris Masonb4ce94d2009-02-04 09:25:08 -05002350}
2351
2352
2353/*
Chris Masond3977122009-01-05 21:25:51 -05002354 * when we walk down the tree, it is usually safe to unlock the higher layers
2355 * in the tree. The exceptions are when our path goes through slot 0, because
2356 * operations on the tree might require changing key pointers higher up in the
2357 * tree.
Chris Masond352ac62008-09-29 15:18:18 -04002358 *
Chris Masond3977122009-01-05 21:25:51 -05002359 * callers might also have set path->keep_locks, which tells this code to keep
2360 * the lock if the path points to the last slot in the block. This is part of
2361 * walking through the tree, and selecting the next slot in the higher block.
Chris Masond352ac62008-09-29 15:18:18 -04002362 *
Chris Masond3977122009-01-05 21:25:51 -05002363 * lowest_unlock sets the lowest level in the tree we're allowed to unlock. so
2364 * if lowest_unlock is 1, level 0 won't be unlocked
Chris Masond352ac62008-09-29 15:18:18 -04002365 */
Chris Masone02119d2008-09-05 16:13:11 -04002366static noinline void unlock_up(struct btrfs_path *path, int level,
Chris Masonf7c79f32012-03-19 15:54:38 -04002367 int lowest_unlock, int min_write_lock_level,
2368 int *write_lock_level)
Chris Mason925baed2008-06-25 16:01:30 -04002369{
2370 int i;
2371 int skip_level = level;
Chris Mason051e1b92008-06-25 16:01:30 -04002372 int no_skips = 0;
Chris Mason925baed2008-06-25 16:01:30 -04002373 struct extent_buffer *t;
2374
2375 for (i = level; i < BTRFS_MAX_LEVEL; i++) {
2376 if (!path->nodes[i])
2377 break;
2378 if (!path->locks[i])
2379 break;
Chris Mason051e1b92008-06-25 16:01:30 -04002380 if (!no_skips && path->slots[i] == 0) {
Chris Mason925baed2008-06-25 16:01:30 -04002381 skip_level = i + 1;
2382 continue;
2383 }
Chris Mason051e1b92008-06-25 16:01:30 -04002384 if (!no_skips && path->keep_locks) {
Chris Mason925baed2008-06-25 16:01:30 -04002385 u32 nritems;
2386 t = path->nodes[i];
2387 nritems = btrfs_header_nritems(t);
Chris Mason051e1b92008-06-25 16:01:30 -04002388 if (nritems < 1 || path->slots[i] >= nritems - 1) {
Chris Mason925baed2008-06-25 16:01:30 -04002389 skip_level = i + 1;
2390 continue;
2391 }
2392 }
Chris Mason051e1b92008-06-25 16:01:30 -04002393 if (skip_level < i && i >= lowest_unlock)
2394 no_skips = 1;
2395
Chris Mason925baed2008-06-25 16:01:30 -04002396 t = path->nodes[i];
2397 if (i >= lowest_unlock && i > skip_level && path->locks[i]) {
Chris Masonbd681512011-07-16 15:23:14 -04002398 btrfs_tree_unlock_rw(t, path->locks[i]);
Chris Mason925baed2008-06-25 16:01:30 -04002399 path->locks[i] = 0;
Chris Masonf7c79f32012-03-19 15:54:38 -04002400 if (write_lock_level &&
2401 i > min_write_lock_level &&
2402 i <= *write_lock_level) {
2403 *write_lock_level = i - 1;
2404 }
Chris Mason925baed2008-06-25 16:01:30 -04002405 }
2406 }
2407}
2408
Chris Mason3c69fae2007-08-07 15:52:22 -04002409/*
Chris Masonb4ce94d2009-02-04 09:25:08 -05002410 * This releases any locks held in the path starting at level and
2411 * going all the way up to the root.
2412 *
2413 * btrfs_search_slot will keep the lock held on higher nodes in a few
2414 * corner cases, such as COW of the block at slot zero in the node. This
2415 * ignores those rules, and it should only be called when there are no
2416 * more updates to be done higher up in the tree.
2417 */
2418noinline void btrfs_unlock_up_safe(struct btrfs_path *path, int level)
2419{
2420 int i;
2421
Josef Bacik09a2a8f92013-04-05 16:51:15 -04002422 if (path->keep_locks)
Chris Masonb4ce94d2009-02-04 09:25:08 -05002423 return;
2424
2425 for (i = level; i < BTRFS_MAX_LEVEL; i++) {
2426 if (!path->nodes[i])
Chris Mason12f4dac2009-02-04 09:31:42 -05002427 continue;
Chris Masonb4ce94d2009-02-04 09:25:08 -05002428 if (!path->locks[i])
Chris Mason12f4dac2009-02-04 09:31:42 -05002429 continue;
Chris Masonbd681512011-07-16 15:23:14 -04002430 btrfs_tree_unlock_rw(path->nodes[i], path->locks[i]);
Chris Masonb4ce94d2009-02-04 09:25:08 -05002431 path->locks[i] = 0;
2432 }
2433}
2434
2435/*
Chris Masonc8c42862009-04-03 10:14:18 -04002436 * helper function for btrfs_search_slot. The goal is to find a block
2437 * in cache without setting the path to blocking. If we find the block
2438 * we return zero and the path is unchanged.
2439 *
2440 * If we can't find the block, we set the path blocking and do some
2441 * reada. -EAGAIN is returned and the search must be repeated.
2442 */
2443static int
2444read_block_for_search(struct btrfs_trans_handle *trans,
2445 struct btrfs_root *root, struct btrfs_path *p,
2446 struct extent_buffer **eb_ret, int level, int slot,
Jan Schmidt5d9e75c42012-05-16 18:25:47 +02002447 struct btrfs_key *key, u64 time_seq)
Chris Masonc8c42862009-04-03 10:14:18 -04002448{
2449 u64 blocknr;
2450 u64 gen;
Chris Masonc8c42862009-04-03 10:14:18 -04002451 struct extent_buffer *b = *eb_ret;
2452 struct extent_buffer *tmp;
Chris Mason76a05b32009-05-14 13:24:30 -04002453 int ret;
Chris Masonc8c42862009-04-03 10:14:18 -04002454
2455 blocknr = btrfs_node_blockptr(b, slot);
2456 gen = btrfs_node_ptr_generation(b, slot);
Chris Masonc8c42862009-04-03 10:14:18 -04002457
Daniel Dressler01d58472014-11-21 17:15:07 +09002458 tmp = btrfs_find_tree_block(root->fs_info, blocknr);
Chris Masoncb449212010-10-24 11:01:27 -04002459 if (tmp) {
Chris Masonb9fab912012-05-06 07:23:47 -04002460 /* first we do an atomic uptodate check */
Josef Bacikbdf7c002013-06-17 13:44:48 -04002461 if (btrfs_buffer_uptodate(tmp, gen, 1) > 0) {
2462 *eb_ret = tmp;
2463 return 0;
Chris Masoncb449212010-10-24 11:01:27 -04002464 }
Josef Bacikbdf7c002013-06-17 13:44:48 -04002465
2466 /* the pages were up to date, but we failed
2467 * the generation number check. Do a full
2468 * read for the generation number that is correct.
2469 * We must do this without dropping locks so
2470 * we can trust our generation number
2471 */
2472 btrfs_set_path_blocking(p);
2473
2474 /* now we're allowed to do a blocking uptodate check */
2475 ret = btrfs_read_buffer(tmp, gen);
2476 if (!ret) {
2477 *eb_ret = tmp;
2478 return 0;
2479 }
2480 free_extent_buffer(tmp);
2481 btrfs_release_path(p);
2482 return -EIO;
Chris Masonc8c42862009-04-03 10:14:18 -04002483 }
2484
2485 /*
2486 * reduce lock contention at high levels
2487 * of the btree by dropping locks before
Chris Mason76a05b32009-05-14 13:24:30 -04002488 * we read. Don't release the lock on the current
2489 * level because we need to walk this node to figure
2490 * out which blocks to read.
Chris Masonc8c42862009-04-03 10:14:18 -04002491 */
Chris Mason8c594ea2009-04-20 15:50:10 -04002492 btrfs_unlock_up_safe(p, level + 1);
2493 btrfs_set_path_blocking(p);
2494
Chris Masoncb449212010-10-24 11:01:27 -04002495 free_extent_buffer(tmp);
Chris Masonc8c42862009-04-03 10:14:18 -04002496 if (p->reada)
2497 reada_for_search(root, p, level, slot, key->objectid);
2498
David Sterbab3b4aa72011-04-21 01:20:15 +02002499 btrfs_release_path(p);
Chris Mason76a05b32009-05-14 13:24:30 -04002500
2501 ret = -EAGAIN;
David Sterbace86cd52014-06-15 01:07:32 +02002502 tmp = read_tree_block(root, blocknr, 0);
Liu Bo64c043d2015-05-25 17:30:15 +08002503 if (!IS_ERR(tmp)) {
Chris Mason76a05b32009-05-14 13:24:30 -04002504 /*
2505 * If the read above didn't mark this buffer up to date,
2506 * it will never end up being up to date. Set ret to EIO now
2507 * and give up so that our caller doesn't loop forever
2508 * on our EAGAINs.
2509 */
Chris Masonb9fab912012-05-06 07:23:47 -04002510 if (!btrfs_buffer_uptodate(tmp, 0, 0))
Chris Mason76a05b32009-05-14 13:24:30 -04002511 ret = -EIO;
Chris Masonc8c42862009-04-03 10:14:18 -04002512 free_extent_buffer(tmp);
Chris Mason76a05b32009-05-14 13:24:30 -04002513 }
2514 return ret;
Chris Masonc8c42862009-04-03 10:14:18 -04002515}
2516
2517/*
2518 * helper function for btrfs_search_slot. This does all of the checks
2519 * for node-level blocks and does any balancing required based on
2520 * the ins_len.
2521 *
2522 * If no extra work was required, zero is returned. If we had to
2523 * drop the path, -EAGAIN is returned and btrfs_search_slot must
2524 * start over
2525 */
2526static int
2527setup_nodes_for_search(struct btrfs_trans_handle *trans,
2528 struct btrfs_root *root, struct btrfs_path *p,
Chris Masonbd681512011-07-16 15:23:14 -04002529 struct extent_buffer *b, int level, int ins_len,
2530 int *write_lock_level)
Chris Masonc8c42862009-04-03 10:14:18 -04002531{
2532 int ret;
2533 if ((p->search_for_split || ins_len > 0) && btrfs_header_nritems(b) >=
2534 BTRFS_NODEPTRS_PER_BLOCK(root) - 3) {
2535 int sret;
2536
Chris Masonbd681512011-07-16 15:23:14 -04002537 if (*write_lock_level < level + 1) {
2538 *write_lock_level = level + 1;
2539 btrfs_release_path(p);
2540 goto again;
2541 }
2542
Chris Masonc8c42862009-04-03 10:14:18 -04002543 btrfs_set_path_blocking(p);
Josef Bacik0b088512013-06-17 14:23:02 -04002544 reada_for_balance(root, p, level);
Chris Masonc8c42862009-04-03 10:14:18 -04002545 sret = split_node(trans, root, p, level);
Chris Masonbd681512011-07-16 15:23:14 -04002546 btrfs_clear_path_blocking(p, NULL, 0);
Chris Masonc8c42862009-04-03 10:14:18 -04002547
2548 BUG_ON(sret > 0);
2549 if (sret) {
2550 ret = sret;
2551 goto done;
2552 }
2553 b = p->nodes[level];
2554 } else if (ins_len < 0 && btrfs_header_nritems(b) <
Chris Masoncfbb9302009-05-18 10:41:58 -04002555 BTRFS_NODEPTRS_PER_BLOCK(root) / 2) {
Chris Masonc8c42862009-04-03 10:14:18 -04002556 int sret;
2557
Chris Masonbd681512011-07-16 15:23:14 -04002558 if (*write_lock_level < level + 1) {
2559 *write_lock_level = level + 1;
2560 btrfs_release_path(p);
2561 goto again;
2562 }
2563
Chris Masonc8c42862009-04-03 10:14:18 -04002564 btrfs_set_path_blocking(p);
Josef Bacik0b088512013-06-17 14:23:02 -04002565 reada_for_balance(root, p, level);
Chris Masonc8c42862009-04-03 10:14:18 -04002566 sret = balance_level(trans, root, p, level);
Chris Masonbd681512011-07-16 15:23:14 -04002567 btrfs_clear_path_blocking(p, NULL, 0);
Chris Masonc8c42862009-04-03 10:14:18 -04002568
2569 if (sret) {
2570 ret = sret;
2571 goto done;
2572 }
2573 b = p->nodes[level];
2574 if (!b) {
David Sterbab3b4aa72011-04-21 01:20:15 +02002575 btrfs_release_path(p);
Chris Masonc8c42862009-04-03 10:14:18 -04002576 goto again;
2577 }
2578 BUG_ON(btrfs_header_nritems(b) == 1);
2579 }
2580 return 0;
2581
2582again:
2583 ret = -EAGAIN;
2584done:
2585 return ret;
2586}
2587
Filipe David Borba Mananad7396f02013-08-30 15:46:43 +01002588static void key_search_validate(struct extent_buffer *b,
2589 struct btrfs_key *key,
2590 int level)
2591{
2592#ifdef CONFIG_BTRFS_ASSERT
2593 struct btrfs_disk_key disk_key;
2594
2595 btrfs_cpu_key_to_disk(&disk_key, key);
2596
2597 if (level == 0)
2598 ASSERT(!memcmp_extent_buffer(b, &disk_key,
2599 offsetof(struct btrfs_leaf, items[0].key),
2600 sizeof(disk_key)));
2601 else
2602 ASSERT(!memcmp_extent_buffer(b, &disk_key,
2603 offsetof(struct btrfs_node, ptrs[0].key),
2604 sizeof(disk_key)));
2605#endif
2606}
2607
2608static int key_search(struct extent_buffer *b, struct btrfs_key *key,
2609 int level, int *prev_cmp, int *slot)
2610{
2611 if (*prev_cmp != 0) {
2612 *prev_cmp = bin_search(b, key, level, slot);
2613 return *prev_cmp;
2614 }
2615
2616 key_search_validate(b, key, level);
2617 *slot = 0;
2618
2619 return 0;
2620}
2621
David Sterba381cf652015-01-02 18:45:16 +01002622int btrfs_find_item(struct btrfs_root *fs_root, struct btrfs_path *path,
Kelley Nielsene33d5c32013-11-04 19:33:33 -08002623 u64 iobjectid, u64 ioff, u8 key_type,
2624 struct btrfs_key *found_key)
2625{
2626 int ret;
2627 struct btrfs_key key;
2628 struct extent_buffer *eb;
David Sterba381cf652015-01-02 18:45:16 +01002629
2630 ASSERT(path);
David Sterba1d4c08e2015-01-02 19:36:14 +01002631 ASSERT(found_key);
Kelley Nielsene33d5c32013-11-04 19:33:33 -08002632
2633 key.type = key_type;
2634 key.objectid = iobjectid;
2635 key.offset = ioff;
2636
2637 ret = btrfs_search_slot(NULL, fs_root, &key, path, 0, 0);
David Sterba1d4c08e2015-01-02 19:36:14 +01002638 if (ret < 0)
Kelley Nielsene33d5c32013-11-04 19:33:33 -08002639 return ret;
2640
2641 eb = path->nodes[0];
2642 if (ret && path->slots[0] >= btrfs_header_nritems(eb)) {
2643 ret = btrfs_next_leaf(fs_root, path);
2644 if (ret)
2645 return ret;
2646 eb = path->nodes[0];
2647 }
2648
2649 btrfs_item_key_to_cpu(eb, found_key, path->slots[0]);
2650 if (found_key->type != key.type ||
2651 found_key->objectid != key.objectid)
2652 return 1;
2653
2654 return 0;
2655}
2656
Chris Masonc8c42862009-04-03 10:14:18 -04002657/*
Chris Mason74123bd2007-02-02 11:05:29 -05002658 * look for key in the tree. path is filled in with nodes along the way
2659 * if key is found, we return zero and you can find the item in the leaf
2660 * level of the path (level 0)
2661 *
2662 * If the key isn't found, the path points to the slot where it should
Chris Masonaa5d6be2007-02-28 16:35:06 -05002663 * be inserted, and 1 is returned. If there are other errors during the
2664 * search a negative error number is returned.
Chris Mason97571fd2007-02-24 13:39:08 -05002665 *
2666 * if ins_len > 0, nodes and leaves will be split as we walk down the
2667 * tree. if ins_len < 0, nodes will be merged as we walk down the tree (if
2668 * possible)
Chris Mason74123bd2007-02-02 11:05:29 -05002669 */
Chris Masone089f052007-03-16 16:20:31 -04002670int btrfs_search_slot(struct btrfs_trans_handle *trans, struct btrfs_root
2671 *root, struct btrfs_key *key, struct btrfs_path *p, int
2672 ins_len, int cow)
Chris Masonbe0e5c02007-01-26 15:51:26 -05002673{
Chris Mason5f39d392007-10-15 16:14:19 -04002674 struct extent_buffer *b;
Chris Masonbe0e5c02007-01-26 15:51:26 -05002675 int slot;
2676 int ret;
Yan Zheng33c66f42009-07-22 09:59:00 -04002677 int err;
Chris Masonbe0e5c02007-01-26 15:51:26 -05002678 int level;
Chris Mason925baed2008-06-25 16:01:30 -04002679 int lowest_unlock = 1;
Chris Masonbd681512011-07-16 15:23:14 -04002680 int root_lock;
2681 /* everything at write_lock_level or lower must be write locked */
2682 int write_lock_level = 0;
Chris Mason9f3a7422007-08-07 15:52:19 -04002683 u8 lowest_level = 0;
Chris Masonf7c79f32012-03-19 15:54:38 -04002684 int min_write_lock_level;
Filipe David Borba Mananad7396f02013-08-30 15:46:43 +01002685 int prev_cmp;
Chris Mason9f3a7422007-08-07 15:52:19 -04002686
Chris Mason6702ed42007-08-07 16:15:09 -04002687 lowest_level = p->lowest_level;
Chris Mason323ac952008-10-01 19:05:46 -04002688 WARN_ON(lowest_level && ins_len > 0);
Chris Mason22b0ebd2007-03-30 08:47:31 -04002689 WARN_ON(p->nodes[0] != NULL);
Filipe David Borba Mananaeb653de2013-12-23 11:53:02 +00002690 BUG_ON(!cow && ins_len);
Josef Bacik25179202008-10-29 14:49:05 -04002691
Chris Masonbd681512011-07-16 15:23:14 -04002692 if (ins_len < 0) {
Chris Mason925baed2008-06-25 16:01:30 -04002693 lowest_unlock = 2;
Chris Mason65b51a02008-08-01 15:11:20 -04002694
Chris Masonbd681512011-07-16 15:23:14 -04002695 /* when we are removing items, we might have to go up to level
2696 * two as we update tree pointers Make sure we keep write
2697 * for those levels as well
2698 */
2699 write_lock_level = 2;
2700 } else if (ins_len > 0) {
2701 /*
2702 * for inserting items, make sure we have a write lock on
2703 * level 1 so we can update keys
2704 */
2705 write_lock_level = 1;
2706 }
2707
2708 if (!cow)
2709 write_lock_level = -1;
2710
Josef Bacik09a2a8f92013-04-05 16:51:15 -04002711 if (cow && (p->keep_locks || p->lowest_level))
Chris Masonbd681512011-07-16 15:23:14 -04002712 write_lock_level = BTRFS_MAX_LEVEL;
2713
Chris Masonf7c79f32012-03-19 15:54:38 -04002714 min_write_lock_level = write_lock_level;
2715
Chris Masonbb803952007-03-01 12:04:21 -05002716again:
Filipe David Borba Mananad7396f02013-08-30 15:46:43 +01002717 prev_cmp = -1;
Chris Masonbd681512011-07-16 15:23:14 -04002718 /*
2719 * we try very hard to do read locks on the root
2720 */
2721 root_lock = BTRFS_READ_LOCK;
2722 level = 0;
Yan Zheng5d4f98a2009-06-10 10:45:14 -04002723 if (p->search_commit_root) {
Chris Masonbd681512011-07-16 15:23:14 -04002724 /*
2725 * the commit roots are read only
2726 * so we always do read locks
2727 */
Josef Bacik3f8a18c2014-03-28 17:16:01 -04002728 if (p->need_commit_sem)
2729 down_read(&root->fs_info->commit_root_sem);
Yan Zheng5d4f98a2009-06-10 10:45:14 -04002730 b = root->commit_root;
2731 extent_buffer_get(b);
Chris Masonbd681512011-07-16 15:23:14 -04002732 level = btrfs_header_level(b);
Josef Bacik3f8a18c2014-03-28 17:16:01 -04002733 if (p->need_commit_sem)
2734 up_read(&root->fs_info->commit_root_sem);
Yan Zheng5d4f98a2009-06-10 10:45:14 -04002735 if (!p->skip_locking)
Chris Masonbd681512011-07-16 15:23:14 -04002736 btrfs_tree_read_lock(b);
Yan Zheng5d4f98a2009-06-10 10:45:14 -04002737 } else {
Chris Masonbd681512011-07-16 15:23:14 -04002738 if (p->skip_locking) {
Yan Zheng5d4f98a2009-06-10 10:45:14 -04002739 b = btrfs_root_node(root);
Chris Masonbd681512011-07-16 15:23:14 -04002740 level = btrfs_header_level(b);
2741 } else {
2742 /* we don't know the level of the root node
2743 * until we actually have it read locked
2744 */
2745 b = btrfs_read_lock_root_node(root);
2746 level = btrfs_header_level(b);
2747 if (level <= write_lock_level) {
2748 /* whoops, must trade for write lock */
2749 btrfs_tree_read_unlock(b);
2750 free_extent_buffer(b);
2751 b = btrfs_lock_root_node(root);
2752 root_lock = BTRFS_WRITE_LOCK;
2753
2754 /* the level might have changed, check again */
2755 level = btrfs_header_level(b);
2756 }
2757 }
Yan Zheng5d4f98a2009-06-10 10:45:14 -04002758 }
Chris Masonbd681512011-07-16 15:23:14 -04002759 p->nodes[level] = b;
2760 if (!p->skip_locking)
2761 p->locks[level] = root_lock;
Chris Mason925baed2008-06-25 16:01:30 -04002762
Chris Masoneb60cea2007-02-02 09:18:22 -05002763 while (b) {
Chris Mason5f39d392007-10-15 16:14:19 -04002764 level = btrfs_header_level(b);
Chris Mason65b51a02008-08-01 15:11:20 -04002765
2766 /*
2767 * setup the path here so we can release it under lock
2768 * contention with the cow code
2769 */
Chris Mason02217ed2007-03-02 16:08:05 -05002770 if (cow) {
Chris Masonc8c42862009-04-03 10:14:18 -04002771 /*
2772 * if we don't really need to cow this block
2773 * then we don't want to set the path blocking,
2774 * so we test it here
2775 */
Yan Zheng5d4f98a2009-06-10 10:45:14 -04002776 if (!should_cow_block(trans, root, b))
Chris Mason65b51a02008-08-01 15:11:20 -04002777 goto cow_done;
Yan Zheng5d4f98a2009-06-10 10:45:14 -04002778
Chris Masonbd681512011-07-16 15:23:14 -04002779 /*
2780 * must have write locks on this node and the
2781 * parent
2782 */
Josef Bacik5124e002012-11-07 13:44:13 -05002783 if (level > write_lock_level ||
2784 (level + 1 > write_lock_level &&
2785 level + 1 < BTRFS_MAX_LEVEL &&
2786 p->nodes[level + 1])) {
Chris Masonbd681512011-07-16 15:23:14 -04002787 write_lock_level = level + 1;
2788 btrfs_release_path(p);
2789 goto again;
2790 }
2791
Filipe Manana160f4082014-07-28 19:37:17 +01002792 btrfs_set_path_blocking(p);
Yan Zheng33c66f42009-07-22 09:59:00 -04002793 err = btrfs_cow_block(trans, root, b,
2794 p->nodes[level + 1],
2795 p->slots[level + 1], &b);
2796 if (err) {
Yan Zheng33c66f42009-07-22 09:59:00 -04002797 ret = err;
Chris Mason65b51a02008-08-01 15:11:20 -04002798 goto done;
Chris Mason54aa1f42007-06-22 14:16:25 -04002799 }
Chris Mason02217ed2007-03-02 16:08:05 -05002800 }
Chris Mason65b51a02008-08-01 15:11:20 -04002801cow_done:
Chris Masoneb60cea2007-02-02 09:18:22 -05002802 p->nodes[level] = b;
Chris Masonbd681512011-07-16 15:23:14 -04002803 btrfs_clear_path_blocking(p, NULL, 0);
Chris Masonb4ce94d2009-02-04 09:25:08 -05002804
2805 /*
2806 * we have a lock on b and as long as we aren't changing
2807 * the tree, there is no way to for the items in b to change.
2808 * It is safe to drop the lock on our parent before we
2809 * go through the expensive btree search on b.
2810 *
Filipe David Borba Mananaeb653de2013-12-23 11:53:02 +00002811 * If we're inserting or deleting (ins_len != 0), then we might
2812 * be changing slot zero, which may require changing the parent.
2813 * So, we can't drop the lock until after we know which slot
2814 * we're operating on.
Chris Masonb4ce94d2009-02-04 09:25:08 -05002815 */
Filipe David Borba Mananaeb653de2013-12-23 11:53:02 +00002816 if (!ins_len && !p->keep_locks) {
2817 int u = level + 1;
2818
2819 if (u < BTRFS_MAX_LEVEL && p->locks[u]) {
2820 btrfs_tree_unlock_rw(p->nodes[u], p->locks[u]);
2821 p->locks[u] = 0;
2822 }
2823 }
Chris Masonb4ce94d2009-02-04 09:25:08 -05002824
Filipe David Borba Mananad7396f02013-08-30 15:46:43 +01002825 ret = key_search(b, key, level, &prev_cmp, &slot);
Chris Masonb4ce94d2009-02-04 09:25:08 -05002826
Chris Mason5f39d392007-10-15 16:14:19 -04002827 if (level != 0) {
Yan Zheng33c66f42009-07-22 09:59:00 -04002828 int dec = 0;
2829 if (ret && slot > 0) {
2830 dec = 1;
Chris Masonbe0e5c02007-01-26 15:51:26 -05002831 slot -= 1;
Yan Zheng33c66f42009-07-22 09:59:00 -04002832 }
Chris Masonbe0e5c02007-01-26 15:51:26 -05002833 p->slots[level] = slot;
Yan Zheng33c66f42009-07-22 09:59:00 -04002834 err = setup_nodes_for_search(trans, root, p, b, level,
Chris Masonbd681512011-07-16 15:23:14 -04002835 ins_len, &write_lock_level);
Yan Zheng33c66f42009-07-22 09:59:00 -04002836 if (err == -EAGAIN)
Chris Masonc8c42862009-04-03 10:14:18 -04002837 goto again;
Yan Zheng33c66f42009-07-22 09:59:00 -04002838 if (err) {
2839 ret = err;
Chris Masonc8c42862009-04-03 10:14:18 -04002840 goto done;
Yan Zheng33c66f42009-07-22 09:59:00 -04002841 }
Chris Masonc8c42862009-04-03 10:14:18 -04002842 b = p->nodes[level];
2843 slot = p->slots[level];
Chris Masonb4ce94d2009-02-04 09:25:08 -05002844
Chris Masonbd681512011-07-16 15:23:14 -04002845 /*
2846 * slot 0 is special, if we change the key
2847 * we have to update the parent pointer
2848 * which means we must have a write lock
2849 * on the parent
2850 */
Filipe David Borba Mananaeb653de2013-12-23 11:53:02 +00002851 if (slot == 0 && ins_len &&
Chris Masonbd681512011-07-16 15:23:14 -04002852 write_lock_level < level + 1) {
2853 write_lock_level = level + 1;
2854 btrfs_release_path(p);
2855 goto again;
2856 }
2857
Chris Masonf7c79f32012-03-19 15:54:38 -04002858 unlock_up(p, level, lowest_unlock,
2859 min_write_lock_level, &write_lock_level);
Chris Masonf9efa9c2008-06-25 16:14:04 -04002860
Chris Mason925baed2008-06-25 16:01:30 -04002861 if (level == lowest_level) {
Yan Zheng33c66f42009-07-22 09:59:00 -04002862 if (dec)
2863 p->slots[level]++;
Zheng Yan5b21f2e2008-09-26 10:05:38 -04002864 goto done;
Chris Mason925baed2008-06-25 16:01:30 -04002865 }
Chris Masonca7a79a2008-05-12 12:59:19 -04002866
Yan Zheng33c66f42009-07-22 09:59:00 -04002867 err = read_block_for_search(trans, root, p,
Jan Schmidt5d9e75c42012-05-16 18:25:47 +02002868 &b, level, slot, key, 0);
Yan Zheng33c66f42009-07-22 09:59:00 -04002869 if (err == -EAGAIN)
Chris Masonc8c42862009-04-03 10:14:18 -04002870 goto again;
Yan Zheng33c66f42009-07-22 09:59:00 -04002871 if (err) {
2872 ret = err;
Chris Mason76a05b32009-05-14 13:24:30 -04002873 goto done;
Yan Zheng33c66f42009-07-22 09:59:00 -04002874 }
Chris Mason76a05b32009-05-14 13:24:30 -04002875
Chris Masonb4ce94d2009-02-04 09:25:08 -05002876 if (!p->skip_locking) {
Chris Masonbd681512011-07-16 15:23:14 -04002877 level = btrfs_header_level(b);
2878 if (level <= write_lock_level) {
2879 err = btrfs_try_tree_write_lock(b);
2880 if (!err) {
2881 btrfs_set_path_blocking(p);
2882 btrfs_tree_lock(b);
2883 btrfs_clear_path_blocking(p, b,
2884 BTRFS_WRITE_LOCK);
2885 }
2886 p->locks[level] = BTRFS_WRITE_LOCK;
2887 } else {
Chris Masonf82c4582014-11-19 10:25:09 -08002888 err = btrfs_tree_read_lock_atomic(b);
Chris Masonbd681512011-07-16 15:23:14 -04002889 if (!err) {
2890 btrfs_set_path_blocking(p);
2891 btrfs_tree_read_lock(b);
2892 btrfs_clear_path_blocking(p, b,
2893 BTRFS_READ_LOCK);
2894 }
2895 p->locks[level] = BTRFS_READ_LOCK;
Chris Masonb4ce94d2009-02-04 09:25:08 -05002896 }
Chris Masonbd681512011-07-16 15:23:14 -04002897 p->nodes[level] = b;
Chris Masonb4ce94d2009-02-04 09:25:08 -05002898 }
Chris Masonbe0e5c02007-01-26 15:51:26 -05002899 } else {
2900 p->slots[level] = slot;
Yan Zheng87b29b22008-12-17 10:21:48 -05002901 if (ins_len > 0 &&
2902 btrfs_leaf_free_space(root, b) < ins_len) {
Chris Masonbd681512011-07-16 15:23:14 -04002903 if (write_lock_level < 1) {
2904 write_lock_level = 1;
2905 btrfs_release_path(p);
2906 goto again;
2907 }
2908
Chris Masonb4ce94d2009-02-04 09:25:08 -05002909 btrfs_set_path_blocking(p);
Yan Zheng33c66f42009-07-22 09:59:00 -04002910 err = split_leaf(trans, root, key,
2911 p, ins_len, ret == 0);
Chris Masonbd681512011-07-16 15:23:14 -04002912 btrfs_clear_path_blocking(p, NULL, 0);
Chris Masonb4ce94d2009-02-04 09:25:08 -05002913
Yan Zheng33c66f42009-07-22 09:59:00 -04002914 BUG_ON(err > 0);
2915 if (err) {
2916 ret = err;
Chris Mason65b51a02008-08-01 15:11:20 -04002917 goto done;
2918 }
Chris Mason5c680ed2007-02-22 11:39:13 -05002919 }
Chris Mason459931e2008-12-10 09:10:46 -05002920 if (!p->search_for_split)
Chris Masonf7c79f32012-03-19 15:54:38 -04002921 unlock_up(p, level, lowest_unlock,
2922 min_write_lock_level, &write_lock_level);
Chris Mason65b51a02008-08-01 15:11:20 -04002923 goto done;
Chris Masonbe0e5c02007-01-26 15:51:26 -05002924 }
2925 }
Chris Mason65b51a02008-08-01 15:11:20 -04002926 ret = 1;
2927done:
Chris Masonb4ce94d2009-02-04 09:25:08 -05002928 /*
2929 * we don't really know what they plan on doing with the path
2930 * from here on, so for now just mark it as blocking
2931 */
Chris Masonb9473432009-03-13 11:00:37 -04002932 if (!p->leave_spinning)
2933 btrfs_set_path_blocking(p);
Filipe Manana5f5bc6b2014-11-09 08:38:39 +00002934 if (ret < 0 && !p->skip_release_on_error)
David Sterbab3b4aa72011-04-21 01:20:15 +02002935 btrfs_release_path(p);
Chris Mason65b51a02008-08-01 15:11:20 -04002936 return ret;
Chris Masonbe0e5c02007-01-26 15:51:26 -05002937}
2938
Chris Mason74123bd2007-02-02 11:05:29 -05002939/*
Jan Schmidt5d9e75c42012-05-16 18:25:47 +02002940 * Like btrfs_search_slot, this looks for a key in the given tree. It uses the
2941 * current state of the tree together with the operations recorded in the tree
2942 * modification log to search for the key in a previous version of this tree, as
2943 * denoted by the time_seq parameter.
2944 *
2945 * Naturally, there is no support for insert, delete or cow operations.
2946 *
2947 * The resulting path and return value will be set up as if we called
2948 * btrfs_search_slot at that point in time with ins_len and cow both set to 0.
2949 */
2950int btrfs_search_old_slot(struct btrfs_root *root, struct btrfs_key *key,
2951 struct btrfs_path *p, u64 time_seq)
2952{
2953 struct extent_buffer *b;
2954 int slot;
2955 int ret;
2956 int err;
2957 int level;
2958 int lowest_unlock = 1;
2959 u8 lowest_level = 0;
Josef Bacikd4b40872013-09-24 14:09:34 -04002960 int prev_cmp = -1;
Jan Schmidt5d9e75c42012-05-16 18:25:47 +02002961
2962 lowest_level = p->lowest_level;
2963 WARN_ON(p->nodes[0] != NULL);
2964
2965 if (p->search_commit_root) {
2966 BUG_ON(time_seq);
2967 return btrfs_search_slot(NULL, root, key, p, 0, 0);
2968 }
2969
2970again:
Jan Schmidt5d9e75c42012-05-16 18:25:47 +02002971 b = get_old_root(root, time_seq);
Jan Schmidt5d9e75c42012-05-16 18:25:47 +02002972 level = btrfs_header_level(b);
Jan Schmidt5d9e75c42012-05-16 18:25:47 +02002973 p->locks[level] = BTRFS_READ_LOCK;
2974
2975 while (b) {
2976 level = btrfs_header_level(b);
2977 p->nodes[level] = b;
2978 btrfs_clear_path_blocking(p, NULL, 0);
2979
2980 /*
2981 * we have a lock on b and as long as we aren't changing
2982 * the tree, there is no way to for the items in b to change.
2983 * It is safe to drop the lock on our parent before we
2984 * go through the expensive btree search on b.
2985 */
2986 btrfs_unlock_up_safe(p, level + 1);
2987
Josef Bacikd4b40872013-09-24 14:09:34 -04002988 /*
2989 * Since we can unwind eb's we want to do a real search every
2990 * time.
2991 */
2992 prev_cmp = -1;
Filipe David Borba Mananad7396f02013-08-30 15:46:43 +01002993 ret = key_search(b, key, level, &prev_cmp, &slot);
Jan Schmidt5d9e75c42012-05-16 18:25:47 +02002994
2995 if (level != 0) {
2996 int dec = 0;
2997 if (ret && slot > 0) {
2998 dec = 1;
2999 slot -= 1;
3000 }
3001 p->slots[level] = slot;
3002 unlock_up(p, level, lowest_unlock, 0, NULL);
3003
3004 if (level == lowest_level) {
3005 if (dec)
3006 p->slots[level]++;
3007 goto done;
3008 }
3009
3010 err = read_block_for_search(NULL, root, p, &b, level,
3011 slot, key, time_seq);
3012 if (err == -EAGAIN)
3013 goto again;
3014 if (err) {
3015 ret = err;
3016 goto done;
3017 }
3018
3019 level = btrfs_header_level(b);
Chris Masonf82c4582014-11-19 10:25:09 -08003020 err = btrfs_tree_read_lock_atomic(b);
Jan Schmidt5d9e75c42012-05-16 18:25:47 +02003021 if (!err) {
3022 btrfs_set_path_blocking(p);
3023 btrfs_tree_read_lock(b);
3024 btrfs_clear_path_blocking(p, b,
3025 BTRFS_READ_LOCK);
3026 }
Josef Bacik9ec72672013-08-07 16:57:23 -04003027 b = tree_mod_log_rewind(root->fs_info, p, b, time_seq);
Josef Bacikdb7f3432013-08-07 14:54:37 -04003028 if (!b) {
3029 ret = -ENOMEM;
3030 goto done;
3031 }
Jan Schmidt5d9e75c42012-05-16 18:25:47 +02003032 p->locks[level] = BTRFS_READ_LOCK;
3033 p->nodes[level] = b;
Jan Schmidt5d9e75c42012-05-16 18:25:47 +02003034 } else {
3035 p->slots[level] = slot;
3036 unlock_up(p, level, lowest_unlock, 0, NULL);
3037 goto done;
3038 }
3039 }
3040 ret = 1;
3041done:
3042 if (!p->leave_spinning)
3043 btrfs_set_path_blocking(p);
3044 if (ret < 0)
3045 btrfs_release_path(p);
3046
3047 return ret;
3048}
3049
3050/*
Arne Jansen2f38b3e2011-09-13 11:18:10 +02003051 * helper to use instead of search slot if no exact match is needed but
3052 * instead the next or previous item should be returned.
3053 * When find_higher is true, the next higher item is returned, the next lower
3054 * otherwise.
3055 * When return_any and find_higher are both true, and no higher item is found,
3056 * return the next lower instead.
3057 * When return_any is true and find_higher is false, and no lower item is found,
3058 * return the next higher instead.
3059 * It returns 0 if any item is found, 1 if none is found (tree empty), and
3060 * < 0 on error
3061 */
3062int btrfs_search_slot_for_read(struct btrfs_root *root,
3063 struct btrfs_key *key, struct btrfs_path *p,
3064 int find_higher, int return_any)
3065{
3066 int ret;
3067 struct extent_buffer *leaf;
3068
3069again:
3070 ret = btrfs_search_slot(NULL, root, key, p, 0, 0);
3071 if (ret <= 0)
3072 return ret;
3073 /*
3074 * a return value of 1 means the path is at the position where the
3075 * item should be inserted. Normally this is the next bigger item,
3076 * but in case the previous item is the last in a leaf, path points
3077 * to the first free slot in the previous leaf, i.e. at an invalid
3078 * item.
3079 */
3080 leaf = p->nodes[0];
3081
3082 if (find_higher) {
3083 if (p->slots[0] >= btrfs_header_nritems(leaf)) {
3084 ret = btrfs_next_leaf(root, p);
3085 if (ret <= 0)
3086 return ret;
3087 if (!return_any)
3088 return 1;
3089 /*
3090 * no higher item found, return the next
3091 * lower instead
3092 */
3093 return_any = 0;
3094 find_higher = 0;
3095 btrfs_release_path(p);
3096 goto again;
3097 }
3098 } else {
Arne Jansene6793762011-09-13 11:18:10 +02003099 if (p->slots[0] == 0) {
3100 ret = btrfs_prev_leaf(root, p);
3101 if (ret < 0)
3102 return ret;
3103 if (!ret) {
Filipe David Borba Manana23c6bf62014-01-11 21:28:54 +00003104 leaf = p->nodes[0];
3105 if (p->slots[0] == btrfs_header_nritems(leaf))
3106 p->slots[0]--;
Arne Jansene6793762011-09-13 11:18:10 +02003107 return 0;
Arne Jansen2f38b3e2011-09-13 11:18:10 +02003108 }
Arne Jansene6793762011-09-13 11:18:10 +02003109 if (!return_any)
3110 return 1;
3111 /*
3112 * no lower item found, return the next
3113 * higher instead
3114 */
3115 return_any = 0;
3116 find_higher = 1;
3117 btrfs_release_path(p);
3118 goto again;
3119 } else {
Arne Jansen2f38b3e2011-09-13 11:18:10 +02003120 --p->slots[0];
3121 }
3122 }
3123 return 0;
3124}
3125
3126/*
Chris Mason74123bd2007-02-02 11:05:29 -05003127 * adjust the pointers going up the tree, starting at level
3128 * making sure the right key of each node is points to 'key'.
3129 * This is used after shifting pointers to the left, so it stops
3130 * fixing up pointers when a given leaf/node is not in slot 0 of the
3131 * higher levels
Chris Masonaa5d6be2007-02-28 16:35:06 -05003132 *
Chris Mason74123bd2007-02-02 11:05:29 -05003133 */
Daniel Dresslerb7a03652014-11-12 13:43:09 +09003134static void fixup_low_keys(struct btrfs_fs_info *fs_info,
3135 struct btrfs_path *path,
Jeff Mahoney143bede2012-03-01 14:56:26 +01003136 struct btrfs_disk_key *key, int level)
Chris Masonbe0e5c02007-01-26 15:51:26 -05003137{
3138 int i;
Chris Mason5f39d392007-10-15 16:14:19 -04003139 struct extent_buffer *t;
3140
Chris Mason234b63a2007-03-13 10:46:10 -04003141 for (i = level; i < BTRFS_MAX_LEVEL; i++) {
Chris Masonbe0e5c02007-01-26 15:51:26 -05003142 int tslot = path->slots[i];
Chris Masoneb60cea2007-02-02 09:18:22 -05003143 if (!path->nodes[i])
Chris Masonbe0e5c02007-01-26 15:51:26 -05003144 break;
Chris Mason5f39d392007-10-15 16:14:19 -04003145 t = path->nodes[i];
Daniel Dresslerb7a03652014-11-12 13:43:09 +09003146 tree_mod_log_set_node_key(fs_info, t, tslot, 1);
Chris Mason5f39d392007-10-15 16:14:19 -04003147 btrfs_set_node_key(t, key, tslot);
Chris Masond6025572007-03-30 14:27:56 -04003148 btrfs_mark_buffer_dirty(path->nodes[i]);
Chris Masonbe0e5c02007-01-26 15:51:26 -05003149 if (tslot != 0)
3150 break;
3151 }
3152}
3153
Chris Mason74123bd2007-02-02 11:05:29 -05003154/*
Zheng Yan31840ae2008-09-23 13:14:14 -04003155 * update item key.
3156 *
3157 * This function isn't completely safe. It's the caller's responsibility
3158 * that the new key won't break the order
3159 */
Daniel Dresslerb7a03652014-11-12 13:43:09 +09003160void btrfs_set_item_key_safe(struct btrfs_fs_info *fs_info,
3161 struct btrfs_path *path,
Jeff Mahoney143bede2012-03-01 14:56:26 +01003162 struct btrfs_key *new_key)
Zheng Yan31840ae2008-09-23 13:14:14 -04003163{
3164 struct btrfs_disk_key disk_key;
3165 struct extent_buffer *eb;
3166 int slot;
3167
3168 eb = path->nodes[0];
3169 slot = path->slots[0];
3170 if (slot > 0) {
3171 btrfs_item_key(eb, &disk_key, slot - 1);
Jeff Mahoney143bede2012-03-01 14:56:26 +01003172 BUG_ON(comp_keys(&disk_key, new_key) >= 0);
Zheng Yan31840ae2008-09-23 13:14:14 -04003173 }
3174 if (slot < btrfs_header_nritems(eb) - 1) {
3175 btrfs_item_key(eb, &disk_key, slot + 1);
Jeff Mahoney143bede2012-03-01 14:56:26 +01003176 BUG_ON(comp_keys(&disk_key, new_key) <= 0);
Zheng Yan31840ae2008-09-23 13:14:14 -04003177 }
3178
3179 btrfs_cpu_key_to_disk(&disk_key, new_key);
3180 btrfs_set_item_key(eb, &disk_key, slot);
3181 btrfs_mark_buffer_dirty(eb);
3182 if (slot == 0)
Daniel Dresslerb7a03652014-11-12 13:43:09 +09003183 fixup_low_keys(fs_info, path, &disk_key, 1);
Zheng Yan31840ae2008-09-23 13:14:14 -04003184}
3185
3186/*
Chris Mason74123bd2007-02-02 11:05:29 -05003187 * try to push data from one node into the next node left in the
Chris Mason79f95c82007-03-01 15:16:26 -05003188 * tree.
Chris Masonaa5d6be2007-02-28 16:35:06 -05003189 *
3190 * returns 0 if some ptrs were pushed left, < 0 if there was some horrible
3191 * error, and > 0 if there was no room in the left hand block.
Chris Mason74123bd2007-02-02 11:05:29 -05003192 */
Chris Mason98ed5172008-01-03 10:01:48 -05003193static int push_node_left(struct btrfs_trans_handle *trans,
3194 struct btrfs_root *root, struct extent_buffer *dst,
Chris Mason971a1f62008-04-24 10:54:32 -04003195 struct extent_buffer *src, int empty)
Chris Masonbe0e5c02007-01-26 15:51:26 -05003196{
Chris Masonbe0e5c02007-01-26 15:51:26 -05003197 int push_items = 0;
Chris Masonbb803952007-03-01 12:04:21 -05003198 int src_nritems;
3199 int dst_nritems;
Chris Masonaa5d6be2007-02-28 16:35:06 -05003200 int ret = 0;
Chris Masonbe0e5c02007-01-26 15:51:26 -05003201
Chris Mason5f39d392007-10-15 16:14:19 -04003202 src_nritems = btrfs_header_nritems(src);
3203 dst_nritems = btrfs_header_nritems(dst);
Chris Mason123abc82007-03-14 14:14:43 -04003204 push_items = BTRFS_NODEPTRS_PER_BLOCK(root) - dst_nritems;
Chris Mason7bb86312007-12-11 09:25:06 -05003205 WARN_ON(btrfs_header_generation(src) != trans->transid);
3206 WARN_ON(btrfs_header_generation(dst) != trans->transid);
Chris Mason54aa1f42007-06-22 14:16:25 -04003207
Chris Masonbce4eae2008-04-24 14:42:46 -04003208 if (!empty && src_nritems <= 8)
Chris Mason971a1f62008-04-24 10:54:32 -04003209 return 1;
3210
Chris Masond3977122009-01-05 21:25:51 -05003211 if (push_items <= 0)
Chris Masonbe0e5c02007-01-26 15:51:26 -05003212 return 1;
3213
Chris Masonbce4eae2008-04-24 14:42:46 -04003214 if (empty) {
Chris Mason971a1f62008-04-24 10:54:32 -04003215 push_items = min(src_nritems, push_items);
Chris Masonbce4eae2008-04-24 14:42:46 -04003216 if (push_items < src_nritems) {
3217 /* leave at least 8 pointers in the node if
3218 * we aren't going to empty it
3219 */
3220 if (src_nritems - push_items < 8) {
3221 if (push_items <= 8)
3222 return 1;
3223 push_items -= 8;
3224 }
3225 }
3226 } else
3227 push_items = min(src_nritems - 8, push_items);
Chris Mason79f95c82007-03-01 15:16:26 -05003228
Filipe David Borba Manana5de865e2013-12-20 15:17:46 +00003229 ret = tree_mod_log_eb_copy(root->fs_info, dst, src, dst_nritems, 0,
3230 push_items);
3231 if (ret) {
3232 btrfs_abort_transaction(trans, root, ret);
3233 return ret;
3234 }
Chris Mason5f39d392007-10-15 16:14:19 -04003235 copy_extent_buffer(dst, src,
3236 btrfs_node_key_ptr_offset(dst_nritems),
3237 btrfs_node_key_ptr_offset(0),
Chris Masond3977122009-01-05 21:25:51 -05003238 push_items * sizeof(struct btrfs_key_ptr));
Chris Mason5f39d392007-10-15 16:14:19 -04003239
Chris Masonbb803952007-03-01 12:04:21 -05003240 if (push_items < src_nritems) {
Jan Schmidt57911b82012-10-19 09:22:03 +02003241 /*
3242 * don't call tree_mod_log_eb_move here, key removal was already
3243 * fully logged by tree_mod_log_eb_copy above.
3244 */
Chris Mason5f39d392007-10-15 16:14:19 -04003245 memmove_extent_buffer(src, btrfs_node_key_ptr_offset(0),
3246 btrfs_node_key_ptr_offset(push_items),
3247 (src_nritems - push_items) *
3248 sizeof(struct btrfs_key_ptr));
Chris Masonbb803952007-03-01 12:04:21 -05003249 }
Chris Mason5f39d392007-10-15 16:14:19 -04003250 btrfs_set_header_nritems(src, src_nritems - push_items);
3251 btrfs_set_header_nritems(dst, dst_nritems + push_items);
3252 btrfs_mark_buffer_dirty(src);
3253 btrfs_mark_buffer_dirty(dst);
Zheng Yan31840ae2008-09-23 13:14:14 -04003254
Chris Masonbb803952007-03-01 12:04:21 -05003255 return ret;
Chris Masonbe0e5c02007-01-26 15:51:26 -05003256}
3257
Chris Mason97571fd2007-02-24 13:39:08 -05003258/*
Chris Mason79f95c82007-03-01 15:16:26 -05003259 * try to push data from one node into the next node right in the
3260 * tree.
3261 *
3262 * returns 0 if some ptrs were pushed, < 0 if there was some horrible
3263 * error, and > 0 if there was no room in the right hand block.
3264 *
3265 * this will only push up to 1/2 the contents of the left node over
3266 */
Chris Mason5f39d392007-10-15 16:14:19 -04003267static int balance_node_right(struct btrfs_trans_handle *trans,
3268 struct btrfs_root *root,
3269 struct extent_buffer *dst,
3270 struct extent_buffer *src)
Chris Mason79f95c82007-03-01 15:16:26 -05003271{
Chris Mason79f95c82007-03-01 15:16:26 -05003272 int push_items = 0;
3273 int max_push;
3274 int src_nritems;
3275 int dst_nritems;
3276 int ret = 0;
Chris Mason79f95c82007-03-01 15:16:26 -05003277
Chris Mason7bb86312007-12-11 09:25:06 -05003278 WARN_ON(btrfs_header_generation(src) != trans->transid);
3279 WARN_ON(btrfs_header_generation(dst) != trans->transid);
3280
Chris Mason5f39d392007-10-15 16:14:19 -04003281 src_nritems = btrfs_header_nritems(src);
3282 dst_nritems = btrfs_header_nritems(dst);
Chris Mason123abc82007-03-14 14:14:43 -04003283 push_items = BTRFS_NODEPTRS_PER_BLOCK(root) - dst_nritems;
Chris Masond3977122009-01-05 21:25:51 -05003284 if (push_items <= 0)
Chris Mason79f95c82007-03-01 15:16:26 -05003285 return 1;
Chris Masonbce4eae2008-04-24 14:42:46 -04003286
Chris Masond3977122009-01-05 21:25:51 -05003287 if (src_nritems < 4)
Chris Masonbce4eae2008-04-24 14:42:46 -04003288 return 1;
Chris Mason79f95c82007-03-01 15:16:26 -05003289
3290 max_push = src_nritems / 2 + 1;
3291 /* don't try to empty the node */
Chris Masond3977122009-01-05 21:25:51 -05003292 if (max_push >= src_nritems)
Chris Mason79f95c82007-03-01 15:16:26 -05003293 return 1;
Yan252c38f2007-08-29 09:11:44 -04003294
Chris Mason79f95c82007-03-01 15:16:26 -05003295 if (max_push < push_items)
3296 push_items = max_push;
3297
Jan Schmidtf2304752012-05-26 11:43:17 +02003298 tree_mod_log_eb_move(root->fs_info, dst, push_items, 0, dst_nritems);
Chris Mason5f39d392007-10-15 16:14:19 -04003299 memmove_extent_buffer(dst, btrfs_node_key_ptr_offset(push_items),
3300 btrfs_node_key_ptr_offset(0),
3301 (dst_nritems) *
3302 sizeof(struct btrfs_key_ptr));
Chris Masond6025572007-03-30 14:27:56 -04003303
Filipe David Borba Manana5de865e2013-12-20 15:17:46 +00003304 ret = tree_mod_log_eb_copy(root->fs_info, dst, src, 0,
3305 src_nritems - push_items, push_items);
3306 if (ret) {
3307 btrfs_abort_transaction(trans, root, ret);
3308 return ret;
3309 }
Chris Mason5f39d392007-10-15 16:14:19 -04003310 copy_extent_buffer(dst, src,
3311 btrfs_node_key_ptr_offset(0),
3312 btrfs_node_key_ptr_offset(src_nritems - push_items),
Chris Masond3977122009-01-05 21:25:51 -05003313 push_items * sizeof(struct btrfs_key_ptr));
Chris Mason79f95c82007-03-01 15:16:26 -05003314
Chris Mason5f39d392007-10-15 16:14:19 -04003315 btrfs_set_header_nritems(src, src_nritems - push_items);
3316 btrfs_set_header_nritems(dst, dst_nritems + push_items);
Chris Mason79f95c82007-03-01 15:16:26 -05003317
Chris Mason5f39d392007-10-15 16:14:19 -04003318 btrfs_mark_buffer_dirty(src);
3319 btrfs_mark_buffer_dirty(dst);
Zheng Yan31840ae2008-09-23 13:14:14 -04003320
Chris Mason79f95c82007-03-01 15:16:26 -05003321 return ret;
3322}
3323
3324/*
Chris Mason97571fd2007-02-24 13:39:08 -05003325 * helper function to insert a new root level in the tree.
3326 * A new node is allocated, and a single item is inserted to
3327 * point to the existing root
Chris Masonaa5d6be2007-02-28 16:35:06 -05003328 *
3329 * returns zero on success or < 0 on failure.
Chris Mason97571fd2007-02-24 13:39:08 -05003330 */
Chris Masond3977122009-01-05 21:25:51 -05003331static noinline int insert_new_root(struct btrfs_trans_handle *trans,
Chris Mason5f39d392007-10-15 16:14:19 -04003332 struct btrfs_root *root,
Liu Bofdd99c72013-05-22 12:06:51 +00003333 struct btrfs_path *path, int level)
Chris Mason5c680ed2007-02-22 11:39:13 -05003334{
Chris Mason7bb86312007-12-11 09:25:06 -05003335 u64 lower_gen;
Chris Mason5f39d392007-10-15 16:14:19 -04003336 struct extent_buffer *lower;
3337 struct extent_buffer *c;
Chris Mason925baed2008-06-25 16:01:30 -04003338 struct extent_buffer *old;
Chris Mason5f39d392007-10-15 16:14:19 -04003339 struct btrfs_disk_key lower_key;
Chris Mason5c680ed2007-02-22 11:39:13 -05003340
3341 BUG_ON(path->nodes[level]);
3342 BUG_ON(path->nodes[level-1] != root->node);
3343
Chris Mason7bb86312007-12-11 09:25:06 -05003344 lower = path->nodes[level-1];
3345 if (level == 1)
3346 btrfs_item_key(lower, &lower_key, 0);
3347 else
3348 btrfs_node_key(lower, &lower_key, 0);
3349
David Sterba4d75f8a2014-06-15 01:54:12 +02003350 c = btrfs_alloc_tree_block(trans, root, 0, root->root_key.objectid,
3351 &lower_key, level, root->node->start, 0);
Chris Mason5f39d392007-10-15 16:14:19 -04003352 if (IS_ERR(c))
3353 return PTR_ERR(c);
Chris Mason925baed2008-06-25 16:01:30 -04003354
Yan, Zhengf0486c62010-05-16 10:46:25 -04003355 root_add_used(root, root->nodesize);
3356
Yan Zheng5d4f98a2009-06-10 10:45:14 -04003357 memset_extent_buffer(c, 0, 0, sizeof(struct btrfs_header));
Chris Mason5f39d392007-10-15 16:14:19 -04003358 btrfs_set_header_nritems(c, 1);
3359 btrfs_set_header_level(c, level);
Chris Masondb945352007-10-15 16:15:53 -04003360 btrfs_set_header_bytenr(c, c->start);
Chris Mason5f39d392007-10-15 16:14:19 -04003361 btrfs_set_header_generation(c, trans->transid);
Yan Zheng5d4f98a2009-06-10 10:45:14 -04003362 btrfs_set_header_backref_rev(c, BTRFS_MIXED_BACKREF_REV);
Chris Mason5f39d392007-10-15 16:14:19 -04003363 btrfs_set_header_owner(c, root->root_key.objectid);
Chris Masond5719762007-03-23 10:01:08 -04003364
Ross Kirk0a4e5582013-09-24 10:12:38 +01003365 write_extent_buffer(c, root->fs_info->fsid, btrfs_header_fsid(),
Chris Mason5f39d392007-10-15 16:14:19 -04003366 BTRFS_FSID_SIZE);
Chris Masone17cade2008-04-15 15:41:47 -04003367
3368 write_extent_buffer(c, root->fs_info->chunk_tree_uuid,
Geert Uytterhoevenb308bc22013-08-20 13:20:15 +02003369 btrfs_header_chunk_tree_uuid(c), BTRFS_UUID_SIZE);
Chris Masone17cade2008-04-15 15:41:47 -04003370
Chris Mason5f39d392007-10-15 16:14:19 -04003371 btrfs_set_node_key(c, &lower_key, 0);
Chris Masondb945352007-10-15 16:15:53 -04003372 btrfs_set_node_blockptr(c, 0, lower->start);
Chris Mason7bb86312007-12-11 09:25:06 -05003373 lower_gen = btrfs_header_generation(lower);
Zheng Yan31840ae2008-09-23 13:14:14 -04003374 WARN_ON(lower_gen != trans->transid);
Chris Mason7bb86312007-12-11 09:25:06 -05003375
3376 btrfs_set_node_ptr_generation(c, 0, lower_gen);
Chris Mason5f39d392007-10-15 16:14:19 -04003377
3378 btrfs_mark_buffer_dirty(c);
Chris Masond5719762007-03-23 10:01:08 -04003379
Chris Mason925baed2008-06-25 16:01:30 -04003380 old = root->node;
Liu Bofdd99c72013-05-22 12:06:51 +00003381 tree_mod_log_set_root_pointer(root, c, 0);
Chris Mason240f62c2011-03-23 14:54:42 -04003382 rcu_assign_pointer(root->node, c);
Chris Mason925baed2008-06-25 16:01:30 -04003383
3384 /* the super has an extra ref to root->node */
3385 free_extent_buffer(old);
3386
Chris Mason0b86a832008-03-24 15:01:56 -04003387 add_root_to_dirty_list(root);
Chris Mason5f39d392007-10-15 16:14:19 -04003388 extent_buffer_get(c);
3389 path->nodes[level] = c;
chandan95449a12015-01-15 12:22:03 +05303390 path->locks[level] = BTRFS_WRITE_LOCK_BLOCKING;
Chris Mason5c680ed2007-02-22 11:39:13 -05003391 path->slots[level] = 0;
3392 return 0;
3393}
3394
Chris Mason74123bd2007-02-02 11:05:29 -05003395/*
3396 * worker function to insert a single pointer in a node.
3397 * the node should have enough room for the pointer already
Chris Mason97571fd2007-02-24 13:39:08 -05003398 *
Chris Mason74123bd2007-02-02 11:05:29 -05003399 * slot and level indicate where you want the key to go, and
3400 * blocknr is the block the key points to.
3401 */
Jeff Mahoney143bede2012-03-01 14:56:26 +01003402static void insert_ptr(struct btrfs_trans_handle *trans,
3403 struct btrfs_root *root, struct btrfs_path *path,
3404 struct btrfs_disk_key *key, u64 bytenr,
Jan Schmidtc3e06962012-06-21 11:01:06 +02003405 int slot, int level)
Chris Mason74123bd2007-02-02 11:05:29 -05003406{
Chris Mason5f39d392007-10-15 16:14:19 -04003407 struct extent_buffer *lower;
Chris Mason74123bd2007-02-02 11:05:29 -05003408 int nritems;
Jan Schmidtf3ea38d2012-05-26 11:45:21 +02003409 int ret;
Chris Mason5c680ed2007-02-22 11:39:13 -05003410
3411 BUG_ON(!path->nodes[level]);
Yan, Zhengf0486c62010-05-16 10:46:25 -04003412 btrfs_assert_tree_locked(path->nodes[level]);
Chris Mason5f39d392007-10-15 16:14:19 -04003413 lower = path->nodes[level];
3414 nritems = btrfs_header_nritems(lower);
Stoyan Gaydarovc2934982009-04-02 17:05:11 -04003415 BUG_ON(slot > nritems);
Jeff Mahoney143bede2012-03-01 14:56:26 +01003416 BUG_ON(nritems == BTRFS_NODEPTRS_PER_BLOCK(root));
Chris Mason74123bd2007-02-02 11:05:29 -05003417 if (slot != nritems) {
Jan Schmidtc3e06962012-06-21 11:01:06 +02003418 if (level)
Jan Schmidtf3ea38d2012-05-26 11:45:21 +02003419 tree_mod_log_eb_move(root->fs_info, lower, slot + 1,
3420 slot, nritems - slot);
Chris Mason5f39d392007-10-15 16:14:19 -04003421 memmove_extent_buffer(lower,
3422 btrfs_node_key_ptr_offset(slot + 1),
3423 btrfs_node_key_ptr_offset(slot),
Chris Masond6025572007-03-30 14:27:56 -04003424 (nritems - slot) * sizeof(struct btrfs_key_ptr));
Chris Mason74123bd2007-02-02 11:05:29 -05003425 }
Jan Schmidtc3e06962012-06-21 11:01:06 +02003426 if (level) {
Jan Schmidtf3ea38d2012-05-26 11:45:21 +02003427 ret = tree_mod_log_insert_key(root->fs_info, lower, slot,
Josef Bacikc8cc6342013-07-01 16:18:19 -04003428 MOD_LOG_KEY_ADD, GFP_NOFS);
Jan Schmidtf3ea38d2012-05-26 11:45:21 +02003429 BUG_ON(ret < 0);
3430 }
Chris Mason5f39d392007-10-15 16:14:19 -04003431 btrfs_set_node_key(lower, key, slot);
Chris Masondb945352007-10-15 16:15:53 -04003432 btrfs_set_node_blockptr(lower, slot, bytenr);
Chris Mason74493f72007-12-11 09:25:06 -05003433 WARN_ON(trans->transid == 0);
3434 btrfs_set_node_ptr_generation(lower, slot, trans->transid);
Chris Mason5f39d392007-10-15 16:14:19 -04003435 btrfs_set_header_nritems(lower, nritems + 1);
3436 btrfs_mark_buffer_dirty(lower);
Chris Mason74123bd2007-02-02 11:05:29 -05003437}
3438
Chris Mason97571fd2007-02-24 13:39:08 -05003439/*
3440 * split the node at the specified level in path in two.
3441 * The path is corrected to point to the appropriate node after the split
3442 *
3443 * Before splitting this tries to make some room in the node by pushing
3444 * left and right, if either one works, it returns right away.
Chris Masonaa5d6be2007-02-28 16:35:06 -05003445 *
3446 * returns 0 on success and < 0 on failure
Chris Mason97571fd2007-02-24 13:39:08 -05003447 */
Chris Masone02119d2008-09-05 16:13:11 -04003448static noinline int split_node(struct btrfs_trans_handle *trans,
3449 struct btrfs_root *root,
3450 struct btrfs_path *path, int level)
Chris Masonbe0e5c02007-01-26 15:51:26 -05003451{
Chris Mason5f39d392007-10-15 16:14:19 -04003452 struct extent_buffer *c;
3453 struct extent_buffer *split;
3454 struct btrfs_disk_key disk_key;
Chris Masonbe0e5c02007-01-26 15:51:26 -05003455 int mid;
Chris Mason5c680ed2007-02-22 11:39:13 -05003456 int ret;
Chris Mason7518a232007-03-12 12:01:18 -04003457 u32 c_nritems;
Chris Masonbe0e5c02007-01-26 15:51:26 -05003458
Chris Mason5f39d392007-10-15 16:14:19 -04003459 c = path->nodes[level];
Chris Mason7bb86312007-12-11 09:25:06 -05003460 WARN_ON(btrfs_header_generation(c) != trans->transid);
Chris Mason5f39d392007-10-15 16:14:19 -04003461 if (c == root->node) {
Jan Schmidtd9abbf12013-03-20 13:49:48 +00003462 /*
Jan Schmidt90f8d622013-04-13 13:19:53 +00003463 * trying to split the root, lets make a new one
3464 *
Liu Bofdd99c72013-05-22 12:06:51 +00003465 * tree mod log: We don't log_removal old root in
Jan Schmidt90f8d622013-04-13 13:19:53 +00003466 * insert_new_root, because that root buffer will be kept as a
3467 * normal node. We are going to log removal of half of the
3468 * elements below with tree_mod_log_eb_copy. We're holding a
3469 * tree lock on the buffer, which is why we cannot race with
3470 * other tree_mod_log users.
Jan Schmidtd9abbf12013-03-20 13:49:48 +00003471 */
Liu Bofdd99c72013-05-22 12:06:51 +00003472 ret = insert_new_root(trans, root, path, level + 1);
Chris Mason5c680ed2007-02-22 11:39:13 -05003473 if (ret)
3474 return ret;
Chris Masonb3612422009-05-13 19:12:15 -04003475 } else {
Chris Masone66f7092007-04-20 13:16:02 -04003476 ret = push_nodes_for_insert(trans, root, path, level);
Chris Mason5f39d392007-10-15 16:14:19 -04003477 c = path->nodes[level];
3478 if (!ret && btrfs_header_nritems(c) <
Chris Masonc448acf2008-04-24 09:34:34 -04003479 BTRFS_NODEPTRS_PER_BLOCK(root) - 3)
Chris Masone66f7092007-04-20 13:16:02 -04003480 return 0;
Chris Mason54aa1f42007-06-22 14:16:25 -04003481 if (ret < 0)
3482 return ret;
Chris Masonbe0e5c02007-01-26 15:51:26 -05003483 }
Chris Masone66f7092007-04-20 13:16:02 -04003484
Chris Mason5f39d392007-10-15 16:14:19 -04003485 c_nritems = btrfs_header_nritems(c);
Yan Zheng5d4f98a2009-06-10 10:45:14 -04003486 mid = (c_nritems + 1) / 2;
3487 btrfs_node_key(c, &disk_key, mid);
Chris Mason7bb86312007-12-11 09:25:06 -05003488
David Sterba4d75f8a2014-06-15 01:54:12 +02003489 split = btrfs_alloc_tree_block(trans, root, 0, root->root_key.objectid,
3490 &disk_key, level, c->start, 0);
Chris Mason5f39d392007-10-15 16:14:19 -04003491 if (IS_ERR(split))
3492 return PTR_ERR(split);
Chris Mason54aa1f42007-06-22 14:16:25 -04003493
Yan, Zhengf0486c62010-05-16 10:46:25 -04003494 root_add_used(root, root->nodesize);
3495
Yan Zheng5d4f98a2009-06-10 10:45:14 -04003496 memset_extent_buffer(split, 0, 0, sizeof(struct btrfs_header));
Chris Mason5f39d392007-10-15 16:14:19 -04003497 btrfs_set_header_level(split, btrfs_header_level(c));
Chris Masondb945352007-10-15 16:15:53 -04003498 btrfs_set_header_bytenr(split, split->start);
Chris Mason5f39d392007-10-15 16:14:19 -04003499 btrfs_set_header_generation(split, trans->transid);
Yan Zheng5d4f98a2009-06-10 10:45:14 -04003500 btrfs_set_header_backref_rev(split, BTRFS_MIXED_BACKREF_REV);
Chris Mason5f39d392007-10-15 16:14:19 -04003501 btrfs_set_header_owner(split, root->root_key.objectid);
3502 write_extent_buffer(split, root->fs_info->fsid,
Ross Kirk0a4e5582013-09-24 10:12:38 +01003503 btrfs_header_fsid(), BTRFS_FSID_SIZE);
Chris Masone17cade2008-04-15 15:41:47 -04003504 write_extent_buffer(split, root->fs_info->chunk_tree_uuid,
Geert Uytterhoevenb308bc22013-08-20 13:20:15 +02003505 btrfs_header_chunk_tree_uuid(split),
Chris Masone17cade2008-04-15 15:41:47 -04003506 BTRFS_UUID_SIZE);
Chris Mason5f39d392007-10-15 16:14:19 -04003507
Filipe David Borba Manana5de865e2013-12-20 15:17:46 +00003508 ret = tree_mod_log_eb_copy(root->fs_info, split, c, 0,
3509 mid, c_nritems - mid);
3510 if (ret) {
3511 btrfs_abort_transaction(trans, root, ret);
3512 return ret;
3513 }
Chris Mason5f39d392007-10-15 16:14:19 -04003514 copy_extent_buffer(split, c,
3515 btrfs_node_key_ptr_offset(0),
3516 btrfs_node_key_ptr_offset(mid),
3517 (c_nritems - mid) * sizeof(struct btrfs_key_ptr));
3518 btrfs_set_header_nritems(split, c_nritems - mid);
3519 btrfs_set_header_nritems(c, mid);
Chris Masonaa5d6be2007-02-28 16:35:06 -05003520 ret = 0;
3521
Chris Mason5f39d392007-10-15 16:14:19 -04003522 btrfs_mark_buffer_dirty(c);
3523 btrfs_mark_buffer_dirty(split);
3524
Jeff Mahoney143bede2012-03-01 14:56:26 +01003525 insert_ptr(trans, root, path, &disk_key, split->start,
Jan Schmidtc3e06962012-06-21 11:01:06 +02003526 path->slots[level + 1] + 1, level + 1);
Chris Masonaa5d6be2007-02-28 16:35:06 -05003527
Chris Mason5de08d72007-02-24 06:24:44 -05003528 if (path->slots[level] >= mid) {
Chris Mason5c680ed2007-02-22 11:39:13 -05003529 path->slots[level] -= mid;
Chris Mason925baed2008-06-25 16:01:30 -04003530 btrfs_tree_unlock(c);
Chris Mason5f39d392007-10-15 16:14:19 -04003531 free_extent_buffer(c);
3532 path->nodes[level] = split;
Chris Mason5c680ed2007-02-22 11:39:13 -05003533 path->slots[level + 1] += 1;
3534 } else {
Chris Mason925baed2008-06-25 16:01:30 -04003535 btrfs_tree_unlock(split);
Chris Mason5f39d392007-10-15 16:14:19 -04003536 free_extent_buffer(split);
Chris Masonbe0e5c02007-01-26 15:51:26 -05003537 }
Chris Masonaa5d6be2007-02-28 16:35:06 -05003538 return ret;
Chris Masonbe0e5c02007-01-26 15:51:26 -05003539}
3540
Chris Mason74123bd2007-02-02 11:05:29 -05003541/*
3542 * how many bytes are required to store the items in a leaf. start
3543 * and nr indicate which items in the leaf to check. This totals up the
3544 * space used both by the item structs and the item data
3545 */
Chris Mason5f39d392007-10-15 16:14:19 -04003546static int leaf_space_used(struct extent_buffer *l, int start, int nr)
Chris Masonbe0e5c02007-01-26 15:51:26 -05003547{
Josef Bacik41be1f32012-10-15 13:43:18 -04003548 struct btrfs_item *start_item;
3549 struct btrfs_item *end_item;
3550 struct btrfs_map_token token;
Chris Masonbe0e5c02007-01-26 15:51:26 -05003551 int data_len;
Chris Mason5f39d392007-10-15 16:14:19 -04003552 int nritems = btrfs_header_nritems(l);
Chris Masond4dbff92007-04-04 14:08:15 -04003553 int end = min(nritems, start + nr) - 1;
Chris Masonbe0e5c02007-01-26 15:51:26 -05003554
3555 if (!nr)
3556 return 0;
Josef Bacik41be1f32012-10-15 13:43:18 -04003557 btrfs_init_map_token(&token);
Ross Kirkdd3cc162013-09-16 15:58:09 +01003558 start_item = btrfs_item_nr(start);
3559 end_item = btrfs_item_nr(end);
Josef Bacik41be1f32012-10-15 13:43:18 -04003560 data_len = btrfs_token_item_offset(l, start_item, &token) +
3561 btrfs_token_item_size(l, start_item, &token);
3562 data_len = data_len - btrfs_token_item_offset(l, end_item, &token);
Chris Mason0783fcf2007-03-12 20:12:07 -04003563 data_len += sizeof(struct btrfs_item) * nr;
Chris Masond4dbff92007-04-04 14:08:15 -04003564 WARN_ON(data_len < 0);
Chris Masonbe0e5c02007-01-26 15:51:26 -05003565 return data_len;
3566}
3567
Chris Mason74123bd2007-02-02 11:05:29 -05003568/*
Chris Masond4dbff92007-04-04 14:08:15 -04003569 * The space between the end of the leaf items and
3570 * the start of the leaf data. IOW, how much room
3571 * the leaf has left for both items and data
3572 */
Chris Masond3977122009-01-05 21:25:51 -05003573noinline int btrfs_leaf_free_space(struct btrfs_root *root,
Chris Masone02119d2008-09-05 16:13:11 -04003574 struct extent_buffer *leaf)
Chris Masond4dbff92007-04-04 14:08:15 -04003575{
Chris Mason5f39d392007-10-15 16:14:19 -04003576 int nritems = btrfs_header_nritems(leaf);
3577 int ret;
3578 ret = BTRFS_LEAF_DATA_SIZE(root) - leaf_space_used(leaf, 0, nritems);
3579 if (ret < 0) {
Frank Holtonefe120a2013-12-20 11:37:06 -05003580 btrfs_crit(root->fs_info,
3581 "leaf free space ret %d, leaf data size %lu, used %d nritems %d",
Jens Axboeae2f5412007-10-19 09:22:59 -04003582 ret, (unsigned long) BTRFS_LEAF_DATA_SIZE(root),
Chris Mason5f39d392007-10-15 16:14:19 -04003583 leaf_space_used(leaf, 0, nritems), nritems);
3584 }
3585 return ret;
Chris Masond4dbff92007-04-04 14:08:15 -04003586}
3587
Chris Mason99d8f832010-07-07 10:51:48 -04003588/*
3589 * min slot controls the lowest index we're willing to push to the
3590 * right. We'll push up to and including min_slot, but no lower
3591 */
Chris Mason44871b12009-03-13 10:04:31 -04003592static noinline int __push_leaf_right(struct btrfs_trans_handle *trans,
3593 struct btrfs_root *root,
3594 struct btrfs_path *path,
3595 int data_size, int empty,
3596 struct extent_buffer *right,
Chris Mason99d8f832010-07-07 10:51:48 -04003597 int free_space, u32 left_nritems,
3598 u32 min_slot)
Chris Mason00ec4c52007-02-24 12:47:20 -05003599{
Chris Mason5f39d392007-10-15 16:14:19 -04003600 struct extent_buffer *left = path->nodes[0];
Chris Mason44871b12009-03-13 10:04:31 -04003601 struct extent_buffer *upper = path->nodes[1];
Chris Masoncfed81a2012-03-03 07:40:03 -05003602 struct btrfs_map_token token;
Chris Mason5f39d392007-10-15 16:14:19 -04003603 struct btrfs_disk_key disk_key;
Chris Mason00ec4c52007-02-24 12:47:20 -05003604 int slot;
Chris Mason34a38212007-11-07 13:31:03 -05003605 u32 i;
Chris Mason00ec4c52007-02-24 12:47:20 -05003606 int push_space = 0;
3607 int push_items = 0;
Chris Mason0783fcf2007-03-12 20:12:07 -04003608 struct btrfs_item *item;
Chris Mason34a38212007-11-07 13:31:03 -05003609 u32 nr;
Chris Mason7518a232007-03-12 12:01:18 -04003610 u32 right_nritems;
Chris Mason5f39d392007-10-15 16:14:19 -04003611 u32 data_end;
Chris Masondb945352007-10-15 16:15:53 -04003612 u32 this_item_size;
Chris Mason00ec4c52007-02-24 12:47:20 -05003613
Chris Masoncfed81a2012-03-03 07:40:03 -05003614 btrfs_init_map_token(&token);
3615
Chris Mason34a38212007-11-07 13:31:03 -05003616 if (empty)
3617 nr = 0;
3618 else
Chris Mason99d8f832010-07-07 10:51:48 -04003619 nr = max_t(u32, 1, min_slot);
Chris Mason34a38212007-11-07 13:31:03 -05003620
Zheng Yan31840ae2008-09-23 13:14:14 -04003621 if (path->slots[0] >= left_nritems)
Yan Zheng87b29b22008-12-17 10:21:48 -05003622 push_space += data_size;
Zheng Yan31840ae2008-09-23 13:14:14 -04003623
Chris Mason44871b12009-03-13 10:04:31 -04003624 slot = path->slots[1];
Chris Mason34a38212007-11-07 13:31:03 -05003625 i = left_nritems - 1;
3626 while (i >= nr) {
Ross Kirkdd3cc162013-09-16 15:58:09 +01003627 item = btrfs_item_nr(i);
Chris Masondb945352007-10-15 16:15:53 -04003628
Zheng Yan31840ae2008-09-23 13:14:14 -04003629 if (!empty && push_items > 0) {
3630 if (path->slots[0] > i)
3631 break;
3632 if (path->slots[0] == i) {
3633 int space = btrfs_leaf_free_space(root, left);
3634 if (space + push_space * 2 > free_space)
3635 break;
3636 }
3637 }
3638
Chris Mason00ec4c52007-02-24 12:47:20 -05003639 if (path->slots[0] == i)
Yan Zheng87b29b22008-12-17 10:21:48 -05003640 push_space += data_size;
Chris Masondb945352007-10-15 16:15:53 -04003641
Chris Masondb945352007-10-15 16:15:53 -04003642 this_item_size = btrfs_item_size(left, item);
3643 if (this_item_size + sizeof(*item) + push_space > free_space)
Chris Mason00ec4c52007-02-24 12:47:20 -05003644 break;
Zheng Yan31840ae2008-09-23 13:14:14 -04003645
Chris Mason00ec4c52007-02-24 12:47:20 -05003646 push_items++;
Chris Masondb945352007-10-15 16:15:53 -04003647 push_space += this_item_size + sizeof(*item);
Chris Mason34a38212007-11-07 13:31:03 -05003648 if (i == 0)
3649 break;
3650 i--;
Chris Masondb945352007-10-15 16:15:53 -04003651 }
Chris Mason5f39d392007-10-15 16:14:19 -04003652
Chris Mason925baed2008-06-25 16:01:30 -04003653 if (push_items == 0)
3654 goto out_unlock;
Chris Mason5f39d392007-10-15 16:14:19 -04003655
Julia Lawall6c1500f2012-11-03 20:30:18 +00003656 WARN_ON(!empty && push_items == left_nritems);
Chris Mason5f39d392007-10-15 16:14:19 -04003657
Chris Mason00ec4c52007-02-24 12:47:20 -05003658 /* push left to right */
Chris Mason5f39d392007-10-15 16:14:19 -04003659 right_nritems = btrfs_header_nritems(right);
Chris Mason34a38212007-11-07 13:31:03 -05003660
Chris Mason5f39d392007-10-15 16:14:19 -04003661 push_space = btrfs_item_end_nr(left, left_nritems - push_items);
Chris Mason123abc82007-03-14 14:14:43 -04003662 push_space -= leaf_data_end(root, left);
Chris Mason5f39d392007-10-15 16:14:19 -04003663
Chris Mason00ec4c52007-02-24 12:47:20 -05003664 /* make room in the right data area */
Chris Mason5f39d392007-10-15 16:14:19 -04003665 data_end = leaf_data_end(root, right);
3666 memmove_extent_buffer(right,
3667 btrfs_leaf_data(right) + data_end - push_space,
3668 btrfs_leaf_data(right) + data_end,
3669 BTRFS_LEAF_DATA_SIZE(root) - data_end);
3670
Chris Mason00ec4c52007-02-24 12:47:20 -05003671 /* copy from the left data area */
Chris Mason5f39d392007-10-15 16:14:19 -04003672 copy_extent_buffer(right, left, btrfs_leaf_data(right) +
Chris Masond6025572007-03-30 14:27:56 -04003673 BTRFS_LEAF_DATA_SIZE(root) - push_space,
3674 btrfs_leaf_data(left) + leaf_data_end(root, left),
3675 push_space);
Chris Mason5f39d392007-10-15 16:14:19 -04003676
3677 memmove_extent_buffer(right, btrfs_item_nr_offset(push_items),
3678 btrfs_item_nr_offset(0),
3679 right_nritems * sizeof(struct btrfs_item));
3680
Chris Mason00ec4c52007-02-24 12:47:20 -05003681 /* copy the items from left to right */
Chris Mason5f39d392007-10-15 16:14:19 -04003682 copy_extent_buffer(right, left, btrfs_item_nr_offset(0),
3683 btrfs_item_nr_offset(left_nritems - push_items),
3684 push_items * sizeof(struct btrfs_item));
Chris Mason00ec4c52007-02-24 12:47:20 -05003685
3686 /* update the item pointers */
Chris Mason7518a232007-03-12 12:01:18 -04003687 right_nritems += push_items;
Chris Mason5f39d392007-10-15 16:14:19 -04003688 btrfs_set_header_nritems(right, right_nritems);
Chris Mason123abc82007-03-14 14:14:43 -04003689 push_space = BTRFS_LEAF_DATA_SIZE(root);
Chris Mason7518a232007-03-12 12:01:18 -04003690 for (i = 0; i < right_nritems; i++) {
Ross Kirkdd3cc162013-09-16 15:58:09 +01003691 item = btrfs_item_nr(i);
Chris Masoncfed81a2012-03-03 07:40:03 -05003692 push_space -= btrfs_token_item_size(right, item, &token);
3693 btrfs_set_token_item_offset(right, item, push_space, &token);
Chris Masondb945352007-10-15 16:15:53 -04003694 }
3695
Chris Mason7518a232007-03-12 12:01:18 -04003696 left_nritems -= push_items;
Chris Mason5f39d392007-10-15 16:14:19 -04003697 btrfs_set_header_nritems(left, left_nritems);
Chris Mason00ec4c52007-02-24 12:47:20 -05003698
Chris Mason34a38212007-11-07 13:31:03 -05003699 if (left_nritems)
3700 btrfs_mark_buffer_dirty(left);
Yan, Zhengf0486c62010-05-16 10:46:25 -04003701 else
Daniel Dressler01d58472014-11-21 17:15:07 +09003702 clean_tree_block(trans, root->fs_info, left);
Yan, Zhengf0486c62010-05-16 10:46:25 -04003703
Chris Mason5f39d392007-10-15 16:14:19 -04003704 btrfs_mark_buffer_dirty(right);
Chris Masona429e512007-04-18 16:15:28 -04003705
Chris Mason5f39d392007-10-15 16:14:19 -04003706 btrfs_item_key(right, &disk_key, 0);
3707 btrfs_set_node_key(upper, &disk_key, slot + 1);
Chris Masond6025572007-03-30 14:27:56 -04003708 btrfs_mark_buffer_dirty(upper);
Chris Mason02217ed2007-03-02 16:08:05 -05003709
Chris Mason00ec4c52007-02-24 12:47:20 -05003710 /* then fixup the leaf pointer in the path */
Chris Mason7518a232007-03-12 12:01:18 -04003711 if (path->slots[0] >= left_nritems) {
3712 path->slots[0] -= left_nritems;
Chris Mason925baed2008-06-25 16:01:30 -04003713 if (btrfs_header_nritems(path->nodes[0]) == 0)
Daniel Dressler01d58472014-11-21 17:15:07 +09003714 clean_tree_block(trans, root->fs_info, path->nodes[0]);
Chris Mason925baed2008-06-25 16:01:30 -04003715 btrfs_tree_unlock(path->nodes[0]);
Chris Mason5f39d392007-10-15 16:14:19 -04003716 free_extent_buffer(path->nodes[0]);
3717 path->nodes[0] = right;
Chris Mason00ec4c52007-02-24 12:47:20 -05003718 path->slots[1] += 1;
3719 } else {
Chris Mason925baed2008-06-25 16:01:30 -04003720 btrfs_tree_unlock(right);
Chris Mason5f39d392007-10-15 16:14:19 -04003721 free_extent_buffer(right);
Chris Mason00ec4c52007-02-24 12:47:20 -05003722 }
3723 return 0;
Chris Mason925baed2008-06-25 16:01:30 -04003724
3725out_unlock:
3726 btrfs_tree_unlock(right);
3727 free_extent_buffer(right);
3728 return 1;
Chris Mason00ec4c52007-02-24 12:47:20 -05003729}
Chris Mason925baed2008-06-25 16:01:30 -04003730
Chris Mason00ec4c52007-02-24 12:47:20 -05003731/*
Chris Mason44871b12009-03-13 10:04:31 -04003732 * push some data in the path leaf to the right, trying to free up at
3733 * least data_size bytes. returns zero if the push worked, nonzero otherwise
3734 *
3735 * returns 1 if the push failed because the other node didn't have enough
3736 * room, 0 if everything worked out and < 0 if there were major errors.
Chris Mason99d8f832010-07-07 10:51:48 -04003737 *
3738 * this will push starting from min_slot to the end of the leaf. It won't
3739 * push any slot lower than min_slot
Chris Mason44871b12009-03-13 10:04:31 -04003740 */
3741static int push_leaf_right(struct btrfs_trans_handle *trans, struct btrfs_root
Chris Mason99d8f832010-07-07 10:51:48 -04003742 *root, struct btrfs_path *path,
3743 int min_data_size, int data_size,
3744 int empty, u32 min_slot)
Chris Mason44871b12009-03-13 10:04:31 -04003745{
3746 struct extent_buffer *left = path->nodes[0];
3747 struct extent_buffer *right;
3748 struct extent_buffer *upper;
3749 int slot;
3750 int free_space;
3751 u32 left_nritems;
3752 int ret;
3753
3754 if (!path->nodes[1])
3755 return 1;
3756
3757 slot = path->slots[1];
3758 upper = path->nodes[1];
3759 if (slot >= btrfs_header_nritems(upper) - 1)
3760 return 1;
3761
3762 btrfs_assert_tree_locked(path->nodes[1]);
3763
3764 right = read_node_slot(root, upper, slot + 1);
Tsutomu Itoh91ca3382011-01-05 02:32:22 +00003765 if (right == NULL)
3766 return 1;
3767
Chris Mason44871b12009-03-13 10:04:31 -04003768 btrfs_tree_lock(right);
3769 btrfs_set_lock_blocking(right);
3770
3771 free_space = btrfs_leaf_free_space(root, right);
3772 if (free_space < data_size)
3773 goto out_unlock;
3774
3775 /* cow and double check */
3776 ret = btrfs_cow_block(trans, root, right, upper,
3777 slot + 1, &right);
3778 if (ret)
3779 goto out_unlock;
3780
3781 free_space = btrfs_leaf_free_space(root, right);
3782 if (free_space < data_size)
3783 goto out_unlock;
3784
3785 left_nritems = btrfs_header_nritems(left);
3786 if (left_nritems == 0)
3787 goto out_unlock;
3788
Filipe David Borba Manana2ef1fed2013-12-04 22:17:39 +00003789 if (path->slots[0] == left_nritems && !empty) {
3790 /* Key greater than all keys in the leaf, right neighbor has
3791 * enough room for it and we're not emptying our leaf to delete
3792 * it, therefore use right neighbor to insert the new item and
3793 * no need to touch/dirty our left leaft. */
3794 btrfs_tree_unlock(left);
3795 free_extent_buffer(left);
3796 path->nodes[0] = right;
3797 path->slots[0] = 0;
3798 path->slots[1]++;
3799 return 0;
3800 }
3801
Chris Mason99d8f832010-07-07 10:51:48 -04003802 return __push_leaf_right(trans, root, path, min_data_size, empty,
3803 right, free_space, left_nritems, min_slot);
Chris Mason44871b12009-03-13 10:04:31 -04003804out_unlock:
3805 btrfs_tree_unlock(right);
3806 free_extent_buffer(right);
3807 return 1;
3808}
3809
3810/*
Chris Mason74123bd2007-02-02 11:05:29 -05003811 * push some data in the path leaf to the left, trying to free up at
3812 * least data_size bytes. returns zero if the push worked, nonzero otherwise
Chris Mason99d8f832010-07-07 10:51:48 -04003813 *
3814 * max_slot can put a limit on how far into the leaf we'll push items. The
3815 * item at 'max_slot' won't be touched. Use (u32)-1 to make us do all the
3816 * items
Chris Mason74123bd2007-02-02 11:05:29 -05003817 */
Chris Mason44871b12009-03-13 10:04:31 -04003818static noinline int __push_leaf_left(struct btrfs_trans_handle *trans,
3819 struct btrfs_root *root,
3820 struct btrfs_path *path, int data_size,
3821 int empty, struct extent_buffer *left,
Chris Mason99d8f832010-07-07 10:51:48 -04003822 int free_space, u32 right_nritems,
3823 u32 max_slot)
Chris Masonbe0e5c02007-01-26 15:51:26 -05003824{
Chris Mason5f39d392007-10-15 16:14:19 -04003825 struct btrfs_disk_key disk_key;
3826 struct extent_buffer *right = path->nodes[0];
Chris Masonbe0e5c02007-01-26 15:51:26 -05003827 int i;
Chris Masonbe0e5c02007-01-26 15:51:26 -05003828 int push_space = 0;
3829 int push_items = 0;
Chris Mason0783fcf2007-03-12 20:12:07 -04003830 struct btrfs_item *item;
Chris Mason7518a232007-03-12 12:01:18 -04003831 u32 old_left_nritems;
Chris Mason34a38212007-11-07 13:31:03 -05003832 u32 nr;
Chris Masonaa5d6be2007-02-28 16:35:06 -05003833 int ret = 0;
Chris Masondb945352007-10-15 16:15:53 -04003834 u32 this_item_size;
3835 u32 old_left_item_size;
Chris Masoncfed81a2012-03-03 07:40:03 -05003836 struct btrfs_map_token token;
3837
3838 btrfs_init_map_token(&token);
Chris Masonbe0e5c02007-01-26 15:51:26 -05003839
Chris Mason34a38212007-11-07 13:31:03 -05003840 if (empty)
Chris Mason99d8f832010-07-07 10:51:48 -04003841 nr = min(right_nritems, max_slot);
Chris Mason34a38212007-11-07 13:31:03 -05003842 else
Chris Mason99d8f832010-07-07 10:51:48 -04003843 nr = min(right_nritems - 1, max_slot);
Chris Mason34a38212007-11-07 13:31:03 -05003844
3845 for (i = 0; i < nr; i++) {
Ross Kirkdd3cc162013-09-16 15:58:09 +01003846 item = btrfs_item_nr(i);
Chris Masondb945352007-10-15 16:15:53 -04003847
Zheng Yan31840ae2008-09-23 13:14:14 -04003848 if (!empty && push_items > 0) {
3849 if (path->slots[0] < i)
3850 break;
3851 if (path->slots[0] == i) {
3852 int space = btrfs_leaf_free_space(root, right);
3853 if (space + push_space * 2 > free_space)
3854 break;
3855 }
3856 }
3857
Chris Masonbe0e5c02007-01-26 15:51:26 -05003858 if (path->slots[0] == i)
Yan Zheng87b29b22008-12-17 10:21:48 -05003859 push_space += data_size;
Chris Masondb945352007-10-15 16:15:53 -04003860
3861 this_item_size = btrfs_item_size(right, item);
3862 if (this_item_size + sizeof(*item) + push_space > free_space)
Chris Masonbe0e5c02007-01-26 15:51:26 -05003863 break;
Chris Masondb945352007-10-15 16:15:53 -04003864
Chris Masonbe0e5c02007-01-26 15:51:26 -05003865 push_items++;
Chris Masondb945352007-10-15 16:15:53 -04003866 push_space += this_item_size + sizeof(*item);
Chris Masonbe0e5c02007-01-26 15:51:26 -05003867 }
Chris Masondb945352007-10-15 16:15:53 -04003868
Chris Masonbe0e5c02007-01-26 15:51:26 -05003869 if (push_items == 0) {
Chris Mason925baed2008-06-25 16:01:30 -04003870 ret = 1;
3871 goto out;
Chris Masonbe0e5c02007-01-26 15:51:26 -05003872 }
Dulshani Gunawardhanafae7f212013-10-31 10:30:08 +05303873 WARN_ON(!empty && push_items == btrfs_header_nritems(right));
Chris Mason5f39d392007-10-15 16:14:19 -04003874
Chris Masonbe0e5c02007-01-26 15:51:26 -05003875 /* push data from right to left */
Chris Mason5f39d392007-10-15 16:14:19 -04003876 copy_extent_buffer(left, right,
3877 btrfs_item_nr_offset(btrfs_header_nritems(left)),
3878 btrfs_item_nr_offset(0),
3879 push_items * sizeof(struct btrfs_item));
3880
Chris Mason123abc82007-03-14 14:14:43 -04003881 push_space = BTRFS_LEAF_DATA_SIZE(root) -
Chris Masond3977122009-01-05 21:25:51 -05003882 btrfs_item_offset_nr(right, push_items - 1);
Chris Mason5f39d392007-10-15 16:14:19 -04003883
3884 copy_extent_buffer(left, right, btrfs_leaf_data(left) +
Chris Masond6025572007-03-30 14:27:56 -04003885 leaf_data_end(root, left) - push_space,
3886 btrfs_leaf_data(right) +
Chris Mason5f39d392007-10-15 16:14:19 -04003887 btrfs_item_offset_nr(right, push_items - 1),
Chris Masond6025572007-03-30 14:27:56 -04003888 push_space);
Chris Mason5f39d392007-10-15 16:14:19 -04003889 old_left_nritems = btrfs_header_nritems(left);
Yan Zheng87b29b22008-12-17 10:21:48 -05003890 BUG_ON(old_left_nritems <= 0);
Chris Masoneb60cea2007-02-02 09:18:22 -05003891
Chris Masondb945352007-10-15 16:15:53 -04003892 old_left_item_size = btrfs_item_offset_nr(left, old_left_nritems - 1);
Chris Mason0783fcf2007-03-12 20:12:07 -04003893 for (i = old_left_nritems; i < old_left_nritems + push_items; i++) {
Chris Mason5f39d392007-10-15 16:14:19 -04003894 u32 ioff;
Chris Masondb945352007-10-15 16:15:53 -04003895
Ross Kirkdd3cc162013-09-16 15:58:09 +01003896 item = btrfs_item_nr(i);
Chris Masondb945352007-10-15 16:15:53 -04003897
Chris Masoncfed81a2012-03-03 07:40:03 -05003898 ioff = btrfs_token_item_offset(left, item, &token);
3899 btrfs_set_token_item_offset(left, item,
3900 ioff - (BTRFS_LEAF_DATA_SIZE(root) - old_left_item_size),
3901 &token);
Chris Masonbe0e5c02007-01-26 15:51:26 -05003902 }
Chris Mason5f39d392007-10-15 16:14:19 -04003903 btrfs_set_header_nritems(left, old_left_nritems + push_items);
Chris Masonbe0e5c02007-01-26 15:51:26 -05003904
3905 /* fixup right node */
Julia Lawall31b1a2b2012-11-03 10:58:34 +00003906 if (push_items > right_nritems)
3907 WARN(1, KERN_CRIT "push items %d nr %u\n", push_items,
Chris Masond3977122009-01-05 21:25:51 -05003908 right_nritems);
Chris Mason5f39d392007-10-15 16:14:19 -04003909
Chris Mason34a38212007-11-07 13:31:03 -05003910 if (push_items < right_nritems) {
3911 push_space = btrfs_item_offset_nr(right, push_items - 1) -
3912 leaf_data_end(root, right);
3913 memmove_extent_buffer(right, btrfs_leaf_data(right) +
3914 BTRFS_LEAF_DATA_SIZE(root) - push_space,
3915 btrfs_leaf_data(right) +
3916 leaf_data_end(root, right), push_space);
3917
3918 memmove_extent_buffer(right, btrfs_item_nr_offset(0),
Chris Mason5f39d392007-10-15 16:14:19 -04003919 btrfs_item_nr_offset(push_items),
3920 (btrfs_header_nritems(right) - push_items) *
3921 sizeof(struct btrfs_item));
Chris Mason34a38212007-11-07 13:31:03 -05003922 }
Yaneef1c492007-11-26 10:58:13 -05003923 right_nritems -= push_items;
3924 btrfs_set_header_nritems(right, right_nritems);
Chris Mason123abc82007-03-14 14:14:43 -04003925 push_space = BTRFS_LEAF_DATA_SIZE(root);
Chris Mason5f39d392007-10-15 16:14:19 -04003926 for (i = 0; i < right_nritems; i++) {
Ross Kirkdd3cc162013-09-16 15:58:09 +01003927 item = btrfs_item_nr(i);
Chris Masondb945352007-10-15 16:15:53 -04003928
Chris Masoncfed81a2012-03-03 07:40:03 -05003929 push_space = push_space - btrfs_token_item_size(right,
3930 item, &token);
3931 btrfs_set_token_item_offset(right, item, push_space, &token);
Chris Masondb945352007-10-15 16:15:53 -04003932 }
Chris Masoneb60cea2007-02-02 09:18:22 -05003933
Chris Mason5f39d392007-10-15 16:14:19 -04003934 btrfs_mark_buffer_dirty(left);
Chris Mason34a38212007-11-07 13:31:03 -05003935 if (right_nritems)
3936 btrfs_mark_buffer_dirty(right);
Yan, Zhengf0486c62010-05-16 10:46:25 -04003937 else
Daniel Dressler01d58472014-11-21 17:15:07 +09003938 clean_tree_block(trans, root->fs_info, right);
Chris Mason098f59c2007-05-11 11:33:21 -04003939
Chris Mason5f39d392007-10-15 16:14:19 -04003940 btrfs_item_key(right, &disk_key, 0);
Daniel Dresslerb7a03652014-11-12 13:43:09 +09003941 fixup_low_keys(root->fs_info, path, &disk_key, 1);
Chris Masonbe0e5c02007-01-26 15:51:26 -05003942
3943 /* then fixup the leaf pointer in the path */
3944 if (path->slots[0] < push_items) {
3945 path->slots[0] += old_left_nritems;
Chris Mason925baed2008-06-25 16:01:30 -04003946 btrfs_tree_unlock(path->nodes[0]);
Chris Mason5f39d392007-10-15 16:14:19 -04003947 free_extent_buffer(path->nodes[0]);
3948 path->nodes[0] = left;
Chris Masonbe0e5c02007-01-26 15:51:26 -05003949 path->slots[1] -= 1;
3950 } else {
Chris Mason925baed2008-06-25 16:01:30 -04003951 btrfs_tree_unlock(left);
Chris Mason5f39d392007-10-15 16:14:19 -04003952 free_extent_buffer(left);
Chris Masonbe0e5c02007-01-26 15:51:26 -05003953 path->slots[0] -= push_items;
3954 }
Chris Masoneb60cea2007-02-02 09:18:22 -05003955 BUG_ON(path->slots[0] < 0);
Chris Masonaa5d6be2007-02-28 16:35:06 -05003956 return ret;
Chris Mason925baed2008-06-25 16:01:30 -04003957out:
3958 btrfs_tree_unlock(left);
3959 free_extent_buffer(left);
3960 return ret;
Chris Masonbe0e5c02007-01-26 15:51:26 -05003961}
3962
Chris Mason74123bd2007-02-02 11:05:29 -05003963/*
Chris Mason44871b12009-03-13 10:04:31 -04003964 * push some data in the path leaf to the left, trying to free up at
3965 * least data_size bytes. returns zero if the push worked, nonzero otherwise
Chris Mason99d8f832010-07-07 10:51:48 -04003966 *
3967 * max_slot can put a limit on how far into the leaf we'll push items. The
3968 * item at 'max_slot' won't be touched. Use (u32)-1 to make us push all the
3969 * items
Chris Mason44871b12009-03-13 10:04:31 -04003970 */
3971static int push_leaf_left(struct btrfs_trans_handle *trans, struct btrfs_root
Chris Mason99d8f832010-07-07 10:51:48 -04003972 *root, struct btrfs_path *path, int min_data_size,
3973 int data_size, int empty, u32 max_slot)
Chris Mason44871b12009-03-13 10:04:31 -04003974{
3975 struct extent_buffer *right = path->nodes[0];
3976 struct extent_buffer *left;
3977 int slot;
3978 int free_space;
3979 u32 right_nritems;
3980 int ret = 0;
3981
3982 slot = path->slots[1];
3983 if (slot == 0)
3984 return 1;
3985 if (!path->nodes[1])
3986 return 1;
3987
3988 right_nritems = btrfs_header_nritems(right);
3989 if (right_nritems == 0)
3990 return 1;
3991
3992 btrfs_assert_tree_locked(path->nodes[1]);
3993
3994 left = read_node_slot(root, path->nodes[1], slot - 1);
Tsutomu Itoh91ca3382011-01-05 02:32:22 +00003995 if (left == NULL)
3996 return 1;
3997
Chris Mason44871b12009-03-13 10:04:31 -04003998 btrfs_tree_lock(left);
3999 btrfs_set_lock_blocking(left);
4000
4001 free_space = btrfs_leaf_free_space(root, left);
4002 if (free_space < data_size) {
4003 ret = 1;
4004 goto out;
4005 }
4006
4007 /* cow and double check */
4008 ret = btrfs_cow_block(trans, root, left,
4009 path->nodes[1], slot - 1, &left);
4010 if (ret) {
4011 /* we hit -ENOSPC, but it isn't fatal here */
Jeff Mahoney79787ea2012-03-12 16:03:00 +01004012 if (ret == -ENOSPC)
4013 ret = 1;
Chris Mason44871b12009-03-13 10:04:31 -04004014 goto out;
4015 }
4016
4017 free_space = btrfs_leaf_free_space(root, left);
4018 if (free_space < data_size) {
4019 ret = 1;
4020 goto out;
4021 }
4022
Chris Mason99d8f832010-07-07 10:51:48 -04004023 return __push_leaf_left(trans, root, path, min_data_size,
4024 empty, left, free_space, right_nritems,
4025 max_slot);
Chris Mason44871b12009-03-13 10:04:31 -04004026out:
4027 btrfs_tree_unlock(left);
4028 free_extent_buffer(left);
4029 return ret;
4030}
4031
4032/*
Chris Mason74123bd2007-02-02 11:05:29 -05004033 * split the path's leaf in two, making sure there is at least data_size
4034 * available for the resulting leaf level of the path.
4035 */
Jeff Mahoney143bede2012-03-01 14:56:26 +01004036static noinline void copy_for_split(struct btrfs_trans_handle *trans,
4037 struct btrfs_root *root,
4038 struct btrfs_path *path,
4039 struct extent_buffer *l,
4040 struct extent_buffer *right,
4041 int slot, int mid, int nritems)
Chris Masonbe0e5c02007-01-26 15:51:26 -05004042{
Chris Masonbe0e5c02007-01-26 15:51:26 -05004043 int data_copy_size;
4044 int rt_data_off;
4045 int i;
Chris Masond4dbff92007-04-04 14:08:15 -04004046 struct btrfs_disk_key disk_key;
Chris Masoncfed81a2012-03-03 07:40:03 -05004047 struct btrfs_map_token token;
4048
4049 btrfs_init_map_token(&token);
Chris Masonbe0e5c02007-01-26 15:51:26 -05004050
Chris Mason5f39d392007-10-15 16:14:19 -04004051 nritems = nritems - mid;
4052 btrfs_set_header_nritems(right, nritems);
4053 data_copy_size = btrfs_item_end_nr(l, mid) - leaf_data_end(root, l);
4054
4055 copy_extent_buffer(right, l, btrfs_item_nr_offset(0),
4056 btrfs_item_nr_offset(mid),
4057 nritems * sizeof(struct btrfs_item));
4058
4059 copy_extent_buffer(right, l,
Chris Masond6025572007-03-30 14:27:56 -04004060 btrfs_leaf_data(right) + BTRFS_LEAF_DATA_SIZE(root) -
4061 data_copy_size, btrfs_leaf_data(l) +
4062 leaf_data_end(root, l), data_copy_size);
Chris Mason74123bd2007-02-02 11:05:29 -05004063
Chris Mason5f39d392007-10-15 16:14:19 -04004064 rt_data_off = BTRFS_LEAF_DATA_SIZE(root) -
4065 btrfs_item_end_nr(l, mid);
4066
4067 for (i = 0; i < nritems; i++) {
Ross Kirkdd3cc162013-09-16 15:58:09 +01004068 struct btrfs_item *item = btrfs_item_nr(i);
Chris Masondb945352007-10-15 16:15:53 -04004069 u32 ioff;
4070
Chris Masoncfed81a2012-03-03 07:40:03 -05004071 ioff = btrfs_token_item_offset(right, item, &token);
4072 btrfs_set_token_item_offset(right, item,
4073 ioff + rt_data_off, &token);
Chris Mason0783fcf2007-03-12 20:12:07 -04004074 }
Chris Mason74123bd2007-02-02 11:05:29 -05004075
Chris Mason5f39d392007-10-15 16:14:19 -04004076 btrfs_set_header_nritems(l, mid);
Chris Mason5f39d392007-10-15 16:14:19 -04004077 btrfs_item_key(right, &disk_key, 0);
Jeff Mahoney143bede2012-03-01 14:56:26 +01004078 insert_ptr(trans, root, path, &disk_key, right->start,
Jan Schmidtc3e06962012-06-21 11:01:06 +02004079 path->slots[1] + 1, 1);
Chris Mason5f39d392007-10-15 16:14:19 -04004080
4081 btrfs_mark_buffer_dirty(right);
4082 btrfs_mark_buffer_dirty(l);
Chris Masoneb60cea2007-02-02 09:18:22 -05004083 BUG_ON(path->slots[0] != slot);
Chris Mason5f39d392007-10-15 16:14:19 -04004084
Chris Masonbe0e5c02007-01-26 15:51:26 -05004085 if (mid <= slot) {
Chris Mason925baed2008-06-25 16:01:30 -04004086 btrfs_tree_unlock(path->nodes[0]);
Chris Mason5f39d392007-10-15 16:14:19 -04004087 free_extent_buffer(path->nodes[0]);
4088 path->nodes[0] = right;
Chris Masonbe0e5c02007-01-26 15:51:26 -05004089 path->slots[0] -= mid;
4090 path->slots[1] += 1;
Chris Mason925baed2008-06-25 16:01:30 -04004091 } else {
4092 btrfs_tree_unlock(right);
Chris Mason5f39d392007-10-15 16:14:19 -04004093 free_extent_buffer(right);
Chris Mason925baed2008-06-25 16:01:30 -04004094 }
Chris Mason5f39d392007-10-15 16:14:19 -04004095
Chris Masoneb60cea2007-02-02 09:18:22 -05004096 BUG_ON(path->slots[0] < 0);
Chris Mason44871b12009-03-13 10:04:31 -04004097}
4098
4099/*
Chris Mason99d8f832010-07-07 10:51:48 -04004100 * double splits happen when we need to insert a big item in the middle
4101 * of a leaf. A double split can leave us with 3 mostly empty leaves:
4102 * leaf: [ slots 0 - N] [ our target ] [ N + 1 - total in leaf ]
4103 * A B C
4104 *
4105 * We avoid this by trying to push the items on either side of our target
4106 * into the adjacent leaves. If all goes well we can avoid the double split
4107 * completely.
4108 */
4109static noinline int push_for_double_split(struct btrfs_trans_handle *trans,
4110 struct btrfs_root *root,
4111 struct btrfs_path *path,
4112 int data_size)
4113{
4114 int ret;
4115 int progress = 0;
4116 int slot;
4117 u32 nritems;
Filipe David Borba Manana5a4267c2013-11-25 03:20:46 +00004118 int space_needed = data_size;
Chris Mason99d8f832010-07-07 10:51:48 -04004119
4120 slot = path->slots[0];
Filipe David Borba Manana5a4267c2013-11-25 03:20:46 +00004121 if (slot < btrfs_header_nritems(path->nodes[0]))
4122 space_needed -= btrfs_leaf_free_space(root, path->nodes[0]);
Chris Mason99d8f832010-07-07 10:51:48 -04004123
4124 /*
4125 * try to push all the items after our slot into the
4126 * right leaf
4127 */
Filipe David Borba Manana5a4267c2013-11-25 03:20:46 +00004128 ret = push_leaf_right(trans, root, path, 1, space_needed, 0, slot);
Chris Mason99d8f832010-07-07 10:51:48 -04004129 if (ret < 0)
4130 return ret;
4131
4132 if (ret == 0)
4133 progress++;
4134
4135 nritems = btrfs_header_nritems(path->nodes[0]);
4136 /*
4137 * our goal is to get our slot at the start or end of a leaf. If
4138 * we've done so we're done
4139 */
4140 if (path->slots[0] == 0 || path->slots[0] == nritems)
4141 return 0;
4142
4143 if (btrfs_leaf_free_space(root, path->nodes[0]) >= data_size)
4144 return 0;
4145
4146 /* try to push all the items before our slot into the next leaf */
4147 slot = path->slots[0];
Filipe David Borba Manana5a4267c2013-11-25 03:20:46 +00004148 ret = push_leaf_left(trans, root, path, 1, space_needed, 0, slot);
Chris Mason99d8f832010-07-07 10:51:48 -04004149 if (ret < 0)
4150 return ret;
4151
4152 if (ret == 0)
4153 progress++;
4154
4155 if (progress)
4156 return 0;
4157 return 1;
4158}
4159
4160/*
Chris Mason44871b12009-03-13 10:04:31 -04004161 * split the path's leaf in two, making sure there is at least data_size
4162 * available for the resulting leaf level of the path.
4163 *
4164 * returns 0 if all went well and < 0 on failure.
4165 */
4166static noinline int split_leaf(struct btrfs_trans_handle *trans,
4167 struct btrfs_root *root,
4168 struct btrfs_key *ins_key,
4169 struct btrfs_path *path, int data_size,
4170 int extend)
4171{
Yan Zheng5d4f98a2009-06-10 10:45:14 -04004172 struct btrfs_disk_key disk_key;
Chris Mason44871b12009-03-13 10:04:31 -04004173 struct extent_buffer *l;
4174 u32 nritems;
4175 int mid;
4176 int slot;
4177 struct extent_buffer *right;
Daniel Dresslerb7a03652014-11-12 13:43:09 +09004178 struct btrfs_fs_info *fs_info = root->fs_info;
Chris Mason44871b12009-03-13 10:04:31 -04004179 int ret = 0;
4180 int wret;
Yan Zheng5d4f98a2009-06-10 10:45:14 -04004181 int split;
Chris Mason44871b12009-03-13 10:04:31 -04004182 int num_doubles = 0;
Chris Mason99d8f832010-07-07 10:51:48 -04004183 int tried_avoid_double = 0;
Chris Mason44871b12009-03-13 10:04:31 -04004184
Yan, Zhenga5719522009-09-24 09:17:31 -04004185 l = path->nodes[0];
4186 slot = path->slots[0];
4187 if (extend && data_size + btrfs_item_size_nr(l, slot) +
4188 sizeof(struct btrfs_item) > BTRFS_LEAF_DATA_SIZE(root))
4189 return -EOVERFLOW;
4190
Chris Mason44871b12009-03-13 10:04:31 -04004191 /* first try to make some room by pushing left and right */
Liu Bo33157e02013-05-22 12:07:06 +00004192 if (data_size && path->nodes[1]) {
Filipe David Borba Manana5a4267c2013-11-25 03:20:46 +00004193 int space_needed = data_size;
4194
4195 if (slot < btrfs_header_nritems(l))
4196 space_needed -= btrfs_leaf_free_space(root, l);
4197
4198 wret = push_leaf_right(trans, root, path, space_needed,
4199 space_needed, 0, 0);
Chris Mason44871b12009-03-13 10:04:31 -04004200 if (wret < 0)
4201 return wret;
4202 if (wret) {
Filipe David Borba Manana5a4267c2013-11-25 03:20:46 +00004203 wret = push_leaf_left(trans, root, path, space_needed,
4204 space_needed, 0, (u32)-1);
Chris Mason44871b12009-03-13 10:04:31 -04004205 if (wret < 0)
4206 return wret;
4207 }
4208 l = path->nodes[0];
4209
4210 /* did the pushes work? */
4211 if (btrfs_leaf_free_space(root, l) >= data_size)
4212 return 0;
4213 }
4214
4215 if (!path->nodes[1]) {
Liu Bofdd99c72013-05-22 12:06:51 +00004216 ret = insert_new_root(trans, root, path, 1);
Chris Mason44871b12009-03-13 10:04:31 -04004217 if (ret)
4218 return ret;
4219 }
4220again:
Yan Zheng5d4f98a2009-06-10 10:45:14 -04004221 split = 1;
Chris Mason44871b12009-03-13 10:04:31 -04004222 l = path->nodes[0];
4223 slot = path->slots[0];
4224 nritems = btrfs_header_nritems(l);
4225 mid = (nritems + 1) / 2;
4226
Yan Zheng5d4f98a2009-06-10 10:45:14 -04004227 if (mid <= slot) {
4228 if (nritems == 1 ||
4229 leaf_space_used(l, mid, nritems - mid) + data_size >
4230 BTRFS_LEAF_DATA_SIZE(root)) {
4231 if (slot >= nritems) {
4232 split = 0;
4233 } else {
4234 mid = slot;
4235 if (mid != nritems &&
4236 leaf_space_used(l, mid, nritems - mid) +
4237 data_size > BTRFS_LEAF_DATA_SIZE(root)) {
Chris Mason99d8f832010-07-07 10:51:48 -04004238 if (data_size && !tried_avoid_double)
4239 goto push_for_double;
Yan Zheng5d4f98a2009-06-10 10:45:14 -04004240 split = 2;
4241 }
4242 }
4243 }
4244 } else {
4245 if (leaf_space_used(l, 0, mid) + data_size >
4246 BTRFS_LEAF_DATA_SIZE(root)) {
4247 if (!extend && data_size && slot == 0) {
4248 split = 0;
4249 } else if ((extend || !data_size) && slot == 0) {
4250 mid = 1;
4251 } else {
4252 mid = slot;
4253 if (mid != nritems &&
4254 leaf_space_used(l, mid, nritems - mid) +
4255 data_size > BTRFS_LEAF_DATA_SIZE(root)) {
Chris Mason99d8f832010-07-07 10:51:48 -04004256 if (data_size && !tried_avoid_double)
4257 goto push_for_double;
Dulshani Gunawardhana67871252013-10-31 10:33:04 +05304258 split = 2;
Yan Zheng5d4f98a2009-06-10 10:45:14 -04004259 }
4260 }
4261 }
4262 }
4263
4264 if (split == 0)
4265 btrfs_cpu_key_to_disk(&disk_key, ins_key);
4266 else
4267 btrfs_item_key(l, &disk_key, mid);
4268
David Sterba4d75f8a2014-06-15 01:54:12 +02004269 right = btrfs_alloc_tree_block(trans, root, 0, root->root_key.objectid,
4270 &disk_key, 0, l->start, 0);
Yan, Zhengf0486c62010-05-16 10:46:25 -04004271 if (IS_ERR(right))
Chris Mason44871b12009-03-13 10:04:31 -04004272 return PTR_ERR(right);
Yan, Zhengf0486c62010-05-16 10:46:25 -04004273
David Sterba707e8a02014-06-04 19:22:26 +02004274 root_add_used(root, root->nodesize);
Chris Mason44871b12009-03-13 10:04:31 -04004275
4276 memset_extent_buffer(right, 0, 0, sizeof(struct btrfs_header));
4277 btrfs_set_header_bytenr(right, right->start);
4278 btrfs_set_header_generation(right, trans->transid);
Yan Zheng5d4f98a2009-06-10 10:45:14 -04004279 btrfs_set_header_backref_rev(right, BTRFS_MIXED_BACKREF_REV);
Chris Mason44871b12009-03-13 10:04:31 -04004280 btrfs_set_header_owner(right, root->root_key.objectid);
4281 btrfs_set_header_level(right, 0);
Daniel Dresslerb7a03652014-11-12 13:43:09 +09004282 write_extent_buffer(right, fs_info->fsid,
Ross Kirk0a4e5582013-09-24 10:12:38 +01004283 btrfs_header_fsid(), BTRFS_FSID_SIZE);
Chris Mason44871b12009-03-13 10:04:31 -04004284
Daniel Dresslerb7a03652014-11-12 13:43:09 +09004285 write_extent_buffer(right, fs_info->chunk_tree_uuid,
Geert Uytterhoevenb308bc22013-08-20 13:20:15 +02004286 btrfs_header_chunk_tree_uuid(right),
Chris Mason44871b12009-03-13 10:04:31 -04004287 BTRFS_UUID_SIZE);
4288
Yan Zheng5d4f98a2009-06-10 10:45:14 -04004289 if (split == 0) {
4290 if (mid <= slot) {
4291 btrfs_set_header_nritems(right, 0);
Jeff Mahoney143bede2012-03-01 14:56:26 +01004292 insert_ptr(trans, root, path, &disk_key, right->start,
Jan Schmidtc3e06962012-06-21 11:01:06 +02004293 path->slots[1] + 1, 1);
Yan Zheng5d4f98a2009-06-10 10:45:14 -04004294 btrfs_tree_unlock(path->nodes[0]);
4295 free_extent_buffer(path->nodes[0]);
4296 path->nodes[0] = right;
4297 path->slots[0] = 0;
4298 path->slots[1] += 1;
4299 } else {
4300 btrfs_set_header_nritems(right, 0);
Jeff Mahoney143bede2012-03-01 14:56:26 +01004301 insert_ptr(trans, root, path, &disk_key, right->start,
Jan Schmidtc3e06962012-06-21 11:01:06 +02004302 path->slots[1], 1);
Yan Zheng5d4f98a2009-06-10 10:45:14 -04004303 btrfs_tree_unlock(path->nodes[0]);
4304 free_extent_buffer(path->nodes[0]);
4305 path->nodes[0] = right;
4306 path->slots[0] = 0;
Jeff Mahoney143bede2012-03-01 14:56:26 +01004307 if (path->slots[1] == 0)
Daniel Dresslerb7a03652014-11-12 13:43:09 +09004308 fixup_low_keys(fs_info, path, &disk_key, 1);
Chris Mason44871b12009-03-13 10:04:31 -04004309 }
Yan Zheng5d4f98a2009-06-10 10:45:14 -04004310 btrfs_mark_buffer_dirty(right);
4311 return ret;
Chris Mason44871b12009-03-13 10:04:31 -04004312 }
4313
Jeff Mahoney143bede2012-03-01 14:56:26 +01004314 copy_for_split(trans, root, path, l, right, slot, mid, nritems);
Chris Mason44871b12009-03-13 10:04:31 -04004315
Yan Zheng5d4f98a2009-06-10 10:45:14 -04004316 if (split == 2) {
Chris Masoncc0c5532007-10-25 15:42:57 -04004317 BUG_ON(num_doubles != 0);
4318 num_doubles++;
4319 goto again;
Chris Mason3326d1b2007-10-15 16:18:25 -04004320 }
Chris Mason44871b12009-03-13 10:04:31 -04004321
Jeff Mahoney143bede2012-03-01 14:56:26 +01004322 return 0;
Chris Mason99d8f832010-07-07 10:51:48 -04004323
4324push_for_double:
4325 push_for_double_split(trans, root, path, data_size);
4326 tried_avoid_double = 1;
4327 if (btrfs_leaf_free_space(root, path->nodes[0]) >= data_size)
4328 return 0;
4329 goto again;
Chris Masonbe0e5c02007-01-26 15:51:26 -05004330}
4331
Yan, Zhengad48fd752009-11-12 09:33:58 +00004332static noinline int setup_leaf_for_split(struct btrfs_trans_handle *trans,
4333 struct btrfs_root *root,
4334 struct btrfs_path *path, int ins_len)
Chris Mason459931e2008-12-10 09:10:46 -05004335{
Yan, Zhengad48fd752009-11-12 09:33:58 +00004336 struct btrfs_key key;
Chris Mason459931e2008-12-10 09:10:46 -05004337 struct extent_buffer *leaf;
Yan, Zhengad48fd752009-11-12 09:33:58 +00004338 struct btrfs_file_extent_item *fi;
4339 u64 extent_len = 0;
4340 u32 item_size;
4341 int ret;
Chris Mason459931e2008-12-10 09:10:46 -05004342
4343 leaf = path->nodes[0];
Yan, Zhengad48fd752009-11-12 09:33:58 +00004344 btrfs_item_key_to_cpu(leaf, &key, path->slots[0]);
4345
4346 BUG_ON(key.type != BTRFS_EXTENT_DATA_KEY &&
4347 key.type != BTRFS_EXTENT_CSUM_KEY);
4348
4349 if (btrfs_leaf_free_space(root, leaf) >= ins_len)
4350 return 0;
Chris Mason459931e2008-12-10 09:10:46 -05004351
4352 item_size = btrfs_item_size_nr(leaf, path->slots[0]);
Yan, Zhengad48fd752009-11-12 09:33:58 +00004353 if (key.type == BTRFS_EXTENT_DATA_KEY) {
4354 fi = btrfs_item_ptr(leaf, path->slots[0],
4355 struct btrfs_file_extent_item);
4356 extent_len = btrfs_file_extent_num_bytes(leaf, fi);
4357 }
David Sterbab3b4aa72011-04-21 01:20:15 +02004358 btrfs_release_path(path);
Chris Mason459931e2008-12-10 09:10:46 -05004359
Chris Mason459931e2008-12-10 09:10:46 -05004360 path->keep_locks = 1;
Yan, Zhengad48fd752009-11-12 09:33:58 +00004361 path->search_for_split = 1;
4362 ret = btrfs_search_slot(trans, root, &key, path, 0, 1);
Chris Mason459931e2008-12-10 09:10:46 -05004363 path->search_for_split = 0;
Filipe Mananaa8df6fe2015-01-20 12:40:53 +00004364 if (ret > 0)
4365 ret = -EAGAIN;
Yan, Zhengad48fd752009-11-12 09:33:58 +00004366 if (ret < 0)
4367 goto err;
Chris Mason459931e2008-12-10 09:10:46 -05004368
Yan, Zhengad48fd752009-11-12 09:33:58 +00004369 ret = -EAGAIN;
4370 leaf = path->nodes[0];
Filipe Mananaa8df6fe2015-01-20 12:40:53 +00004371 /* if our item isn't there, return now */
4372 if (item_size != btrfs_item_size_nr(leaf, path->slots[0]))
Yan, Zhengad48fd752009-11-12 09:33:58 +00004373 goto err;
4374
Chris Mason109f6ae2010-04-02 09:20:18 -04004375 /* the leaf has changed, it now has room. return now */
4376 if (btrfs_leaf_free_space(root, path->nodes[0]) >= ins_len)
4377 goto err;
4378
Yan, Zhengad48fd752009-11-12 09:33:58 +00004379 if (key.type == BTRFS_EXTENT_DATA_KEY) {
4380 fi = btrfs_item_ptr(leaf, path->slots[0],
4381 struct btrfs_file_extent_item);
4382 if (extent_len != btrfs_file_extent_num_bytes(leaf, fi))
4383 goto err;
Chris Mason459931e2008-12-10 09:10:46 -05004384 }
4385
Chris Masonb9473432009-03-13 11:00:37 -04004386 btrfs_set_path_blocking(path);
Yan, Zhengad48fd752009-11-12 09:33:58 +00004387 ret = split_leaf(trans, root, &key, path, ins_len, 1);
Yan, Zhengf0486c62010-05-16 10:46:25 -04004388 if (ret)
4389 goto err;
Chris Mason459931e2008-12-10 09:10:46 -05004390
Yan, Zhengad48fd752009-11-12 09:33:58 +00004391 path->keep_locks = 0;
Chris Masonb9473432009-03-13 11:00:37 -04004392 btrfs_unlock_up_safe(path, 1);
Yan, Zhengad48fd752009-11-12 09:33:58 +00004393 return 0;
4394err:
4395 path->keep_locks = 0;
4396 return ret;
4397}
4398
4399static noinline int split_item(struct btrfs_trans_handle *trans,
4400 struct btrfs_root *root,
4401 struct btrfs_path *path,
4402 struct btrfs_key *new_key,
4403 unsigned long split_offset)
4404{
4405 struct extent_buffer *leaf;
4406 struct btrfs_item *item;
4407 struct btrfs_item *new_item;
4408 int slot;
4409 char *buf;
4410 u32 nritems;
4411 u32 item_size;
4412 u32 orig_offset;
4413 struct btrfs_disk_key disk_key;
4414
Chris Masonb9473432009-03-13 11:00:37 -04004415 leaf = path->nodes[0];
4416 BUG_ON(btrfs_leaf_free_space(root, leaf) < sizeof(struct btrfs_item));
4417
Chris Masonb4ce94d2009-02-04 09:25:08 -05004418 btrfs_set_path_blocking(path);
4419
Ross Kirkdd3cc162013-09-16 15:58:09 +01004420 item = btrfs_item_nr(path->slots[0]);
Chris Mason459931e2008-12-10 09:10:46 -05004421 orig_offset = btrfs_item_offset(leaf, item);
4422 item_size = btrfs_item_size(leaf, item);
4423
Chris Mason459931e2008-12-10 09:10:46 -05004424 buf = kmalloc(item_size, GFP_NOFS);
Yan, Zhengad48fd752009-11-12 09:33:58 +00004425 if (!buf)
4426 return -ENOMEM;
4427
Chris Mason459931e2008-12-10 09:10:46 -05004428 read_extent_buffer(leaf, buf, btrfs_item_ptr_offset(leaf,
4429 path->slots[0]), item_size);
Yan, Zhengad48fd752009-11-12 09:33:58 +00004430
Chris Mason459931e2008-12-10 09:10:46 -05004431 slot = path->slots[0] + 1;
Chris Mason459931e2008-12-10 09:10:46 -05004432 nritems = btrfs_header_nritems(leaf);
Chris Mason459931e2008-12-10 09:10:46 -05004433 if (slot != nritems) {
4434 /* shift the items */
4435 memmove_extent_buffer(leaf, btrfs_item_nr_offset(slot + 1),
Yan, Zhengad48fd752009-11-12 09:33:58 +00004436 btrfs_item_nr_offset(slot),
4437 (nritems - slot) * sizeof(struct btrfs_item));
Chris Mason459931e2008-12-10 09:10:46 -05004438 }
4439
4440 btrfs_cpu_key_to_disk(&disk_key, new_key);
4441 btrfs_set_item_key(leaf, &disk_key, slot);
4442
Ross Kirkdd3cc162013-09-16 15:58:09 +01004443 new_item = btrfs_item_nr(slot);
Chris Mason459931e2008-12-10 09:10:46 -05004444
4445 btrfs_set_item_offset(leaf, new_item, orig_offset);
4446 btrfs_set_item_size(leaf, new_item, item_size - split_offset);
4447
4448 btrfs_set_item_offset(leaf, item,
4449 orig_offset + item_size - split_offset);
4450 btrfs_set_item_size(leaf, item, split_offset);
4451
4452 btrfs_set_header_nritems(leaf, nritems + 1);
4453
4454 /* write the data for the start of the original item */
4455 write_extent_buffer(leaf, buf,
4456 btrfs_item_ptr_offset(leaf, path->slots[0]),
4457 split_offset);
4458
4459 /* write the data for the new item */
4460 write_extent_buffer(leaf, buf + split_offset,
4461 btrfs_item_ptr_offset(leaf, slot),
4462 item_size - split_offset);
4463 btrfs_mark_buffer_dirty(leaf);
4464
Yan, Zhengad48fd752009-11-12 09:33:58 +00004465 BUG_ON(btrfs_leaf_free_space(root, leaf) < 0);
Chris Mason459931e2008-12-10 09:10:46 -05004466 kfree(buf);
Yan, Zhengad48fd752009-11-12 09:33:58 +00004467 return 0;
4468}
4469
4470/*
4471 * This function splits a single item into two items,
4472 * giving 'new_key' to the new item and splitting the
4473 * old one at split_offset (from the start of the item).
4474 *
4475 * The path may be released by this operation. After
4476 * the split, the path is pointing to the old item. The
4477 * new item is going to be in the same node as the old one.
4478 *
4479 * Note, the item being split must be smaller enough to live alone on
4480 * a tree block with room for one extra struct btrfs_item
4481 *
4482 * This allows us to split the item in place, keeping a lock on the
4483 * leaf the entire time.
4484 */
4485int btrfs_split_item(struct btrfs_trans_handle *trans,
4486 struct btrfs_root *root,
4487 struct btrfs_path *path,
4488 struct btrfs_key *new_key,
4489 unsigned long split_offset)
4490{
4491 int ret;
4492 ret = setup_leaf_for_split(trans, root, path,
4493 sizeof(struct btrfs_item));
4494 if (ret)
4495 return ret;
4496
4497 ret = split_item(trans, root, path, new_key, split_offset);
Chris Mason459931e2008-12-10 09:10:46 -05004498 return ret;
4499}
4500
4501/*
Yan, Zhengad48fd752009-11-12 09:33:58 +00004502 * This function duplicate a item, giving 'new_key' to the new item.
4503 * It guarantees both items live in the same tree leaf and the new item
4504 * is contiguous with the original item.
4505 *
4506 * This allows us to split file extent in place, keeping a lock on the
4507 * leaf the entire time.
4508 */
4509int btrfs_duplicate_item(struct btrfs_trans_handle *trans,
4510 struct btrfs_root *root,
4511 struct btrfs_path *path,
4512 struct btrfs_key *new_key)
4513{
4514 struct extent_buffer *leaf;
4515 int ret;
4516 u32 item_size;
4517
4518 leaf = path->nodes[0];
4519 item_size = btrfs_item_size_nr(leaf, path->slots[0]);
4520 ret = setup_leaf_for_split(trans, root, path,
4521 item_size + sizeof(struct btrfs_item));
4522 if (ret)
4523 return ret;
4524
4525 path->slots[0]++;
Tsutomu Itohafe5fea2013-04-16 05:18:22 +00004526 setup_items_for_insert(root, path, new_key, &item_size,
Jeff Mahoney143bede2012-03-01 14:56:26 +01004527 item_size, item_size +
4528 sizeof(struct btrfs_item), 1);
Yan, Zhengad48fd752009-11-12 09:33:58 +00004529 leaf = path->nodes[0];
4530 memcpy_extent_buffer(leaf,
4531 btrfs_item_ptr_offset(leaf, path->slots[0]),
4532 btrfs_item_ptr_offset(leaf, path->slots[0] - 1),
4533 item_size);
4534 return 0;
4535}
4536
4537/*
Chris Masond352ac62008-09-29 15:18:18 -04004538 * make the item pointed to by the path smaller. new_size indicates
4539 * how small to make it, and from_end tells us if we just chop bytes
4540 * off the end of the item or if we shift the item to chop bytes off
4541 * the front.
4542 */
Tsutomu Itohafe5fea2013-04-16 05:18:22 +00004543void btrfs_truncate_item(struct btrfs_root *root, struct btrfs_path *path,
Jeff Mahoney143bede2012-03-01 14:56:26 +01004544 u32 new_size, int from_end)
Chris Masonb18c6682007-04-17 13:26:50 -04004545{
Chris Masonb18c6682007-04-17 13:26:50 -04004546 int slot;
Chris Mason5f39d392007-10-15 16:14:19 -04004547 struct extent_buffer *leaf;
4548 struct btrfs_item *item;
Chris Masonb18c6682007-04-17 13:26:50 -04004549 u32 nritems;
4550 unsigned int data_end;
4551 unsigned int old_data_start;
4552 unsigned int old_size;
4553 unsigned int size_diff;
4554 int i;
Chris Masoncfed81a2012-03-03 07:40:03 -05004555 struct btrfs_map_token token;
4556
4557 btrfs_init_map_token(&token);
Chris Masonb18c6682007-04-17 13:26:50 -04004558
Chris Mason5f39d392007-10-15 16:14:19 -04004559 leaf = path->nodes[0];
Chris Mason179e29e2007-11-01 11:28:41 -04004560 slot = path->slots[0];
4561
4562 old_size = btrfs_item_size_nr(leaf, slot);
4563 if (old_size == new_size)
Jeff Mahoney143bede2012-03-01 14:56:26 +01004564 return;
Chris Masonb18c6682007-04-17 13:26:50 -04004565
Chris Mason5f39d392007-10-15 16:14:19 -04004566 nritems = btrfs_header_nritems(leaf);
Chris Masonb18c6682007-04-17 13:26:50 -04004567 data_end = leaf_data_end(root, leaf);
4568
Chris Mason5f39d392007-10-15 16:14:19 -04004569 old_data_start = btrfs_item_offset_nr(leaf, slot);
Chris Mason179e29e2007-11-01 11:28:41 -04004570
Chris Masonb18c6682007-04-17 13:26:50 -04004571 size_diff = old_size - new_size;
4572
4573 BUG_ON(slot < 0);
4574 BUG_ON(slot >= nritems);
4575
4576 /*
4577 * item0..itemN ... dataN.offset..dataN.size .. data0.size
4578 */
4579 /* first correct the data pointers */
4580 for (i = slot; i < nritems; i++) {
Chris Mason5f39d392007-10-15 16:14:19 -04004581 u32 ioff;
Ross Kirkdd3cc162013-09-16 15:58:09 +01004582 item = btrfs_item_nr(i);
Chris Masondb945352007-10-15 16:15:53 -04004583
Chris Masoncfed81a2012-03-03 07:40:03 -05004584 ioff = btrfs_token_item_offset(leaf, item, &token);
4585 btrfs_set_token_item_offset(leaf, item,
4586 ioff + size_diff, &token);
Chris Masonb18c6682007-04-17 13:26:50 -04004587 }
Chris Masondb945352007-10-15 16:15:53 -04004588
Chris Masonb18c6682007-04-17 13:26:50 -04004589 /* shift the data */
Chris Mason179e29e2007-11-01 11:28:41 -04004590 if (from_end) {
4591 memmove_extent_buffer(leaf, btrfs_leaf_data(leaf) +
4592 data_end + size_diff, btrfs_leaf_data(leaf) +
4593 data_end, old_data_start + new_size - data_end);
4594 } else {
4595 struct btrfs_disk_key disk_key;
4596 u64 offset;
4597
4598 btrfs_item_key(leaf, &disk_key, slot);
4599
4600 if (btrfs_disk_key_type(&disk_key) == BTRFS_EXTENT_DATA_KEY) {
4601 unsigned long ptr;
4602 struct btrfs_file_extent_item *fi;
4603
4604 fi = btrfs_item_ptr(leaf, slot,
4605 struct btrfs_file_extent_item);
4606 fi = (struct btrfs_file_extent_item *)(
4607 (unsigned long)fi - size_diff);
4608
4609 if (btrfs_file_extent_type(leaf, fi) ==
4610 BTRFS_FILE_EXTENT_INLINE) {
4611 ptr = btrfs_item_ptr_offset(leaf, slot);
4612 memmove_extent_buffer(leaf, ptr,
Chris Masond3977122009-01-05 21:25:51 -05004613 (unsigned long)fi,
David Sterba7ec20af2014-07-24 17:34:58 +02004614 BTRFS_FILE_EXTENT_INLINE_DATA_START);
Chris Mason179e29e2007-11-01 11:28:41 -04004615 }
4616 }
4617
4618 memmove_extent_buffer(leaf, btrfs_leaf_data(leaf) +
4619 data_end + size_diff, btrfs_leaf_data(leaf) +
4620 data_end, old_data_start - data_end);
4621
4622 offset = btrfs_disk_key_offset(&disk_key);
4623 btrfs_set_disk_key_offset(&disk_key, offset + size_diff);
4624 btrfs_set_item_key(leaf, &disk_key, slot);
4625 if (slot == 0)
Daniel Dresslerb7a03652014-11-12 13:43:09 +09004626 fixup_low_keys(root->fs_info, path, &disk_key, 1);
Chris Mason179e29e2007-11-01 11:28:41 -04004627 }
Chris Mason5f39d392007-10-15 16:14:19 -04004628
Ross Kirkdd3cc162013-09-16 15:58:09 +01004629 item = btrfs_item_nr(slot);
Chris Mason5f39d392007-10-15 16:14:19 -04004630 btrfs_set_item_size(leaf, item, new_size);
4631 btrfs_mark_buffer_dirty(leaf);
Chris Masonb18c6682007-04-17 13:26:50 -04004632
Chris Mason5f39d392007-10-15 16:14:19 -04004633 if (btrfs_leaf_free_space(root, leaf) < 0) {
4634 btrfs_print_leaf(root, leaf);
Chris Masonb18c6682007-04-17 13:26:50 -04004635 BUG();
Chris Mason5f39d392007-10-15 16:14:19 -04004636 }
Chris Masonb18c6682007-04-17 13:26:50 -04004637}
4638
Chris Masond352ac62008-09-29 15:18:18 -04004639/*
Stefan Behrens8f69dbd2013-05-07 10:23:30 +00004640 * make the item pointed to by the path bigger, data_size is the added size.
Chris Masond352ac62008-09-29 15:18:18 -04004641 */
Tsutomu Itoh4b90c682013-04-16 05:18:49 +00004642void btrfs_extend_item(struct btrfs_root *root, struct btrfs_path *path,
Jeff Mahoney143bede2012-03-01 14:56:26 +01004643 u32 data_size)
Chris Mason6567e832007-04-16 09:22:45 -04004644{
Chris Mason6567e832007-04-16 09:22:45 -04004645 int slot;
Chris Mason5f39d392007-10-15 16:14:19 -04004646 struct extent_buffer *leaf;
4647 struct btrfs_item *item;
Chris Mason6567e832007-04-16 09:22:45 -04004648 u32 nritems;
4649 unsigned int data_end;
4650 unsigned int old_data;
4651 unsigned int old_size;
4652 int i;
Chris Masoncfed81a2012-03-03 07:40:03 -05004653 struct btrfs_map_token token;
4654
4655 btrfs_init_map_token(&token);
Chris Mason6567e832007-04-16 09:22:45 -04004656
Chris Mason5f39d392007-10-15 16:14:19 -04004657 leaf = path->nodes[0];
Chris Mason6567e832007-04-16 09:22:45 -04004658
Chris Mason5f39d392007-10-15 16:14:19 -04004659 nritems = btrfs_header_nritems(leaf);
Chris Mason6567e832007-04-16 09:22:45 -04004660 data_end = leaf_data_end(root, leaf);
4661
Chris Mason5f39d392007-10-15 16:14:19 -04004662 if (btrfs_leaf_free_space(root, leaf) < data_size) {
4663 btrfs_print_leaf(root, leaf);
Chris Mason6567e832007-04-16 09:22:45 -04004664 BUG();
Chris Mason5f39d392007-10-15 16:14:19 -04004665 }
Chris Mason6567e832007-04-16 09:22:45 -04004666 slot = path->slots[0];
Chris Mason5f39d392007-10-15 16:14:19 -04004667 old_data = btrfs_item_end_nr(leaf, slot);
Chris Mason6567e832007-04-16 09:22:45 -04004668
4669 BUG_ON(slot < 0);
Chris Mason3326d1b2007-10-15 16:18:25 -04004670 if (slot >= nritems) {
4671 btrfs_print_leaf(root, leaf);
Frank Holtonefe120a2013-12-20 11:37:06 -05004672 btrfs_crit(root->fs_info, "slot %d too large, nritems %d",
Chris Masond3977122009-01-05 21:25:51 -05004673 slot, nritems);
Chris Mason3326d1b2007-10-15 16:18:25 -04004674 BUG_ON(1);
4675 }
Chris Mason6567e832007-04-16 09:22:45 -04004676
4677 /*
4678 * item0..itemN ... dataN.offset..dataN.size .. data0.size
4679 */
4680 /* first correct the data pointers */
4681 for (i = slot; i < nritems; i++) {
Chris Mason5f39d392007-10-15 16:14:19 -04004682 u32 ioff;
Ross Kirkdd3cc162013-09-16 15:58:09 +01004683 item = btrfs_item_nr(i);
Chris Masondb945352007-10-15 16:15:53 -04004684
Chris Masoncfed81a2012-03-03 07:40:03 -05004685 ioff = btrfs_token_item_offset(leaf, item, &token);
4686 btrfs_set_token_item_offset(leaf, item,
4687 ioff - data_size, &token);
Chris Mason6567e832007-04-16 09:22:45 -04004688 }
Chris Mason5f39d392007-10-15 16:14:19 -04004689
Chris Mason6567e832007-04-16 09:22:45 -04004690 /* shift the data */
Chris Mason5f39d392007-10-15 16:14:19 -04004691 memmove_extent_buffer(leaf, btrfs_leaf_data(leaf) +
Chris Mason6567e832007-04-16 09:22:45 -04004692 data_end - data_size, btrfs_leaf_data(leaf) +
4693 data_end, old_data - data_end);
Chris Mason5f39d392007-10-15 16:14:19 -04004694
Chris Mason6567e832007-04-16 09:22:45 -04004695 data_end = old_data;
Chris Mason5f39d392007-10-15 16:14:19 -04004696 old_size = btrfs_item_size_nr(leaf, slot);
Ross Kirkdd3cc162013-09-16 15:58:09 +01004697 item = btrfs_item_nr(slot);
Chris Mason5f39d392007-10-15 16:14:19 -04004698 btrfs_set_item_size(leaf, item, old_size + data_size);
4699 btrfs_mark_buffer_dirty(leaf);
Chris Mason6567e832007-04-16 09:22:45 -04004700
Chris Mason5f39d392007-10-15 16:14:19 -04004701 if (btrfs_leaf_free_space(root, leaf) < 0) {
4702 btrfs_print_leaf(root, leaf);
Chris Mason6567e832007-04-16 09:22:45 -04004703 BUG();
Chris Mason5f39d392007-10-15 16:14:19 -04004704 }
Chris Mason6567e832007-04-16 09:22:45 -04004705}
4706
Chris Mason74123bd2007-02-02 11:05:29 -05004707/*
Chris Mason44871b12009-03-13 10:04:31 -04004708 * this is a helper for btrfs_insert_empty_items, the main goal here is
4709 * to save stack depth by doing the bulk of the work in a function
4710 * that doesn't call btrfs_search_slot
Chris Mason74123bd2007-02-02 11:05:29 -05004711 */
Tsutomu Itohafe5fea2013-04-16 05:18:22 +00004712void setup_items_for_insert(struct btrfs_root *root, struct btrfs_path *path,
Jeff Mahoney143bede2012-03-01 14:56:26 +01004713 struct btrfs_key *cpu_key, u32 *data_size,
4714 u32 total_data, u32 total_size, int nr)
Chris Masonbe0e5c02007-01-26 15:51:26 -05004715{
Chris Mason5f39d392007-10-15 16:14:19 -04004716 struct btrfs_item *item;
Chris Mason9c583092008-01-29 15:15:18 -05004717 int i;
Chris Mason7518a232007-03-12 12:01:18 -04004718 u32 nritems;
Chris Masonbe0e5c02007-01-26 15:51:26 -05004719 unsigned int data_end;
Chris Masone2fa7222007-03-12 16:22:34 -04004720 struct btrfs_disk_key disk_key;
Chris Mason44871b12009-03-13 10:04:31 -04004721 struct extent_buffer *leaf;
4722 int slot;
Chris Masoncfed81a2012-03-03 07:40:03 -05004723 struct btrfs_map_token token;
4724
Filipe Manana24cdc842014-07-28 19:34:35 +01004725 if (path->slots[0] == 0) {
4726 btrfs_cpu_key_to_disk(&disk_key, cpu_key);
Daniel Dresslerb7a03652014-11-12 13:43:09 +09004727 fixup_low_keys(root->fs_info, path, &disk_key, 1);
Filipe Manana24cdc842014-07-28 19:34:35 +01004728 }
4729 btrfs_unlock_up_safe(path, 1);
4730
Chris Masoncfed81a2012-03-03 07:40:03 -05004731 btrfs_init_map_token(&token);
Chris Masone2fa7222007-03-12 16:22:34 -04004732
Chris Mason5f39d392007-10-15 16:14:19 -04004733 leaf = path->nodes[0];
Chris Mason44871b12009-03-13 10:04:31 -04004734 slot = path->slots[0];
Chris Mason74123bd2007-02-02 11:05:29 -05004735
Chris Mason5f39d392007-10-15 16:14:19 -04004736 nritems = btrfs_header_nritems(leaf);
Chris Mason123abc82007-03-14 14:14:43 -04004737 data_end = leaf_data_end(root, leaf);
Chris Masoneb60cea2007-02-02 09:18:22 -05004738
Chris Masonf25956c2008-09-12 15:32:53 -04004739 if (btrfs_leaf_free_space(root, leaf) < total_size) {
Chris Mason3326d1b2007-10-15 16:18:25 -04004740 btrfs_print_leaf(root, leaf);
Frank Holtonefe120a2013-12-20 11:37:06 -05004741 btrfs_crit(root->fs_info, "not enough freespace need %u have %d",
Chris Mason9c583092008-01-29 15:15:18 -05004742 total_size, btrfs_leaf_free_space(root, leaf));
Chris Masonbe0e5c02007-01-26 15:51:26 -05004743 BUG();
Chris Masond4dbff92007-04-04 14:08:15 -04004744 }
Chris Mason5f39d392007-10-15 16:14:19 -04004745
Chris Masonbe0e5c02007-01-26 15:51:26 -05004746 if (slot != nritems) {
Chris Mason5f39d392007-10-15 16:14:19 -04004747 unsigned int old_data = btrfs_item_end_nr(leaf, slot);
Chris Masonbe0e5c02007-01-26 15:51:26 -05004748
Chris Mason5f39d392007-10-15 16:14:19 -04004749 if (old_data < data_end) {
4750 btrfs_print_leaf(root, leaf);
Frank Holtonefe120a2013-12-20 11:37:06 -05004751 btrfs_crit(root->fs_info, "slot %d old_data %d data_end %d",
Chris Mason5f39d392007-10-15 16:14:19 -04004752 slot, old_data, data_end);
4753 BUG_ON(1);
4754 }
Chris Masonbe0e5c02007-01-26 15:51:26 -05004755 /*
4756 * item0..itemN ... dataN.offset..dataN.size .. data0.size
4757 */
4758 /* first correct the data pointers */
Chris Mason0783fcf2007-03-12 20:12:07 -04004759 for (i = slot; i < nritems; i++) {
Chris Mason5f39d392007-10-15 16:14:19 -04004760 u32 ioff;
Chris Masondb945352007-10-15 16:15:53 -04004761
Ross Kirkdd3cc162013-09-16 15:58:09 +01004762 item = btrfs_item_nr( i);
Chris Masoncfed81a2012-03-03 07:40:03 -05004763 ioff = btrfs_token_item_offset(leaf, item, &token);
4764 btrfs_set_token_item_offset(leaf, item,
4765 ioff - total_data, &token);
Chris Mason0783fcf2007-03-12 20:12:07 -04004766 }
Chris Masonbe0e5c02007-01-26 15:51:26 -05004767 /* shift the items */
Chris Mason9c583092008-01-29 15:15:18 -05004768 memmove_extent_buffer(leaf, btrfs_item_nr_offset(slot + nr),
Chris Mason5f39d392007-10-15 16:14:19 -04004769 btrfs_item_nr_offset(slot),
Chris Masond6025572007-03-30 14:27:56 -04004770 (nritems - slot) * sizeof(struct btrfs_item));
Chris Masonbe0e5c02007-01-26 15:51:26 -05004771
4772 /* shift the data */
Chris Mason5f39d392007-10-15 16:14:19 -04004773 memmove_extent_buffer(leaf, btrfs_leaf_data(leaf) +
Chris Mason9c583092008-01-29 15:15:18 -05004774 data_end - total_data, btrfs_leaf_data(leaf) +
Chris Masond6025572007-03-30 14:27:56 -04004775 data_end, old_data - data_end);
Chris Masonbe0e5c02007-01-26 15:51:26 -05004776 data_end = old_data;
4777 }
Chris Mason5f39d392007-10-15 16:14:19 -04004778
Chris Mason62e27492007-03-15 12:56:47 -04004779 /* setup the item for the new data */
Chris Mason9c583092008-01-29 15:15:18 -05004780 for (i = 0; i < nr; i++) {
4781 btrfs_cpu_key_to_disk(&disk_key, cpu_key + i);
4782 btrfs_set_item_key(leaf, &disk_key, slot + i);
Ross Kirkdd3cc162013-09-16 15:58:09 +01004783 item = btrfs_item_nr(slot + i);
Chris Masoncfed81a2012-03-03 07:40:03 -05004784 btrfs_set_token_item_offset(leaf, item,
4785 data_end - data_size[i], &token);
Chris Mason9c583092008-01-29 15:15:18 -05004786 data_end -= data_size[i];
Chris Masoncfed81a2012-03-03 07:40:03 -05004787 btrfs_set_token_item_size(leaf, item, data_size[i], &token);
Chris Mason9c583092008-01-29 15:15:18 -05004788 }
Chris Mason44871b12009-03-13 10:04:31 -04004789
Chris Mason9c583092008-01-29 15:15:18 -05004790 btrfs_set_header_nritems(leaf, nritems + nr);
Chris Masonb9473432009-03-13 11:00:37 -04004791 btrfs_mark_buffer_dirty(leaf);
Chris Masonaa5d6be2007-02-28 16:35:06 -05004792
Chris Mason5f39d392007-10-15 16:14:19 -04004793 if (btrfs_leaf_free_space(root, leaf) < 0) {
4794 btrfs_print_leaf(root, leaf);
Chris Masonbe0e5c02007-01-26 15:51:26 -05004795 BUG();
Chris Mason5f39d392007-10-15 16:14:19 -04004796 }
Chris Mason44871b12009-03-13 10:04:31 -04004797}
4798
4799/*
4800 * Given a key and some data, insert items into the tree.
4801 * This does all the path init required, making room in the tree if needed.
4802 */
4803int btrfs_insert_empty_items(struct btrfs_trans_handle *trans,
4804 struct btrfs_root *root,
4805 struct btrfs_path *path,
4806 struct btrfs_key *cpu_key, u32 *data_size,
4807 int nr)
4808{
Chris Mason44871b12009-03-13 10:04:31 -04004809 int ret = 0;
4810 int slot;
4811 int i;
4812 u32 total_size = 0;
4813 u32 total_data = 0;
4814
4815 for (i = 0; i < nr; i++)
4816 total_data += data_size[i];
4817
4818 total_size = total_data + (nr * sizeof(struct btrfs_item));
4819 ret = btrfs_search_slot(trans, root, cpu_key, path, total_size, 1);
4820 if (ret == 0)
4821 return -EEXIST;
4822 if (ret < 0)
Jeff Mahoney143bede2012-03-01 14:56:26 +01004823 return ret;
Chris Mason44871b12009-03-13 10:04:31 -04004824
Chris Mason44871b12009-03-13 10:04:31 -04004825 slot = path->slots[0];
4826 BUG_ON(slot < 0);
4827
Tsutomu Itohafe5fea2013-04-16 05:18:22 +00004828 setup_items_for_insert(root, path, cpu_key, data_size,
Chris Mason44871b12009-03-13 10:04:31 -04004829 total_data, total_size, nr);
Jeff Mahoney143bede2012-03-01 14:56:26 +01004830 return 0;
Chris Mason62e27492007-03-15 12:56:47 -04004831}
4832
4833/*
4834 * Given a key and some data, insert an item into the tree.
4835 * This does all the path init required, making room in the tree if needed.
4836 */
Chris Masone089f052007-03-16 16:20:31 -04004837int btrfs_insert_item(struct btrfs_trans_handle *trans, struct btrfs_root
4838 *root, struct btrfs_key *cpu_key, void *data, u32
4839 data_size)
Chris Mason62e27492007-03-15 12:56:47 -04004840{
4841 int ret = 0;
Chris Mason2c90e5d2007-04-02 10:50:19 -04004842 struct btrfs_path *path;
Chris Mason5f39d392007-10-15 16:14:19 -04004843 struct extent_buffer *leaf;
4844 unsigned long ptr;
Chris Mason62e27492007-03-15 12:56:47 -04004845
Chris Mason2c90e5d2007-04-02 10:50:19 -04004846 path = btrfs_alloc_path();
Tsutomu Itohdb5b4932011-03-23 08:14:16 +00004847 if (!path)
4848 return -ENOMEM;
Chris Mason2c90e5d2007-04-02 10:50:19 -04004849 ret = btrfs_insert_empty_item(trans, root, path, cpu_key, data_size);
Chris Mason62e27492007-03-15 12:56:47 -04004850 if (!ret) {
Chris Mason5f39d392007-10-15 16:14:19 -04004851 leaf = path->nodes[0];
4852 ptr = btrfs_item_ptr_offset(leaf, path->slots[0]);
4853 write_extent_buffer(leaf, data, ptr, data_size);
4854 btrfs_mark_buffer_dirty(leaf);
Chris Mason62e27492007-03-15 12:56:47 -04004855 }
Chris Mason2c90e5d2007-04-02 10:50:19 -04004856 btrfs_free_path(path);
Chris Masonaa5d6be2007-02-28 16:35:06 -05004857 return ret;
Chris Masonbe0e5c02007-01-26 15:51:26 -05004858}
4859
Chris Mason74123bd2007-02-02 11:05:29 -05004860/*
Chris Mason5de08d72007-02-24 06:24:44 -05004861 * delete the pointer from a given node.
Chris Mason74123bd2007-02-02 11:05:29 -05004862 *
Chris Masond352ac62008-09-29 15:18:18 -04004863 * the tree should have been previously balanced so the deletion does not
4864 * empty a node.
Chris Mason74123bd2007-02-02 11:05:29 -05004865 */
Tsutomu Itohafe5fea2013-04-16 05:18:22 +00004866static void del_ptr(struct btrfs_root *root, struct btrfs_path *path,
4867 int level, int slot)
Chris Masonbe0e5c02007-01-26 15:51:26 -05004868{
Chris Mason5f39d392007-10-15 16:14:19 -04004869 struct extent_buffer *parent = path->nodes[level];
Chris Mason7518a232007-03-12 12:01:18 -04004870 u32 nritems;
Jan Schmidtf3ea38d2012-05-26 11:45:21 +02004871 int ret;
Chris Masonbe0e5c02007-01-26 15:51:26 -05004872
Chris Mason5f39d392007-10-15 16:14:19 -04004873 nritems = btrfs_header_nritems(parent);
Chris Masond3977122009-01-05 21:25:51 -05004874 if (slot != nritems - 1) {
Liu Bo0e411ec2012-10-19 09:50:54 +00004875 if (level)
Jan Schmidtf3ea38d2012-05-26 11:45:21 +02004876 tree_mod_log_eb_move(root->fs_info, parent, slot,
4877 slot + 1, nritems - slot - 1);
Chris Mason5f39d392007-10-15 16:14:19 -04004878 memmove_extent_buffer(parent,
4879 btrfs_node_key_ptr_offset(slot),
4880 btrfs_node_key_ptr_offset(slot + 1),
Chris Masond6025572007-03-30 14:27:56 -04004881 sizeof(struct btrfs_key_ptr) *
4882 (nritems - slot - 1));
Chris Mason57ba86c2012-12-18 19:35:32 -05004883 } else if (level) {
4884 ret = tree_mod_log_insert_key(root->fs_info, parent, slot,
Josef Bacikc8cc6342013-07-01 16:18:19 -04004885 MOD_LOG_KEY_REMOVE, GFP_NOFS);
Chris Mason57ba86c2012-12-18 19:35:32 -05004886 BUG_ON(ret < 0);
Chris Masonbb803952007-03-01 12:04:21 -05004887 }
Jan Schmidtf3ea38d2012-05-26 11:45:21 +02004888
Chris Mason7518a232007-03-12 12:01:18 -04004889 nritems--;
Chris Mason5f39d392007-10-15 16:14:19 -04004890 btrfs_set_header_nritems(parent, nritems);
Chris Mason7518a232007-03-12 12:01:18 -04004891 if (nritems == 0 && parent == root->node) {
Chris Mason5f39d392007-10-15 16:14:19 -04004892 BUG_ON(btrfs_header_level(root->node) != 1);
Chris Masonbb803952007-03-01 12:04:21 -05004893 /* just turn the root into a leaf and break */
Chris Mason5f39d392007-10-15 16:14:19 -04004894 btrfs_set_header_level(root->node, 0);
Chris Masonbb803952007-03-01 12:04:21 -05004895 } else if (slot == 0) {
Chris Mason5f39d392007-10-15 16:14:19 -04004896 struct btrfs_disk_key disk_key;
4897
4898 btrfs_node_key(parent, &disk_key, 0);
Daniel Dresslerb7a03652014-11-12 13:43:09 +09004899 fixup_low_keys(root->fs_info, path, &disk_key, level + 1);
Chris Masonbe0e5c02007-01-26 15:51:26 -05004900 }
Chris Masond6025572007-03-30 14:27:56 -04004901 btrfs_mark_buffer_dirty(parent);
Chris Masonbe0e5c02007-01-26 15:51:26 -05004902}
4903
Chris Mason74123bd2007-02-02 11:05:29 -05004904/*
Chris Mason323ac952008-10-01 19:05:46 -04004905 * a helper function to delete the leaf pointed to by path->slots[1] and
Yan Zheng5d4f98a2009-06-10 10:45:14 -04004906 * path->nodes[1].
Chris Mason323ac952008-10-01 19:05:46 -04004907 *
4908 * This deletes the pointer in path->nodes[1] and frees the leaf
4909 * block extent. zero is returned if it all worked out, < 0 otherwise.
4910 *
4911 * The path must have already been setup for deleting the leaf, including
4912 * all the proper balancing. path->nodes[1] must be locked.
4913 */
Jeff Mahoney143bede2012-03-01 14:56:26 +01004914static noinline void btrfs_del_leaf(struct btrfs_trans_handle *trans,
4915 struct btrfs_root *root,
4916 struct btrfs_path *path,
4917 struct extent_buffer *leaf)
Chris Mason323ac952008-10-01 19:05:46 -04004918{
Yan Zheng5d4f98a2009-06-10 10:45:14 -04004919 WARN_ON(btrfs_header_generation(leaf) != trans->transid);
Tsutomu Itohafe5fea2013-04-16 05:18:22 +00004920 del_ptr(root, path, 1, path->slots[1]);
Chris Mason323ac952008-10-01 19:05:46 -04004921
Chris Mason4d081c42009-02-04 09:31:28 -05004922 /*
4923 * btrfs_free_extent is expensive, we want to make sure we
4924 * aren't holding any locks when we call it
4925 */
4926 btrfs_unlock_up_safe(path, 0);
4927
Yan, Zhengf0486c62010-05-16 10:46:25 -04004928 root_sub_used(root, leaf->len);
4929
Josef Bacik3083ee22012-03-09 16:01:49 -05004930 extent_buffer_get(leaf);
Jan Schmidt5581a512012-05-16 17:04:52 +02004931 btrfs_free_tree_block(trans, root, leaf, 0, 1);
Josef Bacik3083ee22012-03-09 16:01:49 -05004932 free_extent_buffer_stale(leaf);
Chris Mason323ac952008-10-01 19:05:46 -04004933}
4934/*
Chris Mason74123bd2007-02-02 11:05:29 -05004935 * delete the item at the leaf level in path. If that empties
4936 * the leaf, remove it from the tree
4937 */
Chris Mason85e21ba2008-01-29 15:11:36 -05004938int btrfs_del_items(struct btrfs_trans_handle *trans, struct btrfs_root *root,
4939 struct btrfs_path *path, int slot, int nr)
Chris Masonbe0e5c02007-01-26 15:51:26 -05004940{
Chris Mason5f39d392007-10-15 16:14:19 -04004941 struct extent_buffer *leaf;
4942 struct btrfs_item *item;
Alexandru Moisece0eac22015-08-23 16:01:42 +00004943 u32 last_off;
4944 u32 dsize = 0;
Chris Masonaa5d6be2007-02-28 16:35:06 -05004945 int ret = 0;
4946 int wret;
Chris Mason85e21ba2008-01-29 15:11:36 -05004947 int i;
Chris Mason7518a232007-03-12 12:01:18 -04004948 u32 nritems;
Chris Masoncfed81a2012-03-03 07:40:03 -05004949 struct btrfs_map_token token;
4950
4951 btrfs_init_map_token(&token);
Chris Masonbe0e5c02007-01-26 15:51:26 -05004952
Chris Mason5f39d392007-10-15 16:14:19 -04004953 leaf = path->nodes[0];
Chris Mason85e21ba2008-01-29 15:11:36 -05004954 last_off = btrfs_item_offset_nr(leaf, slot + nr - 1);
4955
4956 for (i = 0; i < nr; i++)
4957 dsize += btrfs_item_size_nr(leaf, slot + i);
4958
Chris Mason5f39d392007-10-15 16:14:19 -04004959 nritems = btrfs_header_nritems(leaf);
Chris Masonbe0e5c02007-01-26 15:51:26 -05004960
Chris Mason85e21ba2008-01-29 15:11:36 -05004961 if (slot + nr != nritems) {
Chris Mason123abc82007-03-14 14:14:43 -04004962 int data_end = leaf_data_end(root, leaf);
Chris Mason5f39d392007-10-15 16:14:19 -04004963
4964 memmove_extent_buffer(leaf, btrfs_leaf_data(leaf) +
Chris Masond6025572007-03-30 14:27:56 -04004965 data_end + dsize,
4966 btrfs_leaf_data(leaf) + data_end,
Chris Mason85e21ba2008-01-29 15:11:36 -05004967 last_off - data_end);
Chris Mason5f39d392007-10-15 16:14:19 -04004968
Chris Mason85e21ba2008-01-29 15:11:36 -05004969 for (i = slot + nr; i < nritems; i++) {
Chris Mason5f39d392007-10-15 16:14:19 -04004970 u32 ioff;
Chris Masondb945352007-10-15 16:15:53 -04004971
Ross Kirkdd3cc162013-09-16 15:58:09 +01004972 item = btrfs_item_nr(i);
Chris Masoncfed81a2012-03-03 07:40:03 -05004973 ioff = btrfs_token_item_offset(leaf, item, &token);
4974 btrfs_set_token_item_offset(leaf, item,
4975 ioff + dsize, &token);
Chris Mason0783fcf2007-03-12 20:12:07 -04004976 }
Chris Masondb945352007-10-15 16:15:53 -04004977
Chris Mason5f39d392007-10-15 16:14:19 -04004978 memmove_extent_buffer(leaf, btrfs_item_nr_offset(slot),
Chris Mason85e21ba2008-01-29 15:11:36 -05004979 btrfs_item_nr_offset(slot + nr),
Chris Masond6025572007-03-30 14:27:56 -04004980 sizeof(struct btrfs_item) *
Chris Mason85e21ba2008-01-29 15:11:36 -05004981 (nritems - slot - nr));
Chris Masonbe0e5c02007-01-26 15:51:26 -05004982 }
Chris Mason85e21ba2008-01-29 15:11:36 -05004983 btrfs_set_header_nritems(leaf, nritems - nr);
4984 nritems -= nr;
Chris Mason5f39d392007-10-15 16:14:19 -04004985
Chris Mason74123bd2007-02-02 11:05:29 -05004986 /* delete the leaf if we've emptied it */
Chris Mason7518a232007-03-12 12:01:18 -04004987 if (nritems == 0) {
Chris Mason5f39d392007-10-15 16:14:19 -04004988 if (leaf == root->node) {
4989 btrfs_set_header_level(leaf, 0);
Chris Mason9a8dd152007-02-23 08:38:36 -05004990 } else {
Yan, Zhengf0486c62010-05-16 10:46:25 -04004991 btrfs_set_path_blocking(path);
Daniel Dressler01d58472014-11-21 17:15:07 +09004992 clean_tree_block(trans, root->fs_info, leaf);
Jeff Mahoney143bede2012-03-01 14:56:26 +01004993 btrfs_del_leaf(trans, root, path, leaf);
Chris Mason9a8dd152007-02-23 08:38:36 -05004994 }
Chris Masonbe0e5c02007-01-26 15:51:26 -05004995 } else {
Chris Mason7518a232007-03-12 12:01:18 -04004996 int used = leaf_space_used(leaf, 0, nritems);
Chris Masonaa5d6be2007-02-28 16:35:06 -05004997 if (slot == 0) {
Chris Mason5f39d392007-10-15 16:14:19 -04004998 struct btrfs_disk_key disk_key;
4999
5000 btrfs_item_key(leaf, &disk_key, 0);
Daniel Dresslerb7a03652014-11-12 13:43:09 +09005001 fixup_low_keys(root->fs_info, path, &disk_key, 1);
Chris Masonaa5d6be2007-02-28 16:35:06 -05005002 }
Chris Masonaa5d6be2007-02-28 16:35:06 -05005003
Chris Mason74123bd2007-02-02 11:05:29 -05005004 /* delete the leaf if it is mostly empty */
Yan Zhengd717aa12009-07-24 12:42:46 -04005005 if (used < BTRFS_LEAF_DATA_SIZE(root) / 3) {
Chris Masonbe0e5c02007-01-26 15:51:26 -05005006 /* push_leaf_left fixes the path.
5007 * make sure the path still points to our leaf
5008 * for possible call to del_ptr below
5009 */
Chris Mason4920c9a2007-01-26 16:38:42 -05005010 slot = path->slots[1];
Chris Mason5f39d392007-10-15 16:14:19 -04005011 extent_buffer_get(leaf);
5012
Chris Masonb9473432009-03-13 11:00:37 -04005013 btrfs_set_path_blocking(path);
Chris Mason99d8f832010-07-07 10:51:48 -04005014 wret = push_leaf_left(trans, root, path, 1, 1,
5015 1, (u32)-1);
Chris Mason54aa1f42007-06-22 14:16:25 -04005016 if (wret < 0 && wret != -ENOSPC)
Chris Masonaa5d6be2007-02-28 16:35:06 -05005017 ret = wret;
Chris Mason5f39d392007-10-15 16:14:19 -04005018
5019 if (path->nodes[0] == leaf &&
5020 btrfs_header_nritems(leaf)) {
Chris Mason99d8f832010-07-07 10:51:48 -04005021 wret = push_leaf_right(trans, root, path, 1,
5022 1, 1, 0);
Chris Mason54aa1f42007-06-22 14:16:25 -04005023 if (wret < 0 && wret != -ENOSPC)
Chris Masonaa5d6be2007-02-28 16:35:06 -05005024 ret = wret;
5025 }
Chris Mason5f39d392007-10-15 16:14:19 -04005026
5027 if (btrfs_header_nritems(leaf) == 0) {
Chris Mason323ac952008-10-01 19:05:46 -04005028 path->slots[1] = slot;
Jeff Mahoney143bede2012-03-01 14:56:26 +01005029 btrfs_del_leaf(trans, root, path, leaf);
Chris Mason5f39d392007-10-15 16:14:19 -04005030 free_extent_buffer(leaf);
Jeff Mahoney143bede2012-03-01 14:56:26 +01005031 ret = 0;
Chris Mason5de08d72007-02-24 06:24:44 -05005032 } else {
Chris Mason925baed2008-06-25 16:01:30 -04005033 /* if we're still in the path, make sure
5034 * we're dirty. Otherwise, one of the
5035 * push_leaf functions must have already
5036 * dirtied this buffer
5037 */
5038 if (path->nodes[0] == leaf)
5039 btrfs_mark_buffer_dirty(leaf);
Chris Mason5f39d392007-10-15 16:14:19 -04005040 free_extent_buffer(leaf);
Chris Masonbe0e5c02007-01-26 15:51:26 -05005041 }
Chris Masond5719762007-03-23 10:01:08 -04005042 } else {
Chris Mason5f39d392007-10-15 16:14:19 -04005043 btrfs_mark_buffer_dirty(leaf);
Chris Masonbe0e5c02007-01-26 15:51:26 -05005044 }
5045 }
Chris Masonaa5d6be2007-02-28 16:35:06 -05005046 return ret;
Chris Masonbe0e5c02007-01-26 15:51:26 -05005047}
5048
Chris Mason97571fd2007-02-24 13:39:08 -05005049/*
Chris Mason925baed2008-06-25 16:01:30 -04005050 * search the tree again to find a leaf with lesser keys
Chris Mason7bb86312007-12-11 09:25:06 -05005051 * returns 0 if it found something or 1 if there are no lesser leaves.
5052 * returns < 0 on io errors.
Chris Masond352ac62008-09-29 15:18:18 -04005053 *
5054 * This may release the path, and so you may lose any locks held at the
5055 * time you call it.
Chris Mason7bb86312007-12-11 09:25:06 -05005056 */
Josef Bacik16e75492013-10-22 12:18:51 -04005057int btrfs_prev_leaf(struct btrfs_root *root, struct btrfs_path *path)
Chris Mason7bb86312007-12-11 09:25:06 -05005058{
Chris Mason925baed2008-06-25 16:01:30 -04005059 struct btrfs_key key;
5060 struct btrfs_disk_key found_key;
5061 int ret;
Chris Mason7bb86312007-12-11 09:25:06 -05005062
Chris Mason925baed2008-06-25 16:01:30 -04005063 btrfs_item_key_to_cpu(path->nodes[0], &key, 0);
Chris Mason7bb86312007-12-11 09:25:06 -05005064
Filipe David Borba Mananae8b0d722013-10-15 00:12:27 +01005065 if (key.offset > 0) {
Chris Mason925baed2008-06-25 16:01:30 -04005066 key.offset--;
Filipe David Borba Mananae8b0d722013-10-15 00:12:27 +01005067 } else if (key.type > 0) {
Chris Mason925baed2008-06-25 16:01:30 -04005068 key.type--;
Filipe David Borba Mananae8b0d722013-10-15 00:12:27 +01005069 key.offset = (u64)-1;
5070 } else if (key.objectid > 0) {
Chris Mason925baed2008-06-25 16:01:30 -04005071 key.objectid--;
Filipe David Borba Mananae8b0d722013-10-15 00:12:27 +01005072 key.type = (u8)-1;
5073 key.offset = (u64)-1;
5074 } else {
Chris Mason925baed2008-06-25 16:01:30 -04005075 return 1;
Filipe David Borba Mananae8b0d722013-10-15 00:12:27 +01005076 }
Chris Mason7bb86312007-12-11 09:25:06 -05005077
David Sterbab3b4aa72011-04-21 01:20:15 +02005078 btrfs_release_path(path);
Chris Mason925baed2008-06-25 16:01:30 -04005079 ret = btrfs_search_slot(NULL, root, &key, path, 0, 0);
5080 if (ret < 0)
5081 return ret;
5082 btrfs_item_key(path->nodes[0], &found_key, 0);
5083 ret = comp_keys(&found_key, &key);
Filipe Manana337c6f62014-06-09 13:22:13 +01005084 /*
5085 * We might have had an item with the previous key in the tree right
5086 * before we released our path. And after we released our path, that
5087 * item might have been pushed to the first slot (0) of the leaf we
5088 * were holding due to a tree balance. Alternatively, an item with the
5089 * previous key can exist as the only element of a leaf (big fat item).
5090 * Therefore account for these 2 cases, so that our callers (like
5091 * btrfs_previous_item) don't miss an existing item with a key matching
5092 * the previous key we computed above.
5093 */
5094 if (ret <= 0)
Chris Mason925baed2008-06-25 16:01:30 -04005095 return 0;
5096 return 1;
Chris Mason7bb86312007-12-11 09:25:06 -05005097}
5098
Chris Mason3f157a22008-06-25 16:01:31 -04005099/*
5100 * A helper function to walk down the tree starting at min_key, and looking
Eric Sandeende78b512013-01-31 18:21:12 +00005101 * for nodes or leaves that are have a minimum transaction id.
5102 * This is used by the btree defrag code, and tree logging
Chris Mason3f157a22008-06-25 16:01:31 -04005103 *
5104 * This does not cow, but it does stuff the starting key it finds back
5105 * into min_key, so you can call btrfs_search_slot with cow=1 on the
5106 * key and get a writable path.
5107 *
5108 * This does lock as it descends, and path->keep_locks should be set
5109 * to 1 by the caller.
5110 *
5111 * This honors path->lowest_level to prevent descent past a given level
5112 * of the tree.
5113 *
Chris Masond352ac62008-09-29 15:18:18 -04005114 * min_trans indicates the oldest transaction that you are interested
5115 * in walking through. Any nodes or leaves older than min_trans are
5116 * skipped over (without reading them).
5117 *
Chris Mason3f157a22008-06-25 16:01:31 -04005118 * returns zero if something useful was found, < 0 on error and 1 if there
5119 * was nothing in the tree that matched the search criteria.
5120 */
5121int btrfs_search_forward(struct btrfs_root *root, struct btrfs_key *min_key,
Eric Sandeende78b512013-01-31 18:21:12 +00005122 struct btrfs_path *path,
Chris Mason3f157a22008-06-25 16:01:31 -04005123 u64 min_trans)
5124{
5125 struct extent_buffer *cur;
5126 struct btrfs_key found_key;
5127 int slot;
Yan96524802008-07-24 12:19:49 -04005128 int sret;
Chris Mason3f157a22008-06-25 16:01:31 -04005129 u32 nritems;
5130 int level;
5131 int ret = 1;
Filipe Mananaf98de9b2014-08-04 19:37:21 +01005132 int keep_locks = path->keep_locks;
Chris Mason3f157a22008-06-25 16:01:31 -04005133
Filipe Mananaf98de9b2014-08-04 19:37:21 +01005134 path->keep_locks = 1;
Chris Mason3f157a22008-06-25 16:01:31 -04005135again:
Chris Masonbd681512011-07-16 15:23:14 -04005136 cur = btrfs_read_lock_root_node(root);
Chris Mason3f157a22008-06-25 16:01:31 -04005137 level = btrfs_header_level(cur);
Chris Masone02119d2008-09-05 16:13:11 -04005138 WARN_ON(path->nodes[level]);
Chris Mason3f157a22008-06-25 16:01:31 -04005139 path->nodes[level] = cur;
Chris Masonbd681512011-07-16 15:23:14 -04005140 path->locks[level] = BTRFS_READ_LOCK;
Chris Mason3f157a22008-06-25 16:01:31 -04005141
5142 if (btrfs_header_generation(cur) < min_trans) {
5143 ret = 1;
5144 goto out;
5145 }
Chris Masond3977122009-01-05 21:25:51 -05005146 while (1) {
Chris Mason3f157a22008-06-25 16:01:31 -04005147 nritems = btrfs_header_nritems(cur);
5148 level = btrfs_header_level(cur);
Yan96524802008-07-24 12:19:49 -04005149 sret = bin_search(cur, min_key, level, &slot);
Chris Mason3f157a22008-06-25 16:01:31 -04005150
Chris Mason323ac952008-10-01 19:05:46 -04005151 /* at the lowest level, we're done, setup the path and exit */
5152 if (level == path->lowest_level) {
Chris Masone02119d2008-09-05 16:13:11 -04005153 if (slot >= nritems)
5154 goto find_next_key;
Chris Mason3f157a22008-06-25 16:01:31 -04005155 ret = 0;
5156 path->slots[level] = slot;
5157 btrfs_item_key_to_cpu(cur, &found_key, slot);
5158 goto out;
5159 }
Yan96524802008-07-24 12:19:49 -04005160 if (sret && slot > 0)
5161 slot--;
Chris Mason3f157a22008-06-25 16:01:31 -04005162 /*
Eric Sandeende78b512013-01-31 18:21:12 +00005163 * check this node pointer against the min_trans parameters.
5164 * If it is too old, old, skip to the next one.
Chris Mason3f157a22008-06-25 16:01:31 -04005165 */
Chris Masond3977122009-01-05 21:25:51 -05005166 while (slot < nritems) {
Chris Mason3f157a22008-06-25 16:01:31 -04005167 u64 gen;
Chris Masone02119d2008-09-05 16:13:11 -04005168
Chris Mason3f157a22008-06-25 16:01:31 -04005169 gen = btrfs_node_ptr_generation(cur, slot);
5170 if (gen < min_trans) {
5171 slot++;
5172 continue;
5173 }
Eric Sandeende78b512013-01-31 18:21:12 +00005174 break;
Chris Mason3f157a22008-06-25 16:01:31 -04005175 }
Chris Masone02119d2008-09-05 16:13:11 -04005176find_next_key:
Chris Mason3f157a22008-06-25 16:01:31 -04005177 /*
5178 * we didn't find a candidate key in this node, walk forward
5179 * and find another one
5180 */
5181 if (slot >= nritems) {
Chris Masone02119d2008-09-05 16:13:11 -04005182 path->slots[level] = slot;
Chris Masonb4ce94d2009-02-04 09:25:08 -05005183 btrfs_set_path_blocking(path);
Chris Masone02119d2008-09-05 16:13:11 -04005184 sret = btrfs_find_next_key(root, path, min_key, level,
Eric Sandeende78b512013-01-31 18:21:12 +00005185 min_trans);
Chris Masone02119d2008-09-05 16:13:11 -04005186 if (sret == 0) {
David Sterbab3b4aa72011-04-21 01:20:15 +02005187 btrfs_release_path(path);
Chris Mason3f157a22008-06-25 16:01:31 -04005188 goto again;
5189 } else {
5190 goto out;
5191 }
5192 }
5193 /* save our key for returning back */
5194 btrfs_node_key_to_cpu(cur, &found_key, slot);
5195 path->slots[level] = slot;
5196 if (level == path->lowest_level) {
5197 ret = 0;
Chris Mason3f157a22008-06-25 16:01:31 -04005198 goto out;
5199 }
Chris Masonb4ce94d2009-02-04 09:25:08 -05005200 btrfs_set_path_blocking(path);
Chris Mason3f157a22008-06-25 16:01:31 -04005201 cur = read_node_slot(root, cur, slot);
Jeff Mahoney79787ea2012-03-12 16:03:00 +01005202 BUG_ON(!cur); /* -ENOMEM */
Chris Mason3f157a22008-06-25 16:01:31 -04005203
Chris Masonbd681512011-07-16 15:23:14 -04005204 btrfs_tree_read_lock(cur);
Chris Masonb4ce94d2009-02-04 09:25:08 -05005205
Chris Masonbd681512011-07-16 15:23:14 -04005206 path->locks[level - 1] = BTRFS_READ_LOCK;
Chris Mason3f157a22008-06-25 16:01:31 -04005207 path->nodes[level - 1] = cur;
Chris Masonf7c79f32012-03-19 15:54:38 -04005208 unlock_up(path, level, 1, 0, NULL);
Chris Masonbd681512011-07-16 15:23:14 -04005209 btrfs_clear_path_blocking(path, NULL, 0);
Chris Mason3f157a22008-06-25 16:01:31 -04005210 }
5211out:
Filipe Mananaf98de9b2014-08-04 19:37:21 +01005212 path->keep_locks = keep_locks;
5213 if (ret == 0) {
5214 btrfs_unlock_up_safe(path, path->lowest_level + 1);
5215 btrfs_set_path_blocking(path);
Chris Mason3f157a22008-06-25 16:01:31 -04005216 memcpy(min_key, &found_key, sizeof(found_key));
Filipe Mananaf98de9b2014-08-04 19:37:21 +01005217 }
Chris Mason3f157a22008-06-25 16:01:31 -04005218 return ret;
5219}
5220
Alexander Block70698302012-06-05 21:07:48 +02005221static void tree_move_down(struct btrfs_root *root,
5222 struct btrfs_path *path,
5223 int *level, int root_level)
5224{
Chris Mason74dd17f2012-08-07 16:25:13 -04005225 BUG_ON(*level == 0);
Alexander Block70698302012-06-05 21:07:48 +02005226 path->nodes[*level - 1] = read_node_slot(root, path->nodes[*level],
5227 path->slots[*level]);
5228 path->slots[*level - 1] = 0;
5229 (*level)--;
5230}
5231
5232static int tree_move_next_or_upnext(struct btrfs_root *root,
5233 struct btrfs_path *path,
5234 int *level, int root_level)
5235{
5236 int ret = 0;
5237 int nritems;
5238 nritems = btrfs_header_nritems(path->nodes[*level]);
5239
5240 path->slots[*level]++;
5241
Chris Mason74dd17f2012-08-07 16:25:13 -04005242 while (path->slots[*level] >= nritems) {
Alexander Block70698302012-06-05 21:07:48 +02005243 if (*level == root_level)
5244 return -1;
5245
5246 /* move upnext */
5247 path->slots[*level] = 0;
5248 free_extent_buffer(path->nodes[*level]);
5249 path->nodes[*level] = NULL;
5250 (*level)++;
5251 path->slots[*level]++;
5252
5253 nritems = btrfs_header_nritems(path->nodes[*level]);
5254 ret = 1;
5255 }
5256 return ret;
5257}
5258
5259/*
5260 * Returns 1 if it had to move up and next. 0 is returned if it moved only next
5261 * or down.
5262 */
5263static int tree_advance(struct btrfs_root *root,
5264 struct btrfs_path *path,
5265 int *level, int root_level,
5266 int allow_down,
5267 struct btrfs_key *key)
5268{
5269 int ret;
5270
5271 if (*level == 0 || !allow_down) {
5272 ret = tree_move_next_or_upnext(root, path, level, root_level);
5273 } else {
5274 tree_move_down(root, path, level, root_level);
5275 ret = 0;
5276 }
5277 if (ret >= 0) {
5278 if (*level == 0)
5279 btrfs_item_key_to_cpu(path->nodes[*level], key,
5280 path->slots[*level]);
5281 else
5282 btrfs_node_key_to_cpu(path->nodes[*level], key,
5283 path->slots[*level]);
5284 }
5285 return ret;
5286}
5287
5288static int tree_compare_item(struct btrfs_root *left_root,
5289 struct btrfs_path *left_path,
5290 struct btrfs_path *right_path,
5291 char *tmp_buf)
5292{
5293 int cmp;
5294 int len1, len2;
5295 unsigned long off1, off2;
5296
5297 len1 = btrfs_item_size_nr(left_path->nodes[0], left_path->slots[0]);
5298 len2 = btrfs_item_size_nr(right_path->nodes[0], right_path->slots[0]);
5299 if (len1 != len2)
5300 return 1;
5301
5302 off1 = btrfs_item_ptr_offset(left_path->nodes[0], left_path->slots[0]);
5303 off2 = btrfs_item_ptr_offset(right_path->nodes[0],
5304 right_path->slots[0]);
5305
5306 read_extent_buffer(left_path->nodes[0], tmp_buf, off1, len1);
5307
5308 cmp = memcmp_extent_buffer(right_path->nodes[0], tmp_buf, off2, len1);
5309 if (cmp)
5310 return 1;
5311 return 0;
5312}
5313
5314#define ADVANCE 1
5315#define ADVANCE_ONLY_NEXT -1
5316
5317/*
5318 * This function compares two trees and calls the provided callback for
5319 * every changed/new/deleted item it finds.
5320 * If shared tree blocks are encountered, whole subtrees are skipped, making
5321 * the compare pretty fast on snapshotted subvolumes.
5322 *
5323 * This currently works on commit roots only. As commit roots are read only,
5324 * we don't do any locking. The commit roots are protected with transactions.
5325 * Transactions are ended and rejoined when a commit is tried in between.
5326 *
5327 * This function checks for modifications done to the trees while comparing.
5328 * If it detects a change, it aborts immediately.
5329 */
5330int btrfs_compare_trees(struct btrfs_root *left_root,
5331 struct btrfs_root *right_root,
5332 btrfs_changed_cb_t changed_cb, void *ctx)
5333{
5334 int ret;
5335 int cmp;
Alexander Block70698302012-06-05 21:07:48 +02005336 struct btrfs_path *left_path = NULL;
5337 struct btrfs_path *right_path = NULL;
5338 struct btrfs_key left_key;
5339 struct btrfs_key right_key;
5340 char *tmp_buf = NULL;
5341 int left_root_level;
5342 int right_root_level;
5343 int left_level;
5344 int right_level;
5345 int left_end_reached;
5346 int right_end_reached;
5347 int advance_left;
5348 int advance_right;
5349 u64 left_blockptr;
5350 u64 right_blockptr;
Filipe Manana6baa4292014-02-20 21:15:25 +00005351 u64 left_gen;
5352 u64 right_gen;
Alexander Block70698302012-06-05 21:07:48 +02005353
5354 left_path = btrfs_alloc_path();
5355 if (!left_path) {
5356 ret = -ENOMEM;
5357 goto out;
5358 }
5359 right_path = btrfs_alloc_path();
5360 if (!right_path) {
5361 ret = -ENOMEM;
5362 goto out;
5363 }
5364
David Sterba707e8a02014-06-04 19:22:26 +02005365 tmp_buf = kmalloc(left_root->nodesize, GFP_NOFS);
Alexander Block70698302012-06-05 21:07:48 +02005366 if (!tmp_buf) {
5367 ret = -ENOMEM;
5368 goto out;
5369 }
5370
5371 left_path->search_commit_root = 1;
5372 left_path->skip_locking = 1;
5373 right_path->search_commit_root = 1;
5374 right_path->skip_locking = 1;
5375
Alexander Block70698302012-06-05 21:07:48 +02005376 /*
5377 * Strategy: Go to the first items of both trees. Then do
5378 *
5379 * If both trees are at level 0
5380 * Compare keys of current items
5381 * If left < right treat left item as new, advance left tree
5382 * and repeat
5383 * If left > right treat right item as deleted, advance right tree
5384 * and repeat
5385 * If left == right do deep compare of items, treat as changed if
5386 * needed, advance both trees and repeat
5387 * If both trees are at the same level but not at level 0
5388 * Compare keys of current nodes/leafs
5389 * If left < right advance left tree and repeat
5390 * If left > right advance right tree and repeat
5391 * If left == right compare blockptrs of the next nodes/leafs
5392 * If they match advance both trees but stay at the same level
5393 * and repeat
5394 * If they don't match advance both trees while allowing to go
5395 * deeper and repeat
5396 * If tree levels are different
5397 * Advance the tree that needs it and repeat
5398 *
5399 * Advancing a tree means:
5400 * If we are at level 0, try to go to the next slot. If that's not
5401 * possible, go one level up and repeat. Stop when we found a level
5402 * where we could go to the next slot. We may at this point be on a
5403 * node or a leaf.
5404 *
5405 * If we are not at level 0 and not on shared tree blocks, go one
5406 * level deeper.
5407 *
5408 * If we are not at level 0 and on shared tree blocks, go one slot to
5409 * the right if possible or go up and right.
5410 */
5411
Josef Bacik3f8a18c2014-03-28 17:16:01 -04005412 down_read(&left_root->fs_info->commit_root_sem);
Alexander Block70698302012-06-05 21:07:48 +02005413 left_level = btrfs_header_level(left_root->commit_root);
5414 left_root_level = left_level;
5415 left_path->nodes[left_level] = left_root->commit_root;
5416 extent_buffer_get(left_path->nodes[left_level]);
5417
5418 right_level = btrfs_header_level(right_root->commit_root);
5419 right_root_level = right_level;
5420 right_path->nodes[right_level] = right_root->commit_root;
5421 extent_buffer_get(right_path->nodes[right_level]);
Josef Bacik3f8a18c2014-03-28 17:16:01 -04005422 up_read(&left_root->fs_info->commit_root_sem);
Alexander Block70698302012-06-05 21:07:48 +02005423
5424 if (left_level == 0)
5425 btrfs_item_key_to_cpu(left_path->nodes[left_level],
5426 &left_key, left_path->slots[left_level]);
5427 else
5428 btrfs_node_key_to_cpu(left_path->nodes[left_level],
5429 &left_key, left_path->slots[left_level]);
5430 if (right_level == 0)
5431 btrfs_item_key_to_cpu(right_path->nodes[right_level],
5432 &right_key, right_path->slots[right_level]);
5433 else
5434 btrfs_node_key_to_cpu(right_path->nodes[right_level],
5435 &right_key, right_path->slots[right_level]);
5436
5437 left_end_reached = right_end_reached = 0;
5438 advance_left = advance_right = 0;
5439
5440 while (1) {
Alexander Block70698302012-06-05 21:07:48 +02005441 if (advance_left && !left_end_reached) {
5442 ret = tree_advance(left_root, left_path, &left_level,
5443 left_root_level,
5444 advance_left != ADVANCE_ONLY_NEXT,
5445 &left_key);
5446 if (ret < 0)
5447 left_end_reached = ADVANCE;
5448 advance_left = 0;
5449 }
5450 if (advance_right && !right_end_reached) {
5451 ret = tree_advance(right_root, right_path, &right_level,
5452 right_root_level,
5453 advance_right != ADVANCE_ONLY_NEXT,
5454 &right_key);
5455 if (ret < 0)
5456 right_end_reached = ADVANCE;
5457 advance_right = 0;
5458 }
5459
5460 if (left_end_reached && right_end_reached) {
5461 ret = 0;
5462 goto out;
5463 } else if (left_end_reached) {
5464 if (right_level == 0) {
5465 ret = changed_cb(left_root, right_root,
5466 left_path, right_path,
5467 &right_key,
5468 BTRFS_COMPARE_TREE_DELETED,
5469 ctx);
5470 if (ret < 0)
5471 goto out;
5472 }
5473 advance_right = ADVANCE;
5474 continue;
5475 } else if (right_end_reached) {
5476 if (left_level == 0) {
5477 ret = changed_cb(left_root, right_root,
5478 left_path, right_path,
5479 &left_key,
5480 BTRFS_COMPARE_TREE_NEW,
5481 ctx);
5482 if (ret < 0)
5483 goto out;
5484 }
5485 advance_left = ADVANCE;
5486 continue;
5487 }
5488
5489 if (left_level == 0 && right_level == 0) {
5490 cmp = btrfs_comp_cpu_keys(&left_key, &right_key);
5491 if (cmp < 0) {
5492 ret = changed_cb(left_root, right_root,
5493 left_path, right_path,
5494 &left_key,
5495 BTRFS_COMPARE_TREE_NEW,
5496 ctx);
5497 if (ret < 0)
5498 goto out;
5499 advance_left = ADVANCE;
5500 } else if (cmp > 0) {
5501 ret = changed_cb(left_root, right_root,
5502 left_path, right_path,
5503 &right_key,
5504 BTRFS_COMPARE_TREE_DELETED,
5505 ctx);
5506 if (ret < 0)
5507 goto out;
5508 advance_right = ADVANCE;
5509 } else {
Fabian Frederickb99d9a62014-09-25 19:35:02 +02005510 enum btrfs_compare_tree_result result;
Josef Bacikba5e8f22013-08-16 16:52:55 -04005511
Chris Mason74dd17f2012-08-07 16:25:13 -04005512 WARN_ON(!extent_buffer_uptodate(left_path->nodes[0]));
Alexander Block70698302012-06-05 21:07:48 +02005513 ret = tree_compare_item(left_root, left_path,
5514 right_path, tmp_buf);
Josef Bacikba5e8f22013-08-16 16:52:55 -04005515 if (ret)
Fabian Frederickb99d9a62014-09-25 19:35:02 +02005516 result = BTRFS_COMPARE_TREE_CHANGED;
Josef Bacikba5e8f22013-08-16 16:52:55 -04005517 else
Fabian Frederickb99d9a62014-09-25 19:35:02 +02005518 result = BTRFS_COMPARE_TREE_SAME;
Josef Bacikba5e8f22013-08-16 16:52:55 -04005519 ret = changed_cb(left_root, right_root,
5520 left_path, right_path,
Fabian Frederickb99d9a62014-09-25 19:35:02 +02005521 &left_key, result, ctx);
Josef Bacikba5e8f22013-08-16 16:52:55 -04005522 if (ret < 0)
5523 goto out;
Alexander Block70698302012-06-05 21:07:48 +02005524 advance_left = ADVANCE;
5525 advance_right = ADVANCE;
5526 }
5527 } else if (left_level == right_level) {
5528 cmp = btrfs_comp_cpu_keys(&left_key, &right_key);
5529 if (cmp < 0) {
5530 advance_left = ADVANCE;
5531 } else if (cmp > 0) {
5532 advance_right = ADVANCE;
5533 } else {
5534 left_blockptr = btrfs_node_blockptr(
5535 left_path->nodes[left_level],
5536 left_path->slots[left_level]);
5537 right_blockptr = btrfs_node_blockptr(
5538 right_path->nodes[right_level],
5539 right_path->slots[right_level]);
Filipe Manana6baa4292014-02-20 21:15:25 +00005540 left_gen = btrfs_node_ptr_generation(
5541 left_path->nodes[left_level],
5542 left_path->slots[left_level]);
5543 right_gen = btrfs_node_ptr_generation(
5544 right_path->nodes[right_level],
5545 right_path->slots[right_level]);
5546 if (left_blockptr == right_blockptr &&
5547 left_gen == right_gen) {
Alexander Block70698302012-06-05 21:07:48 +02005548 /*
5549 * As we're on a shared block, don't
5550 * allow to go deeper.
5551 */
5552 advance_left = ADVANCE_ONLY_NEXT;
5553 advance_right = ADVANCE_ONLY_NEXT;
5554 } else {
5555 advance_left = ADVANCE;
5556 advance_right = ADVANCE;
5557 }
5558 }
5559 } else if (left_level < right_level) {
5560 advance_right = ADVANCE;
5561 } else {
5562 advance_left = ADVANCE;
5563 }
5564 }
5565
5566out:
5567 btrfs_free_path(left_path);
5568 btrfs_free_path(right_path);
5569 kfree(tmp_buf);
Alexander Block70698302012-06-05 21:07:48 +02005570 return ret;
5571}
5572
Chris Mason3f157a22008-06-25 16:01:31 -04005573/*
5574 * this is similar to btrfs_next_leaf, but does not try to preserve
5575 * and fixup the path. It looks for and returns the next key in the
Eric Sandeende78b512013-01-31 18:21:12 +00005576 * tree based on the current path and the min_trans parameters.
Chris Mason3f157a22008-06-25 16:01:31 -04005577 *
5578 * 0 is returned if another key is found, < 0 if there are any errors
5579 * and 1 is returned if there are no higher keys in the tree
5580 *
5581 * path->keep_locks should be set to 1 on the search made before
5582 * calling this function.
5583 */
Chris Masone7a84562008-06-25 16:01:31 -04005584int btrfs_find_next_key(struct btrfs_root *root, struct btrfs_path *path,
Eric Sandeende78b512013-01-31 18:21:12 +00005585 struct btrfs_key *key, int level, u64 min_trans)
Chris Masone7a84562008-06-25 16:01:31 -04005586{
Chris Masone7a84562008-06-25 16:01:31 -04005587 int slot;
5588 struct extent_buffer *c;
5589
Chris Mason934d3752008-12-08 16:43:10 -05005590 WARN_ON(!path->keep_locks);
Chris Masond3977122009-01-05 21:25:51 -05005591 while (level < BTRFS_MAX_LEVEL) {
Chris Masone7a84562008-06-25 16:01:31 -04005592 if (!path->nodes[level])
5593 return 1;
5594
5595 slot = path->slots[level] + 1;
5596 c = path->nodes[level];
Chris Mason3f157a22008-06-25 16:01:31 -04005597next:
Chris Masone7a84562008-06-25 16:01:31 -04005598 if (slot >= btrfs_header_nritems(c)) {
Yan Zheng33c66f42009-07-22 09:59:00 -04005599 int ret;
5600 int orig_lowest;
5601 struct btrfs_key cur_key;
5602 if (level + 1 >= BTRFS_MAX_LEVEL ||
5603 !path->nodes[level + 1])
Chris Masone7a84562008-06-25 16:01:31 -04005604 return 1;
Yan Zheng33c66f42009-07-22 09:59:00 -04005605
5606 if (path->locks[level + 1]) {
5607 level++;
5608 continue;
5609 }
5610
5611 slot = btrfs_header_nritems(c) - 1;
5612 if (level == 0)
5613 btrfs_item_key_to_cpu(c, &cur_key, slot);
5614 else
5615 btrfs_node_key_to_cpu(c, &cur_key, slot);
5616
5617 orig_lowest = path->lowest_level;
David Sterbab3b4aa72011-04-21 01:20:15 +02005618 btrfs_release_path(path);
Yan Zheng33c66f42009-07-22 09:59:00 -04005619 path->lowest_level = level;
5620 ret = btrfs_search_slot(NULL, root, &cur_key, path,
5621 0, 0);
5622 path->lowest_level = orig_lowest;
5623 if (ret < 0)
5624 return ret;
5625
5626 c = path->nodes[level];
5627 slot = path->slots[level];
5628 if (ret == 0)
5629 slot++;
5630 goto next;
Chris Masone7a84562008-06-25 16:01:31 -04005631 }
Yan Zheng33c66f42009-07-22 09:59:00 -04005632
Chris Masone7a84562008-06-25 16:01:31 -04005633 if (level == 0)
5634 btrfs_item_key_to_cpu(c, key, slot);
Chris Mason3f157a22008-06-25 16:01:31 -04005635 else {
Chris Mason3f157a22008-06-25 16:01:31 -04005636 u64 gen = btrfs_node_ptr_generation(c, slot);
5637
Chris Mason3f157a22008-06-25 16:01:31 -04005638 if (gen < min_trans) {
5639 slot++;
5640 goto next;
5641 }
Chris Masone7a84562008-06-25 16:01:31 -04005642 btrfs_node_key_to_cpu(c, key, slot);
Chris Mason3f157a22008-06-25 16:01:31 -04005643 }
Chris Masone7a84562008-06-25 16:01:31 -04005644 return 0;
5645 }
5646 return 1;
5647}
5648
Chris Mason7bb86312007-12-11 09:25:06 -05005649/*
Chris Mason925baed2008-06-25 16:01:30 -04005650 * search the tree again to find a leaf with greater keys
Chris Mason0f70abe2007-02-28 16:46:22 -05005651 * returns 0 if it found something or 1 if there are no greater leaves.
5652 * returns < 0 on io errors.
Chris Mason97571fd2007-02-24 13:39:08 -05005653 */
Chris Mason234b63a2007-03-13 10:46:10 -04005654int btrfs_next_leaf(struct btrfs_root *root, struct btrfs_path *path)
Chris Masond97e63b2007-02-20 16:40:44 -05005655{
Jan Schmidt3d7806e2012-06-11 08:29:29 +02005656 return btrfs_next_old_leaf(root, path, 0);
5657}
5658
5659int btrfs_next_old_leaf(struct btrfs_root *root, struct btrfs_path *path,
5660 u64 time_seq)
5661{
Chris Masond97e63b2007-02-20 16:40:44 -05005662 int slot;
Chris Mason8e73f272009-04-03 10:14:18 -04005663 int level;
Chris Mason5f39d392007-10-15 16:14:19 -04005664 struct extent_buffer *c;
Chris Mason8e73f272009-04-03 10:14:18 -04005665 struct extent_buffer *next;
Chris Mason925baed2008-06-25 16:01:30 -04005666 struct btrfs_key key;
5667 u32 nritems;
5668 int ret;
Chris Mason8e73f272009-04-03 10:14:18 -04005669 int old_spinning = path->leave_spinning;
Chris Masonbd681512011-07-16 15:23:14 -04005670 int next_rw_lock = 0;
Chris Mason925baed2008-06-25 16:01:30 -04005671
5672 nritems = btrfs_header_nritems(path->nodes[0]);
Chris Masond3977122009-01-05 21:25:51 -05005673 if (nritems == 0)
Chris Mason925baed2008-06-25 16:01:30 -04005674 return 1;
Chris Mason925baed2008-06-25 16:01:30 -04005675
Chris Mason8e73f272009-04-03 10:14:18 -04005676 btrfs_item_key_to_cpu(path->nodes[0], &key, nritems - 1);
5677again:
5678 level = 1;
5679 next = NULL;
Chris Masonbd681512011-07-16 15:23:14 -04005680 next_rw_lock = 0;
David Sterbab3b4aa72011-04-21 01:20:15 +02005681 btrfs_release_path(path);
Chris Mason8e73f272009-04-03 10:14:18 -04005682
Chris Masona2135012008-06-25 16:01:30 -04005683 path->keep_locks = 1;
Chris Mason31533fb2011-07-26 16:01:59 -04005684 path->leave_spinning = 1;
Chris Mason8e73f272009-04-03 10:14:18 -04005685
Jan Schmidt3d7806e2012-06-11 08:29:29 +02005686 if (time_seq)
5687 ret = btrfs_search_old_slot(root, &key, path, time_seq);
5688 else
5689 ret = btrfs_search_slot(NULL, root, &key, path, 0, 0);
Chris Mason925baed2008-06-25 16:01:30 -04005690 path->keep_locks = 0;
5691
5692 if (ret < 0)
5693 return ret;
5694
Chris Masona2135012008-06-25 16:01:30 -04005695 nritems = btrfs_header_nritems(path->nodes[0]);
Chris Mason168fd7d2008-06-25 16:01:30 -04005696 /*
5697 * by releasing the path above we dropped all our locks. A balance
5698 * could have added more items next to the key that used to be
5699 * at the very end of the block. So, check again here and
5700 * advance the path if there are now more items available.
5701 */
Chris Masona2135012008-06-25 16:01:30 -04005702 if (nritems > 0 && path->slots[0] < nritems - 1) {
Yan Zhenge457afe2009-07-22 09:59:00 -04005703 if (ret == 0)
5704 path->slots[0]++;
Chris Mason8e73f272009-04-03 10:14:18 -04005705 ret = 0;
Chris Mason925baed2008-06-25 16:01:30 -04005706 goto done;
5707 }
Liu Bo0b43e042014-06-09 11:04:49 +08005708 /*
5709 * So the above check misses one case:
5710 * - after releasing the path above, someone has removed the item that
5711 * used to be at the very end of the block, and balance between leafs
5712 * gets another one with bigger key.offset to replace it.
5713 *
5714 * This one should be returned as well, or we can get leaf corruption
5715 * later(esp. in __btrfs_drop_extents()).
5716 *
5717 * And a bit more explanation about this check,
5718 * with ret > 0, the key isn't found, the path points to the slot
5719 * where it should be inserted, so the path->slots[0] item must be the
5720 * bigger one.
5721 */
5722 if (nritems > 0 && ret > 0 && path->slots[0] == nritems - 1) {
5723 ret = 0;
5724 goto done;
5725 }
Chris Masond97e63b2007-02-20 16:40:44 -05005726
Chris Masond3977122009-01-05 21:25:51 -05005727 while (level < BTRFS_MAX_LEVEL) {
Chris Mason8e73f272009-04-03 10:14:18 -04005728 if (!path->nodes[level]) {
5729 ret = 1;
5730 goto done;
5731 }
Chris Mason5f39d392007-10-15 16:14:19 -04005732
Chris Masond97e63b2007-02-20 16:40:44 -05005733 slot = path->slots[level] + 1;
5734 c = path->nodes[level];
Chris Mason5f39d392007-10-15 16:14:19 -04005735 if (slot >= btrfs_header_nritems(c)) {
Chris Masond97e63b2007-02-20 16:40:44 -05005736 level++;
Chris Mason8e73f272009-04-03 10:14:18 -04005737 if (level == BTRFS_MAX_LEVEL) {
5738 ret = 1;
5739 goto done;
5740 }
Chris Masond97e63b2007-02-20 16:40:44 -05005741 continue;
5742 }
Chris Mason5f39d392007-10-15 16:14:19 -04005743
Chris Mason925baed2008-06-25 16:01:30 -04005744 if (next) {
Chris Masonbd681512011-07-16 15:23:14 -04005745 btrfs_tree_unlock_rw(next, next_rw_lock);
Chris Mason5f39d392007-10-15 16:14:19 -04005746 free_extent_buffer(next);
Chris Mason925baed2008-06-25 16:01:30 -04005747 }
Chris Mason5f39d392007-10-15 16:14:19 -04005748
Chris Mason8e73f272009-04-03 10:14:18 -04005749 next = c;
Chris Masonbd681512011-07-16 15:23:14 -04005750 next_rw_lock = path->locks[level];
Chris Mason8e73f272009-04-03 10:14:18 -04005751 ret = read_block_for_search(NULL, root, path, &next, level,
Jan Schmidt5d9e75c42012-05-16 18:25:47 +02005752 slot, &key, 0);
Chris Mason8e73f272009-04-03 10:14:18 -04005753 if (ret == -EAGAIN)
5754 goto again;
Chris Mason5f39d392007-10-15 16:14:19 -04005755
Chris Mason76a05b32009-05-14 13:24:30 -04005756 if (ret < 0) {
David Sterbab3b4aa72011-04-21 01:20:15 +02005757 btrfs_release_path(path);
Chris Mason76a05b32009-05-14 13:24:30 -04005758 goto done;
5759 }
5760
Chris Mason5cd57b22008-06-25 16:01:30 -04005761 if (!path->skip_locking) {
Chris Masonbd681512011-07-16 15:23:14 -04005762 ret = btrfs_try_tree_read_lock(next);
Jan Schmidtd42244a2012-06-22 14:51:15 +02005763 if (!ret && time_seq) {
5764 /*
5765 * If we don't get the lock, we may be racing
5766 * with push_leaf_left, holding that lock while
5767 * itself waiting for the leaf we've currently
5768 * locked. To solve this situation, we give up
5769 * on our lock and cycle.
5770 */
Jan Schmidtcf538832012-07-04 15:42:48 +02005771 free_extent_buffer(next);
Jan Schmidtd42244a2012-06-22 14:51:15 +02005772 btrfs_release_path(path);
5773 cond_resched();
5774 goto again;
5775 }
Chris Mason8e73f272009-04-03 10:14:18 -04005776 if (!ret) {
5777 btrfs_set_path_blocking(path);
Chris Masonbd681512011-07-16 15:23:14 -04005778 btrfs_tree_read_lock(next);
Chris Mason31533fb2011-07-26 16:01:59 -04005779 btrfs_clear_path_blocking(path, next,
Chris Masonbd681512011-07-16 15:23:14 -04005780 BTRFS_READ_LOCK);
Chris Mason8e73f272009-04-03 10:14:18 -04005781 }
Chris Mason31533fb2011-07-26 16:01:59 -04005782 next_rw_lock = BTRFS_READ_LOCK;
Chris Mason5cd57b22008-06-25 16:01:30 -04005783 }
Chris Masond97e63b2007-02-20 16:40:44 -05005784 break;
5785 }
5786 path->slots[level] = slot;
Chris Masond3977122009-01-05 21:25:51 -05005787 while (1) {
Chris Masond97e63b2007-02-20 16:40:44 -05005788 level--;
5789 c = path->nodes[level];
Chris Mason925baed2008-06-25 16:01:30 -04005790 if (path->locks[level])
Chris Masonbd681512011-07-16 15:23:14 -04005791 btrfs_tree_unlock_rw(c, path->locks[level]);
Chris Mason8e73f272009-04-03 10:14:18 -04005792
Chris Mason5f39d392007-10-15 16:14:19 -04005793 free_extent_buffer(c);
Chris Masond97e63b2007-02-20 16:40:44 -05005794 path->nodes[level] = next;
5795 path->slots[level] = 0;
Chris Masona74a4b92008-06-25 16:01:31 -04005796 if (!path->skip_locking)
Chris Masonbd681512011-07-16 15:23:14 -04005797 path->locks[level] = next_rw_lock;
Chris Masond97e63b2007-02-20 16:40:44 -05005798 if (!level)
5799 break;
Chris Masonb4ce94d2009-02-04 09:25:08 -05005800
Chris Mason8e73f272009-04-03 10:14:18 -04005801 ret = read_block_for_search(NULL, root, path, &next, level,
Jan Schmidt5d9e75c42012-05-16 18:25:47 +02005802 0, &key, 0);
Chris Mason8e73f272009-04-03 10:14:18 -04005803 if (ret == -EAGAIN)
5804 goto again;
5805
Chris Mason76a05b32009-05-14 13:24:30 -04005806 if (ret < 0) {
David Sterbab3b4aa72011-04-21 01:20:15 +02005807 btrfs_release_path(path);
Chris Mason76a05b32009-05-14 13:24:30 -04005808 goto done;
5809 }
5810
Chris Mason5cd57b22008-06-25 16:01:30 -04005811 if (!path->skip_locking) {
Chris Masonbd681512011-07-16 15:23:14 -04005812 ret = btrfs_try_tree_read_lock(next);
Chris Mason8e73f272009-04-03 10:14:18 -04005813 if (!ret) {
5814 btrfs_set_path_blocking(path);
Chris Masonbd681512011-07-16 15:23:14 -04005815 btrfs_tree_read_lock(next);
Chris Mason31533fb2011-07-26 16:01:59 -04005816 btrfs_clear_path_blocking(path, next,
Chris Masonbd681512011-07-16 15:23:14 -04005817 BTRFS_READ_LOCK);
Chris Mason8e73f272009-04-03 10:14:18 -04005818 }
Chris Mason31533fb2011-07-26 16:01:59 -04005819 next_rw_lock = BTRFS_READ_LOCK;
Chris Mason5cd57b22008-06-25 16:01:30 -04005820 }
Chris Masond97e63b2007-02-20 16:40:44 -05005821 }
Chris Mason8e73f272009-04-03 10:14:18 -04005822 ret = 0;
Chris Mason925baed2008-06-25 16:01:30 -04005823done:
Chris Masonf7c79f32012-03-19 15:54:38 -04005824 unlock_up(path, 0, 1, 0, NULL);
Chris Mason8e73f272009-04-03 10:14:18 -04005825 path->leave_spinning = old_spinning;
5826 if (!old_spinning)
5827 btrfs_set_path_blocking(path);
5828
5829 return ret;
Chris Masond97e63b2007-02-20 16:40:44 -05005830}
Chris Mason0b86a832008-03-24 15:01:56 -04005831
Chris Mason3f157a22008-06-25 16:01:31 -04005832/*
5833 * this uses btrfs_prev_leaf to walk backwards in the tree, and keeps
5834 * searching until it gets past min_objectid or finds an item of 'type'
5835 *
5836 * returns 0 if something is found, 1 if nothing was found and < 0 on error
5837 */
Chris Mason0b86a832008-03-24 15:01:56 -04005838int btrfs_previous_item(struct btrfs_root *root,
5839 struct btrfs_path *path, u64 min_objectid,
5840 int type)
5841{
5842 struct btrfs_key found_key;
5843 struct extent_buffer *leaf;
Chris Masone02119d2008-09-05 16:13:11 -04005844 u32 nritems;
Chris Mason0b86a832008-03-24 15:01:56 -04005845 int ret;
5846
Chris Masond3977122009-01-05 21:25:51 -05005847 while (1) {
Chris Mason0b86a832008-03-24 15:01:56 -04005848 if (path->slots[0] == 0) {
Chris Masonb4ce94d2009-02-04 09:25:08 -05005849 btrfs_set_path_blocking(path);
Chris Mason0b86a832008-03-24 15:01:56 -04005850 ret = btrfs_prev_leaf(root, path);
5851 if (ret != 0)
5852 return ret;
5853 } else {
5854 path->slots[0]--;
5855 }
5856 leaf = path->nodes[0];
Chris Masone02119d2008-09-05 16:13:11 -04005857 nritems = btrfs_header_nritems(leaf);
5858 if (nritems == 0)
5859 return 1;
5860 if (path->slots[0] == nritems)
5861 path->slots[0]--;
5862
Chris Mason0b86a832008-03-24 15:01:56 -04005863 btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]);
Chris Masone02119d2008-09-05 16:13:11 -04005864 if (found_key.objectid < min_objectid)
5865 break;
Yan Zheng0a4eefb2009-07-24 11:06:53 -04005866 if (found_key.type == type)
5867 return 0;
Chris Masone02119d2008-09-05 16:13:11 -04005868 if (found_key.objectid == min_objectid &&
5869 found_key.type < type)
5870 break;
Chris Mason0b86a832008-03-24 15:01:56 -04005871 }
5872 return 1;
5873}
Wang Shilongade2e0b2014-01-12 21:38:33 +08005874
5875/*
5876 * search in extent tree to find a previous Metadata/Data extent item with
5877 * min objecitd.
5878 *
5879 * returns 0 if something is found, 1 if nothing was found and < 0 on error
5880 */
5881int btrfs_previous_extent_item(struct btrfs_root *root,
5882 struct btrfs_path *path, u64 min_objectid)
5883{
5884 struct btrfs_key found_key;
5885 struct extent_buffer *leaf;
5886 u32 nritems;
5887 int ret;
5888
5889 while (1) {
5890 if (path->slots[0] == 0) {
5891 btrfs_set_path_blocking(path);
5892 ret = btrfs_prev_leaf(root, path);
5893 if (ret != 0)
5894 return ret;
5895 } else {
5896 path->slots[0]--;
5897 }
5898 leaf = path->nodes[0];
5899 nritems = btrfs_header_nritems(leaf);
5900 if (nritems == 0)
5901 return 1;
5902 if (path->slots[0] == nritems)
5903 path->slots[0]--;
5904
5905 btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]);
5906 if (found_key.objectid < min_objectid)
5907 break;
5908 if (found_key.type == BTRFS_EXTENT_ITEM_KEY ||
5909 found_key.type == BTRFS_METADATA_ITEM_KEY)
5910 return 0;
5911 if (found_key.objectid == min_objectid &&
5912 found_key.type < BTRFS_EXTENT_ITEM_KEY)
5913 break;
5914 }
5915 return 1;
5916}