blob: d7e6baf1b2054f1998608902385bc5016020c2cf [file] [log] [blame]
Chris Mason925baed2008-06-25 16:01:30 -04001/*
2 * Copyright (C) 2008 Oracle. All rights reserved.
3 *
4 * This program is free software; you can redistribute it and/or
5 * modify it under the terms of the GNU General Public
6 * License v2 as published by the Free Software Foundation.
7 *
8 * This program is distributed in the hope that it will be useful,
9 * but WITHOUT ANY WARRANTY; without even the implied warranty of
10 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
11 * General Public License for more details.
12 *
13 * You should have received a copy of the GNU General Public
14 * License along with this program; if not, write to the
15 * Free Software Foundation, Inc., 59 Temple Place - Suite 330,
16 * Boston, MA 021110-1307, USA.
17 */
18#include <linux/sched.h>
Chris Mason925baed2008-06-25 16:01:30 -040019#include <linux/pagemap.h>
20#include <linux/spinlock.h>
21#include <linux/page-flags.h>
Chris Mason4881ee52008-07-24 09:51:08 -040022#include <asm/bug.h>
Chris Mason925baed2008-06-25 16:01:30 -040023#include "ctree.h"
24#include "extent_io.h"
25#include "locking.h"
26
Eric Sandeen48a3b632013-04-25 20:41:01 +000027static void btrfs_assert_tree_read_locked(struct extent_buffer *eb);
Chris Masonbd681512011-07-16 15:23:14 -040028
29/*
30 * if we currently have a spinning reader or writer lock
31 * (indicated by the rw flag) this will bump the count
32 * of blocking holders and drop the spinlock.
33 */
34void btrfs_set_lock_blocking_rw(struct extent_buffer *eb, int rw)
Chris Masonb4ce94d2009-02-04 09:25:08 -050035{
Chris Masonea4ebde2014-06-19 14:16:52 -070036 /*
37 * no lock is required. The lock owner may change if
38 * we have a read lock, but it won't change to or away
39 * from us. If we have the write lock, we are the owner
40 * and it'll never change.
41 */
42 if (eb->lock_nested && current->pid == eb->lock_owner)
43 return;
Chris Masonbd681512011-07-16 15:23:14 -040044 if (rw == BTRFS_WRITE_LOCK) {
45 if (atomic_read(&eb->blocking_writers) == 0) {
46 WARN_ON(atomic_read(&eb->spinning_writers) != 1);
47 atomic_dec(&eb->spinning_writers);
48 btrfs_assert_tree_locked(eb);
49 atomic_inc(&eb->blocking_writers);
50 write_unlock(&eb->lock);
51 }
52 } else if (rw == BTRFS_READ_LOCK) {
53 btrfs_assert_tree_read_locked(eb);
54 atomic_inc(&eb->blocking_readers);
55 WARN_ON(atomic_read(&eb->spinning_readers) == 0);
56 atomic_dec(&eb->spinning_readers);
57 read_unlock(&eb->lock);
58 }
59 return;
Chris Masonb4ce94d2009-02-04 09:25:08 -050060}
Chris Masond3977122009-01-05 21:25:51 -050061
Chris Masonb4ce94d2009-02-04 09:25:08 -050062/*
Chris Masonbd681512011-07-16 15:23:14 -040063 * if we currently have a blocking lock, take the spinlock
64 * and drop our blocking count
Chris Masonb4ce94d2009-02-04 09:25:08 -050065 */
Chris Masonbd681512011-07-16 15:23:14 -040066void btrfs_clear_lock_blocking_rw(struct extent_buffer *eb, int rw)
Chris Masonb4ce94d2009-02-04 09:25:08 -050067{
Chris Masonea4ebde2014-06-19 14:16:52 -070068 /*
69 * no lock is required. The lock owner may change if
70 * we have a read lock, but it won't change to or away
71 * from us. If we have the write lock, we are the owner
72 * and it'll never change.
73 */
74 if (eb->lock_nested && current->pid == eb->lock_owner)
75 return;
76
Chris Masonbd681512011-07-16 15:23:14 -040077 if (rw == BTRFS_WRITE_LOCK_BLOCKING) {
78 BUG_ON(atomic_read(&eb->blocking_writers) != 1);
79 write_lock(&eb->lock);
80 WARN_ON(atomic_read(&eb->spinning_writers));
81 atomic_inc(&eb->spinning_writers);
Chris Masoncbea5ac2012-07-23 15:25:05 -040082 if (atomic_dec_and_test(&eb->blocking_writers) &&
83 waitqueue_active(&eb->write_lock_wq))
Chris Masonbd681512011-07-16 15:23:14 -040084 wake_up(&eb->write_lock_wq);
85 } else if (rw == BTRFS_READ_LOCK_BLOCKING) {
86 BUG_ON(atomic_read(&eb->blocking_readers) == 0);
87 read_lock(&eb->lock);
88 atomic_inc(&eb->spinning_readers);
Chris Masoncbea5ac2012-07-23 15:25:05 -040089 if (atomic_dec_and_test(&eb->blocking_readers) &&
90 waitqueue_active(&eb->read_lock_wq))
Chris Masonbd681512011-07-16 15:23:14 -040091 wake_up(&eb->read_lock_wq);
Chris Masonb4ce94d2009-02-04 09:25:08 -050092 }
Chris Masonbd681512011-07-16 15:23:14 -040093 return;
Chris Masonb4ce94d2009-02-04 09:25:08 -050094}
95
96/*
Chris Masonbd681512011-07-16 15:23:14 -040097 * take a spinning read lock. This will wait for any blocking
98 * writers
Chris Masonb4ce94d2009-02-04 09:25:08 -050099 */
Chris Masonbd681512011-07-16 15:23:14 -0400100void btrfs_tree_read_lock(struct extent_buffer *eb)
Chris Masonb4ce94d2009-02-04 09:25:08 -0500101{
Chris Masonbd681512011-07-16 15:23:14 -0400102again:
Chris Masonea4ebde2014-06-19 14:16:52 -0700103 BUG_ON(!atomic_read(&eb->blocking_writers) &&
104 current->pid == eb->lock_owner);
105
Arne Jansen5b25f702011-09-13 10:55:48 +0200106 read_lock(&eb->lock);
107 if (atomic_read(&eb->blocking_writers) &&
108 current->pid == eb->lock_owner) {
109 /*
110 * This extent is already write-locked by our thread. We allow
111 * an additional read lock to be added because it's for the same
112 * thread. btrfs_find_all_roots() depends on this as it may be
113 * called on a partly (write-)locked tree.
114 */
115 BUG_ON(eb->lock_nested);
116 eb->lock_nested = 1;
117 read_unlock(&eb->lock);
118 return;
119 }
Chris Masonbd681512011-07-16 15:23:14 -0400120 if (atomic_read(&eb->blocking_writers)) {
121 read_unlock(&eb->lock);
Liu Bo39f9d022012-12-27 09:01:22 +0000122 wait_event(eb->write_lock_wq,
123 atomic_read(&eb->blocking_writers) == 0);
Chris Masonbd681512011-07-16 15:23:14 -0400124 goto again;
Chris Masonb4ce94d2009-02-04 09:25:08 -0500125 }
Chris Masonbd681512011-07-16 15:23:14 -0400126 atomic_inc(&eb->read_locks);
127 atomic_inc(&eb->spinning_readers);
Chris Masonb4ce94d2009-02-04 09:25:08 -0500128}
129
130/*
Chris Masonf82c4582014-11-19 10:25:09 -0800131 * take a spinning read lock.
132 * returns 1 if we get the read lock and 0 if we don't
133 * this won't wait for blocking writers
134 */
135int btrfs_tree_read_lock_atomic(struct extent_buffer *eb)
136{
137 if (atomic_read(&eb->blocking_writers))
138 return 0;
139
140 read_lock(&eb->lock);
141 if (atomic_read(&eb->blocking_writers)) {
142 read_unlock(&eb->lock);
143 return 0;
144 }
145 atomic_inc(&eb->read_locks);
146 atomic_inc(&eb->spinning_readers);
147 return 1;
148}
149
150/*
Chris Masonbd681512011-07-16 15:23:14 -0400151 * returns 1 if we get the read lock and 0 if we don't
152 * this won't wait for blocking writers
Chris Masonb4ce94d2009-02-04 09:25:08 -0500153 */
Chris Masonbd681512011-07-16 15:23:14 -0400154int btrfs_try_tree_read_lock(struct extent_buffer *eb)
Chris Mason925baed2008-06-25 16:01:30 -0400155{
Chris Masonbd681512011-07-16 15:23:14 -0400156 if (atomic_read(&eb->blocking_writers))
157 return 0;
Chris Mason66d7e852009-03-12 20:12:45 -0400158
Chris Masonea4ebde2014-06-19 14:16:52 -0700159 if (!read_trylock(&eb->lock))
160 return 0;
161
Chris Masonbd681512011-07-16 15:23:14 -0400162 if (atomic_read(&eb->blocking_writers)) {
163 read_unlock(&eb->lock);
164 return 0;
Chris Masonf9efa9c2008-06-25 16:14:04 -0400165 }
Chris Masonbd681512011-07-16 15:23:14 -0400166 atomic_inc(&eb->read_locks);
167 atomic_inc(&eb->spinning_readers);
Chris Masonb4ce94d2009-02-04 09:25:08 -0500168 return 1;
169}
170
171/*
Chris Masonbd681512011-07-16 15:23:14 -0400172 * returns 1 if we get the read lock and 0 if we don't
173 * this won't wait for blocking writers or readers
174 */
175int btrfs_try_tree_write_lock(struct extent_buffer *eb)
176{
177 if (atomic_read(&eb->blocking_writers) ||
178 atomic_read(&eb->blocking_readers))
179 return 0;
Chris Masonea4ebde2014-06-19 14:16:52 -0700180
Chris Masonf82c4582014-11-19 10:25:09 -0800181 write_lock(&eb->lock);
Chris Masonbd681512011-07-16 15:23:14 -0400182 if (atomic_read(&eb->blocking_writers) ||
183 atomic_read(&eb->blocking_readers)) {
184 write_unlock(&eb->lock);
185 return 0;
186 }
187 atomic_inc(&eb->write_locks);
188 atomic_inc(&eb->spinning_writers);
Arne Jansen5b25f702011-09-13 10:55:48 +0200189 eb->lock_owner = current->pid;
Chris Masonbd681512011-07-16 15:23:14 -0400190 return 1;
191}
192
193/*
194 * drop a spinning read lock
195 */
196void btrfs_tree_read_unlock(struct extent_buffer *eb)
197{
Chris Masonea4ebde2014-06-19 14:16:52 -0700198 /*
199 * if we're nested, we have the write lock. No new locking
200 * is needed as long as we are the lock owner.
201 * The write unlock will do a barrier for us, and the lock_nested
202 * field only matters to the lock owner.
203 */
204 if (eb->lock_nested && current->pid == eb->lock_owner) {
205 eb->lock_nested = 0;
206 return;
Arne Jansen5b25f702011-09-13 10:55:48 +0200207 }
Chris Masonbd681512011-07-16 15:23:14 -0400208 btrfs_assert_tree_read_locked(eb);
209 WARN_ON(atomic_read(&eb->spinning_readers) == 0);
210 atomic_dec(&eb->spinning_readers);
211 atomic_dec(&eb->read_locks);
212 read_unlock(&eb->lock);
213}
214
215/*
216 * drop a blocking read lock
217 */
218void btrfs_tree_read_unlock_blocking(struct extent_buffer *eb)
219{
Chris Masonea4ebde2014-06-19 14:16:52 -0700220 /*
221 * if we're nested, we have the write lock. No new locking
222 * is needed as long as we are the lock owner.
223 * The write unlock will do a barrier for us, and the lock_nested
224 * field only matters to the lock owner.
225 */
226 if (eb->lock_nested && current->pid == eb->lock_owner) {
227 eb->lock_nested = 0;
228 return;
Arne Jansen5b25f702011-09-13 10:55:48 +0200229 }
Chris Masonbd681512011-07-16 15:23:14 -0400230 btrfs_assert_tree_read_locked(eb);
231 WARN_ON(atomic_read(&eb->blocking_readers) == 0);
Chris Masoncbea5ac2012-07-23 15:25:05 -0400232 if (atomic_dec_and_test(&eb->blocking_readers) &&
233 waitqueue_active(&eb->read_lock_wq))
Chris Masonbd681512011-07-16 15:23:14 -0400234 wake_up(&eb->read_lock_wq);
235 atomic_dec(&eb->read_locks);
236}
237
238/*
239 * take a spinning write lock. This will wait for both
240 * blocking readers or writers
Chris Masonb4ce94d2009-02-04 09:25:08 -0500241 */
Jeff Mahoney143bede2012-03-01 14:56:26 +0100242void btrfs_tree_lock(struct extent_buffer *eb)
Chris Masonb4ce94d2009-02-04 09:25:08 -0500243{
Zhaolei166f66d2015-08-06 22:39:36 +0800244 WARN_ON(eb->lock_owner == current->pid);
Chris Masonbd681512011-07-16 15:23:14 -0400245again:
246 wait_event(eb->read_lock_wq, atomic_read(&eb->blocking_readers) == 0);
247 wait_event(eb->write_lock_wq, atomic_read(&eb->blocking_writers) == 0);
248 write_lock(&eb->lock);
249 if (atomic_read(&eb->blocking_readers)) {
250 write_unlock(&eb->lock);
251 wait_event(eb->read_lock_wq,
252 atomic_read(&eb->blocking_readers) == 0);
253 goto again;
Chris Masonb4ce94d2009-02-04 09:25:08 -0500254 }
Chris Masonbd681512011-07-16 15:23:14 -0400255 if (atomic_read(&eb->blocking_writers)) {
256 write_unlock(&eb->lock);
257 wait_event(eb->write_lock_wq,
258 atomic_read(&eb->blocking_writers) == 0);
259 goto again;
260 }
261 WARN_ON(atomic_read(&eb->spinning_writers));
262 atomic_inc(&eb->spinning_writers);
263 atomic_inc(&eb->write_locks);
Arne Jansen5b25f702011-09-13 10:55:48 +0200264 eb->lock_owner = current->pid;
Chris Masonb4ce94d2009-02-04 09:25:08 -0500265}
266
Chris Masonbd681512011-07-16 15:23:14 -0400267/*
268 * drop a spinning or a blocking write lock.
269 */
Jeff Mahoney143bede2012-03-01 14:56:26 +0100270void btrfs_tree_unlock(struct extent_buffer *eb)
Chris Mason925baed2008-06-25 16:01:30 -0400271{
Chris Masonbd681512011-07-16 15:23:14 -0400272 int blockers = atomic_read(&eb->blocking_writers);
Chris Masonb4ce94d2009-02-04 09:25:08 -0500273
Chris Masonbd681512011-07-16 15:23:14 -0400274 BUG_ON(blockers > 1);
275
276 btrfs_assert_tree_locked(eb);
Chris Masonea4ebde2014-06-19 14:16:52 -0700277 eb->lock_owner = 0;
Chris Masonbd681512011-07-16 15:23:14 -0400278 atomic_dec(&eb->write_locks);
279
280 if (blockers) {
281 WARN_ON(atomic_read(&eb->spinning_writers));
282 atomic_dec(&eb->blocking_writers);
Chris Masoncbea5ac2012-07-23 15:25:05 -0400283 smp_mb();
284 if (waitqueue_active(&eb->write_lock_wq))
285 wake_up(&eb->write_lock_wq);
Chris Masonbd681512011-07-16 15:23:14 -0400286 } else {
287 WARN_ON(atomic_read(&eb->spinning_writers) != 1);
288 atomic_dec(&eb->spinning_writers);
289 write_unlock(&eb->lock);
290 }
Chris Mason925baed2008-06-25 16:01:30 -0400291}
292
Chris Masonb9447ef2009-03-09 11:45:38 -0400293void btrfs_assert_tree_locked(struct extent_buffer *eb)
Chris Mason925baed2008-06-25 16:01:30 -0400294{
Chris Masonbd681512011-07-16 15:23:14 -0400295 BUG_ON(!atomic_read(&eb->write_locks));
296}
297
Eric Sandeen48a3b632013-04-25 20:41:01 +0000298static void btrfs_assert_tree_read_locked(struct extent_buffer *eb)
Chris Masonbd681512011-07-16 15:23:14 -0400299{
300 BUG_ON(!atomic_read(&eb->read_locks));
Chris Mason925baed2008-06-25 16:01:30 -0400301}