blob: 49e8e16308e15dd36dbd8f1481e68af8e58ee87d [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
Paul E. McKenneya71fca52009-09-18 10:28:19 -07002 * Read-Copy Update mechanism for mutual exclusion
Linus Torvalds1da177e2005-04-16 15:20:36 -07003 *
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License as published by
6 * the Free Software Foundation; either version 2 of the License, or
7 * (at your option) any later version.
8 *
9 * This program is distributed in the hope that it will be useful,
10 * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 * GNU General Public License for more details.
13 *
14 * You should have received a copy of the GNU General Public License
15 * along with this program; if not, write to the Free Software
16 * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
17 *
Paul E. McKenney01c1c662008-01-25 21:08:24 +010018 * Copyright IBM Corporation, 2001
Linus Torvalds1da177e2005-04-16 15:20:36 -070019 *
20 * Author: Dipankar Sarma <dipankar@in.ibm.com>
Paul E. McKenneya71fca52009-09-18 10:28:19 -070021 *
Josh Triplett595182b2006-10-04 02:17:21 -070022 * Based on the original work by Paul McKenney <paulmck@us.ibm.com>
Linus Torvalds1da177e2005-04-16 15:20:36 -070023 * and inputs from Rusty Russell, Andrea Arcangeli and Andi Kleen.
24 * Papers:
25 * http://www.rdrop.com/users/paulmck/paper/rclockpdcsproof.pdf
26 * http://lse.sourceforge.net/locking/rclock_OLS.2001.05.01c.sc.pdf (OLS2001)
27 *
28 * For detailed explanation of Read-Copy Update mechanism see -
Paul E. McKenneya71fca52009-09-18 10:28:19 -070029 * http://lse.sourceforge.net/locking/rcupdate.html
Linus Torvalds1da177e2005-04-16 15:20:36 -070030 *
31 */
32
33#ifndef __LINUX_RCUPDATE_H
34#define __LINUX_RCUPDATE_H
35
Linus Torvalds1da177e2005-04-16 15:20:36 -070036#include <linux/cache.h>
37#include <linux/spinlock.h>
38#include <linux/threads.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070039#include <linux/cpumask.h>
40#include <linux/seqlock.h>
Peter Zijlstra851a67b2007-10-11 22:11:12 +020041#include <linux/lockdep.h>
Paul E. McKenney4446a362008-05-12 21:21:05 +020042#include <linux/completion.h>
Mathieu Desnoyers551d55a2010-04-17 08:48:42 -040043#include <linux/debugobjects.h>
Paul E. McKenneyca5ecdd2010-04-28 14:39:09 -070044#include <linux/compiler.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070045
Dave Younge5ab6772010-03-10 15:24:05 -080046#ifdef CONFIG_RCU_TORTURE_TEST
47extern int rcutorture_runnable; /* for sysctl */
48#endif /* #ifdef CONFIG_RCU_TORTURE_TEST */
49
Paul E. McKenneya3dc3fb2010-08-13 16:16:25 -070050#define ULONG_CMP_GE(a, b) (ULONG_MAX / 2 >= (a) - (b))
51#define ULONG_CMP_LT(a, b) (ULONG_MAX / 2 < (a) - (b))
52
Linus Torvalds1da177e2005-04-16 15:20:36 -070053/**
54 * struct rcu_head - callback structure for use with RCU
55 * @next: next update requests in a list
56 * @func: actual update function to call after the grace period.
57 */
58struct rcu_head {
59 struct rcu_head *next;
60 void (*func)(struct rcu_head *head);
61};
62
Paul E. McKenney03b042b2009-06-25 09:08:16 -070063/* Exported common interfaces */
Paul E. McKenney7b0b7592010-08-17 14:18:46 -070064extern void call_rcu_sched(struct rcu_head *head,
65 void (*func)(struct rcu_head *rcu));
66extern void synchronize_sched(void);
Paul E. McKenney03b042b2009-06-25 09:08:16 -070067extern void rcu_barrier_bh(void);
68extern void rcu_barrier_sched(void);
Paul E. McKenney03b042b2009-06-25 09:08:16 -070069extern int sched_expedited_torture_stats(char *page);
70
Paul E. McKenney7b0b7592010-08-17 14:18:46 -070071static inline void __rcu_read_lock_bh(void)
72{
73 local_bh_disable();
74}
75
76static inline void __rcu_read_unlock_bh(void)
77{
78 local_bh_enable();
79}
Paul E. McKenneya6826042009-02-25 18:03:42 -080080
Paul E. McKenneya3dc3fb2010-08-13 16:16:25 -070081#ifdef CONFIG_PREEMPT_RCU
82
Paul E. McKenney7b0b7592010-08-17 14:18:46 -070083extern void __rcu_read_lock(void);
84extern void __rcu_read_unlock(void);
85void synchronize_rcu(void);
86
Paul E. McKenneya3dc3fb2010-08-13 16:16:25 -070087/*
88 * Defined as a macro as it is a very low level header included from
89 * areas that don't even know about current. This gives the rcu_read_lock()
90 * nesting depth, but makes sense only if CONFIG_PREEMPT_RCU -- in other
91 * types of kernel builds, the rcu_read_lock() nesting depth is unknowable.
92 */
93#define rcu_preempt_depth() (current->rcu_read_lock_nesting)
94
Paul E. McKenney7b0b7592010-08-17 14:18:46 -070095#else /* #ifdef CONFIG_PREEMPT_RCU */
96
97static inline void __rcu_read_lock(void)
98{
99 preempt_disable();
100}
101
102static inline void __rcu_read_unlock(void)
103{
104 preempt_enable();
105}
106
107static inline void synchronize_rcu(void)
108{
109 synchronize_sched();
110}
111
112static inline int rcu_preempt_depth(void)
113{
114 return 0;
115}
116
117#endif /* #else #ifdef CONFIG_PREEMPT_RCU */
118
119/* Internal to kernel */
Paul E. McKenney7b0b7592010-08-17 14:18:46 -0700120extern void rcu_sched_qs(int cpu);
121extern void rcu_bh_qs(int cpu);
122extern void rcu_check_callbacks(int cpu, int user);
123struct notifier_block;
124
125#ifdef CONFIG_NO_HZ
126
127extern void rcu_enter_nohz(void);
128extern void rcu_exit_nohz(void);
129
130#else /* #ifdef CONFIG_NO_HZ */
131
132static inline void rcu_enter_nohz(void)
133{
134}
135
136static inline void rcu_exit_nohz(void)
137{
138}
139
140#endif /* #else #ifdef CONFIG_NO_HZ */
Paul E. McKenneya3dc3fb2010-08-13 16:16:25 -0700141
Paul E. McKenneyf41d9112009-08-22 13:56:52 -0700142#if defined(CONFIG_TREE_RCU) || defined(CONFIG_TREE_PREEMPT_RCU)
Paul E. McKenney64db4cf2008-12-18 21:55:32 +0100143#include <linux/rcutree.h>
Paul E. McKenneya57eb942010-06-29 16:49:16 -0700144#elif defined(CONFIG_TINY_RCU) || defined(CONFIG_TINY_PREEMPT_RCU)
Paul E. McKenney9b1d82f2009-10-25 19:03:50 -0700145#include <linux/rcutiny.h>
Paul E. McKenney64db4cf2008-12-18 21:55:32 +0100146#else
147#error "Unknown RCU implementation specified to kernel configuration"
Paul E. McKenney6b3ef482009-08-22 13:56:53 -0700148#endif
Paul E. McKenney01c1c662008-01-25 21:08:24 +0100149
Mathieu Desnoyers551d55a2010-04-17 08:48:42 -0400150/*
151 * init_rcu_head_on_stack()/destroy_rcu_head_on_stack() are needed for dynamic
152 * initialization and destruction of rcu_head on the stack. rcu_head structures
153 * allocated dynamically in the heap or defined statically don't need any
154 * initialization.
155 */
156#ifdef CONFIG_DEBUG_OBJECTS_RCU_HEAD
157extern void init_rcu_head_on_stack(struct rcu_head *head);
158extern void destroy_rcu_head_on_stack(struct rcu_head *head);
159#else /* !CONFIG_DEBUG_OBJECTS_RCU_HEAD */
Mathieu Desnoyers43760302010-04-17 08:48:39 -0400160static inline void init_rcu_head_on_stack(struct rcu_head *head)
161{
162}
163
164static inline void destroy_rcu_head_on_stack(struct rcu_head *head)
165{
166}
Mathieu Desnoyers551d55a2010-04-17 08:48:42 -0400167#endif /* #else !CONFIG_DEBUG_OBJECTS_RCU_HEAD */
Mathieu Desnoyers43760302010-04-17 08:48:39 -0400168
Paul E. McKenneybc33f242009-08-22 13:56:47 -0700169#ifdef CONFIG_DEBUG_LOCK_ALLOC
Paul E. McKenney632ee202010-02-22 17:04:45 -0800170
Paul E. McKenneybc33f242009-08-22 13:56:47 -0700171extern struct lockdep_map rcu_lock_map;
Paul E. McKenney632ee202010-02-22 17:04:45 -0800172# define rcu_read_acquire() \
173 lock_acquire(&rcu_lock_map, 0, 0, 2, 1, NULL, _THIS_IP_)
Paul E. McKenneybc33f242009-08-22 13:56:47 -0700174# define rcu_read_release() lock_release(&rcu_lock_map, 1, _THIS_IP_)
Paul E. McKenney632ee202010-02-22 17:04:45 -0800175
176extern struct lockdep_map rcu_bh_lock_map;
177# define rcu_read_acquire_bh() \
178 lock_acquire(&rcu_bh_lock_map, 0, 0, 2, 1, NULL, _THIS_IP_)
179# define rcu_read_release_bh() lock_release(&rcu_bh_lock_map, 1, _THIS_IP_)
180
181extern struct lockdep_map rcu_sched_lock_map;
182# define rcu_read_acquire_sched() \
183 lock_acquire(&rcu_sched_lock_map, 0, 0, 2, 1, NULL, _THIS_IP_)
184# define rcu_read_release_sched() \
185 lock_release(&rcu_sched_lock_map, 1, _THIS_IP_)
186
Paul E. McKenneybc293d62010-04-15 12:50:39 -0700187extern int debug_lockdep_rcu_enabled(void);
Paul E. McKenney54dbf962010-03-03 07:46:57 -0800188
Paul E. McKenney632ee202010-02-22 17:04:45 -0800189/**
Paul E. McKenneyca5ecdd2010-04-28 14:39:09 -0700190 * rcu_read_lock_held() - might we be in RCU read-side critical section?
Paul E. McKenney632ee202010-02-22 17:04:45 -0800191 *
Paul E. McKenneyd20200b2010-03-30 10:52:21 -0700192 * If CONFIG_DEBUG_LOCK_ALLOC is selected, returns nonzero iff in an RCU
193 * read-side critical section. In absence of CONFIG_DEBUG_LOCK_ALLOC,
Paul E. McKenney632ee202010-02-22 17:04:45 -0800194 * this assumes we are in an RCU read-side critical section unless it can
Paul E. McKenneyca5ecdd2010-04-28 14:39:09 -0700195 * prove otherwise. This is useful for debug checks in functions that
196 * require that they be called within an RCU read-side critical section.
Paul E. McKenney54dbf962010-03-03 07:46:57 -0800197 *
Paul E. McKenneyca5ecdd2010-04-28 14:39:09 -0700198 * Checks debug_lockdep_rcu_enabled() to prevent false positives during boot
Paul E. McKenney32c141a2010-03-30 10:59:28 -0700199 * and while lockdep is disabled.
Paul E. McKenney632ee202010-02-22 17:04:45 -0800200 */
201static inline int rcu_read_lock_held(void)
202{
Paul E. McKenney54dbf962010-03-03 07:46:57 -0800203 if (!debug_lockdep_rcu_enabled())
204 return 1;
205 return lock_is_held(&rcu_lock_map);
Paul E. McKenney632ee202010-02-22 17:04:45 -0800206}
207
Paul E. McKenneye3818b82010-03-15 17:03:43 -0700208/*
209 * rcu_read_lock_bh_held() is defined out of line to avoid #include-file
210 * hell.
Paul E. McKenney632ee202010-02-22 17:04:45 -0800211 */
Paul E. McKenneye3818b82010-03-15 17:03:43 -0700212extern int rcu_read_lock_bh_held(void);
Paul E. McKenney632ee202010-02-22 17:04:45 -0800213
214/**
Paul E. McKenneyca5ecdd2010-04-28 14:39:09 -0700215 * rcu_read_lock_sched_held() - might we be in RCU-sched read-side critical section?
Paul E. McKenney632ee202010-02-22 17:04:45 -0800216 *
Paul E. McKenneyd20200b2010-03-30 10:52:21 -0700217 * If CONFIG_DEBUG_LOCK_ALLOC is selected, returns nonzero iff in an
218 * RCU-sched read-side critical section. In absence of
219 * CONFIG_DEBUG_LOCK_ALLOC, this assumes we are in an RCU-sched read-side
220 * critical section unless it can prove otherwise. Note that disabling
221 * of preemption (including disabling irqs) counts as an RCU-sched
Paul E. McKenneyca5ecdd2010-04-28 14:39:09 -0700222 * read-side critical section. This is useful for debug checks in functions
223 * that required that they be called within an RCU-sched read-side
224 * critical section.
Paul E. McKenney54dbf962010-03-03 07:46:57 -0800225 *
Paul E. McKenney32c141a2010-03-30 10:59:28 -0700226 * Check debug_lockdep_rcu_enabled() to prevent false positives during boot
227 * and while lockdep is disabled.
Paul E. McKenney632ee202010-02-22 17:04:45 -0800228 */
Paul E. McKenneye6033e32010-03-03 17:50:16 -0800229#ifdef CONFIG_PREEMPT
Paul E. McKenney632ee202010-02-22 17:04:45 -0800230static inline int rcu_read_lock_sched_held(void)
231{
232 int lockdep_opinion = 0;
233
Paul E. McKenney54dbf962010-03-03 07:46:57 -0800234 if (!debug_lockdep_rcu_enabled())
235 return 1;
Paul E. McKenney632ee202010-02-22 17:04:45 -0800236 if (debug_locks)
237 lockdep_opinion = lock_is_held(&rcu_sched_lock_map);
Lai Jiangshan0cff8102010-03-18 12:25:33 -0700238 return lockdep_opinion || preempt_count() != 0 || irqs_disabled();
Paul E. McKenney632ee202010-02-22 17:04:45 -0800239}
Paul E. McKenneye6033e32010-03-03 17:50:16 -0800240#else /* #ifdef CONFIG_PREEMPT */
241static inline int rcu_read_lock_sched_held(void)
242{
243 return 1;
244}
245#endif /* #else #ifdef CONFIG_PREEMPT */
Paul E. McKenney632ee202010-02-22 17:04:45 -0800246
247#else /* #ifdef CONFIG_DEBUG_LOCK_ALLOC */
248
249# define rcu_read_acquire() do { } while (0)
250# define rcu_read_release() do { } while (0)
251# define rcu_read_acquire_bh() do { } while (0)
252# define rcu_read_release_bh() do { } while (0)
253# define rcu_read_acquire_sched() do { } while (0)
254# define rcu_read_release_sched() do { } while (0)
255
256static inline int rcu_read_lock_held(void)
257{
258 return 1;
259}
260
261static inline int rcu_read_lock_bh_held(void)
262{
263 return 1;
264}
265
Paul E. McKenneye6033e32010-03-03 17:50:16 -0800266#ifdef CONFIG_PREEMPT
Paul E. McKenney632ee202010-02-22 17:04:45 -0800267static inline int rcu_read_lock_sched_held(void)
268{
Paul E. McKenneybbad9372010-04-02 16:17:17 -0700269 return preempt_count() != 0 || irqs_disabled();
Paul E. McKenney632ee202010-02-22 17:04:45 -0800270}
Paul E. McKenneye6033e32010-03-03 17:50:16 -0800271#else /* #ifdef CONFIG_PREEMPT */
272static inline int rcu_read_lock_sched_held(void)
273{
274 return 1;
275}
276#endif /* #else #ifdef CONFIG_PREEMPT */
Paul E. McKenney632ee202010-02-22 17:04:45 -0800277
278#endif /* #else #ifdef CONFIG_DEBUG_LOCK_ALLOC */
279
280#ifdef CONFIG_PROVE_RCU
281
Paul E. McKenneyee84b822010-05-06 09:28:41 -0700282extern int rcu_my_thread_group_empty(void);
283
Tetsuo Handa4221a992010-06-26 01:08:19 +0900284/**
285 * rcu_lockdep_assert - emit lockdep splat if specified condition not met
286 * @c: condition to check
287 */
288#define rcu_lockdep_assert(c) \
Lai Jiangshan2b3fc352010-04-20 16:23:07 +0800289 do { \
290 static bool __warned; \
291 if (debug_lockdep_rcu_enabled() && !__warned && !(c)) { \
292 __warned = true; \
293 lockdep_rcu_dereference(__FILE__, __LINE__); \
294 } \
295 } while (0)
296
Paul E. McKenneyca5ecdd2010-04-28 14:39:09 -0700297#else /* #ifdef CONFIG_PROVE_RCU */
298
Tetsuo Handa4221a992010-06-26 01:08:19 +0900299#define rcu_lockdep_assert(c) do { } while (0)
Paul E. McKenneyca5ecdd2010-04-28 14:39:09 -0700300
301#endif /* #else #ifdef CONFIG_PROVE_RCU */
302
303/*
304 * Helper functions for rcu_dereference_check(), rcu_dereference_protected()
305 * and rcu_assign_pointer(). Some of these could be folded into their
306 * callers, but they are left separate in order to ease introduction of
307 * multiple flavors of pointers to match the multiple flavors of RCU
308 * (e.g., __rcu_bh, * __rcu_sched, and __srcu), should this make sense in
309 * the future.
310 */
Paul E. McKenney53ecfba2010-09-13 17:24:21 -0700311
312#ifdef __CHECKER__
313#define rcu_dereference_sparse(p, space) \
314 ((void)(((typeof(*p) space *)p) == p))
315#else /* #ifdef __CHECKER__ */
316#define rcu_dereference_sparse(p, space)
317#endif /* #else #ifdef __CHECKER__ */
318
Paul E. McKenneyca5ecdd2010-04-28 14:39:09 -0700319#define __rcu_access_pointer(p, space) \
320 ({ \
321 typeof(*p) *_________p1 = (typeof(*p)*__force )ACCESS_ONCE(p); \
Paul E. McKenney53ecfba2010-09-13 17:24:21 -0700322 rcu_dereference_sparse(p, space); \
Paul E. McKenneyca5ecdd2010-04-28 14:39:09 -0700323 ((typeof(*p) __force __kernel *)(_________p1)); \
324 })
325#define __rcu_dereference_check(p, c, space) \
326 ({ \
327 typeof(*p) *_________p1 = (typeof(*p)*__force )ACCESS_ONCE(p); \
Tetsuo Handa4221a992010-06-26 01:08:19 +0900328 rcu_lockdep_assert(c); \
Paul E. McKenney53ecfba2010-09-13 17:24:21 -0700329 rcu_dereference_sparse(p, space); \
Paul E. McKenneyca5ecdd2010-04-28 14:39:09 -0700330 smp_read_barrier_depends(); \
331 ((typeof(*p) __force __kernel *)(_________p1)); \
332 })
333#define __rcu_dereference_protected(p, c, space) \
334 ({ \
Tetsuo Handa4221a992010-06-26 01:08:19 +0900335 rcu_lockdep_assert(c); \
Paul E. McKenney53ecfba2010-09-13 17:24:21 -0700336 rcu_dereference_sparse(p, space); \
Paul E. McKenneyca5ecdd2010-04-28 14:39:09 -0700337 ((typeof(*p) __force __kernel *)(p)); \
338 })
339
340#define __rcu_dereference_index_check(p, c) \
341 ({ \
342 typeof(p) _________p1 = ACCESS_ONCE(p); \
Tetsuo Handa4221a992010-06-26 01:08:19 +0900343 rcu_lockdep_assert(c); \
Paul E. McKenneyca5ecdd2010-04-28 14:39:09 -0700344 smp_read_barrier_depends(); \
345 (_________p1); \
346 })
347#define __rcu_assign_pointer(p, v, space) \
348 ({ \
349 if (!__builtin_constant_p(v) || \
350 ((v) != NULL)) \
351 smp_wmb(); \
352 (p) = (typeof(*v) __force space *)(v); \
353 })
354
355
Paul E. McKenney632ee202010-02-22 17:04:45 -0800356/**
Paul E. McKenneyca5ecdd2010-04-28 14:39:09 -0700357 * rcu_access_pointer() - fetch RCU pointer with no dereferencing
358 * @p: The pointer to read
359 *
360 * Return the value of the specified RCU-protected pointer, but omit the
361 * smp_read_barrier_depends() and keep the ACCESS_ONCE(). This is useful
362 * when the value of this pointer is accessed, but the pointer is not
363 * dereferenced, for example, when testing an RCU-protected pointer against
364 * NULL. Although rcu_access_pointer() may also be used in cases where
365 * update-side locks prevent the value of the pointer from changing, you
366 * should instead use rcu_dereference_protected() for this use case.
367 */
368#define rcu_access_pointer(p) __rcu_access_pointer((p), __rcu)
369
370/**
371 * rcu_dereference_check() - rcu_dereference with debug checking
David Howellsc08c68d2010-04-09 15:39:11 -0700372 * @p: The pointer to read, prior to dereferencing
373 * @c: The conditions under which the dereference will take place
Paul E. McKenney632ee202010-02-22 17:04:45 -0800374 *
David Howellsc08c68d2010-04-09 15:39:11 -0700375 * Do an rcu_dereference(), but check that the conditions under which the
Paul E. McKenneyca5ecdd2010-04-28 14:39:09 -0700376 * dereference will take place are correct. Typically the conditions
377 * indicate the various locking conditions that should be held at that
378 * point. The check should return true if the conditions are satisfied.
379 * An implicit check for being in an RCU read-side critical section
380 * (rcu_read_lock()) is included.
David Howellsc08c68d2010-04-09 15:39:11 -0700381 *
382 * For example:
383 *
Paul E. McKenneyca5ecdd2010-04-28 14:39:09 -0700384 * bar = rcu_dereference_check(foo->bar, lockdep_is_held(&foo->lock));
David Howellsc08c68d2010-04-09 15:39:11 -0700385 *
386 * could be used to indicate to lockdep that foo->bar may only be dereferenced
Paul E. McKenneyca5ecdd2010-04-28 14:39:09 -0700387 * if either rcu_read_lock() is held, or that the lock required to replace
David Howellsc08c68d2010-04-09 15:39:11 -0700388 * the bar struct at foo->bar is held.
389 *
390 * Note that the list of conditions may also include indications of when a lock
391 * need not be held, for example during initialisation or destruction of the
392 * target struct:
393 *
Paul E. McKenneyca5ecdd2010-04-28 14:39:09 -0700394 * bar = rcu_dereference_check(foo->bar, lockdep_is_held(&foo->lock) ||
David Howellsc08c68d2010-04-09 15:39:11 -0700395 * atomic_read(&foo->usage) == 0);
Paul E. McKenneyca5ecdd2010-04-28 14:39:09 -0700396 *
397 * Inserts memory barriers on architectures that require them
398 * (currently only the Alpha), prevents the compiler from refetching
399 * (and from merging fetches), and, more importantly, documents exactly
400 * which pointers are protected by RCU and checks that the pointer is
401 * annotated as __rcu.
Paul E. McKenney632ee202010-02-22 17:04:45 -0800402 */
403#define rcu_dereference_check(p, c) \
Paul E. McKenneyca5ecdd2010-04-28 14:39:09 -0700404 __rcu_dereference_check((p), rcu_read_lock_held() || (c), __rcu)
Paul E. McKenney632ee202010-02-22 17:04:45 -0800405
Paul E. McKenneyb62730b2010-04-09 15:39:10 -0700406/**
Paul E. McKenneyca5ecdd2010-04-28 14:39:09 -0700407 * rcu_dereference_bh_check() - rcu_dereference_bh with debug checking
408 * @p: The pointer to read, prior to dereferencing
409 * @c: The conditions under which the dereference will take place
410 *
411 * This is the RCU-bh counterpart to rcu_dereference_check().
412 */
413#define rcu_dereference_bh_check(p, c) \
414 __rcu_dereference_check((p), rcu_read_lock_bh_held() || (c), __rcu)
415
416/**
417 * rcu_dereference_sched_check() - rcu_dereference_sched with debug checking
418 * @p: The pointer to read, prior to dereferencing
419 * @c: The conditions under which the dereference will take place
420 *
421 * This is the RCU-sched counterpart to rcu_dereference_check().
422 */
423#define rcu_dereference_sched_check(p, c) \
424 __rcu_dereference_check((p), rcu_read_lock_sched_held() || (c), \
425 __rcu)
426
427#define rcu_dereference_raw(p) rcu_dereference_check(p, 1) /*@@@ needed? @@@*/
428
429/**
430 * rcu_dereference_index_check() - rcu_dereference for indices with debug checking
431 * @p: The pointer to read, prior to dereferencing
432 * @c: The conditions under which the dereference will take place
433 *
434 * Similar to rcu_dereference_check(), but omits the sparse checking.
435 * This allows rcu_dereference_index_check() to be used on integers,
436 * which can then be used as array indices. Attempting to use
437 * rcu_dereference_check() on an integer will give compiler warnings
438 * because the sparse address-space mechanism relies on dereferencing
439 * the RCU-protected pointer. Dereferencing integers is not something
440 * that even gcc will put up with.
441 *
442 * Note that this function does not implicitly check for RCU read-side
443 * critical sections. If this function gains lots of uses, it might
444 * make sense to provide versions for each flavor of RCU, but it does
445 * not make sense as of early 2010.
446 */
447#define rcu_dereference_index_check(p, c) \
448 __rcu_dereference_index_check((p), (c))
449
450/**
451 * rcu_dereference_protected() - fetch RCU pointer when updates prevented
452 * @p: The pointer to read, prior to dereferencing
453 * @c: The conditions under which the dereference will take place
Paul E. McKenneyb62730b2010-04-09 15:39:10 -0700454 *
455 * Return the value of the specified RCU-protected pointer, but omit
456 * both the smp_read_barrier_depends() and the ACCESS_ONCE(). This
457 * is useful in cases where update-side locks prevent the value of the
458 * pointer from changing. Please note that this primitive does -not-
459 * prevent the compiler from repeating this reference or combining it
460 * with other references, so it should not be used without protection
461 * of appropriate locks.
Paul E. McKenneyca5ecdd2010-04-28 14:39:09 -0700462 *
463 * This function is only for update-side use. Using this function
464 * when protected only by rcu_read_lock() will result in infrequent
465 * but very ugly failures.
Paul E. McKenneyb62730b2010-04-09 15:39:10 -0700466 */
467#define rcu_dereference_protected(p, c) \
Paul E. McKenneyca5ecdd2010-04-28 14:39:09 -0700468 __rcu_dereference_protected((p), (c), __rcu)
Paul E. McKenneybc33f242009-08-22 13:56:47 -0700469
Linus Torvalds1da177e2005-04-16 15:20:36 -0700470/**
Paul E. McKenneyca5ecdd2010-04-28 14:39:09 -0700471 * rcu_dereference_bh_protected() - fetch RCU-bh pointer when updates prevented
472 * @p: The pointer to read, prior to dereferencing
473 * @c: The conditions under which the dereference will take place
Paul E. McKenneyb62730b2010-04-09 15:39:10 -0700474 *
Paul E. McKenneyca5ecdd2010-04-28 14:39:09 -0700475 * This is the RCU-bh counterpart to rcu_dereference_protected().
Paul E. McKenneyb62730b2010-04-09 15:39:10 -0700476 */
Paul E. McKenneyca5ecdd2010-04-28 14:39:09 -0700477#define rcu_dereference_bh_protected(p, c) \
478 __rcu_dereference_protected((p), (c), __rcu)
Paul E. McKenneyb62730b2010-04-09 15:39:10 -0700479
480/**
Paul E. McKenneyca5ecdd2010-04-28 14:39:09 -0700481 * rcu_dereference_sched_protected() - fetch RCU-sched pointer when updates prevented
482 * @p: The pointer to read, prior to dereferencing
483 * @c: The conditions under which the dereference will take place
484 *
485 * This is the RCU-sched counterpart to rcu_dereference_protected().
486 */
487#define rcu_dereference_sched_protected(p, c) \
488 __rcu_dereference_protected((p), (c), __rcu)
489
490
491/**
492 * rcu_dereference() - fetch RCU-protected pointer for dereferencing
493 * @p: The pointer to read, prior to dereferencing
494 *
495 * This is a simple wrapper around rcu_dereference_check().
496 */
497#define rcu_dereference(p) rcu_dereference_check(p, 0)
498
499/**
500 * rcu_dereference_bh() - fetch an RCU-bh-protected pointer for dereferencing
501 * @p: The pointer to read, prior to dereferencing
502 *
503 * Makes rcu_dereference_check() do the dirty work.
504 */
505#define rcu_dereference_bh(p) rcu_dereference_bh_check(p, 0)
506
507/**
508 * rcu_dereference_sched() - fetch RCU-sched-protected pointer for dereferencing
509 * @p: The pointer to read, prior to dereferencing
510 *
511 * Makes rcu_dereference_check() do the dirty work.
512 */
513#define rcu_dereference_sched(p) rcu_dereference_sched_check(p, 0)
514
515/**
516 * rcu_read_lock() - mark the beginning of an RCU read-side critical section
Linus Torvalds1da177e2005-04-16 15:20:36 -0700517 *
Paul E. McKenney9b06e812005-05-01 08:59:04 -0700518 * When synchronize_rcu() is invoked on one CPU while other CPUs
Linus Torvalds1da177e2005-04-16 15:20:36 -0700519 * are within RCU read-side critical sections, then the
Paul E. McKenney9b06e812005-05-01 08:59:04 -0700520 * synchronize_rcu() is guaranteed to block until after all the other
Linus Torvalds1da177e2005-04-16 15:20:36 -0700521 * CPUs exit their critical sections. Similarly, if call_rcu() is invoked
522 * on one CPU while other CPUs are within RCU read-side critical
523 * sections, invocation of the corresponding RCU callback is deferred
524 * until after the all the other CPUs exit their critical sections.
525 *
526 * Note, however, that RCU callbacks are permitted to run concurrently
Paul E. McKenney77d84852010-07-08 17:38:59 -0700527 * with new RCU read-side critical sections. One way that this can happen
Linus Torvalds1da177e2005-04-16 15:20:36 -0700528 * is via the following sequence of events: (1) CPU 0 enters an RCU
529 * read-side critical section, (2) CPU 1 invokes call_rcu() to register
530 * an RCU callback, (3) CPU 0 exits the RCU read-side critical section,
531 * (4) CPU 2 enters a RCU read-side critical section, (5) the RCU
532 * callback is invoked. This is legal, because the RCU read-side critical
533 * section that was running concurrently with the call_rcu() (and which
534 * therefore might be referencing something that the corresponding RCU
535 * callback would free up) has completed before the corresponding
536 * RCU callback is invoked.
537 *
538 * RCU read-side critical sections may be nested. Any deferred actions
539 * will be deferred until the outermost RCU read-side critical section
540 * completes.
541 *
Paul E. McKenney9079fd72010-08-07 21:59:54 -0700542 * You can avoid reading and understanding the next paragraph by
543 * following this rule: don't put anything in an rcu_read_lock() RCU
544 * read-side critical section that would block in a !PREEMPT kernel.
545 * But if you want the full story, read on!
546 *
547 * In non-preemptible RCU implementations (TREE_RCU and TINY_RCU), it
548 * is illegal to block while in an RCU read-side critical section. In
549 * preemptible RCU implementations (TREE_PREEMPT_RCU and TINY_PREEMPT_RCU)
550 * in CONFIG_PREEMPT kernel builds, RCU read-side critical sections may
551 * be preempted, but explicit blocking is illegal. Finally, in preemptible
552 * RCU implementations in real-time (CONFIG_PREEMPT_RT) kernel builds,
553 * RCU read-side critical sections may be preempted and they may also
554 * block, but only when acquiring spinlocks that are subject to priority
555 * inheritance.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700556 */
Paul E. McKenneybc33f242009-08-22 13:56:47 -0700557static inline void rcu_read_lock(void)
558{
559 __rcu_read_lock();
560 __acquire(RCU);
561 rcu_read_acquire();
562}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700563
Linus Torvalds1da177e2005-04-16 15:20:36 -0700564/*
565 * So where is rcu_write_lock()? It does not exist, as there is no
566 * way for writers to lock out RCU readers. This is a feature, not
567 * a bug -- this property is what provides RCU's performance benefits.
568 * Of course, writers must coordinate with each other. The normal
569 * spinlock primitives work well for this, but any other technique may be
570 * used as well. RCU does not care how the writers keep out of each
571 * others' way, as long as they do so.
572 */
Paul E. McKenney3d76c082009-09-28 07:46:32 -0700573
574/**
Paul E. McKenneyca5ecdd2010-04-28 14:39:09 -0700575 * rcu_read_unlock() - marks the end of an RCU read-side critical section.
Paul E. McKenney3d76c082009-09-28 07:46:32 -0700576 *
577 * See rcu_read_lock() for more information.
578 */
Paul E. McKenneybc33f242009-08-22 13:56:47 -0700579static inline void rcu_read_unlock(void)
580{
581 rcu_read_release();
582 __release(RCU);
583 __rcu_read_unlock();
584}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700585
586/**
Paul E. McKenneyca5ecdd2010-04-28 14:39:09 -0700587 * rcu_read_lock_bh() - mark the beginning of an RCU-bh critical section
Linus Torvalds1da177e2005-04-16 15:20:36 -0700588 *
589 * This is equivalent of rcu_read_lock(), but to be used when updates
Paul E. McKenneyca5ecdd2010-04-28 14:39:09 -0700590 * are being done using call_rcu_bh() or synchronize_rcu_bh(). Since
591 * both call_rcu_bh() and synchronize_rcu_bh() consider completion of a
592 * softirq handler to be a quiescent state, a process in RCU read-side
593 * critical section must be protected by disabling softirqs. Read-side
594 * critical sections in interrupt context can use just rcu_read_lock(),
595 * though this should at least be commented to avoid confusing people
596 * reading the code.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700597 */
Paul E. McKenneybc33f242009-08-22 13:56:47 -0700598static inline void rcu_read_lock_bh(void)
599{
600 __rcu_read_lock_bh();
601 __acquire(RCU_BH);
Paul E. McKenney632ee202010-02-22 17:04:45 -0800602 rcu_read_acquire_bh();
Paul E. McKenneybc33f242009-08-22 13:56:47 -0700603}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700604
605/*
606 * rcu_read_unlock_bh - marks the end of a softirq-only RCU critical section
607 *
608 * See rcu_read_lock_bh() for more information.
609 */
Paul E. McKenneybc33f242009-08-22 13:56:47 -0700610static inline void rcu_read_unlock_bh(void)
611{
Paul E. McKenney632ee202010-02-22 17:04:45 -0800612 rcu_read_release_bh();
Paul E. McKenneybc33f242009-08-22 13:56:47 -0700613 __release(RCU_BH);
614 __rcu_read_unlock_bh();
615}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700616
617/**
Paul E. McKenneyca5ecdd2010-04-28 14:39:09 -0700618 * rcu_read_lock_sched() - mark the beginning of a RCU-sched critical section
Mathieu Desnoyers1c50b722008-09-29 11:06:46 -0400619 *
Paul E. McKenneyca5ecdd2010-04-28 14:39:09 -0700620 * This is equivalent of rcu_read_lock(), but to be used when updates
621 * are being done using call_rcu_sched() or synchronize_rcu_sched().
622 * Read-side critical sections can also be introduced by anything that
623 * disables preemption, including local_irq_disable() and friends.
Mathieu Desnoyers1c50b722008-09-29 11:06:46 -0400624 */
Paul E. McKenneyd6714c22009-08-22 13:56:46 -0700625static inline void rcu_read_lock_sched(void)
626{
627 preempt_disable();
Paul E. McKenneybc33f242009-08-22 13:56:47 -0700628 __acquire(RCU_SCHED);
Paul E. McKenney632ee202010-02-22 17:04:45 -0800629 rcu_read_acquire_sched();
Paul E. McKenneyd6714c22009-08-22 13:56:46 -0700630}
Paul E. McKenney1eba8f82009-09-23 09:50:42 -0700631
632/* Used by lockdep and tracing: cannot be traced, cannot call lockdep. */
Paul E. McKenney7c614d62009-08-24 09:42:00 -0700633static inline notrace void rcu_read_lock_sched_notrace(void)
Paul E. McKenneyd6714c22009-08-22 13:56:46 -0700634{
635 preempt_disable_notrace();
Paul E. McKenneybc33f242009-08-22 13:56:47 -0700636 __acquire(RCU_SCHED);
Paul E. McKenneyd6714c22009-08-22 13:56:46 -0700637}
Mathieu Desnoyers1c50b722008-09-29 11:06:46 -0400638
639/*
640 * rcu_read_unlock_sched - marks the end of a RCU-classic critical section
641 *
642 * See rcu_read_lock_sched for more information.
643 */
Paul E. McKenneyd6714c22009-08-22 13:56:46 -0700644static inline void rcu_read_unlock_sched(void)
645{
Paul E. McKenney632ee202010-02-22 17:04:45 -0800646 rcu_read_release_sched();
Paul E. McKenneybc33f242009-08-22 13:56:47 -0700647 __release(RCU_SCHED);
Paul E. McKenneyd6714c22009-08-22 13:56:46 -0700648 preempt_enable();
649}
Paul E. McKenney1eba8f82009-09-23 09:50:42 -0700650
651/* Used by lockdep and tracing: cannot be traced, cannot call lockdep. */
Paul E. McKenney7c614d62009-08-24 09:42:00 -0700652static inline notrace void rcu_read_unlock_sched_notrace(void)
Paul E. McKenneyd6714c22009-08-22 13:56:46 -0700653{
Paul E. McKenneybc33f242009-08-22 13:56:47 -0700654 __release(RCU_SCHED);
Paul E. McKenneyd6714c22009-08-22 13:56:46 -0700655 preempt_enable_notrace();
656}
Mathieu Desnoyers1c50b722008-09-29 11:06:46 -0400657
Mathieu Desnoyers1c50b722008-09-29 11:06:46 -0400658/**
Paul E. McKenneyca5ecdd2010-04-28 14:39:09 -0700659 * rcu_assign_pointer() - assign to RCU-protected pointer
660 * @p: pointer to assign to
661 * @v: value to assign (publish)
Paul E. McKenneyc26d34a2010-02-22 17:04:46 -0800662 *
Paul E. McKenneyca5ecdd2010-04-28 14:39:09 -0700663 * Assigns the specified value to the specified RCU-protected
664 * pointer, ensuring that any concurrent RCU readers will see
665 * any prior initialization. Returns the value assigned.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700666 *
667 * Inserts memory barriers on architectures that require them
668 * (pretty much all of them other than x86), and also prevents
669 * the compiler from reordering the code that initializes the
670 * structure after the pointer assignment. More importantly, this
671 * call documents which pointers will be dereferenced by RCU read-side
672 * code.
673 */
Paul E. McKenneyd99c4f62008-02-06 01:37:25 -0800674#define rcu_assign_pointer(p, v) \
Paul E. McKenneyca5ecdd2010-04-28 14:39:09 -0700675 __rcu_assign_pointer((p), (v), __rcu)
676
677/**
678 * RCU_INIT_POINTER() - initialize an RCU protected pointer
679 *
680 * Initialize an RCU-protected pointer in such a way to avoid RCU-lockdep
681 * splats.
682 */
683#define RCU_INIT_POINTER(p, v) \
684 p = (typeof(*v) __force __rcu *)(v)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700685
Paul E. McKenney4446a362008-05-12 21:21:05 +0200686/* Infrastructure to implement the synchronize_() primitives. */
687
688struct rcu_synchronize {
689 struct rcu_head head;
690 struct completion completion;
691};
692
693extern void wakeme_after_rcu(struct rcu_head *head);
694
Paul E. McKenney7b0b7592010-08-17 14:18:46 -0700695#ifdef CONFIG_PREEMPT_RCU
696
Paul E. McKenney9b06e812005-05-01 08:59:04 -0700697/**
Paul E. McKenneyca5ecdd2010-04-28 14:39:09 -0700698 * call_rcu() - Queue an RCU callback for invocation after a grace period.
Paul E. McKenney01c1c662008-01-25 21:08:24 +0100699 * @head: structure to be used for queueing the RCU updates.
Paul E. McKenney77d84852010-07-08 17:38:59 -0700700 * @func: actual callback function to be invoked after the grace period
Paul E. McKenney01c1c662008-01-25 21:08:24 +0100701 *
Paul E. McKenney77d84852010-07-08 17:38:59 -0700702 * The callback function will be invoked some time after a full grace
703 * period elapses, in other words after all pre-existing RCU read-side
704 * critical sections have completed. However, the callback function
705 * might well execute concurrently with RCU read-side critical sections
706 * that started after call_rcu() was invoked. RCU read-side critical
Paul E. McKenney01c1c662008-01-25 21:08:24 +0100707 * sections are delimited by rcu_read_lock() and rcu_read_unlock(),
708 * and may be nested.
709 */
710extern void call_rcu(struct rcu_head *head,
711 void (*func)(struct rcu_head *head));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700712
Paul E. McKenney7b0b7592010-08-17 14:18:46 -0700713#else /* #ifdef CONFIG_PREEMPT_RCU */
714
715/* In classic RCU, call_rcu() is just call_rcu_sched(). */
716#define call_rcu call_rcu_sched
717
718#endif /* #else #ifdef CONFIG_PREEMPT_RCU */
719
Paul E. McKenney01c1c662008-01-25 21:08:24 +0100720/**
Paul E. McKenneyca5ecdd2010-04-28 14:39:09 -0700721 * call_rcu_bh() - Queue an RCU for invocation after a quicker grace period.
Paul E. McKenney01c1c662008-01-25 21:08:24 +0100722 * @head: structure to be used for queueing the RCU updates.
Paul E. McKenney77d84852010-07-08 17:38:59 -0700723 * @func: actual callback function to be invoked after the grace period
Paul E. McKenney01c1c662008-01-25 21:08:24 +0100724 *
Paul E. McKenney77d84852010-07-08 17:38:59 -0700725 * The callback function will be invoked some time after a full grace
Paul E. McKenney01c1c662008-01-25 21:08:24 +0100726 * period elapses, in other words after all currently executing RCU
727 * read-side critical sections have completed. call_rcu_bh() assumes
728 * that the read-side critical sections end on completion of a softirq
729 * handler. This means that read-side critical sections in process
730 * context must not be interrupted by softirqs. This interface is to be
731 * used when most of the read-side critical sections are in softirq context.
732 * RCU read-side critical sections are delimited by :
733 * - rcu_read_lock() and rcu_read_unlock(), if in interrupt context.
734 * OR
735 * - rcu_read_lock_bh() and rcu_read_unlock_bh(), if in process context.
736 * These may be nested.
737 */
738extern void call_rcu_bh(struct rcu_head *head,
739 void (*func)(struct rcu_head *head));
740
Mathieu Desnoyers551d55a2010-04-17 08:48:42 -0400741/*
742 * debug_rcu_head_queue()/debug_rcu_head_unqueue() are used internally
743 * by call_rcu() and rcu callback execution, and are therefore not part of the
744 * RCU API. Leaving in rcupdate.h because they are used by all RCU flavors.
745 */
746
747#ifdef CONFIG_DEBUG_OBJECTS_RCU_HEAD
748# define STATE_RCU_HEAD_READY 0
749# define STATE_RCU_HEAD_QUEUED 1
750
751extern struct debug_obj_descr rcuhead_debug_descr;
752
753static inline void debug_rcu_head_queue(struct rcu_head *head)
754{
755 debug_object_activate(head, &rcuhead_debug_descr);
756 debug_object_active_state(head, &rcuhead_debug_descr,
757 STATE_RCU_HEAD_READY,
758 STATE_RCU_HEAD_QUEUED);
759}
760
761static inline void debug_rcu_head_unqueue(struct rcu_head *head)
762{
763 debug_object_active_state(head, &rcuhead_debug_descr,
764 STATE_RCU_HEAD_QUEUED,
765 STATE_RCU_HEAD_READY);
766 debug_object_deactivate(head, &rcuhead_debug_descr);
767}
768#else /* !CONFIG_DEBUG_OBJECTS_RCU_HEAD */
769static inline void debug_rcu_head_queue(struct rcu_head *head)
770{
771}
772
773static inline void debug_rcu_head_unqueue(struct rcu_head *head)
774{
775}
776#endif /* #else !CONFIG_DEBUG_OBJECTS_RCU_HEAD */
777
Linus Torvalds1da177e2005-04-16 15:20:36 -0700778#endif /* __LINUX_RCUPDATE_H */