blob: 26f6417f0264f75c2d91f7b71097b7ef168e7626 [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
Paul E. McKenneya71fca52009-09-18 10:28:19 -07002 * Read-Copy Update mechanism for mutual exclusion
Linus Torvalds1da177e2005-04-16 15:20:36 -07003 *
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License as published by
6 * the Free Software Foundation; either version 2 of the License, or
7 * (at your option) any later version.
8 *
9 * This program is distributed in the hope that it will be useful,
10 * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 * GNU General Public License for more details.
13 *
14 * You should have received a copy of the GNU General Public License
15 * along with this program; if not, write to the Free Software
16 * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
17 *
Paul E. McKenney01c1c662008-01-25 21:08:24 +010018 * Copyright IBM Corporation, 2001
Linus Torvalds1da177e2005-04-16 15:20:36 -070019 *
20 * Author: Dipankar Sarma <dipankar@in.ibm.com>
Paul E. McKenneya71fca52009-09-18 10:28:19 -070021 *
Josh Triplett595182b2006-10-04 02:17:21 -070022 * Based on the original work by Paul McKenney <paulmck@us.ibm.com>
Linus Torvalds1da177e2005-04-16 15:20:36 -070023 * and inputs from Rusty Russell, Andrea Arcangeli and Andi Kleen.
24 * Papers:
25 * http://www.rdrop.com/users/paulmck/paper/rclockpdcsproof.pdf
26 * http://lse.sourceforge.net/locking/rclock_OLS.2001.05.01c.sc.pdf (OLS2001)
27 *
28 * For detailed explanation of Read-Copy Update mechanism see -
Paul E. McKenneya71fca52009-09-18 10:28:19 -070029 * http://lse.sourceforge.net/locking/rcupdate.html
Linus Torvalds1da177e2005-04-16 15:20:36 -070030 *
31 */
32
33#ifndef __LINUX_RCUPDATE_H
34#define __LINUX_RCUPDATE_H
35
Paul E. McKenney99098752011-05-31 21:03:55 -070036#include <linux/types.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070037#include <linux/cache.h>
38#include <linux/spinlock.h>
39#include <linux/threads.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070040#include <linux/cpumask.h>
41#include <linux/seqlock.h>
Peter Zijlstra851a67b2007-10-11 22:11:12 +020042#include <linux/lockdep.h>
Paul E. McKenney4446a362008-05-12 21:21:05 +020043#include <linux/completion.h>
Mathieu Desnoyers551d55a2010-04-17 08:48:42 -040044#include <linux/debugobjects.h>
Paul Gortmaker187f1882011-11-23 20:12:59 -050045#include <linux/bug.h>
Paul E. McKenneyca5ecdd2010-04-28 14:39:09 -070046#include <linux/compiler.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070047
Dave Younge5ab6772010-03-10 15:24:05 -080048#ifdef CONFIG_RCU_TORTURE_TEST
49extern int rcutorture_runnable; /* for sysctl */
50#endif /* #ifdef CONFIG_RCU_TORTURE_TEST */
51
Paul E. McKenney4a298652011-04-03 21:33:51 -070052#if defined(CONFIG_TREE_RCU) || defined(CONFIG_TREE_PREEMPT_RCU)
53extern void rcutorture_record_test_transition(void);
54extern void rcutorture_record_progress(unsigned long vernum);
Paul E. McKenney91afaf32011-10-02 07:44:32 -070055extern void do_trace_rcu_torture_read(char *rcutorturename,
56 struct rcu_head *rhp);
Paul E. McKenney4a298652011-04-03 21:33:51 -070057#else
58static inline void rcutorture_record_test_transition(void)
59{
60}
61static inline void rcutorture_record_progress(unsigned long vernum)
62{
63}
Paul E. McKenney91afaf32011-10-02 07:44:32 -070064#ifdef CONFIG_RCU_TRACE
65extern void do_trace_rcu_torture_read(char *rcutorturename,
66 struct rcu_head *rhp);
67#else
68#define do_trace_rcu_torture_read(rcutorturename, rhp) do { } while (0)
69#endif
Paul E. McKenney4a298652011-04-03 21:33:51 -070070#endif
71
Tejun Heoe27fc962010-11-22 21:36:11 -080072#define UINT_CMP_GE(a, b) (UINT_MAX / 2 >= (a) - (b))
73#define UINT_CMP_LT(a, b) (UINT_MAX / 2 < (a) - (b))
Paul E. McKenneya3dc3fb2010-08-13 16:16:25 -070074#define ULONG_CMP_GE(a, b) (ULONG_MAX / 2 >= (a) - (b))
75#define ULONG_CMP_LT(a, b) (ULONG_MAX / 2 < (a) - (b))
76
Paul E. McKenney03b042b2009-06-25 09:08:16 -070077/* Exported common interfaces */
Paul E. McKenney2c428182011-05-26 22:14:36 -070078
79#ifdef CONFIG_PREEMPT_RCU
80
81/**
82 * call_rcu() - Queue an RCU callback for invocation after a grace period.
83 * @head: structure to be used for queueing the RCU updates.
84 * @func: actual callback function to be invoked after the grace period
85 *
86 * The callback function will be invoked some time after a full grace
87 * period elapses, in other words after all pre-existing RCU read-side
88 * critical sections have completed. However, the callback function
89 * might well execute concurrently with RCU read-side critical sections
90 * that started after call_rcu() was invoked. RCU read-side critical
91 * sections are delimited by rcu_read_lock() and rcu_read_unlock(),
92 * and may be nested.
93 */
94extern void call_rcu(struct rcu_head *head,
95 void (*func)(struct rcu_head *head));
96
97#else /* #ifdef CONFIG_PREEMPT_RCU */
98
99/* In classic RCU, call_rcu() is just call_rcu_sched(). */
100#define call_rcu call_rcu_sched
101
102#endif /* #else #ifdef CONFIG_PREEMPT_RCU */
103
104/**
105 * call_rcu_bh() - Queue an RCU for invocation after a quicker grace period.
106 * @head: structure to be used for queueing the RCU updates.
107 * @func: actual callback function to be invoked after the grace period
108 *
109 * The callback function will be invoked some time after a full grace
110 * period elapses, in other words after all currently executing RCU
111 * read-side critical sections have completed. call_rcu_bh() assumes
112 * that the read-side critical sections end on completion of a softirq
113 * handler. This means that read-side critical sections in process
114 * context must not be interrupted by softirqs. This interface is to be
115 * used when most of the read-side critical sections are in softirq context.
116 * RCU read-side critical sections are delimited by :
117 * - rcu_read_lock() and rcu_read_unlock(), if in interrupt context.
118 * OR
119 * - rcu_read_lock_bh() and rcu_read_unlock_bh(), if in process context.
120 * These may be nested.
121 */
122extern void call_rcu_bh(struct rcu_head *head,
123 void (*func)(struct rcu_head *head));
124
125/**
126 * call_rcu_sched() - Queue an RCU for invocation after sched grace period.
127 * @head: structure to be used for queueing the RCU updates.
128 * @func: actual callback function to be invoked after the grace period
129 *
130 * The callback function will be invoked some time after a full grace
131 * period elapses, in other words after all currently executing RCU
132 * read-side critical sections have completed. call_rcu_sched() assumes
133 * that the read-side critical sections end on enabling of preemption
134 * or on voluntary preemption.
135 * RCU read-side critical sections are delimited by :
136 * - rcu_read_lock_sched() and rcu_read_unlock_sched(),
137 * OR
138 * anything that disables preemption.
139 * These may be nested.
140 */
Paul E. McKenney7b0b7592010-08-17 14:18:46 -0700141extern void call_rcu_sched(struct rcu_head *head,
142 void (*func)(struct rcu_head *rcu));
Paul E. McKenney2c428182011-05-26 22:14:36 -0700143
Paul E. McKenney7b0b7592010-08-17 14:18:46 -0700144extern void synchronize_sched(void);
Paul E. McKenney03b042b2009-06-25 09:08:16 -0700145
Paul E. McKenneya3dc3fb2010-08-13 16:16:25 -0700146#ifdef CONFIG_PREEMPT_RCU
147
Paul E. McKenney7b0b7592010-08-17 14:18:46 -0700148extern void __rcu_read_lock(void);
149extern void __rcu_read_unlock(void);
Paul E. McKenney2a3fa842012-05-21 11:58:36 -0700150extern void rcu_read_unlock_special(struct task_struct *t);
Paul E. McKenney7b0b7592010-08-17 14:18:46 -0700151void synchronize_rcu(void);
152
Paul E. McKenneya3dc3fb2010-08-13 16:16:25 -0700153/*
154 * Defined as a macro as it is a very low level header included from
155 * areas that don't even know about current. This gives the rcu_read_lock()
156 * nesting depth, but makes sense only if CONFIG_PREEMPT_RCU -- in other
157 * types of kernel builds, the rcu_read_lock() nesting depth is unknowable.
158 */
159#define rcu_preempt_depth() (current->rcu_read_lock_nesting)
160
Paul E. McKenney7b0b7592010-08-17 14:18:46 -0700161#else /* #ifdef CONFIG_PREEMPT_RCU */
162
163static inline void __rcu_read_lock(void)
164{
165 preempt_disable();
166}
167
168static inline void __rcu_read_unlock(void)
169{
170 preempt_enable();
171}
172
173static inline void synchronize_rcu(void)
174{
175 synchronize_sched();
176}
177
178static inline int rcu_preempt_depth(void)
179{
180 return 0;
181}
182
183#endif /* #else #ifdef CONFIG_PREEMPT_RCU */
184
185/* Internal to kernel */
Paul E. McKenney7b0b7592010-08-17 14:18:46 -0700186extern void rcu_sched_qs(int cpu);
187extern void rcu_bh_qs(int cpu);
188extern void rcu_check_callbacks(int cpu, int user);
189struct notifier_block;
Paul E. McKenney9b2e4f12011-09-30 12:10:22 -0700190extern void rcu_idle_enter(void);
191extern void rcu_idle_exit(void);
192extern void rcu_irq_enter(void);
193extern void rcu_irq_exit(void);
Paul E. McKenney9dd8fb12012-04-13 12:54:22 -0700194extern void exit_rcu(void);
Paul E. McKenneya3dc3fb2010-08-13 16:16:25 -0700195
Paul E. McKenney8a2ecf42012-02-02 15:42:04 -0800196/**
197 * RCU_NONIDLE - Indicate idle-loop code that needs RCU readers
198 * @a: Code that RCU needs to pay attention to.
199 *
200 * RCU, RCU-bh, and RCU-sched read-side critical sections are forbidden
201 * in the inner idle loop, that is, between the rcu_idle_enter() and
202 * the rcu_idle_exit() -- RCU will happily ignore any such read-side
203 * critical sections. However, things like powertop need tracepoints
204 * in the inner idle loop.
205 *
206 * This macro provides the way out: RCU_NONIDLE(do_something_with_RCU())
207 * will tell RCU that it needs to pay attending, invoke its argument
208 * (in this example, a call to the do_something_with_RCU() function),
209 * and then tell RCU to go back to ignoring this CPU. It is permissible
210 * to nest RCU_NONIDLE() wrappers, but the nesting level is currently
211 * quite limited. If deeper nesting is required, it will be necessary
212 * to adjust DYNTICK_TASK_NESTING_VALUE accordingly.
213 *
214 * This macro may be used from process-level code only.
215 */
216#define RCU_NONIDLE(a) \
217 do { \
218 rcu_idle_exit(); \
219 do { a; } while (0); \
220 rcu_idle_enter(); \
221 } while (0)
222
Paul E. McKenney2c428182011-05-26 22:14:36 -0700223/*
224 * Infrastructure to implement the synchronize_() primitives in
225 * TREE_RCU and rcu_barrier_() primitives in TINY_RCU.
226 */
227
228typedef void call_rcu_func_t(struct rcu_head *head,
229 void (*func)(struct rcu_head *head));
230void wait_rcu_gp(call_rcu_func_t crf);
231
Paul E. McKenneyf41d9112009-08-22 13:56:52 -0700232#if defined(CONFIG_TREE_RCU) || defined(CONFIG_TREE_PREEMPT_RCU)
Paul E. McKenney64db4cf2008-12-18 21:55:32 +0100233#include <linux/rcutree.h>
Paul E. McKenneya57eb942010-06-29 16:49:16 -0700234#elif defined(CONFIG_TINY_RCU) || defined(CONFIG_TINY_PREEMPT_RCU)
Paul E. McKenney9b1d82f2009-10-25 19:03:50 -0700235#include <linux/rcutiny.h>
Paul E. McKenney64db4cf2008-12-18 21:55:32 +0100236#else
237#error "Unknown RCU implementation specified to kernel configuration"
Paul E. McKenney6b3ef482009-08-22 13:56:53 -0700238#endif
Paul E. McKenney01c1c662008-01-25 21:08:24 +0100239
Mathieu Desnoyers551d55a2010-04-17 08:48:42 -0400240/*
241 * init_rcu_head_on_stack()/destroy_rcu_head_on_stack() are needed for dynamic
242 * initialization and destruction of rcu_head on the stack. rcu_head structures
243 * allocated dynamically in the heap or defined statically don't need any
244 * initialization.
245 */
246#ifdef CONFIG_DEBUG_OBJECTS_RCU_HEAD
247extern void init_rcu_head_on_stack(struct rcu_head *head);
248extern void destroy_rcu_head_on_stack(struct rcu_head *head);
249#else /* !CONFIG_DEBUG_OBJECTS_RCU_HEAD */
Mathieu Desnoyers43760302010-04-17 08:48:39 -0400250static inline void init_rcu_head_on_stack(struct rcu_head *head)
251{
252}
253
254static inline void destroy_rcu_head_on_stack(struct rcu_head *head)
255{
256}
Mathieu Desnoyers551d55a2010-04-17 08:48:42 -0400257#endif /* #else !CONFIG_DEBUG_OBJECTS_RCU_HEAD */
Mathieu Desnoyers43760302010-04-17 08:48:39 -0400258
Paul E. McKenney62fde6e2012-05-22 22:10:24 -0700259#if defined(CONFIG_DEBUG_LOCK_ALLOC) || defined(CONFIG_SMP)
260extern int rcu_is_cpu_idle(void);
261#endif /* #if defined(CONFIG_DEBUG_LOCK_ALLOC) || defined(CONFIG_SMP) */
262
Paul E. McKenneyc0d6d012012-01-23 12:41:26 -0800263#if defined(CONFIG_HOTPLUG_CPU) && defined(CONFIG_PROVE_RCU)
264bool rcu_lockdep_current_cpu_online(void);
265#else /* #if defined(CONFIG_HOTPLUG_CPU) && defined(CONFIG_PROVE_RCU) */
266static inline bool rcu_lockdep_current_cpu_online(void)
267{
268 return 1;
269}
270#endif /* #else #if defined(CONFIG_HOTPLUG_CPU) && defined(CONFIG_PROVE_RCU) */
271
Paul E. McKenneybc33f242009-08-22 13:56:47 -0700272#ifdef CONFIG_DEBUG_LOCK_ALLOC
Paul E. McKenney632ee202010-02-22 17:04:45 -0800273
Frederic Weisbecker00f49e52011-10-07 18:22:02 +0200274static inline void rcu_lock_acquire(struct lockdep_map *map)
275{
Frederic Weisbecker00f49e52011-10-07 18:22:02 +0200276 lock_acquire(map, 0, 0, 2, 1, NULL, _THIS_IP_);
277}
278
279static inline void rcu_lock_release(struct lockdep_map *map)
280{
Frederic Weisbecker00f49e52011-10-07 18:22:02 +0200281 lock_release(map, 1, _THIS_IP_);
282}
283
Paul E. McKenneybc33f242009-08-22 13:56:47 -0700284extern struct lockdep_map rcu_lock_map;
Paul E. McKenney632ee202010-02-22 17:04:45 -0800285extern struct lockdep_map rcu_bh_lock_map;
Paul E. McKenney632ee202010-02-22 17:04:45 -0800286extern struct lockdep_map rcu_sched_lock_map;
Paul E. McKenneybc293d62010-04-15 12:50:39 -0700287extern int debug_lockdep_rcu_enabled(void);
Paul E. McKenney54dbf962010-03-03 07:46:57 -0800288
Paul E. McKenney632ee202010-02-22 17:04:45 -0800289/**
Paul E. McKenneyca5ecdd2010-04-28 14:39:09 -0700290 * rcu_read_lock_held() - might we be in RCU read-side critical section?
Paul E. McKenney632ee202010-02-22 17:04:45 -0800291 *
Paul E. McKenneyd20200b2010-03-30 10:52:21 -0700292 * If CONFIG_DEBUG_LOCK_ALLOC is selected, returns nonzero iff in an RCU
293 * read-side critical section. In absence of CONFIG_DEBUG_LOCK_ALLOC,
Paul E. McKenney632ee202010-02-22 17:04:45 -0800294 * this assumes we are in an RCU read-side critical section unless it can
Paul E. McKenneyca5ecdd2010-04-28 14:39:09 -0700295 * prove otherwise. This is useful for debug checks in functions that
296 * require that they be called within an RCU read-side critical section.
Paul E. McKenney54dbf962010-03-03 07:46:57 -0800297 *
Paul E. McKenneyca5ecdd2010-04-28 14:39:09 -0700298 * Checks debug_lockdep_rcu_enabled() to prevent false positives during boot
Paul E. McKenney32c141a2010-03-30 10:59:28 -0700299 * and while lockdep is disabled.
Paul E. McKenney3842a082011-11-28 10:42:42 -0800300 *
301 * Note that rcu_read_lock() and the matching rcu_read_unlock() must
302 * occur in the same context, for example, it is illegal to invoke
303 * rcu_read_unlock() in process context if the matching rcu_read_lock()
304 * was invoked from within an irq handler.
Paul E. McKenneyc0d6d012012-01-23 12:41:26 -0800305 *
306 * Note that rcu_read_lock() is disallowed if the CPU is either idle or
307 * offline from an RCU perspective, so check for those as well.
Paul E. McKenney632ee202010-02-22 17:04:45 -0800308 */
309static inline int rcu_read_lock_held(void)
310{
Paul E. McKenney54dbf962010-03-03 07:46:57 -0800311 if (!debug_lockdep_rcu_enabled())
312 return 1;
Frederic Weisbeckere6b80a32011-10-07 16:25:18 -0700313 if (rcu_is_cpu_idle())
314 return 0;
Paul E. McKenneyc0d6d012012-01-23 12:41:26 -0800315 if (!rcu_lockdep_current_cpu_online())
316 return 0;
Paul E. McKenney54dbf962010-03-03 07:46:57 -0800317 return lock_is_held(&rcu_lock_map);
Paul E. McKenney632ee202010-02-22 17:04:45 -0800318}
319
Paul E. McKenneye3818b82010-03-15 17:03:43 -0700320/*
321 * rcu_read_lock_bh_held() is defined out of line to avoid #include-file
322 * hell.
Paul E. McKenney632ee202010-02-22 17:04:45 -0800323 */
Paul E. McKenneye3818b82010-03-15 17:03:43 -0700324extern int rcu_read_lock_bh_held(void);
Paul E. McKenney632ee202010-02-22 17:04:45 -0800325
326/**
Paul E. McKenneyca5ecdd2010-04-28 14:39:09 -0700327 * rcu_read_lock_sched_held() - might we be in RCU-sched read-side critical section?
Paul E. McKenney632ee202010-02-22 17:04:45 -0800328 *
Paul E. McKenneyd20200b2010-03-30 10:52:21 -0700329 * If CONFIG_DEBUG_LOCK_ALLOC is selected, returns nonzero iff in an
330 * RCU-sched read-side critical section. In absence of
331 * CONFIG_DEBUG_LOCK_ALLOC, this assumes we are in an RCU-sched read-side
332 * critical section unless it can prove otherwise. Note that disabling
333 * of preemption (including disabling irqs) counts as an RCU-sched
Paul E. McKenneyca5ecdd2010-04-28 14:39:09 -0700334 * read-side critical section. This is useful for debug checks in functions
335 * that required that they be called within an RCU-sched read-side
336 * critical section.
Paul E. McKenney54dbf962010-03-03 07:46:57 -0800337 *
Paul E. McKenney32c141a2010-03-30 10:59:28 -0700338 * Check debug_lockdep_rcu_enabled() to prevent false positives during boot
339 * and while lockdep is disabled.
Frederic Weisbeckere6b80a32011-10-07 16:25:18 -0700340 *
341 * Note that if the CPU is in the idle loop from an RCU point of
342 * view (ie: that we are in the section between rcu_idle_enter() and
343 * rcu_idle_exit()) then rcu_read_lock_held() returns false even if the CPU
344 * did an rcu_read_lock(). The reason for this is that RCU ignores CPUs
345 * that are in such a section, considering these as in extended quiescent
346 * state, so such a CPU is effectively never in an RCU read-side critical
347 * section regardless of what RCU primitives it invokes. This state of
348 * affairs is required --- we need to keep an RCU-free window in idle
349 * where the CPU may possibly enter into low power mode. This way we can
350 * notice an extended quiescent state to other CPUs that started a grace
351 * period. Otherwise we would delay any grace period as long as we run in
352 * the idle task.
Paul E. McKenneyc0d6d012012-01-23 12:41:26 -0800353 *
354 * Similarly, we avoid claiming an SRCU read lock held if the current
355 * CPU is offline.
Paul E. McKenney632ee202010-02-22 17:04:45 -0800356 */
Frederic Weisbeckerbdd4e852011-06-08 01:13:27 +0200357#ifdef CONFIG_PREEMPT_COUNT
Paul E. McKenney632ee202010-02-22 17:04:45 -0800358static inline int rcu_read_lock_sched_held(void)
359{
360 int lockdep_opinion = 0;
361
Paul E. McKenney54dbf962010-03-03 07:46:57 -0800362 if (!debug_lockdep_rcu_enabled())
363 return 1;
Frederic Weisbeckere6b80a32011-10-07 16:25:18 -0700364 if (rcu_is_cpu_idle())
365 return 0;
Paul E. McKenneyc0d6d012012-01-23 12:41:26 -0800366 if (!rcu_lockdep_current_cpu_online())
367 return 0;
Paul E. McKenney632ee202010-02-22 17:04:45 -0800368 if (debug_locks)
369 lockdep_opinion = lock_is_held(&rcu_sched_lock_map);
Lai Jiangshan0cff8102010-03-18 12:25:33 -0700370 return lockdep_opinion || preempt_count() != 0 || irqs_disabled();
Paul E. McKenney632ee202010-02-22 17:04:45 -0800371}
Frederic Weisbeckerbdd4e852011-06-08 01:13:27 +0200372#else /* #ifdef CONFIG_PREEMPT_COUNT */
Paul E. McKenneye6033e32010-03-03 17:50:16 -0800373static inline int rcu_read_lock_sched_held(void)
374{
375 return 1;
376}
Frederic Weisbeckerbdd4e852011-06-08 01:13:27 +0200377#endif /* #else #ifdef CONFIG_PREEMPT_COUNT */
Paul E. McKenney632ee202010-02-22 17:04:45 -0800378
379#else /* #ifdef CONFIG_DEBUG_LOCK_ALLOC */
380
Paul E. McKenneyd8ab29f2011-10-07 18:22:03 +0200381# define rcu_lock_acquire(a) do { } while (0)
382# define rcu_lock_release(a) do { } while (0)
Paul E. McKenney632ee202010-02-22 17:04:45 -0800383
384static inline int rcu_read_lock_held(void)
385{
386 return 1;
387}
388
389static inline int rcu_read_lock_bh_held(void)
390{
391 return 1;
392}
393
Frederic Weisbeckerbdd4e852011-06-08 01:13:27 +0200394#ifdef CONFIG_PREEMPT_COUNT
Paul E. McKenney632ee202010-02-22 17:04:45 -0800395static inline int rcu_read_lock_sched_held(void)
396{
Paul E. McKenneybbad9372010-04-02 16:17:17 -0700397 return preempt_count() != 0 || irqs_disabled();
Paul E. McKenney632ee202010-02-22 17:04:45 -0800398}
Frederic Weisbeckerbdd4e852011-06-08 01:13:27 +0200399#else /* #ifdef CONFIG_PREEMPT_COUNT */
Paul E. McKenneye6033e32010-03-03 17:50:16 -0800400static inline int rcu_read_lock_sched_held(void)
401{
402 return 1;
403}
Frederic Weisbeckerbdd4e852011-06-08 01:13:27 +0200404#endif /* #else #ifdef CONFIG_PREEMPT_COUNT */
Paul E. McKenney632ee202010-02-22 17:04:45 -0800405
406#endif /* #else #ifdef CONFIG_DEBUG_LOCK_ALLOC */
407
408#ifdef CONFIG_PROVE_RCU
409
Paul E. McKenneyee84b822010-05-06 09:28:41 -0700410extern int rcu_my_thread_group_empty(void);
411
Tetsuo Handa4221a992010-06-26 01:08:19 +0900412/**
413 * rcu_lockdep_assert - emit lockdep splat if specified condition not met
414 * @c: condition to check
Paul E. McKenneyb3fbab02011-05-24 08:31:09 -0700415 * @s: informative message
Tetsuo Handa4221a992010-06-26 01:08:19 +0900416 */
Paul E. McKenneyb3fbab02011-05-24 08:31:09 -0700417#define rcu_lockdep_assert(c, s) \
Lai Jiangshan2b3fc352010-04-20 16:23:07 +0800418 do { \
Jan Beulich7ccaba52012-03-23 15:01:52 -0700419 static bool __section(.data.unlikely) __warned; \
Lai Jiangshan2b3fc352010-04-20 16:23:07 +0800420 if (debug_lockdep_rcu_enabled() && !__warned && !(c)) { \
421 __warned = true; \
Paul E. McKenneyb3fbab02011-05-24 08:31:09 -0700422 lockdep_rcu_suspicious(__FILE__, __LINE__, s); \
Lai Jiangshan2b3fc352010-04-20 16:23:07 +0800423 } \
424 } while (0)
425
Paul E. McKenney50406b92012-01-12 13:49:19 -0800426#if defined(CONFIG_PROVE_RCU) && !defined(CONFIG_PREEMPT_RCU)
427static inline void rcu_preempt_sleep_check(void)
428{
429 rcu_lockdep_assert(!lock_is_held(&rcu_lock_map),
430 "Illegal context switch in RCU read-side "
431 "critical section");
432}
433#else /* #ifdef CONFIG_PROVE_RCU */
434static inline void rcu_preempt_sleep_check(void)
435{
436}
437#endif /* #else #ifdef CONFIG_PROVE_RCU */
438
Paul E. McKenneyb3fbab02011-05-24 08:31:09 -0700439#define rcu_sleep_check() \
440 do { \
Paul E. McKenney50406b92012-01-12 13:49:19 -0800441 rcu_preempt_sleep_check(); \
Paul E. McKenneyb3fbab02011-05-24 08:31:09 -0700442 rcu_lockdep_assert(!lock_is_held(&rcu_bh_lock_map), \
443 "Illegal context switch in RCU-bh" \
444 " read-side critical section"); \
445 rcu_lockdep_assert(!lock_is_held(&rcu_sched_lock_map), \
446 "Illegal context switch in RCU-sched"\
447 " read-side critical section"); \
448 } while (0)
449
Paul E. McKenneyca5ecdd2010-04-28 14:39:09 -0700450#else /* #ifdef CONFIG_PROVE_RCU */
451
Paul E. McKenneyb3fbab02011-05-24 08:31:09 -0700452#define rcu_lockdep_assert(c, s) do { } while (0)
453#define rcu_sleep_check() do { } while (0)
Paul E. McKenneyca5ecdd2010-04-28 14:39:09 -0700454
455#endif /* #else #ifdef CONFIG_PROVE_RCU */
456
457/*
458 * Helper functions for rcu_dereference_check(), rcu_dereference_protected()
459 * and rcu_assign_pointer(). Some of these could be folded into their
460 * callers, but they are left separate in order to ease introduction of
461 * multiple flavors of pointers to match the multiple flavors of RCU
462 * (e.g., __rcu_bh, * __rcu_sched, and __srcu), should this make sense in
463 * the future.
464 */
Paul E. McKenney53ecfba2010-09-13 17:24:21 -0700465
466#ifdef __CHECKER__
467#define rcu_dereference_sparse(p, space) \
468 ((void)(((typeof(*p) space *)p) == p))
469#else /* #ifdef __CHECKER__ */
470#define rcu_dereference_sparse(p, space)
471#endif /* #else #ifdef __CHECKER__ */
472
Paul E. McKenneyca5ecdd2010-04-28 14:39:09 -0700473#define __rcu_access_pointer(p, space) \
474 ({ \
475 typeof(*p) *_________p1 = (typeof(*p)*__force )ACCESS_ONCE(p); \
Paul E. McKenney53ecfba2010-09-13 17:24:21 -0700476 rcu_dereference_sparse(p, space); \
Paul E. McKenneyca5ecdd2010-04-28 14:39:09 -0700477 ((typeof(*p) __force __kernel *)(_________p1)); \
478 })
479#define __rcu_dereference_check(p, c, space) \
480 ({ \
481 typeof(*p) *_________p1 = (typeof(*p)*__force )ACCESS_ONCE(p); \
Paul E. McKenneyb3fbab02011-05-24 08:31:09 -0700482 rcu_lockdep_assert(c, "suspicious rcu_dereference_check()" \
483 " usage"); \
Paul E. McKenney53ecfba2010-09-13 17:24:21 -0700484 rcu_dereference_sparse(p, space); \
Paul E. McKenneyca5ecdd2010-04-28 14:39:09 -0700485 smp_read_barrier_depends(); \
486 ((typeof(*p) __force __kernel *)(_________p1)); \
487 })
488#define __rcu_dereference_protected(p, c, space) \
489 ({ \
Paul E. McKenneyb3fbab02011-05-24 08:31:09 -0700490 rcu_lockdep_assert(c, "suspicious rcu_dereference_protected()" \
491 " usage"); \
Paul E. McKenney53ecfba2010-09-13 17:24:21 -0700492 rcu_dereference_sparse(p, space); \
Paul E. McKenneyca5ecdd2010-04-28 14:39:09 -0700493 ((typeof(*p) __force __kernel *)(p)); \
494 })
495
Paul E. McKenneya4dd9922011-04-01 07:15:14 -0700496#define __rcu_access_index(p, space) \
497 ({ \
498 typeof(p) _________p1 = ACCESS_ONCE(p); \
499 rcu_dereference_sparse(p, space); \
500 (_________p1); \
501 })
Paul E. McKenneyca5ecdd2010-04-28 14:39:09 -0700502#define __rcu_dereference_index_check(p, c) \
503 ({ \
504 typeof(p) _________p1 = ACCESS_ONCE(p); \
Paul E. McKenneyb3fbab02011-05-24 08:31:09 -0700505 rcu_lockdep_assert(c, \
506 "suspicious rcu_dereference_index_check()" \
507 " usage"); \
Paul E. McKenneyca5ecdd2010-04-28 14:39:09 -0700508 smp_read_barrier_depends(); \
509 (_________p1); \
510 })
511#define __rcu_assign_pointer(p, v, space) \
Paul E. McKenneye9023c42012-05-16 15:51:08 -0700512 do { \
Eric Dumazetd322f452011-07-31 22:09:25 -0700513 smp_wmb(); \
Paul E. McKenneyca5ecdd2010-04-28 14:39:09 -0700514 (p) = (typeof(*v) __force space *)(v); \
Paul E. McKenneye9023c42012-05-16 15:51:08 -0700515 } while (0)
Paul E. McKenneyca5ecdd2010-04-28 14:39:09 -0700516
517
Paul E. McKenney632ee202010-02-22 17:04:45 -0800518/**
Paul E. McKenneyca5ecdd2010-04-28 14:39:09 -0700519 * rcu_access_pointer() - fetch RCU pointer with no dereferencing
520 * @p: The pointer to read
521 *
522 * Return the value of the specified RCU-protected pointer, but omit the
523 * smp_read_barrier_depends() and keep the ACCESS_ONCE(). This is useful
524 * when the value of this pointer is accessed, but the pointer is not
525 * dereferenced, for example, when testing an RCU-protected pointer against
526 * NULL. Although rcu_access_pointer() may also be used in cases where
527 * update-side locks prevent the value of the pointer from changing, you
528 * should instead use rcu_dereference_protected() for this use case.
Paul E. McKenney5e1ee6e2012-01-12 17:21:20 -0800529 *
530 * It is also permissible to use rcu_access_pointer() when read-side
531 * access to the pointer was removed at least one grace period ago, as
532 * is the case in the context of the RCU callback that is freeing up
533 * the data, or after a synchronize_rcu() returns. This can be useful
534 * when tearing down multi-linked structures after a grace period
535 * has elapsed.
Paul E. McKenneyca5ecdd2010-04-28 14:39:09 -0700536 */
537#define rcu_access_pointer(p) __rcu_access_pointer((p), __rcu)
538
539/**
540 * rcu_dereference_check() - rcu_dereference with debug checking
David Howellsc08c68d2010-04-09 15:39:11 -0700541 * @p: The pointer to read, prior to dereferencing
542 * @c: The conditions under which the dereference will take place
Paul E. McKenney632ee202010-02-22 17:04:45 -0800543 *
David Howellsc08c68d2010-04-09 15:39:11 -0700544 * Do an rcu_dereference(), but check that the conditions under which the
Paul E. McKenneyca5ecdd2010-04-28 14:39:09 -0700545 * dereference will take place are correct. Typically the conditions
546 * indicate the various locking conditions that should be held at that
547 * point. The check should return true if the conditions are satisfied.
548 * An implicit check for being in an RCU read-side critical section
549 * (rcu_read_lock()) is included.
David Howellsc08c68d2010-04-09 15:39:11 -0700550 *
551 * For example:
552 *
Paul E. McKenneyca5ecdd2010-04-28 14:39:09 -0700553 * bar = rcu_dereference_check(foo->bar, lockdep_is_held(&foo->lock));
David Howellsc08c68d2010-04-09 15:39:11 -0700554 *
555 * could be used to indicate to lockdep that foo->bar may only be dereferenced
Paul E. McKenneyca5ecdd2010-04-28 14:39:09 -0700556 * if either rcu_read_lock() is held, or that the lock required to replace
David Howellsc08c68d2010-04-09 15:39:11 -0700557 * the bar struct at foo->bar is held.
558 *
559 * Note that the list of conditions may also include indications of when a lock
560 * need not be held, for example during initialisation or destruction of the
561 * target struct:
562 *
Paul E. McKenneyca5ecdd2010-04-28 14:39:09 -0700563 * bar = rcu_dereference_check(foo->bar, lockdep_is_held(&foo->lock) ||
David Howellsc08c68d2010-04-09 15:39:11 -0700564 * atomic_read(&foo->usage) == 0);
Paul E. McKenneyca5ecdd2010-04-28 14:39:09 -0700565 *
566 * Inserts memory barriers on architectures that require them
567 * (currently only the Alpha), prevents the compiler from refetching
568 * (and from merging fetches), and, more importantly, documents exactly
569 * which pointers are protected by RCU and checks that the pointer is
570 * annotated as __rcu.
Paul E. McKenney632ee202010-02-22 17:04:45 -0800571 */
572#define rcu_dereference_check(p, c) \
Paul E. McKenneyca5ecdd2010-04-28 14:39:09 -0700573 __rcu_dereference_check((p), rcu_read_lock_held() || (c), __rcu)
Paul E. McKenney632ee202010-02-22 17:04:45 -0800574
Paul E. McKenneyb62730b2010-04-09 15:39:10 -0700575/**
Paul E. McKenneyca5ecdd2010-04-28 14:39:09 -0700576 * rcu_dereference_bh_check() - rcu_dereference_bh with debug checking
577 * @p: The pointer to read, prior to dereferencing
578 * @c: The conditions under which the dereference will take place
579 *
580 * This is the RCU-bh counterpart to rcu_dereference_check().
581 */
582#define rcu_dereference_bh_check(p, c) \
583 __rcu_dereference_check((p), rcu_read_lock_bh_held() || (c), __rcu)
584
585/**
586 * rcu_dereference_sched_check() - rcu_dereference_sched with debug checking
587 * @p: The pointer to read, prior to dereferencing
588 * @c: The conditions under which the dereference will take place
589 *
590 * This is the RCU-sched counterpart to rcu_dereference_check().
591 */
592#define rcu_dereference_sched_check(p, c) \
593 __rcu_dereference_check((p), rcu_read_lock_sched_held() || (c), \
594 __rcu)
595
596#define rcu_dereference_raw(p) rcu_dereference_check(p, 1) /*@@@ needed? @@@*/
597
598/**
Paul E. McKenneya4dd9922011-04-01 07:15:14 -0700599 * rcu_access_index() - fetch RCU index with no dereferencing
600 * @p: The index to read
601 *
602 * Return the value of the specified RCU-protected index, but omit the
603 * smp_read_barrier_depends() and keep the ACCESS_ONCE(). This is useful
604 * when the value of this index is accessed, but the index is not
605 * dereferenced, for example, when testing an RCU-protected index against
606 * -1. Although rcu_access_index() may also be used in cases where
607 * update-side locks prevent the value of the index from changing, you
608 * should instead use rcu_dereference_index_protected() for this use case.
609 */
610#define rcu_access_index(p) __rcu_access_index((p), __rcu)
611
612/**
Paul E. McKenneyca5ecdd2010-04-28 14:39:09 -0700613 * rcu_dereference_index_check() - rcu_dereference for indices with debug checking
614 * @p: The pointer to read, prior to dereferencing
615 * @c: The conditions under which the dereference will take place
616 *
617 * Similar to rcu_dereference_check(), but omits the sparse checking.
618 * This allows rcu_dereference_index_check() to be used on integers,
619 * which can then be used as array indices. Attempting to use
620 * rcu_dereference_check() on an integer will give compiler warnings
621 * because the sparse address-space mechanism relies on dereferencing
622 * the RCU-protected pointer. Dereferencing integers is not something
623 * that even gcc will put up with.
624 *
625 * Note that this function does not implicitly check for RCU read-side
626 * critical sections. If this function gains lots of uses, it might
627 * make sense to provide versions for each flavor of RCU, but it does
628 * not make sense as of early 2010.
629 */
630#define rcu_dereference_index_check(p, c) \
631 __rcu_dereference_index_check((p), (c))
632
633/**
634 * rcu_dereference_protected() - fetch RCU pointer when updates prevented
635 * @p: The pointer to read, prior to dereferencing
636 * @c: The conditions under which the dereference will take place
Paul E. McKenneyb62730b2010-04-09 15:39:10 -0700637 *
638 * Return the value of the specified RCU-protected pointer, but omit
639 * both the smp_read_barrier_depends() and the ACCESS_ONCE(). This
640 * is useful in cases where update-side locks prevent the value of the
641 * pointer from changing. Please note that this primitive does -not-
642 * prevent the compiler from repeating this reference or combining it
643 * with other references, so it should not be used without protection
644 * of appropriate locks.
Paul E. McKenneyca5ecdd2010-04-28 14:39:09 -0700645 *
646 * This function is only for update-side use. Using this function
647 * when protected only by rcu_read_lock() will result in infrequent
648 * but very ugly failures.
Paul E. McKenneyb62730b2010-04-09 15:39:10 -0700649 */
650#define rcu_dereference_protected(p, c) \
Paul E. McKenneyca5ecdd2010-04-28 14:39:09 -0700651 __rcu_dereference_protected((p), (c), __rcu)
Paul E. McKenneybc33f242009-08-22 13:56:47 -0700652
Paul E. McKenneyca5ecdd2010-04-28 14:39:09 -0700653
654/**
655 * rcu_dereference() - fetch RCU-protected pointer for dereferencing
656 * @p: The pointer to read, prior to dereferencing
657 *
658 * This is a simple wrapper around rcu_dereference_check().
659 */
660#define rcu_dereference(p) rcu_dereference_check(p, 0)
661
662/**
663 * rcu_dereference_bh() - fetch an RCU-bh-protected pointer for dereferencing
664 * @p: The pointer to read, prior to dereferencing
665 *
666 * Makes rcu_dereference_check() do the dirty work.
667 */
668#define rcu_dereference_bh(p) rcu_dereference_bh_check(p, 0)
669
670/**
671 * rcu_dereference_sched() - fetch RCU-sched-protected pointer for dereferencing
672 * @p: The pointer to read, prior to dereferencing
673 *
674 * Makes rcu_dereference_check() do the dirty work.
675 */
676#define rcu_dereference_sched(p) rcu_dereference_sched_check(p, 0)
677
678/**
679 * rcu_read_lock() - mark the beginning of an RCU read-side critical section
Linus Torvalds1da177e2005-04-16 15:20:36 -0700680 *
Paul E. McKenney9b06e812005-05-01 08:59:04 -0700681 * When synchronize_rcu() is invoked on one CPU while other CPUs
Linus Torvalds1da177e2005-04-16 15:20:36 -0700682 * are within RCU read-side critical sections, then the
Paul E. McKenney9b06e812005-05-01 08:59:04 -0700683 * synchronize_rcu() is guaranteed to block until after all the other
Linus Torvalds1da177e2005-04-16 15:20:36 -0700684 * CPUs exit their critical sections. Similarly, if call_rcu() is invoked
685 * on one CPU while other CPUs are within RCU read-side critical
686 * sections, invocation of the corresponding RCU callback is deferred
687 * until after the all the other CPUs exit their critical sections.
688 *
689 * Note, however, that RCU callbacks are permitted to run concurrently
Paul E. McKenney77d84852010-07-08 17:38:59 -0700690 * with new RCU read-side critical sections. One way that this can happen
Linus Torvalds1da177e2005-04-16 15:20:36 -0700691 * is via the following sequence of events: (1) CPU 0 enters an RCU
692 * read-side critical section, (2) CPU 1 invokes call_rcu() to register
693 * an RCU callback, (3) CPU 0 exits the RCU read-side critical section,
694 * (4) CPU 2 enters a RCU read-side critical section, (5) the RCU
695 * callback is invoked. This is legal, because the RCU read-side critical
696 * section that was running concurrently with the call_rcu() (and which
697 * therefore might be referencing something that the corresponding RCU
698 * callback would free up) has completed before the corresponding
699 * RCU callback is invoked.
700 *
701 * RCU read-side critical sections may be nested. Any deferred actions
702 * will be deferred until the outermost RCU read-side critical section
703 * completes.
704 *
Paul E. McKenney9079fd72010-08-07 21:59:54 -0700705 * You can avoid reading and understanding the next paragraph by
706 * following this rule: don't put anything in an rcu_read_lock() RCU
707 * read-side critical section that would block in a !PREEMPT kernel.
708 * But if you want the full story, read on!
709 *
710 * In non-preemptible RCU implementations (TREE_RCU and TINY_RCU), it
711 * is illegal to block while in an RCU read-side critical section. In
712 * preemptible RCU implementations (TREE_PREEMPT_RCU and TINY_PREEMPT_RCU)
713 * in CONFIG_PREEMPT kernel builds, RCU read-side critical sections may
714 * be preempted, but explicit blocking is illegal. Finally, in preemptible
715 * RCU implementations in real-time (CONFIG_PREEMPT_RT) kernel builds,
716 * RCU read-side critical sections may be preempted and they may also
717 * block, but only when acquiring spinlocks that are subject to priority
718 * inheritance.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700719 */
Paul E. McKenneybc33f242009-08-22 13:56:47 -0700720static inline void rcu_read_lock(void)
721{
722 __rcu_read_lock();
723 __acquire(RCU);
Paul E. McKenneyd8ab29f2011-10-07 18:22:03 +0200724 rcu_lock_acquire(&rcu_lock_map);
Heiko Carstensbde23c62012-02-01 10:30:46 -0800725 rcu_lockdep_assert(!rcu_is_cpu_idle(),
726 "rcu_read_lock() used illegally while idle");
Paul E. McKenneybc33f242009-08-22 13:56:47 -0700727}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700728
Linus Torvalds1da177e2005-04-16 15:20:36 -0700729/*
730 * So where is rcu_write_lock()? It does not exist, as there is no
731 * way for writers to lock out RCU readers. This is a feature, not
732 * a bug -- this property is what provides RCU's performance benefits.
733 * Of course, writers must coordinate with each other. The normal
734 * spinlock primitives work well for this, but any other technique may be
735 * used as well. RCU does not care how the writers keep out of each
736 * others' way, as long as they do so.
737 */
Paul E. McKenney3d76c082009-09-28 07:46:32 -0700738
739/**
Paul E. McKenneyca5ecdd2010-04-28 14:39:09 -0700740 * rcu_read_unlock() - marks the end of an RCU read-side critical section.
Paul E. McKenney3d76c082009-09-28 07:46:32 -0700741 *
742 * See rcu_read_lock() for more information.
743 */
Paul E. McKenneybc33f242009-08-22 13:56:47 -0700744static inline void rcu_read_unlock(void)
745{
Heiko Carstensbde23c62012-02-01 10:30:46 -0800746 rcu_lockdep_assert(!rcu_is_cpu_idle(),
747 "rcu_read_unlock() used illegally while idle");
Paul E. McKenneyd8ab29f2011-10-07 18:22:03 +0200748 rcu_lock_release(&rcu_lock_map);
Paul E. McKenneybc33f242009-08-22 13:56:47 -0700749 __release(RCU);
750 __rcu_read_unlock();
751}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700752
753/**
Paul E. McKenneyca5ecdd2010-04-28 14:39:09 -0700754 * rcu_read_lock_bh() - mark the beginning of an RCU-bh critical section
Linus Torvalds1da177e2005-04-16 15:20:36 -0700755 *
756 * This is equivalent of rcu_read_lock(), but to be used when updates
Paul E. McKenneyca5ecdd2010-04-28 14:39:09 -0700757 * are being done using call_rcu_bh() or synchronize_rcu_bh(). Since
758 * both call_rcu_bh() and synchronize_rcu_bh() consider completion of a
759 * softirq handler to be a quiescent state, a process in RCU read-side
760 * critical section must be protected by disabling softirqs. Read-side
761 * critical sections in interrupt context can use just rcu_read_lock(),
762 * though this should at least be commented to avoid confusing people
763 * reading the code.
Paul E. McKenney3842a082011-11-28 10:42:42 -0800764 *
765 * Note that rcu_read_lock_bh() and the matching rcu_read_unlock_bh()
766 * must occur in the same context, for example, it is illegal to invoke
767 * rcu_read_unlock_bh() from one task if the matching rcu_read_lock_bh()
768 * was invoked from some other task.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700769 */
Paul E. McKenneybc33f242009-08-22 13:56:47 -0700770static inline void rcu_read_lock_bh(void)
771{
Paul E. McKenney6206ab92011-08-01 06:22:11 -0700772 local_bh_disable();
Paul E. McKenneybc33f242009-08-22 13:56:47 -0700773 __acquire(RCU_BH);
Paul E. McKenneyd8ab29f2011-10-07 18:22:03 +0200774 rcu_lock_acquire(&rcu_bh_lock_map);
Heiko Carstensbde23c62012-02-01 10:30:46 -0800775 rcu_lockdep_assert(!rcu_is_cpu_idle(),
776 "rcu_read_lock_bh() used illegally while idle");
Paul E. McKenneybc33f242009-08-22 13:56:47 -0700777}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700778
779/*
780 * rcu_read_unlock_bh - marks the end of a softirq-only RCU critical section
781 *
782 * See rcu_read_lock_bh() for more information.
783 */
Paul E. McKenneybc33f242009-08-22 13:56:47 -0700784static inline void rcu_read_unlock_bh(void)
785{
Heiko Carstensbde23c62012-02-01 10:30:46 -0800786 rcu_lockdep_assert(!rcu_is_cpu_idle(),
787 "rcu_read_unlock_bh() used illegally while idle");
Paul E. McKenneyd8ab29f2011-10-07 18:22:03 +0200788 rcu_lock_release(&rcu_bh_lock_map);
Paul E. McKenneybc33f242009-08-22 13:56:47 -0700789 __release(RCU_BH);
Paul E. McKenney6206ab92011-08-01 06:22:11 -0700790 local_bh_enable();
Paul E. McKenneybc33f242009-08-22 13:56:47 -0700791}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700792
793/**
Paul E. McKenneyca5ecdd2010-04-28 14:39:09 -0700794 * rcu_read_lock_sched() - mark the beginning of a RCU-sched critical section
Mathieu Desnoyers1c50b722008-09-29 11:06:46 -0400795 *
Paul E. McKenneyca5ecdd2010-04-28 14:39:09 -0700796 * This is equivalent of rcu_read_lock(), but to be used when updates
797 * are being done using call_rcu_sched() or synchronize_rcu_sched().
798 * Read-side critical sections can also be introduced by anything that
799 * disables preemption, including local_irq_disable() and friends.
Paul E. McKenney3842a082011-11-28 10:42:42 -0800800 *
801 * Note that rcu_read_lock_sched() and the matching rcu_read_unlock_sched()
802 * must occur in the same context, for example, it is illegal to invoke
803 * rcu_read_unlock_sched() from process context if the matching
804 * rcu_read_lock_sched() was invoked from an NMI handler.
Mathieu Desnoyers1c50b722008-09-29 11:06:46 -0400805 */
Paul E. McKenneyd6714c22009-08-22 13:56:46 -0700806static inline void rcu_read_lock_sched(void)
807{
808 preempt_disable();
Paul E. McKenneybc33f242009-08-22 13:56:47 -0700809 __acquire(RCU_SCHED);
Paul E. McKenneyd8ab29f2011-10-07 18:22:03 +0200810 rcu_lock_acquire(&rcu_sched_lock_map);
Heiko Carstensbde23c62012-02-01 10:30:46 -0800811 rcu_lockdep_assert(!rcu_is_cpu_idle(),
812 "rcu_read_lock_sched() used illegally while idle");
Paul E. McKenneyd6714c22009-08-22 13:56:46 -0700813}
Paul E. McKenney1eba8f82009-09-23 09:50:42 -0700814
815/* Used by lockdep and tracing: cannot be traced, cannot call lockdep. */
Paul E. McKenney7c614d62009-08-24 09:42:00 -0700816static inline notrace void rcu_read_lock_sched_notrace(void)
Paul E. McKenneyd6714c22009-08-22 13:56:46 -0700817{
818 preempt_disable_notrace();
Paul E. McKenneybc33f242009-08-22 13:56:47 -0700819 __acquire(RCU_SCHED);
Paul E. McKenneyd6714c22009-08-22 13:56:46 -0700820}
Mathieu Desnoyers1c50b722008-09-29 11:06:46 -0400821
822/*
823 * rcu_read_unlock_sched - marks the end of a RCU-classic critical section
824 *
825 * See rcu_read_lock_sched for more information.
826 */
Paul E. McKenneyd6714c22009-08-22 13:56:46 -0700827static inline void rcu_read_unlock_sched(void)
828{
Heiko Carstensbde23c62012-02-01 10:30:46 -0800829 rcu_lockdep_assert(!rcu_is_cpu_idle(),
830 "rcu_read_unlock_sched() used illegally while idle");
Paul E. McKenneyd8ab29f2011-10-07 18:22:03 +0200831 rcu_lock_release(&rcu_sched_lock_map);
Paul E. McKenneybc33f242009-08-22 13:56:47 -0700832 __release(RCU_SCHED);
Paul E. McKenneyd6714c22009-08-22 13:56:46 -0700833 preempt_enable();
834}
Paul E. McKenney1eba8f82009-09-23 09:50:42 -0700835
836/* Used by lockdep and tracing: cannot be traced, cannot call lockdep. */
Paul E. McKenney7c614d62009-08-24 09:42:00 -0700837static inline notrace void rcu_read_unlock_sched_notrace(void)
Paul E. McKenneyd6714c22009-08-22 13:56:46 -0700838{
Paul E. McKenneybc33f242009-08-22 13:56:47 -0700839 __release(RCU_SCHED);
Paul E. McKenneyd6714c22009-08-22 13:56:46 -0700840 preempt_enable_notrace();
841}
Mathieu Desnoyers1c50b722008-09-29 11:06:46 -0400842
Mathieu Desnoyers1c50b722008-09-29 11:06:46 -0400843/**
Paul E. McKenneyca5ecdd2010-04-28 14:39:09 -0700844 * rcu_assign_pointer() - assign to RCU-protected pointer
845 * @p: pointer to assign to
846 * @v: value to assign (publish)
Paul E. McKenneyc26d34a2010-02-22 17:04:46 -0800847 *
Paul E. McKenneyca5ecdd2010-04-28 14:39:09 -0700848 * Assigns the specified value to the specified RCU-protected
849 * pointer, ensuring that any concurrent RCU readers will see
Paul E. McKenneye9023c42012-05-16 15:51:08 -0700850 * any prior initialization.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700851 *
852 * Inserts memory barriers on architectures that require them
Paul E. McKenney6846c0c2011-07-31 22:33:02 -0700853 * (which is most of them), and also prevents the compiler from
854 * reordering the code that initializes the structure after the pointer
855 * assignment. More importantly, this call documents which pointers
856 * will be dereferenced by RCU read-side code.
857 *
858 * In some special cases, you may use RCU_INIT_POINTER() instead
859 * of rcu_assign_pointer(). RCU_INIT_POINTER() is a bit faster due
860 * to the fact that it does not constrain either the CPU or the compiler.
861 * That said, using RCU_INIT_POINTER() when you should have used
862 * rcu_assign_pointer() is a very bad thing that results in
863 * impossible-to-diagnose memory corruption. So please be careful.
864 * See the RCU_INIT_POINTER() comment header for details.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700865 */
Paul E. McKenneyd99c4f62008-02-06 01:37:25 -0800866#define rcu_assign_pointer(p, v) \
Paul E. McKenneyca5ecdd2010-04-28 14:39:09 -0700867 __rcu_assign_pointer((p), (v), __rcu)
868
869/**
870 * RCU_INIT_POINTER() - initialize an RCU protected pointer
871 *
Paul E. McKenney6846c0c2011-07-31 22:33:02 -0700872 * Initialize an RCU-protected pointer in special cases where readers
873 * do not need ordering constraints on the CPU or the compiler. These
874 * special cases are:
875 *
876 * 1. This use of RCU_INIT_POINTER() is NULLing out the pointer -or-
877 * 2. The caller has taken whatever steps are required to prevent
878 * RCU readers from concurrently accessing this pointer -or-
879 * 3. The referenced data structure has already been exposed to
880 * readers either at compile time or via rcu_assign_pointer() -and-
881 * a. You have not made -any- reader-visible changes to
882 * this structure since then -or-
883 * b. It is OK for readers accessing this structure from its
884 * new location to see the old state of the structure. (For
885 * example, the changes were to statistical counters or to
886 * other state where exact synchronization is not required.)
887 *
888 * Failure to follow these rules governing use of RCU_INIT_POINTER() will
889 * result in impossible-to-diagnose memory corruption. As in the structures
890 * will look OK in crash dumps, but any concurrent RCU readers might
891 * see pre-initialized values of the referenced data structure. So
892 * please be very careful how you use RCU_INIT_POINTER()!!!
893 *
894 * If you are creating an RCU-protected linked structure that is accessed
895 * by a single external-to-structure RCU-protected pointer, then you may
896 * use RCU_INIT_POINTER() to initialize the internal RCU-protected
897 * pointers, but you must use rcu_assign_pointer() to initialize the
898 * external-to-structure pointer -after- you have completely initialized
899 * the reader-accessible portions of the linked structure.
Paul E. McKenneyca5ecdd2010-04-28 14:39:09 -0700900 */
901#define RCU_INIT_POINTER(p, v) \
Paul E. McKenneyd1b88eb2012-05-16 15:42:30 -0700902 do { \
903 p = (typeof(*v) __force __rcu *)(v); \
904 } while (0)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700905
Paul E. McKenney172708d2012-05-16 15:23:45 -0700906/**
907 * RCU_POINTER_INITIALIZER() - statically initialize an RCU protected pointer
908 *
909 * GCC-style initialization for an RCU-protected pointer in a structure field.
910 */
911#define RCU_POINTER_INITIALIZER(p, v) \
912 .p = (typeof(*v) __force __rcu *)(v)
913
Jan Engelhardtd8169d42012-04-19 11:44:39 -0700914/*
915 * Does the specified offset indicate that the corresponding rcu_head
916 * structure can be handled by kfree_rcu()?
917 */
918#define __is_kfree_rcu_offset(offset) ((offset) < 4096)
919
920/*
921 * Helper macro for kfree_rcu() to prevent argument-expansion eyestrain.
922 */
923#define __kfree_rcu(head, offset) \
924 do { \
925 BUILD_BUG_ON(!__is_kfree_rcu_offset(offset)); \
926 call_rcu(head, (void (*)(struct rcu_head *))(unsigned long)(offset)); \
927 } while (0)
928
Lai Jiangshan9ab15442011-03-18 11:15:47 +0800929/**
930 * kfree_rcu() - kfree an object after a grace period.
931 * @ptr: pointer to kfree
932 * @rcu_head: the name of the struct rcu_head within the type of @ptr.
933 *
934 * Many rcu callbacks functions just call kfree() on the base structure.
935 * These functions are trivial, but their size adds up, and furthermore
936 * when they are used in a kernel module, that module must invoke the
937 * high-latency rcu_barrier() function at module-unload time.
938 *
939 * The kfree_rcu() function handles this issue. Rather than encoding a
940 * function address in the embedded rcu_head structure, kfree_rcu() instead
941 * encodes the offset of the rcu_head structure within the base structure.
942 * Because the functions are not allowed in the low-order 4096 bytes of
943 * kernel virtual memory, offsets up to 4095 bytes can be accommodated.
944 * If the offset is larger than 4095 bytes, a compile-time error will
945 * be generated in __kfree_rcu(). If this error is triggered, you can
946 * either fall back to use of call_rcu() or rearrange the structure to
947 * position the rcu_head structure into the first 4096 bytes.
948 *
949 * Note that the allowable offset might decrease in the future, for example,
950 * to allow something like kmem_cache_free_rcu().
Jan Engelhardtd8169d42012-04-19 11:44:39 -0700951 *
952 * The BUILD_BUG_ON check must not involve any function calls, hence the
953 * checks are done in macros here.
Lai Jiangshan9ab15442011-03-18 11:15:47 +0800954 */
955#define kfree_rcu(ptr, rcu_head) \
956 __kfree_rcu(&((ptr)->rcu_head), offsetof(typeof(*(ptr)), rcu_head))
957
Linus Torvalds1da177e2005-04-16 15:20:36 -0700958#endif /* __LINUX_RCUPDATE_H */