blob: 7d537ced949aa83b33b4c20e32c991d2123ca5a5 [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001#ifndef __LINUX_SPINLOCK_H
2#define __LINUX_SPINLOCK_H
3
4/*
Ingo Molnarfb1c8f92005-09-10 00:25:56 -07005 * include/linux/spinlock.h - generic spinlock/rwlock declarations
6 *
7 * here's the role of the various spinlock/rwlock related include files:
8 *
9 * on SMP builds:
10 *
Thomas Gleixnerfb3a6bb2009-12-03 20:01:19 +010011 * asm/spinlock_types.h: contains the arch_spinlock_t/arch_rwlock_t and the
Ingo Molnarfb1c8f92005-09-10 00:25:56 -070012 * initializers
13 *
14 * linux/spinlock_types.h:
15 * defines the generic type and initializers
16 *
Thomas Gleixner0199c4e2009-12-02 20:01:25 +010017 * asm/spinlock.h: contains the arch_spin_*()/etc. lowlevel
Ingo Molnarfb1c8f92005-09-10 00:25:56 -070018 * implementations, mostly inline assembly code
19 *
20 * (also included on UP-debug builds:)
21 *
22 * linux/spinlock_api_smp.h:
23 * contains the prototypes for the _spin_*() APIs.
24 *
25 * linux/spinlock.h: builds the final spin_*() APIs.
26 *
27 * on UP builds:
28 *
29 * linux/spinlock_type_up.h:
30 * contains the generic, simplified UP spinlock type.
31 * (which is an empty structure on non-debug builds)
32 *
33 * linux/spinlock_types.h:
34 * defines the generic type and initializers
35 *
36 * linux/spinlock_up.h:
Thomas Gleixner0199c4e2009-12-02 20:01:25 +010037 * contains the arch_spin_*()/etc. version of UP
Ingo Molnarfb1c8f92005-09-10 00:25:56 -070038 * builds. (which are NOPs on non-debug, non-preempt
39 * builds)
40 *
41 * (included on UP-non-debug builds:)
42 *
43 * linux/spinlock_api_up.h:
44 * builds the _spin_*() APIs.
45 *
46 * linux/spinlock.h: builds the final spin_*() APIs.
Linus Torvalds1da177e2005-04-16 15:20:36 -070047 */
48
Steven Rostedt3f3078912008-07-25 01:45:25 -070049#include <linux/typecheck.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070050#include <linux/preempt.h>
51#include <linux/linkage.h>
52#include <linux/compiler.h>
David Howellsdf9ee292010-10-07 14:08:55 +010053#include <linux/irqflags.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070054#include <linux/thread_info.h>
55#include <linux/kernel.h>
56#include <linux/stringify.h>
Andrew Morton676dcb82006-12-06 20:31:30 -080057#include <linux/bottom_half.h>
David Howells96f951e2012-03-28 18:30:03 +010058#include <asm/barrier.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070059
Linus Torvalds1da177e2005-04-16 15:20:36 -070060
61/*
62 * Must define these before including other files, inline functions need them
63 */
Denys Vlasenko75ddb0e2010-02-20 01:03:48 +010064#define LOCK_SECTION_NAME ".text..lock."KBUILD_BASENAME
Linus Torvalds1da177e2005-04-16 15:20:36 -070065
66#define LOCK_SECTION_START(extra) \
67 ".subsection 1\n\t" \
68 extra \
69 ".ifndef " LOCK_SECTION_NAME "\n\t" \
70 LOCK_SECTION_NAME ":\n\t" \
71 ".endif\n"
72
73#define LOCK_SECTION_END \
74 ".previous\n\t"
75
Harvey Harrisonec701582008-02-08 04:19:55 -080076#define __lockfunc __attribute__((section(".spinlock.text")))
Linus Torvalds1da177e2005-04-16 15:20:36 -070077
78/*
Thomas Gleixnerfb3a6bb2009-12-03 20:01:19 +010079 * Pull the arch_spinlock_t and arch_rwlock_t definitions:
Linus Torvalds1da177e2005-04-16 15:20:36 -070080 */
Ingo Molnarfb1c8f92005-09-10 00:25:56 -070081#include <linux/spinlock_types.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070082
Ingo Molnarfb1c8f92005-09-10 00:25:56 -070083/*
Lucas De Marchi25985ed2011-03-30 22:57:33 -030084 * Pull the arch_spin*() functions/declarations (UP-nondebug doesn't need them):
Ingo Molnarfb1c8f92005-09-10 00:25:56 -070085 */
Ingo Molnar8a25d5d2006-07-03 00:24:54 -070086#ifdef CONFIG_SMP
Ingo Molnarfb1c8f92005-09-10 00:25:56 -070087# include <asm/spinlock.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070088#else
Ingo Molnarfb1c8f92005-09-10 00:25:56 -070089# include <linux/spinlock_up.h>
90#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -070091
Ingo Molnar8a25d5d2006-07-03 00:24:54 -070092#ifdef CONFIG_DEBUG_SPINLOCK
Thomas Gleixnerc2f21ce2009-12-02 20:02:59 +010093 extern void __raw_spin_lock_init(raw_spinlock_t *lock, const char *name,
94 struct lock_class_key *key);
95# define raw_spin_lock_init(lock) \
Ingo Molnar8a25d5d2006-07-03 00:24:54 -070096do { \
97 static struct lock_class_key __key; \
98 \
Thomas Gleixnerc2f21ce2009-12-02 20:02:59 +010099 __raw_spin_lock_init((lock), #lock, &__key); \
Ingo Molnar8a25d5d2006-07-03 00:24:54 -0700100} while (0)
101
102#else
Thomas Gleixnerc2f21ce2009-12-02 20:02:59 +0100103# define raw_spin_lock_init(lock) \
104 do { *(lock) = __RAW_SPIN_LOCK_UNLOCKED(lock); } while (0)
Ingo Molnar8a25d5d2006-07-03 00:24:54 -0700105#endif
106
Thomas Gleixnerc2f21ce2009-12-02 20:02:59 +0100107#define raw_spin_is_locked(lock) arch_spin_is_locked(&(lock)->raw_lock)
Ingo Molnarfb1c8f92005-09-10 00:25:56 -0700108
Nick Piggin95c354f2008-01-30 13:31:20 +0100109#ifdef CONFIG_GENERIC_LOCKBREAK
Thomas Gleixnerc2f21ce2009-12-02 20:02:59 +0100110#define raw_spin_is_contended(lock) ((lock)->break_lock)
Nick Piggin95c354f2008-01-30 13:31:20 +0100111#else
Kyle McMartina5ef7ca2009-02-08 17:39:58 -0500112
Thomas Gleixner0199c4e2009-12-02 20:01:25 +0100113#ifdef arch_spin_is_contended
Thomas Gleixnerc2f21ce2009-12-02 20:02:59 +0100114#define raw_spin_is_contended(lock) arch_spin_is_contended(&(lock)->raw_lock)
Kyle McMartina5ef7ca2009-02-08 17:39:58 -0500115#else
Thomas Gleixnerc2f21ce2009-12-02 20:02:59 +0100116#define raw_spin_is_contended(lock) (((void)(lock), 0))
Thomas Gleixner0199c4e2009-12-02 20:01:25 +0100117#endif /*arch_spin_is_contended*/
Nick Piggin95c354f2008-01-30 13:31:20 +0100118#endif
119
Jiri Olsaad462762009-07-08 12:10:31 +0000120/* The lock does not imply full memory barrier. */
121#ifndef ARCH_HAS_SMP_MB_AFTER_LOCK
122static inline void smp_mb__after_lock(void) { smp_mb(); }
123#endif
124
Ingo Molnarfb1c8f92005-09-10 00:25:56 -0700125/**
Thomas Gleixnerc2f21ce2009-12-02 20:02:59 +0100126 * raw_spin_unlock_wait - wait until the spinlock gets unlocked
Ingo Molnarfb1c8f92005-09-10 00:25:56 -0700127 * @lock: the spinlock in question.
128 */
Thomas Gleixnerc2f21ce2009-12-02 20:02:59 +0100129#define raw_spin_unlock_wait(lock) arch_spin_unlock_wait(&(lock)->raw_lock)
Ingo Molnarfb1c8f92005-09-10 00:25:56 -0700130
Linus Torvalds1da177e2005-04-16 15:20:36 -0700131#ifdef CONFIG_DEBUG_SPINLOCK
Luca Barbierib97c4bc2010-03-11 14:08:45 -0800132 extern void do_raw_spin_lock(raw_spinlock_t *lock) __acquires(lock);
Thomas Gleixner9828ea92009-12-03 20:55:53 +0100133#define do_raw_spin_lock_flags(lock, flags) do_raw_spin_lock(lock)
134 extern int do_raw_spin_trylock(raw_spinlock_t *lock);
Luca Barbierib97c4bc2010-03-11 14:08:45 -0800135 extern void do_raw_spin_unlock(raw_spinlock_t *lock) __releases(lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700136#else
Luca Barbierib97c4bc2010-03-11 14:08:45 -0800137static inline void do_raw_spin_lock(raw_spinlock_t *lock) __acquires(lock)
Thomas Gleixnerc2f21ce2009-12-02 20:02:59 +0100138{
Luca Barbierib97c4bc2010-03-11 14:08:45 -0800139 __acquire(lock);
Thomas Gleixnerc2f21ce2009-12-02 20:02:59 +0100140 arch_spin_lock(&lock->raw_lock);
141}
142
143static inline void
Luca Barbierib97c4bc2010-03-11 14:08:45 -0800144do_raw_spin_lock_flags(raw_spinlock_t *lock, unsigned long *flags) __acquires(lock)
Thomas Gleixnerc2f21ce2009-12-02 20:02:59 +0100145{
Luca Barbierib97c4bc2010-03-11 14:08:45 -0800146 __acquire(lock);
Thomas Gleixnerc2f21ce2009-12-02 20:02:59 +0100147 arch_spin_lock_flags(&lock->raw_lock, *flags);
148}
149
Thomas Gleixner9828ea92009-12-03 20:55:53 +0100150static inline int do_raw_spin_trylock(raw_spinlock_t *lock)
Thomas Gleixnerc2f21ce2009-12-02 20:02:59 +0100151{
152 return arch_spin_trylock(&(lock)->raw_lock);
153}
154
Luca Barbierib97c4bc2010-03-11 14:08:45 -0800155static inline void do_raw_spin_unlock(raw_spinlock_t *lock) __releases(lock)
Thomas Gleixnerc2f21ce2009-12-02 20:02:59 +0100156{
157 arch_spin_unlock(&lock->raw_lock);
Luca Barbierib97c4bc2010-03-11 14:08:45 -0800158 __release(lock);
Thomas Gleixnerc2f21ce2009-12-02 20:02:59 +0100159}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700160#endif
161
Linus Torvalds1da177e2005-04-16 15:20:36 -0700162/*
Thomas Gleixneref12f102009-11-07 23:04:15 +0100163 * Define the various spin_lock methods. Note we define these
164 * regardless of whether CONFIG_SMP or CONFIG_PREEMPT are set. The
165 * various methods are defined as nops in the case they are not
166 * required.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700167 */
Thomas Gleixner9c1721a2009-12-03 21:52:18 +0100168#define raw_spin_trylock(lock) __cond_lock(lock, _raw_spin_trylock(lock))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700169
Thomas Gleixner9c1721a2009-12-03 21:52:18 +0100170#define raw_spin_lock(lock) _raw_spin_lock(lock)
Ingo Molnar8a25d5d2006-07-03 00:24:54 -0700171
172#ifdef CONFIG_DEBUG_LOCK_ALLOC
Thomas Gleixner9c1721a2009-12-03 21:52:18 +0100173# define raw_spin_lock_nested(lock, subclass) \
174 _raw_spin_lock_nested(lock, subclass)
175
Thomas Gleixnerc2f21ce2009-12-02 20:02:59 +0100176# define raw_spin_lock_nest_lock(lock, nest_lock) \
Peter Zijlstrab7d39af2008-08-11 09:30:24 +0200177 do { \
178 typecheck(struct lockdep_map *, &(nest_lock)->dep_map);\
Thomas Gleixner9c1721a2009-12-03 21:52:18 +0100179 _raw_spin_lock_nest_lock(lock, &(nest_lock)->dep_map); \
Peter Zijlstrab7d39af2008-08-11 09:30:24 +0200180 } while (0)
Ingo Molnar8a25d5d2006-07-03 00:24:54 -0700181#else
Thomas Gleixner9c1721a2009-12-03 21:52:18 +0100182# define raw_spin_lock_nested(lock, subclass) _raw_spin_lock(lock)
183# define raw_spin_lock_nest_lock(lock, nest_lock) _raw_spin_lock(lock)
Ingo Molnar8a25d5d2006-07-03 00:24:54 -0700184#endif
185
Ingo Molnarfb1c8f92005-09-10 00:25:56 -0700186#if defined(CONFIG_SMP) || defined(CONFIG_DEBUG_SPINLOCK)
Linus Torvaldsb8e6ec82006-11-26 16:27:17 -0800187
Thomas Gleixnerc2f21ce2009-12-02 20:02:59 +0100188#define raw_spin_lock_irqsave(lock, flags) \
Steven Rostedt3f3078912008-07-25 01:45:25 -0700189 do { \
190 typecheck(unsigned long, flags); \
Thomas Gleixner9c1721a2009-12-03 21:52:18 +0100191 flags = _raw_spin_lock_irqsave(lock); \
Steven Rostedt3f3078912008-07-25 01:45:25 -0700192 } while (0)
Arjan van de Vencfd3ef22006-11-25 11:09:37 -0800193
194#ifdef CONFIG_DEBUG_LOCK_ALLOC
Thomas Gleixnerc2f21ce2009-12-02 20:02:59 +0100195#define raw_spin_lock_irqsave_nested(lock, flags, subclass) \
Steven Rostedt3f3078912008-07-25 01:45:25 -0700196 do { \
197 typecheck(unsigned long, flags); \
Thomas Gleixner9c1721a2009-12-03 21:52:18 +0100198 flags = _raw_spin_lock_irqsave_nested(lock, subclass); \
Steven Rostedt3f3078912008-07-25 01:45:25 -0700199 } while (0)
Arjan van de Vencfd3ef22006-11-25 11:09:37 -0800200#else
Thomas Gleixnerc2f21ce2009-12-02 20:02:59 +0100201#define raw_spin_lock_irqsave_nested(lock, flags, subclass) \
Steven Rostedt3f3078912008-07-25 01:45:25 -0700202 do { \
203 typecheck(unsigned long, flags); \
Thomas Gleixner9c1721a2009-12-03 21:52:18 +0100204 flags = _raw_spin_lock_irqsave(lock); \
Steven Rostedt3f3078912008-07-25 01:45:25 -0700205 } while (0)
Arjan van de Vencfd3ef22006-11-25 11:09:37 -0800206#endif
207
Linus Torvalds1da177e2005-04-16 15:20:36 -0700208#else
Linus Torvaldsb8e6ec82006-11-26 16:27:17 -0800209
Thomas Gleixnerc2f21ce2009-12-02 20:02:59 +0100210#define raw_spin_lock_irqsave(lock, flags) \
Steven Rostedt3f3078912008-07-25 01:45:25 -0700211 do { \
212 typecheck(unsigned long, flags); \
Thomas Gleixner9c1721a2009-12-03 21:52:18 +0100213 _raw_spin_lock_irqsave(lock, flags); \
Steven Rostedt3f3078912008-07-25 01:45:25 -0700214 } while (0)
Thomas Gleixneref12f102009-11-07 23:04:15 +0100215
Thomas Gleixnerc2f21ce2009-12-02 20:02:59 +0100216#define raw_spin_lock_irqsave_nested(lock, flags, subclass) \
217 raw_spin_lock_irqsave(lock, flags)
Arjan van de Vencfd3ef22006-11-25 11:09:37 -0800218
Linus Torvalds1da177e2005-04-16 15:20:36 -0700219#endif
220
Thomas Gleixner9c1721a2009-12-03 21:52:18 +0100221#define raw_spin_lock_irq(lock) _raw_spin_lock_irq(lock)
222#define raw_spin_lock_bh(lock) _raw_spin_lock_bh(lock)
223#define raw_spin_unlock(lock) _raw_spin_unlock(lock)
224#define raw_spin_unlock_irq(lock) _raw_spin_unlock_irq(lock)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700225
Thomas Gleixnerc2f21ce2009-12-02 20:02:59 +0100226#define raw_spin_unlock_irqrestore(lock, flags) \
227 do { \
228 typecheck(unsigned long, flags); \
Thomas Gleixner9c1721a2009-12-03 21:52:18 +0100229 _raw_spin_unlock_irqrestore(lock, flags); \
Steven Rostedt3f3078912008-07-25 01:45:25 -0700230 } while (0)
Thomas Gleixner9c1721a2009-12-03 21:52:18 +0100231#define raw_spin_unlock_bh(lock) _raw_spin_unlock_bh(lock)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700232
Thomas Gleixner9c1721a2009-12-03 21:52:18 +0100233#define raw_spin_trylock_bh(lock) \
234 __cond_lock(lock, _raw_spin_trylock_bh(lock))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700235
Thomas Gleixnerc2f21ce2009-12-02 20:02:59 +0100236#define raw_spin_trylock_irq(lock) \
Linus Torvalds1da177e2005-04-16 15:20:36 -0700237({ \
238 local_irq_disable(); \
Thomas Gleixnerc2f21ce2009-12-02 20:02:59 +0100239 raw_spin_trylock(lock) ? \
Ingo Molnarfb1c8f92005-09-10 00:25:56 -0700240 1 : ({ local_irq_enable(); 0; }); \
Linus Torvalds1da177e2005-04-16 15:20:36 -0700241})
242
Thomas Gleixnerc2f21ce2009-12-02 20:02:59 +0100243#define raw_spin_trylock_irqsave(lock, flags) \
Linus Torvalds1da177e2005-04-16 15:20:36 -0700244({ \
245 local_irq_save(flags); \
Thomas Gleixnerc2f21ce2009-12-02 20:02:59 +0100246 raw_spin_trylock(lock) ? \
Ingo Molnarfb1c8f92005-09-10 00:25:56 -0700247 1 : ({ local_irq_restore(flags); 0; }); \
Linus Torvalds1da177e2005-04-16 15:20:36 -0700248})
249
Thomas Gleixnerc2f21ce2009-12-02 20:02:59 +0100250/**
251 * raw_spin_can_lock - would raw_spin_trylock() succeed?
252 * @lock: the spinlock in question.
253 */
254#define raw_spin_can_lock(lock) (!raw_spin_is_locked(lock))
255
256/* Include rwlock functions */
257#include <linux/rwlock.h>
258
259/*
260 * Pull the _spin_*()/_read_*()/_write_*() functions/declarations:
261 */
262#if defined(CONFIG_SMP) || defined(CONFIG_DEBUG_SPINLOCK)
263# include <linux/spinlock_api_smp.h>
264#else
265# include <linux/spinlock_api_up.h>
266#endif
267
268/*
269 * Map the spin_lock functions to the raw variants for PREEMPT_RT=n
270 */
271
272static inline raw_spinlock_t *spinlock_check(spinlock_t *lock)
273{
274 return &lock->rlock;
275}
276
277#define spin_lock_init(_lock) \
278do { \
279 spinlock_check(_lock); \
280 raw_spin_lock_init(&(_lock)->rlock); \
281} while (0)
282
283static inline void spin_lock(spinlock_t *lock)
284{
285 raw_spin_lock(&lock->rlock);
286}
287
288static inline void spin_lock_bh(spinlock_t *lock)
289{
290 raw_spin_lock_bh(&lock->rlock);
291}
292
293static inline int spin_trylock(spinlock_t *lock)
294{
295 return raw_spin_trylock(&lock->rlock);
296}
297
298#define spin_lock_nested(lock, subclass) \
299do { \
300 raw_spin_lock_nested(spinlock_check(lock), subclass); \
301} while (0)
302
303#define spin_lock_nest_lock(lock, nest_lock) \
304do { \
305 raw_spin_lock_nest_lock(spinlock_check(lock), nest_lock); \
306} while (0)
307
308static inline void spin_lock_irq(spinlock_t *lock)
309{
310 raw_spin_lock_irq(&lock->rlock);
311}
312
313#define spin_lock_irqsave(lock, flags) \
314do { \
315 raw_spin_lock_irqsave(spinlock_check(lock), flags); \
316} while (0)
317
318#define spin_lock_irqsave_nested(lock, flags, subclass) \
319do { \
320 raw_spin_lock_irqsave_nested(spinlock_check(lock), flags, subclass); \
321} while (0)
322
323static inline void spin_unlock(spinlock_t *lock)
324{
325 raw_spin_unlock(&lock->rlock);
326}
327
328static inline void spin_unlock_bh(spinlock_t *lock)
329{
330 raw_spin_unlock_bh(&lock->rlock);
331}
332
333static inline void spin_unlock_irq(spinlock_t *lock)
334{
335 raw_spin_unlock_irq(&lock->rlock);
336}
337
338static inline void spin_unlock_irqrestore(spinlock_t *lock, unsigned long flags)
339{
340 raw_spin_unlock_irqrestore(&lock->rlock, flags);
341}
342
343static inline int spin_trylock_bh(spinlock_t *lock)
344{
345 return raw_spin_trylock_bh(&lock->rlock);
346}
347
348static inline int spin_trylock_irq(spinlock_t *lock)
349{
350 return raw_spin_trylock_irq(&lock->rlock);
351}
352
353#define spin_trylock_irqsave(lock, flags) \
354({ \
355 raw_spin_trylock_irqsave(spinlock_check(lock), flags); \
356})
357
358static inline void spin_unlock_wait(spinlock_t *lock)
359{
360 raw_spin_unlock_wait(&lock->rlock);
361}
362
363static inline int spin_is_locked(spinlock_t *lock)
364{
365 return raw_spin_is_locked(&lock->rlock);
366}
367
368static inline int spin_is_contended(spinlock_t *lock)
369{
370 return raw_spin_is_contended(&lock->rlock);
371}
372
373static inline int spin_can_lock(spinlock_t *lock)
374{
375 return raw_spin_can_lock(&lock->rlock);
376}
377
Paul Gortmaker4ebc1b42012-01-20 18:20:37 -0500378#define assert_spin_locked(lock) assert_raw_spin_locked(&(lock)->rlock)
Thomas Gleixnerc2f21ce2009-12-02 20:02:59 +0100379
Ingo Molnarfb1c8f92005-09-10 00:25:56 -0700380/*
381 * Pull the atomic_t declaration:
382 * (asm-mips/atomic.h needs above definitions)
383 */
Arun Sharma600634972011-07-26 16:09:06 -0700384#include <linux/atomic.h>
Ingo Molnarfb1c8f92005-09-10 00:25:56 -0700385/**
386 * atomic_dec_and_lock - lock on reaching reference count zero
387 * @atomic: the atomic counter
388 * @lock: the spinlock in question
J. Bruce Fieldsdc07e722008-04-07 15:59:05 -0400389 *
390 * Decrements @atomic by 1. If the result is 0, returns true and locks
391 * @lock. Returns false for all other cases.
Ingo Molnarfb1c8f92005-09-10 00:25:56 -0700392 */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700393extern int _atomic_dec_and_lock(atomic_t *atomic, spinlock_t *lock);
Ingo Molnarfb1c8f92005-09-10 00:25:56 -0700394#define atomic_dec_and_lock(atomic, lock) \
Josh Triplettdcc8e552006-09-29 02:01:03 -0700395 __cond_lock(lock, _atomic_dec_and_lock(atomic, lock))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700396
Linus Torvalds1da177e2005-04-16 15:20:36 -0700397#endif /* __LINUX_SPINLOCK_H */