blob: 5344268e6e62fe7db66e2e305829309e968c4d2f [file] [log] [blame]
Ingo Molnarfb1c8f92005-09-10 00:25:56 -07001#ifndef __LINUX_SPINLOCK_API_SMP_H
2#define __LINUX_SPINLOCK_API_SMP_H
3
4#ifndef __LINUX_SPINLOCK_H
5# error "please don't include this file directly"
6#endif
7
8/*
9 * include/linux/spinlock_api_smp.h
10 *
11 * spinlock API declarations on SMP (and debug)
12 * (implemented in kernel/spinlock.c)
13 *
14 * portions Copyright 2005, Red Hat, Inc., Ingo Molnar
15 * Released under the General Public License (GPL).
16 */
17
18int in_lock_functions(unsigned long addr);
19
Thomas Gleixnerc2f21ce2009-12-02 20:02:59 +010020#define assert_raw_spin_locked(x) BUG_ON(!raw_spin_is_locked(x))
Ingo Molnarfb1c8f92005-09-10 00:25:56 -070021
Thomas Gleixner9c1721a2009-12-03 21:52:18 +010022void __lockfunc _raw_spin_lock(raw_spinlock_t *lock) __acquires(lock);
23void __lockfunc _raw_spin_lock_nested(raw_spinlock_t *lock, int subclass)
24 __acquires(lock);
Thomas Graf113948d2015-01-02 23:00:19 +010025void __lockfunc _raw_spin_lock_bh_nested(raw_spinlock_t *lock, int subclass)
26 __acquires(lock);
Thomas Gleixnerc2f21ce2009-12-02 20:02:59 +010027void __lockfunc
Thomas Gleixner9c1721a2009-12-03 21:52:18 +010028_raw_spin_lock_nest_lock(raw_spinlock_t *lock, struct lockdep_map *map)
29 __acquires(lock);
30void __lockfunc _raw_spin_lock_bh(raw_spinlock_t *lock) __acquires(lock);
31void __lockfunc _raw_spin_lock_irq(raw_spinlock_t *lock)
32 __acquires(lock);
Thomas Gleixner6b6b4792009-11-16 18:48:37 +010033
Thomas Gleixner9c1721a2009-12-03 21:52:18 +010034unsigned long __lockfunc _raw_spin_lock_irqsave(raw_spinlock_t *lock)
35 __acquires(lock);
Thomas Gleixnerc2f21ce2009-12-02 20:02:59 +010036unsigned long __lockfunc
Thomas Gleixner9c1721a2009-12-03 21:52:18 +010037_raw_spin_lock_irqsave_nested(raw_spinlock_t *lock, int subclass)
38 __acquires(lock);
39int __lockfunc _raw_spin_trylock(raw_spinlock_t *lock);
40int __lockfunc _raw_spin_trylock_bh(raw_spinlock_t *lock);
41void __lockfunc _raw_spin_unlock(raw_spinlock_t *lock) __releases(lock);
42void __lockfunc _raw_spin_unlock_bh(raw_spinlock_t *lock) __releases(lock);
43void __lockfunc _raw_spin_unlock_irq(raw_spinlock_t *lock) __releases(lock);
Thomas Gleixnerc2f21ce2009-12-02 20:02:59 +010044void __lockfunc
Thomas Gleixner9c1721a2009-12-03 21:52:18 +010045_raw_spin_unlock_irqrestore(raw_spinlock_t *lock, unsigned long flags)
46 __releases(lock);
Ingo Molnarfb1c8f92005-09-10 00:25:56 -070047
Thomas Gleixner6beb0002009-11-09 15:21:34 +000048#ifdef CONFIG_INLINE_SPIN_LOCK
Thomas Gleixner9c1721a2009-12-03 21:52:18 +010049#define _raw_spin_lock(lock) __raw_spin_lock(lock)
Heiko Carstens892a7c62009-08-31 14:43:37 +020050#endif
51
Thomas Gleixner6beb0002009-11-09 15:21:34 +000052#ifdef CONFIG_INLINE_SPIN_LOCK_BH
Thomas Gleixner9c1721a2009-12-03 21:52:18 +010053#define _raw_spin_lock_bh(lock) __raw_spin_lock_bh(lock)
Heiko Carstens892a7c62009-08-31 14:43:37 +020054#endif
55
Thomas Gleixner6beb0002009-11-09 15:21:34 +000056#ifdef CONFIG_INLINE_SPIN_LOCK_IRQ
Thomas Gleixner9c1721a2009-12-03 21:52:18 +010057#define _raw_spin_lock_irq(lock) __raw_spin_lock_irq(lock)
Heiko Carstens892a7c62009-08-31 14:43:37 +020058#endif
59
Thomas Gleixner6beb0002009-11-09 15:21:34 +000060#ifdef CONFIG_INLINE_SPIN_LOCK_IRQSAVE
Thomas Gleixner9c1721a2009-12-03 21:52:18 +010061#define _raw_spin_lock_irqsave(lock) __raw_spin_lock_irqsave(lock)
Heiko Carstens892a7c62009-08-31 14:43:37 +020062#endif
63
Thomas Gleixner6beb0002009-11-09 15:21:34 +000064#ifdef CONFIG_INLINE_SPIN_TRYLOCK
Thomas Gleixner9c1721a2009-12-03 21:52:18 +010065#define _raw_spin_trylock(lock) __raw_spin_trylock(lock)
Heiko Carstens892a7c62009-08-31 14:43:37 +020066#endif
67
Thomas Gleixner6beb0002009-11-09 15:21:34 +000068#ifdef CONFIG_INLINE_SPIN_TRYLOCK_BH
Thomas Gleixner9c1721a2009-12-03 21:52:18 +010069#define _raw_spin_trylock_bh(lock) __raw_spin_trylock_bh(lock)
Heiko Carstens892a7c62009-08-31 14:43:37 +020070#endif
71
Raghavendra K Te335e3e2012-03-22 15:25:08 +053072#ifndef CONFIG_UNINLINE_SPIN_UNLOCK
Thomas Gleixner9c1721a2009-12-03 21:52:18 +010073#define _raw_spin_unlock(lock) __raw_spin_unlock(lock)
Heiko Carstens892a7c62009-08-31 14:43:37 +020074#endif
75
Thomas Gleixner6beb0002009-11-09 15:21:34 +000076#ifdef CONFIG_INLINE_SPIN_UNLOCK_BH
Thomas Gleixner9c1721a2009-12-03 21:52:18 +010077#define _raw_spin_unlock_bh(lock) __raw_spin_unlock_bh(lock)
Heiko Carstens892a7c62009-08-31 14:43:37 +020078#endif
79
Thomas Gleixner6beb0002009-11-09 15:21:34 +000080#ifdef CONFIG_INLINE_SPIN_UNLOCK_IRQ
Thomas Gleixner9c1721a2009-12-03 21:52:18 +010081#define _raw_spin_unlock_irq(lock) __raw_spin_unlock_irq(lock)
Heiko Carstens892a7c62009-08-31 14:43:37 +020082#endif
83
Thomas Gleixner6beb0002009-11-09 15:21:34 +000084#ifdef CONFIG_INLINE_SPIN_UNLOCK_IRQRESTORE
Thomas Gleixner9c1721a2009-12-03 21:52:18 +010085#define _raw_spin_unlock_irqrestore(lock, flags) __raw_spin_unlock_irqrestore(lock, flags)
Heiko Carstens892a7c62009-08-31 14:43:37 +020086#endif
87
Thomas Gleixner9c1721a2009-12-03 21:52:18 +010088static inline int __raw_spin_trylock(raw_spinlock_t *lock)
Heiko Carstens69d0ee72009-08-31 14:43:36 +020089{
90 preempt_disable();
Thomas Gleixner9828ea92009-12-03 20:55:53 +010091 if (do_raw_spin_trylock(lock)) {
Heiko Carstens69d0ee72009-08-31 14:43:36 +020092 spin_acquire(&lock->dep_map, 0, 1, _RET_IP_);
93 return 1;
94 }
95 preempt_enable();
96 return 0;
97}
98
Heiko Carstens69d0ee72009-08-31 14:43:36 +020099/*
100 * If lockdep is enabled then we use the non-preemption spin-ops
101 * even on CONFIG_PREEMPT, because lockdep assumes that interrupts are
102 * not re-enabled during lock-acquire (which the preempt-spin-ops do):
103 */
104#if !defined(CONFIG_GENERIC_LOCKBREAK) || defined(CONFIG_DEBUG_LOCK_ALLOC)
105
Thomas Gleixner9c1721a2009-12-03 21:52:18 +0100106static inline unsigned long __raw_spin_lock_irqsave(raw_spinlock_t *lock)
Heiko Carstens69d0ee72009-08-31 14:43:36 +0200107{
108 unsigned long flags;
109
110 local_irq_save(flags);
111 preempt_disable();
112 spin_acquire(&lock->dep_map, 0, 0, _RET_IP_);
113 /*
114 * On lockdep we dont want the hand-coded irq-enable of
Thomas Gleixner9828ea92009-12-03 20:55:53 +0100115 * do_raw_spin_lock_flags() code, because lockdep assumes
Heiko Carstens69d0ee72009-08-31 14:43:36 +0200116 * that interrupts are not re-enabled during lock-acquire:
117 */
118#ifdef CONFIG_LOCKDEP
Thomas Gleixner9828ea92009-12-03 20:55:53 +0100119 LOCK_CONTENDED(lock, do_raw_spin_trylock, do_raw_spin_lock);
Heiko Carstens69d0ee72009-08-31 14:43:36 +0200120#else
Thomas Gleixner9828ea92009-12-03 20:55:53 +0100121 do_raw_spin_lock_flags(lock, &flags);
Heiko Carstens69d0ee72009-08-31 14:43:36 +0200122#endif
123 return flags;
124}
125
Thomas Gleixner9c1721a2009-12-03 21:52:18 +0100126static inline void __raw_spin_lock_irq(raw_spinlock_t *lock)
Heiko Carstens69d0ee72009-08-31 14:43:36 +0200127{
128 local_irq_disable();
129 preempt_disable();
130 spin_acquire(&lock->dep_map, 0, 0, _RET_IP_);
Thomas Gleixner9828ea92009-12-03 20:55:53 +0100131 LOCK_CONTENDED(lock, do_raw_spin_trylock, do_raw_spin_lock);
Heiko Carstens69d0ee72009-08-31 14:43:36 +0200132}
133
Thomas Gleixner9c1721a2009-12-03 21:52:18 +0100134static inline void __raw_spin_lock_bh(raw_spinlock_t *lock)
Heiko Carstens69d0ee72009-08-31 14:43:36 +0200135{
Peter Zijlstra9ea4c382013-11-19 16:13:38 +0100136 __local_bh_disable_ip(_RET_IP_, SOFTIRQ_LOCK_OFFSET);
Heiko Carstens69d0ee72009-08-31 14:43:36 +0200137 spin_acquire(&lock->dep_map, 0, 0, _RET_IP_);
Thomas Gleixner9828ea92009-12-03 20:55:53 +0100138 LOCK_CONTENDED(lock, do_raw_spin_trylock, do_raw_spin_lock);
Heiko Carstens69d0ee72009-08-31 14:43:36 +0200139}
140
Thomas Gleixner9c1721a2009-12-03 21:52:18 +0100141static inline void __raw_spin_lock(raw_spinlock_t *lock)
Heiko Carstens69d0ee72009-08-31 14:43:36 +0200142{
143 preempt_disable();
144 spin_acquire(&lock->dep_map, 0, 0, _RET_IP_);
Thomas Gleixner9828ea92009-12-03 20:55:53 +0100145 LOCK_CONTENDED(lock, do_raw_spin_trylock, do_raw_spin_lock);
Heiko Carstens69d0ee72009-08-31 14:43:36 +0200146}
147
Chen Gangae584032013-06-19 11:15:46 +0800148#endif /* !CONFIG_GENERIC_LOCKBREAK || CONFIG_DEBUG_LOCK_ALLOC */
Heiko Carstens69d0ee72009-08-31 14:43:36 +0200149
Thomas Gleixner9c1721a2009-12-03 21:52:18 +0100150static inline void __raw_spin_unlock(raw_spinlock_t *lock)
Heiko Carstens69d0ee72009-08-31 14:43:36 +0200151{
152 spin_release(&lock->dep_map, 1, _RET_IP_);
Thomas Gleixner9828ea92009-12-03 20:55:53 +0100153 do_raw_spin_unlock(lock);
Heiko Carstens69d0ee72009-08-31 14:43:36 +0200154 preempt_enable();
155}
156
Thomas Gleixner9c1721a2009-12-03 21:52:18 +0100157static inline void __raw_spin_unlock_irqrestore(raw_spinlock_t *lock,
Heiko Carstens69d0ee72009-08-31 14:43:36 +0200158 unsigned long flags)
159{
160 spin_release(&lock->dep_map, 1, _RET_IP_);
Thomas Gleixner9828ea92009-12-03 20:55:53 +0100161 do_raw_spin_unlock(lock);
Heiko Carstens69d0ee72009-08-31 14:43:36 +0200162 local_irq_restore(flags);
163 preempt_enable();
164}
165
Thomas Gleixner9c1721a2009-12-03 21:52:18 +0100166static inline void __raw_spin_unlock_irq(raw_spinlock_t *lock)
Heiko Carstens69d0ee72009-08-31 14:43:36 +0200167{
168 spin_release(&lock->dep_map, 1, _RET_IP_);
Thomas Gleixner9828ea92009-12-03 20:55:53 +0100169 do_raw_spin_unlock(lock);
Heiko Carstens69d0ee72009-08-31 14:43:36 +0200170 local_irq_enable();
171 preempt_enable();
172}
173
Thomas Gleixner9c1721a2009-12-03 21:52:18 +0100174static inline void __raw_spin_unlock_bh(raw_spinlock_t *lock)
Heiko Carstens69d0ee72009-08-31 14:43:36 +0200175{
176 spin_release(&lock->dep_map, 1, _RET_IP_);
Thomas Gleixner9828ea92009-12-03 20:55:53 +0100177 do_raw_spin_unlock(lock);
Peter Zijlstra9ea4c382013-11-19 16:13:38 +0100178 __local_bh_enable_ip(_RET_IP_, SOFTIRQ_LOCK_OFFSET);
Heiko Carstens69d0ee72009-08-31 14:43:36 +0200179}
180
Thomas Gleixner9c1721a2009-12-03 21:52:18 +0100181static inline int __raw_spin_trylock_bh(raw_spinlock_t *lock)
Heiko Carstens69d0ee72009-08-31 14:43:36 +0200182{
Peter Zijlstra9ea4c382013-11-19 16:13:38 +0100183 __local_bh_disable_ip(_RET_IP_, SOFTIRQ_LOCK_OFFSET);
Thomas Gleixner9828ea92009-12-03 20:55:53 +0100184 if (do_raw_spin_trylock(lock)) {
Heiko Carstens69d0ee72009-08-31 14:43:36 +0200185 spin_acquire(&lock->dep_map, 0, 1, _RET_IP_);
186 return 1;
187 }
Peter Zijlstra9ea4c382013-11-19 16:13:38 +0100188 __local_bh_enable_ip(_RET_IP_, SOFTIRQ_LOCK_OFFSET);
Heiko Carstens69d0ee72009-08-31 14:43:36 +0200189 return 0;
190}
191
Thomas Gleixner6b6b4792009-11-16 18:48:37 +0100192#include <linux/rwlock_api_smp.h>
193
Ingo Molnarfb1c8f92005-09-10 00:25:56 -0700194#endif /* __LINUX_SPINLOCK_API_SMP_H */