blob: d816c62a7a1d1d9fe94ea16c1a7ebf2747a6e0ac [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001#ifndef __ASM_SPINLOCK_H
2#define __ASM_SPINLOCK_H
3
4#include <asm/atomic.h>
5#include <asm/rwlock.h>
6#include <asm/page.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -07007#include <linux/compiler.h>
8
Linus Torvalds1da177e2005-04-16 15:20:36 -07009/*
10 * Your basic SMP spinlocks, allowing only a single CPU anywhere
Ingo Molnarfb1c8f92005-09-10 00:25:56 -070011 *
Linus Torvalds1da177e2005-04-16 15:20:36 -070012 * Simple spin lock operations. There are two variants, one clears IRQ's
13 * on the local processor, one does not.
14 *
15 * We make no fairness assumptions. They have a cost.
Ingo Molnarfb1c8f92005-09-10 00:25:56 -070016 *
17 * (the type definitions are in asm/spinlock_types.h)
Linus Torvalds1da177e2005-04-16 15:20:36 -070018 */
19
Ingo Molnarfb1c8f92005-09-10 00:25:56 -070020#define __raw_spin_is_locked(x) \
21 (*(volatile signed char *)(&(x)->slock) <= 0)
Linus Torvalds1da177e2005-04-16 15:20:36 -070022
Ingo Molnarfb1c8f92005-09-10 00:25:56 -070023#define __raw_spin_lock_string \
Linus Torvalds1da177e2005-04-16 15:20:36 -070024 "\n1:\t" \
25 "lock ; decb %0\n\t" \
26 "jns 3f\n" \
27 "2:\t" \
28 "rep;nop\n\t" \
29 "cmpb $0,%0\n\t" \
30 "jle 2b\n\t" \
31 "jmp 1b\n" \
32 "3:\n\t"
33
Ingo Molnar55f327f2006-07-03 00:24:43 -070034/*
35 * NOTE: there's an irqs-on section here, which normally would have to be
36 * irq-traced, but on CONFIG_TRACE_IRQFLAGS we never use
37 * __raw_spin_lock_string_flags().
38 */
Ingo Molnarfb1c8f92005-09-10 00:25:56 -070039#define __raw_spin_lock_string_flags \
Linus Torvalds1da177e2005-04-16 15:20:36 -070040 "\n1:\t" \
41 "lock ; decb %0\n\t" \
Chuck Ebbert42c059e02006-03-23 02:59:55 -080042 "jns 5f\n" \
Linus Torvalds1da177e2005-04-16 15:20:36 -070043 "2:\t" \
44 "testl $0x200, %1\n\t" \
Chuck Ebbert42c059e02006-03-23 02:59:55 -080045 "jz 4f\n\t" \
46 "sti\n" \
Linus Torvalds1da177e2005-04-16 15:20:36 -070047 "3:\t" \
48 "rep;nop\n\t" \
49 "cmpb $0, %0\n\t" \
50 "jle 3b\n\t" \
51 "cli\n\t" \
52 "jmp 1b\n" \
Chuck Ebbert42c059e02006-03-23 02:59:55 -080053 "4:\t" \
54 "rep;nop\n\t" \
55 "cmpb $0, %0\n\t" \
56 "jg 1b\n\t" \
57 "jmp 4b\n" \
58 "5:\n\t"
Linus Torvalds1da177e2005-04-16 15:20:36 -070059
Gerd Hoffmann9a0b5812006-03-23 02:59:32 -080060#define __raw_spin_lock_string_up \
61 "\n\tdecb %0"
62
Ingo Molnarfb1c8f92005-09-10 00:25:56 -070063static inline void __raw_spin_lock(raw_spinlock_t *lock)
Linus Torvalds1da177e2005-04-16 15:20:36 -070064{
Gerd Hoffmann9a0b5812006-03-23 02:59:32 -080065 alternative_smp(
66 __raw_spin_lock_string,
67 __raw_spin_lock_string_up,
Linus Torvaldsb862f3b2006-07-08 15:24:18 -070068 "+m" (lock->slock) : : "memory");
Linus Torvalds1da177e2005-04-16 15:20:36 -070069}
70
Ingo Molnar8a25d5d2006-07-03 00:24:54 -070071/*
72 * It is easier for the lock validator if interrupts are not re-enabled
73 * in the middle of a lock-acquire. This is a performance feature anyway
74 * so we turn it off:
75 */
76#ifndef CONFIG_PROVE_LOCKING
Ingo Molnarfb1c8f92005-09-10 00:25:56 -070077static inline void __raw_spin_lock_flags(raw_spinlock_t *lock, unsigned long flags)
Linus Torvalds1da177e2005-04-16 15:20:36 -070078{
Gerd Hoffmann9a0b5812006-03-23 02:59:32 -080079 alternative_smp(
80 __raw_spin_lock_string_flags,
81 __raw_spin_lock_string_up,
Linus Torvaldsb862f3b2006-07-08 15:24:18 -070082 "+m" (lock->slock) : "r" (flags) : "memory");
Linus Torvalds1da177e2005-04-16 15:20:36 -070083}
Ingo Molnar8a25d5d2006-07-03 00:24:54 -070084#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -070085
Ingo Molnarfb1c8f92005-09-10 00:25:56 -070086static inline int __raw_spin_trylock(raw_spinlock_t *lock)
Linus Torvalds1da177e2005-04-16 15:20:36 -070087{
88 char oldval;
89 __asm__ __volatile__(
90 "xchgb %b0,%1"
Linus Torvaldsb862f3b2006-07-08 15:24:18 -070091 :"=q" (oldval), "+m" (lock->slock)
Linus Torvalds1da177e2005-04-16 15:20:36 -070092 :"0" (0) : "memory");
93 return oldval > 0;
94}
95
Ingo Molnarfb1c8f92005-09-10 00:25:56 -070096/*
97 * __raw_spin_unlock based on writing $1 to the low byte.
98 * This method works. Despite all the confusion.
99 * (except on PPro SMP or if we are using OOSTORE, so we use xchgb there)
100 * (PPro errata 66, 92)
101 */
102
103#if !defined(CONFIG_X86_OOSTORE) && !defined(CONFIG_X86_PPRO_FENCE)
104
105#define __raw_spin_unlock_string \
106 "movb $1,%0" \
Linus Torvaldsb862f3b2006-07-08 15:24:18 -0700107 :"+m" (lock->slock) : : "memory"
Ingo Molnarfb1c8f92005-09-10 00:25:56 -0700108
109
110static inline void __raw_spin_unlock(raw_spinlock_t *lock)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700111{
Linus Torvalds1da177e2005-04-16 15:20:36 -0700112 __asm__ __volatile__(
Ingo Molnarfb1c8f92005-09-10 00:25:56 -0700113 __raw_spin_unlock_string
114 );
Linus Torvalds1da177e2005-04-16 15:20:36 -0700115}
116
Ingo Molnarfb1c8f92005-09-10 00:25:56 -0700117#else
118
119#define __raw_spin_unlock_string \
120 "xchgb %b0, %1" \
Linus Torvaldsb862f3b2006-07-08 15:24:18 -0700121 :"=q" (oldval), "+m" (lock->slock) \
Ingo Molnarfb1c8f92005-09-10 00:25:56 -0700122 :"0" (oldval) : "memory"
123
124static inline void __raw_spin_unlock(raw_spinlock_t *lock)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700125{
Ingo Molnarfb1c8f92005-09-10 00:25:56 -0700126 char oldval = 1;
127
Linus Torvalds1da177e2005-04-16 15:20:36 -0700128 __asm__ __volatile__(
Ingo Molnarfb1c8f92005-09-10 00:25:56 -0700129 __raw_spin_unlock_string
130 );
Linus Torvalds1da177e2005-04-16 15:20:36 -0700131}
132
Ingo Molnarfb1c8f92005-09-10 00:25:56 -0700133#endif
134
135#define __raw_spin_unlock_wait(lock) \
136 do { while (__raw_spin_is_locked(lock)) cpu_relax(); } while (0)
137
Linus Torvalds1da177e2005-04-16 15:20:36 -0700138/*
139 * Read-write spinlocks, allowing multiple readers
140 * but only one writer.
141 *
142 * NOTE! it is quite common to have readers in interrupts
143 * but no interrupt writers. For those circumstances we
144 * can "mix" irq-safe locks - any writer needs to get a
145 * irq-safe write-lock, but readers can get non-irqsafe
146 * read-locks.
Ingo Molnarfb1c8f92005-09-10 00:25:56 -0700147 *
Linus Torvalds1da177e2005-04-16 15:20:36 -0700148 * On x86, we implement read-write locks as a 32-bit counter
149 * with the high bit (sign) being the "contended" bit.
150 *
151 * The inline assembly is non-obvious. Think about it.
152 *
153 * Changed to use the same technique as rw semaphores. See
154 * semaphore.h for details. -ben
Ingo Molnarfb1c8f92005-09-10 00:25:56 -0700155 *
156 * the helpers are in arch/i386/kernel/semaphore.c
Linus Torvalds1da177e2005-04-16 15:20:36 -0700157 */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700158
Ingo Molnarfb1c8f92005-09-10 00:25:56 -0700159/**
160 * read_can_lock - would read_trylock() succeed?
161 * @lock: the rwlock in question.
162 */
163#define __raw_read_can_lock(x) ((int)(x)->lock > 0)
164
165/**
166 * write_can_lock - would write_trylock() succeed?
167 * @lock: the rwlock in question.
168 */
169#define __raw_write_can_lock(x) ((x)->lock == RW_LOCK_BIAS)
170
171static inline void __raw_read_lock(raw_rwlock_t *rw)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700172{
Linus Torvalds1da177e2005-04-16 15:20:36 -0700173 __build_read_lock(rw, "__read_lock_failed");
174}
175
Ingo Molnarfb1c8f92005-09-10 00:25:56 -0700176static inline void __raw_write_lock(raw_rwlock_t *rw)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700177{
Linus Torvalds1da177e2005-04-16 15:20:36 -0700178 __build_write_lock(rw, "__write_lock_failed");
179}
180
Ingo Molnarfb1c8f92005-09-10 00:25:56 -0700181static inline int __raw_read_trylock(raw_rwlock_t *lock)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700182{
183 atomic_t *count = (atomic_t *)lock;
184 atomic_dec(count);
185 if (atomic_read(count) >= 0)
186 return 1;
187 atomic_inc(count);
188 return 0;
189}
190
Ingo Molnarfb1c8f92005-09-10 00:25:56 -0700191static inline int __raw_write_trylock(raw_rwlock_t *lock)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700192{
193 atomic_t *count = (atomic_t *)lock;
194 if (atomic_sub_and_test(RW_LOCK_BIAS, count))
195 return 1;
196 atomic_add(RW_LOCK_BIAS, count);
197 return 0;
198}
199
Ingo Molnarfb1c8f92005-09-10 00:25:56 -0700200static inline void __raw_read_unlock(raw_rwlock_t *rw)
201{
Linus Torvaldsb862f3b2006-07-08 15:24:18 -0700202 asm volatile(LOCK_PREFIX "incl %0" :"+m" (rw->lock) : : "memory");
Ingo Molnarfb1c8f92005-09-10 00:25:56 -0700203}
204
205static inline void __raw_write_unlock(raw_rwlock_t *rw)
206{
Gerd Hoffmann9a0b5812006-03-23 02:59:32 -0800207 asm volatile(LOCK_PREFIX "addl $" RW_LOCK_BIAS_STR ", %0"
Linus Torvaldsb862f3b2006-07-08 15:24:18 -0700208 : "+m" (rw->lock) : : "memory");
Ingo Molnarfb1c8f92005-09-10 00:25:56 -0700209}
210
Linus Torvalds1da177e2005-04-16 15:20:36 -0700211#endif /* __ASM_SPINLOCK_H */