Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1 | #ifndef __ASM_SPINLOCK_H |
| 2 | #define __ASM_SPINLOCK_H |
| 3 | |
| 4 | #include <asm/atomic.h> |
| 5 | #include <asm/rwlock.h> |
| 6 | #include <asm/page.h> |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 7 | #include <linux/compiler.h> |
| 8 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 9 | /* |
| 10 | * Your basic SMP spinlocks, allowing only a single CPU anywhere |
Ingo Molnar | fb1c8f9 | 2005-09-10 00:25:56 -0700 | [diff] [blame] | 11 | * |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 12 | * Simple spin lock operations. There are two variants, one clears IRQ's |
| 13 | * on the local processor, one does not. |
| 14 | * |
| 15 | * We make no fairness assumptions. They have a cost. |
Ingo Molnar | fb1c8f9 | 2005-09-10 00:25:56 -0700 | [diff] [blame] | 16 | * |
| 17 | * (the type definitions are in asm/spinlock_types.h) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 18 | */ |
| 19 | |
Ingo Molnar | fb1c8f9 | 2005-09-10 00:25:56 -0700 | [diff] [blame] | 20 | #define __raw_spin_is_locked(x) \ |
| 21 | (*(volatile signed char *)(&(x)->slock) <= 0) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 22 | |
Ingo Molnar | fb1c8f9 | 2005-09-10 00:25:56 -0700 | [diff] [blame] | 23 | #define __raw_spin_lock_string \ |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 24 | "\n1:\t" \ |
| 25 | "lock ; decb %0\n\t" \ |
| 26 | "jns 3f\n" \ |
| 27 | "2:\t" \ |
| 28 | "rep;nop\n\t" \ |
| 29 | "cmpb $0,%0\n\t" \ |
| 30 | "jle 2b\n\t" \ |
| 31 | "jmp 1b\n" \ |
| 32 | "3:\n\t" |
| 33 | |
Ingo Molnar | 55f327f | 2006-07-03 00:24:43 -0700 | [diff] [blame] | 34 | /* |
| 35 | * NOTE: there's an irqs-on section here, which normally would have to be |
| 36 | * irq-traced, but on CONFIG_TRACE_IRQFLAGS we never use |
| 37 | * __raw_spin_lock_string_flags(). |
| 38 | */ |
Ingo Molnar | fb1c8f9 | 2005-09-10 00:25:56 -0700 | [diff] [blame] | 39 | #define __raw_spin_lock_string_flags \ |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 40 | "\n1:\t" \ |
| 41 | "lock ; decb %0\n\t" \ |
Chuck Ebbert | 42c059e0 | 2006-03-23 02:59:55 -0800 | [diff] [blame] | 42 | "jns 5f\n" \ |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 43 | "2:\t" \ |
| 44 | "testl $0x200, %1\n\t" \ |
Chuck Ebbert | 42c059e0 | 2006-03-23 02:59:55 -0800 | [diff] [blame] | 45 | "jz 4f\n\t" \ |
| 46 | "sti\n" \ |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 47 | "3:\t" \ |
| 48 | "rep;nop\n\t" \ |
| 49 | "cmpb $0, %0\n\t" \ |
| 50 | "jle 3b\n\t" \ |
| 51 | "cli\n\t" \ |
| 52 | "jmp 1b\n" \ |
Chuck Ebbert | 42c059e0 | 2006-03-23 02:59:55 -0800 | [diff] [blame] | 53 | "4:\t" \ |
| 54 | "rep;nop\n\t" \ |
| 55 | "cmpb $0, %0\n\t" \ |
| 56 | "jg 1b\n\t" \ |
| 57 | "jmp 4b\n" \ |
| 58 | "5:\n\t" |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 59 | |
Gerd Hoffmann | 9a0b581 | 2006-03-23 02:59:32 -0800 | [diff] [blame] | 60 | #define __raw_spin_lock_string_up \ |
| 61 | "\n\tdecb %0" |
| 62 | |
Ingo Molnar | fb1c8f9 | 2005-09-10 00:25:56 -0700 | [diff] [blame] | 63 | static inline void __raw_spin_lock(raw_spinlock_t *lock) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 64 | { |
Gerd Hoffmann | 9a0b581 | 2006-03-23 02:59:32 -0800 | [diff] [blame] | 65 | alternative_smp( |
| 66 | __raw_spin_lock_string, |
| 67 | __raw_spin_lock_string_up, |
| 68 | "=m" (lock->slock) : : "memory"); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 69 | } |
| 70 | |
Ingo Molnar | fb1c8f9 | 2005-09-10 00:25:56 -0700 | [diff] [blame] | 71 | static inline void __raw_spin_lock_flags(raw_spinlock_t *lock, unsigned long flags) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 72 | { |
Gerd Hoffmann | 9a0b581 | 2006-03-23 02:59:32 -0800 | [diff] [blame] | 73 | alternative_smp( |
| 74 | __raw_spin_lock_string_flags, |
| 75 | __raw_spin_lock_string_up, |
| 76 | "=m" (lock->slock) : "r" (flags) : "memory"); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 77 | } |
| 78 | |
Ingo Molnar | fb1c8f9 | 2005-09-10 00:25:56 -0700 | [diff] [blame] | 79 | static inline int __raw_spin_trylock(raw_spinlock_t *lock) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 80 | { |
| 81 | char oldval; |
| 82 | __asm__ __volatile__( |
| 83 | "xchgb %b0,%1" |
| 84 | :"=q" (oldval), "=m" (lock->slock) |
| 85 | :"0" (0) : "memory"); |
| 86 | return oldval > 0; |
| 87 | } |
| 88 | |
Ingo Molnar | fb1c8f9 | 2005-09-10 00:25:56 -0700 | [diff] [blame] | 89 | /* |
| 90 | * __raw_spin_unlock based on writing $1 to the low byte. |
| 91 | * This method works. Despite all the confusion. |
| 92 | * (except on PPro SMP or if we are using OOSTORE, so we use xchgb there) |
| 93 | * (PPro errata 66, 92) |
| 94 | */ |
| 95 | |
| 96 | #if !defined(CONFIG_X86_OOSTORE) && !defined(CONFIG_X86_PPRO_FENCE) |
| 97 | |
| 98 | #define __raw_spin_unlock_string \ |
| 99 | "movb $1,%0" \ |
| 100 | :"=m" (lock->slock) : : "memory" |
| 101 | |
| 102 | |
| 103 | static inline void __raw_spin_unlock(raw_spinlock_t *lock) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 104 | { |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 105 | __asm__ __volatile__( |
Ingo Molnar | fb1c8f9 | 2005-09-10 00:25:56 -0700 | [diff] [blame] | 106 | __raw_spin_unlock_string |
| 107 | ); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 108 | } |
| 109 | |
Ingo Molnar | fb1c8f9 | 2005-09-10 00:25:56 -0700 | [diff] [blame] | 110 | #else |
| 111 | |
| 112 | #define __raw_spin_unlock_string \ |
| 113 | "xchgb %b0, %1" \ |
| 114 | :"=q" (oldval), "=m" (lock->slock) \ |
| 115 | :"0" (oldval) : "memory" |
| 116 | |
| 117 | static inline void __raw_spin_unlock(raw_spinlock_t *lock) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 118 | { |
Ingo Molnar | fb1c8f9 | 2005-09-10 00:25:56 -0700 | [diff] [blame] | 119 | char oldval = 1; |
| 120 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 121 | __asm__ __volatile__( |
Ingo Molnar | fb1c8f9 | 2005-09-10 00:25:56 -0700 | [diff] [blame] | 122 | __raw_spin_unlock_string |
| 123 | ); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 124 | } |
| 125 | |
Ingo Molnar | fb1c8f9 | 2005-09-10 00:25:56 -0700 | [diff] [blame] | 126 | #endif |
| 127 | |
| 128 | #define __raw_spin_unlock_wait(lock) \ |
| 129 | do { while (__raw_spin_is_locked(lock)) cpu_relax(); } while (0) |
| 130 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 131 | /* |
| 132 | * Read-write spinlocks, allowing multiple readers |
| 133 | * but only one writer. |
| 134 | * |
| 135 | * NOTE! it is quite common to have readers in interrupts |
| 136 | * but no interrupt writers. For those circumstances we |
| 137 | * can "mix" irq-safe locks - any writer needs to get a |
| 138 | * irq-safe write-lock, but readers can get non-irqsafe |
| 139 | * read-locks. |
Ingo Molnar | fb1c8f9 | 2005-09-10 00:25:56 -0700 | [diff] [blame] | 140 | * |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 141 | * On x86, we implement read-write locks as a 32-bit counter |
| 142 | * with the high bit (sign) being the "contended" bit. |
| 143 | * |
| 144 | * The inline assembly is non-obvious. Think about it. |
| 145 | * |
| 146 | * Changed to use the same technique as rw semaphores. See |
| 147 | * semaphore.h for details. -ben |
Ingo Molnar | fb1c8f9 | 2005-09-10 00:25:56 -0700 | [diff] [blame] | 148 | * |
| 149 | * the helpers are in arch/i386/kernel/semaphore.c |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 150 | */ |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 151 | |
Ingo Molnar | fb1c8f9 | 2005-09-10 00:25:56 -0700 | [diff] [blame] | 152 | /** |
| 153 | * read_can_lock - would read_trylock() succeed? |
| 154 | * @lock: the rwlock in question. |
| 155 | */ |
| 156 | #define __raw_read_can_lock(x) ((int)(x)->lock > 0) |
| 157 | |
| 158 | /** |
| 159 | * write_can_lock - would write_trylock() succeed? |
| 160 | * @lock: the rwlock in question. |
| 161 | */ |
| 162 | #define __raw_write_can_lock(x) ((x)->lock == RW_LOCK_BIAS) |
| 163 | |
| 164 | static inline void __raw_read_lock(raw_rwlock_t *rw) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 165 | { |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 166 | __build_read_lock(rw, "__read_lock_failed"); |
| 167 | } |
| 168 | |
Ingo Molnar | fb1c8f9 | 2005-09-10 00:25:56 -0700 | [diff] [blame] | 169 | static inline void __raw_write_lock(raw_rwlock_t *rw) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 170 | { |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 171 | __build_write_lock(rw, "__write_lock_failed"); |
| 172 | } |
| 173 | |
Ingo Molnar | fb1c8f9 | 2005-09-10 00:25:56 -0700 | [diff] [blame] | 174 | static inline int __raw_read_trylock(raw_rwlock_t *lock) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 175 | { |
| 176 | atomic_t *count = (atomic_t *)lock; |
| 177 | atomic_dec(count); |
| 178 | if (atomic_read(count) >= 0) |
| 179 | return 1; |
| 180 | atomic_inc(count); |
| 181 | return 0; |
| 182 | } |
| 183 | |
Ingo Molnar | fb1c8f9 | 2005-09-10 00:25:56 -0700 | [diff] [blame] | 184 | static inline int __raw_write_trylock(raw_rwlock_t *lock) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 185 | { |
| 186 | atomic_t *count = (atomic_t *)lock; |
| 187 | if (atomic_sub_and_test(RW_LOCK_BIAS, count)) |
| 188 | return 1; |
| 189 | atomic_add(RW_LOCK_BIAS, count); |
| 190 | return 0; |
| 191 | } |
| 192 | |
Ingo Molnar | fb1c8f9 | 2005-09-10 00:25:56 -0700 | [diff] [blame] | 193 | static inline void __raw_read_unlock(raw_rwlock_t *rw) |
| 194 | { |
Gerd Hoffmann | 9a0b581 | 2006-03-23 02:59:32 -0800 | [diff] [blame] | 195 | asm volatile(LOCK_PREFIX "incl %0" :"=m" (rw->lock) : : "memory"); |
Ingo Molnar | fb1c8f9 | 2005-09-10 00:25:56 -0700 | [diff] [blame] | 196 | } |
| 197 | |
| 198 | static inline void __raw_write_unlock(raw_rwlock_t *rw) |
| 199 | { |
Gerd Hoffmann | 9a0b581 | 2006-03-23 02:59:32 -0800 | [diff] [blame] | 200 | asm volatile(LOCK_PREFIX "addl $" RW_LOCK_BIAS_STR ", %0" |
Ingo Molnar | fb1c8f9 | 2005-09-10 00:25:56 -0700 | [diff] [blame] | 201 | : "=m" (rw->lock) : : "memory"); |
| 202 | } |
| 203 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 204 | #endif /* __ASM_SPINLOCK_H */ |