Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1 | /* |
| 2 | * This file is subject to the terms and conditions of the GNU General Public |
| 3 | * License. See the file "COPYING" in the main directory of this archive |
| 4 | * for more details. |
| 5 | * |
Ralf Baechle | f65e4fa | 2006-09-28 01:45:21 +0100 | [diff] [blame] | 6 | * Copyright (C) 1999, 2000, 06 Ralf Baechle (ralf@linux-mips.org) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 7 | * Copyright (C) 1999, 2000 Silicon Graphics, Inc. |
| 8 | */ |
| 9 | #ifndef _ASM_SPINLOCK_H |
| 10 | #define _ASM_SPINLOCK_H |
| 11 | |
Ralf Baechle | 2a31b03 | 2008-08-28 15:17:49 +0100 | [diff] [blame] | 12 | #include <linux/compiler.h> |
| 13 | |
Ralf Baechle | 0004a9d | 2006-10-31 03:45:07 +0000 | [diff] [blame] | 14 | #include <asm/barrier.h> |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 15 | #include <asm/war.h> |
| 16 | |
| 17 | /* |
| 18 | * Your basic SMP spinlocks, allowing only a single CPU anywhere |
Ralf Baechle | 2a31b03 | 2008-08-28 15:17:49 +0100 | [diff] [blame] | 19 | * |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 20 | * Simple spin lock operations. There are two variants, one clears IRQ's |
| 21 | * on the local processor, one does not. |
| 22 | * |
Ralf Baechle | 2a31b03 | 2008-08-28 15:17:49 +0100 | [diff] [blame] | 23 | * These are fair FIFO ticket locks |
| 24 | * |
| 25 | * (the type definitions are in asm/spinlock_types.h) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 26 | */ |
| 27 | |
Ralf Baechle | 2a31b03 | 2008-08-28 15:17:49 +0100 | [diff] [blame] | 28 | |
| 29 | /* |
| 30 | * Ticket locks are conceptually two parts, one indicating the current head of |
| 31 | * the queue, and the other indicating the current tail. The lock is acquired |
| 32 | * by atomically noting the tail and incrementing it by one (thus adding |
| 33 | * ourself to the queue and noting our position), then waiting until the head |
| 34 | * becomes equal to the the initial value of the tail. |
| 35 | */ |
| 36 | |
| 37 | static inline int __raw_spin_is_locked(raw_spinlock_t *lock) |
| 38 | { |
| 39 | unsigned int counters = ACCESS_ONCE(lock->lock); |
| 40 | |
| 41 | return ((counters >> 14) ^ counters) & 0x1fff; |
| 42 | } |
| 43 | |
| 44 | #define __raw_spin_lock_flags(lock, flags) __raw_spin_lock(lock) |
| 45 | #define __raw_spin_unlock_wait(x) \ |
| 46 | while (__raw_spin_is_locked(x)) { cpu_relax(); } |
| 47 | |
| 48 | static inline int __raw_spin_is_contended(raw_spinlock_t *lock) |
| 49 | { |
| 50 | unsigned int counters = ACCESS_ONCE(lock->lock); |
| 51 | |
| 52 | return (((counters >> 14) - counters) & 0x1fff) > 1; |
| 53 | } |
Kyle McMartin | a5ef7ca | 2009-02-08 17:39:58 -0500 | [diff] [blame] | 54 | #define __raw_spin_is_contended __raw_spin_is_contended |
Ralf Baechle | 2a31b03 | 2008-08-28 15:17:49 +0100 | [diff] [blame] | 55 | |
Ingo Molnar | fb1c8f9 | 2005-09-10 00:25:56 -0700 | [diff] [blame] | 56 | static inline void __raw_spin_lock(raw_spinlock_t *lock) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 57 | { |
Ralf Baechle | 2a31b03 | 2008-08-28 15:17:49 +0100 | [diff] [blame] | 58 | int my_ticket; |
| 59 | int tmp; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 60 | |
| 61 | if (R10000_LLSC_WAR) { |
Ralf Baechle | 2a31b03 | 2008-08-28 15:17:49 +0100 | [diff] [blame] | 62 | __asm__ __volatile__ ( |
| 63 | " .set push # __raw_spin_lock \n" |
| 64 | " .set noreorder \n" |
| 65 | " \n" |
| 66 | "1: ll %[ticket], %[ticket_ptr] \n" |
| 67 | " addiu %[my_ticket], %[ticket], 0x4000 \n" |
| 68 | " sc %[my_ticket], %[ticket_ptr] \n" |
| 69 | " beqzl %[my_ticket], 1b \n" |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 70 | " nop \n" |
Ralf Baechle | 2a31b03 | 2008-08-28 15:17:49 +0100 | [diff] [blame] | 71 | " srl %[my_ticket], %[ticket], 14 \n" |
| 72 | " andi %[my_ticket], %[my_ticket], 0x1fff \n" |
| 73 | " andi %[ticket], %[ticket], 0x1fff \n" |
| 74 | " bne %[ticket], %[my_ticket], 4f \n" |
| 75 | " subu %[ticket], %[my_ticket], %[ticket] \n" |
| 76 | "2: \n" |
Ralf Baechle | f65e4fa | 2006-09-28 01:45:21 +0100 | [diff] [blame] | 77 | " .subsection 2 \n" |
Ralf Baechle | 2a31b03 | 2008-08-28 15:17:49 +0100 | [diff] [blame] | 78 | "4: andi %[ticket], %[ticket], 0x1fff \n" |
David Daney | 0e6826c | 2009-03-27 10:07:02 -0700 | [diff] [blame] | 79 | " sll %[ticket], 5 \n" |
Ralf Baechle | 2a31b03 | 2008-08-28 15:17:49 +0100 | [diff] [blame] | 80 | " \n" |
| 81 | "6: bnez %[ticket], 6b \n" |
| 82 | " subu %[ticket], 1 \n" |
| 83 | " \n" |
| 84 | " lw %[ticket], %[ticket_ptr] \n" |
| 85 | " andi %[ticket], %[ticket], 0x1fff \n" |
| 86 | " beq %[ticket], %[my_ticket], 2b \n" |
| 87 | " subu %[ticket], %[my_ticket], %[ticket] \n" |
David Daney | 0e6826c | 2009-03-27 10:07:02 -0700 | [diff] [blame] | 88 | " b 4b \n" |
Ralf Baechle | 2a31b03 | 2008-08-28 15:17:49 +0100 | [diff] [blame] | 89 | " subu %[ticket], %[ticket], 1 \n" |
Ralf Baechle | f65e4fa | 2006-09-28 01:45:21 +0100 | [diff] [blame] | 90 | " .previous \n" |
Ralf Baechle | 2a31b03 | 2008-08-28 15:17:49 +0100 | [diff] [blame] | 91 | " .set pop \n" |
| 92 | : [ticket_ptr] "+m" (lock->lock), |
| 93 | [ticket] "=&r" (tmp), |
| 94 | [my_ticket] "=&r" (my_ticket)); |
| 95 | } else { |
| 96 | __asm__ __volatile__ ( |
| 97 | " .set push # __raw_spin_lock \n" |
| 98 | " .set noreorder \n" |
| 99 | " \n" |
| 100 | " ll %[ticket], %[ticket_ptr] \n" |
| 101 | "1: addiu %[my_ticket], %[ticket], 0x4000 \n" |
| 102 | " sc %[my_ticket], %[ticket_ptr] \n" |
| 103 | " beqz %[my_ticket], 3f \n" |
| 104 | " nop \n" |
| 105 | " srl %[my_ticket], %[ticket], 14 \n" |
| 106 | " andi %[my_ticket], %[my_ticket], 0x1fff \n" |
| 107 | " andi %[ticket], %[ticket], 0x1fff \n" |
| 108 | " bne %[ticket], %[my_ticket], 4f \n" |
| 109 | " subu %[ticket], %[my_ticket], %[ticket] \n" |
| 110 | "2: \n" |
| 111 | " .subsection 2 \n" |
| 112 | "3: b 1b \n" |
| 113 | " ll %[ticket], %[ticket_ptr] \n" |
| 114 | " \n" |
| 115 | "4: andi %[ticket], %[ticket], 0x1fff \n" |
David Daney | 0e6826c | 2009-03-27 10:07:02 -0700 | [diff] [blame] | 116 | " sll %[ticket], 5 \n" |
Ralf Baechle | 2a31b03 | 2008-08-28 15:17:49 +0100 | [diff] [blame] | 117 | " \n" |
| 118 | "6: bnez %[ticket], 6b \n" |
| 119 | " subu %[ticket], 1 \n" |
| 120 | " \n" |
| 121 | " lw %[ticket], %[ticket_ptr] \n" |
| 122 | " andi %[ticket], %[ticket], 0x1fff \n" |
| 123 | " beq %[ticket], %[my_ticket], 2b \n" |
| 124 | " subu %[ticket], %[my_ticket], %[ticket] \n" |
David Daney | 0e6826c | 2009-03-27 10:07:02 -0700 | [diff] [blame] | 125 | " b 4b \n" |
Ralf Baechle | 2a31b03 | 2008-08-28 15:17:49 +0100 | [diff] [blame] | 126 | " subu %[ticket], %[ticket], 1 \n" |
| 127 | " .previous \n" |
| 128 | " .set pop \n" |
| 129 | : [ticket_ptr] "+m" (lock->lock), |
| 130 | [ticket] "=&r" (tmp), |
| 131 | [my_ticket] "=&r" (my_ticket)); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 132 | } |
Ralf Baechle | 0004a9d | 2006-10-31 03:45:07 +0000 | [diff] [blame] | 133 | |
Ralf Baechle | 17099b1 | 2007-07-14 13:24:05 +0100 | [diff] [blame] | 134 | smp_llsc_mb(); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 135 | } |
| 136 | |
Ingo Molnar | fb1c8f9 | 2005-09-10 00:25:56 -0700 | [diff] [blame] | 137 | static inline void __raw_spin_unlock(raw_spinlock_t *lock) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 138 | { |
Ralf Baechle | 2a31b03 | 2008-08-28 15:17:49 +0100 | [diff] [blame] | 139 | int tmp; |
Ralf Baechle | 0004a9d | 2006-10-31 03:45:07 +0000 | [diff] [blame] | 140 | |
Ralf Baechle | 2a31b03 | 2008-08-28 15:17:49 +0100 | [diff] [blame] | 141 | smp_llsc_mb(); |
| 142 | |
| 143 | if (R10000_LLSC_WAR) { |
| 144 | __asm__ __volatile__ ( |
| 145 | " # __raw_spin_unlock \n" |
| 146 | "1: ll %[ticket], %[ticket_ptr] \n" |
| 147 | " addiu %[ticket], %[ticket], 1 \n" |
| 148 | " ori %[ticket], %[ticket], 0x2000 \n" |
| 149 | " xori %[ticket], %[ticket], 0x2000 \n" |
| 150 | " sc %[ticket], %[ticket_ptr] \n" |
Johannes Dickgreber | 9b8f386 | 2008-10-13 19:33:32 +0200 | [diff] [blame] | 151 | " beqzl %[ticket], 1b \n" |
Ralf Baechle | 2a31b03 | 2008-08-28 15:17:49 +0100 | [diff] [blame] | 152 | : [ticket_ptr] "+m" (lock->lock), |
| 153 | [ticket] "=&r" (tmp)); |
| 154 | } else { |
| 155 | __asm__ __volatile__ ( |
| 156 | " .set push # __raw_spin_unlock \n" |
| 157 | " .set noreorder \n" |
| 158 | " \n" |
| 159 | " ll %[ticket], %[ticket_ptr] \n" |
| 160 | "1: addiu %[ticket], %[ticket], 1 \n" |
| 161 | " ori %[ticket], %[ticket], 0x2000 \n" |
| 162 | " xori %[ticket], %[ticket], 0x2000 \n" |
| 163 | " sc %[ticket], %[ticket_ptr] \n" |
| 164 | " beqz %[ticket], 2f \n" |
| 165 | " nop \n" |
| 166 | " \n" |
| 167 | " .subsection 2 \n" |
| 168 | "2: b 1b \n" |
| 169 | " ll %[ticket], %[ticket_ptr] \n" |
| 170 | " .previous \n" |
| 171 | " .set pop \n" |
| 172 | : [ticket_ptr] "+m" (lock->lock), |
| 173 | [ticket] "=&r" (tmp)); |
| 174 | } |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 175 | } |
| 176 | |
Ingo Molnar | fb1c8f9 | 2005-09-10 00:25:56 -0700 | [diff] [blame] | 177 | static inline unsigned int __raw_spin_trylock(raw_spinlock_t *lock) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 178 | { |
Ralf Baechle | 2a31b03 | 2008-08-28 15:17:49 +0100 | [diff] [blame] | 179 | int tmp, tmp2, tmp3; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 180 | |
| 181 | if (R10000_LLSC_WAR) { |
Ralf Baechle | 2a31b03 | 2008-08-28 15:17:49 +0100 | [diff] [blame] | 182 | __asm__ __volatile__ ( |
| 183 | " .set push # __raw_spin_trylock \n" |
| 184 | " .set noreorder \n" |
| 185 | " \n" |
| 186 | "1: ll %[ticket], %[ticket_ptr] \n" |
| 187 | " srl %[my_ticket], %[ticket], 14 \n" |
| 188 | " andi %[my_ticket], %[my_ticket], 0x1fff \n" |
| 189 | " andi %[now_serving], %[ticket], 0x1fff \n" |
| 190 | " bne %[my_ticket], %[now_serving], 3f \n" |
| 191 | " addiu %[ticket], %[ticket], 0x4000 \n" |
| 192 | " sc %[ticket], %[ticket_ptr] \n" |
| 193 | " beqzl %[ticket], 1b \n" |
| 194 | " li %[ticket], 1 \n" |
| 195 | "2: \n" |
Ralf Baechle | f65e4fa | 2006-09-28 01:45:21 +0100 | [diff] [blame] | 196 | " .subsection 2 \n" |
Ralf Baechle | 2a31b03 | 2008-08-28 15:17:49 +0100 | [diff] [blame] | 197 | "3: b 2b \n" |
| 198 | " li %[ticket], 0 \n" |
Ralf Baechle | f65e4fa | 2006-09-28 01:45:21 +0100 | [diff] [blame] | 199 | " .previous \n" |
Ralf Baechle | 2a31b03 | 2008-08-28 15:17:49 +0100 | [diff] [blame] | 200 | " .set pop \n" |
| 201 | : [ticket_ptr] "+m" (lock->lock), |
| 202 | [ticket] "=&r" (tmp), |
| 203 | [my_ticket] "=&r" (tmp2), |
| 204 | [now_serving] "=&r" (tmp3)); |
| 205 | } else { |
| 206 | __asm__ __volatile__ ( |
| 207 | " .set push # __raw_spin_trylock \n" |
| 208 | " .set noreorder \n" |
| 209 | " \n" |
| 210 | " ll %[ticket], %[ticket_ptr] \n" |
| 211 | "1: srl %[my_ticket], %[ticket], 14 \n" |
| 212 | " andi %[my_ticket], %[my_ticket], 0x1fff \n" |
| 213 | " andi %[now_serving], %[ticket], 0x1fff \n" |
| 214 | " bne %[my_ticket], %[now_serving], 3f \n" |
| 215 | " addiu %[ticket], %[ticket], 0x4000 \n" |
| 216 | " sc %[ticket], %[ticket_ptr] \n" |
| 217 | " beqz %[ticket], 4f \n" |
| 218 | " li %[ticket], 1 \n" |
| 219 | "2: \n" |
| 220 | " .subsection 2 \n" |
| 221 | "3: b 2b \n" |
| 222 | " li %[ticket], 0 \n" |
| 223 | "4: b 1b \n" |
| 224 | " ll %[ticket], %[ticket_ptr] \n" |
| 225 | " .previous \n" |
| 226 | " .set pop \n" |
| 227 | : [ticket_ptr] "+m" (lock->lock), |
| 228 | [ticket] "=&r" (tmp), |
| 229 | [my_ticket] "=&r" (tmp2), |
| 230 | [now_serving] "=&r" (tmp3)); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 231 | } |
| 232 | |
Ralf Baechle | 17099b1 | 2007-07-14 13:24:05 +0100 | [diff] [blame] | 233 | smp_llsc_mb(); |
Ralf Baechle | 0004a9d | 2006-10-31 03:45:07 +0000 | [diff] [blame] | 234 | |
Ralf Baechle | 2a31b03 | 2008-08-28 15:17:49 +0100 | [diff] [blame] | 235 | return tmp; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 236 | } |
| 237 | |
| 238 | /* |
| 239 | * Read-write spinlocks, allowing multiple readers but only one writer. |
| 240 | * |
| 241 | * NOTE! it is quite common to have readers in interrupts but no interrupt |
| 242 | * writers. For those circumstances we can "mix" irq-safe locks - any writer |
| 243 | * needs to get a irq-safe write-lock, but readers can get non-irqsafe |
| 244 | * read-locks. |
| 245 | */ |
| 246 | |
Ralf Baechle | e3c4807 | 2005-02-03 13:34:45 +0000 | [diff] [blame] | 247 | /* |
| 248 | * read_can_lock - would read_trylock() succeed? |
| 249 | * @lock: the rwlock in question. |
| 250 | */ |
| 251 | #define __raw_read_can_lock(rw) ((rw)->lock >= 0) |
| 252 | |
| 253 | /* |
| 254 | * write_can_lock - would write_trylock() succeed? |
| 255 | * @lock: the rwlock in question. |
| 256 | */ |
| 257 | #define __raw_write_can_lock(rw) (!(rw)->lock) |
| 258 | |
Ingo Molnar | fb1c8f9 | 2005-09-10 00:25:56 -0700 | [diff] [blame] | 259 | static inline void __raw_read_lock(raw_rwlock_t *rw) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 260 | { |
| 261 | unsigned int tmp; |
| 262 | |
| 263 | if (R10000_LLSC_WAR) { |
| 264 | __asm__ __volatile__( |
Ingo Molnar | fb1c8f9 | 2005-09-10 00:25:56 -0700 | [diff] [blame] | 265 | " .set noreorder # __raw_read_lock \n" |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 266 | "1: ll %1, %2 \n" |
| 267 | " bltz %1, 1b \n" |
| 268 | " addu %1, 1 \n" |
| 269 | " sc %1, %0 \n" |
| 270 | " beqzl %1, 1b \n" |
| 271 | " nop \n" |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 272 | " .set reorder \n" |
| 273 | : "=m" (rw->lock), "=&r" (tmp) |
| 274 | : "m" (rw->lock) |
| 275 | : "memory"); |
| 276 | } else { |
| 277 | __asm__ __volatile__( |
Ingo Molnar | fb1c8f9 | 2005-09-10 00:25:56 -0700 | [diff] [blame] | 278 | " .set noreorder # __raw_read_lock \n" |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 279 | "1: ll %1, %2 \n" |
Ralf Baechle | f65e4fa | 2006-09-28 01:45:21 +0100 | [diff] [blame] | 280 | " bltz %1, 2f \n" |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 281 | " addu %1, 1 \n" |
| 282 | " sc %1, %0 \n" |
| 283 | " beqz %1, 1b \n" |
Ralf Baechle | 0004a9d | 2006-10-31 03:45:07 +0000 | [diff] [blame] | 284 | " nop \n" |
Ralf Baechle | f65e4fa | 2006-09-28 01:45:21 +0100 | [diff] [blame] | 285 | " .subsection 2 \n" |
| 286 | "2: ll %1, %2 \n" |
| 287 | " bltz %1, 2b \n" |
| 288 | " addu %1, 1 \n" |
| 289 | " b 1b \n" |
| 290 | " nop \n" |
| 291 | " .previous \n" |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 292 | " .set reorder \n" |
| 293 | : "=m" (rw->lock), "=&r" (tmp) |
| 294 | : "m" (rw->lock) |
| 295 | : "memory"); |
| 296 | } |
Ralf Baechle | 0004a9d | 2006-10-31 03:45:07 +0000 | [diff] [blame] | 297 | |
Ralf Baechle | 17099b1 | 2007-07-14 13:24:05 +0100 | [diff] [blame] | 298 | smp_llsc_mb(); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 299 | } |
| 300 | |
| 301 | /* Note the use of sub, not subu which will make the kernel die with an |
| 302 | overflow exception if we ever try to unlock an rwlock that is already |
| 303 | unlocked or is being held by a writer. */ |
Ingo Molnar | fb1c8f9 | 2005-09-10 00:25:56 -0700 | [diff] [blame] | 304 | static inline void __raw_read_unlock(raw_rwlock_t *rw) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 305 | { |
| 306 | unsigned int tmp; |
| 307 | |
Ralf Baechle | 17099b1 | 2007-07-14 13:24:05 +0100 | [diff] [blame] | 308 | smp_llsc_mb(); |
Ralf Baechle | 0004a9d | 2006-10-31 03:45:07 +0000 | [diff] [blame] | 309 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 310 | if (R10000_LLSC_WAR) { |
| 311 | __asm__ __volatile__( |
Ingo Molnar | fb1c8f9 | 2005-09-10 00:25:56 -0700 | [diff] [blame] | 312 | "1: ll %1, %2 # __raw_read_unlock \n" |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 313 | " sub %1, 1 \n" |
| 314 | " sc %1, %0 \n" |
| 315 | " beqzl %1, 1b \n" |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 316 | : "=m" (rw->lock), "=&r" (tmp) |
| 317 | : "m" (rw->lock) |
| 318 | : "memory"); |
| 319 | } else { |
| 320 | __asm__ __volatile__( |
Ingo Molnar | fb1c8f9 | 2005-09-10 00:25:56 -0700 | [diff] [blame] | 321 | " .set noreorder # __raw_read_unlock \n" |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 322 | "1: ll %1, %2 \n" |
| 323 | " sub %1, 1 \n" |
| 324 | " sc %1, %0 \n" |
Ralf Baechle | f65e4fa | 2006-09-28 01:45:21 +0100 | [diff] [blame] | 325 | " beqz %1, 2f \n" |
Ralf Baechle | 0004a9d | 2006-10-31 03:45:07 +0000 | [diff] [blame] | 326 | " nop \n" |
Ralf Baechle | f65e4fa | 2006-09-28 01:45:21 +0100 | [diff] [blame] | 327 | " .subsection 2 \n" |
| 328 | "2: b 1b \n" |
| 329 | " nop \n" |
| 330 | " .previous \n" |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 331 | " .set reorder \n" |
| 332 | : "=m" (rw->lock), "=&r" (tmp) |
| 333 | : "m" (rw->lock) |
| 334 | : "memory"); |
| 335 | } |
| 336 | } |
| 337 | |
Ingo Molnar | fb1c8f9 | 2005-09-10 00:25:56 -0700 | [diff] [blame] | 338 | static inline void __raw_write_lock(raw_rwlock_t *rw) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 339 | { |
| 340 | unsigned int tmp; |
| 341 | |
| 342 | if (R10000_LLSC_WAR) { |
| 343 | __asm__ __volatile__( |
Ingo Molnar | fb1c8f9 | 2005-09-10 00:25:56 -0700 | [diff] [blame] | 344 | " .set noreorder # __raw_write_lock \n" |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 345 | "1: ll %1, %2 \n" |
| 346 | " bnez %1, 1b \n" |
| 347 | " lui %1, 0x8000 \n" |
| 348 | " sc %1, %0 \n" |
| 349 | " beqzl %1, 1b \n" |
Ralf Baechle | 0004a9d | 2006-10-31 03:45:07 +0000 | [diff] [blame] | 350 | " nop \n" |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 351 | " .set reorder \n" |
| 352 | : "=m" (rw->lock), "=&r" (tmp) |
| 353 | : "m" (rw->lock) |
| 354 | : "memory"); |
| 355 | } else { |
| 356 | __asm__ __volatile__( |
Ingo Molnar | fb1c8f9 | 2005-09-10 00:25:56 -0700 | [diff] [blame] | 357 | " .set noreorder # __raw_write_lock \n" |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 358 | "1: ll %1, %2 \n" |
Ralf Baechle | f65e4fa | 2006-09-28 01:45:21 +0100 | [diff] [blame] | 359 | " bnez %1, 2f \n" |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 360 | " lui %1, 0x8000 \n" |
| 361 | " sc %1, %0 \n" |
Ralf Baechle | f65e4fa | 2006-09-28 01:45:21 +0100 | [diff] [blame] | 362 | " beqz %1, 2f \n" |
Ralf Baechle | 0004a9d | 2006-10-31 03:45:07 +0000 | [diff] [blame] | 363 | " nop \n" |
Ralf Baechle | f65e4fa | 2006-09-28 01:45:21 +0100 | [diff] [blame] | 364 | " .subsection 2 \n" |
| 365 | "2: ll %1, %2 \n" |
| 366 | " bnez %1, 2b \n" |
| 367 | " lui %1, 0x8000 \n" |
| 368 | " b 1b \n" |
| 369 | " nop \n" |
| 370 | " .previous \n" |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 371 | " .set reorder \n" |
| 372 | : "=m" (rw->lock), "=&r" (tmp) |
| 373 | : "m" (rw->lock) |
| 374 | : "memory"); |
| 375 | } |
Ralf Baechle | 0004a9d | 2006-10-31 03:45:07 +0000 | [diff] [blame] | 376 | |
Ralf Baechle | 17099b1 | 2007-07-14 13:24:05 +0100 | [diff] [blame] | 377 | smp_llsc_mb(); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 378 | } |
| 379 | |
Ingo Molnar | fb1c8f9 | 2005-09-10 00:25:56 -0700 | [diff] [blame] | 380 | static inline void __raw_write_unlock(raw_rwlock_t *rw) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 381 | { |
Ralf Baechle | 0004a9d | 2006-10-31 03:45:07 +0000 | [diff] [blame] | 382 | smp_mb(); |
| 383 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 384 | __asm__ __volatile__( |
Ralf Baechle | 0004a9d | 2006-10-31 03:45:07 +0000 | [diff] [blame] | 385 | " # __raw_write_unlock \n" |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 386 | " sw $0, %0 \n" |
| 387 | : "=m" (rw->lock) |
| 388 | : "m" (rw->lock) |
| 389 | : "memory"); |
| 390 | } |
| 391 | |
Ralf Baechle | 65316fd | 2006-08-31 14:16:06 +0100 | [diff] [blame] | 392 | static inline int __raw_read_trylock(raw_rwlock_t *rw) |
| 393 | { |
| 394 | unsigned int tmp; |
| 395 | int ret; |
| 396 | |
| 397 | if (R10000_LLSC_WAR) { |
| 398 | __asm__ __volatile__( |
| 399 | " .set noreorder # __raw_read_trylock \n" |
| 400 | " li %2, 0 \n" |
| 401 | "1: ll %1, %3 \n" |
Dave Johnson | d52c2d5 | 2007-03-05 20:50:27 -0500 | [diff] [blame] | 402 | " bltz %1, 2f \n" |
Ralf Baechle | 65316fd | 2006-08-31 14:16:06 +0100 | [diff] [blame] | 403 | " addu %1, 1 \n" |
| 404 | " sc %1, %0 \n" |
Ralf Baechle | 65316fd | 2006-08-31 14:16:06 +0100 | [diff] [blame] | 405 | " .set reorder \n" |
Ralf Baechle | 0004a9d | 2006-10-31 03:45:07 +0000 | [diff] [blame] | 406 | " beqzl %1, 1b \n" |
| 407 | " nop \n" |
Ralf Baechle | 17099b1 | 2007-07-14 13:24:05 +0100 | [diff] [blame] | 408 | __WEAK_LLSC_MB |
Ralf Baechle | 65316fd | 2006-08-31 14:16:06 +0100 | [diff] [blame] | 409 | " li %2, 1 \n" |
| 410 | "2: \n" |
| 411 | : "=m" (rw->lock), "=&r" (tmp), "=&r" (ret) |
| 412 | : "m" (rw->lock) |
| 413 | : "memory"); |
| 414 | } else { |
| 415 | __asm__ __volatile__( |
| 416 | " .set noreorder # __raw_read_trylock \n" |
| 417 | " li %2, 0 \n" |
| 418 | "1: ll %1, %3 \n" |
Dave Johnson | d52c2d5 | 2007-03-05 20:50:27 -0500 | [diff] [blame] | 419 | " bltz %1, 2f \n" |
Ralf Baechle | 65316fd | 2006-08-31 14:16:06 +0100 | [diff] [blame] | 420 | " addu %1, 1 \n" |
| 421 | " sc %1, %0 \n" |
| 422 | " beqz %1, 1b \n" |
Ralf Baechle | 0004a9d | 2006-10-31 03:45:07 +0000 | [diff] [blame] | 423 | " nop \n" |
Ralf Baechle | 65316fd | 2006-08-31 14:16:06 +0100 | [diff] [blame] | 424 | " .set reorder \n" |
Ralf Baechle | 17099b1 | 2007-07-14 13:24:05 +0100 | [diff] [blame] | 425 | __WEAK_LLSC_MB |
Ralf Baechle | 65316fd | 2006-08-31 14:16:06 +0100 | [diff] [blame] | 426 | " li %2, 1 \n" |
| 427 | "2: \n" |
| 428 | : "=m" (rw->lock), "=&r" (tmp), "=&r" (ret) |
| 429 | : "m" (rw->lock) |
| 430 | : "memory"); |
| 431 | } |
| 432 | |
| 433 | return ret; |
| 434 | } |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 435 | |
Ingo Molnar | fb1c8f9 | 2005-09-10 00:25:56 -0700 | [diff] [blame] | 436 | static inline int __raw_write_trylock(raw_rwlock_t *rw) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 437 | { |
| 438 | unsigned int tmp; |
| 439 | int ret; |
| 440 | |
| 441 | if (R10000_LLSC_WAR) { |
| 442 | __asm__ __volatile__( |
Ingo Molnar | fb1c8f9 | 2005-09-10 00:25:56 -0700 | [diff] [blame] | 443 | " .set noreorder # __raw_write_trylock \n" |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 444 | " li %2, 0 \n" |
| 445 | "1: ll %1, %3 \n" |
| 446 | " bnez %1, 2f \n" |
| 447 | " lui %1, 0x8000 \n" |
| 448 | " sc %1, %0 \n" |
| 449 | " beqzl %1, 1b \n" |
Ralf Baechle | 0004a9d | 2006-10-31 03:45:07 +0000 | [diff] [blame] | 450 | " nop \n" |
Ralf Baechle | 17099b1 | 2007-07-14 13:24:05 +0100 | [diff] [blame] | 451 | __WEAK_LLSC_MB |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 452 | " li %2, 1 \n" |
| 453 | " .set reorder \n" |
| 454 | "2: \n" |
| 455 | : "=m" (rw->lock), "=&r" (tmp), "=&r" (ret) |
| 456 | : "m" (rw->lock) |
| 457 | : "memory"); |
| 458 | } else { |
| 459 | __asm__ __volatile__( |
Ingo Molnar | fb1c8f9 | 2005-09-10 00:25:56 -0700 | [diff] [blame] | 460 | " .set noreorder # __raw_write_trylock \n" |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 461 | " li %2, 0 \n" |
| 462 | "1: ll %1, %3 \n" |
| 463 | " bnez %1, 2f \n" |
| 464 | " lui %1, 0x8000 \n" |
| 465 | " sc %1, %0 \n" |
Ralf Baechle | f65e4fa | 2006-09-28 01:45:21 +0100 | [diff] [blame] | 466 | " beqz %1, 3f \n" |
| 467 | " li %2, 1 \n" |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 468 | "2: \n" |
Ralf Baechle | 17099b1 | 2007-07-14 13:24:05 +0100 | [diff] [blame] | 469 | __WEAK_LLSC_MB |
Ralf Baechle | f65e4fa | 2006-09-28 01:45:21 +0100 | [diff] [blame] | 470 | " .subsection 2 \n" |
| 471 | "3: b 1b \n" |
| 472 | " li %2, 0 \n" |
| 473 | " .previous \n" |
| 474 | " .set reorder \n" |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 475 | : "=m" (rw->lock), "=&r" (tmp), "=&r" (ret) |
| 476 | : "m" (rw->lock) |
| 477 | : "memory"); |
| 478 | } |
| 479 | |
| 480 | return ret; |
| 481 | } |
| 482 | |
Ralf Baechle | 65316fd | 2006-08-31 14:16:06 +0100 | [diff] [blame] | 483 | |
Martin Schwidefsky | ef6edc9 | 2006-09-30 23:27:43 -0700 | [diff] [blame] | 484 | #define _raw_spin_relax(lock) cpu_relax() |
| 485 | #define _raw_read_relax(lock) cpu_relax() |
| 486 | #define _raw_write_relax(lock) cpu_relax() |
| 487 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 488 | #endif /* _ASM_SPINLOCK_H */ |