Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1 | /* |
| 2 | * This file is subject to the terms and conditions of the GNU General Public |
| 3 | * License. See the file "COPYING" in the main directory of this archive |
| 4 | * for more details. |
| 5 | * |
| 6 | * Copyright (C) 1994, 95, 96, 97, 98, 99, 2003 by Ralf Baechle |
| 7 | * Copyright (C) 1996 by Paul M. Antoine |
| 8 | * Copyright (C) 1999 Silicon Graphics |
| 9 | * Kevin D. Kissell, kevink@mips.org and Carsten Langgaard, carstenl@mips.com |
| 10 | * Copyright (C) 2000 MIPS Technologies, Inc. |
| 11 | */ |
| 12 | #ifndef _ASM_SYSTEM_H |
| 13 | #define _ASM_SYSTEM_H |
| 14 | |
| 15 | #include <linux/config.h> |
| 16 | #include <linux/types.h> |
| 17 | |
| 18 | #include <asm/addrspace.h> |
| 19 | #include <asm/cpu-features.h> |
Ralf Baechle | e50c0a8 | 2005-05-31 11:49:19 +0000 | [diff] [blame] | 20 | #include <asm/dsp.h> |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 21 | #include <asm/ptrace.h> |
| 22 | #include <asm/war.h> |
| 23 | #include <asm/interrupt.h> |
| 24 | |
| 25 | /* |
| 26 | * read_barrier_depends - Flush all pending reads that subsequents reads |
| 27 | * depend on. |
| 28 | * |
| 29 | * No data-dependent reads from memory-like regions are ever reordered |
| 30 | * over this barrier. All reads preceding this primitive are guaranteed |
| 31 | * to access memory (but not necessarily other CPUs' caches) before any |
| 32 | * reads following this primitive that depend on the data return by |
| 33 | * any of the preceding reads. This primitive is much lighter weight than |
| 34 | * rmb() on most CPUs, and is never heavier weight than is |
| 35 | * rmb(). |
| 36 | * |
| 37 | * These ordering constraints are respected by both the local CPU |
| 38 | * and the compiler. |
| 39 | * |
| 40 | * Ordering is not guaranteed by anything other than these primitives, |
| 41 | * not even by data dependencies. See the documentation for |
| 42 | * memory_barrier() for examples and URLs to more information. |
| 43 | * |
| 44 | * For example, the following code would force ordering (the initial |
| 45 | * value of "a" is zero, "b" is one, and "p" is "&a"): |
| 46 | * |
| 47 | * <programlisting> |
| 48 | * CPU 0 CPU 1 |
| 49 | * |
| 50 | * b = 2; |
| 51 | * memory_barrier(); |
| 52 | * p = &b; q = p; |
| 53 | * read_barrier_depends(); |
| 54 | * d = *q; |
| 55 | * </programlisting> |
| 56 | * |
| 57 | * because the read of "*q" depends on the read of "p" and these |
| 58 | * two reads are separated by a read_barrier_depends(). However, |
| 59 | * the following code, with the same initial values for "a" and "b": |
| 60 | * |
| 61 | * <programlisting> |
| 62 | * CPU 0 CPU 1 |
| 63 | * |
| 64 | * a = 2; |
| 65 | * memory_barrier(); |
| 66 | * b = 3; y = b; |
| 67 | * read_barrier_depends(); |
| 68 | * x = a; |
| 69 | * </programlisting> |
| 70 | * |
| 71 | * does not enforce ordering, since there is no data dependency between |
| 72 | * the read of "a" and the read of "b". Therefore, on some CPUs, such |
| 73 | * as Alpha, "y" could be set to 3 and "x" to 0. Use rmb() |
Ralf Baechle | 3fd5646 | 2005-08-16 16:54:12 +0000 | [diff] [blame] | 74 | * in cases like this where there are no data dependencies. |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 75 | */ |
| 76 | |
| 77 | #define read_barrier_depends() do { } while(0) |
| 78 | |
| 79 | #ifdef CONFIG_CPU_HAS_SYNC |
| 80 | #define __sync() \ |
| 81 | __asm__ __volatile__( \ |
| 82 | ".set push\n\t" \ |
| 83 | ".set noreorder\n\t" \ |
| 84 | ".set mips2\n\t" \ |
| 85 | "sync\n\t" \ |
| 86 | ".set pop" \ |
| 87 | : /* no output */ \ |
| 88 | : /* no input */ \ |
| 89 | : "memory") |
| 90 | #else |
| 91 | #define __sync() do { } while(0) |
| 92 | #endif |
| 93 | |
| 94 | #define __fast_iob() \ |
| 95 | __asm__ __volatile__( \ |
| 96 | ".set push\n\t" \ |
| 97 | ".set noreorder\n\t" \ |
| 98 | "lw $0,%0\n\t" \ |
| 99 | "nop\n\t" \ |
| 100 | ".set pop" \ |
| 101 | : /* no output */ \ |
| 102 | : "m" (*(int *)CKSEG1) \ |
| 103 | : "memory") |
| 104 | |
| 105 | #define fast_wmb() __sync() |
| 106 | #define fast_rmb() __sync() |
| 107 | #define fast_mb() __sync() |
| 108 | #define fast_iob() \ |
| 109 | do { \ |
| 110 | __sync(); \ |
| 111 | __fast_iob(); \ |
| 112 | } while (0) |
| 113 | |
| 114 | #ifdef CONFIG_CPU_HAS_WB |
| 115 | |
| 116 | #include <asm/wbflush.h> |
| 117 | |
| 118 | #define wmb() fast_wmb() |
| 119 | #define rmb() fast_rmb() |
| 120 | #define mb() wbflush() |
| 121 | #define iob() wbflush() |
| 122 | |
| 123 | #else /* !CONFIG_CPU_HAS_WB */ |
| 124 | |
| 125 | #define wmb() fast_wmb() |
| 126 | #define rmb() fast_rmb() |
| 127 | #define mb() fast_mb() |
| 128 | #define iob() fast_iob() |
| 129 | |
| 130 | #endif /* !CONFIG_CPU_HAS_WB */ |
| 131 | |
| 132 | #ifdef CONFIG_SMP |
| 133 | #define smp_mb() mb() |
| 134 | #define smp_rmb() rmb() |
| 135 | #define smp_wmb() wmb() |
| 136 | #define smp_read_barrier_depends() read_barrier_depends() |
| 137 | #else |
| 138 | #define smp_mb() barrier() |
| 139 | #define smp_rmb() barrier() |
| 140 | #define smp_wmb() barrier() |
| 141 | #define smp_read_barrier_depends() do { } while(0) |
| 142 | #endif |
| 143 | |
| 144 | #define set_mb(var, value) \ |
| 145 | do { var = value; mb(); } while (0) |
| 146 | |
| 147 | #define set_wmb(var, value) \ |
| 148 | do { var = value; wmb(); } while (0) |
| 149 | |
| 150 | /* |
| 151 | * switch_to(n) should switch tasks to task nr n, first |
| 152 | * checking that n isn't the current task, in which case it does nothing. |
| 153 | */ |
| 154 | extern asmlinkage void *resume(void *last, void *next, void *next_ti); |
| 155 | |
| 156 | struct task_struct; |
| 157 | |
Ralf Baechle | e50c0a8 | 2005-05-31 11:49:19 +0000 | [diff] [blame] | 158 | #define switch_to(prev,next,last) \ |
| 159 | do { \ |
| 160 | if (cpu_has_dsp) \ |
| 161 | __save_dsp(prev); \ |
Al Viro | 40bc9c6 | 2006-01-12 01:06:07 -0800 | [diff] [blame] | 162 | (last) = resume(prev, next, task_thread_info(next)); \ |
Ralf Baechle | e50c0a8 | 2005-05-31 11:49:19 +0000 | [diff] [blame] | 163 | if (cpu_has_dsp) \ |
| 164 | __restore_dsp(current); \ |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 165 | } while(0) |
| 166 | |
Ingo Molnar | 4dc7a0b | 2006-01-12 01:05:27 -0800 | [diff] [blame] | 167 | /* |
| 168 | * On SMP systems, when the scheduler does migration-cost autodetection, |
| 169 | * it needs a way to flush as much of the CPU's caches as possible. |
| 170 | * |
| 171 | * TODO: fill this in! |
| 172 | */ |
| 173 | static inline void sched_cacheflush(void) |
| 174 | { |
| 175 | } |
| 176 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 177 | static inline unsigned long __xchg_u32(volatile int * m, unsigned int val) |
| 178 | { |
| 179 | __u32 retval; |
| 180 | |
| 181 | if (cpu_has_llsc && R10000_LLSC_WAR) { |
| 182 | unsigned long dummy; |
| 183 | |
| 184 | __asm__ __volatile__( |
Maciej W. Rozycki | c4559f6 | 2005-06-23 15:57:15 +0000 | [diff] [blame] | 185 | " .set mips3 \n" |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 186 | "1: ll %0, %3 # xchg_u32 \n" |
Ralf Baechle | 7222424 | 2005-06-29 13:35:19 +0000 | [diff] [blame] | 187 | " .set mips0 \n" |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 188 | " move %2, %z4 \n" |
Ralf Baechle | 7222424 | 2005-06-29 13:35:19 +0000 | [diff] [blame] | 189 | " .set mips3 \n" |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 190 | " sc %2, %1 \n" |
| 191 | " beqzl %2, 1b \n" |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 192 | #ifdef CONFIG_SMP |
| 193 | " sync \n" |
| 194 | #endif |
Maciej W. Rozycki | aac8aa7 | 2005-06-14 17:35:03 +0000 | [diff] [blame] | 195 | " .set mips0 \n" |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 196 | : "=&r" (retval), "=m" (*m), "=&r" (dummy) |
| 197 | : "R" (*m), "Jr" (val) |
| 198 | : "memory"); |
| 199 | } else if (cpu_has_llsc) { |
| 200 | unsigned long dummy; |
| 201 | |
| 202 | __asm__ __volatile__( |
Maciej W. Rozycki | c4559f6 | 2005-06-23 15:57:15 +0000 | [diff] [blame] | 203 | " .set mips3 \n" |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 204 | "1: ll %0, %3 # xchg_u32 \n" |
Ralf Baechle | 7222424 | 2005-06-29 13:35:19 +0000 | [diff] [blame] | 205 | " .set mips0 \n" |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 206 | " move %2, %z4 \n" |
Ralf Baechle | 7222424 | 2005-06-29 13:35:19 +0000 | [diff] [blame] | 207 | " .set mips3 \n" |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 208 | " sc %2, %1 \n" |
| 209 | " beqz %2, 1b \n" |
| 210 | #ifdef CONFIG_SMP |
| 211 | " sync \n" |
| 212 | #endif |
Maciej W. Rozycki | aac8aa7 | 2005-06-14 17:35:03 +0000 | [diff] [blame] | 213 | " .set mips0 \n" |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 214 | : "=&r" (retval), "=m" (*m), "=&r" (dummy) |
| 215 | : "R" (*m), "Jr" (val) |
| 216 | : "memory"); |
| 217 | } else { |
| 218 | unsigned long flags; |
| 219 | |
| 220 | local_irq_save(flags); |
| 221 | retval = *m; |
| 222 | *m = val; |
| 223 | local_irq_restore(flags); /* implies memory barrier */ |
| 224 | } |
| 225 | |
| 226 | return retval; |
| 227 | } |
| 228 | |
Ralf Baechle | 875d43e | 2005-09-03 15:56:16 -0700 | [diff] [blame] | 229 | #ifdef CONFIG_64BIT |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 230 | static inline __u64 __xchg_u64(volatile __u64 * m, __u64 val) |
| 231 | { |
| 232 | __u64 retval; |
| 233 | |
| 234 | if (cpu_has_llsc && R10000_LLSC_WAR) { |
| 235 | unsigned long dummy; |
| 236 | |
| 237 | __asm__ __volatile__( |
Maciej W. Rozycki | aac8aa7 | 2005-06-14 17:35:03 +0000 | [diff] [blame] | 238 | " .set mips3 \n" |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 239 | "1: lld %0, %3 # xchg_u64 \n" |
| 240 | " move %2, %z4 \n" |
| 241 | " scd %2, %1 \n" |
| 242 | " beqzl %2, 1b \n" |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 243 | #ifdef CONFIG_SMP |
| 244 | " sync \n" |
| 245 | #endif |
Maciej W. Rozycki | aac8aa7 | 2005-06-14 17:35:03 +0000 | [diff] [blame] | 246 | " .set mips0 \n" |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 247 | : "=&r" (retval), "=m" (*m), "=&r" (dummy) |
| 248 | : "R" (*m), "Jr" (val) |
| 249 | : "memory"); |
| 250 | } else if (cpu_has_llsc) { |
| 251 | unsigned long dummy; |
| 252 | |
| 253 | __asm__ __volatile__( |
Maciej W. Rozycki | aac8aa7 | 2005-06-14 17:35:03 +0000 | [diff] [blame] | 254 | " .set mips3 \n" |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 255 | "1: lld %0, %3 # xchg_u64 \n" |
| 256 | " move %2, %z4 \n" |
| 257 | " scd %2, %1 \n" |
| 258 | " beqz %2, 1b \n" |
| 259 | #ifdef CONFIG_SMP |
| 260 | " sync \n" |
| 261 | #endif |
Maciej W. Rozycki | aac8aa7 | 2005-06-14 17:35:03 +0000 | [diff] [blame] | 262 | " .set mips0 \n" |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 263 | : "=&r" (retval), "=m" (*m), "=&r" (dummy) |
| 264 | : "R" (*m), "Jr" (val) |
| 265 | : "memory"); |
| 266 | } else { |
| 267 | unsigned long flags; |
| 268 | |
| 269 | local_irq_save(flags); |
| 270 | retval = *m; |
| 271 | *m = val; |
| 272 | local_irq_restore(flags); /* implies memory barrier */ |
| 273 | } |
| 274 | |
| 275 | return retval; |
| 276 | } |
| 277 | #else |
| 278 | extern __u64 __xchg_u64_unsupported_on_32bit_kernels(volatile __u64 * m, __u64 val); |
| 279 | #define __xchg_u64 __xchg_u64_unsupported_on_32bit_kernels |
| 280 | #endif |
| 281 | |
| 282 | /* This function doesn't exist, so you'll get a linker error |
| 283 | if something tries to do an invalid xchg(). */ |
| 284 | extern void __xchg_called_with_bad_pointer(void); |
| 285 | |
| 286 | static inline unsigned long __xchg(unsigned long x, volatile void * ptr, int size) |
| 287 | { |
| 288 | switch (size) { |
| 289 | case 4: |
| 290 | return __xchg_u32(ptr, x); |
| 291 | case 8: |
| 292 | return __xchg_u64(ptr, x); |
| 293 | } |
| 294 | __xchg_called_with_bad_pointer(); |
| 295 | return x; |
| 296 | } |
| 297 | |
| 298 | #define xchg(ptr,x) ((__typeof__(*(ptr)))__xchg((unsigned long)(x),(ptr),sizeof(*(ptr)))) |
| 299 | #define tas(ptr) (xchg((ptr),1)) |
| 300 | |
| 301 | #define __HAVE_ARCH_CMPXCHG 1 |
| 302 | |
| 303 | static inline unsigned long __cmpxchg_u32(volatile int * m, unsigned long old, |
| 304 | unsigned long new) |
| 305 | { |
| 306 | __u32 retval; |
| 307 | |
| 308 | if (cpu_has_llsc && R10000_LLSC_WAR) { |
| 309 | __asm__ __volatile__( |
Maciej W. Rozycki | aac8aa7 | 2005-06-14 17:35:03 +0000 | [diff] [blame] | 310 | " .set push \n" |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 311 | " .set noat \n" |
Maciej W. Rozycki | c4559f6 | 2005-06-23 15:57:15 +0000 | [diff] [blame] | 312 | " .set mips3 \n" |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 313 | "1: ll %0, %2 # __cmpxchg_u32 \n" |
| 314 | " bne %0, %z3, 2f \n" |
Ralf Baechle | f99d302 | 2005-08-25 16:22:09 +0000 | [diff] [blame] | 315 | " .set mips0 \n" |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 316 | " move $1, %z4 \n" |
Ralf Baechle | f99d302 | 2005-08-25 16:22:09 +0000 | [diff] [blame] | 317 | " .set mips3 \n" |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 318 | " sc $1, %1 \n" |
| 319 | " beqzl $1, 1b \n" |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 320 | #ifdef CONFIG_SMP |
| 321 | " sync \n" |
| 322 | #endif |
| 323 | "2: \n" |
Maciej W. Rozycki | aac8aa7 | 2005-06-14 17:35:03 +0000 | [diff] [blame] | 324 | " .set pop \n" |
Ralf Baechle | 3e6cb2d | 2006-02-21 18:32:14 +0000 | [diff] [blame] | 325 | : "=&r" (retval), "=R" (*m) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 326 | : "R" (*m), "Jr" (old), "Jr" (new) |
| 327 | : "memory"); |
| 328 | } else if (cpu_has_llsc) { |
| 329 | __asm__ __volatile__( |
Maciej W. Rozycki | aac8aa7 | 2005-06-14 17:35:03 +0000 | [diff] [blame] | 330 | " .set push \n" |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 331 | " .set noat \n" |
Maciej W. Rozycki | c4559f6 | 2005-06-23 15:57:15 +0000 | [diff] [blame] | 332 | " .set mips3 \n" |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 333 | "1: ll %0, %2 # __cmpxchg_u32 \n" |
| 334 | " bne %0, %z3, 2f \n" |
Ralf Baechle | f99d302 | 2005-08-25 16:22:09 +0000 | [diff] [blame] | 335 | " .set mips0 \n" |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 336 | " move $1, %z4 \n" |
Ralf Baechle | f99d302 | 2005-08-25 16:22:09 +0000 | [diff] [blame] | 337 | " .set mips3 \n" |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 338 | " sc $1, %1 \n" |
| 339 | " beqz $1, 1b \n" |
| 340 | #ifdef CONFIG_SMP |
| 341 | " sync \n" |
| 342 | #endif |
| 343 | "2: \n" |
Maciej W. Rozycki | aac8aa7 | 2005-06-14 17:35:03 +0000 | [diff] [blame] | 344 | " .set pop \n" |
Ralf Baechle | 3e6cb2d | 2006-02-21 18:32:14 +0000 | [diff] [blame] | 345 | : "=&r" (retval), "=R" (*m) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 346 | : "R" (*m), "Jr" (old), "Jr" (new) |
| 347 | : "memory"); |
| 348 | } else { |
| 349 | unsigned long flags; |
| 350 | |
| 351 | local_irq_save(flags); |
| 352 | retval = *m; |
| 353 | if (retval == old) |
| 354 | *m = new; |
| 355 | local_irq_restore(flags); /* implies memory barrier */ |
| 356 | } |
| 357 | |
| 358 | return retval; |
| 359 | } |
| 360 | |
Ralf Baechle | 875d43e | 2005-09-03 15:56:16 -0700 | [diff] [blame] | 361 | #ifdef CONFIG_64BIT |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 362 | static inline unsigned long __cmpxchg_u64(volatile int * m, unsigned long old, |
| 363 | unsigned long new) |
| 364 | { |
| 365 | __u64 retval; |
| 366 | |
| 367 | if (cpu_has_llsc) { |
| 368 | __asm__ __volatile__( |
Maciej W. Rozycki | aac8aa7 | 2005-06-14 17:35:03 +0000 | [diff] [blame] | 369 | " .set push \n" |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 370 | " .set noat \n" |
Maciej W. Rozycki | aac8aa7 | 2005-06-14 17:35:03 +0000 | [diff] [blame] | 371 | " .set mips3 \n" |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 372 | "1: lld %0, %2 # __cmpxchg_u64 \n" |
| 373 | " bne %0, %z3, 2f \n" |
| 374 | " move $1, %z4 \n" |
| 375 | " scd $1, %1 \n" |
| 376 | " beqzl $1, 1b \n" |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 377 | #ifdef CONFIG_SMP |
| 378 | " sync \n" |
| 379 | #endif |
| 380 | "2: \n" |
Maciej W. Rozycki | aac8aa7 | 2005-06-14 17:35:03 +0000 | [diff] [blame] | 381 | " .set pop \n" |
Ralf Baechle | 3e6cb2d | 2006-02-21 18:32:14 +0000 | [diff] [blame] | 382 | : "=&r" (retval), "=R" (*m) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 383 | : "R" (*m), "Jr" (old), "Jr" (new) |
| 384 | : "memory"); |
| 385 | } else if (cpu_has_llsc) { |
| 386 | __asm__ __volatile__( |
Maciej W. Rozycki | aac8aa7 | 2005-06-14 17:35:03 +0000 | [diff] [blame] | 387 | " .set push \n" |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 388 | " .set noat \n" |
Maciej W. Rozycki | c4559f6 | 2005-06-23 15:57:15 +0000 | [diff] [blame] | 389 | " .set mips3 \n" |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 390 | "1: lld %0, %2 # __cmpxchg_u64 \n" |
| 391 | " bne %0, %z3, 2f \n" |
| 392 | " move $1, %z4 \n" |
| 393 | " scd $1, %1 \n" |
| 394 | " beqz $1, 1b \n" |
| 395 | #ifdef CONFIG_SMP |
| 396 | " sync \n" |
| 397 | #endif |
| 398 | "2: \n" |
Maciej W. Rozycki | aac8aa7 | 2005-06-14 17:35:03 +0000 | [diff] [blame] | 399 | " .set pop \n" |
Ralf Baechle | 3e6cb2d | 2006-02-21 18:32:14 +0000 | [diff] [blame] | 400 | : "=&r" (retval), "=R" (*m) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 401 | : "R" (*m), "Jr" (old), "Jr" (new) |
| 402 | : "memory"); |
| 403 | } else { |
| 404 | unsigned long flags; |
| 405 | |
| 406 | local_irq_save(flags); |
| 407 | retval = *m; |
| 408 | if (retval == old) |
| 409 | *m = new; |
| 410 | local_irq_restore(flags); /* implies memory barrier */ |
| 411 | } |
| 412 | |
| 413 | return retval; |
| 414 | } |
| 415 | #else |
| 416 | extern unsigned long __cmpxchg_u64_unsupported_on_32bit_kernels( |
| 417 | volatile int * m, unsigned long old, unsigned long new); |
| 418 | #define __cmpxchg_u64 __cmpxchg_u64_unsupported_on_32bit_kernels |
| 419 | #endif |
| 420 | |
| 421 | /* This function doesn't exist, so you'll get a linker error |
| 422 | if something tries to do an invalid cmpxchg(). */ |
| 423 | extern void __cmpxchg_called_with_bad_pointer(void); |
| 424 | |
| 425 | static inline unsigned long __cmpxchg(volatile void * ptr, unsigned long old, |
| 426 | unsigned long new, int size) |
| 427 | { |
| 428 | switch (size) { |
| 429 | case 4: |
| 430 | return __cmpxchg_u32(ptr, old, new); |
| 431 | case 8: |
| 432 | return __cmpxchg_u64(ptr, old, new); |
| 433 | } |
| 434 | __cmpxchg_called_with_bad_pointer(); |
| 435 | return old; |
| 436 | } |
| 437 | |
| 438 | #define cmpxchg(ptr,old,new) ((__typeof__(*(ptr)))__cmpxchg((ptr), (unsigned long)(old), (unsigned long)(new),sizeof(*(ptr)))) |
| 439 | |
Ralf Baechle | e01402b | 2005-07-14 15:57:16 +0000 | [diff] [blame] | 440 | extern void set_handler (unsigned long offset, void *addr, unsigned long len); |
| 441 | extern void set_uncached_handler (unsigned long offset, void *addr, unsigned long len); |
| 442 | extern void *set_vi_handler (int n, void *addr); |
| 443 | extern void *set_vi_srs_handler (int n, void *addr, int regset); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 444 | extern void *set_except_vector(int n, void *addr); |
| 445 | extern void per_cpu_trap_init(void); |
| 446 | |
Ralf Baechle | 178086c | 2005-10-13 17:07:54 +0100 | [diff] [blame] | 447 | extern NORET_TYPE void die(const char *, struct pt_regs *); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 448 | |
Ralf Baechle | 178086c | 2005-10-13 17:07:54 +0100 | [diff] [blame] | 449 | static inline void die_if_kernel(const char *str, struct pt_regs *regs) |
| 450 | { |
| 451 | if (unlikely(!user_mode(regs))) |
| 452 | die(str, regs); |
| 453 | } |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 454 | |
| 455 | extern int stop_a_enabled; |
| 456 | |
| 457 | /* |
Nick Piggin | 4866cde | 2005-06-25 14:57:23 -0700 | [diff] [blame] | 458 | * See include/asm-ia64/system.h; prevents deadlock on SMP |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 459 | * systems. |
| 460 | */ |
Nick Piggin | 4866cde | 2005-06-25 14:57:23 -0700 | [diff] [blame] | 461 | #define __ARCH_WANT_UNLOCKED_CTXSW |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 462 | |
| 463 | #define arch_align_stack(x) (x) |
| 464 | |
| 465 | #endif /* _ASM_SYSTEM_H */ |