Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1 | /* |
| 2 | * This file is subject to the terms and conditions of the GNU General Public |
| 3 | * License. See the file "COPYING" in the main directory of this archive |
| 4 | * for more details. |
| 5 | * |
Ralf Baechle | 102fa15 | 2007-02-16 17:18:50 +0000 | [diff] [blame] | 6 | * Copyright (c) 1994 - 1997, 99, 2000, 06, 07 Ralf Baechle (ralf@linux-mips.org) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 7 | * Copyright (c) 1999, 2000 Silicon Graphics, Inc. |
| 8 | */ |
| 9 | #ifndef _ASM_BITOPS_H |
| 10 | #define _ASM_BITOPS_H |
| 11 | |
Jiri Slaby | 0624517 | 2007-10-18 23:40:26 -0700 | [diff] [blame^] | 12 | #ifndef _LINUX_BITOPS_H |
| 13 | #error only <linux/bitops.h> can be included directly |
| 14 | #endif |
| 15 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 16 | #include <linux/compiler.h> |
Ralf Baechle | 4ffd8b3 | 2006-11-30 01:14:50 +0000 | [diff] [blame] | 17 | #include <linux/irqflags.h> |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 18 | #include <linux/types.h> |
Ralf Baechle | 0004a9d | 2006-10-31 03:45:07 +0000 | [diff] [blame] | 19 | #include <asm/barrier.h> |
Ralf Baechle | ec917c2c | 2005-10-07 16:58:15 +0100 | [diff] [blame] | 20 | #include <asm/bug.h> |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 21 | #include <asm/byteorder.h> /* sigh ... */ |
| 22 | #include <asm/cpu-features.h> |
Ralf Baechle | 4ffd8b3 | 2006-11-30 01:14:50 +0000 | [diff] [blame] | 23 | #include <asm/sgidefs.h> |
| 24 | #include <asm/war.h> |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 25 | |
Ralf Baechle | 49a89ef | 2007-10-11 23:46:15 +0100 | [diff] [blame] | 26 | #if _MIPS_SZLONG == 32 |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 27 | #define SZLONG_LOG 5 |
| 28 | #define SZLONG_MASK 31UL |
Maciej W. Rozycki | aac8aa7 | 2005-06-14 17:35:03 +0000 | [diff] [blame] | 29 | #define __LL "ll " |
| 30 | #define __SC "sc " |
Ralf Baechle | 102fa15 | 2007-02-16 17:18:50 +0000 | [diff] [blame] | 31 | #define __INS "ins " |
| 32 | #define __EXT "ext " |
Ralf Baechle | 49a89ef | 2007-10-11 23:46:15 +0100 | [diff] [blame] | 33 | #elif _MIPS_SZLONG == 64 |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 34 | #define SZLONG_LOG 6 |
| 35 | #define SZLONG_MASK 63UL |
Maciej W. Rozycki | aac8aa7 | 2005-06-14 17:35:03 +0000 | [diff] [blame] | 36 | #define __LL "lld " |
| 37 | #define __SC "scd " |
Ralf Baechle | 102fa15 | 2007-02-16 17:18:50 +0000 | [diff] [blame] | 38 | #define __INS "dins " |
| 39 | #define __EXT "dext " |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 40 | #endif |
| 41 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 42 | /* |
| 43 | * clear_bit() doesn't provide any barrier for the compiler. |
| 44 | */ |
Ralf Baechle | 17099b1 | 2007-07-14 13:24:05 +0100 | [diff] [blame] | 45 | #define smp_mb__before_clear_bit() smp_llsc_mb() |
| 46 | #define smp_mb__after_clear_bit() smp_llsc_mb() |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 47 | |
| 48 | /* |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 49 | * set_bit - Atomically set a bit in memory |
| 50 | * @nr: the bit to set |
| 51 | * @addr: the address to start counting from |
| 52 | * |
| 53 | * This function is atomic and may not be reordered. See __set_bit() |
| 54 | * if you do not require the atomic guarantees. |
| 55 | * Note that @nr may be almost arbitrarily large; this function is not |
| 56 | * restricted to acting on a single-word quantity. |
| 57 | */ |
| 58 | static inline void set_bit(unsigned long nr, volatile unsigned long *addr) |
| 59 | { |
| 60 | unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG); |
Ralf Baechle | b961153 | 2007-03-05 00:56:15 +0000 | [diff] [blame] | 61 | unsigned short bit = nr & SZLONG_MASK; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 62 | unsigned long temp; |
| 63 | |
| 64 | if (cpu_has_llsc && R10000_LLSC_WAR) { |
| 65 | __asm__ __volatile__( |
Maciej W. Rozycki | c4559f6 | 2005-06-23 15:57:15 +0000 | [diff] [blame] | 66 | " .set mips3 \n" |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 67 | "1: " __LL "%0, %1 # set_bit \n" |
| 68 | " or %0, %2 \n" |
Maciej W. Rozycki | aac8aa7 | 2005-06-14 17:35:03 +0000 | [diff] [blame] | 69 | " " __SC "%0, %1 \n" |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 70 | " beqzl %0, 1b \n" |
Maciej W. Rozycki | aac8aa7 | 2005-06-14 17:35:03 +0000 | [diff] [blame] | 71 | " .set mips0 \n" |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 72 | : "=&r" (temp), "=m" (*m) |
Ralf Baechle | b961153 | 2007-03-05 00:56:15 +0000 | [diff] [blame] | 73 | : "ir" (1UL << bit), "m" (*m)); |
Ralf Baechle | 102fa15 | 2007-02-16 17:18:50 +0000 | [diff] [blame] | 74 | #ifdef CONFIG_CPU_MIPSR2 |
Ralf Baechle | b961153 | 2007-03-05 00:56:15 +0000 | [diff] [blame] | 75 | } else if (__builtin_constant_p(bit)) { |
Ralf Baechle | 102fa15 | 2007-02-16 17:18:50 +0000 | [diff] [blame] | 76 | __asm__ __volatile__( |
| 77 | "1: " __LL "%0, %1 # set_bit \n" |
| 78 | " " __INS "%0, %4, %2, 1 \n" |
| 79 | " " __SC "%0, %1 \n" |
| 80 | " beqz %0, 2f \n" |
| 81 | " .subsection 2 \n" |
| 82 | "2: b 1b \n" |
| 83 | " .previous \n" |
| 84 | : "=&r" (temp), "=m" (*m) |
Ralf Baechle | b961153 | 2007-03-05 00:56:15 +0000 | [diff] [blame] | 85 | : "ir" (bit), "m" (*m), "r" (~0)); |
Ralf Baechle | 102fa15 | 2007-02-16 17:18:50 +0000 | [diff] [blame] | 86 | #endif /* CONFIG_CPU_MIPSR2 */ |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 87 | } else if (cpu_has_llsc) { |
| 88 | __asm__ __volatile__( |
Maciej W. Rozycki | c4559f6 | 2005-06-23 15:57:15 +0000 | [diff] [blame] | 89 | " .set mips3 \n" |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 90 | "1: " __LL "%0, %1 # set_bit \n" |
| 91 | " or %0, %2 \n" |
Maciej W. Rozycki | aac8aa7 | 2005-06-14 17:35:03 +0000 | [diff] [blame] | 92 | " " __SC "%0, %1 \n" |
Ralf Baechle | f65e4fa | 2006-09-28 01:45:21 +0100 | [diff] [blame] | 93 | " beqz %0, 2f \n" |
| 94 | " .subsection 2 \n" |
| 95 | "2: b 1b \n" |
| 96 | " .previous \n" |
Maciej W. Rozycki | aac8aa7 | 2005-06-14 17:35:03 +0000 | [diff] [blame] | 97 | " .set mips0 \n" |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 98 | : "=&r" (temp), "=m" (*m) |
Ralf Baechle | b961153 | 2007-03-05 00:56:15 +0000 | [diff] [blame] | 99 | : "ir" (1UL << bit), "m" (*m)); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 100 | } else { |
| 101 | volatile unsigned long *a = addr; |
| 102 | unsigned long mask; |
Ralf Baechle | 4ffd8b3 | 2006-11-30 01:14:50 +0000 | [diff] [blame] | 103 | unsigned long flags; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 104 | |
| 105 | a += nr >> SZLONG_LOG; |
Ralf Baechle | b961153 | 2007-03-05 00:56:15 +0000 | [diff] [blame] | 106 | mask = 1UL << bit; |
Ralf Baechle | 49edd09 | 2007-03-16 16:10:36 +0000 | [diff] [blame] | 107 | raw_local_irq_save(flags); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 108 | *a |= mask; |
Ralf Baechle | 49edd09 | 2007-03-16 16:10:36 +0000 | [diff] [blame] | 109 | raw_local_irq_restore(flags); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 110 | } |
| 111 | } |
| 112 | |
| 113 | /* |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 114 | * clear_bit - Clears a bit in memory |
| 115 | * @nr: Bit to clear |
| 116 | * @addr: Address to start counting from |
| 117 | * |
| 118 | * clear_bit() is atomic and may not be reordered. However, it does |
| 119 | * not contain a memory barrier, so if it is used for locking purposes, |
| 120 | * you should call smp_mb__before_clear_bit() and/or smp_mb__after_clear_bit() |
| 121 | * in order to ensure changes are visible on other processors. |
| 122 | */ |
| 123 | static inline void clear_bit(unsigned long nr, volatile unsigned long *addr) |
| 124 | { |
| 125 | unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG); |
Ralf Baechle | b961153 | 2007-03-05 00:56:15 +0000 | [diff] [blame] | 126 | unsigned short bit = nr & SZLONG_MASK; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 127 | unsigned long temp; |
| 128 | |
| 129 | if (cpu_has_llsc && R10000_LLSC_WAR) { |
| 130 | __asm__ __volatile__( |
Maciej W. Rozycki | c4559f6 | 2005-06-23 15:57:15 +0000 | [diff] [blame] | 131 | " .set mips3 \n" |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 132 | "1: " __LL "%0, %1 # clear_bit \n" |
| 133 | " and %0, %2 \n" |
| 134 | " " __SC "%0, %1 \n" |
| 135 | " beqzl %0, 1b \n" |
Maciej W. Rozycki | aac8aa7 | 2005-06-14 17:35:03 +0000 | [diff] [blame] | 136 | " .set mips0 \n" |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 137 | : "=&r" (temp), "=m" (*m) |
Ralf Baechle | b961153 | 2007-03-05 00:56:15 +0000 | [diff] [blame] | 138 | : "ir" (~(1UL << bit)), "m" (*m)); |
Ralf Baechle | 102fa15 | 2007-02-16 17:18:50 +0000 | [diff] [blame] | 139 | #ifdef CONFIG_CPU_MIPSR2 |
Ralf Baechle | b961153 | 2007-03-05 00:56:15 +0000 | [diff] [blame] | 140 | } else if (__builtin_constant_p(bit)) { |
Ralf Baechle | 102fa15 | 2007-02-16 17:18:50 +0000 | [diff] [blame] | 141 | __asm__ __volatile__( |
| 142 | "1: " __LL "%0, %1 # clear_bit \n" |
| 143 | " " __INS "%0, $0, %2, 1 \n" |
| 144 | " " __SC "%0, %1 \n" |
| 145 | " beqz %0, 2f \n" |
| 146 | " .subsection 2 \n" |
| 147 | "2: b 1b \n" |
| 148 | " .previous \n" |
| 149 | : "=&r" (temp), "=m" (*m) |
Ralf Baechle | b961153 | 2007-03-05 00:56:15 +0000 | [diff] [blame] | 150 | : "ir" (bit), "m" (*m)); |
Ralf Baechle | 102fa15 | 2007-02-16 17:18:50 +0000 | [diff] [blame] | 151 | #endif /* CONFIG_CPU_MIPSR2 */ |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 152 | } else if (cpu_has_llsc) { |
| 153 | __asm__ __volatile__( |
Maciej W. Rozycki | c4559f6 | 2005-06-23 15:57:15 +0000 | [diff] [blame] | 154 | " .set mips3 \n" |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 155 | "1: " __LL "%0, %1 # clear_bit \n" |
| 156 | " and %0, %2 \n" |
| 157 | " " __SC "%0, %1 \n" |
Ralf Baechle | f65e4fa | 2006-09-28 01:45:21 +0100 | [diff] [blame] | 158 | " beqz %0, 2f \n" |
| 159 | " .subsection 2 \n" |
| 160 | "2: b 1b \n" |
| 161 | " .previous \n" |
Maciej W. Rozycki | aac8aa7 | 2005-06-14 17:35:03 +0000 | [diff] [blame] | 162 | " .set mips0 \n" |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 163 | : "=&r" (temp), "=m" (*m) |
Ralf Baechle | b961153 | 2007-03-05 00:56:15 +0000 | [diff] [blame] | 164 | : "ir" (~(1UL << bit)), "m" (*m)); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 165 | } else { |
| 166 | volatile unsigned long *a = addr; |
| 167 | unsigned long mask; |
Ralf Baechle | 4ffd8b3 | 2006-11-30 01:14:50 +0000 | [diff] [blame] | 168 | unsigned long flags; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 169 | |
| 170 | a += nr >> SZLONG_LOG; |
Ralf Baechle | b961153 | 2007-03-05 00:56:15 +0000 | [diff] [blame] | 171 | mask = 1UL << bit; |
Ralf Baechle | 49edd09 | 2007-03-16 16:10:36 +0000 | [diff] [blame] | 172 | raw_local_irq_save(flags); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 173 | *a &= ~mask; |
Ralf Baechle | 49edd09 | 2007-03-16 16:10:36 +0000 | [diff] [blame] | 174 | raw_local_irq_restore(flags); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 175 | } |
| 176 | } |
| 177 | |
| 178 | /* |
Nick Piggin | 728697c | 2007-10-18 03:06:53 -0700 | [diff] [blame] | 179 | * clear_bit_unlock - Clears a bit in memory |
| 180 | * @nr: Bit to clear |
| 181 | * @addr: Address to start counting from |
| 182 | * |
| 183 | * clear_bit() is atomic and implies release semantics before the memory |
| 184 | * operation. It can be used for an unlock. |
| 185 | */ |
| 186 | static inline void clear_bit_unlock(unsigned long nr, volatile unsigned long *addr) |
| 187 | { |
| 188 | smp_mb__before_clear_bit(); |
| 189 | clear_bit(nr, addr); |
| 190 | } |
| 191 | |
| 192 | /* |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 193 | * change_bit - Toggle a bit in memory |
| 194 | * @nr: Bit to change |
| 195 | * @addr: Address to start counting from |
| 196 | * |
| 197 | * change_bit() is atomic and may not be reordered. |
| 198 | * Note that @nr may be almost arbitrarily large; this function is not |
| 199 | * restricted to acting on a single-word quantity. |
| 200 | */ |
| 201 | static inline void change_bit(unsigned long nr, volatile unsigned long *addr) |
| 202 | { |
Ralf Baechle | b961153 | 2007-03-05 00:56:15 +0000 | [diff] [blame] | 203 | unsigned short bit = nr & SZLONG_MASK; |
| 204 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 205 | if (cpu_has_llsc && R10000_LLSC_WAR) { |
| 206 | unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG); |
| 207 | unsigned long temp; |
| 208 | |
| 209 | __asm__ __volatile__( |
Maciej W. Rozycki | c4559f6 | 2005-06-23 15:57:15 +0000 | [diff] [blame] | 210 | " .set mips3 \n" |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 211 | "1: " __LL "%0, %1 # change_bit \n" |
| 212 | " xor %0, %2 \n" |
Maciej W. Rozycki | aac8aa7 | 2005-06-14 17:35:03 +0000 | [diff] [blame] | 213 | " " __SC "%0, %1 \n" |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 214 | " beqzl %0, 1b \n" |
Maciej W. Rozycki | aac8aa7 | 2005-06-14 17:35:03 +0000 | [diff] [blame] | 215 | " .set mips0 \n" |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 216 | : "=&r" (temp), "=m" (*m) |
Ralf Baechle | b961153 | 2007-03-05 00:56:15 +0000 | [diff] [blame] | 217 | : "ir" (1UL << bit), "m" (*m)); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 218 | } else if (cpu_has_llsc) { |
| 219 | unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG); |
| 220 | unsigned long temp; |
| 221 | |
| 222 | __asm__ __volatile__( |
Maciej W. Rozycki | c4559f6 | 2005-06-23 15:57:15 +0000 | [diff] [blame] | 223 | " .set mips3 \n" |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 224 | "1: " __LL "%0, %1 # change_bit \n" |
| 225 | " xor %0, %2 \n" |
Maciej W. Rozycki | aac8aa7 | 2005-06-14 17:35:03 +0000 | [diff] [blame] | 226 | " " __SC "%0, %1 \n" |
Ralf Baechle | f65e4fa | 2006-09-28 01:45:21 +0100 | [diff] [blame] | 227 | " beqz %0, 2f \n" |
| 228 | " .subsection 2 \n" |
| 229 | "2: b 1b \n" |
| 230 | " .previous \n" |
Maciej W. Rozycki | aac8aa7 | 2005-06-14 17:35:03 +0000 | [diff] [blame] | 231 | " .set mips0 \n" |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 232 | : "=&r" (temp), "=m" (*m) |
Ralf Baechle | b961153 | 2007-03-05 00:56:15 +0000 | [diff] [blame] | 233 | : "ir" (1UL << bit), "m" (*m)); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 234 | } else { |
| 235 | volatile unsigned long *a = addr; |
| 236 | unsigned long mask; |
Ralf Baechle | 4ffd8b3 | 2006-11-30 01:14:50 +0000 | [diff] [blame] | 237 | unsigned long flags; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 238 | |
| 239 | a += nr >> SZLONG_LOG; |
Ralf Baechle | b961153 | 2007-03-05 00:56:15 +0000 | [diff] [blame] | 240 | mask = 1UL << bit; |
Ralf Baechle | 49edd09 | 2007-03-16 16:10:36 +0000 | [diff] [blame] | 241 | raw_local_irq_save(flags); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 242 | *a ^= mask; |
Ralf Baechle | 49edd09 | 2007-03-16 16:10:36 +0000 | [diff] [blame] | 243 | raw_local_irq_restore(flags); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 244 | } |
| 245 | } |
| 246 | |
| 247 | /* |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 248 | * test_and_set_bit - Set a bit and return its old value |
| 249 | * @nr: Bit to set |
| 250 | * @addr: Address to count from |
| 251 | * |
| 252 | * This operation is atomic and cannot be reordered. |
| 253 | * It also implies a memory barrier. |
| 254 | */ |
| 255 | static inline int test_and_set_bit(unsigned long nr, |
| 256 | volatile unsigned long *addr) |
| 257 | { |
Ralf Baechle | b961153 | 2007-03-05 00:56:15 +0000 | [diff] [blame] | 258 | unsigned short bit = nr & SZLONG_MASK; |
Ralf Baechle | ff72b7a | 2007-06-07 13:17:30 +0100 | [diff] [blame] | 259 | unsigned long res; |
Ralf Baechle | b961153 | 2007-03-05 00:56:15 +0000 | [diff] [blame] | 260 | |
Nick Piggin | c8f30ae | 2007-10-18 03:06:52 -0700 | [diff] [blame] | 261 | smp_llsc_mb(); |
| 262 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 263 | if (cpu_has_llsc && R10000_LLSC_WAR) { |
| 264 | unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG); |
Ralf Baechle | ff72b7a | 2007-06-07 13:17:30 +0100 | [diff] [blame] | 265 | unsigned long temp; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 266 | |
| 267 | __asm__ __volatile__( |
Maciej W. Rozycki | c4559f6 | 2005-06-23 15:57:15 +0000 | [diff] [blame] | 268 | " .set mips3 \n" |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 269 | "1: " __LL "%0, %1 # test_and_set_bit \n" |
| 270 | " or %2, %0, %3 \n" |
| 271 | " " __SC "%2, %1 \n" |
| 272 | " beqzl %2, 1b \n" |
| 273 | " and %2, %0, %3 \n" |
Maciej W. Rozycki | aac8aa7 | 2005-06-14 17:35:03 +0000 | [diff] [blame] | 274 | " .set mips0 \n" |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 275 | : "=&r" (temp), "=m" (*m), "=&r" (res) |
Ralf Baechle | b961153 | 2007-03-05 00:56:15 +0000 | [diff] [blame] | 276 | : "r" (1UL << bit), "m" (*m) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 277 | : "memory"); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 278 | } else if (cpu_has_llsc) { |
| 279 | unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG); |
Ralf Baechle | ff72b7a | 2007-06-07 13:17:30 +0100 | [diff] [blame] | 280 | unsigned long temp; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 281 | |
| 282 | __asm__ __volatile__( |
Maciej W. Rozycki | aac8aa7 | 2005-06-14 17:35:03 +0000 | [diff] [blame] | 283 | " .set push \n" |
| 284 | " .set noreorder \n" |
Maciej W. Rozycki | c4559f6 | 2005-06-23 15:57:15 +0000 | [diff] [blame] | 285 | " .set mips3 \n" |
Maciej W. Rozycki | aac8aa7 | 2005-06-14 17:35:03 +0000 | [diff] [blame] | 286 | "1: " __LL "%0, %1 # test_and_set_bit \n" |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 287 | " or %2, %0, %3 \n" |
| 288 | " " __SC "%2, %1 \n" |
Ralf Baechle | f65e4fa | 2006-09-28 01:45:21 +0100 | [diff] [blame] | 289 | " beqz %2, 2f \n" |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 290 | " and %2, %0, %3 \n" |
Ralf Baechle | f65e4fa | 2006-09-28 01:45:21 +0100 | [diff] [blame] | 291 | " .subsection 2 \n" |
| 292 | "2: b 1b \n" |
| 293 | " nop \n" |
| 294 | " .previous \n" |
Maciej W. Rozycki | aac8aa7 | 2005-06-14 17:35:03 +0000 | [diff] [blame] | 295 | " .set pop \n" |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 296 | : "=&r" (temp), "=m" (*m), "=&r" (res) |
Ralf Baechle | b961153 | 2007-03-05 00:56:15 +0000 | [diff] [blame] | 297 | : "r" (1UL << bit), "m" (*m) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 298 | : "memory"); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 299 | } else { |
| 300 | volatile unsigned long *a = addr; |
| 301 | unsigned long mask; |
Ralf Baechle | 4ffd8b3 | 2006-11-30 01:14:50 +0000 | [diff] [blame] | 302 | unsigned long flags; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 303 | |
| 304 | a += nr >> SZLONG_LOG; |
Ralf Baechle | b961153 | 2007-03-05 00:56:15 +0000 | [diff] [blame] | 305 | mask = 1UL << bit; |
Ralf Baechle | 49edd09 | 2007-03-16 16:10:36 +0000 | [diff] [blame] | 306 | raw_local_irq_save(flags); |
Ralf Baechle | ff72b7a | 2007-06-07 13:17:30 +0100 | [diff] [blame] | 307 | res = (mask & *a); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 308 | *a |= mask; |
Ralf Baechle | 49edd09 | 2007-03-16 16:10:36 +0000 | [diff] [blame] | 309 | raw_local_irq_restore(flags); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 310 | } |
Ralf Baechle | 0004a9d | 2006-10-31 03:45:07 +0000 | [diff] [blame] | 311 | |
Ralf Baechle | 17099b1 | 2007-07-14 13:24:05 +0100 | [diff] [blame] | 312 | smp_llsc_mb(); |
Ralf Baechle | ff72b7a | 2007-06-07 13:17:30 +0100 | [diff] [blame] | 313 | |
| 314 | return res != 0; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 315 | } |
| 316 | |
| 317 | /* |
Nick Piggin | 728697c | 2007-10-18 03:06:53 -0700 | [diff] [blame] | 318 | * test_and_set_bit_lock - Set a bit and return its old value |
| 319 | * @nr: Bit to set |
| 320 | * @addr: Address to count from |
| 321 | * |
| 322 | * This operation is atomic and implies acquire ordering semantics |
| 323 | * after the memory operation. |
| 324 | */ |
| 325 | static inline int test_and_set_bit_lock(unsigned long nr, |
| 326 | volatile unsigned long *addr) |
| 327 | { |
| 328 | unsigned short bit = nr & SZLONG_MASK; |
| 329 | unsigned long res; |
| 330 | |
| 331 | if (cpu_has_llsc && R10000_LLSC_WAR) { |
| 332 | unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG); |
| 333 | unsigned long temp; |
| 334 | |
| 335 | __asm__ __volatile__( |
| 336 | " .set mips3 \n" |
| 337 | "1: " __LL "%0, %1 # test_and_set_bit \n" |
| 338 | " or %2, %0, %3 \n" |
| 339 | " " __SC "%2, %1 \n" |
| 340 | " beqzl %2, 1b \n" |
| 341 | " and %2, %0, %3 \n" |
| 342 | " .set mips0 \n" |
| 343 | : "=&r" (temp), "=m" (*m), "=&r" (res) |
| 344 | : "r" (1UL << bit), "m" (*m) |
| 345 | : "memory"); |
| 346 | } else if (cpu_has_llsc) { |
| 347 | unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG); |
| 348 | unsigned long temp; |
| 349 | |
| 350 | __asm__ __volatile__( |
| 351 | " .set push \n" |
| 352 | " .set noreorder \n" |
| 353 | " .set mips3 \n" |
| 354 | "1: " __LL "%0, %1 # test_and_set_bit \n" |
| 355 | " or %2, %0, %3 \n" |
| 356 | " " __SC "%2, %1 \n" |
| 357 | " beqz %2, 2f \n" |
| 358 | " and %2, %0, %3 \n" |
| 359 | " .subsection 2 \n" |
| 360 | "2: b 1b \n" |
| 361 | " nop \n" |
| 362 | " .previous \n" |
| 363 | " .set pop \n" |
| 364 | : "=&r" (temp), "=m" (*m), "=&r" (res) |
| 365 | : "r" (1UL << bit), "m" (*m) |
| 366 | : "memory"); |
| 367 | } else { |
| 368 | volatile unsigned long *a = addr; |
| 369 | unsigned long mask; |
| 370 | unsigned long flags; |
| 371 | |
| 372 | a += nr >> SZLONG_LOG; |
| 373 | mask = 1UL << bit; |
| 374 | raw_local_irq_save(flags); |
| 375 | res = (mask & *a); |
| 376 | *a |= mask; |
| 377 | raw_local_irq_restore(flags); |
| 378 | } |
| 379 | |
| 380 | smp_llsc_mb(); |
| 381 | |
| 382 | return res != 0; |
| 383 | } |
| 384 | /* |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 385 | * test_and_clear_bit - Clear a bit and return its old value |
| 386 | * @nr: Bit to clear |
| 387 | * @addr: Address to count from |
| 388 | * |
| 389 | * This operation is atomic and cannot be reordered. |
| 390 | * It also implies a memory barrier. |
| 391 | */ |
| 392 | static inline int test_and_clear_bit(unsigned long nr, |
| 393 | volatile unsigned long *addr) |
| 394 | { |
Ralf Baechle | b961153 | 2007-03-05 00:56:15 +0000 | [diff] [blame] | 395 | unsigned short bit = nr & SZLONG_MASK; |
Ralf Baechle | ff72b7a | 2007-06-07 13:17:30 +0100 | [diff] [blame] | 396 | unsigned long res; |
Ralf Baechle | b961153 | 2007-03-05 00:56:15 +0000 | [diff] [blame] | 397 | |
Nick Piggin | c8f30ae | 2007-10-18 03:06:52 -0700 | [diff] [blame] | 398 | smp_llsc_mb(); |
| 399 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 400 | if (cpu_has_llsc && R10000_LLSC_WAR) { |
| 401 | unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG); |
Atsushi Nemoto | 8e09ffb | 2007-06-14 00:56:31 +0900 | [diff] [blame] | 402 | unsigned long temp; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 403 | |
| 404 | __asm__ __volatile__( |
Maciej W. Rozycki | c4559f6 | 2005-06-23 15:57:15 +0000 | [diff] [blame] | 405 | " .set mips3 \n" |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 406 | "1: " __LL "%0, %1 # test_and_clear_bit \n" |
| 407 | " or %2, %0, %3 \n" |
| 408 | " xor %2, %3 \n" |
Maciej W. Rozycki | aac8aa7 | 2005-06-14 17:35:03 +0000 | [diff] [blame] | 409 | " " __SC "%2, %1 \n" |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 410 | " beqzl %2, 1b \n" |
| 411 | " and %2, %0, %3 \n" |
Maciej W. Rozycki | aac8aa7 | 2005-06-14 17:35:03 +0000 | [diff] [blame] | 412 | " .set mips0 \n" |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 413 | : "=&r" (temp), "=m" (*m), "=&r" (res) |
Ralf Baechle | b961153 | 2007-03-05 00:56:15 +0000 | [diff] [blame] | 414 | : "r" (1UL << bit), "m" (*m) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 415 | : "memory"); |
Ralf Baechle | 102fa15 | 2007-02-16 17:18:50 +0000 | [diff] [blame] | 416 | #ifdef CONFIG_CPU_MIPSR2 |
| 417 | } else if (__builtin_constant_p(nr)) { |
| 418 | unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG); |
Ralf Baechle | ff72b7a | 2007-06-07 13:17:30 +0100 | [diff] [blame] | 419 | unsigned long temp; |
Ralf Baechle | 102fa15 | 2007-02-16 17:18:50 +0000 | [diff] [blame] | 420 | |
| 421 | __asm__ __volatile__( |
| 422 | "1: " __LL "%0, %1 # test_and_clear_bit \n" |
| 423 | " " __EXT "%2, %0, %3, 1 \n" |
| 424 | " " __INS "%0, $0, %3, 1 \n" |
| 425 | " " __SC "%0, %1 \n" |
| 426 | " beqz %0, 2f \n" |
| 427 | " .subsection 2 \n" |
| 428 | "2: b 1b \n" |
| 429 | " .previous \n" |
| 430 | : "=&r" (temp), "=m" (*m), "=&r" (res) |
Ralf Baechle | b961153 | 2007-03-05 00:56:15 +0000 | [diff] [blame] | 431 | : "ri" (bit), "m" (*m) |
Ralf Baechle | 102fa15 | 2007-02-16 17:18:50 +0000 | [diff] [blame] | 432 | : "memory"); |
Ralf Baechle | 102fa15 | 2007-02-16 17:18:50 +0000 | [diff] [blame] | 433 | #endif |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 434 | } else if (cpu_has_llsc) { |
| 435 | unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG); |
Ralf Baechle | ff72b7a | 2007-06-07 13:17:30 +0100 | [diff] [blame] | 436 | unsigned long temp; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 437 | |
| 438 | __asm__ __volatile__( |
Maciej W. Rozycki | aac8aa7 | 2005-06-14 17:35:03 +0000 | [diff] [blame] | 439 | " .set push \n" |
| 440 | " .set noreorder \n" |
Maciej W. Rozycki | c4559f6 | 2005-06-23 15:57:15 +0000 | [diff] [blame] | 441 | " .set mips3 \n" |
Maciej W. Rozycki | aac8aa7 | 2005-06-14 17:35:03 +0000 | [diff] [blame] | 442 | "1: " __LL "%0, %1 # test_and_clear_bit \n" |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 443 | " or %2, %0, %3 \n" |
| 444 | " xor %2, %3 \n" |
Maciej W. Rozycki | aac8aa7 | 2005-06-14 17:35:03 +0000 | [diff] [blame] | 445 | " " __SC "%2, %1 \n" |
Ralf Baechle | f65e4fa | 2006-09-28 01:45:21 +0100 | [diff] [blame] | 446 | " beqz %2, 2f \n" |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 447 | " and %2, %0, %3 \n" |
Ralf Baechle | f65e4fa | 2006-09-28 01:45:21 +0100 | [diff] [blame] | 448 | " .subsection 2 \n" |
| 449 | "2: b 1b \n" |
| 450 | " nop \n" |
| 451 | " .previous \n" |
Maciej W. Rozycki | aac8aa7 | 2005-06-14 17:35:03 +0000 | [diff] [blame] | 452 | " .set pop \n" |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 453 | : "=&r" (temp), "=m" (*m), "=&r" (res) |
Ralf Baechle | b961153 | 2007-03-05 00:56:15 +0000 | [diff] [blame] | 454 | : "r" (1UL << bit), "m" (*m) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 455 | : "memory"); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 456 | } else { |
| 457 | volatile unsigned long *a = addr; |
| 458 | unsigned long mask; |
Ralf Baechle | 4ffd8b3 | 2006-11-30 01:14:50 +0000 | [diff] [blame] | 459 | unsigned long flags; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 460 | |
| 461 | a += nr >> SZLONG_LOG; |
Ralf Baechle | b961153 | 2007-03-05 00:56:15 +0000 | [diff] [blame] | 462 | mask = 1UL << bit; |
Ralf Baechle | 49edd09 | 2007-03-16 16:10:36 +0000 | [diff] [blame] | 463 | raw_local_irq_save(flags); |
Ralf Baechle | ff72b7a | 2007-06-07 13:17:30 +0100 | [diff] [blame] | 464 | res = (mask & *a); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 465 | *a &= ~mask; |
Ralf Baechle | 49edd09 | 2007-03-16 16:10:36 +0000 | [diff] [blame] | 466 | raw_local_irq_restore(flags); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 467 | } |
Ralf Baechle | 0004a9d | 2006-10-31 03:45:07 +0000 | [diff] [blame] | 468 | |
Ralf Baechle | 17099b1 | 2007-07-14 13:24:05 +0100 | [diff] [blame] | 469 | smp_llsc_mb(); |
Ralf Baechle | ff72b7a | 2007-06-07 13:17:30 +0100 | [diff] [blame] | 470 | |
| 471 | return res != 0; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 472 | } |
| 473 | |
| 474 | /* |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 475 | * test_and_change_bit - Change a bit and return its old value |
| 476 | * @nr: Bit to change |
| 477 | * @addr: Address to count from |
| 478 | * |
| 479 | * This operation is atomic and cannot be reordered. |
| 480 | * It also implies a memory barrier. |
| 481 | */ |
| 482 | static inline int test_and_change_bit(unsigned long nr, |
| 483 | volatile unsigned long *addr) |
| 484 | { |
Ralf Baechle | b961153 | 2007-03-05 00:56:15 +0000 | [diff] [blame] | 485 | unsigned short bit = nr & SZLONG_MASK; |
Ralf Baechle | ff72b7a | 2007-06-07 13:17:30 +0100 | [diff] [blame] | 486 | unsigned long res; |
Ralf Baechle | b961153 | 2007-03-05 00:56:15 +0000 | [diff] [blame] | 487 | |
Nick Piggin | c8f30ae | 2007-10-18 03:06:52 -0700 | [diff] [blame] | 488 | smp_llsc_mb(); |
| 489 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 490 | if (cpu_has_llsc && R10000_LLSC_WAR) { |
| 491 | unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG); |
Ralf Baechle | ff72b7a | 2007-06-07 13:17:30 +0100 | [diff] [blame] | 492 | unsigned long temp; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 493 | |
| 494 | __asm__ __volatile__( |
Maciej W. Rozycki | c4559f6 | 2005-06-23 15:57:15 +0000 | [diff] [blame] | 495 | " .set mips3 \n" |
Maciej W. Rozycki | aac8aa7 | 2005-06-14 17:35:03 +0000 | [diff] [blame] | 496 | "1: " __LL "%0, %1 # test_and_change_bit \n" |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 497 | " xor %2, %0, %3 \n" |
Maciej W. Rozycki | aac8aa7 | 2005-06-14 17:35:03 +0000 | [diff] [blame] | 498 | " " __SC "%2, %1 \n" |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 499 | " beqzl %2, 1b \n" |
| 500 | " and %2, %0, %3 \n" |
Maciej W. Rozycki | aac8aa7 | 2005-06-14 17:35:03 +0000 | [diff] [blame] | 501 | " .set mips0 \n" |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 502 | : "=&r" (temp), "=m" (*m), "=&r" (res) |
Ralf Baechle | b961153 | 2007-03-05 00:56:15 +0000 | [diff] [blame] | 503 | : "r" (1UL << bit), "m" (*m) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 504 | : "memory"); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 505 | } else if (cpu_has_llsc) { |
| 506 | unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG); |
Ralf Baechle | ff72b7a | 2007-06-07 13:17:30 +0100 | [diff] [blame] | 507 | unsigned long temp; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 508 | |
| 509 | __asm__ __volatile__( |
Maciej W. Rozycki | aac8aa7 | 2005-06-14 17:35:03 +0000 | [diff] [blame] | 510 | " .set push \n" |
| 511 | " .set noreorder \n" |
Maciej W. Rozycki | c4559f6 | 2005-06-23 15:57:15 +0000 | [diff] [blame] | 512 | " .set mips3 \n" |
Maciej W. Rozycki | aac8aa7 | 2005-06-14 17:35:03 +0000 | [diff] [blame] | 513 | "1: " __LL "%0, %1 # test_and_change_bit \n" |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 514 | " xor %2, %0, %3 \n" |
Maciej W. Rozycki | aac8aa7 | 2005-06-14 17:35:03 +0000 | [diff] [blame] | 515 | " " __SC "\t%2, %1 \n" |
Ralf Baechle | f65e4fa | 2006-09-28 01:45:21 +0100 | [diff] [blame] | 516 | " beqz %2, 2f \n" |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 517 | " and %2, %0, %3 \n" |
Ralf Baechle | f65e4fa | 2006-09-28 01:45:21 +0100 | [diff] [blame] | 518 | " .subsection 2 \n" |
| 519 | "2: b 1b \n" |
| 520 | " nop \n" |
| 521 | " .previous \n" |
Maciej W. Rozycki | aac8aa7 | 2005-06-14 17:35:03 +0000 | [diff] [blame] | 522 | " .set pop \n" |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 523 | : "=&r" (temp), "=m" (*m), "=&r" (res) |
Ralf Baechle | b961153 | 2007-03-05 00:56:15 +0000 | [diff] [blame] | 524 | : "r" (1UL << bit), "m" (*m) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 525 | : "memory"); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 526 | } else { |
| 527 | volatile unsigned long *a = addr; |
Ralf Baechle | ff72b7a | 2007-06-07 13:17:30 +0100 | [diff] [blame] | 528 | unsigned long mask; |
Ralf Baechle | 4ffd8b3 | 2006-11-30 01:14:50 +0000 | [diff] [blame] | 529 | unsigned long flags; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 530 | |
| 531 | a += nr >> SZLONG_LOG; |
Ralf Baechle | b961153 | 2007-03-05 00:56:15 +0000 | [diff] [blame] | 532 | mask = 1UL << bit; |
Ralf Baechle | 49edd09 | 2007-03-16 16:10:36 +0000 | [diff] [blame] | 533 | raw_local_irq_save(flags); |
Ralf Baechle | ff72b7a | 2007-06-07 13:17:30 +0100 | [diff] [blame] | 534 | res = (mask & *a); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 535 | *a ^= mask; |
Ralf Baechle | 49edd09 | 2007-03-16 16:10:36 +0000 | [diff] [blame] | 536 | raw_local_irq_restore(flags); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 537 | } |
Ralf Baechle | 0004a9d | 2006-10-31 03:45:07 +0000 | [diff] [blame] | 538 | |
Ralf Baechle | 17099b1 | 2007-07-14 13:24:05 +0100 | [diff] [blame] | 539 | smp_llsc_mb(); |
Ralf Baechle | ff72b7a | 2007-06-07 13:17:30 +0100 | [diff] [blame] | 540 | |
| 541 | return res != 0; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 542 | } |
| 543 | |
Akinobu Mita | 3c9ee7e | 2006-03-26 01:39:30 -0800 | [diff] [blame] | 544 | #include <asm-generic/bitops/non-atomic.h> |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 545 | |
Ralf Baechle | 6590326 | 2005-07-12 12:50:30 +0000 | [diff] [blame] | 546 | /* |
Nick Piggin | 728697c | 2007-10-18 03:06:53 -0700 | [diff] [blame] | 547 | * __clear_bit_unlock - Clears a bit in memory |
| 548 | * @nr: Bit to clear |
| 549 | * @addr: Address to start counting from |
| 550 | * |
| 551 | * __clear_bit() is non-atomic and implies release semantics before the memory |
| 552 | * operation. It can be used for an unlock if no other CPUs can concurrently |
| 553 | * modify other bits in the word. |
| 554 | */ |
| 555 | static inline void __clear_bit_unlock(unsigned long nr, volatile unsigned long *addr) |
| 556 | { |
| 557 | smp_mb(); |
| 558 | __clear_bit(nr, addr); |
| 559 | } |
| 560 | |
| 561 | /* |
Ralf Baechle | 6590326 | 2005-07-12 12:50:30 +0000 | [diff] [blame] | 562 | * Return the bit position (0..63) of the most significant 1 bit in a word |
| 563 | * Returns -1 if no 1 bit exists |
| 564 | */ |
Ralf Baechle | ec917c2c | 2005-10-07 16:58:15 +0100 | [diff] [blame] | 565 | static inline int __ilog2(unsigned long x) |
Ralf Baechle | 6590326 | 2005-07-12 12:50:30 +0000 | [diff] [blame] | 566 | { |
| 567 | int lz; |
| 568 | |
Ralf Baechle | ec917c2c | 2005-10-07 16:58:15 +0100 | [diff] [blame] | 569 | if (sizeof(x) == 4) { |
Ralf Baechle | 49a89ef | 2007-10-11 23:46:15 +0100 | [diff] [blame] | 570 | __asm__( |
Ralf Baechle | ec917c2c | 2005-10-07 16:58:15 +0100 | [diff] [blame] | 571 | " .set push \n" |
| 572 | " .set mips32 \n" |
| 573 | " clz %0, %1 \n" |
| 574 | " .set pop \n" |
| 575 | : "=r" (lz) |
| 576 | : "r" (x)); |
| 577 | |
| 578 | return 31 - lz; |
| 579 | } |
| 580 | |
| 581 | BUG_ON(sizeof(x) != 8); |
| 582 | |
Ralf Baechle | 49a89ef | 2007-10-11 23:46:15 +0100 | [diff] [blame] | 583 | __asm__( |
Ralf Baechle | 6590326 | 2005-07-12 12:50:30 +0000 | [diff] [blame] | 584 | " .set push \n" |
| 585 | " .set mips64 \n" |
| 586 | " dclz %0, %1 \n" |
| 587 | " .set pop \n" |
| 588 | : "=r" (lz) |
| 589 | : "r" (x)); |
| 590 | |
| 591 | return 63 - lz; |
| 592 | } |
Ralf Baechle | 6590326 | 2005-07-12 12:50:30 +0000 | [diff] [blame] | 593 | |
Akinobu Mita | 3c9ee7e | 2006-03-26 01:39:30 -0800 | [diff] [blame] | 594 | #if defined(CONFIG_CPU_MIPS32) || defined(CONFIG_CPU_MIPS64) |
| 595 | |
Ralf Baechle | 6590326 | 2005-07-12 12:50:30 +0000 | [diff] [blame] | 596 | /* |
| 597 | * __ffs - find first bit in word. |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 598 | * @word: The word to search |
| 599 | * |
Ralf Baechle | 6590326 | 2005-07-12 12:50:30 +0000 | [diff] [blame] | 600 | * Returns 0..SZLONG-1 |
| 601 | * Undefined if no bit exists, so code should check against 0 first. |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 602 | */ |
Ralf Baechle | 6590326 | 2005-07-12 12:50:30 +0000 | [diff] [blame] | 603 | static inline unsigned long __ffs(unsigned long word) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 604 | { |
Ralf Baechle | 6590326 | 2005-07-12 12:50:30 +0000 | [diff] [blame] | 605 | return __ilog2(word & -word); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 606 | } |
| 607 | |
| 608 | /* |
Atsushi Nemoto | bc81824 | 2006-04-17 21:19:12 +0900 | [diff] [blame] | 609 | * fls - find last bit set. |
| 610 | * @word: The word to search |
| 611 | * |
| 612 | * This is defined the same way as ffs. |
| 613 | * Note fls(0) = 0, fls(1) = 1, fls(0x80000000) = 32. |
| 614 | */ |
| 615 | static inline int fls(int word) |
| 616 | { |
Ralf Baechle | 49a89ef | 2007-10-11 23:46:15 +0100 | [diff] [blame] | 617 | __asm__("clz %0, %1" : "=r" (word) : "r" (word)); |
Atsushi Nemoto | bc81824 | 2006-04-17 21:19:12 +0900 | [diff] [blame] | 618 | |
| 619 | return 32 - word; |
| 620 | } |
| 621 | |
| 622 | #if defined(CONFIG_64BIT) && defined(CONFIG_CPU_MIPS64) |
| 623 | static inline int fls64(__u64 word) |
| 624 | { |
Ralf Baechle | 49a89ef | 2007-10-11 23:46:15 +0100 | [diff] [blame] | 625 | __asm__("dclz %0, %1" : "=r" (word) : "r" (word)); |
Atsushi Nemoto | bc81824 | 2006-04-17 21:19:12 +0900 | [diff] [blame] | 626 | |
| 627 | return 64 - word; |
| 628 | } |
| 629 | #else |
| 630 | #include <asm-generic/bitops/fls64.h> |
| 631 | #endif |
| 632 | |
| 633 | /* |
Ralf Baechle | 6590326 | 2005-07-12 12:50:30 +0000 | [diff] [blame] | 634 | * ffs - find first bit set. |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 635 | * @word: The word to search |
| 636 | * |
Atsushi Nemoto | bc81824 | 2006-04-17 21:19:12 +0900 | [diff] [blame] | 637 | * This is defined the same way as |
| 638 | * the libc and compiler builtin ffs routines, therefore |
| 639 | * differs in spirit from the above ffz (man ffs). |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 640 | */ |
Atsushi Nemoto | bc81824 | 2006-04-17 21:19:12 +0900 | [diff] [blame] | 641 | static inline int ffs(int word) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 642 | { |
Ralf Baechle | 6590326 | 2005-07-12 12:50:30 +0000 | [diff] [blame] | 643 | if (!word) |
| 644 | return 0; |
| 645 | |
Atsushi Nemoto | bc81824 | 2006-04-17 21:19:12 +0900 | [diff] [blame] | 646 | return fls(word & -word); |
Ralf Baechle | 6590326 | 2005-07-12 12:50:30 +0000 | [diff] [blame] | 647 | } |
Ralf Baechle | 2caf190 | 2006-01-30 17:14:41 +0000 | [diff] [blame] | 648 | |
Akinobu Mita | 3c9ee7e | 2006-03-26 01:39:30 -0800 | [diff] [blame] | 649 | #else |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 650 | |
Akinobu Mita | 3c9ee7e | 2006-03-26 01:39:30 -0800 | [diff] [blame] | 651 | #include <asm-generic/bitops/__ffs.h> |
| 652 | #include <asm-generic/bitops/ffs.h> |
Akinobu Mita | 3c9ee7e | 2006-03-26 01:39:30 -0800 | [diff] [blame] | 653 | #include <asm-generic/bitops/fls.h> |
Atsushi Nemoto | bc81824 | 2006-04-17 21:19:12 +0900 | [diff] [blame] | 654 | #include <asm-generic/bitops/fls64.h> |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 655 | |
Akinobu Mita | 3c9ee7e | 2006-03-26 01:39:30 -0800 | [diff] [blame] | 656 | #endif /*defined(CONFIG_CPU_MIPS32) || defined(CONFIG_CPU_MIPS64) */ |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 657 | |
Atsushi Nemoto | bc81824 | 2006-04-17 21:19:12 +0900 | [diff] [blame] | 658 | #include <asm-generic/bitops/ffz.h> |
Akinobu Mita | 3c9ee7e | 2006-03-26 01:39:30 -0800 | [diff] [blame] | 659 | #include <asm-generic/bitops/find.h> |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 660 | |
| 661 | #ifdef __KERNEL__ |
| 662 | |
Akinobu Mita | 3c9ee7e | 2006-03-26 01:39:30 -0800 | [diff] [blame] | 663 | #include <asm-generic/bitops/sched.h> |
| 664 | #include <asm-generic/bitops/hweight.h> |
| 665 | #include <asm-generic/bitops/ext2-non-atomic.h> |
| 666 | #include <asm-generic/bitops/ext2-atomic.h> |
| 667 | #include <asm-generic/bitops/minix.h> |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 668 | |
| 669 | #endif /* __KERNEL__ */ |
| 670 | |
| 671 | #endif /* _ASM_BITOPS_H */ |