Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1 | /* |
| 2 | * This file is subject to the terms and conditions of the GNU General Public |
| 3 | * License. See the file "COPYING" in the main directory of this archive |
| 4 | * for more details. |
| 5 | * |
| 6 | * Copyright (c) 1994 - 1997, 1999, 2000 Ralf Baechle (ralf@gnu.org) |
| 7 | * Copyright (c) 1999, 2000 Silicon Graphics, Inc. |
| 8 | */ |
| 9 | #ifndef _ASM_BITOPS_H |
| 10 | #define _ASM_BITOPS_H |
| 11 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 12 | #include <linux/compiler.h> |
Ralf Baechle | 4ffd8b3 | 2006-11-30 01:14:50 +0000 | [diff] [blame] | 13 | #include <linux/irqflags.h> |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 14 | #include <linux/types.h> |
Ralf Baechle | ec917c2c | 2005-10-07 16:58:15 +0100 | [diff] [blame] | 15 | #include <asm/bug.h> |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 16 | #include <asm/byteorder.h> /* sigh ... */ |
| 17 | #include <asm/cpu-features.h> |
Ralf Baechle | 4ffd8b3 | 2006-11-30 01:14:50 +0000 | [diff] [blame] | 18 | #include <asm/sgidefs.h> |
| 19 | #include <asm/war.h> |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 20 | |
| 21 | #if (_MIPS_SZLONG == 32) |
| 22 | #define SZLONG_LOG 5 |
| 23 | #define SZLONG_MASK 31UL |
Maciej W. Rozycki | aac8aa7 | 2005-06-14 17:35:03 +0000 | [diff] [blame] | 24 | #define __LL "ll " |
| 25 | #define __SC "sc " |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 26 | #elif (_MIPS_SZLONG == 64) |
| 27 | #define SZLONG_LOG 6 |
| 28 | #define SZLONG_MASK 63UL |
Maciej W. Rozycki | aac8aa7 | 2005-06-14 17:35:03 +0000 | [diff] [blame] | 29 | #define __LL "lld " |
| 30 | #define __SC "scd " |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 31 | #endif |
| 32 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 33 | /* |
| 34 | * clear_bit() doesn't provide any barrier for the compiler. |
| 35 | */ |
| 36 | #define smp_mb__before_clear_bit() smp_mb() |
| 37 | #define smp_mb__after_clear_bit() smp_mb() |
| 38 | |
| 39 | /* |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 40 | * set_bit - Atomically set a bit in memory |
| 41 | * @nr: the bit to set |
| 42 | * @addr: the address to start counting from |
| 43 | * |
| 44 | * This function is atomic and may not be reordered. See __set_bit() |
| 45 | * if you do not require the atomic guarantees. |
| 46 | * Note that @nr may be almost arbitrarily large; this function is not |
| 47 | * restricted to acting on a single-word quantity. |
| 48 | */ |
| 49 | static inline void set_bit(unsigned long nr, volatile unsigned long *addr) |
| 50 | { |
| 51 | unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG); |
| 52 | unsigned long temp; |
| 53 | |
| 54 | if (cpu_has_llsc && R10000_LLSC_WAR) { |
| 55 | __asm__ __volatile__( |
Maciej W. Rozycki | c4559f6 | 2005-06-23 15:57:15 +0000 | [diff] [blame] | 56 | " .set mips3 \n" |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 57 | "1: " __LL "%0, %1 # set_bit \n" |
| 58 | " or %0, %2 \n" |
Maciej W. Rozycki | aac8aa7 | 2005-06-14 17:35:03 +0000 | [diff] [blame] | 59 | " " __SC "%0, %1 \n" |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 60 | " beqzl %0, 1b \n" |
Maciej W. Rozycki | aac8aa7 | 2005-06-14 17:35:03 +0000 | [diff] [blame] | 61 | " .set mips0 \n" |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 62 | : "=&r" (temp), "=m" (*m) |
| 63 | : "ir" (1UL << (nr & SZLONG_MASK)), "m" (*m)); |
| 64 | } else if (cpu_has_llsc) { |
| 65 | __asm__ __volatile__( |
Maciej W. Rozycki | c4559f6 | 2005-06-23 15:57:15 +0000 | [diff] [blame] | 66 | " .set mips3 \n" |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 67 | "1: " __LL "%0, %1 # set_bit \n" |
| 68 | " or %0, %2 \n" |
Maciej W. Rozycki | aac8aa7 | 2005-06-14 17:35:03 +0000 | [diff] [blame] | 69 | " " __SC "%0, %1 \n" |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 70 | " beqz %0, 1b \n" |
Maciej W. Rozycki | aac8aa7 | 2005-06-14 17:35:03 +0000 | [diff] [blame] | 71 | " .set mips0 \n" |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 72 | : "=&r" (temp), "=m" (*m) |
| 73 | : "ir" (1UL << (nr & SZLONG_MASK)), "m" (*m)); |
| 74 | } else { |
| 75 | volatile unsigned long *a = addr; |
| 76 | unsigned long mask; |
Ralf Baechle | 4ffd8b3 | 2006-11-30 01:14:50 +0000 | [diff] [blame] | 77 | unsigned long flags; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 78 | |
| 79 | a += nr >> SZLONG_LOG; |
| 80 | mask = 1UL << (nr & SZLONG_MASK); |
Ralf Baechle | 4ffd8b3 | 2006-11-30 01:14:50 +0000 | [diff] [blame] | 81 | local_irq_save(flags); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 82 | *a |= mask; |
Ralf Baechle | 4ffd8b3 | 2006-11-30 01:14:50 +0000 | [diff] [blame] | 83 | local_irq_restore(flags); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 84 | } |
| 85 | } |
| 86 | |
| 87 | /* |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 88 | * clear_bit - Clears a bit in memory |
| 89 | * @nr: Bit to clear |
| 90 | * @addr: Address to start counting from |
| 91 | * |
| 92 | * clear_bit() is atomic and may not be reordered. However, it does |
| 93 | * not contain a memory barrier, so if it is used for locking purposes, |
| 94 | * you should call smp_mb__before_clear_bit() and/or smp_mb__after_clear_bit() |
| 95 | * in order to ensure changes are visible on other processors. |
| 96 | */ |
| 97 | static inline void clear_bit(unsigned long nr, volatile unsigned long *addr) |
| 98 | { |
| 99 | unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG); |
| 100 | unsigned long temp; |
| 101 | |
| 102 | if (cpu_has_llsc && R10000_LLSC_WAR) { |
| 103 | __asm__ __volatile__( |
Maciej W. Rozycki | c4559f6 | 2005-06-23 15:57:15 +0000 | [diff] [blame] | 104 | " .set mips3 \n" |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 105 | "1: " __LL "%0, %1 # clear_bit \n" |
| 106 | " and %0, %2 \n" |
| 107 | " " __SC "%0, %1 \n" |
| 108 | " beqzl %0, 1b \n" |
Maciej W. Rozycki | aac8aa7 | 2005-06-14 17:35:03 +0000 | [diff] [blame] | 109 | " .set mips0 \n" |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 110 | : "=&r" (temp), "=m" (*m) |
| 111 | : "ir" (~(1UL << (nr & SZLONG_MASK))), "m" (*m)); |
| 112 | } else if (cpu_has_llsc) { |
| 113 | __asm__ __volatile__( |
Maciej W. Rozycki | c4559f6 | 2005-06-23 15:57:15 +0000 | [diff] [blame] | 114 | " .set mips3 \n" |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 115 | "1: " __LL "%0, %1 # clear_bit \n" |
| 116 | " and %0, %2 \n" |
| 117 | " " __SC "%0, %1 \n" |
| 118 | " beqz %0, 1b \n" |
Maciej W. Rozycki | aac8aa7 | 2005-06-14 17:35:03 +0000 | [diff] [blame] | 119 | " .set mips0 \n" |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 120 | : "=&r" (temp), "=m" (*m) |
| 121 | : "ir" (~(1UL << (nr & SZLONG_MASK))), "m" (*m)); |
| 122 | } else { |
| 123 | volatile unsigned long *a = addr; |
| 124 | unsigned long mask; |
Ralf Baechle | 4ffd8b3 | 2006-11-30 01:14:50 +0000 | [diff] [blame] | 125 | unsigned long flags; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 126 | |
| 127 | a += nr >> SZLONG_LOG; |
| 128 | mask = 1UL << (nr & SZLONG_MASK); |
Ralf Baechle | 4ffd8b3 | 2006-11-30 01:14:50 +0000 | [diff] [blame] | 129 | local_irq_save(flags); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 130 | *a &= ~mask; |
Ralf Baechle | 4ffd8b3 | 2006-11-30 01:14:50 +0000 | [diff] [blame] | 131 | local_irq_restore(flags); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 132 | } |
| 133 | } |
| 134 | |
| 135 | /* |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 136 | * change_bit - Toggle a bit in memory |
| 137 | * @nr: Bit to change |
| 138 | * @addr: Address to start counting from |
| 139 | * |
| 140 | * change_bit() is atomic and may not be reordered. |
| 141 | * Note that @nr may be almost arbitrarily large; this function is not |
| 142 | * restricted to acting on a single-word quantity. |
| 143 | */ |
| 144 | static inline void change_bit(unsigned long nr, volatile unsigned long *addr) |
| 145 | { |
| 146 | if (cpu_has_llsc && R10000_LLSC_WAR) { |
| 147 | unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG); |
| 148 | unsigned long temp; |
| 149 | |
| 150 | __asm__ __volatile__( |
Maciej W. Rozycki | c4559f6 | 2005-06-23 15:57:15 +0000 | [diff] [blame] | 151 | " .set mips3 \n" |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 152 | "1: " __LL "%0, %1 # change_bit \n" |
| 153 | " xor %0, %2 \n" |
Maciej W. Rozycki | aac8aa7 | 2005-06-14 17:35:03 +0000 | [diff] [blame] | 154 | " " __SC "%0, %1 \n" |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 155 | " beqzl %0, 1b \n" |
Maciej W. Rozycki | aac8aa7 | 2005-06-14 17:35:03 +0000 | [diff] [blame] | 156 | " .set mips0 \n" |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 157 | : "=&r" (temp), "=m" (*m) |
| 158 | : "ir" (1UL << (nr & SZLONG_MASK)), "m" (*m)); |
| 159 | } else if (cpu_has_llsc) { |
| 160 | unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG); |
| 161 | unsigned long temp; |
| 162 | |
| 163 | __asm__ __volatile__( |
Maciej W. Rozycki | c4559f6 | 2005-06-23 15:57:15 +0000 | [diff] [blame] | 164 | " .set mips3 \n" |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 165 | "1: " __LL "%0, %1 # change_bit \n" |
| 166 | " xor %0, %2 \n" |
Maciej W. Rozycki | aac8aa7 | 2005-06-14 17:35:03 +0000 | [diff] [blame] | 167 | " " __SC "%0, %1 \n" |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 168 | " beqz %0, 1b \n" |
Maciej W. Rozycki | aac8aa7 | 2005-06-14 17:35:03 +0000 | [diff] [blame] | 169 | " .set mips0 \n" |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 170 | : "=&r" (temp), "=m" (*m) |
| 171 | : "ir" (1UL << (nr & SZLONG_MASK)), "m" (*m)); |
| 172 | } else { |
| 173 | volatile unsigned long *a = addr; |
| 174 | unsigned long mask; |
Ralf Baechle | 4ffd8b3 | 2006-11-30 01:14:50 +0000 | [diff] [blame] | 175 | unsigned long flags; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 176 | |
| 177 | a += nr >> SZLONG_LOG; |
| 178 | mask = 1UL << (nr & SZLONG_MASK); |
Ralf Baechle | 4ffd8b3 | 2006-11-30 01:14:50 +0000 | [diff] [blame] | 179 | local_irq_save(flags); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 180 | *a ^= mask; |
Ralf Baechle | 4ffd8b3 | 2006-11-30 01:14:50 +0000 | [diff] [blame] | 181 | local_irq_restore(flags); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 182 | } |
| 183 | } |
| 184 | |
| 185 | /* |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 186 | * test_and_set_bit - Set a bit and return its old value |
| 187 | * @nr: Bit to set |
| 188 | * @addr: Address to count from |
| 189 | * |
| 190 | * This operation is atomic and cannot be reordered. |
| 191 | * It also implies a memory barrier. |
| 192 | */ |
| 193 | static inline int test_and_set_bit(unsigned long nr, |
| 194 | volatile unsigned long *addr) |
| 195 | { |
| 196 | if (cpu_has_llsc && R10000_LLSC_WAR) { |
| 197 | unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG); |
| 198 | unsigned long temp, res; |
| 199 | |
| 200 | __asm__ __volatile__( |
Maciej W. Rozycki | c4559f6 | 2005-06-23 15:57:15 +0000 | [diff] [blame] | 201 | " .set mips3 \n" |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 202 | "1: " __LL "%0, %1 # test_and_set_bit \n" |
| 203 | " or %2, %0, %3 \n" |
| 204 | " " __SC "%2, %1 \n" |
| 205 | " beqzl %2, 1b \n" |
| 206 | " and %2, %0, %3 \n" |
| 207 | #ifdef CONFIG_SMP |
Maciej W. Rozycki | aac8aa7 | 2005-06-14 17:35:03 +0000 | [diff] [blame] | 208 | " sync \n" |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 209 | #endif |
Maciej W. Rozycki | aac8aa7 | 2005-06-14 17:35:03 +0000 | [diff] [blame] | 210 | " .set mips0 \n" |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 211 | : "=&r" (temp), "=m" (*m), "=&r" (res) |
| 212 | : "r" (1UL << (nr & SZLONG_MASK)), "m" (*m) |
| 213 | : "memory"); |
| 214 | |
| 215 | return res != 0; |
| 216 | } else if (cpu_has_llsc) { |
| 217 | unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG); |
| 218 | unsigned long temp, res; |
| 219 | |
| 220 | __asm__ __volatile__( |
Maciej W. Rozycki | aac8aa7 | 2005-06-14 17:35:03 +0000 | [diff] [blame] | 221 | " .set push \n" |
| 222 | " .set noreorder \n" |
Maciej W. Rozycki | c4559f6 | 2005-06-23 15:57:15 +0000 | [diff] [blame] | 223 | " .set mips3 \n" |
Maciej W. Rozycki | aac8aa7 | 2005-06-14 17:35:03 +0000 | [diff] [blame] | 224 | "1: " __LL "%0, %1 # test_and_set_bit \n" |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 225 | " or %2, %0, %3 \n" |
| 226 | " " __SC "%2, %1 \n" |
| 227 | " beqz %2, 1b \n" |
| 228 | " and %2, %0, %3 \n" |
| 229 | #ifdef CONFIG_SMP |
Maciej W. Rozycki | aac8aa7 | 2005-06-14 17:35:03 +0000 | [diff] [blame] | 230 | " sync \n" |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 231 | #endif |
Maciej W. Rozycki | aac8aa7 | 2005-06-14 17:35:03 +0000 | [diff] [blame] | 232 | " .set pop \n" |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 233 | : "=&r" (temp), "=m" (*m), "=&r" (res) |
| 234 | : "r" (1UL << (nr & SZLONG_MASK)), "m" (*m) |
| 235 | : "memory"); |
| 236 | |
| 237 | return res != 0; |
| 238 | } else { |
| 239 | volatile unsigned long *a = addr; |
| 240 | unsigned long mask; |
| 241 | int retval; |
Ralf Baechle | 4ffd8b3 | 2006-11-30 01:14:50 +0000 | [diff] [blame] | 242 | unsigned long flags; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 243 | |
| 244 | a += nr >> SZLONG_LOG; |
| 245 | mask = 1UL << (nr & SZLONG_MASK); |
Ralf Baechle | 4ffd8b3 | 2006-11-30 01:14:50 +0000 | [diff] [blame] | 246 | local_irq_save(flags); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 247 | retval = (mask & *a) != 0; |
| 248 | *a |= mask; |
Ralf Baechle | 4ffd8b3 | 2006-11-30 01:14:50 +0000 | [diff] [blame] | 249 | local_irq_restore(flags); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 250 | |
| 251 | return retval; |
| 252 | } |
| 253 | } |
| 254 | |
| 255 | /* |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 256 | * test_and_clear_bit - Clear a bit and return its old value |
| 257 | * @nr: Bit to clear |
| 258 | * @addr: Address to count from |
| 259 | * |
| 260 | * This operation is atomic and cannot be reordered. |
| 261 | * It also implies a memory barrier. |
| 262 | */ |
| 263 | static inline int test_and_clear_bit(unsigned long nr, |
| 264 | volatile unsigned long *addr) |
| 265 | { |
| 266 | if (cpu_has_llsc && R10000_LLSC_WAR) { |
| 267 | unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG); |
| 268 | unsigned long temp, res; |
| 269 | |
| 270 | __asm__ __volatile__( |
Maciej W. Rozycki | c4559f6 | 2005-06-23 15:57:15 +0000 | [diff] [blame] | 271 | " .set mips3 \n" |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 272 | "1: " __LL "%0, %1 # test_and_clear_bit \n" |
| 273 | " or %2, %0, %3 \n" |
| 274 | " xor %2, %3 \n" |
Maciej W. Rozycki | aac8aa7 | 2005-06-14 17:35:03 +0000 | [diff] [blame] | 275 | " " __SC "%2, %1 \n" |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 276 | " beqzl %2, 1b \n" |
| 277 | " and %2, %0, %3 \n" |
| 278 | #ifdef CONFIG_SMP |
| 279 | " sync \n" |
| 280 | #endif |
Maciej W. Rozycki | aac8aa7 | 2005-06-14 17:35:03 +0000 | [diff] [blame] | 281 | " .set mips0 \n" |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 282 | : "=&r" (temp), "=m" (*m), "=&r" (res) |
| 283 | : "r" (1UL << (nr & SZLONG_MASK)), "m" (*m) |
| 284 | : "memory"); |
| 285 | |
| 286 | return res != 0; |
| 287 | } else if (cpu_has_llsc) { |
| 288 | unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG); |
| 289 | unsigned long temp, res; |
| 290 | |
| 291 | __asm__ __volatile__( |
Maciej W. Rozycki | aac8aa7 | 2005-06-14 17:35:03 +0000 | [diff] [blame] | 292 | " .set push \n" |
| 293 | " .set noreorder \n" |
Maciej W. Rozycki | c4559f6 | 2005-06-23 15:57:15 +0000 | [diff] [blame] | 294 | " .set mips3 \n" |
Maciej W. Rozycki | aac8aa7 | 2005-06-14 17:35:03 +0000 | [diff] [blame] | 295 | "1: " __LL "%0, %1 # test_and_clear_bit \n" |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 296 | " or %2, %0, %3 \n" |
| 297 | " xor %2, %3 \n" |
Maciej W. Rozycki | aac8aa7 | 2005-06-14 17:35:03 +0000 | [diff] [blame] | 298 | " " __SC "%2, %1 \n" |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 299 | " beqz %2, 1b \n" |
| 300 | " and %2, %0, %3 \n" |
| 301 | #ifdef CONFIG_SMP |
| 302 | " sync \n" |
| 303 | #endif |
Maciej W. Rozycki | aac8aa7 | 2005-06-14 17:35:03 +0000 | [diff] [blame] | 304 | " .set pop \n" |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 305 | : "=&r" (temp), "=m" (*m), "=&r" (res) |
| 306 | : "r" (1UL << (nr & SZLONG_MASK)), "m" (*m) |
| 307 | : "memory"); |
| 308 | |
| 309 | return res != 0; |
| 310 | } else { |
| 311 | volatile unsigned long *a = addr; |
| 312 | unsigned long mask; |
| 313 | int retval; |
Ralf Baechle | 4ffd8b3 | 2006-11-30 01:14:50 +0000 | [diff] [blame] | 314 | unsigned long flags; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 315 | |
| 316 | a += nr >> SZLONG_LOG; |
| 317 | mask = 1UL << (nr & SZLONG_MASK); |
Ralf Baechle | 4ffd8b3 | 2006-11-30 01:14:50 +0000 | [diff] [blame] | 318 | local_irq_save(flags); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 319 | retval = (mask & *a) != 0; |
| 320 | *a &= ~mask; |
Ralf Baechle | 4ffd8b3 | 2006-11-30 01:14:50 +0000 | [diff] [blame] | 321 | local_irq_restore(flags); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 322 | |
| 323 | return retval; |
| 324 | } |
| 325 | } |
| 326 | |
| 327 | /* |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 328 | * test_and_change_bit - Change a bit and return its old value |
| 329 | * @nr: Bit to change |
| 330 | * @addr: Address to count from |
| 331 | * |
| 332 | * This operation is atomic and cannot be reordered. |
| 333 | * It also implies a memory barrier. |
| 334 | */ |
| 335 | static inline int test_and_change_bit(unsigned long nr, |
| 336 | volatile unsigned long *addr) |
| 337 | { |
| 338 | if (cpu_has_llsc && R10000_LLSC_WAR) { |
| 339 | unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG); |
| 340 | unsigned long temp, res; |
| 341 | |
| 342 | __asm__ __volatile__( |
Maciej W. Rozycki | c4559f6 | 2005-06-23 15:57:15 +0000 | [diff] [blame] | 343 | " .set mips3 \n" |
Maciej W. Rozycki | aac8aa7 | 2005-06-14 17:35:03 +0000 | [diff] [blame] | 344 | "1: " __LL "%0, %1 # test_and_change_bit \n" |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 345 | " xor %2, %0, %3 \n" |
Maciej W. Rozycki | aac8aa7 | 2005-06-14 17:35:03 +0000 | [diff] [blame] | 346 | " " __SC "%2, %1 \n" |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 347 | " beqzl %2, 1b \n" |
| 348 | " and %2, %0, %3 \n" |
| 349 | #ifdef CONFIG_SMP |
| 350 | " sync \n" |
| 351 | #endif |
Maciej W. Rozycki | aac8aa7 | 2005-06-14 17:35:03 +0000 | [diff] [blame] | 352 | " .set mips0 \n" |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 353 | : "=&r" (temp), "=m" (*m), "=&r" (res) |
| 354 | : "r" (1UL << (nr & SZLONG_MASK)), "m" (*m) |
| 355 | : "memory"); |
| 356 | |
| 357 | return res != 0; |
| 358 | } else if (cpu_has_llsc) { |
| 359 | unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG); |
| 360 | unsigned long temp, res; |
| 361 | |
| 362 | __asm__ __volatile__( |
Maciej W. Rozycki | aac8aa7 | 2005-06-14 17:35:03 +0000 | [diff] [blame] | 363 | " .set push \n" |
| 364 | " .set noreorder \n" |
Maciej W. Rozycki | c4559f6 | 2005-06-23 15:57:15 +0000 | [diff] [blame] | 365 | " .set mips3 \n" |
Maciej W. Rozycki | aac8aa7 | 2005-06-14 17:35:03 +0000 | [diff] [blame] | 366 | "1: " __LL "%0, %1 # test_and_change_bit \n" |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 367 | " xor %2, %0, %3 \n" |
Maciej W. Rozycki | aac8aa7 | 2005-06-14 17:35:03 +0000 | [diff] [blame] | 368 | " " __SC "\t%2, %1 \n" |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 369 | " beqz %2, 1b \n" |
| 370 | " and %2, %0, %3 \n" |
| 371 | #ifdef CONFIG_SMP |
| 372 | " sync \n" |
| 373 | #endif |
Maciej W. Rozycki | aac8aa7 | 2005-06-14 17:35:03 +0000 | [diff] [blame] | 374 | " .set pop \n" |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 375 | : "=&r" (temp), "=m" (*m), "=&r" (res) |
| 376 | : "r" (1UL << (nr & SZLONG_MASK)), "m" (*m) |
| 377 | : "memory"); |
| 378 | |
| 379 | return res != 0; |
| 380 | } else { |
| 381 | volatile unsigned long *a = addr; |
| 382 | unsigned long mask, retval; |
Ralf Baechle | 4ffd8b3 | 2006-11-30 01:14:50 +0000 | [diff] [blame] | 383 | unsigned long flags; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 384 | |
| 385 | a += nr >> SZLONG_LOG; |
| 386 | mask = 1UL << (nr & SZLONG_MASK); |
Ralf Baechle | 4ffd8b3 | 2006-11-30 01:14:50 +0000 | [diff] [blame] | 387 | local_irq_save(flags); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 388 | retval = (mask & *a) != 0; |
| 389 | *a ^= mask; |
Ralf Baechle | 4ffd8b3 | 2006-11-30 01:14:50 +0000 | [diff] [blame] | 390 | local_irq_restore(flags); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 391 | |
| 392 | return retval; |
| 393 | } |
| 394 | } |
| 395 | |
Akinobu Mita | 3c9ee7e | 2006-03-26 01:39:30 -0800 | [diff] [blame] | 396 | #include <asm-generic/bitops/non-atomic.h> |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 397 | |
Ralf Baechle | 6590326 | 2005-07-12 12:50:30 +0000 | [diff] [blame] | 398 | /* |
| 399 | * Return the bit position (0..63) of the most significant 1 bit in a word |
| 400 | * Returns -1 if no 1 bit exists |
| 401 | */ |
Ralf Baechle | ec917c2c | 2005-10-07 16:58:15 +0100 | [diff] [blame] | 402 | static inline int __ilog2(unsigned long x) |
Ralf Baechle | 6590326 | 2005-07-12 12:50:30 +0000 | [diff] [blame] | 403 | { |
| 404 | int lz; |
| 405 | |
Ralf Baechle | ec917c2c | 2005-10-07 16:58:15 +0100 | [diff] [blame] | 406 | if (sizeof(x) == 4) { |
| 407 | __asm__ ( |
| 408 | " .set push \n" |
| 409 | " .set mips32 \n" |
| 410 | " clz %0, %1 \n" |
| 411 | " .set pop \n" |
| 412 | : "=r" (lz) |
| 413 | : "r" (x)); |
| 414 | |
| 415 | return 31 - lz; |
| 416 | } |
| 417 | |
| 418 | BUG_ON(sizeof(x) != 8); |
| 419 | |
Ralf Baechle | 6590326 | 2005-07-12 12:50:30 +0000 | [diff] [blame] | 420 | __asm__ ( |
| 421 | " .set push \n" |
| 422 | " .set mips64 \n" |
| 423 | " dclz %0, %1 \n" |
| 424 | " .set pop \n" |
| 425 | : "=r" (lz) |
| 426 | : "r" (x)); |
| 427 | |
| 428 | return 63 - lz; |
| 429 | } |
Ralf Baechle | 6590326 | 2005-07-12 12:50:30 +0000 | [diff] [blame] | 430 | |
Akinobu Mita | 3c9ee7e | 2006-03-26 01:39:30 -0800 | [diff] [blame] | 431 | #if defined(CONFIG_CPU_MIPS32) || defined(CONFIG_CPU_MIPS64) |
| 432 | |
Ralf Baechle | 6590326 | 2005-07-12 12:50:30 +0000 | [diff] [blame] | 433 | /* |
| 434 | * __ffs - find first bit in word. |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 435 | * @word: The word to search |
| 436 | * |
Ralf Baechle | 6590326 | 2005-07-12 12:50:30 +0000 | [diff] [blame] | 437 | * Returns 0..SZLONG-1 |
| 438 | * Undefined if no bit exists, so code should check against 0 first. |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 439 | */ |
Ralf Baechle | 6590326 | 2005-07-12 12:50:30 +0000 | [diff] [blame] | 440 | static inline unsigned long __ffs(unsigned long word) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 441 | { |
Ralf Baechle | 6590326 | 2005-07-12 12:50:30 +0000 | [diff] [blame] | 442 | return __ilog2(word & -word); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 443 | } |
| 444 | |
| 445 | /* |
Atsushi Nemoto | bc81824 | 2006-04-17 21:19:12 +0900 | [diff] [blame] | 446 | * fls - find last bit set. |
| 447 | * @word: The word to search |
| 448 | * |
| 449 | * This is defined the same way as ffs. |
| 450 | * Note fls(0) = 0, fls(1) = 1, fls(0x80000000) = 32. |
| 451 | */ |
| 452 | static inline int fls(int word) |
| 453 | { |
| 454 | __asm__ ("clz %0, %1" : "=r" (word) : "r" (word)); |
| 455 | |
| 456 | return 32 - word; |
| 457 | } |
| 458 | |
| 459 | #if defined(CONFIG_64BIT) && defined(CONFIG_CPU_MIPS64) |
| 460 | static inline int fls64(__u64 word) |
| 461 | { |
| 462 | __asm__ ("dclz %0, %1" : "=r" (word) : "r" (word)); |
| 463 | |
| 464 | return 64 - word; |
| 465 | } |
| 466 | #else |
| 467 | #include <asm-generic/bitops/fls64.h> |
| 468 | #endif |
| 469 | |
| 470 | /* |
Ralf Baechle | 6590326 | 2005-07-12 12:50:30 +0000 | [diff] [blame] | 471 | * ffs - find first bit set. |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 472 | * @word: The word to search |
| 473 | * |
Atsushi Nemoto | bc81824 | 2006-04-17 21:19:12 +0900 | [diff] [blame] | 474 | * This is defined the same way as |
| 475 | * the libc and compiler builtin ffs routines, therefore |
| 476 | * differs in spirit from the above ffz (man ffs). |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 477 | */ |
Atsushi Nemoto | bc81824 | 2006-04-17 21:19:12 +0900 | [diff] [blame] | 478 | static inline int ffs(int word) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 479 | { |
Ralf Baechle | 6590326 | 2005-07-12 12:50:30 +0000 | [diff] [blame] | 480 | if (!word) |
| 481 | return 0; |
| 482 | |
Atsushi Nemoto | bc81824 | 2006-04-17 21:19:12 +0900 | [diff] [blame] | 483 | return fls(word & -word); |
Ralf Baechle | 6590326 | 2005-07-12 12:50:30 +0000 | [diff] [blame] | 484 | } |
Ralf Baechle | 2caf190 | 2006-01-30 17:14:41 +0000 | [diff] [blame] | 485 | |
Akinobu Mita | 3c9ee7e | 2006-03-26 01:39:30 -0800 | [diff] [blame] | 486 | #else |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 487 | |
Akinobu Mita | 3c9ee7e | 2006-03-26 01:39:30 -0800 | [diff] [blame] | 488 | #include <asm-generic/bitops/__ffs.h> |
| 489 | #include <asm-generic/bitops/ffs.h> |
Akinobu Mita | 3c9ee7e | 2006-03-26 01:39:30 -0800 | [diff] [blame] | 490 | #include <asm-generic/bitops/fls.h> |
Atsushi Nemoto | bc81824 | 2006-04-17 21:19:12 +0900 | [diff] [blame] | 491 | #include <asm-generic/bitops/fls64.h> |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 492 | |
Akinobu Mita | 3c9ee7e | 2006-03-26 01:39:30 -0800 | [diff] [blame] | 493 | #endif /*defined(CONFIG_CPU_MIPS32) || defined(CONFIG_CPU_MIPS64) */ |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 494 | |
Atsushi Nemoto | bc81824 | 2006-04-17 21:19:12 +0900 | [diff] [blame] | 495 | #include <asm-generic/bitops/ffz.h> |
Akinobu Mita | 3c9ee7e | 2006-03-26 01:39:30 -0800 | [diff] [blame] | 496 | #include <asm-generic/bitops/find.h> |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 497 | |
| 498 | #ifdef __KERNEL__ |
| 499 | |
Akinobu Mita | 3c9ee7e | 2006-03-26 01:39:30 -0800 | [diff] [blame] | 500 | #include <asm-generic/bitops/sched.h> |
| 501 | #include <asm-generic/bitops/hweight.h> |
| 502 | #include <asm-generic/bitops/ext2-non-atomic.h> |
| 503 | #include <asm-generic/bitops/ext2-atomic.h> |
| 504 | #include <asm-generic/bitops/minix.h> |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 505 | |
| 506 | #endif /* __KERNEL__ */ |
| 507 | |
| 508 | #endif /* _ASM_BITOPS_H */ |