Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1 | /* |
| 2 | * This file is subject to the terms and conditions of the GNU General Public |
| 3 | * License. See the file "COPYING" in the main directory of this archive |
| 4 | * for more details. |
| 5 | * |
Ralf Baechle | 102fa15 | 2007-02-16 17:18:50 +0000 | [diff] [blame] | 6 | * Copyright (c) 1994 - 1997, 99, 2000, 06, 07 Ralf Baechle (ralf@linux-mips.org) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 7 | * Copyright (c) 1999, 2000 Silicon Graphics, Inc. |
| 8 | */ |
| 9 | #ifndef _ASM_BITOPS_H |
| 10 | #define _ASM_BITOPS_H |
| 11 | |
Jiri Slaby | 0624517 | 2007-10-18 23:40:26 -0700 | [diff] [blame] | 12 | #ifndef _LINUX_BITOPS_H |
| 13 | #error only <linux/bitops.h> can be included directly |
| 14 | #endif |
| 15 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 16 | #include <linux/compiler.h> |
| 17 | #include <linux/types.h> |
Ralf Baechle | 0004a9d | 2006-10-31 03:45:07 +0000 | [diff] [blame] | 18 | #include <asm/barrier.h> |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 19 | #include <asm/byteorder.h> /* sigh ... */ |
Maciej W. Rozycki | b0984c4 | 2014-11-15 22:08:48 +0000 | [diff] [blame] | 20 | #include <asm/compiler.h> |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 21 | #include <asm/cpu-features.h> |
Ralf Baechle | 0549062 | 2016-04-15 10:25:33 +0200 | [diff] [blame] | 22 | #include <asm/llsc.h> |
Ralf Baechle | 4ffd8b3 | 2006-11-30 01:14:50 +0000 | [diff] [blame] | 23 | #include <asm/sgidefs.h> |
| 24 | #include <asm/war.h> |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 25 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 26 | /* |
Jim Quinlan | 92d1159 | 2012-09-06 11:36:55 -0400 | [diff] [blame] | 27 | * These are the "slower" versions of the functions and are in bitops.c. |
| 28 | * These functions call raw_local_irq_{save,restore}(). |
| 29 | */ |
| 30 | void __mips_set_bit(unsigned long nr, volatile unsigned long *addr); |
| 31 | void __mips_clear_bit(unsigned long nr, volatile unsigned long *addr); |
| 32 | void __mips_change_bit(unsigned long nr, volatile unsigned long *addr); |
| 33 | int __mips_test_and_set_bit(unsigned long nr, |
| 34 | volatile unsigned long *addr); |
| 35 | int __mips_test_and_set_bit_lock(unsigned long nr, |
| 36 | volatile unsigned long *addr); |
| 37 | int __mips_test_and_clear_bit(unsigned long nr, |
| 38 | volatile unsigned long *addr); |
| 39 | int __mips_test_and_change_bit(unsigned long nr, |
| 40 | volatile unsigned long *addr); |
| 41 | |
| 42 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 43 | /* |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 44 | * set_bit - Atomically set a bit in memory |
| 45 | * @nr: the bit to set |
| 46 | * @addr: the address to start counting from |
| 47 | * |
| 48 | * This function is atomic and may not be reordered. See __set_bit() |
| 49 | * if you do not require the atomic guarantees. |
| 50 | * Note that @nr may be almost arbitrarily large; this function is not |
| 51 | * restricted to acting on a single-word quantity. |
| 52 | */ |
| 53 | static inline void set_bit(unsigned long nr, volatile unsigned long *addr) |
| 54 | { |
| 55 | unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG); |
Jim Quinlan | 9de79c5 | 2012-09-06 11:36:54 -0400 | [diff] [blame] | 56 | int bit = nr & SZLONG_MASK; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 57 | unsigned long temp; |
| 58 | |
David Daney | b791d11 | 2009-07-13 11:15:19 -0700 | [diff] [blame] | 59 | if (kernel_uses_llsc && R10000_LLSC_WAR) { |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 60 | __asm__ __volatile__( |
Ralf Baechle | a809d46 | 2014-03-30 13:20:10 +0200 | [diff] [blame] | 61 | " .set arch=r4000 \n" |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 62 | "1: " __LL "%0, %1 # set_bit \n" |
| 63 | " or %0, %2 \n" |
Maciej W. Rozycki | aac8aa7 | 2005-06-14 17:35:03 +0000 | [diff] [blame] | 64 | " " __SC "%0, %1 \n" |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 65 | " beqzl %0, 1b \n" |
Maciej W. Rozycki | aac8aa7 | 2005-06-14 17:35:03 +0000 | [diff] [blame] | 66 | " .set mips0 \n" |
Markos Chandras | 94bfb75 | 2015-01-26 12:44:11 +0000 | [diff] [blame] | 67 | : "=&r" (temp), "=" GCC_OFF_SMALL_ASM() (*m) |
| 68 | : "ir" (1UL << bit), GCC_OFF_SMALL_ASM() (*m)); |
Markos Chandras | 87a927e | 2014-11-20 13:58:30 +0000 | [diff] [blame] | 69 | #if defined(CONFIG_CPU_MIPSR2) || defined(CONFIG_CPU_MIPSR6) |
David Daney | b791d11 | 2009-07-13 11:15:19 -0700 | [diff] [blame] | 70 | } else if (kernel_uses_llsc && __builtin_constant_p(bit)) { |
Ralf Baechle | 7837314 | 2010-10-29 19:08:24 +0100 | [diff] [blame] | 71 | do { |
| 72 | __asm__ __volatile__( |
| 73 | " " __LL "%0, %1 # set_bit \n" |
| 74 | " " __INS "%0, %3, %2, 1 \n" |
| 75 | " " __SC "%0, %1 \n" |
Markos Chandras | 94bfb75 | 2015-01-26 12:44:11 +0000 | [diff] [blame] | 76 | : "=&r" (temp), "+" GCC_OFF_SMALL_ASM() (*m) |
Ralf Baechle | 7837314 | 2010-10-29 19:08:24 +0100 | [diff] [blame] | 77 | : "ir" (bit), "r" (~0)); |
| 78 | } while (unlikely(!temp)); |
Markos Chandras | 87a927e | 2014-11-20 13:58:30 +0000 | [diff] [blame] | 79 | #endif /* CONFIG_CPU_MIPSR2 || CONFIG_CPU_MIPSR6 */ |
David Daney | b791d11 | 2009-07-13 11:15:19 -0700 | [diff] [blame] | 80 | } else if (kernel_uses_llsc) { |
Ralf Baechle | 7837314 | 2010-10-29 19:08:24 +0100 | [diff] [blame] | 81 | do { |
| 82 | __asm__ __volatile__( |
Markos Chandras | 87a927e | 2014-11-20 13:58:30 +0000 | [diff] [blame] | 83 | " .set "MIPS_ISA_ARCH_LEVEL" \n" |
Ralf Baechle | 7837314 | 2010-10-29 19:08:24 +0100 | [diff] [blame] | 84 | " " __LL "%0, %1 # set_bit \n" |
| 85 | " or %0, %2 \n" |
| 86 | " " __SC "%0, %1 \n" |
| 87 | " .set mips0 \n" |
Markos Chandras | 94bfb75 | 2015-01-26 12:44:11 +0000 | [diff] [blame] | 88 | : "=&r" (temp), "+" GCC_OFF_SMALL_ASM() (*m) |
Ralf Baechle | 7837314 | 2010-10-29 19:08:24 +0100 | [diff] [blame] | 89 | : "ir" (1UL << bit)); |
| 90 | } while (unlikely(!temp)); |
Jim Quinlan | 92d1159 | 2012-09-06 11:36:55 -0400 | [diff] [blame] | 91 | } else |
| 92 | __mips_set_bit(nr, addr); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 93 | } |
| 94 | |
| 95 | /* |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 96 | * clear_bit - Clears a bit in memory |
| 97 | * @nr: Bit to clear |
| 98 | * @addr: Address to start counting from |
| 99 | * |
| 100 | * clear_bit() is atomic and may not be reordered. However, it does |
| 101 | * not contain a memory barrier, so if it is used for locking purposes, |
Peter Zijlstra | 91bbefe | 2014-03-13 19:00:36 +0100 | [diff] [blame] | 102 | * you should call smp_mb__before_atomic() and/or smp_mb__after_atomic() |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 103 | * in order to ensure changes are visible on other processors. |
| 104 | */ |
| 105 | static inline void clear_bit(unsigned long nr, volatile unsigned long *addr) |
| 106 | { |
| 107 | unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG); |
Jim Quinlan | 9de79c5 | 2012-09-06 11:36:54 -0400 | [diff] [blame] | 108 | int bit = nr & SZLONG_MASK; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 109 | unsigned long temp; |
| 110 | |
David Daney | b791d11 | 2009-07-13 11:15:19 -0700 | [diff] [blame] | 111 | if (kernel_uses_llsc && R10000_LLSC_WAR) { |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 112 | __asm__ __volatile__( |
Ralf Baechle | a809d46 | 2014-03-30 13:20:10 +0200 | [diff] [blame] | 113 | " .set arch=r4000 \n" |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 114 | "1: " __LL "%0, %1 # clear_bit \n" |
| 115 | " and %0, %2 \n" |
| 116 | " " __SC "%0, %1 \n" |
| 117 | " beqzl %0, 1b \n" |
Maciej W. Rozycki | aac8aa7 | 2005-06-14 17:35:03 +0000 | [diff] [blame] | 118 | " .set mips0 \n" |
Markos Chandras | 94bfb75 | 2015-01-26 12:44:11 +0000 | [diff] [blame] | 119 | : "=&r" (temp), "+" GCC_OFF_SMALL_ASM() (*m) |
Ralf Baechle | 7837314 | 2010-10-29 19:08:24 +0100 | [diff] [blame] | 120 | : "ir" (~(1UL << bit))); |
Markos Chandras | 87a927e | 2014-11-20 13:58:30 +0000 | [diff] [blame] | 121 | #if defined(CONFIG_CPU_MIPSR2) || defined(CONFIG_CPU_MIPSR6) |
David Daney | b791d11 | 2009-07-13 11:15:19 -0700 | [diff] [blame] | 122 | } else if (kernel_uses_llsc && __builtin_constant_p(bit)) { |
Ralf Baechle | 7837314 | 2010-10-29 19:08:24 +0100 | [diff] [blame] | 123 | do { |
| 124 | __asm__ __volatile__( |
| 125 | " " __LL "%0, %1 # clear_bit \n" |
| 126 | " " __INS "%0, $0, %2, 1 \n" |
| 127 | " " __SC "%0, %1 \n" |
Markos Chandras | 94bfb75 | 2015-01-26 12:44:11 +0000 | [diff] [blame] | 128 | : "=&r" (temp), "+" GCC_OFF_SMALL_ASM() (*m) |
Ralf Baechle | 7837314 | 2010-10-29 19:08:24 +0100 | [diff] [blame] | 129 | : "ir" (bit)); |
| 130 | } while (unlikely(!temp)); |
Markos Chandras | 87a927e | 2014-11-20 13:58:30 +0000 | [diff] [blame] | 131 | #endif /* CONFIG_CPU_MIPSR2 || CONFIG_CPU_MIPSR6 */ |
David Daney | b791d11 | 2009-07-13 11:15:19 -0700 | [diff] [blame] | 132 | } else if (kernel_uses_llsc) { |
Ralf Baechle | 7837314 | 2010-10-29 19:08:24 +0100 | [diff] [blame] | 133 | do { |
| 134 | __asm__ __volatile__( |
Markos Chandras | 87a927e | 2014-11-20 13:58:30 +0000 | [diff] [blame] | 135 | " .set "MIPS_ISA_ARCH_LEVEL" \n" |
Ralf Baechle | 7837314 | 2010-10-29 19:08:24 +0100 | [diff] [blame] | 136 | " " __LL "%0, %1 # clear_bit \n" |
| 137 | " and %0, %2 \n" |
| 138 | " " __SC "%0, %1 \n" |
| 139 | " .set mips0 \n" |
Markos Chandras | 94bfb75 | 2015-01-26 12:44:11 +0000 | [diff] [blame] | 140 | : "=&r" (temp), "+" GCC_OFF_SMALL_ASM() (*m) |
Ralf Baechle | 7837314 | 2010-10-29 19:08:24 +0100 | [diff] [blame] | 141 | : "ir" (~(1UL << bit))); |
| 142 | } while (unlikely(!temp)); |
Jim Quinlan | 92d1159 | 2012-09-06 11:36:55 -0400 | [diff] [blame] | 143 | } else |
| 144 | __mips_clear_bit(nr, addr); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 145 | } |
| 146 | |
| 147 | /* |
Nick Piggin | 728697c | 2007-10-18 03:06:53 -0700 | [diff] [blame] | 148 | * clear_bit_unlock - Clears a bit in memory |
| 149 | * @nr: Bit to clear |
| 150 | * @addr: Address to start counting from |
| 151 | * |
| 152 | * clear_bit() is atomic and implies release semantics before the memory |
| 153 | * operation. It can be used for an unlock. |
| 154 | */ |
| 155 | static inline void clear_bit_unlock(unsigned long nr, volatile unsigned long *addr) |
| 156 | { |
Peter Zijlstra | 91bbefe | 2014-03-13 19:00:36 +0100 | [diff] [blame] | 157 | smp_mb__before_atomic(); |
Nick Piggin | 728697c | 2007-10-18 03:06:53 -0700 | [diff] [blame] | 158 | clear_bit(nr, addr); |
| 159 | } |
| 160 | |
| 161 | /* |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 162 | * change_bit - Toggle a bit in memory |
| 163 | * @nr: Bit to change |
| 164 | * @addr: Address to start counting from |
| 165 | * |
| 166 | * change_bit() is atomic and may not be reordered. |
| 167 | * Note that @nr may be almost arbitrarily large; this function is not |
| 168 | * restricted to acting on a single-word quantity. |
| 169 | */ |
| 170 | static inline void change_bit(unsigned long nr, volatile unsigned long *addr) |
| 171 | { |
Jim Quinlan | 9de79c5 | 2012-09-06 11:36:54 -0400 | [diff] [blame] | 172 | int bit = nr & SZLONG_MASK; |
Ralf Baechle | b961153 | 2007-03-05 00:56:15 +0000 | [diff] [blame] | 173 | |
David Daney | b791d11 | 2009-07-13 11:15:19 -0700 | [diff] [blame] | 174 | if (kernel_uses_llsc && R10000_LLSC_WAR) { |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 175 | unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG); |
| 176 | unsigned long temp; |
| 177 | |
| 178 | __asm__ __volatile__( |
Ralf Baechle | a809d46 | 2014-03-30 13:20:10 +0200 | [diff] [blame] | 179 | " .set arch=r4000 \n" |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 180 | "1: " __LL "%0, %1 # change_bit \n" |
| 181 | " xor %0, %2 \n" |
Maciej W. Rozycki | aac8aa7 | 2005-06-14 17:35:03 +0000 | [diff] [blame] | 182 | " " __SC "%0, %1 \n" |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 183 | " beqzl %0, 1b \n" |
Maciej W. Rozycki | aac8aa7 | 2005-06-14 17:35:03 +0000 | [diff] [blame] | 184 | " .set mips0 \n" |
Markos Chandras | 94bfb75 | 2015-01-26 12:44:11 +0000 | [diff] [blame] | 185 | : "=&r" (temp), "+" GCC_OFF_SMALL_ASM() (*m) |
Ralf Baechle | 7837314 | 2010-10-29 19:08:24 +0100 | [diff] [blame] | 186 | : "ir" (1UL << bit)); |
David Daney | b791d11 | 2009-07-13 11:15:19 -0700 | [diff] [blame] | 187 | } else if (kernel_uses_llsc) { |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 188 | unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG); |
| 189 | unsigned long temp; |
| 190 | |
Ralf Baechle | 7837314 | 2010-10-29 19:08:24 +0100 | [diff] [blame] | 191 | do { |
| 192 | __asm__ __volatile__( |
Markos Chandras | 87a927e | 2014-11-20 13:58:30 +0000 | [diff] [blame] | 193 | " .set "MIPS_ISA_ARCH_LEVEL" \n" |
Ralf Baechle | 7837314 | 2010-10-29 19:08:24 +0100 | [diff] [blame] | 194 | " " __LL "%0, %1 # change_bit \n" |
| 195 | " xor %0, %2 \n" |
| 196 | " " __SC "%0, %1 \n" |
| 197 | " .set mips0 \n" |
Markos Chandras | 94bfb75 | 2015-01-26 12:44:11 +0000 | [diff] [blame] | 198 | : "=&r" (temp), "+" GCC_OFF_SMALL_ASM() (*m) |
Ralf Baechle | 7837314 | 2010-10-29 19:08:24 +0100 | [diff] [blame] | 199 | : "ir" (1UL << bit)); |
| 200 | } while (unlikely(!temp)); |
Jim Quinlan | 92d1159 | 2012-09-06 11:36:55 -0400 | [diff] [blame] | 201 | } else |
| 202 | __mips_change_bit(nr, addr); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 203 | } |
| 204 | |
| 205 | /* |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 206 | * test_and_set_bit - Set a bit and return its old value |
| 207 | * @nr: Bit to set |
| 208 | * @addr: Address to count from |
| 209 | * |
| 210 | * This operation is atomic and cannot be reordered. |
| 211 | * It also implies a memory barrier. |
| 212 | */ |
| 213 | static inline int test_and_set_bit(unsigned long nr, |
| 214 | volatile unsigned long *addr) |
| 215 | { |
Jim Quinlan | 9de79c5 | 2012-09-06 11:36:54 -0400 | [diff] [blame] | 216 | int bit = nr & SZLONG_MASK; |
Ralf Baechle | ff72b7a | 2007-06-07 13:17:30 +0100 | [diff] [blame] | 217 | unsigned long res; |
Ralf Baechle | b961153 | 2007-03-05 00:56:15 +0000 | [diff] [blame] | 218 | |
David Daney | f252ffd | 2010-01-08 17:17:43 -0800 | [diff] [blame] | 219 | smp_mb__before_llsc(); |
Nick Piggin | c8f30ae | 2007-10-18 03:06:52 -0700 | [diff] [blame] | 220 | |
David Daney | b791d11 | 2009-07-13 11:15:19 -0700 | [diff] [blame] | 221 | if (kernel_uses_llsc && R10000_LLSC_WAR) { |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 222 | unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG); |
Ralf Baechle | ff72b7a | 2007-06-07 13:17:30 +0100 | [diff] [blame] | 223 | unsigned long temp; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 224 | |
| 225 | __asm__ __volatile__( |
Ralf Baechle | a809d46 | 2014-03-30 13:20:10 +0200 | [diff] [blame] | 226 | " .set arch=r4000 \n" |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 227 | "1: " __LL "%0, %1 # test_and_set_bit \n" |
| 228 | " or %2, %0, %3 \n" |
| 229 | " " __SC "%2, %1 \n" |
| 230 | " beqzl %2, 1b \n" |
| 231 | " and %2, %0, %3 \n" |
Maciej W. Rozycki | aac8aa7 | 2005-06-14 17:35:03 +0000 | [diff] [blame] | 232 | " .set mips0 \n" |
Markos Chandras | 94bfb75 | 2015-01-26 12:44:11 +0000 | [diff] [blame] | 233 | : "=&r" (temp), "+" GCC_OFF_SMALL_ASM() (*m), "=&r" (res) |
Ralf Baechle | 7837314 | 2010-10-29 19:08:24 +0100 | [diff] [blame] | 234 | : "r" (1UL << bit) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 235 | : "memory"); |
David Daney | b791d11 | 2009-07-13 11:15:19 -0700 | [diff] [blame] | 236 | } else if (kernel_uses_llsc) { |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 237 | unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG); |
Ralf Baechle | ff72b7a | 2007-06-07 13:17:30 +0100 | [diff] [blame] | 238 | unsigned long temp; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 239 | |
Ralf Baechle | 7837314 | 2010-10-29 19:08:24 +0100 | [diff] [blame] | 240 | do { |
| 241 | __asm__ __volatile__( |
Markos Chandras | 87a927e | 2014-11-20 13:58:30 +0000 | [diff] [blame] | 242 | " .set "MIPS_ISA_ARCH_LEVEL" \n" |
Ralf Baechle | 7837314 | 2010-10-29 19:08:24 +0100 | [diff] [blame] | 243 | " " __LL "%0, %1 # test_and_set_bit \n" |
| 244 | " or %2, %0, %3 \n" |
| 245 | " " __SC "%2, %1 \n" |
| 246 | " .set mips0 \n" |
Markos Chandras | 94bfb75 | 2015-01-26 12:44:11 +0000 | [diff] [blame] | 247 | : "=&r" (temp), "+" GCC_OFF_SMALL_ASM() (*m), "=&r" (res) |
Ralf Baechle | 7837314 | 2010-10-29 19:08:24 +0100 | [diff] [blame] | 248 | : "r" (1UL << bit) |
| 249 | : "memory"); |
| 250 | } while (unlikely(!res)); |
| 251 | |
| 252 | res = temp & (1UL << bit); |
Jim Quinlan | 92d1159 | 2012-09-06 11:36:55 -0400 | [diff] [blame] | 253 | } else |
| 254 | res = __mips_test_and_set_bit(nr, addr); |
Ralf Baechle | 0004a9d | 2006-10-31 03:45:07 +0000 | [diff] [blame] | 255 | |
Ralf Baechle | 17099b1 | 2007-07-14 13:24:05 +0100 | [diff] [blame] | 256 | smp_llsc_mb(); |
Ralf Baechle | ff72b7a | 2007-06-07 13:17:30 +0100 | [diff] [blame] | 257 | |
| 258 | return res != 0; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 259 | } |
| 260 | |
| 261 | /* |
Nick Piggin | 728697c | 2007-10-18 03:06:53 -0700 | [diff] [blame] | 262 | * test_and_set_bit_lock - Set a bit and return its old value |
| 263 | * @nr: Bit to set |
| 264 | * @addr: Address to count from |
| 265 | * |
| 266 | * This operation is atomic and implies acquire ordering semantics |
| 267 | * after the memory operation. |
| 268 | */ |
| 269 | static inline int test_and_set_bit_lock(unsigned long nr, |
| 270 | volatile unsigned long *addr) |
| 271 | { |
Jim Quinlan | 9de79c5 | 2012-09-06 11:36:54 -0400 | [diff] [blame] | 272 | int bit = nr & SZLONG_MASK; |
Nick Piggin | 728697c | 2007-10-18 03:06:53 -0700 | [diff] [blame] | 273 | unsigned long res; |
| 274 | |
David Daney | b791d11 | 2009-07-13 11:15:19 -0700 | [diff] [blame] | 275 | if (kernel_uses_llsc && R10000_LLSC_WAR) { |
Nick Piggin | 728697c | 2007-10-18 03:06:53 -0700 | [diff] [blame] | 276 | unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG); |
| 277 | unsigned long temp; |
| 278 | |
| 279 | __asm__ __volatile__( |
Ralf Baechle | a809d46 | 2014-03-30 13:20:10 +0200 | [diff] [blame] | 280 | " .set arch=r4000 \n" |
Nick Piggin | 728697c | 2007-10-18 03:06:53 -0700 | [diff] [blame] | 281 | "1: " __LL "%0, %1 # test_and_set_bit \n" |
| 282 | " or %2, %0, %3 \n" |
| 283 | " " __SC "%2, %1 \n" |
| 284 | " beqzl %2, 1b \n" |
| 285 | " and %2, %0, %3 \n" |
| 286 | " .set mips0 \n" |
Ralf Baechle | 7837314 | 2010-10-29 19:08:24 +0100 | [diff] [blame] | 287 | : "=&r" (temp), "+m" (*m), "=&r" (res) |
| 288 | : "r" (1UL << bit) |
Nick Piggin | 728697c | 2007-10-18 03:06:53 -0700 | [diff] [blame] | 289 | : "memory"); |
David Daney | b791d11 | 2009-07-13 11:15:19 -0700 | [diff] [blame] | 290 | } else if (kernel_uses_llsc) { |
Nick Piggin | 728697c | 2007-10-18 03:06:53 -0700 | [diff] [blame] | 291 | unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG); |
| 292 | unsigned long temp; |
| 293 | |
Ralf Baechle | 7837314 | 2010-10-29 19:08:24 +0100 | [diff] [blame] | 294 | do { |
| 295 | __asm__ __volatile__( |
Markos Chandras | 87a927e | 2014-11-20 13:58:30 +0000 | [diff] [blame] | 296 | " .set "MIPS_ISA_ARCH_LEVEL" \n" |
Ralf Baechle | 7837314 | 2010-10-29 19:08:24 +0100 | [diff] [blame] | 297 | " " __LL "%0, %1 # test_and_set_bit \n" |
| 298 | " or %2, %0, %3 \n" |
| 299 | " " __SC "%2, %1 \n" |
| 300 | " .set mips0 \n" |
Markos Chandras | 94bfb75 | 2015-01-26 12:44:11 +0000 | [diff] [blame] | 301 | : "=&r" (temp), "+" GCC_OFF_SMALL_ASM() (*m), "=&r" (res) |
Ralf Baechle | 7837314 | 2010-10-29 19:08:24 +0100 | [diff] [blame] | 302 | : "r" (1UL << bit) |
| 303 | : "memory"); |
| 304 | } while (unlikely(!res)); |
| 305 | |
| 306 | res = temp & (1UL << bit); |
Jim Quinlan | 92d1159 | 2012-09-06 11:36:55 -0400 | [diff] [blame] | 307 | } else |
| 308 | res = __mips_test_and_set_bit_lock(nr, addr); |
Nick Piggin | 728697c | 2007-10-18 03:06:53 -0700 | [diff] [blame] | 309 | |
| 310 | smp_llsc_mb(); |
| 311 | |
| 312 | return res != 0; |
| 313 | } |
| 314 | /* |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 315 | * test_and_clear_bit - Clear a bit and return its old value |
| 316 | * @nr: Bit to clear |
| 317 | * @addr: Address to count from |
| 318 | * |
| 319 | * This operation is atomic and cannot be reordered. |
| 320 | * It also implies a memory barrier. |
| 321 | */ |
| 322 | static inline int test_and_clear_bit(unsigned long nr, |
| 323 | volatile unsigned long *addr) |
| 324 | { |
Jim Quinlan | 9de79c5 | 2012-09-06 11:36:54 -0400 | [diff] [blame] | 325 | int bit = nr & SZLONG_MASK; |
Ralf Baechle | ff72b7a | 2007-06-07 13:17:30 +0100 | [diff] [blame] | 326 | unsigned long res; |
Ralf Baechle | b961153 | 2007-03-05 00:56:15 +0000 | [diff] [blame] | 327 | |
David Daney | f252ffd | 2010-01-08 17:17:43 -0800 | [diff] [blame] | 328 | smp_mb__before_llsc(); |
Nick Piggin | c8f30ae | 2007-10-18 03:06:52 -0700 | [diff] [blame] | 329 | |
David Daney | b791d11 | 2009-07-13 11:15:19 -0700 | [diff] [blame] | 330 | if (kernel_uses_llsc && R10000_LLSC_WAR) { |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 331 | unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG); |
Atsushi Nemoto | 8e09ffb | 2007-06-14 00:56:31 +0900 | [diff] [blame] | 332 | unsigned long temp; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 333 | |
| 334 | __asm__ __volatile__( |
Ralf Baechle | a809d46 | 2014-03-30 13:20:10 +0200 | [diff] [blame] | 335 | " .set arch=r4000 \n" |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 336 | "1: " __LL "%0, %1 # test_and_clear_bit \n" |
| 337 | " or %2, %0, %3 \n" |
| 338 | " xor %2, %3 \n" |
Ralf Baechle | 7034228 | 2013-01-22 12:59:30 +0100 | [diff] [blame] | 339 | " " __SC "%2, %1 \n" |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 340 | " beqzl %2, 1b \n" |
| 341 | " and %2, %0, %3 \n" |
Maciej W. Rozycki | aac8aa7 | 2005-06-14 17:35:03 +0000 | [diff] [blame] | 342 | " .set mips0 \n" |
Markos Chandras | 94bfb75 | 2015-01-26 12:44:11 +0000 | [diff] [blame] | 343 | : "=&r" (temp), "+" GCC_OFF_SMALL_ASM() (*m), "=&r" (res) |
Ralf Baechle | 7837314 | 2010-10-29 19:08:24 +0100 | [diff] [blame] | 344 | : "r" (1UL << bit) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 345 | : "memory"); |
Markos Chandras | 87a927e | 2014-11-20 13:58:30 +0000 | [diff] [blame] | 346 | #if defined(CONFIG_CPU_MIPSR2) || defined(CONFIG_CPU_MIPSR6) |
David Daney | b791d11 | 2009-07-13 11:15:19 -0700 | [diff] [blame] | 347 | } else if (kernel_uses_llsc && __builtin_constant_p(nr)) { |
Ralf Baechle | 102fa15 | 2007-02-16 17:18:50 +0000 | [diff] [blame] | 348 | unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG); |
Ralf Baechle | ff72b7a | 2007-06-07 13:17:30 +0100 | [diff] [blame] | 349 | unsigned long temp; |
Ralf Baechle | 102fa15 | 2007-02-16 17:18:50 +0000 | [diff] [blame] | 350 | |
Ralf Baechle | 7837314 | 2010-10-29 19:08:24 +0100 | [diff] [blame] | 351 | do { |
| 352 | __asm__ __volatile__( |
Ralf Baechle | 7034228 | 2013-01-22 12:59:30 +0100 | [diff] [blame] | 353 | " " __LL "%0, %1 # test_and_clear_bit \n" |
Ralf Baechle | 7837314 | 2010-10-29 19:08:24 +0100 | [diff] [blame] | 354 | " " __EXT "%2, %0, %3, 1 \n" |
Ralf Baechle | 7034228 | 2013-01-22 12:59:30 +0100 | [diff] [blame] | 355 | " " __INS "%0, $0, %3, 1 \n" |
| 356 | " " __SC "%0, %1 \n" |
Markos Chandras | 94bfb75 | 2015-01-26 12:44:11 +0000 | [diff] [blame] | 357 | : "=&r" (temp), "+" GCC_OFF_SMALL_ASM() (*m), "=&r" (res) |
Ralf Baechle | 7837314 | 2010-10-29 19:08:24 +0100 | [diff] [blame] | 358 | : "ir" (bit) |
| 359 | : "memory"); |
| 360 | } while (unlikely(!temp)); |
Ralf Baechle | 102fa15 | 2007-02-16 17:18:50 +0000 | [diff] [blame] | 361 | #endif |
David Daney | b791d11 | 2009-07-13 11:15:19 -0700 | [diff] [blame] | 362 | } else if (kernel_uses_llsc) { |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 363 | unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG); |
Ralf Baechle | ff72b7a | 2007-06-07 13:17:30 +0100 | [diff] [blame] | 364 | unsigned long temp; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 365 | |
Ralf Baechle | 7837314 | 2010-10-29 19:08:24 +0100 | [diff] [blame] | 366 | do { |
| 367 | __asm__ __volatile__( |
Markos Chandras | 87a927e | 2014-11-20 13:58:30 +0000 | [diff] [blame] | 368 | " .set "MIPS_ISA_ARCH_LEVEL" \n" |
Ralf Baechle | 7034228 | 2013-01-22 12:59:30 +0100 | [diff] [blame] | 369 | " " __LL "%0, %1 # test_and_clear_bit \n" |
Ralf Baechle | 7837314 | 2010-10-29 19:08:24 +0100 | [diff] [blame] | 370 | " or %2, %0, %3 \n" |
| 371 | " xor %2, %3 \n" |
Ralf Baechle | 7034228 | 2013-01-22 12:59:30 +0100 | [diff] [blame] | 372 | " " __SC "%2, %1 \n" |
Ralf Baechle | 7837314 | 2010-10-29 19:08:24 +0100 | [diff] [blame] | 373 | " .set mips0 \n" |
Markos Chandras | 94bfb75 | 2015-01-26 12:44:11 +0000 | [diff] [blame] | 374 | : "=&r" (temp), "+" GCC_OFF_SMALL_ASM() (*m), "=&r" (res) |
Ralf Baechle | 7837314 | 2010-10-29 19:08:24 +0100 | [diff] [blame] | 375 | : "r" (1UL << bit) |
| 376 | : "memory"); |
| 377 | } while (unlikely(!res)); |
| 378 | |
| 379 | res = temp & (1UL << bit); |
Jim Quinlan | 92d1159 | 2012-09-06 11:36:55 -0400 | [diff] [blame] | 380 | } else |
| 381 | res = __mips_test_and_clear_bit(nr, addr); |
Ralf Baechle | 0004a9d | 2006-10-31 03:45:07 +0000 | [diff] [blame] | 382 | |
Ralf Baechle | 17099b1 | 2007-07-14 13:24:05 +0100 | [diff] [blame] | 383 | smp_llsc_mb(); |
Ralf Baechle | ff72b7a | 2007-06-07 13:17:30 +0100 | [diff] [blame] | 384 | |
| 385 | return res != 0; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 386 | } |
| 387 | |
| 388 | /* |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 389 | * test_and_change_bit - Change a bit and return its old value |
| 390 | * @nr: Bit to change |
| 391 | * @addr: Address to count from |
| 392 | * |
| 393 | * This operation is atomic and cannot be reordered. |
| 394 | * It also implies a memory barrier. |
| 395 | */ |
| 396 | static inline int test_and_change_bit(unsigned long nr, |
| 397 | volatile unsigned long *addr) |
| 398 | { |
Jim Quinlan | 9de79c5 | 2012-09-06 11:36:54 -0400 | [diff] [blame] | 399 | int bit = nr & SZLONG_MASK; |
Ralf Baechle | ff72b7a | 2007-06-07 13:17:30 +0100 | [diff] [blame] | 400 | unsigned long res; |
Ralf Baechle | b961153 | 2007-03-05 00:56:15 +0000 | [diff] [blame] | 401 | |
David Daney | f252ffd | 2010-01-08 17:17:43 -0800 | [diff] [blame] | 402 | smp_mb__before_llsc(); |
Nick Piggin | c8f30ae | 2007-10-18 03:06:52 -0700 | [diff] [blame] | 403 | |
David Daney | b791d11 | 2009-07-13 11:15:19 -0700 | [diff] [blame] | 404 | if (kernel_uses_llsc && R10000_LLSC_WAR) { |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 405 | unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG); |
Ralf Baechle | ff72b7a | 2007-06-07 13:17:30 +0100 | [diff] [blame] | 406 | unsigned long temp; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 407 | |
| 408 | __asm__ __volatile__( |
Ralf Baechle | a809d46 | 2014-03-30 13:20:10 +0200 | [diff] [blame] | 409 | " .set arch=r4000 \n" |
Maciej W. Rozycki | aac8aa7 | 2005-06-14 17:35:03 +0000 | [diff] [blame] | 410 | "1: " __LL "%0, %1 # test_and_change_bit \n" |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 411 | " xor %2, %0, %3 \n" |
Maciej W. Rozycki | aac8aa7 | 2005-06-14 17:35:03 +0000 | [diff] [blame] | 412 | " " __SC "%2, %1 \n" |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 413 | " beqzl %2, 1b \n" |
| 414 | " and %2, %0, %3 \n" |
Maciej W. Rozycki | aac8aa7 | 2005-06-14 17:35:03 +0000 | [diff] [blame] | 415 | " .set mips0 \n" |
Markos Chandras | 94bfb75 | 2015-01-26 12:44:11 +0000 | [diff] [blame] | 416 | : "=&r" (temp), "+" GCC_OFF_SMALL_ASM() (*m), "=&r" (res) |
Ralf Baechle | 7837314 | 2010-10-29 19:08:24 +0100 | [diff] [blame] | 417 | : "r" (1UL << bit) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 418 | : "memory"); |
David Daney | b791d11 | 2009-07-13 11:15:19 -0700 | [diff] [blame] | 419 | } else if (kernel_uses_llsc) { |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 420 | unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG); |
Ralf Baechle | ff72b7a | 2007-06-07 13:17:30 +0100 | [diff] [blame] | 421 | unsigned long temp; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 422 | |
Ralf Baechle | 7837314 | 2010-10-29 19:08:24 +0100 | [diff] [blame] | 423 | do { |
| 424 | __asm__ __volatile__( |
Markos Chandras | 87a927e | 2014-11-20 13:58:30 +0000 | [diff] [blame] | 425 | " .set "MIPS_ISA_ARCH_LEVEL" \n" |
Ralf Baechle | 7034228 | 2013-01-22 12:59:30 +0100 | [diff] [blame] | 426 | " " __LL "%0, %1 # test_and_change_bit \n" |
Ralf Baechle | 7837314 | 2010-10-29 19:08:24 +0100 | [diff] [blame] | 427 | " xor %2, %0, %3 \n" |
| 428 | " " __SC "\t%2, %1 \n" |
| 429 | " .set mips0 \n" |
Markos Chandras | 94bfb75 | 2015-01-26 12:44:11 +0000 | [diff] [blame] | 430 | : "=&r" (temp), "+" GCC_OFF_SMALL_ASM() (*m), "=&r" (res) |
Ralf Baechle | 7837314 | 2010-10-29 19:08:24 +0100 | [diff] [blame] | 431 | : "r" (1UL << bit) |
| 432 | : "memory"); |
| 433 | } while (unlikely(!res)); |
| 434 | |
| 435 | res = temp & (1UL << bit); |
Jim Quinlan | 92d1159 | 2012-09-06 11:36:55 -0400 | [diff] [blame] | 436 | } else |
| 437 | res = __mips_test_and_change_bit(nr, addr); |
Ralf Baechle | 0004a9d | 2006-10-31 03:45:07 +0000 | [diff] [blame] | 438 | |
Ralf Baechle | 17099b1 | 2007-07-14 13:24:05 +0100 | [diff] [blame] | 439 | smp_llsc_mb(); |
Ralf Baechle | ff72b7a | 2007-06-07 13:17:30 +0100 | [diff] [blame] | 440 | |
| 441 | return res != 0; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 442 | } |
| 443 | |
Akinobu Mita | 3c9ee7e | 2006-03-26 01:39:30 -0800 | [diff] [blame] | 444 | #include <asm-generic/bitops/non-atomic.h> |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 445 | |
Ralf Baechle | 6590326 | 2005-07-12 12:50:30 +0000 | [diff] [blame] | 446 | /* |
Nick Piggin | 728697c | 2007-10-18 03:06:53 -0700 | [diff] [blame] | 447 | * __clear_bit_unlock - Clears a bit in memory |
| 448 | * @nr: Bit to clear |
| 449 | * @addr: Address to start counting from |
| 450 | * |
| 451 | * __clear_bit() is non-atomic and implies release semantics before the memory |
| 452 | * operation. It can be used for an unlock if no other CPUs can concurrently |
| 453 | * modify other bits in the word. |
| 454 | */ |
| 455 | static inline void __clear_bit_unlock(unsigned long nr, volatile unsigned long *addr) |
| 456 | { |
Leonid Yegoshin | 6f6ed48 | 2015-06-01 17:09:52 -0700 | [diff] [blame] | 457 | smp_mb__before_llsc(); |
Nick Piggin | 728697c | 2007-10-18 03:06:53 -0700 | [diff] [blame] | 458 | __clear_bit(nr, addr); |
| 459 | } |
| 460 | |
| 461 | /* |
Ralf Baechle | 6590326 | 2005-07-12 12:50:30 +0000 | [diff] [blame] | 462 | * Return the bit position (0..63) of the most significant 1 bit in a word |
| 463 | * Returns -1 if no 1 bit exists |
| 464 | */ |
Ralf Baechle | 4816227 | 2008-10-28 09:40:35 +0000 | [diff] [blame] | 465 | static inline unsigned long __fls(unsigned long word) |
Ralf Baechle | 6590326 | 2005-07-12 12:50:30 +0000 | [diff] [blame] | 466 | { |
Ralf Baechle | 4816227 | 2008-10-28 09:40:35 +0000 | [diff] [blame] | 467 | int num; |
Ralf Baechle | 6590326 | 2005-07-12 12:50:30 +0000 | [diff] [blame] | 468 | |
Maciej W. Rozycki | cb5d4aa | 2015-04-03 23:25:00 +0100 | [diff] [blame] | 469 | if (BITS_PER_LONG == 32 && !__builtin_constant_p(word) && |
Ralf Baechle | 47740eb | 2009-04-19 03:21:22 +0200 | [diff] [blame] | 470 | __builtin_constant_p(cpu_has_clo_clz) && cpu_has_clo_clz) { |
Ralf Baechle | 49a89ef | 2007-10-11 23:46:15 +0100 | [diff] [blame] | 471 | __asm__( |
Ralf Baechle | ec917c2c | 2005-10-07 16:58:15 +0100 | [diff] [blame] | 472 | " .set push \n" |
Markos Chandras | 87a927e | 2014-11-20 13:58:30 +0000 | [diff] [blame] | 473 | " .set "MIPS_ISA_LEVEL" \n" |
Ralf Baechle | ec917c2c | 2005-10-07 16:58:15 +0100 | [diff] [blame] | 474 | " clz %0, %1 \n" |
| 475 | " .set pop \n" |
Ralf Baechle | 4816227 | 2008-10-28 09:40:35 +0000 | [diff] [blame] | 476 | : "=r" (num) |
| 477 | : "r" (word)); |
Ralf Baechle | ec917c2c | 2005-10-07 16:58:15 +0100 | [diff] [blame] | 478 | |
Ralf Baechle | 4816227 | 2008-10-28 09:40:35 +0000 | [diff] [blame] | 479 | return 31 - num; |
Ralf Baechle | ec917c2c | 2005-10-07 16:58:15 +0100 | [diff] [blame] | 480 | } |
| 481 | |
Maciej W. Rozycki | cb5d4aa | 2015-04-03 23:25:00 +0100 | [diff] [blame] | 482 | if (BITS_PER_LONG == 64 && !__builtin_constant_p(word) && |
Ralf Baechle | 4816227 | 2008-10-28 09:40:35 +0000 | [diff] [blame] | 483 | __builtin_constant_p(cpu_has_mips64) && cpu_has_mips64) { |
| 484 | __asm__( |
| 485 | " .set push \n" |
Markos Chandras | 87a927e | 2014-11-20 13:58:30 +0000 | [diff] [blame] | 486 | " .set "MIPS_ISA_LEVEL" \n" |
Ralf Baechle | 4816227 | 2008-10-28 09:40:35 +0000 | [diff] [blame] | 487 | " dclz %0, %1 \n" |
| 488 | " .set pop \n" |
| 489 | : "=r" (num) |
| 490 | : "r" (word)); |
Ralf Baechle | ec917c2c | 2005-10-07 16:58:15 +0100 | [diff] [blame] | 491 | |
Ralf Baechle | 4816227 | 2008-10-28 09:40:35 +0000 | [diff] [blame] | 492 | return 63 - num; |
| 493 | } |
Ralf Baechle | 6590326 | 2005-07-12 12:50:30 +0000 | [diff] [blame] | 494 | |
Ralf Baechle | 4816227 | 2008-10-28 09:40:35 +0000 | [diff] [blame] | 495 | num = BITS_PER_LONG - 1; |
| 496 | |
| 497 | #if BITS_PER_LONG == 64 |
| 498 | if (!(word & (~0ul << 32))) { |
| 499 | num -= 32; |
| 500 | word <<= 32; |
| 501 | } |
| 502 | #endif |
| 503 | if (!(word & (~0ul << (BITS_PER_LONG-16)))) { |
| 504 | num -= 16; |
| 505 | word <<= 16; |
| 506 | } |
| 507 | if (!(word & (~0ul << (BITS_PER_LONG-8)))) { |
| 508 | num -= 8; |
| 509 | word <<= 8; |
| 510 | } |
| 511 | if (!(word & (~0ul << (BITS_PER_LONG-4)))) { |
| 512 | num -= 4; |
| 513 | word <<= 4; |
| 514 | } |
| 515 | if (!(word & (~0ul << (BITS_PER_LONG-2)))) { |
| 516 | num -= 2; |
| 517 | word <<= 2; |
| 518 | } |
| 519 | if (!(word & (~0ul << (BITS_PER_LONG-1)))) |
| 520 | num -= 1; |
| 521 | return num; |
Ralf Baechle | 6590326 | 2005-07-12 12:50:30 +0000 | [diff] [blame] | 522 | } |
Ralf Baechle | 6590326 | 2005-07-12 12:50:30 +0000 | [diff] [blame] | 523 | |
| 524 | /* |
| 525 | * __ffs - find first bit in word. |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 526 | * @word: The word to search |
| 527 | * |
Ralf Baechle | 6590326 | 2005-07-12 12:50:30 +0000 | [diff] [blame] | 528 | * Returns 0..SZLONG-1 |
| 529 | * Undefined if no bit exists, so code should check against 0 first. |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 530 | */ |
Ralf Baechle | 6590326 | 2005-07-12 12:50:30 +0000 | [diff] [blame] | 531 | static inline unsigned long __ffs(unsigned long word) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 532 | { |
Ralf Baechle | ddc0d00 | 2008-05-04 14:53:53 +0100 | [diff] [blame] | 533 | return __fls(word & -word); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 534 | } |
| 535 | |
| 536 | /* |
Atsushi Nemoto | bc81824 | 2006-04-17 21:19:12 +0900 | [diff] [blame] | 537 | * fls - find last bit set. |
| 538 | * @word: The word to search |
| 539 | * |
| 540 | * This is defined the same way as ffs. |
| 541 | * Note fls(0) = 0, fls(1) = 1, fls(0x80000000) = 32. |
| 542 | */ |
Ralf Baechle | 4816227 | 2008-10-28 09:40:35 +0000 | [diff] [blame] | 543 | static inline int fls(int x) |
Atsushi Nemoto | bc81824 | 2006-04-17 21:19:12 +0900 | [diff] [blame] | 544 | { |
Ralf Baechle | 4816227 | 2008-10-28 09:40:35 +0000 | [diff] [blame] | 545 | int r; |
Atsushi Nemoto | bc81824 | 2006-04-17 21:19:12 +0900 | [diff] [blame] | 546 | |
Maciej W. Rozycki | cb5d4aa | 2015-04-03 23:25:00 +0100 | [diff] [blame] | 547 | if (!__builtin_constant_p(x) && |
| 548 | __builtin_constant_p(cpu_has_clo_clz) && cpu_has_clo_clz) { |
Maciej W. Rozycki | db87313 | 2014-06-29 00:26:20 +0100 | [diff] [blame] | 549 | __asm__( |
| 550 | " .set push \n" |
Markos Chandras | 87a927e | 2014-11-20 13:58:30 +0000 | [diff] [blame] | 551 | " .set "MIPS_ISA_LEVEL" \n" |
Maciej W. Rozycki | db87313 | 2014-06-29 00:26:20 +0100 | [diff] [blame] | 552 | " clz %0, %1 \n" |
| 553 | " .set pop \n" |
| 554 | : "=r" (x) |
| 555 | : "r" (x)); |
Ralf Baechle | 4816227 | 2008-10-28 09:40:35 +0000 | [diff] [blame] | 556 | |
| 557 | return 32 - x; |
| 558 | } |
| 559 | |
| 560 | r = 32; |
| 561 | if (!x) |
| 562 | return 0; |
| 563 | if (!(x & 0xffff0000u)) { |
| 564 | x <<= 16; |
| 565 | r -= 16; |
| 566 | } |
| 567 | if (!(x & 0xff000000u)) { |
| 568 | x <<= 8; |
| 569 | r -= 8; |
| 570 | } |
| 571 | if (!(x & 0xf0000000u)) { |
| 572 | x <<= 4; |
| 573 | r -= 4; |
| 574 | } |
| 575 | if (!(x & 0xc0000000u)) { |
| 576 | x <<= 2; |
| 577 | r -= 2; |
| 578 | } |
| 579 | if (!(x & 0x80000000u)) { |
| 580 | x <<= 1; |
| 581 | r -= 1; |
| 582 | } |
| 583 | return r; |
Atsushi Nemoto | bc81824 | 2006-04-17 21:19:12 +0900 | [diff] [blame] | 584 | } |
| 585 | |
Atsushi Nemoto | bc81824 | 2006-04-17 21:19:12 +0900 | [diff] [blame] | 586 | #include <asm-generic/bitops/fls64.h> |
Atsushi Nemoto | bc81824 | 2006-04-17 21:19:12 +0900 | [diff] [blame] | 587 | |
| 588 | /* |
Ralf Baechle | 6590326 | 2005-07-12 12:50:30 +0000 | [diff] [blame] | 589 | * ffs - find first bit set. |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 590 | * @word: The word to search |
| 591 | * |
Atsushi Nemoto | bc81824 | 2006-04-17 21:19:12 +0900 | [diff] [blame] | 592 | * This is defined the same way as |
| 593 | * the libc and compiler builtin ffs routines, therefore |
| 594 | * differs in spirit from the above ffz (man ffs). |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 595 | */ |
Atsushi Nemoto | bc81824 | 2006-04-17 21:19:12 +0900 | [diff] [blame] | 596 | static inline int ffs(int word) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 597 | { |
Ralf Baechle | 6590326 | 2005-07-12 12:50:30 +0000 | [diff] [blame] | 598 | if (!word) |
| 599 | return 0; |
| 600 | |
Atsushi Nemoto | bc81824 | 2006-04-17 21:19:12 +0900 | [diff] [blame] | 601 | return fls(word & -word); |
Ralf Baechle | 6590326 | 2005-07-12 12:50:30 +0000 | [diff] [blame] | 602 | } |
Ralf Baechle | 2caf190 | 2006-01-30 17:14:41 +0000 | [diff] [blame] | 603 | |
Atsushi Nemoto | bc81824 | 2006-04-17 21:19:12 +0900 | [diff] [blame] | 604 | #include <asm-generic/bitops/ffz.h> |
Akinobu Mita | 3c9ee7e | 2006-03-26 01:39:30 -0800 | [diff] [blame] | 605 | #include <asm-generic/bitops/find.h> |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 606 | |
| 607 | #ifdef __KERNEL__ |
| 608 | |
Akinobu Mita | 3c9ee7e | 2006-03-26 01:39:30 -0800 | [diff] [blame] | 609 | #include <asm-generic/bitops/sched.h> |
David Daney | 1a403d1 | 2010-06-25 16:46:07 -0700 | [diff] [blame] | 610 | |
| 611 | #include <asm/arch_hweight.h> |
| 612 | #include <asm-generic/bitops/const_hweight.h> |
| 613 | |
Akinobu Mita | 861b5ae | 2011-03-23 16:42:02 -0700 | [diff] [blame] | 614 | #include <asm-generic/bitops/le.h> |
Akinobu Mita | 3c9ee7e | 2006-03-26 01:39:30 -0800 | [diff] [blame] | 615 | #include <asm-generic/bitops/ext2-atomic.h> |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 616 | |
| 617 | #endif /* __KERNEL__ */ |
| 618 | |
| 619 | #endif /* _ASM_BITOPS_H */ |