blob: ec75ce4cdb8c3e90307e51350c2d7dc4b950a4d5 [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
2 * This file is subject to the terms and conditions of the GNU General Public
3 * License. See the file "COPYING" in the main directory of this archive
4 * for more details.
5 *
Ralf Baechle102fa152007-02-16 17:18:50 +00006 * Copyright (c) 1994 - 1997, 99, 2000, 06, 07 Ralf Baechle (ralf@linux-mips.org)
Linus Torvalds1da177e2005-04-16 15:20:36 -07007 * Copyright (c) 1999, 2000 Silicon Graphics, Inc.
8 */
9#ifndef _ASM_BITOPS_H
10#define _ASM_BITOPS_H
11
Jiri Slaby06245172007-10-18 23:40:26 -070012#ifndef _LINUX_BITOPS_H
13#error only <linux/bitops.h> can be included directly
14#endif
15
Linus Torvalds1da177e2005-04-16 15:20:36 -070016#include <linux/compiler.h>
Ralf Baechle4ffd8b32006-11-30 01:14:50 +000017#include <linux/irqflags.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070018#include <linux/types.h>
Ralf Baechle0004a9d2006-10-31 03:45:07 +000019#include <asm/barrier.h>
Ralf Baechleec917c2c2005-10-07 16:58:15 +010020#include <asm/bug.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070021#include <asm/byteorder.h> /* sigh ... */
22#include <asm/cpu-features.h>
Ralf Baechle4ffd8b32006-11-30 01:14:50 +000023#include <asm/sgidefs.h>
24#include <asm/war.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070025
Ralf Baechle49a89ef2007-10-11 23:46:15 +010026#if _MIPS_SZLONG == 32
Linus Torvalds1da177e2005-04-16 15:20:36 -070027#define SZLONG_LOG 5
28#define SZLONG_MASK 31UL
Maciej W. Rozyckiaac8aa72005-06-14 17:35:03 +000029#define __LL "ll "
30#define __SC "sc "
Ralf Baechle102fa152007-02-16 17:18:50 +000031#define __INS "ins "
32#define __EXT "ext "
Ralf Baechle49a89ef2007-10-11 23:46:15 +010033#elif _MIPS_SZLONG == 64
Linus Torvalds1da177e2005-04-16 15:20:36 -070034#define SZLONG_LOG 6
35#define SZLONG_MASK 63UL
Maciej W. Rozyckiaac8aa72005-06-14 17:35:03 +000036#define __LL "lld "
37#define __SC "scd "
Ralf Baechle102fa152007-02-16 17:18:50 +000038#define __INS "dins "
39#define __EXT "dext "
Linus Torvalds1da177e2005-04-16 15:20:36 -070040#endif
41
Linus Torvalds1da177e2005-04-16 15:20:36 -070042/*
43 * clear_bit() doesn't provide any barrier for the compiler.
44 */
Ralf Baechle17099b12007-07-14 13:24:05 +010045#define smp_mb__before_clear_bit() smp_llsc_mb()
46#define smp_mb__after_clear_bit() smp_llsc_mb()
Linus Torvalds1da177e2005-04-16 15:20:36 -070047
48/*
Linus Torvalds1da177e2005-04-16 15:20:36 -070049 * set_bit - Atomically set a bit in memory
50 * @nr: the bit to set
51 * @addr: the address to start counting from
52 *
53 * This function is atomic and may not be reordered. See __set_bit()
54 * if you do not require the atomic guarantees.
55 * Note that @nr may be almost arbitrarily large; this function is not
56 * restricted to acting on a single-word quantity.
57 */
58static inline void set_bit(unsigned long nr, volatile unsigned long *addr)
59{
60 unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG);
Ralf Baechleb9611532007-03-05 00:56:15 +000061 unsigned short bit = nr & SZLONG_MASK;
Linus Torvalds1da177e2005-04-16 15:20:36 -070062 unsigned long temp;
63
64 if (cpu_has_llsc && R10000_LLSC_WAR) {
65 __asm__ __volatile__(
Maciej W. Rozyckic4559f62005-06-23 15:57:15 +000066 " .set mips3 \n"
Linus Torvalds1da177e2005-04-16 15:20:36 -070067 "1: " __LL "%0, %1 # set_bit \n"
68 " or %0, %2 \n"
Maciej W. Rozyckiaac8aa72005-06-14 17:35:03 +000069 " " __SC "%0, %1 \n"
Linus Torvalds1da177e2005-04-16 15:20:36 -070070 " beqzl %0, 1b \n"
Maciej W. Rozyckiaac8aa72005-06-14 17:35:03 +000071 " .set mips0 \n"
Linus Torvalds1da177e2005-04-16 15:20:36 -070072 : "=&r" (temp), "=m" (*m)
Ralf Baechleb9611532007-03-05 00:56:15 +000073 : "ir" (1UL << bit), "m" (*m));
Ralf Baechle102fa152007-02-16 17:18:50 +000074#ifdef CONFIG_CPU_MIPSR2
Ralf Baechleb9611532007-03-05 00:56:15 +000075 } else if (__builtin_constant_p(bit)) {
Ralf Baechle102fa152007-02-16 17:18:50 +000076 __asm__ __volatile__(
77 "1: " __LL "%0, %1 # set_bit \n"
78 " " __INS "%0, %4, %2, 1 \n"
79 " " __SC "%0, %1 \n"
80 " beqz %0, 2f \n"
81 " .subsection 2 \n"
82 "2: b 1b \n"
83 " .previous \n"
84 : "=&r" (temp), "=m" (*m)
Ralf Baechleb9611532007-03-05 00:56:15 +000085 : "ir" (bit), "m" (*m), "r" (~0));
Ralf Baechle102fa152007-02-16 17:18:50 +000086#endif /* CONFIG_CPU_MIPSR2 */
Linus Torvalds1da177e2005-04-16 15:20:36 -070087 } else if (cpu_has_llsc) {
88 __asm__ __volatile__(
Maciej W. Rozyckic4559f62005-06-23 15:57:15 +000089 " .set mips3 \n"
Linus Torvalds1da177e2005-04-16 15:20:36 -070090 "1: " __LL "%0, %1 # set_bit \n"
91 " or %0, %2 \n"
Maciej W. Rozyckiaac8aa72005-06-14 17:35:03 +000092 " " __SC "%0, %1 \n"
Ralf Baechlef65e4fa2006-09-28 01:45:21 +010093 " beqz %0, 2f \n"
94 " .subsection 2 \n"
95 "2: b 1b \n"
96 " .previous \n"
Maciej W. Rozyckiaac8aa72005-06-14 17:35:03 +000097 " .set mips0 \n"
Linus Torvalds1da177e2005-04-16 15:20:36 -070098 : "=&r" (temp), "=m" (*m)
Ralf Baechleb9611532007-03-05 00:56:15 +000099 : "ir" (1UL << bit), "m" (*m));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700100 } else {
101 volatile unsigned long *a = addr;
102 unsigned long mask;
Ralf Baechle4ffd8b32006-11-30 01:14:50 +0000103 unsigned long flags;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700104
105 a += nr >> SZLONG_LOG;
Ralf Baechleb9611532007-03-05 00:56:15 +0000106 mask = 1UL << bit;
Ralf Baechle49edd092007-03-16 16:10:36 +0000107 raw_local_irq_save(flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700108 *a |= mask;
Ralf Baechle49edd092007-03-16 16:10:36 +0000109 raw_local_irq_restore(flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700110 }
111}
112
113/*
Linus Torvalds1da177e2005-04-16 15:20:36 -0700114 * clear_bit - Clears a bit in memory
115 * @nr: Bit to clear
116 * @addr: Address to start counting from
117 *
118 * clear_bit() is atomic and may not be reordered. However, it does
119 * not contain a memory barrier, so if it is used for locking purposes,
120 * you should call smp_mb__before_clear_bit() and/or smp_mb__after_clear_bit()
121 * in order to ensure changes are visible on other processors.
122 */
123static inline void clear_bit(unsigned long nr, volatile unsigned long *addr)
124{
125 unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG);
Ralf Baechleb9611532007-03-05 00:56:15 +0000126 unsigned short bit = nr & SZLONG_MASK;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700127 unsigned long temp;
128
129 if (cpu_has_llsc && R10000_LLSC_WAR) {
130 __asm__ __volatile__(
Maciej W. Rozyckic4559f62005-06-23 15:57:15 +0000131 " .set mips3 \n"
Linus Torvalds1da177e2005-04-16 15:20:36 -0700132 "1: " __LL "%0, %1 # clear_bit \n"
133 " and %0, %2 \n"
134 " " __SC "%0, %1 \n"
135 " beqzl %0, 1b \n"
Maciej W. Rozyckiaac8aa72005-06-14 17:35:03 +0000136 " .set mips0 \n"
Linus Torvalds1da177e2005-04-16 15:20:36 -0700137 : "=&r" (temp), "=m" (*m)
Ralf Baechleb9611532007-03-05 00:56:15 +0000138 : "ir" (~(1UL << bit)), "m" (*m));
Ralf Baechle102fa152007-02-16 17:18:50 +0000139#ifdef CONFIG_CPU_MIPSR2
Ralf Baechleb9611532007-03-05 00:56:15 +0000140 } else if (__builtin_constant_p(bit)) {
Ralf Baechle102fa152007-02-16 17:18:50 +0000141 __asm__ __volatile__(
142 "1: " __LL "%0, %1 # clear_bit \n"
143 " " __INS "%0, $0, %2, 1 \n"
144 " " __SC "%0, %1 \n"
145 " beqz %0, 2f \n"
146 " .subsection 2 \n"
147 "2: b 1b \n"
148 " .previous \n"
149 : "=&r" (temp), "=m" (*m)
Ralf Baechleb9611532007-03-05 00:56:15 +0000150 : "ir" (bit), "m" (*m));
Ralf Baechle102fa152007-02-16 17:18:50 +0000151#endif /* CONFIG_CPU_MIPSR2 */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700152 } else if (cpu_has_llsc) {
153 __asm__ __volatile__(
Maciej W. Rozyckic4559f62005-06-23 15:57:15 +0000154 " .set mips3 \n"
Linus Torvalds1da177e2005-04-16 15:20:36 -0700155 "1: " __LL "%0, %1 # clear_bit \n"
156 " and %0, %2 \n"
157 " " __SC "%0, %1 \n"
Ralf Baechlef65e4fa2006-09-28 01:45:21 +0100158 " beqz %0, 2f \n"
159 " .subsection 2 \n"
160 "2: b 1b \n"
161 " .previous \n"
Maciej W. Rozyckiaac8aa72005-06-14 17:35:03 +0000162 " .set mips0 \n"
Linus Torvalds1da177e2005-04-16 15:20:36 -0700163 : "=&r" (temp), "=m" (*m)
Ralf Baechleb9611532007-03-05 00:56:15 +0000164 : "ir" (~(1UL << bit)), "m" (*m));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700165 } else {
166 volatile unsigned long *a = addr;
167 unsigned long mask;
Ralf Baechle4ffd8b32006-11-30 01:14:50 +0000168 unsigned long flags;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700169
170 a += nr >> SZLONG_LOG;
Ralf Baechleb9611532007-03-05 00:56:15 +0000171 mask = 1UL << bit;
Ralf Baechle49edd092007-03-16 16:10:36 +0000172 raw_local_irq_save(flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700173 *a &= ~mask;
Ralf Baechle49edd092007-03-16 16:10:36 +0000174 raw_local_irq_restore(flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700175 }
176}
177
178/*
Nick Piggin728697c2007-10-18 03:06:53 -0700179 * clear_bit_unlock - Clears a bit in memory
180 * @nr: Bit to clear
181 * @addr: Address to start counting from
182 *
183 * clear_bit() is atomic and implies release semantics before the memory
184 * operation. It can be used for an unlock.
185 */
186static inline void clear_bit_unlock(unsigned long nr, volatile unsigned long *addr)
187{
188 smp_mb__before_clear_bit();
189 clear_bit(nr, addr);
190}
191
192/*
Linus Torvalds1da177e2005-04-16 15:20:36 -0700193 * change_bit - Toggle a bit in memory
194 * @nr: Bit to change
195 * @addr: Address to start counting from
196 *
197 * change_bit() is atomic and may not be reordered.
198 * Note that @nr may be almost arbitrarily large; this function is not
199 * restricted to acting on a single-word quantity.
200 */
201static inline void change_bit(unsigned long nr, volatile unsigned long *addr)
202{
Ralf Baechleb9611532007-03-05 00:56:15 +0000203 unsigned short bit = nr & SZLONG_MASK;
204
Linus Torvalds1da177e2005-04-16 15:20:36 -0700205 if (cpu_has_llsc && R10000_LLSC_WAR) {
206 unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG);
207 unsigned long temp;
208
209 __asm__ __volatile__(
Maciej W. Rozyckic4559f62005-06-23 15:57:15 +0000210 " .set mips3 \n"
Linus Torvalds1da177e2005-04-16 15:20:36 -0700211 "1: " __LL "%0, %1 # change_bit \n"
212 " xor %0, %2 \n"
Maciej W. Rozyckiaac8aa72005-06-14 17:35:03 +0000213 " " __SC "%0, %1 \n"
Linus Torvalds1da177e2005-04-16 15:20:36 -0700214 " beqzl %0, 1b \n"
Maciej W. Rozyckiaac8aa72005-06-14 17:35:03 +0000215 " .set mips0 \n"
Linus Torvalds1da177e2005-04-16 15:20:36 -0700216 : "=&r" (temp), "=m" (*m)
Ralf Baechleb9611532007-03-05 00:56:15 +0000217 : "ir" (1UL << bit), "m" (*m));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700218 } else if (cpu_has_llsc) {
219 unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG);
220 unsigned long temp;
221
222 __asm__ __volatile__(
Maciej W. Rozyckic4559f62005-06-23 15:57:15 +0000223 " .set mips3 \n"
Linus Torvalds1da177e2005-04-16 15:20:36 -0700224 "1: " __LL "%0, %1 # change_bit \n"
225 " xor %0, %2 \n"
Maciej W. Rozyckiaac8aa72005-06-14 17:35:03 +0000226 " " __SC "%0, %1 \n"
Ralf Baechlef65e4fa2006-09-28 01:45:21 +0100227 " beqz %0, 2f \n"
228 " .subsection 2 \n"
229 "2: b 1b \n"
230 " .previous \n"
Maciej W. Rozyckiaac8aa72005-06-14 17:35:03 +0000231 " .set mips0 \n"
Linus Torvalds1da177e2005-04-16 15:20:36 -0700232 : "=&r" (temp), "=m" (*m)
Ralf Baechleb9611532007-03-05 00:56:15 +0000233 : "ir" (1UL << bit), "m" (*m));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700234 } else {
235 volatile unsigned long *a = addr;
236 unsigned long mask;
Ralf Baechle4ffd8b32006-11-30 01:14:50 +0000237 unsigned long flags;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700238
239 a += nr >> SZLONG_LOG;
Ralf Baechleb9611532007-03-05 00:56:15 +0000240 mask = 1UL << bit;
Ralf Baechle49edd092007-03-16 16:10:36 +0000241 raw_local_irq_save(flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700242 *a ^= mask;
Ralf Baechle49edd092007-03-16 16:10:36 +0000243 raw_local_irq_restore(flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700244 }
245}
246
247/*
Linus Torvalds1da177e2005-04-16 15:20:36 -0700248 * test_and_set_bit - Set a bit and return its old value
249 * @nr: Bit to set
250 * @addr: Address to count from
251 *
252 * This operation is atomic and cannot be reordered.
253 * It also implies a memory barrier.
254 */
255static inline int test_and_set_bit(unsigned long nr,
256 volatile unsigned long *addr)
257{
Ralf Baechleb9611532007-03-05 00:56:15 +0000258 unsigned short bit = nr & SZLONG_MASK;
Ralf Baechleff72b7a2007-06-07 13:17:30 +0100259 unsigned long res;
Ralf Baechleb9611532007-03-05 00:56:15 +0000260
Nick Pigginc8f30ae2007-10-18 03:06:52 -0700261 smp_llsc_mb();
262
Linus Torvalds1da177e2005-04-16 15:20:36 -0700263 if (cpu_has_llsc && R10000_LLSC_WAR) {
264 unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG);
Ralf Baechleff72b7a2007-06-07 13:17:30 +0100265 unsigned long temp;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700266
267 __asm__ __volatile__(
Maciej W. Rozyckic4559f62005-06-23 15:57:15 +0000268 " .set mips3 \n"
Linus Torvalds1da177e2005-04-16 15:20:36 -0700269 "1: " __LL "%0, %1 # test_and_set_bit \n"
270 " or %2, %0, %3 \n"
271 " " __SC "%2, %1 \n"
272 " beqzl %2, 1b \n"
273 " and %2, %0, %3 \n"
Maciej W. Rozyckiaac8aa72005-06-14 17:35:03 +0000274 " .set mips0 \n"
Linus Torvalds1da177e2005-04-16 15:20:36 -0700275 : "=&r" (temp), "=m" (*m), "=&r" (res)
Ralf Baechleb9611532007-03-05 00:56:15 +0000276 : "r" (1UL << bit), "m" (*m)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700277 : "memory");
Linus Torvalds1da177e2005-04-16 15:20:36 -0700278 } else if (cpu_has_llsc) {
279 unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG);
Ralf Baechleff72b7a2007-06-07 13:17:30 +0100280 unsigned long temp;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700281
282 __asm__ __volatile__(
Maciej W. Rozyckiaac8aa72005-06-14 17:35:03 +0000283 " .set push \n"
284 " .set noreorder \n"
Maciej W. Rozyckic4559f62005-06-23 15:57:15 +0000285 " .set mips3 \n"
Maciej W. Rozyckiaac8aa72005-06-14 17:35:03 +0000286 "1: " __LL "%0, %1 # test_and_set_bit \n"
Linus Torvalds1da177e2005-04-16 15:20:36 -0700287 " or %2, %0, %3 \n"
288 " " __SC "%2, %1 \n"
Ralf Baechlef65e4fa2006-09-28 01:45:21 +0100289 " beqz %2, 2f \n"
Linus Torvalds1da177e2005-04-16 15:20:36 -0700290 " and %2, %0, %3 \n"
Ralf Baechlef65e4fa2006-09-28 01:45:21 +0100291 " .subsection 2 \n"
292 "2: b 1b \n"
293 " nop \n"
294 " .previous \n"
Maciej W. Rozyckiaac8aa72005-06-14 17:35:03 +0000295 " .set pop \n"
Linus Torvalds1da177e2005-04-16 15:20:36 -0700296 : "=&r" (temp), "=m" (*m), "=&r" (res)
Ralf Baechleb9611532007-03-05 00:56:15 +0000297 : "r" (1UL << bit), "m" (*m)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700298 : "memory");
Linus Torvalds1da177e2005-04-16 15:20:36 -0700299 } else {
300 volatile unsigned long *a = addr;
301 unsigned long mask;
Ralf Baechle4ffd8b32006-11-30 01:14:50 +0000302 unsigned long flags;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700303
304 a += nr >> SZLONG_LOG;
Ralf Baechleb9611532007-03-05 00:56:15 +0000305 mask = 1UL << bit;
Ralf Baechle49edd092007-03-16 16:10:36 +0000306 raw_local_irq_save(flags);
Ralf Baechleff72b7a2007-06-07 13:17:30 +0100307 res = (mask & *a);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700308 *a |= mask;
Ralf Baechle49edd092007-03-16 16:10:36 +0000309 raw_local_irq_restore(flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700310 }
Ralf Baechle0004a9d2006-10-31 03:45:07 +0000311
Ralf Baechle17099b12007-07-14 13:24:05 +0100312 smp_llsc_mb();
Ralf Baechleff72b7a2007-06-07 13:17:30 +0100313
314 return res != 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700315}
316
317/*
Nick Piggin728697c2007-10-18 03:06:53 -0700318 * test_and_set_bit_lock - Set a bit and return its old value
319 * @nr: Bit to set
320 * @addr: Address to count from
321 *
322 * This operation is atomic and implies acquire ordering semantics
323 * after the memory operation.
324 */
325static inline int test_and_set_bit_lock(unsigned long nr,
326 volatile unsigned long *addr)
327{
328 unsigned short bit = nr & SZLONG_MASK;
329 unsigned long res;
330
331 if (cpu_has_llsc && R10000_LLSC_WAR) {
332 unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG);
333 unsigned long temp;
334
335 __asm__ __volatile__(
336 " .set mips3 \n"
337 "1: " __LL "%0, %1 # test_and_set_bit \n"
338 " or %2, %0, %3 \n"
339 " " __SC "%2, %1 \n"
340 " beqzl %2, 1b \n"
341 " and %2, %0, %3 \n"
342 " .set mips0 \n"
343 : "=&r" (temp), "=m" (*m), "=&r" (res)
344 : "r" (1UL << bit), "m" (*m)
345 : "memory");
346 } else if (cpu_has_llsc) {
347 unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG);
348 unsigned long temp;
349
350 __asm__ __volatile__(
351 " .set push \n"
352 " .set noreorder \n"
353 " .set mips3 \n"
354 "1: " __LL "%0, %1 # test_and_set_bit \n"
355 " or %2, %0, %3 \n"
356 " " __SC "%2, %1 \n"
357 " beqz %2, 2f \n"
358 " and %2, %0, %3 \n"
359 " .subsection 2 \n"
360 "2: b 1b \n"
361 " nop \n"
362 " .previous \n"
363 " .set pop \n"
364 : "=&r" (temp), "=m" (*m), "=&r" (res)
365 : "r" (1UL << bit), "m" (*m)
366 : "memory");
367 } else {
368 volatile unsigned long *a = addr;
369 unsigned long mask;
370 unsigned long flags;
371
372 a += nr >> SZLONG_LOG;
373 mask = 1UL << bit;
374 raw_local_irq_save(flags);
375 res = (mask & *a);
376 *a |= mask;
377 raw_local_irq_restore(flags);
378 }
379
380 smp_llsc_mb();
381
382 return res != 0;
383}
384/*
Linus Torvalds1da177e2005-04-16 15:20:36 -0700385 * test_and_clear_bit - Clear a bit and return its old value
386 * @nr: Bit to clear
387 * @addr: Address to count from
388 *
389 * This operation is atomic and cannot be reordered.
390 * It also implies a memory barrier.
391 */
392static inline int test_and_clear_bit(unsigned long nr,
393 volatile unsigned long *addr)
394{
Ralf Baechleb9611532007-03-05 00:56:15 +0000395 unsigned short bit = nr & SZLONG_MASK;
Ralf Baechleff72b7a2007-06-07 13:17:30 +0100396 unsigned long res;
Ralf Baechleb9611532007-03-05 00:56:15 +0000397
Nick Pigginc8f30ae2007-10-18 03:06:52 -0700398 smp_llsc_mb();
399
Linus Torvalds1da177e2005-04-16 15:20:36 -0700400 if (cpu_has_llsc && R10000_LLSC_WAR) {
401 unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG);
Atsushi Nemoto8e09ffb2007-06-14 00:56:31 +0900402 unsigned long temp;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700403
404 __asm__ __volatile__(
Maciej W. Rozyckic4559f62005-06-23 15:57:15 +0000405 " .set mips3 \n"
Linus Torvalds1da177e2005-04-16 15:20:36 -0700406 "1: " __LL "%0, %1 # test_and_clear_bit \n"
407 " or %2, %0, %3 \n"
408 " xor %2, %3 \n"
Maciej W. Rozyckiaac8aa72005-06-14 17:35:03 +0000409 " " __SC "%2, %1 \n"
Linus Torvalds1da177e2005-04-16 15:20:36 -0700410 " beqzl %2, 1b \n"
411 " and %2, %0, %3 \n"
Maciej W. Rozyckiaac8aa72005-06-14 17:35:03 +0000412 " .set mips0 \n"
Linus Torvalds1da177e2005-04-16 15:20:36 -0700413 : "=&r" (temp), "=m" (*m), "=&r" (res)
Ralf Baechleb9611532007-03-05 00:56:15 +0000414 : "r" (1UL << bit), "m" (*m)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700415 : "memory");
Ralf Baechle102fa152007-02-16 17:18:50 +0000416#ifdef CONFIG_CPU_MIPSR2
417 } else if (__builtin_constant_p(nr)) {
418 unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG);
Ralf Baechleff72b7a2007-06-07 13:17:30 +0100419 unsigned long temp;
Ralf Baechle102fa152007-02-16 17:18:50 +0000420
421 __asm__ __volatile__(
422 "1: " __LL "%0, %1 # test_and_clear_bit \n"
423 " " __EXT "%2, %0, %3, 1 \n"
424 " " __INS "%0, $0, %3, 1 \n"
425 " " __SC "%0, %1 \n"
426 " beqz %0, 2f \n"
427 " .subsection 2 \n"
428 "2: b 1b \n"
429 " .previous \n"
430 : "=&r" (temp), "=m" (*m), "=&r" (res)
Ralf Baechleb9611532007-03-05 00:56:15 +0000431 : "ri" (bit), "m" (*m)
Ralf Baechle102fa152007-02-16 17:18:50 +0000432 : "memory");
Ralf Baechle102fa152007-02-16 17:18:50 +0000433#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -0700434 } else if (cpu_has_llsc) {
435 unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG);
Ralf Baechleff72b7a2007-06-07 13:17:30 +0100436 unsigned long temp;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700437
438 __asm__ __volatile__(
Maciej W. Rozyckiaac8aa72005-06-14 17:35:03 +0000439 " .set push \n"
440 " .set noreorder \n"
Maciej W. Rozyckic4559f62005-06-23 15:57:15 +0000441 " .set mips3 \n"
Maciej W. Rozyckiaac8aa72005-06-14 17:35:03 +0000442 "1: " __LL "%0, %1 # test_and_clear_bit \n"
Linus Torvalds1da177e2005-04-16 15:20:36 -0700443 " or %2, %0, %3 \n"
444 " xor %2, %3 \n"
Maciej W. Rozyckiaac8aa72005-06-14 17:35:03 +0000445 " " __SC "%2, %1 \n"
Ralf Baechlef65e4fa2006-09-28 01:45:21 +0100446 " beqz %2, 2f \n"
Linus Torvalds1da177e2005-04-16 15:20:36 -0700447 " and %2, %0, %3 \n"
Ralf Baechlef65e4fa2006-09-28 01:45:21 +0100448 " .subsection 2 \n"
449 "2: b 1b \n"
450 " nop \n"
451 " .previous \n"
Maciej W. Rozyckiaac8aa72005-06-14 17:35:03 +0000452 " .set pop \n"
Linus Torvalds1da177e2005-04-16 15:20:36 -0700453 : "=&r" (temp), "=m" (*m), "=&r" (res)
Ralf Baechleb9611532007-03-05 00:56:15 +0000454 : "r" (1UL << bit), "m" (*m)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700455 : "memory");
Linus Torvalds1da177e2005-04-16 15:20:36 -0700456 } else {
457 volatile unsigned long *a = addr;
458 unsigned long mask;
Ralf Baechle4ffd8b32006-11-30 01:14:50 +0000459 unsigned long flags;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700460
461 a += nr >> SZLONG_LOG;
Ralf Baechleb9611532007-03-05 00:56:15 +0000462 mask = 1UL << bit;
Ralf Baechle49edd092007-03-16 16:10:36 +0000463 raw_local_irq_save(flags);
Ralf Baechleff72b7a2007-06-07 13:17:30 +0100464 res = (mask & *a);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700465 *a &= ~mask;
Ralf Baechle49edd092007-03-16 16:10:36 +0000466 raw_local_irq_restore(flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700467 }
Ralf Baechle0004a9d2006-10-31 03:45:07 +0000468
Ralf Baechle17099b12007-07-14 13:24:05 +0100469 smp_llsc_mb();
Ralf Baechleff72b7a2007-06-07 13:17:30 +0100470
471 return res != 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700472}
473
474/*
Linus Torvalds1da177e2005-04-16 15:20:36 -0700475 * test_and_change_bit - Change a bit and return its old value
476 * @nr: Bit to change
477 * @addr: Address to count from
478 *
479 * This operation is atomic and cannot be reordered.
480 * It also implies a memory barrier.
481 */
482static inline int test_and_change_bit(unsigned long nr,
483 volatile unsigned long *addr)
484{
Ralf Baechleb9611532007-03-05 00:56:15 +0000485 unsigned short bit = nr & SZLONG_MASK;
Ralf Baechleff72b7a2007-06-07 13:17:30 +0100486 unsigned long res;
Ralf Baechleb9611532007-03-05 00:56:15 +0000487
Nick Pigginc8f30ae2007-10-18 03:06:52 -0700488 smp_llsc_mb();
489
Linus Torvalds1da177e2005-04-16 15:20:36 -0700490 if (cpu_has_llsc && R10000_LLSC_WAR) {
491 unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG);
Ralf Baechleff72b7a2007-06-07 13:17:30 +0100492 unsigned long temp;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700493
494 __asm__ __volatile__(
Maciej W. Rozyckic4559f62005-06-23 15:57:15 +0000495 " .set mips3 \n"
Maciej W. Rozyckiaac8aa72005-06-14 17:35:03 +0000496 "1: " __LL "%0, %1 # test_and_change_bit \n"
Linus Torvalds1da177e2005-04-16 15:20:36 -0700497 " xor %2, %0, %3 \n"
Maciej W. Rozyckiaac8aa72005-06-14 17:35:03 +0000498 " " __SC "%2, %1 \n"
Linus Torvalds1da177e2005-04-16 15:20:36 -0700499 " beqzl %2, 1b \n"
500 " and %2, %0, %3 \n"
Maciej W. Rozyckiaac8aa72005-06-14 17:35:03 +0000501 " .set mips0 \n"
Linus Torvalds1da177e2005-04-16 15:20:36 -0700502 : "=&r" (temp), "=m" (*m), "=&r" (res)
Ralf Baechleb9611532007-03-05 00:56:15 +0000503 : "r" (1UL << bit), "m" (*m)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700504 : "memory");
Linus Torvalds1da177e2005-04-16 15:20:36 -0700505 } else if (cpu_has_llsc) {
506 unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG);
Ralf Baechleff72b7a2007-06-07 13:17:30 +0100507 unsigned long temp;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700508
509 __asm__ __volatile__(
Maciej W. Rozyckiaac8aa72005-06-14 17:35:03 +0000510 " .set push \n"
511 " .set noreorder \n"
Maciej W. Rozyckic4559f62005-06-23 15:57:15 +0000512 " .set mips3 \n"
Maciej W. Rozyckiaac8aa72005-06-14 17:35:03 +0000513 "1: " __LL "%0, %1 # test_and_change_bit \n"
Linus Torvalds1da177e2005-04-16 15:20:36 -0700514 " xor %2, %0, %3 \n"
Maciej W. Rozyckiaac8aa72005-06-14 17:35:03 +0000515 " " __SC "\t%2, %1 \n"
Ralf Baechlef65e4fa2006-09-28 01:45:21 +0100516 " beqz %2, 2f \n"
Linus Torvalds1da177e2005-04-16 15:20:36 -0700517 " and %2, %0, %3 \n"
Ralf Baechlef65e4fa2006-09-28 01:45:21 +0100518 " .subsection 2 \n"
519 "2: b 1b \n"
520 " nop \n"
521 " .previous \n"
Maciej W. Rozyckiaac8aa72005-06-14 17:35:03 +0000522 " .set pop \n"
Linus Torvalds1da177e2005-04-16 15:20:36 -0700523 : "=&r" (temp), "=m" (*m), "=&r" (res)
Ralf Baechleb9611532007-03-05 00:56:15 +0000524 : "r" (1UL << bit), "m" (*m)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700525 : "memory");
Linus Torvalds1da177e2005-04-16 15:20:36 -0700526 } else {
527 volatile unsigned long *a = addr;
Ralf Baechleff72b7a2007-06-07 13:17:30 +0100528 unsigned long mask;
Ralf Baechle4ffd8b32006-11-30 01:14:50 +0000529 unsigned long flags;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700530
531 a += nr >> SZLONG_LOG;
Ralf Baechleb9611532007-03-05 00:56:15 +0000532 mask = 1UL << bit;
Ralf Baechle49edd092007-03-16 16:10:36 +0000533 raw_local_irq_save(flags);
Ralf Baechleff72b7a2007-06-07 13:17:30 +0100534 res = (mask & *a);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700535 *a ^= mask;
Ralf Baechle49edd092007-03-16 16:10:36 +0000536 raw_local_irq_restore(flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700537 }
Ralf Baechle0004a9d2006-10-31 03:45:07 +0000538
Ralf Baechle17099b12007-07-14 13:24:05 +0100539 smp_llsc_mb();
Ralf Baechleff72b7a2007-06-07 13:17:30 +0100540
541 return res != 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700542}
543
Akinobu Mita3c9ee7e2006-03-26 01:39:30 -0800544#include <asm-generic/bitops/non-atomic.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -0700545
Ralf Baechle65903262005-07-12 12:50:30 +0000546/*
Nick Piggin728697c2007-10-18 03:06:53 -0700547 * __clear_bit_unlock - Clears a bit in memory
548 * @nr: Bit to clear
549 * @addr: Address to start counting from
550 *
551 * __clear_bit() is non-atomic and implies release semantics before the memory
552 * operation. It can be used for an unlock if no other CPUs can concurrently
553 * modify other bits in the word.
554 */
555static inline void __clear_bit_unlock(unsigned long nr, volatile unsigned long *addr)
556{
557 smp_mb();
558 __clear_bit(nr, addr);
559}
560
561/*
Ralf Baechle65903262005-07-12 12:50:30 +0000562 * Return the bit position (0..63) of the most significant 1 bit in a word
563 * Returns -1 if no 1 bit exists
564 */
Ralf Baechleec917c2c2005-10-07 16:58:15 +0100565static inline int __ilog2(unsigned long x)
Ralf Baechle65903262005-07-12 12:50:30 +0000566{
567 int lz;
568
Ralf Baechleec917c2c2005-10-07 16:58:15 +0100569 if (sizeof(x) == 4) {
Ralf Baechle49a89ef2007-10-11 23:46:15 +0100570 __asm__(
Ralf Baechleec917c2c2005-10-07 16:58:15 +0100571 " .set push \n"
572 " .set mips32 \n"
573 " clz %0, %1 \n"
574 " .set pop \n"
575 : "=r" (lz)
576 : "r" (x));
577
578 return 31 - lz;
579 }
580
581 BUG_ON(sizeof(x) != 8);
582
Ralf Baechle49a89ef2007-10-11 23:46:15 +0100583 __asm__(
Ralf Baechle65903262005-07-12 12:50:30 +0000584 " .set push \n"
585 " .set mips64 \n"
586 " dclz %0, %1 \n"
587 " .set pop \n"
588 : "=r" (lz)
589 : "r" (x));
590
591 return 63 - lz;
592}
Ralf Baechle65903262005-07-12 12:50:30 +0000593
Akinobu Mita3c9ee7e2006-03-26 01:39:30 -0800594#if defined(CONFIG_CPU_MIPS32) || defined(CONFIG_CPU_MIPS64)
595
Ralf Baechle65903262005-07-12 12:50:30 +0000596/*
597 * __ffs - find first bit in word.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700598 * @word: The word to search
599 *
Ralf Baechle65903262005-07-12 12:50:30 +0000600 * Returns 0..SZLONG-1
601 * Undefined if no bit exists, so code should check against 0 first.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700602 */
Ralf Baechle65903262005-07-12 12:50:30 +0000603static inline unsigned long __ffs(unsigned long word)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700604{
Ralf Baechle65903262005-07-12 12:50:30 +0000605 return __ilog2(word & -word);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700606}
607
608/*
Atsushi Nemotobc818242006-04-17 21:19:12 +0900609 * fls - find last bit set.
610 * @word: The word to search
611 *
612 * This is defined the same way as ffs.
613 * Note fls(0) = 0, fls(1) = 1, fls(0x80000000) = 32.
614 */
615static inline int fls(int word)
616{
Ralf Baechle49a89ef2007-10-11 23:46:15 +0100617 __asm__("clz %0, %1" : "=r" (word) : "r" (word));
Atsushi Nemotobc818242006-04-17 21:19:12 +0900618
619 return 32 - word;
620}
621
622#if defined(CONFIG_64BIT) && defined(CONFIG_CPU_MIPS64)
623static inline int fls64(__u64 word)
624{
Ralf Baechle49a89ef2007-10-11 23:46:15 +0100625 __asm__("dclz %0, %1" : "=r" (word) : "r" (word));
Atsushi Nemotobc818242006-04-17 21:19:12 +0900626
627 return 64 - word;
628}
629#else
630#include <asm-generic/bitops/fls64.h>
631#endif
632
633/*
Ralf Baechle65903262005-07-12 12:50:30 +0000634 * ffs - find first bit set.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700635 * @word: The word to search
636 *
Atsushi Nemotobc818242006-04-17 21:19:12 +0900637 * This is defined the same way as
638 * the libc and compiler builtin ffs routines, therefore
639 * differs in spirit from the above ffz (man ffs).
Linus Torvalds1da177e2005-04-16 15:20:36 -0700640 */
Atsushi Nemotobc818242006-04-17 21:19:12 +0900641static inline int ffs(int word)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700642{
Ralf Baechle65903262005-07-12 12:50:30 +0000643 if (!word)
644 return 0;
645
Atsushi Nemotobc818242006-04-17 21:19:12 +0900646 return fls(word & -word);
Ralf Baechle65903262005-07-12 12:50:30 +0000647}
Ralf Baechle2caf1902006-01-30 17:14:41 +0000648
Akinobu Mita3c9ee7e2006-03-26 01:39:30 -0800649#else
Linus Torvalds1da177e2005-04-16 15:20:36 -0700650
Akinobu Mita3c9ee7e2006-03-26 01:39:30 -0800651#include <asm-generic/bitops/__ffs.h>
652#include <asm-generic/bitops/ffs.h>
Akinobu Mita3c9ee7e2006-03-26 01:39:30 -0800653#include <asm-generic/bitops/fls.h>
Atsushi Nemotobc818242006-04-17 21:19:12 +0900654#include <asm-generic/bitops/fls64.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -0700655
Akinobu Mita3c9ee7e2006-03-26 01:39:30 -0800656#endif /*defined(CONFIG_CPU_MIPS32) || defined(CONFIG_CPU_MIPS64) */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700657
Atsushi Nemotobc818242006-04-17 21:19:12 +0900658#include <asm-generic/bitops/ffz.h>
Akinobu Mita3c9ee7e2006-03-26 01:39:30 -0800659#include <asm-generic/bitops/find.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -0700660
661#ifdef __KERNEL__
662
Akinobu Mita3c9ee7e2006-03-26 01:39:30 -0800663#include <asm-generic/bitops/sched.h>
664#include <asm-generic/bitops/hweight.h>
665#include <asm-generic/bitops/ext2-non-atomic.h>
666#include <asm-generic/bitops/ext2-atomic.h>
667#include <asm-generic/bitops/minix.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -0700668
669#endif /* __KERNEL__ */
670
671#endif /* _ASM_BITOPS_H */