blob: f7419ef9c8f99f57a0495bf31453580c6ed43893 [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
2 * Copyright 1995, Russell King.
3 * Various bits and pieces copyrights include:
4 * Linus Torvalds (test_bit).
5 * Big endian support: Copyright 2001, Nicolas Pitre
6 * reworked by rmk.
7 *
8 * bit 0 is the LSB of an "unsigned long" quantity.
9 *
10 * Please note that the code in this file should never be included
11 * from user space. Many of these are not implemented in assembler
12 * since they would be too costly. Also, they require privileged
13 * instructions (which are not available from user mode) to ensure
14 * that they are atomic.
15 */
16
17#ifndef __ASM_ARM_BITOPS_H
18#define __ASM_ARM_BITOPS_H
19
20#ifdef __KERNEL__
21
Jiri Slaby06245172007-10-18 23:40:26 -070022#ifndef _LINUX_BITOPS_H
23#error only <linux/bitops.h> can be included directly
24#endif
25
Russell King8dc39b82005-11-16 17:23:57 +000026#include <linux/compiler.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070027#include <asm/system.h>
28
Heechul Yun4cde7e02011-07-19 13:26:54 +010029#define smp_mb__before_clear_bit() smp_mb()
30#define smp_mb__after_clear_bit() smp_mb()
Linus Torvalds1da177e2005-04-16 15:20:36 -070031
32/*
33 * These functions are the basis of our bit ops.
34 *
35 * First, the atomic bitops. These use native endian.
36 */
37static inline void ____atomic_set_bit(unsigned int bit, volatile unsigned long *p)
38{
39 unsigned long flags;
40 unsigned long mask = 1UL << (bit & 31);
41
42 p += bit >> 5;
43
Lennert Buytenheke7cc2c52006-09-21 03:35:20 +010044 raw_local_irq_save(flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -070045 *p |= mask;
Lennert Buytenheke7cc2c52006-09-21 03:35:20 +010046 raw_local_irq_restore(flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -070047}
48
49static inline void ____atomic_clear_bit(unsigned int bit, volatile unsigned long *p)
50{
51 unsigned long flags;
52 unsigned long mask = 1UL << (bit & 31);
53
54 p += bit >> 5;
55
Lennert Buytenheke7cc2c52006-09-21 03:35:20 +010056 raw_local_irq_save(flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -070057 *p &= ~mask;
Lennert Buytenheke7cc2c52006-09-21 03:35:20 +010058 raw_local_irq_restore(flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -070059}
60
61static inline void ____atomic_change_bit(unsigned int bit, volatile unsigned long *p)
62{
63 unsigned long flags;
64 unsigned long mask = 1UL << (bit & 31);
65
66 p += bit >> 5;
67
Lennert Buytenheke7cc2c52006-09-21 03:35:20 +010068 raw_local_irq_save(flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -070069 *p ^= mask;
Lennert Buytenheke7cc2c52006-09-21 03:35:20 +010070 raw_local_irq_restore(flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -070071}
72
73static inline int
74____atomic_test_and_set_bit(unsigned int bit, volatile unsigned long *p)
75{
76 unsigned long flags;
77 unsigned int res;
78 unsigned long mask = 1UL << (bit & 31);
79
80 p += bit >> 5;
81
Lennert Buytenheke7cc2c52006-09-21 03:35:20 +010082 raw_local_irq_save(flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -070083 res = *p;
84 *p = res | mask;
Lennert Buytenheke7cc2c52006-09-21 03:35:20 +010085 raw_local_irq_restore(flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -070086
Johannes Weinere9ac8292009-07-21 17:08:28 +020087 return (res & mask) != 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -070088}
89
90static inline int
91____atomic_test_and_clear_bit(unsigned int bit, volatile unsigned long *p)
92{
93 unsigned long flags;
94 unsigned int res;
95 unsigned long mask = 1UL << (bit & 31);
96
97 p += bit >> 5;
98
Lennert Buytenheke7cc2c52006-09-21 03:35:20 +010099 raw_local_irq_save(flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700100 res = *p;
101 *p = res & ~mask;
Lennert Buytenheke7cc2c52006-09-21 03:35:20 +0100102 raw_local_irq_restore(flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700103
Johannes Weinere9ac8292009-07-21 17:08:28 +0200104 return (res & mask) != 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700105}
106
107static inline int
108____atomic_test_and_change_bit(unsigned int bit, volatile unsigned long *p)
109{
110 unsigned long flags;
111 unsigned int res;
112 unsigned long mask = 1UL << (bit & 31);
113
114 p += bit >> 5;
115
Lennert Buytenheke7cc2c52006-09-21 03:35:20 +0100116 raw_local_irq_save(flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700117 res = *p;
118 *p = res ^ mask;
Lennert Buytenheke7cc2c52006-09-21 03:35:20 +0100119 raw_local_irq_restore(flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700120
Johannes Weinere9ac8292009-07-21 17:08:28 +0200121 return (res & mask) != 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700122}
123
Akinobu Mitab89c3b12006-03-26 01:39:19 -0800124#include <asm-generic/bitops/non-atomic.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -0700125
126/*
127 * A note about Endian-ness.
128 * -------------------------
129 *
130 * When the ARM is put into big endian mode via CR15, the processor
131 * merely swaps the order of bytes within words, thus:
132 *
133 * ------------ physical data bus bits -----------
134 * D31 ... D24 D23 ... D16 D15 ... D8 D7 ... D0
135 * little byte 3 byte 2 byte 1 byte 0
136 * big byte 0 byte 1 byte 2 byte 3
137 *
138 * This means that reading a 32-bit word at address 0 returns the same
139 * value irrespective of the endian mode bit.
140 *
141 * Peripheral devices should be connected with the data bus reversed in
142 * "Big Endian" mode. ARM Application Note 61 is applicable, and is
143 * available from http://www.arm.com/.
144 *
145 * The following assumes that the data bus connectivity for big endian
146 * mode has been followed.
147 *
148 * Note that bit 0 is defined to be 32-bit word bit 0, not byte 0 bit 0.
149 */
150
151/*
Russell King6323f0c2011-01-16 18:02:17 +0000152 * Native endian assembly bitops. nr = 0 -> word 0 bit 0.
153 */
154extern void _set_bit(int nr, volatile unsigned long * p);
155extern void _clear_bit(int nr, volatile unsigned long * p);
156extern void _change_bit(int nr, volatile unsigned long * p);
157extern int _test_and_set_bit(int nr, volatile unsigned long * p);
158extern int _test_and_clear_bit(int nr, volatile unsigned long * p);
159extern int _test_and_change_bit(int nr, volatile unsigned long * p);
160
161/*
Linus Torvalds1da177e2005-04-16 15:20:36 -0700162 * Little endian assembly bitops. nr = 0 -> byte 0 bit 0.
163 */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700164extern int _find_first_zero_bit_le(const void * p, unsigned size);
165extern int _find_next_zero_bit_le(const void * p, int size, int offset);
166extern int _find_first_bit_le(const unsigned long *p, unsigned size);
167extern int _find_next_bit_le(const unsigned long *p, int size, int offset);
168
169/*
170 * Big endian assembly bitops. nr = 0 -> byte 3 bit 0.
171 */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700172extern int _find_first_zero_bit_be(const void * p, unsigned size);
173extern int _find_next_zero_bit_be(const void * p, int size, int offset);
174extern int _find_first_bit_be(const unsigned long *p, unsigned size);
175extern int _find_next_bit_be(const unsigned long *p, int size, int offset);
176
Russell Kinge7ec0292005-07-28 20:36:26 +0100177#ifndef CONFIG_SMP
Linus Torvalds1da177e2005-04-16 15:20:36 -0700178/*
179 * The __* form of bitops are non-atomic and may be reordered.
180 */
Russell King6323f0c2011-01-16 18:02:17 +0000181#define ATOMIC_BITOP(name,nr,p) \
182 (__builtin_constant_p(nr) ? ____atomic_##name(nr, p) : _##name(nr,p))
Russell Kinge7ec0292005-07-28 20:36:26 +0100183#else
Russell King6323f0c2011-01-16 18:02:17 +0000184#define ATOMIC_BITOP(name,nr,p) _##name(nr,p)
Russell Kinge7ec0292005-07-28 20:36:26 +0100185#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -0700186
Russell King6323f0c2011-01-16 18:02:17 +0000187/*
188 * Native endian atomic definitions.
189 */
190#define set_bit(nr,p) ATOMIC_BITOP(set_bit,nr,p)
191#define clear_bit(nr,p) ATOMIC_BITOP(clear_bit,nr,p)
192#define change_bit(nr,p) ATOMIC_BITOP(change_bit,nr,p)
193#define test_and_set_bit(nr,p) ATOMIC_BITOP(test_and_set_bit,nr,p)
194#define test_and_clear_bit(nr,p) ATOMIC_BITOP(test_and_clear_bit,nr,p)
195#define test_and_change_bit(nr,p) ATOMIC_BITOP(test_and_change_bit,nr,p)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700196
197#ifndef __ARMEB__
198/*
199 * These are the little endian, atomic definitions.
200 */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700201#define find_first_zero_bit(p,sz) _find_first_zero_bit_le(p,sz)
202#define find_next_zero_bit(p,sz,off) _find_next_zero_bit_le(p,sz,off)
203#define find_first_bit(p,sz) _find_first_bit_le(p,sz)
204#define find_next_bit(p,sz,off) _find_next_bit_le(p,sz,off)
205
Linus Torvalds1da177e2005-04-16 15:20:36 -0700206#else
Linus Torvalds1da177e2005-04-16 15:20:36 -0700207/*
208 * These are the big endian, atomic definitions.
209 */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700210#define find_first_zero_bit(p,sz) _find_first_zero_bit_be(p,sz)
211#define find_next_zero_bit(p,sz,off) _find_next_zero_bit_be(p,sz,off)
212#define find_first_bit(p,sz) _find_first_bit_be(p,sz)
213#define find_next_bit(p,sz,off) _find_next_bit_be(p,sz,off)
214
Linus Torvalds1da177e2005-04-16 15:20:36 -0700215#endif
216
217#if __LINUX_ARM_ARCH__ < 5
218
Akinobu Mitab89c3b12006-03-26 01:39:19 -0800219#include <asm-generic/bitops/ffz.h>
Nicolas Pitre94fc7332008-12-04 03:59:41 +0100220#include <asm-generic/bitops/__fls.h>
Akinobu Mitab89c3b12006-03-26 01:39:19 -0800221#include <asm-generic/bitops/__ffs.h>
222#include <asm-generic/bitops/fls.h>
223#include <asm-generic/bitops/ffs.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -0700224
225#else
226
Akinobu Mita9363513362006-03-26 01:38:59 -0800227static inline int constant_fls(int x)
228{
229 int r = 32;
230
231 if (!x)
232 return 0;
233 if (!(x & 0xffff0000u)) {
234 x <<= 16;
235 r -= 16;
236 }
237 if (!(x & 0xff000000u)) {
238 x <<= 8;
239 r -= 8;
240 }
241 if (!(x & 0xf0000000u)) {
242 x <<= 4;
243 r -= 4;
244 }
245 if (!(x & 0xc0000000u)) {
246 x <<= 2;
247 r -= 2;
248 }
249 if (!(x & 0x80000000u)) {
250 x <<= 1;
251 r -= 1;
252 }
253 return r;
254}
255
Linus Torvalds1da177e2005-04-16 15:20:36 -0700256/*
257 * On ARMv5 and above those functions can be implemented around
258 * the clz instruction for much better code efficiency.
259 */
260
Andrew Morton0c65f452008-07-23 15:35:22 -0700261static inline int fls(int x)
262{
Nicolas Pitre94fc7332008-12-04 03:59:41 +0100263 int ret;
264
265 if (__builtin_constant_p(x))
266 return constant_fls(x);
267
Rabin Vincente163d522011-01-12 14:38:52 +0100268 asm("clz\t%0, %1" : "=r" (ret) : "r" (x));
Nicolas Pitre94fc7332008-12-04 03:59:41 +0100269 ret = 32 - ret;
270 return ret;
Andrew Morton0c65f452008-07-23 15:35:22 -0700271}
272
Nicolas Pitre94fc7332008-12-04 03:59:41 +0100273#define __fls(x) (fls(x) - 1)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700274#define ffs(x) ({ unsigned long __t = (x); fls(__t & -__t); })
275#define __ffs(x) (ffs(x) - 1)
276#define ffz(x) __ffs( ~(x) )
277
278#endif
279
Akinobu Mitab89c3b12006-03-26 01:39:19 -0800280#include <asm-generic/bitops/fls64.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -0700281
Akinobu Mitab89c3b12006-03-26 01:39:19 -0800282#include <asm-generic/bitops/sched.h>
283#include <asm-generic/bitops/hweight.h>
Nick Piggin26333572007-10-18 03:06:39 -0700284#include <asm-generic/bitops/lock.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -0700285
Akinobu Mita04b18ff2011-05-26 16:26:11 -0700286#ifdef __ARMEB__
Akinobu Mitaf6b57e32011-03-23 16:41:57 -0700287
288static inline int find_first_zero_bit_le(const void *p, unsigned size)
289{
290 return _find_first_zero_bit_le(p, size);
291}
Akinobu Mitaa2812e12011-05-26 16:26:06 -0700292#define find_first_zero_bit_le find_first_zero_bit_le
Akinobu Mitaf6b57e32011-03-23 16:41:57 -0700293
294static inline int find_next_zero_bit_le(const void *p, int size, int offset)
295{
296 return _find_next_zero_bit_le(p, size, offset);
297}
Akinobu Mitaa2812e12011-05-26 16:26:06 -0700298#define find_next_zero_bit_le find_next_zero_bit_le
Akinobu Mitaf6b57e32011-03-23 16:41:57 -0700299
300static inline int find_next_bit_le(const void *p, int size, int offset)
301{
302 return _find_next_bit_le(p, size, offset);
303}
Akinobu Mitaa2812e12011-05-26 16:26:06 -0700304#define find_next_bit_le find_next_bit_le
Akinobu Mitaf6b57e32011-03-23 16:41:57 -0700305
Akinobu Mita04b18ff2011-05-26 16:26:11 -0700306#endif
307
308#include <asm-generic/bitops/le.h>
309
Linus Torvalds1da177e2005-04-16 15:20:36 -0700310/*
311 * Ext2 is defined to use little-endian byte ordering.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700312 */
Akinobu Mita148817b2011-07-26 16:09:04 -0700313#include <asm-generic/bitops/ext2-atomic-setbit.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -0700314
Linus Torvalds1da177e2005-04-16 15:20:36 -0700315#endif /* __KERNEL__ */
316
317#endif /* _ARM_BITOPS_H */