blob: bb3c0ab7e66771d5494b3798528fef3db602bc00 [file] [log] [blame]
Becky Brucefeaf7cf2005-09-22 14:20:04 -05001#ifndef _ASM_POWERPC_ATOMIC_H_
2#define _ASM_POWERPC_ATOMIC_H_
3
Linus Torvalds1da177e2005-04-16 15:20:36 -07004/*
5 * PowerPC atomic operations
6 */
7
Linus Torvalds1da177e2005-04-16 15:20:36 -07008typedef struct { volatile int counter; } atomic_t;
9
10#ifdef __KERNEL__
Nick Pigginf055aff2006-02-20 10:41:40 +010011#include <linux/compiler.h>
Becky Brucefeaf7cf2005-09-22 14:20:04 -050012#include <asm/synch.h>
David Gibson3ddfbcf2005-11-10 12:56:55 +110013#include <asm/asm-compat.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070014
Becky Brucefeaf7cf2005-09-22 14:20:04 -050015#define ATOMIC_INIT(i) { (i) }
Linus Torvalds1da177e2005-04-16 15:20:36 -070016
17#define atomic_read(v) ((v)->counter)
18#define atomic_set(v,i) (((v)->counter) = (i))
19
Linus Torvalds1da177e2005-04-16 15:20:36 -070020static __inline__ void atomic_add(int a, atomic_t *v)
21{
22 int t;
23
24 __asm__ __volatile__(
25"1: lwarx %0,0,%3 # atomic_add\n\
26 add %0,%2,%0\n"
27 PPC405_ERR77(0,%3)
28" stwcx. %0,0,%3 \n\
29 bne- 1b"
30 : "=&r" (t), "=m" (v->counter)
31 : "r" (a), "r" (&v->counter), "m" (v->counter)
32 : "cc");
33}
34
35static __inline__ int atomic_add_return(int a, atomic_t *v)
36{
37 int t;
38
39 __asm__ __volatile__(
Anton Blanchard144b9c12006-01-13 15:37:17 +110040 LWSYNC_ON_SMP
Linus Torvalds1da177e2005-04-16 15:20:36 -070041"1: lwarx %0,0,%2 # atomic_add_return\n\
42 add %0,%1,%0\n"
43 PPC405_ERR77(0,%2)
44" stwcx. %0,0,%2 \n\
45 bne- 1b"
Becky Brucefeaf7cf2005-09-22 14:20:04 -050046 ISYNC_ON_SMP
Linus Torvalds1da177e2005-04-16 15:20:36 -070047 : "=&r" (t)
48 : "r" (a), "r" (&v->counter)
49 : "cc", "memory");
50
51 return t;
52}
53
54#define atomic_add_negative(a, v) (atomic_add_return((a), (v)) < 0)
55
56static __inline__ void atomic_sub(int a, atomic_t *v)
57{
58 int t;
59
60 __asm__ __volatile__(
61"1: lwarx %0,0,%3 # atomic_sub\n\
62 subf %0,%2,%0\n"
63 PPC405_ERR77(0,%3)
64" stwcx. %0,0,%3 \n\
65 bne- 1b"
66 : "=&r" (t), "=m" (v->counter)
67 : "r" (a), "r" (&v->counter), "m" (v->counter)
68 : "cc");
69}
70
71static __inline__ int atomic_sub_return(int a, atomic_t *v)
72{
73 int t;
74
75 __asm__ __volatile__(
Anton Blanchard144b9c12006-01-13 15:37:17 +110076 LWSYNC_ON_SMP
Linus Torvalds1da177e2005-04-16 15:20:36 -070077"1: lwarx %0,0,%2 # atomic_sub_return\n\
78 subf %0,%1,%0\n"
79 PPC405_ERR77(0,%2)
80" stwcx. %0,0,%2 \n\
81 bne- 1b"
Becky Brucefeaf7cf2005-09-22 14:20:04 -050082 ISYNC_ON_SMP
Linus Torvalds1da177e2005-04-16 15:20:36 -070083 : "=&r" (t)
84 : "r" (a), "r" (&v->counter)
85 : "cc", "memory");
86
87 return t;
88}
89
90static __inline__ void atomic_inc(atomic_t *v)
91{
92 int t;
93
94 __asm__ __volatile__(
95"1: lwarx %0,0,%2 # atomic_inc\n\
96 addic %0,%0,1\n"
97 PPC405_ERR77(0,%2)
98" stwcx. %0,0,%2 \n\
99 bne- 1b"
100 : "=&r" (t), "=m" (v->counter)
101 : "r" (&v->counter), "m" (v->counter)
102 : "cc");
103}
104
105static __inline__ int atomic_inc_return(atomic_t *v)
106{
107 int t;
108
109 __asm__ __volatile__(
Anton Blanchard144b9c12006-01-13 15:37:17 +1100110 LWSYNC_ON_SMP
Linus Torvalds1da177e2005-04-16 15:20:36 -0700111"1: lwarx %0,0,%1 # atomic_inc_return\n\
112 addic %0,%0,1\n"
113 PPC405_ERR77(0,%1)
114" stwcx. %0,0,%1 \n\
115 bne- 1b"
Becky Brucefeaf7cf2005-09-22 14:20:04 -0500116 ISYNC_ON_SMP
Linus Torvalds1da177e2005-04-16 15:20:36 -0700117 : "=&r" (t)
118 : "r" (&v->counter)
119 : "cc", "memory");
120
121 return t;
122}
123
124/*
125 * atomic_inc_and_test - increment and test
126 * @v: pointer of type atomic_t
127 *
128 * Atomically increments @v by 1
129 * and returns true if the result is zero, or false for all
130 * other cases.
131 */
132#define atomic_inc_and_test(v) (atomic_inc_return(v) == 0)
133
134static __inline__ void atomic_dec(atomic_t *v)
135{
136 int t;
137
138 __asm__ __volatile__(
139"1: lwarx %0,0,%2 # atomic_dec\n\
140 addic %0,%0,-1\n"
141 PPC405_ERR77(0,%2)\
142" stwcx. %0,0,%2\n\
143 bne- 1b"
144 : "=&r" (t), "=m" (v->counter)
145 : "r" (&v->counter), "m" (v->counter)
146 : "cc");
147}
148
149static __inline__ int atomic_dec_return(atomic_t *v)
150{
151 int t;
152
153 __asm__ __volatile__(
Anton Blanchard144b9c12006-01-13 15:37:17 +1100154 LWSYNC_ON_SMP
Linus Torvalds1da177e2005-04-16 15:20:36 -0700155"1: lwarx %0,0,%1 # atomic_dec_return\n\
156 addic %0,%0,-1\n"
157 PPC405_ERR77(0,%1)
158" stwcx. %0,0,%1\n\
159 bne- 1b"
Becky Brucefeaf7cf2005-09-22 14:20:04 -0500160 ISYNC_ON_SMP
Linus Torvalds1da177e2005-04-16 15:20:36 -0700161 : "=&r" (t)
162 : "r" (&v->counter)
163 : "cc", "memory");
164
165 return t;
166}
167
Nick Piggin4a6dae62005-11-13 16:07:24 -0800168#define atomic_cmpxchg(v, o, n) ((int)cmpxchg(&((v)->counter), (o), (n)))
Ingo Molnarffbf6702006-01-09 15:59:17 -0800169#define atomic_xchg(v, new) (xchg(&((v)->counter), new))
Nick Piggin4a6dae62005-11-13 16:07:24 -0800170
Nick Piggin8426e1f2005-11-13 16:07:25 -0800171/**
172 * atomic_add_unless - add unless the number is a given value
173 * @v: pointer of type atomic_t
174 * @a: the amount to add to v...
175 * @u: ...unless v is equal to u.
176 *
177 * Atomically adds @a to @v, so long as it was not @u.
178 * Returns non-zero if @v was not @u, and zero otherwise.
179 */
Nick Pigginf055aff2006-02-20 10:41:40 +0100180static __inline__ int atomic_add_unless(atomic_t *v, int a, int u)
181{
182 int t;
183
184 __asm__ __volatile__ (
185 LWSYNC_ON_SMP
186"1: lwarx %0,0,%1 # atomic_add_unless\n\
187 cmpw 0,%0,%3 \n\
188 beq- 2f \n\
189 add %0,%2,%0 \n"
190 PPC405_ERR77(0,%2)
191" stwcx. %0,0,%1 \n\
192 bne- 1b \n"
193 ISYNC_ON_SMP
194" subf %0,%2,%0 \n\
1952:"
196 : "=&r" (t)
197 : "r" (&v->counter), "r" (a), "r" (u)
198 : "cc", "memory");
199
200 return t != u;
201}
202
Nick Piggin8426e1f2005-11-13 16:07:25 -0800203#define atomic_inc_not_zero(v) atomic_add_unless((v), 1, 0)
204
Linus Torvalds1da177e2005-04-16 15:20:36 -0700205#define atomic_sub_and_test(a, v) (atomic_sub_return((a), (v)) == 0)
206#define atomic_dec_and_test(v) (atomic_dec_return((v)) == 0)
207
208/*
209 * Atomically test *v and decrement if it is greater than 0.
210 * The function returns the old value of *v minus 1.
211 */
212static __inline__ int atomic_dec_if_positive(atomic_t *v)
213{
214 int t;
215
216 __asm__ __volatile__(
Anton Blanchard144b9c12006-01-13 15:37:17 +1100217 LWSYNC_ON_SMP
Linus Torvalds1da177e2005-04-16 15:20:36 -0700218"1: lwarx %0,0,%1 # atomic_dec_if_positive\n\
219 addic. %0,%0,-1\n\
220 blt- 2f\n"
221 PPC405_ERR77(0,%1)
222" stwcx. %0,0,%1\n\
223 bne- 1b"
Becky Brucefeaf7cf2005-09-22 14:20:04 -0500224 ISYNC_ON_SMP
Linus Torvalds1da177e2005-04-16 15:20:36 -0700225 "\n\
2262:" : "=&r" (t)
227 : "r" (&v->counter)
228 : "cc", "memory");
229
230 return t;
231}
232
Becky Brucefeaf7cf2005-09-22 14:20:04 -0500233#define smp_mb__before_atomic_dec() smp_mb()
234#define smp_mb__after_atomic_dec() smp_mb()
235#define smp_mb__before_atomic_inc() smp_mb()
236#define smp_mb__after_atomic_inc() smp_mb()
Linus Torvalds1da177e2005-04-16 15:20:36 -0700237
Stephen Rothwell06a98db2005-11-10 15:51:14 +1100238#ifdef __powerpc64__
239
240typedef struct { volatile long counter; } atomic64_t;
241
242#define ATOMIC64_INIT(i) { (i) }
243
244#define atomic64_read(v) ((v)->counter)
245#define atomic64_set(v,i) (((v)->counter) = (i))
246
247static __inline__ void atomic64_add(long a, atomic64_t *v)
248{
249 long t;
250
251 __asm__ __volatile__(
252"1: ldarx %0,0,%3 # atomic64_add\n\
253 add %0,%2,%0\n\
254 stdcx. %0,0,%3 \n\
255 bne- 1b"
256 : "=&r" (t), "=m" (v->counter)
257 : "r" (a), "r" (&v->counter), "m" (v->counter)
258 : "cc");
259}
260
261static __inline__ long atomic64_add_return(long a, atomic64_t *v)
262{
263 long t;
264
265 __asm__ __volatile__(
Anton Blanchard144b9c12006-01-13 15:37:17 +1100266 LWSYNC_ON_SMP
Stephen Rothwell06a98db2005-11-10 15:51:14 +1100267"1: ldarx %0,0,%2 # atomic64_add_return\n\
268 add %0,%1,%0\n\
269 stdcx. %0,0,%2 \n\
270 bne- 1b"
271 ISYNC_ON_SMP
272 : "=&r" (t)
273 : "r" (a), "r" (&v->counter)
274 : "cc", "memory");
275
276 return t;
277}
278
279#define atomic64_add_negative(a, v) (atomic64_add_return((a), (v)) < 0)
280
281static __inline__ void atomic64_sub(long a, atomic64_t *v)
282{
283 long t;
284
285 __asm__ __volatile__(
286"1: ldarx %0,0,%3 # atomic64_sub\n\
287 subf %0,%2,%0\n\
288 stdcx. %0,0,%3 \n\
289 bne- 1b"
290 : "=&r" (t), "=m" (v->counter)
291 : "r" (a), "r" (&v->counter), "m" (v->counter)
292 : "cc");
293}
294
295static __inline__ long atomic64_sub_return(long a, atomic64_t *v)
296{
297 long t;
298
299 __asm__ __volatile__(
Anton Blanchard144b9c12006-01-13 15:37:17 +1100300 LWSYNC_ON_SMP
Stephen Rothwell06a98db2005-11-10 15:51:14 +1100301"1: ldarx %0,0,%2 # atomic64_sub_return\n\
302 subf %0,%1,%0\n\
303 stdcx. %0,0,%2 \n\
304 bne- 1b"
305 ISYNC_ON_SMP
306 : "=&r" (t)
307 : "r" (a), "r" (&v->counter)
308 : "cc", "memory");
309
310 return t;
311}
312
313static __inline__ void atomic64_inc(atomic64_t *v)
314{
315 long t;
316
317 __asm__ __volatile__(
318"1: ldarx %0,0,%2 # atomic64_inc\n\
319 addic %0,%0,1\n\
320 stdcx. %0,0,%2 \n\
321 bne- 1b"
322 : "=&r" (t), "=m" (v->counter)
323 : "r" (&v->counter), "m" (v->counter)
324 : "cc");
325}
326
327static __inline__ long atomic64_inc_return(atomic64_t *v)
328{
329 long t;
330
331 __asm__ __volatile__(
Anton Blanchard144b9c12006-01-13 15:37:17 +1100332 LWSYNC_ON_SMP
Stephen Rothwell06a98db2005-11-10 15:51:14 +1100333"1: ldarx %0,0,%1 # atomic64_inc_return\n\
334 addic %0,%0,1\n\
335 stdcx. %0,0,%1 \n\
336 bne- 1b"
337 ISYNC_ON_SMP
338 : "=&r" (t)
339 : "r" (&v->counter)
340 : "cc", "memory");
341
342 return t;
343}
344
345/*
346 * atomic64_inc_and_test - increment and test
347 * @v: pointer of type atomic64_t
348 *
349 * Atomically increments @v by 1
350 * and returns true if the result is zero, or false for all
351 * other cases.
352 */
353#define atomic64_inc_and_test(v) (atomic64_inc_return(v) == 0)
354
355static __inline__ void atomic64_dec(atomic64_t *v)
356{
357 long t;
358
359 __asm__ __volatile__(
360"1: ldarx %0,0,%2 # atomic64_dec\n\
361 addic %0,%0,-1\n\
362 stdcx. %0,0,%2\n\
363 bne- 1b"
364 : "=&r" (t), "=m" (v->counter)
365 : "r" (&v->counter), "m" (v->counter)
366 : "cc");
367}
368
369static __inline__ long atomic64_dec_return(atomic64_t *v)
370{
371 long t;
372
373 __asm__ __volatile__(
Anton Blanchard144b9c12006-01-13 15:37:17 +1100374 LWSYNC_ON_SMP
Stephen Rothwell06a98db2005-11-10 15:51:14 +1100375"1: ldarx %0,0,%1 # atomic64_dec_return\n\
376 addic %0,%0,-1\n\
377 stdcx. %0,0,%1\n\
378 bne- 1b"
379 ISYNC_ON_SMP
380 : "=&r" (t)
381 : "r" (&v->counter)
382 : "cc", "memory");
383
384 return t;
385}
386
387#define atomic64_sub_and_test(a, v) (atomic64_sub_return((a), (v)) == 0)
388#define atomic64_dec_and_test(v) (atomic64_dec_return((v)) == 0)
389
390/*
391 * Atomically test *v and decrement if it is greater than 0.
392 * The function returns the old value of *v minus 1.
393 */
394static __inline__ long atomic64_dec_if_positive(atomic64_t *v)
395{
396 long t;
397
398 __asm__ __volatile__(
Anton Blanchard144b9c12006-01-13 15:37:17 +1100399 LWSYNC_ON_SMP
Stephen Rothwell06a98db2005-11-10 15:51:14 +1100400"1: ldarx %0,0,%1 # atomic64_dec_if_positive\n\
401 addic. %0,%0,-1\n\
402 blt- 2f\n\
403 stdcx. %0,0,%1\n\
404 bne- 1b"
405 ISYNC_ON_SMP
406 "\n\
4072:" : "=&r" (t)
408 : "r" (&v->counter)
409 : "cc", "memory");
410
411 return t;
412}
413
414#endif /* __powerpc64__ */
415
Christoph Lameterd3cb4872006-01-06 00:11:20 -0800416#include <asm-generic/atomic.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -0700417#endif /* __KERNEL__ */
Becky Brucefeaf7cf2005-09-22 14:20:04 -0500418#endif /* _ASM_POWERPC_ATOMIC_H_ */