blob: 6423ffa195a409f59b3d6a6881d1b1b866c29dbe [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
2 * Atomic operations that C can't guarantee us. Useful for
3 * resource counting etc..
4 *
5 * But use these as seldom as possible since they are much more slower
6 * than regular operations.
7 *
8 * This file is subject to the terms and conditions of the GNU General Public
9 * License. See the file "COPYING" in the main directory of this archive
10 * for more details.
11 *
Ralf Baechlee303e082006-11-30 01:14:50 +000012 * Copyright (C) 1996, 97, 99, 2000, 03, 04, 06 by Ralf Baechle
Linus Torvalds1da177e2005-04-16 15:20:36 -070013 */
Linus Torvalds1da177e2005-04-16 15:20:36 -070014#ifndef _ASM_ATOMIC_H
15#define _ASM_ATOMIC_H
16
Ralf Baechle192ef362006-07-07 14:07:18 +010017#include <linux/irqflags.h>
Ralf Baechle0004a9d2006-10-31 03:45:07 +000018#include <asm/barrier.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070019#include <asm/cpu-features.h>
20#include <asm/war.h>
21
Linus Torvalds1da177e2005-04-16 15:20:36 -070022typedef struct { volatile int counter; } atomic_t;
23
24#define ATOMIC_INIT(i) { (i) }
25
26/*
27 * atomic_read - read atomic variable
28 * @v: pointer of type atomic_t
29 *
30 * Atomically reads the value of @v.
31 */
32#define atomic_read(v) ((v)->counter)
33
34/*
35 * atomic_set - set atomic variable
36 * @v: pointer of type atomic_t
37 * @i: required value
38 *
39 * Atomically sets the value of @v to @i.
40 */
41#define atomic_set(v,i) ((v)->counter = (i))
42
43/*
44 * atomic_add - add integer to atomic variable
45 * @i: integer value to add
46 * @v: pointer of type atomic_t
47 *
48 * Atomically adds @i to @v.
49 */
50static __inline__ void atomic_add(int i, atomic_t * v)
51{
52 if (cpu_has_llsc && R10000_LLSC_WAR) {
53 unsigned long temp;
54
55 __asm__ __volatile__(
Maciej W. Rozyckic4559f62005-06-23 15:57:15 +000056 " .set mips3 \n"
Linus Torvalds1da177e2005-04-16 15:20:36 -070057 "1: ll %0, %1 # atomic_add \n"
58 " addu %0, %2 \n"
59 " sc %0, %1 \n"
60 " beqzl %0, 1b \n"
Maciej W. Rozyckiaac8aa72005-06-14 17:35:03 +000061 " .set mips0 \n"
Linus Torvalds1da177e2005-04-16 15:20:36 -070062 : "=&r" (temp), "=m" (v->counter)
63 : "Ir" (i), "m" (v->counter));
64 } else if (cpu_has_llsc) {
65 unsigned long temp;
66
67 __asm__ __volatile__(
Maciej W. Rozyckic4559f62005-06-23 15:57:15 +000068 " .set mips3 \n"
Linus Torvalds1da177e2005-04-16 15:20:36 -070069 "1: ll %0, %1 # atomic_add \n"
70 " addu %0, %2 \n"
71 " sc %0, %1 \n"
Ralf Baechlef65e4fa2006-09-28 01:45:21 +010072 " beqz %0, 2f \n"
73 " .subsection 2 \n"
74 "2: b 1b \n"
75 " .previous \n"
Maciej W. Rozyckiaac8aa72005-06-14 17:35:03 +000076 " .set mips0 \n"
Linus Torvalds1da177e2005-04-16 15:20:36 -070077 : "=&r" (temp), "=m" (v->counter)
78 : "Ir" (i), "m" (v->counter));
79 } else {
80 unsigned long flags;
81
Ralf Baechle49edd092007-03-16 16:10:36 +000082 raw_local_irq_save(flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -070083 v->counter += i;
Ralf Baechle49edd092007-03-16 16:10:36 +000084 raw_local_irq_restore(flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -070085 }
86}
87
88/*
89 * atomic_sub - subtract the atomic variable
90 * @i: integer value to subtract
91 * @v: pointer of type atomic_t
92 *
93 * Atomically subtracts @i from @v.
94 */
95static __inline__ void atomic_sub(int i, atomic_t * v)
96{
97 if (cpu_has_llsc && R10000_LLSC_WAR) {
98 unsigned long temp;
99
100 __asm__ __volatile__(
Maciej W. Rozyckic4559f62005-06-23 15:57:15 +0000101 " .set mips3 \n"
Linus Torvalds1da177e2005-04-16 15:20:36 -0700102 "1: ll %0, %1 # atomic_sub \n"
103 " subu %0, %2 \n"
104 " sc %0, %1 \n"
105 " beqzl %0, 1b \n"
Maciej W. Rozyckiaac8aa72005-06-14 17:35:03 +0000106 " .set mips0 \n"
Linus Torvalds1da177e2005-04-16 15:20:36 -0700107 : "=&r" (temp), "=m" (v->counter)
108 : "Ir" (i), "m" (v->counter));
109 } else if (cpu_has_llsc) {
110 unsigned long temp;
111
112 __asm__ __volatile__(
Maciej W. Rozyckic4559f62005-06-23 15:57:15 +0000113 " .set mips3 \n"
Linus Torvalds1da177e2005-04-16 15:20:36 -0700114 "1: ll %0, %1 # atomic_sub \n"
115 " subu %0, %2 \n"
116 " sc %0, %1 \n"
Ralf Baechlef65e4fa2006-09-28 01:45:21 +0100117 " beqz %0, 2f \n"
118 " .subsection 2 \n"
119 "2: b 1b \n"
120 " .previous \n"
Maciej W. Rozyckiaac8aa72005-06-14 17:35:03 +0000121 " .set mips0 \n"
Linus Torvalds1da177e2005-04-16 15:20:36 -0700122 : "=&r" (temp), "=m" (v->counter)
123 : "Ir" (i), "m" (v->counter));
124 } else {
125 unsigned long flags;
126
Ralf Baechle49edd092007-03-16 16:10:36 +0000127 raw_local_irq_save(flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700128 v->counter -= i;
Ralf Baechle49edd092007-03-16 16:10:36 +0000129 raw_local_irq_restore(flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700130 }
131}
132
133/*
134 * Same as above, but return the result value
135 */
136static __inline__ int atomic_add_return(int i, atomic_t * v)
137{
138 unsigned long result;
139
Ralf Baechle0004a9d2006-10-31 03:45:07 +0000140 smp_mb();
141
Linus Torvalds1da177e2005-04-16 15:20:36 -0700142 if (cpu_has_llsc && R10000_LLSC_WAR) {
143 unsigned long temp;
144
145 __asm__ __volatile__(
Maciej W. Rozyckic4559f62005-06-23 15:57:15 +0000146 " .set mips3 \n"
Linus Torvalds1da177e2005-04-16 15:20:36 -0700147 "1: ll %1, %2 # atomic_add_return \n"
148 " addu %0, %1, %3 \n"
149 " sc %0, %2 \n"
150 " beqzl %0, 1b \n"
151 " addu %0, %1, %3 \n"
Maciej W. Rozyckiaac8aa72005-06-14 17:35:03 +0000152 " .set mips0 \n"
Linus Torvalds1da177e2005-04-16 15:20:36 -0700153 : "=&r" (result), "=&r" (temp), "=m" (v->counter)
154 : "Ir" (i), "m" (v->counter)
155 : "memory");
156 } else if (cpu_has_llsc) {
157 unsigned long temp;
158
159 __asm__ __volatile__(
Maciej W. Rozyckic4559f62005-06-23 15:57:15 +0000160 " .set mips3 \n"
Linus Torvalds1da177e2005-04-16 15:20:36 -0700161 "1: ll %1, %2 # atomic_add_return \n"
162 " addu %0, %1, %3 \n"
163 " sc %0, %2 \n"
Ralf Baechlef65e4fa2006-09-28 01:45:21 +0100164 " beqz %0, 2f \n"
Linus Torvalds1da177e2005-04-16 15:20:36 -0700165 " addu %0, %1, %3 \n"
Ralf Baechlef65e4fa2006-09-28 01:45:21 +0100166 " .subsection 2 \n"
167 "2: b 1b \n"
168 " .previous \n"
Maciej W. Rozyckiaac8aa72005-06-14 17:35:03 +0000169 " .set mips0 \n"
Linus Torvalds1da177e2005-04-16 15:20:36 -0700170 : "=&r" (result), "=&r" (temp), "=m" (v->counter)
171 : "Ir" (i), "m" (v->counter)
172 : "memory");
173 } else {
174 unsigned long flags;
175
Ralf Baechle49edd092007-03-16 16:10:36 +0000176 raw_local_irq_save(flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700177 result = v->counter;
178 result += i;
179 v->counter = result;
Ralf Baechle49edd092007-03-16 16:10:36 +0000180 raw_local_irq_restore(flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700181 }
182
Ralf Baechle0004a9d2006-10-31 03:45:07 +0000183 smp_mb();
184
Linus Torvalds1da177e2005-04-16 15:20:36 -0700185 return result;
186}
187
188static __inline__ int atomic_sub_return(int i, atomic_t * v)
189{
190 unsigned long result;
191
Ralf Baechle0004a9d2006-10-31 03:45:07 +0000192 smp_mb();
193
Linus Torvalds1da177e2005-04-16 15:20:36 -0700194 if (cpu_has_llsc && R10000_LLSC_WAR) {
195 unsigned long temp;
196
197 __asm__ __volatile__(
Maciej W. Rozyckic4559f62005-06-23 15:57:15 +0000198 " .set mips3 \n"
Linus Torvalds1da177e2005-04-16 15:20:36 -0700199 "1: ll %1, %2 # atomic_sub_return \n"
200 " subu %0, %1, %3 \n"
201 " sc %0, %2 \n"
202 " beqzl %0, 1b \n"
203 " subu %0, %1, %3 \n"
Maciej W. Rozyckiaac8aa72005-06-14 17:35:03 +0000204 " .set mips0 \n"
Linus Torvalds1da177e2005-04-16 15:20:36 -0700205 : "=&r" (result), "=&r" (temp), "=m" (v->counter)
206 : "Ir" (i), "m" (v->counter)
207 : "memory");
208 } else if (cpu_has_llsc) {
209 unsigned long temp;
210
211 __asm__ __volatile__(
Maciej W. Rozyckic4559f62005-06-23 15:57:15 +0000212 " .set mips3 \n"
Linus Torvalds1da177e2005-04-16 15:20:36 -0700213 "1: ll %1, %2 # atomic_sub_return \n"
214 " subu %0, %1, %3 \n"
215 " sc %0, %2 \n"
Ralf Baechlef65e4fa2006-09-28 01:45:21 +0100216 " beqz %0, 2f \n"
Linus Torvalds1da177e2005-04-16 15:20:36 -0700217 " subu %0, %1, %3 \n"
Ralf Baechlef65e4fa2006-09-28 01:45:21 +0100218 " .subsection 2 \n"
219 "2: b 1b \n"
220 " .previous \n"
Maciej W. Rozyckiaac8aa72005-06-14 17:35:03 +0000221 " .set mips0 \n"
Linus Torvalds1da177e2005-04-16 15:20:36 -0700222 : "=&r" (result), "=&r" (temp), "=m" (v->counter)
223 : "Ir" (i), "m" (v->counter)
224 : "memory");
225 } else {
226 unsigned long flags;
227
Ralf Baechle49edd092007-03-16 16:10:36 +0000228 raw_local_irq_save(flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700229 result = v->counter;
230 result -= i;
231 v->counter = result;
Ralf Baechle49edd092007-03-16 16:10:36 +0000232 raw_local_irq_restore(flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700233 }
234
Ralf Baechle0004a9d2006-10-31 03:45:07 +0000235 smp_mb();
236
Linus Torvalds1da177e2005-04-16 15:20:36 -0700237 return result;
238}
239
240/*
Arnaud Gierschf10d14d2005-11-13 00:38:18 +0100241 * atomic_sub_if_positive - conditionally subtract integer from atomic variable
242 * @i: integer value to subtract
Linus Torvalds1da177e2005-04-16 15:20:36 -0700243 * @v: pointer of type atomic_t
244 *
Arnaud Gierschf10d14d2005-11-13 00:38:18 +0100245 * Atomically test @v and subtract @i if @v is greater or equal than @i.
246 * The function returns the old value of @v minus @i.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700247 */
248static __inline__ int atomic_sub_if_positive(int i, atomic_t * v)
249{
250 unsigned long result;
251
Ralf Baechle0004a9d2006-10-31 03:45:07 +0000252 smp_mb();
253
Linus Torvalds1da177e2005-04-16 15:20:36 -0700254 if (cpu_has_llsc && R10000_LLSC_WAR) {
255 unsigned long temp;
256
257 __asm__ __volatile__(
Maciej W. Rozyckic4559f62005-06-23 15:57:15 +0000258 " .set mips3 \n"
Linus Torvalds1da177e2005-04-16 15:20:36 -0700259 "1: ll %1, %2 # atomic_sub_if_positive\n"
260 " subu %0, %1, %3 \n"
261 " bltz %0, 1f \n"
262 " sc %0, %2 \n"
Ralf Baechle92f22c12006-02-23 14:10:53 +0000263 " .set noreorder \n"
Linus Torvalds1da177e2005-04-16 15:20:36 -0700264 " beqzl %0, 1b \n"
Ralf Baechle92f22c12006-02-23 14:10:53 +0000265 " subu %0, %1, %3 \n"
266 " .set reorder \n"
Linus Torvalds1da177e2005-04-16 15:20:36 -0700267 "1: \n"
Maciej W. Rozyckiaac8aa72005-06-14 17:35:03 +0000268 " .set mips0 \n"
Linus Torvalds1da177e2005-04-16 15:20:36 -0700269 : "=&r" (result), "=&r" (temp), "=m" (v->counter)
270 : "Ir" (i), "m" (v->counter)
271 : "memory");
272 } else if (cpu_has_llsc) {
273 unsigned long temp;
274
275 __asm__ __volatile__(
Maciej W. Rozyckic4559f62005-06-23 15:57:15 +0000276 " .set mips3 \n"
Linus Torvalds1da177e2005-04-16 15:20:36 -0700277 "1: ll %1, %2 # atomic_sub_if_positive\n"
278 " subu %0, %1, %3 \n"
279 " bltz %0, 1f \n"
280 " sc %0, %2 \n"
Ralf Baechle92f22c12006-02-23 14:10:53 +0000281 " .set noreorder \n"
Ralf Baechlef65e4fa2006-09-28 01:45:21 +0100282 " beqz %0, 2f \n"
Ralf Baechle92f22c12006-02-23 14:10:53 +0000283 " subu %0, %1, %3 \n"
284 " .set reorder \n"
Linus Torvalds1da177e2005-04-16 15:20:36 -0700285 "1: \n"
Ralf Baechlef65e4fa2006-09-28 01:45:21 +0100286 " .subsection 2 \n"
287 "2: b 1b \n"
288 " .previous \n"
Maciej W. Rozyckiaac8aa72005-06-14 17:35:03 +0000289 " .set mips0 \n"
Linus Torvalds1da177e2005-04-16 15:20:36 -0700290 : "=&r" (result), "=&r" (temp), "=m" (v->counter)
291 : "Ir" (i), "m" (v->counter)
292 : "memory");
293 } else {
294 unsigned long flags;
295
Ralf Baechle49edd092007-03-16 16:10:36 +0000296 raw_local_irq_save(flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700297 result = v->counter;
298 result -= i;
299 if (result >= 0)
300 v->counter = result;
Ralf Baechle49edd092007-03-16 16:10:36 +0000301 raw_local_irq_restore(flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700302 }
303
Ralf Baechle0004a9d2006-10-31 03:45:07 +0000304 smp_mb();
305
Linus Torvalds1da177e2005-04-16 15:20:36 -0700306 return result;
307}
308
Mathieu Desnoyerse12f6442007-05-08 00:34:24 -0700309#define atomic_cmpxchg(v, o, n) (cmpxchg(&((v)->counter), (o), (n)))
310#define atomic_xchg(v, new) (xchg(&((v)->counter), (new)))
Nick Piggin4a6dae62005-11-13 16:07:24 -0800311
Nick Piggin8426e1f2005-11-13 16:07:25 -0800312/**
313 * atomic_add_unless - add unless the number is a given value
314 * @v: pointer of type atomic_t
315 * @a: the amount to add to v...
316 * @u: ...unless v is equal to u.
317 *
318 * Atomically adds @a to @v, so long as it was not @u.
319 * Returns non-zero if @v was not @u, and zero otherwise.
320 */
321#define atomic_add_unless(v, a, u) \
322({ \
Mathieu Desnoyerse12f6442007-05-08 00:34:24 -0700323 __typeof__((v)->counter) c, old; \
Nick Piggin8426e1f2005-11-13 16:07:25 -0800324 c = atomic_read(v); \
325 while (c != (u) && (old = atomic_cmpxchg((v), c, c + (a))) != c) \
326 c = old; \
327 c != (u); \
328})
329#define atomic_inc_not_zero(v) atomic_add_unless((v), 1, 0)
330
Linus Torvalds1da177e2005-04-16 15:20:36 -0700331#define atomic_dec_return(v) atomic_sub_return(1,(v))
332#define atomic_inc_return(v) atomic_add_return(1,(v))
333
334/*
335 * atomic_sub_and_test - subtract value from variable and test result
336 * @i: integer value to subtract
337 * @v: pointer of type atomic_t
338 *
339 * Atomically subtracts @i from @v and returns
340 * true if the result is zero, or false for all
341 * other cases.
342 */
343#define atomic_sub_and_test(i,v) (atomic_sub_return((i), (v)) == 0)
344
345/*
346 * atomic_inc_and_test - increment and test
347 * @v: pointer of type atomic_t
348 *
349 * Atomically increments @v by 1
350 * and returns true if the result is zero, or false for all
351 * other cases.
352 */
353#define atomic_inc_and_test(v) (atomic_inc_return(v) == 0)
354
355/*
356 * atomic_dec_and_test - decrement by 1 and test
357 * @v: pointer of type atomic_t
358 *
359 * Atomically decrements @v by 1 and
360 * returns true if the result is 0, or false for all other
361 * cases.
362 */
363#define atomic_dec_and_test(v) (atomic_sub_return(1, (v)) == 0)
364
365/*
366 * atomic_dec_if_positive - decrement by 1 if old value positive
367 * @v: pointer of type atomic_t
368 */
369#define atomic_dec_if_positive(v) atomic_sub_if_positive(1, v)
370
371/*
372 * atomic_inc - increment atomic variable
373 * @v: pointer of type atomic_t
374 *
375 * Atomically increments @v by 1.
376 */
377#define atomic_inc(v) atomic_add(1,(v))
378
379/*
380 * atomic_dec - decrement and test
381 * @v: pointer of type atomic_t
382 *
383 * Atomically decrements @v by 1.
384 */
385#define atomic_dec(v) atomic_sub(1,(v))
386
387/*
388 * atomic_add_negative - add and test if negative
389 * @v: pointer of type atomic_t
390 * @i: integer value to add
391 *
392 * Atomically adds @i to @v and returns true
393 * if the result is negative, or false when
394 * result is greater than or equal to zero.
395 */
396#define atomic_add_negative(i,v) (atomic_add_return(i, (v)) < 0)
397
Ralf Baechle875d43e2005-09-03 15:56:16 -0700398#ifdef CONFIG_64BIT
Linus Torvalds1da177e2005-04-16 15:20:36 -0700399
Ralf Baechle4f8b5c72006-11-30 15:38:10 +0000400typedef struct { volatile long counter; } atomic64_t;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700401
402#define ATOMIC64_INIT(i) { (i) }
403
404/*
405 * atomic64_read - read atomic variable
406 * @v: pointer of type atomic64_t
407 *
408 */
409#define atomic64_read(v) ((v)->counter)
410
411/*
412 * atomic64_set - set atomic variable
413 * @v: pointer of type atomic64_t
414 * @i: required value
415 */
416#define atomic64_set(v,i) ((v)->counter = (i))
417
418/*
419 * atomic64_add - add integer to atomic variable
420 * @i: integer value to add
421 * @v: pointer of type atomic64_t
422 *
423 * Atomically adds @i to @v.
424 */
425static __inline__ void atomic64_add(long i, atomic64_t * v)
426{
427 if (cpu_has_llsc && R10000_LLSC_WAR) {
428 unsigned long temp;
429
430 __asm__ __volatile__(
Maciej W. Rozyckiaac8aa72005-06-14 17:35:03 +0000431 " .set mips3 \n"
Linus Torvalds1da177e2005-04-16 15:20:36 -0700432 "1: lld %0, %1 # atomic64_add \n"
433 " addu %0, %2 \n"
434 " scd %0, %1 \n"
435 " beqzl %0, 1b \n"
Maciej W. Rozyckiaac8aa72005-06-14 17:35:03 +0000436 " .set mips0 \n"
Linus Torvalds1da177e2005-04-16 15:20:36 -0700437 : "=&r" (temp), "=m" (v->counter)
438 : "Ir" (i), "m" (v->counter));
439 } else if (cpu_has_llsc) {
440 unsigned long temp;
441
442 __asm__ __volatile__(
Maciej W. Rozyckiaac8aa72005-06-14 17:35:03 +0000443 " .set mips3 \n"
Linus Torvalds1da177e2005-04-16 15:20:36 -0700444 "1: lld %0, %1 # atomic64_add \n"
445 " addu %0, %2 \n"
446 " scd %0, %1 \n"
Ralf Baechlef65e4fa2006-09-28 01:45:21 +0100447 " beqz %0, 2f \n"
448 " .subsection 2 \n"
449 "2: b 1b \n"
450 " .previous \n"
Maciej W. Rozyckiaac8aa72005-06-14 17:35:03 +0000451 " .set mips0 \n"
Linus Torvalds1da177e2005-04-16 15:20:36 -0700452 : "=&r" (temp), "=m" (v->counter)
453 : "Ir" (i), "m" (v->counter));
454 } else {
455 unsigned long flags;
456
Ralf Baechle49edd092007-03-16 16:10:36 +0000457 raw_local_irq_save(flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700458 v->counter += i;
Ralf Baechle49edd092007-03-16 16:10:36 +0000459 raw_local_irq_restore(flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700460 }
461}
462
463/*
464 * atomic64_sub - subtract the atomic variable
465 * @i: integer value to subtract
466 * @v: pointer of type atomic64_t
467 *
468 * Atomically subtracts @i from @v.
469 */
470static __inline__ void atomic64_sub(long i, atomic64_t * v)
471{
472 if (cpu_has_llsc && R10000_LLSC_WAR) {
473 unsigned long temp;
474
475 __asm__ __volatile__(
Maciej W. Rozyckiaac8aa72005-06-14 17:35:03 +0000476 " .set mips3 \n"
Linus Torvalds1da177e2005-04-16 15:20:36 -0700477 "1: lld %0, %1 # atomic64_sub \n"
478 " subu %0, %2 \n"
479 " scd %0, %1 \n"
480 " beqzl %0, 1b \n"
Maciej W. Rozyckiaac8aa72005-06-14 17:35:03 +0000481 " .set mips0 \n"
Linus Torvalds1da177e2005-04-16 15:20:36 -0700482 : "=&r" (temp), "=m" (v->counter)
483 : "Ir" (i), "m" (v->counter));
484 } else if (cpu_has_llsc) {
485 unsigned long temp;
486
487 __asm__ __volatile__(
Maciej W. Rozyckiaac8aa72005-06-14 17:35:03 +0000488 " .set mips3 \n"
Linus Torvalds1da177e2005-04-16 15:20:36 -0700489 "1: lld %0, %1 # atomic64_sub \n"
490 " subu %0, %2 \n"
491 " scd %0, %1 \n"
Ralf Baechlef65e4fa2006-09-28 01:45:21 +0100492 " beqz %0, 2f \n"
493 " .subsection 2 \n"
494 "2: b 1b \n"
495 " .previous \n"
Maciej W. Rozyckiaac8aa72005-06-14 17:35:03 +0000496 " .set mips0 \n"
Linus Torvalds1da177e2005-04-16 15:20:36 -0700497 : "=&r" (temp), "=m" (v->counter)
498 : "Ir" (i), "m" (v->counter));
499 } else {
500 unsigned long flags;
501
Ralf Baechle49edd092007-03-16 16:10:36 +0000502 raw_local_irq_save(flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700503 v->counter -= i;
Ralf Baechle49edd092007-03-16 16:10:36 +0000504 raw_local_irq_restore(flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700505 }
506}
507
508/*
509 * Same as above, but return the result value
510 */
511static __inline__ long atomic64_add_return(long i, atomic64_t * v)
512{
513 unsigned long result;
514
Ralf Baechle0004a9d2006-10-31 03:45:07 +0000515 smp_mb();
516
Linus Torvalds1da177e2005-04-16 15:20:36 -0700517 if (cpu_has_llsc && R10000_LLSC_WAR) {
518 unsigned long temp;
519
520 __asm__ __volatile__(
Maciej W. Rozyckiaac8aa72005-06-14 17:35:03 +0000521 " .set mips3 \n"
Linus Torvalds1da177e2005-04-16 15:20:36 -0700522 "1: lld %1, %2 # atomic64_add_return \n"
523 " addu %0, %1, %3 \n"
524 " scd %0, %2 \n"
525 " beqzl %0, 1b \n"
526 " addu %0, %1, %3 \n"
Maciej W. Rozyckiaac8aa72005-06-14 17:35:03 +0000527 " .set mips0 \n"
Linus Torvalds1da177e2005-04-16 15:20:36 -0700528 : "=&r" (result), "=&r" (temp), "=m" (v->counter)
529 : "Ir" (i), "m" (v->counter)
530 : "memory");
531 } else if (cpu_has_llsc) {
532 unsigned long temp;
533
534 __asm__ __volatile__(
Maciej W. Rozyckiaac8aa72005-06-14 17:35:03 +0000535 " .set mips3 \n"
Linus Torvalds1da177e2005-04-16 15:20:36 -0700536 "1: lld %1, %2 # atomic64_add_return \n"
537 " addu %0, %1, %3 \n"
538 " scd %0, %2 \n"
Ralf Baechlef65e4fa2006-09-28 01:45:21 +0100539 " beqz %0, 2f \n"
Linus Torvalds1da177e2005-04-16 15:20:36 -0700540 " addu %0, %1, %3 \n"
Ralf Baechlef65e4fa2006-09-28 01:45:21 +0100541 " .subsection 2 \n"
542 "2: b 1b \n"
543 " .previous \n"
Maciej W. Rozyckiaac8aa72005-06-14 17:35:03 +0000544 " .set mips0 \n"
Linus Torvalds1da177e2005-04-16 15:20:36 -0700545 : "=&r" (result), "=&r" (temp), "=m" (v->counter)
546 : "Ir" (i), "m" (v->counter)
547 : "memory");
548 } else {
549 unsigned long flags;
550
Ralf Baechle49edd092007-03-16 16:10:36 +0000551 raw_local_irq_save(flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700552 result = v->counter;
553 result += i;
554 v->counter = result;
Ralf Baechle49edd092007-03-16 16:10:36 +0000555 raw_local_irq_restore(flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700556 }
557
Ralf Baechle0004a9d2006-10-31 03:45:07 +0000558 smp_mb();
559
Linus Torvalds1da177e2005-04-16 15:20:36 -0700560 return result;
561}
562
563static __inline__ long atomic64_sub_return(long i, atomic64_t * v)
564{
565 unsigned long result;
566
Ralf Baechle0004a9d2006-10-31 03:45:07 +0000567 smp_mb();
568
Linus Torvalds1da177e2005-04-16 15:20:36 -0700569 if (cpu_has_llsc && R10000_LLSC_WAR) {
570 unsigned long temp;
571
572 __asm__ __volatile__(
Maciej W. Rozyckiaac8aa72005-06-14 17:35:03 +0000573 " .set mips3 \n"
Linus Torvalds1da177e2005-04-16 15:20:36 -0700574 "1: lld %1, %2 # atomic64_sub_return \n"
575 " subu %0, %1, %3 \n"
576 " scd %0, %2 \n"
577 " beqzl %0, 1b \n"
578 " subu %0, %1, %3 \n"
Maciej W. Rozyckiaac8aa72005-06-14 17:35:03 +0000579 " .set mips0 \n"
Linus Torvalds1da177e2005-04-16 15:20:36 -0700580 : "=&r" (result), "=&r" (temp), "=m" (v->counter)
581 : "Ir" (i), "m" (v->counter)
582 : "memory");
583 } else if (cpu_has_llsc) {
584 unsigned long temp;
585
586 __asm__ __volatile__(
Maciej W. Rozyckiaac8aa72005-06-14 17:35:03 +0000587 " .set mips3 \n"
Linus Torvalds1da177e2005-04-16 15:20:36 -0700588 "1: lld %1, %2 # atomic64_sub_return \n"
589 " subu %0, %1, %3 \n"
590 " scd %0, %2 \n"
Ralf Baechlef65e4fa2006-09-28 01:45:21 +0100591 " beqz %0, 2f \n"
Linus Torvalds1da177e2005-04-16 15:20:36 -0700592 " subu %0, %1, %3 \n"
Ralf Baechlef65e4fa2006-09-28 01:45:21 +0100593 " .subsection 2 \n"
594 "2: b 1b \n"
595 " .previous \n"
Maciej W. Rozyckiaac8aa72005-06-14 17:35:03 +0000596 " .set mips0 \n"
Linus Torvalds1da177e2005-04-16 15:20:36 -0700597 : "=&r" (result), "=&r" (temp), "=m" (v->counter)
598 : "Ir" (i), "m" (v->counter)
599 : "memory");
600 } else {
601 unsigned long flags;
602
Ralf Baechle49edd092007-03-16 16:10:36 +0000603 raw_local_irq_save(flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700604 result = v->counter;
605 result -= i;
606 v->counter = result;
Ralf Baechle49edd092007-03-16 16:10:36 +0000607 raw_local_irq_restore(flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700608 }
609
Ralf Baechle0004a9d2006-10-31 03:45:07 +0000610 smp_mb();
611
Linus Torvalds1da177e2005-04-16 15:20:36 -0700612 return result;
613}
614
615/*
Arnaud Gierschf10d14d2005-11-13 00:38:18 +0100616 * atomic64_sub_if_positive - conditionally subtract integer from atomic variable
617 * @i: integer value to subtract
Linus Torvalds1da177e2005-04-16 15:20:36 -0700618 * @v: pointer of type atomic64_t
619 *
Arnaud Gierschf10d14d2005-11-13 00:38:18 +0100620 * Atomically test @v and subtract @i if @v is greater or equal than @i.
621 * The function returns the old value of @v minus @i.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700622 */
623static __inline__ long atomic64_sub_if_positive(long i, atomic64_t * v)
624{
625 unsigned long result;
626
Ralf Baechle0004a9d2006-10-31 03:45:07 +0000627 smp_mb();
628
Linus Torvalds1da177e2005-04-16 15:20:36 -0700629 if (cpu_has_llsc && R10000_LLSC_WAR) {
630 unsigned long temp;
631
632 __asm__ __volatile__(
Maciej W. Rozyckiaac8aa72005-06-14 17:35:03 +0000633 " .set mips3 \n"
Linus Torvalds1da177e2005-04-16 15:20:36 -0700634 "1: lld %1, %2 # atomic64_sub_if_positive\n"
635 " dsubu %0, %1, %3 \n"
636 " bltz %0, 1f \n"
637 " scd %0, %2 \n"
Ralf Baechle92f22c12006-02-23 14:10:53 +0000638 " .set noreorder \n"
Linus Torvalds1da177e2005-04-16 15:20:36 -0700639 " beqzl %0, 1b \n"
Ralf Baechle92f22c12006-02-23 14:10:53 +0000640 " dsubu %0, %1, %3 \n"
641 " .set reorder \n"
Linus Torvalds1da177e2005-04-16 15:20:36 -0700642 "1: \n"
Maciej W. Rozyckiaac8aa72005-06-14 17:35:03 +0000643 " .set mips0 \n"
Linus Torvalds1da177e2005-04-16 15:20:36 -0700644 : "=&r" (result), "=&r" (temp), "=m" (v->counter)
645 : "Ir" (i), "m" (v->counter)
646 : "memory");
647 } else if (cpu_has_llsc) {
648 unsigned long temp;
649
650 __asm__ __volatile__(
Maciej W. Rozyckiaac8aa72005-06-14 17:35:03 +0000651 " .set mips3 \n"
Linus Torvalds1da177e2005-04-16 15:20:36 -0700652 "1: lld %1, %2 # atomic64_sub_if_positive\n"
653 " dsubu %0, %1, %3 \n"
654 " bltz %0, 1f \n"
655 " scd %0, %2 \n"
Ralf Baechle92f22c12006-02-23 14:10:53 +0000656 " .set noreorder \n"
Ralf Baechlef65e4fa2006-09-28 01:45:21 +0100657 " beqz %0, 2f \n"
Ralf Baechle92f22c12006-02-23 14:10:53 +0000658 " dsubu %0, %1, %3 \n"
659 " .set reorder \n"
Linus Torvalds1da177e2005-04-16 15:20:36 -0700660 "1: \n"
Ralf Baechlef65e4fa2006-09-28 01:45:21 +0100661 " .subsection 2 \n"
662 "2: b 1b \n"
663 " .previous \n"
Maciej W. Rozyckiaac8aa72005-06-14 17:35:03 +0000664 " .set mips0 \n"
Linus Torvalds1da177e2005-04-16 15:20:36 -0700665 : "=&r" (result), "=&r" (temp), "=m" (v->counter)
666 : "Ir" (i), "m" (v->counter)
667 : "memory");
668 } else {
669 unsigned long flags;
670
Ralf Baechle49edd092007-03-16 16:10:36 +0000671 raw_local_irq_save(flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700672 result = v->counter;
673 result -= i;
674 if (result >= 0)
675 v->counter = result;
Ralf Baechle49edd092007-03-16 16:10:36 +0000676 raw_local_irq_restore(flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700677 }
678
Ralf Baechle0004a9d2006-10-31 03:45:07 +0000679 smp_mb();
680
Linus Torvalds1da177e2005-04-16 15:20:36 -0700681 return result;
682}
683
Mathieu Desnoyerse12f6442007-05-08 00:34:24 -0700684#define atomic64_cmpxchg(v, o, n) \
685 (((__typeof__((v)->counter)))cmpxchg(&((v)->counter), (o), (n)))
686#define atomic64_xchg(v, new) (xchg(&((v)->counter), (new)))
687
688/**
689 * atomic64_add_unless - add unless the number is a given value
690 * @v: pointer of type atomic64_t
691 * @a: the amount to add to v...
692 * @u: ...unless v is equal to u.
693 *
694 * Atomically adds @a to @v, so long as it was not @u.
695 * Returns non-zero if @v was not @u, and zero otherwise.
696 */
697#define atomic64_add_unless(v, a, u) \
698({ \
699 __typeof__((v)->counter) c, old; \
700 c = atomic_read(v); \
701 while (c != (u) && (old = atomic64_cmpxchg((v), c, c + (a))) != c) \
702 c = old; \
703 c != (u); \
704})
705#define atomic64_inc_not_zero(v) atomic64_add_unless((v), 1, 0)
706
Linus Torvalds1da177e2005-04-16 15:20:36 -0700707#define atomic64_dec_return(v) atomic64_sub_return(1,(v))
708#define atomic64_inc_return(v) atomic64_add_return(1,(v))
709
710/*
711 * atomic64_sub_and_test - subtract value from variable and test result
712 * @i: integer value to subtract
713 * @v: pointer of type atomic64_t
714 *
715 * Atomically subtracts @i from @v and returns
716 * true if the result is zero, or false for all
717 * other cases.
718 */
719#define atomic64_sub_and_test(i,v) (atomic64_sub_return((i), (v)) == 0)
720
721/*
722 * atomic64_inc_and_test - increment and test
723 * @v: pointer of type atomic64_t
724 *
725 * Atomically increments @v by 1
726 * and returns true if the result is zero, or false for all
727 * other cases.
728 */
729#define atomic64_inc_and_test(v) (atomic64_inc_return(v) == 0)
730
731/*
732 * atomic64_dec_and_test - decrement by 1 and test
733 * @v: pointer of type atomic64_t
734 *
735 * Atomically decrements @v by 1 and
736 * returns true if the result is 0, or false for all other
737 * cases.
738 */
739#define atomic64_dec_and_test(v) (atomic64_sub_return(1, (v)) == 0)
740
741/*
742 * atomic64_dec_if_positive - decrement by 1 if old value positive
743 * @v: pointer of type atomic64_t
744 */
745#define atomic64_dec_if_positive(v) atomic64_sub_if_positive(1, v)
746
747/*
748 * atomic64_inc - increment atomic variable
749 * @v: pointer of type atomic64_t
750 *
751 * Atomically increments @v by 1.
752 */
753#define atomic64_inc(v) atomic64_add(1,(v))
754
755/*
756 * atomic64_dec - decrement and test
757 * @v: pointer of type atomic64_t
758 *
759 * Atomically decrements @v by 1.
760 */
761#define atomic64_dec(v) atomic64_sub(1,(v))
762
763/*
764 * atomic64_add_negative - add and test if negative
765 * @v: pointer of type atomic64_t
766 * @i: integer value to add
767 *
768 * Atomically adds @i to @v and returns true
769 * if the result is negative, or false when
770 * result is greater than or equal to zero.
771 */
772#define atomic64_add_negative(i,v) (atomic64_add_return(i, (v)) < 0)
773
Ralf Baechle875d43e2005-09-03 15:56:16 -0700774#endif /* CONFIG_64BIT */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700775
776/*
777 * atomic*_return operations are serializing but not the non-*_return
778 * versions.
779 */
780#define smp_mb__before_atomic_dec() smp_mb()
781#define smp_mb__after_atomic_dec() smp_mb()
782#define smp_mb__before_atomic_inc() smp_mb()
783#define smp_mb__after_atomic_inc() smp_mb()
784
Christoph Lameterd3cb4872006-01-06 00:11:20 -0800785#include <asm-generic/atomic.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -0700786#endif /* _ASM_ATOMIC_H */