blob: ff90c6e3fcb4ab846f3bcc013279f8f64c566b35 [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001#ifndef __ARCH_I386_ATOMIC__
2#define __ARCH_I386_ATOMIC__
3
Linus Torvalds1da177e2005-04-16 15:20:36 -07004#include <linux/compiler.h>
5#include <asm/processor.h>
6
7/*
8 * Atomic operations that C can't guarantee us. Useful for
9 * resource counting etc..
10 */
11
Linus Torvalds1da177e2005-04-16 15:20:36 -070012/*
13 * Make sure gcc doesn't try to be clever and move things around
14 * on us. We need to use _exactly_ the address the user gave us,
15 * not some alias that contains the same information.
16 */
Linus Torvaldsf9e9dcb2006-12-06 14:42:57 -080017typedef struct { int counter; } atomic_t;
Linus Torvalds1da177e2005-04-16 15:20:36 -070018
19#define ATOMIC_INIT(i) { (i) }
20
21/**
22 * atomic_read - read atomic variable
23 * @v: pointer of type atomic_t
24 *
25 * Atomically reads the value of @v.
26 */
27#define atomic_read(v) ((v)->counter)
28
29/**
30 * atomic_set - set atomic variable
31 * @v: pointer of type atomic_t
32 * @i: required value
33 *
34 * Atomically sets the value of @v to @i.
35 */
36#define atomic_set(v,i) (((v)->counter) = (i))
37
38/**
39 * atomic_add - add integer to atomic variable
40 * @i: integer value to add
41 * @v: pointer of type atomic_t
42 *
43 * Atomically adds @i to @v.
44 */
45static __inline__ void atomic_add(int i, atomic_t *v)
46{
47 __asm__ __volatile__(
Gerd Hoffmann9a0b5812006-03-23 02:59:32 -080048 LOCK_PREFIX "addl %1,%0"
Linus Torvaldsb862f3b2006-07-08 15:24:18 -070049 :"+m" (v->counter)
50 :"ir" (i));
Linus Torvalds1da177e2005-04-16 15:20:36 -070051}
52
53/**
54 * atomic_sub - subtract the atomic variable
55 * @i: integer value to subtract
56 * @v: pointer of type atomic_t
57 *
58 * Atomically subtracts @i from @v.
59 */
60static __inline__ void atomic_sub(int i, atomic_t *v)
61{
62 __asm__ __volatile__(
Gerd Hoffmann9a0b5812006-03-23 02:59:32 -080063 LOCK_PREFIX "subl %1,%0"
Linus Torvaldsb862f3b2006-07-08 15:24:18 -070064 :"+m" (v->counter)
65 :"ir" (i));
Linus Torvalds1da177e2005-04-16 15:20:36 -070066}
67
68/**
69 * atomic_sub_and_test - subtract value from variable and test result
70 * @i: integer value to subtract
71 * @v: pointer of type atomic_t
72 *
73 * Atomically subtracts @i from @v and returns
74 * true if the result is zero, or false for all
75 * other cases.
76 */
77static __inline__ int atomic_sub_and_test(int i, atomic_t *v)
78{
79 unsigned char c;
80
81 __asm__ __volatile__(
Gerd Hoffmann9a0b5812006-03-23 02:59:32 -080082 LOCK_PREFIX "subl %2,%0; sete %1"
Linus Torvaldsb862f3b2006-07-08 15:24:18 -070083 :"+m" (v->counter), "=qm" (c)
84 :"ir" (i) : "memory");
Linus Torvalds1da177e2005-04-16 15:20:36 -070085 return c;
86}
87
88/**
89 * atomic_inc - increment atomic variable
90 * @v: pointer of type atomic_t
91 *
92 * Atomically increments @v by 1.
93 */
94static __inline__ void atomic_inc(atomic_t *v)
95{
96 __asm__ __volatile__(
Gerd Hoffmann9a0b5812006-03-23 02:59:32 -080097 LOCK_PREFIX "incl %0"
Linus Torvaldsb862f3b2006-07-08 15:24:18 -070098 :"+m" (v->counter));
Linus Torvalds1da177e2005-04-16 15:20:36 -070099}
100
101/**
102 * atomic_dec - decrement atomic variable
103 * @v: pointer of type atomic_t
104 *
105 * Atomically decrements @v by 1.
106 */
107static __inline__ void atomic_dec(atomic_t *v)
108{
109 __asm__ __volatile__(
Gerd Hoffmann9a0b5812006-03-23 02:59:32 -0800110 LOCK_PREFIX "decl %0"
Linus Torvaldsb862f3b2006-07-08 15:24:18 -0700111 :"+m" (v->counter));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700112}
113
114/**
115 * atomic_dec_and_test - decrement and test
116 * @v: pointer of type atomic_t
117 *
118 * Atomically decrements @v by 1 and
119 * returns true if the result is 0, or false for all other
120 * cases.
121 */
122static __inline__ int atomic_dec_and_test(atomic_t *v)
123{
124 unsigned char c;
125
126 __asm__ __volatile__(
Gerd Hoffmann9a0b5812006-03-23 02:59:32 -0800127 LOCK_PREFIX "decl %0; sete %1"
Linus Torvaldsb862f3b2006-07-08 15:24:18 -0700128 :"+m" (v->counter), "=qm" (c)
129 : : "memory");
Linus Torvalds1da177e2005-04-16 15:20:36 -0700130 return c != 0;
131}
132
133/**
134 * atomic_inc_and_test - increment and test
135 * @v: pointer of type atomic_t
136 *
137 * Atomically increments @v by 1
138 * and returns true if the result is zero, or false for all
139 * other cases.
140 */
141static __inline__ int atomic_inc_and_test(atomic_t *v)
142{
143 unsigned char c;
144
145 __asm__ __volatile__(
Gerd Hoffmann9a0b5812006-03-23 02:59:32 -0800146 LOCK_PREFIX "incl %0; sete %1"
Linus Torvaldsb862f3b2006-07-08 15:24:18 -0700147 :"+m" (v->counter), "=qm" (c)
148 : : "memory");
Linus Torvalds1da177e2005-04-16 15:20:36 -0700149 return c != 0;
150}
151
152/**
153 * atomic_add_negative - add and test if negative
154 * @v: pointer of type atomic_t
155 * @i: integer value to add
156 *
157 * Atomically adds @i to @v and returns true
158 * if the result is negative, or false when
159 * result is greater than or equal to zero.
160 */
161static __inline__ int atomic_add_negative(int i, atomic_t *v)
162{
163 unsigned char c;
164
165 __asm__ __volatile__(
Gerd Hoffmann9a0b5812006-03-23 02:59:32 -0800166 LOCK_PREFIX "addl %2,%0; sets %1"
Linus Torvaldsb862f3b2006-07-08 15:24:18 -0700167 :"+m" (v->counter), "=qm" (c)
168 :"ir" (i) : "memory");
Linus Torvalds1da177e2005-04-16 15:20:36 -0700169 return c;
170}
171
172/**
173 * atomic_add_return - add and return
174 * @v: pointer of type atomic_t
175 * @i: integer value to add
176 *
177 * Atomically adds @i to @v and returns @i + @v
178 */
179static __inline__ int atomic_add_return(int i, atomic_t *v)
180{
181 int __i;
182#ifdef CONFIG_M386
lepton1bb858f2006-04-18 22:21:10 -0700183 unsigned long flags;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700184 if(unlikely(boot_cpu_data.x86==3))
185 goto no_xadd;
186#endif
187 /* Modern 486+ processor */
188 __i = i;
189 __asm__ __volatile__(
Duncan Sandse4b522d2006-12-07 02:14:13 +0100190 LOCK_PREFIX "xaddl %0, %1"
191 :"+r" (i), "+m" (v->counter)
192 : : "memory");
Linus Torvalds1da177e2005-04-16 15:20:36 -0700193 return i + __i;
194
195#ifdef CONFIG_M386
196no_xadd: /* Legacy 386 processor */
lepton1bb858f2006-04-18 22:21:10 -0700197 local_irq_save(flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700198 __i = atomic_read(v);
199 atomic_set(v, i + __i);
lepton1bb858f2006-04-18 22:21:10 -0700200 local_irq_restore(flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700201 return i + __i;
202#endif
203}
204
205static __inline__ int atomic_sub_return(int i, atomic_t *v)
206{
207 return atomic_add_return(-i,v);
208}
209
Mathieu Desnoyerse656e242007-05-08 00:34:20 -0700210#define atomic_cmpxchg(v, old, new) (cmpxchg(&((v)->counter), (old), (new)))
211#define atomic_xchg(v, new) (xchg(&((v)->counter), (new)))
Nick Piggin4a6dae62005-11-13 16:07:24 -0800212
Nick Piggin8426e1f2005-11-13 16:07:25 -0800213/**
Robert P. J. Day72fd4a32007-02-10 01:45:59 -0800214 * atomic_add_unless - add unless the number is already a given value
Nick Piggin8426e1f2005-11-13 16:07:25 -0800215 * @v: pointer of type atomic_t
216 * @a: the amount to add to v...
217 * @u: ...unless v is equal to u.
218 *
Robert P. J. Day72fd4a32007-02-10 01:45:59 -0800219 * Atomically adds @a to @v, so long as @v was not already @u.
Nick Piggin8426e1f2005-11-13 16:07:25 -0800220 * Returns non-zero if @v was not @u, and zero otherwise.
221 */
Mathieu Desnoyers2856f5e2007-05-08 00:34:38 -0700222static __inline__ int atomic_add_unless(atomic_t *v, int a, int u)
223{
224 int c, old;
225 c = atomic_read(v);
226 for (;;) {
227 if (unlikely(c == (u)))
228 break;
229 old = atomic_cmpxchg((v), c, c + (a));
230 if (likely(old == c))
231 break;
232 c = old;
233 }
234 return c != (u);
235}
236
Nick Piggin8426e1f2005-11-13 16:07:25 -0800237#define atomic_inc_not_zero(v) atomic_add_unless((v), 1, 0)
238
Linus Torvalds1da177e2005-04-16 15:20:36 -0700239#define atomic_inc_return(v) (atomic_add_return(1,v))
240#define atomic_dec_return(v) (atomic_sub_return(1,v))
241
242/* These are x86-specific, used by some header files */
243#define atomic_clear_mask(mask, addr) \
Gerd Hoffmann9a0b5812006-03-23 02:59:32 -0800244__asm__ __volatile__(LOCK_PREFIX "andl %0,%1" \
Linus Torvalds1da177e2005-04-16 15:20:36 -0700245: : "r" (~(mask)),"m" (*addr) : "memory")
246
247#define atomic_set_mask(mask, addr) \
Gerd Hoffmann9a0b5812006-03-23 02:59:32 -0800248__asm__ __volatile__(LOCK_PREFIX "orl %0,%1" \
Linus Torvalds1da177e2005-04-16 15:20:36 -0700249: : "r" (mask),"m" (*(addr)) : "memory")
250
251/* Atomic operations are already serializing on x86 */
252#define smp_mb__before_atomic_dec() barrier()
253#define smp_mb__after_atomic_dec() barrier()
254#define smp_mb__before_atomic_inc() barrier()
255#define smp_mb__after_atomic_inc() barrier()
256
Christoph Lameterd3cb4872006-01-06 00:11:20 -0800257#include <asm-generic/atomic.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -0700258#endif