blob: 21a4825148c09e34c70db728f68951ff32e631f8 [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001#ifndef __ARCH_I386_ATOMIC__
2#define __ARCH_I386_ATOMIC__
3
Linus Torvalds1da177e2005-04-16 15:20:36 -07004#include <linux/compiler.h>
5#include <asm/processor.h>
Jeff Dikea436ed92007-05-08 00:35:02 -07006#include <asm/cmpxchg.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -07007
8/*
9 * Atomic operations that C can't guarantee us. Useful for
10 * resource counting etc..
11 */
12
Linus Torvalds1da177e2005-04-16 15:20:36 -070013/*
14 * Make sure gcc doesn't try to be clever and move things around
15 * on us. We need to use _exactly_ the address the user gave us,
16 * not some alias that contains the same information.
17 */
Joe Perches78ff12e2008-03-23 01:01:41 -070018typedef struct {
19 int counter;
20} atomic_t;
Linus Torvalds1da177e2005-04-16 15:20:36 -070021
22#define ATOMIC_INIT(i) { (i) }
23
24/**
25 * atomic_read - read atomic variable
26 * @v: pointer of type atomic_t
Joe Perches78ff12e2008-03-23 01:01:41 -070027 *
Linus Torvalds1da177e2005-04-16 15:20:36 -070028 * Atomically reads the value of @v.
Joe Perches78ff12e2008-03-23 01:01:41 -070029 */
Linus Torvalds1da177e2005-04-16 15:20:36 -070030#define atomic_read(v) ((v)->counter)
31
32/**
33 * atomic_set - set atomic variable
34 * @v: pointer of type atomic_t
35 * @i: required value
Joe Perches78ff12e2008-03-23 01:01:41 -070036 *
Linus Torvalds1da177e2005-04-16 15:20:36 -070037 * Atomically sets the value of @v to @i.
Joe Perches78ff12e2008-03-23 01:01:41 -070038 */
39#define atomic_set(v, i) (((v)->counter) = (i))
Linus Torvalds1da177e2005-04-16 15:20:36 -070040
41/**
42 * atomic_add - add integer to atomic variable
43 * @i: integer value to add
44 * @v: pointer of type atomic_t
Joe Perches78ff12e2008-03-23 01:01:41 -070045 *
Linus Torvalds1da177e2005-04-16 15:20:36 -070046 * Atomically adds @i to @v.
47 */
Joe Perches78ff12e2008-03-23 01:01:41 -070048static inline void atomic_add(int i, atomic_t *v)
Linus Torvalds1da177e2005-04-16 15:20:36 -070049{
Joe Perches78ff12e2008-03-23 01:01:41 -070050 asm volatile(LOCK_PREFIX "addl %1,%0"
51 : "+m" (v->counter)
52 : "ir" (i));
Linus Torvalds1da177e2005-04-16 15:20:36 -070053}
54
55/**
Robert P. J. Daycc386822007-05-08 00:35:08 -070056 * atomic_sub - subtract integer from atomic variable
Linus Torvalds1da177e2005-04-16 15:20:36 -070057 * @i: integer value to subtract
58 * @v: pointer of type atomic_t
Joe Perches78ff12e2008-03-23 01:01:41 -070059 *
Linus Torvalds1da177e2005-04-16 15:20:36 -070060 * Atomically subtracts @i from @v.
61 */
Joe Perches78ff12e2008-03-23 01:01:41 -070062static inline void atomic_sub(int i, atomic_t *v)
Linus Torvalds1da177e2005-04-16 15:20:36 -070063{
Joe Perches78ff12e2008-03-23 01:01:41 -070064 asm volatile(LOCK_PREFIX "subl %1,%0"
65 : "+m" (v->counter)
66 : "ir" (i));
Linus Torvalds1da177e2005-04-16 15:20:36 -070067}
68
69/**
70 * atomic_sub_and_test - subtract value from variable and test result
71 * @i: integer value to subtract
72 * @v: pointer of type atomic_t
Joe Perches78ff12e2008-03-23 01:01:41 -070073 *
Linus Torvalds1da177e2005-04-16 15:20:36 -070074 * Atomically subtracts @i from @v and returns
75 * true if the result is zero, or false for all
76 * other cases.
77 */
Joe Perches78ff12e2008-03-23 01:01:41 -070078static inline int atomic_sub_and_test(int i, atomic_t *v)
Linus Torvalds1da177e2005-04-16 15:20:36 -070079{
80 unsigned char c;
81
Joe Perches78ff12e2008-03-23 01:01:41 -070082 asm volatile(LOCK_PREFIX "subl %2,%0; sete %1"
83 : "+m" (v->counter), "=qm" (c)
84 : "ir" (i) : "memory");
Linus Torvalds1da177e2005-04-16 15:20:36 -070085 return c;
86}
87
88/**
89 * atomic_inc - increment atomic variable
90 * @v: pointer of type atomic_t
Joe Perches78ff12e2008-03-23 01:01:41 -070091 *
Linus Torvalds1da177e2005-04-16 15:20:36 -070092 * Atomically increments @v by 1.
Joe Perches78ff12e2008-03-23 01:01:41 -070093 */
94static inline void atomic_inc(atomic_t *v)
Linus Torvalds1da177e2005-04-16 15:20:36 -070095{
Joe Perches78ff12e2008-03-23 01:01:41 -070096 asm volatile(LOCK_PREFIX "incl %0"
97 : "+m" (v->counter));
Linus Torvalds1da177e2005-04-16 15:20:36 -070098}
99
100/**
101 * atomic_dec - decrement atomic variable
102 * @v: pointer of type atomic_t
Joe Perches78ff12e2008-03-23 01:01:41 -0700103 *
Linus Torvalds1da177e2005-04-16 15:20:36 -0700104 * Atomically decrements @v by 1.
Joe Perches78ff12e2008-03-23 01:01:41 -0700105 */
106static inline void atomic_dec(atomic_t *v)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700107{
Joe Perches78ff12e2008-03-23 01:01:41 -0700108 asm volatile(LOCK_PREFIX "decl %0"
109 : "+m" (v->counter));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700110}
111
112/**
113 * atomic_dec_and_test - decrement and test
114 * @v: pointer of type atomic_t
Joe Perches78ff12e2008-03-23 01:01:41 -0700115 *
Linus Torvalds1da177e2005-04-16 15:20:36 -0700116 * Atomically decrements @v by 1 and
117 * returns true if the result is 0, or false for all other
118 * cases.
Joe Perches78ff12e2008-03-23 01:01:41 -0700119 */
120static inline int atomic_dec_and_test(atomic_t *v)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700121{
122 unsigned char c;
123
Joe Perches78ff12e2008-03-23 01:01:41 -0700124 asm volatile(LOCK_PREFIX "decl %0; sete %1"
125 : "+m" (v->counter), "=qm" (c)
126 : : "memory");
Linus Torvalds1da177e2005-04-16 15:20:36 -0700127 return c != 0;
128}
129
130/**
Joe Perches78ff12e2008-03-23 01:01:41 -0700131 * atomic_inc_and_test - increment and test
Linus Torvalds1da177e2005-04-16 15:20:36 -0700132 * @v: pointer of type atomic_t
Joe Perches78ff12e2008-03-23 01:01:41 -0700133 *
Linus Torvalds1da177e2005-04-16 15:20:36 -0700134 * Atomically increments @v by 1
135 * and returns true if the result is zero, or false for all
136 * other cases.
Joe Perches78ff12e2008-03-23 01:01:41 -0700137 */
138static inline int atomic_inc_and_test(atomic_t *v)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700139{
140 unsigned char c;
141
Joe Perches78ff12e2008-03-23 01:01:41 -0700142 asm volatile(LOCK_PREFIX "incl %0; sete %1"
143 : "+m" (v->counter), "=qm" (c)
144 : : "memory");
Linus Torvalds1da177e2005-04-16 15:20:36 -0700145 return c != 0;
146}
147
148/**
149 * atomic_add_negative - add and test if negative
150 * @v: pointer of type atomic_t
151 * @i: integer value to add
Joe Perches78ff12e2008-03-23 01:01:41 -0700152 *
Linus Torvalds1da177e2005-04-16 15:20:36 -0700153 * Atomically adds @i to @v and returns true
154 * if the result is negative, or false when
155 * result is greater than or equal to zero.
Joe Perches78ff12e2008-03-23 01:01:41 -0700156 */
157static inline int atomic_add_negative(int i, atomic_t *v)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700158{
159 unsigned char c;
160
Joe Perches78ff12e2008-03-23 01:01:41 -0700161 asm volatile(LOCK_PREFIX "addl %2,%0; sets %1"
162 : "+m" (v->counter), "=qm" (c)
163 : "ir" (i) : "memory");
Linus Torvalds1da177e2005-04-16 15:20:36 -0700164 return c;
165}
166
167/**
Robert P. J. Daycc386822007-05-08 00:35:08 -0700168 * atomic_add_return - add integer and return
Linus Torvalds1da177e2005-04-16 15:20:36 -0700169 * @v: pointer of type atomic_t
170 * @i: integer value to add
171 *
172 * Atomically adds @i to @v and returns @i + @v
173 */
Joe Perches78ff12e2008-03-23 01:01:41 -0700174static inline int atomic_add_return(int i, atomic_t *v)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700175{
176 int __i;
177#ifdef CONFIG_M386
lepton1bb858f2006-04-18 22:21:10 -0700178 unsigned long flags;
Joe Perches78ff12e2008-03-23 01:01:41 -0700179 if (unlikely(boot_cpu_data.x86 <= 3))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700180 goto no_xadd;
181#endif
182 /* Modern 486+ processor */
183 __i = i;
Joe Perches78ff12e2008-03-23 01:01:41 -0700184 asm volatile(LOCK_PREFIX "xaddl %0, %1"
185 : "+r" (i), "+m" (v->counter)
186 : : "memory");
Linus Torvalds1da177e2005-04-16 15:20:36 -0700187 return i + __i;
188
189#ifdef CONFIG_M386
190no_xadd: /* Legacy 386 processor */
lepton1bb858f2006-04-18 22:21:10 -0700191 local_irq_save(flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700192 __i = atomic_read(v);
193 atomic_set(v, i + __i);
lepton1bb858f2006-04-18 22:21:10 -0700194 local_irq_restore(flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700195 return i + __i;
196#endif
197}
198
Robert P. J. Daycc386822007-05-08 00:35:08 -0700199/**
200 * atomic_sub_return - subtract integer and return
201 * @v: pointer of type atomic_t
202 * @i: integer value to subtract
203 *
204 * Atomically subtracts @i from @v and returns @v - @i
205 */
Joe Perches78ff12e2008-03-23 01:01:41 -0700206static inline int atomic_sub_return(int i, atomic_t *v)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700207{
Joe Perches78ff12e2008-03-23 01:01:41 -0700208 return atomic_add_return(-i, v);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700209}
210
Mathieu Desnoyerse656e242007-05-08 00:34:20 -0700211#define atomic_cmpxchg(v, old, new) (cmpxchg(&((v)->counter), (old), (new)))
212#define atomic_xchg(v, new) (xchg(&((v)->counter), (new)))
Nick Piggin4a6dae62005-11-13 16:07:24 -0800213
Nick Piggin8426e1f2005-11-13 16:07:25 -0800214/**
Robert P. J. Day72fd4a32007-02-10 01:45:59 -0800215 * atomic_add_unless - add unless the number is already a given value
Nick Piggin8426e1f2005-11-13 16:07:25 -0800216 * @v: pointer of type atomic_t
217 * @a: the amount to add to v...
218 * @u: ...unless v is equal to u.
219 *
Robert P. J. Day72fd4a32007-02-10 01:45:59 -0800220 * Atomically adds @a to @v, so long as @v was not already @u.
Nick Piggin8426e1f2005-11-13 16:07:25 -0800221 * Returns non-zero if @v was not @u, and zero otherwise.
222 */
Joe Perches78ff12e2008-03-23 01:01:41 -0700223static inline int atomic_add_unless(atomic_t *v, int a, int u)
Mathieu Desnoyers2856f5e2007-05-08 00:34:38 -0700224{
225 int c, old;
226 c = atomic_read(v);
227 for (;;) {
228 if (unlikely(c == (u)))
229 break;
230 old = atomic_cmpxchg((v), c, c + (a));
231 if (likely(old == c))
232 break;
233 c = old;
234 }
235 return c != (u);
236}
237
Nick Piggin8426e1f2005-11-13 16:07:25 -0800238#define atomic_inc_not_zero(v) atomic_add_unless((v), 1, 0)
239
Joe Perches78ff12e2008-03-23 01:01:41 -0700240#define atomic_inc_return(v) (atomic_add_return(1, v))
241#define atomic_dec_return(v) (atomic_sub_return(1, v))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700242
243/* These are x86-specific, used by some header files */
Joe Perches78ff12e2008-03-23 01:01:41 -0700244#define atomic_clear_mask(mask, addr) \
245 asm volatile(LOCK_PREFIX "andl %0,%1" \
246 : : "r" (~(mask)), "m" (*(addr)) : "memory")
Linus Torvalds1da177e2005-04-16 15:20:36 -0700247
Joe Perches78ff12e2008-03-23 01:01:41 -0700248#define atomic_set_mask(mask, addr) \
249 asm volatile(LOCK_PREFIX "orl %0,%1" \
250 : : "r" (mask), "m" (*(addr)) : "memory")
Linus Torvalds1da177e2005-04-16 15:20:36 -0700251
252/* Atomic operations are already serializing on x86 */
253#define smp_mb__before_atomic_dec() barrier()
254#define smp_mb__after_atomic_dec() barrier()
255#define smp_mb__before_atomic_inc() barrier()
256#define smp_mb__after_atomic_inc() barrier()
257
Christoph Lameterd3cb4872006-01-06 00:11:20 -0800258#include <asm-generic/atomic.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -0700259#endif