blob: c1cf59d72f096e3825010681889eb6625c662d16 [file] [log] [blame]
H. Peter Anvin1965aae2008-10-22 22:26:29 -07001#ifndef _ASM_X86_CMPXCHG_32_H
2#define _ASM_X86_CMPXCHG_32_H
Jeff Dikea436ed92007-05-08 00:35:02 -07003
4#include <linux/bitops.h> /* for LOCK_PREFIX */
5
Avi Kivity2d9ce172007-07-19 14:30:14 +03006/*
7 * Note: if you use set64_bit(), __cmpxchg64(), or their variants, you
8 * you need to test for the feature in boot_cpu_data.
9 */
10
Peter Zijlstraf3834b92009-10-09 10:12:46 +020011extern void __xchg_wrong_size(void);
12
13/*
14 * Note: no "lock" prefix even on SMP: xchg always implies lock anyway
15 * Note 2: xchg has side effect, so that attribute volatile is necessary,
16 * but generally the primitive is invalid, *ptr is output argument. --ANK
17 */
Jeff Dikea436ed92007-05-08 00:35:02 -070018
Joe Perches81210192008-03-23 01:01:51 -070019struct __xchg_dummy {
20 unsigned long a[100];
21};
Jeff Dikea436ed92007-05-08 00:35:02 -070022#define __xg(x) ((struct __xchg_dummy *)(x))
23
Peter Zijlstraf3834b92009-10-09 10:12:46 +020024#define __xchg(x, ptr, size) \
25({ \
26 __typeof(*(ptr)) __x = (x); \
27 switch (size) { \
28 case 1: \
29 asm volatile("xchgb %b0,%1" \
H. Peter Anvin113fc5a2010-07-27 17:01:49 -070030 : "=q" (__x), "+m" (*__xg(ptr)) \
31 : "0" (__x) \
Peter Zijlstraf3834b92009-10-09 10:12:46 +020032 : "memory"); \
33 break; \
34 case 2: \
35 asm volatile("xchgw %w0,%1" \
H. Peter Anvin113fc5a2010-07-27 17:01:49 -070036 : "=r" (__x), "+m" (*__xg(ptr)) \
37 : "0" (__x) \
Peter Zijlstraf3834b92009-10-09 10:12:46 +020038 : "memory"); \
39 break; \
40 case 4: \
41 asm volatile("xchgl %0,%1" \
H. Peter Anvin113fc5a2010-07-27 17:01:49 -070042 : "=r" (__x), "+m" (*__xg(ptr)) \
43 : "0" (__x) \
Peter Zijlstraf3834b92009-10-09 10:12:46 +020044 : "memory"); \
45 break; \
46 default: \
47 __xchg_wrong_size(); \
48 } \
49 __x; \
50})
51
52#define xchg(ptr, v) \
53 __xchg((v), (ptr), sizeof(*ptr))
54
Jeff Dikea436ed92007-05-08 00:35:02 -070055/*
56 * The semantics of XCHGCMP8B are a bit strange, this is why
57 * there is a loop and the loading of %%eax and %%edx has to
58 * be inside. This inlines well in most cases, the cached
59 * cost is around ~38 cycles. (in the future we might want
60 * to do an SIMD/3DNOW!/MMX/FPU 64-bit store here, but that
61 * might have an implicit FPU-save as a cost, so it's not
62 * clear which path to go.)
63 *
64 * cmpxchg8b must be used with the lock prefix here to allow
65 * the instruction to be executed atomically, see page 3-102
66 * of the instruction set reference 24319102.pdf. We need
67 * the reader side to see the coherent 64bit value.
68 */
Joe Perches81210192008-03-23 01:01:51 -070069static inline void __set_64bit(unsigned long long *ptr,
70 unsigned int low, unsigned int high)
Jeff Dikea436ed92007-05-08 00:35:02 -070071{
Joe Perches81210192008-03-23 01:01:51 -070072 asm volatile("\n1:\t"
H. Peter Anvin113fc5a2010-07-27 17:01:49 -070073 "movl (%1), %%eax\n\t"
74 "movl 4(%1), %%edx\n\t"
75 LOCK_PREFIX "cmpxchg8b (%1)\n\t"
Joe Perches81210192008-03-23 01:01:51 -070076 "jnz 1b"
H. Peter Anvin113fc5a2010-07-27 17:01:49 -070077 : "=m" (*ptr)
78 : "D" (ptr),
79 "b" (low),
80 "c" (high)
Joe Perches81210192008-03-23 01:01:51 -070081 : "ax", "dx", "memory");
Jeff Dikea436ed92007-05-08 00:35:02 -070082}
83
Joe Perches81210192008-03-23 01:01:51 -070084static inline void __set_64bit_constant(unsigned long long *ptr,
85 unsigned long long value)
Jeff Dikea436ed92007-05-08 00:35:02 -070086{
Joe Perches81210192008-03-23 01:01:51 -070087 __set_64bit(ptr, (unsigned int)value, (unsigned int)(value >> 32));
Jeff Dikea436ed92007-05-08 00:35:02 -070088}
89
Joe Perches81210192008-03-23 01:01:51 -070090#define ll_low(x) *(((unsigned int *)&(x)) + 0)
91#define ll_high(x) *(((unsigned int *)&(x)) + 1)
Jeff Dikea436ed92007-05-08 00:35:02 -070092
Joe Perches81210192008-03-23 01:01:51 -070093static inline void __set_64bit_var(unsigned long long *ptr,
94 unsigned long long value)
95{
96 __set_64bit(ptr, ll_low(value), ll_high(value));
97}
98
99#define set_64bit(ptr, value) \
100 (__builtin_constant_p((value)) \
101 ? __set_64bit_constant((ptr), (value)) \
102 : __set_64bit_var((ptr), (value)))
103
104#define _set_64bit(ptr, value) \
105 (__builtin_constant_p(value) \
106 ? __set_64bit(ptr, (unsigned int)(value), \
107 (unsigned int)((value) >> 32)) \
108 : __set_64bit(ptr, ll_low((value)), ll_high((value))))
Jeff Dikea436ed92007-05-08 00:35:02 -0700109
Peter Zijlstraf3834b92009-10-09 10:12:46 +0200110extern void __cmpxchg_wrong_size(void);
Jeff Dikea436ed92007-05-08 00:35:02 -0700111
112/*
113 * Atomic compare and exchange. Compare OLD with MEM, if identical,
114 * store NEW in MEM. Return the initial value in MEM. Success is
115 * indicated by comparing RETURN with OLD.
116 */
Peter Zijlstraf3834b92009-10-09 10:12:46 +0200117#define __raw_cmpxchg(ptr, old, new, size, lock) \
118({ \
119 __typeof__(*(ptr)) __ret; \
120 __typeof__(*(ptr)) __old = (old); \
121 __typeof__(*(ptr)) __new = (new); \
122 switch (size) { \
123 case 1: \
H. Peter Anvin113fc5a2010-07-27 17:01:49 -0700124 asm volatile(lock "cmpxchgb %b2,%1" \
125 : "=a" (__ret), "+m" (*__xg(ptr)) \
126 : "q" (__new), "0" (__old) \
Peter Zijlstraf3834b92009-10-09 10:12:46 +0200127 : "memory"); \
128 break; \
129 case 2: \
H. Peter Anvin113fc5a2010-07-27 17:01:49 -0700130 asm volatile(lock "cmpxchgw %w2,%1" \
131 : "=a" (__ret), "+m" (*__xg(ptr)) \
132 : "r" (__new), "0" (__old) \
Peter Zijlstraf3834b92009-10-09 10:12:46 +0200133 : "memory"); \
134 break; \
135 case 4: \
H. Peter Anvin113fc5a2010-07-27 17:01:49 -0700136 asm volatile(lock "cmpxchgl %2,%1" \
137 : "=a" (__ret), "+m" (*__xg(ptr)) \
138 : "r" (__new), "0" (__old) \
Peter Zijlstraf3834b92009-10-09 10:12:46 +0200139 : "memory"); \
140 break; \
141 default: \
142 __cmpxchg_wrong_size(); \
143 } \
144 __ret; \
145})
146
147#define __cmpxchg(ptr, old, new, size) \
148 __raw_cmpxchg((ptr), (old), (new), (size), LOCK_PREFIX)
149
150#define __sync_cmpxchg(ptr, old, new, size) \
151 __raw_cmpxchg((ptr), (old), (new), (size), "lock; ")
152
153#define __cmpxchg_local(ptr, old, new, size) \
154 __raw_cmpxchg((ptr), (old), (new), (size), "")
Jeff Dikea436ed92007-05-08 00:35:02 -0700155
156#ifdef CONFIG_X86_CMPXCHG
157#define __HAVE_ARCH_CMPXCHG 1
Peter Zijlstraf3834b92009-10-09 10:12:46 +0200158
159#define cmpxchg(ptr, old, new) \
160 __cmpxchg((ptr), (old), (new), sizeof(*ptr))
161
162#define sync_cmpxchg(ptr, old, new) \
163 __sync_cmpxchg((ptr), (old), (new), sizeof(*ptr))
164
165#define cmpxchg_local(ptr, old, new) \
166 __cmpxchg_local((ptr), (old), (new), sizeof(*ptr))
Mathieu Desnoyers2c0b8a72008-01-30 13:30:47 +0100167#endif
168
169#ifdef CONFIG_X86_CMPXCHG64
Joe Perches81210192008-03-23 01:01:51 -0700170#define cmpxchg64(ptr, o, n) \
171 ((__typeof__(*(ptr)))__cmpxchg64((ptr), (unsigned long long)(o), \
172 (unsigned long long)(n)))
173#define cmpxchg64_local(ptr, o, n) \
174 ((__typeof__(*(ptr)))__cmpxchg64_local((ptr), (unsigned long long)(o), \
175 (unsigned long long)(n)))
Jeff Dikea436ed92007-05-08 00:35:02 -0700176#endif
177
Mathieu Desnoyers2c0b8a72008-01-30 13:30:47 +0100178static inline unsigned long long __cmpxchg64(volatile void *ptr,
Joe Perches81210192008-03-23 01:01:51 -0700179 unsigned long long old,
180 unsigned long long new)
Jeff Dikea436ed92007-05-08 00:35:02 -0700181{
182 unsigned long long prev;
H. Peter Anvin113fc5a2010-07-27 17:01:49 -0700183 asm volatile(LOCK_PREFIX "cmpxchg8b %1"
184 : "=A" (prev),
185 "+m" (*__xg(ptr))
186 : "b" ((unsigned long)new),
187 "c" ((unsigned long)(new >> 32)),
188 "0" (old)
Joe Perches81210192008-03-23 01:01:51 -0700189 : "memory");
Jeff Dikea436ed92007-05-08 00:35:02 -0700190 return prev;
191}
192
193static inline unsigned long long __cmpxchg64_local(volatile void *ptr,
Joe Perches81210192008-03-23 01:01:51 -0700194 unsigned long long old,
195 unsigned long long new)
Jeff Dikea436ed92007-05-08 00:35:02 -0700196{
197 unsigned long long prev;
H. Peter Anvin113fc5a2010-07-27 17:01:49 -0700198 asm volatile("cmpxchg8b %1"
199 : "=A" (prev),
200 "+m" (*__xg(ptr))
201 : "b" ((unsigned long)new),
202 "c" ((unsigned long)(new >> 32)),
203 "0" (old)
Joe Perches81210192008-03-23 01:01:51 -0700204 : "memory");
Jeff Dikea436ed92007-05-08 00:35:02 -0700205 return prev;
206}
207
Mathieu Desnoyers2c0b8a72008-01-30 13:30:47 +0100208#ifndef CONFIG_X86_CMPXCHG
209/*
210 * Building a kernel capable running on 80386. It may be necessary to
211 * simulate the cmpxchg on the 80386 CPU. For that purpose we define
212 * a function for each of the sizes we support.
213 */
214
215extern unsigned long cmpxchg_386_u8(volatile void *, u8, u8);
216extern unsigned long cmpxchg_386_u16(volatile void *, u16, u16);
217extern unsigned long cmpxchg_386_u32(volatile void *, u32, u32);
218
219static inline unsigned long cmpxchg_386(volatile void *ptr, unsigned long old,
Joe Perches81210192008-03-23 01:01:51 -0700220 unsigned long new, int size)
Mathieu Desnoyers2c0b8a72008-01-30 13:30:47 +0100221{
222 switch (size) {
223 case 1:
224 return cmpxchg_386_u8(ptr, old, new);
225 case 2:
226 return cmpxchg_386_u16(ptr, old, new);
227 case 4:
228 return cmpxchg_386_u32(ptr, old, new);
229 }
230 return old;
231}
232
233#define cmpxchg(ptr, o, n) \
234({ \
235 __typeof__(*(ptr)) __ret; \
236 if (likely(boot_cpu_data.x86 > 3)) \
Mathieu Desnoyers3078b792008-03-06 13:45:46 +0100237 __ret = (__typeof__(*(ptr)))__cmpxchg((ptr), \
238 (unsigned long)(o), (unsigned long)(n), \
239 sizeof(*(ptr))); \
Mathieu Desnoyers2c0b8a72008-01-30 13:30:47 +0100240 else \
Mathieu Desnoyers3078b792008-03-06 13:45:46 +0100241 __ret = (__typeof__(*(ptr)))cmpxchg_386((ptr), \
242 (unsigned long)(o), (unsigned long)(n), \
243 sizeof(*(ptr))); \
Mathieu Desnoyers2c0b8a72008-01-30 13:30:47 +0100244 __ret; \
245})
246#define cmpxchg_local(ptr, o, n) \
247({ \
248 __typeof__(*(ptr)) __ret; \
249 if (likely(boot_cpu_data.x86 > 3)) \
Mathieu Desnoyers3078b792008-03-06 13:45:46 +0100250 __ret = (__typeof__(*(ptr)))__cmpxchg_local((ptr), \
251 (unsigned long)(o), (unsigned long)(n), \
252 sizeof(*(ptr))); \
Mathieu Desnoyers2c0b8a72008-01-30 13:30:47 +0100253 else \
Mathieu Desnoyers3078b792008-03-06 13:45:46 +0100254 __ret = (__typeof__(*(ptr)))cmpxchg_386((ptr), \
255 (unsigned long)(o), (unsigned long)(n), \
256 sizeof(*(ptr))); \
Mathieu Desnoyers2c0b8a72008-01-30 13:30:47 +0100257 __ret; \
258})
259#endif
260
261#ifndef CONFIG_X86_CMPXCHG64
262/*
263 * Building a kernel capable running on 80386 and 80486. It may be necessary
264 * to simulate the cmpxchg8b on the 80386 and 80486 CPU.
265 */
266
267extern unsigned long long cmpxchg_486_u64(volatile void *, u64, u64);
268
Arjan van de Ven79e1dd02009-09-30 17:07:54 +0200269#define cmpxchg64(ptr, o, n) \
270({ \
271 __typeof__(*(ptr)) __ret; \
272 __typeof__(*(ptr)) __old = (o); \
273 __typeof__(*(ptr)) __new = (n); \
Luca Barbieri9c76b382010-02-24 10:54:23 +0100274 alternative_io(LOCK_PREFIX_HERE \
275 "call cmpxchg8b_emu", \
Arjan van de Ven79e1dd02009-09-30 17:07:54 +0200276 "lock; cmpxchg8b (%%esi)" , \
277 X86_FEATURE_CX8, \
278 "=A" (__ret), \
279 "S" ((ptr)), "0" (__old), \
280 "b" ((unsigned int)__new), \
281 "c" ((unsigned int)(__new>>32)) \
282 : "memory"); \
283 __ret; })
284
285
286
Mathieu Desnoyers2c0b8a72008-01-30 13:30:47 +0100287#define cmpxchg64_local(ptr, o, n) \
288({ \
289 __typeof__(*(ptr)) __ret; \
290 if (likely(boot_cpu_data.x86 > 4)) \
Mathieu Desnoyers3078b792008-03-06 13:45:46 +0100291 __ret = (__typeof__(*(ptr)))__cmpxchg64_local((ptr), \
292 (unsigned long long)(o), \
Mathieu Desnoyers2c0b8a72008-01-30 13:30:47 +0100293 (unsigned long long)(n)); \
294 else \
Mathieu Desnoyers3078b792008-03-06 13:45:46 +0100295 __ret = (__typeof__(*(ptr)))cmpxchg_486_u64((ptr), \
296 (unsigned long long)(o), \
Mathieu Desnoyers2c0b8a72008-01-30 13:30:47 +0100297 (unsigned long long)(n)); \
298 __ret; \
299})
300
301#endif
302
H. Peter Anvin1965aae2008-10-22 22:26:29 -0700303#endif /* _ASM_X86_CMPXCHG_32_H */