blob: 3b59f94b5a3d3f345bf5bfeff53440063ae3f426 [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
2 * linux/include/asm-arm/atomic.h
3 *
4 * Copyright (C) 1996 Russell King.
5 * Copyright (C) 2002 Deep Blue Solutions Ltd.
6 *
7 * This program is free software; you can redistribute it and/or modify
8 * it under the terms of the GNU General Public License version 2 as
9 * published by the Free Software Foundation.
10 */
11#ifndef __ASM_ARM_ATOMIC_H
12#define __ASM_ARM_ATOMIC_H
13
Russell King8dc39b82005-11-16 17:23:57 +000014#include <linux/compiler.h>
Mathieu Desnoyers2856f5e2007-05-08 00:34:38 -070015#include <asm/system.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070016
17typedef struct { volatile int counter; } atomic_t;
18
19#define ATOMIC_INIT(i) { (i) }
20
21#ifdef __KERNEL__
22
23#define atomic_read(v) ((v)->counter)
24
25#if __LINUX_ARM_ARCH__ >= 6
26
27/*
28 * ARMv6 UP and SMP safe atomic ops. We use load exclusive and
29 * store exclusive to ensure that these are atomic. We may loop
30 * to ensure that the update happens. Writing to 'v->counter'
31 * without using the following operations WILL break the atomic
32 * nature of these ops.
33 */
34static inline void atomic_set(atomic_t *v, int i)
35{
36 unsigned long tmp;
37
38 __asm__ __volatile__("@ atomic_set\n"
39"1: ldrex %0, [%1]\n"
40" strex %0, %2, [%1]\n"
41" teq %0, #0\n"
42" bne 1b"
43 : "=&r" (tmp)
44 : "r" (&v->counter), "r" (i)
45 : "cc");
46}
47
48static inline int atomic_add_return(int i, atomic_t *v)
49{
50 unsigned long tmp;
51 int result;
52
53 __asm__ __volatile__("@ atomic_add_return\n"
54"1: ldrex %0, [%2]\n"
55" add %0, %0, %3\n"
56" strex %1, %0, [%2]\n"
57" teq %1, #0\n"
58" bne 1b"
59 : "=&r" (result), "=&r" (tmp)
60 : "r" (&v->counter), "Ir" (i)
61 : "cc");
62
63 return result;
64}
65
66static inline int atomic_sub_return(int i, atomic_t *v)
67{
68 unsigned long tmp;
69 int result;
70
71 __asm__ __volatile__("@ atomic_sub_return\n"
72"1: ldrex %0, [%2]\n"
73" sub %0, %0, %3\n"
74" strex %1, %0, [%2]\n"
75" teq %1, #0\n"
76" bne 1b"
77 : "=&r" (result), "=&r" (tmp)
78 : "r" (&v->counter), "Ir" (i)
79 : "cc");
80
81 return result;
82}
83
Nick Piggin4a6dae62005-11-13 16:07:24 -080084static inline int atomic_cmpxchg(atomic_t *ptr, int old, int new)
85{
Russell King49ee57a2005-11-16 18:03:10 +000086 unsigned long oldval, res;
Nick Piggin4a6dae62005-11-13 16:07:24 -080087
88 do {
89 __asm__ __volatile__("@ atomic_cmpxchg\n"
90 "ldrex %1, [%2]\n"
Nicolas Pitrea7d06832005-11-16 15:05:11 +000091 "mov %0, #0\n"
Nick Piggin4a6dae62005-11-13 16:07:24 -080092 "teq %1, %3\n"
93 "strexeq %0, %4, [%2]\n"
94 : "=&r" (res), "=&r" (oldval)
95 : "r" (&ptr->counter), "Ir" (old), "r" (new)
96 : "cc");
97 } while (res);
98
99 return oldval;
100}
101
Linus Torvalds1da177e2005-04-16 15:20:36 -0700102static inline void atomic_clear_mask(unsigned long mask, unsigned long *addr)
103{
104 unsigned long tmp, tmp2;
105
106 __asm__ __volatile__("@ atomic_clear_mask\n"
Stelian Pop0803c302007-03-15 16:54:27 +0100107"1: ldrex %0, [%2]\n"
Linus Torvalds1da177e2005-04-16 15:20:36 -0700108" bic %0, %0, %3\n"
Stelian Pop0803c302007-03-15 16:54:27 +0100109" strex %1, %0, [%2]\n"
Linus Torvalds1da177e2005-04-16 15:20:36 -0700110" teq %1, #0\n"
111" bne 1b"
112 : "=&r" (tmp), "=&r" (tmp2)
113 : "r" (addr), "Ir" (mask)
114 : "cc");
115}
116
117#else /* ARM_ARCH_6 */
118
119#include <asm/system.h>
120
121#ifdef CONFIG_SMP
122#error SMP not supported on pre-ARMv6 CPUs
123#endif
124
125#define atomic_set(v,i) (((v)->counter) = (i))
126
127static inline int atomic_add_return(int i, atomic_t *v)
128{
129 unsigned long flags;
130 int val;
131
Lennert Buytenhek8dd5c842006-09-16 10:47:18 +0100132 raw_local_irq_save(flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700133 val = v->counter;
134 v->counter = val += i;
Lennert Buytenhek8dd5c842006-09-16 10:47:18 +0100135 raw_local_irq_restore(flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700136
137 return val;
138}
139
140static inline int atomic_sub_return(int i, atomic_t *v)
141{
142 unsigned long flags;
143 int val;
144
Lennert Buytenhek8dd5c842006-09-16 10:47:18 +0100145 raw_local_irq_save(flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700146 val = v->counter;
147 v->counter = val -= i;
Lennert Buytenhek8dd5c842006-09-16 10:47:18 +0100148 raw_local_irq_restore(flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700149
150 return val;
151}
152
Nick Piggin4a6dae62005-11-13 16:07:24 -0800153static inline int atomic_cmpxchg(atomic_t *v, int old, int new)
154{
155 int ret;
156 unsigned long flags;
157
Lennert Buytenhek8dd5c842006-09-16 10:47:18 +0100158 raw_local_irq_save(flags);
Nick Piggin4a6dae62005-11-13 16:07:24 -0800159 ret = v->counter;
160 if (likely(ret == old))
161 v->counter = new;
Lennert Buytenhek8dd5c842006-09-16 10:47:18 +0100162 raw_local_irq_restore(flags);
Nick Piggin4a6dae62005-11-13 16:07:24 -0800163
164 return ret;
165}
166
Linus Torvalds1da177e2005-04-16 15:20:36 -0700167static inline void atomic_clear_mask(unsigned long mask, unsigned long *addr)
168{
169 unsigned long flags;
170
Lennert Buytenhek8dd5c842006-09-16 10:47:18 +0100171 raw_local_irq_save(flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700172 *addr &= ~mask;
Lennert Buytenhek8dd5c842006-09-16 10:47:18 +0100173 raw_local_irq_restore(flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700174}
175
176#endif /* __LINUX_ARM_ARCH__ */
177
Ingo Molnarffbf6702006-01-09 15:59:17 -0800178#define atomic_xchg(v, new) (xchg(&((v)->counter), new))
179
Nick Piggin8426e1f2005-11-13 16:07:25 -0800180static inline int atomic_add_unless(atomic_t *v, int a, int u)
181{
182 int c, old;
183
184 c = atomic_read(v);
185 while (c != u && (old = atomic_cmpxchg((v), c, c + a)) != c)
186 c = old;
187 return c != u;
188}
189#define atomic_inc_not_zero(v) atomic_add_unless((v), 1, 0)
190
Linus Torvalds1da177e2005-04-16 15:20:36 -0700191#define atomic_add(i, v) (void) atomic_add_return(i, v)
192#define atomic_inc(v) (void) atomic_add_return(1, v)
193#define atomic_sub(i, v) (void) atomic_sub_return(i, v)
194#define atomic_dec(v) (void) atomic_sub_return(1, v)
195
196#define atomic_inc_and_test(v) (atomic_add_return(1, v) == 0)
197#define atomic_dec_and_test(v) (atomic_sub_return(1, v) == 0)
198#define atomic_inc_return(v) (atomic_add_return(1, v))
199#define atomic_dec_return(v) (atomic_sub_return(1, v))
200#define atomic_sub_and_test(i, v) (atomic_sub_return(i, v) == 0)
201
202#define atomic_add_negative(i,v) (atomic_add_return(i, v) < 0)
203
204/* Atomic operations are already serializing on ARM */
205#define smp_mb__before_atomic_dec() barrier()
206#define smp_mb__after_atomic_dec() barrier()
207#define smp_mb__before_atomic_inc() barrier()
208#define smp_mb__after_atomic_inc() barrier()
209
Christoph Lameterd3cb4872006-01-06 00:11:20 -0800210#include <asm-generic/atomic.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -0700211#endif
212#endif