Heiko Carstens | 1275105 | 2009-09-11 10:28:34 +0200 | [diff] [blame] | 1 | /* |
Heiko Carstens | a53c8fa | 2012-07-20 11:15:04 +0200 | [diff] [blame] | 2 | * Copyright IBM Corp. 1999, 2009 |
Heiko Carstens | 1275105 | 2009-09-11 10:28:34 +0200 | [diff] [blame] | 3 | * Author(s): Martin Schwidefsky <schwidefsky@de.ibm.com>, |
| 4 | * Denis Joseph Barrow, |
| 5 | * Arnd Bergmann <arndb@de.ibm.com>, |
| 6 | * |
| 7 | * Atomic operations that C can't guarantee us. |
| 8 | * Useful for resource counting etc. |
Lucas De Marchi | 25985ed | 2011-03-30 22:57:33 -0300 | [diff] [blame] | 9 | * s390 uses 'Compare And Swap' for atomicity in SMP environment. |
Heiko Carstens | 1275105 | 2009-09-11 10:28:34 +0200 | [diff] [blame] | 10 | * |
| 11 | */ |
| 12 | |
Heiko Carstens | a53c8fa | 2012-07-20 11:15:04 +0200 | [diff] [blame] | 13 | #ifndef __ARCH_S390_ATOMIC__ |
| 14 | #define __ARCH_S390_ATOMIC__ |
| 15 | |
Dave Jones | 5bd1db6 | 2006-04-10 22:53:51 -0700 | [diff] [blame] | 16 | #include <linux/compiler.h> |
Matthew Wilcox | ea435467 | 2009-01-06 14:40:39 -0800 | [diff] [blame] | 17 | #include <linux/types.h> |
Heiko Carstens | 0ccc8b7 | 2014-03-20 08:55:00 +0100 | [diff] [blame] | 18 | #include <asm/barrier.h> |
David Howells | a0616cd | 2012-03-28 18:30:02 +0100 | [diff] [blame] | 19 | #include <asm/cmpxchg.h> |
Dave Jones | 5bd1db6 | 2006-04-10 22:53:51 -0700 | [diff] [blame] | 20 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 21 | #define ATOMIC_INIT(i) { (i) } |
| 22 | |
Heiko Carstens | 0ccc8b7 | 2014-03-20 08:55:00 +0100 | [diff] [blame] | 23 | #define __ATOMIC_NO_BARRIER "\n" |
| 24 | |
Heiko Carstens | 7528743 | 2013-09-11 16:34:04 +0200 | [diff] [blame] | 25 | #ifdef CONFIG_HAVE_MARCH_Z196_FEATURES |
| 26 | |
| 27 | #define __ATOMIC_OR "lao" |
| 28 | #define __ATOMIC_AND "lan" |
| 29 | #define __ATOMIC_ADD "laa" |
Peter Zijlstra | ae8c35c | 2014-04-23 20:15:48 +0200 | [diff] [blame] | 30 | #define __ATOMIC_XOR "lax" |
Heiko Carstens | 0ccc8b7 | 2014-03-20 08:55:00 +0100 | [diff] [blame] | 31 | #define __ATOMIC_BARRIER "bcr 14,0\n" |
Heiko Carstens | 7528743 | 2013-09-11 16:34:04 +0200 | [diff] [blame] | 32 | |
Heiko Carstens | 0ccc8b7 | 2014-03-20 08:55:00 +0100 | [diff] [blame] | 33 | #define __ATOMIC_LOOP(ptr, op_val, op_string, __barrier) \ |
Heiko Carstens | 7528743 | 2013-09-11 16:34:04 +0200 | [diff] [blame] | 34 | ({ \ |
| 35 | int old_val; \ |
Heiko Carstens | 9a70a42 | 2013-09-16 11:07:49 +0200 | [diff] [blame] | 36 | \ |
| 37 | typecheck(atomic_t *, ptr); \ |
Heiko Carstens | 7528743 | 2013-09-11 16:34:04 +0200 | [diff] [blame] | 38 | asm volatile( \ |
| 39 | op_string " %0,%2,%1\n" \ |
Heiko Carstens | 0ccc8b7 | 2014-03-20 08:55:00 +0100 | [diff] [blame] | 40 | __barrier \ |
Heiko Carstens | 9a70a42 | 2013-09-16 11:07:49 +0200 | [diff] [blame] | 41 | : "=d" (old_val), "+Q" ((ptr)->counter) \ |
Heiko Carstens | 7528743 | 2013-09-11 16:34:04 +0200 | [diff] [blame] | 42 | : "d" (op_val) \ |
| 43 | : "cc", "memory"); \ |
| 44 | old_val; \ |
| 45 | }) |
| 46 | |
| 47 | #else /* CONFIG_HAVE_MARCH_Z196_FEATURES */ |
| 48 | |
| 49 | #define __ATOMIC_OR "or" |
| 50 | #define __ATOMIC_AND "nr" |
| 51 | #define __ATOMIC_ADD "ar" |
Peter Zijlstra | ae8c35c | 2014-04-23 20:15:48 +0200 | [diff] [blame] | 52 | #define __ATOMIC_XOR "xr" |
Heiko Carstens | 0ccc8b7 | 2014-03-20 08:55:00 +0100 | [diff] [blame] | 53 | #define __ATOMIC_BARRIER "\n" |
Heiko Carstens | 7528743 | 2013-09-11 16:34:04 +0200 | [diff] [blame] | 54 | |
Heiko Carstens | 0ccc8b7 | 2014-03-20 08:55:00 +0100 | [diff] [blame] | 55 | #define __ATOMIC_LOOP(ptr, op_val, op_string, __barrier) \ |
Heiko Carstens | 7528743 | 2013-09-11 16:34:04 +0200 | [diff] [blame] | 56 | ({ \ |
Martin Schwidefsky | 3947517 | 2009-12-07 12:52:05 +0100 | [diff] [blame] | 57 | int old_val, new_val; \ |
Heiko Carstens | 9a70a42 | 2013-09-16 11:07:49 +0200 | [diff] [blame] | 58 | \ |
| 59 | typecheck(atomic_t *, ptr); \ |
Martin Schwidefsky | 94c12cc | 2006-09-28 16:56:43 +0200 | [diff] [blame] | 60 | asm volatile( \ |
| 61 | " l %0,%2\n" \ |
| 62 | "0: lr %1,%0\n" \ |
| 63 | op_string " %1,%3\n" \ |
| 64 | " cs %0,%1,%2\n" \ |
| 65 | " jl 0b" \ |
Heiko Carstens | 9a70a42 | 2013-09-16 11:07:49 +0200 | [diff] [blame] | 66 | : "=&d" (old_val), "=&d" (new_val), "+Q" ((ptr)->counter)\ |
| 67 | : "d" (op_val) \ |
Martin Schwidefsky | 94c12cc | 2006-09-28 16:56:43 +0200 | [diff] [blame] | 68 | : "cc", "memory"); \ |
Heiko Carstens | 7528743 | 2013-09-11 16:34:04 +0200 | [diff] [blame] | 69 | old_val; \ |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 70 | }) |
Martin Schwidefsky | 94c12cc | 2006-09-28 16:56:43 +0200 | [diff] [blame] | 71 | |
Heiko Carstens | 7528743 | 2013-09-11 16:34:04 +0200 | [diff] [blame] | 72 | #endif /* CONFIG_HAVE_MARCH_Z196_FEATURES */ |
| 73 | |
Heiko Carstens | c51b962 | 2007-08-22 13:51:45 +0200 | [diff] [blame] | 74 | static inline int atomic_read(const atomic_t *v) |
| 75 | { |
Heiko Carstens | 7657e41 | 2011-02-17 13:13:58 +0100 | [diff] [blame] | 76 | int c; |
| 77 | |
| 78 | asm volatile( |
| 79 | " l %0,%1\n" |
| 80 | : "=d" (c) : "Q" (v->counter)); |
| 81 | return c; |
Heiko Carstens | c51b962 | 2007-08-22 13:51:45 +0200 | [diff] [blame] | 82 | } |
| 83 | |
| 84 | static inline void atomic_set(atomic_t *v, int i) |
| 85 | { |
Heiko Carstens | 7657e41 | 2011-02-17 13:13:58 +0100 | [diff] [blame] | 86 | asm volatile( |
| 87 | " st %1,%0\n" |
| 88 | : "=Q" (v->counter) : "d" (i)); |
Heiko Carstens | c51b962 | 2007-08-22 13:51:45 +0200 | [diff] [blame] | 89 | } |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 90 | |
Heiko Carstens | bfe3349b | 2009-09-11 10:28:35 +0200 | [diff] [blame] | 91 | static inline int atomic_add_return(int i, atomic_t *v) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 92 | { |
Heiko Carstens | 0ccc8b7 | 2014-03-20 08:55:00 +0100 | [diff] [blame] | 93 | return __ATOMIC_LOOP(v, i, __ATOMIC_ADD, __ATOMIC_BARRIER) + i; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 94 | } |
Heiko Carstens | 7528743 | 2013-09-11 16:34:04 +0200 | [diff] [blame] | 95 | |
Peter Zijlstra | 56fefbb | 2016-04-18 01:16:05 +0200 | [diff] [blame] | 96 | static inline int atomic_fetch_add(int i, atomic_t *v) |
| 97 | { |
| 98 | return __ATOMIC_LOOP(v, i, __ATOMIC_ADD, __ATOMIC_BARRIER); |
| 99 | } |
| 100 | |
Heiko Carstens | 5692e4d | 2013-09-16 10:44:54 +0200 | [diff] [blame] | 101 | static inline void atomic_add(int i, atomic_t *v) |
| 102 | { |
| 103 | #ifdef CONFIG_HAVE_MARCH_Z196_FEATURES |
| 104 | if (__builtin_constant_p(i) && (i > -129) && (i < 128)) { |
| 105 | asm volatile( |
| 106 | "asi %0,%1\n" |
| 107 | : "+Q" (v->counter) |
| 108 | : "i" (i) |
| 109 | : "cc", "memory"); |
Heiko Carstens | 0ccc8b7 | 2014-03-20 08:55:00 +0100 | [diff] [blame] | 110 | return; |
Heiko Carstens | 5692e4d | 2013-09-16 10:44:54 +0200 | [diff] [blame] | 111 | } |
Heiko Carstens | 5692e4d | 2013-09-16 10:44:54 +0200 | [diff] [blame] | 112 | #endif |
Heiko Carstens | 0ccc8b7 | 2014-03-20 08:55:00 +0100 | [diff] [blame] | 113 | __ATOMIC_LOOP(v, i, __ATOMIC_ADD, __ATOMIC_NO_BARRIER); |
Heiko Carstens | 5692e4d | 2013-09-16 10:44:54 +0200 | [diff] [blame] | 114 | } |
| 115 | |
Martin Schwidefsky | 973bd99 | 2006-01-06 00:19:07 -0800 | [diff] [blame] | 116 | #define atomic_add_negative(_i, _v) (atomic_add_return(_i, _v) < 0) |
Heiko Carstens | 5692e4d | 2013-09-16 10:44:54 +0200 | [diff] [blame] | 117 | #define atomic_inc(_v) atomic_add(1, _v) |
Martin Schwidefsky | 973bd99 | 2006-01-06 00:19:07 -0800 | [diff] [blame] | 118 | #define atomic_inc_return(_v) atomic_add_return(1, _v) |
| 119 | #define atomic_inc_and_test(_v) (atomic_add_return(1, _v) == 0) |
Heiko Carstens | 5692e4d | 2013-09-16 10:44:54 +0200 | [diff] [blame] | 120 | #define atomic_sub(_i, _v) atomic_add(-(int)(_i), _v) |
Heiko Carstens | 86d51bc | 2013-09-11 14:28:47 +0200 | [diff] [blame] | 121 | #define atomic_sub_return(_i, _v) atomic_add_return(-(int)(_i), _v) |
Peter Zijlstra | 56fefbb | 2016-04-18 01:16:05 +0200 | [diff] [blame] | 122 | #define atomic_fetch_sub(_i, _v) atomic_fetch_add(-(int)(_i), _v) |
Martin Schwidefsky | 973bd99 | 2006-01-06 00:19:07 -0800 | [diff] [blame] | 123 | #define atomic_sub_and_test(_i, _v) (atomic_sub_return(_i, _v) == 0) |
Heiko Carstens | 5692e4d | 2013-09-16 10:44:54 +0200 | [diff] [blame] | 124 | #define atomic_dec(_v) atomic_sub(1, _v) |
Martin Schwidefsky | 973bd99 | 2006-01-06 00:19:07 -0800 | [diff] [blame] | 125 | #define atomic_dec_return(_v) atomic_sub_return(1, _v) |
| 126 | #define atomic_dec_and_test(_v) (atomic_sub_return(1, _v) == 0) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 127 | |
Peter Zijlstra | 56fefbb | 2016-04-18 01:16:05 +0200 | [diff] [blame] | 128 | #define ATOMIC_OPS(op, OP) \ |
Peter Zijlstra | ae8c35c | 2014-04-23 20:15:48 +0200 | [diff] [blame] | 129 | static inline void atomic_##op(int i, atomic_t *v) \ |
| 130 | { \ |
| 131 | __ATOMIC_LOOP(v, i, __ATOMIC_##OP, __ATOMIC_NO_BARRIER); \ |
Peter Zijlstra | 56fefbb | 2016-04-18 01:16:05 +0200 | [diff] [blame] | 132 | } \ |
| 133 | static inline int atomic_fetch_##op(int i, atomic_t *v) \ |
| 134 | { \ |
| 135 | return __ATOMIC_LOOP(v, i, __ATOMIC_##OP, __ATOMIC_BARRIER); \ |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 136 | } |
Martin Schwidefsky | 973bd99 | 2006-01-06 00:19:07 -0800 | [diff] [blame] | 137 | |
Peter Zijlstra | 56fefbb | 2016-04-18 01:16:05 +0200 | [diff] [blame] | 138 | ATOMIC_OPS(and, AND) |
| 139 | ATOMIC_OPS(or, OR) |
| 140 | ATOMIC_OPS(xor, XOR) |
| 141 | |
| 142 | #undef ATOMIC_OPS |
Peter Zijlstra | ae8c35c | 2014-04-23 20:15:48 +0200 | [diff] [blame] | 143 | |
Ingo Molnar | ffbf670 | 2006-01-09 15:59:17 -0800 | [diff] [blame] | 144 | #define atomic_xchg(v, new) (xchg(&((v)->counter), new)) |
| 145 | |
Heiko Carstens | bfe3349b | 2009-09-11 10:28:35 +0200 | [diff] [blame] | 146 | static inline int atomic_cmpxchg(atomic_t *v, int old, int new) |
Martin Schwidefsky | 973bd99 | 2006-01-06 00:19:07 -0800 | [diff] [blame] | 147 | { |
Martin Schwidefsky | 94c12cc | 2006-09-28 16:56:43 +0200 | [diff] [blame] | 148 | asm volatile( |
| 149 | " cs %0,%2,%1" |
Heiko Carstens | 9a70a42 | 2013-09-16 11:07:49 +0200 | [diff] [blame] | 150 | : "+d" (old), "+Q" (v->counter) |
| 151 | : "d" (new) |
Martin Schwidefsky | 94c12cc | 2006-09-28 16:56:43 +0200 | [diff] [blame] | 152 | : "cc", "memory"); |
Martin Schwidefsky | 973bd99 | 2006-01-06 00:19:07 -0800 | [diff] [blame] | 153 | return old; |
| 154 | } |
| 155 | |
Arun Sharma | f24219b | 2011-07-26 16:09:07 -0700 | [diff] [blame] | 156 | static inline int __atomic_add_unless(atomic_t *v, int a, int u) |
Martin Schwidefsky | 973bd99 | 2006-01-06 00:19:07 -0800 | [diff] [blame] | 157 | { |
| 158 | int c, old; |
Martin Schwidefsky | 973bd99 | 2006-01-06 00:19:07 -0800 | [diff] [blame] | 159 | c = atomic_read(v); |
Nick Piggin | 0b2fcfd | 2006-03-23 03:01:02 -0800 | [diff] [blame] | 160 | for (;;) { |
| 161 | if (unlikely(c == u)) |
| 162 | break; |
| 163 | old = atomic_cmpxchg(v, c, c + a); |
| 164 | if (likely(old == c)) |
| 165 | break; |
Martin Schwidefsky | 973bd99 | 2006-01-06 00:19:07 -0800 | [diff] [blame] | 166 | c = old; |
Nick Piggin | 0b2fcfd | 2006-03-23 03:01:02 -0800 | [diff] [blame] | 167 | } |
Arun Sharma | f24219b | 2011-07-26 16:09:07 -0700 | [diff] [blame] | 168 | return c; |
Martin Schwidefsky | 973bd99 | 2006-01-06 00:19:07 -0800 | [diff] [blame] | 169 | } |
| 170 | |
Martin Schwidefsky | 973bd99 | 2006-01-06 00:19:07 -0800 | [diff] [blame] | 171 | |
Heiko Carstens | 7528743 | 2013-09-11 16:34:04 +0200 | [diff] [blame] | 172 | #undef __ATOMIC_LOOP |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 173 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 174 | #define ATOMIC64_INIT(i) { (i) } |
| 175 | |
Heiko Carstens | 0ccc8b7 | 2014-03-20 08:55:00 +0100 | [diff] [blame] | 176 | #define __ATOMIC64_NO_BARRIER "\n" |
| 177 | |
Heiko Carstens | 7528743 | 2013-09-11 16:34:04 +0200 | [diff] [blame] | 178 | #ifdef CONFIG_HAVE_MARCH_Z196_FEATURES |
| 179 | |
| 180 | #define __ATOMIC64_OR "laog" |
| 181 | #define __ATOMIC64_AND "lang" |
| 182 | #define __ATOMIC64_ADD "laag" |
Peter Zijlstra | ae8c35c | 2014-04-23 20:15:48 +0200 | [diff] [blame] | 183 | #define __ATOMIC64_XOR "laxg" |
Heiko Carstens | 0ccc8b7 | 2014-03-20 08:55:00 +0100 | [diff] [blame] | 184 | #define __ATOMIC64_BARRIER "bcr 14,0\n" |
Heiko Carstens | 7528743 | 2013-09-11 16:34:04 +0200 | [diff] [blame] | 185 | |
Heiko Carstens | 0ccc8b7 | 2014-03-20 08:55:00 +0100 | [diff] [blame] | 186 | #define __ATOMIC64_LOOP(ptr, op_val, op_string, __barrier) \ |
Heiko Carstens | 7528743 | 2013-09-11 16:34:04 +0200 | [diff] [blame] | 187 | ({ \ |
| 188 | long long old_val; \ |
Heiko Carstens | 9a70a42 | 2013-09-16 11:07:49 +0200 | [diff] [blame] | 189 | \ |
| 190 | typecheck(atomic64_t *, ptr); \ |
Heiko Carstens | 7528743 | 2013-09-11 16:34:04 +0200 | [diff] [blame] | 191 | asm volatile( \ |
| 192 | op_string " %0,%2,%1\n" \ |
Heiko Carstens | 0ccc8b7 | 2014-03-20 08:55:00 +0100 | [diff] [blame] | 193 | __barrier \ |
Heiko Carstens | 9a70a42 | 2013-09-16 11:07:49 +0200 | [diff] [blame] | 194 | : "=d" (old_val), "+Q" ((ptr)->counter) \ |
Heiko Carstens | 7528743 | 2013-09-11 16:34:04 +0200 | [diff] [blame] | 195 | : "d" (op_val) \ |
| 196 | : "cc", "memory"); \ |
| 197 | old_val; \ |
| 198 | }) |
| 199 | |
| 200 | #else /* CONFIG_HAVE_MARCH_Z196_FEATURES */ |
| 201 | |
| 202 | #define __ATOMIC64_OR "ogr" |
| 203 | #define __ATOMIC64_AND "ngr" |
| 204 | #define __ATOMIC64_ADD "agr" |
Peter Zijlstra | ae8c35c | 2014-04-23 20:15:48 +0200 | [diff] [blame] | 205 | #define __ATOMIC64_XOR "xgr" |
Heiko Carstens | 0ccc8b7 | 2014-03-20 08:55:00 +0100 | [diff] [blame] | 206 | #define __ATOMIC64_BARRIER "\n" |
Heiko Carstens | 7528743 | 2013-09-11 16:34:04 +0200 | [diff] [blame] | 207 | |
Heiko Carstens | 0ccc8b7 | 2014-03-20 08:55:00 +0100 | [diff] [blame] | 208 | #define __ATOMIC64_LOOP(ptr, op_val, op_string, __barrier) \ |
Heiko Carstens | 7528743 | 2013-09-11 16:34:04 +0200 | [diff] [blame] | 209 | ({ \ |
Martin Schwidefsky | 3947517 | 2009-12-07 12:52:05 +0100 | [diff] [blame] | 210 | long long old_val, new_val; \ |
Heiko Carstens | 9a70a42 | 2013-09-16 11:07:49 +0200 | [diff] [blame] | 211 | \ |
| 212 | typecheck(atomic64_t *, ptr); \ |
Martin Schwidefsky | 94c12cc | 2006-09-28 16:56:43 +0200 | [diff] [blame] | 213 | asm volatile( \ |
| 214 | " lg %0,%2\n" \ |
| 215 | "0: lgr %1,%0\n" \ |
| 216 | op_string " %1,%3\n" \ |
| 217 | " csg %0,%1,%2\n" \ |
| 218 | " jl 0b" \ |
Heiko Carstens | 9a70a42 | 2013-09-16 11:07:49 +0200 | [diff] [blame] | 219 | : "=&d" (old_val), "=&d" (new_val), "+Q" ((ptr)->counter)\ |
| 220 | : "d" (op_val) \ |
Heiko Carstens | bfe3349b | 2009-09-11 10:28:35 +0200 | [diff] [blame] | 221 | : "cc", "memory"); \ |
Heiko Carstens | 7528743 | 2013-09-11 16:34:04 +0200 | [diff] [blame] | 222 | old_val; \ |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 223 | }) |
Martin Schwidefsky | 94c12cc | 2006-09-28 16:56:43 +0200 | [diff] [blame] | 224 | |
Heiko Carstens | 7528743 | 2013-09-11 16:34:04 +0200 | [diff] [blame] | 225 | #endif /* CONFIG_HAVE_MARCH_Z196_FEATURES */ |
| 226 | |
Heiko Carstens | c51b962 | 2007-08-22 13:51:45 +0200 | [diff] [blame] | 227 | static inline long long atomic64_read(const atomic64_t *v) |
| 228 | { |
Heiko Carstens | 7657e41 | 2011-02-17 13:13:58 +0100 | [diff] [blame] | 229 | long long c; |
| 230 | |
| 231 | asm volatile( |
| 232 | " lg %0,%1\n" |
| 233 | : "=d" (c) : "Q" (v->counter)); |
| 234 | return c; |
Heiko Carstens | c51b962 | 2007-08-22 13:51:45 +0200 | [diff] [blame] | 235 | } |
| 236 | |
| 237 | static inline void atomic64_set(atomic64_t *v, long long i) |
| 238 | { |
Heiko Carstens | 7657e41 | 2011-02-17 13:13:58 +0100 | [diff] [blame] | 239 | asm volatile( |
| 240 | " stg %1,%0\n" |
| 241 | : "=Q" (v->counter) : "d" (i)); |
Heiko Carstens | c51b962 | 2007-08-22 13:51:45 +0200 | [diff] [blame] | 242 | } |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 243 | |
Heiko Carstens | bfe3349b | 2009-09-11 10:28:35 +0200 | [diff] [blame] | 244 | static inline long long atomic64_add_return(long long i, atomic64_t *v) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 245 | { |
Heiko Carstens | 0ccc8b7 | 2014-03-20 08:55:00 +0100 | [diff] [blame] | 246 | return __ATOMIC64_LOOP(v, i, __ATOMIC64_ADD, __ATOMIC64_BARRIER) + i; |
| 247 | } |
| 248 | |
Peter Zijlstra | 56fefbb | 2016-04-18 01:16:05 +0200 | [diff] [blame] | 249 | static inline long long atomic64_fetch_add(long long i, atomic64_t *v) |
| 250 | { |
| 251 | return __ATOMIC64_LOOP(v, i, __ATOMIC64_ADD, __ATOMIC64_BARRIER); |
| 252 | } |
| 253 | |
Heiko Carstens | 0ccc8b7 | 2014-03-20 08:55:00 +0100 | [diff] [blame] | 254 | static inline void atomic64_add(long long i, atomic64_t *v) |
| 255 | { |
| 256 | #ifdef CONFIG_HAVE_MARCH_Z196_FEATURES |
| 257 | if (__builtin_constant_p(i) && (i > -129) && (i < 128)) { |
| 258 | asm volatile( |
| 259 | "agsi %0,%1\n" |
| 260 | : "+Q" (v->counter) |
| 261 | : "i" (i) |
| 262 | : "cc", "memory"); |
| 263 | return; |
| 264 | } |
| 265 | #endif |
| 266 | __ATOMIC64_LOOP(v, i, __ATOMIC64_ADD, __ATOMIC64_NO_BARRIER); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 267 | } |
Martin Schwidefsky | 973bd99 | 2006-01-06 00:19:07 -0800 | [diff] [blame] | 268 | |
Mathieu Desnoyers | 3a5f10e | 2007-02-21 10:55:59 +0100 | [diff] [blame] | 269 | #define atomic64_xchg(v, new) (xchg(&((v)->counter), new)) |
| 270 | |
Heiko Carstens | bfe3349b | 2009-09-11 10:28:35 +0200 | [diff] [blame] | 271 | static inline long long atomic64_cmpxchg(atomic64_t *v, |
Martin Schwidefsky | 973bd99 | 2006-01-06 00:19:07 -0800 | [diff] [blame] | 272 | long long old, long long new) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 273 | { |
Martin Schwidefsky | 94c12cc | 2006-09-28 16:56:43 +0200 | [diff] [blame] | 274 | asm volatile( |
| 275 | " csg %0,%2,%1" |
Heiko Carstens | 9a70a42 | 2013-09-16 11:07:49 +0200 | [diff] [blame] | 276 | : "+d" (old), "+Q" (v->counter) |
| 277 | : "d" (new) |
Martin Schwidefsky | 94c12cc | 2006-09-28 16:56:43 +0200 | [diff] [blame] | 278 | : "cc", "memory"); |
Martin Schwidefsky | 973bd99 | 2006-01-06 00:19:07 -0800 | [diff] [blame] | 279 | return old; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 280 | } |
| 281 | |
Peter Zijlstra | 56fefbb | 2016-04-18 01:16:05 +0200 | [diff] [blame] | 282 | #define ATOMIC64_OPS(op, OP) \ |
Peter Zijlstra | ae8c35c | 2014-04-23 20:15:48 +0200 | [diff] [blame] | 283 | static inline void atomic64_##op(long i, atomic64_t *v) \ |
| 284 | { \ |
| 285 | __ATOMIC64_LOOP(v, i, __ATOMIC64_##OP, __ATOMIC64_NO_BARRIER); \ |
Peter Zijlstra | 56fefbb | 2016-04-18 01:16:05 +0200 | [diff] [blame] | 286 | } \ |
| 287 | static inline long atomic64_fetch_##op(long i, atomic64_t *v) \ |
| 288 | { \ |
| 289 | return __ATOMIC64_LOOP(v, i, __ATOMIC64_##OP, __ATOMIC64_BARRIER); \ |
Peter Zijlstra | ae8c35c | 2014-04-23 20:15:48 +0200 | [diff] [blame] | 290 | } |
| 291 | |
Peter Zijlstra | 56fefbb | 2016-04-18 01:16:05 +0200 | [diff] [blame] | 292 | ATOMIC64_OPS(and, AND) |
| 293 | ATOMIC64_OPS(or, OR) |
| 294 | ATOMIC64_OPS(xor, XOR) |
Peter Zijlstra | ae8c35c | 2014-04-23 20:15:48 +0200 | [diff] [blame] | 295 | |
Peter Zijlstra | 56fefbb | 2016-04-18 01:16:05 +0200 | [diff] [blame] | 296 | #undef ATOMIC64_OPS |
Heiko Carstens | 7528743 | 2013-09-11 16:34:04 +0200 | [diff] [blame] | 297 | #undef __ATOMIC64_LOOP |
Heiko Carstens | 1275105 | 2009-09-11 10:28:34 +0200 | [diff] [blame] | 298 | |
Heiko Carstens | 9a70a42 | 2013-09-16 11:07:49 +0200 | [diff] [blame] | 299 | static inline int atomic64_add_unless(atomic64_t *v, long long i, long long u) |
Martin Schwidefsky | 973bd99 | 2006-01-06 00:19:07 -0800 | [diff] [blame] | 300 | { |
| 301 | long long c, old; |
Heiko Carstens | 2ddb3ec | 2010-05-26 23:26:18 +0200 | [diff] [blame] | 302 | |
Martin Schwidefsky | 973bd99 | 2006-01-06 00:19:07 -0800 | [diff] [blame] | 303 | c = atomic64_read(v); |
Nick Piggin | 0b2fcfd | 2006-03-23 03:01:02 -0800 | [diff] [blame] | 304 | for (;;) { |
| 305 | if (unlikely(c == u)) |
| 306 | break; |
Heiko Carstens | 9a70a42 | 2013-09-16 11:07:49 +0200 | [diff] [blame] | 307 | old = atomic64_cmpxchg(v, c, c + i); |
Nick Piggin | 0b2fcfd | 2006-03-23 03:01:02 -0800 | [diff] [blame] | 308 | if (likely(old == c)) |
| 309 | break; |
Martin Schwidefsky | 973bd99 | 2006-01-06 00:19:07 -0800 | [diff] [blame] | 310 | c = old; |
Nick Piggin | 0b2fcfd | 2006-03-23 03:01:02 -0800 | [diff] [blame] | 311 | } |
Martin Schwidefsky | 973bd99 | 2006-01-06 00:19:07 -0800 | [diff] [blame] | 312 | return c != u; |
| 313 | } |
| 314 | |
Heiko Carstens | 2ddb3ec | 2010-05-26 23:26:18 +0200 | [diff] [blame] | 315 | static inline long long atomic64_dec_if_positive(atomic64_t *v) |
| 316 | { |
| 317 | long long c, old, dec; |
| 318 | |
| 319 | c = atomic64_read(v); |
| 320 | for (;;) { |
| 321 | dec = c - 1; |
| 322 | if (unlikely(dec < 0)) |
| 323 | break; |
| 324 | old = atomic64_cmpxchg((v), c, dec); |
| 325 | if (likely(old == c)) |
| 326 | break; |
| 327 | c = old; |
| 328 | } |
| 329 | return dec; |
| 330 | } |
| 331 | |
Heiko Carstens | 1275105 | 2009-09-11 10:28:34 +0200 | [diff] [blame] | 332 | #define atomic64_add_negative(_i, _v) (atomic64_add_return(_i, _v) < 0) |
Heiko Carstens | 5692e4d | 2013-09-16 10:44:54 +0200 | [diff] [blame] | 333 | #define atomic64_inc(_v) atomic64_add(1, _v) |
Heiko Carstens | 1275105 | 2009-09-11 10:28:34 +0200 | [diff] [blame] | 334 | #define atomic64_inc_return(_v) atomic64_add_return(1, _v) |
| 335 | #define atomic64_inc_and_test(_v) (atomic64_add_return(1, _v) == 0) |
Heiko Carstens | 86d51bc | 2013-09-11 14:28:47 +0200 | [diff] [blame] | 336 | #define atomic64_sub_return(_i, _v) atomic64_add_return(-(long long)(_i), _v) |
Peter Zijlstra | 56fefbb | 2016-04-18 01:16:05 +0200 | [diff] [blame] | 337 | #define atomic64_fetch_sub(_i, _v) atomic64_fetch_add(-(long long)(_i), _v) |
Heiko Carstens | 5692e4d | 2013-09-16 10:44:54 +0200 | [diff] [blame] | 338 | #define atomic64_sub(_i, _v) atomic64_add(-(long long)(_i), _v) |
Heiko Carstens | 1275105 | 2009-09-11 10:28:34 +0200 | [diff] [blame] | 339 | #define atomic64_sub_and_test(_i, _v) (atomic64_sub_return(_i, _v) == 0) |
Heiko Carstens | 5692e4d | 2013-09-16 10:44:54 +0200 | [diff] [blame] | 340 | #define atomic64_dec(_v) atomic64_sub(1, _v) |
Heiko Carstens | 1275105 | 2009-09-11 10:28:34 +0200 | [diff] [blame] | 341 | #define atomic64_dec_return(_v) atomic64_sub_return(1, _v) |
| 342 | #define atomic64_dec_and_test(_v) (atomic64_sub_return(1, _v) == 0) |
| 343 | #define atomic64_inc_not_zero(v) atomic64_add_unless((v), 1, 0) |
Nick Piggin | 8426e1f | 2005-11-13 16:07:25 -0800 | [diff] [blame] | 344 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 345 | #endif /* __ARCH_S390_ATOMIC__ */ |