Ingo Molnar | b7882b7 | 2009-07-03 13:26:39 +0200 | [diff] [blame] | 1 | #include <linux/compiler.h> |
Ingo Molnar | 1fde902 | 2009-07-03 17:28:57 +0200 | [diff] [blame^] | 2 | #include <linux/module.h> |
Ingo Molnar | b7882b7 | 2009-07-03 13:26:39 +0200 | [diff] [blame] | 3 | #include <linux/types.h> |
Ingo Molnar | 1fde902 | 2009-07-03 17:28:57 +0200 | [diff] [blame^] | 4 | |
Ingo Molnar | b7882b7 | 2009-07-03 13:26:39 +0200 | [diff] [blame] | 5 | #include <asm/processor.h> |
| 6 | #include <asm/cmpxchg.h> |
| 7 | #include <asm/atomic.h> |
| 8 | |
Ingo Molnar | 3ac805d | 2009-07-03 12:51:19 +0200 | [diff] [blame] | 9 | static noinline u64 cmpxchg8b(u64 *ptr, u64 old, u64 new) |
Ingo Molnar | b7882b7 | 2009-07-03 13:26:39 +0200 | [diff] [blame] | 10 | { |
Eric Dumazet | 69237f9 | 2009-07-03 13:26:41 +0200 | [diff] [blame] | 11 | u32 low = new; |
| 12 | u32 high = new >> 32; |
| 13 | |
Ingo Molnar | b7882b7 | 2009-07-03 13:26:39 +0200 | [diff] [blame] | 14 | asm volatile( |
Eric Dumazet | 69237f9 | 2009-07-03 13:26:41 +0200 | [diff] [blame] | 15 | LOCK_PREFIX "cmpxchg8b %1\n" |
| 16 | : "+A" (old), "+m" (*ptr) |
| 17 | : "b" (low), "c" (high) |
| 18 | ); |
Ingo Molnar | b7882b7 | 2009-07-03 13:26:39 +0200 | [diff] [blame] | 19 | return old; |
| 20 | } |
| 21 | |
| 22 | u64 atomic64_cmpxchg(atomic64_t *ptr, u64 old_val, u64 new_val) |
| 23 | { |
| 24 | return cmpxchg8b(&ptr->counter, old_val, new_val); |
| 25 | } |
Ingo Molnar | 1fde902 | 2009-07-03 17:28:57 +0200 | [diff] [blame^] | 26 | EXPORT_SYMBOL(atomic64_cmpxchg); |
Ingo Molnar | b7882b7 | 2009-07-03 13:26:39 +0200 | [diff] [blame] | 27 | |
| 28 | /** |
| 29 | * atomic64_xchg - xchg atomic64 variable |
| 30 | * @ptr: pointer to type atomic64_t |
| 31 | * @new_val: value to assign |
| 32 | * |
| 33 | * Atomically xchgs the value of @ptr to @new_val and returns |
| 34 | * the old value. |
| 35 | */ |
| 36 | |
| 37 | u64 atomic64_xchg(atomic64_t *ptr, u64 new_val) |
| 38 | { |
| 39 | u64 old_val; |
| 40 | |
| 41 | do { |
Ingo Molnar | 199e237 | 2009-07-03 13:02:39 +0200 | [diff] [blame] | 42 | old_val = __atomic64_read(ptr); |
Ingo Molnar | b7882b7 | 2009-07-03 13:26:39 +0200 | [diff] [blame] | 43 | } while (atomic64_cmpxchg(ptr, old_val, new_val) != old_val); |
| 44 | |
| 45 | return old_val; |
| 46 | } |
Ingo Molnar | 1fde902 | 2009-07-03 17:28:57 +0200 | [diff] [blame^] | 47 | EXPORT_SYMBOL(atomic64_xchg); |
Ingo Molnar | b7882b7 | 2009-07-03 13:26:39 +0200 | [diff] [blame] | 48 | |
| 49 | /** |
| 50 | * atomic64_set - set atomic64 variable |
| 51 | * @ptr: pointer to type atomic64_t |
| 52 | * @new_val: value to assign |
| 53 | * |
| 54 | * Atomically sets the value of @ptr to @new_val. |
| 55 | */ |
| 56 | void atomic64_set(atomic64_t *ptr, u64 new_val) |
| 57 | { |
| 58 | atomic64_xchg(ptr, new_val); |
| 59 | } |
Ingo Molnar | 1fde902 | 2009-07-03 17:28:57 +0200 | [diff] [blame^] | 60 | EXPORT_SYMBOL(atomic64_read); |
Ingo Molnar | b7882b7 | 2009-07-03 13:26:39 +0200 | [diff] [blame] | 61 | |
| 62 | /** |
| 63 | * atomic64_read - read atomic64 variable |
| 64 | * @ptr: pointer to type atomic64_t |
| 65 | * |
| 66 | * Atomically reads the value of @ptr and returns it. |
| 67 | */ |
| 68 | u64 atomic64_read(atomic64_t *ptr) |
| 69 | { |
Eric Dumazet | 67d7178 | 2009-07-03 13:23:02 +0200 | [diff] [blame] | 70 | u64 res; |
Ingo Molnar | b7882b7 | 2009-07-03 13:26:39 +0200 | [diff] [blame] | 71 | |
Eric Dumazet | 67d7178 | 2009-07-03 13:23:02 +0200 | [diff] [blame] | 72 | asm volatile( |
| 73 | "mov %%ebx, %%eax\n\t" |
| 74 | "mov %%ecx, %%edx\n\t" |
| 75 | LOCK_PREFIX "cmpxchg8b %1\n" |
| 76 | : "+A" (res) |
| 77 | : "m" (*ptr) |
| 78 | ); |
| 79 | |
| 80 | return res; |
Ingo Molnar | b7882b7 | 2009-07-03 13:26:39 +0200 | [diff] [blame] | 81 | } |
Ingo Molnar | 1fde902 | 2009-07-03 17:28:57 +0200 | [diff] [blame^] | 82 | EXPORT_SYMBOL(atomic64_read); |
Ingo Molnar | b7882b7 | 2009-07-03 13:26:39 +0200 | [diff] [blame] | 83 | |
| 84 | /** |
| 85 | * atomic64_add_return - add and return |
| 86 | * @delta: integer value to add |
| 87 | * @ptr: pointer to type atomic64_t |
| 88 | * |
| 89 | * Atomically adds @delta to @ptr and returns @delta + *@ptr |
| 90 | */ |
Ingo Molnar | 3ac805d | 2009-07-03 12:51:19 +0200 | [diff] [blame] | 91 | noinline u64 atomic64_add_return(u64 delta, atomic64_t *ptr) |
Ingo Molnar | b7882b7 | 2009-07-03 13:26:39 +0200 | [diff] [blame] | 92 | { |
Ingo Molnar | 824975e | 2009-07-03 12:39:07 +0200 | [diff] [blame] | 93 | /* |
| 94 | * Try first with a (probably incorrect) assumption about |
| 95 | * what we have there. We'll do two loops most likely, |
| 96 | * but we'll get an ownership MESI transaction straight away |
| 97 | * instead of a read transaction followed by a |
| 98 | * flush-for-ownership transaction: |
| 99 | */ |
| 100 | u64 old_val, new_val, real_val = 1ULL << 32; |
Ingo Molnar | b7882b7 | 2009-07-03 13:26:39 +0200 | [diff] [blame] | 101 | |
| 102 | do { |
Ingo Molnar | 824975e | 2009-07-03 12:39:07 +0200 | [diff] [blame] | 103 | old_val = real_val; |
Ingo Molnar | b7882b7 | 2009-07-03 13:26:39 +0200 | [diff] [blame] | 104 | new_val = old_val + delta; |
| 105 | |
Ingo Molnar | 824975e | 2009-07-03 12:39:07 +0200 | [diff] [blame] | 106 | real_val = atomic64_cmpxchg(ptr, old_val, new_val); |
| 107 | |
| 108 | } while (real_val != old_val); |
Ingo Molnar | b7882b7 | 2009-07-03 13:26:39 +0200 | [diff] [blame] | 109 | |
| 110 | return new_val; |
| 111 | } |
Ingo Molnar | 1fde902 | 2009-07-03 17:28:57 +0200 | [diff] [blame^] | 112 | EXPORT_SYMBOL(atomic64_add_return); |
Ingo Molnar | b7882b7 | 2009-07-03 13:26:39 +0200 | [diff] [blame] | 113 | |
| 114 | u64 atomic64_sub_return(u64 delta, atomic64_t *ptr) |
| 115 | { |
| 116 | return atomic64_add_return(-delta, ptr); |
| 117 | } |
Ingo Molnar | 1fde902 | 2009-07-03 17:28:57 +0200 | [diff] [blame^] | 118 | EXPORT_SYMBOL(atomic64_sub_return); |
Ingo Molnar | b7882b7 | 2009-07-03 13:26:39 +0200 | [diff] [blame] | 119 | |
| 120 | u64 atomic64_inc_return(atomic64_t *ptr) |
| 121 | { |
| 122 | return atomic64_add_return(1, ptr); |
| 123 | } |
Ingo Molnar | 1fde902 | 2009-07-03 17:28:57 +0200 | [diff] [blame^] | 124 | EXPORT_SYMBOL(atomic64_inc_return); |
Ingo Molnar | b7882b7 | 2009-07-03 13:26:39 +0200 | [diff] [blame] | 125 | |
| 126 | u64 atomic64_dec_return(atomic64_t *ptr) |
| 127 | { |
| 128 | return atomic64_sub_return(1, ptr); |
| 129 | } |
Ingo Molnar | 1fde902 | 2009-07-03 17:28:57 +0200 | [diff] [blame^] | 130 | EXPORT_SYMBOL(atomic64_dec_return); |
Ingo Molnar | b7882b7 | 2009-07-03 13:26:39 +0200 | [diff] [blame] | 131 | |
| 132 | /** |
| 133 | * atomic64_add - add integer to atomic64 variable |
| 134 | * @delta: integer value to add |
| 135 | * @ptr: pointer to type atomic64_t |
| 136 | * |
| 137 | * Atomically adds @delta to @ptr. |
| 138 | */ |
| 139 | void atomic64_add(u64 delta, atomic64_t *ptr) |
| 140 | { |
| 141 | atomic64_add_return(delta, ptr); |
| 142 | } |
Ingo Molnar | 1fde902 | 2009-07-03 17:28:57 +0200 | [diff] [blame^] | 143 | EXPORT_SYMBOL(atomic64_add); |
Ingo Molnar | b7882b7 | 2009-07-03 13:26:39 +0200 | [diff] [blame] | 144 | |
| 145 | /** |
| 146 | * atomic64_sub - subtract the atomic64 variable |
| 147 | * @delta: integer value to subtract |
| 148 | * @ptr: pointer to type atomic64_t |
| 149 | * |
| 150 | * Atomically subtracts @delta from @ptr. |
| 151 | */ |
| 152 | void atomic64_sub(u64 delta, atomic64_t *ptr) |
| 153 | { |
| 154 | atomic64_add(-delta, ptr); |
| 155 | } |
Ingo Molnar | 1fde902 | 2009-07-03 17:28:57 +0200 | [diff] [blame^] | 156 | EXPORT_SYMBOL(atomic64_sub); |
Ingo Molnar | b7882b7 | 2009-07-03 13:26:39 +0200 | [diff] [blame] | 157 | |
| 158 | /** |
| 159 | * atomic64_sub_and_test - subtract value from variable and test result |
| 160 | * @delta: integer value to subtract |
| 161 | * @ptr: pointer to type atomic64_t |
| 162 | * |
| 163 | * Atomically subtracts @delta from @ptr and returns |
| 164 | * true if the result is zero, or false for all |
| 165 | * other cases. |
| 166 | */ |
| 167 | int atomic64_sub_and_test(u64 delta, atomic64_t *ptr) |
| 168 | { |
| 169 | u64 old_val = atomic64_sub_return(delta, ptr); |
| 170 | |
| 171 | return old_val == 0; |
| 172 | } |
Ingo Molnar | 1fde902 | 2009-07-03 17:28:57 +0200 | [diff] [blame^] | 173 | EXPORT_SYMBOL(atomic64_sub_and_test); |
Ingo Molnar | b7882b7 | 2009-07-03 13:26:39 +0200 | [diff] [blame] | 174 | |
| 175 | /** |
| 176 | * atomic64_inc - increment atomic64 variable |
| 177 | * @ptr: pointer to type atomic64_t |
| 178 | * |
| 179 | * Atomically increments @ptr by 1. |
| 180 | */ |
| 181 | void atomic64_inc(atomic64_t *ptr) |
| 182 | { |
| 183 | atomic64_add(1, ptr); |
| 184 | } |
Ingo Molnar | 1fde902 | 2009-07-03 17:28:57 +0200 | [diff] [blame^] | 185 | EXPORT_SYMBOL(atomic64_inc); |
Ingo Molnar | b7882b7 | 2009-07-03 13:26:39 +0200 | [diff] [blame] | 186 | |
| 187 | /** |
| 188 | * atomic64_dec - decrement atomic64 variable |
| 189 | * @ptr: pointer to type atomic64_t |
| 190 | * |
| 191 | * Atomically decrements @ptr by 1. |
| 192 | */ |
| 193 | void atomic64_dec(atomic64_t *ptr) |
| 194 | { |
| 195 | atomic64_sub(1, ptr); |
| 196 | } |
Ingo Molnar | 1fde902 | 2009-07-03 17:28:57 +0200 | [diff] [blame^] | 197 | EXPORT_SYMBOL(atomic64_dec); |
Ingo Molnar | b7882b7 | 2009-07-03 13:26:39 +0200 | [diff] [blame] | 198 | |
| 199 | /** |
| 200 | * atomic64_dec_and_test - decrement and test |
| 201 | * @ptr: pointer to type atomic64_t |
| 202 | * |
| 203 | * Atomically decrements @ptr by 1 and |
| 204 | * returns true if the result is 0, or false for all other |
| 205 | * cases. |
| 206 | */ |
| 207 | int atomic64_dec_and_test(atomic64_t *ptr) |
| 208 | { |
| 209 | return atomic64_sub_and_test(1, ptr); |
| 210 | } |
Ingo Molnar | 1fde902 | 2009-07-03 17:28:57 +0200 | [diff] [blame^] | 211 | EXPORT_SYMBOL(atomic64_dec_and_test); |
Ingo Molnar | b7882b7 | 2009-07-03 13:26:39 +0200 | [diff] [blame] | 212 | |
| 213 | /** |
| 214 | * atomic64_inc_and_test - increment and test |
| 215 | * @ptr: pointer to type atomic64_t |
| 216 | * |
| 217 | * Atomically increments @ptr by 1 |
| 218 | * and returns true if the result is zero, or false for all |
| 219 | * other cases. |
| 220 | */ |
| 221 | int atomic64_inc_and_test(atomic64_t *ptr) |
| 222 | { |
| 223 | return atomic64_sub_and_test(-1, ptr); |
| 224 | } |
Ingo Molnar | 1fde902 | 2009-07-03 17:28:57 +0200 | [diff] [blame^] | 225 | EXPORT_SYMBOL(atomic64_inc_and_test); |
Ingo Molnar | b7882b7 | 2009-07-03 13:26:39 +0200 | [diff] [blame] | 226 | |
| 227 | /** |
| 228 | * atomic64_add_negative - add and test if negative |
| 229 | * @delta: integer value to add |
| 230 | * @ptr: pointer to type atomic64_t |
| 231 | * |
| 232 | * Atomically adds @delta to @ptr and returns true |
| 233 | * if the result is negative, or false when |
| 234 | * result is greater than or equal to zero. |
| 235 | */ |
| 236 | int atomic64_add_negative(u64 delta, atomic64_t *ptr) |
| 237 | { |
| 238 | long long old_val = atomic64_add_return(delta, ptr); |
| 239 | |
| 240 | return old_val < 0; |
| 241 | } |
Ingo Molnar | 1fde902 | 2009-07-03 17:28:57 +0200 | [diff] [blame^] | 242 | EXPORT_SYMBOL(atomic64_add_negative); |