Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 1 | // Copyright 2012 the V8 project authors. All rights reserved. |
| 2 | // Use of this source code is governed by a BSD-style license that can be |
| 3 | // found in the LICENSE file. |
| 4 | |
| 5 | |
| 6 | // This file is an internal atomic implementation for compiler-based |
| 7 | // ThreadSanitizer. Use base/atomicops.h instead. |
| 8 | |
| 9 | #ifndef V8_BASE_ATOMICOPS_INTERNALS_TSAN_H_ |
| 10 | #define V8_BASE_ATOMICOPS_INTERNALS_TSAN_H_ |
| 11 | |
| 12 | namespace v8 { |
| 13 | namespace base { |
| 14 | |
| 15 | #ifndef TSAN_INTERFACE_ATOMIC_H |
| 16 | #define TSAN_INTERFACE_ATOMIC_H |
| 17 | |
| 18 | |
| 19 | extern "C" { |
| 20 | typedef char __tsan_atomic8; |
| 21 | typedef short __tsan_atomic16; // NOLINT |
| 22 | typedef int __tsan_atomic32; |
| 23 | typedef long __tsan_atomic64; // NOLINT |
| 24 | |
| 25 | #if defined(__SIZEOF_INT128__) \ |
| 26 | || (__clang_major__ * 100 + __clang_minor__ >= 302) |
| 27 | typedef __int128 __tsan_atomic128; |
| 28 | #define __TSAN_HAS_INT128 1 |
| 29 | #else |
| 30 | typedef char __tsan_atomic128; |
| 31 | #define __TSAN_HAS_INT128 0 |
| 32 | #endif |
| 33 | |
| 34 | typedef enum { |
| 35 | __tsan_memory_order_relaxed, |
| 36 | __tsan_memory_order_consume, |
| 37 | __tsan_memory_order_acquire, |
| 38 | __tsan_memory_order_release, |
| 39 | __tsan_memory_order_acq_rel, |
| 40 | __tsan_memory_order_seq_cst, |
| 41 | } __tsan_memory_order; |
| 42 | |
| 43 | __tsan_atomic8 __tsan_atomic8_load(const volatile __tsan_atomic8* a, |
| 44 | __tsan_memory_order mo); |
| 45 | __tsan_atomic16 __tsan_atomic16_load(const volatile __tsan_atomic16* a, |
| 46 | __tsan_memory_order mo); |
| 47 | __tsan_atomic32 __tsan_atomic32_load(const volatile __tsan_atomic32* a, |
| 48 | __tsan_memory_order mo); |
| 49 | __tsan_atomic64 __tsan_atomic64_load(const volatile __tsan_atomic64* a, |
| 50 | __tsan_memory_order mo); |
| 51 | __tsan_atomic128 __tsan_atomic128_load(const volatile __tsan_atomic128* a, |
| 52 | __tsan_memory_order mo); |
| 53 | |
| 54 | void __tsan_atomic8_store(volatile __tsan_atomic8* a, __tsan_atomic8 v, |
| 55 | __tsan_memory_order mo); |
| 56 | void __tsan_atomic16_store(volatile __tsan_atomic16* a, __tsan_atomic16 v, |
| 57 | __tsan_memory_order mo); |
| 58 | void __tsan_atomic32_store(volatile __tsan_atomic32* a, __tsan_atomic32 v, |
| 59 | __tsan_memory_order mo); |
| 60 | void __tsan_atomic64_store(volatile __tsan_atomic64* a, __tsan_atomic64 v, |
| 61 | __tsan_memory_order mo); |
| 62 | void __tsan_atomic128_store(volatile __tsan_atomic128* a, __tsan_atomic128 v, |
| 63 | __tsan_memory_order mo); |
| 64 | |
| 65 | __tsan_atomic8 __tsan_atomic8_exchange(volatile __tsan_atomic8* a, |
| 66 | __tsan_atomic8 v, __tsan_memory_order mo); |
| 67 | __tsan_atomic16 __tsan_atomic16_exchange(volatile __tsan_atomic16* a, |
| 68 | __tsan_atomic16 v, __tsan_memory_order mo); |
| 69 | __tsan_atomic32 __tsan_atomic32_exchange(volatile __tsan_atomic32* a, |
| 70 | __tsan_atomic32 v, __tsan_memory_order mo); |
| 71 | __tsan_atomic64 __tsan_atomic64_exchange(volatile __tsan_atomic64* a, |
| 72 | __tsan_atomic64 v, __tsan_memory_order mo); |
| 73 | __tsan_atomic128 __tsan_atomic128_exchange(volatile __tsan_atomic128* a, |
| 74 | __tsan_atomic128 v, __tsan_memory_order mo); |
| 75 | |
| 76 | __tsan_atomic8 __tsan_atomic8_fetch_add(volatile __tsan_atomic8* a, |
| 77 | __tsan_atomic8 v, __tsan_memory_order mo); |
| 78 | __tsan_atomic16 __tsan_atomic16_fetch_add(volatile __tsan_atomic16* a, |
| 79 | __tsan_atomic16 v, __tsan_memory_order mo); |
| 80 | __tsan_atomic32 __tsan_atomic32_fetch_add(volatile __tsan_atomic32* a, |
| 81 | __tsan_atomic32 v, __tsan_memory_order mo); |
| 82 | __tsan_atomic64 __tsan_atomic64_fetch_add(volatile __tsan_atomic64* a, |
| 83 | __tsan_atomic64 v, __tsan_memory_order mo); |
| 84 | __tsan_atomic128 __tsan_atomic128_fetch_add(volatile __tsan_atomic128* a, |
| 85 | __tsan_atomic128 v, __tsan_memory_order mo); |
| 86 | |
| 87 | __tsan_atomic8 __tsan_atomic8_fetch_and(volatile __tsan_atomic8* a, |
| 88 | __tsan_atomic8 v, __tsan_memory_order mo); |
| 89 | __tsan_atomic16 __tsan_atomic16_fetch_and(volatile __tsan_atomic16* a, |
| 90 | __tsan_atomic16 v, __tsan_memory_order mo); |
| 91 | __tsan_atomic32 __tsan_atomic32_fetch_and(volatile __tsan_atomic32* a, |
| 92 | __tsan_atomic32 v, __tsan_memory_order mo); |
| 93 | __tsan_atomic64 __tsan_atomic64_fetch_and(volatile __tsan_atomic64* a, |
| 94 | __tsan_atomic64 v, __tsan_memory_order mo); |
| 95 | __tsan_atomic128 __tsan_atomic128_fetch_and(volatile __tsan_atomic128* a, |
| 96 | __tsan_atomic128 v, __tsan_memory_order mo); |
| 97 | |
| 98 | __tsan_atomic8 __tsan_atomic8_fetch_or(volatile __tsan_atomic8* a, |
| 99 | __tsan_atomic8 v, __tsan_memory_order mo); |
| 100 | __tsan_atomic16 __tsan_atomic16_fetch_or(volatile __tsan_atomic16* a, |
| 101 | __tsan_atomic16 v, __tsan_memory_order mo); |
| 102 | __tsan_atomic32 __tsan_atomic32_fetch_or(volatile __tsan_atomic32* a, |
| 103 | __tsan_atomic32 v, __tsan_memory_order mo); |
| 104 | __tsan_atomic64 __tsan_atomic64_fetch_or(volatile __tsan_atomic64* a, |
| 105 | __tsan_atomic64 v, __tsan_memory_order mo); |
| 106 | __tsan_atomic128 __tsan_atomic128_fetch_or(volatile __tsan_atomic128* a, |
| 107 | __tsan_atomic128 v, __tsan_memory_order mo); |
| 108 | |
| 109 | __tsan_atomic8 __tsan_atomic8_fetch_xor(volatile __tsan_atomic8* a, |
| 110 | __tsan_atomic8 v, __tsan_memory_order mo); |
| 111 | __tsan_atomic16 __tsan_atomic16_fetch_xor(volatile __tsan_atomic16* a, |
| 112 | __tsan_atomic16 v, __tsan_memory_order mo); |
| 113 | __tsan_atomic32 __tsan_atomic32_fetch_xor(volatile __tsan_atomic32* a, |
| 114 | __tsan_atomic32 v, __tsan_memory_order mo); |
| 115 | __tsan_atomic64 __tsan_atomic64_fetch_xor(volatile __tsan_atomic64* a, |
| 116 | __tsan_atomic64 v, __tsan_memory_order mo); |
| 117 | __tsan_atomic128 __tsan_atomic128_fetch_xor(volatile __tsan_atomic128* a, |
| 118 | __tsan_atomic128 v, __tsan_memory_order mo); |
| 119 | |
| 120 | __tsan_atomic8 __tsan_atomic8_fetch_nand(volatile __tsan_atomic8* a, |
| 121 | __tsan_atomic8 v, __tsan_memory_order mo); |
| 122 | __tsan_atomic16 __tsan_atomic16_fetch_nand(volatile __tsan_atomic16* a, |
| 123 | __tsan_atomic16 v, __tsan_memory_order mo); |
| 124 | __tsan_atomic32 __tsan_atomic32_fetch_nand(volatile __tsan_atomic32* a, |
| 125 | __tsan_atomic32 v, __tsan_memory_order mo); |
| 126 | __tsan_atomic64 __tsan_atomic64_fetch_nand(volatile __tsan_atomic64* a, |
| 127 | __tsan_atomic64 v, __tsan_memory_order mo); |
| 128 | __tsan_atomic128 __tsan_atomic128_fetch_nand(volatile __tsan_atomic128* a, |
| 129 | __tsan_atomic128 v, __tsan_memory_order mo); |
| 130 | |
| 131 | int __tsan_atomic8_compare_exchange_weak(volatile __tsan_atomic8* a, |
| 132 | __tsan_atomic8* c, __tsan_atomic8 v, __tsan_memory_order mo, |
| 133 | __tsan_memory_order fail_mo); |
| 134 | int __tsan_atomic16_compare_exchange_weak(volatile __tsan_atomic16* a, |
| 135 | __tsan_atomic16* c, __tsan_atomic16 v, __tsan_memory_order mo, |
| 136 | __tsan_memory_order fail_mo); |
| 137 | int __tsan_atomic32_compare_exchange_weak(volatile __tsan_atomic32* a, |
| 138 | __tsan_atomic32* c, __tsan_atomic32 v, __tsan_memory_order mo, |
| 139 | __tsan_memory_order fail_mo); |
| 140 | int __tsan_atomic64_compare_exchange_weak(volatile __tsan_atomic64* a, |
| 141 | __tsan_atomic64* c, __tsan_atomic64 v, __tsan_memory_order mo, |
| 142 | __tsan_memory_order fail_mo); |
| 143 | int __tsan_atomic128_compare_exchange_weak(volatile __tsan_atomic128* a, |
| 144 | __tsan_atomic128* c, __tsan_atomic128 v, __tsan_memory_order mo, |
| 145 | __tsan_memory_order fail_mo); |
| 146 | |
| 147 | int __tsan_atomic8_compare_exchange_strong(volatile __tsan_atomic8* a, |
| 148 | __tsan_atomic8* c, __tsan_atomic8 v, __tsan_memory_order mo, |
| 149 | __tsan_memory_order fail_mo); |
| 150 | int __tsan_atomic16_compare_exchange_strong(volatile __tsan_atomic16* a, |
| 151 | __tsan_atomic16* c, __tsan_atomic16 v, __tsan_memory_order mo, |
| 152 | __tsan_memory_order fail_mo); |
| 153 | int __tsan_atomic32_compare_exchange_strong(volatile __tsan_atomic32* a, |
| 154 | __tsan_atomic32* c, __tsan_atomic32 v, __tsan_memory_order mo, |
| 155 | __tsan_memory_order fail_mo); |
| 156 | int __tsan_atomic64_compare_exchange_strong(volatile __tsan_atomic64* a, |
| 157 | __tsan_atomic64* c, __tsan_atomic64 v, __tsan_memory_order mo, |
| 158 | __tsan_memory_order fail_mo); |
| 159 | int __tsan_atomic128_compare_exchange_strong(volatile __tsan_atomic128* a, |
| 160 | __tsan_atomic128* c, __tsan_atomic128 v, __tsan_memory_order mo, |
| 161 | __tsan_memory_order fail_mo); |
| 162 | |
| 163 | __tsan_atomic8 __tsan_atomic8_compare_exchange_val( |
| 164 | volatile __tsan_atomic8* a, __tsan_atomic8 c, __tsan_atomic8 v, |
| 165 | __tsan_memory_order mo, __tsan_memory_order fail_mo); |
| 166 | __tsan_atomic16 __tsan_atomic16_compare_exchange_val( |
| 167 | volatile __tsan_atomic16* a, __tsan_atomic16 c, __tsan_atomic16 v, |
| 168 | __tsan_memory_order mo, __tsan_memory_order fail_mo); |
| 169 | __tsan_atomic32 __tsan_atomic32_compare_exchange_val( |
| 170 | volatile __tsan_atomic32* a, __tsan_atomic32 c, __tsan_atomic32 v, |
| 171 | __tsan_memory_order mo, __tsan_memory_order fail_mo); |
| 172 | __tsan_atomic64 __tsan_atomic64_compare_exchange_val( |
| 173 | volatile __tsan_atomic64* a, __tsan_atomic64 c, __tsan_atomic64 v, |
| 174 | __tsan_memory_order mo, __tsan_memory_order fail_mo); |
| 175 | __tsan_atomic128 __tsan_atomic128_compare_exchange_val( |
| 176 | volatile __tsan_atomic128* a, __tsan_atomic128 c, __tsan_atomic128 v, |
| 177 | __tsan_memory_order mo, __tsan_memory_order fail_mo); |
| 178 | |
| 179 | void __tsan_atomic_thread_fence(__tsan_memory_order mo); |
| 180 | void __tsan_atomic_signal_fence(__tsan_memory_order mo); |
| 181 | } // extern "C" |
| 182 | |
| 183 | #endif // #ifndef TSAN_INTERFACE_ATOMIC_H |
| 184 | |
| 185 | inline Atomic32 NoBarrier_CompareAndSwap(volatile Atomic32* ptr, |
| 186 | Atomic32 old_value, |
| 187 | Atomic32 new_value) { |
| 188 | Atomic32 cmp = old_value; |
| 189 | __tsan_atomic32_compare_exchange_strong(ptr, &cmp, new_value, |
| 190 | __tsan_memory_order_relaxed, __tsan_memory_order_relaxed); |
| 191 | return cmp; |
| 192 | } |
| 193 | |
| 194 | inline Atomic32 NoBarrier_AtomicExchange(volatile Atomic32* ptr, |
| 195 | Atomic32 new_value) { |
| 196 | return __tsan_atomic32_exchange(ptr, new_value, |
| 197 | __tsan_memory_order_relaxed); |
| 198 | } |
| 199 | |
| 200 | inline Atomic32 Acquire_AtomicExchange(volatile Atomic32* ptr, |
| 201 | Atomic32 new_value) { |
| 202 | return __tsan_atomic32_exchange(ptr, new_value, |
| 203 | __tsan_memory_order_acquire); |
| 204 | } |
| 205 | |
| 206 | inline Atomic32 Release_AtomicExchange(volatile Atomic32* ptr, |
| 207 | Atomic32 new_value) { |
| 208 | return __tsan_atomic32_exchange(ptr, new_value, |
| 209 | __tsan_memory_order_release); |
| 210 | } |
| 211 | |
| 212 | inline Atomic32 NoBarrier_AtomicIncrement(volatile Atomic32* ptr, |
| 213 | Atomic32 increment) { |
| 214 | return increment + __tsan_atomic32_fetch_add(ptr, increment, |
| 215 | __tsan_memory_order_relaxed); |
| 216 | } |
| 217 | |
| 218 | inline Atomic32 Barrier_AtomicIncrement(volatile Atomic32* ptr, |
| 219 | Atomic32 increment) { |
| 220 | return increment + __tsan_atomic32_fetch_add(ptr, increment, |
| 221 | __tsan_memory_order_acq_rel); |
| 222 | } |
| 223 | |
| 224 | inline Atomic32 Acquire_CompareAndSwap(volatile Atomic32* ptr, |
| 225 | Atomic32 old_value, |
| 226 | Atomic32 new_value) { |
| 227 | Atomic32 cmp = old_value; |
| 228 | __tsan_atomic32_compare_exchange_strong(ptr, &cmp, new_value, |
| 229 | __tsan_memory_order_acquire, __tsan_memory_order_acquire); |
| 230 | return cmp; |
| 231 | } |
| 232 | |
| 233 | inline Atomic32 Release_CompareAndSwap(volatile Atomic32* ptr, |
| 234 | Atomic32 old_value, |
| 235 | Atomic32 new_value) { |
| 236 | Atomic32 cmp = old_value; |
| 237 | __tsan_atomic32_compare_exchange_strong(ptr, &cmp, new_value, |
| 238 | __tsan_memory_order_release, __tsan_memory_order_relaxed); |
| 239 | return cmp; |
| 240 | } |
| 241 | |
| 242 | inline void NoBarrier_Store(volatile Atomic8* ptr, Atomic8 value) { |
| 243 | __tsan_atomic8_store(ptr, value, __tsan_memory_order_relaxed); |
| 244 | } |
| 245 | |
| 246 | inline void NoBarrier_Store(volatile Atomic32* ptr, Atomic32 value) { |
| 247 | __tsan_atomic32_store(ptr, value, __tsan_memory_order_relaxed); |
| 248 | } |
| 249 | |
| 250 | inline void Acquire_Store(volatile Atomic32* ptr, Atomic32 value) { |
| 251 | __tsan_atomic32_store(ptr, value, __tsan_memory_order_relaxed); |
| 252 | __tsan_atomic_thread_fence(__tsan_memory_order_seq_cst); |
| 253 | } |
| 254 | |
| 255 | inline void Release_Store(volatile Atomic32* ptr, Atomic32 value) { |
| 256 | __tsan_atomic32_store(ptr, value, __tsan_memory_order_release); |
| 257 | } |
| 258 | |
| 259 | inline Atomic8 NoBarrier_Load(volatile const Atomic8* ptr) { |
| 260 | return __tsan_atomic8_load(ptr, __tsan_memory_order_relaxed); |
| 261 | } |
| 262 | |
| 263 | inline Atomic32 NoBarrier_Load(volatile const Atomic32* ptr) { |
| 264 | return __tsan_atomic32_load(ptr, __tsan_memory_order_relaxed); |
| 265 | } |
| 266 | |
| 267 | inline Atomic32 Acquire_Load(volatile const Atomic32* ptr) { |
| 268 | return __tsan_atomic32_load(ptr, __tsan_memory_order_acquire); |
| 269 | } |
| 270 | |
| 271 | inline Atomic32 Release_Load(volatile const Atomic32* ptr) { |
| 272 | __tsan_atomic_thread_fence(__tsan_memory_order_seq_cst); |
| 273 | return __tsan_atomic32_load(ptr, __tsan_memory_order_relaxed); |
| 274 | } |
| 275 | |
| 276 | inline Atomic64 NoBarrier_CompareAndSwap(volatile Atomic64* ptr, |
| 277 | Atomic64 old_value, |
| 278 | Atomic64 new_value) { |
| 279 | Atomic64 cmp = old_value; |
| 280 | __tsan_atomic64_compare_exchange_strong(ptr, &cmp, new_value, |
| 281 | __tsan_memory_order_relaxed, __tsan_memory_order_relaxed); |
| 282 | return cmp; |
| 283 | } |
| 284 | |
| 285 | inline Atomic64 NoBarrier_AtomicExchange(volatile Atomic64* ptr, |
| 286 | Atomic64 new_value) { |
| 287 | return __tsan_atomic64_exchange(ptr, new_value, __tsan_memory_order_relaxed); |
| 288 | } |
| 289 | |
| 290 | inline Atomic64 Acquire_AtomicExchange(volatile Atomic64* ptr, |
| 291 | Atomic64 new_value) { |
| 292 | return __tsan_atomic64_exchange(ptr, new_value, __tsan_memory_order_acquire); |
| 293 | } |
| 294 | |
| 295 | inline Atomic64 Release_AtomicExchange(volatile Atomic64* ptr, |
| 296 | Atomic64 new_value) { |
| 297 | return __tsan_atomic64_exchange(ptr, new_value, __tsan_memory_order_release); |
| 298 | } |
| 299 | |
| 300 | inline Atomic64 NoBarrier_AtomicIncrement(volatile Atomic64* ptr, |
| 301 | Atomic64 increment) { |
| 302 | return increment + __tsan_atomic64_fetch_add(ptr, increment, |
| 303 | __tsan_memory_order_relaxed); |
| 304 | } |
| 305 | |
| 306 | inline Atomic64 Barrier_AtomicIncrement(volatile Atomic64* ptr, |
| 307 | Atomic64 increment) { |
| 308 | return increment + __tsan_atomic64_fetch_add(ptr, increment, |
| 309 | __tsan_memory_order_acq_rel); |
| 310 | } |
| 311 | |
| 312 | inline void NoBarrier_Store(volatile Atomic64* ptr, Atomic64 value) { |
| 313 | __tsan_atomic64_store(ptr, value, __tsan_memory_order_relaxed); |
| 314 | } |
| 315 | |
| 316 | inline void Acquire_Store(volatile Atomic64* ptr, Atomic64 value) { |
| 317 | __tsan_atomic64_store(ptr, value, __tsan_memory_order_relaxed); |
| 318 | __tsan_atomic_thread_fence(__tsan_memory_order_seq_cst); |
| 319 | } |
| 320 | |
| 321 | inline void Release_Store(volatile Atomic64* ptr, Atomic64 value) { |
| 322 | __tsan_atomic64_store(ptr, value, __tsan_memory_order_release); |
| 323 | } |
| 324 | |
| 325 | inline Atomic64 NoBarrier_Load(volatile const Atomic64* ptr) { |
| 326 | return __tsan_atomic64_load(ptr, __tsan_memory_order_relaxed); |
| 327 | } |
| 328 | |
| 329 | inline Atomic64 Acquire_Load(volatile const Atomic64* ptr) { |
| 330 | return __tsan_atomic64_load(ptr, __tsan_memory_order_acquire); |
| 331 | } |
| 332 | |
| 333 | inline Atomic64 Release_Load(volatile const Atomic64* ptr) { |
| 334 | __tsan_atomic_thread_fence(__tsan_memory_order_seq_cst); |
| 335 | return __tsan_atomic64_load(ptr, __tsan_memory_order_relaxed); |
| 336 | } |
| 337 | |
| 338 | inline Atomic64 Acquire_CompareAndSwap(volatile Atomic64* ptr, |
| 339 | Atomic64 old_value, |
| 340 | Atomic64 new_value) { |
| 341 | Atomic64 cmp = old_value; |
| 342 | __tsan_atomic64_compare_exchange_strong(ptr, &cmp, new_value, |
| 343 | __tsan_memory_order_acquire, __tsan_memory_order_acquire); |
| 344 | return cmp; |
| 345 | } |
| 346 | |
| 347 | inline Atomic64 Release_CompareAndSwap(volatile Atomic64* ptr, |
| 348 | Atomic64 old_value, |
| 349 | Atomic64 new_value) { |
| 350 | Atomic64 cmp = old_value; |
| 351 | __tsan_atomic64_compare_exchange_strong(ptr, &cmp, new_value, |
| 352 | __tsan_memory_order_release, __tsan_memory_order_relaxed); |
| 353 | return cmp; |
| 354 | } |
| 355 | |
| 356 | inline void MemoryBarrier() { |
| 357 | __tsan_atomic_thread_fence(__tsan_memory_order_seq_cst); |
| 358 | } |
| 359 | |
| 360 | } // namespace base |
| 361 | } // namespace v8 |
| 362 | |
| 363 | #endif // V8_BASE_ATOMICOPS_INTERNALS_TSAN_H_ |