Martin v. Löwis | 4d0d471 | 2010-12-03 20:14:31 +0000 | [diff] [blame] | 1 | #ifndef Py_LIMITED_API |
Jeffrey Yasskin | 3937083 | 2010-05-03 19:29:34 +0000 | [diff] [blame] | 2 | #ifndef Py_ATOMIC_H |
| 3 | #define Py_ATOMIC_H |
| 4 | /* XXX: When compilers start offering a stdatomic.h with lock-free |
| 5 | atomic_int and atomic_address types, include that here and rewrite |
| 6 | the atomic operations in terms of it. */ |
| 7 | |
| 8 | #include "dynamic_annotations.h" |
| 9 | |
| 10 | #ifdef __cplusplus |
| 11 | extern "C" { |
| 12 | #endif |
| 13 | |
| 14 | /* This is modeled after the atomics interface from C1x, according to |
| 15 | * the draft at |
| 16 | * http://www.open-std.org/JTC1/SC22/wg14/www/docs/n1425.pdf. |
| 17 | * Operations and types are named the same except with a _Py_ prefix |
| 18 | * and have the same semantics. |
| 19 | * |
| 20 | * Beware, the implementations here are deep magic. |
| 21 | */ |
| 22 | |
| 23 | typedef enum _Py_memory_order { |
| 24 | _Py_memory_order_relaxed, |
| 25 | _Py_memory_order_acquire, |
| 26 | _Py_memory_order_release, |
| 27 | _Py_memory_order_acq_rel, |
| 28 | _Py_memory_order_seq_cst |
| 29 | } _Py_memory_order; |
| 30 | |
| 31 | typedef struct _Py_atomic_address { |
| 32 | void *_value; |
| 33 | } _Py_atomic_address; |
| 34 | |
| 35 | typedef struct _Py_atomic_int { |
| 36 | int _value; |
| 37 | } _Py_atomic_int; |
| 38 | |
| 39 | /* Only support GCC (for expression statements) and x86 (for simple |
| 40 | * atomic semantics) for now */ |
| 41 | #if defined(__GNUC__) && (defined(__i386__) || defined(__amd64)) |
| 42 | |
| 43 | static __inline__ void |
| 44 | _Py_atomic_signal_fence(_Py_memory_order order) |
| 45 | { |
| 46 | if (order != _Py_memory_order_relaxed) |
| 47 | __asm__ volatile("":::"memory"); |
| 48 | } |
| 49 | |
| 50 | static __inline__ void |
| 51 | _Py_atomic_thread_fence(_Py_memory_order order) |
| 52 | { |
| 53 | if (order != _Py_memory_order_relaxed) |
| 54 | __asm__ volatile("mfence":::"memory"); |
| 55 | } |
| 56 | |
| 57 | /* Tell the race checker about this operation's effects. */ |
| 58 | static __inline__ void |
| 59 | _Py_ANNOTATE_MEMORY_ORDER(const volatile void *address, _Py_memory_order order) |
| 60 | { |
| 61 | switch(order) { |
| 62 | case _Py_memory_order_release: |
| 63 | case _Py_memory_order_acq_rel: |
| 64 | case _Py_memory_order_seq_cst: |
| 65 | _Py_ANNOTATE_HAPPENS_BEFORE(address); |
| 66 | break; |
| 67 | default: |
| 68 | break; |
| 69 | } |
| 70 | switch(order) { |
| 71 | case _Py_memory_order_acquire: |
| 72 | case _Py_memory_order_acq_rel: |
| 73 | case _Py_memory_order_seq_cst: |
| 74 | _Py_ANNOTATE_HAPPENS_AFTER(address); |
| 75 | break; |
| 76 | default: |
| 77 | break; |
| 78 | } |
| 79 | } |
| 80 | |
| 81 | #define _Py_atomic_store_explicit(ATOMIC_VAL, NEW_VAL, ORDER) \ |
| 82 | __extension__ ({ \ |
| 83 | __typeof__(ATOMIC_VAL) atomic_val = ATOMIC_VAL; \ |
| 84 | __typeof__(atomic_val->_value) new_val = NEW_VAL;\ |
| 85 | volatile __typeof__(new_val) *volatile_data = &atomic_val->_value; \ |
| 86 | _Py_memory_order order = ORDER; \ |
| 87 | _Py_ANNOTATE_MEMORY_ORDER(atomic_val, order); \ |
| 88 | \ |
| 89 | /* Perform the operation. */ \ |
| 90 | _Py_ANNOTATE_IGNORE_WRITES_BEGIN(); \ |
| 91 | switch(order) { \ |
| 92 | case _Py_memory_order_release: \ |
| 93 | _Py_atomic_signal_fence(_Py_memory_order_release); \ |
| 94 | /* fallthrough */ \ |
| 95 | case _Py_memory_order_relaxed: \ |
| 96 | *volatile_data = new_val; \ |
| 97 | break; \ |
| 98 | \ |
| 99 | case _Py_memory_order_acquire: \ |
| 100 | case _Py_memory_order_acq_rel: \ |
| 101 | case _Py_memory_order_seq_cst: \ |
| 102 | __asm__ volatile("xchg %0, %1" \ |
| 103 | : "+r"(new_val) \ |
| 104 | : "m"(atomic_val->_value) \ |
| 105 | : "memory"); \ |
| 106 | break; \ |
| 107 | } \ |
| 108 | _Py_ANNOTATE_IGNORE_WRITES_END(); \ |
| 109 | }) |
| 110 | |
| 111 | #define _Py_atomic_load_explicit(ATOMIC_VAL, ORDER) \ |
| 112 | __extension__ ({ \ |
| 113 | __typeof__(ATOMIC_VAL) atomic_val = ATOMIC_VAL; \ |
| 114 | __typeof__(atomic_val->_value) result; \ |
| 115 | volatile __typeof__(result) *volatile_data = &atomic_val->_value; \ |
| 116 | _Py_memory_order order = ORDER; \ |
| 117 | _Py_ANNOTATE_MEMORY_ORDER(atomic_val, order); \ |
| 118 | \ |
| 119 | /* Perform the operation. */ \ |
| 120 | _Py_ANNOTATE_IGNORE_READS_BEGIN(); \ |
| 121 | switch(order) { \ |
| 122 | case _Py_memory_order_release: \ |
| 123 | case _Py_memory_order_acq_rel: \ |
| 124 | case _Py_memory_order_seq_cst: \ |
| 125 | /* Loads on x86 are not releases by default, so need a */ \ |
| 126 | /* thread fence. */ \ |
| 127 | _Py_atomic_thread_fence(_Py_memory_order_release); \ |
| 128 | break; \ |
| 129 | default: \ |
| 130 | /* No fence */ \ |
| 131 | break; \ |
| 132 | } \ |
| 133 | result = *volatile_data; \ |
| 134 | switch(order) { \ |
| 135 | case _Py_memory_order_acquire: \ |
| 136 | case _Py_memory_order_acq_rel: \ |
| 137 | case _Py_memory_order_seq_cst: \ |
| 138 | /* Loads on x86 are automatically acquire operations so */ \ |
| 139 | /* can get by with just a compiler fence. */ \ |
| 140 | _Py_atomic_signal_fence(_Py_memory_order_acquire); \ |
| 141 | break; \ |
| 142 | default: \ |
| 143 | /* No fence */ \ |
| 144 | break; \ |
| 145 | } \ |
| 146 | _Py_ANNOTATE_IGNORE_READS_END(); \ |
| 147 | result; \ |
| 148 | }) |
| 149 | |
| 150 | #else /* !gcc x86 */ |
| 151 | /* Fall back to other compilers and processors by assuming that simple |
| 152 | volatile accesses are atomic. This is false, so people should port |
| 153 | this. */ |
| 154 | #define _Py_atomic_signal_fence(/*memory_order*/ ORDER) ((void)0) |
| 155 | #define _Py_atomic_thread_fence(/*memory_order*/ ORDER) ((void)0) |
| 156 | #define _Py_atomic_store_explicit(ATOMIC_VAL, NEW_VAL, ORDER) \ |
| 157 | ((ATOMIC_VAL)->_value = NEW_VAL) |
| 158 | #define _Py_atomic_load_explicit(ATOMIC_VAL, ORDER) \ |
| 159 | ((ATOMIC_VAL)->_value) |
| 160 | |
| 161 | #endif /* !gcc x86 */ |
| 162 | |
| 163 | /* Standardized shortcuts. */ |
| 164 | #define _Py_atomic_store(ATOMIC_VAL, NEW_VAL) \ |
| 165 | _Py_atomic_store_explicit(ATOMIC_VAL, NEW_VAL, _Py_memory_order_seq_cst) |
| 166 | #define _Py_atomic_load(ATOMIC_VAL) \ |
| 167 | _Py_atomic_load_explicit(ATOMIC_VAL, _Py_memory_order_seq_cst) |
| 168 | |
| 169 | /* Python-local extensions */ |
| 170 | |
| 171 | #define _Py_atomic_store_relaxed(ATOMIC_VAL, NEW_VAL) \ |
| 172 | _Py_atomic_store_explicit(ATOMIC_VAL, NEW_VAL, _Py_memory_order_relaxed) |
| 173 | #define _Py_atomic_load_relaxed(ATOMIC_VAL) \ |
| 174 | _Py_atomic_load_explicit(ATOMIC_VAL, _Py_memory_order_relaxed) |
| 175 | |
| 176 | #ifdef __cplusplus |
| 177 | } |
| 178 | #endif |
| 179 | |
| 180 | #endif /* Py_ATOMIC_H */ |
Martin v. Löwis | 4d0d471 | 2010-12-03 20:14:31 +0000 | [diff] [blame] | 181 | #endif /* Py_LIMITED_API */ |