Elliott Hughes | 5ea047b | 2011-09-13 14:38:18 -0700 | [diff] [blame] | 1 | /* |
| 2 | * Copyright (C) 2010 The Android Open Source Project |
| 3 | * |
| 4 | * Licensed under the Apache License, Version 2.0 (the "License"); |
| 5 | * you may not use this file except in compliance with the License. |
| 6 | * You may obtain a copy of the License at |
| 7 | * |
| 8 | * http://www.apache.org/licenses/LICENSE-2.0 |
| 9 | * |
| 10 | * Unless required by applicable law or agreed to in writing, software |
| 11 | * distributed under the License is distributed on an "AS IS" BASIS, |
| 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 13 | * See the License for the specific language governing permissions and |
| 14 | * limitations under the License. |
| 15 | */ |
| 16 | |
| 17 | #include "atomic.h" |
| 18 | |
Elliott Hughes | 7c6169d | 2012-05-02 16:11:48 -0700 | [diff] [blame] | 19 | #include <pthread.h> |
Ian Rogers | 25fd14b | 2012-09-05 10:56:38 -0700 | [diff] [blame] | 20 | #include <vector> |
Elliott Hughes | 7c6169d | 2012-05-02 16:11:48 -0700 | [diff] [blame] | 21 | |
| 22 | #include "mutex.h" |
| 23 | #include "stl_util.h" |
| 24 | #include "stringprintf.h" |
| 25 | |
| 26 | #if defined(__APPLE__) |
| 27 | #include <libkern/OSAtomic.h> |
| 28 | #endif |
| 29 | #if defined(__arm__) |
| 30 | #include <machine/cpu-features.h> |
| 31 | #endif |
Elliott Hughes | 5ea047b | 2011-09-13 14:38:18 -0700 | [diff] [blame] | 32 | |
| 33 | namespace art { |
| 34 | |
Elliott Hughes | 5ea047b | 2011-09-13 14:38:18 -0700 | [diff] [blame] | 35 | #if defined(HAVE_MACOSX_IPC) |
Elliott Hughes | 7c6169d | 2012-05-02 16:11:48 -0700 | [diff] [blame] | 36 | #define NEED_MAC_QUASI_ATOMICS 1 |
Elliott Hughes | 5ea047b | 2011-09-13 14:38:18 -0700 | [diff] [blame] | 37 | |
| 38 | #elif defined(__i386__) || defined(__x86_64__) |
Elliott Hughes | 7c6169d | 2012-05-02 16:11:48 -0700 | [diff] [blame] | 39 | #define NEED_PTHREADS_QUASI_ATOMICS 1 |
Elliott Hughes | 5ea047b | 2011-09-13 14:38:18 -0700 | [diff] [blame] | 40 | |
Elliott Hughes | 7c6169d | 2012-05-02 16:11:48 -0700 | [diff] [blame] | 41 | #elif defined(__mips__) |
| 42 | #define NEED_PTHREADS_QUASI_ATOMICS 1 |
Elliott Hughes | 5ea047b | 2011-09-13 14:38:18 -0700 | [diff] [blame] | 43 | |
Elliott Hughes | 7c6169d | 2012-05-02 16:11:48 -0700 | [diff] [blame] | 44 | #elif defined(__arm__) |
| 45 | |
| 46 | #if defined(__ARM_HAVE_LDREXD) |
| 47 | #define NEED_ARM_LDREXD_QUASI_ATOMICS 1 |
| 48 | #else |
| 49 | #define NEED_PTHREADS_QUASI_ATOMICS 1 |
| 50 | #endif |
| 51 | |
Elliott Hughes | 7c6169d | 2012-05-02 16:11:48 -0700 | [diff] [blame] | 52 | #else |
| 53 | #error "QuasiAtomic unsupported on this platform" |
| 54 | #endif |
| 55 | |
| 56 | // ***************************************************************************** |
| 57 | |
| 58 | #if NEED_ARM_LDREXD_QUASI_ATOMICS |
| 59 | |
Elliott Hughes | 557e027 | 2011-09-29 10:52:22 -0700 | [diff] [blame] | 60 | static inline int64_t QuasiAtomicSwap64Impl(int64_t new_value, volatile int64_t* addr) { |
Elliott Hughes | 5ea047b | 2011-09-13 14:38:18 -0700 | [diff] [blame] | 61 | int64_t prev; |
| 62 | int status; |
| 63 | do { |
Elliott Hughes | 7c6169d | 2012-05-02 16:11:48 -0700 | [diff] [blame] | 64 | __asm__ __volatile__("@ QuasiAtomic::Swap64\n" |
Elliott Hughes | 5ea047b | 2011-09-13 14:38:18 -0700 | [diff] [blame] | 65 | "ldrexd %0, %H0, [%3]\n" |
| 66 | "strexd %1, %4, %H4, [%3]" |
| 67 | : "=&r" (prev), "=&r" (status), "+m"(*addr) |
| 68 | : "r" (addr), "r" (new_value) |
| 69 | : "cc"); |
| 70 | } while (__builtin_expect(status != 0, 0)); |
| 71 | return prev; |
| 72 | } |
| 73 | |
Elliott Hughes | 7c6169d | 2012-05-02 16:11:48 -0700 | [diff] [blame] | 74 | int64_t QuasiAtomic::Swap64(int64_t new_value, volatile int64_t* addr) { |
Elliott Hughes | 557e027 | 2011-09-29 10:52:22 -0700 | [diff] [blame] | 75 | return QuasiAtomicSwap64Impl(new_value, addr); |
| 76 | } |
| 77 | |
Elliott Hughes | 7c6169d | 2012-05-02 16:11:48 -0700 | [diff] [blame] | 78 | int64_t QuasiAtomic::Swap64Sync(int64_t new_value, volatile int64_t* addr) { |
Elliott Hughes | 557e027 | 2011-09-29 10:52:22 -0700 | [diff] [blame] | 79 | ANDROID_MEMBAR_STORE(); |
| 80 | int64_t old_value = QuasiAtomicSwap64Impl(new_value, addr); |
| 81 | ANDROID_MEMBAR_FULL(); |
| 82 | return old_value; |
| 83 | } |
| 84 | |
Elliott Hughes | 7c6169d | 2012-05-02 16:11:48 -0700 | [diff] [blame] | 85 | int64_t QuasiAtomic::Read64(volatile const int64_t* addr) { |
| 86 | int64_t value; |
| 87 | __asm__ __volatile__("@ QuasiAtomic::Read64\n" |
| 88 | "ldrexd %0, %H0, [%1]" |
| 89 | : "=&r" (value) |
| 90 | : "r" (addr)); |
| 91 | return value; |
| 92 | } |
| 93 | |
| 94 | int QuasiAtomic::Cas64(int64_t old_value, int64_t new_value, volatile int64_t* addr) { |
Elliott Hughes | 5ea047b | 2011-09-13 14:38:18 -0700 | [diff] [blame] | 95 | int64_t prev; |
| 96 | int status; |
| 97 | do { |
Elliott Hughes | 7c6169d | 2012-05-02 16:11:48 -0700 | [diff] [blame] | 98 | __asm__ __volatile__("@ QuasiAtomic::Cas64\n" |
Elliott Hughes | 5ea047b | 2011-09-13 14:38:18 -0700 | [diff] [blame] | 99 | "ldrexd %0, %H0, [%3]\n" |
| 100 | "mov %1, #0\n" |
| 101 | "teq %0, %4\n" |
| 102 | "teqeq %H0, %H4\n" |
| 103 | "strexdeq %1, %5, %H5, [%3]" |
| 104 | : "=&r" (prev), "=&r" (status), "+m"(*addr) |
| 105 | : "r" (addr), "Ir" (old_value), "r" (new_value) |
| 106 | : "cc"); |
| 107 | } while (__builtin_expect(status != 0, 0)); |
| 108 | return prev != old_value; |
| 109 | } |
| 110 | |
Elliott Hughes | 7c6169d | 2012-05-02 16:11:48 -0700 | [diff] [blame] | 111 | #endif |
Elliott Hughes | 5ea047b | 2011-09-13 14:38:18 -0700 | [diff] [blame] | 112 | |
Elliott Hughes | 7c6169d | 2012-05-02 16:11:48 -0700 | [diff] [blame] | 113 | // ***************************************************************************** |
Elliott Hughes | 5ea047b | 2011-09-13 14:38:18 -0700 | [diff] [blame] | 114 | |
Elliott Hughes | 7c6169d | 2012-05-02 16:11:48 -0700 | [diff] [blame] | 115 | #if NEED_MAC_QUASI_ATOMICS |
Elliott Hughes | 5ea047b | 2011-09-13 14:38:18 -0700 | [diff] [blame] | 116 | |
Elliott Hughes | 7c6169d | 2012-05-02 16:11:48 -0700 | [diff] [blame] | 117 | static inline int64_t QuasiAtomicSwap64Impl(int64_t value, volatile int64_t* addr) { |
| 118 | int64_t old_value; |
| 119 | do { |
| 120 | old_value = *addr; |
| 121 | } while (QuasiAtomic::Cas64(old_value, value, addr)); |
Elliott Hughes | 557e027 | 2011-09-29 10:52:22 -0700 | [diff] [blame] | 122 | return old_value; |
| 123 | } |
| 124 | |
Elliott Hughes | 7c6169d | 2012-05-02 16:11:48 -0700 | [diff] [blame] | 125 | int64_t QuasiAtomic::Swap64(int64_t value, volatile int64_t* addr) { |
| 126 | return QuasiAtomicSwap64Impl(value, addr); |
Elliott Hughes | 5ea047b | 2011-09-13 14:38:18 -0700 | [diff] [blame] | 127 | } |
| 128 | |
Elliott Hughes | 7c6169d | 2012-05-02 16:11:48 -0700 | [diff] [blame] | 129 | int64_t QuasiAtomic::Swap64Sync(int64_t value, volatile int64_t* addr) { |
| 130 | ANDROID_MEMBAR_STORE(); |
| 131 | int64_t old_value = QuasiAtomicSwap64Impl(value, addr); |
| 132 | // TUNING: barriers can be avoided on some architectures. |
| 133 | ANDROID_MEMBAR_FULL(); |
| 134 | return old_value; |
| 135 | } |
Elliott Hughes | 5ea047b | 2011-09-13 14:38:18 -0700 | [diff] [blame] | 136 | |
Elliott Hughes | 7c6169d | 2012-05-02 16:11:48 -0700 | [diff] [blame] | 137 | int64_t QuasiAtomic::Read64(volatile const int64_t* addr) { |
| 138 | return OSAtomicAdd64Barrier(0, const_cast<volatile int64_t*>(addr)); |
| 139 | } |
Elliott Hughes | 5ea047b | 2011-09-13 14:38:18 -0700 | [diff] [blame] | 140 | |
Elliott Hughes | 7c6169d | 2012-05-02 16:11:48 -0700 | [diff] [blame] | 141 | int QuasiAtomic::Cas64(int64_t old_value, int64_t new_value, volatile int64_t* addr) { |
| 142 | return OSAtomicCompareAndSwap64Barrier(old_value, new_value, const_cast<int64_t*>(addr)) == 0; |
| 143 | } |
| 144 | |
| 145 | #endif |
| 146 | |
| 147 | // ***************************************************************************** |
| 148 | |
| 149 | #if NEED_PTHREADS_QUASI_ATOMICS |
| 150 | |
| 151 | // In the absence of a better implementation, we implement the 64-bit atomic |
| 152 | // operations through mutex locking. |
| 153 | |
| 154 | // We stripe across a bunch of different mutexes to reduce contention. |
| 155 | static const size_t kSwapLockCount = 32; |
| 156 | static std::vector<Mutex*>* gSwapLocks; |
| 157 | |
| 158 | void QuasiAtomic::Startup() { |
| 159 | gSwapLocks = new std::vector<Mutex*>; |
| 160 | for (size_t i = 0; i < kSwapLockCount; ++i) { |
| 161 | gSwapLocks->push_back(new Mutex(StringPrintf("QuasiAtomic stripe %d", i).c_str())); |
| 162 | } |
| 163 | } |
| 164 | |
| 165 | void QuasiAtomic::Shutdown() { |
| 166 | STLDeleteElements(gSwapLocks); |
| 167 | delete gSwapLocks; |
| 168 | } |
| 169 | |
| 170 | static inline Mutex& GetSwapLock(const volatile int64_t* addr) { |
| 171 | return *(*gSwapLocks)[((unsigned)(void*)(addr) >> 3U) % kSwapLockCount]; |
| 172 | } |
| 173 | |
| 174 | int64_t QuasiAtomic::Swap64(int64_t value, volatile int64_t* addr) { |
| 175 | MutexLock mu(GetSwapLock(addr)); |
| 176 | int64_t old_value = *addr; |
| 177 | *addr = value; |
| 178 | return old_value; |
| 179 | } |
| 180 | |
| 181 | int64_t QuasiAtomic::Swap64Sync(int64_t value, volatile int64_t* addr) { |
| 182 | // Same as QuasiAtomicSwap64 - mutex handles barrier. |
| 183 | return QuasiAtomic::Swap64(value, addr); |
| 184 | } |
| 185 | |
| 186 | int QuasiAtomic::Cas64(int64_t old_value, int64_t new_value, volatile int64_t* addr) { |
| 187 | MutexLock mu(GetSwapLock(addr)); |
Elliott Hughes | 5ea047b | 2011-09-13 14:38:18 -0700 | [diff] [blame] | 188 | if (*addr == old_value) { |
| 189 | *addr = new_value; |
Elliott Hughes | 7c6169d | 2012-05-02 16:11:48 -0700 | [diff] [blame] | 190 | return 0; |
Elliott Hughes | 5ea047b | 2011-09-13 14:38:18 -0700 | [diff] [blame] | 191 | } |
Elliott Hughes | 7c6169d | 2012-05-02 16:11:48 -0700 | [diff] [blame] | 192 | return 1; |
Elliott Hughes | 5ea047b | 2011-09-13 14:38:18 -0700 | [diff] [blame] | 193 | } |
| 194 | |
Elliott Hughes | 7c6169d | 2012-05-02 16:11:48 -0700 | [diff] [blame] | 195 | int64_t QuasiAtomic::Read64(volatile const int64_t* addr) { |
| 196 | MutexLock mu(GetSwapLock(addr)); |
| 197 | return *addr; |
Elliott Hughes | 5ea047b | 2011-09-13 14:38:18 -0700 | [diff] [blame] | 198 | } |
| 199 | |
Elliott Hughes | 5ea047b | 2011-09-13 14:38:18 -0700 | [diff] [blame] | 200 | #else |
Elliott Hughes | 7c6169d | 2012-05-02 16:11:48 -0700 | [diff] [blame] | 201 | |
| 202 | // The other implementations don't need any special setup. |
| 203 | void QuasiAtomic::Startup() {} |
| 204 | void QuasiAtomic::Shutdown() {} |
| 205 | |
Elliott Hughes | 5ea047b | 2011-09-13 14:38:18 -0700 | [diff] [blame] | 206 | #endif |
| 207 | |
Elliott Hughes | 5ea047b | 2011-09-13 14:38:18 -0700 | [diff] [blame] | 208 | } // namespace art |