Elliott Hughes | 5ea047b | 2011-09-13 14:38:18 -0700 | [diff] [blame] | 1 | /* |
| 2 | * Copyright (C) 2010 The Android Open Source Project |
| 3 | * |
| 4 | * Licensed under the Apache License, Version 2.0 (the "License"); |
| 5 | * you may not use this file except in compliance with the License. |
| 6 | * You may obtain a copy of the License at |
| 7 | * |
| 8 | * http://www.apache.org/licenses/LICENSE-2.0 |
| 9 | * |
| 10 | * Unless required by applicable law or agreed to in writing, software |
| 11 | * distributed under the License is distributed on an "AS IS" BASIS, |
| 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 13 | * See the License for the specific language governing permissions and |
| 14 | * limitations under the License. |
| 15 | */ |
| 16 | |
| 17 | #include "atomic.h" |
| 18 | |
Ian Rogers | 9adbff5 | 2013-01-23 18:19:03 -0800 | [diff] [blame] | 19 | #define NEED_SWAP_MUTEXES !defined(__arm__) && !defined(__i386__) |
Elliott Hughes | e222ee0 | 2012-12-13 14:41:43 -0800 | [diff] [blame] | 20 | |
Ian Rogers | 9adbff5 | 2013-01-23 18:19:03 -0800 | [diff] [blame] | 21 | #if NEED_SWAP_MUTEXES |
Ian Rogers | 25fd14b | 2012-09-05 10:56:38 -0700 | [diff] [blame] | 22 | #include <vector> |
Elliott Hughes | 76b6167 | 2012-12-12 17:47:30 -0800 | [diff] [blame] | 23 | #include "base/mutex.h" |
Elliott Hughes | 1aa246d | 2012-12-13 09:29:36 -0800 | [diff] [blame] | 24 | #include "base/stl_util.h" |
Elliott Hughes | e222ee0 | 2012-12-13 14:41:43 -0800 | [diff] [blame] | 25 | #include "base/stringprintf.h" |
Brian Carlstrom | 7846396 | 2013-11-07 15:48:35 -0800 | [diff] [blame] | 26 | #include "thread-inl.h" |
Elliott Hughes | 7c6169d | 2012-05-02 16:11:48 -0700 | [diff] [blame] | 27 | #endif |
Elliott Hughes | 5ea047b | 2011-09-13 14:38:18 -0700 | [diff] [blame] | 28 | |
| 29 | namespace art { |
| 30 | |
Ian Rogers | 9adbff5 | 2013-01-23 18:19:03 -0800 | [diff] [blame] | 31 | #if NEED_SWAP_MUTEXES |
| 32 | // We stripe across a bunch of different mutexes to reduce contention. |
| 33 | static const size_t kSwapMutexCount = 32; |
| 34 | static std::vector<Mutex*>* gSwapMutexes; |
Elliott Hughes | 5ea047b | 2011-09-13 14:38:18 -0700 | [diff] [blame] | 35 | |
Ian Rogers | 9adbff5 | 2013-01-23 18:19:03 -0800 | [diff] [blame] | 36 | static Mutex& GetSwapMutex(const volatile int64_t* addr) { |
Brian Carlstrom | 2d88862 | 2013-07-18 17:02:00 -0700 | [diff] [blame] | 37 | return *(*gSwapMutexes)[(reinterpret_cast<unsigned>(addr) >> 3U) % kSwapMutexCount]; |
Ian Rogers | 9adbff5 | 2013-01-23 18:19:03 -0800 | [diff] [blame] | 38 | } |
Elliott Hughes | 7c6169d | 2012-05-02 16:11:48 -0700 | [diff] [blame] | 39 | #endif |
| 40 | |
Ian Rogers | 9adbff5 | 2013-01-23 18:19:03 -0800 | [diff] [blame] | 41 | void QuasiAtomic::Startup() { |
| 42 | #if NEED_SWAP_MUTEXES |
| 43 | gSwapMutexes = new std::vector<Mutex*>; |
| 44 | for (size_t i = 0; i < kSwapMutexCount; ++i) { |
Ian Rogers | 0fed328 | 2013-04-19 11:13:20 -0700 | [diff] [blame] | 45 | gSwapMutexes->push_back(new Mutex("QuasiAtomic stripe")); |
Ian Rogers | 9adbff5 | 2013-01-23 18:19:03 -0800 | [diff] [blame] | 46 | } |
Elliott Hughes | 7c6169d | 2012-05-02 16:11:48 -0700 | [diff] [blame] | 47 | #endif |
Elliott Hughes | 5ea047b | 2011-09-13 14:38:18 -0700 | [diff] [blame] | 48 | } |
| 49 | |
Ian Rogers | 9adbff5 | 2013-01-23 18:19:03 -0800 | [diff] [blame] | 50 | void QuasiAtomic::Shutdown() { |
| 51 | #if NEED_SWAP_MUTEXES |
| 52 | STLDeleteElements(gSwapMutexes); |
| 53 | delete gSwapMutexes; |
| 54 | #endif |
Elliott Hughes | 557e027 | 2011-09-29 10:52:22 -0700 | [diff] [blame] | 55 | } |
| 56 | |
Elliott Hughes | 7c6169d | 2012-05-02 16:11:48 -0700 | [diff] [blame] | 57 | int64_t QuasiAtomic::Read64(volatile const int64_t* addr) { |
| 58 | int64_t value; |
Ian Rogers | 0fed328 | 2013-04-19 11:13:20 -0700 | [diff] [blame] | 59 | #if NEED_SWAP_MUTEXES |
| 60 | MutexLock mu(Thread::Current(), GetSwapMutex(addr)); |
| 61 | value = *addr; |
| 62 | #elif defined(__arm__) |
Ian Rogers | 9adbff5 | 2013-01-23 18:19:03 -0800 | [diff] [blame] | 63 | // Exclusive loads are defined not to tear, clearing the exclusive state isn't necessary. If we |
| 64 | // have LPAE (such as Cortex-A15) then ldrd would suffice. |
Elliott Hughes | 7c6169d | 2012-05-02 16:11:48 -0700 | [diff] [blame] | 65 | __asm__ __volatile__("@ QuasiAtomic::Read64\n" |
| 66 | "ldrexd %0, %H0, [%1]" |
| 67 | : "=&r" (value) |
| 68 | : "r" (addr)); |
Ian Rogers | 9adbff5 | 2013-01-23 18:19:03 -0800 | [diff] [blame] | 69 | #elif defined(__i386__) |
| 70 | __asm__ __volatile__( |
| 71 | "movq %1, %0\n" |
| 72 | : "=x" (value) |
| 73 | : "m" (*addr)); |
| 74 | #else |
Ian Rogers | 0fed328 | 2013-04-19 11:13:20 -0700 | [diff] [blame] | 75 | #error Unexpected architecture |
Ian Rogers | 9adbff5 | 2013-01-23 18:19:03 -0800 | [diff] [blame] | 76 | #endif |
Elliott Hughes | 7c6169d | 2012-05-02 16:11:48 -0700 | [diff] [blame] | 77 | return value; |
| 78 | } |
| 79 | |
Ian Rogers | 9adbff5 | 2013-01-23 18:19:03 -0800 | [diff] [blame] | 80 | void QuasiAtomic::Write64(volatile int64_t* addr, int64_t value) { |
Ian Rogers | 0fed328 | 2013-04-19 11:13:20 -0700 | [diff] [blame] | 81 | #if NEED_SWAP_MUTEXES |
| 82 | MutexLock mu(Thread::Current(), GetSwapMutex(addr)); |
| 83 | *addr = value; |
| 84 | #elif defined(__arm__) |
Ian Rogers | 9adbff5 | 2013-01-23 18:19:03 -0800 | [diff] [blame] | 85 | // The write is done as a swap so that the cache-line is in the exclusive state for the store. If |
| 86 | // we know that ARM architecture has LPAE (such as Cortex-A15) this isn't necessary and strd will |
| 87 | // suffice. |
| 88 | int64_t prev; |
| 89 | int status; |
| 90 | do { |
| 91 | __asm__ __volatile__("@ QuasiAtomic::Write64\n" |
| 92 | "ldrexd %0, %H0, [%3]\n" |
| 93 | "strexd %1, %4, %H4, [%3]" |
| 94 | : "=&r" (prev), "=&r" (status), "+m"(*addr) |
| 95 | : "r" (addr), "r" (value) |
| 96 | : "cc"); |
| 97 | } while (__builtin_expect(status != 0, 0)); |
| 98 | #elif defined(__i386__) |
| 99 | __asm__ __volatile__( |
| 100 | "movq %1, %0" |
| 101 | : "=m" (*addr) |
| 102 | : "x" (value)); |
| 103 | #else |
Ian Rogers | 0fed328 | 2013-04-19 11:13:20 -0700 | [diff] [blame] | 104 | #error Unexpected architecture |
Ian Rogers | 9adbff5 | 2013-01-23 18:19:03 -0800 | [diff] [blame] | 105 | #endif |
| 106 | } |
| 107 | |
| 108 | |
| 109 | bool QuasiAtomic::Cas64(int64_t old_value, int64_t new_value, volatile int64_t* addr) { |
Ian Rogers | 0fed328 | 2013-04-19 11:13:20 -0700 | [diff] [blame] | 110 | #if NEED_SWAP_MUTEXES |
| 111 | MutexLock mu(Thread::Current(), GetSwapMutex(addr)); |
| 112 | if (*addr == old_value) { |
| 113 | *addr = new_value; |
| 114 | return true; |
| 115 | } |
| 116 | return false; |
| 117 | #elif defined(__arm__) |
Elliott Hughes | 5ea047b | 2011-09-13 14:38:18 -0700 | [diff] [blame] | 118 | int64_t prev; |
| 119 | int status; |
| 120 | do { |
Elliott Hughes | 7c6169d | 2012-05-02 16:11:48 -0700 | [diff] [blame] | 121 | __asm__ __volatile__("@ QuasiAtomic::Cas64\n" |
Elliott Hughes | 5ea047b | 2011-09-13 14:38:18 -0700 | [diff] [blame] | 122 | "ldrexd %0, %H0, [%3]\n" |
| 123 | "mov %1, #0\n" |
| 124 | "teq %0, %4\n" |
| 125 | "teqeq %H0, %H4\n" |
| 126 | "strexdeq %1, %5, %H5, [%3]" |
| 127 | : "=&r" (prev), "=&r" (status), "+m"(*addr) |
| 128 | : "r" (addr), "Ir" (old_value), "r" (new_value) |
| 129 | : "cc"); |
| 130 | } while (__builtin_expect(status != 0, 0)); |
Ian Rogers | 0794b6a | 2013-01-30 16:26:20 -0800 | [diff] [blame] | 131 | return prev == old_value; |
Ian Rogers | 9adbff5 | 2013-01-23 18:19:03 -0800 | [diff] [blame] | 132 | #elif defined(__i386__) |
Ian Rogers | 1c653d5 | 2013-06-13 15:04:30 -0700 | [diff] [blame] | 133 | // The compiler does the right job and works better than inline assembly, especially with -O0 |
| 134 | // compilation. |
| 135 | return __sync_bool_compare_and_swap(addr, old_value, new_value); |
Elliott Hughes | 5ea047b | 2011-09-13 14:38:18 -0700 | [diff] [blame] | 136 | #else |
Ian Rogers | 0fed328 | 2013-04-19 11:13:20 -0700 | [diff] [blame] | 137 | #error Unexpected architecture |
Elliott Hughes | 5ea047b | 2011-09-13 14:38:18 -0700 | [diff] [blame] | 138 | #endif |
Ian Rogers | 9adbff5 | 2013-01-23 18:19:03 -0800 | [diff] [blame] | 139 | } |
| 140 | |
| 141 | bool QuasiAtomic::LongAtomicsUseMutexes() { |
| 142 | #if NEED_SWAP_MUTEXES |
| 143 | return true; |
| 144 | #else |
| 145 | return false; |
| 146 | #endif |
| 147 | } |
Elliott Hughes | 5ea047b | 2011-09-13 14:38:18 -0700 | [diff] [blame] | 148 | |
Elliott Hughes | 5ea047b | 2011-09-13 14:38:18 -0700 | [diff] [blame] | 149 | } // namespace art |