blob: 87de506a851c316835507fd0edcb895d10df5d76 [file] [log] [blame]
Elliott Hughes5ea047b2011-09-13 14:38:18 -07001/*
2 * Copyright (C) 2008 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
Brian Carlstromfc0e3212013-07-17 14:40:12 -070017#ifndef ART_RUNTIME_ATOMIC_H_
18#define ART_RUNTIME_ATOMIC_H_
Elliott Hughes5ea047b2011-09-13 14:38:18 -070019
Elliott Hughes7c6169d2012-05-02 16:11:48 -070020#include <stdint.h>
Ian Rogers3e5cf302014-05-20 16:40:37 -070021#include <atomic>
Ian Rogers3e5cf302014-05-20 16:40:37 -070022#include <limits>
Ian Rogersb122a4b2013-11-19 18:00:50 -080023#include <vector>
Elliott Hughes7c6169d2012-05-02 16:11:48 -070024
Ian Rogersa9844542014-04-21 17:01:02 -070025#include "base/logging.h"
Elliott Hughes76160052012-12-12 16:31:20 -080026#include "base/macros.h"
Elliott Hughes5ea047b2011-09-13 14:38:18 -070027
28namespace art {
29
Ian Rogersb122a4b2013-11-19 18:00:50 -080030class Mutex;
31
Hans Boehm30359612014-05-21 17:46:23 -070032// QuasiAtomic encapsulates two separate facilities that we are
33// trying to move away from: "quasiatomic" 64 bit operations
34// and custom memory fences. For the time being, they remain
35// exposed. Clients should be converted to use either class Atomic
36// below whenever possible, and should eventually use C++11 atomics.
37// The two facilities that do not have a good C++11 analog are
38// ThreadFenceForConstructor and Atomic::*JavaData.
39//
Elliott Hughes7c6169d2012-05-02 16:11:48 -070040// NOTE: Two "quasiatomic" operations on the exact same memory address
41// are guaranteed to operate atomically with respect to each other,
42// but no guarantees are made about quasiatomic operations mixed with
43// non-quasiatomic operations on the same address, nor about
44// quasiatomic operations that are performed on partially-overlapping
45// memory.
Elliott Hughes7c6169d2012-05-02 16:11:48 -070046class QuasiAtomic {
Ian Rogers936b37f2014-02-14 00:52:24 -080047#if defined(__mips__) && !defined(__LP64__)
Ian Rogersb122a4b2013-11-19 18:00:50 -080048 static constexpr bool kNeedSwapMutexes = true;
Andreas Gampe1a5c4062015-01-15 12:10:47 -080049#elif defined(__mips__) && defined(__LP64__)
50 // TODO - mips64 still need this for Cas64 ???
51 static constexpr bool kNeedSwapMutexes = true;
Ian Rogersb122a4b2013-11-19 18:00:50 -080052#else
53 static constexpr bool kNeedSwapMutexes = false;
54#endif
55
Elliott Hughes7c6169d2012-05-02 16:11:48 -070056 public:
57 static void Startup();
Elliott Hughes5ea047b2011-09-13 14:38:18 -070058
Elliott Hughes7c6169d2012-05-02 16:11:48 -070059 static void Shutdown();
Elliott Hughes5ea047b2011-09-13 14:38:18 -070060
Ian Rogers9adbff52013-01-23 18:19:03 -080061 // Reads the 64-bit value at "addr" without tearing.
Ian Rogersb122a4b2013-11-19 18:00:50 -080062 static int64_t Read64(volatile const int64_t* addr) {
63 if (!kNeedSwapMutexes) {
Ian Rogersa9844542014-04-21 17:01:02 -070064 int64_t value;
65#if defined(__LP64__)
66 value = *addr;
67#else
68#if defined(__arm__)
69#if defined(__ARM_FEATURE_LPAE)
70 // With LPAE support (such as Cortex-A15) then ldrd is defined not to tear.
71 __asm__ __volatile__("@ QuasiAtomic::Read64\n"
72 "ldrd %0, %H0, %1"
73 : "=r" (value)
74 : "m" (*addr));
75#else
76 // Exclusive loads are defined not to tear, clearing the exclusive state isn't necessary.
77 __asm__ __volatile__("@ QuasiAtomic::Read64\n"
78 "ldrexd %0, %H0, %1"
79 : "=r" (value)
80 : "Q" (*addr));
81#endif
82#elif defined(__i386__)
83 __asm__ __volatile__(
84 "movq %1, %0\n"
85 : "=x" (value)
86 : "m" (*addr));
87#else
88 LOG(FATAL) << "Unsupported architecture";
89#endif
90#endif // defined(__LP64__)
91 return value;
Ian Rogersb122a4b2013-11-19 18:00:50 -080092 } else {
93 return SwapMutexRead64(addr);
94 }
95 }
Elliott Hughes7c6169d2012-05-02 16:11:48 -070096
Ian Rogers9adbff52013-01-23 18:19:03 -080097 // Writes to the 64-bit value at "addr" without tearing.
Ian Rogersa9844542014-04-21 17:01:02 -070098 static void Write64(volatile int64_t* addr, int64_t value) {
Ian Rogersb122a4b2013-11-19 18:00:50 -080099 if (!kNeedSwapMutexes) {
Ian Rogersa9844542014-04-21 17:01:02 -0700100#if defined(__LP64__)
101 *addr = value;
102#else
103#if defined(__arm__)
104#if defined(__ARM_FEATURE_LPAE)
105 // If we know that ARM architecture has LPAE (such as Cortex-A15) strd is defined not to tear.
106 __asm__ __volatile__("@ QuasiAtomic::Write64\n"
107 "strd %1, %H1, %0"
108 : "=m"(*addr)
109 : "r" (value));
110#else
111 // The write is done as a swap so that the cache-line is in the exclusive state for the store.
112 int64_t prev;
113 int status;
114 do {
115 __asm__ __volatile__("@ QuasiAtomic::Write64\n"
116 "ldrexd %0, %H0, %2\n"
117 "strexd %1, %3, %H3, %2"
118 : "=&r" (prev), "=&r" (status), "+Q"(*addr)
119 : "r" (value)
120 : "cc");
121 } while (UNLIKELY(status != 0));
122#endif
123#elif defined(__i386__)
124 __asm__ __volatile__(
125 "movq %1, %0"
126 : "=m" (*addr)
127 : "x" (value));
128#else
129 LOG(FATAL) << "Unsupported architecture";
130#endif
131#endif // defined(__LP64__)
Ian Rogersb122a4b2013-11-19 18:00:50 -0800132 } else {
Ian Rogersa9844542014-04-21 17:01:02 -0700133 SwapMutexWrite64(addr, value);
Ian Rogersb122a4b2013-11-19 18:00:50 -0800134 }
135 }
Ian Rogers9adbff52013-01-23 18:19:03 -0800136
137 // Atomically compare the value at "addr" to "old_value", if equal replace it with "new_value"
138 // and return true. Otherwise, don't swap, and return false.
Hans Boehm30359612014-05-21 17:46:23 -0700139 // This is fully ordered, i.e. it has C++11 memory_order_seq_cst
140 // semantics (assuming all other accesses use a mutex if this one does).
141 // This has "strong" semantics; if it fails then it is guaranteed that
142 // at some point during the execution of Cas64, *addr was not equal to
143 // old_value.
Ian Rogersb122a4b2013-11-19 18:00:50 -0800144 static bool Cas64(int64_t old_value, int64_t new_value, volatile int64_t* addr) {
145 if (!kNeedSwapMutexes) {
146 return __sync_bool_compare_and_swap(addr, old_value, new_value);
147 } else {
148 return SwapMutexCas64(old_value, new_value, addr);
149 }
150 }
Ian Rogers9adbff52013-01-23 18:19:03 -0800151
152 // Does the architecture provide reasonable atomic long operations or do we fall back on mutexes?
Ian Rogersb122a4b2013-11-19 18:00:50 -0800153 static bool LongAtomicsUseMutexes() {
Ian Rogers63c5dd02014-05-19 22:55:00 -0700154 return kNeedSwapMutexes;
Ian Rogersb122a4b2013-11-19 18:00:50 -0800155 }
156
Hans Boehma1ec0652014-06-06 17:13:03 -0700157 static void ThreadFenceAcquire() {
Hans Boehm30359612014-05-21 17:46:23 -0700158 std::atomic_thread_fence(std::memory_order_acquire);
159 }
160
Hans Boehma1ec0652014-06-06 17:13:03 -0700161 static void ThreadFenceRelease() {
Hans Boehm30359612014-05-21 17:46:23 -0700162 std::atomic_thread_fence(std::memory_order_release);
163 }
164
165 static void ThreadFenceForConstructor() {
166 #if defined(__aarch64__)
167 __asm__ __volatile__("dmb ishst" : : : "memory");
168 #else
169 std::atomic_thread_fence(std::memory_order_release);
170 #endif
171 }
172
173 static void ThreadFenceSequentiallyConsistent() {
174 std::atomic_thread_fence(std::memory_order_seq_cst);
175 }
176
Elliott Hughes7c6169d2012-05-02 16:11:48 -0700177 private:
Ian Rogersb122a4b2013-11-19 18:00:50 -0800178 static Mutex* GetSwapMutex(const volatile int64_t* addr);
179 static int64_t SwapMutexRead64(volatile const int64_t* addr);
180 static void SwapMutexWrite64(volatile int64_t* addr, int64_t val);
181 static bool SwapMutexCas64(int64_t old_value, int64_t new_value, volatile int64_t* addr);
182
183 // We stripe across a bunch of different mutexes to reduce contention.
184 static constexpr size_t kSwapMutexCount = 32;
185 static std::vector<Mutex*>* gSwapMutexes;
186
Elliott Hughes7c6169d2012-05-02 16:11:48 -0700187 DISALLOW_COPY_AND_ASSIGN(QuasiAtomic);
188};
Elliott Hughes5ea047b2011-09-13 14:38:18 -0700189
Hans Boehm30359612014-05-21 17:46:23 -0700190template<typename T>
Dan Albertaab0f862014-08-11 16:38:02 -0700191class PACKED(sizeof(T)) Atomic : public std::atomic<T> {
Hans Boehm30359612014-05-21 17:46:23 -0700192 public:
Dan Albert6a3f8d92014-08-12 11:48:34 -0700193 Atomic<T>() : std::atomic<T>(0) { }
Hans Boehm30359612014-05-21 17:46:23 -0700194
195 explicit Atomic<T>(T value) : std::atomic<T>(value) { }
196
197 // Load from memory without ordering or synchronization constraints.
198 T LoadRelaxed() const {
199 return this->load(std::memory_order_relaxed);
200 }
201
202 // Word tearing allowed, but may race.
203 // TODO: Optimize?
204 // There has been some discussion of eventually disallowing word
205 // tearing for Java data loads.
206 T LoadJavaData() const {
207 return this->load(std::memory_order_relaxed);
208 }
209
210 // Load from memory with a total ordering.
211 // Corresponds exactly to a Java volatile load.
212 T LoadSequentiallyConsistent() const {
213 return this->load(std::memory_order_seq_cst);
214 }
215
216 // Store to memory without ordering or synchronization constraints.
217 void StoreRelaxed(T desired) {
218 this->store(desired, std::memory_order_relaxed);
219 }
220
221 // Word tearing allowed, but may race.
222 void StoreJavaData(T desired) {
223 this->store(desired, std::memory_order_relaxed);
224 }
225
226 // Store to memory with release ordering.
227 void StoreRelease(T desired) {
228 this->store(desired, std::memory_order_release);
229 }
230
231 // Store to memory with a total ordering.
232 void StoreSequentiallyConsistent(T desired) {
233 this->store(desired, std::memory_order_seq_cst);
234 }
235
236 // Atomically replace the value with desired value if it matches the expected value.
237 // Participates in total ordering of atomic operations.
238 bool CompareExchangeStrongSequentiallyConsistent(T expected_value, T desired_value) {
239 return this->compare_exchange_strong(expected_value, desired_value, std::memory_order_seq_cst);
240 }
241
242 // The same, except it may fail spuriously.
243 bool CompareExchangeWeakSequentiallyConsistent(T expected_value, T desired_value) {
244 return this->compare_exchange_weak(expected_value, desired_value, std::memory_order_seq_cst);
245 }
246
247 // Atomically replace the value with desired value if it matches the expected value. Doesn't
248 // imply ordering or synchronization constraints.
249 bool CompareExchangeStrongRelaxed(T expected_value, T desired_value) {
250 return this->compare_exchange_strong(expected_value, desired_value, std::memory_order_relaxed);
251 }
252
253 // The same, except it may fail spuriously.
254 bool CompareExchangeWeakRelaxed(T expected_value, T desired_value) {
255 return this->compare_exchange_weak(expected_value, desired_value, std::memory_order_relaxed);
256 }
257
258 // Atomically replace the value with desired value if it matches the expected value. Prior writes
259 // made to other memory locations by the thread that did the release become visible in this
260 // thread.
261 bool CompareExchangeWeakAcquire(T expected_value, T desired_value) {
262 return this->compare_exchange_weak(expected_value, desired_value, std::memory_order_acquire);
263 }
264
265 // Atomically replace the value with desired value if it matches the expected value. prior writes
266 // to other memory locations become visible to the threads that do a consume or an acquire on the
267 // same location.
268 bool CompareExchangeWeakRelease(T expected_value, T desired_value) {
269 return this->compare_exchange_weak(expected_value, desired_value, std::memory_order_release);
270 }
271
272 T FetchAndAddSequentiallyConsistent(const T value) {
273 return this->fetch_add(value, std::memory_order_seq_cst); // Return old_value.
274 }
275
276 T FetchAndSubSequentiallyConsistent(const T value) {
277 return this->fetch_sub(value, std::memory_order_seq_cst); // Return old value.
278 }
279
Ian Rogers8c1b5f72014-07-09 22:02:36 -0700280 T FetchAndOrSequentiallyConsistent(const T value) {
281 return this->fetch_or(value, std::memory_order_seq_cst); // Return old_value.
282 }
283
284 T FetchAndAndSequentiallyConsistent(const T value) {
285 return this->fetch_and(value, std::memory_order_seq_cst); // Return old_value.
286 }
287
Hans Boehm30359612014-05-21 17:46:23 -0700288 volatile T* Address() {
289 return reinterpret_cast<T*>(this);
290 }
291
292 static T MaxValue() {
293 return std::numeric_limits<T>::max();
294 }
Hans Boehm30359612014-05-21 17:46:23 -0700295};
296
Hans Boehm30359612014-05-21 17:46:23 -0700297typedef Atomic<int32_t> AtomicInteger;
298
Andreas Gampe575e78c2014-11-03 23:41:03 -0800299static_assert(sizeof(AtomicInteger) == sizeof(int32_t), "Weird AtomicInteger size");
300static_assert(alignof(AtomicInteger) == alignof(int32_t),
301 "AtomicInteger alignment differs from that of underlyingtype");
302static_assert(sizeof(Atomic<int64_t>) == sizeof(int64_t), "Weird Atomic<int64> size");
Dan Albertaab0f862014-08-11 16:38:02 -0700303
304// Assert the alignment of 64-bit integers is 64-bit. This isn't true on certain 32-bit
305// architectures (e.g. x86-32) but we know that 64-bit integers here are arranged to be 8-byte
306// aligned.
Hans Boehm2f4a2ed2014-06-06 18:17:43 -0700307#if defined(__LP64__)
Andreas Gampe575e78c2014-11-03 23:41:03 -0800308 static_assert(alignof(Atomic<int64_t>) == alignof(int64_t),
309 "Atomic<int64> alignment differs from that of underlying type");
Hans Boehm2f4a2ed2014-06-06 18:17:43 -0700310#endif
Ian Rogers3e5cf302014-05-20 16:40:37 -0700311
Elliott Hughes5ea047b2011-09-13 14:38:18 -0700312} // namespace art
313
Brian Carlstromfc0e3212013-07-17 14:40:12 -0700314#endif // ART_RUNTIME_ATOMIC_H_