blob: a472b67fcdce5b2e3310e14c1e1188941395fa91 [file] [log] [blame]
Elliott Hughes8daa0922011-09-11 13:46:25 -07001/*
2 * Copyright (C) 2011 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17#include "mutex.h"
18
19#include <errno.h>
Ian Rogersc604d732012-10-14 16:09:54 -070020#include <sys/time.h>
Elliott Hughes8daa0922011-09-11 13:46:25 -070021
Andreas Gampe46ee31b2016-12-14 10:11:49 -080022#include "android-base/stringprintf.h"
23
Hiroshi Yamauchi1afde132013-08-06 17:09:30 -070024#include "atomic.h"
Elliott Hughes07ed66b2012-12-12 18:34:25 -080025#include "base/logging.h"
Vladimir Marko80afd022015-05-19 18:08:00 +010026#include "base/time_utils.h"
Mathieu Chartier32ce2ad2016-03-04 14:58:03 -080027#include "base/systrace.h"
Ian Rogerscf7f1912014-10-22 22:06:39 -070028#include "base/value_object.h"
Ian Rogers693ff612013-02-01 10:56:12 -080029#include "mutex-inl.h"
Mathieu Chartier0795f232016-09-27 18:43:30 -070030#include "scoped_thread_state_change-inl.h"
Ian Rogers04d7aa92013-03-16 14:29:17 -070031#include "thread-inl.h"
Elliott Hughes8daa0922011-09-11 13:46:25 -070032
33namespace art {
34
Andreas Gampe46ee31b2016-12-14 10:11:49 -080035using android::base::StringPrintf;
36
David Sehrf42eb2c2016-10-19 13:20:45 -070037static Atomic<Locks::ClientCallback*> safe_to_call_abort_callback(nullptr);
38
Ian Rogers719d1a32014-03-06 12:13:39 -080039Mutex* Locks::abort_lock_ = nullptr;
Brian Carlstrom306db812014-09-05 13:01:41 -070040Mutex* Locks::alloc_tracker_lock_ = nullptr;
Andreas Gampe74240812014-04-17 10:35:09 -070041Mutex* Locks::allocated_monitor_ids_lock_ = nullptr;
Chao-ying Fu9e369312014-05-21 11:20:52 -070042Mutex* Locks::allocated_thread_ids_lock_ = nullptr;
Sebastien Hertzed2be172014-08-19 15:33:43 +020043ReaderWriterMutex* Locks::breakpoint_lock_ = nullptr;
Ian Rogers719d1a32014-03-06 12:13:39 -080044ReaderWriterMutex* Locks::classlinker_classes_lock_ = nullptr;
Brian Carlstrom306db812014-09-05 13:01:41 -070045Mutex* Locks::deoptimization_lock_ = nullptr;
Ian Rogers719d1a32014-03-06 12:13:39 -080046ReaderWriterMutex* Locks::heap_bitmap_lock_ = nullptr;
Mathieu Chartier9ef78b52014-09-25 17:03:12 -070047Mutex* Locks::instrument_entrypoints_lock_ = nullptr;
Mathieu Chartiera5a53ef2014-09-12 12:58:05 -070048Mutex* Locks::intern_table_lock_ = nullptr;
Andreas Gampec8089542017-01-16 12:41:12 -080049Mutex* Locks::jni_function_table_lock_ = nullptr;
Ian Rogers68d8b422014-07-17 11:09:10 -070050Mutex* Locks::jni_libraries_lock_ = nullptr;
Ian Rogers719d1a32014-03-06 12:13:39 -080051Mutex* Locks::logging_lock_ = nullptr;
Chao-ying Fu9e369312014-05-21 11:20:52 -070052Mutex* Locks::modify_ldt_lock_ = nullptr;
Yu Lieac44242015-06-29 10:50:03 +080053MutatorMutex* Locks::mutator_lock_ = nullptr;
Brian Carlstrom306db812014-09-05 13:01:41 -070054Mutex* Locks::profiler_lock_ = nullptr;
Nicolas Geoffray340dafa2016-11-18 16:03:10 +000055ReaderWriterMutex* Locks::verifier_deps_lock_ = nullptr;
Mathieu Chartierf9c6fc62015-10-07 11:44:05 -070056ReaderWriterMutex* Locks::oat_file_manager_lock_ = nullptr;
Richard Uhlera206c742016-05-24 15:04:22 -070057Mutex* Locks::host_dlopen_handles_lock_ = nullptr;
Mathieu Chartiera5a53ef2014-09-12 12:58:05 -070058Mutex* Locks::reference_processor_lock_ = nullptr;
59Mutex* Locks::reference_queue_cleared_references_lock_ = nullptr;
60Mutex* Locks::reference_queue_finalizer_references_lock_ = nullptr;
61Mutex* Locks::reference_queue_phantom_references_lock_ = nullptr;
62Mutex* Locks::reference_queue_soft_references_lock_ = nullptr;
63Mutex* Locks::reference_queue_weak_references_lock_ = nullptr;
Ian Rogers719d1a32014-03-06 12:13:39 -080064Mutex* Locks::runtime_shutdown_lock_ = nullptr;
Mingyao Yang063fc772016-08-02 11:02:54 -070065Mutex* Locks::cha_lock_ = nullptr;
Ian Rogers719d1a32014-03-06 12:13:39 -080066Mutex* Locks::thread_list_lock_ = nullptr;
Mathieu Chartier91e56692015-03-03 13:51:04 -080067ConditionVariable* Locks::thread_exit_cond_ = nullptr;
Ian Rogers719d1a32014-03-06 12:13:39 -080068Mutex* Locks::thread_suspend_count_lock_ = nullptr;
69Mutex* Locks::trace_lock_ = nullptr;
Ian Rogers719d1a32014-03-06 12:13:39 -080070Mutex* Locks::unexpected_signal_lock_ = nullptr;
Alex Light88fd7202017-06-30 08:31:59 -070071Mutex* Locks::user_code_suspension_lock_ = nullptr;
Andreas Gampe5bdb6552015-07-22 23:44:55 -070072Uninterruptible Roles::uninterruptible_;
Andreas Gampe05a364c2016-10-14 13:27:12 -070073ReaderWriterMutex* Locks::jni_globals_lock_ = nullptr;
74Mutex* Locks::jni_weak_globals_lock_ = nullptr;
Andreas Gampecc1b5352016-12-01 16:58:38 -080075ReaderWriterMutex* Locks::dex_lock_ = nullptr;
Hiroshi Yamauchia2224042017-02-08 16:35:45 -080076std::vector<BaseMutex*> Locks::expected_mutexes_on_weak_ref_access_;
Hiroshi Yamauchi8a433242017-03-07 14:39:22 -080077Atomic<const BaseMutex*> Locks::expected_mutexes_on_weak_ref_access_guard_;
Ian Rogers719d1a32014-03-06 12:13:39 -080078
79struct AllMutexData {
80 // A guard for all_mutexes_ that's not a mutex (Mutexes must CAS to acquire and busy wait).
81 Atomic<const BaseMutex*> all_mutexes_guard;
82 // All created mutexes guarded by all_mutexes_guard_.
83 std::set<BaseMutex*>* all_mutexes;
Mathieu Chartier2cebb242015-04-21 16:50:40 -070084 AllMutexData() : all_mutexes(nullptr) {}
Ian Rogers719d1a32014-03-06 12:13:39 -080085};
86static struct AllMutexData gAllMutexData[kAllMutexDataSize];
87
Ian Rogersc604d732012-10-14 16:09:54 -070088#if ART_USE_FUTEXES
89static bool ComputeRelativeTimeSpec(timespec* result_ts, const timespec& lhs, const timespec& rhs) {
Brian Carlstromfb6996f2013-07-18 18:21:14 -070090 const int32_t one_sec = 1000 * 1000 * 1000; // one second in nanoseconds.
Ian Rogersc604d732012-10-14 16:09:54 -070091 result_ts->tv_sec = lhs.tv_sec - rhs.tv_sec;
92 result_ts->tv_nsec = lhs.tv_nsec - rhs.tv_nsec;
93 if (result_ts->tv_nsec < 0) {
94 result_ts->tv_sec--;
95 result_ts->tv_nsec += one_sec;
96 } else if (result_ts->tv_nsec > one_sec) {
97 result_ts->tv_sec++;
98 result_ts->tv_nsec -= one_sec;
99 }
100 return result_ts->tv_sec < 0;
101}
102#endif
103
Ian Rogers6f3dbba2014-10-14 17:41:57 -0700104class ScopedAllMutexesLock FINAL {
Ian Rogers56edc432013-01-18 16:51:51 -0800105 public:
Brian Carlstrom93ba8932013-07-17 21:31:49 -0700106 explicit ScopedAllMutexesLock(const BaseMutex* mutex) : mutex_(mutex) {
Ian Rogers3e5cf302014-05-20 16:40:37 -0700107 while (!gAllMutexData->all_mutexes_guard.CompareExchangeWeakAcquire(0, mutex)) {
Ian Rogers56edc432013-01-18 16:51:51 -0800108 NanoSleep(100);
109 }
110 }
Ian Rogers6f3dbba2014-10-14 17:41:57 -0700111
Ian Rogers56edc432013-01-18 16:51:51 -0800112 ~ScopedAllMutexesLock() {
Ian Rogers3e5cf302014-05-20 16:40:37 -0700113 while (!gAllMutexData->all_mutexes_guard.CompareExchangeWeakRelease(mutex_, 0)) {
Ian Rogers56edc432013-01-18 16:51:51 -0800114 NanoSleep(100);
115 }
116 }
Ian Rogers6f3dbba2014-10-14 17:41:57 -0700117
Ian Rogers56edc432013-01-18 16:51:51 -0800118 private:
119 const BaseMutex* const mutex_;
120};
Ian Rogers56edc432013-01-18 16:51:51 -0800121
Hiroshi Yamauchi8a433242017-03-07 14:39:22 -0800122class Locks::ScopedExpectedMutexesOnWeakRefAccessLock FINAL {
123 public:
124 explicit ScopedExpectedMutexesOnWeakRefAccessLock(const BaseMutex* mutex) : mutex_(mutex) {
125 while (!Locks::expected_mutexes_on_weak_ref_access_guard_.CompareExchangeWeakAcquire(0,
126 mutex)) {
127 NanoSleep(100);
128 }
129 }
130
131 ~ScopedExpectedMutexesOnWeakRefAccessLock() {
132 while (!Locks::expected_mutexes_on_weak_ref_access_guard_.CompareExchangeWeakRelease(mutex_,
133 0)) {
134 NanoSleep(100);
135 }
136 }
137
138 private:
139 const BaseMutex* const mutex_;
140};
141
Ian Rogerscf7f1912014-10-22 22:06:39 -0700142// Scoped class that generates events at the beginning and end of lock contention.
143class ScopedContentionRecorder FINAL : public ValueObject {
144 public:
145 ScopedContentionRecorder(BaseMutex* mutex, uint64_t blocked_tid, uint64_t owner_tid)
Mathieu Chartier2cebb242015-04-21 16:50:40 -0700146 : mutex_(kLogLockContentions ? mutex : nullptr),
Ian Rogerscf7f1912014-10-22 22:06:39 -0700147 blocked_tid_(kLogLockContentions ? blocked_tid : 0),
148 owner_tid_(kLogLockContentions ? owner_tid : 0),
149 start_nano_time_(kLogLockContentions ? NanoTime() : 0) {
150 if (ATRACE_ENABLED()) {
151 std::string msg = StringPrintf("Lock contention on %s (owner tid: %" PRIu64 ")",
152 mutex->GetName(), owner_tid);
153 ATRACE_BEGIN(msg.c_str());
154 }
155 }
156
157 ~ScopedContentionRecorder() {
158 ATRACE_END();
159 if (kLogLockContentions) {
160 uint64_t end_nano_time = NanoTime();
161 mutex_->RecordContention(blocked_tid_, owner_tid_, end_nano_time - start_nano_time_);
162 }
163 }
164
165 private:
166 BaseMutex* const mutex_;
167 const uint64_t blocked_tid_;
168 const uint64_t owner_tid_;
169 const uint64_t start_nano_time_;
170};
171
Hiroshi Yamauchia2224042017-02-08 16:35:45 -0800172BaseMutex::BaseMutex(const char* name, LockLevel level)
173 : level_(level),
174 name_(name),
175 should_respond_to_empty_checkpoint_request_(false) {
Hiroshi Yamauchi1afde132013-08-06 17:09:30 -0700176 if (kLogLockContentions) {
177 ScopedAllMutexesLock mu(this);
Ian Rogersd9c4fc92013-10-01 19:45:43 -0700178 std::set<BaseMutex*>** all_mutexes_ptr = &gAllMutexData->all_mutexes;
Mathieu Chartier2cebb242015-04-21 16:50:40 -0700179 if (*all_mutexes_ptr == nullptr) {
Hiroshi Yamauchi1afde132013-08-06 17:09:30 -0700180 // We leak the global set of all mutexes to avoid ordering issues in global variable
181 // construction/destruction.
182 *all_mutexes_ptr = new std::set<BaseMutex*>();
183 }
184 (*all_mutexes_ptr)->insert(this);
Ian Rogers56edc432013-01-18 16:51:51 -0800185 }
Ian Rogers56edc432013-01-18 16:51:51 -0800186}
187
188BaseMutex::~BaseMutex() {
Hiroshi Yamauchi1afde132013-08-06 17:09:30 -0700189 if (kLogLockContentions) {
190 ScopedAllMutexesLock mu(this);
Ian Rogersd9c4fc92013-10-01 19:45:43 -0700191 gAllMutexData->all_mutexes->erase(this);
Hiroshi Yamauchi1afde132013-08-06 17:09:30 -0700192 }
Ian Rogers56edc432013-01-18 16:51:51 -0800193}
194
195void BaseMutex::DumpAll(std::ostream& os) {
Hiroshi Yamauchi1afde132013-08-06 17:09:30 -0700196 if (kLogLockContentions) {
197 os << "Mutex logging:\n";
198 ScopedAllMutexesLock mu(reinterpret_cast<const BaseMutex*>(-1));
Ian Rogersd9c4fc92013-10-01 19:45:43 -0700199 std::set<BaseMutex*>* all_mutexes = gAllMutexData->all_mutexes;
Mathieu Chartier2cebb242015-04-21 16:50:40 -0700200 if (all_mutexes == nullptr) {
Hiroshi Yamauchi1afde132013-08-06 17:09:30 -0700201 // No mutexes have been created yet during at startup.
202 return;
203 }
204 typedef std::set<BaseMutex*>::const_iterator It;
Ian Rogersd9c4fc92013-10-01 19:45:43 -0700205 os << "(Contended)\n";
Hiroshi Yamauchi1afde132013-08-06 17:09:30 -0700206 for (It it = all_mutexes->begin(); it != all_mutexes->end(); ++it) {
207 BaseMutex* mutex = *it;
208 if (mutex->HasEverContended()) {
209 mutex->Dump(os);
210 os << "\n";
211 }
212 }
213 os << "(Never contented)\n";
214 for (It it = all_mutexes->begin(); it != all_mutexes->end(); ++it) {
215 BaseMutex* mutex = *it;
216 if (!mutex->HasEverContended()) {
217 mutex->Dump(os);
218 os << "\n";
219 }
220 }
Ian Rogers56edc432013-01-18 16:51:51 -0800221 }
Ian Rogers56edc432013-01-18 16:51:51 -0800222}
Ian Rogers00f7d0e2012-07-19 15:28:27 -0700223
Ian Rogers81d425b2012-09-27 16:03:43 -0700224void BaseMutex::CheckSafeToWait(Thread* self) {
Mathieu Chartier2cebb242015-04-21 16:50:40 -0700225 if (self == nullptr) {
Ian Rogers00f7d0e2012-07-19 15:28:27 -0700226 CheckUnattachedThread(level_);
227 return;
228 }
Ian Rogers25fd14b2012-09-05 10:56:38 -0700229 if (kDebugLocking) {
Ian Rogersd9c4fc92013-10-01 19:45:43 -0700230 CHECK(self->GetHeldMutex(level_) == this || level_ == kMonitorLock)
231 << "Waiting on unacquired mutex: " << name_;
Ian Rogers25fd14b2012-09-05 10:56:38 -0700232 bool bad_mutexes_held = false;
Elliott Hughes0f827162013-02-26 12:12:58 -0800233 for (int i = kLockLevelCount - 1; i >= 0; --i) {
Ian Rogers25fd14b2012-09-05 10:56:38 -0700234 if (i != level_) {
Ian Rogers81d425b2012-09-27 16:03:43 -0700235 BaseMutex* held_mutex = self->GetHeldMutex(static_cast<LockLevel>(i));
Ian Rogersf3d874c2014-07-17 18:52:42 -0700236 // We expect waits to happen while holding the thread list suspend thread lock.
Mathieu Chartier2cebb242015-04-21 16:50:40 -0700237 if (held_mutex != nullptr) {
Elliott Hughes0f827162013-02-26 12:12:58 -0800238 LOG(ERROR) << "Holding \"" << held_mutex->name_ << "\" "
239 << "(level " << LockLevel(i) << ") while performing wait on "
240 << "\"" << name_ << "\" (level " << level_ << ")";
Ian Rogers25fd14b2012-09-05 10:56:38 -0700241 bad_mutexes_held = true;
242 }
Ian Rogers00f7d0e2012-07-19 15:28:27 -0700243 }
244 }
Nicolas Geoffraydb978712014-12-09 13:33:38 +0000245 if (gAborting == 0) { // Avoid recursive aborts.
246 CHECK(!bad_mutexes_held);
247 }
Ian Rogers00f7d0e2012-07-19 15:28:27 -0700248 }
Ian Rogers00f7d0e2012-07-19 15:28:27 -0700249}
250
Ian Rogers37f3c962014-07-17 11:25:30 -0700251void BaseMutex::ContentionLogData::AddToWaitTime(uint64_t value) {
Hiroshi Yamauchi1afde132013-08-06 17:09:30 -0700252 if (kLogLockContentions) {
253 // Atomically add value to wait_time.
Ian Rogers37f3c962014-07-17 11:25:30 -0700254 wait_time.FetchAndAddSequentiallyConsistent(value);
Hiroshi Yamauchi1afde132013-08-06 17:09:30 -0700255 }
256}
257
Brian Carlstrom0de79852013-07-25 22:29:58 -0700258void BaseMutex::RecordContention(uint64_t blocked_tid,
259 uint64_t owner_tid,
Hiroshi Yamauchi1afde132013-08-06 17:09:30 -0700260 uint64_t nano_time_blocked) {
261 if (kLogLockContentions) {
Ian Rogers3e5cf302014-05-20 16:40:37 -0700262 ContentionLogData* data = contention_log_data_;
Hiroshi Yamauchi1afde132013-08-06 17:09:30 -0700263 ++(data->contention_count);
264 data->AddToWaitTime(nano_time_blocked);
265 ContentionLogEntry* log = data->contention_log;
266 // This code is intentionally racy as it is only used for diagnostics.
Ian Rogers3e5cf302014-05-20 16:40:37 -0700267 uint32_t slot = data->cur_content_log_entry.LoadRelaxed();
Hiroshi Yamauchi1afde132013-08-06 17:09:30 -0700268 if (log[slot].blocked_tid == blocked_tid &&
269 log[slot].owner_tid == blocked_tid) {
270 ++log[slot].count;
271 } else {
272 uint32_t new_slot;
273 do {
Ian Rogers3e5cf302014-05-20 16:40:37 -0700274 slot = data->cur_content_log_entry.LoadRelaxed();
Hiroshi Yamauchi1afde132013-08-06 17:09:30 -0700275 new_slot = (slot + 1) % kContentionLogSize;
Ian Rogers3e5cf302014-05-20 16:40:37 -0700276 } while (!data->cur_content_log_entry.CompareExchangeWeakRelaxed(slot, new_slot));
Hiroshi Yamauchi1afde132013-08-06 17:09:30 -0700277 log[new_slot].blocked_tid = blocked_tid;
278 log[new_slot].owner_tid = owner_tid;
Ian Rogers3e5cf302014-05-20 16:40:37 -0700279 log[new_slot].count.StoreRelaxed(1);
Hiroshi Yamauchi1afde132013-08-06 17:09:30 -0700280 }
Ian Rogers56edc432013-01-18 16:51:51 -0800281 }
Ian Rogers56edc432013-01-18 16:51:51 -0800282}
283
Ian Rogers56edc432013-01-18 16:51:51 -0800284void BaseMutex::DumpContention(std::ostream& os) const {
Hiroshi Yamauchi1afde132013-08-06 17:09:30 -0700285 if (kLogLockContentions) {
Ian Rogers3e5cf302014-05-20 16:40:37 -0700286 const ContentionLogData* data = contention_log_data_;
Hiroshi Yamauchi1afde132013-08-06 17:09:30 -0700287 const ContentionLogEntry* log = data->contention_log;
Ian Rogers37f3c962014-07-17 11:25:30 -0700288 uint64_t wait_time = data->wait_time.LoadRelaxed();
Ian Rogers3e5cf302014-05-20 16:40:37 -0700289 uint32_t contention_count = data->contention_count.LoadRelaxed();
Hiroshi Yamauchi1afde132013-08-06 17:09:30 -0700290 if (contention_count == 0) {
291 os << "never contended";
292 } else {
293 os << "contended " << contention_count
Mathieu Chartier73d1e172014-04-11 17:53:48 -0700294 << " total wait of contender " << PrettyDuration(wait_time)
295 << " average " << PrettyDuration(wait_time / contention_count);
Hiroshi Yamauchi1afde132013-08-06 17:09:30 -0700296 SafeMap<uint64_t, size_t> most_common_blocker;
297 SafeMap<uint64_t, size_t> most_common_blocked;
Hiroshi Yamauchi1afde132013-08-06 17:09:30 -0700298 for (size_t i = 0; i < kContentionLogSize; ++i) {
299 uint64_t blocked_tid = log[i].blocked_tid;
300 uint64_t owner_tid = log[i].owner_tid;
Ian Rogers3e5cf302014-05-20 16:40:37 -0700301 uint32_t count = log[i].count.LoadRelaxed();
Hiroshi Yamauchi1afde132013-08-06 17:09:30 -0700302 if (count > 0) {
Mathieu Chartier73d1e172014-04-11 17:53:48 -0700303 auto it = most_common_blocked.find(blocked_tid);
Hiroshi Yamauchi1afde132013-08-06 17:09:30 -0700304 if (it != most_common_blocked.end()) {
305 most_common_blocked.Overwrite(blocked_tid, it->second + count);
306 } else {
307 most_common_blocked.Put(blocked_tid, count);
308 }
309 it = most_common_blocker.find(owner_tid);
310 if (it != most_common_blocker.end()) {
311 most_common_blocker.Overwrite(owner_tid, it->second + count);
312 } else {
313 most_common_blocker.Put(owner_tid, count);
314 }
Ian Rogers56edc432013-01-18 16:51:51 -0800315 }
316 }
Hiroshi Yamauchi1afde132013-08-06 17:09:30 -0700317 uint64_t max_tid = 0;
318 size_t max_tid_count = 0;
Mathieu Chartier73d1e172014-04-11 17:53:48 -0700319 for (const auto& pair : most_common_blocked) {
320 if (pair.second > max_tid_count) {
321 max_tid = pair.first;
322 max_tid_count = pair.second;
Hiroshi Yamauchi1afde132013-08-06 17:09:30 -0700323 }
Ian Rogers56edc432013-01-18 16:51:51 -0800324 }
Hiroshi Yamauchi1afde132013-08-06 17:09:30 -0700325 if (max_tid != 0) {
326 os << " sample shows most blocked tid=" << max_tid;
Ian Rogers56edc432013-01-18 16:51:51 -0800327 }
Hiroshi Yamauchi1afde132013-08-06 17:09:30 -0700328 max_tid = 0;
329 max_tid_count = 0;
Mathieu Chartier73d1e172014-04-11 17:53:48 -0700330 for (const auto& pair : most_common_blocker) {
331 if (pair.second > max_tid_count) {
332 max_tid = pair.first;
333 max_tid_count = pair.second;
Hiroshi Yamauchi1afde132013-08-06 17:09:30 -0700334 }
335 }
336 if (max_tid != 0) {
337 os << " sample shows tid=" << max_tid << " owning during this time";
338 }
Ian Rogers56edc432013-01-18 16:51:51 -0800339 }
340 }
Ian Rogers56edc432013-01-18 16:51:51 -0800341}
342
343
Ian Rogers81d425b2012-09-27 16:03:43 -0700344Mutex::Mutex(const char* name, LockLevel level, bool recursive)
Ian Rogers00f7d0e2012-07-19 15:28:27 -0700345 : BaseMutex(name, level), recursive_(recursive), recursion_count_(0) {
Ian Rogersc604d732012-10-14 16:09:54 -0700346#if ART_USE_FUTEXES
Ian Rogersc7190692014-07-08 23:50:26 -0700347 DCHECK_EQ(0, state_.LoadRelaxed());
Ian Rogers3e5cf302014-05-20 16:40:37 -0700348 DCHECK_EQ(0, num_contenders_.LoadRelaxed());
Ian Rogers00f7d0e2012-07-19 15:28:27 -0700349#else
Ian Rogersc5f17732014-06-05 20:48:42 -0700350 CHECK_MUTEX_CALL(pthread_mutex_init, (&mutex_, nullptr));
Ian Rogers00f7d0e2012-07-19 15:28:27 -0700351#endif
Ian Rogersc5f17732014-06-05 20:48:42 -0700352 exclusive_owner_ = 0;
Elliott Hughes8daa0922011-09-11 13:46:25 -0700353}
354
David Sehrf42eb2c2016-10-19 13:20:45 -0700355// Helper to allow checking shutdown while locking for thread safety.
356static bool IsSafeToCallAbortSafe() {
357 MutexLock mu(Thread::Current(), *Locks::runtime_shutdown_lock_);
358 return Locks::IsSafeToCallAbortRacy();
Andreas Gampe8f1fa102015-01-22 19:48:51 -0800359}
360
Elliott Hughes8daa0922011-09-11 13:46:25 -0700361Mutex::~Mutex() {
David Sehrf42eb2c2016-10-19 13:20:45 -0700362 bool safe_to_call_abort = Locks::IsSafeToCallAbortRacy();
Ian Rogersc604d732012-10-14 16:09:54 -0700363#if ART_USE_FUTEXES
Ian Rogersc7190692014-07-08 23:50:26 -0700364 if (state_.LoadRelaxed() != 0) {
David Sehrf42eb2c2016-10-19 13:20:45 -0700365 LOG(safe_to_call_abort ? FATAL : WARNING)
366 << "destroying mutex with owner: " << exclusive_owner_;
Ian Rogersc604d732012-10-14 16:09:54 -0700367 } else {
Andreas Gampe8f1fa102015-01-22 19:48:51 -0800368 if (exclusive_owner_ != 0) {
David Sehrf42eb2c2016-10-19 13:20:45 -0700369 LOG(safe_to_call_abort ? FATAL : WARNING)
370 << "unexpectedly found an owner on unlocked mutex " << name_;
Andreas Gampe8f1fa102015-01-22 19:48:51 -0800371 }
372 if (num_contenders_.LoadSequentiallyConsistent() != 0) {
David Sehrf42eb2c2016-10-19 13:20:45 -0700373 LOG(safe_to_call_abort ? FATAL : WARNING)
374 << "unexpectedly found a contender on mutex " << name_;
Mathieu Chartiercef50f02014-12-09 17:38:52 -0800375 }
Ian Rogersc604d732012-10-14 16:09:54 -0700376 }
377#else
Elliott Hughese62934d2012-04-09 11:24:29 -0700378 // We can't use CHECK_MUTEX_CALL here because on shutdown a suspended daemon thread
379 // may still be using locks.
Elliott Hughes6b355752012-01-13 16:49:08 -0800380 int rc = pthread_mutex_destroy(&mutex_);
381 if (rc != 0) {
382 errno = rc;
David Sehrf42eb2c2016-10-19 13:20:45 -0700383 PLOG(safe_to_call_abort ? FATAL : WARNING)
384 << "pthread_mutex_destroy failed for " << name_;
Elliott Hughes6b355752012-01-13 16:49:08 -0800385 }
Ian Rogersc604d732012-10-14 16:09:54 -0700386#endif
Elliott Hughes8daa0922011-09-11 13:46:25 -0700387}
388
Ian Rogers81d425b2012-09-27 16:03:43 -0700389void Mutex::ExclusiveLock(Thread* self) {
Mathieu Chartier2cebb242015-04-21 16:50:40 -0700390 DCHECK(self == nullptr || self == Thread::Current());
Ian Rogers25fd14b2012-09-05 10:56:38 -0700391 if (kDebugLocking && !recursive_) {
Ian Rogers81d425b2012-09-27 16:03:43 -0700392 AssertNotHeld(self);
Ian Rogers25fd14b2012-09-05 10:56:38 -0700393 }
Ian Rogers81d425b2012-09-27 16:03:43 -0700394 if (!recursive_ || !IsExclusiveHeld(self)) {
Ian Rogersc604d732012-10-14 16:09:54 -0700395#if ART_USE_FUTEXES
396 bool done = false;
397 do {
Ian Rogersc7190692014-07-08 23:50:26 -0700398 int32_t cur_state = state_.LoadRelaxed();
Hiroshi Yamauchi967a0ad2013-09-10 16:24:21 -0700399 if (LIKELY(cur_state == 0)) {
Ian Rogersc7190692014-07-08 23:50:26 -0700400 // Change state from 0 to 1 and impose load/store ordering appropriate for lock acquisition.
401 done = state_.CompareExchangeWeakAcquire(0 /* cur_state */, 1 /* new state */);
Ian Rogersc604d732012-10-14 16:09:54 -0700402 } else {
403 // Failed to acquire, hang up.
Hiroshi Yamauchib3733082013-08-12 17:28:49 -0700404 ScopedContentionRecorder scr(this, SafeGetTid(self), GetExclusiveOwnerTid());
Ian Rogersb122a4b2013-11-19 18:00:50 -0800405 num_contenders_++;
Hiroshi Yamauchia2224042017-02-08 16:35:45 -0800406 if (UNLIKELY(should_respond_to_empty_checkpoint_request_)) {
407 self->CheckEmptyCheckpointFromMutex();
408 }
Mathieu Chartier2cebb242015-04-21 16:50:40 -0700409 if (futex(state_.Address(), FUTEX_WAIT, 1, nullptr, nullptr, 0) != 0) {
Brian Carlstrom0de79852013-07-25 22:29:58 -0700410 // EAGAIN and EINTR both indicate a spurious failure, try again from the beginning.
411 // We don't use TEMP_FAILURE_RETRY so we can intentionally retry to acquire the lock.
412 if ((errno != EAGAIN) && (errno != EINTR)) {
Ian Rogersc604d732012-10-14 16:09:54 -0700413 PLOG(FATAL) << "futex wait failed for " << name_;
414 }
415 }
Ian Rogersb122a4b2013-11-19 18:00:50 -0800416 num_contenders_--;
Ian Rogersc604d732012-10-14 16:09:54 -0700417 }
Brian Carlstromdf629502013-07-17 22:39:56 -0700418 } while (!done);
Ian Rogersc7190692014-07-08 23:50:26 -0700419 DCHECK_EQ(state_.LoadRelaxed(), 1);
Ian Rogersc604d732012-10-14 16:09:54 -0700420#else
Ian Rogers00f7d0e2012-07-19 15:28:27 -0700421 CHECK_MUTEX_CALL(pthread_mutex_lock, (&mutex_));
Ian Rogersc604d732012-10-14 16:09:54 -0700422#endif
Ian Rogersc5f17732014-06-05 20:48:42 -0700423 DCHECK_EQ(exclusive_owner_, 0U);
424 exclusive_owner_ = SafeGetTid(self);
Ian Rogers81d425b2012-09-27 16:03:43 -0700425 RegisterAsLocked(self);
Ian Rogers00f7d0e2012-07-19 15:28:27 -0700426 }
427 recursion_count_++;
Ian Rogers25fd14b2012-09-05 10:56:38 -0700428 if (kDebugLocking) {
429 CHECK(recursion_count_ == 1 || recursive_) << "Unexpected recursion count on mutex: "
430 << name_ << " " << recursion_count_;
Ian Rogers81d425b2012-09-27 16:03:43 -0700431 AssertHeld(self);
Ian Rogers25fd14b2012-09-05 10:56:38 -0700432 }
Elliott Hughes8daa0922011-09-11 13:46:25 -0700433}
434
Ian Rogers81d425b2012-09-27 16:03:43 -0700435bool Mutex::ExclusiveTryLock(Thread* self) {
Mathieu Chartier2cebb242015-04-21 16:50:40 -0700436 DCHECK(self == nullptr || self == Thread::Current());
Ian Rogers25fd14b2012-09-05 10:56:38 -0700437 if (kDebugLocking && !recursive_) {
Ian Rogers81d425b2012-09-27 16:03:43 -0700438 AssertNotHeld(self);
Ian Rogers25fd14b2012-09-05 10:56:38 -0700439 }
Ian Rogers81d425b2012-09-27 16:03:43 -0700440 if (!recursive_ || !IsExclusiveHeld(self)) {
Ian Rogersc604d732012-10-14 16:09:54 -0700441#if ART_USE_FUTEXES
442 bool done = false;
443 do {
Ian Rogersc7190692014-07-08 23:50:26 -0700444 int32_t cur_state = state_.LoadRelaxed();
Ian Rogersc604d732012-10-14 16:09:54 -0700445 if (cur_state == 0) {
Ian Rogersc7190692014-07-08 23:50:26 -0700446 // Change state from 0 to 1 and impose load/store ordering appropriate for lock acquisition.
447 done = state_.CompareExchangeWeakAcquire(0 /* cur_state */, 1 /* new state */);
Ian Rogersc604d732012-10-14 16:09:54 -0700448 } else {
449 return false;
450 }
Brian Carlstromdf629502013-07-17 22:39:56 -0700451 } while (!done);
Ian Rogersc7190692014-07-08 23:50:26 -0700452 DCHECK_EQ(state_.LoadRelaxed(), 1);
Ian Rogersc604d732012-10-14 16:09:54 -0700453#else
Ian Rogers00f7d0e2012-07-19 15:28:27 -0700454 int result = pthread_mutex_trylock(&mutex_);
455 if (result == EBUSY) {
456 return false;
457 }
458 if (result != 0) {
459 errno = result;
460 PLOG(FATAL) << "pthread_mutex_trylock failed for " << name_;
461 }
Ian Rogersc604d732012-10-14 16:09:54 -0700462#endif
Ian Rogersc5f17732014-06-05 20:48:42 -0700463 DCHECK_EQ(exclusive_owner_, 0U);
464 exclusive_owner_ = SafeGetTid(self);
Ian Rogers81d425b2012-09-27 16:03:43 -0700465 RegisterAsLocked(self);
Elliott Hughes8daa0922011-09-11 13:46:25 -0700466 }
Ian Rogers00f7d0e2012-07-19 15:28:27 -0700467 recursion_count_++;
Ian Rogers25fd14b2012-09-05 10:56:38 -0700468 if (kDebugLocking) {
469 CHECK(recursion_count_ == 1 || recursive_) << "Unexpected recursion count on mutex: "
470 << name_ << " " << recursion_count_;
Ian Rogers81d425b2012-09-27 16:03:43 -0700471 AssertHeld(self);
Ian Rogers25fd14b2012-09-05 10:56:38 -0700472 }
Elliott Hughes8daa0922011-09-11 13:46:25 -0700473 return true;
474}
475
Ian Rogers81d425b2012-09-27 16:03:43 -0700476void Mutex::ExclusiveUnlock(Thread* self) {
Mathieu Chartiereb0a1792014-12-15 17:23:45 -0800477 if (kIsDebugBuild && self != nullptr && self != Thread::Current()) {
478 std::string name1 = "<null>";
479 std::string name2 = "<null>";
480 if (self != nullptr) {
481 self->GetThreadName(name1);
482 }
483 if (Thread::Current() != nullptr) {
484 Thread::Current()->GetThreadName(name2);
485 }
Mathieu Chartier4c101102015-01-27 17:14:16 -0800486 LOG(FATAL) << GetName() << " level=" << level_ << " self=" << name1
487 << " Thread::Current()=" << name2;
Mathieu Chartiereb0a1792014-12-15 17:23:45 -0800488 }
Ian Rogers81d425b2012-09-27 16:03:43 -0700489 AssertHeld(self);
Ian Rogersc5f17732014-06-05 20:48:42 -0700490 DCHECK_NE(exclusive_owner_, 0U);
Ian Rogers00f7d0e2012-07-19 15:28:27 -0700491 recursion_count_--;
492 if (!recursive_ || recursion_count_ == 0) {
Ian Rogers25fd14b2012-09-05 10:56:38 -0700493 if (kDebugLocking) {
494 CHECK(recursion_count_ == 0 || recursive_) << "Unexpected recursion count on mutex: "
495 << name_ << " " << recursion_count_;
496 }
Ian Rogers81d425b2012-09-27 16:03:43 -0700497 RegisterAsUnlocked(self);
Ian Rogersc604d732012-10-14 16:09:54 -0700498#if ART_USE_FUTEXES
Ian Rogersc5f17732014-06-05 20:48:42 -0700499 bool done = false;
500 do {
Ian Rogersc7190692014-07-08 23:50:26 -0700501 int32_t cur_state = state_.LoadRelaxed();
Ian Rogersc5f17732014-06-05 20:48:42 -0700502 if (LIKELY(cur_state == 1)) {
Ian Rogersc5f17732014-06-05 20:48:42 -0700503 // We're no longer the owner.
504 exclusive_owner_ = 0;
Ian Rogersc7190692014-07-08 23:50:26 -0700505 // Change state to 0 and impose load/store ordering appropriate for lock release.
506 // Note, the relaxed loads below musn't reorder before the CompareExchange.
507 // TODO: the ordering here is non-trivial as state is split across 3 fields, fix by placing
508 // a status bit into the state on contention.
509 done = state_.CompareExchangeWeakSequentiallyConsistent(cur_state, 0 /* new state */);
Ian Rogersc5f17732014-06-05 20:48:42 -0700510 if (LIKELY(done)) { // Spurious fail?
Ian Rogersc7190692014-07-08 23:50:26 -0700511 // Wake a contender.
Ian Rogersc5f17732014-06-05 20:48:42 -0700512 if (UNLIKELY(num_contenders_.LoadRelaxed() > 0)) {
Mathieu Chartier2cebb242015-04-21 16:50:40 -0700513 futex(state_.Address(), FUTEX_WAKE, 1, nullptr, nullptr, 0);
Ian Rogersc5f17732014-06-05 20:48:42 -0700514 }
515 }
516 } else {
517 // Logging acquires the logging lock, avoid infinite recursion in that case.
518 if (this != Locks::logging_lock_) {
519 LOG(FATAL) << "Unexpected state_ in unlock " << cur_state << " for " << name_;
520 } else {
Andreas Gampe3fec9ac2016-09-13 10:47:28 -0700521 LogHelper::LogLineLowStack(__FILE__,
522 __LINE__,
523 ::android::base::FATAL_WITHOUT_ABORT,
524 StringPrintf("Unexpected state_ %d in unlock for %s",
525 cur_state, name_).c_str());
Ian Rogersc5f17732014-06-05 20:48:42 -0700526 _exit(1);
Ian Rogersc604d732012-10-14 16:09:54 -0700527 }
528 }
Ian Rogersc5f17732014-06-05 20:48:42 -0700529 } while (!done);
Ian Rogersc604d732012-10-14 16:09:54 -0700530#else
Ian Rogersc5f17732014-06-05 20:48:42 -0700531 exclusive_owner_ = 0;
Ian Rogers00f7d0e2012-07-19 15:28:27 -0700532 CHECK_MUTEX_CALL(pthread_mutex_unlock, (&mutex_));
Ian Rogersc604d732012-10-14 16:09:54 -0700533#endif
Ian Rogers00f7d0e2012-07-19 15:28:27 -0700534 }
Elliott Hughes8daa0922011-09-11 13:46:25 -0700535}
536
Ian Rogers56edc432013-01-18 16:51:51 -0800537void Mutex::Dump(std::ostream& os) const {
538 os << (recursive_ ? "recursive " : "non-recursive ")
539 << name_
540 << " level=" << static_cast<int>(level_)
541 << " rec=" << recursion_count_
542 << " owner=" << GetExclusiveOwnerTid() << " ";
543 DumpContention(os);
Ian Rogers01ae5802012-09-28 16:14:01 -0700544}
545
546std::ostream& operator<<(std::ostream& os, const Mutex& mu) {
Ian Rogers56edc432013-01-18 16:51:51 -0800547 mu.Dump(os);
548 return os;
Ian Rogers01ae5802012-09-28 16:14:01 -0700549}
550
Hiroshi Yamauchia2224042017-02-08 16:35:45 -0800551void Mutex::WakeupToRespondToEmptyCheckpoint() {
552#if ART_USE_FUTEXES
553 // Wake up all the waiters so they will respond to the emtpy checkpoint.
554 DCHECK(should_respond_to_empty_checkpoint_request_);
555 if (UNLIKELY(num_contenders_.LoadRelaxed() > 0)) {
556 futex(state_.Address(), FUTEX_WAKE, -1, nullptr, nullptr, 0);
557 }
558#else
559 LOG(FATAL) << "Non futex case isn't supported.";
560#endif
561}
562
Brian Carlstrom02c8cc62013-07-18 15:54:44 -0700563ReaderWriterMutex::ReaderWriterMutex(const char* name, LockLevel level)
564 : BaseMutex(name, level)
Ian Rogers81d425b2012-09-27 16:03:43 -0700565#if ART_USE_FUTEXES
Ian Rogersc5f17732014-06-05 20:48:42 -0700566 , state_(0), num_pending_readers_(0), num_pending_writers_(0)
Ian Rogers81d425b2012-09-27 16:03:43 -0700567#endif
Brian Carlstrom7934ac22013-07-26 10:54:15 -0700568{ // NOLINT(whitespace/braces)
Ian Rogers81d425b2012-09-27 16:03:43 -0700569#if !ART_USE_FUTEXES
Ian Rogersc5f17732014-06-05 20:48:42 -0700570 CHECK_MUTEX_CALL(pthread_rwlock_init, (&rwlock_, nullptr));
Ian Rogers81d425b2012-09-27 16:03:43 -0700571#endif
Ian Rogersc5f17732014-06-05 20:48:42 -0700572 exclusive_owner_ = 0;
Ian Rogers00f7d0e2012-07-19 15:28:27 -0700573}
574
575ReaderWriterMutex::~ReaderWriterMutex() {
Ian Rogers81d425b2012-09-27 16:03:43 -0700576#if ART_USE_FUTEXES
Ian Rogersc7190692014-07-08 23:50:26 -0700577 CHECK_EQ(state_.LoadRelaxed(), 0);
Ian Rogers81d425b2012-09-27 16:03:43 -0700578 CHECK_EQ(exclusive_owner_, 0U);
Ian Rogersc7190692014-07-08 23:50:26 -0700579 CHECK_EQ(num_pending_readers_.LoadRelaxed(), 0);
Ian Rogers3e5cf302014-05-20 16:40:37 -0700580 CHECK_EQ(num_pending_writers_.LoadRelaxed(), 0);
Ian Rogers81d425b2012-09-27 16:03:43 -0700581#else
Ian Rogers00f7d0e2012-07-19 15:28:27 -0700582 // We can't use CHECK_MUTEX_CALL here because on shutdown a suspended daemon thread
583 // may still be using locks.
584 int rc = pthread_rwlock_destroy(&rwlock_);
585 if (rc != 0) {
586 errno = rc;
David Sehrf42eb2c2016-10-19 13:20:45 -0700587 bool is_safe_to_call_abort = IsSafeToCallAbortSafe();
588 PLOG(is_safe_to_call_abort ? FATAL : WARNING) << "pthread_rwlock_destroy failed for " << name_;
Brian Carlstromcd74c4b2012-01-23 13:21:00 -0800589 }
Ian Rogers81d425b2012-09-27 16:03:43 -0700590#endif
Ian Rogers00f7d0e2012-07-19 15:28:27 -0700591}
592
Ian Rogers81d425b2012-09-27 16:03:43 -0700593void ReaderWriterMutex::ExclusiveLock(Thread* self) {
Mathieu Chartier2cebb242015-04-21 16:50:40 -0700594 DCHECK(self == nullptr || self == Thread::Current());
Ian Rogers81d425b2012-09-27 16:03:43 -0700595 AssertNotExclusiveHeld(self);
596#if ART_USE_FUTEXES
597 bool done = false;
598 do {
Ian Rogersc7190692014-07-08 23:50:26 -0700599 int32_t cur_state = state_.LoadRelaxed();
Hiroshi Yamauchi967a0ad2013-09-10 16:24:21 -0700600 if (LIKELY(cur_state == 0)) {
Ian Rogersc7190692014-07-08 23:50:26 -0700601 // Change state from 0 to -1 and impose load/store ordering appropriate for lock acquisition.
602 done = state_.CompareExchangeWeakAcquire(0 /* cur_state*/, -1 /* new state */);
Ian Rogers81d425b2012-09-27 16:03:43 -0700603 } else {
604 // Failed to acquire, hang up.
Hiroshi Yamauchib3733082013-08-12 17:28:49 -0700605 ScopedContentionRecorder scr(this, SafeGetTid(self), GetExclusiveOwnerTid());
Ian Rogersc7190692014-07-08 23:50:26 -0700606 ++num_pending_writers_;
Hiroshi Yamauchia2224042017-02-08 16:35:45 -0800607 if (UNLIKELY(should_respond_to_empty_checkpoint_request_)) {
608 self->CheckEmptyCheckpointFromMutex();
609 }
Mathieu Chartier2cebb242015-04-21 16:50:40 -0700610 if (futex(state_.Address(), FUTEX_WAIT, cur_state, nullptr, nullptr, 0) != 0) {
Brian Carlstrom0de79852013-07-25 22:29:58 -0700611 // EAGAIN and EINTR both indicate a spurious failure, try again from the beginning.
612 // We don't use TEMP_FAILURE_RETRY so we can intentionally retry to acquire the lock.
613 if ((errno != EAGAIN) && (errno != EINTR)) {
Ian Rogers81d425b2012-09-27 16:03:43 -0700614 PLOG(FATAL) << "futex wait failed for " << name_;
615 }
616 }
Ian Rogersc7190692014-07-08 23:50:26 -0700617 --num_pending_writers_;
Ian Rogers81d425b2012-09-27 16:03:43 -0700618 }
Brian Carlstromdf629502013-07-17 22:39:56 -0700619 } while (!done);
Ian Rogersc7190692014-07-08 23:50:26 -0700620 DCHECK_EQ(state_.LoadRelaxed(), -1);
Ian Rogers81d425b2012-09-27 16:03:43 -0700621#else
Ian Rogers00f7d0e2012-07-19 15:28:27 -0700622 CHECK_MUTEX_CALL(pthread_rwlock_wrlock, (&rwlock_));
Ian Rogers81d425b2012-09-27 16:03:43 -0700623#endif
Ian Rogersc5f17732014-06-05 20:48:42 -0700624 DCHECK_EQ(exclusive_owner_, 0U);
625 exclusive_owner_ = SafeGetTid(self);
Ian Rogers81d425b2012-09-27 16:03:43 -0700626 RegisterAsLocked(self);
627 AssertExclusiveHeld(self);
Ian Rogers00f7d0e2012-07-19 15:28:27 -0700628}
629
Ian Rogers81d425b2012-09-27 16:03:43 -0700630void ReaderWriterMutex::ExclusiveUnlock(Thread* self) {
Mathieu Chartier2cebb242015-04-21 16:50:40 -0700631 DCHECK(self == nullptr || self == Thread::Current());
Ian Rogers81d425b2012-09-27 16:03:43 -0700632 AssertExclusiveHeld(self);
633 RegisterAsUnlocked(self);
Ian Rogersc5f17732014-06-05 20:48:42 -0700634 DCHECK_NE(exclusive_owner_, 0U);
Ian Rogers81d425b2012-09-27 16:03:43 -0700635#if ART_USE_FUTEXES
636 bool done = false;
637 do {
Ian Rogersc7190692014-07-08 23:50:26 -0700638 int32_t cur_state = state_.LoadRelaxed();
Hiroshi Yamauchi967a0ad2013-09-10 16:24:21 -0700639 if (LIKELY(cur_state == -1)) {
Ian Rogers81d425b2012-09-27 16:03:43 -0700640 // We're no longer the owner.
641 exclusive_owner_ = 0;
Ian Rogersc7190692014-07-08 23:50:26 -0700642 // Change state from -1 to 0 and impose load/store ordering appropriate for lock release.
643 // Note, the relaxed loads below musn't reorder before the CompareExchange.
644 // TODO: the ordering here is non-trivial as state is split across 3 fields, fix by placing
645 // a status bit into the state on contention.
646 done = state_.CompareExchangeWeakSequentiallyConsistent(-1 /* cur_state*/, 0 /* new state */);
647 if (LIKELY(done)) { // Weak CAS may fail spuriously.
Ian Rogers81d425b2012-09-27 16:03:43 -0700648 // Wake any waiters.
Ian Rogersc7190692014-07-08 23:50:26 -0700649 if (UNLIKELY(num_pending_readers_.LoadRelaxed() > 0 ||
650 num_pending_writers_.LoadRelaxed() > 0)) {
Mathieu Chartier2cebb242015-04-21 16:50:40 -0700651 futex(state_.Address(), FUTEX_WAKE, -1, nullptr, nullptr, 0);
Ian Rogers81d425b2012-09-27 16:03:43 -0700652 }
653 }
654 } else {
655 LOG(FATAL) << "Unexpected state_:" << cur_state << " for " << name_;
656 }
Brian Carlstromdf629502013-07-17 22:39:56 -0700657 } while (!done);
Ian Rogers81d425b2012-09-27 16:03:43 -0700658#else
Ian Rogersc5f17732014-06-05 20:48:42 -0700659 exclusive_owner_ = 0;
Ian Rogers00f7d0e2012-07-19 15:28:27 -0700660 CHECK_MUTEX_CALL(pthread_rwlock_unlock, (&rwlock_));
Ian Rogers81d425b2012-09-27 16:03:43 -0700661#endif
Ian Rogers00f7d0e2012-07-19 15:28:27 -0700662}
663
Ian Rogers66aee5c2012-08-15 17:17:47 -0700664#if HAVE_TIMED_RWLOCK
Ian Rogersc604d732012-10-14 16:09:54 -0700665bool ReaderWriterMutex::ExclusiveLockWithTimeout(Thread* self, int64_t ms, int32_t ns) {
Mathieu Chartier2cebb242015-04-21 16:50:40 -0700666 DCHECK(self == nullptr || self == Thread::Current());
Ian Rogers81d425b2012-09-27 16:03:43 -0700667#if ART_USE_FUTEXES
668 bool done = false;
Ian Rogersc604d732012-10-14 16:09:54 -0700669 timespec end_abs_ts;
tony.ys_liu071e48e2015-01-14 18:28:03 +0800670 InitTimeSpec(true, CLOCK_MONOTONIC, ms, ns, &end_abs_ts);
Ian Rogers81d425b2012-09-27 16:03:43 -0700671 do {
Ian Rogersc7190692014-07-08 23:50:26 -0700672 int32_t cur_state = state_.LoadRelaxed();
Ian Rogers81d425b2012-09-27 16:03:43 -0700673 if (cur_state == 0) {
Ian Rogersc7190692014-07-08 23:50:26 -0700674 // Change state from 0 to -1 and impose load/store ordering appropriate for lock acquisition.
675 done = state_.CompareExchangeWeakAcquire(0 /* cur_state */, -1 /* new state */);
Ian Rogers81d425b2012-09-27 16:03:43 -0700676 } else {
677 // Failed to acquire, hang up.
Ian Rogersc604d732012-10-14 16:09:54 -0700678 timespec now_abs_ts;
tony.ys_liu071e48e2015-01-14 18:28:03 +0800679 InitTimeSpec(true, CLOCK_MONOTONIC, 0, 0, &now_abs_ts);
Ian Rogersc604d732012-10-14 16:09:54 -0700680 timespec rel_ts;
681 if (ComputeRelativeTimeSpec(&rel_ts, end_abs_ts, now_abs_ts)) {
682 return false; // Timed out.
683 }
Hiroshi Yamauchib3733082013-08-12 17:28:49 -0700684 ScopedContentionRecorder scr(this, SafeGetTid(self), GetExclusiveOwnerTid());
Ian Rogersc7190692014-07-08 23:50:26 -0700685 ++num_pending_writers_;
Hiroshi Yamauchia2224042017-02-08 16:35:45 -0800686 if (UNLIKELY(should_respond_to_empty_checkpoint_request_)) {
687 self->CheckEmptyCheckpointFromMutex();
688 }
Mathieu Chartier2cebb242015-04-21 16:50:40 -0700689 if (futex(state_.Address(), FUTEX_WAIT, cur_state, &rel_ts, nullptr, 0) != 0) {
Ian Rogers81d425b2012-09-27 16:03:43 -0700690 if (errno == ETIMEDOUT) {
Ian Rogersc7190692014-07-08 23:50:26 -0700691 --num_pending_writers_;
Ian Rogersc604d732012-10-14 16:09:54 -0700692 return false; // Timed out.
Brian Carlstrom0de79852013-07-25 22:29:58 -0700693 } else if ((errno != EAGAIN) && (errno != EINTR)) {
694 // EAGAIN and EINTR both indicate a spurious failure,
695 // recompute the relative time out from now and try again.
696 // We don't use TEMP_FAILURE_RETRY so we can recompute rel_ts;
Ian Rogers81d425b2012-09-27 16:03:43 -0700697 PLOG(FATAL) << "timed futex wait failed for " << name_;
698 }
699 }
Ian Rogersc7190692014-07-08 23:50:26 -0700700 --num_pending_writers_;
Ian Rogers81d425b2012-09-27 16:03:43 -0700701 }
Brian Carlstromdf629502013-07-17 22:39:56 -0700702 } while (!done);
Ian Rogers81d425b2012-09-27 16:03:43 -0700703#else
Ian Rogersc604d732012-10-14 16:09:54 -0700704 timespec ts;
Brian Carlstrombcc29262012-11-02 11:36:03 -0700705 InitTimeSpec(true, CLOCK_REALTIME, ms, ns, &ts);
Ian Rogersc604d732012-10-14 16:09:54 -0700706 int result = pthread_rwlock_timedwrlock(&rwlock_, &ts);
Ian Rogers00f7d0e2012-07-19 15:28:27 -0700707 if (result == ETIMEDOUT) {
708 return false;
709 }
710 if (result != 0) {
711 errno = result;
Ian Rogersa5acfd32012-08-15 11:50:10 -0700712 PLOG(FATAL) << "pthread_rwlock_timedwrlock failed for " << name_;
Ian Rogers00f7d0e2012-07-19 15:28:27 -0700713 }
Ian Rogers81d425b2012-09-27 16:03:43 -0700714#endif
Ian Rogersc5f17732014-06-05 20:48:42 -0700715 exclusive_owner_ = SafeGetTid(self);
Ian Rogers81d425b2012-09-27 16:03:43 -0700716 RegisterAsLocked(self);
717 AssertSharedHeld(self);
Ian Rogers00f7d0e2012-07-19 15:28:27 -0700718 return true;
719}
Ian Rogers66aee5c2012-08-15 17:17:47 -0700720#endif
Ian Rogers00f7d0e2012-07-19 15:28:27 -0700721
Ian Rogers51d212e2014-10-23 17:48:20 -0700722#if ART_USE_FUTEXES
Ian Rogerscf7f1912014-10-22 22:06:39 -0700723void ReaderWriterMutex::HandleSharedLockContention(Thread* self, int32_t cur_state) {
724 // Owner holds it exclusively, hang up.
725 ScopedContentionRecorder scr(this, GetExclusiveOwnerTid(), SafeGetTid(self));
726 ++num_pending_readers_;
Hiroshi Yamauchia2224042017-02-08 16:35:45 -0800727 if (UNLIKELY(should_respond_to_empty_checkpoint_request_)) {
728 self->CheckEmptyCheckpointFromMutex();
729 }
Mathieu Chartier2cebb242015-04-21 16:50:40 -0700730 if (futex(state_.Address(), FUTEX_WAIT, cur_state, nullptr, nullptr, 0) != 0) {
Daniel Colascione6f4d1022016-11-21 14:35:42 -0800731 if (errno != EAGAIN && errno != EINTR) {
Ian Rogerscf7f1912014-10-22 22:06:39 -0700732 PLOG(FATAL) << "futex wait failed for " << name_;
733 }
734 }
735 --num_pending_readers_;
736}
Ian Rogers51d212e2014-10-23 17:48:20 -0700737#endif
Ian Rogerscf7f1912014-10-22 22:06:39 -0700738
Ian Rogers81d425b2012-09-27 16:03:43 -0700739bool ReaderWriterMutex::SharedTryLock(Thread* self) {
Mathieu Chartier2cebb242015-04-21 16:50:40 -0700740 DCHECK(self == nullptr || self == Thread::Current());
Ian Rogers81d425b2012-09-27 16:03:43 -0700741#if ART_USE_FUTEXES
742 bool done = false;
743 do {
Ian Rogersc7190692014-07-08 23:50:26 -0700744 int32_t cur_state = state_.LoadRelaxed();
Ian Rogers81d425b2012-09-27 16:03:43 -0700745 if (cur_state >= 0) {
Ian Rogersc7190692014-07-08 23:50:26 -0700746 // Add as an extra reader and impose load/store ordering appropriate for lock acquisition.
747 done = state_.CompareExchangeWeakAcquire(cur_state, cur_state + 1);
Ian Rogers81d425b2012-09-27 16:03:43 -0700748 } else {
749 // Owner holds it exclusively.
750 return false;
751 }
Brian Carlstromdf629502013-07-17 22:39:56 -0700752 } while (!done);
Ian Rogers81d425b2012-09-27 16:03:43 -0700753#else
Ian Rogers00f7d0e2012-07-19 15:28:27 -0700754 int result = pthread_rwlock_tryrdlock(&rwlock_);
755 if (result == EBUSY) {
756 return false;
757 }
758 if (result != 0) {
759 errno = result;
760 PLOG(FATAL) << "pthread_mutex_trylock failed for " << name_;
761 }
Ian Rogers81d425b2012-09-27 16:03:43 -0700762#endif
763 RegisterAsLocked(self);
764 AssertSharedHeld(self);
Ian Rogers00f7d0e2012-07-19 15:28:27 -0700765 return true;
766}
767
Ian Rogers81d425b2012-09-27 16:03:43 -0700768bool ReaderWriterMutex::IsSharedHeld(const Thread* self) const {
Mathieu Chartier2cebb242015-04-21 16:50:40 -0700769 DCHECK(self == nullptr || self == Thread::Current());
Ian Rogers00f7d0e2012-07-19 15:28:27 -0700770 bool result;
Mathieu Chartier2cebb242015-04-21 16:50:40 -0700771 if (UNLIKELY(self == nullptr)) { // Handle unattached threads.
Ian Rogers01ae5802012-09-28 16:14:01 -0700772 result = IsExclusiveHeld(self); // TODO: a better best effort here.
Ian Rogers00f7d0e2012-07-19 15:28:27 -0700773 } else {
774 result = (self->GetHeldMutex(level_) == this);
775 }
776 return result;
777}
778
Ian Rogers56edc432013-01-18 16:51:51 -0800779void ReaderWriterMutex::Dump(std::ostream& os) const {
780 os << name_
781 << " level=" << static_cast<int>(level_)
Mathieu Chartier5869a2c2014-10-08 14:26:23 -0700782 << " owner=" << GetExclusiveOwnerTid()
783#if ART_USE_FUTEXES
784 << " state=" << state_.LoadSequentiallyConsistent()
785 << " num_pending_writers=" << num_pending_writers_.LoadSequentiallyConsistent()
786 << " num_pending_readers=" << num_pending_readers_.LoadSequentiallyConsistent()
787#endif
788 << " ";
Ian Rogers56edc432013-01-18 16:51:51 -0800789 DumpContention(os);
Ian Rogers01ae5802012-09-28 16:14:01 -0700790}
791
792std::ostream& operator<<(std::ostream& os, const ReaderWriterMutex& mu) {
Ian Rogers56edc432013-01-18 16:51:51 -0800793 mu.Dump(os);
794 return os;
Ian Rogers01ae5802012-09-28 16:14:01 -0700795}
796
Yu Lieac44242015-06-29 10:50:03 +0800797std::ostream& operator<<(std::ostream& os, const MutatorMutex& mu) {
798 mu.Dump(os);
799 return os;
800}
801
Hiroshi Yamauchia2224042017-02-08 16:35:45 -0800802void ReaderWriterMutex::WakeupToRespondToEmptyCheckpoint() {
803#if ART_USE_FUTEXES
804 // Wake up all the waiters so they will respond to the emtpy checkpoint.
805 DCHECK(should_respond_to_empty_checkpoint_request_);
806 if (UNLIKELY(num_pending_readers_.LoadRelaxed() > 0 ||
807 num_pending_writers_.LoadRelaxed() > 0)) {
808 futex(state_.Address(), FUTEX_WAKE, -1, nullptr, nullptr, 0);
809 }
810#else
811 LOG(FATAL) << "Non futex case isn't supported.";
812#endif
813}
814
Ian Rogers23055dc2013-04-18 16:29:16 -0700815ConditionVariable::ConditionVariable(const char* name, Mutex& guard)
Ian Rogersc604d732012-10-14 16:09:54 -0700816 : name_(name), guard_(guard) {
817#if ART_USE_FUTEXES
Ian Rogers3e5cf302014-05-20 16:40:37 -0700818 DCHECK_EQ(0, sequence_.LoadRelaxed());
Ian Rogersc604d732012-10-14 16:09:54 -0700819 num_waiters_ = 0;
Ian Rogersc604d732012-10-14 16:09:54 -0700820#else
Narayan Kamath51b71022014-03-04 11:57:09 +0000821 pthread_condattr_t cond_attrs;
Ian Rogersc5f17732014-06-05 20:48:42 -0700822 CHECK_MUTEX_CALL(pthread_condattr_init, (&cond_attrs));
Narayan Kamath51b71022014-03-04 11:57:09 +0000823#if !defined(__APPLE__)
824 // Apple doesn't have CLOCK_MONOTONIC or pthread_condattr_setclock.
Ian Rogers51d212e2014-10-23 17:48:20 -0700825 CHECK_MUTEX_CALL(pthread_condattr_setclock, (&cond_attrs, CLOCK_MONOTONIC));
Narayan Kamath51b71022014-03-04 11:57:09 +0000826#endif
827 CHECK_MUTEX_CALL(pthread_cond_init, (&cond_, &cond_attrs));
Ian Rogersc604d732012-10-14 16:09:54 -0700828#endif
Elliott Hughes5f791332011-09-15 17:45:30 -0700829}
830
831ConditionVariable::~ConditionVariable() {
Ian Rogers5bd97c42012-11-27 02:38:26 -0800832#if ART_USE_FUTEXES
833 if (num_waiters_!= 0) {
David Sehrf42eb2c2016-10-19 13:20:45 -0700834 bool is_safe_to_call_abort = IsSafeToCallAbortSafe();
835 LOG(is_safe_to_call_abort ? FATAL : WARNING)
Andreas Gampe3fec9ac2016-09-13 10:47:28 -0700836 << "ConditionVariable::~ConditionVariable for " << name_
Ian Rogersd45f2012012-11-28 11:46:23 -0800837 << " called with " << num_waiters_ << " waiters.";
Ian Rogers5bd97c42012-11-27 02:38:26 -0800838 }
839#else
Elliott Hughese62934d2012-04-09 11:24:29 -0700840 // We can't use CHECK_MUTEX_CALL here because on shutdown a suspended daemon thread
841 // may still be using condition variables.
842 int rc = pthread_cond_destroy(&cond_);
843 if (rc != 0) {
844 errno = rc;
David Sehrf42eb2c2016-10-19 13:20:45 -0700845 bool is_safe_to_call_abort = IsSafeToCallAbortSafe();
846 PLOG(is_safe_to_call_abort ? FATAL : WARNING) << "pthread_cond_destroy failed for " << name_;
Elliott Hughese62934d2012-04-09 11:24:29 -0700847 }
Ian Rogersc604d732012-10-14 16:09:54 -0700848#endif
Elliott Hughes5f791332011-09-15 17:45:30 -0700849}
850
Ian Rogersc604d732012-10-14 16:09:54 -0700851void ConditionVariable::Broadcast(Thread* self) {
Mathieu Chartier2cebb242015-04-21 16:50:40 -0700852 DCHECK(self == nullptr || self == Thread::Current());
Ian Rogersc604d732012-10-14 16:09:54 -0700853 // TODO: enable below, there's a race in thread creation that causes false failures currently.
854 // guard_.AssertExclusiveHeld(self);
Mathieu Chartiere46cd752012-10-31 16:56:18 -0700855 DCHECK_EQ(guard_.GetExclusiveOwnerTid(), SafeGetTid(self));
Ian Rogersc604d732012-10-14 16:09:54 -0700856#if ART_USE_FUTEXES
Ian Rogersd45f2012012-11-28 11:46:23 -0800857 if (num_waiters_ > 0) {
Ian Rogersb122a4b2013-11-19 18:00:50 -0800858 sequence_++; // Indicate the broadcast occurred.
Ian Rogersc604d732012-10-14 16:09:54 -0700859 bool done = false;
860 do {
Ian Rogers3e5cf302014-05-20 16:40:37 -0700861 int32_t cur_sequence = sequence_.LoadRelaxed();
Ian Rogersd45f2012012-11-28 11:46:23 -0800862 // Requeue waiters onto mutex. The waiter holds the contender count on the mutex high ensuring
863 // mutex unlocks will awaken the requeued waiter thread.
Ian Rogersb122a4b2013-11-19 18:00:50 -0800864 done = futex(sequence_.Address(), FUTEX_CMP_REQUEUE, 0,
Ian Rogers5bd97c42012-11-27 02:38:26 -0800865 reinterpret_cast<const timespec*>(std::numeric_limits<int32_t>::max()),
Ian Rogersc7190692014-07-08 23:50:26 -0700866 guard_.state_.Address(), cur_sequence) != -1;
Ian Rogers5bd97c42012-11-27 02:38:26 -0800867 if (!done) {
Daniel Colascione6f4d1022016-11-21 14:35:42 -0800868 if (errno != EAGAIN && errno != EINTR) {
Ian Rogers5bd97c42012-11-27 02:38:26 -0800869 PLOG(FATAL) << "futex cmp requeue failed for " << name_;
870 }
Ian Rogers5bd97c42012-11-27 02:38:26 -0800871 }
Ian Rogersc604d732012-10-14 16:09:54 -0700872 } while (!done);
873 }
874#else
Elliott Hughes5f791332011-09-15 17:45:30 -0700875 CHECK_MUTEX_CALL(pthread_cond_broadcast, (&cond_));
Ian Rogersc604d732012-10-14 16:09:54 -0700876#endif
Elliott Hughes5f791332011-09-15 17:45:30 -0700877}
878
Ian Rogersc604d732012-10-14 16:09:54 -0700879void ConditionVariable::Signal(Thread* self) {
Mathieu Chartier2cebb242015-04-21 16:50:40 -0700880 DCHECK(self == nullptr || self == Thread::Current());
Ian Rogersc604d732012-10-14 16:09:54 -0700881 guard_.AssertExclusiveHeld(self);
882#if ART_USE_FUTEXES
Ian Rogersd45f2012012-11-28 11:46:23 -0800883 if (num_waiters_ > 0) {
Ian Rogersb122a4b2013-11-19 18:00:50 -0800884 sequence_++; // Indicate a signal occurred.
Ian Rogersc604d732012-10-14 16:09:54 -0700885 // Futex wake 1 waiter who will then come and in contend on mutex. It'd be nice to requeue them
886 // to avoid this, however, requeueing can only move all waiters.
Mathieu Chartier2cebb242015-04-21 16:50:40 -0700887 int num_woken = futex(sequence_.Address(), FUTEX_WAKE, 1, nullptr, nullptr, 0);
Ian Rogersd45f2012012-11-28 11:46:23 -0800888 // Check something was woken or else we changed sequence_ before they had chance to wait.
Ian Rogers5bd97c42012-11-27 02:38:26 -0800889 CHECK((num_woken == 0) || (num_woken == 1));
Ian Rogersc604d732012-10-14 16:09:54 -0700890 }
891#else
Elliott Hughes5f791332011-09-15 17:45:30 -0700892 CHECK_MUTEX_CALL(pthread_cond_signal, (&cond_));
Ian Rogersc604d732012-10-14 16:09:54 -0700893#endif
Elliott Hughes5f791332011-09-15 17:45:30 -0700894}
895
Ian Rogersc604d732012-10-14 16:09:54 -0700896void ConditionVariable::Wait(Thread* self) {
Ian Rogers1d54e732013-05-02 21:10:01 -0700897 guard_.CheckSafeToWait(self);
898 WaitHoldingLocks(self);
899}
900
901void ConditionVariable::WaitHoldingLocks(Thread* self) {
Mathieu Chartier2cebb242015-04-21 16:50:40 -0700902 DCHECK(self == nullptr || self == Thread::Current());
Ian Rogersc604d732012-10-14 16:09:54 -0700903 guard_.AssertExclusiveHeld(self);
Ian Rogersc604d732012-10-14 16:09:54 -0700904 unsigned int old_recursion_count = guard_.recursion_count_;
905#if ART_USE_FUTEXES
Ian Rogersc604d732012-10-14 16:09:54 -0700906 num_waiters_++;
Ian Rogersd45f2012012-11-28 11:46:23 -0800907 // Ensure the Mutex is contended so that requeued threads are awoken.
Ian Rogersb122a4b2013-11-19 18:00:50 -0800908 guard_.num_contenders_++;
Ian Rogersc604d732012-10-14 16:09:54 -0700909 guard_.recursion_count_ = 1;
Ian Rogers3e5cf302014-05-20 16:40:37 -0700910 int32_t cur_sequence = sequence_.LoadRelaxed();
Ian Rogersc604d732012-10-14 16:09:54 -0700911 guard_.ExclusiveUnlock(self);
Mathieu Chartier2cebb242015-04-21 16:50:40 -0700912 if (futex(sequence_.Address(), FUTEX_WAIT, cur_sequence, nullptr, nullptr, 0) != 0) {
Ian Rogersd45f2012012-11-28 11:46:23 -0800913 // Futex failed, check it is an expected error.
914 // EAGAIN == EWOULDBLK, so we let the caller try again.
915 // EINTR implies a signal was sent to this thread.
916 if ((errno != EINTR) && (errno != EAGAIN)) {
Ian Rogersc604d732012-10-14 16:09:54 -0700917 PLOG(FATAL) << "futex wait failed for " << name_;
918 }
919 }
Mathieu Chartier4d87df62016-01-07 15:14:19 -0800920 if (self != nullptr) {
921 JNIEnvExt* const env = self->GetJniEnv();
922 if (UNLIKELY(env != nullptr && env->runtime_deleted)) {
923 CHECK(self->IsDaemon());
924 // If the runtime has been deleted, then we cannot proceed. Just sleep forever. This may
925 // occur for user daemon threads that get a spurious wakeup. This occurs for test 132 with
926 // --host and --gdb.
927 // After we wake up, the runtime may have been shutdown, which means that this condition may
928 // have been deleted. It is not safe to retry the wait.
929 SleepForever();
930 }
931 }
Ian Rogersc604d732012-10-14 16:09:54 -0700932 guard_.ExclusiveLock(self);
Ian Rogersd45f2012012-11-28 11:46:23 -0800933 CHECK_GE(num_waiters_, 0);
Ian Rogersc604d732012-10-14 16:09:54 -0700934 num_waiters_--;
Ian Rogersd45f2012012-11-28 11:46:23 -0800935 // We awoke and so no longer require awakes from the guard_'s unlock.
Ian Rogers3e5cf302014-05-20 16:40:37 -0700936 CHECK_GE(guard_.num_contenders_.LoadRelaxed(), 0);
Ian Rogersb122a4b2013-11-19 18:00:50 -0800937 guard_.num_contenders_--;
Ian Rogersc604d732012-10-14 16:09:54 -0700938#else
Ian Rogersc5f17732014-06-05 20:48:42 -0700939 uint64_t old_owner = guard_.exclusive_owner_;
940 guard_.exclusive_owner_ = 0;
Ian Rogersc604d732012-10-14 16:09:54 -0700941 guard_.recursion_count_ = 0;
942 CHECK_MUTEX_CALL(pthread_cond_wait, (&cond_, &guard_.mutex_));
Ian Rogersc5f17732014-06-05 20:48:42 -0700943 guard_.exclusive_owner_ = old_owner;
Ian Rogersc604d732012-10-14 16:09:54 -0700944#endif
945 guard_.recursion_count_ = old_recursion_count;
Elliott Hughes5f791332011-09-15 17:45:30 -0700946}
947
Ian Rogers7b078e82014-09-10 14:44:24 -0700948bool ConditionVariable::TimedWait(Thread* self, int64_t ms, int32_t ns) {
Mathieu Chartier2cebb242015-04-21 16:50:40 -0700949 DCHECK(self == nullptr || self == Thread::Current());
Ian Rogers7b078e82014-09-10 14:44:24 -0700950 bool timed_out = false;
Ian Rogersc604d732012-10-14 16:09:54 -0700951 guard_.AssertExclusiveHeld(self);
Ian Rogers1d54e732013-05-02 21:10:01 -0700952 guard_.CheckSafeToWait(self);
Ian Rogersc604d732012-10-14 16:09:54 -0700953 unsigned int old_recursion_count = guard_.recursion_count_;
954#if ART_USE_FUTEXES
Ian Rogersc604d732012-10-14 16:09:54 -0700955 timespec rel_ts;
Ian Rogers5bd97c42012-11-27 02:38:26 -0800956 InitTimeSpec(false, CLOCK_REALTIME, ms, ns, &rel_ts);
Ian Rogersc604d732012-10-14 16:09:54 -0700957 num_waiters_++;
Ian Rogersd45f2012012-11-28 11:46:23 -0800958 // Ensure the Mutex is contended so that requeued threads are awoken.
Ian Rogersb122a4b2013-11-19 18:00:50 -0800959 guard_.num_contenders_++;
Ian Rogersc604d732012-10-14 16:09:54 -0700960 guard_.recursion_count_ = 1;
Ian Rogers3e5cf302014-05-20 16:40:37 -0700961 int32_t cur_sequence = sequence_.LoadRelaxed();
Ian Rogersc604d732012-10-14 16:09:54 -0700962 guard_.ExclusiveUnlock(self);
Mathieu Chartier2cebb242015-04-21 16:50:40 -0700963 if (futex(sequence_.Address(), FUTEX_WAIT, cur_sequence, &rel_ts, nullptr, 0) != 0) {
Ian Rogersc604d732012-10-14 16:09:54 -0700964 if (errno == ETIMEDOUT) {
Ian Rogersd45f2012012-11-28 11:46:23 -0800965 // Timed out we're done.
Ian Rogers7b078e82014-09-10 14:44:24 -0700966 timed_out = true;
Brian Carlstrom0de79852013-07-25 22:29:58 -0700967 } else if ((errno == EAGAIN) || (errno == EINTR)) {
Ian Rogersd45f2012012-11-28 11:46:23 -0800968 // A signal or ConditionVariable::Signal/Broadcast has come in.
Ian Rogersc604d732012-10-14 16:09:54 -0700969 } else {
970 PLOG(FATAL) << "timed futex wait failed for " << name_;
971 }
972 }
973 guard_.ExclusiveLock(self);
Ian Rogersd45f2012012-11-28 11:46:23 -0800974 CHECK_GE(num_waiters_, 0);
Ian Rogersc604d732012-10-14 16:09:54 -0700975 num_waiters_--;
Ian Rogersd45f2012012-11-28 11:46:23 -0800976 // We awoke and so no longer require awakes from the guard_'s unlock.
Ian Rogers3e5cf302014-05-20 16:40:37 -0700977 CHECK_GE(guard_.num_contenders_.LoadRelaxed(), 0);
Ian Rogersb122a4b2013-11-19 18:00:50 -0800978 guard_.num_contenders_--;
Ian Rogersc604d732012-10-14 16:09:54 -0700979#else
Narayan Kamath51b71022014-03-04 11:57:09 +0000980#if !defined(__APPLE__)
Ian Rogersc604d732012-10-14 16:09:54 -0700981 int clock = CLOCK_MONOTONIC;
Elliott Hughes5f791332011-09-15 17:45:30 -0700982#else
Ian Rogersc604d732012-10-14 16:09:54 -0700983 int clock = CLOCK_REALTIME;
Elliott Hughes5f791332011-09-15 17:45:30 -0700984#endif
Ian Rogersc5f17732014-06-05 20:48:42 -0700985 uint64_t old_owner = guard_.exclusive_owner_;
986 guard_.exclusive_owner_ = 0;
Ian Rogersc604d732012-10-14 16:09:54 -0700987 guard_.recursion_count_ = 0;
988 timespec ts;
Brian Carlstrombcc29262012-11-02 11:36:03 -0700989 InitTimeSpec(true, clock, ms, ns, &ts);
Narayan Kamath51b71022014-03-04 11:57:09 +0000990 int rc = TEMP_FAILURE_RETRY(pthread_cond_timedwait(&cond_, &guard_.mutex_, &ts));
Ian Rogers7b078e82014-09-10 14:44:24 -0700991 if (rc == ETIMEDOUT) {
992 timed_out = true;
993 } else if (rc != 0) {
Elliott Hughes5f791332011-09-15 17:45:30 -0700994 errno = rc;
995 PLOG(FATAL) << "TimedWait failed for " << name_;
996 }
Ian Rogersc5f17732014-06-05 20:48:42 -0700997 guard_.exclusive_owner_ = old_owner;
Ian Rogersc604d732012-10-14 16:09:54 -0700998#endif
999 guard_.recursion_count_ = old_recursion_count;
Ian Rogers7b078e82014-09-10 14:44:24 -07001000 return timed_out;
Elliott Hughes5f791332011-09-15 17:45:30 -07001001}
1002
Ian Rogers719d1a32014-03-06 12:13:39 -08001003void Locks::Init() {
1004 if (logging_lock_ != nullptr) {
1005 // Already initialized.
Nicolas Geoffray7f0a6d62014-05-26 14:01:46 +01001006 if (kRuntimeISA == kX86 || kRuntimeISA == kX86_64) {
Chao-ying Fu9e369312014-05-21 11:20:52 -07001007 DCHECK(modify_ldt_lock_ != nullptr);
1008 } else {
1009 DCHECK(modify_ldt_lock_ == nullptr);
1010 }
Ian Rogers719d1a32014-03-06 12:13:39 -08001011 DCHECK(abort_lock_ != nullptr);
Brian Carlstrom306db812014-09-05 13:01:41 -07001012 DCHECK(alloc_tracker_lock_ != nullptr);
Andreas Gampe74240812014-04-17 10:35:09 -07001013 DCHECK(allocated_monitor_ids_lock_ != nullptr);
Chao-ying Fu9e369312014-05-21 11:20:52 -07001014 DCHECK(allocated_thread_ids_lock_ != nullptr);
Ian Rogers719d1a32014-03-06 12:13:39 -08001015 DCHECK(breakpoint_lock_ != nullptr);
Ian Rogers719d1a32014-03-06 12:13:39 -08001016 DCHECK(classlinker_classes_lock_ != nullptr);
Brian Carlstrom306db812014-09-05 13:01:41 -07001017 DCHECK(deoptimization_lock_ != nullptr);
Ian Rogers719d1a32014-03-06 12:13:39 -08001018 DCHECK(heap_bitmap_lock_ != nullptr);
Mathieu Chartierf9c6fc62015-10-07 11:44:05 -07001019 DCHECK(oat_file_manager_lock_ != nullptr);
David Brazdilca3c8c32016-09-06 14:04:48 +01001020 DCHECK(verifier_deps_lock_ != nullptr);
Richard Uhlera206c742016-05-24 15:04:22 -07001021 DCHECK(host_dlopen_handles_lock_ != nullptr);
Brian Carlstrom306db812014-09-05 13:01:41 -07001022 DCHECK(intern_table_lock_ != nullptr);
Andreas Gampec8089542017-01-16 12:41:12 -08001023 DCHECK(jni_function_table_lock_ != nullptr);
Ian Rogers68d8b422014-07-17 11:09:10 -07001024 DCHECK(jni_libraries_lock_ != nullptr);
Ian Rogers719d1a32014-03-06 12:13:39 -08001025 DCHECK(logging_lock_ != nullptr);
1026 DCHECK(mutator_lock_ != nullptr);
Brian Carlstrom306db812014-09-05 13:01:41 -07001027 DCHECK(profiler_lock_ != nullptr);
Mingyao Yang063fc772016-08-02 11:02:54 -07001028 DCHECK(cha_lock_ != nullptr);
Ian Rogers719d1a32014-03-06 12:13:39 -08001029 DCHECK(thread_list_lock_ != nullptr);
1030 DCHECK(thread_suspend_count_lock_ != nullptr);
1031 DCHECK(trace_lock_ != nullptr);
Ian Rogers719d1a32014-03-06 12:13:39 -08001032 DCHECK(unexpected_signal_lock_ != nullptr);
Alex Light88fd7202017-06-30 08:31:59 -07001033 DCHECK(user_code_suspension_lock_ != nullptr);
Andreas Gampecc1b5352016-12-01 16:58:38 -08001034 DCHECK(dex_lock_ != nullptr);
Ian Rogers719d1a32014-03-06 12:13:39 -08001035 } else {
Chao-ying Fu9e369312014-05-21 11:20:52 -07001036 // Create global locks in level order from highest lock level to lowest.
Ian Rogers4ad5cd32014-11-11 23:08:07 -08001037 LockLevel current_lock_level = kInstrumentEntrypointsLock;
1038 DCHECK(instrument_entrypoints_lock_ == nullptr);
1039 instrument_entrypoints_lock_ = new Mutex("instrument entrypoint lock", current_lock_level);
Ian Rogers719d1a32014-03-06 12:13:39 -08001040
Chao-ying Fu9e369312014-05-21 11:20:52 -07001041 #define UPDATE_CURRENT_LOCK_LEVEL(new_level) \
Chih-Hung Hsiehfba39972016-05-11 11:26:48 -07001042 if ((new_level) >= current_lock_level) { \
Brian Carlstrom306db812014-09-05 13:01:41 -07001043 /* Do not use CHECKs or FATAL here, abort_lock_ is not setup yet. */ \
1044 fprintf(stderr, "New local level %d is not less than current level %d\n", \
1045 new_level, current_lock_level); \
1046 exit(1); \
1047 } \
Ian Rogersf3d874c2014-07-17 18:52:42 -07001048 current_lock_level = new_level;
1049
Alex Light88fd7202017-06-30 08:31:59 -07001050 UPDATE_CURRENT_LOCK_LEVEL(kUserCodeSuspensionLock);
1051 DCHECK(user_code_suspension_lock_ == nullptr);
1052 user_code_suspension_lock_ = new Mutex("user code suspension lock", current_lock_level);
1053
Ian Rogersf3d874c2014-07-17 18:52:42 -07001054 UPDATE_CURRENT_LOCK_LEVEL(kMutatorLock);
1055 DCHECK(mutator_lock_ == nullptr);
Yu Lieac44242015-06-29 10:50:03 +08001056 mutator_lock_ = new MutatorMutex("mutator lock", current_lock_level);
Chao-ying Fu9e369312014-05-21 11:20:52 -07001057
1058 UPDATE_CURRENT_LOCK_LEVEL(kHeapBitmapLock);
1059 DCHECK(heap_bitmap_lock_ == nullptr);
1060 heap_bitmap_lock_ = new ReaderWriterMutex("heap bitmap lock", current_lock_level);
1061
Jeff Hao69dbec62014-09-15 18:03:41 -07001062 UPDATE_CURRENT_LOCK_LEVEL(kTraceLock);
1063 DCHECK(trace_lock_ == nullptr);
1064 trace_lock_ = new Mutex("trace lock", current_lock_level);
1065
Chao-ying Fu9e369312014-05-21 11:20:52 -07001066 UPDATE_CURRENT_LOCK_LEVEL(kRuntimeShutdownLock);
1067 DCHECK(runtime_shutdown_lock_ == nullptr);
1068 runtime_shutdown_lock_ = new Mutex("runtime shutdown lock", current_lock_level);
1069
1070 UPDATE_CURRENT_LOCK_LEVEL(kProfilerLock);
1071 DCHECK(profiler_lock_ == nullptr);
1072 profiler_lock_ = new Mutex("profiler lock", current_lock_level);
1073
Brian Carlstrom306db812014-09-05 13:01:41 -07001074 UPDATE_CURRENT_LOCK_LEVEL(kDeoptimizationLock);
1075 DCHECK(deoptimization_lock_ == nullptr);
1076 deoptimization_lock_ = new Mutex("Deoptimization lock", current_lock_level);
1077
1078 UPDATE_CURRENT_LOCK_LEVEL(kAllocTrackerLock);
1079 DCHECK(alloc_tracker_lock_ == nullptr);
1080 alloc_tracker_lock_ = new Mutex("AllocTracker lock", current_lock_level);
1081
Chao-ying Fu9e369312014-05-21 11:20:52 -07001082 UPDATE_CURRENT_LOCK_LEVEL(kThreadListLock);
1083 DCHECK(thread_list_lock_ == nullptr);
1084 thread_list_lock_ = new Mutex("thread list lock", current_lock_level);
1085
Ian Rogers68d8b422014-07-17 11:09:10 -07001086 UPDATE_CURRENT_LOCK_LEVEL(kJniLoadLibraryLock);
1087 DCHECK(jni_libraries_lock_ == nullptr);
1088 jni_libraries_lock_ = new Mutex("JNI shared libraries map lock", current_lock_level);
1089
Chao-ying Fu9e369312014-05-21 11:20:52 -07001090 UPDATE_CURRENT_LOCK_LEVEL(kBreakpointLock);
Ian Rogers719d1a32014-03-06 12:13:39 -08001091 DCHECK(breakpoint_lock_ == nullptr);
Sebastien Hertzed2be172014-08-19 15:33:43 +02001092 breakpoint_lock_ = new ReaderWriterMutex("breakpoint lock", current_lock_level);
Chao-ying Fu9e369312014-05-21 11:20:52 -07001093
Mingyao Yang063fc772016-08-02 11:02:54 -07001094 UPDATE_CURRENT_LOCK_LEVEL(kCHALock);
1095 DCHECK(cha_lock_ == nullptr);
1096 cha_lock_ = new Mutex("CHA lock", current_lock_level);
1097
Chao-ying Fu9e369312014-05-21 11:20:52 -07001098 UPDATE_CURRENT_LOCK_LEVEL(kClassLinkerClassesLock);
Ian Rogers719d1a32014-03-06 12:13:39 -08001099 DCHECK(classlinker_classes_lock_ == nullptr);
1100 classlinker_classes_lock_ = new ReaderWriterMutex("ClassLinker classes lock",
Chao-ying Fu9e369312014-05-21 11:20:52 -07001101 current_lock_level);
1102
Andreas Gampe74240812014-04-17 10:35:09 -07001103 UPDATE_CURRENT_LOCK_LEVEL(kMonitorPoolLock);
1104 DCHECK(allocated_monitor_ids_lock_ == nullptr);
1105 allocated_monitor_ids_lock_ = new Mutex("allocated monitor ids lock", current_lock_level);
1106
Chao-ying Fu9e369312014-05-21 11:20:52 -07001107 UPDATE_CURRENT_LOCK_LEVEL(kAllocatedThreadIdsLock);
1108 DCHECK(allocated_thread_ids_lock_ == nullptr);
1109 allocated_thread_ids_lock_ = new Mutex("allocated thread ids lock", current_lock_level);
1110
Nicolas Geoffray7f0a6d62014-05-26 14:01:46 +01001111 if (kRuntimeISA == kX86 || kRuntimeISA == kX86_64) {
Chao-ying Fu9e369312014-05-21 11:20:52 -07001112 UPDATE_CURRENT_LOCK_LEVEL(kModifyLdtLock);
1113 DCHECK(modify_ldt_lock_ == nullptr);
1114 modify_ldt_lock_ = new Mutex("modify_ldt lock", current_lock_level);
1115 }
1116
Andreas Gampecc1b5352016-12-01 16:58:38 -08001117 UPDATE_CURRENT_LOCK_LEVEL(kDexLock);
1118 DCHECK(dex_lock_ == nullptr);
1119 dex_lock_ = new ReaderWriterMutex("ClassLinker dex lock", current_lock_level);
1120
Mathieu Chartierf9c6fc62015-10-07 11:44:05 -07001121 UPDATE_CURRENT_LOCK_LEVEL(kOatFileManagerLock);
1122 DCHECK(oat_file_manager_lock_ == nullptr);
1123 oat_file_manager_lock_ = new ReaderWriterMutex("OatFile manager lock", current_lock_level);
1124
David Brazdilca3c8c32016-09-06 14:04:48 +01001125 UPDATE_CURRENT_LOCK_LEVEL(kVerifierDepsLock);
1126 DCHECK(verifier_deps_lock_ == nullptr);
Nicolas Geoffray340dafa2016-11-18 16:03:10 +00001127 verifier_deps_lock_ = new ReaderWriterMutex("verifier deps lock", current_lock_level);
David Brazdilca3c8c32016-09-06 14:04:48 +01001128
Richard Uhlera206c742016-05-24 15:04:22 -07001129 UPDATE_CURRENT_LOCK_LEVEL(kHostDlOpenHandlesLock);
1130 DCHECK(host_dlopen_handles_lock_ == nullptr);
1131 host_dlopen_handles_lock_ = new Mutex("host dlopen handles lock", current_lock_level);
Mathieu Chartiere58991b2015-10-13 07:59:34 -07001132
Chao-ying Fu9e369312014-05-21 11:20:52 -07001133 UPDATE_CURRENT_LOCK_LEVEL(kInternTableLock);
Ian Rogers719d1a32014-03-06 12:13:39 -08001134 DCHECK(intern_table_lock_ == nullptr);
Chao-ying Fu9e369312014-05-21 11:20:52 -07001135 intern_table_lock_ = new Mutex("InternTable lock", current_lock_level);
1136
Mathieu Chartiera5a53ef2014-09-12 12:58:05 -07001137 UPDATE_CURRENT_LOCK_LEVEL(kReferenceProcessorLock);
1138 DCHECK(reference_processor_lock_ == nullptr);
1139 reference_processor_lock_ = new Mutex("ReferenceProcessor lock", current_lock_level);
1140
1141 UPDATE_CURRENT_LOCK_LEVEL(kReferenceQueueClearedReferencesLock);
1142 DCHECK(reference_queue_cleared_references_lock_ == nullptr);
1143 reference_queue_cleared_references_lock_ = new Mutex("ReferenceQueue cleared references lock", current_lock_level);
1144
1145 UPDATE_CURRENT_LOCK_LEVEL(kReferenceQueueWeakReferencesLock);
1146 DCHECK(reference_queue_weak_references_lock_ == nullptr);
1147 reference_queue_weak_references_lock_ = new Mutex("ReferenceQueue cleared references lock", current_lock_level);
1148
1149 UPDATE_CURRENT_LOCK_LEVEL(kReferenceQueueFinalizerReferencesLock);
1150 DCHECK(reference_queue_finalizer_references_lock_ == nullptr);
1151 reference_queue_finalizer_references_lock_ = new Mutex("ReferenceQueue finalizer references lock", current_lock_level);
1152
1153 UPDATE_CURRENT_LOCK_LEVEL(kReferenceQueuePhantomReferencesLock);
1154 DCHECK(reference_queue_phantom_references_lock_ == nullptr);
1155 reference_queue_phantom_references_lock_ = new Mutex("ReferenceQueue phantom references lock", current_lock_level);
1156
1157 UPDATE_CURRENT_LOCK_LEVEL(kReferenceQueueSoftReferencesLock);
1158 DCHECK(reference_queue_soft_references_lock_ == nullptr);
1159 reference_queue_soft_references_lock_ = new Mutex("ReferenceQueue soft references lock", current_lock_level);
1160
Andreas Gampe05a364c2016-10-14 13:27:12 -07001161 UPDATE_CURRENT_LOCK_LEVEL(kJniGlobalsLock);
1162 DCHECK(jni_globals_lock_ == nullptr);
1163 jni_globals_lock_ =
1164 new ReaderWriterMutex("JNI global reference table lock", current_lock_level);
1165
1166 UPDATE_CURRENT_LOCK_LEVEL(kJniWeakGlobalsLock);
1167 DCHECK(jni_weak_globals_lock_ == nullptr);
1168 jni_weak_globals_lock_ = new Mutex("JNI weak global reference table lock", current_lock_level);
1169
Andreas Gampec8089542017-01-16 12:41:12 -08001170 UPDATE_CURRENT_LOCK_LEVEL(kJniFunctionTableLock);
1171 DCHECK(jni_function_table_lock_ == nullptr);
1172 jni_function_table_lock_ = new Mutex("JNI function table lock", current_lock_level);
1173
Chao-ying Fu9e369312014-05-21 11:20:52 -07001174 UPDATE_CURRENT_LOCK_LEVEL(kAbortLock);
1175 DCHECK(abort_lock_ == nullptr);
1176 abort_lock_ = new Mutex("abort lock", current_lock_level, true);
1177
1178 UPDATE_CURRENT_LOCK_LEVEL(kThreadSuspendCountLock);
1179 DCHECK(thread_suspend_count_lock_ == nullptr);
1180 thread_suspend_count_lock_ = new Mutex("thread suspend count lock", current_lock_level);
1181
1182 UPDATE_CURRENT_LOCK_LEVEL(kUnexpectedSignalLock);
1183 DCHECK(unexpected_signal_lock_ == nullptr);
1184 unexpected_signal_lock_ = new Mutex("unexpected signal lock", current_lock_level, true);
1185
1186 UPDATE_CURRENT_LOCK_LEVEL(kLoggingLock);
1187 DCHECK(logging_lock_ == nullptr);
1188 logging_lock_ = new Mutex("logging lock", current_lock_level, true);
1189
1190 #undef UPDATE_CURRENT_LOCK_LEVEL
Mathieu Chartier91e56692015-03-03 13:51:04 -08001191
Hiroshi Yamauchia2224042017-02-08 16:35:45 -08001192 // List of mutexes that we may hold when accessing a weak ref.
Hiroshi Yamauchi8a433242017-03-07 14:39:22 -08001193 AddToExpectedMutexesOnWeakRefAccess(dex_lock_, /*need_lock*/ false);
1194 AddToExpectedMutexesOnWeakRefAccess(classlinker_classes_lock_, /*need_lock*/ false);
1195 AddToExpectedMutexesOnWeakRefAccess(jni_libraries_lock_, /*need_lock*/ false);
Hiroshi Yamauchia2224042017-02-08 16:35:45 -08001196
Mathieu Chartier91e56692015-03-03 13:51:04 -08001197 InitConditions();
Ian Rogers719d1a32014-03-06 12:13:39 -08001198 }
1199}
1200
Mathieu Chartier91e56692015-03-03 13:51:04 -08001201void Locks::InitConditions() {
1202 thread_exit_cond_ = new ConditionVariable("thread exit condition variable", *thread_list_lock_);
1203}
Ian Rogers719d1a32014-03-06 12:13:39 -08001204
David Sehrf42eb2c2016-10-19 13:20:45 -07001205void Locks::SetClientCallback(ClientCallback* safe_to_call_abort_cb) {
1206 safe_to_call_abort_callback.StoreRelease(safe_to_call_abort_cb);
1207}
1208
1209// Helper to allow checking shutdown while ignoring locking requirements.
1210bool Locks::IsSafeToCallAbortRacy() {
1211 Locks::ClientCallback* safe_to_call_abort_cb = safe_to_call_abort_callback.LoadAcquire();
1212 return safe_to_call_abort_cb != nullptr && safe_to_call_abort_cb();
1213}
1214
Hiroshi Yamauchi8a433242017-03-07 14:39:22 -08001215void Locks::AddToExpectedMutexesOnWeakRefAccess(BaseMutex* mutex, bool need_lock) {
1216 if (need_lock) {
1217 ScopedExpectedMutexesOnWeakRefAccessLock mu(mutex);
1218 mutex->SetShouldRespondToEmptyCheckpointRequest(true);
1219 expected_mutexes_on_weak_ref_access_.push_back(mutex);
1220 } else {
1221 mutex->SetShouldRespondToEmptyCheckpointRequest(true);
1222 expected_mutexes_on_weak_ref_access_.push_back(mutex);
1223 }
1224}
1225
1226void Locks::RemoveFromExpectedMutexesOnWeakRefAccess(BaseMutex* mutex, bool need_lock) {
1227 if (need_lock) {
1228 ScopedExpectedMutexesOnWeakRefAccessLock mu(mutex);
1229 mutex->SetShouldRespondToEmptyCheckpointRequest(false);
1230 std::vector<BaseMutex*>& list = expected_mutexes_on_weak_ref_access_;
1231 auto it = std::find(list.begin(), list.end(), mutex);
1232 DCHECK(it != list.end());
1233 list.erase(it);
1234 } else {
1235 mutex->SetShouldRespondToEmptyCheckpointRequest(false);
1236 std::vector<BaseMutex*>& list = expected_mutexes_on_weak_ref_access_;
1237 auto it = std::find(list.begin(), list.end(), mutex);
1238 DCHECK(it != list.end());
1239 list.erase(it);
1240 }
1241}
1242
1243bool Locks::IsExpectedOnWeakRefAccess(BaseMutex* mutex) {
1244 ScopedExpectedMutexesOnWeakRefAccessLock mu(mutex);
1245 std::vector<BaseMutex*>& list = expected_mutexes_on_weak_ref_access_;
1246 return std::find(list.begin(), list.end(), mutex) != list.end();
1247}
1248
Elliott Hughese62934d2012-04-09 11:24:29 -07001249} // namespace art