Dynamic Tools Team | 517193e | 2019-09-11 14:48:41 +0000 | [diff] [blame] | 1 | //===-- tsd_exclusive.h -----------------------------------------*- C++ -*-===// |
| 2 | // |
| 3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| 4 | // See https://llvm.org/LICENSE.txt for license information. |
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| 6 | // |
| 7 | //===----------------------------------------------------------------------===// |
| 8 | |
| 9 | #ifndef SCUDO_TSD_EXCLUSIVE_H_ |
| 10 | #define SCUDO_TSD_EXCLUSIVE_H_ |
| 11 | |
| 12 | #include "tsd.h" |
| 13 | |
Dynamic Tools Team | 517193e | 2019-09-11 14:48:41 +0000 | [diff] [blame] | 14 | namespace scudo { |
| 15 | |
Peter Collingbourne | 33f8e1a | 2020-09-09 19:15:26 -0700 | [diff] [blame] | 16 | struct ThreadState { |
| 17 | bool DisableMemInit : 1; |
| 18 | enum { |
| 19 | NotInitialized = 0, |
| 20 | Initialized, |
| 21 | TornDown, |
| 22 | } InitState : 2; |
Dynamic Tools Team | 517193e | 2019-09-11 14:48:41 +0000 | [diff] [blame] | 23 | }; |
| 24 | |
| 25 | template <class Allocator> void teardownThread(void *Ptr); |
| 26 | |
| 27 | template <class Allocator> struct TSDRegistryExT { |
Dynamic Tools Team | 517193e | 2019-09-11 14:48:41 +0000 | [diff] [blame] | 28 | void init(Allocator *Instance) { |
Kostya Kortchinsky | 4a435d2 | 2021-05-25 15:00:58 -0700 | [diff] [blame] | 29 | DCHECK(!Initialized); |
| 30 | Instance->init(); |
| 31 | CHECK_EQ(pthread_key_create(&PThreadKey, teardownThread<Allocator>), 0); |
| 32 | FallbackTSD.init(Instance); |
| 33 | Initialized = true; |
Dynamic Tools Team | 517193e | 2019-09-11 14:48:41 +0000 | [diff] [blame] | 34 | } |
| 35 | |
Peter Collingbourne | cc3d493 | 2020-12-21 18:39:03 -0800 | [diff] [blame] | 36 | void initOnceMaybe(Allocator *Instance) { |
| 37 | ScopedLock L(Mutex); |
| 38 | if (LIKELY(Initialized)) |
| 39 | return; |
Kostya Kortchinsky | 4a435d2 | 2021-05-25 15:00:58 -0700 | [diff] [blame] | 40 | init(Instance); // Sets Initialized. |
Peter Collingbourne | cc3d493 | 2020-12-21 18:39:03 -0800 | [diff] [blame] | 41 | } |
| 42 | |
Kostya Kortchinsky | 4a435d2 | 2021-05-25 15:00:58 -0700 | [diff] [blame] | 43 | void unmapTestOnly(Allocator *Instance) { |
| 44 | DCHECK(Instance); |
| 45 | if (reinterpret_cast<Allocator *>(pthread_getspecific(PThreadKey))) { |
| 46 | DCHECK_EQ(reinterpret_cast<Allocator *>(pthread_getspecific(PThreadKey)), |
| 47 | Instance); |
| 48 | ThreadTSD.commitBack(Instance); |
| 49 | ThreadTSD = {}; |
| 50 | } |
| 51 | CHECK_EQ(pthread_key_delete(PThreadKey), 0); |
| 52 | PThreadKey = {}; |
| 53 | FallbackTSD.commitBack(Instance); |
| 54 | FallbackTSD = {}; |
Vitaly Buka | 0457156 | 2021-04-01 12:40:28 -0700 | [diff] [blame] | 55 | State = {}; |
Kostya Kortchinsky | 4a435d2 | 2021-05-25 15:00:58 -0700 | [diff] [blame] | 56 | Initialized = false; |
Vitaly Buka | 0457156 | 2021-04-01 12:40:28 -0700 | [diff] [blame] | 57 | } |
Dynamic Tools Team | 517193e | 2019-09-11 14:48:41 +0000 | [diff] [blame] | 58 | |
| 59 | ALWAYS_INLINE void initThreadMaybe(Allocator *Instance, bool MinimalInit) { |
Peter Collingbourne | 33f8e1a | 2020-09-09 19:15:26 -0700 | [diff] [blame] | 60 | if (LIKELY(State.InitState != ThreadState::NotInitialized)) |
Dynamic Tools Team | 517193e | 2019-09-11 14:48:41 +0000 | [diff] [blame] | 61 | return; |
| 62 | initThread(Instance, MinimalInit); |
| 63 | } |
| 64 | |
| 65 | ALWAYS_INLINE TSD<Allocator> *getTSDAndLock(bool *UnlockRequired) { |
Peter Collingbourne | 33f8e1a | 2020-09-09 19:15:26 -0700 | [diff] [blame] | 66 | if (LIKELY(State.InitState == ThreadState::Initialized && |
Dynamic Tools Team | 2c0f7b6 | 2019-12-19 10:37:12 -0800 | [diff] [blame] | 67 | !atomic_load(&Disabled, memory_order_acquire))) { |
Dynamic Tools Team | 517193e | 2019-09-11 14:48:41 +0000 | [diff] [blame] | 68 | *UnlockRequired = false; |
| 69 | return &ThreadTSD; |
| 70 | } |
Dynamic Tools Team | 2e7fec2 | 2020-02-16 15:29:46 -0800 | [diff] [blame] | 71 | FallbackTSD.lock(); |
Dynamic Tools Team | 517193e | 2019-09-11 14:48:41 +0000 | [diff] [blame] | 72 | *UnlockRequired = true; |
Dynamic Tools Team | 2e7fec2 | 2020-02-16 15:29:46 -0800 | [diff] [blame] | 73 | return &FallbackTSD; |
Dynamic Tools Team | 517193e | 2019-09-11 14:48:41 +0000 | [diff] [blame] | 74 | } |
| 75 | |
Dynamic Tools Team | 2c0f7b6 | 2019-12-19 10:37:12 -0800 | [diff] [blame] | 76 | // To disable the exclusive TSD registry, we effectively lock the fallback TSD |
| 77 | // and force all threads to attempt to use it instead of their local one. |
| 78 | void disable() { |
Dynamic Tools Team | 83eaa51 | 2020-01-09 11:43:16 -0800 | [diff] [blame] | 79 | Mutex.lock(); |
Dynamic Tools Team | 2e7fec2 | 2020-02-16 15:29:46 -0800 | [diff] [blame] | 80 | FallbackTSD.lock(); |
Dynamic Tools Team | 2c0f7b6 | 2019-12-19 10:37:12 -0800 | [diff] [blame] | 81 | atomic_store(&Disabled, 1U, memory_order_release); |
| 82 | } |
| 83 | |
| 84 | void enable() { |
| 85 | atomic_store(&Disabled, 0U, memory_order_release); |
Dynamic Tools Team | 2e7fec2 | 2020-02-16 15:29:46 -0800 | [diff] [blame] | 86 | FallbackTSD.unlock(); |
Dynamic Tools Team | 83eaa51 | 2020-01-09 11:43:16 -0800 | [diff] [blame] | 87 | Mutex.unlock(); |
Dynamic Tools Team | 2c0f7b6 | 2019-12-19 10:37:12 -0800 | [diff] [blame] | 88 | } |
| 89 | |
Kostya Kortchinsky | c72ca56 | 2020-07-27 09:13:42 -0700 | [diff] [blame] | 90 | bool setOption(Option O, UNUSED sptr Value) { |
Peter Collingbourne | 33f8e1a | 2020-09-09 19:15:26 -0700 | [diff] [blame] | 91 | if (O == Option::ThreadDisableMemInit) |
| 92 | State.DisableMemInit = Value; |
Kostya Kortchinsky | c72ca56 | 2020-07-27 09:13:42 -0700 | [diff] [blame] | 93 | if (O == Option::MaxTSDsCount) |
| 94 | return false; |
| 95 | return true; |
| 96 | } |
| 97 | |
Peter Collingbourne | 33f8e1a | 2020-09-09 19:15:26 -0700 | [diff] [blame] | 98 | bool getDisableMemInit() { return State.DisableMemInit; } |
| 99 | |
Dynamic Tools Team | 517193e | 2019-09-11 14:48:41 +0000 | [diff] [blame] | 100 | private: |
Dynamic Tools Team | 517193e | 2019-09-11 14:48:41 +0000 | [diff] [blame] | 101 | // Using minimal initialization allows for global initialization while keeping |
| 102 | // the thread specific structure untouched. The fallback structure will be |
| 103 | // used instead. |
| 104 | NOINLINE void initThread(Allocator *Instance, bool MinimalInit) { |
| 105 | initOnceMaybe(Instance); |
| 106 | if (UNLIKELY(MinimalInit)) |
| 107 | return; |
| 108 | CHECK_EQ( |
| 109 | pthread_setspecific(PThreadKey, reinterpret_cast<void *>(Instance)), 0); |
Kostya Kortchinsky | 4a435d2 | 2021-05-25 15:00:58 -0700 | [diff] [blame] | 110 | ThreadTSD.init(Instance); |
Peter Collingbourne | 33f8e1a | 2020-09-09 19:15:26 -0700 | [diff] [blame] | 111 | State.InitState = ThreadState::Initialized; |
Dynamic Tools Team | 83eaa51 | 2020-01-09 11:43:16 -0800 | [diff] [blame] | 112 | Instance->callPostInitCallback(); |
Dynamic Tools Team | 517193e | 2019-09-11 14:48:41 +0000 | [diff] [blame] | 113 | } |
| 114 | |
Vitaly Buka | 5d3d727 | 2021-04-29 01:19:51 -0700 | [diff] [blame] | 115 | pthread_key_t PThreadKey = {}; |
| 116 | bool Initialized = false; |
| 117 | atomic_u8 Disabled = {}; |
Dynamic Tools Team | 2e7fec2 | 2020-02-16 15:29:46 -0800 | [diff] [blame] | 118 | TSD<Allocator> FallbackTSD; |
Dynamic Tools Team | 517193e | 2019-09-11 14:48:41 +0000 | [diff] [blame] | 119 | HybridMutex Mutex; |
Peter Collingbourne | 0d4ff65 | 2020-09-10 12:38:42 -0700 | [diff] [blame] | 120 | static thread_local ThreadState State; |
| 121 | static thread_local TSD<Allocator> ThreadTSD; |
Dynamic Tools Team | 517193e | 2019-09-11 14:48:41 +0000 | [diff] [blame] | 122 | |
| 123 | friend void teardownThread<Allocator>(void *Ptr); |
| 124 | }; |
| 125 | |
| 126 | template <class Allocator> |
Peter Collingbourne | 0d4ff65 | 2020-09-10 12:38:42 -0700 | [diff] [blame] | 127 | thread_local TSD<Allocator> TSDRegistryExT<Allocator>::ThreadTSD; |
Dynamic Tools Team | 517193e | 2019-09-11 14:48:41 +0000 | [diff] [blame] | 128 | template <class Allocator> |
Peter Collingbourne | 0d4ff65 | 2020-09-10 12:38:42 -0700 | [diff] [blame] | 129 | thread_local ThreadState TSDRegistryExT<Allocator>::State; |
Dynamic Tools Team | 517193e | 2019-09-11 14:48:41 +0000 | [diff] [blame] | 130 | |
| 131 | template <class Allocator> void teardownThread(void *Ptr) { |
| 132 | typedef TSDRegistryExT<Allocator> TSDRegistryT; |
| 133 | Allocator *Instance = reinterpret_cast<Allocator *>(Ptr); |
| 134 | // The glibc POSIX thread-local-storage deallocation routine calls user |
| 135 | // provided destructors in a loop of PTHREAD_DESTRUCTOR_ITERATIONS. |
| 136 | // We want to be called last since other destructors might call free and the |
| 137 | // like, so we wait until PTHREAD_DESTRUCTOR_ITERATIONS before draining the |
| 138 | // quarantine and swallowing the cache. |
| 139 | if (TSDRegistryT::ThreadTSD.DestructorIterations > 1) { |
| 140 | TSDRegistryT::ThreadTSD.DestructorIterations--; |
| 141 | // If pthread_setspecific fails, we will go ahead with the teardown. |
| 142 | if (LIKELY(pthread_setspecific(Instance->getTSDRegistry()->PThreadKey, |
| 143 | Ptr) == 0)) |
| 144 | return; |
| 145 | } |
| 146 | TSDRegistryT::ThreadTSD.commitBack(Instance); |
Peter Collingbourne | 33f8e1a | 2020-09-09 19:15:26 -0700 | [diff] [blame] | 147 | TSDRegistryT::State.InitState = ThreadState::TornDown; |
Dynamic Tools Team | 517193e | 2019-09-11 14:48:41 +0000 | [diff] [blame] | 148 | } |
| 149 | |
| 150 | } // namespace scudo |
| 151 | |
| 152 | #endif // SCUDO_TSD_EXCLUSIVE_H_ |