Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 1 | //===-- tsan_rtl.h ----------------------------------------------*- C++ -*-===// |
| 2 | // |
| 3 | // The LLVM Compiler Infrastructure |
| 4 | // |
| 5 | // This file is distributed under the University of Illinois Open Source |
| 6 | // License. See LICENSE.TXT for details. |
| 7 | // |
| 8 | //===----------------------------------------------------------------------===// |
| 9 | // |
| 10 | // This file is a part of ThreadSanitizer (TSan), a race detector. |
| 11 | // |
| 12 | // Main internal TSan header file. |
| 13 | // |
| 14 | // Ground rules: |
| 15 | // - C++ run-time should not be used (static CTORs, RTTI, exceptions, static |
| 16 | // function-scope locals) |
| 17 | // - All functions/classes/etc reside in namespace __tsan, except for those |
| 18 | // declared in tsan_interface.h. |
| 19 | // - Platform-specific files should be used instead of ifdefs (*). |
| 20 | // - No system headers included in header files (*). |
| 21 | // - Platform specific headres included only into platform-specific files (*). |
| 22 | // |
| 23 | // (*) Except when inlining is critical for performance. |
| 24 | //===----------------------------------------------------------------------===// |
| 25 | |
| 26 | #ifndef TSAN_RTL_H |
| 27 | #define TSAN_RTL_H |
| 28 | |
Kostya Serebryany | 72166ca | 2012-12-05 10:09:15 +0000 | [diff] [blame] | 29 | #include "sanitizer_common/sanitizer_allocator.h" |
Alexey Samsonov | 1f3c2fe | 2013-05-29 09:15:39 +0000 | [diff] [blame] | 30 | #include "sanitizer_common/sanitizer_allocator_internal.h" |
Stephen Hines | 2d1fdb2 | 2014-05-28 23:58:16 -0700 | [diff] [blame] | 31 | #include "sanitizer_common/sanitizer_asm.h" |
Alexey Samsonov | 2bbd8be | 2013-03-15 13:48:44 +0000 | [diff] [blame] | 32 | #include "sanitizer_common/sanitizer_common.h" |
Stephen Hines | 2d1fdb2 | 2014-05-28 23:58:16 -0700 | [diff] [blame] | 33 | #include "sanitizer_common/sanitizer_deadlock_detector_interface.h" |
Dmitry Vyukov | 4af0f21 | 2013-10-03 13:37:17 +0000 | [diff] [blame] | 34 | #include "sanitizer_common/sanitizer_libignore.h" |
Sergey Matveev | a52e5c6 | 2013-06-26 15:37:14 +0000 | [diff] [blame] | 35 | #include "sanitizer_common/sanitizer_suppressions.h" |
Alexey Samsonov | 2bbd8be | 2013-03-15 13:48:44 +0000 | [diff] [blame] | 36 | #include "sanitizer_common/sanitizer_thread_registry.h" |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 37 | #include "tsan_clock.h" |
| 38 | #include "tsan_defs.h" |
| 39 | #include "tsan_flags.h" |
| 40 | #include "tsan_sync.h" |
| 41 | #include "tsan_trace.h" |
| 42 | #include "tsan_vector.h" |
| 43 | #include "tsan_report.h" |
Dmitry Vyukov | 385542a | 2012-11-28 10:35:31 +0000 | [diff] [blame] | 44 | #include "tsan_platform.h" |
Dmitry Vyukov | ad9da37 | 2012-12-06 12:16:15 +0000 | [diff] [blame] | 45 | #include "tsan_mutexset.h" |
Stephen Hines | 2d1fdb2 | 2014-05-28 23:58:16 -0700 | [diff] [blame] | 46 | #include "tsan_ignoreset.h" |
Stephen Hines | 6a211c5 | 2014-07-21 00:49:56 -0700 | [diff] [blame] | 47 | #include "tsan_stack_trace.h" |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 48 | |
Kostya Serebryany | 503a3af | 2012-12-04 15:13:30 +0000 | [diff] [blame] | 49 | #if SANITIZER_WORDSIZE != 64 |
| 50 | # error "ThreadSanitizer is supported only on 64-bit platforms" |
| 51 | #endif |
| 52 | |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 53 | namespace __tsan { |
| 54 | |
Stephen Hines | 86277eb | 2015-03-23 12:06:32 -0700 | [diff] [blame] | 55 | #ifndef SANITIZER_GO |
Dmitry Vyukov | e93e505 | 2013-03-18 10:32:21 +0000 | [diff] [blame] | 56 | struct MapUnmapCallback; |
Pirama Arumuga Nainar | 799172d | 2016-03-03 15:50:30 -0800 | [diff] [blame^] | 57 | #if defined(__mips64) || defined(__aarch64__) || defined(__powerpc__) |
Stephen Hines | 86277eb | 2015-03-23 12:06:32 -0700 | [diff] [blame] | 58 | static const uptr kAllocatorSpace = 0; |
| 59 | static const uptr kAllocatorSize = SANITIZER_MMAP_RANGE_SIZE; |
| 60 | static const uptr kAllocatorRegionSizeLog = 20; |
| 61 | static const uptr kAllocatorNumRegions = |
| 62 | kAllocatorSize >> kAllocatorRegionSizeLog; |
| 63 | typedef TwoLevelByteMap<(kAllocatorNumRegions >> 12), 1 << 12, |
| 64 | MapUnmapCallback> ByteMap; |
| 65 | typedef SizeClassAllocator32<kAllocatorSpace, kAllocatorSize, 0, |
| 66 | CompactSizeClassMap, kAllocatorRegionSizeLog, ByteMap, |
| 67 | MapUnmapCallback> PrimaryAllocator; |
| 68 | #else |
Pirama Arumuga Nainar | 799172d | 2016-03-03 15:50:30 -0800 | [diff] [blame^] | 69 | typedef SizeClassAllocator64<Mapping::kHeapMemBeg, |
| 70 | Mapping::kHeapMemEnd - Mapping::kHeapMemBeg, 0, |
Dmitry Vyukov | e93e505 | 2013-03-18 10:32:21 +0000 | [diff] [blame] | 71 | DefaultSizeClassMap, MapUnmapCallback> PrimaryAllocator; |
Stephen Hines | 86277eb | 2015-03-23 12:06:32 -0700 | [diff] [blame] | 72 | #endif |
Kostya Serebryany | 82de942 | 2012-12-04 14:15:17 +0000 | [diff] [blame] | 73 | typedef SizeClassAllocatorLocalCache<PrimaryAllocator> AllocatorCache; |
Dmitry Vyukov | e93e505 | 2013-03-18 10:32:21 +0000 | [diff] [blame] | 74 | typedef LargeMmapAllocator<MapUnmapCallback> SecondaryAllocator; |
Dmitry Vyukov | 2e87051 | 2012-08-15 15:35:15 +0000 | [diff] [blame] | 75 | typedef CombinedAllocator<PrimaryAllocator, AllocatorCache, |
| 76 | SecondaryAllocator> Allocator; |
Dmitry Vyukov | ff35f1d | 2012-08-30 13:02:30 +0000 | [diff] [blame] | 77 | Allocator *allocator(); |
Dmitry Vyukov | 2e87051 | 2012-08-15 15:35:15 +0000 | [diff] [blame] | 78 | #endif |
| 79 | |
Alexey Samsonov | 591616d | 2012-09-11 09:44:48 +0000 | [diff] [blame] | 80 | void TsanCheckFailed(const char *file, int line, const char *cond, |
| 81 | u64 v1, u64 v2); |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 82 | |
Dmitry Vyukov | 9743d74 | 2013-03-20 10:31:53 +0000 | [diff] [blame] | 83 | const u64 kShadowRodata = (u64)-1; // .rodata shadow marker |
| 84 | |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 85 | // FastState (from most significant bit): |
Dmitry Vyukov | 0d35d9d | 2012-11-28 10:49:27 +0000 | [diff] [blame] | 86 | // ignore : 1 |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 87 | // tid : kTidBits |
Dmitry Vyukov | 069ce82 | 2012-05-17 14:17:51 +0000 | [diff] [blame] | 88 | // unused : - |
Dmitry Vyukov | d698edc | 2012-11-28 12:19:50 +0000 | [diff] [blame] | 89 | // history_size : 3 |
Stephen Hines | 6a211c5 | 2014-07-21 00:49:56 -0700 | [diff] [blame] | 90 | // epoch : kClkBits |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 91 | class FastState { |
| 92 | public: |
| 93 | FastState(u64 tid, u64 epoch) { |
Dmitry Vyukov | 069ce82 | 2012-05-17 14:17:51 +0000 | [diff] [blame] | 94 | x_ = tid << kTidShift; |
Stephen Hines | 6a211c5 | 2014-07-21 00:49:56 -0700 | [diff] [blame] | 95 | x_ |= epoch; |
Dmitry Vyukov | 0d35d9d | 2012-11-28 10:49:27 +0000 | [diff] [blame] | 96 | DCHECK_EQ(tid, this->tid()); |
| 97 | DCHECK_EQ(epoch, this->epoch()); |
| 98 | DCHECK_EQ(GetIgnoreBit(), false); |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 99 | } |
| 100 | |
| 101 | explicit FastState(u64 x) |
| 102 | : x_(x) { |
| 103 | } |
| 104 | |
Dmitry Vyukov | 332c62b | 2012-08-16 15:08:49 +0000 | [diff] [blame] | 105 | u64 raw() const { |
| 106 | return x_; |
| 107 | } |
| 108 | |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 109 | u64 tid() const { |
Dmitry Vyukov | c8f0a00 | 2012-11-30 20:02:11 +0000 | [diff] [blame] | 110 | u64 res = (x_ & ~kIgnoreBit) >> kTidShift; |
| 111 | return res; |
| 112 | } |
| 113 | |
| 114 | u64 TidWithIgnore() const { |
Dmitry Vyukov | 069ce82 | 2012-05-17 14:17:51 +0000 | [diff] [blame] | 115 | u64 res = x_ >> kTidShift; |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 116 | return res; |
| 117 | } |
Dmitry Vyukov | 069ce82 | 2012-05-17 14:17:51 +0000 | [diff] [blame] | 118 | |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 119 | u64 epoch() const { |
Stephen Hines | 6a211c5 | 2014-07-21 00:49:56 -0700 | [diff] [blame] | 120 | u64 res = x_ & ((1ull << kClkBits) - 1); |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 121 | return res; |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 122 | } |
Dmitry Vyukov | 069ce82 | 2012-05-17 14:17:51 +0000 | [diff] [blame] | 123 | |
| 124 | void IncrementEpoch() { |
| 125 | u64 old_epoch = epoch(); |
Stephen Hines | 6a211c5 | 2014-07-21 00:49:56 -0700 | [diff] [blame] | 126 | x_ += 1; |
Dmitry Vyukov | e784ad4 | 2012-05-21 10:20:53 +0000 | [diff] [blame] | 127 | DCHECK_EQ(old_epoch + 1, epoch()); |
Dmitry Vyukov | 069ce82 | 2012-05-17 14:17:51 +0000 | [diff] [blame] | 128 | (void)old_epoch; |
| 129 | } |
| 130 | |
| 131 | void SetIgnoreBit() { x_ |= kIgnoreBit; } |
| 132 | void ClearIgnoreBit() { x_ &= ~kIgnoreBit; } |
Dmitry Vyukov | 0d35d9d | 2012-11-28 10:49:27 +0000 | [diff] [blame] | 133 | bool GetIgnoreBit() const { return (s64)x_ < 0; } |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 134 | |
Dmitry Vyukov | d698edc | 2012-11-28 12:19:50 +0000 | [diff] [blame] | 135 | void SetHistorySize(int hs) { |
| 136 | CHECK_GE(hs, 0); |
| 137 | CHECK_LE(hs, 7); |
Stephen Hines | 6a211c5 | 2014-07-21 00:49:56 -0700 | [diff] [blame] | 138 | x_ = (x_ & ~(kHistoryMask << kHistoryShift)) | (u64(hs) << kHistoryShift); |
Dmitry Vyukov | d698edc | 2012-11-28 12:19:50 +0000 | [diff] [blame] | 139 | } |
| 140 | |
Stephen Hines | 6a211c5 | 2014-07-21 00:49:56 -0700 | [diff] [blame] | 141 | ALWAYS_INLINE |
Dmitry Vyukov | d698edc | 2012-11-28 12:19:50 +0000 | [diff] [blame] | 142 | int GetHistorySize() const { |
Stephen Hines | 6a211c5 | 2014-07-21 00:49:56 -0700 | [diff] [blame] | 143 | return (int)((x_ >> kHistoryShift) & kHistoryMask); |
Dmitry Vyukov | d698edc | 2012-11-28 12:19:50 +0000 | [diff] [blame] | 144 | } |
| 145 | |
| 146 | void ClearHistorySize() { |
Stephen Hines | 6a211c5 | 2014-07-21 00:49:56 -0700 | [diff] [blame] | 147 | SetHistorySize(0); |
Dmitry Vyukov | d698edc | 2012-11-28 12:19:50 +0000 | [diff] [blame] | 148 | } |
| 149 | |
Stephen Hines | 6a211c5 | 2014-07-21 00:49:56 -0700 | [diff] [blame] | 150 | ALWAYS_INLINE |
Dmitry Vyukov | d698edc | 2012-11-28 12:19:50 +0000 | [diff] [blame] | 151 | u64 GetTracePos() const { |
| 152 | const int hs = GetHistorySize(); |
| 153 | // When hs == 0, the trace consists of 2 parts. |
| 154 | const u64 mask = (1ull << (kTracePartSizeBits + hs + 1)) - 1; |
| 155 | return epoch() & mask; |
| 156 | } |
| 157 | |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 158 | private: |
| 159 | friend class Shadow; |
Dmitry Vyukov | 069ce82 | 2012-05-17 14:17:51 +0000 | [diff] [blame] | 160 | static const int kTidShift = 64 - kTidBits - 1; |
Dmitry Vyukov | 0d35d9d | 2012-11-28 10:49:27 +0000 | [diff] [blame] | 161 | static const u64 kIgnoreBit = 1ull << 63; |
Dmitry Vyukov | 069ce82 | 2012-05-17 14:17:51 +0000 | [diff] [blame] | 162 | static const u64 kFreedBit = 1ull << 63; |
Stephen Hines | 6a211c5 | 2014-07-21 00:49:56 -0700 | [diff] [blame] | 163 | static const u64 kHistoryShift = kClkBits; |
| 164 | static const u64 kHistoryMask = 7; |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 165 | u64 x_; |
| 166 | }; |
| 167 | |
| 168 | // Shadow (from most significant bit): |
Dmitry Vyukov | 069ce82 | 2012-05-17 14:17:51 +0000 | [diff] [blame] | 169 | // freed : 1 |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 170 | // tid : kTidBits |
Dmitry Vyukov | 334553e | 2013-02-01 09:42:06 +0000 | [diff] [blame] | 171 | // is_atomic : 1 |
Dmitry Vyukov | 33a040a | 2013-02-01 10:02:55 +0000 | [diff] [blame] | 172 | // is_read : 1 |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 173 | // size_log : 2 |
| 174 | // addr0 : 3 |
Stephen Hines | 6a211c5 | 2014-07-21 00:49:56 -0700 | [diff] [blame] | 175 | // epoch : kClkBits |
Dmitry Vyukov | e963666 | 2012-06-27 16:05:06 +0000 | [diff] [blame] | 176 | class Shadow : public FastState { |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 177 | public: |
Dmitry Vyukov | d698edc | 2012-11-28 12:19:50 +0000 | [diff] [blame] | 178 | explicit Shadow(u64 x) |
| 179 | : FastState(x) { |
| 180 | } |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 181 | |
Dmitry Vyukov | d698edc | 2012-11-28 12:19:50 +0000 | [diff] [blame] | 182 | explicit Shadow(const FastState &s) |
| 183 | : FastState(s.x_) { |
| 184 | ClearHistorySize(); |
| 185 | } |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 186 | |
| 187 | void SetAddr0AndSizeLog(u64 addr0, unsigned kAccessSizeLog) { |
Stephen Hines | 6a211c5 | 2014-07-21 00:49:56 -0700 | [diff] [blame] | 188 | DCHECK_EQ((x_ >> kClkBits) & 31, 0); |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 189 | DCHECK_LE(addr0, 7); |
| 190 | DCHECK_LE(kAccessSizeLog, 3); |
Stephen Hines | 6a211c5 | 2014-07-21 00:49:56 -0700 | [diff] [blame] | 191 | x_ |= ((kAccessSizeLog << 3) | addr0) << kClkBits; |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 192 | DCHECK_EQ(kAccessSizeLog, size_log()); |
| 193 | DCHECK_EQ(addr0, this->addr0()); |
| 194 | } |
| 195 | |
| 196 | void SetWrite(unsigned kAccessIsWrite) { |
Dmitry Vyukov | 33a040a | 2013-02-01 10:02:55 +0000 | [diff] [blame] | 197 | DCHECK_EQ(x_ & kReadBit, 0); |
| 198 | if (!kAccessIsWrite) |
| 199 | x_ |= kReadBit; |
Dmitry Vyukov | 334553e | 2013-02-01 09:42:06 +0000 | [diff] [blame] | 200 | DCHECK_EQ(kAccessIsWrite, IsWrite()); |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 201 | } |
| 202 | |
Dmitry Vyukov | 334553e | 2013-02-01 09:42:06 +0000 | [diff] [blame] | 203 | void SetAtomic(bool kIsAtomic) { |
| 204 | DCHECK(!IsAtomic()); |
| 205 | if (kIsAtomic) |
| 206 | x_ |= kAtomicBit; |
| 207 | DCHECK_EQ(IsAtomic(), kIsAtomic); |
| 208 | } |
| 209 | |
| 210 | bool IsAtomic() const { |
| 211 | return x_ & kAtomicBit; |
| 212 | } |
| 213 | |
| 214 | bool IsZero() const { |
| 215 | return x_ == 0; |
| 216 | } |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 217 | |
Dmitry Vyukov | adfb650 | 2012-05-22 18:07:45 +0000 | [diff] [blame] | 218 | static inline bool TidsAreEqual(const Shadow s1, const Shadow s2) { |
Dmitry Vyukov | 069ce82 | 2012-05-17 14:17:51 +0000 | [diff] [blame] | 219 | u64 shifted_xor = (s1.x_ ^ s2.x_) >> kTidShift; |
Dmitry Vyukov | c8f0a00 | 2012-11-30 20:02:11 +0000 | [diff] [blame] | 220 | DCHECK_EQ(shifted_xor == 0, s1.TidWithIgnore() == s2.TidWithIgnore()); |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 221 | return shifted_xor == 0; |
| 222 | } |
Dmitry Vyukov | adfb650 | 2012-05-22 18:07:45 +0000 | [diff] [blame] | 223 | |
Stephen Hines | 6a211c5 | 2014-07-21 00:49:56 -0700 | [diff] [blame] | 224 | static ALWAYS_INLINE |
| 225 | bool Addr0AndSizeAreEqual(const Shadow s1, const Shadow s2) { |
| 226 | u64 masked_xor = ((s1.x_ ^ s2.x_) >> kClkBits) & 31; |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 227 | return masked_xor == 0; |
| 228 | } |
| 229 | |
Stephen Hines | 6a211c5 | 2014-07-21 00:49:56 -0700 | [diff] [blame] | 230 | static ALWAYS_INLINE bool TwoRangesIntersect(Shadow s1, Shadow s2, |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 231 | unsigned kS2AccessSize) { |
| 232 | bool res = false; |
| 233 | u64 diff = s1.addr0() - s2.addr0(); |
| 234 | if ((s64)diff < 0) { // s1.addr0 < s2.addr0 // NOLINT |
| 235 | // if (s1.addr0() + size1) > s2.addr0()) return true; |
Stephen Hines | 6a211c5 | 2014-07-21 00:49:56 -0700 | [diff] [blame] | 236 | if (s1.size() > -diff) |
| 237 | res = true; |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 238 | } else { |
| 239 | // if (s2.addr0() + kS2AccessSize > s1.addr0()) return true; |
Stephen Hines | 6a211c5 | 2014-07-21 00:49:56 -0700 | [diff] [blame] | 240 | if (kS2AccessSize > diff) |
| 241 | res = true; |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 242 | } |
Stephen Hines | 6a211c5 | 2014-07-21 00:49:56 -0700 | [diff] [blame] | 243 | DCHECK_EQ(res, TwoRangesIntersectSlow(s1, s2)); |
| 244 | DCHECK_EQ(res, TwoRangesIntersectSlow(s2, s1)); |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 245 | return res; |
| 246 | } |
| 247 | |
Stephen Hines | 6a211c5 | 2014-07-21 00:49:56 -0700 | [diff] [blame] | 248 | u64 ALWAYS_INLINE addr0() const { return (x_ >> kClkBits) & 7; } |
| 249 | u64 ALWAYS_INLINE size() const { return 1ull << size_log(); } |
| 250 | bool ALWAYS_INLINE IsWrite() const { return !IsRead(); } |
| 251 | bool ALWAYS_INLINE IsRead() const { return x_ & kReadBit; } |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 252 | |
Dmitry Vyukov | 069ce82 | 2012-05-17 14:17:51 +0000 | [diff] [blame] | 253 | // The idea behind the freed bit is as follows. |
| 254 | // When the memory is freed (or otherwise unaccessible) we write to the shadow |
| 255 | // values with tid/epoch related to the free and the freed bit set. |
| 256 | // During memory accesses processing the freed bit is considered |
| 257 | // as msb of tid. So any access races with shadow with freed bit set |
| 258 | // (it is as if write from a thread with which we never synchronized before). |
| 259 | // This allows us to detect accesses to freed memory w/o additional |
| 260 | // overheads in memory access processing and at the same time restore |
| 261 | // tid/epoch of free. |
| 262 | void MarkAsFreed() { |
| 263 | x_ |= kFreedBit; |
| 264 | } |
| 265 | |
Dmitry Vyukov | 3285866 | 2013-02-01 14:41:58 +0000 | [diff] [blame] | 266 | bool IsFreed() const { |
| 267 | return x_ & kFreedBit; |
| 268 | } |
| 269 | |
Dmitry Vyukov | 069ce82 | 2012-05-17 14:17:51 +0000 | [diff] [blame] | 270 | bool GetFreedAndReset() { |
| 271 | bool res = x_ & kFreedBit; |
| 272 | x_ &= ~kFreedBit; |
| 273 | return res; |
| 274 | } |
| 275 | |
Stephen Hines | 6a211c5 | 2014-07-21 00:49:56 -0700 | [diff] [blame] | 276 | bool ALWAYS_INLINE IsBothReadsOrAtomic(bool kIsWrite, bool kIsAtomic) const { |
| 277 | bool v = x_ & ((u64(kIsWrite ^ 1) << kReadShift) |
| 278 | | (u64(kIsAtomic) << kAtomicShift)); |
Dmitry Vyukov | 334553e | 2013-02-01 09:42:06 +0000 | [diff] [blame] | 279 | DCHECK_EQ(v, (!IsWrite() && !kIsWrite) || (IsAtomic() && kIsAtomic)); |
| 280 | return v; |
| 281 | } |
| 282 | |
Stephen Hines | 6a211c5 | 2014-07-21 00:49:56 -0700 | [diff] [blame] | 283 | bool ALWAYS_INLINE IsRWNotWeaker(bool kIsWrite, bool kIsAtomic) const { |
Dmitry Vyukov | 33a040a | 2013-02-01 10:02:55 +0000 | [diff] [blame] | 284 | bool v = ((x_ >> kReadShift) & 3) |
Dmitry Vyukov | 334553e | 2013-02-01 09:42:06 +0000 | [diff] [blame] | 285 | <= u64((kIsWrite ^ 1) | (kIsAtomic << 1)); |
| 286 | DCHECK_EQ(v, (IsAtomic() < kIsAtomic) || |
| 287 | (IsAtomic() == kIsAtomic && !IsWrite() <= !kIsWrite)); |
| 288 | return v; |
| 289 | } |
| 290 | |
Stephen Hines | 6a211c5 | 2014-07-21 00:49:56 -0700 | [diff] [blame] | 291 | bool ALWAYS_INLINE IsRWWeakerOrEqual(bool kIsWrite, bool kIsAtomic) const { |
Dmitry Vyukov | 33a040a | 2013-02-01 10:02:55 +0000 | [diff] [blame] | 292 | bool v = ((x_ >> kReadShift) & 3) |
Dmitry Vyukov | 334553e | 2013-02-01 09:42:06 +0000 | [diff] [blame] | 293 | >= u64((kIsWrite ^ 1) | (kIsAtomic << 1)); |
| 294 | DCHECK_EQ(v, (IsAtomic() > kIsAtomic) || |
| 295 | (IsAtomic() == kIsAtomic && !IsWrite() >= !kIsWrite)); |
| 296 | return v; |
| 297 | } |
| 298 | |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 299 | private: |
Stephen Hines | 6a211c5 | 2014-07-21 00:49:56 -0700 | [diff] [blame] | 300 | static const u64 kReadShift = 5 + kClkBits; |
Dmitry Vyukov | 33a040a | 2013-02-01 10:02:55 +0000 | [diff] [blame] | 301 | static const u64 kReadBit = 1ull << kReadShift; |
Stephen Hines | 6a211c5 | 2014-07-21 00:49:56 -0700 | [diff] [blame] | 302 | static const u64 kAtomicShift = 6 + kClkBits; |
Dmitry Vyukov | 334553e | 2013-02-01 09:42:06 +0000 | [diff] [blame] | 303 | static const u64 kAtomicBit = 1ull << kAtomicShift; |
| 304 | |
Stephen Hines | 6a211c5 | 2014-07-21 00:49:56 -0700 | [diff] [blame] | 305 | u64 size_log() const { return (x_ >> (3 + kClkBits)) & 3; } |
Dmitry Vyukov | adfb650 | 2012-05-22 18:07:45 +0000 | [diff] [blame] | 306 | |
Stephen Hines | 6a211c5 | 2014-07-21 00:49:56 -0700 | [diff] [blame] | 307 | static bool TwoRangesIntersectSlow(const Shadow s1, const Shadow s2) { |
Dmitry Vyukov | adfb650 | 2012-05-22 18:07:45 +0000 | [diff] [blame] | 308 | if (s1.addr0() == s2.addr0()) return true; |
| 309 | if (s1.addr0() < s2.addr0() && s1.addr0() + s1.size() > s2.addr0()) |
| 310 | return true; |
| 311 | if (s2.addr0() < s1.addr0() && s2.addr0() + s2.size() > s1.addr0()) |
| 312 | return true; |
| 313 | return false; |
| 314 | } |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 315 | }; |
| 316 | |
Pirama Arumuga Nainar | 7c91505 | 2015-04-08 08:58:29 -0700 | [diff] [blame] | 317 | struct ThreadSignalContext; |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 318 | |
Dmitry Vyukov | 8b30c25 | 2013-03-25 10:10:44 +0000 | [diff] [blame] | 319 | struct JmpBuf { |
| 320 | uptr sp; |
| 321 | uptr mangled_sp; |
Stephen Hines | 6d18623 | 2014-11-26 17:56:19 -0800 | [diff] [blame] | 322 | int int_signal_send; |
| 323 | bool in_blocking_func; |
| 324 | uptr in_signal_handler; |
Dmitry Vyukov | 8b30c25 | 2013-03-25 10:10:44 +0000 | [diff] [blame] | 325 | uptr *shadow_stack_pos; |
| 326 | }; |
| 327 | |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 328 | // This struct is stored in TLS. |
| 329 | struct ThreadState { |
| 330 | FastState fast_state; |
| 331 | // Synch epoch represents the threads's epoch before the last synchronization |
| 332 | // action. It allows to reduce number of shadow state updates. |
| 333 | // For example, fast_synch_epoch=100, last write to addr X was at epoch=150, |
| 334 | // if we are processing write to X from the same thread at epoch=200, |
| 335 | // we do nothing, because both writes happen in the same 'synch epoch'. |
| 336 | // That is, if another memory access does not race with the former write, |
| 337 | // it does not race with the latter as well. |
| 338 | // QUESTION: can we can squeeze this into ThreadState::Fast? |
| 339 | // E.g. ThreadState::Fast is a 44-bit, 32 are taken by synch_epoch and 12 are |
| 340 | // taken by epoch between synchs. |
| 341 | // This way we can save one load from tls. |
| 342 | u64 fast_synch_epoch; |
| 343 | // This is a slow path flag. On fast path, fast_state.GetIgnoreBit() is read. |
| 344 | // We do not distinguish beteween ignoring reads and writes |
| 345 | // for better performance. |
| 346 | int ignore_reads_and_writes; |
Dmitry Vyukov | e1ddbf9 | 2013-10-10 15:58:12 +0000 | [diff] [blame] | 347 | int ignore_sync; |
Stephen Hines | 2d1fdb2 | 2014-05-28 23:58:16 -0700 | [diff] [blame] | 348 | // Go does not support ignores. |
Stephen Hines | 86277eb | 2015-03-23 12:06:32 -0700 | [diff] [blame] | 349 | #ifndef SANITIZER_GO |
Stephen Hines | 2d1fdb2 | 2014-05-28 23:58:16 -0700 | [diff] [blame] | 350 | IgnoreSet mop_ignore_set; |
| 351 | IgnoreSet sync_ignore_set; |
| 352 | #endif |
Dmitry Vyukov | 01a7ce8 | 2013-10-16 15:35:12 +0000 | [diff] [blame] | 353 | // C/C++ uses fixed size shadow stack embed into Trace. |
| 354 | // Go uses malloc-allocated shadow stack with dynamic size. |
| 355 | uptr *shadow_stack; |
| 356 | uptr *shadow_stack_end; |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 357 | uptr *shadow_stack_pos; |
| 358 | u64 *racy_shadow_addr; |
| 359 | u64 racy_state[2]; |
Dmitry Vyukov | ad9da37 | 2012-12-06 12:16:15 +0000 | [diff] [blame] | 360 | MutexSet mset; |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 361 | ThreadClock clock; |
Stephen Hines | 86277eb | 2015-03-23 12:06:32 -0700 | [diff] [blame] | 362 | #ifndef SANITIZER_GO |
Dmitry Vyukov | 2e87051 | 2012-08-15 15:35:15 +0000 | [diff] [blame] | 363 | AllocatorCache alloc_cache; |
Alexey Samsonov | 1f3c2fe | 2013-05-29 09:15:39 +0000 | [diff] [blame] | 364 | InternalAllocatorCache internal_alloc_cache; |
Dmitry Vyukov | 8b30c25 | 2013-03-25 10:10:44 +0000 | [diff] [blame] | 365 | Vector<JmpBuf> jmp_bufs; |
Stephen Hines | 2d1fdb2 | 2014-05-28 23:58:16 -0700 | [diff] [blame] | 366 | int ignore_interceptors; |
Dmitry Vyukov | 2e87051 | 2012-08-15 15:35:15 +0000 | [diff] [blame] | 367 | #endif |
Stephen Hines | 86277eb | 2015-03-23 12:06:32 -0700 | [diff] [blame] | 368 | #if TSAN_COLLECT_STATS |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 369 | u64 stat[StatCnt]; |
Stephen Hines | 86277eb | 2015-03-23 12:06:32 -0700 | [diff] [blame] | 370 | #endif |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 371 | const int tid; |
Dmitry Vyukov | ff35f1d | 2012-08-30 13:02:30 +0000 | [diff] [blame] | 372 | const int unique_id; |
Dmitry Vyukov | 4e81d0e | 2013-01-29 13:03:07 +0000 | [diff] [blame] | 373 | bool in_symbolizer; |
Dmitry Vyukov | 4af0f21 | 2013-10-03 13:37:17 +0000 | [diff] [blame] | 374 | bool in_ignored_lib; |
Pirama Arumuga Nainar | 7c91505 | 2015-04-08 08:58:29 -0700 | [diff] [blame] | 375 | bool is_inited; |
Stephen Hines | 6d18623 | 2014-11-26 17:56:19 -0800 | [diff] [blame] | 376 | bool is_dead; |
Dmitry Vyukov | 3285866 | 2013-02-01 14:41:58 +0000 | [diff] [blame] | 377 | bool is_freeing; |
Dmitry Vyukov | 0dc47b6 | 2013-03-21 15:37:39 +0000 | [diff] [blame] | 378 | bool is_vptr_access; |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 379 | const uptr stk_addr; |
| 380 | const uptr stk_size; |
| 381 | const uptr tls_addr; |
| 382 | const uptr tls_size; |
Stephen Hines | 2d1fdb2 | 2014-05-28 23:58:16 -0700 | [diff] [blame] | 383 | ThreadContext *tctx; |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 384 | |
Stephen Hines | 86277eb | 2015-03-23 12:06:32 -0700 | [diff] [blame] | 385 | #if SANITIZER_DEBUG && !SANITIZER_GO |
Stephen Hines | 2d1fdb2 | 2014-05-28 23:58:16 -0700 | [diff] [blame] | 386 | InternalDeadlockDetector internal_deadlock_detector; |
Stephen Hines | 86277eb | 2015-03-23 12:06:32 -0700 | [diff] [blame] | 387 | #endif |
Stephen Hines | 2d1fdb2 | 2014-05-28 23:58:16 -0700 | [diff] [blame] | 388 | DDPhysicalThread *dd_pt; |
| 389 | DDLogicalThread *dd_lt; |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 390 | |
Stephen Hines | 6d18623 | 2014-11-26 17:56:19 -0800 | [diff] [blame] | 391 | atomic_uintptr_t in_signal_handler; |
Pirama Arumuga Nainar | 7c91505 | 2015-04-08 08:58:29 -0700 | [diff] [blame] | 392 | ThreadSignalContext *signal_ctx; |
Dmitry Vyukov | e963666 | 2012-06-27 16:05:06 +0000 | [diff] [blame] | 393 | |
Stephen Hines | 6a211c5 | 2014-07-21 00:49:56 -0700 | [diff] [blame] | 394 | DenseSlabAllocCache block_cache; |
| 395 | DenseSlabAllocCache sync_cache; |
Stephen Hines | 6d18623 | 2014-11-26 17:56:19 -0800 | [diff] [blame] | 396 | DenseSlabAllocCache clock_cache; |
Stephen Hines | 6a211c5 | 2014-07-21 00:49:56 -0700 | [diff] [blame] | 397 | |
Stephen Hines | 86277eb | 2015-03-23 12:06:32 -0700 | [diff] [blame] | 398 | #ifndef SANITIZER_GO |
Dmitry Vyukov | 8485311 | 2012-08-31 17:27:49 +0000 | [diff] [blame] | 399 | u32 last_sleep_stack_id; |
| 400 | ThreadClock last_sleep_clock; |
| 401 | #endif |
| 402 | |
Dmitry Vyukov | 9ad7c32 | 2012-06-22 11:08:55 +0000 | [diff] [blame] | 403 | // Set in regions of runtime that must be signal-safe and fork-safe. |
| 404 | // If set, malloc must not be called. |
| 405 | int nomalloc; |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 406 | |
Dmitry Vyukov | ff35f1d | 2012-08-30 13:02:30 +0000 | [diff] [blame] | 407 | explicit ThreadState(Context *ctx, int tid, int unique_id, u64 epoch, |
Stephen Hines | 2d1fdb2 | 2014-05-28 23:58:16 -0700 | [diff] [blame] | 408 | unsigned reuse_count, |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 409 | uptr stk_addr, uptr stk_size, |
| 410 | uptr tls_addr, uptr tls_size); |
| 411 | }; |
| 412 | |
Stephen Hines | 86277eb | 2015-03-23 12:06:32 -0700 | [diff] [blame] | 413 | #ifndef SANITIZER_GO |
Pirama Arumuga Nainar | 799172d | 2016-03-03 15:50:30 -0800 | [diff] [blame^] | 414 | #if SANITIZER_MAC |
| 415 | ThreadState *cur_thread(); |
| 416 | void cur_thread_finalize(); |
| 417 | #else |
Stephen Hines | 2d1fdb2 | 2014-05-28 23:58:16 -0700 | [diff] [blame] | 418 | __attribute__((tls_model("initial-exec"))) |
Dmitry Vyukov | b78caa6 | 2012-07-05 16:18:28 +0000 | [diff] [blame] | 419 | extern THREADLOCAL char cur_thread_placeholder[]; |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 420 | INLINE ThreadState *cur_thread() { |
| 421 | return reinterpret_cast<ThreadState *>(&cur_thread_placeholder); |
| 422 | } |
Pirama Arumuga Nainar | 799172d | 2016-03-03 15:50:30 -0800 | [diff] [blame^] | 423 | INLINE void cur_thread_finalize() { } |
| 424 | #endif // SANITIZER_MAC |
| 425 | #endif // SANITIZER_GO |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 426 | |
Alexey Samsonov | 2bbd8be | 2013-03-15 13:48:44 +0000 | [diff] [blame] | 427 | class ThreadContext : public ThreadContextBase { |
| 428 | public: |
| 429 | explicit ThreadContext(int tid); |
Dmitry Vyukov | 6af642e | 2013-03-18 10:10:15 +0000 | [diff] [blame] | 430 | ~ThreadContext(); |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 431 | ThreadState *thr; |
Dmitry Vyukov | 2c5284e | 2013-03-18 09:02:27 +0000 | [diff] [blame] | 432 | u32 creation_stack_id; |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 433 | SyncClock sync; |
| 434 | // Epoch at which the thread had started. |
| 435 | // If we see an event from the thread stamped by an older epoch, |
| 436 | // the event is from a dead thread that shared tid with this thread. |
| 437 | u64 epoch0; |
| 438 | u64 epoch1; |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 439 | |
Alexey Samsonov | 2bbd8be | 2013-03-15 13:48:44 +0000 | [diff] [blame] | 440 | // Override superclass callbacks. |
Pirama Arumuga Nainar | 259f706 | 2015-05-06 11:49:53 -0700 | [diff] [blame] | 441 | void OnDead() override; |
| 442 | void OnJoined(void *arg) override; |
| 443 | void OnFinished() override; |
| 444 | void OnStarted(void *arg) override; |
| 445 | void OnCreated(void *arg) override; |
| 446 | void OnReset() override; |
| 447 | void OnDetached(void *arg) override; |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 448 | }; |
| 449 | |
| 450 | struct RacyStacks { |
| 451 | MD5Hash hash[2]; |
| 452 | bool operator==(const RacyStacks &other) const { |
| 453 | if (hash[0] == other.hash[0] && hash[1] == other.hash[1]) |
| 454 | return true; |
| 455 | if (hash[0] == other.hash[1] && hash[1] == other.hash[0]) |
| 456 | return true; |
| 457 | return false; |
| 458 | } |
| 459 | }; |
| 460 | |
| 461 | struct RacyAddress { |
| 462 | uptr addr_min; |
| 463 | uptr addr_max; |
| 464 | }; |
| 465 | |
Dmitry Vyukov | 158c6ac | 2012-10-05 15:51:32 +0000 | [diff] [blame] | 466 | struct FiredSuppression { |
| 467 | ReportType type; |
Pirama Arumuga Nainar | 799172d | 2016-03-03 15:50:30 -0800 | [diff] [blame^] | 468 | uptr pc_or_addr; |
Dmitry Vyukov | f754eb5 | 2013-03-27 17:59:57 +0000 | [diff] [blame] | 469 | Suppression *supp; |
Dmitry Vyukov | 158c6ac | 2012-10-05 15:51:32 +0000 | [diff] [blame] | 470 | }; |
| 471 | |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 472 | struct Context { |
| 473 | Context(); |
| 474 | |
| 475 | bool initialized; |
Stephen Hines | 2d1fdb2 | 2014-05-28 23:58:16 -0700 | [diff] [blame] | 476 | bool after_multithreaded_fork; |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 477 | |
Stephen Hines | 6a211c5 | 2014-07-21 00:49:56 -0700 | [diff] [blame] | 478 | MetaMap metamap; |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 479 | |
| 480 | Mutex report_mtx; |
| 481 | int nreported; |
| 482 | int nmissed_expected; |
Dmitry Vyukov | a38e40f | 2013-03-21 07:02:36 +0000 | [diff] [blame] | 483 | atomic_uint64_t last_symbolize_time_ns; |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 484 | |
Stephen Hines | 2d1fdb2 | 2014-05-28 23:58:16 -0700 | [diff] [blame] | 485 | void *background_thread; |
| 486 | atomic_uint32_t stop_background_thread; |
| 487 | |
Alexey Samsonov | 2bbd8be | 2013-03-15 13:48:44 +0000 | [diff] [blame] | 488 | ThreadRegistry *thread_registry; |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 489 | |
Pirama Arumuga Nainar | 799172d | 2016-03-03 15:50:30 -0800 | [diff] [blame^] | 490 | Mutex racy_mtx; |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 491 | Vector<RacyStacks> racy_stacks; |
| 492 | Vector<RacyAddress> racy_addresses; |
Alexey Samsonov | 0a05e5f | 2013-06-14 11:18:58 +0000 | [diff] [blame] | 493 | // Number of fired suppressions may be large enough. |
Pirama Arumuga Nainar | 799172d | 2016-03-03 15:50:30 -0800 | [diff] [blame^] | 494 | Mutex fired_suppressions_mtx; |
Alexey Samsonov | 0a05e5f | 2013-06-14 11:18:58 +0000 | [diff] [blame] | 495 | InternalMmapVector<FiredSuppression> fired_suppressions; |
Stephen Hines | 2d1fdb2 | 2014-05-28 23:58:16 -0700 | [diff] [blame] | 496 | DDetector *dd; |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 497 | |
Stephen Hines | 6d18623 | 2014-11-26 17:56:19 -0800 | [diff] [blame] | 498 | ClockAlloc clock_alloc; |
| 499 | |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 500 | Flags flags; |
| 501 | |
| 502 | u64 stat[StatCnt]; |
| 503 | u64 int_alloc_cnt[MBlockTypeCount]; |
| 504 | u64 int_alloc_siz[MBlockTypeCount]; |
| 505 | }; |
| 506 | |
Stephen Hines | 2d1fdb2 | 2014-05-28 23:58:16 -0700 | [diff] [blame] | 507 | extern Context *ctx; // The one and the only global runtime context. |
| 508 | |
| 509 | struct ScopedIgnoreInterceptors { |
| 510 | ScopedIgnoreInterceptors() { |
Stephen Hines | 86277eb | 2015-03-23 12:06:32 -0700 | [diff] [blame] | 511 | #ifndef SANITIZER_GO |
Stephen Hines | 2d1fdb2 | 2014-05-28 23:58:16 -0700 | [diff] [blame] | 512 | cur_thread()->ignore_interceptors++; |
| 513 | #endif |
| 514 | } |
| 515 | |
| 516 | ~ScopedIgnoreInterceptors() { |
Stephen Hines | 86277eb | 2015-03-23 12:06:32 -0700 | [diff] [blame] | 517 | #ifndef SANITIZER_GO |
Stephen Hines | 2d1fdb2 | 2014-05-28 23:58:16 -0700 | [diff] [blame] | 518 | cur_thread()->ignore_interceptors--; |
| 519 | #endif |
| 520 | } |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 521 | }; |
| 522 | |
| 523 | class ScopedReport { |
| 524 | public: |
| 525 | explicit ScopedReport(ReportType typ); |
| 526 | ~ScopedReport(); |
| 527 | |
Stephen Hines | 6d18623 | 2014-11-26 17:56:19 -0800 | [diff] [blame] | 528 | void AddMemoryAccess(uptr addr, Shadow s, StackTrace stack, |
Dmitry Vyukov | ad9da37 | 2012-12-06 12:16:15 +0000 | [diff] [blame] | 529 | const MutexSet *mset); |
Stephen Hines | 6d18623 | 2014-11-26 17:56:19 -0800 | [diff] [blame] | 530 | void AddStack(StackTrace stack, bool suppressable = false); |
Stephen Hines | 6a211c5 | 2014-07-21 00:49:56 -0700 | [diff] [blame] | 531 | void AddThread(const ThreadContext *tctx, bool suppressable = false); |
| 532 | void AddThread(int unique_tid, bool suppressable = false); |
Stephen Hines | 2d1fdb2 | 2014-05-28 23:58:16 -0700 | [diff] [blame] | 533 | void AddUniqueTid(int unique_tid); |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 534 | void AddMutex(const SyncVar *s); |
Stephen Hines | 2d1fdb2 | 2014-05-28 23:58:16 -0700 | [diff] [blame] | 535 | u64 AddMutex(u64 id); |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 536 | void AddLocation(uptr addr, uptr size); |
Dmitry Vyukov | 8485311 | 2012-08-31 17:27:49 +0000 | [diff] [blame] | 537 | void AddSleep(u32 stack_id); |
Dmitry Vyukov | 4536cb1 | 2013-03-21 16:55:17 +0000 | [diff] [blame] | 538 | void SetCount(int count); |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 539 | |
| 540 | const ReportDesc *GetReport() const; |
| 541 | |
| 542 | private: |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 543 | ReportDesc *rep_; |
Stephen Hines | 2d1fdb2 | 2014-05-28 23:58:16 -0700 | [diff] [blame] | 544 | // Symbolizer makes lots of intercepted calls. If we try to process them, |
| 545 | // at best it will cause deadlocks on internal mutexes. |
| 546 | ScopedIgnoreInterceptors ignore_interceptors_; |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 547 | |
Stephen Hines | 2d1fdb2 | 2014-05-28 23:58:16 -0700 | [diff] [blame] | 548 | void AddDeadMutex(u64 id); |
Dmitry Vyukov | ad9da37 | 2012-12-06 12:16:15 +0000 | [diff] [blame] | 549 | |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 550 | ScopedReport(const ScopedReport&); |
| 551 | void operator = (const ScopedReport&); |
| 552 | }; |
| 553 | |
Stephen Hines | 6d18623 | 2014-11-26 17:56:19 -0800 | [diff] [blame] | 554 | void RestoreStack(int tid, const u64 epoch, VarSizeStackTrace *stk, |
| 555 | MutexSet *mset); |
| 556 | |
| 557 | template<typename StackTraceTy> |
| 558 | void ObtainCurrentStack(ThreadState *thr, uptr toppc, StackTraceTy *stack) { |
| 559 | uptr size = thr->shadow_stack_pos - thr->shadow_stack; |
| 560 | uptr start = 0; |
| 561 | if (size + !!toppc > kStackTraceMax) { |
| 562 | start = size + !!toppc - kStackTraceMax; |
| 563 | size = kStackTraceMax - !!toppc; |
| 564 | } |
| 565 | stack->Init(&thr->shadow_stack[start], size, toppc); |
| 566 | } |
| 567 | |
Dmitry Vyukov | 332c62b | 2012-08-16 15:08:49 +0000 | [diff] [blame] | 568 | |
Stephen Hines | 86277eb | 2015-03-23 12:06:32 -0700 | [diff] [blame] | 569 | #if TSAN_COLLECT_STATS |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 570 | void StatAggregate(u64 *dst, u64 *src); |
| 571 | void StatOutput(u64 *stat); |
Stephen Hines | 86277eb | 2015-03-23 12:06:32 -0700 | [diff] [blame] | 572 | #endif |
| 573 | |
Timur Iskhodzhanov | abfdbdf | 2013-03-28 18:52:40 +0000 | [diff] [blame] | 574 | void ALWAYS_INLINE StatInc(ThreadState *thr, StatType typ, u64 n = 1) { |
Stephen Hines | 86277eb | 2015-03-23 12:06:32 -0700 | [diff] [blame] | 575 | #if TSAN_COLLECT_STATS |
| 576 | thr->stat[typ] += n; |
| 577 | #endif |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 578 | } |
Timur Iskhodzhanov | abfdbdf | 2013-03-28 18:52:40 +0000 | [diff] [blame] | 579 | void ALWAYS_INLINE StatSet(ThreadState *thr, StatType typ, u64 n) { |
Stephen Hines | 86277eb | 2015-03-23 12:06:32 -0700 | [diff] [blame] | 580 | #if TSAN_COLLECT_STATS |
| 581 | thr->stat[typ] = n; |
| 582 | #endif |
Alexey Samsonov | 2bbd8be | 2013-03-15 13:48:44 +0000 | [diff] [blame] | 583 | } |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 584 | |
Dmitry Vyukov | a05fcc1 | 2012-11-06 16:00:16 +0000 | [diff] [blame] | 585 | void MapShadow(uptr addr, uptr size); |
Pirama Arumuga Nainar | cdce50b | 2015-07-01 12:26:56 -0700 | [diff] [blame] | 586 | void MapThreadTrace(uptr addr, uptr size, const char *name); |
Dmitry Vyukov | 7ac33ac | 2013-03-18 15:49:07 +0000 | [diff] [blame] | 587 | void DontNeedShadowFor(uptr addr, uptr size); |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 588 | void InitializeShadowMemory(); |
| 589 | void InitializeInterceptors(); |
Dmitry Vyukov | 4af0f21 | 2013-10-03 13:37:17 +0000 | [diff] [blame] | 590 | void InitializeLibIgnore(); |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 591 | void InitializeDynamicAnnotations(); |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 592 | |
Stephen Hines | 2d1fdb2 | 2014-05-28 23:58:16 -0700 | [diff] [blame] | 593 | void ForkBefore(ThreadState *thr, uptr pc); |
| 594 | void ForkParentAfter(ThreadState *thr, uptr pc); |
| 595 | void ForkChildAfter(ThreadState *thr, uptr pc); |
| 596 | |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 597 | void ReportRace(ThreadState *thr); |
Stephen Hines | 6a211c5 | 2014-07-21 00:49:56 -0700 | [diff] [blame] | 598 | bool OutputReport(ThreadState *thr, const ScopedReport &srep); |
Pirama Arumuga Nainar | 799172d | 2016-03-03 15:50:30 -0800 | [diff] [blame^] | 599 | bool IsFiredSuppression(Context *ctx, ReportType type, StackTrace trace); |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 600 | bool IsExpectedReport(uptr addr, uptr size); |
Dmitry Vyukov | 0fd908c | 2013-03-28 16:21:19 +0000 | [diff] [blame] | 601 | void PrintMatchedBenignRaces(); |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 602 | |
| 603 | #if defined(TSAN_DEBUG_OUTPUT) && TSAN_DEBUG_OUTPUT >= 1 |
Alexey Samsonov | b1fe302 | 2012-11-02 12:17:51 +0000 | [diff] [blame] | 604 | # define DPrintf Printf |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 605 | #else |
| 606 | # define DPrintf(...) |
| 607 | #endif |
| 608 | |
| 609 | #if defined(TSAN_DEBUG_OUTPUT) && TSAN_DEBUG_OUTPUT >= 2 |
Alexey Samsonov | b1fe302 | 2012-11-02 12:17:51 +0000 | [diff] [blame] | 610 | # define DPrintf2 Printf |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 611 | #else |
| 612 | # define DPrintf2(...) |
| 613 | #endif |
| 614 | |
Dmitry Vyukov | 8485311 | 2012-08-31 17:27:49 +0000 | [diff] [blame] | 615 | u32 CurrentStackId(ThreadState *thr, uptr pc); |
Stephen Hines | 2d1fdb2 | 2014-05-28 23:58:16 -0700 | [diff] [blame] | 616 | ReportStack *SymbolizeStackId(u32 stack_id); |
Dmitry Vyukov | 1da1056 | 2012-09-01 12:13:18 +0000 | [diff] [blame] | 617 | void PrintCurrentStack(ThreadState *thr, uptr pc); |
Stephen Hines | 6d18623 | 2014-11-26 17:56:19 -0800 | [diff] [blame] | 618 | void PrintCurrentStackSlow(uptr pc); // uses libunwind |
Dmitry Vyukov | 8485311 | 2012-08-31 17:27:49 +0000 | [diff] [blame] | 619 | |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 620 | void Initialize(ThreadState *thr); |
| 621 | int Finalize(ThreadState *thr); |
| 622 | |
Stephen Hines | 6a211c5 | 2014-07-21 00:49:56 -0700 | [diff] [blame] | 623 | void OnUserAlloc(ThreadState *thr, uptr pc, uptr p, uptr sz, bool write); |
| 624 | void OnUserFree(ThreadState *thr, uptr pc, uptr p, bool write); |
Dmitry Vyukov | 21cc85d | 2012-12-20 17:29:34 +0000 | [diff] [blame] | 625 | |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 626 | void MemoryAccess(ThreadState *thr, uptr pc, uptr addr, |
Dmitry Vyukov | 334553e | 2013-02-01 09:42:06 +0000 | [diff] [blame] | 627 | int kAccessSizeLog, bool kAccessIsWrite, bool kIsAtomic); |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 628 | void MemoryAccessImpl(ThreadState *thr, uptr addr, |
Dmitry Vyukov | 334553e | 2013-02-01 09:42:06 +0000 | [diff] [blame] | 629 | int kAccessSizeLog, bool kAccessIsWrite, bool kIsAtomic, |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 630 | u64 *shadow_mem, Shadow cur); |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 631 | void MemoryAccessRange(ThreadState *thr, uptr pc, uptr addr, |
Dmitry Vyukov | 334553e | 2013-02-01 09:42:06 +0000 | [diff] [blame] | 632 | uptr size, bool is_write); |
Dmitry Vyukov | eaa0190 | 2013-02-13 13:05:36 +0000 | [diff] [blame] | 633 | void MemoryAccessRangeStep(ThreadState *thr, uptr pc, uptr addr, |
| 634 | uptr size, uptr step, bool is_write); |
Dmitry Vyukov | 8ecd0e5 | 2013-04-30 11:56:56 +0000 | [diff] [blame] | 635 | void UnalignedMemoryAccess(ThreadState *thr, uptr pc, uptr addr, |
| 636 | int size, bool kAccessIsWrite, bool kIsAtomic); |
Dmitry Vyukov | 334553e | 2013-02-01 09:42:06 +0000 | [diff] [blame] | 637 | |
| 638 | const int kSizeLog1 = 0; |
| 639 | const int kSizeLog2 = 1; |
| 640 | const int kSizeLog4 = 2; |
| 641 | const int kSizeLog8 = 3; |
| 642 | |
Timur Iskhodzhanov | abfdbdf | 2013-03-28 18:52:40 +0000 | [diff] [blame] | 643 | void ALWAYS_INLINE MemoryRead(ThreadState *thr, uptr pc, |
Dmitry Vyukov | 334553e | 2013-02-01 09:42:06 +0000 | [diff] [blame] | 644 | uptr addr, int kAccessSizeLog) { |
| 645 | MemoryAccess(thr, pc, addr, kAccessSizeLog, false, false); |
| 646 | } |
| 647 | |
Timur Iskhodzhanov | abfdbdf | 2013-03-28 18:52:40 +0000 | [diff] [blame] | 648 | void ALWAYS_INLINE MemoryWrite(ThreadState *thr, uptr pc, |
Dmitry Vyukov | 334553e | 2013-02-01 09:42:06 +0000 | [diff] [blame] | 649 | uptr addr, int kAccessSizeLog) { |
| 650 | MemoryAccess(thr, pc, addr, kAccessSizeLog, true, false); |
| 651 | } |
| 652 | |
Timur Iskhodzhanov | abfdbdf | 2013-03-28 18:52:40 +0000 | [diff] [blame] | 653 | void ALWAYS_INLINE MemoryReadAtomic(ThreadState *thr, uptr pc, |
Dmitry Vyukov | 334553e | 2013-02-01 09:42:06 +0000 | [diff] [blame] | 654 | uptr addr, int kAccessSizeLog) { |
| 655 | MemoryAccess(thr, pc, addr, kAccessSizeLog, false, true); |
| 656 | } |
| 657 | |
Timur Iskhodzhanov | abfdbdf | 2013-03-28 18:52:40 +0000 | [diff] [blame] | 658 | void ALWAYS_INLINE MemoryWriteAtomic(ThreadState *thr, uptr pc, |
Dmitry Vyukov | 334553e | 2013-02-01 09:42:06 +0000 | [diff] [blame] | 659 | uptr addr, int kAccessSizeLog) { |
| 660 | MemoryAccess(thr, pc, addr, kAccessSizeLog, true, true); |
| 661 | } |
| 662 | |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 663 | void MemoryResetRange(ThreadState *thr, uptr pc, uptr addr, uptr size); |
| 664 | void MemoryRangeFreed(ThreadState *thr, uptr pc, uptr addr, uptr size); |
Dmitry Vyukov | 26af893 | 2012-08-15 16:52:19 +0000 | [diff] [blame] | 665 | void MemoryRangeImitateWrite(ThreadState *thr, uptr pc, uptr addr, uptr size); |
Dmitry Vyukov | e1ddbf9 | 2013-10-10 15:58:12 +0000 | [diff] [blame] | 666 | |
Stephen Hines | 2d1fdb2 | 2014-05-28 23:58:16 -0700 | [diff] [blame] | 667 | void ThreadIgnoreBegin(ThreadState *thr, uptr pc); |
| 668 | void ThreadIgnoreEnd(ThreadState *thr, uptr pc); |
| 669 | void ThreadIgnoreSyncBegin(ThreadState *thr, uptr pc); |
| 670 | void ThreadIgnoreSyncEnd(ThreadState *thr, uptr pc); |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 671 | |
| 672 | void FuncEntry(ThreadState *thr, uptr pc); |
| 673 | void FuncExit(ThreadState *thr); |
| 674 | |
| 675 | int ThreadCreate(ThreadState *thr, uptr pc, uptr uid, bool detached); |
Dmitry Vyukov | e0023f7 | 2012-10-02 12:58:14 +0000 | [diff] [blame] | 676 | void ThreadStart(ThreadState *thr, int tid, uptr os_id); |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 677 | void ThreadFinish(ThreadState *thr); |
| 678 | int ThreadTid(ThreadState *thr, uptr pc, uptr uid); |
| 679 | void ThreadJoin(ThreadState *thr, uptr pc, int tid); |
| 680 | void ThreadDetach(ThreadState *thr, uptr pc, int tid); |
| 681 | void ThreadFinalize(ThreadState *thr); |
Dmitry Vyukov | aecf2e5 | 2012-12-04 15:46:05 +0000 | [diff] [blame] | 682 | void ThreadSetName(ThreadState *thr, const char *name); |
Dmitry Vyukov | 54e0a9a | 2012-11-07 16:41:57 +0000 | [diff] [blame] | 683 | int ThreadCount(ThreadState *thr); |
Dmitry Vyukov | ee8ee24 | 2012-11-15 17:40:49 +0000 | [diff] [blame] | 684 | void ProcessPendingSignals(ThreadState *thr); |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 685 | |
Dmitry Vyukov | c20e9ba | 2012-08-16 13:29:41 +0000 | [diff] [blame] | 686 | void MutexCreate(ThreadState *thr, uptr pc, uptr addr, |
| 687 | bool rw, bool recursive, bool linker_init); |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 688 | void MutexDestroy(ThreadState *thr, uptr pc, uptr addr); |
Stephen Hines | 2d1fdb2 | 2014-05-28 23:58:16 -0700 | [diff] [blame] | 689 | void MutexLock(ThreadState *thr, uptr pc, uptr addr, int rec = 1, |
| 690 | bool try_lock = false); |
Dmitry Vyukov | 8354fae | 2013-05-17 12:03:46 +0000 | [diff] [blame] | 691 | int MutexUnlock(ThreadState *thr, uptr pc, uptr addr, bool all = false); |
Stephen Hines | 2d1fdb2 | 2014-05-28 23:58:16 -0700 | [diff] [blame] | 692 | void MutexReadLock(ThreadState *thr, uptr pc, uptr addr, bool try_lock = false); |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 693 | void MutexReadUnlock(ThreadState *thr, uptr pc, uptr addr); |
| 694 | void MutexReadOrWriteUnlock(ThreadState *thr, uptr pc, uptr addr); |
Dmitry Vyukov | 11f5309 | 2013-11-15 16:58:12 +0000 | [diff] [blame] | 695 | void MutexRepair(ThreadState *thr, uptr pc, uptr addr); // call on EOWNERDEAD |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 696 | |
| 697 | void Acquire(ThreadState *thr, uptr pc, uptr addr); |
Stephen Hines | 6d18623 | 2014-11-26 17:56:19 -0800 | [diff] [blame] | 698 | // AcquireGlobal synchronizes the current thread with all other threads. |
| 699 | // In terms of happens-before relation, it draws a HB edge from all threads |
| 700 | // (where they happen to execute right now) to the current thread. We use it to |
| 701 | // handle Go finalizers. Namely, finalizer goroutine executes AcquireGlobal |
| 702 | // right before executing finalizers. This provides a coarse, but simple |
| 703 | // approximation of the actual required synchronization. |
Dmitry Vyukov | 538f1ba | 2012-11-07 15:08:20 +0000 | [diff] [blame] | 704 | void AcquireGlobal(ThreadState *thr, uptr pc); |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 705 | void Release(ThreadState *thr, uptr pc, uptr addr); |
Dmitry Vyukov | 9d150bd | 2012-07-28 15:27:41 +0000 | [diff] [blame] | 706 | void ReleaseStore(ThreadState *thr, uptr pc, uptr addr); |
Dmitry Vyukov | 8485311 | 2012-08-31 17:27:49 +0000 | [diff] [blame] | 707 | void AfterSleep(ThreadState *thr, uptr pc); |
Dmitry Vyukov | e1ddbf9 | 2013-10-10 15:58:12 +0000 | [diff] [blame] | 708 | void AcquireImpl(ThreadState *thr, uptr pc, SyncClock *c); |
| 709 | void ReleaseImpl(ThreadState *thr, uptr pc, SyncClock *c); |
| 710 | void ReleaseStoreImpl(ThreadState *thr, uptr pc, SyncClock *c); |
| 711 | void AcquireReleaseImpl(ThreadState *thr, uptr pc, SyncClock *c); |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 712 | |
| 713 | // The hacky call uses custom calling convention and an assembly thunk. |
| 714 | // It is considerably faster that a normal call for the caller |
| 715 | // if it is not executed (it is intended for slow paths from hot functions). |
| 716 | // The trick is that the call preserves all registers and the compiler |
| 717 | // does not treat it as a call. |
| 718 | // If it does not work for you, use normal call. |
Pirama Arumuga Nainar | 799172d | 2016-03-03 15:50:30 -0800 | [diff] [blame^] | 719 | #if !SANITIZER_DEBUG && defined(__x86_64__) && !SANITIZER_MAC |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 720 | // The caller may not create the stack frame for itself at all, |
| 721 | // so we create a reserve stack frame for it (1024b must be enough). |
| 722 | #define HACKY_CALL(f) \ |
Dmitry Vyukov | 41e8153 | 2012-09-02 11:24:07 +0000 | [diff] [blame] | 723 | __asm__ __volatile__("sub $1024, %%rsp;" \ |
Stephen Hines | 2d1fdb2 | 2014-05-28 23:58:16 -0700 | [diff] [blame] | 724 | CFI_INL_ADJUST_CFA_OFFSET(1024) \ |
Dmitry Vyukov | f5d526f | 2012-11-26 14:20:26 +0000 | [diff] [blame] | 725 | ".hidden " #f "_thunk;" \ |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 726 | "call " #f "_thunk;" \ |
Dmitry Vyukov | 41e8153 | 2012-09-02 11:24:07 +0000 | [diff] [blame] | 727 | "add $1024, %%rsp;" \ |
Stephen Hines | 2d1fdb2 | 2014-05-28 23:58:16 -0700 | [diff] [blame] | 728 | CFI_INL_ADJUST_CFA_OFFSET(-1024) \ |
Dmitry Vyukov | 41e8153 | 2012-09-02 11:24:07 +0000 | [diff] [blame] | 729 | ::: "memory", "cc"); |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 730 | #else |
| 731 | #define HACKY_CALL(f) f() |
| 732 | #endif |
| 733 | |
Dmitry Vyukov | b78caa6 | 2012-07-05 16:18:28 +0000 | [diff] [blame] | 734 | void TraceSwitch(ThreadState *thr); |
Dmitry Vyukov | 385542a | 2012-11-28 10:35:31 +0000 | [diff] [blame] | 735 | uptr TraceTopPC(ThreadState *thr); |
Dmitry Vyukov | d698edc | 2012-11-28 12:19:50 +0000 | [diff] [blame] | 736 | uptr TraceSize(); |
Dmitry Vyukov | 0415ac0 | 2012-12-04 12:19:53 +0000 | [diff] [blame] | 737 | uptr TraceParts(); |
Dmitry Vyukov | 9743d74 | 2013-03-20 10:31:53 +0000 | [diff] [blame] | 738 | Trace *ThreadTrace(int tid); |
Dmitry Vyukov | b78caa6 | 2012-07-05 16:18:28 +0000 | [diff] [blame] | 739 | |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 740 | extern "C" void __tsan_trace_switch(); |
Timur Iskhodzhanov | abfdbdf | 2013-03-28 18:52:40 +0000 | [diff] [blame] | 741 | void ALWAYS_INLINE TraceAddEvent(ThreadState *thr, FastState fs, |
Dmitry Vyukov | ad9da37 | 2012-12-06 12:16:15 +0000 | [diff] [blame] | 742 | EventType typ, u64 addr) { |
Stephen Hines | 2d1fdb2 | 2014-05-28 23:58:16 -0700 | [diff] [blame] | 743 | if (!kCollectHistory) |
| 744 | return; |
Dmitry Vyukov | ad9da37 | 2012-12-06 12:16:15 +0000 | [diff] [blame] | 745 | DCHECK_GE((int)typ, 0); |
| 746 | DCHECK_LE((int)typ, 7); |
| 747 | DCHECK_EQ(GetLsb(addr, 61), addr); |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 748 | StatInc(thr, StatEvents); |
Dmitry Vyukov | 3fb70e3 | 2012-11-28 13:01:32 +0000 | [diff] [blame] | 749 | u64 pos = fs.GetTracePos(); |
| 750 | if (UNLIKELY((pos % kTracePartSize) == 0)) { |
Stephen Hines | 86277eb | 2015-03-23 12:06:32 -0700 | [diff] [blame] | 751 | #ifndef SANITIZER_GO |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 752 | HACKY_CALL(__tsan_trace_switch); |
Dmitry Vyukov | b78caa6 | 2012-07-05 16:18:28 +0000 | [diff] [blame] | 753 | #else |
| 754 | TraceSwitch(thr); |
| 755 | #endif |
| 756 | } |
Dmitry Vyukov | 385542a | 2012-11-28 10:35:31 +0000 | [diff] [blame] | 757 | Event *trace = (Event*)GetThreadTrace(fs.tid()); |
Dmitry Vyukov | 3fb70e3 | 2012-11-28 13:01:32 +0000 | [diff] [blame] | 758 | Event *evp = &trace[pos]; |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 759 | Event ev = (u64)addr | ((u64)typ << 61); |
| 760 | *evp = ev; |
| 761 | } |
| 762 | |
Stephen Hines | 86277eb | 2015-03-23 12:06:32 -0700 | [diff] [blame] | 763 | #ifndef SANITIZER_GO |
| 764 | uptr ALWAYS_INLINE HeapEnd() { |
Pirama Arumuga Nainar | 799172d | 2016-03-03 15:50:30 -0800 | [diff] [blame^] | 765 | return HeapMemEnd() + PrimaryAllocator::AdditionalSize(); |
Stephen Hines | 86277eb | 2015-03-23 12:06:32 -0700 | [diff] [blame] | 766 | } |
| 767 | #endif |
| 768 | |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 769 | } // namespace __tsan |
| 770 | |
| 771 | #endif // TSAN_RTL_H |