Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 1 | //===-- tsan_rtl.h ----------------------------------------------*- C++ -*-===// |
| 2 | // |
| 3 | // The LLVM Compiler Infrastructure |
| 4 | // |
| 5 | // This file is distributed under the University of Illinois Open Source |
| 6 | // License. See LICENSE.TXT for details. |
| 7 | // |
| 8 | //===----------------------------------------------------------------------===// |
| 9 | // |
| 10 | // This file is a part of ThreadSanitizer (TSan), a race detector. |
| 11 | // |
| 12 | // Main internal TSan header file. |
| 13 | // |
| 14 | // Ground rules: |
| 15 | // - C++ run-time should not be used (static CTORs, RTTI, exceptions, static |
| 16 | // function-scope locals) |
| 17 | // - All functions/classes/etc reside in namespace __tsan, except for those |
| 18 | // declared in tsan_interface.h. |
| 19 | // - Platform-specific files should be used instead of ifdefs (*). |
| 20 | // - No system headers included in header files (*). |
| 21 | // - Platform specific headres included only into platform-specific files (*). |
| 22 | // |
| 23 | // (*) Except when inlining is critical for performance. |
| 24 | //===----------------------------------------------------------------------===// |
| 25 | |
| 26 | #ifndef TSAN_RTL_H |
| 27 | #define TSAN_RTL_H |
| 28 | |
Kostya Serebryany | 72166ca | 2012-12-05 10:09:15 +0000 | [diff] [blame] | 29 | #include "sanitizer_common/sanitizer_allocator.h" |
Alexey Samsonov | 1f3c2fe | 2013-05-29 09:15:39 +0000 | [diff] [blame] | 30 | #include "sanitizer_common/sanitizer_allocator_internal.h" |
Stephen Hines | 2d1fdb2 | 2014-05-28 23:58:16 -0700 | [diff] [blame] | 31 | #include "sanitizer_common/sanitizer_asm.h" |
Alexey Samsonov | 2bbd8be | 2013-03-15 13:48:44 +0000 | [diff] [blame] | 32 | #include "sanitizer_common/sanitizer_common.h" |
Stephen Hines | 2d1fdb2 | 2014-05-28 23:58:16 -0700 | [diff] [blame] | 33 | #include "sanitizer_common/sanitizer_deadlock_detector_interface.h" |
Dmitry Vyukov | 4af0f21 | 2013-10-03 13:37:17 +0000 | [diff] [blame] | 34 | #include "sanitizer_common/sanitizer_libignore.h" |
Sergey Matveev | a52e5c6 | 2013-06-26 15:37:14 +0000 | [diff] [blame] | 35 | #include "sanitizer_common/sanitizer_suppressions.h" |
Alexey Samsonov | 2bbd8be | 2013-03-15 13:48:44 +0000 | [diff] [blame] | 36 | #include "sanitizer_common/sanitizer_thread_registry.h" |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 37 | #include "tsan_clock.h" |
| 38 | #include "tsan_defs.h" |
| 39 | #include "tsan_flags.h" |
| 40 | #include "tsan_sync.h" |
| 41 | #include "tsan_trace.h" |
| 42 | #include "tsan_vector.h" |
| 43 | #include "tsan_report.h" |
Dmitry Vyukov | 385542a | 2012-11-28 10:35:31 +0000 | [diff] [blame] | 44 | #include "tsan_platform.h" |
Dmitry Vyukov | ad9da37 | 2012-12-06 12:16:15 +0000 | [diff] [blame] | 45 | #include "tsan_mutexset.h" |
Stephen Hines | 2d1fdb2 | 2014-05-28 23:58:16 -0700 | [diff] [blame] | 46 | #include "tsan_ignoreset.h" |
Stephen Hines | 6a211c5 | 2014-07-21 00:49:56 -0700 | [diff] [blame] | 47 | #include "tsan_stack_trace.h" |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 48 | |
Kostya Serebryany | 503a3af | 2012-12-04 15:13:30 +0000 | [diff] [blame] | 49 | #if SANITIZER_WORDSIZE != 64 |
| 50 | # error "ThreadSanitizer is supported only on 64-bit platforms" |
| 51 | #endif |
| 52 | |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 53 | namespace __tsan { |
| 54 | |
Stephen Hines | 86277eb | 2015-03-23 12:06:32 -0700 | [diff] [blame] | 55 | #ifndef SANITIZER_GO |
Dmitry Vyukov | e93e505 | 2013-03-18 10:32:21 +0000 | [diff] [blame] | 56 | struct MapUnmapCallback; |
Stephen Hines | 86277eb | 2015-03-23 12:06:32 -0700 | [diff] [blame] | 57 | #ifdef __mips64 |
| 58 | static const uptr kAllocatorSpace = 0; |
| 59 | static const uptr kAllocatorSize = SANITIZER_MMAP_RANGE_SIZE; |
| 60 | static const uptr kAllocatorRegionSizeLog = 20; |
| 61 | static const uptr kAllocatorNumRegions = |
| 62 | kAllocatorSize >> kAllocatorRegionSizeLog; |
| 63 | typedef TwoLevelByteMap<(kAllocatorNumRegions >> 12), 1 << 12, |
| 64 | MapUnmapCallback> ByteMap; |
| 65 | typedef SizeClassAllocator32<kAllocatorSpace, kAllocatorSize, 0, |
| 66 | CompactSizeClassMap, kAllocatorRegionSizeLog, ByteMap, |
| 67 | MapUnmapCallback> PrimaryAllocator; |
| 68 | #else |
Stephen Hines | 6d18623 | 2014-11-26 17:56:19 -0800 | [diff] [blame] | 69 | typedef SizeClassAllocator64<kHeapMemBeg, kHeapMemEnd - kHeapMemBeg, 0, |
Dmitry Vyukov | e93e505 | 2013-03-18 10:32:21 +0000 | [diff] [blame] | 70 | DefaultSizeClassMap, MapUnmapCallback> PrimaryAllocator; |
Stephen Hines | 86277eb | 2015-03-23 12:06:32 -0700 | [diff] [blame] | 71 | #endif |
Kostya Serebryany | 82de942 | 2012-12-04 14:15:17 +0000 | [diff] [blame] | 72 | typedef SizeClassAllocatorLocalCache<PrimaryAllocator> AllocatorCache; |
Dmitry Vyukov | e93e505 | 2013-03-18 10:32:21 +0000 | [diff] [blame] | 73 | typedef LargeMmapAllocator<MapUnmapCallback> SecondaryAllocator; |
Dmitry Vyukov | 2e87051 | 2012-08-15 15:35:15 +0000 | [diff] [blame] | 74 | typedef CombinedAllocator<PrimaryAllocator, AllocatorCache, |
| 75 | SecondaryAllocator> Allocator; |
Dmitry Vyukov | ff35f1d | 2012-08-30 13:02:30 +0000 | [diff] [blame] | 76 | Allocator *allocator(); |
Dmitry Vyukov | 2e87051 | 2012-08-15 15:35:15 +0000 | [diff] [blame] | 77 | #endif |
| 78 | |
Alexey Samsonov | 591616d | 2012-09-11 09:44:48 +0000 | [diff] [blame] | 79 | void TsanCheckFailed(const char *file, int line, const char *cond, |
| 80 | u64 v1, u64 v2); |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 81 | |
Dmitry Vyukov | 9743d74 | 2013-03-20 10:31:53 +0000 | [diff] [blame] | 82 | const u64 kShadowRodata = (u64)-1; // .rodata shadow marker |
| 83 | |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 84 | // FastState (from most significant bit): |
Dmitry Vyukov | 0d35d9d | 2012-11-28 10:49:27 +0000 | [diff] [blame] | 85 | // ignore : 1 |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 86 | // tid : kTidBits |
Dmitry Vyukov | 069ce82 | 2012-05-17 14:17:51 +0000 | [diff] [blame] | 87 | // unused : - |
Dmitry Vyukov | d698edc | 2012-11-28 12:19:50 +0000 | [diff] [blame] | 88 | // history_size : 3 |
Stephen Hines | 6a211c5 | 2014-07-21 00:49:56 -0700 | [diff] [blame] | 89 | // epoch : kClkBits |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 90 | class FastState { |
| 91 | public: |
| 92 | FastState(u64 tid, u64 epoch) { |
Dmitry Vyukov | 069ce82 | 2012-05-17 14:17:51 +0000 | [diff] [blame] | 93 | x_ = tid << kTidShift; |
Stephen Hines | 6a211c5 | 2014-07-21 00:49:56 -0700 | [diff] [blame] | 94 | x_ |= epoch; |
Dmitry Vyukov | 0d35d9d | 2012-11-28 10:49:27 +0000 | [diff] [blame] | 95 | DCHECK_EQ(tid, this->tid()); |
| 96 | DCHECK_EQ(epoch, this->epoch()); |
| 97 | DCHECK_EQ(GetIgnoreBit(), false); |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 98 | } |
| 99 | |
| 100 | explicit FastState(u64 x) |
| 101 | : x_(x) { |
| 102 | } |
| 103 | |
Dmitry Vyukov | 332c62b | 2012-08-16 15:08:49 +0000 | [diff] [blame] | 104 | u64 raw() const { |
| 105 | return x_; |
| 106 | } |
| 107 | |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 108 | u64 tid() const { |
Dmitry Vyukov | c8f0a00 | 2012-11-30 20:02:11 +0000 | [diff] [blame] | 109 | u64 res = (x_ & ~kIgnoreBit) >> kTidShift; |
| 110 | return res; |
| 111 | } |
| 112 | |
| 113 | u64 TidWithIgnore() const { |
Dmitry Vyukov | 069ce82 | 2012-05-17 14:17:51 +0000 | [diff] [blame] | 114 | u64 res = x_ >> kTidShift; |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 115 | return res; |
| 116 | } |
Dmitry Vyukov | 069ce82 | 2012-05-17 14:17:51 +0000 | [diff] [blame] | 117 | |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 118 | u64 epoch() const { |
Stephen Hines | 6a211c5 | 2014-07-21 00:49:56 -0700 | [diff] [blame] | 119 | u64 res = x_ & ((1ull << kClkBits) - 1); |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 120 | return res; |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 121 | } |
Dmitry Vyukov | 069ce82 | 2012-05-17 14:17:51 +0000 | [diff] [blame] | 122 | |
| 123 | void IncrementEpoch() { |
| 124 | u64 old_epoch = epoch(); |
Stephen Hines | 6a211c5 | 2014-07-21 00:49:56 -0700 | [diff] [blame] | 125 | x_ += 1; |
Dmitry Vyukov | e784ad4 | 2012-05-21 10:20:53 +0000 | [diff] [blame] | 126 | DCHECK_EQ(old_epoch + 1, epoch()); |
Dmitry Vyukov | 069ce82 | 2012-05-17 14:17:51 +0000 | [diff] [blame] | 127 | (void)old_epoch; |
| 128 | } |
| 129 | |
| 130 | void SetIgnoreBit() { x_ |= kIgnoreBit; } |
| 131 | void ClearIgnoreBit() { x_ &= ~kIgnoreBit; } |
Dmitry Vyukov | 0d35d9d | 2012-11-28 10:49:27 +0000 | [diff] [blame] | 132 | bool GetIgnoreBit() const { return (s64)x_ < 0; } |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 133 | |
Dmitry Vyukov | d698edc | 2012-11-28 12:19:50 +0000 | [diff] [blame] | 134 | void SetHistorySize(int hs) { |
| 135 | CHECK_GE(hs, 0); |
| 136 | CHECK_LE(hs, 7); |
Stephen Hines | 6a211c5 | 2014-07-21 00:49:56 -0700 | [diff] [blame] | 137 | x_ = (x_ & ~(kHistoryMask << kHistoryShift)) | (u64(hs) << kHistoryShift); |
Dmitry Vyukov | d698edc | 2012-11-28 12:19:50 +0000 | [diff] [blame] | 138 | } |
| 139 | |
Stephen Hines | 6a211c5 | 2014-07-21 00:49:56 -0700 | [diff] [blame] | 140 | ALWAYS_INLINE |
Dmitry Vyukov | d698edc | 2012-11-28 12:19:50 +0000 | [diff] [blame] | 141 | int GetHistorySize() const { |
Stephen Hines | 6a211c5 | 2014-07-21 00:49:56 -0700 | [diff] [blame] | 142 | return (int)((x_ >> kHistoryShift) & kHistoryMask); |
Dmitry Vyukov | d698edc | 2012-11-28 12:19:50 +0000 | [diff] [blame] | 143 | } |
| 144 | |
| 145 | void ClearHistorySize() { |
Stephen Hines | 6a211c5 | 2014-07-21 00:49:56 -0700 | [diff] [blame] | 146 | SetHistorySize(0); |
Dmitry Vyukov | d698edc | 2012-11-28 12:19:50 +0000 | [diff] [blame] | 147 | } |
| 148 | |
Stephen Hines | 6a211c5 | 2014-07-21 00:49:56 -0700 | [diff] [blame] | 149 | ALWAYS_INLINE |
Dmitry Vyukov | d698edc | 2012-11-28 12:19:50 +0000 | [diff] [blame] | 150 | u64 GetTracePos() const { |
| 151 | const int hs = GetHistorySize(); |
| 152 | // When hs == 0, the trace consists of 2 parts. |
| 153 | const u64 mask = (1ull << (kTracePartSizeBits + hs + 1)) - 1; |
| 154 | return epoch() & mask; |
| 155 | } |
| 156 | |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 157 | private: |
| 158 | friend class Shadow; |
Dmitry Vyukov | 069ce82 | 2012-05-17 14:17:51 +0000 | [diff] [blame] | 159 | static const int kTidShift = 64 - kTidBits - 1; |
Dmitry Vyukov | 0d35d9d | 2012-11-28 10:49:27 +0000 | [diff] [blame] | 160 | static const u64 kIgnoreBit = 1ull << 63; |
Dmitry Vyukov | 069ce82 | 2012-05-17 14:17:51 +0000 | [diff] [blame] | 161 | static const u64 kFreedBit = 1ull << 63; |
Stephen Hines | 6a211c5 | 2014-07-21 00:49:56 -0700 | [diff] [blame] | 162 | static const u64 kHistoryShift = kClkBits; |
| 163 | static const u64 kHistoryMask = 7; |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 164 | u64 x_; |
| 165 | }; |
| 166 | |
| 167 | // Shadow (from most significant bit): |
Dmitry Vyukov | 069ce82 | 2012-05-17 14:17:51 +0000 | [diff] [blame] | 168 | // freed : 1 |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 169 | // tid : kTidBits |
Dmitry Vyukov | 334553e | 2013-02-01 09:42:06 +0000 | [diff] [blame] | 170 | // is_atomic : 1 |
Dmitry Vyukov | 33a040a | 2013-02-01 10:02:55 +0000 | [diff] [blame] | 171 | // is_read : 1 |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 172 | // size_log : 2 |
| 173 | // addr0 : 3 |
Stephen Hines | 6a211c5 | 2014-07-21 00:49:56 -0700 | [diff] [blame] | 174 | // epoch : kClkBits |
Dmitry Vyukov | e963666 | 2012-06-27 16:05:06 +0000 | [diff] [blame] | 175 | class Shadow : public FastState { |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 176 | public: |
Dmitry Vyukov | d698edc | 2012-11-28 12:19:50 +0000 | [diff] [blame] | 177 | explicit Shadow(u64 x) |
| 178 | : FastState(x) { |
| 179 | } |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 180 | |
Dmitry Vyukov | d698edc | 2012-11-28 12:19:50 +0000 | [diff] [blame] | 181 | explicit Shadow(const FastState &s) |
| 182 | : FastState(s.x_) { |
| 183 | ClearHistorySize(); |
| 184 | } |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 185 | |
| 186 | void SetAddr0AndSizeLog(u64 addr0, unsigned kAccessSizeLog) { |
Stephen Hines | 6a211c5 | 2014-07-21 00:49:56 -0700 | [diff] [blame] | 187 | DCHECK_EQ((x_ >> kClkBits) & 31, 0); |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 188 | DCHECK_LE(addr0, 7); |
| 189 | DCHECK_LE(kAccessSizeLog, 3); |
Stephen Hines | 6a211c5 | 2014-07-21 00:49:56 -0700 | [diff] [blame] | 190 | x_ |= ((kAccessSizeLog << 3) | addr0) << kClkBits; |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 191 | DCHECK_EQ(kAccessSizeLog, size_log()); |
| 192 | DCHECK_EQ(addr0, this->addr0()); |
| 193 | } |
| 194 | |
| 195 | void SetWrite(unsigned kAccessIsWrite) { |
Dmitry Vyukov | 33a040a | 2013-02-01 10:02:55 +0000 | [diff] [blame] | 196 | DCHECK_EQ(x_ & kReadBit, 0); |
| 197 | if (!kAccessIsWrite) |
| 198 | x_ |= kReadBit; |
Dmitry Vyukov | 334553e | 2013-02-01 09:42:06 +0000 | [diff] [blame] | 199 | DCHECK_EQ(kAccessIsWrite, IsWrite()); |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 200 | } |
| 201 | |
Dmitry Vyukov | 334553e | 2013-02-01 09:42:06 +0000 | [diff] [blame] | 202 | void SetAtomic(bool kIsAtomic) { |
| 203 | DCHECK(!IsAtomic()); |
| 204 | if (kIsAtomic) |
| 205 | x_ |= kAtomicBit; |
| 206 | DCHECK_EQ(IsAtomic(), kIsAtomic); |
| 207 | } |
| 208 | |
| 209 | bool IsAtomic() const { |
| 210 | return x_ & kAtomicBit; |
| 211 | } |
| 212 | |
| 213 | bool IsZero() const { |
| 214 | return x_ == 0; |
| 215 | } |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 216 | |
Dmitry Vyukov | adfb650 | 2012-05-22 18:07:45 +0000 | [diff] [blame] | 217 | static inline bool TidsAreEqual(const Shadow s1, const Shadow s2) { |
Dmitry Vyukov | 069ce82 | 2012-05-17 14:17:51 +0000 | [diff] [blame] | 218 | u64 shifted_xor = (s1.x_ ^ s2.x_) >> kTidShift; |
Dmitry Vyukov | c8f0a00 | 2012-11-30 20:02:11 +0000 | [diff] [blame] | 219 | DCHECK_EQ(shifted_xor == 0, s1.TidWithIgnore() == s2.TidWithIgnore()); |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 220 | return shifted_xor == 0; |
| 221 | } |
Dmitry Vyukov | adfb650 | 2012-05-22 18:07:45 +0000 | [diff] [blame] | 222 | |
Stephen Hines | 6a211c5 | 2014-07-21 00:49:56 -0700 | [diff] [blame] | 223 | static ALWAYS_INLINE |
| 224 | bool Addr0AndSizeAreEqual(const Shadow s1, const Shadow s2) { |
| 225 | u64 masked_xor = ((s1.x_ ^ s2.x_) >> kClkBits) & 31; |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 226 | return masked_xor == 0; |
| 227 | } |
| 228 | |
Stephen Hines | 6a211c5 | 2014-07-21 00:49:56 -0700 | [diff] [blame] | 229 | static ALWAYS_INLINE bool TwoRangesIntersect(Shadow s1, Shadow s2, |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 230 | unsigned kS2AccessSize) { |
| 231 | bool res = false; |
| 232 | u64 diff = s1.addr0() - s2.addr0(); |
| 233 | if ((s64)diff < 0) { // s1.addr0 < s2.addr0 // NOLINT |
| 234 | // if (s1.addr0() + size1) > s2.addr0()) return true; |
Stephen Hines | 6a211c5 | 2014-07-21 00:49:56 -0700 | [diff] [blame] | 235 | if (s1.size() > -diff) |
| 236 | res = true; |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 237 | } else { |
| 238 | // if (s2.addr0() + kS2AccessSize > s1.addr0()) return true; |
Stephen Hines | 6a211c5 | 2014-07-21 00:49:56 -0700 | [diff] [blame] | 239 | if (kS2AccessSize > diff) |
| 240 | res = true; |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 241 | } |
Stephen Hines | 6a211c5 | 2014-07-21 00:49:56 -0700 | [diff] [blame] | 242 | DCHECK_EQ(res, TwoRangesIntersectSlow(s1, s2)); |
| 243 | DCHECK_EQ(res, TwoRangesIntersectSlow(s2, s1)); |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 244 | return res; |
| 245 | } |
| 246 | |
Stephen Hines | 6a211c5 | 2014-07-21 00:49:56 -0700 | [diff] [blame] | 247 | u64 ALWAYS_INLINE addr0() const { return (x_ >> kClkBits) & 7; } |
| 248 | u64 ALWAYS_INLINE size() const { return 1ull << size_log(); } |
| 249 | bool ALWAYS_INLINE IsWrite() const { return !IsRead(); } |
| 250 | bool ALWAYS_INLINE IsRead() const { return x_ & kReadBit; } |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 251 | |
Dmitry Vyukov | 069ce82 | 2012-05-17 14:17:51 +0000 | [diff] [blame] | 252 | // The idea behind the freed bit is as follows. |
| 253 | // When the memory is freed (or otherwise unaccessible) we write to the shadow |
| 254 | // values with tid/epoch related to the free and the freed bit set. |
| 255 | // During memory accesses processing the freed bit is considered |
| 256 | // as msb of tid. So any access races with shadow with freed bit set |
| 257 | // (it is as if write from a thread with which we never synchronized before). |
| 258 | // This allows us to detect accesses to freed memory w/o additional |
| 259 | // overheads in memory access processing and at the same time restore |
| 260 | // tid/epoch of free. |
| 261 | void MarkAsFreed() { |
| 262 | x_ |= kFreedBit; |
| 263 | } |
| 264 | |
Dmitry Vyukov | 3285866 | 2013-02-01 14:41:58 +0000 | [diff] [blame] | 265 | bool IsFreed() const { |
| 266 | return x_ & kFreedBit; |
| 267 | } |
| 268 | |
Dmitry Vyukov | 069ce82 | 2012-05-17 14:17:51 +0000 | [diff] [blame] | 269 | bool GetFreedAndReset() { |
| 270 | bool res = x_ & kFreedBit; |
| 271 | x_ &= ~kFreedBit; |
| 272 | return res; |
| 273 | } |
| 274 | |
Stephen Hines | 6a211c5 | 2014-07-21 00:49:56 -0700 | [diff] [blame] | 275 | bool ALWAYS_INLINE IsBothReadsOrAtomic(bool kIsWrite, bool kIsAtomic) const { |
| 276 | bool v = x_ & ((u64(kIsWrite ^ 1) << kReadShift) |
| 277 | | (u64(kIsAtomic) << kAtomicShift)); |
Dmitry Vyukov | 334553e | 2013-02-01 09:42:06 +0000 | [diff] [blame] | 278 | DCHECK_EQ(v, (!IsWrite() && !kIsWrite) || (IsAtomic() && kIsAtomic)); |
| 279 | return v; |
| 280 | } |
| 281 | |
Stephen Hines | 6a211c5 | 2014-07-21 00:49:56 -0700 | [diff] [blame] | 282 | bool ALWAYS_INLINE IsRWNotWeaker(bool kIsWrite, bool kIsAtomic) const { |
Dmitry Vyukov | 33a040a | 2013-02-01 10:02:55 +0000 | [diff] [blame] | 283 | bool v = ((x_ >> kReadShift) & 3) |
Dmitry Vyukov | 334553e | 2013-02-01 09:42:06 +0000 | [diff] [blame] | 284 | <= u64((kIsWrite ^ 1) | (kIsAtomic << 1)); |
| 285 | DCHECK_EQ(v, (IsAtomic() < kIsAtomic) || |
| 286 | (IsAtomic() == kIsAtomic && !IsWrite() <= !kIsWrite)); |
| 287 | return v; |
| 288 | } |
| 289 | |
Stephen Hines | 6a211c5 | 2014-07-21 00:49:56 -0700 | [diff] [blame] | 290 | bool ALWAYS_INLINE IsRWWeakerOrEqual(bool kIsWrite, bool kIsAtomic) const { |
Dmitry Vyukov | 33a040a | 2013-02-01 10:02:55 +0000 | [diff] [blame] | 291 | bool v = ((x_ >> kReadShift) & 3) |
Dmitry Vyukov | 334553e | 2013-02-01 09:42:06 +0000 | [diff] [blame] | 292 | >= u64((kIsWrite ^ 1) | (kIsAtomic << 1)); |
| 293 | DCHECK_EQ(v, (IsAtomic() > kIsAtomic) || |
| 294 | (IsAtomic() == kIsAtomic && !IsWrite() >= !kIsWrite)); |
| 295 | return v; |
| 296 | } |
| 297 | |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 298 | private: |
Stephen Hines | 6a211c5 | 2014-07-21 00:49:56 -0700 | [diff] [blame] | 299 | static const u64 kReadShift = 5 + kClkBits; |
Dmitry Vyukov | 33a040a | 2013-02-01 10:02:55 +0000 | [diff] [blame] | 300 | static const u64 kReadBit = 1ull << kReadShift; |
Stephen Hines | 6a211c5 | 2014-07-21 00:49:56 -0700 | [diff] [blame] | 301 | static const u64 kAtomicShift = 6 + kClkBits; |
Dmitry Vyukov | 334553e | 2013-02-01 09:42:06 +0000 | [diff] [blame] | 302 | static const u64 kAtomicBit = 1ull << kAtomicShift; |
| 303 | |
Stephen Hines | 6a211c5 | 2014-07-21 00:49:56 -0700 | [diff] [blame] | 304 | u64 size_log() const { return (x_ >> (3 + kClkBits)) & 3; } |
Dmitry Vyukov | adfb650 | 2012-05-22 18:07:45 +0000 | [diff] [blame] | 305 | |
Stephen Hines | 6a211c5 | 2014-07-21 00:49:56 -0700 | [diff] [blame] | 306 | static bool TwoRangesIntersectSlow(const Shadow s1, const Shadow s2) { |
Dmitry Vyukov | adfb650 | 2012-05-22 18:07:45 +0000 | [diff] [blame] | 307 | if (s1.addr0() == s2.addr0()) return true; |
| 308 | if (s1.addr0() < s2.addr0() && s1.addr0() + s1.size() > s2.addr0()) |
| 309 | return true; |
| 310 | if (s2.addr0() < s1.addr0() && s2.addr0() + s2.size() > s1.addr0()) |
| 311 | return true; |
| 312 | return false; |
| 313 | } |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 314 | }; |
| 315 | |
Pirama Arumuga Nainar | 7c91505 | 2015-04-08 08:58:29 -0700 | [diff] [blame] | 316 | struct ThreadSignalContext; |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 317 | |
Dmitry Vyukov | 8b30c25 | 2013-03-25 10:10:44 +0000 | [diff] [blame] | 318 | struct JmpBuf { |
| 319 | uptr sp; |
| 320 | uptr mangled_sp; |
Stephen Hines | 6d18623 | 2014-11-26 17:56:19 -0800 | [diff] [blame] | 321 | int int_signal_send; |
| 322 | bool in_blocking_func; |
| 323 | uptr in_signal_handler; |
Dmitry Vyukov | 8b30c25 | 2013-03-25 10:10:44 +0000 | [diff] [blame] | 324 | uptr *shadow_stack_pos; |
| 325 | }; |
| 326 | |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 327 | // This struct is stored in TLS. |
| 328 | struct ThreadState { |
| 329 | FastState fast_state; |
| 330 | // Synch epoch represents the threads's epoch before the last synchronization |
| 331 | // action. It allows to reduce number of shadow state updates. |
| 332 | // For example, fast_synch_epoch=100, last write to addr X was at epoch=150, |
| 333 | // if we are processing write to X from the same thread at epoch=200, |
| 334 | // we do nothing, because both writes happen in the same 'synch epoch'. |
| 335 | // That is, if another memory access does not race with the former write, |
| 336 | // it does not race with the latter as well. |
| 337 | // QUESTION: can we can squeeze this into ThreadState::Fast? |
| 338 | // E.g. ThreadState::Fast is a 44-bit, 32 are taken by synch_epoch and 12 are |
| 339 | // taken by epoch between synchs. |
| 340 | // This way we can save one load from tls. |
| 341 | u64 fast_synch_epoch; |
| 342 | // This is a slow path flag. On fast path, fast_state.GetIgnoreBit() is read. |
| 343 | // We do not distinguish beteween ignoring reads and writes |
| 344 | // for better performance. |
| 345 | int ignore_reads_and_writes; |
Dmitry Vyukov | e1ddbf9 | 2013-10-10 15:58:12 +0000 | [diff] [blame] | 346 | int ignore_sync; |
Stephen Hines | 2d1fdb2 | 2014-05-28 23:58:16 -0700 | [diff] [blame] | 347 | // Go does not support ignores. |
Stephen Hines | 86277eb | 2015-03-23 12:06:32 -0700 | [diff] [blame] | 348 | #ifndef SANITIZER_GO |
Stephen Hines | 2d1fdb2 | 2014-05-28 23:58:16 -0700 | [diff] [blame] | 349 | IgnoreSet mop_ignore_set; |
| 350 | IgnoreSet sync_ignore_set; |
| 351 | #endif |
Dmitry Vyukov | 01a7ce8 | 2013-10-16 15:35:12 +0000 | [diff] [blame] | 352 | // C/C++ uses fixed size shadow stack embed into Trace. |
| 353 | // Go uses malloc-allocated shadow stack with dynamic size. |
| 354 | uptr *shadow_stack; |
| 355 | uptr *shadow_stack_end; |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 356 | uptr *shadow_stack_pos; |
| 357 | u64 *racy_shadow_addr; |
| 358 | u64 racy_state[2]; |
Dmitry Vyukov | ad9da37 | 2012-12-06 12:16:15 +0000 | [diff] [blame] | 359 | MutexSet mset; |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 360 | ThreadClock clock; |
Stephen Hines | 86277eb | 2015-03-23 12:06:32 -0700 | [diff] [blame] | 361 | #ifndef SANITIZER_GO |
Dmitry Vyukov | 2e87051 | 2012-08-15 15:35:15 +0000 | [diff] [blame] | 362 | AllocatorCache alloc_cache; |
Alexey Samsonov | 1f3c2fe | 2013-05-29 09:15:39 +0000 | [diff] [blame] | 363 | InternalAllocatorCache internal_alloc_cache; |
Dmitry Vyukov | 8b30c25 | 2013-03-25 10:10:44 +0000 | [diff] [blame] | 364 | Vector<JmpBuf> jmp_bufs; |
Stephen Hines | 2d1fdb2 | 2014-05-28 23:58:16 -0700 | [diff] [blame] | 365 | int ignore_interceptors; |
Dmitry Vyukov | 2e87051 | 2012-08-15 15:35:15 +0000 | [diff] [blame] | 366 | #endif |
Stephen Hines | 86277eb | 2015-03-23 12:06:32 -0700 | [diff] [blame] | 367 | #if TSAN_COLLECT_STATS |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 368 | u64 stat[StatCnt]; |
Stephen Hines | 86277eb | 2015-03-23 12:06:32 -0700 | [diff] [blame] | 369 | #endif |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 370 | const int tid; |
Dmitry Vyukov | ff35f1d | 2012-08-30 13:02:30 +0000 | [diff] [blame] | 371 | const int unique_id; |
Dmitry Vyukov | 4e81d0e | 2013-01-29 13:03:07 +0000 | [diff] [blame] | 372 | bool in_symbolizer; |
Dmitry Vyukov | 4af0f21 | 2013-10-03 13:37:17 +0000 | [diff] [blame] | 373 | bool in_ignored_lib; |
Pirama Arumuga Nainar | 7c91505 | 2015-04-08 08:58:29 -0700 | [diff] [blame] | 374 | bool is_inited; |
Stephen Hines | 6d18623 | 2014-11-26 17:56:19 -0800 | [diff] [blame] | 375 | bool is_dead; |
Dmitry Vyukov | 3285866 | 2013-02-01 14:41:58 +0000 | [diff] [blame] | 376 | bool is_freeing; |
Dmitry Vyukov | 0dc47b6 | 2013-03-21 15:37:39 +0000 | [diff] [blame] | 377 | bool is_vptr_access; |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 378 | const uptr stk_addr; |
| 379 | const uptr stk_size; |
| 380 | const uptr tls_addr; |
| 381 | const uptr tls_size; |
Stephen Hines | 2d1fdb2 | 2014-05-28 23:58:16 -0700 | [diff] [blame] | 382 | ThreadContext *tctx; |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 383 | |
Stephen Hines | 86277eb | 2015-03-23 12:06:32 -0700 | [diff] [blame] | 384 | #if SANITIZER_DEBUG && !SANITIZER_GO |
Stephen Hines | 2d1fdb2 | 2014-05-28 23:58:16 -0700 | [diff] [blame] | 385 | InternalDeadlockDetector internal_deadlock_detector; |
Stephen Hines | 86277eb | 2015-03-23 12:06:32 -0700 | [diff] [blame] | 386 | #endif |
Stephen Hines | 2d1fdb2 | 2014-05-28 23:58:16 -0700 | [diff] [blame] | 387 | DDPhysicalThread *dd_pt; |
| 388 | DDLogicalThread *dd_lt; |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 389 | |
Stephen Hines | 6d18623 | 2014-11-26 17:56:19 -0800 | [diff] [blame] | 390 | atomic_uintptr_t in_signal_handler; |
Pirama Arumuga Nainar | 7c91505 | 2015-04-08 08:58:29 -0700 | [diff] [blame] | 391 | ThreadSignalContext *signal_ctx; |
Dmitry Vyukov | e963666 | 2012-06-27 16:05:06 +0000 | [diff] [blame] | 392 | |
Stephen Hines | 6a211c5 | 2014-07-21 00:49:56 -0700 | [diff] [blame] | 393 | DenseSlabAllocCache block_cache; |
| 394 | DenseSlabAllocCache sync_cache; |
Stephen Hines | 6d18623 | 2014-11-26 17:56:19 -0800 | [diff] [blame] | 395 | DenseSlabAllocCache clock_cache; |
Stephen Hines | 6a211c5 | 2014-07-21 00:49:56 -0700 | [diff] [blame] | 396 | |
Stephen Hines | 86277eb | 2015-03-23 12:06:32 -0700 | [diff] [blame] | 397 | #ifndef SANITIZER_GO |
Dmitry Vyukov | 8485311 | 2012-08-31 17:27:49 +0000 | [diff] [blame] | 398 | u32 last_sleep_stack_id; |
| 399 | ThreadClock last_sleep_clock; |
| 400 | #endif |
| 401 | |
Dmitry Vyukov | 9ad7c32 | 2012-06-22 11:08:55 +0000 | [diff] [blame] | 402 | // Set in regions of runtime that must be signal-safe and fork-safe. |
| 403 | // If set, malloc must not be called. |
| 404 | int nomalloc; |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 405 | |
Dmitry Vyukov | ff35f1d | 2012-08-30 13:02:30 +0000 | [diff] [blame] | 406 | explicit ThreadState(Context *ctx, int tid, int unique_id, u64 epoch, |
Stephen Hines | 2d1fdb2 | 2014-05-28 23:58:16 -0700 | [diff] [blame] | 407 | unsigned reuse_count, |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 408 | uptr stk_addr, uptr stk_size, |
| 409 | uptr tls_addr, uptr tls_size); |
| 410 | }; |
| 411 | |
Stephen Hines | 86277eb | 2015-03-23 12:06:32 -0700 | [diff] [blame] | 412 | #ifndef SANITIZER_GO |
Stephen Hines | 2d1fdb2 | 2014-05-28 23:58:16 -0700 | [diff] [blame] | 413 | __attribute__((tls_model("initial-exec"))) |
Dmitry Vyukov | b78caa6 | 2012-07-05 16:18:28 +0000 | [diff] [blame] | 414 | extern THREADLOCAL char cur_thread_placeholder[]; |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 415 | INLINE ThreadState *cur_thread() { |
| 416 | return reinterpret_cast<ThreadState *>(&cur_thread_placeholder); |
| 417 | } |
Dmitry Vyukov | b78caa6 | 2012-07-05 16:18:28 +0000 | [diff] [blame] | 418 | #endif |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 419 | |
Alexey Samsonov | 2bbd8be | 2013-03-15 13:48:44 +0000 | [diff] [blame] | 420 | class ThreadContext : public ThreadContextBase { |
| 421 | public: |
| 422 | explicit ThreadContext(int tid); |
Dmitry Vyukov | 6af642e | 2013-03-18 10:10:15 +0000 | [diff] [blame] | 423 | ~ThreadContext(); |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 424 | ThreadState *thr; |
Dmitry Vyukov | 2c5284e | 2013-03-18 09:02:27 +0000 | [diff] [blame] | 425 | u32 creation_stack_id; |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 426 | SyncClock sync; |
| 427 | // Epoch at which the thread had started. |
| 428 | // If we see an event from the thread stamped by an older epoch, |
| 429 | // the event is from a dead thread that shared tid with this thread. |
| 430 | u64 epoch0; |
| 431 | u64 epoch1; |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 432 | |
Alexey Samsonov | 2bbd8be | 2013-03-15 13:48:44 +0000 | [diff] [blame] | 433 | // Override superclass callbacks. |
Pirama Arumuga Nainar | 259f706 | 2015-05-06 11:49:53 -0700 | [diff] [blame] | 434 | void OnDead() override; |
| 435 | void OnJoined(void *arg) override; |
| 436 | void OnFinished() override; |
| 437 | void OnStarted(void *arg) override; |
| 438 | void OnCreated(void *arg) override; |
| 439 | void OnReset() override; |
| 440 | void OnDetached(void *arg) override; |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 441 | }; |
| 442 | |
| 443 | struct RacyStacks { |
| 444 | MD5Hash hash[2]; |
| 445 | bool operator==(const RacyStacks &other) const { |
| 446 | if (hash[0] == other.hash[0] && hash[1] == other.hash[1]) |
| 447 | return true; |
| 448 | if (hash[0] == other.hash[1] && hash[1] == other.hash[0]) |
| 449 | return true; |
| 450 | return false; |
| 451 | } |
| 452 | }; |
| 453 | |
| 454 | struct RacyAddress { |
| 455 | uptr addr_min; |
| 456 | uptr addr_max; |
| 457 | }; |
| 458 | |
Dmitry Vyukov | 158c6ac | 2012-10-05 15:51:32 +0000 | [diff] [blame] | 459 | struct FiredSuppression { |
| 460 | ReportType type; |
| 461 | uptr pc; |
Dmitry Vyukov | f754eb5 | 2013-03-27 17:59:57 +0000 | [diff] [blame] | 462 | Suppression *supp; |
Dmitry Vyukov | 158c6ac | 2012-10-05 15:51:32 +0000 | [diff] [blame] | 463 | }; |
| 464 | |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 465 | struct Context { |
| 466 | Context(); |
| 467 | |
| 468 | bool initialized; |
Stephen Hines | 2d1fdb2 | 2014-05-28 23:58:16 -0700 | [diff] [blame] | 469 | bool after_multithreaded_fork; |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 470 | |
Stephen Hines | 6a211c5 | 2014-07-21 00:49:56 -0700 | [diff] [blame] | 471 | MetaMap metamap; |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 472 | |
| 473 | Mutex report_mtx; |
| 474 | int nreported; |
| 475 | int nmissed_expected; |
Dmitry Vyukov | a38e40f | 2013-03-21 07:02:36 +0000 | [diff] [blame] | 476 | atomic_uint64_t last_symbolize_time_ns; |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 477 | |
Stephen Hines | 2d1fdb2 | 2014-05-28 23:58:16 -0700 | [diff] [blame] | 478 | void *background_thread; |
| 479 | atomic_uint32_t stop_background_thread; |
| 480 | |
Alexey Samsonov | 2bbd8be | 2013-03-15 13:48:44 +0000 | [diff] [blame] | 481 | ThreadRegistry *thread_registry; |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 482 | |
| 483 | Vector<RacyStacks> racy_stacks; |
| 484 | Vector<RacyAddress> racy_addresses; |
Alexey Samsonov | 0a05e5f | 2013-06-14 11:18:58 +0000 | [diff] [blame] | 485 | // Number of fired suppressions may be large enough. |
| 486 | InternalMmapVector<FiredSuppression> fired_suppressions; |
Stephen Hines | 2d1fdb2 | 2014-05-28 23:58:16 -0700 | [diff] [blame] | 487 | DDetector *dd; |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 488 | |
Stephen Hines | 6d18623 | 2014-11-26 17:56:19 -0800 | [diff] [blame] | 489 | ClockAlloc clock_alloc; |
| 490 | |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 491 | Flags flags; |
| 492 | |
| 493 | u64 stat[StatCnt]; |
| 494 | u64 int_alloc_cnt[MBlockTypeCount]; |
| 495 | u64 int_alloc_siz[MBlockTypeCount]; |
| 496 | }; |
| 497 | |
Stephen Hines | 2d1fdb2 | 2014-05-28 23:58:16 -0700 | [diff] [blame] | 498 | extern Context *ctx; // The one and the only global runtime context. |
| 499 | |
| 500 | struct ScopedIgnoreInterceptors { |
| 501 | ScopedIgnoreInterceptors() { |
Stephen Hines | 86277eb | 2015-03-23 12:06:32 -0700 | [diff] [blame] | 502 | #ifndef SANITIZER_GO |
Stephen Hines | 2d1fdb2 | 2014-05-28 23:58:16 -0700 | [diff] [blame] | 503 | cur_thread()->ignore_interceptors++; |
| 504 | #endif |
| 505 | } |
| 506 | |
| 507 | ~ScopedIgnoreInterceptors() { |
Stephen Hines | 86277eb | 2015-03-23 12:06:32 -0700 | [diff] [blame] | 508 | #ifndef SANITIZER_GO |
Stephen Hines | 2d1fdb2 | 2014-05-28 23:58:16 -0700 | [diff] [blame] | 509 | cur_thread()->ignore_interceptors--; |
| 510 | #endif |
| 511 | } |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 512 | }; |
| 513 | |
| 514 | class ScopedReport { |
| 515 | public: |
| 516 | explicit ScopedReport(ReportType typ); |
| 517 | ~ScopedReport(); |
| 518 | |
Stephen Hines | 6d18623 | 2014-11-26 17:56:19 -0800 | [diff] [blame] | 519 | void AddMemoryAccess(uptr addr, Shadow s, StackTrace stack, |
Dmitry Vyukov | ad9da37 | 2012-12-06 12:16:15 +0000 | [diff] [blame] | 520 | const MutexSet *mset); |
Stephen Hines | 6d18623 | 2014-11-26 17:56:19 -0800 | [diff] [blame] | 521 | void AddStack(StackTrace stack, bool suppressable = false); |
Stephen Hines | 6a211c5 | 2014-07-21 00:49:56 -0700 | [diff] [blame] | 522 | void AddThread(const ThreadContext *tctx, bool suppressable = false); |
| 523 | void AddThread(int unique_tid, bool suppressable = false); |
Stephen Hines | 2d1fdb2 | 2014-05-28 23:58:16 -0700 | [diff] [blame] | 524 | void AddUniqueTid(int unique_tid); |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 525 | void AddMutex(const SyncVar *s); |
Stephen Hines | 2d1fdb2 | 2014-05-28 23:58:16 -0700 | [diff] [blame] | 526 | u64 AddMutex(u64 id); |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 527 | void AddLocation(uptr addr, uptr size); |
Dmitry Vyukov | 8485311 | 2012-08-31 17:27:49 +0000 | [diff] [blame] | 528 | void AddSleep(u32 stack_id); |
Dmitry Vyukov | 4536cb1 | 2013-03-21 16:55:17 +0000 | [diff] [blame] | 529 | void SetCount(int count); |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 530 | |
| 531 | const ReportDesc *GetReport() const; |
| 532 | |
| 533 | private: |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 534 | ReportDesc *rep_; |
Stephen Hines | 2d1fdb2 | 2014-05-28 23:58:16 -0700 | [diff] [blame] | 535 | // Symbolizer makes lots of intercepted calls. If we try to process them, |
| 536 | // at best it will cause deadlocks on internal mutexes. |
| 537 | ScopedIgnoreInterceptors ignore_interceptors_; |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 538 | |
Stephen Hines | 2d1fdb2 | 2014-05-28 23:58:16 -0700 | [diff] [blame] | 539 | void AddDeadMutex(u64 id); |
Dmitry Vyukov | ad9da37 | 2012-12-06 12:16:15 +0000 | [diff] [blame] | 540 | |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 541 | ScopedReport(const ScopedReport&); |
| 542 | void operator = (const ScopedReport&); |
| 543 | }; |
| 544 | |
Stephen Hines | 6d18623 | 2014-11-26 17:56:19 -0800 | [diff] [blame] | 545 | void RestoreStack(int tid, const u64 epoch, VarSizeStackTrace *stk, |
| 546 | MutexSet *mset); |
| 547 | |
| 548 | template<typename StackTraceTy> |
| 549 | void ObtainCurrentStack(ThreadState *thr, uptr toppc, StackTraceTy *stack) { |
| 550 | uptr size = thr->shadow_stack_pos - thr->shadow_stack; |
| 551 | uptr start = 0; |
| 552 | if (size + !!toppc > kStackTraceMax) { |
| 553 | start = size + !!toppc - kStackTraceMax; |
| 554 | size = kStackTraceMax - !!toppc; |
| 555 | } |
| 556 | stack->Init(&thr->shadow_stack[start], size, toppc); |
| 557 | } |
| 558 | |
Dmitry Vyukov | 332c62b | 2012-08-16 15:08:49 +0000 | [diff] [blame] | 559 | |
Stephen Hines | 86277eb | 2015-03-23 12:06:32 -0700 | [diff] [blame] | 560 | #if TSAN_COLLECT_STATS |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 561 | void StatAggregate(u64 *dst, u64 *src); |
| 562 | void StatOutput(u64 *stat); |
Stephen Hines | 86277eb | 2015-03-23 12:06:32 -0700 | [diff] [blame] | 563 | #endif |
| 564 | |
Timur Iskhodzhanov | abfdbdf | 2013-03-28 18:52:40 +0000 | [diff] [blame] | 565 | void ALWAYS_INLINE StatInc(ThreadState *thr, StatType typ, u64 n = 1) { |
Stephen Hines | 86277eb | 2015-03-23 12:06:32 -0700 | [diff] [blame] | 566 | #if TSAN_COLLECT_STATS |
| 567 | thr->stat[typ] += n; |
| 568 | #endif |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 569 | } |
Timur Iskhodzhanov | abfdbdf | 2013-03-28 18:52:40 +0000 | [diff] [blame] | 570 | void ALWAYS_INLINE StatSet(ThreadState *thr, StatType typ, u64 n) { |
Stephen Hines | 86277eb | 2015-03-23 12:06:32 -0700 | [diff] [blame] | 571 | #if TSAN_COLLECT_STATS |
| 572 | thr->stat[typ] = n; |
| 573 | #endif |
Alexey Samsonov | 2bbd8be | 2013-03-15 13:48:44 +0000 | [diff] [blame] | 574 | } |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 575 | |
Dmitry Vyukov | a05fcc1 | 2012-11-06 16:00:16 +0000 | [diff] [blame] | 576 | void MapShadow(uptr addr, uptr size); |
Pirama Arumuga Nainar | cdce50b | 2015-07-01 12:26:56 -0700 | [diff] [blame] | 577 | void MapThreadTrace(uptr addr, uptr size, const char *name); |
Dmitry Vyukov | 7ac33ac | 2013-03-18 15:49:07 +0000 | [diff] [blame] | 578 | void DontNeedShadowFor(uptr addr, uptr size); |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 579 | void InitializeShadowMemory(); |
| 580 | void InitializeInterceptors(); |
Dmitry Vyukov | 4af0f21 | 2013-10-03 13:37:17 +0000 | [diff] [blame] | 581 | void InitializeLibIgnore(); |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 582 | void InitializeDynamicAnnotations(); |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 583 | |
Stephen Hines | 2d1fdb2 | 2014-05-28 23:58:16 -0700 | [diff] [blame] | 584 | void ForkBefore(ThreadState *thr, uptr pc); |
| 585 | void ForkParentAfter(ThreadState *thr, uptr pc); |
| 586 | void ForkChildAfter(ThreadState *thr, uptr pc); |
| 587 | |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 588 | void ReportRace(ThreadState *thr); |
Stephen Hines | 6a211c5 | 2014-07-21 00:49:56 -0700 | [diff] [blame] | 589 | bool OutputReport(ThreadState *thr, const ScopedReport &srep); |
Stephen Hines | 6d18623 | 2014-11-26 17:56:19 -0800 | [diff] [blame] | 590 | bool IsFiredSuppression(Context *ctx, const ScopedReport &srep, |
| 591 | StackTrace trace); |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 592 | bool IsExpectedReport(uptr addr, uptr size); |
Dmitry Vyukov | 0fd908c | 2013-03-28 16:21:19 +0000 | [diff] [blame] | 593 | void PrintMatchedBenignRaces(); |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 594 | |
| 595 | #if defined(TSAN_DEBUG_OUTPUT) && TSAN_DEBUG_OUTPUT >= 1 |
Alexey Samsonov | b1fe302 | 2012-11-02 12:17:51 +0000 | [diff] [blame] | 596 | # define DPrintf Printf |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 597 | #else |
| 598 | # define DPrintf(...) |
| 599 | #endif |
| 600 | |
| 601 | #if defined(TSAN_DEBUG_OUTPUT) && TSAN_DEBUG_OUTPUT >= 2 |
Alexey Samsonov | b1fe302 | 2012-11-02 12:17:51 +0000 | [diff] [blame] | 602 | # define DPrintf2 Printf |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 603 | #else |
| 604 | # define DPrintf2(...) |
| 605 | #endif |
| 606 | |
Dmitry Vyukov | 8485311 | 2012-08-31 17:27:49 +0000 | [diff] [blame] | 607 | u32 CurrentStackId(ThreadState *thr, uptr pc); |
Stephen Hines | 2d1fdb2 | 2014-05-28 23:58:16 -0700 | [diff] [blame] | 608 | ReportStack *SymbolizeStackId(u32 stack_id); |
Dmitry Vyukov | 1da1056 | 2012-09-01 12:13:18 +0000 | [diff] [blame] | 609 | void PrintCurrentStack(ThreadState *thr, uptr pc); |
Stephen Hines | 6d18623 | 2014-11-26 17:56:19 -0800 | [diff] [blame] | 610 | void PrintCurrentStackSlow(uptr pc); // uses libunwind |
Dmitry Vyukov | 8485311 | 2012-08-31 17:27:49 +0000 | [diff] [blame] | 611 | |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 612 | void Initialize(ThreadState *thr); |
| 613 | int Finalize(ThreadState *thr); |
| 614 | |
Stephen Hines | 6a211c5 | 2014-07-21 00:49:56 -0700 | [diff] [blame] | 615 | void OnUserAlloc(ThreadState *thr, uptr pc, uptr p, uptr sz, bool write); |
| 616 | void OnUserFree(ThreadState *thr, uptr pc, uptr p, bool write); |
Dmitry Vyukov | 21cc85d | 2012-12-20 17:29:34 +0000 | [diff] [blame] | 617 | |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 618 | void MemoryAccess(ThreadState *thr, uptr pc, uptr addr, |
Dmitry Vyukov | 334553e | 2013-02-01 09:42:06 +0000 | [diff] [blame] | 619 | int kAccessSizeLog, bool kAccessIsWrite, bool kIsAtomic); |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 620 | void MemoryAccessImpl(ThreadState *thr, uptr addr, |
Dmitry Vyukov | 334553e | 2013-02-01 09:42:06 +0000 | [diff] [blame] | 621 | int kAccessSizeLog, bool kAccessIsWrite, bool kIsAtomic, |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 622 | u64 *shadow_mem, Shadow cur); |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 623 | void MemoryAccessRange(ThreadState *thr, uptr pc, uptr addr, |
Dmitry Vyukov | 334553e | 2013-02-01 09:42:06 +0000 | [diff] [blame] | 624 | uptr size, bool is_write); |
Dmitry Vyukov | eaa0190 | 2013-02-13 13:05:36 +0000 | [diff] [blame] | 625 | void MemoryAccessRangeStep(ThreadState *thr, uptr pc, uptr addr, |
| 626 | uptr size, uptr step, bool is_write); |
Dmitry Vyukov | 8ecd0e5 | 2013-04-30 11:56:56 +0000 | [diff] [blame] | 627 | void UnalignedMemoryAccess(ThreadState *thr, uptr pc, uptr addr, |
| 628 | int size, bool kAccessIsWrite, bool kIsAtomic); |
Dmitry Vyukov | 334553e | 2013-02-01 09:42:06 +0000 | [diff] [blame] | 629 | |
| 630 | const int kSizeLog1 = 0; |
| 631 | const int kSizeLog2 = 1; |
| 632 | const int kSizeLog4 = 2; |
| 633 | const int kSizeLog8 = 3; |
| 634 | |
Timur Iskhodzhanov | abfdbdf | 2013-03-28 18:52:40 +0000 | [diff] [blame] | 635 | void ALWAYS_INLINE MemoryRead(ThreadState *thr, uptr pc, |
Dmitry Vyukov | 334553e | 2013-02-01 09:42:06 +0000 | [diff] [blame] | 636 | uptr addr, int kAccessSizeLog) { |
| 637 | MemoryAccess(thr, pc, addr, kAccessSizeLog, false, false); |
| 638 | } |
| 639 | |
Timur Iskhodzhanov | abfdbdf | 2013-03-28 18:52:40 +0000 | [diff] [blame] | 640 | void ALWAYS_INLINE MemoryWrite(ThreadState *thr, uptr pc, |
Dmitry Vyukov | 334553e | 2013-02-01 09:42:06 +0000 | [diff] [blame] | 641 | uptr addr, int kAccessSizeLog) { |
| 642 | MemoryAccess(thr, pc, addr, kAccessSizeLog, true, false); |
| 643 | } |
| 644 | |
Timur Iskhodzhanov | abfdbdf | 2013-03-28 18:52:40 +0000 | [diff] [blame] | 645 | void ALWAYS_INLINE MemoryReadAtomic(ThreadState *thr, uptr pc, |
Dmitry Vyukov | 334553e | 2013-02-01 09:42:06 +0000 | [diff] [blame] | 646 | uptr addr, int kAccessSizeLog) { |
| 647 | MemoryAccess(thr, pc, addr, kAccessSizeLog, false, true); |
| 648 | } |
| 649 | |
Timur Iskhodzhanov | abfdbdf | 2013-03-28 18:52:40 +0000 | [diff] [blame] | 650 | void ALWAYS_INLINE MemoryWriteAtomic(ThreadState *thr, uptr pc, |
Dmitry Vyukov | 334553e | 2013-02-01 09:42:06 +0000 | [diff] [blame] | 651 | uptr addr, int kAccessSizeLog) { |
| 652 | MemoryAccess(thr, pc, addr, kAccessSizeLog, true, true); |
| 653 | } |
| 654 | |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 655 | void MemoryResetRange(ThreadState *thr, uptr pc, uptr addr, uptr size); |
| 656 | void MemoryRangeFreed(ThreadState *thr, uptr pc, uptr addr, uptr size); |
Dmitry Vyukov | 26af893 | 2012-08-15 16:52:19 +0000 | [diff] [blame] | 657 | void MemoryRangeImitateWrite(ThreadState *thr, uptr pc, uptr addr, uptr size); |
Dmitry Vyukov | e1ddbf9 | 2013-10-10 15:58:12 +0000 | [diff] [blame] | 658 | |
Stephen Hines | 2d1fdb2 | 2014-05-28 23:58:16 -0700 | [diff] [blame] | 659 | void ThreadIgnoreBegin(ThreadState *thr, uptr pc); |
| 660 | void ThreadIgnoreEnd(ThreadState *thr, uptr pc); |
| 661 | void ThreadIgnoreSyncBegin(ThreadState *thr, uptr pc); |
| 662 | void ThreadIgnoreSyncEnd(ThreadState *thr, uptr pc); |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 663 | |
| 664 | void FuncEntry(ThreadState *thr, uptr pc); |
| 665 | void FuncExit(ThreadState *thr); |
| 666 | |
| 667 | int ThreadCreate(ThreadState *thr, uptr pc, uptr uid, bool detached); |
Dmitry Vyukov | e0023f7 | 2012-10-02 12:58:14 +0000 | [diff] [blame] | 668 | void ThreadStart(ThreadState *thr, int tid, uptr os_id); |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 669 | void ThreadFinish(ThreadState *thr); |
| 670 | int ThreadTid(ThreadState *thr, uptr pc, uptr uid); |
| 671 | void ThreadJoin(ThreadState *thr, uptr pc, int tid); |
| 672 | void ThreadDetach(ThreadState *thr, uptr pc, int tid); |
| 673 | void ThreadFinalize(ThreadState *thr); |
Dmitry Vyukov | aecf2e5 | 2012-12-04 15:46:05 +0000 | [diff] [blame] | 674 | void ThreadSetName(ThreadState *thr, const char *name); |
Dmitry Vyukov | 54e0a9a | 2012-11-07 16:41:57 +0000 | [diff] [blame] | 675 | int ThreadCount(ThreadState *thr); |
Dmitry Vyukov | ee8ee24 | 2012-11-15 17:40:49 +0000 | [diff] [blame] | 676 | void ProcessPendingSignals(ThreadState *thr); |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 677 | |
Dmitry Vyukov | c20e9ba | 2012-08-16 13:29:41 +0000 | [diff] [blame] | 678 | void MutexCreate(ThreadState *thr, uptr pc, uptr addr, |
| 679 | bool rw, bool recursive, bool linker_init); |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 680 | void MutexDestroy(ThreadState *thr, uptr pc, uptr addr); |
Stephen Hines | 2d1fdb2 | 2014-05-28 23:58:16 -0700 | [diff] [blame] | 681 | void MutexLock(ThreadState *thr, uptr pc, uptr addr, int rec = 1, |
| 682 | bool try_lock = false); |
Dmitry Vyukov | 8354fae | 2013-05-17 12:03:46 +0000 | [diff] [blame] | 683 | int MutexUnlock(ThreadState *thr, uptr pc, uptr addr, bool all = false); |
Stephen Hines | 2d1fdb2 | 2014-05-28 23:58:16 -0700 | [diff] [blame] | 684 | void MutexReadLock(ThreadState *thr, uptr pc, uptr addr, bool try_lock = false); |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 685 | void MutexReadUnlock(ThreadState *thr, uptr pc, uptr addr); |
| 686 | void MutexReadOrWriteUnlock(ThreadState *thr, uptr pc, uptr addr); |
Dmitry Vyukov | 11f5309 | 2013-11-15 16:58:12 +0000 | [diff] [blame] | 687 | void MutexRepair(ThreadState *thr, uptr pc, uptr addr); // call on EOWNERDEAD |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 688 | |
| 689 | void Acquire(ThreadState *thr, uptr pc, uptr addr); |
Stephen Hines | 6d18623 | 2014-11-26 17:56:19 -0800 | [diff] [blame] | 690 | // AcquireGlobal synchronizes the current thread with all other threads. |
| 691 | // In terms of happens-before relation, it draws a HB edge from all threads |
| 692 | // (where they happen to execute right now) to the current thread. We use it to |
| 693 | // handle Go finalizers. Namely, finalizer goroutine executes AcquireGlobal |
| 694 | // right before executing finalizers. This provides a coarse, but simple |
| 695 | // approximation of the actual required synchronization. |
Dmitry Vyukov | 538f1ba | 2012-11-07 15:08:20 +0000 | [diff] [blame] | 696 | void AcquireGlobal(ThreadState *thr, uptr pc); |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 697 | void Release(ThreadState *thr, uptr pc, uptr addr); |
Dmitry Vyukov | 9d150bd | 2012-07-28 15:27:41 +0000 | [diff] [blame] | 698 | void ReleaseStore(ThreadState *thr, uptr pc, uptr addr); |
Dmitry Vyukov | 8485311 | 2012-08-31 17:27:49 +0000 | [diff] [blame] | 699 | void AfterSleep(ThreadState *thr, uptr pc); |
Dmitry Vyukov | e1ddbf9 | 2013-10-10 15:58:12 +0000 | [diff] [blame] | 700 | void AcquireImpl(ThreadState *thr, uptr pc, SyncClock *c); |
| 701 | void ReleaseImpl(ThreadState *thr, uptr pc, SyncClock *c); |
| 702 | void ReleaseStoreImpl(ThreadState *thr, uptr pc, SyncClock *c); |
| 703 | void AcquireReleaseImpl(ThreadState *thr, uptr pc, SyncClock *c); |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 704 | |
| 705 | // The hacky call uses custom calling convention and an assembly thunk. |
| 706 | // It is considerably faster that a normal call for the caller |
| 707 | // if it is not executed (it is intended for slow paths from hot functions). |
| 708 | // The trick is that the call preserves all registers and the compiler |
| 709 | // does not treat it as a call. |
| 710 | // If it does not work for you, use normal call. |
Stephen Hines | 86277eb | 2015-03-23 12:06:32 -0700 | [diff] [blame] | 711 | #if !SANITIZER_DEBUG && defined(__x86_64__) |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 712 | // The caller may not create the stack frame for itself at all, |
| 713 | // so we create a reserve stack frame for it (1024b must be enough). |
| 714 | #define HACKY_CALL(f) \ |
Dmitry Vyukov | 41e8153 | 2012-09-02 11:24:07 +0000 | [diff] [blame] | 715 | __asm__ __volatile__("sub $1024, %%rsp;" \ |
Stephen Hines | 2d1fdb2 | 2014-05-28 23:58:16 -0700 | [diff] [blame] | 716 | CFI_INL_ADJUST_CFA_OFFSET(1024) \ |
Dmitry Vyukov | f5d526f | 2012-11-26 14:20:26 +0000 | [diff] [blame] | 717 | ".hidden " #f "_thunk;" \ |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 718 | "call " #f "_thunk;" \ |
Dmitry Vyukov | 41e8153 | 2012-09-02 11:24:07 +0000 | [diff] [blame] | 719 | "add $1024, %%rsp;" \ |
Stephen Hines | 2d1fdb2 | 2014-05-28 23:58:16 -0700 | [diff] [blame] | 720 | CFI_INL_ADJUST_CFA_OFFSET(-1024) \ |
Dmitry Vyukov | 41e8153 | 2012-09-02 11:24:07 +0000 | [diff] [blame] | 721 | ::: "memory", "cc"); |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 722 | #else |
| 723 | #define HACKY_CALL(f) f() |
| 724 | #endif |
| 725 | |
Dmitry Vyukov | b78caa6 | 2012-07-05 16:18:28 +0000 | [diff] [blame] | 726 | void TraceSwitch(ThreadState *thr); |
Dmitry Vyukov | 385542a | 2012-11-28 10:35:31 +0000 | [diff] [blame] | 727 | uptr TraceTopPC(ThreadState *thr); |
Dmitry Vyukov | d698edc | 2012-11-28 12:19:50 +0000 | [diff] [blame] | 728 | uptr TraceSize(); |
Dmitry Vyukov | 0415ac0 | 2012-12-04 12:19:53 +0000 | [diff] [blame] | 729 | uptr TraceParts(); |
Dmitry Vyukov | 9743d74 | 2013-03-20 10:31:53 +0000 | [diff] [blame] | 730 | Trace *ThreadTrace(int tid); |
Dmitry Vyukov | b78caa6 | 2012-07-05 16:18:28 +0000 | [diff] [blame] | 731 | |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 732 | extern "C" void __tsan_trace_switch(); |
Timur Iskhodzhanov | abfdbdf | 2013-03-28 18:52:40 +0000 | [diff] [blame] | 733 | void ALWAYS_INLINE TraceAddEvent(ThreadState *thr, FastState fs, |
Dmitry Vyukov | ad9da37 | 2012-12-06 12:16:15 +0000 | [diff] [blame] | 734 | EventType typ, u64 addr) { |
Stephen Hines | 2d1fdb2 | 2014-05-28 23:58:16 -0700 | [diff] [blame] | 735 | if (!kCollectHistory) |
| 736 | return; |
Dmitry Vyukov | ad9da37 | 2012-12-06 12:16:15 +0000 | [diff] [blame] | 737 | DCHECK_GE((int)typ, 0); |
| 738 | DCHECK_LE((int)typ, 7); |
| 739 | DCHECK_EQ(GetLsb(addr, 61), addr); |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 740 | StatInc(thr, StatEvents); |
Dmitry Vyukov | 3fb70e3 | 2012-11-28 13:01:32 +0000 | [diff] [blame] | 741 | u64 pos = fs.GetTracePos(); |
| 742 | if (UNLIKELY((pos % kTracePartSize) == 0)) { |
Stephen Hines | 86277eb | 2015-03-23 12:06:32 -0700 | [diff] [blame] | 743 | #ifndef SANITIZER_GO |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 744 | HACKY_CALL(__tsan_trace_switch); |
Dmitry Vyukov | b78caa6 | 2012-07-05 16:18:28 +0000 | [diff] [blame] | 745 | #else |
| 746 | TraceSwitch(thr); |
| 747 | #endif |
| 748 | } |
Dmitry Vyukov | 385542a | 2012-11-28 10:35:31 +0000 | [diff] [blame] | 749 | Event *trace = (Event*)GetThreadTrace(fs.tid()); |
Dmitry Vyukov | 3fb70e3 | 2012-11-28 13:01:32 +0000 | [diff] [blame] | 750 | Event *evp = &trace[pos]; |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 751 | Event ev = (u64)addr | ((u64)typ << 61); |
| 752 | *evp = ev; |
| 753 | } |
| 754 | |
Stephen Hines | 86277eb | 2015-03-23 12:06:32 -0700 | [diff] [blame] | 755 | #ifndef SANITIZER_GO |
| 756 | uptr ALWAYS_INLINE HeapEnd() { |
| 757 | #if SANITIZER_CAN_USE_ALLOCATOR64 |
| 758 | return kHeapMemEnd + PrimaryAllocator::AdditionalSize(); |
| 759 | #else |
| 760 | return kHeapMemEnd; |
| 761 | #endif |
| 762 | } |
| 763 | #endif |
| 764 | |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 765 | } // namespace __tsan |
| 766 | |
| 767 | #endif // TSAN_RTL_H |