Alexey Samsonov | 603c4be | 2012-06-04 13:55:19 +0000 | [diff] [blame] | 1 | //===-- tsan_rtl.cc -------------------------------------------------------===// |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 2 | // |
| 3 | // The LLVM Compiler Infrastructure |
| 4 | // |
| 5 | // This file is distributed under the University of Illinois Open Source |
| 6 | // License. See LICENSE.TXT for details. |
| 7 | // |
| 8 | //===----------------------------------------------------------------------===// |
| 9 | // |
| 10 | // This file is a part of ThreadSanitizer (TSan), a race detector. |
| 11 | // |
| 12 | // Main file (entry points) for the TSan run-time. |
| 13 | //===----------------------------------------------------------------------===// |
| 14 | |
Dmitry Vyukov | fce5bd4 | 2012-06-29 16:58:33 +0000 | [diff] [blame] | 15 | #include "sanitizer_common/sanitizer_atomic.h" |
Alexey Samsonov | 0969bcf | 2012-06-18 08:44:30 +0000 | [diff] [blame] | 16 | #include "sanitizer_common/sanitizer_common.h" |
Kostya Serebryany | 16e0075 | 2012-05-31 13:42:53 +0000 | [diff] [blame] | 17 | #include "sanitizer_common/sanitizer_libc.h" |
Dmitry Vyukov | 8485311 | 2012-08-31 17:27:49 +0000 | [diff] [blame] | 18 | #include "sanitizer_common/sanitizer_stackdepot.h" |
Alexey Samsonov | 47b1634 | 2012-06-07 09:50:16 +0000 | [diff] [blame] | 19 | #include "sanitizer_common/sanitizer_placement_new.h" |
Alexey Samsonov | 8cc1f81 | 2012-09-06 08:48:43 +0000 | [diff] [blame] | 20 | #include "sanitizer_common/sanitizer_symbolizer.h" |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 21 | #include "tsan_defs.h" |
| 22 | #include "tsan_platform.h" |
| 23 | #include "tsan_rtl.h" |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 24 | #include "tsan_mman.h" |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 25 | #include "tsan_suppressions.h" |
Dmitry Vyukov | a38e40f | 2013-03-21 07:02:36 +0000 | [diff] [blame] | 26 | #include "tsan_symbolize.h" |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 27 | |
Dmitry Vyukov | adfb650 | 2012-05-22 18:07:45 +0000 | [diff] [blame] | 28 | volatile int __tsan_resumed = 0; |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 29 | |
| 30 | extern "C" void __tsan_resume() { |
Dmitry Vyukov | adfb650 | 2012-05-22 18:07:45 +0000 | [diff] [blame] | 31 | __tsan_resumed = 1; |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 32 | } |
| 33 | |
| 34 | namespace __tsan { |
| 35 | |
Dmitry Vyukov | b78caa6 | 2012-07-05 16:18:28 +0000 | [diff] [blame] | 36 | #ifndef TSAN_GO |
Alexey Samsonov | 0a4c906 | 2012-06-05 13:50:57 +0000 | [diff] [blame] | 37 | THREADLOCAL char cur_thread_placeholder[sizeof(ThreadState)] ALIGNED(64); |
Dmitry Vyukov | b78caa6 | 2012-07-05 16:18:28 +0000 | [diff] [blame] | 38 | #endif |
Alexey Samsonov | 0a4c906 | 2012-06-05 13:50:57 +0000 | [diff] [blame] | 39 | static char ctx_placeholder[sizeof(Context)] ALIGNED(64); |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 40 | |
Dmitry Vyukov | 22881ec | 2013-01-30 09:24:00 +0000 | [diff] [blame] | 41 | // Can be overriden by a front-end. |
Dmitry Vyukov | 7c984ca | 2013-01-30 09:46:53 +0000 | [diff] [blame] | 42 | bool CPP_WEAK OnFinalize(bool failed) { |
Dmitry Vyukov | 22881ec | 2013-01-30 09:24:00 +0000 | [diff] [blame] | 43 | return failed; |
| 44 | } |
Dmitry Vyukov | 22881ec | 2013-01-30 09:24:00 +0000 | [diff] [blame] | 45 | |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 46 | static Context *ctx; |
| 47 | Context *CTX() { |
| 48 | return ctx; |
| 49 | } |
| 50 | |
Alexey Samsonov | 2bbd8be | 2013-03-15 13:48:44 +0000 | [diff] [blame] | 51 | static char thread_registry_placeholder[sizeof(ThreadRegistry)]; |
| 52 | |
| 53 | static ThreadContextBase *CreateThreadContext(u32 tid) { |
Alexey Samsonov | 2bbd8be | 2013-03-15 13:48:44 +0000 | [diff] [blame] | 54 | // Map thread trace when context is created. |
| 55 | MapThreadTrace(GetThreadTrace(tid), TraceSize() * sizeof(Event)); |
Dmitry Vyukov | 9743d74 | 2013-03-20 10:31:53 +0000 | [diff] [blame] | 56 | MapThreadTrace(GetThreadTraceHeader(tid), sizeof(Trace)); |
| 57 | new(ThreadTrace(tid)) Trace(); |
| 58 | void *mem = internal_alloc(MBlockThreadContex, sizeof(ThreadContext)); |
Alexey Samsonov | 2bbd8be | 2013-03-15 13:48:44 +0000 | [diff] [blame] | 59 | return new(mem) ThreadContext(tid); |
| 60 | } |
| 61 | |
| 62 | #ifndef TSAN_GO |
| 63 | static const u32 kThreadQuarantineSize = 16; |
| 64 | #else |
| 65 | static const u32 kThreadQuarantineSize = 64; |
| 66 | #endif |
| 67 | |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 68 | Context::Context() |
| 69 | : initialized() |
| 70 | , report_mtx(MutexTypeReport, StatMtxReport) |
| 71 | , nreported() |
| 72 | , nmissed_expected() |
Alexey Samsonov | 2bbd8be | 2013-03-15 13:48:44 +0000 | [diff] [blame] | 73 | , thread_registry(new(thread_registry_placeholder) ThreadRegistry( |
| 74 | CreateThreadContext, kMaxTid, kThreadQuarantineSize)) |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 75 | , racy_stacks(MBlockRacyStacks) |
Dmitry Vyukov | 158c6ac | 2012-10-05 15:51:32 +0000 | [diff] [blame] | 76 | , racy_addresses(MBlockRacyAddresses) |
| 77 | , fired_suppressions(MBlockRacyAddresses) { |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 78 | } |
| 79 | |
| 80 | // The objects are allocated in TLS, so one may rely on zero-initialization. |
Dmitry Vyukov | ff35f1d | 2012-08-30 13:02:30 +0000 | [diff] [blame] | 81 | ThreadState::ThreadState(Context *ctx, int tid, int unique_id, u64 epoch, |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 82 | uptr stk_addr, uptr stk_size, |
| 83 | uptr tls_addr, uptr tls_size) |
| 84 | : fast_state(tid, epoch) |
| 85 | // Do not touch these, rely on zero initialization, |
| 86 | // they may be accessed before the ctor. |
| 87 | // , fast_ignore_reads() |
| 88 | // , fast_ignore_writes() |
| 89 | // , in_rtl() |
| 90 | , shadow_stack_pos(&shadow_stack[0]) |
Dmitry Vyukov | 8b30c25 | 2013-03-25 10:10:44 +0000 | [diff] [blame] | 91 | #ifndef TSAN_GO |
| 92 | , jmp_bufs(MBlockJmpBuf) |
| 93 | #endif |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 94 | , tid(tid) |
Dmitry Vyukov | ff35f1d | 2012-08-30 13:02:30 +0000 | [diff] [blame] | 95 | , unique_id(unique_id) |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 96 | , stk_addr(stk_addr) |
| 97 | , stk_size(stk_size) |
| 98 | , tls_addr(tls_addr) |
| 99 | , tls_size(tls_size) { |
| 100 | } |
| 101 | |
Dmitry Vyukov | a38e40f | 2013-03-21 07:02:36 +0000 | [diff] [blame] | 102 | static void MemoryProfiler(Context *ctx, fd_t fd, int i) { |
Dmitry Vyukov | 4bebe7b | 2013-03-21 06:24:31 +0000 | [diff] [blame] | 103 | uptr n_threads; |
| 104 | uptr n_running_threads; |
| 105 | ctx->thread_registry->GetNumberOfThreads(&n_threads, &n_running_threads); |
Dmitry Vyukov | a38e40f | 2013-03-21 07:02:36 +0000 | [diff] [blame] | 106 | InternalScopedBuffer<char> buf(4096); |
Dmitry Vyukov | 4bebe7b | 2013-03-21 06:24:31 +0000 | [diff] [blame] | 107 | internal_snprintf(buf.data(), buf.size(), "%d: nthr=%d nlive=%d\n", |
| 108 | i, n_threads, n_running_threads); |
| 109 | internal_write(fd, buf.data(), internal_strlen(buf.data())); |
| 110 | WriteMemoryProfile(buf.data(), buf.size()); |
| 111 | internal_write(fd, buf.data(), internal_strlen(buf.data())); |
Dmitry Vyukov | 2612773 | 2012-05-22 11:33:03 +0000 | [diff] [blame] | 112 | } |
| 113 | |
Dmitry Vyukov | 4bebe7b | 2013-03-21 06:24:31 +0000 | [diff] [blame] | 114 | static void BackgroundThread(void *arg) { |
| 115 | ScopedInRtl in_rtl; |
Dmitry Vyukov | a38e40f | 2013-03-21 07:02:36 +0000 | [diff] [blame] | 116 | Context *ctx = CTX(); |
Dmitry Vyukov | f63dde3 | 2013-03-21 13:01:50 +0000 | [diff] [blame] | 117 | const u64 kMs2Ns = 1000 * 1000; |
Dmitry Vyukov | 4bebe7b | 2013-03-21 06:24:31 +0000 | [diff] [blame] | 118 | |
| 119 | fd_t mprof_fd = kInvalidFd; |
| 120 | if (flags()->profile_memory && flags()->profile_memory[0]) { |
| 121 | InternalScopedBuffer<char> filename(4096); |
| 122 | internal_snprintf(filename.data(), filename.size(), "%s.%d", |
| 123 | flags()->profile_memory, GetPid()); |
| 124 | mprof_fd = OpenFile(filename.data(), true); |
| 125 | if (mprof_fd == kInvalidFd) { |
| 126 | Printf("ThreadSanitizer: failed to open memory profile file '%s'\n", |
| 127 | &filename[0]); |
| 128 | } |
Dmitry Vyukov | 2612773 | 2012-05-22 11:33:03 +0000 | [diff] [blame] | 129 | } |
Dmitry Vyukov | 4bebe7b | 2013-03-21 06:24:31 +0000 | [diff] [blame] | 130 | |
| 131 | u64 last_flush = NanoTime(); |
| 132 | for (int i = 0; ; i++) { |
| 133 | SleepForSeconds(1); |
| 134 | u64 now = NanoTime(); |
| 135 | |
Dmitry Vyukov | a38e40f | 2013-03-21 07:02:36 +0000 | [diff] [blame] | 136 | // Flush memory if requested. |
Dmitry Vyukov | 4bebe7b | 2013-03-21 06:24:31 +0000 | [diff] [blame] | 137 | if (flags()->flush_memory_ms) { |
Dmitry Vyukov | f63dde3 | 2013-03-21 13:01:50 +0000 | [diff] [blame] | 138 | if (last_flush + flags()->flush_memory_ms * kMs2Ns < now) { |
Dmitry Vyukov | 4bebe7b | 2013-03-21 06:24:31 +0000 | [diff] [blame] | 139 | FlushShadowMemory(); |
| 140 | last_flush = NanoTime(); |
| 141 | } |
| 142 | } |
| 143 | |
Dmitry Vyukov | a38e40f | 2013-03-21 07:02:36 +0000 | [diff] [blame] | 144 | // Write memory profile if requested. |
Dmitry Vyukov | 4bebe7b | 2013-03-21 06:24:31 +0000 | [diff] [blame] | 145 | if (mprof_fd != kInvalidFd) |
Dmitry Vyukov | a38e40f | 2013-03-21 07:02:36 +0000 | [diff] [blame] | 146 | MemoryProfiler(ctx, mprof_fd, i); |
| 147 | |
| 148 | #ifndef TSAN_GO |
Dmitry Vyukov | f63dde3 | 2013-03-21 13:01:50 +0000 | [diff] [blame] | 149 | // Flush symbolizer cache if requested. |
| 150 | if (flags()->flush_symbolizer_ms > 0) { |
| 151 | u64 last = atomic_load(&ctx->last_symbolize_time_ns, |
| 152 | memory_order_relaxed); |
| 153 | if (last != 0 && last + flags()->flush_symbolizer_ms * kMs2Ns < now) { |
| 154 | Lock l(&ctx->report_mtx); |
| 155 | SymbolizeFlush(); |
| 156 | atomic_store(&ctx->last_symbolize_time_ns, 0, memory_order_relaxed); |
| 157 | } |
Dmitry Vyukov | a38e40f | 2013-03-21 07:02:36 +0000 | [diff] [blame] | 158 | } |
| 159 | #endif |
Dmitry Vyukov | 4bebe7b | 2013-03-21 06:24:31 +0000 | [diff] [blame] | 160 | } |
Dmitry Vyukov | 2612773 | 2012-05-22 11:33:03 +0000 | [diff] [blame] | 161 | } |
| 162 | |
Dmitry Vyukov | 7ac33ac | 2013-03-18 15:49:07 +0000 | [diff] [blame] | 163 | void DontNeedShadowFor(uptr addr, uptr size) { |
| 164 | uptr shadow_beg = MemToShadow(addr); |
| 165 | uptr shadow_end = MemToShadow(addr + size); |
| 166 | FlushUnneededShadowMemory(shadow_beg, shadow_end - shadow_beg); |
| 167 | } |
| 168 | |
Dmitry Vyukov | a05fcc1 | 2012-11-06 16:00:16 +0000 | [diff] [blame] | 169 | void MapShadow(uptr addr, uptr size) { |
Dmitry Vyukov | 6b641c5 | 2012-11-06 16:48:46 +0000 | [diff] [blame] | 170 | MmapFixedNoReserve(MemToShadow(addr), size * kShadowMultiplier); |
Dmitry Vyukov | a05fcc1 | 2012-11-06 16:00:16 +0000 | [diff] [blame] | 171 | } |
| 172 | |
Dmitry Vyukov | 6535c31 | 2012-12-13 08:14:02 +0000 | [diff] [blame] | 173 | void MapThreadTrace(uptr addr, uptr size) { |
Dmitry Vyukov | dae1251 | 2012-12-21 12:30:52 +0000 | [diff] [blame] | 174 | DPrintf("#0: Mapping trace at %p-%p(0x%zx)\n", addr, addr + size, size); |
Dmitry Vyukov | 6535c31 | 2012-12-13 08:14:02 +0000 | [diff] [blame] | 175 | CHECK_GE(addr, kTraceMemBegin); |
| 176 | CHECK_LE(addr + size, kTraceMemBegin + kTraceMemSize); |
| 177 | if (addr != (uptr)MmapFixedNoReserve(addr, size)) { |
| 178 | Printf("FATAL: ThreadSanitizer can not mmap thread trace\n"); |
| 179 | Die(); |
| 180 | } |
| 181 | } |
| 182 | |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 183 | void Initialize(ThreadState *thr) { |
| 184 | // Thread safe because done before all threads exist. |
| 185 | static bool is_initialized = false; |
| 186 | if (is_initialized) |
| 187 | return; |
| 188 | is_initialized = true; |
Kostya Serebryany | 859778a | 2013-01-31 14:11:21 +0000 | [diff] [blame] | 189 | SanitizerToolName = "ThreadSanitizer"; |
Alexey Samsonov | 591616d | 2012-09-11 09:44:48 +0000 | [diff] [blame] | 190 | // Install tool-specific callbacks in sanitizer_common. |
| 191 | SetCheckFailedCallback(TsanCheckFailed); |
| 192 | |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 193 | ScopedInRtl in_rtl; |
Dmitry Vyukov | bbbb20b | 2012-08-16 19:36:45 +0000 | [diff] [blame] | 194 | #ifndef TSAN_GO |
Dmitry Vyukov | 2e87051 | 2012-08-15 15:35:15 +0000 | [diff] [blame] | 195 | InitializeAllocator(); |
Dmitry Vyukov | bbbb20b | 2012-08-16 19:36:45 +0000 | [diff] [blame] | 196 | #endif |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 197 | InitializeInterceptors(); |
| 198 | const char *env = InitializePlatform(); |
| 199 | InitializeMutex(); |
| 200 | InitializeDynamicAnnotations(); |
| 201 | ctx = new(ctx_placeholder) Context; |
Dmitry Vyukov | a05fcc1 | 2012-11-06 16:00:16 +0000 | [diff] [blame] | 202 | #ifndef TSAN_GO |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 203 | InitializeShadowMemory(); |
Dmitry Vyukov | a05fcc1 | 2012-11-06 16:00:16 +0000 | [diff] [blame] | 204 | #endif |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 205 | InitializeFlags(&ctx->flags, env); |
Alexey Samsonov | b1fe302 | 2012-11-02 12:17:51 +0000 | [diff] [blame] | 206 | // Setup correct file descriptor for error reports. |
Dmitry Vyukov | cec6068 | 2012-11-28 12:56:52 +0000 | [diff] [blame] | 207 | if (internal_strcmp(flags()->log_path, "stdout") == 0) |
| 208 | __sanitizer_set_report_fd(kStdoutFd); |
| 209 | else if (internal_strcmp(flags()->log_path, "stderr") == 0) |
| 210 | __sanitizer_set_report_fd(kStderrFd); |
| 211 | else |
| 212 | __sanitizer_set_report_path(flags()->log_path); |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 213 | InitializeSuppressions(); |
Dmitry Vyukov | 85a6dad | 2012-09-19 04:39:36 +0000 | [diff] [blame] | 214 | #ifndef TSAN_GO |
Alexey Samsonov | 68bdcc4 | 2012-09-25 12:35:47 +0000 | [diff] [blame] | 215 | // Initialize external symbolizer before internal threads are started. |
Alexey Samsonov | 8cc1f81 | 2012-09-06 08:48:43 +0000 | [diff] [blame] | 216 | const char *external_symbolizer = flags()->external_symbolizer_path; |
| 217 | if (external_symbolizer != 0 && external_symbolizer[0] != '\0') { |
Alexey Samsonov | 93b4caf | 2012-11-09 14:45:30 +0000 | [diff] [blame] | 218 | if (!InitializeExternalSymbolizer(external_symbolizer)) { |
| 219 | Printf("Failed to start external symbolizer: '%s'\n", |
| 220 | external_symbolizer); |
| 221 | Die(); |
| 222 | } |
Alexey Samsonov | 8cc1f81 | 2012-09-06 08:48:43 +0000 | [diff] [blame] | 223 | } |
Dmitry Vyukov | 85a6dad | 2012-09-19 04:39:36 +0000 | [diff] [blame] | 224 | #endif |
Dmitry Vyukov | 4bebe7b | 2013-03-21 06:24:31 +0000 | [diff] [blame] | 225 | internal_start_thread(&BackgroundThread, 0); |
Alexey Samsonov | 8cc1f81 | 2012-09-06 08:48:43 +0000 | [diff] [blame] | 226 | |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 227 | if (ctx->flags.verbosity) |
Alexey Samsonov | b1fe302 | 2012-11-02 12:17:51 +0000 | [diff] [blame] | 228 | Printf("***** Running under ThreadSanitizer v2 (pid %d) *****\n", |
Alexey Samsonov | 67a64dd | 2012-06-06 10:13:27 +0000 | [diff] [blame] | 229 | GetPid()); |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 230 | |
| 231 | // Initialize thread 0. |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 232 | int tid = ThreadCreate(thr, 0, 0, true); |
| 233 | CHECK_EQ(tid, 0); |
Dmitry Vyukov | 7dccf3f | 2012-10-02 11:52:05 +0000 | [diff] [blame] | 234 | ThreadStart(thr, tid, GetPid()); |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 235 | CHECK_EQ(thr->in_rtl, 1); |
| 236 | ctx->initialized = true; |
| 237 | |
Dmitry Vyukov | adfb650 | 2012-05-22 18:07:45 +0000 | [diff] [blame] | 238 | if (flags()->stop_on_start) { |
Alexey Samsonov | b1fe302 | 2012-11-02 12:17:51 +0000 | [diff] [blame] | 239 | Printf("ThreadSanitizer is suspended at startup (pid %d)." |
Dmitry Vyukov | adfb650 | 2012-05-22 18:07:45 +0000 | [diff] [blame] | 240 | " Call __tsan_resume().\n", |
| 241 | GetPid()); |
Alexey Samsonov | ba5e996 | 2013-01-30 07:45:58 +0000 | [diff] [blame] | 242 | while (__tsan_resumed == 0) {} |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 243 | } |
| 244 | } |
| 245 | |
| 246 | int Finalize(ThreadState *thr) { |
| 247 | ScopedInRtl in_rtl; |
| 248 | Context *ctx = __tsan::ctx; |
| 249 | bool failed = false; |
| 250 | |
Dmitry Vyukov | 54e0a9a | 2012-11-07 16:41:57 +0000 | [diff] [blame] | 251 | if (flags()->atexit_sleep_ms > 0 && ThreadCount(thr) > 1) |
| 252 | SleepForMillis(flags()->atexit_sleep_ms); |
| 253 | |
Dmitry Vyukov | d0a51c0 | 2012-10-02 12:07:16 +0000 | [diff] [blame] | 254 | // Wait for pending reports. |
| 255 | ctx->report_mtx.Lock(); |
| 256 | ctx->report_mtx.Unlock(); |
| 257 | |
Dmitry Vyukov | bdd844c | 2013-01-24 09:08:03 +0000 | [diff] [blame] | 258 | #ifndef TSAN_GO |
| 259 | if (ctx->flags.verbosity) |
| 260 | AllocatorPrintStats(); |
| 261 | #endif |
| 262 | |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 263 | ThreadFinalize(thr); |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 264 | |
| 265 | if (ctx->nreported) { |
| 266 | failed = true; |
Dmitry Vyukov | b3b2123 | 2012-10-07 14:21:24 +0000 | [diff] [blame] | 267 | #ifndef TSAN_GO |
Alexey Samsonov | b1fe302 | 2012-11-02 12:17:51 +0000 | [diff] [blame] | 268 | Printf("ThreadSanitizer: reported %d warnings\n", ctx->nreported); |
Dmitry Vyukov | b3b2123 | 2012-10-07 14:21:24 +0000 | [diff] [blame] | 269 | #else |
Alexey Samsonov | b1fe302 | 2012-11-02 12:17:51 +0000 | [diff] [blame] | 270 | Printf("Found %d data race(s)\n", ctx->nreported); |
Dmitry Vyukov | b3b2123 | 2012-10-07 14:21:24 +0000 | [diff] [blame] | 271 | #endif |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 272 | } |
| 273 | |
| 274 | if (ctx->nmissed_expected) { |
| 275 | failed = true; |
Alexey Samsonov | b1fe302 | 2012-11-02 12:17:51 +0000 | [diff] [blame] | 276 | Printf("ThreadSanitizer: missed %d expected races\n", |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 277 | ctx->nmissed_expected); |
| 278 | } |
| 279 | |
Dmitry Vyukov | f754eb5 | 2013-03-27 17:59:57 +0000 | [diff] [blame^] | 280 | if (flags()->print_suppressions) |
| 281 | PrintMatchedSuppressions(); |
| 282 | |
Dmitry Vyukov | 22881ec | 2013-01-30 09:24:00 +0000 | [diff] [blame] | 283 | failed = OnFinalize(failed); |
Dmitry Vyukov | 22881ec | 2013-01-30 09:24:00 +0000 | [diff] [blame] | 284 | |
Dmitry Vyukov | 08adb18 | 2012-11-13 13:53:43 +0000 | [diff] [blame] | 285 | StatAggregate(ctx->stat, thr->stat); |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 286 | StatOutput(ctx->stat); |
Dmitry Vyukov | b7b6b1c | 2012-05-17 15:00:27 +0000 | [diff] [blame] | 287 | return failed ? flags()->exitcode : 0; |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 288 | } |
| 289 | |
Dmitry Vyukov | 0ab628c | 2012-09-06 15:18:14 +0000 | [diff] [blame] | 290 | #ifndef TSAN_GO |
Dmitry Vyukov | 8485311 | 2012-08-31 17:27:49 +0000 | [diff] [blame] | 291 | u32 CurrentStackId(ThreadState *thr, uptr pc) { |
| 292 | if (thr->shadow_stack_pos == 0) // May happen during bootstrap. |
| 293 | return 0; |
| 294 | if (pc) { |
| 295 | thr->shadow_stack_pos[0] = pc; |
| 296 | thr->shadow_stack_pos++; |
| 297 | } |
| 298 | u32 id = StackDepotPut(thr->shadow_stack, |
| 299 | thr->shadow_stack_pos - thr->shadow_stack); |
| 300 | if (pc) |
| 301 | thr->shadow_stack_pos--; |
| 302 | return id; |
| 303 | } |
Dmitry Vyukov | 0ab628c | 2012-09-06 15:18:14 +0000 | [diff] [blame] | 304 | #endif |
Dmitry Vyukov | 8485311 | 2012-08-31 17:27:49 +0000 | [diff] [blame] | 305 | |
Dmitry Vyukov | b78caa6 | 2012-07-05 16:18:28 +0000 | [diff] [blame] | 306 | void TraceSwitch(ThreadState *thr) { |
Dmitry Vyukov | 9ad7c32 | 2012-06-22 11:08:55 +0000 | [diff] [blame] | 307 | thr->nomalloc++; |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 308 | ScopedInRtl in_rtl; |
Dmitry Vyukov | 9743d74 | 2013-03-20 10:31:53 +0000 | [diff] [blame] | 309 | Trace *thr_trace = ThreadTrace(thr->tid); |
| 310 | Lock l(&thr_trace->mtx); |
Dmitry Vyukov | 0415ac0 | 2012-12-04 12:19:53 +0000 | [diff] [blame] | 311 | unsigned trace = (thr->fast_state.epoch() / kTracePartSize) % TraceParts(); |
Dmitry Vyukov | 9743d74 | 2013-03-20 10:31:53 +0000 | [diff] [blame] | 312 | TraceHeader *hdr = &thr_trace->headers[trace]; |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 313 | hdr->epoch0 = thr->fast_state.epoch(); |
| 314 | hdr->stack0.ObtainCurrent(thr, 0); |
Dmitry Vyukov | ad9da37 | 2012-12-06 12:16:15 +0000 | [diff] [blame] | 315 | hdr->mset0 = thr->mset; |
Dmitry Vyukov | 9ad7c32 | 2012-06-22 11:08:55 +0000 | [diff] [blame] | 316 | thr->nomalloc--; |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 317 | } |
| 318 | |
Dmitry Vyukov | 9743d74 | 2013-03-20 10:31:53 +0000 | [diff] [blame] | 319 | Trace *ThreadTrace(int tid) { |
| 320 | return (Trace*)GetThreadTraceHeader(tid); |
| 321 | } |
| 322 | |
Dmitry Vyukov | 385542a | 2012-11-28 10:35:31 +0000 | [diff] [blame] | 323 | uptr TraceTopPC(ThreadState *thr) { |
| 324 | Event *events = (Event*)GetThreadTrace(thr->tid); |
Dmitry Vyukov | d698edc | 2012-11-28 12:19:50 +0000 | [diff] [blame] | 325 | uptr pc = events[thr->fast_state.GetTracePos()]; |
Dmitry Vyukov | 385542a | 2012-11-28 10:35:31 +0000 | [diff] [blame] | 326 | return pc; |
| 327 | } |
| 328 | |
Dmitry Vyukov | d698edc | 2012-11-28 12:19:50 +0000 | [diff] [blame] | 329 | uptr TraceSize() { |
| 330 | return (uptr)(1ull << (kTracePartSizeBits + flags()->history_size + 1)); |
| 331 | } |
| 332 | |
Dmitry Vyukov | 0415ac0 | 2012-12-04 12:19:53 +0000 | [diff] [blame] | 333 | uptr TraceParts() { |
| 334 | return TraceSize() / kTracePartSize; |
| 335 | } |
| 336 | |
Dmitry Vyukov | b78caa6 | 2012-07-05 16:18:28 +0000 | [diff] [blame] | 337 | #ifndef TSAN_GO |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 338 | extern "C" void __tsan_trace_switch() { |
| 339 | TraceSwitch(cur_thread()); |
| 340 | } |
| 341 | |
| 342 | extern "C" void __tsan_report_race() { |
| 343 | ReportRace(cur_thread()); |
| 344 | } |
Dmitry Vyukov | b78caa6 | 2012-07-05 16:18:28 +0000 | [diff] [blame] | 345 | #endif |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 346 | |
| 347 | ALWAYS_INLINE |
| 348 | static Shadow LoadShadow(u64 *p) { |
| 349 | u64 raw = atomic_load((atomic_uint64_t*)p, memory_order_relaxed); |
| 350 | return Shadow(raw); |
| 351 | } |
| 352 | |
| 353 | ALWAYS_INLINE |
| 354 | static void StoreShadow(u64 *sp, u64 s) { |
| 355 | atomic_store((atomic_uint64_t*)sp, s, memory_order_relaxed); |
| 356 | } |
| 357 | |
| 358 | ALWAYS_INLINE |
| 359 | static void StoreIfNotYetStored(u64 *sp, u64 *s) { |
| 360 | StoreShadow(sp, *s); |
| 361 | *s = 0; |
| 362 | } |
| 363 | |
| 364 | static inline void HandleRace(ThreadState *thr, u64 *shadow_mem, |
| 365 | Shadow cur, Shadow old) { |
| 366 | thr->racy_state[0] = cur.raw(); |
| 367 | thr->racy_state[1] = old.raw(); |
| 368 | thr->racy_shadow_addr = shadow_mem; |
Dmitry Vyukov | b78caa6 | 2012-07-05 16:18:28 +0000 | [diff] [blame] | 369 | #ifndef TSAN_GO |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 370 | HACKY_CALL(__tsan_report_race); |
Dmitry Vyukov | b78caa6 | 2012-07-05 16:18:28 +0000 | [diff] [blame] | 371 | #else |
| 372 | ReportRace(thr); |
| 373 | #endif |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 374 | } |
| 375 | |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 376 | static inline bool OldIsInSameSynchEpoch(Shadow old, ThreadState *thr) { |
| 377 | return old.epoch() >= thr->fast_synch_epoch; |
| 378 | } |
| 379 | |
| 380 | static inline bool HappensBefore(Shadow old, ThreadState *thr) { |
Dmitry Vyukov | c8f0a00 | 2012-11-30 20:02:11 +0000 | [diff] [blame] | 381 | return thr->clock.get(old.TidWithIgnore()) >= old.epoch(); |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 382 | } |
| 383 | |
| 384 | ALWAYS_INLINE |
| 385 | void MemoryAccessImpl(ThreadState *thr, uptr addr, |
Dmitry Vyukov | 334553e | 2013-02-01 09:42:06 +0000 | [diff] [blame] | 386 | int kAccessSizeLog, bool kAccessIsWrite, bool kIsAtomic, |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 387 | u64 *shadow_mem, Shadow cur) { |
| 388 | StatInc(thr, StatMop); |
| 389 | StatInc(thr, kAccessIsWrite ? StatMopWrite : StatMopRead); |
| 390 | StatInc(thr, (StatType)(StatMop1 + kAccessSizeLog)); |
| 391 | |
| 392 | // This potentially can live in an MMX/SSE scratch register. |
| 393 | // The required intrinsics are: |
| 394 | // __m128i _mm_move_epi64(__m128i*); |
| 395 | // _mm_storel_epi64(u64*, __m128i); |
| 396 | u64 store_word = cur.raw(); |
| 397 | |
| 398 | // scan all the shadow values and dispatch to 4 categories: |
| 399 | // same, replace, candidate and race (see comments below). |
| 400 | // we consider only 3 cases regarding access sizes: |
| 401 | // equal, intersect and not intersect. initially I considered |
| 402 | // larger and smaller as well, it allowed to replace some |
| 403 | // 'candidates' with 'same' or 'replace', but I think |
| 404 | // it's just not worth it (performance- and complexity-wise). |
| 405 | |
Dmitry Vyukov | 286c914 | 2013-03-20 11:22:03 +0000 | [diff] [blame] | 406 | Shadow old(0); |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 407 | if (kShadowCnt == 1) { |
| 408 | int idx = 0; |
| 409 | #include "tsan_update_shadow_word_inl.h" |
| 410 | } else if (kShadowCnt == 2) { |
| 411 | int idx = 0; |
| 412 | #include "tsan_update_shadow_word_inl.h" |
| 413 | idx = 1; |
| 414 | #include "tsan_update_shadow_word_inl.h" |
| 415 | } else if (kShadowCnt == 4) { |
| 416 | int idx = 0; |
| 417 | #include "tsan_update_shadow_word_inl.h" |
| 418 | idx = 1; |
| 419 | #include "tsan_update_shadow_word_inl.h" |
| 420 | idx = 2; |
| 421 | #include "tsan_update_shadow_word_inl.h" |
| 422 | idx = 3; |
| 423 | #include "tsan_update_shadow_word_inl.h" |
| 424 | } else if (kShadowCnt == 8) { |
| 425 | int idx = 0; |
| 426 | #include "tsan_update_shadow_word_inl.h" |
| 427 | idx = 1; |
| 428 | #include "tsan_update_shadow_word_inl.h" |
| 429 | idx = 2; |
| 430 | #include "tsan_update_shadow_word_inl.h" |
| 431 | idx = 3; |
| 432 | #include "tsan_update_shadow_word_inl.h" |
| 433 | idx = 4; |
| 434 | #include "tsan_update_shadow_word_inl.h" |
| 435 | idx = 5; |
| 436 | #include "tsan_update_shadow_word_inl.h" |
| 437 | idx = 6; |
| 438 | #include "tsan_update_shadow_word_inl.h" |
| 439 | idx = 7; |
| 440 | #include "tsan_update_shadow_word_inl.h" |
| 441 | } else { |
| 442 | CHECK(false); |
| 443 | } |
| 444 | |
| 445 | // we did not find any races and had already stored |
| 446 | // the current access info, so we are done |
| 447 | if (LIKELY(store_word == 0)) |
| 448 | return; |
| 449 | // choose a random candidate slot and replace it |
| 450 | StoreShadow(shadow_mem + (cur.epoch() % kShadowCnt), store_word); |
| 451 | StatInc(thr, StatShadowReplace); |
| 452 | return; |
| 453 | RACE: |
| 454 | HandleRace(thr, shadow_mem, cur, old); |
| 455 | return; |
| 456 | } |
| 457 | |
| 458 | ALWAYS_INLINE |
| 459 | void MemoryAccess(ThreadState *thr, uptr pc, uptr addr, |
Dmitry Vyukov | 334553e | 2013-02-01 09:42:06 +0000 | [diff] [blame] | 460 | int kAccessSizeLog, bool kAccessIsWrite, bool kIsAtomic) { |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 461 | u64 *shadow_mem = (u64*)MemToShadow(addr); |
Dmitry Vyukov | 68230a1 | 2012-12-07 19:23:59 +0000 | [diff] [blame] | 462 | DPrintf2("#%d: MemoryAccess: @%p %p size=%d" |
Alexey Samsonov | e954101 | 2012-06-06 13:11:29 +0000 | [diff] [blame] | 463 | " is_write=%d shadow_mem=%p {%zx, %zx, %zx, %zx}\n", |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 464 | (int)thr->fast_state.tid(), (void*)pc, (void*)addr, |
| 465 | (int)(1 << kAccessSizeLog), kAccessIsWrite, shadow_mem, |
Alexey Samsonov | e954101 | 2012-06-06 13:11:29 +0000 | [diff] [blame] | 466 | (uptr)shadow_mem[0], (uptr)shadow_mem[1], |
| 467 | (uptr)shadow_mem[2], (uptr)shadow_mem[3]); |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 468 | #if TSAN_DEBUG |
| 469 | if (!IsAppMem(addr)) { |
Alexey Samsonov | b1fe302 | 2012-11-02 12:17:51 +0000 | [diff] [blame] | 470 | Printf("Access to non app mem %zx\n", addr); |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 471 | DCHECK(IsAppMem(addr)); |
| 472 | } |
| 473 | if (!IsShadowMem((uptr)shadow_mem)) { |
Alexey Samsonov | b1fe302 | 2012-11-02 12:17:51 +0000 | [diff] [blame] | 474 | Printf("Bad shadow addr %p (%zx)\n", shadow_mem, addr); |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 475 | DCHECK(IsShadowMem((uptr)shadow_mem)); |
| 476 | } |
| 477 | #endif |
| 478 | |
Dmitry Vyukov | 82dbc51 | 2013-03-20 13:21:50 +0000 | [diff] [blame] | 479 | if (*shadow_mem == kShadowRodata) { |
| 480 | // Access to .rodata section, no races here. |
| 481 | // Measurements show that it can be 10-20% of all memory accesses. |
| 482 | StatInc(thr, StatMop); |
| 483 | StatInc(thr, kAccessIsWrite ? StatMopWrite : StatMopRead); |
| 484 | StatInc(thr, (StatType)(StatMop1 + kAccessSizeLog)); |
| 485 | StatInc(thr, StatMopRodata); |
| 486 | return; |
| 487 | } |
| 488 | |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 489 | FastState fast_state = thr->fast_state; |
| 490 | if (fast_state.GetIgnoreBit()) |
| 491 | return; |
| 492 | fast_state.IncrementEpoch(); |
| 493 | thr->fast_state = fast_state; |
| 494 | Shadow cur(fast_state); |
| 495 | cur.SetAddr0AndSizeLog(addr & 7, kAccessSizeLog); |
| 496 | cur.SetWrite(kAccessIsWrite); |
Dmitry Vyukov | 334553e | 2013-02-01 09:42:06 +0000 | [diff] [blame] | 497 | cur.SetAtomic(kIsAtomic); |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 498 | |
| 499 | // We must not store to the trace if we do not store to the shadow. |
| 500 | // That is, this call must be moved somewhere below. |
Dmitry Vyukov | 385542a | 2012-11-28 10:35:31 +0000 | [diff] [blame] | 501 | TraceAddEvent(thr, fast_state, EventTypeMop, pc); |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 502 | |
Dmitry Vyukov | 334553e | 2013-02-01 09:42:06 +0000 | [diff] [blame] | 503 | MemoryAccessImpl(thr, addr, kAccessSizeLog, kAccessIsWrite, kIsAtomic, |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 504 | shadow_mem, cur); |
| 505 | } |
| 506 | |
| 507 | static void MemoryRangeSet(ThreadState *thr, uptr pc, uptr addr, uptr size, |
| 508 | u64 val) { |
Dmitry Vyukov | 74172de | 2013-03-18 16:56:48 +0000 | [diff] [blame] | 509 | (void)thr; |
| 510 | (void)pc; |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 511 | if (size == 0) |
| 512 | return; |
| 513 | // FIXME: fix me. |
| 514 | uptr offset = addr % kShadowCell; |
| 515 | if (offset) { |
| 516 | offset = kShadowCell - offset; |
| 517 | if (size <= offset) |
| 518 | return; |
| 519 | addr += offset; |
| 520 | size -= offset; |
| 521 | } |
Dmitry Vyukov | aaac6e2 | 2012-09-02 12:04:51 +0000 | [diff] [blame] | 522 | DCHECK_EQ(addr % 8, 0); |
| 523 | // If a user passes some insane arguments (memset(0)), |
| 524 | // let it just crash as usual. |
| 525 | if (!IsAppMem(addr) || !IsAppMem(addr + size - 1)) |
| 526 | return; |
Dmitry Vyukov | 74172de | 2013-03-18 16:56:48 +0000 | [diff] [blame] | 527 | // Don't want to touch lots of shadow memory. |
| 528 | // If a program maps 10MB stack, there is no need reset the whole range. |
Dmitry Vyukov | 26af893 | 2012-08-15 16:52:19 +0000 | [diff] [blame] | 529 | size = (size + (kShadowCell - 1)) & ~(kShadowCell - 1); |
Dmitry Vyukov | 74172de | 2013-03-18 16:56:48 +0000 | [diff] [blame] | 530 | if (size < 64*1024) { |
| 531 | u64 *p = (u64*)MemToShadow(addr); |
| 532 | CHECK(IsShadowMem((uptr)p)); |
| 533 | CHECK(IsShadowMem((uptr)(p + size * kShadowCnt / kShadowCell - 1))); |
| 534 | // FIXME: may overwrite a part outside the region |
| 535 | for (uptr i = 0; i < size / kShadowCell * kShadowCnt;) { |
| 536 | p[i++] = val; |
| 537 | for (uptr j = 1; j < kShadowCnt; j++) |
| 538 | p[i++] = 0; |
| 539 | } |
| 540 | } else { |
| 541 | // The region is big, reset only beginning and end. |
| 542 | const uptr kPageSize = 4096; |
| 543 | u64 *begin = (u64*)MemToShadow(addr); |
| 544 | u64 *end = begin + size / kShadowCell * kShadowCnt; |
| 545 | u64 *p = begin; |
| 546 | // Set at least first kPageSize/2 to page boundary. |
| 547 | while ((p < begin + kPageSize / kShadowSize / 2) || ((uptr)p % kPageSize)) { |
| 548 | *p++ = val; |
| 549 | for (uptr j = 1; j < kShadowCnt; j++) |
| 550 | *p++ = 0; |
| 551 | } |
| 552 | // Reset middle part. |
| 553 | u64 *p1 = p; |
| 554 | p = RoundDown(end, kPageSize); |
| 555 | UnmapOrDie((void*)p1, (uptr)p - (uptr)p1); |
| 556 | MmapFixedNoReserve((uptr)p1, (uptr)p - (uptr)p1); |
| 557 | // Set the ending. |
| 558 | while (p < end) { |
| 559 | *p++ = val; |
| 560 | for (uptr j = 1; j < kShadowCnt; j++) |
| 561 | *p++ = 0; |
| 562 | } |
Dmitry Vyukov | 26af893 | 2012-08-15 16:52:19 +0000 | [diff] [blame] | 563 | } |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 564 | } |
| 565 | |
| 566 | void MemoryResetRange(ThreadState *thr, uptr pc, uptr addr, uptr size) { |
| 567 | MemoryRangeSet(thr, pc, addr, size, 0); |
| 568 | } |
| 569 | |
| 570 | void MemoryRangeFreed(ThreadState *thr, uptr pc, uptr addr, uptr size) { |
Dmitry Vyukov | 74172de | 2013-03-18 16:56:48 +0000 | [diff] [blame] | 571 | // Processing more than 1k (4k of shadow) is expensive, |
| 572 | // can cause excessive memory consumption (user does not necessary touch |
| 573 | // the whole range) and most likely unnecessary. |
| 574 | if (size > 1024) |
| 575 | size = 1024; |
Dmitry Vyukov | 3285866 | 2013-02-01 14:41:58 +0000 | [diff] [blame] | 576 | CHECK_EQ(thr->is_freeing, false); |
| 577 | thr->is_freeing = true; |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 578 | MemoryAccessRange(thr, pc, addr, size, true); |
Dmitry Vyukov | 3285866 | 2013-02-01 14:41:58 +0000 | [diff] [blame] | 579 | thr->is_freeing = false; |
Dmitry Vyukov | 069ce82 | 2012-05-17 14:17:51 +0000 | [diff] [blame] | 580 | Shadow s(thr->fast_state); |
Dmitry Vyukov | 064c847 | 2012-11-30 06:39:01 +0000 | [diff] [blame] | 581 | s.ClearIgnoreBit(); |
Dmitry Vyukov | 069ce82 | 2012-05-17 14:17:51 +0000 | [diff] [blame] | 582 | s.MarkAsFreed(); |
| 583 | s.SetWrite(true); |
| 584 | s.SetAddr0AndSizeLog(0, 3); |
| 585 | MemoryRangeSet(thr, pc, addr, size, s.raw()); |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 586 | } |
| 587 | |
Dmitry Vyukov | 26af893 | 2012-08-15 16:52:19 +0000 | [diff] [blame] | 588 | void MemoryRangeImitateWrite(ThreadState *thr, uptr pc, uptr addr, uptr size) { |
| 589 | Shadow s(thr->fast_state); |
Dmitry Vyukov | 064c847 | 2012-11-30 06:39:01 +0000 | [diff] [blame] | 590 | s.ClearIgnoreBit(); |
Dmitry Vyukov | 26af893 | 2012-08-15 16:52:19 +0000 | [diff] [blame] | 591 | s.SetWrite(true); |
| 592 | s.SetAddr0AndSizeLog(0, 3); |
| 593 | MemoryRangeSet(thr, pc, addr, size, s.raw()); |
| 594 | } |
| 595 | |
Dmitry Vyukov | 6fa4cc3 | 2012-11-23 07:14:11 +0000 | [diff] [blame] | 596 | ALWAYS_INLINE |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 597 | void FuncEntry(ThreadState *thr, uptr pc) { |
| 598 | DCHECK_EQ(thr->in_rtl, 0); |
| 599 | StatInc(thr, StatFuncEnter); |
Dmitry Vyukov | 25d1c79 | 2012-07-16 16:44:47 +0000 | [diff] [blame] | 600 | DPrintf2("#%d: FuncEntry %p\n", (int)thr->fast_state.tid(), (void*)pc); |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 601 | thr->fast_state.IncrementEpoch(); |
Dmitry Vyukov | 385542a | 2012-11-28 10:35:31 +0000 | [diff] [blame] | 602 | TraceAddEvent(thr, thr->fast_state, EventTypeFuncEnter, pc); |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 603 | |
| 604 | // Shadow stack maintenance can be replaced with |
| 605 | // stack unwinding during trace switch (which presumably must be faster). |
Dmitry Vyukov | 769544e | 2012-05-28 07:45:35 +0000 | [diff] [blame] | 606 | DCHECK_GE(thr->shadow_stack_pos, &thr->shadow_stack[0]); |
Dmitry Vyukov | 25d1c79 | 2012-07-16 16:44:47 +0000 | [diff] [blame] | 607 | #ifndef TSAN_GO |
Dmitry Vyukov | 769544e | 2012-05-28 07:45:35 +0000 | [diff] [blame] | 608 | DCHECK_LT(thr->shadow_stack_pos, &thr->shadow_stack[kShadowStackSize]); |
Dmitry Vyukov | 25d1c79 | 2012-07-16 16:44:47 +0000 | [diff] [blame] | 609 | #else |
| 610 | if (thr->shadow_stack_pos == thr->shadow_stack_end) { |
| 611 | const int sz = thr->shadow_stack_end - thr->shadow_stack; |
| 612 | const int newsz = 2 * sz; |
| 613 | uptr *newstack = (uptr*)internal_alloc(MBlockShadowStack, |
| 614 | newsz * sizeof(uptr)); |
| 615 | internal_memcpy(newstack, thr->shadow_stack, sz * sizeof(uptr)); |
| 616 | internal_free(thr->shadow_stack); |
| 617 | thr->shadow_stack = newstack; |
| 618 | thr->shadow_stack_pos = newstack + sz; |
| 619 | thr->shadow_stack_end = newstack + newsz; |
| 620 | } |
| 621 | #endif |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 622 | thr->shadow_stack_pos[0] = pc; |
| 623 | thr->shadow_stack_pos++; |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 624 | } |
| 625 | |
Dmitry Vyukov | 6fa4cc3 | 2012-11-23 07:14:11 +0000 | [diff] [blame] | 626 | ALWAYS_INLINE |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 627 | void FuncExit(ThreadState *thr) { |
| 628 | DCHECK_EQ(thr->in_rtl, 0); |
| 629 | StatInc(thr, StatFuncExit); |
Dmitry Vyukov | 25d1c79 | 2012-07-16 16:44:47 +0000 | [diff] [blame] | 630 | DPrintf2("#%d: FuncExit\n", (int)thr->fast_state.tid()); |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 631 | thr->fast_state.IncrementEpoch(); |
Dmitry Vyukov | 385542a | 2012-11-28 10:35:31 +0000 | [diff] [blame] | 632 | TraceAddEvent(thr, thr->fast_state, EventTypeFuncExit, 0); |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 633 | |
Dmitry Vyukov | 769544e | 2012-05-28 07:45:35 +0000 | [diff] [blame] | 634 | DCHECK_GT(thr->shadow_stack_pos, &thr->shadow_stack[0]); |
Dmitry Vyukov | 25d1c79 | 2012-07-16 16:44:47 +0000 | [diff] [blame] | 635 | #ifndef TSAN_GO |
Dmitry Vyukov | 769544e | 2012-05-28 07:45:35 +0000 | [diff] [blame] | 636 | DCHECK_LT(thr->shadow_stack_pos, &thr->shadow_stack[kShadowStackSize]); |
Dmitry Vyukov | 25d1c79 | 2012-07-16 16:44:47 +0000 | [diff] [blame] | 637 | #endif |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 638 | thr->shadow_stack_pos--; |
| 639 | } |
| 640 | |
| 641 | void IgnoreCtl(ThreadState *thr, bool write, bool begin) { |
| 642 | DPrintf("#%d: IgnoreCtl(%d, %d)\n", thr->tid, write, begin); |
| 643 | thr->ignore_reads_and_writes += begin ? 1 : -1; |
| 644 | CHECK_GE(thr->ignore_reads_and_writes, 0); |
| 645 | if (thr->ignore_reads_and_writes) |
| 646 | thr->fast_state.SetIgnoreBit(); |
| 647 | else |
| 648 | thr->fast_state.ClearIgnoreBit(); |
| 649 | } |
| 650 | |
Dmitry Vyukov | b78caa6 | 2012-07-05 16:18:28 +0000 | [diff] [blame] | 651 | bool MD5Hash::operator==(const MD5Hash &other) const { |
| 652 | return hash[0] == other.hash[0] && hash[1] == other.hash[1]; |
| 653 | } |
| 654 | |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 655 | #if TSAN_DEBUG |
| 656 | void build_consistency_debug() {} |
| 657 | #else |
| 658 | void build_consistency_release() {} |
| 659 | #endif |
| 660 | |
| 661 | #if TSAN_COLLECT_STATS |
| 662 | void build_consistency_stats() {} |
| 663 | #else |
| 664 | void build_consistency_nostats() {} |
| 665 | #endif |
| 666 | |
| 667 | #if TSAN_SHADOW_COUNT == 1 |
| 668 | void build_consistency_shadow1() {} |
| 669 | #elif TSAN_SHADOW_COUNT == 2 |
| 670 | void build_consistency_shadow2() {} |
| 671 | #elif TSAN_SHADOW_COUNT == 4 |
| 672 | void build_consistency_shadow4() {} |
| 673 | #else |
| 674 | void build_consistency_shadow8() {} |
| 675 | #endif |
| 676 | |
| 677 | } // namespace __tsan |
| 678 | |
Dmitry Vyukov | b78caa6 | 2012-07-05 16:18:28 +0000 | [diff] [blame] | 679 | #ifndef TSAN_GO |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 680 | // Must be included in this file to make sure everything is inlined. |
| 681 | #include "tsan_interface_inl.h" |
Dmitry Vyukov | b78caa6 | 2012-07-05 16:18:28 +0000 | [diff] [blame] | 682 | #endif |