Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 1 | //===-- tsan_rtl.cc ---------------------------------------------*- C++ -*-===// |
| 2 | // |
| 3 | // The LLVM Compiler Infrastructure |
| 4 | // |
| 5 | // This file is distributed under the University of Illinois Open Source |
| 6 | // License. See LICENSE.TXT for details. |
| 7 | // |
| 8 | //===----------------------------------------------------------------------===// |
| 9 | // |
| 10 | // This file is a part of ThreadSanitizer (TSan), a race detector. |
| 11 | // |
| 12 | // Main file (entry points) for the TSan run-time. |
| 13 | //===----------------------------------------------------------------------===// |
| 14 | |
| 15 | #include "tsan_defs.h" |
| 16 | #include "tsan_platform.h" |
| 17 | #include "tsan_rtl.h" |
| 18 | #include "tsan_interface.h" |
| 19 | #include "tsan_atomic.h" |
| 20 | #include "tsan_mman.h" |
| 21 | #include "tsan_placement_new.h" |
| 22 | #include "tsan_suppressions.h" |
| 23 | |
| 24 | volatile int __tsan_stop = 0; |
| 25 | |
| 26 | extern "C" void __tsan_resume() { |
| 27 | __tsan_stop = 0; |
| 28 | } |
| 29 | |
| 30 | namespace __tsan { |
| 31 | |
| 32 | THREADLOCAL char cur_thread_placeholder[sizeof(ThreadState)] ALIGN(64); |
| 33 | static char ctx_placeholder[sizeof(Context)] ALIGN(64); |
| 34 | |
| 35 | static Context *ctx; |
| 36 | Context *CTX() { |
| 37 | return ctx; |
| 38 | } |
| 39 | |
| 40 | Context::Context() |
| 41 | : initialized() |
| 42 | , report_mtx(MutexTypeReport, StatMtxReport) |
| 43 | , nreported() |
| 44 | , nmissed_expected() |
| 45 | , thread_mtx(MutexTypeThreads, StatMtxThreads) |
| 46 | , racy_stacks(MBlockRacyStacks) |
| 47 | , racy_addresses(MBlockRacyAddresses) { |
| 48 | } |
| 49 | |
| 50 | // The objects are allocated in TLS, so one may rely on zero-initialization. |
| 51 | ThreadState::ThreadState(Context *ctx, int tid, u64 epoch, |
| 52 | uptr stk_addr, uptr stk_size, |
| 53 | uptr tls_addr, uptr tls_size) |
| 54 | : fast_state(tid, epoch) |
| 55 | // Do not touch these, rely on zero initialization, |
| 56 | // they may be accessed before the ctor. |
| 57 | // , fast_ignore_reads() |
| 58 | // , fast_ignore_writes() |
| 59 | // , in_rtl() |
| 60 | , shadow_stack_pos(&shadow_stack[0]) |
| 61 | , tid(tid) |
| 62 | , func_call_count() |
| 63 | , stk_addr(stk_addr) |
| 64 | , stk_size(stk_size) |
| 65 | , tls_addr(tls_addr) |
| 66 | , tls_size(tls_size) { |
| 67 | } |
| 68 | |
| 69 | ThreadContext::ThreadContext(int tid) |
| 70 | : tid(tid) |
| 71 | , unique_id() |
| 72 | , user_id() |
| 73 | , thr() |
| 74 | , status(ThreadStatusInvalid) |
| 75 | , detached() |
| 76 | , reuse_count() |
| 77 | , epoch0() |
| 78 | , epoch1() |
| 79 | , dead_next() { |
| 80 | } |
| 81 | |
| 82 | void Initialize(ThreadState *thr) { |
| 83 | // Thread safe because done before all threads exist. |
| 84 | static bool is_initialized = false; |
| 85 | if (is_initialized) |
| 86 | return; |
| 87 | is_initialized = true; |
| 88 | ScopedInRtl in_rtl; |
| 89 | InitializeInterceptors(); |
| 90 | const char *env = InitializePlatform(); |
| 91 | InitializeMutex(); |
| 92 | InitializeDynamicAnnotations(); |
| 93 | ctx = new(ctx_placeholder) Context; |
| 94 | InitializeShadowMemory(); |
| 95 | ctx->dead_list_size = 0; |
| 96 | ctx->dead_list_head = 0; |
| 97 | ctx->dead_list_tail = 0; |
| 98 | InitializeFlags(&ctx->flags, env); |
| 99 | InitializeSuppressions(); |
| 100 | |
| 101 | if (ctx->flags.verbosity) |
| 102 | Printf("***** Running under ThreadSanitizer v2 (pid=%d) *****\n", GetPid()); |
| 103 | |
| 104 | // Initialize thread 0. |
| 105 | ctx->thread_seq = 0; |
| 106 | int tid = ThreadCreate(thr, 0, 0, true); |
| 107 | CHECK_EQ(tid, 0); |
| 108 | ThreadStart(thr, tid); |
| 109 | CHECK_EQ(thr->in_rtl, 1); |
| 110 | ctx->initialized = true; |
| 111 | |
| 112 | if (__tsan_stop) { |
| 113 | Printf("ThreadSanitizer is suspended at startup.\n"); |
| 114 | while (__tsan_stop); |
| 115 | } |
| 116 | } |
| 117 | |
| 118 | int Finalize(ThreadState *thr) { |
| 119 | ScopedInRtl in_rtl; |
| 120 | Context *ctx = __tsan::ctx; |
| 121 | bool failed = false; |
| 122 | |
| 123 | // Be very careful beyond that point. |
| 124 | // All bets are off. Everything is destroyed. |
| 125 | ThreadFinish(thr); |
| 126 | ThreadFinalize(thr); |
| 127 | FinalizeFlags(&ctx->flags); |
| 128 | |
| 129 | if (ctx->nreported) { |
| 130 | failed = true; |
| 131 | Printf("ThreadSanitizer: reported %d warnings\n", ctx->nreported); |
| 132 | } |
| 133 | |
| 134 | if (ctx->nmissed_expected) { |
| 135 | failed = true; |
| 136 | Printf("ThreadSanitizer: missed %d expected races\n", |
| 137 | ctx->nmissed_expected); |
| 138 | } |
| 139 | |
| 140 | StatOutput(ctx->stat); |
| 141 | FinalizeSuppressions(); |
| 142 | FinalizePlatform(); |
| 143 | |
| 144 | const int exitcode = failed ? flags()->exitcode : 0; |
| 145 | const int log_fileno = flags()->log_fileno; |
| 146 | __tsan::ctx->~Context(); |
| 147 | __tsan::ctx = 0; |
| 148 | |
| 149 | InternalAllocStatAggregate(ctx, thr); |
| 150 | |
| 151 | for (int i = 0; i < (int)MBlockTypeCount; i++) { |
| 152 | if (ctx->int_alloc_cnt[i] == 0 && ctx->int_alloc_siz[i] == 0) |
| 153 | continue; |
| 154 | InternalScopedBuf<char> tmp(1024); |
| 155 | Snprintf(tmp, tmp.Size(), "ThreadSanitizer: Internal memory leak: " |
| 156 | "type=%d count=%lld size=%lld\n", |
| 157 | (int)i, ctx->int_alloc_cnt[i], ctx->int_alloc_siz[i]); |
| 158 | internal_write(log_fileno, tmp, internal_strlen(tmp)); |
| 159 | } |
| 160 | |
| 161 | return exitcode; |
| 162 | } |
| 163 | |
| 164 | static void TraceSwitch(ThreadState *thr) { |
| 165 | ScopedInRtl in_rtl; |
| 166 | Lock l(&thr->trace.mtx); |
| 167 | unsigned trace = (thr->fast_state.epoch() / kTracePartSize) % kTraceParts; |
| 168 | TraceHeader *hdr = &thr->trace.headers[trace]; |
| 169 | hdr->epoch0 = thr->fast_state.epoch(); |
| 170 | hdr->stack0.ObtainCurrent(thr, 0); |
| 171 | } |
| 172 | |
| 173 | extern "C" void __tsan_trace_switch() { |
| 174 | TraceSwitch(cur_thread()); |
| 175 | } |
| 176 | |
| 177 | extern "C" void __tsan_report_race() { |
| 178 | ReportRace(cur_thread()); |
| 179 | } |
| 180 | |
| 181 | ALWAYS_INLINE |
| 182 | static Shadow LoadShadow(u64 *p) { |
| 183 | u64 raw = atomic_load((atomic_uint64_t*)p, memory_order_relaxed); |
| 184 | return Shadow(raw); |
| 185 | } |
| 186 | |
| 187 | ALWAYS_INLINE |
| 188 | static void StoreShadow(u64 *sp, u64 s) { |
| 189 | atomic_store((atomic_uint64_t*)sp, s, memory_order_relaxed); |
| 190 | } |
| 191 | |
| 192 | ALWAYS_INLINE |
| 193 | static void StoreIfNotYetStored(u64 *sp, u64 *s) { |
| 194 | StoreShadow(sp, *s); |
| 195 | *s = 0; |
| 196 | } |
| 197 | |
| 198 | static inline void HandleRace(ThreadState *thr, u64 *shadow_mem, |
| 199 | Shadow cur, Shadow old) { |
| 200 | thr->racy_state[0] = cur.raw(); |
| 201 | thr->racy_state[1] = old.raw(); |
| 202 | thr->racy_shadow_addr = shadow_mem; |
| 203 | HACKY_CALL(__tsan_report_race); |
| 204 | } |
| 205 | |
| 206 | static inline bool BothReads(Shadow s, int kAccessIsWrite) { |
| 207 | return !kAccessIsWrite && !s.is_write(); |
| 208 | } |
| 209 | |
| 210 | static inline bool OldIsRWStronger(Shadow old, int kAccessIsWrite) { |
| 211 | return old.is_write() || !kAccessIsWrite; |
| 212 | } |
| 213 | |
| 214 | static inline bool OldIsRWWeaker(Shadow old, int kAccessIsWrite) { |
| 215 | return !old.is_write() || kAccessIsWrite; |
| 216 | } |
| 217 | |
| 218 | static inline bool OldIsInSameSynchEpoch(Shadow old, ThreadState *thr) { |
| 219 | return old.epoch() >= thr->fast_synch_epoch; |
| 220 | } |
| 221 | |
| 222 | static inline bool HappensBefore(Shadow old, ThreadState *thr) { |
| 223 | return thr->clock.get(old.tid()) >= old.epoch(); |
| 224 | } |
| 225 | |
| 226 | ALWAYS_INLINE |
| 227 | void MemoryAccessImpl(ThreadState *thr, uptr addr, |
| 228 | int kAccessSizeLog, bool kAccessIsWrite, FastState fast_state, |
| 229 | u64 *shadow_mem, Shadow cur) { |
| 230 | StatInc(thr, StatMop); |
| 231 | StatInc(thr, kAccessIsWrite ? StatMopWrite : StatMopRead); |
| 232 | StatInc(thr, (StatType)(StatMop1 + kAccessSizeLog)); |
| 233 | |
| 234 | // This potentially can live in an MMX/SSE scratch register. |
| 235 | // The required intrinsics are: |
| 236 | // __m128i _mm_move_epi64(__m128i*); |
| 237 | // _mm_storel_epi64(u64*, __m128i); |
| 238 | u64 store_word = cur.raw(); |
| 239 | |
| 240 | // scan all the shadow values and dispatch to 4 categories: |
| 241 | // same, replace, candidate and race (see comments below). |
| 242 | // we consider only 3 cases regarding access sizes: |
| 243 | // equal, intersect and not intersect. initially I considered |
| 244 | // larger and smaller as well, it allowed to replace some |
| 245 | // 'candidates' with 'same' or 'replace', but I think |
| 246 | // it's just not worth it (performance- and complexity-wise). |
| 247 | |
| 248 | Shadow old(0); |
| 249 | if (kShadowCnt == 1) { |
| 250 | int idx = 0; |
| 251 | #include "tsan_update_shadow_word_inl.h" |
| 252 | } else if (kShadowCnt == 2) { |
| 253 | int idx = 0; |
| 254 | #include "tsan_update_shadow_word_inl.h" |
| 255 | idx = 1; |
| 256 | #include "tsan_update_shadow_word_inl.h" |
| 257 | } else if (kShadowCnt == 4) { |
| 258 | int idx = 0; |
| 259 | #include "tsan_update_shadow_word_inl.h" |
| 260 | idx = 1; |
| 261 | #include "tsan_update_shadow_word_inl.h" |
| 262 | idx = 2; |
| 263 | #include "tsan_update_shadow_word_inl.h" |
| 264 | idx = 3; |
| 265 | #include "tsan_update_shadow_word_inl.h" |
| 266 | } else if (kShadowCnt == 8) { |
| 267 | int idx = 0; |
| 268 | #include "tsan_update_shadow_word_inl.h" |
| 269 | idx = 1; |
| 270 | #include "tsan_update_shadow_word_inl.h" |
| 271 | idx = 2; |
| 272 | #include "tsan_update_shadow_word_inl.h" |
| 273 | idx = 3; |
| 274 | #include "tsan_update_shadow_word_inl.h" |
| 275 | idx = 4; |
| 276 | #include "tsan_update_shadow_word_inl.h" |
| 277 | idx = 5; |
| 278 | #include "tsan_update_shadow_word_inl.h" |
| 279 | idx = 6; |
| 280 | #include "tsan_update_shadow_word_inl.h" |
| 281 | idx = 7; |
| 282 | #include "tsan_update_shadow_word_inl.h" |
| 283 | } else { |
| 284 | CHECK(false); |
| 285 | } |
| 286 | |
| 287 | // we did not find any races and had already stored |
| 288 | // the current access info, so we are done |
| 289 | if (LIKELY(store_word == 0)) |
| 290 | return; |
| 291 | // choose a random candidate slot and replace it |
| 292 | StoreShadow(shadow_mem + (cur.epoch() % kShadowCnt), store_word); |
| 293 | StatInc(thr, StatShadowReplace); |
| 294 | return; |
| 295 | RACE: |
| 296 | HandleRace(thr, shadow_mem, cur, old); |
| 297 | return; |
| 298 | } |
| 299 | |
| 300 | ALWAYS_INLINE |
| 301 | void MemoryAccess(ThreadState *thr, uptr pc, uptr addr, |
| 302 | int kAccessSizeLog, bool kAccessIsWrite) { |
| 303 | u64 *shadow_mem = (u64*)MemToShadow(addr); |
| 304 | DPrintf2("#%d: tsan::OnMemoryAccess: @%p %p size=%d" |
| 305 | " is_write=%d shadow_mem=%p {%llx, %llx, %llx, %llx}\n", |
| 306 | (int)thr->fast_state.tid(), (void*)pc, (void*)addr, |
| 307 | (int)(1 << kAccessSizeLog), kAccessIsWrite, shadow_mem, |
| 308 | shadow_mem[0], shadow_mem[1], shadow_mem[2], shadow_mem[3]); |
| 309 | #if TSAN_DEBUG |
| 310 | if (!IsAppMem(addr)) { |
| 311 | Printf("Access to non app mem %lx\n", addr); |
| 312 | DCHECK(IsAppMem(addr)); |
| 313 | } |
| 314 | if (!IsShadowMem((uptr)shadow_mem)) { |
| 315 | Printf("Bad shadow addr %p (%lx)\n", shadow_mem, addr); |
| 316 | DCHECK(IsShadowMem((uptr)shadow_mem)); |
| 317 | } |
| 318 | #endif |
| 319 | |
| 320 | FastState fast_state = thr->fast_state; |
| 321 | if (fast_state.GetIgnoreBit()) |
| 322 | return; |
| 323 | fast_state.IncrementEpoch(); |
| 324 | thr->fast_state = fast_state; |
| 325 | Shadow cur(fast_state); |
| 326 | cur.SetAddr0AndSizeLog(addr & 7, kAccessSizeLog); |
| 327 | cur.SetWrite(kAccessIsWrite); |
| 328 | |
| 329 | // We must not store to the trace if we do not store to the shadow. |
| 330 | // That is, this call must be moved somewhere below. |
| 331 | TraceAddEvent(thr, fast_state.epoch(), EventTypeMop, pc); |
| 332 | |
| 333 | MemoryAccessImpl(thr, addr, kAccessSizeLog, kAccessIsWrite, fast_state, |
| 334 | shadow_mem, cur); |
| 335 | } |
| 336 | |
| 337 | static void MemoryRangeSet(ThreadState *thr, uptr pc, uptr addr, uptr size, |
| 338 | u64 val) { |
| 339 | if (size == 0) |
| 340 | return; |
| 341 | // FIXME: fix me. |
| 342 | uptr offset = addr % kShadowCell; |
| 343 | if (offset) { |
| 344 | offset = kShadowCell - offset; |
| 345 | if (size <= offset) |
| 346 | return; |
| 347 | addr += offset; |
| 348 | size -= offset; |
| 349 | } |
| 350 | CHECK_EQ(addr % 8, 0); |
| 351 | CHECK(IsAppMem(addr)); |
| 352 | CHECK(IsAppMem(addr + size - 1)); |
| 353 | (void)thr; |
| 354 | (void)pc; |
| 355 | // Some programs mmap like hundreds of GBs but actually used a small part. |
| 356 | // So, it's better to report a false positive on the memory |
| 357 | // then to hang here senselessly. |
| 358 | const uptr kMaxResetSize = 1024*1024*1024; |
| 359 | if (size > kMaxResetSize) |
| 360 | size = kMaxResetSize; |
| 361 | size = (size + 7) & ~7; |
| 362 | u64 *p = (u64*)MemToShadow(addr); |
| 363 | CHECK(IsShadowMem((uptr)p)); |
| 364 | CHECK(IsShadowMem((uptr)(p + size * kShadowCnt / kShadowCell - 1))); |
| 365 | // FIXME: may overwrite a part outside the region |
| 366 | for (uptr i = 0; i < size * kShadowCnt / kShadowCell; i++) |
| 367 | p[i] = val; |
| 368 | } |
| 369 | |
| 370 | void MemoryResetRange(ThreadState *thr, uptr pc, uptr addr, uptr size) { |
| 371 | MemoryRangeSet(thr, pc, addr, size, 0); |
| 372 | } |
| 373 | |
| 374 | void MemoryRangeFreed(ThreadState *thr, uptr pc, uptr addr, uptr size) { |
| 375 | MemoryAccessRange(thr, pc, addr, size, true); |
Dmitry Vyukov | 069ce82 | 2012-05-17 14:17:51 +0000 | [diff] [blame^] | 376 | Shadow s(thr->fast_state); |
| 377 | s.MarkAsFreed(); |
| 378 | s.SetWrite(true); |
| 379 | s.SetAddr0AndSizeLog(0, 3); |
| 380 | MemoryRangeSet(thr, pc, addr, size, s.raw()); |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 381 | } |
| 382 | |
| 383 | void FuncEntry(ThreadState *thr, uptr pc) { |
| 384 | DCHECK_EQ(thr->in_rtl, 0); |
| 385 | StatInc(thr, StatFuncEnter); |
| 386 | DPrintf2("#%d: tsan::FuncEntry %p\n", (int)thr->fast_state.tid(), (void*)pc); |
| 387 | thr->fast_state.IncrementEpoch(); |
| 388 | TraceAddEvent(thr, thr->fast_state.epoch(), EventTypeFuncEnter, pc); |
| 389 | |
| 390 | // Shadow stack maintenance can be replaced with |
| 391 | // stack unwinding during trace switch (which presumably must be faster). |
| 392 | DCHECK(thr->shadow_stack_pos >= &thr->shadow_stack[0]); |
| 393 | DCHECK(thr->shadow_stack_pos < &thr->shadow_stack[kShadowStackSize]); |
| 394 | thr->shadow_stack_pos[0] = pc; |
| 395 | thr->shadow_stack_pos++; |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 396 | } |
| 397 | |
| 398 | void FuncExit(ThreadState *thr) { |
| 399 | DCHECK_EQ(thr->in_rtl, 0); |
| 400 | StatInc(thr, StatFuncExit); |
| 401 | DPrintf2("#%d: tsan::FuncExit\n", (int)thr->fast_state.tid()); |
| 402 | thr->fast_state.IncrementEpoch(); |
| 403 | TraceAddEvent(thr, thr->fast_state.epoch(), EventTypeFuncExit, 0); |
| 404 | |
| 405 | DCHECK(thr->shadow_stack_pos > &thr->shadow_stack[0]); |
| 406 | DCHECK(thr->shadow_stack_pos < &thr->shadow_stack[kShadowStackSize]); |
| 407 | thr->shadow_stack_pos--; |
| 408 | } |
| 409 | |
| 410 | void IgnoreCtl(ThreadState *thr, bool write, bool begin) { |
| 411 | DPrintf("#%d: IgnoreCtl(%d, %d)\n", thr->tid, write, begin); |
| 412 | thr->ignore_reads_and_writes += begin ? 1 : -1; |
| 413 | CHECK_GE(thr->ignore_reads_and_writes, 0); |
| 414 | if (thr->ignore_reads_and_writes) |
| 415 | thr->fast_state.SetIgnoreBit(); |
| 416 | else |
| 417 | thr->fast_state.ClearIgnoreBit(); |
| 418 | } |
| 419 | |
| 420 | void InternalAllocStatAggregate(Context *ctx, ThreadState *thr) { |
| 421 | for (int i = 0; i < (int)MBlockTypeCount; i++) { |
| 422 | ctx->int_alloc_cnt[i] += thr->int_alloc_cnt[i]; |
| 423 | ctx->int_alloc_siz[i] += thr->int_alloc_siz[i]; |
| 424 | thr->int_alloc_cnt[i] = 0; |
| 425 | thr->int_alloc_siz[i] = 0; |
| 426 | } |
| 427 | } |
| 428 | |
| 429 | #if TSAN_DEBUG |
| 430 | void build_consistency_debug() {} |
| 431 | #else |
| 432 | void build_consistency_release() {} |
| 433 | #endif |
| 434 | |
| 435 | #if TSAN_COLLECT_STATS |
| 436 | void build_consistency_stats() {} |
| 437 | #else |
| 438 | void build_consistency_nostats() {} |
| 439 | #endif |
| 440 | |
| 441 | #if TSAN_SHADOW_COUNT == 1 |
| 442 | void build_consistency_shadow1() {} |
| 443 | #elif TSAN_SHADOW_COUNT == 2 |
| 444 | void build_consistency_shadow2() {} |
| 445 | #elif TSAN_SHADOW_COUNT == 4 |
| 446 | void build_consistency_shadow4() {} |
| 447 | #else |
| 448 | void build_consistency_shadow8() {} |
| 449 | #endif |
| 450 | |
| 451 | } // namespace __tsan |
| 452 | |
| 453 | // Must be included in this file to make sure everything is inlined. |
| 454 | #include "tsan_interface_inl.h" |