Alexey Samsonov | 603c4be | 2012-06-04 13:55:19 +0000 | [diff] [blame] | 1 | //===-- tsan_rtl.cc -------------------------------------------------------===// |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 2 | // |
| 3 | // The LLVM Compiler Infrastructure |
| 4 | // |
| 5 | // This file is distributed under the University of Illinois Open Source |
| 6 | // License. See LICENSE.TXT for details. |
| 7 | // |
| 8 | //===----------------------------------------------------------------------===// |
| 9 | // |
| 10 | // This file is a part of ThreadSanitizer (TSan), a race detector. |
| 11 | // |
| 12 | // Main file (entry points) for the TSan run-time. |
| 13 | //===----------------------------------------------------------------------===// |
| 14 | |
Kostya Serebryany | 16e0075 | 2012-05-31 13:42:53 +0000 | [diff] [blame] | 15 | #include "sanitizer_common/sanitizer_libc.h" |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 16 | #include "tsan_defs.h" |
| 17 | #include "tsan_platform.h" |
| 18 | #include "tsan_rtl.h" |
| 19 | #include "tsan_interface.h" |
| 20 | #include "tsan_atomic.h" |
| 21 | #include "tsan_mman.h" |
| 22 | #include "tsan_placement_new.h" |
| 23 | #include "tsan_suppressions.h" |
| 24 | |
Dmitry Vyukov | adfb650 | 2012-05-22 18:07:45 +0000 | [diff] [blame] | 25 | volatile int __tsan_resumed = 0; |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 26 | |
| 27 | extern "C" void __tsan_resume() { |
Dmitry Vyukov | adfb650 | 2012-05-22 18:07:45 +0000 | [diff] [blame] | 28 | __tsan_resumed = 1; |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 29 | } |
| 30 | |
| 31 | namespace __tsan { |
| 32 | |
Alexey Samsonov | 0a4c906 | 2012-06-05 13:50:57 +0000 | [diff] [blame] | 33 | THREADLOCAL char cur_thread_placeholder[sizeof(ThreadState)] ALIGNED(64); |
| 34 | static char ctx_placeholder[sizeof(Context)] ALIGNED(64); |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 35 | |
| 36 | static Context *ctx; |
| 37 | Context *CTX() { |
| 38 | return ctx; |
| 39 | } |
| 40 | |
| 41 | Context::Context() |
| 42 | : initialized() |
| 43 | , report_mtx(MutexTypeReport, StatMtxReport) |
| 44 | , nreported() |
| 45 | , nmissed_expected() |
| 46 | , thread_mtx(MutexTypeThreads, StatMtxThreads) |
| 47 | , racy_stacks(MBlockRacyStacks) |
| 48 | , racy_addresses(MBlockRacyAddresses) { |
| 49 | } |
| 50 | |
| 51 | // The objects are allocated in TLS, so one may rely on zero-initialization. |
| 52 | ThreadState::ThreadState(Context *ctx, int tid, u64 epoch, |
| 53 | uptr stk_addr, uptr stk_size, |
| 54 | uptr tls_addr, uptr tls_size) |
| 55 | : fast_state(tid, epoch) |
| 56 | // Do not touch these, rely on zero initialization, |
| 57 | // they may be accessed before the ctor. |
| 58 | // , fast_ignore_reads() |
| 59 | // , fast_ignore_writes() |
| 60 | // , in_rtl() |
| 61 | , shadow_stack_pos(&shadow_stack[0]) |
| 62 | , tid(tid) |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 63 | , stk_addr(stk_addr) |
| 64 | , stk_size(stk_size) |
| 65 | , tls_addr(tls_addr) |
| 66 | , tls_size(tls_size) { |
| 67 | } |
| 68 | |
| 69 | ThreadContext::ThreadContext(int tid) |
| 70 | : tid(tid) |
| 71 | , unique_id() |
| 72 | , user_id() |
| 73 | , thr() |
| 74 | , status(ThreadStatusInvalid) |
| 75 | , detached() |
| 76 | , reuse_count() |
| 77 | , epoch0() |
| 78 | , epoch1() |
Dmitry Vyukov | 9d2ffc2 | 2012-05-22 14:34:43 +0000 | [diff] [blame] | 79 | , dead_info() |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 80 | , dead_next() { |
| 81 | } |
| 82 | |
Dmitry Vyukov | 2612773 | 2012-05-22 11:33:03 +0000 | [diff] [blame] | 83 | static void WriteMemoryProfile(char *buf, uptr buf_size, int num) { |
| 84 | uptr shadow = GetShadowMemoryConsumption(); |
| 85 | |
| 86 | int nthread = 0; |
| 87 | int nlivethread = 0; |
| 88 | uptr threadmem = 0; |
| 89 | { |
| 90 | Lock l(&ctx->thread_mtx); |
| 91 | for (unsigned i = 0; i < kMaxTid; i++) { |
| 92 | ThreadContext *tctx = ctx->threads[i]; |
| 93 | if (tctx == 0) |
| 94 | continue; |
| 95 | nthread += 1; |
| 96 | threadmem += sizeof(ThreadContext); |
| 97 | if (tctx->status != ThreadStatusRunning) |
| 98 | continue; |
| 99 | nlivethread += 1; |
| 100 | threadmem += sizeof(ThreadState); |
| 101 | } |
| 102 | } |
| 103 | |
| 104 | uptr nsync = 0; |
| 105 | uptr syncmem = CTX()->synctab.GetMemoryConsumption(&nsync); |
| 106 | |
Alexey Samsonov | e954101 | 2012-06-06 13:11:29 +0000 | [diff] [blame^] | 107 | SNPrintf(buf, buf_size, "%d: shadow=%zuMB" |
| 108 | " thread=%zuMB(total=%d/live=%d)" |
| 109 | " sync=%zuMB(cnt=%zu)\n", |
Dmitry Vyukov | 2612773 | 2012-05-22 11:33:03 +0000 | [diff] [blame] | 110 | num, |
| 111 | shadow >> 20, |
| 112 | threadmem >> 20, nthread, nlivethread, |
| 113 | syncmem >> 20, nsync); |
| 114 | } |
| 115 | |
| 116 | static void MemoryProfileThread(void *arg) { |
| 117 | ScopedInRtl in_rtl; |
| 118 | fd_t fd = (fd_t)(uptr)arg; |
| 119 | for (int i = 0; ; i++) { |
| 120 | InternalScopedBuf<char> buf(4096); |
| 121 | WriteMemoryProfile(buf.Ptr(), buf.Size(), i); |
| 122 | internal_write(fd, buf.Ptr(), internal_strlen(buf.Ptr())); |
| 123 | internal_sleep_ms(1000); |
| 124 | } |
| 125 | } |
| 126 | |
| 127 | static void InitializeMemoryProfile() { |
| 128 | if (flags()->profile_memory == 0 || flags()->profile_memory[0] == 0) |
| 129 | return; |
| 130 | InternalScopedBuf<char> filename(4096); |
Alexey Samsonov | 67a64dd | 2012-06-06 10:13:27 +0000 | [diff] [blame] | 131 | SNPrintf(filename.Ptr(), filename.Size(), "%s.%d", |
Dmitry Vyukov | 2612773 | 2012-05-22 11:33:03 +0000 | [diff] [blame] | 132 | flags()->profile_memory, GetPid()); |
| 133 | fd_t fd = internal_open(filename.Ptr(), true); |
| 134 | if (fd == kInvalidFd) { |
Alexey Samsonov | 67a64dd | 2012-06-06 10:13:27 +0000 | [diff] [blame] | 135 | TsanPrintf("Failed to open memory profile file '%s'\n", &filename[0]); |
Dmitry Vyukov | 2612773 | 2012-05-22 11:33:03 +0000 | [diff] [blame] | 136 | Die(); |
| 137 | } |
| 138 | internal_start_thread(&MemoryProfileThread, (void*)(uptr)fd); |
| 139 | } |
| 140 | |
Dmitry Vyukov | adfb650 | 2012-05-22 18:07:45 +0000 | [diff] [blame] | 141 | static void MemoryFlushThread(void *arg) { |
| 142 | ScopedInRtl in_rtl; |
| 143 | for (int i = 0; ; i++) { |
| 144 | internal_sleep_ms(flags()->flush_memory_ms); |
| 145 | FlushShadowMemory(); |
| 146 | } |
| 147 | } |
| 148 | |
| 149 | static void InitializeMemoryFlush() { |
| 150 | if (flags()->flush_memory_ms == 0) |
| 151 | return; |
| 152 | if (flags()->flush_memory_ms < 100) |
| 153 | flags()->flush_memory_ms = 100; |
| 154 | internal_start_thread(&MemoryFlushThread, 0); |
| 155 | } |
| 156 | |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 157 | void Initialize(ThreadState *thr) { |
Kostya Serebryany | b3cedf9 | 2012-05-29 12:18:18 +0000 | [diff] [blame] | 158 | MiniLibcStub(); |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 159 | // Thread safe because done before all threads exist. |
| 160 | static bool is_initialized = false; |
| 161 | if (is_initialized) |
| 162 | return; |
| 163 | is_initialized = true; |
| 164 | ScopedInRtl in_rtl; |
| 165 | InitializeInterceptors(); |
| 166 | const char *env = InitializePlatform(); |
| 167 | InitializeMutex(); |
| 168 | InitializeDynamicAnnotations(); |
| 169 | ctx = new(ctx_placeholder) Context; |
| 170 | InitializeShadowMemory(); |
| 171 | ctx->dead_list_size = 0; |
| 172 | ctx->dead_list_head = 0; |
| 173 | ctx->dead_list_tail = 0; |
| 174 | InitializeFlags(&ctx->flags, env); |
| 175 | InitializeSuppressions(); |
Dmitry Vyukov | 2612773 | 2012-05-22 11:33:03 +0000 | [diff] [blame] | 176 | InitializeMemoryProfile(); |
Dmitry Vyukov | adfb650 | 2012-05-22 18:07:45 +0000 | [diff] [blame] | 177 | InitializeMemoryFlush(); |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 178 | |
| 179 | if (ctx->flags.verbosity) |
Alexey Samsonov | 67a64dd | 2012-06-06 10:13:27 +0000 | [diff] [blame] | 180 | TsanPrintf("***** Running under ThreadSanitizer v2 (pid %d) *****\n", |
| 181 | GetPid()); |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 182 | |
| 183 | // Initialize thread 0. |
| 184 | ctx->thread_seq = 0; |
| 185 | int tid = ThreadCreate(thr, 0, 0, true); |
| 186 | CHECK_EQ(tid, 0); |
| 187 | ThreadStart(thr, tid); |
| 188 | CHECK_EQ(thr->in_rtl, 1); |
| 189 | ctx->initialized = true; |
| 190 | |
Dmitry Vyukov | adfb650 | 2012-05-22 18:07:45 +0000 | [diff] [blame] | 191 | if (flags()->stop_on_start) { |
Alexey Samsonov | 67a64dd | 2012-06-06 10:13:27 +0000 | [diff] [blame] | 192 | TsanPrintf("ThreadSanitizer is suspended at startup (pid %d)." |
Dmitry Vyukov | adfb650 | 2012-05-22 18:07:45 +0000 | [diff] [blame] | 193 | " Call __tsan_resume().\n", |
| 194 | GetPid()); |
| 195 | while (__tsan_resumed == 0); |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 196 | } |
| 197 | } |
| 198 | |
| 199 | int Finalize(ThreadState *thr) { |
| 200 | ScopedInRtl in_rtl; |
| 201 | Context *ctx = __tsan::ctx; |
| 202 | bool failed = false; |
| 203 | |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 204 | ThreadFinalize(thr); |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 205 | |
| 206 | if (ctx->nreported) { |
| 207 | failed = true; |
Alexey Samsonov | 67a64dd | 2012-06-06 10:13:27 +0000 | [diff] [blame] | 208 | TsanPrintf("ThreadSanitizer: reported %d warnings\n", ctx->nreported); |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 209 | } |
| 210 | |
| 211 | if (ctx->nmissed_expected) { |
| 212 | failed = true; |
Alexey Samsonov | 67a64dd | 2012-06-06 10:13:27 +0000 | [diff] [blame] | 213 | TsanPrintf("ThreadSanitizer: missed %d expected races\n", |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 214 | ctx->nmissed_expected); |
| 215 | } |
| 216 | |
| 217 | StatOutput(ctx->stat); |
Dmitry Vyukov | b7b6b1c | 2012-05-17 15:00:27 +0000 | [diff] [blame] | 218 | return failed ? flags()->exitcode : 0; |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 219 | } |
| 220 | |
| 221 | static void TraceSwitch(ThreadState *thr) { |
| 222 | ScopedInRtl in_rtl; |
| 223 | Lock l(&thr->trace.mtx); |
| 224 | unsigned trace = (thr->fast_state.epoch() / kTracePartSize) % kTraceParts; |
| 225 | TraceHeader *hdr = &thr->trace.headers[trace]; |
| 226 | hdr->epoch0 = thr->fast_state.epoch(); |
| 227 | hdr->stack0.ObtainCurrent(thr, 0); |
| 228 | } |
| 229 | |
| 230 | extern "C" void __tsan_trace_switch() { |
| 231 | TraceSwitch(cur_thread()); |
| 232 | } |
| 233 | |
| 234 | extern "C" void __tsan_report_race() { |
| 235 | ReportRace(cur_thread()); |
| 236 | } |
| 237 | |
| 238 | ALWAYS_INLINE |
| 239 | static Shadow LoadShadow(u64 *p) { |
| 240 | u64 raw = atomic_load((atomic_uint64_t*)p, memory_order_relaxed); |
| 241 | return Shadow(raw); |
| 242 | } |
| 243 | |
| 244 | ALWAYS_INLINE |
| 245 | static void StoreShadow(u64 *sp, u64 s) { |
| 246 | atomic_store((atomic_uint64_t*)sp, s, memory_order_relaxed); |
| 247 | } |
| 248 | |
| 249 | ALWAYS_INLINE |
| 250 | static void StoreIfNotYetStored(u64 *sp, u64 *s) { |
| 251 | StoreShadow(sp, *s); |
| 252 | *s = 0; |
| 253 | } |
| 254 | |
| 255 | static inline void HandleRace(ThreadState *thr, u64 *shadow_mem, |
| 256 | Shadow cur, Shadow old) { |
| 257 | thr->racy_state[0] = cur.raw(); |
| 258 | thr->racy_state[1] = old.raw(); |
| 259 | thr->racy_shadow_addr = shadow_mem; |
| 260 | HACKY_CALL(__tsan_report_race); |
| 261 | } |
| 262 | |
| 263 | static inline bool BothReads(Shadow s, int kAccessIsWrite) { |
| 264 | return !kAccessIsWrite && !s.is_write(); |
| 265 | } |
| 266 | |
| 267 | static inline bool OldIsRWStronger(Shadow old, int kAccessIsWrite) { |
| 268 | return old.is_write() || !kAccessIsWrite; |
| 269 | } |
| 270 | |
| 271 | static inline bool OldIsRWWeaker(Shadow old, int kAccessIsWrite) { |
| 272 | return !old.is_write() || kAccessIsWrite; |
| 273 | } |
| 274 | |
| 275 | static inline bool OldIsInSameSynchEpoch(Shadow old, ThreadState *thr) { |
| 276 | return old.epoch() >= thr->fast_synch_epoch; |
| 277 | } |
| 278 | |
| 279 | static inline bool HappensBefore(Shadow old, ThreadState *thr) { |
| 280 | return thr->clock.get(old.tid()) >= old.epoch(); |
| 281 | } |
| 282 | |
| 283 | ALWAYS_INLINE |
| 284 | void MemoryAccessImpl(ThreadState *thr, uptr addr, |
| 285 | int kAccessSizeLog, bool kAccessIsWrite, FastState fast_state, |
| 286 | u64 *shadow_mem, Shadow cur) { |
| 287 | StatInc(thr, StatMop); |
| 288 | StatInc(thr, kAccessIsWrite ? StatMopWrite : StatMopRead); |
| 289 | StatInc(thr, (StatType)(StatMop1 + kAccessSizeLog)); |
| 290 | |
| 291 | // This potentially can live in an MMX/SSE scratch register. |
| 292 | // The required intrinsics are: |
| 293 | // __m128i _mm_move_epi64(__m128i*); |
| 294 | // _mm_storel_epi64(u64*, __m128i); |
| 295 | u64 store_word = cur.raw(); |
| 296 | |
| 297 | // scan all the shadow values and dispatch to 4 categories: |
| 298 | // same, replace, candidate and race (see comments below). |
| 299 | // we consider only 3 cases regarding access sizes: |
| 300 | // equal, intersect and not intersect. initially I considered |
| 301 | // larger and smaller as well, it allowed to replace some |
| 302 | // 'candidates' with 'same' or 'replace', but I think |
| 303 | // it's just not worth it (performance- and complexity-wise). |
| 304 | |
| 305 | Shadow old(0); |
| 306 | if (kShadowCnt == 1) { |
| 307 | int idx = 0; |
| 308 | #include "tsan_update_shadow_word_inl.h" |
| 309 | } else if (kShadowCnt == 2) { |
| 310 | int idx = 0; |
| 311 | #include "tsan_update_shadow_word_inl.h" |
| 312 | idx = 1; |
| 313 | #include "tsan_update_shadow_word_inl.h" |
| 314 | } else if (kShadowCnt == 4) { |
| 315 | int idx = 0; |
| 316 | #include "tsan_update_shadow_word_inl.h" |
| 317 | idx = 1; |
| 318 | #include "tsan_update_shadow_word_inl.h" |
| 319 | idx = 2; |
| 320 | #include "tsan_update_shadow_word_inl.h" |
| 321 | idx = 3; |
| 322 | #include "tsan_update_shadow_word_inl.h" |
| 323 | } else if (kShadowCnt == 8) { |
| 324 | int idx = 0; |
| 325 | #include "tsan_update_shadow_word_inl.h" |
| 326 | idx = 1; |
| 327 | #include "tsan_update_shadow_word_inl.h" |
| 328 | idx = 2; |
| 329 | #include "tsan_update_shadow_word_inl.h" |
| 330 | idx = 3; |
| 331 | #include "tsan_update_shadow_word_inl.h" |
| 332 | idx = 4; |
| 333 | #include "tsan_update_shadow_word_inl.h" |
| 334 | idx = 5; |
| 335 | #include "tsan_update_shadow_word_inl.h" |
| 336 | idx = 6; |
| 337 | #include "tsan_update_shadow_word_inl.h" |
| 338 | idx = 7; |
| 339 | #include "tsan_update_shadow_word_inl.h" |
| 340 | } else { |
| 341 | CHECK(false); |
| 342 | } |
| 343 | |
| 344 | // we did not find any races and had already stored |
| 345 | // the current access info, so we are done |
| 346 | if (LIKELY(store_word == 0)) |
| 347 | return; |
| 348 | // choose a random candidate slot and replace it |
| 349 | StoreShadow(shadow_mem + (cur.epoch() % kShadowCnt), store_word); |
| 350 | StatInc(thr, StatShadowReplace); |
| 351 | return; |
| 352 | RACE: |
| 353 | HandleRace(thr, shadow_mem, cur, old); |
| 354 | return; |
| 355 | } |
| 356 | |
| 357 | ALWAYS_INLINE |
| 358 | void MemoryAccess(ThreadState *thr, uptr pc, uptr addr, |
| 359 | int kAccessSizeLog, bool kAccessIsWrite) { |
| 360 | u64 *shadow_mem = (u64*)MemToShadow(addr); |
| 361 | DPrintf2("#%d: tsan::OnMemoryAccess: @%p %p size=%d" |
Alexey Samsonov | e954101 | 2012-06-06 13:11:29 +0000 | [diff] [blame^] | 362 | " is_write=%d shadow_mem=%p {%zx, %zx, %zx, %zx}\n", |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 363 | (int)thr->fast_state.tid(), (void*)pc, (void*)addr, |
| 364 | (int)(1 << kAccessSizeLog), kAccessIsWrite, shadow_mem, |
Alexey Samsonov | e954101 | 2012-06-06 13:11:29 +0000 | [diff] [blame^] | 365 | (uptr)shadow_mem[0], (uptr)shadow_mem[1], |
| 366 | (uptr)shadow_mem[2], (uptr)shadow_mem[3]); |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 367 | #if TSAN_DEBUG |
| 368 | if (!IsAppMem(addr)) { |
Alexey Samsonov | e954101 | 2012-06-06 13:11:29 +0000 | [diff] [blame^] | 369 | TsanPrintf("Access to non app mem %zx\n", addr); |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 370 | DCHECK(IsAppMem(addr)); |
| 371 | } |
| 372 | if (!IsShadowMem((uptr)shadow_mem)) { |
Alexey Samsonov | e954101 | 2012-06-06 13:11:29 +0000 | [diff] [blame^] | 373 | TsanPrintf("Bad shadow addr %p (%zx)\n", shadow_mem, addr); |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 374 | DCHECK(IsShadowMem((uptr)shadow_mem)); |
| 375 | } |
| 376 | #endif |
| 377 | |
| 378 | FastState fast_state = thr->fast_state; |
| 379 | if (fast_state.GetIgnoreBit()) |
| 380 | return; |
| 381 | fast_state.IncrementEpoch(); |
| 382 | thr->fast_state = fast_state; |
| 383 | Shadow cur(fast_state); |
| 384 | cur.SetAddr0AndSizeLog(addr & 7, kAccessSizeLog); |
| 385 | cur.SetWrite(kAccessIsWrite); |
| 386 | |
| 387 | // We must not store to the trace if we do not store to the shadow. |
| 388 | // That is, this call must be moved somewhere below. |
| 389 | TraceAddEvent(thr, fast_state.epoch(), EventTypeMop, pc); |
| 390 | |
| 391 | MemoryAccessImpl(thr, addr, kAccessSizeLog, kAccessIsWrite, fast_state, |
| 392 | shadow_mem, cur); |
| 393 | } |
| 394 | |
| 395 | static void MemoryRangeSet(ThreadState *thr, uptr pc, uptr addr, uptr size, |
| 396 | u64 val) { |
| 397 | if (size == 0) |
| 398 | return; |
| 399 | // FIXME: fix me. |
| 400 | uptr offset = addr % kShadowCell; |
| 401 | if (offset) { |
| 402 | offset = kShadowCell - offset; |
| 403 | if (size <= offset) |
| 404 | return; |
| 405 | addr += offset; |
| 406 | size -= offset; |
| 407 | } |
| 408 | CHECK_EQ(addr % 8, 0); |
| 409 | CHECK(IsAppMem(addr)); |
| 410 | CHECK(IsAppMem(addr + size - 1)); |
| 411 | (void)thr; |
| 412 | (void)pc; |
| 413 | // Some programs mmap like hundreds of GBs but actually used a small part. |
| 414 | // So, it's better to report a false positive on the memory |
| 415 | // then to hang here senselessly. |
| 416 | const uptr kMaxResetSize = 1024*1024*1024; |
| 417 | if (size > kMaxResetSize) |
| 418 | size = kMaxResetSize; |
| 419 | size = (size + 7) & ~7; |
| 420 | u64 *p = (u64*)MemToShadow(addr); |
| 421 | CHECK(IsShadowMem((uptr)p)); |
| 422 | CHECK(IsShadowMem((uptr)(p + size * kShadowCnt / kShadowCell - 1))); |
| 423 | // FIXME: may overwrite a part outside the region |
| 424 | for (uptr i = 0; i < size * kShadowCnt / kShadowCell; i++) |
| 425 | p[i] = val; |
| 426 | } |
| 427 | |
| 428 | void MemoryResetRange(ThreadState *thr, uptr pc, uptr addr, uptr size) { |
| 429 | MemoryRangeSet(thr, pc, addr, size, 0); |
| 430 | } |
| 431 | |
| 432 | void MemoryRangeFreed(ThreadState *thr, uptr pc, uptr addr, uptr size) { |
| 433 | MemoryAccessRange(thr, pc, addr, size, true); |
Dmitry Vyukov | 069ce82 | 2012-05-17 14:17:51 +0000 | [diff] [blame] | 434 | Shadow s(thr->fast_state); |
| 435 | s.MarkAsFreed(); |
| 436 | s.SetWrite(true); |
| 437 | s.SetAddr0AndSizeLog(0, 3); |
| 438 | MemoryRangeSet(thr, pc, addr, size, s.raw()); |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 439 | } |
| 440 | |
| 441 | void FuncEntry(ThreadState *thr, uptr pc) { |
| 442 | DCHECK_EQ(thr->in_rtl, 0); |
| 443 | StatInc(thr, StatFuncEnter); |
| 444 | DPrintf2("#%d: tsan::FuncEntry %p\n", (int)thr->fast_state.tid(), (void*)pc); |
| 445 | thr->fast_state.IncrementEpoch(); |
| 446 | TraceAddEvent(thr, thr->fast_state.epoch(), EventTypeFuncEnter, pc); |
| 447 | |
| 448 | // Shadow stack maintenance can be replaced with |
| 449 | // stack unwinding during trace switch (which presumably must be faster). |
Dmitry Vyukov | 769544e | 2012-05-28 07:45:35 +0000 | [diff] [blame] | 450 | DCHECK_GE(thr->shadow_stack_pos, &thr->shadow_stack[0]); |
| 451 | DCHECK_LT(thr->shadow_stack_pos, &thr->shadow_stack[kShadowStackSize]); |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 452 | thr->shadow_stack_pos[0] = pc; |
| 453 | thr->shadow_stack_pos++; |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 454 | } |
| 455 | |
| 456 | void FuncExit(ThreadState *thr) { |
| 457 | DCHECK_EQ(thr->in_rtl, 0); |
| 458 | StatInc(thr, StatFuncExit); |
| 459 | DPrintf2("#%d: tsan::FuncExit\n", (int)thr->fast_state.tid()); |
| 460 | thr->fast_state.IncrementEpoch(); |
| 461 | TraceAddEvent(thr, thr->fast_state.epoch(), EventTypeFuncExit, 0); |
| 462 | |
Dmitry Vyukov | 769544e | 2012-05-28 07:45:35 +0000 | [diff] [blame] | 463 | DCHECK_GT(thr->shadow_stack_pos, &thr->shadow_stack[0]); |
| 464 | DCHECK_LT(thr->shadow_stack_pos, &thr->shadow_stack[kShadowStackSize]); |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 465 | thr->shadow_stack_pos--; |
| 466 | } |
| 467 | |
| 468 | void IgnoreCtl(ThreadState *thr, bool write, bool begin) { |
| 469 | DPrintf("#%d: IgnoreCtl(%d, %d)\n", thr->tid, write, begin); |
| 470 | thr->ignore_reads_and_writes += begin ? 1 : -1; |
| 471 | CHECK_GE(thr->ignore_reads_and_writes, 0); |
| 472 | if (thr->ignore_reads_and_writes) |
| 473 | thr->fast_state.SetIgnoreBit(); |
| 474 | else |
| 475 | thr->fast_state.ClearIgnoreBit(); |
| 476 | } |
| 477 | |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 478 | #if TSAN_DEBUG |
| 479 | void build_consistency_debug() {} |
| 480 | #else |
| 481 | void build_consistency_release() {} |
| 482 | #endif |
| 483 | |
| 484 | #if TSAN_COLLECT_STATS |
| 485 | void build_consistency_stats() {} |
| 486 | #else |
| 487 | void build_consistency_nostats() {} |
| 488 | #endif |
| 489 | |
| 490 | #if TSAN_SHADOW_COUNT == 1 |
| 491 | void build_consistency_shadow1() {} |
| 492 | #elif TSAN_SHADOW_COUNT == 2 |
| 493 | void build_consistency_shadow2() {} |
| 494 | #elif TSAN_SHADOW_COUNT == 4 |
| 495 | void build_consistency_shadow4() {} |
| 496 | #else |
| 497 | void build_consistency_shadow8() {} |
| 498 | #endif |
| 499 | |
| 500 | } // namespace __tsan |
| 501 | |
| 502 | // Must be included in this file to make sure everything is inlined. |
| 503 | #include "tsan_interface_inl.h" |