Alexey Samsonov | 603c4be | 2012-06-04 13:55:19 +0000 | [diff] [blame] | 1 | //===-- tsan_mman.cc ------------------------------------------------------===// |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 2 | // |
| 3 | // The LLVM Compiler Infrastructure |
| 4 | // |
| 5 | // This file is distributed under the University of Illinois Open Source |
| 6 | // License. See LICENSE.TXT for details. |
| 7 | // |
| 8 | //===----------------------------------------------------------------------===// |
| 9 | // |
| 10 | // This file is a part of ThreadSanitizer (TSan), a race detector. |
| 11 | // |
| 12 | //===----------------------------------------------------------------------===// |
Stephen Hines | 6a211c5 | 2014-07-21 00:49:56 -0700 | [diff] [blame] | 13 | #include "sanitizer_common/sanitizer_allocator_interface.h" |
Alexey Samsonov | f7667cc | 2012-06-07 11:54:08 +0000 | [diff] [blame] | 14 | #include "sanitizer_common/sanitizer_common.h" |
Dmitry Vyukov | 2e87051 | 2012-08-15 15:35:15 +0000 | [diff] [blame] | 15 | #include "sanitizer_common/sanitizer_placement_new.h" |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 16 | #include "tsan_mman.h" |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 17 | #include "tsan_rtl.h" |
| 18 | #include "tsan_report.h" |
| 19 | #include "tsan_flags.h" |
| 20 | |
Alexey Samsonov | 4f0ea39 | 2012-09-24 13:19:47 +0000 | [diff] [blame] | 21 | // May be overriden by front-end. |
Stephen Hines | 6a211c5 | 2014-07-21 00:49:56 -0700 | [diff] [blame] | 22 | extern "C" void WEAK __sanitizer_malloc_hook(void *ptr, uptr size) { |
| 23 | (void)ptr; |
| 24 | (void)size; |
| 25 | } |
Alexey Samsonov | 4f0ea39 | 2012-09-24 13:19:47 +0000 | [diff] [blame] | 26 | |
Stephen Hines | 6a211c5 | 2014-07-21 00:49:56 -0700 | [diff] [blame] | 27 | extern "C" void WEAK __sanitizer_free_hook(void *ptr) { |
| 28 | (void)ptr; |
| 29 | } |
Alexey Samsonov | 4f0ea39 | 2012-09-24 13:19:47 +0000 | [diff] [blame] | 30 | |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 31 | namespace __tsan { |
| 32 | |
Dmitry Vyukov | e93e505 | 2013-03-18 10:32:21 +0000 | [diff] [blame] | 33 | struct MapUnmapCallback { |
| 34 | void OnMap(uptr p, uptr size) const { } |
| 35 | void OnUnmap(uptr p, uptr size) const { |
| 36 | // We are about to unmap a chunk of user memory. |
| 37 | // Mark the corresponding shadow memory as not needed. |
Dmitry Vyukov | 7ac33ac | 2013-03-18 15:49:07 +0000 | [diff] [blame] | 38 | DontNeedShadowFor(p, size); |
Dmitry Vyukov | e93e505 | 2013-03-18 10:32:21 +0000 | [diff] [blame] | 39 | } |
| 40 | }; |
| 41 | |
Dmitry Vyukov | ff35f1d | 2012-08-30 13:02:30 +0000 | [diff] [blame] | 42 | static char allocator_placeholder[sizeof(Allocator)] ALIGNED(64); |
| 43 | Allocator *allocator() { |
Dmitry Vyukov | 2e87051 | 2012-08-15 15:35:15 +0000 | [diff] [blame] | 44 | return reinterpret_cast<Allocator*>(&allocator_placeholder); |
| 45 | } |
| 46 | |
| 47 | void InitializeAllocator() { |
Stephen Hines | 86277eb | 2015-03-23 12:06:32 -0700 | [diff] [blame] | 48 | allocator()->Init(common_flags()->allocator_may_return_null); |
Dmitry Vyukov | 2e87051 | 2012-08-15 15:35:15 +0000 | [diff] [blame] | 49 | } |
| 50 | |
Dmitry Vyukov | bdd844c | 2013-01-24 09:08:03 +0000 | [diff] [blame] | 51 | void AllocatorThreadStart(ThreadState *thr) { |
| 52 | allocator()->InitCache(&thr->alloc_cache); |
Alexey Samsonov | 1f3c2fe | 2013-05-29 09:15:39 +0000 | [diff] [blame] | 53 | internal_allocator()->InitCache(&thr->internal_alloc_cache); |
Dmitry Vyukov | bdd844c | 2013-01-24 09:08:03 +0000 | [diff] [blame] | 54 | } |
| 55 | |
| 56 | void AllocatorThreadFinish(ThreadState *thr) { |
| 57 | allocator()->DestroyCache(&thr->alloc_cache); |
Alexey Samsonov | 1f3c2fe | 2013-05-29 09:15:39 +0000 | [diff] [blame] | 58 | internal_allocator()->DestroyCache(&thr->internal_alloc_cache); |
Dmitry Vyukov | bdd844c | 2013-01-24 09:08:03 +0000 | [diff] [blame] | 59 | } |
| 60 | |
| 61 | void AllocatorPrintStats() { |
| 62 | allocator()->PrintStats(); |
Dmitry Vyukov | 2e87051 | 2012-08-15 15:35:15 +0000 | [diff] [blame] | 63 | } |
| 64 | |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 65 | static void SignalUnsafeCall(ThreadState *thr, uptr pc) { |
Stephen Hines | 6d18623 | 2014-11-26 17:56:19 -0800 | [diff] [blame] | 66 | if (atomic_load(&thr->in_signal_handler, memory_order_relaxed) == 0 || |
| 67 | !flags()->report_signal_unsafe) |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 68 | return; |
Stephen Hines | 6d18623 | 2014-11-26 17:56:19 -0800 | [diff] [blame] | 69 | VarSizeStackTrace stack; |
| 70 | ObtainCurrentStack(thr, pc, &stack); |
Alexey Samsonov | 2bbd8be | 2013-03-15 13:48:44 +0000 | [diff] [blame] | 71 | ThreadRegistryLock l(ctx->thread_registry); |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 72 | ScopedReport rep(ReportTypeSignalUnsafe); |
Dmitry Vyukov | 158c6ac | 2012-10-05 15:51:32 +0000 | [diff] [blame] | 73 | if (!IsFiredSuppression(ctx, rep, stack)) { |
Stephen Hines | 6d18623 | 2014-11-26 17:56:19 -0800 | [diff] [blame] | 74 | rep.AddStack(stack, true); |
Stephen Hines | 6a211c5 | 2014-07-21 00:49:56 -0700 | [diff] [blame] | 75 | OutputReport(thr, rep); |
Dmitry Vyukov | 158c6ac | 2012-10-05 15:51:32 +0000 | [diff] [blame] | 76 | } |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 77 | } |
| 78 | |
Stephen Hines | 6d18623 | 2014-11-26 17:56:19 -0800 | [diff] [blame] | 79 | void *user_alloc(ThreadState *thr, uptr pc, uptr sz, uptr align, bool signal) { |
Dmitry Vyukov | 7423c78 | 2013-03-22 17:06:22 +0000 | [diff] [blame] | 80 | if ((sz >= (1ull << 40)) || (align >= (1ull << 40))) |
Stephen Hines | 86277eb | 2015-03-23 12:06:32 -0700 | [diff] [blame] | 81 | return allocator()->ReturnNullOrDie(); |
Dmitry Vyukov | 2e87051 | 2012-08-15 15:35:15 +0000 | [diff] [blame] | 82 | void *p = allocator()->Allocate(&thr->alloc_cache, sz, align); |
| 83 | if (p == 0) |
Dmitry Vyukov | 7b8bee1 | 2012-05-18 09:41:52 +0000 | [diff] [blame] | 84 | return 0; |
Stephen Hines | 6a211c5 | 2014-07-21 00:49:56 -0700 | [diff] [blame] | 85 | if (ctx && ctx->initialized) |
| 86 | OnUserAlloc(thr, pc, (uptr)p, sz, true); |
Stephen Hines | 6d18623 | 2014-11-26 17:56:19 -0800 | [diff] [blame] | 87 | if (signal) |
| 88 | SignalUnsafeCall(thr, pc); |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 89 | return p; |
| 90 | } |
| 91 | |
Stephen Hines | 86277eb | 2015-03-23 12:06:32 -0700 | [diff] [blame] | 92 | void *user_calloc(ThreadState *thr, uptr pc, uptr size, uptr n) { |
| 93 | if (CallocShouldReturnNullDueToOverflow(size, n)) |
| 94 | return allocator()->ReturnNullOrDie(); |
| 95 | void *p = user_alloc(thr, pc, n * size); |
| 96 | if (p) |
| 97 | internal_memset(p, 0, n * size); |
| 98 | return p; |
| 99 | } |
| 100 | |
Stephen Hines | 6d18623 | 2014-11-26 17:56:19 -0800 | [diff] [blame] | 101 | void user_free(ThreadState *thr, uptr pc, void *p, bool signal) { |
Stephen Hines | 6a211c5 | 2014-07-21 00:49:56 -0700 | [diff] [blame] | 102 | if (ctx && ctx->initialized) |
| 103 | OnUserFree(thr, pc, (uptr)p, true); |
Dmitry Vyukov | 2e87051 | 2012-08-15 15:35:15 +0000 | [diff] [blame] | 104 | allocator()->Deallocate(&thr->alloc_cache, p); |
Stephen Hines | 6d18623 | 2014-11-26 17:56:19 -0800 | [diff] [blame] | 105 | if (signal) |
| 106 | SignalUnsafeCall(thr, pc); |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 107 | } |
| 108 | |
Stephen Hines | 6a211c5 | 2014-07-21 00:49:56 -0700 | [diff] [blame] | 109 | void OnUserAlloc(ThreadState *thr, uptr pc, uptr p, uptr sz, bool write) { |
| 110 | DPrintf("#%d: alloc(%zu) = %p\n", thr->tid, sz, p); |
| 111 | ctx->metamap.AllocBlock(thr, pc, p, sz); |
| 112 | if (write && thr->ignore_reads_and_writes == 0) |
| 113 | MemoryRangeImitateWrite(thr, pc, (uptr)p, sz); |
| 114 | else |
| 115 | MemoryResetRange(thr, pc, (uptr)p, sz); |
| 116 | } |
| 117 | |
| 118 | void OnUserFree(ThreadState *thr, uptr pc, uptr p, bool write) { |
| 119 | CHECK_NE(p, (void*)0); |
| 120 | uptr sz = ctx->metamap.FreeBlock(thr, pc, p); |
| 121 | DPrintf("#%d: free(%p, %zu)\n", thr->tid, p, sz); |
| 122 | if (write && thr->ignore_reads_and_writes == 0) |
| 123 | MemoryRangeFreed(thr, pc, (uptr)p, sz); |
| 124 | } |
| 125 | |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 126 | void *user_realloc(ThreadState *thr, uptr pc, void *p, uptr sz) { |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 127 | void *p2 = 0; |
| 128 | // FIXME: Handle "shrinking" more efficiently, |
| 129 | // it seems that some software actually does this. |
| 130 | if (sz) { |
| 131 | p2 = user_alloc(thr, pc, sz); |
Dmitry Vyukov | efd9582 | 2012-05-21 06:46:27 +0000 | [diff] [blame] | 132 | if (p2 == 0) |
| 133 | return 0; |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 134 | if (p) { |
Stephen Hines | 6a211c5 | 2014-07-21 00:49:56 -0700 | [diff] [blame] | 135 | uptr oldsz = user_alloc_usable_size(p); |
| 136 | internal_memcpy(p2, p, min(oldsz, sz)); |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 137 | } |
| 138 | } |
Dmitry Vyukov | f51c386 | 2013-03-18 19:47:36 +0000 | [diff] [blame] | 139 | if (p) |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 140 | user_free(thr, pc, p); |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 141 | return p2; |
| 142 | } |
| 143 | |
Stephen Hines | 6a211c5 | 2014-07-21 00:49:56 -0700 | [diff] [blame] | 144 | uptr user_alloc_usable_size(const void *p) { |
Alexey Samsonov | 8a6b5e5 | 2013-02-25 08:43:10 +0000 | [diff] [blame] | 145 | if (p == 0) |
| 146 | return 0; |
Stephen Hines | 6a211c5 | 2014-07-21 00:49:56 -0700 | [diff] [blame] | 147 | MBlock *b = ctx->metamap.GetBlock((uptr)p); |
| 148 | return b ? b->siz : 0; |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 149 | } |
| 150 | |
Alexey Samsonov | 4f0ea39 | 2012-09-24 13:19:47 +0000 | [diff] [blame] | 151 | void invoke_malloc_hook(void *ptr, uptr size) { |
Alexey Samsonov | 4f0ea39 | 2012-09-24 13:19:47 +0000 | [diff] [blame] | 152 | ThreadState *thr = cur_thread(); |
Stephen Hines | 2d1fdb2 | 2014-05-28 23:58:16 -0700 | [diff] [blame] | 153 | if (ctx == 0 || !ctx->initialized || thr->ignore_interceptors) |
Alexey Samsonov | 4f0ea39 | 2012-09-24 13:19:47 +0000 | [diff] [blame] | 154 | return; |
Stephen Hines | 6a211c5 | 2014-07-21 00:49:56 -0700 | [diff] [blame] | 155 | __sanitizer_malloc_hook(ptr, size); |
Alexey Samsonov | 4f0ea39 | 2012-09-24 13:19:47 +0000 | [diff] [blame] | 156 | } |
| 157 | |
| 158 | void invoke_free_hook(void *ptr) { |
Alexey Samsonov | 4f0ea39 | 2012-09-24 13:19:47 +0000 | [diff] [blame] | 159 | ThreadState *thr = cur_thread(); |
Stephen Hines | 2d1fdb2 | 2014-05-28 23:58:16 -0700 | [diff] [blame] | 160 | if (ctx == 0 || !ctx->initialized || thr->ignore_interceptors) |
Alexey Samsonov | 4f0ea39 | 2012-09-24 13:19:47 +0000 | [diff] [blame] | 161 | return; |
Stephen Hines | 6a211c5 | 2014-07-21 00:49:56 -0700 | [diff] [blame] | 162 | __sanitizer_free_hook(ptr); |
Alexey Samsonov | 4f0ea39 | 2012-09-24 13:19:47 +0000 | [diff] [blame] | 163 | } |
| 164 | |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 165 | void *internal_alloc(MBlockType typ, uptr sz) { |
| 166 | ThreadState *thr = cur_thread(); |
Dmitry Vyukov | 9ad7c32 | 2012-06-22 11:08:55 +0000 | [diff] [blame] | 167 | if (thr->nomalloc) { |
| 168 | thr->nomalloc = 0; // CHECK calls internal_malloc(). |
| 169 | CHECK(0); |
| 170 | } |
Alexey Samsonov | 1f3c2fe | 2013-05-29 09:15:39 +0000 | [diff] [blame] | 171 | return InternalAlloc(sz, &thr->internal_alloc_cache); |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 172 | } |
| 173 | |
| 174 | void internal_free(void *p) { |
| 175 | ThreadState *thr = cur_thread(); |
Dmitry Vyukov | 9ad7c32 | 2012-06-22 11:08:55 +0000 | [diff] [blame] | 176 | if (thr->nomalloc) { |
| 177 | thr->nomalloc = 0; // CHECK calls internal_malloc(). |
| 178 | CHECK(0); |
| 179 | } |
Alexey Samsonov | 1f3c2fe | 2013-05-29 09:15:39 +0000 | [diff] [blame] | 180 | InternalFree(p, &thr->internal_alloc_cache); |
Kostya Serebryany | 7ac4148 | 2012-05-10 13:48:04 +0000 | [diff] [blame] | 181 | } |
| 182 | |
| 183 | } // namespace __tsan |
Dmitry Vyukov | 1253082 | 2013-01-23 12:08:03 +0000 | [diff] [blame] | 184 | |
| 185 | using namespace __tsan; |
| 186 | |
| 187 | extern "C" { |
Stephen Hines | 6a211c5 | 2014-07-21 00:49:56 -0700 | [diff] [blame] | 188 | uptr __sanitizer_get_current_allocated_bytes() { |
| 189 | uptr stats[AllocatorStatCount]; |
| 190 | allocator()->GetStats(stats); |
| 191 | return stats[AllocatorStatAllocated]; |
| 192 | } |
Dmitry Vyukov | 1253082 | 2013-01-23 12:08:03 +0000 | [diff] [blame] | 193 | |
Stephen Hines | 6a211c5 | 2014-07-21 00:49:56 -0700 | [diff] [blame] | 194 | uptr __sanitizer_get_heap_size() { |
| 195 | uptr stats[AllocatorStatCount]; |
| 196 | allocator()->GetStats(stats); |
| 197 | return stats[AllocatorStatMapped]; |
| 198 | } |
Dmitry Vyukov | 1253082 | 2013-01-23 12:08:03 +0000 | [diff] [blame] | 199 | |
Stephen Hines | 6a211c5 | 2014-07-21 00:49:56 -0700 | [diff] [blame] | 200 | uptr __sanitizer_get_free_bytes() { |
| 201 | return 1; |
| 202 | } |
Dmitry Vyukov | 1253082 | 2013-01-23 12:08:03 +0000 | [diff] [blame] | 203 | |
Stephen Hines | 6a211c5 | 2014-07-21 00:49:56 -0700 | [diff] [blame] | 204 | uptr __sanitizer_get_unmapped_bytes() { |
| 205 | return 1; |
| 206 | } |
Dmitry Vyukov | 1253082 | 2013-01-23 12:08:03 +0000 | [diff] [blame] | 207 | |
Stephen Hines | 6a211c5 | 2014-07-21 00:49:56 -0700 | [diff] [blame] | 208 | uptr __sanitizer_get_estimated_allocated_size(uptr size) { |
Dmitry Vyukov | 1253082 | 2013-01-23 12:08:03 +0000 | [diff] [blame] | 209 | return size; |
| 210 | } |
Dmitry Vyukov | 1253082 | 2013-01-23 12:08:03 +0000 | [diff] [blame] | 211 | |
Stephen Hines | 6a211c5 | 2014-07-21 00:49:56 -0700 | [diff] [blame] | 212 | int __sanitizer_get_ownership(const void *p) { |
| 213 | return allocator()->GetBlockBegin(p) != 0; |
| 214 | } |
Stephen Hines | 6a211c5 | 2014-07-21 00:49:56 -0700 | [diff] [blame] | 215 | |
| 216 | uptr __sanitizer_get_allocated_size(const void *p) { |
| 217 | return user_alloc_usable_size(p); |
| 218 | } |
Dmitry Vyukov | 3ce2170 | 2013-03-18 17:21:15 +0000 | [diff] [blame] | 219 | |
| 220 | void __tsan_on_thread_idle() { |
| 221 | ThreadState *thr = cur_thread(); |
| 222 | allocator()->SwallowCache(&thr->alloc_cache); |
Alexey Samsonov | 1f3c2fe | 2013-05-29 09:15:39 +0000 | [diff] [blame] | 223 | internal_allocator()->SwallowCache(&thr->internal_alloc_cache); |
Stephen Hines | 6a211c5 | 2014-07-21 00:49:56 -0700 | [diff] [blame] | 224 | ctx->metamap.OnThreadIdle(thr); |
Dmitry Vyukov | 3ce2170 | 2013-03-18 17:21:15 +0000 | [diff] [blame] | 225 | } |
Dmitry Vyukov | 1253082 | 2013-01-23 12:08:03 +0000 | [diff] [blame] | 226 | } // extern "C" |