Alexey Samsonov | 6e0c3a4 | 2012-06-07 08:52:56 +0000 | [diff] [blame] | 1 | //===-- sanitizer_allocator.cc --------------------------------------------===// |
| 2 | // |
| 3 | // The LLVM Compiler Infrastructure |
| 4 | // |
| 5 | // This file is distributed under the University of Illinois Open Source |
| 6 | // License. See LICENSE.TXT for details. |
| 7 | // |
| 8 | //===----------------------------------------------------------------------===// |
| 9 | // |
| 10 | // This file is shared between AddressSanitizer and ThreadSanitizer |
| 11 | // run-time libraries. |
Alexey Samsonov | 1f3c2fe | 2013-05-29 09:15:39 +0000 | [diff] [blame] | 12 | // This allocator is used inside run-times. |
Alexey Samsonov | 6e0c3a4 | 2012-06-07 08:52:56 +0000 | [diff] [blame] | 13 | //===----------------------------------------------------------------------===// |
Alexey Samsonov | 1f3c2fe | 2013-05-29 09:15:39 +0000 | [diff] [blame] | 14 | #include "sanitizer_allocator.h" |
| 15 | #include "sanitizer_allocator_internal.h" |
Alexey Samsonov | 6e0c3a4 | 2012-06-07 08:52:56 +0000 | [diff] [blame] | 16 | #include "sanitizer_common.h" |
| 17 | |
Alexey Samsonov | 6e0c3a4 | 2012-06-07 08:52:56 +0000 | [diff] [blame] | 18 | namespace __sanitizer { |
| 19 | |
Alexey Samsonov | 1f3c2fe | 2013-05-29 09:15:39 +0000 | [diff] [blame] | 20 | // ThreadSanitizer for Go uses libc malloc/free. |
Stephen Hines | 2d1fdb2 | 2014-05-28 23:58:16 -0700 | [diff] [blame] | 21 | #if defined(SANITIZER_GO) || defined(SANITIZER_USE_MALLOC) |
Alexey Samsonov | 1f3c2fe | 2013-05-29 09:15:39 +0000 | [diff] [blame] | 22 | # if SANITIZER_LINUX && !SANITIZER_ANDROID |
| 23 | extern "C" void *__libc_malloc(uptr size); |
| 24 | extern "C" void __libc_free(void *ptr); |
| 25 | # define LIBC_MALLOC __libc_malloc |
| 26 | # define LIBC_FREE __libc_free |
| 27 | # else |
| 28 | # include <stdlib.h> |
| 29 | # define LIBC_MALLOC malloc |
| 30 | # define LIBC_FREE free |
| 31 | # endif |
| 32 | |
| 33 | static void *RawInternalAlloc(uptr size, InternalAllocatorCache *cache) { |
| 34 | (void)cache; |
| 35 | return LIBC_MALLOC(size); |
| 36 | } |
| 37 | |
| 38 | static void RawInternalFree(void *ptr, InternalAllocatorCache *cache) { |
| 39 | (void)cache; |
| 40 | LIBC_FREE(ptr); |
| 41 | } |
| 42 | |
| 43 | InternalAllocator *internal_allocator() { |
| 44 | return 0; |
| 45 | } |
| 46 | |
| 47 | #else // SANITIZER_GO |
| 48 | |
Alexey Samsonov | 845abaf | 2013-05-29 10:41:53 +0000 | [diff] [blame] | 49 | static ALIGNED(64) char internal_alloc_placeholder[sizeof(InternalAllocator)]; |
Alexey Samsonov | 1f3c2fe | 2013-05-29 09:15:39 +0000 | [diff] [blame] | 50 | static atomic_uint8_t internal_allocator_initialized; |
| 51 | static StaticSpinMutex internal_alloc_init_mu; |
| 52 | |
| 53 | static InternalAllocatorCache internal_allocator_cache; |
| 54 | static StaticSpinMutex internal_allocator_cache_mu; |
| 55 | |
| 56 | InternalAllocator *internal_allocator() { |
| 57 | InternalAllocator *internal_allocator_instance = |
| 58 | reinterpret_cast<InternalAllocator *>(&internal_alloc_placeholder); |
| 59 | if (atomic_load(&internal_allocator_initialized, memory_order_acquire) == 0) { |
| 60 | SpinMutexLock l(&internal_alloc_init_mu); |
| 61 | if (atomic_load(&internal_allocator_initialized, memory_order_relaxed) == |
| 62 | 0) { |
Stephen Hines | 86277eb | 2015-03-23 12:06:32 -0700 | [diff] [blame] | 63 | internal_allocator_instance->Init(/* may_return_null*/ false); |
Alexey Samsonov | 1f3c2fe | 2013-05-29 09:15:39 +0000 | [diff] [blame] | 64 | atomic_store(&internal_allocator_initialized, 1, memory_order_release); |
| 65 | } |
| 66 | } |
| 67 | return internal_allocator_instance; |
| 68 | } |
| 69 | |
| 70 | static void *RawInternalAlloc(uptr size, InternalAllocatorCache *cache) { |
| 71 | if (cache == 0) { |
| 72 | SpinMutexLock l(&internal_allocator_cache_mu); |
| 73 | return internal_allocator()->Allocate(&internal_allocator_cache, size, 8, |
| 74 | false); |
| 75 | } |
| 76 | return internal_allocator()->Allocate(cache, size, 8, false); |
| 77 | } |
| 78 | |
| 79 | static void RawInternalFree(void *ptr, InternalAllocatorCache *cache) { |
| 80 | if (cache == 0) { |
| 81 | SpinMutexLock l(&internal_allocator_cache_mu); |
| 82 | return internal_allocator()->Deallocate(&internal_allocator_cache, ptr); |
| 83 | } |
| 84 | internal_allocator()->Deallocate(cache, ptr); |
| 85 | } |
| 86 | |
| 87 | #endif // SANITIZER_GO |
| 88 | |
Dmitry Vyukov | 225f531 | 2012-06-25 15:09:24 +0000 | [diff] [blame] | 89 | const u64 kBlockMagic = 0x6A6CB03ABCEBC041ull; |
Alexey Samsonov | 6e0c3a4 | 2012-06-07 08:52:56 +0000 | [diff] [blame] | 90 | |
Alexey Samsonov | 1f3c2fe | 2013-05-29 09:15:39 +0000 | [diff] [blame] | 91 | void *InternalAlloc(uptr size, InternalAllocatorCache *cache) { |
Dmitry Vyukov | b78caa6 | 2012-07-05 16:18:28 +0000 | [diff] [blame] | 92 | if (size + sizeof(u64) < size) |
| 93 | return 0; |
Alexey Samsonov | 1f3c2fe | 2013-05-29 09:15:39 +0000 | [diff] [blame] | 94 | void *p = RawInternalAlloc(size + sizeof(u64), cache); |
Dmitry Vyukov | b78caa6 | 2012-07-05 16:18:28 +0000 | [diff] [blame] | 95 | if (p == 0) |
| 96 | return 0; |
Dmitry Vyukov | 225f531 | 2012-06-25 15:09:24 +0000 | [diff] [blame] | 97 | ((u64*)p)[0] = kBlockMagic; |
Alexey Samsonov | 6e0c3a4 | 2012-06-07 08:52:56 +0000 | [diff] [blame] | 98 | return (char*)p + sizeof(u64); |
| 99 | } |
| 100 | |
Alexey Samsonov | 1f3c2fe | 2013-05-29 09:15:39 +0000 | [diff] [blame] | 101 | void InternalFree(void *addr, InternalAllocatorCache *cache) { |
Dmitry Vyukov | b78caa6 | 2012-07-05 16:18:28 +0000 | [diff] [blame] | 102 | if (addr == 0) |
| 103 | return; |
Alexey Samsonov | 6e0c3a4 | 2012-06-07 08:52:56 +0000 | [diff] [blame] | 104 | addr = (char*)addr - sizeof(u64); |
Alexey Samsonov | 1f3c2fe | 2013-05-29 09:15:39 +0000 | [diff] [blame] | 105 | CHECK_EQ(kBlockMagic, ((u64*)addr)[0]); |
Alexey Samsonov | 6e0c3a4 | 2012-06-07 08:52:56 +0000 | [diff] [blame] | 106 | ((u64*)addr)[0] = 0; |
Alexey Samsonov | 1f3c2fe | 2013-05-29 09:15:39 +0000 | [diff] [blame] | 107 | RawInternalFree(addr, cache); |
Alexey Samsonov | 6e0c3a4 | 2012-06-07 08:52:56 +0000 | [diff] [blame] | 108 | } |
| 109 | |
Alexey Samsonov | 70e177e | 2012-08-27 09:30:58 +0000 | [diff] [blame] | 110 | // LowLevelAllocator |
| 111 | static LowLevelAllocateCallback low_level_alloc_callback; |
| 112 | |
| 113 | void *LowLevelAllocator::Allocate(uptr size) { |
Alexey Samsonov | d883c80 | 2012-08-27 14:51:36 +0000 | [diff] [blame] | 114 | // Align allocation size. |
| 115 | size = RoundUpTo(size, 8); |
Alexey Samsonov | 70e177e | 2012-08-27 09:30:58 +0000 | [diff] [blame] | 116 | if (allocated_end_ - allocated_current_ < (sptr)size) { |
Kostya Serebryany | e89f184 | 2012-11-24 05:03:11 +0000 | [diff] [blame] | 117 | uptr size_to_allocate = Max(size, GetPageSizeCached()); |
Alexey Samsonov | 70e177e | 2012-08-27 09:30:58 +0000 | [diff] [blame] | 118 | allocated_current_ = |
Stephen Hines | 2d1fdb2 | 2014-05-28 23:58:16 -0700 | [diff] [blame] | 119 | (char*)MmapOrDie(size_to_allocate, __func__); |
Alexey Samsonov | 70e177e | 2012-08-27 09:30:58 +0000 | [diff] [blame] | 120 | allocated_end_ = allocated_current_ + size_to_allocate; |
| 121 | if (low_level_alloc_callback) { |
| 122 | low_level_alloc_callback((uptr)allocated_current_, |
| 123 | size_to_allocate); |
| 124 | } |
| 125 | } |
| 126 | CHECK(allocated_end_ - allocated_current_ >= (sptr)size); |
| 127 | void *res = allocated_current_; |
| 128 | allocated_current_ += size; |
| 129 | return res; |
| 130 | } |
| 131 | |
| 132 | void SetLowLevelAllocateCallback(LowLevelAllocateCallback callback) { |
| 133 | low_level_alloc_callback = callback; |
| 134 | } |
| 135 | |
Kostya Serebryany | 65199f1 | 2013-01-25 11:46:22 +0000 | [diff] [blame] | 136 | bool CallocShouldReturnNullDueToOverflow(uptr size, uptr n) { |
Kostya Serebryany | b2efe13 | 2013-01-25 12:22:21 +0000 | [diff] [blame] | 137 | if (!size) return false; |
| 138 | uptr max = (uptr)-1L; |
| 139 | return (max / size) < n; |
Kostya Serebryany | 65199f1 | 2013-01-25 11:46:22 +0000 | [diff] [blame] | 140 | } |
| 141 | |
Stephen Hines | 86277eb | 2015-03-23 12:06:32 -0700 | [diff] [blame] | 142 | void NORETURN ReportAllocatorCannotReturnNull() { |
Kostya Serebryany | 9150f39 | 2013-09-06 09:25:11 +0000 | [diff] [blame] | 143 | Report("%s's allocator is terminating the process instead of returning 0\n", |
| 144 | SanitizerToolName); |
| 145 | Report("If you don't like this behavior set allocator_may_return_null=1\n"); |
| 146 | CHECK(0); |
Stephen Hines | 86277eb | 2015-03-23 12:06:32 -0700 | [diff] [blame] | 147 | Die(); |
Kostya Serebryany | 9150f39 | 2013-09-06 09:25:11 +0000 | [diff] [blame] | 148 | } |
| 149 | |
Alexey Samsonov | 6e0c3a4 | 2012-06-07 08:52:56 +0000 | [diff] [blame] | 150 | } // namespace __sanitizer |