Evgeniy Stepanov | 78c56c3 | 2012-12-11 12:27:27 +0000 | [diff] [blame] | 1 | //===-- msan_allocator.cc --------------------------- ---------------------===// |
| 2 | // |
| 3 | // The LLVM Compiler Infrastructure |
| 4 | // |
| 5 | // This file is distributed under the University of Illinois Open Source |
| 6 | // License. See LICENSE.TXT for details. |
| 7 | // |
| 8 | //===----------------------------------------------------------------------===// |
| 9 | // |
| 10 | // This file is a part of MemorySanitizer. |
| 11 | // |
| 12 | // MemorySanitizer allocator. |
| 13 | //===----------------------------------------------------------------------===// |
| 14 | |
| 15 | #include "sanitizer_common/sanitizer_allocator.h" |
| 16 | #include "sanitizer_common/sanitizer_stackdepot.h" |
| 17 | #include "msan.h" |
| 18 | |
| 19 | namespace __msan { |
| 20 | |
| 21 | struct Metadata { |
| 22 | uptr requested_size; |
| 23 | }; |
| 24 | |
| 25 | static const uptr kAllocatorSpace = 0x600000000000ULL; |
| 26 | static const uptr kAllocatorSize = 0x80000000000; // 8T. |
| 27 | static const uptr kMetadataSize = sizeof(Metadata); |
Evgeniy Stepanov | 600d516 | 2013-10-15 11:33:48 +0000 | [diff] [blame] | 28 | static const uptr kMaxAllowedMallocSize = 8UL << 30; |
Evgeniy Stepanov | 78c56c3 | 2012-12-11 12:27:27 +0000 | [diff] [blame] | 29 | |
| 30 | typedef SizeClassAllocator64<kAllocatorSpace, kAllocatorSize, kMetadataSize, |
| 31 | DefaultSizeClassMap> PrimaryAllocator; |
| 32 | typedef SizeClassAllocatorLocalCache<PrimaryAllocator> AllocatorCache; |
Kostya Serebryany | 214621f | 2012-12-12 14:32:18 +0000 | [diff] [blame] | 33 | typedef LargeMmapAllocator<> SecondaryAllocator; |
Evgeniy Stepanov | 78c56c3 | 2012-12-11 12:27:27 +0000 | [diff] [blame] | 34 | typedef CombinedAllocator<PrimaryAllocator, AllocatorCache, |
| 35 | SecondaryAllocator> Allocator; |
| 36 | |
| 37 | static THREADLOCAL AllocatorCache cache; |
| 38 | static Allocator allocator; |
| 39 | |
| 40 | static int inited = 0; |
| 41 | |
| 42 | static inline void Init() { |
| 43 | if (inited) return; |
| 44 | __msan_init(); |
| 45 | inited = true; // this must happen before any threads are created. |
| 46 | allocator.Init(); |
| 47 | } |
| 48 | |
Evgeniy Stepanov | 7c6bd40 | 2013-10-22 14:31:30 +0000 | [diff] [blame] | 49 | void MsanAllocatorThreadFinish() { |
| 50 | allocator.SwallowCache(&cache); |
| 51 | } |
| 52 | |
Evgeniy Stepanov | 78c56c3 | 2012-12-11 12:27:27 +0000 | [diff] [blame] | 53 | static void *MsanAllocate(StackTrace *stack, uptr size, |
| 54 | uptr alignment, bool zeroise) { |
| 55 | Init(); |
Evgeniy Stepanov | 600d516 | 2013-10-15 11:33:48 +0000 | [diff] [blame] | 56 | if (size > kMaxAllowedMallocSize) { |
| 57 | Report("WARNING: MemorySanitizer failed to allocate %p bytes\n", |
| 58 | (void *)size); |
| 59 | return AllocatorReturnNull(); |
| 60 | } |
Evgeniy Stepanov | 78c56c3 | 2012-12-11 12:27:27 +0000 | [diff] [blame] | 61 | void *res = allocator.Allocate(&cache, size, alignment, false); |
| 62 | Metadata *meta = reinterpret_cast<Metadata*>(allocator.GetMetaData(res)); |
| 63 | meta->requested_size = size; |
Evgeniy Stepanov | effdc7e | 2013-09-16 11:03:31 +0000 | [diff] [blame] | 64 | if (zeroise) { |
Evgeniy Stepanov | 78c56c3 | 2012-12-11 12:27:27 +0000 | [diff] [blame] | 65 | __msan_clear_and_unpoison(res, size); |
Evgeniy Stepanov | effdc7e | 2013-09-16 11:03:31 +0000 | [diff] [blame] | 66 | } else if (flags()->poison_in_malloc) { |
Evgeniy Stepanov | 78c56c3 | 2012-12-11 12:27:27 +0000 | [diff] [blame] | 67 | __msan_poison(res, size); |
Evgeniy Stepanov | effdc7e | 2013-09-16 11:03:31 +0000 | [diff] [blame] | 68 | if (__msan_get_track_origins()) { |
| 69 | u32 stack_id = StackDepotPut(stack->trace, stack->size); |
| 70 | CHECK(stack_id); |
| 71 | CHECK_EQ((stack_id >> 31), |
| 72 | 0); // Higher bit is occupied by stack origins. |
| 73 | __msan_set_origin(res, size, stack_id); |
| 74 | } |
Evgeniy Stepanov | 78c56c3 | 2012-12-11 12:27:27 +0000 | [diff] [blame] | 75 | } |
Evgeniy Stepanov | 5c48a8c | 2013-08-02 14:26:58 +0000 | [diff] [blame] | 76 | MSAN_MALLOC_HOOK(res, size); |
Evgeniy Stepanov | 78c56c3 | 2012-12-11 12:27:27 +0000 | [diff] [blame] | 77 | return res; |
| 78 | } |
| 79 | |
Evgeniy Stepanov | effdc7e | 2013-09-16 11:03:31 +0000 | [diff] [blame] | 80 | void MsanDeallocate(StackTrace *stack, void *p) { |
Evgeniy Stepanov | 78c56c3 | 2012-12-11 12:27:27 +0000 | [diff] [blame] | 81 | CHECK(p); |
| 82 | Init(); |
Evgeniy Stepanov | 5c48a8c | 2013-08-02 14:26:58 +0000 | [diff] [blame] | 83 | MSAN_FREE_HOOK(p); |
Evgeniy Stepanov | 78c56c3 | 2012-12-11 12:27:27 +0000 | [diff] [blame] | 84 | Metadata *meta = reinterpret_cast<Metadata*>(allocator.GetMetaData(p)); |
| 85 | uptr size = meta->requested_size; |
Evgeniy Stepanov | 5c48a8c | 2013-08-02 14:26:58 +0000 | [diff] [blame] | 86 | meta->requested_size = 0; |
Evgeniy Stepanov | 78c56c3 | 2012-12-11 12:27:27 +0000 | [diff] [blame] | 87 | // This memory will not be reused by anyone else, so we are free to keep it |
| 88 | // poisoned. |
Evgeniy Stepanov | effdc7e | 2013-09-16 11:03:31 +0000 | [diff] [blame] | 89 | if (flags()->poison_in_free) { |
| 90 | __msan_poison(p, size); |
| 91 | if (__msan_get_track_origins()) { |
| 92 | u32 stack_id = StackDepotPut(stack->trace, stack->size); |
| 93 | CHECK(stack_id); |
| 94 | CHECK_EQ((stack_id >> 31), |
| 95 | 0); // Higher bit is occupied by stack origins. |
| 96 | __msan_set_origin(p, size, stack_id); |
| 97 | } |
| 98 | } |
Evgeniy Stepanov | 78c56c3 | 2012-12-11 12:27:27 +0000 | [diff] [blame] | 99 | allocator.Deallocate(&cache, p); |
| 100 | } |
| 101 | |
| 102 | void *MsanReallocate(StackTrace *stack, void *old_p, uptr new_size, |
| 103 | uptr alignment, bool zeroise) { |
| 104 | if (!old_p) |
| 105 | return MsanAllocate(stack, new_size, alignment, zeroise); |
| 106 | if (!new_size) { |
Evgeniy Stepanov | effdc7e | 2013-09-16 11:03:31 +0000 | [diff] [blame] | 107 | MsanDeallocate(stack, old_p); |
Evgeniy Stepanov | 78c56c3 | 2012-12-11 12:27:27 +0000 | [diff] [blame] | 108 | return 0; |
| 109 | } |
| 110 | Metadata *meta = reinterpret_cast<Metadata*>(allocator.GetMetaData(old_p)); |
| 111 | uptr old_size = meta->requested_size; |
| 112 | uptr actually_allocated_size = allocator.GetActuallyAllocatedSize(old_p); |
| 113 | if (new_size <= actually_allocated_size) { |
| 114 | // We are not reallocating here. |
| 115 | meta->requested_size = new_size; |
| 116 | if (new_size > old_size) |
| 117 | __msan_poison((char*)old_p + old_size, new_size - old_size); |
| 118 | return old_p; |
| 119 | } |
| 120 | uptr memcpy_size = Min(new_size, old_size); |
| 121 | void *new_p = MsanAllocate(stack, new_size, alignment, zeroise); |
| 122 | // Printf("realloc: old_size %zd new_size %zd\n", old_size, new_size); |
Evgeniy Stepanov | 600d516 | 2013-10-15 11:33:48 +0000 | [diff] [blame] | 123 | if (new_p) { |
Evgeniy Stepanov | 78c56c3 | 2012-12-11 12:27:27 +0000 | [diff] [blame] | 124 | __msan_memcpy(new_p, old_p, memcpy_size); |
Evgeniy Stepanov | 600d516 | 2013-10-15 11:33:48 +0000 | [diff] [blame] | 125 | MsanDeallocate(stack, old_p); |
| 126 | } |
Evgeniy Stepanov | 78c56c3 | 2012-12-11 12:27:27 +0000 | [diff] [blame] | 127 | return new_p; |
| 128 | } |
| 129 | |
Evgeniy Stepanov | 5c48a8c | 2013-08-02 14:26:58 +0000 | [diff] [blame] | 130 | static uptr AllocationSize(const void *p) { |
| 131 | if (p == 0) |
| 132 | return 0; |
| 133 | const void *beg = allocator.GetBlockBegin(p); |
| 134 | if (beg != p) |
| 135 | return 0; |
| 136 | Metadata *b = (Metadata*)allocator.GetMetaData(p); |
| 137 | return b->requested_size; |
| 138 | } |
| 139 | |
Evgeniy Stepanov | 78c56c3 | 2012-12-11 12:27:27 +0000 | [diff] [blame] | 140 | } // namespace __msan |
Evgeniy Stepanov | 5c48a8c | 2013-08-02 14:26:58 +0000 | [diff] [blame] | 141 | |
| 142 | using namespace __msan; |
| 143 | |
| 144 | uptr __msan_get_current_allocated_bytes() { |
| 145 | u64 stats[AllocatorStatCount]; |
| 146 | allocator.GetStats(stats); |
| 147 | u64 m = stats[AllocatorStatMalloced]; |
| 148 | u64 f = stats[AllocatorStatFreed]; |
| 149 | return m >= f ? m - f : 1; |
| 150 | } |
| 151 | |
| 152 | uptr __msan_get_heap_size() { |
| 153 | u64 stats[AllocatorStatCount]; |
| 154 | allocator.GetStats(stats); |
| 155 | u64 m = stats[AllocatorStatMmapped]; |
| 156 | u64 f = stats[AllocatorStatUnmapped]; |
| 157 | return m >= f ? m - f : 1; |
| 158 | } |
| 159 | |
| 160 | uptr __msan_get_free_bytes() { |
| 161 | return 1; |
| 162 | } |
| 163 | |
| 164 | uptr __msan_get_unmapped_bytes() { |
| 165 | return 1; |
| 166 | } |
| 167 | |
| 168 | uptr __msan_get_estimated_allocated_size(uptr size) { |
| 169 | return size; |
| 170 | } |
| 171 | |
Evgeniy Stepanov | 13c3790 | 2013-09-10 11:04:37 +0000 | [diff] [blame] | 172 | int __msan_get_ownership(const void *p) { |
Evgeniy Stepanov | 5c48a8c | 2013-08-02 14:26:58 +0000 | [diff] [blame] | 173 | return AllocationSize(p) != 0; |
| 174 | } |
| 175 | |
| 176 | uptr __msan_get_allocated_size(const void *p) { |
| 177 | return AllocationSize(p); |
| 178 | } |