blob: 606439ea1d4148c7cc60d63807884fb37e179827 [file] [log] [blame]
Kostya Serebryany712fc982016-06-07 01:20:26 +00001//===-- scudo_allocator.cpp -------------------------------------*- C++ -*-===//
2//
3// The LLVM Compiler Infrastructure
4//
5// This file is distributed under the University of Illinois Open Source
6// License. See LICENSE.TXT for details.
7//
8//===----------------------------------------------------------------------===//
9///
10/// Scudo Hardened Allocator implementation.
11/// It uses the sanitizer_common allocator as a base and aims at mitigating
12/// heap corruption vulnerabilities. It provides a checksum-guarded chunk
13/// header, a delayed free list, and additional sanity checks.
14///
15//===----------------------------------------------------------------------===//
16
17#include "scudo_allocator.h"
Kostya Kortchinskyb0e96eb2017-05-09 15:12:38 +000018#include "scudo_crc32.h"
Kostya Kortchinsky43917722017-08-16 16:40:48 +000019#include "scudo_flags.h"
Kostya Kortchinsky36b34342017-04-27 20:21:16 +000020#include "scudo_tls.h"
Kostya Serebryany712fc982016-06-07 01:20:26 +000021#include "scudo_utils.h"
22
Alex Shlyapnikov42bea012017-07-18 19:11:04 +000023#include "sanitizer_common/sanitizer_allocator_checks.h"
Kostya Serebryany712fc982016-06-07 01:20:26 +000024#include "sanitizer_common/sanitizer_allocator_interface.h"
Alex Shlyapnikovdf18cbb2017-07-14 21:17:16 +000025#include "sanitizer_common/sanitizer_errno.h"
Kostya Serebryany712fc982016-06-07 01:20:26 +000026#include "sanitizer_common/sanitizer_quarantine.h"
27
Kostya Kortchinsky006805d2017-04-20 15:11:00 +000028#include <string.h>
Kostya Serebryany712fc982016-06-07 01:20:26 +000029
30namespace __scudo {
31
Kostya Serebryany712fc982016-06-07 01:20:26 +000032// Global static cookie, initialized at start-up.
Kostya Kortchinsky1148dc52016-11-30 17:32:20 +000033static uptr Cookie;
34
Kostya Kortchinsky1148dc52016-11-30 17:32:20 +000035// We default to software CRC32 if the alternatives are not supported, either
36// at compilation or at runtime.
37static atomic_uint8_t HashAlgorithm = { CRC32Software };
38
Kostya Kortchinsky43917722017-08-16 16:40:48 +000039INLINE u32 computeCRC32(u32 Crc, uptr Value, uptr *Array, uptr ArraySize) {
Kostya Kortchinskyb0e96eb2017-05-09 15:12:38 +000040 // If the hardware CRC32 feature is defined here, it was enabled everywhere,
41 // as opposed to only for scudo_crc32.cpp. This means that other hardware
42 // specific instructions were likely emitted at other places, and as a
43 // result there is no reason to not use it here.
Kostya Kortchinskyb39dff42017-01-18 17:11:17 +000044#if defined(__SSE4_2__) || defined(__ARM_FEATURE_CRC32)
Kostya Kortchinskyb0e96eb2017-05-09 15:12:38 +000045 Crc = CRC32_INTRINSIC(Crc, Value);
46 for (uptr i = 0; i < ArraySize; i++)
47 Crc = CRC32_INTRINSIC(Crc, Array[i]);
48 return Crc;
Kostya Kortchinskyb39dff42017-01-18 17:11:17 +000049#else
Kostya Kortchinskyb0e96eb2017-05-09 15:12:38 +000050 if (atomic_load_relaxed(&HashAlgorithm) == CRC32Hardware) {
51 Crc = computeHardwareCRC32(Crc, Value);
52 for (uptr i = 0; i < ArraySize; i++)
53 Crc = computeHardwareCRC32(Crc, Array[i]);
54 return Crc;
55 }
56 Crc = computeSoftwareCRC32(Crc, Value);
57 for (uptr i = 0; i < ArraySize; i++)
58 Crc = computeSoftwareCRC32(Crc, Array[i]);
59 return Crc;
60#endif // defined(__SSE4_2__) || defined(__ARM_FEATURE_CRC32)
Kostya Kortchinsky1148dc52016-11-30 17:32:20 +000061}
Kostya Serebryany712fc982016-06-07 01:20:26 +000062
Kostya Kortchinsky36b34342017-04-27 20:21:16 +000063static ScudoBackendAllocator &getBackendAllocator();
64
Kostya Serebryany712fc982016-06-07 01:20:26 +000065struct ScudoChunk : UnpackedHeader {
66 // We can't use the offset member of the chunk itself, as we would double
67 // fetch it without any warranty that it wouldn't have been tampered. To
68 // prevent this, we work with a local copy of the header.
Kostya Kortchinsky71dcc332016-10-26 16:16:58 +000069 void *getAllocBeg(UnpackedHeader *Header) {
Kostya Serebryany712fc982016-06-07 01:20:26 +000070 return reinterpret_cast<void *>(
71 reinterpret_cast<uptr>(this) - (Header->Offset << MinAlignmentLog));
72 }
73
Kostya Kortchinsky1148dc52016-11-30 17:32:20 +000074 // Returns the usable size for a chunk, meaning the amount of bytes from the
75 // beginning of the user data to the end of the backend allocated chunk.
76 uptr getUsableSize(UnpackedHeader *Header) {
Kostya Kortchinsky01a66fc2017-05-11 21:40:45 +000077 uptr Size =
Kostya Kortchinskyb44364d2017-07-13 21:01:19 +000078 getBackendAllocator().getActuallyAllocatedSize(getAllocBeg(Header),
Kostya Kortchinsky01a66fc2017-05-11 21:40:45 +000079 Header->FromPrimary);
Kostya Kortchinsky1148dc52016-11-30 17:32:20 +000080 if (Size == 0)
Kostya Kortchinsky006805d2017-04-20 15:11:00 +000081 return 0;
Kostya Kortchinsky1148dc52016-11-30 17:32:20 +000082 return Size - AlignedChunkHeaderSize - (Header->Offset << MinAlignmentLog);
83 }
84
85 // Compute the checksum of the Chunk pointer and its ChunkHeader.
Kostya Kortchinsky71dcc332016-10-26 16:16:58 +000086 u16 computeChecksum(UnpackedHeader *Header) const {
Kostya Kortchinsky1148dc52016-11-30 17:32:20 +000087 UnpackedHeader ZeroChecksumHeader = *Header;
88 ZeroChecksumHeader.Checksum = 0;
89 uptr HeaderHolder[sizeof(UnpackedHeader) / sizeof(uptr)];
90 memcpy(&HeaderHolder, &ZeroChecksumHeader, sizeof(HeaderHolder));
Kostya Kortchinsky43917722017-08-16 16:40:48 +000091 u32 Crc = computeCRC32(static_cast<u32>(Cookie),
92 reinterpret_cast<uptr>(this), HeaderHolder,
Kostya Kortchinskyb0e96eb2017-05-09 15:12:38 +000093 ARRAY_SIZE(HeaderHolder));
Kostya Kortchinskyb39dff42017-01-18 17:11:17 +000094 return static_cast<u16>(Crc);
Kostya Kortchinsky1148dc52016-11-30 17:32:20 +000095 }
96
Kostya Kortchinsky006805d2017-04-20 15:11:00 +000097 // Checks the validity of a chunk by verifying its checksum. It doesn't
98 // incur termination in the event of an invalid chunk.
Kostya Kortchinsky1148dc52016-11-30 17:32:20 +000099 bool isValid() {
100 UnpackedHeader NewUnpackedHeader;
101 const AtomicPackedHeader *AtomicHeader =
102 reinterpret_cast<const AtomicPackedHeader *>(this);
Kostya Kortchinskya00b9222017-01-20 18:32:18 +0000103 PackedHeader NewPackedHeader = atomic_load_relaxed(AtomicHeader);
Kostya Kortchinsky1148dc52016-11-30 17:32:20 +0000104 NewUnpackedHeader = bit_cast<UnpackedHeader>(NewPackedHeader);
105 return (NewUnpackedHeader.Checksum == computeChecksum(&NewUnpackedHeader));
Kostya Serebryany712fc982016-06-07 01:20:26 +0000106 }
107
Kostya Kortchinsky006805d2017-04-20 15:11:00 +0000108 // Nulls out a chunk header. When returning the chunk to the backend, there
109 // is no need to store a valid ChunkAvailable header, as this would be
110 // computationally expensive. Zeroing out serves the same purpose by making
111 // the header invalid. In the extremely rare event where 0 would be a valid
112 // checksum for the chunk, the state of the chunk is ChunkAvailable anyway.
113 COMPILER_CHECK(ChunkAvailable == 0);
114 void eraseHeader() {
115 PackedHeader NullPackedHeader = 0;
116 AtomicPackedHeader *AtomicHeader =
117 reinterpret_cast<AtomicPackedHeader *>(this);
118 atomic_store_relaxed(AtomicHeader, NullPackedHeader);
119 }
120
Kostya Serebryany712fc982016-06-07 01:20:26 +0000121 // Loads and unpacks the header, verifying the checksum in the process.
122 void loadHeader(UnpackedHeader *NewUnpackedHeader) const {
123 const AtomicPackedHeader *AtomicHeader =
124 reinterpret_cast<const AtomicPackedHeader *>(this);
Kostya Kortchinskya00b9222017-01-20 18:32:18 +0000125 PackedHeader NewPackedHeader = atomic_load_relaxed(AtomicHeader);
Kostya Serebryany712fc982016-06-07 01:20:26 +0000126 *NewUnpackedHeader = bit_cast<UnpackedHeader>(NewPackedHeader);
Kostya Kortchinsky006805d2017-04-20 15:11:00 +0000127 if (UNLIKELY(NewUnpackedHeader->Checksum !=
128 computeChecksum(NewUnpackedHeader))) {
Kostya Serebryany712fc982016-06-07 01:20:26 +0000129 dieWithMessage("ERROR: corrupted chunk header at address %p\n", this);
130 }
131 }
132
133 // Packs and stores the header, computing the checksum in the process.
134 void storeHeader(UnpackedHeader *NewUnpackedHeader) {
Kostya Kortchinsky71dcc332016-10-26 16:16:58 +0000135 NewUnpackedHeader->Checksum = computeChecksum(NewUnpackedHeader);
Kostya Serebryany712fc982016-06-07 01:20:26 +0000136 PackedHeader NewPackedHeader = bit_cast<PackedHeader>(*NewUnpackedHeader);
137 AtomicPackedHeader *AtomicHeader =
138 reinterpret_cast<AtomicPackedHeader *>(this);
Kostya Kortchinskya00b9222017-01-20 18:32:18 +0000139 atomic_store_relaxed(AtomicHeader, NewPackedHeader);
Kostya Serebryany712fc982016-06-07 01:20:26 +0000140 }
141
142 // Packs and stores the header, computing the checksum in the process. We
143 // compare the current header with the expected provided one to ensure that
144 // we are not being raced by a corruption occurring in another thread.
145 void compareExchangeHeader(UnpackedHeader *NewUnpackedHeader,
146 UnpackedHeader *OldUnpackedHeader) {
Kostya Kortchinsky71dcc332016-10-26 16:16:58 +0000147 NewUnpackedHeader->Checksum = computeChecksum(NewUnpackedHeader);
Kostya Serebryany712fc982016-06-07 01:20:26 +0000148 PackedHeader NewPackedHeader = bit_cast<PackedHeader>(*NewUnpackedHeader);
149 PackedHeader OldPackedHeader = bit_cast<PackedHeader>(*OldUnpackedHeader);
150 AtomicPackedHeader *AtomicHeader =
151 reinterpret_cast<AtomicPackedHeader *>(this);
Kostya Kortchinsky006805d2017-04-20 15:11:00 +0000152 if (UNLIKELY(!atomic_compare_exchange_strong(AtomicHeader,
153 &OldPackedHeader,
154 NewPackedHeader,
155 memory_order_relaxed))) {
Kostya Serebryany712fc982016-06-07 01:20:26 +0000156 dieWithMessage("ERROR: race on chunk header at address %p\n", this);
157 }
158 }
159};
160
Kostya Kortchinsky006805d2017-04-20 15:11:00 +0000161ScudoChunk *getScudoChunk(uptr UserBeg) {
162 return reinterpret_cast<ScudoChunk *>(UserBeg - AlignedChunkHeaderSize);
163}
164
Kostya Kortchinsky36b34342017-04-27 20:21:16 +0000165struct AllocatorOptions {
Kostya Kortchinsky2d944052017-07-24 15:29:38 +0000166 u32 QuarantineSizeKb;
Kostya Kortchinsky36b34342017-04-27 20:21:16 +0000167 u32 ThreadLocalQuarantineSizeKb;
Kostya Kortchinsky2d944052017-07-24 15:29:38 +0000168 u32 QuarantineChunksUpToSize;
Kostya Kortchinsky36b34342017-04-27 20:21:16 +0000169 bool MayReturnNull;
170 s32 ReleaseToOSIntervalMs;
171 bool DeallocationTypeMismatch;
172 bool DeleteSizeMismatch;
173 bool ZeroContents;
Kostya Serebryany712fc982016-06-07 01:20:26 +0000174
Kostya Kortchinsky36b34342017-04-27 20:21:16 +0000175 void setFrom(const Flags *f, const CommonFlags *cf);
Kostya Kortchinsky36b34342017-04-27 20:21:16 +0000176};
Kostya Serebryany712fc982016-06-07 01:20:26 +0000177
Kostya Kortchinsky36b34342017-04-27 20:21:16 +0000178void AllocatorOptions::setFrom(const Flags *f, const CommonFlags *cf) {
179 MayReturnNull = cf->allocator_may_return_null;
180 ReleaseToOSIntervalMs = cf->allocator_release_to_os_interval_ms;
Kostya Kortchinsky2d944052017-07-24 15:29:38 +0000181 QuarantineSizeKb = f->QuarantineSizeKb;
Kostya Kortchinsky36b34342017-04-27 20:21:16 +0000182 ThreadLocalQuarantineSizeKb = f->ThreadLocalQuarantineSizeKb;
Kostya Kortchinsky2d944052017-07-24 15:29:38 +0000183 QuarantineChunksUpToSize = f->QuarantineChunksUpToSize;
Kostya Kortchinsky36b34342017-04-27 20:21:16 +0000184 DeallocationTypeMismatch = f->DeallocationTypeMismatch;
185 DeleteSizeMismatch = f->DeleteSizeMismatch;
186 ZeroContents = f->ZeroContents;
Kostya Serebryany712fc982016-06-07 01:20:26 +0000187}
188
Kostya Kortchinsky36b34342017-04-27 20:21:16 +0000189static void initScudoInternal(const AllocatorOptions &Options);
190
191static bool ScudoInitIsRunning = false;
192
193void initScudo() {
Kostya Serebryany712fc982016-06-07 01:20:26 +0000194 SanitizerToolName = "Scudo";
195 CHECK(!ScudoInitIsRunning && "Scudo init calls itself!");
196 ScudoInitIsRunning = true;
197
Kostya Kortchinskyb0e96eb2017-05-09 15:12:38 +0000198 // Check if hardware CRC32 is supported in the binary and by the platform, if
199 // so, opt for the CRC32 hardware version of the checksum.
200 if (computeHardwareCRC32 && testCPUFeature(CRC32CPUFeature))
Kostya Kortchinsky1148dc52016-11-30 17:32:20 +0000201 atomic_store_relaxed(&HashAlgorithm, CRC32Hardware);
Kostya Kortchinsky1148dc52016-11-30 17:32:20 +0000202
Kostya Serebryany712fc982016-06-07 01:20:26 +0000203 initFlags();
204
205 AllocatorOptions Options;
206 Options.setFrom(getFlags(), common_flags());
Kostya Kortchinsky36b34342017-04-27 20:21:16 +0000207 initScudoInternal(Options);
Kostya Serebryany712fc982016-06-07 01:20:26 +0000208
Kostya Kortchinsky36b34342017-04-27 20:21:16 +0000209 // TODO(kostyak): determine if MaybeStartBackgroudThread could be of some use.
Kostya Kortchinsky71dcc332016-10-26 16:16:58 +0000210
Kostya Serebryany712fc982016-06-07 01:20:26 +0000211 ScudoInitIsRunning = false;
212}
213
Kostya Serebryany712fc982016-06-07 01:20:26 +0000214struct QuarantineCallback {
215 explicit QuarantineCallback(AllocatorCache *Cache)
216 : Cache_(Cache) {}
217
Kostya Kortchinsky01a66fc2017-05-11 21:40:45 +0000218 // Chunk recycling function, returns a quarantined chunk to the backend,
219 // first making sure it hasn't been tampered with.
Kostya Serebryany712fc982016-06-07 01:20:26 +0000220 void Recycle(ScudoChunk *Chunk) {
221 UnpackedHeader Header;
222 Chunk->loadHeader(&Header);
Kostya Kortchinsky006805d2017-04-20 15:11:00 +0000223 if (UNLIKELY(Header.State != ChunkQuarantine)) {
Kostya Serebryany712fc982016-06-07 01:20:26 +0000224 dieWithMessage("ERROR: invalid chunk state when recycling address %p\n",
225 Chunk);
226 }
Kostya Kortchinsky006805d2017-04-20 15:11:00 +0000227 Chunk->eraseHeader();
Kostya Kortchinsky71dcc332016-10-26 16:16:58 +0000228 void *Ptr = Chunk->getAllocBeg(&Header);
Kostya Kortchinskyb44364d2017-07-13 21:01:19 +0000229 if (Header.FromPrimary)
230 getBackendAllocator().deallocatePrimary(Cache_, Ptr);
231 else
232 getBackendAllocator().deallocateSecondary(Ptr);
Kostya Serebryany712fc982016-06-07 01:20:26 +0000233 }
234
Kostya Kortchinsky01a66fc2017-05-11 21:40:45 +0000235 // Internal quarantine allocation and deallocation functions. We first check
236 // that the batches are indeed serviced by the Primary.
237 // TODO(kostyak): figure out the best way to protect the batches.
238 COMPILER_CHECK(sizeof(QuarantineBatch) < SizeClassMap::kMaxSize);
Kostya Serebryany712fc982016-06-07 01:20:26 +0000239 void *Allocate(uptr Size) {
Kostya Kortchinskyb44364d2017-07-13 21:01:19 +0000240 return getBackendAllocator().allocatePrimary(Cache_, Size);
Kostya Serebryany712fc982016-06-07 01:20:26 +0000241 }
242
243 void Deallocate(void *Ptr) {
Kostya Kortchinskyb44364d2017-07-13 21:01:19 +0000244 getBackendAllocator().deallocatePrimary(Cache_, Ptr);
Kostya Serebryany712fc982016-06-07 01:20:26 +0000245 }
246
247 AllocatorCache *Cache_;
248};
249
250typedef Quarantine<QuarantineCallback, ScudoChunk> ScudoQuarantine;
Kostya Kortchinsky006805d2017-04-20 15:11:00 +0000251typedef ScudoQuarantine::Cache ScudoQuarantineCache;
Kostya Kortchinsky36b34342017-04-27 20:21:16 +0000252COMPILER_CHECK(sizeof(ScudoQuarantineCache) <=
Kostya Kortchinsky39248092017-09-22 15:35:37 +0000253 sizeof(ScudoTSD::QuarantineCachePlaceHolder));
Kostya Serebryany712fc982016-06-07 01:20:26 +0000254
Kostya Kortchinsky39248092017-09-22 15:35:37 +0000255ScudoQuarantineCache *getQuarantineCache(ScudoTSD *TSD) {
256 return reinterpret_cast<ScudoQuarantineCache *>(
257 TSD->QuarantineCachePlaceHolder);
Kostya Serebryany712fc982016-06-07 01:20:26 +0000258}
259
Kostya Kortchinsky006805d2017-04-20 15:11:00 +0000260struct ScudoAllocator {
Kostya Kortchinsky71dcc332016-10-26 16:16:58 +0000261 static const uptr MaxAllowedMallocSize =
262 FIRST_32_SECOND_64(2UL << 30, 1ULL << 40);
Kostya Serebryany712fc982016-06-07 01:20:26 +0000263
Alex Shlyapnikovccab11b2017-06-20 21:23:02 +0000264 typedef ReturnNullOrDieOnFailure FailureHandler;
265
Kostya Kortchinsky006805d2017-04-20 15:11:00 +0000266 ScudoBackendAllocator BackendAllocator;
Kostya Serebryany712fc982016-06-07 01:20:26 +0000267 ScudoQuarantine AllocatorQuarantine;
268
Kostya Kortchinskyb44364d2017-07-13 21:01:19 +0000269 StaticSpinMutex GlobalPrngMutex;
270 ScudoPrng GlobalPrng;
271
Kostya Kortchinsky2d944052017-07-24 15:29:38 +0000272 u32 QuarantineChunksUpToSize;
273
Kostya Serebryany712fc982016-06-07 01:20:26 +0000274 bool DeallocationTypeMismatch;
275 bool ZeroContents;
276 bool DeleteSizeMismatch;
277
Kostya Kortchinsky006805d2017-04-20 15:11:00 +0000278 explicit ScudoAllocator(LinkerInitialized)
Kostya Kortchinsky22396c22017-09-25 15:12:08 +0000279 : AllocatorQuarantine(LINKER_INITIALIZED) {}
Kostya Serebryany712fc982016-06-07 01:20:26 +0000280
281 void init(const AllocatorOptions &Options) {
Kostya Serebryany712fc982016-06-07 01:20:26 +0000282 // Verify that the header offset field can hold the maximum offset. In the
Kostya Kortchinsky71dcc332016-10-26 16:16:58 +0000283 // case of the Secondary allocator, it takes care of alignment and the
284 // offset will always be 0. In the case of the Primary, the worst case
285 // scenario happens in the last size class, when the backend allocation
286 // would already be aligned on the requested alignment, which would happen
287 // to be the maximum alignment that would fit in that size class. As a
288 // result, the maximum offset will be at most the maximum alignment for the
289 // last size class minus the header size, in multiples of MinAlignment.
Kostya Serebryany712fc982016-06-07 01:20:26 +0000290 UnpackedHeader Header = {};
Kostya Kortchinskyb44364d2017-07-13 21:01:19 +0000291 uptr MaxPrimaryAlignment =
292 1 << MostSignificantSetBitIndex(SizeClassMap::kMaxSize - MinAlignment);
293 uptr MaxOffset =
294 (MaxPrimaryAlignment - AlignedChunkHeaderSize) >> MinAlignmentLog;
Kostya Kortchinsky1148dc52016-11-30 17:32:20 +0000295 Header.Offset = MaxOffset;
296 if (Header.Offset != MaxOffset) {
Kostya Serebryany712fc982016-06-07 01:20:26 +0000297 dieWithMessage("ERROR: the maximum possible offset doesn't fit in the "
298 "header\n");
299 }
Kostya Kortchinskyfff8e062017-04-20 18:07:17 +0000300 // Verify that we can fit the maximum size or amount of unused bytes in the
301 // header. Given that the Secondary fits the allocation to a page, the worst
302 // case scenario happens in the Primary. It will depend on the second to
303 // last and last class sizes, as well as the dynamic base for the Primary.
304 // The following is an over-approximation that works for our needs.
305 uptr MaxSizeOrUnusedBytes = SizeClassMap::kMaxSize - 1;
306 Header.SizeOrUnusedBytes = MaxSizeOrUnusedBytes;
307 if (Header.SizeOrUnusedBytes != MaxSizeOrUnusedBytes) {
Kostya Kortchinsky1148dc52016-11-30 17:32:20 +0000308 dieWithMessage("ERROR: the maximum possible unused bytes doesn't fit in "
309 "the header\n");
310 }
Kostya Serebryany712fc982016-06-07 01:20:26 +0000311
312 DeallocationTypeMismatch = Options.DeallocationTypeMismatch;
313 DeleteSizeMismatch = Options.DeleteSizeMismatch;
314 ZeroContents = Options.ZeroContents;
Alex Shlyapnikovccab11b2017-06-20 21:23:02 +0000315 SetAllocatorMayReturnNull(Options.MayReturnNull);
Kostya Kortchinskyb44364d2017-07-13 21:01:19 +0000316 BackendAllocator.init(Options.ReleaseToOSIntervalMs);
Kostya Kortchinsky71dcc332016-10-26 16:16:58 +0000317 AllocatorQuarantine.Init(
Kostya Kortchinsky2d944052017-07-24 15:29:38 +0000318 static_cast<uptr>(Options.QuarantineSizeKb) << 10,
Kostya Kortchinsky71dcc332016-10-26 16:16:58 +0000319 static_cast<uptr>(Options.ThreadLocalQuarantineSizeKb) << 10);
Kostya Kortchinsky2d944052017-07-24 15:29:38 +0000320 QuarantineChunksUpToSize = Options.QuarantineChunksUpToSize;
Kostya Kortchinskyb44364d2017-07-13 21:01:19 +0000321 GlobalPrng.init();
322 Cookie = GlobalPrng.getU64();
Kostya Serebryany712fc982016-06-07 01:20:26 +0000323 }
324
Kostya Kortchinsky006805d2017-04-20 15:11:00 +0000325 // Helper function that checks for a valid Scudo chunk. nullptr isn't.
Kostya Kortchinsky1148dc52016-11-30 17:32:20 +0000326 bool isValidPointer(const void *UserPtr) {
Kostya Kortchinsky36b34342017-04-27 20:21:16 +0000327 initThreadMaybe();
Kostya Kortchinsky0ce49992017-06-29 16:45:20 +0000328 if (UNLIKELY(!UserPtr))
Kostya Kortchinsky1148dc52016-11-30 17:32:20 +0000329 return false;
Kostya Kortchinsky006805d2017-04-20 15:11:00 +0000330 uptr UserBeg = reinterpret_cast<uptr>(UserPtr);
331 if (!IsAligned(UserBeg, MinAlignment))
332 return false;
333 return getScudoChunk(UserBeg)->isValid();
Kostya Kortchinsky1148dc52016-11-30 17:32:20 +0000334 }
335
Kostya Serebryany712fc982016-06-07 01:20:26 +0000336 // Allocates a chunk.
Kostya Kortchinsky006805d2017-04-20 15:11:00 +0000337 void *allocate(uptr Size, uptr Alignment, AllocType Type,
338 bool ForceZeroContents = false) {
Kostya Kortchinsky36b34342017-04-27 20:21:16 +0000339 initThreadMaybe();
Kostya Kortchinsky0ce49992017-06-29 16:45:20 +0000340 if (UNLIKELY(Alignment > MaxAlignment))
Alex Shlyapnikovccab11b2017-06-20 21:23:02 +0000341 return FailureHandler::OnBadRequest();
Kostya Kortchinsky0ce49992017-06-29 16:45:20 +0000342 if (UNLIKELY(Alignment < MinAlignment))
Kostya Serebryany712fc982016-06-07 01:20:26 +0000343 Alignment = MinAlignment;
Kostya Kortchinsky0ce49992017-06-29 16:45:20 +0000344 if (UNLIKELY(Size >= MaxAllowedMallocSize))
Alex Shlyapnikovccab11b2017-06-20 21:23:02 +0000345 return FailureHandler::OnBadRequest();
Kostya Kortchinsky0ce49992017-06-29 16:45:20 +0000346 if (UNLIKELY(Size == 0))
Kostya Kortchinsky006805d2017-04-20 15:11:00 +0000347 Size = 1;
Kostya Kortchinskyc74da7c2016-12-13 19:31:54 +0000348
349 uptr NeededSize = RoundUpTo(Size, MinAlignment) + AlignedChunkHeaderSize;
Kostya Kortchinsky01a66fc2017-05-11 21:40:45 +0000350 uptr AlignedSize = (Alignment > MinAlignment) ?
351 NeededSize + (Alignment - AlignedChunkHeaderSize) : NeededSize;
Kostya Kortchinsky0ce49992017-06-29 16:45:20 +0000352 if (UNLIKELY(AlignedSize >= MaxAllowedMallocSize))
Alex Shlyapnikovccab11b2017-06-20 21:23:02 +0000353 return FailureHandler::OnBadRequest();
Kostya Kortchinskyc74da7c2016-12-13 19:31:54 +0000354
Kostya Kortchinsky01a66fc2017-05-11 21:40:45 +0000355 // Primary and Secondary backed allocations have a different treatment. We
356 // deal with alignment requirements of Primary serviced allocations here,
357 // but the Secondary will take care of its own alignment needs.
358 bool FromPrimary = PrimaryAllocator::CanAllocate(AlignedSize, MinAlignment);
Kostya Serebryany712fc982016-06-07 01:20:26 +0000359
360 void *Ptr;
Kostya Kortchinsky00582562017-07-12 15:29:08 +0000361 u8 Salt;
Kostya Kortchinskyb44364d2017-07-13 21:01:19 +0000362 uptr AllocSize;
363 if (FromPrimary) {
364 AllocSize = AlignedSize;
Kostya Kortchinsky39248092017-09-22 15:35:37 +0000365 ScudoTSD *TSD = getTSDAndLock();
Kostya Kortchinsky22396c22017-09-25 15:12:08 +0000366 Salt = TSD->Prng.getU8();
367 Ptr = BackendAllocator.allocatePrimary(&TSD->Cache, AllocSize);
368 TSD->unlock();
Kostya Serebryany712fc982016-06-07 01:20:26 +0000369 } else {
Kostya Kortchinskyb44364d2017-07-13 21:01:19 +0000370 {
371 SpinMutexLock l(&GlobalPrngMutex);
372 Salt = GlobalPrng.getU8();
373 }
374 AllocSize = NeededSize;
375 Ptr = BackendAllocator.allocateSecondary(AllocSize, Alignment);
Kostya Serebryany712fc982016-06-07 01:20:26 +0000376 }
Kostya Kortchinsky0ce49992017-06-29 16:45:20 +0000377 if (UNLIKELY(!Ptr))
Alex Shlyapnikovccab11b2017-06-20 21:23:02 +0000378 return FailureHandler::OnOOM();
Kostya Serebryany712fc982016-06-07 01:20:26 +0000379
Kostya Kortchinsky1148dc52016-11-30 17:32:20 +0000380 // If requested, we will zero out the entire contents of the returned chunk.
Kostya Kortchinsky006805d2017-04-20 15:11:00 +0000381 if ((ForceZeroContents || ZeroContents) && FromPrimary)
Kostya Kortchinskyb44364d2017-07-13 21:01:19 +0000382 memset(Ptr, 0, BackendAllocator.getActuallyAllocatedSize(
383 Ptr, /*FromPrimary=*/true));
Kostya Kortchinsky1148dc52016-11-30 17:32:20 +0000384
Kostya Serebryany712fc982016-06-07 01:20:26 +0000385 UnpackedHeader Header = {};
Kostya Kortchinsky01a66fc2017-05-11 21:40:45 +0000386 uptr AllocBeg = reinterpret_cast<uptr>(Ptr);
387 uptr UserBeg = AllocBeg + AlignedChunkHeaderSize;
Kostya Kortchinsky0ce49992017-06-29 16:45:20 +0000388 if (UNLIKELY(!IsAligned(UserBeg, Alignment))) {
Kostya Kortchinsky01a66fc2017-05-11 21:40:45 +0000389 // Since the Secondary takes care of alignment, a non-aligned pointer
390 // means it is from the Primary. It is also the only case where the offset
391 // field of the header would be non-zero.
392 CHECK(FromPrimary);
393 UserBeg = RoundUpTo(UserBeg, Alignment);
394 uptr Offset = UserBeg - AlignedChunkHeaderSize - AllocBeg;
395 Header.Offset = Offset >> MinAlignmentLog;
396 }
Kostya Kortchinskyb44364d2017-07-13 21:01:19 +0000397 CHECK_LE(UserBeg + Size, AllocBeg + AllocSize);
Kostya Serebryany712fc982016-06-07 01:20:26 +0000398 Header.State = ChunkAllocated;
Kostya Serebryany712fc982016-06-07 01:20:26 +0000399 Header.AllocType = Type;
Kostya Kortchinskyfff8e062017-04-20 18:07:17 +0000400 if (FromPrimary) {
Kostya Kortchinskyb44364d2017-07-13 21:01:19 +0000401 Header.FromPrimary = 1;
Kostya Kortchinskyfff8e062017-04-20 18:07:17 +0000402 Header.SizeOrUnusedBytes = Size;
403 } else {
404 // The secondary fits the allocations to a page, so the amount of unused
405 // bytes is the difference between the end of the user allocation and the
406 // next page boundary.
407 uptr PageSize = GetPageSizeCached();
408 uptr TrailingBytes = (UserBeg + Size) & (PageSize - 1);
409 if (TrailingBytes)
410 Header.SizeOrUnusedBytes = PageSize - TrailingBytes;
411 }
Kostya Kortchinskyb44364d2017-07-13 21:01:19 +0000412 Header.Salt = Salt;
Kostya Kortchinsky006805d2017-04-20 15:11:00 +0000413 getScudoChunk(UserBeg)->storeHeader(&Header);
414 void *UserPtr = reinterpret_cast<void *>(UserBeg);
Kostya Serebryany712fc982016-06-07 01:20:26 +0000415 // if (&__sanitizer_malloc_hook) __sanitizer_malloc_hook(UserPtr, Size);
416 return UserPtr;
417 }
418
Kostya Kortchinsky2d944052017-07-24 15:29:38 +0000419 // Place a chunk in the quarantine or directly deallocate it in the event of
420 // a zero-sized quarantine, or if the size of the chunk is greater than the
421 // quarantine chunk size threshold.
Kostya Kortchinskyf1a54fd2017-04-21 18:10:53 +0000422 void quarantineOrDeallocateChunk(ScudoChunk *Chunk, UnpackedHeader *Header,
423 uptr Size) {
Kostya Kortchinsky2d944052017-07-24 15:29:38 +0000424 const bool BypassQuarantine = (AllocatorQuarantine.GetCacheSize() == 0) ||
425 (Size > QuarantineChunksUpToSize);
Kostya Kortchinskyf1a54fd2017-04-21 18:10:53 +0000426 if (BypassQuarantine) {
427 Chunk->eraseHeader();
428 void *Ptr = Chunk->getAllocBeg(Header);
Kostya Kortchinsky2d944052017-07-24 15:29:38 +0000429 if (Header->FromPrimary) {
Kostya Kortchinsky39248092017-09-22 15:35:37 +0000430 ScudoTSD *TSD = getTSDAndLock();
Kostya Kortchinsky22396c22017-09-25 15:12:08 +0000431 getBackendAllocator().deallocatePrimary(&TSD->Cache, Ptr);
432 TSD->unlock();
Kostya Kortchinskyf1a54fd2017-04-21 18:10:53 +0000433 } else {
Kostya Kortchinskyb44364d2017-07-13 21:01:19 +0000434 getBackendAllocator().deallocateSecondary(Ptr);
Kostya Kortchinskyf1a54fd2017-04-21 18:10:53 +0000435 }
436 } else {
Kostya Kortchinsky2d944052017-07-24 15:29:38 +0000437 // If a small memory amount was allocated with a larger alignment, we want
438 // to take that into account. Otherwise the Quarantine would be filled
439 // with tiny chunks, taking a lot of VA memory. This is an approximation
440 // of the usable size, that allows us to not call
441 // GetActuallyAllocatedSize.
442 uptr EstimatedSize = Size + (Header->Offset << MinAlignmentLog);
Kostya Kortchinskyf1a54fd2017-04-21 18:10:53 +0000443 UnpackedHeader NewHeader = *Header;
444 NewHeader.State = ChunkQuarantine;
445 Chunk->compareExchangeHeader(&NewHeader, Header);
Kostya Kortchinsky39248092017-09-22 15:35:37 +0000446 ScudoTSD *TSD = getTSDAndLock();
Kostya Kortchinsky22396c22017-09-25 15:12:08 +0000447 AllocatorQuarantine.Put(getQuarantineCache(TSD),
448 QuarantineCallback(&TSD->Cache),
449 Chunk, EstimatedSize);
450 TSD->unlock();
Kostya Kortchinskyf1a54fd2017-04-21 18:10:53 +0000451 }
452 }
453
Kostya Serebryany712fc982016-06-07 01:20:26 +0000454 // Deallocates a Chunk, which means adding it to the delayed free list (or
455 // Quarantine).
456 void deallocate(void *UserPtr, uptr DeleteSize, AllocType Type) {
Kostya Kortchinsky040c2112017-09-11 19:59:40 +0000457 // For a deallocation, we only ensure minimal initialization, meaning thread
458 // local data will be left uninitialized for now (when using ELF TLS). The
459 // fallback cache will be used instead. This is a workaround for a situation
460 // where the only heap operation performed in a thread would be a free past
461 // the TLS destructors, ending up in initialized thread specific data never
462 // being destroyed properly. Any other heap operation will do a full init.
463 initThreadMaybe(/*MinimalInit=*/true);
Kostya Serebryany712fc982016-06-07 01:20:26 +0000464 // if (&__sanitizer_free_hook) __sanitizer_free_hook(UserPtr);
Kostya Kortchinsky0ce49992017-06-29 16:45:20 +0000465 if (UNLIKELY(!UserPtr))
Kostya Serebryany712fc982016-06-07 01:20:26 +0000466 return;
Kostya Kortchinsky006805d2017-04-20 15:11:00 +0000467 uptr UserBeg = reinterpret_cast<uptr>(UserPtr);
468 if (UNLIKELY(!IsAligned(UserBeg, MinAlignment))) {
Kostya Serebryany712fc982016-06-07 01:20:26 +0000469 dieWithMessage("ERROR: attempted to deallocate a chunk not properly "
470 "aligned at address %p\n", UserPtr);
471 }
Kostya Kortchinsky006805d2017-04-20 15:11:00 +0000472 ScudoChunk *Chunk = getScudoChunk(UserBeg);
Kostya Kortchinsky2d944052017-07-24 15:29:38 +0000473 UnpackedHeader Header;
474 Chunk->loadHeader(&Header);
475 if (UNLIKELY(Header.State != ChunkAllocated)) {
Kostya Serebryany712fc982016-06-07 01:20:26 +0000476 dieWithMessage("ERROR: invalid chunk state when deallocating address "
Kostya Kortchinsky1148dc52016-11-30 17:32:20 +0000477 "%p\n", UserPtr);
Kostya Serebryany712fc982016-06-07 01:20:26 +0000478 }
Kostya Serebryany712fc982016-06-07 01:20:26 +0000479 if (DeallocationTypeMismatch) {
480 // The deallocation type has to match the allocation one.
Kostya Kortchinsky2d944052017-07-24 15:29:38 +0000481 if (Header.AllocType != Type) {
Kostya Serebryany712fc982016-06-07 01:20:26 +0000482 // With the exception of memalign'd Chunks, that can be still be free'd.
Kostya Kortchinsky2d944052017-07-24 15:29:38 +0000483 if (Header.AllocType != FromMemalign || Type != FromMalloc) {
Kostya Kortchinsky43917722017-08-16 16:40:48 +0000484 dieWithMessage("ERROR: allocation type mismatch when deallocating "
485 "address %p\n", UserPtr);
Kostya Serebryany712fc982016-06-07 01:20:26 +0000486 }
487 }
488 }
Kostya Kortchinsky2d944052017-07-24 15:29:38 +0000489 uptr Size = Header.FromPrimary ? Header.SizeOrUnusedBytes :
490 Chunk->getUsableSize(&Header) - Header.SizeOrUnusedBytes;
Kostya Serebryany712fc982016-06-07 01:20:26 +0000491 if (DeleteSizeMismatch) {
492 if (DeleteSize && DeleteSize != Size) {
493 dieWithMessage("ERROR: invalid sized delete on chunk at address %p\n",
Kostya Kortchinsky006805d2017-04-20 15:11:00 +0000494 UserPtr);
Kostya Serebryany712fc982016-06-07 01:20:26 +0000495 }
496 }
Kostya Kortchinsky2d944052017-07-24 15:29:38 +0000497 quarantineOrDeallocateChunk(Chunk, &Header, Size);
Kostya Serebryany712fc982016-06-07 01:20:26 +0000498 }
499
Kostya Serebryany712fc982016-06-07 01:20:26 +0000500 // Reallocates a chunk. We can save on a new allocation if the new requested
501 // size still fits in the chunk.
502 void *reallocate(void *OldPtr, uptr NewSize) {
Kostya Kortchinsky36b34342017-04-27 20:21:16 +0000503 initThreadMaybe();
Kostya Kortchinsky006805d2017-04-20 15:11:00 +0000504 uptr UserBeg = reinterpret_cast<uptr>(OldPtr);
505 if (UNLIKELY(!IsAligned(UserBeg, MinAlignment))) {
506 dieWithMessage("ERROR: attempted to reallocate a chunk not properly "
507 "aligned at address %p\n", OldPtr);
508 }
509 ScudoChunk *Chunk = getScudoChunk(UserBeg);
Kostya Kortchinsky1148dc52016-11-30 17:32:20 +0000510 UnpackedHeader OldHeader;
511 Chunk->loadHeader(&OldHeader);
Kostya Kortchinsky006805d2017-04-20 15:11:00 +0000512 if (UNLIKELY(OldHeader.State != ChunkAllocated)) {
Kostya Kortchinsky1148dc52016-11-30 17:32:20 +0000513 dieWithMessage("ERROR: invalid chunk state when reallocating address "
514 "%p\n", OldPtr);
515 }
Kostya Kortchinsky43917722017-08-16 16:40:48 +0000516 if (DeallocationTypeMismatch) {
517 if (UNLIKELY(OldHeader.AllocType != FromMalloc)) {
518 dieWithMessage("ERROR: allocation type mismatch when reallocating "
519 "address %p\n", OldPtr);
520 }
Kostya Serebryany712fc982016-06-07 01:20:26 +0000521 }
Kostya Kortchinsky006805d2017-04-20 15:11:00 +0000522 uptr UsableSize = Chunk->getUsableSize(&OldHeader);
Kostya Kortchinskyfff8e062017-04-20 18:07:17 +0000523 // The new size still fits in the current chunk, and the size difference
524 // is reasonable.
525 if (NewSize <= UsableSize &&
526 (UsableSize - NewSize) < (SizeClassMap::kMaxSize / 2)) {
Kostya Kortchinskyf1a54fd2017-04-21 18:10:53 +0000527 UnpackedHeader NewHeader = OldHeader;
Kostya Kortchinskyfff8e062017-04-20 18:07:17 +0000528 NewHeader.SizeOrUnusedBytes =
529 OldHeader.FromPrimary ? NewSize : UsableSize - NewSize;
Kostya Serebryany712fc982016-06-07 01:20:26 +0000530 Chunk->compareExchangeHeader(&NewHeader, &OldHeader);
531 return OldPtr;
532 }
533 // Otherwise, we have to allocate a new chunk and copy the contents of the
534 // old one.
535 void *NewPtr = allocate(NewSize, MinAlignment, FromMalloc);
536 if (NewPtr) {
Kostya Kortchinskyfff8e062017-04-20 18:07:17 +0000537 uptr OldSize = OldHeader.FromPrimary ? OldHeader.SizeOrUnusedBytes :
538 UsableSize - OldHeader.SizeOrUnusedBytes;
Kostya Kortchinsky43917722017-08-16 16:40:48 +0000539 memcpy(NewPtr, OldPtr, Min(NewSize, UsableSize));
Kostya Kortchinsky2d944052017-07-24 15:29:38 +0000540 quarantineOrDeallocateChunk(Chunk, &OldHeader, OldSize);
Kostya Serebryany712fc982016-06-07 01:20:26 +0000541 }
542 return NewPtr;
543 }
544
Kostya Kortchinsky1148dc52016-11-30 17:32:20 +0000545 // Helper function that returns the actual usable size of a chunk.
546 uptr getUsableSize(const void *Ptr) {
Kostya Kortchinsky36b34342017-04-27 20:21:16 +0000547 initThreadMaybe();
Kostya Kortchinsky0ce49992017-06-29 16:45:20 +0000548 if (UNLIKELY(!Ptr))
Kostya Kortchinsky1148dc52016-11-30 17:32:20 +0000549 return 0;
Kostya Kortchinsky006805d2017-04-20 15:11:00 +0000550 uptr UserBeg = reinterpret_cast<uptr>(Ptr);
551 ScudoChunk *Chunk = getScudoChunk(UserBeg);
Kostya Kortchinsky1148dc52016-11-30 17:32:20 +0000552 UnpackedHeader Header;
553 Chunk->loadHeader(&Header);
554 // Getting the usable size of a chunk only makes sense if it's allocated.
Kostya Kortchinsky006805d2017-04-20 15:11:00 +0000555 if (UNLIKELY(Header.State != ChunkAllocated)) {
Kostya Kortchinsky1148dc52016-11-30 17:32:20 +0000556 dieWithMessage("ERROR: invalid chunk state when sizing address %p\n",
557 Ptr);
558 }
559 return Chunk->getUsableSize(&Header);
560 }
561
Kostya Serebryany712fc982016-06-07 01:20:26 +0000562 void *calloc(uptr NMemB, uptr Size) {
Kostya Kortchinsky36b34342017-04-27 20:21:16 +0000563 initThreadMaybe();
Kostya Kortchinskyb44364d2017-07-13 21:01:19 +0000564 if (UNLIKELY(CheckForCallocOverflow(NMemB, Size)))
Alex Shlyapnikovccab11b2017-06-20 21:23:02 +0000565 return FailureHandler::OnBadRequest();
Kostya Kortchinsky0ce49992017-06-29 16:45:20 +0000566 return allocate(NMemB * Size, MinAlignment, FromMalloc, true);
Kostya Serebryany712fc982016-06-07 01:20:26 +0000567 }
568
Kostya Kortchinsky39248092017-09-22 15:35:37 +0000569 void commitBack(ScudoTSD *TSD) {
570 AllocatorQuarantine.Drain(getQuarantineCache(TSD),
571 QuarantineCallback(&TSD->Cache));
572 BackendAllocator.destroyCache(&TSD->Cache);
Kostya Serebryany712fc982016-06-07 01:20:26 +0000573 }
Kostya Kortchinsky8d6257b2017-02-03 20:49:42 +0000574
575 uptr getStats(AllocatorStat StatType) {
Kostya Kortchinsky36b34342017-04-27 20:21:16 +0000576 initThreadMaybe();
Kostya Kortchinsky8d6257b2017-02-03 20:49:42 +0000577 uptr stats[AllocatorStatCount];
Kostya Kortchinskyb44364d2017-07-13 21:01:19 +0000578 BackendAllocator.getStats(stats);
Kostya Kortchinsky8d6257b2017-02-03 20:49:42 +0000579 return stats[StatType];
580 }
Kostya Kortchinsky26e689f2017-09-14 20:34:32 +0000581
582 void *handleBadRequest() {
583 initThreadMaybe();
584 return FailureHandler::OnBadRequest();
585 }
Kostya Serebryany712fc982016-06-07 01:20:26 +0000586};
587
Kostya Kortchinsky006805d2017-04-20 15:11:00 +0000588static ScudoAllocator Instance(LINKER_INITIALIZED);
Kostya Serebryany712fc982016-06-07 01:20:26 +0000589
Kostya Kortchinsky006805d2017-04-20 15:11:00 +0000590static ScudoBackendAllocator &getBackendAllocator() {
Kostya Serebryany712fc982016-06-07 01:20:26 +0000591 return Instance.BackendAllocator;
592}
593
Kostya Kortchinsky36b34342017-04-27 20:21:16 +0000594static void initScudoInternal(const AllocatorOptions &Options) {
Kostya Serebryany712fc982016-06-07 01:20:26 +0000595 Instance.init(Options);
596}
597
Kostya Kortchinsky22396c22017-09-25 15:12:08 +0000598void ScudoTSD::init(bool Shared) {
599 UnlockRequired = Shared;
Kostya Kortchinskyb44364d2017-07-13 21:01:19 +0000600 getBackendAllocator().initCache(&Cache);
Kostya Kortchinsky00582562017-07-12 15:29:08 +0000601 Prng.init();
Kostya Kortchinsky36b34342017-04-27 20:21:16 +0000602 memset(QuarantineCachePlaceHolder, 0, sizeof(QuarantineCachePlaceHolder));
603}
604
Kostya Kortchinsky39248092017-09-22 15:35:37 +0000605void ScudoTSD::commitBack() {
Kostya Kortchinsky36b34342017-04-27 20:21:16 +0000606 Instance.commitBack(this);
Kostya Serebryany712fc982016-06-07 01:20:26 +0000607}
608
609void *scudoMalloc(uptr Size, AllocType Type) {
Alex Shlyapnikov42bea012017-07-18 19:11:04 +0000610 return SetErrnoOnNull(Instance.allocate(Size, MinAlignment, Type));
Kostya Serebryany712fc982016-06-07 01:20:26 +0000611}
612
613void scudoFree(void *Ptr, AllocType Type) {
614 Instance.deallocate(Ptr, 0, Type);
615}
616
617void scudoSizedFree(void *Ptr, uptr Size, AllocType Type) {
618 Instance.deallocate(Ptr, Size, Type);
619}
620
621void *scudoRealloc(void *Ptr, uptr Size) {
622 if (!Ptr)
Alex Shlyapnikov42bea012017-07-18 19:11:04 +0000623 return SetErrnoOnNull(Instance.allocate(Size, MinAlignment, FromMalloc));
Kostya Serebryany712fc982016-06-07 01:20:26 +0000624 if (Size == 0) {
625 Instance.deallocate(Ptr, 0, FromMalloc);
626 return nullptr;
627 }
Alex Shlyapnikov42bea012017-07-18 19:11:04 +0000628 return SetErrnoOnNull(Instance.reallocate(Ptr, Size));
Kostya Serebryany712fc982016-06-07 01:20:26 +0000629}
630
631void *scudoCalloc(uptr NMemB, uptr Size) {
Alex Shlyapnikov42bea012017-07-18 19:11:04 +0000632 return SetErrnoOnNull(Instance.calloc(NMemB, Size));
Kostya Serebryany712fc982016-06-07 01:20:26 +0000633}
634
635void *scudoValloc(uptr Size) {
Alex Shlyapnikov42bea012017-07-18 19:11:04 +0000636 return SetErrnoOnNull(
637 Instance.allocate(Size, GetPageSizeCached(), FromMemalign));
Kostya Serebryany712fc982016-06-07 01:20:26 +0000638}
639
Kostya Serebryany712fc982016-06-07 01:20:26 +0000640void *scudoPvalloc(uptr Size) {
641 uptr PageSize = GetPageSizeCached();
Kostya Kortchinsky65fdf672017-07-25 21:18:02 +0000642 if (UNLIKELY(CheckForPvallocOverflow(Size, PageSize))) {
643 errno = errno_ENOMEM;
Kostya Kortchinsky26e689f2017-09-14 20:34:32 +0000644 return Instance.handleBadRequest();
Kostya Kortchinsky65fdf672017-07-25 21:18:02 +0000645 }
Alex Shlyapnikovdf18cbb2017-07-14 21:17:16 +0000646 // pvalloc(0) should allocate one page.
647 Size = Size ? RoundUpTo(Size, PageSize) : PageSize;
Alex Shlyapnikov42bea012017-07-18 19:11:04 +0000648 return SetErrnoOnNull(Instance.allocate(Size, PageSize, FromMemalign));
Kostya Serebryany712fc982016-06-07 01:20:26 +0000649}
650
Kostya Kortchinsky0ce49992017-06-29 16:45:20 +0000651void *scudoMemalign(uptr Alignment, uptr Size) {
Alex Shlyapnikovdf18cbb2017-07-14 21:17:16 +0000652 if (UNLIKELY(!IsPowerOfTwo(Alignment))) {
653 errno = errno_EINVAL;
Kostya Kortchinsky26e689f2017-09-14 20:34:32 +0000654 return Instance.handleBadRequest();
Alex Shlyapnikovdf18cbb2017-07-14 21:17:16 +0000655 }
Alex Shlyapnikov42bea012017-07-18 19:11:04 +0000656 return SetErrnoOnNull(Instance.allocate(Size, Alignment, FromMemalign));
Kostya Kortchinsky0ce49992017-06-29 16:45:20 +0000657}
658
Kostya Serebryany712fc982016-06-07 01:20:26 +0000659int scudoPosixMemalign(void **MemPtr, uptr Alignment, uptr Size) {
Alex Shlyapnikov42bea012017-07-18 19:11:04 +0000660 if (UNLIKELY(!CheckPosixMemalignAlignment(Alignment))) {
Kostya Kortchinsky26e689f2017-09-14 20:34:32 +0000661 Instance.handleBadRequest();
Alex Shlyapnikovdf18cbb2017-07-14 21:17:16 +0000662 return errno_EINVAL;
Kostya Kortchinsky0ce49992017-06-29 16:45:20 +0000663 }
Alex Shlyapnikovdf18cbb2017-07-14 21:17:16 +0000664 void *Ptr = Instance.allocate(Size, Alignment, FromMemalign);
Alex Shlyapnikov42bea012017-07-18 19:11:04 +0000665 if (UNLIKELY(!Ptr))
Alex Shlyapnikovdf18cbb2017-07-14 21:17:16 +0000666 return errno_ENOMEM;
667 *MemPtr = Ptr;
Kostya Serebryany712fc982016-06-07 01:20:26 +0000668 return 0;
669}
670
671void *scudoAlignedAlloc(uptr Alignment, uptr Size) {
Alex Shlyapnikov42bea012017-07-18 19:11:04 +0000672 if (UNLIKELY(!CheckAlignedAllocAlignmentAndSize(Alignment, Size))) {
Alex Shlyapnikovdf18cbb2017-07-14 21:17:16 +0000673 errno = errno_EINVAL;
Kostya Kortchinsky26e689f2017-09-14 20:34:32 +0000674 return Instance.handleBadRequest();
Alex Shlyapnikovdf18cbb2017-07-14 21:17:16 +0000675 }
Alex Shlyapnikov42bea012017-07-18 19:11:04 +0000676 return SetErrnoOnNull(Instance.allocate(Size, Alignment, FromMalloc));
Kostya Serebryany712fc982016-06-07 01:20:26 +0000677}
678
679uptr scudoMallocUsableSize(void *Ptr) {
680 return Instance.getUsableSize(Ptr);
681}
682
Kostya Kortchinsky1148dc52016-11-30 17:32:20 +0000683} // namespace __scudo
Kostya Serebryany712fc982016-06-07 01:20:26 +0000684
685using namespace __scudo;
686
687// MallocExtension helper functions
688
689uptr __sanitizer_get_current_allocated_bytes() {
Kostya Kortchinsky8d6257b2017-02-03 20:49:42 +0000690 return Instance.getStats(AllocatorStatAllocated);
Kostya Serebryany712fc982016-06-07 01:20:26 +0000691}
692
693uptr __sanitizer_get_heap_size() {
Kostya Kortchinsky8d6257b2017-02-03 20:49:42 +0000694 return Instance.getStats(AllocatorStatMapped);
Kostya Serebryany712fc982016-06-07 01:20:26 +0000695}
696
697uptr __sanitizer_get_free_bytes() {
698 return 1;
699}
700
701uptr __sanitizer_get_unmapped_bytes() {
702 return 1;
703}
704
705uptr __sanitizer_get_estimated_allocated_size(uptr size) {
706 return size;
707}
708
Kostya Kortchinsky1148dc52016-11-30 17:32:20 +0000709int __sanitizer_get_ownership(const void *Ptr) {
710 return Instance.isValidPointer(Ptr);
Kostya Serebryany712fc982016-06-07 01:20:26 +0000711}
712
Kostya Kortchinsky1148dc52016-11-30 17:32:20 +0000713uptr __sanitizer_get_allocated_size(const void *Ptr) {
714 return Instance.getUsableSize(Ptr);
Kostya Serebryany712fc982016-06-07 01:20:26 +0000715}