[automerger skipped] [scudo] Rework dieOnMapUnmapError am: 4d022f5d4f -s ours
am skip reason: Merged-In Idc91a4187ddc9ad1aef8ab85ca021a6e2cc65566 with SHA-1 c79ab1b2c1 is already in history
Original change: https://googleplex-android-review.googlesource.com/c/platform/external/scudo/+/14967144
Change-Id: I76aad8f73e3f1c45fe1d67fb18167831a6182449
diff --git a/standalone/allocator_config.h b/standalone/allocator_config.h
index 8e103f2..7f55636 100644
--- a/standalone/allocator_config.h
+++ b/standalone/allocator_config.h
@@ -40,6 +40,12 @@
// // eg: Ptr = Base + (CompactPtr << Scale).
// typedef u32 PrimaryCompactPtrT;
// static const uptr PrimaryCompactPtrScale = SCUDO_MIN_ALIGNMENT_LOG;
+// // Indicates support for offsetting the start of a region by
+// // a random number of pages. Only used with primary64.
+// static const bool PrimaryEnableRandomOffset = true;
+// // Call map for user memory with at least this size. Only used with
+// // primary64.
+// static const uptr PrimaryMapSizeIncrement = 1UL << 18;
// // Defines the minimal & maximal release interval that can be set.
// static const s32 PrimaryMinReleaseToOsIntervalMs = INT32_MIN;
// static const s32 PrimaryMaxReleaseToOsIntervalMs = INT32_MAX;
@@ -61,6 +67,8 @@
static const uptr PrimaryRegionSizeLog = 32U;
typedef uptr PrimaryCompactPtrT;
static const uptr PrimaryCompactPtrScale = 0;
+ static const bool PrimaryEnableRandomOffset = true;
+ static const uptr PrimaryMapSizeIncrement = 1UL << 18;
#else
typedef SizeClassAllocator32<DefaultConfig> Primary;
static const uptr PrimaryRegionSizeLog = 19U;
@@ -89,6 +97,8 @@
static const uptr PrimaryRegionSizeLog = 28U;
typedef u32 PrimaryCompactPtrT;
static const uptr PrimaryCompactPtrScale = SCUDO_MIN_ALIGNMENT_LOG;
+ static const bool PrimaryEnableRandomOffset = true;
+ static const uptr PrimaryMapSizeIncrement = 1UL << 18;
#else
typedef SizeClassAllocator32<AndroidConfig> Primary;
static const uptr PrimaryRegionSizeLog = 18U;
@@ -118,6 +128,8 @@
static const uptr PrimaryRegionSizeLog = 27U;
typedef u32 PrimaryCompactPtrT;
static const uptr PrimaryCompactPtrScale = SCUDO_MIN_ALIGNMENT_LOG;
+ static const bool PrimaryEnableRandomOffset = true;
+ static const uptr PrimaryMapSizeIncrement = 1UL << 18;
#else
typedef SizeClassAllocator32<AndroidSvelteConfig> Primary;
static const uptr PrimaryRegionSizeLog = 16U;
@@ -140,12 +152,14 @@
#if SCUDO_CAN_USE_PRIMARY64
struct FuchsiaConfig {
- using SizeClassMap = DefaultSizeClassMap;
+ using SizeClassMap = FuchsiaSizeClassMap;
static const bool MaySupportMemoryTagging = false;
typedef SizeClassAllocator64<FuchsiaConfig> Primary;
static const uptr PrimaryRegionSizeLog = 30U;
typedef u32 PrimaryCompactPtrT;
+ static const bool PrimaryEnableRandomOffset = true;
+ static const uptr PrimaryMapSizeIncrement = 1UL << 18;
static const uptr PrimaryCompactPtrScale = SCUDO_MIN_ALIGNMENT_LOG;
static const s32 PrimaryMinReleaseToOsIntervalMs = INT32_MIN;
static const s32 PrimaryMaxReleaseToOsIntervalMs = INT32_MAX;
@@ -154,12 +168,34 @@
template <class A>
using TSDRegistryT = TSDRegistrySharedT<A, 8U, 4U>; // Shared, max 8 TSDs.
};
+
+struct TrustyConfig {
+ using SizeClassMap = TrustySizeClassMap;
+ static const bool MaySupportMemoryTagging = false;
+
+ typedef SizeClassAllocator64<TrustyConfig> Primary;
+ // Some apps have 1 page of heap total so small regions are necessary.
+ static const uptr PrimaryRegionSizeLog = 10U;
+ typedef u32 PrimaryCompactPtrT;
+ static const bool PrimaryEnableRandomOffset = false;
+ // Trusty is extremely memory-constrained so minimally round up map calls.
+ static const uptr PrimaryMapSizeIncrement = 1UL << 4;
+ static const uptr PrimaryCompactPtrScale = SCUDO_MIN_ALIGNMENT_LOG;
+ static const s32 PrimaryMinReleaseToOsIntervalMs = INT32_MIN;
+ static const s32 PrimaryMaxReleaseToOsIntervalMs = INT32_MAX;
+
+ typedef MapAllocatorNoCache SecondaryCache;
+ template <class A>
+ using TSDRegistryT = TSDRegistrySharedT<A, 1U, 1U>; // Shared, max 1 TSD.
+};
#endif
#if SCUDO_ANDROID
typedef AndroidConfig Config;
#elif SCUDO_FUCHSIA
typedef FuchsiaConfig Config;
+#elif SCUDO_TRUSTY
+typedef TrustyConfig Config;
#else
typedef DefaultConfig Config;
#endif
diff --git a/standalone/benchmarks/malloc_benchmark.cpp b/standalone/benchmarks/malloc_benchmark.cpp
index 661fff4..2adec88 100644
--- a/standalone/benchmarks/malloc_benchmark.cpp
+++ b/standalone/benchmarks/malloc_benchmark.cpp
@@ -29,7 +29,6 @@
std::unique_ptr<AllocatorT, decltype(Deleter)> Allocator(new AllocatorT,
Deleter);
CurrentAllocator = Allocator.get();
- Allocator->reset();
const size_t NBytes = State.range(0);
size_t PageSize = scudo::getPageSizeCached();
@@ -70,7 +69,6 @@
std::unique_ptr<AllocatorT, decltype(Deleter)> Allocator(new AllocatorT,
Deleter);
CurrentAllocator = Allocator.get();
- Allocator->reset();
const size_t NumIters = State.range(0);
size_t PageSize = scudo::getPageSizeCached();
diff --git a/standalone/bytemap.h b/standalone/bytemap.h
index e0d54f4..248e096 100644
--- a/standalone/bytemap.h
+++ b/standalone/bytemap.h
@@ -17,10 +17,9 @@
template <uptr Size> class FlatByteMap {
public:
- void initLinkerInitialized() {}
- void init() { memset(Map, 0, sizeof(Map)); }
+ void init() { DCHECK(Size == 0 || Map[0] == 0); }
- void unmapTestOnly() {}
+ void unmapTestOnly() { memset(Map, 0, Size); }
void set(uptr Index, u8 Value) {
DCHECK_LT(Index, Size);
@@ -36,7 +35,7 @@
void enable() {}
private:
- u8 Map[Size];
+ u8 Map[Size] = {};
};
} // namespace scudo
diff --git a/standalone/combined.h b/standalone/combined.h
index 8080d67..079edab 100644
--- a/standalone/combined.h
+++ b/standalone/combined.h
@@ -132,7 +132,7 @@
typedef GlobalQuarantine<QuarantineCallback, void> QuarantineT;
typedef typename QuarantineT::CacheT QuarantineCacheT;
- void initLinkerInitialized() {
+ void init() {
performSanityChecks();
// Check if hardware CRC32 is supported in the binary and by the platform,
@@ -166,11 +166,10 @@
QuarantineMaxChunkSize =
static_cast<u32>(getFlags()->quarantine_max_chunk_size);
- Stats.initLinkerInitialized();
+ Stats.init();
const s32 ReleaseToOsIntervalMs = getFlags()->release_to_os_interval_ms;
- Primary.initLinkerInitialized(ReleaseToOsIntervalMs);
- Secondary.initLinkerInitialized(&Stats, ReleaseToOsIntervalMs);
-
+ Primary.init(ReleaseToOsIntervalMs);
+ Secondary.init(&Stats, ReleaseToOsIntervalMs);
Quarantine.init(
static_cast<uptr>(getFlags()->quarantine_size_kb << 10),
static_cast<uptr>(getFlags()->thread_local_quarantine_size_kb << 10));
@@ -210,11 +209,10 @@
TSDRegistry.initThreadMaybe(this, MinimalInit);
}
- void reset() { memset(this, 0, sizeof(*this)); }
-
void unmapTestOnly() {
- TSDRegistry.unmapTestOnly();
+ TSDRegistry.unmapTestOnly(this);
Primary.unmapTestOnly();
+ Secondary.unmapTestOnly();
#ifdef GWP_ASAN_HOOKS
if (getFlags()->GWP_ASAN_InstallSignalHandlers)
gwp_asan::segv_handler::uninstallSignalHandlers();
@@ -225,9 +223,7 @@
TSDRegistryT *getTSDRegistry() { return &TSDRegistry; }
// The Cache must be provided zero-initialized.
- void initCache(CacheT *Cache) {
- Cache->initLinkerInitialized(&Stats, &Primary);
- }
+ void initCache(CacheT *Cache) { Cache->init(&Stats, &Primary); }
// Release the resources used by a TSD, which involves:
// - draining the local quarantine cache to the global quarantine;
@@ -698,7 +694,7 @@
// function. This can be called with a null buffer or zero size for buffer
// sizing purposes.
uptr getStats(char *Buffer, uptr Size) {
- ScopedString Str(1024);
+ ScopedString Str;
disable();
const uptr Length = getStats(&Str) + 1;
enable();
@@ -712,7 +708,7 @@
}
void printStats() {
- ScopedString Str(1024);
+ ScopedString Str;
disable();
getStats(&Str);
enable();
@@ -731,6 +727,8 @@
void iterateOverChunks(uptr Base, uptr Size, iterate_callback Callback,
void *Arg) {
initThreadMaybe();
+ if (archSupportsMemoryTagging())
+ Base = untagPointer(Base);
const uptr From = Base;
const uptr To = Base + Size;
bool MayHaveTaggedPrimary = allocatorSupportsMemoryTagging<Params>() &&
diff --git a/standalone/common.h b/standalone/common.h
index 3f27a3d..bc3dfec 100644
--- a/standalone/common.h
+++ b/standalone/common.h
@@ -13,6 +13,7 @@
#include "fuchsia.h"
#include "linux.h"
+#include "trusty.h"
#include <stddef.h>
#include <string.h>
diff --git a/standalone/flags.inc b/standalone/flags.inc
index b5cab47..690d889 100644
--- a/standalone/flags.inc
+++ b/standalone/flags.inc
@@ -37,12 +37,6 @@
SCUDO_FLAG(bool, pattern_fill_contents, false,
"Pattern fill chunk contents on allocation.")
-SCUDO_FLAG(int, rss_limit_mb, -1,
- "Enforce an upper limit (in megabytes) to the process RSS. The "
- "allocator will terminate or return NULL when allocations are "
- "attempted past that limit (depending on may_return_null). Negative "
- "values disable the feature.")
-
SCUDO_FLAG(bool, may_return_null, true,
"Indicate whether the allocator should terminate instead of "
"returning NULL in otherwise non-fatal error scenarios, eg: OOM, "
diff --git a/standalone/internal_defs.h b/standalone/internal_defs.h
index bbf7631..c9ffad1 100644
--- a/standalone/internal_defs.h
+++ b/standalone/internal_defs.h
@@ -105,14 +105,11 @@
void NORETURN reportCheckFailed(const char *File, int Line,
const char *Condition, u64 Value1, u64 Value2);
-
#define CHECK_IMPL(C1, Op, C2) \
do { \
- scudo::u64 V1 = (scudo::u64)(C1); \
- scudo::u64 V2 = (scudo::u64)(C2); \
- if (UNLIKELY(!(V1 Op V2))) { \
- scudo::reportCheckFailed(__FILE__, __LINE__, \
- "(" #C1 ") " #Op " (" #C2 ")", V1, V2); \
+ if (UNLIKELY(!(C1 Op C2))) { \
+ scudo::reportCheckFailed(__FILE__, __LINE__, #C1 " " #Op " " #C2, \
+ (scudo::u64)C1, (scudo::u64)C2); \
scudo::die(); \
} \
} while (false)
diff --git a/standalone/linux.cpp b/standalone/linux.cpp
index 301bdcd..dedab61 100644
--- a/standalone/linux.cpp
+++ b/standalone/linux.cpp
@@ -10,7 +10,6 @@
#if SCUDO_LINUX
-#include "atomic_helpers.h"
#include "common.h"
#include "linux.h"
#include "mutex.h"
@@ -90,41 +89,10 @@
dieOnMapUnmapError();
}
-static bool madviseNeedsMemset() {
- const uptr Size = getPageSizeCached();
- char *P = reinterpret_cast<char *>(mmap(0, Size, PROT_READ | PROT_WRITE,
- MAP_PRIVATE | MAP_ANONYMOUS, -1, 0));
- if (!P)
- dieOnMapUnmapError(errno == ENOMEM ? Size : 0);
- *P = 1;
- while (madvise(P, Size, MADV_DONTNEED) == -1 && errno == EAGAIN) {
- }
- const bool R = (*P != 0);
- if (munmap(P, Size) != 0)
- dieOnMapUnmapError();
- return R;
-}
-
-static bool madviseNeedsMemsetCached() {
- static atomic_u8 Cache;
- enum State : u8 { Unknown = 0, Yes = 1, No = 2 };
- State NeedsMemset = static_cast<State>(atomic_load_relaxed(&Cache));
- if (NeedsMemset == Unknown) {
- NeedsMemset = madviseNeedsMemset() ? Yes : No;
- atomic_store_relaxed(&Cache, NeedsMemset);
- }
- return NeedsMemset == Yes;
-}
-
void releasePagesToOS(uptr BaseAddress, uptr Offset, uptr Size,
UNUSED MapPlatformData *Data) {
void *Addr = reinterpret_cast<void *>(BaseAddress + Offset);
- if (madviseNeedsMemsetCached()) {
- // Workaround for QEMU-user ignoring MADV_DONTNEED.
- // https://github.com/qemu/qemu/blob/b1cffefa1b163bce9aebc3416f562c1d3886eeaa/linux-user/syscall.c#L11941
- // https://bugs.launchpad.net/qemu/+bug/1926521
- memset(Addr, 0, Size);
- }
+
while (madvise(Addr, Size, MADV_DONTNEED) == -1 && errno == EAGAIN) {
}
}
diff --git a/standalone/local_cache.h b/standalone/local_cache.h
index 5003937..f46645f 100644
--- a/standalone/local_cache.h
+++ b/standalone/local_cache.h
@@ -49,18 +49,14 @@
CompactPtrT Batch[MaxNumCached];
};
- void initLinkerInitialized(GlobalStats *S, SizeClassAllocator *A) {
- Stats.initLinkerInitialized();
+ void init(GlobalStats *S, SizeClassAllocator *A) {
+ DCHECK(isEmpty());
+ Stats.init();
if (LIKELY(S))
S->link(&Stats);
Allocator = A;
}
- void init(GlobalStats *S, SizeClassAllocator *A) {
- memset(this, 0, sizeof(*this));
- initLinkerInitialized(S, A);
- }
-
void destroy(GlobalStats *S) {
drain();
if (LIKELY(S))
diff --git a/standalone/memtag.h b/standalone/memtag.h
index 4bdce16..0c47f67 100644
--- a/standalone/memtag.h
+++ b/standalone/memtag.h
@@ -67,15 +67,27 @@
}
inline bool systemDetectsMemoryTagFaultsTestOnly() {
+#ifndef PR_SET_TAGGED_ADDR_CTRL
+#define PR_SET_TAGGED_ADDR_CTRL 54
+#endif
#ifndef PR_GET_TAGGED_ADDR_CTRL
#define PR_GET_TAGGED_ADDR_CTRL 56
#endif
+#ifndef PR_TAGGED_ADDR_ENABLE
+#define PR_TAGGED_ADDR_ENABLE (1UL << 0)
+#endif
#ifndef PR_MTE_TCF_SHIFT
#define PR_MTE_TCF_SHIFT 1
#endif
+#ifndef PR_MTE_TAG_SHIFT
+#define PR_MTE_TAG_SHIFT 3
+#endif
#ifndef PR_MTE_TCF_NONE
#define PR_MTE_TCF_NONE (0UL << PR_MTE_TCF_SHIFT)
#endif
+#ifndef PR_MTE_TCF_SYNC
+#define PR_MTE_TCF_SYNC (1UL << PR_MTE_TCF_SHIFT)
+#endif
#ifndef PR_MTE_TCF_MASK
#define PR_MTE_TCF_MASK (3UL << PR_MTE_TCF_SHIFT)
#endif
@@ -84,32 +96,28 @@
PR_MTE_TCF_MASK) != PR_MTE_TCF_NONE;
}
+inline void enableSystemMemoryTaggingTestOnly() {
+ prctl(PR_SET_TAGGED_ADDR_CTRL,
+ PR_TAGGED_ADDR_ENABLE | PR_MTE_TCF_SYNC | (0xfffe << PR_MTE_TAG_SHIFT),
+ 0, 0, 0);
+}
+
#else // !SCUDO_LINUX
inline bool systemSupportsMemoryTagging() { return false; }
-inline bool systemDetectsMemoryTagFaultsTestOnly() { return false; }
+inline bool systemDetectsMemoryTagFaultsTestOnly() {
+ UNREACHABLE("memory tagging not supported");
+}
+
+inline void enableSystemMemoryTaggingTestOnly() {
+ UNREACHABLE("memory tagging not supported");
+}
#endif // SCUDO_LINUX
-inline void disableMemoryTagChecksTestOnly() {
- __asm__ __volatile__(
- R"(
- .arch_extension memtag
- msr tco, #1
- )");
-}
-
-inline void enableMemoryTagChecksTestOnly() {
- __asm__ __volatile__(
- R"(
- .arch_extension memtag
- msr tco, #0
- )");
-}
-
class ScopedDisableMemoryTagChecks {
- size_t PrevTCO;
+ uptr PrevTCO;
public:
ScopedDisableMemoryTagChecks() {
@@ -134,6 +142,7 @@
};
inline uptr selectRandomTag(uptr Ptr, uptr ExcludeMask) {
+ ExcludeMask |= 1; // Always exclude Tag 0.
uptr TaggedPtr;
__asm__ __volatile__(
R"(
@@ -145,10 +154,14 @@
return TaggedPtr;
}
-inline uptr addFixedTag(uptr Ptr, uptr Tag) { return Ptr | (Tag << 56); }
+inline uptr addFixedTag(uptr Ptr, uptr Tag) {
+ DCHECK_LT(Tag, 16);
+ DCHECK_EQ(untagPointer(Ptr), Ptr);
+ return Ptr | (Tag << 56);
+}
inline uptr storeTags(uptr Begin, uptr End) {
- DCHECK(Begin % 16 == 0);
+ DCHECK_EQ(0, Begin % 16);
uptr LineSize, Next, Tmp;
__asm__ __volatile__(
R"(
@@ -208,10 +221,12 @@
[Tmp] "=&r"(Tmp)
: [End] "r"(End)
: "memory");
+ DCHECK_EQ(0, Begin % 16);
return Begin;
}
inline void storeTag(uptr Ptr) {
+ DCHECK_EQ(0, Ptr % 16);
__asm__ __volatile__(R"(
.arch_extension memtag
stg %0, [%0]
@@ -222,6 +237,7 @@
}
inline uptr loadTag(uptr Ptr) {
+ DCHECK_EQ(0, Ptr % 16);
uptr TaggedPtr = Ptr;
__asm__ __volatile__(
R"(
@@ -244,11 +260,7 @@
UNREACHABLE("memory tagging not supported");
}
-inline void disableMemoryTagChecksTestOnly() {
- UNREACHABLE("memory tagging not supported");
-}
-
-inline void enableMemoryTagChecksTestOnly() {
+inline void enableSystemMemoryTaggingTestOnly() {
UNREACHABLE("memory tagging not supported");
}
diff --git a/standalone/mutex.h b/standalone/mutex.h
index a654d35..c8504c0 100644
--- a/standalone/mutex.h
+++ b/standalone/mutex.h
@@ -22,7 +22,6 @@
class HybridMutex {
public:
- void init() { M = {}; }
bool tryLock();
NOINLINE void lock() {
if (LIKELY(tryLock()))
diff --git a/standalone/platform.h b/standalone/platform.h
index a4c2a0b..36378d1 100644
--- a/standalone/platform.h
+++ b/standalone/platform.h
@@ -12,7 +12,7 @@
// Transitive includes of stdint.h specify some of the defines checked below.
#include <stdint.h>
-#if defined(__linux__)
+#if defined(__linux__) && !defined(__TRUSTY__)
#define SCUDO_LINUX 1
#else
#define SCUDO_LINUX 0
@@ -31,6 +31,12 @@
#define SCUDO_FUCHSIA 0
#endif
+#if defined(__TRUSTY__)
+#define SCUDO_TRUSTY 1
+#else
+#define SCUDO_TRUSTY 0
+#endif
+
#if __LP64__
#define SCUDO_WORDSIZE 64U
#else
diff --git a/standalone/primary32.h b/standalone/primary32.h
index 33d8175..36ae083 100644
--- a/standalone/primary32.h
+++ b/standalone/primary32.h
@@ -60,11 +60,14 @@
static bool canAllocate(uptr Size) { return Size <= SizeClassMap::MaxSize; }
- void initLinkerInitialized(s32 ReleaseToOsInterval) {
+ void init(s32 ReleaseToOsInterval) {
if (SCUDO_FUCHSIA)
reportError("SizeClassAllocator32 is not supported on Fuchsia");
- PossibleRegions.initLinkerInitialized();
+ if (SCUDO_TRUSTY)
+ reportError("SizeClassAllocator32 is not supported on Trusty");
+
+ PossibleRegions.init();
u32 Seed;
const u64 Time = getMonotonicTime();
@@ -80,10 +83,6 @@
}
setOption(Option::ReleaseInterval, static_cast<sptr>(ReleaseToOsInterval));
}
- void init(s32 ReleaseToOsInterval) {
- memset(this, 0, sizeof(*this));
- initLinkerInitialized(ReleaseToOsInterval);
- }
void unmapTestOnly() {
while (NumberOfStashedRegions > 0)
@@ -96,6 +95,7 @@
MinRegionIndex = Sci->MinRegionIndex;
if (Sci->MaxRegionIndex > MaxRegionIndex)
MaxRegionIndex = Sci->MaxRegionIndex;
+ *Sci = {};
}
for (uptr I = MinRegionIndex; I < MaxRegionIndex; I++)
if (PossibleRegions[I])
diff --git a/standalone/primary64.h b/standalone/primary64.h
index 94375fc..27634c9 100644
--- a/standalone/primary64.h
+++ b/standalone/primary64.h
@@ -25,8 +25,9 @@
//
// It starts by reserving NumClasses * 2^RegionSizeLog bytes, equally divided in
// Regions, specific to each size class. Note that the base of that mapping is
-// random (based to the platform specific map() capabilities), and that each
-// Region actually starts at a random offset from its base.
+// random (based to the platform specific map() capabilities). If
+// PrimaryEnableRandomOffset is set, each Region actually starts at a random
+// offset from its base.
//
// Regions are mapped incrementally on demand to fulfill allocation requests,
// those mappings being split into equally sized Blocks based on the size class
@@ -57,7 +58,8 @@
static bool canAllocate(uptr Size) { return Size <= SizeClassMap::MaxSize; }
- void initLinkerInitialized(s32 ReleaseToOsInterval) {
+ void init(s32 ReleaseToOsInterval) {
+ DCHECK_EQ(PrimaryBase, 0U);
// Reserve the space required for the Primary.
PrimaryBase = reinterpret_cast<uptr>(
map(nullptr, PrimarySize, nullptr, MAP_NOACCESS, &Data));
@@ -69,21 +71,25 @@
const uptr PageSize = getPageSizeCached();
for (uptr I = 0; I < NumClasses; I++) {
RegionInfo *Region = getRegionInfo(I);
- // The actual start of a region is offseted by a random number of pages.
- Region->RegionBeg =
- getRegionBaseByClassId(I) + (getRandomModN(&Seed, 16) + 1) * PageSize;
+ // The actual start of a region is offset by a random number of pages
+ // when PrimaryEnableRandomOffset is set.
+ Region->RegionBeg = getRegionBaseByClassId(I) +
+ (Config::PrimaryEnableRandomOffset
+ ? ((getRandomModN(&Seed, 16) + 1) * PageSize)
+ : 0);
Region->RandState = getRandomU32(&Seed);
Region->ReleaseInfo.LastReleaseAtNs = Time;
}
setOption(Option::ReleaseInterval, static_cast<sptr>(ReleaseToOsInterval));
}
- void init(s32 ReleaseToOsInterval) {
- memset(this, 0, sizeof(*this));
- initLinkerInitialized(ReleaseToOsInterval);
- }
void unmapTestOnly() {
+ for (uptr I = 0; I < NumClasses; I++) {
+ RegionInfo *Region = getRegionInfo(I);
+ *Region = {};
+ }
unmap(reinterpret_cast<void *>(PrimaryBase), PrimarySize, UNMAP_ALL, &Data);
+ PrimaryBase = 0U;
}
TransferBatch *popBatch(CacheT *C, uptr ClassId) {
@@ -265,8 +271,7 @@
static const uptr NumClasses = SizeClassMap::NumClasses;
static const uptr PrimarySize = RegionSize * NumClasses;
- // Call map for user memory with at least this size.
- static const uptr MapSizeIncrement = 1UL << 18;
+ static const uptr MapSizeIncrement = Config::PrimaryMapSizeIncrement;
// Fill at most this number of batches from the newly map'd memory.
static const u32 MaxNumBatches = SCUDO_ANDROID ? 4U : 8U;
@@ -339,7 +344,7 @@
if (UNLIKELY(RegionBase + MappedUser + MapSize > RegionSize)) {
if (!Region->Exhausted) {
Region->Exhausted = true;
- ScopedString Str(1024);
+ ScopedString Str;
getStats(&Str);
Str.append(
"Scudo OOM: The process has exhausted %zuM for size class %zu.\n",
diff --git a/standalone/quarantine.h b/standalone/quarantine.h
index 8d4b38e..84eb90c 100644
--- a/standalone/quarantine.h
+++ b/standalone/quarantine.h
@@ -64,11 +64,7 @@
// Per-thread cache of memory blocks.
template <typename Callback> class QuarantineCache {
public:
- void initLinkerInitialized() {}
- void init() {
- memset(this, 0, sizeof(*this));
- initLinkerInitialized();
- }
+ void init() { DCHECK_EQ(atomic_load_relaxed(&Size), 0U); }
// Total memory used, including internal accounting.
uptr getSize() const { return atomic_load_relaxed(&Size); }
@@ -175,7 +171,10 @@
public:
typedef QuarantineCache<Callback> CacheT;
- void initLinkerInitialized(uptr Size, uptr CacheSize) {
+ void init(uptr Size, uptr CacheSize) {
+ DCHECK_EQ(atomic_load_relaxed(&MaxSize), 0U);
+ DCHECK_EQ(atomic_load_relaxed(&MinSize), 0U);
+ DCHECK_EQ(atomic_load_relaxed(&MaxCacheSize), 0U);
// Thread local quarantine size can be zero only when global quarantine size
// is zero (it allows us to perform just one atomic read per put() call).
CHECK((Size == 0 && CacheSize == 0) || CacheSize != 0);
@@ -184,16 +183,7 @@
atomic_store_relaxed(&MinSize, Size / 10 * 9); // 90% of max size.
atomic_store_relaxed(&MaxCacheSize, CacheSize);
- Cache.initLinkerInitialized();
- }
- void init(uptr Size, uptr CacheSize) {
- CacheMutex.init();
Cache.init();
- RecycleMutex.init();
- MinSize = {};
- MaxSize = {};
- MaxCacheSize = {};
- initLinkerInitialized(Size, CacheSize);
}
uptr getMaxSize() const { return atomic_load_relaxed(&MaxSize); }
diff --git a/standalone/report.cpp b/standalone/report.cpp
index 80cc6ed..561c7c5 100644
--- a/standalone/report.cpp
+++ b/standalone/report.cpp
@@ -17,7 +17,7 @@
class ScopedErrorReport {
public:
- ScopedErrorReport() : Message(512) { Message.append("Scudo ERROR: "); }
+ ScopedErrorReport() : Message() { Message.append("Scudo ERROR: "); }
void append(const char *Format, ...) {
va_list Args;
va_start(Args, Format);
@@ -45,8 +45,8 @@
trap();
}
ScopedErrorReport Report;
- Report.append("CHECK failed @ %s:%d %s (%llu, %llu)\n", File, Line, Condition,
- Value1, Value2);
+ Report.append("CHECK failed @ %s:%d %s ((u64)op1=%llu, (u64)op2=%llu)\n",
+ File, Line, Condition, Value1, Value2);
}
// Generic string fatal error message.
diff --git a/standalone/secondary.h b/standalone/secondary.h
index ea5d680..630e64d 100644
--- a/standalone/secondary.h
+++ b/standalone/secondary.h
@@ -28,7 +28,10 @@
namespace LargeBlock {
-struct Header {
+struct alignas(Max<uptr>(archSupportsMemoryTagging()
+ ? archMemoryTagGranuleSize()
+ : 1,
+ 1U << SCUDO_MIN_ALIGNMENT_LOG)) Header {
LargeBlock::Header *Prev;
LargeBlock::Header *Next;
uptr CommitBase;
@@ -38,9 +41,12 @@
[[no_unique_address]] MapPlatformData Data;
};
-constexpr uptr getHeaderSize() {
- return roundUpTo(sizeof(Header), 1U << SCUDO_MIN_ALIGNMENT_LOG);
-}
+static_assert(sizeof(Header) % (1U << SCUDO_MIN_ALIGNMENT_LOG) == 0, "");
+static_assert(!archSupportsMemoryTagging() ||
+ sizeof(Header) % archMemoryTagGranuleSize() == 0,
+ "");
+
+constexpr uptr getHeaderSize() { return sizeof(Header); }
template <typename Config> static uptr addHeaderTag(uptr Ptr) {
if (allocatorSupportsMemoryTagging<Config>())
@@ -49,8 +55,7 @@
}
template <typename Config> static Header *getHeader(uptr Ptr) {
- return reinterpret_cast<Header *>(addHeaderTag<Config>(Ptr) -
- getHeaderSize());
+ return reinterpret_cast<Header *>(addHeaderTag<Config>(Ptr)) - 1;
}
template <typename Config> static Header *getHeader(const void *Ptr) {
@@ -66,7 +71,6 @@
class MapAllocatorNoCache {
public:
- void initLinkerInitialized(UNUSED s32 ReleaseToOsInterval) {}
void init(UNUSED s32 ReleaseToOsInterval) {}
bool retrieve(UNUSED Options Options, UNUSED uptr Size, UNUSED uptr Alignment,
UNUSED LargeBlock::Header **H, UNUSED bool *Zeroed) {
@@ -78,6 +82,7 @@
void enable() {}
void releaseToOS() {}
void disableMemoryTagging() {}
+ void unmapTestOnly() {}
bool setOption(Option O, UNUSED sptr Value) {
if (O == Option::ReleaseInterval || O == Option::MaxCacheEntriesCount ||
O == Option::MaxCacheEntrySize)
@@ -115,17 +120,14 @@
Config::SecondaryCacheEntriesArraySize,
"");
- void initLinkerInitialized(s32 ReleaseToOsInterval) {
+ void init(s32 ReleaseToOsInterval) {
+ DCHECK_EQ(EntriesCount, 0U);
setOption(Option::MaxCacheEntriesCount,
static_cast<sptr>(Config::SecondaryCacheDefaultMaxEntriesCount));
setOption(Option::MaxCacheEntrySize,
static_cast<sptr>(Config::SecondaryCacheDefaultMaxEntrySize));
setOption(Option::ReleaseInterval, static_cast<sptr>(ReleaseToOsInterval));
}
- void init(s32 ReleaseToOsInterval) {
- memset(this, 0, sizeof(*this));
- initLinkerInitialized(ReleaseToOsInterval);
- }
void store(Options Options, LargeBlock::Header *H) {
if (!canCache(H->CommitSize))
@@ -321,6 +323,8 @@
void enable() { Mutex.unlock(); }
+ void unmapTestOnly() { empty(); }
+
private:
void empty() {
struct {
@@ -396,16 +400,14 @@
template <typename Config> class MapAllocator {
public:
- void initLinkerInitialized(GlobalStats *S, s32 ReleaseToOsInterval = -1) {
- Cache.initLinkerInitialized(ReleaseToOsInterval);
- Stats.initLinkerInitialized();
+ void init(GlobalStats *S, s32 ReleaseToOsInterval = -1) {
+ DCHECK_EQ(AllocatedBytes, 0U);
+ DCHECK_EQ(FreedBytes, 0U);
+ Cache.init(ReleaseToOsInterval);
+ Stats.init();
if (LIKELY(S))
S->link(&Stats);
}
- void init(GlobalStats *S, s32 ReleaseToOsInterval = -1) {
- memset(this, 0, sizeof(*this));
- initLinkerInitialized(S, ReleaseToOsInterval);
- }
void *allocate(Options Options, uptr Size, uptr AlignmentHint = 0,
uptr *BlockEnd = nullptr,
@@ -451,6 +453,8 @@
void disableMemoryTagging() { Cache.disableMemoryTagging(); }
+ void unmapTestOnly() { Cache.unmapTestOnly(); }
+
private:
typename Config::SecondaryCache Cache;
diff --git a/standalone/size_class_map.h b/standalone/size_class_map.h
index 1948802..ba0f784 100644
--- a/standalone/size_class_map.h
+++ b/standalone/size_class_map.h
@@ -64,12 +64,10 @@
static const u8 S = Config::NumBits - 1;
static const uptr M = (1UL << S) - 1;
- static const uptr SizeDelta = Chunk::getHeaderSize();
-
public:
static const u32 MaxNumCachedHint = Config::MaxNumCachedHint;
- static const uptr MaxSize = (1UL << Config::MaxSizeLog) + SizeDelta;
+ static const uptr MaxSize = (1UL << Config::MaxSizeLog) + Config::SizeDelta;
static const uptr NumClasses =
MidClass + ((Config::MaxSizeLog - Config::MidSizeLog) << S) + 1;
static_assert(NumClasses <= 256, "");
@@ -79,24 +77,22 @@
static uptr getSizeByClassId(uptr ClassId) {
DCHECK_NE(ClassId, BatchClassId);
if (ClassId <= MidClass)
- return (ClassId << Config::MinSizeLog) + SizeDelta;
+ return (ClassId << Config::MinSizeLog) + Config::SizeDelta;
ClassId -= MidClass;
const uptr T = MidSize << (ClassId >> S);
- return T + (T >> S) * (ClassId & M) + SizeDelta;
+ return T + (T >> S) * (ClassId & M) + Config::SizeDelta;
}
static u8 getSizeLSBByClassId(uptr ClassId) {
return u8(getLeastSignificantSetBitIndex(getSizeByClassId(ClassId)));
}
- static constexpr bool usesCompressedLSBFormat() {
- return false;
- }
+ static constexpr bool usesCompressedLSBFormat() { return false; }
static uptr getClassIdBySize(uptr Size) {
- if (Size <= SizeDelta + (1 << Config::MinSizeLog))
+ if (Size <= Config::SizeDelta + (1 << Config::MinSizeLog))
return 1;
- Size -= SizeDelta;
+ Size -= Config::SizeDelta;
DCHECK_LE(Size, MaxSize);
if (Size <= MidSize)
return (Size + MinSize - 1) >> Config::MinSizeLog;
@@ -227,12 +223,25 @@
static const uptr MinSizeLog = 5;
static const uptr MidSizeLog = 8;
static const uptr MaxSizeLog = 17;
- static const u32 MaxNumCachedHint = 10;
+ static const u32 MaxNumCachedHint = 14;
static const uptr MaxBytesCachedLog = 10;
+ static const uptr SizeDelta = 0;
};
typedef FixedSizeClassMap<DefaultSizeClassConfig> DefaultSizeClassMap;
+struct FuchsiaSizeClassConfig {
+ static const uptr NumBits = 3;
+ static const uptr MinSizeLog = 5;
+ static const uptr MidSizeLog = 8;
+ static const uptr MaxSizeLog = 17;
+ static const u32 MaxNumCachedHint = 10;
+ static const uptr MaxBytesCachedLog = 10;
+ static const uptr SizeDelta = Chunk::getHeaderSize();
+};
+
+typedef FixedSizeClassMap<FuchsiaSizeClassConfig> FuchsiaSizeClassMap;
+
struct AndroidSizeClassConfig {
#if SCUDO_WORDSIZE == 64U
static const uptr NumBits = 7;
@@ -285,6 +294,7 @@
static const uptr MaxSizeLog = 14;
static const u32 MaxNumCachedHint = 13;
static const uptr MaxBytesCachedLog = 10;
+ static const uptr SizeDelta = Chunk::getHeaderSize();
#else
static const uptr NumBits = 4;
static const uptr MinSizeLog = 3;
@@ -292,13 +302,28 @@
static const uptr MaxSizeLog = 14;
static const u32 MaxNumCachedHint = 14;
static const uptr MaxBytesCachedLog = 10;
+ static const uptr SizeDelta = Chunk::getHeaderSize();
#endif
};
typedef FixedSizeClassMap<SvelteSizeClassConfig> SvelteSizeClassMap;
+// Trusty is configured to only have one region containing blocks of size
+// 2^7 bytes.
+struct TrustySizeClassConfig {
+ static const uptr NumBits = 1;
+ static const uptr MinSizeLog = 7;
+ static const uptr MidSizeLog = 7;
+ static const uptr MaxSizeLog = 7;
+ static const u32 MaxNumCachedHint = 8;
+ static const uptr MaxBytesCachedLog = 10;
+ static const uptr SizeDelta = 0;
+};
+
+typedef FixedSizeClassMap<TrustySizeClassConfig> TrustySizeClassMap;
+
template <typename SCMap> inline void printMap() {
- ScopedString Buffer(1024);
+ ScopedString Buffer;
uptr PrevS = 0;
uptr TotalCached = 0;
for (uptr I = 0; I < SCMap::NumClasses; I++) {
diff --git a/standalone/stats.h b/standalone/stats.h
index e15c056..be5bf2d 100644
--- a/standalone/stats.h
+++ b/standalone/stats.h
@@ -29,8 +29,10 @@
// LocalStats::add'ing, this is OK, we will still get a meaningful number.
class LocalStats {
public:
- void initLinkerInitialized() {}
- void init() { memset(this, 0, sizeof(*this)); }
+ void init() {
+ for (uptr I = 0; I < StatCount; I++)
+ DCHECK_EQ(get(static_cast<StatType>(I)), 0U);
+ }
void add(StatType I, uptr V) {
V += atomic_load_relaxed(&StatsArray[I]);
@@ -56,13 +58,7 @@
// Global stats, used for aggregation and querying.
class GlobalStats : public LocalStats {
public:
- void initLinkerInitialized() {}
- void init() {
- LocalStats::init();
- Mutex.init();
- StatsList = {};
- initLinkerInitialized();
- }
+ void init() { LocalStats::init(); }
void link(LocalStats *S) {
ScopedLock L(Mutex);
diff --git a/standalone/string_utils.cpp b/standalone/string_utils.cpp
index 25bddbc..acf8588 100644
--- a/standalone/string_utils.cpp
+++ b/standalone/string_utils.cpp
@@ -219,7 +219,6 @@
}
void ScopedString::append(const char *Format, va_list Args) {
- DCHECK_LT(Length, String.size());
va_list ArgsCopy;
va_copy(ArgsCopy, Args);
// formatString doesn't currently support a null buffer or zero buffer length,
@@ -228,11 +227,13 @@
char C[1];
const uptr AdditionalLength =
static_cast<uptr>(formatString(C, sizeof(C), Format, Args)) + 1;
+ const uptr Length = length();
String.resize(Length + AdditionalLength);
- formatString(String.data() + Length, AdditionalLength, Format, ArgsCopy);
+ const uptr FormattedLength = static_cast<uptr>(formatString(
+ String.data() + Length, String.size() - Length, Format, ArgsCopy));
+ RAW_CHECK(data()[length()] == '\0');
+ RAW_CHECK(FormattedLength + 1 == AdditionalLength);
va_end(ArgsCopy);
- Length = strlen(String.data());
- CHECK_LT(Length, String.size());
}
FORMAT(2, 3)
@@ -247,7 +248,7 @@
void Printf(const char *Format, ...) {
va_list Args;
va_start(Args, Format);
- ScopedString Msg(1024);
+ ScopedString Msg;
Msg.append(Format, Args);
outputRaw(Msg.data());
va_end(Args);
diff --git a/standalone/string_utils.h b/standalone/string_utils.h
index 4880fa1..06d23d4 100644
--- a/standalone/string_utils.h
+++ b/standalone/string_utils.h
@@ -18,14 +18,12 @@
class ScopedString {
public:
- explicit ScopedString(uptr MaxLength) : String(MaxLength), Length(0) {
- String[0] = '\0';
- }
- uptr length() { return Length; }
+ explicit ScopedString() { String.push_back('\0'); }
+ uptr length() { return String.size() - 1; }
const char *data() { return String.data(); }
void clear() {
- String[0] = '\0';
- Length = 0;
+ String.clear();
+ String.push_back('\0');
}
void append(const char *Format, va_list Args);
void append(const char *Format, ...);
@@ -33,7 +31,6 @@
private:
Vector<char> String;
- uptr Length;
};
int formatString(char *Buffer, uptr BufferLength, const char *Format, ...);
diff --git a/standalone/tests/combined_test.cpp b/standalone/tests/combined_test.cpp
index 5db249d..6716d5d 100644
--- a/standalone/tests/combined_test.cpp
+++ b/standalone/tests/combined_test.cpp
@@ -6,6 +6,7 @@
//
//===----------------------------------------------------------------------===//
+#include "memtag.h"
#include "tests/scudo_unit_test.h"
#include "allocator_config.h"
@@ -68,7 +69,6 @@
template <typename Config> struct TestAllocator : scudo::Allocator<Config> {
TestAllocator() {
- this->reset();
this->initThreadMaybe();
if (scudo::archSupportsMemoryTagging() &&
!scudo::systemDetectsMemoryTagFaultsTestOnly())
@@ -97,7 +97,7 @@
void RunTest();
- void BasicTest(scudo::uptr SizeLogMin, scudo::uptr SizeLogMax);
+ void BasicTest(scudo::uptr SizeLog);
using AllocatorT = TestAllocator<TypeParam>;
std::unique_ptr<AllocatorT> Allocator;
@@ -141,37 +141,56 @@
}
template <class Config>
-void ScudoCombinedTest<Config>::BasicTest(scudo::uptr SizeLogMin,
- scudo::uptr SizeLogMax) {
+void ScudoCombinedTest<Config>::BasicTest(scudo::uptr SizeLog) {
auto *Allocator = this->Allocator.get();
// This allocates and deallocates a bunch of chunks, with a wide range of
// sizes and alignments, with a focus on sizes that could trigger weird
// behaviors (plus or minus a small delta of a power of two for example).
- for (scudo::uptr SizeLog = SizeLogMin; SizeLog <= SizeLogMax; SizeLog++) {
- for (scudo::uptr AlignLog = MinAlignLog; AlignLog <= 16U; AlignLog++) {
- const scudo::uptr Align = 1U << AlignLog;
- for (scudo::sptr Delta = -32; Delta <= 32; Delta++) {
- if (static_cast<scudo::sptr>(1U << SizeLog) + Delta <= 0)
- continue;
- const scudo::uptr Size = (1U << SizeLog) + Delta;
- void *P = Allocator->allocate(Size, Origin, Align);
- EXPECT_NE(P, nullptr);
- EXPECT_TRUE(Allocator->isOwned(P));
- EXPECT_TRUE(scudo::isAligned(reinterpret_cast<scudo::uptr>(P), Align));
- EXPECT_LE(Size, Allocator->getUsableSize(P));
- memset(P, 0xaa, Size);
- checkMemoryTaggingMaybe(Allocator, P, Size, Align);
- Allocator->deallocate(P, Origin, Size);
- }
+ for (scudo::uptr AlignLog = MinAlignLog; AlignLog <= 16U; AlignLog++) {
+ const scudo::uptr Align = 1U << AlignLog;
+ for (scudo::sptr Delta = -32; Delta <= 32; Delta++) {
+ if (static_cast<scudo::sptr>(1U << SizeLog) + Delta <= 0)
+ continue;
+ const scudo::uptr Size = (1U << SizeLog) + Delta;
+ void *P = Allocator->allocate(Size, Origin, Align);
+ EXPECT_NE(P, nullptr);
+ EXPECT_TRUE(Allocator->isOwned(P));
+ EXPECT_TRUE(scudo::isAligned(reinterpret_cast<scudo::uptr>(P), Align));
+ EXPECT_LE(Size, Allocator->getUsableSize(P));
+ memset(P, 0xaa, Size);
+ checkMemoryTaggingMaybe(Allocator, P, Size, Align);
+ Allocator->deallocate(P, Origin, Size);
}
}
}
-SCUDO_TYPED_TEST(ScudoCombinedTest, BasicCombined0) { this->BasicTest(0, 16); }
-SCUDO_TYPED_TEST(ScudoCombinedTest, BasicCombined1) { this->BasicTest(17, 18); }
-SCUDO_TYPED_TEST(ScudoCombinedTest, BasicCombined2) { this->BasicTest(19, 19); }
-SCUDO_TYPED_TEST(ScudoCombinedTest, BasicCombined3) { this->BasicTest(20, 20); }
+#define SCUDO_MAKE_BASIC_TEST(SizeLog) \
+ SCUDO_TYPED_TEST(ScudoCombinedTest, BasicCombined##SizeLog) { \
+ this->BasicTest(SizeLog); \
+ }
+
+SCUDO_MAKE_BASIC_TEST(0)
+SCUDO_MAKE_BASIC_TEST(1)
+SCUDO_MAKE_BASIC_TEST(2)
+SCUDO_MAKE_BASIC_TEST(3)
+SCUDO_MAKE_BASIC_TEST(4)
+SCUDO_MAKE_BASIC_TEST(5)
+SCUDO_MAKE_BASIC_TEST(6)
+SCUDO_MAKE_BASIC_TEST(7)
+SCUDO_MAKE_BASIC_TEST(8)
+SCUDO_MAKE_BASIC_TEST(9)
+SCUDO_MAKE_BASIC_TEST(10)
+SCUDO_MAKE_BASIC_TEST(11)
+SCUDO_MAKE_BASIC_TEST(12)
+SCUDO_MAKE_BASIC_TEST(13)
+SCUDO_MAKE_BASIC_TEST(14)
+SCUDO_MAKE_BASIC_TEST(15)
+SCUDO_MAKE_BASIC_TEST(16)
+SCUDO_MAKE_BASIC_TEST(17)
+SCUDO_MAKE_BASIC_TEST(18)
+SCUDO_MAKE_BASIC_TEST(19)
+SCUDO_MAKE_BASIC_TEST(20)
SCUDO_TYPED_TEST(ScudoCombinedTest, ZeroContents) {
auto *Allocator = this->Allocator.get();
@@ -193,7 +212,7 @@
SCUDO_TYPED_TEST(ScudoCombinedTest, ZeroFill) {
auto *Allocator = this->Allocator.get();
- // Ensure that specifying ZeroContents returns a zero'd out block.
+ // Ensure that specifying ZeroFill returns a zero'd out block.
Allocator->setFillContents(scudo::ZeroFill);
for (scudo::uptr SizeLog = 0U; SizeLog <= 20U; SizeLog++) {
for (scudo::uptr Delta = 0U; Delta <= 4U; Delta++) {
@@ -253,7 +272,28 @@
EXPECT_TRUE(Found);
}
-SCUDO_TYPED_TEST(ScudoCombinedTest, ReallocateLarge) {
+SCUDO_TYPED_TEST(ScudoCombinedTest, ReallocateLargeIncreasing) {
+ auto *Allocator = this->Allocator.get();
+
+ // Reallocate a chunk all the way up to a secondary allocation, verifying that
+ // we preserve the data in the process.
+ scudo::uptr Size = 16;
+ void *P = Allocator->allocate(Size, Origin);
+ const char Marker = 0xab;
+ memset(P, Marker, Size);
+ while (Size < TypeParam::Primary::SizeClassMap::MaxSize * 4) {
+ void *NewP = Allocator->reallocate(P, Size * 2);
+ EXPECT_NE(NewP, nullptr);
+ for (scudo::uptr J = 0; J < Size; J++)
+ EXPECT_EQ((reinterpret_cast<char *>(NewP))[J], Marker);
+ memset(reinterpret_cast<char *>(NewP) + Size, Marker, Size);
+ Size *= 2U;
+ P = NewP;
+ }
+ Allocator->deallocate(P, Origin);
+}
+
+SCUDO_TYPED_TEST(ScudoCombinedTest, ReallocateLargeDecreasing) {
auto *Allocator = this->Allocator.get();
// Reallocate a large chunk all the way down to a byte, verifying that we
@@ -359,7 +399,7 @@
// Check that disabling memory tagging works correctly.
void *P = Allocator->allocate(2048, Origin);
EXPECT_DEATH(reinterpret_cast<char *>(P)[2048] = 0xaa, "");
- scudo::disableMemoryTagChecksTestOnly();
+ scudo::ScopedDisableMemoryTagChecks NoTagChecks;
Allocator->disableMemoryTagging();
reinterpret_cast<char *>(P)[2048] = 0xaa;
Allocator->deallocate(P, Origin);
@@ -370,10 +410,6 @@
Allocator->deallocate(P, Origin);
Allocator->releaseToOS();
-
- // Disabling memory tag checks may interfere with subsequent tests.
- // Re-enable them now.
- scudo::enableMemoryTagChecksTestOnly();
}
}
@@ -452,12 +488,6 @@
Allocator->releaseToOS();
}
-#if SCUDO_FUCHSIA
-#define SKIP_ON_FUCHSIA(T) DISABLED_##T
-#else
-#define SKIP_ON_FUCHSIA(T) T
-#endif
-
// Test that multiple instantiations of the allocator have not messed up the
// process's signal handlers (GWP-ASan used to do this).
TEST(ScudoCombinedTest, SKIP_ON_FUCHSIA(testSEGV)) {
@@ -476,6 +506,7 @@
static const scudo::uptr MaxSizeLog = 13;
static const scudo::u32 MaxNumCachedHint = 4;
static const scudo::uptr MaxBytesCachedLog = 12;
+ static const scudo::uptr SizeDelta = 0;
};
static const scudo::uptr DeathRegionSizeLog = 20U;
@@ -490,6 +521,8 @@
static const scudo::s32 PrimaryMaxReleaseToOsIntervalMs = INT32_MAX;
typedef scudo::uptr PrimaryCompactPtrT;
static const scudo::uptr PrimaryCompactPtrScale = 0;
+ static const bool PrimaryEnableRandomOffset = true;
+ static const scudo::uptr PrimaryMapSizeIncrement = 1UL << 18;
typedef scudo::MapAllocatorNoCache SecondaryCache;
template <class A> using TSDRegistryT = scudo::TSDRegistrySharedT<A, 1U, 1U>;
@@ -528,15 +561,6 @@
EXPECT_DEATH(Allocator->getUsableSize(P), "");
}
-// Ensure that releaseToOS can be called prior to any other allocator
-// operation without issue.
-TEST(ScudoCombinedTest, ReleaseToOS) {
- using AllocatorT = TestAllocator<DeathConfig>;
- auto Allocator = std::unique_ptr<AllocatorT>(new AllocatorT());
-
- Allocator->releaseToOS();
-}
-
// Verify that when a region gets full, the allocator will still manage to
// fulfill the allocation through a larger size class.
TEST(ScudoCombinedTest, FullRegion) {
@@ -569,10 +593,15 @@
EXPECT_EQ(FailedAllocationsCount, 0U);
}
-TEST(ScudoCombinedTest, OddEven) {
- using AllocatorT = TestAllocator<scudo::AndroidConfig>;
- using SizeClassMap = AllocatorT::PrimaryT::SizeClassMap;
- auto Allocator = std::unique_ptr<AllocatorT>(new AllocatorT());
+// Ensure that releaseToOS can be called prior to any other allocator
+// operation without issue.
+SCUDO_TYPED_TEST(ScudoCombinedTest, ReleaseToOS) {
+ auto *Allocator = this->Allocator.get();
+ Allocator->releaseToOS();
+}
+
+SCUDO_TYPED_TEST(ScudoCombinedTest, OddEven) {
+ auto *Allocator = this->Allocator.get();
if (!Allocator->useMemoryTaggingTestOnly())
return;
@@ -583,6 +612,7 @@
EXPECT_NE(Tag1 % 2, Tag2 % 2);
};
+ using SizeClassMap = typename TypeParam::Primary::SizeClassMap;
for (scudo::uptr ClassId = 1U; ClassId <= SizeClassMap::LargestClassId;
ClassId++) {
const scudo::uptr Size = SizeClassMap::getSizeByClassId(ClassId);
@@ -608,10 +638,8 @@
}
}
-TEST(ScudoCombinedTest, DisableMemInit) {
- using AllocatorT = TestAllocator<scudo::AndroidConfig>;
- using SizeClassMap = AllocatorT::PrimaryT::SizeClassMap;
- auto Allocator = std::unique_ptr<AllocatorT>(new AllocatorT());
+SCUDO_TYPED_TEST(ScudoCombinedTest, DisableMemInit) {
+ auto *Allocator = this->Allocator.get();
std::vector<void *> Ptrs(65536, nullptr);
@@ -623,6 +651,7 @@
// expected. This is tricky to ensure when MTE is enabled, so this test tries
// to exercise the relevant code on our MTE path.
for (scudo::uptr ClassId = 1U; ClassId <= 8; ClassId++) {
+ using SizeClassMap = typename TypeParam::Primary::SizeClassMap;
const scudo::uptr Size =
SizeClassMap::getSizeByClassId(ClassId) - scudo::Chunk::getHeaderSize();
if (Size < 8)
diff --git a/standalone/tests/common_test.cpp b/standalone/tests/common_test.cpp
new file mode 100644
index 0000000..711e3b2
--- /dev/null
+++ b/standalone/tests/common_test.cpp
@@ -0,0 +1,72 @@
+//===-- common_test.cpp -----------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "internal_defs.h"
+#include "tests/scudo_unit_test.h"
+
+#include "common.h"
+#include <algorithm>
+#include <fstream>
+
+namespace scudo {
+
+static uptr getResidentMemorySize() {
+ if (!SCUDO_LINUX)
+ UNREACHABLE("Not implemented!");
+ uptr Size;
+ uptr Resident;
+ std::ifstream IFS("/proc/self/statm");
+ IFS >> Size;
+ IFS >> Resident;
+ return Resident * getPageSizeCached();
+}
+
+// Fuchsia needs getResidentMemorySize implementation.
+TEST(ScudoCommonTest, SKIP_ON_FUCHSIA(ResidentMemorySize)) {
+ uptr OnStart = getResidentMemorySize();
+ EXPECT_GT(OnStart, 0UL);
+
+ const uptr Size = 1ull << 30;
+ const uptr Threshold = Size >> 3;
+
+ MapPlatformData Data = {};
+ void *P = map(nullptr, Size, "ResidentMemorySize", 0, &Data);
+ ASSERT_NE(nullptr, P);
+ EXPECT_LT(getResidentMemorySize(), OnStart + Threshold);
+
+ memset(P, 1, Size);
+ EXPECT_GT(getResidentMemorySize(), OnStart + Size - Threshold);
+
+ releasePagesToOS((uptr)P, 0, Size, &Data);
+ EXPECT_LT(getResidentMemorySize(), OnStart + Threshold);
+
+ memset(P, 1, Size);
+ EXPECT_GT(getResidentMemorySize(), OnStart + Size - Threshold);
+
+ unmap(P, Size, 0, &Data);
+}
+
+TEST(ScudoCommonTest, Zeros) {
+ const uptr Size = 1ull << 20;
+
+ MapPlatformData Data = {};
+ uptr *P = reinterpret_cast<uptr *>(map(nullptr, Size, "Zeros", 0, &Data));
+ const ptrdiff_t N = Size / sizeof(*P);
+ ASSERT_NE(nullptr, P);
+ EXPECT_EQ(std::count(P, P + N, 0), N);
+
+ memset(P, 1, Size);
+ EXPECT_EQ(std::count(P, P + N, 0), 0);
+
+ releasePagesToOS((uptr)P, 0, Size, &Data);
+ EXPECT_EQ(std::count(P, P + N, 0), N);
+
+ unmap(P, Size, 0, &Data);
+}
+
+} // namespace scudo
diff --git a/standalone/tests/map_test.cpp b/standalone/tests/map_test.cpp
index 7c40b73..149a704 100644
--- a/standalone/tests/map_test.cpp
+++ b/standalone/tests/map_test.cpp
@@ -31,11 +31,19 @@
TEST(ScudoMapTest, MapUnmap) {
const scudo::uptr Size = 4 * scudo::getPageSizeCached();
- void *P = scudo::map(nullptr, Size, MappingName, 0, nullptr);
- EXPECT_NE(P, nullptr);
- memset(P, 0xaa, Size);
- scudo::unmap(P, Size, 0, nullptr);
- EXPECT_DEATH(memset(P, 0xbb, Size), "");
+ EXPECT_DEATH(
+ {
+ // Repeat few time to avoid missing crash if it's mmaped by unrelated
+ // code.
+ for (int i = 0; i < 10; ++i) {
+ void *P = scudo::map(nullptr, Size, MappingName, 0, nullptr);
+ if (!P)
+ continue;
+ scudo::unmap(P, Size, 0, nullptr);
+ memset(P, 0xbb, Size);
+ }
+ },
+ "");
}
TEST(ScudoMapTest, MapWithGuardUnmap) {
diff --git a/standalone/tests/memtag_test.cpp b/standalone/tests/memtag_test.cpp
new file mode 100644
index 0000000..50ba0fc
--- /dev/null
+++ b/standalone/tests/memtag_test.cpp
@@ -0,0 +1,186 @@
+//===-- memtag_test.cpp -----------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "common.h"
+#include "memtag.h"
+#include "platform.h"
+#include "tests/scudo_unit_test.h"
+
+#if SCUDO_LINUX
+namespace scudo {
+
+TEST(MemtagBasicTest, Unsupported) {
+ if (archSupportsMemoryTagging())
+ GTEST_SKIP();
+
+ EXPECT_DEATH(archMemoryTagGranuleSize(), "not supported");
+ EXPECT_DEATH(untagPointer((uptr)0), "not supported");
+ EXPECT_DEATH(extractTag((uptr)0), "not supported");
+
+ EXPECT_DEATH(systemSupportsMemoryTagging(), "not supported");
+ EXPECT_DEATH(systemDetectsMemoryTagFaultsTestOnly(), "not supported");
+ EXPECT_DEATH(enableSystemMemoryTaggingTestOnly(), "not supported");
+
+ EXPECT_DEATH(selectRandomTag((uptr)0, 0), "not supported");
+ EXPECT_DEATH(addFixedTag((uptr)0, 1), "not supported");
+ EXPECT_DEATH(storeTags((uptr)0, (uptr)0 + sizeof(0)), "not supported");
+ EXPECT_DEATH(storeTag((uptr)0), "not supported");
+ EXPECT_DEATH(loadTag((uptr)0), "not supported");
+
+ EXPECT_DEATH(setRandomTag(nullptr, 64, 0, nullptr, nullptr), "not supported");
+ EXPECT_DEATH(untagPointer(nullptr), "not supported");
+ EXPECT_DEATH(loadTag(nullptr), "not supported");
+ EXPECT_DEATH(addFixedTag(nullptr, 0), "not supported");
+}
+
+class MemtagTest : public ::testing::Test {
+protected:
+ void SetUp() override {
+ if (!archSupportsMemoryTagging() || !systemDetectsMemoryTagFaultsTestOnly())
+ GTEST_SKIP() << "Memory tagging is not supported";
+
+ BufferSize = getPageSizeCached();
+ Buffer = reinterpret_cast<u8 *>(
+ map(nullptr, BufferSize, "MemtagTest", MAP_MEMTAG, &Data));
+ Addr = reinterpret_cast<uptr>(Buffer);
+ EXPECT_TRUE(isAligned(Addr, archMemoryTagGranuleSize()));
+ EXPECT_EQ(Addr, untagPointer(Addr));
+ }
+
+ void TearDown() override {
+ if (Buffer)
+ unmap(Buffer, BufferSize, 0, &Data);
+ }
+
+ uptr BufferSize = 0;
+ MapPlatformData Data = {};
+ u8 *Buffer = nullptr;
+ uptr Addr = 0;
+};
+
+TEST_F(MemtagTest, ArchMemoryTagGranuleSize) {
+ EXPECT_GT(archMemoryTagGranuleSize(), 1u);
+ EXPECT_TRUE(isPowerOfTwo(archMemoryTagGranuleSize()));
+}
+
+TEST_F(MemtagTest, ExtractTag) {
+ uptr Tags = 0;
+ // Try all value for the top byte and check the tags values are in the
+ // expected range.
+ for (u64 Top = 0; Top < 0x100; ++Top)
+ Tags = Tags | (1u << extractTag(Addr | (Top << 56)));
+ EXPECT_EQ(0xffffull, Tags);
+}
+
+TEST_F(MemtagTest, AddFixedTag) {
+ for (uptr Tag = 0; Tag < 0x10; ++Tag)
+ EXPECT_EQ(Tag, extractTag(addFixedTag(Addr, Tag)));
+ if (SCUDO_DEBUG) {
+ EXPECT_DEBUG_DEATH(addFixedTag(Addr, 16), "");
+ EXPECT_DEBUG_DEATH(addFixedTag(~Addr, 0), "");
+ }
+}
+
+TEST_F(MemtagTest, UntagPointer) {
+ uptr UnTagMask = untagPointer(~uptr(0));
+ for (u64 Top = 0; Top < 0x100; ++Top) {
+ uptr Ptr = (Addr | (Top << 56)) & UnTagMask;
+ EXPECT_EQ(addFixedTag(Ptr, 0), untagPointer(Ptr));
+ }
+}
+
+TEST_F(MemtagTest, ScopedDisableMemoryTagChecks) {
+ u8 *P = reinterpret_cast<u8 *>(addFixedTag(Addr, 1));
+ EXPECT_NE(P, Buffer);
+
+ EXPECT_DEATH(*P = 20, "");
+ ScopedDisableMemoryTagChecks Disable;
+ *P = 10;
+}
+
+TEST_F(MemtagTest, SelectRandomTag) {
+ for (uptr SrcTag = 0; SrcTag < 0x10; ++SrcTag) {
+ uptr Ptr = addFixedTag(Addr, SrcTag);
+ uptr Tags = 0;
+ for (uptr I = 0; I < 100000; ++I)
+ Tags = Tags | (1u << extractTag(selectRandomTag(Ptr, 0)));
+ EXPECT_EQ(0xfffeull, Tags);
+ }
+}
+
+TEST_F(MemtagTest, SelectRandomTagWithMask) {
+ for (uptr j = 0; j < 32; ++j) {
+ for (uptr i = 0; i < 1000; ++i)
+ EXPECT_NE(j, extractTag(selectRandomTag(Addr, 1ull << j)));
+ }
+}
+
+TEST_F(MemtagTest, SKIP_NO_DEBUG(LoadStoreTagUnaligned)) {
+ for (uptr P = Addr; P < Addr + 4 * archMemoryTagGranuleSize(); ++P) {
+ if (P % archMemoryTagGranuleSize() == 0)
+ continue;
+ EXPECT_DEBUG_DEATH(loadTag(P), "");
+ EXPECT_DEBUG_DEATH(storeTag(P), "");
+ }
+}
+
+TEST_F(MemtagTest, LoadStoreTag) {
+ uptr Base = Addr + 0x100;
+ uptr Tagged = addFixedTag(Base, 7);
+ storeTag(Tagged);
+
+ EXPECT_EQ(Base - archMemoryTagGranuleSize(),
+ loadTag(Base - archMemoryTagGranuleSize()));
+ EXPECT_EQ(Tagged, loadTag(Base));
+ EXPECT_EQ(Base + archMemoryTagGranuleSize(),
+ loadTag(Base + archMemoryTagGranuleSize()));
+}
+
+TEST_F(MemtagTest, SKIP_NO_DEBUG(StoreTagsUnaligned)) {
+ for (uptr P = Addr; P < Addr + 4 * archMemoryTagGranuleSize(); ++P) {
+ uptr Tagged = addFixedTag(P, 5);
+ if (Tagged % archMemoryTagGranuleSize() == 0)
+ continue;
+ EXPECT_DEBUG_DEATH(storeTags(Tagged, Tagged), "");
+ }
+}
+
+TEST_F(MemtagTest, StoreTags) {
+ const uptr MaxTaggedSize = 4 * archMemoryTagGranuleSize();
+ for (uptr Size = 0; Size <= MaxTaggedSize; ++Size) {
+ uptr NoTagBegin = Addr + archMemoryTagGranuleSize();
+ uptr NoTagEnd = NoTagBegin + Size;
+
+ u8 Tag = 5;
+
+ uptr TaggedBegin = addFixedTag(NoTagBegin, Tag);
+ uptr TaggedEnd = addFixedTag(NoTagEnd, Tag);
+
+ EXPECT_EQ(roundUpTo(TaggedEnd, archMemoryTagGranuleSize()),
+ storeTags(TaggedBegin, TaggedEnd));
+
+ uptr LoadPtr = Addr;
+ // Untagged left granule.
+ EXPECT_EQ(LoadPtr, loadTag(LoadPtr));
+
+ for (LoadPtr += archMemoryTagGranuleSize(); LoadPtr < NoTagEnd;
+ LoadPtr += archMemoryTagGranuleSize()) {
+ EXPECT_EQ(addFixedTag(LoadPtr, 5), loadTag(LoadPtr));
+ }
+
+ // Untagged right granule.
+ EXPECT_EQ(LoadPtr, loadTag(LoadPtr));
+
+ // Reset tags without using StoreTags.
+ releasePagesToOS(Addr, 0, BufferSize, &Data);
+ }
+}
+
+} // namespace scudo
+
+#endif
diff --git a/standalone/tests/mutex_test.cpp b/standalone/tests/mutex_test.cpp
index ed56cb5..efee6fe 100644
--- a/standalone/tests/mutex_test.cpp
+++ b/standalone/tests/mutex_test.cpp
@@ -82,7 +82,6 @@
TEST(ScudoMutexTest, Mutex) {
scudo::HybridMutex M;
- M.init();
TestData Data(M);
pthread_t Threads[NumberOfThreads];
for (scudo::u32 I = 0; I < NumberOfThreads; I++)
@@ -93,7 +92,6 @@
TEST(ScudoMutexTest, MutexTry) {
scudo::HybridMutex M;
- M.init();
TestData Data(M);
pthread_t Threads[NumberOfThreads];
for (scudo::u32 I = 0; I < NumberOfThreads; I++)
diff --git a/standalone/tests/primary_test.cpp b/standalone/tests/primary_test.cpp
index e7aa6f7..5ec4361 100644
--- a/standalone/tests/primary_test.cpp
+++ b/standalone/tests/primary_test.cpp
@@ -29,24 +29,40 @@
static const bool MaySupportMemoryTagging = false;
typedef scudo::uptr PrimaryCompactPtrT;
static const scudo::uptr PrimaryCompactPtrScale = 0;
+ static const bool PrimaryEnableRandomOffset = true;
+ static const scudo::uptr PrimaryMapSizeIncrement = 1UL << 18;
};
struct TestConfig2 {
+#if defined(__mips__)
+ // Unable to allocate greater size on QEMU-user.
+ static const scudo::uptr PrimaryRegionSizeLog = 23U;
+#else
static const scudo::uptr PrimaryRegionSizeLog = 24U;
+#endif
static const scudo::s32 PrimaryMinReleaseToOsIntervalMs = INT32_MIN;
static const scudo::s32 PrimaryMaxReleaseToOsIntervalMs = INT32_MAX;
static const bool MaySupportMemoryTagging = false;
typedef scudo::uptr PrimaryCompactPtrT;
static const scudo::uptr PrimaryCompactPtrScale = 0;
+ static const bool PrimaryEnableRandomOffset = true;
+ static const scudo::uptr PrimaryMapSizeIncrement = 1UL << 18;
};
struct TestConfig3 {
+#if defined(__mips__)
+ // Unable to allocate greater size on QEMU-user.
+ static const scudo::uptr PrimaryRegionSizeLog = 23U;
+#else
static const scudo::uptr PrimaryRegionSizeLog = 24U;
+#endif
static const scudo::s32 PrimaryMinReleaseToOsIntervalMs = INT32_MIN;
static const scudo::s32 PrimaryMaxReleaseToOsIntervalMs = INT32_MAX;
static const bool MaySupportMemoryTagging = true;
typedef scudo::uptr PrimaryCompactPtrT;
static const scudo::uptr PrimaryCompactPtrScale = 0;
+ static const bool PrimaryEnableRandomOffset = true;
+ static const scudo::uptr PrimaryMapSizeIncrement = 1UL << 18;
};
template <typename BaseConfig, typename SizeClassMapT>
@@ -122,7 +138,7 @@
}
Cache.destroy(nullptr);
Allocator->releaseToOS();
- scudo::ScopedString Str(1024);
+ scudo::ScopedString Str;
Allocator->getStats(&Str);
Str.output();
}
@@ -135,6 +151,8 @@
static const bool MaySupportMemoryTagging = false;
typedef scudo::uptr PrimaryCompactPtrT;
static const scudo::uptr PrimaryCompactPtrScale = 0;
+ static const bool PrimaryEnableRandomOffset = true;
+ static const scudo::uptr PrimaryMapSizeIncrement = 1UL << 18;
};
// The 64-bit SizeClassAllocator can be easily OOM'd with small region sizes.
@@ -168,7 +186,7 @@
}
Cache.destroy(nullptr);
Allocator.releaseToOS();
- scudo::ScopedString Str(1024);
+ scudo::ScopedString Str;
Allocator.getStats(&Str);
Str.output();
EXPECT_EQ(AllocationFailed, true);
@@ -206,7 +224,7 @@
}
Cache.destroy(nullptr);
Allocator->releaseToOS();
- scudo::ScopedString Str(1024);
+ scudo::ScopedString Str;
Allocator->getStats(&Str);
Str.output();
}
@@ -253,7 +271,7 @@
for (auto &T : Threads)
T.join();
Allocator->releaseToOS();
- scudo::ScopedString Str(1024);
+ scudo::ScopedString Str;
Allocator->getStats(&Str);
Str.output();
}
diff --git a/standalone/tests/quarantine_test.cpp b/standalone/tests/quarantine_test.cpp
index 91de56a..972c98d 100644
--- a/standalone/tests/quarantine_test.cpp
+++ b/standalone/tests/quarantine_test.cpp
@@ -214,7 +214,7 @@
Quarantine.drainAndRecycle(&Cache, Cb);
EXPECT_EQ(Cache.getSize(), 0UL);
- scudo::ScopedString Str(1024);
+ scudo::ScopedString Str;
Quarantine.getStats(&Str);
Str.output();
}
@@ -246,7 +246,7 @@
for (scudo::uptr I = 0; I < NumberOfThreads; I++)
pthread_join(T[I].Thread, 0);
- scudo::ScopedString Str(1024);
+ scudo::ScopedString Str;
Quarantine.getStats(&Str);
Str.output();
diff --git a/standalone/tests/report_test.cpp b/standalone/tests/report_test.cpp
index 09f03f1..374b6b8 100644
--- a/standalone/tests/report_test.cpp
+++ b/standalone/tests/report_test.cpp
@@ -10,6 +10,13 @@
#include "report.h"
+TEST(ScudoReportTest, Check) {
+ CHECK_LT(-1, 1);
+ EXPECT_DEATH(CHECK_GT(-1, 1),
+ "\\(-1\\) > \\(1\\) \\(\\(u64\\)op1=18446744073709551615, "
+ "\\(u64\\)op2=1");
+}
+
TEST(ScudoReportTest, Generic) {
// Potentially unused if EXPECT_DEATH isn't defined.
UNUSED void *P = reinterpret_cast<void *>(0x42424242U);
diff --git a/standalone/tests/scudo_unit_test.h b/standalone/tests/scudo_unit_test.h
index 555a935..1665fa8 100644
--- a/standalone/tests/scudo_unit_test.h
+++ b/standalone/tests/scudo_unit_test.h
@@ -33,4 +33,16 @@
#define EXPECT_STREQ(X, Y) EXPECT_EQ(strcmp(X, Y), 0)
#endif
+#if SCUDO_FUCHSIA
+#define SKIP_ON_FUCHSIA(T) DISABLED_##T
+#else
+#define SKIP_ON_FUCHSIA(T) T
+#endif
+
+#if SCUDO_DEBUG
+#define SKIP_NO_DEBUG(T) T
+#else
+#define SKIP_NO_DEBUG(T) DISABLED_##T
+#endif
+
extern bool UseQuarantine;
diff --git a/standalone/tests/scudo_unit_test_main.cpp b/standalone/tests/scudo_unit_test_main.cpp
index 9bbf6e7..1983b80 100644
--- a/standalone/tests/scudo_unit_test_main.cpp
+++ b/standalone/tests/scudo_unit_test_main.cpp
@@ -6,6 +6,7 @@
//
//===----------------------------------------------------------------------===//
+#include "memtag.h"
#include "tests/scudo_unit_test.h"
// Match Android's default configuration, which disables Scudo's mismatch
@@ -33,6 +34,8 @@
// for Fuchsia builds.
#if !SCUDO_FUCHSIA
int main(int argc, char **argv) {
+ if (scudo::archSupportsMemoryTagging())
+ scudo::enableSystemMemoryTaggingTestOnly();
testing::InitGoogleTest(&argc, argv);
return RUN_ALL_TESTS();
}
diff --git a/standalone/tests/secondary_test.cpp b/standalone/tests/secondary_test.cpp
index a557042..2dc041b 100644
--- a/standalone/tests/secondary_test.cpp
+++ b/standalone/tests/secondary_test.cpp
@@ -32,9 +32,21 @@
memset(P, 'A', Size);
EXPECT_GE(SecondaryT::getBlockSize(P), Size);
L->deallocate(scudo::Options{}, P);
+
// If the Secondary can't cache that pointer, it will be unmapped.
- if (!L->canCache(Size))
- EXPECT_DEATH(memset(P, 'A', Size), "");
+ if (!L->canCache(Size)) {
+ EXPECT_DEATH(
+ {
+ // Repeat few time to avoid missing crash if it's mmaped by unrelated
+ // code.
+ for (int i = 0; i < 10; ++i) {
+ P = L->allocate(scudo::Options{}, Size);
+ L->deallocate(scudo::Options{}, P);
+ memset(P, 'A', Size);
+ }
+ },
+ "");
+ }
const scudo::uptr Align = 1U << 16;
P = L->allocate(scudo::Options{}, Size + Align, Align);
@@ -52,9 +64,10 @@
L->deallocate(scudo::Options{}, V.back());
V.pop_back();
}
- scudo::ScopedString Str(1024);
+ scudo::ScopedString Str;
L->getStats(&Str);
Str.output();
+ L->unmapTestOnly();
}
struct NoCacheConfig {
@@ -109,9 +122,10 @@
}
}
}
- scudo::ScopedString Str(1024);
+ scudo::ScopedString Str;
L->getStats(&Str);
Str.output();
+ L->unmapTestOnly();
}
TEST(ScudoSecondaryTest, SecondaryIterate) {
@@ -132,9 +146,10 @@
L->deallocate(scudo::Options{}, V.back());
V.pop_back();
}
- scudo::ScopedString Str(1024);
+ scudo::ScopedString Str;
L->getStats(&Str);
Str.output();
+ L->unmapTestOnly();
}
TEST(ScudoSecondaryTest, SecondaryOptions) {
@@ -158,6 +173,7 @@
EXPECT_TRUE(L->setOption(scudo::Option::MaxCacheEntrySize, 1UL << 20));
EXPECT_TRUE(L->canCache(1UL << 16));
}
+ L->unmapTestOnly();
}
static std::mutex Mutex;
@@ -201,7 +217,8 @@
}
for (auto &T : Threads)
T.join();
- scudo::ScopedString Str(1024);
+ scudo::ScopedString Str;
L->getStats(&Str);
Str.output();
+ L->unmapTestOnly();
}
diff --git a/standalone/tests/size_class_map_test.cpp b/standalone/tests/size_class_map_test.cpp
index 88859de..076f36f 100644
--- a/standalone/tests/size_class_map_test.cpp
+++ b/standalone/tests/size_class_map_test.cpp
@@ -35,6 +35,7 @@
static const scudo::uptr MaxSizeLog = 5;
static const scudo::u32 MaxNumCachedHint = 0;
static const scudo::uptr MaxBytesCachedLog = 0;
+ static const scudo::uptr SizeDelta = 0;
};
TEST(ScudoSizeClassMapTest, OneClassSizeClassMap) {
@@ -49,6 +50,7 @@
static const scudo::uptr MaxSizeLog = 63;
static const scudo::u32 MaxNumCachedHint = 128;
static const scudo::uptr MaxBytesCachedLog = 16;
+ static const scudo::uptr SizeDelta = 0;
};
TEST(ScudoSizeClassMapTest, LargeMaxSizeClassMap) {
diff --git a/standalone/tests/strings_test.cpp b/standalone/tests/strings_test.cpp
index eed174d..6d7e78a 100644
--- a/standalone/tests/strings_test.cpp
+++ b/standalone/tests/strings_test.cpp
@@ -12,8 +12,14 @@
#include <limits.h>
+TEST(ScudoStringsTest, Constructor) {
+ scudo::ScopedString Str;
+ EXPECT_EQ(0ul, Str.length());
+ EXPECT_EQ('\0', *Str.data());
+}
+
TEST(ScudoStringsTest, Basic) {
- scudo::ScopedString Str(128);
+ scudo::ScopedString Str;
Str.append("a%db%zdc%ue%zuf%xh%zxq%pe%sr", static_cast<int>(-1),
static_cast<scudo::uptr>(-2), static_cast<unsigned>(-4),
static_cast<scudo::uptr>(5), static_cast<unsigned>(10),
@@ -28,8 +34,25 @@
EXPECT_STREQ(expectedString.c_str(), Str.data());
}
+TEST(ScudoStringsTest, Clear) {
+ scudo::ScopedString Str;
+ Str.append("123");
+ Str.clear();
+ EXPECT_EQ(0ul, Str.length());
+ EXPECT_EQ('\0', *Str.data());
+}
+
+TEST(ScudoStringsTest, ClearLarge) {
+ scudo::ScopedString Str;
+ for (int i = 0; i < 10000; ++i)
+ Str.append("123");
+ Str.clear();
+ EXPECT_EQ(0ul, Str.length());
+ EXPECT_EQ('\0', *Str.data());
+}
+
TEST(ScudoStringsTest, Precision) {
- scudo::ScopedString Str(128);
+ scudo::ScopedString Str;
Str.append("%.*s", 3, "12345");
EXPECT_EQ(Str.length(), strlen(Str.data()));
EXPECT_STREQ("123", Str.data());
@@ -52,7 +75,7 @@
// Use a ScopedString that spans a page, and attempt to write past the end
// of it with variations of append. The expectation is for nothing to crash.
const scudo::uptr PageSize = scudo::getPageSizeCached();
- scudo::ScopedString Str(PageSize);
+ scudo::ScopedString Str;
Str.clear();
fillString(Str, 2 * PageSize);
Str.clear();
@@ -68,7 +91,7 @@
template <typename T>
static void testAgainstLibc(const char *Format, T Arg1, T Arg2) {
- scudo::ScopedString Str(128);
+ scudo::ScopedString Str;
Str.append(Format, Arg1, Arg2);
char Buffer[128];
snprintf(Buffer, sizeof(Buffer), Format, Arg1, Arg2);
diff --git a/standalone/tests/tsd_test.cpp b/standalone/tests/tsd_test.cpp
index 58ac9e7..35e579d 100644
--- a/standalone/tests/tsd_test.cpp
+++ b/standalone/tests/tsd_test.cpp
@@ -26,15 +26,14 @@
using CacheT = struct MockCache { volatile scudo::uptr Canary; };
using QuarantineCacheT = struct MockQuarantine {};
- void initLinkerInitialized() {
+ void init() {
// This should only be called once by the registry.
EXPECT_FALSE(Initialized);
Initialized = true;
}
- void reset() { memset(this, 0, sizeof(*this)); }
- void unmapTestOnly() { TSDRegistry.unmapTestOnly(); }
- void initCache(CacheT *Cache) { memset(Cache, 0, sizeof(*Cache)); }
+ void unmapTestOnly() { TSDRegistry.unmapTestOnly(this); }
+ void initCache(CacheT *Cache) { *Cache = {}; }
void commitBack(scudo::TSD<MockAllocator> *TSD) {}
TSDRegistryT *getTSDRegistry() { return &TSDRegistry; }
void callPostInitCallback() {}
@@ -42,7 +41,7 @@
bool isInitialized() { return Initialized; }
private:
- bool Initialized;
+ bool Initialized = false;
TSDRegistryT TSDRegistry;
};
@@ -69,11 +68,10 @@
};
std::unique_ptr<AllocatorT, decltype(Deleter)> Allocator(new AllocatorT,
Deleter);
- Allocator->reset();
EXPECT_FALSE(Allocator->isInitialized());
auto Registry = Allocator->getTSDRegistry();
- Registry->initLinkerInitialized(Allocator.get());
+ Registry->init(Allocator.get());
EXPECT_TRUE(Allocator->isInitialized());
}
@@ -84,7 +82,6 @@
};
std::unique_ptr<AllocatorT, decltype(Deleter)> Allocator(new AllocatorT,
Deleter);
- Allocator->reset();
EXPECT_FALSE(Allocator->isInitialized());
auto Registry = Allocator->getTSDRegistry();
@@ -153,7 +150,6 @@
};
std::unique_ptr<AllocatorT, decltype(Deleter)> Allocator(new AllocatorT,
Deleter);
- Allocator->reset();
std::thread Threads[32];
for (scudo::uptr I = 0; I < ARRAY_SIZE(Threads); I++)
Threads[I] = std::thread(stressCache<AllocatorT>, Allocator.get());
@@ -209,7 +205,6 @@
};
std::unique_ptr<AllocatorT, decltype(Deleter)> Allocator(new AllocatorT,
Deleter);
- Allocator->reset();
// We attempt to use as many TSDs as the shared cache offers by creating a
// decent amount of threads that will be run concurrently and attempt to get
// and lock TSDs. We put them all in a set and count the number of entries
diff --git a/standalone/tests/vector_test.cpp b/standalone/tests/vector_test.cpp
index d2c6a9b..dc23c2a 100644
--- a/standalone/tests/vector_test.cpp
+++ b/standalone/tests/vector_test.cpp
@@ -23,14 +23,14 @@
}
TEST(ScudoVectorTest, Stride) {
- scudo::Vector<int> V;
- for (int i = 0; i < 1000; i++) {
- V.push_back(i);
- EXPECT_EQ(V.size(), i + 1U);
- EXPECT_EQ(V[i], i);
+ scudo::Vector<scudo::uptr> V;
+ for (scudo::uptr I = 0; I < 1000; I++) {
+ V.push_back(I);
+ EXPECT_EQ(V.size(), I + 1U);
+ EXPECT_EQ(V[I], I);
}
- for (int i = 0; i < 1000; i++)
- EXPECT_EQ(V[i], i);
+ for (scudo::uptr I = 0; I < 1000; I++)
+ EXPECT_EQ(V[I], I);
}
TEST(ScudoVectorTest, ResizeReduction) {
diff --git a/standalone/tests/wrappers_c_test.cpp b/standalone/tests/wrappers_c_test.cpp
index eed8f03..b82b5fb 100644
--- a/standalone/tests/wrappers_c_test.cpp
+++ b/standalone/tests/wrappers_c_test.cpp
@@ -6,6 +6,7 @@
//
//===----------------------------------------------------------------------===//
+#include "memtag.h"
#include "scudo/interface.h"
#include "tests/scudo_unit_test.h"
@@ -277,6 +278,10 @@
static size_t Count;
static void callback(uintptr_t Base, size_t Size, void *Arg) {
+ if (scudo::archSupportsMemoryTagging()) {
+ Base = scudo::untagPointer(Base);
+ BoundaryP = scudo::untagPointer(BoundaryP);
+ }
if (Base == BoundaryP)
Count++;
}
@@ -366,7 +371,7 @@
TEST(ScudoWrappersCTest, Fork) {
void *P;
pid_t Pid = fork();
- EXPECT_GE(Pid, 0);
+ EXPECT_GE(Pid, 0) << strerror(errno);
if (Pid == 0) {
P = malloc(Size);
EXPECT_NE(P, nullptr);
diff --git a/standalone/trusty.cpp b/standalone/trusty.cpp
new file mode 100644
index 0000000..81d6bc5
--- /dev/null
+++ b/standalone/trusty.cpp
@@ -0,0 +1,100 @@
+//===-- trusty.cpp ---------------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "platform.h"
+
+#if SCUDO_TRUSTY
+
+#include "common.h"
+#include "mutex.h"
+#include "string_utils.h"
+#include "trusty.h"
+
+#include <errno.h> // for errno
+#include <stdio.h> // for printf()
+#include <stdlib.h> // for getenv()
+#include <sys/auxv.h> // for getauxval()
+#include <time.h> // for clock_gettime()
+#include <trusty_syscalls.h> // for _trusty_brk()
+
+#define SBRK_ALIGN 32
+
+namespace scudo {
+
+uptr getPageSize() { return getauxval(AT_PAGESZ); }
+
+void NORETURN die() { abort(); }
+
+void *map(UNUSED void *Addr, uptr Size, UNUSED const char *Name, uptr Flags,
+ UNUSED MapPlatformData *Data) {
+ // Calling _trusty_brk(0) returns the current program break.
+ uptr ProgramBreak = reinterpret_cast<uptr>(_trusty_brk(0));
+ uptr Start;
+ uptr End;
+
+ Start = roundUpTo(ProgramBreak, SBRK_ALIGN);
+ // Don't actually extend the heap if MAP_NOACCESS flag is set since this is
+ // the case where Scudo tries to reserve a memory region without mapping
+ // physical pages.
+ if (Flags & MAP_NOACCESS)
+ return reinterpret_cast<void *>(Start);
+
+ // Attempt to extend the heap by Size bytes using _trusty_brk.
+ End = roundUpTo(Start + Size, SBRK_ALIGN);
+ ProgramBreak =
+ reinterpret_cast<uptr>(_trusty_brk(reinterpret_cast<void *>(End)));
+ if (ProgramBreak < End) {
+ errno = ENOMEM;
+ dieOnMapUnmapError(Size);
+ return nullptr;
+ }
+ return reinterpret_cast<void *>(Start); // Base of new reserved region.
+}
+
+// Unmap is a no-op since Trusty uses sbrk instead of memory mapping.
+void unmap(UNUSED void *Addr, UNUSED uptr Size, UNUSED uptr Flags,
+ UNUSED MapPlatformData *Data) {}
+
+void setMemoryPermission(UNUSED uptr Addr, UNUSED uptr Size, UNUSED uptr Flags,
+ UNUSED MapPlatformData *Data) {}
+
+void releasePagesToOS(UNUSED uptr BaseAddress, UNUSED uptr Offset,
+ UNUSED uptr Size, UNUSED MapPlatformData *Data) {}
+
+const char *getEnv(const char *Name) { return getenv(Name); }
+
+// All mutex operations are a no-op since Trusty doesn't currently support
+// threads.
+bool HybridMutex::tryLock() { return true; }
+
+void HybridMutex::lockSlow() {}
+
+void HybridMutex::unlock() {}
+
+u64 getMonotonicTime() {
+ timespec TS;
+ clock_gettime(CLOCK_MONOTONIC, &TS);
+ return static_cast<u64>(TS.tv_sec) * (1000ULL * 1000 * 1000) +
+ static_cast<u64>(TS.tv_nsec);
+}
+
+u32 getNumberOfCPUs() { return 0; }
+
+u32 getThreadID() { return 0; }
+
+bool getRandom(UNUSED void *Buffer, UNUSED uptr Length, UNUSED bool Blocking) {
+ return false;
+}
+
+void outputRaw(const char *Buffer) { printf("%s", Buffer); }
+
+void setAbortMessage(UNUSED const char *Message) {}
+
+} // namespace scudo
+
+#endif // SCUDO_TRUSTY
diff --git a/standalone/trusty.h b/standalone/trusty.h
new file mode 100644
index 0000000..50edd1c
--- /dev/null
+++ b/standalone/trusty.h
@@ -0,0 +1,24 @@
+//===-- trusty.h -----------------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef SCUDO_TRUSTY_H_
+#define SCUDO_TRUSTY_H_
+
+#include "platform.h"
+
+#if SCUDO_TRUSTY
+
+namespace scudo {
+// MapPlatformData is unused on Trusty, define it as a minimially sized
+// structure.
+struct MapPlatformData {};
+} // namespace scudo
+
+#endif // SCUDO_TRUSTY
+
+#endif // SCUDO_TRUSTY_H_
diff --git a/standalone/tsd.h b/standalone/tsd.h
index a6e669b..e376df0 100644
--- a/standalone/tsd.h
+++ b/standalone/tsd.h
@@ -28,14 +28,11 @@
typename Allocator::QuarantineCacheT QuarantineCache;
u8 DestructorIterations = 0;
- void initLinkerInitialized(Allocator *Instance) {
+ void init(Allocator *Instance) {
+ DCHECK_EQ(DestructorIterations, 0U);
Instance->initCache(&Cache);
DestructorIterations = PTHREAD_DESTRUCTOR_ITERATIONS;
}
- void init(Allocator *Instance) {
- memset(this, 0, sizeof(*this));
- initLinkerInitialized(Instance);
- }
void commitBack(Allocator *Instance) { Instance->commitBack(this); }
diff --git a/standalone/tsd_exclusive.h b/standalone/tsd_exclusive.h
index a907ed4..bba0c27 100644
--- a/standalone/tsd_exclusive.h
+++ b/standalone/tsd_exclusive.h
@@ -25,31 +25,35 @@
template <class Allocator> void teardownThread(void *Ptr);
template <class Allocator> struct TSDRegistryExT {
- void initLinkerInitialized(Allocator *Instance) {
- Instance->initLinkerInitialized();
- CHECK_EQ(pthread_key_create(&PThreadKey, teardownThread<Allocator>), 0);
- FallbackTSD.initLinkerInitialized(Instance);
- Initialized = true;
- }
void init(Allocator *Instance) {
- memset(this, 0, sizeof(*this));
- initLinkerInitialized(Instance);
+ DCHECK(!Initialized);
+ Instance->init();
+ CHECK_EQ(pthread_key_create(&PThreadKey, teardownThread<Allocator>), 0);
+ FallbackTSD.init(Instance);
+ Initialized = true;
}
void initOnceMaybe(Allocator *Instance) {
ScopedLock L(Mutex);
if (LIKELY(Initialized))
return;
- initLinkerInitialized(Instance); // Sets Initialized.
+ init(Instance); // Sets Initialized.
}
- void unmapTestOnly() {
- Allocator *Instance =
- reinterpret_cast<Allocator *>(pthread_getspecific(PThreadKey));
- if (!Instance)
- return;
- ThreadTSD.commitBack(Instance);
+ void unmapTestOnly(Allocator *Instance) {
+ DCHECK(Instance);
+ if (reinterpret_cast<Allocator *>(pthread_getspecific(PThreadKey))) {
+ DCHECK_EQ(reinterpret_cast<Allocator *>(pthread_getspecific(PThreadKey)),
+ Instance);
+ ThreadTSD.commitBack(Instance);
+ ThreadTSD = {};
+ }
+ CHECK_EQ(pthread_key_delete(PThreadKey), 0);
+ PThreadKey = {};
+ FallbackTSD.commitBack(Instance);
+ FallbackTSD = {};
State = {};
+ Initialized = false;
}
ALWAYS_INLINE void initThreadMaybe(Allocator *Instance, bool MinimalInit) {
@@ -103,7 +107,7 @@
return;
CHECK_EQ(
pthread_setspecific(PThreadKey, reinterpret_cast<void *>(Instance)), 0);
- ThreadTSD.initLinkerInitialized(Instance);
+ ThreadTSD.init(Instance);
State.InitState = ThreadState::Initialized;
Instance->callPostInitCallback();
}
diff --git a/standalone/tsd_shared.h b/standalone/tsd_shared.h
index afe3623..1c2a880 100644
--- a/standalone/tsd_shared.h
+++ b/standalone/tsd_shared.h
@@ -24,28 +24,32 @@
template <class Allocator, u32 TSDsArraySize, u32 DefaultTSDCount>
struct TSDRegistrySharedT {
- void initLinkerInitialized(Allocator *Instance) {
- Instance->initLinkerInitialized();
+ void init(Allocator *Instance) {
+ DCHECK(!Initialized);
+ Instance->init();
for (u32 I = 0; I < TSDsArraySize; I++)
- TSDs[I].initLinkerInitialized(Instance);
+ TSDs[I].init(Instance);
const u32 NumberOfCPUs = getNumberOfCPUs();
setNumberOfTSDs((NumberOfCPUs == 0) ? DefaultTSDCount
: Min(NumberOfCPUs, DefaultTSDCount));
Initialized = true;
}
- void init(Allocator *Instance) {
- memset(this, 0, sizeof(*this));
- initLinkerInitialized(Instance);
- }
void initOnceMaybe(Allocator *Instance) {
ScopedLock L(Mutex);
if (LIKELY(Initialized))
return;
- initLinkerInitialized(Instance); // Sets Initialized.
+ init(Instance); // Sets Initialized.
}
- void unmapTestOnly() { setCurrentTSD(nullptr); }
+ void unmapTestOnly(Allocator *Instance) {
+ for (u32 I = 0; I < TSDsArraySize; I++) {
+ TSDs[I].commitBack(Instance);
+ TSDs[I] = {};
+ }
+ setCurrentTSD(nullptr);
+ Initialized = false;
+ }
ALWAYS_INLINE void initThreadMaybe(Allocator *Instance,
UNUSED bool MinimalInit) {
diff --git a/standalone/vector.h b/standalone/vector.h
index 6ca350a..2c9a6e2 100644
--- a/standalone/vector.h
+++ b/standalone/vector.h
@@ -19,14 +19,13 @@
// small vectors. The current implementation supports only POD types.
template <typename T> class VectorNoCtor {
public:
- void init(uptr InitialCapacity) {
- CapacityBytes = 0;
- Size = 0;
- Data = nullptr;
+ void init(uptr InitialCapacity = 0) {
+ Data = reinterpret_cast<T *>(&LocalData[0]);
+ CapacityBytes = sizeof(LocalData);
reserve(InitialCapacity);
}
void destroy() {
- if (Data)
+ if (Data != reinterpret_cast<T *>(&LocalData[0]))
unmap(Data, CapacityBytes);
}
T &operator[](uptr I) {
@@ -82,26 +81,24 @@
void reallocate(uptr NewCapacity) {
DCHECK_GT(NewCapacity, 0);
DCHECK_LE(Size, NewCapacity);
- const uptr NewCapacityBytes =
- roundUpTo(NewCapacity * sizeof(T), getPageSizeCached());
+ NewCapacity = roundUpTo(NewCapacity * sizeof(T), getPageSizeCached());
T *NewData =
- reinterpret_cast<T *>(map(nullptr, NewCapacityBytes, "scudo:vector"));
- if (Data) {
- memcpy(NewData, Data, Size * sizeof(T));
- unmap(Data, CapacityBytes);
- }
+ reinterpret_cast<T *>(map(nullptr, NewCapacity, "scudo:vector"));
+ memcpy(NewData, Data, Size * sizeof(T));
+ destroy();
Data = NewData;
- CapacityBytes = NewCapacityBytes;
+ CapacityBytes = NewCapacity;
}
- T *Data;
- uptr CapacityBytes;
- uptr Size;
+ T *Data = nullptr;
+ u8 LocalData[256] = {};
+ uptr CapacityBytes = 0;
+ uptr Size = 0;
};
template <typename T> class Vector : public VectorNoCtor<T> {
public:
- Vector() { VectorNoCtor<T>::init(1); }
+ Vector() { VectorNoCtor<T>::init(); }
explicit Vector(uptr Count) {
VectorNoCtor<T>::init(Count);
this->resize(Count);