Dynamic Tools Team | 517193e | 2019-09-11 14:48:41 +0000 | [diff] [blame] | 1 | //===-- secondary_test.cpp --------------------------------------*- C++ -*-===// |
| 2 | // |
| 3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| 4 | // See https://llvm.org/LICENSE.txt for license information. |
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| 6 | // |
| 7 | //===----------------------------------------------------------------------===// |
| 8 | |
Vitaly Buka | ace22c7 | 2021-06-30 23:39:12 -0700 | [diff] [blame] | 9 | #include "memtag.h" |
Dynamic Tools Team | 09e6d48 | 2019-11-26 18:18:14 -0800 | [diff] [blame] | 10 | #include "tests/scudo_unit_test.h" |
Dynamic Tools Team | 517193e | 2019-09-11 14:48:41 +0000 | [diff] [blame] | 11 | |
Peter Collingbourne | 7488a17 | 2020-12-14 13:57:59 -0800 | [diff] [blame] | 12 | #include "allocator_config.h" |
Dynamic Tools Team | 09e6d48 | 2019-11-26 18:18:14 -0800 | [diff] [blame] | 13 | #include "secondary.h" |
Dynamic Tools Team | 517193e | 2019-09-11 14:48:41 +0000 | [diff] [blame] | 14 | |
Dynamic Tools Team | 517193e | 2019-09-11 14:48:41 +0000 | [diff] [blame] | 15 | #include <condition_variable> |
Vitaly Buka | d319932 | 2021-07-01 10:22:35 -0700 | [diff] [blame] | 16 | #include <memory> |
Dynamic Tools Team | 517193e | 2019-09-11 14:48:41 +0000 | [diff] [blame] | 17 | #include <mutex> |
Dynamic Tools Team | d2740c5 | 2019-11-19 13:58:06 -0800 | [diff] [blame] | 18 | #include <random> |
Vitaly Buka | d319932 | 2021-07-01 10:22:35 -0700 | [diff] [blame] | 19 | #include <stdio.h> |
Dynamic Tools Team | 517193e | 2019-09-11 14:48:41 +0000 | [diff] [blame] | 20 | #include <thread> |
Dynamic Tools Team | 09e6d48 | 2019-11-26 18:18:14 -0800 | [diff] [blame] | 21 | #include <vector> |
Dynamic Tools Team | 517193e | 2019-09-11 14:48:41 +0000 | [diff] [blame] | 22 | |
Vitaly Buka | 97742dd | 2021-07-01 11:56:11 -0700 | [diff] [blame] | 23 | template <typename Config> static scudo::Options getOptionsForConfig() { |
Vitaly Buka | ace22c7 | 2021-06-30 23:39:12 -0700 | [diff] [blame] | 24 | if (!Config::MaySupportMemoryTagging || !scudo::archSupportsMemoryTagging()) |
| 25 | return {}; |
| 26 | scudo::AtomicOptions AO; |
| 27 | AO.set(scudo::OptionBit::UseMemoryTagging); |
| 28 | return AO.load(); |
Vitaly Buka | 97742dd | 2021-07-01 11:56:11 -0700 | [diff] [blame] | 29 | } |
| 30 | |
Peter Collingbourne | 4edf4ef | 2020-12-15 15:32:32 -0800 | [diff] [blame] | 31 | template <typename Config> static void testSecondaryBasic(void) { |
| 32 | using SecondaryT = scudo::MapAllocator<Config>; |
Vitaly Buka | 97742dd | 2021-07-01 11:56:11 -0700 | [diff] [blame] | 33 | scudo::Options Options = getOptionsForConfig<Config>(); |
Peter Collingbourne | 4edf4ef | 2020-12-15 15:32:32 -0800 | [diff] [blame] | 34 | |
Dynamic Tools Team | 517193e | 2019-09-11 14:48:41 +0000 | [diff] [blame] | 35 | scudo::GlobalStats S; |
| 36 | S.init(); |
Kostya Kortchinsky | c72ca56 | 2020-07-27 09:13:42 -0700 | [diff] [blame] | 37 | std::unique_ptr<SecondaryT> L(new SecondaryT); |
Dynamic Tools Team | 517193e | 2019-09-11 14:48:41 +0000 | [diff] [blame] | 38 | L->init(&S); |
| 39 | const scudo::uptr Size = 1U << 16; |
Vitaly Buka | 97742dd | 2021-07-01 11:56:11 -0700 | [diff] [blame] | 40 | void *P = L->allocate(Options, Size); |
Dynamic Tools Team | 517193e | 2019-09-11 14:48:41 +0000 | [diff] [blame] | 41 | EXPECT_NE(P, nullptr); |
| 42 | memset(P, 'A', Size); |
Dynamic Tools Team | d29271f | 2019-10-31 10:31:49 -0700 | [diff] [blame] | 43 | EXPECT_GE(SecondaryT::getBlockSize(P), Size); |
Vitaly Buka | 97742dd | 2021-07-01 11:56:11 -0700 | [diff] [blame] | 44 | L->deallocate(Options, P); |
Mitch Phillips | a99b779 | 2021-05-20 10:56:47 -0700 | [diff] [blame] | 45 | |
Dynamic Tools Team | c5d5abc | 2020-01-27 14:03:21 -0800 | [diff] [blame] | 46 | // If the Secondary can't cache that pointer, it will be unmapped. |
Vitaly Buka | 5cb9636 | 2021-05-20 16:17:25 -0700 | [diff] [blame] | 47 | if (!L->canCache(Size)) { |
| 48 | EXPECT_DEATH( |
| 49 | { |
| 50 | // Repeat few time to avoid missing crash if it's mmaped by unrelated |
| 51 | // code. |
| 52 | for (int i = 0; i < 10; ++i) { |
Vitaly Buka | 97742dd | 2021-07-01 11:56:11 -0700 | [diff] [blame] | 53 | P = L->allocate(Options, Size); |
| 54 | L->deallocate(Options, P); |
Vitaly Buka | 5cb9636 | 2021-05-20 16:17:25 -0700 | [diff] [blame] | 55 | memset(P, 'A', Size); |
| 56 | } |
| 57 | }, |
| 58 | ""); |
Vitaly Buka | 5cb9636 | 2021-05-20 16:17:25 -0700 | [diff] [blame] | 59 | } |
Dynamic Tools Team | 517193e | 2019-09-11 14:48:41 +0000 | [diff] [blame] | 60 | |
| 61 | const scudo::uptr Align = 1U << 16; |
Vitaly Buka | 97742dd | 2021-07-01 11:56:11 -0700 | [diff] [blame] | 62 | P = L->allocate(Options, Size + Align, Align); |
Dynamic Tools Team | 517193e | 2019-09-11 14:48:41 +0000 | [diff] [blame] | 63 | EXPECT_NE(P, nullptr); |
| 64 | void *AlignedP = reinterpret_cast<void *>( |
| 65 | scudo::roundUpTo(reinterpret_cast<scudo::uptr>(P), Align)); |
| 66 | memset(AlignedP, 'A', Size); |
Vitaly Buka | 97742dd | 2021-07-01 11:56:11 -0700 | [diff] [blame] | 67 | L->deallocate(Options, P); |
Dynamic Tools Team | 517193e | 2019-09-11 14:48:41 +0000 | [diff] [blame] | 68 | |
| 69 | std::vector<void *> V; |
| 70 | for (scudo::uptr I = 0; I < 32U; I++) |
Vitaly Buka | 97742dd | 2021-07-01 11:56:11 -0700 | [diff] [blame] | 71 | V.push_back(L->allocate(Options, Size)); |
Dynamic Tools Team | d2740c5 | 2019-11-19 13:58:06 -0800 | [diff] [blame] | 72 | std::shuffle(V.begin(), V.end(), std::mt19937(std::random_device()())); |
Dynamic Tools Team | 517193e | 2019-09-11 14:48:41 +0000 | [diff] [blame] | 73 | while (!V.empty()) { |
Vitaly Buka | 97742dd | 2021-07-01 11:56:11 -0700 | [diff] [blame] | 74 | L->deallocate(Options, V.back()); |
Dynamic Tools Team | 517193e | 2019-09-11 14:48:41 +0000 | [diff] [blame] | 75 | V.pop_back(); |
| 76 | } |
Kostya Kortchinsky | 53aea5c | 2021-06-03 12:11:05 -0700 | [diff] [blame] | 77 | scudo::ScopedString Str; |
Dynamic Tools Team | 3e8c65b | 2019-10-18 20:00:32 +0000 | [diff] [blame] | 78 | L->getStats(&Str); |
| 79 | Str.output(); |
Mitch Phillips | da86a54 | 2021-05-24 16:08:57 -0700 | [diff] [blame] | 80 | L->unmapTestOnly(); |
Dynamic Tools Team | 517193e | 2019-09-11 14:48:41 +0000 | [diff] [blame] | 81 | } |
| 82 | |
Peter Collingbourne | 4edf4ef | 2020-12-15 15:32:32 -0800 | [diff] [blame] | 83 | struct NoCacheConfig { |
| 84 | typedef scudo::MapAllocatorNoCache SecondaryCache; |
Peter Collingbourne | cc3d493 | 2020-12-21 18:39:03 -0800 | [diff] [blame] | 85 | static const bool MaySupportMemoryTagging = false; |
Peter Collingbourne | 4edf4ef | 2020-12-15 15:32:32 -0800 | [diff] [blame] | 86 | }; |
| 87 | |
Peter Collingbourne | 7488a17 | 2020-12-14 13:57:59 -0800 | [diff] [blame] | 88 | struct TestConfig { |
Peter Collingbourne | 4edf4ef | 2020-12-15 15:32:32 -0800 | [diff] [blame] | 89 | typedef scudo::MapAllocatorCache<TestConfig> SecondaryCache; |
Peter Collingbourne | cc3d493 | 2020-12-21 18:39:03 -0800 | [diff] [blame] | 90 | static const bool MaySupportMemoryTagging = false; |
Peter Collingbourne | 7488a17 | 2020-12-14 13:57:59 -0800 | [diff] [blame] | 91 | static const scudo::u32 SecondaryCacheEntriesArraySize = 128U; |
Peter Collingbourne | cc3d493 | 2020-12-21 18:39:03 -0800 | [diff] [blame] | 92 | static const scudo::u32 SecondaryCacheQuarantineSize = 0U; |
Peter Collingbourne | 7488a17 | 2020-12-14 13:57:59 -0800 | [diff] [blame] | 93 | static const scudo::u32 SecondaryCacheDefaultMaxEntriesCount = 64U; |
| 94 | static const scudo::uptr SecondaryCacheDefaultMaxEntrySize = 1UL << 20; |
| 95 | static const scudo::s32 SecondaryCacheMinReleaseToOsIntervalMs = INT32_MIN; |
| 96 | static const scudo::s32 SecondaryCacheMaxReleaseToOsIntervalMs = INT32_MAX; |
| 97 | }; |
| 98 | |
Dynamic Tools Team | d29271f | 2019-10-31 10:31:49 -0700 | [diff] [blame] | 99 | TEST(ScudoSecondaryTest, SecondaryBasic) { |
Peter Collingbourne | 4edf4ef | 2020-12-15 15:32:32 -0800 | [diff] [blame] | 100 | testSecondaryBasic<NoCacheConfig>(); |
| 101 | testSecondaryBasic<scudo::DefaultConfig>(); |
| 102 | testSecondaryBasic<TestConfig>(); |
Dynamic Tools Team | d29271f | 2019-10-31 10:31:49 -0700 | [diff] [blame] | 103 | } |
| 104 | |
Vitaly Buka | d319932 | 2021-07-01 10:22:35 -0700 | [diff] [blame] | 105 | struct MapAllocatorTest : public Test { |
Vitaly Buka | 97742dd | 2021-07-01 11:56:11 -0700 | [diff] [blame] | 106 | using Config = scudo::DefaultConfig; |
| 107 | using LargeAllocator = scudo::MapAllocator<Config>; |
| 108 | |
Vitaly Buka | d319932 | 2021-07-01 10:22:35 -0700 | [diff] [blame] | 109 | void SetUp() override { Allocator->init(nullptr); } |
| 110 | |
| 111 | void TearDown() override { Allocator->unmapTestOnly(); } |
| 112 | |
| 113 | std::unique_ptr<LargeAllocator> Allocator = |
| 114 | std::make_unique<LargeAllocator>(); |
Vitaly Buka | 97742dd | 2021-07-01 11:56:11 -0700 | [diff] [blame] | 115 | scudo::Options Options = getOptionsForConfig<Config>(); |
Vitaly Buka | d319932 | 2021-07-01 10:22:35 -0700 | [diff] [blame] | 116 | }; |
| 117 | |
Dynamic Tools Team | 517193e | 2019-09-11 14:48:41 +0000 | [diff] [blame] | 118 | // This exercises a variety of combinations of size and alignment for the |
| 119 | // MapAllocator. The size computation done here mimic the ones done by the |
| 120 | // combined allocator. |
Vitaly Buka | d319932 | 2021-07-01 10:22:35 -0700 | [diff] [blame] | 121 | TEST_F(MapAllocatorTest, SecondaryCombinations) { |
Dynamic Tools Team | 517193e | 2019-09-11 14:48:41 +0000 | [diff] [blame] | 122 | constexpr scudo::uptr MinAlign = FIRST_32_SECOND_64(8, 16); |
| 123 | constexpr scudo::uptr HeaderSize = scudo::roundUpTo(8, MinAlign); |
Dynamic Tools Team | 517193e | 2019-09-11 14:48:41 +0000 | [diff] [blame] | 124 | for (scudo::uptr SizeLog = 0; SizeLog <= 20; SizeLog++) { |
| 125 | for (scudo::uptr AlignLog = FIRST_32_SECOND_64(3, 4); AlignLog <= 16; |
| 126 | AlignLog++) { |
| 127 | const scudo::uptr Align = 1U << AlignLog; |
| 128 | for (scudo::sptr Delta = -128; Delta <= 128; Delta += 8) { |
| 129 | if (static_cast<scudo::sptr>(1U << SizeLog) + Delta <= 0) |
| 130 | continue; |
| 131 | const scudo::uptr UserSize = |
| 132 | scudo::roundUpTo((1U << SizeLog) + Delta, MinAlign); |
| 133 | const scudo::uptr Size = |
| 134 | HeaderSize + UserSize + (Align > MinAlign ? Align - HeaderSize : 0); |
Vitaly Buka | d319932 | 2021-07-01 10:22:35 -0700 | [diff] [blame] | 135 | void *P = Allocator->allocate(Options, Size, Align); |
Dynamic Tools Team | 517193e | 2019-09-11 14:48:41 +0000 | [diff] [blame] | 136 | EXPECT_NE(P, nullptr); |
| 137 | void *AlignedP = reinterpret_cast<void *>( |
| 138 | scudo::roundUpTo(reinterpret_cast<scudo::uptr>(P), Align)); |
| 139 | memset(AlignedP, 0xff, UserSize); |
Vitaly Buka | d319932 | 2021-07-01 10:22:35 -0700 | [diff] [blame] | 140 | Allocator->deallocate(Options, P); |
Dynamic Tools Team | 517193e | 2019-09-11 14:48:41 +0000 | [diff] [blame] | 141 | } |
| 142 | } |
| 143 | } |
Kostya Kortchinsky | 53aea5c | 2021-06-03 12:11:05 -0700 | [diff] [blame] | 144 | scudo::ScopedString Str; |
Vitaly Buka | d319932 | 2021-07-01 10:22:35 -0700 | [diff] [blame] | 145 | Allocator->getStats(&Str); |
Dynamic Tools Team | 3e8c65b | 2019-10-18 20:00:32 +0000 | [diff] [blame] | 146 | Str.output(); |
Dynamic Tools Team | 517193e | 2019-09-11 14:48:41 +0000 | [diff] [blame] | 147 | } |
| 148 | |
Vitaly Buka | d319932 | 2021-07-01 10:22:35 -0700 | [diff] [blame] | 149 | TEST_F(MapAllocatorTest, SecondaryIterate) { |
Dynamic Tools Team | 517193e | 2019-09-11 14:48:41 +0000 | [diff] [blame] | 150 | std::vector<void *> V; |
| 151 | const scudo::uptr PageSize = scudo::getPageSizeCached(); |
| 152 | for (scudo::uptr I = 0; I < 32U; I++) |
Vitaly Buka | d319932 | 2021-07-01 10:22:35 -0700 | [diff] [blame] | 153 | V.push_back(Allocator->allocate(Options, (std::rand() % 16) * PageSize)); |
Dynamic Tools Team | 517193e | 2019-09-11 14:48:41 +0000 | [diff] [blame] | 154 | auto Lambda = [V](scudo::uptr Block) { |
| 155 | EXPECT_NE(std::find(V.begin(), V.end(), reinterpret_cast<void *>(Block)), |
| 156 | V.end()); |
| 157 | }; |
Vitaly Buka | d319932 | 2021-07-01 10:22:35 -0700 | [diff] [blame] | 158 | Allocator->disable(); |
| 159 | Allocator->iterateOverBlocks(Lambda); |
| 160 | Allocator->enable(); |
Dynamic Tools Team | 517193e | 2019-09-11 14:48:41 +0000 | [diff] [blame] | 161 | while (!V.empty()) { |
Vitaly Buka | d319932 | 2021-07-01 10:22:35 -0700 | [diff] [blame] | 162 | Allocator->deallocate(Options, V.back()); |
Dynamic Tools Team | 517193e | 2019-09-11 14:48:41 +0000 | [diff] [blame] | 163 | V.pop_back(); |
| 164 | } |
Kostya Kortchinsky | 53aea5c | 2021-06-03 12:11:05 -0700 | [diff] [blame] | 165 | scudo::ScopedString Str; |
Vitaly Buka | d319932 | 2021-07-01 10:22:35 -0700 | [diff] [blame] | 166 | Allocator->getStats(&Str); |
Dynamic Tools Team | 3e8c65b | 2019-10-18 20:00:32 +0000 | [diff] [blame] | 167 | Str.output(); |
Dynamic Tools Team | 517193e | 2019-09-11 14:48:41 +0000 | [diff] [blame] | 168 | } |
| 169 | |
Vitaly Buka | d319932 | 2021-07-01 10:22:35 -0700 | [diff] [blame] | 170 | TEST_F(MapAllocatorTest, SecondaryOptions) { |
Kostya Kortchinsky | c72ca56 | 2020-07-27 09:13:42 -0700 | [diff] [blame] | 171 | // Attempt to set a maximum number of entries higher than the array size. |
Vitaly Buka | d319932 | 2021-07-01 10:22:35 -0700 | [diff] [blame] | 172 | EXPECT_FALSE( |
| 173 | Allocator->setOption(scudo::Option::MaxCacheEntriesCount, 4096U)); |
Kostya Kortchinsky | c72ca56 | 2020-07-27 09:13:42 -0700 | [diff] [blame] | 174 | // A negative number will be cast to a scudo::u32, and fail. |
Vitaly Buka | d319932 | 2021-07-01 10:22:35 -0700 | [diff] [blame] | 175 | EXPECT_FALSE(Allocator->setOption(scudo::Option::MaxCacheEntriesCount, -1)); |
| 176 | if (Allocator->canCache(0U)) { |
Kostya Kortchinsky | c72ca56 | 2020-07-27 09:13:42 -0700 | [diff] [blame] | 177 | // Various valid combinations. |
Vitaly Buka | d319932 | 2021-07-01 10:22:35 -0700 | [diff] [blame] | 178 | EXPECT_TRUE(Allocator->setOption(scudo::Option::MaxCacheEntriesCount, 4U)); |
| 179 | EXPECT_TRUE( |
| 180 | Allocator->setOption(scudo::Option::MaxCacheEntrySize, 1UL << 20)); |
| 181 | EXPECT_TRUE(Allocator->canCache(1UL << 18)); |
| 182 | EXPECT_TRUE( |
| 183 | Allocator->setOption(scudo::Option::MaxCacheEntrySize, 1UL << 17)); |
| 184 | EXPECT_FALSE(Allocator->canCache(1UL << 18)); |
| 185 | EXPECT_TRUE(Allocator->canCache(1UL << 16)); |
| 186 | EXPECT_TRUE(Allocator->setOption(scudo::Option::MaxCacheEntriesCount, 0U)); |
| 187 | EXPECT_FALSE(Allocator->canCache(1UL << 16)); |
| 188 | EXPECT_TRUE(Allocator->setOption(scudo::Option::MaxCacheEntriesCount, 4U)); |
| 189 | EXPECT_TRUE( |
| 190 | Allocator->setOption(scudo::Option::MaxCacheEntrySize, 1UL << 20)); |
| 191 | EXPECT_TRUE(Allocator->canCache(1UL << 16)); |
Dynamic Tools Team | 517193e | 2019-09-11 14:48:41 +0000 | [diff] [blame] | 192 | } |
| 193 | } |
| 194 | |
Vitaly Buka | d319932 | 2021-07-01 10:22:35 -0700 | [diff] [blame] | 195 | struct MapAllocatorWithReleaseTest : public MapAllocatorTest { |
| 196 | void SetUp() override { Allocator->init(nullptr, /*ReleaseToOsInterval=*/0); } |
| 197 | |
| 198 | void performAllocations() { |
| 199 | std::vector<void *> V; |
| 200 | const scudo::uptr PageSize = scudo::getPageSizeCached(); |
| 201 | { |
| 202 | std::unique_lock<std::mutex> Lock(Mutex); |
| 203 | while (!Ready) |
| 204 | Cv.wait(Lock); |
| 205 | } |
| 206 | for (scudo::uptr I = 0; I < 128U; I++) { |
| 207 | // Deallocate 75% of the blocks. |
| 208 | const bool Deallocate = (rand() & 3) != 0; |
| 209 | void *P = Allocator->allocate(Options, (std::rand() % 16) * PageSize); |
| 210 | if (Deallocate) |
| 211 | Allocator->deallocate(Options, P); |
| 212 | else |
| 213 | V.push_back(P); |
| 214 | } |
| 215 | while (!V.empty()) { |
| 216 | Allocator->deallocate(Options, V.back()); |
| 217 | V.pop_back(); |
| 218 | } |
| 219 | } |
| 220 | |
| 221 | std::mutex Mutex; |
| 222 | std::condition_variable Cv; |
| 223 | bool Ready = false; |
| 224 | }; |
| 225 | |
| 226 | TEST_F(MapAllocatorWithReleaseTest, SecondaryThreadsRace) { |
Dynamic Tools Team | fa69c70 | 2020-02-05 09:58:52 -0800 | [diff] [blame] | 227 | std::thread Threads[16]; |
| 228 | for (scudo::uptr I = 0; I < ARRAY_SIZE(Threads); I++) |
Vitaly Buka | d319932 | 2021-07-01 10:22:35 -0700 | [diff] [blame] | 229 | Threads[I] = |
| 230 | std::thread(&MapAllocatorWithReleaseTest::performAllocations, this); |
Dynamic Tools Team | 517193e | 2019-09-11 14:48:41 +0000 | [diff] [blame] | 231 | { |
| 232 | std::unique_lock<std::mutex> Lock(Mutex); |
| 233 | Ready = true; |
| 234 | Cv.notify_all(); |
| 235 | } |
| 236 | for (auto &T : Threads) |
| 237 | T.join(); |
Kostya Kortchinsky | 53aea5c | 2021-06-03 12:11:05 -0700 | [diff] [blame] | 238 | scudo::ScopedString Str; |
Vitaly Buka | d319932 | 2021-07-01 10:22:35 -0700 | [diff] [blame] | 239 | Allocator->getStats(&Str); |
Dynamic Tools Team | 3e8c65b | 2019-10-18 20:00:32 +0000 | [diff] [blame] | 240 | Str.output(); |
Dynamic Tools Team | 517193e | 2019-09-11 14:48:41 +0000 | [diff] [blame] | 241 | } |