Dynamic Tools Team | 517193e | 2019-09-11 14:48:41 +0000 | [diff] [blame] | 1 | //===-- secondary_test.cpp --------------------------------------*- C++ -*-===// |
| 2 | // |
| 3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| 4 | // See https://llvm.org/LICENSE.txt for license information. |
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| 6 | // |
| 7 | //===----------------------------------------------------------------------===// |
| 8 | |
Vitaly Buka | ace22c7 | 2021-06-30 23:39:12 -0700 | [diff] [blame] | 9 | #include "memtag.h" |
Dynamic Tools Team | 09e6d48 | 2019-11-26 18:18:14 -0800 | [diff] [blame] | 10 | #include "tests/scudo_unit_test.h" |
Dynamic Tools Team | 517193e | 2019-09-11 14:48:41 +0000 | [diff] [blame] | 11 | |
Peter Collingbourne | 7488a17 | 2020-12-14 13:57:59 -0800 | [diff] [blame] | 12 | #include "allocator_config.h" |
Dynamic Tools Team | 09e6d48 | 2019-11-26 18:18:14 -0800 | [diff] [blame] | 13 | #include "secondary.h" |
Dynamic Tools Team | 517193e | 2019-09-11 14:48:41 +0000 | [diff] [blame] | 14 | |
Dynamic Tools Team | 517193e | 2019-09-11 14:48:41 +0000 | [diff] [blame] | 15 | #include <condition_variable> |
Vitaly Buka | d319932 | 2021-07-01 10:22:35 -0700 | [diff] [blame] | 16 | #include <memory> |
Dynamic Tools Team | 517193e | 2019-09-11 14:48:41 +0000 | [diff] [blame] | 17 | #include <mutex> |
Dynamic Tools Team | d2740c5 | 2019-11-19 13:58:06 -0800 | [diff] [blame] | 18 | #include <random> |
Vitaly Buka | d319932 | 2021-07-01 10:22:35 -0700 | [diff] [blame] | 19 | #include <stdio.h> |
Dynamic Tools Team | 517193e | 2019-09-11 14:48:41 +0000 | [diff] [blame] | 20 | #include <thread> |
Dynamic Tools Team | 09e6d48 | 2019-11-26 18:18:14 -0800 | [diff] [blame] | 21 | #include <vector> |
Dynamic Tools Team | 517193e | 2019-09-11 14:48:41 +0000 | [diff] [blame] | 22 | |
Vitaly Buka | 97742dd | 2021-07-01 11:56:11 -0700 | [diff] [blame] | 23 | template <typename Config> static scudo::Options getOptionsForConfig() { |
Vitaly Buka | 7878d87 | 2021-07-01 21:40:04 -0700 | [diff] [blame^] | 24 | if (!Config::MaySupportMemoryTagging || !scudo::archSupportsMemoryTagging() || |
| 25 | !scudo::systemSupportsMemoryTagging()) |
Vitaly Buka | ace22c7 | 2021-06-30 23:39:12 -0700 | [diff] [blame] | 26 | return {}; |
| 27 | scudo::AtomicOptions AO; |
| 28 | AO.set(scudo::OptionBit::UseMemoryTagging); |
| 29 | return AO.load(); |
Vitaly Buka | 97742dd | 2021-07-01 11:56:11 -0700 | [diff] [blame] | 30 | } |
| 31 | |
Peter Collingbourne | 4edf4ef | 2020-12-15 15:32:32 -0800 | [diff] [blame] | 32 | template <typename Config> static void testSecondaryBasic(void) { |
| 33 | using SecondaryT = scudo::MapAllocator<Config>; |
Vitaly Buka | 97742dd | 2021-07-01 11:56:11 -0700 | [diff] [blame] | 34 | scudo::Options Options = getOptionsForConfig<Config>(); |
Peter Collingbourne | 4edf4ef | 2020-12-15 15:32:32 -0800 | [diff] [blame] | 35 | |
Dynamic Tools Team | 517193e | 2019-09-11 14:48:41 +0000 | [diff] [blame] | 36 | scudo::GlobalStats S; |
| 37 | S.init(); |
Kostya Kortchinsky | c72ca56 | 2020-07-27 09:13:42 -0700 | [diff] [blame] | 38 | std::unique_ptr<SecondaryT> L(new SecondaryT); |
Dynamic Tools Team | 517193e | 2019-09-11 14:48:41 +0000 | [diff] [blame] | 39 | L->init(&S); |
| 40 | const scudo::uptr Size = 1U << 16; |
Vitaly Buka | 97742dd | 2021-07-01 11:56:11 -0700 | [diff] [blame] | 41 | void *P = L->allocate(Options, Size); |
Dynamic Tools Team | 517193e | 2019-09-11 14:48:41 +0000 | [diff] [blame] | 42 | EXPECT_NE(P, nullptr); |
| 43 | memset(P, 'A', Size); |
Dynamic Tools Team | d29271f | 2019-10-31 10:31:49 -0700 | [diff] [blame] | 44 | EXPECT_GE(SecondaryT::getBlockSize(P), Size); |
Vitaly Buka | 97742dd | 2021-07-01 11:56:11 -0700 | [diff] [blame] | 45 | L->deallocate(Options, P); |
Mitch Phillips | a99b779 | 2021-05-20 10:56:47 -0700 | [diff] [blame] | 46 | |
Dynamic Tools Team | c5d5abc | 2020-01-27 14:03:21 -0800 | [diff] [blame] | 47 | // If the Secondary can't cache that pointer, it will be unmapped. |
Vitaly Buka | 5cb9636 | 2021-05-20 16:17:25 -0700 | [diff] [blame] | 48 | if (!L->canCache(Size)) { |
| 49 | EXPECT_DEATH( |
| 50 | { |
| 51 | // Repeat few time to avoid missing crash if it's mmaped by unrelated |
| 52 | // code. |
| 53 | for (int i = 0; i < 10; ++i) { |
Vitaly Buka | 97742dd | 2021-07-01 11:56:11 -0700 | [diff] [blame] | 54 | P = L->allocate(Options, Size); |
| 55 | L->deallocate(Options, P); |
Vitaly Buka | 5cb9636 | 2021-05-20 16:17:25 -0700 | [diff] [blame] | 56 | memset(P, 'A', Size); |
| 57 | } |
| 58 | }, |
| 59 | ""); |
Vitaly Buka | 5cb9636 | 2021-05-20 16:17:25 -0700 | [diff] [blame] | 60 | } |
Dynamic Tools Team | 517193e | 2019-09-11 14:48:41 +0000 | [diff] [blame] | 61 | |
| 62 | const scudo::uptr Align = 1U << 16; |
Vitaly Buka | 97742dd | 2021-07-01 11:56:11 -0700 | [diff] [blame] | 63 | P = L->allocate(Options, Size + Align, Align); |
Dynamic Tools Team | 517193e | 2019-09-11 14:48:41 +0000 | [diff] [blame] | 64 | EXPECT_NE(P, nullptr); |
| 65 | void *AlignedP = reinterpret_cast<void *>( |
| 66 | scudo::roundUpTo(reinterpret_cast<scudo::uptr>(P), Align)); |
| 67 | memset(AlignedP, 'A', Size); |
Vitaly Buka | 97742dd | 2021-07-01 11:56:11 -0700 | [diff] [blame] | 68 | L->deallocate(Options, P); |
Dynamic Tools Team | 517193e | 2019-09-11 14:48:41 +0000 | [diff] [blame] | 69 | |
| 70 | std::vector<void *> V; |
| 71 | for (scudo::uptr I = 0; I < 32U; I++) |
Vitaly Buka | 97742dd | 2021-07-01 11:56:11 -0700 | [diff] [blame] | 72 | V.push_back(L->allocate(Options, Size)); |
Dynamic Tools Team | d2740c5 | 2019-11-19 13:58:06 -0800 | [diff] [blame] | 73 | std::shuffle(V.begin(), V.end(), std::mt19937(std::random_device()())); |
Dynamic Tools Team | 517193e | 2019-09-11 14:48:41 +0000 | [diff] [blame] | 74 | while (!V.empty()) { |
Vitaly Buka | 97742dd | 2021-07-01 11:56:11 -0700 | [diff] [blame] | 75 | L->deallocate(Options, V.back()); |
Dynamic Tools Team | 517193e | 2019-09-11 14:48:41 +0000 | [diff] [blame] | 76 | V.pop_back(); |
| 77 | } |
Kostya Kortchinsky | 53aea5c | 2021-06-03 12:11:05 -0700 | [diff] [blame] | 78 | scudo::ScopedString Str; |
Dynamic Tools Team | 3e8c65b | 2019-10-18 20:00:32 +0000 | [diff] [blame] | 79 | L->getStats(&Str); |
| 80 | Str.output(); |
Mitch Phillips | da86a54 | 2021-05-24 16:08:57 -0700 | [diff] [blame] | 81 | L->unmapTestOnly(); |
Dynamic Tools Team | 517193e | 2019-09-11 14:48:41 +0000 | [diff] [blame] | 82 | } |
| 83 | |
Peter Collingbourne | 4edf4ef | 2020-12-15 15:32:32 -0800 | [diff] [blame] | 84 | struct NoCacheConfig { |
| 85 | typedef scudo::MapAllocatorNoCache SecondaryCache; |
Peter Collingbourne | cc3d493 | 2020-12-21 18:39:03 -0800 | [diff] [blame] | 86 | static const bool MaySupportMemoryTagging = false; |
Peter Collingbourne | 4edf4ef | 2020-12-15 15:32:32 -0800 | [diff] [blame] | 87 | }; |
| 88 | |
Peter Collingbourne | 7488a17 | 2020-12-14 13:57:59 -0800 | [diff] [blame] | 89 | struct TestConfig { |
Peter Collingbourne | 4edf4ef | 2020-12-15 15:32:32 -0800 | [diff] [blame] | 90 | typedef scudo::MapAllocatorCache<TestConfig> SecondaryCache; |
Peter Collingbourne | cc3d493 | 2020-12-21 18:39:03 -0800 | [diff] [blame] | 91 | static const bool MaySupportMemoryTagging = false; |
Peter Collingbourne | 7488a17 | 2020-12-14 13:57:59 -0800 | [diff] [blame] | 92 | static const scudo::u32 SecondaryCacheEntriesArraySize = 128U; |
Peter Collingbourne | cc3d493 | 2020-12-21 18:39:03 -0800 | [diff] [blame] | 93 | static const scudo::u32 SecondaryCacheQuarantineSize = 0U; |
Peter Collingbourne | 7488a17 | 2020-12-14 13:57:59 -0800 | [diff] [blame] | 94 | static const scudo::u32 SecondaryCacheDefaultMaxEntriesCount = 64U; |
| 95 | static const scudo::uptr SecondaryCacheDefaultMaxEntrySize = 1UL << 20; |
| 96 | static const scudo::s32 SecondaryCacheMinReleaseToOsIntervalMs = INT32_MIN; |
| 97 | static const scudo::s32 SecondaryCacheMaxReleaseToOsIntervalMs = INT32_MAX; |
| 98 | }; |
| 99 | |
Dynamic Tools Team | d29271f | 2019-10-31 10:31:49 -0700 | [diff] [blame] | 100 | TEST(ScudoSecondaryTest, SecondaryBasic) { |
Peter Collingbourne | 4edf4ef | 2020-12-15 15:32:32 -0800 | [diff] [blame] | 101 | testSecondaryBasic<NoCacheConfig>(); |
| 102 | testSecondaryBasic<scudo::DefaultConfig>(); |
| 103 | testSecondaryBasic<TestConfig>(); |
Dynamic Tools Team | d29271f | 2019-10-31 10:31:49 -0700 | [diff] [blame] | 104 | } |
| 105 | |
Vitaly Buka | d319932 | 2021-07-01 10:22:35 -0700 | [diff] [blame] | 106 | struct MapAllocatorTest : public Test { |
Vitaly Buka | 97742dd | 2021-07-01 11:56:11 -0700 | [diff] [blame] | 107 | using Config = scudo::DefaultConfig; |
| 108 | using LargeAllocator = scudo::MapAllocator<Config>; |
| 109 | |
Vitaly Buka | d319932 | 2021-07-01 10:22:35 -0700 | [diff] [blame] | 110 | void SetUp() override { Allocator->init(nullptr); } |
| 111 | |
| 112 | void TearDown() override { Allocator->unmapTestOnly(); } |
| 113 | |
| 114 | std::unique_ptr<LargeAllocator> Allocator = |
| 115 | std::make_unique<LargeAllocator>(); |
Vitaly Buka | 97742dd | 2021-07-01 11:56:11 -0700 | [diff] [blame] | 116 | scudo::Options Options = getOptionsForConfig<Config>(); |
Vitaly Buka | d319932 | 2021-07-01 10:22:35 -0700 | [diff] [blame] | 117 | }; |
| 118 | |
Dynamic Tools Team | 517193e | 2019-09-11 14:48:41 +0000 | [diff] [blame] | 119 | // This exercises a variety of combinations of size and alignment for the |
| 120 | // MapAllocator. The size computation done here mimic the ones done by the |
| 121 | // combined allocator. |
Vitaly Buka | d319932 | 2021-07-01 10:22:35 -0700 | [diff] [blame] | 122 | TEST_F(MapAllocatorTest, SecondaryCombinations) { |
Dynamic Tools Team | 517193e | 2019-09-11 14:48:41 +0000 | [diff] [blame] | 123 | constexpr scudo::uptr MinAlign = FIRST_32_SECOND_64(8, 16); |
| 124 | constexpr scudo::uptr HeaderSize = scudo::roundUpTo(8, MinAlign); |
Dynamic Tools Team | 517193e | 2019-09-11 14:48:41 +0000 | [diff] [blame] | 125 | for (scudo::uptr SizeLog = 0; SizeLog <= 20; SizeLog++) { |
| 126 | for (scudo::uptr AlignLog = FIRST_32_SECOND_64(3, 4); AlignLog <= 16; |
| 127 | AlignLog++) { |
| 128 | const scudo::uptr Align = 1U << AlignLog; |
| 129 | for (scudo::sptr Delta = -128; Delta <= 128; Delta += 8) { |
| 130 | if (static_cast<scudo::sptr>(1U << SizeLog) + Delta <= 0) |
| 131 | continue; |
| 132 | const scudo::uptr UserSize = |
| 133 | scudo::roundUpTo((1U << SizeLog) + Delta, MinAlign); |
| 134 | const scudo::uptr Size = |
| 135 | HeaderSize + UserSize + (Align > MinAlign ? Align - HeaderSize : 0); |
Vitaly Buka | d319932 | 2021-07-01 10:22:35 -0700 | [diff] [blame] | 136 | void *P = Allocator->allocate(Options, Size, Align); |
Dynamic Tools Team | 517193e | 2019-09-11 14:48:41 +0000 | [diff] [blame] | 137 | EXPECT_NE(P, nullptr); |
| 138 | void *AlignedP = reinterpret_cast<void *>( |
| 139 | scudo::roundUpTo(reinterpret_cast<scudo::uptr>(P), Align)); |
| 140 | memset(AlignedP, 0xff, UserSize); |
Vitaly Buka | d319932 | 2021-07-01 10:22:35 -0700 | [diff] [blame] | 141 | Allocator->deallocate(Options, P); |
Dynamic Tools Team | 517193e | 2019-09-11 14:48:41 +0000 | [diff] [blame] | 142 | } |
| 143 | } |
| 144 | } |
Kostya Kortchinsky | 53aea5c | 2021-06-03 12:11:05 -0700 | [diff] [blame] | 145 | scudo::ScopedString Str; |
Vitaly Buka | d319932 | 2021-07-01 10:22:35 -0700 | [diff] [blame] | 146 | Allocator->getStats(&Str); |
Dynamic Tools Team | 3e8c65b | 2019-10-18 20:00:32 +0000 | [diff] [blame] | 147 | Str.output(); |
Dynamic Tools Team | 517193e | 2019-09-11 14:48:41 +0000 | [diff] [blame] | 148 | } |
| 149 | |
Vitaly Buka | d319932 | 2021-07-01 10:22:35 -0700 | [diff] [blame] | 150 | TEST_F(MapAllocatorTest, SecondaryIterate) { |
Dynamic Tools Team | 517193e | 2019-09-11 14:48:41 +0000 | [diff] [blame] | 151 | std::vector<void *> V; |
| 152 | const scudo::uptr PageSize = scudo::getPageSizeCached(); |
| 153 | for (scudo::uptr I = 0; I < 32U; I++) |
Vitaly Buka | d319932 | 2021-07-01 10:22:35 -0700 | [diff] [blame] | 154 | V.push_back(Allocator->allocate(Options, (std::rand() % 16) * PageSize)); |
Dynamic Tools Team | 517193e | 2019-09-11 14:48:41 +0000 | [diff] [blame] | 155 | auto Lambda = [V](scudo::uptr Block) { |
| 156 | EXPECT_NE(std::find(V.begin(), V.end(), reinterpret_cast<void *>(Block)), |
| 157 | V.end()); |
| 158 | }; |
Vitaly Buka | d319932 | 2021-07-01 10:22:35 -0700 | [diff] [blame] | 159 | Allocator->disable(); |
| 160 | Allocator->iterateOverBlocks(Lambda); |
| 161 | Allocator->enable(); |
Dynamic Tools Team | 517193e | 2019-09-11 14:48:41 +0000 | [diff] [blame] | 162 | while (!V.empty()) { |
Vitaly Buka | d319932 | 2021-07-01 10:22:35 -0700 | [diff] [blame] | 163 | Allocator->deallocate(Options, V.back()); |
Dynamic Tools Team | 517193e | 2019-09-11 14:48:41 +0000 | [diff] [blame] | 164 | V.pop_back(); |
| 165 | } |
Kostya Kortchinsky | 53aea5c | 2021-06-03 12:11:05 -0700 | [diff] [blame] | 166 | scudo::ScopedString Str; |
Vitaly Buka | d319932 | 2021-07-01 10:22:35 -0700 | [diff] [blame] | 167 | Allocator->getStats(&Str); |
Dynamic Tools Team | 3e8c65b | 2019-10-18 20:00:32 +0000 | [diff] [blame] | 168 | Str.output(); |
Dynamic Tools Team | 517193e | 2019-09-11 14:48:41 +0000 | [diff] [blame] | 169 | } |
| 170 | |
Vitaly Buka | d319932 | 2021-07-01 10:22:35 -0700 | [diff] [blame] | 171 | TEST_F(MapAllocatorTest, SecondaryOptions) { |
Kostya Kortchinsky | c72ca56 | 2020-07-27 09:13:42 -0700 | [diff] [blame] | 172 | // Attempt to set a maximum number of entries higher than the array size. |
Vitaly Buka | d319932 | 2021-07-01 10:22:35 -0700 | [diff] [blame] | 173 | EXPECT_FALSE( |
| 174 | Allocator->setOption(scudo::Option::MaxCacheEntriesCount, 4096U)); |
Kostya Kortchinsky | c72ca56 | 2020-07-27 09:13:42 -0700 | [diff] [blame] | 175 | // A negative number will be cast to a scudo::u32, and fail. |
Vitaly Buka | d319932 | 2021-07-01 10:22:35 -0700 | [diff] [blame] | 176 | EXPECT_FALSE(Allocator->setOption(scudo::Option::MaxCacheEntriesCount, -1)); |
| 177 | if (Allocator->canCache(0U)) { |
Kostya Kortchinsky | c72ca56 | 2020-07-27 09:13:42 -0700 | [diff] [blame] | 178 | // Various valid combinations. |
Vitaly Buka | d319932 | 2021-07-01 10:22:35 -0700 | [diff] [blame] | 179 | EXPECT_TRUE(Allocator->setOption(scudo::Option::MaxCacheEntriesCount, 4U)); |
| 180 | EXPECT_TRUE( |
| 181 | Allocator->setOption(scudo::Option::MaxCacheEntrySize, 1UL << 20)); |
| 182 | EXPECT_TRUE(Allocator->canCache(1UL << 18)); |
| 183 | EXPECT_TRUE( |
| 184 | Allocator->setOption(scudo::Option::MaxCacheEntrySize, 1UL << 17)); |
| 185 | EXPECT_FALSE(Allocator->canCache(1UL << 18)); |
| 186 | EXPECT_TRUE(Allocator->canCache(1UL << 16)); |
| 187 | EXPECT_TRUE(Allocator->setOption(scudo::Option::MaxCacheEntriesCount, 0U)); |
| 188 | EXPECT_FALSE(Allocator->canCache(1UL << 16)); |
| 189 | EXPECT_TRUE(Allocator->setOption(scudo::Option::MaxCacheEntriesCount, 4U)); |
| 190 | EXPECT_TRUE( |
| 191 | Allocator->setOption(scudo::Option::MaxCacheEntrySize, 1UL << 20)); |
| 192 | EXPECT_TRUE(Allocator->canCache(1UL << 16)); |
Dynamic Tools Team | 517193e | 2019-09-11 14:48:41 +0000 | [diff] [blame] | 193 | } |
| 194 | } |
| 195 | |
Vitaly Buka | d319932 | 2021-07-01 10:22:35 -0700 | [diff] [blame] | 196 | struct MapAllocatorWithReleaseTest : public MapAllocatorTest { |
| 197 | void SetUp() override { Allocator->init(nullptr, /*ReleaseToOsInterval=*/0); } |
| 198 | |
| 199 | void performAllocations() { |
| 200 | std::vector<void *> V; |
| 201 | const scudo::uptr PageSize = scudo::getPageSizeCached(); |
| 202 | { |
| 203 | std::unique_lock<std::mutex> Lock(Mutex); |
| 204 | while (!Ready) |
| 205 | Cv.wait(Lock); |
| 206 | } |
| 207 | for (scudo::uptr I = 0; I < 128U; I++) { |
| 208 | // Deallocate 75% of the blocks. |
| 209 | const bool Deallocate = (rand() & 3) != 0; |
| 210 | void *P = Allocator->allocate(Options, (std::rand() % 16) * PageSize); |
| 211 | if (Deallocate) |
| 212 | Allocator->deallocate(Options, P); |
| 213 | else |
| 214 | V.push_back(P); |
| 215 | } |
| 216 | while (!V.empty()) { |
| 217 | Allocator->deallocate(Options, V.back()); |
| 218 | V.pop_back(); |
| 219 | } |
| 220 | } |
| 221 | |
| 222 | std::mutex Mutex; |
| 223 | std::condition_variable Cv; |
| 224 | bool Ready = false; |
| 225 | }; |
| 226 | |
| 227 | TEST_F(MapAllocatorWithReleaseTest, SecondaryThreadsRace) { |
Dynamic Tools Team | fa69c70 | 2020-02-05 09:58:52 -0800 | [diff] [blame] | 228 | std::thread Threads[16]; |
| 229 | for (scudo::uptr I = 0; I < ARRAY_SIZE(Threads); I++) |
Vitaly Buka | d319932 | 2021-07-01 10:22:35 -0700 | [diff] [blame] | 230 | Threads[I] = |
| 231 | std::thread(&MapAllocatorWithReleaseTest::performAllocations, this); |
Dynamic Tools Team | 517193e | 2019-09-11 14:48:41 +0000 | [diff] [blame] | 232 | { |
| 233 | std::unique_lock<std::mutex> Lock(Mutex); |
| 234 | Ready = true; |
| 235 | Cv.notify_all(); |
| 236 | } |
| 237 | for (auto &T : Threads) |
| 238 | T.join(); |
Kostya Kortchinsky | 53aea5c | 2021-06-03 12:11:05 -0700 | [diff] [blame] | 239 | scudo::ScopedString Str; |
Vitaly Buka | d319932 | 2021-07-01 10:22:35 -0700 | [diff] [blame] | 240 | Allocator->getStats(&Str); |
Dynamic Tools Team | 3e8c65b | 2019-10-18 20:00:32 +0000 | [diff] [blame] | 241 | Str.output(); |
Dynamic Tools Team | 517193e | 2019-09-11 14:48:41 +0000 | [diff] [blame] | 242 | } |