Dynamic Tools Team | 517193e | 2019-09-11 14:48:41 +0000 | [diff] [blame] | 1 | //===-- primary64.h ---------------------------------------------*- C++ -*-===// |
| 2 | // |
| 3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| 4 | // See https://llvm.org/LICENSE.txt for license information. |
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| 6 | // |
| 7 | //===----------------------------------------------------------------------===// |
| 8 | |
| 9 | #ifndef SCUDO_PRIMARY64_H_ |
| 10 | #define SCUDO_PRIMARY64_H_ |
| 11 | |
| 12 | #include "bytemap.h" |
| 13 | #include "common.h" |
| 14 | #include "list.h" |
| 15 | #include "local_cache.h" |
Dynamic Tools Team | 48429c7 | 2019-12-04 17:46:15 -0800 | [diff] [blame^] | 16 | #include "memtag.h" |
Dynamic Tools Team | 517193e | 2019-09-11 14:48:41 +0000 | [diff] [blame] | 17 | #include "release.h" |
| 18 | #include "stats.h" |
| 19 | #include "string_utils.h" |
| 20 | |
| 21 | namespace scudo { |
| 22 | |
| 23 | // SizeClassAllocator64 is an allocator tuned for 64-bit address space. |
| 24 | // |
| 25 | // It starts by reserving NumClasses * 2^RegionSizeLog bytes, equally divided in |
| 26 | // Regions, specific to each size class. Note that the base of that mapping is |
| 27 | // random (based to the platform specific map() capabilities), and that each |
| 28 | // Region actually starts at a random offset from its base. |
| 29 | // |
| 30 | // Regions are mapped incrementally on demand to fulfill allocation requests, |
| 31 | // those mappings being split into equally sized Blocks based on the size class |
| 32 | // they belong to. The Blocks created are shuffled to prevent predictable |
| 33 | // address patterns (the predictability increases with the size of the Blocks). |
| 34 | // |
| 35 | // The 1st Region (for size class 0) holds the TransferBatches. This is a |
| 36 | // structure used to transfer arrays of available pointers from the class size |
| 37 | // freelist to the thread specific freelist, and back. |
| 38 | // |
| 39 | // The memory used by this allocator is never unmapped, but can be partially |
| 40 | // released if the platform allows for it. |
| 41 | |
Dynamic Tools Team | 48429c7 | 2019-12-04 17:46:15 -0800 | [diff] [blame^] | 42 | template <class SizeClassMapT, uptr RegionSizeLog, |
| 43 | bool MaySupportMemoryTagging = false> |
| 44 | class SizeClassAllocator64 { |
Dynamic Tools Team | 517193e | 2019-09-11 14:48:41 +0000 | [diff] [blame] | 45 | public: |
| 46 | typedef SizeClassMapT SizeClassMap; |
Dynamic Tools Team | 48429c7 | 2019-12-04 17:46:15 -0800 | [diff] [blame^] | 47 | typedef SizeClassAllocator64<SizeClassMap, RegionSizeLog, |
| 48 | MaySupportMemoryTagging> |
| 49 | ThisT; |
Dynamic Tools Team | 517193e | 2019-09-11 14:48:41 +0000 | [diff] [blame] | 50 | typedef SizeClassAllocatorLocalCache<ThisT> CacheT; |
| 51 | typedef typename CacheT::TransferBatch TransferBatch; |
Dynamic Tools Team | 48429c7 | 2019-12-04 17:46:15 -0800 | [diff] [blame^] | 52 | static const bool SupportsMemoryTagging = |
| 53 | MaySupportMemoryTagging && archSupportsMemoryTagging(); |
Dynamic Tools Team | 517193e | 2019-09-11 14:48:41 +0000 | [diff] [blame] | 54 | |
| 55 | static uptr getSizeByClassId(uptr ClassId) { |
| 56 | return (ClassId == SizeClassMap::BatchClassId) |
| 57 | ? sizeof(TransferBatch) |
| 58 | : SizeClassMap::getSizeByClassId(ClassId); |
| 59 | } |
| 60 | |
| 61 | static bool canAllocate(uptr Size) { return Size <= SizeClassMap::MaxSize; } |
| 62 | |
| 63 | void initLinkerInitialized(s32 ReleaseToOsInterval) { |
| 64 | // Reserve the space required for the Primary. |
| 65 | PrimaryBase = reinterpret_cast<uptr>( |
| 66 | map(nullptr, PrimarySize, "scudo:primary", MAP_NOACCESS, &Data)); |
| 67 | |
| 68 | RegionInfoArray = reinterpret_cast<RegionInfo *>( |
| 69 | map(nullptr, sizeof(RegionInfo) * NumClasses, "scudo:regioninfo")); |
| 70 | DCHECK_EQ(reinterpret_cast<uptr>(RegionInfoArray) % SCUDO_CACHE_LINE_SIZE, |
| 71 | 0); |
| 72 | |
| 73 | u32 Seed; |
| 74 | if (UNLIKELY(!getRandom(reinterpret_cast<void *>(&Seed), sizeof(Seed)))) |
| 75 | Seed = static_cast<u32>(getMonotonicTime() ^ (PrimaryBase >> 12)); |
| 76 | const uptr PageSize = getPageSizeCached(); |
| 77 | for (uptr I = 0; I < NumClasses; I++) { |
| 78 | RegionInfo *Region = getRegionInfo(I); |
| 79 | // The actual start of a region is offseted by a random number of pages. |
| 80 | Region->RegionBeg = |
| 81 | getRegionBaseByClassId(I) + (getRandomModN(&Seed, 16) + 1) * PageSize; |
| 82 | // Releasing smaller size classes doesn't necessarily yield to a |
| 83 | // meaningful RSS impact: there are more blocks per page, they are |
| 84 | // randomized around, and thus pages are less likely to be entirely empty. |
| 85 | // On top of this, attempting to release those require more iterations and |
| 86 | // memory accesses which ends up being fairly costly. The current lower |
| 87 | // limit is mostly arbitrary and based on empirical observations. |
| 88 | // TODO(kostyak): make the lower limit a runtime option |
Dynamic Tools Team | c283bab | 2019-10-07 17:37:39 +0000 | [diff] [blame] | 89 | Region->CanRelease = (ReleaseToOsInterval >= 0) && |
Dynamic Tools Team | 517193e | 2019-09-11 14:48:41 +0000 | [diff] [blame] | 90 | (I != SizeClassMap::BatchClassId) && |
Dynamic Tools Team | c283bab | 2019-10-07 17:37:39 +0000 | [diff] [blame] | 91 | (getSizeByClassId(I) >= (PageSize / 32)); |
Dynamic Tools Team | 517193e | 2019-09-11 14:48:41 +0000 | [diff] [blame] | 92 | Region->RandState = getRandomU32(&Seed); |
| 93 | } |
| 94 | ReleaseToOsIntervalMs = ReleaseToOsInterval; |
Dynamic Tools Team | 48429c7 | 2019-12-04 17:46:15 -0800 | [diff] [blame^] | 95 | |
| 96 | if (SupportsMemoryTagging) |
| 97 | UseMemoryTagging = systemSupportsMemoryTagging(); |
Dynamic Tools Team | 517193e | 2019-09-11 14:48:41 +0000 | [diff] [blame] | 98 | } |
| 99 | void init(s32 ReleaseToOsInterval) { |
| 100 | memset(this, 0, sizeof(*this)); |
| 101 | initLinkerInitialized(ReleaseToOsInterval); |
| 102 | } |
| 103 | |
| 104 | void unmapTestOnly() { |
| 105 | unmap(reinterpret_cast<void *>(PrimaryBase), PrimarySize, UNMAP_ALL, &Data); |
| 106 | unmap(reinterpret_cast<void *>(RegionInfoArray), |
| 107 | sizeof(RegionInfo) * NumClasses); |
| 108 | } |
| 109 | |
| 110 | TransferBatch *popBatch(CacheT *C, uptr ClassId) { |
| 111 | DCHECK_LT(ClassId, NumClasses); |
| 112 | RegionInfo *Region = getRegionInfo(ClassId); |
| 113 | ScopedLock L(Region->Mutex); |
| 114 | TransferBatch *B = Region->FreeList.front(); |
| 115 | if (B) { |
| 116 | Region->FreeList.pop_front(); |
| 117 | } else { |
| 118 | B = populateFreeList(C, ClassId, Region); |
| 119 | if (UNLIKELY(!B)) |
| 120 | return nullptr; |
| 121 | } |
| 122 | DCHECK_GT(B->getCount(), 0); |
| 123 | Region->Stats.PoppedBlocks += B->getCount(); |
| 124 | return B; |
| 125 | } |
| 126 | |
| 127 | void pushBatch(uptr ClassId, TransferBatch *B) { |
| 128 | DCHECK_GT(B->getCount(), 0); |
| 129 | RegionInfo *Region = getRegionInfo(ClassId); |
| 130 | ScopedLock L(Region->Mutex); |
| 131 | Region->FreeList.push_front(B); |
| 132 | Region->Stats.PushedBlocks += B->getCount(); |
| 133 | if (Region->CanRelease) |
| 134 | releaseToOSMaybe(Region, ClassId); |
| 135 | } |
| 136 | |
| 137 | void disable() { |
Dynamic Tools Team | 83eaa51 | 2020-01-09 11:43:16 -0800 | [diff] [blame] | 138 | // The BatchClassId must be locked last since other classes can use it. |
| 139 | for (sptr I = static_cast<sptr>(NumClasses) - 1; I >= 0; I--) { |
| 140 | if (static_cast<uptr>(I) == SizeClassMap::BatchClassId) |
| 141 | continue; |
| 142 | getRegionInfo(static_cast<uptr>(I))->Mutex.lock(); |
| 143 | } |
| 144 | getRegionInfo(SizeClassMap::BatchClassId)->Mutex.lock(); |
Dynamic Tools Team | 517193e | 2019-09-11 14:48:41 +0000 | [diff] [blame] | 145 | } |
| 146 | |
| 147 | void enable() { |
Dynamic Tools Team | 83eaa51 | 2020-01-09 11:43:16 -0800 | [diff] [blame] | 148 | getRegionInfo(SizeClassMap::BatchClassId)->Mutex.unlock(); |
| 149 | for (uptr I = 0; I < NumClasses; I++) { |
| 150 | if (I == SizeClassMap::BatchClassId) |
| 151 | continue; |
| 152 | getRegionInfo(I)->Mutex.unlock(); |
| 153 | } |
Dynamic Tools Team | 517193e | 2019-09-11 14:48:41 +0000 | [diff] [blame] | 154 | } |
| 155 | |
| 156 | template <typename F> void iterateOverBlocks(F Callback) const { |
| 157 | for (uptr I = 0; I < NumClasses; I++) { |
| 158 | if (I == SizeClassMap::BatchClassId) |
| 159 | continue; |
| 160 | const RegionInfo *Region = getRegionInfo(I); |
| 161 | const uptr BlockSize = getSizeByClassId(I); |
| 162 | const uptr From = Region->RegionBeg; |
| 163 | const uptr To = From + Region->AllocatedUser; |
| 164 | for (uptr Block = From; Block < To; Block += BlockSize) |
| 165 | Callback(Block); |
| 166 | } |
| 167 | } |
| 168 | |
Dynamic Tools Team | 3e8c65b | 2019-10-18 20:00:32 +0000 | [diff] [blame] | 169 | void getStats(ScopedString *Str) const { |
Dynamic Tools Team | 517193e | 2019-09-11 14:48:41 +0000 | [diff] [blame] | 170 | // TODO(kostyak): get the RSS per region. |
| 171 | uptr TotalMapped = 0; |
| 172 | uptr PoppedBlocks = 0; |
| 173 | uptr PushedBlocks = 0; |
| 174 | for (uptr I = 0; I < NumClasses; I++) { |
| 175 | RegionInfo *Region = getRegionInfo(I); |
| 176 | if (Region->MappedUser) |
| 177 | TotalMapped += Region->MappedUser; |
| 178 | PoppedBlocks += Region->Stats.PoppedBlocks; |
| 179 | PushedBlocks += Region->Stats.PushedBlocks; |
| 180 | } |
Dynamic Tools Team | 3e8c65b | 2019-10-18 20:00:32 +0000 | [diff] [blame] | 181 | Str->append("Stats: SizeClassAllocator64: %zuM mapped (%zuM rss) in %zu " |
| 182 | "allocations; remains %zu\n", |
| 183 | TotalMapped >> 20, 0, PoppedBlocks, |
| 184 | PoppedBlocks - PushedBlocks); |
Dynamic Tools Team | 517193e | 2019-09-11 14:48:41 +0000 | [diff] [blame] | 185 | |
| 186 | for (uptr I = 0; I < NumClasses; I++) |
Dynamic Tools Team | 3e8c65b | 2019-10-18 20:00:32 +0000 | [diff] [blame] | 187 | getStats(Str, I, 0); |
Dynamic Tools Team | 517193e | 2019-09-11 14:48:41 +0000 | [diff] [blame] | 188 | } |
| 189 | |
Dynamic Tools Team | c283bab | 2019-10-07 17:37:39 +0000 | [diff] [blame] | 190 | uptr releaseToOS() { |
| 191 | uptr TotalReleasedBytes = 0; |
Dynamic Tools Team | 517193e | 2019-09-11 14:48:41 +0000 | [diff] [blame] | 192 | for (uptr I = 0; I < NumClasses; I++) { |
| 193 | if (I == SizeClassMap::BatchClassId) |
| 194 | continue; |
| 195 | RegionInfo *Region = getRegionInfo(I); |
| 196 | ScopedLock L(Region->Mutex); |
Dynamic Tools Team | c283bab | 2019-10-07 17:37:39 +0000 | [diff] [blame] | 197 | TotalReleasedBytes += releaseToOSMaybe(Region, I, /*Force=*/true); |
Dynamic Tools Team | 517193e | 2019-09-11 14:48:41 +0000 | [diff] [blame] | 198 | } |
Dynamic Tools Team | c283bab | 2019-10-07 17:37:39 +0000 | [diff] [blame] | 199 | return TotalReleasedBytes; |
Dynamic Tools Team | 517193e | 2019-09-11 14:48:41 +0000 | [diff] [blame] | 200 | } |
| 201 | |
Dynamic Tools Team | 48429c7 | 2019-12-04 17:46:15 -0800 | [diff] [blame^] | 202 | bool useMemoryTagging() const { |
| 203 | return SupportsMemoryTagging && UseMemoryTagging; |
| 204 | } |
| 205 | void disableMemoryTagging() { UseMemoryTagging = false; } |
| 206 | |
Dynamic Tools Team | 517193e | 2019-09-11 14:48:41 +0000 | [diff] [blame] | 207 | private: |
| 208 | static const uptr RegionSize = 1UL << RegionSizeLog; |
| 209 | static const uptr NumClasses = SizeClassMap::NumClasses; |
| 210 | static const uptr PrimarySize = RegionSize * NumClasses; |
| 211 | |
| 212 | // Call map for user memory with at least this size. |
| 213 | static const uptr MapSizeIncrement = 1UL << 17; |
Dynamic Tools Team | aa15246 | 2019-11-19 10:18:38 -0800 | [diff] [blame] | 214 | // Fill at most this number of batches from the newly map'd memory. |
| 215 | static const u32 MaxNumBatches = 8U; |
Dynamic Tools Team | 517193e | 2019-09-11 14:48:41 +0000 | [diff] [blame] | 216 | |
| 217 | struct RegionStats { |
| 218 | uptr PoppedBlocks; |
| 219 | uptr PushedBlocks; |
| 220 | }; |
| 221 | |
| 222 | struct ReleaseToOsInfo { |
| 223 | uptr PushedBlocksAtLastRelease; |
| 224 | uptr RangesReleased; |
| 225 | uptr LastReleasedBytes; |
| 226 | u64 LastReleaseAtNs; |
| 227 | }; |
| 228 | |
| 229 | struct ALIGNED(SCUDO_CACHE_LINE_SIZE) RegionInfo { |
| 230 | HybridMutex Mutex; |
Dynamic Tools Team | 3599770 | 2019-10-28 15:06:10 -0700 | [diff] [blame] | 231 | SinglyLinkedList<TransferBatch> FreeList; |
Dynamic Tools Team | 517193e | 2019-09-11 14:48:41 +0000 | [diff] [blame] | 232 | RegionStats Stats; |
| 233 | bool CanRelease; |
| 234 | bool Exhausted; |
| 235 | u32 RandState; |
| 236 | uptr RegionBeg; |
| 237 | uptr MappedUser; // Bytes mapped for user memory. |
| 238 | uptr AllocatedUser; // Bytes allocated for user memory. |
| 239 | MapPlatformData Data; |
| 240 | ReleaseToOsInfo ReleaseInfo; |
| 241 | }; |
Dynamic Tools Team | 09e6d48 | 2019-11-26 18:18:14 -0800 | [diff] [blame] | 242 | static_assert(sizeof(RegionInfo) % SCUDO_CACHE_LINE_SIZE == 0, ""); |
Dynamic Tools Team | 517193e | 2019-09-11 14:48:41 +0000 | [diff] [blame] | 243 | |
| 244 | uptr PrimaryBase; |
| 245 | RegionInfo *RegionInfoArray; |
| 246 | MapPlatformData Data; |
| 247 | s32 ReleaseToOsIntervalMs; |
Dynamic Tools Team | 48429c7 | 2019-12-04 17:46:15 -0800 | [diff] [blame^] | 248 | bool UseMemoryTagging; |
Dynamic Tools Team | 517193e | 2019-09-11 14:48:41 +0000 | [diff] [blame] | 249 | |
| 250 | RegionInfo *getRegionInfo(uptr ClassId) const { |
| 251 | DCHECK_LT(ClassId, NumClasses); |
| 252 | return &RegionInfoArray[ClassId]; |
| 253 | } |
| 254 | |
| 255 | uptr getRegionBaseByClassId(uptr ClassId) const { |
| 256 | return PrimaryBase + (ClassId << RegionSizeLog); |
| 257 | } |
| 258 | |
| 259 | bool populateBatches(CacheT *C, RegionInfo *Region, uptr ClassId, |
| 260 | TransferBatch **CurrentBatch, u32 MaxCount, |
| 261 | void **PointersArray, u32 Count) { |
| 262 | // No need to shuffle the batches size class. |
| 263 | if (ClassId != SizeClassMap::BatchClassId) |
| 264 | shuffle(PointersArray, Count, &Region->RandState); |
| 265 | TransferBatch *B = *CurrentBatch; |
| 266 | for (uptr I = 0; I < Count; I++) { |
| 267 | if (B && B->getCount() == MaxCount) { |
| 268 | Region->FreeList.push_back(B); |
| 269 | B = nullptr; |
| 270 | } |
| 271 | if (!B) { |
| 272 | B = C->createBatch(ClassId, PointersArray[I]); |
| 273 | if (UNLIKELY(!B)) |
| 274 | return false; |
| 275 | B->clear(); |
| 276 | } |
| 277 | B->add(PointersArray[I]); |
| 278 | } |
| 279 | *CurrentBatch = B; |
| 280 | return true; |
| 281 | } |
| 282 | |
| 283 | NOINLINE TransferBatch *populateFreeList(CacheT *C, uptr ClassId, |
| 284 | RegionInfo *Region) { |
| 285 | const uptr Size = getSizeByClassId(ClassId); |
| 286 | const u32 MaxCount = TransferBatch::getMaxCached(Size); |
| 287 | |
| 288 | const uptr RegionBeg = Region->RegionBeg; |
| 289 | const uptr MappedUser = Region->MappedUser; |
| 290 | const uptr TotalUserBytes = Region->AllocatedUser + MaxCount * Size; |
| 291 | // Map more space for blocks, if necessary. |
Dynamic Tools Team | c283bab | 2019-10-07 17:37:39 +0000 | [diff] [blame] | 292 | if (TotalUserBytes > MappedUser) { |
Dynamic Tools Team | 517193e | 2019-09-11 14:48:41 +0000 | [diff] [blame] | 293 | // Do the mmap for the user memory. |
| 294 | const uptr UserMapSize = |
| 295 | roundUpTo(TotalUserBytes - MappedUser, MapSizeIncrement); |
| 296 | const uptr RegionBase = RegionBeg - getRegionBaseByClassId(ClassId); |
| 297 | if (UNLIKELY(RegionBase + MappedUser + UserMapSize > RegionSize)) { |
| 298 | if (!Region->Exhausted) { |
| 299 | Region->Exhausted = true; |
Dynamic Tools Team | 3e8c65b | 2019-10-18 20:00:32 +0000 | [diff] [blame] | 300 | ScopedString Str(1024); |
| 301 | getStats(&Str); |
| 302 | Str.append( |
Dynamic Tools Team | 517193e | 2019-09-11 14:48:41 +0000 | [diff] [blame] | 303 | "Scudo OOM: The process has Exhausted %zuM for size class %zu.\n", |
| 304 | RegionSize >> 20, Size); |
Dynamic Tools Team | 3e8c65b | 2019-10-18 20:00:32 +0000 | [diff] [blame] | 305 | Str.output(); |
Dynamic Tools Team | 517193e | 2019-09-11 14:48:41 +0000 | [diff] [blame] | 306 | } |
| 307 | return nullptr; |
| 308 | } |
| 309 | if (UNLIKELY(MappedUser == 0)) |
| 310 | Region->Data = Data; |
| 311 | if (UNLIKELY(!map(reinterpret_cast<void *>(RegionBeg + MappedUser), |
| 312 | UserMapSize, "scudo:primary", |
Dynamic Tools Team | 48429c7 | 2019-12-04 17:46:15 -0800 | [diff] [blame^] | 313 | MAP_ALLOWNOMEM | MAP_RESIZABLE | |
| 314 | (useMemoryTagging() ? MAP_MEMTAG : 0), |
| 315 | &Region->Data))) |
Dynamic Tools Team | 517193e | 2019-09-11 14:48:41 +0000 | [diff] [blame] | 316 | return nullptr; |
| 317 | Region->MappedUser += UserMapSize; |
| 318 | C->getStats().add(StatMapped, UserMapSize); |
| 319 | } |
| 320 | |
Dynamic Tools Team | aa15246 | 2019-11-19 10:18:38 -0800 | [diff] [blame] | 321 | const u32 NumberOfBlocks = Min( |
| 322 | MaxNumBatches * MaxCount, |
| 323 | static_cast<u32>((Region->MappedUser - Region->AllocatedUser) / Size)); |
Dynamic Tools Team | 517193e | 2019-09-11 14:48:41 +0000 | [diff] [blame] | 324 | DCHECK_GT(NumberOfBlocks, 0); |
| 325 | |
| 326 | TransferBatch *B = nullptr; |
Dynamic Tools Team | aa15246 | 2019-11-19 10:18:38 -0800 | [diff] [blame] | 327 | constexpr u32 ShuffleArraySize = |
| 328 | MaxNumBatches * TransferBatch::MaxNumCached; |
Dynamic Tools Team | 517193e | 2019-09-11 14:48:41 +0000 | [diff] [blame] | 329 | void *ShuffleArray[ShuffleArraySize]; |
| 330 | u32 Count = 0; |
| 331 | const uptr P = RegionBeg + Region->AllocatedUser; |
Dynamic Tools Team | aa15246 | 2019-11-19 10:18:38 -0800 | [diff] [blame] | 332 | const uptr AllocatedUser = Size * NumberOfBlocks; |
Dynamic Tools Team | 517193e | 2019-09-11 14:48:41 +0000 | [diff] [blame] | 333 | for (uptr I = P; I < P + AllocatedUser; I += Size) { |
| 334 | ShuffleArray[Count++] = reinterpret_cast<void *>(I); |
| 335 | if (Count == ShuffleArraySize) { |
| 336 | if (UNLIKELY(!populateBatches(C, Region, ClassId, &B, MaxCount, |
| 337 | ShuffleArray, Count))) |
| 338 | return nullptr; |
| 339 | Count = 0; |
| 340 | } |
| 341 | } |
| 342 | if (Count) { |
| 343 | if (UNLIKELY(!populateBatches(C, Region, ClassId, &B, MaxCount, |
| 344 | ShuffleArray, Count))) |
| 345 | return nullptr; |
| 346 | } |
| 347 | DCHECK(B); |
Dynamic Tools Team | aa15246 | 2019-11-19 10:18:38 -0800 | [diff] [blame] | 348 | if (!Region->FreeList.empty()) { |
| 349 | Region->FreeList.push_back(B); |
| 350 | B = Region->FreeList.front(); |
| 351 | Region->FreeList.pop_front(); |
| 352 | } |
Dynamic Tools Team | 517193e | 2019-09-11 14:48:41 +0000 | [diff] [blame] | 353 | DCHECK_GT(B->getCount(), 0); |
| 354 | |
| 355 | C->getStats().add(StatFree, AllocatedUser); |
| 356 | Region->AllocatedUser += AllocatedUser; |
| 357 | Region->Exhausted = false; |
| 358 | if (Region->CanRelease) |
| 359 | Region->ReleaseInfo.LastReleaseAtNs = getMonotonicTime(); |
| 360 | |
| 361 | return B; |
| 362 | } |
| 363 | |
Dynamic Tools Team | 3e8c65b | 2019-10-18 20:00:32 +0000 | [diff] [blame] | 364 | void getStats(ScopedString *Str, uptr ClassId, uptr Rss) const { |
Dynamic Tools Team | 517193e | 2019-09-11 14:48:41 +0000 | [diff] [blame] | 365 | RegionInfo *Region = getRegionInfo(ClassId); |
| 366 | if (Region->MappedUser == 0) |
| 367 | return; |
| 368 | const uptr InUse = Region->Stats.PoppedBlocks - Region->Stats.PushedBlocks; |
Dynamic Tools Team | c283bab | 2019-10-07 17:37:39 +0000 | [diff] [blame] | 369 | const uptr TotalChunks = Region->AllocatedUser / getSizeByClassId(ClassId); |
Dynamic Tools Team | 3e8c65b | 2019-10-18 20:00:32 +0000 | [diff] [blame] | 370 | Str->append("%s %02zu (%6zu): mapped: %6zuK popped: %7zu pushed: %7zu " |
| 371 | "inuse: %6zu total: %6zu rss: %6zuK releases: %6zu last " |
| 372 | "released: %6zuK region: 0x%zx (0x%zx)\n", |
| 373 | Region->Exhausted ? "F" : " ", ClassId, |
| 374 | getSizeByClassId(ClassId), Region->MappedUser >> 10, |
| 375 | Region->Stats.PoppedBlocks, Region->Stats.PushedBlocks, InUse, |
| 376 | TotalChunks, Rss >> 10, Region->ReleaseInfo.RangesReleased, |
| 377 | Region->ReleaseInfo.LastReleasedBytes >> 10, Region->RegionBeg, |
| 378 | getRegionBaseByClassId(ClassId)); |
Dynamic Tools Team | 517193e | 2019-09-11 14:48:41 +0000 | [diff] [blame] | 379 | } |
| 380 | |
Dynamic Tools Team | c283bab | 2019-10-07 17:37:39 +0000 | [diff] [blame] | 381 | NOINLINE uptr releaseToOSMaybe(RegionInfo *Region, uptr ClassId, |
Dynamic Tools Team | 517193e | 2019-09-11 14:48:41 +0000 | [diff] [blame] | 382 | bool Force = false) { |
| 383 | const uptr BlockSize = getSizeByClassId(ClassId); |
| 384 | const uptr PageSize = getPageSizeCached(); |
| 385 | |
| 386 | CHECK_GE(Region->Stats.PoppedBlocks, Region->Stats.PushedBlocks); |
Dynamic Tools Team | c283bab | 2019-10-07 17:37:39 +0000 | [diff] [blame] | 387 | const uptr BytesInFreeList = |
| 388 | Region->AllocatedUser - |
| 389 | (Region->Stats.PoppedBlocks - Region->Stats.PushedBlocks) * BlockSize; |
| 390 | if (BytesInFreeList < PageSize) |
| 391 | return 0; // No chance to release anything. |
Dynamic Tools Team | 517193e | 2019-09-11 14:48:41 +0000 | [diff] [blame] | 392 | if ((Region->Stats.PushedBlocks - |
| 393 | Region->ReleaseInfo.PushedBlocksAtLastRelease) * |
| 394 | BlockSize < |
| 395 | PageSize) { |
Dynamic Tools Team | c283bab | 2019-10-07 17:37:39 +0000 | [diff] [blame] | 396 | return 0; // Nothing new to release. |
Dynamic Tools Team | 517193e | 2019-09-11 14:48:41 +0000 | [diff] [blame] | 397 | } |
| 398 | |
| 399 | if (!Force) { |
| 400 | const s32 IntervalMs = ReleaseToOsIntervalMs; |
| 401 | if (IntervalMs < 0) |
Dynamic Tools Team | c283bab | 2019-10-07 17:37:39 +0000 | [diff] [blame] | 402 | return 0; |
Dynamic Tools Team | 517193e | 2019-09-11 14:48:41 +0000 | [diff] [blame] | 403 | if (Region->ReleaseInfo.LastReleaseAtNs + |
| 404 | static_cast<uptr>(IntervalMs) * 1000000ULL > |
| 405 | getMonotonicTime()) { |
Dynamic Tools Team | c283bab | 2019-10-07 17:37:39 +0000 | [diff] [blame] | 406 | return 0; // Memory was returned recently. |
Dynamic Tools Team | 517193e | 2019-09-11 14:48:41 +0000 | [diff] [blame] | 407 | } |
| 408 | } |
| 409 | |
| 410 | ReleaseRecorder Recorder(Region->RegionBeg, &Region->Data); |
Dynamic Tools Team | 3599770 | 2019-10-28 15:06:10 -0700 | [diff] [blame] | 411 | releaseFreeMemoryToOS(Region->FreeList, Region->RegionBeg, |
Dynamic Tools Team | 517193e | 2019-09-11 14:48:41 +0000 | [diff] [blame] | 412 | roundUpTo(Region->AllocatedUser, PageSize) / PageSize, |
| 413 | BlockSize, &Recorder); |
| 414 | |
| 415 | if (Recorder.getReleasedRangesCount() > 0) { |
| 416 | Region->ReleaseInfo.PushedBlocksAtLastRelease = |
| 417 | Region->Stats.PushedBlocks; |
| 418 | Region->ReleaseInfo.RangesReleased += Recorder.getReleasedRangesCount(); |
| 419 | Region->ReleaseInfo.LastReleasedBytes = Recorder.getReleasedBytes(); |
| 420 | } |
| 421 | Region->ReleaseInfo.LastReleaseAtNs = getMonotonicTime(); |
Dynamic Tools Team | c283bab | 2019-10-07 17:37:39 +0000 | [diff] [blame] | 422 | return Recorder.getReleasedBytes(); |
Dynamic Tools Team | 517193e | 2019-09-11 14:48:41 +0000 | [diff] [blame] | 423 | } |
| 424 | }; |
| 425 | |
| 426 | } // namespace scudo |
| 427 | |
| 428 | #endif // SCUDO_PRIMARY64_H_ |