buzbee | 862a760 | 2013-04-05 10:58:54 -0700 | [diff] [blame] | 1 | /* |
| 2 | * Copyright (C) 2013 The Android Open Source Project |
| 3 | * |
| 4 | * Licensed under the Apache License, Version 2.0 (the "License"); |
| 5 | * you may not use this file except in compliance with the License. |
| 6 | * You may obtain a copy of the License at |
| 7 | * |
| 8 | * http://www.apache.org/licenses/LICENSE-2.0 |
| 9 | * |
| 10 | * Unless required by applicable law or agreed to in writing, software |
| 11 | * distributed under the License is distributed on an "AS IS" BASIS, |
| 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 13 | * See the License for the specific language governing permissions and |
| 14 | * limitations under the License. |
| 15 | */ |
| 16 | |
Andreas Gampe | d490129 | 2017-05-30 18:41:34 -0700 | [diff] [blame] | 17 | #include "arena_allocator-inl.h" |
| 18 | |
| 19 | #include <sys/mman.h> |
| 20 | |
Vladimir Marko | 83cc7ae | 2014-02-12 18:02:05 +0000 | [diff] [blame] | 21 | #include <algorithm> |
Vladimir Marko | f44d36c | 2017-03-14 14:18:46 +0000 | [diff] [blame] | 22 | #include <cstddef> |
Ian Rogers | 6f3dbba | 2014-10-14 17:41:57 -0700 | [diff] [blame] | 23 | #include <iomanip> |
Vladimir Marko | 83cc7ae | 2014-02-12 18:02:05 +0000 | [diff] [blame] | 24 | #include <numeric> |
| 25 | |
Mathieu Chartier | b666f48 | 2015-02-18 14:33:14 -0800 | [diff] [blame] | 26 | #include "logging.h" |
Vladimir Marko | 3481ba2 | 2015-04-13 12:22:36 +0100 | [diff] [blame] | 27 | #include "mem_map.h" |
Mathieu Chartier | b666f48 | 2015-02-18 14:33:14 -0800 | [diff] [blame] | 28 | #include "mutex.h" |
Andreas Gampe | b486a98 | 2017-06-01 13:45:54 -0700 | [diff] [blame] | 29 | #include "thread-current-inl.h" |
Mathieu Chartier | 32ce2ad | 2016-03-04 14:58:03 -0800 | [diff] [blame] | 30 | #include "systrace.h" |
buzbee | 862a760 | 2013-04-05 10:58:54 -0700 | [diff] [blame] | 31 | |
| 32 | namespace art { |
| 33 | |
Vladimir Marko | f44d36c | 2017-03-14 14:18:46 +0000 | [diff] [blame] | 34 | constexpr size_t kMemoryToolRedZoneBytes = 8; |
Mathieu Chartier | f6c4b3b | 2013-08-24 16:11:37 -0700 | [diff] [blame] | 35 | |
Vladimir Marko | bd9e9db | 2014-03-07 19:41:05 +0000 | [diff] [blame] | 36 | template <bool kCount> |
Vladimir Marko | 8dea81c | 2014-06-06 14:50:36 +0100 | [diff] [blame] | 37 | const char* const ArenaAllocatorStatsImpl<kCount>::kAllocNames[] = { |
Igor Murashkin | d01745e | 2017-04-05 16:40:31 -0700 | [diff] [blame] | 38 | // Every name should have the same width and end with a space. Abbreviate if necessary: |
Vladimir Marko | f9f6441 | 2015-09-02 14:05:49 +0100 | [diff] [blame] | 39 | "Misc ", |
Vladimir Marko | f9f6441 | 2015-09-02 14:05:49 +0100 | [diff] [blame] | 40 | "SwitchTbl ", |
Vladimir Marko | f9f6441 | 2015-09-02 14:05:49 +0100 | [diff] [blame] | 41 | "SlowPaths ", |
Vladimir Marko | f9f6441 | 2015-09-02 14:05:49 +0100 | [diff] [blame] | 42 | "GrowBitMap ", |
Vladimir Marko | f9f6441 | 2015-09-02 14:05:49 +0100 | [diff] [blame] | 43 | "STL ", |
Vladimir Marko | 2aaa4b5 | 2015-09-17 17:03:26 +0100 | [diff] [blame] | 44 | "GraphBuilder ", |
Vladimir Marko | f9f6441 | 2015-09-02 14:05:49 +0100 | [diff] [blame] | 45 | "Graph ", |
| 46 | "BasicBlock ", |
Vladimir Marko | fa6b93c | 2015-09-15 10:15:55 +0100 | [diff] [blame] | 47 | "BlockList ", |
| 48 | "RevPostOrder ", |
| 49 | "LinearOrder ", |
| 50 | "ConstantsMap ", |
Vladimir Marko | 6058455 | 2015-09-03 13:35:12 +0000 | [diff] [blame] | 51 | "Predecessors ", |
| 52 | "Successors ", |
| 53 | "Dominated ", |
Vladimir Marko | f9f6441 | 2015-09-02 14:05:49 +0100 | [diff] [blame] | 54 | "Instruction ", |
Igor Murashkin | d01745e | 2017-04-05 16:40:31 -0700 | [diff] [blame] | 55 | "CtorFenceIns ", |
Vladimir Marko | fa6b93c | 2015-09-15 10:15:55 +0100 | [diff] [blame] | 56 | "InvokeInputs ", |
| 57 | "PhiInputs ", |
Vladimir Marko | f9f6441 | 2015-09-02 14:05:49 +0100 | [diff] [blame] | 58 | "LoopInfo ", |
Vladimir Marko | fa6b93c | 2015-09-15 10:15:55 +0100 | [diff] [blame] | 59 | "LIBackEdges ", |
Vladimir Marko | f9f6441 | 2015-09-02 14:05:49 +0100 | [diff] [blame] | 60 | "TryCatchInf ", |
| 61 | "UseListNode ", |
| 62 | "Environment ", |
Vladimir Marko | fa6b93c | 2015-09-15 10:15:55 +0100 | [diff] [blame] | 63 | "EnvVRegs ", |
| 64 | "EnvLocations ", |
Vladimir Marko | 2aaa4b5 | 2015-09-17 17:03:26 +0100 | [diff] [blame] | 65 | "LocSummary ", |
Vladimir Marko | 71bf809 | 2015-09-15 15:33:14 +0100 | [diff] [blame] | 66 | "SsaBuilder ", |
Vladimir Marko | f9f6441 | 2015-09-02 14:05:49 +0100 | [diff] [blame] | 67 | "MoveOperands ", |
| 68 | "CodeBuffer ", |
| 69 | "StackMaps ", |
Vladimir Marko | f9f6441 | 2015-09-02 14:05:49 +0100 | [diff] [blame] | 70 | "Optimization ", |
Vladimir Marko | 2aaa4b5 | 2015-09-17 17:03:26 +0100 | [diff] [blame] | 71 | "GVN ", |
Vladimir Marko | 5233f93 | 2015-09-29 19:01:15 +0100 | [diff] [blame] | 72 | "InductionVar ", |
| 73 | "BCE ", |
Vladimir Marko | f6a35de | 2016-03-21 12:01:50 +0000 | [diff] [blame] | 74 | "DCE ", |
| 75 | "LSE ", |
| 76 | "LICM ", |
Aart Bik | 9620230 | 2016-10-04 17:33:56 -0700 | [diff] [blame] | 77 | "LoopOpt ", |
Vladimir Marko | 2aaa4b5 | 2015-09-17 17:03:26 +0100 | [diff] [blame] | 78 | "SsaLiveness ", |
| 79 | "SsaPhiElim ", |
| 80 | "RefTypeProp ", |
Vladimir Marko | 2aaa4b5 | 2015-09-17 17:03:26 +0100 | [diff] [blame] | 81 | "SideEffects ", |
| 82 | "RegAllocator ", |
Vladimir Marko | f6a35de | 2016-03-21 12:01:50 +0000 | [diff] [blame] | 83 | "RegAllocVldt ", |
Vladimir Marko | 225b646 | 2015-09-28 12:17:40 +0100 | [diff] [blame] | 84 | "StackMapStm ", |
Vladimir Marko | d38ba0a | 2017-04-13 13:43:43 +0100 | [diff] [blame] | 85 | "VectorNode ", |
Vladimir Marko | 225b646 | 2015-09-28 12:17:40 +0100 | [diff] [blame] | 86 | "CodeGen ", |
Vladimir Marko | 93205e3 | 2016-04-13 11:59:46 +0100 | [diff] [blame] | 87 | "Assembler ", |
Vladimir Marko | 225b646 | 2015-09-28 12:17:40 +0100 | [diff] [blame] | 88 | "ParallelMove ", |
Vladimir Marko | 655e585 | 2015-10-12 10:38:28 +0100 | [diff] [blame] | 89 | "GraphChecker ", |
Mathieu Chartier | de40d47 | 2015-10-15 17:47:48 -0700 | [diff] [blame] | 90 | "Verifier ", |
Vladimir Marko | 93205e3 | 2016-04-13 11:59:46 +0100 | [diff] [blame] | 91 | "CallingConv ", |
Vladimir Marko | 4e335d0 | 2016-12-19 16:04:33 +0000 | [diff] [blame] | 92 | "CHA ", |
Alexandre Rames | 22aa54b | 2016-10-18 09:32:29 +0100 | [diff] [blame] | 93 | "Scheduler ", |
Calin Juravle | cc3171a | 2017-05-19 16:47:53 -0700 | [diff] [blame] | 94 | "Profile ", |
buzbee | 862a760 | 2013-04-05 10:58:54 -0700 | [diff] [blame] | 95 | }; |
| 96 | |
Vladimir Marko | 83cc7ae | 2014-02-12 18:02:05 +0000 | [diff] [blame] | 97 | template <bool kCount> |
| 98 | ArenaAllocatorStatsImpl<kCount>::ArenaAllocatorStatsImpl() |
Vladimir Marko | 4e335d0 | 2016-12-19 16:04:33 +0000 | [diff] [blame] | 99 | : num_allocations_(0u), |
| 100 | alloc_stats_(kNumArenaAllocKinds, 0u) { |
Vladimir Marko | 83cc7ae | 2014-02-12 18:02:05 +0000 | [diff] [blame] | 101 | } |
| 102 | |
| 103 | template <bool kCount> |
| 104 | void ArenaAllocatorStatsImpl<kCount>::Copy(const ArenaAllocatorStatsImpl& other) { |
| 105 | num_allocations_ = other.num_allocations_; |
Vladimir Marko | 4e335d0 | 2016-12-19 16:04:33 +0000 | [diff] [blame] | 106 | std::copy_n(other.alloc_stats_.begin(), kNumArenaAllocKinds, alloc_stats_.begin()); |
Vladimir Marko | 83cc7ae | 2014-02-12 18:02:05 +0000 | [diff] [blame] | 107 | } |
| 108 | |
| 109 | template <bool kCount> |
| 110 | void ArenaAllocatorStatsImpl<kCount>::RecordAlloc(size_t bytes, ArenaAllocKind kind) { |
| 111 | alloc_stats_[kind] += bytes; |
| 112 | ++num_allocations_; |
| 113 | } |
| 114 | |
| 115 | template <bool kCount> |
| 116 | size_t ArenaAllocatorStatsImpl<kCount>::NumAllocations() const { |
| 117 | return num_allocations_; |
| 118 | } |
| 119 | |
| 120 | template <bool kCount> |
| 121 | size_t ArenaAllocatorStatsImpl<kCount>::BytesAllocated() const { |
| 122 | const size_t init = 0u; // Initial value of the correct type. |
Vladimir Marko | 4e335d0 | 2016-12-19 16:04:33 +0000 | [diff] [blame] | 123 | return std::accumulate(alloc_stats_.begin(), alloc_stats_.end(), init); |
Vladimir Marko | 83cc7ae | 2014-02-12 18:02:05 +0000 | [diff] [blame] | 124 | } |
| 125 | |
| 126 | template <bool kCount> |
| 127 | void ArenaAllocatorStatsImpl<kCount>::Dump(std::ostream& os, const Arena* first, |
| 128 | ssize_t lost_bytes_adjustment) const { |
| 129 | size_t malloc_bytes = 0u; |
| 130 | size_t lost_bytes = 0u; |
| 131 | size_t num_arenas = 0u; |
| 132 | for (const Arena* arena = first; arena != nullptr; arena = arena->next_) { |
| 133 | malloc_bytes += arena->Size(); |
| 134 | lost_bytes += arena->RemainingSpace(); |
| 135 | ++num_arenas; |
| 136 | } |
| 137 | // The lost_bytes_adjustment is used to make up for the fact that the current arena |
| 138 | // may not have the bytes_allocated_ updated correctly. |
| 139 | lost_bytes += lost_bytes_adjustment; |
| 140 | const size_t bytes_allocated = BytesAllocated(); |
| 141 | os << " MEM: used: " << bytes_allocated << ", allocated: " << malloc_bytes |
| 142 | << ", lost: " << lost_bytes << "\n"; |
Vladimir Marko | bd9e9db | 2014-03-07 19:41:05 +0000 | [diff] [blame] | 143 | size_t num_allocations = NumAllocations(); |
Vladimir Marko | 83cc7ae | 2014-02-12 18:02:05 +0000 | [diff] [blame] | 144 | if (num_allocations != 0) { |
| 145 | os << "Number of arenas allocated: " << num_arenas << ", Number of allocations: " |
| 146 | << num_allocations << ", avg size: " << bytes_allocated / num_allocations << "\n"; |
| 147 | } |
| 148 | os << "===== Allocation by kind\n"; |
Andreas Gampe | 785d2f2 | 2014-11-03 22:57:30 -0800 | [diff] [blame] | 149 | static_assert(arraysize(kAllocNames) == kNumArenaAllocKinds, "arraysize of kAllocNames"); |
Vladimir Marko | 83cc7ae | 2014-02-12 18:02:05 +0000 | [diff] [blame] | 150 | for (int i = 0; i < kNumArenaAllocKinds; i++) { |
Vladimir Marko | bd9e9db | 2014-03-07 19:41:05 +0000 | [diff] [blame] | 151 | os << kAllocNames[i] << std::setw(10) << alloc_stats_[i] << "\n"; |
Vladimir Marko | 83cc7ae | 2014-02-12 18:02:05 +0000 | [diff] [blame] | 152 | } |
| 153 | } |
| 154 | |
Stephen Hines | 47b2247 | 2017-01-26 00:50:31 -0800 | [diff] [blame] | 155 | #pragma GCC diagnostic push |
Jayant Chowdhary | 5f9da55 | 2017-02-02 12:48:10 -0800 | [diff] [blame] | 156 | #if __clang_major__ >= 4 |
Stephen Hines | 47b2247 | 2017-01-26 00:50:31 -0800 | [diff] [blame] | 157 | #pragma GCC diagnostic ignored "-Winstantiation-after-specialization" |
Jayant Chowdhary | 5f9da55 | 2017-02-02 12:48:10 -0800 | [diff] [blame] | 158 | #endif |
Vladimir Marko | d38ba0a | 2017-04-13 13:43:43 +0100 | [diff] [blame] | 159 | // We're going to use ArenaAllocatorStatsImpl<kArenaAllocatorCountAllocations> which needs |
| 160 | // to be explicitly instantiated if kArenaAllocatorCountAllocations is true. Explicit |
| 161 | // instantiation of the specialization ArenaAllocatorStatsImpl<false> does not do anything |
| 162 | // but requires the warning "-Winstantiation-after-specialization" to be turned off. |
| 163 | // |
| 164 | // To avoid bit-rot of the ArenaAllocatorStatsImpl<true>, instantiate it also in debug builds |
| 165 | // (but keep the unnecessary code out of release builds) as we do not usually compile with |
| 166 | // kArenaAllocatorCountAllocations set to true. |
| 167 | template class ArenaAllocatorStatsImpl<kArenaAllocatorCountAllocations || kIsDebugBuild>; |
Stephen Hines | 47b2247 | 2017-01-26 00:50:31 -0800 | [diff] [blame] | 168 | #pragma GCC diagnostic pop |
Vladimir Marko | 83cc7ae | 2014-02-12 18:02:05 +0000 | [diff] [blame] | 169 | |
Vladimir Marko | 7bda3b6 | 2015-10-07 12:44:31 +0000 | [diff] [blame] | 170 | void ArenaAllocatorMemoryTool::DoMakeDefined(void* ptr, size_t size) { |
| 171 | MEMORY_TOOL_MAKE_DEFINED(ptr, size); |
| 172 | } |
| 173 | |
| 174 | void ArenaAllocatorMemoryTool::DoMakeUndefined(void* ptr, size_t size) { |
| 175 | MEMORY_TOOL_MAKE_UNDEFINED(ptr, size); |
| 176 | } |
| 177 | |
| 178 | void ArenaAllocatorMemoryTool::DoMakeInaccessible(void* ptr, size_t size) { |
| 179 | MEMORY_TOOL_MAKE_NOACCESS(ptr, size); |
| 180 | } |
| 181 | |
Andreas Gampe | d9911ee | 2017-03-27 13:27:24 -0700 | [diff] [blame] | 182 | Arena::Arena() : bytes_allocated_(0), memory_(nullptr), size_(0), next_(nullptr) { |
Ian Rogers | e7a5b7d | 2013-04-18 20:09:02 -0700 | [diff] [blame] | 183 | } |
| 184 | |
Vladimir Marko | f44d36c | 2017-03-14 14:18:46 +0000 | [diff] [blame] | 185 | class MallocArena FINAL : public Arena { |
| 186 | public: |
Andreas Gampe | 121f148 | 2017-05-12 10:28:35 -0700 | [diff] [blame] | 187 | explicit MallocArena(size_t size = arena_allocator::kArenaDefaultSize); |
Vladimir Marko | f44d36c | 2017-03-14 14:18:46 +0000 | [diff] [blame] | 188 | virtual ~MallocArena(); |
| 189 | private: |
| 190 | static constexpr size_t RequiredOverallocation() { |
| 191 | return (alignof(std::max_align_t) < ArenaAllocator::kArenaAlignment) |
| 192 | ? ArenaAllocator::kArenaAlignment - alignof(std::max_align_t) |
| 193 | : 0u; |
| 194 | } |
| 195 | |
| 196 | uint8_t* unaligned_memory_; |
| 197 | }; |
| 198 | |
Mathieu Chartier | c6201fa | 2015-03-12 10:06:33 -0700 | [diff] [blame] | 199 | MallocArena::MallocArena(size_t size) { |
Vladimir Marko | f44d36c | 2017-03-14 14:18:46 +0000 | [diff] [blame] | 200 | // We need to guarantee kArenaAlignment aligned allocation for the new arena. |
| 201 | // TODO: Use std::aligned_alloc() when it becomes available with C++17. |
| 202 | constexpr size_t overallocation = RequiredOverallocation(); |
| 203 | unaligned_memory_ = reinterpret_cast<uint8_t*>(calloc(1, size + overallocation)); |
| 204 | CHECK(unaligned_memory_ != nullptr); // Abort on OOM. |
| 205 | DCHECK_ALIGNED(unaligned_memory_, alignof(std::max_align_t)); |
| 206 | if (overallocation == 0u) { |
| 207 | memory_ = unaligned_memory_; |
| 208 | } else { |
| 209 | memory_ = AlignUp(unaligned_memory_, ArenaAllocator::kArenaAlignment); |
| 210 | if (UNLIKELY(RUNNING_ON_MEMORY_TOOL > 0)) { |
| 211 | size_t head = memory_ - unaligned_memory_; |
| 212 | size_t tail = overallocation - head; |
| 213 | MEMORY_TOOL_MAKE_NOACCESS(unaligned_memory_, head); |
| 214 | MEMORY_TOOL_MAKE_NOACCESS(memory_ + size, tail); |
| 215 | } |
| 216 | } |
| 217 | DCHECK_ALIGNED(memory_, ArenaAllocator::kArenaAlignment); |
Mathieu Chartier | c6201fa | 2015-03-12 10:06:33 -0700 | [diff] [blame] | 218 | size_ = size; |
buzbee | 862a760 | 2013-04-05 10:58:54 -0700 | [diff] [blame] | 219 | } |
| 220 | |
Mathieu Chartier | c6201fa | 2015-03-12 10:06:33 -0700 | [diff] [blame] | 221 | MallocArena::~MallocArena() { |
Vladimir Marko | f44d36c | 2017-03-14 14:18:46 +0000 | [diff] [blame] | 222 | constexpr size_t overallocation = RequiredOverallocation(); |
| 223 | if (overallocation != 0u && UNLIKELY(RUNNING_ON_MEMORY_TOOL > 0)) { |
| 224 | size_t head = memory_ - unaligned_memory_; |
| 225 | size_t tail = overallocation - head; |
| 226 | MEMORY_TOOL_MAKE_UNDEFINED(unaligned_memory_, head); |
| 227 | MEMORY_TOOL_MAKE_UNDEFINED(memory_ + size_, tail); |
| 228 | } |
| 229 | free(reinterpret_cast<void*>(unaligned_memory_)); |
Mathieu Chartier | c6201fa | 2015-03-12 10:06:33 -0700 | [diff] [blame] | 230 | } |
| 231 | |
Vladimir Marko | f44d36c | 2017-03-14 14:18:46 +0000 | [diff] [blame] | 232 | class MemMapArena FINAL : public Arena { |
| 233 | public: |
| 234 | MemMapArena(size_t size, bool low_4gb, const char* name); |
| 235 | virtual ~MemMapArena(); |
| 236 | void Release() OVERRIDE; |
| 237 | |
| 238 | private: |
| 239 | std::unique_ptr<MemMap> map_; |
| 240 | }; |
| 241 | |
Nicolas Geoffray | 25e0456 | 2016-03-01 13:17:58 +0000 | [diff] [blame] | 242 | MemMapArena::MemMapArena(size_t size, bool low_4gb, const char* name) { |
Vladimir Marko | f44d36c | 2017-03-14 14:18:46 +0000 | [diff] [blame] | 243 | // Round up to a full page as that's the smallest unit of allocation for mmap() |
| 244 | // and we want to be able to use all memory that we actually allocate. |
| 245 | size = RoundUp(size, kPageSize); |
Mathieu Chartier | c6201fa | 2015-03-12 10:06:33 -0700 | [diff] [blame] | 246 | std::string error_msg; |
Mathieu Chartier | c785344 | 2015-03-27 14:35:38 -0700 | [diff] [blame] | 247 | map_.reset(MemMap::MapAnonymous( |
Nicolas Geoffray | 25e0456 | 2016-03-01 13:17:58 +0000 | [diff] [blame] | 248 | name, nullptr, size, PROT_READ | PROT_WRITE, low_4gb, false, &error_msg)); |
Mathieu Chartier | c6201fa | 2015-03-12 10:06:33 -0700 | [diff] [blame] | 249 | CHECK(map_.get() != nullptr) << error_msg; |
| 250 | memory_ = map_->Begin(); |
Vladimir Marko | f44d36c | 2017-03-14 14:18:46 +0000 | [diff] [blame] | 251 | static_assert(ArenaAllocator::kArenaAlignment <= kPageSize, |
| 252 | "Arena should not need stronger alignment than kPageSize."); |
| 253 | DCHECK_ALIGNED(memory_, ArenaAllocator::kArenaAlignment); |
Mathieu Chartier | c6201fa | 2015-03-12 10:06:33 -0700 | [diff] [blame] | 254 | size_ = map_->Size(); |
| 255 | } |
| 256 | |
Vladimir Marko | 3481ba2 | 2015-04-13 12:22:36 +0100 | [diff] [blame] | 257 | MemMapArena::~MemMapArena() { |
| 258 | // Destroys MemMap via std::unique_ptr<>. |
| 259 | } |
| 260 | |
Mathieu Chartier | c6201fa | 2015-03-12 10:06:33 -0700 | [diff] [blame] | 261 | void MemMapArena::Release() { |
| 262 | if (bytes_allocated_ > 0) { |
Mathieu Chartier | 9b34b24 | 2015-03-09 11:30:17 -0700 | [diff] [blame] | 263 | map_->MadviseDontNeedAndZero(); |
| 264 | bytes_allocated_ = 0; |
| 265 | } |
| 266 | } |
| 267 | |
Mathieu Chartier | f6c4b3b | 2013-08-24 16:11:37 -0700 | [diff] [blame] | 268 | void Arena::Reset() { |
Mathieu Chartier | 9b34b24 | 2015-03-09 11:30:17 -0700 | [diff] [blame] | 269 | if (bytes_allocated_ > 0) { |
Mathieu Chartier | c6201fa | 2015-03-12 10:06:33 -0700 | [diff] [blame] | 270 | memset(Begin(), 0, bytes_allocated_); |
Mathieu Chartier | f6c4b3b | 2013-08-24 16:11:37 -0700 | [diff] [blame] | 271 | bytes_allocated_ = 0; |
buzbee | 862a760 | 2013-04-05 10:58:54 -0700 | [diff] [blame] | 272 | } |
buzbee | 862a760 | 2013-04-05 10:58:54 -0700 | [diff] [blame] | 273 | } |
| 274 | |
Nicolas Geoffray | 25e0456 | 2016-03-01 13:17:58 +0000 | [diff] [blame] | 275 | ArenaPool::ArenaPool(bool use_malloc, bool low_4gb, const char* name) |
| 276 | : use_malloc_(use_malloc), |
| 277 | lock_("Arena pool lock", kArenaPoolLock), |
| 278 | free_arenas_(nullptr), |
| 279 | low_4gb_(low_4gb), |
| 280 | name_(name) { |
Mathieu Chartier | c785344 | 2015-03-27 14:35:38 -0700 | [diff] [blame] | 281 | if (low_4gb) { |
| 282 | CHECK(!use_malloc) << "low4gb must use map implementation"; |
| 283 | } |
Mathieu Chartier | c6201fa | 2015-03-12 10:06:33 -0700 | [diff] [blame] | 284 | if (!use_malloc) { |
Mathieu Chartier | 9b34b24 | 2015-03-09 11:30:17 -0700 | [diff] [blame] | 285 | MemMap::Init(); |
| 286 | } |
Mathieu Chartier | f6c4b3b | 2013-08-24 16:11:37 -0700 | [diff] [blame] | 287 | } |
| 288 | |
| 289 | ArenaPool::~ArenaPool() { |
Jean-Philippe Halimi | ca76a1a | 2016-02-02 19:48:52 +0100 | [diff] [blame] | 290 | ReclaimMemory(); |
| 291 | } |
| 292 | |
| 293 | void ArenaPool::ReclaimMemory() { |
Mathieu Chartier | f6c4b3b | 2013-08-24 16:11:37 -0700 | [diff] [blame] | 294 | while (free_arenas_ != nullptr) { |
| 295 | auto* arena = free_arenas_; |
| 296 | free_arenas_ = free_arenas_->next_; |
| 297 | delete arena; |
| 298 | } |
| 299 | } |
| 300 | |
Jean-Philippe Halimi | ca76a1a | 2016-02-02 19:48:52 +0100 | [diff] [blame] | 301 | void ArenaPool::LockReclaimMemory() { |
| 302 | MutexLock lock(Thread::Current(), lock_); |
| 303 | ReclaimMemory(); |
| 304 | } |
| 305 | |
Mathieu Chartier | f6c4b3b | 2013-08-24 16:11:37 -0700 | [diff] [blame] | 306 | Arena* ArenaPool::AllocArena(size_t size) { |
| 307 | Thread* self = Thread::Current(); |
| 308 | Arena* ret = nullptr; |
| 309 | { |
| 310 | MutexLock lock(self, lock_); |
| 311 | if (free_arenas_ != nullptr && LIKELY(free_arenas_->Size() >= size)) { |
| 312 | ret = free_arenas_; |
| 313 | free_arenas_ = free_arenas_->next_; |
| 314 | } |
| 315 | } |
| 316 | if (ret == nullptr) { |
Mathieu Chartier | c785344 | 2015-03-27 14:35:38 -0700 | [diff] [blame] | 317 | ret = use_malloc_ ? static_cast<Arena*>(new MallocArena(size)) : |
Nicolas Geoffray | 25e0456 | 2016-03-01 13:17:58 +0000 | [diff] [blame] | 318 | new MemMapArena(size, low_4gb_, name_); |
Mathieu Chartier | f6c4b3b | 2013-08-24 16:11:37 -0700 | [diff] [blame] | 319 | } |
| 320 | ret->Reset(); |
| 321 | return ret; |
| 322 | } |
| 323 | |
Mathieu Chartier | 9b34b24 | 2015-03-09 11:30:17 -0700 | [diff] [blame] | 324 | void ArenaPool::TrimMaps() { |
Mathieu Chartier | c6201fa | 2015-03-12 10:06:33 -0700 | [diff] [blame] | 325 | if (!use_malloc_) { |
Mathieu Chartier | 32ce2ad | 2016-03-04 14:58:03 -0800 | [diff] [blame] | 326 | ScopedTrace trace(__PRETTY_FUNCTION__); |
Mathieu Chartier | c6201fa | 2015-03-12 10:06:33 -0700 | [diff] [blame] | 327 | // Doesn't work for malloc. |
| 328 | MutexLock lock(Thread::Current(), lock_); |
| 329 | for (auto* arena = free_arenas_; arena != nullptr; arena = arena->next_) { |
| 330 | arena->Release(); |
| 331 | } |
Mathieu Chartier | 9b34b24 | 2015-03-09 11:30:17 -0700 | [diff] [blame] | 332 | } |
| 333 | } |
| 334 | |
Mathieu Chartier | 49285c5 | 2014-12-02 15:43:48 -0800 | [diff] [blame] | 335 | size_t ArenaPool::GetBytesAllocated() const { |
| 336 | size_t total = 0; |
| 337 | MutexLock lock(Thread::Current(), lock_); |
| 338 | for (Arena* arena = free_arenas_; arena != nullptr; arena = arena->next_) { |
| 339 | total += arena->GetBytesAllocated(); |
| 340 | } |
| 341 | return total; |
| 342 | } |
| 343 | |
Vladimir Marko | 83cc7ae | 2014-02-12 18:02:05 +0000 | [diff] [blame] | 344 | void ArenaPool::FreeArenaChain(Arena* first) { |
Evgenii Stepanov | 1e13374 | 2015-05-20 12:30:59 -0700 | [diff] [blame] | 345 | if (UNLIKELY(RUNNING_ON_MEMORY_TOOL > 0)) { |
Vladimir Marko | 83cc7ae | 2014-02-12 18:02:05 +0000 | [diff] [blame] | 346 | for (Arena* arena = first; arena != nullptr; arena = arena->next_) { |
Evgenii Stepanov | 1e13374 | 2015-05-20 12:30:59 -0700 | [diff] [blame] | 347 | MEMORY_TOOL_MAKE_UNDEFINED(arena->memory_, arena->bytes_allocated_); |
Vladimir Marko | 83cc7ae | 2014-02-12 18:02:05 +0000 | [diff] [blame] | 348 | } |
Mathieu Chartier | 75165d0 | 2013-09-12 14:00:31 -0700 | [diff] [blame] | 349 | } |
Andreas Gampe | 121f148 | 2017-05-12 10:28:35 -0700 | [diff] [blame] | 350 | |
| 351 | if (arena_allocator::kArenaAllocatorPreciseTracking) { |
| 352 | // Do not reuse arenas when tracking. |
| 353 | while (first != nullptr) { |
| 354 | Arena* next = first->next_; |
| 355 | delete first; |
| 356 | first = next; |
| 357 | } |
| 358 | return; |
| 359 | } |
| 360 | |
Vladimir Marko | 83cc7ae | 2014-02-12 18:02:05 +0000 | [diff] [blame] | 361 | if (first != nullptr) { |
| 362 | Arena* last = first; |
| 363 | while (last->next_ != nullptr) { |
| 364 | last = last->next_; |
| 365 | } |
| 366 | Thread* self = Thread::Current(); |
Mathieu Chartier | f6c4b3b | 2013-08-24 16:11:37 -0700 | [diff] [blame] | 367 | MutexLock lock(self, lock_); |
Vladimir Marko | 83cc7ae | 2014-02-12 18:02:05 +0000 | [diff] [blame] | 368 | last->next_ = free_arenas_; |
| 369 | free_arenas_ = first; |
Mathieu Chartier | f6c4b3b | 2013-08-24 16:11:37 -0700 | [diff] [blame] | 370 | } |
| 371 | } |
| 372 | |
| 373 | size_t ArenaAllocator::BytesAllocated() const { |
Vladimir Marko | 83cc7ae | 2014-02-12 18:02:05 +0000 | [diff] [blame] | 374 | return ArenaAllocatorStats::BytesAllocated(); |
Mathieu Chartier | f6c4b3b | 2013-08-24 16:11:37 -0700 | [diff] [blame] | 375 | } |
| 376 | |
Mathieu Chartier | c785344 | 2015-03-27 14:35:38 -0700 | [diff] [blame] | 377 | size_t ArenaAllocator::BytesUsed() const { |
| 378 | size_t total = ptr_ - begin_; |
| 379 | if (arena_head_ != nullptr) { |
| 380 | for (Arena* cur_arena = arena_head_->next_; cur_arena != nullptr; |
| 381 | cur_arena = cur_arena->next_) { |
| 382 | total += cur_arena->GetBytesAllocated(); |
| 383 | } |
| 384 | } |
| 385 | return total; |
| 386 | } |
| 387 | |
Mathieu Chartier | f6c4b3b | 2013-08-24 16:11:37 -0700 | [diff] [blame] | 388 | ArenaAllocator::ArenaAllocator(ArenaPool* pool) |
| 389 | : pool_(pool), |
| 390 | begin_(nullptr), |
| 391 | end_(nullptr), |
| 392 | ptr_(nullptr), |
Vladimir Marko | 2a408a3 | 2015-09-18 14:11:00 +0100 | [diff] [blame] | 393 | arena_head_(nullptr) { |
Mathieu Chartier | f6c4b3b | 2013-08-24 16:11:37 -0700 | [diff] [blame] | 394 | } |
| 395 | |
| 396 | void ArenaAllocator::UpdateBytesAllocated() { |
| 397 | if (arena_head_ != nullptr) { |
| 398 | // Update how many bytes we have allocated into the arena so that the arena pool knows how |
| 399 | // much memory to zero out. |
| 400 | arena_head_->bytes_allocated_ = ptr_ - begin_; |
| 401 | } |
| 402 | } |
| 403 | |
Vladimir Marko | 2a408a3 | 2015-09-18 14:11:00 +0100 | [diff] [blame] | 404 | void* ArenaAllocator::AllocWithMemoryTool(size_t bytes, ArenaAllocKind kind) { |
Vladimir Marko | 7500193 | 2015-11-10 20:54:22 +0000 | [diff] [blame] | 405 | // We mark all memory for a newly retrieved arena as inaccessible and then |
| 406 | // mark only the actually allocated memory as defined. That leaves red zones |
| 407 | // and padding between allocations marked as inaccessible. |
Evgenii Stepanov | 1e13374 | 2015-05-20 12:30:59 -0700 | [diff] [blame] | 408 | size_t rounded_bytes = RoundUp(bytes + kMemoryToolRedZoneBytes, 8); |
Vladimir Marko | 83cc7ae | 2014-02-12 18:02:05 +0000 | [diff] [blame] | 409 | ArenaAllocatorStats::RecordAlloc(rounded_bytes, kind); |
Vladimir Marko | 3f84f2c | 2016-04-25 19:40:34 +0100 | [diff] [blame] | 410 | uint8_t* ret; |
Vladimir Marko | 3e0e717 | 2016-04-22 18:07:13 +0100 | [diff] [blame] | 411 | if (UNLIKELY(rounded_bytes > static_cast<size_t>(end_ - ptr_))) { |
Vladimir Marko | f44d36c | 2017-03-14 14:18:46 +0000 | [diff] [blame] | 412 | ret = AllocFromNewArenaWithMemoryTool(rounded_bytes); |
Vladimir Marko | 3f84f2c | 2016-04-25 19:40:34 +0100 | [diff] [blame] | 413 | } else { |
| 414 | ret = ptr_; |
| 415 | ptr_ += rounded_bytes; |
Vladimir Marko | 3e0e717 | 2016-04-22 18:07:13 +0100 | [diff] [blame] | 416 | } |
Vladimir Marko | 2a408a3 | 2015-09-18 14:11:00 +0100 | [diff] [blame] | 417 | MEMORY_TOOL_MAKE_DEFINED(ret, bytes); |
Vladimir Marko | 7500193 | 2015-11-10 20:54:22 +0000 | [diff] [blame] | 418 | // Check that the memory is already zeroed out. |
| 419 | DCHECK(std::all_of(ret, ret + bytes, [](uint8_t val) { return val == 0u; })); |
Mathieu Chartier | 75165d0 | 2013-09-12 14:00:31 -0700 | [diff] [blame] | 420 | return ret; |
| 421 | } |
| 422 | |
Vladimir Marko | f44d36c | 2017-03-14 14:18:46 +0000 | [diff] [blame] | 423 | void* ArenaAllocator::AllocWithMemoryToolAlign16(size_t bytes, ArenaAllocKind kind) { |
| 424 | // We mark all memory for a newly retrieved arena as inaccessible and then |
| 425 | // mark only the actually allocated memory as defined. That leaves red zones |
| 426 | // and padding between allocations marked as inaccessible. |
| 427 | size_t rounded_bytes = bytes + kMemoryToolRedZoneBytes; |
| 428 | DCHECK_ALIGNED(rounded_bytes, 8); // `bytes` is 16-byte aligned, red zone is 8-byte aligned. |
| 429 | uintptr_t padding = |
| 430 | ((reinterpret_cast<uintptr_t>(ptr_) + 15u) & 15u) - reinterpret_cast<uintptr_t>(ptr_); |
| 431 | ArenaAllocatorStats::RecordAlloc(rounded_bytes, kind); |
| 432 | uint8_t* ret; |
| 433 | if (UNLIKELY(padding + rounded_bytes > static_cast<size_t>(end_ - ptr_))) { |
| 434 | static_assert(kArenaAlignment >= 16, "Expecting sufficient alignment for new Arena."); |
| 435 | ret = AllocFromNewArenaWithMemoryTool(rounded_bytes); |
| 436 | } else { |
| 437 | ptr_ += padding; // Leave padding inaccessible. |
| 438 | ret = ptr_; |
| 439 | ptr_ += rounded_bytes; |
| 440 | } |
| 441 | MEMORY_TOOL_MAKE_DEFINED(ret, bytes); |
| 442 | // Check that the memory is already zeroed out. |
| 443 | DCHECK(std::all_of(ret, ret + bytes, [](uint8_t val) { return val == 0u; })); |
| 444 | return ret; |
| 445 | } |
| 446 | |
Mathieu Chartier | f6c4b3b | 2013-08-24 16:11:37 -0700 | [diff] [blame] | 447 | ArenaAllocator::~ArenaAllocator() { |
| 448 | // Reclaim all the arenas by giving them back to the thread pool. |
| 449 | UpdateBytesAllocated(); |
Vladimir Marko | 83cc7ae | 2014-02-12 18:02:05 +0000 | [diff] [blame] | 450 | pool_->FreeArenaChain(arena_head_); |
Mathieu Chartier | f6c4b3b | 2013-08-24 16:11:37 -0700 | [diff] [blame] | 451 | } |
| 452 | |
Vladimir Marko | 3f84f2c | 2016-04-25 19:40:34 +0100 | [diff] [blame] | 453 | uint8_t* ArenaAllocator::AllocFromNewArena(size_t bytes) { |
Andreas Gampe | 121f148 | 2017-05-12 10:28:35 -0700 | [diff] [blame] | 454 | Arena* new_arena = pool_->AllocArena(std::max(arena_allocator::kArenaDefaultSize, bytes)); |
Vladimir Marko | 3e0e717 | 2016-04-22 18:07:13 +0100 | [diff] [blame] | 455 | DCHECK(new_arena != nullptr); |
| 456 | DCHECK_LE(bytes, new_arena->Size()); |
| 457 | if (static_cast<size_t>(end_ - ptr_) > new_arena->Size() - bytes) { |
| 458 | // The old arena has more space remaining than the new one, so keep using it. |
| 459 | // This can happen when the requested size is over half of the default size. |
| 460 | DCHECK(arena_head_ != nullptr); |
| 461 | new_arena->bytes_allocated_ = bytes; // UpdateBytesAllocated() on the new_arena. |
| 462 | new_arena->next_ = arena_head_->next_; |
| 463 | arena_head_->next_ = new_arena; |
| 464 | } else { |
| 465 | UpdateBytesAllocated(); |
| 466 | new_arena->next_ = arena_head_; |
| 467 | arena_head_ = new_arena; |
| 468 | // Update our internal data structures. |
| 469 | begin_ = new_arena->Begin(); |
Andreas Gampe | f6dd829 | 2016-08-19 20:22:19 -0700 | [diff] [blame] | 470 | DCHECK_ALIGNED(begin_, kAlignment); |
Vladimir Marko | 3e0e717 | 2016-04-22 18:07:13 +0100 | [diff] [blame] | 471 | ptr_ = begin_ + bytes; |
| 472 | end_ = new_arena->End(); |
| 473 | } |
| 474 | return new_arena->Begin(); |
Mathieu Chartier | f6c4b3b | 2013-08-24 16:11:37 -0700 | [diff] [blame] | 475 | } |
| 476 | |
Vladimir Marko | f44d36c | 2017-03-14 14:18:46 +0000 | [diff] [blame] | 477 | uint8_t* ArenaAllocator::AllocFromNewArenaWithMemoryTool(size_t bytes) { |
| 478 | uint8_t* ret = AllocFromNewArena(bytes); |
| 479 | uint8_t* noaccess_begin = ret + bytes; |
| 480 | uint8_t* noaccess_end; |
| 481 | if (ret == arena_head_->Begin()) { |
| 482 | DCHECK(ptr_ - bytes == ret); |
| 483 | noaccess_end = end_; |
| 484 | } else { |
| 485 | // We're still using the old arena but `ret` comes from a new one just after it. |
| 486 | DCHECK(arena_head_->next_ != nullptr); |
| 487 | DCHECK(ret == arena_head_->next_->Begin()); |
| 488 | DCHECK_EQ(bytes, arena_head_->next_->GetBytesAllocated()); |
| 489 | noaccess_end = arena_head_->next_->End(); |
| 490 | } |
| 491 | MEMORY_TOOL_MAKE_NOACCESS(noaccess_begin, noaccess_end - noaccess_begin); |
| 492 | return ret; |
| 493 | } |
| 494 | |
Mathieu Chartier | e401d14 | 2015-04-22 13:56:20 -0700 | [diff] [blame] | 495 | bool ArenaAllocator::Contains(const void* ptr) const { |
| 496 | if (ptr >= begin_ && ptr < end_) { |
| 497 | return true; |
| 498 | } |
| 499 | for (const Arena* cur_arena = arena_head_; cur_arena != nullptr; cur_arena = cur_arena->next_) { |
| 500 | if (cur_arena->Contains(ptr)) { |
| 501 | return true; |
| 502 | } |
| 503 | } |
| 504 | return false; |
| 505 | } |
| 506 | |
Vladimir Marko | f44d36c | 2017-03-14 14:18:46 +0000 | [diff] [blame] | 507 | MemStats::MemStats(const char* name, |
| 508 | const ArenaAllocatorStats* stats, |
| 509 | const Arena* first_arena, |
Vladimir Marko | 83cc7ae | 2014-02-12 18:02:05 +0000 | [diff] [blame] | 510 | ssize_t lost_bytes_adjustment) |
| 511 | : name_(name), |
| 512 | stats_(stats), |
| 513 | first_arena_(first_arena), |
| 514 | lost_bytes_adjustment_(lost_bytes_adjustment) { |
| 515 | } |
| 516 | |
| 517 | void MemStats::Dump(std::ostream& os) const { |
| 518 | os << name_ << " stats:\n"; |
| 519 | stats_->Dump(os, first_arena_, lost_bytes_adjustment_); |
| 520 | } |
| 521 | |
Mathieu Chartier | f6c4b3b | 2013-08-24 16:11:37 -0700 | [diff] [blame] | 522 | // Dump memory usage stats. |
Vladimir Marko | 83cc7ae | 2014-02-12 18:02:05 +0000 | [diff] [blame] | 523 | MemStats ArenaAllocator::GetMemStats() const { |
| 524 | ssize_t lost_bytes_adjustment = |
| 525 | (arena_head_ == nullptr) ? 0 : (end_ - ptr_) - arena_head_->RemainingSpace(); |
| 526 | return MemStats("ArenaAllocator", this, arena_head_, lost_bytes_adjustment); |
buzbee | 862a760 | 2013-04-05 10:58:54 -0700 | [diff] [blame] | 527 | } |
| 528 | |
| 529 | } // namespace art |