Hiroshi Yamauchi | 7cb7bbc | 2013-11-18 17:27:37 -0800 | [diff] [blame] | 1 | |
Hiroshi Yamauchi | cf58d4a | 2013-09-26 14:21:22 -0700 | [diff] [blame] | 2 | /* |
| 3 | * Copyright (C) 2013 The Android Open Source Project |
| 4 | * |
| 5 | * Licensed under the Apache License, Version 2.0 (the "License"); |
| 6 | * you may not use this file except in compliance with the License. |
| 7 | * You may obtain a copy of the License at |
| 8 | * |
| 9 | * http://www.apache.org/licenses/LICENSE-2.0 |
| 10 | * |
| 11 | * Unless required by applicable law or agreed to in writing, software |
| 12 | * distributed under the License is distributed on an "AS IS" BASIS, |
| 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 14 | * See the License for the specific language governing permissions and |
| 15 | * limitations under the License. |
| 16 | */ |
| 17 | |
Hiroshi Yamauchi | cf58d4a | 2013-09-26 14:21:22 -0700 | [diff] [blame] | 18 | #include "rosalloc_space-inl.h" |
Mathieu Chartier | a8e8f9c | 2014-04-09 14:51:05 -0700 | [diff] [blame] | 19 | |
Andreas Gampe | 170331f | 2017-12-07 18:41:03 -0800 | [diff] [blame] | 20 | #include "base/logging.h" // For VLOG. |
Vladimir Marko | 80afd02 | 2015-05-19 18:08:00 +0100 | [diff] [blame] | 21 | #include "base/time_utils.h" |
David Sehr | c431b9d | 2018-03-02 12:01:51 -0800 | [diff] [blame] | 22 | #include "base/utils.h" |
Hiroshi Yamauchi | cf58d4a | 2013-09-26 14:21:22 -0700 | [diff] [blame] | 23 | #include "gc/accounting/card_table.h" |
Mathieu Chartier | a8e8f9c | 2014-04-09 14:51:05 -0700 | [diff] [blame] | 24 | #include "gc/accounting/space_bitmap-inl.h" |
Hiroshi Yamauchi | cf58d4a | 2013-09-26 14:21:22 -0700 | [diff] [blame] | 25 | #include "gc/heap.h" |
Andreas Gampe | 8cf9cb3 | 2017-07-19 09:28:38 -0700 | [diff] [blame] | 26 | #include "memory_tool_malloc_space-inl.h" |
Hiroshi Yamauchi | cf58d4a | 2013-09-26 14:21:22 -0700 | [diff] [blame] | 27 | #include "mirror/class-inl.h" |
| 28 | #include "mirror/object-inl.h" |
| 29 | #include "runtime.h" |
Andreas Gampe | 508fdf3 | 2017-06-05 16:42:13 -0700 | [diff] [blame] | 30 | #include "scoped_thread_state_change-inl.h" |
Hiroshi Yamauchi | cf58d4a | 2013-09-26 14:21:22 -0700 | [diff] [blame] | 31 | #include "thread.h" |
| 32 | #include "thread_list.h" |
Hiroshi Yamauchi | cf58d4a | 2013-09-26 14:21:22 -0700 | [diff] [blame] | 33 | |
| 34 | namespace art { |
| 35 | namespace gc { |
| 36 | namespace space { |
| 37 | |
Mathieu Chartier | 73d1e17 | 2014-04-11 17:53:48 -0700 | [diff] [blame] | 38 | static constexpr bool kPrefetchDuringRosAllocFreeList = false; |
Mathieu Chartier | 8585bad | 2014-04-11 17:53:48 -0700 | [diff] [blame] | 39 | static constexpr size_t kPrefetchLookAhead = 8; |
| 40 | // Use this only for verification, it is not safe to use since the class of the object may have |
| 41 | // been freed. |
| 42 | static constexpr bool kVerifyFreedBytes = false; |
Ian Rogers | 6fac447 | 2014-02-25 17:01:10 -0800 | [diff] [blame] | 43 | |
Mathieu Chartier | 31f4414 | 2014-04-08 14:40:03 -0700 | [diff] [blame] | 44 | // TODO: Fix |
Evgenii Stepanov | 1e13374 | 2015-05-20 12:30:59 -0700 | [diff] [blame] | 45 | // template class MemoryToolMallocSpace<RosAllocSpace, allocator::RosAlloc*>; |
Hiroshi Yamauchi | cf58d4a | 2013-09-26 14:21:22 -0700 | [diff] [blame] | 46 | |
Andreas Gampe | d757632 | 2014-10-24 22:13:45 -0700 | [diff] [blame] | 47 | RosAllocSpace::RosAllocSpace(MemMap* mem_map, size_t initial_size, const std::string& name, |
Ian Rogers | 1373595 | 2014-10-08 12:43:28 -0700 | [diff] [blame] | 48 | art::gc::allocator::RosAlloc* rosalloc, uint8_t* begin, uint8_t* end, |
| 49 | uint8_t* limit, size_t growth_limit, bool can_move_objects, |
Andreas Gampe | d757632 | 2014-10-24 22:13:45 -0700 | [diff] [blame] | 50 | size_t starting_size, bool low_memory_mode) |
Mathieu Chartier | 31f4414 | 2014-04-08 14:40:03 -0700 | [diff] [blame] | 51 | : MallocSpace(name, mem_map, begin, end, limit, growth_limit, true, can_move_objects, |
| 52 | starting_size, initial_size), |
| 53 | rosalloc_(rosalloc), low_memory_mode_(low_memory_mode) { |
| 54 | CHECK(rosalloc != nullptr); |
Hiroshi Yamauchi | cf58d4a | 2013-09-26 14:21:22 -0700 | [diff] [blame] | 55 | } |
| 56 | |
Mathieu Chartier | e6da9af | 2013-12-16 11:54:42 -0800 | [diff] [blame] | 57 | RosAllocSpace* RosAllocSpace::CreateFromMemMap(MemMap* mem_map, const std::string& name, |
Ian Rogers | a55cf41 | 2014-02-27 00:31:26 -0800 | [diff] [blame] | 58 | size_t starting_size, size_t initial_size, |
| 59 | size_t growth_limit, size_t capacity, |
Mathieu Chartier | 31f4414 | 2014-04-08 14:40:03 -0700 | [diff] [blame] | 60 | bool low_memory_mode, bool can_move_objects) { |
Mathieu Chartier | e6da9af | 2013-12-16 11:54:42 -0800 | [diff] [blame] | 61 | DCHECK(mem_map != nullptr); |
Andreas Gampe | d757632 | 2014-10-24 22:13:45 -0700 | [diff] [blame] | 62 | |
Evgenii Stepanov | 1e13374 | 2015-05-20 12:30:59 -0700 | [diff] [blame] | 63 | bool running_on_memory_tool = Runtime::Current()->IsRunningOnMemoryTool(); |
Andreas Gampe | d757632 | 2014-10-24 22:13:45 -0700 | [diff] [blame] | 64 | |
Mathieu Chartier | e6da9af | 2013-12-16 11:54:42 -0800 | [diff] [blame] | 65 | allocator::RosAlloc* rosalloc = CreateRosAlloc(mem_map->Begin(), starting_size, initial_size, |
Evgenii Stepanov | 1e13374 | 2015-05-20 12:30:59 -0700 | [diff] [blame] | 66 | capacity, low_memory_mode, running_on_memory_tool); |
Mathieu Chartier | 2cebb24 | 2015-04-21 16:50:40 -0700 | [diff] [blame] | 67 | if (rosalloc == nullptr) { |
Mathieu Chartier | e6da9af | 2013-12-16 11:54:42 -0800 | [diff] [blame] | 68 | LOG(ERROR) << "Failed to initialize rosalloc for alloc space (" << name << ")"; |
Mathieu Chartier | 2cebb24 | 2015-04-21 16:50:40 -0700 | [diff] [blame] | 69 | return nullptr; |
Mathieu Chartier | e6da9af | 2013-12-16 11:54:42 -0800 | [diff] [blame] | 70 | } |
| 71 | |
lzang1 | 385de73 | 2014-02-21 14:15:01 +0800 | [diff] [blame] | 72 | // Protect memory beyond the starting size. MoreCore will add r/w permissions when necessory |
Ian Rogers | 1373595 | 2014-10-08 12:43:28 -0700 | [diff] [blame] | 73 | uint8_t* end = mem_map->Begin() + starting_size; |
lzang1 | 385de73 | 2014-02-21 14:15:01 +0800 | [diff] [blame] | 74 | if (capacity - starting_size > 0) { |
Mathieu Chartier | 3425d02 | 2017-10-03 16:22:05 -0700 | [diff] [blame] | 75 | CheckedCall(mprotect, name.c_str(), end, capacity - starting_size, PROT_NONE); |
Mathieu Chartier | e6da9af | 2013-12-16 11:54:42 -0800 | [diff] [blame] | 76 | } |
| 77 | |
| 78 | // Everything is set so record in immutable structure and leave |
Ian Rogers | 1373595 | 2014-10-08 12:43:28 -0700 | [diff] [blame] | 79 | uint8_t* begin = mem_map->Begin(); |
Andreas Gampe | 8b362a8 | 2018-05-22 20:54:14 +0000 | [diff] [blame] | 80 | // TODO: Fix RosAllocSpace to support Valgrind/ASan. There is currently some issues with |
Mathieu Chartier | 661974a | 2014-01-09 11:23:53 -0800 | [diff] [blame] | 81 | // AllocationSize caused by redzones. b/12944686 |
Evgenii Stepanov | 1e13374 | 2015-05-20 12:30:59 -0700 | [diff] [blame] | 82 | if (running_on_memory_tool) { |
| 83 | return new MemoryToolMallocSpace<RosAllocSpace, kDefaultMemoryToolRedZoneBytes, false, true>( |
Andreas Gampe | d757632 | 2014-10-24 22:13:45 -0700 | [diff] [blame] | 84 | mem_map, initial_size, name, rosalloc, begin, end, begin + capacity, growth_limit, |
| 85 | can_move_objects, starting_size, low_memory_mode); |
Mathieu Chartier | e6da9af | 2013-12-16 11:54:42 -0800 | [diff] [blame] | 86 | } else { |
Andreas Gampe | d757632 | 2014-10-24 22:13:45 -0700 | [diff] [blame] | 87 | return new RosAllocSpace(mem_map, initial_size, name, rosalloc, begin, end, begin + capacity, |
| 88 | growth_limit, can_move_objects, starting_size, low_memory_mode); |
Mathieu Chartier | e6da9af | 2013-12-16 11:54:42 -0800 | [diff] [blame] | 89 | } |
Mathieu Chartier | e6da9af | 2013-12-16 11:54:42 -0800 | [diff] [blame] | 90 | } |
| 91 | |
Mathieu Chartier | 661974a | 2014-01-09 11:23:53 -0800 | [diff] [blame] | 92 | RosAllocSpace::~RosAllocSpace() { |
| 93 | delete rosalloc_; |
| 94 | } |
| 95 | |
Ian Rogers | 6fac447 | 2014-02-25 17:01:10 -0800 | [diff] [blame] | 96 | RosAllocSpace* RosAllocSpace::Create(const std::string& name, size_t initial_size, |
Ian Rogers | 1373595 | 2014-10-08 12:43:28 -0700 | [diff] [blame] | 97 | size_t growth_limit, size_t capacity, uint8_t* requested_begin, |
Mathieu Chartier | 31f4414 | 2014-04-08 14:40:03 -0700 | [diff] [blame] | 98 | bool low_memory_mode, bool can_move_objects) { |
Hiroshi Yamauchi | cf58d4a | 2013-09-26 14:21:22 -0700 | [diff] [blame] | 99 | uint64_t start_time = 0; |
| 100 | if (VLOG_IS_ON(heap) || VLOG_IS_ON(startup)) { |
| 101 | start_time = NanoTime(); |
| 102 | VLOG(startup) << "RosAllocSpace::Create entering " << name |
| 103 | << " initial_size=" << PrettySize(initial_size) |
| 104 | << " growth_limit=" << PrettySize(growth_limit) |
| 105 | << " capacity=" << PrettySize(capacity) |
| 106 | << " requested_begin=" << reinterpret_cast<void*>(requested_begin); |
| 107 | } |
| 108 | |
| 109 | // Memory we promise to rosalloc before it asks for morecore. |
| 110 | // Note: making this value large means that large allocations are unlikely to succeed as rosalloc |
| 111 | // will ask for this memory from sys_alloc which will fail as the footprint (this value plus the |
| 112 | // size of the large allocation) will be greater than the footprint limit. |
Hiroshi Yamauchi | 5ccd498 | 2014-03-11 12:19:04 -0700 | [diff] [blame] | 113 | size_t starting_size = Heap::kDefaultStartingSize; |
Hiroshi Yamauchi | cf58d4a | 2013-09-26 14:21:22 -0700 | [diff] [blame] | 114 | MemMap* mem_map = CreateMemMap(name, starting_size, &initial_size, &growth_limit, &capacity, |
| 115 | requested_begin); |
Mathieu Chartier | 2cebb24 | 2015-04-21 16:50:40 -0700 | [diff] [blame] | 116 | if (mem_map == nullptr) { |
Hiroshi Yamauchi | cf58d4a | 2013-09-26 14:21:22 -0700 | [diff] [blame] | 117 | LOG(ERROR) << "Failed to create mem map for alloc space (" << name << ") of size " |
| 118 | << PrettySize(capacity); |
Mathieu Chartier | 2cebb24 | 2015-04-21 16:50:40 -0700 | [diff] [blame] | 119 | return nullptr; |
Hiroshi Yamauchi | cf58d4a | 2013-09-26 14:21:22 -0700 | [diff] [blame] | 120 | } |
Hiroshi Yamauchi | cf58d4a | 2013-09-26 14:21:22 -0700 | [diff] [blame] | 121 | |
Mathieu Chartier | e6da9af | 2013-12-16 11:54:42 -0800 | [diff] [blame] | 122 | RosAllocSpace* space = CreateFromMemMap(mem_map, name, starting_size, initial_size, |
Mathieu Chartier | 31f4414 | 2014-04-08 14:40:03 -0700 | [diff] [blame] | 123 | growth_limit, capacity, low_memory_mode, |
| 124 | can_move_objects); |
Hiroshi Yamauchi | cf58d4a | 2013-09-26 14:21:22 -0700 | [diff] [blame] | 125 | // We start out with only the initial size possibly containing objects. |
| 126 | if (VLOG_IS_ON(heap) || VLOG_IS_ON(startup)) { |
| 127 | LOG(INFO) << "RosAllocSpace::Create exiting (" << PrettyDuration(NanoTime() - start_time) |
| 128 | << " ) " << *space; |
| 129 | } |
| 130 | return space; |
| 131 | } |
| 132 | |
Mathieu Chartier | 31f4414 | 2014-04-08 14:40:03 -0700 | [diff] [blame] | 133 | allocator::RosAlloc* RosAllocSpace::CreateRosAlloc(void* begin, size_t morecore_start, |
| 134 | size_t initial_size, |
Andreas Gampe | d757632 | 2014-10-24 22:13:45 -0700 | [diff] [blame] | 135 | size_t maximum_size, bool low_memory_mode, |
Evgenii Stepanov | 1e13374 | 2015-05-20 12:30:59 -0700 | [diff] [blame] | 136 | bool running_on_memory_tool) { |
Hiroshi Yamauchi | cf58d4a | 2013-09-26 14:21:22 -0700 | [diff] [blame] | 137 | // clear errno to allow PLOG on error |
| 138 | errno = 0; |
| 139 | // create rosalloc using our backing storage starting at begin and |
| 140 | // with a footprint of morecore_start. When morecore_start bytes of |
| 141 | // memory is exhaused morecore will be called. |
Hiroshi Yamauchi | 573f7d2 | 2013-12-17 11:54:23 -0800 | [diff] [blame] | 142 | allocator::RosAlloc* rosalloc = new art::gc::allocator::RosAlloc( |
Hiroshi Yamauchi | 26d69ff | 2014-02-27 11:27:10 -0800 | [diff] [blame] | 143 | begin, morecore_start, maximum_size, |
Hiroshi Yamauchi | 573f7d2 | 2013-12-17 11:54:23 -0800 | [diff] [blame] | 144 | low_memory_mode ? |
| 145 | art::gc::allocator::RosAlloc::kPageReleaseModeAll : |
Andreas Gampe | d757632 | 2014-10-24 22:13:45 -0700 | [diff] [blame] | 146 | art::gc::allocator::RosAlloc::kPageReleaseModeSizeAndEnd, |
Evgenii Stepanov | 1e13374 | 2015-05-20 12:30:59 -0700 | [diff] [blame] | 147 | running_on_memory_tool); |
Mathieu Chartier | 2cebb24 | 2015-04-21 16:50:40 -0700 | [diff] [blame] | 148 | if (rosalloc != nullptr) { |
Hiroshi Yamauchi | cf58d4a | 2013-09-26 14:21:22 -0700 | [diff] [blame] | 149 | rosalloc->SetFootprintLimit(initial_size); |
| 150 | } else { |
| 151 | PLOG(ERROR) << "RosAlloc::Create failed"; |
Mathieu Chartier | e6da9af | 2013-12-16 11:54:42 -0800 | [diff] [blame] | 152 | } |
Hiroshi Yamauchi | cf58d4a | 2013-09-26 14:21:22 -0700 | [diff] [blame] | 153 | return rosalloc; |
| 154 | } |
| 155 | |
Ian Rogers | 6fac447 | 2014-02-25 17:01:10 -0800 | [diff] [blame] | 156 | mirror::Object* RosAllocSpace::AllocWithGrowth(Thread* self, size_t num_bytes, |
Hiroshi Yamauchi | 4460a84 | 2015-03-09 11:57:48 -0700 | [diff] [blame] | 157 | size_t* bytes_allocated, size_t* usable_size, |
| 158 | size_t* bytes_tl_bulk_allocated) { |
Hiroshi Yamauchi | cf58d4a | 2013-09-26 14:21:22 -0700 | [diff] [blame] | 159 | mirror::Object* result; |
| 160 | { |
| 161 | MutexLock mu(self, lock_); |
| 162 | // Grow as much as possible within the space. |
| 163 | size_t max_allowed = Capacity(); |
| 164 | rosalloc_->SetFootprintLimit(max_allowed); |
| 165 | // Try the allocation. |
Hiroshi Yamauchi | 4460a84 | 2015-03-09 11:57:48 -0700 | [diff] [blame] | 166 | result = AllocCommon(self, num_bytes, bytes_allocated, usable_size, |
| 167 | bytes_tl_bulk_allocated); |
Hiroshi Yamauchi | cf58d4a | 2013-09-26 14:21:22 -0700 | [diff] [blame] | 168 | // Shrink back down as small as possible. |
| 169 | size_t footprint = rosalloc_->Footprint(); |
| 170 | rosalloc_->SetFootprintLimit(footprint); |
| 171 | } |
| 172 | // Note RosAlloc zeroes memory internally. |
Mathieu Chartier | 2cebb24 | 2015-04-21 16:50:40 -0700 | [diff] [blame] | 173 | // Return the new allocation or null. |
Mathieu Chartier | 0651d41 | 2014-04-29 14:37:57 -0700 | [diff] [blame] | 174 | CHECK(!kDebugSpaces || result == nullptr || Contains(result)); |
Hiroshi Yamauchi | cf58d4a | 2013-09-26 14:21:22 -0700 | [diff] [blame] | 175 | return result; |
| 176 | } |
| 177 | |
Andreas Gampe | d757632 | 2014-10-24 22:13:45 -0700 | [diff] [blame] | 178 | MallocSpace* RosAllocSpace::CreateInstance(MemMap* mem_map, const std::string& name, |
| 179 | void* allocator, uint8_t* begin, uint8_t* end, |
| 180 | uint8_t* limit, size_t growth_limit, |
Mathieu Chartier | 31f4414 | 2014-04-08 14:40:03 -0700 | [diff] [blame] | 181 | bool can_move_objects) { |
Evgenii Stepanov | 1e13374 | 2015-05-20 12:30:59 -0700 | [diff] [blame] | 182 | if (Runtime::Current()->IsRunningOnMemoryTool()) { |
| 183 | return new MemoryToolMallocSpace<RosAllocSpace, kDefaultMemoryToolRedZoneBytes, false, true>( |
Andreas Gampe | d757632 | 2014-10-24 22:13:45 -0700 | [diff] [blame] | 184 | mem_map, initial_size_, name, reinterpret_cast<allocator::RosAlloc*>(allocator), begin, end, |
| 185 | limit, growth_limit, can_move_objects, starting_size_, low_memory_mode_); |
| 186 | } else { |
| 187 | return new RosAllocSpace(mem_map, initial_size_, name, |
| 188 | reinterpret_cast<allocator::RosAlloc*>(allocator), begin, end, limit, |
| 189 | growth_limit, can_move_objects, starting_size_, low_memory_mode_); |
| 190 | } |
Hiroshi Yamauchi | cf58d4a | 2013-09-26 14:21:22 -0700 | [diff] [blame] | 191 | } |
| 192 | |
| 193 | size_t RosAllocSpace::Free(Thread* self, mirror::Object* ptr) { |
| 194 | if (kDebugSpaces) { |
Mathieu Chartier | 2cebb24 | 2015-04-21 16:50:40 -0700 | [diff] [blame] | 195 | CHECK(ptr != nullptr); |
Hiroshi Yamauchi | cf58d4a | 2013-09-26 14:21:22 -0700 | [diff] [blame] | 196 | CHECK(Contains(ptr)) << "Free (" << ptr << ") not in bounds of heap " << *this; |
| 197 | } |
Hiroshi Yamauchi | cf58d4a | 2013-09-26 14:21:22 -0700 | [diff] [blame] | 198 | if (kRecentFreeCount > 0) { |
| 199 | MutexLock mu(self, lock_); |
| 200 | RegisterRecentFree(ptr); |
| 201 | } |
Mathieu Chartier | 8585bad | 2014-04-11 17:53:48 -0700 | [diff] [blame] | 202 | return rosalloc_->Free(self, ptr); |
Hiroshi Yamauchi | cf58d4a | 2013-09-26 14:21:22 -0700 | [diff] [blame] | 203 | } |
| 204 | |
| 205 | size_t RosAllocSpace::FreeList(Thread* self, size_t num_ptrs, mirror::Object** ptrs) { |
Mathieu Chartier | 8585bad | 2014-04-11 17:53:48 -0700 | [diff] [blame] | 206 | DCHECK(ptrs != nullptr); |
Hiroshi Yamauchi | cf58d4a | 2013-09-26 14:21:22 -0700 | [diff] [blame] | 207 | |
Mathieu Chartier | 8585bad | 2014-04-11 17:53:48 -0700 | [diff] [blame] | 208 | size_t verify_bytes = 0; |
Hiroshi Yamauchi | cf58d4a | 2013-09-26 14:21:22 -0700 | [diff] [blame] | 209 | for (size_t i = 0; i < num_ptrs; i++) { |
Mathieu Chartier | 8585bad | 2014-04-11 17:53:48 -0700 | [diff] [blame] | 210 | if (kPrefetchDuringRosAllocFreeList && i + kPrefetchLookAhead < num_ptrs) { |
| 211 | __builtin_prefetch(reinterpret_cast<char*>(ptrs[i + kPrefetchLookAhead])); |
Hiroshi Yamauchi | cf58d4a | 2013-09-26 14:21:22 -0700 | [diff] [blame] | 212 | } |
Mathieu Chartier | 8585bad | 2014-04-11 17:53:48 -0700 | [diff] [blame] | 213 | if (kVerifyFreedBytes) { |
Hiroshi Yamauchi | 4460a84 | 2015-03-09 11:57:48 -0700 | [diff] [blame] | 214 | verify_bytes += AllocationSizeNonvirtual<true>(ptrs[i], nullptr); |
Mathieu Chartier | 8585bad | 2014-04-11 17:53:48 -0700 | [diff] [blame] | 215 | } |
Hiroshi Yamauchi | cf58d4a | 2013-09-26 14:21:22 -0700 | [diff] [blame] | 216 | } |
| 217 | |
| 218 | if (kRecentFreeCount > 0) { |
| 219 | MutexLock mu(self, lock_); |
| 220 | for (size_t i = 0; i < num_ptrs; i++) { |
| 221 | RegisterRecentFree(ptrs[i]); |
| 222 | } |
| 223 | } |
| 224 | |
| 225 | if (kDebugSpaces) { |
| 226 | size_t num_broken_ptrs = 0; |
| 227 | for (size_t i = 0; i < num_ptrs; i++) { |
| 228 | if (!Contains(ptrs[i])) { |
| 229 | num_broken_ptrs++; |
| 230 | LOG(ERROR) << "FreeList[" << i << "] (" << ptrs[i] << ") not in bounds of heap " << *this; |
| 231 | } else { |
| 232 | size_t size = rosalloc_->UsableSize(ptrs[i]); |
| 233 | memset(ptrs[i], 0xEF, size); |
| 234 | } |
| 235 | } |
| 236 | CHECK_EQ(num_broken_ptrs, 0u); |
| 237 | } |
| 238 | |
Mathieu Chartier | 8585bad | 2014-04-11 17:53:48 -0700 | [diff] [blame] | 239 | const size_t bytes_freed = rosalloc_->BulkFree(self, reinterpret_cast<void**>(ptrs), num_ptrs); |
| 240 | if (kVerifyFreedBytes) { |
| 241 | CHECK_EQ(verify_bytes, bytes_freed); |
| 242 | } |
Hiroshi Yamauchi | cf58d4a | 2013-09-26 14:21:22 -0700 | [diff] [blame] | 243 | return bytes_freed; |
| 244 | } |
| 245 | |
Hiroshi Yamauchi | cf58d4a | 2013-09-26 14:21:22 -0700 | [diff] [blame] | 246 | size_t RosAllocSpace::Trim() { |
Hiroshi Yamauchi | d9a88de | 2014-04-07 13:52:31 -0700 | [diff] [blame] | 247 | VLOG(heap) << "RosAllocSpace::Trim() "; |
Hiroshi Yamauchi | 573f7d2 | 2013-12-17 11:54:23 -0800 | [diff] [blame] | 248 | { |
Mathieu Chartier | a9d82fe | 2016-01-25 20:06:11 -0800 | [diff] [blame] | 249 | Thread* const self = Thread::Current(); |
| 250 | // SOA required for Rosalloc::Trim() -> ArtRosAllocMoreCore() -> Heap::GetRosAllocSpace. |
| 251 | ScopedObjectAccess soa(self); |
| 252 | MutexLock mu(self, lock_); |
Hiroshi Yamauchi | 573f7d2 | 2013-12-17 11:54:23 -0800 | [diff] [blame] | 253 | // Trim to release memory at the end of the space. |
| 254 | rosalloc_->Trim(); |
| 255 | } |
| 256 | // Attempt to release pages if it does not release all empty pages. |
| 257 | if (!rosalloc_->DoesReleaseAllPages()) { |
Hiroshi Yamauchi | d9a88de | 2014-04-07 13:52:31 -0700 | [diff] [blame] | 258 | return rosalloc_->ReleasePages(); |
Hiroshi Yamauchi | 573f7d2 | 2013-12-17 11:54:23 -0800 | [diff] [blame] | 259 | } |
Hiroshi Yamauchi | cf58d4a | 2013-09-26 14:21:22 -0700 | [diff] [blame] | 260 | return 0; |
| 261 | } |
| 262 | |
| 263 | void RosAllocSpace::Walk(void(*callback)(void *start, void *end, size_t num_bytes, void* callback_arg), |
| 264 | void* arg) { |
Hiroshi Yamauchi | 1cd53db | 2014-03-28 15:26:48 -0700 | [diff] [blame] | 265 | InspectAllRosAlloc(callback, arg, true); |
Hiroshi Yamauchi | cf58d4a | 2013-09-26 14:21:22 -0700 | [diff] [blame] | 266 | } |
| 267 | |
| 268 | size_t RosAllocSpace::GetFootprint() { |
| 269 | MutexLock mu(Thread::Current(), lock_); |
| 270 | return rosalloc_->Footprint(); |
| 271 | } |
| 272 | |
| 273 | size_t RosAllocSpace::GetFootprintLimit() { |
| 274 | MutexLock mu(Thread::Current(), lock_); |
| 275 | return rosalloc_->FootprintLimit(); |
| 276 | } |
| 277 | |
| 278 | void RosAllocSpace::SetFootprintLimit(size_t new_size) { |
| 279 | MutexLock mu(Thread::Current(), lock_); |
| 280 | VLOG(heap) << "RosAllocSpace::SetFootprintLimit " << PrettySize(new_size); |
| 281 | // Compare against the actual footprint, rather than the Size(), because the heap may not have |
| 282 | // grown all the way to the allowed size yet. |
| 283 | size_t current_space_size = rosalloc_->Footprint(); |
| 284 | if (new_size < current_space_size) { |
| 285 | // Don't let the space grow any more. |
| 286 | new_size = current_space_size; |
| 287 | } |
| 288 | rosalloc_->SetFootprintLimit(new_size); |
| 289 | } |
| 290 | |
| 291 | uint64_t RosAllocSpace::GetBytesAllocated() { |
Hiroshi Yamauchi | 4ce1f00 | 2013-11-18 14:49:09 -0800 | [diff] [blame] | 292 | size_t bytes_allocated = 0; |
Hiroshi Yamauchi | 1cd53db | 2014-03-28 15:26:48 -0700 | [diff] [blame] | 293 | InspectAllRosAlloc(art::gc::allocator::RosAlloc::BytesAllocatedCallback, &bytes_allocated, false); |
Hiroshi Yamauchi | 4ce1f00 | 2013-11-18 14:49:09 -0800 | [diff] [blame] | 294 | return bytes_allocated; |
Hiroshi Yamauchi | cf58d4a | 2013-09-26 14:21:22 -0700 | [diff] [blame] | 295 | } |
| 296 | |
| 297 | uint64_t RosAllocSpace::GetObjectsAllocated() { |
Hiroshi Yamauchi | 4ce1f00 | 2013-11-18 14:49:09 -0800 | [diff] [blame] | 298 | size_t objects_allocated = 0; |
Hiroshi Yamauchi | 1cd53db | 2014-03-28 15:26:48 -0700 | [diff] [blame] | 299 | InspectAllRosAlloc(art::gc::allocator::RosAlloc::ObjectsAllocatedCallback, &objects_allocated, false); |
Hiroshi Yamauchi | 4ce1f00 | 2013-11-18 14:49:09 -0800 | [diff] [blame] | 300 | return objects_allocated; |
Hiroshi Yamauchi | cf58d4a | 2013-09-26 14:21:22 -0700 | [diff] [blame] | 301 | } |
| 302 | |
Hiroshi Yamauchi | 1cd53db | 2014-03-28 15:26:48 -0700 | [diff] [blame] | 303 | void RosAllocSpace::InspectAllRosAllocWithSuspendAll( |
| 304 | void (*callback)(void *start, void *end, size_t num_bytes, void* callback_arg), |
| 305 | void* arg, bool do_null_callback_at_end) NO_THREAD_SAFETY_ANALYSIS { |
| 306 | // TODO: NO_THREAD_SAFETY_ANALYSIS. |
| 307 | Thread* self = Thread::Current(); |
Mathieu Chartier | 4f55e22 | 2015-09-04 13:26:21 -0700 | [diff] [blame] | 308 | ScopedSuspendAll ssa(__FUNCTION__); |
| 309 | MutexLock mu(self, *Locks::runtime_shutdown_lock_); |
| 310 | MutexLock mu2(self, *Locks::thread_list_lock_); |
| 311 | rosalloc_->InspectAll(callback, arg); |
| 312 | if (do_null_callback_at_end) { |
| 313 | callback(nullptr, nullptr, 0, arg); // Indicate end of a space. |
Hiroshi Yamauchi | 1cd53db | 2014-03-28 15:26:48 -0700 | [diff] [blame] | 314 | } |
Hiroshi Yamauchi | 1cd53db | 2014-03-28 15:26:48 -0700 | [diff] [blame] | 315 | } |
| 316 | |
Hiroshi Yamauchi | cf58d4a | 2013-09-26 14:21:22 -0700 | [diff] [blame] | 317 | void RosAllocSpace::InspectAllRosAlloc(void (*callback)(void *start, void *end, size_t num_bytes, void* callback_arg), |
Hiroshi Yamauchi | 1cd53db | 2014-03-28 15:26:48 -0700 | [diff] [blame] | 318 | void* arg, bool do_null_callback_at_end) NO_THREAD_SAFETY_ANALYSIS { |
Hiroshi Yamauchi | cf58d4a | 2013-09-26 14:21:22 -0700 | [diff] [blame] | 319 | // TODO: NO_THREAD_SAFETY_ANALYSIS. |
| 320 | Thread* self = Thread::Current(); |
| 321 | if (Locks::mutator_lock_->IsExclusiveHeld(self)) { |
| 322 | // The mutators are already suspended. For example, a call path |
| 323 | // from SignalCatcher::HandleSigQuit(). |
| 324 | rosalloc_->InspectAll(callback, arg); |
Hiroshi Yamauchi | 1cd53db | 2014-03-28 15:26:48 -0700 | [diff] [blame] | 325 | if (do_null_callback_at_end) { |
Mathieu Chartier | 2cebb24 | 2015-04-21 16:50:40 -0700 | [diff] [blame] | 326 | callback(nullptr, nullptr, 0, arg); // Indicate end of a space. |
Hiroshi Yamauchi | cf58d4a | 2013-09-26 14:21:22 -0700 | [diff] [blame] | 327 | } |
Hiroshi Yamauchi | 1cd53db | 2014-03-28 15:26:48 -0700 | [diff] [blame] | 328 | } else if (Locks::mutator_lock_->IsSharedHeld(self)) { |
| 329 | // The mutators are not suspended yet and we have a shared access |
| 330 | // to the mutator lock. Temporarily release the shared access by |
| 331 | // transitioning to the suspend state, and suspend the mutators. |
Mathieu Chartier | f1d666e | 2015-09-03 16:13:34 -0700 | [diff] [blame] | 332 | ScopedThreadSuspension sts(self, kSuspended); |
Hiroshi Yamauchi | 1cd53db | 2014-03-28 15:26:48 -0700 | [diff] [blame] | 333 | InspectAllRosAllocWithSuspendAll(callback, arg, do_null_callback_at_end); |
Hiroshi Yamauchi | 1cd53db | 2014-03-28 15:26:48 -0700 | [diff] [blame] | 334 | } else { |
| 335 | // The mutators are not suspended yet. Suspend the mutators. |
| 336 | InspectAllRosAllocWithSuspendAll(callback, arg, do_null_callback_at_end); |
Hiroshi Yamauchi | cf58d4a | 2013-09-26 14:21:22 -0700 | [diff] [blame] | 337 | } |
| 338 | } |
| 339 | |
Hiroshi Yamauchi | 4460a84 | 2015-03-09 11:57:48 -0700 | [diff] [blame] | 340 | size_t RosAllocSpace::RevokeThreadLocalBuffers(Thread* thread) { |
| 341 | return rosalloc_->RevokeThreadLocalRuns(thread); |
Hiroshi Yamauchi | cf58d4a | 2013-09-26 14:21:22 -0700 | [diff] [blame] | 342 | } |
| 343 | |
Hiroshi Yamauchi | 4460a84 | 2015-03-09 11:57:48 -0700 | [diff] [blame] | 344 | size_t RosAllocSpace::RevokeAllThreadLocalBuffers() { |
| 345 | return rosalloc_->RevokeAllThreadLocalRuns(); |
Hiroshi Yamauchi | cf58d4a | 2013-09-26 14:21:22 -0700 | [diff] [blame] | 346 | } |
| 347 | |
Ian Rogers | 68d8b42 | 2014-07-17 11:09:10 -0700 | [diff] [blame] | 348 | void RosAllocSpace::AssertThreadLocalBuffersAreRevoked(Thread* thread) { |
| 349 | if (kIsDebugBuild) { |
| 350 | rosalloc_->AssertThreadLocalRunsAreRevoked(thread); |
| 351 | } |
| 352 | } |
| 353 | |
Hiroshi Yamauchi | c93c530 | 2014-03-20 16:15:37 -0700 | [diff] [blame] | 354 | void RosAllocSpace::AssertAllThreadLocalBuffersAreRevoked() { |
| 355 | if (kIsDebugBuild) { |
| 356 | rosalloc_->AssertAllThreadLocalRunsAreRevoked(); |
| 357 | } |
| 358 | } |
| 359 | |
Mathieu Chartier | e6da9af | 2013-12-16 11:54:42 -0800 | [diff] [blame] | 360 | void RosAllocSpace::Clear() { |
Mathieu Chartier | 31f4414 | 2014-04-08 14:40:03 -0700 | [diff] [blame] | 361 | size_t footprint_limit = GetFootprintLimit(); |
Mathieu Chartier | e6da9af | 2013-12-16 11:54:42 -0800 | [diff] [blame] | 362 | madvise(GetMemMap()->Begin(), GetMemMap()->Size(), MADV_DONTNEED); |
Mathieu Chartier | 31f4414 | 2014-04-08 14:40:03 -0700 | [diff] [blame] | 363 | live_bitmap_->Clear(); |
| 364 | mark_bitmap_->Clear(); |
Ian Rogers | be2a1df | 2014-07-10 00:56:36 -0700 | [diff] [blame] | 365 | SetEnd(begin_ + starting_size_); |
Mathieu Chartier | 31f4414 | 2014-04-08 14:40:03 -0700 | [diff] [blame] | 366 | delete rosalloc_; |
Hiroshi Yamauchi | 94c41df | 2014-12-18 21:19:24 -0800 | [diff] [blame] | 367 | rosalloc_ = CreateRosAlloc(mem_map_->Begin(), starting_size_, initial_size_, |
| 368 | NonGrowthLimitCapacity(), low_memory_mode_, |
Evgenii Stepanov | 1e13374 | 2015-05-20 12:30:59 -0700 | [diff] [blame] | 369 | Runtime::Current()->IsRunningOnMemoryTool()); |
Mathieu Chartier | 31f4414 | 2014-04-08 14:40:03 -0700 | [diff] [blame] | 370 | SetFootprintLimit(footprint_limit); |
Mathieu Chartier | 15d3402 | 2014-02-26 17:16:38 -0800 | [diff] [blame] | 371 | } |
| 372 | |
Hiroshi Yamauchi | 565c2d9 | 2016-03-23 12:53:26 -0700 | [diff] [blame] | 373 | void RosAllocSpace::DumpStats(std::ostream& os) { |
| 374 | ScopedSuspendAll ssa(__FUNCTION__); |
| 375 | rosalloc_->DumpStats(os); |
| 376 | } |
| 377 | |
Andreas Gampe | d490129 | 2017-05-30 18:41:34 -0700 | [diff] [blame] | 378 | template<bool kMaybeIsRunningOnMemoryTool> |
| 379 | size_t RosAllocSpace::AllocationSizeNonvirtual(mirror::Object* obj, size_t* usable_size) { |
| 380 | // obj is a valid object. Use its class in the header to get the size. |
| 381 | // Don't use verification since the object may be dead if we are sweeping. |
| 382 | size_t size = obj->SizeOf<kVerifyNone>(); |
| 383 | bool add_redzones = false; |
| 384 | if (kMaybeIsRunningOnMemoryTool) { |
Andreas Gampe | 8b362a8 | 2018-05-22 20:54:14 +0000 | [diff] [blame] | 385 | add_redzones = RUNNING_ON_MEMORY_TOOL ? kMemoryToolAddsRedzones : 0; |
Andreas Gampe | d490129 | 2017-05-30 18:41:34 -0700 | [diff] [blame] | 386 | if (add_redzones) { |
| 387 | size += 2 * kDefaultMemoryToolRedZoneBytes; |
| 388 | } |
| 389 | } else { |
Andreas Gampe | 8b362a8 | 2018-05-22 20:54:14 +0000 | [diff] [blame] | 390 | DCHECK_EQ(RUNNING_ON_MEMORY_TOOL, 0U); |
Andreas Gampe | d490129 | 2017-05-30 18:41:34 -0700 | [diff] [blame] | 391 | } |
| 392 | size_t size_by_size = rosalloc_->UsableSize(size); |
| 393 | if (kIsDebugBuild) { |
| 394 | // On memory tool, the red zone has an impact... |
| 395 | const uint8_t* obj_ptr = reinterpret_cast<const uint8_t*>(obj); |
| 396 | size_t size_by_ptr = rosalloc_->UsableSize( |
| 397 | obj_ptr - (add_redzones ? kDefaultMemoryToolRedZoneBytes : 0)); |
| 398 | if (size_by_size != size_by_ptr) { |
| 399 | LOG(INFO) << "Found a bad sized obj of size " << size |
| 400 | << " at " << std::hex << reinterpret_cast<intptr_t>(obj_ptr) << std::dec |
| 401 | << " size_by_size=" << size_by_size << " size_by_ptr=" << size_by_ptr; |
| 402 | } |
| 403 | DCHECK_EQ(size_by_size, size_by_ptr); |
| 404 | } |
| 405 | if (usable_size != nullptr) { |
| 406 | *usable_size = size_by_size; |
| 407 | } |
| 408 | return size_by_size; |
| 409 | } |
| 410 | |
Hiroshi Yamauchi | cf58d4a | 2013-09-26 14:21:22 -0700 | [diff] [blame] | 411 | } // namespace space |
Andreas Gampe | 277ccbd | 2014-11-03 21:36:10 -0800 | [diff] [blame] | 412 | |
| 413 | namespace allocator { |
| 414 | |
| 415 | // Callback from rosalloc when it needs to increase the footprint. |
Mathieu Chartier | a9d82fe | 2016-01-25 20:06:11 -0800 | [diff] [blame] | 416 | void* ArtRosAllocMoreCore(allocator::RosAlloc* rosalloc, intptr_t increment) |
Andreas Gampe | bdf7f1c | 2016-08-30 16:38:47 -0700 | [diff] [blame] | 417 | REQUIRES_SHARED(Locks::mutator_lock_) { |
Andreas Gampe | 277ccbd | 2014-11-03 21:36:10 -0800 | [diff] [blame] | 418 | Heap* heap = Runtime::Current()->GetHeap(); |
| 419 | art::gc::space::RosAllocSpace* rosalloc_space = heap->GetRosAllocSpace(rosalloc); |
| 420 | DCHECK(rosalloc_space != nullptr); |
| 421 | DCHECK_EQ(rosalloc_space->GetRosAlloc(), rosalloc); |
| 422 | return rosalloc_space->MoreCore(increment); |
| 423 | } |
| 424 | |
| 425 | } // namespace allocator |
| 426 | |
Hiroshi Yamauchi | cf58d4a | 2013-09-26 14:21:22 -0700 | [diff] [blame] | 427 | } // namespace gc |
| 428 | } // namespace art |