Hiroshi Yamauchi | cf58d4a | 2013-09-26 14:21:22 -0700 | [diff] [blame] | 1 | /* |
| 2 | * Copyright (C) 2013 The Android Open Source Project |
| 3 | * |
| 4 | * Licensed under the Apache License, Version 2.0 (the "License"); |
| 5 | * you may not use this file except in compliance with the License. |
| 6 | * You may obtain a copy of the License at |
| 7 | * |
| 8 | * http://www.apache.org/licenses/LICENSE-2.0 |
| 9 | * |
| 10 | * Unless required by applicable law or agreed to in writing, software |
| 11 | * distributed under the License is distributed on an "AS IS" BASIS, |
| 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 13 | * See the License for the specific language governing permissions and |
| 14 | * limitations under the License. |
| 15 | */ |
| 16 | |
| 17 | #include "malloc_space.h" |
| 18 | |
Mathieu Chartier | ec05007 | 2014-01-07 16:00:07 -0800 | [diff] [blame] | 19 | #include "gc/accounting/card_table-inl.h" |
| 20 | #include "gc/accounting/space_bitmap-inl.h" |
Hiroshi Yamauchi | cf58d4a | 2013-09-26 14:21:22 -0700 | [diff] [blame] | 21 | #include "gc/heap.h" |
| 22 | #include "mirror/class-inl.h" |
| 23 | #include "mirror/object-inl.h" |
| 24 | #include "runtime.h" |
| 25 | #include "thread.h" |
| 26 | #include "thread_list.h" |
| 27 | #include "utils.h" |
| 28 | |
| 29 | namespace art { |
| 30 | namespace gc { |
| 31 | namespace space { |
| 32 | |
| 33 | size_t MallocSpace::bitmap_index_ = 0; |
| 34 | |
| 35 | MallocSpace::MallocSpace(const std::string& name, MemMap* mem_map, |
| 36 | byte* begin, byte* end, byte* limit, size_t growth_limit) |
| 37 | : ContinuousMemMapAllocSpace(name, mem_map, begin, end, limit, kGcRetentionPolicyAlwaysCollect), |
| 38 | recent_free_pos_(0), lock_("allocation space lock", kAllocSpaceLock), |
| 39 | growth_limit_(growth_limit) { |
| 40 | size_t bitmap_index = bitmap_index_++; |
| 41 | static const uintptr_t kGcCardSize = static_cast<uintptr_t>(accounting::CardTable::kCardSize); |
| 42 | CHECK(IsAligned<kGcCardSize>(reinterpret_cast<uintptr_t>(mem_map->Begin()))); |
| 43 | CHECK(IsAligned<kGcCardSize>(reinterpret_cast<uintptr_t>(mem_map->End()))); |
| 44 | live_bitmap_.reset(accounting::SpaceBitmap::Create( |
| 45 | StringPrintf("allocspace %s live-bitmap %d", name.c_str(), static_cast<int>(bitmap_index)), |
| 46 | Begin(), Capacity())); |
| 47 | DCHECK(live_bitmap_.get() != NULL) << "could not create allocspace live bitmap #" << bitmap_index; |
| 48 | mark_bitmap_.reset(accounting::SpaceBitmap::Create( |
| 49 | StringPrintf("allocspace %s mark-bitmap %d", name.c_str(), static_cast<int>(bitmap_index)), |
| 50 | Begin(), Capacity())); |
| 51 | DCHECK(live_bitmap_.get() != NULL) << "could not create allocspace mark bitmap #" << bitmap_index; |
| 52 | for (auto& freed : recent_freed_objects_) { |
| 53 | freed.first = nullptr; |
| 54 | freed.second = nullptr; |
| 55 | } |
| 56 | } |
| 57 | |
| 58 | MemMap* MallocSpace::CreateMemMap(const std::string& name, size_t starting_size, size_t* initial_size, |
| 59 | size_t* growth_limit, size_t* capacity, byte* requested_begin) { |
| 60 | // Sanity check arguments |
| 61 | if (starting_size > *initial_size) { |
| 62 | *initial_size = starting_size; |
| 63 | } |
| 64 | if (*initial_size > *growth_limit) { |
| 65 | LOG(ERROR) << "Failed to create alloc space (" << name << ") where the initial size (" |
| 66 | << PrettySize(*initial_size) << ") is larger than its capacity (" |
| 67 | << PrettySize(*growth_limit) << ")"; |
| 68 | return NULL; |
| 69 | } |
| 70 | if (*growth_limit > *capacity) { |
| 71 | LOG(ERROR) << "Failed to create alloc space (" << name << ") where the growth limit capacity (" |
| 72 | << PrettySize(*growth_limit) << ") is larger than the capacity (" |
| 73 | << PrettySize(*capacity) << ")"; |
| 74 | return NULL; |
| 75 | } |
| 76 | |
| 77 | // Page align growth limit and capacity which will be used to manage mmapped storage |
| 78 | *growth_limit = RoundUp(*growth_limit, kPageSize); |
| 79 | *capacity = RoundUp(*capacity, kPageSize); |
| 80 | |
| 81 | std::string error_msg; |
| 82 | MemMap* mem_map = MemMap::MapAnonymous(name.c_str(), requested_begin, *capacity, |
| 83 | PROT_READ | PROT_WRITE, &error_msg); |
Mathieu Chartier | e6da9af | 2013-12-16 11:54:42 -0800 | [diff] [blame] | 84 | if (mem_map == nullptr) { |
Hiroshi Yamauchi | cf58d4a | 2013-09-26 14:21:22 -0700 | [diff] [blame] | 85 | LOG(ERROR) << "Failed to allocate pages for alloc space (" << name << ") of size " |
| 86 | << PrettySize(*capacity) << ": " << error_msg; |
Hiroshi Yamauchi | cf58d4a | 2013-09-26 14:21:22 -0700 | [diff] [blame] | 87 | } |
| 88 | return mem_map; |
| 89 | } |
| 90 | |
| 91 | void MallocSpace::SwapBitmaps() { |
| 92 | live_bitmap_.swap(mark_bitmap_); |
| 93 | // Swap names to get more descriptive diagnostics. |
| 94 | std::string temp_name(live_bitmap_->GetName()); |
| 95 | live_bitmap_->SetName(mark_bitmap_->GetName()); |
| 96 | mark_bitmap_->SetName(temp_name); |
| 97 | } |
| 98 | |
| 99 | mirror::Class* MallocSpace::FindRecentFreedObject(const mirror::Object* obj) { |
| 100 | size_t pos = recent_free_pos_; |
| 101 | // Start at the most recently freed object and work our way back since there may be duplicates |
| 102 | // caused by dlmalloc reusing memory. |
| 103 | if (kRecentFreeCount > 0) { |
| 104 | for (size_t i = 0; i + 1 < kRecentFreeCount + 1; ++i) { |
| 105 | pos = pos != 0 ? pos - 1 : kRecentFreeMask; |
| 106 | if (recent_freed_objects_[pos].first == obj) { |
| 107 | return recent_freed_objects_[pos].second; |
| 108 | } |
| 109 | } |
| 110 | } |
| 111 | return nullptr; |
| 112 | } |
| 113 | |
| 114 | void MallocSpace::RegisterRecentFree(mirror::Object* ptr) { |
| 115 | recent_freed_objects_[recent_free_pos_] = std::make_pair(ptr, ptr->GetClass()); |
| 116 | recent_free_pos_ = (recent_free_pos_ + 1) & kRecentFreeMask; |
| 117 | } |
| 118 | |
| 119 | void MallocSpace::SetGrowthLimit(size_t growth_limit) { |
| 120 | growth_limit = RoundUp(growth_limit, kPageSize); |
| 121 | growth_limit_ = growth_limit; |
| 122 | if (Size() > growth_limit_) { |
| 123 | end_ = begin_ + growth_limit; |
| 124 | } |
| 125 | } |
| 126 | |
| 127 | void* MallocSpace::MoreCore(intptr_t increment) { |
| 128 | CheckMoreCoreForPrecondition(); |
| 129 | byte* original_end = end_; |
| 130 | if (increment != 0) { |
| 131 | VLOG(heap) << "MallocSpace::MoreCore " << PrettySize(increment); |
| 132 | byte* new_end = original_end + increment; |
| 133 | if (increment > 0) { |
| 134 | // Should never be asked to increase the allocation beyond the capacity of the space. Enforced |
| 135 | // by mspace_set_footprint_limit. |
| 136 | CHECK_LE(new_end, Begin() + Capacity()); |
| 137 | CHECK_MEMORY_CALL(mprotect, (original_end, increment, PROT_READ | PROT_WRITE), GetName()); |
| 138 | } else { |
| 139 | // Should never be asked for negative footprint (ie before begin). Zero footprint is ok. |
| 140 | CHECK_GE(original_end + increment, Begin()); |
| 141 | // Advise we don't need the pages and protect them |
| 142 | // TODO: by removing permissions to the pages we may be causing TLB shoot-down which can be |
| 143 | // expensive (note the same isn't true for giving permissions to a page as the protected |
| 144 | // page shouldn't be in a TLB). We should investigate performance impact of just |
| 145 | // removing ignoring the memory protection change here and in Space::CreateAllocSpace. It's |
| 146 | // likely just a useful debug feature. |
| 147 | size_t size = -increment; |
| 148 | CHECK_MEMORY_CALL(madvise, (new_end, size, MADV_DONTNEED), GetName()); |
| 149 | CHECK_MEMORY_CALL(mprotect, (new_end, size, PROT_NONE), GetName()); |
| 150 | } |
| 151 | // Update end_ |
| 152 | end_ = new_end; |
| 153 | } |
| 154 | return original_end; |
| 155 | } |
| 156 | |
| 157 | // Returns the old mark bitmap. |
| 158 | accounting::SpaceBitmap* MallocSpace::BindLiveToMarkBitmap() { |
| 159 | accounting::SpaceBitmap* live_bitmap = GetLiveBitmap(); |
| 160 | accounting::SpaceBitmap* mark_bitmap = mark_bitmap_.release(); |
| 161 | temp_bitmap_.reset(mark_bitmap); |
| 162 | mark_bitmap_.reset(live_bitmap); |
| 163 | return mark_bitmap; |
| 164 | } |
| 165 | |
| 166 | bool MallocSpace::HasBoundBitmaps() const { |
| 167 | return temp_bitmap_.get() != nullptr; |
| 168 | } |
| 169 | |
| 170 | void MallocSpace::UnBindBitmaps() { |
| 171 | CHECK(HasBoundBitmaps()); |
| 172 | // At this point, the temp_bitmap holds our old mark bitmap. |
| 173 | accounting::SpaceBitmap* new_bitmap = temp_bitmap_.release(); |
| 174 | CHECK_EQ(mark_bitmap_.release(), live_bitmap_.get()); |
| 175 | mark_bitmap_.reset(new_bitmap); |
| 176 | DCHECK(temp_bitmap_.get() == NULL); |
| 177 | } |
| 178 | |
Hiroshi Yamauchi | 573f7d2 | 2013-12-17 11:54:23 -0800 | [diff] [blame] | 179 | MallocSpace* MallocSpace::CreateZygoteSpace(const char* alloc_space_name, bool low_memory_mode) { |
Hiroshi Yamauchi | cf58d4a | 2013-09-26 14:21:22 -0700 | [diff] [blame] | 180 | // For RosAlloc, revoke thread local runs before creating a new |
| 181 | // alloc space so that we won't mix thread local runs from different |
| 182 | // alloc spaces. |
| 183 | RevokeAllThreadLocalBuffers(); |
| 184 | end_ = reinterpret_cast<byte*>(RoundUp(reinterpret_cast<uintptr_t>(end_), kPageSize)); |
| 185 | DCHECK(IsAligned<accounting::CardTable::kCardSize>(begin_)); |
| 186 | DCHECK(IsAligned<accounting::CardTable::kCardSize>(end_)); |
| 187 | DCHECK(IsAligned<kPageSize>(begin_)); |
| 188 | DCHECK(IsAligned<kPageSize>(end_)); |
| 189 | size_t size = RoundUp(Size(), kPageSize); |
Mathieu Chartier | 85a43c0 | 2014-01-07 17:59:00 -0800 | [diff] [blame^] | 190 | // Trimming the heap should be done by the caller since we may have invalidated the accounting |
| 191 | // stored in between objects. |
Hiroshi Yamauchi | cf58d4a | 2013-09-26 14:21:22 -0700 | [diff] [blame] | 192 | // Remaining size is for the new alloc space. |
| 193 | const size_t growth_limit = growth_limit_ - size; |
| 194 | const size_t capacity = Capacity() - size; |
| 195 | VLOG(heap) << "Begin " << reinterpret_cast<const void*>(begin_) << "\n" |
| 196 | << "End " << reinterpret_cast<const void*>(end_) << "\n" |
| 197 | << "Size " << size << "\n" |
| 198 | << "GrowthLimit " << growth_limit_ << "\n" |
| 199 | << "Capacity " << Capacity(); |
| 200 | SetGrowthLimit(RoundUp(size, kPageSize)); |
| 201 | SetFootprintLimit(RoundUp(size, kPageSize)); |
Mathieu Chartier | e6da9af | 2013-12-16 11:54:42 -0800 | [diff] [blame] | 202 | |
| 203 | // TODO: Not hardcode these in? |
| 204 | const size_t starting_size = kPageSize; |
| 205 | const size_t initial_size = 2 * MB; |
Hiroshi Yamauchi | cf58d4a | 2013-09-26 14:21:22 -0700 | [diff] [blame] | 206 | // FIXME: Do we need reference counted pointers here? |
| 207 | // Make the two spaces share the same mark bitmaps since the bitmaps span both of the spaces. |
| 208 | VLOG(heap) << "Creating new AllocSpace: "; |
| 209 | VLOG(heap) << "Size " << GetMemMap()->Size(); |
| 210 | VLOG(heap) << "GrowthLimit " << PrettySize(growth_limit); |
| 211 | VLOG(heap) << "Capacity " << PrettySize(capacity); |
| 212 | // Remap the tail. |
| 213 | std::string error_msg; |
| 214 | UniquePtr<MemMap> mem_map(GetMemMap()->RemapAtEnd(end_, alloc_space_name, |
| 215 | PROT_READ | PROT_WRITE, &error_msg)); |
| 216 | CHECK(mem_map.get() != nullptr) << error_msg; |
Hiroshi Yamauchi | 573f7d2 | 2013-12-17 11:54:23 -0800 | [diff] [blame] | 217 | void* allocator = CreateAllocator(end_, starting_size, initial_size, low_memory_mode); |
Hiroshi Yamauchi | cf58d4a | 2013-09-26 14:21:22 -0700 | [diff] [blame] | 218 | // Protect memory beyond the initial size. |
| 219 | byte* end = mem_map->Begin() + starting_size; |
| 220 | if (capacity - initial_size > 0) { |
| 221 | CHECK_MEMORY_CALL(mprotect, (end, capacity - initial_size, PROT_NONE), alloc_space_name); |
| 222 | } |
| 223 | MallocSpace* alloc_space = CreateInstance(alloc_space_name, mem_map.release(), allocator, |
| 224 | end_, end, limit_, growth_limit); |
| 225 | SetLimit(End()); |
| 226 | live_bitmap_->SetHeapLimit(reinterpret_cast<uintptr_t>(End())); |
| 227 | CHECK_EQ(live_bitmap_->HeapLimit(), reinterpret_cast<uintptr_t>(End())); |
| 228 | mark_bitmap_->SetHeapLimit(reinterpret_cast<uintptr_t>(End())); |
| 229 | CHECK_EQ(mark_bitmap_->HeapLimit(), reinterpret_cast<uintptr_t>(End())); |
| 230 | VLOG(heap) << "zygote space creation done"; |
| 231 | return alloc_space; |
| 232 | } |
| 233 | |
| 234 | void MallocSpace::Dump(std::ostream& os) const { |
| 235 | os << GetType() |
| 236 | << " begin=" << reinterpret_cast<void*>(Begin()) |
| 237 | << ",end=" << reinterpret_cast<void*>(End()) |
| 238 | << ",size=" << PrettySize(Size()) << ",capacity=" << PrettySize(Capacity()) |
| 239 | << ",name=\"" << GetName() << "\"]"; |
| 240 | } |
| 241 | |
Mathieu Chartier | ec05007 | 2014-01-07 16:00:07 -0800 | [diff] [blame] | 242 | struct SweepCallbackContext { |
| 243 | bool swap_bitmaps; |
| 244 | Heap* heap; |
| 245 | space::MallocSpace* space; |
| 246 | Thread* self; |
| 247 | size_t freed_objects; |
| 248 | size_t freed_bytes; |
| 249 | }; |
| 250 | |
| 251 | static void SweepCallback(size_t num_ptrs, mirror::Object** ptrs, void* arg) { |
| 252 | SweepCallbackContext* context = static_cast<SweepCallbackContext*>(arg); |
| 253 | space::AllocSpace* space = context->space; |
| 254 | Thread* self = context->self; |
| 255 | Locks::heap_bitmap_lock_->AssertExclusiveHeld(self); |
| 256 | // If the bitmaps aren't swapped we need to clear the bits since the GC isn't going to re-swap |
| 257 | // the bitmaps as an optimization. |
| 258 | if (!context->swap_bitmaps) { |
| 259 | accounting::SpaceBitmap* bitmap = context->space->GetLiveBitmap(); |
| 260 | for (size_t i = 0; i < num_ptrs; ++i) { |
| 261 | bitmap->Clear(ptrs[i]); |
| 262 | } |
| 263 | } |
| 264 | // Use a bulk free, that merges consecutive objects before freeing or free per object? |
| 265 | // Documentation suggests better free performance with merging, but this may be at the expensive |
| 266 | // of allocation. |
| 267 | context->freed_objects += num_ptrs; |
| 268 | context->freed_bytes += space->FreeList(self, num_ptrs, ptrs); |
| 269 | } |
| 270 | |
| 271 | static void ZygoteSweepCallback(size_t num_ptrs, mirror::Object** ptrs, void* arg) { |
| 272 | SweepCallbackContext* context = static_cast<SweepCallbackContext*>(arg); |
| 273 | Locks::heap_bitmap_lock_->AssertExclusiveHeld(context->self); |
| 274 | accounting::CardTable* card_table = context->heap->GetCardTable(); |
| 275 | // If the bitmaps aren't swapped we need to clear the bits since the GC isn't going to re-swap |
| 276 | // the bitmaps as an optimization. |
| 277 | if (!context->swap_bitmaps) { |
| 278 | accounting::SpaceBitmap* bitmap = context->space->GetLiveBitmap(); |
| 279 | for (size_t i = 0; i < num_ptrs; ++i) { |
| 280 | bitmap->Clear(ptrs[i]); |
| 281 | } |
| 282 | } |
| 283 | // We don't free any actual memory to avoid dirtying the shared zygote pages. |
| 284 | for (size_t i = 0; i < num_ptrs; ++i) { |
| 285 | // Need to mark the card since this will update the mod-union table next GC cycle. |
| 286 | card_table->MarkCard(ptrs[i]); |
| 287 | } |
| 288 | } |
| 289 | |
| 290 | void MallocSpace::Sweep(bool swap_bitmaps, size_t* freed_objects, size_t* freed_bytes) { |
| 291 | DCHECK(freed_objects != nullptr); |
| 292 | DCHECK(freed_bytes != nullptr); |
| 293 | accounting::SpaceBitmap* live_bitmap = GetLiveBitmap(); |
| 294 | accounting::SpaceBitmap* mark_bitmap = GetMarkBitmap(); |
| 295 | // If the bitmaps are bound then sweeping this space clearly won't do anything. |
| 296 | if (live_bitmap == mark_bitmap) { |
| 297 | return; |
| 298 | } |
| 299 | SweepCallbackContext scc; |
| 300 | scc.swap_bitmaps = swap_bitmaps; |
| 301 | scc.heap = Runtime::Current()->GetHeap(); |
| 302 | scc.self = Thread::Current(); |
| 303 | scc.space = this; |
| 304 | scc.freed_objects = 0; |
| 305 | scc.freed_bytes = 0; |
| 306 | if (swap_bitmaps) { |
| 307 | std::swap(live_bitmap, mark_bitmap); |
| 308 | } |
| 309 | // Bitmaps are pre-swapped for optimization which enables sweeping with the heap unlocked. |
| 310 | accounting::SpaceBitmap::SweepWalk(*live_bitmap, *mark_bitmap, |
| 311 | reinterpret_cast<uintptr_t>(Begin()), |
| 312 | reinterpret_cast<uintptr_t>(End()), |
| 313 | IsZygoteSpace() ? &ZygoteSweepCallback : &SweepCallback, |
| 314 | reinterpret_cast<void*>(&scc)); |
| 315 | *freed_objects += scc.freed_objects; |
| 316 | *freed_bytes += scc.freed_bytes; |
| 317 | } |
| 318 | |
Hiroshi Yamauchi | cf58d4a | 2013-09-26 14:21:22 -0700 | [diff] [blame] | 319 | } // namespace space |
| 320 | } // namespace gc |
| 321 | } // namespace art |