blob: b591486f59b2d3f15cca26a6de629a09306ffe56 [file] [log] [blame]
Elliott Hughes2faa5f12012-01-30 14:42:07 -08001/*
2 * Copyright (C) 2011 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
Hiroshi Yamauchicf58d4a2013-09-26 14:21:22 -070016
Ian Rogers1d54e732013-05-02 21:10:01 -070017#include "dlmalloc_space.h"
Hiroshi Yamauchicf58d4a2013-09-26 14:21:22 -070018
Hiroshi Yamauchi50b29282013-07-30 13:58:37 -070019#include "dlmalloc_space-inl.h"
Ian Rogers1d54e732013-05-02 21:10:01 -070020#include "gc/accounting/card_table.h"
21#include "gc/heap.h"
Hiroshi Yamauchicf58d4a2013-09-26 14:21:22 -070022#include "mirror/class-inl.h"
Mathieu Chartier0f72e412013-09-06 16:40:01 -070023#include "mirror/object-inl.h"
Ian Rogers2dd0e2c2013-01-24 12:42:14 -080024#include "runtime.h"
Ian Rogers2dd0e2c2013-01-24 12:42:14 -080025#include "thread.h"
Hiroshi Yamauchicf58d4a2013-09-26 14:21:22 -070026#include "thread_list.h"
Brian Carlstrom578bbdc2011-07-21 14:07:47 -070027#include "utils.h"
Ian Rogers6fac4472014-02-25 17:01:10 -080028#include "valgrind_malloc_space-inl.h"
Ian Rogers1d54e732013-05-02 21:10:01 -070029
Carl Shapiro69759ea2011-07-21 18:13:35 -070030namespace art {
Ian Rogers1d54e732013-05-02 21:10:01 -070031namespace gc {
32namespace space {
Ian Rogers30fab402012-01-23 15:43:46 -080033
Ian Rogers6fac4472014-02-25 17:01:10 -080034static constexpr bool kPrefetchDuringDlMallocFreeList = true;
35
36template class ValgrindMallocSpace<DlMallocSpace, void*>;
Mathieu Chartier2fde5332012-09-14 14:51:54 -070037
Mathieu Chartier1c23e1e2012-10-12 14:14:11 -070038DlMallocSpace::DlMallocSpace(const std::string& name, MemMap* mem_map, void* mspace, byte* begin,
Hiroshi Yamauchicf58d4a2013-09-26 14:21:22 -070039 byte* end, byte* limit, size_t growth_limit)
40 : MallocSpace(name, mem_map, begin, end, limit, growth_limit),
Hiroshi Yamauchie48780b2013-12-17 17:19:53 -080041 mspace_(mspace), mspace_for_alloc_(mspace) {
Mathieu Chartierb062fdd2012-07-03 09:51:48 -070042 CHECK(mspace != NULL);
Mathieu Chartierb062fdd2012-07-03 09:51:48 -070043}
44
Mathieu Chartiere6da9af2013-12-16 11:54:42 -080045DlMallocSpace* DlMallocSpace::CreateFromMemMap(MemMap* mem_map, const std::string& name,
46 size_t starting_size,
47 size_t initial_size, size_t growth_limit,
48 size_t capacity) {
49 DCHECK(mem_map != nullptr);
Hiroshi Yamauchicf58d4a2013-09-26 14:21:22 -070050 void* mspace = CreateMspace(mem_map->Begin(), starting_size, initial_size);
Mathieu Chartiere6da9af2013-12-16 11:54:42 -080051 if (mspace == nullptr) {
Ian Rogers30fab402012-01-23 15:43:46 -080052 LOG(ERROR) << "Failed to initialize mspace for alloc space (" << name << ")";
Mathieu Chartiere6da9af2013-12-16 11:54:42 -080053 return nullptr;
Ian Rogers30fab402012-01-23 15:43:46 -080054 }
55
lzang1385de732014-02-21 14:15:01 +080056 // Protect memory beyond the starting size. morecore will add r/w permissions when necessory
Ian Rogers3bb17a62012-01-27 23:56:44 -080057 byte* end = mem_map->Begin() + starting_size;
lzang1385de732014-02-21 14:15:01 +080058 if (capacity - starting_size > 0) {
59 CHECK_MEMORY_CALL(mprotect, (end, capacity - starting_size, PROT_NONE), name);
Ian Rogers30fab402012-01-23 15:43:46 -080060 }
61
62 // Everything is set so record in immutable structure and leave
Hiroshi Yamauchicf58d4a2013-09-26 14:21:22 -070063 byte* begin = mem_map->Begin();
Ian Rogers1d54e732013-05-02 21:10:01 -070064 if (RUNNING_ON_VALGRIND > 0) {
Mathieu Chartiere6da9af2013-12-16 11:54:42 -080065 return new ValgrindMallocSpace<DlMallocSpace, void*>(
Hiroshi Yamauchi7cb7bbc2013-11-18 17:27:37 -080066 name, mem_map, mspace, begin, end, begin + capacity, growth_limit, initial_size);
Ian Rogers1d54e732013-05-02 21:10:01 -070067 } else {
Mathieu Chartiere6da9af2013-12-16 11:54:42 -080068 return new DlMallocSpace(name, mem_map, mspace, begin, end, begin + capacity, growth_limit);
Ian Rogers1d54e732013-05-02 21:10:01 -070069 }
Mathieu Chartiere6da9af2013-12-16 11:54:42 -080070}
71
72DlMallocSpace* DlMallocSpace::Create(const std::string& name, size_t initial_size, size_t growth_limit,
73 size_t capacity, byte* requested_begin) {
74 uint64_t start_time = 0;
75 if (VLOG_IS_ON(heap) || VLOG_IS_ON(startup)) {
76 start_time = NanoTime();
77 LOG(INFO) << "DlMallocSpace::Create entering " << name
78 << " initial_size=" << PrettySize(initial_size)
79 << " growth_limit=" << PrettySize(growth_limit)
80 << " capacity=" << PrettySize(capacity)
81 << " requested_begin=" << reinterpret_cast<void*>(requested_begin);
82 }
83
84 // Memory we promise to dlmalloc before it asks for morecore.
85 // Note: making this value large means that large allocations are unlikely to succeed as dlmalloc
86 // will ask for this memory from sys_alloc which will fail as the footprint (this value plus the
87 // size of the large allocation) will be greater than the footprint limit.
88 size_t starting_size = kPageSize;
89 MemMap* mem_map = CreateMemMap(name, starting_size, &initial_size, &growth_limit, &capacity,
90 requested_begin);
91 if (mem_map == nullptr) {
92 LOG(ERROR) << "Failed to create mem map for alloc space (" << name << ") of size "
93 << PrettySize(capacity);
94 return nullptr;
95 }
96 DlMallocSpace* space = CreateFromMemMap(mem_map, name, starting_size, initial_size,
97 growth_limit, capacity);
Mathieu Chartier590fee92013-09-13 13:46:47 -070098 // We start out with only the initial size possibly containing objects.
Ian Rogers30fab402012-01-23 15:43:46 -080099 if (VLOG_IS_ON(heap) || VLOG_IS_ON(startup)) {
Hiroshi Yamauchicf58d4a2013-09-26 14:21:22 -0700100 LOG(INFO) << "DlMallocSpace::Create exiting (" << PrettyDuration(NanoTime() - start_time)
Ian Rogers3bb17a62012-01-27 23:56:44 -0800101 << " ) " << *space;
Ian Rogers30fab402012-01-23 15:43:46 -0800102 }
103 return space;
Carl Shapiro69759ea2011-07-21 18:13:35 -0700104}
105
Hiroshi Yamauchicf58d4a2013-09-26 14:21:22 -0700106void* DlMallocSpace::CreateMspace(void* begin, size_t morecore_start, size_t initial_size) {
Ian Rogers30fab402012-01-23 15:43:46 -0800107 // clear errno to allow PLOG on error
Carl Shapiro69759ea2011-07-21 18:13:35 -0700108 errno = 0;
Ian Rogers3bb17a62012-01-27 23:56:44 -0800109 // create mspace using our backing storage starting at begin and with a footprint of
110 // morecore_start. Don't use an internal dlmalloc lock (as we already hold heap lock). When
111 // morecore_start bytes of memory is exhaused morecore will be called.
112 void* msp = create_mspace_with_base(begin, morecore_start, false /*locked*/);
Mathieu Chartiere6da9af2013-12-16 11:54:42 -0800113 if (msp != nullptr) {
Ian Rogers30fab402012-01-23 15:43:46 -0800114 // Do not allow morecore requests to succeed beyond the initial size of the heap
Ian Rogers3bb17a62012-01-27 23:56:44 -0800115 mspace_set_footprint_limit(msp, initial_size);
Carl Shapiro69759ea2011-07-21 18:13:35 -0700116 } else {
Ian Rogers30fab402012-01-23 15:43:46 -0800117 PLOG(ERROR) << "create_mspace_with_base failed";
Carl Shapiro69759ea2011-07-21 18:13:35 -0700118 }
119 return msp;
120}
121
Ian Rogers6fac4472014-02-25 17:01:10 -0800122mirror::Object* DlMallocSpace::AllocWithGrowth(Thread* self, size_t num_bytes,
123 size_t* bytes_allocated, size_t* usable_size) {
Hiroshi Yamauchi50b29282013-07-30 13:58:37 -0700124 mirror::Object* result;
125 {
126 MutexLock mu(self, lock_);
Hiroshi Yamauchicf58d4a2013-09-26 14:21:22 -0700127 // Grow as much as possible within the space.
Hiroshi Yamauchi50b29282013-07-30 13:58:37 -0700128 size_t max_allowed = Capacity();
129 mspace_set_footprint_limit(mspace_, max_allowed);
130 // Try the allocation.
Ian Rogers6fac4472014-02-25 17:01:10 -0800131 result = AllocWithoutGrowthLocked(self, num_bytes, bytes_allocated, usable_size);
Hiroshi Yamauchi50b29282013-07-30 13:58:37 -0700132 // Shrink back down as small as possible.
133 size_t footprint = mspace_footprint(mspace_);
134 mspace_set_footprint_limit(mspace_, footprint);
jeffhaoc1160702011-10-27 15:48:45 -0700135 }
Hiroshi Yamauchi50b29282013-07-30 13:58:37 -0700136 if (result != NULL) {
137 // Zero freshly allocated memory, done while not holding the space's lock.
138 memset(result, 0, num_bytes);
139 }
Ian Rogers30fab402012-01-23 15:43:46 -0800140 // Return the new allocation or NULL.
Mathieu Chartier8e9a1492012-10-04 12:25:40 -0700141 CHECK(!kDebugSpaces || result == NULL || Contains(result));
Ian Rogers30fab402012-01-23 15:43:46 -0800142 return result;
Brian Carlstrom4a289ed2011-08-16 17:17:49 -0700143}
144
Ian Rogers6fac4472014-02-25 17:01:10 -0800145MallocSpace* DlMallocSpace::CreateInstance(const std::string& name, MemMap* mem_map,
146 void* allocator, byte* begin, byte* end,
Hiroshi Yamauchicf58d4a2013-09-26 14:21:22 -0700147 byte* limit, size_t growth_limit) {
148 return new DlMallocSpace(name, mem_map, allocator, begin, end, limit, growth_limit);
Mathieu Chartier0f72e412013-09-06 16:40:01 -0700149}
150
Ian Rogers2dd0e2c2013-01-24 12:42:14 -0800151size_t DlMallocSpace::Free(Thread* self, mirror::Object* ptr) {
Ian Rogers50b35e22012-10-04 10:09:15 -0700152 MutexLock mu(self, lock_);
Mathieu Chartier8e9a1492012-10-04 12:25:40 -0700153 if (kDebugSpaces) {
Mathieu Chartier2fde5332012-09-14 14:51:54 -0700154 CHECK(ptr != NULL);
155 CHECK(Contains(ptr)) << "Free (" << ptr << ") not in bounds of heap " << *this;
156 }
Ian Rogers6fac4472014-02-25 17:01:10 -0800157 const size_t bytes_freed = AllocationSizeNonvirtual(ptr, nullptr);
Mathieu Chartier0f72e412013-09-06 16:40:01 -0700158 if (kRecentFreeCount > 0) {
159 RegisterRecentFree(ptr);
160 }
Ian Rogers30fab402012-01-23 15:43:46 -0800161 mspace_free(mspace_, ptr);
Mathieu Chartier1c23e1e2012-10-12 14:14:11 -0700162 return bytes_freed;
Ian Rogers30fab402012-01-23 15:43:46 -0800163}
164
Ian Rogers2dd0e2c2013-01-24 12:42:14 -0800165size_t DlMallocSpace::FreeList(Thread* self, size_t num_ptrs, mirror::Object** ptrs) {
Mathieu Chartierd22d5482012-11-06 17:14:12 -0800166 DCHECK(ptrs != NULL);
167
Mathieu Chartier1c23e1e2012-10-12 14:14:11 -0700168 // Don't need the lock to calculate the size of the freed pointers.
169 size_t bytes_freed = 0;
170 for (size_t i = 0; i < num_ptrs; i++) {
Ian Rogers2dd0e2c2013-01-24 12:42:14 -0800171 mirror::Object* ptr = ptrs[i];
Mathieu Chartierd22d5482012-11-06 17:14:12 -0800172 const size_t look_ahead = 8;
173 if (kPrefetchDuringDlMallocFreeList && i + look_ahead < num_ptrs) {
174 // The head of chunk for the allocation is sizeof(size_t) behind the allocation.
175 __builtin_prefetch(reinterpret_cast<char*>(ptrs[i + look_ahead]) - sizeof(size_t));
176 }
Ian Rogers6fac4472014-02-25 17:01:10 -0800177 bytes_freed += AllocationSizeNonvirtual(ptr, nullptr);
Mathieu Chartier1c23e1e2012-10-12 14:14:11 -0700178 }
179
Mathieu Chartier0f72e412013-09-06 16:40:01 -0700180 if (kRecentFreeCount > 0) {
181 MutexLock mu(self, lock_);
182 for (size_t i = 0; i < num_ptrs; i++) {
183 RegisterRecentFree(ptrs[i]);
184 }
185 }
186
Mathieu Chartier8e9a1492012-10-04 12:25:40 -0700187 if (kDebugSpaces) {
Mathieu Chartier2fde5332012-09-14 14:51:54 -0700188 size_t num_broken_ptrs = 0;
189 for (size_t i = 0; i < num_ptrs; i++) {
190 if (!Contains(ptrs[i])) {
191 num_broken_ptrs++;
192 LOG(ERROR) << "FreeList[" << i << "] (" << ptrs[i] << ") not in bounds of heap " << *this;
193 } else {
194 size_t size = mspace_usable_size(ptrs[i]);
195 memset(ptrs[i], 0xEF, size);
196 }
Ian Rogers30fab402012-01-23 15:43:46 -0800197 }
Mathieu Chartier2fde5332012-09-14 14:51:54 -0700198 CHECK_EQ(num_broken_ptrs, 0u);
Ian Rogers30fab402012-01-23 15:43:46 -0800199 }
Mathieu Chartierd22d5482012-11-06 17:14:12 -0800200
201 {
202 MutexLock mu(self, lock_);
Mathieu Chartierd22d5482012-11-06 17:14:12 -0800203 mspace_bulk_free(mspace_, reinterpret_cast<void**>(ptrs), num_ptrs);
204 return bytes_freed;
205 }
Ian Rogers30fab402012-01-23 15:43:46 -0800206}
207
208// Callback from dlmalloc when it needs to increase the footprint
209extern "C" void* art_heap_morecore(void* mspace, intptr_t increment) {
Elliott Hughesb3bd5f02012-03-08 21:05:27 -0800210 Heap* heap = Runtime::Current()->GetHeap();
Mathieu Chartiere6da9af2013-12-16 11:54:42 -0800211 DlMallocSpace* dlmalloc_space = heap->GetDlMallocSpace();
212 // Support for multiple DlMalloc provided by a slow path.
213 if (UNLIKELY(dlmalloc_space == nullptr || dlmalloc_space->GetMspace() != mspace)) {
214 dlmalloc_space = nullptr;
215 for (space::ContinuousSpace* space : heap->GetContinuousSpaces()) {
216 if (space->IsDlMallocSpace()) {
217 DlMallocSpace* cur_dlmalloc_space = space->AsDlMallocSpace();
218 if (cur_dlmalloc_space->GetMspace() == mspace) {
219 dlmalloc_space = cur_dlmalloc_space;
220 break;
221 }
222 }
223 }
224 CHECK(dlmalloc_space != nullptr) << "Couldn't find DlmMallocSpace with mspace=" << mspace;
225 }
226 return dlmalloc_space->MoreCore(increment);
Ian Rogers30fab402012-01-23 15:43:46 -0800227}
228
Ian Rogers48931882013-01-22 14:35:16 -0800229size_t DlMallocSpace::Trim() {
Ian Rogers50b35e22012-10-04 10:09:15 -0700230 MutexLock mu(Thread::Current(), lock_);
Elliott Hughes9eebd3b2012-06-08 13:56:31 -0700231 // Trim to release memory at the end of the space.
232 mspace_trim(mspace_, 0);
233 // Visit space looking for page-sized holes to advise the kernel we don't need.
Ian Rogers48931882013-01-22 14:35:16 -0800234 size_t reclaimed = 0;
Ian Rogers1d54e732013-05-02 21:10:01 -0700235 mspace_inspect_all(mspace_, DlmallocMadviseCallback, &reclaimed);
Ian Rogers48931882013-01-22 14:35:16 -0800236 return reclaimed;
Elliott Hughes9eebd3b2012-06-08 13:56:31 -0700237}
Ian Rogers30fab402012-01-23 15:43:46 -0800238
Mathieu Chartier1c23e1e2012-10-12 14:14:11 -0700239void DlMallocSpace::Walk(void(*callback)(void *start, void *end, size_t num_bytes, void* callback_arg),
Ian Rogers30fab402012-01-23 15:43:46 -0800240 void* arg) {
Ian Rogers50b35e22012-10-04 10:09:15 -0700241 MutexLock mu(Thread::Current(), lock_);
Ian Rogers30fab402012-01-23 15:43:46 -0800242 mspace_inspect_all(mspace_, callback, arg);
Ian Rogers15bf2d32012-08-28 17:33:04 -0700243 callback(NULL, NULL, 0, arg); // Indicate end of a space.
Ian Rogers30fab402012-01-23 15:43:46 -0800244}
245
Hiroshi Yamauchi09b07a92013-07-15 13:17:06 -0700246size_t DlMallocSpace::GetFootprint() {
247 MutexLock mu(Thread::Current(), lock_);
248 return mspace_footprint(mspace_);
249}
250
Mathieu Chartier1c23e1e2012-10-12 14:14:11 -0700251size_t DlMallocSpace::GetFootprintLimit() {
Ian Rogers50b35e22012-10-04 10:09:15 -0700252 MutexLock mu(Thread::Current(), lock_);
Ian Rogers30fab402012-01-23 15:43:46 -0800253 return mspace_footprint_limit(mspace_);
254}
255
Mathieu Chartier1c23e1e2012-10-12 14:14:11 -0700256void DlMallocSpace::SetFootprintLimit(size_t new_size) {
Ian Rogers50b35e22012-10-04 10:09:15 -0700257 MutexLock mu(Thread::Current(), lock_);
Hiroshi Yamauchicf58d4a2013-09-26 14:21:22 -0700258 VLOG(heap) << "DlMallocSpace::SetFootprintLimit " << PrettySize(new_size);
Ian Rogers30fab402012-01-23 15:43:46 -0800259 // Compare against the actual footprint, rather than the Size(), because the heap may not have
260 // grown all the way to the allowed size yet.
Ian Rogers30fab402012-01-23 15:43:46 -0800261 size_t current_space_size = mspace_footprint(mspace_);
262 if (new_size < current_space_size) {
263 // Don't let the space grow any more.
264 new_size = current_space_size;
265 }
266 mspace_set_footprint_limit(mspace_, new_size);
267}
268
Hiroshi Yamauchibe031ff2013-10-08 16:42:37 -0700269uint64_t DlMallocSpace::GetBytesAllocated() {
Hiroshi Yamauchi4ce1f002013-11-18 14:49:09 -0800270 MutexLock mu(Thread::Current(), lock_);
271 size_t bytes_allocated = 0;
272 mspace_inspect_all(mspace_, DlmallocBytesAllocatedCallback, &bytes_allocated);
273 return bytes_allocated;
Hiroshi Yamauchibe031ff2013-10-08 16:42:37 -0700274}
275
276uint64_t DlMallocSpace::GetObjectsAllocated() {
Hiroshi Yamauchi4ce1f002013-11-18 14:49:09 -0800277 MutexLock mu(Thread::Current(), lock_);
278 size_t objects_allocated = 0;
279 mspace_inspect_all(mspace_, DlmallocObjectsAllocatedCallback, &objects_allocated);
280 return objects_allocated;
Hiroshi Yamauchibe031ff2013-10-08 16:42:37 -0700281}
282
Mathieu Chartiere6da9af2013-12-16 11:54:42 -0800283void DlMallocSpace::Clear() {
284 madvise(GetMemMap()->Begin(), GetMemMap()->Size(), MADV_DONTNEED);
285 GetLiveBitmap()->Clear();
286 GetMarkBitmap()->Clear();
287}
288
Mathieu Chartier15d34022014-02-26 17:16:38 -0800289void DlMallocSpace::Reset() {
290 // TODO: Delete and create new mspace here.
291}
292
Hiroshi Yamauchicf58d4a2013-09-26 14:21:22 -0700293#ifndef NDEBUG
294void DlMallocSpace::CheckMoreCoreForPrecondition() {
295 lock_.AssertHeld(Thread::Current());
296}
297#endif
298
Ian Rogers1d54e732013-05-02 21:10:01 -0700299} // namespace space
300} // namespace gc
Carl Shapiro69759ea2011-07-21 18:13:35 -0700301} // namespace art