Mathieu Chartier | 590fee9 | 2013-09-13 13:46:47 -0700 | [diff] [blame] | 1 | /* |
| 2 | * Copyright (C) 2013 The Android Open Source Project |
| 3 | * |
| 4 | * Licensed under the Apache License, Version 2.0 (the "License"); |
| 5 | * you may not use this file except in compliance with the License. |
| 6 | * You may obtain a copy of the License at |
| 7 | * |
| 8 | * http://www.apache.org/licenses/LICENSE-2.0 |
| 9 | * |
| 10 | * Unless required by applicable law or agreed to in writing, software |
| 11 | * distributed under the License is distributed on an "AS IS" BASIS, |
| 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 13 | * See the License for the specific language governing permissions and |
| 14 | * limitations under the License. |
| 15 | */ |
| 16 | |
| 17 | #include "bump_pointer_space.h" |
| 18 | #include "bump_pointer_space-inl.h" |
| 19 | #include "mirror/object-inl.h" |
| 20 | #include "mirror/class-inl.h" |
Mathieu Chartier | 692fafd | 2013-11-29 17:24:40 -0800 | [diff] [blame] | 21 | #include "thread_list.h" |
Mathieu Chartier | 590fee9 | 2013-09-13 13:46:47 -0700 | [diff] [blame] | 22 | |
| 23 | namespace art { |
| 24 | namespace gc { |
| 25 | namespace space { |
| 26 | |
| 27 | BumpPointerSpace* BumpPointerSpace::Create(const std::string& name, size_t capacity, |
Ian Rogers | 1373595 | 2014-10-08 12:43:28 -0700 | [diff] [blame] | 28 | uint8_t* requested_begin) { |
Mathieu Chartier | 590fee9 | 2013-09-13 13:46:47 -0700 | [diff] [blame] | 29 | capacity = RoundUp(capacity, kPageSize); |
| 30 | std::string error_msg; |
Ian Rogers | 700a402 | 2014-05-19 16:49:03 -0700 | [diff] [blame] | 31 | std::unique_ptr<MemMap> mem_map(MemMap::MapAnonymous(name.c_str(), requested_begin, capacity, |
Vladimir Marko | 5c42c29 | 2015-02-25 12:02:49 +0000 | [diff] [blame] | 32 | PROT_READ | PROT_WRITE, true, false, |
| 33 | &error_msg)); |
Mathieu Chartier | 590fee9 | 2013-09-13 13:46:47 -0700 | [diff] [blame] | 34 | if (mem_map.get() == nullptr) { |
| 35 | LOG(ERROR) << "Failed to allocate pages for alloc space (" << name << ") of size " |
| 36 | << PrettySize(capacity) << " with message " << error_msg; |
| 37 | return nullptr; |
| 38 | } |
| 39 | return new BumpPointerSpace(name, mem_map.release()); |
| 40 | } |
| 41 | |
Mathieu Chartier | 31f4414 | 2014-04-08 14:40:03 -0700 | [diff] [blame] | 42 | BumpPointerSpace* BumpPointerSpace::CreateFromMemMap(const std::string& name, MemMap* mem_map) { |
| 43 | return new BumpPointerSpace(name, mem_map); |
| 44 | } |
| 45 | |
Ian Rogers | 1373595 | 2014-10-08 12:43:28 -0700 | [diff] [blame] | 46 | BumpPointerSpace::BumpPointerSpace(const std::string& name, uint8_t* begin, uint8_t* limit) |
Mathieu Chartier | 590fee9 | 2013-09-13 13:46:47 -0700 | [diff] [blame] | 47 | : ContinuousMemMapAllocSpace(name, nullptr, begin, begin, limit, |
| 48 | kGcRetentionPolicyAlwaysCollect), |
Mathieu Chartier | 692fafd | 2013-11-29 17:24:40 -0800 | [diff] [blame] | 49 | growth_end_(limit), |
| 50 | objects_allocated_(0), bytes_allocated_(0), |
| 51 | block_lock_("Block lock"), |
Mathieu Chartier | fc4c27e | 2014-02-11 11:05:41 -0800 | [diff] [blame] | 52 | main_block_size_(0), |
Mathieu Chartier | 692fafd | 2013-11-29 17:24:40 -0800 | [diff] [blame] | 53 | num_blocks_(0) { |
Mathieu Chartier | 590fee9 | 2013-09-13 13:46:47 -0700 | [diff] [blame] | 54 | } |
| 55 | |
| 56 | BumpPointerSpace::BumpPointerSpace(const std::string& name, MemMap* mem_map) |
| 57 | : ContinuousMemMapAllocSpace(name, mem_map, mem_map->Begin(), mem_map->Begin(), mem_map->End(), |
| 58 | kGcRetentionPolicyAlwaysCollect), |
Mathieu Chartier | 692fafd | 2013-11-29 17:24:40 -0800 | [diff] [blame] | 59 | growth_end_(mem_map->End()), |
| 60 | objects_allocated_(0), bytes_allocated_(0), |
Hiroshi Yamauchi | 2cd334a | 2015-01-09 14:03:35 -0800 | [diff] [blame] | 61 | block_lock_("Block lock", kBumpPointerSpaceBlockLock), |
Mathieu Chartier | fc4c27e | 2014-02-11 11:05:41 -0800 | [diff] [blame] | 62 | main_block_size_(0), |
Mathieu Chartier | 692fafd | 2013-11-29 17:24:40 -0800 | [diff] [blame] | 63 | num_blocks_(0) { |
Mathieu Chartier | 590fee9 | 2013-09-13 13:46:47 -0700 | [diff] [blame] | 64 | } |
| 65 | |
Mathieu Chartier | 590fee9 | 2013-09-13 13:46:47 -0700 | [diff] [blame] | 66 | void BumpPointerSpace::Clear() { |
| 67 | // Release the pages back to the operating system. |
Ian Rogers | c5f1773 | 2014-06-05 20:48:42 -0700 | [diff] [blame] | 68 | if (!kMadviseZeroes) { |
| 69 | memset(Begin(), 0, Limit() - Begin()); |
| 70 | } |
Mathieu Chartier | 590fee9 | 2013-09-13 13:46:47 -0700 | [diff] [blame] | 71 | CHECK_NE(madvise(Begin(), Limit() - Begin(), MADV_DONTNEED), -1) << "madvise failed"; |
| 72 | // Reset the end of the space back to the beginning, we move the end forward as we allocate |
| 73 | // objects. |
Mathieu Chartier | fc4c27e | 2014-02-11 11:05:41 -0800 | [diff] [blame] | 74 | SetEnd(Begin()); |
Ian Rogers | 3e5cf30 | 2014-05-20 16:40:37 -0700 | [diff] [blame] | 75 | objects_allocated_.StoreRelaxed(0); |
| 76 | bytes_allocated_.StoreRelaxed(0); |
Mathieu Chartier | 590fee9 | 2013-09-13 13:46:47 -0700 | [diff] [blame] | 77 | growth_end_ = Limit(); |
Mathieu Chartier | 692fafd | 2013-11-29 17:24:40 -0800 | [diff] [blame] | 78 | { |
| 79 | MutexLock mu(Thread::Current(), block_lock_); |
| 80 | num_blocks_ = 0; |
Mathieu Chartier | fc4c27e | 2014-02-11 11:05:41 -0800 | [diff] [blame] | 81 | main_block_size_ = 0; |
Mathieu Chartier | 692fafd | 2013-11-29 17:24:40 -0800 | [diff] [blame] | 82 | } |
Mathieu Chartier | 590fee9 | 2013-09-13 13:46:47 -0700 | [diff] [blame] | 83 | } |
| 84 | |
| 85 | void BumpPointerSpace::Dump(std::ostream& os) const { |
Mathieu Chartier | 15d3402 | 2014-02-26 17:16:38 -0800 | [diff] [blame] | 86 | os << GetName() << " " |
| 87 | << reinterpret_cast<void*>(Begin()) << "-" << reinterpret_cast<void*>(End()) << " - " |
| 88 | << reinterpret_cast<void*>(Limit()); |
Mathieu Chartier | 590fee9 | 2013-09-13 13:46:47 -0700 | [diff] [blame] | 89 | } |
| 90 | |
| 91 | mirror::Object* BumpPointerSpace::GetNextObject(mirror::Object* obj) { |
| 92 | const uintptr_t position = reinterpret_cast<uintptr_t>(obj) + obj->SizeOf(); |
| 93 | return reinterpret_cast<mirror::Object*>(RoundUp(position, kAlignment)); |
| 94 | } |
| 95 | |
Hiroshi Yamauchi | 4460a84 | 2015-03-09 11:57:48 -0700 | [diff] [blame] | 96 | size_t BumpPointerSpace::RevokeThreadLocalBuffers(Thread* thread) { |
Mathieu Chartier | 692fafd | 2013-11-29 17:24:40 -0800 | [diff] [blame] | 97 | MutexLock mu(Thread::Current(), block_lock_); |
| 98 | RevokeThreadLocalBuffersLocked(thread); |
Hiroshi Yamauchi | 4460a84 | 2015-03-09 11:57:48 -0700 | [diff] [blame] | 99 | return 0U; |
Mathieu Chartier | 692fafd | 2013-11-29 17:24:40 -0800 | [diff] [blame] | 100 | } |
| 101 | |
Hiroshi Yamauchi | 4460a84 | 2015-03-09 11:57:48 -0700 | [diff] [blame] | 102 | size_t BumpPointerSpace::RevokeAllThreadLocalBuffers() { |
Mathieu Chartier | 692fafd | 2013-11-29 17:24:40 -0800 | [diff] [blame] | 103 | Thread* self = Thread::Current(); |
| 104 | MutexLock mu(self, *Locks::runtime_shutdown_lock_); |
| 105 | MutexLock mu2(self, *Locks::thread_list_lock_); |
| 106 | // TODO: Not do a copy of the thread list? |
| 107 | std::list<Thread*> thread_list = Runtime::Current()->GetThreadList()->GetList(); |
| 108 | for (Thread* thread : thread_list) { |
| 109 | RevokeThreadLocalBuffers(thread); |
| 110 | } |
Hiroshi Yamauchi | 4460a84 | 2015-03-09 11:57:48 -0700 | [diff] [blame] | 111 | return 0U; |
Mathieu Chartier | 692fafd | 2013-11-29 17:24:40 -0800 | [diff] [blame] | 112 | } |
| 113 | |
Hiroshi Yamauchi | c93c530 | 2014-03-20 16:15:37 -0700 | [diff] [blame] | 114 | void BumpPointerSpace::AssertThreadLocalBuffersAreRevoked(Thread* thread) { |
| 115 | if (kIsDebugBuild) { |
| 116 | MutexLock mu(Thread::Current(), block_lock_); |
| 117 | DCHECK(!thread->HasTlab()); |
| 118 | } |
| 119 | } |
| 120 | |
| 121 | void BumpPointerSpace::AssertAllThreadLocalBuffersAreRevoked() { |
| 122 | if (kIsDebugBuild) { |
| 123 | Thread* self = Thread::Current(); |
| 124 | MutexLock mu(self, *Locks::runtime_shutdown_lock_); |
| 125 | MutexLock mu2(self, *Locks::thread_list_lock_); |
| 126 | // TODO: Not do a copy of the thread list? |
| 127 | std::list<Thread*> thread_list = Runtime::Current()->GetThreadList()->GetList(); |
| 128 | for (Thread* thread : thread_list) { |
| 129 | AssertThreadLocalBuffersAreRevoked(thread); |
| 130 | } |
| 131 | } |
| 132 | } |
| 133 | |
Mathieu Chartier | 692fafd | 2013-11-29 17:24:40 -0800 | [diff] [blame] | 134 | void BumpPointerSpace::UpdateMainBlock() { |
Mathieu Chartier | 692fafd | 2013-11-29 17:24:40 -0800 | [diff] [blame] | 135 | DCHECK_EQ(num_blocks_, 0U); |
Mathieu Chartier | fc4c27e | 2014-02-11 11:05:41 -0800 | [diff] [blame] | 136 | main_block_size_ = Size(); |
Mathieu Chartier | 692fafd | 2013-11-29 17:24:40 -0800 | [diff] [blame] | 137 | } |
| 138 | |
| 139 | // Returns the start of the storage. |
Ian Rogers | 1373595 | 2014-10-08 12:43:28 -0700 | [diff] [blame] | 140 | uint8_t* BumpPointerSpace::AllocBlock(size_t bytes) { |
Mathieu Chartier | 692fafd | 2013-11-29 17:24:40 -0800 | [diff] [blame] | 141 | bytes = RoundUp(bytes, kAlignment); |
| 142 | if (!num_blocks_) { |
| 143 | UpdateMainBlock(); |
| 144 | } |
Ian Rogers | 1373595 | 2014-10-08 12:43:28 -0700 | [diff] [blame] | 145 | uint8_t* storage = reinterpret_cast<uint8_t*>( |
Mathieu Chartier | 692fafd | 2013-11-29 17:24:40 -0800 | [diff] [blame] | 146 | AllocNonvirtualWithoutAccounting(bytes + sizeof(BlockHeader))); |
| 147 | if (LIKELY(storage != nullptr)) { |
| 148 | BlockHeader* header = reinterpret_cast<BlockHeader*>(storage); |
| 149 | header->size_ = bytes; // Write out the block header. |
| 150 | storage += sizeof(BlockHeader); |
| 151 | ++num_blocks_; |
| 152 | } |
| 153 | return storage; |
| 154 | } |
| 155 | |
Mathieu Chartier | 83c8ee0 | 2014-01-28 14:50:23 -0800 | [diff] [blame] | 156 | void BumpPointerSpace::Walk(ObjectCallback* callback, void* arg) { |
Ian Rogers | 1373595 | 2014-10-08 12:43:28 -0700 | [diff] [blame] | 157 | uint8_t* pos = Begin(); |
| 158 | uint8_t* end = End(); |
| 159 | uint8_t* main_end = pos; |
Mathieu Chartier | 692fafd | 2013-11-29 17:24:40 -0800 | [diff] [blame] | 160 | { |
| 161 | MutexLock mu(Thread::Current(), block_lock_); |
| 162 | // If we have 0 blocks then we need to update the main header since we have bump pointer style |
| 163 | // allocation into an unbounded region (actually bounded by Capacity()). |
| 164 | if (num_blocks_ == 0) { |
| 165 | UpdateMainBlock(); |
| 166 | } |
Mathieu Chartier | 8544b46 | 2014-02-12 17:47:42 -0800 | [diff] [blame] | 167 | main_end = Begin() + main_block_size_; |
| 168 | if (num_blocks_ == 0) { |
| 169 | // We don't have any other blocks, this means someone else may be allocating into the main |
| 170 | // block. In this case, we don't want to try and visit the other blocks after the main block |
| 171 | // since these could actually be part of the main block. |
| 172 | end = main_end; |
| 173 | } |
Mathieu Chartier | 692fafd | 2013-11-29 17:24:40 -0800 | [diff] [blame] | 174 | } |
Mathieu Chartier | fc4c27e | 2014-02-11 11:05:41 -0800 | [diff] [blame] | 175 | // Walk all of the objects in the main block first. |
| 176 | while (pos < main_end) { |
| 177 | mirror::Object* obj = reinterpret_cast<mirror::Object*>(pos); |
Hiroshi Yamauchi | 2cd334a | 2015-01-09 14:03:35 -0800 | [diff] [blame] | 178 | // No read barrier because obj may not be a valid object. |
| 179 | if (obj->GetClass<kDefaultVerifyFlags, kWithoutReadBarrier>() == nullptr) { |
Mathieu Chartier | 8544b46 | 2014-02-12 17:47:42 -0800 | [diff] [blame] | 180 | // There is a race condition where a thread has just allocated an object but not set the |
| 181 | // class. We can't know the size of this object, so we don't visit it and exit the function |
| 182 | // since there is guaranteed to be not other blocks. |
| 183 | return; |
| 184 | } else { |
| 185 | callback(obj, arg); |
Ian Rogers | 1373595 | 2014-10-08 12:43:28 -0700 | [diff] [blame] | 186 | pos = reinterpret_cast<uint8_t*>(GetNextObject(obj)); |
Mathieu Chartier | 8544b46 | 2014-02-12 17:47:42 -0800 | [diff] [blame] | 187 | } |
Mathieu Chartier | fc4c27e | 2014-02-11 11:05:41 -0800 | [diff] [blame] | 188 | } |
| 189 | // Walk the other blocks (currently only TLABs). |
Mathieu Chartier | 8544b46 | 2014-02-12 17:47:42 -0800 | [diff] [blame] | 190 | while (pos < end) { |
Mathieu Chartier | 692fafd | 2013-11-29 17:24:40 -0800 | [diff] [blame] | 191 | BlockHeader* header = reinterpret_cast<BlockHeader*>(pos); |
| 192 | size_t block_size = header->size_; |
| 193 | pos += sizeof(BlockHeader); // Skip the header so that we know where the objects |
| 194 | mirror::Object* obj = reinterpret_cast<mirror::Object*>(pos); |
Andreas Gampe | 277ccbd | 2014-11-03 21:36:10 -0800 | [diff] [blame] | 195 | const mirror::Object* end_obj = reinterpret_cast<const mirror::Object*>(pos + block_size); |
| 196 | CHECK_LE(reinterpret_cast<const uint8_t*>(end_obj), End()); |
Mathieu Chartier | 692fafd | 2013-11-29 17:24:40 -0800 | [diff] [blame] | 197 | // We don't know how many objects are allocated in the current block. When we hit a null class |
| 198 | // assume its the end. TODO: Have a thread update the header when it flushes the block? |
Hiroshi Yamauchi | 2cd334a | 2015-01-09 14:03:35 -0800 | [diff] [blame] | 199 | // No read barrier because obj may not be a valid object. |
| 200 | while (obj < end_obj && obj->GetClass<kDefaultVerifyFlags, kWithoutReadBarrier>() != nullptr) { |
Mathieu Chartier | 692fafd | 2013-11-29 17:24:40 -0800 | [diff] [blame] | 201 | callback(obj, arg); |
| 202 | obj = GetNextObject(obj); |
| 203 | } |
| 204 | pos += block_size; |
| 205 | } |
| 206 | } |
| 207 | |
Mathieu Chartier | a8e8f9c | 2014-04-09 14:51:05 -0700 | [diff] [blame] | 208 | accounting::ContinuousSpaceBitmap::SweepCallback* BumpPointerSpace::GetSweepCallback() { |
Ian Rogers | 2c4257b | 2014-10-24 14:20:06 -0700 | [diff] [blame] | 209 | UNIMPLEMENTED(FATAL); |
| 210 | UNREACHABLE(); |
Mathieu Chartier | 692fafd | 2013-11-29 17:24:40 -0800 | [diff] [blame] | 211 | } |
| 212 | |
| 213 | uint64_t BumpPointerSpace::GetBytesAllocated() { |
| 214 | // Start out pre-determined amount (blocks which are not being allocated into). |
Ian Rogers | 3e5cf30 | 2014-05-20 16:40:37 -0700 | [diff] [blame] | 215 | uint64_t total = static_cast<uint64_t>(bytes_allocated_.LoadRelaxed()); |
Mathieu Chartier | 692fafd | 2013-11-29 17:24:40 -0800 | [diff] [blame] | 216 | Thread* self = Thread::Current(); |
| 217 | MutexLock mu(self, *Locks::runtime_shutdown_lock_); |
| 218 | MutexLock mu2(self, *Locks::thread_list_lock_); |
| 219 | std::list<Thread*> thread_list = Runtime::Current()->GetThreadList()->GetList(); |
| 220 | MutexLock mu3(Thread::Current(), block_lock_); |
| 221 | // If we don't have any blocks, we don't have any thread local buffers. This check is required |
| 222 | // since there can exist multiple bump pointer spaces which exist at the same time. |
| 223 | if (num_blocks_ > 0) { |
| 224 | for (Thread* thread : thread_list) { |
Ian Rogers | dd7624d | 2014-03-14 17:43:00 -0700 | [diff] [blame] | 225 | total += thread->GetThreadLocalBytesAllocated(); |
Mathieu Chartier | 692fafd | 2013-11-29 17:24:40 -0800 | [diff] [blame] | 226 | } |
| 227 | } |
| 228 | return total; |
| 229 | } |
| 230 | |
| 231 | uint64_t BumpPointerSpace::GetObjectsAllocated() { |
| 232 | // Start out pre-determined amount (blocks which are not being allocated into). |
Ian Rogers | 3e5cf30 | 2014-05-20 16:40:37 -0700 | [diff] [blame] | 233 | uint64_t total = static_cast<uint64_t>(objects_allocated_.LoadRelaxed()); |
Mathieu Chartier | 692fafd | 2013-11-29 17:24:40 -0800 | [diff] [blame] | 234 | Thread* self = Thread::Current(); |
| 235 | MutexLock mu(self, *Locks::runtime_shutdown_lock_); |
| 236 | MutexLock mu2(self, *Locks::thread_list_lock_); |
| 237 | std::list<Thread*> thread_list = Runtime::Current()->GetThreadList()->GetList(); |
| 238 | MutexLock mu3(Thread::Current(), block_lock_); |
| 239 | // If we don't have any blocks, we don't have any thread local buffers. This check is required |
| 240 | // since there can exist multiple bump pointer spaces which exist at the same time. |
| 241 | if (num_blocks_ > 0) { |
| 242 | for (Thread* thread : thread_list) { |
Ian Rogers | dd7624d | 2014-03-14 17:43:00 -0700 | [diff] [blame] | 243 | total += thread->GetThreadLocalObjectsAllocated(); |
Mathieu Chartier | 692fafd | 2013-11-29 17:24:40 -0800 | [diff] [blame] | 244 | } |
| 245 | } |
| 246 | return total; |
| 247 | } |
| 248 | |
| 249 | void BumpPointerSpace::RevokeThreadLocalBuffersLocked(Thread* thread) { |
Ian Rogers | 3e5cf30 | 2014-05-20 16:40:37 -0700 | [diff] [blame] | 250 | objects_allocated_.FetchAndAddSequentiallyConsistent(thread->GetThreadLocalObjectsAllocated()); |
| 251 | bytes_allocated_.FetchAndAddSequentiallyConsistent(thread->GetThreadLocalBytesAllocated()); |
Mathieu Chartier | e6da9af | 2013-12-16 11:54:42 -0800 | [diff] [blame] | 252 | thread->SetTlab(nullptr, nullptr); |
Mathieu Chartier | 692fafd | 2013-11-29 17:24:40 -0800 | [diff] [blame] | 253 | } |
| 254 | |
Mathieu Chartier | e6da9af | 2013-12-16 11:54:42 -0800 | [diff] [blame] | 255 | bool BumpPointerSpace::AllocNewTlab(Thread* self, size_t bytes) { |
Mathieu Chartier | 692fafd | 2013-11-29 17:24:40 -0800 | [diff] [blame] | 256 | MutexLock mu(Thread::Current(), block_lock_); |
| 257 | RevokeThreadLocalBuffersLocked(self); |
Ian Rogers | 1373595 | 2014-10-08 12:43:28 -0700 | [diff] [blame] | 258 | uint8_t* start = AllocBlock(bytes); |
Mathieu Chartier | 692fafd | 2013-11-29 17:24:40 -0800 | [diff] [blame] | 259 | if (start == nullptr) { |
| 260 | return false; |
| 261 | } |
Mathieu Chartier | e6da9af | 2013-12-16 11:54:42 -0800 | [diff] [blame] | 262 | self->SetTlab(start, start + bytes); |
Mathieu Chartier | 692fafd | 2013-11-29 17:24:40 -0800 | [diff] [blame] | 263 | return true; |
| 264 | } |
| 265 | |
Mathieu Chartier | b363f66 | 2014-07-16 13:28:58 -0700 | [diff] [blame] | 266 | void BumpPointerSpace::LogFragmentationAllocFailure(std::ostream& os, |
| 267 | size_t /* failed_alloc_bytes */) { |
| 268 | size_t max_contiguous_allocation = Limit() - End(); |
| 269 | os << "; failed due to fragmentation (largest possible contiguous allocation " |
| 270 | << max_contiguous_allocation << " bytes)"; |
| 271 | // Caller's job to print failed_alloc_bytes. |
| 272 | } |
| 273 | |
Mathieu Chartier | 590fee9 | 2013-09-13 13:46:47 -0700 | [diff] [blame] | 274 | } // namespace space |
| 275 | } // namespace gc |
| 276 | } // namespace art |