Hiroshi Yamauchi | d5307ec | 2014-03-27 21:07:51 -0700 | [diff] [blame] | 1 | /* |
| 2 | * Copyright (C) 2014 The Android Open Source Project |
| 3 | * |
| 4 | * Licensed under the Apache License, Version 2.0 (the "License"); |
| 5 | * you may not use this file except in compliance with the License. |
| 6 | * You may obtain a copy of the License at |
| 7 | * |
| 8 | * http://www.apache.org/licenses/LICENSE-2.0 |
| 9 | * |
| 10 | * Unless required by applicable law or agreed to in writing, software |
| 11 | * distributed under the License is distributed on an "AS IS" BASIS, |
| 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 13 | * See the License for the specific language governing permissions and |
| 14 | * limitations under the License. |
| 15 | */ |
| 16 | |
| 17 | #ifndef ART_RUNTIME_GC_COLLECTOR_CONCURRENT_COPYING_H_ |
| 18 | #define ART_RUNTIME_GC_COLLECTOR_CONCURRENT_COPYING_H_ |
| 19 | |
Hiroshi Yamauchi | 2cd334a | 2015-01-09 14:03:35 -0800 | [diff] [blame] | 20 | #include "barrier.h" |
Hiroshi Yamauchi | d5307ec | 2014-03-27 21:07:51 -0700 | [diff] [blame] | 21 | #include "garbage_collector.h" |
Mathieu Chartier | 763a31e | 2015-11-16 16:05:55 -0800 | [diff] [blame] | 22 | #include "immune_spaces.h" |
Hiroshi Yamauchi | 2cd334a | 2015-01-09 14:03:35 -0800 | [diff] [blame] | 23 | #include "jni.h" |
Hiroshi Yamauchi | 2cd334a | 2015-01-09 14:03:35 -0800 | [diff] [blame] | 24 | #include "offsets.h" |
Hiroshi Yamauchi | 2cd334a | 2015-01-09 14:03:35 -0800 | [diff] [blame] | 25 | #include "mirror/object_reference.h" |
| 26 | #include "safe_map.h" |
| 27 | |
| 28 | #include <unordered_map> |
| 29 | #include <vector> |
Hiroshi Yamauchi | d5307ec | 2014-03-27 21:07:51 -0700 | [diff] [blame] | 30 | |
| 31 | namespace art { |
Hiroshi Yamauchi | febd0cf | 2016-09-14 19:31:25 -0700 | [diff] [blame] | 32 | class Closure; |
Hiroshi Yamauchi | 2cd334a | 2015-01-09 14:03:35 -0800 | [diff] [blame] | 33 | class RootInfo; |
| 34 | |
Andreas Gampe | 5d08fcc | 2017-06-05 17:56:46 -0700 | [diff] [blame] | 35 | namespace mirror { |
| 36 | class Object; |
| 37 | } // namespace mirror |
| 38 | |
Hiroshi Yamauchi | d5307ec | 2014-03-27 21:07:51 -0700 | [diff] [blame] | 39 | namespace gc { |
Hiroshi Yamauchi | 2cd334a | 2015-01-09 14:03:35 -0800 | [diff] [blame] | 40 | |
| 41 | namespace accounting { |
Andreas Gampe | 291ce17 | 2017-04-24 13:22:18 -0700 | [diff] [blame] | 42 | template<typename T> class AtomicStack; |
| 43 | typedef AtomicStack<mirror::Object> ObjectStack; |
Andreas Gampe | d490129 | 2017-05-30 18:41:34 -0700 | [diff] [blame] | 44 | template <size_t kAlignment> class SpaceBitmap; |
Hiroshi Yamauchi | 2cd334a | 2015-01-09 14:03:35 -0800 | [diff] [blame] | 45 | typedef SpaceBitmap<kObjectAlignment> ContinuousSpaceBitmap; |
| 46 | class HeapBitmap; |
Andreas Gampe | 291ce17 | 2017-04-24 13:22:18 -0700 | [diff] [blame] | 47 | class ReadBarrierTable; |
Hiroshi Yamauchi | 2cd334a | 2015-01-09 14:03:35 -0800 | [diff] [blame] | 48 | } // namespace accounting |
| 49 | |
| 50 | namespace space { |
| 51 | class RegionSpace; |
| 52 | } // namespace space |
| 53 | |
Hiroshi Yamauchi | d5307ec | 2014-03-27 21:07:51 -0700 | [diff] [blame] | 54 | namespace collector { |
| 55 | |
| 56 | class ConcurrentCopying : public GarbageCollector { |
| 57 | public: |
Hiroshi Yamauchi | 2cd334a | 2015-01-09 14:03:35 -0800 | [diff] [blame] | 58 | // Enable the no-from-space-refs verification at the pause. |
Hiroshi Yamauchi | daf61a1 | 2016-06-10 14:27:38 -0700 | [diff] [blame] | 59 | static constexpr bool kEnableNoFromSpaceRefsVerification = kIsDebugBuild; |
Hiroshi Yamauchi | 2cd334a | 2015-01-09 14:03:35 -0800 | [diff] [blame] | 60 | // Enable the from-space bytes/objects check. |
Hiroshi Yamauchi | daf61a1 | 2016-06-10 14:27:38 -0700 | [diff] [blame] | 61 | static constexpr bool kEnableFromSpaceAccountingCheck = kIsDebugBuild; |
Hiroshi Yamauchi | 2cd334a | 2015-01-09 14:03:35 -0800 | [diff] [blame] | 62 | // Enable verbose mode. |
Hiroshi Yamauchi | 3c44893 | 2016-01-22 16:26:50 -0800 | [diff] [blame] | 63 | static constexpr bool kVerboseMode = false; |
Mathieu Chartier | 36a270a | 2016-07-28 18:08:51 -0700 | [diff] [blame] | 64 | // If kGrayDirtyImmuneObjects is true then we gray dirty objects in the GC pause to prevent dirty |
| 65 | // pages. |
| 66 | static constexpr bool kGrayDirtyImmuneObjects = true; |
Hiroshi Yamauchi | d5307ec | 2014-03-27 21:07:51 -0700 | [diff] [blame] | 67 | |
Chih-Hung Hsieh | a593118 | 2016-09-01 15:08:13 -0700 | [diff] [blame] | 68 | explicit ConcurrentCopying(Heap* heap, |
| 69 | const std::string& name_prefix = "", |
| 70 | bool measure_read_barrier_slow_path = false); |
Hiroshi Yamauchi | 2cd334a | 2015-01-09 14:03:35 -0800 | [diff] [blame] | 71 | ~ConcurrentCopying(); |
Hiroshi Yamauchi | d5307ec | 2014-03-27 21:07:51 -0700 | [diff] [blame] | 72 | |
Hiroshi Yamauchi | d8db5a2 | 2016-06-28 14:07:41 -0700 | [diff] [blame] | 73 | virtual void RunPhases() OVERRIDE |
Mathieu Chartier | 56fe258 | 2016-07-14 13:30:03 -0700 | [diff] [blame] | 74 | REQUIRES(!immune_gray_stack_lock_, |
| 75 | !mark_stack_lock_, |
| 76 | !rb_slow_path_histogram_lock_, |
| 77 | !skipped_blocks_lock_); |
Andreas Gampe | bdf7f1c | 2016-08-30 16:38:47 -0700 | [diff] [blame] | 78 | void InitializePhase() REQUIRES_SHARED(Locks::mutator_lock_) |
Hiroshi Yamauchi | d8db5a2 | 2016-06-28 14:07:41 -0700 | [diff] [blame] | 79 | REQUIRES(!mark_stack_lock_, !immune_gray_stack_lock_); |
Andreas Gampe | bdf7f1c | 2016-08-30 16:38:47 -0700 | [diff] [blame] | 80 | void MarkingPhase() REQUIRES_SHARED(Locks::mutator_lock_) |
Hiroshi Yamauchi | d8db5a2 | 2016-06-28 14:07:41 -0700 | [diff] [blame] | 81 | REQUIRES(!mark_stack_lock_, !skipped_blocks_lock_, !immune_gray_stack_lock_); |
Andreas Gampe | bdf7f1c | 2016-08-30 16:38:47 -0700 | [diff] [blame] | 82 | void ReclaimPhase() REQUIRES_SHARED(Locks::mutator_lock_) REQUIRES(!mark_stack_lock_); |
Mathieu Chartier | 56fe258 | 2016-07-14 13:30:03 -0700 | [diff] [blame] | 83 | void FinishPhase() REQUIRES(!mark_stack_lock_, |
| 84 | !rb_slow_path_histogram_lock_, |
| 85 | !skipped_blocks_lock_); |
Hiroshi Yamauchi | 2cd334a | 2015-01-09 14:03:35 -0800 | [diff] [blame] | 86 | |
Andreas Gampe | bdf7f1c | 2016-08-30 16:38:47 -0700 | [diff] [blame] | 87 | void BindBitmaps() REQUIRES_SHARED(Locks::mutator_lock_) |
Mathieu Chartier | 9044347 | 2015-07-16 20:32:27 -0700 | [diff] [blame] | 88 | REQUIRES(!Locks::heap_bitmap_lock_); |
Hiroshi Yamauchi | d5307ec | 2014-03-27 21:07:51 -0700 | [diff] [blame] | 89 | virtual GcType GetGcType() const OVERRIDE { |
| 90 | return kGcTypePartial; |
| 91 | } |
| 92 | virtual CollectorType GetCollectorType() const OVERRIDE { |
| 93 | return kCollectorTypeCC; |
| 94 | } |
Hiroshi Yamauchi | 2cd334a | 2015-01-09 14:03:35 -0800 | [diff] [blame] | 95 | virtual void RevokeAllThreadLocalBuffers() OVERRIDE; |
| 96 | void SetRegionSpace(space::RegionSpace* region_space) { |
| 97 | DCHECK(region_space != nullptr); |
| 98 | region_space_ = region_space; |
| 99 | } |
| 100 | space::RegionSpace* RegionSpace() { |
| 101 | return region_space_; |
| 102 | } |
| 103 | void AssertToSpaceInvariant(mirror::Object* obj, MemberOffset offset, mirror::Object* ref) |
Andreas Gampe | bdf7f1c | 2016-08-30 16:38:47 -0700 | [diff] [blame] | 104 | REQUIRES_SHARED(Locks::mutator_lock_); |
Hiroshi Yamauchi | 3f64f25 | 2015-06-12 18:35:06 -0700 | [diff] [blame] | 105 | void AssertToSpaceInvariant(GcRootSource* gc_root_source, mirror::Object* ref) |
Andreas Gampe | bdf7f1c | 2016-08-30 16:38:47 -0700 | [diff] [blame] | 106 | REQUIRES_SHARED(Locks::mutator_lock_); |
| 107 | bool IsInToSpace(mirror::Object* ref) REQUIRES_SHARED(Locks::mutator_lock_) { |
Hiroshi Yamauchi | 2cd334a | 2015-01-09 14:03:35 -0800 | [diff] [blame] | 108 | DCHECK(ref != nullptr); |
| 109 | return IsMarked(ref) == ref; |
| 110 | } |
Mathieu Chartier | c381c36 | 2016-08-23 13:27:53 -0700 | [diff] [blame] | 111 | template<bool kGrayImmuneObject = true, bool kFromGCThread = false> |
Mathieu Chartier | 1ca6890 | 2017-04-18 11:26:22 -0700 | [diff] [blame] | 112 | ALWAYS_INLINE mirror::Object* Mark(mirror::Object* from_ref, |
| 113 | mirror::Object* holder = nullptr, |
| 114 | MemberOffset offset = MemberOffset(0)) |
Andreas Gampe | bdf7f1c | 2016-08-30 16:38:47 -0700 | [diff] [blame] | 115 | REQUIRES_SHARED(Locks::mutator_lock_) |
Mathieu Chartier | 56fe258 | 2016-07-14 13:30:03 -0700 | [diff] [blame] | 116 | REQUIRES(!mark_stack_lock_, !skipped_blocks_lock_, !immune_gray_stack_lock_); |
| 117 | ALWAYS_INLINE mirror::Object* MarkFromReadBarrier(mirror::Object* from_ref) |
Andreas Gampe | bdf7f1c | 2016-08-30 16:38:47 -0700 | [diff] [blame] | 118 | REQUIRES_SHARED(Locks::mutator_lock_) |
Hiroshi Yamauchi | d8db5a2 | 2016-06-28 14:07:41 -0700 | [diff] [blame] | 119 | REQUIRES(!mark_stack_lock_, !skipped_blocks_lock_, !immune_gray_stack_lock_); |
Hiroshi Yamauchi | 2cd334a | 2015-01-09 14:03:35 -0800 | [diff] [blame] | 120 | bool IsMarking() const { |
| 121 | return is_marking_; |
| 122 | } |
Mathieu Chartier | 3768ade | 2017-05-02 14:04:39 -0700 | [diff] [blame] | 123 | // We may want to use read barrier entrypoints before is_marking_ is true since concurrent graying |
| 124 | // creates a small window where we might dispatch on these entrypoints. |
| 125 | bool IsUsingReadBarrierEntrypoints() const { |
| 126 | return is_using_read_barrier_entrypoints_; |
| 127 | } |
Hiroshi Yamauchi | 2cd334a | 2015-01-09 14:03:35 -0800 | [diff] [blame] | 128 | bool IsActive() const { |
| 129 | return is_active_; |
| 130 | } |
| 131 | Barrier& GetBarrier() { |
| 132 | return *gc_barrier_; |
| 133 | } |
Hiroshi Yamauchi | febd0cf | 2016-09-14 19:31:25 -0700 | [diff] [blame] | 134 | bool IsWeakRefAccessEnabled() REQUIRES(Locks::thread_list_lock_) { |
| 135 | return weak_ref_access_enabled_; |
Hiroshi Yamauchi | 0b71357 | 2015-06-16 18:29:23 -0700 | [diff] [blame] | 136 | } |
Andreas Gampe | bdf7f1c | 2016-08-30 16:38:47 -0700 | [diff] [blame] | 137 | void RevokeThreadLocalMarkStack(Thread* thread) REQUIRES_SHARED(Locks::mutator_lock_) |
Mathieu Chartier | 9044347 | 2015-07-16 20:32:27 -0700 | [diff] [blame] | 138 | REQUIRES(!mark_stack_lock_); |
Hiroshi Yamauchi | d5307ec | 2014-03-27 21:07:51 -0700 | [diff] [blame] | 139 | |
Nicolas Geoffray | 13056a1 | 2017-05-11 11:48:28 +0000 | [diff] [blame] | 140 | virtual mirror::Object* IsMarked(mirror::Object* from_ref) OVERRIDE |
| 141 | REQUIRES_SHARED(Locks::mutator_lock_); |
| 142 | |
Hiroshi Yamauchi | d5307ec | 2014-03-27 21:07:51 -0700 | [diff] [blame] | 143 | private: |
Andreas Gampe | bdf7f1c | 2016-08-30 16:38:47 -0700 | [diff] [blame] | 144 | void PushOntoMarkStack(mirror::Object* obj) REQUIRES_SHARED(Locks::mutator_lock_) |
Mathieu Chartier | 9044347 | 2015-07-16 20:32:27 -0700 | [diff] [blame] | 145 | REQUIRES(!mark_stack_lock_); |
Mathieu Chartier | ef496d9 | 2017-04-28 18:58:59 -0700 | [diff] [blame] | 146 | mirror::Object* Copy(mirror::Object* from_ref, |
| 147 | mirror::Object* holder, |
| 148 | MemberOffset offset) |
| 149 | REQUIRES_SHARED(Locks::mutator_lock_) |
Mathieu Chartier | d6636d3 | 2016-07-28 11:02:38 -0700 | [diff] [blame] | 150 | REQUIRES(!mark_stack_lock_, !skipped_blocks_lock_, !immune_gray_stack_lock_); |
Andreas Gampe | bdf7f1c | 2016-08-30 16:38:47 -0700 | [diff] [blame] | 151 | void Scan(mirror::Object* to_ref) REQUIRES_SHARED(Locks::mutator_lock_) |
Mathieu Chartier | 9044347 | 2015-07-16 20:32:27 -0700 | [diff] [blame] | 152 | REQUIRES(!mark_stack_lock_); |
Hiroshi Yamauchi | 2cd334a | 2015-01-09 14:03:35 -0800 | [diff] [blame] | 153 | void Process(mirror::Object* obj, MemberOffset offset) |
Andreas Gampe | bdf7f1c | 2016-08-30 16:38:47 -0700 | [diff] [blame] | 154 | REQUIRES_SHARED(Locks::mutator_lock_) |
Hiroshi Yamauchi | d8db5a2 | 2016-06-28 14:07:41 -0700 | [diff] [blame] | 155 | REQUIRES(!mark_stack_lock_ , !skipped_blocks_lock_, !immune_gray_stack_lock_); |
Mathieu Chartier | bb87e0f | 2015-04-03 11:21:55 -0700 | [diff] [blame] | 156 | virtual void VisitRoots(mirror::Object*** roots, size_t count, const RootInfo& info) |
Andreas Gampe | bdf7f1c | 2016-08-30 16:38:47 -0700 | [diff] [blame] | 157 | OVERRIDE REQUIRES_SHARED(Locks::mutator_lock_) |
Hiroshi Yamauchi | d8db5a2 | 2016-06-28 14:07:41 -0700 | [diff] [blame] | 158 | REQUIRES(!mark_stack_lock_, !skipped_blocks_lock_, !immune_gray_stack_lock_); |
| 159 | template<bool kGrayImmuneObject> |
Mathieu Chartier | da7c650 | 2015-07-23 16:01:26 -0700 | [diff] [blame] | 160 | void MarkRoot(mirror::CompressedReference<mirror::Object>* root) |
Andreas Gampe | bdf7f1c | 2016-08-30 16:38:47 -0700 | [diff] [blame] | 161 | REQUIRES_SHARED(Locks::mutator_lock_) |
Hiroshi Yamauchi | d8db5a2 | 2016-06-28 14:07:41 -0700 | [diff] [blame] | 162 | REQUIRES(!mark_stack_lock_, !skipped_blocks_lock_, !immune_gray_stack_lock_); |
Mathieu Chartier | bb87e0f | 2015-04-03 11:21:55 -0700 | [diff] [blame] | 163 | virtual void VisitRoots(mirror::CompressedReference<mirror::Object>** roots, size_t count, |
| 164 | const RootInfo& info) |
Andreas Gampe | bdf7f1c | 2016-08-30 16:38:47 -0700 | [diff] [blame] | 165 | OVERRIDE REQUIRES_SHARED(Locks::mutator_lock_) |
Hiroshi Yamauchi | d8db5a2 | 2016-06-28 14:07:41 -0700 | [diff] [blame] | 166 | REQUIRES(!mark_stack_lock_, !skipped_blocks_lock_, !immune_gray_stack_lock_); |
Mathieu Chartier | 9044347 | 2015-07-16 20:32:27 -0700 | [diff] [blame] | 167 | void VerifyNoFromSpaceReferences() REQUIRES(Locks::mutator_lock_); |
Hiroshi Yamauchi | 2cd334a | 2015-01-09 14:03:35 -0800 | [diff] [blame] | 168 | accounting::ObjectStack* GetAllocationStack(); |
| 169 | accounting::ObjectStack* GetLiveStack(); |
Andreas Gampe | bdf7f1c | 2016-08-30 16:38:47 -0700 | [diff] [blame] | 170 | virtual void ProcessMarkStack() OVERRIDE REQUIRES_SHARED(Locks::mutator_lock_) |
Mathieu Chartier | 9044347 | 2015-07-16 20:32:27 -0700 | [diff] [blame] | 171 | REQUIRES(!mark_stack_lock_); |
Andreas Gampe | bdf7f1c | 2016-08-30 16:38:47 -0700 | [diff] [blame] | 172 | bool ProcessMarkStackOnce() REQUIRES_SHARED(Locks::mutator_lock_) REQUIRES(!mark_stack_lock_); |
| 173 | void ProcessMarkStackRef(mirror::Object* to_ref) REQUIRES_SHARED(Locks::mutator_lock_) |
Mathieu Chartier | 9044347 | 2015-07-16 20:32:27 -0700 | [diff] [blame] | 174 | REQUIRES(!mark_stack_lock_); |
Mathieu Chartier | 21328a1 | 2016-07-22 10:47:45 -0700 | [diff] [blame] | 175 | void GrayAllDirtyImmuneObjects() |
| 176 | REQUIRES(Locks::mutator_lock_) |
| 177 | REQUIRES(!mark_stack_lock_); |
Mathieu Chartier | 3768ade | 2017-05-02 14:04:39 -0700 | [diff] [blame] | 178 | void GrayAllNewlyDirtyImmuneObjects() |
| 179 | REQUIRES(Locks::mutator_lock_) |
| 180 | REQUIRES(!mark_stack_lock_); |
Mathieu Chartier | 21328a1 | 2016-07-22 10:47:45 -0700 | [diff] [blame] | 181 | void VerifyGrayImmuneObjects() |
| 182 | REQUIRES(Locks::mutator_lock_) |
| 183 | REQUIRES(!mark_stack_lock_); |
Mathieu Chartier | a1467d0 | 2017-02-22 09:22:50 -0800 | [diff] [blame] | 184 | void VerifyNoMissingCardMarks() |
| 185 | REQUIRES(Locks::mutator_lock_) |
| 186 | REQUIRES(!mark_stack_lock_); |
Hiroshi Yamauchi | febd0cf | 2016-09-14 19:31:25 -0700 | [diff] [blame] | 187 | size_t ProcessThreadLocalMarkStacks(bool disable_weak_ref_access, Closure* checkpoint_callback) |
Andreas Gampe | bdf7f1c | 2016-08-30 16:38:47 -0700 | [diff] [blame] | 188 | REQUIRES_SHARED(Locks::mutator_lock_) REQUIRES(!mark_stack_lock_); |
Hiroshi Yamauchi | febd0cf | 2016-09-14 19:31:25 -0700 | [diff] [blame] | 189 | void RevokeThreadLocalMarkStacks(bool disable_weak_ref_access, Closure* checkpoint_callback) |
Andreas Gampe | bdf7f1c | 2016-08-30 16:38:47 -0700 | [diff] [blame] | 190 | REQUIRES_SHARED(Locks::mutator_lock_); |
| 191 | void SwitchToSharedMarkStackMode() REQUIRES_SHARED(Locks::mutator_lock_) |
Mathieu Chartier | 9044347 | 2015-07-16 20:32:27 -0700 | [diff] [blame] | 192 | REQUIRES(!mark_stack_lock_); |
Andreas Gampe | bdf7f1c | 2016-08-30 16:38:47 -0700 | [diff] [blame] | 193 | void SwitchToGcExclusiveMarkStackMode() REQUIRES_SHARED(Locks::mutator_lock_); |
Mathieu Chartier | 31e8822 | 2016-10-14 18:43:19 -0700 | [diff] [blame] | 194 | virtual void DelayReferenceReferent(ObjPtr<mirror::Class> klass, |
| 195 | ObjPtr<mirror::Reference> reference) OVERRIDE |
Andreas Gampe | bdf7f1c | 2016-08-30 16:38:47 -0700 | [diff] [blame] | 196 | REQUIRES_SHARED(Locks::mutator_lock_); |
| 197 | void ProcessReferences(Thread* self) REQUIRES_SHARED(Locks::mutator_lock_); |
Mathieu Chartier | 9750995 | 2015-07-13 14:35:43 -0700 | [diff] [blame] | 198 | virtual mirror::Object* MarkObject(mirror::Object* from_ref) OVERRIDE |
Andreas Gampe | bdf7f1c | 2016-08-30 16:38:47 -0700 | [diff] [blame] | 199 | REQUIRES_SHARED(Locks::mutator_lock_) |
Hiroshi Yamauchi | d8db5a2 | 2016-06-28 14:07:41 -0700 | [diff] [blame] | 200 | REQUIRES(!mark_stack_lock_, !skipped_blocks_lock_, !immune_gray_stack_lock_); |
Hiroshi Yamauchi | 057d977 | 2017-02-17 15:33:23 -0800 | [diff] [blame] | 201 | virtual void MarkHeapReference(mirror::HeapReference<mirror::Object>* from_ref, |
| 202 | bool do_atomic_update) OVERRIDE |
Andreas Gampe | bdf7f1c | 2016-08-30 16:38:47 -0700 | [diff] [blame] | 203 | REQUIRES_SHARED(Locks::mutator_lock_) |
Hiroshi Yamauchi | d8db5a2 | 2016-06-28 14:07:41 -0700 | [diff] [blame] | 204 | REQUIRES(!mark_stack_lock_, !skipped_blocks_lock_, !immune_gray_stack_lock_); |
Mathieu Chartier | c381c36 | 2016-08-23 13:27:53 -0700 | [diff] [blame] | 205 | bool IsMarkedInUnevacFromSpace(mirror::Object* from_ref) |
Andreas Gampe | bdf7f1c | 2016-08-30 16:38:47 -0700 | [diff] [blame] | 206 | REQUIRES_SHARED(Locks::mutator_lock_); |
Hiroshi Yamauchi | 65f5f24 | 2016-12-19 11:44:47 -0800 | [diff] [blame] | 207 | virtual bool IsNullOrMarkedHeapReference(mirror::HeapReference<mirror::Object>* field, |
| 208 | bool do_atomic_update) OVERRIDE |
Andreas Gampe | bdf7f1c | 2016-08-30 16:38:47 -0700 | [diff] [blame] | 209 | REQUIRES_SHARED(Locks::mutator_lock_); |
Hiroshi Yamauchi | 2cd334a | 2015-01-09 14:03:35 -0800 | [diff] [blame] | 210 | void SweepSystemWeaks(Thread* self) |
Andreas Gampe | bdf7f1c | 2016-08-30 16:38:47 -0700 | [diff] [blame] | 211 | REQUIRES_SHARED(Locks::mutator_lock_) REQUIRES(!Locks::heap_bitmap_lock_); |
Hiroshi Yamauchi | 2cd334a | 2015-01-09 14:03:35 -0800 | [diff] [blame] | 212 | void Sweep(bool swap_bitmaps) |
Andreas Gampe | bdf7f1c | 2016-08-30 16:38:47 -0700 | [diff] [blame] | 213 | REQUIRES_SHARED(Locks::mutator_lock_) REQUIRES(Locks::heap_bitmap_lock_, !mark_stack_lock_); |
Hiroshi Yamauchi | 2cd334a | 2015-01-09 14:03:35 -0800 | [diff] [blame] | 214 | void SweepLargeObjects(bool swap_bitmaps) |
Andreas Gampe | bdf7f1c | 2016-08-30 16:38:47 -0700 | [diff] [blame] | 215 | REQUIRES_SHARED(Locks::mutator_lock_) REQUIRES(Locks::heap_bitmap_lock_); |
Mathieu Chartier | 962cd7a | 2016-08-16 12:15:59 -0700 | [diff] [blame] | 216 | void MarkZygoteLargeObjects() |
Andreas Gampe | bdf7f1c | 2016-08-30 16:38:47 -0700 | [diff] [blame] | 217 | REQUIRES_SHARED(Locks::mutator_lock_); |
Hiroshi Yamauchi | 2cd334a | 2015-01-09 14:03:35 -0800 | [diff] [blame] | 218 | void FillWithDummyObject(mirror::Object* dummy_obj, size_t byte_size) |
Mathieu Chartier | d6636d3 | 2016-07-28 11:02:38 -0700 | [diff] [blame] | 219 | REQUIRES(!mark_stack_lock_, !skipped_blocks_lock_, !immune_gray_stack_lock_) |
Andreas Gampe | bdf7f1c | 2016-08-30 16:38:47 -0700 | [diff] [blame] | 220 | REQUIRES_SHARED(Locks::mutator_lock_); |
Hiroshi Yamauchi | 2cd334a | 2015-01-09 14:03:35 -0800 | [diff] [blame] | 221 | mirror::Object* AllocateInSkippedBlock(size_t alloc_size) |
Mathieu Chartier | d6636d3 | 2016-07-28 11:02:38 -0700 | [diff] [blame] | 222 | REQUIRES(!mark_stack_lock_, !skipped_blocks_lock_, !immune_gray_stack_lock_) |
Andreas Gampe | bdf7f1c | 2016-08-30 16:38:47 -0700 | [diff] [blame] | 223 | REQUIRES_SHARED(Locks::mutator_lock_); |
| 224 | void CheckEmptyMarkStack() REQUIRES_SHARED(Locks::mutator_lock_) REQUIRES(!mark_stack_lock_); |
| 225 | void IssueEmptyCheckpoint() REQUIRES_SHARED(Locks::mutator_lock_); |
| 226 | bool IsOnAllocStack(mirror::Object* ref) REQUIRES_SHARED(Locks::mutator_lock_); |
Hiroshi Yamauchi | 2cd334a | 2015-01-09 14:03:35 -0800 | [diff] [blame] | 227 | mirror::Object* GetFwdPtr(mirror::Object* from_ref) |
Andreas Gampe | bdf7f1c | 2016-08-30 16:38:47 -0700 | [diff] [blame] | 228 | REQUIRES_SHARED(Locks::mutator_lock_); |
Mathieu Chartier | 9044347 | 2015-07-16 20:32:27 -0700 | [diff] [blame] | 229 | void FlipThreadRoots() REQUIRES(!Locks::mutator_lock_); |
Andreas Gampe | bdf7f1c | 2016-08-30 16:38:47 -0700 | [diff] [blame] | 230 | void SwapStacks() REQUIRES_SHARED(Locks::mutator_lock_); |
Hiroshi Yamauchi | 2cd334a | 2015-01-09 14:03:35 -0800 | [diff] [blame] | 231 | void RecordLiveStackFreezeSize(Thread* self); |
| 232 | void ComputeUnevacFromSpaceLiveRatio(); |
Hiroshi Yamauchi | 3f64f25 | 2015-06-12 18:35:06 -0700 | [diff] [blame] | 233 | void LogFromSpaceRefHolder(mirror::Object* obj, MemberOffset offset) |
Andreas Gampe | bdf7f1c | 2016-08-30 16:38:47 -0700 | [diff] [blame] | 234 | REQUIRES_SHARED(Locks::mutator_lock_); |
Hiroshi Yamauchi | 3f64f25 | 2015-06-12 18:35:06 -0700 | [diff] [blame] | 235 | void AssertToSpaceInvariantInNonMovingSpace(mirror::Object* obj, mirror::Object* ref) |
Andreas Gampe | bdf7f1c | 2016-08-30 16:38:47 -0700 | [diff] [blame] | 236 | REQUIRES_SHARED(Locks::mutator_lock_); |
| 237 | void ReenableWeakRefAccess(Thread* self) REQUIRES_SHARED(Locks::mutator_lock_); |
| 238 | void DisableMarking() REQUIRES_SHARED(Locks::mutator_lock_); |
| 239 | void IssueDisableMarkingCheckpoint() REQUIRES_SHARED(Locks::mutator_lock_); |
| 240 | void ExpandGcMarkStack() REQUIRES_SHARED(Locks::mutator_lock_); |
Mathieu Chartier | 1ca6890 | 2017-04-18 11:26:22 -0700 | [diff] [blame] | 241 | mirror::Object* MarkNonMoving(mirror::Object* from_ref, |
| 242 | mirror::Object* holder = nullptr, |
| 243 | MemberOffset offset = MemberOffset(0)) |
| 244 | REQUIRES_SHARED(Locks::mutator_lock_) |
Hiroshi Yamauchi | 723e6ce | 2015-10-28 20:59:47 -0700 | [diff] [blame] | 245 | REQUIRES(!mark_stack_lock_, !skipped_blocks_lock_); |
Hiroshi Yamauchi | d8db5a2 | 2016-06-28 14:07:41 -0700 | [diff] [blame] | 246 | ALWAYS_INLINE mirror::Object* MarkUnevacFromSpaceRegion(mirror::Object* from_ref, |
Hiroshi Yamauchi | 8e67465 | 2015-12-22 11:09:18 -0800 | [diff] [blame] | 247 | accounting::SpaceBitmap<kObjectAlignment>* bitmap) |
Andreas Gampe | bdf7f1c | 2016-08-30 16:38:47 -0700 | [diff] [blame] | 248 | REQUIRES_SHARED(Locks::mutator_lock_) |
Hiroshi Yamauchi | 8e67465 | 2015-12-22 11:09:18 -0800 | [diff] [blame] | 249 | REQUIRES(!mark_stack_lock_, !skipped_blocks_lock_); |
Hiroshi Yamauchi | d8db5a2 | 2016-06-28 14:07:41 -0700 | [diff] [blame] | 250 | template<bool kGrayImmuneObject> |
| 251 | ALWAYS_INLINE mirror::Object* MarkImmuneSpace(mirror::Object* from_ref) |
Andreas Gampe | bdf7f1c | 2016-08-30 16:38:47 -0700 | [diff] [blame] | 252 | REQUIRES_SHARED(Locks::mutator_lock_) REQUIRES(!immune_gray_stack_lock_); |
| 253 | void PushOntoFalseGrayStack(mirror::Object* obj) REQUIRES_SHARED(Locks::mutator_lock_) |
Hiroshi Yamauchi | 8e67465 | 2015-12-22 11:09:18 -0800 | [diff] [blame] | 254 | REQUIRES(!mark_stack_lock_); |
Andreas Gampe | bdf7f1c | 2016-08-30 16:38:47 -0700 | [diff] [blame] | 255 | void ProcessFalseGrayStack() REQUIRES_SHARED(Locks::mutator_lock_) |
Hiroshi Yamauchi | 8e67465 | 2015-12-22 11:09:18 -0800 | [diff] [blame] | 256 | REQUIRES(!mark_stack_lock_); |
Hiroshi Yamauchi | d8db5a2 | 2016-06-28 14:07:41 -0700 | [diff] [blame] | 257 | void ScanImmuneObject(mirror::Object* obj) |
Andreas Gampe | bdf7f1c | 2016-08-30 16:38:47 -0700 | [diff] [blame] | 258 | REQUIRES_SHARED(Locks::mutator_lock_) REQUIRES(!mark_stack_lock_); |
Mathieu Chartier | 56fe258 | 2016-07-14 13:30:03 -0700 | [diff] [blame] | 259 | mirror::Object* MarkFromReadBarrierWithMeasurements(mirror::Object* from_ref) |
Andreas Gampe | bdf7f1c | 2016-08-30 16:38:47 -0700 | [diff] [blame] | 260 | REQUIRES_SHARED(Locks::mutator_lock_) |
Mathieu Chartier | 56fe258 | 2016-07-14 13:30:03 -0700 | [diff] [blame] | 261 | REQUIRES(!mark_stack_lock_, !skipped_blocks_lock_, !immune_gray_stack_lock_); |
| 262 | void DumpPerformanceInfo(std::ostream& os) OVERRIDE REQUIRES(!rb_slow_path_histogram_lock_); |
Mathieu Chartier | 3768ade | 2017-05-02 14:04:39 -0700 | [diff] [blame] | 263 | // Set the read barrier mark entrypoints to non-null. |
| 264 | void ActivateReadBarrierEntrypoints(); |
Hiroshi Yamauchi | 2cd334a | 2015-01-09 14:03:35 -0800 | [diff] [blame] | 265 | |
| 266 | space::RegionSpace* region_space_; // The underlying region space. |
| 267 | std::unique_ptr<Barrier> gc_barrier_; |
Hiroshi Yamauchi | 0b71357 | 2015-06-16 18:29:23 -0700 | [diff] [blame] | 268 | std::unique_ptr<accounting::ObjectStack> gc_mark_stack_; |
Mathieu Chartier | 36a270a | 2016-07-28 18:08:51 -0700 | [diff] [blame] | 269 | std::unique_ptr<accounting::ObjectStack> rb_mark_bit_stack_; |
| 270 | bool rb_mark_bit_stack_full_; |
Hiroshi Yamauchi | 8e67465 | 2015-12-22 11:09:18 -0800 | [diff] [blame] | 271 | std::vector<mirror::Object*> false_gray_stack_ GUARDED_BY(mark_stack_lock_); |
Hiroshi Yamauchi | 0b71357 | 2015-06-16 18:29:23 -0700 | [diff] [blame] | 272 | Mutex mark_stack_lock_ DEFAULT_MUTEX_ACQUIRED_AFTER; |
| 273 | std::vector<accounting::ObjectStack*> revoked_mark_stacks_ |
| 274 | GUARDED_BY(mark_stack_lock_); |
| 275 | static constexpr size_t kMarkStackSize = kPageSize; |
| 276 | static constexpr size_t kMarkStackPoolSize = 256; |
| 277 | std::vector<accounting::ObjectStack*> pooled_mark_stacks_ |
| 278 | GUARDED_BY(mark_stack_lock_); |
| 279 | Thread* thread_running_gc_; |
Hiroshi Yamauchi | 2cd334a | 2015-01-09 14:03:35 -0800 | [diff] [blame] | 280 | bool is_marking_; // True while marking is ongoing. |
Mathieu Chartier | 3768ade | 2017-05-02 14:04:39 -0700 | [diff] [blame] | 281 | // True while we might dispatch on the read barrier entrypoints. |
| 282 | bool is_using_read_barrier_entrypoints_; |
Hiroshi Yamauchi | 2cd334a | 2015-01-09 14:03:35 -0800 | [diff] [blame] | 283 | bool is_active_; // True while the collection is ongoing. |
| 284 | bool is_asserting_to_space_invariant_; // True while asserting the to-space invariant. |
Mathieu Chartier | 763a31e | 2015-11-16 16:05:55 -0800 | [diff] [blame] | 285 | ImmuneSpaces immune_spaces_; |
Andreas Gampe | d490129 | 2017-05-30 18:41:34 -0700 | [diff] [blame] | 286 | accounting::ContinuousSpaceBitmap* region_space_bitmap_; |
Hiroshi Yamauchi | 2cd334a | 2015-01-09 14:03:35 -0800 | [diff] [blame] | 287 | // A cache of Heap::GetMarkBitmap(). |
| 288 | accounting::HeapBitmap* heap_mark_bitmap_; |
| 289 | size_t live_stack_freeze_size_; |
| 290 | size_t from_space_num_objects_at_first_pause_; |
| 291 | size_t from_space_num_bytes_at_first_pause_; |
Hiroshi Yamauchi | 0b71357 | 2015-06-16 18:29:23 -0700 | [diff] [blame] | 292 | Atomic<int> is_mark_stack_push_disallowed_; |
| 293 | enum MarkStackMode { |
| 294 | kMarkStackModeOff = 0, // Mark stack is off. |
| 295 | kMarkStackModeThreadLocal, // All threads except for the GC-running thread push refs onto |
| 296 | // thread-local mark stacks. The GC-running thread pushes onto and |
| 297 | // pops off the GC mark stack without a lock. |
| 298 | kMarkStackModeShared, // All threads share the GC mark stack with a lock. |
| 299 | kMarkStackModeGcExclusive // The GC-running thread pushes onto and pops from the GC mark stack |
| 300 | // without a lock. Other threads won't access the mark stack. |
| 301 | }; |
| 302 | Atomic<MarkStackMode> mark_stack_mode_; |
Hiroshi Yamauchi | febd0cf | 2016-09-14 19:31:25 -0700 | [diff] [blame] | 303 | bool weak_ref_access_enabled_ GUARDED_BY(Locks::thread_list_lock_); |
Hiroshi Yamauchi | 2cd334a | 2015-01-09 14:03:35 -0800 | [diff] [blame] | 304 | |
| 305 | // How many objects and bytes we moved. Used for accounting. |
| 306 | Atomic<size_t> bytes_moved_; |
| 307 | Atomic<size_t> objects_moved_; |
Mathieu Chartier | cca44a0 | 2016-08-17 10:07:29 -0700 | [diff] [blame] | 308 | Atomic<uint64_t> cumulative_bytes_moved_; |
| 309 | Atomic<uint64_t> cumulative_objects_moved_; |
Hiroshi Yamauchi | 2cd334a | 2015-01-09 14:03:35 -0800 | [diff] [blame] | 310 | |
| 311 | // The skipped blocks are memory blocks/chucks that were copies of |
| 312 | // objects that were unused due to lost races (cas failures) at |
| 313 | // object copy/forward pointer install. They are reused. |
| 314 | Mutex skipped_blocks_lock_ DEFAULT_MUTEX_ACQUIRED_AFTER; |
| 315 | std::multimap<size_t, uint8_t*> skipped_blocks_map_ GUARDED_BY(skipped_blocks_lock_); |
| 316 | Atomic<size_t> to_space_bytes_skipped_; |
| 317 | Atomic<size_t> to_space_objects_skipped_; |
| 318 | |
Mathieu Chartier | 56fe258 | 2016-07-14 13:30:03 -0700 | [diff] [blame] | 319 | // If measure_read_barrier_slow_path_ is true, we count how long is spent in MarkFromReadBarrier |
| 320 | // and also log. |
| 321 | bool measure_read_barrier_slow_path_; |
| 322 | // mark_from_read_barrier_measurements_ is true if systrace is enabled or |
| 323 | // measure_read_barrier_time_ is true. |
| 324 | bool mark_from_read_barrier_measurements_; |
| 325 | Atomic<uint64_t> rb_slow_path_ns_; |
| 326 | Atomic<uint64_t> rb_slow_path_count_; |
| 327 | Atomic<uint64_t> rb_slow_path_count_gc_; |
| 328 | mutable Mutex rb_slow_path_histogram_lock_ DEFAULT_MUTEX_ACQUIRED_AFTER; |
| 329 | Histogram<uint64_t> rb_slow_path_time_histogram_ GUARDED_BY(rb_slow_path_histogram_lock_); |
| 330 | uint64_t rb_slow_path_count_total_ GUARDED_BY(rb_slow_path_histogram_lock_); |
| 331 | uint64_t rb_slow_path_count_gc_total_ GUARDED_BY(rb_slow_path_histogram_lock_); |
| 332 | |
Hiroshi Yamauchi | 2cd334a | 2015-01-09 14:03:35 -0800 | [diff] [blame] | 333 | accounting::ReadBarrierTable* rb_table_; |
| 334 | bool force_evacuate_all_; // True if all regions are evacuated. |
Hiroshi Yamauchi | d8db5a2 | 2016-06-28 14:07:41 -0700 | [diff] [blame] | 335 | Atomic<bool> updated_all_immune_objects_; |
| 336 | bool gc_grays_immune_objects_; |
| 337 | Mutex immune_gray_stack_lock_ DEFAULT_MUTEX_ACQUIRED_AFTER; |
| 338 | std::vector<mirror::Object*> immune_gray_stack_ GUARDED_BY(immune_gray_stack_lock_); |
Hiroshi Yamauchi | 2cd334a | 2015-01-09 14:03:35 -0800 | [diff] [blame] | 339 | |
Mathieu Chartier | 3ed8ec1 | 2017-04-20 19:28:54 -0700 | [diff] [blame] | 340 | // Class of java.lang.Object. Filled in from WellKnownClasses in FlipCallback. Must |
| 341 | // be filled in before flipping thread roots so that FillDummyObject can run. Not |
| 342 | // ObjPtr since the GC may transition to suspended and runnable between phases. |
| 343 | mirror::Class* java_lang_Object_; |
| 344 | |
Mathieu Chartier | 3768ade | 2017-05-02 14:04:39 -0700 | [diff] [blame] | 345 | class ActivateReadBarrierEntrypointsCallback; |
| 346 | class ActivateReadBarrierEntrypointsCheckpoint; |
Mathieu Chartier | a07f559 | 2016-06-16 11:44:28 -0700 | [diff] [blame] | 347 | class AssertToSpaceInvariantFieldVisitor; |
Mathieu Chartier | a07f559 | 2016-06-16 11:44:28 -0700 | [diff] [blame] | 348 | class AssertToSpaceInvariantRefsVisitor; |
| 349 | class ClearBlackPtrsVisitor; |
| 350 | class ComputeUnevacFromSpaceLiveRatioVisitor; |
Hiroshi Yamauchi | febd0cf | 2016-09-14 19:31:25 -0700 | [diff] [blame] | 351 | class DisableMarkingCallback; |
Mathieu Chartier | a07f559 | 2016-06-16 11:44:28 -0700 | [diff] [blame] | 352 | class DisableMarkingCheckpoint; |
Hiroshi Yamauchi | febd0cf | 2016-09-14 19:31:25 -0700 | [diff] [blame] | 353 | class DisableWeakRefAccessCallback; |
Mathieu Chartier | a07f559 | 2016-06-16 11:44:28 -0700 | [diff] [blame] | 354 | class FlipCallback; |
Mathieu Chartier | 3768ade | 2017-05-02 14:04:39 -0700 | [diff] [blame] | 355 | template <bool kConcurrent> class GrayImmuneObjectVisitor; |
Hiroshi Yamauchi | d8db5a2 | 2016-06-28 14:07:41 -0700 | [diff] [blame] | 356 | class ImmuneSpaceScanObjVisitor; |
Mathieu Chartier | a07f559 | 2016-06-16 11:44:28 -0700 | [diff] [blame] | 357 | class LostCopyVisitor; |
| 358 | class RefFieldsVisitor; |
| 359 | class RevokeThreadLocalMarkStackCheckpoint; |
Hiroshi Yamauchi | d8db5a2 | 2016-06-28 14:07:41 -0700 | [diff] [blame] | 360 | class ScopedGcGraysImmuneObjects; |
| 361 | class ThreadFlipVisitor; |
Mathieu Chartier | 21328a1 | 2016-07-22 10:47:45 -0700 | [diff] [blame] | 362 | class VerifyGrayImmuneObjectsVisitor; |
Mathieu Chartier | a07f559 | 2016-06-16 11:44:28 -0700 | [diff] [blame] | 363 | class VerifyNoFromSpaceRefsFieldVisitor; |
Mathieu Chartier | a07f559 | 2016-06-16 11:44:28 -0700 | [diff] [blame] | 364 | class VerifyNoFromSpaceRefsVisitor; |
Mathieu Chartier | a1467d0 | 2017-02-22 09:22:50 -0800 | [diff] [blame] | 365 | class VerifyNoMissingCardMarkVisitor; |
Hiroshi Yamauchi | 2cd334a | 2015-01-09 14:03:35 -0800 | [diff] [blame] | 366 | |
Mathieu Chartier | 3130cdf | 2015-05-03 15:20:23 -0700 | [diff] [blame] | 367 | DISALLOW_IMPLICIT_CONSTRUCTORS(ConcurrentCopying); |
Hiroshi Yamauchi | d5307ec | 2014-03-27 21:07:51 -0700 | [diff] [blame] | 368 | }; |
| 369 | |
| 370 | } // namespace collector |
| 371 | } // namespace gc |
| 372 | } // namespace art |
| 373 | |
| 374 | #endif // ART_RUNTIME_GC_COLLECTOR_CONCURRENT_COPYING_H_ |