ART: Add precise root visiting
Add VisitRootFlags::kVisitRootFlagPrecise to signal a
request for precise RootInfo objects.
Move VisitRootFlags to gc_root.h. Refactor VisitRoot
functions to pass flags. Add code in Thread visiting
to decode vregs in compiled code.
Bug: 31385354
Test: m test-art-host
Change-Id: I8e7280f0ab682871f729f2a1c6b18670cf2dbf82
diff --git a/runtime/gc_root.h b/runtime/gc_root.h
index b795409..79e80f1 100644
--- a/runtime/gc_root.h
+++ b/runtime/gc_root.h
@@ -86,6 +86,22 @@
return os;
}
+// Not all combinations of flags are valid. You may not visit all roots as well as the new roots
+// (no logical reason to do this). You also may not start logging new roots and stop logging new
+// roots (also no logical reason to do this).
+//
+// The precise flag ensures that more metadata is supplied. An example is vreg data for compiled
+// method frames.
+enum VisitRootFlags : uint8_t {
+ kVisitRootFlagAllRoots = 0x1,
+ kVisitRootFlagNewRoots = 0x2,
+ kVisitRootFlagStartLoggingNewRoots = 0x4,
+ kVisitRootFlagStopLoggingNewRoots = 0x8,
+ kVisitRootFlagClearRootLog = 0x10,
+ kVisitRootFlagClassLoader = 0x20,
+ kVisitRootFlagPrecise = 0x80,
+};
+
class RootVisitor {
public:
virtual ~RootVisitor() { }
diff --git a/runtime/runtime.cc b/runtime/runtime.cc
index bf34548..14628f0 100644
--- a/runtime/runtime.cc
+++ b/runtime/runtime.cc
@@ -1695,13 +1695,13 @@
VisitTransactionRoots(visitor);
}
-void Runtime::VisitNonConcurrentRoots(RootVisitor* visitor) {
- thread_list_->VisitRoots(visitor);
+void Runtime::VisitNonConcurrentRoots(RootVisitor* visitor, VisitRootFlags flags) {
+ VisitThreadRoots(visitor, flags);
VisitNonThreadRoots(visitor);
}
-void Runtime::VisitThreadRoots(RootVisitor* visitor) {
- thread_list_->VisitRoots(visitor);
+void Runtime::VisitThreadRoots(RootVisitor* visitor, VisitRootFlags flags) {
+ thread_list_->VisitRoots(visitor, flags);
}
size_t Runtime::FlipThreadRoots(Closure* thread_flip_visitor, Closure* flip_callback,
@@ -1710,7 +1710,7 @@
}
void Runtime::VisitRoots(RootVisitor* visitor, VisitRootFlags flags) {
- VisitNonConcurrentRoots(visitor);
+ VisitNonConcurrentRoots(visitor, flags);
VisitConcurrentRoots(visitor, flags);
}
diff --git a/runtime/runtime.h b/runtime/runtime.h
index e6b3128..d40c631 100644
--- a/runtime/runtime.h
+++ b/runtime/runtime.h
@@ -100,18 +100,6 @@
typedef std::vector<std::pair<std::string, const void*>> RuntimeOptions;
-// Not all combinations of flags are valid. You may not visit all roots as well as the new roots
-// (no logical reason to do this). You also may not start logging new roots and stop logging new
-// roots (also no logical reason to do this).
-enum VisitRootFlags : uint8_t {
- kVisitRootFlagAllRoots = 0x1,
- kVisitRootFlagNewRoots = 0x2,
- kVisitRootFlagStartLoggingNewRoots = 0x4,
- kVisitRootFlagStopLoggingNewRoots = 0x8,
- kVisitRootFlagClearRootLog = 0x10,
- kVisitRootFlagClassLoader = 0x20,
-};
-
class Runtime {
public:
// Parse raw runtime options.
@@ -349,28 +337,16 @@
void VisitTransactionRoots(RootVisitor* visitor)
REQUIRES_SHARED(Locks::mutator_lock_);
- // Visit all of the thread roots.
- void VisitThreadRoots(RootVisitor* visitor) REQUIRES_SHARED(Locks::mutator_lock_);
-
// Flip thread roots from from-space refs to to-space refs.
size_t FlipThreadRoots(Closure* thread_flip_visitor, Closure* flip_callback,
gc::collector::GarbageCollector* collector)
REQUIRES(!Locks::mutator_lock_);
- // Visit all other roots which must be done with mutators suspended.
- void VisitNonConcurrentRoots(RootVisitor* visitor)
- REQUIRES_SHARED(Locks::mutator_lock_);
-
// Sweep system weaks, the system weak is deleted if the visitor return null. Otherwise, the
// system weak is updated to be the visitor's returned value.
void SweepSystemWeaks(IsMarkedVisitor* visitor)
REQUIRES_SHARED(Locks::mutator_lock_);
- // Constant roots are the roots which never change after the runtime is initialized, they only
- // need to be visited once per GC cycle.
- void VisitConstantRoots(RootVisitor* visitor)
- REQUIRES_SHARED(Locks::mutator_lock_);
-
// Returns a special method that calls into a trampoline for runtime method resolution
ArtMethod* GetResolutionMethod();
@@ -702,6 +678,19 @@
void MaybeSaveJitProfilingInfo();
+ // Visit all of the thread roots.
+ void VisitThreadRoots(RootVisitor* visitor, VisitRootFlags flags)
+ REQUIRES_SHARED(Locks::mutator_lock_);
+
+ // Visit all other roots which must be done with mutators suspended.
+ void VisitNonConcurrentRoots(RootVisitor* visitor, VisitRootFlags flags)
+ REQUIRES_SHARED(Locks::mutator_lock_);
+
+ // Constant roots are the roots which never change after the runtime is initialized, they only
+ // need to be visited once per GC cycle.
+ void VisitConstantRoots(RootVisitor* visitor)
+ REQUIRES_SHARED(Locks::mutator_lock_);
+
// A pointer to the active runtime or null.
static Runtime* instance_;
diff --git a/runtime/stack.cc b/runtime/stack.cc
index f20aa20..792da88 100644
--- a/runtime/stack.cc
+++ b/runtime/stack.cc
@@ -614,12 +614,6 @@
return result;
}
-static instrumentation::InstrumentationStackFrame& GetInstrumentationStackFrame(Thread* thread,
- uint32_t depth) {
- CHECK_LT(depth, thread->GetInstrumentationStack()->size());
- return thread->GetInstrumentationStack()->at(depth);
-}
-
static void AssertPcIsWithinQuickCode(ArtMethod* method, uintptr_t pc)
REQUIRES_SHARED(Locks::mutator_lock_) {
if (method->IsNative() || method->IsRuntimeMethod() || method->IsProxyMethod()) {
@@ -777,6 +771,7 @@
return QuickMethodFrameInfo(frame_size, callee_info.CoreSpillMask(), callee_info.FpSpillMask());
}
+template <StackVisitor::CountTransitions kCount>
void StackVisitor::WalkStack(bool include_transitions) {
DCHECK(thread_ == Thread::Current() || thread_->IsSuspended());
CHECK_EQ(cur_depth_, 0U);
@@ -842,8 +837,9 @@
// While profiling, the return pc is restored from the side stack, except when walking
// the stack for an exception where the side stack will be unwound in VisitFrame.
if (reinterpret_cast<uintptr_t>(GetQuickInstrumentationExitPc()) == return_pc) {
+ CHECK_LT(instrumentation_stack_depth, thread_->GetInstrumentationStack()->size());
const instrumentation::InstrumentationStackFrame& instrumentation_frame =
- GetInstrumentationStackFrame(thread_, instrumentation_stack_depth);
+ thread_->GetInstrumentationStack()->at(instrumentation_stack_depth);
instrumentation_stack_depth++;
if (GetMethod() ==
Runtime::Current()->GetCalleeSaveMethod(Runtime::kSaveAllCalleeSaves)) {
@@ -907,13 +903,18 @@
return;
}
}
- cur_depth_++;
+ if (kCount == CountTransitions::kYes) {
+ cur_depth_++;
+ }
}
if (num_frames_ != 0) {
CHECK_EQ(cur_depth_, num_frames_);
}
}
+template void StackVisitor::WalkStack<StackVisitor::CountTransitions::kYes>(bool);
+template void StackVisitor::WalkStack<StackVisitor::CountTransitions::kNo>(bool);
+
void JavaFrameRootInfo::Describe(std::ostream& os) const {
const StackVisitor* visitor = stack_visitor_;
CHECK(visitor != nullptr);
diff --git a/runtime/stack.h b/runtime/stack.h
index d02e4b7..b1e99e5 100644
--- a/runtime/stack.h
+++ b/runtime/stack.h
@@ -595,6 +595,12 @@
// Return 'true' if we should continue to visit more frames, 'false' to stop.
virtual bool VisitFrame() REQUIRES_SHARED(Locks::mutator_lock_) = 0;
+ enum class CountTransitions {
+ kYes,
+ kNo,
+ };
+
+ template <CountTransitions kCount = CountTransitions::kYes>
void WalkStack(bool include_transitions = false)
REQUIRES_SHARED(Locks::mutator_lock_);
diff --git a/runtime/thread.cc b/runtime/thread.cc
index 17f5513..bc133d1 100644
--- a/runtime/thread.cc
+++ b/runtime/thread.cc
@@ -2802,7 +2802,7 @@
}
// RootVisitor parameters are: (const Object* obj, size_t vreg, const StackVisitor* visitor).
-template <typename RootVisitor>
+template <typename RootVisitor, bool kPrecise = false>
class ReferenceMapVisitor : public StackVisitor {
public:
ReferenceMapVisitor(Thread* thread, Context* context, RootVisitor& visitor)
@@ -2889,7 +2889,9 @@
}
}
- void VisitQuickFrame() REQUIRES_SHARED(Locks::mutator_lock_) {
+ template <typename T>
+ ALWAYS_INLINE
+ inline void VisitQuickFrameWithVregCallback() REQUIRES_SHARED(Locks::mutator_lock_) {
ArtMethod** cur_quick_frame = GetCurrentQuickFrame();
DCHECK(cur_quick_frame != nullptr);
ArtMethod* m = *cur_quick_frame;
@@ -2906,6 +2908,9 @@
CodeInfoEncoding encoding = code_info.ExtractEncoding();
StackMap map = code_info.GetStackMapForNativePcOffset(native_pc_offset, encoding);
DCHECK(map.IsValid());
+
+ T vreg_info(m, code_info, encoding, map, visitor_);
+
// Visit stack entries that hold pointers.
size_t number_of_bits = map.GetNumberOfStackMaskBits(encoding.stack_map_encoding);
for (size_t i = 0; i < number_of_bits; ++i) {
@@ -2914,7 +2919,7 @@
mirror::Object* ref = ref_addr->AsMirrorPtr();
if (ref != nullptr) {
mirror::Object* new_ref = ref;
- visitor_(&new_ref, -1, this);
+ vreg_info.VisitStack(&new_ref, i, this);
if (ref != new_ref) {
ref_addr->Assign(new_ref);
}
@@ -2935,13 +2940,119 @@
<< "set in register_mask=" << register_mask << " at " << DescribeLocation();
}
if (*ref_addr != nullptr) {
- visitor_(ref_addr, -1, this);
+ vreg_info.VisitRegister(ref_addr, i, this);
}
}
}
}
}
+ void VisitQuickFrame() REQUIRES_SHARED(Locks::mutator_lock_) {
+ if (kPrecise) {
+ VisitQuickFramePrecise();
+ } else {
+ VisitQuickFrameNonPrecise();
+ }
+ }
+
+ void VisitQuickFrameNonPrecise() REQUIRES_SHARED(Locks::mutator_lock_) {
+ struct UndefinedVRegInfo {
+ UndefinedVRegInfo(ArtMethod* method ATTRIBUTE_UNUSED,
+ const CodeInfo& code_info ATTRIBUTE_UNUSED,
+ const CodeInfoEncoding& encoding ATTRIBUTE_UNUSED,
+ const StackMap& map ATTRIBUTE_UNUSED,
+ RootVisitor& _visitor)
+ : visitor(_visitor) {
+ }
+
+ ALWAYS_INLINE
+ void VisitStack(mirror::Object** ref,
+ size_t stack_index ATTRIBUTE_UNUSED,
+ const StackVisitor* stack_visitor)
+ REQUIRES_SHARED(Locks::mutator_lock_) {
+ visitor(ref, -1, stack_visitor);
+ }
+
+ ALWAYS_INLINE
+ void VisitRegister(mirror::Object** ref,
+ size_t register_index ATTRIBUTE_UNUSED,
+ const StackVisitor* stack_visitor)
+ REQUIRES_SHARED(Locks::mutator_lock_) {
+ visitor(ref, -1, stack_visitor);
+ }
+
+ RootVisitor& visitor;
+ };
+ VisitQuickFrameWithVregCallback<UndefinedVRegInfo>();
+ }
+
+ void VisitQuickFramePrecise() REQUIRES_SHARED(Locks::mutator_lock_) {
+ struct StackMapVRegInfo {
+ StackMapVRegInfo(ArtMethod* method,
+ const CodeInfo& _code_info,
+ const CodeInfoEncoding& _encoding,
+ const StackMap& map,
+ RootVisitor& _visitor)
+ : number_of_dex_registers(method->GetCodeItem()->registers_size_),
+ code_info(_code_info),
+ encoding(_encoding),
+ dex_register_map(code_info.GetDexRegisterMapOf(map,
+ encoding,
+ number_of_dex_registers)),
+ visitor(_visitor) {
+ }
+
+ // TODO: If necessary, we should consider caching a reverse map instead of the linear
+ // lookups for each location.
+ void FindWithType(const size_t index,
+ const DexRegisterLocation::Kind kind,
+ mirror::Object** ref,
+ const StackVisitor* stack_visitor)
+ REQUIRES_SHARED(Locks::mutator_lock_) {
+ bool found = false;
+ for (size_t dex_reg = 0; dex_reg != number_of_dex_registers; ++dex_reg) {
+ DexRegisterLocation location = dex_register_map.GetDexRegisterLocation(
+ dex_reg, number_of_dex_registers, code_info, encoding);
+ if (location.GetKind() == kind && static_cast<size_t>(location.GetValue()) == index) {
+ visitor(ref, dex_reg, stack_visitor);
+ found = true;
+ }
+ }
+
+ if (!found) {
+ // If nothing found, report with -1.
+ visitor(ref, -1, stack_visitor);
+ }
+ }
+
+ void VisitStack(mirror::Object** ref, size_t stack_index, const StackVisitor* stack_visitor)
+ REQUIRES_SHARED(Locks::mutator_lock_) {
+ const size_t stack_offset = stack_index * kFrameSlotSize;
+ FindWithType(stack_offset,
+ DexRegisterLocation::Kind::kInStack,
+ ref,
+ stack_visitor);
+ }
+
+ void VisitRegister(mirror::Object** ref,
+ size_t register_index,
+ const StackVisitor* stack_visitor)
+ REQUIRES_SHARED(Locks::mutator_lock_) {
+ FindWithType(register_index,
+ DexRegisterLocation::Kind::kInRegister,
+ ref,
+ stack_visitor);
+ }
+
+ size_t number_of_dex_registers;
+ const CodeInfo& code_info;
+ const CodeInfoEncoding& encoding;
+ DexRegisterMap dex_register_map;
+ RootVisitor& visitor;
+ };
+ VisitQuickFrameWithVregCallback<StackMapVRegInfo>();
+ }
+
// Visitor for when we visit a root.
RootVisitor& visitor_;
};
@@ -2960,6 +3071,7 @@
const uint32_t tid_;
};
+template <bool kPrecise>
void Thread::VisitRoots(RootVisitor* visitor) {
const uint32_t thread_id = GetThreadId();
visitor->VisitRootIfNonNull(&tlsPtr_.opeer, RootInfo(kRootThreadObject, thread_id));
@@ -2977,7 +3089,7 @@
// Visit roots for deoptimization.
if (tlsPtr_.stacked_shadow_frame_record != nullptr) {
RootCallbackVisitor visitor_to_callback(visitor, thread_id);
- ReferenceMapVisitor<RootCallbackVisitor> mapper(this, nullptr, visitor_to_callback);
+ ReferenceMapVisitor<RootCallbackVisitor, kPrecise> mapper(this, nullptr, visitor_to_callback);
for (StackedShadowFrameRecord* record = tlsPtr_.stacked_shadow_frame_record;
record != nullptr;
record = record->GetLink()) {
@@ -3000,7 +3112,7 @@
}
if (tlsPtr_.frame_id_to_shadow_frame != nullptr) {
RootCallbackVisitor visitor_to_callback(visitor, thread_id);
- ReferenceMapVisitor<RootCallbackVisitor> mapper(this, nullptr, visitor_to_callback);
+ ReferenceMapVisitor<RootCallbackVisitor, kPrecise> mapper(this, nullptr, visitor_to_callback);
for (FrameIdToShadowFrame* record = tlsPtr_.frame_id_to_shadow_frame;
record != nullptr;
record = record->GetNext()) {
@@ -3013,14 +3125,22 @@
// Visit roots on this thread's stack
Context* context = GetLongJumpContext();
RootCallbackVisitor visitor_to_callback(visitor, thread_id);
- ReferenceMapVisitor<RootCallbackVisitor> mapper(this, context, visitor_to_callback);
- mapper.WalkStack();
+ ReferenceMapVisitor<RootCallbackVisitor, kPrecise> mapper(this, context, visitor_to_callback);
+ mapper.template WalkStack<StackVisitor::CountTransitions::kNo>(false);
ReleaseLongJumpContext(context);
for (instrumentation::InstrumentationStackFrame& frame : *GetInstrumentationStack()) {
visitor->VisitRootIfNonNull(&frame.this_object_, RootInfo(kRootVMInternal, thread_id));
}
}
+void Thread::VisitRoots(RootVisitor* visitor, VisitRootFlags flags) {
+ if ((flags & VisitRootFlags::kVisitRootFlagPrecise) != 0) {
+ VisitRoots<true>(visitor);
+ } else {
+ VisitRoots<false>(visitor);
+ }
+}
+
class VerifyRootVisitor : public SingleRootVisitor {
public:
void VisitRoot(mirror::Object* root, const RootInfo& info ATTRIBUTE_UNUSED)
diff --git a/runtime/thread.h b/runtime/thread.h
index b80fdc7..31cd0eb 100644
--- a/runtime/thread.h
+++ b/runtime/thread.h
@@ -549,7 +549,8 @@
return tlsPtr_.frame_id_to_shadow_frame != nullptr;
}
- void VisitRoots(RootVisitor* visitor) REQUIRES_SHARED(Locks::mutator_lock_);
+ void VisitRoots(RootVisitor* visitor, VisitRootFlags flags = kVisitRootFlagAllRoots)
+ REQUIRES_SHARED(Locks::mutator_lock_);
ALWAYS_INLINE void VerifyStack() REQUIRES_SHARED(Locks::mutator_lock_);
@@ -1245,6 +1246,9 @@
// Install the protected region for implicit stack checks.
void InstallImplicitProtection();
+ template <bool kPrecise>
+ void VisitRoots(RootVisitor* visitor) REQUIRES_SHARED(Locks::mutator_lock_);
+
static bool IsAotCompiler();
// 32 bits of atomically changed state and flags. Keeping as 32 bits allows and atomic CAS to
diff --git a/runtime/thread_list.cc b/runtime/thread_list.cc
index a6bd83d..664eeb4 100644
--- a/runtime/thread_list.cc
+++ b/runtime/thread_list.cc
@@ -1395,10 +1395,10 @@
}
}
-void ThreadList::VisitRoots(RootVisitor* visitor) const {
+void ThreadList::VisitRoots(RootVisitor* visitor, VisitRootFlags flags) const {
MutexLock mu(Thread::Current(), *Locks::thread_list_lock_);
for (const auto& thread : list_) {
- thread->VisitRoots(visitor);
+ thread->VisitRoots(visitor, flags);
}
}
diff --git a/runtime/thread_list.h b/runtime/thread_list.h
index 1acabcb..658db00 100644
--- a/runtime/thread_list.h
+++ b/runtime/thread_list.h
@@ -155,7 +155,7 @@
!Locks::thread_list_lock_,
!Locks::thread_suspend_count_lock_);
- void VisitRoots(RootVisitor* visitor) const
+ void VisitRoots(RootVisitor* visitor, VisitRootFlags flags) const
REQUIRES_SHARED(Locks::mutator_lock_);
void VisitRootsForSuspendedThreads(RootVisitor* visitor)