Merge "Use entrypoint switching to reduce code size of GcRoot read barrier"
diff --git a/compiler/optimizing/code_generator_arm.cc b/compiler/optimizing/code_generator_arm.cc
index 32642e1..7c72d00 100644
--- a/compiler/optimizing/code_generator_arm.cc
+++ b/compiler/optimizing/code_generator_arm.cc
@@ -620,8 +620,10 @@
// reference (different from `ref`) in `obj.field`).
class ReadBarrierMarkSlowPathARM : public SlowPathCodeARM {
public:
- ReadBarrierMarkSlowPathARM(HInstruction* instruction, Location ref)
- : SlowPathCodeARM(instruction), ref_(ref) {
+ ReadBarrierMarkSlowPathARM(HInstruction* instruction,
+ Location ref,
+ Location entrypoint = Location::NoLocation())
+ : SlowPathCodeARM(instruction), ref_(ref), entrypoint_(entrypoint) {
DCHECK(kEmitCompilerReadBarrier);
}
@@ -676,10 +678,15 @@
//
// rX <- ReadBarrierMarkRegX(rX)
//
- int32_t entry_point_offset =
- CodeGenerator::GetReadBarrierMarkEntryPointsOffset<kArmPointerSize>(ref_reg);
- // This runtime call does not require a stack map.
- arm_codegen->InvokeRuntimeWithoutRecordingPcInfo(entry_point_offset, instruction_, this);
+ if (entrypoint_.IsValid()) {
+ arm_codegen->ValidateInvokeRuntimeWithoutRecordingPcInfo(instruction_, this);
+ __ blx(entrypoint_.AsRegister<Register>());
+ } else {
+ int32_t entry_point_offset =
+ CodeGenerator::GetReadBarrierMarkEntryPointsOffset<kArmPointerSize>(ref_reg);
+ // This runtime call does not require a stack map.
+ arm_codegen->InvokeRuntimeWithoutRecordingPcInfo(entry_point_offset, instruction_, this);
+ }
__ b(GetExitLabel());
}
@@ -687,6 +694,9 @@
// The location (register) of the marked object reference.
const Location ref_;
+ // The location of the entrypoint if already loaded.
+ const Location entrypoint_;
+
DISALLOW_COPY_AND_ASSIGN(ReadBarrierMarkSlowPathARM);
};
@@ -6836,8 +6846,9 @@
// Baker's read barrier are used:
//
// root = obj.field;
- // if (Thread::Current()->GetIsGcMarking()) {
- // root = ReadBarrier::Mark(root)
+ // temp = Thread::Current()->pReadBarrierMarkReg ## root.reg()
+ // if (temp != null) {
+ // root = temp(root)
// }
// /* GcRoot<mirror::Object> */ root = *(obj + offset)
@@ -6851,14 +6862,23 @@
"have different sizes.");
// Slow path marking the GC root `root`.
+ Location temp = Location::RegisterLocation(LR);
SlowPathCodeARM* slow_path =
- new (GetGraph()->GetArena()) ReadBarrierMarkSlowPathARM(instruction, root);
+ new (GetGraph()->GetArena()) ReadBarrierMarkSlowPathARM(
+ instruction,
+ root,
+ /*entrypoint*/ temp);
codegen_->AddSlowPath(slow_path);
- // IP = Thread::Current()->GetIsGcMarking()
- __ LoadFromOffset(
- kLoadWord, IP, TR, Thread::IsGcMarkingOffset<kArmPointerSize>().Int32Value());
- __ CompareAndBranchIfNonZero(IP, slow_path->GetEntryLabel());
+ // temp = Thread::Current()->pReadBarrierMarkReg ## root.reg()
+ const int32_t entry_point_offset =
+ CodeGenerator::GetReadBarrierMarkEntryPointsOffset<kArmPointerSize>(root.reg());
+ // Loading the entrypoint does not require a load acquire since it is only changed when
+ // threads are suspended or running a checkpoint.
+ __ LoadFromOffset(kLoadWord, temp.AsRegister<Register>(), TR, entry_point_offset);
+ // The entrypoint is null when the GC is not marking, this prevents one load compared to
+ // checking GetIsGcMarking.
+ __ CompareAndBranchIfNonZero(temp.AsRegister<Register>(), slow_path->GetEntryLabel());
__ Bind(slow_path->GetExitLabel());
} else {
// GC root loaded through a slow path for read barriers other
diff --git a/compiler/optimizing/code_generator_arm64.cc b/compiler/optimizing/code_generator_arm64.cc
index ef4e511..35b1605 100644
--- a/compiler/optimizing/code_generator_arm64.cc
+++ b/compiler/optimizing/code_generator_arm64.cc
@@ -607,10 +607,16 @@
// probably still be a from-space reference (unless it gets updated by
// another thread, or if another thread installed another object
// reference (different from `ref`) in `obj.field`).
+// If entrypoint is a valid location it is assumed to already be holding the entrypoint. The case
+// where the entrypoint is passed in is for the GcRoot read barrier.
class ReadBarrierMarkSlowPathARM64 : public SlowPathCodeARM64 {
public:
- ReadBarrierMarkSlowPathARM64(HInstruction* instruction, Location ref)
- : SlowPathCodeARM64(instruction), ref_(ref) {
+ ReadBarrierMarkSlowPathARM64(HInstruction* instruction,
+ Location ref,
+ Location entrypoint = Location::NoLocation())
+ : SlowPathCodeARM64(instruction),
+ ref_(ref),
+ entrypoint_(entrypoint) {
DCHECK(kEmitCompilerReadBarrier);
}
@@ -665,10 +671,16 @@
//
// rX <- ReadBarrierMarkRegX(rX)
//
- int32_t entry_point_offset =
- CodeGenerator::GetReadBarrierMarkEntryPointsOffset<kArm64PointerSize>(ref_.reg());
- // This runtime call does not require a stack map.
- arm64_codegen->InvokeRuntimeWithoutRecordingPcInfo(entry_point_offset, instruction_, this);
+ if (entrypoint_.IsValid()) {
+ arm64_codegen->ValidateInvokeRuntimeWithoutRecordingPcInfo(instruction_, this);
+ __ Blr(XRegisterFrom(entrypoint_));
+ } else {
+ // Entrypoint is not already loaded, load from the thread.
+ int32_t entry_point_offset =
+ CodeGenerator::GetReadBarrierMarkEntryPointsOffset<kArm64PointerSize>(ref_.reg());
+ // This runtime call does not require a stack map.
+ arm64_codegen->InvokeRuntimeWithoutRecordingPcInfo(entry_point_offset, instruction_, this);
+ }
__ B(GetExitLabel());
}
@@ -676,6 +688,9 @@
// The location (register) of the marked object reference.
const Location ref_;
+ // The location of the entrypoint if it is already loaded.
+ const Location entrypoint_;
+
DISALLOW_COPY_AND_ASSIGN(ReadBarrierMarkSlowPathARM64);
};
@@ -5379,8 +5394,9 @@
// Baker's read barrier are used:
//
// root = obj.field;
- // if (Thread::Current()->GetIsGcMarking()) {
- // root = ReadBarrier::Mark(root)
+ // temp = Thread::Current()->pReadBarrierMarkReg ## root.reg()
+ // if (temp != null) {
+ // root = temp(root)
// }
// /* GcRoot<mirror::Object> */ root = *(obj + offset)
@@ -5397,16 +5413,22 @@
"art::mirror::CompressedReference<mirror::Object> and int32_t "
"have different sizes.");
- // Slow path marking the GC root `root`.
- SlowPathCodeARM64* slow_path =
- new (GetGraph()->GetArena()) ReadBarrierMarkSlowPathARM64(instruction, root);
- codegen_->AddSlowPath(slow_path);
+ Register temp = lr;
- MacroAssembler* masm = GetVIXLAssembler();
- UseScratchRegisterScope temps(masm);
- Register temp = temps.AcquireW();
- // temp = Thread::Current()->GetIsGcMarking()
- __ Ldr(temp, MemOperand(tr, Thread::IsGcMarkingOffset<kArm64PointerSize>().Int32Value()));
+ // Slow path marking the GC root `root`. The entrypoint will alrady be loaded in temp.
+ SlowPathCodeARM64* slow_path =
+ new (GetGraph()->GetArena()) ReadBarrierMarkSlowPathARM64(instruction,
+ root,
+ LocationFrom(temp));
+ codegen_->AddSlowPath(slow_path);
+ const int32_t entry_point_offset =
+ CodeGenerator::GetReadBarrierMarkEntryPointsOffset<kArm64PointerSize>(root.reg());
+ // temp = Thread::Current()->pReadBarrierMarkReg ## root.reg()
+ // Loading the entrypoint does not require a load acquire since it is only changed when
+ // threads are suspended or running a checkpoint.
+ __ Ldr(temp, MemOperand(tr, entry_point_offset));
+ // The entrypoint is null when the GC is not marking, this prevents one load compared to
+ // checking GetIsGcMarking.
__ Cbnz(temp, slow_path->GetEntryLabel());
__ Bind(slow_path->GetExitLabel());
} else {
diff --git a/runtime/arch/arm/entrypoints_init_arm.cc b/runtime/arch/arm/entrypoints_init_arm.cc
index 01b3f34..5bd6b56 100644
--- a/runtime/arch/arm/entrypoints_init_arm.cc
+++ b/runtime/arch/arm/entrypoints_init_arm.cc
@@ -67,6 +67,21 @@
// Long long arithmetics - REM_LONG[_2ADDR] and DIV_LONG[_2ADDR]
extern "C" int64_t __aeabi_ldivmod(int64_t, int64_t);
+void UpdateReadBarrierEntrypoints(QuickEntryPoints* qpoints, bool is_marking) {
+ qpoints->pReadBarrierMarkReg00 = is_marking ? art_quick_read_barrier_mark_reg00 : nullptr;
+ qpoints->pReadBarrierMarkReg01 = is_marking ? art_quick_read_barrier_mark_reg01 : nullptr;
+ qpoints->pReadBarrierMarkReg02 = is_marking ? art_quick_read_barrier_mark_reg02 : nullptr;
+ qpoints->pReadBarrierMarkReg03 = is_marking ? art_quick_read_barrier_mark_reg03 : nullptr;
+ qpoints->pReadBarrierMarkReg04 = is_marking ? art_quick_read_barrier_mark_reg04 : nullptr;
+ qpoints->pReadBarrierMarkReg05 = is_marking ? art_quick_read_barrier_mark_reg05 : nullptr;
+ qpoints->pReadBarrierMarkReg06 = is_marking ? art_quick_read_barrier_mark_reg06 : nullptr;
+ qpoints->pReadBarrierMarkReg07 = is_marking ? art_quick_read_barrier_mark_reg07 : nullptr;
+ qpoints->pReadBarrierMarkReg08 = is_marking ? art_quick_read_barrier_mark_reg08 : nullptr;
+ qpoints->pReadBarrierMarkReg09 = is_marking ? art_quick_read_barrier_mark_reg09 : nullptr;
+ qpoints->pReadBarrierMarkReg10 = is_marking ? art_quick_read_barrier_mark_reg10 : nullptr;
+ qpoints->pReadBarrierMarkReg11 = is_marking ? art_quick_read_barrier_mark_reg11 : nullptr;
+}
+
void InitEntryPoints(JniEntryPoints* jpoints, QuickEntryPoints* qpoints) {
DefaultInitEntryPoints(jpoints, qpoints);
@@ -123,18 +138,7 @@
// Read barrier.
qpoints->pReadBarrierJni = ReadBarrierJni;
- qpoints->pReadBarrierMarkReg00 = art_quick_read_barrier_mark_reg00;
- qpoints->pReadBarrierMarkReg01 = art_quick_read_barrier_mark_reg01;
- qpoints->pReadBarrierMarkReg02 = art_quick_read_barrier_mark_reg02;
- qpoints->pReadBarrierMarkReg03 = art_quick_read_barrier_mark_reg03;
- qpoints->pReadBarrierMarkReg04 = art_quick_read_barrier_mark_reg04;
- qpoints->pReadBarrierMarkReg05 = art_quick_read_barrier_mark_reg05;
- qpoints->pReadBarrierMarkReg06 = art_quick_read_barrier_mark_reg06;
- qpoints->pReadBarrierMarkReg07 = art_quick_read_barrier_mark_reg07;
- qpoints->pReadBarrierMarkReg08 = art_quick_read_barrier_mark_reg08;
- qpoints->pReadBarrierMarkReg09 = art_quick_read_barrier_mark_reg09;
- qpoints->pReadBarrierMarkReg10 = art_quick_read_barrier_mark_reg10;
- qpoints->pReadBarrierMarkReg11 = art_quick_read_barrier_mark_reg11;
+ UpdateReadBarrierEntrypoints(qpoints, /*is_marking*/ false);
qpoints->pReadBarrierMarkReg12 = nullptr; // Cannot use register 12 (IP) to pass arguments.
qpoints->pReadBarrierMarkReg13 = nullptr; // Cannot use register 13 (SP) to pass arguments.
qpoints->pReadBarrierMarkReg14 = nullptr; // Cannot use register 14 (LR) to pass arguments.
diff --git a/runtime/arch/arm64/entrypoints_init_arm64.cc b/runtime/arch/arm64/entrypoints_init_arm64.cc
index 3c77672..e7c9fef 100644
--- a/runtime/arch/arm64/entrypoints_init_arm64.cc
+++ b/runtime/arch/arm64/entrypoints_init_arm64.cc
@@ -70,6 +70,47 @@
extern "C" mirror::Object* art_quick_read_barrier_mark_reg28(mirror::Object*);
extern "C" mirror::Object* art_quick_read_barrier_mark_reg29(mirror::Object*);
+void UpdateReadBarrierEntrypoints(QuickEntryPoints* qpoints, bool is_marking) {
+ // ARM64 is the architecture with the largest number of core
+ // registers (32) that supports the read barrier configuration.
+ // Because registers 30 (LR) and 31 (SP/XZR) cannot be used to pass
+ // arguments, only define ReadBarrierMarkRegX entrypoints for the
+ // first 30 registers. This limitation is not a problem on other
+ // supported architectures (ARM, x86 and x86-64) either, as they
+ // have less core registers (resp. 16, 8 and 16). (We may have to
+ // revise that design choice if read barrier support is added for
+ // MIPS and/or MIPS64.)
+ qpoints->pReadBarrierMarkReg00 = is_marking ? art_quick_read_barrier_mark_reg00 : nullptr;
+ qpoints->pReadBarrierMarkReg01 = is_marking ? art_quick_read_barrier_mark_reg01 : nullptr;
+ qpoints->pReadBarrierMarkReg02 = is_marking ? art_quick_read_barrier_mark_reg02 : nullptr;
+ qpoints->pReadBarrierMarkReg03 = is_marking ? art_quick_read_barrier_mark_reg03 : nullptr;
+ qpoints->pReadBarrierMarkReg04 = is_marking ? art_quick_read_barrier_mark_reg04 : nullptr;
+ qpoints->pReadBarrierMarkReg05 = is_marking ? art_quick_read_barrier_mark_reg05 : nullptr;
+ qpoints->pReadBarrierMarkReg06 = is_marking ? art_quick_read_barrier_mark_reg06 : nullptr;
+ qpoints->pReadBarrierMarkReg07 = is_marking ? art_quick_read_barrier_mark_reg07 : nullptr;
+ qpoints->pReadBarrierMarkReg08 = is_marking ? art_quick_read_barrier_mark_reg08 : nullptr;
+ qpoints->pReadBarrierMarkReg09 = is_marking ? art_quick_read_barrier_mark_reg09 : nullptr;
+ qpoints->pReadBarrierMarkReg10 = is_marking ? art_quick_read_barrier_mark_reg10 : nullptr;
+ qpoints->pReadBarrierMarkReg11 = is_marking ? art_quick_read_barrier_mark_reg11 : nullptr;
+ qpoints->pReadBarrierMarkReg12 = is_marking ? art_quick_read_barrier_mark_reg12 : nullptr;
+ qpoints->pReadBarrierMarkReg13 = is_marking ? art_quick_read_barrier_mark_reg13 : nullptr;
+ qpoints->pReadBarrierMarkReg14 = is_marking ? art_quick_read_barrier_mark_reg14 : nullptr;
+ qpoints->pReadBarrierMarkReg15 = is_marking ? art_quick_read_barrier_mark_reg15 : nullptr;
+ qpoints->pReadBarrierMarkReg17 = is_marking ? art_quick_read_barrier_mark_reg17 : nullptr;
+ qpoints->pReadBarrierMarkReg18 = is_marking ? art_quick_read_barrier_mark_reg18 : nullptr;
+ qpoints->pReadBarrierMarkReg19 = is_marking ? art_quick_read_barrier_mark_reg19 : nullptr;
+ qpoints->pReadBarrierMarkReg20 = is_marking ? art_quick_read_barrier_mark_reg20 : nullptr;
+ qpoints->pReadBarrierMarkReg21 = is_marking ? art_quick_read_barrier_mark_reg21 : nullptr;
+ qpoints->pReadBarrierMarkReg22 = is_marking ? art_quick_read_barrier_mark_reg22 : nullptr;
+ qpoints->pReadBarrierMarkReg23 = is_marking ? art_quick_read_barrier_mark_reg23 : nullptr;
+ qpoints->pReadBarrierMarkReg24 = is_marking ? art_quick_read_barrier_mark_reg24 : nullptr;
+ qpoints->pReadBarrierMarkReg25 = is_marking ? art_quick_read_barrier_mark_reg25 : nullptr;
+ qpoints->pReadBarrierMarkReg26 = is_marking ? art_quick_read_barrier_mark_reg26 : nullptr;
+ qpoints->pReadBarrierMarkReg27 = is_marking ? art_quick_read_barrier_mark_reg27 : nullptr;
+ qpoints->pReadBarrierMarkReg28 = is_marking ? art_quick_read_barrier_mark_reg28 : nullptr;
+ qpoints->pReadBarrierMarkReg29 = is_marking ? art_quick_read_barrier_mark_reg29 : nullptr;
+}
+
void InitEntryPoints(JniEntryPoints* jpoints, QuickEntryPoints* qpoints) {
DefaultInitEntryPoints(jpoints, qpoints);
@@ -126,45 +167,8 @@
// Read barrier.
qpoints->pReadBarrierJni = ReadBarrierJni;
- // ARM64 is the architecture with the largest number of core
- // registers (32) that supports the read barrier configuration.
- // Because registers 30 (LR) and 31 (SP/XZR) cannot be used to pass
- // arguments, only define ReadBarrierMarkRegX entrypoints for the
- // first 30 registers. This limitation is not a problem on other
- // supported architectures (ARM, x86 and x86-64) either, as they
- // have less core registers (resp. 16, 8 and 16). (We may have to
- // revise that design choice if read barrier support is added for
- // MIPS and/or MIPS64.)
- qpoints->pReadBarrierMarkReg00 = art_quick_read_barrier_mark_reg00;
- qpoints->pReadBarrierMarkReg01 = art_quick_read_barrier_mark_reg01;
- qpoints->pReadBarrierMarkReg02 = art_quick_read_barrier_mark_reg02;
- qpoints->pReadBarrierMarkReg03 = art_quick_read_barrier_mark_reg03;
- qpoints->pReadBarrierMarkReg04 = art_quick_read_barrier_mark_reg04;
- qpoints->pReadBarrierMarkReg05 = art_quick_read_barrier_mark_reg05;
- qpoints->pReadBarrierMarkReg06 = art_quick_read_barrier_mark_reg06;
- qpoints->pReadBarrierMarkReg07 = art_quick_read_barrier_mark_reg07;
- qpoints->pReadBarrierMarkReg08 = art_quick_read_barrier_mark_reg08;
- qpoints->pReadBarrierMarkReg09 = art_quick_read_barrier_mark_reg09;
- qpoints->pReadBarrierMarkReg10 = art_quick_read_barrier_mark_reg10;
- qpoints->pReadBarrierMarkReg11 = art_quick_read_barrier_mark_reg11;
- qpoints->pReadBarrierMarkReg12 = art_quick_read_barrier_mark_reg12;
- qpoints->pReadBarrierMarkReg13 = art_quick_read_barrier_mark_reg13;
- qpoints->pReadBarrierMarkReg14 = art_quick_read_barrier_mark_reg14;
- qpoints->pReadBarrierMarkReg15 = art_quick_read_barrier_mark_reg15;
qpoints->pReadBarrierMarkReg16 = nullptr; // IP0 is used as a temp by the asm stub.
- qpoints->pReadBarrierMarkReg17 = art_quick_read_barrier_mark_reg17;
- qpoints->pReadBarrierMarkReg18 = art_quick_read_barrier_mark_reg18;
- qpoints->pReadBarrierMarkReg19 = art_quick_read_barrier_mark_reg19;
- qpoints->pReadBarrierMarkReg20 = art_quick_read_barrier_mark_reg20;
- qpoints->pReadBarrierMarkReg21 = art_quick_read_barrier_mark_reg21;
- qpoints->pReadBarrierMarkReg22 = art_quick_read_barrier_mark_reg22;
- qpoints->pReadBarrierMarkReg23 = art_quick_read_barrier_mark_reg23;
- qpoints->pReadBarrierMarkReg24 = art_quick_read_barrier_mark_reg24;
- qpoints->pReadBarrierMarkReg25 = art_quick_read_barrier_mark_reg25;
- qpoints->pReadBarrierMarkReg26 = art_quick_read_barrier_mark_reg26;
- qpoints->pReadBarrierMarkReg27 = art_quick_read_barrier_mark_reg27;
- qpoints->pReadBarrierMarkReg28 = art_quick_read_barrier_mark_reg28;
- qpoints->pReadBarrierMarkReg29 = art_quick_read_barrier_mark_reg29;
+ UpdateReadBarrierEntrypoints(qpoints, /*is_marking*/ false);
qpoints->pReadBarrierSlow = artReadBarrierSlow;
qpoints->pReadBarrierForRootSlow = artReadBarrierForRootSlow;
};
diff --git a/runtime/arch/mips/entrypoints_init_mips.cc b/runtime/arch/mips/entrypoints_init_mips.cc
index e3230f6..8c4df0f 100644
--- a/runtime/arch/mips/entrypoints_init_mips.cc
+++ b/runtime/arch/mips/entrypoints_init_mips.cc
@@ -59,6 +59,9 @@
extern "C" int64_t __divdi3(int64_t, int64_t);
extern "C" int64_t __moddi3(int64_t, int64_t);
+// No read barrier entrypoints for marking registers.
+void UpdateReadBarrierEntrypoints(QuickEntryPoints* qpoints, bool is_marking) {}
+
void InitEntryPoints(JniEntryPoints* jpoints, QuickEntryPoints* qpoints) {
// Note: MIPS has asserts checking for the type of entrypoint. Don't move it
// to InitDefaultEntryPoints().
diff --git a/runtime/arch/mips64/entrypoints_init_mips64.cc b/runtime/arch/mips64/entrypoints_init_mips64.cc
index 43b73f1..0bd4e14 100644
--- a/runtime/arch/mips64/entrypoints_init_mips64.cc
+++ b/runtime/arch/mips64/entrypoints_init_mips64.cc
@@ -59,6 +59,9 @@
extern "C" int64_t __divdi3(int64_t, int64_t);
extern "C" int64_t __moddi3(int64_t, int64_t);
+// No read barrier entrypoints for marking registers.
+void UpdateReadBarrierEntrypoints(QuickEntryPoints* qpoints, bool is_marking) {}
+
void InitEntryPoints(JniEntryPoints* jpoints, QuickEntryPoints* qpoints) {
DefaultInitEntryPoints(jpoints, qpoints);
diff --git a/runtime/arch/x86/entrypoints_init_x86.cc b/runtime/arch/x86/entrypoints_init_x86.cc
index 877df8f..94fea69 100644
--- a/runtime/arch/x86/entrypoints_init_x86.cc
+++ b/runtime/arch/x86/entrypoints_init_x86.cc
@@ -44,6 +44,16 @@
extern "C" mirror::Object* art_quick_read_barrier_slow(mirror::Object*, mirror::Object*, uint32_t);
extern "C" mirror::Object* art_quick_read_barrier_for_root_slow(GcRoot<mirror::Object>*);
+void UpdateReadBarrierEntrypoints(QuickEntryPoints* qpoints, bool is_marking) {
+ qpoints->pReadBarrierMarkReg00 = is_marking ? art_quick_read_barrier_mark_reg00 : nullptr;
+ qpoints->pReadBarrierMarkReg01 = is_marking ? art_quick_read_barrier_mark_reg01 : nullptr;
+ qpoints->pReadBarrierMarkReg02 = is_marking ? art_quick_read_barrier_mark_reg02 : nullptr;
+ qpoints->pReadBarrierMarkReg03 = is_marking ? art_quick_read_barrier_mark_reg03 : nullptr;
+ qpoints->pReadBarrierMarkReg05 = is_marking ? art_quick_read_barrier_mark_reg05 : nullptr;
+ qpoints->pReadBarrierMarkReg06 = is_marking ? art_quick_read_barrier_mark_reg06 : nullptr;
+ qpoints->pReadBarrierMarkReg07 = is_marking ? art_quick_read_barrier_mark_reg07 : nullptr;
+}
+
void InitEntryPoints(JniEntryPoints* jpoints, QuickEntryPoints* qpoints) {
DefaultInitEntryPoints(jpoints, qpoints);
@@ -87,14 +97,8 @@
// Read barrier.
qpoints->pReadBarrierJni = ReadBarrierJni;
- qpoints->pReadBarrierMarkReg00 = art_quick_read_barrier_mark_reg00;
- qpoints->pReadBarrierMarkReg01 = art_quick_read_barrier_mark_reg01;
- qpoints->pReadBarrierMarkReg02 = art_quick_read_barrier_mark_reg02;
- qpoints->pReadBarrierMarkReg03 = art_quick_read_barrier_mark_reg03;
+ UpdateReadBarrierEntrypoints(qpoints, /*is_marking*/ false);
qpoints->pReadBarrierMarkReg04 = nullptr; // Cannot use register 4 (ESP) to pass arguments.
- qpoints->pReadBarrierMarkReg05 = art_quick_read_barrier_mark_reg05;
- qpoints->pReadBarrierMarkReg06 = art_quick_read_barrier_mark_reg06;
- qpoints->pReadBarrierMarkReg07 = art_quick_read_barrier_mark_reg07;
// x86 has only 8 core registers.
qpoints->pReadBarrierMarkReg08 = nullptr;
qpoints->pReadBarrierMarkReg09 = nullptr;
diff --git a/runtime/arch/x86_64/entrypoints_init_x86_64.cc b/runtime/arch/x86_64/entrypoints_init_x86_64.cc
index 59c9dfe..6b66e62 100644
--- a/runtime/arch/x86_64/entrypoints_init_x86_64.cc
+++ b/runtime/arch/x86_64/entrypoints_init_x86_64.cc
@@ -55,6 +55,24 @@
extern "C" mirror::Object* art_quick_read_barrier_slow(mirror::Object*, mirror::Object*, uint32_t);
extern "C" mirror::Object* art_quick_read_barrier_for_root_slow(GcRoot<mirror::Object>*);
+void UpdateReadBarrierEntrypoints(QuickEntryPoints* qpoints, bool is_marking) {
+ qpoints->pReadBarrierMarkReg00 = is_marking ? art_quick_read_barrier_mark_reg00 : nullptr;
+ qpoints->pReadBarrierMarkReg01 = is_marking ? art_quick_read_barrier_mark_reg01 : nullptr;
+ qpoints->pReadBarrierMarkReg02 = is_marking ? art_quick_read_barrier_mark_reg02 : nullptr;
+ qpoints->pReadBarrierMarkReg03 = is_marking ? art_quick_read_barrier_mark_reg03 : nullptr;
+ qpoints->pReadBarrierMarkReg05 = is_marking ? art_quick_read_barrier_mark_reg05 : nullptr;
+ qpoints->pReadBarrierMarkReg06 = is_marking ? art_quick_read_barrier_mark_reg06 : nullptr;
+ qpoints->pReadBarrierMarkReg07 = is_marking ? art_quick_read_barrier_mark_reg07 : nullptr;
+ qpoints->pReadBarrierMarkReg08 = is_marking ? art_quick_read_barrier_mark_reg08 : nullptr;
+ qpoints->pReadBarrierMarkReg09 = is_marking ? art_quick_read_barrier_mark_reg09 : nullptr;
+ qpoints->pReadBarrierMarkReg10 = is_marking ? art_quick_read_barrier_mark_reg10 : nullptr;
+ qpoints->pReadBarrierMarkReg11 = is_marking ? art_quick_read_barrier_mark_reg11 : nullptr;
+ qpoints->pReadBarrierMarkReg12 = is_marking ? art_quick_read_barrier_mark_reg12 : nullptr;
+ qpoints->pReadBarrierMarkReg13 = is_marking ? art_quick_read_barrier_mark_reg13 : nullptr;
+ qpoints->pReadBarrierMarkReg14 = is_marking ? art_quick_read_barrier_mark_reg14 : nullptr;
+ qpoints->pReadBarrierMarkReg15 = is_marking ? art_quick_read_barrier_mark_reg15 : nullptr;
+}
+
void InitEntryPoints(JniEntryPoints* jpoints, QuickEntryPoints* qpoints) {
#if defined(__APPLE__)
UNUSED(jpoints, qpoints);
@@ -101,22 +119,8 @@
// Read barrier.
qpoints->pReadBarrierJni = ReadBarrierJni;
- qpoints->pReadBarrierMarkReg00 = art_quick_read_barrier_mark_reg00;
- qpoints->pReadBarrierMarkReg01 = art_quick_read_barrier_mark_reg01;
- qpoints->pReadBarrierMarkReg02 = art_quick_read_barrier_mark_reg02;
- qpoints->pReadBarrierMarkReg03 = art_quick_read_barrier_mark_reg03;
+ UpdateReadBarrierEntrypoints(qpoints, /*is_marking*/ false);
qpoints->pReadBarrierMarkReg04 = nullptr; // Cannot use register 4 (RSP) to pass arguments.
- qpoints->pReadBarrierMarkReg05 = art_quick_read_barrier_mark_reg05;
- qpoints->pReadBarrierMarkReg06 = art_quick_read_barrier_mark_reg06;
- qpoints->pReadBarrierMarkReg07 = art_quick_read_barrier_mark_reg07;
- qpoints->pReadBarrierMarkReg08 = art_quick_read_barrier_mark_reg08;
- qpoints->pReadBarrierMarkReg09 = art_quick_read_barrier_mark_reg09;
- qpoints->pReadBarrierMarkReg10 = art_quick_read_barrier_mark_reg10;
- qpoints->pReadBarrierMarkReg11 = art_quick_read_barrier_mark_reg11;
- qpoints->pReadBarrierMarkReg12 = art_quick_read_barrier_mark_reg12;
- qpoints->pReadBarrierMarkReg13 = art_quick_read_barrier_mark_reg13;
- qpoints->pReadBarrierMarkReg14 = art_quick_read_barrier_mark_reg14;
- qpoints->pReadBarrierMarkReg15 = art_quick_read_barrier_mark_reg15;
// x86-64 has only 16 core registers.
qpoints->pReadBarrierMarkReg16 = nullptr;
qpoints->pReadBarrierMarkReg17 = nullptr;
diff --git a/runtime/gc/collector/concurrent_copying.cc b/runtime/gc/collector/concurrent_copying.cc
index 11d6849..2e72ada 100644
--- a/runtime/gc/collector/concurrent_copying.cc
+++ b/runtime/gc/collector/concurrent_copying.cc
@@ -246,7 +246,7 @@
Thread* self = Thread::Current();
CHECK(thread == self || thread->IsSuspended() || thread->GetState() == kWaitingPerformingGc)
<< thread->GetState() << " thread " << thread << " self " << self;
- thread->SetIsGcMarking(true);
+ thread->SetIsGcMarkingAndUpdateEntrypoints(true);
if (use_tlab_ && thread->HasTlab()) {
if (ConcurrentCopying::kEnableFromSpaceAccountingCheck) {
// This must come before the revoke.
@@ -746,7 +746,7 @@
// Disable the thread-local is_gc_marking flag.
// Note a thread that has just started right before this checkpoint may have already this flag
// set to false, which is ok.
- thread->SetIsGcMarking(false);
+ thread->SetIsGcMarkingAndUpdateEntrypoints(false);
// If thread is a running mutator, then act on behalf of the garbage collector.
// See the code in ThreadList::RunCheckpoint.
concurrent_copying_->GetBarrier().Pass(self);
diff --git a/runtime/thread.cc b/runtime/thread.cc
index 8ce9661..23c077c 100644
--- a/runtime/thread.cc
+++ b/runtime/thread.cc
@@ -116,6 +116,13 @@
}
void InitEntryPoints(JniEntryPoints* jpoints, QuickEntryPoints* qpoints);
+void UpdateReadBarrierEntrypoints(QuickEntryPoints* qpoints, bool is_marking);
+
+void Thread::SetIsGcMarkingAndUpdateEntrypoints(bool is_marking) {
+ CHECK(kUseReadBarrier);
+ tls32_.is_gc_marking = is_marking;
+ UpdateReadBarrierEntrypoints(&tlsPtr_.quick_entrypoints, is_marking);
+}
void Thread::InitTlsEntryPoints() {
// Insert a placeholder so we can easily tell if we call an unimplemented entry point.
diff --git a/runtime/thread.h b/runtime/thread.h
index f3001be..faa77e1 100644
--- a/runtime/thread.h
+++ b/runtime/thread.h
@@ -845,10 +845,7 @@
return tls32_.is_gc_marking;
}
- void SetIsGcMarking(bool is_marking) {
- CHECK(kUseReadBarrier);
- tls32_.is_gc_marking = is_marking;
- }
+ void SetIsGcMarkingAndUpdateEntrypoints(bool is_marking);
bool GetWeakRefAccessEnabled() const {
CHECK(kUseReadBarrier);
diff --git a/runtime/thread_list.cc b/runtime/thread_list.cc
index eba6666..8a3bb15 100644
--- a/runtime/thread_list.cc
+++ b/runtime/thread_list.cc
@@ -1242,7 +1242,7 @@
// Initialize according to the state of the CC collector.
bool is_gc_marking =
Runtime::Current()->GetHeap()->ConcurrentCopyingCollector()->IsMarking();
- self->SetIsGcMarking(is_gc_marking);
+ self->SetIsGcMarkingAndUpdateEntrypoints(is_gc_marking);
bool weak_ref_access_enabled =
Runtime::Current()->GetHeap()->ConcurrentCopyingCollector()->IsWeakRefAccessEnabled();
self->SetWeakRefAccessEnabled(weak_ref_access_enabled);