Merge "Catch signals that happen inside a fault handler."
diff --git a/compiler/Android.bp b/compiler/Android.bp
index 1ee2a21..c59e36b 100644
--- a/compiler/Android.bp
+++ b/compiler/Android.bp
@@ -417,6 +417,7 @@
shared_libs: [
"libartd-compiler",
+ "libartd-simulator",
"libvixld-arm",
"libvixld-arm64",
diff --git a/compiler/compiled_method.h b/compiler/compiled_method.h
index e2a0942..00e2d62 100644
--- a/compiler/compiled_method.h
+++ b/compiler/compiled_method.h
@@ -170,7 +170,6 @@
// choose to squeeze the Type into fewer than 8 bits, we'll have to declare
// patch_type_ as an uintN_t and do explicit static_cast<>s.
enum class Type : uint8_t {
- kRecordPosition, // Just record patch position for patchoat.
kMethod,
kCall,
kCallRelative, // NOTE: Actual patching is instruction_set-dependent.
@@ -183,10 +182,6 @@
kDexCacheArray, // NOTE: Actual patching is instruction_set-dependent.
};
- static LinkerPatch RecordPosition(size_t literal_offset) {
- return LinkerPatch(literal_offset, Type::kRecordPosition, /* target_dex_file */ nullptr);
- }
-
static LinkerPatch MethodPatch(size_t literal_offset,
const DexFile* target_dex_file,
uint32_t target_method_idx) {
diff --git a/compiler/dex/dex_to_dex_decompiler.cc b/compiler/dex/dex_to_dex_decompiler.cc
index 5360103..85d5784 100644
--- a/compiler/dex/dex_to_dex_decompiler.cc
+++ b/compiler/dex/dex_to_dex_decompiler.cc
@@ -185,7 +185,7 @@
}
if (quickened_info_ptr_ != quickened_info_end_) {
- LOG(ERROR) << "Failed to use all values in quickening info."
+ LOG(FATAL) << "Failed to use all values in quickening info."
<< " Actual: " << std::hex << quickened_info_ptr_
<< " Expected: " << quickened_info_end_;
return false;
diff --git a/compiler/driver/compiler_driver.cc b/compiler/driver/compiler_driver.cc
index 057e3c9..9950987 100644
--- a/compiler/driver/compiler_driver.cc
+++ b/compiler/driver/compiler_driver.cc
@@ -480,7 +480,9 @@
DCHECK(!it.HasNext());
}
}
- DCHECK_EQ(quickening_info_ptr, quickening_info_end) << "Failed to use all quickening info";
+ if (quickening_info_ptr != quickening_info_end) {
+ LOG(FATAL) << "Failed to use all quickening info";
+ }
}
void CompilerDriver::CompileAll(jobject class_loader,
diff --git a/compiler/driver/compiler_driver.h b/compiler/driver/compiler_driver.h
index 1e5c43d..cbde587 100644
--- a/compiler/driver/compiler_driver.h
+++ b/compiler/driver/compiler_driver.h
@@ -355,6 +355,10 @@
return current_dex_to_dex_methods_;
}
+ const ProfileCompilationInfo* GetProfileCompilationInfo() const {
+ return profile_compilation_info_;
+ }
+
private:
// Can `referrer_class` access the resolved `member`?
// Dispatch call to mirror::Class::CanAccessResolvedField or
diff --git a/compiler/driver/compiler_options.cc b/compiler/driver/compiler_options.cc
index c222f90..34ad1c5 100644
--- a/compiler/driver/compiler_options.cc
+++ b/compiler/driver/compiler_options.cc
@@ -32,7 +32,6 @@
no_inline_from_(nullptr),
boot_image_(false),
app_image_(false),
- include_patch_information_(kDefaultIncludePatchInformation),
top_k_profile_threshold_(kDefaultTopKProfileThreshold),
debuggable_(false),
generate_debug_info_(kDefaultGenerateDebugInfo),
@@ -66,7 +65,6 @@
size_t inline_depth_limit,
size_t inline_max_code_units,
const std::vector<const DexFile*>* no_inline_from,
- bool include_patch_information,
double top_k_profile_threshold,
bool debuggable,
bool generate_debug_info,
@@ -93,7 +91,6 @@
no_inline_from_(no_inline_from),
boot_image_(false),
app_image_(false),
- include_patch_information_(include_patch_information),
top_k_profile_threshold_(top_k_profile_threshold),
debuggable_(debuggable),
generate_debug_info_(generate_debug_info),
@@ -206,10 +203,6 @@
debuggable_ = true;
} else if (option.starts_with("--top-k-profile-threshold=")) {
ParseDouble(option.data(), '=', 0.0, 100.0, &top_k_profile_threshold_, Usage);
- } else if (option == "--include-patch-information") {
- include_patch_information_ = true;
- } else if (option == "--no-include-patch-information") {
- include_patch_information_ = false;
} else if (option == "--abort-on-hard-verifier-error") {
abort_on_hard_verifier_failure_ = true;
} else if (option.starts_with("--dump-init-failures=")) {
diff --git a/compiler/driver/compiler_options.h b/compiler/driver/compiler_options.h
index 6894cd5..2e3e55f 100644
--- a/compiler/driver/compiler_options.h
+++ b/compiler/driver/compiler_options.h
@@ -46,7 +46,6 @@
static constexpr double kDefaultTopKProfileThreshold = 90.0;
static const bool kDefaultGenerateDebugInfo = false;
static const bool kDefaultGenerateMiniDebugInfo = false;
- static const bool kDefaultIncludePatchInformation = false;
static const size_t kDefaultInlineDepthLimit = 3;
static const size_t kDefaultInlineMaxCodeUnits = 32;
static constexpr size_t kUnsetInlineDepthLimit = -1;
@@ -68,7 +67,6 @@
size_t inline_depth_limit,
size_t inline_max_code_units,
const std::vector<const DexFile*>* no_inline_from,
- bool include_patch_information,
double top_k_profile_threshold,
bool debuggable,
bool generate_debug_info,
@@ -213,10 +211,6 @@
return implicit_suspend_checks_;
}
- bool GetIncludePatchInformation() const {
- return include_patch_information_;
- }
-
bool IsBootImage() const {
return boot_image_;
}
@@ -305,7 +299,6 @@
bool boot_image_;
bool app_image_;
- bool include_patch_information_;
// When using a profile file only the top K% of the profiled samples will be compiled.
double top_k_profile_threshold_;
bool debuggable_;
diff --git a/compiler/elf_writer.h b/compiler/elf_writer.h
index d55f745..7baae52 100644
--- a/compiler/elf_writer.h
+++ b/compiler/elf_writer.h
@@ -63,7 +63,6 @@
virtual void EndText(OutputStream* text) = 0;
virtual void WriteDynamicSection() = 0;
virtual void WriteDebugInfo(const ArrayRef<const debug::MethodDebugInfo>& method_infos) = 0;
- virtual void WritePatchLocations(const ArrayRef<const uintptr_t>& patch_locations) = 0;
virtual bool End() = 0;
// Get the ELF writer's stream. This stream can be used for writing data directly
diff --git a/compiler/elf_writer_quick.cc b/compiler/elf_writer_quick.cc
index 0d6575c..28c35e9 100644
--- a/compiler/elf_writer_quick.cc
+++ b/compiler/elf_writer_quick.cc
@@ -105,7 +105,6 @@
void EndText(OutputStream* text) OVERRIDE;
void WriteDynamicSection() OVERRIDE;
void WriteDebugInfo(const ArrayRef<const debug::MethodDebugInfo>& method_infos) OVERRIDE;
- void WritePatchLocations(const ArrayRef<const uintptr_t>& patch_locations) OVERRIDE;
bool End() OVERRIDE;
virtual OutputStream* GetStream() OVERRIDE;
@@ -268,17 +267,6 @@
}
template <typename ElfTypes>
-void ElfWriterQuick<ElfTypes>::WritePatchLocations(
- const ArrayRef<const uintptr_t>& patch_locations) {
- // Add relocation section for .text.
- if (compiler_options_->GetIncludePatchInformation()) {
- // Note that ElfWriter::Fixup will be called regardless and therefore
- // we need to include oat_patches for debug sections unconditionally.
- builder_->WritePatches(".text.oat_patches", patch_locations);
- }
-}
-
-template <typename ElfTypes>
bool ElfWriterQuick<ElfTypes>::End() {
builder_->End();
if (compiler_options_->GetGenerateBuildId()) {
diff --git a/compiler/image_test.cc b/compiler/image_test.cc
index b0225a3..89e8a67 100644
--- a/compiler/image_test.cc
+++ b/compiler/image_test.cc
@@ -318,7 +318,6 @@
elf_writer->WriteDynamicSection();
elf_writer->WriteDebugInfo(oat_writer->GetMethodDebugInfo());
- elf_writer->WritePatchLocations(oat_writer->GetAbsolutePatchLocations());
bool success = elf_writer->End();
ASSERT_TRUE(success);
diff --git a/compiler/jit/jit_compiler.cc b/compiler/jit/jit_compiler.cc
index cbd831a..3ae7974 100644
--- a/compiler/jit/jit_compiler.cc
+++ b/compiler/jit/jit_compiler.cc
@@ -100,7 +100,6 @@
CompilerOptions::kDefaultInlineDepthLimit,
CompilerOptions::kDefaultInlineMaxCodeUnits,
/* no_inline_from */ nullptr,
- /* include_patch_information */ false,
CompilerOptions::kDefaultTopKProfileThreshold,
Runtime::Current()->IsJavaDebuggable(),
CompilerOptions::kDefaultGenerateDebugInfo,
diff --git a/compiler/oat_test.cc b/compiler/oat_test.cc
index e2233e4..97b1374 100644
--- a/compiler/oat_test.cc
+++ b/compiler/oat_test.cc
@@ -250,7 +250,6 @@
elf_writer->WriteDynamicSection();
elf_writer->WriteDebugInfo(oat_writer.GetMethodDebugInfo());
- elf_writer->WritePatchLocations(oat_writer.GetAbsolutePatchLocations());
if (!elf_writer->End()) {
return false;
diff --git a/compiler/oat_writer.cc b/compiler/oat_writer.cc
index 8ab44d2..afcdf5e 100644
--- a/compiler/oat_writer.cc
+++ b/compiler/oat_writer.cc
@@ -1225,7 +1225,7 @@
break;
}
default: {
- DCHECK_EQ(patch.GetType(), LinkerPatch::Type::kRecordPosition);
+ DCHECK(false) << "Unexpected linker patch type: " << patch.GetType();
break;
}
}
@@ -2270,28 +2270,11 @@
/* verify */ true,
/* verify_checksum */ true,
&error_msg);
- } else if (oat_dex_file->source_.IsRawFile()) {
+ } else {
+ CHECK(oat_dex_file->source_.IsRawFile())
+ << static_cast<size_t>(oat_dex_file->source_.GetType());
File* raw_file = oat_dex_file->source_.GetRawFile();
dex_file = DexFile::OpenDex(raw_file->Fd(), location, /* verify_checksum */ true, &error_msg);
- } else {
- // The source data is a vdex file.
- CHECK(oat_dex_file->source_.IsRawData())
- << static_cast<size_t>(oat_dex_file->source_.GetType());
- const uint8_t* raw_dex_file = oat_dex_file->source_.GetRawData();
- // Note: The raw data has already been checked to contain the header
- // and all the data that the header specifies as the file size.
- DCHECK(raw_dex_file != nullptr);
- DCHECK(ValidateDexFileHeader(raw_dex_file, oat_dex_file->GetLocation()));
- const UnalignedDexFileHeader* header = AsUnalignedDexFileHeader(raw_dex_file);
- // Since the source may have had its layout changed, don't verify the checksum.
- dex_file = DexFile::Open(raw_dex_file,
- header->file_size_,
- location,
- oat_dex_file->dex_file_location_checksum_,
- nullptr,
- /* verify */ true,
- /* verify_checksum */ false,
- &error_msg);
}
if (dex_file == nullptr) {
LOG(ERROR) << "Failed to open dex file for layout: " << error_msg;
diff --git a/compiler/oat_writer.h b/compiler/oat_writer.h
index db84166..5113714 100644
--- a/compiler/oat_writer.h
+++ b/compiler/oat_writer.h
@@ -225,10 +225,6 @@
return oat_data_offset_;
}
- ArrayRef<const uintptr_t> GetAbsolutePatchLocations() const {
- return ArrayRef<const uintptr_t>(absolute_patch_locations_);
- }
-
~OatWriter();
void AddMethodDebugInfos(const std::vector<debug::MethodDebugInfo>& infos) {
diff --git a/compiler/optimizing/code_generator_arm.cc b/compiler/optimizing/code_generator_arm.cc
index 7b84ef8..bab626f 100644
--- a/compiler/optimizing/code_generator_arm.cc
+++ b/compiler/optimizing/code_generator_arm.cc
@@ -636,56 +636,25 @@
DISALLOW_COPY_AND_ASSIGN(ArraySetSlowPathARM);
};
-// Slow path marking an object reference `ref` during a read
-// barrier. The field `obj.field` in the object `obj` holding this
-// reference does not get updated by this slow path after marking (see
-// ReadBarrierMarkAndUpdateFieldSlowPathARM below for that).
+// Abstract base class for read barrier slow paths marking a reference
+// `ref`.
//
-// This means that after the execution of this slow path, `ref` will
-// always be up-to-date, but `obj.field` may not; i.e., after the
-// flip, `ref` will be a to-space reference, but `obj.field` will
-// probably still be a from-space reference (unless it gets updated by
-// another thread, or if another thread installed another object
-// reference (different from `ref`) in `obj.field`).
-//
-// If `entrypoint` is a valid location it is assumed to already be
-// holding the entrypoint. The case where the entrypoint is passed in
-// is for the GcRoot read barrier.
-class ReadBarrierMarkSlowPathARM : public SlowPathCodeARM {
- public:
- ReadBarrierMarkSlowPathARM(HInstruction* instruction,
- Location ref,
- Location entrypoint = Location::NoLocation())
+// Argument `entrypoint` must be a register location holding the read
+// barrier marking runtime entry point to be invoked.
+class ReadBarrierMarkSlowPathBaseARM : public SlowPathCodeARM {
+ protected:
+ ReadBarrierMarkSlowPathBaseARM(HInstruction* instruction, Location ref, Location entrypoint)
: SlowPathCodeARM(instruction), ref_(ref), entrypoint_(entrypoint) {
DCHECK(kEmitCompilerReadBarrier);
}
- const char* GetDescription() const OVERRIDE { return "ReadBarrierMarkSlowPathARM"; }
+ const char* GetDescription() const OVERRIDE { return "ReadBarrierMarkSlowPathBaseARM"; }
- void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
- LocationSummary* locations = instruction_->GetLocations();
+ // Generate assembly code calling the read barrier marking runtime
+ // entry point (ReadBarrierMarkRegX).
+ void GenerateReadBarrierMarkRuntimeCall(CodeGenerator* codegen) {
Register ref_reg = ref_.AsRegister<Register>();
- DCHECK(locations->CanCall());
- DCHECK(!locations->GetLiveRegisters()->ContainsCoreRegister(ref_reg)) << ref_reg;
- DCHECK(instruction_->IsInstanceFieldGet() ||
- instruction_->IsStaticFieldGet() ||
- instruction_->IsArrayGet() ||
- instruction_->IsArraySet() ||
- instruction_->IsLoadClass() ||
- instruction_->IsLoadString() ||
- instruction_->IsInstanceOf() ||
- instruction_->IsCheckCast() ||
- (instruction_->IsInvokeVirtual() && instruction_->GetLocations()->Intrinsified()) ||
- (instruction_->IsInvokeStaticOrDirect() && instruction_->GetLocations()->Intrinsified()))
- << "Unexpected instruction in read barrier marking slow path: "
- << instruction_->DebugName();
- // The read barrier instrumentation of object ArrayGet
- // instructions does not support the HIntermediateAddress
- // instruction.
- DCHECK(!(instruction_->IsArrayGet() &&
- instruction_->AsArrayGet()->GetArray()->IsIntermediateAddress()));
- __ Bind(GetEntryLabel());
// No need to save live registers; it's taken care of by the
// entrypoint. Also, there is no need to update the stack mask,
// as this runtime call will not trigger a garbage collection.
@@ -715,116 +684,331 @@
arm_codegen->ValidateInvokeRuntimeWithoutRecordingPcInfo(instruction_, this);
__ blx(entrypoint_.AsRegister<Register>());
} else {
+ // Entrypoint is not already loaded, load from the thread.
int32_t entry_point_offset =
CodeGenerator::GetReadBarrierMarkEntryPointsOffset<kArmPointerSize>(ref_reg);
// This runtime call does not require a stack map.
arm_codegen->InvokeRuntimeWithoutRecordingPcInfo(entry_point_offset, instruction_, this);
}
+ }
+
+ // The location (register) of the marked object reference.
+ const Location ref_;
+
+ // The location of the entrypoint if it is already loaded.
+ const Location entrypoint_;
+
+ private:
+ DISALLOW_COPY_AND_ASSIGN(ReadBarrierMarkSlowPathBaseARM);
+};
+
+// Slow path marking an object reference `ref` during a read
+// barrier. The field `obj.field` in the object `obj` holding this
+// reference does not get updated by this slow path after marking.
+//
+// This means that after the execution of this slow path, `ref` will
+// always be up-to-date, but `obj.field` may not; i.e., after the
+// flip, `ref` will be a to-space reference, but `obj.field` will
+// probably still be a from-space reference (unless it gets updated by
+// another thread, or if another thread installed another object
+// reference (different from `ref`) in `obj.field`).
+//
+// If `entrypoint` is a valid location it is assumed to already be
+// holding the entrypoint. The case where the entrypoint is passed in
+// is when the decision to mark is based on whether the GC is marking.
+class ReadBarrierMarkSlowPathARM : public ReadBarrierMarkSlowPathBaseARM {
+ public:
+ ReadBarrierMarkSlowPathARM(HInstruction* instruction,
+ Location ref,
+ Location entrypoint = Location::NoLocation())
+ : ReadBarrierMarkSlowPathBaseARM(instruction, ref, entrypoint) {
+ DCHECK(kEmitCompilerReadBarrier);
+ }
+
+ const char* GetDescription() const OVERRIDE { return "ReadBarrierMarkSlowPathARM"; }
+
+ void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
+ LocationSummary* locations = instruction_->GetLocations();
+ DCHECK(locations->CanCall());
+ if (kIsDebugBuild) {
+ Register ref_reg = ref_.AsRegister<Register>();
+ DCHECK(!locations->GetLiveRegisters()->ContainsCoreRegister(ref_reg)) << ref_reg;
+ }
+ DCHECK(instruction_->IsLoadClass() || instruction_->IsLoadString())
+ << "Unexpected instruction in read barrier marking slow path: "
+ << instruction_->DebugName();
+
+ __ Bind(GetEntryLabel());
+ GenerateReadBarrierMarkRuntimeCall(codegen);
__ b(GetExitLabel());
}
private:
- // The location (register) of the marked object reference.
- const Location ref_;
-
- // The location of the entrypoint if already loaded.
- const Location entrypoint_;
-
DISALLOW_COPY_AND_ASSIGN(ReadBarrierMarkSlowPathARM);
};
-// Slow path marking an object reference `ref` during a read barrier,
-// and if needed, atomically updating the field `obj.field` in the
-// object `obj` holding this reference after marking (contrary to
-// ReadBarrierMarkSlowPathARM above, which never tries to update
-// `obj.field`).
+// Slow path loading `obj`'s lock word, loading a reference from
+// object `*(obj + offset + (index << scale_factor))` into `ref`, and
+// marking `ref` if `obj` is gray according to the lock word (Baker
+// read barrier). The field `obj.field` in the object `obj` holding
+// this reference does not get updated by this slow path after marking
+// (see LoadReferenceWithBakerReadBarrierAndUpdateFieldSlowPathARM
+// below for that).
//
-// This means that after the execution of this slow path, both `ref`
-// and `obj.field` will be up-to-date; i.e., after the flip, both will
-// hold the same to-space reference (unless another thread installed
-// another object reference (different from `ref`) in `obj.field`).
-class ReadBarrierMarkAndUpdateFieldSlowPathARM : public SlowPathCodeARM {
+// This means that after the execution of this slow path, `ref` will
+// always be up-to-date, but `obj.field` may not; i.e., after the
+// flip, `ref` will be a to-space reference, but `obj.field` will
+// probably still be a from-space reference (unless it gets updated by
+// another thread, or if another thread installed another object
+// reference (different from `ref`) in `obj.field`).
+//
+// Argument `entrypoint` must be a register location holding the read
+// barrier marking runtime entry point to be invoked.
+class LoadReferenceWithBakerReadBarrierSlowPathARM : public ReadBarrierMarkSlowPathBaseARM {
public:
- ReadBarrierMarkAndUpdateFieldSlowPathARM(HInstruction* instruction,
- Location ref,
- Register obj,
- Location field_offset,
- Register temp1,
- Register temp2)
- : SlowPathCodeARM(instruction),
- ref_(ref),
+ LoadReferenceWithBakerReadBarrierSlowPathARM(HInstruction* instruction,
+ Location ref,
+ Register obj,
+ uint32_t offset,
+ Location index,
+ ScaleFactor scale_factor,
+ bool needs_null_check,
+ Register temp,
+ Location entrypoint)
+ : ReadBarrierMarkSlowPathBaseARM(instruction, ref, entrypoint),
obj_(obj),
- field_offset_(field_offset),
- temp1_(temp1),
- temp2_(temp2) {
+ offset_(offset),
+ index_(index),
+ scale_factor_(scale_factor),
+ needs_null_check_(needs_null_check),
+ temp_(temp) {
DCHECK(kEmitCompilerReadBarrier);
+ DCHECK(kUseBakerReadBarrier);
}
- const char* GetDescription() const OVERRIDE { return "ReadBarrierMarkAndUpdateFieldSlowPathARM"; }
+ const char* GetDescription() const OVERRIDE {
+ return "LoadReferenceWithBakerReadBarrierSlowPathARM";
+ }
void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
LocationSummary* locations = instruction_->GetLocations();
Register ref_reg = ref_.AsRegister<Register>();
DCHECK(locations->CanCall());
DCHECK(!locations->GetLiveRegisters()->ContainsCoreRegister(ref_reg)) << ref_reg;
- // This slow path is only used by the UnsafeCASObject intrinsic.
+ DCHECK_NE(ref_reg, temp_);
+ DCHECK(instruction_->IsInstanceFieldGet() ||
+ instruction_->IsStaticFieldGet() ||
+ instruction_->IsArrayGet() ||
+ instruction_->IsArraySet() ||
+ instruction_->IsInstanceOf() ||
+ instruction_->IsCheckCast() ||
+ (instruction_->IsInvokeVirtual() && instruction_->GetLocations()->Intrinsified()) ||
+ (instruction_->IsInvokeStaticOrDirect() && instruction_->GetLocations()->Intrinsified()))
+ << "Unexpected instruction in read barrier marking slow path: "
+ << instruction_->DebugName();
+ // The read barrier instrumentation of object ArrayGet
+ // instructions does not support the HIntermediateAddress
+ // instruction.
+ DCHECK(!(instruction_->IsArrayGet() &&
+ instruction_->AsArrayGet()->GetArray()->IsIntermediateAddress()));
+
+ __ Bind(GetEntryLabel());
+
+ // When using MaybeGenerateReadBarrierSlow, the read barrier call is
+ // inserted after the original load. However, in fast path based
+ // Baker's read barriers, we need to perform the load of
+ // mirror::Object::monitor_ *before* the original reference load.
+ // This load-load ordering is required by the read barrier.
+ // The fast path/slow path (for Baker's algorithm) should look like:
+ //
+ // uint32_t rb_state = Lockword(obj->monitor_).ReadBarrierState();
+ // lfence; // Load fence or artificial data dependency to prevent load-load reordering
+ // HeapReference<mirror::Object> ref = *src; // Original reference load.
+ // bool is_gray = (rb_state == ReadBarrier::GrayState());
+ // if (is_gray) {
+ // ref = entrypoint(ref); // ref = ReadBarrier::Mark(ref); // Runtime entry point call.
+ // }
+ //
+ // Note: the original implementation in ReadBarrier::Barrier is
+ // slightly more complex as it performs additional checks that we do
+ // not do here for performance reasons.
+
+ // /* int32_t */ monitor = obj->monitor_
+ uint32_t monitor_offset = mirror::Object::MonitorOffset().Int32Value();
+ __ LoadFromOffset(kLoadWord, temp_, obj_, monitor_offset);
+ if (needs_null_check_) {
+ codegen->MaybeRecordImplicitNullCheck(instruction_);
+ }
+ // /* LockWord */ lock_word = LockWord(monitor)
+ static_assert(sizeof(LockWord) == sizeof(int32_t),
+ "art::LockWord and int32_t have different sizes.");
+
+ // Introduce a dependency on the lock_word including the rb_state,
+ // which shall prevent load-load reordering without using
+ // a memory barrier (which would be more expensive).
+ // `obj` is unchanged by this operation, but its value now depends
+ // on `temp`.
+ __ add(obj_, obj_, ShifterOperand(temp_, LSR, 32));
+
+ // The actual reference load.
+ // A possible implicit null check has already been handled above.
+ CodeGeneratorARM* arm_codegen = down_cast<CodeGeneratorARM*>(codegen);
+ arm_codegen->GenerateRawReferenceLoad(
+ instruction_, ref_, obj_, offset_, index_, scale_factor_, /* needs_null_check */ false);
+
+ // Mark the object `ref` when `obj` is gray.
+ //
+ // if (rb_state == ReadBarrier::GrayState())
+ // ref = ReadBarrier::Mark(ref);
+ //
+ // Given the numeric representation, it's enough to check the low bit of the
+ // rb_state. We do that by shifting the bit out of the lock word with LSRS
+ // which can be a 16-bit instruction unlike the TST immediate.
+ static_assert(ReadBarrier::WhiteState() == 0, "Expecting white to have value 0");
+ static_assert(ReadBarrier::GrayState() == 1, "Expecting gray to have value 1");
+ __ Lsrs(temp_, temp_, LockWord::kReadBarrierStateShift + 1);
+ __ b(GetExitLabel(), CC); // Carry flag is the last bit shifted out by LSRS.
+ GenerateReadBarrierMarkRuntimeCall(codegen);
+
+ __ b(GetExitLabel());
+ }
+
+ private:
+ // The register containing the object holding the marked object reference field.
+ Register obj_;
+ // The offset, index and scale factor to access the reference in `obj_`.
+ uint32_t offset_;
+ Location index_;
+ ScaleFactor scale_factor_;
+ // Is a null check required?
+ bool needs_null_check_;
+ // A temporary register used to hold the lock word of `obj_`.
+ Register temp_;
+
+ DISALLOW_COPY_AND_ASSIGN(LoadReferenceWithBakerReadBarrierSlowPathARM);
+};
+
+// Slow path loading `obj`'s lock word, loading a reference from
+// object `*(obj + offset + (index << scale_factor))` into `ref`, and
+// marking `ref` if `obj` is gray according to the lock word (Baker
+// read barrier). If needed, this slow path also atomically updates
+// the field `obj.field` in the object `obj` holding this reference
+// after marking (contrary to
+// LoadReferenceWithBakerReadBarrierSlowPathARM above, which never
+// tries to update `obj.field`).
+//
+// This means that after the execution of this slow path, both `ref`
+// and `obj.field` will be up-to-date; i.e., after the flip, both will
+// hold the same to-space reference (unless another thread installed
+// another object reference (different from `ref`) in `obj.field`).
+//
+// Argument `entrypoint` must be a register location holding the read
+// barrier marking runtime entry point to be invoked.
+class LoadReferenceWithBakerReadBarrierAndUpdateFieldSlowPathARM
+ : public ReadBarrierMarkSlowPathBaseARM {
+ public:
+ LoadReferenceWithBakerReadBarrierAndUpdateFieldSlowPathARM(HInstruction* instruction,
+ Location ref,
+ Register obj,
+ uint32_t offset,
+ Location index,
+ ScaleFactor scale_factor,
+ bool needs_null_check,
+ Register temp1,
+ Register temp2,
+ Location entrypoint)
+ : ReadBarrierMarkSlowPathBaseARM(instruction, ref, entrypoint),
+ obj_(obj),
+ offset_(offset),
+ index_(index),
+ scale_factor_(scale_factor),
+ needs_null_check_(needs_null_check),
+ temp1_(temp1),
+ temp2_(temp2) {
+ DCHECK(kEmitCompilerReadBarrier);
+ DCHECK(kUseBakerReadBarrier);
+ }
+
+ const char* GetDescription() const OVERRIDE {
+ return "LoadReferenceWithBakerReadBarrierAndUpdateFieldSlowPathARM";
+ }
+
+ void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
+ LocationSummary* locations = instruction_->GetLocations();
+ Register ref_reg = ref_.AsRegister<Register>();
+ DCHECK(locations->CanCall());
+ DCHECK(!locations->GetLiveRegisters()->ContainsCoreRegister(ref_reg)) << ref_reg;
+ DCHECK_NE(ref_reg, temp1_);
+
+ // This slow path is only used by the UnsafeCASObject intrinsic at the moment.
DCHECK((instruction_->IsInvokeVirtual() && instruction_->GetLocations()->Intrinsified()))
<< "Unexpected instruction in read barrier marking and field updating slow path: "
<< instruction_->DebugName();
DCHECK(instruction_->GetLocations()->Intrinsified());
DCHECK_EQ(instruction_->AsInvoke()->GetIntrinsic(), Intrinsics::kUnsafeCASObject);
- DCHECK(field_offset_.IsRegisterPair()) << field_offset_;
+ DCHECK_EQ(offset_, 0u);
+ DCHECK_EQ(scale_factor_, ScaleFactor::TIMES_1);
+ // The location of the offset of the marked reference field within `obj_`.
+ Location field_offset = index_;
+ DCHECK(field_offset.IsRegisterPair()) << field_offset;
__ Bind(GetEntryLabel());
- // Save the old reference.
+ // /* int32_t */ monitor = obj->monitor_
+ uint32_t monitor_offset = mirror::Object::MonitorOffset().Int32Value();
+ __ LoadFromOffset(kLoadWord, temp1_, obj_, monitor_offset);
+ if (needs_null_check_) {
+ codegen->MaybeRecordImplicitNullCheck(instruction_);
+ }
+ // /* LockWord */ lock_word = LockWord(monitor)
+ static_assert(sizeof(LockWord) == sizeof(int32_t),
+ "art::LockWord and int32_t have different sizes.");
+
+ // Introduce a dependency on the lock_word including the rb_state,
+ // which shall prevent load-load reordering without using
+ // a memory barrier (which would be more expensive).
+ // `obj` is unchanged by this operation, but its value now depends
+ // on `temp1`.
+ __ add(obj_, obj_, ShifterOperand(temp1_, LSR, 32));
+
+ // The actual reference load.
+ // A possible implicit null check has already been handled above.
+ CodeGeneratorARM* arm_codegen = down_cast<CodeGeneratorARM*>(codegen);
+ arm_codegen->GenerateRawReferenceLoad(
+ instruction_, ref_, obj_, offset_, index_, scale_factor_, /* needs_null_check */ false);
+
+ // Mark the object `ref` when `obj` is gray.
+ //
+ // if (rb_state == ReadBarrier::GrayState())
+ // ref = ReadBarrier::Mark(ref);
+ //
+ // Given the numeric representation, it's enough to check the low bit of the
+ // rb_state. We do that by shifting the bit out of the lock word with LSRS
+ // which can be a 16-bit instruction unlike the TST immediate.
+ static_assert(ReadBarrier::WhiteState() == 0, "Expecting white to have value 0");
+ static_assert(ReadBarrier::GrayState() == 1, "Expecting gray to have value 1");
+ __ Lsrs(temp1_, temp1_, LockWord::kReadBarrierStateShift + 1);
+ __ b(GetExitLabel(), CC); // Carry flag is the last bit shifted out by LSRS.
+
+ // Save the old value of the reference before marking it.
// Note that we cannot use IP to save the old reference, as IP is
// used internally by the ReadBarrierMarkRegX entry point, and we
// need the old reference after the call to that entry point.
DCHECK_NE(temp1_, IP);
__ Mov(temp1_, ref_reg);
- // No need to save live registers; it's taken care of by the
- // entrypoint. Also, there is no need to update the stack mask,
- // as this runtime call will not trigger a garbage collection.
- CodeGeneratorARM* arm_codegen = down_cast<CodeGeneratorARM*>(codegen);
- DCHECK_NE(ref_reg, SP);
- DCHECK_NE(ref_reg, LR);
- DCHECK_NE(ref_reg, PC);
- // IP is used internally by the ReadBarrierMarkRegX entry point
- // as a temporary, it cannot be the entry point's input/output.
- DCHECK_NE(ref_reg, IP);
- DCHECK(0 <= ref_reg && ref_reg < kNumberOfCoreRegisters) << ref_reg;
- // "Compact" slow path, saving two moves.
- //
- // Instead of using the standard runtime calling convention (input
- // and output in R0):
- //
- // R0 <- ref
- // R0 <- ReadBarrierMark(R0)
- // ref <- R0
- //
- // we just use rX (the register containing `ref`) as input and output
- // of a dedicated entrypoint:
- //
- // rX <- ReadBarrierMarkRegX(rX)
- //
- int32_t entry_point_offset =
- CodeGenerator::GetReadBarrierMarkEntryPointsOffset<kArmPointerSize>(ref_reg);
- // This runtime call does not require a stack map.
- arm_codegen->InvokeRuntimeWithoutRecordingPcInfo(entry_point_offset, instruction_, this);
+ GenerateReadBarrierMarkRuntimeCall(codegen);
// If the new reference is different from the old reference,
- // update the field in the holder (`*(obj_ + field_offset_)`).
+ // update the field in the holder (`*(obj_ + field_offset)`).
//
// Note that this field could also hold a different object, if
// another thread had concurrently changed it. In that case, the
// LDREX/SUBS/ITNE sequence of instructions in the compare-and-set
// (CAS) operation below would abort the CAS, leaving the field
// as-is.
- Label done;
__ cmp(temp1_, ShifterOperand(ref_reg));
- __ b(&done, EQ);
+ __ b(GetExitLabel(), EQ);
// Update the the holder's field atomically. This may fail if
// mutator updates before us, but it's OK. This is achieved
@@ -837,7 +1021,7 @@
// The UnsafeCASObject intrinsic uses a register pair as field
// offset ("long offset"), of which only the low part contains
// data.
- Register offset = field_offset_.AsRegisterPairLow<Register>();
+ Register offset = field_offset.AsRegisterPairLow<Register>();
Register expected = temp1_;
Register value = ref_reg;
Register tmp_ptr = IP; // Pointer to actual memory.
@@ -887,22 +1071,27 @@
}
}
- __ Bind(&done);
__ b(GetExitLabel());
}
private:
- // The location (register) of the marked object reference.
- const Location ref_;
// The register containing the object holding the marked object reference field.
const Register obj_;
- // The location of the offset of the marked reference field within `obj_`.
- Location field_offset_;
-
+ // The offset, index and scale factor to access the reference in `obj_`.
+ uint32_t offset_;
+ Location index_;
+ ScaleFactor scale_factor_;
+ // Is a null check required?
+ bool needs_null_check_;
+ // A temporary register used to hold the lock word of `obj_`; and
+ // also to hold the original reference value, when the reference is
+ // marked.
const Register temp1_;
+ // A temporary register used in the implementation of the CAS, to
+ // update the object's reference field.
const Register temp2_;
- DISALLOW_COPY_AND_ASSIGN(ReadBarrierMarkAndUpdateFieldSlowPathARM);
+ DISALLOW_COPY_AND_ASSIGN(LoadReferenceWithBakerReadBarrierAndUpdateFieldSlowPathARM);
};
// Slow path generating a read barrier for a heap reference.
@@ -1375,10 +1564,332 @@
}
}
+static void GenerateVcmp(HInstruction* instruction, CodeGeneratorARM* codegen) {
+ Primitive::Type type = instruction->InputAt(0)->GetType();
+ Location lhs_loc = instruction->GetLocations()->InAt(0);
+ Location rhs_loc = instruction->GetLocations()->InAt(1);
+ if (rhs_loc.IsConstant()) {
+ // 0.0 is the only immediate that can be encoded directly in
+ // a VCMP instruction.
+ //
+ // Both the JLS (section 15.20.1) and the JVMS (section 6.5)
+ // specify that in a floating-point comparison, positive zero
+ // and negative zero are considered equal, so we can use the
+ // literal 0.0 for both cases here.
+ //
+ // Note however that some methods (Float.equal, Float.compare,
+ // Float.compareTo, Double.equal, Double.compare,
+ // Double.compareTo, Math.max, Math.min, StrictMath.max,
+ // StrictMath.min) consider 0.0 to be (strictly) greater than
+ // -0.0. So if we ever translate calls to these methods into a
+ // HCompare instruction, we must handle the -0.0 case with
+ // care here.
+ DCHECK(rhs_loc.GetConstant()->IsArithmeticZero());
+ if (type == Primitive::kPrimFloat) {
+ __ vcmpsz(lhs_loc.AsFpuRegister<SRegister>());
+ } else {
+ DCHECK_EQ(type, Primitive::kPrimDouble);
+ __ vcmpdz(FromLowSToD(lhs_loc.AsFpuRegisterPairLow<SRegister>()));
+ }
+ } else {
+ if (type == Primitive::kPrimFloat) {
+ __ vcmps(lhs_loc.AsFpuRegister<SRegister>(), rhs_loc.AsFpuRegister<SRegister>());
+ } else {
+ DCHECK_EQ(type, Primitive::kPrimDouble);
+ __ vcmpd(FromLowSToD(lhs_loc.AsFpuRegisterPairLow<SRegister>()),
+ FromLowSToD(rhs_loc.AsFpuRegisterPairLow<SRegister>()));
+ }
+ }
+}
+
+static Condition GenerateLongTestConstant(HCondition* condition,
+ bool invert,
+ CodeGeneratorARM* codegen) {
+ DCHECK_EQ(condition->GetLeft()->GetType(), Primitive::kPrimLong);
+
+ const LocationSummary* const locations = condition->GetLocations();
+ IfCondition cond = invert ? condition->GetOppositeCondition() : condition->GetCondition();
+ Condition ret = EQ;
+ const Location left = locations->InAt(0);
+ const Location right = locations->InAt(1);
+
+ DCHECK(right.IsConstant());
+
+ const Register left_high = left.AsRegisterPairHigh<Register>();
+ const Register left_low = left.AsRegisterPairLow<Register>();
+ int64_t value = right.GetConstant()->AsLongConstant()->GetValue();
+
+ switch (cond) {
+ case kCondEQ:
+ case kCondNE:
+ case kCondB:
+ case kCondBE:
+ case kCondA:
+ case kCondAE:
+ __ CmpConstant(left_high, High32Bits(value));
+ __ it(EQ);
+ __ cmp(left_low, ShifterOperand(Low32Bits(value)), EQ);
+ ret = ARMUnsignedCondition(cond);
+ break;
+ case kCondLE:
+ case kCondGT:
+ // Trivially true or false.
+ if (value == std::numeric_limits<int64_t>::max()) {
+ __ cmp(left_low, ShifterOperand(left_low));
+ ret = cond == kCondLE ? EQ : NE;
+ break;
+ }
+
+ if (cond == kCondLE) {
+ cond = kCondLT;
+ } else {
+ DCHECK_EQ(cond, kCondGT);
+ cond = kCondGE;
+ }
+
+ value++;
+ FALLTHROUGH_INTENDED;
+ case kCondGE:
+ case kCondLT:
+ __ CmpConstant(left_low, Low32Bits(value));
+ __ sbcs(IP, left_high, ShifterOperand(High32Bits(value)));
+ ret = ARMCondition(cond);
+ break;
+ default:
+ LOG(FATAL) << "Unreachable";
+ UNREACHABLE();
+ }
+
+ return ret;
+}
+
+static Condition GenerateLongTest(HCondition* condition,
+ bool invert,
+ CodeGeneratorARM* codegen) {
+ DCHECK_EQ(condition->GetLeft()->GetType(), Primitive::kPrimLong);
+
+ const LocationSummary* const locations = condition->GetLocations();
+ IfCondition cond = invert ? condition->GetOppositeCondition() : condition->GetCondition();
+ Condition ret = EQ;
+ Location left = locations->InAt(0);
+ Location right = locations->InAt(1);
+
+ DCHECK(right.IsRegisterPair());
+
+ switch (cond) {
+ case kCondEQ:
+ case kCondNE:
+ case kCondB:
+ case kCondBE:
+ case kCondA:
+ case kCondAE:
+ __ cmp(left.AsRegisterPairHigh<Register>(),
+ ShifterOperand(right.AsRegisterPairHigh<Register>()));
+ __ it(EQ);
+ __ cmp(left.AsRegisterPairLow<Register>(),
+ ShifterOperand(right.AsRegisterPairLow<Register>()),
+ EQ);
+ ret = ARMUnsignedCondition(cond);
+ break;
+ case kCondLE:
+ case kCondGT:
+ if (cond == kCondLE) {
+ cond = kCondGE;
+ } else {
+ DCHECK_EQ(cond, kCondGT);
+ cond = kCondLT;
+ }
+
+ std::swap(left, right);
+ FALLTHROUGH_INTENDED;
+ case kCondGE:
+ case kCondLT:
+ __ cmp(left.AsRegisterPairLow<Register>(),
+ ShifterOperand(right.AsRegisterPairLow<Register>()));
+ __ sbcs(IP,
+ left.AsRegisterPairHigh<Register>(),
+ ShifterOperand(right.AsRegisterPairHigh<Register>()));
+ ret = ARMCondition(cond);
+ break;
+ default:
+ LOG(FATAL) << "Unreachable";
+ UNREACHABLE();
+ }
+
+ return ret;
+}
+
+static Condition GenerateTest(HInstruction* instruction,
+ Location loc,
+ bool invert,
+ CodeGeneratorARM* codegen) {
+ DCHECK(!instruction->IsConstant());
+
+ Condition ret = invert ? EQ : NE;
+
+ if (IsBooleanValueOrMaterializedCondition(instruction)) {
+ __ CmpConstant(loc.AsRegister<Register>(), 0);
+ } else {
+ HCondition* const condition = instruction->AsCondition();
+ const LocationSummary* const locations = condition->GetLocations();
+ const Primitive::Type type = condition->GetLeft()->GetType();
+ const IfCondition cond = invert ? condition->GetOppositeCondition() : condition->GetCondition();
+ const Location right = locations->InAt(1);
+
+ if (type == Primitive::kPrimLong) {
+ ret = condition->GetLocations()->InAt(1).IsConstant()
+ ? GenerateLongTestConstant(condition, invert, codegen)
+ : GenerateLongTest(condition, invert, codegen);
+ } else if (Primitive::IsFloatingPointType(type)) {
+ GenerateVcmp(condition, codegen);
+ __ vmstat();
+ ret = ARMFPCondition(cond, condition->IsGtBias());
+ } else {
+ DCHECK(Primitive::IsIntegralType(type) || type == Primitive::kPrimNot) << type;
+
+ const Register left = locations->InAt(0).AsRegister<Register>();
+
+ if (right.IsRegister()) {
+ __ cmp(left, ShifterOperand(right.AsRegister<Register>()));
+ } else {
+ DCHECK(right.IsConstant());
+ __ CmpConstant(left, CodeGenerator::GetInt32ValueOf(right.GetConstant()));
+ }
+
+ ret = ARMCondition(cond);
+ }
+ }
+
+ return ret;
+}
+
+static bool CanGenerateTest(HInstruction* condition, ArmAssembler* assembler) {
+ if (!IsBooleanValueOrMaterializedCondition(condition)) {
+ const HCondition* const cond = condition->AsCondition();
+
+ if (cond->GetLeft()->GetType() == Primitive::kPrimLong) {
+ const LocationSummary* const locations = cond->GetLocations();
+ const IfCondition c = cond->GetCondition();
+
+ if (locations->InAt(1).IsConstant()) {
+ const int64_t value = locations->InAt(1).GetConstant()->AsLongConstant()->GetValue();
+ ShifterOperand so;
+
+ if (c < kCondLT || c > kCondGE) {
+ // Since IT blocks longer than a 16-bit instruction are deprecated by ARMv8,
+ // we check that the least significant half of the first input to be compared
+ // is in a low register (the other half is read outside an IT block), and
+ // the constant fits in an 8-bit unsigned integer, so that a 16-bit CMP
+ // encoding can be used.
+ if (!ArmAssembler::IsLowRegister(locations->InAt(0).AsRegisterPairLow<Register>()) ||
+ !IsUint<8>(Low32Bits(value))) {
+ return false;
+ }
+ } else if (c == kCondLE || c == kCondGT) {
+ if (value < std::numeric_limits<int64_t>::max() &&
+ !assembler->ShifterOperandCanHold(kNoRegister,
+ kNoRegister,
+ SBC,
+ High32Bits(value + 1),
+ kCcSet,
+ &so)) {
+ return false;
+ }
+ } else if (!assembler->ShifterOperandCanHold(kNoRegister,
+ kNoRegister,
+ SBC,
+ High32Bits(value),
+ kCcSet,
+ &so)) {
+ return false;
+ }
+ }
+ }
+ }
+
+ return true;
+}
+
+static bool CanEncodeConstantAs8BitImmediate(HConstant* constant) {
+ const Primitive::Type type = constant->GetType();
+ bool ret = false;
+
+ DCHECK(Primitive::IsIntegralType(type) || type == Primitive::kPrimNot) << type;
+
+ if (type == Primitive::kPrimLong) {
+ const uint64_t value = constant->AsLongConstant()->GetValueAsUint64();
+
+ ret = IsUint<8>(Low32Bits(value)) && IsUint<8>(High32Bits(value));
+ } else {
+ ret = IsUint<8>(CodeGenerator::GetInt32ValueOf(constant));
+ }
+
+ return ret;
+}
+
+static Location Arm8BitEncodableConstantOrRegister(HInstruction* constant) {
+ DCHECK(!Primitive::IsFloatingPointType(constant->GetType()));
+
+ if (constant->IsConstant() && CanEncodeConstantAs8BitImmediate(constant->AsConstant())) {
+ return Location::ConstantLocation(constant->AsConstant());
+ }
+
+ return Location::RequiresRegister();
+}
+
+static bool CanGenerateConditionalMove(const Location& out, const Location& src) {
+ // Since IT blocks longer than a 16-bit instruction are deprecated by ARMv8,
+ // we check that we are not dealing with floating-point output (there is no
+ // 16-bit VMOV encoding).
+ if (!out.IsRegister() && !out.IsRegisterPair()) {
+ return false;
+ }
+
+ // For constants, we also check that the output is in one or two low registers,
+ // and that the constants fit in an 8-bit unsigned integer, so that a 16-bit
+ // MOV encoding can be used.
+ if (src.IsConstant()) {
+ if (!CanEncodeConstantAs8BitImmediate(src.GetConstant())) {
+ return false;
+ }
+
+ if (out.IsRegister()) {
+ if (!ArmAssembler::IsLowRegister(out.AsRegister<Register>())) {
+ return false;
+ }
+ } else {
+ DCHECK(out.IsRegisterPair());
+
+ if (!ArmAssembler::IsLowRegister(out.AsRegisterPairHigh<Register>())) {
+ return false;
+ }
+ }
+ }
+
+ return true;
+}
+
#undef __
// NOLINT on __ macro to suppress wrong warning/fix (misc-macro-parentheses) from clang-tidy.
#define __ down_cast<ArmAssembler*>(GetAssembler())-> // NOLINT
+Label* CodeGeneratorARM::GetFinalLabel(HInstruction* instruction, Label* final_label) {
+ DCHECK(!instruction->IsControlFlow() && !instruction->IsSuspendCheck());
+
+ const HBasicBlock* const block = instruction->GetBlock();
+ const HLoopInformation* const info = block->GetLoopInformation();
+ HInstruction* const next = instruction->GetNext();
+
+ // Avoid a branch to a branch.
+ if (next->IsGoto() && (info == nullptr ||
+ !info->IsBackEdge(*block) ||
+ !info->HasSuspendCheck())) {
+ final_label = GetLabelOf(next->AsGoto()->GetSuccessor());
+ }
+
+ return final_label;
+}
+
void CodeGeneratorARM::DumpCoreRegister(std::ostream& stream, int reg) const {
stream << Register(reg);
}
@@ -1437,8 +1948,6 @@
graph->GetArena()->Adapter(kArenaAllocCodeGenerator)),
pc_relative_type_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)),
type_bss_entry_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)),
- boot_image_address_patches_(std::less<uint32_t>(),
- graph->GetArena()->Adapter(kArenaAllocCodeGenerator)),
jit_string_patches_(StringReferenceValueComparator(),
graph->GetArena()->Adapter(kArenaAllocCodeGenerator)),
jit_class_patches_(TypeReferenceValueComparator(),
@@ -1905,44 +2414,6 @@
void InstructionCodeGeneratorARM::VisitExit(HExit* exit ATTRIBUTE_UNUSED) {
}
-void InstructionCodeGeneratorARM::GenerateVcmp(HInstruction* instruction) {
- Primitive::Type type = instruction->InputAt(0)->GetType();
- Location lhs_loc = instruction->GetLocations()->InAt(0);
- Location rhs_loc = instruction->GetLocations()->InAt(1);
- if (rhs_loc.IsConstant()) {
- // 0.0 is the only immediate that can be encoded directly in
- // a VCMP instruction.
- //
- // Both the JLS (section 15.20.1) and the JVMS (section 6.5)
- // specify that in a floating-point comparison, positive zero
- // and negative zero are considered equal, so we can use the
- // literal 0.0 for both cases here.
- //
- // Note however that some methods (Float.equal, Float.compare,
- // Float.compareTo, Double.equal, Double.compare,
- // Double.compareTo, Math.max, Math.min, StrictMath.max,
- // StrictMath.min) consider 0.0 to be (strictly) greater than
- // -0.0. So if we ever translate calls to these methods into a
- // HCompare instruction, we must handle the -0.0 case with
- // care here.
- DCHECK(rhs_loc.GetConstant()->IsArithmeticZero());
- if (type == Primitive::kPrimFloat) {
- __ vcmpsz(lhs_loc.AsFpuRegister<SRegister>());
- } else {
- DCHECK_EQ(type, Primitive::kPrimDouble);
- __ vcmpdz(FromLowSToD(lhs_loc.AsFpuRegisterPairLow<SRegister>()));
- }
- } else {
- if (type == Primitive::kPrimFloat) {
- __ vcmps(lhs_loc.AsFpuRegister<SRegister>(), rhs_loc.AsFpuRegister<SRegister>());
- } else {
- DCHECK_EQ(type, Primitive::kPrimDouble);
- __ vcmpd(FromLowSToD(lhs_loc.AsFpuRegisterPairLow<SRegister>()),
- FromLowSToD(rhs_loc.AsFpuRegisterPairLow<SRegister>()));
- }
- }
-}
-
void InstructionCodeGeneratorARM::GenerateFPJumps(HCondition* cond,
Label* true_label,
Label* false_label ATTRIBUTE_UNUSED) {
@@ -2050,7 +2521,7 @@
break;
case Primitive::kPrimFloat:
case Primitive::kPrimDouble:
- GenerateVcmp(condition);
+ GenerateVcmp(condition, codegen_);
GenerateFPJumps(condition, true_target, false_target);
break;
default:
@@ -2120,20 +2591,38 @@
return;
}
+ Label* non_fallthrough_target;
+ Condition arm_cond;
LocationSummary* locations = cond->GetLocations();
DCHECK(locations->InAt(0).IsRegister());
Register left = locations->InAt(0).AsRegister<Register>();
Location right = locations->InAt(1);
- if (right.IsRegister()) {
- __ cmp(left, ShifterOperand(right.AsRegister<Register>()));
- } else {
- DCHECK(right.IsConstant());
- __ CmpConstant(left, CodeGenerator::GetInt32ValueOf(right.GetConstant()));
- }
+
if (true_target == nullptr) {
- __ b(false_target, ARMCondition(condition->GetOppositeCondition()));
+ arm_cond = ARMCondition(condition->GetOppositeCondition());
+ non_fallthrough_target = false_target;
} else {
- __ b(true_target, ARMCondition(condition->GetCondition()));
+ arm_cond = ARMCondition(condition->GetCondition());
+ non_fallthrough_target = true_target;
+ }
+
+ if (right.IsConstant() && (arm_cond == NE || arm_cond == EQ) &&
+ CodeGenerator::GetInt32ValueOf(right.GetConstant()) == 0) {
+ if (arm_cond == EQ) {
+ __ CompareAndBranchIfZero(left, non_fallthrough_target);
+ } else {
+ DCHECK_EQ(arm_cond, NE);
+ __ CompareAndBranchIfNonZero(left, non_fallthrough_target);
+ }
+ } else {
+ if (right.IsRegister()) {
+ __ cmp(left, ShifterOperand(right.AsRegister<Register>()));
+ } else {
+ DCHECK(right.IsConstant());
+ __ CmpConstant(left, CodeGenerator::GetInt32ValueOf(right.GetConstant()));
+ }
+
+ __ b(non_fallthrough_target, arm_cond);
}
}
@@ -2193,28 +2682,140 @@
void LocationsBuilderARM::VisitSelect(HSelect* select) {
LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(select);
- if (Primitive::IsFloatingPointType(select->GetType())) {
+ const bool is_floating_point = Primitive::IsFloatingPointType(select->GetType());
+
+ if (is_floating_point) {
locations->SetInAt(0, Location::RequiresFpuRegister());
- locations->SetInAt(1, Location::RequiresFpuRegister());
+ locations->SetInAt(1, Location::FpuRegisterOrConstant(select->GetTrueValue()));
} else {
locations->SetInAt(0, Location::RequiresRegister());
- locations->SetInAt(1, Location::RequiresRegister());
+ locations->SetInAt(1, Arm8BitEncodableConstantOrRegister(select->GetTrueValue()));
}
+
if (IsBooleanValueOrMaterializedCondition(select->GetCondition())) {
- locations->SetInAt(2, Location::RequiresRegister());
+ locations->SetInAt(2, Location::RegisterOrConstant(select->GetCondition()));
+ // The code generator handles overlap with the values, but not with the condition.
+ locations->SetOut(Location::SameAsFirstInput());
+ } else if (is_floating_point) {
+ locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap);
+ } else {
+ if (!locations->InAt(1).IsConstant()) {
+ locations->SetInAt(0, Arm8BitEncodableConstantOrRegister(select->GetFalseValue()));
+ }
+
+ locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
}
- locations->SetOut(Location::SameAsFirstInput());
}
void InstructionCodeGeneratorARM::VisitSelect(HSelect* select) {
- LocationSummary* locations = select->GetLocations();
- Label false_target;
- GenerateTestAndBranch(select,
- /* condition_input_index */ 2,
- /* true_target */ nullptr,
- &false_target);
- codegen_->MoveLocation(locations->Out(), locations->InAt(1), select->GetType());
- __ Bind(&false_target);
+ HInstruction* const condition = select->GetCondition();
+ const LocationSummary* const locations = select->GetLocations();
+ const Primitive::Type type = select->GetType();
+ const Location first = locations->InAt(0);
+ const Location out = locations->Out();
+ const Location second = locations->InAt(1);
+ Location src;
+
+ if (condition->IsIntConstant()) {
+ if (condition->AsIntConstant()->IsFalse()) {
+ src = first;
+ } else {
+ src = second;
+ }
+
+ codegen_->MoveLocation(out, src, type);
+ return;
+ }
+
+ if (!Primitive::IsFloatingPointType(type) &&
+ CanGenerateTest(condition, codegen_->GetAssembler())) {
+ bool invert = false;
+
+ if (out.Equals(second)) {
+ src = first;
+ invert = true;
+ } else if (out.Equals(first)) {
+ src = second;
+ } else if (second.IsConstant()) {
+ DCHECK(CanEncodeConstantAs8BitImmediate(second.GetConstant()));
+ src = second;
+ } else if (first.IsConstant()) {
+ DCHECK(CanEncodeConstantAs8BitImmediate(first.GetConstant()));
+ src = first;
+ invert = true;
+ } else {
+ src = second;
+ }
+
+ if (CanGenerateConditionalMove(out, src)) {
+ if (!out.Equals(first) && !out.Equals(second)) {
+ codegen_->MoveLocation(out, src.Equals(first) ? second : first, type);
+ }
+
+ const Condition cond = GenerateTest(condition, locations->InAt(2), invert, codegen_);
+
+ if (out.IsRegister()) {
+ ShifterOperand operand;
+
+ if (src.IsConstant()) {
+ operand = ShifterOperand(CodeGenerator::GetInt32ValueOf(src.GetConstant()));
+ } else {
+ DCHECK(src.IsRegister());
+ operand = ShifterOperand(src.AsRegister<Register>());
+ }
+
+ __ it(cond);
+ __ mov(out.AsRegister<Register>(), operand, cond);
+ } else {
+ DCHECK(out.IsRegisterPair());
+
+ ShifterOperand operand_high;
+ ShifterOperand operand_low;
+
+ if (src.IsConstant()) {
+ const int64_t value = src.GetConstant()->AsLongConstant()->GetValue();
+
+ operand_high = ShifterOperand(High32Bits(value));
+ operand_low = ShifterOperand(Low32Bits(value));
+ } else {
+ DCHECK(src.IsRegisterPair());
+ operand_high = ShifterOperand(src.AsRegisterPairHigh<Register>());
+ operand_low = ShifterOperand(src.AsRegisterPairLow<Register>());
+ }
+
+ __ it(cond);
+ __ mov(out.AsRegisterPairLow<Register>(), operand_low, cond);
+ __ it(cond);
+ __ mov(out.AsRegisterPairHigh<Register>(), operand_high, cond);
+ }
+
+ return;
+ }
+ }
+
+ Label* false_target = nullptr;
+ Label* true_target = nullptr;
+ Label select_end;
+ Label* target = codegen_->GetFinalLabel(select, &select_end);
+
+ if (out.Equals(second)) {
+ true_target = target;
+ src = first;
+ } else {
+ false_target = target;
+ src = second;
+
+ if (!out.Equals(first)) {
+ codegen_->MoveLocation(out, first, type);
+ }
+ }
+
+ GenerateTestAndBranch(select, 2, true_target, false_target);
+ codegen_->MoveLocation(out, src, type);
+
+ if (select_end.IsLinked()) {
+ __ Bind(&select_end);
+ }
}
void LocationsBuilderARM::VisitNativeDebugInfo(HNativeDebugInfo* info) {
@@ -2293,7 +2894,7 @@
break;
case Primitive::kPrimFloat:
case Primitive::kPrimDouble:
- GenerateVcmp(cond);
+ GenerateVcmp(cond, codegen_);
GenerateFPJumps(cond, &true_label, &false_label);
break;
}
@@ -4347,7 +4948,7 @@
case Primitive::kPrimFloat:
case Primitive::kPrimDouble: {
__ LoadImmediate(out, 0);
- GenerateVcmp(compare);
+ GenerateVcmp(compare, codegen_);
__ vmstat(); // transfer FP status register to ARM APSR.
less_cond = ARMFPCondition(kCondLT, compare->IsGtBias());
break;
@@ -4710,6 +5311,7 @@
case ADC: neg_opcode = SBC; value = ~value; break;
case SUB: neg_opcode = ADD; value = -value; break;
case SBC: neg_opcode = ADC; value = ~value; break;
+ case MOV: neg_opcode = MVN; value = ~value; break;
default:
return false;
}
@@ -7185,14 +7787,35 @@
DCHECK(kEmitCompilerReadBarrier);
if (kUseBakerReadBarrier) {
// Fast path implementation of art::ReadBarrier::BarrierForRoot when
- // Baker's read barrier are used:
+ // Baker's read barrier are used.
//
- // root = obj.field;
+ // Note that we do not actually check the value of
+ // `GetIsGcMarking()` to decide whether to mark the loaded GC
+ // root or not. Instead, we load into `temp` the read barrier
+ // mark entry point corresponding to register `root`. If `temp`
+ // is null, it means that `GetIsGcMarking()` is false, and vice
+ // versa.
+ //
// temp = Thread::Current()->pReadBarrierMarkReg ## root.reg()
- // if (temp != null) {
- // root = temp(root)
+ // GcRoot<mirror::Object> root = *(obj+offset); // Original reference load.
+ // if (temp != nullptr) { // <=> Thread::Current()->GetIsGcMarking()
+ // // Slow path.
+ // root = temp(root); // root = ReadBarrier::Mark(root); // Runtime entry point call.
// }
+ // Slow path marking the GC root `root`. The entrypoint will already be loaded in `temp`.
+ Location temp = Location::RegisterLocation(LR);
+ SlowPathCodeARM* slow_path = new (GetGraph()->GetArena()) ReadBarrierMarkSlowPathARM(
+ instruction, root, /* entrypoint */ temp);
+ codegen_->AddSlowPath(slow_path);
+
+ // temp = Thread::Current()->pReadBarrierMarkReg ## root.reg()
+ const int32_t entry_point_offset =
+ CodeGenerator::GetReadBarrierMarkEntryPointsOffset<kArmPointerSize>(root.reg());
+ // Loading the entrypoint does not require a load acquire since it is only changed when
+ // threads are suspended or running a checkpoint.
+ __ LoadFromOffset(kLoadWord, temp.AsRegister<Register>(), TR, entry_point_offset);
+
// /* GcRoot<mirror::Object> */ root = *(obj + offset)
__ LoadFromOffset(kLoadWord, root_reg, obj, offset);
static_assert(
@@ -7203,21 +7826,6 @@
"art::mirror::CompressedReference<mirror::Object> and int32_t "
"have different sizes.");
- // Slow path marking the GC root `root`.
- Location temp = Location::RegisterLocation(LR);
- SlowPathCodeARM* slow_path =
- new (GetGraph()->GetArena()) ReadBarrierMarkSlowPathARM(
- instruction,
- root,
- /*entrypoint*/ temp);
- codegen_->AddSlowPath(slow_path);
-
- // temp = Thread::Current()->pReadBarrierMarkReg ## root.reg()
- const int32_t entry_point_offset =
- CodeGenerator::GetReadBarrierMarkEntryPointsOffset<kArmPointerSize>(root.reg());
- // Loading the entrypoint does not require a load acquire since it is only changed when
- // threads are suspended or running a checkpoint.
- __ LoadFromOffset(kLoadWord, temp.AsRegister<Register>(), TR, entry_point_offset);
// The entrypoint is null when the GC is not marking, this prevents one load compared to
// checking GetIsGcMarking.
__ CompareAndBranchIfNonZero(temp.AsRegister<Register>(), slow_path->GetEntryLabel());
@@ -7288,51 +7896,101 @@
DCHECK(kEmitCompilerReadBarrier);
DCHECK(kUseBakerReadBarrier);
- // In slow path based read barriers, the read barrier call is
- // inserted after the original load. However, in fast path based
- // Baker's read barriers, we need to perform the load of
- // mirror::Object::monitor_ *before* the original reference load.
- // This load-load ordering is required by the read barrier.
- // The fast path/slow path (for Baker's algorithm) should look like:
+ // Query `art::Thread::Current()->GetIsGcMarking()` to decide
+ // whether we need to enter the slow path to mark the reference.
+ // Then, in the slow path, check the gray bit in the lock word of
+ // the reference's holder (`obj`) to decide whether to mark `ref` or
+ // not.
//
- // uint32_t rb_state = Lockword(obj->monitor_).ReadBarrierState();
- // lfence; // Load fence or artificial data dependency to prevent load-load reordering
- // HeapReference<Object> ref = *src; // Original reference load.
- // bool is_gray = (rb_state == ReadBarrier::GrayState());
- // if (is_gray) {
- // ref = ReadBarrier::Mark(ref); // Performed by runtime entrypoint slow path.
+ // Note that we do not actually check the value of `GetIsGcMarking()`;
+ // instead, we load into `temp3` the read barrier mark entry point
+ // corresponding to register `ref`. If `temp3` is null, it means
+ // that `GetIsGcMarking()` is false, and vice versa.
+ //
+ // temp3 = Thread::Current()->pReadBarrierMarkReg ## root.reg()
+ // if (temp3 != nullptr) { // <=> Thread::Current()->GetIsGcMarking()
+ // // Slow path.
+ // uint32_t rb_state = Lockword(obj->monitor_).ReadBarrierState();
+ // lfence; // Load fence or artificial data dependency to prevent load-load reordering
+ // HeapReference<mirror::Object> ref = *src; // Original reference load.
+ // bool is_gray = (rb_state == ReadBarrier::GrayState());
+ // if (is_gray) {
+ // ref = temp3(ref); // ref = ReadBarrier::Mark(ref); // Runtime entry point call.
+ // }
+ // } else {
+ // HeapReference<mirror::Object> ref = *src; // Original reference load.
// }
- //
- // Note: the original implementation in ReadBarrier::Barrier is
- // slightly more complex as it performs additional checks that we do
- // not do here for performance reasons.
- Register ref_reg = ref.AsRegister<Register>();
Register temp_reg = temp.AsRegister<Register>();
- uint32_t monitor_offset = mirror::Object::MonitorOffset().Int32Value();
- // /* int32_t */ monitor = obj->monitor_
- __ LoadFromOffset(kLoadWord, temp_reg, obj, monitor_offset);
- if (needs_null_check) {
- MaybeRecordImplicitNullCheck(instruction);
+ // Slow path marking the object `ref` when the GC is marking. The
+ // entrypoint will already be loaded in `temp3`.
+ Location temp3 = Location::RegisterLocation(LR);
+ SlowPathCodeARM* slow_path;
+ if (always_update_field) {
+ DCHECK(temp2 != nullptr);
+ // LoadReferenceWithBakerReadBarrierAndUpdateFieldSlowPathARM only
+ // supports address of the form `obj + field_offset`, where `obj`
+ // is a register and `field_offset` is a register pair (of which
+ // only the lower half is used). Thus `offset` and `scale_factor`
+ // above are expected to be null in this code path.
+ DCHECK_EQ(offset, 0u);
+ DCHECK_EQ(scale_factor, ScaleFactor::TIMES_1);
+ Location field_offset = index;
+ slow_path =
+ new (GetGraph()->GetArena()) LoadReferenceWithBakerReadBarrierAndUpdateFieldSlowPathARM(
+ instruction,
+ ref,
+ obj,
+ offset,
+ /* index */ field_offset,
+ scale_factor,
+ needs_null_check,
+ temp_reg,
+ *temp2,
+ /* entrypoint */ temp3);
+ } else {
+ slow_path = new (GetGraph()->GetArena()) LoadReferenceWithBakerReadBarrierSlowPathARM(
+ instruction,
+ ref,
+ obj,
+ offset,
+ index,
+ scale_factor,
+ needs_null_check,
+ temp_reg,
+ /* entrypoint */ temp3);
}
- // /* LockWord */ lock_word = LockWord(monitor)
- static_assert(sizeof(LockWord) == sizeof(int32_t),
- "art::LockWord and int32_t have different sizes.");
+ AddSlowPath(slow_path);
- // Introduce a dependency on the lock_word including the rb_state,
- // which shall prevent load-load reordering without using
- // a memory barrier (which would be more expensive).
- // `obj` is unchanged by this operation, but its value now depends
- // on `temp_reg`.
- __ add(obj, obj, ShifterOperand(temp_reg, LSR, 32));
+ // temp3 = Thread::Current()->pReadBarrierMarkReg ## ref.reg()
+ const int32_t entry_point_offset =
+ CodeGenerator::GetReadBarrierMarkEntryPointsOffset<kArmPointerSize>(ref.reg());
+ // Loading the entrypoint does not require a load acquire since it is only changed when
+ // threads are suspended or running a checkpoint.
+ __ LoadFromOffset(kLoadWord, temp3.AsRegister<Register>(), TR, entry_point_offset);
+ // The entrypoint is null when the GC is not marking, this prevents one load compared to
+ // checking GetIsGcMarking.
+ __ CompareAndBranchIfNonZero(temp3.AsRegister<Register>(), slow_path->GetEntryLabel());
+ // Fast path: just load the reference.
+ GenerateRawReferenceLoad(instruction, ref, obj, offset, index, scale_factor, needs_null_check);
+ __ Bind(slow_path->GetExitLabel());
+}
- // The actual reference load.
+void CodeGeneratorARM::GenerateRawReferenceLoad(HInstruction* instruction,
+ Location ref,
+ Register obj,
+ uint32_t offset,
+ Location index,
+ ScaleFactor scale_factor,
+ bool needs_null_check) {
+ Register ref_reg = ref.AsRegister<Register>();
+
if (index.IsValid()) {
// Load types involving an "index": ArrayGet,
// UnsafeGetObject/UnsafeGetObjectVolatile and UnsafeCASObject
// intrinsics.
- // /* HeapReference<Object> */ ref = *(obj + offset + (index << scale_factor))
+ // /* HeapReference<mirror::Object> */ ref = *(obj + offset + (index << scale_factor))
if (index.IsConstant()) {
size_t computed_offset =
(index.GetConstant()->AsIntConstant()->GetValue() << scale_factor) + offset;
@@ -7349,41 +8007,16 @@
__ LoadFromOffset(kLoadWord, ref_reg, IP, offset);
}
} else {
- // /* HeapReference<Object> */ ref = *(obj + offset)
+ // /* HeapReference<mirror::Object> */ ref = *(obj + offset)
__ LoadFromOffset(kLoadWord, ref_reg, obj, offset);
}
+ if (needs_null_check) {
+ MaybeRecordImplicitNullCheck(instruction);
+ }
+
// Object* ref = ref_addr->AsMirrorPtr()
__ MaybeUnpoisonHeapReference(ref_reg);
-
- // Slow path marking the object `ref` when it is gray.
- SlowPathCodeARM* slow_path;
- if (always_update_field) {
- DCHECK(temp2 != nullptr);
- // ReadBarrierMarkAndUpdateFieldSlowPathARM only supports address
- // of the form `obj + field_offset`, where `obj` is a register and
- // `field_offset` is a register pair (of which only the lower half
- // is used). Thus `offset` and `scale_factor` above are expected
- // to be null in this code path.
- DCHECK_EQ(offset, 0u);
- DCHECK_EQ(scale_factor, ScaleFactor::TIMES_1);
- slow_path = new (GetGraph()->GetArena()) ReadBarrierMarkAndUpdateFieldSlowPathARM(
- instruction, ref, obj, /* field_offset */ index, temp_reg, *temp2);
- } else {
- slow_path = new (GetGraph()->GetArena()) ReadBarrierMarkSlowPathARM(instruction, ref);
- }
- AddSlowPath(slow_path);
-
- // if (rb_state == ReadBarrier::GrayState())
- // ref = ReadBarrier::Mark(ref);
- // Given the numeric representation, it's enough to check the low bit of the
- // rb_state. We do that by shifting the bit out of the lock word with LSRS
- // which can be a 16-bit instruction unlike the TST immediate.
- static_assert(ReadBarrier::WhiteState() == 0, "Expecting white to have value 0");
- static_assert(ReadBarrier::GrayState() == 1, "Expecting gray to have value 1");
- __ Lsrs(temp_reg, temp_reg, LockWord::kReadBarrierStateShift + 1);
- __ b(slow_path->GetEntryLabel(), CS); // Carry flag is the last bit shifted out by LSRS.
- __ Bind(slow_path->GetExitLabel());
}
void CodeGeneratorARM::GenerateReadBarrierSlow(HInstruction* instruction,
@@ -7626,9 +8259,7 @@
}
Literal* CodeGeneratorARM::DeduplicateBootImageAddressLiteral(uint32_t address) {
- bool needs_patch = GetCompilerOptions().GetIncludePatchInformation();
- Uint32ToLiteralMap* map = needs_patch ? &boot_image_address_patches_ : &uint32_literals_;
- return DeduplicateUint32Literal(dchecked_integral_cast<uint32_t>(address), map);
+ return DeduplicateUint32Literal(dchecked_integral_cast<uint32_t>(address), &uint32_literals_);
}
Literal* CodeGeneratorARM::DeduplicateJitStringLiteral(const DexFile& dex_file,
@@ -7679,8 +8310,7 @@
/* MOVW+MOVT for each entry */ 2u * pc_relative_string_patches_.size() +
boot_image_type_patches_.size() +
/* MOVW+MOVT for each entry */ 2u * pc_relative_type_patches_.size() +
- /* MOVW+MOVT for each entry */ 2u * type_bss_entry_patches_.size() +
- boot_image_address_patches_.size();
+ /* MOVW+MOVT for each entry */ 2u * type_bss_entry_patches_.size();
linker_patches->reserve(size);
EmitPcRelativeLinkerPatches<LinkerPatch::DexCacheArrayPatch>(pc_relative_dex_cache_patches_,
linker_patches);
@@ -7714,13 +8344,6 @@
target_type.dex_file,
target_type.type_index.index_));
}
- for (const auto& entry : boot_image_address_patches_) {
- DCHECK(GetCompilerOptions().GetIncludePatchInformation());
- Literal* literal = entry.second;
- DCHECK(literal->GetLabel()->IsBound());
- uint32_t literal_offset = literal->GetLabel()->Position();
- linker_patches->push_back(LinkerPatch::RecordPosition(literal_offset));
- }
DCHECK_EQ(size, linker_patches->size());
}
diff --git a/compiler/optimizing/code_generator_arm.h b/compiler/optimizing/code_generator_arm.h
index df2dbc7..5b15902 100644
--- a/compiler/optimizing/code_generator_arm.h
+++ b/compiler/optimizing/code_generator_arm.h
@@ -299,7 +299,6 @@
void GenerateCompareTestAndBranch(HCondition* condition,
Label* true_target,
Label* false_target);
- void GenerateVcmp(HInstruction* instruction);
void GenerateFPJumps(HCondition* cond, Label* true_label, Label* false_label);
void GenerateLongComparesAndJumps(HCondition* cond, Label* true_label, Label* false_label);
void DivRemOneOrMinusOne(HBinaryOperation* instruction);
@@ -422,6 +421,8 @@
return CommonGetLabelOf<Label>(block_labels_, block);
}
+ Label* GetFinalLabel(HInstruction* instruction, Label* final_label);
+
void Initialize() OVERRIDE {
block_labels_ = CommonInitializeLabels<Label>();
}
@@ -520,9 +521,6 @@
Location index,
Location temp,
bool needs_null_check);
- // Factored implementation used by GenerateFieldLoadWithBakerReadBarrier
- // and GenerateArrayLoadWithBakerReadBarrier.
-
// Factored implementation, used by GenerateFieldLoadWithBakerReadBarrier,
// GenerateArrayLoadWithBakerReadBarrier and some intrinsics.
//
@@ -545,6 +543,15 @@
bool always_update_field = false,
Register* temp2 = nullptr);
+ // Generate a heap reference load (with no read barrier).
+ void GenerateRawReferenceLoad(HInstruction* instruction,
+ Location ref,
+ Register obj,
+ uint32_t offset,
+ Location index,
+ ScaleFactor scale_factor,
+ bool needs_null_check);
+
// Generate a read barrier for a heap reference within `instruction`
// using a slow path.
//
@@ -642,8 +649,6 @@
ArenaDeque<PcRelativePatchInfo> pc_relative_type_patches_;
// PC-relative type patch info for kBssEntry.
ArenaDeque<PcRelativePatchInfo> type_bss_entry_patches_;
- // Deduplication map for patchable boot image addresses.
- Uint32ToLiteralMap boot_image_address_patches_;
// Patches for string literals in JIT compiled code.
StringToLiteralMap jit_string_patches_;
diff --git a/compiler/optimizing/code_generator_arm64.cc b/compiler/optimizing/code_generator_arm64.cc
index 18c95b3..97b61ed 100644
--- a/compiler/optimizing/code_generator_arm64.cc
+++ b/compiler/optimizing/code_generator_arm64.cc
@@ -633,58 +633,23 @@
}
}
-// Slow path marking an object reference `ref` during a read
-// barrier. The field `obj.field` in the object `obj` holding this
-// reference does not get updated by this slow path after marking (see
-// ReadBarrierMarkAndUpdateFieldSlowPathARM64 below for that).
+// Abstract base class for read barrier slow paths marking a reference
+// `ref`.
//
-// This means that after the execution of this slow path, `ref` will
-// always be up-to-date, but `obj.field` may not; i.e., after the
-// flip, `ref` will be a to-space reference, but `obj.field` will
-// probably still be a from-space reference (unless it gets updated by
-// another thread, or if another thread installed another object
-// reference (different from `ref`) in `obj.field`).
-//
-// If `entrypoint` is a valid location it is assumed to already be
-// holding the entrypoint. The case where the entrypoint is passed in
-// is for the GcRoot read barrier.
-class ReadBarrierMarkSlowPathARM64 : public SlowPathCodeARM64 {
- public:
- ReadBarrierMarkSlowPathARM64(HInstruction* instruction,
- Location ref,
- Location entrypoint = Location::NoLocation())
- : SlowPathCodeARM64(instruction),
- ref_(ref),
- entrypoint_(entrypoint) {
+// Argument `entrypoint` must be a register location holding the read
+// barrier marking runtime entry point to be invoked.
+class ReadBarrierMarkSlowPathBaseARM64 : public SlowPathCodeARM64 {
+ protected:
+ ReadBarrierMarkSlowPathBaseARM64(HInstruction* instruction, Location ref, Location entrypoint)
+ : SlowPathCodeARM64(instruction), ref_(ref), entrypoint_(entrypoint) {
DCHECK(kEmitCompilerReadBarrier);
}
- const char* GetDescription() const OVERRIDE { return "ReadBarrierMarkSlowPathARM64"; }
+ const char* GetDescription() const OVERRIDE { return "ReadBarrierMarkSlowPathBaseARM64"; }
- void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
- LocationSummary* locations = instruction_->GetLocations();
- DCHECK(locations->CanCall());
- DCHECK(ref_.IsRegister()) << ref_;
- DCHECK(!locations->GetLiveRegisters()->ContainsCoreRegister(ref_.reg())) << ref_.reg();
- DCHECK(instruction_->IsInstanceFieldGet() ||
- instruction_->IsStaticFieldGet() ||
- instruction_->IsArrayGet() ||
- instruction_->IsArraySet() ||
- instruction_->IsLoadClass() ||
- instruction_->IsLoadString() ||
- instruction_->IsInstanceOf() ||
- instruction_->IsCheckCast() ||
- (instruction_->IsInvokeVirtual() && instruction_->GetLocations()->Intrinsified()) ||
- (instruction_->IsInvokeStaticOrDirect() && instruction_->GetLocations()->Intrinsified()))
- << "Unexpected instruction in read barrier marking slow path: "
- << instruction_->DebugName();
- // The read barrier instrumentation of object ArrayGet
- // instructions does not support the HIntermediateAddress
- // instruction.
- DCHECK(!(instruction_->IsArrayGet() &&
- instruction_->AsArrayGet()->GetArray()->IsIntermediateAddress()));
-
- __ Bind(GetEntryLabel());
+ // Generate assembly code calling the read barrier marking runtime
+ // entry point (ReadBarrierMarkRegX).
+ void GenerateReadBarrierMarkRuntimeCall(CodeGenerator* codegen) {
// No need to save live registers; it's taken care of by the
// entrypoint. Also, there is no need to update the stack mask,
// as this runtime call will not trigger a garbage collection.
@@ -720,46 +685,261 @@
// This runtime call does not require a stack map.
arm64_codegen->InvokeRuntimeWithoutRecordingPcInfo(entry_point_offset, instruction_, this);
}
- __ B(GetExitLabel());
}
- private:
// The location (register) of the marked object reference.
const Location ref_;
// The location of the entrypoint if it is already loaded.
const Location entrypoint_;
+ private:
+ DISALLOW_COPY_AND_ASSIGN(ReadBarrierMarkSlowPathBaseARM64);
+};
+
+// Slow path marking an object reference `ref` during a read
+// barrier. The field `obj.field` in the object `obj` holding this
+// reference does not get updated by this slow path after marking.
+//
+// This means that after the execution of this slow path, `ref` will
+// always be up-to-date, but `obj.field` may not; i.e., after the
+// flip, `ref` will be a to-space reference, but `obj.field` will
+// probably still be a from-space reference (unless it gets updated by
+// another thread, or if another thread installed another object
+// reference (different from `ref`) in `obj.field`).
+//
+// If `entrypoint` is a valid location it is assumed to already be
+// holding the entrypoint. The case where the entrypoint is passed in
+// is when the decision to mark is based on whether the GC is marking.
+class ReadBarrierMarkSlowPathARM64 : public ReadBarrierMarkSlowPathBaseARM64 {
+ public:
+ ReadBarrierMarkSlowPathARM64(HInstruction* instruction,
+ Location ref,
+ Location entrypoint = Location::NoLocation())
+ : ReadBarrierMarkSlowPathBaseARM64(instruction, ref, entrypoint) {
+ DCHECK(kEmitCompilerReadBarrier);
+ }
+
+ const char* GetDescription() const OVERRIDE { return "ReadBarrierMarkSlowPathARM64"; }
+
+ void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
+ LocationSummary* locations = instruction_->GetLocations();
+ DCHECK(locations->CanCall());
+ DCHECK(ref_.IsRegister()) << ref_;
+ DCHECK(!locations->GetLiveRegisters()->ContainsCoreRegister(ref_.reg())) << ref_.reg();
+ DCHECK(instruction_->IsLoadClass() || instruction_->IsLoadString())
+ << "Unexpected instruction in read barrier marking slow path: "
+ << instruction_->DebugName();
+
+ __ Bind(GetEntryLabel());
+ GenerateReadBarrierMarkRuntimeCall(codegen);
+ __ B(GetExitLabel());
+ }
+
+ private:
DISALLOW_COPY_AND_ASSIGN(ReadBarrierMarkSlowPathARM64);
};
-// Slow path marking an object reference `ref` during a read barrier,
-// and if needed, atomically updating the field `obj.field` in the
-// object `obj` holding this reference after marking (contrary to
-// ReadBarrierMarkSlowPathARM64 above, which never tries to update
-// `obj.field`).
+// Slow path loading `obj`'s lock word, loading a reference from
+// object `*(obj + offset + (index << scale_factor))` into `ref`, and
+// marking `ref` if `obj` is gray according to the lock word (Baker
+// read barrier). The field `obj.field` in the object `obj` holding
+// this reference does not get updated by this slow path after marking
+// (see LoadReferenceWithBakerReadBarrierAndUpdateFieldSlowPathARM64
+// below for that).
+//
+// This means that after the execution of this slow path, `ref` will
+// always be up-to-date, but `obj.field` may not; i.e., after the
+// flip, `ref` will be a to-space reference, but `obj.field` will
+// probably still be a from-space reference (unless it gets updated by
+// another thread, or if another thread installed another object
+// reference (different from `ref`) in `obj.field`).
+//
+// Argument `entrypoint` must be a register location holding the read
+// barrier marking runtime entry point to be invoked.
+class LoadReferenceWithBakerReadBarrierSlowPathARM64 : public ReadBarrierMarkSlowPathBaseARM64 {
+ public:
+ LoadReferenceWithBakerReadBarrierSlowPathARM64(HInstruction* instruction,
+ Location ref,
+ Register obj,
+ uint32_t offset,
+ Location index,
+ size_t scale_factor,
+ bool needs_null_check,
+ bool use_load_acquire,
+ Register temp,
+ Location entrypoint)
+ : ReadBarrierMarkSlowPathBaseARM64(instruction, ref, entrypoint),
+ obj_(obj),
+ offset_(offset),
+ index_(index),
+ scale_factor_(scale_factor),
+ needs_null_check_(needs_null_check),
+ use_load_acquire_(use_load_acquire),
+ temp_(temp) {
+ DCHECK(kEmitCompilerReadBarrier);
+ DCHECK(kUseBakerReadBarrier);
+ }
+
+ const char* GetDescription() const OVERRIDE {
+ return "LoadReferenceWithBakerReadBarrierSlowPathARM64";
+ }
+
+ void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
+ LocationSummary* locations = instruction_->GetLocations();
+ DCHECK(locations->CanCall());
+ DCHECK(ref_.IsRegister()) << ref_;
+ DCHECK(!locations->GetLiveRegisters()->ContainsCoreRegister(ref_.reg())) << ref_.reg();
+ DCHECK(obj_.IsW());
+ DCHECK_NE(ref_.reg(), LocationFrom(temp_).reg());
+ DCHECK(instruction_->IsInstanceFieldGet() ||
+ instruction_->IsStaticFieldGet() ||
+ instruction_->IsArrayGet() ||
+ instruction_->IsArraySet() ||
+ instruction_->IsInstanceOf() ||
+ instruction_->IsCheckCast() ||
+ (instruction_->IsInvokeVirtual() && instruction_->GetLocations()->Intrinsified()) ||
+ (instruction_->IsInvokeStaticOrDirect() && instruction_->GetLocations()->Intrinsified()))
+ << "Unexpected instruction in read barrier marking slow path: "
+ << instruction_->DebugName();
+ // The read barrier instrumentation of object ArrayGet
+ // instructions does not support the HIntermediateAddress
+ // instruction.
+ DCHECK(!(instruction_->IsArrayGet() &&
+ instruction_->AsArrayGet()->GetArray()->IsIntermediateAddress()));
+
+ // Temporary register `temp_`, used to store the lock word, must
+ // not be IP0 nor IP1, as we may use them to emit the reference
+ // load (in the call to GenerateRawReferenceLoad below), and we
+ // need the lock word to still be in `temp_` after the reference
+ // load.
+ DCHECK_NE(LocationFrom(temp_).reg(), IP0);
+ DCHECK_NE(LocationFrom(temp_).reg(), IP1);
+
+ __ Bind(GetEntryLabel());
+
+ // When using MaybeGenerateReadBarrierSlow, the read barrier call is
+ // inserted after the original load. However, in fast path based
+ // Baker's read barriers, we need to perform the load of
+ // mirror::Object::monitor_ *before* the original reference load.
+ // This load-load ordering is required by the read barrier.
+ // The fast path/slow path (for Baker's algorithm) should look like:
+ //
+ // uint32_t rb_state = Lockword(obj->monitor_).ReadBarrierState();
+ // lfence; // Load fence or artificial data dependency to prevent load-load reordering
+ // HeapReference<mirror::Object> ref = *src; // Original reference load.
+ // bool is_gray = (rb_state == ReadBarrier::GrayState());
+ // if (is_gray) {
+ // ref = entrypoint(ref); // ref = ReadBarrier::Mark(ref); // Runtime entry point call.
+ // }
+ //
+ // Note: the original implementation in ReadBarrier::Barrier is
+ // slightly more complex as it performs additional checks that we do
+ // not do here for performance reasons.
+
+ // /* int32_t */ monitor = obj->monitor_
+ uint32_t monitor_offset = mirror::Object::MonitorOffset().Int32Value();
+ __ Ldr(temp_, HeapOperand(obj_, monitor_offset));
+ if (needs_null_check_) {
+ codegen->MaybeRecordImplicitNullCheck(instruction_);
+ }
+ // /* LockWord */ lock_word = LockWord(monitor)
+ static_assert(sizeof(LockWord) == sizeof(int32_t),
+ "art::LockWord and int32_t have different sizes.");
+
+ // Introduce a dependency on the lock_word including rb_state,
+ // to prevent load-load reordering, and without using
+ // a memory barrier (which would be more expensive).
+ // `obj` is unchanged by this operation, but its value now depends
+ // on `temp`.
+ __ Add(obj_.X(), obj_.X(), Operand(temp_.X(), LSR, 32));
+
+ // The actual reference load.
+ // A possible implicit null check has already been handled above.
+ CodeGeneratorARM64* arm64_codegen = down_cast<CodeGeneratorARM64*>(codegen);
+ arm64_codegen->GenerateRawReferenceLoad(instruction_,
+ ref_,
+ obj_,
+ offset_,
+ index_,
+ scale_factor_,
+ /* needs_null_check */ false,
+ use_load_acquire_);
+
+ // Mark the object `ref` when `obj` is gray.
+ //
+ // if (rb_state == ReadBarrier::GrayState())
+ // ref = ReadBarrier::Mark(ref);
+ //
+ // Given the numeric representation, it's enough to check the low bit of the rb_state.
+ static_assert(ReadBarrier::WhiteState() == 0, "Expecting white to have value 0");
+ static_assert(ReadBarrier::GrayState() == 1, "Expecting gray to have value 1");
+ __ Tbz(temp_, LockWord::kReadBarrierStateShift, GetExitLabel());
+ GenerateReadBarrierMarkRuntimeCall(codegen);
+
+ __ B(GetExitLabel());
+ }
+
+ private:
+ // The register containing the object holding the marked object reference field.
+ Register obj_;
+ // The offset, index and scale factor to access the reference in `obj_`.
+ uint32_t offset_;
+ Location index_;
+ size_t scale_factor_;
+ // Is a null check required?
+ bool needs_null_check_;
+ // Should this reference load use Load-Acquire semantics?
+ bool use_load_acquire_;
+ // A temporary register used to hold the lock word of `obj_`.
+ Register temp_;
+
+ DISALLOW_COPY_AND_ASSIGN(LoadReferenceWithBakerReadBarrierSlowPathARM64);
+};
+
+// Slow path loading `obj`'s lock word, loading a reference from
+// object `*(obj + offset + (index << scale_factor))` into `ref`, and
+// marking `ref` if `obj` is gray according to the lock word (Baker
+// read barrier). If needed, this slow path also atomically updates
+// the field `obj.field` in the object `obj` holding this reference
+// after marking (contrary to
+// LoadReferenceWithBakerReadBarrierSlowPathARM64 above, which never
+// tries to update `obj.field`).
//
// This means that after the execution of this slow path, both `ref`
// and `obj.field` will be up-to-date; i.e., after the flip, both will
// hold the same to-space reference (unless another thread installed
// another object reference (different from `ref`) in `obj.field`).
-class ReadBarrierMarkAndUpdateFieldSlowPathARM64 : public SlowPathCodeARM64 {
+//
+// Argument `entrypoint` must be a register location holding the read
+// barrier marking runtime entry point to be invoked.
+class LoadReferenceWithBakerReadBarrierAndUpdateFieldSlowPathARM64
+ : public ReadBarrierMarkSlowPathBaseARM64 {
public:
- ReadBarrierMarkAndUpdateFieldSlowPathARM64(HInstruction* instruction,
- Location ref,
- Register obj,
- Location field_offset,
- Register temp)
- : SlowPathCodeARM64(instruction),
- ref_(ref),
+ LoadReferenceWithBakerReadBarrierAndUpdateFieldSlowPathARM64(HInstruction* instruction,
+ Location ref,
+ Register obj,
+ uint32_t offset,
+ Location index,
+ size_t scale_factor,
+ bool needs_null_check,
+ bool use_load_acquire,
+ Register temp,
+ Location entrypoint)
+ : ReadBarrierMarkSlowPathBaseARM64(instruction, ref, entrypoint),
obj_(obj),
- field_offset_(field_offset),
+ offset_(offset),
+ index_(index),
+ scale_factor_(scale_factor),
+ needs_null_check_(needs_null_check),
+ use_load_acquire_(use_load_acquire),
temp_(temp) {
DCHECK(kEmitCompilerReadBarrier);
+ DCHECK(kUseBakerReadBarrier);
}
const char* GetDescription() const OVERRIDE {
- return "ReadBarrierMarkAndUpdateFieldSlowPathARM64";
+ return "LoadReferenceWithBakerReadBarrierAndUpdateFieldSlowPathARM64";
}
void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
@@ -768,64 +948,90 @@
DCHECK(locations->CanCall());
DCHECK(ref_.IsRegister()) << ref_;
DCHECK(!locations->GetLiveRegisters()->ContainsCoreRegister(ref_.reg())) << ref_.reg();
- // This slow path is only used by the UnsafeCASObject intrinsic.
+ DCHECK(obj_.IsW());
+ DCHECK_NE(ref_.reg(), LocationFrom(temp_).reg());
+
+ // This slow path is only used by the UnsafeCASObject intrinsic at the moment.
DCHECK((instruction_->IsInvokeVirtual() && instruction_->GetLocations()->Intrinsified()))
<< "Unexpected instruction in read barrier marking and field updating slow path: "
<< instruction_->DebugName();
DCHECK(instruction_->GetLocations()->Intrinsified());
DCHECK_EQ(instruction_->AsInvoke()->GetIntrinsic(), Intrinsics::kUnsafeCASObject);
- DCHECK(field_offset_.IsRegister()) << field_offset_;
+ DCHECK_EQ(offset_, 0u);
+ DCHECK_EQ(scale_factor_, 0u);
+ DCHECK_EQ(use_load_acquire_, false);
+ // The location of the offset of the marked reference field within `obj_`.
+ Location field_offset = index_;
+ DCHECK(field_offset.IsRegister()) << field_offset;
+
+ // Temporary register `temp_`, used to store the lock word, must
+ // not be IP0 nor IP1, as we may use them to emit the reference
+ // load (in the call to GenerateRawReferenceLoad below), and we
+ // need the lock word to still be in `temp_` after the reference
+ // load.
+ DCHECK_NE(LocationFrom(temp_).reg(), IP0);
+ DCHECK_NE(LocationFrom(temp_).reg(), IP1);
__ Bind(GetEntryLabel());
- // Save the old reference.
+ // /* int32_t */ monitor = obj->monitor_
+ uint32_t monitor_offset = mirror::Object::MonitorOffset().Int32Value();
+ __ Ldr(temp_, HeapOperand(obj_, monitor_offset));
+ if (needs_null_check_) {
+ codegen->MaybeRecordImplicitNullCheck(instruction_);
+ }
+ // /* LockWord */ lock_word = LockWord(monitor)
+ static_assert(sizeof(LockWord) == sizeof(int32_t),
+ "art::LockWord and int32_t have different sizes.");
+
+ // Introduce a dependency on the lock_word including rb_state,
+ // to prevent load-load reordering, and without using
+ // a memory barrier (which would be more expensive).
+ // `obj` is unchanged by this operation, but its value now depends
+ // on `temp`.
+ __ Add(obj_.X(), obj_.X(), Operand(temp_.X(), LSR, 32));
+
+ // The actual reference load.
+ // A possible implicit null check has already been handled above.
+ CodeGeneratorARM64* arm64_codegen = down_cast<CodeGeneratorARM64*>(codegen);
+ arm64_codegen->GenerateRawReferenceLoad(instruction_,
+ ref_,
+ obj_,
+ offset_,
+ index_,
+ scale_factor_,
+ /* needs_null_check */ false,
+ use_load_acquire_);
+
+ // Mark the object `ref` when `obj` is gray.
+ //
+ // if (rb_state == ReadBarrier::GrayState())
+ // ref = ReadBarrier::Mark(ref);
+ //
+ // Given the numeric representation, it's enough to check the low bit of the rb_state.
+ static_assert(ReadBarrier::WhiteState() == 0, "Expecting white to have value 0");
+ static_assert(ReadBarrier::GrayState() == 1, "Expecting gray to have value 1");
+ __ Tbz(temp_, LockWord::kReadBarrierStateShift, GetExitLabel());
+
+ // Save the old value of the reference before marking it.
// Note that we cannot use IP to save the old reference, as IP is
// used internally by the ReadBarrierMarkRegX entry point, and we
// need the old reference after the call to that entry point.
DCHECK_NE(LocationFrom(temp_).reg(), IP0);
__ Mov(temp_.W(), ref_reg);
- // No need to save live registers; it's taken care of by the
- // entrypoint. Also, there is no need to update the stack mask,
- // as this runtime call will not trigger a garbage collection.
- CodeGeneratorARM64* arm64_codegen = down_cast<CodeGeneratorARM64*>(codegen);
- DCHECK_NE(ref_.reg(), LR);
- DCHECK_NE(ref_.reg(), WSP);
- DCHECK_NE(ref_.reg(), WZR);
- // IP0 is used internally by the ReadBarrierMarkRegX entry point
- // as a temporary, it cannot be the entry point's input/output.
- DCHECK_NE(ref_.reg(), IP0);
- DCHECK(0 <= ref_.reg() && ref_.reg() < kNumberOfWRegisters) << ref_.reg();
- // "Compact" slow path, saving two moves.
- //
- // Instead of using the standard runtime calling convention (input
- // and output in W0):
- //
- // W0 <- ref
- // W0 <- ReadBarrierMark(W0)
- // ref <- W0
- //
- // we just use rX (the register containing `ref`) as input and output
- // of a dedicated entrypoint:
- //
- // rX <- ReadBarrierMarkRegX(rX)
- //
- int32_t entry_point_offset =
- CodeGenerator::GetReadBarrierMarkEntryPointsOffset<kArm64PointerSize>(ref_.reg());
- // This runtime call does not require a stack map.
- arm64_codegen->InvokeRuntimeWithoutRecordingPcInfo(entry_point_offset, instruction_, this);
+ GenerateReadBarrierMarkRuntimeCall(codegen);
// If the new reference is different from the old reference,
- // update the field in the holder (`*(obj_ + field_offset_)`).
+ // update the field in the holder (`*(obj_ + field_offset)`).
//
// Note that this field could also hold a different object, if
// another thread had concurrently changed it. In that case, the
// LDXR/CMP/BNE sequence of instructions in the compare-and-set
// (CAS) operation below would abort the CAS, leaving the field
// as-is.
- vixl::aarch64::Label done;
__ Cmp(temp_.W(), ref_reg);
- __ B(eq, &done);
+ __ B(eq, GetExitLabel());
// Update the the holder's field atomically. This may fail if
// mutator updates before us, but it's OK. This is achieved
@@ -838,7 +1044,7 @@
// Convenience aliases.
Register base = obj_.W();
- Register offset = XRegisterFrom(field_offset_);
+ Register offset = XRegisterFrom(field_offset);
Register expected = temp_.W();
Register value = ref_reg;
Register tmp_ptr = temps.AcquireX(); // Pointer to actual memory.
@@ -882,21 +1088,26 @@
}
}
- __ Bind(&done);
__ B(GetExitLabel());
}
private:
- // The location (register) of the marked object reference.
- const Location ref_;
// The register containing the object holding the marked object reference field.
const Register obj_;
- // The location of the offset of the marked reference field within `obj_`.
- Location field_offset_;
-
+ // The offset, index and scale factor to access the reference in `obj_`.
+ uint32_t offset_;
+ Location index_;
+ size_t scale_factor_;
+ // Is a null check required?
+ bool needs_null_check_;
+ // Should this reference load use Load-Acquire semantics?
+ bool use_load_acquire_;
+ // A temporary register used to hold the lock word of `obj_`; and
+ // also to hold the original reference value, when the reference is
+ // marked.
const Register temp_;
- DISALLOW_COPY_AND_ASSIGN(ReadBarrierMarkAndUpdateFieldSlowPathARM64);
+ DISALLOW_COPY_AND_ASSIGN(LoadReferenceWithBakerReadBarrierAndUpdateFieldSlowPathARM64);
};
// Slow path generating a read barrier for a heap reference.
@@ -1200,8 +1411,6 @@
graph->GetArena()->Adapter(kArenaAllocCodeGenerator)),
pc_relative_type_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)),
type_bss_entry_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)),
- boot_image_address_patches_(std::less<uint32_t>(),
- graph->GetArena()->Adapter(kArenaAllocCodeGenerator)),
jit_string_patches_(StringReferenceValueComparator(),
graph->GetArena()->Adapter(kArenaAllocCodeGenerator)),
jit_class_patches_(TypeReferenceValueComparator(),
@@ -2427,6 +2636,9 @@
LocationSummary::kNoCall);
if (object_array_get_with_read_barrier && kUseBakerReadBarrier) {
locations->SetCustomSlowPathCallerSaves(RegisterSet::Empty()); // No caller-save registers.
+ // We need a temporary register for the read barrier marking slow
+ // path in CodeGeneratorARM64::GenerateArrayLoadWithBakerReadBarrier.
+ locations->AddTemp(Location::RequiresRegister());
}
locations->SetInAt(0, Location::RequiresRegister());
locations->SetInAt(1, Location::RegisterOrConstant(instruction->InputAt(1)));
@@ -2462,7 +2674,7 @@
if (type == Primitive::kPrimNot && kEmitCompilerReadBarrier && kUseBakerReadBarrier) {
// Object ArrayGet with Baker's read barrier case.
- Register temp = temps.AcquireW();
+ Register temp = WRegisterFrom(locations->GetTemp(0));
// Note that a potential implicit null check is handled in the
// CodeGeneratorARM64::GenerateArrayLoadWithBakerReadBarrier call.
codegen_->GenerateArrayLoadWithBakerReadBarrier(
@@ -3419,7 +3631,7 @@
if (Primitive::IsFloatingPointType(select->GetType())) {
locations->SetInAt(0, Location::RequiresFpuRegister());
locations->SetInAt(1, Location::RequiresFpuRegister());
- locations->SetOut(Location::RequiresFpuRegister());
+ locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap);
} else {
HConstant* cst_true_value = select->GetTrueValue()->AsConstant();
HConstant* cst_false_value = select->GetFalseValue()->AsConstant();
@@ -3442,7 +3654,7 @@
: Location::ConstantLocation(cst_true_value));
locations->SetInAt(0, false_value_in_register ? Location::RequiresRegister()
: Location::ConstantLocation(cst_false_value));
- locations->SetOut(Location::RequiresRegister());
+ locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
}
if (IsBooleanValueOrMaterializedCondition(select->GetCondition())) {
@@ -4328,9 +4540,7 @@
vixl::aarch64::Literal<uint32_t>* CodeGeneratorARM64::DeduplicateBootImageAddressLiteral(
uint64_t address) {
- bool needs_patch = GetCompilerOptions().GetIncludePatchInformation();
- Uint32ToLiteralMap* map = needs_patch ? &boot_image_address_patches_ : &uint32_literals_;
- return DeduplicateUint32Literal(dchecked_integral_cast<uint32_t>(address), map);
+ return DeduplicateUint32Literal(dchecked_integral_cast<uint32_t>(address), &uint32_literals_);
}
vixl::aarch64::Literal<uint32_t>* CodeGeneratorARM64::DeduplicateJitStringLiteral(
@@ -4398,8 +4608,7 @@
pc_relative_string_patches_.size() +
boot_image_type_patches_.size() +
pc_relative_type_patches_.size() +
- type_bss_entry_patches_.size() +
- boot_image_address_patches_.size();
+ type_bss_entry_patches_.size();
linker_patches->reserve(size);
for (const PcRelativePatchInfo& info : pc_relative_dex_cache_patches_) {
linker_patches->push_back(LinkerPatch::DexCacheArrayPatch(info.label.GetLocation(),
@@ -4433,11 +4642,6 @@
target_type.dex_file,
target_type.type_index.index_));
}
- for (const auto& entry : boot_image_address_patches_) {
- DCHECK(GetCompilerOptions().GetIncludePatchInformation());
- vixl::aarch64::Literal<uint32_t>* literal = entry.second;
- linker_patches->push_back(LinkerPatch::RecordPosition(literal->GetOffset()));
- }
DCHECK_EQ(size, linker_patches->size());
}
@@ -5614,14 +5818,35 @@
DCHECK(kEmitCompilerReadBarrier);
if (kUseBakerReadBarrier) {
// Fast path implementation of art::ReadBarrier::BarrierForRoot when
- // Baker's read barrier are used:
+ // Baker's read barrier are used.
//
- // root = obj.field;
+ // Note that we do not actually check the value of
+ // `GetIsGcMarking()` to decide whether to mark the loaded GC
+ // root or not. Instead, we load into `temp` the read barrier
+ // mark entry point corresponding to register `root`. If `temp`
+ // is null, it means that `GetIsGcMarking()` is false, and vice
+ // versa.
+ //
// temp = Thread::Current()->pReadBarrierMarkReg ## root.reg()
- // if (temp != null) {
- // root = temp(root)
+ // GcRoot<mirror::Object> root = *(obj+offset); // Original reference load.
+ // if (temp != nullptr) { // <=> Thread::Current()->GetIsGcMarking()
+ // // Slow path.
+ // root = temp(root); // root = ReadBarrier::Mark(root); // Runtime entry point call.
// }
+ // Slow path marking the GC root `root`. The entrypoint will already be loaded in `temp`.
+ Register temp = lr;
+ SlowPathCodeARM64* slow_path = new (GetGraph()->GetArena()) ReadBarrierMarkSlowPathARM64(
+ instruction, root, /* entrypoint */ LocationFrom(temp));
+ codegen_->AddSlowPath(slow_path);
+
+ // temp = Thread::Current()->pReadBarrierMarkReg ## root.reg()
+ const int32_t entry_point_offset =
+ CodeGenerator::GetReadBarrierMarkEntryPointsOffset<kArm64PointerSize>(root.reg());
+ // Loading the entrypoint does not require a load acquire since it is only changed when
+ // threads are suspended or running a checkpoint.
+ __ Ldr(temp, MemOperand(tr, entry_point_offset));
+
// /* GcRoot<mirror::Object> */ root = *(obj + offset)
if (fixup_label == nullptr) {
__ Ldr(root_reg, MemOperand(obj, offset));
@@ -5636,20 +5861,6 @@
"art::mirror::CompressedReference<mirror::Object> and int32_t "
"have different sizes.");
- Register temp = lr;
-
- // Slow path marking the GC root `root`. The entrypoint will alrady be loaded in temp.
- SlowPathCodeARM64* slow_path =
- new (GetGraph()->GetArena()) ReadBarrierMarkSlowPathARM64(instruction,
- root,
- LocationFrom(temp));
- codegen_->AddSlowPath(slow_path);
- const int32_t entry_point_offset =
- CodeGenerator::GetReadBarrierMarkEntryPointsOffset<kArm64PointerSize>(root.reg());
- // temp = Thread::Current()->pReadBarrierMarkReg ## root.reg()
- // Loading the entrypoint does not require a load acquire since it is only changed when
- // threads are suspended or running a checkpoint.
- __ Ldr(temp, MemOperand(tr, entry_point_offset));
// The entrypoint is null when the GC is not marking, this prevents one load compared to
// checking GetIsGcMarking.
__ Cbnz(temp, slow_path->GetEntryLabel());
@@ -5751,54 +5962,103 @@
// `instruction->IsArrayGet()` => `!use_load_acquire`.
DCHECK(!instruction->IsArrayGet() || !use_load_acquire);
- MacroAssembler* masm = GetVIXLAssembler();
- UseScratchRegisterScope temps(masm);
-
- // In slow path based read barriers, the read barrier call is
- // inserted after the original load. However, in fast path based
- // Baker's read barriers, we need to perform the load of
- // mirror::Object::monitor_ *before* the original reference load.
- // This load-load ordering is required by the read barrier.
- // The fast path/slow path (for Baker's algorithm) should look like:
+ // Query `art::Thread::Current()->GetIsGcMarking()` to decide
+ // whether we need to enter the slow path to mark the reference.
+ // Then, in the slow path, check the gray bit in the lock word of
+ // the reference's holder (`obj`) to decide whether to mark `ref` or
+ // not.
//
- // uint32_t rb_state = Lockword(obj->monitor_).ReadBarrierState();
- // lfence; // Load fence or artificial data dependency to prevent load-load reordering
- // HeapReference<Object> ref = *src; // Original reference load.
- // bool is_gray = (rb_state == ReadBarrier::GrayState());
- // if (is_gray) {
- // ref = ReadBarrier::Mark(ref); // Performed by runtime entrypoint slow path.
+ // Note that we do not actually check the value of `GetIsGcMarking()`;
+ // instead, we load into `temp2` the read barrier mark entry point
+ // corresponding to register `ref`. If `temp2` is null, it means
+ // that `GetIsGcMarking()` is false, and vice versa.
+ //
+ // temp2 = Thread::Current()->pReadBarrierMarkReg ## root.reg()
+ // if (temp2 != nullptr) { // <=> Thread::Current()->GetIsGcMarking()
+ // // Slow path.
+ // uint32_t rb_state = Lockword(obj->monitor_).ReadBarrierState();
+ // lfence; // Load fence or artificial data dependency to prevent load-load reordering
+ // HeapReference<mirror::Object> ref = *src; // Original reference load.
+ // bool is_gray = (rb_state == ReadBarrier::GrayState());
+ // if (is_gray) {
+ // ref = temp2(ref); // ref = ReadBarrier::Mark(ref); // Runtime entry point call.
+ // }
+ // } else {
+ // HeapReference<mirror::Object> ref = *src; // Original reference load.
// }
- //
- // Note: the original implementation in ReadBarrier::Barrier is
- // slightly more complex as it performs additional checks that we do
- // not do here for performance reasons.
+ // Slow path marking the object `ref` when the GC is marking. The
+ // entrypoint will already be loaded in `temp2`.
+ Register temp2 = lr;
+ Location temp2_loc = LocationFrom(temp2);
+ SlowPathCodeARM64* slow_path;
+ if (always_update_field) {
+ // LoadReferenceWithBakerReadBarrierAndUpdateFieldSlowPathARM64
+ // only supports address of the form `obj + field_offset`, where
+ // `obj` is a register and `field_offset` is a register. Thus
+ // `offset` and `scale_factor` above are expected to be null in
+ // this code path.
+ DCHECK_EQ(offset, 0u);
+ DCHECK_EQ(scale_factor, 0u); /* "times 1" */
+ Location field_offset = index;
+ slow_path =
+ new (GetGraph()->GetArena()) LoadReferenceWithBakerReadBarrierAndUpdateFieldSlowPathARM64(
+ instruction,
+ ref,
+ obj,
+ offset,
+ /* index */ field_offset,
+ scale_factor,
+ needs_null_check,
+ use_load_acquire,
+ temp,
+ /* entrypoint */ temp2_loc);
+ } else {
+ slow_path = new (GetGraph()->GetArena()) LoadReferenceWithBakerReadBarrierSlowPathARM64(
+ instruction,
+ ref,
+ obj,
+ offset,
+ index,
+ scale_factor,
+ needs_null_check,
+ use_load_acquire,
+ temp,
+ /* entrypoint */ temp2_loc);
+ }
+ AddSlowPath(slow_path);
+
+ // temp2 = Thread::Current()->pReadBarrierMarkReg ## ref.reg()
+ const int32_t entry_point_offset =
+ CodeGenerator::GetReadBarrierMarkEntryPointsOffset<kArm64PointerSize>(ref.reg());
+ // Loading the entrypoint does not require a load acquire since it is only changed when
+ // threads are suspended or running a checkpoint.
+ __ Ldr(temp2, MemOperand(tr, entry_point_offset));
+ // The entrypoint is null when the GC is not marking, this prevents one load compared to
+ // checking GetIsGcMarking.
+ __ Cbnz(temp2, slow_path->GetEntryLabel());
+ // Fast path: just load the reference.
+ GenerateRawReferenceLoad(
+ instruction, ref, obj, offset, index, scale_factor, needs_null_check, use_load_acquire);
+ __ Bind(slow_path->GetExitLabel());
+}
+
+void CodeGeneratorARM64::GenerateRawReferenceLoad(HInstruction* instruction,
+ Location ref,
+ Register obj,
+ uint32_t offset,
+ Location index,
+ size_t scale_factor,
+ bool needs_null_check,
+ bool use_load_acquire) {
+ DCHECK(obj.IsW());
Primitive::Type type = Primitive::kPrimNot;
Register ref_reg = RegisterFrom(ref, type);
- DCHECK(obj.IsW());
- uint32_t monitor_offset = mirror::Object::MonitorOffset().Int32Value();
- {
- // Ensure that between load and MaybeRecordImplicitNullCheck there are no pools emitted.
- EmissionCheckScope guard(GetVIXLAssembler(), kMaxMacroInstructionSizeInBytes);
- // /* int32_t */ monitor = obj->monitor_
- __ Ldr(temp, HeapOperand(obj, monitor_offset));
- if (needs_null_check) {
- MaybeRecordImplicitNullCheck(instruction);
- }
- }
- // /* LockWord */ lock_word = LockWord(monitor)
- static_assert(sizeof(LockWord) == sizeof(int32_t),
- "art::LockWord and int32_t have different sizes.");
+ // If needed, vixl::EmissionCheckScope guards are used to ensure
+ // that no pools are emitted between the load (macro) instruction
+ // and MaybeRecordImplicitNullCheck.
- // Introduce a dependency on the lock_word including rb_state,
- // to prevent load-load reordering, and without using
- // a memory barrier (which would be more expensive).
- // `obj` is unchanged by this operation, but its value now depends
- // on `temp`.
- __ Add(obj.X(), obj.X(), Operand(temp.X(), LSR, 32));
-
- // The actual reference load.
if (index.IsValid()) {
// Load types involving an "index": ArrayGet,
// UnsafeGetObject/UnsafeGetObjectVolatile and UnsafeCASObject
@@ -5813,59 +6073,50 @@
<< instruction->AsInvoke()->GetIntrinsic();
DCHECK_EQ(offset, 0u);
DCHECK_EQ(scale_factor, 0u);
- DCHECK_EQ(needs_null_check, 0u);
- // /* HeapReference<Object> */ ref = *(obj + index)
+ DCHECK_EQ(needs_null_check, false);
+ // /* HeapReference<mirror::Object> */ ref = *(obj + index)
MemOperand field = HeapOperand(obj, XRegisterFrom(index));
LoadAcquire(instruction, ref_reg, field, /* needs_null_check */ false);
} else {
- // ArrayGet and UnsafeGetObject intrinsics cases.
- // /* HeapReference<Object> */ ref = *(obj + offset + (index << scale_factor))
+ // ArrayGet and UnsafeGetObject and UnsafeCASObject intrinsics cases.
+ // /* HeapReference<mirror::Object> */ ref = *(obj + offset + (index << scale_factor))
if (index.IsConstant()) {
uint32_t computed_offset = offset + (Int64ConstantFrom(index) << scale_factor);
+ EmissionCheckScope guard(GetVIXLAssembler(), kMaxMacroInstructionSizeInBytes);
Load(type, ref_reg, HeapOperand(obj, computed_offset));
+ if (needs_null_check) {
+ MaybeRecordImplicitNullCheck(instruction);
+ }
} else {
- Register temp3 = temps.AcquireW();
- __ Add(temp3, obj, offset);
- Load(type, ref_reg, HeapOperand(temp3, XRegisterFrom(index), LSL, scale_factor));
- temps.Release(temp3);
+ UseScratchRegisterScope temps(GetVIXLAssembler());
+ Register temp = temps.AcquireW();
+ __ Add(temp, obj, offset);
+ {
+ EmissionCheckScope guard(GetVIXLAssembler(), kMaxMacroInstructionSizeInBytes);
+ Load(type, ref_reg, HeapOperand(temp, XRegisterFrom(index), LSL, scale_factor));
+ if (needs_null_check) {
+ MaybeRecordImplicitNullCheck(instruction);
+ }
+ }
}
}
} else {
- // /* HeapReference<Object> */ ref = *(obj + offset)
+ // /* HeapReference<mirror::Object> */ ref = *(obj + offset)
MemOperand field = HeapOperand(obj, offset);
if (use_load_acquire) {
- LoadAcquire(instruction, ref_reg, field, /* needs_null_check */ false);
+ // Implicit null checks are handled by CodeGeneratorARM64::LoadAcquire.
+ LoadAcquire(instruction, ref_reg, field, needs_null_check);
} else {
+ EmissionCheckScope guard(GetVIXLAssembler(), kMaxMacroInstructionSizeInBytes);
Load(type, ref_reg, field);
+ if (needs_null_check) {
+ MaybeRecordImplicitNullCheck(instruction);
+ }
}
}
// Object* ref = ref_addr->AsMirrorPtr()
GetAssembler()->MaybeUnpoisonHeapReference(ref_reg);
-
- // Slow path marking the object `ref` when it is gray.
- SlowPathCodeARM64* slow_path;
- if (always_update_field) {
- // ReadBarrierMarkAndUpdateFieldSlowPathARM64 only supports
- // address of the form `obj + field_offset`, where `obj` is a
- // register and `field_offset` is a register. Thus `offset` and
- // `scale_factor` above are expected to be null in this code path.
- DCHECK_EQ(offset, 0u);
- DCHECK_EQ(scale_factor, 0u); /* "times 1" */
- slow_path = new (GetGraph()->GetArena()) ReadBarrierMarkAndUpdateFieldSlowPathARM64(
- instruction, ref, obj, /* field_offset */ index, temp);
- } else {
- slow_path = new (GetGraph()->GetArena()) ReadBarrierMarkSlowPathARM64(instruction, ref);
- }
- AddSlowPath(slow_path);
-
- // if (rb_state == ReadBarrier::GrayState())
- // ref = ReadBarrier::Mark(ref);
- // Given the numeric representation, it's enough to check the low bit of the rb_state.
- static_assert(ReadBarrier::WhiteState() == 0, "Expecting white to have value 0");
- static_assert(ReadBarrier::GrayState() == 1, "Expecting gray to have value 1");
- __ Tbnz(temp, LockWord::kReadBarrierStateShift, slow_path->GetEntryLabel());
- __ Bind(slow_path->GetExitLabel());
}
void CodeGeneratorARM64::GenerateReadBarrierSlow(HInstruction* instruction,
diff --git a/compiler/optimizing/code_generator_arm64.h b/compiler/optimizing/code_generator_arm64.h
index 5faf29a..7471cd5 100644
--- a/compiler/optimizing/code_generator_arm64.h
+++ b/compiler/optimizing/code_generator_arm64.h
@@ -616,8 +616,8 @@
Location index,
vixl::aarch64::Register temp,
bool needs_null_check);
- // Factored implementation used by GenerateFieldLoadWithBakerReadBarrier
- // and GenerateArrayLoadWithBakerReadBarrier.
+ // Factored implementation, used by GenerateFieldLoadWithBakerReadBarrier,
+ // GenerateArrayLoadWithBakerReadBarrier and some intrinsics.
//
// Load the object reference located at the address
// `obj + offset + (index << scale_factor)`, held by object `obj`, into
@@ -636,6 +636,16 @@
bool use_load_acquire,
bool always_update_field = false);
+ // Generate a heap reference load (with no read barrier).
+ void GenerateRawReferenceLoad(HInstruction* instruction,
+ Location ref,
+ vixl::aarch64::Register obj,
+ uint32_t offset,
+ Location index,
+ size_t scale_factor,
+ bool needs_null_check,
+ bool use_load_acquire);
+
// Generate a read barrier for a heap reference within `instruction`
// using a slow path.
//
@@ -761,8 +771,6 @@
ArenaDeque<PcRelativePatchInfo> pc_relative_type_patches_;
// PC-relative type patch info for kBssEntry.
ArenaDeque<PcRelativePatchInfo> type_bss_entry_patches_;
- // Deduplication map for patchable boot image addresses.
- Uint32ToLiteralMap boot_image_address_patches_;
// Patches for string literals in JIT compiled code.
StringToLiteralMap jit_string_patches_;
diff --git a/compiler/optimizing/code_generator_arm_vixl.cc b/compiler/optimizing/code_generator_arm_vixl.cc
index 6bfbe4a..a1f30cd 100644
--- a/compiler/optimizing/code_generator_arm_vixl.cc
+++ b/compiler/optimizing/code_generator_arm_vixl.cc
@@ -42,6 +42,7 @@
using helpers::DWARFReg;
using helpers::HighDRegisterFrom;
using helpers::HighRegisterFrom;
+using helpers::InputDRegisterAt;
using helpers::InputOperandAt;
using helpers::InputRegister;
using helpers::InputRegisterAt;
@@ -53,6 +54,7 @@
using helpers::LocationFrom;
using helpers::LowRegisterFrom;
using helpers::LowSRegisterFrom;
+using helpers::OperandFrom;
using helpers::OutputRegister;
using helpers::OutputSRegister;
using helpers::OutputVRegister;
@@ -657,52 +659,25 @@
DISALLOW_COPY_AND_ASSIGN(ArraySetSlowPathARMVIXL);
};
-// Slow path marking an object reference `ref` during a read
-// barrier. The field `obj.field` in the object `obj` holding this
-// reference does not get updated by this slow path after marking (see
-// ReadBarrierMarkAndUpdateFieldSlowPathARM below for that).
+// Abstract base class for read barrier slow paths marking a reference
+// `ref`.
//
-// This means that after the execution of this slow path, `ref` will
-// always be up-to-date, but `obj.field` may not; i.e., after the
-// flip, `ref` will be a to-space reference, but `obj.field` will
-// probably still be a from-space reference (unless it gets updated by
-// another thread, or if another thread installed another object
-// reference (different from `ref`) in `obj.field`).
-class ReadBarrierMarkSlowPathARMVIXL : public SlowPathCodeARMVIXL {
- public:
- ReadBarrierMarkSlowPathARMVIXL(HInstruction* instruction,
- Location ref,
- Location entrypoint = Location::NoLocation())
+// Argument `entrypoint` must be a register location holding the read
+// barrier marking runtime entry point to be invoked.
+class ReadBarrierMarkSlowPathBaseARMVIXL : public SlowPathCodeARMVIXL {
+ protected:
+ ReadBarrierMarkSlowPathBaseARMVIXL(HInstruction* instruction, Location ref, Location entrypoint)
: SlowPathCodeARMVIXL(instruction), ref_(ref), entrypoint_(entrypoint) {
DCHECK(kEmitCompilerReadBarrier);
}
- const char* GetDescription() const OVERRIDE { return "ReadBarrierMarkSlowPathARMVIXL"; }
+ const char* GetDescription() const OVERRIDE { return "ReadBarrierMarkSlowPathBaseARMVIXL"; }
- void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
- LocationSummary* locations = instruction_->GetLocations();
+ // Generate assembly code calling the read barrier marking runtime
+ // entry point (ReadBarrierMarkRegX).
+ void GenerateReadBarrierMarkRuntimeCall(CodeGenerator* codegen) {
vixl32::Register ref_reg = RegisterFrom(ref_);
- DCHECK(locations->CanCall());
- DCHECK(!locations->GetLiveRegisters()->ContainsCoreRegister(ref_reg.GetCode())) << ref_reg;
- DCHECK(instruction_->IsInstanceFieldGet() ||
- instruction_->IsStaticFieldGet() ||
- instruction_->IsArrayGet() ||
- instruction_->IsArraySet() ||
- instruction_->IsLoadClass() ||
- instruction_->IsLoadString() ||
- instruction_->IsInstanceOf() ||
- instruction_->IsCheckCast() ||
- (instruction_->IsInvokeVirtual() && instruction_->GetLocations()->Intrinsified()) ||
- (instruction_->IsInvokeStaticOrDirect() && instruction_->GetLocations()->Intrinsified()))
- << "Unexpected instruction in read barrier marking slow path: "
- << instruction_->DebugName();
- // The read barrier instrumentation of object ArrayGet
- // instructions does not support the HIntermediateAddress
- // instruction.
- DCHECK(!(instruction_->IsArrayGet() &&
- instruction_->AsArrayGet()->GetArray()->IsIntermediateAddress()));
- __ Bind(GetEntryLabel());
// No need to save live registers; it's taken care of by the
// entrypoint. Also, there is no need to update the stack mask,
// as this runtime call will not trigger a garbage collection.
@@ -732,53 +707,108 @@
arm_codegen->ValidateInvokeRuntimeWithoutRecordingPcInfo(instruction_, this);
__ Blx(RegisterFrom(entrypoint_));
} else {
+ // Entrypoint is not already loaded, load from the thread.
int32_t entry_point_offset =
CodeGenerator::GetReadBarrierMarkEntryPointsOffset<kArmPointerSize>(ref_reg.GetCode());
// This runtime call does not require a stack map.
arm_codegen->InvokeRuntimeWithoutRecordingPcInfo(entry_point_offset, instruction_, this);
}
- __ B(GetExitLabel());
}
- private:
// The location (register) of the marked object reference.
const Location ref_;
// The location of the entrypoint if already loaded.
const Location entrypoint_;
- DISALLOW_COPY_AND_ASSIGN(ReadBarrierMarkSlowPathARMVIXL);
+ private:
+ DISALLOW_COPY_AND_ASSIGN(ReadBarrierMarkSlowPathBaseARMVIXL);
};
-// Slow path marking an object reference `ref` during a read barrier,
-// and if needed, atomically updating the field `obj.field` in the
-// object `obj` holding this reference after marking (contrary to
-// ReadBarrierMarkSlowPathARM above, which never tries to update
-// `obj.field`).
+// Slow path marking an object reference `ref` during a read
+// barrier. The field `obj.field` in the object `obj` holding this
+// reference does not get updated by this slow path after marking.
//
-// This means that after the execution of this slow path, both `ref`
-// and `obj.field` will be up-to-date; i.e., after the flip, both will
-// hold the same to-space reference (unless another thread installed
-// another object reference (different from `ref`) in `obj.field`).
-class ReadBarrierMarkAndUpdateFieldSlowPathARMVIXL : public SlowPathCodeARMVIXL {
+// This means that after the execution of this slow path, `ref` will
+// always be up-to-date, but `obj.field` may not; i.e., after the
+// flip, `ref` will be a to-space reference, but `obj.field` will
+// probably still be a from-space reference (unless it gets updated by
+// another thread, or if another thread installed another object
+// reference (different from `ref`) in `obj.field`).
+//
+// If `entrypoint` is a valid location it is assumed to already be
+// holding the entrypoint. The case where the entrypoint is passed in
+// is when the decision to mark is based on whether the GC is marking.
+class ReadBarrierMarkSlowPathARMVIXL : public ReadBarrierMarkSlowPathBaseARMVIXL {
public:
- ReadBarrierMarkAndUpdateFieldSlowPathARMVIXL(HInstruction* instruction,
- Location ref,
- vixl32::Register obj,
- Location field_offset,
- vixl32::Register temp1,
- vixl32::Register temp2)
- : SlowPathCodeARMVIXL(instruction),
- ref_(ref),
- obj_(obj),
- field_offset_(field_offset),
- temp1_(temp1),
- temp2_(temp2) {
+ ReadBarrierMarkSlowPathARMVIXL(HInstruction* instruction,
+ Location ref,
+ Location entrypoint = Location::NoLocation())
+ : ReadBarrierMarkSlowPathBaseARMVIXL(instruction, ref, entrypoint) {
DCHECK(kEmitCompilerReadBarrier);
}
+ const char* GetDescription() const OVERRIDE { return "ReadBarrierMarkSlowPathARMVIXL"; }
+
+ void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
+ LocationSummary* locations = instruction_->GetLocations();
+ DCHECK(locations->CanCall());
+ DCHECK(ref_.IsRegister()) << ref_;
+ DCHECK(!locations->GetLiveRegisters()->ContainsCoreRegister(ref_.reg())) << ref_.reg();
+ DCHECK(instruction_->IsLoadClass() || instruction_->IsLoadString())
+ << "Unexpected instruction in read barrier marking slow path: "
+ << instruction_->DebugName();
+
+ __ Bind(GetEntryLabel());
+ GenerateReadBarrierMarkRuntimeCall(codegen);
+ __ B(GetExitLabel());
+ }
+
+ private:
+ DISALLOW_COPY_AND_ASSIGN(ReadBarrierMarkSlowPathARMVIXL);
+};
+
+// Slow path loading `obj`'s lock word, loading a reference from
+// object `*(obj + offset + (index << scale_factor))` into `ref`, and
+// marking `ref` if `obj` is gray according to the lock word (Baker
+// read barrier). The field `obj.field` in the object `obj` holding
+// this reference does not get updated by this slow path after marking
+// (see LoadReferenceWithBakerReadBarrierAndUpdateFieldSlowPathARMVIXL
+// below for that).
+//
+// This means that after the execution of this slow path, `ref` will
+// always be up-to-date, but `obj.field` may not; i.e., after the
+// flip, `ref` will be a to-space reference, but `obj.field` will
+// probably still be a from-space reference (unless it gets updated by
+// another thread, or if another thread installed another object
+// reference (different from `ref`) in `obj.field`).
+//
+// Argument `entrypoint` must be a register location holding the read
+// barrier marking runtime entry point to be invoked.
+class LoadReferenceWithBakerReadBarrierSlowPathARMVIXL : public ReadBarrierMarkSlowPathBaseARMVIXL {
+ public:
+ LoadReferenceWithBakerReadBarrierSlowPathARMVIXL(HInstruction* instruction,
+ Location ref,
+ vixl32::Register obj,
+ uint32_t offset,
+ Location index,
+ ScaleFactor scale_factor,
+ bool needs_null_check,
+ vixl32::Register temp,
+ Location entrypoint)
+ : ReadBarrierMarkSlowPathBaseARMVIXL(instruction, ref, entrypoint),
+ obj_(obj),
+ offset_(offset),
+ index_(index),
+ scale_factor_(scale_factor),
+ needs_null_check_(needs_null_check),
+ temp_(temp) {
+ DCHECK(kEmitCompilerReadBarrier);
+ DCHECK(kUseBakerReadBarrier);
+ }
+
const char* GetDescription() const OVERRIDE {
- return "ReadBarrierMarkAndUpdateFieldSlowPathARMVIXL";
+ return "LoadReferenceWithBakerReadBarrierSlowPathARMVIXL";
}
void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
@@ -786,64 +816,233 @@
vixl32::Register ref_reg = RegisterFrom(ref_);
DCHECK(locations->CanCall());
DCHECK(!locations->GetLiveRegisters()->ContainsCoreRegister(ref_reg.GetCode())) << ref_reg;
- // This slow path is only used by the UnsafeCASObject intrinsic.
+ DCHECK(instruction_->IsInstanceFieldGet() ||
+ instruction_->IsStaticFieldGet() ||
+ instruction_->IsArrayGet() ||
+ instruction_->IsArraySet() ||
+ instruction_->IsInstanceOf() ||
+ instruction_->IsCheckCast() ||
+ (instruction_->IsInvokeVirtual() && instruction_->GetLocations()->Intrinsified()) ||
+ (instruction_->IsInvokeStaticOrDirect() && instruction_->GetLocations()->Intrinsified()))
+ << "Unexpected instruction in read barrier marking slow path: "
+ << instruction_->DebugName();
+ // The read barrier instrumentation of object ArrayGet
+ // instructions does not support the HIntermediateAddress
+ // instruction.
+ DCHECK(!(instruction_->IsArrayGet() &&
+ instruction_->AsArrayGet()->GetArray()->IsIntermediateAddress()));
+
+ // Temporary register `temp_`, used to store the lock word, must
+ // not be IP, as we may use it to emit the reference load (in the
+ // call to GenerateRawReferenceLoad below), and we need the lock
+ // word to still be in `temp_` after the reference load.
+ DCHECK(!temp_.Is(ip));
+
+ __ Bind(GetEntryLabel());
+
+ // When using MaybeGenerateReadBarrierSlow, the read barrier call is
+ // inserted after the original load. However, in fast path based
+ // Baker's read barriers, we need to perform the load of
+ // mirror::Object::monitor_ *before* the original reference load.
+ // This load-load ordering is required by the read barrier.
+ // The fast path/slow path (for Baker's algorithm) should look like:
+ //
+ // uint32_t rb_state = Lockword(obj->monitor_).ReadBarrierState();
+ // lfence; // Load fence or artificial data dependency to prevent load-load reordering
+ // HeapReference<mirror::Object> ref = *src; // Original reference load.
+ // bool is_gray = (rb_state == ReadBarrier::GrayState());
+ // if (is_gray) {
+ // ref = entrypoint(ref); // ref = ReadBarrier::Mark(ref); // Runtime entry point call.
+ // }
+ //
+ // Note: the original implementation in ReadBarrier::Barrier is
+ // slightly more complex as it performs additional checks that we do
+ // not do here for performance reasons.
+
+ CodeGeneratorARMVIXL* arm_codegen = down_cast<CodeGeneratorARMVIXL*>(codegen);
+
+ // /* int32_t */ monitor = obj->monitor_
+ uint32_t monitor_offset = mirror::Object::MonitorOffset().Int32Value();
+ arm_codegen->GetAssembler()->LoadFromOffset(kLoadWord, temp_, obj_, monitor_offset);
+ if (needs_null_check_) {
+ codegen->MaybeRecordImplicitNullCheck(instruction_);
+ }
+ // /* LockWord */ lock_word = LockWord(monitor)
+ static_assert(sizeof(LockWord) == sizeof(int32_t),
+ "art::LockWord and int32_t have different sizes.");
+
+ // Introduce a dependency on the lock_word including the rb_state,
+ // which shall prevent load-load reordering without using
+ // a memory barrier (which would be more expensive).
+ // `obj` is unchanged by this operation, but its value now depends
+ // on `temp`.
+ __ Add(obj_, obj_, Operand(temp_, ShiftType::LSR, 32));
+
+ // The actual reference load.
+ // A possible implicit null check has already been handled above.
+ arm_codegen->GenerateRawReferenceLoad(
+ instruction_, ref_, obj_, offset_, index_, scale_factor_, /* needs_null_check */ false);
+
+ // Mark the object `ref` when `obj` is gray.
+ //
+ // if (rb_state == ReadBarrier::GrayState())
+ // ref = ReadBarrier::Mark(ref);
+ //
+ // Given the numeric representation, it's enough to check the low bit of the
+ // rb_state. We do that by shifting the bit out of the lock word with LSRS
+ // which can be a 16-bit instruction unlike the TST immediate.
+ static_assert(ReadBarrier::WhiteState() == 0, "Expecting white to have value 0");
+ static_assert(ReadBarrier::GrayState() == 1, "Expecting gray to have value 1");
+ __ Lsrs(temp_, temp_, LockWord::kReadBarrierStateShift + 1);
+ __ B(cc, GetExitLabel()); // Carry flag is the last bit shifted out by LSRS.
+ GenerateReadBarrierMarkRuntimeCall(codegen);
+
+ __ B(GetExitLabel());
+ }
+
+ private:
+ // The register containing the object holding the marked object reference field.
+ vixl32::Register obj_;
+ // The offset, index and scale factor to access the reference in `obj_`.
+ uint32_t offset_;
+ Location index_;
+ ScaleFactor scale_factor_;
+ // Is a null check required?
+ bool needs_null_check_;
+ // A temporary register used to hold the lock word of `obj_`.
+ vixl32::Register temp_;
+
+ DISALLOW_COPY_AND_ASSIGN(LoadReferenceWithBakerReadBarrierSlowPathARMVIXL);
+};
+
+// Slow path loading `obj`'s lock word, loading a reference from
+// object `*(obj + offset + (index << scale_factor))` into `ref`, and
+// marking `ref` if `obj` is gray according to the lock word (Baker
+// read barrier). If needed, this slow path also atomically updates
+// the field `obj.field` in the object `obj` holding this reference
+// after marking (contrary to
+// LoadReferenceWithBakerReadBarrierSlowPathARMVIXL above, which never
+// tries to update `obj.field`).
+//
+// This means that after the execution of this slow path, both `ref`
+// and `obj.field` will be up-to-date; i.e., after the flip, both will
+// hold the same to-space reference (unless another thread installed
+// another object reference (different from `ref`) in `obj.field`).
+//
+//
+// Argument `entrypoint` must be a register location holding the read
+// barrier marking runtime entry point to be invoked.
+class LoadReferenceWithBakerReadBarrierAndUpdateFieldSlowPathARMVIXL
+ : public ReadBarrierMarkSlowPathBaseARMVIXL {
+ public:
+ LoadReferenceWithBakerReadBarrierAndUpdateFieldSlowPathARMVIXL(HInstruction* instruction,
+ Location ref,
+ vixl32::Register obj,
+ uint32_t offset,
+ Location index,
+ ScaleFactor scale_factor,
+ bool needs_null_check,
+ vixl32::Register temp1,
+ vixl32::Register temp2,
+ Location entrypoint)
+ : ReadBarrierMarkSlowPathBaseARMVIXL(instruction, ref, entrypoint),
+ obj_(obj),
+ offset_(offset),
+ index_(index),
+ scale_factor_(scale_factor),
+ needs_null_check_(needs_null_check),
+ temp1_(temp1),
+ temp2_(temp2) {
+ DCHECK(kEmitCompilerReadBarrier);
+ DCHECK(kUseBakerReadBarrier);
+ }
+
+ const char* GetDescription() const OVERRIDE {
+ return "LoadReferenceWithBakerReadBarrierAndUpdateFieldSlowPathARMVIXL";
+ }
+
+ void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
+ LocationSummary* locations = instruction_->GetLocations();
+ vixl32::Register ref_reg = RegisterFrom(ref_);
+ DCHECK(locations->CanCall());
+ DCHECK(!locations->GetLiveRegisters()->ContainsCoreRegister(ref_reg.GetCode())) << ref_reg;
+ DCHECK_NE(ref_.reg(), LocationFrom(temp1_).reg());
+
+ // This slow path is only used by the UnsafeCASObject intrinsic at the moment.
DCHECK((instruction_->IsInvokeVirtual() && instruction_->GetLocations()->Intrinsified()))
<< "Unexpected instruction in read barrier marking and field updating slow path: "
<< instruction_->DebugName();
DCHECK(instruction_->GetLocations()->Intrinsified());
DCHECK_EQ(instruction_->AsInvoke()->GetIntrinsic(), Intrinsics::kUnsafeCASObject);
- DCHECK(field_offset_.IsRegisterPair()) << field_offset_;
+ DCHECK_EQ(offset_, 0u);
+ DCHECK_EQ(scale_factor_, ScaleFactor::TIMES_1);
+ Location field_offset = index_;
+ DCHECK(field_offset.IsRegisterPair()) << field_offset;
+
+ // Temporary register `temp1_`, used to store the lock word, must
+ // not be IP, as we may use it to emit the reference load (in the
+ // call to GenerateRawReferenceLoad below), and we need the lock
+ // word to still be in `temp1_` after the reference load.
+ DCHECK(!temp1_.Is(ip));
__ Bind(GetEntryLabel());
- // Save the old reference.
+ CodeGeneratorARMVIXL* arm_codegen = down_cast<CodeGeneratorARMVIXL*>(codegen);
+
+ // /* int32_t */ monitor = obj->monitor_
+ uint32_t monitor_offset = mirror::Object::MonitorOffset().Int32Value();
+ arm_codegen->GetAssembler()->LoadFromOffset(kLoadWord, temp1_, obj_, monitor_offset);
+ if (needs_null_check_) {
+ codegen->MaybeRecordImplicitNullCheck(instruction_);
+ }
+ // /* LockWord */ lock_word = LockWord(monitor)
+ static_assert(sizeof(LockWord) == sizeof(int32_t),
+ "art::LockWord and int32_t have different sizes.");
+
+ // Introduce a dependency on the lock_word including the rb_state,
+ // which shall prevent load-load reordering without using
+ // a memory barrier (which would be more expensive).
+ // `obj` is unchanged by this operation, but its value now depends
+ // on `temp`.
+ __ Add(obj_, obj_, Operand(temp1_, ShiftType::LSR, 32));
+
+ // The actual reference load.
+ // A possible implicit null check has already been handled above.
+ arm_codegen->GenerateRawReferenceLoad(
+ instruction_, ref_, obj_, offset_, index_, scale_factor_, /* needs_null_check */ false);
+
+ // Mark the object `ref` when `obj` is gray.
+ //
+ // if (rb_state == ReadBarrier::GrayState())
+ // ref = ReadBarrier::Mark(ref);
+ //
+ // Given the numeric representation, it's enough to check the low bit of the
+ // rb_state. We do that by shifting the bit out of the lock word with LSRS
+ // which can be a 16-bit instruction unlike the TST immediate.
+ static_assert(ReadBarrier::WhiteState() == 0, "Expecting white to have value 0");
+ static_assert(ReadBarrier::GrayState() == 1, "Expecting gray to have value 1");
+ __ Lsrs(temp1_, temp1_, LockWord::kReadBarrierStateShift + 1);
+ __ B(cc, GetExitLabel()); // Carry flag is the last bit shifted out by LSRS.
+
+ // Save the old value of the reference before marking it.
// Note that we cannot use IP to save the old reference, as IP is
// used internally by the ReadBarrierMarkRegX entry point, and we
// need the old reference after the call to that entry point.
DCHECK(!temp1_.Is(ip));
__ Mov(temp1_, ref_reg);
- // No need to save live registers; it's taken care of by the
- // entrypoint. Also, there is no need to update the stack mask,
- // as this runtime call will not trigger a garbage collection.
- CodeGeneratorARMVIXL* arm_codegen = down_cast<CodeGeneratorARMVIXL*>(codegen);
- DCHECK(!ref_reg.Is(sp));
- DCHECK(!ref_reg.Is(lr));
- DCHECK(!ref_reg.Is(pc));
- // IP is used internally by the ReadBarrierMarkRegX entry point
- // as a temporary, it cannot be the entry point's input/output.
- DCHECK(!ref_reg.Is(ip));
- DCHECK(ref_reg.IsRegister()) << ref_reg;
- // "Compact" slow path, saving two moves.
- //
- // Instead of using the standard runtime calling convention (input
- // and output in R0):
- //
- // R0 <- ref
- // R0 <- ReadBarrierMark(R0)
- // ref <- R0
- //
- // we just use rX (the register containing `ref`) as input and output
- // of a dedicated entrypoint:
- //
- // rX <- ReadBarrierMarkRegX(rX)
- //
- int32_t entry_point_offset =
- CodeGenerator::GetReadBarrierMarkEntryPointsOffset<kArmPointerSize>(ref_reg.GetCode());
- // This runtime call does not require a stack map.
- arm_codegen->InvokeRuntimeWithoutRecordingPcInfo(entry_point_offset, instruction_, this);
+ GenerateReadBarrierMarkRuntimeCall(codegen);
// If the new reference is different from the old reference,
- // update the field in the holder (`*(obj_ + field_offset_)`).
+ // update the field in the holder (`*(obj_ + field_offset)`).
//
// Note that this field could also hold a different object, if
// another thread had concurrently changed it. In that case, the
// LDREX/SUBS/ITNE sequence of instructions in the compare-and-set
// (CAS) operation below would abort the CAS, leaving the field
// as-is.
- vixl32::Label done;
__ Cmp(temp1_, ref_reg);
- __ B(eq, &done, /* far_target */ false);
+ __ B(eq, GetExitLabel());
// Update the the holder's field atomically. This may fail if
// mutator updates before us, but it's OK. This is achieved
@@ -857,7 +1056,7 @@
// The UnsafeCASObject intrinsic uses a register pair as field
// offset ("long offset"), of which only the low part contains
// data.
- vixl32::Register offset = LowRegisterFrom(field_offset_);
+ vixl32::Register offset = LowRegisterFrom(field_offset);
vixl32::Register expected = temp1_;
vixl32::Register value = ref_reg;
vixl32::Register tmp_ptr = temps.Acquire(); // Pointer to actual memory.
@@ -913,22 +1112,27 @@
}
}
- __ Bind(&done);
__ B(GetExitLabel());
}
private:
- // The location (register) of the marked object reference.
- const Location ref_;
// The register containing the object holding the marked object reference field.
const vixl32::Register obj_;
- // The location of the offset of the marked reference field within `obj_`.
- Location field_offset_;
-
+ // The offset, index and scale factor to access the reference in `obj_`.
+ uint32_t offset_;
+ Location index_;
+ ScaleFactor scale_factor_;
+ // Is a null check required?
+ bool needs_null_check_;
+ // A temporary register used to hold the lock word of `obj_`; and
+ // also to hold the original reference value, when the reference is
+ // marked.
const vixl32::Register temp1_;
+ // A temporary register used in the implementation of the CAS, to
+ // update the object's reference field.
const vixl32::Register temp2_;
- DISALLOW_COPY_AND_ASSIGN(ReadBarrierMarkAndUpdateFieldSlowPathARMVIXL);
+ DISALLOW_COPY_AND_ASSIGN(LoadReferenceWithBakerReadBarrierAndUpdateFieldSlowPathARMVIXL);
};
// Slow path generating a read barrier for a heap reference.
@@ -1450,8 +1654,317 @@
}
}
+static void GenerateVcmp(HInstruction* instruction, CodeGeneratorARMVIXL* codegen) {
+ const Location rhs_loc = instruction->GetLocations()->InAt(1);
+ if (rhs_loc.IsConstant()) {
+ // 0.0 is the only immediate that can be encoded directly in
+ // a VCMP instruction.
+ //
+ // Both the JLS (section 15.20.1) and the JVMS (section 6.5)
+ // specify that in a floating-point comparison, positive zero
+ // and negative zero are considered equal, so we can use the
+ // literal 0.0 for both cases here.
+ //
+ // Note however that some methods (Float.equal, Float.compare,
+ // Float.compareTo, Double.equal, Double.compare,
+ // Double.compareTo, Math.max, Math.min, StrictMath.max,
+ // StrictMath.min) consider 0.0 to be (strictly) greater than
+ // -0.0. So if we ever translate calls to these methods into a
+ // HCompare instruction, we must handle the -0.0 case with
+ // care here.
+ DCHECK(rhs_loc.GetConstant()->IsArithmeticZero());
+
+ const Primitive::Type type = instruction->InputAt(0)->GetType();
+
+ if (type == Primitive::kPrimFloat) {
+ __ Vcmp(F32, InputSRegisterAt(instruction, 0), 0.0);
+ } else {
+ DCHECK_EQ(type, Primitive::kPrimDouble);
+ __ Vcmp(F64, InputDRegisterAt(instruction, 0), 0.0);
+ }
+ } else {
+ __ Vcmp(InputVRegisterAt(instruction, 0), InputVRegisterAt(instruction, 1));
+ }
+}
+
+static vixl32::Condition GenerateLongTestConstant(HCondition* condition,
+ bool invert,
+ CodeGeneratorARMVIXL* codegen) {
+ DCHECK_EQ(condition->GetLeft()->GetType(), Primitive::kPrimLong);
+
+ const LocationSummary* const locations = condition->GetLocations();
+ IfCondition cond = invert ? condition->GetOppositeCondition() : condition->GetCondition();
+ vixl32::Condition ret = eq;
+ const Location left = locations->InAt(0);
+ const Location right = locations->InAt(1);
+
+ DCHECK(right.IsConstant());
+
+ const vixl32::Register left_high = HighRegisterFrom(left);
+ const vixl32::Register left_low = LowRegisterFrom(left);
+ int64_t value = Int64ConstantFrom(right);
+
+ switch (cond) {
+ case kCondEQ:
+ case kCondNE:
+ case kCondB:
+ case kCondBE:
+ case kCondA:
+ case kCondAE: {
+ __ Cmp(left_high, High32Bits(value));
+
+ ExactAssemblyScope guard(codegen->GetVIXLAssembler(),
+ 2 * vixl32::k16BitT32InstructionSizeInBytes,
+ CodeBufferCheckScope::kExactSize);
+
+ __ it(eq);
+ __ cmp(eq, left_low, Low32Bits(value));
+ ret = ARMUnsignedCondition(cond);
+ break;
+ }
+ case kCondLE:
+ case kCondGT:
+ // Trivially true or false.
+ if (value == std::numeric_limits<int64_t>::max()) {
+ __ Cmp(left_low, left_low);
+ ret = cond == kCondLE ? eq : ne;
+ break;
+ }
+
+ if (cond == kCondLE) {
+ cond = kCondLT;
+ } else {
+ DCHECK_EQ(cond, kCondGT);
+ cond = kCondGE;
+ }
+
+ value++;
+ FALLTHROUGH_INTENDED;
+ case kCondGE:
+ case kCondLT: {
+ UseScratchRegisterScope temps(codegen->GetVIXLAssembler());
+
+ __ Cmp(left_low, Low32Bits(value));
+ __ Sbcs(temps.Acquire(), left_high, High32Bits(value));
+ ret = ARMCondition(cond);
+ break;
+ }
+ default:
+ LOG(FATAL) << "Unreachable";
+ UNREACHABLE();
+ }
+
+ return ret;
+}
+
+static vixl32::Condition GenerateLongTest(HCondition* condition,
+ bool invert,
+ CodeGeneratorARMVIXL* codegen) {
+ DCHECK_EQ(condition->GetLeft()->GetType(), Primitive::kPrimLong);
+
+ const LocationSummary* const locations = condition->GetLocations();
+ IfCondition cond = invert ? condition->GetOppositeCondition() : condition->GetCondition();
+ vixl32::Condition ret = eq;
+ Location left = locations->InAt(0);
+ Location right = locations->InAt(1);
+
+ DCHECK(right.IsRegisterPair());
+
+ switch (cond) {
+ case kCondEQ:
+ case kCondNE:
+ case kCondB:
+ case kCondBE:
+ case kCondA:
+ case kCondAE: {
+ __ Cmp(HighRegisterFrom(left), HighRegisterFrom(right));
+
+ ExactAssemblyScope guard(codegen->GetVIXLAssembler(),
+ 2 * vixl32::k16BitT32InstructionSizeInBytes,
+ CodeBufferCheckScope::kExactSize);
+
+ __ it(eq);
+ __ cmp(eq, LowRegisterFrom(left), LowRegisterFrom(right));
+ ret = ARMUnsignedCondition(cond);
+ break;
+ }
+ case kCondLE:
+ case kCondGT:
+ if (cond == kCondLE) {
+ cond = kCondGE;
+ } else {
+ DCHECK_EQ(cond, kCondGT);
+ cond = kCondLT;
+ }
+
+ std::swap(left, right);
+ FALLTHROUGH_INTENDED;
+ case kCondGE:
+ case kCondLT: {
+ UseScratchRegisterScope temps(codegen->GetVIXLAssembler());
+
+ __ Cmp(LowRegisterFrom(left), LowRegisterFrom(right));
+ __ Sbcs(temps.Acquire(), HighRegisterFrom(left), HighRegisterFrom(right));
+ ret = ARMCondition(cond);
+ break;
+ }
+ default:
+ LOG(FATAL) << "Unreachable";
+ UNREACHABLE();
+ }
+
+ return ret;
+}
+
+static vixl32::Condition GenerateTest(HInstruction* instruction,
+ Location loc,
+ bool invert,
+ CodeGeneratorARMVIXL* codegen) {
+ DCHECK(!instruction->IsConstant());
+
+ vixl32::Condition ret = invert ? eq : ne;
+
+ if (IsBooleanValueOrMaterializedCondition(instruction)) {
+ __ Cmp(RegisterFrom(loc), 0);
+ } else {
+ HCondition* const condition = instruction->AsCondition();
+ const Primitive::Type type = condition->GetLeft()->GetType();
+ const IfCondition cond = invert ? condition->GetOppositeCondition() : condition->GetCondition();
+
+ if (type == Primitive::kPrimLong) {
+ ret = condition->GetLocations()->InAt(1).IsConstant()
+ ? GenerateLongTestConstant(condition, invert, codegen)
+ : GenerateLongTest(condition, invert, codegen);
+ } else if (Primitive::IsFloatingPointType(type)) {
+ GenerateVcmp(condition, codegen);
+ __ Vmrs(RegisterOrAPSR_nzcv(kPcCode), FPSCR);
+ ret = ARMFPCondition(cond, condition->IsGtBias());
+ } else {
+ DCHECK(Primitive::IsIntegralType(type) || type == Primitive::kPrimNot) << type;
+ __ Cmp(InputRegisterAt(condition, 0), InputOperandAt(condition, 1));
+ ret = ARMCondition(cond);
+ }
+ }
+
+ return ret;
+}
+
+static bool CanGenerateTest(HInstruction* condition, ArmVIXLAssembler* assembler) {
+ if (!IsBooleanValueOrMaterializedCondition(condition)) {
+ const HCondition* const cond = condition->AsCondition();
+
+ if (cond->GetLeft()->GetType() == Primitive::kPrimLong) {
+ const LocationSummary* const locations = cond->GetLocations();
+ const IfCondition c = cond->GetCondition();
+
+ if (locations->InAt(1).IsConstant()) {
+ const int64_t value = Int64ConstantFrom(locations->InAt(1));
+
+ if (c < kCondLT || c > kCondGE) {
+ // Since IT blocks longer than a 16-bit instruction are deprecated by ARMv8,
+ // we check that the least significant half of the first input to be compared
+ // is in a low register (the other half is read outside an IT block), and
+ // the constant fits in an 8-bit unsigned integer, so that a 16-bit CMP
+ // encoding can be used.
+ if (!LowRegisterFrom(locations->InAt(0)).IsLow() || !IsUint<8>(Low32Bits(value))) {
+ return false;
+ }
+ // TODO(VIXL): The rest of the checks are there to keep the backend in sync with
+ // the previous one, but are not strictly necessary.
+ } else if (c == kCondLE || c == kCondGT) {
+ if (value < std::numeric_limits<int64_t>::max() &&
+ !assembler->ShifterOperandCanHold(SBC, High32Bits(value + 1), kCcSet)) {
+ return false;
+ }
+ } else if (!assembler->ShifterOperandCanHold(SBC, High32Bits(value), kCcSet)) {
+ return false;
+ }
+ }
+ }
+ }
+
+ return true;
+}
+
+static bool CanEncodeConstantAs8BitImmediate(HConstant* constant) {
+ const Primitive::Type type = constant->GetType();
+ bool ret = false;
+
+ DCHECK(Primitive::IsIntegralType(type) || type == Primitive::kPrimNot) << type;
+
+ if (type == Primitive::kPrimLong) {
+ const uint64_t value = Uint64ConstantFrom(constant);
+
+ ret = IsUint<8>(Low32Bits(value)) && IsUint<8>(High32Bits(value));
+ } else {
+ ret = IsUint<8>(Int32ConstantFrom(constant));
+ }
+
+ return ret;
+}
+
+static Location Arm8BitEncodableConstantOrRegister(HInstruction* constant) {
+ DCHECK(!Primitive::IsFloatingPointType(constant->GetType()));
+
+ if (constant->IsConstant() && CanEncodeConstantAs8BitImmediate(constant->AsConstant())) {
+ return Location::ConstantLocation(constant->AsConstant());
+ }
+
+ return Location::RequiresRegister();
+}
+
+static bool CanGenerateConditionalMove(const Location& out, const Location& src) {
+ // Since IT blocks longer than a 16-bit instruction are deprecated by ARMv8,
+ // we check that we are not dealing with floating-point output (there is no
+ // 16-bit VMOV encoding).
+ if (!out.IsRegister() && !out.IsRegisterPair()) {
+ return false;
+ }
+
+ // For constants, we also check that the output is in one or two low registers,
+ // and that the constants fit in an 8-bit unsigned integer, so that a 16-bit
+ // MOV encoding can be used.
+ if (src.IsConstant()) {
+ if (!CanEncodeConstantAs8BitImmediate(src.GetConstant())) {
+ return false;
+ }
+
+ if (out.IsRegister()) {
+ if (!RegisterFrom(out).IsLow()) {
+ return false;
+ }
+ } else {
+ DCHECK(out.IsRegisterPair());
+
+ if (!HighRegisterFrom(out).IsLow()) {
+ return false;
+ }
+ }
+ }
+
+ return true;
+}
+
#undef __
+vixl32::Label* CodeGeneratorARMVIXL::GetFinalLabel(HInstruction* instruction,
+ vixl32::Label* final_label) {
+ DCHECK(!instruction->IsControlFlow() && !instruction->IsSuspendCheck());
+
+ const HBasicBlock* const block = instruction->GetBlock();
+ const HLoopInformation* const info = block->GetLoopInformation();
+ HInstruction* const next = instruction->GetNext();
+
+ // Avoid a branch to a branch.
+ if (next->IsGoto() && (info == nullptr ||
+ !info->IsBackEdge(*block) ||
+ !info->HasSuspendCheck())) {
+ final_label = GetLabelOf(next->AsGoto()->GetSuccessor());
+ }
+
+ return final_label;
+}
+
CodeGeneratorARMVIXL::CodeGeneratorARMVIXL(HGraph* graph,
const ArmInstructionSetFeatures& isa_features,
const CompilerOptions& compiler_options,
@@ -1481,8 +1994,6 @@
graph->GetArena()->Adapter(kArenaAllocCodeGenerator)),
pc_relative_type_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)),
type_bss_entry_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)),
- boot_image_address_patches_(std::less<uint32_t>(),
- graph->GetArena()->Adapter(kArenaAllocCodeGenerator)),
jit_string_patches_(StringReferenceValueComparator(),
graph->GetArena()->Adapter(kArenaAllocCodeGenerator)),
jit_class_patches_(TypeReferenceValueComparator(),
@@ -1945,43 +2456,6 @@
void InstructionCodeGeneratorARMVIXL::VisitExit(HExit* exit ATTRIBUTE_UNUSED) {
}
-void InstructionCodeGeneratorARMVIXL::GenerateVcmp(HInstruction* instruction) {
- Primitive::Type type = instruction->InputAt(0)->GetType();
- Location lhs_loc = instruction->GetLocations()->InAt(0);
- Location rhs_loc = instruction->GetLocations()->InAt(1);
- if (rhs_loc.IsConstant()) {
- // 0.0 is the only immediate that can be encoded directly in
- // a VCMP instruction.
- //
- // Both the JLS (section 15.20.1) and the JVMS (section 6.5)
- // specify that in a floating-point comparison, positive zero
- // and negative zero are considered equal, so we can use the
- // literal 0.0 for both cases here.
- //
- // Note however that some methods (Float.equal, Float.compare,
- // Float.compareTo, Double.equal, Double.compare,
- // Double.compareTo, Math.max, Math.min, StrictMath.max,
- // StrictMath.min) consider 0.0 to be (strictly) greater than
- // -0.0. So if we ever translate calls to these methods into a
- // HCompare instruction, we must handle the -0.0 case with
- // care here.
- DCHECK(rhs_loc.GetConstant()->IsArithmeticZero());
- if (type == Primitive::kPrimFloat) {
- __ Vcmp(F32, InputSRegisterAt(instruction, 0), 0.0);
- } else {
- DCHECK_EQ(type, Primitive::kPrimDouble);
- __ Vcmp(F64, DRegisterFrom(lhs_loc), 0.0);
- }
- } else {
- if (type == Primitive::kPrimFloat) {
- __ Vcmp(InputSRegisterAt(instruction, 0), InputSRegisterAt(instruction, 1));
- } else {
- DCHECK_EQ(type, Primitive::kPrimDouble);
- __ Vcmp(DRegisterFrom(lhs_loc), DRegisterFrom(rhs_loc));
- }
- }
-}
-
void InstructionCodeGeneratorARMVIXL::GenerateFPJumps(HCondition* cond,
vixl32::Label* true_label,
vixl32::Label* false_label ATTRIBUTE_UNUSED) {
@@ -2090,7 +2564,7 @@
break;
case Primitive::kPrimFloat:
case Primitive::kPrimDouble:
- GenerateVcmp(condition);
+ GenerateVcmp(condition, codegen_);
GenerateFPJumps(condition, true_target, false_target);
break;
default:
@@ -2167,20 +2641,29 @@
return;
}
- LocationSummary* locations = cond->GetLocations();
- DCHECK(locations->InAt(0).IsRegister());
- vixl32::Register left = InputRegisterAt(cond, 0);
- Location right = locations->InAt(1);
- if (right.IsRegister()) {
- __ Cmp(left, InputRegisterAt(cond, 1));
- } else {
- DCHECK(right.IsConstant());
- __ Cmp(left, CodeGenerator::GetInt32ValueOf(right.GetConstant()));
- }
+ vixl32::Label* non_fallthrough_target;
+ vixl32::Condition arm_cond = vixl32::Condition::None();
+ const vixl32::Register left = InputRegisterAt(cond, 0);
+ const Operand right = InputOperandAt(cond, 1);
+
if (true_target == nullptr) {
- __ B(ARMCondition(condition->GetOppositeCondition()), false_target);
+ arm_cond = ARMCondition(condition->GetOppositeCondition());
+ non_fallthrough_target = false_target;
} else {
- __ B(ARMCondition(condition->GetCondition()), true_target);
+ arm_cond = ARMCondition(condition->GetCondition());
+ non_fallthrough_target = true_target;
+ }
+
+ if (right.IsImmediate() && right.GetImmediate() == 0 && (arm_cond.Is(ne) || arm_cond.Is(eq))) {
+ if (arm_cond.Is(eq)) {
+ __ CompareAndBranchIfZero(left, non_fallthrough_target);
+ } else {
+ DCHECK(arm_cond.Is(ne));
+ __ CompareAndBranchIfNonZero(left, non_fallthrough_target);
+ }
+ } else {
+ __ Cmp(left, right);
+ __ B(arm_cond, non_fallthrough_target);
}
}
@@ -2241,29 +2724,135 @@
void LocationsBuilderARMVIXL::VisitSelect(HSelect* select) {
LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(select);
- if (Primitive::IsFloatingPointType(select->GetType())) {
+ const bool is_floating_point = Primitive::IsFloatingPointType(select->GetType());
+
+ if (is_floating_point) {
locations->SetInAt(0, Location::RequiresFpuRegister());
- locations->SetInAt(1, Location::RequiresFpuRegister());
+ locations->SetInAt(1, Location::FpuRegisterOrConstant(select->GetTrueValue()));
} else {
locations->SetInAt(0, Location::RequiresRegister());
- locations->SetInAt(1, Location::RequiresRegister());
+ locations->SetInAt(1, Arm8BitEncodableConstantOrRegister(select->GetTrueValue()));
}
+
if (IsBooleanValueOrMaterializedCondition(select->GetCondition())) {
- locations->SetInAt(2, Location::RequiresRegister());
+ locations->SetInAt(2, Location::RegisterOrConstant(select->GetCondition()));
+ // The code generator handles overlap with the values, but not with the condition.
+ locations->SetOut(Location::SameAsFirstInput());
+ } else if (is_floating_point) {
+ locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap);
+ } else {
+ if (!locations->InAt(1).IsConstant()) {
+ locations->SetInAt(0, Arm8BitEncodableConstantOrRegister(select->GetFalseValue()));
+ }
+
+ locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
}
- locations->SetOut(Location::SameAsFirstInput());
}
void InstructionCodeGeneratorARMVIXL::VisitSelect(HSelect* select) {
- LocationSummary* locations = select->GetLocations();
- vixl32::Label false_target;
- GenerateTestAndBranch(select,
- /* condition_input_index */ 2,
- /* true_target */ nullptr,
- &false_target,
- /* far_target */ false);
- codegen_->MoveLocation(locations->Out(), locations->InAt(1), select->GetType());
- __ Bind(&false_target);
+ HInstruction* const condition = select->GetCondition();
+ const LocationSummary* const locations = select->GetLocations();
+ const Primitive::Type type = select->GetType();
+ const Location first = locations->InAt(0);
+ const Location out = locations->Out();
+ const Location second = locations->InAt(1);
+ Location src;
+
+ if (condition->IsIntConstant()) {
+ if (condition->AsIntConstant()->IsFalse()) {
+ src = first;
+ } else {
+ src = second;
+ }
+
+ codegen_->MoveLocation(out, src, type);
+ return;
+ }
+
+ if (!Primitive::IsFloatingPointType(type) &&
+ CanGenerateTest(condition, codegen_->GetAssembler())) {
+ bool invert = false;
+
+ if (out.Equals(second)) {
+ src = first;
+ invert = true;
+ } else if (out.Equals(first)) {
+ src = second;
+ } else if (second.IsConstant()) {
+ DCHECK(CanEncodeConstantAs8BitImmediate(second.GetConstant()));
+ src = second;
+ } else if (first.IsConstant()) {
+ DCHECK(CanEncodeConstantAs8BitImmediate(first.GetConstant()));
+ src = first;
+ invert = true;
+ } else {
+ src = second;
+ }
+
+ if (CanGenerateConditionalMove(out, src)) {
+ if (!out.Equals(first) && !out.Equals(second)) {
+ codegen_->MoveLocation(out, src.Equals(first) ? second : first, type);
+ }
+
+ const vixl32::Condition cond = GenerateTest(condition, locations->InAt(2), invert, codegen_);
+ const size_t instr_count = out.IsRegisterPair() ? 4 : 2;
+ ExactAssemblyScope guard(GetVIXLAssembler(),
+ instr_count * vixl32::k16BitT32InstructionSizeInBytes,
+ CodeBufferCheckScope::kExactSize);
+
+ if (out.IsRegister()) {
+ __ it(cond);
+ __ mov(cond, RegisterFrom(out), OperandFrom(src, type));
+ } else {
+ DCHECK(out.IsRegisterPair());
+
+ Operand operand_high(0);
+ Operand operand_low(0);
+
+ if (src.IsConstant()) {
+ const int64_t value = Int64ConstantFrom(src);
+
+ operand_high = High32Bits(value);
+ operand_low = Low32Bits(value);
+ } else {
+ DCHECK(src.IsRegisterPair());
+ operand_high = HighRegisterFrom(src);
+ operand_low = LowRegisterFrom(src);
+ }
+
+ __ it(cond);
+ __ mov(cond, LowRegisterFrom(out), operand_low);
+ __ it(cond);
+ __ mov(cond, HighRegisterFrom(out), operand_high);
+ }
+
+ return;
+ }
+ }
+
+ vixl32::Label* false_target = nullptr;
+ vixl32::Label* true_target = nullptr;
+ vixl32::Label select_end;
+ vixl32::Label* const target = codegen_->GetFinalLabel(select, &select_end);
+
+ if (out.Equals(second)) {
+ true_target = target;
+ src = first;
+ } else {
+ false_target = target;
+ src = second;
+
+ if (!out.Equals(first)) {
+ codegen_->MoveLocation(out, first, type);
+ }
+ }
+
+ GenerateTestAndBranch(select, 2, true_target, false_target, /* far_target */ false);
+ codegen_->MoveLocation(out, src, type);
+
+ if (select_end.IsReferenced()) {
+ __ Bind(&select_end);
+ }
}
void LocationsBuilderARMVIXL::VisitNativeDebugInfo(HNativeDebugInfo* info) {
@@ -2341,7 +2930,7 @@
break;
case Primitive::kPrimFloat:
case Primitive::kPrimDouble:
- GenerateVcmp(cond);
+ GenerateVcmp(cond, codegen_);
GenerateFPJumps(cond, &true_label, &false_label);
break;
}
@@ -4356,7 +4945,7 @@
case Primitive::kPrimFloat:
case Primitive::kPrimDouble: {
__ Mov(out, 0);
- GenerateVcmp(compare);
+ GenerateVcmp(compare, codegen_);
// To branch on the FP compare result we transfer FPSCR to APSR (encoded as PC in VMRS).
__ Vmrs(RegisterOrAPSR_nzcv(kPcCode), FPSCR);
less_cond = ARMFPCondition(kCondLT, compare->IsGtBias());
@@ -4733,6 +5322,7 @@
case ADC: neg_opcode = SBC; value = ~value; break;
case SUB: neg_opcode = ADD; value = -value; break;
case SBC: neg_opcode = ADC; value = ~value; break;
+ case MOV: neg_opcode = MVN; value = ~value; break;
default:
return false;
}
@@ -7263,14 +7853,35 @@
DCHECK(kEmitCompilerReadBarrier);
if (kUseBakerReadBarrier) {
// Fast path implementation of art::ReadBarrier::BarrierForRoot when
- // Baker's read barrier are used:
+ // Baker's read barrier are used.
//
- // root = obj.field;
+ // Note that we do not actually check the value of
+ // `GetIsGcMarking()` to decide whether to mark the loaded GC
+ // root or not. Instead, we load into `temp` the read barrier
+ // mark entry point corresponding to register `root`. If `temp`
+ // is null, it means that `GetIsGcMarking()` is false, and vice
+ // versa.
+ //
// temp = Thread::Current()->pReadBarrierMarkReg ## root.reg()
- // if (temp != null) {
- // root = temp(root)
+ // GcRoot<mirror::Object> root = *(obj+offset); // Original reference load.
+ // if (temp != nullptr) { // <=> Thread::Current()->GetIsGcMarking()
+ // // Slow path.
+ // root = temp(root); // root = ReadBarrier::Mark(root); // Runtime entry point call.
// }
+ // Slow path marking the GC root `root`. The entrypoint will already be loaded in `temp`.
+ Location temp = LocationFrom(lr);
+ SlowPathCodeARMVIXL* slow_path = new (GetGraph()->GetArena()) ReadBarrierMarkSlowPathARMVIXL(
+ instruction, root, /* entrypoint */ temp);
+ codegen_->AddSlowPath(slow_path);
+
+ // temp = Thread::Current()->pReadBarrierMarkReg ## root.reg()
+ const int32_t entry_point_offset =
+ CodeGenerator::GetReadBarrierMarkEntryPointsOffset<kArmPointerSize>(root.reg());
+ // Loading the entrypoint does not require a load acquire since it is only changed when
+ // threads are suspended or running a checkpoint.
+ GetAssembler()->LoadFromOffset(kLoadWord, RegisterFrom(temp), tr, entry_point_offset);
+
// /* GcRoot<mirror::Object> */ root = *(obj + offset)
GetAssembler()->LoadFromOffset(kLoadWord, root_reg, obj, offset);
static_assert(
@@ -7281,21 +7892,6 @@
"art::mirror::CompressedReference<mirror::Object> and int32_t "
"have different sizes.");
- // Slow path marking the GC root `root`.
- Location temp = LocationFrom(lr);
- SlowPathCodeARMVIXL* slow_path =
- new (GetGraph()->GetArena()) ReadBarrierMarkSlowPathARMVIXL(
- instruction,
- root,
- /*entrypoint*/ temp);
- codegen_->AddSlowPath(slow_path);
-
- // temp = Thread::Current()->pReadBarrierMarkReg ## root.reg()
- const int32_t entry_point_offset =
- CodeGenerator::GetReadBarrierMarkEntryPointsOffset<kArmPointerSize>(root.reg());
- // Loading the entrypoint does not require a load acquire since it is only changed when
- // threads are suspended or running a checkpoint.
- GetAssembler()->LoadFromOffset(kLoadWord, RegisterFrom(temp), tr, entry_point_offset);
// The entrypoint is null when the GC is not marking, this prevents one load compared to
// checking GetIsGcMarking.
__ CompareAndBranchIfNonZero(RegisterFrom(temp), slow_path->GetEntryLabel());
@@ -7366,55 +7962,114 @@
DCHECK(kEmitCompilerReadBarrier);
DCHECK(kUseBakerReadBarrier);
- // In slow path based read barriers, the read barrier call is
- // inserted after the original load. However, in fast path based
- // Baker's read barriers, we need to perform the load of
- // mirror::Object::monitor_ *before* the original reference load.
- // This load-load ordering is required by the read barrier.
- // The fast path/slow path (for Baker's algorithm) should look like:
+ // Query `art::Thread::Current()->GetIsGcMarking()` to decide
+ // whether we need to enter the slow path to mark the reference.
+ // Then, in the slow path, check the gray bit in the lock word of
+ // the reference's holder (`obj`) to decide whether to mark `ref` or
+ // not.
//
- // uint32_t rb_state = Lockword(obj->monitor_).ReadBarrierState();
- // lfence; // Load fence or artificial data dependency to prevent load-load reordering
- // HeapReference<Object> ref = *src; // Original reference load.
- // bool is_gray = (rb_state == ReadBarrier::GrayState());
- // if (is_gray) {
- // ref = ReadBarrier::Mark(ref); // Performed by runtime entrypoint slow path.
+ // Note that we do not actually check the value of `GetIsGcMarking()`;
+ // instead, we load into `temp3` the read barrier mark entry point
+ // corresponding to register `ref`. If `temp3` is null, it means
+ // that `GetIsGcMarking()` is false, and vice versa.
+ //
+ // temp3 = Thread::Current()->pReadBarrierMarkReg ## root.reg()
+ // if (temp3 != nullptr) { // <=> Thread::Current()->GetIsGcMarking()
+ // // Slow path.
+ // uint32_t rb_state = Lockword(obj->monitor_).ReadBarrierState();
+ // lfence; // Load fence or artificial data dependency to prevent load-load reordering
+ // HeapReference<mirror::Object> ref = *src; // Original reference load.
+ // bool is_gray = (rb_state == ReadBarrier::GrayState());
+ // if (is_gray) {
+ // ref = temp3(ref); // ref = ReadBarrier::Mark(ref); // Runtime entry point call.
+ // }
+ // } else {
+ // HeapReference<mirror::Object> ref = *src; // Original reference load.
// }
- //
- // Note: the original implementation in ReadBarrier::Barrier is
- // slightly more complex as it performs additional checks that we do
- // not do here for performance reasons.
- vixl32::Register ref_reg = RegisterFrom(ref);
vixl32::Register temp_reg = RegisterFrom(temp);
- uint32_t monitor_offset = mirror::Object::MonitorOffset().Int32Value();
- // /* int32_t */ monitor = obj->monitor_
- GetAssembler()->LoadFromOffset(kLoadWord, temp_reg, obj, monitor_offset);
- if (needs_null_check) {
- MaybeRecordImplicitNullCheck(instruction);
+ // Slow path marking the object `ref` when the GC is marking. The
+ // entrypoint will already be loaded in `temp3`.
+ Location temp3 = LocationFrom(lr);
+ SlowPathCodeARMVIXL* slow_path;
+ if (always_update_field) {
+ DCHECK(temp2 != nullptr);
+ // LoadReferenceWithBakerReadBarrierAndUpdateFieldSlowPathARMVIXL
+ // only supports address of the form `obj + field_offset`, where
+ // `obj` is a register and `field_offset` is a register pair (of
+ // which only the lower half is used). Thus `offset` and
+ // `scale_factor` above are expected to be null in this code path.
+ DCHECK_EQ(offset, 0u);
+ DCHECK_EQ(scale_factor, ScaleFactor::TIMES_1);
+ Location field_offset = index;
+ slow_path =
+ new (GetGraph()->GetArena()) LoadReferenceWithBakerReadBarrierAndUpdateFieldSlowPathARMVIXL(
+ instruction,
+ ref,
+ obj,
+ offset,
+ /* index */ field_offset,
+ scale_factor,
+ needs_null_check,
+ temp_reg,
+ *temp2,
+ /* entrypoint */ temp3);
+ } else {
+ slow_path = new (GetGraph()->GetArena()) LoadReferenceWithBakerReadBarrierSlowPathARMVIXL(
+ instruction,
+ ref,
+ obj,
+ offset,
+ index,
+ scale_factor,
+ needs_null_check,
+ temp_reg,
+ /* entrypoint */ temp3);
}
- // /* LockWord */ lock_word = LockWord(monitor)
- static_assert(sizeof(LockWord) == sizeof(int32_t),
- "art::LockWord and int32_t have different sizes.");
+ AddSlowPath(slow_path);
- // Introduce a dependency on the lock_word including the rb_state,
- // which shall prevent load-load reordering without using
- // a memory barrier (which would be more expensive).
- // `obj` is unchanged by this operation, but its value now depends
- // on `temp_reg`.
- __ Add(obj, obj, Operand(temp_reg, ShiftType::LSR, 32));
+ // temp3 = Thread::Current()->pReadBarrierMarkReg ## ref.reg()
+ const int32_t entry_point_offset =
+ CodeGenerator::GetReadBarrierMarkEntryPointsOffset<kArmPointerSize>(ref.reg());
+ // Loading the entrypoint does not require a load acquire since it is only changed when
+ // threads are suspended or running a checkpoint.
+ GetAssembler()->LoadFromOffset(kLoadWord, RegisterFrom(temp3), tr, entry_point_offset);
+ // The entrypoint is null when the GC is not marking, this prevents one load compared to
+ // checking GetIsGcMarking.
+ __ CompareAndBranchIfNonZero(RegisterFrom(temp3), slow_path->GetEntryLabel());
+ // Fast path: just load the reference.
+ GenerateRawReferenceLoad(instruction, ref, obj, offset, index, scale_factor, needs_null_check);
+ __ Bind(slow_path->GetExitLabel());
+}
- // The actual reference load.
+void CodeGeneratorARMVIXL::GenerateRawReferenceLoad(HInstruction* instruction,
+ Location ref,
+ vixl::aarch32::Register obj,
+ uint32_t offset,
+ Location index,
+ ScaleFactor scale_factor,
+ bool needs_null_check) {
+ Primitive::Type type = Primitive::kPrimNot;
+ vixl32::Register ref_reg = RegisterFrom(ref, type);
+
+ // If needed, vixl::EmissionCheckScope guards are used to ensure
+ // that no pools are emitted between the load (macro) instruction
+ // and MaybeRecordImplicitNullCheck.
+
if (index.IsValid()) {
// Load types involving an "index": ArrayGet,
// UnsafeGetObject/UnsafeGetObjectVolatile and UnsafeCASObject
// intrinsics.
- // /* HeapReference<Object> */ ref = *(obj + offset + (index << scale_factor))
+ // /* HeapReference<mirror::Object> */ ref = *(obj + offset + (index << scale_factor))
if (index.IsConstant()) {
size_t computed_offset =
(Int32ConstantFrom(index) << scale_factor) + offset;
+ vixl::EmissionCheckScope guard(GetVIXLAssembler(), kMaxMacroInstructionSizeInBytes);
GetAssembler()->LoadFromOffset(kLoadWord, ref_reg, obj, computed_offset);
+ if (needs_null_check) {
+ MaybeRecordImplicitNullCheck(instruction);
+ }
} else {
// Handle the special case of the
// UnsafeGetObject/UnsafeGetObjectVolatile and UnsafeCASObject
@@ -7424,46 +8079,27 @@
? LowRegisterFrom(index)
: RegisterFrom(index);
UseScratchRegisterScope temps(GetVIXLAssembler());
- const vixl32::Register temp3 = temps.Acquire();
- __ Add(temp3, obj, Operand(index_reg, ShiftType::LSL, scale_factor));
- GetAssembler()->LoadFromOffset(kLoadWord, ref_reg, temp3, offset);
+ vixl32::Register temp = temps.Acquire();
+ __ Add(temp, obj, Operand(index_reg, ShiftType::LSL, scale_factor));
+ {
+ vixl::EmissionCheckScope guard(GetVIXLAssembler(), kMaxMacroInstructionSizeInBytes);
+ GetAssembler()->LoadFromOffset(kLoadWord, ref_reg, temp, offset);
+ if (needs_null_check) {
+ MaybeRecordImplicitNullCheck(instruction);
+ }
+ }
}
} else {
- // /* HeapReference<Object> */ ref = *(obj + offset)
+ // /* HeapReference<mirror::Object> */ ref = *(obj + offset)
+ vixl::EmissionCheckScope guard(GetVIXLAssembler(), kMaxMacroInstructionSizeInBytes);
GetAssembler()->LoadFromOffset(kLoadWord, ref_reg, obj, offset);
+ if (needs_null_check) {
+ MaybeRecordImplicitNullCheck(instruction);
+ }
}
// Object* ref = ref_addr->AsMirrorPtr()
GetAssembler()->MaybeUnpoisonHeapReference(ref_reg);
-
- // Slow path marking the object `ref` when it is gray.
- SlowPathCodeARMVIXL* slow_path;
- if (always_update_field) {
- DCHECK(temp2 != nullptr);
- // ReadBarrierMarkAndUpdateFieldSlowPathARMVIXL only supports address
- // of the form `obj + field_offset`, where `obj` is a register and
- // `field_offset` is a register pair (of which only the lower half
- // is used). Thus `offset` and `scale_factor` above are expected
- // to be null in this code path.
- DCHECK_EQ(offset, 0u);
- DCHECK_EQ(scale_factor, ScaleFactor::TIMES_1);
- slow_path = new (GetGraph()->GetArena()) ReadBarrierMarkAndUpdateFieldSlowPathARMVIXL(
- instruction, ref, obj, /* field_offset */ index, temp_reg, *temp2);
- } else {
- slow_path = new (GetGraph()->GetArena()) ReadBarrierMarkSlowPathARMVIXL(instruction, ref);
- }
- AddSlowPath(slow_path);
-
- // if (rb_state == ReadBarrier::GrayState())
- // ref = ReadBarrier::Mark(ref);
- // Given the numeric representation, it's enough to check the low bit of the
- // rb_state. We do that by shifting the bit out of the lock word with LSRS
- // which can be a 16-bit instruction unlike the TST immediate.
- static_assert(ReadBarrier::WhiteState() == 0, "Expecting white to have value 0");
- static_assert(ReadBarrier::GrayState() == 1, "Expecting gray to have value 1");
- __ Lsrs(temp_reg, temp_reg, LockWord::kReadBarrierStateShift + 1);
- __ B(cs, slow_path->GetEntryLabel()); // Carry flag is the last bit shifted out by LSRS.
- __ Bind(slow_path->GetExitLabel());
}
void CodeGeneratorARMVIXL::GenerateReadBarrierSlow(HInstruction* instruction,
@@ -7738,9 +8374,7 @@
}
VIXLUInt32Literal* CodeGeneratorARMVIXL::DeduplicateBootImageAddressLiteral(uint32_t address) {
- bool needs_patch = GetCompilerOptions().GetIncludePatchInformation();
- Uint32ToLiteralMap* map = needs_patch ? &boot_image_address_patches_ : &uint32_literals_;
- return DeduplicateUint32Literal(dchecked_integral_cast<uint32_t>(address), map);
+ return DeduplicateUint32Literal(dchecked_integral_cast<uint32_t>(address), &uint32_literals_);
}
VIXLUInt32Literal* CodeGeneratorARMVIXL::DeduplicateDexCacheAddressLiteral(uint32_t address) {
@@ -7800,8 +8434,7 @@
/* MOVW+MOVT for each entry */ 2u * pc_relative_string_patches_.size() +
boot_image_type_patches_.size() +
/* MOVW+MOVT for each entry */ 2u * pc_relative_type_patches_.size() +
- /* MOVW+MOVT for each entry */ 2u * type_bss_entry_patches_.size() +
- boot_image_address_patches_.size();
+ /* MOVW+MOVT for each entry */ 2u * type_bss_entry_patches_.size();
linker_patches->reserve(size);
EmitPcRelativeLinkerPatches<LinkerPatch::DexCacheArrayPatch>(pc_relative_dex_cache_patches_,
linker_patches);
@@ -7835,13 +8468,6 @@
target_type.dex_file,
target_type.type_index.index_));
}
- for (const auto& entry : boot_image_address_patches_) {
- DCHECK(GetCompilerOptions().GetIncludePatchInformation());
- VIXLUInt32Literal* literal = entry.second;
- DCHECK(literal->IsBound());
- uint32_t literal_offset = literal->GetLocation();
- linker_patches->push_back(LinkerPatch::RecordPosition(literal_offset));
- }
DCHECK_EQ(size, linker_patches->size());
}
diff --git a/compiler/optimizing/code_generator_arm_vixl.h b/compiler/optimizing/code_generator_arm_vixl.h
index 3f52c72..781027a 100644
--- a/compiler/optimizing/code_generator_arm_vixl.h
+++ b/compiler/optimizing/code_generator_arm_vixl.h
@@ -45,6 +45,11 @@
namespace art {
namespace arm {
+// This constant is used as an approximate margin when emission of veneer and literal pools
+// must be blocked.
+static constexpr int kMaxMacroInstructionSizeInBytes =
+ 15 * vixl::aarch32::kMaxInstructionSizeInBytes;
+
static const vixl::aarch32::Register kParameterCoreRegistersVIXL[] = {
vixl::aarch32::r1,
vixl::aarch32::r2,
@@ -396,7 +401,6 @@
void GenerateCompareTestAndBranch(HCondition* condition,
vixl::aarch32::Label* true_target,
vixl::aarch32::Label* false_target);
- void GenerateVcmp(HInstruction* instruction);
void GenerateFPJumps(HCondition* cond,
vixl::aarch32::Label* true_label,
vixl::aarch32::Label* false_label);
@@ -505,6 +509,8 @@
return &(block_labels_[block->GetBlockId()]);
}
+ vixl32::Label* GetFinalLabel(HInstruction* instruction, vixl32::Label* final_label);
+
void Initialize() OVERRIDE {
block_labels_.resize(GetGraph()->GetBlocks().size());
}
@@ -625,6 +631,15 @@
bool always_update_field = false,
vixl::aarch32::Register* temp2 = nullptr);
+ // Generate a heap reference load (with no read barrier).
+ void GenerateRawReferenceLoad(HInstruction* instruction,
+ Location ref,
+ vixl::aarch32::Register obj,
+ uint32_t offset,
+ Location index,
+ ScaleFactor scale_factor,
+ bool needs_null_check);
+
// Generate a read barrier for a heap reference within `instruction`
// using a slow path.
//
@@ -738,8 +753,6 @@
ArenaDeque<PcRelativePatchInfo> pc_relative_type_patches_;
// PC-relative type patch info for kBssEntry.
ArenaDeque<PcRelativePatchInfo> type_bss_entry_patches_;
- // Deduplication map for patchable boot image addresses.
- Uint32ToLiteralMap boot_image_address_patches_;
// Patches for string literals in JIT compiled code.
StringToLiteralMap jit_string_patches_;
diff --git a/compiler/optimizing/code_generator_mips.cc b/compiler/optimizing/code_generator_mips.cc
index 51dd898..5f02a52 100644
--- a/compiler/optimizing/code_generator_mips.cc
+++ b/compiler/optimizing/code_generator_mips.cc
@@ -391,7 +391,8 @@
class TypeCheckSlowPathMIPS : public SlowPathCodeMIPS {
public:
- explicit TypeCheckSlowPathMIPS(HInstruction* instruction) : SlowPathCodeMIPS(instruction) {}
+ explicit TypeCheckSlowPathMIPS(HInstruction* instruction, bool is_fatal)
+ : SlowPathCodeMIPS(instruction), is_fatal_(is_fatal) {}
void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
LocationSummary* locations = instruction_->GetLocations();
@@ -401,7 +402,9 @@
CodeGeneratorMIPS* mips_codegen = down_cast<CodeGeneratorMIPS*>(codegen);
__ Bind(GetEntryLabel());
- SaveLiveRegisters(codegen, locations);
+ if (!is_fatal_) {
+ SaveLiveRegisters(codegen, locations);
+ }
// We're moving two locations to locations that could overlap, so we need a parallel
// move resolver.
@@ -424,13 +427,19 @@
CheckEntrypointTypes<kQuickCheckInstanceOf, void, mirror::Object*, mirror::Class*>();
}
- RestoreLiveRegisters(codegen, locations);
- __ B(GetExitLabel());
+ if (!is_fatal_) {
+ RestoreLiveRegisters(codegen, locations);
+ __ B(GetExitLabel());
+ }
}
const char* GetDescription() const OVERRIDE { return "TypeCheckSlowPathMIPS"; }
+ bool IsFatal() const OVERRIDE { return is_fatal_; }
+
private:
+ const bool is_fatal_;
+
DISALLOW_COPY_AND_ASSIGN(TypeCheckSlowPathMIPS);
};
@@ -482,8 +491,6 @@
graph->GetArena()->Adapter(kArenaAllocCodeGenerator)),
pc_relative_type_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)),
type_bss_entry_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)),
- boot_image_address_patches_(std::less<uint32_t>(),
- graph->GetArena()->Adapter(kArenaAllocCodeGenerator)),
jit_string_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)),
jit_class_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)),
clobbered_ra_(false) {
@@ -1026,8 +1033,7 @@
pc_relative_type_patches_.size() +
type_bss_entry_patches_.size() +
boot_image_string_patches_.size() +
- boot_image_type_patches_.size() +
- boot_image_address_patches_.size();
+ boot_image_type_patches_.size();
linker_patches->reserve(size);
EmitPcRelativeLinkerPatches<LinkerPatch::DexCacheArrayPatch>(pc_relative_dex_cache_patches_,
linker_patches);
@@ -1061,13 +1067,6 @@
target_type.dex_file,
target_type.type_index.index_));
}
- for (const auto& entry : boot_image_address_patches_) {
- DCHECK(GetCompilerOptions().GetIncludePatchInformation());
- Literal* literal = entry.second;
- DCHECK(literal->GetLabel()->IsBound());
- uint32_t literal_offset = __ GetLabelLocation(literal->GetLabel());
- linker_patches->push_back(LinkerPatch::RecordPosition(literal_offset));
- }
DCHECK_EQ(size, linker_patches->size());
}
@@ -1125,9 +1124,7 @@
}
Literal* CodeGeneratorMIPS::DeduplicateBootImageAddressLiteral(uint32_t address) {
- bool needs_patch = GetCompilerOptions().GetIncludePatchInformation();
- Uint32ToLiteralMap* map = needs_patch ? &boot_image_address_patches_ : &uint32_literals_;
- return DeduplicateUint32Literal(dchecked_integral_cast<uint32_t>(address), map);
+ return DeduplicateUint32Literal(dchecked_integral_cast<uint32_t>(address), &uint32_literals_);
}
void CodeGeneratorMIPS::EmitPcRelativeAddressPlaceholderHigh(PcRelativePatchInfo* info,
@@ -2331,30 +2328,178 @@
}
void LocationsBuilderMIPS::VisitCheckCast(HCheckCast* instruction) {
- LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(
- instruction,
- LocationSummary::kCallOnSlowPath);
+ LocationSummary::CallKind call_kind = LocationSummary::kNoCall;
+ bool throws_into_catch = instruction->CanThrowIntoCatchBlock();
+
+ TypeCheckKind type_check_kind = instruction->GetTypeCheckKind();
+ switch (type_check_kind) {
+ case TypeCheckKind::kExactCheck:
+ case TypeCheckKind::kAbstractClassCheck:
+ case TypeCheckKind::kClassHierarchyCheck:
+ case TypeCheckKind::kArrayObjectCheck:
+ call_kind = throws_into_catch
+ ? LocationSummary::kCallOnSlowPath
+ : LocationSummary::kNoCall; // In fact, call on a fatal (non-returning) slow path.
+ break;
+ case TypeCheckKind::kArrayCheck:
+ case TypeCheckKind::kUnresolvedCheck:
+ case TypeCheckKind::kInterfaceCheck:
+ call_kind = LocationSummary::kCallOnSlowPath;
+ break;
+ }
+
+ LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instruction, call_kind);
locations->SetInAt(0, Location::RequiresRegister());
locations->SetInAt(1, Location::RequiresRegister());
- // Note that TypeCheckSlowPathMIPS uses this register too.
locations->AddTemp(Location::RequiresRegister());
}
void InstructionCodeGeneratorMIPS::VisitCheckCast(HCheckCast* instruction) {
+ TypeCheckKind type_check_kind = instruction->GetTypeCheckKind();
LocationSummary* locations = instruction->GetLocations();
Register obj = locations->InAt(0).AsRegister<Register>();
Register cls = locations->InAt(1).AsRegister<Register>();
- Register obj_cls = locations->GetTemp(0).AsRegister<Register>();
+ Register temp = locations->GetTemp(0).AsRegister<Register>();
+ const uint32_t class_offset = mirror::Object::ClassOffset().Int32Value();
+ const uint32_t super_offset = mirror::Class::SuperClassOffset().Int32Value();
+ const uint32_t component_offset = mirror::Class::ComponentTypeOffset().Int32Value();
+ const uint32_t primitive_offset = mirror::Class::PrimitiveTypeOffset().Int32Value();
+ const uint32_t iftable_offset = mirror::Class::IfTableOffset().Uint32Value();
+ const uint32_t array_length_offset = mirror::Array::LengthOffset().Uint32Value();
+ const uint32_t object_array_data_offset =
+ mirror::Array::DataOffset(kHeapReferenceSize).Uint32Value();
+ MipsLabel done;
- SlowPathCodeMIPS* slow_path = new (GetGraph()->GetArena()) TypeCheckSlowPathMIPS(instruction);
+ // Always false for read barriers since we may need to go to the entrypoint for non-fatal cases
+ // from false negatives. The false negatives may come from avoiding read barriers below. Avoiding
+ // read barriers is done for performance and code size reasons.
+ bool is_type_check_slow_path_fatal = false;
+ if (!kEmitCompilerReadBarrier) {
+ is_type_check_slow_path_fatal =
+ (type_check_kind == TypeCheckKind::kExactCheck ||
+ type_check_kind == TypeCheckKind::kAbstractClassCheck ||
+ type_check_kind == TypeCheckKind::kClassHierarchyCheck ||
+ type_check_kind == TypeCheckKind::kArrayObjectCheck) &&
+ !instruction->CanThrowIntoCatchBlock();
+ }
+ SlowPathCodeMIPS* slow_path =
+ new (GetGraph()->GetArena()) TypeCheckSlowPathMIPS(instruction,
+ is_type_check_slow_path_fatal);
codegen_->AddSlowPath(slow_path);
- // TODO: avoid this check if we know obj is not null.
- __ Beqz(obj, slow_path->GetExitLabel());
- // Compare the class of `obj` with `cls`.
- __ LoadFromOffset(kLoadWord, obj_cls, obj, mirror::Object::ClassOffset().Int32Value());
- __ MaybeUnpoisonHeapReference(obj_cls);
- __ Bne(obj_cls, cls, slow_path->GetEntryLabel());
+ // Avoid this check if we know `obj` is not null.
+ if (instruction->MustDoNullCheck()) {
+ __ Beqz(obj, &done);
+ }
+
+ switch (type_check_kind) {
+ case TypeCheckKind::kExactCheck:
+ case TypeCheckKind::kArrayCheck: {
+ // /* HeapReference<Class> */ temp = obj->klass_
+ __ LoadFromOffset(kLoadWord, temp, obj, class_offset);
+ __ MaybeUnpoisonHeapReference(temp);
+ // Jump to slow path for throwing the exception or doing a
+ // more involved array check.
+ __ Bne(temp, cls, slow_path->GetEntryLabel());
+ break;
+ }
+
+ case TypeCheckKind::kAbstractClassCheck: {
+ // /* HeapReference<Class> */ temp = obj->klass_
+ __ LoadFromOffset(kLoadWord, temp, obj, class_offset);
+ __ MaybeUnpoisonHeapReference(temp);
+ // If the class is abstract, we eagerly fetch the super class of the
+ // object to avoid doing a comparison we know will fail.
+ MipsLabel loop;
+ __ Bind(&loop);
+ // /* HeapReference<Class> */ temp = temp->super_class_
+ __ LoadFromOffset(kLoadWord, temp, temp, super_offset);
+ __ MaybeUnpoisonHeapReference(temp);
+ // If the class reference currently in `temp` is null, jump to the slow path to throw the
+ // exception.
+ __ Beqz(temp, slow_path->GetEntryLabel());
+ // Otherwise, compare the classes.
+ __ Bne(temp, cls, &loop);
+ break;
+ }
+
+ case TypeCheckKind::kClassHierarchyCheck: {
+ // /* HeapReference<Class> */ temp = obj->klass_
+ __ LoadFromOffset(kLoadWord, temp, obj, class_offset);
+ __ MaybeUnpoisonHeapReference(temp);
+ // Walk over the class hierarchy to find a match.
+ MipsLabel loop;
+ __ Bind(&loop);
+ __ Beq(temp, cls, &done);
+ // /* HeapReference<Class> */ temp = temp->super_class_
+ __ LoadFromOffset(kLoadWord, temp, temp, super_offset);
+ __ MaybeUnpoisonHeapReference(temp);
+ // If the class reference currently in `temp` is null, jump to the slow path to throw the
+ // exception. Otherwise, jump to the beginning of the loop.
+ __ Bnez(temp, &loop);
+ __ B(slow_path->GetEntryLabel());
+ break;
+ }
+
+ case TypeCheckKind::kArrayObjectCheck: {
+ // /* HeapReference<Class> */ temp = obj->klass_
+ __ LoadFromOffset(kLoadWord, temp, obj, class_offset);
+ __ MaybeUnpoisonHeapReference(temp);
+ // Do an exact check.
+ __ Beq(temp, cls, &done);
+ // Otherwise, we need to check that the object's class is a non-primitive array.
+ // /* HeapReference<Class> */ temp = temp->component_type_
+ __ LoadFromOffset(kLoadWord, temp, temp, component_offset);
+ __ MaybeUnpoisonHeapReference(temp);
+ // If the component type is null, jump to the slow path to throw the exception.
+ __ Beqz(temp, slow_path->GetEntryLabel());
+ // Otherwise, the object is indeed an array, further check that this component
+ // type is not a primitive type.
+ __ LoadFromOffset(kLoadUnsignedHalfword, temp, temp, primitive_offset);
+ static_assert(Primitive::kPrimNot == 0, "Expected 0 for kPrimNot");
+ __ Bnez(temp, slow_path->GetEntryLabel());
+ break;
+ }
+
+ case TypeCheckKind::kUnresolvedCheck:
+ // We always go into the type check slow path for the unresolved check case.
+ // We cannot directly call the CheckCast runtime entry point
+ // without resorting to a type checking slow path here (i.e. by
+ // calling InvokeRuntime directly), as it would require to
+ // assign fixed registers for the inputs of this HInstanceOf
+ // instruction (following the runtime calling convention), which
+ // might be cluttered by the potential first read barrier
+ // emission at the beginning of this method.
+ __ B(slow_path->GetEntryLabel());
+ break;
+
+ case TypeCheckKind::kInterfaceCheck: {
+ // Avoid read barriers to improve performance of the fast path. We can not get false
+ // positives by doing this.
+ // /* HeapReference<Class> */ temp = obj->klass_
+ __ LoadFromOffset(kLoadWord, temp, obj, class_offset);
+ __ MaybeUnpoisonHeapReference(temp);
+ // /* HeapReference<Class> */ temp = temp->iftable_
+ __ LoadFromOffset(kLoadWord, temp, temp, iftable_offset);
+ __ MaybeUnpoisonHeapReference(temp);
+ // Iftable is never null.
+ __ Lw(TMP, temp, array_length_offset);
+ // Loop through the iftable and check if any class matches.
+ MipsLabel loop;
+ __ Bind(&loop);
+ __ Addiu(temp, temp, 2 * kHeapReferenceSize); // Possibly in delay slot on R2.
+ __ Beqz(TMP, slow_path->GetEntryLabel());
+ __ Lw(AT, temp, object_array_data_offset - 2 * kHeapReferenceSize);
+ __ MaybeUnpoisonHeapReference(AT);
+ // Go to next interface.
+ __ Addiu(TMP, TMP, -2);
+ // Compare the classes and continue the loop if they do not match.
+ __ Bne(AT, cls, &loop);
+ break;
+ }
+ }
+
+ __ Bind(&done);
__ Bind(slow_path->GetExitLabel());
}
@@ -5193,8 +5338,22 @@
}
void LocationsBuilderMIPS::VisitInstanceOf(HInstanceOf* instruction) {
- LocationSummary::CallKind call_kind =
- instruction->IsExactCheck() ? LocationSummary::kNoCall : LocationSummary::kCallOnSlowPath;
+ LocationSummary::CallKind call_kind = LocationSummary::kNoCall;
+ TypeCheckKind type_check_kind = instruction->GetTypeCheckKind();
+ switch (type_check_kind) {
+ case TypeCheckKind::kExactCheck:
+ case TypeCheckKind::kAbstractClassCheck:
+ case TypeCheckKind::kClassHierarchyCheck:
+ case TypeCheckKind::kArrayObjectCheck:
+ call_kind = LocationSummary::kNoCall;
+ break;
+ case TypeCheckKind::kArrayCheck:
+ case TypeCheckKind::kUnresolvedCheck:
+ case TypeCheckKind::kInterfaceCheck:
+ call_kind = LocationSummary::kCallOnSlowPath;
+ break;
+ }
+
LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instruction, call_kind);
locations->SetInAt(0, Location::RequiresRegister());
locations->SetInAt(1, Location::RequiresRegister());
@@ -5204,36 +5363,143 @@
}
void InstructionCodeGeneratorMIPS::VisitInstanceOf(HInstanceOf* instruction) {
+ TypeCheckKind type_check_kind = instruction->GetTypeCheckKind();
LocationSummary* locations = instruction->GetLocations();
Register obj = locations->InAt(0).AsRegister<Register>();
Register cls = locations->InAt(1).AsRegister<Register>();
Register out = locations->Out().AsRegister<Register>();
-
+ uint32_t class_offset = mirror::Object::ClassOffset().Int32Value();
+ uint32_t super_offset = mirror::Class::SuperClassOffset().Int32Value();
+ uint32_t component_offset = mirror::Class::ComponentTypeOffset().Int32Value();
+ uint32_t primitive_offset = mirror::Class::PrimitiveTypeOffset().Int32Value();
MipsLabel done;
+ SlowPathCodeMIPS* slow_path = nullptr;
// Return 0 if `obj` is null.
- // TODO: Avoid this check if we know `obj` is not null.
- __ Move(out, ZERO);
- __ Beqz(obj, &done);
+ // Avoid this check if we know `obj` is not null.
+ if (instruction->MustDoNullCheck()) {
+ __ Move(out, ZERO);
+ __ Beqz(obj, &done);
+ }
- // Compare the class of `obj` with `cls`.
- __ LoadFromOffset(kLoadWord, out, obj, mirror::Object::ClassOffset().Int32Value());
- __ MaybeUnpoisonHeapReference(out);
- if (instruction->IsExactCheck()) {
- // Classes must be equal for the instanceof to succeed.
- __ Xor(out, out, cls);
- __ Sltiu(out, out, 1);
- } else {
- // If the classes are not equal, we go into a slow path.
- DCHECK(locations->OnlyCallsOnSlowPath());
- SlowPathCodeMIPS* slow_path = new (GetGraph()->GetArena()) TypeCheckSlowPathMIPS(instruction);
- codegen_->AddSlowPath(slow_path);
- __ Bne(out, cls, slow_path->GetEntryLabel());
- __ LoadConst32(out, 1);
- __ Bind(slow_path->GetExitLabel());
+ switch (type_check_kind) {
+ case TypeCheckKind::kExactCheck: {
+ // /* HeapReference<Class> */ out = obj->klass_
+ __ LoadFromOffset(kLoadWord, out, obj, class_offset);
+ __ MaybeUnpoisonHeapReference(out);
+ // Classes must be equal for the instanceof to succeed.
+ __ Xor(out, out, cls);
+ __ Sltiu(out, out, 1);
+ break;
+ }
+
+ case TypeCheckKind::kAbstractClassCheck: {
+ // /* HeapReference<Class> */ out = obj->klass_
+ __ LoadFromOffset(kLoadWord, out, obj, class_offset);
+ __ MaybeUnpoisonHeapReference(out);
+ // If the class is abstract, we eagerly fetch the super class of the
+ // object to avoid doing a comparison we know will fail.
+ MipsLabel loop;
+ __ Bind(&loop);
+ // /* HeapReference<Class> */ out = out->super_class_
+ __ LoadFromOffset(kLoadWord, out, out, super_offset);
+ __ MaybeUnpoisonHeapReference(out);
+ // If `out` is null, we use it for the result, and jump to `done`.
+ __ Beqz(out, &done);
+ __ Bne(out, cls, &loop);
+ __ LoadConst32(out, 1);
+ break;
+ }
+
+ case TypeCheckKind::kClassHierarchyCheck: {
+ // /* HeapReference<Class> */ out = obj->klass_
+ __ LoadFromOffset(kLoadWord, out, obj, class_offset);
+ __ MaybeUnpoisonHeapReference(out);
+ // Walk over the class hierarchy to find a match.
+ MipsLabel loop, success;
+ __ Bind(&loop);
+ __ Beq(out, cls, &success);
+ // /* HeapReference<Class> */ out = out->super_class_
+ __ LoadFromOffset(kLoadWord, out, out, super_offset);
+ __ MaybeUnpoisonHeapReference(out);
+ __ Bnez(out, &loop);
+ // If `out` is null, we use it for the result, and jump to `done`.
+ __ B(&done);
+ __ Bind(&success);
+ __ LoadConst32(out, 1);
+ break;
+ }
+
+ case TypeCheckKind::kArrayObjectCheck: {
+ // /* HeapReference<Class> */ out = obj->klass_
+ __ LoadFromOffset(kLoadWord, out, obj, class_offset);
+ __ MaybeUnpoisonHeapReference(out);
+ // Do an exact check.
+ MipsLabel success;
+ __ Beq(out, cls, &success);
+ // Otherwise, we need to check that the object's class is a non-primitive array.
+ // /* HeapReference<Class> */ out = out->component_type_
+ __ LoadFromOffset(kLoadWord, out, out, component_offset);
+ __ MaybeUnpoisonHeapReference(out);
+ // If `out` is null, we use it for the result, and jump to `done`.
+ __ Beqz(out, &done);
+ __ LoadFromOffset(kLoadUnsignedHalfword, out, out, primitive_offset);
+ static_assert(Primitive::kPrimNot == 0, "Expected 0 for kPrimNot");
+ __ Sltiu(out, out, 1);
+ __ B(&done);
+ __ Bind(&success);
+ __ LoadConst32(out, 1);
+ break;
+ }
+
+ case TypeCheckKind::kArrayCheck: {
+ // No read barrier since the slow path will retry upon failure.
+ // /* HeapReference<Class> */ out = obj->klass_
+ __ LoadFromOffset(kLoadWord, out, obj, class_offset);
+ __ MaybeUnpoisonHeapReference(out);
+ DCHECK(locations->OnlyCallsOnSlowPath());
+ slow_path = new (GetGraph()->GetArena()) TypeCheckSlowPathMIPS(instruction,
+ /* is_fatal */ false);
+ codegen_->AddSlowPath(slow_path);
+ __ Bne(out, cls, slow_path->GetEntryLabel());
+ __ LoadConst32(out, 1);
+ break;
+ }
+
+ case TypeCheckKind::kUnresolvedCheck:
+ case TypeCheckKind::kInterfaceCheck: {
+ // Note that we indeed only call on slow path, but we always go
+ // into the slow path for the unresolved and interface check
+ // cases.
+ //
+ // We cannot directly call the InstanceofNonTrivial runtime
+ // entry point without resorting to a type checking slow path
+ // here (i.e. by calling InvokeRuntime directly), as it would
+ // require to assign fixed registers for the inputs of this
+ // HInstanceOf instruction (following the runtime calling
+ // convention), which might be cluttered by the potential first
+ // read barrier emission at the beginning of this method.
+ //
+ // TODO: Introduce a new runtime entry point taking the object
+ // to test (instead of its class) as argument, and let it deal
+ // with the read barrier issues. This will let us refactor this
+ // case of the `switch` code as it was previously (with a direct
+ // call to the runtime not using a type checking slow path).
+ // This should also be beneficial for the other cases above.
+ DCHECK(locations->OnlyCallsOnSlowPath());
+ slow_path = new (GetGraph()->GetArena()) TypeCheckSlowPathMIPS(instruction,
+ /* is_fatal */ false);
+ codegen_->AddSlowPath(slow_path);
+ __ B(slow_path->GetEntryLabel());
+ break;
+ }
}
__ Bind(&done);
+
+ if (slow_path != nullptr) {
+ __ Bind(slow_path->GetExitLabel());
+ }
}
void LocationsBuilderMIPS::VisitIntConstant(HIntConstant* constant) {
diff --git a/compiler/optimizing/code_generator_mips.h b/compiler/optimizing/code_generator_mips.h
index 0ccd80a..98fee24 100644
--- a/compiler/optimizing/code_generator_mips.h
+++ b/compiler/optimizing/code_generator_mips.h
@@ -535,8 +535,6 @@
ArenaDeque<PcRelativePatchInfo> pc_relative_type_patches_;
// PC-relative type patch info for kBssEntry.
ArenaDeque<PcRelativePatchInfo> type_bss_entry_patches_;
- // Deduplication map for patchable boot image addresses.
- Uint32ToLiteralMap boot_image_address_patches_;
// Patches for string root accesses in JIT compiled code.
ArenaDeque<JitPatchInfo> jit_string_patches_;
// Patches for class root accesses in JIT compiled code.
diff --git a/compiler/optimizing/code_generator_mips64.cc b/compiler/optimizing/code_generator_mips64.cc
index 138ebe6..02c3ad6 100644
--- a/compiler/optimizing/code_generator_mips64.cc
+++ b/compiler/optimizing/code_generator_mips64.cc
@@ -336,7 +336,8 @@
class TypeCheckSlowPathMIPS64 : public SlowPathCodeMIPS64 {
public:
- explicit TypeCheckSlowPathMIPS64(HInstruction* instruction) : SlowPathCodeMIPS64(instruction) {}
+ explicit TypeCheckSlowPathMIPS64(HInstruction* instruction, bool is_fatal)
+ : SlowPathCodeMIPS64(instruction), is_fatal_(is_fatal) {}
void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
LocationSummary* locations = instruction_->GetLocations();
@@ -347,7 +348,9 @@
CodeGeneratorMIPS64* mips64_codegen = down_cast<CodeGeneratorMIPS64*>(codegen);
__ Bind(GetEntryLabel());
- SaveLiveRegisters(codegen, locations);
+ if (!is_fatal_) {
+ SaveLiveRegisters(codegen, locations);
+ }
// We're moving two locations to locations that could overlap, so we need a parallel
// move resolver.
@@ -370,13 +373,19 @@
CheckEntrypointTypes<kQuickCheckInstanceOf, void, mirror::Object*, mirror::Class*>();
}
- RestoreLiveRegisters(codegen, locations);
- __ Bc(GetExitLabel());
+ if (!is_fatal_) {
+ RestoreLiveRegisters(codegen, locations);
+ __ Bc(GetExitLabel());
+ }
}
const char* GetDescription() const OVERRIDE { return "TypeCheckSlowPathMIPS64"; }
+ bool IsFatal() const OVERRIDE { return is_fatal_; }
+
private:
+ const bool is_fatal_;
+
DISALLOW_COPY_AND_ASSIGN(TypeCheckSlowPathMIPS64);
};
@@ -430,8 +439,6 @@
graph->GetArena()->Adapter(kArenaAllocCodeGenerator)),
pc_relative_type_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)),
type_bss_entry_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)),
- boot_image_address_patches_(std::less<uint32_t>(),
- graph->GetArena()->Adapter(kArenaAllocCodeGenerator)),
jit_string_patches_(StringReferenceValueComparator(),
graph->GetArena()->Adapter(kArenaAllocCodeGenerator)),
jit_class_patches_(TypeReferenceValueComparator(),
@@ -937,8 +944,7 @@
pc_relative_type_patches_.size() +
type_bss_entry_patches_.size() +
boot_image_string_patches_.size() +
- boot_image_type_patches_.size() +
- boot_image_address_patches_.size();
+ boot_image_type_patches_.size();
linker_patches->reserve(size);
EmitPcRelativeLinkerPatches<LinkerPatch::DexCacheArrayPatch>(pc_relative_dex_cache_patches_,
linker_patches);
@@ -972,13 +978,6 @@
target_type.dex_file,
target_type.type_index.index_));
}
- for (const auto& entry : boot_image_address_patches_) {
- DCHECK(GetCompilerOptions().GetIncludePatchInformation());
- Literal* literal = entry.second;
- DCHECK(literal->GetLabel()->IsBound());
- uint32_t literal_offset = __ GetLabelLocation(literal->GetLabel());
- linker_patches->push_back(LinkerPatch::RecordPosition(literal_offset));
- }
DCHECK_EQ(size, linker_patches->size());
}
@@ -1042,9 +1041,7 @@
}
Literal* CodeGeneratorMIPS64::DeduplicateBootImageAddressLiteral(uint64_t address) {
- bool needs_patch = GetCompilerOptions().GetIncludePatchInformation();
- Uint32ToLiteralMap* map = needs_patch ? &boot_image_address_patches_ : &uint32_literals_;
- return DeduplicateUint32Literal(dchecked_integral_cast<uint32_t>(address), map);
+ return DeduplicateUint32Literal(dchecked_integral_cast<uint32_t>(address), &uint32_literals_);
}
void CodeGeneratorMIPS64::EmitPcRelativeAddressPlaceholderHigh(PcRelativePatchInfo* info,
@@ -1889,31 +1886,178 @@
}
void LocationsBuilderMIPS64::VisitCheckCast(HCheckCast* instruction) {
- LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(
- instruction,
- LocationSummary::kCallOnSlowPath);
+ LocationSummary::CallKind call_kind = LocationSummary::kNoCall;
+ bool throws_into_catch = instruction->CanThrowIntoCatchBlock();
+
+ TypeCheckKind type_check_kind = instruction->GetTypeCheckKind();
+ switch (type_check_kind) {
+ case TypeCheckKind::kExactCheck:
+ case TypeCheckKind::kAbstractClassCheck:
+ case TypeCheckKind::kClassHierarchyCheck:
+ case TypeCheckKind::kArrayObjectCheck:
+ call_kind = throws_into_catch
+ ? LocationSummary::kCallOnSlowPath
+ : LocationSummary::kNoCall; // In fact, call on a fatal (non-returning) slow path.
+ break;
+ case TypeCheckKind::kArrayCheck:
+ case TypeCheckKind::kUnresolvedCheck:
+ case TypeCheckKind::kInterfaceCheck:
+ call_kind = LocationSummary::kCallOnSlowPath;
+ break;
+ }
+
+ LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instruction, call_kind);
locations->SetInAt(0, Location::RequiresRegister());
locations->SetInAt(1, Location::RequiresRegister());
- // Note that TypeCheckSlowPathMIPS64 uses this register too.
locations->AddTemp(Location::RequiresRegister());
}
void InstructionCodeGeneratorMIPS64::VisitCheckCast(HCheckCast* instruction) {
+ TypeCheckKind type_check_kind = instruction->GetTypeCheckKind();
LocationSummary* locations = instruction->GetLocations();
GpuRegister obj = locations->InAt(0).AsRegister<GpuRegister>();
GpuRegister cls = locations->InAt(1).AsRegister<GpuRegister>();
- GpuRegister obj_cls = locations->GetTemp(0).AsRegister<GpuRegister>();
+ GpuRegister temp = locations->GetTemp(0).AsRegister<GpuRegister>();
+ const uint32_t class_offset = mirror::Object::ClassOffset().Int32Value();
+ const uint32_t super_offset = mirror::Class::SuperClassOffset().Int32Value();
+ const uint32_t component_offset = mirror::Class::ComponentTypeOffset().Int32Value();
+ const uint32_t primitive_offset = mirror::Class::PrimitiveTypeOffset().Int32Value();
+ const uint32_t iftable_offset = mirror::Class::IfTableOffset().Uint32Value();
+ const uint32_t array_length_offset = mirror::Array::LengthOffset().Uint32Value();
+ const uint32_t object_array_data_offset =
+ mirror::Array::DataOffset(kHeapReferenceSize).Uint32Value();
+ Mips64Label done;
+ // Always false for read barriers since we may need to go to the entrypoint for non-fatal cases
+ // from false negatives. The false negatives may come from avoiding read barriers below. Avoiding
+ // read barriers is done for performance and code size reasons.
+ bool is_type_check_slow_path_fatal = false;
+ if (!kEmitCompilerReadBarrier) {
+ is_type_check_slow_path_fatal =
+ (type_check_kind == TypeCheckKind::kExactCheck ||
+ type_check_kind == TypeCheckKind::kAbstractClassCheck ||
+ type_check_kind == TypeCheckKind::kClassHierarchyCheck ||
+ type_check_kind == TypeCheckKind::kArrayObjectCheck) &&
+ !instruction->CanThrowIntoCatchBlock();
+ }
SlowPathCodeMIPS64* slow_path =
- new (GetGraph()->GetArena()) TypeCheckSlowPathMIPS64(instruction);
+ new (GetGraph()->GetArena()) TypeCheckSlowPathMIPS64(instruction,
+ is_type_check_slow_path_fatal);
codegen_->AddSlowPath(slow_path);
- // TODO: avoid this check if we know obj is not null.
- __ Beqzc(obj, slow_path->GetExitLabel());
- // Compare the class of `obj` with `cls`.
- __ LoadFromOffset(kLoadUnsignedWord, obj_cls, obj, mirror::Object::ClassOffset().Int32Value());
- __ MaybeUnpoisonHeapReference(obj_cls);
- __ Bnec(obj_cls, cls, slow_path->GetEntryLabel());
+ // Avoid this check if we know `obj` is not null.
+ if (instruction->MustDoNullCheck()) {
+ __ Beqzc(obj, &done);
+ }
+
+ switch (type_check_kind) {
+ case TypeCheckKind::kExactCheck:
+ case TypeCheckKind::kArrayCheck: {
+ // /* HeapReference<Class> */ temp = obj->klass_
+ __ LoadFromOffset(kLoadUnsignedWord, temp, obj, class_offset);
+ __ MaybeUnpoisonHeapReference(temp);
+ // Jump to slow path for throwing the exception or doing a
+ // more involved array check.
+ __ Bnec(temp, cls, slow_path->GetEntryLabel());
+ break;
+ }
+
+ case TypeCheckKind::kAbstractClassCheck: {
+ // /* HeapReference<Class> */ temp = obj->klass_
+ __ LoadFromOffset(kLoadUnsignedWord, temp, obj, class_offset);
+ __ MaybeUnpoisonHeapReference(temp);
+ // If the class is abstract, we eagerly fetch the super class of the
+ // object to avoid doing a comparison we know will fail.
+ Mips64Label loop;
+ __ Bind(&loop);
+ // /* HeapReference<Class> */ temp = temp->super_class_
+ __ LoadFromOffset(kLoadUnsignedWord, temp, temp, super_offset);
+ __ MaybeUnpoisonHeapReference(temp);
+ // If the class reference currently in `temp` is null, jump to the slow path to throw the
+ // exception.
+ __ Beqzc(temp, slow_path->GetEntryLabel());
+ // Otherwise, compare the classes.
+ __ Bnec(temp, cls, &loop);
+ break;
+ }
+
+ case TypeCheckKind::kClassHierarchyCheck: {
+ // /* HeapReference<Class> */ temp = obj->klass_
+ __ LoadFromOffset(kLoadUnsignedWord, temp, obj, class_offset);
+ __ MaybeUnpoisonHeapReference(temp);
+ // Walk over the class hierarchy to find a match.
+ Mips64Label loop;
+ __ Bind(&loop);
+ __ Beqc(temp, cls, &done);
+ // /* HeapReference<Class> */ temp = temp->super_class_
+ __ LoadFromOffset(kLoadUnsignedWord, temp, temp, super_offset);
+ __ MaybeUnpoisonHeapReference(temp);
+ // If the class reference currently in `temp` is null, jump to the slow path to throw the
+ // exception. Otherwise, jump to the beginning of the loop.
+ __ Bnezc(temp, &loop);
+ __ Bc(slow_path->GetEntryLabel());
+ break;
+ }
+
+ case TypeCheckKind::kArrayObjectCheck: {
+ // /* HeapReference<Class> */ temp = obj->klass_
+ __ LoadFromOffset(kLoadUnsignedWord, temp, obj, class_offset);
+ __ MaybeUnpoisonHeapReference(temp);
+ // Do an exact check.
+ __ Beqc(temp, cls, &done);
+ // Otherwise, we need to check that the object's class is a non-primitive array.
+ // /* HeapReference<Class> */ temp = temp->component_type_
+ __ LoadFromOffset(kLoadUnsignedWord, temp, temp, component_offset);
+ __ MaybeUnpoisonHeapReference(temp);
+ // If the component type is null, jump to the slow path to throw the exception.
+ __ Beqzc(temp, slow_path->GetEntryLabel());
+ // Otherwise, the object is indeed an array, further check that this component
+ // type is not a primitive type.
+ __ LoadFromOffset(kLoadUnsignedHalfword, temp, temp, primitive_offset);
+ static_assert(Primitive::kPrimNot == 0, "Expected 0 for kPrimNot");
+ __ Bnezc(temp, slow_path->GetEntryLabel());
+ break;
+ }
+
+ case TypeCheckKind::kUnresolvedCheck:
+ // We always go into the type check slow path for the unresolved check case.
+ // We cannot directly call the CheckCast runtime entry point
+ // without resorting to a type checking slow path here (i.e. by
+ // calling InvokeRuntime directly), as it would require to
+ // assign fixed registers for the inputs of this HInstanceOf
+ // instruction (following the runtime calling convention), which
+ // might be cluttered by the potential first read barrier
+ // emission at the beginning of this method.
+ __ Bc(slow_path->GetEntryLabel());
+ break;
+
+ case TypeCheckKind::kInterfaceCheck: {
+ // Avoid read barriers to improve performance of the fast path. We can not get false
+ // positives by doing this.
+ // /* HeapReference<Class> */ temp = obj->klass_
+ __ LoadFromOffset(kLoadUnsignedWord, temp, obj, class_offset);
+ __ MaybeUnpoisonHeapReference(temp);
+ // /* HeapReference<Class> */ temp = temp->iftable_
+ __ LoadFromOffset(kLoadUnsignedWord, temp, temp, iftable_offset);
+ __ MaybeUnpoisonHeapReference(temp);
+ // Iftable is never null.
+ __ Lw(TMP, temp, array_length_offset);
+ // Loop through the iftable and check if any class matches.
+ Mips64Label loop;
+ __ Bind(&loop);
+ __ Beqzc(TMP, slow_path->GetEntryLabel());
+ __ Lwu(AT, temp, object_array_data_offset);
+ __ MaybeUnpoisonHeapReference(AT);
+ // Go to next interface.
+ __ Daddiu(temp, temp, 2 * kHeapReferenceSize);
+ __ Addiu(TMP, TMP, -2);
+ // Compare the classes and continue the loop if they do not match.
+ __ Bnec(AT, cls, &loop);
+ break;
+ }
+ }
+
+ __ Bind(&done);
__ Bind(slow_path->GetExitLabel());
}
@@ -3282,8 +3426,22 @@
}
void LocationsBuilderMIPS64::VisitInstanceOf(HInstanceOf* instruction) {
- LocationSummary::CallKind call_kind =
- instruction->IsExactCheck() ? LocationSummary::kNoCall : LocationSummary::kCallOnSlowPath;
+ LocationSummary::CallKind call_kind = LocationSummary::kNoCall;
+ TypeCheckKind type_check_kind = instruction->GetTypeCheckKind();
+ switch (type_check_kind) {
+ case TypeCheckKind::kExactCheck:
+ case TypeCheckKind::kAbstractClassCheck:
+ case TypeCheckKind::kClassHierarchyCheck:
+ case TypeCheckKind::kArrayObjectCheck:
+ call_kind = LocationSummary::kNoCall;
+ break;
+ case TypeCheckKind::kArrayCheck:
+ case TypeCheckKind::kUnresolvedCheck:
+ case TypeCheckKind::kInterfaceCheck:
+ call_kind = LocationSummary::kCallOnSlowPath;
+ break;
+ }
+
LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instruction, call_kind);
locations->SetInAt(0, Location::RequiresRegister());
locations->SetInAt(1, Location::RequiresRegister());
@@ -3293,37 +3451,143 @@
}
void InstructionCodeGeneratorMIPS64::VisitInstanceOf(HInstanceOf* instruction) {
+ TypeCheckKind type_check_kind = instruction->GetTypeCheckKind();
LocationSummary* locations = instruction->GetLocations();
GpuRegister obj = locations->InAt(0).AsRegister<GpuRegister>();
GpuRegister cls = locations->InAt(1).AsRegister<GpuRegister>();
GpuRegister out = locations->Out().AsRegister<GpuRegister>();
-
+ uint32_t class_offset = mirror::Object::ClassOffset().Int32Value();
+ uint32_t super_offset = mirror::Class::SuperClassOffset().Int32Value();
+ uint32_t component_offset = mirror::Class::ComponentTypeOffset().Int32Value();
+ uint32_t primitive_offset = mirror::Class::PrimitiveTypeOffset().Int32Value();
Mips64Label done;
+ SlowPathCodeMIPS64* slow_path = nullptr;
// Return 0 if `obj` is null.
- // TODO: Avoid this check if we know `obj` is not null.
- __ Move(out, ZERO);
- __ Beqzc(obj, &done);
+ // Avoid this check if we know `obj` is not null.
+ if (instruction->MustDoNullCheck()) {
+ __ Move(out, ZERO);
+ __ Beqzc(obj, &done);
+ }
- // Compare the class of `obj` with `cls`.
- __ LoadFromOffset(kLoadUnsignedWord, out, obj, mirror::Object::ClassOffset().Int32Value());
- __ MaybeUnpoisonHeapReference(out);
- if (instruction->IsExactCheck()) {
- // Classes must be equal for the instanceof to succeed.
- __ Xor(out, out, cls);
- __ Sltiu(out, out, 1);
- } else {
- // If the classes are not equal, we go into a slow path.
- DCHECK(locations->OnlyCallsOnSlowPath());
- SlowPathCodeMIPS64* slow_path =
- new (GetGraph()->GetArena()) TypeCheckSlowPathMIPS64(instruction);
- codegen_->AddSlowPath(slow_path);
- __ Bnec(out, cls, slow_path->GetEntryLabel());
- __ LoadConst32(out, 1);
- __ Bind(slow_path->GetExitLabel());
+ switch (type_check_kind) {
+ case TypeCheckKind::kExactCheck: {
+ // /* HeapReference<Class> */ out = obj->klass_
+ __ LoadFromOffset(kLoadUnsignedWord, out, obj, class_offset);
+ __ MaybeUnpoisonHeapReference(out);
+ // Classes must be equal for the instanceof to succeed.
+ __ Xor(out, out, cls);
+ __ Sltiu(out, out, 1);
+ break;
+ }
+
+ case TypeCheckKind::kAbstractClassCheck: {
+ // /* HeapReference<Class> */ out = obj->klass_
+ __ LoadFromOffset(kLoadUnsignedWord, out, obj, class_offset);
+ __ MaybeUnpoisonHeapReference(out);
+ // If the class is abstract, we eagerly fetch the super class of the
+ // object to avoid doing a comparison we know will fail.
+ Mips64Label loop;
+ __ Bind(&loop);
+ // /* HeapReference<Class> */ out = out->super_class_
+ __ LoadFromOffset(kLoadUnsignedWord, out, out, super_offset);
+ __ MaybeUnpoisonHeapReference(out);
+ // If `out` is null, we use it for the result, and jump to `done`.
+ __ Beqzc(out, &done);
+ __ Bnec(out, cls, &loop);
+ __ LoadConst32(out, 1);
+ break;
+ }
+
+ case TypeCheckKind::kClassHierarchyCheck: {
+ // /* HeapReference<Class> */ out = obj->klass_
+ __ LoadFromOffset(kLoadUnsignedWord, out, obj, class_offset);
+ __ MaybeUnpoisonHeapReference(out);
+ // Walk over the class hierarchy to find a match.
+ Mips64Label loop, success;
+ __ Bind(&loop);
+ __ Beqc(out, cls, &success);
+ // /* HeapReference<Class> */ out = out->super_class_
+ __ LoadFromOffset(kLoadUnsignedWord, out, out, super_offset);
+ __ MaybeUnpoisonHeapReference(out);
+ __ Bnezc(out, &loop);
+ // If `out` is null, we use it for the result, and jump to `done`.
+ __ Bc(&done);
+ __ Bind(&success);
+ __ LoadConst32(out, 1);
+ break;
+ }
+
+ case TypeCheckKind::kArrayObjectCheck: {
+ // /* HeapReference<Class> */ out = obj->klass_
+ __ LoadFromOffset(kLoadUnsignedWord, out, obj, class_offset);
+ __ MaybeUnpoisonHeapReference(out);
+ // Do an exact check.
+ Mips64Label success;
+ __ Beqc(out, cls, &success);
+ // Otherwise, we need to check that the object's class is a non-primitive array.
+ // /* HeapReference<Class> */ out = out->component_type_
+ __ LoadFromOffset(kLoadUnsignedWord, out, out, component_offset);
+ __ MaybeUnpoisonHeapReference(out);
+ // If `out` is null, we use it for the result, and jump to `done`.
+ __ Beqzc(out, &done);
+ __ LoadFromOffset(kLoadUnsignedHalfword, out, out, primitive_offset);
+ static_assert(Primitive::kPrimNot == 0, "Expected 0 for kPrimNot");
+ __ Sltiu(out, out, 1);
+ __ Bc(&done);
+ __ Bind(&success);
+ __ LoadConst32(out, 1);
+ break;
+ }
+
+ case TypeCheckKind::kArrayCheck: {
+ // No read barrier since the slow path will retry upon failure.
+ // /* HeapReference<Class> */ out = obj->klass_
+ __ LoadFromOffset(kLoadUnsignedWord, out, obj, class_offset);
+ __ MaybeUnpoisonHeapReference(out);
+ DCHECK(locations->OnlyCallsOnSlowPath());
+ slow_path = new (GetGraph()->GetArena()) TypeCheckSlowPathMIPS64(instruction,
+ /* is_fatal */ false);
+ codegen_->AddSlowPath(slow_path);
+ __ Bnec(out, cls, slow_path->GetEntryLabel());
+ __ LoadConst32(out, 1);
+ break;
+ }
+
+ case TypeCheckKind::kUnresolvedCheck:
+ case TypeCheckKind::kInterfaceCheck: {
+ // Note that we indeed only call on slow path, but we always go
+ // into the slow path for the unresolved and interface check
+ // cases.
+ //
+ // We cannot directly call the InstanceofNonTrivial runtime
+ // entry point without resorting to a type checking slow path
+ // here (i.e. by calling InvokeRuntime directly), as it would
+ // require to assign fixed registers for the inputs of this
+ // HInstanceOf instruction (following the runtime calling
+ // convention), which might be cluttered by the potential first
+ // read barrier emission at the beginning of this method.
+ //
+ // TODO: Introduce a new runtime entry point taking the object
+ // to test (instead of its class) as argument, and let it deal
+ // with the read barrier issues. This will let us refactor this
+ // case of the `switch` code as it was previously (with a direct
+ // call to the runtime not using a type checking slow path).
+ // This should also be beneficial for the other cases above.
+ DCHECK(locations->OnlyCallsOnSlowPath());
+ slow_path = new (GetGraph()->GetArena()) TypeCheckSlowPathMIPS64(instruction,
+ /* is_fatal */ false);
+ codegen_->AddSlowPath(slow_path);
+ __ Bc(slow_path->GetEntryLabel());
+ break;
+ }
}
__ Bind(&done);
+
+ if (slow_path != nullptr) {
+ __ Bind(slow_path->GetExitLabel());
+ }
}
void LocationsBuilderMIPS64::VisitIntConstant(HIntConstant* constant) {
diff --git a/compiler/optimizing/code_generator_mips64.h b/compiler/optimizing/code_generator_mips64.h
index 26cc7dc..3056f7f 100644
--- a/compiler/optimizing/code_generator_mips64.h
+++ b/compiler/optimizing/code_generator_mips64.h
@@ -492,8 +492,6 @@
ArenaDeque<PcRelativePatchInfo> pc_relative_type_patches_;
// PC-relative type patch info for kBssEntry.
ArenaDeque<PcRelativePatchInfo> type_bss_entry_patches_;
- // Deduplication map for patchable boot image addresses.
- Uint32ToLiteralMap boot_image_address_patches_;
// Patches for string root accesses in JIT compiled code.
StringToLiteralMap jit_string_patches_;
// Patches for class root accesses in JIT compiled code.
diff --git a/compiler/optimizing/code_generator_x86.cc b/compiler/optimizing/code_generator_x86.cc
index b779aed..0b50619 100644
--- a/compiler/optimizing/code_generator_x86.cc
+++ b/compiler/optimizing/code_generator_x86.cc
@@ -1015,7 +1015,6 @@
assembler_(graph->GetArena()),
isa_features_(isa_features),
pc_relative_dex_cache_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)),
- simple_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)),
string_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)),
boot_image_type_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)),
type_bss_entry_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)),
@@ -4603,13 +4602,6 @@
temp, ArtMethod::EntryPointFromQuickCompiledCodeOffset(kX86PointerSize).Int32Value()));
}
-void CodeGeneratorX86::RecordSimplePatch() {
- if (GetCompilerOptions().GetIncludePatchInformation()) {
- simple_patches_.emplace_back();
- __ Bind(&simple_patches_.back());
- }
-}
-
void CodeGeneratorX86::RecordBootStringPatch(HLoadString* load_string) {
DCHECK(GetCompilerOptions().IsBootImage());
HX86ComputeBaseMethodAddress* address = nullptr;
@@ -4682,17 +4674,12 @@
DCHECK(linker_patches->empty());
size_t size =
pc_relative_dex_cache_patches_.size() +
- simple_patches_.size() +
string_patches_.size() +
boot_image_type_patches_.size() +
type_bss_entry_patches_.size();
linker_patches->reserve(size);
EmitPcRelativeLinkerPatches<LinkerPatch::DexCacheArrayPatch>(pc_relative_dex_cache_patches_,
linker_patches);
- for (const Label& label : simple_patches_) {
- uint32_t literal_offset = label.Position() - kLabelPositionToLiteralOffsetAdjustment;
- linker_patches->push_back(LinkerPatch::RecordPosition(literal_offset));
- }
if (!GetCompilerOptions().IsBootImage()) {
DCHECK(boot_image_type_patches_.empty());
EmitPcRelativeLinkerPatches<LinkerPatch::StringBssEntryPatch>(string_patches_, linker_patches);
@@ -6154,7 +6141,6 @@
reinterpret_cast<uintptr_t>(cls->GetClass().Get()));
DCHECK_NE(address, 0u);
__ movl(out, Immediate(address));
- codegen_->RecordSimplePatch();
break;
}
case HLoadClass::LoadKind::kBssEntry: {
@@ -6311,7 +6297,6 @@
reinterpret_cast<uintptr_t>(load->GetString().Get()));
DCHECK_NE(address, 0u);
__ movl(out, Immediate(address));
- codegen_->RecordSimplePatch();
return; // No dex cache slow path.
}
case HLoadString::LoadKind::kBssEntry: {
diff --git a/compiler/optimizing/code_generator_x86.h b/compiler/optimizing/code_generator_x86.h
index 5360dc9..65ee383 100644
--- a/compiler/optimizing/code_generator_x86.h
+++ b/compiler/optimizing/code_generator_x86.h
@@ -412,7 +412,6 @@
// Generate a call to a virtual method.
void GenerateVirtualCall(HInvokeVirtual* invoke, Location temp) OVERRIDE;
- void RecordSimplePatch();
void RecordBootStringPatch(HLoadString* load_string);
void RecordBootTypePatch(HLoadClass* load_class);
Label* NewTypeBssEntryPatch(HLoadClass* load_class);
@@ -633,8 +632,6 @@
// PC-relative DexCache access info.
ArenaDeque<X86PcRelativePatchInfo> pc_relative_dex_cache_patches_;
- // Patch locations for patchoat where the linker doesn't do any other work.
- ArenaDeque<Label> simple_patches_;
// String patch locations; type depends on configuration (app .bss or boot image PIC/non-PIC).
ArenaDeque<X86PcRelativePatchInfo> string_patches_;
// Type patch locations for boot image; type depends on configuration (boot image PIC/non-PIC).
diff --git a/compiler/optimizing/code_generator_x86_64.cc b/compiler/optimizing/code_generator_x86_64.cc
index 179bf6d..644fcee 100644
--- a/compiler/optimizing/code_generator_x86_64.cc
+++ b/compiler/optimizing/code_generator_x86_64.cc
@@ -1070,13 +1070,6 @@
kX86_64PointerSize).SizeValue()));
}
-void CodeGeneratorX86_64::RecordSimplePatch() {
- if (GetCompilerOptions().GetIncludePatchInformation()) {
- simple_patches_.emplace_back();
- __ Bind(&simple_patches_.back());
- }
-}
-
void CodeGeneratorX86_64::RecordBootStringPatch(HLoadString* load_string) {
DCHECK(GetCompilerOptions().IsBootImage());
string_patches_.emplace_back(load_string->GetDexFile(), load_string->GetStringIndex().index_);
@@ -1126,17 +1119,12 @@
DCHECK(linker_patches->empty());
size_t size =
pc_relative_dex_cache_patches_.size() +
- simple_patches_.size() +
string_patches_.size() +
boot_image_type_patches_.size() +
type_bss_entry_patches_.size();
linker_patches->reserve(size);
EmitPcRelativeLinkerPatches<LinkerPatch::DexCacheArrayPatch>(pc_relative_dex_cache_patches_,
linker_patches);
- for (const Label& label : simple_patches_) {
- uint32_t literal_offset = label.Position() - kLabelPositionToLiteralOffsetAdjustment;
- linker_patches->push_back(LinkerPatch::RecordPosition(literal_offset));
- }
if (!GetCompilerOptions().IsBootImage()) {
DCHECK(boot_image_type_patches_.empty());
EmitPcRelativeLinkerPatches<LinkerPatch::StringBssEntryPatch>(string_patches_, linker_patches);
@@ -1227,7 +1215,6 @@
isa_features_(isa_features),
constant_area_start_(0),
pc_relative_dex_cache_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)),
- simple_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)),
string_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)),
boot_image_type_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)),
type_bss_entry_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)),
@@ -5545,7 +5532,6 @@
reinterpret_cast<uintptr_t>(cls->GetClass().Get()));
DCHECK_NE(address, 0u);
__ movl(out, Immediate(address)); // Zero-extended.
- codegen_->RecordSimplePatch();
break;
}
case HLoadClass::LoadKind::kBssEntry: {
@@ -5681,7 +5667,6 @@
reinterpret_cast<uintptr_t>(load->GetString().Get()));
DCHECK_NE(address, 0u);
__ movl(out, Immediate(address)); // Zero-extended.
- codegen_->RecordSimplePatch();
return; // No dex cache slow path.
}
case HLoadString::LoadKind::kBssEntry: {
diff --git a/compiler/optimizing/code_generator_x86_64.h b/compiler/optimizing/code_generator_x86_64.h
index 3a83731..376c3ce 100644
--- a/compiler/optimizing/code_generator_x86_64.h
+++ b/compiler/optimizing/code_generator_x86_64.h
@@ -406,7 +406,6 @@
void GenerateStaticOrDirectCall(HInvokeStaticOrDirect* invoke, Location temp) OVERRIDE;
void GenerateVirtualCall(HInvokeVirtual* invoke, Location temp) OVERRIDE;
- void RecordSimplePatch();
void RecordBootStringPatch(HLoadString* load_string);
void RecordBootTypePatch(HLoadClass* load_class);
Label* NewTypeBssEntryPatch(HLoadClass* load_class);
@@ -602,8 +601,6 @@
// PC-relative DexCache access info.
ArenaDeque<PatchInfo<Label>> pc_relative_dex_cache_patches_;
- // Patch locations for patchoat where the linker doesn't do any other work.
- ArenaDeque<Label> simple_patches_;
// String patch locations; type depends on configuration (app .bss or boot image PIC).
ArenaDeque<PatchInfo<Label>> string_patches_;
// Type patch locations for boot image (always PIC).
diff --git a/compiler/optimizing/inliner.cc b/compiler/optimizing/inliner.cc
index e3926c5..664b95a 100644
--- a/compiler/optimizing/inliner.cc
+++ b/compiler/optimizing/inliner.cc
@@ -249,20 +249,25 @@
ProfilingInfo* const profiling_info_;
};
-static bool IsMonomorphic(Handle<mirror::ObjectArray<mirror::Class>> classes)
- REQUIRES_SHARED(Locks::mutator_lock_) {
- DCHECK_GE(InlineCache::kIndividualCacheSize, 2);
- return classes->Get(0) != nullptr && classes->Get(1) == nullptr;
-}
-
-static bool IsMegamorphic(Handle<mirror::ObjectArray<mirror::Class>> classes)
- REQUIRES_SHARED(Locks::mutator_lock_) {
- for (size_t i = 0; i < InlineCache::kIndividualCacheSize; ++i) {
- if (classes->Get(i) == nullptr) {
- return false;
+HInliner::InlineCacheType HInliner::GetInlineCacheType(
+ const Handle<mirror::ObjectArray<mirror::Class>>& classes)
+ REQUIRES_SHARED(Locks::mutator_lock_) {
+ uint8_t number_of_types = 0;
+ for (; number_of_types < InlineCache::kIndividualCacheSize; ++number_of_types) {
+ if (classes->Get(number_of_types) == nullptr) {
+ break;
}
}
- return true;
+
+ if (number_of_types == 0) {
+ return kInlineCacheUninitialized;
+ } else if (number_of_types == 1) {
+ return kInlineCacheMonomorphic;
+ } else if (number_of_types == InlineCache::kIndividualCacheSize) {
+ return kInlineCacheMegamorphic;
+ } else {
+ return kInlineCachePolymorphic;
+ }
}
static mirror::Class* GetMonomorphicType(Handle<mirror::ObjectArray<mirror::Class>> classes)
@@ -271,18 +276,6 @@
return classes->Get(0);
}
-static bool IsUninitialized(Handle<mirror::ObjectArray<mirror::Class>> classes)
- REQUIRES_SHARED(Locks::mutator_lock_) {
- return classes->Get(0) == nullptr;
-}
-
-static bool IsPolymorphic(Handle<mirror::ObjectArray<mirror::Class>> classes)
- REQUIRES_SHARED(Locks::mutator_lock_) {
- DCHECK_GE(InlineCache::kIndividualCacheSize, 3);
- return classes->Get(1) != nullptr &&
- classes->Get(InlineCache::kIndividualCacheSize - 1) == nullptr;
-}
-
ArtMethod* HInliner::TryCHADevirtualization(ArtMethod* resolved_method) {
if (!resolved_method->HasSingleImplementation()) {
return nullptr;
@@ -353,67 +346,209 @@
}
return result;
}
-
DCHECK(!invoke_instruction->IsInvokeStaticOrDirect());
- // Check if we can use an inline cache.
- ArtMethod* caller = graph_->GetArtMethod();
- if (Runtime::Current()->UseJitCompilation()) {
- // Under JIT, we should always know the caller.
- DCHECK(caller != nullptr);
- ScopedProfilingInfoInlineUse spiis(caller, soa.Self());
- ProfilingInfo* profiling_info = spiis.GetProfilingInfo();
- if (profiling_info != nullptr) {
- StackHandleScope<1> hs(soa.Self());
- ClassLinker* class_linker = caller_compilation_unit_.GetClassLinker();
- Handle<mirror::ObjectArray<mirror::Class>> inline_cache = hs.NewHandle(
- mirror::ObjectArray<mirror::Class>::Alloc(
- soa.Self(),
- class_linker->GetClassRoot(ClassLinker::kClassArrayClass),
- InlineCache::kIndividualCacheSize));
- if (inline_cache == nullptr) {
- // We got an OOME. Just clear the exception, and don't inline.
- DCHECK(soa.Self()->IsExceptionPending());
- soa.Self()->ClearException();
- VLOG(compiler) << "Out of memory in the compiler when trying to inline";
- return false;
+ // Try using inline caches.
+ return TryInlineFromInlineCache(caller_dex_file, invoke_instruction, resolved_method);
+}
+
+static Handle<mirror::ObjectArray<mirror::Class>> AllocateInlineCacheHolder(
+ const DexCompilationUnit& compilation_unit,
+ StackHandleScope<1>* hs)
+ REQUIRES_SHARED(Locks::mutator_lock_) {
+ Thread* self = Thread::Current();
+ ClassLinker* class_linker = compilation_unit.GetClassLinker();
+ Handle<mirror::ObjectArray<mirror::Class>> inline_cache = hs->NewHandle(
+ mirror::ObjectArray<mirror::Class>::Alloc(
+ self,
+ class_linker->GetClassRoot(ClassLinker::kClassArrayClass),
+ InlineCache::kIndividualCacheSize));
+ if (inline_cache == nullptr) {
+ // We got an OOME. Just clear the exception, and don't inline.
+ DCHECK(self->IsExceptionPending());
+ self->ClearException();
+ VLOG(compiler) << "Out of memory in the compiler when trying to inline";
+ }
+ return inline_cache;
+}
+
+bool HInliner::TryInlineFromInlineCache(const DexFile& caller_dex_file,
+ HInvoke* invoke_instruction,
+ ArtMethod* resolved_method)
+ REQUIRES_SHARED(Locks::mutator_lock_) {
+ StackHandleScope<1> hs(Thread::Current());
+ Handle<mirror::ObjectArray<mirror::Class>> inline_cache;
+ InlineCacheType inline_cache_type = Runtime::Current()->IsAotCompiler()
+ ? GetInlineCacheAOT(caller_dex_file, invoke_instruction, &hs, &inline_cache)
+ : GetInlineCacheJIT(invoke_instruction, &hs, &inline_cache);
+
+ switch (inline_cache_type) {
+ case kInlineCacheNoData:
+ break;
+
+ case kInlineCacheUninitialized:
+ VLOG(compiler) << "Interface or virtual call to "
+ << caller_dex_file.PrettyMethod(invoke_instruction->GetDexMethodIndex())
+ << " is not hit and not inlined";
+ return false;
+
+ case kInlineCacheMonomorphic:
+ MaybeRecordStat(kMonomorphicCall);
+ if (outermost_graph_->IsCompilingOsr()) {
+ // If we are compiling OSR, we pretend this call is polymorphic, as we may come from the
+ // interpreter and it may have seen different receiver types.
+ return TryInlinePolymorphicCall(invoke_instruction, resolved_method, inline_cache);
} else {
- Runtime::Current()->GetJit()->GetCodeCache()->CopyInlineCacheInto(
- *profiling_info->GetInlineCache(invoke_instruction->GetDexPc()),
- inline_cache);
- if (IsUninitialized(inline_cache)) {
- VLOG(compiler) << "Interface or virtual call to "
- << caller_dex_file.PrettyMethod(method_index)
- << " is not hit and not inlined";
- return false;
- } else if (IsMonomorphic(inline_cache)) {
- MaybeRecordStat(kMonomorphicCall);
- if (outermost_graph_->IsCompilingOsr()) {
- // If we are compiling OSR, we pretend this call is polymorphic, as we may come from the
- // interpreter and it may have seen different receiver types.
- return TryInlinePolymorphicCall(invoke_instruction, resolved_method, inline_cache);
- } else {
- return TryInlineMonomorphicCall(invoke_instruction, resolved_method, inline_cache);
- }
- } else if (IsPolymorphic(inline_cache)) {
- MaybeRecordStat(kPolymorphicCall);
- return TryInlinePolymorphicCall(invoke_instruction, resolved_method, inline_cache);
- } else {
- DCHECK(IsMegamorphic(inline_cache));
- VLOG(compiler) << "Interface or virtual call to "
- << caller_dex_file.PrettyMethod(method_index)
- << " is megamorphic and not inlined";
- MaybeRecordStat(kMegamorphicCall);
- return false;
- }
+ return TryInlineMonomorphicCall(invoke_instruction, resolved_method, inline_cache);
}
+
+ case kInlineCachePolymorphic:
+ MaybeRecordStat(kPolymorphicCall);
+ return TryInlinePolymorphicCall(invoke_instruction, resolved_method, inline_cache);
+
+ case kInlineCacheMegamorphic:
+ VLOG(compiler) << "Interface or virtual call to "
+ << caller_dex_file.PrettyMethod(invoke_instruction->GetDexMethodIndex())
+ << " is megamorphic and not inlined";
+ MaybeRecordStat(kMegamorphicCall);
+ return false;
+
+ case kInlineCacheMissingTypes:
+ VLOG(compiler) << "Interface or virtual call to "
+ << caller_dex_file.PrettyMethod(invoke_instruction->GetDexMethodIndex())
+ << " is missing types and not inlined";
+ return false;
+ }
+ UNREACHABLE();
+}
+
+HInliner::InlineCacheType HInliner::GetInlineCacheJIT(
+ HInvoke* invoke_instruction,
+ StackHandleScope<1>* hs,
+ /*out*/Handle<mirror::ObjectArray<mirror::Class>>* inline_cache)
+ REQUIRES_SHARED(Locks::mutator_lock_) {
+ DCHECK(Runtime::Current()->UseJitCompilation());
+
+ ArtMethod* caller = graph_->GetArtMethod();
+ // Under JIT, we should always know the caller.
+ DCHECK(caller != nullptr);
+ ScopedProfilingInfoInlineUse spiis(caller, Thread::Current());
+ ProfilingInfo* profiling_info = spiis.GetProfilingInfo();
+
+ if (profiling_info == nullptr) {
+ return kInlineCacheNoData;
+ }
+
+ *inline_cache = AllocateInlineCacheHolder(caller_compilation_unit_, hs);
+ if (inline_cache->Get() == nullptr) {
+ // We can't extract any data if we failed to allocate;
+ return kInlineCacheNoData;
+ } else {
+ Runtime::Current()->GetJit()->GetCodeCache()->CopyInlineCacheInto(
+ *profiling_info->GetInlineCache(invoke_instruction->GetDexPc()),
+ *inline_cache);
+ return GetInlineCacheType(*inline_cache);
+ }
+}
+
+HInliner::InlineCacheType HInliner::GetInlineCacheAOT(
+ const DexFile& caller_dex_file,
+ HInvoke* invoke_instruction,
+ StackHandleScope<1>* hs,
+ /*out*/Handle<mirror::ObjectArray<mirror::Class>>* inline_cache)
+ REQUIRES_SHARED(Locks::mutator_lock_) {
+ DCHECK(Runtime::Current()->IsAotCompiler());
+ const ProfileCompilationInfo* pci = compiler_driver_->GetProfileCompilationInfo();
+ if (pci == nullptr) {
+ return kInlineCacheNoData;
+ }
+
+ ProfileCompilationInfo::OfflineProfileMethodInfo offline_profile;
+ bool found = pci->GetMethod(caller_dex_file.GetLocation(),
+ caller_dex_file.GetLocationChecksum(),
+ caller_compilation_unit_.GetDexMethodIndex(),
+ &offline_profile);
+ if (!found) {
+ return kInlineCacheNoData; // no profile information for this invocation.
+ }
+
+ *inline_cache = AllocateInlineCacheHolder(caller_compilation_unit_, hs);
+ if (inline_cache == nullptr) {
+ // We can't extract any data if we failed to allocate;
+ return kInlineCacheNoData;
+ } else {
+ return ExtractClassesFromOfflineProfile(invoke_instruction,
+ offline_profile,
+ *inline_cache);
+ }
+}
+
+HInliner::InlineCacheType HInliner::ExtractClassesFromOfflineProfile(
+ const HInvoke* invoke_instruction,
+ const ProfileCompilationInfo::OfflineProfileMethodInfo& offline_profile,
+ /*out*/Handle<mirror::ObjectArray<mirror::Class>> inline_cache)
+ REQUIRES_SHARED(Locks::mutator_lock_) {
+ const auto it = offline_profile.inline_caches.find(invoke_instruction->GetDexPc());
+ if (it == offline_profile.inline_caches.end()) {
+ return kInlineCacheUninitialized;
+ }
+
+ const ProfileCompilationInfo::DexPcData& dex_pc_data = it->second;
+
+ if (dex_pc_data.is_missing_types) {
+ return kInlineCacheMissingTypes;
+ }
+ if (dex_pc_data.is_megamorphic) {
+ return kInlineCacheMegamorphic;
+ }
+
+ DCHECK_LE(dex_pc_data.classes.size(), InlineCache::kIndividualCacheSize);
+ Thread* self = Thread::Current();
+ // We need to resolve the class relative to the containing dex file.
+ // So first, build a mapping from the index of dex file in the profile to
+ // its dex cache. This will avoid repeating the lookup when walking over
+ // the inline cache types.
+ std::vector<ObjPtr<mirror::DexCache>> dex_profile_index_to_dex_cache(
+ offline_profile.dex_references.size());
+ for (size_t i = 0; i < offline_profile.dex_references.size(); i++) {
+ bool found = false;
+ for (const DexFile* dex_file : compiler_driver_->GetDexFilesForOatFile()) {
+ if (offline_profile.dex_references[i].MatchesDex(dex_file)) {
+ dex_profile_index_to_dex_cache[i] =
+ caller_compilation_unit_.GetClassLinker()->FindDexCache(self, *dex_file);
+ found = true;
+ }
+ }
+ if (!found) {
+ VLOG(compiler) << "Could not find profiled dex file: "
+ << offline_profile.dex_references[i].dex_location;
+ return kInlineCacheMissingTypes;
}
}
- VLOG(compiler) << "Interface or virtual call to "
- << caller_dex_file.PrettyMethod(method_index)
- << " could not be statically determined";
- return false;
+ // Walk over the classes and resolve them. If we cannot find a type we return
+ // kInlineCacheMissingTypes.
+ int ic_index = 0;
+ for (const ProfileCompilationInfo::ClassReference& class_ref : dex_pc_data.classes) {
+ ObjPtr<mirror::DexCache> dex_cache =
+ dex_profile_index_to_dex_cache[class_ref.dex_profile_index];
+ DCHECK(dex_cache != nullptr);
+ ObjPtr<mirror::Class> clazz = ClassLinker::LookupResolvedType(
+ class_ref.type_index,
+ dex_cache,
+ caller_compilation_unit_.GetClassLoader().Get());
+ if (clazz != nullptr) {
+ inline_cache->Set(ic_index++, clazz);
+ } else {
+ VLOG(compiler) << "Could not resolve class from inline cache in AOT mode "
+ << caller_compilation_unit_.GetDexFile()->PrettyMethod(
+ invoke_instruction->GetDexMethodIndex()) << " : "
+ << caller_compilation_unit_
+ .GetDexFile()->StringByTypeIdx(class_ref.type_index);
+ return kInlineCacheMissingTypes;
+ }
+ }
+ return GetInlineCacheType(inline_cache);
}
HInstanceFieldGet* HInliner::BuildGetReceiverClass(ClassLinker* class_linker,
@@ -556,6 +691,13 @@
// Insert before setting the kind, as setting the kind affects the inputs.
bb_cursor->InsertInstructionAfter(load_class, receiver_class);
load_class->SetLoadKind(kind);
+ // In AOT mode, we will most likely load the class from BSS, which will involve a call
+ // to the runtime. In this case, the load instruction will need an environment so copy
+ // it from the invoke instruction.
+ if (load_class->NeedsEnvironment()) {
+ DCHECK(Runtime::Current()->IsAotCompiler());
+ load_class->CopyEnvironmentFrom(invoke_instruction->GetEnvironment());
+ }
HNotEqual* compare = new (graph_->GetArena()) HNotEqual(load_class, receiver_class);
bb_cursor->InsertInstructionAfter(compare, load_class);
@@ -746,7 +888,10 @@
ArtMethod* resolved_method,
Handle<mirror::ObjectArray<mirror::Class>> classes) {
// This optimization only works under JIT for now.
- DCHECK(Runtime::Current()->UseJitCompilation());
+ if (!Runtime::Current()->UseJitCompilation()) {
+ return false;
+ }
+
if (graph_->GetInstructionSet() == kMips64) {
// TODO: Support HClassTableGet for mips64.
return false;
@@ -1380,6 +1525,13 @@
<< " could not be inlined because one branch always throws and"
<< " caller does not have an exit block";
return false;
+ } else if (graph_->HasIrreducibleLoops()) {
+ // TODO(ngeoffray): Support re-computing loop information to graphs with
+ // irreducible loops?
+ VLOG(compiler) << "Method " << callee_dex_file.PrettyMethod(method_index)
+ << " could not be inlined because one branch always throws and"
+ << " caller has irreducible loops";
+ return false;
}
} else {
has_one_return = true;
diff --git a/compiler/optimizing/inliner.h b/compiler/optimizing/inliner.h
index 75d025a..8f8b268 100644
--- a/compiler/optimizing/inliner.h
+++ b/compiler/optimizing/inliner.h
@@ -20,6 +20,7 @@
#include "dex_file_types.h"
#include "invoke_type.h"
#include "optimization.h"
+#include "jit/profile_compilation_info.h"
namespace art {
@@ -59,6 +60,15 @@
static constexpr const char* kInlinerPassName = "inliner";
private:
+ enum InlineCacheType {
+ kInlineCacheNoData = 0,
+ kInlineCacheUninitialized = 1,
+ kInlineCacheMonomorphic = 2,
+ kInlineCachePolymorphic = 3,
+ kInlineCacheMegamorphic = 4,
+ kInlineCacheMissingTypes = 5
+ };
+
bool TryInline(HInvoke* invoke_instruction);
// Try to inline `resolved_method` in place of `invoke_instruction`. `do_rtp` is whether
@@ -106,6 +116,45 @@
HInstruction* obj,
HInstruction* value);
+ // Try inlining the invoke instruction using inline caches.
+ bool TryInlineFromInlineCache(
+ const DexFile& caller_dex_file,
+ HInvoke* invoke_instruction,
+ ArtMethod* resolved_method)
+ REQUIRES_SHARED(Locks::mutator_lock_);
+
+ // Try getting the inline cache from JIT code cache.
+ // Return true if the inline cache was successfully allocated and the
+ // invoke info was found in the profile info.
+ InlineCacheType GetInlineCacheJIT(
+ HInvoke* invoke_instruction,
+ StackHandleScope<1>* hs,
+ /*out*/Handle<mirror::ObjectArray<mirror::Class>>* inline_cache)
+ REQUIRES_SHARED(Locks::mutator_lock_);
+
+ // Try getting the inline cache from AOT offline profile.
+ // Return true if the inline cache was successfully allocated and the
+ // invoke info was found in the profile info.
+ InlineCacheType GetInlineCacheAOT(const DexFile& caller_dex_file,
+ HInvoke* invoke_instruction,
+ StackHandleScope<1>* hs,
+ /*out*/Handle<mirror::ObjectArray<mirror::Class>>* inline_cache)
+ REQUIRES_SHARED(Locks::mutator_lock_);
+
+ // Extract the mirror classes from the offline profile and add them to the `inline_cache`.
+ // Note that even if we have profile data for the invoke the inline_cache might contain
+ // only null entries if the types cannot be resolved.
+ InlineCacheType ExtractClassesFromOfflineProfile(
+ const HInvoke* invoke_instruction,
+ const ProfileCompilationInfo::OfflineProfileMethodInfo& offline_profile,
+ /*out*/Handle<mirror::ObjectArray<mirror::Class>> inline_cache)
+ REQUIRES_SHARED(Locks::mutator_lock_);
+
+ // Compute the inline cache type.
+ InlineCacheType GetInlineCacheType(
+ const Handle<mirror::ObjectArray<mirror::Class>>& classes)
+ REQUIRES_SHARED(Locks::mutator_lock_);
+
// Try to inline the target of a monomorphic call. If successful, the code
// in the graph will look like:
// if (receiver.getClass() != ic.GetMonomorphicType()) deopt
diff --git a/compiler/optimizing/intrinsics_arm.cc b/compiler/optimizing/intrinsics_arm.cc
index 86000e9..28095c4 100644
--- a/compiler/optimizing/intrinsics_arm.cc
+++ b/compiler/optimizing/intrinsics_arm.cc
@@ -1947,6 +1947,8 @@
}
if (kEmitCompilerReadBarrier && kUseBakerReadBarrier) {
+ // TODO: Also convert this intrinsic to the IsGcMarking strategy?
+
// The base destination address is computed later, as `temp2` is
// used for intermediate computations.
diff --git a/compiler/optimizing/intrinsics_arm64.cc b/compiler/optimizing/intrinsics_arm64.cc
index 6c3938c..934ba1b 100644
--- a/compiler/optimizing/intrinsics_arm64.cc
+++ b/compiler/optimizing/intrinsics_arm64.cc
@@ -853,7 +853,6 @@
DCHECK((type == Primitive::kPrimInt) ||
(type == Primitive::kPrimLong) ||
(type == Primitive::kPrimNot));
- MacroAssembler* masm = codegen->GetVIXLAssembler();
Location base_loc = locations->InAt(1);
Register base = WRegisterFrom(base_loc); // Object pointer.
Location offset_loc = locations->InAt(2);
@@ -863,8 +862,7 @@
if (type == Primitive::kPrimNot && kEmitCompilerReadBarrier && kUseBakerReadBarrier) {
// UnsafeGetObject/UnsafeGetObjectVolatile with Baker's read barrier case.
- UseScratchRegisterScope temps(masm);
- Register temp = temps.AcquireW();
+ Register temp = WRegisterFrom(locations->GetTemp(0));
codegen->GenerateReferenceLoadWithBakerReadBarrier(invoke,
trg_loc,
base,
@@ -901,6 +899,9 @@
kIntrinsified);
if (can_call && kUseBakerReadBarrier) {
locations->SetCustomSlowPathCallerSaves(RegisterSet::Empty()); // No caller-save registers.
+ // We need a temporary register for the read barrier marking slow
+ // path in CodeGeneratorARM64::GenerateReferenceLoadWithBakerReadBarrier.
+ locations->AddTemp(Location::RequiresRegister());
}
locations->SetInAt(0, Location::NoLocation()); // Unused receiver.
locations->SetInAt(1, Location::RequiresRegister());
@@ -2381,9 +2382,14 @@
// Temporary register IP0, obtained from the VIXL scratch register
// pool, cannot be used in ReadBarrierSystemArrayCopySlowPathARM64
// (because that register is clobbered by ReadBarrierMarkRegX
- // entry points). Get an extra temporary register from the
- // register allocator.
+ // entry points). It cannot be used in calls to
+ // CodeGeneratorARM64::GenerateFieldLoadWithBakerReadBarrier
+ // either. For these reasons, get a third extra temporary register
+ // from the register allocator.
locations->AddTemp(Location::RequiresRegister());
+ } else {
+ // Cases other than Baker read barriers: the third temporary will
+ // be acquired from the VIXL scratch register pool.
}
}
@@ -2494,11 +2500,12 @@
// We use a block to end the scratch scope before the write barrier, thus
// freeing the temporary registers so they can be used in `MarkGCCard`.
UseScratchRegisterScope temps(masm);
- // Note: Because it is acquired from VIXL's scratch register pool,
- // `temp3` might be IP0, and thus cannot be used as `ref` argument
- // of CodeGeneratorARM64::GenerateFieldLoadWithBakerReadBarrier
- // calls below (see ReadBarrierMarkSlowPathARM64 for more details).
- Register temp3 = temps.AcquireW();
+ Register temp3;
+ if (kEmitCompilerReadBarrier && kUseBakerReadBarrier) {
+ temp3 = WRegisterFrom(locations->GetTemp(2));
+ } else {
+ temp3 = temps.AcquireW();
+ }
if (!optimizations.GetDoesNotNeedTypeCheck()) {
// Check whether all elements of the source array are assignable to the component
@@ -2704,19 +2711,7 @@
Register src_curr_addr = temp1.X();
Register dst_curr_addr = temp2.X();
- Register src_stop_addr;
- if (kEmitCompilerReadBarrier && kUseBakerReadBarrier) {
- // Temporary register IP0, obtained from the VIXL scratch
- // register pool as `temp3`, cannot be used in
- // ReadBarrierSystemArrayCopySlowPathARM64 (because that
- // register is clobbered by ReadBarrierMarkRegX entry points).
- // So another temporary register allocated by the register
- // allocator instead.
- DCHECK_EQ(LocationFrom(temp3).reg(), IP0);
- src_stop_addr = XRegisterFrom(locations->GetTemp(2));
- } else {
- src_stop_addr = temp3.X();
- }
+ Register src_stop_addr = temp3.X();
GenSystemArrayCopyAddresses(masm,
Primitive::kPrimNot,
@@ -2732,6 +2727,8 @@
const int32_t element_size = Primitive::ComponentSize(Primitive::kPrimNot);
if (kEmitCompilerReadBarrier && kUseBakerReadBarrier) {
+ // TODO: Also convert this intrinsic to the IsGcMarking strategy?
+
// SystemArrayCopy implementation for Baker read barriers (see
// also CodeGeneratorARM::GenerateReferenceLoadWithBakerReadBarrier):
//
@@ -2758,10 +2755,11 @@
__ Cmp(src_curr_addr, src_stop_addr);
__ B(&done, eq);
- Register tmp = temps.AcquireW();
// Make sure `tmp` is not IP0, as it is clobbered by
// ReadBarrierMarkRegX entry points in
// ReadBarrierSystemArrayCopySlowPathARM64.
+ temps.Exclude(ip0);
+ Register tmp = temps.AcquireW();
DCHECK_NE(LocationFrom(tmp).reg(), IP0);
// /* int32_t */ monitor = src->monitor_
diff --git a/compiler/optimizing/intrinsics_arm_vixl.cc b/compiler/optimizing/intrinsics_arm_vixl.cc
index aa89dea..60bcf2c 100644
--- a/compiler/optimizing/intrinsics_arm_vixl.cc
+++ b/compiler/optimizing/intrinsics_arm_vixl.cc
@@ -2265,6 +2265,8 @@
}
if (kEmitCompilerReadBarrier && kUseBakerReadBarrier) {
+ // TODO: Also convert this intrinsic to the IsGcMarking strategy?
+
// The base destination address is computed later, as `temp2` is
// used for intermediate computations.
diff --git a/compiler/optimizing/load_store_elimination.cc b/compiler/optimizing/load_store_elimination.cc
index 46ba048..48699b3 100644
--- a/compiler/optimizing/load_store_elimination.cc
+++ b/compiler/optimizing/load_store_elimination.cc
@@ -69,6 +69,13 @@
return is_singleton_and_not_returned_ && is_singleton_and_not_deopt_visible_;
}
+ // Returns true if reference_ is a singleton and returned to the caller or
+ // used as an environment local of an HDeoptimize instruction.
+ bool IsSingletonAndNonRemovable() const {
+ return is_singleton_ &&
+ (!is_singleton_and_not_returned_ || !is_singleton_and_not_deopt_visible_);
+ }
+
bool HasIndexAliasing() {
return has_index_aliasing_;
}
@@ -339,11 +346,7 @@
return false;
}
ReferenceInfo* ref_info = loc1->GetReferenceInfo();
- if (ref_info->IsSingleton()) {
- // This is guaranteed by the CanReferencesAlias() test above.
- DCHECK_EQ(ref_info, loc2->GetReferenceInfo());
- ref_info->SetHasIndexAliasing(true);
- }
+ ref_info->SetHasIndexAliasing(true);
}
return true;
}
@@ -628,14 +631,17 @@
for (size_t i = 0; i < heap_values.size(); i++) {
HeapLocation* location = heap_location_collector_.GetHeapLocation(i);
ReferenceInfo* ref_info = location->GetReferenceInfo();
- if (!ref_info->IsSingleton() || location->IsValueKilledByLoopSideEffects()) {
- // heap value is killed by loop side effects (stored into directly, or due to
- // aliasing).
+ if (ref_info->IsSingletonAndRemovable() &&
+ !location->IsValueKilledByLoopSideEffects()) {
+ // A removable singleton's field that's not stored into inside a loop is
+ // invariant throughout the loop. Nothing to do.
+ DCHECK(ref_info->IsSingletonAndRemovable());
+ } else {
+ // heap value is killed by loop side effects (stored into directly, or
+ // due to aliasing). Or the heap value may be needed after method return
+ // or deoptimization.
KeepIfIsStore(pre_header_heap_values[i]);
heap_values[i] = kUnknownHeapValue;
- } else {
- // A singleton's field that's not stored into inside a loop is invariant throughout
- // the loop.
}
}
}
@@ -654,7 +660,7 @@
bool from_all_predecessors = true;
ReferenceInfo* ref_info = heap_location_collector_.GetHeapLocation(i)->GetReferenceInfo();
HInstruction* singleton_ref = nullptr;
- if (ref_info->IsSingletonAndRemovable()) {
+ if (ref_info->IsSingleton()) {
// We do more analysis of liveness when merging heap values for such
// cases since stores into such references may potentially be eliminated.
singleton_ref = ref_info->GetReference();
@@ -680,8 +686,9 @@
}
}
- if (merged_value == kUnknownHeapValue) {
- // There are conflicting heap values from different predecessors.
+ if (merged_value == kUnknownHeapValue || ref_info->IsSingletonAndNonRemovable()) {
+ // There are conflicting heap values from different predecessors,
+ // or the heap value may be needed after method return or deoptimization.
// Keep the last store in each predecessor since future loads cannot be eliminated.
for (HBasicBlock* predecessor : predecessors) {
ArenaVector<HInstruction*>& pred_values = heap_values_for_[predecessor->GetBlockId()];
@@ -828,15 +835,15 @@
// Store into the heap location with the same value.
same_value = true;
} else if (index != nullptr && ref_info->HasIndexAliasing()) {
- // For array element, don't eliminate stores if the index can be
- // aliased.
- } else if (ref_info->IsSingletonAndRemovable()) {
- // Store into a field/element of a singleton instance/array that's not returned.
- // The value cannot be killed due to aliasing/invocation. It can be redundant since
- // future loads can directly get the value set by this instruction. The value can
- // still be killed due to merging or loop side effects. Stores whose values are
- // killed due to merging/loop side effects later will be removed from
- // possibly_removed_stores_ when that is detected.
+ // For array element, don't eliminate stores if the index can be aliased.
+ } else if (ref_info->IsSingleton()) {
+ // Store into a field of a singleton. The value cannot be killed due to
+ // aliasing/invocation. It can be redundant since future loads can
+ // directly get the value set by this instruction. The value can still be killed due to
+ // merging or loop side effects. Stores whose values are killed due to merging/loop side
+ // effects later will be removed from possibly_removed_stores_ when that is detected.
+ // Stores whose values may be needed after method return or deoptimization
+ // are also removed from possibly_removed_stores_ when that is detected.
possibly_redundant = true;
HNewInstance* new_instance = ref_info->GetReference()->AsNewInstance();
if (new_instance != nullptr && new_instance->IsFinalizable()) {
@@ -945,6 +952,33 @@
value);
}
+ void VisitDeoptimize(HDeoptimize* instruction) {
+ const ArenaVector<HInstruction*>& heap_values =
+ heap_values_for_[instruction->GetBlock()->GetBlockId()];
+ for (HInstruction* heap_value : heap_values) {
+ // Filter out fake instructions before checking instruction kind below.
+ if (heap_value == kUnknownHeapValue || heap_value == kDefaultHeapValue) {
+ continue;
+ }
+ // A store is kept as the heap value for possibly removed stores.
+ if (heap_value->IsInstanceFieldSet() || heap_value->IsArraySet()) {
+ // Check whether the reference for a store is used by an environment local of
+ // HDeoptimize.
+ HInstruction* reference = heap_value->InputAt(0);
+ DCHECK(heap_location_collector_.FindReferenceInfoOf(reference)->IsSingleton());
+ for (const HUseListNode<HEnvironment*>& use : reference->GetEnvUses()) {
+ HEnvironment* user = use.GetUser();
+ if (user->GetHolder() == instruction) {
+ // The singleton for the store is visible at this deoptimization
+ // point. Need to keep the store so that the heap value is
+ // seen by the interpreter.
+ KeepIfIsStore(heap_value);
+ }
+ }
+ }
+ }
+ }
+
void HandleInvoke(HInstruction* invoke) {
ArenaVector<HInstruction*>& heap_values =
heap_values_for_[invoke->GetBlock()->GetBlockId()];
diff --git a/compiler/optimizing/nodes.cc b/compiler/optimizing/nodes.cc
index 62c8910..020e446 100644
--- a/compiler/optimizing/nodes.cc
+++ b/compiler/optimizing/nodes.cc
@@ -2179,6 +2179,9 @@
}
}
if (rerun_loop_analysis) {
+ DCHECK(!outer_graph->HasIrreducibleLoops())
+ << "Recomputing loop information in graphs with irreducible loops "
+ << "is unsupported, as it could lead to loop header changes";
outer_graph->ClearLoopInformation();
outer_graph->ClearDominanceInformation();
outer_graph->BuildDominatorTree();
diff --git a/compiler/optimizing/nodes.h b/compiler/optimizing/nodes.h
index f912d7d..542b218 100644
--- a/compiler/optimizing/nodes.h
+++ b/compiler/optimizing/nodes.h
@@ -5545,8 +5545,6 @@
// Use a known boot image Class* address, embedded in the code by the codegen.
// Used for boot image classes referenced by apps in AOT- and JIT-compiled code.
- // Note: codegen needs to emit a linker patch if indicated by compiler options'
- // GetIncludePatchInformation().
kBootImageAddress,
// Load from an entry in the .bss section using a PC-relative load.
@@ -5750,8 +5748,6 @@
// Use a known boot image String* address, embedded in the code by the codegen.
// Used for boot image strings referenced by apps in AOT- and JIT-compiled code.
- // Note: codegen needs to emit a linker patch if indicated by compiler options'
- // GetIncludePatchInformation().
kBootImageAddress,
// Load from an entry in the .bss section using a PC-relative load.
diff --git a/compiler/optimizing/register_allocator_linear_scan.cc b/compiler/optimizing/register_allocator_linear_scan.cc
index 1a391ce..6354e76 100644
--- a/compiler/optimizing/register_allocator_linear_scan.cc
+++ b/compiler/optimizing/register_allocator_linear_scan.cc
@@ -629,21 +629,21 @@
if (!locations->OutputCanOverlapWithInputs() && locations->Out().IsUnallocated()) {
HInputsRef inputs = defined_by->GetInputs();
for (size_t i = 0; i < inputs.size(); ++i) {
- // Take the last interval of the input. It is the location of that interval
- // that will be used at `defined_by`.
- LiveInterval* interval = inputs[i]->GetLiveInterval()->GetLastSibling();
- // Note that interval may have not been processed yet.
- // TODO: Handle non-split intervals last in the work list.
- if (locations->InAt(i).IsValid()
- && interval->HasRegister()
- && interval->SameRegisterKind(*current)) {
- // The input must be live until the end of `defined_by`, to comply to
- // the linear scan algorithm. So we use `defined_by`'s end lifetime
- // position to check whether the input is dead or is inactive after
- // `defined_by`.
- DCHECK(interval->CoversSlow(defined_by->GetLifetimePosition()));
- size_t position = defined_by->GetLifetimePosition() + 1;
- FreeIfNotCoverAt(interval, position, free_until);
+ if (locations->InAt(i).IsValid()) {
+ // Take the last interval of the input. It is the location of that interval
+ // that will be used at `defined_by`.
+ LiveInterval* interval = inputs[i]->GetLiveInterval()->GetLastSibling();
+ // Note that interval may have not been processed yet.
+ // TODO: Handle non-split intervals last in the work list.
+ if (interval->HasRegister() && interval->SameRegisterKind(*current)) {
+ // The input must be live until the end of `defined_by`, to comply to
+ // the linear scan algorithm. So we use `defined_by`'s end lifetime
+ // position to check whether the input is dead or is inactive after
+ // `defined_by`.
+ DCHECK(interval->CoversSlow(defined_by->GetLifetimePosition()));
+ size_t position = defined_by->GetLifetimePosition() + 1;
+ FreeIfNotCoverAt(interval, position, free_until);
+ }
}
}
}
diff --git a/compiler/optimizing/sharpening.cc b/compiler/optimizing/sharpening.cc
index 8f1827b..7bd38c7 100644
--- a/compiler/optimizing/sharpening.cc
+++ b/compiler/optimizing/sharpening.cc
@@ -65,9 +65,7 @@
}
static bool AOTCanEmbedMethod(ArtMethod* method, const CompilerOptions& options) {
- // Including patch information means the AOT code will be patched, which we don't
- // support in the compiler, and is anyways moving away b/33192586.
- return IsInBootImage(method) && !options.GetCompilePic() && !options.GetIncludePatchInformation();
+ return IsInBootImage(method) && !options.GetCompilePic();
}
diff --git a/dex2oat/dex2oat.cc b/dex2oat/dex2oat.cc
index f535557..3fa30fa 100644
--- a/dex2oat/dex2oat.cc
+++ b/dex2oat/dex2oat.cc
@@ -328,11 +328,6 @@
UsageError("");
UsageError(" --dump-timing: display a breakdown of where time was spent");
UsageError("");
- UsageError(" --include-patch-information: Include patching information so the generated code");
- UsageError(" can have its base address moved without full recompilation.");
- UsageError("");
- UsageError(" --no-include-patch-information: Do not include patching information.");
- UsageError("");
UsageError(" -g");
UsageError(" --generate-debug-info: Generate debug information for native debugging,");
UsageError(" such as stack unwinding information, ELF symbols and DWARF sections.");
@@ -1958,7 +1953,6 @@
elf_writer->WriteDynamicSection();
elf_writer->WriteDebugInfo(oat_writer->GetMethodDebugInfo());
- elf_writer->WritePatchLocations(oat_writer->GetAbsolutePatchLocations());
if (!elf_writer->End()) {
LOG(ERROR) << "Failed to write ELF file " << oat_file->GetPath();
@@ -2106,6 +2100,10 @@
return DoProfileGuidedOptimizations();
}
+ bool HasInputVdexFile() const {
+ return input_vdex_file_ != nullptr || input_vdex_fd_ != -1 || !input_vdex_.empty();
+ }
+
bool LoadProfile() {
DCHECK(UseProfile());
@@ -2891,6 +2889,13 @@
}
}
+ if (dex2oat->DoDexLayoutOptimizations()) {
+ if (dex2oat->HasInputVdexFile()) {
+ LOG(ERROR) << "Dexlayout is incompatible with an input VDEX";
+ return EXIT_FAILURE;
+ }
+ }
+
art::MemMap::Init(); // For ZipEntry::ExtractToMemMap, and vdex.
// Check early that the result of compilation can be written
diff --git a/dex2oat/dex2oat_test.cc b/dex2oat/dex2oat_test.cc
index e7277bc..289b8ab 100644
--- a/dex2oat/dex2oat_test.cc
+++ b/dex2oat/dex2oat_test.cc
@@ -604,7 +604,8 @@
const std::string& app_image_file_name,
bool use_fd,
size_t num_profile_classes,
- const std::vector<std::string>& extra_args = {}) {
+ const std::vector<std::string>& extra_args = {},
+ bool expect_success = true) {
const std::string profile_location = GetScratchDir() + "/primary.prof";
const char* location = dex_location.c_str();
std::string error_msg;
@@ -631,7 +632,7 @@
odex_location,
CompilerFilter::kSpeedProfile,
copy,
- /* expect_success */ true,
+ expect_success,
use_fd);
if (app_image_file != nullptr) {
ASSERT_EQ(app_image_file->FlushCloseOrErase(), 0) << "Could not flush and close art file";
@@ -709,6 +710,7 @@
EXPECT_GT(vdex_file1->GetLength(), 0u);
}
{
+ // Test that vdex and dexlayout fail gracefully.
std::string input_vdex = StringPrintf("--input-vdex-fd=%d", vdex_file1->Fd());
std::string output_vdex = StringPrintf("--output-vdex-fd=%d", vdex_file2.GetFd());
CompileProfileOdex(dex_location,
@@ -716,13 +718,13 @@
app_image_file_name,
/* use_fd */ true,
/* num_profile_classes */ 1,
- { input_vdex, output_vdex });
- EXPECT_GT(vdex_file2.GetFile()->GetLength(), 0u);
+ { input_vdex, output_vdex },
+ /* expect_success */ false);
+ EXPECT_EQ(vdex_file2.GetFile()->GetLength(), 0u);
}
ASSERT_EQ(vdex_file1->FlushCloseOrErase(), 0) << "Could not flush and close vdex file";
CheckValidity();
- ASSERT_TRUE(success_);
- CheckResult(dex_location, odex_location, app_image_file_name);
+ ASSERT_FALSE(success_);
}
void CheckResult(const std::string& dex_location,
diff --git a/runtime/arch/arm/quick_entrypoints_arm.S b/runtime/arch/arm/quick_entrypoints_arm.S
index 8531091..72aa785 100644
--- a/runtime/arch/arm/quick_entrypoints_arm.S
+++ b/runtime/arch/arm/quick_entrypoints_arm.S
@@ -1487,6 +1487,7 @@
.Lconflict_trampoline:
// Call the runtime stub to populate the ImtConflictTable and jump to the
// resolved method.
+ mov r0, r12 // Load interface method
INVOKE_TRAMPOLINE_BODY artInvokeInterfaceTrampoline
END art_quick_imt_conflict_trampoline
diff --git a/runtime/arch/arm64/quick_entrypoints_arm64.S b/runtime/arch/arm64/quick_entrypoints_arm64.S
index 7cb50b7..5b5d2ef 100644
--- a/runtime/arch/arm64/quick_entrypoints_arm64.S
+++ b/runtime/arch/arm64/quick_entrypoints_arm64.S
@@ -1966,6 +1966,7 @@
.Lconflict_trampoline:
// Call the runtime stub to populate the ImtConflictTable and jump to the
// resolved method.
+ mov x0, xIP0 // Load interface method
INVOKE_TRAMPOLINE_BODY artInvokeInterfaceTrampoline
END art_quick_imt_conflict_trampoline
diff --git a/runtime/arch/mips/quick_entrypoints_mips.S b/runtime/arch/mips/quick_entrypoints_mips.S
index 4f7b495..5d61539 100644
--- a/runtime/arch/mips/quick_entrypoints_mips.S
+++ b/runtime/arch/mips/quick_entrypoints_mips.S
@@ -1763,6 +1763,7 @@
.Lconflict_trampoline:
# Call the runtime stub to populate the ImtConflictTable and jump to the resolved method.
+ move $a0, $t7 # Load interface method.
INVOKE_TRAMPOLINE_BODY artInvokeInterfaceTrampoline
END art_quick_imt_conflict_trampoline
diff --git a/runtime/arch/mips64/quick_entrypoints_mips64.S b/runtime/arch/mips64/quick_entrypoints_mips64.S
index 28d7c77..3ee9c4a 100644
--- a/runtime/arch/mips64/quick_entrypoints_mips64.S
+++ b/runtime/arch/mips64/quick_entrypoints_mips64.S
@@ -1715,6 +1715,7 @@
.Lconflict_trampoline:
# Call the runtime stub to populate the ImtConflictTable and jump to the resolved method.
+ move $a0, $t0 # Load interface method.
INVOKE_TRAMPOLINE_BODY artInvokeInterfaceTrampoline
END art_quick_imt_conflict_trampoline
diff --git a/runtime/arch/x86/quick_entrypoints_x86.S b/runtime/arch/x86/quick_entrypoints_x86.S
index 791ec39..5f38dc8 100644
--- a/runtime/arch/x86/quick_entrypoints_x86.S
+++ b/runtime/arch/x86/quick_entrypoints_x86.S
@@ -1806,6 +1806,7 @@
.Lconflict_trampoline:
// Call the runtime stub to populate the ImtConflictTable and jump to the
// resolved method.
+ movl %edi, %eax // Load interface method
POP EDI
INVOKE_TRAMPOLINE_BODY artInvokeInterfaceTrampoline
END_FUNCTION art_quick_imt_conflict_trampoline
diff --git a/runtime/arch/x86_64/quick_entrypoints_x86_64.S b/runtime/arch/x86_64/quick_entrypoints_x86_64.S
index 822b411..e87b165 100644
--- a/runtime/arch/x86_64/quick_entrypoints_x86_64.S
+++ b/runtime/arch/x86_64/quick_entrypoints_x86_64.S
@@ -1662,6 +1662,7 @@
.Lconflict_trampoline:
// Call the runtime stub to populate the ImtConflictTable and jump to the
// resolved method.
+ movq %r10, %rdi // Load interface method
INVOKE_TRAMPOLINE_BODY artInvokeInterfaceTrampoline
#endif // __APPLE__
END_FUNCTION art_quick_imt_conflict_trampoline
diff --git a/runtime/entrypoints/quick/quick_trampoline_entrypoints.cc b/runtime/entrypoints/quick/quick_trampoline_entrypoints.cc
index 4c3990a..3fd20a6 100644
--- a/runtime/entrypoints/quick/quick_trampoline_entrypoints.cc
+++ b/runtime/entrypoints/quick/quick_trampoline_entrypoints.cc
@@ -2323,48 +2323,26 @@
return artInvokeCommon<kVirtual, true>(method_idx, this_object, self, sp);
}
-// Determine target of interface dispatch. This object is known non-null. First argument
-// is there for consistency but should not be used, as some architectures overwrite it
-// in the assembly trampoline.
-extern "C" TwoWordReturn artInvokeInterfaceTrampoline(uint32_t deadbeef ATTRIBUTE_UNUSED,
+// Determine target of interface dispatch. The interface method and this object are known non-null.
+// The interface method is the method returned by the dex cache in the conflict trampoline.
+extern "C" TwoWordReturn artInvokeInterfaceTrampoline(ArtMethod* interface_method,
mirror::Object* raw_this_object,
Thread* self,
ArtMethod** sp)
REQUIRES_SHARED(Locks::mutator_lock_) {
+ CHECK(interface_method != nullptr);
ObjPtr<mirror::Object> this_object(raw_this_object);
ScopedQuickEntrypointChecks sqec(self);
StackHandleScope<1> hs(self);
Handle<mirror::Class> cls(hs.NewHandle(this_object->GetClass()));
ArtMethod* caller_method = QuickArgumentVisitor::GetCallingMethod(sp);
-
- // Fetch the dex_method_idx of the target interface method from the caller.
- uint32_t dex_pc = QuickArgumentVisitor::GetCallingDexPc(sp);
-
- const DexFile::CodeItem* code_item = caller_method->GetCodeItem();
- CHECK_LT(dex_pc, code_item->insns_size_in_code_units_);
- const Instruction* instr = Instruction::At(&code_item->insns_[dex_pc]);
- Instruction::Code instr_code = instr->Opcode();
- CHECK(instr_code == Instruction::INVOKE_INTERFACE ||
- instr_code == Instruction::INVOKE_INTERFACE_RANGE)
- << "Unexpected call into interface trampoline: " << instr->DumpString(nullptr);
- uint32_t dex_method_idx;
- if (instr_code == Instruction::INVOKE_INTERFACE) {
- dex_method_idx = instr->VRegB_35c();
- } else {
- CHECK_EQ(instr_code, Instruction::INVOKE_INTERFACE_RANGE);
- dex_method_idx = instr->VRegB_3rc();
- }
-
- ArtMethod* interface_method = caller_method->GetDexCacheResolvedMethod(
- dex_method_idx, kRuntimePointerSize);
- DCHECK(interface_method != nullptr) << dex_method_idx << " " << caller_method->PrettyMethod();
ArtMethod* method = nullptr;
ImTable* imt = cls->GetImt(kRuntimePointerSize);
if (LIKELY(interface_method->GetDexMethodIndex() != DexFile::kDexNoIndex)) {
- // If the dex cache already resolved the interface method, look whether we have
- // a match in the ImtConflictTable.
+ // If the interface method is already resolved, look whether we have a match in the
+ // ImtConflictTable.
ArtMethod* conflict_method = imt->Get(ImTable::GetImtIndex(interface_method),
kRuntimePointerSize);
if (LIKELY(conflict_method->IsRuntimeMethod())) {
@@ -2389,9 +2367,26 @@
return GetTwoWordFailureValue(); // Failure.
}
} else {
- // The dex cache did not resolve the method, look it up in the dex file
- // of the caller,
+ // The interface method is unresolved, so look it up in the dex file of the caller.
DCHECK_EQ(interface_method, Runtime::Current()->GetResolutionMethod());
+
+ // Fetch the dex_method_idx of the target interface method from the caller.
+ uint32_t dex_method_idx;
+ uint32_t dex_pc = QuickArgumentVisitor::GetCallingDexPc(sp);
+ const DexFile::CodeItem* code_item = caller_method->GetCodeItem();
+ DCHECK_LT(dex_pc, code_item->insns_size_in_code_units_);
+ const Instruction* instr = Instruction::At(&code_item->insns_[dex_pc]);
+ Instruction::Code instr_code = instr->Opcode();
+ DCHECK(instr_code == Instruction::INVOKE_INTERFACE ||
+ instr_code == Instruction::INVOKE_INTERFACE_RANGE)
+ << "Unexpected call into interface trampoline: " << instr->DumpString(nullptr);
+ if (instr_code == Instruction::INVOKE_INTERFACE) {
+ dex_method_idx = instr->VRegB_35c();
+ } else {
+ DCHECK_EQ(instr_code, Instruction::INVOKE_INTERFACE_RANGE);
+ dex_method_idx = instr->VRegB_3rc();
+ }
+
const DexFile* dex_file = caller_method->GetDeclaringClass()->GetDexCache()
->GetDexFile();
uint32_t shorty_len;
diff --git a/runtime/gc/scoped_gc_critical_section.cc b/runtime/gc/scoped_gc_critical_section.cc
index b5eb979..f937d2c 100644
--- a/runtime/gc/scoped_gc_critical_section.cc
+++ b/runtime/gc/scoped_gc_critical_section.cc
@@ -29,10 +29,14 @@
CollectorType collector_type)
: self_(self) {
Runtime::Current()->GetHeap()->StartGC(self, cause, collector_type);
- old_cause_ = self->StartAssertNoThreadSuspension("ScopedGCCriticalSection");
+ if (self != nullptr) {
+ old_cause_ = self->StartAssertNoThreadSuspension("ScopedGCCriticalSection");
+ }
}
ScopedGCCriticalSection::~ScopedGCCriticalSection() {
- self_->EndAssertNoThreadSuspension(old_cause_);
+ if (self_ != nullptr) {
+ self_->EndAssertNoThreadSuspension(old_cause_);
+ }
Runtime::Current()->GetHeap()->FinishGC(self_, collector::kGcTypeNone);
}
diff --git a/runtime/interpreter/unstarted_runtime.cc b/runtime/interpreter/unstarted_runtime.cc
index 80554c2..eb0a9d1 100644
--- a/runtime/interpreter/unstarted_runtime.cc
+++ b/runtime/interpreter/unstarted_runtime.cc
@@ -1672,6 +1672,12 @@
}
}
+void UnstartedRuntime::UnstartedSystemIdentityHashCode(
+ Thread* self ATTRIBUTE_UNUSED, ShadowFrame* shadow_frame, JValue* result, size_t arg_offset)
+ REQUIRES_SHARED(Locks::mutator_lock_) {
+ mirror::Object* obj = shadow_frame->GetVRegReference(arg_offset);
+ result->SetI((obj != nullptr) ? obj->IdentityHashCode() : 0);
+}
void UnstartedRuntime::UnstartedJNIVMRuntimeNewUnpaddedArray(
Thread* self, ArtMethod* method ATTRIBUTE_UNUSED, mirror::Object* receiver ATTRIBUTE_UNUSED,
@@ -1836,13 +1842,6 @@
}
}
-void UnstartedRuntime::UnstartedJNISystemIdentityHashCode(
- Thread* self ATTRIBUTE_UNUSED, ArtMethod* method ATTRIBUTE_UNUSED,
- mirror::Object* receiver ATTRIBUTE_UNUSED, uint32_t* args, JValue* result) {
- mirror::Object* obj = reinterpret_cast<mirror::Object*>(args[0]);
- result->SetI((obj != nullptr) ? obj->IdentityHashCode() : 0);
-}
-
void UnstartedRuntime::UnstartedJNIByteOrderIsLittleEndian(
Thread* self ATTRIBUTE_UNUSED, ArtMethod* method ATTRIBUTE_UNUSED,
mirror::Object* receiver ATTRIBUTE_UNUSED, uint32_t* args ATTRIBUTE_UNUSED, JValue* result) {
diff --git a/runtime/interpreter/unstarted_runtime_list.h b/runtime/interpreter/unstarted_runtime_list.h
index e9435e4..2560a92 100644
--- a/runtime/interpreter/unstarted_runtime_list.h
+++ b/runtime/interpreter/unstarted_runtime_list.h
@@ -76,7 +76,8 @@
V(UnsafePutObjectVolatile, "void sun.misc.Unsafe.putObjectVolatile(java.lang.Object, long, java.lang.Object)") \
V(UnsafePutOrderedObject, "void sun.misc.Unsafe.putOrderedObject(java.lang.Object, long, java.lang.Object)") \
V(IntegerParseInt, "int java.lang.Integer.parseInt(java.lang.String)") \
- V(LongParseLong, "long java.lang.Long.parseLong(java.lang.String)")
+ V(LongParseLong, "long java.lang.Long.parseLong(java.lang.String)") \
+ V(SystemIdentityHashCode, "int java.lang.System.identityHashCode(java.lang.Object)")
// Methods that are native.
#define UNSTARTED_RUNTIME_JNI_LIST(V) \
@@ -98,7 +99,6 @@
V(ArrayCreateMultiArray, "java.lang.Object java.lang.reflect.Array.createMultiArray(java.lang.Class, int[])") \
V(ArrayCreateObjectArray, "java.lang.Object java.lang.reflect.Array.createObjectArray(java.lang.Class, int)") \
V(ThrowableNativeFillInStackTrace, "java.lang.Object java.lang.Throwable.nativeFillInStackTrace()") \
- V(SystemIdentityHashCode, "int java.lang.System.identityHashCode(java.lang.Object)") \
V(ByteOrderIsLittleEndian, "boolean java.nio.ByteOrder.isLittleEndian()") \
V(UnsafeCompareAndSwapInt, "boolean sun.misc.Unsafe.compareAndSwapInt(java.lang.Object, long, int, int)") \
V(UnsafeGetIntVolatile, "int sun.misc.Unsafe.getIntVolatile(java.lang.Object, long)") \
diff --git a/runtime/interpreter/unstarted_runtime_test.cc b/runtime/interpreter/unstarted_runtime_test.cc
index db222fa..56e261c 100644
--- a/runtime/interpreter/unstarted_runtime_test.cc
+++ b/runtime/interpreter/unstarted_runtime_test.cc
@@ -1367,5 +1367,26 @@
ShadowFrame::DeleteDeoptimizedFrame(shadow_frame);
}
+TEST_F(UnstartedRuntimeTest, IdentityHashCode) {
+ Thread* self = Thread::Current();
+ ScopedObjectAccess soa(self);
+ ShadowFrame* tmp = ShadowFrame::CreateDeoptimizedFrame(10, nullptr, nullptr, 0);
+
+ JValue result;
+ UnstartedSystemIdentityHashCode(self, tmp, &result, 0);
+
+ EXPECT_EQ(0, result.GetI());
+ ASSERT_FALSE(self->IsExceptionPending());
+
+ ObjPtr<mirror::String> str = mirror::String::AllocFromModifiedUtf8(self, "abd");
+ tmp->SetVRegReference(0, str.Ptr());
+ UnstartedSystemIdentityHashCode(self, tmp, &result, 0);
+ EXPECT_NE(0, result.GetI());
+ EXPECT_EQ(str->IdentityHashCode(), result.GetI());
+ ASSERT_FALSE(self->IsExceptionPending());
+
+ ShadowFrame::DeleteDeoptimizedFrame(tmp);
+}
+
} // namespace interpreter
} // namespace art
diff --git a/runtime/jit/jit_code_cache.cc b/runtime/jit/jit_code_cache.cc
index 8b2a2b4..e7b23dc 100644
--- a/runtime/jit/jit_code_cache.cc
+++ b/runtime/jit/jit_code_cache.cc
@@ -1262,6 +1262,7 @@
for (size_t i = 0; i < info->number_of_inline_caches_; ++i) {
std::vector<ProfileMethodInfo::ProfileClassReference> profile_classes;
const InlineCache& cache = info->cache_[i];
+ ArtMethod* caller = info->GetMethod();
bool is_missing_types = false;
for (size_t k = 0; k < InlineCache::kIndividualCacheSize; k++) {
mirror::Class* cls = cache.classes_[k].Read();
@@ -1269,6 +1270,15 @@
break;
}
+ // Check if the receiver is in the boot class path or if it's in the
+ // same class loader as the caller. If not, skip it, as there is not
+ // much we can do during AOT.
+ if (!cls->IsBootStrapClassLoaded() &&
+ caller->GetClassLoader() != cls->GetClassLoader()) {
+ is_missing_types = true;
+ continue;
+ }
+
const DexFile* class_dex_file = nullptr;
dex::TypeIndex type_index;
diff --git a/runtime/lock_word.h b/runtime/lock_word.h
index 2f2565b..edc64f3 100644
--- a/runtime/lock_word.h
+++ b/runtime/lock_word.h
@@ -57,7 +57,8 @@
* |10|9|87654321098765432109876543210|
* |11|0| ForwardingAddress |
*
- * The rb bits store the read barrier state.
+ * The `r` bit stores the read barrier state.
+ * The `m` bit stores the mark state.
*/
class LockWord {
public:
diff --git a/runtime/mirror/class.h b/runtime/mirror/class.h
index b68eedc..c52b66a 100644
--- a/runtime/mirror/class.h
+++ b/runtime/mirror/class.h
@@ -595,7 +595,7 @@
// The size of java.lang.Class.class.
static uint32_t ClassClassSize(PointerSize pointer_size) {
// The number of vtable entries in java.lang.Class.
- uint32_t vtable_entries = Object::kVTableLength + 73;
+ uint32_t vtable_entries = Object::kVTableLength + 70;
return ComputeClassSize(true, vtable_entries, 0, 0, 4, 1, 0, pointer_size);
}
diff --git a/runtime/oat_file_manager.cc b/runtime/oat_file_manager.cc
index 6e30f14..d04dbbe 100644
--- a/runtime/oat_file_manager.cc
+++ b/runtime/oat_file_manager.cc
@@ -23,6 +23,7 @@
#include "android-base/stringprintf.h"
#include "art_field-inl.h"
+#include "base/bit_vector-inl.h"
#include "base/logging.h"
#include "base/stl_util.h"
#include "base/systrace.h"
@@ -145,13 +146,52 @@
return oat_files;
}
+class TypeIndexInfo {
+ public:
+ explicit TypeIndexInfo(const DexFile* dex_file)
+ : type_indexes_(GenerateTypeIndexes(dex_file)),
+ iter_(type_indexes_.Indexes().begin()),
+ end_(type_indexes_.Indexes().end()) { }
+
+ BitVector& GetTypeIndexes() {
+ return type_indexes_;
+ }
+ BitVector::IndexIterator& GetIterator() {
+ return iter_;
+ }
+ BitVector::IndexIterator& GetIteratorEnd() {
+ return end_;
+ }
+ void AdvanceIterator() {
+ iter_++;
+ }
+
+ private:
+ static BitVector GenerateTypeIndexes(const DexFile* dex_file) {
+ BitVector type_indexes(/*start_bits*/0, /*expandable*/true, Allocator::GetMallocAllocator());
+ for (uint16_t i = 0; i < dex_file->NumClassDefs(); ++i) {
+ const DexFile::ClassDef& class_def = dex_file->GetClassDef(i);
+ uint16_t type_idx = class_def.class_idx_.index_;
+ type_indexes.SetBit(type_idx);
+ }
+ return type_indexes;
+ }
+
+ // BitVector with bits set for the type indexes of all classes in the input dex file.
+ BitVector type_indexes_;
+ BitVector::IndexIterator iter_;
+ BitVector::IndexIterator end_;
+};
+
class DexFileAndClassPair : ValueObject {
public:
- DexFileAndClassPair(const DexFile* dex_file, size_t current_class_index, bool from_loaded_oat)
- : cached_descriptor_(GetClassDescriptor(dex_file, current_class_index)),
+ DexFileAndClassPair(const DexFile* dex_file, TypeIndexInfo* type_info, bool from_loaded_oat)
+ : type_info_(type_info),
dex_file_(dex_file),
- current_class_index_(current_class_index),
- from_loaded_oat_(from_loaded_oat) {}
+ cached_descriptor_(dex_file_->StringByTypeIdx(dex::TypeIndex(*type_info->GetIterator()))),
+ from_loaded_oat_(from_loaded_oat) {
+ type_info_->AdvanceIterator();
+ }
DexFileAndClassPair(const DexFileAndClassPair& rhs) = default;
@@ -172,16 +212,12 @@
}
bool DexFileHasMoreClasses() const {
- return current_class_index_ + 1 < dex_file_->NumClassDefs();
+ return type_info_->GetIterator() != type_info_->GetIteratorEnd();
}
void Next() {
- ++current_class_index_;
- cached_descriptor_ = GetClassDescriptor(dex_file_, current_class_index_);
- }
-
- size_t GetCurrentClassIndex() const {
- return current_class_index_;
+ cached_descriptor_ = dex_file_->StringByTypeIdx(dex::TypeIndex(*type_info_->GetIterator()));
+ type_info_->AdvanceIterator();
}
bool FromLoadedOat() const {
@@ -193,42 +229,36 @@
}
private:
- static const char* GetClassDescriptor(const DexFile* dex_file, size_t index) {
- DCHECK(IsUint<16>(index));
- const DexFile::ClassDef& class_def = dex_file->GetClassDef(static_cast<uint16_t>(index));
- return dex_file->StringByTypeIdx(class_def.class_idx_);
- }
-
- const char* cached_descriptor_;
+ TypeIndexInfo* type_info_;
const DexFile* dex_file_;
- size_t current_class_index_;
+ const char* cached_descriptor_;
bool from_loaded_oat_; // We only need to compare mismatches between what we load now
// and what was loaded before. Any old duplicates must have been
// OK, and any new "internal" duplicates are as well (they must
// be from multidex, which resolves correctly).
};
-static void AddDexFilesFromOat(const OatFile* oat_file,
- bool already_loaded,
- /*out*/std::priority_queue<DexFileAndClassPair>* heap,
- std::vector<std::unique_ptr<const DexFile>>* opened_dex_files) {
+static void AddDexFilesFromOat(
+ const OatFile* oat_file,
+ /*out*/std::vector<const DexFile*>* dex_files,
+ std::vector<std::unique_ptr<const DexFile>>* opened_dex_files) {
for (const OatDexFile* oat_dex_file : oat_file->GetOatDexFiles()) {
std::string error;
std::unique_ptr<const DexFile> dex_file = oat_dex_file->OpenDexFile(&error);
if (dex_file == nullptr) {
LOG(WARNING) << "Could not create dex file from oat file: " << error;
} else if (dex_file->NumClassDefs() > 0U) {
- heap->emplace(dex_file.get(), /*current_class_index*/0U, already_loaded);
+ dex_files->push_back(dex_file.get());
opened_dex_files->push_back(std::move(dex_file));
}
}
}
-static void AddNext(/*inout*/DexFileAndClassPair* original,
- /*inout*/std::priority_queue<DexFileAndClassPair>* heap) {
- if (original->DexFileHasMoreClasses()) {
- original->Next();
- heap->push(std::move(*original));
+static void AddNext(/*inout*/DexFileAndClassPair& original,
+ /*inout*/std::priority_queue<DexFileAndClassPair>& heap) {
+ if (original.DexFileHasMoreClasses()) {
+ original.Next();
+ heap.push(std::move(original));
}
}
@@ -297,7 +327,8 @@
static bool GetDexFilesFromClassLoader(
ScopedObjectAccessAlreadyRunnable& soa,
mirror::ClassLoader* class_loader,
- std::priority_queue<DexFileAndClassPair>* queue) REQUIRES_SHARED(Locks::mutator_lock_) {
+ std::vector<const DexFile*>* dex_files)
+ REQUIRES_SHARED(Locks::mutator_lock_) {
if (ClassLinker::IsBootClassLoader(soa, class_loader)) {
// The boot class loader. We don't load any of these files, as we know we compiled against
// them correctly.
@@ -312,7 +343,7 @@
return false;
}
- bool recursive_result = GetDexFilesFromClassLoader(soa, class_loader->GetParent(), queue);
+ bool recursive_result = GetDexFilesFromClassLoader(soa, class_loader->GetParent(), dex_files);
if (!recursive_result) {
// Something wrong up the chain.
return false;
@@ -322,7 +353,7 @@
auto GetDexFilesFn = [&] (const DexFile* cp_dex_file)
REQUIRES_SHARED(Locks::mutator_lock_) {
if (cp_dex_file->NumClassDefs() > 0) {
- queue->emplace(cp_dex_file, 0U, true);
+ dex_files->push_back(cp_dex_file);
}
return true; // Continue looking.
};
@@ -341,7 +372,8 @@
static void GetDexFilesFromDexElementsArray(
ScopedObjectAccessAlreadyRunnable& soa,
Handle<mirror::ObjectArray<mirror::Object>> dex_elements,
- std::priority_queue<DexFileAndClassPair>* queue) REQUIRES_SHARED(Locks::mutator_lock_) {
+ std::vector<const DexFile*>* dex_files)
+ REQUIRES_SHARED(Locks::mutator_lock_) {
if (dex_elements == nullptr) {
// Nothing to do.
return;
@@ -360,7 +392,7 @@
auto GetDexFilesFn = [&] (const DexFile* cp_dex_file)
REQUIRES_SHARED(Locks::mutator_lock_) {
if (cp_dex_file != nullptr && cp_dex_file->NumClassDefs() > 0) {
- queue->emplace(cp_dex_file, 0U, true);
+ dex_files->push_back(cp_dex_file);
}
return true; // Continue looking.
};
@@ -389,43 +421,95 @@
}
static bool AreSharedLibrariesOk(const std::string& shared_libraries,
- std::priority_queue<DexFileAndClassPair>& queue) {
+ std::vector<const DexFile*>& dex_files) {
+ // If no shared libraries, we expect no dex files.
if (shared_libraries.empty()) {
- if (queue.empty()) {
- // No shared libraries or oat files, as expected.
- return true;
- }
- } else {
- if (shared_libraries.compare(OatFile::kSpecialSharedLibrary) == 0) {
- // If we find the special shared library, skip the shared libraries check.
- return true;
- }
- // Shared libraries is a series of dex file paths and their checksums, each separated by '*'.
- std::vector<std::string> shared_libraries_split;
- Split(shared_libraries, '*', &shared_libraries_split);
-
- size_t index = 0;
- std::priority_queue<DexFileAndClassPair> temp = queue;
- while (!temp.empty() && index < shared_libraries_split.size() - 1) {
- DexFileAndClassPair pair(temp.top());
- const DexFile* dex_file = pair.GetDexFile();
- const std::string& dex_filename = dex_file->GetLocation();
- if (dex_filename != shared_libraries_split[index]) {
- break;
- }
- char* end;
- size_t shared_lib_checksum = strtoul(shared_libraries_split[index + 1].c_str(), &end, 10);
- uint32_t dex_checksum = dex_file->GetLocationChecksum();
- if (*end != '\0' || dex_checksum != shared_lib_checksum) {
- break;
- }
- temp.pop();
- index += 2;
- }
-
- // Check is successful if it made it through the queue and all the shared libraries.
- return temp.empty() && index == shared_libraries_split.size();
+ return dex_files.empty();
}
+ // If we find the special shared library, skip the shared libraries check.
+ if (shared_libraries.compare(OatFile::kSpecialSharedLibrary) == 0) {
+ return true;
+ }
+ // Shared libraries is a series of dex file paths and their checksums, each separated by '*'.
+ std::vector<std::string> shared_libraries_split;
+ Split(shared_libraries, '*', &shared_libraries_split);
+
+ // Sanity check size of dex files and split shared libraries. Should be 2x as many entries in
+ // the split shared libraries since it contains pairs of filename/checksum.
+ if (dex_files.size() * 2 != shared_libraries_split.size()) {
+ return false;
+ }
+
+ for (size_t i = 0; i < dex_files.size(); ++i) {
+ if (dex_files[i]->GetLocation() != shared_libraries_split[i * 2]) {
+ return false;
+ }
+ char* end;
+ size_t shared_lib_checksum = strtoul(shared_libraries_split[i * 2 + 1].c_str(), &end, 10);
+ uint32_t dex_checksum = dex_files[i]->GetLocationChecksum();
+ if (*end != '\0' || dex_checksum != shared_lib_checksum) {
+ return false;
+ }
+ }
+
+ return true;
+}
+
+static bool CollisionCheck(std::vector<const DexFile*>& dex_files_loaded,
+ std::vector<const DexFile*>& dex_files_unloaded,
+ std::string* error_msg /*out*/) {
+ // Generate type index information for each dex file.
+ std::vector<TypeIndexInfo> loaded_types;
+ for (const DexFile* dex_file : dex_files_loaded) {
+ loaded_types.push_back(TypeIndexInfo(dex_file));
+ }
+ std::vector<TypeIndexInfo> unloaded_types;
+ for (const DexFile* dex_file : dex_files_unloaded) {
+ unloaded_types.push_back(TypeIndexInfo(dex_file));
+ }
+
+ // Populate the queue of dex file and class pairs with the loaded and unloaded dex files.
+ std::priority_queue<DexFileAndClassPair> queue;
+ for (size_t i = 0; i < dex_files_loaded.size(); ++i) {
+ if (loaded_types[i].GetIterator() != loaded_types[i].GetIteratorEnd()) {
+ queue.emplace(dex_files_loaded[i], &loaded_types[i], /*from_loaded_oat*/true);
+ }
+ }
+ for (size_t i = 0; i < dex_files_unloaded.size(); ++i) {
+ if (unloaded_types[i].GetIterator() != unloaded_types[i].GetIteratorEnd()) {
+ queue.emplace(dex_files_unloaded[i], &unloaded_types[i], /*from_loaded_oat*/false);
+ }
+ }
+
+ // Now drain the queue.
+ while (!queue.empty()) {
+ // Modifying the top element is only safe if we pop right after.
+ DexFileAndClassPair compare_pop(queue.top());
+ queue.pop();
+
+ // Compare against the following elements.
+ while (!queue.empty()) {
+ DexFileAndClassPair top(queue.top());
+ if (strcmp(compare_pop.GetCachedDescriptor(), top.GetCachedDescriptor()) == 0) {
+ // Same descriptor. Check whether it's crossing old-oat-files to new-oat-files.
+ if (compare_pop.FromLoadedOat() != top.FromLoadedOat()) {
+ *error_msg =
+ StringPrintf("Found duplicated class when checking oat files: '%s' in %s and %s",
+ compare_pop.GetCachedDescriptor(),
+ compare_pop.GetDexFile()->GetLocation().c_str(),
+ top.GetDexFile()->GetLocation().c_str());
+ return true;
+ }
+ queue.pop();
+ AddNext(top, queue);
+ } else {
+ // Something else. Done here.
+ break;
+ }
+ }
+ AddNext(compare_pop, queue);
+ }
+
return false;
}
@@ -450,7 +534,7 @@
DCHECK(oat_file != nullptr);
DCHECK(error_msg != nullptr);
- std::priority_queue<DexFileAndClassPair> queue;
+ std::vector<const DexFile*> dex_files_loaded;
// Try to get dex files from the given class loader. If the class loader is null, or we do
// not support one of the class loaders in the chain, conservatively compare against all
@@ -464,12 +548,12 @@
Handle<mirror::ObjectArray<mirror::Object>> h_dex_elements =
hs.NewHandle(soa.Decode<mirror::ObjectArray<mirror::Object>>(dex_elements));
if (h_class_loader != nullptr &&
- GetDexFilesFromClassLoader(soa, h_class_loader.Get(), &queue)) {
+ GetDexFilesFromClassLoader(soa, h_class_loader.Get(), &dex_files_loaded)) {
class_loader_ok = true;
// In this case, also take into account the dex_elements array, if given. We don't need to
// read it otherwise, as we'll compare against all open oat files anyways.
- GetDexFilesFromDexElementsArray(soa, h_dex_elements, &queue);
+ GetDexFilesFromDexElementsArray(soa, h_dex_elements, &dex_files_loaded);
} else if (h_class_loader != nullptr) {
VLOG(class_linker) << "Something unsupported with "
<< mirror::Class::PrettyClass(h_class_loader->GetClass());
@@ -486,10 +570,8 @@
if (!class_loader_ok) {
// Add dex files from already loaded oat files, but skip boot.
- // Clean up the queue.
- while (!queue.empty()) {
- queue.pop();
- }
+ // Clean up the dex files.
+ dex_files_loaded.clear();
std::vector<const OatFile*> boot_oat_files = GetBootOatFiles();
// The same OatFile can be loaded multiple times at different addresses. In this case, we don't
@@ -503,10 +585,7 @@
boot_oat_files.end() && location != oat_file->GetLocation() &&
unique_locations.find(location) == unique_locations.end()) {
unique_locations.insert(location);
- AddDexFilesFromOat(loaded_oat_file.get(),
- /*already_loaded*/true,
- &queue,
- /*out*/&opened_dex_files);
+ AddDexFilesFromOat(loaded_oat_file.get(), &dex_files_loaded, &opened_dex_files);
}
}
}
@@ -514,46 +593,15 @@
// Exit if shared libraries are ok. Do a full duplicate classes check otherwise.
const std::string
shared_libraries(oat_file->GetOatHeader().GetStoreValueByKey(OatHeader::kClassPathKey));
- if (AreSharedLibrariesOk(shared_libraries, queue)) {
+ if (AreSharedLibrariesOk(shared_libraries, dex_files_loaded)) {
return false;
}
ScopedTrace st("Collision check");
-
// Add dex files from the oat file to check.
- AddDexFilesFromOat(oat_file, /*already_loaded*/false, &queue, &opened_dex_files);
-
- // Now drain the queue.
- while (!queue.empty()) {
- // Modifying the top element is only safe if we pop right after.
- DexFileAndClassPair compare_pop(queue.top());
- queue.pop();
-
- // Compare against the following elements.
- while (!queue.empty()) {
- DexFileAndClassPair top(queue.top());
-
- if (strcmp(compare_pop.GetCachedDescriptor(), top.GetCachedDescriptor()) == 0) {
- // Same descriptor. Check whether it's crossing old-oat-files to new-oat-files.
- if (compare_pop.FromLoadedOat() != top.FromLoadedOat()) {
- *error_msg =
- StringPrintf("Found duplicated class when checking oat files: '%s' in %s and %s",
- compare_pop.GetCachedDescriptor(),
- compare_pop.GetDexFile()->GetLocation().c_str(),
- top.GetDexFile()->GetLocation().c_str());
- return true;
- }
- queue.pop();
- AddNext(&top, &queue);
- } else {
- // Something else. Done here.
- break;
- }
- }
- AddNext(&compare_pop, &queue);
- }
-
- return false;
+ std::vector<const DexFile*> dex_files_unloaded;
+ AddDexFilesFromOat(oat_file, &dex_files_unloaded, &opened_dex_files);
+ return CollisionCheck(dex_files_loaded, dex_files_unloaded, error_msg);
}
std::vector<std::unique_ptr<const DexFile>> OatFileManager::OpenDexFilesFromOat(
diff --git a/runtime/openjdkjvmti/OpenjdkJvmTi.cc b/runtime/openjdkjvmti/OpenjdkJvmTi.cc
index c016728..5e0d4bd 100644
--- a/runtime/openjdkjvmti/OpenjdkJvmTi.cc
+++ b/runtime/openjdkjvmti/OpenjdkJvmTi.cc
@@ -1426,6 +1426,7 @@
ClassUtil::Register(&gEventHandler);
DumpUtil::Register(&gEventHandler);
SearchUtil::Register();
+ HeapUtil::Register();
runtime->GetJavaVM()->AddEnvironmentHook(GetEnvHandler);
@@ -1438,6 +1439,7 @@
ClassUtil::Unregister();
DumpUtil::Unregister();
SearchUtil::Unregister();
+ HeapUtil::Unregister();
return true;
}
diff --git a/runtime/openjdkjvmti/jvmti_weak_table-inl.h b/runtime/openjdkjvmti/jvmti_weak_table-inl.h
new file mode 100644
index 0000000..f67fffc
--- /dev/null
+++ b/runtime/openjdkjvmti/jvmti_weak_table-inl.h
@@ -0,0 +1,389 @@
+/* Copyright (C) 2017 The Android Open Source Project
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This file implements interfaces from the file jvmti.h. This implementation
+ * is licensed under the same terms as the file jvmti.h. The
+ * copyright and license information for the file jvmti.h follows.
+ *
+ * Copyright (c) 2003, 2011, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation. Oracle designates this
+ * particular file as subject to the "Classpath" exception as provided
+ * by Oracle in the LICENSE file that accompanied this code.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#ifndef ART_RUNTIME_OPENJDKJVMTI_JVMTI_WEAK_TABLE_INL_H_
+#define ART_RUNTIME_OPENJDKJVMTI_JVMTI_WEAK_TABLE_INL_H_
+
+#include "jvmti_weak_table.h"
+
+#include <limits>
+
+#include "art_jvmti.h"
+#include "base/logging.h"
+#include "gc/allocation_listener.h"
+#include "instrumentation.h"
+#include "jni_env_ext-inl.h"
+#include "jvmti_allocator.h"
+#include "mirror/class.h"
+#include "mirror/object.h"
+#include "runtime.h"
+#include "ScopedLocalRef.h"
+
+namespace openjdkjvmti {
+
+template <typename T>
+void JvmtiWeakTable<T>::Lock() {
+ allow_disallow_lock_.ExclusiveLock(art::Thread::Current());
+}
+template <typename T>
+void JvmtiWeakTable<T>::Unlock() {
+ allow_disallow_lock_.ExclusiveUnlock(art::Thread::Current());
+}
+template <typename T>
+void JvmtiWeakTable<T>::AssertLocked() {
+ allow_disallow_lock_.AssertHeld(art::Thread::Current());
+}
+
+template <typename T>
+void JvmtiWeakTable<T>::UpdateTableWithReadBarrier() {
+ update_since_last_sweep_ = true;
+
+ auto WithReadBarrierUpdater = [&](const art::GcRoot<art::mirror::Object>& original_root,
+ art::mirror::Object* original_obj ATTRIBUTE_UNUSED)
+ REQUIRES_SHARED(art::Locks::mutator_lock_) {
+ return original_root.Read<art::kWithReadBarrier>();
+ };
+
+ UpdateTableWith<decltype(WithReadBarrierUpdater), kIgnoreNull>(WithReadBarrierUpdater);
+}
+
+template <typename T>
+bool JvmtiWeakTable<T>::GetTagSlowPath(art::Thread* self, art::mirror::Object* obj, T* result) {
+ // Under concurrent GC, there is a window between moving objects and sweeping of system
+ // weaks in which mutators are active. We may receive a to-space object pointer in obj,
+ // but still have from-space pointers in the table. Explicitly update the table once.
+ // Note: this will keep *all* objects in the table live, but should be a rare occurrence.
+ UpdateTableWithReadBarrier();
+ return GetTagLocked(self, obj, result);
+}
+
+template <typename T>
+bool JvmtiWeakTable<T>::Remove(art::mirror::Object* obj, /* out */ T* tag) {
+ art::Thread* self = art::Thread::Current();
+ art::MutexLock mu(self, allow_disallow_lock_);
+ Wait(self);
+
+ return RemoveLocked(self, obj, tag);
+}
+template <typename T>
+bool JvmtiWeakTable<T>::RemoveLocked(art::mirror::Object* obj, T* tag) {
+ art::Thread* self = art::Thread::Current();
+ allow_disallow_lock_.AssertHeld(self);
+ Wait(self);
+
+ return RemoveLocked(self, obj, tag);
+}
+
+template <typename T>
+bool JvmtiWeakTable<T>::RemoveLocked(art::Thread* self, art::mirror::Object* obj, T* tag) {
+ auto it = tagged_objects_.find(art::GcRoot<art::mirror::Object>(obj));
+ if (it != tagged_objects_.end()) {
+ if (tag != nullptr) {
+ *tag = it->second;
+ }
+ tagged_objects_.erase(it);
+ return true;
+ }
+
+ if (art::kUseReadBarrier && self->GetIsGcMarking() && !update_since_last_sweep_) {
+ // Under concurrent GC, there is a window between moving objects and sweeping of system
+ // weaks in which mutators are active. We may receive a to-space object pointer in obj,
+ // but still have from-space pointers in the table. Explicitly update the table once.
+ // Note: this will keep *all* objects in the table live, but should be a rare occurrence.
+
+ // Update the table.
+ UpdateTableWithReadBarrier();
+
+ // And try again.
+ return RemoveLocked(self, obj, tag);
+ }
+
+ // Not in here.
+ return false;
+}
+
+template <typename T>
+bool JvmtiWeakTable<T>::Set(art::mirror::Object* obj, T new_tag) {
+ art::Thread* self = art::Thread::Current();
+ art::MutexLock mu(self, allow_disallow_lock_);
+ Wait(self);
+
+ return SetLocked(self, obj, new_tag);
+}
+template <typename T>
+bool JvmtiWeakTable<T>::SetLocked(art::mirror::Object* obj, T new_tag) {
+ art::Thread* self = art::Thread::Current();
+ allow_disallow_lock_.AssertHeld(self);
+ Wait(self);
+
+ return SetLocked(self, obj, new_tag);
+}
+
+template <typename T>
+bool JvmtiWeakTable<T>::SetLocked(art::Thread* self, art::mirror::Object* obj, T new_tag) {
+ auto it = tagged_objects_.find(art::GcRoot<art::mirror::Object>(obj));
+ if (it != tagged_objects_.end()) {
+ it->second = new_tag;
+ return true;
+ }
+
+ if (art::kUseReadBarrier && self->GetIsGcMarking() && !update_since_last_sweep_) {
+ // Under concurrent GC, there is a window between moving objects and sweeping of system
+ // weaks in which mutators are active. We may receive a to-space object pointer in obj,
+ // but still have from-space pointers in the table. Explicitly update the table once.
+ // Note: this will keep *all* objects in the table live, but should be a rare occurrence.
+
+ // Update the table.
+ UpdateTableWithReadBarrier();
+
+ // And try again.
+ return SetLocked(self, obj, new_tag);
+ }
+
+ // New element.
+ auto insert_it = tagged_objects_.emplace(art::GcRoot<art::mirror::Object>(obj), new_tag);
+ DCHECK(insert_it.second);
+ return false;
+}
+
+template <typename T>
+void JvmtiWeakTable<T>::Sweep(art::IsMarkedVisitor* visitor) {
+ if (DoesHandleNullOnSweep()) {
+ SweepImpl<true>(visitor);
+ } else {
+ SweepImpl<false>(visitor);
+ }
+
+ // Under concurrent GC, there is a window between moving objects and sweeping of system
+ // weaks in which mutators are active. We may receive a to-space object pointer in obj,
+ // but still have from-space pointers in the table. We explicitly update the table then
+ // to ensure we compare against to-space pointers. But we want to do this only once. Once
+ // sweeping is done, we know all objects are to-space pointers until the next GC cycle,
+ // so we re-enable the explicit update for the next marking.
+ update_since_last_sweep_ = false;
+}
+
+template <typename T>
+template <bool kHandleNull>
+void JvmtiWeakTable<T>::SweepImpl(art::IsMarkedVisitor* visitor) {
+ art::Thread* self = art::Thread::Current();
+ art::MutexLock mu(self, allow_disallow_lock_);
+
+ auto IsMarkedUpdater = [&](const art::GcRoot<art::mirror::Object>& original_root ATTRIBUTE_UNUSED,
+ art::mirror::Object* original_obj) {
+ return visitor->IsMarked(original_obj);
+ };
+
+ UpdateTableWith<decltype(IsMarkedUpdater),
+ kHandleNull ? kCallHandleNull : kRemoveNull>(IsMarkedUpdater);
+}
+
+template <typename T>
+template <typename Updater, typename JvmtiWeakTable<T>::TableUpdateNullTarget kTargetNull>
+ALWAYS_INLINE inline void JvmtiWeakTable<T>::UpdateTableWith(Updater& updater) {
+ // We optimistically hope that elements will still be well-distributed when re-inserting them.
+ // So play with the map mechanics, and postpone rehashing. This avoids the need of a side
+ // vector and two passes.
+ float original_max_load_factor = tagged_objects_.max_load_factor();
+ tagged_objects_.max_load_factor(std::numeric_limits<float>::max());
+ // For checking that a max load-factor actually does what we expect.
+ size_t original_bucket_count = tagged_objects_.bucket_count();
+
+ for (auto it = tagged_objects_.begin(); it != tagged_objects_.end();) {
+ DCHECK(!it->first.IsNull());
+ art::mirror::Object* original_obj = it->first.template Read<art::kWithoutReadBarrier>();
+ art::mirror::Object* target_obj = updater(it->first, original_obj);
+ if (original_obj != target_obj) {
+ if (kTargetNull == kIgnoreNull && target_obj == nullptr) {
+ // Ignore null target, don't do anything.
+ } else {
+ T tag = it->second;
+ it = tagged_objects_.erase(it);
+ if (target_obj != nullptr) {
+ tagged_objects_.emplace(art::GcRoot<art::mirror::Object>(target_obj), tag);
+ DCHECK_EQ(original_bucket_count, tagged_objects_.bucket_count());
+ } else if (kTargetNull == kCallHandleNull) {
+ HandleNullSweep(tag);
+ }
+ continue; // Iterator was implicitly updated by erase.
+ }
+ }
+ it++;
+ }
+
+ tagged_objects_.max_load_factor(original_max_load_factor);
+ // TODO: consider rehash here.
+}
+
+template <typename T>
+template <typename Storage, class Allocator>
+struct JvmtiWeakTable<T>::ReleasableContainer {
+ using allocator_type = Allocator;
+
+ explicit ReleasableContainer(const allocator_type& alloc, size_t reserve = 10)
+ : allocator(alloc),
+ data(reserve > 0 ? allocator.allocate(reserve) : nullptr),
+ size(0),
+ capacity(reserve) {
+ }
+
+ ~ReleasableContainer() {
+ if (data != nullptr) {
+ allocator.deallocate(data, capacity);
+ capacity = 0;
+ size = 0;
+ }
+ }
+
+ Storage* Release() {
+ Storage* tmp = data;
+
+ data = nullptr;
+ size = 0;
+ capacity = 0;
+
+ return tmp;
+ }
+
+ void Resize(size_t new_capacity) {
+ CHECK_GT(new_capacity, capacity);
+
+ Storage* tmp = allocator.allocate(new_capacity);
+ DCHECK(tmp != nullptr);
+ if (data != nullptr) {
+ memcpy(tmp, data, sizeof(Storage) * size);
+ }
+ Storage* old = data;
+ data = tmp;
+ allocator.deallocate(old, capacity);
+ capacity = new_capacity;
+ }
+
+ void Pushback(const Storage& elem) {
+ if (size == capacity) {
+ size_t new_capacity = 2 * capacity + 1;
+ Resize(new_capacity);
+ }
+ data[size++] = elem;
+ }
+
+ Allocator allocator;
+ Storage* data;
+ size_t size;
+ size_t capacity;
+};
+
+template <typename T>
+jvmtiError JvmtiWeakTable<T>::GetTaggedObjects(jvmtiEnv* jvmti_env,
+ jint tag_count,
+ const T* tags,
+ jint* count_ptr,
+ jobject** object_result_ptr,
+ T** tag_result_ptr) {
+ if (tag_count < 0) {
+ return ERR(ILLEGAL_ARGUMENT);
+ }
+ if (tag_count > 0) {
+ for (size_t i = 0; i != static_cast<size_t>(tag_count); ++i) {
+ if (tags[i] == 0) {
+ return ERR(ILLEGAL_ARGUMENT);
+ }
+ }
+ }
+ if (tags == nullptr) {
+ return ERR(NULL_POINTER);
+ }
+ if (count_ptr == nullptr) {
+ return ERR(NULL_POINTER);
+ }
+
+ art::Thread* self = art::Thread::Current();
+ art::MutexLock mu(self, allow_disallow_lock_);
+ Wait(self);
+
+ art::JNIEnvExt* jni_env = self->GetJniEnv();
+
+ constexpr size_t kDefaultSize = 10;
+ size_t initial_object_size;
+ size_t initial_tag_size;
+ if (tag_count == 0) {
+ initial_object_size = (object_result_ptr != nullptr) ? tagged_objects_.size() : 0;
+ initial_tag_size = (tag_result_ptr != nullptr) ? tagged_objects_.size() : 0;
+ } else {
+ initial_object_size = initial_tag_size = kDefaultSize;
+ }
+ JvmtiAllocator<void> allocator(jvmti_env);
+ ReleasableContainer<jobject, JvmtiAllocator<jobject>> selected_objects(allocator,
+ initial_object_size);
+ ReleasableContainer<T, JvmtiAllocator<T>> selected_tags(allocator, initial_tag_size);
+
+ size_t count = 0;
+ for (auto& pair : tagged_objects_) {
+ bool select;
+ if (tag_count > 0) {
+ select = false;
+ for (size_t i = 0; i != static_cast<size_t>(tag_count); ++i) {
+ if (tags[i] == pair.second) {
+ select = true;
+ break;
+ }
+ }
+ } else {
+ select = true;
+ }
+
+ if (select) {
+ art::mirror::Object* obj = pair.first.template Read<art::kWithReadBarrier>();
+ if (obj != nullptr) {
+ count++;
+ if (object_result_ptr != nullptr) {
+ selected_objects.Pushback(jni_env->AddLocalReference<jobject>(obj));
+ }
+ if (tag_result_ptr != nullptr) {
+ selected_tags.Pushback(pair.second);
+ }
+ }
+ }
+ }
+
+ if (object_result_ptr != nullptr) {
+ *object_result_ptr = selected_objects.Release();
+ }
+ if (tag_result_ptr != nullptr) {
+ *tag_result_ptr = selected_tags.Release();
+ }
+ *count_ptr = static_cast<jint>(count);
+ return ERR(NONE);
+}
+
+} // namespace openjdkjvmti
+
+#endif // ART_RUNTIME_OPENJDKJVMTI_JVMTI_WEAK_TABLE_INL_H_
diff --git a/runtime/openjdkjvmti/jvmti_weak_table.h b/runtime/openjdkjvmti/jvmti_weak_table.h
new file mode 100644
index 0000000..ae36122
--- /dev/null
+++ b/runtime/openjdkjvmti/jvmti_weak_table.h
@@ -0,0 +1,219 @@
+/* Copyright (C) 2017 The Android Open Source Project
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This file implements interfaces from the file jvmti.h. This implementation
+ * is licensed under the same terms as the file jvmti.h. The
+ * copyright and license information for the file jvmti.h follows.
+ *
+ * Copyright (c) 2003, 2011, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation. Oracle designates this
+ * particular file as subject to the "Classpath" exception as provided
+ * by Oracle in the LICENSE file that accompanied this code.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#ifndef ART_RUNTIME_OPENJDKJVMTI_JVMTI_WEAK_TABLE_H_
+#define ART_RUNTIME_OPENJDKJVMTI_JVMTI_WEAK_TABLE_H_
+
+#include <unordered_map>
+
+#include "base/macros.h"
+#include "base/mutex.h"
+#include "gc/system_weak.h"
+#include "gc_root-inl.h"
+#include "globals.h"
+#include "jvmti.h"
+#include "mirror/object.h"
+#include "thread-inl.h"
+
+namespace openjdkjvmti {
+
+class EventHandler;
+
+// A system-weak container mapping objects to elements of the template type. This corresponds
+// to a weak hash map. For historical reasons the stored value is called "tag."
+template <typename T>
+class JvmtiWeakTable : public art::gc::SystemWeakHolder {
+ public:
+ JvmtiWeakTable()
+ : art::gc::SystemWeakHolder(kTaggingLockLevel),
+ update_since_last_sweep_(false) {
+ }
+
+ // Remove the mapping for the given object, returning whether such a mapping existed (and the old
+ // value).
+ bool Remove(art::mirror::Object* obj, /* out */ T* tag)
+ REQUIRES_SHARED(art::Locks::mutator_lock_)
+ REQUIRES(!allow_disallow_lock_);
+ bool RemoveLocked(art::mirror::Object* obj, /* out */ T* tag)
+ REQUIRES_SHARED(art::Locks::mutator_lock_)
+ REQUIRES(allow_disallow_lock_);
+
+ // Set the mapping for the given object. Returns true if this overwrites an already existing
+ // mapping.
+ virtual bool Set(art::mirror::Object* obj, T tag)
+ REQUIRES_SHARED(art::Locks::mutator_lock_)
+ REQUIRES(!allow_disallow_lock_);
+ virtual bool SetLocked(art::mirror::Object* obj, T tag)
+ REQUIRES_SHARED(art::Locks::mutator_lock_)
+ REQUIRES(allow_disallow_lock_);
+
+ // Return the value associated with the given object. Returns true if the mapping exists, false
+ // otherwise.
+ bool GetTag(art::mirror::Object* obj, /* out */ T* result)
+ REQUIRES_SHARED(art::Locks::mutator_lock_)
+ REQUIRES(!allow_disallow_lock_) {
+ art::Thread* self = art::Thread::Current();
+ art::MutexLock mu(self, allow_disallow_lock_);
+ Wait(self);
+
+ return GetTagLocked(self, obj, result);
+ }
+ bool GetTagLocked(art::mirror::Object* obj, /* out */ T* result)
+ REQUIRES_SHARED(art::Locks::mutator_lock_)
+ REQUIRES(allow_disallow_lock_) {
+ art::Thread* self = art::Thread::Current();
+ allow_disallow_lock_.AssertHeld(self);
+ Wait(self);
+
+ return GetTagLocked(self, obj, result);
+ }
+
+ // Sweep the container. DO NOT CALL MANUALLY.
+ void Sweep(art::IsMarkedVisitor* visitor)
+ REQUIRES_SHARED(art::Locks::mutator_lock_)
+ REQUIRES(!allow_disallow_lock_);
+
+ // Return all objects that have a value mapping in tags.
+ jvmtiError GetTaggedObjects(jvmtiEnv* jvmti_env,
+ jint tag_count,
+ const T* tags,
+ /* out */ jint* count_ptr,
+ /* out */ jobject** object_result_ptr,
+ /* out */ T** tag_result_ptr)
+ REQUIRES_SHARED(art::Locks::mutator_lock_)
+ REQUIRES(!allow_disallow_lock_);
+
+ // Locking functions, to allow coarse-grained locking and amortization.
+ void Lock() ACQUIRE(allow_disallow_lock_);
+ void Unlock() RELEASE(allow_disallow_lock_);
+ void AssertLocked() ASSERT_CAPABILITY(allow_disallow_lock_);
+
+ protected:
+ // Should HandleNullSweep be called when Sweep detects the release of an object?
+ virtual bool DoesHandleNullOnSweep() {
+ return false;
+ }
+ // If DoesHandleNullOnSweep returns true, this function will be called.
+ virtual void HandleNullSweep(T tag ATTRIBUTE_UNUSED) {}
+
+ private:
+ bool SetLocked(art::Thread* self, art::mirror::Object* obj, T tag)
+ REQUIRES_SHARED(art::Locks::mutator_lock_)
+ REQUIRES(allow_disallow_lock_);
+
+ bool RemoveLocked(art::Thread* self, art::mirror::Object* obj, /* out */ T* tag)
+ REQUIRES_SHARED(art::Locks::mutator_lock_)
+ REQUIRES(allow_disallow_lock_);
+
+ bool GetTagLocked(art::Thread* self, art::mirror::Object* obj, /* out */ T* result)
+ REQUIRES_SHARED(art::Locks::mutator_lock_)
+ REQUIRES(allow_disallow_lock_) {
+ auto it = tagged_objects_.find(art::GcRoot<art::mirror::Object>(obj));
+ if (it != tagged_objects_.end()) {
+ *result = it->second;
+ return true;
+ }
+
+ // Performance optimization: To avoid multiple table updates, ensure that during GC we
+ // only update once. See the comment on the implementation of GetTagSlowPath.
+ if (art::kUseReadBarrier &&
+ self != nullptr &&
+ self->GetIsGcMarking() &&
+ !update_since_last_sweep_) {
+ return GetTagSlowPath(self, obj, result);
+ }
+
+ return false;
+ }
+
+ // Slow-path for GetTag. We didn't find the object, but we might be storing from-pointers and
+ // are asked to retrieve with a to-pointer.
+ bool GetTagSlowPath(art::Thread* self, art::mirror::Object* obj, /* out */ T* result)
+ REQUIRES_SHARED(art::Locks::mutator_lock_)
+ REQUIRES(allow_disallow_lock_);
+
+ // Update the table by doing read barriers on each element, ensuring that to-space pointers
+ // are stored.
+ void UpdateTableWithReadBarrier()
+ REQUIRES_SHARED(art::Locks::mutator_lock_)
+ REQUIRES(allow_disallow_lock_);
+
+ template <bool kHandleNull>
+ void SweepImpl(art::IsMarkedVisitor* visitor)
+ REQUIRES_SHARED(art::Locks::mutator_lock_)
+ REQUIRES(!allow_disallow_lock_);
+
+ enum TableUpdateNullTarget {
+ kIgnoreNull,
+ kRemoveNull,
+ kCallHandleNull
+ };
+
+ template <typename Updater, TableUpdateNullTarget kTargetNull>
+ void UpdateTableWith(Updater& updater)
+ REQUIRES_SHARED(art::Locks::mutator_lock_)
+ REQUIRES(allow_disallow_lock_);
+
+ template <typename Storage, class Allocator = std::allocator<T>>
+ struct ReleasableContainer;
+
+ struct HashGcRoot {
+ size_t operator()(const art::GcRoot<art::mirror::Object>& r) const
+ REQUIRES_SHARED(art::Locks::mutator_lock_) {
+ return reinterpret_cast<uintptr_t>(r.Read<art::kWithoutReadBarrier>());
+ }
+ };
+
+ struct EqGcRoot {
+ bool operator()(const art::GcRoot<art::mirror::Object>& r1,
+ const art::GcRoot<art::mirror::Object>& r2) const
+ REQUIRES_SHARED(art::Locks::mutator_lock_) {
+ return r1.Read<art::kWithoutReadBarrier>() == r2.Read<art::kWithoutReadBarrier>();
+ }
+ };
+
+ // The tag table is used when visiting roots. So it needs to have a low lock level.
+ static constexpr art::LockLevel kTaggingLockLevel =
+ static_cast<art::LockLevel>(art::LockLevel::kAbortLock + 1);
+
+ std::unordered_map<art::GcRoot<art::mirror::Object>,
+ T,
+ HashGcRoot,
+ EqGcRoot> tagged_objects_
+ GUARDED_BY(allow_disallow_lock_)
+ GUARDED_BY(art::Locks::mutator_lock_);
+ // To avoid repeatedly scanning the whole table, remember if we did that since the last sweep.
+ bool update_since_last_sweep_;
+};
+
+} // namespace openjdkjvmti
+
+#endif // ART_RUNTIME_OPENJDKJVMTI_JVMTI_WEAK_TABLE_H_
diff --git a/runtime/openjdkjvmti/object_tagging.cc b/runtime/openjdkjvmti/object_tagging.cc
index b27c2a3..4215588 100644
--- a/runtime/openjdkjvmti/object_tagging.cc
+++ b/runtime/openjdkjvmti/object_tagging.cc
@@ -33,355 +33,34 @@
#include <limits>
-#include "art_jvmti.h"
-#include "base/logging.h"
#include "events-inl.h"
-#include "gc/allocation_listener.h"
-#include "instrumentation.h"
-#include "jni_env_ext-inl.h"
-#include "jvmti_allocator.h"
-#include "mirror/class.h"
-#include "mirror/object.h"
-#include "runtime.h"
-#include "ScopedLocalRef.h"
+#include "jvmti_weak_table-inl.h"
namespace openjdkjvmti {
-void ObjectTagTable::Lock() {
- allow_disallow_lock_.ExclusiveLock(art::Thread::Current());
-}
-void ObjectTagTable::Unlock() {
- allow_disallow_lock_.ExclusiveUnlock(art::Thread::Current());
-}
-void ObjectTagTable::AssertLocked() {
- allow_disallow_lock_.AssertHeld(art::Thread::Current());
-}
-
-void ObjectTagTable::UpdateTableWithReadBarrier() {
- update_since_last_sweep_ = true;
-
- auto WithReadBarrierUpdater = [&](const art::GcRoot<art::mirror::Object>& original_root,
- art::mirror::Object* original_obj ATTRIBUTE_UNUSED)
- REQUIRES_SHARED(art::Locks::mutator_lock_) {
- return original_root.Read<art::kWithReadBarrier>();
- };
-
- UpdateTableWith<decltype(WithReadBarrierUpdater), kIgnoreNull>(WithReadBarrierUpdater);
-}
-
-bool ObjectTagTable::GetTagSlowPath(art::Thread* self, art::mirror::Object* obj, jlong* result) {
- // Under concurrent GC, there is a window between moving objects and sweeping of system
- // weaks in which mutators are active. We may receive a to-space object pointer in obj,
- // but still have from-space pointers in the table. Explicitly update the table once.
- // Note: this will keep *all* objects in the table live, but should be a rare occurrence.
- UpdateTableWithReadBarrier();
- return GetTagLocked(self, obj, result);
-}
-
-void ObjectTagTable::Add(art::mirror::Object* obj, jlong tag) {
- // Same as Set(), as we don't have duplicates in an unordered_map.
- Set(obj, tag);
-}
-
-bool ObjectTagTable::Remove(art::mirror::Object* obj, jlong* tag) {
- art::Thread* self = art::Thread::Current();
- art::MutexLock mu(self, allow_disallow_lock_);
- Wait(self);
-
- return RemoveLocked(self, obj, tag);
-}
-bool ObjectTagTable::RemoveLocked(art::mirror::Object* obj, jlong* tag) {
- art::Thread* self = art::Thread::Current();
- allow_disallow_lock_.AssertHeld(self);
- Wait(self);
-
- return RemoveLocked(self, obj, tag);
-}
-
-bool ObjectTagTable::RemoveLocked(art::Thread* self, art::mirror::Object* obj, jlong* tag) {
- auto it = tagged_objects_.find(art::GcRoot<art::mirror::Object>(obj));
- if (it != tagged_objects_.end()) {
- if (tag != nullptr) {
- *tag = it->second;
- }
- tagged_objects_.erase(it);
- return true;
- }
-
- if (art::kUseReadBarrier && self->GetIsGcMarking() && !update_since_last_sweep_) {
- // Under concurrent GC, there is a window between moving objects and sweeping of system
- // weaks in which mutators are active. We may receive a to-space object pointer in obj,
- // but still have from-space pointers in the table. Explicitly update the table once.
- // Note: this will keep *all* objects in the table live, but should be a rare occurrence.
-
- // Update the table.
- UpdateTableWithReadBarrier();
-
- // And try again.
- return RemoveLocked(self, obj, tag);
- }
-
- // Not in here.
- return false;
-}
+// Instantiate for jlong = JVMTI tags.
+template class JvmtiWeakTable<jlong>;
bool ObjectTagTable::Set(art::mirror::Object* obj, jlong new_tag) {
if (new_tag == 0) {
jlong tmp;
return Remove(obj, &tmp);
}
-
- art::Thread* self = art::Thread::Current();
- art::MutexLock mu(self, allow_disallow_lock_);
- Wait(self);
-
- return SetLocked(self, obj, new_tag);
+ return JvmtiWeakTable<jlong>::Set(obj, new_tag);
}
bool ObjectTagTable::SetLocked(art::mirror::Object* obj, jlong new_tag) {
if (new_tag == 0) {
jlong tmp;
return RemoveLocked(obj, &tmp);
}
-
- art::Thread* self = art::Thread::Current();
- allow_disallow_lock_.AssertHeld(self);
- Wait(self);
-
- return SetLocked(self, obj, new_tag);
+ return JvmtiWeakTable<jlong>::SetLocked(obj, new_tag);
}
-bool ObjectTagTable::SetLocked(art::Thread* self, art::mirror::Object* obj, jlong new_tag) {
- auto it = tagged_objects_.find(art::GcRoot<art::mirror::Object>(obj));
- if (it != tagged_objects_.end()) {
- it->second = new_tag;
- return true;
- }
-
- if (art::kUseReadBarrier && self->GetIsGcMarking() && !update_since_last_sweep_) {
- // Under concurrent GC, there is a window between moving objects and sweeping of system
- // weaks in which mutators are active. We may receive a to-space object pointer in obj,
- // but still have from-space pointers in the table. Explicitly update the table once.
- // Note: this will keep *all* objects in the table live, but should be a rare occurrence.
-
- // Update the table.
- UpdateTableWithReadBarrier();
-
- // And try again.
- return SetLocked(self, obj, new_tag);
- }
-
- // New element.
- auto insert_it = tagged_objects_.emplace(art::GcRoot<art::mirror::Object>(obj), new_tag);
- DCHECK(insert_it.second);
- return false;
+bool ObjectTagTable::DoesHandleNullOnSweep() {
+ return event_handler_->IsEventEnabledAnywhere(ArtJvmtiEvent::kObjectFree);
}
-
-void ObjectTagTable::Sweep(art::IsMarkedVisitor* visitor) {
- if (event_handler_->IsEventEnabledAnywhere(ArtJvmtiEvent::kObjectFree)) {
- SweepImpl<true>(visitor);
- } else {
- SweepImpl<false>(visitor);
- }
-
- // Under concurrent GC, there is a window between moving objects and sweeping of system
- // weaks in which mutators are active. We may receive a to-space object pointer in obj,
- // but still have from-space pointers in the table. We explicitly update the table then
- // to ensure we compare against to-space pointers. But we want to do this only once. Once
- // sweeping is done, we know all objects are to-space pointers until the next GC cycle,
- // so we re-enable the explicit update for the next marking.
- update_since_last_sweep_ = false;
-}
-
-template <bool kHandleNull>
-void ObjectTagTable::SweepImpl(art::IsMarkedVisitor* visitor) {
- art::Thread* self = art::Thread::Current();
- art::MutexLock mu(self, allow_disallow_lock_);
-
- auto IsMarkedUpdater = [&](const art::GcRoot<art::mirror::Object>& original_root ATTRIBUTE_UNUSED,
- art::mirror::Object* original_obj) {
- return visitor->IsMarked(original_obj);
- };
-
- UpdateTableWith<decltype(IsMarkedUpdater),
- kHandleNull ? kCallHandleNull : kRemoveNull>(IsMarkedUpdater);
-}
-
void ObjectTagTable::HandleNullSweep(jlong tag) {
event_handler_->DispatchEvent<ArtJvmtiEvent::kObjectFree>(nullptr, tag);
}
-template <typename T, ObjectTagTable::TableUpdateNullTarget kTargetNull>
-ALWAYS_INLINE inline void ObjectTagTable::UpdateTableWith(T& updater) {
- // We optimistically hope that elements will still be well-distributed when re-inserting them.
- // So play with the map mechanics, and postpone rehashing. This avoids the need of a side
- // vector and two passes.
- float original_max_load_factor = tagged_objects_.max_load_factor();
- tagged_objects_.max_load_factor(std::numeric_limits<float>::max());
- // For checking that a max load-factor actually does what we expect.
- size_t original_bucket_count = tagged_objects_.bucket_count();
-
- for (auto it = tagged_objects_.begin(); it != tagged_objects_.end();) {
- DCHECK(!it->first.IsNull());
- art::mirror::Object* original_obj = it->first.Read<art::kWithoutReadBarrier>();
- art::mirror::Object* target_obj = updater(it->first, original_obj);
- if (original_obj != target_obj) {
- if (kTargetNull == kIgnoreNull && target_obj == nullptr) {
- // Ignore null target, don't do anything.
- } else {
- jlong tag = it->second;
- it = tagged_objects_.erase(it);
- if (target_obj != nullptr) {
- tagged_objects_.emplace(art::GcRoot<art::mirror::Object>(target_obj), tag);
- DCHECK_EQ(original_bucket_count, tagged_objects_.bucket_count());
- } else if (kTargetNull == kCallHandleNull) {
- HandleNullSweep(tag);
- }
- continue; // Iterator was implicitly updated by erase.
- }
- }
- it++;
- }
-
- tagged_objects_.max_load_factor(original_max_load_factor);
- // TODO: consider rehash here.
-}
-
-template <typename T, class Allocator = std::allocator<T>>
-struct ReleasableContainer {
- using allocator_type = Allocator;
-
- explicit ReleasableContainer(const allocator_type& alloc, size_t reserve = 10)
- : allocator(alloc),
- data(reserve > 0 ? allocator.allocate(reserve) : nullptr),
- size(0),
- capacity(reserve) {
- }
-
- ~ReleasableContainer() {
- if (data != nullptr) {
- allocator.deallocate(data, capacity);
- capacity = 0;
- size = 0;
- }
- }
-
- T* Release() {
- T* tmp = data;
-
- data = nullptr;
- size = 0;
- capacity = 0;
-
- return tmp;
- }
-
- void Resize(size_t new_capacity) {
- CHECK_GT(new_capacity, capacity);
-
- T* tmp = allocator.allocate(new_capacity);
- DCHECK(tmp != nullptr);
- if (data != nullptr) {
- memcpy(tmp, data, sizeof(T) * size);
- }
- T* old = data;
- data = tmp;
- allocator.deallocate(old, capacity);
- capacity = new_capacity;
- }
-
- void Pushback(const T& elem) {
- if (size == capacity) {
- size_t new_capacity = 2 * capacity + 1;
- Resize(new_capacity);
- }
- data[size++] = elem;
- }
-
- Allocator allocator;
- T* data;
- size_t size;
- size_t capacity;
-};
-
-jvmtiError ObjectTagTable::GetTaggedObjects(jvmtiEnv* jvmti_env,
- jint tag_count,
- const jlong* tags,
- jint* count_ptr,
- jobject** object_result_ptr,
- jlong** tag_result_ptr) {
- if (tag_count < 0) {
- return ERR(ILLEGAL_ARGUMENT);
- }
- if (tag_count > 0) {
- for (size_t i = 0; i != static_cast<size_t>(tag_count); ++i) {
- if (tags[i] == 0) {
- return ERR(ILLEGAL_ARGUMENT);
- }
- }
- }
- if (tags == nullptr) {
- return ERR(NULL_POINTER);
- }
- if (count_ptr == nullptr) {
- return ERR(NULL_POINTER);
- }
-
- art::Thread* self = art::Thread::Current();
- art::MutexLock mu(self, allow_disallow_lock_);
- Wait(self);
-
- art::JNIEnvExt* jni_env = self->GetJniEnv();
-
- constexpr size_t kDefaultSize = 10;
- size_t initial_object_size;
- size_t initial_tag_size;
- if (tag_count == 0) {
- initial_object_size = (object_result_ptr != nullptr) ? tagged_objects_.size() : 0;
- initial_tag_size = (tag_result_ptr != nullptr) ? tagged_objects_.size() : 0;
- } else {
- initial_object_size = initial_tag_size = kDefaultSize;
- }
- JvmtiAllocator<void> allocator(jvmti_env);
- ReleasableContainer<jobject, JvmtiAllocator<jobject>> selected_objects(allocator, initial_object_size);
- ReleasableContainer<jlong, JvmtiAllocator<jlong>> selected_tags(allocator, initial_tag_size);
-
- size_t count = 0;
- for (auto& pair : tagged_objects_) {
- bool select;
- if (tag_count > 0) {
- select = false;
- for (size_t i = 0; i != static_cast<size_t>(tag_count); ++i) {
- if (tags[i] == pair.second) {
- select = true;
- break;
- }
- }
- } else {
- select = true;
- }
-
- if (select) {
- art::mirror::Object* obj = pair.first.Read<art::kWithReadBarrier>();
- if (obj != nullptr) {
- count++;
- if (object_result_ptr != nullptr) {
- selected_objects.Pushback(jni_env->AddLocalReference<jobject>(obj));
- }
- if (tag_result_ptr != nullptr) {
- selected_tags.Pushback(pair.second);
- }
- }
- }
- }
-
- if (object_result_ptr != nullptr) {
- *object_result_ptr = selected_objects.Release();
- }
- if (tag_result_ptr != nullptr) {
- *tag_result_ptr = selected_tags.Release();
- }
- *count_ptr = static_cast<jint>(count);
- return ERR(NONE);
-}
-
} // namespace openjdkjvmti
diff --git a/runtime/openjdkjvmti/object_tagging.h b/runtime/openjdkjvmti/object_tagging.h
index 0296f1a..b5a601c 100644
--- a/runtime/openjdkjvmti/object_tagging.h
+++ b/runtime/openjdkjvmti/object_tagging.h
@@ -1,17 +1,32 @@
-/*
- * Copyright (C) 2016 The Android Open Source Project
+/* Copyright (C) 2016 The Android Open Source Project
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
+ * This file implements interfaces from the file jvmti.h. This implementation
+ * is licensed under the same terms as the file jvmti.h. The
+ * copyright and license information for the file jvmti.h follows.
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ * Copyright (c) 2003, 2011, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation. Oracle designates this
+ * particular file as subject to the "Classpath" exception as provided
+ * by Oracle in the LICENSE file that accompanied this code.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
*/
#ifndef ART_RUNTIME_OPENJDKJVMTI_OBJECT_TAGGING_H_
@@ -20,62 +35,27 @@
#include <unordered_map>
#include "base/mutex.h"
-#include "gc/system_weak.h"
-#include "gc_root-inl.h"
#include "globals.h"
#include "jvmti.h"
+#include "jvmti_weak_table.h"
#include "mirror/object.h"
-#include "thread-inl.h"
namespace openjdkjvmti {
class EventHandler;
-class ObjectTagTable : public art::gc::SystemWeakHolder {
+class ObjectTagTable FINAL : public JvmtiWeakTable<jlong> {
public:
- explicit ObjectTagTable(EventHandler* event_handler)
- : art::gc::SystemWeakHolder(kTaggingLockLevel),
- update_since_last_sweep_(false),
- event_handler_(event_handler) {
+ explicit ObjectTagTable(EventHandler* event_handler) : event_handler_(event_handler) {
}
- void Add(art::mirror::Object* obj, jlong tag)
+ bool Set(art::mirror::Object* obj, jlong tag) OVERRIDE
REQUIRES_SHARED(art::Locks::mutator_lock_)
REQUIRES(!allow_disallow_lock_);
-
- bool Remove(art::mirror::Object* obj, jlong* tag)
- REQUIRES_SHARED(art::Locks::mutator_lock_)
- REQUIRES(!allow_disallow_lock_);
- bool RemoveLocked(art::mirror::Object* obj, jlong* tag)
+ bool SetLocked(art::mirror::Object* obj, jlong tag) OVERRIDE
REQUIRES_SHARED(art::Locks::mutator_lock_)
REQUIRES(allow_disallow_lock_);
- bool Set(art::mirror::Object* obj, jlong tag)
- REQUIRES_SHARED(art::Locks::mutator_lock_)
- REQUIRES(!allow_disallow_lock_);
- bool SetLocked(art::mirror::Object* obj, jlong tag)
- REQUIRES_SHARED(art::Locks::mutator_lock_)
- REQUIRES(allow_disallow_lock_);
-
- bool GetTag(art::mirror::Object* obj, jlong* result)
- REQUIRES_SHARED(art::Locks::mutator_lock_)
- REQUIRES(!allow_disallow_lock_) {
- art::Thread* self = art::Thread::Current();
- art::MutexLock mu(self, allow_disallow_lock_);
- Wait(self);
-
- return GetTagLocked(self, obj, result);
- }
- bool GetTagLocked(art::mirror::Object* obj, jlong* result)
- REQUIRES_SHARED(art::Locks::mutator_lock_)
- REQUIRES(allow_disallow_lock_) {
- art::Thread* self = art::Thread::Current();
- allow_disallow_lock_.AssertHeld(self);
- Wait(self);
-
- return GetTagLocked(self, obj, result);
- }
-
jlong GetTagOrZero(art::mirror::Object* obj)
REQUIRES_SHARED(art::Locks::mutator_lock_)
REQUIRES(!allow_disallow_lock_) {
@@ -91,108 +71,11 @@
return tmp;
}
- void Sweep(art::IsMarkedVisitor* visitor)
- REQUIRES_SHARED(art::Locks::mutator_lock_)
- REQUIRES(!allow_disallow_lock_);
-
- jvmtiError GetTaggedObjects(jvmtiEnv* jvmti_env,
- jint tag_count,
- const jlong* tags,
- jint* count_ptr,
- jobject** object_result_ptr,
- jlong** tag_result_ptr)
- REQUIRES_SHARED(art::Locks::mutator_lock_)
- REQUIRES(!allow_disallow_lock_);
-
- void Lock() ACQUIRE(allow_disallow_lock_);
- void Unlock() RELEASE(allow_disallow_lock_);
- void AssertLocked() ASSERT_CAPABILITY(allow_disallow_lock_);
+ protected:
+ bool DoesHandleNullOnSweep() OVERRIDE;
+ void HandleNullSweep(jlong tag) OVERRIDE;
private:
- bool SetLocked(art::Thread* self, art::mirror::Object* obj, jlong tag)
- REQUIRES_SHARED(art::Locks::mutator_lock_)
- REQUIRES(allow_disallow_lock_);
-
- bool RemoveLocked(art::Thread* self, art::mirror::Object* obj, jlong* tag)
- REQUIRES_SHARED(art::Locks::mutator_lock_)
- REQUIRES(allow_disallow_lock_);
-
- bool GetTagLocked(art::Thread* self, art::mirror::Object* obj, jlong* result)
- REQUIRES_SHARED(art::Locks::mutator_lock_)
- REQUIRES(allow_disallow_lock_) {
- auto it = tagged_objects_.find(art::GcRoot<art::mirror::Object>(obj));
- if (it != tagged_objects_.end()) {
- *result = it->second;
- return true;
- }
-
- if (art::kUseReadBarrier &&
- self != nullptr &&
- self->GetIsGcMarking() &&
- !update_since_last_sweep_) {
- return GetTagSlowPath(self, obj, result);
- }
-
- return false;
- }
-
- // Slow-path for GetTag. We didn't find the object, but we might be storing from-pointers and
- // are asked to retrieve with a to-pointer.
- bool GetTagSlowPath(art::Thread* self, art::mirror::Object* obj, jlong* result)
- REQUIRES_SHARED(art::Locks::mutator_lock_)
- REQUIRES(allow_disallow_lock_);
-
- // Update the table by doing read barriers on each element, ensuring that to-space pointers
- // are stored.
- void UpdateTableWithReadBarrier()
- REQUIRES_SHARED(art::Locks::mutator_lock_)
- REQUIRES(allow_disallow_lock_);
-
- template <bool kHandleNull>
- void SweepImpl(art::IsMarkedVisitor* visitor)
- REQUIRES_SHARED(art::Locks::mutator_lock_)
- REQUIRES(!allow_disallow_lock_);
- void HandleNullSweep(jlong tag);
-
- enum TableUpdateNullTarget {
- kIgnoreNull,
- kRemoveNull,
- kCallHandleNull
- };
-
- template <typename T, TableUpdateNullTarget kTargetNull>
- void UpdateTableWith(T& updater)
- REQUIRES_SHARED(art::Locks::mutator_lock_)
- REQUIRES(allow_disallow_lock_);
-
- struct HashGcRoot {
- size_t operator()(const art::GcRoot<art::mirror::Object>& r) const
- REQUIRES_SHARED(art::Locks::mutator_lock_) {
- return reinterpret_cast<uintptr_t>(r.Read<art::kWithoutReadBarrier>());
- }
- };
-
- struct EqGcRoot {
- bool operator()(const art::GcRoot<art::mirror::Object>& r1,
- const art::GcRoot<art::mirror::Object>& r2) const
- REQUIRES_SHARED(art::Locks::mutator_lock_) {
- return r1.Read<art::kWithoutReadBarrier>() == r2.Read<art::kWithoutReadBarrier>();
- }
- };
-
- // The tag table is used when visiting roots. So it needs to have a low lock level.
- static constexpr art::LockLevel kTaggingLockLevel =
- static_cast<art::LockLevel>(art::LockLevel::kAbortLock + 1);
-
- std::unordered_map<art::GcRoot<art::mirror::Object>,
- jlong,
- HashGcRoot,
- EqGcRoot> tagged_objects_
- GUARDED_BY(allow_disallow_lock_)
- GUARDED_BY(art::Locks::mutator_lock_);
- // To avoid repeatedly scanning the whole table, remember if we did that since the last sweep.
- bool update_since_last_sweep_;
-
EventHandler* event_handler_;
};
diff --git a/runtime/openjdkjvmti/ti_heap.cc b/runtime/openjdkjvmti/ti_heap.cc
index d52f0ea..c2495e3 100644
--- a/runtime/openjdkjvmti/ti_heap.cc
+++ b/runtime/openjdkjvmti/ti_heap.cc
@@ -25,6 +25,7 @@
#include "gc_root-inl.h"
#include "jni_env_ext.h"
#include "jni_internal.h"
+#include "jvmti_weak_table-inl.h"
#include "mirror/class.h"
#include "mirror/object-inl.h"
#include "mirror/object_array-inl.h"
@@ -41,6 +42,21 @@
namespace {
+struct IndexCache {
+ // The number of interface fields implemented by the class. This is a prefix to all assigned
+ // field indices.
+ size_t interface_fields;
+
+ // It would be nice to also cache the following, but it is complicated to wire up into the
+ // generic visit:
+ // The number of fields in interfaces and superclasses. This is the first index assigned to
+ // fields of the class.
+ // size_t superclass_fields;
+};
+using IndexCachingTable = JvmtiWeakTable<IndexCache>;
+
+static IndexCachingTable gIndexCachingTable;
+
// Report the contents of a string, if a callback is set.
jint ReportString(art::ObjPtr<art::mirror::Object> obj,
jvmtiEnv* env,
@@ -402,6 +418,12 @@
// "non-marker" interfaces (= interfaces with methods).
static size_t CountInterfaceFields(art::ObjPtr<art::mirror::Class> klass)
REQUIRES_SHARED(art::Locks::mutator_lock_) {
+ // Do we have a cached value?
+ IndexCache tmp;
+ if (gIndexCachingTable.GetTag(klass.Ptr(), &tmp)) {
+ return tmp.interface_fields;
+ }
+
size_t count = 0;
auto visitor = [&count](art::ObjPtr<art::mirror::Class> inf_klass)
REQUIRES_SHARED(art::Locks::mutator_lock_) {
@@ -410,9 +432,12 @@
count += inf_klass->NumStaticFields();
};
RecursiveInterfaceVisit<decltype(visitor)>::VisitStatic(art::Thread::Current(), klass, visitor);
- return count;
- // TODO: Implement caching.
+ // Store this into the cache.
+ tmp.interface_fields = count;
+ gIndexCachingTable.Set(klass.Ptr(), tmp);
+
+ return count;
}
UserData* user_data_;
@@ -618,6 +643,14 @@
} // namespace
+void HeapUtil::Register() {
+ art::Runtime::Current()->AddSystemWeakHolder(&gIndexCachingTable);
+}
+
+void HeapUtil::Unregister() {
+ art::Runtime::Current()->RemoveSystemWeakHolder(&gIndexCachingTable);
+}
+
struct IterateThroughHeapData {
IterateThroughHeapData(HeapUtil* _heap_util,
jvmtiEnv* _env,
@@ -1004,7 +1037,6 @@
jvmtiHeapReferenceInfo reference_info;
memset(&reference_info, 0, sizeof(reference_info));
- // TODO: Implement spec-compliant numbering.
reference_info.field.index = field_index;
jvmtiHeapReferenceKind kind =
diff --git a/runtime/openjdkjvmti/ti_heap.h b/runtime/openjdkjvmti/ti_heap.h
index 72ee097..dccecb4 100644
--- a/runtime/openjdkjvmti/ti_heap.h
+++ b/runtime/openjdkjvmti/ti_heap.h
@@ -49,6 +49,9 @@
return tags_;
}
+ static void Register();
+ static void Unregister();
+
private:
ObjectTagTable* tags_;
};
diff --git a/runtime/type_lookup_table.h b/runtime/type_lookup_table.h
index 3f6f76f..fd68deb 100644
--- a/runtime/type_lookup_table.h
+++ b/runtime/type_lookup_table.h
@@ -148,7 +148,7 @@
return mask_;
}
- // Attempt to set an entry on it's hash' slot. If there is alrady something there, return false.
+ // Attempt to set an entry on its hash's slot. If there is already something there, return false.
// Otherwise return true.
bool SetOnInitialPos(const Entry& entry, uint32_t hash);
diff --git a/runtime/vdex_file.h b/runtime/vdex_file.h
index 7daf2f8..5048bad 100644
--- a/runtime/vdex_file.h
+++ b/runtime/vdex_file.h
@@ -61,7 +61,7 @@
private:
static constexpr uint8_t kVdexMagic[] = { 'v', 'd', 'e', 'x' };
- static constexpr uint8_t kVdexVersion[] = { '0', '0', '3', '\0' }; // Remove verify-profile
+ static constexpr uint8_t kVdexVersion[] = { '0', '0', '4', '\0' }; // dexlayout incompatibility
uint8_t magic_[4];
uint8_t version_[4];
diff --git a/test/530-checker-lse/src/Main.java b/test/530-checker-lse/src/Main.java
index 14a40ef..6632503 100644
--- a/test/530-checker-lse/src/Main.java
+++ b/test/530-checker-lse/src/Main.java
@@ -747,6 +747,69 @@
return 1.0f;
}
+ /// CHECK-START: TestClass2 Main.testStoreStore() load_store_elimination (before)
+ /// CHECK: NewInstance
+ /// CHECK: InstanceFieldSet
+ /// CHECK: InstanceFieldSet
+ /// CHECK: InstanceFieldSet
+ /// CHECK: InstanceFieldSet
+
+ /// CHECK-START: TestClass2 Main.testStoreStore() load_store_elimination (after)
+ /// CHECK: NewInstance
+ /// CHECK: InstanceFieldSet
+ /// CHECK: InstanceFieldSet
+ /// CHECK-NOT: InstanceFieldSet
+
+ private static TestClass2 testStoreStore() {
+ TestClass2 obj = new TestClass2();
+ obj.i = 41;
+ obj.j = 42;
+ obj.i = 41;
+ obj.j = 43;
+ return obj;
+ }
+
+ /// CHECK-START: int Main.testStoreStoreWithDeoptimize(int[]) load_store_elimination (before)
+ /// CHECK: NewInstance
+ /// CHECK: InstanceFieldSet
+ /// CHECK: InstanceFieldSet
+ /// CHECK: InstanceFieldSet
+ /// CHECK: InstanceFieldSet
+ /// CHECK: Deoptimize
+ /// CHECK: ArraySet
+ /// CHECK: ArraySet
+ /// CHECK: ArraySet
+ /// CHECK: ArraySet
+ /// CHECK: ArrayGet
+ /// CHECK: ArrayGet
+ /// CHECK: ArrayGet
+ /// CHECK: ArrayGet
+
+ /// CHECK-START: int Main.testStoreStoreWithDeoptimize(int[]) load_store_elimination (after)
+ /// CHECK: NewInstance
+ /// CHECK: InstanceFieldSet
+ /// CHECK: InstanceFieldSet
+ /// CHECK-NOT: InstanceFieldSet
+ /// CHECK: Deoptimize
+ /// CHECK: ArraySet
+ /// CHECK: ArraySet
+ /// CHECK: ArraySet
+ /// CHECK: ArraySet
+ /// CHECK-NOT: ArrayGet
+
+ private static int testStoreStoreWithDeoptimize(int[] arr) {
+ TestClass2 obj = new TestClass2();
+ obj.i = 41;
+ obj.j = 42;
+ obj.i = 41;
+ obj.j = 43;
+ arr[0] = 1; // One HDeoptimize here.
+ arr[1] = 1;
+ arr[2] = 1;
+ arr[3] = 1;
+ return arr[0] + arr[1] + arr[2] + arr[3];
+ }
+
/// CHECK-START: double Main.getCircleArea(double, boolean) load_store_elimination (before)
/// CHECK: NewInstance
@@ -950,6 +1013,10 @@
assertIntEquals(testAllocationEliminationOfArray2(), 11);
assertIntEquals(testAllocationEliminationOfArray3(2), 4);
assertIntEquals(testAllocationEliminationOfArray4(2), 6);
+
+ assertIntEquals(testStoreStore().i, 41);
+ assertIntEquals(testStoreStore().j, 43);
+ assertIntEquals(testStoreStoreWithDeoptimize(new int[4]), 4);
}
static boolean sFlag;
diff --git a/test/570-checker-select/src/Main.java b/test/570-checker-select/src/Main.java
index e0a76ca..3ac6f89 100644
--- a/test/570-checker-select/src/Main.java
+++ b/test/570-checker-select/src/Main.java
@@ -371,6 +371,49 @@
return a > b ? x : y;
}
+ /// CHECK-START-ARM: long Main.$noinline$LongEqNonmatCond_LongVarVar(long, long, long, long) disassembly (after)
+ /// CHECK: Select
+ /// CHECK-NEXT: cmp {{r\d+}}, {{r\d+}}
+ /// CHECK-NEXT: it eq
+ /// CHECK-NEXT: cmpeq {{r\d+}}, {{r\d+}}
+ /// CHECK-NEXT: it eq
+
+ public static long $noinline$LongEqNonmatCond_LongVarVar(long a, long b, long x, long y) {
+ return a == b ? x : y;
+ }
+
+ /// CHECK-START-ARM: long Main.$noinline$LongNonmatCondCst_LongVarVar(long, long, long) disassembly (after)
+ /// CHECK: Select
+ /// CHECK-NEXT: mov ip, #52720
+ /// CHECK-NEXT: movt ip, #35243
+ /// CHECK-NEXT: cmp {{r\d+}}, ip
+ /// CHECK-NEXT: sbcs ip, {{r\d+}}, #{{\d+}}
+ /// CHECK-NEXT: it ge
+
+ public static long $noinline$LongNonmatCondCst_LongVarVar(long a, long x, long y) {
+ return a > 0x89ABCDEFL ? x : y;
+ }
+
+ /// CHECK-START-ARM: long Main.$noinline$LongNonmatCondCst_LongVarVar2(long, long, long) disassembly (after)
+ /// CHECK: Select
+ /// CHECK-NEXT: mov ip, #{{\d+}}
+ /// CHECK-NEXT: movt ip, #{{\d+}}
+ /// CHECK-NEXT: cmp {{r\d+}}, ip
+
+ public static long $noinline$LongNonmatCondCst_LongVarVar2(long a, long x, long y) {
+ return a > 0x0123456789ABCDEFL ? x : y;
+ }
+
+ /// CHECK-START-ARM: long Main.$noinline$LongNonmatCondCst_LongVarVar3(long, long, long) disassembly (after)
+ /// CHECK: Select
+ /// CHECK-NEXT: cmp {{r\d+}}, {{r\d+}}
+ /// CHECK-NOT: sbcs
+ /// CHECK-NOT: cmp
+
+ public static long $noinline$LongNonmatCondCst_LongVarVar3(long a, long x, long y) {
+ return a > 0x7FFFFFFFFFFFFFFFL ? x : y;
+ }
+
/// CHECK-START: long Main.LongMatCond_LongVarVar(long, long, long, long) register (after)
/// CHECK: <<Cond:z\d+>> LessThanOrEqual [{{j\d+}},{{j\d+}}]
/// CHECK: <<Sel1:j\d+>> Select [{{j\d+}},{{j\d+}},<<Cond>>]
@@ -612,6 +655,39 @@
assertEqual(5, IntMatCond_IntVarVar(3, 2, 5, 7));
assertEqual(8, IntMatCond_IntVarVar(2, 3, 5, 7));
+ assertEqual(0xAAAAAAAA55555555L,
+ LongNonmatCond_LongVarVar(3L, 2L, 0xAAAAAAAA55555555L, 0x8888888877777777L));
+ assertEqual(0x8888888877777777L,
+ LongNonmatCond_LongVarVar(2L, 2L, 0xAAAAAAAA55555555L, 0x8888888877777777L));
+ assertEqual(0x8888888877777777L,
+ LongNonmatCond_LongVarVar(2L, 3L, 0xAAAAAAAA55555555L, 0x8888888877777777L));
+ assertEqual(0xAAAAAAAA55555555L, LongNonmatCond_LongVarVar(0x0000000100000000L,
+ 0x00000000FFFFFFFFL,
+ 0xAAAAAAAA55555555L,
+ 0x8888888877777777L));
+ assertEqual(0x8888888877777777L, LongNonmatCond_LongVarVar(0x00000000FFFFFFFFL,
+ 0x0000000100000000L,
+ 0xAAAAAAAA55555555L,
+ 0x8888888877777777L));
+
+ assertEqual(0x8888888877777777L, $noinline$LongEqNonmatCond_LongVarVar(2L,
+ 3L,
+ 0xAAAAAAAA55555555L,
+ 0x8888888877777777L));
+ assertEqual(0xAAAAAAAA55555555L, $noinline$LongEqNonmatCond_LongVarVar(2L,
+ 2L,
+ 0xAAAAAAAA55555555L,
+ 0x8888888877777777L));
+ assertEqual(0x8888888877777777L, $noinline$LongEqNonmatCond_LongVarVar(0x10000000000L,
+ 0L,
+ 0xAAAAAAAA55555555L,
+ 0x8888888877777777L));
+
+ assertEqual(5L, $noinline$LongNonmatCondCst_LongVarVar2(0x7FFFFFFFFFFFFFFFL, 5L, 7L));
+ assertEqual(7L, $noinline$LongNonmatCondCst_LongVarVar2(2L, 5L, 7L));
+
+ assertEqual(7L, $noinline$LongNonmatCondCst_LongVarVar3(2L, 5L, 7L));
+
assertEqual(5, FloatLtNonmatCond_IntVarVar(3, 2, 5, 7));
assertEqual(7, FloatLtNonmatCond_IntVarVar(2, 3, 5, 7));
assertEqual(7, FloatLtNonmatCond_IntVarVar(Float.NaN, 2, 5, 7));
diff --git a/test/626-checker-arm64-scratch-register/src/Main.java b/test/626-checker-arm64-scratch-register/src/Main.java
index 6dd4374..1394917 100644
--- a/test/626-checker-arm64-scratch-register/src/Main.java
+++ b/test/626-checker-arm64-scratch-register/src/Main.java
@@ -70,7 +70,7 @@
/// CHECK: end_block
/// CHECK: begin_block
/// CHECK: name "<<ElseBlock>>"
- /// CHECK: ParallelMove moves:[#100->d17,32(sp)->d1,36(sp)->d2,d17->d3,d3->d4,d4->d5,d5->d6,d6->d7,d7->d18,d18->d19,d19->d20,d20->d21,d21->d22,d22->d23,d23->d10,d10->d11,d11->d12,24(sp)->d13,28(sp)->d14,d14->16(sp),d12->20(sp),d13->24(sp),d1->28(sp),d2->32(sp),16(sp)->36(sp),20(sp)->40(sp)]
+ /// CHECK: ParallelMove moves:[40(sp)->d0,24(sp)->32(sp),28(sp)->36(sp),d0->d3,d3->d4,d2->d5,d4->d6,d5->d7,d6->d18,d7->d19,d18->d20,d19->d21,d20->d22,d21->d23,d22->d10,d23->d11,16(sp)->24(sp),20(sp)->28(sp),d10->d14,d11->d12,d12->d13,d13->d1,d14->d2,32(sp)->16(sp),36(sp)->20(sp)]
/// CHECK: end_block
/// CHECK-START-ARM64: void Main.test() disassembly (after)
@@ -85,7 +85,7 @@
/// CHECK: end_block
/// CHECK: begin_block
/// CHECK: name "<<ElseBlock>>"
- /// CHECK: ParallelMove moves:[invalid->invalid,invalid->invalid,invalid->invalid,invalid->invalid,invalid->invalid,invalid->invalid,invalid->invalid,invalid->invalid,invalid->invalid,invalid->invalid,invalid->invalid,invalid->invalid,invalid->invalid,invalid->invalid,invalid->invalid,invalid->invalid,invalid->invalid,invalid->invalid,invalid->invalid,invalid->invalid,invalid->invalid,invalid->invalid,invalid->invalid,invalid->invalid,invalid->invalid,invalid->invalid]
+ /// CHECK: ParallelMove moves:[invalid->invalid,invalid->invalid,invalid->invalid,invalid->invalid,invalid->invalid,invalid->invalid,invalid->invalid,invalid->invalid,invalid->invalid,invalid->invalid,invalid->invalid,invalid->invalid,invalid->invalid,invalid->invalid,invalid->invalid,invalid->invalid,invalid->invalid,invalid->invalid,invalid->invalid,invalid->invalid,invalid->invalid,invalid->invalid,invalid->invalid,invalid->invalid,invalid->invalid]
/// CHECK: fmov d31, d2
/// CHECK: ldr s2, [sp, #36]
/// CHECK: ldr w16, [sp, #16]
@@ -111,11 +111,10 @@
/// CHECK: fmov d6, d5
/// CHECK: fmov d5, d4
/// CHECK: fmov d4, d3
- /// CHECK: fmov d3, d17
- /// CHECK: fmov d17, d13
+ /// CHECK: fmov d3, d13
/// CHECK: ldr s13, [sp, #24]
- /// CHECK: str s17, [sp, #24]
- /// CHECK: ldr s17, pc+{{\d+}} (addr {{0x[0-9a-f]+}}) (100)
+ /// CHECK: str s3, [sp, #24]
+ /// CHECK: ldr s3, pc+{{\d+}} (addr {{0x[0-9a-f]+}}) (100)
/// CHECK: end_block
public void test() {
diff --git a/test/638-checker-inline-caches/expected.txt b/test/638-checker-inline-caches/expected.txt
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/test/638-checker-inline-caches/expected.txt
diff --git a/test/638-checker-inline-caches/info.txt b/test/638-checker-inline-caches/info.txt
new file mode 100644
index 0000000..1fac628
--- /dev/null
+++ b/test/638-checker-inline-caches/info.txt
@@ -0,0 +1 @@
+Verify the use of inline caches in AOT mode.
diff --git a/test/638-checker-inline-caches/multidex.jpp b/test/638-checker-inline-caches/multidex.jpp
new file mode 100644
index 0000000..69a2cc1
--- /dev/null
+++ b/test/638-checker-inline-caches/multidex.jpp
@@ -0,0 +1,12 @@
+Main:
+ @@com.android.jack.annotations.ForceInMainDex
+ class Main
+Super:
+ @@com.android.jack.annotations.ForceInMainDex
+ class Super
+SubA:
+ @@com.android.jack.annotations.ForceInMainDex
+ class SubA
+SubB
+ @@com.android.jack.annotations.ForceInMainDex
+ class SubB
diff --git a/test/638-checker-inline-caches/profile b/test/638-checker-inline-caches/profile
new file mode 100644
index 0000000..1ca6d7b
--- /dev/null
+++ b/test/638-checker-inline-caches/profile
@@ -0,0 +1,6 @@
+LMain;->inlineMonomorphicSubA(LSuper;)I+LSubA;
+LMain;->inlinePolymophicSubASubB(LSuper;)I+LSubA;,LSubB;
+LMain;->inlinePolymophicCrossDexSubASubC(LSuper;)I+LSubA;,LSubC;
+LMain;->inlineMegamorphic(LSuper;)I+LSubA;,LSubB;,LSubC;,LSubD;,LSubE;
+LMain;->inlineMissingTypes(LSuper;)I+missing_types
+LMain;->noInlineCache(LSuper;)I
diff --git a/test/638-checker-inline-caches/run b/test/638-checker-inline-caches/run
new file mode 100644
index 0000000..146e180
--- /dev/null
+++ b/test/638-checker-inline-caches/run
@@ -0,0 +1,17 @@
+#!/bin/bash
+#
+# Copyright (C) 2017 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+exec ${RUN} $@ --profile -Xcompiler-option --compiler-filter=speed-profile
diff --git a/test/638-checker-inline-caches/src-multidex/SubC.java b/test/638-checker-inline-caches/src-multidex/SubC.java
new file mode 100644
index 0000000..f7e3c08
--- /dev/null
+++ b/test/638-checker-inline-caches/src-multidex/SubC.java
@@ -0,0 +1,19 @@
+/*
+ * Copyright (C) 2017 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+public class SubC extends Super {
+ public int getValue() { return 24; }
+}
diff --git a/test/638-checker-inline-caches/src/Main.java b/test/638-checker-inline-caches/src/Main.java
new file mode 100644
index 0000000..2cee47e
--- /dev/null
+++ b/test/638-checker-inline-caches/src/Main.java
@@ -0,0 +1,192 @@
+/*
+ * Copyright (C) 2017 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+class SubA extends Super {
+ int getValue() { return 42; }
+}
+
+class SubB extends Super {
+ int getValue() { return 38; }
+}
+
+class SubD extends Super {
+ int getValue() { return 10; }
+}
+
+class SubE extends Super {
+ int getValue() { return -4; }
+}
+
+public class Main {
+
+ /// CHECK-START: int Main.inlineMonomorphicSubA(Super) inliner (before)
+ /// CHECK: InvokeVirtual method_name:Super.getValue
+
+ /// CHECK-START: int Main.inlineMonomorphicSubA(Super) inliner (after)
+ /// CHECK-NOT: InvokeVirtual method_name:Super.getValue
+
+ /// CHECK-START: int Main.inlineMonomorphicSubA(Super) inliner (after)
+ /// CHECK: <<SubARet:i\d+>> IntConstant 42
+ /// CHECK: <<ObjClass:l\d+>> InstanceFieldGet field_name:java.lang.Object.shadow$_klass_
+ /// CHECK: <<InlineClass:l\d+>> LoadClass class_name:SubA
+ /// CHECK: <<Test:z\d+>> NotEqual [<<InlineClass>>,<<ObjClass>>]
+ /// CHECK: Deoptimize [<<Test>>]
+ /// CHECK: Return [<<SubARet>>]
+ public static int inlineMonomorphicSubA(Super a) {
+ return a.getValue();
+ }
+
+ /// CHECK-START: int Main.inlinePolymophicSubASubB(Super) inliner (before)
+ /// CHECK: InvokeVirtual method_name:Super.getValue
+
+ /// CHECK-START: int Main.inlinePolymophicSubASubB(Super) inliner (after)
+ /// CHECK-NOT: InvokeVirtual method_name:Super.getValue
+
+ // Note that the order in which the types are added to the inline cache in the profile matters.
+
+ /// CHECK-START: int Main.inlinePolymophicSubASubB(Super) inliner (after)
+ /// CHECK-DAG: <<SubARet:i\d+>> IntConstant 42
+ /// CHECK-DAG: <<SubBRet:i\d+>> IntConstant 38
+ /// CHECK: <<ObjClassSubA:l\d+>> InstanceFieldGet field_name:java.lang.Object.shadow$_klass_
+ /// CHECK: <<InlineClassSubA:l\d+>> LoadClass class_name:SubA
+ /// CHECK: <<TestSubA:z\d+>> NotEqual [<<InlineClassSubA>>,<<ObjClassSubA>>]
+ /// CHECK: If [<<TestSubA>>]
+
+ /// CHECK: <<ObjClassSubB:l\d+>> InstanceFieldGet field_name:java.lang.Object.shadow$_klass_
+ /// CHECK: <<InlineClassSubB:l\d+>> LoadClass class_name:SubB
+ /// CHECK: <<TestSubB:z\d+>> NotEqual [<<InlineClassSubB>>,<<ObjClassSubB>>]
+ /// CHECK: Deoptimize [<<TestSubB>>]
+
+ /// CHECK: <<Ret:i\d+>> Phi [<<SubARet>>,<<SubBRet>>]
+ /// CHECK: Return [<<Ret>>]
+ public static int inlinePolymophicSubASubB(Super a) {
+ return a.getValue();
+ }
+
+ /// CHECK-START: int Main.inlinePolymophicCrossDexSubASubC(Super) inliner (before)
+ /// CHECK: InvokeVirtual method_name:Super.getValue
+
+ /// CHECK-START: int Main.inlinePolymophicCrossDexSubASubC(Super) inliner (after)
+ /// CHECK-NOT: InvokeVirtual method_name:Super.getValue
+
+ // Note that the order in which the types are added to the inline cache in the profile matters.
+
+ /// CHECK-START: int Main.inlinePolymophicCrossDexSubASubC(Super) inliner (after)
+ /// CHECK-DAG: <<SubARet:i\d+>> IntConstant 42
+ /// CHECK-DAG: <<SubCRet:i\d+>> IntConstant 24
+ /// CHECK: <<ObjClassSubA:l\d+>> InstanceFieldGet field_name:java.lang.Object.shadow$_klass_
+ /// CHECK: <<InlineClassSubA:l\d+>> LoadClass class_name:SubA
+ /// CHECK: <<TestSubA:z\d+>> NotEqual [<<InlineClassSubA>>,<<ObjClassSubA>>]
+ /// CHECK: If [<<TestSubA>>]
+
+ /// CHECK: <<ObjClassSubC:l\d+>> InstanceFieldGet field_name:java.lang.Object.shadow$_klass_
+ /// CHECK: <<InlineClassSubC:l\d+>> LoadClass class_name:SubC
+ /// CHECK: <<TestSubC:z\d+>> NotEqual [<<InlineClassSubC>>,<<ObjClassSubC>>]
+ /// CHECK: Deoptimize [<<TestSubC>>]
+
+ /// CHECK: <<Ret:i\d+>> Phi [<<SubARet>>,<<SubCRet>>]
+ /// CHECK: Return [<<Ret>>]
+ public static int inlinePolymophicCrossDexSubASubC(Super a) {
+ return a.getValue();
+ }
+
+ /// CHECK-START: int Main.inlineMegamorphic(Super) inliner (before)
+ /// CHECK: InvokeVirtual method_name:Super.getValue
+
+ /// CHECK-START: int Main.inlineMegamorphic(Super) inliner (after)
+ /// CHECK: InvokeVirtual method_name:Super.getValue
+ public static int inlineMegamorphic(Super a) {
+ return a.getValue();
+ }
+
+ /// CHECK-START: int Main.inlineMissingTypes(Super) inliner (before)
+ /// CHECK: InvokeVirtual method_name:Super.getValue
+
+ /// CHECK-START: int Main.inlineMissingTypes(Super) inliner (after)
+ /// CHECK: InvokeVirtual method_name:Super.getValue
+ public static int inlineMissingTypes(Super a) {
+ return a.getValue();
+ }
+
+ /// CHECK-START: int Main.noInlineCache(Super) inliner (before)
+ /// CHECK: InvokeVirtual method_name:Super.getValue
+
+ /// CHECK-START: int Main.noInlineCache(Super) inliner (after)
+ /// CHECK: InvokeVirtual method_name:Super.getValue
+ public static int noInlineCache(Super a) {
+ return a.getValue();
+ }
+
+ public static void testInlineMonomorphic() {
+ if (inlineMonomorphicSubA(new SubA()) != 42) {
+ throw new Error("Expected 42");
+ }
+
+ // Call with a different type than the one from the inline cache.
+ if (inlineMonomorphicSubA(new SubB()) != 38) {
+ throw new Error("Expected 38");
+ }
+ }
+
+ public static void testInlinePolymorhic() {
+ if (inlinePolymophicSubASubB(new SubA()) != 42) {
+ throw new Error("Expected 42");
+ }
+
+ if (inlinePolymophicSubASubB(new SubB()) != 38) {
+ throw new Error("Expected 38");
+ }
+
+ // Call with a different type than the one from the inline cache.
+ if (inlinePolymophicSubASubB(new SubC()) != 24) {
+ throw new Error("Expected 25");
+ }
+
+ if (inlinePolymophicCrossDexSubASubC(new SubA()) != 42) {
+ throw new Error("Expected 42");
+ }
+
+ if (inlinePolymophicCrossDexSubASubC(new SubC()) != 24) {
+ throw new Error("Expected 24");
+ }
+
+ // Call with a different type than the one from the inline cache.
+ if (inlinePolymophicCrossDexSubASubC(new SubB()) != 38) {
+ throw new Error("Expected 38");
+ }
+ }
+
+ public static void testInlineMegamorphic() {
+ if (inlineMegamorphic(new SubA()) != 42) {
+ throw new Error("Expected 42");
+ }
+ }
+
+
+ public static void testNoInlineCache() {
+ if (noInlineCache(new SubA()) != 42) {
+ throw new Error("Expected 42");
+ }
+ }
+
+ public static void main(String[] args) {
+ testInlineMonomorphic();
+ testInlinePolymorhic();
+ testInlineMegamorphic();
+ testNoInlineCache();
+ }
+
+}
diff --git a/test/638-checker-inline-caches/src/Super.java b/test/638-checker-inline-caches/src/Super.java
new file mode 100644
index 0000000..30cdf30
--- /dev/null
+++ b/test/638-checker-inline-caches/src/Super.java
@@ -0,0 +1,19 @@
+/*
+ * Copyright (C) 2017 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+public abstract class Super {
+ abstract int getValue();
+}
diff --git a/test/641-irreducible-inline/expected.txt b/test/641-irreducible-inline/expected.txt
new file mode 100644
index 0000000..d81cc07
--- /dev/null
+++ b/test/641-irreducible-inline/expected.txt
@@ -0,0 +1 @@
+42
diff --git a/test/641-irreducible-inline/info.txt b/test/641-irreducible-inline/info.txt
new file mode 100644
index 0000000..ec6d0d2
--- /dev/null
+++ b/test/641-irreducible-inline/info.txt
@@ -0,0 +1,2 @@
+Regression test for optimizing in the presence of
+inlining a method that throws in an irreducible loop
diff --git a/test/641-irreducible-inline/smali/IrreducibleLoop.smali b/test/641-irreducible-inline/smali/IrreducibleLoop.smali
new file mode 100644
index 0000000..3e6c1f1
--- /dev/null
+++ b/test/641-irreducible-inline/smali/IrreducibleLoop.smali
@@ -0,0 +1,54 @@
+# Copyright (C) 2017 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+.class public LIrreducibleLoop;
+
+.super Ljava/lang/Object;
+
+.method public static simpleLoop(I)I
+ .registers 3
+ const/16 v0, 42
+ if-eqz p0, :loop_entry
+ goto :other_loop_pre_entry
+
+ # The then part: beginning of the irreducible loop.
+ :loop_entry
+ if-nez p0, :exit
+ invoke-static {v0},LIrreducibleLoop;->foo(I)V
+ :other_loop_entry
+ goto :loop_entry
+
+ # The else part.
+ :other_loop_pre_entry
+ if-eqz p0, :other_loop_entry
+ invoke-static {v0},LIrreducibleLoop;->foo(I)V
+ goto :other_loop_entry
+
+ :exit
+ return v0
+.end method
+
+.method public static foo(I)V
+ .registers 3
+ const/16 v0, 0
+ sget-boolean v1,LIrreducibleLoop;->doThrow:Z
+ if-eqz v1, :exit
+ # Inlining a method that throws requires re-computing loop information
+ # which is unsupported when the caller has an irreducible loop.
+ throw v0
+ :exit
+ return-void
+.end method
+
+.field public static doThrow:Z
diff --git a/test/641-irreducible-inline/src/Main.java b/test/641-irreducible-inline/src/Main.java
new file mode 100644
index 0000000..53244f7
--- /dev/null
+++ b/test/641-irreducible-inline/src/Main.java
@@ -0,0 +1,29 @@
+/*
+ * Copyright (C) 2017 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.lang.reflect.Method;
+
+public class Main {
+ // Workaround for b/18051191.
+ class InnerClass {}
+
+ public static void main(String[] args) throws Exception {
+ Class<?> c = Class.forName("IrreducibleLoop");
+ Method m = c.getMethod("simpleLoop", int.class);
+ Object[] arguments = { 42 };
+ System.out.println(m.invoke(null, arguments));
+ }
+}
diff --git a/test/924-threads/src/Main.java b/test/924-threads/src/Main.java
index f18d70e..716f59e 100644
--- a/test/924-threads/src/Main.java
+++ b/test/924-threads/src/Main.java
@@ -135,8 +135,12 @@
synchronized(cdl3_2) {
cdl3_1.countDown();
cdl3_2.await();
- Thread.yield();
- Thread.sleep(100);
+ // While the latch improves the chances to make good progress, scheduling might still be
+ // messy. Wait till we get the right Java-side Thread state.
+ do {
+ Thread.yield();
+ } while (t.getState() != Thread.State.BLOCKED);
+ Thread.sleep(10);
printThreadState(t);
}
diff --git a/test/etc/run-test-jar b/test/etc/run-test-jar
index 1473149..de16256 100755
--- a/test/etc/run-test-jar
+++ b/test/etc/run-test-jar
@@ -63,6 +63,7 @@
TEST_IS_NDEBUG="n"
APP_IMAGE="y"
VDEX_FILTER=""
+PROFILE="n"
# if "y", run 'sync' before dalvikvm to make sure all files from
# build step (e.g. dex2oat) were finished writing.
@@ -269,6 +270,9 @@
elif [ "x$1" = "x--sync" ]; then
SYNC_BEFORE_RUN="y"
shift
+ elif [ "x$1" = "x--profile" ]; then
+ PROFILE="y"
+ shift
elif expr "x$1" : "x--" >/dev/null 2>&1; then
echo "unknown $0 option: $1" 1>&2
exit 1
@@ -441,8 +445,8 @@
JNI_OPTS="-Xjnigreflimit:512 -Xcheck:jni"
if [ "$RELOCATE" = "y" ]; then
- COMPILE_FLAGS="${COMPILE_FLAGS} --include-patch-information --runtime-arg -Xnorelocate"
- FLAGS="${FLAGS} -Xrelocate -Xcompiler-option --include-patch-information"
+ COMPILE_FLAGS="${COMPILE_FLAGS} --runtime-arg -Xnorelocate"
+ FLAGS="${FLAGS} -Xrelocate"
if [ "$HOST" = "y" ]; then
# Run test sets a fairly draconian ulimit that we will likely blow right over
# since we are relocating. Get the total size of the /system/framework directory
@@ -511,6 +515,7 @@
exit 1
fi
+profman_cmdline="true"
dex2oat_cmdline="true"
vdex_cmdline="true"
mkdir_locations="${DEX_LOCATION}/dalvik-cache/$ISA"
@@ -536,6 +541,15 @@
dex2oat_cmdline="${dex2oat_cmdline} --instruction-set-features=${INSTRUCTION_SET_FEATURES}"
fi
+ if [ "$PROFILE" = "y" ]; then
+ profman_cmdline="${ANDROID_ROOT}/bin/profman \
+ --apk=$DEX_LOCATION/$TEST_NAME.jar \
+ --dex-location=$DEX_LOCATION/$TEST_NAME.jar \
+ --create-profile-from=$DEX_LOCATION/profile \
+ --reference-profile-file=$TEST_NAME.prof"
+ dex2oat_cmdline="${dex2oat_cmdline} --profile-file=$TEST_NAME.prof"
+ fi
+
# Add in a timeout. This is important for testing the compilation/verification time of
# pathological cases.
# Note: as we don't know how decent targets are (e.g., emulator), only do this on the host for
@@ -594,6 +608,7 @@
dex2oat_cmdline=$(echo $dex2oat_cmdline)
dalvikvm_cmdline=$(echo $dalvikvm_cmdline)
vdex_cmdline=$(echo $vdex_cmdline)
+profman_cmdline=$(echo $profman_cmdline)
if [ "$HOST" = "n" ]; then
adb root > /dev/null
@@ -603,11 +618,18 @@
adb shell mkdir -p $DEX_LOCATION
adb push $TEST_NAME.jar $DEX_LOCATION
adb push $TEST_NAME-ex.jar $DEX_LOCATION
+ if [ "$PROFILE" = "y" ]; then
+ adb push profile $DEX_LOCATION
+ fi
else
adb shell rm -r $DEX_LOCATION >/dev/null 2>&1
adb shell mkdir -p $DEX_LOCATION >/dev/null 2>&1
adb push $TEST_NAME.jar $DEX_LOCATION >/dev/null 2>&1
adb push $TEST_NAME-ex.jar $DEX_LOCATION >/dev/null 2>&1
+ if [ "$PROFILE" = "y" ]; then
+ adb push profile $DEX_LOCATION >/dev/null 2>&1
+ fi
+
fi
LD_LIBRARY_PATH=/data/$TEST_DIRECTORY/art/$ISA
@@ -634,6 +656,7 @@
mkdir -p ${mkdir_locations} && \
export LD_LIBRARY_PATH=$LD_LIBRARY_PATH && \
export PATH=$ANDROID_ROOT/bin:$PATH && \
+ $profman_cmdline && \
$dex2oat_cmdline && \
$vdex_cmdline && \
$strip_cmdline && \
@@ -710,13 +733,14 @@
fi
if [ "$DEV_MODE" = "y" ]; then
- echo "mkdir -p ${mkdir_locations} && $dex2oat_cmdline && $vdex_cmdline && $strip_cmdline && $sync_cmdline && $cmdline"
+ echo "mkdir -p ${mkdir_locations} && $profman_cmdline && $dex2oat_cmdline && $vdex_cmdline && $strip_cmdline && $sync_cmdline && $cmdline"
fi
cd $ANDROID_BUILD_TOP
rm -rf ${DEX_LOCATION}/dalvik-cache/
mkdir -p ${mkdir_locations} || exit 1
+ $profman_cmdline || { echo "Profman failed." >&2 ; exit 2; }
$dex2oat_cmdline || { echo "Dex2oat failed." >&2 ; exit 2; }
$vdex_cmdline || { echo "Dex2oat failed." >&2 ; exit 2; }
$strip_cmdline || { echo "Strip failed." >&2 ; exit 3; }
diff --git a/test/testrunner/run_build_test_target.py b/test/testrunner/run_build_test_target.py
new file mode 100755
index 0000000..0cd1dde
--- /dev/null
+++ b/test/testrunner/run_build_test_target.py
@@ -0,0 +1,58 @@
+#!/usr/bin/env python
+#
+# Copyright 2017, The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import argparse
+import os
+import subprocess
+
+from target_config import target_config
+import env
+
+parser = argparse.ArgumentParser()
+parser.add_argument('--build-target', required=True, dest='build_target')
+parser.add_argument('-j', default='1', dest='n_threads')
+options = parser.parse_args()
+
+target = target_config[options.build_target]
+n_threads = options.n_threads
+custom_env = target.get('env', {})
+custom_env['SOONG_ALLOW_MISSING_DEPENDENCIES'] = 'true'
+print custom_env
+os.environ.update(custom_env)
+
+
+if target.get('target'):
+ build_command = 'make'
+ build_command += ' -j' + str(n_threads)
+ build_command += ' -C ' + env.ANDROID_BUILD_TOP
+ build_command += ' ' + target.get('target')
+ print build_command.split()
+ if subprocess.call(build_command.split()):
+ sys.exit(1)
+
+else:
+ run_test_command = [os.path.join(env.ANDROID_BUILD_TOP,
+ 'art/test/testrunner/testrunner.py')]
+ run_test_command += target.get('flags', [])
+ run_test_command += ['-j', str(n_threads)]
+ run_test_command += ['-b']
+ run_test_command += ['--verbose']
+
+ print run_test_command
+ if subprocess.call(run_test_command):
+ sys.exit(1)
+
+sys.exit(0)
diff --git a/test/testrunner/target_config.py b/test/testrunner/target_config.py
new file mode 100644
index 0000000..5387d6a
--- /dev/null
+++ b/test/testrunner/target_config.py
@@ -0,0 +1,257 @@
+target_config = {
+ 'art-test' : {
+ 'flags' : [],
+ 'env' : {
+ 'ART_USE_READ_BARRIER' : 'false'
+ }
+ },
+ 'art-interpreter' : {
+ 'flags' : ['--interpreter'],
+ 'env' : {
+ 'ART_USE_READ_BARRIER' : 'false'
+ }
+ },
+ 'art-interpreter-access-checks' : {
+ 'flags' : ['--interp-ac'],
+ 'env' : {
+ 'ART_USE_READ_BARRIER' : 'false'
+ }
+ },
+ 'art-jit' : {
+ 'flags' : ['--jit'],
+ 'env' : {
+ 'ART_USE_READ_BARRIER' : 'false'
+ }
+ },
+ 'art-gcstress-gcverify': {
+ 'flags' : ['--gcstress',
+ '--gcverify'],
+ 'env' : {
+ 'ART_USE_READ_BARRIER' : 'false',
+ 'ART_DEFAULT_GC_TYPE' : 'SS'
+ }
+ },
+ 'art-interpreter-gcstress' : {
+ 'flags': ['--interpreter',
+ '--gcstress'],
+ 'env' : {
+ 'ART_USE_READ_BARRIER' : 'false',
+ 'ART_DEFAULT_GC_TYPE' : 'SS'
+ }
+ },
+ 'art-optimizing-gcstress' : {
+ 'flags': ['--gcstress',
+ '--optimizing'],
+ 'env' : {
+ 'ART_USE_READ_BARRIER' : 'false',
+ 'ART_DEFAULT_GC_TYPE' : 'SS'
+ }
+ },
+ 'art-jit-gcstress' : {
+ 'flags': ['--jit',
+ '--gcstress'],
+ 'env' : {
+ 'ART_USE_READ_BARRIER' : 'false'
+ }
+ },
+ 'art-read-barrier' : {
+ 'flags': ['--interpreter',
+ '--optimizing'],
+ 'env' : {
+ 'ART_USE_READ_BARRIER' : 'true',
+ 'ART_HEAP_POISONING' : 'true'
+ }
+ },
+ 'art-read-barrier-gcstress' : {
+ 'flags' : ['--interpreter',
+ '--optimizing',
+ '--gcstress'],
+ 'env' : {
+ 'ART_USE_READ_BARRIER' : 'true',
+ 'ART_HEAP_POISONING' : 'true'
+ }
+ },
+ 'art-read-barrier-table-lookup' : {
+ 'flags' : ['--interpreter',
+ '--optimizing'],
+ 'env' : {
+ 'ART_USE_READ_BARRIER' : 'true',
+ 'ART_READ_BARRIER_TYPE' : 'TABLELOOKUP',
+ 'ART_HEAP_POISONING' : 'true'
+ }
+ },
+ 'art-debug-gc' : {
+ 'flags' : ['--interpreter',
+ '--optimizing'],
+ 'env' : {
+ 'ART_TEST_DEBUG_GC' : 'true',
+ 'ART_USE_READ_BARRIER' : 'false'
+ }
+ },
+ 'art-ss-gc' : {
+ 'flags' : ['--interpreter',
+ '--optimizing',
+ '--jit'],
+ 'env' : {
+ 'ART_DEFAULT_GC_TYPE' : 'SS',
+ 'ART_USE_READ_BARRIER' : 'false'
+ }
+ },
+ 'art-gss-gc' : {
+ 'flags' : ['--interpreter',
+ '--optimizing',
+ '--jit'],
+ 'env' : {
+ 'ART_DEFAULT_GC_TYPE' : 'GSS',
+ 'ART_USE_READ_BARRIER' : 'false'
+ }
+ },
+ 'art-ss-gc-tlab' : {
+ 'flags' : ['--interpreter',
+ '--optimizing',
+ '--jit'],
+ 'env' : {
+ 'ART_DEFAULT_GC_TYPE' : 'SS',
+ 'ART_USE_TLAB' : 'true',
+ 'ART_USE_READ_BARRIER' : 'false'
+ }
+ },
+ 'art-gss-gc-tlab' : {
+ 'flags' : ['--interpreter',
+ '--optimizing',
+ '--jit'],
+ 'env' : {
+ 'ART_DEFAULT_GC_TYPE' : 'GSS',
+ 'ART_USE_TLAB' : 'true',
+ 'ART_USE_READ_BARRIER' : 'false'
+ }
+ },
+ 'art-tracing' : {
+ 'flags' : ['--trace'],
+ 'env' : {
+ 'ART_USE_READ_BARRIER' : 'false'
+ }
+ },
+ 'art-interpreter-tracing' : {
+ 'flags' : ['--interpreter',
+ '--trace'],
+ 'env' : {
+ 'ART_USE_READ_BARRIER' : 'false',
+ }
+ },
+ 'art-forcecopy' : {
+ 'flags' : ['--forcecopy'],
+ 'env' : {
+ 'ART_USE_READ_BARRIER' : 'false',
+ }
+ },
+ 'art-no-prebuild' : {
+ 'flags' : ['--no-prebuild'],
+ 'env' : {
+ 'ART_USE_READ_BARRIER' : 'false',
+ }
+ },
+ 'art-no-image' : {
+ 'flags' : ['--no-image'],
+ 'env' : {
+ 'ART_USE_READ_BARRIER' : 'false',
+ }
+ },
+ 'art-interpreter-no-image' : {
+ 'flags' : ['--interpreter',
+ '--no-image'],
+ 'env' : {
+ 'ART_USE_READ_BARRIER' : 'false',
+ }
+ },
+ 'art-relocate-no-patchoat' : {
+ 'flags' : ['--relocate-npatchoat'],
+ 'env' : {
+ 'ART_USE_READ_BARRIER' : 'false',
+ }
+ },
+ 'art-no-dex2oat' : {
+ 'flags' : ['--no-dex2oat'],
+ 'env' : {
+ 'ART_USE_READ_BARRIER' : 'false',
+ }
+ },
+ 'art-heap-poisoning' : {
+ 'flags' : ['--interpreter',
+ '--optimizing'],
+ 'env' : {
+ 'ART_USE_READ_BARRIER' : 'false',
+ 'ART_HEAP_POISONING' : 'true'
+ }
+ },
+ 'art-gtest' : {
+ 'target' : 'test-art-gtest',
+ 'env' : {
+ 'ART_USE_READ_BARRIER' : 'true'
+ }
+ },
+ 'art-gtest-read-barrier': {
+ 'target' : 'test-art-gtest',
+ 'env' : {
+ 'ART_USE_READ_BARRIER' : 'true',
+ 'ART_HEAP_POISONING' : 'true'
+ }
+ },
+ 'art-gtest-read-barrier-table-lookup': {
+ 'target' : 'test-art-gtest',
+ 'env': {
+ 'ART_USE_READ_BARRIER' : 'true',
+ 'ART_READ_BARRIER_TYPE' : 'TABLELOOKUP',
+ 'ART_HEAP_POISONING' : 'true'
+ }
+ },
+ 'art-gtest-ss-gc': {
+ 'target' : 'test-art-gtest',
+ 'env': {
+ 'ART_DEFAULT_GC_TYPE' : 'SS',
+ 'ART_USE_READ_BARRIER' : 'false'
+ }
+ },
+ 'art-gtest-gss-gc': {
+ 'target' : 'test-art-gtest',
+ 'env' : {
+ 'ART_DEFAULT_GC_TYPE' : 'GSS',
+ 'ART_USE_READ_BARRIER' : 'false'
+ }
+ },
+ 'art-gtest-ss-gc-tlab': {
+ 'target' : 'test-art-gtest',
+ 'env': {
+ 'ART_DEFAULT_GC_TYPE' : 'SS',
+ 'ART_USE_TLAB' : 'true',
+ 'ART_USE_READ_BARRIER' : 'false',
+ }
+ },
+ 'art-gtest-gss-gc-tlab': {
+ 'target' : 'test-art-gtest',
+ 'env': {
+ 'ART_DEFAULT_GC_TYPE' : 'GSS',
+ 'ART_USE_TLAB' : 'true',
+ 'ART_USE_READ_BARRIER' : 'false'
+ }
+ },
+ 'art-gtest-valgrind32': {
+ 'target' : 'valgrind-test-art-host32',
+ 'env': {
+ 'ART_USE_READ_BARRIER' : 'false'
+ }
+ },
+ 'art-gtest-valgrind64': {
+ 'target' : 'valgrind-test-art-host64',
+ 'env': {
+ 'ART_USE_READ_BARRIER' : 'false'
+ }
+ },
+ 'art-gtest-heap-poisoning': {
+ 'target' : 'valgrind-test-art-host64',
+ 'env' : {
+ 'ART_HEAP_POISONING' : 'true',
+ 'ART_USE_READ_BARRIER' : 'false'
+ }
+ }
+}
diff --git a/test/testrunner/testrunner.py b/test/testrunner/testrunner.py
index 9c8d3b8..f77e9ad 100755
--- a/test/testrunner/testrunner.py
+++ b/test/testrunner/testrunner.py
@@ -56,6 +56,7 @@
import time
import env
+from target_config import target_config
TARGET_TYPES = set()
RUN_TYPES = set()
@@ -704,6 +705,25 @@
return {match.group(12)}
raise ValueError(test_name + " is not a valid test")
+
+def setup_env_for_build_target(build_target, parser, options):
+ """Setup environment for the build target
+
+ The method setup environment for the master-art-host targets.
+ """
+ os.environ.update(build_target['env'])
+ os.environ['SOONG_ALLOW_MISSING_DEPENDENCIES'] = 'true'
+ print_text('%s\n' % (str(os.environ)))
+
+ target_options = vars(parser.parse_args(build_target['flags']))
+ target_options['host'] = True
+ target_options['verbose'] = True
+ target_options['build'] = True
+ target_options['n_thread'] = options['n_thread']
+ target_options['dry_run'] = options['dry_run']
+
+ return target_options
+
def parse_option():
global verbose
global dry_run
@@ -733,90 +753,95 @@
action='store_true', dest='build',
help="Build dependencies under all circumstances. By default we will " +
"not build dependencies unless ART_TEST_RUN_TEST_BUILD=true.")
+ parser.add_argument('--build-target', dest='build_target', help='master-art-host targets')
parser.set_defaults(build = env.ART_TEST_RUN_TEST_BUILD)
parser.add_argument('--gdb', action='store_true', dest='gdb')
parser.add_argument('--gdb-arg', dest='gdb_arg')
- options = parser.parse_args()
+ options = vars(parser.parse_args())
+ if options['build_target']:
+ options = setup_env_for_build_target(target_config[options['build_target']],
+ parser, options)
+
test = ''
- env.EXTRA_DISABLED_TESTS.update(set(options.skips))
- if options.test:
- test = parse_test_name(options.test)
- if options.pictest:
+ env.EXTRA_DISABLED_TESTS.update(set(options['skips']))
+ if options['test']:
+ test = parse_test_name(options['test'])
+ if options['pictest']:
PICTEST_TYPES.add('pictest')
- if options.ndebug:
+ if options['ndebug']:
RUN_TYPES.add('ndebug')
- if options.interp_ac:
+ if options['interp_ac']:
COMPILER_TYPES.add('interp-ac')
- if options.picimage:
+ if options['picimage']:
IMAGE_TYPES.add('picimage')
- if options.n64:
+ if options['n64']:
ADDRESS_SIZES.add('64')
- if options.interpreter:
+ if options['interpreter']:
COMPILER_TYPES.add('interpreter')
- if options.jni:
+ if options['jni']:
JNI_TYPES.add('jni')
- if options.relocate_npatchoat:
+ if options['relocate_npatchoat']:
RELOCATE_TYPES.add('relocate-npatchoat')
- if options.no_prebuild:
+ if options['no_prebuild']:
PREBUILD_TYPES.add('no-prebuild')
- if options.npictest:
+ if options['npictest']:
PICTEST_TYPES.add('npictest')
- if options.no_dex2oat:
+ if options['no_dex2oat']:
PREBUILD_TYPES.add('no-dex2oat')
- if options.jit:
+ if options['jit']:
COMPILER_TYPES.add('jit')
- if options.relocate:
+ if options['relocate']:
RELOCATE_TYPES.add('relocate')
- if options.ndebuggable:
+ if options['ndebuggable']:
DEBUGGABLE_TYPES.add('ndebuggable')
- if options.no_image:
+ if options['no_image']:
IMAGE_TYPES.add('no-image')
- if options.optimizing:
+ if options['optimizing']:
COMPILER_TYPES.add('optimizing')
- if options.trace:
+ if options['trace']:
TRACE_TYPES.add('trace')
- if options.gcstress:
+ if options['gcstress']:
GC_TYPES.add('gcstress')
- if options.no_relocate:
+ if options['no_relocate']:
RELOCATE_TYPES.add('no-relocate')
- if options.target:
+ if options['target']:
TARGET_TYPES.add('target')
- if options.forcecopy:
+ if options['forcecopy']:
JNI_TYPES.add('forcecopy')
- if options.n32:
+ if options['n32']:
ADDRESS_SIZES.add('32')
- if options.host:
+ if options['host']:
TARGET_TYPES.add('host')
- if options.gcverify:
+ if options['gcverify']:
GC_TYPES.add('gcverify')
- if options.debuggable:
+ if options['debuggable']:
DEBUGGABLE_TYPES.add('debuggable')
- if options.prebuild:
+ if options['prebuild']:
PREBUILD_TYPES.add('prebuild')
- if options.debug:
+ if options['debug']:
RUN_TYPES.add('debug')
- if options.checkjni:
+ if options['checkjni']:
JNI_TYPES.add('checkjni')
- if options.ntrace:
+ if options['ntrace']:
TRACE_TYPES.add('ntrace')
- if options.cms:
+ if options['cms']:
GC_TYPES.add('cms')
- if options.multipicimage:
+ if options['multipicimage']:
IMAGE_TYPES.add('multipicimage')
- if options.verbose:
+ if options['verbose']:
verbose = True
- if options.n_thread:
- n_thread = max(1, options.n_thread)
- if options.dry_run:
+ if options['n_thread']:
+ n_thread = max(1, options['n_thread'])
+ if options['dry_run']:
dry_run = True
verbose = True
- build = options.build
- if options.gdb:
+ build = options['build']
+ if options['gdb']:
n_thread = 1
gdb = True
- if options.gdb_arg:
- gdb_arg = options.gdb_arg
+ if options['gdb_arg']:
+ gdb_arg = options['gdb_arg']
return test