Optimizing: Improve const-string code generation.
For strings in the boot image, use either direct pointers
or pc-relative addresses. For other strings, use PC-relative
access to the dex cache arrays for AOT and direct address of
the string's dex cache slot for JIT.
For aosp_flounder-userdebug:
- 32-bit boot.oat: -692KiB (-0.9%)
- 64-bit boot.oat: -948KiB (-1.1%)
- 32-bit dalvik cache total: -900KiB (-0.9%)
- 64-bit dalvik cache total: -3672KiB (-1.5%)
(contains more files than the 32-bit dalvik cache)
For aosp_flounder-userdebug forced to compile PIC:
- 32-bit boot.oat: -380KiB (-0.5%)
- 64-bit boot.oat: -928KiB (-1.0%)
- 32-bit dalvik cache total: -468KiB (-0.4%)
- 64-bit dalvik cache total: -1928KiB (-0.8%)
(contains more files than the 32-bit dalvik cache)
Bug: 26884697
Change-Id: Iec7266ce67e6fedc107be78fab2e742a8dab2696
diff --git a/compiler/optimizing/builder.cc b/compiler/optimizing/builder.cc
index 1b62531..b6b8322 100644
--- a/compiler/optimizing/builder.cc
+++ b/compiler/optimizing/builder.cc
@@ -2745,20 +2745,16 @@
case Instruction::CONST_STRING: {
uint32_t string_index = instruction.VRegB_21c();
- bool in_dex_cache = compiler_driver_->CanAssumeStringIsPresentInDexCache(
- *dex_file_, string_index);
current_block_->AddInstruction(
- new (arena_) HLoadString(graph_->GetCurrentMethod(), string_index, dex_pc, in_dex_cache));
+ new (arena_) HLoadString(graph_->GetCurrentMethod(), string_index, *dex_file_, dex_pc));
UpdateLocal(instruction.VRegA_21c(), current_block_->GetLastInstruction(), dex_pc);
break;
}
case Instruction::CONST_STRING_JUMBO: {
uint32_t string_index = instruction.VRegB_31c();
- bool in_dex_cache = compiler_driver_->CanAssumeStringIsPresentInDexCache(
- *dex_file_, string_index);
current_block_->AddInstruction(
- new (arena_) HLoadString(graph_->GetCurrentMethod(), string_index, dex_pc, in_dex_cache));
+ new (arena_) HLoadString(graph_->GetCurrentMethod(), string_index, *dex_file_, dex_pc));
UpdateLocal(instruction.VRegA_31c(), current_block_->GetLastInstruction(), dex_pc);
break;
}
diff --git a/compiler/optimizing/code_generator.h b/compiler/optimizing/code_generator.h
index e56323f..cad5529 100644
--- a/compiler/optimizing/code_generator.h
+++ b/compiler/optimizing/code_generator.h
@@ -443,6 +443,11 @@
uint32_t dex_pc,
SlowPathCode* slow_path) = 0;
+ // Check if the desired_string_load_kind is supported. If it is, return it,
+ // otherwise return a fall-back info that should be used instead.
+ virtual HLoadString::LoadKind GetSupportedLoadStringKind(
+ HLoadString::LoadKind desired_string_load_kind) = 0;
+
// Check if the desired_dispatch_info is supported. If it is, return it,
// otherwise return a fall-back info that should be used instead.
virtual HInvokeStaticOrDirect::DispatchInfo GetSupportedInvokeStaticOrDirectDispatch(
@@ -471,6 +476,18 @@
LabelType label;
};
+ // String patch info used for recording locations of required linker patches and
+ // target strings. The actual string address can be absolute or PC-relative.
+ template <typename LabelType>
+ struct StringPatchInfo {
+ StringPatchInfo(const DexFile& df, uint32_t index)
+ : dex_file(df), string_index(index), label() { }
+
+ const DexFile& dex_file;
+ uint32_t string_index;
+ LabelType label;
+ };
+
CodeGenerator(HGraph* graph,
size_t number_of_core_registers,
size_t number_of_fpu_registers,
diff --git a/compiler/optimizing/code_generator_arm.cc b/compiler/optimizing/code_generator_arm.cc
index 3a18a0d..98577d6 100644
--- a/compiler/optimizing/code_generator_arm.cc
+++ b/compiler/optimizing/code_generator_arm.cc
@@ -779,13 +779,19 @@
move_resolver_(graph->GetArena(), this),
assembler_(),
isa_features_(isa_features),
+ uint32_literals_(std::less<uint32_t>(),
+ graph->GetArena()->Adapter(kArenaAllocCodeGenerator)),
method_patches_(MethodReferenceComparator(),
graph->GetArena()->Adapter(kArenaAllocCodeGenerator)),
call_patches_(MethodReferenceComparator(),
graph->GetArena()->Adapter(kArenaAllocCodeGenerator)),
relative_call_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)),
- dex_cache_arrays_base_labels_(std::less<HArmDexCacheArraysBase*>(),
- graph->GetArena()->Adapter(kArenaAllocCodeGenerator)) {
+ pc_relative_dex_cache_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)),
+ boot_image_string_patches_(StringReferenceValueComparator(),
+ graph->GetArena()->Adapter(kArenaAllocCodeGenerator)),
+ pc_relative_string_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)),
+ boot_image_address_patches_(std::less<uint32_t>(),
+ graph->GetArena()->Adapter(kArenaAllocCodeGenerator)) {
// Always save the LR register to mimic Quick.
AddAllocatedRegister(Location::RegisterLocation(LR));
}
@@ -5221,12 +5227,57 @@
__ Bind(slow_path->GetExitLabel());
}
+HLoadString::LoadKind CodeGeneratorARM::GetSupportedLoadStringKind(
+ HLoadString::LoadKind desired_string_load_kind) {
+ if (kEmitCompilerReadBarrier) {
+ switch (desired_string_load_kind) {
+ case HLoadString::LoadKind::kBootImageLinkTimeAddress:
+ case HLoadString::LoadKind::kBootImageLinkTimePcRelative:
+ case HLoadString::LoadKind::kBootImageAddress:
+ // TODO: Implement for read barrier.
+ return HLoadString::LoadKind::kDexCacheViaMethod;
+ default:
+ break;
+ }
+ }
+ switch (desired_string_load_kind) {
+ case HLoadString::LoadKind::kBootImageLinkTimeAddress:
+ DCHECK(!GetCompilerOptions().GetCompilePic());
+ break;
+ case HLoadString::LoadKind::kBootImageLinkTimePcRelative:
+ DCHECK(GetCompilerOptions().GetCompilePic());
+ break;
+ case HLoadString::LoadKind::kBootImageAddress:
+ break;
+ case HLoadString::LoadKind::kDexCacheAddress:
+ DCHECK(Runtime::Current()->UseJit());
+ break;
+ case HLoadString::LoadKind::kDexCachePcRelative:
+ DCHECK(!Runtime::Current()->UseJit());
+ // We disable pc-relative load when there is an irreducible loop, as the optimization
+ // is incompatible with it.
+ // TODO: Create as many ArmDexCacheArraysBase instructions as needed for methods
+ // with irreducible loops.
+ if (GetGraph()->HasIrreducibleLoops()) {
+ return HLoadString::LoadKind::kDexCacheViaMethod;
+ }
+ break;
+ case HLoadString::LoadKind::kDexCacheViaMethod:
+ break;
+ }
+ return desired_string_load_kind;
+}
+
void LocationsBuilderARM::VisitLoadString(HLoadString* load) {
- LocationSummary::CallKind call_kind = (!load->IsInDexCache() || kEmitCompilerReadBarrier)
+ LocationSummary::CallKind call_kind = (load->NeedsEnvironment() || kEmitCompilerReadBarrier)
? LocationSummary::kCallOnSlowPath
: LocationSummary::kNoCall;
LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(load, call_kind);
- locations->SetInAt(0, Location::RequiresRegister());
+ HLoadString::LoadKind load_kind = load->GetLoadKind();
+ if (load_kind == HLoadString::LoadKind::kDexCacheViaMethod ||
+ load_kind == HLoadString::LoadKind::kDexCachePcRelative) {
+ locations->SetInAt(0, Location::RequiresRegister());
+ }
locations->SetOut(Location::RequiresRegister());
}
@@ -5234,16 +5285,73 @@
LocationSummary* locations = load->GetLocations();
Location out_loc = locations->Out();
Register out = out_loc.AsRegister<Register>();
- Register current_method = locations->InAt(0).AsRegister<Register>();
- // /* GcRoot<mirror::Class> */ out = current_method->declaring_class_
- GenerateGcRootFieldLoad(
- load, out_loc, current_method, ArtMethod::DeclaringClassOffset().Int32Value());
- // /* GcRoot<mirror::String>[] */ out = out->dex_cache_strings_
- __ LoadFromOffset(kLoadWord, out, out, mirror::Class::DexCacheStringsOffset().Int32Value());
- // /* GcRoot<mirror::String> */ out = out[string_index]
- GenerateGcRootFieldLoad(
- load, out_loc, out, CodeGenerator::GetCacheOffset(load->GetStringIndex()));
+ switch (load->GetLoadKind()) {
+ case HLoadString::LoadKind::kBootImageLinkTimeAddress: {
+ DCHECK(!kEmitCompilerReadBarrier);
+ __ LoadLiteral(out, codegen_->DeduplicateBootImageStringLiteral(load->GetDexFile(),
+ load->GetStringIndex()));
+ return; // No dex cache slow path.
+ }
+ case HLoadString::LoadKind::kBootImageLinkTimePcRelative: {
+ DCHECK(!kEmitCompilerReadBarrier);
+ CodeGeneratorARM::PcRelativePatchInfo* labels =
+ codegen_->NewPcRelativeStringPatch(load->GetDexFile(), load->GetStringIndex());
+ __ BindTrackedLabel(&labels->movw_label);
+ __ movw(out, /* placeholder */ 0u);
+ __ BindTrackedLabel(&labels->movt_label);
+ __ movt(out, /* placeholder */ 0u);
+ __ BindTrackedLabel(&labels->add_pc_label);
+ __ add(out, out, ShifterOperand(PC));
+ return; // No dex cache slow path.
+ }
+ case HLoadString::LoadKind::kBootImageAddress: {
+ DCHECK(!kEmitCompilerReadBarrier);
+ DCHECK_NE(load->GetAddress(), 0u);
+ uint32_t address = dchecked_integral_cast<uint32_t>(load->GetAddress());
+ __ LoadLiteral(out, codegen_->DeduplicateBootImageAddressLiteral(address));
+ return; // No dex cache slow path.
+ }
+ case HLoadString::LoadKind::kDexCacheAddress: {
+ DCHECK_NE(load->GetAddress(), 0u);
+ uint32_t address = dchecked_integral_cast<uint32_t>(load->GetAddress());
+ // 16-bit LDR immediate has a 5-bit offset multiplied by the size and that gives
+ // a 128B range. To try and reduce the number of literals if we load multiple strings,
+ // simply split the dex cache address to a 128B aligned base loaded from a literal
+ // and the remaining offset embedded in the load.
+ static_assert(sizeof(GcRoot<mirror::String>) == 4u, "Expected GC root to be 4 bytes.");
+ DCHECK_ALIGNED(load->GetAddress(), 4u);
+ constexpr size_t offset_bits = /* encoded bits */ 5 + /* scale */ 2;
+ uint32_t base_address = address & ~MaxInt<uint32_t>(offset_bits);
+ uint32_t offset = address & MaxInt<uint32_t>(offset_bits);
+ __ LoadLiteral(out, codegen_->DeduplicateDexCacheAddressLiteral(base_address));
+ GenerateGcRootFieldLoad(load, out_loc, out, offset);
+ break;
+ }
+ case HLoadString::LoadKind::kDexCachePcRelative: {
+ Register base_reg = locations->InAt(0).AsRegister<Register>();
+ HArmDexCacheArraysBase* base = load->InputAt(0)->AsArmDexCacheArraysBase();
+ int32_t offset = load->GetDexCacheElementOffset() - base->GetElementOffset();
+ GenerateGcRootFieldLoad(load, out_loc, base_reg, offset);
+ break;
+ }
+ case HLoadString::LoadKind::kDexCacheViaMethod: {
+ Register current_method = locations->InAt(0).AsRegister<Register>();
+
+ // /* GcRoot<mirror::Class> */ out = current_method->declaring_class_
+ GenerateGcRootFieldLoad(
+ load, out_loc, current_method, ArtMethod::DeclaringClassOffset().Int32Value());
+ // /* GcRoot<mirror::String>[] */ out = out->dex_cache_strings_
+ __ LoadFromOffset(kLoadWord, out, out, mirror::Class::DexCacheStringsOffset().Int32Value());
+ // /* GcRoot<mirror::String> */ out = out[string_index]
+ GenerateGcRootFieldLoad(
+ load, out_loc, out, CodeGenerator::GetCacheOffset(load->GetStringIndex()));
+ break;
+ }
+ default:
+ LOG(FATAL) << "Unexpected load kind: " << load->GetLoadKind();
+ UNREACHABLE();
+ }
if (!load->IsInDexCache()) {
SlowPathCode* slow_path = new (GetGraph()->GetArena()) LoadStringSlowPathARM(load);
@@ -6220,6 +6328,8 @@
HInvokeStaticOrDirect::DispatchInfo dispatch_info = desired_dispatch_info;
// We disable pc-relative load when there is an irreducible loop, as the optimization
// is incompatible with it.
+ // TODO: Create as many ArmDexCacheArraysBase instructions as needed for methods
+ // with irreducible loops.
if (GetGraph()->HasIrreducibleLoops() &&
(dispatch_info.method_load_kind ==
HInvokeStaticOrDirect::MethodLoadKind::kDexCachePcRelative)) {
@@ -6399,13 +6509,49 @@
__ blx(LR);
}
+CodeGeneratorARM::PcRelativePatchInfo* CodeGeneratorARM::NewPcRelativeStringPatch(
+ const DexFile& dex_file, uint32_t string_index) {
+ return NewPcRelativePatch(dex_file, string_index, &pc_relative_string_patches_);
+}
+
+CodeGeneratorARM::PcRelativePatchInfo* CodeGeneratorARM::NewPcRelativeDexCacheArrayPatch(
+ const DexFile& dex_file, uint32_t element_offset) {
+ return NewPcRelativePatch(dex_file, element_offset, &pc_relative_dex_cache_patches_);
+}
+
+CodeGeneratorARM::PcRelativePatchInfo* CodeGeneratorARM::NewPcRelativePatch(
+ const DexFile& dex_file, uint32_t offset_or_index, ArenaDeque<PcRelativePatchInfo>* patches) {
+ patches->emplace_back(dex_file, offset_or_index);
+ return &patches->back();
+}
+
+Literal* CodeGeneratorARM::DeduplicateBootImageStringLiteral(const DexFile& dex_file,
+ uint32_t string_index) {
+ return boot_image_string_patches_.GetOrCreate(
+ StringReference(&dex_file, string_index),
+ [this]() { return __ NewLiteral<uint32_t>(/* placeholder */ 0u); });
+}
+
+Literal* CodeGeneratorARM::DeduplicateBootImageAddressLiteral(uint32_t address) {
+ bool needs_patch = GetCompilerOptions().GetIncludePatchInformation();
+ Uint32ToLiteralMap* map = needs_patch ? &boot_image_address_patches_ : &uint32_literals_;
+ return DeduplicateUint32Literal(dchecked_integral_cast<uint32_t>(address), map);
+}
+
+Literal* CodeGeneratorARM::DeduplicateDexCacheAddressLiteral(uint32_t address) {
+ return DeduplicateUint32Literal(address, &uint32_literals_);
+}
+
void CodeGeneratorARM::EmitLinkerPatches(ArenaVector<LinkerPatch>* linker_patches) {
DCHECK(linker_patches->empty());
size_t size =
method_patches_.size() +
call_patches_.size() +
relative_call_patches_.size() +
- /* MOVW+MOVT for each base */ 2u * dex_cache_arrays_base_labels_.size();
+ /* MOVW+MOVT for each base */ 2u * pc_relative_dex_cache_patches_.size() +
+ boot_image_string_patches_.size() +
+ /* MOVW+MOVT for each base */ 2u * pc_relative_string_patches_.size() +
+ boot_image_address_patches_.size();
linker_patches->reserve(size);
for (const auto& entry : method_patches_) {
const MethodReference& target_method = entry.first;
@@ -6431,41 +6577,75 @@
info.target_method.dex_file,
info.target_method.dex_method_index));
}
- for (const auto& pair : dex_cache_arrays_base_labels_) {
- HArmDexCacheArraysBase* base = pair.first;
- const DexCacheArraysBaseLabels* labels = &pair.second;
- const DexFile& dex_file = base->GetDexFile();
- size_t base_element_offset = base->GetElementOffset();
- DCHECK(labels->add_pc_label.IsBound());
- uint32_t add_pc_offset = dchecked_integral_cast<uint32_t>(labels->add_pc_label.Position());
+ for (const PcRelativePatchInfo& info : pc_relative_dex_cache_patches_) {
+ const DexFile& dex_file = info.target_dex_file;
+ size_t base_element_offset = info.offset_or_index;
+ DCHECK(info.add_pc_label.IsBound());
+ uint32_t add_pc_offset = dchecked_integral_cast<uint32_t>(info.add_pc_label.Position());
// Add MOVW patch.
- DCHECK(labels->movw_label.IsBound());
- uint32_t movw_offset = dchecked_integral_cast<uint32_t>(labels->movw_label.Position());
+ DCHECK(info.movw_label.IsBound());
+ uint32_t movw_offset = dchecked_integral_cast<uint32_t>(info.movw_label.Position());
linker_patches->push_back(LinkerPatch::DexCacheArrayPatch(movw_offset,
&dex_file,
add_pc_offset,
base_element_offset));
// Add MOVT patch.
- DCHECK(labels->movt_label.IsBound());
- uint32_t movt_offset = dchecked_integral_cast<uint32_t>(labels->movt_label.Position());
+ DCHECK(info.movt_label.IsBound());
+ uint32_t movt_offset = dchecked_integral_cast<uint32_t>(info.movt_label.Position());
linker_patches->push_back(LinkerPatch::DexCacheArrayPatch(movt_offset,
&dex_file,
add_pc_offset,
base_element_offset));
}
+ for (const auto& entry : boot_image_string_patches_) {
+ const StringReference& target_string = entry.first;
+ Literal* literal = entry.second;
+ DCHECK(literal->GetLabel()->IsBound());
+ uint32_t literal_offset = literal->GetLabel()->Position();
+ linker_patches->push_back(LinkerPatch::StringPatch(literal_offset,
+ target_string.dex_file,
+ target_string.string_index));
+ }
+ for (const PcRelativePatchInfo& info : pc_relative_string_patches_) {
+ const DexFile& dex_file = info.target_dex_file;
+ uint32_t string_index = info.offset_or_index;
+ DCHECK(info.add_pc_label.IsBound());
+ uint32_t add_pc_offset = dchecked_integral_cast<uint32_t>(info.add_pc_label.Position());
+ // Add MOVW patch.
+ DCHECK(info.movw_label.IsBound());
+ uint32_t movw_offset = dchecked_integral_cast<uint32_t>(info.movw_label.Position());
+ linker_patches->push_back(LinkerPatch::RelativeStringPatch(movw_offset,
+ &dex_file,
+ add_pc_offset,
+ string_index));
+ // Add MOVT patch.
+ DCHECK(info.movt_label.IsBound());
+ uint32_t movt_offset = dchecked_integral_cast<uint32_t>(info.movt_label.Position());
+ linker_patches->push_back(LinkerPatch::RelativeStringPatch(movt_offset,
+ &dex_file,
+ add_pc_offset,
+ string_index));
+ }
+ for (const auto& entry : boot_image_address_patches_) {
+ DCHECK(GetCompilerOptions().GetIncludePatchInformation());
+ Literal* literal = entry.second;
+ DCHECK(literal->GetLabel()->IsBound());
+ uint32_t literal_offset = literal->GetLabel()->Position();
+ linker_patches->push_back(LinkerPatch::RecordPosition(literal_offset));
+ }
+}
+
+Literal* CodeGeneratorARM::DeduplicateUint32Literal(uint32_t value, Uint32ToLiteralMap* map) {
+ return map->GetOrCreate(
+ value,
+ [this, value]() { return __ NewLiteral<uint32_t>(value); });
}
Literal* CodeGeneratorARM::DeduplicateMethodLiteral(MethodReference target_method,
MethodToLiteralMap* map) {
- // Look up the literal for target_method.
- auto lb = map->lower_bound(target_method);
- if (lb != map->end() && !map->key_comp()(target_method, lb->first)) {
- return lb->second;
- }
- // We don't have a literal for this method yet, insert a new one.
- Literal* literal = __ NewLiteral<uint32_t>(0u);
- map->PutBefore(lb, target_method, literal);
- return literal;
+ return map->GetOrCreate(
+ target_method,
+ [this]() { return __ NewLiteral<uint32_t>(/* placeholder */ 0u); });
}
Literal* CodeGeneratorARM::DeduplicateMethodAddressLiteral(MethodReference target_method) {
@@ -6600,16 +6780,16 @@
void LocationsBuilderARM::VisitArmDexCacheArraysBase(HArmDexCacheArraysBase* base) {
LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(base);
locations->SetOut(Location::RequiresRegister());
- codegen_->AddDexCacheArraysBase(base);
}
void InstructionCodeGeneratorARM::VisitArmDexCacheArraysBase(HArmDexCacheArraysBase* base) {
Register base_reg = base->GetLocations()->Out().AsRegister<Register>();
- CodeGeneratorARM::DexCacheArraysBaseLabels* labels = codegen_->GetDexCacheArraysBaseLabels(base);
+ CodeGeneratorARM::PcRelativePatchInfo* labels =
+ codegen_->NewPcRelativeDexCacheArrayPatch(base->GetDexFile(), base->GetElementOffset());
__ BindTrackedLabel(&labels->movw_label);
- __ movw(base_reg, 0u);
+ __ movw(base_reg, /* placeholder */ 0u);
__ BindTrackedLabel(&labels->movt_label);
- __ movt(base_reg, 0u);
+ __ movt(base_reg, /* placeholder */ 0u);
__ BindTrackedLabel(&labels->add_pc_label);
__ add(base_reg, base_reg, ShifterOperand(PC));
}
diff --git a/compiler/optimizing/code_generator_arm.h b/compiler/optimizing/code_generator_arm.h
index cc4aa14..8434128 100644
--- a/compiler/optimizing/code_generator_arm.h
+++ b/compiler/optimizing/code_generator_arm.h
@@ -23,6 +23,7 @@
#include "nodes.h"
#include "parallel_move_resolver.h"
#include "utils/arm/assembler_thumb2.h"
+#include "utils/string_reference.h"
namespace art {
namespace arm {
@@ -403,6 +404,11 @@
Label* GetFrameEntryLabel() { return &frame_entry_label_; }
+ // Check if the desired_string_load_kind is supported. If it is, return it,
+ // otherwise return a fall-back kind that should be used instead.
+ HLoadString::LoadKind GetSupportedLoadStringKind(
+ HLoadString::LoadKind desired_string_load_kind) OVERRIDE;
+
// Check if the desired_dispatch_info is supported. If it is, return it,
// otherwise return a fall-back info that should be used instead.
HInvokeStaticOrDirect::DispatchInfo GetSupportedInvokeStaticOrDirectDispatch(
@@ -414,32 +420,34 @@
void MoveFromReturnRegister(Location trg, Primitive::Type type) OVERRIDE;
- void EmitLinkerPatches(ArenaVector<LinkerPatch>* linker_patches) OVERRIDE;
-
- // The PC-relative base address is loaded with three instructions, MOVW+MOVT
+ // The PcRelativePatchInfo is used for PC-relative addressing of dex cache arrays
+ // and boot image strings. The only difference is the interpretation of the offset_or_index.
+ // The PC-relative address is loaded with three instructions, MOVW+MOVT
// to load the offset to base_reg and then ADD base_reg, PC. The offset is
// calculated from the ADD's effective PC, i.e. PC+4 on Thumb2. Though we
// currently emit these 3 instructions together, instruction scheduling could
// split this sequence apart, so we keep separate labels for each of them.
- struct DexCacheArraysBaseLabels {
- DexCacheArraysBaseLabels() = default;
- DexCacheArraysBaseLabels(DexCacheArraysBaseLabels&& other) = default;
+ struct PcRelativePatchInfo {
+ PcRelativePatchInfo(const DexFile& dex_file, uint32_t off_or_idx)
+ : target_dex_file(dex_file), offset_or_index(off_or_idx) { }
+ PcRelativePatchInfo(PcRelativePatchInfo&& other) = default;
+ const DexFile& target_dex_file;
+ // Either the dex cache array element offset or the string index.
+ uint32_t offset_or_index;
Label movw_label;
Label movt_label;
Label add_pc_label;
};
- void AddDexCacheArraysBase(HArmDexCacheArraysBase* base) {
- DexCacheArraysBaseLabels labels;
- dex_cache_arrays_base_labels_.Put(base, std::move(labels));
- }
+ PcRelativePatchInfo* NewPcRelativeStringPatch(const DexFile& dex_file, uint32_t string_index);
+ PcRelativePatchInfo* NewPcRelativeDexCacheArrayPatch(const DexFile& dex_file,
+ uint32_t element_offset);
+ Literal* DeduplicateBootImageStringLiteral(const DexFile& dex_file, uint32_t string_index);
+ Literal* DeduplicateBootImageAddressLiteral(uint32_t address);
+ Literal* DeduplicateDexCacheAddressLiteral(uint32_t address);
- DexCacheArraysBaseLabels* GetDexCacheArraysBaseLabels(HArmDexCacheArraysBase* base) {
- auto it = dex_cache_arrays_base_labels_.find(base);
- DCHECK(it != dex_cache_arrays_base_labels_.end());
- return &it->second;
- }
+ void EmitLinkerPatches(ArenaVector<LinkerPatch>* linker_patches) OVERRIDE;
// Fast path implementation of ReadBarrier::Barrier for a heap
// reference field load when Baker's read barriers are used.
@@ -525,14 +533,19 @@
Register GetInvokeStaticOrDirectExtraParameter(HInvokeStaticOrDirect* invoke, Register temp);
+ using Uint32ToLiteralMap = ArenaSafeMap<uint32_t, Literal*>;
using MethodToLiteralMap = ArenaSafeMap<MethodReference, Literal*, MethodReferenceComparator>;
- using DexCacheArraysBaseToLabelsMap = ArenaSafeMap<HArmDexCacheArraysBase*,
- DexCacheArraysBaseLabels,
- std::less<HArmDexCacheArraysBase*>>;
+ using BootStringToLiteralMap = ArenaSafeMap<StringReference,
+ Literal*,
+ StringReferenceValueComparator>;
+ Literal* DeduplicateUint32Literal(uint32_t value, Uint32ToLiteralMap* map);
Literal* DeduplicateMethodLiteral(MethodReference target_method, MethodToLiteralMap* map);
Literal* DeduplicateMethodAddressLiteral(MethodReference target_method);
Literal* DeduplicateMethodCodeLiteral(MethodReference target_method);
+ PcRelativePatchInfo* NewPcRelativePatch(const DexFile& dex_file,
+ uint32_t offset_or_index,
+ ArenaDeque<PcRelativePatchInfo>* patches);
// Labels for each block that will be compiled.
Label* block_labels_; // Indexed by block id.
@@ -543,14 +556,22 @@
Thumb2Assembler assembler_;
const ArmInstructionSetFeatures& isa_features_;
+ // Deduplication map for 32-bit literals, used for non-patchable boot image addresses.
+ Uint32ToLiteralMap uint32_literals_;
// Method patch info, map MethodReference to a literal for method address and method code.
MethodToLiteralMap method_patches_;
MethodToLiteralMap call_patches_;
// Relative call patch info.
// Using ArenaDeque<> which retains element addresses on push/emplace_back().
ArenaDeque<MethodPatchInfo<Label>> relative_call_patches_;
-
- DexCacheArraysBaseToLabelsMap dex_cache_arrays_base_labels_;
+ // PC-relative patch info for each HArmDexCacheArraysBase.
+ ArenaDeque<PcRelativePatchInfo> pc_relative_dex_cache_patches_;
+ // Deduplication map for boot string literals for kBootImageLinkTimeAddress.
+ BootStringToLiteralMap boot_image_string_patches_;
+ // PC-relative String patch info.
+ ArenaDeque<PcRelativePatchInfo> pc_relative_string_patches_;
+ // Deduplication map for patchable boot image addresses.
+ Uint32ToLiteralMap boot_image_address_patches_;
DISALLOW_COPY_AND_ASSIGN(CodeGeneratorARM);
};
diff --git a/compiler/optimizing/code_generator_arm64.cc b/compiler/optimizing/code_generator_arm64.cc
index 1f577b3..491014d 100644
--- a/compiler/optimizing/code_generator_arm64.cc
+++ b/compiler/optimizing/code_generator_arm64.cc
@@ -905,6 +905,8 @@
instruction_visitor_(graph, this),
move_resolver_(graph->GetArena(), this),
isa_features_(isa_features),
+ uint32_literals_(std::less<uint32_t>(),
+ graph->GetArena()->Adapter(kArenaAllocCodeGenerator)),
uint64_literals_(std::less<uint64_t>(),
graph->GetArena()->Adapter(kArenaAllocCodeGenerator)),
method_patches_(MethodReferenceComparator(),
@@ -912,7 +914,12 @@
call_patches_(MethodReferenceComparator(),
graph->GetArena()->Adapter(kArenaAllocCodeGenerator)),
relative_call_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)),
- pc_relative_dex_cache_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)) {
+ pc_relative_dex_cache_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)),
+ boot_image_string_patches_(StringReferenceValueComparator(),
+ graph->GetArena()->Adapter(kArenaAllocCodeGenerator)),
+ pc_relative_string_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)),
+ boot_image_address_patches_(std::less<uint32_t>(),
+ graph->GetArena()->Adapter(kArenaAllocCodeGenerator)) {
// Save the link register (containing the return address) to mimic Quick.
AddAllocatedRegister(LocationFrom(lr));
}
@@ -3662,23 +3669,21 @@
break;
case HInvokeStaticOrDirect::MethodLoadKind::kDexCachePcRelative: {
// Add ADRP with its PC-relative DexCache access patch.
- pc_relative_dex_cache_patches_.emplace_back(*invoke->GetTargetMethod().dex_file,
- invoke->GetDexCacheArrayOffset());
- vixl::Label* pc_insn_label = &pc_relative_dex_cache_patches_.back().label;
+ const DexFile& dex_file = *invoke->GetTargetMethod().dex_file;
+ uint32_t element_offset = invoke->GetDexCacheArrayOffset();
+ vixl::Label* adrp_label = NewPcRelativeDexCacheArrayPatch(dex_file, element_offset);
{
vixl::SingleEmissionCheckScope guard(GetVIXLAssembler());
- __ Bind(pc_insn_label);
- __ adrp(XRegisterFrom(temp), 0);
+ __ Bind(adrp_label);
+ __ adrp(XRegisterFrom(temp), /* offset placeholder */ 0);
}
- pc_relative_dex_cache_patches_.back().pc_insn_label = pc_insn_label;
// Add LDR with its PC-relative DexCache access patch.
- pc_relative_dex_cache_patches_.emplace_back(*invoke->GetTargetMethod().dex_file,
- invoke->GetDexCacheArrayOffset());
+ vixl::Label* ldr_label =
+ NewPcRelativeDexCacheArrayPatch(dex_file, element_offset, adrp_label);
{
vixl::SingleEmissionCheckScope guard(GetVIXLAssembler());
- __ Bind(&pc_relative_dex_cache_patches_.back().label);
- __ ldr(XRegisterFrom(temp), MemOperand(XRegisterFrom(temp), 0));
- pc_relative_dex_cache_patches_.back().pc_insn_label = pc_insn_label;
+ __ Bind(ldr_label);
+ __ ldr(XRegisterFrom(temp), MemOperand(XRegisterFrom(temp), /* offset placeholder */ 0));
}
break;
}
@@ -3772,13 +3777,58 @@
__ Blr(lr);
}
+vixl::Label* CodeGeneratorARM64::NewPcRelativeStringPatch(const DexFile& dex_file,
+ uint32_t string_index,
+ vixl::Label* adrp_label) {
+ return NewPcRelativePatch(dex_file, string_index, adrp_label, &pc_relative_string_patches_);
+}
+
+vixl::Label* CodeGeneratorARM64::NewPcRelativeDexCacheArrayPatch(const DexFile& dex_file,
+ uint32_t element_offset,
+ vixl::Label* adrp_label) {
+ return NewPcRelativePatch(dex_file, element_offset, adrp_label, &pc_relative_dex_cache_patches_);
+}
+
+vixl::Label* CodeGeneratorARM64::NewPcRelativePatch(const DexFile& dex_file,
+ uint32_t offset_or_index,
+ vixl::Label* adrp_label,
+ ArenaDeque<PcRelativePatchInfo>* patches) {
+ // Add a patch entry and return the label.
+ patches->emplace_back(dex_file, offset_or_index);
+ PcRelativePatchInfo* info = &patches->back();
+ vixl::Label* label = &info->label;
+ // If adrp_label is null, this is the ADRP patch and needs to point to its own label.
+ info->pc_insn_label = (adrp_label != nullptr) ? adrp_label : label;
+ return label;
+}
+
+vixl::Literal<uint32_t>* CodeGeneratorARM64::DeduplicateBootImageStringLiteral(
+ const DexFile& dex_file, uint32_t string_index) {
+ return boot_image_string_patches_.GetOrCreate(
+ StringReference(&dex_file, string_index),
+ [this]() { return __ CreateLiteralDestroyedWithPool<uint32_t>(/* placeholder */ 0u); });
+}
+
+vixl::Literal<uint32_t>* CodeGeneratorARM64::DeduplicateBootImageAddressLiteral(uint64_t address) {
+ bool needs_patch = GetCompilerOptions().GetIncludePatchInformation();
+ Uint32ToLiteralMap* map = needs_patch ? &boot_image_address_patches_ : &uint32_literals_;
+ return DeduplicateUint32Literal(dchecked_integral_cast<uint32_t>(address), map);
+}
+
+vixl::Literal<uint64_t>* CodeGeneratorARM64::DeduplicateDexCacheAddressLiteral(uint64_t address) {
+ return DeduplicateUint64Literal(address);
+}
+
void CodeGeneratorARM64::EmitLinkerPatches(ArenaVector<LinkerPatch>* linker_patches) {
DCHECK(linker_patches->empty());
size_t size =
method_patches_.size() +
call_patches_.size() +
relative_call_patches_.size() +
- pc_relative_dex_cache_patches_.size();
+ pc_relative_dex_cache_patches_.size() +
+ boot_image_string_patches_.size() +
+ pc_relative_string_patches_.size() +
+ boot_image_address_patches_.size();
linker_patches->reserve(size);
for (const auto& entry : method_patches_) {
const MethodReference& target_method = entry.first;
@@ -3799,38 +3849,51 @@
info.target_method.dex_file,
info.target_method.dex_method_index));
}
- for (const PcRelativeDexCacheAccessInfo& info : pc_relative_dex_cache_patches_) {
+ for (const PcRelativePatchInfo& info : pc_relative_dex_cache_patches_) {
linker_patches->push_back(LinkerPatch::DexCacheArrayPatch(info.label.location(),
&info.target_dex_file,
info.pc_insn_label->location(),
- info.element_offset));
+ info.offset_or_index));
+ }
+ for (const auto& entry : boot_image_string_patches_) {
+ const StringReference& target_string = entry.first;
+ vixl::Literal<uint32_t>* literal = entry.second;
+ linker_patches->push_back(LinkerPatch::StringPatch(literal->offset(),
+ target_string.dex_file,
+ target_string.string_index));
+ }
+ for (const PcRelativePatchInfo& info : pc_relative_string_patches_) {
+ linker_patches->push_back(LinkerPatch::RelativeStringPatch(info.label.location(),
+ &info.target_dex_file,
+ info.pc_insn_label->location(),
+ info.offset_or_index));
+ }
+ for (const auto& entry : boot_image_address_patches_) {
+ DCHECK(GetCompilerOptions().GetIncludePatchInformation());
+ vixl::Literal<uint32_t>* literal = entry.second;
+ linker_patches->push_back(LinkerPatch::RecordPosition(literal->offset()));
}
}
+vixl::Literal<uint32_t>* CodeGeneratorARM64::DeduplicateUint32Literal(uint32_t value,
+ Uint32ToLiteralMap* map) {
+ return map->GetOrCreate(
+ value,
+ [this, value]() { return __ CreateLiteralDestroyedWithPool<uint32_t>(value); });
+}
+
vixl::Literal<uint64_t>* CodeGeneratorARM64::DeduplicateUint64Literal(uint64_t value) {
- // Look up the literal for value.
- auto lb = uint64_literals_.lower_bound(value);
- if (lb != uint64_literals_.end() && !uint64_literals_.key_comp()(value, lb->first)) {
- return lb->second;
- }
- // We don't have a literal for this value, insert a new one.
- vixl::Literal<uint64_t>* literal = __ CreateLiteralDestroyedWithPool<uint64_t>(value);
- uint64_literals_.PutBefore(lb, value, literal);
- return literal;
+ return uint64_literals_.GetOrCreate(
+ value,
+ [this, value]() { return __ CreateLiteralDestroyedWithPool<uint64_t>(value); });
}
vixl::Literal<uint64_t>* CodeGeneratorARM64::DeduplicateMethodLiteral(
MethodReference target_method,
MethodToLiteralMap* map) {
- // Look up the literal for target_method.
- auto lb = map->lower_bound(target_method);
- if (lb != map->end() && !map->key_comp()(target_method, lb->first)) {
- return lb->second;
- }
- // We don't have a literal for this method yet, insert a new one.
- vixl::Literal<uint64_t>* literal = __ CreateLiteralDestroyedWithPool<uint64_t>(0u);
- map->PutBefore(lb, target_method, literal);
- return literal;
+ return map->GetOrCreate(
+ target_method,
+ [this]() { return __ CreateLiteralDestroyedWithPool<uint64_t>(/* placeholder */ 0u); });
}
vixl::Literal<uint64_t>* CodeGeneratorARM64::DeduplicateMethodAddressLiteral(
@@ -3955,28 +4018,135 @@
// Nothing to do, this is driven by the code generator.
}
+HLoadString::LoadKind CodeGeneratorARM64::GetSupportedLoadStringKind(
+ HLoadString::LoadKind desired_string_load_kind) {
+ if (kEmitCompilerReadBarrier) {
+ switch (desired_string_load_kind) {
+ case HLoadString::LoadKind::kBootImageLinkTimeAddress:
+ case HLoadString::LoadKind::kBootImageLinkTimePcRelative:
+ case HLoadString::LoadKind::kBootImageAddress:
+ // TODO: Implement for read barrier.
+ return HLoadString::LoadKind::kDexCacheViaMethod;
+ default:
+ break;
+ }
+ }
+ switch (desired_string_load_kind) {
+ case HLoadString::LoadKind::kBootImageLinkTimeAddress:
+ DCHECK(!GetCompilerOptions().GetCompilePic());
+ break;
+ case HLoadString::LoadKind::kBootImageLinkTimePcRelative:
+ DCHECK(GetCompilerOptions().GetCompilePic());
+ break;
+ case HLoadString::LoadKind::kBootImageAddress:
+ break;
+ case HLoadString::LoadKind::kDexCacheAddress:
+ DCHECK(Runtime::Current()->UseJit());
+ break;
+ case HLoadString::LoadKind::kDexCachePcRelative:
+ DCHECK(!Runtime::Current()->UseJit());
+ break;
+ case HLoadString::LoadKind::kDexCacheViaMethod:
+ break;
+ }
+ return desired_string_load_kind;
+}
+
void LocationsBuilderARM64::VisitLoadString(HLoadString* load) {
- LocationSummary::CallKind call_kind = (!load->IsInDexCache() || kEmitCompilerReadBarrier)
+ LocationSummary::CallKind call_kind = (load->NeedsEnvironment() || kEmitCompilerReadBarrier)
? LocationSummary::kCallOnSlowPath
: LocationSummary::kNoCall;
LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(load, call_kind);
- locations->SetInAt(0, Location::RequiresRegister());
+ if (load->GetLoadKind() == HLoadString::LoadKind::kDexCacheViaMethod) {
+ locations->SetInAt(0, Location::RequiresRegister());
+ }
locations->SetOut(Location::RequiresRegister());
}
void InstructionCodeGeneratorARM64::VisitLoadString(HLoadString* load) {
Location out_loc = load->GetLocations()->Out();
Register out = OutputRegister(load);
- Register current_method = InputRegisterAt(load, 0);
- // /* GcRoot<mirror::Class> */ out = current_method->declaring_class_
- GenerateGcRootFieldLoad(
- load, out_loc, current_method, ArtMethod::DeclaringClassOffset().Int32Value());
- // /* GcRoot<mirror::String>[] */ out = out->dex_cache_strings_
- __ Ldr(out.X(), HeapOperand(out, mirror::Class::DexCacheStringsOffset().Uint32Value()));
- // /* GcRoot<mirror::String> */ out = out[string_index]
- GenerateGcRootFieldLoad(
- load, out_loc, out.X(), CodeGenerator::GetCacheOffset(load->GetStringIndex()));
+ switch (load->GetLoadKind()) {
+ case HLoadString::LoadKind::kBootImageLinkTimeAddress:
+ DCHECK(!kEmitCompilerReadBarrier);
+ __ Ldr(out, codegen_->DeduplicateBootImageStringLiteral(load->GetDexFile(),
+ load->GetStringIndex()));
+ return; // No dex cache slow path.
+ case HLoadString::LoadKind::kBootImageLinkTimePcRelative: {
+ DCHECK(!kEmitCompilerReadBarrier);
+ // Add ADRP with its PC-relative String patch.
+ const DexFile& dex_file = load->GetDexFile();
+ uint32_t string_index = load->GetStringIndex();
+ vixl::Label* adrp_label = codegen_->NewPcRelativeStringPatch(dex_file, string_index);
+ {
+ vixl::SingleEmissionCheckScope guard(GetVIXLAssembler());
+ __ Bind(adrp_label);
+ __ adrp(out.X(), /* offset placeholder */ 0);
+ }
+ // Add ADD with its PC-relative String patch.
+ vixl::Label* add_label =
+ codegen_->NewPcRelativeStringPatch(dex_file, string_index, adrp_label);
+ {
+ vixl::SingleEmissionCheckScope guard(GetVIXLAssembler());
+ __ Bind(add_label);
+ __ add(out.X(), out.X(), Operand(/* offset placeholder */ 0));
+ }
+ return; // No dex cache slow path.
+ }
+ case HLoadString::LoadKind::kBootImageAddress: {
+ DCHECK(!kEmitCompilerReadBarrier);
+ DCHECK(load->GetAddress() != 0u && IsUint<32>(load->GetAddress()));
+ __ Ldr(out.W(), codegen_->DeduplicateBootImageAddressLiteral(load->GetAddress()));
+ return; // No dex cache slow path.
+ }
+ case HLoadString::LoadKind::kDexCacheAddress: {
+ DCHECK_NE(load->GetAddress(), 0u);
+ // LDR immediate has a 12-bit offset multiplied by the size and for 32-bit loads
+ // that gives a 16KiB range. To try and reduce the number of literals if we load
+ // multiple strings, simply split the dex cache address to a 16KiB aligned base
+ // loaded from a literal and the remaining offset embedded in the load.
+ static_assert(sizeof(GcRoot<mirror::String>) == 4u, "Expected GC root to be 4 bytes.");
+ DCHECK_ALIGNED(load->GetAddress(), 4u);
+ constexpr size_t offset_bits = /* encoded bits */ 12 + /* scale */ 2;
+ uint64_t base_address = load->GetAddress() & ~MaxInt<uint64_t>(offset_bits);
+ uint32_t offset = load->GetAddress() & MaxInt<uint64_t>(offset_bits);
+ __ Ldr(out.X(), codegen_->DeduplicateDexCacheAddressLiteral(base_address));
+ GenerateGcRootFieldLoad(load, out_loc, out.X(), offset);
+ break;
+ }
+ case HLoadString::LoadKind::kDexCachePcRelative: {
+ // Add ADRP with its PC-relative DexCache access patch.
+ const DexFile& dex_file = load->GetDexFile();
+ uint32_t element_offset = load->GetDexCacheElementOffset();
+ vixl::Label* adrp_label = codegen_->NewPcRelativeDexCacheArrayPatch(dex_file, element_offset);
+ {
+ vixl::SingleEmissionCheckScope guard(GetVIXLAssembler());
+ __ Bind(adrp_label);
+ __ adrp(out.X(), /* offset placeholder */ 0);
+ }
+ // Add LDR with its PC-relative DexCache access patch.
+ vixl::Label* ldr_label =
+ codegen_->NewPcRelativeDexCacheArrayPatch(dex_file, element_offset, adrp_label);
+ GenerateGcRootFieldLoad(load, out_loc, out.X(), /* offset placeholder */ 0, ldr_label);
+ break;
+ }
+ case HLoadString::LoadKind::kDexCacheViaMethod: {
+ Register current_method = InputRegisterAt(load, 0);
+ // /* GcRoot<mirror::Class> */ out = current_method->declaring_class_
+ GenerateGcRootFieldLoad(
+ load, out_loc, current_method, ArtMethod::DeclaringClassOffset().Int32Value());
+ // /* GcRoot<mirror::String>[] */ out = out->dex_cache_strings_
+ __ Ldr(out.X(), HeapOperand(out, mirror::Class::DexCacheStringsOffset().Uint32Value()));
+ // /* GcRoot<mirror::String> */ out = out[string_index]
+ GenerateGcRootFieldLoad(
+ load, out_loc, out.X(), CodeGenerator::GetCacheOffset(load->GetStringIndex()));
+ break;
+ }
+ default:
+ LOG(FATAL) << "Unexpected load kind: " << load->GetLoadKind();
+ UNREACHABLE();
+ }
if (!load->IsInDexCache()) {
SlowPathCodeARM64* slow_path = new (GetGraph()->GetArena()) LoadStringSlowPathARM64(load);
@@ -4791,7 +4961,8 @@
void InstructionCodeGeneratorARM64::GenerateGcRootFieldLoad(HInstruction* instruction,
Location root,
vixl::Register obj,
- uint32_t offset) {
+ uint32_t offset,
+ vixl::Label* fixup_label) {
Register root_reg = RegisterFrom(root, Primitive::kPrimNot);
if (kEmitCompilerReadBarrier) {
if (kUseBakerReadBarrier) {
@@ -4804,7 +4975,13 @@
// }
// /* GcRoot<mirror::Object> */ root = *(obj + offset)
- __ Ldr(root_reg, MemOperand(obj, offset));
+ if (fixup_label == nullptr) {
+ __ Ldr(root_reg, MemOperand(obj, offset));
+ } else {
+ vixl::SingleEmissionCheckScope guard(GetVIXLAssembler());
+ __ Bind(fixup_label);
+ __ ldr(root_reg, MemOperand(obj, offset));
+ }
static_assert(
sizeof(mirror::CompressedReference<mirror::Object>) == sizeof(GcRoot<mirror::Object>),
"art::mirror::CompressedReference<mirror::Object> and art::GcRoot<mirror::Object> "
@@ -4829,14 +5006,26 @@
// GC root loaded through a slow path for read barriers other
// than Baker's.
// /* GcRoot<mirror::Object>* */ root = obj + offset
- __ Add(root_reg.X(), obj.X(), offset);
+ if (fixup_label == nullptr) {
+ __ Add(root_reg.X(), obj.X(), offset);
+ } else {
+ vixl::SingleEmissionCheckScope guard(GetVIXLAssembler());
+ __ Bind(fixup_label);
+ __ add(root_reg.X(), obj.X(), offset);
+ }
// /* mirror::Object* */ root = root->Read()
codegen_->GenerateReadBarrierForRootSlow(instruction, root, root);
}
} else {
// Plain GC root load with no read barrier.
// /* GcRoot<mirror::Object> */ root = *(obj + offset)
- __ Ldr(root_reg, MemOperand(obj, offset));
+ if (fixup_label == nullptr) {
+ __ Ldr(root_reg, MemOperand(obj, offset));
+ } else {
+ vixl::SingleEmissionCheckScope guard(GetVIXLAssembler());
+ __ Bind(fixup_label);
+ __ ldr(root_reg, MemOperand(obj, offset));
+ }
// Note that GC roots are not affected by heap poisoning, thus we
// do not have to unpoison `root_reg` here.
}
diff --git a/compiler/optimizing/code_generator_arm64.h b/compiler/optimizing/code_generator_arm64.h
index cf9dc1b..8ec7531 100644
--- a/compiler/optimizing/code_generator_arm64.h
+++ b/compiler/optimizing/code_generator_arm64.h
@@ -17,6 +17,7 @@
#ifndef ART_COMPILER_OPTIMIZING_CODE_GENERATOR_ARM64_H_
#define ART_COMPILER_OPTIMIZING_CODE_GENERATOR_ARM64_H_
+#include "arch/arm64/quick_method_frame_info_arm64.h"
#include "code_generator.h"
#include "common_arm64.h"
#include "dex/compiler_enums.h"
@@ -24,9 +25,9 @@
#include "nodes.h"
#include "parallel_move_resolver.h"
#include "utils/arm64/assembler_arm64.h"
+#include "utils/string_reference.h"
#include "vixl/a64/disasm-a64.h"
#include "vixl/a64/macro-assembler-a64.h"
-#include "arch/arm64/quick_method_frame_info_arm64.h"
namespace art {
namespace arm64 {
@@ -255,7 +256,8 @@
void GenerateGcRootFieldLoad(HInstruction* instruction,
Location root,
vixl::Register obj,
- uint32_t offset);
+ uint32_t offset,
+ vixl::Label* fixup_label = nullptr);
// Generate a floating-point comparison.
void GenerateFcmp(HInstruction* instruction);
@@ -453,6 +455,11 @@
return false;
}
+ // Check if the desired_string_load_kind is supported. If it is, return it,
+ // otherwise return a fall-back kind that should be used instead.
+ HLoadString::LoadKind GetSupportedLoadStringKind(
+ HLoadString::LoadKind desired_string_load_kind) OVERRIDE;
+
// Check if the desired_dispatch_info is supported. If it is, return it,
// otherwise return a fall-back info that should be used instead.
HInvokeStaticOrDirect::DispatchInfo GetSupportedInvokeStaticOrDirectDispatch(
@@ -467,6 +474,27 @@
UNIMPLEMENTED(FATAL);
}
+ // Add a new PC-relative string patch for an instruction and return the label
+ // to be bound before the instruction. The instruction will be either the
+ // ADRP (pass `adrp_label = null`) or the ADD (pass `adrp_label` pointing
+ // to the associated ADRP patch label).
+ vixl::Label* NewPcRelativeStringPatch(const DexFile& dex_file,
+ uint32_t string_index,
+ vixl::Label* adrp_label = nullptr);
+
+ // Add a new PC-relative dex cache array patch for an instruction and return
+ // the label to be bound before the instruction. The instruction will be
+ // either the ADRP (pass `adrp_label = null`) or the LDR (pass `adrp_label`
+ // pointing to the associated ADRP patch label).
+ vixl::Label* NewPcRelativeDexCacheArrayPatch(const DexFile& dex_file,
+ uint32_t element_offset,
+ vixl::Label* adrp_label = nullptr);
+
+ vixl::Literal<uint32_t>* DeduplicateBootImageStringLiteral(const DexFile& dex_file,
+ uint32_t string_index);
+ vixl::Literal<uint32_t>* DeduplicateBootImageAddressLiteral(uint64_t address);
+ vixl::Literal<uint64_t>* DeduplicateDexCacheAddressLiteral(uint64_t address);
+
void EmitLinkerPatches(ArenaVector<LinkerPatch>* linker_patches) OVERRIDE;
// Fast path implementation of ReadBarrier::Barrier for a heap
@@ -554,26 +582,39 @@
bool use_load_acquire);
using Uint64ToLiteralMap = ArenaSafeMap<uint64_t, vixl::Literal<uint64_t>*>;
+ using Uint32ToLiteralMap = ArenaSafeMap<uint32_t, vixl::Literal<uint32_t>*>;
using MethodToLiteralMap = ArenaSafeMap<MethodReference,
vixl::Literal<uint64_t>*,
MethodReferenceComparator>;
+ using BootStringToLiteralMap = ArenaSafeMap<StringReference,
+ vixl::Literal<uint32_t>*,
+ StringReferenceValueComparator>;
+ vixl::Literal<uint32_t>* DeduplicateUint32Literal(uint32_t value, Uint32ToLiteralMap* map);
vixl::Literal<uint64_t>* DeduplicateUint64Literal(uint64_t value);
vixl::Literal<uint64_t>* DeduplicateMethodLiteral(MethodReference target_method,
MethodToLiteralMap* map);
vixl::Literal<uint64_t>* DeduplicateMethodAddressLiteral(MethodReference target_method);
vixl::Literal<uint64_t>* DeduplicateMethodCodeLiteral(MethodReference target_method);
- struct PcRelativeDexCacheAccessInfo {
- PcRelativeDexCacheAccessInfo(const DexFile& dex_file, uint32_t element_off)
- : target_dex_file(dex_file), element_offset(element_off), label(), pc_insn_label() { }
+ // The PcRelativePatchInfo is used for PC-relative addressing of dex cache arrays
+ // and boot image strings. The only difference is the interpretation of the offset_or_index.
+ struct PcRelativePatchInfo {
+ PcRelativePatchInfo(const DexFile& dex_file, uint32_t off_or_idx)
+ : target_dex_file(dex_file), offset_or_index(off_or_idx), label(), pc_insn_label() { }
const DexFile& target_dex_file;
- uint32_t element_offset;
+ // Either the dex cache array element offset or the string index.
+ uint32_t offset_or_index;
vixl::Label label;
vixl::Label* pc_insn_label;
};
+ vixl::Label* NewPcRelativePatch(const DexFile& dex_file,
+ uint32_t offset_or_index,
+ vixl::Label* adrp_label,
+ ArenaDeque<PcRelativePatchInfo>* patches);
+
void EmitJumpTables();
// Labels for each block that will be compiled.
@@ -587,7 +628,10 @@
Arm64Assembler assembler_;
const Arm64InstructionSetFeatures& isa_features_;
- // Deduplication map for 64-bit literals, used for non-patchable method address and method code.
+ // Deduplication map for 32-bit literals, used for non-patchable boot image addresses.
+ Uint32ToLiteralMap uint32_literals_;
+ // Deduplication map for 64-bit literals, used for non-patchable method address, method code
+ // or string dex cache address.
Uint64ToLiteralMap uint64_literals_;
// Method patch info, map MethodReference to a literal for method address and method code.
MethodToLiteralMap method_patches_;
@@ -596,7 +640,13 @@
// Using ArenaDeque<> which retains element addresses on push/emplace_back().
ArenaDeque<MethodPatchInfo<vixl::Label>> relative_call_patches_;
// PC-relative DexCache access info.
- ArenaDeque<PcRelativeDexCacheAccessInfo> pc_relative_dex_cache_patches_;
+ ArenaDeque<PcRelativePatchInfo> pc_relative_dex_cache_patches_;
+ // Deduplication map for boot string literals for kBootImageLinkTimeAddress.
+ BootStringToLiteralMap boot_image_string_patches_;
+ // PC-relative String patch info.
+ ArenaDeque<PcRelativePatchInfo> pc_relative_string_patches_;
+ // Deduplication map for patchable boot image addresses.
+ Uint32ToLiteralMap boot_image_address_patches_;
DISALLOW_COPY_AND_ASSIGN(CodeGeneratorARM64);
};
diff --git a/compiler/optimizing/code_generator_mips.cc b/compiler/optimizing/code_generator_mips.cc
index a29d839..8b19f84 100644
--- a/compiler/optimizing/code_generator_mips.cc
+++ b/compiler/optimizing/code_generator_mips.cc
@@ -3816,6 +3816,12 @@
return false;
}
+HLoadString::LoadKind CodeGeneratorMIPS::GetSupportedLoadStringKind(
+ HLoadString::LoadKind desired_string_load_kind ATTRIBUTE_UNUSED) {
+ // TODO: Implement other kinds.
+ return HLoadString::LoadKind::kDexCacheViaMethod;
+}
+
HInvokeStaticOrDirect::DispatchInfo CodeGeneratorMIPS::GetSupportedInvokeStaticOrDirectDispatch(
const HInvokeStaticOrDirect::DispatchInfo& desired_dispatch_info,
MethodReference target_method ATTRIBUTE_UNUSED) {
@@ -4066,9 +4072,9 @@
}
void LocationsBuilderMIPS::VisitLoadString(HLoadString* load) {
- LocationSummary::CallKind call_kind = load->IsInDexCache()
- ? LocationSummary::kNoCall
- : LocationSummary::kCallOnSlowPath;
+ LocationSummary::CallKind call_kind = load->NeedsEnvironment()
+ ? LocationSummary::kCallOnSlowPath
+ : LocationSummary::kNoCall;
LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(load, call_kind);
locations->SetInAt(0, Location::RequiresRegister());
locations->SetOut(Location::RequiresRegister());
diff --git a/compiler/optimizing/code_generator_mips.h b/compiler/optimizing/code_generator_mips.h
index b720573..afe7917 100644
--- a/compiler/optimizing/code_generator_mips.h
+++ b/compiler/optimizing/code_generator_mips.h
@@ -345,6 +345,11 @@
return type == Primitive::kPrimLong;
}
+ // Check if the desired_string_load_kind is supported. If it is, return it,
+ // otherwise return a fall-back kind that should be used instead.
+ HLoadString::LoadKind GetSupportedLoadStringKind(
+ HLoadString::LoadKind desired_string_load_kind) OVERRIDE;
+
// Check if the desired_dispatch_info is supported. If it is, return it,
// otherwise return a fall-back info that should be used instead.
HInvokeStaticOrDirect::DispatchInfo GetSupportedInvokeStaticOrDirectDispatch(
diff --git a/compiler/optimizing/code_generator_mips64.cc b/compiler/optimizing/code_generator_mips64.cc
index 72ef499..2f9eca6 100644
--- a/compiler/optimizing/code_generator_mips64.cc
+++ b/compiler/optimizing/code_generator_mips64.cc
@@ -3030,6 +3030,12 @@
return false;
}
+HLoadString::LoadKind CodeGeneratorMIPS64::GetSupportedLoadStringKind(
+ HLoadString::LoadKind desired_string_load_kind ATTRIBUTE_UNUSED) {
+ // TODO: Implement other kinds.
+ return HLoadString::LoadKind::kDexCacheViaMethod;
+}
+
HInvokeStaticOrDirect::DispatchInfo CodeGeneratorMIPS64::GetSupportedInvokeStaticOrDirectDispatch(
const HInvokeStaticOrDirect::DispatchInfo& desired_dispatch_info,
MethodReference target_method ATTRIBUTE_UNUSED) {
@@ -3284,9 +3290,9 @@
}
void LocationsBuilderMIPS64::VisitLoadString(HLoadString* load) {
- LocationSummary::CallKind call_kind = load->IsInDexCache()
- ? LocationSummary::kNoCall
- : LocationSummary::kCallOnSlowPath;
+ LocationSummary::CallKind call_kind = load->NeedsEnvironment()
+ ? LocationSummary::kCallOnSlowPath
+ : LocationSummary::kNoCall;
LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(load, call_kind);
locations->SetInAt(0, Location::RequiresRegister());
locations->SetOut(Location::RequiresRegister());
diff --git a/compiler/optimizing/code_generator_mips64.h b/compiler/optimizing/code_generator_mips64.h
index 9464a14..94767cb 100644
--- a/compiler/optimizing/code_generator_mips64.h
+++ b/compiler/optimizing/code_generator_mips64.h
@@ -337,6 +337,11 @@
bool NeedsTwoRegisters(Primitive::Type type ATTRIBUTE_UNUSED) const { return false; }
+ // Check if the desired_string_load_kind is supported. If it is, return it,
+ // otherwise return a fall-back kind that should be used instead.
+ HLoadString::LoadKind GetSupportedLoadStringKind(
+ HLoadString::LoadKind desired_string_load_kind) OVERRIDE;
+
// Check if the desired_dispatch_info is supported. If it is, return it,
// otherwise return a fall-back info that should be used instead.
HInvokeStaticOrDirect::DispatchInfo GetSupportedInvokeStaticOrDirectDispatch(
diff --git a/compiler/optimizing/code_generator_x86.cc b/compiler/optimizing/code_generator_x86.cc
index 394f4ee..715b5be 100644
--- a/compiler/optimizing/code_generator_x86.cc
+++ b/compiler/optimizing/code_generator_x86.cc
@@ -799,6 +799,8 @@
method_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)),
relative_call_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)),
pc_relative_dex_cache_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)),
+ simple_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)),
+ string_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)),
fixups_to_jump_tables_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)) {
// Use a fake return address register to mimic Quick.
AddAllocatedRegister(Location::RegisterLocation(kFakeReturnRegister));
@@ -4340,6 +4342,8 @@
// We disable pc-relative load when there is an irreducible loop, as the optimization
// is incompatible with it.
+ // TODO: Create as many X86ComputeBaseMethodAddress instructions
+ // as needed for methods with irreducible loops.
if (GetGraph()->HasIrreducibleLoops() &&
(dispatch_info.method_load_kind ==
HInvokeStaticOrDirect::MethodLoadKind::kDexCachePcRelative)) {
@@ -4401,18 +4405,17 @@
__ movl(temp.AsRegister<Register>(), Immediate(invoke->GetMethodAddress()));
break;
case HInvokeStaticOrDirect::MethodLoadKind::kDirectAddressWithFixup:
- __ movl(temp.AsRegister<Register>(), Immediate(0)); // Placeholder.
+ __ movl(temp.AsRegister<Register>(), Immediate(/* placeholder */ 0));
method_patches_.emplace_back(invoke->GetTargetMethod());
__ Bind(&method_patches_.back().label); // Bind the label at the end of the "movl" insn.
break;
case HInvokeStaticOrDirect::MethodLoadKind::kDexCachePcRelative: {
Register base_reg = GetInvokeStaticOrDirectExtraParameter(invoke,
temp.AsRegister<Register>());
- uint32_t offset = invoke->GetDexCacheArrayOffset();
__ movl(temp.AsRegister<Register>(), Address(base_reg, kDummy32BitOffset));
- // Add the patch entry and bind its label at the end of the instruction.
- pc_relative_dex_cache_patches_.emplace_back(*invoke->GetTargetMethod().dex_file, offset);
- __ Bind(&pc_relative_dex_cache_patches_.back().label);
+ // Bind a new fixup label at the end of the "movl" insn.
+ uint32_t offset = invoke->GetDexCacheArrayOffset();
+ __ Bind(NewPcRelativeDexCacheArrayPatch(*invoke->GetTargetMethod().dex_file, offset));
break;
}
case HInvokeStaticOrDirect::MethodLoadKind::kDexCacheViaMethod: {
@@ -4494,12 +4497,33 @@
temp, ArtMethod::EntryPointFromQuickCompiledCodeOffset(kX86WordSize).Int32Value()));
}
+void CodeGeneratorX86::RecordSimplePatch() {
+ if (GetCompilerOptions().GetIncludePatchInformation()) {
+ simple_patches_.emplace_back();
+ __ Bind(&simple_patches_.back());
+ }
+}
+
+void CodeGeneratorX86::RecordStringPatch(HLoadString* load_string) {
+ string_patches_.emplace_back(load_string->GetDexFile(), load_string->GetStringIndex());
+ __ Bind(&string_patches_.back().label);
+}
+
+Label* CodeGeneratorX86::NewPcRelativeDexCacheArrayPatch(const DexFile& dex_file,
+ uint32_t element_offset) {
+ // Add the patch entry and bind its label at the end of the instruction.
+ pc_relative_dex_cache_patches_.emplace_back(dex_file, element_offset);
+ return &pc_relative_dex_cache_patches_.back().label;
+}
+
void CodeGeneratorX86::EmitLinkerPatches(ArenaVector<LinkerPatch>* linker_patches) {
DCHECK(linker_patches->empty());
size_t size =
method_patches_.size() +
relative_call_patches_.size() +
- pc_relative_dex_cache_patches_.size();
+ pc_relative_dex_cache_patches_.size() +
+ simple_patches_.size() +
+ string_patches_.size();
linker_patches->reserve(size);
// The label points to the end of the "movl" insn but the literal offset for method
// patch needs to point to the embedded constant which occupies the last 4 bytes.
@@ -4523,6 +4547,26 @@
GetMethodAddressOffset(),
info.element_offset));
}
+ for (const Label& label : simple_patches_) {
+ uint32_t literal_offset = label.Position() - kLabelPositionToLiteralOffsetAdjustment;
+ linker_patches->push_back(LinkerPatch::RecordPosition(literal_offset));
+ }
+ if (GetCompilerOptions().GetCompilePic()) {
+ for (const StringPatchInfo<Label>& info : string_patches_) {
+ uint32_t literal_offset = info.label.Position() - kLabelPositionToLiteralOffsetAdjustment;
+ linker_patches->push_back(LinkerPatch::RelativeStringPatch(literal_offset,
+ &info.dex_file,
+ GetMethodAddressOffset(),
+ info.string_index));
+ }
+ } else {
+ for (const StringPatchInfo<Label>& info : string_patches_) {
+ uint32_t literal_offset = info.label.Position() - kLabelPositionToLiteralOffsetAdjustment;
+ linker_patches->push_back(LinkerPatch::StringPatch(literal_offset,
+ &info.dex_file,
+ info.string_index));
+ }
+ }
}
void CodeGeneratorX86::MarkGCCard(Register temp,
@@ -5916,14 +5960,15 @@
DCHECK(!cls->MustGenerateClinitCheck());
// /* GcRoot<mirror::Class> */ out = current_method->declaring_class_
GenerateGcRootFieldLoad(
- cls, out_loc, current_method, ArtMethod::DeclaringClassOffset().Int32Value());
+ cls, out_loc, Address(current_method, ArtMethod::DeclaringClassOffset().Int32Value()));
} else {
// /* GcRoot<mirror::Class>[] */ out =
// current_method.ptr_sized_fields_->dex_cache_resolved_types_
__ movl(out, Address(current_method,
ArtMethod::DexCacheResolvedTypesOffset(kX86PointerSize).Int32Value()));
// /* GcRoot<mirror::Class> */ out = out[type_index]
- GenerateGcRootFieldLoad(cls, out_loc, out, CodeGenerator::GetCacheOffset(cls->GetTypeIndex()));
+ GenerateGcRootFieldLoad(
+ cls, out_loc, Address(out, CodeGenerator::GetCacheOffset(cls->GetTypeIndex())));
if (!cls->IsInDexCache() || cls->MustGenerateClinitCheck()) {
DCHECK(cls->CanCallRuntime());
@@ -5972,12 +6017,58 @@
// No need for memory fence, thanks to the X86 memory model.
}
+HLoadString::LoadKind CodeGeneratorX86::GetSupportedLoadStringKind(
+ HLoadString::LoadKind desired_string_load_kind) {
+ if (kEmitCompilerReadBarrier) {
+ switch (desired_string_load_kind) {
+ case HLoadString::LoadKind::kBootImageLinkTimeAddress:
+ case HLoadString::LoadKind::kBootImageLinkTimePcRelative:
+ case HLoadString::LoadKind::kBootImageAddress:
+ // TODO: Implement for read barrier.
+ return HLoadString::LoadKind::kDexCacheViaMethod;
+ default:
+ break;
+ }
+ }
+ switch (desired_string_load_kind) {
+ case HLoadString::LoadKind::kBootImageLinkTimeAddress:
+ DCHECK(!GetCompilerOptions().GetCompilePic());
+ break;
+ case HLoadString::LoadKind::kBootImageLinkTimePcRelative:
+ DCHECK(GetCompilerOptions().GetCompilePic());
+ FALLTHROUGH_INTENDED;
+ case HLoadString::LoadKind::kDexCachePcRelative:
+ DCHECK(!Runtime::Current()->UseJit()); // Note: boot image is also non-JIT.
+ // We disable pc-relative load when there is an irreducible loop, as the optimization
+ // is incompatible with it.
+ // TODO: Create as many X86ComputeBaseMethodAddress instructions as needed for methods
+ // with irreducible loops.
+ if (GetGraph()->HasIrreducibleLoops()) {
+ return HLoadString::LoadKind::kDexCacheViaMethod;
+ }
+ break;
+ case HLoadString::LoadKind::kBootImageAddress:
+ break;
+ case HLoadString::LoadKind::kDexCacheAddress:
+ DCHECK(Runtime::Current()->UseJit());
+ break;
+ case HLoadString::LoadKind::kDexCacheViaMethod:
+ break;
+ }
+ return desired_string_load_kind;
+}
+
void LocationsBuilderX86::VisitLoadString(HLoadString* load) {
- LocationSummary::CallKind call_kind = (!load->IsInDexCache() || kEmitCompilerReadBarrier)
+ LocationSummary::CallKind call_kind = (load->NeedsEnvironment() || kEmitCompilerReadBarrier)
? LocationSummary::kCallOnSlowPath
: LocationSummary::kNoCall;
LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(load, call_kind);
- locations->SetInAt(0, Location::RequiresRegister());
+ HLoadString::LoadKind load_kind = load->GetLoadKind();
+ if (load_kind == HLoadString::LoadKind::kDexCacheViaMethod ||
+ load_kind == HLoadString::LoadKind::kBootImageLinkTimePcRelative ||
+ load_kind == HLoadString::LoadKind::kDexCachePcRelative) {
+ locations->SetInAt(0, Location::RequiresRegister());
+ }
locations->SetOut(Location::RequiresRegister());
}
@@ -5985,16 +6076,61 @@
LocationSummary* locations = load->GetLocations();
Location out_loc = locations->Out();
Register out = out_loc.AsRegister<Register>();
- Register current_method = locations->InAt(0).AsRegister<Register>();
- // /* GcRoot<mirror::Class> */ out = current_method->declaring_class_
- GenerateGcRootFieldLoad(
- load, out_loc, current_method, ArtMethod::DeclaringClassOffset().Int32Value());
- // /* GcRoot<mirror::String>[] */ out = out->dex_cache_strings_
- __ movl(out, Address(out, mirror::Class::DexCacheStringsOffset().Int32Value()));
- // /* GcRoot<mirror::String> */ out = out[string_index]
- GenerateGcRootFieldLoad(
- load, out_loc, out, CodeGenerator::GetCacheOffset(load->GetStringIndex()));
+ switch (load->GetLoadKind()) {
+ case HLoadString::LoadKind::kBootImageLinkTimeAddress: {
+ DCHECK(!kEmitCompilerReadBarrier);
+ __ movl(out, Immediate(/* placeholder */ 0));
+ codegen_->RecordStringPatch(load);
+ return; // No dex cache slow path.
+ }
+ case HLoadString::LoadKind::kBootImageLinkTimePcRelative: {
+ DCHECK(!kEmitCompilerReadBarrier);
+ Register method_address = locations->InAt(0).AsRegister<Register>();
+ __ leal(out, Address(method_address, CodeGeneratorX86::kDummy32BitOffset));
+ codegen_->RecordStringPatch(load);
+ return; // No dex cache slow path.
+ }
+ case HLoadString::LoadKind::kBootImageAddress: {
+ DCHECK(!kEmitCompilerReadBarrier);
+ DCHECK_NE(load->GetAddress(), 0u);
+ uint32_t address = dchecked_integral_cast<uint32_t>(load->GetAddress());
+ __ movl(out, Immediate(address));
+ codegen_->RecordSimplePatch();
+ return; // No dex cache slow path.
+ }
+ case HLoadString::LoadKind::kDexCacheAddress: {
+ DCHECK_NE(load->GetAddress(), 0u);
+ uint32_t address = dchecked_integral_cast<uint32_t>(load->GetAddress());
+ GenerateGcRootFieldLoad(load, out_loc, Address::Absolute(address));
+ break;
+ }
+ case HLoadString::LoadKind::kDexCachePcRelative: {
+ Register base_reg = locations->InAt(0).AsRegister<Register>();
+ uint32_t offset = load->GetDexCacheElementOffset();
+ Label* fixup_label = codegen_->NewPcRelativeDexCacheArrayPatch(load->GetDexFile(), offset);
+ GenerateGcRootFieldLoad(
+ load, out_loc, Address(base_reg, CodeGeneratorX86::kDummy32BitOffset), fixup_label);
+ break;
+ }
+ case HLoadString::LoadKind::kDexCacheViaMethod: {
+ Register current_method = locations->InAt(0).AsRegister<Register>();
+
+ // /* GcRoot<mirror::Class> */ out = current_method->declaring_class_
+ GenerateGcRootFieldLoad(
+ load, out_loc, Address(current_method, ArtMethod::DeclaringClassOffset().Int32Value()));
+
+ // /* GcRoot<mirror::String>[] */ out = out->dex_cache_strings_
+ __ movl(out, Address(out, mirror::Class::DexCacheStringsOffset().Int32Value()));
+ // /* GcRoot<mirror::String> */ out = out[string_index]
+ GenerateGcRootFieldLoad(
+ load, out_loc, Address(out, CodeGenerator::GetCacheOffset(load->GetStringIndex())));
+ break;
+ }
+ default:
+ LOG(FATAL) << "Unexpected load kind: " << load->GetLoadKind();
+ UNREACHABLE();
+ }
if (!load->IsInDexCache()) {
SlowPathCode* slow_path = new (GetGraph()->GetArena()) LoadStringSlowPathX86(load);
@@ -6692,21 +6828,24 @@
void InstructionCodeGeneratorX86::GenerateGcRootFieldLoad(HInstruction* instruction,
Location root,
- Register obj,
- uint32_t offset) {
+ const Address& address,
+ Label* fixup_label) {
Register root_reg = root.AsRegister<Register>();
if (kEmitCompilerReadBarrier) {
if (kUseBakerReadBarrier) {
// Fast path implementation of art::ReadBarrier::BarrierForRoot when
// Baker's read barrier are used:
//
- // root = obj.field;
+ // root = *address;
// if (Thread::Current()->GetIsGcMarking()) {
// root = ReadBarrier::Mark(root)
// }
- // /* GcRoot<mirror::Object> */ root = *(obj + offset)
- __ movl(root_reg, Address(obj, offset));
+ // /* GcRoot<mirror::Object> */ root = *address
+ __ movl(root_reg, address);
+ if (fixup_label != nullptr) {
+ __ Bind(fixup_label);
+ }
static_assert(
sizeof(mirror::CompressedReference<mirror::Object>) == sizeof(GcRoot<mirror::Object>),
"art::mirror::CompressedReference<mirror::Object> and art::GcRoot<mirror::Object> "
@@ -6727,15 +6866,21 @@
} else {
// GC root loaded through a slow path for read barriers other
// than Baker's.
- // /* GcRoot<mirror::Object>* */ root = obj + offset
- __ leal(root_reg, Address(obj, offset));
+ // /* GcRoot<mirror::Object>* */ root = address
+ __ leal(root_reg, address);
+ if (fixup_label != nullptr) {
+ __ Bind(fixup_label);
+ }
// /* mirror::Object* */ root = root->Read()
codegen_->GenerateReadBarrierForRootSlow(instruction, root, root);
}
} else {
// Plain GC root load with no read barrier.
- // /* GcRoot<mirror::Object> */ root = *(obj + offset)
- __ movl(root_reg, Address(obj, offset));
+ // /* GcRoot<mirror::Object> */ root = *address
+ __ movl(root_reg, address);
+ if (fixup_label != nullptr) {
+ __ Bind(fixup_label);
+ }
// Note that GC roots are not affected by heap poisoning, thus we
// do not have to unpoison `root_reg` here.
}
diff --git a/compiler/optimizing/code_generator_x86.h b/compiler/optimizing/code_generator_x86.h
index c397899..1fa22fc 100644
--- a/compiler/optimizing/code_generator_x86.h
+++ b/compiler/optimizing/code_generator_x86.h
@@ -258,13 +258,13 @@
Location maybe_temp);
// Generate a GC root reference load:
//
- // root <- *(obj + offset)
+ // root <- *address
//
// while honoring read barriers (if any).
void GenerateGcRootFieldLoad(HInstruction* instruction,
Location root,
- Register obj,
- uint32_t offset);
+ const Address& address,
+ Label* fixup_label = nullptr);
// Push value to FPU stack. `is_fp` specifies whether the value is floating point or not.
// `is_wide` specifies whether it is long/double or not.
@@ -388,6 +388,11 @@
// Helper method to move a 64bits value between two locations.
void Move64(Location destination, Location source);
+ // Check if the desired_string_load_kind is supported. If it is, return it,
+ // otherwise return a fall-back kind that should be used instead.
+ HLoadString::LoadKind GetSupportedLoadStringKind(
+ HLoadString::LoadKind desired_string_load_kind) OVERRIDE;
+
// Check if the desired_dispatch_info is supported. If it is, return it,
// otherwise return a fall-back info that should be used instead.
HInvokeStaticOrDirect::DispatchInfo GetSupportedInvokeStaticOrDirectDispatch(
@@ -399,6 +404,10 @@
// Generate a call to a virtual method.
void GenerateVirtualCall(HInvokeVirtual* invoke, Location temp) OVERRIDE;
+ void RecordSimplePatch();
+ void RecordStringPatch(HLoadString* load_string);
+ Label* NewPcRelativeDexCacheArrayPatch(const DexFile& dex_file, uint32_t element_offset);
+
void MoveFromReturnRegister(Location trg, Primitive::Type type) OVERRIDE;
// Emit linker patches.
@@ -542,6 +551,10 @@
void GenerateImplicitNullCheck(HNullCheck* instruction);
void GenerateExplicitNullCheck(HNullCheck* instruction);
+ // When we don't know the proper offset for the value, we use kDummy32BitOffset.
+ // The correct value will be inserted when processing Assembler fixups.
+ static constexpr int32_t kDummy32BitOffset = 256;
+
private:
// Factored implementation of GenerateFieldLoadWithBakerReadBarrier
// and GenerateArrayLoadWithBakerReadBarrier.
@@ -578,6 +591,10 @@
ArenaDeque<MethodPatchInfo<Label>> relative_call_patches_;
// PC-relative DexCache access info.
ArenaDeque<PcRelativeDexCacheAccessInfo> pc_relative_dex_cache_patches_;
+ // Patch locations for patchoat where the linker doesn't do any other work.
+ ArenaDeque<Label> simple_patches_;
+ // String patch locations.
+ ArenaDeque<StringPatchInfo<Label>> string_patches_;
// Offset to the start of the constant area in the assembled code.
// Used for fixups to the constant area.
@@ -592,10 +609,6 @@
// instruction gives the address of the start of this method.
int32_t method_address_offset_;
- // When we don't know the proper offset for the value, we use kDummy32BitOffset.
- // The correct value will be inserted when processing Assembler fixups.
- static constexpr int32_t kDummy32BitOffset = 256;
-
DISALLOW_COPY_AND_ASSIGN(CodeGeneratorX86);
};
diff --git a/compiler/optimizing/code_generator_x86_64.cc b/compiler/optimizing/code_generator_x86_64.cc
index d24b5bb..504eaa8 100644
--- a/compiler/optimizing/code_generator_x86_64.cc
+++ b/compiler/optimizing/code_generator_x86_64.cc
@@ -784,14 +784,14 @@
method_patches_.emplace_back(invoke->GetTargetMethod());
__ Bind(&method_patches_.back().label); // Bind the label at the end of the "movl" insn.
break;
- case HInvokeStaticOrDirect::MethodLoadKind::kDexCachePcRelative:
- pc_relative_dex_cache_patches_.emplace_back(*invoke->GetTargetMethod().dex_file,
- invoke->GetDexCacheArrayOffset());
+ case HInvokeStaticOrDirect::MethodLoadKind::kDexCachePcRelative: {
__ movq(temp.AsRegister<CpuRegister>(),
Address::Absolute(kDummy32BitOffset, /* no_rip */ false));
- // Bind the label at the end of the "movl" insn.
- __ Bind(&pc_relative_dex_cache_patches_.back().label);
+ // Bind a new fixup label at the end of the "movl" insn.
+ uint32_t offset = invoke->GetDexCacheArrayOffset();
+ __ Bind(NewPcRelativeDexCacheArrayPatch(*invoke->GetTargetMethod().dex_file, offset));
break;
+ }
case HInvokeStaticOrDirect::MethodLoadKind::kDexCacheViaMethod: {
Location current_method = invoke->GetLocations()->InAt(invoke->GetSpecialInputIndex());
Register method_reg;
@@ -873,12 +873,33 @@
kX86_64WordSize).SizeValue()));
}
+void CodeGeneratorX86_64::RecordSimplePatch() {
+ if (GetCompilerOptions().GetIncludePatchInformation()) {
+ simple_patches_.emplace_back();
+ __ Bind(&simple_patches_.back());
+ }
+}
+
+void CodeGeneratorX86_64::RecordStringPatch(HLoadString* load_string) {
+ string_patches_.emplace_back(load_string->GetDexFile(), load_string->GetStringIndex());
+ __ Bind(&string_patches_.back().label);
+}
+
+Label* CodeGeneratorX86_64::NewPcRelativeDexCacheArrayPatch(const DexFile& dex_file,
+ uint32_t element_offset) {
+ // Add a patch entry and return the label.
+ pc_relative_dex_cache_patches_.emplace_back(dex_file, element_offset);
+ return &pc_relative_dex_cache_patches_.back().label;
+}
+
void CodeGeneratorX86_64::EmitLinkerPatches(ArenaVector<LinkerPatch>* linker_patches) {
DCHECK(linker_patches->empty());
size_t size =
method_patches_.size() +
relative_call_patches_.size() +
- pc_relative_dex_cache_patches_.size();
+ pc_relative_dex_cache_patches_.size() +
+ simple_patches_.size() +
+ string_patches_.size();
linker_patches->reserve(size);
// The label points to the end of the "movl" insn but the literal offset for method
// patch needs to point to the embedded constant which occupies the last 4 bytes.
@@ -902,6 +923,18 @@
info.label.Position(),
info.element_offset));
}
+ for (const Label& label : simple_patches_) {
+ uint32_t literal_offset = label.Position() - kLabelPositionToLiteralOffsetAdjustment;
+ linker_patches->push_back(LinkerPatch::RecordPosition(literal_offset));
+ }
+ for (const StringPatchInfo<Label>& info : string_patches_) {
+ // These are always PC-relative, see GetSupportedLoadStringKind().
+ uint32_t literal_offset = info.label.Position() - kLabelPositionToLiteralOffsetAdjustment;
+ linker_patches->push_back(LinkerPatch::RelativeStringPatch(literal_offset,
+ &info.dex_file,
+ info.label.Position(),
+ info.string_index));
+ }
}
void CodeGeneratorX86_64::DumpCoreRegister(std::ostream& stream, int reg) const {
@@ -978,6 +1011,8 @@
method_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)),
relative_call_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)),
pc_relative_dex_cache_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)),
+ simple_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)),
+ string_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)),
fixups_to_jump_tables_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)) {
AddAllocatedRegister(Location::RegisterLocation(kFakeReturnRegister));
}
@@ -5365,14 +5400,15 @@
DCHECK(!cls->MustGenerateClinitCheck());
// /* GcRoot<mirror::Class> */ out = current_method->declaring_class_
GenerateGcRootFieldLoad(
- cls, out_loc, current_method, ArtMethod::DeclaringClassOffset().Int32Value());
+ cls, out_loc, Address(current_method, ArtMethod::DeclaringClassOffset().Int32Value()));
} else {
// /* GcRoot<mirror::Class>[] */ out =
// current_method.ptr_sized_fields_->dex_cache_resolved_types_
__ movq(out, Address(current_method,
ArtMethod::DexCacheResolvedTypesOffset(kX86_64PointerSize).Int32Value()));
// /* GcRoot<mirror::Class> */ out = out[type_index]
- GenerateGcRootFieldLoad(cls, out_loc, out, CodeGenerator::GetCacheOffset(cls->GetTypeIndex()));
+ GenerateGcRootFieldLoad(
+ cls, out_loc, Address(out, CodeGenerator::GetCacheOffset(cls->GetTypeIndex())));
if (!cls->IsInDexCache() || cls->MustGenerateClinitCheck()) {
DCHECK(cls->CanCallRuntime());
@@ -5410,12 +5446,49 @@
check->GetLocations()->InAt(0).AsRegister<CpuRegister>());
}
+HLoadString::LoadKind CodeGeneratorX86_64::GetSupportedLoadStringKind(
+ HLoadString::LoadKind desired_string_load_kind) {
+ if (kEmitCompilerReadBarrier) {
+ switch (desired_string_load_kind) {
+ case HLoadString::LoadKind::kBootImageLinkTimeAddress:
+ case HLoadString::LoadKind::kBootImageLinkTimePcRelative:
+ case HLoadString::LoadKind::kBootImageAddress:
+ // TODO: Implement for read barrier.
+ return HLoadString::LoadKind::kDexCacheViaMethod;
+ default:
+ break;
+ }
+ }
+ switch (desired_string_load_kind) {
+ case HLoadString::LoadKind::kBootImageLinkTimeAddress:
+ DCHECK(!GetCompilerOptions().GetCompilePic());
+ // We prefer the always-available RIP-relative address for the x86-64 boot image.
+ return HLoadString::LoadKind::kBootImageLinkTimePcRelative;
+ case HLoadString::LoadKind::kBootImageLinkTimePcRelative:
+ DCHECK(GetCompilerOptions().GetCompilePic());
+ break;
+ case HLoadString::LoadKind::kBootImageAddress:
+ break;
+ case HLoadString::LoadKind::kDexCacheAddress:
+ DCHECK(Runtime::Current()->UseJit());
+ break;
+ case HLoadString::LoadKind::kDexCachePcRelative:
+ DCHECK(!Runtime::Current()->UseJit());
+ break;
+ case HLoadString::LoadKind::kDexCacheViaMethod:
+ break;
+ }
+ return desired_string_load_kind;
+}
+
void LocationsBuilderX86_64::VisitLoadString(HLoadString* load) {
- LocationSummary::CallKind call_kind = (!load->IsInDexCache() || kEmitCompilerReadBarrier)
+ LocationSummary::CallKind call_kind = (load->NeedsEnvironment() || kEmitCompilerReadBarrier)
? LocationSummary::kCallOnSlowPath
: LocationSummary::kNoCall;
LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(load, call_kind);
- locations->SetInAt(0, Location::RequiresRegister());
+ if (load->GetLoadKind() == HLoadString::LoadKind::kDexCacheViaMethod) {
+ locations->SetInAt(0, Location::RequiresRegister());
+ }
locations->SetOut(Location::RequiresRegister());
}
@@ -5423,16 +5496,59 @@
LocationSummary* locations = load->GetLocations();
Location out_loc = locations->Out();
CpuRegister out = out_loc.AsRegister<CpuRegister>();
- CpuRegister current_method = locations->InAt(0).AsRegister<CpuRegister>();
- // /* GcRoot<mirror::Class> */ out = current_method->declaring_class_
- GenerateGcRootFieldLoad(
- load, out_loc, current_method, ArtMethod::DeclaringClassOffset().Int32Value());
- // /* GcRoot<mirror::String>[] */ out = out->dex_cache_strings_
- __ movq(out, Address(out, mirror::Class::DexCacheStringsOffset().Uint32Value()));
- // /* GcRoot<mirror::String> */ out = out[string_index]
- GenerateGcRootFieldLoad(
- load, out_loc, out, CodeGenerator::GetCacheOffset(load->GetStringIndex()));
+ switch (load->GetLoadKind()) {
+ case HLoadString::LoadKind::kBootImageLinkTimePcRelative: {
+ DCHECK(!kEmitCompilerReadBarrier);
+ __ leal(out, Address::Absolute(CodeGeneratorX86_64::kDummy32BitOffset, /* no_rip */ false));
+ codegen_->RecordStringPatch(load);
+ return; // No dex cache slow path.
+ }
+ case HLoadString::LoadKind::kBootImageAddress: {
+ DCHECK(!kEmitCompilerReadBarrier);
+ DCHECK_NE(load->GetAddress(), 0u);
+ uint32_t address = dchecked_integral_cast<uint32_t>(load->GetAddress());
+ __ movl(out, Immediate(address)); // Zero-extended.
+ codegen_->RecordSimplePatch();
+ return; // No dex cache slow path.
+ }
+ case HLoadString::LoadKind::kDexCacheAddress: {
+ DCHECK_NE(load->GetAddress(), 0u);
+ if (IsUint<32>(load->GetAddress())) {
+ Address address = Address::Absolute(load->GetAddress(), /* no_rip */ true);
+ GenerateGcRootFieldLoad(load, out_loc, address);
+ } else {
+ // TODO: Consider using opcode A1, i.e. movl eax, moff32 (with 64-bit address).
+ __ movq(out, Immediate(load->GetAddress()));
+ GenerateGcRootFieldLoad(load, out_loc, Address(out, 0));
+ }
+ break;
+ }
+ case HLoadString::LoadKind::kDexCachePcRelative: {
+ uint32_t offset = load->GetDexCacheElementOffset();
+ Label* fixup_label = codegen_->NewPcRelativeDexCacheArrayPatch(load->GetDexFile(), offset);
+ Address address = Address::Absolute(CodeGeneratorX86_64::kDummy32BitOffset,
+ /* no_rip */ false);
+ GenerateGcRootFieldLoad(load, out_loc, address, fixup_label);
+ break;
+ }
+ case HLoadString::LoadKind::kDexCacheViaMethod: {
+ CpuRegister current_method = locations->InAt(0).AsRegister<CpuRegister>();
+
+ // /* GcRoot<mirror::Class> */ out = current_method->declaring_class_
+ GenerateGcRootFieldLoad(
+ load, out_loc, Address(current_method, ArtMethod::DeclaringClassOffset().Int32Value()));
+ // /* GcRoot<mirror::String>[] */ out = out->dex_cache_strings_
+ __ movq(out, Address(out, mirror::Class::DexCacheStringsOffset().Uint32Value()));
+ // /* GcRoot<mirror::String> */ out = out[string_index]
+ GenerateGcRootFieldLoad(
+ load, out_loc, Address(out, CodeGenerator::GetCacheOffset(load->GetStringIndex())));
+ break;
+ }
+ default:
+ LOG(FATAL) << "Unexpected load kind: " << load->GetLoadKind();
+ UNREACHABLE();
+ }
if (!load->IsInDexCache()) {
SlowPathCode* slow_path = new (GetGraph()->GetArena()) LoadStringSlowPathX86_64(load);
@@ -6171,21 +6287,24 @@
void InstructionCodeGeneratorX86_64::GenerateGcRootFieldLoad(HInstruction* instruction,
Location root,
- CpuRegister obj,
- uint32_t offset) {
+ const Address& address,
+ Label* fixup_label) {
CpuRegister root_reg = root.AsRegister<CpuRegister>();
if (kEmitCompilerReadBarrier) {
if (kUseBakerReadBarrier) {
// Fast path implementation of art::ReadBarrier::BarrierForRoot when
// Baker's read barrier are used:
//
- // root = obj.field;
+ // root = *address;
// if (Thread::Current()->GetIsGcMarking()) {
// root = ReadBarrier::Mark(root)
// }
- // /* GcRoot<mirror::Object> */ root = *(obj + offset)
- __ movl(root_reg, Address(obj, offset));
+ // /* GcRoot<mirror::Object> */ root = *address
+ __ movl(root_reg, address);
+ if (fixup_label != nullptr) {
+ __ Bind(fixup_label);
+ }
static_assert(
sizeof(mirror::CompressedReference<mirror::Object>) == sizeof(GcRoot<mirror::Object>),
"art::mirror::CompressedReference<mirror::Object> and art::GcRoot<mirror::Object> "
@@ -6207,15 +6326,21 @@
} else {
// GC root loaded through a slow path for read barriers other
// than Baker's.
- // /* GcRoot<mirror::Object>* */ root = obj + offset
- __ leaq(root_reg, Address(obj, offset));
+ // /* GcRoot<mirror::Object>* */ root = address
+ __ leaq(root_reg, address);
+ if (fixup_label != nullptr) {
+ __ Bind(fixup_label);
+ }
// /* mirror::Object* */ root = root->Read()
codegen_->GenerateReadBarrierForRootSlow(instruction, root, root);
}
} else {
// Plain GC root load with no read barrier.
- // /* GcRoot<mirror::Object> */ root = *(obj + offset)
- __ movl(root_reg, Address(obj, offset));
+ // /* GcRoot<mirror::Object> */ root = *address
+ __ movl(root_reg, address);
+ if (fixup_label != nullptr) {
+ __ Bind(fixup_label);
+ }
// Note that GC roots are not affected by heap poisoning, thus we
// do not have to unpoison `root_reg` here.
}
diff --git a/compiler/optimizing/code_generator_x86_64.h b/compiler/optimizing/code_generator_x86_64.h
index c3fce6e..fdcf3c3 100644
--- a/compiler/optimizing/code_generator_x86_64.h
+++ b/compiler/optimizing/code_generator_x86_64.h
@@ -252,13 +252,13 @@
Location maybe_temp);
// Generate a GC root reference load:
//
- // root <- *(obj + offset)
+ // root <- *address
//
// while honoring read barriers (if any).
void GenerateGcRootFieldLoad(HInstruction* instruction,
Location root,
- CpuRegister obj,
- uint32_t offset);
+ const Address& address,
+ Label* fixup_label = nullptr);
void PushOntoFPStack(Location source, uint32_t temp_offset,
uint32_t stack_adjustment, bool is_float);
@@ -384,6 +384,11 @@
return false;
}
+ // Check if the desired_string_load_kind is supported. If it is, return it,
+ // otherwise return a fall-back kind that should be used instead.
+ HLoadString::LoadKind GetSupportedLoadStringKind(
+ HLoadString::LoadKind desired_string_load_kind) OVERRIDE;
+
// Check if the desired_dispatch_info is supported. If it is, return it,
// otherwise return a fall-back info that should be used instead.
HInvokeStaticOrDirect::DispatchInfo GetSupportedInvokeStaticOrDirectDispatch(
@@ -393,6 +398,10 @@
void GenerateStaticOrDirectCall(HInvokeStaticOrDirect* invoke, Location temp) OVERRIDE;
void GenerateVirtualCall(HInvokeVirtual* invoke, Location temp) OVERRIDE;
+ void RecordSimplePatch();
+ void RecordStringPatch(HLoadString* load_string);
+ Label* NewPcRelativeDexCacheArrayPatch(const DexFile& dex_file, uint32_t element_offset);
+
void MoveFromReturnRegister(Location trg, Primitive::Type type) OVERRIDE;
void EmitLinkerPatches(ArenaVector<LinkerPatch>* linker_patches) OVERRIDE;
@@ -515,6 +524,10 @@
void GenerateImplicitNullCheck(HNullCheck* instruction);
void GenerateExplicitNullCheck(HNullCheck* instruction);
+ // When we don't know the proper offset for the value, we use kDummy32BitOffset.
+ // We will fix this up in the linker later to have the right value.
+ static constexpr int32_t kDummy32BitOffset = 256;
+
private:
// Factored implementation of GenerateFieldLoadWithBakerReadBarrier
// and GenerateArrayLoadWithBakerReadBarrier.
@@ -552,10 +565,10 @@
ArenaDeque<MethodPatchInfo<Label>> relative_call_patches_;
// PC-relative DexCache access info.
ArenaDeque<PcRelativeDexCacheAccessInfo> pc_relative_dex_cache_patches_;
-
- // When we don't know the proper offset for the value, we use kDummy32BitOffset.
- // We will fix this up in the linker later to have the right value.
- static constexpr int32_t kDummy32BitOffset = 256;
+ // Patch locations for patchoat where the linker doesn't do any other work.
+ ArenaDeque<Label> simple_patches_;
+ // String patch locations.
+ ArenaDeque<StringPatchInfo<Label>> string_patches_;
// Fixups for jump tables need to be handled specially.
ArenaVector<JumpTableRIPFixup*> fixups_to_jump_tables_;
diff --git a/compiler/optimizing/dex_cache_array_fixups_arm.cc b/compiler/optimizing/dex_cache_array_fixups_arm.cc
index 3db254a..e9072b9 100644
--- a/compiler/optimizing/dex_cache_array_fixups_arm.cc
+++ b/compiler/optimizing/dex_cache_array_fixups_arm.cc
@@ -44,6 +44,21 @@
}
private:
+ void VisitLoadString(HLoadString* load_string) OVERRIDE {
+ // If this is a load with PC-relative access to the dex cache methods array,
+ // we need to add the dex cache arrays base as the special input.
+ if (load_string->GetLoadKind() == HLoadString::LoadKind::kDexCachePcRelative) {
+ // Initialize base for target dex file if needed.
+ const DexFile& dex_file = load_string->GetDexFile();
+ HArmDexCacheArraysBase* base = GetOrCreateDexCacheArrayBase(dex_file);
+ // Update the element offset in base.
+ DexCacheArraysLayout layout(kArmPointerSize, &dex_file);
+ base->UpdateElementOffset(layout.StringOffset(load_string->GetStringIndex()));
+ // Add the special argument base to the load.
+ load_string->AddSpecialInput(base);
+ }
+ }
+
void VisitInvokeStaticOrDirect(HInvokeStaticOrDirect* invoke) OVERRIDE {
// If this is an invoke with PC-relative access to the dex cache methods array,
// we need to add the dex cache arrays base as the special input.
diff --git a/compiler/optimizing/graph_visualizer.cc b/compiler/optimizing/graph_visualizer.cc
index 4b5b919..bcdd3e7 100644
--- a/compiler/optimizing/graph_visualizer.cc
+++ b/compiler/optimizing/graph_visualizer.cc
@@ -367,6 +367,10 @@
<< load_class->NeedsAccessCheck() << std::noboolalpha;
}
+ void VisitLoadString(HLoadString* load_string) OVERRIDE {
+ StartAttributeStream("load_kind") << load_string->GetLoadKind();
+ }
+
void VisitCheckCast(HCheckCast* check_cast) OVERRIDE {
StartAttributeStream("check_kind") << check_cast->GetTypeCheckKind();
StartAttributeStream("must_do_null_check") << std::boolalpha
diff --git a/compiler/optimizing/nodes.cc b/compiler/optimizing/nodes.cc
index 1a426d5..9504481 100644
--- a/compiler/optimizing/nodes.cc
+++ b/compiler/optimizing/nodes.cc
@@ -86,11 +86,7 @@
}
}
-static void RemoveAsUser(HInstruction* instruction) {
- for (size_t i = 0; i < instruction->InputCount(); i++) {
- instruction->RemoveAsUserOfInput(i);
- }
-
+static void RemoveEnvironmentUses(HInstruction* instruction) {
for (HEnvironment* environment = instruction->GetEnvironment();
environment != nullptr;
environment = environment->GetParent()) {
@@ -102,6 +98,14 @@
}
}
+static void RemoveAsUser(HInstruction* instruction) {
+ for (size_t i = 0; i < instruction->InputCount(); i++) {
+ instruction->RemoveAsUserOfInput(i);
+ }
+
+ RemoveEnvironmentUses(instruction);
+}
+
void HGraph::RemoveInstructionsAsUsersFromDeadBlocks(const ArenaBitVector& visited) const {
for (size_t i = 0; i < blocks_.size(); ++i) {
if (!visited.IsBitSet(i)) {
@@ -1007,6 +1011,11 @@
}
}
+void HInstruction::RemoveEnvironment() {
+ RemoveEnvironmentUses(this);
+ environment_ = nullptr;
+}
+
void HInstruction::ReplaceWith(HInstruction* other) {
DCHECK(other != nullptr);
for (HUseIterator<HInstruction*> it(GetUses()); !it.Done(); it.Advance()) {
@@ -2387,6 +2396,59 @@
}
}
+bool HLoadString::InstructionDataEquals(HInstruction* other) const {
+ HLoadString* other_load_string = other->AsLoadString();
+ if (string_index_ != other_load_string->string_index_ ||
+ GetPackedFields() != other_load_string->GetPackedFields()) {
+ return false;
+ }
+ LoadKind load_kind = GetLoadKind();
+ if (HasAddress(load_kind)) {
+ return GetAddress() == other_load_string->GetAddress();
+ } else if (HasStringReference(load_kind)) {
+ return IsSameDexFile(GetDexFile(), other_load_string->GetDexFile());
+ } else {
+ DCHECK(HasDexCacheReference(load_kind)) << load_kind;
+ // If the string indexes and dex files are the same, dex cache element offsets
+ // must also be the same, so we don't need to compare them.
+ return IsSameDexFile(GetDexFile(), other_load_string->GetDexFile());
+ }
+}
+
+void HLoadString::SetLoadKindInternal(LoadKind load_kind) {
+ // Once sharpened, the load kind should not be changed again.
+ DCHECK_EQ(GetLoadKind(), LoadKind::kDexCacheViaMethod);
+ SetPackedField<LoadKindField>(load_kind);
+
+ if (load_kind != LoadKind::kDexCacheViaMethod) {
+ RemoveAsUserOfInput(0u);
+ SetRawInputAt(0u, nullptr);
+ }
+ if (!NeedsEnvironment()) {
+ RemoveEnvironment();
+ }
+}
+
+std::ostream& operator<<(std::ostream& os, HLoadString::LoadKind rhs) {
+ switch (rhs) {
+ case HLoadString::LoadKind::kBootImageLinkTimeAddress:
+ return os << "BootImageLinkTimeAddress";
+ case HLoadString::LoadKind::kBootImageLinkTimePcRelative:
+ return os << "BootImageLinkTimePcRelative";
+ case HLoadString::LoadKind::kBootImageAddress:
+ return os << "BootImageAddress";
+ case HLoadString::LoadKind::kDexCacheAddress:
+ return os << "DexCacheAddress";
+ case HLoadString::LoadKind::kDexCachePcRelative:
+ return os << "DexCachePcRelative";
+ case HLoadString::LoadKind::kDexCacheViaMethod:
+ return os << "DexCacheViaMethod";
+ default:
+ LOG(FATAL) << "Unknown HLoadString::LoadKind: " << static_cast<int>(rhs);
+ UNREACHABLE();
+ }
+}
+
void HInstruction::RemoveEnvironmentUsers() {
for (HUseIterator<HEnvironment*> use_it(GetEnvUses()); !use_it.Done(); use_it.Advance()) {
HUseListNode<HEnvironment*>* user_node = use_it.Current();
diff --git a/compiler/optimizing/nodes.h b/compiler/optimizing/nodes.h
index e9a42cb..ba42421 100644
--- a/compiler/optimizing/nodes.h
+++ b/compiler/optimizing/nodes.h
@@ -1995,6 +1995,8 @@
environment_ = environment;
}
+ void RemoveEnvironment();
+
// Set the environment of this instruction, copying it from `environment`. While
// copying, the uses lists are being updated.
void CopyEnvironmentFrom(HEnvironment* environment) {
@@ -5557,32 +5559,117 @@
class HLoadString : public HExpression<1> {
public:
+ // Determines how to load the String.
+ enum class LoadKind {
+ // Use boot image String* address that will be known at link time.
+ // Used for boot image strings referenced by boot image code in non-PIC mode.
+ kBootImageLinkTimeAddress,
+
+ // Use PC-relative boot image String* address that will be known at link time.
+ // Used for boot image strings referenced by boot image code in PIC mode.
+ kBootImageLinkTimePcRelative,
+
+ // Use a known boot image String* address, embedded in the code by the codegen.
+ // Used for boot image strings referenced by apps in AOT- and JIT-compiled code.
+ // Note: codegen needs to emit a linker patch if indicated by compiler options'
+ // GetIncludePatchInformation().
+ kBootImageAddress,
+
+ // Load from the resolved strings array at an absolute address.
+ // Used for strings outside the boot image referenced by JIT-compiled code.
+ kDexCacheAddress,
+
+ // Load from resolved strings array in the dex cache using a PC-relative load.
+ // Used for strings outside boot image when we know that we can access
+ // the dex cache arrays using a PC-relative load.
+ kDexCachePcRelative,
+
+ // Load from resolved strings array accessed through the class loaded from
+ // the compiled method's own ArtMethod*. This is the default access type when
+ // all other types are unavailable.
+ kDexCacheViaMethod,
+
+ kLast = kDexCacheViaMethod
+ };
+
HLoadString(HCurrentMethod* current_method,
uint32_t string_index,
- uint32_t dex_pc,
- bool is_in_dex_cache)
+ const DexFile& dex_file,
+ uint32_t dex_pc)
: HExpression(Primitive::kPrimNot, SideEffectsForArchRuntimeCalls(), dex_pc),
string_index_(string_index) {
- SetPackedFlag<kFlagIsInDexCache>(is_in_dex_cache);
+ SetPackedFlag<kFlagIsInDexCache>(false);
+ SetPackedField<LoadKindField>(LoadKind::kDexCacheViaMethod);
+ load_data_.ref.dex_file = &dex_file;
SetRawInputAt(0, current_method);
}
+ void SetLoadKindWithAddress(LoadKind load_kind, uint64_t address) {
+ DCHECK(HasAddress(load_kind));
+ load_data_.address = address;
+ SetLoadKindInternal(load_kind);
+ }
+
+ void SetLoadKindWithStringReference(LoadKind load_kind,
+ const DexFile& dex_file,
+ uint32_t string_index) {
+ DCHECK(HasStringReference(load_kind));
+ load_data_.ref.dex_file = &dex_file;
+ string_index_ = string_index;
+ SetLoadKindInternal(load_kind);
+ }
+
+ void SetLoadKindWithDexCacheReference(LoadKind load_kind,
+ const DexFile& dex_file,
+ uint32_t element_index) {
+ DCHECK(HasDexCacheReference(load_kind));
+ load_data_.ref.dex_file = &dex_file;
+ load_data_.ref.dex_cache_element_index = element_index;
+ SetLoadKindInternal(load_kind);
+ }
+
+ LoadKind GetLoadKind() const {
+ return GetPackedField<LoadKindField>();
+ }
+
+ const DexFile& GetDexFile() const;
+
+ uint32_t GetStringIndex() const {
+ DCHECK(HasStringReference(GetLoadKind()) || /* For slow paths. */ !IsInDexCache());
+ return string_index_;
+ }
+
+ uint32_t GetDexCacheElementOffset() const;
+
+ uint64_t GetAddress() const {
+ DCHECK(HasAddress(GetLoadKind()));
+ return load_data_.address;
+ }
+
bool CanBeMoved() const OVERRIDE { return true; }
- bool InstructionDataEquals(HInstruction* other) const OVERRIDE {
- return other->AsLoadString()->string_index_ == string_index_;
- }
+ bool InstructionDataEquals(HInstruction* other) const OVERRIDE;
size_t ComputeHashCode() const OVERRIDE { return string_index_; }
- uint32_t GetStringIndex() const { return string_index_; }
+ // Will call the runtime if we need to load the string through
+ // the dex cache and the string is not guaranteed to be there yet.
+ bool NeedsEnvironment() const OVERRIDE {
+ LoadKind load_kind = GetLoadKind();
+ if (load_kind == LoadKind::kBootImageLinkTimeAddress ||
+ load_kind == LoadKind::kBootImageLinkTimePcRelative ||
+ load_kind == LoadKind::kBootImageAddress) {
+ return false;
+ }
+ return !IsInDexCache();
+ }
- // Will call the runtime if the string is not already in the dex cache.
- bool NeedsEnvironment() const OVERRIDE { return !IsInDexCache(); }
+ bool NeedsDexCacheOfDeclaringClass() const OVERRIDE {
+ return GetLoadKind() == LoadKind::kDexCacheViaMethod;
+ }
- bool NeedsDexCacheOfDeclaringClass() const OVERRIDE { return true; }
bool CanBeNull() const OVERRIDE { return false; }
- bool CanThrow() const OVERRIDE { return !IsInDexCache(); }
+ bool CanThrow() const OVERRIDE { return NeedsEnvironment(); }
static SideEffects SideEffectsForArchRuntimeCalls() {
return SideEffects::CanTriggerGC();
@@ -5590,17 +5677,83 @@
bool IsInDexCache() const { return GetPackedFlag<kFlagIsInDexCache>(); }
+ void MarkInDexCache() {
+ SetPackedFlag<kFlagIsInDexCache>(true);
+ DCHECK(!NeedsEnvironment());
+ RemoveEnvironment();
+ }
+
+ size_t InputCount() const OVERRIDE {
+ return (InputAt(0) != nullptr) ? 1u : 0u;
+ }
+
+ void AddSpecialInput(HInstruction* special_input);
+
DECLARE_INSTRUCTION(LoadString);
private:
static constexpr size_t kFlagIsInDexCache = kNumberOfExpressionPackedBits;
- static constexpr size_t kNumberOfLoadStringPackedBits = kFlagIsInDexCache + 1;
+ static constexpr size_t kFieldLoadKind = kFlagIsInDexCache + 1;
+ static constexpr size_t kFieldLoadKindSize =
+ MinimumBitsToStore(static_cast<size_t>(LoadKind::kLast));
+ static constexpr size_t kNumberOfLoadStringPackedBits = kFieldLoadKind + kFieldLoadKindSize;
static_assert(kNumberOfLoadStringPackedBits <= kMaxNumberOfPackedBits, "Too many packed fields.");
+ using LoadKindField = BitField<LoadKind, kFieldLoadKind, kFieldLoadKindSize>;
- const uint32_t string_index_;
+ static bool HasStringReference(LoadKind load_kind) {
+ return load_kind == LoadKind::kBootImageLinkTimeAddress ||
+ load_kind == LoadKind::kBootImageLinkTimePcRelative ||
+ load_kind == LoadKind::kDexCacheViaMethod;
+ }
+
+ static bool HasAddress(LoadKind load_kind) {
+ return load_kind == LoadKind::kBootImageAddress || load_kind == LoadKind::kDexCacheAddress;
+ }
+
+ static bool HasDexCacheReference(LoadKind load_kind) {
+ return load_kind == LoadKind::kDexCachePcRelative;
+ }
+
+ void SetLoadKindInternal(LoadKind load_kind);
+
+ // String index serves also as the hash code and it's also needed for slow-paths,
+ // so it must not be overwritten with other load data.
+ uint32_t string_index_;
+
+ union {
+ struct {
+ const DexFile* dex_file; // For string reference and dex cache reference.
+ uint32_t dex_cache_element_index; // Only for dex cache reference.
+ } ref;
+ uint64_t address; // Up to 64-bit, needed for kDexCacheAddress on 64-bit targets.
+ } load_data_;
DISALLOW_COPY_AND_ASSIGN(HLoadString);
};
+std::ostream& operator<<(std::ostream& os, HLoadString::LoadKind rhs);
+
+// Note: defined outside class to see operator<<(., HLoadString::LoadKind).
+inline const DexFile& HLoadString::GetDexFile() const {
+ DCHECK(HasStringReference(GetLoadKind()) || HasDexCacheReference(GetLoadKind()))
+ << GetLoadKind();
+ return *load_data_.ref.dex_file;
+}
+
+// Note: defined outside class to see operator<<(., HLoadString::LoadKind).
+inline uint32_t HLoadString::GetDexCacheElementOffset() const {
+ DCHECK(HasDexCacheReference(GetLoadKind())) << GetLoadKind();
+ return load_data_.ref.dex_cache_element_index;
+}
+
+// Note: defined outside class to see operator<<(., HLoadString::LoadKind).
+inline void HLoadString::AddSpecialInput(HInstruction* special_input) {
+ // The special input is used for PC-relative loads on some architectures.
+ DCHECK(GetLoadKind() == LoadKind::kBootImageLinkTimePcRelative ||
+ GetLoadKind() == LoadKind::kDexCachePcRelative) << GetLoadKind();
+ DCHECK(InputAt(0) == nullptr);
+ SetRawInputAt(0u, special_input);
+ special_input->AddUseAt(this, 0);
+}
/**
* Performs an initialization check on its Class object input.
diff --git a/compiler/optimizing/pc_relative_fixups_x86.cc b/compiler/optimizing/pc_relative_fixups_x86.cc
index d281a9f..dafbd3d 100644
--- a/compiler/optimizing/pc_relative_fixups_x86.cc
+++ b/compiler/optimizing/pc_relative_fixups_x86.cc
@@ -80,6 +80,15 @@
HandleInvoke(invoke);
}
+ void VisitLoadString(HLoadString* load_string) OVERRIDE {
+ HLoadString::LoadKind load_kind = load_string->GetLoadKind();
+ if (load_kind == HLoadString::LoadKind::kBootImageLinkTimePcRelative ||
+ load_kind == HLoadString::LoadKind::kDexCachePcRelative) {
+ InitializePCRelativeBasePointer();
+ load_string->AddSpecialInput(base_);
+ }
+ }
+
void BinaryFP(HBinaryOperation* bin) {
HConstant* rhs = bin->InputAt(1)->AsConstant();
if (rhs != nullptr && Primitive::IsFloatingPointType(rhs->GetType())) {
diff --git a/compiler/optimizing/sharpening.cc b/compiler/optimizing/sharpening.cc
index 5e1d1d9..45ae336 100644
--- a/compiler/optimizing/sharpening.cc
+++ b/compiler/optimizing/sharpening.cc
@@ -16,11 +16,19 @@
#include "sharpening.h"
+#include "class_linker.h"
#include "code_generator.h"
+#include "driver/dex_compilation_unit.h"
#include "utils/dex_cache_arrays_layout-inl.h"
#include "driver/compiler_driver.h"
+#include "gc/heap.h"
+#include "gc/space/image_space.h"
+#include "handle_scope-inl.h"
+#include "mirror/dex_cache.h"
+#include "mirror/string.h"
#include "nodes.h"
#include "runtime.h"
+#include "scoped_thread_state_change.h"
namespace art {
@@ -31,12 +39,13 @@
HInstruction* instruction = it.Current();
if (instruction->IsInvokeStaticOrDirect()) {
ProcessInvokeStaticOrDirect(instruction->AsInvokeStaticOrDirect());
+ } else if (instruction->IsLoadString()) {
+ ProcessLoadString(instruction->AsLoadString());
}
// TODO: Move the sharpening of invoke-virtual/-interface/-super from HGraphBuilder
// here. Rewrite it to avoid the CompilerDriver's reliance on verifier data
// because we know the type better when inlining.
- // TODO: HLoadClass, HLoadString - select PC relative dex cache array access if
- // available.
+ // TODO: HLoadClass - select better load kind if available.
}
}
}
@@ -143,4 +152,105 @@
invoke->SetDispatchInfo(dispatch_info);
}
+void HSharpening::ProcessLoadString(HLoadString* load_string) {
+ DCHECK_EQ(load_string->GetLoadKind(), HLoadString::LoadKind::kDexCacheViaMethod);
+ DCHECK(!load_string->IsInDexCache());
+
+ const DexFile& dex_file = load_string->GetDexFile();
+ uint32_t string_index = load_string->GetStringIndex();
+
+ bool is_in_dex_cache = false;
+ HLoadString::LoadKind desired_load_kind;
+ uint64_t address = 0u; // String or dex cache element address.
+ {
+ Runtime* runtime = Runtime::Current();
+ ClassLinker* class_linker = runtime->GetClassLinker();
+ ScopedObjectAccess soa(Thread::Current());
+ StackHandleScope<1> hs(soa.Self());
+ Handle<mirror::DexCache> dex_cache = IsSameDexFile(dex_file, *compilation_unit_.GetDexFile())
+ ? compilation_unit_.GetDexCache()
+ : hs.NewHandle(class_linker->FindDexCache(soa.Self(), dex_file));
+
+ if (compiler_driver_->IsBootImage()) {
+ // Compiling boot image. Resolve the string and allocate it if needed.
+ DCHECK(!runtime->UseJit());
+ mirror::String* string = class_linker->ResolveString(dex_file, string_index, dex_cache);
+ CHECK(string != nullptr);
+ if (!compiler_driver_->GetSupportBootImageFixup()) {
+ // MIPS/MIPS64 or compiler_driver_test. Do not sharpen.
+ desired_load_kind = HLoadString::LoadKind::kDexCacheViaMethod;
+ } else {
+ DCHECK(ContainsElement(compiler_driver_->GetDexFilesForOatFile(),
+ &load_string->GetDexFile()));
+ is_in_dex_cache = true;
+ desired_load_kind = codegen_->GetCompilerOptions().GetCompilePic()
+ ? HLoadString::LoadKind::kBootImageLinkTimePcRelative
+ : HLoadString::LoadKind::kBootImageLinkTimeAddress;
+ }
+ } else {
+ // Not compiling boot image. Try to lookup the string without allocating if not found.
+ mirror::String* string = class_linker->LookupString(dex_file, string_index, dex_cache);
+ if (runtime->UseJit()) {
+ // TODO: Make sure we don't set the "compile PIC" flag for JIT as that's bogus.
+ // DCHECK(!codegen_->GetCompilerOptions().GetCompilePic());
+ is_in_dex_cache = (string != nullptr);
+ if (string != nullptr && runtime->GetHeap()->ObjectIsInBootImageSpace(string)) {
+ desired_load_kind = HLoadString::LoadKind::kBootImageAddress;
+ // Convert to uintptr_t first to avoid sign-extension if a 32-bit pointer is "signed."
+ address = static_cast<uint64_t>(reinterpret_cast<uintptr_t>(string));
+ } else {
+ // Note: If the string is not in the dex cache, the instruction needs environment
+ // and will not be inlined across dex files. Within a dex file, the slow-path helper
+ // loads the correct string and inlined frames are used correctly for OOM stack trace.
+ // TODO: Write a test for this.
+ desired_load_kind = HLoadString::LoadKind::kDexCacheAddress;
+ void* dex_cache_element_address = &dex_cache->GetStrings()[string_index];
+ // Convert to uintptr_t first to avoid sign-extension if a 32-bit pointer is "signed."
+ address = static_cast<uint64_t>(reinterpret_cast<uintptr_t>(dex_cache_element_address));
+ }
+ } else if (string != nullptr && runtime->GetHeap()->ObjectIsInBootImageSpace(string)) {
+ if (codegen_->GetCompilerOptions().GetCompilePic()) {
+ // Use PC-relative load from the dex cache if the dex file belongs
+ // to the oat file that we're currently compiling.
+ desired_load_kind =
+ ContainsElement(compiler_driver_->GetDexFilesForOatFile(), &load_string->GetDexFile())
+ ? HLoadString::LoadKind::kDexCachePcRelative
+ : HLoadString::LoadKind::kDexCacheViaMethod;
+ } else {
+ desired_load_kind = HLoadString::LoadKind::kBootImageAddress;
+ // Convert to uintptr_t first to avoid sign-extension if a 32-bit pointer is "signed."
+ address = static_cast<uint64_t>(reinterpret_cast<uintptr_t>(string));
+ }
+ } else {
+ // Not JIT and the string is not in boot image.
+ desired_load_kind = HLoadString::LoadKind::kDexCachePcRelative;
+ }
+ }
+ }
+ if (is_in_dex_cache) {
+ load_string->MarkInDexCache();
+ }
+
+ HLoadString::LoadKind load_kind = codegen_->GetSupportedLoadStringKind(desired_load_kind);
+ switch (load_kind) {
+ case HLoadString::LoadKind::kBootImageLinkTimeAddress:
+ case HLoadString::LoadKind::kBootImageLinkTimePcRelative:
+ case HLoadString::LoadKind::kDexCacheViaMethod:
+ load_string->SetLoadKindWithStringReference(load_kind, dex_file, string_index);
+ break;
+ case HLoadString::LoadKind::kBootImageAddress:
+ case HLoadString::LoadKind::kDexCacheAddress:
+ DCHECK_NE(address, 0u);
+ load_string->SetLoadKindWithAddress(load_kind, address);
+ break;
+ case HLoadString::LoadKind::kDexCachePcRelative: {
+ size_t pointer_size = InstructionSetPointerSize(codegen_->GetInstructionSet());
+ DexCacheArraysLayout layout(pointer_size, &dex_file);
+ size_t element_index = layout.StringOffset(string_index);
+ load_string->SetLoadKindWithDexCacheReference(load_kind, dex_file, element_index);
+ break;
+ }
+ }
+}
+
} // namespace art
diff --git a/compiler/optimizing/sharpening.h b/compiler/optimizing/sharpening.h
index adae700..24152f6 100644
--- a/compiler/optimizing/sharpening.h
+++ b/compiler/optimizing/sharpening.h
@@ -47,6 +47,7 @@
private:
void ProcessInvokeStaticOrDirect(HInvokeStaticOrDirect* invoke);
+ void ProcessLoadString(HLoadString* load_string);
CodeGenerator* codegen_;
const DexCompilationUnit& compilation_unit_;