Merge "Pack stack map entries on bit level to save space."
diff --git a/build/Android.common_path.mk b/build/Android.common_path.mk
index ecc9e76..7be1894 100644
--- a/build/Android.common_path.mk
+++ b/build/Android.common_path.mk
@@ -93,11 +93,10 @@
HOST_CORE_DEX_FILES := $(foreach jar,$(HOST_CORE_JARS), $(call intermediates-dir-for,JAVA_LIBRARIES,$(jar),t,COMMON)/javalib.jar)
TARGET_CORE_DEX_FILES := $(foreach jar,$(TARGET_CORE_JARS),$(call intermediates-dir-for,JAVA_LIBRARIES,$(jar), ,COMMON)/javalib.jar)
-ifeq ($(ANDROID_COMPILE_WITH_JACK),true)
# Classpath for Jack compilation: we only need core-libart.
HOST_JACK_CLASSPATH_DEPENDENCIES := $(call intermediates-dir-for,JAVA_LIBRARIES,core-oj-hostdex,t,COMMON)/classes.jack $(call intermediates-dir-for,JAVA_LIBRARIES,core-libart-hostdex,t,COMMON)/classes.jack
HOST_JACK_CLASSPATH := $(abspath $(call intermediates-dir-for,JAVA_LIBRARIES,core-oj-hostdex,t,COMMON)/classes.jack):$(abspath $(call intermediates-dir-for,JAVA_LIBRARIES,core-libart-hostdex,t,COMMON)/classes.jack)
TARGET_JACK_CLASSPATH_DEPENDENCIES := $(call intermediates-dir-for,JAVA_LIBRARIES,core-oj, ,COMMON)/classes.jack $(call intermediates-dir-for,JAVA_LIBRARIES,core-libart, ,COMMON)/classes.jack
TARGET_JACK_CLASSPATH := $(abspath $(call intermediates-dir-for,JAVA_LIBRARIES,core-oj, ,COMMON)/classes.jack):$(abspath $(call intermediates-dir-for,JAVA_LIBRARIES,core-libart, ,COMMON)/classes.jack)
-endif
+
endif # ART_ANDROID_COMMON_PATH_MK
diff --git a/build/Android.common_test.mk b/build/Android.common_test.mk
index cde41e0..df7df26 100644
--- a/build/Android.common_test.mk
+++ b/build/Android.common_test.mk
@@ -201,7 +201,6 @@
LOCAL_MODULE_PATH := $(3)
LOCAL_DEX_PREOPT_IMAGE_LOCATION := $(TARGET_CORE_IMG_OUT)
ifneq ($(wildcard $(LOCAL_PATH)/$(2)/main.list),)
- LOCAL_DX_FLAGS := --multi-dex --main-dex-list=$(LOCAL_PATH)/$(2)/main.list --minimal-main-dex
LOCAL_JACK_FLAGS := -D jack.dex.output.policy=minimal-multidex -D jack.preprocessor=true -D jack.preprocessor.file=$(LOCAL_PATH)/$(2)/main.jpp
endif
include $(BUILD_JAVA_LIBRARY)
@@ -217,7 +216,6 @@
LOCAL_JAVA_LIBRARIES := $(HOST_CORE_JARS)
LOCAL_DEX_PREOPT_IMAGE := $(HOST_CORE_IMG_LOCATION)
ifneq ($(wildcard $(LOCAL_PATH)/$(2)/main.list),)
- LOCAL_DX_FLAGS := --multi-dex --main-dex-list=$(LOCAL_PATH)/$(2)/main.list --minimal-main-dex
LOCAL_JACK_FLAGS := -D jack.dex.output.policy=minimal-multidex -D jack.preprocessor=true -D jack.preprocessor.file=$(LOCAL_PATH)/$(2)/main.jpp
endif
include $(BUILD_HOST_DALVIK_JAVA_LIBRARY)
diff --git a/compiler/Android.mk b/compiler/Android.mk
index e6ff8f6..9a416e2 100644
--- a/compiler/Android.mk
+++ b/compiler/Android.mk
@@ -148,6 +148,7 @@
LIBART_COMPILER_CFLAGS :=
LIBART_COMPILER_ENUM_OPERATOR_OUT_HEADER_FILES := \
+ compiled_method.h \
dex/compiler_enums.h \
dex/dex_to_dex_compiler.h \
driver/compiler_driver.h \
diff --git a/compiler/compiled_method.h b/compiler/compiled_method.h
index 5887620..70161eb 100644
--- a/compiler/compiled_method.h
+++ b/compiler/compiled_method.h
@@ -18,6 +18,7 @@
#define ART_COMPILER_COMPILED_METHOD_H_
#include <memory>
+#include <iosfwd>
#include <string>
#include <vector>
@@ -158,21 +159,34 @@
using DefaultSrcMap = SrcMap<std::allocator<SrcMapElem>>;
-
-enum LinkerPatchType {
- kLinkerPatchMethod,
- kLinkerPatchCall,
- kLinkerPatchCallRelative, // NOTE: Actual patching is instruction_set-dependent.
- kLinkerPatchType,
- kLinkerPatchDexCacheArray, // NOTE: Actual patching is instruction_set-dependent.
-};
-
class LinkerPatch {
public:
+ // Note: We explicitly specify the underlying type of the enum because GCC
+ // would otherwise select a bigger underlying type and then complain that
+ // 'art::LinkerPatch::patch_type_' is too small to hold all
+ // values of 'enum class art::LinkerPatch::Type'
+ // which is ridiculous given we have only a handful of values here. If we
+ // choose to squeeze the Type into fewer than 8 bits, we'll have to declare
+ // patch_type_ as an uintN_t and do explicit static_cast<>s.
+ enum class Type : uint8_t {
+ kRecordPosition, // Just record patch position for patchoat.
+ kMethod,
+ kCall,
+ kCallRelative, // NOTE: Actual patching is instruction_set-dependent.
+ kType,
+ kString,
+ kStringRelative, // NOTE: Actual patching is instruction_set-dependent.
+ kDexCacheArray, // NOTE: Actual patching is instruction_set-dependent.
+ };
+
+ static LinkerPatch RecordPosition(size_t literal_offset) {
+ return LinkerPatch(literal_offset, Type::kRecordPosition, /* target_dex_file */ nullptr);
+ }
+
static LinkerPatch MethodPatch(size_t literal_offset,
const DexFile* target_dex_file,
uint32_t target_method_idx) {
- LinkerPatch patch(literal_offset, kLinkerPatchMethod, target_dex_file);
+ LinkerPatch patch(literal_offset, Type::kMethod, target_dex_file);
patch.method_idx_ = target_method_idx;
return patch;
}
@@ -180,7 +194,7 @@
static LinkerPatch CodePatch(size_t literal_offset,
const DexFile* target_dex_file,
uint32_t target_method_idx) {
- LinkerPatch patch(literal_offset, kLinkerPatchCall, target_dex_file);
+ LinkerPatch patch(literal_offset, Type::kCall, target_dex_file);
patch.method_idx_ = target_method_idx;
return patch;
}
@@ -188,7 +202,7 @@
static LinkerPatch RelativeCodePatch(size_t literal_offset,
const DexFile* target_dex_file,
uint32_t target_method_idx) {
- LinkerPatch patch(literal_offset, kLinkerPatchCallRelative, target_dex_file);
+ LinkerPatch patch(literal_offset, Type::kCallRelative, target_dex_file);
patch.method_idx_ = target_method_idx;
return patch;
}
@@ -196,17 +210,35 @@
static LinkerPatch TypePatch(size_t literal_offset,
const DexFile* target_dex_file,
uint32_t target_type_idx) {
- LinkerPatch patch(literal_offset, kLinkerPatchType, target_dex_file);
+ LinkerPatch patch(literal_offset, Type::kType, target_dex_file);
patch.type_idx_ = target_type_idx;
return patch;
}
+ static LinkerPatch StringPatch(size_t literal_offset,
+ const DexFile* target_dex_file,
+ uint32_t target_string_idx) {
+ LinkerPatch patch(literal_offset, Type::kString, target_dex_file);
+ patch.string_idx_ = target_string_idx;
+ return patch;
+ }
+
+ static LinkerPatch RelativeStringPatch(size_t literal_offset,
+ const DexFile* target_dex_file,
+ uint32_t pc_insn_offset,
+ uint32_t target_string_idx) {
+ LinkerPatch patch(literal_offset, Type::kStringRelative, target_dex_file);
+ patch.string_idx_ = target_string_idx;
+ patch.pc_insn_offset_ = pc_insn_offset;
+ return patch;
+ }
+
static LinkerPatch DexCacheArrayPatch(size_t literal_offset,
const DexFile* target_dex_file,
uint32_t pc_insn_offset,
size_t element_offset) {
DCHECK(IsUint<32>(element_offset));
- LinkerPatch patch(literal_offset, kLinkerPatchDexCacheArray, target_dex_file);
+ LinkerPatch patch(literal_offset, Type::kDexCacheArray, target_dex_file);
patch.pc_insn_offset_ = pc_insn_offset;
patch.element_offset_ = element_offset;
return patch;
@@ -219,47 +251,65 @@
return literal_offset_;
}
- LinkerPatchType Type() const {
+ Type GetType() const {
return patch_type_;
}
bool IsPcRelative() const {
- return Type() == kLinkerPatchCallRelative || Type() == kLinkerPatchDexCacheArray;
+ switch (GetType()) {
+ case Type::kCallRelative:
+ case Type::kStringRelative:
+ case Type::kDexCacheArray:
+ return true;
+ default:
+ return false;
+ }
}
MethodReference TargetMethod() const {
- DCHECK(patch_type_ == kLinkerPatchMethod ||
- patch_type_ == kLinkerPatchCall || patch_type_ == kLinkerPatchCallRelative);
+ DCHECK(patch_type_ == Type::kMethod ||
+ patch_type_ == Type::kCall ||
+ patch_type_ == Type::kCallRelative);
return MethodReference(target_dex_file_, method_idx_);
}
const DexFile* TargetTypeDexFile() const {
- DCHECK(patch_type_ == kLinkerPatchType);
+ DCHECK(patch_type_ == Type::kType);
return target_dex_file_;
}
uint32_t TargetTypeIndex() const {
- DCHECK(patch_type_ == kLinkerPatchType);
+ DCHECK(patch_type_ == Type::kType);
return type_idx_;
}
+ const DexFile* TargetStringDexFile() const {
+ DCHECK(patch_type_ == Type::kString || patch_type_ == Type::kStringRelative);
+ return target_dex_file_;
+ }
+
+ uint32_t TargetStringIndex() const {
+ DCHECK(patch_type_ == Type::kString || patch_type_ == Type::kStringRelative);
+ return string_idx_;
+ }
+
const DexFile* TargetDexCacheDexFile() const {
- DCHECK(patch_type_ == kLinkerPatchDexCacheArray);
+ DCHECK(patch_type_ == Type::kDexCacheArray);
return target_dex_file_;
}
size_t TargetDexCacheElementOffset() const {
- DCHECK(patch_type_ == kLinkerPatchDexCacheArray);
+ DCHECK(patch_type_ == Type::kDexCacheArray);
return element_offset_;
}
uint32_t PcInsnOffset() const {
- DCHECK(patch_type_ == kLinkerPatchDexCacheArray);
+ DCHECK(patch_type_ == Type::kStringRelative || patch_type_ == Type::kDexCacheArray);
return pc_insn_offset_;
}
private:
- LinkerPatch(size_t literal_offset, LinkerPatchType patch_type, const DexFile* target_dex_file)
+ LinkerPatch(size_t literal_offset, Type patch_type, const DexFile* target_dex_file)
: target_dex_file_(target_dex_file),
literal_offset_(literal_offset),
patch_type_(patch_type) {
@@ -272,14 +322,16 @@
const DexFile* target_dex_file_;
uint32_t literal_offset_ : 24; // Method code size up to 16MiB.
- LinkerPatchType patch_type_ : 8;
+ Type patch_type_ : 8;
union {
uint32_t cmp1_; // Used for relational operators.
uint32_t method_idx_; // Method index for Call/Method patches.
uint32_t type_idx_; // Type index for Type patches.
+ uint32_t string_idx_; // String index for String patches.
uint32_t element_offset_; // Element offset in the dex cache arrays.
static_assert(sizeof(method_idx_) == sizeof(cmp1_), "needed by relational operators");
static_assert(sizeof(type_idx_) == sizeof(cmp1_), "needed by relational operators");
+ static_assert(sizeof(string_idx_) == sizeof(cmp1_), "needed by relational operators");
static_assert(sizeof(element_offset_) == sizeof(cmp1_), "needed by relational operators");
};
union {
@@ -295,6 +347,7 @@
friend bool operator==(const LinkerPatch& lhs, const LinkerPatch& rhs);
friend bool operator<(const LinkerPatch& lhs, const LinkerPatch& rhs);
};
+std::ostream& operator<<(std::ostream& os, const LinkerPatch::Type& type);
inline bool operator==(const LinkerPatch& lhs, const LinkerPatch& rhs) {
return lhs.literal_offset_ == rhs.literal_offset_ &&
diff --git a/compiler/dex/verification_results.cc b/compiler/dex/verification_results.cc
index 7c9ce1e..1491a18 100644
--- a/compiler/dex/verification_results.cc
+++ b/compiler/dex/verification_results.cc
@@ -64,7 +64,6 @@
DCHECK_EQ(it->second->GetDevirtMap().size(), verified_method->GetDevirtMap().size());
DCHECK_EQ(it->second->GetSafeCastSet().size(), verified_method->GetSafeCastSet().size());
}
- DCHECK_EQ(it->second->GetDexGcMap().size(), verified_method->GetDexGcMap().size());
// Delete the new verified method since there was already an existing one registered. It
// is unsafe to replace the existing one since the JIT may be using it to generate a
// native GC map.
diff --git a/compiler/dex/verified_method.cc b/compiler/dex/verified_method.cc
index 9ae2164..5c0253c 100644
--- a/compiler/dex/verified_method.cc
+++ b/compiler/dex/verified_method.cc
@@ -30,7 +30,6 @@
#include "mirror/dex_cache-inl.h"
#include "mirror/object-inl.h"
#include "utils.h"
-#include "verifier/dex_gc_map.h"
#include "verifier/method_verifier-inl.h"
#include "verifier/reg_type-inl.h"
#include "verifier/register_line-inl.h"
@@ -49,14 +48,6 @@
method_verifier->HasInstructionThatWillThrow()));
if (compile) {
- /* Generate a register map. */
- if (!verified_method->GenerateGcMap(method_verifier)) {
- return nullptr; // Not a real failure, but a failure to encode.
- }
- if (kIsDebugBuild) {
- VerifyGcMap(method_verifier, verified_method->dex_gc_map_);
- }
-
// TODO: move this out when DEX-to-DEX supports devirtualization.
if (method_verifier->HasVirtualOrInterfaceInvokes()) {
verified_method->GenerateDevirtMap(method_verifier);
@@ -90,120 +81,6 @@
return std::binary_search(safe_cast_set_.begin(), safe_cast_set_.end(), pc);
}
-bool VerifiedMethod::GenerateGcMap(verifier::MethodVerifier* method_verifier) {
- DCHECK(dex_gc_map_.empty());
- size_t num_entries, ref_bitmap_bits, pc_bits;
- ComputeGcMapSizes(method_verifier, &num_entries, &ref_bitmap_bits, &pc_bits);
- const size_t ref_bitmap_bytes = RoundUp(ref_bitmap_bits, kBitsPerByte) / kBitsPerByte;
- static constexpr size_t kFormatBits = 3;
- // We have 16 - kFormatBits available for the ref_bitmap_bytes.
- if ((ref_bitmap_bytes >> (16u - kFormatBits)) != 0) {
- LOG(WARNING) << "Cannot encode GC map for method with " << ref_bitmap_bits << " registers: "
- << PrettyMethod(method_verifier->GetMethodReference().dex_method_index,
- *method_verifier->GetMethodReference().dex_file);
- return false;
- }
- // There are 2 bytes to encode the number of entries.
- if (num_entries > std::numeric_limits<uint16_t>::max()) {
- LOG(WARNING) << "Cannot encode GC map for method with " << num_entries << " entries: "
- << PrettyMethod(method_verifier->GetMethodReference().dex_method_index,
- *method_verifier->GetMethodReference().dex_file);
- return false;
- }
- size_t pc_bytes;
- verifier::RegisterMapFormat format;
- if (pc_bits <= kBitsPerByte) {
- format = verifier::kRegMapFormatCompact8;
- pc_bytes = 1;
- } else if (pc_bits <= kBitsPerByte * 2) {
- format = verifier::kRegMapFormatCompact16;
- pc_bytes = 2;
- } else {
- LOG(WARNING) << "Cannot encode GC map for method with "
- << (1 << pc_bits) << " instructions (number is rounded up to nearest power of 2): "
- << PrettyMethod(method_verifier->GetMethodReference().dex_method_index,
- *method_verifier->GetMethodReference().dex_file);
- return false;
- }
- size_t table_size = ((pc_bytes + ref_bitmap_bytes) * num_entries) + 4;
- dex_gc_map_.reserve(table_size);
- // Write table header.
- dex_gc_map_.push_back(format | ((ref_bitmap_bytes & ~0xFF) >> (kBitsPerByte - kFormatBits)));
- dex_gc_map_.push_back(ref_bitmap_bytes & 0xFF);
- dex_gc_map_.push_back(num_entries & 0xFF);
- dex_gc_map_.push_back((num_entries >> 8) & 0xFF);
- // Write table data.
- const DexFile::CodeItem* code_item = method_verifier->CodeItem();
- for (size_t i = 0; i < code_item->insns_size_in_code_units_; i++) {
- if (method_verifier->GetInstructionFlags(i).IsCompileTimeInfoPoint()) {
- dex_gc_map_.push_back(i & 0xFF);
- if (pc_bytes == 2) {
- dex_gc_map_.push_back((i >> 8) & 0xFF);
- }
- verifier::RegisterLine* line = method_verifier->GetRegLine(i);
- line->WriteReferenceBitMap(method_verifier, &dex_gc_map_, ref_bitmap_bytes);
- }
- }
- DCHECK_EQ(dex_gc_map_.size(), table_size);
- return true;
-}
-
-void VerifiedMethod::VerifyGcMap(verifier::MethodVerifier* method_verifier,
- const std::vector<uint8_t>& data) {
- // Check that for every GC point there is a map entry, there aren't entries for non-GC points,
- // that the table data is well formed and all references are marked (or not) in the bitmap.
- verifier::DexPcToReferenceMap map(&data[0]);
- CHECK_EQ(data.size(), map.RawSize()) << map.NumEntries() << " " << map.RegWidth();
- size_t map_index = 0;
- const DexFile::CodeItem* code_item = method_verifier->CodeItem();
- for (size_t i = 0; i < code_item->insns_size_in_code_units_; i++) {
- const uint8_t* reg_bitmap = map.FindBitMap(i, false);
- if (method_verifier->GetInstructionFlags(i).IsCompileTimeInfoPoint()) {
- DCHECK_LT(map_index, map.NumEntries());
- DCHECK_EQ(map.GetDexPc(map_index), i);
- DCHECK_EQ(map.GetBitMap(map_index), reg_bitmap);
- map_index++;
- verifier::RegisterLine* line = method_verifier->GetRegLine(i);
- for (size_t j = 0; j < code_item->registers_size_; j++) {
- if (line->GetRegisterType(method_verifier, j).IsNonZeroReferenceTypes()) {
- DCHECK_LT(j / kBitsPerByte, map.RegWidth());
- DCHECK_EQ((reg_bitmap[j / kBitsPerByte] >> (j % kBitsPerByte)) & 1, 1);
- } else if ((j / kBitsPerByte) < map.RegWidth()) {
- DCHECK_EQ((reg_bitmap[j / kBitsPerByte] >> (j % kBitsPerByte)) & 1, 0);
- } else {
- // If a register doesn't contain a reference then the bitmap may be shorter than the line.
- }
- }
- } else {
- DCHECK(i >= 65536 || reg_bitmap == nullptr);
- }
- }
-}
-
-void VerifiedMethod::ComputeGcMapSizes(verifier::MethodVerifier* method_verifier,
- size_t* gc_points, size_t* ref_bitmap_bits,
- size_t* log2_max_gc_pc) {
- size_t local_gc_points = 0;
- size_t max_insn = 0;
- size_t max_ref_reg = -1;
- const DexFile::CodeItem* code_item = method_verifier->CodeItem();
- for (size_t i = 0; i < code_item->insns_size_in_code_units_; i++) {
- if (method_verifier->GetInstructionFlags(i).IsCompileTimeInfoPoint()) {
- local_gc_points++;
- max_insn = i;
- verifier::RegisterLine* line = method_verifier->GetRegLine(i);
- max_ref_reg = line->GetMaxNonZeroReferenceReg(method_verifier, max_ref_reg);
- }
- }
- *gc_points = local_gc_points;
- *ref_bitmap_bits = max_ref_reg + 1; // If max register is 0 we need 1 bit to encode (ie +1).
- size_t i = 0;
- while ((1U << i) <= max_insn) {
- i++;
- }
- *log2_max_gc_pc = i;
-}
-
bool VerifiedMethod::GenerateDequickenMap(verifier::MethodVerifier* method_verifier) {
if (method_verifier->HasFailures()) {
return false;
diff --git a/compiler/dex/verified_method.h b/compiler/dex/verified_method.h
index 12d0219..495acf0 100644
--- a/compiler/dex/verified_method.h
+++ b/compiler/dex/verified_method.h
@@ -47,10 +47,6 @@
SHARED_REQUIRES(Locks::mutator_lock_);
~VerifiedMethod() = default;
- const std::vector<uint8_t>& GetDexGcMap() const {
- return dex_gc_map_;
- }
-
const DevirtualizationMap& GetDevirtMap() const {
return devirt_map_;
}
@@ -114,7 +110,6 @@
void GenerateSafeCastSet(verifier::MethodVerifier* method_verifier)
SHARED_REQUIRES(Locks::mutator_lock_);
- std::vector<uint8_t> dex_gc_map_;
DevirtualizationMap devirt_map_;
// Dequicken map is required for compiling quickened byte codes. The quicken maps from
// dex PC to dex method index or dex field index based on the instruction.
diff --git a/compiler/driver/compiler_driver.cc b/compiler/driver/compiler_driver.cc
index ea16cb2..d29d528 100644
--- a/compiler/driver/compiler_driver.cc
+++ b/compiler/driver/compiler_driver.cc
@@ -2212,7 +2212,8 @@
class VerifyClassVisitor : public CompilationVisitor {
public:
- explicit VerifyClassVisitor(const ParallelCompilationManager* manager) : manager_(manager) {}
+ VerifyClassVisitor(const ParallelCompilationManager* manager, LogSeverity log_level)
+ : manager_(manager), log_level_(log_level) {}
virtual void Visit(size_t class_def_index) REQUIRES(!Locks::mutator_lock_) OVERRIDE {
ATRACE_CALL();
@@ -2250,7 +2251,7 @@
&class_def,
Runtime::Current()->GetCompilerCallbacks(),
true /* allow soft failures */,
- true /* log hard failures */,
+ log_level_,
&error_msg) ==
verifier::MethodVerifier::kHardFailure) {
LOG(ERROR) << "Verification failed on class " << PrettyDescriptor(descriptor)
@@ -2259,7 +2260,7 @@
}
} else if (!SkipClass(jclass_loader, dex_file, klass.Get())) {
CHECK(klass->IsResolved()) << PrettyClass(klass.Get());
- class_linker->VerifyClass(soa.Self(), klass);
+ class_linker->VerifyClass(soa.Self(), klass, log_level_);
if (klass->IsErroneous()) {
// ClassLinker::VerifyClass throws, which isn't useful in the compiler.
@@ -2282,6 +2283,7 @@
private:
const ParallelCompilationManager* const manager_;
+ const LogSeverity log_level_;
};
void CompilerDriver::VerifyDexFile(jobject class_loader,
@@ -2294,7 +2296,10 @@
ClassLinker* class_linker = Runtime::Current()->GetClassLinker();
ParallelCompilationManager context(class_linker, class_loader, this, &dex_file, dex_files,
thread_pool);
- VerifyClassVisitor visitor(&context);
+ LogSeverity log_level = GetCompilerOptions().AbortOnHardVerifierFailure()
+ ? LogSeverity::INTERNAL_FATAL
+ : LogSeverity::WARNING;
+ VerifyClassVisitor visitor(&context, log_level);
context.ForAll(0, dex_file.NumClassDefs(), &visitor, thread_count);
}
diff --git a/compiler/linker/arm/relative_patcher_arm_base.cc b/compiler/linker/arm/relative_patcher_arm_base.cc
index 682b008..d4dd978 100644
--- a/compiler/linker/arm/relative_patcher_arm_base.cc
+++ b/compiler/linker/arm/relative_patcher_arm_base.cc
@@ -112,7 +112,7 @@
}
}
for (const LinkerPatch& patch : compiled_method->GetPatches()) {
- if (patch.Type() == kLinkerPatchCallRelative) {
+ if (patch.GetType() == LinkerPatch::Type::kCallRelative) {
unprocessed_patches_.emplace_back(patch.TargetMethod(),
quick_code_offset + patch.LiteralOffset());
}
diff --git a/compiler/linker/arm/relative_patcher_thumb2.cc b/compiler/linker/arm/relative_patcher_thumb2.cc
index c090dff..582ecb3 100644
--- a/compiler/linker/arm/relative_patcher_thumb2.cc
+++ b/compiler/linker/arm/relative_patcher_thumb2.cc
@@ -56,10 +56,10 @@
SetInsn32(code, literal_offset, value);
}
-void Thumb2RelativePatcher::PatchDexCacheReference(std::vector<uint8_t>* code,
- const LinkerPatch& patch,
- uint32_t patch_offset,
- uint32_t target_offset) {
+void Thumb2RelativePatcher::PatchPcRelativeReference(std::vector<uint8_t>* code,
+ const LinkerPatch& patch,
+ uint32_t patch_offset,
+ uint32_t target_offset) {
uint32_t literal_offset = patch.LiteralOffset();
uint32_t pc_literal_offset = patch.PcInsnOffset();
uint32_t pc_base = patch_offset + (pc_literal_offset - literal_offset) + 4u /* PC adjustment */;
diff --git a/compiler/linker/arm/relative_patcher_thumb2.h b/compiler/linker/arm/relative_patcher_thumb2.h
index 0d903c0..d85739c 100644
--- a/compiler/linker/arm/relative_patcher_thumb2.h
+++ b/compiler/linker/arm/relative_patcher_thumb2.h
@@ -30,10 +30,10 @@
uint32_t literal_offset,
uint32_t patch_offset,
uint32_t target_offset) OVERRIDE;
- void PatchDexCacheReference(std::vector<uint8_t>* code,
- const LinkerPatch& patch,
- uint32_t patch_offset,
- uint32_t target_offset) OVERRIDE;
+ void PatchPcRelativeReference(std::vector<uint8_t>* code,
+ const LinkerPatch& patch,
+ uint32_t patch_offset,
+ uint32_t target_offset) OVERRIDE;
private:
static std::vector<uint8_t> CompileThunkCode();
diff --git a/compiler/linker/arm/relative_patcher_thumb2_test.cc b/compiler/linker/arm/relative_patcher_thumb2_test.cc
index a259cda..a8078e3 100644
--- a/compiler/linker/arm/relative_patcher_thumb2_test.cc
+++ b/compiler/linker/arm/relative_patcher_thumb2_test.cc
@@ -30,6 +30,9 @@
static const ArrayRef<const uint8_t> kCallCode;
static const uint8_t kNopRawCode[];
static const ArrayRef<const uint8_t> kNopCode;
+ static const uint8_t kUnpatchedPcRelativeRawCode[];
+ static const ArrayRef<const uint8_t> kUnpatchedPcRelativeCode;
+ static const uint32_t kPcInsnOffset;
// Branches within range [-256, 256) can be created from these by adding the low 8 bits.
static constexpr uint32_t kBlPlus0 = 0xf000f800;
@@ -123,47 +126,9 @@
return result;
}
- void TestDexCachereference(uint32_t dex_cache_arrays_begin, uint32_t element_offset) {
- dex_cache_arrays_begin_ = dex_cache_arrays_begin;
- static const uint8_t raw_code[] = {
- 0x40, 0xf2, 0x00, 0x00, // MOVW r0, #0 (placeholder)
- 0xc0, 0xf2, 0x00, 0x00, // MOVT r0, #0 (placeholder)
- 0x78, 0x44, // ADD r0, pc
- };
- constexpr uint32_t pc_insn_offset = 8u;
- const ArrayRef<const uint8_t> code(raw_code);
- LinkerPatch patches[] = {
- LinkerPatch::DexCacheArrayPatch(0u, nullptr, pc_insn_offset, element_offset),
- LinkerPatch::DexCacheArrayPatch(4u, nullptr, pc_insn_offset, element_offset),
- };
- AddCompiledMethod(MethodRef(1u), code, ArrayRef<const LinkerPatch>(patches));
- Link();
-
- uint32_t method1_offset = GetMethodOffset(1u);
- uint32_t pc_base_offset = method1_offset + pc_insn_offset + 4u /* PC adjustment */;
- uint32_t diff = dex_cache_arrays_begin_ + element_offset - pc_base_offset;
- // Distribute the bits of the diff between the MOVW and MOVT:
- uint32_t diffw = diff & 0xffffu;
- uint32_t difft = diff >> 16;
- uint32_t movw = 0xf2400000u | // MOVW r0, #0 (placeholder),
- ((diffw & 0xf000u) << (16 - 12)) | // move imm4 from bits 12-15 to bits 16-19,
- ((diffw & 0x0800u) << (26 - 11)) | // move imm from bit 11 to bit 26,
- ((diffw & 0x0700u) << (12 - 8)) | // move imm3 from bits 8-10 to bits 12-14,
- ((diffw & 0x00ffu)); // keep imm8 at bits 0-7.
- uint32_t movt = 0xf2c00000u | // MOVT r0, #0 (placeholder),
- ((difft & 0xf000u) << (16 - 12)) | // move imm4 from bits 12-15 to bits 16-19,
- ((difft & 0x0800u) << (26 - 11)) | // move imm from bit 11 to bit 26,
- ((difft & 0x0700u) << (12 - 8)) | // move imm3 from bits 8-10 to bits 12-14,
- ((difft & 0x00ffu)); // keep imm8 at bits 0-7.
- const uint8_t expected_code[] = {
- static_cast<uint8_t>(movw >> 16), static_cast<uint8_t>(movw >> 24),
- static_cast<uint8_t>(movw >> 0), static_cast<uint8_t>(movw >> 8),
- static_cast<uint8_t>(movt >> 16), static_cast<uint8_t>(movt >> 24),
- static_cast<uint8_t>(movt >> 0), static_cast<uint8_t>(movt >> 8),
- 0x78, 0x44,
- };
- EXPECT_TRUE(CheckLinkedMethod(MethodRef(1u), ArrayRef<const uint8_t>(expected_code)));
- }
+ void TestDexCacheReference(uint32_t dex_cache_arrays_begin, uint32_t element_offset);
+ void TestStringReference(uint32_t string_offset);
+ void CheckPcRelativePatch(const ArrayRef<const LinkerPatch>& patches, uint32_t target_offset);
};
const uint8_t Thumb2RelativePatcherTest::kCallRawCode[] = {
@@ -178,6 +143,67 @@
const ArrayRef<const uint8_t> Thumb2RelativePatcherTest::kNopCode(kNopRawCode);
+const uint8_t Thumb2RelativePatcherTest::kUnpatchedPcRelativeRawCode[] = {
+ 0x40, 0xf2, 0x00, 0x00, // MOVW r0, #0 (placeholder)
+ 0xc0, 0xf2, 0x00, 0x00, // MOVT r0, #0 (placeholder)
+ 0x78, 0x44, // ADD r0, pc
+};
+const ArrayRef<const uint8_t> Thumb2RelativePatcherTest::kUnpatchedPcRelativeCode(
+ kUnpatchedPcRelativeRawCode);
+const uint32_t Thumb2RelativePatcherTest::kPcInsnOffset = 8u;
+
+void Thumb2RelativePatcherTest::TestDexCacheReference(uint32_t dex_cache_arrays_begin,
+ uint32_t element_offset) {
+ dex_cache_arrays_begin_ = dex_cache_arrays_begin;
+ LinkerPatch patches[] = {
+ LinkerPatch::DexCacheArrayPatch(0u, nullptr, kPcInsnOffset, element_offset),
+ LinkerPatch::DexCacheArrayPatch(4u, nullptr, kPcInsnOffset, element_offset),
+ };
+ CheckPcRelativePatch(ArrayRef<const LinkerPatch>(patches),
+ dex_cache_arrays_begin_ + element_offset);
+}
+
+void Thumb2RelativePatcherTest::TestStringReference(uint32_t string_offset) {
+ constexpr uint32_t kStringIndex = 1u;
+ string_index_to_offset_map_.Put(kStringIndex, string_offset);
+ LinkerPatch patches[] = {
+ LinkerPatch::RelativeStringPatch(0u, nullptr, kPcInsnOffset, kStringIndex),
+ LinkerPatch::RelativeStringPatch(4u, nullptr, kPcInsnOffset, kStringIndex),
+ };
+ CheckPcRelativePatch(ArrayRef<const LinkerPatch>(patches), string_offset);
+}
+
+void Thumb2RelativePatcherTest::CheckPcRelativePatch(const ArrayRef<const LinkerPatch>& patches,
+ uint32_t target_offset) {
+ AddCompiledMethod(MethodRef(1u), kUnpatchedPcRelativeCode, ArrayRef<const LinkerPatch>(patches));
+ Link();
+
+ uint32_t method1_offset = GetMethodOffset(1u);
+ uint32_t pc_base_offset = method1_offset + kPcInsnOffset + 4u /* PC adjustment */;
+ uint32_t diff = target_offset - pc_base_offset;
+ // Distribute the bits of the diff between the MOVW and MOVT:
+ uint32_t diffw = diff & 0xffffu;
+ uint32_t difft = diff >> 16;
+ uint32_t movw = 0xf2400000u | // MOVW r0, #0 (placeholder),
+ ((diffw & 0xf000u) << (16 - 12)) | // move imm4 from bits 12-15 to bits 16-19,
+ ((diffw & 0x0800u) << (26 - 11)) | // move imm from bit 11 to bit 26,
+ ((diffw & 0x0700u) << (12 - 8)) | // move imm3 from bits 8-10 to bits 12-14,
+ ((diffw & 0x00ffu)); // keep imm8 at bits 0-7.
+ uint32_t movt = 0xf2c00000u | // MOVT r0, #0 (placeholder),
+ ((difft & 0xf000u) << (16 - 12)) | // move imm4 from bits 12-15 to bits 16-19,
+ ((difft & 0x0800u) << (26 - 11)) | // move imm from bit 11 to bit 26,
+ ((difft & 0x0700u) << (12 - 8)) | // move imm3 from bits 8-10 to bits 12-14,
+ ((difft & 0x00ffu)); // keep imm8 at bits 0-7.
+ const uint8_t expected_code[] = {
+ static_cast<uint8_t>(movw >> 16), static_cast<uint8_t>(movw >> 24),
+ static_cast<uint8_t>(movw >> 0), static_cast<uint8_t>(movw >> 8),
+ static_cast<uint8_t>(movt >> 16), static_cast<uint8_t>(movt >> 24),
+ static_cast<uint8_t>(movt >> 0), static_cast<uint8_t>(movt >> 8),
+ 0x78, 0x44,
+ };
+ EXPECT_TRUE(CheckLinkedMethod(MethodRef(1u), ArrayRef<const uint8_t>(expected_code)));
+}
+
TEST_F(Thumb2RelativePatcherTest, CallSelf) {
LinkerPatch patches[] = {
LinkerPatch::RelativeCodePatch(0u, nullptr, 1u),
@@ -366,23 +392,43 @@
EXPECT_TRUE(CheckThunk(thunk_offset));
}
-TEST_F(Thumb2RelativePatcherTest, DexCacheReferenceImm8) {
- TestDexCachereference(0x00ff0000u, 0x00fcu);
+TEST_F(Thumb2RelativePatcherTest, DexCacheReference1) {
+ TestDexCacheReference(0x00ff0000u, 0x00fcu);
ASSERT_LT(GetMethodOffset(1u), 0xfcu);
}
-TEST_F(Thumb2RelativePatcherTest, DexCacheReferenceImm3) {
- TestDexCachereference(0x02ff0000u, 0x05fcu);
+TEST_F(Thumb2RelativePatcherTest, DexCacheReference2) {
+ TestDexCacheReference(0x02ff0000u, 0x05fcu);
ASSERT_LT(GetMethodOffset(1u), 0xfcu);
}
-TEST_F(Thumb2RelativePatcherTest, DexCacheReferenceImm) {
- TestDexCachereference(0x08ff0000u, 0x08fcu);
+TEST_F(Thumb2RelativePatcherTest, DexCacheReference3) {
+ TestDexCacheReference(0x08ff0000u, 0x08fcu);
ASSERT_LT(GetMethodOffset(1u), 0xfcu);
}
-TEST_F(Thumb2RelativePatcherTest, DexCacheReferenceimm4) {
- TestDexCachereference(0xd0ff0000u, 0x60fcu);
+TEST_F(Thumb2RelativePatcherTest, DexCacheReference4) {
+ TestDexCacheReference(0xd0ff0000u, 0x60fcu);
+ ASSERT_LT(GetMethodOffset(1u), 0xfcu);
+}
+
+TEST_F(Thumb2RelativePatcherTest, StringReference1) {
+ TestStringReference(0x00ff00fcu);
+ ASSERT_LT(GetMethodOffset(1u), 0xfcu);
+}
+
+TEST_F(Thumb2RelativePatcherTest, StringReference2) {
+ TestStringReference(0x02ff05fcu);
+ ASSERT_LT(GetMethodOffset(1u), 0xfcu);
+}
+
+TEST_F(Thumb2RelativePatcherTest, StringReference3) {
+ TestStringReference(0x08ff08fcu);
+ ASSERT_LT(GetMethodOffset(1u), 0xfcu);
+}
+
+TEST_F(Thumb2RelativePatcherTest, StringReference4) {
+ TestStringReference(0xd0ff60fcu);
ASSERT_LT(GetMethodOffset(1u), 0xfcu);
}
diff --git a/compiler/linker/arm64/relative_patcher_arm64.cc b/compiler/linker/arm64/relative_patcher_arm64.cc
index a81c85c..e3e3121 100644
--- a/compiler/linker/arm64/relative_patcher_arm64.cc
+++ b/compiler/linker/arm64/relative_patcher_arm64.cc
@@ -28,6 +28,17 @@
namespace art {
namespace linker {
+namespace {
+
+inline bool IsAdrpPatch(const LinkerPatch& patch) {
+ LinkerPatch::Type type = patch.GetType();
+ return
+ (type == LinkerPatch::Type::kStringRelative || type == LinkerPatch::Type::kDexCacheArray) &&
+ patch.LiteralOffset() == patch.PcInsnOffset();
+}
+
+} // anonymous namespace
+
Arm64RelativePatcher::Arm64RelativePatcher(RelativePatcherTargetProvider* provider,
const Arm64InstructionSetFeatures* features)
: ArmBaseRelativePatcher(provider, kArm64, CompileThunkCode(),
@@ -61,8 +72,7 @@
size_t num_adrp = 0u;
DCHECK(compiled_method != nullptr);
for (const LinkerPatch& patch : compiled_method->GetPatches()) {
- if (patch.Type() == kLinkerPatchDexCacheArray &&
- patch.LiteralOffset() == patch.PcInsnOffset()) { // ADRP patch
+ if (IsAdrpPatch(patch)) {
++num_adrp;
}
}
@@ -78,8 +88,7 @@
uint32_t thunk_offset = compiled_method->AlignCode(quick_code_offset + code.size());
DCHECK(compiled_method != nullptr);
for (const LinkerPatch& patch : compiled_method->GetPatches()) {
- if (patch.Type() == kLinkerPatchDexCacheArray &&
- patch.LiteralOffset() == patch.PcInsnOffset()) { // ADRP patch
+ if (IsAdrpPatch(patch)) {
uint32_t patch_offset = quick_code_offset + patch.LiteralOffset();
if (NeedsErratum843419Thunk(code, patch.LiteralOffset(), patch_offset)) {
adrp_thunk_locations_.emplace_back(patch_offset, thunk_offset);
@@ -151,10 +160,10 @@
SetInsn(code, literal_offset, insn);
}
-void Arm64RelativePatcher::PatchDexCacheReference(std::vector<uint8_t>* code,
- const LinkerPatch& patch,
- uint32_t patch_offset,
- uint32_t target_offset) {
+void Arm64RelativePatcher::PatchPcRelativeReference(std::vector<uint8_t>* code,
+ const LinkerPatch& patch,
+ uint32_t patch_offset,
+ uint32_t target_offset) {
DCHECK_EQ(patch_offset & 3u, 0u);
DCHECK_EQ(target_offset & 3u, 0u);
uint32_t literal_offset = patch.LiteralOffset();
@@ -199,8 +208,15 @@
// Write the new ADRP (or B to the erratum 843419 thunk).
SetInsn(code, literal_offset, insn);
} else {
- // LDR 32-bit or 64-bit with imm12 == 0 (unset).
- DCHECK_EQ(insn & 0xbffffc00, 0xb9400000) << insn;
+ if ((insn & 0xfffffc00) == 0x91000000) {
+ // ADD immediate, 64-bit with imm12 == 0 (unset).
+ DCHECK(patch.GetType() == LinkerPatch::Type::kStringRelative) << patch.GetType();
+ shift = 0u; // No shift for ADD.
+ } else {
+ // LDR 32-bit or 64-bit with imm12 == 0 (unset).
+ DCHECK(patch.GetType() == LinkerPatch::Type::kDexCacheArray) << patch.GetType();
+ DCHECK_EQ(insn & 0xbffffc00, 0xb9400000) << std::hex << insn;
+ }
if (kIsDebugBuild) {
uint32_t adrp = GetInsn(code, pc_insn_offset);
if ((adrp & 0x9f000000u) != 0x90000000u) {
@@ -263,7 +279,7 @@
DCHECK_EQ(patch_offset & 0x3u, 0u);
if ((patch_offset & 0xff8) == 0xff8) { // ...ff8 or ...ffc
uint32_t adrp = GetInsn(code, literal_offset);
- DCHECK_EQ(adrp & 0xff000000, 0x90000000);
+ DCHECK_EQ(adrp & 0x9f000000, 0x90000000);
uint32_t next_offset = patch_offset + 4u;
uint32_t next_insn = GetInsn(code, literal_offset + 4u);
@@ -277,6 +293,15 @@
return false;
}
+ // And since LinkerPatch::Type::kStringRelative is using the result of the ADRP
+ // for an ADD immediate, check for that as well. We generalize a bit to include
+ // ADD/ADDS/SUB/SUBS immediate that either uses the ADRP destination or stores
+ // the result to a different register.
+ if ((next_insn & 0x1f000000) == 0x11000000 &&
+ ((((next_insn >> 5) ^ adrp) & 0x1f) == 0 || ((next_insn ^ adrp) & 0x1f) != 0)) {
+ return false;
+ }
+
// LDR <Wt>, <label> is always aligned and thus it doesn't cause boundary crossing.
if ((next_insn & 0xff000000) == 0x18000000) {
return false;
diff --git a/compiler/linker/arm64/relative_patcher_arm64.h b/compiler/linker/arm64/relative_patcher_arm64.h
index f9b76e6..48ad105 100644
--- a/compiler/linker/arm64/relative_patcher_arm64.h
+++ b/compiler/linker/arm64/relative_patcher_arm64.h
@@ -37,10 +37,10 @@
uint32_t literal_offset,
uint32_t patch_offset,
uint32_t target_offset) OVERRIDE;
- void PatchDexCacheReference(std::vector<uint8_t>* code,
- const LinkerPatch& patch,
- uint32_t patch_offset,
- uint32_t target_offset) OVERRIDE;
+ void PatchPcRelativeReference(std::vector<uint8_t>* code,
+ const LinkerPatch& patch,
+ uint32_t patch_offset,
+ uint32_t target_offset) OVERRIDE;
private:
static std::vector<uint8_t> CompileThunkCode();
diff --git a/compiler/linker/arm64/relative_patcher_arm64_test.cc b/compiler/linker/arm64/relative_patcher_arm64_test.cc
index 0bfef5e..09729fd 100644
--- a/compiler/linker/arm64/relative_patcher_arm64_test.cc
+++ b/compiler/linker/arm64/relative_patcher_arm64_test.cc
@@ -40,6 +40,15 @@
static constexpr uint32_t kBlPlusMax = 0x95ffffffu;
static constexpr uint32_t kBlMinusMax = 0x96000000u;
+ // LDR immediate, 32-bit.
+ static constexpr uint32_t kLdrWInsn = 0xb9400000u;
+
+ // ADD/ADDS/SUB/SUBS immediate, 64-bit.
+ static constexpr uint32_t kAddXInsn = 0x91000000u;
+ static constexpr uint32_t kAddsXInsn = 0xb1000000u;
+ static constexpr uint32_t kSubXInsn = 0xd1000000u;
+ static constexpr uint32_t kSubsXInsn = 0xf1000000u;
+
// LDUR x2, [sp, #4], i.e. unaligned load crossing 64-bit boundary (assuming aligned sp).
static constexpr uint32_t kLdurInsn = 0xf840405fu;
@@ -109,7 +118,7 @@
uint32_t GetMethodOffset(uint32_t method_idx) {
auto result = method_offset_map_.FindMethodOffset(MethodRef(method_idx));
CHECK(result.first);
- CHECK_EQ(result.second & 3u, 0u);
+ CHECK_ALIGNED(result.second, 4u);
return result.second;
}
@@ -147,20 +156,29 @@
return result;
}
- std::vector<uint8_t> GenNopsAndAdrpLdr(size_t num_nops,
- uint32_t method_offset, uint32_t target_offset) {
+ std::vector<uint8_t> GenNopsAndAdrpAndUse(size_t num_nops,
+ uint32_t method_offset,
+ uint32_t target_offset,
+ uint32_t use_insn) {
std::vector<uint8_t> result;
result.reserve(num_nops * 4u + 8u);
for (size_t i = 0; i != num_nops; ++i) {
result.insert(result.end(), kNopCode.begin(), kNopCode.end());
}
- DCHECK_EQ(method_offset & 3u, 0u);
- DCHECK_EQ(target_offset & 3u, 0u);
+ CHECK_ALIGNED(method_offset, 4u);
+ CHECK_ALIGNED(target_offset, 4u);
uint32_t adrp_offset = method_offset + num_nops * 4u;
uint32_t disp = target_offset - (adrp_offset & ~0xfffu);
- DCHECK_EQ(disp & 3u, 0u);
- uint32_t ldr = 0xb9400001 | // LDR w1, [x0, #(imm12 * 2)]
- ((disp & 0xfffu) << (10 - 2)); // imm12 = ((disp & 0xfffu) >> 2) is at bit 10.
+ if (use_insn == kLdrWInsn) {
+ DCHECK_ALIGNED(disp, 1u << 2);
+ use_insn |= 1 | // LDR x1, [x0, #(imm12 << 2)]
+ ((disp & 0xfffu) << (10 - 2)); // imm12 = ((disp & 0xfffu) >> 2) is at bit 10.
+ } else if (use_insn == kAddXInsn) {
+ use_insn |= 1 | // ADD x1, x0, #imm
+ (disp & 0xfffu) << 10; // imm12 = (disp & 0xfffu) is at bit 10.
+ } else {
+ LOG(FATAL) << "Unexpected instruction: 0x" << std::hex << use_insn;
+ }
uint32_t adrp = 0x90000000 | // ADRP x0, +SignExtend(immhi:immlo:Zeros(12), 64)
((disp & 0x3000u) << (29 - 12)) | // immlo = ((disp & 0x3000u) >> 12) is at bit 29,
((disp & 0xffffc000) >> (14 - 5)) | // immhi = (disp >> 14) is at bit 5,
@@ -170,13 +188,19 @@
result.push_back(static_cast<uint8_t>(adrp >> 8));
result.push_back(static_cast<uint8_t>(adrp >> 16));
result.push_back(static_cast<uint8_t>(adrp >> 24));
- result.push_back(static_cast<uint8_t>(ldr));
- result.push_back(static_cast<uint8_t>(ldr >> 8));
- result.push_back(static_cast<uint8_t>(ldr >> 16));
- result.push_back(static_cast<uint8_t>(ldr >> 24));
+ result.push_back(static_cast<uint8_t>(use_insn));
+ result.push_back(static_cast<uint8_t>(use_insn >> 8));
+ result.push_back(static_cast<uint8_t>(use_insn >> 16));
+ result.push_back(static_cast<uint8_t>(use_insn >> 24));
return result;
}
+ std::vector<uint8_t> GenNopsAndAdrpLdr(size_t num_nops,
+ uint32_t method_offset,
+ uint32_t target_offset) {
+ return GenNopsAndAdrpAndUse(num_nops, method_offset, target_offset, kLdrWInsn);
+ }
+
void TestNopsAdrpLdr(size_t num_nops, uint32_t dex_cache_arrays_begin, uint32_t element_offset) {
dex_cache_arrays_begin_ = dex_cache_arrays_begin;
auto code = GenNopsAndAdrpLdr(num_nops, 0u, 0u); // Unpatched.
@@ -184,7 +208,8 @@
LinkerPatch::DexCacheArrayPatch(num_nops * 4u , nullptr, num_nops * 4u, element_offset),
LinkerPatch::DexCacheArrayPatch(num_nops * 4u + 4u, nullptr, num_nops * 4u, element_offset),
};
- AddCompiledMethod(MethodRef(1u), ArrayRef<const uint8_t>(code),
+ AddCompiledMethod(MethodRef(1u),
+ ArrayRef<const uint8_t>(code),
ArrayRef<const LinkerPatch>(patches));
Link();
@@ -194,6 +219,30 @@
EXPECT_TRUE(CheckLinkedMethod(MethodRef(1u), ArrayRef<const uint8_t>(expected_code)));
}
+ std::vector<uint8_t> GenNopsAndAdrpAdd(size_t num_nops,
+ uint32_t method_offset,
+ uint32_t target_offset) {
+ return GenNopsAndAdrpAndUse(num_nops, method_offset, target_offset, kAddXInsn);
+ }
+
+ void TestNopsAdrpAdd(size_t num_nops, uint32_t string_offset) {
+ constexpr uint32_t kStringIndex = 1u;
+ string_index_to_offset_map_.Put(kStringIndex, string_offset);
+ auto code = GenNopsAndAdrpAdd(num_nops, 0u, 0u); // Unpatched.
+ LinkerPatch patches[] = {
+ LinkerPatch::RelativeStringPatch(num_nops * 4u , nullptr, num_nops * 4u, kStringIndex),
+ LinkerPatch::RelativeStringPatch(num_nops * 4u + 4u, nullptr, num_nops * 4u, kStringIndex),
+ };
+ AddCompiledMethod(MethodRef(1u),
+ ArrayRef<const uint8_t>(code),
+ ArrayRef<const LinkerPatch>(patches));
+ Link();
+
+ uint32_t method1_offset = GetMethodOffset(1u);
+ auto expected_code = GenNopsAndAdrpAdd(num_nops, method1_offset, string_offset);
+ EXPECT_TRUE(CheckLinkedMethod(MethodRef(1u), ArrayRef<const uint8_t>(expected_code)));
+ }
+
void InsertInsn(std::vector<uint8_t>* code, size_t pos, uint32_t insn) {
CHECK_LE(pos, code->size());
const uint8_t insn_code[] = {
@@ -204,8 +253,10 @@
code->insert(code->begin() + pos, insn_code, insn_code + sizeof(insn_code));
}
- void PrepareNopsAdrpInsn2Ldr(size_t num_nops, uint32_t insn2,
- uint32_t dex_cache_arrays_begin, uint32_t element_offset) {
+ void PrepareNopsAdrpInsn2Ldr(size_t num_nops,
+ uint32_t insn2,
+ uint32_t dex_cache_arrays_begin,
+ uint32_t element_offset) {
dex_cache_arrays_begin_ = dex_cache_arrays_begin;
auto code = GenNopsAndAdrpLdr(num_nops, 0u, 0u); // Unpatched.
InsertInsn(&code, num_nops * 4u + 4u, insn2);
@@ -213,26 +264,41 @@
LinkerPatch::DexCacheArrayPatch(num_nops * 4u , nullptr, num_nops * 4u, element_offset),
LinkerPatch::DexCacheArrayPatch(num_nops * 4u + 8u, nullptr, num_nops * 4u, element_offset),
};
- AddCompiledMethod(MethodRef(1u), ArrayRef<const uint8_t>(code),
+ AddCompiledMethod(MethodRef(1u),
+ ArrayRef<const uint8_t>(code),
ArrayRef<const LinkerPatch>(patches));
Link();
}
- void TestNopsAdrpInsn2Ldr(size_t num_nops, uint32_t insn2,
- uint32_t dex_cache_arrays_begin, uint32_t element_offset) {
- PrepareNopsAdrpInsn2Ldr(num_nops, insn2, dex_cache_arrays_begin, element_offset);
+ void PrepareNopsAdrpInsn2Add(size_t num_nops, uint32_t insn2, uint32_t string_offset) {
+ constexpr uint32_t kStringIndex = 1u;
+ string_index_to_offset_map_.Put(kStringIndex, string_offset);
+ auto code = GenNopsAndAdrpAdd(num_nops, 0u, 0u); // Unpatched.
+ InsertInsn(&code, num_nops * 4u + 4u, insn2);
+ LinkerPatch patches[] = {
+ LinkerPatch::RelativeStringPatch(num_nops * 4u , nullptr, num_nops * 4u, kStringIndex),
+ LinkerPatch::RelativeStringPatch(num_nops * 4u + 8u, nullptr, num_nops * 4u, kStringIndex),
+ };
+ AddCompiledMethod(MethodRef(1u),
+ ArrayRef<const uint8_t>(code),
+ ArrayRef<const LinkerPatch>(patches));
+ Link();
+ }
+ void TestNopsAdrpInsn2AndUse(size_t num_nops,
+ uint32_t insn2,
+ uint32_t target_offset,
+ uint32_t use_insn) {
uint32_t method1_offset = GetMethodOffset(1u);
- uint32_t target_offset = dex_cache_arrays_begin_ + element_offset;
- auto expected_code = GenNopsAndAdrpLdr(num_nops, method1_offset, target_offset);
+ auto expected_code = GenNopsAndAdrpAndUse(num_nops, method1_offset, target_offset, use_insn);
InsertInsn(&expected_code, num_nops * 4u + 4u, insn2);
EXPECT_TRUE(CheckLinkedMethod(MethodRef(1u), ArrayRef<const uint8_t>(expected_code)));
}
- void TestNopsAdrpInsn2LdrHasThunk(size_t num_nops, uint32_t insn2,
- uint32_t dex_cache_arrays_begin, uint32_t element_offset) {
- PrepareNopsAdrpInsn2Ldr(num_nops, insn2, dex_cache_arrays_begin, element_offset);
-
+ void TestNopsAdrpInsn2AndUseHasThunk(size_t num_nops,
+ uint32_t insn2,
+ uint32_t target_offset,
+ uint32_t use_insn) {
uint32_t method1_offset = GetMethodOffset(1u);
CHECK(!compiled_method_refs_.empty());
CHECK_EQ(compiled_method_refs_[0].dex_method_index, 1u);
@@ -240,13 +306,12 @@
uint32_t method1_size = compiled_methods_[0]->GetQuickCode().size();
uint32_t thunk_offset = CompiledCode::AlignCode(method1_offset + method1_size, kArm64);
uint32_t b_diff = thunk_offset - (method1_offset + num_nops * 4u);
- ASSERT_EQ(b_diff & 3u, 0u);
+ CHECK_ALIGNED(b_diff, 4u);
ASSERT_LT(b_diff, 128 * MB);
uint32_t b_out = kBPlus0 + ((b_diff >> 2) & 0x03ffffffu);
uint32_t b_in = kBPlus0 + ((-b_diff >> 2) & 0x03ffffffu);
- uint32_t target_offset = dex_cache_arrays_begin_ + element_offset;
- auto expected_code = GenNopsAndAdrpLdr(num_nops, method1_offset, target_offset);
+ auto expected_code = GenNopsAndAdrpAndUse(num_nops, method1_offset, target_offset, use_insn);
InsertInsn(&expected_code, num_nops * 4u + 4u, insn2);
// Replace adrp with bl.
expected_code.erase(expected_code.begin() + num_nops * 4u,
@@ -270,29 +335,39 @@
}
}
- void TestAdrpInsn2Ldr(uint32_t insn2, uint32_t adrp_offset, bool has_thunk,
- uint32_t dex_cache_arrays_begin, uint32_t element_offset) {
+ void TestAdrpInsn2Ldr(uint32_t insn2,
+ uint32_t adrp_offset,
+ bool has_thunk,
+ uint32_t dex_cache_arrays_begin,
+ uint32_t element_offset) {
uint32_t method1_offset =
CompiledCode::AlignCode(kTrampolineSize, kArm64) + sizeof(OatQuickMethodHeader);
ASSERT_LT(method1_offset, adrp_offset);
- ASSERT_EQ(adrp_offset & 3u, 0u);
+ CHECK_ALIGNED(adrp_offset, 4u);
uint32_t num_nops = (adrp_offset - method1_offset) / 4u;
+ PrepareNopsAdrpInsn2Ldr(num_nops, insn2, dex_cache_arrays_begin, element_offset);
+ uint32_t target_offset = dex_cache_arrays_begin_ + element_offset;
if (has_thunk) {
- TestNopsAdrpInsn2LdrHasThunk(num_nops, insn2, dex_cache_arrays_begin, element_offset);
+ TestNopsAdrpInsn2AndUseHasThunk(num_nops, insn2, target_offset, kLdrWInsn);
} else {
- TestNopsAdrpInsn2Ldr(num_nops, insn2, dex_cache_arrays_begin, element_offset);
+ TestNopsAdrpInsn2AndUse(num_nops, insn2, target_offset, kLdrWInsn);
}
ASSERT_EQ(method1_offset, GetMethodOffset(1u)); // If this fails, num_nops is wrong.
}
- void TestAdrpLdurLdr(uint32_t adrp_offset, bool has_thunk,
- uint32_t dex_cache_arrays_begin, uint32_t element_offset) {
+ void TestAdrpLdurLdr(uint32_t adrp_offset,
+ bool has_thunk,
+ uint32_t dex_cache_arrays_begin,
+ uint32_t element_offset) {
TestAdrpInsn2Ldr(kLdurInsn, adrp_offset, has_thunk, dex_cache_arrays_begin, element_offset);
}
- void TestAdrpLdrPcRelLdr(uint32_t pcrel_ldr_insn, int32_t pcrel_disp,
- uint32_t adrp_offset, bool has_thunk,
- uint32_t dex_cache_arrays_begin, uint32_t element_offset) {
+ void TestAdrpLdrPcRelLdr(uint32_t pcrel_ldr_insn,
+ int32_t pcrel_disp,
+ uint32_t adrp_offset,
+ bool has_thunk,
+ uint32_t dex_cache_arrays_begin,
+ uint32_t element_offset) {
ASSERT_LT(pcrel_disp, 0x100000);
ASSERT_GE(pcrel_disp, -0x100000);
ASSERT_EQ(pcrel_disp & 0x3, 0);
@@ -300,13 +375,60 @@
TestAdrpInsn2Ldr(insn2, adrp_offset, has_thunk, dex_cache_arrays_begin, element_offset);
}
- void TestAdrpLdrSpRelLdr(uint32_t sprel_ldr_insn, uint32_t sprel_disp_in_load_units,
- uint32_t adrp_offset, bool has_thunk,
- uint32_t dex_cache_arrays_begin, uint32_t element_offset) {
+ void TestAdrpLdrSpRelLdr(uint32_t sprel_ldr_insn,
+ uint32_t sprel_disp_in_load_units,
+ uint32_t adrp_offset,
+ bool has_thunk,
+ uint32_t dex_cache_arrays_begin,
+ uint32_t element_offset) {
ASSERT_LT(sprel_disp_in_load_units, 0x1000u);
uint32_t insn2 = sprel_ldr_insn | ((sprel_disp_in_load_units & 0xfffu) << 10);
TestAdrpInsn2Ldr(insn2, adrp_offset, has_thunk, dex_cache_arrays_begin, element_offset);
}
+
+ void TestAdrpInsn2Add(uint32_t insn2,
+ uint32_t adrp_offset,
+ bool has_thunk,
+ uint32_t string_offset) {
+ uint32_t method1_offset =
+ CompiledCode::AlignCode(kTrampolineSize, kArm64) + sizeof(OatQuickMethodHeader);
+ ASSERT_LT(method1_offset, adrp_offset);
+ CHECK_ALIGNED(adrp_offset, 4u);
+ uint32_t num_nops = (adrp_offset - method1_offset) / 4u;
+ PrepareNopsAdrpInsn2Add(num_nops, insn2, string_offset);
+ if (has_thunk) {
+ TestNopsAdrpInsn2AndUseHasThunk(num_nops, insn2, string_offset, kAddXInsn);
+ } else {
+ TestNopsAdrpInsn2AndUse(num_nops, insn2, string_offset, kAddXInsn);
+ }
+ ASSERT_EQ(method1_offset, GetMethodOffset(1u)); // If this fails, num_nops is wrong.
+ }
+
+ void TestAdrpLdurAdd(uint32_t adrp_offset, bool has_thunk, uint32_t string_offset) {
+ TestAdrpInsn2Add(kLdurInsn, adrp_offset, has_thunk, string_offset);
+ }
+
+ void TestAdrpLdrPcRelAdd(uint32_t pcrel_ldr_insn,
+ int32_t pcrel_disp,
+ uint32_t adrp_offset,
+ bool has_thunk,
+ uint32_t string_offset) {
+ ASSERT_LT(pcrel_disp, 0x100000);
+ ASSERT_GE(pcrel_disp, -0x100000);
+ ASSERT_EQ(pcrel_disp & 0x3, 0);
+ uint32_t insn2 = pcrel_ldr_insn | (((static_cast<uint32_t>(pcrel_disp) >> 2) & 0x7ffffu) << 5);
+ TestAdrpInsn2Add(insn2, adrp_offset, has_thunk, string_offset);
+ }
+
+ void TestAdrpLdrSpRelAdd(uint32_t sprel_ldr_insn,
+ uint32_t sprel_disp_in_load_units,
+ uint32_t adrp_offset,
+ bool has_thunk,
+ uint32_t string_offset) {
+ ASSERT_LT(sprel_disp_in_load_units, 0x1000u);
+ uint32_t insn2 = sprel_ldr_insn | ((sprel_disp_in_load_units & 0xfffu) << 10);
+ TestAdrpInsn2Add(insn2, adrp_offset, has_thunk, string_offset);
+ }
};
const uint8_t Arm64RelativePatcherTest::kCallRawCode[] = {
@@ -358,14 +480,14 @@
uint32_t method1_offset = GetMethodOffset(1u);
uint32_t method2_offset = GetMethodOffset(2u);
uint32_t diff_after = method2_offset - method1_offset;
- ASSERT_EQ(diff_after & 3u, 0u);
+ CHECK_ALIGNED(diff_after, 4u);
ASSERT_LT(diff_after >> 2, 1u << 8); // Simple encoding, (diff_after >> 2) fits into 8 bits.
static const uint8_t method1_expected_code[] = {
static_cast<uint8_t>(diff_after >> 2), 0x00, 0x00, 0x94
};
EXPECT_TRUE(CheckLinkedMethod(MethodRef(1u), ArrayRef<const uint8_t>(method1_expected_code)));
uint32_t diff_before = method1_offset - method2_offset;
- ASSERT_EQ(diff_before & 3u, 0u);
+ CHECK_ALIGNED(diff_before, 4u);
ASSERT_GE(diff_before, -1u << 27);
auto method2_expected_code = GenNopsAndBl(0u, kBlPlus0 | ((diff_before >> 2) & 0x03ffffffu));
EXPECT_TRUE(CheckLinkedMethod(MethodRef(2u), ArrayRef<const uint8_t>(method2_expected_code)));
@@ -411,7 +533,7 @@
uint32_t thunk_offset =
CompiledCode::AlignCode(last_method_offset + last_method_code.size(), kArm64);
uint32_t diff = thunk_offset - (last_method_offset + bl_offset_in_last_method);
- ASSERT_EQ(diff & 3u, 0u);
+ CHECK_ALIGNED(diff, 4u);
ASSERT_LT(diff, 128 * MB);
auto expected_code = GenNopsAndBl(1u, kBlPlus0 | (diff >> 2));
EXPECT_TRUE(CheckLinkedMethod(MethodRef(last_method_idx),
@@ -497,7 +619,7 @@
uint32_t thunk_offset = last_method_header_offset - CompiledCode::AlignCode(ThunkSize(), kArm64);
ASSERT_TRUE(IsAligned<kArm64Alignment>(thunk_offset));
uint32_t diff = thunk_offset - (method1_offset + bl_offset_in_method1);
- ASSERT_EQ(diff & 3u, 0u);
+ CHECK_ALIGNED(diff, 4u);
ASSERT_LT(diff, 128 * MB);
auto expected_code = GenNopsAndBl(0u, kBlPlus0 | (diff >> 2));
EXPECT_TRUE(CheckLinkedMethod(MethodRef(1u), ArrayRef<const uint8_t>(expected_code)));
@@ -527,7 +649,7 @@
uint32_t thunk_offset =
CompiledCode::AlignCode(last_method_offset + last_method_code.size(), kArm64);
uint32_t diff = thunk_offset - (last_method_offset + bl_offset_in_last_method);
- ASSERT_EQ(diff & 3u, 0u);
+ CHECK_ALIGNED(diff, 4u);
ASSERT_LT(diff, 128 * MB);
auto expected_code = GenNopsAndBl(1u, kBlPlus0 | (diff >> 2));
EXPECT_TRUE(CheckLinkedMethod(MethodRef(last_method_idx),
@@ -551,74 +673,158 @@
TestNopsAdrpLdr(0u, 0x12345000u, 0x4000u);
}
-TEST_F(Arm64RelativePatcherTestDefault, DexCacheReference0xff4) {
- TestAdrpLdurLdr(0xff4u, false, 0x12345678u, 0x1234u);
+TEST_F(Arm64RelativePatcherTestDefault, StringReference1) {
+ TestNopsAdrpAdd(0u, 0x12345678u);
}
-TEST_F(Arm64RelativePatcherTestDefault, DexCacheReference0xff8) {
- TestAdrpLdurLdr(0xff8u, true, 0x12345678u, 0x1234u);
+TEST_F(Arm64RelativePatcherTestDefault, StringReference2) {
+ TestNopsAdrpAdd(0u, -0x12345678u);
}
-TEST_F(Arm64RelativePatcherTestDefault, DexCacheReference0xffc) {
- TestAdrpLdurLdr(0xffcu, true, 0x12345678u, 0x1234u);
+TEST_F(Arm64RelativePatcherTestDefault, StringReference3) {
+ TestNopsAdrpAdd(0u, 0x12345000u);
}
-TEST_F(Arm64RelativePatcherTestDefault, DexCacheReference0x1000) {
- TestAdrpLdurLdr(0x1000u, false, 0x12345678u, 0x1234u);
-}
-
-TEST_F(Arm64RelativePatcherTestDenver64, DexCacheReference0xff4) {
- TestAdrpLdurLdr(0xff4u, false, 0x12345678u, 0x1234u);
-}
-
-TEST_F(Arm64RelativePatcherTestDenver64, DexCacheReference0xff8) {
- TestAdrpLdurLdr(0xff8u, false, 0x12345678u, 0x1234u);
-}
-
-TEST_F(Arm64RelativePatcherTestDenver64, DexCacheReference0xffc) {
- TestAdrpLdurLdr(0xffcu, false, 0x12345678u, 0x1234u);
-}
-
-TEST_F(Arm64RelativePatcherTestDenver64, DexCacheReference0x1000) {
- TestAdrpLdurLdr(0x1000u, false, 0x12345678u, 0x1234u);
+TEST_F(Arm64RelativePatcherTestDefault, StringReference4) {
+ TestNopsAdrpAdd(0u, 0x12345ffcu);
}
#define TEST_FOR_OFFSETS(test, disp1, disp2) \
test(0xff4u, disp1) test(0xff8u, disp1) test(0xffcu, disp1) test(0x1000u, disp1) \
test(0xff4u, disp2) test(0xff8u, disp2) test(0xffcu, disp2) test(0x1000u, disp2)
+#define DEFAULT_LDUR_LDR_TEST(adrp_offset, disp) \
+ TEST_F(Arm64RelativePatcherTestDefault, DexCacheReference ## adrp_offset ## Ldur ## disp) { \
+ bool has_thunk = (adrp_offset == 0xff8u || adrp_offset == 0xffcu); \
+ TestAdrpLdurLdr(adrp_offset, has_thunk, 0x12345678u, disp); \
+ }
+
+TEST_FOR_OFFSETS(DEFAULT_LDUR_LDR_TEST, 0x1234, 0x1238)
+
+#define DENVER64_LDUR_LDR_TEST(adrp_offset, disp) \
+ TEST_F(Arm64RelativePatcherTestDenver64, DexCacheReference ## adrp_offset ## Ldur ## disp) { \
+ TestAdrpLdurLdr(adrp_offset, false, 0x12345678u, disp); \
+ }
+
+TEST_FOR_OFFSETS(DENVER64_LDUR_LDR_TEST, 0x1234, 0x1238)
+
// LDR <Wt>, <label> is always aligned. We should never have to use a fixup.
-#define LDRW_PCREL_TEST(adrp_offset, disp) \
+#define LDRW_PCREL_LDR_TEST(adrp_offset, disp) \
TEST_F(Arm64RelativePatcherTestDefault, DexCacheReference ## adrp_offset ## WPcRel ## disp) { \
TestAdrpLdrPcRelLdr(kLdrWPcRelInsn, disp, adrp_offset, false, 0x12345678u, 0x1234u); \
}
-TEST_FOR_OFFSETS(LDRW_PCREL_TEST, 0x1234, 0x1238)
+TEST_FOR_OFFSETS(LDRW_PCREL_LDR_TEST, 0x1234, 0x1238)
// LDR <Xt>, <label> is aligned when offset + displacement is a multiple of 8.
-#define LDRX_PCREL_TEST(adrp_offset, disp) \
+#define LDRX_PCREL_LDR_TEST(adrp_offset, disp) \
TEST_F(Arm64RelativePatcherTestDefault, DexCacheReference ## adrp_offset ## XPcRel ## disp) { \
- bool unaligned = ((adrp_offset + 4u + static_cast<uint32_t>(disp)) & 7u) != 0; \
+ bool unaligned = !IsAligned<8u>(adrp_offset + 4u + static_cast<uint32_t>(disp)); \
bool has_thunk = (adrp_offset == 0xff8u || adrp_offset == 0xffcu) && unaligned; \
TestAdrpLdrPcRelLdr(kLdrXPcRelInsn, disp, adrp_offset, has_thunk, 0x12345678u, 0x1234u); \
}
-TEST_FOR_OFFSETS(LDRX_PCREL_TEST, 0x1234, 0x1238)
+TEST_FOR_OFFSETS(LDRX_PCREL_LDR_TEST, 0x1234, 0x1238)
// LDR <Wt>, [SP, #<pimm>] and LDR <Xt>, [SP, #<pimm>] are always aligned. No fixup needed.
-#define LDRW_SPREL_TEST(adrp_offset, disp) \
+#define LDRW_SPREL_LDR_TEST(adrp_offset, disp) \
TEST_F(Arm64RelativePatcherTestDefault, DexCacheReference ## adrp_offset ## WSpRel ## disp) { \
TestAdrpLdrSpRelLdr(kLdrWSpRelInsn, disp >> 2, adrp_offset, false, 0x12345678u, 0x1234u); \
}
-TEST_FOR_OFFSETS(LDRW_SPREL_TEST, 0, 4)
+TEST_FOR_OFFSETS(LDRW_SPREL_LDR_TEST, 0, 4)
-#define LDRX_SPREL_TEST(adrp_offset, disp) \
+#define LDRX_SPREL_LDR_TEST(adrp_offset, disp) \
TEST_F(Arm64RelativePatcherTestDefault, DexCacheReference ## adrp_offset ## XSpRel ## disp) { \
TestAdrpLdrSpRelLdr(kLdrXSpRelInsn, disp >> 3, adrp_offset, false, 0x12345678u, 0x1234u); \
}
-TEST_FOR_OFFSETS(LDRX_SPREL_TEST, 0, 8)
+TEST_FOR_OFFSETS(LDRX_SPREL_LDR_TEST, 0, 8)
+
+#define DEFAULT_LDUR_ADD_TEST(adrp_offset, disp) \
+ TEST_F(Arm64RelativePatcherTestDefault, StringReference ## adrp_offset ## Ldur ## disp) { \
+ bool has_thunk = (adrp_offset == 0xff8u || adrp_offset == 0xffcu); \
+ TestAdrpLdurAdd(adrp_offset, has_thunk, disp); \
+ }
+
+TEST_FOR_OFFSETS(DEFAULT_LDUR_ADD_TEST, 0x12345678, 0xffffc840)
+
+#define DENVER64_LDUR_ADD_TEST(adrp_offset, disp) \
+ TEST_F(Arm64RelativePatcherTestDenver64, StringReference ## adrp_offset ## Ldur ## disp) { \
+ TestAdrpLdurAdd(adrp_offset, false, disp); \
+ }
+
+TEST_FOR_OFFSETS(DENVER64_LDUR_ADD_TEST, 0x12345678, 0xffffc840)
+
+#define DEFAULT_SUBX3X2_ADD_TEST(adrp_offset, disp) \
+ TEST_F(Arm64RelativePatcherTestDefault, StringReference ## adrp_offset ## SubX3X2 ## disp) { \
+ /* SUB unrelated to "ADRP x0, addr". */ \
+ uint32_t sub = kSubXInsn | (100 << 10) | (2u << 5) | 3u; /* SUB x3, x2, #100 */ \
+ TestAdrpInsn2Add(sub, adrp_offset, false, disp); \
+ }
+
+TEST_FOR_OFFSETS(DEFAULT_SUBX3X2_ADD_TEST, 0x12345678, 0xffffc840)
+
+#define DEFAULT_SUBSX3X0_ADD_TEST(adrp_offset, disp) \
+ TEST_F(Arm64RelativePatcherTestDefault, StringReference ## adrp_offset ## SubsX3X0 ## disp) { \
+ /* SUBS that uses the result of "ADRP x0, addr". */ \
+ uint32_t subs = kSubsXInsn | (100 << 10) | (0u << 5) | 3u; /* SUBS x3, x0, #100 */ \
+ TestAdrpInsn2Add(subs, adrp_offset, false, disp); \
+ }
+
+TEST_FOR_OFFSETS(DEFAULT_SUBSX3X0_ADD_TEST, 0x12345678, 0xffffc840)
+
+#define DEFAULT_ADDX0X0_ADD_TEST(adrp_offset, disp) \
+ TEST_F(Arm64RelativePatcherTestDefault, StringReference ## adrp_offset ## AddX0X0 ## disp) { \
+ /* ADD that uses the result register of "ADRP x0, addr" as both source and destination. */ \
+ uint32_t add = kSubXInsn | (100 << 10) | (0u << 5) | 0u; /* ADD x0, x0, #100 */ \
+ TestAdrpInsn2Add(add, adrp_offset, false, disp); \
+ }
+
+TEST_FOR_OFFSETS(DEFAULT_ADDX0X0_ADD_TEST, 0x12345678, 0xffffc840)
+
+#define DEFAULT_ADDSX0X2_ADD_TEST(adrp_offset, disp) \
+ TEST_F(Arm64RelativePatcherTestDefault, StringReference ## adrp_offset ## AddsX0X2 ## disp) { \
+ /* ADDS that does not use the result of "ADRP x0, addr" but overwrites that register. */ \
+ uint32_t adds = kAddsXInsn | (100 << 10) | (2u << 5) | 0u; /* ADDS x0, x2, #100 */ \
+ bool has_thunk = (adrp_offset == 0xff8u || adrp_offset == 0xffcu); \
+ TestAdrpInsn2Add(adds, adrp_offset, has_thunk, disp); \
+ }
+
+TEST_FOR_OFFSETS(DEFAULT_ADDSX0X2_ADD_TEST, 0x12345678, 0xffffc840)
+
+// LDR <Wt>, <label> is always aligned. We should never have to use a fixup.
+#define LDRW_PCREL_ADD_TEST(adrp_offset, disp) \
+ TEST_F(Arm64RelativePatcherTestDefault, StringReference ## adrp_offset ## WPcRel ## disp) { \
+ TestAdrpLdrPcRelAdd(kLdrWPcRelInsn, disp, adrp_offset, false, 0x12345678u); \
+ }
+
+TEST_FOR_OFFSETS(LDRW_PCREL_ADD_TEST, 0x1234, 0x1238)
+
+// LDR <Xt>, <label> is aligned when offset + displacement is a multiple of 8.
+#define LDRX_PCREL_ADD_TEST(adrp_offset, disp) \
+ TEST_F(Arm64RelativePatcherTestDefault, StringReference ## adrp_offset ## XPcRel ## disp) { \
+ bool unaligned = !IsAligned<8u>(adrp_offset + 4u + static_cast<uint32_t>(disp)); \
+ bool has_thunk = (adrp_offset == 0xff8u || adrp_offset == 0xffcu) && unaligned; \
+ TestAdrpLdrPcRelAdd(kLdrXPcRelInsn, disp, adrp_offset, has_thunk, 0x12345678u); \
+ }
+
+TEST_FOR_OFFSETS(LDRX_PCREL_ADD_TEST, 0x1234, 0x1238)
+
+// LDR <Wt>, [SP, #<pimm>] and LDR <Xt>, [SP, #<pimm>] are always aligned. No fixup needed.
+#define LDRW_SPREL_ADD_TEST(adrp_offset, disp) \
+ TEST_F(Arm64RelativePatcherTestDefault, StringReference ## adrp_offset ## WSpRel ## disp) { \
+ TestAdrpLdrSpRelAdd(kLdrWSpRelInsn, disp >> 2, adrp_offset, false, 0x12345678u); \
+ }
+
+TEST_FOR_OFFSETS(LDRW_SPREL_ADD_TEST, 0, 4)
+
+#define LDRX_SPREL_ADD_TEST(adrp_offset, disp) \
+ TEST_F(Arm64RelativePatcherTestDefault, StringReference ## adrp_offset ## XSpRel ## disp) { \
+ TestAdrpLdrSpRelAdd(kLdrXSpRelInsn, disp >> 3, adrp_offset, false, 0x12345678u); \
+ }
+
+TEST_FOR_OFFSETS(LDRX_SPREL_ADD_TEST, 0, 8)
} // namespace linker
} // namespace art
diff --git a/compiler/linker/multi_oat_relative_patcher.h b/compiler/linker/multi_oat_relative_patcher.h
index 1727d52..dbda03f 100644
--- a/compiler/linker/multi_oat_relative_patcher.h
+++ b/compiler/linker/multi_oat_relative_patcher.h
@@ -103,13 +103,13 @@
}
// Wrapper around RelativePatcher::PatchDexCacheReference(), doing offset adjustment.
- void PatchDexCacheReference(std::vector<uint8_t>* code,
- const LinkerPatch& patch,
- uint32_t patch_offset,
- uint32_t target_offset) {
+ void PatchPcRelativeReference(std::vector<uint8_t>* code,
+ const LinkerPatch& patch,
+ uint32_t patch_offset,
+ uint32_t target_offset) {
patch_offset += adjustment_;
target_offset += adjustment_;
- relative_patcher_->PatchDexCacheReference(code, patch, patch_offset, target_offset);
+ relative_patcher_->PatchPcRelativeReference(code, patch, patch_offset, target_offset);
}
// Wrappers around RelativePatcher for statistics retrieval.
diff --git a/compiler/linker/multi_oat_relative_patcher_test.cc b/compiler/linker/multi_oat_relative_patcher_test.cc
index 792cdfe..92a96a0 100644
--- a/compiler/linker/multi_oat_relative_patcher_test.cc
+++ b/compiler/linker/multi_oat_relative_patcher_test.cc
@@ -86,10 +86,10 @@
last_target_offset_ = target_offset;
}
- void PatchDexCacheReference(std::vector<uint8_t>* code ATTRIBUTE_UNUSED,
- const LinkerPatch& patch,
- uint32_t patch_offset,
- uint32_t target_offset) OVERRIDE {
+ void PatchPcRelativeReference(std::vector<uint8_t>* code ATTRIBUTE_UNUSED,
+ const LinkerPatch& patch,
+ uint32_t patch_offset,
+ uint32_t target_offset) OVERRIDE {
last_literal_offset_ = patch.LiteralOffset();
last_patch_offset_ = patch_offset;
last_target_offset_ = target_offset;
@@ -277,7 +277,7 @@
uint32_t method2_target_offset = 0xccccu;
LinkerPatch method2_patch =
LinkerPatch::DexCacheArrayPatch(method2_literal_offset, nullptr, 0u, 1234u);
- patcher_.PatchDexCacheReference(
+ patcher_.PatchPcRelativeReference(
&code, method2_patch, method2_patch_offset, method2_target_offset);
DCHECK_EQ(method2_literal_offset, mock_->last_literal_offset_);
DCHECK_EQ(method2_patch_offset + adjustment1, mock_->last_patch_offset_);
diff --git a/compiler/linker/relative_patcher.cc b/compiler/linker/relative_patcher.cc
index 6727c17..3a22983 100644
--- a/compiler/linker/relative_patcher.cc
+++ b/compiler/linker/relative_patcher.cc
@@ -62,10 +62,10 @@
LOG(FATAL) << "Unexpected relative call patch.";
}
- virtual void PatchDexCacheReference(std::vector<uint8_t>* code ATTRIBUTE_UNUSED,
- const LinkerPatch& patch ATTRIBUTE_UNUSED,
- uint32_t patch_offset ATTRIBUTE_UNUSED,
- uint32_t target_offset ATTRIBUTE_UNUSED) {
+ void PatchPcRelativeReference(std::vector<uint8_t>* code ATTRIBUTE_UNUSED,
+ const LinkerPatch& patch ATTRIBUTE_UNUSED,
+ uint32_t patch_offset ATTRIBUTE_UNUSED,
+ uint32_t target_offset ATTRIBUTE_UNUSED) OVERRIDE {
LOG(FATAL) << "Unexpected relative dex cache array patch.";
}
diff --git a/compiler/linker/relative_patcher.h b/compiler/linker/relative_patcher.h
index ba37451..a22b9f2 100644
--- a/compiler/linker/relative_patcher.h
+++ b/compiler/linker/relative_patcher.h
@@ -104,10 +104,10 @@
uint32_t target_offset) = 0;
// Patch a reference to a dex cache location.
- virtual void PatchDexCacheReference(std::vector<uint8_t>* code,
- const LinkerPatch& patch,
- uint32_t patch_offset,
- uint32_t target_offset) = 0;
+ virtual void PatchPcRelativeReference(std::vector<uint8_t>* code,
+ const LinkerPatch& patch,
+ uint32_t patch_offset,
+ uint32_t target_offset) = 0;
protected:
RelativePatcher()
diff --git a/compiler/linker/relative_patcher_test.h b/compiler/linker/relative_patcher_test.h
index 704135a..bf61ea0 100644
--- a/compiler/linker/relative_patcher_test.h
+++ b/compiler/linker/relative_patcher_test.h
@@ -142,20 +142,27 @@
patched_code_.assign(code.begin(), code.end());
code = ArrayRef<const uint8_t>(patched_code_);
for (const LinkerPatch& patch : compiled_method->GetPatches()) {
- if (patch.Type() == kLinkerPatchCallRelative) {
+ if (patch.GetType() == LinkerPatch::Type::kCallRelative) {
auto result = method_offset_map_.FindMethodOffset(patch.TargetMethod());
uint32_t target_offset =
result.first ? result.second : kTrampolineOffset + compiled_method->CodeDelta();
patcher_->PatchCall(&patched_code_, patch.LiteralOffset(),
offset + patch.LiteralOffset(), target_offset);
- } else if (patch.Type() == kLinkerPatchDexCacheArray) {
+ } else if (patch.GetType() == LinkerPatch::Type::kDexCacheArray) {
uint32_t target_offset = dex_cache_arrays_begin_ + patch.TargetDexCacheElementOffset();
- patcher_->PatchDexCacheReference(&patched_code_,
- patch,
- offset + patch.LiteralOffset(),
- target_offset);
+ patcher_->PatchPcRelativeReference(&patched_code_,
+ patch,
+ offset + patch.LiteralOffset(),
+ target_offset);
+ } else if (patch.GetType() == LinkerPatch::Type::kStringRelative) {
+ uint32_t target_offset = string_index_to_offset_map_.Get(patch.TargetStringIndex());
+ patcher_->PatchPcRelativeReference(&patched_code_,
+ patch,
+ offset + patch.LiteralOffset(),
+ target_offset);
} else {
- LOG(FATAL) << "Bad patch type.";
+ LOG(FATAL) << "Bad patch type. " << patch.GetType();
+ UNREACHABLE();
}
}
}
@@ -257,6 +264,7 @@
MethodOffsetMap method_offset_map_;
std::unique_ptr<RelativePatcher> patcher_;
uint32_t dex_cache_arrays_begin_;
+ SafeMap<uint32_t, uint32_t> string_index_to_offset_map_;
std::vector<MethodReference> compiled_method_refs_;
std::vector<std::unique_ptr<CompiledMethod>> compiled_methods_;
std::vector<uint8_t> patched_code_;
diff --git a/compiler/linker/x86/relative_patcher_x86.cc b/compiler/linker/x86/relative_patcher_x86.cc
index 24b1481..768d31a 100644
--- a/compiler/linker/x86/relative_patcher_x86.cc
+++ b/compiler/linker/x86/relative_patcher_x86.cc
@@ -21,10 +21,10 @@
namespace art {
namespace linker {
-void X86RelativePatcher::PatchDexCacheReference(std::vector<uint8_t>* code,
- const LinkerPatch& patch,
- uint32_t patch_offset,
- uint32_t target_offset) {
+void X86RelativePatcher::PatchPcRelativeReference(std::vector<uint8_t>* code,
+ const LinkerPatch& patch,
+ uint32_t patch_offset,
+ uint32_t target_offset) {
uint32_t anchor_literal_offset = patch.PcInsnOffset();
uint32_t literal_offset = patch.LiteralOffset();
diff --git a/compiler/linker/x86/relative_patcher_x86.h b/compiler/linker/x86/relative_patcher_x86.h
index ddc244c..fbf9ad4 100644
--- a/compiler/linker/x86/relative_patcher_x86.h
+++ b/compiler/linker/x86/relative_patcher_x86.h
@@ -26,10 +26,10 @@
public:
X86RelativePatcher() { }
- void PatchDexCacheReference(std::vector<uint8_t>* code,
- const LinkerPatch& patch,
- uint32_t patch_offset,
- uint32_t target_offset) OVERRIDE;
+ void PatchPcRelativeReference(std::vector<uint8_t>* code,
+ const LinkerPatch& patch,
+ uint32_t patch_offset,
+ uint32_t target_offset) OVERRIDE;
};
} // namespace linker
diff --git a/compiler/linker/x86/relative_patcher_x86_test.cc b/compiler/linker/x86/relative_patcher_x86_test.cc
index 7acc330..2a44b79 100644
--- a/compiler/linker/x86/relative_patcher_x86_test.cc
+++ b/compiler/linker/x86/relative_patcher_x86_test.cc
@@ -70,15 +70,19 @@
uint32_t diff_after = method2_offset - (method1_offset + kCallCode.size() /* PC adjustment */);
static const uint8_t method1_expected_code[] = {
0xe8,
- static_cast<uint8_t>(diff_after), static_cast<uint8_t>(diff_after >> 8),
- static_cast<uint8_t>(diff_after >> 16), static_cast<uint8_t>(diff_after >> 24)
+ static_cast<uint8_t>(diff_after),
+ static_cast<uint8_t>(diff_after >> 8),
+ static_cast<uint8_t>(diff_after >> 16),
+ static_cast<uint8_t>(diff_after >> 24)
};
EXPECT_TRUE(CheckLinkedMethod(MethodRef(1u), ArrayRef<const uint8_t>(method1_expected_code)));
uint32_t diff_before = method1_offset - (method2_offset + kCallCode.size() /* PC adjustment */);
static const uint8_t method2_expected_code[] = {
0xe8,
- static_cast<uint8_t>(diff_before), static_cast<uint8_t>(diff_before >> 8),
- static_cast<uint8_t>(diff_before >> 16), static_cast<uint8_t>(diff_before >> 24)
+ static_cast<uint8_t>(diff_before),
+ static_cast<uint8_t>(diff_before >> 8),
+ static_cast<uint8_t>(diff_before >> 16),
+ static_cast<uint8_t>(diff_before >> 24)
};
EXPECT_TRUE(CheckLinkedMethod(MethodRef(2u), ArrayRef<const uint8_t>(method2_expected_code)));
}
@@ -95,8 +99,10 @@
uint32_t diff = kTrampolineOffset - (result.second + kCallCode.size());
static const uint8_t expected_code[] = {
0xe8,
- static_cast<uint8_t>(diff), static_cast<uint8_t>(diff >> 8),
- static_cast<uint8_t>(diff >> 16), static_cast<uint8_t>(diff >> 24)
+ static_cast<uint8_t>(diff),
+ static_cast<uint8_t>(diff >> 8),
+ static_cast<uint8_t>(diff >> 16),
+ static_cast<uint8_t>(diff >> 24)
};
EXPECT_TRUE(CheckLinkedMethod(MethodRef(1u), ArrayRef<const uint8_t>(expected_code)));
}
@@ -125,8 +131,42 @@
0xe8, 0x00, 0x00, 0x00, 0x00, // call +0
0x5b, // pop ebx
0x8b, 0x83, // mov eax, [ebx + diff]
- static_cast<uint8_t>(diff), static_cast<uint8_t>(diff >> 8),
- static_cast<uint8_t>(diff >> 16), static_cast<uint8_t>(diff >> 24)
+ static_cast<uint8_t>(diff),
+ static_cast<uint8_t>(diff >> 8),
+ static_cast<uint8_t>(diff >> 16),
+ static_cast<uint8_t>(diff >> 24)
+ };
+ EXPECT_TRUE(CheckLinkedMethod(MethodRef(1u), ArrayRef<const uint8_t>(expected_code)));
+}
+
+TEST_F(X86RelativePatcherTest, StringReference) {
+ constexpr uint32_t kStringIndex = 1u;
+ constexpr uint32_t kStringOffset = 0x12345678;
+ string_index_to_offset_map_.Put(kStringIndex, kStringOffset);
+ static const uint8_t raw_code[] = {
+ 0xe8, 0x00, 0x00, 0x00, 0x00, // call +0
+ 0x5b, // pop ebx
+ 0x8d, 0x83, 0x00, 0x01, 0x00, 0x00, // lea eax, [ebx + 256 (kDummy32BitValue)]
+ };
+ constexpr uint32_t anchor_offset = 5u; // After call +0.
+ ArrayRef<const uint8_t> code(raw_code);
+ LinkerPatch patches[] = {
+ LinkerPatch::RelativeStringPatch(code.size() - 4u, nullptr, anchor_offset, kStringIndex),
+ };
+ AddCompiledMethod(MethodRef(1u), code, ArrayRef<const LinkerPatch>(patches));
+ Link();
+
+ auto result = method_offset_map_.FindMethodOffset(MethodRef(1u));
+ ASSERT_TRUE(result.first);
+ uint32_t diff = kStringOffset - (result.second + anchor_offset);
+ static const uint8_t expected_code[] = {
+ 0xe8, 0x00, 0x00, 0x00, 0x00, // call +0
+ 0x5b, // pop ebx
+ 0x8d, 0x83, // lea eax, [ebx + diff]
+ static_cast<uint8_t>(diff),
+ static_cast<uint8_t>(diff >> 8),
+ static_cast<uint8_t>(diff >> 16),
+ static_cast<uint8_t>(diff >> 24)
};
EXPECT_TRUE(CheckLinkedMethod(MethodRef(1u), ArrayRef<const uint8_t>(expected_code)));
}
diff --git a/compiler/linker/x86_64/relative_patcher_x86_64.cc b/compiler/linker/x86_64/relative_patcher_x86_64.cc
index e571f50..2ff6930 100644
--- a/compiler/linker/x86_64/relative_patcher_x86_64.cc
+++ b/compiler/linker/x86_64/relative_patcher_x86_64.cc
@@ -21,10 +21,10 @@
namespace art {
namespace linker {
-void X86_64RelativePatcher::PatchDexCacheReference(std::vector<uint8_t>* code,
- const LinkerPatch& patch,
- uint32_t patch_offset,
- uint32_t target_offset) {
+void X86_64RelativePatcher::PatchPcRelativeReference(std::vector<uint8_t>* code,
+ const LinkerPatch& patch,
+ uint32_t patch_offset,
+ uint32_t target_offset) {
DCHECK_LE(patch.LiteralOffset() + 4u, code->size());
// Unsigned arithmetic with its well-defined overflow behavior is just fine here.
uint32_t displacement = target_offset - patch_offset;
diff --git a/compiler/linker/x86_64/relative_patcher_x86_64.h b/compiler/linker/x86_64/relative_patcher_x86_64.h
index feecb3a..11bb6d5 100644
--- a/compiler/linker/x86_64/relative_patcher_x86_64.h
+++ b/compiler/linker/x86_64/relative_patcher_x86_64.h
@@ -26,10 +26,10 @@
public:
X86_64RelativePatcher() { }
- void PatchDexCacheReference(std::vector<uint8_t>* code,
- const LinkerPatch& patch,
- uint32_t patch_offset,
- uint32_t target_offset) OVERRIDE;
+ void PatchPcRelativeReference(std::vector<uint8_t>* code,
+ const LinkerPatch& patch,
+ uint32_t patch_offset,
+ uint32_t target_offset) OVERRIDE;
};
} // namespace linker
diff --git a/compiler/linker/x86_64/relative_patcher_x86_64_test.cc b/compiler/linker/x86_64/relative_patcher_x86_64_test.cc
index 36e0f01..2b46453 100644
--- a/compiler/linker/x86_64/relative_patcher_x86_64_test.cc
+++ b/compiler/linker/x86_64/relative_patcher_x86_64_test.cc
@@ -29,6 +29,8 @@
static const ArrayRef<const uint8_t> kCallCode;
static const uint8_t kDexCacheLoadRawCode[];
static const ArrayRef<const uint8_t> kDexCacheLoadCode;
+ static const uint8_t kStringReferenceRawCode[];
+ static const ArrayRef<const uint8_t> kStringReferenceCode;
uint32_t GetMethodOffset(uint32_t method_idx) {
auto result = method_offset_map_.FindMethodOffset(MethodRef(method_idx));
@@ -51,6 +53,14 @@
const ArrayRef<const uint8_t> X86_64RelativePatcherTest::kDexCacheLoadCode(
kDexCacheLoadRawCode);
+const uint8_t X86_64RelativePatcherTest::kStringReferenceRawCode[] = {
+ 0x8d, 0x05, // lea eax, [rip + <offset>]
+ 0x00, 0x01, 0x00, 0x00
+};
+
+const ArrayRef<const uint8_t> X86_64RelativePatcherTest::kStringReferenceCode(
+ kStringReferenceRawCode);
+
TEST_F(X86_64RelativePatcherTest, CallSelf) {
LinkerPatch patches[] = {
LinkerPatch::RelativeCodePatch(kCallCode.size() - 4u, nullptr, 1u),
@@ -80,15 +90,19 @@
uint32_t diff_after = method2_offset - (method1_offset + kCallCode.size() /* PC adjustment */);
static const uint8_t method1_expected_code[] = {
0xe8,
- static_cast<uint8_t>(diff_after), static_cast<uint8_t>(diff_after >> 8),
- static_cast<uint8_t>(diff_after >> 16), static_cast<uint8_t>(diff_after >> 24)
+ static_cast<uint8_t>(diff_after),
+ static_cast<uint8_t>(diff_after >> 8),
+ static_cast<uint8_t>(diff_after >> 16),
+ static_cast<uint8_t>(diff_after >> 24)
};
EXPECT_TRUE(CheckLinkedMethod(MethodRef(1u), ArrayRef<const uint8_t>(method1_expected_code)));
uint32_t diff_before = method1_offset - (method2_offset + kCallCode.size() /* PC adjustment */);
static const uint8_t method2_expected_code[] = {
0xe8,
- static_cast<uint8_t>(diff_before), static_cast<uint8_t>(diff_before >> 8),
- static_cast<uint8_t>(diff_before >> 16), static_cast<uint8_t>(diff_before >> 24)
+ static_cast<uint8_t>(diff_before),
+ static_cast<uint8_t>(diff_before >> 8),
+ static_cast<uint8_t>(diff_before >> 16),
+ static_cast<uint8_t>(diff_before >> 24)
};
EXPECT_TRUE(CheckLinkedMethod(MethodRef(2u), ArrayRef<const uint8_t>(method2_expected_code)));
}
@@ -105,8 +119,10 @@
uint32_t diff = kTrampolineOffset - (result.second + kCallCode.size());
static const uint8_t expected_code[] = {
0xe8,
- static_cast<uint8_t>(diff), static_cast<uint8_t>(diff >> 8),
- static_cast<uint8_t>(diff >> 16), static_cast<uint8_t>(diff >> 24)
+ static_cast<uint8_t>(diff),
+ static_cast<uint8_t>(diff >> 8),
+ static_cast<uint8_t>(diff >> 16),
+ static_cast<uint8_t>(diff >> 24)
};
EXPECT_TRUE(CheckLinkedMethod(MethodRef(1u), ArrayRef<const uint8_t>(expected_code)));
}
@@ -126,8 +142,34 @@
dex_cache_arrays_begin_ + kElementOffset - (result.second + kDexCacheLoadCode.size());
static const uint8_t expected_code[] = {
0x8b, 0x05,
- static_cast<uint8_t>(diff), static_cast<uint8_t>(diff >> 8),
- static_cast<uint8_t>(diff >> 16), static_cast<uint8_t>(diff >> 24)
+ static_cast<uint8_t>(diff),
+ static_cast<uint8_t>(diff >> 8),
+ static_cast<uint8_t>(diff >> 16),
+ static_cast<uint8_t>(diff >> 24)
+ };
+ EXPECT_TRUE(CheckLinkedMethod(MethodRef(1u), ArrayRef<const uint8_t>(expected_code)));
+}
+
+TEST_F(X86_64RelativePatcherTest, StringReference) {
+ constexpr uint32_t kStringIndex = 1u;
+ constexpr uint32_t kStringOffset = 0x12345678;
+ string_index_to_offset_map_.Put(kStringIndex, kStringOffset);
+ LinkerPatch patches[] = {
+ LinkerPatch::RelativeStringPatch(
+ kStringReferenceCode.size() - 4u, nullptr, 0u, kStringIndex),
+ };
+ AddCompiledMethod(MethodRef(1u), kStringReferenceCode, ArrayRef<const LinkerPatch>(patches));
+ Link();
+
+ auto result = method_offset_map_.FindMethodOffset(MethodRef(1u));
+ ASSERT_TRUE(result.first);
+ uint32_t diff = kStringOffset - (result.second + kStringReferenceCode.size());
+ static const uint8_t expected_code[] = {
+ 0x8d, 0x05,
+ static_cast<uint8_t>(diff),
+ static_cast<uint8_t>(diff >> 8),
+ static_cast<uint8_t>(diff >> 16),
+ static_cast<uint8_t>(diff >> 24)
};
EXPECT_TRUE(CheckLinkedMethod(MethodRef(1u), ArrayRef<const uint8_t>(expected_code)));
}
diff --git a/compiler/oat_writer.cc b/compiler/oat_writer.cc
index c2f19c9..3a67b1e 100644
--- a/compiler/oat_writer.cc
+++ b/compiler/oat_writer.cc
@@ -1046,6 +1046,7 @@
OatDexMethodVisitor::StartClass(dex_file, class_def_index);
if (dex_cache_ == nullptr || dex_cache_->GetDexFile() != dex_file) {
dex_cache_ = class_linker_->FindDexCache(Thread::Current(), *dex_file);
+ DCHECK(dex_cache_ != nullptr);
}
return true;
}
@@ -1115,28 +1116,56 @@
quick_code = ArrayRef<const uint8_t>(patched_code_);
for (const LinkerPatch& patch : compiled_method->GetPatches()) {
uint32_t literal_offset = patch.LiteralOffset();
- if (patch.Type() == kLinkerPatchCallRelative) {
- // NOTE: Relative calls across oat files are not supported.
- uint32_t target_offset = GetTargetOffset(patch);
- writer_->relative_patcher_->PatchCall(&patched_code_,
- literal_offset,
- offset_ + literal_offset,
- target_offset);
- } else if (patch.Type() == kLinkerPatchDexCacheArray) {
- uint32_t target_offset = GetDexCacheOffset(patch);
- writer_->relative_patcher_->PatchDexCacheReference(&patched_code_,
- patch,
- offset_ + literal_offset,
- target_offset);
- } else if (patch.Type() == kLinkerPatchCall) {
- uint32_t target_offset = GetTargetOffset(patch);
- PatchCodeAddress(&patched_code_, literal_offset, target_offset);
- } else if (patch.Type() == kLinkerPatchMethod) {
- ArtMethod* method = GetTargetMethod(patch);
- PatchMethodAddress(&patched_code_, literal_offset, method);
- } else if (patch.Type() == kLinkerPatchType) {
- mirror::Class* type = GetTargetType(patch);
- PatchObjectAddress(&patched_code_, literal_offset, type);
+ switch (patch.GetType()) {
+ case LinkerPatch::Type::kCallRelative: {
+ // NOTE: Relative calls across oat files are not supported.
+ uint32_t target_offset = GetTargetOffset(patch);
+ writer_->relative_patcher_->PatchCall(&patched_code_,
+ literal_offset,
+ offset_ + literal_offset,
+ target_offset);
+ break;
+ }
+ case LinkerPatch::Type::kDexCacheArray: {
+ uint32_t target_offset = GetDexCacheOffset(patch);
+ writer_->relative_patcher_->PatchPcRelativeReference(&patched_code_,
+ patch,
+ offset_ + literal_offset,
+ target_offset);
+ break;
+ }
+ case LinkerPatch::Type::kStringRelative: {
+ uint32_t target_offset = GetTargetObjectOffset(GetTargetString(patch));
+ writer_->relative_patcher_->PatchPcRelativeReference(&patched_code_,
+ patch,
+ offset_ + literal_offset,
+ target_offset);
+ break;
+ }
+ case LinkerPatch::Type::kCall: {
+ uint32_t target_offset = GetTargetOffset(patch);
+ PatchCodeAddress(&patched_code_, literal_offset, target_offset);
+ break;
+ }
+ case LinkerPatch::Type::kMethod: {
+ ArtMethod* method = GetTargetMethod(patch);
+ PatchMethodAddress(&patched_code_, literal_offset, method);
+ break;
+ }
+ case LinkerPatch::Type::kString: {
+ mirror::String* string = GetTargetString(patch);
+ PatchObjectAddress(&patched_code_, literal_offset, string);
+ break;
+ }
+ case LinkerPatch::Type::kType: {
+ mirror::Class* type = GetTargetType(patch);
+ PatchObjectAddress(&patched_code_, literal_offset, type);
+ break;
+ }
+ default: {
+ DCHECK_EQ(patch.GetType(), LinkerPatch::Type::kRecordPosition);
+ break;
+ }
}
}
}
@@ -1205,15 +1234,23 @@
return target_offset;
}
- mirror::Class* GetTargetType(const LinkerPatch& patch)
- SHARED_REQUIRES(Locks::mutator_lock_) {
+ mirror::Class* GetTargetType(const LinkerPatch& patch) SHARED_REQUIRES(Locks::mutator_lock_) {
mirror::DexCache* dex_cache = (dex_file_ == patch.TargetTypeDexFile())
- ? dex_cache_ : class_linker_->FindDexCache(Thread::Current(), *patch.TargetTypeDexFile());
+ ? dex_cache_
+ : class_linker_->FindDexCache(Thread::Current(), *patch.TargetTypeDexFile());
mirror::Class* type = dex_cache->GetResolvedType(patch.TargetTypeIndex());
CHECK(type != nullptr);
return type;
}
+ mirror::String* GetTargetString(const LinkerPatch& patch) SHARED_REQUIRES(Locks::mutator_lock_) {
+ mirror::String* string = dex_cache_->GetResolvedString(patch.TargetStringIndex());
+ DCHECK(string != nullptr);
+ DCHECK(writer_->HasBootImage() ||
+ Runtime::Current()->GetHeap()->ObjectIsInBootImageSpace(string));
+ return string;
+ }
+
uint32_t GetDexCacheOffset(const LinkerPatch& patch) SHARED_REQUIRES(Locks::mutator_lock_) {
if (writer_->HasBootImage()) {
uintptr_t element = writer_->image_writer_->GetDexCacheArrayElementImageAddress<uintptr_t>(
@@ -1227,6 +1264,15 @@
}
}
+ uint32_t GetTargetObjectOffset(mirror::Object* object) SHARED_REQUIRES(Locks::mutator_lock_) {
+ DCHECK(writer_->HasBootImage());
+ object = writer_->image_writer_->GetImageAddress(object);
+ size_t oat_index = writer_->image_writer_->GetOatIndexForDexFile(dex_file_);
+ uintptr_t oat_data_begin = writer_->image_writer_->GetOatDataBegin(oat_index);
+ // TODO: Clean up offset types. The target offset must be treated as signed.
+ return static_cast<uint32_t>(reinterpret_cast<uintptr_t>(object) - oat_data_begin);
+ }
+
void PatchObjectAddress(std::vector<uint8_t>* code, uint32_t offset, mirror::Object* object)
SHARED_REQUIRES(Locks::mutator_lock_) {
if (writer_->HasBootImage()) {
diff --git a/compiler/optimizing/builder.cc b/compiler/optimizing/builder.cc
index 1b62531..b6b8322 100644
--- a/compiler/optimizing/builder.cc
+++ b/compiler/optimizing/builder.cc
@@ -2745,20 +2745,16 @@
case Instruction::CONST_STRING: {
uint32_t string_index = instruction.VRegB_21c();
- bool in_dex_cache = compiler_driver_->CanAssumeStringIsPresentInDexCache(
- *dex_file_, string_index);
current_block_->AddInstruction(
- new (arena_) HLoadString(graph_->GetCurrentMethod(), string_index, dex_pc, in_dex_cache));
+ new (arena_) HLoadString(graph_->GetCurrentMethod(), string_index, *dex_file_, dex_pc));
UpdateLocal(instruction.VRegA_21c(), current_block_->GetLastInstruction(), dex_pc);
break;
}
case Instruction::CONST_STRING_JUMBO: {
uint32_t string_index = instruction.VRegB_31c();
- bool in_dex_cache = compiler_driver_->CanAssumeStringIsPresentInDexCache(
- *dex_file_, string_index);
current_block_->AddInstruction(
- new (arena_) HLoadString(graph_->GetCurrentMethod(), string_index, dex_pc, in_dex_cache));
+ new (arena_) HLoadString(graph_->GetCurrentMethod(), string_index, *dex_file_, dex_pc));
UpdateLocal(instruction.VRegA_31c(), current_block_->GetLastInstruction(), dex_pc);
break;
}
diff --git a/compiler/optimizing/code_generator.cc b/compiler/optimizing/code_generator.cc
index 216f03d..7cf9072 100644
--- a/compiler/optimizing/code_generator.cc
+++ b/compiler/optimizing/code_generator.cc
@@ -54,7 +54,6 @@
#include "parallel_move_resolver.h"
#include "ssa_liveness_analysis.h"
#include "utils/assembler.h"
-#include "verifier/dex_gc_map.h"
#include "vmap_table.h"
namespace art {
diff --git a/compiler/optimizing/code_generator.h b/compiler/optimizing/code_generator.h
index e56323f..cad5529 100644
--- a/compiler/optimizing/code_generator.h
+++ b/compiler/optimizing/code_generator.h
@@ -443,6 +443,11 @@
uint32_t dex_pc,
SlowPathCode* slow_path) = 0;
+ // Check if the desired_string_load_kind is supported. If it is, return it,
+ // otherwise return a fall-back info that should be used instead.
+ virtual HLoadString::LoadKind GetSupportedLoadStringKind(
+ HLoadString::LoadKind desired_string_load_kind) = 0;
+
// Check if the desired_dispatch_info is supported. If it is, return it,
// otherwise return a fall-back info that should be used instead.
virtual HInvokeStaticOrDirect::DispatchInfo GetSupportedInvokeStaticOrDirectDispatch(
@@ -471,6 +476,18 @@
LabelType label;
};
+ // String patch info used for recording locations of required linker patches and
+ // target strings. The actual string address can be absolute or PC-relative.
+ template <typename LabelType>
+ struct StringPatchInfo {
+ StringPatchInfo(const DexFile& df, uint32_t index)
+ : dex_file(df), string_index(index), label() { }
+
+ const DexFile& dex_file;
+ uint32_t string_index;
+ LabelType label;
+ };
+
CodeGenerator(HGraph* graph,
size_t number_of_core_registers,
size_t number_of_fpu_registers,
diff --git a/compiler/optimizing/code_generator_arm.cc b/compiler/optimizing/code_generator_arm.cc
index 3a18a0d..98577d6 100644
--- a/compiler/optimizing/code_generator_arm.cc
+++ b/compiler/optimizing/code_generator_arm.cc
@@ -779,13 +779,19 @@
move_resolver_(graph->GetArena(), this),
assembler_(),
isa_features_(isa_features),
+ uint32_literals_(std::less<uint32_t>(),
+ graph->GetArena()->Adapter(kArenaAllocCodeGenerator)),
method_patches_(MethodReferenceComparator(),
graph->GetArena()->Adapter(kArenaAllocCodeGenerator)),
call_patches_(MethodReferenceComparator(),
graph->GetArena()->Adapter(kArenaAllocCodeGenerator)),
relative_call_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)),
- dex_cache_arrays_base_labels_(std::less<HArmDexCacheArraysBase*>(),
- graph->GetArena()->Adapter(kArenaAllocCodeGenerator)) {
+ pc_relative_dex_cache_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)),
+ boot_image_string_patches_(StringReferenceValueComparator(),
+ graph->GetArena()->Adapter(kArenaAllocCodeGenerator)),
+ pc_relative_string_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)),
+ boot_image_address_patches_(std::less<uint32_t>(),
+ graph->GetArena()->Adapter(kArenaAllocCodeGenerator)) {
// Always save the LR register to mimic Quick.
AddAllocatedRegister(Location::RegisterLocation(LR));
}
@@ -5221,12 +5227,57 @@
__ Bind(slow_path->GetExitLabel());
}
+HLoadString::LoadKind CodeGeneratorARM::GetSupportedLoadStringKind(
+ HLoadString::LoadKind desired_string_load_kind) {
+ if (kEmitCompilerReadBarrier) {
+ switch (desired_string_load_kind) {
+ case HLoadString::LoadKind::kBootImageLinkTimeAddress:
+ case HLoadString::LoadKind::kBootImageLinkTimePcRelative:
+ case HLoadString::LoadKind::kBootImageAddress:
+ // TODO: Implement for read barrier.
+ return HLoadString::LoadKind::kDexCacheViaMethod;
+ default:
+ break;
+ }
+ }
+ switch (desired_string_load_kind) {
+ case HLoadString::LoadKind::kBootImageLinkTimeAddress:
+ DCHECK(!GetCompilerOptions().GetCompilePic());
+ break;
+ case HLoadString::LoadKind::kBootImageLinkTimePcRelative:
+ DCHECK(GetCompilerOptions().GetCompilePic());
+ break;
+ case HLoadString::LoadKind::kBootImageAddress:
+ break;
+ case HLoadString::LoadKind::kDexCacheAddress:
+ DCHECK(Runtime::Current()->UseJit());
+ break;
+ case HLoadString::LoadKind::kDexCachePcRelative:
+ DCHECK(!Runtime::Current()->UseJit());
+ // We disable pc-relative load when there is an irreducible loop, as the optimization
+ // is incompatible with it.
+ // TODO: Create as many ArmDexCacheArraysBase instructions as needed for methods
+ // with irreducible loops.
+ if (GetGraph()->HasIrreducibleLoops()) {
+ return HLoadString::LoadKind::kDexCacheViaMethod;
+ }
+ break;
+ case HLoadString::LoadKind::kDexCacheViaMethod:
+ break;
+ }
+ return desired_string_load_kind;
+}
+
void LocationsBuilderARM::VisitLoadString(HLoadString* load) {
- LocationSummary::CallKind call_kind = (!load->IsInDexCache() || kEmitCompilerReadBarrier)
+ LocationSummary::CallKind call_kind = (load->NeedsEnvironment() || kEmitCompilerReadBarrier)
? LocationSummary::kCallOnSlowPath
: LocationSummary::kNoCall;
LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(load, call_kind);
- locations->SetInAt(0, Location::RequiresRegister());
+ HLoadString::LoadKind load_kind = load->GetLoadKind();
+ if (load_kind == HLoadString::LoadKind::kDexCacheViaMethod ||
+ load_kind == HLoadString::LoadKind::kDexCachePcRelative) {
+ locations->SetInAt(0, Location::RequiresRegister());
+ }
locations->SetOut(Location::RequiresRegister());
}
@@ -5234,16 +5285,73 @@
LocationSummary* locations = load->GetLocations();
Location out_loc = locations->Out();
Register out = out_loc.AsRegister<Register>();
- Register current_method = locations->InAt(0).AsRegister<Register>();
- // /* GcRoot<mirror::Class> */ out = current_method->declaring_class_
- GenerateGcRootFieldLoad(
- load, out_loc, current_method, ArtMethod::DeclaringClassOffset().Int32Value());
- // /* GcRoot<mirror::String>[] */ out = out->dex_cache_strings_
- __ LoadFromOffset(kLoadWord, out, out, mirror::Class::DexCacheStringsOffset().Int32Value());
- // /* GcRoot<mirror::String> */ out = out[string_index]
- GenerateGcRootFieldLoad(
- load, out_loc, out, CodeGenerator::GetCacheOffset(load->GetStringIndex()));
+ switch (load->GetLoadKind()) {
+ case HLoadString::LoadKind::kBootImageLinkTimeAddress: {
+ DCHECK(!kEmitCompilerReadBarrier);
+ __ LoadLiteral(out, codegen_->DeduplicateBootImageStringLiteral(load->GetDexFile(),
+ load->GetStringIndex()));
+ return; // No dex cache slow path.
+ }
+ case HLoadString::LoadKind::kBootImageLinkTimePcRelative: {
+ DCHECK(!kEmitCompilerReadBarrier);
+ CodeGeneratorARM::PcRelativePatchInfo* labels =
+ codegen_->NewPcRelativeStringPatch(load->GetDexFile(), load->GetStringIndex());
+ __ BindTrackedLabel(&labels->movw_label);
+ __ movw(out, /* placeholder */ 0u);
+ __ BindTrackedLabel(&labels->movt_label);
+ __ movt(out, /* placeholder */ 0u);
+ __ BindTrackedLabel(&labels->add_pc_label);
+ __ add(out, out, ShifterOperand(PC));
+ return; // No dex cache slow path.
+ }
+ case HLoadString::LoadKind::kBootImageAddress: {
+ DCHECK(!kEmitCompilerReadBarrier);
+ DCHECK_NE(load->GetAddress(), 0u);
+ uint32_t address = dchecked_integral_cast<uint32_t>(load->GetAddress());
+ __ LoadLiteral(out, codegen_->DeduplicateBootImageAddressLiteral(address));
+ return; // No dex cache slow path.
+ }
+ case HLoadString::LoadKind::kDexCacheAddress: {
+ DCHECK_NE(load->GetAddress(), 0u);
+ uint32_t address = dchecked_integral_cast<uint32_t>(load->GetAddress());
+ // 16-bit LDR immediate has a 5-bit offset multiplied by the size and that gives
+ // a 128B range. To try and reduce the number of literals if we load multiple strings,
+ // simply split the dex cache address to a 128B aligned base loaded from a literal
+ // and the remaining offset embedded in the load.
+ static_assert(sizeof(GcRoot<mirror::String>) == 4u, "Expected GC root to be 4 bytes.");
+ DCHECK_ALIGNED(load->GetAddress(), 4u);
+ constexpr size_t offset_bits = /* encoded bits */ 5 + /* scale */ 2;
+ uint32_t base_address = address & ~MaxInt<uint32_t>(offset_bits);
+ uint32_t offset = address & MaxInt<uint32_t>(offset_bits);
+ __ LoadLiteral(out, codegen_->DeduplicateDexCacheAddressLiteral(base_address));
+ GenerateGcRootFieldLoad(load, out_loc, out, offset);
+ break;
+ }
+ case HLoadString::LoadKind::kDexCachePcRelative: {
+ Register base_reg = locations->InAt(0).AsRegister<Register>();
+ HArmDexCacheArraysBase* base = load->InputAt(0)->AsArmDexCacheArraysBase();
+ int32_t offset = load->GetDexCacheElementOffset() - base->GetElementOffset();
+ GenerateGcRootFieldLoad(load, out_loc, base_reg, offset);
+ break;
+ }
+ case HLoadString::LoadKind::kDexCacheViaMethod: {
+ Register current_method = locations->InAt(0).AsRegister<Register>();
+
+ // /* GcRoot<mirror::Class> */ out = current_method->declaring_class_
+ GenerateGcRootFieldLoad(
+ load, out_loc, current_method, ArtMethod::DeclaringClassOffset().Int32Value());
+ // /* GcRoot<mirror::String>[] */ out = out->dex_cache_strings_
+ __ LoadFromOffset(kLoadWord, out, out, mirror::Class::DexCacheStringsOffset().Int32Value());
+ // /* GcRoot<mirror::String> */ out = out[string_index]
+ GenerateGcRootFieldLoad(
+ load, out_loc, out, CodeGenerator::GetCacheOffset(load->GetStringIndex()));
+ break;
+ }
+ default:
+ LOG(FATAL) << "Unexpected load kind: " << load->GetLoadKind();
+ UNREACHABLE();
+ }
if (!load->IsInDexCache()) {
SlowPathCode* slow_path = new (GetGraph()->GetArena()) LoadStringSlowPathARM(load);
@@ -6220,6 +6328,8 @@
HInvokeStaticOrDirect::DispatchInfo dispatch_info = desired_dispatch_info;
// We disable pc-relative load when there is an irreducible loop, as the optimization
// is incompatible with it.
+ // TODO: Create as many ArmDexCacheArraysBase instructions as needed for methods
+ // with irreducible loops.
if (GetGraph()->HasIrreducibleLoops() &&
(dispatch_info.method_load_kind ==
HInvokeStaticOrDirect::MethodLoadKind::kDexCachePcRelative)) {
@@ -6399,13 +6509,49 @@
__ blx(LR);
}
+CodeGeneratorARM::PcRelativePatchInfo* CodeGeneratorARM::NewPcRelativeStringPatch(
+ const DexFile& dex_file, uint32_t string_index) {
+ return NewPcRelativePatch(dex_file, string_index, &pc_relative_string_patches_);
+}
+
+CodeGeneratorARM::PcRelativePatchInfo* CodeGeneratorARM::NewPcRelativeDexCacheArrayPatch(
+ const DexFile& dex_file, uint32_t element_offset) {
+ return NewPcRelativePatch(dex_file, element_offset, &pc_relative_dex_cache_patches_);
+}
+
+CodeGeneratorARM::PcRelativePatchInfo* CodeGeneratorARM::NewPcRelativePatch(
+ const DexFile& dex_file, uint32_t offset_or_index, ArenaDeque<PcRelativePatchInfo>* patches) {
+ patches->emplace_back(dex_file, offset_or_index);
+ return &patches->back();
+}
+
+Literal* CodeGeneratorARM::DeduplicateBootImageStringLiteral(const DexFile& dex_file,
+ uint32_t string_index) {
+ return boot_image_string_patches_.GetOrCreate(
+ StringReference(&dex_file, string_index),
+ [this]() { return __ NewLiteral<uint32_t>(/* placeholder */ 0u); });
+}
+
+Literal* CodeGeneratorARM::DeduplicateBootImageAddressLiteral(uint32_t address) {
+ bool needs_patch = GetCompilerOptions().GetIncludePatchInformation();
+ Uint32ToLiteralMap* map = needs_patch ? &boot_image_address_patches_ : &uint32_literals_;
+ return DeduplicateUint32Literal(dchecked_integral_cast<uint32_t>(address), map);
+}
+
+Literal* CodeGeneratorARM::DeduplicateDexCacheAddressLiteral(uint32_t address) {
+ return DeduplicateUint32Literal(address, &uint32_literals_);
+}
+
void CodeGeneratorARM::EmitLinkerPatches(ArenaVector<LinkerPatch>* linker_patches) {
DCHECK(linker_patches->empty());
size_t size =
method_patches_.size() +
call_patches_.size() +
relative_call_patches_.size() +
- /* MOVW+MOVT for each base */ 2u * dex_cache_arrays_base_labels_.size();
+ /* MOVW+MOVT for each base */ 2u * pc_relative_dex_cache_patches_.size() +
+ boot_image_string_patches_.size() +
+ /* MOVW+MOVT for each base */ 2u * pc_relative_string_patches_.size() +
+ boot_image_address_patches_.size();
linker_patches->reserve(size);
for (const auto& entry : method_patches_) {
const MethodReference& target_method = entry.first;
@@ -6431,41 +6577,75 @@
info.target_method.dex_file,
info.target_method.dex_method_index));
}
- for (const auto& pair : dex_cache_arrays_base_labels_) {
- HArmDexCacheArraysBase* base = pair.first;
- const DexCacheArraysBaseLabels* labels = &pair.second;
- const DexFile& dex_file = base->GetDexFile();
- size_t base_element_offset = base->GetElementOffset();
- DCHECK(labels->add_pc_label.IsBound());
- uint32_t add_pc_offset = dchecked_integral_cast<uint32_t>(labels->add_pc_label.Position());
+ for (const PcRelativePatchInfo& info : pc_relative_dex_cache_patches_) {
+ const DexFile& dex_file = info.target_dex_file;
+ size_t base_element_offset = info.offset_or_index;
+ DCHECK(info.add_pc_label.IsBound());
+ uint32_t add_pc_offset = dchecked_integral_cast<uint32_t>(info.add_pc_label.Position());
// Add MOVW patch.
- DCHECK(labels->movw_label.IsBound());
- uint32_t movw_offset = dchecked_integral_cast<uint32_t>(labels->movw_label.Position());
+ DCHECK(info.movw_label.IsBound());
+ uint32_t movw_offset = dchecked_integral_cast<uint32_t>(info.movw_label.Position());
linker_patches->push_back(LinkerPatch::DexCacheArrayPatch(movw_offset,
&dex_file,
add_pc_offset,
base_element_offset));
// Add MOVT patch.
- DCHECK(labels->movt_label.IsBound());
- uint32_t movt_offset = dchecked_integral_cast<uint32_t>(labels->movt_label.Position());
+ DCHECK(info.movt_label.IsBound());
+ uint32_t movt_offset = dchecked_integral_cast<uint32_t>(info.movt_label.Position());
linker_patches->push_back(LinkerPatch::DexCacheArrayPatch(movt_offset,
&dex_file,
add_pc_offset,
base_element_offset));
}
+ for (const auto& entry : boot_image_string_patches_) {
+ const StringReference& target_string = entry.first;
+ Literal* literal = entry.second;
+ DCHECK(literal->GetLabel()->IsBound());
+ uint32_t literal_offset = literal->GetLabel()->Position();
+ linker_patches->push_back(LinkerPatch::StringPatch(literal_offset,
+ target_string.dex_file,
+ target_string.string_index));
+ }
+ for (const PcRelativePatchInfo& info : pc_relative_string_patches_) {
+ const DexFile& dex_file = info.target_dex_file;
+ uint32_t string_index = info.offset_or_index;
+ DCHECK(info.add_pc_label.IsBound());
+ uint32_t add_pc_offset = dchecked_integral_cast<uint32_t>(info.add_pc_label.Position());
+ // Add MOVW patch.
+ DCHECK(info.movw_label.IsBound());
+ uint32_t movw_offset = dchecked_integral_cast<uint32_t>(info.movw_label.Position());
+ linker_patches->push_back(LinkerPatch::RelativeStringPatch(movw_offset,
+ &dex_file,
+ add_pc_offset,
+ string_index));
+ // Add MOVT patch.
+ DCHECK(info.movt_label.IsBound());
+ uint32_t movt_offset = dchecked_integral_cast<uint32_t>(info.movt_label.Position());
+ linker_patches->push_back(LinkerPatch::RelativeStringPatch(movt_offset,
+ &dex_file,
+ add_pc_offset,
+ string_index));
+ }
+ for (const auto& entry : boot_image_address_patches_) {
+ DCHECK(GetCompilerOptions().GetIncludePatchInformation());
+ Literal* literal = entry.second;
+ DCHECK(literal->GetLabel()->IsBound());
+ uint32_t literal_offset = literal->GetLabel()->Position();
+ linker_patches->push_back(LinkerPatch::RecordPosition(literal_offset));
+ }
+}
+
+Literal* CodeGeneratorARM::DeduplicateUint32Literal(uint32_t value, Uint32ToLiteralMap* map) {
+ return map->GetOrCreate(
+ value,
+ [this, value]() { return __ NewLiteral<uint32_t>(value); });
}
Literal* CodeGeneratorARM::DeduplicateMethodLiteral(MethodReference target_method,
MethodToLiteralMap* map) {
- // Look up the literal for target_method.
- auto lb = map->lower_bound(target_method);
- if (lb != map->end() && !map->key_comp()(target_method, lb->first)) {
- return lb->second;
- }
- // We don't have a literal for this method yet, insert a new one.
- Literal* literal = __ NewLiteral<uint32_t>(0u);
- map->PutBefore(lb, target_method, literal);
- return literal;
+ return map->GetOrCreate(
+ target_method,
+ [this]() { return __ NewLiteral<uint32_t>(/* placeholder */ 0u); });
}
Literal* CodeGeneratorARM::DeduplicateMethodAddressLiteral(MethodReference target_method) {
@@ -6600,16 +6780,16 @@
void LocationsBuilderARM::VisitArmDexCacheArraysBase(HArmDexCacheArraysBase* base) {
LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(base);
locations->SetOut(Location::RequiresRegister());
- codegen_->AddDexCacheArraysBase(base);
}
void InstructionCodeGeneratorARM::VisitArmDexCacheArraysBase(HArmDexCacheArraysBase* base) {
Register base_reg = base->GetLocations()->Out().AsRegister<Register>();
- CodeGeneratorARM::DexCacheArraysBaseLabels* labels = codegen_->GetDexCacheArraysBaseLabels(base);
+ CodeGeneratorARM::PcRelativePatchInfo* labels =
+ codegen_->NewPcRelativeDexCacheArrayPatch(base->GetDexFile(), base->GetElementOffset());
__ BindTrackedLabel(&labels->movw_label);
- __ movw(base_reg, 0u);
+ __ movw(base_reg, /* placeholder */ 0u);
__ BindTrackedLabel(&labels->movt_label);
- __ movt(base_reg, 0u);
+ __ movt(base_reg, /* placeholder */ 0u);
__ BindTrackedLabel(&labels->add_pc_label);
__ add(base_reg, base_reg, ShifterOperand(PC));
}
diff --git a/compiler/optimizing/code_generator_arm.h b/compiler/optimizing/code_generator_arm.h
index cc4aa14..8434128 100644
--- a/compiler/optimizing/code_generator_arm.h
+++ b/compiler/optimizing/code_generator_arm.h
@@ -23,6 +23,7 @@
#include "nodes.h"
#include "parallel_move_resolver.h"
#include "utils/arm/assembler_thumb2.h"
+#include "utils/string_reference.h"
namespace art {
namespace arm {
@@ -403,6 +404,11 @@
Label* GetFrameEntryLabel() { return &frame_entry_label_; }
+ // Check if the desired_string_load_kind is supported. If it is, return it,
+ // otherwise return a fall-back kind that should be used instead.
+ HLoadString::LoadKind GetSupportedLoadStringKind(
+ HLoadString::LoadKind desired_string_load_kind) OVERRIDE;
+
// Check if the desired_dispatch_info is supported. If it is, return it,
// otherwise return a fall-back info that should be used instead.
HInvokeStaticOrDirect::DispatchInfo GetSupportedInvokeStaticOrDirectDispatch(
@@ -414,32 +420,34 @@
void MoveFromReturnRegister(Location trg, Primitive::Type type) OVERRIDE;
- void EmitLinkerPatches(ArenaVector<LinkerPatch>* linker_patches) OVERRIDE;
-
- // The PC-relative base address is loaded with three instructions, MOVW+MOVT
+ // The PcRelativePatchInfo is used for PC-relative addressing of dex cache arrays
+ // and boot image strings. The only difference is the interpretation of the offset_or_index.
+ // The PC-relative address is loaded with three instructions, MOVW+MOVT
// to load the offset to base_reg and then ADD base_reg, PC. The offset is
// calculated from the ADD's effective PC, i.e. PC+4 on Thumb2. Though we
// currently emit these 3 instructions together, instruction scheduling could
// split this sequence apart, so we keep separate labels for each of them.
- struct DexCacheArraysBaseLabels {
- DexCacheArraysBaseLabels() = default;
- DexCacheArraysBaseLabels(DexCacheArraysBaseLabels&& other) = default;
+ struct PcRelativePatchInfo {
+ PcRelativePatchInfo(const DexFile& dex_file, uint32_t off_or_idx)
+ : target_dex_file(dex_file), offset_or_index(off_or_idx) { }
+ PcRelativePatchInfo(PcRelativePatchInfo&& other) = default;
+ const DexFile& target_dex_file;
+ // Either the dex cache array element offset or the string index.
+ uint32_t offset_or_index;
Label movw_label;
Label movt_label;
Label add_pc_label;
};
- void AddDexCacheArraysBase(HArmDexCacheArraysBase* base) {
- DexCacheArraysBaseLabels labels;
- dex_cache_arrays_base_labels_.Put(base, std::move(labels));
- }
+ PcRelativePatchInfo* NewPcRelativeStringPatch(const DexFile& dex_file, uint32_t string_index);
+ PcRelativePatchInfo* NewPcRelativeDexCacheArrayPatch(const DexFile& dex_file,
+ uint32_t element_offset);
+ Literal* DeduplicateBootImageStringLiteral(const DexFile& dex_file, uint32_t string_index);
+ Literal* DeduplicateBootImageAddressLiteral(uint32_t address);
+ Literal* DeduplicateDexCacheAddressLiteral(uint32_t address);
- DexCacheArraysBaseLabels* GetDexCacheArraysBaseLabels(HArmDexCacheArraysBase* base) {
- auto it = dex_cache_arrays_base_labels_.find(base);
- DCHECK(it != dex_cache_arrays_base_labels_.end());
- return &it->second;
- }
+ void EmitLinkerPatches(ArenaVector<LinkerPatch>* linker_patches) OVERRIDE;
// Fast path implementation of ReadBarrier::Barrier for a heap
// reference field load when Baker's read barriers are used.
@@ -525,14 +533,19 @@
Register GetInvokeStaticOrDirectExtraParameter(HInvokeStaticOrDirect* invoke, Register temp);
+ using Uint32ToLiteralMap = ArenaSafeMap<uint32_t, Literal*>;
using MethodToLiteralMap = ArenaSafeMap<MethodReference, Literal*, MethodReferenceComparator>;
- using DexCacheArraysBaseToLabelsMap = ArenaSafeMap<HArmDexCacheArraysBase*,
- DexCacheArraysBaseLabels,
- std::less<HArmDexCacheArraysBase*>>;
+ using BootStringToLiteralMap = ArenaSafeMap<StringReference,
+ Literal*,
+ StringReferenceValueComparator>;
+ Literal* DeduplicateUint32Literal(uint32_t value, Uint32ToLiteralMap* map);
Literal* DeduplicateMethodLiteral(MethodReference target_method, MethodToLiteralMap* map);
Literal* DeduplicateMethodAddressLiteral(MethodReference target_method);
Literal* DeduplicateMethodCodeLiteral(MethodReference target_method);
+ PcRelativePatchInfo* NewPcRelativePatch(const DexFile& dex_file,
+ uint32_t offset_or_index,
+ ArenaDeque<PcRelativePatchInfo>* patches);
// Labels for each block that will be compiled.
Label* block_labels_; // Indexed by block id.
@@ -543,14 +556,22 @@
Thumb2Assembler assembler_;
const ArmInstructionSetFeatures& isa_features_;
+ // Deduplication map for 32-bit literals, used for non-patchable boot image addresses.
+ Uint32ToLiteralMap uint32_literals_;
// Method patch info, map MethodReference to a literal for method address and method code.
MethodToLiteralMap method_patches_;
MethodToLiteralMap call_patches_;
// Relative call patch info.
// Using ArenaDeque<> which retains element addresses on push/emplace_back().
ArenaDeque<MethodPatchInfo<Label>> relative_call_patches_;
-
- DexCacheArraysBaseToLabelsMap dex_cache_arrays_base_labels_;
+ // PC-relative patch info for each HArmDexCacheArraysBase.
+ ArenaDeque<PcRelativePatchInfo> pc_relative_dex_cache_patches_;
+ // Deduplication map for boot string literals for kBootImageLinkTimeAddress.
+ BootStringToLiteralMap boot_image_string_patches_;
+ // PC-relative String patch info.
+ ArenaDeque<PcRelativePatchInfo> pc_relative_string_patches_;
+ // Deduplication map for patchable boot image addresses.
+ Uint32ToLiteralMap boot_image_address_patches_;
DISALLOW_COPY_AND_ASSIGN(CodeGeneratorARM);
};
diff --git a/compiler/optimizing/code_generator_arm64.cc b/compiler/optimizing/code_generator_arm64.cc
index 1f577b3..491014d 100644
--- a/compiler/optimizing/code_generator_arm64.cc
+++ b/compiler/optimizing/code_generator_arm64.cc
@@ -905,6 +905,8 @@
instruction_visitor_(graph, this),
move_resolver_(graph->GetArena(), this),
isa_features_(isa_features),
+ uint32_literals_(std::less<uint32_t>(),
+ graph->GetArena()->Adapter(kArenaAllocCodeGenerator)),
uint64_literals_(std::less<uint64_t>(),
graph->GetArena()->Adapter(kArenaAllocCodeGenerator)),
method_patches_(MethodReferenceComparator(),
@@ -912,7 +914,12 @@
call_patches_(MethodReferenceComparator(),
graph->GetArena()->Adapter(kArenaAllocCodeGenerator)),
relative_call_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)),
- pc_relative_dex_cache_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)) {
+ pc_relative_dex_cache_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)),
+ boot_image_string_patches_(StringReferenceValueComparator(),
+ graph->GetArena()->Adapter(kArenaAllocCodeGenerator)),
+ pc_relative_string_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)),
+ boot_image_address_patches_(std::less<uint32_t>(),
+ graph->GetArena()->Adapter(kArenaAllocCodeGenerator)) {
// Save the link register (containing the return address) to mimic Quick.
AddAllocatedRegister(LocationFrom(lr));
}
@@ -3662,23 +3669,21 @@
break;
case HInvokeStaticOrDirect::MethodLoadKind::kDexCachePcRelative: {
// Add ADRP with its PC-relative DexCache access patch.
- pc_relative_dex_cache_patches_.emplace_back(*invoke->GetTargetMethod().dex_file,
- invoke->GetDexCacheArrayOffset());
- vixl::Label* pc_insn_label = &pc_relative_dex_cache_patches_.back().label;
+ const DexFile& dex_file = *invoke->GetTargetMethod().dex_file;
+ uint32_t element_offset = invoke->GetDexCacheArrayOffset();
+ vixl::Label* adrp_label = NewPcRelativeDexCacheArrayPatch(dex_file, element_offset);
{
vixl::SingleEmissionCheckScope guard(GetVIXLAssembler());
- __ Bind(pc_insn_label);
- __ adrp(XRegisterFrom(temp), 0);
+ __ Bind(adrp_label);
+ __ adrp(XRegisterFrom(temp), /* offset placeholder */ 0);
}
- pc_relative_dex_cache_patches_.back().pc_insn_label = pc_insn_label;
// Add LDR with its PC-relative DexCache access patch.
- pc_relative_dex_cache_patches_.emplace_back(*invoke->GetTargetMethod().dex_file,
- invoke->GetDexCacheArrayOffset());
+ vixl::Label* ldr_label =
+ NewPcRelativeDexCacheArrayPatch(dex_file, element_offset, adrp_label);
{
vixl::SingleEmissionCheckScope guard(GetVIXLAssembler());
- __ Bind(&pc_relative_dex_cache_patches_.back().label);
- __ ldr(XRegisterFrom(temp), MemOperand(XRegisterFrom(temp), 0));
- pc_relative_dex_cache_patches_.back().pc_insn_label = pc_insn_label;
+ __ Bind(ldr_label);
+ __ ldr(XRegisterFrom(temp), MemOperand(XRegisterFrom(temp), /* offset placeholder */ 0));
}
break;
}
@@ -3772,13 +3777,58 @@
__ Blr(lr);
}
+vixl::Label* CodeGeneratorARM64::NewPcRelativeStringPatch(const DexFile& dex_file,
+ uint32_t string_index,
+ vixl::Label* adrp_label) {
+ return NewPcRelativePatch(dex_file, string_index, adrp_label, &pc_relative_string_patches_);
+}
+
+vixl::Label* CodeGeneratorARM64::NewPcRelativeDexCacheArrayPatch(const DexFile& dex_file,
+ uint32_t element_offset,
+ vixl::Label* adrp_label) {
+ return NewPcRelativePatch(dex_file, element_offset, adrp_label, &pc_relative_dex_cache_patches_);
+}
+
+vixl::Label* CodeGeneratorARM64::NewPcRelativePatch(const DexFile& dex_file,
+ uint32_t offset_or_index,
+ vixl::Label* adrp_label,
+ ArenaDeque<PcRelativePatchInfo>* patches) {
+ // Add a patch entry and return the label.
+ patches->emplace_back(dex_file, offset_or_index);
+ PcRelativePatchInfo* info = &patches->back();
+ vixl::Label* label = &info->label;
+ // If adrp_label is null, this is the ADRP patch and needs to point to its own label.
+ info->pc_insn_label = (adrp_label != nullptr) ? adrp_label : label;
+ return label;
+}
+
+vixl::Literal<uint32_t>* CodeGeneratorARM64::DeduplicateBootImageStringLiteral(
+ const DexFile& dex_file, uint32_t string_index) {
+ return boot_image_string_patches_.GetOrCreate(
+ StringReference(&dex_file, string_index),
+ [this]() { return __ CreateLiteralDestroyedWithPool<uint32_t>(/* placeholder */ 0u); });
+}
+
+vixl::Literal<uint32_t>* CodeGeneratorARM64::DeduplicateBootImageAddressLiteral(uint64_t address) {
+ bool needs_patch = GetCompilerOptions().GetIncludePatchInformation();
+ Uint32ToLiteralMap* map = needs_patch ? &boot_image_address_patches_ : &uint32_literals_;
+ return DeduplicateUint32Literal(dchecked_integral_cast<uint32_t>(address), map);
+}
+
+vixl::Literal<uint64_t>* CodeGeneratorARM64::DeduplicateDexCacheAddressLiteral(uint64_t address) {
+ return DeduplicateUint64Literal(address);
+}
+
void CodeGeneratorARM64::EmitLinkerPatches(ArenaVector<LinkerPatch>* linker_patches) {
DCHECK(linker_patches->empty());
size_t size =
method_patches_.size() +
call_patches_.size() +
relative_call_patches_.size() +
- pc_relative_dex_cache_patches_.size();
+ pc_relative_dex_cache_patches_.size() +
+ boot_image_string_patches_.size() +
+ pc_relative_string_patches_.size() +
+ boot_image_address_patches_.size();
linker_patches->reserve(size);
for (const auto& entry : method_patches_) {
const MethodReference& target_method = entry.first;
@@ -3799,38 +3849,51 @@
info.target_method.dex_file,
info.target_method.dex_method_index));
}
- for (const PcRelativeDexCacheAccessInfo& info : pc_relative_dex_cache_patches_) {
+ for (const PcRelativePatchInfo& info : pc_relative_dex_cache_patches_) {
linker_patches->push_back(LinkerPatch::DexCacheArrayPatch(info.label.location(),
&info.target_dex_file,
info.pc_insn_label->location(),
- info.element_offset));
+ info.offset_or_index));
+ }
+ for (const auto& entry : boot_image_string_patches_) {
+ const StringReference& target_string = entry.first;
+ vixl::Literal<uint32_t>* literal = entry.second;
+ linker_patches->push_back(LinkerPatch::StringPatch(literal->offset(),
+ target_string.dex_file,
+ target_string.string_index));
+ }
+ for (const PcRelativePatchInfo& info : pc_relative_string_patches_) {
+ linker_patches->push_back(LinkerPatch::RelativeStringPatch(info.label.location(),
+ &info.target_dex_file,
+ info.pc_insn_label->location(),
+ info.offset_or_index));
+ }
+ for (const auto& entry : boot_image_address_patches_) {
+ DCHECK(GetCompilerOptions().GetIncludePatchInformation());
+ vixl::Literal<uint32_t>* literal = entry.second;
+ linker_patches->push_back(LinkerPatch::RecordPosition(literal->offset()));
}
}
+vixl::Literal<uint32_t>* CodeGeneratorARM64::DeduplicateUint32Literal(uint32_t value,
+ Uint32ToLiteralMap* map) {
+ return map->GetOrCreate(
+ value,
+ [this, value]() { return __ CreateLiteralDestroyedWithPool<uint32_t>(value); });
+}
+
vixl::Literal<uint64_t>* CodeGeneratorARM64::DeduplicateUint64Literal(uint64_t value) {
- // Look up the literal for value.
- auto lb = uint64_literals_.lower_bound(value);
- if (lb != uint64_literals_.end() && !uint64_literals_.key_comp()(value, lb->first)) {
- return lb->second;
- }
- // We don't have a literal for this value, insert a new one.
- vixl::Literal<uint64_t>* literal = __ CreateLiteralDestroyedWithPool<uint64_t>(value);
- uint64_literals_.PutBefore(lb, value, literal);
- return literal;
+ return uint64_literals_.GetOrCreate(
+ value,
+ [this, value]() { return __ CreateLiteralDestroyedWithPool<uint64_t>(value); });
}
vixl::Literal<uint64_t>* CodeGeneratorARM64::DeduplicateMethodLiteral(
MethodReference target_method,
MethodToLiteralMap* map) {
- // Look up the literal for target_method.
- auto lb = map->lower_bound(target_method);
- if (lb != map->end() && !map->key_comp()(target_method, lb->first)) {
- return lb->second;
- }
- // We don't have a literal for this method yet, insert a new one.
- vixl::Literal<uint64_t>* literal = __ CreateLiteralDestroyedWithPool<uint64_t>(0u);
- map->PutBefore(lb, target_method, literal);
- return literal;
+ return map->GetOrCreate(
+ target_method,
+ [this]() { return __ CreateLiteralDestroyedWithPool<uint64_t>(/* placeholder */ 0u); });
}
vixl::Literal<uint64_t>* CodeGeneratorARM64::DeduplicateMethodAddressLiteral(
@@ -3955,28 +4018,135 @@
// Nothing to do, this is driven by the code generator.
}
+HLoadString::LoadKind CodeGeneratorARM64::GetSupportedLoadStringKind(
+ HLoadString::LoadKind desired_string_load_kind) {
+ if (kEmitCompilerReadBarrier) {
+ switch (desired_string_load_kind) {
+ case HLoadString::LoadKind::kBootImageLinkTimeAddress:
+ case HLoadString::LoadKind::kBootImageLinkTimePcRelative:
+ case HLoadString::LoadKind::kBootImageAddress:
+ // TODO: Implement for read barrier.
+ return HLoadString::LoadKind::kDexCacheViaMethod;
+ default:
+ break;
+ }
+ }
+ switch (desired_string_load_kind) {
+ case HLoadString::LoadKind::kBootImageLinkTimeAddress:
+ DCHECK(!GetCompilerOptions().GetCompilePic());
+ break;
+ case HLoadString::LoadKind::kBootImageLinkTimePcRelative:
+ DCHECK(GetCompilerOptions().GetCompilePic());
+ break;
+ case HLoadString::LoadKind::kBootImageAddress:
+ break;
+ case HLoadString::LoadKind::kDexCacheAddress:
+ DCHECK(Runtime::Current()->UseJit());
+ break;
+ case HLoadString::LoadKind::kDexCachePcRelative:
+ DCHECK(!Runtime::Current()->UseJit());
+ break;
+ case HLoadString::LoadKind::kDexCacheViaMethod:
+ break;
+ }
+ return desired_string_load_kind;
+}
+
void LocationsBuilderARM64::VisitLoadString(HLoadString* load) {
- LocationSummary::CallKind call_kind = (!load->IsInDexCache() || kEmitCompilerReadBarrier)
+ LocationSummary::CallKind call_kind = (load->NeedsEnvironment() || kEmitCompilerReadBarrier)
? LocationSummary::kCallOnSlowPath
: LocationSummary::kNoCall;
LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(load, call_kind);
- locations->SetInAt(0, Location::RequiresRegister());
+ if (load->GetLoadKind() == HLoadString::LoadKind::kDexCacheViaMethod) {
+ locations->SetInAt(0, Location::RequiresRegister());
+ }
locations->SetOut(Location::RequiresRegister());
}
void InstructionCodeGeneratorARM64::VisitLoadString(HLoadString* load) {
Location out_loc = load->GetLocations()->Out();
Register out = OutputRegister(load);
- Register current_method = InputRegisterAt(load, 0);
- // /* GcRoot<mirror::Class> */ out = current_method->declaring_class_
- GenerateGcRootFieldLoad(
- load, out_loc, current_method, ArtMethod::DeclaringClassOffset().Int32Value());
- // /* GcRoot<mirror::String>[] */ out = out->dex_cache_strings_
- __ Ldr(out.X(), HeapOperand(out, mirror::Class::DexCacheStringsOffset().Uint32Value()));
- // /* GcRoot<mirror::String> */ out = out[string_index]
- GenerateGcRootFieldLoad(
- load, out_loc, out.X(), CodeGenerator::GetCacheOffset(load->GetStringIndex()));
+ switch (load->GetLoadKind()) {
+ case HLoadString::LoadKind::kBootImageLinkTimeAddress:
+ DCHECK(!kEmitCompilerReadBarrier);
+ __ Ldr(out, codegen_->DeduplicateBootImageStringLiteral(load->GetDexFile(),
+ load->GetStringIndex()));
+ return; // No dex cache slow path.
+ case HLoadString::LoadKind::kBootImageLinkTimePcRelative: {
+ DCHECK(!kEmitCompilerReadBarrier);
+ // Add ADRP with its PC-relative String patch.
+ const DexFile& dex_file = load->GetDexFile();
+ uint32_t string_index = load->GetStringIndex();
+ vixl::Label* adrp_label = codegen_->NewPcRelativeStringPatch(dex_file, string_index);
+ {
+ vixl::SingleEmissionCheckScope guard(GetVIXLAssembler());
+ __ Bind(adrp_label);
+ __ adrp(out.X(), /* offset placeholder */ 0);
+ }
+ // Add ADD with its PC-relative String patch.
+ vixl::Label* add_label =
+ codegen_->NewPcRelativeStringPatch(dex_file, string_index, adrp_label);
+ {
+ vixl::SingleEmissionCheckScope guard(GetVIXLAssembler());
+ __ Bind(add_label);
+ __ add(out.X(), out.X(), Operand(/* offset placeholder */ 0));
+ }
+ return; // No dex cache slow path.
+ }
+ case HLoadString::LoadKind::kBootImageAddress: {
+ DCHECK(!kEmitCompilerReadBarrier);
+ DCHECK(load->GetAddress() != 0u && IsUint<32>(load->GetAddress()));
+ __ Ldr(out.W(), codegen_->DeduplicateBootImageAddressLiteral(load->GetAddress()));
+ return; // No dex cache slow path.
+ }
+ case HLoadString::LoadKind::kDexCacheAddress: {
+ DCHECK_NE(load->GetAddress(), 0u);
+ // LDR immediate has a 12-bit offset multiplied by the size and for 32-bit loads
+ // that gives a 16KiB range. To try and reduce the number of literals if we load
+ // multiple strings, simply split the dex cache address to a 16KiB aligned base
+ // loaded from a literal and the remaining offset embedded in the load.
+ static_assert(sizeof(GcRoot<mirror::String>) == 4u, "Expected GC root to be 4 bytes.");
+ DCHECK_ALIGNED(load->GetAddress(), 4u);
+ constexpr size_t offset_bits = /* encoded bits */ 12 + /* scale */ 2;
+ uint64_t base_address = load->GetAddress() & ~MaxInt<uint64_t>(offset_bits);
+ uint32_t offset = load->GetAddress() & MaxInt<uint64_t>(offset_bits);
+ __ Ldr(out.X(), codegen_->DeduplicateDexCacheAddressLiteral(base_address));
+ GenerateGcRootFieldLoad(load, out_loc, out.X(), offset);
+ break;
+ }
+ case HLoadString::LoadKind::kDexCachePcRelative: {
+ // Add ADRP with its PC-relative DexCache access patch.
+ const DexFile& dex_file = load->GetDexFile();
+ uint32_t element_offset = load->GetDexCacheElementOffset();
+ vixl::Label* adrp_label = codegen_->NewPcRelativeDexCacheArrayPatch(dex_file, element_offset);
+ {
+ vixl::SingleEmissionCheckScope guard(GetVIXLAssembler());
+ __ Bind(adrp_label);
+ __ adrp(out.X(), /* offset placeholder */ 0);
+ }
+ // Add LDR with its PC-relative DexCache access patch.
+ vixl::Label* ldr_label =
+ codegen_->NewPcRelativeDexCacheArrayPatch(dex_file, element_offset, adrp_label);
+ GenerateGcRootFieldLoad(load, out_loc, out.X(), /* offset placeholder */ 0, ldr_label);
+ break;
+ }
+ case HLoadString::LoadKind::kDexCacheViaMethod: {
+ Register current_method = InputRegisterAt(load, 0);
+ // /* GcRoot<mirror::Class> */ out = current_method->declaring_class_
+ GenerateGcRootFieldLoad(
+ load, out_loc, current_method, ArtMethod::DeclaringClassOffset().Int32Value());
+ // /* GcRoot<mirror::String>[] */ out = out->dex_cache_strings_
+ __ Ldr(out.X(), HeapOperand(out, mirror::Class::DexCacheStringsOffset().Uint32Value()));
+ // /* GcRoot<mirror::String> */ out = out[string_index]
+ GenerateGcRootFieldLoad(
+ load, out_loc, out.X(), CodeGenerator::GetCacheOffset(load->GetStringIndex()));
+ break;
+ }
+ default:
+ LOG(FATAL) << "Unexpected load kind: " << load->GetLoadKind();
+ UNREACHABLE();
+ }
if (!load->IsInDexCache()) {
SlowPathCodeARM64* slow_path = new (GetGraph()->GetArena()) LoadStringSlowPathARM64(load);
@@ -4791,7 +4961,8 @@
void InstructionCodeGeneratorARM64::GenerateGcRootFieldLoad(HInstruction* instruction,
Location root,
vixl::Register obj,
- uint32_t offset) {
+ uint32_t offset,
+ vixl::Label* fixup_label) {
Register root_reg = RegisterFrom(root, Primitive::kPrimNot);
if (kEmitCompilerReadBarrier) {
if (kUseBakerReadBarrier) {
@@ -4804,7 +4975,13 @@
// }
// /* GcRoot<mirror::Object> */ root = *(obj + offset)
- __ Ldr(root_reg, MemOperand(obj, offset));
+ if (fixup_label == nullptr) {
+ __ Ldr(root_reg, MemOperand(obj, offset));
+ } else {
+ vixl::SingleEmissionCheckScope guard(GetVIXLAssembler());
+ __ Bind(fixup_label);
+ __ ldr(root_reg, MemOperand(obj, offset));
+ }
static_assert(
sizeof(mirror::CompressedReference<mirror::Object>) == sizeof(GcRoot<mirror::Object>),
"art::mirror::CompressedReference<mirror::Object> and art::GcRoot<mirror::Object> "
@@ -4829,14 +5006,26 @@
// GC root loaded through a slow path for read barriers other
// than Baker's.
// /* GcRoot<mirror::Object>* */ root = obj + offset
- __ Add(root_reg.X(), obj.X(), offset);
+ if (fixup_label == nullptr) {
+ __ Add(root_reg.X(), obj.X(), offset);
+ } else {
+ vixl::SingleEmissionCheckScope guard(GetVIXLAssembler());
+ __ Bind(fixup_label);
+ __ add(root_reg.X(), obj.X(), offset);
+ }
// /* mirror::Object* */ root = root->Read()
codegen_->GenerateReadBarrierForRootSlow(instruction, root, root);
}
} else {
// Plain GC root load with no read barrier.
// /* GcRoot<mirror::Object> */ root = *(obj + offset)
- __ Ldr(root_reg, MemOperand(obj, offset));
+ if (fixup_label == nullptr) {
+ __ Ldr(root_reg, MemOperand(obj, offset));
+ } else {
+ vixl::SingleEmissionCheckScope guard(GetVIXLAssembler());
+ __ Bind(fixup_label);
+ __ ldr(root_reg, MemOperand(obj, offset));
+ }
// Note that GC roots are not affected by heap poisoning, thus we
// do not have to unpoison `root_reg` here.
}
diff --git a/compiler/optimizing/code_generator_arm64.h b/compiler/optimizing/code_generator_arm64.h
index cf9dc1b..8ec7531 100644
--- a/compiler/optimizing/code_generator_arm64.h
+++ b/compiler/optimizing/code_generator_arm64.h
@@ -17,6 +17,7 @@
#ifndef ART_COMPILER_OPTIMIZING_CODE_GENERATOR_ARM64_H_
#define ART_COMPILER_OPTIMIZING_CODE_GENERATOR_ARM64_H_
+#include "arch/arm64/quick_method_frame_info_arm64.h"
#include "code_generator.h"
#include "common_arm64.h"
#include "dex/compiler_enums.h"
@@ -24,9 +25,9 @@
#include "nodes.h"
#include "parallel_move_resolver.h"
#include "utils/arm64/assembler_arm64.h"
+#include "utils/string_reference.h"
#include "vixl/a64/disasm-a64.h"
#include "vixl/a64/macro-assembler-a64.h"
-#include "arch/arm64/quick_method_frame_info_arm64.h"
namespace art {
namespace arm64 {
@@ -255,7 +256,8 @@
void GenerateGcRootFieldLoad(HInstruction* instruction,
Location root,
vixl::Register obj,
- uint32_t offset);
+ uint32_t offset,
+ vixl::Label* fixup_label = nullptr);
// Generate a floating-point comparison.
void GenerateFcmp(HInstruction* instruction);
@@ -453,6 +455,11 @@
return false;
}
+ // Check if the desired_string_load_kind is supported. If it is, return it,
+ // otherwise return a fall-back kind that should be used instead.
+ HLoadString::LoadKind GetSupportedLoadStringKind(
+ HLoadString::LoadKind desired_string_load_kind) OVERRIDE;
+
// Check if the desired_dispatch_info is supported. If it is, return it,
// otherwise return a fall-back info that should be used instead.
HInvokeStaticOrDirect::DispatchInfo GetSupportedInvokeStaticOrDirectDispatch(
@@ -467,6 +474,27 @@
UNIMPLEMENTED(FATAL);
}
+ // Add a new PC-relative string patch for an instruction and return the label
+ // to be bound before the instruction. The instruction will be either the
+ // ADRP (pass `adrp_label = null`) or the ADD (pass `adrp_label` pointing
+ // to the associated ADRP patch label).
+ vixl::Label* NewPcRelativeStringPatch(const DexFile& dex_file,
+ uint32_t string_index,
+ vixl::Label* adrp_label = nullptr);
+
+ // Add a new PC-relative dex cache array patch for an instruction and return
+ // the label to be bound before the instruction. The instruction will be
+ // either the ADRP (pass `adrp_label = null`) or the LDR (pass `adrp_label`
+ // pointing to the associated ADRP patch label).
+ vixl::Label* NewPcRelativeDexCacheArrayPatch(const DexFile& dex_file,
+ uint32_t element_offset,
+ vixl::Label* adrp_label = nullptr);
+
+ vixl::Literal<uint32_t>* DeduplicateBootImageStringLiteral(const DexFile& dex_file,
+ uint32_t string_index);
+ vixl::Literal<uint32_t>* DeduplicateBootImageAddressLiteral(uint64_t address);
+ vixl::Literal<uint64_t>* DeduplicateDexCacheAddressLiteral(uint64_t address);
+
void EmitLinkerPatches(ArenaVector<LinkerPatch>* linker_patches) OVERRIDE;
// Fast path implementation of ReadBarrier::Barrier for a heap
@@ -554,26 +582,39 @@
bool use_load_acquire);
using Uint64ToLiteralMap = ArenaSafeMap<uint64_t, vixl::Literal<uint64_t>*>;
+ using Uint32ToLiteralMap = ArenaSafeMap<uint32_t, vixl::Literal<uint32_t>*>;
using MethodToLiteralMap = ArenaSafeMap<MethodReference,
vixl::Literal<uint64_t>*,
MethodReferenceComparator>;
+ using BootStringToLiteralMap = ArenaSafeMap<StringReference,
+ vixl::Literal<uint32_t>*,
+ StringReferenceValueComparator>;
+ vixl::Literal<uint32_t>* DeduplicateUint32Literal(uint32_t value, Uint32ToLiteralMap* map);
vixl::Literal<uint64_t>* DeduplicateUint64Literal(uint64_t value);
vixl::Literal<uint64_t>* DeduplicateMethodLiteral(MethodReference target_method,
MethodToLiteralMap* map);
vixl::Literal<uint64_t>* DeduplicateMethodAddressLiteral(MethodReference target_method);
vixl::Literal<uint64_t>* DeduplicateMethodCodeLiteral(MethodReference target_method);
- struct PcRelativeDexCacheAccessInfo {
- PcRelativeDexCacheAccessInfo(const DexFile& dex_file, uint32_t element_off)
- : target_dex_file(dex_file), element_offset(element_off), label(), pc_insn_label() { }
+ // The PcRelativePatchInfo is used for PC-relative addressing of dex cache arrays
+ // and boot image strings. The only difference is the interpretation of the offset_or_index.
+ struct PcRelativePatchInfo {
+ PcRelativePatchInfo(const DexFile& dex_file, uint32_t off_or_idx)
+ : target_dex_file(dex_file), offset_or_index(off_or_idx), label(), pc_insn_label() { }
const DexFile& target_dex_file;
- uint32_t element_offset;
+ // Either the dex cache array element offset or the string index.
+ uint32_t offset_or_index;
vixl::Label label;
vixl::Label* pc_insn_label;
};
+ vixl::Label* NewPcRelativePatch(const DexFile& dex_file,
+ uint32_t offset_or_index,
+ vixl::Label* adrp_label,
+ ArenaDeque<PcRelativePatchInfo>* patches);
+
void EmitJumpTables();
// Labels for each block that will be compiled.
@@ -587,7 +628,10 @@
Arm64Assembler assembler_;
const Arm64InstructionSetFeatures& isa_features_;
- // Deduplication map for 64-bit literals, used for non-patchable method address and method code.
+ // Deduplication map for 32-bit literals, used for non-patchable boot image addresses.
+ Uint32ToLiteralMap uint32_literals_;
+ // Deduplication map for 64-bit literals, used for non-patchable method address, method code
+ // or string dex cache address.
Uint64ToLiteralMap uint64_literals_;
// Method patch info, map MethodReference to a literal for method address and method code.
MethodToLiteralMap method_patches_;
@@ -596,7 +640,13 @@
// Using ArenaDeque<> which retains element addresses on push/emplace_back().
ArenaDeque<MethodPatchInfo<vixl::Label>> relative_call_patches_;
// PC-relative DexCache access info.
- ArenaDeque<PcRelativeDexCacheAccessInfo> pc_relative_dex_cache_patches_;
+ ArenaDeque<PcRelativePatchInfo> pc_relative_dex_cache_patches_;
+ // Deduplication map for boot string literals for kBootImageLinkTimeAddress.
+ BootStringToLiteralMap boot_image_string_patches_;
+ // PC-relative String patch info.
+ ArenaDeque<PcRelativePatchInfo> pc_relative_string_patches_;
+ // Deduplication map for patchable boot image addresses.
+ Uint32ToLiteralMap boot_image_address_patches_;
DISALLOW_COPY_AND_ASSIGN(CodeGeneratorARM64);
};
diff --git a/compiler/optimizing/code_generator_mips.cc b/compiler/optimizing/code_generator_mips.cc
index a29d839..8b19f84 100644
--- a/compiler/optimizing/code_generator_mips.cc
+++ b/compiler/optimizing/code_generator_mips.cc
@@ -3816,6 +3816,12 @@
return false;
}
+HLoadString::LoadKind CodeGeneratorMIPS::GetSupportedLoadStringKind(
+ HLoadString::LoadKind desired_string_load_kind ATTRIBUTE_UNUSED) {
+ // TODO: Implement other kinds.
+ return HLoadString::LoadKind::kDexCacheViaMethod;
+}
+
HInvokeStaticOrDirect::DispatchInfo CodeGeneratorMIPS::GetSupportedInvokeStaticOrDirectDispatch(
const HInvokeStaticOrDirect::DispatchInfo& desired_dispatch_info,
MethodReference target_method ATTRIBUTE_UNUSED) {
@@ -4066,9 +4072,9 @@
}
void LocationsBuilderMIPS::VisitLoadString(HLoadString* load) {
- LocationSummary::CallKind call_kind = load->IsInDexCache()
- ? LocationSummary::kNoCall
- : LocationSummary::kCallOnSlowPath;
+ LocationSummary::CallKind call_kind = load->NeedsEnvironment()
+ ? LocationSummary::kCallOnSlowPath
+ : LocationSummary::kNoCall;
LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(load, call_kind);
locations->SetInAt(0, Location::RequiresRegister());
locations->SetOut(Location::RequiresRegister());
diff --git a/compiler/optimizing/code_generator_mips.h b/compiler/optimizing/code_generator_mips.h
index b720573..afe7917 100644
--- a/compiler/optimizing/code_generator_mips.h
+++ b/compiler/optimizing/code_generator_mips.h
@@ -345,6 +345,11 @@
return type == Primitive::kPrimLong;
}
+ // Check if the desired_string_load_kind is supported. If it is, return it,
+ // otherwise return a fall-back kind that should be used instead.
+ HLoadString::LoadKind GetSupportedLoadStringKind(
+ HLoadString::LoadKind desired_string_load_kind) OVERRIDE;
+
// Check if the desired_dispatch_info is supported. If it is, return it,
// otherwise return a fall-back info that should be used instead.
HInvokeStaticOrDirect::DispatchInfo GetSupportedInvokeStaticOrDirectDispatch(
diff --git a/compiler/optimizing/code_generator_mips64.cc b/compiler/optimizing/code_generator_mips64.cc
index 72ef499..2f9eca6 100644
--- a/compiler/optimizing/code_generator_mips64.cc
+++ b/compiler/optimizing/code_generator_mips64.cc
@@ -3030,6 +3030,12 @@
return false;
}
+HLoadString::LoadKind CodeGeneratorMIPS64::GetSupportedLoadStringKind(
+ HLoadString::LoadKind desired_string_load_kind ATTRIBUTE_UNUSED) {
+ // TODO: Implement other kinds.
+ return HLoadString::LoadKind::kDexCacheViaMethod;
+}
+
HInvokeStaticOrDirect::DispatchInfo CodeGeneratorMIPS64::GetSupportedInvokeStaticOrDirectDispatch(
const HInvokeStaticOrDirect::DispatchInfo& desired_dispatch_info,
MethodReference target_method ATTRIBUTE_UNUSED) {
@@ -3284,9 +3290,9 @@
}
void LocationsBuilderMIPS64::VisitLoadString(HLoadString* load) {
- LocationSummary::CallKind call_kind = load->IsInDexCache()
- ? LocationSummary::kNoCall
- : LocationSummary::kCallOnSlowPath;
+ LocationSummary::CallKind call_kind = load->NeedsEnvironment()
+ ? LocationSummary::kCallOnSlowPath
+ : LocationSummary::kNoCall;
LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(load, call_kind);
locations->SetInAt(0, Location::RequiresRegister());
locations->SetOut(Location::RequiresRegister());
diff --git a/compiler/optimizing/code_generator_mips64.h b/compiler/optimizing/code_generator_mips64.h
index 9464a14..94767cb 100644
--- a/compiler/optimizing/code_generator_mips64.h
+++ b/compiler/optimizing/code_generator_mips64.h
@@ -337,6 +337,11 @@
bool NeedsTwoRegisters(Primitive::Type type ATTRIBUTE_UNUSED) const { return false; }
+ // Check if the desired_string_load_kind is supported. If it is, return it,
+ // otherwise return a fall-back kind that should be used instead.
+ HLoadString::LoadKind GetSupportedLoadStringKind(
+ HLoadString::LoadKind desired_string_load_kind) OVERRIDE;
+
// Check if the desired_dispatch_info is supported. If it is, return it,
// otherwise return a fall-back info that should be used instead.
HInvokeStaticOrDirect::DispatchInfo GetSupportedInvokeStaticOrDirectDispatch(
diff --git a/compiler/optimizing/code_generator_x86.cc b/compiler/optimizing/code_generator_x86.cc
index 394f4ee..715b5be 100644
--- a/compiler/optimizing/code_generator_x86.cc
+++ b/compiler/optimizing/code_generator_x86.cc
@@ -799,6 +799,8 @@
method_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)),
relative_call_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)),
pc_relative_dex_cache_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)),
+ simple_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)),
+ string_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)),
fixups_to_jump_tables_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)) {
// Use a fake return address register to mimic Quick.
AddAllocatedRegister(Location::RegisterLocation(kFakeReturnRegister));
@@ -4340,6 +4342,8 @@
// We disable pc-relative load when there is an irreducible loop, as the optimization
// is incompatible with it.
+ // TODO: Create as many X86ComputeBaseMethodAddress instructions
+ // as needed for methods with irreducible loops.
if (GetGraph()->HasIrreducibleLoops() &&
(dispatch_info.method_load_kind ==
HInvokeStaticOrDirect::MethodLoadKind::kDexCachePcRelative)) {
@@ -4401,18 +4405,17 @@
__ movl(temp.AsRegister<Register>(), Immediate(invoke->GetMethodAddress()));
break;
case HInvokeStaticOrDirect::MethodLoadKind::kDirectAddressWithFixup:
- __ movl(temp.AsRegister<Register>(), Immediate(0)); // Placeholder.
+ __ movl(temp.AsRegister<Register>(), Immediate(/* placeholder */ 0));
method_patches_.emplace_back(invoke->GetTargetMethod());
__ Bind(&method_patches_.back().label); // Bind the label at the end of the "movl" insn.
break;
case HInvokeStaticOrDirect::MethodLoadKind::kDexCachePcRelative: {
Register base_reg = GetInvokeStaticOrDirectExtraParameter(invoke,
temp.AsRegister<Register>());
- uint32_t offset = invoke->GetDexCacheArrayOffset();
__ movl(temp.AsRegister<Register>(), Address(base_reg, kDummy32BitOffset));
- // Add the patch entry and bind its label at the end of the instruction.
- pc_relative_dex_cache_patches_.emplace_back(*invoke->GetTargetMethod().dex_file, offset);
- __ Bind(&pc_relative_dex_cache_patches_.back().label);
+ // Bind a new fixup label at the end of the "movl" insn.
+ uint32_t offset = invoke->GetDexCacheArrayOffset();
+ __ Bind(NewPcRelativeDexCacheArrayPatch(*invoke->GetTargetMethod().dex_file, offset));
break;
}
case HInvokeStaticOrDirect::MethodLoadKind::kDexCacheViaMethod: {
@@ -4494,12 +4497,33 @@
temp, ArtMethod::EntryPointFromQuickCompiledCodeOffset(kX86WordSize).Int32Value()));
}
+void CodeGeneratorX86::RecordSimplePatch() {
+ if (GetCompilerOptions().GetIncludePatchInformation()) {
+ simple_patches_.emplace_back();
+ __ Bind(&simple_patches_.back());
+ }
+}
+
+void CodeGeneratorX86::RecordStringPatch(HLoadString* load_string) {
+ string_patches_.emplace_back(load_string->GetDexFile(), load_string->GetStringIndex());
+ __ Bind(&string_patches_.back().label);
+}
+
+Label* CodeGeneratorX86::NewPcRelativeDexCacheArrayPatch(const DexFile& dex_file,
+ uint32_t element_offset) {
+ // Add the patch entry and bind its label at the end of the instruction.
+ pc_relative_dex_cache_patches_.emplace_back(dex_file, element_offset);
+ return &pc_relative_dex_cache_patches_.back().label;
+}
+
void CodeGeneratorX86::EmitLinkerPatches(ArenaVector<LinkerPatch>* linker_patches) {
DCHECK(linker_patches->empty());
size_t size =
method_patches_.size() +
relative_call_patches_.size() +
- pc_relative_dex_cache_patches_.size();
+ pc_relative_dex_cache_patches_.size() +
+ simple_patches_.size() +
+ string_patches_.size();
linker_patches->reserve(size);
// The label points to the end of the "movl" insn but the literal offset for method
// patch needs to point to the embedded constant which occupies the last 4 bytes.
@@ -4523,6 +4547,26 @@
GetMethodAddressOffset(),
info.element_offset));
}
+ for (const Label& label : simple_patches_) {
+ uint32_t literal_offset = label.Position() - kLabelPositionToLiteralOffsetAdjustment;
+ linker_patches->push_back(LinkerPatch::RecordPosition(literal_offset));
+ }
+ if (GetCompilerOptions().GetCompilePic()) {
+ for (const StringPatchInfo<Label>& info : string_patches_) {
+ uint32_t literal_offset = info.label.Position() - kLabelPositionToLiteralOffsetAdjustment;
+ linker_patches->push_back(LinkerPatch::RelativeStringPatch(literal_offset,
+ &info.dex_file,
+ GetMethodAddressOffset(),
+ info.string_index));
+ }
+ } else {
+ for (const StringPatchInfo<Label>& info : string_patches_) {
+ uint32_t literal_offset = info.label.Position() - kLabelPositionToLiteralOffsetAdjustment;
+ linker_patches->push_back(LinkerPatch::StringPatch(literal_offset,
+ &info.dex_file,
+ info.string_index));
+ }
+ }
}
void CodeGeneratorX86::MarkGCCard(Register temp,
@@ -5916,14 +5960,15 @@
DCHECK(!cls->MustGenerateClinitCheck());
// /* GcRoot<mirror::Class> */ out = current_method->declaring_class_
GenerateGcRootFieldLoad(
- cls, out_loc, current_method, ArtMethod::DeclaringClassOffset().Int32Value());
+ cls, out_loc, Address(current_method, ArtMethod::DeclaringClassOffset().Int32Value()));
} else {
// /* GcRoot<mirror::Class>[] */ out =
// current_method.ptr_sized_fields_->dex_cache_resolved_types_
__ movl(out, Address(current_method,
ArtMethod::DexCacheResolvedTypesOffset(kX86PointerSize).Int32Value()));
// /* GcRoot<mirror::Class> */ out = out[type_index]
- GenerateGcRootFieldLoad(cls, out_loc, out, CodeGenerator::GetCacheOffset(cls->GetTypeIndex()));
+ GenerateGcRootFieldLoad(
+ cls, out_loc, Address(out, CodeGenerator::GetCacheOffset(cls->GetTypeIndex())));
if (!cls->IsInDexCache() || cls->MustGenerateClinitCheck()) {
DCHECK(cls->CanCallRuntime());
@@ -5972,12 +6017,58 @@
// No need for memory fence, thanks to the X86 memory model.
}
+HLoadString::LoadKind CodeGeneratorX86::GetSupportedLoadStringKind(
+ HLoadString::LoadKind desired_string_load_kind) {
+ if (kEmitCompilerReadBarrier) {
+ switch (desired_string_load_kind) {
+ case HLoadString::LoadKind::kBootImageLinkTimeAddress:
+ case HLoadString::LoadKind::kBootImageLinkTimePcRelative:
+ case HLoadString::LoadKind::kBootImageAddress:
+ // TODO: Implement for read barrier.
+ return HLoadString::LoadKind::kDexCacheViaMethod;
+ default:
+ break;
+ }
+ }
+ switch (desired_string_load_kind) {
+ case HLoadString::LoadKind::kBootImageLinkTimeAddress:
+ DCHECK(!GetCompilerOptions().GetCompilePic());
+ break;
+ case HLoadString::LoadKind::kBootImageLinkTimePcRelative:
+ DCHECK(GetCompilerOptions().GetCompilePic());
+ FALLTHROUGH_INTENDED;
+ case HLoadString::LoadKind::kDexCachePcRelative:
+ DCHECK(!Runtime::Current()->UseJit()); // Note: boot image is also non-JIT.
+ // We disable pc-relative load when there is an irreducible loop, as the optimization
+ // is incompatible with it.
+ // TODO: Create as many X86ComputeBaseMethodAddress instructions as needed for methods
+ // with irreducible loops.
+ if (GetGraph()->HasIrreducibleLoops()) {
+ return HLoadString::LoadKind::kDexCacheViaMethod;
+ }
+ break;
+ case HLoadString::LoadKind::kBootImageAddress:
+ break;
+ case HLoadString::LoadKind::kDexCacheAddress:
+ DCHECK(Runtime::Current()->UseJit());
+ break;
+ case HLoadString::LoadKind::kDexCacheViaMethod:
+ break;
+ }
+ return desired_string_load_kind;
+}
+
void LocationsBuilderX86::VisitLoadString(HLoadString* load) {
- LocationSummary::CallKind call_kind = (!load->IsInDexCache() || kEmitCompilerReadBarrier)
+ LocationSummary::CallKind call_kind = (load->NeedsEnvironment() || kEmitCompilerReadBarrier)
? LocationSummary::kCallOnSlowPath
: LocationSummary::kNoCall;
LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(load, call_kind);
- locations->SetInAt(0, Location::RequiresRegister());
+ HLoadString::LoadKind load_kind = load->GetLoadKind();
+ if (load_kind == HLoadString::LoadKind::kDexCacheViaMethod ||
+ load_kind == HLoadString::LoadKind::kBootImageLinkTimePcRelative ||
+ load_kind == HLoadString::LoadKind::kDexCachePcRelative) {
+ locations->SetInAt(0, Location::RequiresRegister());
+ }
locations->SetOut(Location::RequiresRegister());
}
@@ -5985,16 +6076,61 @@
LocationSummary* locations = load->GetLocations();
Location out_loc = locations->Out();
Register out = out_loc.AsRegister<Register>();
- Register current_method = locations->InAt(0).AsRegister<Register>();
- // /* GcRoot<mirror::Class> */ out = current_method->declaring_class_
- GenerateGcRootFieldLoad(
- load, out_loc, current_method, ArtMethod::DeclaringClassOffset().Int32Value());
- // /* GcRoot<mirror::String>[] */ out = out->dex_cache_strings_
- __ movl(out, Address(out, mirror::Class::DexCacheStringsOffset().Int32Value()));
- // /* GcRoot<mirror::String> */ out = out[string_index]
- GenerateGcRootFieldLoad(
- load, out_loc, out, CodeGenerator::GetCacheOffset(load->GetStringIndex()));
+ switch (load->GetLoadKind()) {
+ case HLoadString::LoadKind::kBootImageLinkTimeAddress: {
+ DCHECK(!kEmitCompilerReadBarrier);
+ __ movl(out, Immediate(/* placeholder */ 0));
+ codegen_->RecordStringPatch(load);
+ return; // No dex cache slow path.
+ }
+ case HLoadString::LoadKind::kBootImageLinkTimePcRelative: {
+ DCHECK(!kEmitCompilerReadBarrier);
+ Register method_address = locations->InAt(0).AsRegister<Register>();
+ __ leal(out, Address(method_address, CodeGeneratorX86::kDummy32BitOffset));
+ codegen_->RecordStringPatch(load);
+ return; // No dex cache slow path.
+ }
+ case HLoadString::LoadKind::kBootImageAddress: {
+ DCHECK(!kEmitCompilerReadBarrier);
+ DCHECK_NE(load->GetAddress(), 0u);
+ uint32_t address = dchecked_integral_cast<uint32_t>(load->GetAddress());
+ __ movl(out, Immediate(address));
+ codegen_->RecordSimplePatch();
+ return; // No dex cache slow path.
+ }
+ case HLoadString::LoadKind::kDexCacheAddress: {
+ DCHECK_NE(load->GetAddress(), 0u);
+ uint32_t address = dchecked_integral_cast<uint32_t>(load->GetAddress());
+ GenerateGcRootFieldLoad(load, out_loc, Address::Absolute(address));
+ break;
+ }
+ case HLoadString::LoadKind::kDexCachePcRelative: {
+ Register base_reg = locations->InAt(0).AsRegister<Register>();
+ uint32_t offset = load->GetDexCacheElementOffset();
+ Label* fixup_label = codegen_->NewPcRelativeDexCacheArrayPatch(load->GetDexFile(), offset);
+ GenerateGcRootFieldLoad(
+ load, out_loc, Address(base_reg, CodeGeneratorX86::kDummy32BitOffset), fixup_label);
+ break;
+ }
+ case HLoadString::LoadKind::kDexCacheViaMethod: {
+ Register current_method = locations->InAt(0).AsRegister<Register>();
+
+ // /* GcRoot<mirror::Class> */ out = current_method->declaring_class_
+ GenerateGcRootFieldLoad(
+ load, out_loc, Address(current_method, ArtMethod::DeclaringClassOffset().Int32Value()));
+
+ // /* GcRoot<mirror::String>[] */ out = out->dex_cache_strings_
+ __ movl(out, Address(out, mirror::Class::DexCacheStringsOffset().Int32Value()));
+ // /* GcRoot<mirror::String> */ out = out[string_index]
+ GenerateGcRootFieldLoad(
+ load, out_loc, Address(out, CodeGenerator::GetCacheOffset(load->GetStringIndex())));
+ break;
+ }
+ default:
+ LOG(FATAL) << "Unexpected load kind: " << load->GetLoadKind();
+ UNREACHABLE();
+ }
if (!load->IsInDexCache()) {
SlowPathCode* slow_path = new (GetGraph()->GetArena()) LoadStringSlowPathX86(load);
@@ -6692,21 +6828,24 @@
void InstructionCodeGeneratorX86::GenerateGcRootFieldLoad(HInstruction* instruction,
Location root,
- Register obj,
- uint32_t offset) {
+ const Address& address,
+ Label* fixup_label) {
Register root_reg = root.AsRegister<Register>();
if (kEmitCompilerReadBarrier) {
if (kUseBakerReadBarrier) {
// Fast path implementation of art::ReadBarrier::BarrierForRoot when
// Baker's read barrier are used:
//
- // root = obj.field;
+ // root = *address;
// if (Thread::Current()->GetIsGcMarking()) {
// root = ReadBarrier::Mark(root)
// }
- // /* GcRoot<mirror::Object> */ root = *(obj + offset)
- __ movl(root_reg, Address(obj, offset));
+ // /* GcRoot<mirror::Object> */ root = *address
+ __ movl(root_reg, address);
+ if (fixup_label != nullptr) {
+ __ Bind(fixup_label);
+ }
static_assert(
sizeof(mirror::CompressedReference<mirror::Object>) == sizeof(GcRoot<mirror::Object>),
"art::mirror::CompressedReference<mirror::Object> and art::GcRoot<mirror::Object> "
@@ -6727,15 +6866,21 @@
} else {
// GC root loaded through a slow path for read barriers other
// than Baker's.
- // /* GcRoot<mirror::Object>* */ root = obj + offset
- __ leal(root_reg, Address(obj, offset));
+ // /* GcRoot<mirror::Object>* */ root = address
+ __ leal(root_reg, address);
+ if (fixup_label != nullptr) {
+ __ Bind(fixup_label);
+ }
// /* mirror::Object* */ root = root->Read()
codegen_->GenerateReadBarrierForRootSlow(instruction, root, root);
}
} else {
// Plain GC root load with no read barrier.
- // /* GcRoot<mirror::Object> */ root = *(obj + offset)
- __ movl(root_reg, Address(obj, offset));
+ // /* GcRoot<mirror::Object> */ root = *address
+ __ movl(root_reg, address);
+ if (fixup_label != nullptr) {
+ __ Bind(fixup_label);
+ }
// Note that GC roots are not affected by heap poisoning, thus we
// do not have to unpoison `root_reg` here.
}
diff --git a/compiler/optimizing/code_generator_x86.h b/compiler/optimizing/code_generator_x86.h
index c397899..1fa22fc 100644
--- a/compiler/optimizing/code_generator_x86.h
+++ b/compiler/optimizing/code_generator_x86.h
@@ -258,13 +258,13 @@
Location maybe_temp);
// Generate a GC root reference load:
//
- // root <- *(obj + offset)
+ // root <- *address
//
// while honoring read barriers (if any).
void GenerateGcRootFieldLoad(HInstruction* instruction,
Location root,
- Register obj,
- uint32_t offset);
+ const Address& address,
+ Label* fixup_label = nullptr);
// Push value to FPU stack. `is_fp` specifies whether the value is floating point or not.
// `is_wide` specifies whether it is long/double or not.
@@ -388,6 +388,11 @@
// Helper method to move a 64bits value between two locations.
void Move64(Location destination, Location source);
+ // Check if the desired_string_load_kind is supported. If it is, return it,
+ // otherwise return a fall-back kind that should be used instead.
+ HLoadString::LoadKind GetSupportedLoadStringKind(
+ HLoadString::LoadKind desired_string_load_kind) OVERRIDE;
+
// Check if the desired_dispatch_info is supported. If it is, return it,
// otherwise return a fall-back info that should be used instead.
HInvokeStaticOrDirect::DispatchInfo GetSupportedInvokeStaticOrDirectDispatch(
@@ -399,6 +404,10 @@
// Generate a call to a virtual method.
void GenerateVirtualCall(HInvokeVirtual* invoke, Location temp) OVERRIDE;
+ void RecordSimplePatch();
+ void RecordStringPatch(HLoadString* load_string);
+ Label* NewPcRelativeDexCacheArrayPatch(const DexFile& dex_file, uint32_t element_offset);
+
void MoveFromReturnRegister(Location trg, Primitive::Type type) OVERRIDE;
// Emit linker patches.
@@ -542,6 +551,10 @@
void GenerateImplicitNullCheck(HNullCheck* instruction);
void GenerateExplicitNullCheck(HNullCheck* instruction);
+ // When we don't know the proper offset for the value, we use kDummy32BitOffset.
+ // The correct value will be inserted when processing Assembler fixups.
+ static constexpr int32_t kDummy32BitOffset = 256;
+
private:
// Factored implementation of GenerateFieldLoadWithBakerReadBarrier
// and GenerateArrayLoadWithBakerReadBarrier.
@@ -578,6 +591,10 @@
ArenaDeque<MethodPatchInfo<Label>> relative_call_patches_;
// PC-relative DexCache access info.
ArenaDeque<PcRelativeDexCacheAccessInfo> pc_relative_dex_cache_patches_;
+ // Patch locations for patchoat where the linker doesn't do any other work.
+ ArenaDeque<Label> simple_patches_;
+ // String patch locations.
+ ArenaDeque<StringPatchInfo<Label>> string_patches_;
// Offset to the start of the constant area in the assembled code.
// Used for fixups to the constant area.
@@ -592,10 +609,6 @@
// instruction gives the address of the start of this method.
int32_t method_address_offset_;
- // When we don't know the proper offset for the value, we use kDummy32BitOffset.
- // The correct value will be inserted when processing Assembler fixups.
- static constexpr int32_t kDummy32BitOffset = 256;
-
DISALLOW_COPY_AND_ASSIGN(CodeGeneratorX86);
};
diff --git a/compiler/optimizing/code_generator_x86_64.cc b/compiler/optimizing/code_generator_x86_64.cc
index d24b5bb..cc46a07 100644
--- a/compiler/optimizing/code_generator_x86_64.cc
+++ b/compiler/optimizing/code_generator_x86_64.cc
@@ -784,14 +784,14 @@
method_patches_.emplace_back(invoke->GetTargetMethod());
__ Bind(&method_patches_.back().label); // Bind the label at the end of the "movl" insn.
break;
- case HInvokeStaticOrDirect::MethodLoadKind::kDexCachePcRelative:
- pc_relative_dex_cache_patches_.emplace_back(*invoke->GetTargetMethod().dex_file,
- invoke->GetDexCacheArrayOffset());
+ case HInvokeStaticOrDirect::MethodLoadKind::kDexCachePcRelative: {
__ movq(temp.AsRegister<CpuRegister>(),
Address::Absolute(kDummy32BitOffset, /* no_rip */ false));
- // Bind the label at the end of the "movl" insn.
- __ Bind(&pc_relative_dex_cache_patches_.back().label);
+ // Bind a new fixup label at the end of the "movl" insn.
+ uint32_t offset = invoke->GetDexCacheArrayOffset();
+ __ Bind(NewPcRelativeDexCacheArrayPatch(*invoke->GetTargetMethod().dex_file, offset));
break;
+ }
case HInvokeStaticOrDirect::MethodLoadKind::kDexCacheViaMethod: {
Location current_method = invoke->GetLocations()->InAt(invoke->GetSpecialInputIndex());
Register method_reg;
@@ -873,12 +873,33 @@
kX86_64WordSize).SizeValue()));
}
+void CodeGeneratorX86_64::RecordSimplePatch() {
+ if (GetCompilerOptions().GetIncludePatchInformation()) {
+ simple_patches_.emplace_back();
+ __ Bind(&simple_patches_.back());
+ }
+}
+
+void CodeGeneratorX86_64::RecordStringPatch(HLoadString* load_string) {
+ string_patches_.emplace_back(load_string->GetDexFile(), load_string->GetStringIndex());
+ __ Bind(&string_patches_.back().label);
+}
+
+Label* CodeGeneratorX86_64::NewPcRelativeDexCacheArrayPatch(const DexFile& dex_file,
+ uint32_t element_offset) {
+ // Add a patch entry and return the label.
+ pc_relative_dex_cache_patches_.emplace_back(dex_file, element_offset);
+ return &pc_relative_dex_cache_patches_.back().label;
+}
+
void CodeGeneratorX86_64::EmitLinkerPatches(ArenaVector<LinkerPatch>* linker_patches) {
DCHECK(linker_patches->empty());
size_t size =
method_patches_.size() +
relative_call_patches_.size() +
- pc_relative_dex_cache_patches_.size();
+ pc_relative_dex_cache_patches_.size() +
+ simple_patches_.size() +
+ string_patches_.size();
linker_patches->reserve(size);
// The label points to the end of the "movl" insn but the literal offset for method
// patch needs to point to the embedded constant which occupies the last 4 bytes.
@@ -902,6 +923,18 @@
info.label.Position(),
info.element_offset));
}
+ for (const Label& label : simple_patches_) {
+ uint32_t literal_offset = label.Position() - kLabelPositionToLiteralOffsetAdjustment;
+ linker_patches->push_back(LinkerPatch::RecordPosition(literal_offset));
+ }
+ for (const StringPatchInfo<Label>& info : string_patches_) {
+ // These are always PC-relative, see GetSupportedLoadStringKind().
+ uint32_t literal_offset = info.label.Position() - kLabelPositionToLiteralOffsetAdjustment;
+ linker_patches->push_back(LinkerPatch::RelativeStringPatch(literal_offset,
+ &info.dex_file,
+ info.label.Position(),
+ info.string_index));
+ }
}
void CodeGeneratorX86_64::DumpCoreRegister(std::ostream& stream, int reg) const {
@@ -978,6 +1011,8 @@
method_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)),
relative_call_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)),
pc_relative_dex_cache_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)),
+ simple_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)),
+ string_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)),
fixups_to_jump_tables_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)) {
AddAllocatedRegister(Location::RegisterLocation(kFakeReturnRegister));
}
@@ -2799,7 +2834,7 @@
} else if (in.IsConstant()) {
int64_t v = in.GetConstant()->AsLongConstant()->GetValue();
XmmRegister dest = out.AsFpuRegister<XmmRegister>();
- codegen_->Load64BitValue(dest, static_cast<double>(v));
+ codegen_->Load32BitValue(dest, static_cast<float>(v));
} else {
__ cvtsi2ss(out.AsFpuRegister<XmmRegister>(),
Address(CpuRegister(RSP), in.GetStackIndex()), true);
@@ -5249,6 +5284,12 @@
CpuRegister(ensure_scratch.GetRegister()));
}
+void ParallelMoveResolverX86_64::Exchange64(CpuRegister reg1, CpuRegister reg2) {
+ __ movq(CpuRegister(TMP), reg1);
+ __ movq(reg1, reg2);
+ __ movq(reg2, CpuRegister(TMP));
+}
+
void ParallelMoveResolverX86_64::Exchange64(CpuRegister reg, int mem) {
__ movq(CpuRegister(TMP), Address(CpuRegister(RSP), mem));
__ movq(Address(CpuRegister(RSP), mem), reg);
@@ -5286,7 +5327,7 @@
Location destination = move->GetDestination();
if (source.IsRegister() && destination.IsRegister()) {
- __ xchgq(destination.AsRegister<CpuRegister>(), source.AsRegister<CpuRegister>());
+ Exchange64(source.AsRegister<CpuRegister>(), destination.AsRegister<CpuRegister>());
} else if (source.IsRegister() && destination.IsStackSlot()) {
Exchange32(source.AsRegister<CpuRegister>(), destination.GetStackIndex());
} else if (source.IsStackSlot() && destination.IsRegister()) {
@@ -5365,14 +5406,15 @@
DCHECK(!cls->MustGenerateClinitCheck());
// /* GcRoot<mirror::Class> */ out = current_method->declaring_class_
GenerateGcRootFieldLoad(
- cls, out_loc, current_method, ArtMethod::DeclaringClassOffset().Int32Value());
+ cls, out_loc, Address(current_method, ArtMethod::DeclaringClassOffset().Int32Value()));
} else {
// /* GcRoot<mirror::Class>[] */ out =
// current_method.ptr_sized_fields_->dex_cache_resolved_types_
__ movq(out, Address(current_method,
ArtMethod::DexCacheResolvedTypesOffset(kX86_64PointerSize).Int32Value()));
// /* GcRoot<mirror::Class> */ out = out[type_index]
- GenerateGcRootFieldLoad(cls, out_loc, out, CodeGenerator::GetCacheOffset(cls->GetTypeIndex()));
+ GenerateGcRootFieldLoad(
+ cls, out_loc, Address(out, CodeGenerator::GetCacheOffset(cls->GetTypeIndex())));
if (!cls->IsInDexCache() || cls->MustGenerateClinitCheck()) {
DCHECK(cls->CanCallRuntime());
@@ -5410,12 +5452,49 @@
check->GetLocations()->InAt(0).AsRegister<CpuRegister>());
}
+HLoadString::LoadKind CodeGeneratorX86_64::GetSupportedLoadStringKind(
+ HLoadString::LoadKind desired_string_load_kind) {
+ if (kEmitCompilerReadBarrier) {
+ switch (desired_string_load_kind) {
+ case HLoadString::LoadKind::kBootImageLinkTimeAddress:
+ case HLoadString::LoadKind::kBootImageLinkTimePcRelative:
+ case HLoadString::LoadKind::kBootImageAddress:
+ // TODO: Implement for read barrier.
+ return HLoadString::LoadKind::kDexCacheViaMethod;
+ default:
+ break;
+ }
+ }
+ switch (desired_string_load_kind) {
+ case HLoadString::LoadKind::kBootImageLinkTimeAddress:
+ DCHECK(!GetCompilerOptions().GetCompilePic());
+ // We prefer the always-available RIP-relative address for the x86-64 boot image.
+ return HLoadString::LoadKind::kBootImageLinkTimePcRelative;
+ case HLoadString::LoadKind::kBootImageLinkTimePcRelative:
+ DCHECK(GetCompilerOptions().GetCompilePic());
+ break;
+ case HLoadString::LoadKind::kBootImageAddress:
+ break;
+ case HLoadString::LoadKind::kDexCacheAddress:
+ DCHECK(Runtime::Current()->UseJit());
+ break;
+ case HLoadString::LoadKind::kDexCachePcRelative:
+ DCHECK(!Runtime::Current()->UseJit());
+ break;
+ case HLoadString::LoadKind::kDexCacheViaMethod:
+ break;
+ }
+ return desired_string_load_kind;
+}
+
void LocationsBuilderX86_64::VisitLoadString(HLoadString* load) {
- LocationSummary::CallKind call_kind = (!load->IsInDexCache() || kEmitCompilerReadBarrier)
+ LocationSummary::CallKind call_kind = (load->NeedsEnvironment() || kEmitCompilerReadBarrier)
? LocationSummary::kCallOnSlowPath
: LocationSummary::kNoCall;
LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(load, call_kind);
- locations->SetInAt(0, Location::RequiresRegister());
+ if (load->GetLoadKind() == HLoadString::LoadKind::kDexCacheViaMethod) {
+ locations->SetInAt(0, Location::RequiresRegister());
+ }
locations->SetOut(Location::RequiresRegister());
}
@@ -5423,16 +5502,59 @@
LocationSummary* locations = load->GetLocations();
Location out_loc = locations->Out();
CpuRegister out = out_loc.AsRegister<CpuRegister>();
- CpuRegister current_method = locations->InAt(0).AsRegister<CpuRegister>();
- // /* GcRoot<mirror::Class> */ out = current_method->declaring_class_
- GenerateGcRootFieldLoad(
- load, out_loc, current_method, ArtMethod::DeclaringClassOffset().Int32Value());
- // /* GcRoot<mirror::String>[] */ out = out->dex_cache_strings_
- __ movq(out, Address(out, mirror::Class::DexCacheStringsOffset().Uint32Value()));
- // /* GcRoot<mirror::String> */ out = out[string_index]
- GenerateGcRootFieldLoad(
- load, out_loc, out, CodeGenerator::GetCacheOffset(load->GetStringIndex()));
+ switch (load->GetLoadKind()) {
+ case HLoadString::LoadKind::kBootImageLinkTimePcRelative: {
+ DCHECK(!kEmitCompilerReadBarrier);
+ __ leal(out, Address::Absolute(CodeGeneratorX86_64::kDummy32BitOffset, /* no_rip */ false));
+ codegen_->RecordStringPatch(load);
+ return; // No dex cache slow path.
+ }
+ case HLoadString::LoadKind::kBootImageAddress: {
+ DCHECK(!kEmitCompilerReadBarrier);
+ DCHECK_NE(load->GetAddress(), 0u);
+ uint32_t address = dchecked_integral_cast<uint32_t>(load->GetAddress());
+ __ movl(out, Immediate(address)); // Zero-extended.
+ codegen_->RecordSimplePatch();
+ return; // No dex cache slow path.
+ }
+ case HLoadString::LoadKind::kDexCacheAddress: {
+ DCHECK_NE(load->GetAddress(), 0u);
+ if (IsUint<32>(load->GetAddress())) {
+ Address address = Address::Absolute(load->GetAddress(), /* no_rip */ true);
+ GenerateGcRootFieldLoad(load, out_loc, address);
+ } else {
+ // TODO: Consider using opcode A1, i.e. movl eax, moff32 (with 64-bit address).
+ __ movq(out, Immediate(load->GetAddress()));
+ GenerateGcRootFieldLoad(load, out_loc, Address(out, 0));
+ }
+ break;
+ }
+ case HLoadString::LoadKind::kDexCachePcRelative: {
+ uint32_t offset = load->GetDexCacheElementOffset();
+ Label* fixup_label = codegen_->NewPcRelativeDexCacheArrayPatch(load->GetDexFile(), offset);
+ Address address = Address::Absolute(CodeGeneratorX86_64::kDummy32BitOffset,
+ /* no_rip */ false);
+ GenerateGcRootFieldLoad(load, out_loc, address, fixup_label);
+ break;
+ }
+ case HLoadString::LoadKind::kDexCacheViaMethod: {
+ CpuRegister current_method = locations->InAt(0).AsRegister<CpuRegister>();
+
+ // /* GcRoot<mirror::Class> */ out = current_method->declaring_class_
+ GenerateGcRootFieldLoad(
+ load, out_loc, Address(current_method, ArtMethod::DeclaringClassOffset().Int32Value()));
+ // /* GcRoot<mirror::String>[] */ out = out->dex_cache_strings_
+ __ movq(out, Address(out, mirror::Class::DexCacheStringsOffset().Uint32Value()));
+ // /* GcRoot<mirror::String> */ out = out[string_index]
+ GenerateGcRootFieldLoad(
+ load, out_loc, Address(out, CodeGenerator::GetCacheOffset(load->GetStringIndex())));
+ break;
+ }
+ default:
+ LOG(FATAL) << "Unexpected load kind: " << load->GetLoadKind();
+ UNREACHABLE();
+ }
if (!load->IsInDexCache()) {
SlowPathCode* slow_path = new (GetGraph()->GetArena()) LoadStringSlowPathX86_64(load);
@@ -6171,21 +6293,24 @@
void InstructionCodeGeneratorX86_64::GenerateGcRootFieldLoad(HInstruction* instruction,
Location root,
- CpuRegister obj,
- uint32_t offset) {
+ const Address& address,
+ Label* fixup_label) {
CpuRegister root_reg = root.AsRegister<CpuRegister>();
if (kEmitCompilerReadBarrier) {
if (kUseBakerReadBarrier) {
// Fast path implementation of art::ReadBarrier::BarrierForRoot when
// Baker's read barrier are used:
//
- // root = obj.field;
+ // root = *address;
// if (Thread::Current()->GetIsGcMarking()) {
// root = ReadBarrier::Mark(root)
// }
- // /* GcRoot<mirror::Object> */ root = *(obj + offset)
- __ movl(root_reg, Address(obj, offset));
+ // /* GcRoot<mirror::Object> */ root = *address
+ __ movl(root_reg, address);
+ if (fixup_label != nullptr) {
+ __ Bind(fixup_label);
+ }
static_assert(
sizeof(mirror::CompressedReference<mirror::Object>) == sizeof(GcRoot<mirror::Object>),
"art::mirror::CompressedReference<mirror::Object> and art::GcRoot<mirror::Object> "
@@ -6207,15 +6332,21 @@
} else {
// GC root loaded through a slow path for read barriers other
// than Baker's.
- // /* GcRoot<mirror::Object>* */ root = obj + offset
- __ leaq(root_reg, Address(obj, offset));
+ // /* GcRoot<mirror::Object>* */ root = address
+ __ leaq(root_reg, address);
+ if (fixup_label != nullptr) {
+ __ Bind(fixup_label);
+ }
// /* mirror::Object* */ root = root->Read()
codegen_->GenerateReadBarrierForRootSlow(instruction, root, root);
}
} else {
// Plain GC root load with no read barrier.
- // /* GcRoot<mirror::Object> */ root = *(obj + offset)
- __ movl(root_reg, Address(obj, offset));
+ // /* GcRoot<mirror::Object> */ root = *address
+ __ movl(root_reg, address);
+ if (fixup_label != nullptr) {
+ __ Bind(fixup_label);
+ }
// Note that GC roots are not affected by heap poisoning, thus we
// do not have to unpoison `root_reg` here.
}
diff --git a/compiler/optimizing/code_generator_x86_64.h b/compiler/optimizing/code_generator_x86_64.h
index c3fce6e..7ebce58 100644
--- a/compiler/optimizing/code_generator_x86_64.h
+++ b/compiler/optimizing/code_generator_x86_64.h
@@ -142,6 +142,7 @@
void Exchange32(CpuRegister reg, int mem);
void Exchange32(XmmRegister reg, int mem);
void Exchange32(int mem1, int mem2);
+ void Exchange64(CpuRegister reg1, CpuRegister reg2);
void Exchange64(CpuRegister reg, int mem);
void Exchange64(XmmRegister reg, int mem);
void Exchange64(int mem1, int mem2);
@@ -252,13 +253,13 @@
Location maybe_temp);
// Generate a GC root reference load:
//
- // root <- *(obj + offset)
+ // root <- *address
//
// while honoring read barriers (if any).
void GenerateGcRootFieldLoad(HInstruction* instruction,
Location root,
- CpuRegister obj,
- uint32_t offset);
+ const Address& address,
+ Label* fixup_label = nullptr);
void PushOntoFPStack(Location source, uint32_t temp_offset,
uint32_t stack_adjustment, bool is_float);
@@ -384,6 +385,11 @@
return false;
}
+ // Check if the desired_string_load_kind is supported. If it is, return it,
+ // otherwise return a fall-back kind that should be used instead.
+ HLoadString::LoadKind GetSupportedLoadStringKind(
+ HLoadString::LoadKind desired_string_load_kind) OVERRIDE;
+
// Check if the desired_dispatch_info is supported. If it is, return it,
// otherwise return a fall-back info that should be used instead.
HInvokeStaticOrDirect::DispatchInfo GetSupportedInvokeStaticOrDirectDispatch(
@@ -393,6 +399,10 @@
void GenerateStaticOrDirectCall(HInvokeStaticOrDirect* invoke, Location temp) OVERRIDE;
void GenerateVirtualCall(HInvokeVirtual* invoke, Location temp) OVERRIDE;
+ void RecordSimplePatch();
+ void RecordStringPatch(HLoadString* load_string);
+ Label* NewPcRelativeDexCacheArrayPatch(const DexFile& dex_file, uint32_t element_offset);
+
void MoveFromReturnRegister(Location trg, Primitive::Type type) OVERRIDE;
void EmitLinkerPatches(ArenaVector<LinkerPatch>* linker_patches) OVERRIDE;
@@ -515,6 +525,10 @@
void GenerateImplicitNullCheck(HNullCheck* instruction);
void GenerateExplicitNullCheck(HNullCheck* instruction);
+ // When we don't know the proper offset for the value, we use kDummy32BitOffset.
+ // We will fix this up in the linker later to have the right value.
+ static constexpr int32_t kDummy32BitOffset = 256;
+
private:
// Factored implementation of GenerateFieldLoadWithBakerReadBarrier
// and GenerateArrayLoadWithBakerReadBarrier.
@@ -552,10 +566,10 @@
ArenaDeque<MethodPatchInfo<Label>> relative_call_patches_;
// PC-relative DexCache access info.
ArenaDeque<PcRelativeDexCacheAccessInfo> pc_relative_dex_cache_patches_;
-
- // When we don't know the proper offset for the value, we use kDummy32BitOffset.
- // We will fix this up in the linker later to have the right value.
- static constexpr int32_t kDummy32BitOffset = 256;
+ // Patch locations for patchoat where the linker doesn't do any other work.
+ ArenaDeque<Label> simple_patches_;
+ // String patch locations.
+ ArenaDeque<StringPatchInfo<Label>> string_patches_;
// Fixups for jump tables need to be handled specially.
ArenaVector<JumpTableRIPFixup*> fixups_to_jump_tables_;
diff --git a/compiler/optimizing/dex_cache_array_fixups_arm.cc b/compiler/optimizing/dex_cache_array_fixups_arm.cc
index 3db254a..e9072b9 100644
--- a/compiler/optimizing/dex_cache_array_fixups_arm.cc
+++ b/compiler/optimizing/dex_cache_array_fixups_arm.cc
@@ -44,6 +44,21 @@
}
private:
+ void VisitLoadString(HLoadString* load_string) OVERRIDE {
+ // If this is a load with PC-relative access to the dex cache methods array,
+ // we need to add the dex cache arrays base as the special input.
+ if (load_string->GetLoadKind() == HLoadString::LoadKind::kDexCachePcRelative) {
+ // Initialize base for target dex file if needed.
+ const DexFile& dex_file = load_string->GetDexFile();
+ HArmDexCacheArraysBase* base = GetOrCreateDexCacheArrayBase(dex_file);
+ // Update the element offset in base.
+ DexCacheArraysLayout layout(kArmPointerSize, &dex_file);
+ base->UpdateElementOffset(layout.StringOffset(load_string->GetStringIndex()));
+ // Add the special argument base to the load.
+ load_string->AddSpecialInput(base);
+ }
+ }
+
void VisitInvokeStaticOrDirect(HInvokeStaticOrDirect* invoke) OVERRIDE {
// If this is an invoke with PC-relative access to the dex cache methods array,
// we need to add the dex cache arrays base as the special input.
diff --git a/compiler/optimizing/graph_checker.cc b/compiler/optimizing/graph_checker.cc
index e5c8c47..c790d01 100644
--- a/compiler/optimizing/graph_checker.cc
+++ b/compiler/optimizing/graph_checker.cc
@@ -32,11 +32,9 @@
// Check consistency with respect to predecessors of `block`.
// Note: Counting duplicates with a sorted vector uses up to 6x less memory
- // than ArenaSafeMap<HBasicBlock*, size_t>.
- ArenaVector<HBasicBlock*> sorted_predecessors(
- block->GetPredecessors().begin(),
- block->GetPredecessors().end(),
- GetGraph()->GetArena()->Adapter(kArenaAllocGraphChecker));
+ // than ArenaSafeMap<HBasicBlock*, size_t> and also allows storage reuse.
+ ArenaVector<HBasicBlock*>& sorted_predecessors = blocks_storage_;
+ sorted_predecessors.assign(block->GetPredecessors().begin(), block->GetPredecessors().end());
std::sort(sorted_predecessors.begin(), sorted_predecessors.end());
for (auto it = sorted_predecessors.begin(), end = sorted_predecessors.end(); it != end; ) {
HBasicBlock* p = *it++;
@@ -57,11 +55,9 @@
// Check consistency with respect to successors of `block`.
// Note: Counting duplicates with a sorted vector uses up to 6x less memory
- // than ArenaSafeMap<HBasicBlock*, size_t>.
- ArenaVector<HBasicBlock*> sorted_successors(
- block->GetSuccessors().begin(),
- block->GetSuccessors().end(),
- GetGraph()->GetArena()->Adapter(kArenaAllocGraphChecker));
+ // than ArenaSafeMap<HBasicBlock*, size_t> and also allows storage reuse.
+ ArenaVector<HBasicBlock*>& sorted_successors = blocks_storage_;
+ sorted_successors.assign(block->GetSuccessors().begin(), block->GetSuccessors().end());
std::sort(sorted_successors.begin(), sorted_successors.end());
for (auto it = sorted_successors.begin(), end = sorted_successors.end(); it != end; ) {
HBasicBlock* s = *it++;
@@ -811,10 +807,11 @@
phi->GetRegNumber(),
type_str.str().c_str()));
} else {
- ArenaBitVector visited(GetGraph()->GetArena(),
- 0,
- /* expandable */ true,
- kArenaAllocGraphChecker);
+ // If we get here, make sure we allocate all the necessary storage at once
+ // because the BitVector reallocation strategy has very bad worst-case behavior.
+ ArenaBitVector& visited = visited_storage_;
+ visited.SetBit(GetGraph()->GetCurrentInstructionId());
+ visited.ClearAllBits();
if (!IsConstantEquivalent(phi, other_phi, &visited)) {
AddError(StringPrintf("Two phis (%d and %d) found for VReg %d but they "
"are not equivalents of constants.",
@@ -1008,4 +1005,19 @@
}
}
+void GraphChecker::VisitTypeConversion(HTypeConversion* instruction) {
+ VisitInstruction(instruction);
+ Primitive::Type result_type = instruction->GetResultType();
+ Primitive::Type input_type = instruction->GetInputType();
+ // Invariant: We should never generate a conversion to a Boolean value.
+ if (result_type == Primitive::kPrimBoolean) {
+ AddError(StringPrintf(
+ "%s %d converts to a %s (from a %s).",
+ instruction->DebugName(),
+ instruction->GetId(),
+ Primitive::PrettyDescriptor(result_type),
+ Primitive::PrettyDescriptor(input_type)));
+ }
+}
+
} // namespace art
diff --git a/compiler/optimizing/graph_checker.h b/compiler/optimizing/graph_checker.h
index 27d5621..83b1984 100644
--- a/compiler/optimizing/graph_checker.h
+++ b/compiler/optimizing/graph_checker.h
@@ -33,7 +33,9 @@
seen_ids_(graph->GetArena(),
graph->GetCurrentInstructionId(),
false,
- kArenaAllocGraphChecker) {}
+ kArenaAllocGraphChecker),
+ blocks_storage_(graph->GetArena()->Adapter(kArenaAllocGraphChecker)),
+ visited_storage_(graph->GetArena(), 0u, true, kArenaAllocGraphChecker) {}
// Check the whole graph (in reverse post-order).
void Run() {
@@ -65,6 +67,7 @@
void VisitReturnVoid(HReturnVoid* ret) OVERRIDE;
void VisitSelect(HSelect* instruction) OVERRIDE;
void VisitTryBoundary(HTryBoundary* try_boundary) OVERRIDE;
+ void VisitTypeConversion(HTypeConversion* instruction) OVERRIDE;
void HandleLoop(HBasicBlock* loop_header);
void HandleBooleanInput(HInstruction* instruction, size_t input_index);
@@ -102,6 +105,10 @@
const char* const dump_prefix_;
ArenaBitVector seen_ids_;
+ // To reduce the total arena memory allocation, we reuse the same storage.
+ ArenaVector<HBasicBlock*> blocks_storage_;
+ ArenaBitVector visited_storage_;
+
DISALLOW_COPY_AND_ASSIGN(GraphChecker);
};
diff --git a/compiler/optimizing/graph_visualizer.cc b/compiler/optimizing/graph_visualizer.cc
index 4b5b919..fe47f7d 100644
--- a/compiler/optimizing/graph_visualizer.cc
+++ b/compiler/optimizing/graph_visualizer.cc
@@ -178,41 +178,47 @@
: nullptr),
indent_(0) {}
+ void Flush() {
+ // We use "\n" instead of std::endl to avoid implicit flushing which
+ // generates too many syscalls during debug-GC tests (b/27826765).
+ output_ << std::flush;
+ }
+
void StartTag(const char* name) {
AddIndent();
- output_ << "begin_" << name << std::endl;
+ output_ << "begin_" << name << "\n";
indent_++;
}
void EndTag(const char* name) {
indent_--;
AddIndent();
- output_ << "end_" << name << std::endl;
+ output_ << "end_" << name << "\n";
}
void PrintProperty(const char* name, const char* property) {
AddIndent();
- output_ << name << " \"" << property << "\"" << std::endl;
+ output_ << name << " \"" << property << "\"\n";
}
void PrintProperty(const char* name, const char* property, int id) {
AddIndent();
- output_ << name << " \"" << property << id << "\"" << std::endl;
+ output_ << name << " \"" << property << id << "\"\n";
}
void PrintEmptyProperty(const char* name) {
AddIndent();
- output_ << name << std::endl;
+ output_ << name << "\n";
}
void PrintTime(const char* name) {
AddIndent();
- output_ << name << " " << time(nullptr) << std::endl;
+ output_ << name << " " << time(nullptr) << "\n";
}
void PrintInt(const char* name, int value) {
AddIndent();
- output_ << name << " " << value << std::endl;
+ output_ << name << " " << value << "\n";
}
void AddIndent() {
@@ -249,7 +255,7 @@
if (block->IsEntryBlock() && (disasm_info_ != nullptr)) {
output_ << " \"" << kDisassemblyBlockFrameEntry << "\" ";
}
- output_<< std::endl;
+ output_<< "\n";
}
void PrintSuccessors(HBasicBlock* block) {
@@ -258,7 +264,7 @@
for (HBasicBlock* successor : block->GetNormalSuccessors()) {
output_ << " \"B" << successor->GetBlockId() << "\" ";
}
- output_<< std::endl;
+ output_<< "\n";
}
void PrintExceptionHandlers(HBasicBlock* block) {
@@ -272,7 +278,7 @@
!disasm_info_->GetSlowPathIntervals().empty()) {
output_ << " \"" << kDisassemblyBlockSlowPaths << "\" ";
}
- output_<< std::endl;
+ output_<< "\n";
}
void DumpLocation(std::ostream& stream, const Location& location) {
@@ -367,6 +373,10 @@
<< load_class->NeedsAccessCheck() << std::noboolalpha;
}
+ void VisitLoadString(HLoadString* load_string) OVERRIDE {
+ StartAttributeStream("load_kind") << load_string->GetLoadKind();
+ }
+
void VisitCheckCast(HCheckCast* check_cast) OVERRIDE {
StartAttributeStream("check_kind") << check_cast->GetTypeCheckKind();
StartAttributeStream("must_do_null_check") << std::boolalpha
@@ -588,7 +598,7 @@
auto it = disasm_info_->GetInstructionIntervals().find(instruction);
if (it != disasm_info_->GetInstructionIntervals().end()
&& it->second.start != it->second.end) {
- output_ << std::endl;
+ output_ << "\n";
disassembler_->Disassemble(output_, it->second.start, it->second.end);
}
}
@@ -608,7 +618,7 @@
output_ << bci << " " << num_uses << " "
<< GetTypeId(instruction->GetType()) << instruction->GetId() << " ";
PrintInstruction(instruction);
- output_ << " " << kEndInstructionMarker << std::endl;
+ output_ << " " << kEndInstructionMarker << "\n";
}
}
@@ -652,10 +662,10 @@
output_ << " 0 0 disasm " << kDisassemblyBlockFrameEntry << " ";
GeneratedCodeInterval frame_entry = disasm_info_->GetFrameEntryInterval();
if (frame_entry.start != frame_entry.end) {
- output_ << std::endl;
+ output_ << "\n";
disassembler_->Disassemble(output_, frame_entry.start, frame_entry.end);
}
- output_ << kEndInstructionMarker << std::endl;
+ output_ << kEndInstructionMarker << "\n";
DumpEndOfDisassemblyBlock();
}
@@ -671,9 +681,9 @@
GetGraph()->HasExitBlock() ? GetGraph()->GetExitBlock()->GetBlockId() : -1,
-1);
for (SlowPathCodeInfo info : disasm_info_->GetSlowPathIntervals()) {
- output_ << " 0 0 disasm " << info.slow_path->GetDescription() << std::endl;
+ output_ << " 0 0 disasm " << info.slow_path->GetDescription() << "\n";
disassembler_->Disassemble(output_, info.code_interval.start, info.code_interval.end);
- output_ << kEndInstructionMarker << std::endl;
+ output_ << kEndInstructionMarker << "\n";
}
DumpEndOfDisassemblyBlock();
}
@@ -694,6 +704,7 @@
DumpDisassemblyBlockForSlowPaths();
}
EndTag("cfg");
+ Flush();
}
void VisitBasicBlock(HBasicBlock* block) OVERRIDE {
@@ -733,7 +744,7 @@
for (HInputIterator inputs(instruction); !inputs.Done(); inputs.Advance()) {
output_ << inputs.Current()->GetId() << " ";
}
- output_ << "]" << std::endl;
+ output_ << "]\n";
}
EndTag("locals");
EndTag("states");
@@ -775,6 +786,7 @@
printer.PrintProperty("method", method_name);
printer.PrintTime("date");
printer.EndTag("compilation");
+ printer.Flush();
}
void HGraphVisualizer::DumpGraph(const char* pass_name,
diff --git a/compiler/optimizing/instruction_simplifier.cc b/compiler/optimizing/instruction_simplifier.cc
index 1249b48..1f66db7 100644
--- a/compiler/optimizing/instruction_simplifier.cc
+++ b/compiler/optimizing/instruction_simplifier.cc
@@ -786,14 +786,21 @@
}
static bool IsTypeConversionImplicit(Primitive::Type input_type, Primitive::Type result_type) {
+ // Invariant: We should never generate a conversion to a Boolean value.
+ DCHECK_NE(Primitive::kPrimBoolean, result_type);
+
// Besides conversion to the same type, widening integral conversions are implicit,
// excluding conversions to long and the byte->char conversion where we need to
// clear the high 16 bits of the 32-bit sign-extended representation of byte.
return result_type == input_type ||
- (result_type == Primitive::kPrimInt && input_type == Primitive::kPrimByte) ||
- (result_type == Primitive::kPrimInt && input_type == Primitive::kPrimShort) ||
- (result_type == Primitive::kPrimInt && input_type == Primitive::kPrimChar) ||
- (result_type == Primitive::kPrimShort && input_type == Primitive::kPrimByte);
+ (result_type == Primitive::kPrimInt && (input_type == Primitive::kPrimBoolean ||
+ input_type == Primitive::kPrimByte ||
+ input_type == Primitive::kPrimShort ||
+ input_type == Primitive::kPrimChar)) ||
+ (result_type == Primitive::kPrimChar && input_type == Primitive::kPrimBoolean) ||
+ (result_type == Primitive::kPrimShort && (input_type == Primitive::kPrimBoolean ||
+ input_type == Primitive::kPrimByte)) ||
+ (result_type == Primitive::kPrimByte && input_type == Primitive::kPrimBoolean);
}
static bool IsTypeConversionLossless(Primitive::Type input_type, Primitive::Type result_type) {
diff --git a/compiler/optimizing/instruction_simplifier_arm64.h b/compiler/optimizing/instruction_simplifier_arm64.h
index 338120b..da26998 100644
--- a/compiler/optimizing/instruction_simplifier_arm64.h
+++ b/compiler/optimizing/instruction_simplifier_arm64.h
@@ -51,6 +51,22 @@
return TryMergeIntoShifterOperand(use, bitfield_op, true);
}
+ /**
+ * This simplifier uses a special-purpose BB visitor.
+ * (1) No need to visit Phi nodes.
+ * (2) Since statements can be removed in a "forward" fashion,
+ * the visitor should test if each statement is still there.
+ */
+ void VisitBasicBlock(HBasicBlock* block) OVERRIDE {
+ // TODO: fragile iteration, provide more robust iterators?
+ for (HInstructionIterator it(block->GetInstructions()); !it.Done(); it.Advance()) {
+ HInstruction* instruction = it.Current();
+ if (instruction->IsInBlock()) {
+ instruction->Accept(this);
+ }
+ }
+ }
+
// HInstruction visitors, sorted alphabetically.
void VisitAnd(HAnd* instruction) OVERRIDE;
void VisitArrayGet(HArrayGet* instruction) OVERRIDE;
diff --git a/compiler/optimizing/nodes.cc b/compiler/optimizing/nodes.cc
index 1a426d5..9504481 100644
--- a/compiler/optimizing/nodes.cc
+++ b/compiler/optimizing/nodes.cc
@@ -86,11 +86,7 @@
}
}
-static void RemoveAsUser(HInstruction* instruction) {
- for (size_t i = 0; i < instruction->InputCount(); i++) {
- instruction->RemoveAsUserOfInput(i);
- }
-
+static void RemoveEnvironmentUses(HInstruction* instruction) {
for (HEnvironment* environment = instruction->GetEnvironment();
environment != nullptr;
environment = environment->GetParent()) {
@@ -102,6 +98,14 @@
}
}
+static void RemoveAsUser(HInstruction* instruction) {
+ for (size_t i = 0; i < instruction->InputCount(); i++) {
+ instruction->RemoveAsUserOfInput(i);
+ }
+
+ RemoveEnvironmentUses(instruction);
+}
+
void HGraph::RemoveInstructionsAsUsersFromDeadBlocks(const ArenaBitVector& visited) const {
for (size_t i = 0; i < blocks_.size(); ++i) {
if (!visited.IsBitSet(i)) {
@@ -1007,6 +1011,11 @@
}
}
+void HInstruction::RemoveEnvironment() {
+ RemoveEnvironmentUses(this);
+ environment_ = nullptr;
+}
+
void HInstruction::ReplaceWith(HInstruction* other) {
DCHECK(other != nullptr);
for (HUseIterator<HInstruction*> it(GetUses()); !it.Done(); it.Advance()) {
@@ -2387,6 +2396,59 @@
}
}
+bool HLoadString::InstructionDataEquals(HInstruction* other) const {
+ HLoadString* other_load_string = other->AsLoadString();
+ if (string_index_ != other_load_string->string_index_ ||
+ GetPackedFields() != other_load_string->GetPackedFields()) {
+ return false;
+ }
+ LoadKind load_kind = GetLoadKind();
+ if (HasAddress(load_kind)) {
+ return GetAddress() == other_load_string->GetAddress();
+ } else if (HasStringReference(load_kind)) {
+ return IsSameDexFile(GetDexFile(), other_load_string->GetDexFile());
+ } else {
+ DCHECK(HasDexCacheReference(load_kind)) << load_kind;
+ // If the string indexes and dex files are the same, dex cache element offsets
+ // must also be the same, so we don't need to compare them.
+ return IsSameDexFile(GetDexFile(), other_load_string->GetDexFile());
+ }
+}
+
+void HLoadString::SetLoadKindInternal(LoadKind load_kind) {
+ // Once sharpened, the load kind should not be changed again.
+ DCHECK_EQ(GetLoadKind(), LoadKind::kDexCacheViaMethod);
+ SetPackedField<LoadKindField>(load_kind);
+
+ if (load_kind != LoadKind::kDexCacheViaMethod) {
+ RemoveAsUserOfInput(0u);
+ SetRawInputAt(0u, nullptr);
+ }
+ if (!NeedsEnvironment()) {
+ RemoveEnvironment();
+ }
+}
+
+std::ostream& operator<<(std::ostream& os, HLoadString::LoadKind rhs) {
+ switch (rhs) {
+ case HLoadString::LoadKind::kBootImageLinkTimeAddress:
+ return os << "BootImageLinkTimeAddress";
+ case HLoadString::LoadKind::kBootImageLinkTimePcRelative:
+ return os << "BootImageLinkTimePcRelative";
+ case HLoadString::LoadKind::kBootImageAddress:
+ return os << "BootImageAddress";
+ case HLoadString::LoadKind::kDexCacheAddress:
+ return os << "DexCacheAddress";
+ case HLoadString::LoadKind::kDexCachePcRelative:
+ return os << "DexCachePcRelative";
+ case HLoadString::LoadKind::kDexCacheViaMethod:
+ return os << "DexCacheViaMethod";
+ default:
+ LOG(FATAL) << "Unknown HLoadString::LoadKind: " << static_cast<int>(rhs);
+ UNREACHABLE();
+ }
+}
+
void HInstruction::RemoveEnvironmentUsers() {
for (HUseIterator<HEnvironment*> use_it(GetEnvUses()); !use_it.Done(); use_it.Advance()) {
HUseListNode<HEnvironment*>* user_node = use_it.Current();
diff --git a/compiler/optimizing/nodes.h b/compiler/optimizing/nodes.h
index e9a42cb..9425ef3 100644
--- a/compiler/optimizing/nodes.h
+++ b/compiler/optimizing/nodes.h
@@ -1995,6 +1995,8 @@
environment_ = environment;
}
+ void RemoveEnvironment();
+
// Set the environment of this instruction, copying it from `environment`. While
// copying, the uses lists are being updated.
void CopyEnvironmentFrom(HEnvironment* environment) {
@@ -4902,6 +4904,8 @@
dex_pc) {
SetRawInputAt(0, input);
DCHECK_NE(input->GetType(), result_type);
+ // Invariant: We should never generate a conversion to a Boolean value.
+ DCHECK_NE(Primitive::kPrimBoolean, result_type);
}
HInstruction* GetInput() const { return InputAt(0); }
@@ -5557,32 +5561,117 @@
class HLoadString : public HExpression<1> {
public:
+ // Determines how to load the String.
+ enum class LoadKind {
+ // Use boot image String* address that will be known at link time.
+ // Used for boot image strings referenced by boot image code in non-PIC mode.
+ kBootImageLinkTimeAddress,
+
+ // Use PC-relative boot image String* address that will be known at link time.
+ // Used for boot image strings referenced by boot image code in PIC mode.
+ kBootImageLinkTimePcRelative,
+
+ // Use a known boot image String* address, embedded in the code by the codegen.
+ // Used for boot image strings referenced by apps in AOT- and JIT-compiled code.
+ // Note: codegen needs to emit a linker patch if indicated by compiler options'
+ // GetIncludePatchInformation().
+ kBootImageAddress,
+
+ // Load from the resolved strings array at an absolute address.
+ // Used for strings outside the boot image referenced by JIT-compiled code.
+ kDexCacheAddress,
+
+ // Load from resolved strings array in the dex cache using a PC-relative load.
+ // Used for strings outside boot image when we know that we can access
+ // the dex cache arrays using a PC-relative load.
+ kDexCachePcRelative,
+
+ // Load from resolved strings array accessed through the class loaded from
+ // the compiled method's own ArtMethod*. This is the default access type when
+ // all other types are unavailable.
+ kDexCacheViaMethod,
+
+ kLast = kDexCacheViaMethod
+ };
+
HLoadString(HCurrentMethod* current_method,
uint32_t string_index,
- uint32_t dex_pc,
- bool is_in_dex_cache)
+ const DexFile& dex_file,
+ uint32_t dex_pc)
: HExpression(Primitive::kPrimNot, SideEffectsForArchRuntimeCalls(), dex_pc),
string_index_(string_index) {
- SetPackedFlag<kFlagIsInDexCache>(is_in_dex_cache);
+ SetPackedFlag<kFlagIsInDexCache>(false);
+ SetPackedField<LoadKindField>(LoadKind::kDexCacheViaMethod);
+ load_data_.ref.dex_file = &dex_file;
SetRawInputAt(0, current_method);
}
+ void SetLoadKindWithAddress(LoadKind load_kind, uint64_t address) {
+ DCHECK(HasAddress(load_kind));
+ load_data_.address = address;
+ SetLoadKindInternal(load_kind);
+ }
+
+ void SetLoadKindWithStringReference(LoadKind load_kind,
+ const DexFile& dex_file,
+ uint32_t string_index) {
+ DCHECK(HasStringReference(load_kind));
+ load_data_.ref.dex_file = &dex_file;
+ string_index_ = string_index;
+ SetLoadKindInternal(load_kind);
+ }
+
+ void SetLoadKindWithDexCacheReference(LoadKind load_kind,
+ const DexFile& dex_file,
+ uint32_t element_index) {
+ DCHECK(HasDexCacheReference(load_kind));
+ load_data_.ref.dex_file = &dex_file;
+ load_data_.ref.dex_cache_element_index = element_index;
+ SetLoadKindInternal(load_kind);
+ }
+
+ LoadKind GetLoadKind() const {
+ return GetPackedField<LoadKindField>();
+ }
+
+ const DexFile& GetDexFile() const;
+
+ uint32_t GetStringIndex() const {
+ DCHECK(HasStringReference(GetLoadKind()) || /* For slow paths. */ !IsInDexCache());
+ return string_index_;
+ }
+
+ uint32_t GetDexCacheElementOffset() const;
+
+ uint64_t GetAddress() const {
+ DCHECK(HasAddress(GetLoadKind()));
+ return load_data_.address;
+ }
+
bool CanBeMoved() const OVERRIDE { return true; }
- bool InstructionDataEquals(HInstruction* other) const OVERRIDE {
- return other->AsLoadString()->string_index_ == string_index_;
- }
+ bool InstructionDataEquals(HInstruction* other) const OVERRIDE;
size_t ComputeHashCode() const OVERRIDE { return string_index_; }
- uint32_t GetStringIndex() const { return string_index_; }
+ // Will call the runtime if we need to load the string through
+ // the dex cache and the string is not guaranteed to be there yet.
+ bool NeedsEnvironment() const OVERRIDE {
+ LoadKind load_kind = GetLoadKind();
+ if (load_kind == LoadKind::kBootImageLinkTimeAddress ||
+ load_kind == LoadKind::kBootImageLinkTimePcRelative ||
+ load_kind == LoadKind::kBootImageAddress) {
+ return false;
+ }
+ return !IsInDexCache();
+ }
- // Will call the runtime if the string is not already in the dex cache.
- bool NeedsEnvironment() const OVERRIDE { return !IsInDexCache(); }
+ bool NeedsDexCacheOfDeclaringClass() const OVERRIDE {
+ return GetLoadKind() == LoadKind::kDexCacheViaMethod;
+ }
- bool NeedsDexCacheOfDeclaringClass() const OVERRIDE { return true; }
bool CanBeNull() const OVERRIDE { return false; }
- bool CanThrow() const OVERRIDE { return !IsInDexCache(); }
+ bool CanThrow() const OVERRIDE { return NeedsEnvironment(); }
static SideEffects SideEffectsForArchRuntimeCalls() {
return SideEffects::CanTriggerGC();
@@ -5590,17 +5679,83 @@
bool IsInDexCache() const { return GetPackedFlag<kFlagIsInDexCache>(); }
+ void MarkInDexCache() {
+ SetPackedFlag<kFlagIsInDexCache>(true);
+ DCHECK(!NeedsEnvironment());
+ RemoveEnvironment();
+ }
+
+ size_t InputCount() const OVERRIDE {
+ return (InputAt(0) != nullptr) ? 1u : 0u;
+ }
+
+ void AddSpecialInput(HInstruction* special_input);
+
DECLARE_INSTRUCTION(LoadString);
private:
static constexpr size_t kFlagIsInDexCache = kNumberOfExpressionPackedBits;
- static constexpr size_t kNumberOfLoadStringPackedBits = kFlagIsInDexCache + 1;
+ static constexpr size_t kFieldLoadKind = kFlagIsInDexCache + 1;
+ static constexpr size_t kFieldLoadKindSize =
+ MinimumBitsToStore(static_cast<size_t>(LoadKind::kLast));
+ static constexpr size_t kNumberOfLoadStringPackedBits = kFieldLoadKind + kFieldLoadKindSize;
static_assert(kNumberOfLoadStringPackedBits <= kMaxNumberOfPackedBits, "Too many packed fields.");
+ using LoadKindField = BitField<LoadKind, kFieldLoadKind, kFieldLoadKindSize>;
- const uint32_t string_index_;
+ static bool HasStringReference(LoadKind load_kind) {
+ return load_kind == LoadKind::kBootImageLinkTimeAddress ||
+ load_kind == LoadKind::kBootImageLinkTimePcRelative ||
+ load_kind == LoadKind::kDexCacheViaMethod;
+ }
+
+ static bool HasAddress(LoadKind load_kind) {
+ return load_kind == LoadKind::kBootImageAddress || load_kind == LoadKind::kDexCacheAddress;
+ }
+
+ static bool HasDexCacheReference(LoadKind load_kind) {
+ return load_kind == LoadKind::kDexCachePcRelative;
+ }
+
+ void SetLoadKindInternal(LoadKind load_kind);
+
+ // String index serves also as the hash code and it's also needed for slow-paths,
+ // so it must not be overwritten with other load data.
+ uint32_t string_index_;
+
+ union {
+ struct {
+ const DexFile* dex_file; // For string reference and dex cache reference.
+ uint32_t dex_cache_element_index; // Only for dex cache reference.
+ } ref;
+ uint64_t address; // Up to 64-bit, needed for kDexCacheAddress on 64-bit targets.
+ } load_data_;
DISALLOW_COPY_AND_ASSIGN(HLoadString);
};
+std::ostream& operator<<(std::ostream& os, HLoadString::LoadKind rhs);
+
+// Note: defined outside class to see operator<<(., HLoadString::LoadKind).
+inline const DexFile& HLoadString::GetDexFile() const {
+ DCHECK(HasStringReference(GetLoadKind()) || HasDexCacheReference(GetLoadKind()))
+ << GetLoadKind();
+ return *load_data_.ref.dex_file;
+}
+
+// Note: defined outside class to see operator<<(., HLoadString::LoadKind).
+inline uint32_t HLoadString::GetDexCacheElementOffset() const {
+ DCHECK(HasDexCacheReference(GetLoadKind())) << GetLoadKind();
+ return load_data_.ref.dex_cache_element_index;
+}
+
+// Note: defined outside class to see operator<<(., HLoadString::LoadKind).
+inline void HLoadString::AddSpecialInput(HInstruction* special_input) {
+ // The special input is used for PC-relative loads on some architectures.
+ DCHECK(GetLoadKind() == LoadKind::kBootImageLinkTimePcRelative ||
+ GetLoadKind() == LoadKind::kDexCachePcRelative) << GetLoadKind();
+ DCHECK(InputAt(0) == nullptr);
+ SetRawInputAt(0u, special_input);
+ special_input->AddUseAt(this, 0);
+}
/**
* Performs an initialization check on its Class object input.
diff --git a/compiler/optimizing/pc_relative_fixups_x86.cc b/compiler/optimizing/pc_relative_fixups_x86.cc
index d281a9f..dafbd3d 100644
--- a/compiler/optimizing/pc_relative_fixups_x86.cc
+++ b/compiler/optimizing/pc_relative_fixups_x86.cc
@@ -80,6 +80,15 @@
HandleInvoke(invoke);
}
+ void VisitLoadString(HLoadString* load_string) OVERRIDE {
+ HLoadString::LoadKind load_kind = load_string->GetLoadKind();
+ if (load_kind == HLoadString::LoadKind::kBootImageLinkTimePcRelative ||
+ load_kind == HLoadString::LoadKind::kDexCachePcRelative) {
+ InitializePCRelativeBasePointer();
+ load_string->AddSpecialInput(base_);
+ }
+ }
+
void BinaryFP(HBinaryOperation* bin) {
HConstant* rhs = bin->InputAt(1)->AsConstant();
if (rhs != nullptr && Primitive::IsFloatingPointType(rhs->GetType())) {
diff --git a/compiler/optimizing/sharpening.cc b/compiler/optimizing/sharpening.cc
index 5e1d1d9..7a1bb31 100644
--- a/compiler/optimizing/sharpening.cc
+++ b/compiler/optimizing/sharpening.cc
@@ -16,11 +16,20 @@
#include "sharpening.h"
+#include "base/casts.h"
+#include "class_linker.h"
#include "code_generator.h"
+#include "driver/dex_compilation_unit.h"
#include "utils/dex_cache_arrays_layout-inl.h"
#include "driver/compiler_driver.h"
+#include "gc/heap.h"
+#include "gc/space/image_space.h"
+#include "handle_scope-inl.h"
+#include "mirror/dex_cache.h"
+#include "mirror/string.h"
#include "nodes.h"
#include "runtime.h"
+#include "scoped_thread_state_change.h"
namespace art {
@@ -31,12 +40,13 @@
HInstruction* instruction = it.Current();
if (instruction->IsInvokeStaticOrDirect()) {
ProcessInvokeStaticOrDirect(instruction->AsInvokeStaticOrDirect());
+ } else if (instruction->IsLoadString()) {
+ ProcessLoadString(instruction->AsLoadString());
}
// TODO: Move the sharpening of invoke-virtual/-interface/-super from HGraphBuilder
// here. Rewrite it to avoid the CompilerDriver's reliance on verifier data
// because we know the type better when inlining.
- // TODO: HLoadClass, HLoadString - select PC relative dex cache array access if
- // available.
+ // TODO: HLoadClass - select better load kind if available.
}
}
}
@@ -143,4 +153,101 @@
invoke->SetDispatchInfo(dispatch_info);
}
+void HSharpening::ProcessLoadString(HLoadString* load_string) {
+ DCHECK_EQ(load_string->GetLoadKind(), HLoadString::LoadKind::kDexCacheViaMethod);
+ DCHECK(!load_string->IsInDexCache());
+
+ const DexFile& dex_file = load_string->GetDexFile();
+ uint32_t string_index = load_string->GetStringIndex();
+
+ bool is_in_dex_cache = false;
+ HLoadString::LoadKind desired_load_kind;
+ uint64_t address = 0u; // String or dex cache element address.
+ {
+ Runtime* runtime = Runtime::Current();
+ ClassLinker* class_linker = runtime->GetClassLinker();
+ ScopedObjectAccess soa(Thread::Current());
+ StackHandleScope<1> hs(soa.Self());
+ Handle<mirror::DexCache> dex_cache = IsSameDexFile(dex_file, *compilation_unit_.GetDexFile())
+ ? compilation_unit_.GetDexCache()
+ : hs.NewHandle(class_linker->FindDexCache(soa.Self(), dex_file));
+
+ if (compiler_driver_->IsBootImage()) {
+ // Compiling boot image. Resolve the string and allocate it if needed.
+ DCHECK(!runtime->UseJit());
+ mirror::String* string = class_linker->ResolveString(dex_file, string_index, dex_cache);
+ CHECK(string != nullptr);
+ if (!compiler_driver_->GetSupportBootImageFixup()) {
+ // MIPS/MIPS64 or compiler_driver_test. Do not sharpen.
+ desired_load_kind = HLoadString::LoadKind::kDexCacheViaMethod;
+ } else {
+ DCHECK(ContainsElement(compiler_driver_->GetDexFilesForOatFile(), &dex_file));
+ is_in_dex_cache = true;
+ desired_load_kind = codegen_->GetCompilerOptions().GetCompilePic()
+ ? HLoadString::LoadKind::kBootImageLinkTimePcRelative
+ : HLoadString::LoadKind::kBootImageLinkTimeAddress;
+ }
+ } else if (runtime->UseJit()) {
+ // TODO: Make sure we don't set the "compile PIC" flag for JIT as that's bogus.
+ // DCHECK(!codegen_->GetCompilerOptions().GetCompilePic());
+ mirror::String* string = dex_cache->GetResolvedString(string_index);
+ is_in_dex_cache = (string != nullptr);
+ if (string != nullptr && runtime->GetHeap()->ObjectIsInBootImageSpace(string)) {
+ desired_load_kind = HLoadString::LoadKind::kBootImageAddress;
+ address = reinterpret_cast64<uint64_t>(string);
+ } else {
+ // Note: If the string is not in the dex cache, the instruction needs environment
+ // and will not be inlined across dex files. Within a dex file, the slow-path helper
+ // loads the correct string and inlined frames are used correctly for OOM stack trace.
+ // TODO: Write a test for this.
+ desired_load_kind = HLoadString::LoadKind::kDexCacheAddress;
+ void* dex_cache_element_address = &dex_cache->GetStrings()[string_index];
+ address = reinterpret_cast64<uint64_t>(dex_cache_element_address);
+ }
+ } else {
+ // AOT app compilation. Try to lookup the string without allocating if not found.
+ mirror::String* string = class_linker->LookupString(dex_file, string_index, dex_cache);
+ if (string != nullptr && runtime->GetHeap()->ObjectIsInBootImageSpace(string)) {
+ if (codegen_->GetCompilerOptions().GetCompilePic()) {
+ // Use PC-relative load from the dex cache if the dex file belongs
+ // to the oat file that we're currently compiling.
+ desired_load_kind = ContainsElement(compiler_driver_->GetDexFilesForOatFile(), &dex_file)
+ ? HLoadString::LoadKind::kDexCachePcRelative
+ : HLoadString::LoadKind::kDexCacheViaMethod;
+ } else {
+ desired_load_kind = HLoadString::LoadKind::kBootImageAddress;
+ address = reinterpret_cast64<uint64_t>(string);
+ }
+ } else {
+ // Not JIT and the string is not in boot image.
+ desired_load_kind = HLoadString::LoadKind::kDexCachePcRelative;
+ }
+ }
+ }
+ if (is_in_dex_cache) {
+ load_string->MarkInDexCache();
+ }
+
+ HLoadString::LoadKind load_kind = codegen_->GetSupportedLoadStringKind(desired_load_kind);
+ switch (load_kind) {
+ case HLoadString::LoadKind::kBootImageLinkTimeAddress:
+ case HLoadString::LoadKind::kBootImageLinkTimePcRelative:
+ case HLoadString::LoadKind::kDexCacheViaMethod:
+ load_string->SetLoadKindWithStringReference(load_kind, dex_file, string_index);
+ break;
+ case HLoadString::LoadKind::kBootImageAddress:
+ case HLoadString::LoadKind::kDexCacheAddress:
+ DCHECK_NE(address, 0u);
+ load_string->SetLoadKindWithAddress(load_kind, address);
+ break;
+ case HLoadString::LoadKind::kDexCachePcRelative: {
+ size_t pointer_size = InstructionSetPointerSize(codegen_->GetInstructionSet());
+ DexCacheArraysLayout layout(pointer_size, &dex_file);
+ size_t element_index = layout.StringOffset(string_index);
+ load_string->SetLoadKindWithDexCacheReference(load_kind, dex_file, element_index);
+ break;
+ }
+ }
+}
+
} // namespace art
diff --git a/compiler/optimizing/sharpening.h b/compiler/optimizing/sharpening.h
index adae700..24152f6 100644
--- a/compiler/optimizing/sharpening.h
+++ b/compiler/optimizing/sharpening.h
@@ -47,6 +47,7 @@
private:
void ProcessInvokeStaticOrDirect(HInvokeStaticOrDirect* invoke);
+ void ProcessLoadString(HLoadString* load_string);
CodeGenerator* codegen_;
const DexCompilationUnit& compilation_unit_;
diff --git a/compiler/utils/string_reference.h b/compiler/utils/string_reference.h
new file mode 100644
index 0000000..72552f2
--- /dev/null
+++ b/compiler/utils/string_reference.h
@@ -0,0 +1,64 @@
+/*
+ * Copyright (C) 2016 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef ART_COMPILER_UTILS_STRING_REFERENCE_H_
+#define ART_COMPILER_UTILS_STRING_REFERENCE_H_
+
+#include <stdint.h>
+
+#include "base/logging.h"
+#include "utf-inl.h"
+
+namespace art {
+
+class DexFile;
+
+// A string is uniquely located by its DexFile and the string_ids_ table index into that DexFile.
+struct StringReference {
+ StringReference(const DexFile* file, uint32_t index) : dex_file(file), string_index(index) { }
+
+ const DexFile* dex_file;
+ uint32_t string_index;
+};
+
+// Compare the actual referenced string values. Used for string reference deduplication.
+struct StringReferenceValueComparator {
+ bool operator()(StringReference sr1, StringReference sr2) const {
+ // Note that we want to deduplicate identical strings even if they are referenced
+ // by different dex files, so we need some (any) total ordering of strings, rather
+ // than references. However, the references should usually be from the same dex file,
+ // so we choose the dex file string ordering so that we can simply compare indexes
+ // and avoid the costly string comparison in the most common case.
+ if (sr1.dex_file == sr2.dex_file) {
+ // Use the string order enforced by the dex file verifier.
+ DCHECK_EQ(
+ sr1.string_index < sr2.string_index,
+ CompareModifiedUtf8ToModifiedUtf8AsUtf16CodePointValues(
+ sr1.dex_file->GetStringData(sr1.dex_file->GetStringId(sr1.string_index)),
+ sr1.dex_file->GetStringData(sr2.dex_file->GetStringId(sr2.string_index))) < 0);
+ return sr1.string_index < sr2.string_index;
+ } else {
+ // Cannot compare indexes, so do the string comparison.
+ return CompareModifiedUtf8ToModifiedUtf8AsUtf16CodePointValues(
+ sr1.dex_file->GetStringData(sr1.dex_file->GetStringId(sr1.string_index)),
+ sr1.dex_file->GetStringData(sr2.dex_file->GetStringId(sr2.string_index))) < 0;
+ }
+ }
+};
+
+} // namespace art
+
+#endif // ART_COMPILER_UTILS_STRING_REFERENCE_H_
diff --git a/compiler/utils/test_dex_file_builder.h b/compiler/utils/test_dex_file_builder.h
index 2958dc6..fb37804 100644
--- a/compiler/utils/test_dex_file_builder.h
+++ b/compiler/utils/test_dex_file_builder.h
@@ -88,7 +88,7 @@
std::memset(header_data.data, 0, sizeof(header_data.data));
DexFile::Header* header = reinterpret_cast<DexFile::Header*>(&header_data.data);
std::copy_n(DexFile::kDexMagic, 4u, header->magic_);
- std::copy_n(DexFile::kDexMagicVersion, 4u, header->magic_ + 4u);
+ std::copy_n(DexFile::kDexMagicVersions[0], 4u, header->magic_ + 4u);
header->header_size_ = sizeof(DexFile::Header);
header->endian_tag_ = DexFile::kDexEndianConstant;
header->link_size_ = 0u; // Unused.
diff --git a/dex2oat/dex2oat.cc b/dex2oat/dex2oat.cc
index ec6f96f..0889098 100644
--- a/dex2oat/dex2oat.cc
+++ b/dex2oat/dex2oat.cc
@@ -1030,6 +1030,9 @@
compiler_options_->GetNativeDebuggable() ? OatHeader::kTrueValue : OatHeader::kFalseValue);
key_value_store_->Put(OatHeader::kCompilerFilter,
CompilerFilter::NameOfFilter(compiler_options_->GetCompilerFilter()));
+ key_value_store_->Put(OatHeader::kHasPatchInfoKey,
+ compiler_options_->GetIncludePatchInformation() ? OatHeader::kTrueValue
+ : OatHeader::kFalseValue);
}
// Parse the arguments from the command line. In case of an unrecognized option or impossible
diff --git a/oatdump/oatdump.cc b/oatdump/oatdump.cc
index a280c64..1150ce9 100644
--- a/oatdump/oatdump.cc
+++ b/oatdump/oatdump.cc
@@ -61,7 +61,6 @@
#include "stack_map.h"
#include "ScopedLocalRef.h"
#include "thread_list.h"
-#include "verifier/dex_gc_map.h"
#include "verifier/method_verifier.h"
#include "vmap_table.h"
#include "well_known_classes.h"
diff --git a/oatdump/oatdump_test.cc b/oatdump/oatdump_test.cc
index 3e420ad..bf062ed 100644
--- a/oatdump/oatdump_test.cc
+++ b/oatdump/oatdump_test.cc
@@ -69,6 +69,7 @@
exec_argv.push_back("--output=" + core_oat_location_ + ".symbolize");
} else if (mode == kModeArt) {
exec_argv.push_back("--image=" + core_art_location_);
+ exec_argv.push_back("--instruction-set=" + std::string(GetInstructionSetString(kRuntimeISA)));
exec_argv.push_back("--output=/dev/null");
} else {
CHECK_EQ(static_cast<size_t>(mode), static_cast<size_t>(kModeOat));
@@ -84,8 +85,8 @@
std::string core_oat_location_;
};
-// Disable tests on arm as they are taking too long to run for hammerhead. b/27824283.
-#ifndef __arm__
+// Disable tests on arm and mips as they are taking too long to run. b/27824283.
+#if !defined(__arm__) && !defined(__mips__)
TEST_F(OatDumpTest, TestImage) {
std::string error_msg;
ASSERT_TRUE(Exec(kModeArt, {}, &error_msg)) << error_msg;
diff --git a/patchoat/patchoat.cc b/patchoat/patchoat.cc
index 63ae342..a1b3c9e 100644
--- a/patchoat/patchoat.cc
+++ b/patchoat/patchoat.cc
@@ -320,11 +320,11 @@
PatchOat& p = space_to_patchoat_map.find(space)->second;
- if (!p.WriteImage(output_image_file.get())) {
- LOG(ERROR) << "Failed to write image file " << output_image_file->GetPath();
+ bool success = p.WriteImage(output_image_file.get());
+ success = FinishFile(output_image_file.get(), success);
+ if (!success) {
return false;
}
- FinishFile(output_image_file.get(), true);
bool skip_patching_oat = space_to_skip_patching_map.find(space)->second;
if (!skip_patching_oat) {
@@ -336,11 +336,11 @@
LOG(ERROR) << "Failed to open output oat file at " << output_oat_filename;
return false;
}
- if (!p.WriteElf(output_oat_file.get())) {
- LOG(ERROR) << "Failed to write oat file " << output_oat_file->GetPath();
+ success = p.WriteElf(output_oat_file.get());
+ success = FinishFile(output_oat_file.get(), success);
+ if (!success) {
return false;
}
- FinishFile(output_oat_file.get(), true);
}
}
return true;
diff --git a/runtime/Android.mk b/runtime/Android.mk
index 0c6541e..542a2c4 100644
--- a/runtime/Android.mk
+++ b/runtime/Android.mk
@@ -187,7 +187,6 @@
type_lookup_table.cc \
utf.cc \
utils.cc \
- verifier/dex_gc_map.cc \
verifier/instruction_flags.cc \
verifier/method_verifier.cc \
verifier/reg_type.cc \
diff --git a/runtime/art_method.h b/runtime/art_method.h
index 3dbcd58..d1ef019 100644
--- a/runtime/art_method.h
+++ b/runtime/art_method.h
@@ -545,6 +545,9 @@
ALWAYS_INLINE GcRoot<mirror::Class>* GetDexCacheResolvedTypes(size_t pointer_size)
SHARED_REQUIRES(Locks::mutator_lock_);
+ // Note, hotness_counter_ updates are non-atomic but it doesn't need to be precise. Also,
+ // given that the counter is only 16 bits wide we can expect wrap-around in some
+ // situations. Consumers of hotness_count_ must be able to deal with that.
uint16_t IncrementCounter() {
return ++hotness_count_;
}
@@ -553,6 +556,14 @@
hotness_count_ = 0;
}
+ void SetCounter(int16_t hotness_count) {
+ hotness_count_ = hotness_count;
+ }
+
+ uint16_t GetCounter() const {
+ return hotness_count_;
+ }
+
const uint8_t* GetQuickenedInfo() SHARED_REQUIRES(Locks::mutator_lock_);
// Returns the method header for the compiled code containing 'pc'. Note that runtime
@@ -597,7 +608,7 @@
// ifTable.
uint16_t method_index_;
- // The hotness we measure for this method. Incremented by the interpreter. Not atomic, as we allow
+ // The hotness we measure for this method. Managed by the interpreter. Not atomic, as we allow
// missing increments: if the method is hot, we will see it eventually.
uint16_t hotness_count_;
diff --git a/runtime/asm_support.h b/runtime/asm_support.h
index 942f9de..d27d2f6 100644
--- a/runtime/asm_support.h
+++ b/runtime/asm_support.h
@@ -20,6 +20,7 @@
#if defined(__cplusplus)
#include "art_method.h"
#include "gc/allocator/rosalloc.h"
+#include "jit/jit_instrumentation.h"
#include "lock_word.h"
#include "mirror/class.h"
#include "mirror/string.h"
@@ -188,7 +189,13 @@
#define SHADOWFRAME_DEX_PC_OFFSET (SHADOWFRAME_NUMBER_OF_VREGS_OFFSET + 4)
ADD_TEST_EQ(SHADOWFRAME_DEX_PC_OFFSET,
static_cast<int32_t>(art::ShadowFrame::DexPCOffset()))
-#define SHADOWFRAME_VREGS_OFFSET (SHADOWFRAME_NUMBER_OF_VREGS_OFFSET + 8)
+#define SHADOWFRAME_CACHED_HOTNESS_COUNTDOWN_OFFSET (SHADOWFRAME_NUMBER_OF_VREGS_OFFSET + 8)
+ADD_TEST_EQ(SHADOWFRAME_CACHED_HOTNESS_COUNTDOWN_OFFSET,
+ static_cast<int32_t>(art::ShadowFrame::CachedHotnessCountdownOffset()))
+#define SHADOWFRAME_HOTNESS_COUNTDOWN_OFFSET (SHADOWFRAME_NUMBER_OF_VREGS_OFFSET + 10)
+ADD_TEST_EQ(SHADOWFRAME_HOTNESS_COUNTDOWN_OFFSET,
+ static_cast<int32_t>(art::ShadowFrame::HotnessCountdownOffset()))
+#define SHADOWFRAME_VREGS_OFFSET (SHADOWFRAME_NUMBER_OF_VREGS_OFFSET + 12)
ADD_TEST_EQ(SHADOWFRAME_VREGS_OFFSET,
static_cast<int32_t>(art::ShadowFrame::VRegsOffset()))
@@ -389,6 +396,12 @@
#define THREAD_CHECKPOINT_REQUEST 2
ADD_TEST_EQ(THREAD_CHECKPOINT_REQUEST, static_cast<int32_t>(art::kCheckpointRequest))
+#define JIT_CHECK_OSR -1
+ADD_TEST_EQ(JIT_CHECK_OSR, static_cast<int32_t>(art::jit::kJitCheckForOSR))
+
+#define JIT_HOTNESS_DISABLE -2
+ADD_TEST_EQ(JIT_HOTNESS_DISABLE, static_cast<int32_t>(art::jit::kJitHotnessDisabled))
+
#if defined(__cplusplus)
} // End of CheckAsmSupportOffsets.
#endif
diff --git a/runtime/base/casts.h b/runtime/base/casts.h
index f884649..6b67864 100644
--- a/runtime/base/casts.h
+++ b/runtime/base/casts.h
@@ -19,6 +19,7 @@
#include <assert.h>
#include <limits>
+#include <stdint.h>
#include <string.h>
#include <type_traits>
@@ -34,7 +35,7 @@
// When you use implicit_cast, the compiler checks that the cast is safe.
// Such explicit implicit_casts are necessary in surprisingly many
// situations where C++ demands an exact type match instead of an
-// argument type convertable to a target type.
+// argument type convertible to a target type.
//
// The From type can be inferred, so the preferred syntax for using
// implicit_cast is the same as for static_cast etc.:
@@ -102,6 +103,29 @@
return static_cast<Dest>(source);
}
+// A version of reinterpret_cast<>() between pointers and int64_t/uint64_t
+// that goes through uintptr_t to avoid treating the pointer as "signed."
+
+template <typename Dest, typename Source>
+inline Dest reinterpret_cast64(Source source) {
+ // This is the overload for casting from int64_t/uint64_t to a pointer.
+ static_assert(std::is_same<Source, int64_t>::value || std::is_same<Source, uint64_t>::value,
+ "Source must be int64_t or uint64_t.");
+ static_assert(std::is_pointer<Dest>::value, "Dest must be a pointer.");
+ // Check that we don't lose any non-0 bits here.
+ DCHECK_EQ(static_cast<Source>(static_cast<uintptr_t>(source)), source);
+ return reinterpret_cast<Dest>(static_cast<uintptr_t>(source));
+}
+
+template <typename Dest, typename Source>
+inline Dest reinterpret_cast64(Source* ptr) {
+ // This is the overload for casting from a pointer to int64_t/uint64_t.
+ static_assert(std::is_same<Dest, int64_t>::value || std::is_same<Dest, uint64_t>::value,
+ "Dest must be int64_t or uint64_t.");
+ static_assert(sizeof(uintptr_t) <= sizeof(Dest), "Expecting at most 64-bit pointers.");
+ return static_cast<Dest>(reinterpret_cast<uintptr_t>(ptr));
+}
+
} // namespace art
#endif // ART_RUNTIME_BASE_CASTS_H_
diff --git a/runtime/base/logging.cc b/runtime/base/logging.cc
index 7a620e3..212e5bd 100644
--- a/runtime/base/logging.cc
+++ b/runtime/base/logging.cc
@@ -185,14 +185,15 @@
LogMessage::LogMessage(const char* file, unsigned int line, LogSeverity severity, int error)
: data_(new LogMessageData(file, line, severity, error)) {
if (PrintDirectly(severity)) {
- static const char* log_characters = "VDIWEFF";
- CHECK_EQ(strlen(log_characters), INTERNAL_FATAL + 1U);
- stream() << ProgramInvocationShortName() << " " << log_characters[static_cast<size_t>(severity)]
+ static constexpr char kLogCharacters[] = { 'N', 'V', 'D', 'I', 'W', 'E', 'F', 'F' };
+ static_assert(arraysize(kLogCharacters) == static_cast<size_t>(INTERNAL_FATAL) + 1,
+ "Wrong character array size");
+ stream() << ProgramInvocationShortName() << " " << kLogCharacters[static_cast<size_t>(severity)]
<< " " << getpid() << " " << ::art::GetTid() << " " << file << ":" << line << "]";
}
}
LogMessage::~LogMessage() {
- if (!PrintDirectly(data_->GetSeverity())) {
+ if (!PrintDirectly(data_->GetSeverity()) && data_->GetSeverity() != LogSeverity::NONE) {
if (data_->GetSeverity() < gMinimumLogSeverity) {
return; // No need to format something we're not going to output.
}
@@ -236,6 +237,7 @@
#ifdef __ANDROID__
static const android_LogPriority kLogSeverityToAndroidLogPriority[] = {
+ ANDROID_LOG_VERBOSE, // NONE, use verbose as stand-in, will never be printed.
ANDROID_LOG_VERBOSE, ANDROID_LOG_DEBUG, ANDROID_LOG_INFO, ANDROID_LOG_WARN,
ANDROID_LOG_ERROR, ANDROID_LOG_FATAL, ANDROID_LOG_FATAL
};
@@ -245,16 +247,20 @@
void LogMessage::LogLine(const char* file, unsigned int line, LogSeverity log_severity,
const char* message) {
+ if (log_severity == LogSeverity::NONE) {
+ return;
+ }
+
#ifdef __ANDROID__
const char* tag = ProgramInvocationShortName();
- int priority = kLogSeverityToAndroidLogPriority[log_severity];
+ int priority = kLogSeverityToAndroidLogPriority[static_cast<size_t>(log_severity)];
if (priority == ANDROID_LOG_FATAL) {
LOG_PRI(priority, tag, "%s:%u] %s", file, line, message);
} else {
LOG_PRI(priority, tag, "%s", message);
}
#else
- static const char* log_characters = "VDIWEFF";
+ static const char* log_characters = "NVDIWEFF";
CHECK_EQ(strlen(log_characters), INTERNAL_FATAL + 1U);
char severity = log_characters[log_severity];
fprintf(stderr, "%s %c %5d %5d %s:%u] %s\n",
@@ -264,10 +270,14 @@
void LogMessage::LogLineLowStack(const char* file, unsigned int line, LogSeverity log_severity,
const char* message) {
+ if (log_severity == LogSeverity::NONE) {
+ return;
+ }
+
#ifdef __ANDROID__
// Use android_writeLog() to avoid stack-based buffers used by android_printLog().
const char* tag = ProgramInvocationShortName();
- int priority = kLogSeverityToAndroidLogPriority[log_severity];
+ int priority = kLogSeverityToAndroidLogPriority[static_cast<size_t>(log_severity)];
char* buf = nullptr;
size_t buf_size = 0u;
if (priority == ANDROID_LOG_FATAL) {
@@ -285,13 +295,14 @@
android_writeLog(priority, tag, message);
}
#else
- static const char* log_characters = "VDIWEFF";
- CHECK_EQ(strlen(log_characters), INTERNAL_FATAL + 1U);
+ static constexpr char kLogCharacters[] = { 'N', 'V', 'D', 'I', 'W', 'E', 'F', 'F' };
+ static_assert(arraysize(kLogCharacters) == static_cast<size_t>(INTERNAL_FATAL) + 1,
+ "Wrong character array size");
const char* program_name = ProgramInvocationShortName();
TEMP_FAILURE_RETRY(write(STDERR_FILENO, program_name, strlen(program_name)));
TEMP_FAILURE_RETRY(write(STDERR_FILENO, " ", 1));
- TEMP_FAILURE_RETRY(write(STDERR_FILENO, &log_characters[log_severity], 1));
+ TEMP_FAILURE_RETRY(write(STDERR_FILENO, &kLogCharacters[static_cast<size_t>(log_severity)], 1));
TEMP_FAILURE_RETRY(write(STDERR_FILENO, " ", 1));
// TODO: pid and tid.
TEMP_FAILURE_RETRY(write(STDERR_FILENO, file, strlen(file)));
diff --git a/runtime/base/logging.h b/runtime/base/logging.h
index 8aaeaac..97280c3 100644
--- a/runtime/base/logging.h
+++ b/runtime/base/logging.h
@@ -24,6 +24,7 @@
namespace art {
enum LogSeverity {
+ NONE, // Fake level, don't log at all.
VERBOSE,
DEBUG,
INFO,
diff --git a/runtime/base/time_utils.cc b/runtime/base/time_utils.cc
index b7cf207..3e5bac8 100644
--- a/runtime/base/time_utils.cc
+++ b/runtime/base/time_utils.cc
@@ -15,6 +15,7 @@
*/
#include <inttypes.h>
+#include <limits>
#include <sstream>
#include "time_utils.h"
@@ -190,9 +191,16 @@
}
int64_t end_sec = ts->tv_sec + ms / 1000;
- if (UNLIKELY(end_sec >= 0x7fffffff)) {
- LOG(INFO) << "Note: end time exceeds INT32_MAX: " << end_sec;
- end_sec = 0x7ffffffe;
+ constexpr int32_t int32_max = std::numeric_limits<int32_t>::max();
+ if (UNLIKELY(end_sec >= int32_max)) {
+ // Either ms was intended to denote an infinite timeout, or we have a
+ // problem. The former generally uses the largest possible millisecond
+ // or nanosecond value. Log only in the latter case.
+ constexpr int64_t int64_max = std::numeric_limits<int64_t>::max();
+ if (ms != int64_max && ms != int64_max / (1000 * 1000)) {
+ LOG(INFO) << "Note: end time exceeds INT32_MAX: " << end_sec;
+ }
+ end_sec = int32_max - 1; // Allow for increment below.
}
ts->tv_sec = end_sec;
ts->tv_nsec = (ts->tv_nsec + (ms % 1000) * 1000000) + ns;
diff --git a/runtime/class_linker.cc b/runtime/class_linker.cc
index 99e38d9..7060593 100644
--- a/runtime/class_linker.cc
+++ b/runtime/class_linker.cc
@@ -50,7 +50,7 @@
#include "experimental_flags.h"
#include "gc_root-inl.h"
#include "gc/accounting/card_table-inl.h"
-#include "gc/accounting/heap_bitmap.h"
+#include "gc/accounting/heap_bitmap-inl.h"
#include "gc/heap.h"
#include "gc/space/image_space.h"
#include "handle_scope-inl.h"
@@ -96,6 +96,7 @@
namespace art {
static constexpr bool kSanityCheckObjects = kIsDebugBuild;
+static constexpr bool kVerifyArtMethodDeclaringClasses = kIsDebugBuild;
static void ThrowNoClassDefFoundError(const char* fmt, ...)
__attribute__((__format__(__printf__, 1, 2)))
@@ -1197,6 +1198,23 @@
ClassTable* const table_;
};
+class VerifyDeclaringClassVisitor : public ArtMethodVisitor {
+ public:
+ VerifyDeclaringClassVisitor() SHARED_REQUIRES(Locks::mutator_lock_, Locks::heap_bitmap_lock_)
+ : live_bitmap_(Runtime::Current()->GetHeap()->GetLiveBitmap()) {}
+
+ virtual void Visit(ArtMethod* method)
+ SHARED_REQUIRES(Locks::mutator_lock_, Locks::heap_bitmap_lock_) {
+ mirror::Class* klass = method->GetDeclaringClassUnchecked();
+ if (klass != nullptr) {
+ CHECK(live_bitmap_->Test(klass)) << "Image method has unmarked declaring class";
+ }
+ }
+
+ private:
+ gc::accounting::HeapBitmap* const live_bitmap_;
+};
+
bool ClassLinker::UpdateAppImageClassLoadersAndDexCaches(
gc::space::ImageSpace* space,
Handle<mirror::ClassLoader> class_loader,
@@ -1416,15 +1434,24 @@
}
}
}
- if (*out_forward_dex_cache_array) {
- ScopedTrace timing("Fixup ArtMethod dex cache arrays");
- FixupArtMethodArrayVisitor visitor(header);
- header.GetImageSection(ImageHeader::kSectionArtMethods).VisitPackedArtMethods(
- &visitor,
- space->Begin(),
- sizeof(void*));
- Runtime::Current()->GetHeap()->WriteBarrierEveryFieldOf(class_loader.Get());
- }
+ }
+ if (*out_forward_dex_cache_array) {
+ ScopedTrace timing("Fixup ArtMethod dex cache arrays");
+ FixupArtMethodArrayVisitor visitor(header);
+ header.GetImageSection(ImageHeader::kSectionArtMethods).VisitPackedArtMethods(
+ &visitor,
+ space->Begin(),
+ sizeof(void*));
+ Runtime::Current()->GetHeap()->WriteBarrierEveryFieldOf(class_loader.Get());
+ }
+ if (kVerifyArtMethodDeclaringClasses) {
+ ScopedTrace timing("Verify declaring classes");
+ ReaderMutexLock rmu(self, *Locks::heap_bitmap_lock_);
+ VerifyDeclaringClassVisitor visitor;
+ header.GetImageSection(ImageHeader::kSectionArtMethods).VisitPackedArtMethods(
+ &visitor,
+ space->Begin(),
+ sizeof(void*));
}
return true;
}
@@ -3715,10 +3742,6 @@
DCHECK(klass.Get() != nullptr);
DCHECK(supertype.Get() != nullptr);
- StackHandleScope<1> hs(self);
- // Acquire lock to prevent races on verifying the super class.
- ObjectLock<mirror::Class> super_lock(self, supertype);
-
if (!supertype->IsVerified() && !supertype->IsErroneous()) {
VerifyClass(self, supertype);
}
@@ -3732,6 +3755,7 @@
PrettyDescriptor(klass.Get()).c_str(),
PrettyDescriptor(supertype.Get()).c_str());
LOG(WARNING) << error_msg << " in " << klass->GetDexCache()->GetLocation()->ToModifiedUtf8();
+ StackHandleScope<1> hs(self);
Handle<mirror::Throwable> cause(hs.NewHandle(self->GetException()));
if (cause.Get() != nullptr) {
// Set during VerifyClass call (if at all).
@@ -3746,44 +3770,57 @@
if (Runtime::Current()->IsAotCompiler()) {
Runtime::Current()->GetCompilerCallbacks()->ClassRejected(ref);
}
+ // Need to grab the lock to change status.
+ ObjectLock<mirror::Class> super_lock(self, klass);
mirror::Class::SetStatus(klass, mirror::Class::kStatusError, self);
return false;
}
-void ClassLinker::VerifyClass(Thread* self, Handle<mirror::Class> klass) {
- // TODO: assert that the monitor on the Class is held
- ObjectLock<mirror::Class> lock(self, klass);
+void ClassLinker::VerifyClass(Thread* self, Handle<mirror::Class> klass, LogSeverity log_level) {
+ {
+ // TODO: assert that the monitor on the Class is held
+ ObjectLock<mirror::Class> lock(self, klass);
- // Don't attempt to re-verify if already sufficiently verified.
- if (klass->IsVerified()) {
- EnsureSkipAccessChecksMethods(klass);
- return;
- }
- if (klass->IsCompileTimeVerified() && Runtime::Current()->IsAotCompiler()) {
- return;
- }
+ // Is somebody verifying this now?
+ mirror::Class::Status old_status = klass->GetStatus();
+ while (old_status == mirror::Class::kStatusVerifying ||
+ old_status == mirror::Class::kStatusVerifyingAtRuntime) {
+ lock.WaitIgnoringInterrupts();
+ CHECK_GT(klass->GetStatus(), old_status);
+ old_status = klass->GetStatus();
+ }
- // The class might already be erroneous, for example at compile time if we attempted to verify
- // this class as a parent to another.
- if (klass->IsErroneous()) {
- ThrowEarlierClassFailure(klass.Get());
- return;
- }
+ // The class might already be erroneous, for example at compile time if we attempted to verify
+ // this class as a parent to another.
+ if (klass->IsErroneous()) {
+ ThrowEarlierClassFailure(klass.Get());
+ return;
+ }
- if (klass->GetStatus() == mirror::Class::kStatusResolved) {
- mirror::Class::SetStatus(klass, mirror::Class::kStatusVerifying, self);
- } else {
- CHECK_EQ(klass->GetStatus(), mirror::Class::kStatusRetryVerificationAtRuntime)
- << PrettyClass(klass.Get());
- CHECK(!Runtime::Current()->IsAotCompiler());
- mirror::Class::SetStatus(klass, mirror::Class::kStatusVerifyingAtRuntime, self);
- }
+ // Don't attempt to re-verify if already sufficiently verified.
+ if (klass->IsVerified()) {
+ EnsureSkipAccessChecksMethods(klass);
+ return;
+ }
+ if (klass->IsCompileTimeVerified() && Runtime::Current()->IsAotCompiler()) {
+ return;
+ }
- // Skip verification if disabled.
- if (!Runtime::Current()->IsVerificationEnabled()) {
- mirror::Class::SetStatus(klass, mirror::Class::kStatusVerified, self);
- EnsureSkipAccessChecksMethods(klass);
- return;
+ if (klass->GetStatus() == mirror::Class::kStatusResolved) {
+ mirror::Class::SetStatus(klass, mirror::Class::kStatusVerifying, self);
+ } else {
+ CHECK_EQ(klass->GetStatus(), mirror::Class::kStatusRetryVerificationAtRuntime)
+ << PrettyClass(klass.Get());
+ CHECK(!Runtime::Current()->IsAotCompiler());
+ mirror::Class::SetStatus(klass, mirror::Class::kStatusVerifyingAtRuntime, self);
+ }
+
+ // Skip verification if disabled.
+ if (!Runtime::Current()->IsVerificationEnabled()) {
+ mirror::Class::SetStatus(klass, mirror::Class::kStatusVerified, self);
+ EnsureSkipAccessChecksMethods(klass);
+ return;
+ }
}
// Verify super class.
@@ -3853,9 +3890,13 @@
klass.Get(),
runtime->GetCompilerCallbacks(),
runtime->IsAotCompiler(),
- runtime->IsAotCompiler(),
+ log_level,
&error_msg);
}
+
+ // Verification is done, grab the lock again.
+ ObjectLock<mirror::Class> lock(self, klass);
+
if (preverified || verifier_failure != verifier::MethodVerifier::kHardFailure) {
if (!preverified && verifier_failure != verifier::MethodVerifier::kNoFailure) {
VLOG(class_linker) << "Soft verification failure in class " << PrettyDescriptor(klass.Get())
@@ -7024,6 +7065,23 @@
return string;
}
+mirror::String* ClassLinker::LookupString(const DexFile& dex_file,
+ uint32_t string_idx,
+ Handle<mirror::DexCache> dex_cache) {
+ DCHECK(dex_cache.Get() != nullptr);
+ mirror::String* resolved = dex_cache->GetResolvedString(string_idx);
+ if (resolved != nullptr) {
+ return resolved;
+ }
+ uint32_t utf16_length;
+ const char* utf8_data = dex_file.StringDataAndUtf16LengthByIdx(string_idx, &utf16_length);
+ mirror::String* string = intern_table_->LookupStrong(Thread::Current(), utf16_length, utf8_data);
+ if (string != nullptr) {
+ dex_cache->SetResolvedString(string_idx, string);
+ }
+ return string;
+}
+
mirror::Class* ClassLinker::ResolveType(const DexFile& dex_file,
uint16_t type_idx,
mirror::Class* referrer) {
diff --git a/runtime/class_linker.h b/runtime/class_linker.h
index b4fbe1c..b4b7f34 100644
--- a/runtime/class_linker.h
+++ b/runtime/class_linker.h
@@ -247,6 +247,12 @@
Handle<mirror::DexCache> dex_cache)
SHARED_REQUIRES(Locks::mutator_lock_);
+ // Find a String with the given index from the DexFile, storing the
+ // result in the DexCache if found. Return null if not found.
+ mirror::String* LookupString(const DexFile& dex_file, uint32_t string_idx,
+ Handle<mirror::DexCache> dex_cache)
+ SHARED_REQUIRES(Locks::mutator_lock_);
+
// Resolve a Type with the given index from the DexFile, storing the
// result in the DexCache. The referrer is used to identity the
// target DexCache and ClassLoader to use for resolution.
@@ -449,7 +455,9 @@
SHARED_REQUIRES(Locks::mutator_lock_)
REQUIRES(!Roles::uninterruptible_);
- void VerifyClass(Thread* self, Handle<mirror::Class> klass)
+ void VerifyClass(Thread* self,
+ Handle<mirror::Class> klass,
+ LogSeverity log_level = LogSeverity::NONE)
SHARED_REQUIRES(Locks::mutator_lock_)
REQUIRES(!dex_lock_);
bool VerifyClassUsingOatFile(const DexFile& dex_file,
diff --git a/runtime/compiler_filter.cc b/runtime/compiler_filter.cc
index 31a1bc1..d617caf 100644
--- a/runtime/compiler_filter.cc
+++ b/runtime/compiler_filter.cc
@@ -83,6 +83,34 @@
UNREACHABLE();
}
+CompilerFilter::Filter CompilerFilter::GetNonProfileDependentFilterFrom(Filter filter) {
+ switch (filter) {
+ case CompilerFilter::kVerifyNone:
+ case CompilerFilter::kVerifyAtRuntime:
+ case CompilerFilter::kInterpretOnly:
+ case CompilerFilter::kSpace:
+ case CompilerFilter::kBalanced:
+ case CompilerFilter::kTime:
+ case CompilerFilter::kSpeed:
+ case CompilerFilter::kEverything:
+ return filter;
+
+ case CompilerFilter::kVerifyProfile:
+ return CompilerFilter::kInterpretOnly;
+
+ case CompilerFilter::kSpaceProfile:
+ return CompilerFilter::kSpace;
+
+ case CompilerFilter::kSpeedProfile:
+ return CompilerFilter::kSpeed;
+
+ case CompilerFilter::kEverythingProfile:
+ return CompilerFilter::kEverything;
+ }
+ UNREACHABLE();
+}
+
+
bool CompilerFilter::IsAsGoodAs(Filter current, Filter target) {
return current >= target;
}
diff --git a/runtime/compiler_filter.h b/runtime/compiler_filter.h
index 1bea8b4..6289d8a 100644
--- a/runtime/compiler_filter.h
+++ b/runtime/compiler_filter.h
@@ -59,6 +59,9 @@
// profile.
static bool DependsOnProfile(Filter filter);
+ // Returns a non-profile-guided version of the given filter.
+ static Filter GetNonProfileDependentFilterFrom(Filter filter);
+
// Returns true if the 'current' compiler filter is considered at least as
// good as the 'target' compilation type.
// For example: kSpeed is as good as kInterpretOnly, but kInterpretOnly is
diff --git a/runtime/debugger.cc b/runtime/debugger.cc
index 109e03d..d832552 100644
--- a/runtime/debugger.cc
+++ b/runtime/debugger.cc
@@ -4818,7 +4818,7 @@
LOG(INFO) << "Tracked allocations, (count=" << count << ")";
for (auto it = records->RBegin(), end = records->REnd();
count > 0 && it != end; count--, it++) {
- const gc::AllocRecord* record = it->second;
+ const gc::AllocRecord* record = &it->second;
LOG(INFO) << StringPrintf(" Thread %-2d %6zd bytes ", record->GetTid(), record->ByteCount())
<< PrettyClass(record->GetClass());
@@ -4957,7 +4957,7 @@
uint16_t count = capped_count;
for (auto it = records->RBegin(), end = records->REnd();
count > 0 && it != end; count--, it++) {
- const gc::AllocRecord* record = it->second;
+ const gc::AllocRecord* record = &it->second;
std::string temp;
class_names.Add(record->GetClassDescriptor(&temp));
for (size_t i = 0, depth = record->GetDepth(); i < depth; i++) {
@@ -5008,7 +5008,7 @@
// (2b) thread id
// (2b) allocated object's class name index
// (1b) stack depth
- const gc::AllocRecord* record = it->second;
+ const gc::AllocRecord* record = &it->second;
size_t stack_depth = record->GetDepth();
size_t allocated_object_class_name_index =
class_names.IndexOf(record->GetClassDescriptor(&temp));
diff --git a/runtime/dex_file.cc b/runtime/dex_file.cc
index 4a0a6fc..e63eaa2 100644
--- a/runtime/dex_file.cc
+++ b/runtime/dex_file.cc
@@ -62,7 +62,12 @@
namespace art {
const uint8_t DexFile::kDexMagic[] = { 'd', 'e', 'x', '\n' };
-const uint8_t DexFile::kDexMagicVersion[] = { '0', '3', '5', '\0' };
+const uint8_t DexFile::kDexMagicVersions[DexFile::kNumDexVersions][DexFile::kDexVersionLen] = {
+ {'0', '3', '5', '\0'},
+ // Dex version 036 skipped because of an old dalvik bug on some versions of android where dex
+ // files with that version number would erroneously be accepted and run.
+ {'0', '3', '7', '\0'}
+};
bool DexFile::GetChecksum(const char* filename, uint32_t* checksum, std::string* error_msg) {
CHECK(checksum != nullptr);
@@ -493,11 +498,16 @@
bool DexFile::IsVersionValid(const uint8_t* magic) {
const uint8_t* version = &magic[sizeof(kDexMagic)];
- return (memcmp(version, kDexMagicVersion, sizeof(kDexMagicVersion)) == 0);
+ for (uint32_t i = 0; i < kNumDexVersions; i++) {
+ if (memcmp(version, kDexMagicVersions[i], kDexVersionLen) == 0) {
+ return true;
+ }
+ }
+ return false;
}
-uint32_t DexFile::GetVersion() const {
- const char* version = reinterpret_cast<const char*>(&GetHeader().magic_[sizeof(kDexMagic)]);
+uint32_t DexFile::Header::GetVersion() const {
+ const char* version = reinterpret_cast<const char*>(&magic_[sizeof(kDexMagic)]);
return atoi(version);
}
diff --git a/runtime/dex_file.h b/runtime/dex_file.h
index e497e9c..1456636 100644
--- a/runtime/dex_file.h
+++ b/runtime/dex_file.h
@@ -58,7 +58,10 @@
class DexFile {
public:
static const uint8_t kDexMagic[];
- static const uint8_t kDexMagicVersion[];
+ static constexpr size_t kNumDexVersions = 2;
+ static constexpr size_t kDexVersionLen = 4;
+ static const uint8_t kDexMagicVersions[kNumDexVersions][kDexVersionLen];
+
static constexpr size_t kSha1DigestSize = 20;
static constexpr uint32_t kDexEndianConstant = 0x12345678;
@@ -71,7 +74,7 @@
// The value of an invalid index.
static const uint16_t kDexNoIndex16 = 0xFFFF;
- // The separator charactor in MultiDex locations.
+ // The separator character in MultiDex locations.
static constexpr char kMultiDexSeparator = ':';
// A string version of the previous. This is a define so that we can merge string literals in the
@@ -104,6 +107,9 @@
uint32_t data_size_; // unused
uint32_t data_off_; // unused
+ // Decode the dex magic version
+ uint32_t GetVersion() const;
+
private:
DISALLOW_COPY_AND_ASSIGN(Header);
};
@@ -476,7 +482,9 @@
}
// Decode the dex magic version
- uint32_t GetVersion() const;
+ uint32_t GetVersion() const {
+ return GetHeader().GetVersion();
+ }
// Returns true if the byte string points to the magic value.
static bool IsMagicValid(const uint8_t* magic);
diff --git a/runtime/dex_file_verifier.cc b/runtime/dex_file_verifier.cc
index 9c9b8c5..811e763 100644
--- a/runtime/dex_file_verifier.cc
+++ b/runtime/dex_file_verifier.cc
@@ -2465,7 +2465,13 @@
GetFieldDescriptionOrError(begin_, header_, idx).c_str(),
field_access_flags,
PrettyJavaAccessFlags(field_access_flags).c_str());
- return false;
+ if (header_->GetVersion() >= 37) {
+ return false;
+ } else {
+ // Allow in older versions, but warn.
+ LOG(WARNING) << "This dex file is invalid and will be rejected in the future. Error is: "
+ << *error_msg;
+ }
}
// Interface fields may be synthetic, but may not have other flags.
constexpr uint32_t kDisallowed = ~(kPublicFinalStatic | kAccSynthetic);
@@ -2474,7 +2480,13 @@
GetFieldDescriptionOrError(begin_, header_, idx).c_str(),
field_access_flags,
PrettyJavaAccessFlags(field_access_flags).c_str());
- return false;
+ if (header_->GetVersion() >= 37) {
+ return false;
+ } else {
+ // Allow in older versions, but warn.
+ LOG(WARNING) << "This dex file is invalid and will be rejected in the future. Error is: "
+ << *error_msg;
+ }
}
return true;
}
@@ -2657,7 +2669,13 @@
*error_msg = StringPrintf("Interface method %" PRIu32 "(%s) is not public and abstract",
method_index,
GetMethodDescriptionOrError(begin_, header_, method_index).c_str());
- return false;
+ if (header_->GetVersion() >= 37) {
+ return false;
+ } else {
+ // Allow in older versions, but warn.
+ LOG(WARNING) << "This dex file is invalid and will be rejected in the future. Error is: "
+ << *error_msg;
+ }
}
// At this point, we know the method is public and abstract. This means that all the checks
// for invalid combinations above applies. In addition, interface methods must not be
diff --git a/runtime/dex_file_verifier_test.cc b/runtime/dex_file_verifier_test.cc
index 44cf2ee..4a5ed5d 100644
--- a/runtime/dex_file_verifier_test.cc
+++ b/runtime/dex_file_verifier_test.cc
@@ -725,6 +725,13 @@
return result;
}
+// Make the Dex file version 37.
+static void MakeDexVersion37(DexFile* dex_file) {
+ size_t offset = OFFSETOF_MEMBER(DexFile::Header, magic_) + 6;
+ CHECK_EQ(*(dex_file->Begin() + offset), '5');
+ *(const_cast<uint8_t*>(dex_file->Begin()) + offset) = '7';
+}
+
TEST_F(DexFileVerifierTest, MethodAccessFlagsInterfaces) {
VerifyModification(
kMethodFlagsInterface,
@@ -733,6 +740,14 @@
ApplyMaskToMethodFlags(dex_file, "foo", ~kAccDeclaredSynchronized);
},
nullptr);
+ VerifyModification(
+ kMethodFlagsInterface,
+ "method_flags_interface_ok37",
+ [](DexFile* dex_file) {
+ MakeDexVersion37(dex_file);
+ ApplyMaskToMethodFlags(dex_file, "foo", ~kAccDeclaredSynchronized);
+ },
+ nullptr);
VerifyModification(
kMethodFlagsInterface,
@@ -742,7 +757,18 @@
ApplyMaskToMethodFlags(dex_file, "foo", ~kAccPublic);
},
+ nullptr); // Should be allowed in older dex versions for backwards compatibility.
+ VerifyModification(
+ kMethodFlagsInterface,
+ "method_flags_interface_non_public",
+ [](DexFile* dex_file) {
+ MakeDexVersion37(dex_file);
+ ApplyMaskToMethodFlags(dex_file, "foo", ~kAccDeclaredSynchronized);
+
+ ApplyMaskToMethodFlags(dex_file, "foo", ~kAccPublic);
+ },
"Interface method 1(LInterfaceMethodFlags;.foo) is not public and abstract");
+
VerifyModification(
kMethodFlagsInterface,
"method_flags_interface_non_abstract",
@@ -781,7 +807,18 @@
ApplyMaskToMethodFlags(dex_file, "foo", ~kAccPublic);
},
+ nullptr); // Should be allowed in older dex versions for backwards compatibility.
+ VerifyModification(
+ kMethodFlagsInterface,
+ "method_flags_interface_non_public",
+ [](DexFile* dex_file) {
+ MakeDexVersion37(dex_file);
+ ApplyMaskToMethodFlags(dex_file, "foo", ~kAccDeclaredSynchronized);
+
+ ApplyMaskToMethodFlags(dex_file, "foo", ~kAccPublic);
+ },
"Interface method 1(LInterfaceMethodFlags;.foo) is not public and abstract");
+
VerifyModification(
kMethodFlagsInterface,
"method_flags_interface_protected",
@@ -791,6 +828,17 @@
ApplyMaskToMethodFlags(dex_file, "foo", ~kAccPublic);
OrMaskToMethodFlags(dex_file, "foo", kAccProtected);
},
+ nullptr); // Should be allowed in older dex versions for backwards compatibility.
+ VerifyModification(
+ kMethodFlagsInterface,
+ "method_flags_interface_protected",
+ [](DexFile* dex_file) {
+ MakeDexVersion37(dex_file);
+ ApplyMaskToMethodFlags(dex_file, "foo", ~kAccDeclaredSynchronized);
+
+ ApplyMaskToMethodFlags(dex_file, "foo", ~kAccPublic);
+ OrMaskToMethodFlags(dex_file, "foo", kAccProtected);
+ },
"Interface method 1(LInterfaceMethodFlags;.foo) is not public and abstract");
constexpr uint32_t kAllMethodFlags =
@@ -1070,6 +1118,14 @@
ApplyMaskToFieldFlags(dex_file, "foo", ~kAccDeclaredSynchronized);
},
nullptr);
+ VerifyModification(
+ kFieldFlagsInterfaceTestDex,
+ "field_flags_interface",
+ [](DexFile* dex_file) {
+ MakeDexVersion37(dex_file);
+ ApplyMaskToFieldFlags(dex_file, "foo", ~kAccDeclaredSynchronized);
+ },
+ nullptr);
VerifyModification(
kFieldFlagsInterfaceTestDex,
@@ -1079,7 +1135,18 @@
ApplyMaskToFieldFlags(dex_file, "foo", ~kAccPublic);
},
+ nullptr); // Should be allowed in older dex versions for backwards compatibility.
+ VerifyModification(
+ kFieldFlagsInterfaceTestDex,
+ "field_flags_interface_non_public",
+ [](DexFile* dex_file) {
+ MakeDexVersion37(dex_file);
+ ApplyMaskToFieldFlags(dex_file, "foo", ~kAccDeclaredSynchronized);
+
+ ApplyMaskToFieldFlags(dex_file, "foo", ~kAccPublic);
+ },
"Interface field is not public final static");
+
VerifyModification(
kFieldFlagsInterfaceTestDex,
"field_flags_interface_non_final",
@@ -1088,7 +1155,18 @@
ApplyMaskToFieldFlags(dex_file, "foo", ~kAccFinal);
},
+ nullptr); // Should be allowed in older dex versions for backwards compatibility.
+ VerifyModification(
+ kFieldFlagsInterfaceTestDex,
+ "field_flags_interface_non_final",
+ [](DexFile* dex_file) {
+ MakeDexVersion37(dex_file);
+ ApplyMaskToFieldFlags(dex_file, "foo", ~kAccDeclaredSynchronized);
+
+ ApplyMaskToFieldFlags(dex_file, "foo", ~kAccFinal);
+ },
"Interface field is not public final static");
+
VerifyModification(
kFieldFlagsInterfaceTestDex,
"field_flags_interface_protected",
@@ -1098,7 +1176,19 @@
ApplyMaskToFieldFlags(dex_file, "foo", ~kAccPublic);
OrMaskToFieldFlags(dex_file, "foo", kAccProtected);
},
+ nullptr); // Should be allowed in older dex versions for backwards compatibility.
+ VerifyModification(
+ kFieldFlagsInterfaceTestDex,
+ "field_flags_interface_protected",
+ [](DexFile* dex_file) {
+ MakeDexVersion37(dex_file);
+ ApplyMaskToFieldFlags(dex_file, "foo", ~kAccDeclaredSynchronized);
+
+ ApplyMaskToFieldFlags(dex_file, "foo", ~kAccPublic);
+ OrMaskToFieldFlags(dex_file, "foo", kAccProtected);
+ },
"Interface field is not public final static");
+
VerifyModification(
kFieldFlagsInterfaceTestDex,
"field_flags_interface_private",
@@ -1108,6 +1198,17 @@
ApplyMaskToFieldFlags(dex_file, "foo", ~kAccPublic);
OrMaskToFieldFlags(dex_file, "foo", kAccPrivate);
},
+ nullptr); // Should be allowed in older dex versions for backwards compatibility.
+ VerifyModification(
+ kFieldFlagsInterfaceTestDex,
+ "field_flags_interface_private",
+ [](DexFile* dex_file) {
+ MakeDexVersion37(dex_file);
+ ApplyMaskToFieldFlags(dex_file, "foo", ~kAccDeclaredSynchronized);
+
+ ApplyMaskToFieldFlags(dex_file, "foo", ~kAccPublic);
+ OrMaskToFieldFlags(dex_file, "foo", kAccPrivate);
+ },
"Interface field is not public final static");
VerifyModification(
@@ -1152,6 +1253,21 @@
}
OrMaskToFieldFlags(dex_file, "foo", mask);
},
+ nullptr); // Should be allowed in older dex versions for backwards compatibility.
+ VerifyModification(
+ kFieldFlagsInterfaceTestDex,
+ "field_flags_interface_disallowed",
+ [&](DexFile* dex_file) {
+ MakeDexVersion37(dex_file);
+ ApplyMaskToFieldFlags(dex_file, "foo", ~kAccDeclaredSynchronized);
+
+ uint32_t mask = ApplyMaskShifted(kInterfaceDisallowed, i);
+ if ((mask & kAccProtected) != 0) {
+ mask &= ~kAccProtected;
+ ApplyMaskToFieldFlags(dex_file, "foo", ~kAccPublic);
+ }
+ OrMaskToFieldFlags(dex_file, "foo", mask);
+ },
"Interface field has disallowed flag");
}
}
@@ -1180,6 +1296,14 @@
[](DexFile* dex_file) {
ApplyMaskToFieldFlags(dex_file, "foo", ~kAccDeclaredSynchronized);
},
+ nullptr); // Should be allowed in older dex versions for backwards compatibility.
+ VerifyModification(
+ kFieldFlagsInterfaceBadTestDex,
+ "field_flags_interface_non_static",
+ [](DexFile* dex_file) {
+ MakeDexVersion37(dex_file);
+ ApplyMaskToFieldFlags(dex_file, "foo", ~kAccDeclaredSynchronized);
+ },
"Interface field is not public final static");
}
diff --git a/runtime/entrypoints/entrypoint_utils-inl.h b/runtime/entrypoints/entrypoint_utils-inl.h
index 5344cdd..116261b 100644
--- a/runtime/entrypoints/entrypoint_utils-inl.h
+++ b/runtime/entrypoints/entrypoint_utils-inl.h
@@ -39,6 +39,7 @@
namespace art {
+template <bool kResolve = true>
inline ArtMethod* GetResolvedMethod(ArtMethod* outer_method,
const InlineInfo& inline_info,
uint8_t inlining_depth)
@@ -50,6 +51,9 @@
if (!caller->IsRuntimeMethod()) {
return caller;
}
+ if (!kResolve) {
+ return nullptr;
+ }
// The method in the dex cache can be the runtime method responsible for invoking
// the stub that will then update the dex cache. Therefore, we need to do the
@@ -64,7 +68,7 @@
if (inlining_depth == 0) {
class_loader.Assign(outer_method->GetClassLoader());
} else {
- caller = GetResolvedMethod(outer_method, inline_info, inlining_depth - 1);
+ caller = GetResolvedMethod<kResolve>(outer_method, inline_info, inlining_depth - 1);
class_loader.Assign(caller->GetClassLoader());
}
diff --git a/runtime/gc/allocation_record.cc b/runtime/gc/allocation_record.cc
index e3714bb..bd023b3 100644
--- a/runtime/gc/allocation_record.cc
+++ b/runtime/gc/allocation_record.cc
@@ -102,15 +102,15 @@
// Only visit the last recent_record_max_ number of allocation records in entries_ and mark the
// klass_ fields as strong roots.
for (auto it = entries_.rbegin(), end = entries_.rend(); it != end; ++it) {
- AllocRecord* record = it->second;
+ AllocRecord& record = it->second;
if (count > 0) {
- buffered_visitor.VisitRootIfNonNull(record->GetClassGcRoot());
+ buffered_visitor.VisitRootIfNonNull(record.GetClassGcRoot());
--count;
}
// Visit all of the stack frames to make sure no methods in the stack traces get unloaded by
// class unloading.
- for (size_t i = 0, depth = record->GetDepth(); i < depth; ++i) {
- const AllocRecordStackTraceElement& element = record->StackElement(i);
+ for (size_t i = 0, depth = record.GetDepth(); i < depth; ++i) {
+ const AllocRecordStackTraceElement& element = record.StackElement(i);
DCHECK(element.GetMethod() != nullptr);
element.GetMethod()->VisitRoots(buffered_visitor, sizeof(void*));
}
@@ -143,15 +143,14 @@
++count;
// This does not need a read barrier because this is called by GC.
mirror::Object* old_object = it->first.Read<kWithoutReadBarrier>();
- AllocRecord* record = it->second;
+ AllocRecord& record = it->second;
mirror::Object* new_object = old_object == nullptr ? nullptr : visitor->IsMarked(old_object);
if (new_object == nullptr) {
if (count > delete_bound) {
it->first = GcRoot<mirror::Object>(nullptr);
- SweepClassObject(record, visitor);
+ SweepClassObject(&record, visitor);
++it;
} else {
- delete record;
it = entries_.erase(it);
++count_deleted;
}
@@ -160,7 +159,7 @@
it->first = GcRoot<mirror::Object>(new_object);
++count_moved;
}
- SweepClassObject(record, visitor);
+ SweepClassObject(&record, visitor);
++it;
}
}
@@ -184,34 +183,32 @@
new_record_condition_.Broadcast(Thread::Current());
}
-struct AllocRecordStackVisitor : public StackVisitor {
- AllocRecordStackVisitor(Thread* thread, AllocRecordStackTrace* trace_in, size_t max)
+class AllocRecordStackVisitor : public StackVisitor {
+ public:
+ AllocRecordStackVisitor(Thread* thread, size_t max_depth, AllocRecordStackTrace* trace_out)
SHARED_REQUIRES(Locks::mutator_lock_)
- : StackVisitor(thread, nullptr, StackVisitor::StackWalkKind::kIncludeInlinedFrames),
- trace(trace_in),
- max_depth(max) {}
+ : StackVisitor(thread, nullptr, StackVisitor::StackWalkKind::kIncludeInlinedFramesNoResolve),
+ max_depth_(max_depth),
+ trace_(trace_out) {}
// TODO: Enable annotalysis. We know lock is held in constructor, but abstraction confuses
// annotalysis.
bool VisitFrame() OVERRIDE NO_THREAD_SAFETY_ANALYSIS {
- if (depth >= max_depth) {
+ if (trace_->GetDepth() >= max_depth_) {
return false;
}
ArtMethod* m = GetMethod();
- if (!m->IsRuntimeMethod()) {
- trace->SetStackElementAt(depth, m, GetDexPc());
- ++depth;
+ // m may be null if we have inlined methods of unresolved classes. b/27858645
+ if (m != nullptr && !m->IsRuntimeMethod()) {
+ m = m->GetInterfaceMethodIfProxy(sizeof(void*));
+ trace_->AddStackElement(AllocRecordStackTraceElement(m, GetDexPc()));
}
return true;
}
- ~AllocRecordStackVisitor() {
- trace->SetDepth(depth);
- }
-
- AllocRecordStackTrace* trace;
- size_t depth = 0u;
- const size_t max_depth;
+ private:
+ const size_t max_depth_;
+ AllocRecordStackTrace* const trace_;
};
void AllocRecordObjectMap::SetAllocTrackingEnabled(bool enable) {
@@ -235,7 +232,6 @@
if (self_name == "JDWP") {
records->alloc_ddm_thread_id_ = self->GetTid();
}
- records->scratch_trace_.SetDepth(records->max_stack_depth_);
size_t sz = sizeof(AllocRecordStackTraceElement) * records->max_stack_depth_ +
sizeof(AllocRecord) + sizeof(AllocRecordStackTrace);
LOG(INFO) << "Enabling alloc tracker (" << records->alloc_record_max_ << " entries of "
@@ -265,27 +261,35 @@
}
}
-void AllocRecordObjectMap::RecordAllocation(Thread* self, mirror::Object* obj, mirror::Class* klass,
+void AllocRecordObjectMap::RecordAllocation(Thread* self,
+ mirror::Object** obj,
size_t byte_count) {
+ // Get stack trace outside of lock in case there are allocations during the stack walk.
+ // b/27858645.
+ AllocRecordStackTrace trace;
+ AllocRecordStackVisitor visitor(self, max_stack_depth_, /*out*/ &trace);
+ {
+ StackHandleScope<1> hs(self);
+ auto obj_wrapper = hs.NewHandleWrapper(obj);
+ visitor.WalkStack();
+ }
+
MutexLock mu(self, *Locks::alloc_tracker_lock_);
- Heap* heap = Runtime::Current()->GetHeap();
+ Heap* const heap = Runtime::Current()->GetHeap();
if (!heap->IsAllocTrackingEnabled()) {
// In the process of shutting down recording, bail.
return;
}
- AllocRecordObjectMap* records = heap->GetAllocationRecords();
- DCHECK(records != nullptr);
-
- // Do not record for DDM thread
- if (records->alloc_ddm_thread_id_ == self->GetTid()) {
+ // Do not record for DDM thread.
+ if (alloc_ddm_thread_id_ == self->GetTid()) {
return;
}
// Wait for GC's sweeping to complete and allow new records
- while (UNLIKELY((!kUseReadBarrier && !records->allow_new_record_) ||
+ while (UNLIKELY((!kUseReadBarrier && !allow_new_record_) ||
(kUseReadBarrier && !self->GetWeakRefAccessEnabled()))) {
- records->new_record_condition_.WaitHoldingLocks(self);
+ new_record_condition_.WaitHoldingLocks(self);
}
if (!heap->IsAllocTrackingEnabled()) {
@@ -294,28 +298,22 @@
return;
}
- DCHECK_LE(records->Size(), records->alloc_record_max_);
+ DCHECK_LE(Size(), alloc_record_max_);
- // Get stack trace.
- // add scope to make "visitor" destroyed promptly, in order to set the scratch_trace_->depth_
- {
- AllocRecordStackVisitor visitor(self, &records->scratch_trace_, records->max_stack_depth_);
- visitor.WalkStack();
- }
- records->scratch_trace_.SetTid(self->GetTid());
- AllocRecordStackTrace* trace = new AllocRecordStackTrace(records->scratch_trace_);
+ // Erase extra unfilled elements.
+ trace.SetTid(self->GetTid());
- // Fill in the basics.
- AllocRecord* record = new AllocRecord(byte_count, klass, trace);
-
- records->Put(obj, record);
- DCHECK_LE(records->Size(), records->alloc_record_max_);
+ // Add the record.
+ Put(*obj, AllocRecord(byte_count, (*obj)->GetClass(), std::move(trace)));
+ DCHECK_LE(Size(), alloc_record_max_);
}
void AllocRecordObjectMap::Clear() {
- STLDeleteValues(&entries_);
entries_.clear();
}
+AllocRecordObjectMap::AllocRecordObjectMap()
+ : new_record_condition_("New allocation record condition", *Locks::alloc_tracker_lock_) {}
+
} // namespace gc
} // namespace art
diff --git a/runtime/gc/allocation_record.h b/runtime/gc/allocation_record.h
index 18cce4d..a2d86cc 100644
--- a/runtime/gc/allocation_record.h
+++ b/runtime/gc/allocation_record.h
@@ -18,6 +18,7 @@
#define ART_RUNTIME_GC_ALLOCATION_RECORD_H_
#include <list>
+#include <memory>
#include "base/mutex.h"
#include "object_callbacks.h"
@@ -37,10 +38,13 @@
class AllocRecordStackTraceElement {
public:
- AllocRecordStackTraceElement() : method_(nullptr), dex_pc_(0) {}
-
int32_t ComputeLineNumber() const SHARED_REQUIRES(Locks::mutator_lock_);
+ AllocRecordStackTraceElement() = default;
+ AllocRecordStackTraceElement(ArtMethod* method, uint32_t dex_pc)
+ : method_(method),
+ dex_pc_(dex_pc) {}
+
ArtMethod* GetMethod() const {
return method_;
}
@@ -58,32 +62,27 @@
}
bool operator==(const AllocRecordStackTraceElement& other) const {
- if (this == &other) return true;
return method_ == other.method_ && dex_pc_ == other.dex_pc_;
}
private:
- ArtMethod* method_;
- uint32_t dex_pc_;
+ ArtMethod* method_ = nullptr;
+ uint32_t dex_pc_ = 0;
};
class AllocRecordStackTrace {
public:
static constexpr size_t kHashMultiplier = 17;
- explicit AllocRecordStackTrace(size_t max_depth)
- : tid_(0), depth_(0), stack_(new AllocRecordStackTraceElement[max_depth]) {}
+ AllocRecordStackTrace() = default;
+
+ AllocRecordStackTrace(AllocRecordStackTrace&& r)
+ : tid_(r.tid_),
+ stack_(std::move(r.stack_)) {}
AllocRecordStackTrace(const AllocRecordStackTrace& r)
- : tid_(r.tid_), depth_(r.depth_), stack_(new AllocRecordStackTraceElement[r.depth_]) {
- for (size_t i = 0; i < depth_; ++i) {
- stack_[i] = r.stack_[i];
- }
- }
-
- ~AllocRecordStackTrace() {
- delete[] stack_;
- }
+ : tid_(r.tid_),
+ stack_(r.stack_) {}
pid_t GetTid() const {
return tid_;
@@ -94,37 +93,32 @@
}
size_t GetDepth() const {
- return depth_;
- }
-
- void SetDepth(size_t depth) {
- depth_ = depth;
+ return stack_.size();
}
const AllocRecordStackTraceElement& GetStackElement(size_t index) const {
- DCHECK_LT(index, depth_);
+ DCHECK_LT(index, GetDepth());
return stack_[index];
}
+ void AddStackElement(const AllocRecordStackTraceElement& element) {
+ stack_.push_back(element);
+ }
+
void SetStackElementAt(size_t index, ArtMethod* m, uint32_t dex_pc) {
+ DCHECK_LT(index, stack_.size());
stack_[index].SetMethod(m);
stack_[index].SetDexPc(dex_pc);
}
bool operator==(const AllocRecordStackTrace& other) const {
if (this == &other) return true;
- if (tid_ != other.tid_) return false;
- if (depth_ != other.depth_) return false;
- for (size_t i = 0; i < depth_; ++i) {
- if (!(stack_[i] == other.stack_[i])) return false;
- }
- return true;
+ return tid_ == other.tid_ && stack_ == other.stack_;
}
private:
- pid_t tid_;
- size_t depth_;
- AllocRecordStackTraceElement* const stack_;
+ pid_t tid_ = 0;
+ std::vector<AllocRecordStackTraceElement> stack_;
};
struct HashAllocRecordTypes {
@@ -161,19 +155,15 @@
class AllocRecord {
public:
// All instances of AllocRecord should be managed by an instance of AllocRecordObjectMap.
- AllocRecord(size_t count, mirror::Class* klass, AllocRecordStackTrace* trace)
- : byte_count_(count), klass_(klass), trace_(trace) {}
-
- ~AllocRecord() {
- delete trace_;
- }
+ AllocRecord(size_t count, mirror::Class* klass, AllocRecordStackTrace&& trace)
+ : byte_count_(count), klass_(klass), trace_(std::move(trace)) {}
size_t GetDepth() const {
- return trace_->GetDepth();
+ return trace_.GetDepth();
}
const AllocRecordStackTrace* GetStackTrace() const {
- return trace_;
+ return &trace_;
}
size_t ByteCount() const {
@@ -181,7 +171,7 @@
}
pid_t GetTid() const {
- return trace_->GetTid();
+ return trace_.GetTid();
}
mirror::Class* GetClass() const SHARED_REQUIRES(Locks::mutator_lock_) {
@@ -196,16 +186,15 @@
}
const AllocRecordStackTraceElement& StackElement(size_t index) const {
- return trace_->GetStackElement(index);
+ return trace_.GetStackElement(index);
}
private:
const size_t byte_count_;
// The klass_ could be a strong or weak root for GC
GcRoot<mirror::Class> klass_;
- // TODO: Currently trace_ is like a std::unique_ptr,
- // but in future with deduplication it could be a std::shared_ptr.
- const AllocRecordStackTrace* const trace_;
+ // TODO: Share between alloc records with identical stack traces.
+ AllocRecordStackTrace trace_;
};
class AllocRecordObjectMap {
@@ -215,36 +204,29 @@
// weak roots). The last recent_record_max_ number of pairs in the list are always kept for DDMS's
// recent allocation tracking, but GcRoot<mirror::Object> pointers in these pairs can become null.
// Both types of pointers need read barriers, do not directly access them.
- typedef std::list<std::pair<GcRoot<mirror::Object>, AllocRecord*>> EntryList;
+ using EntryPair = std::pair<GcRoot<mirror::Object>, AllocRecord>;
+ typedef std::list<EntryPair> EntryList;
- // "static" because it is part of double-checked locking. It needs to check a bool first,
- // in order to make sure the AllocRecordObjectMap object is not null.
- static void RecordAllocation(Thread* self, mirror::Object* obj, mirror::Class* klass,
- size_t byte_count)
+ // Caller needs to check that it is enabled before calling since we read the stack trace before
+ // checking the enabled boolean.
+ void RecordAllocation(Thread* self,
+ mirror::Object** obj,
+ size_t byte_count)
REQUIRES(!Locks::alloc_tracker_lock_)
SHARED_REQUIRES(Locks::mutator_lock_);
static void SetAllocTrackingEnabled(bool enabled) REQUIRES(!Locks::alloc_tracker_lock_);
- AllocRecordObjectMap() REQUIRES(Locks::alloc_tracker_lock_)
- : alloc_record_max_(kDefaultNumAllocRecords),
- recent_record_max_(kDefaultNumRecentRecords),
- max_stack_depth_(kDefaultAllocStackDepth),
- scratch_trace_(kMaxSupportedStackDepth),
- alloc_ddm_thread_id_(0),
- allow_new_record_(true),
- new_record_condition_("New allocation record condition", *Locks::alloc_tracker_lock_) {}
-
+ AllocRecordObjectMap() REQUIRES(Locks::alloc_tracker_lock_);
~AllocRecordObjectMap();
- void Put(mirror::Object* obj, AllocRecord* record)
+ void Put(mirror::Object* obj, AllocRecord&& record)
SHARED_REQUIRES(Locks::mutator_lock_)
REQUIRES(Locks::alloc_tracker_lock_) {
if (entries_.size() == alloc_record_max_) {
- delete entries_.front().second;
entries_.pop_front();
}
- entries_.emplace_back(GcRoot<mirror::Object>(obj), record);
+ entries_.push_back(EntryPair(GcRoot<mirror::Object>(obj), std::move(record)));
}
size_t Size() const SHARED_REQUIRES(Locks::alloc_tracker_lock_) {
@@ -313,12 +295,11 @@
static constexpr size_t kDefaultNumRecentRecords = 64 * 1024 - 1;
static constexpr size_t kDefaultAllocStackDepth = 16;
static constexpr size_t kMaxSupportedStackDepth = 128;
- size_t alloc_record_max_ GUARDED_BY(Locks::alloc_tracker_lock_);
- size_t recent_record_max_ GUARDED_BY(Locks::alloc_tracker_lock_);
- size_t max_stack_depth_ GUARDED_BY(Locks::alloc_tracker_lock_);
- AllocRecordStackTrace scratch_trace_ GUARDED_BY(Locks::alloc_tracker_lock_);
- pid_t alloc_ddm_thread_id_ GUARDED_BY(Locks::alloc_tracker_lock_);
- bool allow_new_record_ GUARDED_BY(Locks::alloc_tracker_lock_);
+ size_t alloc_record_max_ GUARDED_BY(Locks::alloc_tracker_lock_) = kDefaultNumAllocRecords;
+ size_t recent_record_max_ GUARDED_BY(Locks::alloc_tracker_lock_) = kDefaultNumRecentRecords;
+ size_t max_stack_depth_ = kDefaultAllocStackDepth;
+ pid_t alloc_ddm_thread_id_ GUARDED_BY(Locks::alloc_tracker_lock_) = 0;
+ bool allow_new_record_ GUARDED_BY(Locks::alloc_tracker_lock_) = true;
ConditionVariable new_record_condition_ GUARDED_BY(Locks::alloc_tracker_lock_);
// see the comment in typedef of EntryList
EntryList entries_ GUARDED_BY(Locks::alloc_tracker_lock_);
diff --git a/runtime/gc/heap-inl.h b/runtime/gc/heap-inl.h
index 59fd4a6..6aed61a 100644
--- a/runtime/gc/heap-inl.h
+++ b/runtime/gc/heap-inl.h
@@ -176,8 +176,10 @@
}
if (kInstrumented) {
if (IsAllocTrackingEnabled()) {
- // Use obj->GetClass() instead of klass, because PushOnAllocationStack() could move klass
- AllocRecordObjectMap::RecordAllocation(self, obj, obj->GetClass(), bytes_allocated);
+ // allocation_records_ is not null since it never becomes null after allocation tracking is
+ // enabled.
+ DCHECK(allocation_records_ != nullptr);
+ allocation_records_->RecordAllocation(self, &obj, bytes_allocated);
}
} else {
DCHECK(!IsAllocTrackingEnabled());
diff --git a/runtime/gc/heap.h b/runtime/gc/heap.h
index 2925591..fada1a2 100644
--- a/runtime/gc/heap.h
+++ b/runtime/gc/heap.h
@@ -1326,8 +1326,7 @@
// Allocation tracking support
Atomic<bool> alloc_tracking_enabled_;
- std::unique_ptr<AllocRecordObjectMap> allocation_records_
- GUARDED_BY(Locks::alloc_tracker_lock_);
+ std::unique_ptr<AllocRecordObjectMap> allocation_records_;
// GC stress related data structures.
Mutex* backtrace_lock_ DEFAULT_MUTEX_ACQUIRED_AFTER;
diff --git a/runtime/hprof/hprof.cc b/runtime/hprof/hprof.cc
index bb35ec7..3885c60 100644
--- a/runtime/hprof/hprof.cc
+++ b/runtime/hprof/hprof.cc
@@ -828,7 +828,7 @@
continue;
}
++count;
- const gc::AllocRecordStackTrace* trace = it->second->GetStackTrace();
+ const gc::AllocRecordStackTrace* trace = it->second.GetStackTrace();
// Copy the pair into a real hash map to speed up look up.
auto records_result = allocation_records_.emplace(obj, trace);
diff --git a/runtime/instrumentation.h b/runtime/instrumentation.h
index d07f47b..a4c3d41 100644
--- a/runtime/instrumentation.h
+++ b/runtime/instrumentation.h
@@ -303,7 +303,8 @@
bool NonJitProfilingActive() const SHARED_REQUIRES(Locks::mutator_lock_) {
return have_dex_pc_listeners_ || have_method_exit_listeners_ ||
have_field_read_listeners_ || have_field_write_listeners_ ||
- have_exception_caught_listeners_ || have_method_unwind_listeners_;
+ have_exception_caught_listeners_ || have_method_unwind_listeners_ ||
+ have_branch_listeners_;
}
// Inform listeners that a method has been entered. A dex PC is provided as we may install
diff --git a/runtime/intern_table.cc b/runtime/intern_table.cc
index 74a2532..79f24a8 100644
--- a/runtime/intern_table.cc
+++ b/runtime/intern_table.cc
@@ -97,6 +97,17 @@
return LookupStrongLocked(s);
}
+mirror::String* InternTable::LookupStrong(Thread* self,
+ uint32_t utf16_length,
+ const char* utf8_data) {
+ DCHECK_EQ(utf16_length, CountModifiedUtf8Chars(utf8_data));
+ Utf8String string(utf16_length,
+ utf8_data,
+ ComputeUtf16HashFromModifiedUtf8(utf8_data, utf16_length));
+ MutexLock mu(self, *Locks::intern_table_lock_);
+ return strong_interns_.Find(string);
+}
+
mirror::String* InternTable::LookupWeakLocked(mirror::String* s) {
return weak_interns_.Find(s);
}
@@ -365,6 +376,20 @@
return a.Read()->Equals(b.Read());
}
+bool InternTable::StringHashEquals::operator()(const GcRoot<mirror::String>& a,
+ const Utf8String& b) const {
+ if (kIsDebugBuild) {
+ Locks::mutator_lock_->AssertSharedHeld(Thread::Current());
+ }
+ mirror::String* a_string = a.Read();
+ uint32_t a_length = static_cast<uint32_t>(a_string->GetLength());
+ if (a_length != b.GetUtf16Length()) {
+ return false;
+ }
+ const uint16_t* a_value = a_string->GetValue();
+ return CompareModifiedUtf8ToUtf16AsCodePointValues(b.GetUtf8Data(), a_value, a_length) == 0;
+}
+
size_t InternTable::Table::AddTableFromMemory(const uint8_t* ptr) {
size_t read_count = 0;
UnorderedSet set(ptr, /*make copy*/false, &read_count);
@@ -421,6 +446,17 @@
return nullptr;
}
+mirror::String* InternTable::Table::Find(const Utf8String& string) {
+ Locks::intern_table_lock_->AssertHeld(Thread::Current());
+ for (UnorderedSet& table : tables_) {
+ auto it = table.Find(string);
+ if (it != table.end()) {
+ return it->Read();
+ }
+ }
+ return nullptr;
+}
+
void InternTable::Table::AddNewTable() {
tables_.push_back(UnorderedSet());
}
diff --git a/runtime/intern_table.h b/runtime/intern_table.h
index 274f5ad..f845de5 100644
--- a/runtime/intern_table.h
+++ b/runtime/intern_table.h
@@ -88,6 +88,9 @@
mirror::String* LookupStrong(Thread* self, mirror::String* s)
REQUIRES(!Locks::intern_table_lock_)
SHARED_REQUIRES(Locks::mutator_lock_);
+ mirror::String* LookupStrong(Thread* self, uint32_t utf16_length, const char* utf8_data)
+ REQUIRES(!Locks::intern_table_lock_)
+ SHARED_REQUIRES(Locks::mutator_lock_);
// Lookup a weak intern, returns null if not found.
mirror::String* LookupWeak(Thread* self, mirror::String* s)
@@ -136,11 +139,32 @@
REQUIRES(!Locks::intern_table_lock_);
private:
+ // Modified UTF-8-encoded string treated as UTF16.
+ class Utf8String {
+ public:
+ Utf8String(uint32_t utf16_length, const char* utf8_data, int32_t hash)
+ : hash_(hash), utf16_length_(utf16_length), utf8_data_(utf8_data) { }
+
+ int32_t GetHash() const { return hash_; }
+ uint32_t GetUtf16Length() const { return utf16_length_; }
+ const char* GetUtf8Data() const { return utf8_data_; }
+
+ private:
+ int32_t hash_;
+ uint32_t utf16_length_;
+ const char* utf8_data_;
+ };
+
class StringHashEquals {
public:
std::size_t operator()(const GcRoot<mirror::String>& root) const NO_THREAD_SAFETY_ANALYSIS;
bool operator()(const GcRoot<mirror::String>& a, const GcRoot<mirror::String>& b) const
NO_THREAD_SAFETY_ANALYSIS;
+
+ // Utf8String can be used for lookup.
+ std::size_t operator()(const Utf8String& key) const { return key.GetHash(); }
+ bool operator()(const GcRoot<mirror::String>& a, const Utf8String& b) const
+ NO_THREAD_SAFETY_ANALYSIS;
};
class GcRootEmptyFn {
public:
@@ -159,6 +183,8 @@
Table();
mirror::String* Find(mirror::String* s) SHARED_REQUIRES(Locks::mutator_lock_)
REQUIRES(Locks::intern_table_lock_);
+ mirror::String* Find(const Utf8String& string) SHARED_REQUIRES(Locks::mutator_lock_)
+ REQUIRES(Locks::intern_table_lock_);
void Insert(mirror::String* s) SHARED_REQUIRES(Locks::mutator_lock_)
REQUIRES(Locks::intern_table_lock_);
void Remove(mirror::String* s)
diff --git a/runtime/intern_table_test.cc b/runtime/intern_table_test.cc
index b60b32d..fe78bf2 100644
--- a/runtime/intern_table_test.cc
+++ b/runtime/intern_table_test.cc
@@ -35,12 +35,14 @@
Handle<mirror::String> foo_3(
hs.NewHandle(mirror::String::AllocFromModifiedUtf8(soa.Self(), "foo")));
Handle<mirror::String> bar(hs.NewHandle(intern_table.InternStrong(3, "bar")));
+ ASSERT_TRUE(foo_1.Get() != nullptr);
+ ASSERT_TRUE(foo_2.Get() != nullptr);
+ ASSERT_TRUE(foo_3.Get() != nullptr);
+ ASSERT_TRUE(bar.Get() != nullptr);
+ EXPECT_EQ(foo_1.Get(), foo_2.Get());
EXPECT_TRUE(foo_1->Equals("foo"));
EXPECT_TRUE(foo_2->Equals("foo"));
EXPECT_TRUE(foo_3->Equals("foo"));
- EXPECT_TRUE(foo_1.Get() != nullptr);
- EXPECT_TRUE(foo_2.Get() != nullptr);
- EXPECT_EQ(foo_1.Get(), foo_2.Get());
EXPECT_NE(foo_1.Get(), bar.Get());
EXPECT_NE(foo_2.Get(), bar.Get());
EXPECT_NE(foo_3.Get(), bar.Get());
@@ -175,4 +177,39 @@
}
}
+TEST_F(InternTableTest, LookupStrong) {
+ ScopedObjectAccess soa(Thread::Current());
+ InternTable intern_table;
+ StackHandleScope<3> hs(soa.Self());
+ Handle<mirror::String> foo(hs.NewHandle(intern_table.InternStrong(3, "foo")));
+ Handle<mirror::String> bar(hs.NewHandle(intern_table.InternStrong(3, "bar")));
+ Handle<mirror::String> foobar(hs.NewHandle(intern_table.InternStrong(6, "foobar")));
+ ASSERT_TRUE(foo.Get() != nullptr);
+ ASSERT_TRUE(bar.Get() != nullptr);
+ ASSERT_TRUE(foobar.Get() != nullptr);
+ ASSERT_TRUE(foo->Equals("foo"));
+ ASSERT_TRUE(bar->Equals("bar"));
+ ASSERT_TRUE(foobar->Equals("foobar"));
+ ASSERT_NE(foo.Get(), bar.Get());
+ ASSERT_NE(foo.Get(), foobar.Get());
+ ASSERT_NE(bar.Get(), foobar.Get());
+ mirror::String* lookup_foo = intern_table.LookupStrong(soa.Self(), 3, "foo");
+ EXPECT_EQ(lookup_foo, foo.Get());
+ mirror::String* lookup_bar = intern_table.LookupStrong(soa.Self(), 3, "bar");
+ EXPECT_EQ(lookup_bar, bar.Get());
+ mirror::String* lookup_foobar = intern_table.LookupStrong(soa.Self(), 6, "foobar");
+ EXPECT_EQ(lookup_foobar, foobar.Get());
+ mirror::String* lookup_foox = intern_table.LookupStrong(soa.Self(), 4, "foox");
+ EXPECT_TRUE(lookup_foox == nullptr);
+ mirror::String* lookup_fooba = intern_table.LookupStrong(soa.Self(), 5, "fooba");
+ EXPECT_TRUE(lookup_fooba == nullptr);
+ mirror::String* lookup_foobaR = intern_table.LookupStrong(soa.Self(), 6, "foobaR");
+ EXPECT_TRUE(lookup_foobaR == nullptr);
+ // Try a hash conflict.
+ ASSERT_EQ(ComputeUtf16HashFromModifiedUtf8("foobar", 6),
+ ComputeUtf16HashFromModifiedUtf8("foobbS", 6));
+ mirror::String* lookup_foobbS = intern_table.LookupStrong(soa.Self(), 6, "foobbS");
+ EXPECT_TRUE(lookup_foobbS == nullptr);
+}
+
} // namespace art
diff --git a/runtime/interpreter/interpreter_goto_table_impl.cc b/runtime/interpreter/interpreter_goto_table_impl.cc
index 12d6fdc..f365fd0 100644
--- a/runtime/interpreter/interpreter_goto_table_impl.cc
+++ b/runtime/interpreter/interpreter_goto_table_impl.cc
@@ -22,6 +22,7 @@
#include "experimental_flags.h"
#include "interpreter_common.h"
#include "jit/jit.h"
+#include "jit/jit_instrumentation.h"
#include "safe_math.h"
#include <memory> // std::unique_ptr
@@ -64,15 +65,20 @@
currentHandlersTable = handlersTable[ \
Runtime::Current()->GetInstrumentation()->GetInterpreterHandlerTable()]
-#define BRANCH_INSTRUMENTATION(offset) \
- do { \
- ArtMethod* method = shadow_frame.GetMethod(); \
- instrumentation::Instrumentation* instrumentation = Runtime::Current()->GetInstrumentation(); \
- instrumentation->Branch(self, method, dex_pc, offset); \
- JValue result; \
- if (jit::Jit::MaybeDoOnStackReplacement(self, method, dex_pc, offset, &result)) { \
- return result; \
- } \
+#define BRANCH_INSTRUMENTATION(offset) \
+ do { \
+ instrumentation->Branch(self, method, dex_pc, offset); \
+ JValue result; \
+ if (jit::Jit::MaybeDoOnStackReplacement(self, method, dex_pc, offset, &result)) { \
+ return result; \
+ } \
+ } while (false)
+
+#define HOTNESS_UPDATE() \
+ do { \
+ if (jit_instrumentation_cache != nullptr) { \
+ jit_instrumentation_cache->AddSamples(self, method, 1); \
+ } \
} while (false)
#define UNREACHABLE_CODE_CHECK() \
@@ -186,6 +192,13 @@
UPDATE_HANDLER_TABLE();
std::unique_ptr<lambda::ClosureBuilder> lambda_closure_builder;
size_t lambda_captured_variable_index = 0;
+ const auto* const instrumentation = Runtime::Current()->GetInstrumentation();
+ ArtMethod* method = shadow_frame.GetMethod();
+ jit::Jit* jit = Runtime::Current()->GetJit();
+ jit::JitInstrumentationCache* jit_instrumentation_cache = nullptr;
+ if (jit != nullptr) {
+ jit_instrumentation_cache = jit->GetInstrumentationCache();
+ }
// Jump to first instruction.
ADVANCE(0);
@@ -277,7 +290,6 @@
JValue result;
self->AllowThreadSuspension();
HANDLE_MONITOR_CHECKS();
- instrumentation::Instrumentation* instrumentation = Runtime::Current()->GetInstrumentation();
if (UNLIKELY(instrumentation->HasMethodExitListeners())) {
instrumentation->MethodExitEvent(self, shadow_frame.GetThisObject(code_item->ins_size_),
shadow_frame.GetMethod(), dex_pc,
@@ -292,7 +304,6 @@
JValue result;
self->AllowThreadSuspension();
HANDLE_MONITOR_CHECKS();
- instrumentation::Instrumentation* instrumentation = Runtime::Current()->GetInstrumentation();
if (UNLIKELY(instrumentation->HasMethodExitListeners())) {
instrumentation->MethodExitEvent(self, shadow_frame.GetThisObject(code_item->ins_size_),
shadow_frame.GetMethod(), dex_pc,
@@ -308,7 +319,6 @@
result.SetI(shadow_frame.GetVReg(inst->VRegA_11x(inst_data)));
self->AllowThreadSuspension();
HANDLE_MONITOR_CHECKS();
- instrumentation::Instrumentation* instrumentation = Runtime::Current()->GetInstrumentation();
if (UNLIKELY(instrumentation->HasMethodExitListeners())) {
instrumentation->MethodExitEvent(self, shadow_frame.GetThisObject(code_item->ins_size_),
shadow_frame.GetMethod(), dex_pc,
@@ -323,7 +333,6 @@
result.SetJ(shadow_frame.GetVRegLong(inst->VRegA_11x(inst_data)));
self->AllowThreadSuspension();
HANDLE_MONITOR_CHECKS();
- instrumentation::Instrumentation* instrumentation = Runtime::Current()->GetInstrumentation();
if (UNLIKELY(instrumentation->HasMethodExitListeners())) {
instrumentation->MethodExitEvent(self, shadow_frame.GetThisObject(code_item->ins_size_),
shadow_frame.GetMethod(), dex_pc,
@@ -359,7 +368,6 @@
}
}
result.SetL(obj_result);
- instrumentation::Instrumentation* instrumentation = Runtime::Current()->GetInstrumentation();
if (UNLIKELY(instrumentation->HasMethodExitListeners())) {
instrumentation->MethodExitEvent(self, shadow_frame.GetThisObject(code_item->ins_size_),
shadow_frame.GetMethod(), dex_pc,
@@ -630,6 +638,7 @@
int8_t offset = inst->VRegA_10t(inst_data);
BRANCH_INSTRUMENTATION(offset);
if (IsBackwardBranch(offset)) {
+ HOTNESS_UPDATE();
if (UNLIKELY(self->TestAllFlags())) {
self->CheckSuspend();
UPDATE_HANDLER_TABLE();
@@ -643,6 +652,7 @@
int16_t offset = inst->VRegA_20t();
BRANCH_INSTRUMENTATION(offset);
if (IsBackwardBranch(offset)) {
+ HOTNESS_UPDATE();
if (UNLIKELY(self->TestAllFlags())) {
self->CheckSuspend();
UPDATE_HANDLER_TABLE();
@@ -656,6 +666,7 @@
int32_t offset = inst->VRegA_30t();
BRANCH_INSTRUMENTATION(offset);
if (IsBackwardBranch(offset)) {
+ HOTNESS_UPDATE();
if (UNLIKELY(self->TestAllFlags())) {
self->CheckSuspend();
UPDATE_HANDLER_TABLE();
@@ -669,6 +680,7 @@
int32_t offset = DoPackedSwitch(inst, shadow_frame, inst_data);
BRANCH_INSTRUMENTATION(offset);
if (IsBackwardBranch(offset)) {
+ HOTNESS_UPDATE();
if (UNLIKELY(self->TestAllFlags())) {
self->CheckSuspend();
UPDATE_HANDLER_TABLE();
@@ -682,6 +694,7 @@
int32_t offset = DoSparseSwitch(inst, shadow_frame, inst_data);
BRANCH_INSTRUMENTATION(offset);
if (IsBackwardBranch(offset)) {
+ HOTNESS_UPDATE();
if (UNLIKELY(self->TestAllFlags())) {
self->CheckSuspend();
UPDATE_HANDLER_TABLE();
@@ -785,6 +798,7 @@
int16_t offset = inst->VRegC_22t();
BRANCH_INSTRUMENTATION(offset);
if (IsBackwardBranch(offset)) {
+ HOTNESS_UPDATE();
if (UNLIKELY(self->TestAllFlags())) {
self->CheckSuspend();
UPDATE_HANDLER_TABLE();
@@ -804,6 +818,7 @@
int16_t offset = inst->VRegC_22t();
BRANCH_INSTRUMENTATION(offset);
if (IsBackwardBranch(offset)) {
+ HOTNESS_UPDATE();
if (UNLIKELY(self->TestAllFlags())) {
self->CheckSuspend();
UPDATE_HANDLER_TABLE();
@@ -823,6 +838,7 @@
int16_t offset = inst->VRegC_22t();
BRANCH_INSTRUMENTATION(offset);
if (IsBackwardBranch(offset)) {
+ HOTNESS_UPDATE();
if (UNLIKELY(self->TestAllFlags())) {
self->CheckSuspend();
UPDATE_HANDLER_TABLE();
@@ -842,6 +858,7 @@
int16_t offset = inst->VRegC_22t();
BRANCH_INSTRUMENTATION(offset);
if (IsBackwardBranch(offset)) {
+ HOTNESS_UPDATE();
if (UNLIKELY(self->TestAllFlags())) {
self->CheckSuspend();
UPDATE_HANDLER_TABLE();
@@ -861,6 +878,7 @@
int16_t offset = inst->VRegC_22t();
BRANCH_INSTRUMENTATION(offset);
if (IsBackwardBranch(offset)) {
+ HOTNESS_UPDATE();
if (UNLIKELY(self->TestAllFlags())) {
self->CheckSuspend();
UPDATE_HANDLER_TABLE();
@@ -880,6 +898,7 @@
int16_t offset = inst->VRegC_22t();
BRANCH_INSTRUMENTATION(offset);
if (IsBackwardBranch(offset)) {
+ HOTNESS_UPDATE();
if (UNLIKELY(self->TestAllFlags())) {
self->CheckSuspend();
UPDATE_HANDLER_TABLE();
@@ -898,6 +917,7 @@
int16_t offset = inst->VRegB_21t();
BRANCH_INSTRUMENTATION(offset);
if (IsBackwardBranch(offset)) {
+ HOTNESS_UPDATE();
if (UNLIKELY(self->TestAllFlags())) {
self->CheckSuspend();
UPDATE_HANDLER_TABLE();
@@ -916,6 +936,7 @@
int16_t offset = inst->VRegB_21t();
BRANCH_INSTRUMENTATION(offset);
if (IsBackwardBranch(offset)) {
+ HOTNESS_UPDATE();
if (UNLIKELY(self->TestAllFlags())) {
self->CheckSuspend();
UPDATE_HANDLER_TABLE();
@@ -934,6 +955,7 @@
int16_t offset = inst->VRegB_21t();
BRANCH_INSTRUMENTATION(offset);
if (IsBackwardBranch(offset)) {
+ HOTNESS_UPDATE();
if (UNLIKELY(self->TestAllFlags())) {
self->CheckSuspend();
UPDATE_HANDLER_TABLE();
@@ -952,6 +974,7 @@
int16_t offset = inst->VRegB_21t();
BRANCH_INSTRUMENTATION(offset);
if (IsBackwardBranch(offset)) {
+ HOTNESS_UPDATE();
if (UNLIKELY(self->TestAllFlags())) {
self->CheckSuspend();
UPDATE_HANDLER_TABLE();
@@ -970,6 +993,7 @@
int16_t offset = inst->VRegB_21t();
BRANCH_INSTRUMENTATION(offset);
if (IsBackwardBranch(offset)) {
+ HOTNESS_UPDATE();
if (UNLIKELY(self->TestAllFlags())) {
self->CheckSuspend();
UPDATE_HANDLER_TABLE();
@@ -988,6 +1012,7 @@
int16_t offset = inst->VRegB_21t();
BRANCH_INSTRUMENTATION(offset);
if (IsBackwardBranch(offset)) {
+ HOTNESS_UPDATE();
if (UNLIKELY(self->TestAllFlags())) {
self->CheckSuspend();
UPDATE_HANDLER_TABLE();
@@ -2558,7 +2583,6 @@
self->CheckSuspend();
UPDATE_HANDLER_TABLE();
}
- instrumentation::Instrumentation* instrumentation = Runtime::Current()->GetInstrumentation();
uint32_t found_dex_pc = FindNextInstructionFollowingException(self, shadow_frame, dex_pc,
instrumentation);
if (found_dex_pc == DexFile::kDexNoIndex) {
diff --git a/runtime/interpreter/interpreter_switch_impl.cc b/runtime/interpreter/interpreter_switch_impl.cc
index 0488dbf..f9941d2 100644
--- a/runtime/interpreter/interpreter_switch_impl.cc
+++ b/runtime/interpreter/interpreter_switch_impl.cc
@@ -18,6 +18,7 @@
#include "experimental_flags.h"
#include "interpreter_common.h"
#include "jit/jit.h"
+#include "jit/jit_instrumentation.h"
#include "safe_math.h"
#include <memory> // std::unique_ptr
@@ -72,7 +73,6 @@
#define BRANCH_INSTRUMENTATION(offset) \
do { \
- ArtMethod* method = shadow_frame.GetMethod(); \
instrumentation->Branch(self, method, dex_pc, offset); \
JValue result; \
if (jit::Jit::MaybeDoOnStackReplacement(self, method, dex_pc, offset, &result)) { \
@@ -80,6 +80,13 @@
} \
} while (false)
+#define HOTNESS_UPDATE() \
+ do { \
+ if (jit_instrumentation_cache != nullptr) { \
+ jit_instrumentation_cache->AddSamples(self, method, 1); \
+ } \
+ } while (false)
+
static bool IsExperimentalInstructionEnabled(const Instruction *inst) {
DCHECK(inst->IsExperimental());
return Runtime::Current()->AreExperimentalFlagsEnabled(ExperimentalFlags::kLambdas);
@@ -101,6 +108,12 @@
const uint16_t* const insns = code_item->insns_;
const Instruction* inst = Instruction::At(insns + dex_pc);
uint16_t inst_data;
+ ArtMethod* method = shadow_frame.GetMethod();
+ jit::Jit* jit = Runtime::Current()->GetJit();
+ jit::JitInstrumentationCache* jit_instrumentation_cache = nullptr;
+ if (jit != nullptr) {
+ jit_instrumentation_cache = jit->GetInstrumentationCache();
+ }
// TODO: collapse capture-variable+create-lambda into one opcode, then we won't need
// to keep this live for the scope of the entire function call.
@@ -564,6 +577,7 @@
int8_t offset = inst->VRegA_10t(inst_data);
BRANCH_INSTRUMENTATION(offset);
if (IsBackwardBranch(offset)) {
+ HOTNESS_UPDATE();
self->AllowThreadSuspension();
}
inst = inst->RelativeAt(offset);
@@ -574,6 +588,7 @@
int16_t offset = inst->VRegA_20t();
BRANCH_INSTRUMENTATION(offset);
if (IsBackwardBranch(offset)) {
+ HOTNESS_UPDATE();
self->AllowThreadSuspension();
}
inst = inst->RelativeAt(offset);
@@ -584,6 +599,7 @@
int32_t offset = inst->VRegA_30t();
BRANCH_INSTRUMENTATION(offset);
if (IsBackwardBranch(offset)) {
+ HOTNESS_UPDATE();
self->AllowThreadSuspension();
}
inst = inst->RelativeAt(offset);
@@ -594,6 +610,7 @@
int32_t offset = DoPackedSwitch(inst, shadow_frame, inst_data);
BRANCH_INSTRUMENTATION(offset);
if (IsBackwardBranch(offset)) {
+ HOTNESS_UPDATE();
self->AllowThreadSuspension();
}
inst = inst->RelativeAt(offset);
@@ -604,6 +621,7 @@
int32_t offset = DoSparseSwitch(inst, shadow_frame, inst_data);
BRANCH_INSTRUMENTATION(offset);
if (IsBackwardBranch(offset)) {
+ HOTNESS_UPDATE();
self->AllowThreadSuspension();
}
inst = inst->RelativeAt(offset);
@@ -708,6 +726,7 @@
int16_t offset = inst->VRegC_22t();
BRANCH_INSTRUMENTATION(offset);
if (IsBackwardBranch(offset)) {
+ HOTNESS_UPDATE();
self->AllowThreadSuspension();
}
inst = inst->RelativeAt(offset);
@@ -724,6 +743,7 @@
int16_t offset = inst->VRegC_22t();
BRANCH_INSTRUMENTATION(offset);
if (IsBackwardBranch(offset)) {
+ HOTNESS_UPDATE();
self->AllowThreadSuspension();
}
inst = inst->RelativeAt(offset);
@@ -740,6 +760,7 @@
int16_t offset = inst->VRegC_22t();
BRANCH_INSTRUMENTATION(offset);
if (IsBackwardBranch(offset)) {
+ HOTNESS_UPDATE();
self->AllowThreadSuspension();
}
inst = inst->RelativeAt(offset);
@@ -756,6 +777,7 @@
int16_t offset = inst->VRegC_22t();
BRANCH_INSTRUMENTATION(offset);
if (IsBackwardBranch(offset)) {
+ HOTNESS_UPDATE();
self->AllowThreadSuspension();
}
inst = inst->RelativeAt(offset);
@@ -772,6 +794,7 @@
int16_t offset = inst->VRegC_22t();
BRANCH_INSTRUMENTATION(offset);
if (IsBackwardBranch(offset)) {
+ HOTNESS_UPDATE();
self->AllowThreadSuspension();
}
inst = inst->RelativeAt(offset);
@@ -788,6 +811,7 @@
int16_t offset = inst->VRegC_22t();
BRANCH_INSTRUMENTATION(offset);
if (IsBackwardBranch(offset)) {
+ HOTNESS_UPDATE();
self->AllowThreadSuspension();
}
inst = inst->RelativeAt(offset);
@@ -803,6 +827,7 @@
int16_t offset = inst->VRegB_21t();
BRANCH_INSTRUMENTATION(offset);
if (IsBackwardBranch(offset)) {
+ HOTNESS_UPDATE();
self->AllowThreadSuspension();
}
inst = inst->RelativeAt(offset);
@@ -818,6 +843,7 @@
int16_t offset = inst->VRegB_21t();
BRANCH_INSTRUMENTATION(offset);
if (IsBackwardBranch(offset)) {
+ HOTNESS_UPDATE();
self->AllowThreadSuspension();
}
inst = inst->RelativeAt(offset);
@@ -833,6 +859,7 @@
int16_t offset = inst->VRegB_21t();
BRANCH_INSTRUMENTATION(offset);
if (IsBackwardBranch(offset)) {
+ HOTNESS_UPDATE();
self->AllowThreadSuspension();
}
inst = inst->RelativeAt(offset);
@@ -848,6 +875,7 @@
int16_t offset = inst->VRegB_21t();
BRANCH_INSTRUMENTATION(offset);
if (IsBackwardBranch(offset)) {
+ HOTNESS_UPDATE();
self->AllowThreadSuspension();
}
inst = inst->RelativeAt(offset);
@@ -863,6 +891,7 @@
int16_t offset = inst->VRegB_21t();
BRANCH_INSTRUMENTATION(offset);
if (IsBackwardBranch(offset)) {
+ HOTNESS_UPDATE();
self->AllowThreadSuspension();
}
inst = inst->RelativeAt(offset);
@@ -878,6 +907,7 @@
int16_t offset = inst->VRegB_21t();
BRANCH_INSTRUMENTATION(offset);
if (IsBackwardBranch(offset)) {
+ HOTNESS_UPDATE();
self->AllowThreadSuspension();
}
inst = inst->RelativeAt(offset);
diff --git a/runtime/interpreter/mterp/arm/bincmp.S b/runtime/interpreter/mterp/arm/bincmp.S
index cfad714..8fad42f 100644
--- a/runtime/interpreter/mterp/arm/bincmp.S
+++ b/runtime/interpreter/mterp/arm/bincmp.S
@@ -1,7 +1,6 @@
/*
- * Generic two-operand compare-and-branch operation. Provide a "revcmp"
- * fragment that specifies the *reverse* comparison to perform, e.g.
- * for "if-le" you would use "gt".
+ * Generic two-operand compare-and-branch operation. Provide a "condition"
+ * fragment that specifies the comparison to perform.
*
* For: if-eq, if-ne, if-lt, if-ge, if-gt, if-le
*/
@@ -9,23 +8,12 @@
mov r1, rINST, lsr #12 @ r1<- B
ubfx r0, rINST, #8, #4 @ r0<- A
GET_VREG r3, r1 @ r3<- vB
- GET_VREG r2, r0 @ r2<- vA
+ GET_VREG r0, r0 @ r0<- vA
FETCH_S rINST, 1 @ rINST<- branch offset, in code units
- cmp r2, r3 @ compare (vA, vB)
- mov${revcmp} rINST, #2
-#if MTERP_PROFILE_BRANCHES
- @ TUNING: once measurements are complete, remove #if and hand-schedule.
- EXPORT_PC
- mov r0, rSELF
- add r1, rFP, #OFF_FP_SHADOWFRAME
- mov r2, rINST
- bl MterpProfileBranch @ (self, shadow_frame, offset)
- cmp r0, #0
- bne MterpOnStackReplacement @ Note: offset must be in rINST
-#endif
- adds r2, rINST, rINST @ convert to bytes, check sign
- ldr lr, [rSELF, #THREAD_FLAGS_OFFSET]
- FETCH_ADVANCE_INST_RB r2 @ update rPC, load rINST
- bmi MterpCheckSuspendAndContinue
+ cmp r0, r3 @ compare (vA, vB)
+ b${condition} MterpCommonTakenBranchNoFlags
+ cmp rPROFILE, #JIT_CHECK_OSR @ possible OSR re-entry?
+ beq .L_check_not_taken_osr
+ FETCH_ADVANCE_INST 2
GET_INST_OPCODE ip @ extract opcode from rINST
GOTO_OPCODE ip @ jump to next instruction
diff --git a/runtime/interpreter/mterp/arm/entry.S b/runtime/interpreter/mterp/arm/entry.S
index 981c036..a6b131d 100644
--- a/runtime/interpreter/mterp/arm/entry.S
+++ b/runtime/interpreter/mterp/arm/entry.S
@@ -33,10 +33,8 @@
ExecuteMterpImpl:
.fnstart
- .save {r4-r10,fp,lr}
- stmfd sp!, {r4-r10,fp,lr} @ save 9 regs
- .pad #4
- sub sp, sp, #4 @ align 64
+ .save {r3-r10,fp,lr}
+ stmfd sp!, {r3-r10,fp,lr} @ save 10 regs, (r3 just to align 64)
/* Remember the return register */
str r3, [r2, #SHADOWFRAME_RESULT_REGISTER_OFFSET]
@@ -57,6 +55,12 @@
/* Starting ibase */
ldr rIBASE, [rSELF, #THREAD_CURRENT_IBASE_OFFSET]
+ /* Set up for backwards branches & osr profiling */
+ ldr r0, [rFP, #OFF_FP_METHOD]
+ add r1, rFP, #OFF_FP_SHADOWFRAME
+ bl MterpSetUpHotnessCountdown
+ mov rPROFILE, r0 @ Starting hotness countdown to rPROFILE
+
/* start executing the instruction at rPC */
FETCH_INST @ load rINST from rPC
GET_INST_OPCODE ip @ extract opcode from rINST
diff --git a/runtime/interpreter/mterp/arm/footer.S b/runtime/interpreter/mterp/arm/footer.S
index 3456a75..62e573a 100644
--- a/runtime/interpreter/mterp/arm/footer.S
+++ b/runtime/interpreter/mterp/arm/footer.S
@@ -114,21 +114,117 @@
/* NOTE: no fallthrough */
/*
- * Check for suspend check request. Assumes rINST already loaded, rPC advanced and
- * still needs to get the opcode and branch to it, and flags are in lr.
+ * Common handling for branches with support for Jit profiling.
+ * On entry:
+ * rINST <= signed offset
+ * rPROFILE <= signed hotness countdown (expanded to 32 bits)
+ * condition bits <= set to establish sign of offset (use "NoFlags" entry if not)
+ *
+ * We have quite a few different cases for branch profiling, OSR detection and
+ * suspend check support here.
+ *
+ * Taken backward branches:
+ * If profiling active, do hotness countdown and report if we hit zero.
+ * If in osr check mode, see if our target is a compiled loop header entry and do OSR if so.
+ * Is there a pending suspend request? If so, suspend.
+ *
+ * Taken forward branches and not-taken backward branches:
+ * If in osr check mode, see if our target is a compiled loop header entry and do OSR if so.
+ *
+ * Our most common case is expected to be a taken backward branch with active jit profiling,
+ * but no full OSR check and no pending suspend request.
+ * Next most common case is not-taken branch with no full OSR check.
+ *
*/
-MterpCheckSuspendAndContinue:
- ldr rIBASE, [rSELF, #THREAD_CURRENT_IBASE_OFFSET] @ refresh rIBASE
+MterpCommonTakenBranchNoFlags:
+ cmp rINST, #0
+MterpCommonTakenBranch:
+ bgt .L_forward_branch @ don't add forward branches to hotness
+/*
+ * We need to subtract 1 from positive values and we should not see 0 here,
+ * so we may use the result of the comparison with -1.
+ */
+#if JIT_CHECK_OSR != -1
+# error "JIT_CHECK_OSR must be -1."
+#endif
+ cmp rPROFILE, #JIT_CHECK_OSR
+ beq .L_osr_check
+ subgts rPROFILE, #1
+ beq .L_add_batch @ counted down to zero - report
+.L_resume_backward_branch:
+ ldr lr, [rSELF, #THREAD_FLAGS_OFFSET]
+ REFRESH_IBASE
+ add r2, rINST, rINST @ r2<- byte offset
+ FETCH_ADVANCE_INST_RB r2 @ update rPC, load rINST
ands lr, #(THREAD_SUSPEND_REQUEST | THREAD_CHECKPOINT_REQUEST)
- bne 1f
+ bne .L_suspend_request_pending
GET_INST_OPCODE ip @ extract opcode from rINST
GOTO_OPCODE ip @ jump to next instruction
-1:
+
+.L_suspend_request_pending:
EXPORT_PC
mov r0, rSELF
bl MterpSuspendCheck @ (self)
cmp r0, #0
bne MterpFallback
+ REFRESH_IBASE @ might have changed during suspend
+ GET_INST_OPCODE ip @ extract opcode from rINST
+ GOTO_OPCODE ip @ jump to next instruction
+
+.L_no_count_backwards:
+ cmp rPROFILE, #JIT_CHECK_OSR @ possible OSR re-entry?
+ bne .L_resume_backward_branch
+.L_osr_check:
+ mov r0, rSELF
+ add r1, rFP, #OFF_FP_SHADOWFRAME
+ mov r2, rINST
+ EXPORT_PC
+ bl MterpMaybeDoOnStackReplacement @ (self, shadow_frame, offset)
+ cmp r0, #0
+ bne MterpOnStackReplacement
+ b .L_resume_backward_branch
+
+.L_forward_branch:
+ cmp rPROFILE, #JIT_CHECK_OSR @ possible OSR re-entry?
+ beq .L_check_osr_forward
+.L_resume_forward_branch:
+ add r2, rINST, rINST @ r2<- byte offset
+ FETCH_ADVANCE_INST_RB r2 @ update rPC, load rINST
+ GET_INST_OPCODE ip @ extract opcode from rINST
+ GOTO_OPCODE ip @ jump to next instruction
+
+.L_check_osr_forward:
+ mov r0, rSELF
+ add r1, rFP, #OFF_FP_SHADOWFRAME
+ mov r2, rINST
+ EXPORT_PC
+ bl MterpMaybeDoOnStackReplacement @ (self, shadow_frame, offset)
+ cmp r0, #0
+ bne MterpOnStackReplacement
+ b .L_resume_forward_branch
+
+.L_add_batch:
+ add r1, rFP, #OFF_FP_SHADOWFRAME
+ strh rPROFILE, [r1, #SHADOWFRAME_HOTNESS_COUNTDOWN_OFFSET]
+ ldr r0, [rFP, #OFF_FP_METHOD]
+ mov r2, rSELF
+ bl MterpAddHotnessBatch @ (method, shadow_frame, self)
+ mov rPROFILE, r0 @ restore new hotness countdown to rPROFILE
+ b .L_no_count_backwards
+
+/*
+ * Entered from the conditional branch handlers when OSR check request active on
+ * not-taken path. All Dalvik not-taken conditional branch offsets are 2.
+ */
+.L_check_not_taken_osr:
+ mov r0, rSELF
+ add r1, rFP, #OFF_FP_SHADOWFRAME
+ mov r2, #2
+ EXPORT_PC
+ bl MterpMaybeDoOnStackReplacement @ (self, shadow_frame, offset)
+ cmp r0, #0
+ bne MterpOnStackReplacement
+ FETCH_ADVANCE_INST 2
GET_INST_OPCODE ip @ extract opcode from rINST
GOTO_OPCODE ip @ jump to next instruction
@@ -176,9 +272,27 @@
str r1, [r2, #4]
mov r0, #1 @ signal return to caller.
MterpDone:
- add sp, sp, #4 @ un-align 64
- ldmfd sp!, {r4-r10,fp,pc} @ restore 9 regs and return
+/*
+ * At this point, we expect rPROFILE to be non-zero. If negative, hotness is disabled or we're
+ * checking for OSR. If greater than zero, we might have unreported hotness to register
+ * (the difference between the ending rPROFILE and the cached hotness counter). rPROFILE
+ * should only reach zero immediately after a hotness decrement, and is then reset to either
+ * a negative special state or the new non-zero countdown value.
+ */
+ cmp rPROFILE, #0
+ bgt MterpProfileActive @ if > 0, we may have some counts to report.
+ ldmfd sp!, {r3-r10,fp,pc} @ restore 10 regs and return
+MterpProfileActive:
+ mov rINST, r0 @ stash return value
+ /* Report cached hotness counts */
+ ldr r0, [rFP, #OFF_FP_METHOD]
+ add r1, rFP, #OFF_FP_SHADOWFRAME
+ mov r2, rSELF
+ strh rPROFILE, [r1, #SHADOWFRAME_HOTNESS_COUNTDOWN_OFFSET]
+ bl MterpAddHotnessBatch @ (method, shadow_frame, self)
+ mov r0, rINST @ restore return value
+ ldmfd sp!, {r3-r10,fp,pc} @ restore 10 regs and return
.fnend
.size ExecuteMterpImpl, .-ExecuteMterpImpl
diff --git a/runtime/interpreter/mterp/arm/header.S b/runtime/interpreter/mterp/arm/header.S
index 298af8a..039bcbe 100644
--- a/runtime/interpreter/mterp/arm/header.S
+++ b/runtime/interpreter/mterp/arm/header.S
@@ -72,7 +72,8 @@
r6 rSELF self (Thread) pointer
r7 rINST first 16-bit code unit of current instruction
r8 rIBASE interpreted instruction base pointer, used for computed goto
- r11 rREFS base of object references in shadow frame (ideally, we'll get rid of this later).
+ r10 rPROFILE branch profiling countdown
+ r11 rREFS base of object references in shadow frame (ideally, we'll get rid of this later).
Macros are provided for common operations. Each macro MUST emit only
one instruction to make instruction-counting easier. They MUST NOT alter
@@ -90,12 +91,13 @@
/* During bringup, we'll use the shadow frame model instead of rFP */
/* single-purpose registers, given names for clarity */
-#define rPC r4
-#define rFP r5
-#define rSELF r6
-#define rINST r7
-#define rIBASE r8
-#define rREFS r11
+#define rPC r4
+#define rFP r5
+#define rSELF r6
+#define rINST r7
+#define rIBASE r8
+#define rPROFILE r10
+#define rREFS r11
/*
* Instead of holding a pointer to the shadow frame, we keep rFP at the base of the vregs. So,
@@ -109,7 +111,7 @@
#define OFF_FP_RESULT_REGISTER OFF_FP(SHADOWFRAME_RESULT_REGISTER_OFFSET)
#define OFF_FP_DEX_PC_PTR OFF_FP(SHADOWFRAME_DEX_PC_PTR_OFFSET)
#define OFF_FP_CODE_ITEM OFF_FP(SHADOWFRAME_CODE_ITEM_OFFSET)
-#define OFF_FP_SHADOWFRAME (-SHADOWFRAME_VREGS_OFFSET)
+#define OFF_FP_SHADOWFRAME OFF_FP(0)
/*
* "export" the PC to dex_pc field in the shadow frame, f/b/o future exception objects. Must
diff --git a/runtime/interpreter/mterp/arm/op_cmp_long.S b/runtime/interpreter/mterp/arm/op_cmp_long.S
index e57b19c..6626ff0 100644
--- a/runtime/interpreter/mterp/arm/op_cmp_long.S
+++ b/runtime/interpreter/mterp/arm/op_cmp_long.S
@@ -1,22 +1,6 @@
/*
* Compare two 64-bit values. Puts 0, 1, or -1 into the destination
* register based on the results of the comparison.
- *
- * We load the full values with LDM, but in practice many values could
- * be resolved by only looking at the high word. This could be made
- * faster or slower by splitting the LDM into a pair of LDRs.
- *
- * If we just wanted to set condition flags, we could do this:
- * subs ip, r0, r2
- * sbcs ip, r1, r3
- * subeqs ip, r0, r2
- * Leaving { <0, 0, >0 } in ip. However, we have to set it to a specific
- * integer value, which we can do with 2 conditional mov/mvn instructions
- * (set 1, set -1; if they're equal we already have 0 in ip), giving
- * us a constant 5-cycle path plus a branch at the end to the
- * instruction epilogue code. The multi-compare approach below needs
- * 2 or 3 cycles + branch if the high word doesn't match, 6 + branch
- * in the worst case (the 64-bit values are equal).
*/
/* cmp-long vAA, vBB, vCC */
FETCH r0, 1 @ r0<- CCBB
@@ -27,30 +11,13 @@
VREG_INDEX_TO_ADDR r3, r3 @ r3<- &fp[CC]
ldmia r2, {r0-r1} @ r0/r1<- vBB/vBB+1
ldmia r3, {r2-r3} @ r2/r3<- vCC/vCC+1
- cmp r1, r3 @ compare (vBB+1, vCC+1)
- blt .L${opcode}_less @ signed compare on high part
- bgt .L${opcode}_greater
- subs r1, r0, r2 @ r1<- r0 - r2
- bhi .L${opcode}_greater @ unsigned compare on low part
- bne .L${opcode}_less
- b .L${opcode}_finish @ equal; r1 already holds 0
-%break
-
-.L${opcode}_less:
- mvn r1, #0 @ r1<- -1
- @ Want to cond code the next mov so we can avoid branch, but don't see it;
- @ instead, we just replicate the tail end.
+ cmp r0, r2
+ sbcs ip, r1, r3 @ Sets correct CCs for checking LT (but not EQ/NE)
+ mov ip, #0
+ mvnlt ip, #0 @ -1
+ cmpeq r0, r2 @ For correct EQ/NE, we may need to repeat the first CMP
+ orrne ip, #1
FETCH_ADVANCE_INST 2 @ advance rPC, load rINST
- SET_VREG r1, r9 @ vAA<- r1
- GET_INST_OPCODE ip @ extract opcode from rINST
- GOTO_OPCODE ip @ jump to next instruction
-
-.L${opcode}_greater:
- mov r1, #1 @ r1<- 1
- @ fall through to _finish
-
-.L${opcode}_finish:
- FETCH_ADVANCE_INST 2 @ advance rPC, load rINST
- SET_VREG r1, r9 @ vAA<- r1
+ SET_VREG ip, r9 @ vAA<- ip
GET_INST_OPCODE ip @ extract opcode from rINST
GOTO_OPCODE ip @ jump to next instruction
diff --git a/runtime/interpreter/mterp/arm/op_goto.S b/runtime/interpreter/mterp/arm/op_goto.S
index 6861950..aa42dfd 100644
--- a/runtime/interpreter/mterp/arm/op_goto.S
+++ b/runtime/interpreter/mterp/arm/op_goto.S
@@ -5,32 +5,5 @@
* double to get a byte offset.
*/
/* goto +AA */
- /* tuning: use sbfx for 6t2+ targets */
-#if MTERP_PROFILE_BRANCHES
- mov r0, rINST, lsl #16 @ r0<- AAxx0000
- movs rINST, r0, asr #24 @ rINST<- ssssssAA (sign-extended)
- EXPORT_PC
- mov r0, rSELF
- add r1, rFP, #OFF_FP_SHADOWFRAME
- mov r2, rINST
- bl MterpProfileBranch @ (self, shadow_frame, offset)
- cmp r0, #0
- bne MterpOnStackReplacement @ Note: offset must be in rINST
- ldr lr, [rSELF, #THREAD_FLAGS_OFFSET]
- adds r2, rINST, rINST @ r2<- byte offset, set flags
- FETCH_ADVANCE_INST_RB r2 @ update rPC, load rINST
- @ If backwards branch refresh rIBASE
- bmi MterpCheckSuspendAndContinue
- GET_INST_OPCODE ip @ extract opcode from rINST
- GOTO_OPCODE ip @ jump to next instruction
-#else
- mov r0, rINST, lsl #16 @ r0<- AAxx0000
- movs rINST, r0, asr #24 @ rINST<- ssssssAA (sign-extended)
- ldr lr, [rSELF, #THREAD_FLAGS_OFFSET]
- adds r2, rINST, rINST @ r2<- byte offset, set flags
- FETCH_ADVANCE_INST_RB r2 @ update rPC, load rINST
- @ If backwards branch refresh rIBASE
- bmi MterpCheckSuspendAndContinue
- GET_INST_OPCODE ip @ extract opcode from rINST
- GOTO_OPCODE ip @ jump to next instruction
-#endif
+ sbfx rINST, rINST, #8, #8 @ rINST<- ssssssAA (sign-extended)
+ b MterpCommonTakenBranchNoFlags
diff --git a/runtime/interpreter/mterp/arm/op_goto_16.S b/runtime/interpreter/mterp/arm/op_goto_16.S
index 91639ca..12a6bc0 100644
--- a/runtime/interpreter/mterp/arm/op_goto_16.S
+++ b/runtime/interpreter/mterp/arm/op_goto_16.S
@@ -5,27 +5,5 @@
* double to get a byte offset.
*/
/* goto/16 +AAAA */
-#if MTERP_PROFILE_BRANCHES
FETCH_S rINST, 1 @ rINST<- ssssAAAA (sign-extended)
- EXPORT_PC
- mov r0, rSELF
- add r1, rFP, #OFF_FP_SHADOWFRAME
- mov r2, rINST
- bl MterpProfileBranch @ (self, shadow_frame, offset)
- cmp r0, #0
- bne MterpOnStackReplacement @ Note: offset must be in rINST
- ldr lr, [rSELF, #THREAD_FLAGS_OFFSET]
- adds r1, rINST, rINST @ r1<- byte offset, flags set
- FETCH_ADVANCE_INST_RB r1 @ update rPC, load rINST
- bmi MterpCheckSuspendAndContinue
- GET_INST_OPCODE ip @ extract opcode from rINST
- GOTO_OPCODE ip @ jump to next instruction
-#else
- FETCH_S rINST, 1 @ rINST<- ssssAAAA (sign-extended)
- ldr lr, [rSELF, #THREAD_FLAGS_OFFSET]
- adds r1, rINST, rINST @ r1<- byte offset, flags set
- FETCH_ADVANCE_INST_RB r1 @ update rPC, load rINST
- bmi MterpCheckSuspendAndContinue
- GET_INST_OPCODE ip @ extract opcode from rINST
- GOTO_OPCODE ip @ jump to next instruction
-#endif
+ b MterpCommonTakenBranchNoFlags
diff --git a/runtime/interpreter/mterp/arm/op_goto_32.S b/runtime/interpreter/mterp/arm/op_goto_32.S
index e730b52..7325a1c 100644
--- a/runtime/interpreter/mterp/arm/op_goto_32.S
+++ b/runtime/interpreter/mterp/arm/op_goto_32.S
@@ -10,31 +10,7 @@
* offset to byte offset.
*/
/* goto/32 +AAAAAAAA */
-#if MTERP_PROFILE_BRANCHES
FETCH r0, 1 @ r0<- aaaa (lo)
- FETCH r1, 2 @ r1<- AAAA (hi)
- orr rINST, r0, r1, lsl #16 @ rINST<- AAAAaaaa
- EXPORT_PC
- mov r0, rSELF
- add r1, rFP, #OFF_FP_SHADOWFRAME
- mov r2, rINST
- bl MterpProfileBranch @ (self, shadow_frame, offset)
- cmp r0, #0
- bne MterpOnStackReplacement @ Note: offset must be in rINST
- ldr lr, [rSELF, #THREAD_FLAGS_OFFSET]
- adds r1, rINST, rINST @ r1<- byte offset
- FETCH_ADVANCE_INST_RB r1 @ update rPC, load rINST
- ble MterpCheckSuspendAndContinue
- GET_INST_OPCODE ip @ extract opcode from rINST
- GOTO_OPCODE ip @ jump to next instruction
-#else
- FETCH r0, 1 @ r0<- aaaa (lo)
- FETCH r1, 2 @ r1<- AAAA (hi)
- orr rINST, r0, r1, lsl #16 @ rINST<- AAAAaaaa
- ldr lr, [rSELF, #THREAD_FLAGS_OFFSET]
- adds r1, rINST, rINST @ r1<- byte offset
- FETCH_ADVANCE_INST_RB r1 @ update rPC, load rINST
- ble MterpCheckSuspendAndContinue
- GET_INST_OPCODE ip @ extract opcode from rINST
- GOTO_OPCODE ip @ jump to next instruction
-#endif
+ FETCH r3, 2 @ r1<- AAAA (hi)
+ orrs rINST, r0, r3, lsl #16 @ rINST<- AAAAaaaa
+ b MterpCommonTakenBranch
diff --git a/runtime/interpreter/mterp/arm/op_if_eq.S b/runtime/interpreter/mterp/arm/op_if_eq.S
index 5685686..b8b6a6e 100644
--- a/runtime/interpreter/mterp/arm/op_if_eq.S
+++ b/runtime/interpreter/mterp/arm/op_if_eq.S
@@ -1 +1 @@
-%include "arm/bincmp.S" { "revcmp":"ne" }
+%include "arm/bincmp.S" { "condition":"eq" }
diff --git a/runtime/interpreter/mterp/arm/op_if_eqz.S b/runtime/interpreter/mterp/arm/op_if_eqz.S
index 2a9c0f9..7012f61 100644
--- a/runtime/interpreter/mterp/arm/op_if_eqz.S
+++ b/runtime/interpreter/mterp/arm/op_if_eqz.S
@@ -1 +1 @@
-%include "arm/zcmp.S" { "revcmp":"ne" }
+%include "arm/zcmp.S" { "condition":"eq" }
diff --git a/runtime/interpreter/mterp/arm/op_if_ge.S b/runtime/interpreter/mterp/arm/op_if_ge.S
index 60a0307..eb29e63 100644
--- a/runtime/interpreter/mterp/arm/op_if_ge.S
+++ b/runtime/interpreter/mterp/arm/op_if_ge.S
@@ -1 +1 @@
-%include "arm/bincmp.S" { "revcmp":"lt" }
+%include "arm/bincmp.S" { "condition":"ge" }
diff --git a/runtime/interpreter/mterp/arm/op_if_gez.S b/runtime/interpreter/mterp/arm/op_if_gez.S
index 981cdec..d9da374 100644
--- a/runtime/interpreter/mterp/arm/op_if_gez.S
+++ b/runtime/interpreter/mterp/arm/op_if_gez.S
@@ -1 +1 @@
-%include "arm/zcmp.S" { "revcmp":"lt" }
+%include "arm/zcmp.S" { "condition":"ge" }
diff --git a/runtime/interpreter/mterp/arm/op_if_gt.S b/runtime/interpreter/mterp/arm/op_if_gt.S
index ca50cd7..a35eab8 100644
--- a/runtime/interpreter/mterp/arm/op_if_gt.S
+++ b/runtime/interpreter/mterp/arm/op_if_gt.S
@@ -1 +1 @@
-%include "arm/bincmp.S" { "revcmp":"le" }
+%include "arm/bincmp.S" { "condition":"gt" }
diff --git a/runtime/interpreter/mterp/arm/op_if_gtz.S b/runtime/interpreter/mterp/arm/op_if_gtz.S
index c621812..4ef4d8e 100644
--- a/runtime/interpreter/mterp/arm/op_if_gtz.S
+++ b/runtime/interpreter/mterp/arm/op_if_gtz.S
@@ -1 +1 @@
-%include "arm/zcmp.S" { "revcmp":"le" }
+%include "arm/zcmp.S" { "condition":"gt" }
diff --git a/runtime/interpreter/mterp/arm/op_if_le.S b/runtime/interpreter/mterp/arm/op_if_le.S
index 7e060f2..c7c31bc 100644
--- a/runtime/interpreter/mterp/arm/op_if_le.S
+++ b/runtime/interpreter/mterp/arm/op_if_le.S
@@ -1 +1 @@
-%include "arm/bincmp.S" { "revcmp":"gt" }
+%include "arm/bincmp.S" { "condition":"le" }
diff --git a/runtime/interpreter/mterp/arm/op_if_lez.S b/runtime/interpreter/mterp/arm/op_if_lez.S
index f92be23..9fbf6c9 100644
--- a/runtime/interpreter/mterp/arm/op_if_lez.S
+++ b/runtime/interpreter/mterp/arm/op_if_lez.S
@@ -1 +1 @@
-%include "arm/zcmp.S" { "revcmp":"gt" }
+%include "arm/zcmp.S" { "condition":"le" }
diff --git a/runtime/interpreter/mterp/arm/op_if_lt.S b/runtime/interpreter/mterp/arm/op_if_lt.S
index 213344d..9469fbb 100644
--- a/runtime/interpreter/mterp/arm/op_if_lt.S
+++ b/runtime/interpreter/mterp/arm/op_if_lt.S
@@ -1 +1 @@
-%include "arm/bincmp.S" { "revcmp":"ge" }
+%include "arm/bincmp.S" { "condition":"lt" }
diff --git a/runtime/interpreter/mterp/arm/op_if_ltz.S b/runtime/interpreter/mterp/arm/op_if_ltz.S
index dfd4e44..a4fc1b8 100644
--- a/runtime/interpreter/mterp/arm/op_if_ltz.S
+++ b/runtime/interpreter/mterp/arm/op_if_ltz.S
@@ -1 +1 @@
-%include "arm/zcmp.S" { "revcmp":"ge" }
+%include "arm/zcmp.S" { "condition":"lt" }
diff --git a/runtime/interpreter/mterp/arm/op_if_ne.S b/runtime/interpreter/mterp/arm/op_if_ne.S
index 4a58b4a..c945331 100644
--- a/runtime/interpreter/mterp/arm/op_if_ne.S
+++ b/runtime/interpreter/mterp/arm/op_if_ne.S
@@ -1 +1 @@
-%include "arm/bincmp.S" { "revcmp":"eq" }
+%include "arm/bincmp.S" { "condition":"ne" }
diff --git a/runtime/interpreter/mterp/arm/op_if_nez.S b/runtime/interpreter/mterp/arm/op_if_nez.S
index d864ef4..2d81fda 100644
--- a/runtime/interpreter/mterp/arm/op_if_nez.S
+++ b/runtime/interpreter/mterp/arm/op_if_nez.S
@@ -1 +1 @@
-%include "arm/zcmp.S" { "revcmp":"eq" }
+%include "arm/zcmp.S" { "condition":"ne" }
diff --git a/runtime/interpreter/mterp/arm/op_mul_long.S b/runtime/interpreter/mterp/arm/op_mul_long.S
index 8f40f19..a13c803 100644
--- a/runtime/interpreter/mterp/arm/op_mul_long.S
+++ b/runtime/interpreter/mterp/arm/op_mul_long.S
@@ -24,13 +24,13 @@
VREG_INDEX_TO_ADDR r3, r3 @ r3<- &fp[CC]
ldmia r2, {r0-r1} @ r0/r1<- vBB/vBB+1
ldmia r3, {r2-r3} @ r2/r3<- vCC/vCC+1
- mul ip, r2, r1 @ ip<- ZxW
- umull r9, r10, r2, r0 @ r9/r10 <- ZxX
- mla r2, r0, r3, ip @ r2<- YxX + (ZxW)
+ mul ip, r2, r1 @ ip<- ZxW
+ umull r1, lr, r2, r0 @ r1/lr <- ZxX
+ mla r2, r0, r3, ip @ r2<- YxX + (ZxW)
mov r0, rINST, lsr #8 @ r0<- AA
- add r10, r2, r10 @ r10<- r10 + low(ZxW + (YxX))
+ add r2, r2, lr @ r2<- lr + low(ZxW + (YxX))
VREG_INDEX_TO_ADDR r0, r0 @ r0<- &fp[AA]
FETCH_ADVANCE_INST 2 @ advance rPC, load rINST
GET_INST_OPCODE ip @ extract opcode from rINST
- stmia r0, {r9-r10} @ vAA/vAA+1<- r9/r10
+ stmia r0, {r1-r2 } @ vAA/vAA+1<- r1/r2
GOTO_OPCODE ip @ jump to next instruction
diff --git a/runtime/interpreter/mterp/arm/op_mul_long_2addr.S b/runtime/interpreter/mterp/arm/op_mul_long_2addr.S
index 7ef24c5..4c1f058 100644
--- a/runtime/interpreter/mterp/arm/op_mul_long_2addr.S
+++ b/runtime/interpreter/mterp/arm/op_mul_long_2addr.S
@@ -13,12 +13,12 @@
VREG_INDEX_TO_ADDR rINST, r9 @ rINST<- &fp[A]
ldmia r1, {r2-r3} @ r2/r3<- vBB/vBB+1
ldmia rINST, {r0-r1} @ r0/r1<- vAA/vAA+1
- mul ip, r2, r1 @ ip<- ZxW
- umull r9, r10, r2, r0 @ r9/r10 <- ZxX
- mla r2, r0, r3, ip @ r2<- YxX + (ZxW)
+ mul ip, r2, r1 @ ip<- ZxW
+ umull r1, lr, r2, r0 @ r1/lr <- ZxX
+ mla r2, r0, r3, ip @ r2<- YxX + (ZxW)
mov r0, rINST @ r0<- &fp[A] (free up rINST)
FETCH_ADVANCE_INST 1 @ advance rPC, load rINST
- add r10, r2, r10 @ r10<- r10 + low(ZxW + (YxX))
+ add r2, r2, lr @ r2<- r2 + low(ZxW + (YxX))
GET_INST_OPCODE ip @ extract opcode from rINST
- stmia r0, {r9-r10} @ vAA/vAA+1<- r9/r10
+ stmia r0, {r1-r2} @ vAA/vAA+1<- r1/r2
GOTO_OPCODE ip @ jump to next instruction
diff --git a/runtime/interpreter/mterp/arm/op_packed_switch.S b/runtime/interpreter/mterp/arm/op_packed_switch.S
index 4c369cb..412c58f 100644
--- a/runtime/interpreter/mterp/arm/op_packed_switch.S
+++ b/runtime/interpreter/mterp/arm/op_packed_switch.S
@@ -9,7 +9,6 @@
* for: packed-switch, sparse-switch
*/
/* op vAA, +BBBB */
-#if MTERP_PROFILE_BRANCHES
FETCH r0, 1 @ r0<- bbbb (lo)
FETCH r1, 2 @ r1<- BBBB (hi)
mov r3, rINST, lsr #8 @ r3<- AA
@@ -17,33 +16,5 @@
GET_VREG r1, r3 @ r1<- vAA
add r0, rPC, r0, lsl #1 @ r0<- PC + BBBBbbbb*2
bl $func @ r0<- code-unit branch offset
- mov rINST, r0
- EXPORT_PC
- mov r0, rSELF
- add r1, rFP, #OFF_FP_SHADOWFRAME
- mov r2, rINST
- bl MterpProfileBranch @ (self, shadow_frame, offset)
- cmp r0, #0
- bne MterpOnStackReplacement @ Note: offset must be in rINST
- ldr lr, [rSELF, #THREAD_FLAGS_OFFSET]
- adds r1, rINST, rINST @ r1<- byte offset; clear V
- FETCH_ADVANCE_INST_RB r1 @ update rPC, load rINST
- ble MterpCheckSuspendAndContinue
- GET_INST_OPCODE ip @ extract opcode from rINST
- GOTO_OPCODE ip @ jump to next instruction
-#else
- FETCH r0, 1 @ r0<- bbbb (lo)
- FETCH r1, 2 @ r1<- BBBB (hi)
- mov r3, rINST, lsr #8 @ r3<- AA
- orr r0, r0, r1, lsl #16 @ r0<- BBBBbbbb
- GET_VREG r1, r3 @ r1<- vAA
- add r0, rPC, r0, lsl #1 @ r0<- PC + BBBBbbbb*2
- bl $func @ r0<- code-unit branch offset
- mov rINST, r0
- ldr lr, [rSELF, #THREAD_FLAGS_OFFSET]
- adds r1, rINST, rINST @ r1<- byte offset; clear V
- FETCH_ADVANCE_INST_RB r1 @ update rPC, load rINST
- ble MterpCheckSuspendAndContinue
- GET_INST_OPCODE ip @ extract opcode from rINST
- GOTO_OPCODE ip @ jump to next instruction
-#endif
+ movs rINST, r0
+ b MterpCommonTakenBranch
diff --git a/runtime/interpreter/mterp/arm/zcmp.S b/runtime/interpreter/mterp/arm/zcmp.S
index 3d7dec0..5db8b6c 100644
--- a/runtime/interpreter/mterp/arm/zcmp.S
+++ b/runtime/interpreter/mterp/arm/zcmp.S
@@ -1,29 +1,17 @@
/*
- * Generic one-operand compare-and-branch operation. Provide a "revcmp"
- * fragment that specifies the *reverse* comparison to perform, e.g.
- * for "if-le" you would use "gt".
+ * Generic one-operand compare-and-branch operation. Provide a "condition"
+ * fragment that specifies the comparison to perform.
*
* for: if-eqz, if-nez, if-ltz, if-gez, if-gtz, if-lez
*/
/* if-cmp vAA, +BBBB */
mov r0, rINST, lsr #8 @ r0<- AA
- GET_VREG r2, r0 @ r2<- vAA
+ GET_VREG r0, r0 @ r0<- vAA
FETCH_S rINST, 1 @ rINST<- branch offset, in code units
- ldr lr, [rSELF, #THREAD_FLAGS_OFFSET]
- cmp r2, #0 @ compare (vA, 0)
- mov${revcmp} rINST, #2
-#if MTERP_PROFILE_BRANCHES
- @ TUNING: once measurements are complete, remove #if and hand-schedule.
- EXPORT_PC
- mov r0, rSELF
- add r1, rFP, #OFF_FP_SHADOWFRAME
- mov r2, rINST
- bl MterpProfileBranch @ (self, shadow_frame, offset)
- cmp r0, #0
- bne MterpOnStackReplacement @ Note: offset must be in rINST
-#endif
- adds r1, rINST, rINST @ convert to bytes & set flags
- FETCH_ADVANCE_INST_RB r1 @ update rPC, load rINST
- bmi MterpCheckSuspendAndContinue
+ cmp r0, #0 @ compare (vA, 0)
+ b${condition} MterpCommonTakenBranchNoFlags
+ cmp rPROFILE, #JIT_CHECK_OSR @ possible OSR re-entry?
+ beq .L_check_not_taken_osr
+ FETCH_ADVANCE_INST 2
GET_INST_OPCODE ip @ extract opcode from rINST
GOTO_OPCODE ip @ jump to next instruction
diff --git a/runtime/interpreter/mterp/arm64/bincmp.S b/runtime/interpreter/mterp/arm64/bincmp.S
index 2356ecb..8dd4fed 100644
--- a/runtime/interpreter/mterp/arm64/bincmp.S
+++ b/runtime/interpreter/mterp/arm64/bincmp.S
@@ -1,7 +1,6 @@
/*
- * Generic two-operand compare-and-branch operation. Provide a "revcmp"
- * fragment that specifies the *reverse* comparison to perform, e.g.
- * for "if-le" you would use "gt".
+ * Generic two-operand compare-and-branch operation. Provide a "condition"
+ * fragment that specifies the comparison to perform.
*
* For: if-eq, if-ne, if-lt, if-ge, if-gt, if-le
*/
@@ -10,22 +9,11 @@
ubfx w0, wINST, #8, #4 // w0<- A
GET_VREG w3, w1 // w3<- vB
GET_VREG w2, w0 // w2<- vA
- FETCH_S w1, 1 // w1<- branch offset, in code units
- mov w0, #2 // Offset if branch not taken
+ FETCH_S wINST, 1 // wINST<- branch offset, in code units
cmp w2, w3 // compare (vA, vB)
- csel wINST, w1, w0, ${condition} // Branch if true, stashing result in callee save reg.
-#if MTERP_PROFILE_BRANCHES
- // TUINING: once measurements are complete, remove #if and hand-schedule.
- EXPORT_PC
- mov x0, xSELF
- add x1, xFP, #OFF_FP_SHADOWFRAME
- sbfm x2, xINST, 0, 31 // Sign extend branch offset
- bl MterpProfileBranch // (self, shadow_frame, offset)
- cbnz w0, MterpOnStackReplacement // Note: offset must be in xINST
-#endif
- ldr w7, [xSELF, #THREAD_FLAGS_OFFSET]
- adds w2, wINST, wINST // convert to bytes, check sign
- FETCH_ADVANCE_INST_RB w2 // update rPC, load wINST
- b.mi MterpCheckSuspendAndContinue
+ b.${condition} MterpCommonTakenBranchNoFlags
+ cmp wPROFILE, #JIT_CHECK_OSR // possible OSR re-entry?
+ b.eq .L_check_not_taken_osr
+ FETCH_ADVANCE_INST 2
GET_INST_OPCODE ip // extract opcode from wINST
GOTO_OPCODE ip // jump to next instruction
diff --git a/runtime/interpreter/mterp/arm64/entry.S b/runtime/interpreter/mterp/arm64/entry.S
index 23e656e..9fbbbd3 100644
--- a/runtime/interpreter/mterp/arm64/entry.S
+++ b/runtime/interpreter/mterp/arm64/entry.S
@@ -31,11 +31,12 @@
ExecuteMterpImpl:
.cfi_startproc
- stp xIBASE, xREFS, [sp, #-64]!
- stp xSELF, xINST, [sp, #16]
- stp xPC, xFP, [sp, #32]
- stp fp, lr, [sp, #48]
- add fp, sp, #48
+ stp xPROFILE, x27, [sp, #-80]!
+ stp xIBASE, xREFS, [sp, #16]
+ stp xSELF, xINST, [sp, #32]
+ stp xPC, xFP, [sp, #48]
+ stp fp, lr, [sp, #64]
+ add fp, sp, #64
/* Remember the return register */
str x3, [x2, #SHADOWFRAME_RESULT_REGISTER_OFFSET]
@@ -56,6 +57,12 @@
/* Starting ibase */
ldr xIBASE, [xSELF, #THREAD_CURRENT_IBASE_OFFSET]
+ /* Set up for backwards branches & osr profiling */
+ ldr x0, [xFP, #OFF_FP_METHOD]
+ add x1, xFP, #OFF_FP_SHADOWFRAME
+ bl MterpSetUpHotnessCountdown
+ mov wPROFILE, w0 // Starting hotness countdown to xPROFILE
+
/* start executing the instruction at rPC */
FETCH_INST // load wINST from rPC
GET_INST_OPCODE ip // extract opcode from wINST
diff --git a/runtime/interpreter/mterp/arm64/footer.S b/runtime/interpreter/mterp/arm64/footer.S
index aae78de..2d3a11e 100644
--- a/runtime/interpreter/mterp/arm64/footer.S
+++ b/runtime/interpreter/mterp/arm64/footer.S
@@ -107,6 +107,107 @@
GET_INST_OPCODE ip
GOTO_OPCODE ip
/* NOTE: no fallthrough */
+/*
+ * Common handling for branches with support for Jit profiling.
+ * On entry:
+ * wINST <= signed offset
+ * wPROFILE <= signed hotness countdown (expanded to 32 bits)
+ * condition bits <= set to establish sign of offset (use "NoFlags" entry if not)
+ *
+ * We have quite a few different cases for branch profiling, OSR detection and
+ * suspend check support here.
+ *
+ * Taken backward branches:
+ * If profiling active, do hotness countdown and report if we hit zero.
+ * If in osr check mode, see if our target is a compiled loop header entry and do OSR if so.
+ * Is there a pending suspend request? If so, suspend.
+ *
+ * Taken forward branches and not-taken backward branches:
+ * If in osr check mode, see if our target is a compiled loop header entry and do OSR if so.
+ *
+ * Our most common case is expected to be a taken backward branch with active jit profiling,
+ * but no full OSR check and no pending suspend request.
+ * Next most common case is not-taken branch with no full OSR check.
+ *
+ */
+MterpCommonTakenBranchNoFlags:
+ cmp wINST, #0
+ b.gt .L_forward_branch // don't add forward branches to hotness
+ tbnz wPROFILE, #31, .L_no_count_backwards // go if negative
+ subs wPROFILE, wPROFILE, #1 // countdown
+ b.eq .L_add_batch // counted down to zero - report
+.L_resume_backward_branch:
+ ldr lr, [xSELF, #THREAD_FLAGS_OFFSET]
+ add w2, wINST, wINST // w2<- byte offset
+ FETCH_ADVANCE_INST_RB w2 // update rPC, load wINST
+ REFRESH_IBASE
+ ands lr, lr, #(THREAD_SUSPEND_REQUEST | THREAD_CHECKPOINT_REQUEST)
+ b.ne .L_suspend_request_pending
+ GET_INST_OPCODE ip // extract opcode from wINST
+ GOTO_OPCODE ip // jump to next instruction
+
+.L_suspend_request_pending:
+ EXPORT_PC
+ mov x0, xSELF
+ bl MterpSuspendCheck // (self)
+ cbnz x0, MterpFallback
+ REFRESH_IBASE // might have changed during suspend
+ GET_INST_OPCODE ip // extract opcode from wINST
+ GOTO_OPCODE ip // jump to next instruction
+
+.L_no_count_backwards:
+ cmp wPROFILE, #JIT_CHECK_OSR // possible OSR re-entry?
+ b.ne .L_resume_backward_branch
+ mov x0, xSELF
+ add x1, xFP, #OFF_FP_SHADOWFRAME
+ mov x2, xINST
+ EXPORT_PC
+ bl MterpMaybeDoOnStackReplacement // (self, shadow_frame, offset)
+ cbnz x0, MterpOnStackReplacement
+ b .L_resume_backward_branch
+
+.L_forward_branch:
+ cmp wPROFILE, #JIT_CHECK_OSR // possible OSR re-entry?
+ b.eq .L_check_osr_forward
+.L_resume_forward_branch:
+ add w2, wINST, wINST // w2<- byte offset
+ FETCH_ADVANCE_INST_RB w2 // update rPC, load wINST
+ GET_INST_OPCODE ip // extract opcode from wINST
+ GOTO_OPCODE ip // jump to next instruction
+
+.L_check_osr_forward:
+ mov x0, xSELF
+ add x1, xFP, #OFF_FP_SHADOWFRAME
+ mov x2, xINST
+ EXPORT_PC
+ bl MterpMaybeDoOnStackReplacement // (self, shadow_frame, offset)
+ cbnz x0, MterpOnStackReplacement
+ b .L_resume_forward_branch
+
+.L_add_batch:
+ add x1, xFP, #OFF_FP_SHADOWFRAME
+ strh wPROFILE, [x1, #SHADOWFRAME_HOTNESS_COUNTDOWN_OFFSET]
+ ldr x0, [xFP, #OFF_FP_METHOD]
+ mov x2, xSELF
+ bl MterpAddHotnessBatch // (method, shadow_frame, self)
+ mov wPROFILE, w0 // restore new hotness countdown to wPROFILE
+ b .L_no_count_backwards
+
+/*
+ * Entered from the conditional branch handlers when OSR check request active on
+ * not-taken path. All Dalvik not-taken conditional branch offsets are 2.
+ */
+.L_check_not_taken_osr:
+ mov x0, xSELF
+ add x1, xFP, #OFF_FP_SHADOWFRAME
+ mov x2, #2
+ EXPORT_PC
+ bl MterpMaybeDoOnStackReplacement // (self, shadow_frame, offset)
+ cbnz x0, MterpOnStackReplacement
+ FETCH_ADVANCE_INST 2
+ GET_INST_OPCODE ip // extract opcode from wINST
+ GOTO_OPCODE ip // jump to next instruction
+
/*
* Check for suspend check request. Assumes wINST already loaded, xPC advanced and
@@ -175,10 +276,36 @@
check2:
mov x0, #1 // signal return to caller.
MterpDone:
- ldp fp, lr, [sp, #48]
- ldp xPC, xFP, [sp, #32]
- ldp xSELF, xINST, [sp, #16]
- ldp xIBASE, xREFS, [sp], #64
+/*
+ * At this point, we expect wPROFILE to be non-zero. If negative, hotness is disabled or we're
+ * checking for OSR. If greater than zero, we might have unreported hotness to register
+ * (the difference between the ending wPROFILE and the cached hotness counter). wPROFILE
+ * should only reach zero immediately after a hotness decrement, and is then reset to either
+ * a negative special state or the new non-zero countdown value.
+ */
+ cmp wPROFILE, #0
+ bgt MterpProfileActive // if > 0, we may have some counts to report.
+ ldp fp, lr, [sp, #64]
+ ldp xPC, xFP, [sp, #48]
+ ldp xSELF, xINST, [sp, #32]
+ ldp xIBASE, xREFS, [sp, #16]
+ ldp xPROFILE, x27, [sp], #80
+ ret
+
+MterpProfileActive:
+ mov xINST, x0 // stash return value
+ /* Report cached hotness counts */
+ ldr x0, [xFP, #OFF_FP_METHOD]
+ add x1, xFP, #OFF_FP_SHADOWFRAME
+ mov x2, xSELF
+ strh wPROFILE, [x1, #SHADOWFRAME_HOTNESS_COUNTDOWN_OFFSET]
+ bl MterpAddHotnessBatch // (method, shadow_frame, self)
+ mov x0, xINST // restore return value
+ ldp fp, lr, [sp, #64]
+ ldp xPC, xFP, [sp, #48]
+ ldp xSELF, xINST, [sp, #32]
+ ldp xIBASE, xREFS, [sp, #16]
+ ldp xPROFILE, x27, [sp], #80
ret
.cfi_endproc
diff --git a/runtime/interpreter/mterp/arm64/header.S b/runtime/interpreter/mterp/arm64/header.S
index 7101ba9..4257200 100644
--- a/runtime/interpreter/mterp/arm64/header.S
+++ b/runtime/interpreter/mterp/arm64/header.S
@@ -74,6 +74,7 @@
x23 xINST first 16-bit code unit of current instruction
x24 xIBASE interpreted instruction base pointer, used for computed goto
x25 xREFS base of object references in shadow frame (ideally, we'll get rid of this later).
+ x26 wPROFILE jit profile hotness countdown
x16 ip scratch reg
x17 ip2 scratch reg (used by macros)
@@ -92,15 +93,17 @@
/* During bringup, we'll use the shadow frame model instead of xFP */
/* single-purpose registers, given names for clarity */
-#define xPC x20
-#define xFP x21
-#define xSELF x22
-#define xINST x23
-#define wINST w23
-#define xIBASE x24
-#define xREFS x25
-#define ip x16
-#define ip2 x17
+#define xPC x20
+#define xFP x21
+#define xSELF x22
+#define xINST x23
+#define wINST w23
+#define xIBASE x24
+#define xREFS x25
+#define wPROFILE w26
+#define xPROFILE x26
+#define ip x16
+#define ip2 x17
/*
* Instead of holding a pointer to the shadow frame, we keep xFP at the base of the vregs. So,
@@ -114,7 +117,7 @@
#define OFF_FP_RESULT_REGISTER OFF_FP(SHADOWFRAME_RESULT_REGISTER_OFFSET)
#define OFF_FP_DEX_PC_PTR OFF_FP(SHADOWFRAME_DEX_PC_PTR_OFFSET)
#define OFF_FP_CODE_ITEM OFF_FP(SHADOWFRAME_CODE_ITEM_OFFSET)
-#define OFF_FP_SHADOWFRAME (-SHADOWFRAME_VREGS_OFFSET)
+#define OFF_FP_SHADOWFRAME OFF_FP(0)
/*
* "export" the PC to dex_pc field in the shadow frame, f/b/o future exception objects. Must
diff --git a/runtime/interpreter/mterp/arm64/op_goto.S b/runtime/interpreter/mterp/arm64/op_goto.S
index 7e2f6a9..6381e94 100644
--- a/runtime/interpreter/mterp/arm64/op_goto.S
+++ b/runtime/interpreter/mterp/arm64/op_goto.S
@@ -5,21 +5,5 @@
* double to get a byte offset.
*/
/* goto +AA */
- /* tuning: use sbfx for 6t2+ targets */
- lsl w0, wINST, #16 // w0<- AAxx0000
- asr wINST, w0, #24 // wINST<- ssssssAA (sign-extended)
-#if MTERP_PROFILE_BRANCHES
- EXPORT_PC
- mov x0, xSELF
- add x1, xFP, #OFF_FP_SHADOWFRAME
- sbfm x2, xINST, 0, 31
- bl MterpProfileBranch // (self, shadow_frame, offset)
- cbnz w0, MterpOnStackReplacement // Note: offset must be in wINST
-#endif
- ldr w7, [xSELF, #THREAD_FLAGS_OFFSET] // Preload flags for MterpCheckSuspendAndContinue
- adds w1, wINST, wINST // Convert dalvik offset to byte offset, setting flags
- FETCH_ADVANCE_INST_RB w1 // load wINST and advance xPC
- // If backwards branch refresh rIBASE
- b.mi MterpCheckSuspendAndContinue
- GET_INST_OPCODE ip // extract opcode from wINST
- GOTO_OPCODE ip // jump to next instruction
+ sbfx wINST, wINST, #8, #8 // wINST<- ssssssAA (sign-extended)
+ b MterpCommonTakenBranchNoFlags
diff --git a/runtime/interpreter/mterp/arm64/op_goto_16.S b/runtime/interpreter/mterp/arm64/op_goto_16.S
index b2b9924..fb9a80a 100644
--- a/runtime/interpreter/mterp/arm64/op_goto_16.S
+++ b/runtime/interpreter/mterp/arm64/op_goto_16.S
@@ -6,17 +6,4 @@
*/
/* goto/16 +AAAA */
FETCH_S wINST, 1 // wINST<- ssssAAAA (sign-extended)
-#if MTERP_PROFILE_BRANCHES
- EXPORT_PC
- mov x0, xSELF
- add x1, xFP, #OFF_FP_SHADOWFRAME
- sbfm x2, xINST, 0, 31
- bl MterpProfileBranch // (self, shadow_frame, offset)
- cbnz w0, MterpOnStackReplacement // Note: offset must be in xINST
-#endif
- ldr w7, [xSELF, #THREAD_FLAGS_OFFSET]
- adds w1, wINST, wINST // w1<- byte offset, flags set
- FETCH_ADVANCE_INST_RB w1 // update rPC, load rINST
- b.mi MterpCheckSuspendAndContinue
- GET_INST_OPCODE ip // extract opcode from rINST
- GOTO_OPCODE ip // jump to next instruction
+ b MterpCommonTakenBranchNoFlags
diff --git a/runtime/interpreter/mterp/arm64/op_goto_32.S b/runtime/interpreter/mterp/arm64/op_goto_32.S
index b785857..b13cb41 100644
--- a/runtime/interpreter/mterp/arm64/op_goto_32.S
+++ b/runtime/interpreter/mterp/arm64/op_goto_32.S
@@ -13,17 +13,4 @@
FETCH w0, 1 // w0<- aaaa (lo)
FETCH w1, 2 // w1<- AAAA (hi)
orr wINST, w0, w1, lsl #16 // wINST<- AAAAaaaa
-#if MTERP_PROFILE_BRANCHES
- EXPORT_PC
- mov x0, xSELF
- add x1, xFP, #OFF_FP_SHADOWFRAME
- sbfm x2, xINST, 0, 31
- bl MterpProfileBranch // (self, shadow_frame, offset)
- cbnz w0, MterpOnStackReplacement // Note: offset must be in xINST
-#endif
- ldr w7, [xSELF, #THREAD_FLAGS_OFFSET]
- adds w1, wINST, wINST // w1<- byte offset
- FETCH_ADVANCE_INST_RB w1 // update rPC, load xINST
- b.le MterpCheckSuspendAndContinue
- GET_INST_OPCODE ip // extract opcode from xINST
- GOTO_OPCODE ip // jump to next instruction
+ b MterpCommonTakenBranchNoFlags
diff --git a/runtime/interpreter/mterp/arm64/op_packed_switch.S b/runtime/interpreter/mterp/arm64/op_packed_switch.S
index e8b4f04..1456f1a 100644
--- a/runtime/interpreter/mterp/arm64/op_packed_switch.S
+++ b/runtime/interpreter/mterp/arm64/op_packed_switch.S
@@ -17,17 +17,4 @@
add x0, xPC, w0, lsl #1 // w0<- PC + BBBBbbbb*2
bl $func // w0<- code-unit branch offset
sbfm xINST, x0, 0, 31
-#if MTERP_PROFILE_BRANCHES
- EXPORT_PC
- mov x0, xSELF
- add x1, xFP, #OFF_FP_SHADOWFRAME
- mov x2, xINST
- bl MterpProfileBranch // (self, shadow_frame, offset)
- cbnz w0, MterpOnStackReplacement
-#endif
- ldr w7, [xSELF, #THREAD_FLAGS_OFFSET]
- adds w1, wINST, wINST // w1<- byte offset; clear V
- FETCH_ADVANCE_INST_RB w1 // update rPC, load wINST
- b.le MterpCheckSuspendAndContinue
- GET_INST_OPCODE ip // extract opcode from wINST
- GOTO_OPCODE ip // jump to next instruction
+ b MterpCommonTakenBranchNoFlags
diff --git a/runtime/interpreter/mterp/arm64/zcmp.S b/runtime/interpreter/mterp/arm64/zcmp.S
index 3f1e1b1..b303e6a 100644
--- a/runtime/interpreter/mterp/arm64/zcmp.S
+++ b/runtime/interpreter/mterp/arm64/zcmp.S
@@ -1,29 +1,17 @@
/*
- * Generic one-operand compare-and-branch operation. Provide a "revcmp"
- * fragment that specifies the *reverse* comparison to perform, e.g.
- * for "if-le" you would use "gt".
+ * Generic one-operand compare-and-branch operation. Provide a "condition"
+ * fragment that specifies the comparison to perform.
*
* for: if-eqz, if-nez, if-ltz, if-gez, if-gtz, if-lez
*/
/* if-cmp vAA, +BBBB */
lsr w0, wINST, #8 // w0<- AA
GET_VREG w2, w0 // w2<- vAA
- FETCH_S w1, 1 // w1<- branch offset, in code units
- mov w0, #2 // Branch offset if not taken
+ FETCH_S wINST, 1 // w1<- branch offset, in code units
cmp w2, #0 // compare (vA, 0)
- csel wINST, w1, w0, ${condition} // Branch if true, stashing result in callee save reg
-#if MTERP_PROFILE_BRANCHES
- // TUNING: once measurements are complete, remove #if and hand-schedule.
- EXPORT_PC
- mov x0, xSELF
- add x1, xFP, #OFF_FP_SHADOWFRAME
- sbfm x2, xINST, 0, 31
- bl MterpProfileBranch // (self, shadow_frame, offset)
- cbnz w0, MterpOnStackReplacement // Note: offset must be in wINST
-#endif
- ldr w7, [xSELF, #THREAD_FLAGS_OFFSET]
- adds w2, wINST, wINST // convert to bytes & set flags
- FETCH_ADVANCE_INST_RB w2 // update rPC, load wINST
- b.mi MterpCheckSuspendAndContinue
+ b.${condition} MterpCommonTakenBranchNoFlags
+ cmp wPROFILE, #JIT_CHECK_OSR // possible OSR re-entry?
+ b.eq .L_check_not_taken_osr
+ FETCH_ADVANCE_INST 2
GET_INST_OPCODE ip // extract opcode from wINST
GOTO_OPCODE ip // jump to next instruction
diff --git a/runtime/interpreter/mterp/mterp.cc b/runtime/interpreter/mterp/mterp.cc
index 10b19c5..cbfdcc3 100644
--- a/runtime/interpreter/mterp/mterp.cc
+++ b/runtime/interpreter/mterp/mterp.cc
@@ -21,6 +21,7 @@
#include "entrypoints/entrypoint_utils-inl.h"
#include "mterp.h"
#include "jit/jit.h"
+#include "jit/jit_instrumentation.h"
#include "debugger.h"
namespace art {
@@ -432,7 +433,7 @@
}
extern "C" void MterpCheckBefore(Thread* self, ShadowFrame* shadow_frame)
- SHARED_REQUIRES(Locks::mutator_lock_) {
+ SHARED_REQUIRES(Locks::mutator_lock_) {
const Instruction* inst = Instruction::At(shadow_frame->GetDexPCPtr());
uint16_t inst_data = inst->Fetch16(0);
if (inst->Opcode(inst_data) == Instruction::MOVE_EXCEPTION) {
@@ -444,7 +445,7 @@
}
extern "C" void MterpLogDivideByZeroException(Thread* self, ShadowFrame* shadow_frame)
- SHARED_REQUIRES(Locks::mutator_lock_) {
+ SHARED_REQUIRES(Locks::mutator_lock_) {
UNUSED(self);
const Instruction* inst = Instruction::At(shadow_frame->GetDexPCPtr());
uint16_t inst_data = inst->Fetch16(0);
@@ -452,7 +453,7 @@
}
extern "C" void MterpLogArrayIndexException(Thread* self, ShadowFrame* shadow_frame)
- SHARED_REQUIRES(Locks::mutator_lock_) {
+ SHARED_REQUIRES(Locks::mutator_lock_) {
UNUSED(self);
const Instruction* inst = Instruction::At(shadow_frame->GetDexPCPtr());
uint16_t inst_data = inst->Fetch16(0);
@@ -460,7 +461,7 @@
}
extern "C" void MterpLogNegativeArraySizeException(Thread* self, ShadowFrame* shadow_frame)
- SHARED_REQUIRES(Locks::mutator_lock_) {
+ SHARED_REQUIRES(Locks::mutator_lock_) {
UNUSED(self);
const Instruction* inst = Instruction::At(shadow_frame->GetDexPCPtr());
uint16_t inst_data = inst->Fetch16(0);
@@ -468,7 +469,7 @@
}
extern "C" void MterpLogNoSuchMethodException(Thread* self, ShadowFrame* shadow_frame)
- SHARED_REQUIRES(Locks::mutator_lock_) {
+ SHARED_REQUIRES(Locks::mutator_lock_) {
UNUSED(self);
const Instruction* inst = Instruction::At(shadow_frame->GetDexPCPtr());
uint16_t inst_data = inst->Fetch16(0);
@@ -476,7 +477,7 @@
}
extern "C" void MterpLogExceptionThrownException(Thread* self, ShadowFrame* shadow_frame)
- SHARED_REQUIRES(Locks::mutator_lock_) {
+ SHARED_REQUIRES(Locks::mutator_lock_) {
UNUSED(self);
const Instruction* inst = Instruction::At(shadow_frame->GetDexPCPtr());
uint16_t inst_data = inst->Fetch16(0);
@@ -484,7 +485,7 @@
}
extern "C" void MterpLogNullObjectException(Thread* self, ShadowFrame* shadow_frame)
- SHARED_REQUIRES(Locks::mutator_lock_) {
+ SHARED_REQUIRES(Locks::mutator_lock_) {
UNUSED(self);
const Instruction* inst = Instruction::At(shadow_frame->GetDexPCPtr());
uint16_t inst_data = inst->Fetch16(0);
@@ -492,7 +493,7 @@
}
extern "C" void MterpLogFallback(Thread* self, ShadowFrame* shadow_frame)
- SHARED_REQUIRES(Locks::mutator_lock_) {
+ SHARED_REQUIRES(Locks::mutator_lock_) {
UNUSED(self);
const Instruction* inst = Instruction::At(shadow_frame->GetDexPCPtr());
uint16_t inst_data = inst->Fetch16(0);
@@ -501,7 +502,7 @@
}
extern "C" void MterpLogOSR(Thread* self, ShadowFrame* shadow_frame, int32_t offset)
- SHARED_REQUIRES(Locks::mutator_lock_) {
+ SHARED_REQUIRES(Locks::mutator_lock_) {
UNUSED(self);
const Instruction* inst = Instruction::At(shadow_frame->GetDexPCPtr());
uint16_t inst_data = inst->Fetch16(0);
@@ -509,7 +510,7 @@
}
extern "C" void MterpLogSuspendFallback(Thread* self, ShadowFrame* shadow_frame, uint32_t flags)
- SHARED_REQUIRES(Locks::mutator_lock_) {
+ SHARED_REQUIRES(Locks::mutator_lock_) {
UNUSED(self);
const Instruction* inst = Instruction::At(shadow_frame->GetDexPCPtr());
uint16_t inst_data = inst->Fetch16(0);
@@ -521,7 +522,7 @@
}
extern "C" bool MterpSuspendCheck(Thread* self)
- SHARED_REQUIRES(Locks::mutator_lock_) {
+ SHARED_REQUIRES(Locks::mutator_lock_) {
self->AllowThreadSuspension();
return MterpShouldSwitchInterpreters();
}
@@ -617,7 +618,7 @@
}
extern "C" mirror::Object* artAGetObjectFromMterp(mirror::Object* arr, int32_t index)
- SHARED_REQUIRES(Locks::mutator_lock_) {
+ SHARED_REQUIRES(Locks::mutator_lock_) {
if (UNLIKELY(arr == nullptr)) {
ThrowNullPointerExceptionFromInterpreter();
return nullptr;
@@ -631,7 +632,7 @@
}
extern "C" mirror::Object* artIGetObjectFromMterp(mirror::Object* obj, uint32_t field_offset)
- SHARED_REQUIRES(Locks::mutator_lock_) {
+ SHARED_REQUIRES(Locks::mutator_lock_) {
if (UNLIKELY(obj == nullptr)) {
ThrowNullPointerExceptionFromInterpreter();
return nullptr;
@@ -639,13 +640,85 @@
return obj->GetFieldObject<mirror::Object>(MemberOffset(field_offset));
}
+/*
+ * Create a hotness_countdown based on the current method hotness_count and profiling
+ * mode. In short, determine how many hotness events we hit before reporting back
+ * to the full instrumentation via MterpAddHotnessBatch. Called once on entry to the method,
+ * and regenerated following batch updates.
+ */
+extern "C" int MterpSetUpHotnessCountdown(ArtMethod* method, ShadowFrame* shadow_frame)
+ SHARED_REQUIRES(Locks::mutator_lock_) {
+ uint16_t hotness_count = method->GetCounter();
+ int32_t countdown_value = jit::kJitHotnessDisabled;
+ jit::Jit* jit = Runtime::Current()->GetJit();
+ if (jit != nullptr) {
+ jit::JitInstrumentationCache* cache = jit->GetInstrumentationCache();
+ int32_t warm_threshold = cache->WarmMethodThreshold();
+ int32_t hot_threshold = cache->HotMethodThreshold();
+ int32_t osr_threshold = cache->OSRMethodThreshold();
+ if (hotness_count < warm_threshold) {
+ countdown_value = warm_threshold - hotness_count;
+ } else if (hotness_count < hot_threshold) {
+ countdown_value = hot_threshold - hotness_count;
+ } else if (hotness_count < osr_threshold) {
+ countdown_value = osr_threshold - hotness_count;
+ } else {
+ countdown_value = jit::kJitCheckForOSR;
+ }
+ }
+ /*
+ * The actual hotness threshold may exceed the range of our int16_t countdown value. This is
+ * not a problem, though. We can just break it down into smaller chunks.
+ */
+ countdown_value = std::min(countdown_value,
+ static_cast<int32_t>(std::numeric_limits<int16_t>::max()));
+ shadow_frame->SetCachedHotnessCountdown(countdown_value);
+ shadow_frame->SetHotnessCountdown(countdown_value);
+ return countdown_value;
+}
+
+/*
+ * Report a batch of hotness events to the instrumentation and then return the new
+ * countdown value to the next time we should report.
+ */
+extern "C" int16_t MterpAddHotnessBatch(ArtMethod* method,
+ ShadowFrame* shadow_frame,
+ Thread* self)
+ SHARED_REQUIRES(Locks::mutator_lock_) {
+ jit::Jit* jit = Runtime::Current()->GetJit();
+ if (jit != nullptr) {
+ int16_t count = shadow_frame->GetCachedHotnessCountdown() - shadow_frame->GetHotnessCountdown();
+ jit->GetInstrumentationCache()->AddSamples(self, method, count);
+ }
+ return MterpSetUpHotnessCountdown(method, shadow_frame);
+}
+
+// TUNING: Unused by arm/arm64/x86/x86_64. Remove when mips/mips64 mterps support batch updates.
extern "C" bool MterpProfileBranch(Thread* self, ShadowFrame* shadow_frame, int32_t offset)
- SHARED_REQUIRES(Locks::mutator_lock_) {
+ SHARED_REQUIRES(Locks::mutator_lock_) {
ArtMethod* method = shadow_frame->GetMethod();
JValue* result = shadow_frame->GetResultRegister();
uint32_t dex_pc = shadow_frame->GetDexPC();
- const auto* const instrumentation = Runtime::Current()->GetInstrumentation();
- instrumentation->Branch(self, method, dex_pc, offset);
+ jit::Jit* jit = Runtime::Current()->GetJit();
+ if ((jit != nullptr) && (offset <= 0)) {
+ jit->GetInstrumentationCache()->AddSamples(self, method, 1);
+ }
+ int16_t countdown_value = MterpSetUpHotnessCountdown(method, shadow_frame);
+ if (countdown_value == jit::kJitCheckForOSR) {
+ return jit::Jit::MaybeDoOnStackReplacement(self, method, dex_pc, offset, result);
+ } else {
+ return false;
+ }
+}
+
+extern "C" bool MterpMaybeDoOnStackReplacement(Thread* self,
+ ShadowFrame* shadow_frame,
+ int32_t offset)
+ SHARED_REQUIRES(Locks::mutator_lock_) {
+ ArtMethod* method = shadow_frame->GetMethod();
+ JValue* result = shadow_frame->GetResultRegister();
+ uint32_t dex_pc = shadow_frame->GetDexPC();
+ // Assumes caller has already determined that an OSR check is appropriate.
return jit::Jit::MaybeDoOnStackReplacement(self, method, dex_pc, offset, result);
}
diff --git a/runtime/interpreter/mterp/out/mterp_arm.S b/runtime/interpreter/mterp/out/mterp_arm.S
index 092474d..a38a87b 100644
--- a/runtime/interpreter/mterp/out/mterp_arm.S
+++ b/runtime/interpreter/mterp/out/mterp_arm.S
@@ -79,7 +79,8 @@
r6 rSELF self (Thread) pointer
r7 rINST first 16-bit code unit of current instruction
r8 rIBASE interpreted instruction base pointer, used for computed goto
- r11 rREFS base of object references in shadow frame (ideally, we'll get rid of this later).
+ r10 rPROFILE branch profiling countdown
+ r11 rREFS base of object references in shadow frame (ideally, we'll get rid of this later).
Macros are provided for common operations. Each macro MUST emit only
one instruction to make instruction-counting easier. They MUST NOT alter
@@ -97,12 +98,13 @@
/* During bringup, we'll use the shadow frame model instead of rFP */
/* single-purpose registers, given names for clarity */
-#define rPC r4
-#define rFP r5
-#define rSELF r6
-#define rINST r7
-#define rIBASE r8
-#define rREFS r11
+#define rPC r4
+#define rFP r5
+#define rSELF r6
+#define rINST r7
+#define rIBASE r8
+#define rPROFILE r10
+#define rREFS r11
/*
* Instead of holding a pointer to the shadow frame, we keep rFP at the base of the vregs. So,
@@ -116,7 +118,7 @@
#define OFF_FP_RESULT_REGISTER OFF_FP(SHADOWFRAME_RESULT_REGISTER_OFFSET)
#define OFF_FP_DEX_PC_PTR OFF_FP(SHADOWFRAME_DEX_PC_PTR_OFFSET)
#define OFF_FP_CODE_ITEM OFF_FP(SHADOWFRAME_CODE_ITEM_OFFSET)
-#define OFF_FP_SHADOWFRAME (-SHADOWFRAME_VREGS_OFFSET)
+#define OFF_FP_SHADOWFRAME OFF_FP(0)
/*
* "export" the PC to dex_pc field in the shadow frame, f/b/o future exception objects. Must
@@ -329,10 +331,8 @@
ExecuteMterpImpl:
.fnstart
- .save {r4-r10,fp,lr}
- stmfd sp!, {r4-r10,fp,lr} @ save 9 regs
- .pad #4
- sub sp, sp, #4 @ align 64
+ .save {r3-r10,fp,lr}
+ stmfd sp!, {r3-r10,fp,lr} @ save 10 regs, (r3 just to align 64)
/* Remember the return register */
str r3, [r2, #SHADOWFRAME_RESULT_REGISTER_OFFSET]
@@ -353,6 +353,12 @@
/* Starting ibase */
ldr rIBASE, [rSELF, #THREAD_CURRENT_IBASE_OFFSET]
+ /* Set up for backwards branches & osr profiling */
+ ldr r0, [rFP, #OFF_FP_METHOD]
+ add r1, rFP, #OFF_FP_SHADOWFRAME
+ bl MterpSetUpHotnessCountdown
+ mov rPROFILE, r0 @ Starting hotness countdown to rPROFILE
+
/* start executing the instruction at rPC */
FETCH_INST @ load rINST from rPC
GET_INST_OPCODE ip @ extract opcode from rINST
@@ -1103,35 +1109,8 @@
* double to get a byte offset.
*/
/* goto +AA */
- /* tuning: use sbfx for 6t2+ targets */
-#if MTERP_PROFILE_BRANCHES
- mov r0, rINST, lsl #16 @ r0<- AAxx0000
- movs rINST, r0, asr #24 @ rINST<- ssssssAA (sign-extended)
- EXPORT_PC
- mov r0, rSELF
- add r1, rFP, #OFF_FP_SHADOWFRAME
- mov r2, rINST
- bl MterpProfileBranch @ (self, shadow_frame, offset)
- cmp r0, #0
- bne MterpOnStackReplacement @ Note: offset must be in rINST
- ldr lr, [rSELF, #THREAD_FLAGS_OFFSET]
- adds r2, rINST, rINST @ r2<- byte offset, set flags
- FETCH_ADVANCE_INST_RB r2 @ update rPC, load rINST
- @ If backwards branch refresh rIBASE
- bmi MterpCheckSuspendAndContinue
- GET_INST_OPCODE ip @ extract opcode from rINST
- GOTO_OPCODE ip @ jump to next instruction
-#else
- mov r0, rINST, lsl #16 @ r0<- AAxx0000
- movs rINST, r0, asr #24 @ rINST<- ssssssAA (sign-extended)
- ldr lr, [rSELF, #THREAD_FLAGS_OFFSET]
- adds r2, rINST, rINST @ r2<- byte offset, set flags
- FETCH_ADVANCE_INST_RB r2 @ update rPC, load rINST
- @ If backwards branch refresh rIBASE
- bmi MterpCheckSuspendAndContinue
- GET_INST_OPCODE ip @ extract opcode from rINST
- GOTO_OPCODE ip @ jump to next instruction
-#endif
+ sbfx rINST, rINST, #8, #8 @ rINST<- ssssssAA (sign-extended)
+ b MterpCommonTakenBranchNoFlags
/* ------------------------------ */
.balign 128
@@ -1144,30 +1123,8 @@
* double to get a byte offset.
*/
/* goto/16 +AAAA */
-#if MTERP_PROFILE_BRANCHES
FETCH_S rINST, 1 @ rINST<- ssssAAAA (sign-extended)
- EXPORT_PC
- mov r0, rSELF
- add r1, rFP, #OFF_FP_SHADOWFRAME
- mov r2, rINST
- bl MterpProfileBranch @ (self, shadow_frame, offset)
- cmp r0, #0
- bne MterpOnStackReplacement @ Note: offset must be in rINST
- ldr lr, [rSELF, #THREAD_FLAGS_OFFSET]
- adds r1, rINST, rINST @ r1<- byte offset, flags set
- FETCH_ADVANCE_INST_RB r1 @ update rPC, load rINST
- bmi MterpCheckSuspendAndContinue
- GET_INST_OPCODE ip @ extract opcode from rINST
- GOTO_OPCODE ip @ jump to next instruction
-#else
- FETCH_S rINST, 1 @ rINST<- ssssAAAA (sign-extended)
- ldr lr, [rSELF, #THREAD_FLAGS_OFFSET]
- adds r1, rINST, rINST @ r1<- byte offset, flags set
- FETCH_ADVANCE_INST_RB r1 @ update rPC, load rINST
- bmi MterpCheckSuspendAndContinue
- GET_INST_OPCODE ip @ extract opcode from rINST
- GOTO_OPCODE ip @ jump to next instruction
-#endif
+ b MterpCommonTakenBranchNoFlags
/* ------------------------------ */
.balign 128
@@ -1185,34 +1142,10 @@
* offset to byte offset.
*/
/* goto/32 +AAAAAAAA */
-#if MTERP_PROFILE_BRANCHES
FETCH r0, 1 @ r0<- aaaa (lo)
- FETCH r1, 2 @ r1<- AAAA (hi)
- orr rINST, r0, r1, lsl #16 @ rINST<- AAAAaaaa
- EXPORT_PC
- mov r0, rSELF
- add r1, rFP, #OFF_FP_SHADOWFRAME
- mov r2, rINST
- bl MterpProfileBranch @ (self, shadow_frame, offset)
- cmp r0, #0
- bne MterpOnStackReplacement @ Note: offset must be in rINST
- ldr lr, [rSELF, #THREAD_FLAGS_OFFSET]
- adds r1, rINST, rINST @ r1<- byte offset
- FETCH_ADVANCE_INST_RB r1 @ update rPC, load rINST
- ble MterpCheckSuspendAndContinue
- GET_INST_OPCODE ip @ extract opcode from rINST
- GOTO_OPCODE ip @ jump to next instruction
-#else
- FETCH r0, 1 @ r0<- aaaa (lo)
- FETCH r1, 2 @ r1<- AAAA (hi)
- orr rINST, r0, r1, lsl #16 @ rINST<- AAAAaaaa
- ldr lr, [rSELF, #THREAD_FLAGS_OFFSET]
- adds r1, rINST, rINST @ r1<- byte offset
- FETCH_ADVANCE_INST_RB r1 @ update rPC, load rINST
- ble MterpCheckSuspendAndContinue
- GET_INST_OPCODE ip @ extract opcode from rINST
- GOTO_OPCODE ip @ jump to next instruction
-#endif
+ FETCH r3, 2 @ r1<- AAAA (hi)
+ orrs rINST, r0, r3, lsl #16 @ rINST<- AAAAaaaa
+ b MterpCommonTakenBranch
/* ------------------------------ */
.balign 128
@@ -1228,7 +1161,6 @@
* for: packed-switch, sparse-switch
*/
/* op vAA, +BBBB */
-#if MTERP_PROFILE_BRANCHES
FETCH r0, 1 @ r0<- bbbb (lo)
FETCH r1, 2 @ r1<- BBBB (hi)
mov r3, rINST, lsr #8 @ r3<- AA
@@ -1236,36 +1168,8 @@
GET_VREG r1, r3 @ r1<- vAA
add r0, rPC, r0, lsl #1 @ r0<- PC + BBBBbbbb*2
bl MterpDoPackedSwitch @ r0<- code-unit branch offset
- mov rINST, r0
- EXPORT_PC
- mov r0, rSELF
- add r1, rFP, #OFF_FP_SHADOWFRAME
- mov r2, rINST
- bl MterpProfileBranch @ (self, shadow_frame, offset)
- cmp r0, #0
- bne MterpOnStackReplacement @ Note: offset must be in rINST
- ldr lr, [rSELF, #THREAD_FLAGS_OFFSET]
- adds r1, rINST, rINST @ r1<- byte offset; clear V
- FETCH_ADVANCE_INST_RB r1 @ update rPC, load rINST
- ble MterpCheckSuspendAndContinue
- GET_INST_OPCODE ip @ extract opcode from rINST
- GOTO_OPCODE ip @ jump to next instruction
-#else
- FETCH r0, 1 @ r0<- bbbb (lo)
- FETCH r1, 2 @ r1<- BBBB (hi)
- mov r3, rINST, lsr #8 @ r3<- AA
- orr r0, r0, r1, lsl #16 @ r0<- BBBBbbbb
- GET_VREG r1, r3 @ r1<- vAA
- add r0, rPC, r0, lsl #1 @ r0<- PC + BBBBbbbb*2
- bl MterpDoPackedSwitch @ r0<- code-unit branch offset
- mov rINST, r0
- ldr lr, [rSELF, #THREAD_FLAGS_OFFSET]
- adds r1, rINST, rINST @ r1<- byte offset; clear V
- FETCH_ADVANCE_INST_RB r1 @ update rPC, load rINST
- ble MterpCheckSuspendAndContinue
- GET_INST_OPCODE ip @ extract opcode from rINST
- GOTO_OPCODE ip @ jump to next instruction
-#endif
+ movs rINST, r0
+ b MterpCommonTakenBranch
/* ------------------------------ */
.balign 128
@@ -1282,7 +1186,6 @@
* for: packed-switch, sparse-switch
*/
/* op vAA, +BBBB */
-#if MTERP_PROFILE_BRANCHES
FETCH r0, 1 @ r0<- bbbb (lo)
FETCH r1, 2 @ r1<- BBBB (hi)
mov r3, rINST, lsr #8 @ r3<- AA
@@ -1290,36 +1193,8 @@
GET_VREG r1, r3 @ r1<- vAA
add r0, rPC, r0, lsl #1 @ r0<- PC + BBBBbbbb*2
bl MterpDoSparseSwitch @ r0<- code-unit branch offset
- mov rINST, r0
- EXPORT_PC
- mov r0, rSELF
- add r1, rFP, #OFF_FP_SHADOWFRAME
- mov r2, rINST
- bl MterpProfileBranch @ (self, shadow_frame, offset)
- cmp r0, #0
- bne MterpOnStackReplacement @ Note: offset must be in rINST
- ldr lr, [rSELF, #THREAD_FLAGS_OFFSET]
- adds r1, rINST, rINST @ r1<- byte offset; clear V
- FETCH_ADVANCE_INST_RB r1 @ update rPC, load rINST
- ble MterpCheckSuspendAndContinue
- GET_INST_OPCODE ip @ extract opcode from rINST
- GOTO_OPCODE ip @ jump to next instruction
-#else
- FETCH r0, 1 @ r0<- bbbb (lo)
- FETCH r1, 2 @ r1<- BBBB (hi)
- mov r3, rINST, lsr #8 @ r3<- AA
- orr r0, r0, r1, lsl #16 @ r0<- BBBBbbbb
- GET_VREG r1, r3 @ r1<- vAA
- add r0, rPC, r0, lsl #1 @ r0<- PC + BBBBbbbb*2
- bl MterpDoSparseSwitch @ r0<- code-unit branch offset
- mov rINST, r0
- ldr lr, [rSELF, #THREAD_FLAGS_OFFSET]
- adds r1, rINST, rINST @ r1<- byte offset; clear V
- FETCH_ADVANCE_INST_RB r1 @ update rPC, load rINST
- ble MterpCheckSuspendAndContinue
- GET_INST_OPCODE ip @ extract opcode from rINST
- GOTO_OPCODE ip @ jump to next instruction
-#endif
+ movs rINST, r0
+ b MterpCommonTakenBranch
/* ------------------------------ */
@@ -1485,22 +1360,6 @@
/*
* Compare two 64-bit values. Puts 0, 1, or -1 into the destination
* register based on the results of the comparison.
- *
- * We load the full values with LDM, but in practice many values could
- * be resolved by only looking at the high word. This could be made
- * faster or slower by splitting the LDM into a pair of LDRs.
- *
- * If we just wanted to set condition flags, we could do this:
- * subs ip, r0, r2
- * sbcs ip, r1, r3
- * subeqs ip, r0, r2
- * Leaving { <0, 0, >0 } in ip. However, we have to set it to a specific
- * integer value, which we can do with 2 conditional mov/mvn instructions
- * (set 1, set -1; if they're equal we already have 0 in ip), giving
- * us a constant 5-cycle path plus a branch at the end to the
- * instruction epilogue code. The multi-compare approach below needs
- * 2 or 3 cycles + branch if the high word doesn't match, 6 + branch
- * in the worst case (the 64-bit values are equal).
*/
/* cmp-long vAA, vBB, vCC */
FETCH r0, 1 @ r0<- CCBB
@@ -1511,13 +1370,16 @@
VREG_INDEX_TO_ADDR r3, r3 @ r3<- &fp[CC]
ldmia r2, {r0-r1} @ r0/r1<- vBB/vBB+1
ldmia r3, {r2-r3} @ r2/r3<- vCC/vCC+1
- cmp r1, r3 @ compare (vBB+1, vCC+1)
- blt .Lop_cmp_long_less @ signed compare on high part
- bgt .Lop_cmp_long_greater
- subs r1, r0, r2 @ r1<- r0 - r2
- bhi .Lop_cmp_long_greater @ unsigned compare on low part
- bne .Lop_cmp_long_less
- b .Lop_cmp_long_finish @ equal; r1 already holds 0
+ cmp r0, r2
+ sbcs ip, r1, r3 @ Sets correct CCs for checking LT (but not EQ/NE)
+ mov ip, #0
+ mvnlt ip, #0 @ -1
+ cmpeq r0, r2 @ For correct EQ/NE, we may need to repeat the first CMP
+ orrne ip, #1
+ FETCH_ADVANCE_INST 2 @ advance rPC, load rINST
+ SET_VREG ip, r9 @ vAA<- ip
+ GET_INST_OPCODE ip @ extract opcode from rINST
+ GOTO_OPCODE ip @ jump to next instruction
/* ------------------------------ */
.balign 128
@@ -1525,9 +1387,8 @@
/* File: arm/op_if_eq.S */
/* File: arm/bincmp.S */
/*
- * Generic two-operand compare-and-branch operation. Provide a "revcmp"
- * fragment that specifies the *reverse* comparison to perform, e.g.
- * for "if-le" you would use "gt".
+ * Generic two-operand compare-and-branch operation. Provide a "condition"
+ * fragment that specifies the comparison to perform.
*
* For: if-eq, if-ne, if-lt, if-ge, if-gt, if-le
*/
@@ -1535,24 +1396,13 @@
mov r1, rINST, lsr #12 @ r1<- B
ubfx r0, rINST, #8, #4 @ r0<- A
GET_VREG r3, r1 @ r3<- vB
- GET_VREG r2, r0 @ r2<- vA
+ GET_VREG r0, r0 @ r0<- vA
FETCH_S rINST, 1 @ rINST<- branch offset, in code units
- cmp r2, r3 @ compare (vA, vB)
- movne rINST, #2
-#if MTERP_PROFILE_BRANCHES
- @ TUNING: once measurements are complete, remove #if and hand-schedule.
- EXPORT_PC
- mov r0, rSELF
- add r1, rFP, #OFF_FP_SHADOWFRAME
- mov r2, rINST
- bl MterpProfileBranch @ (self, shadow_frame, offset)
- cmp r0, #0
- bne MterpOnStackReplacement @ Note: offset must be in rINST
-#endif
- adds r2, rINST, rINST @ convert to bytes, check sign
- ldr lr, [rSELF, #THREAD_FLAGS_OFFSET]
- FETCH_ADVANCE_INST_RB r2 @ update rPC, load rINST
- bmi MterpCheckSuspendAndContinue
+ cmp r0, r3 @ compare (vA, vB)
+ beq MterpCommonTakenBranchNoFlags
+ cmp rPROFILE, #JIT_CHECK_OSR @ possible OSR re-entry?
+ beq .L_check_not_taken_osr
+ FETCH_ADVANCE_INST 2
GET_INST_OPCODE ip @ extract opcode from rINST
GOTO_OPCODE ip @ jump to next instruction
@@ -1563,9 +1413,8 @@
/* File: arm/op_if_ne.S */
/* File: arm/bincmp.S */
/*
- * Generic two-operand compare-and-branch operation. Provide a "revcmp"
- * fragment that specifies the *reverse* comparison to perform, e.g.
- * for "if-le" you would use "gt".
+ * Generic two-operand compare-and-branch operation. Provide a "condition"
+ * fragment that specifies the comparison to perform.
*
* For: if-eq, if-ne, if-lt, if-ge, if-gt, if-le
*/
@@ -1573,24 +1422,13 @@
mov r1, rINST, lsr #12 @ r1<- B
ubfx r0, rINST, #8, #4 @ r0<- A
GET_VREG r3, r1 @ r3<- vB
- GET_VREG r2, r0 @ r2<- vA
+ GET_VREG r0, r0 @ r0<- vA
FETCH_S rINST, 1 @ rINST<- branch offset, in code units
- cmp r2, r3 @ compare (vA, vB)
- moveq rINST, #2
-#if MTERP_PROFILE_BRANCHES
- @ TUNING: once measurements are complete, remove #if and hand-schedule.
- EXPORT_PC
- mov r0, rSELF
- add r1, rFP, #OFF_FP_SHADOWFRAME
- mov r2, rINST
- bl MterpProfileBranch @ (self, shadow_frame, offset)
- cmp r0, #0
- bne MterpOnStackReplacement @ Note: offset must be in rINST
-#endif
- adds r2, rINST, rINST @ convert to bytes, check sign
- ldr lr, [rSELF, #THREAD_FLAGS_OFFSET]
- FETCH_ADVANCE_INST_RB r2 @ update rPC, load rINST
- bmi MterpCheckSuspendAndContinue
+ cmp r0, r3 @ compare (vA, vB)
+ bne MterpCommonTakenBranchNoFlags
+ cmp rPROFILE, #JIT_CHECK_OSR @ possible OSR re-entry?
+ beq .L_check_not_taken_osr
+ FETCH_ADVANCE_INST 2
GET_INST_OPCODE ip @ extract opcode from rINST
GOTO_OPCODE ip @ jump to next instruction
@@ -1601,9 +1439,8 @@
/* File: arm/op_if_lt.S */
/* File: arm/bincmp.S */
/*
- * Generic two-operand compare-and-branch operation. Provide a "revcmp"
- * fragment that specifies the *reverse* comparison to perform, e.g.
- * for "if-le" you would use "gt".
+ * Generic two-operand compare-and-branch operation. Provide a "condition"
+ * fragment that specifies the comparison to perform.
*
* For: if-eq, if-ne, if-lt, if-ge, if-gt, if-le
*/
@@ -1611,24 +1448,13 @@
mov r1, rINST, lsr #12 @ r1<- B
ubfx r0, rINST, #8, #4 @ r0<- A
GET_VREG r3, r1 @ r3<- vB
- GET_VREG r2, r0 @ r2<- vA
+ GET_VREG r0, r0 @ r0<- vA
FETCH_S rINST, 1 @ rINST<- branch offset, in code units
- cmp r2, r3 @ compare (vA, vB)
- movge rINST, #2
-#if MTERP_PROFILE_BRANCHES
- @ TUNING: once measurements are complete, remove #if and hand-schedule.
- EXPORT_PC
- mov r0, rSELF
- add r1, rFP, #OFF_FP_SHADOWFRAME
- mov r2, rINST
- bl MterpProfileBranch @ (self, shadow_frame, offset)
- cmp r0, #0
- bne MterpOnStackReplacement @ Note: offset must be in rINST
-#endif
- adds r2, rINST, rINST @ convert to bytes, check sign
- ldr lr, [rSELF, #THREAD_FLAGS_OFFSET]
- FETCH_ADVANCE_INST_RB r2 @ update rPC, load rINST
- bmi MterpCheckSuspendAndContinue
+ cmp r0, r3 @ compare (vA, vB)
+ blt MterpCommonTakenBranchNoFlags
+ cmp rPROFILE, #JIT_CHECK_OSR @ possible OSR re-entry?
+ beq .L_check_not_taken_osr
+ FETCH_ADVANCE_INST 2
GET_INST_OPCODE ip @ extract opcode from rINST
GOTO_OPCODE ip @ jump to next instruction
@@ -1639,9 +1465,8 @@
/* File: arm/op_if_ge.S */
/* File: arm/bincmp.S */
/*
- * Generic two-operand compare-and-branch operation. Provide a "revcmp"
- * fragment that specifies the *reverse* comparison to perform, e.g.
- * for "if-le" you would use "gt".
+ * Generic two-operand compare-and-branch operation. Provide a "condition"
+ * fragment that specifies the comparison to perform.
*
* For: if-eq, if-ne, if-lt, if-ge, if-gt, if-le
*/
@@ -1649,24 +1474,13 @@
mov r1, rINST, lsr #12 @ r1<- B
ubfx r0, rINST, #8, #4 @ r0<- A
GET_VREG r3, r1 @ r3<- vB
- GET_VREG r2, r0 @ r2<- vA
+ GET_VREG r0, r0 @ r0<- vA
FETCH_S rINST, 1 @ rINST<- branch offset, in code units
- cmp r2, r3 @ compare (vA, vB)
- movlt rINST, #2
-#if MTERP_PROFILE_BRANCHES
- @ TUNING: once measurements are complete, remove #if and hand-schedule.
- EXPORT_PC
- mov r0, rSELF
- add r1, rFP, #OFF_FP_SHADOWFRAME
- mov r2, rINST
- bl MterpProfileBranch @ (self, shadow_frame, offset)
- cmp r0, #0
- bne MterpOnStackReplacement @ Note: offset must be in rINST
-#endif
- adds r2, rINST, rINST @ convert to bytes, check sign
- ldr lr, [rSELF, #THREAD_FLAGS_OFFSET]
- FETCH_ADVANCE_INST_RB r2 @ update rPC, load rINST
- bmi MterpCheckSuspendAndContinue
+ cmp r0, r3 @ compare (vA, vB)
+ bge MterpCommonTakenBranchNoFlags
+ cmp rPROFILE, #JIT_CHECK_OSR @ possible OSR re-entry?
+ beq .L_check_not_taken_osr
+ FETCH_ADVANCE_INST 2
GET_INST_OPCODE ip @ extract opcode from rINST
GOTO_OPCODE ip @ jump to next instruction
@@ -1677,9 +1491,8 @@
/* File: arm/op_if_gt.S */
/* File: arm/bincmp.S */
/*
- * Generic two-operand compare-and-branch operation. Provide a "revcmp"
- * fragment that specifies the *reverse* comparison to perform, e.g.
- * for "if-le" you would use "gt".
+ * Generic two-operand compare-and-branch operation. Provide a "condition"
+ * fragment that specifies the comparison to perform.
*
* For: if-eq, if-ne, if-lt, if-ge, if-gt, if-le
*/
@@ -1687,24 +1500,13 @@
mov r1, rINST, lsr #12 @ r1<- B
ubfx r0, rINST, #8, #4 @ r0<- A
GET_VREG r3, r1 @ r3<- vB
- GET_VREG r2, r0 @ r2<- vA
+ GET_VREG r0, r0 @ r0<- vA
FETCH_S rINST, 1 @ rINST<- branch offset, in code units
- cmp r2, r3 @ compare (vA, vB)
- movle rINST, #2
-#if MTERP_PROFILE_BRANCHES
- @ TUNING: once measurements are complete, remove #if and hand-schedule.
- EXPORT_PC
- mov r0, rSELF
- add r1, rFP, #OFF_FP_SHADOWFRAME
- mov r2, rINST
- bl MterpProfileBranch @ (self, shadow_frame, offset)
- cmp r0, #0
- bne MterpOnStackReplacement @ Note: offset must be in rINST
-#endif
- adds r2, rINST, rINST @ convert to bytes, check sign
- ldr lr, [rSELF, #THREAD_FLAGS_OFFSET]
- FETCH_ADVANCE_INST_RB r2 @ update rPC, load rINST
- bmi MterpCheckSuspendAndContinue
+ cmp r0, r3 @ compare (vA, vB)
+ bgt MterpCommonTakenBranchNoFlags
+ cmp rPROFILE, #JIT_CHECK_OSR @ possible OSR re-entry?
+ beq .L_check_not_taken_osr
+ FETCH_ADVANCE_INST 2
GET_INST_OPCODE ip @ extract opcode from rINST
GOTO_OPCODE ip @ jump to next instruction
@@ -1715,9 +1517,8 @@
/* File: arm/op_if_le.S */
/* File: arm/bincmp.S */
/*
- * Generic two-operand compare-and-branch operation. Provide a "revcmp"
- * fragment that specifies the *reverse* comparison to perform, e.g.
- * for "if-le" you would use "gt".
+ * Generic two-operand compare-and-branch operation. Provide a "condition"
+ * fragment that specifies the comparison to perform.
*
* For: if-eq, if-ne, if-lt, if-ge, if-gt, if-le
*/
@@ -1725,24 +1526,13 @@
mov r1, rINST, lsr #12 @ r1<- B
ubfx r0, rINST, #8, #4 @ r0<- A
GET_VREG r3, r1 @ r3<- vB
- GET_VREG r2, r0 @ r2<- vA
+ GET_VREG r0, r0 @ r0<- vA
FETCH_S rINST, 1 @ rINST<- branch offset, in code units
- cmp r2, r3 @ compare (vA, vB)
- movgt rINST, #2
-#if MTERP_PROFILE_BRANCHES
- @ TUNING: once measurements are complete, remove #if and hand-schedule.
- EXPORT_PC
- mov r0, rSELF
- add r1, rFP, #OFF_FP_SHADOWFRAME
- mov r2, rINST
- bl MterpProfileBranch @ (self, shadow_frame, offset)
- cmp r0, #0
- bne MterpOnStackReplacement @ Note: offset must be in rINST
-#endif
- adds r2, rINST, rINST @ convert to bytes, check sign
- ldr lr, [rSELF, #THREAD_FLAGS_OFFSET]
- FETCH_ADVANCE_INST_RB r2 @ update rPC, load rINST
- bmi MterpCheckSuspendAndContinue
+ cmp r0, r3 @ compare (vA, vB)
+ ble MterpCommonTakenBranchNoFlags
+ cmp rPROFILE, #JIT_CHECK_OSR @ possible OSR re-entry?
+ beq .L_check_not_taken_osr
+ FETCH_ADVANCE_INST 2
GET_INST_OPCODE ip @ extract opcode from rINST
GOTO_OPCODE ip @ jump to next instruction
@@ -1753,32 +1543,20 @@
/* File: arm/op_if_eqz.S */
/* File: arm/zcmp.S */
/*
- * Generic one-operand compare-and-branch operation. Provide a "revcmp"
- * fragment that specifies the *reverse* comparison to perform, e.g.
- * for "if-le" you would use "gt".
+ * Generic one-operand compare-and-branch operation. Provide a "condition"
+ * fragment that specifies the comparison to perform.
*
* for: if-eqz, if-nez, if-ltz, if-gez, if-gtz, if-lez
*/
/* if-cmp vAA, +BBBB */
mov r0, rINST, lsr #8 @ r0<- AA
- GET_VREG r2, r0 @ r2<- vAA
+ GET_VREG r0, r0 @ r0<- vAA
FETCH_S rINST, 1 @ rINST<- branch offset, in code units
- ldr lr, [rSELF, #THREAD_FLAGS_OFFSET]
- cmp r2, #0 @ compare (vA, 0)
- movne rINST, #2
-#if MTERP_PROFILE_BRANCHES
- @ TUNING: once measurements are complete, remove #if and hand-schedule.
- EXPORT_PC
- mov r0, rSELF
- add r1, rFP, #OFF_FP_SHADOWFRAME
- mov r2, rINST
- bl MterpProfileBranch @ (self, shadow_frame, offset)
- cmp r0, #0
- bne MterpOnStackReplacement @ Note: offset must be in rINST
-#endif
- adds r1, rINST, rINST @ convert to bytes & set flags
- FETCH_ADVANCE_INST_RB r1 @ update rPC, load rINST
- bmi MterpCheckSuspendAndContinue
+ cmp r0, #0 @ compare (vA, 0)
+ beq MterpCommonTakenBranchNoFlags
+ cmp rPROFILE, #JIT_CHECK_OSR @ possible OSR re-entry?
+ beq .L_check_not_taken_osr
+ FETCH_ADVANCE_INST 2
GET_INST_OPCODE ip @ extract opcode from rINST
GOTO_OPCODE ip @ jump to next instruction
@@ -1789,32 +1567,20 @@
/* File: arm/op_if_nez.S */
/* File: arm/zcmp.S */
/*
- * Generic one-operand compare-and-branch operation. Provide a "revcmp"
- * fragment that specifies the *reverse* comparison to perform, e.g.
- * for "if-le" you would use "gt".
+ * Generic one-operand compare-and-branch operation. Provide a "condition"
+ * fragment that specifies the comparison to perform.
*
* for: if-eqz, if-nez, if-ltz, if-gez, if-gtz, if-lez
*/
/* if-cmp vAA, +BBBB */
mov r0, rINST, lsr #8 @ r0<- AA
- GET_VREG r2, r0 @ r2<- vAA
+ GET_VREG r0, r0 @ r0<- vAA
FETCH_S rINST, 1 @ rINST<- branch offset, in code units
- ldr lr, [rSELF, #THREAD_FLAGS_OFFSET]
- cmp r2, #0 @ compare (vA, 0)
- moveq rINST, #2
-#if MTERP_PROFILE_BRANCHES
- @ TUNING: once measurements are complete, remove #if and hand-schedule.
- EXPORT_PC
- mov r0, rSELF
- add r1, rFP, #OFF_FP_SHADOWFRAME
- mov r2, rINST
- bl MterpProfileBranch @ (self, shadow_frame, offset)
- cmp r0, #0
- bne MterpOnStackReplacement @ Note: offset must be in rINST
-#endif
- adds r1, rINST, rINST @ convert to bytes & set flags
- FETCH_ADVANCE_INST_RB r1 @ update rPC, load rINST
- bmi MterpCheckSuspendAndContinue
+ cmp r0, #0 @ compare (vA, 0)
+ bne MterpCommonTakenBranchNoFlags
+ cmp rPROFILE, #JIT_CHECK_OSR @ possible OSR re-entry?
+ beq .L_check_not_taken_osr
+ FETCH_ADVANCE_INST 2
GET_INST_OPCODE ip @ extract opcode from rINST
GOTO_OPCODE ip @ jump to next instruction
@@ -1825,32 +1591,20 @@
/* File: arm/op_if_ltz.S */
/* File: arm/zcmp.S */
/*
- * Generic one-operand compare-and-branch operation. Provide a "revcmp"
- * fragment that specifies the *reverse* comparison to perform, e.g.
- * for "if-le" you would use "gt".
+ * Generic one-operand compare-and-branch operation. Provide a "condition"
+ * fragment that specifies the comparison to perform.
*
* for: if-eqz, if-nez, if-ltz, if-gez, if-gtz, if-lez
*/
/* if-cmp vAA, +BBBB */
mov r0, rINST, lsr #8 @ r0<- AA
- GET_VREG r2, r0 @ r2<- vAA
+ GET_VREG r0, r0 @ r0<- vAA
FETCH_S rINST, 1 @ rINST<- branch offset, in code units
- ldr lr, [rSELF, #THREAD_FLAGS_OFFSET]
- cmp r2, #0 @ compare (vA, 0)
- movge rINST, #2
-#if MTERP_PROFILE_BRANCHES
- @ TUNING: once measurements are complete, remove #if and hand-schedule.
- EXPORT_PC
- mov r0, rSELF
- add r1, rFP, #OFF_FP_SHADOWFRAME
- mov r2, rINST
- bl MterpProfileBranch @ (self, shadow_frame, offset)
- cmp r0, #0
- bne MterpOnStackReplacement @ Note: offset must be in rINST
-#endif
- adds r1, rINST, rINST @ convert to bytes & set flags
- FETCH_ADVANCE_INST_RB r1 @ update rPC, load rINST
- bmi MterpCheckSuspendAndContinue
+ cmp r0, #0 @ compare (vA, 0)
+ blt MterpCommonTakenBranchNoFlags
+ cmp rPROFILE, #JIT_CHECK_OSR @ possible OSR re-entry?
+ beq .L_check_not_taken_osr
+ FETCH_ADVANCE_INST 2
GET_INST_OPCODE ip @ extract opcode from rINST
GOTO_OPCODE ip @ jump to next instruction
@@ -1861,32 +1615,20 @@
/* File: arm/op_if_gez.S */
/* File: arm/zcmp.S */
/*
- * Generic one-operand compare-and-branch operation. Provide a "revcmp"
- * fragment that specifies the *reverse* comparison to perform, e.g.
- * for "if-le" you would use "gt".
+ * Generic one-operand compare-and-branch operation. Provide a "condition"
+ * fragment that specifies the comparison to perform.
*
* for: if-eqz, if-nez, if-ltz, if-gez, if-gtz, if-lez
*/
/* if-cmp vAA, +BBBB */
mov r0, rINST, lsr #8 @ r0<- AA
- GET_VREG r2, r0 @ r2<- vAA
+ GET_VREG r0, r0 @ r0<- vAA
FETCH_S rINST, 1 @ rINST<- branch offset, in code units
- ldr lr, [rSELF, #THREAD_FLAGS_OFFSET]
- cmp r2, #0 @ compare (vA, 0)
- movlt rINST, #2
-#if MTERP_PROFILE_BRANCHES
- @ TUNING: once measurements are complete, remove #if and hand-schedule.
- EXPORT_PC
- mov r0, rSELF
- add r1, rFP, #OFF_FP_SHADOWFRAME
- mov r2, rINST
- bl MterpProfileBranch @ (self, shadow_frame, offset)
- cmp r0, #0
- bne MterpOnStackReplacement @ Note: offset must be in rINST
-#endif
- adds r1, rINST, rINST @ convert to bytes & set flags
- FETCH_ADVANCE_INST_RB r1 @ update rPC, load rINST
- bmi MterpCheckSuspendAndContinue
+ cmp r0, #0 @ compare (vA, 0)
+ bge MterpCommonTakenBranchNoFlags
+ cmp rPROFILE, #JIT_CHECK_OSR @ possible OSR re-entry?
+ beq .L_check_not_taken_osr
+ FETCH_ADVANCE_INST 2
GET_INST_OPCODE ip @ extract opcode from rINST
GOTO_OPCODE ip @ jump to next instruction
@@ -1897,32 +1639,20 @@
/* File: arm/op_if_gtz.S */
/* File: arm/zcmp.S */
/*
- * Generic one-operand compare-and-branch operation. Provide a "revcmp"
- * fragment that specifies the *reverse* comparison to perform, e.g.
- * for "if-le" you would use "gt".
+ * Generic one-operand compare-and-branch operation. Provide a "condition"
+ * fragment that specifies the comparison to perform.
*
* for: if-eqz, if-nez, if-ltz, if-gez, if-gtz, if-lez
*/
/* if-cmp vAA, +BBBB */
mov r0, rINST, lsr #8 @ r0<- AA
- GET_VREG r2, r0 @ r2<- vAA
+ GET_VREG r0, r0 @ r0<- vAA
FETCH_S rINST, 1 @ rINST<- branch offset, in code units
- ldr lr, [rSELF, #THREAD_FLAGS_OFFSET]
- cmp r2, #0 @ compare (vA, 0)
- movle rINST, #2
-#if MTERP_PROFILE_BRANCHES
- @ TUNING: once measurements are complete, remove #if and hand-schedule.
- EXPORT_PC
- mov r0, rSELF
- add r1, rFP, #OFF_FP_SHADOWFRAME
- mov r2, rINST
- bl MterpProfileBranch @ (self, shadow_frame, offset)
- cmp r0, #0
- bne MterpOnStackReplacement @ Note: offset must be in rINST
-#endif
- adds r1, rINST, rINST @ convert to bytes & set flags
- FETCH_ADVANCE_INST_RB r1 @ update rPC, load rINST
- bmi MterpCheckSuspendAndContinue
+ cmp r0, #0 @ compare (vA, 0)
+ bgt MterpCommonTakenBranchNoFlags
+ cmp rPROFILE, #JIT_CHECK_OSR @ possible OSR re-entry?
+ beq .L_check_not_taken_osr
+ FETCH_ADVANCE_INST 2
GET_INST_OPCODE ip @ extract opcode from rINST
GOTO_OPCODE ip @ jump to next instruction
@@ -1933,32 +1663,20 @@
/* File: arm/op_if_lez.S */
/* File: arm/zcmp.S */
/*
- * Generic one-operand compare-and-branch operation. Provide a "revcmp"
- * fragment that specifies the *reverse* comparison to perform, e.g.
- * for "if-le" you would use "gt".
+ * Generic one-operand compare-and-branch operation. Provide a "condition"
+ * fragment that specifies the comparison to perform.
*
* for: if-eqz, if-nez, if-ltz, if-gez, if-gtz, if-lez
*/
/* if-cmp vAA, +BBBB */
mov r0, rINST, lsr #8 @ r0<- AA
- GET_VREG r2, r0 @ r2<- vAA
+ GET_VREG r0, r0 @ r0<- vAA
FETCH_S rINST, 1 @ rINST<- branch offset, in code units
- ldr lr, [rSELF, #THREAD_FLAGS_OFFSET]
- cmp r2, #0 @ compare (vA, 0)
- movgt rINST, #2
-#if MTERP_PROFILE_BRANCHES
- @ TUNING: once measurements are complete, remove #if and hand-schedule.
- EXPORT_PC
- mov r0, rSELF
- add r1, rFP, #OFF_FP_SHADOWFRAME
- mov r2, rINST
- bl MterpProfileBranch @ (self, shadow_frame, offset)
- cmp r0, #0
- bne MterpOnStackReplacement @ Note: offset must be in rINST
-#endif
- adds r1, rINST, rINST @ convert to bytes & set flags
- FETCH_ADVANCE_INST_RB r1 @ update rPC, load rINST
- bmi MterpCheckSuspendAndContinue
+ cmp r0, #0 @ compare (vA, 0)
+ ble MterpCommonTakenBranchNoFlags
+ cmp rPROFILE, #JIT_CHECK_OSR @ possible OSR re-entry?
+ beq .L_check_not_taken_osr
+ FETCH_ADVANCE_INST 2
GET_INST_OPCODE ip @ extract opcode from rINST
GOTO_OPCODE ip @ jump to next instruction
@@ -4711,15 +4429,15 @@
VREG_INDEX_TO_ADDR r3, r3 @ r3<- &fp[CC]
ldmia r2, {r0-r1} @ r0/r1<- vBB/vBB+1
ldmia r3, {r2-r3} @ r2/r3<- vCC/vCC+1
- mul ip, r2, r1 @ ip<- ZxW
- umull r9, r10, r2, r0 @ r9/r10 <- ZxX
- mla r2, r0, r3, ip @ r2<- YxX + (ZxW)
+ mul ip, r2, r1 @ ip<- ZxW
+ umull r1, lr, r2, r0 @ r1/lr <- ZxX
+ mla r2, r0, r3, ip @ r2<- YxX + (ZxW)
mov r0, rINST, lsr #8 @ r0<- AA
- add r10, r2, r10 @ r10<- r10 + low(ZxW + (YxX))
+ add r2, r2, lr @ r2<- lr + low(ZxW + (YxX))
VREG_INDEX_TO_ADDR r0, r0 @ r0<- &fp[AA]
FETCH_ADVANCE_INST 2 @ advance rPC, load rINST
GET_INST_OPCODE ip @ extract opcode from rINST
- stmia r0, {r9-r10} @ vAA/vAA+1<- r9/r10
+ stmia r0, {r1-r2 } @ vAA/vAA+1<- r1/r2
GOTO_OPCODE ip @ jump to next instruction
/* ------------------------------ */
@@ -5877,14 +5595,14 @@
VREG_INDEX_TO_ADDR rINST, r9 @ rINST<- &fp[A]
ldmia r1, {r2-r3} @ r2/r3<- vBB/vBB+1
ldmia rINST, {r0-r1} @ r0/r1<- vAA/vAA+1
- mul ip, r2, r1 @ ip<- ZxW
- umull r9, r10, r2, r0 @ r9/r10 <- ZxX
- mla r2, r0, r3, ip @ r2<- YxX + (ZxW)
+ mul ip, r2, r1 @ ip<- ZxW
+ umull r1, lr, r2, r0 @ r1/lr <- ZxX
+ mla r2, r0, r3, ip @ r2<- YxX + (ZxW)
mov r0, rINST @ r0<- &fp[A] (free up rINST)
FETCH_ADVANCE_INST 1 @ advance rPC, load rINST
- add r10, r2, r10 @ r10<- r10 + low(ZxW + (YxX))
+ add r2, r2, lr @ r2<- r2 + low(ZxW + (YxX))
GET_INST_OPCODE ip @ extract opcode from rINST
- stmia r0, {r9-r10} @ vAA/vAA+1<- r9/r10
+ stmia r0, {r1-r2} @ vAA/vAA+1<- r1/r2
GOTO_OPCODE ip @ jump to next instruction
/* ------------------------------ */
@@ -7616,27 +7334,6 @@
.balign 4
artMterpAsmSisterStart:
-/* continuation for op_cmp_long */
-
-.Lop_cmp_long_less:
- mvn r1, #0 @ r1<- -1
- @ Want to cond code the next mov so we can avoid branch, but don't see it;
- @ instead, we just replicate the tail end.
- FETCH_ADVANCE_INST 2 @ advance rPC, load rINST
- SET_VREG r1, r9 @ vAA<- r1
- GET_INST_OPCODE ip @ extract opcode from rINST
- GOTO_OPCODE ip @ jump to next instruction
-
-.Lop_cmp_long_greater:
- mov r1, #1 @ r1<- 1
- @ fall through to _finish
-
-.Lop_cmp_long_finish:
- FETCH_ADVANCE_INST 2 @ advance rPC, load rINST
- SET_VREG r1, r9 @ vAA<- r1
- GET_INST_OPCODE ip @ extract opcode from rINST
- GOTO_OPCODE ip @ jump to next instruction
-
/* continuation for op_float_to_long */
/*
* Convert the float in r0 to a long in r0/r1.
@@ -12207,21 +11904,117 @@
/* NOTE: no fallthrough */
/*
- * Check for suspend check request. Assumes rINST already loaded, rPC advanced and
- * still needs to get the opcode and branch to it, and flags are in lr.
+ * Common handling for branches with support for Jit profiling.
+ * On entry:
+ * rINST <= signed offset
+ * rPROFILE <= signed hotness countdown (expanded to 32 bits)
+ * condition bits <= set to establish sign of offset (use "NoFlags" entry if not)
+ *
+ * We have quite a few different cases for branch profiling, OSR detection and
+ * suspend check support here.
+ *
+ * Taken backward branches:
+ * If profiling active, do hotness countdown and report if we hit zero.
+ * If in osr check mode, see if our target is a compiled loop header entry and do OSR if so.
+ * Is there a pending suspend request? If so, suspend.
+ *
+ * Taken forward branches and not-taken backward branches:
+ * If in osr check mode, see if our target is a compiled loop header entry and do OSR if so.
+ *
+ * Our most common case is expected to be a taken backward branch with active jit profiling,
+ * but no full OSR check and no pending suspend request.
+ * Next most common case is not-taken branch with no full OSR check.
+ *
*/
-MterpCheckSuspendAndContinue:
- ldr rIBASE, [rSELF, #THREAD_CURRENT_IBASE_OFFSET] @ refresh rIBASE
+MterpCommonTakenBranchNoFlags:
+ cmp rINST, #0
+MterpCommonTakenBranch:
+ bgt .L_forward_branch @ don't add forward branches to hotness
+/*
+ * We need to subtract 1 from positive values and we should not see 0 here,
+ * so we may use the result of the comparison with -1.
+ */
+#if JIT_CHECK_OSR != -1
+# error "JIT_CHECK_OSR must be -1."
+#endif
+ cmp rPROFILE, #JIT_CHECK_OSR
+ beq .L_osr_check
+ subgts rPROFILE, #1
+ beq .L_add_batch @ counted down to zero - report
+.L_resume_backward_branch:
+ ldr lr, [rSELF, #THREAD_FLAGS_OFFSET]
+ REFRESH_IBASE
+ add r2, rINST, rINST @ r2<- byte offset
+ FETCH_ADVANCE_INST_RB r2 @ update rPC, load rINST
ands lr, #(THREAD_SUSPEND_REQUEST | THREAD_CHECKPOINT_REQUEST)
- bne 1f
+ bne .L_suspend_request_pending
GET_INST_OPCODE ip @ extract opcode from rINST
GOTO_OPCODE ip @ jump to next instruction
-1:
+
+.L_suspend_request_pending:
EXPORT_PC
mov r0, rSELF
bl MterpSuspendCheck @ (self)
cmp r0, #0
bne MterpFallback
+ REFRESH_IBASE @ might have changed during suspend
+ GET_INST_OPCODE ip @ extract opcode from rINST
+ GOTO_OPCODE ip @ jump to next instruction
+
+.L_no_count_backwards:
+ cmp rPROFILE, #JIT_CHECK_OSR @ possible OSR re-entry?
+ bne .L_resume_backward_branch
+.L_osr_check:
+ mov r0, rSELF
+ add r1, rFP, #OFF_FP_SHADOWFRAME
+ mov r2, rINST
+ EXPORT_PC
+ bl MterpMaybeDoOnStackReplacement @ (self, shadow_frame, offset)
+ cmp r0, #0
+ bne MterpOnStackReplacement
+ b .L_resume_backward_branch
+
+.L_forward_branch:
+ cmp rPROFILE, #JIT_CHECK_OSR @ possible OSR re-entry?
+ beq .L_check_osr_forward
+.L_resume_forward_branch:
+ add r2, rINST, rINST @ r2<- byte offset
+ FETCH_ADVANCE_INST_RB r2 @ update rPC, load rINST
+ GET_INST_OPCODE ip @ extract opcode from rINST
+ GOTO_OPCODE ip @ jump to next instruction
+
+.L_check_osr_forward:
+ mov r0, rSELF
+ add r1, rFP, #OFF_FP_SHADOWFRAME
+ mov r2, rINST
+ EXPORT_PC
+ bl MterpMaybeDoOnStackReplacement @ (self, shadow_frame, offset)
+ cmp r0, #0
+ bne MterpOnStackReplacement
+ b .L_resume_forward_branch
+
+.L_add_batch:
+ add r1, rFP, #OFF_FP_SHADOWFRAME
+ strh rPROFILE, [r1, #SHADOWFRAME_HOTNESS_COUNTDOWN_OFFSET]
+ ldr r0, [rFP, #OFF_FP_METHOD]
+ mov r2, rSELF
+ bl MterpAddHotnessBatch @ (method, shadow_frame, self)
+ mov rPROFILE, r0 @ restore new hotness countdown to rPROFILE
+ b .L_no_count_backwards
+
+/*
+ * Entered from the conditional branch handlers when OSR check request active on
+ * not-taken path. All Dalvik not-taken conditional branch offsets are 2.
+ */
+.L_check_not_taken_osr:
+ mov r0, rSELF
+ add r1, rFP, #OFF_FP_SHADOWFRAME
+ mov r2, #2
+ EXPORT_PC
+ bl MterpMaybeDoOnStackReplacement @ (self, shadow_frame, offset)
+ cmp r0, #0
+ bne MterpOnStackReplacement
+ FETCH_ADVANCE_INST 2
GET_INST_OPCODE ip @ extract opcode from rINST
GOTO_OPCODE ip @ jump to next instruction
@@ -12269,9 +12062,27 @@
str r1, [r2, #4]
mov r0, #1 @ signal return to caller.
MterpDone:
- add sp, sp, #4 @ un-align 64
- ldmfd sp!, {r4-r10,fp,pc} @ restore 9 regs and return
+/*
+ * At this point, we expect rPROFILE to be non-zero. If negative, hotness is disabled or we're
+ * checking for OSR. If greater than zero, we might have unreported hotness to register
+ * (the difference between the ending rPROFILE and the cached hotness counter). rPROFILE
+ * should only reach zero immediately after a hotness decrement, and is then reset to either
+ * a negative special state or the new non-zero countdown value.
+ */
+ cmp rPROFILE, #0
+ bgt MterpProfileActive @ if > 0, we may have some counts to report.
+ ldmfd sp!, {r3-r10,fp,pc} @ restore 10 regs and return
+MterpProfileActive:
+ mov rINST, r0 @ stash return value
+ /* Report cached hotness counts */
+ ldr r0, [rFP, #OFF_FP_METHOD]
+ add r1, rFP, #OFF_FP_SHADOWFRAME
+ mov r2, rSELF
+ strh rPROFILE, [r1, #SHADOWFRAME_HOTNESS_COUNTDOWN_OFFSET]
+ bl MterpAddHotnessBatch @ (method, shadow_frame, self)
+ mov r0, rINST @ restore return value
+ ldmfd sp!, {r3-r10,fp,pc} @ restore 10 regs and return
.fnend
.size ExecuteMterpImpl, .-ExecuteMterpImpl
diff --git a/runtime/interpreter/mterp/out/mterp_arm64.S b/runtime/interpreter/mterp/out/mterp_arm64.S
index 6ae59d8..55797e6 100644
--- a/runtime/interpreter/mterp/out/mterp_arm64.S
+++ b/runtime/interpreter/mterp/out/mterp_arm64.S
@@ -81,6 +81,7 @@
x23 xINST first 16-bit code unit of current instruction
x24 xIBASE interpreted instruction base pointer, used for computed goto
x25 xREFS base of object references in shadow frame (ideally, we'll get rid of this later).
+ x26 wPROFILE jit profile hotness countdown
x16 ip scratch reg
x17 ip2 scratch reg (used by macros)
@@ -99,15 +100,17 @@
/* During bringup, we'll use the shadow frame model instead of xFP */
/* single-purpose registers, given names for clarity */
-#define xPC x20
-#define xFP x21
-#define xSELF x22
-#define xINST x23
-#define wINST w23
-#define xIBASE x24
-#define xREFS x25
-#define ip x16
-#define ip2 x17
+#define xPC x20
+#define xFP x21
+#define xSELF x22
+#define xINST x23
+#define wINST w23
+#define xIBASE x24
+#define xREFS x25
+#define wPROFILE w26
+#define xPROFILE x26
+#define ip x16
+#define ip2 x17
/*
* Instead of holding a pointer to the shadow frame, we keep xFP at the base of the vregs. So,
@@ -121,7 +124,7 @@
#define OFF_FP_RESULT_REGISTER OFF_FP(SHADOWFRAME_RESULT_REGISTER_OFFSET)
#define OFF_FP_DEX_PC_PTR OFF_FP(SHADOWFRAME_DEX_PC_PTR_OFFSET)
#define OFF_FP_CODE_ITEM OFF_FP(SHADOWFRAME_CODE_ITEM_OFFSET)
-#define OFF_FP_SHADOWFRAME (-SHADOWFRAME_VREGS_OFFSET)
+#define OFF_FP_SHADOWFRAME OFF_FP(0)
/*
* "export" the PC to dex_pc field in the shadow frame, f/b/o future exception objects. Must
@@ -323,11 +326,12 @@
ExecuteMterpImpl:
.cfi_startproc
- stp xIBASE, xREFS, [sp, #-64]!
- stp xSELF, xINST, [sp, #16]
- stp xPC, xFP, [sp, #32]
- stp fp, lr, [sp, #48]
- add fp, sp, #48
+ stp xPROFILE, x27, [sp, #-80]!
+ stp xIBASE, xREFS, [sp, #16]
+ stp xSELF, xINST, [sp, #32]
+ stp xPC, xFP, [sp, #48]
+ stp fp, lr, [sp, #64]
+ add fp, sp, #64
/* Remember the return register */
str x3, [x2, #SHADOWFRAME_RESULT_REGISTER_OFFSET]
@@ -348,6 +352,12 @@
/* Starting ibase */
ldr xIBASE, [xSELF, #THREAD_CURRENT_IBASE_OFFSET]
+ /* Set up for backwards branches & osr profiling */
+ ldr x0, [xFP, #OFF_FP_METHOD]
+ add x1, xFP, #OFF_FP_SHADOWFRAME
+ bl MterpSetUpHotnessCountdown
+ mov wPROFILE, w0 // Starting hotness countdown to xPROFILE
+
/* start executing the instruction at rPC */
FETCH_INST // load wINST from rPC
GET_INST_OPCODE ip // extract opcode from wINST
@@ -1081,24 +1091,8 @@
* double to get a byte offset.
*/
/* goto +AA */
- /* tuning: use sbfx for 6t2+ targets */
- lsl w0, wINST, #16 // w0<- AAxx0000
- asr wINST, w0, #24 // wINST<- ssssssAA (sign-extended)
-#if MTERP_PROFILE_BRANCHES
- EXPORT_PC
- mov x0, xSELF
- add x1, xFP, #OFF_FP_SHADOWFRAME
- sbfm x2, xINST, 0, 31
- bl MterpProfileBranch // (self, shadow_frame, offset)
- cbnz w0, MterpOnStackReplacement // Note: offset must be in wINST
-#endif
- ldr w7, [xSELF, #THREAD_FLAGS_OFFSET] // Preload flags for MterpCheckSuspendAndContinue
- adds w1, wINST, wINST // Convert dalvik offset to byte offset, setting flags
- FETCH_ADVANCE_INST_RB w1 // load wINST and advance xPC
- // If backwards branch refresh rIBASE
- b.mi MterpCheckSuspendAndContinue
- GET_INST_OPCODE ip // extract opcode from wINST
- GOTO_OPCODE ip // jump to next instruction
+ sbfx wINST, wINST, #8, #8 // wINST<- ssssssAA (sign-extended)
+ b MterpCommonTakenBranchNoFlags
/* ------------------------------ */
.balign 128
@@ -1112,20 +1106,7 @@
*/
/* goto/16 +AAAA */
FETCH_S wINST, 1 // wINST<- ssssAAAA (sign-extended)
-#if MTERP_PROFILE_BRANCHES
- EXPORT_PC
- mov x0, xSELF
- add x1, xFP, #OFF_FP_SHADOWFRAME
- sbfm x2, xINST, 0, 31
- bl MterpProfileBranch // (self, shadow_frame, offset)
- cbnz w0, MterpOnStackReplacement // Note: offset must be in xINST
-#endif
- ldr w7, [xSELF, #THREAD_FLAGS_OFFSET]
- adds w1, wINST, wINST // w1<- byte offset, flags set
- FETCH_ADVANCE_INST_RB w1 // update rPC, load rINST
- b.mi MterpCheckSuspendAndContinue
- GET_INST_OPCODE ip // extract opcode from rINST
- GOTO_OPCODE ip // jump to next instruction
+ b MterpCommonTakenBranchNoFlags
/* ------------------------------ */
.balign 128
@@ -1146,20 +1127,7 @@
FETCH w0, 1 // w0<- aaaa (lo)
FETCH w1, 2 // w1<- AAAA (hi)
orr wINST, w0, w1, lsl #16 // wINST<- AAAAaaaa
-#if MTERP_PROFILE_BRANCHES
- EXPORT_PC
- mov x0, xSELF
- add x1, xFP, #OFF_FP_SHADOWFRAME
- sbfm x2, xINST, 0, 31
- bl MterpProfileBranch // (self, shadow_frame, offset)
- cbnz w0, MterpOnStackReplacement // Note: offset must be in xINST
-#endif
- ldr w7, [xSELF, #THREAD_FLAGS_OFFSET]
- adds w1, wINST, wINST // w1<- byte offset
- FETCH_ADVANCE_INST_RB w1 // update rPC, load xINST
- b.le MterpCheckSuspendAndContinue
- GET_INST_OPCODE ip // extract opcode from xINST
- GOTO_OPCODE ip // jump to next instruction
+ b MterpCommonTakenBranchNoFlags
/* ------------------------------ */
.balign 128
@@ -1183,20 +1151,7 @@
add x0, xPC, w0, lsl #1 // w0<- PC + BBBBbbbb*2
bl MterpDoPackedSwitch // w0<- code-unit branch offset
sbfm xINST, x0, 0, 31
-#if MTERP_PROFILE_BRANCHES
- EXPORT_PC
- mov x0, xSELF
- add x1, xFP, #OFF_FP_SHADOWFRAME
- mov x2, xINST
- bl MterpProfileBranch // (self, shadow_frame, offset)
- cbnz w0, MterpOnStackReplacement
-#endif
- ldr w7, [xSELF, #THREAD_FLAGS_OFFSET]
- adds w1, wINST, wINST // w1<- byte offset; clear V
- FETCH_ADVANCE_INST_RB w1 // update rPC, load wINST
- b.le MterpCheckSuspendAndContinue
- GET_INST_OPCODE ip // extract opcode from wINST
- GOTO_OPCODE ip // jump to next instruction
+ b MterpCommonTakenBranchNoFlags
/* ------------------------------ */
.balign 128
@@ -1221,20 +1176,7 @@
add x0, xPC, w0, lsl #1 // w0<- PC + BBBBbbbb*2
bl MterpDoSparseSwitch // w0<- code-unit branch offset
sbfm xINST, x0, 0, 31
-#if MTERP_PROFILE_BRANCHES
- EXPORT_PC
- mov x0, xSELF
- add x1, xFP, #OFF_FP_SHADOWFRAME
- mov x2, xINST
- bl MterpProfileBranch // (self, shadow_frame, offset)
- cbnz w0, MterpOnStackReplacement
-#endif
- ldr w7, [xSELF, #THREAD_FLAGS_OFFSET]
- adds w1, wINST, wINST // w1<- byte offset; clear V
- FETCH_ADVANCE_INST_RB w1 // update rPC, load wINST
- b.le MterpCheckSuspendAndContinue
- GET_INST_OPCODE ip // extract opcode from wINST
- GOTO_OPCODE ip // jump to next instruction
+ b MterpCommonTakenBranchNoFlags
/* ------------------------------ */
@@ -1365,9 +1307,8 @@
/* File: arm64/op_if_eq.S */
/* File: arm64/bincmp.S */
/*
- * Generic two-operand compare-and-branch operation. Provide a "revcmp"
- * fragment that specifies the *reverse* comparison to perform, e.g.
- * for "if-le" you would use "gt".
+ * Generic two-operand compare-and-branch operation. Provide a "condition"
+ * fragment that specifies the comparison to perform.
*
* For: if-eq, if-ne, if-lt, if-ge, if-gt, if-le
*/
@@ -1376,23 +1317,12 @@
ubfx w0, wINST, #8, #4 // w0<- A
GET_VREG w3, w1 // w3<- vB
GET_VREG w2, w0 // w2<- vA
- FETCH_S w1, 1 // w1<- branch offset, in code units
- mov w0, #2 // Offset if branch not taken
+ FETCH_S wINST, 1 // wINST<- branch offset, in code units
cmp w2, w3 // compare (vA, vB)
- csel wINST, w1, w0, eq // Branch if true, stashing result in callee save reg.
-#if MTERP_PROFILE_BRANCHES
- // TUINING: once measurements are complete, remove #if and hand-schedule.
- EXPORT_PC
- mov x0, xSELF
- add x1, xFP, #OFF_FP_SHADOWFRAME
- sbfm x2, xINST, 0, 31 // Sign extend branch offset
- bl MterpProfileBranch // (self, shadow_frame, offset)
- cbnz w0, MterpOnStackReplacement // Note: offset must be in xINST
-#endif
- ldr w7, [xSELF, #THREAD_FLAGS_OFFSET]
- adds w2, wINST, wINST // convert to bytes, check sign
- FETCH_ADVANCE_INST_RB w2 // update rPC, load wINST
- b.mi MterpCheckSuspendAndContinue
+ b.eq MterpCommonTakenBranchNoFlags
+ cmp wPROFILE, #JIT_CHECK_OSR // possible OSR re-entry?
+ b.eq .L_check_not_taken_osr
+ FETCH_ADVANCE_INST 2
GET_INST_OPCODE ip // extract opcode from wINST
GOTO_OPCODE ip // jump to next instruction
@@ -1403,9 +1333,8 @@
/* File: arm64/op_if_ne.S */
/* File: arm64/bincmp.S */
/*
- * Generic two-operand compare-and-branch operation. Provide a "revcmp"
- * fragment that specifies the *reverse* comparison to perform, e.g.
- * for "if-le" you would use "gt".
+ * Generic two-operand compare-and-branch operation. Provide a "condition"
+ * fragment that specifies the comparison to perform.
*
* For: if-eq, if-ne, if-lt, if-ge, if-gt, if-le
*/
@@ -1414,23 +1343,12 @@
ubfx w0, wINST, #8, #4 // w0<- A
GET_VREG w3, w1 // w3<- vB
GET_VREG w2, w0 // w2<- vA
- FETCH_S w1, 1 // w1<- branch offset, in code units
- mov w0, #2 // Offset if branch not taken
+ FETCH_S wINST, 1 // wINST<- branch offset, in code units
cmp w2, w3 // compare (vA, vB)
- csel wINST, w1, w0, ne // Branch if true, stashing result in callee save reg.
-#if MTERP_PROFILE_BRANCHES
- // TUINING: once measurements are complete, remove #if and hand-schedule.
- EXPORT_PC
- mov x0, xSELF
- add x1, xFP, #OFF_FP_SHADOWFRAME
- sbfm x2, xINST, 0, 31 // Sign extend branch offset
- bl MterpProfileBranch // (self, shadow_frame, offset)
- cbnz w0, MterpOnStackReplacement // Note: offset must be in xINST
-#endif
- ldr w7, [xSELF, #THREAD_FLAGS_OFFSET]
- adds w2, wINST, wINST // convert to bytes, check sign
- FETCH_ADVANCE_INST_RB w2 // update rPC, load wINST
- b.mi MterpCheckSuspendAndContinue
+ b.ne MterpCommonTakenBranchNoFlags
+ cmp wPROFILE, #JIT_CHECK_OSR // possible OSR re-entry?
+ b.eq .L_check_not_taken_osr
+ FETCH_ADVANCE_INST 2
GET_INST_OPCODE ip // extract opcode from wINST
GOTO_OPCODE ip // jump to next instruction
@@ -1441,9 +1359,8 @@
/* File: arm64/op_if_lt.S */
/* File: arm64/bincmp.S */
/*
- * Generic two-operand compare-and-branch operation. Provide a "revcmp"
- * fragment that specifies the *reverse* comparison to perform, e.g.
- * for "if-le" you would use "gt".
+ * Generic two-operand compare-and-branch operation. Provide a "condition"
+ * fragment that specifies the comparison to perform.
*
* For: if-eq, if-ne, if-lt, if-ge, if-gt, if-le
*/
@@ -1452,23 +1369,12 @@
ubfx w0, wINST, #8, #4 // w0<- A
GET_VREG w3, w1 // w3<- vB
GET_VREG w2, w0 // w2<- vA
- FETCH_S w1, 1 // w1<- branch offset, in code units
- mov w0, #2 // Offset if branch not taken
+ FETCH_S wINST, 1 // wINST<- branch offset, in code units
cmp w2, w3 // compare (vA, vB)
- csel wINST, w1, w0, lt // Branch if true, stashing result in callee save reg.
-#if MTERP_PROFILE_BRANCHES
- // TUINING: once measurements are complete, remove #if and hand-schedule.
- EXPORT_PC
- mov x0, xSELF
- add x1, xFP, #OFF_FP_SHADOWFRAME
- sbfm x2, xINST, 0, 31 // Sign extend branch offset
- bl MterpProfileBranch // (self, shadow_frame, offset)
- cbnz w0, MterpOnStackReplacement // Note: offset must be in xINST
-#endif
- ldr w7, [xSELF, #THREAD_FLAGS_OFFSET]
- adds w2, wINST, wINST // convert to bytes, check sign
- FETCH_ADVANCE_INST_RB w2 // update rPC, load wINST
- b.mi MterpCheckSuspendAndContinue
+ b.lt MterpCommonTakenBranchNoFlags
+ cmp wPROFILE, #JIT_CHECK_OSR // possible OSR re-entry?
+ b.eq .L_check_not_taken_osr
+ FETCH_ADVANCE_INST 2
GET_INST_OPCODE ip // extract opcode from wINST
GOTO_OPCODE ip // jump to next instruction
@@ -1479,9 +1385,8 @@
/* File: arm64/op_if_ge.S */
/* File: arm64/bincmp.S */
/*
- * Generic two-operand compare-and-branch operation. Provide a "revcmp"
- * fragment that specifies the *reverse* comparison to perform, e.g.
- * for "if-le" you would use "gt".
+ * Generic two-operand compare-and-branch operation. Provide a "condition"
+ * fragment that specifies the comparison to perform.
*
* For: if-eq, if-ne, if-lt, if-ge, if-gt, if-le
*/
@@ -1490,23 +1395,12 @@
ubfx w0, wINST, #8, #4 // w0<- A
GET_VREG w3, w1 // w3<- vB
GET_VREG w2, w0 // w2<- vA
- FETCH_S w1, 1 // w1<- branch offset, in code units
- mov w0, #2 // Offset if branch not taken
+ FETCH_S wINST, 1 // wINST<- branch offset, in code units
cmp w2, w3 // compare (vA, vB)
- csel wINST, w1, w0, ge // Branch if true, stashing result in callee save reg.
-#if MTERP_PROFILE_BRANCHES
- // TUINING: once measurements are complete, remove #if and hand-schedule.
- EXPORT_PC
- mov x0, xSELF
- add x1, xFP, #OFF_FP_SHADOWFRAME
- sbfm x2, xINST, 0, 31 // Sign extend branch offset
- bl MterpProfileBranch // (self, shadow_frame, offset)
- cbnz w0, MterpOnStackReplacement // Note: offset must be in xINST
-#endif
- ldr w7, [xSELF, #THREAD_FLAGS_OFFSET]
- adds w2, wINST, wINST // convert to bytes, check sign
- FETCH_ADVANCE_INST_RB w2 // update rPC, load wINST
- b.mi MterpCheckSuspendAndContinue
+ b.ge MterpCommonTakenBranchNoFlags
+ cmp wPROFILE, #JIT_CHECK_OSR // possible OSR re-entry?
+ b.eq .L_check_not_taken_osr
+ FETCH_ADVANCE_INST 2
GET_INST_OPCODE ip // extract opcode from wINST
GOTO_OPCODE ip // jump to next instruction
@@ -1517,9 +1411,8 @@
/* File: arm64/op_if_gt.S */
/* File: arm64/bincmp.S */
/*
- * Generic two-operand compare-and-branch operation. Provide a "revcmp"
- * fragment that specifies the *reverse* comparison to perform, e.g.
- * for "if-le" you would use "gt".
+ * Generic two-operand compare-and-branch operation. Provide a "condition"
+ * fragment that specifies the comparison to perform.
*
* For: if-eq, if-ne, if-lt, if-ge, if-gt, if-le
*/
@@ -1528,23 +1421,12 @@
ubfx w0, wINST, #8, #4 // w0<- A
GET_VREG w3, w1 // w3<- vB
GET_VREG w2, w0 // w2<- vA
- FETCH_S w1, 1 // w1<- branch offset, in code units
- mov w0, #2 // Offset if branch not taken
+ FETCH_S wINST, 1 // wINST<- branch offset, in code units
cmp w2, w3 // compare (vA, vB)
- csel wINST, w1, w0, gt // Branch if true, stashing result in callee save reg.
-#if MTERP_PROFILE_BRANCHES
- // TUINING: once measurements are complete, remove #if and hand-schedule.
- EXPORT_PC
- mov x0, xSELF
- add x1, xFP, #OFF_FP_SHADOWFRAME
- sbfm x2, xINST, 0, 31 // Sign extend branch offset
- bl MterpProfileBranch // (self, shadow_frame, offset)
- cbnz w0, MterpOnStackReplacement // Note: offset must be in xINST
-#endif
- ldr w7, [xSELF, #THREAD_FLAGS_OFFSET]
- adds w2, wINST, wINST // convert to bytes, check sign
- FETCH_ADVANCE_INST_RB w2 // update rPC, load wINST
- b.mi MterpCheckSuspendAndContinue
+ b.gt MterpCommonTakenBranchNoFlags
+ cmp wPROFILE, #JIT_CHECK_OSR // possible OSR re-entry?
+ b.eq .L_check_not_taken_osr
+ FETCH_ADVANCE_INST 2
GET_INST_OPCODE ip // extract opcode from wINST
GOTO_OPCODE ip // jump to next instruction
@@ -1555,9 +1437,8 @@
/* File: arm64/op_if_le.S */
/* File: arm64/bincmp.S */
/*
- * Generic two-operand compare-and-branch operation. Provide a "revcmp"
- * fragment that specifies the *reverse* comparison to perform, e.g.
- * for "if-le" you would use "gt".
+ * Generic two-operand compare-and-branch operation. Provide a "condition"
+ * fragment that specifies the comparison to perform.
*
* For: if-eq, if-ne, if-lt, if-ge, if-gt, if-le
*/
@@ -1566,23 +1447,12 @@
ubfx w0, wINST, #8, #4 // w0<- A
GET_VREG w3, w1 // w3<- vB
GET_VREG w2, w0 // w2<- vA
- FETCH_S w1, 1 // w1<- branch offset, in code units
- mov w0, #2 // Offset if branch not taken
+ FETCH_S wINST, 1 // wINST<- branch offset, in code units
cmp w2, w3 // compare (vA, vB)
- csel wINST, w1, w0, le // Branch if true, stashing result in callee save reg.
-#if MTERP_PROFILE_BRANCHES
- // TUINING: once measurements are complete, remove #if and hand-schedule.
- EXPORT_PC
- mov x0, xSELF
- add x1, xFP, #OFF_FP_SHADOWFRAME
- sbfm x2, xINST, 0, 31 // Sign extend branch offset
- bl MterpProfileBranch // (self, shadow_frame, offset)
- cbnz w0, MterpOnStackReplacement // Note: offset must be in xINST
-#endif
- ldr w7, [xSELF, #THREAD_FLAGS_OFFSET]
- adds w2, wINST, wINST // convert to bytes, check sign
- FETCH_ADVANCE_INST_RB w2 // update rPC, load wINST
- b.mi MterpCheckSuspendAndContinue
+ b.le MterpCommonTakenBranchNoFlags
+ cmp wPROFILE, #JIT_CHECK_OSR // possible OSR re-entry?
+ b.eq .L_check_not_taken_osr
+ FETCH_ADVANCE_INST 2
GET_INST_OPCODE ip // extract opcode from wINST
GOTO_OPCODE ip // jump to next instruction
@@ -1593,32 +1463,20 @@
/* File: arm64/op_if_eqz.S */
/* File: arm64/zcmp.S */
/*
- * Generic one-operand compare-and-branch operation. Provide a "revcmp"
- * fragment that specifies the *reverse* comparison to perform, e.g.
- * for "if-le" you would use "gt".
+ * Generic one-operand compare-and-branch operation. Provide a "condition"
+ * fragment that specifies the comparison to perform.
*
* for: if-eqz, if-nez, if-ltz, if-gez, if-gtz, if-lez
*/
/* if-cmp vAA, +BBBB */
lsr w0, wINST, #8 // w0<- AA
GET_VREG w2, w0 // w2<- vAA
- FETCH_S w1, 1 // w1<- branch offset, in code units
- mov w0, #2 // Branch offset if not taken
+ FETCH_S wINST, 1 // w1<- branch offset, in code units
cmp w2, #0 // compare (vA, 0)
- csel wINST, w1, w0, eq // Branch if true, stashing result in callee save reg
-#if MTERP_PROFILE_BRANCHES
- // TUNING: once measurements are complete, remove #if and hand-schedule.
- EXPORT_PC
- mov x0, xSELF
- add x1, xFP, #OFF_FP_SHADOWFRAME
- sbfm x2, xINST, 0, 31
- bl MterpProfileBranch // (self, shadow_frame, offset)
- cbnz w0, MterpOnStackReplacement // Note: offset must be in wINST
-#endif
- ldr w7, [xSELF, #THREAD_FLAGS_OFFSET]
- adds w2, wINST, wINST // convert to bytes & set flags
- FETCH_ADVANCE_INST_RB w2 // update rPC, load wINST
- b.mi MterpCheckSuspendAndContinue
+ b.eq MterpCommonTakenBranchNoFlags
+ cmp wPROFILE, #JIT_CHECK_OSR // possible OSR re-entry?
+ b.eq .L_check_not_taken_osr
+ FETCH_ADVANCE_INST 2
GET_INST_OPCODE ip // extract opcode from wINST
GOTO_OPCODE ip // jump to next instruction
@@ -1629,32 +1487,20 @@
/* File: arm64/op_if_nez.S */
/* File: arm64/zcmp.S */
/*
- * Generic one-operand compare-and-branch operation. Provide a "revcmp"
- * fragment that specifies the *reverse* comparison to perform, e.g.
- * for "if-le" you would use "gt".
+ * Generic one-operand compare-and-branch operation. Provide a "condition"
+ * fragment that specifies the comparison to perform.
*
* for: if-eqz, if-nez, if-ltz, if-gez, if-gtz, if-lez
*/
/* if-cmp vAA, +BBBB */
lsr w0, wINST, #8 // w0<- AA
GET_VREG w2, w0 // w2<- vAA
- FETCH_S w1, 1 // w1<- branch offset, in code units
- mov w0, #2 // Branch offset if not taken
+ FETCH_S wINST, 1 // w1<- branch offset, in code units
cmp w2, #0 // compare (vA, 0)
- csel wINST, w1, w0, ne // Branch if true, stashing result in callee save reg
-#if MTERP_PROFILE_BRANCHES
- // TUNING: once measurements are complete, remove #if and hand-schedule.
- EXPORT_PC
- mov x0, xSELF
- add x1, xFP, #OFF_FP_SHADOWFRAME
- sbfm x2, xINST, 0, 31
- bl MterpProfileBranch // (self, shadow_frame, offset)
- cbnz w0, MterpOnStackReplacement // Note: offset must be in wINST
-#endif
- ldr w7, [xSELF, #THREAD_FLAGS_OFFSET]
- adds w2, wINST, wINST // convert to bytes & set flags
- FETCH_ADVANCE_INST_RB w2 // update rPC, load wINST
- b.mi MterpCheckSuspendAndContinue
+ b.ne MterpCommonTakenBranchNoFlags
+ cmp wPROFILE, #JIT_CHECK_OSR // possible OSR re-entry?
+ b.eq .L_check_not_taken_osr
+ FETCH_ADVANCE_INST 2
GET_INST_OPCODE ip // extract opcode from wINST
GOTO_OPCODE ip // jump to next instruction
@@ -1665,32 +1511,20 @@
/* File: arm64/op_if_ltz.S */
/* File: arm64/zcmp.S */
/*
- * Generic one-operand compare-and-branch operation. Provide a "revcmp"
- * fragment that specifies the *reverse* comparison to perform, e.g.
- * for "if-le" you would use "gt".
+ * Generic one-operand compare-and-branch operation. Provide a "condition"
+ * fragment that specifies the comparison to perform.
*
* for: if-eqz, if-nez, if-ltz, if-gez, if-gtz, if-lez
*/
/* if-cmp vAA, +BBBB */
lsr w0, wINST, #8 // w0<- AA
GET_VREG w2, w0 // w2<- vAA
- FETCH_S w1, 1 // w1<- branch offset, in code units
- mov w0, #2 // Branch offset if not taken
+ FETCH_S wINST, 1 // w1<- branch offset, in code units
cmp w2, #0 // compare (vA, 0)
- csel wINST, w1, w0, lt // Branch if true, stashing result in callee save reg
-#if MTERP_PROFILE_BRANCHES
- // TUNING: once measurements are complete, remove #if and hand-schedule.
- EXPORT_PC
- mov x0, xSELF
- add x1, xFP, #OFF_FP_SHADOWFRAME
- sbfm x2, xINST, 0, 31
- bl MterpProfileBranch // (self, shadow_frame, offset)
- cbnz w0, MterpOnStackReplacement // Note: offset must be in wINST
-#endif
- ldr w7, [xSELF, #THREAD_FLAGS_OFFSET]
- adds w2, wINST, wINST // convert to bytes & set flags
- FETCH_ADVANCE_INST_RB w2 // update rPC, load wINST
- b.mi MterpCheckSuspendAndContinue
+ b.lt MterpCommonTakenBranchNoFlags
+ cmp wPROFILE, #JIT_CHECK_OSR // possible OSR re-entry?
+ b.eq .L_check_not_taken_osr
+ FETCH_ADVANCE_INST 2
GET_INST_OPCODE ip // extract opcode from wINST
GOTO_OPCODE ip // jump to next instruction
@@ -1701,32 +1535,20 @@
/* File: arm64/op_if_gez.S */
/* File: arm64/zcmp.S */
/*
- * Generic one-operand compare-and-branch operation. Provide a "revcmp"
- * fragment that specifies the *reverse* comparison to perform, e.g.
- * for "if-le" you would use "gt".
+ * Generic one-operand compare-and-branch operation. Provide a "condition"
+ * fragment that specifies the comparison to perform.
*
* for: if-eqz, if-nez, if-ltz, if-gez, if-gtz, if-lez
*/
/* if-cmp vAA, +BBBB */
lsr w0, wINST, #8 // w0<- AA
GET_VREG w2, w0 // w2<- vAA
- FETCH_S w1, 1 // w1<- branch offset, in code units
- mov w0, #2 // Branch offset if not taken
+ FETCH_S wINST, 1 // w1<- branch offset, in code units
cmp w2, #0 // compare (vA, 0)
- csel wINST, w1, w0, ge // Branch if true, stashing result in callee save reg
-#if MTERP_PROFILE_BRANCHES
- // TUNING: once measurements are complete, remove #if and hand-schedule.
- EXPORT_PC
- mov x0, xSELF
- add x1, xFP, #OFF_FP_SHADOWFRAME
- sbfm x2, xINST, 0, 31
- bl MterpProfileBranch // (self, shadow_frame, offset)
- cbnz w0, MterpOnStackReplacement // Note: offset must be in wINST
-#endif
- ldr w7, [xSELF, #THREAD_FLAGS_OFFSET]
- adds w2, wINST, wINST // convert to bytes & set flags
- FETCH_ADVANCE_INST_RB w2 // update rPC, load wINST
- b.mi MterpCheckSuspendAndContinue
+ b.ge MterpCommonTakenBranchNoFlags
+ cmp wPROFILE, #JIT_CHECK_OSR // possible OSR re-entry?
+ b.eq .L_check_not_taken_osr
+ FETCH_ADVANCE_INST 2
GET_INST_OPCODE ip // extract opcode from wINST
GOTO_OPCODE ip // jump to next instruction
@@ -1737,32 +1559,20 @@
/* File: arm64/op_if_gtz.S */
/* File: arm64/zcmp.S */
/*
- * Generic one-operand compare-and-branch operation. Provide a "revcmp"
- * fragment that specifies the *reverse* comparison to perform, e.g.
- * for "if-le" you would use "gt".
+ * Generic one-operand compare-and-branch operation. Provide a "condition"
+ * fragment that specifies the comparison to perform.
*
* for: if-eqz, if-nez, if-ltz, if-gez, if-gtz, if-lez
*/
/* if-cmp vAA, +BBBB */
lsr w0, wINST, #8 // w0<- AA
GET_VREG w2, w0 // w2<- vAA
- FETCH_S w1, 1 // w1<- branch offset, in code units
- mov w0, #2 // Branch offset if not taken
+ FETCH_S wINST, 1 // w1<- branch offset, in code units
cmp w2, #0 // compare (vA, 0)
- csel wINST, w1, w0, gt // Branch if true, stashing result in callee save reg
-#if MTERP_PROFILE_BRANCHES
- // TUNING: once measurements are complete, remove #if and hand-schedule.
- EXPORT_PC
- mov x0, xSELF
- add x1, xFP, #OFF_FP_SHADOWFRAME
- sbfm x2, xINST, 0, 31
- bl MterpProfileBranch // (self, shadow_frame, offset)
- cbnz w0, MterpOnStackReplacement // Note: offset must be in wINST
-#endif
- ldr w7, [xSELF, #THREAD_FLAGS_OFFSET]
- adds w2, wINST, wINST // convert to bytes & set flags
- FETCH_ADVANCE_INST_RB w2 // update rPC, load wINST
- b.mi MterpCheckSuspendAndContinue
+ b.gt MterpCommonTakenBranchNoFlags
+ cmp wPROFILE, #JIT_CHECK_OSR // possible OSR re-entry?
+ b.eq .L_check_not_taken_osr
+ FETCH_ADVANCE_INST 2
GET_INST_OPCODE ip // extract opcode from wINST
GOTO_OPCODE ip // jump to next instruction
@@ -1773,32 +1583,20 @@
/* File: arm64/op_if_lez.S */
/* File: arm64/zcmp.S */
/*
- * Generic one-operand compare-and-branch operation. Provide a "revcmp"
- * fragment that specifies the *reverse* comparison to perform, e.g.
- * for "if-le" you would use "gt".
+ * Generic one-operand compare-and-branch operation. Provide a "condition"
+ * fragment that specifies the comparison to perform.
*
* for: if-eqz, if-nez, if-ltz, if-gez, if-gtz, if-lez
*/
/* if-cmp vAA, +BBBB */
lsr w0, wINST, #8 // w0<- AA
GET_VREG w2, w0 // w2<- vAA
- FETCH_S w1, 1 // w1<- branch offset, in code units
- mov w0, #2 // Branch offset if not taken
+ FETCH_S wINST, 1 // w1<- branch offset, in code units
cmp w2, #0 // compare (vA, 0)
- csel wINST, w1, w0, le // Branch if true, stashing result in callee save reg
-#if MTERP_PROFILE_BRANCHES
- // TUNING: once measurements are complete, remove #if and hand-schedule.
- EXPORT_PC
- mov x0, xSELF
- add x1, xFP, #OFF_FP_SHADOWFRAME
- sbfm x2, xINST, 0, 31
- bl MterpProfileBranch // (self, shadow_frame, offset)
- cbnz w0, MterpOnStackReplacement // Note: offset must be in wINST
-#endif
- ldr w7, [xSELF, #THREAD_FLAGS_OFFSET]
- adds w2, wINST, wINST // convert to bytes & set flags
- FETCH_ADVANCE_INST_RB w2 // update rPC, load wINST
- b.mi MterpCheckSuspendAndContinue
+ b.le MterpCommonTakenBranchNoFlags
+ cmp wPROFILE, #JIT_CHECK_OSR // possible OSR re-entry?
+ b.eq .L_check_not_taken_osr
+ FETCH_ADVANCE_INST 2
GET_INST_OPCODE ip // extract opcode from wINST
GOTO_OPCODE ip // jump to next instruction
@@ -11596,6 +11394,107 @@
GET_INST_OPCODE ip
GOTO_OPCODE ip
/* NOTE: no fallthrough */
+/*
+ * Common handling for branches with support for Jit profiling.
+ * On entry:
+ * wINST <= signed offset
+ * wPROFILE <= signed hotness countdown (expanded to 32 bits)
+ * condition bits <= set to establish sign of offset (use "NoFlags" entry if not)
+ *
+ * We have quite a few different cases for branch profiling, OSR detection and
+ * suspend check support here.
+ *
+ * Taken backward branches:
+ * If profiling active, do hotness countdown and report if we hit zero.
+ * If in osr check mode, see if our target is a compiled loop header entry and do OSR if so.
+ * Is there a pending suspend request? If so, suspend.
+ *
+ * Taken forward branches and not-taken backward branches:
+ * If in osr check mode, see if our target is a compiled loop header entry and do OSR if so.
+ *
+ * Our most common case is expected to be a taken backward branch with active jit profiling,
+ * but no full OSR check and no pending suspend request.
+ * Next most common case is not-taken branch with no full OSR check.
+ *
+ */
+MterpCommonTakenBranchNoFlags:
+ cmp wINST, #0
+ b.gt .L_forward_branch // don't add forward branches to hotness
+ tbnz wPROFILE, #31, .L_no_count_backwards // go if negative
+ subs wPROFILE, wPROFILE, #1 // countdown
+ b.eq .L_add_batch // counted down to zero - report
+.L_resume_backward_branch:
+ ldr lr, [xSELF, #THREAD_FLAGS_OFFSET]
+ add w2, wINST, wINST // w2<- byte offset
+ FETCH_ADVANCE_INST_RB w2 // update rPC, load wINST
+ REFRESH_IBASE
+ ands lr, lr, #(THREAD_SUSPEND_REQUEST | THREAD_CHECKPOINT_REQUEST)
+ b.ne .L_suspend_request_pending
+ GET_INST_OPCODE ip // extract opcode from wINST
+ GOTO_OPCODE ip // jump to next instruction
+
+.L_suspend_request_pending:
+ EXPORT_PC
+ mov x0, xSELF
+ bl MterpSuspendCheck // (self)
+ cbnz x0, MterpFallback
+ REFRESH_IBASE // might have changed during suspend
+ GET_INST_OPCODE ip // extract opcode from wINST
+ GOTO_OPCODE ip // jump to next instruction
+
+.L_no_count_backwards:
+ cmp wPROFILE, #JIT_CHECK_OSR // possible OSR re-entry?
+ b.ne .L_resume_backward_branch
+ mov x0, xSELF
+ add x1, xFP, #OFF_FP_SHADOWFRAME
+ mov x2, xINST
+ EXPORT_PC
+ bl MterpMaybeDoOnStackReplacement // (self, shadow_frame, offset)
+ cbnz x0, MterpOnStackReplacement
+ b .L_resume_backward_branch
+
+.L_forward_branch:
+ cmp wPROFILE, #JIT_CHECK_OSR // possible OSR re-entry?
+ b.eq .L_check_osr_forward
+.L_resume_forward_branch:
+ add w2, wINST, wINST // w2<- byte offset
+ FETCH_ADVANCE_INST_RB w2 // update rPC, load wINST
+ GET_INST_OPCODE ip // extract opcode from wINST
+ GOTO_OPCODE ip // jump to next instruction
+
+.L_check_osr_forward:
+ mov x0, xSELF
+ add x1, xFP, #OFF_FP_SHADOWFRAME
+ mov x2, xINST
+ EXPORT_PC
+ bl MterpMaybeDoOnStackReplacement // (self, shadow_frame, offset)
+ cbnz x0, MterpOnStackReplacement
+ b .L_resume_forward_branch
+
+.L_add_batch:
+ add x1, xFP, #OFF_FP_SHADOWFRAME
+ strh wPROFILE, [x1, #SHADOWFRAME_HOTNESS_COUNTDOWN_OFFSET]
+ ldr x0, [xFP, #OFF_FP_METHOD]
+ mov x2, xSELF
+ bl MterpAddHotnessBatch // (method, shadow_frame, self)
+ mov wPROFILE, w0 // restore new hotness countdown to wPROFILE
+ b .L_no_count_backwards
+
+/*
+ * Entered from the conditional branch handlers when OSR check request active on
+ * not-taken path. All Dalvik not-taken conditional branch offsets are 2.
+ */
+.L_check_not_taken_osr:
+ mov x0, xSELF
+ add x1, xFP, #OFF_FP_SHADOWFRAME
+ mov x2, #2
+ EXPORT_PC
+ bl MterpMaybeDoOnStackReplacement // (self, shadow_frame, offset)
+ cbnz x0, MterpOnStackReplacement
+ FETCH_ADVANCE_INST 2
+ GET_INST_OPCODE ip // extract opcode from wINST
+ GOTO_OPCODE ip // jump to next instruction
+
/*
* Check for suspend check request. Assumes wINST already loaded, xPC advanced and
@@ -11664,10 +11563,36 @@
check2:
mov x0, #1 // signal return to caller.
MterpDone:
- ldp fp, lr, [sp, #48]
- ldp xPC, xFP, [sp, #32]
- ldp xSELF, xINST, [sp, #16]
- ldp xIBASE, xREFS, [sp], #64
+/*
+ * At this point, we expect wPROFILE to be non-zero. If negative, hotness is disabled or we're
+ * checking for OSR. If greater than zero, we might have unreported hotness to register
+ * (the difference between the ending wPROFILE and the cached hotness counter). wPROFILE
+ * should only reach zero immediately after a hotness decrement, and is then reset to either
+ * a negative special state or the new non-zero countdown value.
+ */
+ cmp wPROFILE, #0
+ bgt MterpProfileActive // if > 0, we may have some counts to report.
+ ldp fp, lr, [sp, #64]
+ ldp xPC, xFP, [sp, #48]
+ ldp xSELF, xINST, [sp, #32]
+ ldp xIBASE, xREFS, [sp, #16]
+ ldp xPROFILE, x27, [sp], #80
+ ret
+
+MterpProfileActive:
+ mov xINST, x0 // stash return value
+ /* Report cached hotness counts */
+ ldr x0, [xFP, #OFF_FP_METHOD]
+ add x1, xFP, #OFF_FP_SHADOWFRAME
+ mov x2, xSELF
+ strh wPROFILE, [x1, #SHADOWFRAME_HOTNESS_COUNTDOWN_OFFSET]
+ bl MterpAddHotnessBatch // (method, shadow_frame, self)
+ mov x0, xINST // restore return value
+ ldp fp, lr, [sp, #64]
+ ldp xPC, xFP, [sp, #48]
+ ldp xSELF, xINST, [sp, #32]
+ ldp xIBASE, xREFS, [sp, #16]
+ ldp xPROFILE, x27, [sp], #80
ret
.cfi_endproc
diff --git a/runtime/interpreter/mterp/out/mterp_x86.S b/runtime/interpreter/mterp/out/mterp_x86.S
index ebac5fc..685b9b6 100644
--- a/runtime/interpreter/mterp/out/mterp_x86.S
+++ b/runtime/interpreter/mterp/out/mterp_x86.S
@@ -124,6 +124,21 @@
.cfi_restore \_reg
.endm
+/*
+ * Instead of holding a pointer to the shadow frame, we keep rFP at the base of the vregs. So,
+ * to access other shadow frame fields, we need to use a backwards offset. Define those here.
+ */
+#define OFF_FP(a) (a - SHADOWFRAME_VREGS_OFFSET)
+#define OFF_FP_NUMBER_OF_VREGS OFF_FP(SHADOWFRAME_NUMBER_OF_VREGS_OFFSET)
+#define OFF_FP_DEX_PC OFF_FP(SHADOWFRAME_DEX_PC_OFFSET)
+#define OFF_FP_LINK OFF_FP(SHADOWFRAME_LINK_OFFSET)
+#define OFF_FP_METHOD OFF_FP(SHADOWFRAME_METHOD_OFFSET)
+#define OFF_FP_RESULT_REGISTER OFF_FP(SHADOWFRAME_RESULT_REGISTER_OFFSET)
+#define OFF_FP_DEX_PC_PTR OFF_FP(SHADOWFRAME_DEX_PC_PTR_OFFSET)
+#define OFF_FP_CODE_ITEM OFF_FP(SHADOWFRAME_CODE_ITEM_OFFSET)
+#define OFF_FP_COUNTDOWN_OFFSET OFF_FP(SHADOWFRAME_HOTNESS_COUNTDOWN_OFFSET)
+#define OFF_FP_SHADOWFRAME OFF_FP(0)
+
/* Frame size must be 16-byte aligned.
* Remember about 4 bytes for return address + 4 * 4 for spills
*/
@@ -155,43 +170,11 @@
#define rINSTbl %bl
#define rIBASE %edx
#define rREFS %ebp
+#define rPROFILE OFF_FP_COUNTDOWN_OFFSET(rFP)
-/*
- * Instead of holding a pointer to the shadow frame, we keep rFP at the base of the vregs. So,
- * to access other shadow frame fields, we need to use a backwards offset. Define those here.
- */
-#define OFF_FP(a) (a - SHADOWFRAME_VREGS_OFFSET)
-#define OFF_FP_NUMBER_OF_VREGS OFF_FP(SHADOWFRAME_NUMBER_OF_VREGS_OFFSET)
-#define OFF_FP_DEX_PC OFF_FP(SHADOWFRAME_DEX_PC_OFFSET)
-#define OFF_FP_LINK OFF_FP(SHADOWFRAME_LINK_OFFSET)
-#define OFF_FP_METHOD OFF_FP(SHADOWFRAME_METHOD_OFFSET)
-#define OFF_FP_RESULT_REGISTER OFF_FP(SHADOWFRAME_RESULT_REGISTER_OFFSET)
-#define OFF_FP_DEX_PC_PTR OFF_FP(SHADOWFRAME_DEX_PC_PTR_OFFSET)
-#define OFF_FP_CODE_ITEM OFF_FP(SHADOWFRAME_CODE_ITEM_OFFSET)
-#define OFF_FP_SHADOWFRAME (-SHADOWFRAME_VREGS_OFFSET)
-
-#define MTERP_PROFILE_BRANCHES 1
#define MTERP_LOGGING 0
/*
- * Profile branch. rINST should contain the offset. %eax is scratch.
- */
-.macro MTERP_PROFILE_BRANCH
-#ifdef MTERP_PROFILE_BRANCHES
- EXPORT_PC
- movl rSELF, %eax
- movl %eax, OUT_ARG0(%esp)
- leal OFF_FP_SHADOWFRAME(rFP), %eax
- movl %eax, OUT_ARG1(%esp)
- movl rINST, OUT_ARG2(%esp)
- call SYMBOL(MterpProfileBranch)
- testb %al, %al
- jnz MterpOnStackReplacement
- RESTORE_IBASE
-#endif
-.endm
-
-/*
* "export" the PC to dex_pc field in the shadow frame, f/b/o future exception objects. Must
* be done *before* something throws.
*
@@ -399,6 +382,13 @@
lea (rPC, %eax, 2), rPC
EXPORT_PC
+ /* Set up for backwards branches & osr profiling */
+ movl OFF_FP_METHOD(rFP), %eax
+ movl %eax, OUT_ARG0(%esp)
+ leal OFF_FP_SHADOWFRAME(rFP), %ecx
+ movl %ecx, OUT_ARG1(%esp)
+ call SYMBOL(MterpSetUpHotnessCountdown)
+
/* Starting ibase */
REFRESH_IBASE
@@ -1099,12 +1089,8 @@
*/
/* goto +AA */
movsbl rINSTbl, rINST # rINST <- ssssssAA
- MTERP_PROFILE_BRANCH
- addl rINST, rINST # rINST <- AA * 2
- leal (rPC, rINST), rPC
- FETCH_INST
- jle MterpCheckSuspendAndContinue # AA * 2 <= 0 => suspend check
- GOTO_NEXT
+ testl rINST, rINST
+ jmp MterpCommonTakenBranch
/* ------------------------------ */
.balign 128
@@ -1118,12 +1104,8 @@
*/
/* goto/16 +AAAA */
movswl 2(rPC), rINST # rINST <- ssssAAAA
- MTERP_PROFILE_BRANCH
- addl rINST, rINST # rINST <- AA * 2
- leal (rPC, rINST), rPC
- FETCH_INST
- jle MterpCheckSuspendAndContinue # AA * 2 <= 0 => suspend check
- GOTO_NEXT
+ testl rINST, rINST
+ jmp MterpCommonTakenBranch
/* ------------------------------ */
.balign 128
@@ -1142,12 +1124,8 @@
*/
/* goto/32 +AAAAAAAA */
movl 2(rPC), rINST # rINST <- AAAAAAAA
- MTERP_PROFILE_BRANCH
- addl rINST, rINST # rINST <- AA * 2
- leal (rPC, rINST), rPC
- FETCH_INST
- jle MterpCheckSuspendAndContinue # AA * 2 <= 0 => suspend check
- GOTO_NEXT
+ testl rINST, rINST
+ jmp MterpCommonTakenBranch
/* ------------------------------ */
.balign 128
@@ -1169,14 +1147,10 @@
movl %eax, OUT_ARG1(%esp) # ARG1 <- vAA
movl %ecx, OUT_ARG0(%esp) # ARG0 <- switchData
call SYMBOL(MterpDoPackedSwitch)
- movl %eax, rINST
- MTERP_PROFILE_BRANCH
- addl rINST, rINST
- leal (rPC, rINST), rPC
- FETCH_INST
REFRESH_IBASE
- jle MterpCheckSuspendAndContinue
- GOTO_NEXT
+ testl %eax, %eax
+ movl %eax, rINST
+ jmp MterpCommonTakenBranch
/* ------------------------------ */
.balign 128
@@ -1199,14 +1173,10 @@
movl %eax, OUT_ARG1(%esp) # ARG1 <- vAA
movl %ecx, OUT_ARG0(%esp) # ARG0 <- switchData
call SYMBOL(MterpDoSparseSwitch)
- movl %eax, rINST
- MTERP_PROFILE_BRANCH
- addl rINST, rINST
- leal (rPC, rINST), rPC
- FETCH_INST
REFRESH_IBASE
- jle MterpCheckSuspendAndContinue
- GOTO_NEXT
+ testl %eax, %eax
+ movl %eax, rINST
+ jmp MterpCommonTakenBranch
/* ------------------------------ */
@@ -1423,16 +1393,14 @@
GET_VREG %eax, %ecx # eax <- vA
sarl $4, rINST # rINST <- B
cmpl VREG_ADDRESS(rINST), %eax # compare (vA, vB)
- movl $2, rINST
jne 1f
movswl 2(rPC), rINST # Get signed branch offset
+ testl rINST, rINST
+ jmp MterpCommonTakenBranch
1:
- MTERP_PROFILE_BRANCH
- addl rINST, rINST # eax <- AA * 2
- leal (rPC, rINST), rPC
- FETCH_INST
- jle MterpCheckSuspendAndContinue # AA * 2 <= 0 => suspend check
- GOTO_NEXT
+ cmpw $JIT_CHECK_OSR, rPROFILE
+ je .L_check_not_taken_osr
+ ADVANCE_PC_FETCH_AND_GOTO_NEXT 2
/* ------------------------------ */
@@ -1453,16 +1421,14 @@
GET_VREG %eax, %ecx # eax <- vA
sarl $4, rINST # rINST <- B
cmpl VREG_ADDRESS(rINST), %eax # compare (vA, vB)
- movl $2, rINST
je 1f
movswl 2(rPC), rINST # Get signed branch offset
+ testl rINST, rINST
+ jmp MterpCommonTakenBranch
1:
- MTERP_PROFILE_BRANCH
- addl rINST, rINST # eax <- AA * 2
- leal (rPC, rINST), rPC
- FETCH_INST
- jle MterpCheckSuspendAndContinue # AA * 2 <= 0 => suspend check
- GOTO_NEXT
+ cmpw $JIT_CHECK_OSR, rPROFILE
+ je .L_check_not_taken_osr
+ ADVANCE_PC_FETCH_AND_GOTO_NEXT 2
/* ------------------------------ */
@@ -1483,16 +1449,14 @@
GET_VREG %eax, %ecx # eax <- vA
sarl $4, rINST # rINST <- B
cmpl VREG_ADDRESS(rINST), %eax # compare (vA, vB)
- movl $2, rINST
jge 1f
movswl 2(rPC), rINST # Get signed branch offset
+ testl rINST, rINST
+ jmp MterpCommonTakenBranch
1:
- MTERP_PROFILE_BRANCH
- addl rINST, rINST # eax <- AA * 2
- leal (rPC, rINST), rPC
- FETCH_INST
- jle MterpCheckSuspendAndContinue # AA * 2 <= 0 => suspend check
- GOTO_NEXT
+ cmpw $JIT_CHECK_OSR, rPROFILE
+ je .L_check_not_taken_osr
+ ADVANCE_PC_FETCH_AND_GOTO_NEXT 2
/* ------------------------------ */
@@ -1513,16 +1477,14 @@
GET_VREG %eax, %ecx # eax <- vA
sarl $4, rINST # rINST <- B
cmpl VREG_ADDRESS(rINST), %eax # compare (vA, vB)
- movl $2, rINST
jl 1f
movswl 2(rPC), rINST # Get signed branch offset
+ testl rINST, rINST
+ jmp MterpCommonTakenBranch
1:
- MTERP_PROFILE_BRANCH
- addl rINST, rINST # eax <- AA * 2
- leal (rPC, rINST), rPC
- FETCH_INST
- jle MterpCheckSuspendAndContinue # AA * 2 <= 0 => suspend check
- GOTO_NEXT
+ cmpw $JIT_CHECK_OSR, rPROFILE
+ je .L_check_not_taken_osr
+ ADVANCE_PC_FETCH_AND_GOTO_NEXT 2
/* ------------------------------ */
@@ -1543,16 +1505,14 @@
GET_VREG %eax, %ecx # eax <- vA
sarl $4, rINST # rINST <- B
cmpl VREG_ADDRESS(rINST), %eax # compare (vA, vB)
- movl $2, rINST
jle 1f
movswl 2(rPC), rINST # Get signed branch offset
+ testl rINST, rINST
+ jmp MterpCommonTakenBranch
1:
- MTERP_PROFILE_BRANCH
- addl rINST, rINST # eax <- AA * 2
- leal (rPC, rINST), rPC
- FETCH_INST
- jle MterpCheckSuspendAndContinue # AA * 2 <= 0 => suspend check
- GOTO_NEXT
+ cmpw $JIT_CHECK_OSR, rPROFILE
+ je .L_check_not_taken_osr
+ ADVANCE_PC_FETCH_AND_GOTO_NEXT 2
/* ------------------------------ */
@@ -1573,16 +1533,14 @@
GET_VREG %eax, %ecx # eax <- vA
sarl $4, rINST # rINST <- B
cmpl VREG_ADDRESS(rINST), %eax # compare (vA, vB)
- movl $2, rINST
jg 1f
movswl 2(rPC), rINST # Get signed branch offset
+ testl rINST, rINST
+ jmp MterpCommonTakenBranch
1:
- MTERP_PROFILE_BRANCH
- addl rINST, rINST # eax <- AA * 2
- leal (rPC, rINST), rPC
- FETCH_INST
- jle MterpCheckSuspendAndContinue # AA * 2 <= 0 => suspend check
- GOTO_NEXT
+ cmpw $JIT_CHECK_OSR, rPROFILE
+ je .L_check_not_taken_osr
+ ADVANCE_PC_FETCH_AND_GOTO_NEXT 2
/* ------------------------------ */
@@ -1599,16 +1557,14 @@
*/
/* if-cmp vAA, +BBBB */
cmpl $0, VREG_ADDRESS(rINST) # compare (vA, 0)
- movl $2, rINST
jne 1f
movswl 2(rPC), rINST # fetch signed displacement
+ testl rINST, rINST
+ jmp MterpCommonTakenBranch
1:
- MTERP_PROFILE_BRANCH
- addl rINST, rINST # eax <- AA * 2
- leal (rPC, rINST), rPC
- FETCH_INST
- jle MterpCheckSuspendAndContinue # AA * 2 <= 0 => suspend check
- GOTO_NEXT
+ cmpw $JIT_CHECK_OSR, rPROFILE
+ je .L_check_not_taken_osr
+ ADVANCE_PC_FETCH_AND_GOTO_NEXT 2
/* ------------------------------ */
@@ -1625,16 +1581,14 @@
*/
/* if-cmp vAA, +BBBB */
cmpl $0, VREG_ADDRESS(rINST) # compare (vA, 0)
- movl $2, rINST
je 1f
movswl 2(rPC), rINST # fetch signed displacement
+ testl rINST, rINST
+ jmp MterpCommonTakenBranch
1:
- MTERP_PROFILE_BRANCH
- addl rINST, rINST # eax <- AA * 2
- leal (rPC, rINST), rPC
- FETCH_INST
- jle MterpCheckSuspendAndContinue # AA * 2 <= 0 => suspend check
- GOTO_NEXT
+ cmpw $JIT_CHECK_OSR, rPROFILE
+ je .L_check_not_taken_osr
+ ADVANCE_PC_FETCH_AND_GOTO_NEXT 2
/* ------------------------------ */
@@ -1651,16 +1605,14 @@
*/
/* if-cmp vAA, +BBBB */
cmpl $0, VREG_ADDRESS(rINST) # compare (vA, 0)
- movl $2, rINST
jge 1f
movswl 2(rPC), rINST # fetch signed displacement
+ testl rINST, rINST
+ jmp MterpCommonTakenBranch
1:
- MTERP_PROFILE_BRANCH
- addl rINST, rINST # eax <- AA * 2
- leal (rPC, rINST), rPC
- FETCH_INST
- jle MterpCheckSuspendAndContinue # AA * 2 <= 0 => suspend check
- GOTO_NEXT
+ cmpw $JIT_CHECK_OSR, rPROFILE
+ je .L_check_not_taken_osr
+ ADVANCE_PC_FETCH_AND_GOTO_NEXT 2
/* ------------------------------ */
@@ -1677,16 +1629,14 @@
*/
/* if-cmp vAA, +BBBB */
cmpl $0, VREG_ADDRESS(rINST) # compare (vA, 0)
- movl $2, rINST
jl 1f
movswl 2(rPC), rINST # fetch signed displacement
+ testl rINST, rINST
+ jmp MterpCommonTakenBranch
1:
- MTERP_PROFILE_BRANCH
- addl rINST, rINST # eax <- AA * 2
- leal (rPC, rINST), rPC
- FETCH_INST
- jle MterpCheckSuspendAndContinue # AA * 2 <= 0 => suspend check
- GOTO_NEXT
+ cmpw $JIT_CHECK_OSR, rPROFILE
+ je .L_check_not_taken_osr
+ ADVANCE_PC_FETCH_AND_GOTO_NEXT 2
/* ------------------------------ */
@@ -1703,16 +1653,14 @@
*/
/* if-cmp vAA, +BBBB */
cmpl $0, VREG_ADDRESS(rINST) # compare (vA, 0)
- movl $2, rINST
jle 1f
movswl 2(rPC), rINST # fetch signed displacement
+ testl rINST, rINST
+ jmp MterpCommonTakenBranch
1:
- MTERP_PROFILE_BRANCH
- addl rINST, rINST # eax <- AA * 2
- leal (rPC, rINST), rPC
- FETCH_INST
- jle MterpCheckSuspendAndContinue # AA * 2 <= 0 => suspend check
- GOTO_NEXT
+ cmpw $JIT_CHECK_OSR, rPROFILE
+ je .L_check_not_taken_osr
+ ADVANCE_PC_FETCH_AND_GOTO_NEXT 2
/* ------------------------------ */
@@ -1729,16 +1677,14 @@
*/
/* if-cmp vAA, +BBBB */
cmpl $0, VREG_ADDRESS(rINST) # compare (vA, 0)
- movl $2, rINST
jg 1f
movswl 2(rPC), rINST # fetch signed displacement
+ testl rINST, rINST
+ jmp MterpCommonTakenBranch
1:
- MTERP_PROFILE_BRANCH
- addl rINST, rINST # eax <- AA * 2
- leal (rPC, rINST), rPC
- FETCH_INST
- jle MterpCheckSuspendAndContinue # AA * 2 <= 0 => suspend check
- GOTO_NEXT
+ cmpw $JIT_CHECK_OSR, rPROFILE
+ je .L_check_not_taken_osr
+ ADVANCE_PC_FETCH_AND_GOTO_NEXT 2
/* ------------------------------ */
@@ -12936,20 +12882,120 @@
/* NOTE: no fallthrough */
/*
- * Check for suspend check request. Assumes rINST already loaded, rPC advanced and
- * still needs to get the opcode and branch to it, and flags are in lr.
+ * Common handling for branches with support for Jit profiling.
+ * On entry:
+ * rINST <= signed offset
+ * condition bits <= set to establish sign of offset (use "NoFlags" entry if not)
+ *
+ * We have quite a few different cases for branch profiling, OSR detection and
+ * suspend check support here.
+ *
+ * Taken backward branches:
+ * If profiling active, do hotness countdown and report if we hit zero.
+ * If in osr check mode, see if our target is a compiled loop header entry and do OSR if so.
+ * Is there a pending suspend request? If so, suspend.
+ *
+ * Taken forward branches and not-taken backward branches:
+ * If in osr check mode, see if our target is a compiled loop header entry and do OSR if so.
+ *
+ * Our most common case is expected to be a taken backward branch with active jit profiling,
+ * but no full OSR check and no pending suspend request.
+ * Next most common case is not-taken branch with no full OSR check.
+ *
*/
-MterpCheckSuspendAndContinue:
+MterpCommonTakenBranch:
+ jg .L_forward_branch # don't add forward branches to hotness
+/*
+ * We need to subtract 1 from positive values and we should not see 0 here,
+ * so we may use the result of the comparison with -1.
+ */
+#if JIT_CHECK_OSR != -1
+# error "JIT_CHECK_OSR must be -1."
+#endif
+ cmpw $JIT_CHECK_OSR, rPROFILE
+ je .L_osr_check
+ decw rPROFILE
+ je .L_add_batch # counted down to zero - report
+.L_resume_backward_branch:
movl rSELF, %eax
- EXPORT_PC
testl $(THREAD_SUSPEND_REQUEST | THREAD_CHECKPOINT_REQUEST), THREAD_FLAGS_OFFSET(%eax)
- jz 1f
- movl %eax, OUT_ARG0(%esp)
- call SYMBOL(MterpSuspendCheck)
+ leal (rPC, rINST, 2), rPC
+ FETCH_INST
+ jnz .L_suspend_request_pending
REFRESH_IBASE
-1:
GOTO_NEXT
+.L_suspend_request_pending:
+ EXPORT_PC
+ movl %eax, OUT_ARG0(%esp) # rSELF in eax
+ call SYMBOL(MterpSuspendCheck) # (self)
+ testb %al, %al
+ jnz MterpFallback
+ REFRESH_IBASE # might have changed during suspend
+ GOTO_NEXT
+
+.L_no_count_backwards:
+ cmpw $JIT_CHECK_OSR, rPROFILE # possible OSR re-entry?
+ jne .L_resume_backward_branch
+.L_osr_check:
+ EXPORT_PC
+ movl rSELF, %eax
+ movl %eax, OUT_ARG0(%esp)
+ leal OFF_FP_SHADOWFRAME(rFP), %ecx
+ movl %ecx, OUT_ARG1(%esp)
+ movl rINST, OUT_ARG2(%esp)
+ call SYMBOL(MterpMaybeDoOnStackReplacement) # (self, shadow_frame, offset)
+ testb %al, %al
+ jz .L_resume_backward_branch
+ jmp MterpOnStackReplacement
+
+.L_forward_branch:
+ cmpw $JIT_CHECK_OSR, rPROFILE # possible OSR re-entry?
+ je .L_check_osr_forward
+.L_resume_forward_branch:
+ leal (rPC, rINST, 2), rPC
+ FETCH_INST
+ GOTO_NEXT
+
+.L_check_osr_forward:
+ EXPORT_PC
+ movl rSELF, %eax
+ movl %eax, OUT_ARG0(%esp)
+ leal OFF_FP_SHADOWFRAME(rFP), %ecx
+ movl %ecx, OUT_ARG1(%esp)
+ movl rINST, OUT_ARG2(%esp)
+ call SYMBOL(MterpMaybeDoOnStackReplacement) # (self, shadow_frame, offset)
+ testb %al, %al
+ REFRESH_IBASE
+ jz .L_resume_forward_branch
+ jmp MterpOnStackReplacement
+
+.L_add_batch:
+ movl OFF_FP_METHOD(rFP), %eax
+ movl %eax, OUT_ARG0(%esp)
+ leal OFF_FP_SHADOWFRAME(rFP), %ecx
+ movl %ecx, OUT_ARG1(%esp)
+ movl rSELF, %eax
+ movl %eax, OUT_ARG2(%esp)
+ call SYMBOL(MterpAddHotnessBatch) # (method, shadow_frame, self)
+ jmp .L_no_count_backwards
+
+/*
+ * Entered from the conditional branch handlers when OSR check request active on
+ * not-taken path. All Dalvik not-taken conditional branch offsets are 2.
+ */
+.L_check_not_taken_osr:
+ movl rSELF, %eax
+ movl %eax, OUT_ARG0(%esp)
+ leal OFF_FP_SHADOWFRAME(rFP), %ecx
+ movl %ecx, OUT_ARG1(%esp)
+ movl rINST, OUT_ARG3(%esp)
+ call SYMBOL(MterpMaybeDoOnStackReplacement) # (self, shadow_frame, offset)
+ testb %al, %al
+ REFRESH_IBASE
+ jnz MterpOnStackReplacement
+ ADVANCE_PC_FETCH_AND_GOTO_NEXT 2
+
/*
* On-stack replacement has happened, and now we've returned from the compiled method.
*/
@@ -12994,7 +13040,29 @@
movl %ecx, 4(%edx)
mov $1, %eax
MterpDone:
+/*
+ * At this point, we expect rPROFILE to be non-zero. If negative, hotness is disabled or we're
+ * checking for OSR. If greater than zero, we might have unreported hotness to register
+ * (the difference between the ending rPROFILE and the cached hotness counter). rPROFILE
+ * should only reach zero immediately after a hotness decrement, and is then reset to either
+ * a negative special state or the new non-zero countdown value.
+ */
+ cmpw $0, rPROFILE
+ jle MRestoreFrame # if > 0, we may have some counts to report.
+
+ movl %eax, rINST # stash return value
+ /* Report cached hotness counts */
+ movl OFF_FP_METHOD(rFP), %eax
+ movl %eax, OUT_ARG0(%esp)
+ leal OFF_FP_SHADOWFRAME(rFP), %ecx
+ movl %ecx, OUT_ARG1(%esp)
+ movl rSELF, %eax
+ movl %eax, OUT_ARG2(%esp)
+ call SYMBOL(MterpAddHotnessBatch) # (method, shadow_frame, self)
+ movl rINST, %eax # restore return value
+
/* pop up frame */
+MRestoreFrame:
addl $FRAME_SIZE, %esp
.cfi_adjust_cfa_offset -FRAME_SIZE
diff --git a/runtime/interpreter/mterp/out/mterp_x86_64.S b/runtime/interpreter/mterp/out/mterp_x86_64.S
index a1360e0..f78bcf0 100644
--- a/runtime/interpreter/mterp/out/mterp_x86_64.S
+++ b/runtime/interpreter/mterp/out/mterp_x86_64.S
@@ -120,6 +120,21 @@
.cfi_restore \_reg
.endm
+/*
+ * Instead of holding a pointer to the shadow frame, we keep rFP at the base of the vregs. So,
+ * to access other shadow frame fields, we need to use a backwards offset. Define those here.
+ */
+#define OFF_FP(a) (a - SHADOWFRAME_VREGS_OFFSET)
+#define OFF_FP_NUMBER_OF_VREGS OFF_FP(SHADOWFRAME_NUMBER_OF_VREGS_OFFSET)
+#define OFF_FP_DEX_PC OFF_FP(SHADOWFRAME_DEX_PC_OFFSET)
+#define OFF_FP_LINK OFF_FP(SHADOWFRAME_LINK_OFFSET)
+#define OFF_FP_METHOD OFF_FP(SHADOWFRAME_METHOD_OFFSET)
+#define OFF_FP_RESULT_REGISTER OFF_FP(SHADOWFRAME_RESULT_REGISTER_OFFSET)
+#define OFF_FP_DEX_PC_PTR OFF_FP(SHADOWFRAME_DEX_PC_PTR_OFFSET)
+#define OFF_FP_CODE_ITEM OFF_FP(SHADOWFRAME_CODE_ITEM_OFFSET)
+#define OFF_FP_COUNTDOWN_OFFSET OFF_FP(SHADOWFRAME_HOTNESS_COUNTDOWN_OFFSET)
+#define OFF_FP_SHADOWFRAME (-SHADOWFRAME_VREGS_OFFSET)
+
/* Frame size must be 16-byte aligned.
* Remember about 8 bytes for return address + 6 * 8 for spills.
*/
@@ -130,6 +145,8 @@
#define IN_ARG2 %rdx
#define IN_ARG1 %rsi
#define IN_ARG0 %rdi
+/* Spill offsets relative to %esp */
+#define SELF_SPILL (FRAME_SIZE - 8)
/* Out Args */
#define OUT_ARG3 %rcx
#define OUT_ARG2 %rdx
@@ -144,7 +161,7 @@
/* During bringup, we'll use the shadow frame model instead of rFP */
/* single-purpose registers, given names for clarity */
-#define rSELF %rbp
+#define rSELF SELF_SPILL(%rsp)
#define rPC %r12
#define rFP %r13
#define rINST %ebx
@@ -154,40 +171,11 @@
#define rINSTbl %bl
#define rIBASE %r14
#define rREFS %r15
+#define rPROFILE %ebp
-/*
- * Instead of holding a pointer to the shadow frame, we keep rFP at the base of the vregs. So,
- * to access other shadow frame fields, we need to use a backwards offset. Define those here.
- */
-#define OFF_FP(a) (a - SHADOWFRAME_VREGS_OFFSET)
-#define OFF_FP_NUMBER_OF_VREGS OFF_FP(SHADOWFRAME_NUMBER_OF_VREGS_OFFSET)
-#define OFF_FP_DEX_PC OFF_FP(SHADOWFRAME_DEX_PC_OFFSET)
-#define OFF_FP_LINK OFF_FP(SHADOWFRAME_LINK_OFFSET)
-#define OFF_FP_METHOD OFF_FP(SHADOWFRAME_METHOD_OFFSET)
-#define OFF_FP_RESULT_REGISTER OFF_FP(SHADOWFRAME_RESULT_REGISTER_OFFSET)
-#define OFF_FP_DEX_PC_PTR OFF_FP(SHADOWFRAME_DEX_PC_PTR_OFFSET)
-#define OFF_FP_CODE_ITEM OFF_FP(SHADOWFRAME_CODE_ITEM_OFFSET)
-#define OFF_FP_SHADOWFRAME (-SHADOWFRAME_VREGS_OFFSET)
-
-#define MTERP_PROFILE_BRANCHES 1
#define MTERP_LOGGING 0
/*
- * Profile branch. rINST should contain the offset. %eax is scratch.
- */
-.macro MTERP_PROFILE_BRANCH
-#ifdef MTERP_PROFILE_BRANCHES
- EXPORT_PC
- movq rSELF, OUT_ARG0
- leaq OFF_FP_SHADOWFRAME(rFP), OUT_ARG1
- movl rINST, OUT_32_ARG2
- call SYMBOL(MterpProfileBranch)
- testb %al, %al
- jnz MterpOnStackReplacement
-#endif
-.endm
-
-/*
* "export" the PC to dex_pc field in the shadow frame, f/b/o future exception objects. Must
* be done *before* something throws.
*
@@ -211,7 +199,8 @@
*
*/
.macro REFRESH_IBASE
- movq THREAD_CURRENT_IBASE_OFFSET(rSELF), rIBASE
+ movq rSELF, rIBASE
+ movq THREAD_CURRENT_IBASE_OFFSET(rIBASE), rIBASE
.endm
/*
@@ -377,6 +366,12 @@
movq IN_ARG0, rSELF
REFRESH_IBASE
+ /* Set up for backwards branches & osr profiling */
+ movq OFF_FP_METHOD(rFP), OUT_ARG0
+ leaq OFF_FP_SHADOWFRAME(rFP), OUT_ARG1
+ call SYMBOL(MterpSetUpHotnessCountdown)
+ movswl %ax, rPROFILE
+
/* start executing the instruction at rPC */
FETCH_INST
GOTO_NEXT
@@ -579,9 +574,10 @@
.L_op_move_exception: /* 0x0d */
/* File: x86_64/op_move_exception.S */
/* move-exception vAA */
- movl THREAD_EXCEPTION_OFFSET(rSELF), %eax
+ movq rSELF, %rcx
+ movl THREAD_EXCEPTION_OFFSET(%rcx), %eax
SET_VREG_OBJECT %eax, rINSTq # fp[AA] <- exception object
- movl $0, THREAD_EXCEPTION_OFFSET(rSELF)
+ movl $0, THREAD_EXCEPTION_OFFSET(%rcx)
ADVANCE_PC_FETCH_AND_GOTO_NEXT 1
/* ------------------------------ */
@@ -590,9 +586,9 @@
/* File: x86_64/op_return_void.S */
.extern MterpThreadFenceForConstructor
call SYMBOL(MterpThreadFenceForConstructor)
- testl $(THREAD_SUSPEND_REQUEST | THREAD_CHECKPOINT_REQUEST), THREAD_FLAGS_OFFSET(rSELF)
- jz 1f
movq rSELF, OUT_ARG0
+ testl $(THREAD_SUSPEND_REQUEST | THREAD_CHECKPOINT_REQUEST), THREAD_FLAGS_OFFSET(OUT_ARG0)
+ jz 1f
call SYMBOL(MterpSuspendCheck)
1:
xorq %rax, %rax
@@ -610,9 +606,9 @@
/* op vAA */
.extern MterpThreadFenceForConstructor
call SYMBOL(MterpThreadFenceForConstructor)
- testl $(THREAD_SUSPEND_REQUEST | THREAD_CHECKPOINT_REQUEST), THREAD_FLAGS_OFFSET(rSELF)
- jz 1f
movq rSELF, OUT_ARG0
+ testl $(THREAD_SUSPEND_REQUEST | THREAD_CHECKPOINT_REQUEST), THREAD_FLAGS_OFFSET(OUT_ARG0)
+ jz 1f
call SYMBOL(MterpSuspendCheck)
1:
GET_VREG %eax, rINSTq # eax <- vAA
@@ -628,9 +624,9 @@
/* return-wide vAA */
.extern MterpThreadFenceForConstructor
call SYMBOL(MterpThreadFenceForConstructor)
- testl $(THREAD_SUSPEND_REQUEST | THREAD_CHECKPOINT_REQUEST), THREAD_FLAGS_OFFSET(rSELF)
- jz 1f
movq rSELF, OUT_ARG0
+ testl $(THREAD_SUSPEND_REQUEST | THREAD_CHECKPOINT_REQUEST), THREAD_FLAGS_OFFSET(OUT_ARG0)
+ jz 1f
call SYMBOL(MterpSuspendCheck)
1:
GET_WIDE_VREG %rax, rINSTq # eax <- v[AA]
@@ -649,9 +645,9 @@
/* op vAA */
.extern MterpThreadFenceForConstructor
call SYMBOL(MterpThreadFenceForConstructor)
- testl $(THREAD_SUSPEND_REQUEST | THREAD_CHECKPOINT_REQUEST), THREAD_FLAGS_OFFSET(rSELF)
- jz 1f
movq rSELF, OUT_ARG0
+ testl $(THREAD_SUSPEND_REQUEST | THREAD_CHECKPOINT_REQUEST), THREAD_FLAGS_OFFSET(OUT_ARG0)
+ jz 1f
call SYMBOL(MterpSuspendCheck)
1:
GET_VREG %eax, rINSTq # eax <- vAA
@@ -854,7 +850,8 @@
movq rSELF, OUT_ARG3
call SYMBOL(MterpInstanceOf) # (index, &obj, method, self)
movsbl %al, %eax
- cmpq $0, THREAD_EXCEPTION_OFFSET(rSELF)
+ movq rSELF, %rcx
+ cmpq $0, THREAD_EXCEPTION_OFFSET(%rcx)
jnz MterpException
andb $0xf, rINSTbl # rINSTbl <- A
SET_VREG %eax, rINSTq
@@ -988,7 +985,8 @@
GET_VREG %eax, rINSTq # eax<- vAA (exception object)
testb %al, %al
jz common_errNullObject
- movq %rax, THREAD_EXCEPTION_OFFSET(rSELF)
+ movq rSELF, %rcx
+ movq %rax, THREAD_EXCEPTION_OFFSET(%rcx)
jmp MterpException
/* ------------------------------ */
@@ -1003,12 +1001,8 @@
*/
/* goto +AA */
movsbq rINSTbl, rINSTq # rINSTq <- ssssssAA
- MTERP_PROFILE_BRANCH
- addq rINSTq, rINSTq # rINSTq <- AA * 2
- leaq (rPC, rINSTq), rPC
- FETCH_INST
- jle MterpCheckSuspendAndContinue # AA * 2 <= 0 => suspend check
- GOTO_NEXT
+ testq rINSTq, rINSTq
+ jmp MterpCommonTakenBranch
/* ------------------------------ */
.balign 128
@@ -1022,12 +1016,8 @@
*/
/* goto/16 +AAAA */
movswq 2(rPC), rINSTq # rINSTq <- ssssAAAA
- MTERP_PROFILE_BRANCH
- addq rINSTq, rINSTq # rINSTq <- AA * 2
- leaq (rPC, rINSTq), rPC
- FETCH_INST
- jle MterpCheckSuspendAndContinue # AA * 2 <= 0 => suspend check
- GOTO_NEXT
+ testq rINSTq, rINSTq
+ jmp MterpCommonTakenBranch
/* ------------------------------ */
.balign 128
@@ -1044,12 +1034,8 @@
*/
/* goto/32 +AAAAAAAA */
movslq 2(rPC), rINSTq # rINSTq <- AAAAAAAA
- MTERP_PROFILE_BRANCH
- addq rINSTq, rINSTq # rINSTq <- AA * 2
- leaq (rPC, rINSTq), rPC
- FETCH_INST
- jle MterpCheckSuspendAndContinue # AA * 2 <= 0 => suspend check
- GOTO_NEXT
+ testq rINSTq, rINSTq
+ jmp MterpCommonTakenBranch
/* ------------------------------ */
.balign 128
@@ -1069,13 +1055,9 @@
leaq (rPC,OUT_ARG0,2), OUT_ARG0 # rcx <- PC + BBBBbbbb*2
GET_VREG OUT_32_ARG1, rINSTq # eax <- vAA
call SYMBOL(MterpDoPackedSwitch)
+ testl %eax, %eax
movslq %eax, rINSTq
- MTERP_PROFILE_BRANCH
- addq rINSTq, rINSTq
- leaq (rPC, rINSTq), rPC
- FETCH_INST
- jle MterpCheckSuspendAndContinue
- GOTO_NEXT
+ jmp MterpCommonTakenBranch
/* ------------------------------ */
.balign 128
@@ -1096,13 +1078,9 @@
leaq (rPC,OUT_ARG0,2), OUT_ARG0 # rcx <- PC + BBBBbbbb*2
GET_VREG OUT_32_ARG1, rINSTq # eax <- vAA
call SYMBOL(MterpDoSparseSwitch)
+ testl %eax, %eax
movslq %eax, rINSTq
- MTERP_PROFILE_BRANCH
- addq rINSTq, rINSTq
- leaq (rPC, rINSTq), rPC
- FETCH_INST
- jle MterpCheckSuspendAndContinue
- GOTO_NEXT
+ jmp MterpCommonTakenBranch
/* ------------------------------ */
@@ -1309,16 +1287,14 @@
andb $0xf, %cl # rcx <- A
GET_VREG %eax, %rcx # eax <- vA
cmpl VREG_ADDRESS(rINSTq), %eax # compare (vA, vB)
- movl $2, rINST # assume not taken
jne 1f
movswq 2(rPC), rINSTq # Get signed branch offset
+ testq rINSTq, rINSTq
+ jmp MterpCommonTakenBranch
1:
- MTERP_PROFILE_BRANCH
- addq rINSTq, rINSTq # rax <- AA * 2
- leaq (rPC, rINSTq), rPC
- FETCH_INST
- jle MterpCheckSuspendAndContinue # AA * 2 <= 0 => suspend check
- GOTO_NEXT
+ cmpl $JIT_CHECK_OSR, rPROFILE
+ je .L_check_not_taken_osr
+ ADVANCE_PC_FETCH_AND_GOTO_NEXT 2
/* ------------------------------ */
@@ -1339,16 +1315,14 @@
andb $0xf, %cl # rcx <- A
GET_VREG %eax, %rcx # eax <- vA
cmpl VREG_ADDRESS(rINSTq), %eax # compare (vA, vB)
- movl $2, rINST # assume not taken
je 1f
movswq 2(rPC), rINSTq # Get signed branch offset
+ testq rINSTq, rINSTq
+ jmp MterpCommonTakenBranch
1:
- MTERP_PROFILE_BRANCH
- addq rINSTq, rINSTq # rax <- AA * 2
- leaq (rPC, rINSTq), rPC
- FETCH_INST
- jle MterpCheckSuspendAndContinue # AA * 2 <= 0 => suspend check
- GOTO_NEXT
+ cmpl $JIT_CHECK_OSR, rPROFILE
+ je .L_check_not_taken_osr
+ ADVANCE_PC_FETCH_AND_GOTO_NEXT 2
/* ------------------------------ */
@@ -1369,16 +1343,14 @@
andb $0xf, %cl # rcx <- A
GET_VREG %eax, %rcx # eax <- vA
cmpl VREG_ADDRESS(rINSTq), %eax # compare (vA, vB)
- movl $2, rINST # assume not taken
jge 1f
movswq 2(rPC), rINSTq # Get signed branch offset
+ testq rINSTq, rINSTq
+ jmp MterpCommonTakenBranch
1:
- MTERP_PROFILE_BRANCH
- addq rINSTq, rINSTq # rax <- AA * 2
- leaq (rPC, rINSTq), rPC
- FETCH_INST
- jle MterpCheckSuspendAndContinue # AA * 2 <= 0 => suspend check
- GOTO_NEXT
+ cmpl $JIT_CHECK_OSR, rPROFILE
+ je .L_check_not_taken_osr
+ ADVANCE_PC_FETCH_AND_GOTO_NEXT 2
/* ------------------------------ */
@@ -1399,16 +1371,14 @@
andb $0xf, %cl # rcx <- A
GET_VREG %eax, %rcx # eax <- vA
cmpl VREG_ADDRESS(rINSTq), %eax # compare (vA, vB)
- movl $2, rINST # assume not taken
jl 1f
movswq 2(rPC), rINSTq # Get signed branch offset
+ testq rINSTq, rINSTq
+ jmp MterpCommonTakenBranch
1:
- MTERP_PROFILE_BRANCH
- addq rINSTq, rINSTq # rax <- AA * 2
- leaq (rPC, rINSTq), rPC
- FETCH_INST
- jle MterpCheckSuspendAndContinue # AA * 2 <= 0 => suspend check
- GOTO_NEXT
+ cmpl $JIT_CHECK_OSR, rPROFILE
+ je .L_check_not_taken_osr
+ ADVANCE_PC_FETCH_AND_GOTO_NEXT 2
/* ------------------------------ */
@@ -1429,16 +1399,14 @@
andb $0xf, %cl # rcx <- A
GET_VREG %eax, %rcx # eax <- vA
cmpl VREG_ADDRESS(rINSTq), %eax # compare (vA, vB)
- movl $2, rINST # assume not taken
jle 1f
movswq 2(rPC), rINSTq # Get signed branch offset
+ testq rINSTq, rINSTq
+ jmp MterpCommonTakenBranch
1:
- MTERP_PROFILE_BRANCH
- addq rINSTq, rINSTq # rax <- AA * 2
- leaq (rPC, rINSTq), rPC
- FETCH_INST
- jle MterpCheckSuspendAndContinue # AA * 2 <= 0 => suspend check
- GOTO_NEXT
+ cmpl $JIT_CHECK_OSR, rPROFILE
+ je .L_check_not_taken_osr
+ ADVANCE_PC_FETCH_AND_GOTO_NEXT 2
/* ------------------------------ */
@@ -1459,16 +1427,14 @@
andb $0xf, %cl # rcx <- A
GET_VREG %eax, %rcx # eax <- vA
cmpl VREG_ADDRESS(rINSTq), %eax # compare (vA, vB)
- movl $2, rINST # assume not taken
jg 1f
movswq 2(rPC), rINSTq # Get signed branch offset
+ testq rINSTq, rINSTq
+ jmp MterpCommonTakenBranch
1:
- MTERP_PROFILE_BRANCH
- addq rINSTq, rINSTq # rax <- AA * 2
- leaq (rPC, rINSTq), rPC
- FETCH_INST
- jle MterpCheckSuspendAndContinue # AA * 2 <= 0 => suspend check
- GOTO_NEXT
+ cmpl $JIT_CHECK_OSR, rPROFILE
+ je .L_check_not_taken_osr
+ ADVANCE_PC_FETCH_AND_GOTO_NEXT 2
/* ------------------------------ */
@@ -1485,16 +1451,14 @@
*/
/* if-cmp vAA, +BBBB */
cmpl $0, VREG_ADDRESS(rINSTq) # compare (vA, 0)
- movl $2, rINST # assume branch not taken
jne 1f
movswq 2(rPC), rINSTq # fetch signed displacement
+ testq rINSTq, rINSTq
+ jmp MterpCommonTakenBranch
1:
- MTERP_PROFILE_BRANCH
- addq rINSTq, rINSTq # rINSTq <- AA * 2
- leaq (rPC, rINSTq), rPC
- FETCH_INST
- jle MterpCheckSuspendAndContinue # AA * 2 <= 0 => suspend check
- GOTO_NEXT
+ cmpl $JIT_CHECK_OSR, rPROFILE
+ je .L_check_not_taken_osr
+ ADVANCE_PC_FETCH_AND_GOTO_NEXT 2
/* ------------------------------ */
@@ -1511,16 +1475,14 @@
*/
/* if-cmp vAA, +BBBB */
cmpl $0, VREG_ADDRESS(rINSTq) # compare (vA, 0)
- movl $2, rINST # assume branch not taken
je 1f
movswq 2(rPC), rINSTq # fetch signed displacement
+ testq rINSTq, rINSTq
+ jmp MterpCommonTakenBranch
1:
- MTERP_PROFILE_BRANCH
- addq rINSTq, rINSTq # rINSTq <- AA * 2
- leaq (rPC, rINSTq), rPC
- FETCH_INST
- jle MterpCheckSuspendAndContinue # AA * 2 <= 0 => suspend check
- GOTO_NEXT
+ cmpl $JIT_CHECK_OSR, rPROFILE
+ je .L_check_not_taken_osr
+ ADVANCE_PC_FETCH_AND_GOTO_NEXT 2
/* ------------------------------ */
@@ -1537,16 +1499,14 @@
*/
/* if-cmp vAA, +BBBB */
cmpl $0, VREG_ADDRESS(rINSTq) # compare (vA, 0)
- movl $2, rINST # assume branch not taken
jge 1f
movswq 2(rPC), rINSTq # fetch signed displacement
+ testq rINSTq, rINSTq
+ jmp MterpCommonTakenBranch
1:
- MTERP_PROFILE_BRANCH
- addq rINSTq, rINSTq # rINSTq <- AA * 2
- leaq (rPC, rINSTq), rPC
- FETCH_INST
- jle MterpCheckSuspendAndContinue # AA * 2 <= 0 => suspend check
- GOTO_NEXT
+ cmpl $JIT_CHECK_OSR, rPROFILE
+ je .L_check_not_taken_osr
+ ADVANCE_PC_FETCH_AND_GOTO_NEXT 2
/* ------------------------------ */
@@ -1563,16 +1523,14 @@
*/
/* if-cmp vAA, +BBBB */
cmpl $0, VREG_ADDRESS(rINSTq) # compare (vA, 0)
- movl $2, rINST # assume branch not taken
jl 1f
movswq 2(rPC), rINSTq # fetch signed displacement
+ testq rINSTq, rINSTq
+ jmp MterpCommonTakenBranch
1:
- MTERP_PROFILE_BRANCH
- addq rINSTq, rINSTq # rINSTq <- AA * 2
- leaq (rPC, rINSTq), rPC
- FETCH_INST
- jle MterpCheckSuspendAndContinue # AA * 2 <= 0 => suspend check
- GOTO_NEXT
+ cmpl $JIT_CHECK_OSR, rPROFILE
+ je .L_check_not_taken_osr
+ ADVANCE_PC_FETCH_AND_GOTO_NEXT 2
/* ------------------------------ */
@@ -1589,16 +1547,14 @@
*/
/* if-cmp vAA, +BBBB */
cmpl $0, VREG_ADDRESS(rINSTq) # compare (vA, 0)
- movl $2, rINST # assume branch not taken
jle 1f
movswq 2(rPC), rINSTq # fetch signed displacement
+ testq rINSTq, rINSTq
+ jmp MterpCommonTakenBranch
1:
- MTERP_PROFILE_BRANCH
- addq rINSTq, rINSTq # rINSTq <- AA * 2
- leaq (rPC, rINSTq), rPC
- FETCH_INST
- jle MterpCheckSuspendAndContinue # AA * 2 <= 0 => suspend check
- GOTO_NEXT
+ cmpl $JIT_CHECK_OSR, rPROFILE
+ je .L_check_not_taken_osr
+ ADVANCE_PC_FETCH_AND_GOTO_NEXT 2
/* ------------------------------ */
@@ -1615,16 +1571,14 @@
*/
/* if-cmp vAA, +BBBB */
cmpl $0, VREG_ADDRESS(rINSTq) # compare (vA, 0)
- movl $2, rINST # assume branch not taken
jg 1f
movswq 2(rPC), rINSTq # fetch signed displacement
+ testq rINSTq, rINSTq
+ jmp MterpCommonTakenBranch
1:
- MTERP_PROFILE_BRANCH
- addq rINSTq, rINSTq # rINSTq <- AA * 2
- leaq (rPC, rINSTq), rPC
- FETCH_INST
- jle MterpCheckSuspendAndContinue # AA * 2 <= 0 => suspend check
- GOTO_NEXT
+ cmpl $JIT_CHECK_OSR, rPROFILE
+ je .L_check_not_taken_osr
+ ADVANCE_PC_FETCH_AND_GOTO_NEXT 2
/* ------------------------------ */
@@ -1767,7 +1721,8 @@
GET_VREG OUT_32_ARG1, %rcx # ecx <- vCC (requested index)
EXPORT_PC
call SYMBOL(artAGetObjectFromMterp) # (array, index)
- cmpq $0, THREAD_EXCEPTION_OFFSET(rSELF)
+ movq rSELF, %rcx
+ cmpq $0, THREAD_EXCEPTION_OFFSET(%rcx)
jnz MterpException
SET_VREG_OBJECT %eax, rINSTq
ADVANCE_PC_FETCH_AND_GOTO_NEXT 2
@@ -2099,7 +2054,8 @@
movq OFF_FP_METHOD(rFP), OUT_ARG2 # referrer
movq rSELF, OUT_ARG3
call SYMBOL(artGet32InstanceFromCode)
- cmpq $0, THREAD_EXCEPTION_OFFSET(rSELF)
+ movq rSELF, %rcx
+ cmpq $0, THREAD_EXCEPTION_OFFSET(%rcx)
jnz MterpException # bail out
andb $0xf, rINSTbl # rINST <- A
.if 0
@@ -2131,7 +2087,8 @@
movq OFF_FP_METHOD(rFP), OUT_ARG2 # referrer
movq rSELF, OUT_ARG3
call SYMBOL(artGet64InstanceFromCode)
- cmpq $0, THREAD_EXCEPTION_OFFSET(rSELF)
+ movq rSELF, %rcx
+ cmpq $0, THREAD_EXCEPTION_OFFSET(%rcx)
jnz MterpException # bail out
andb $0xf, rINSTbl # rINST <- A
.if 0
@@ -2164,7 +2121,8 @@
movq OFF_FP_METHOD(rFP), OUT_ARG2 # referrer
movq rSELF, OUT_ARG3
call SYMBOL(artGetObjInstanceFromCode)
- cmpq $0, THREAD_EXCEPTION_OFFSET(rSELF)
+ movq rSELF, %rcx
+ cmpq $0, THREAD_EXCEPTION_OFFSET(%rcx)
jnz MterpException # bail out
andb $0xf, rINSTbl # rINST <- A
.if 1
@@ -2197,7 +2155,8 @@
movq OFF_FP_METHOD(rFP), OUT_ARG2 # referrer
movq rSELF, OUT_ARG3
call SYMBOL(artGetBooleanInstanceFromCode)
- cmpq $0, THREAD_EXCEPTION_OFFSET(rSELF)
+ movq rSELF, %rcx
+ cmpq $0, THREAD_EXCEPTION_OFFSET(%rcx)
jnz MterpException # bail out
andb $0xf, rINSTbl # rINST <- A
.if 0
@@ -2230,7 +2189,8 @@
movq OFF_FP_METHOD(rFP), OUT_ARG2 # referrer
movq rSELF, OUT_ARG3
call SYMBOL(artGetByteInstanceFromCode)
- cmpq $0, THREAD_EXCEPTION_OFFSET(rSELF)
+ movq rSELF, %rcx
+ cmpq $0, THREAD_EXCEPTION_OFFSET(%rcx)
jnz MterpException # bail out
andb $0xf, rINSTbl # rINST <- A
.if 0
@@ -2263,7 +2223,8 @@
movq OFF_FP_METHOD(rFP), OUT_ARG2 # referrer
movq rSELF, OUT_ARG3
call SYMBOL(artGetCharInstanceFromCode)
- cmpq $0, THREAD_EXCEPTION_OFFSET(rSELF)
+ movq rSELF, %rcx
+ cmpq $0, THREAD_EXCEPTION_OFFSET(%rcx)
jnz MterpException # bail out
andb $0xf, rINSTbl # rINST <- A
.if 0
@@ -2296,7 +2257,8 @@
movq OFF_FP_METHOD(rFP), OUT_ARG2 # referrer
movq rSELF, OUT_ARG3
call SYMBOL(artGetShortInstanceFromCode)
- cmpq $0, THREAD_EXCEPTION_OFFSET(rSELF)
+ movq rSELF, %rcx
+ cmpq $0, THREAD_EXCEPTION_OFFSET(%rcx)
jnz MterpException # bail out
andb $0xf, rINSTbl # rINST <- A
.if 0
@@ -2489,7 +2451,8 @@
movq OFF_FP_METHOD(rFP), OUT_ARG1 # referrer
movq rSELF, OUT_ARG2 # self
call SYMBOL(artGet32StaticFromCode)
- cmpl $0, THREAD_EXCEPTION_OFFSET(rSELF)
+ movq rSELF, %rcx
+ cmpl $0, THREAD_EXCEPTION_OFFSET(%rcx)
jnz MterpException
.if 0
SET_VREG_OBJECT %eax, rINSTq # fp[A] <- value
@@ -2519,7 +2482,8 @@
movq OFF_FP_METHOD(rFP), OUT_ARG1 # referrer
movq rSELF, OUT_ARG2 # self
call SYMBOL(artGet64StaticFromCode)
- cmpl $0, THREAD_EXCEPTION_OFFSET(rSELF)
+ movq rSELF, %rcx
+ cmpl $0, THREAD_EXCEPTION_OFFSET(%rcx)
jnz MterpException
.if 0
SET_VREG_OBJECT %eax, rINSTq # fp[A] <- value
@@ -2550,7 +2514,8 @@
movq OFF_FP_METHOD(rFP), OUT_ARG1 # referrer
movq rSELF, OUT_ARG2 # self
call SYMBOL(artGetObjStaticFromCode)
- cmpl $0, THREAD_EXCEPTION_OFFSET(rSELF)
+ movq rSELF, %rcx
+ cmpl $0, THREAD_EXCEPTION_OFFSET(%rcx)
jnz MterpException
.if 1
SET_VREG_OBJECT %eax, rINSTq # fp[A] <- value
@@ -2581,7 +2546,8 @@
movq OFF_FP_METHOD(rFP), OUT_ARG1 # referrer
movq rSELF, OUT_ARG2 # self
call SYMBOL(artGetBooleanStaticFromCode)
- cmpl $0, THREAD_EXCEPTION_OFFSET(rSELF)
+ movq rSELF, %rcx
+ cmpl $0, THREAD_EXCEPTION_OFFSET(%rcx)
jnz MterpException
.if 0
SET_VREG_OBJECT %eax, rINSTq # fp[A] <- value
@@ -2612,7 +2578,8 @@
movq OFF_FP_METHOD(rFP), OUT_ARG1 # referrer
movq rSELF, OUT_ARG2 # self
call SYMBOL(artGetByteStaticFromCode)
- cmpl $0, THREAD_EXCEPTION_OFFSET(rSELF)
+ movq rSELF, %rcx
+ cmpl $0, THREAD_EXCEPTION_OFFSET(%rcx)
jnz MterpException
.if 0
SET_VREG_OBJECT %eax, rINSTq # fp[A] <- value
@@ -2643,7 +2610,8 @@
movq OFF_FP_METHOD(rFP), OUT_ARG1 # referrer
movq rSELF, OUT_ARG2 # self
call SYMBOL(artGetCharStaticFromCode)
- cmpl $0, THREAD_EXCEPTION_OFFSET(rSELF)
+ movq rSELF, %rcx
+ cmpl $0, THREAD_EXCEPTION_OFFSET(%rcx)
jnz MterpException
.if 0
SET_VREG_OBJECT %eax, rINSTq # fp[A] <- value
@@ -2674,7 +2642,8 @@
movq OFF_FP_METHOD(rFP), OUT_ARG1 # referrer
movq rSELF, OUT_ARG2 # self
call SYMBOL(artGetShortStaticFromCode)
- cmpl $0, THREAD_EXCEPTION_OFFSET(rSELF)
+ movq rSELF, %rcx
+ cmpl $0, THREAD_EXCEPTION_OFFSET(%rcx)
jnz MterpException
.if 0
SET_VREG_OBJECT %eax, rINSTq # fp[A] <- value
@@ -3002,9 +2971,9 @@
.balign 128
.L_op_return_void_no_barrier: /* 0x73 */
/* File: x86_64/op_return_void_no_barrier.S */
- testl $(THREAD_SUSPEND_REQUEST | THREAD_CHECKPOINT_REQUEST), THREAD_FLAGS_OFFSET(rSELF)
- jz 1f
movq rSELF, OUT_ARG0
+ testl $(THREAD_SUSPEND_REQUEST | THREAD_CHECKPOINT_REQUEST), THREAD_FLAGS_OFFSET(OUT_ARG0)
+ jz 1f
call SYMBOL(MterpSuspendCheck)
1:
xorq %rax, %rax
@@ -5712,7 +5681,8 @@
movzwl 2(rPC), OUT_32_ARG1 # eax <- field byte offset
EXPORT_PC
callq SYMBOL(artIGetObjectFromMterp) # (obj, offset)
- cmpq $0, THREAD_EXCEPTION_OFFSET(rSELF)
+ movq rSELF, %rcx
+ cmpq $0, THREAD_EXCEPTION_OFFSET(%rcx)
jnz MterpException # bail out
andb $0xf, rINSTbl # rINST <- A
SET_VREG_OBJECT %eax, rINSTq # fp[A] <- value
@@ -11849,7 +11819,7 @@
#if MTERP_LOGGING
movq rSELF, OUT_ARG0
leaq OFF_FP_SHADOWFRAME(rFP), OUT_ARG1
- movl THREAD_FLAGS_OFFSET(rSELF), OUT_32_ARG2
+ movl THREAD_FLAGS_OFFSET(OUT_ARG0), OUT_32_ARG2
call SYMBOL(MterpLogSuspendFallback)
#endif
jmp MterpCommonFallback
@@ -11860,7 +11830,8 @@
* interpreter.
*/
MterpPossibleException:
- cmpq $0, THREAD_EXCEPTION_OFFSET(rSELF)
+ movq rSELF, %rcx
+ cmpq $0, THREAD_EXCEPTION_OFFSET(%rcx)
jz MterpFallback
/* intentional fallthrough - handle pending exception. */
@@ -11891,19 +11862,113 @@
/* NOTE: no fallthrough */
/*
- * Check for suspend check request. Assumes rINST already loaded, rPC advanced and
- * still needs to get the opcode and branch to it, and flags are in lr.
+ * Common handling for branches with support for Jit profiling.
+ * On entry:
+ * rINST <= signed offset
+ * rPROFILE <= signed hotness countdown (expanded to 32 bits)
+ * condition bits <= set to establish sign of offset (use "NoFlags" entry if not)
+ *
+ * We have quite a few different cases for branch profiling, OSR detection and
+ * suspend check support here.
+ *
+ * Taken backward branches:
+ * If profiling active, do hotness countdown and report if we hit zero.
+ * If in osr check mode, see if our target is a compiled loop header entry and do OSR if so.
+ * Is there a pending suspend request? If so, suspend.
+ *
+ * Taken forward branches and not-taken backward branches:
+ * If in osr check mode, see if our target is a compiled loop header entry and do OSR if so.
+ *
+ * Our most common case is expected to be a taken backward branch with active jit profiling,
+ * but no full OSR check and no pending suspend request.
+ * Next most common case is not-taken branch with no full OSR check.
+ *
*/
-MterpCheckSuspendAndContinue:
+MterpCommonTakenBranch:
+ jg .L_forward_branch # don't add forward branches to hotness
+/*
+ * We need to subtract 1 from positive values and we should not see 0 here,
+ * so we may use the result of the comparison with -1.
+ */
+#if JIT_CHECK_OSR != -1
+# error "JIT_CHECK_OSR must be -1."
+#endif
+ cmpl $JIT_CHECK_OSR, rPROFILE
+ je .L_osr_check
+ decl rPROFILE
+ je .L_add_batch # counted down to zero - report
+.L_resume_backward_branch:
+ movq rSELF, %rax
+ testl $(THREAD_SUSPEND_REQUEST | THREAD_CHECKPOINT_REQUEST), THREAD_FLAGS_OFFSET(%rax)
REFRESH_IBASE
- testl $(THREAD_SUSPEND_REQUEST | THREAD_CHECKPOINT_REQUEST), THREAD_FLAGS_OFFSET(rSELF)
- jz 1f
+ leaq (rPC, rINSTq, 2), rPC
+ FETCH_INST
+ jnz .L_suspend_request_pending
+ GOTO_NEXT
+
+.L_suspend_request_pending:
EXPORT_PC
movq rSELF, OUT_ARG0
- call SYMBOL(MterpSuspendCheck)
-1:
+ call SYMBOL(MterpSuspendCheck) # (self)
+ testb %al, %al
+ jnz MterpFallback
+ REFRESH_IBASE # might have changed during suspend
GOTO_NEXT
+.L_no_count_backwards:
+ cmpl $JIT_CHECK_OSR, rPROFILE # possible OSR re-entry?
+ jne .L_resume_backward_branch
+.L_osr_check:
+ EXPORT_PC
+ movq rSELF, OUT_ARG0
+ leaq OFF_FP_SHADOWFRAME(rFP), OUT_ARG1
+ movq rINSTq, OUT_ARG2
+ call SYMBOL(MterpMaybeDoOnStackReplacement) # (self, shadow_frame, offset)
+ testb %al, %al
+ jz .L_resume_backward_branch
+ jmp MterpOnStackReplacement
+
+.L_forward_branch:
+ cmpl $JIT_CHECK_OSR, rPROFILE # possible OSR re-entry?
+ je .L_check_osr_forward
+.L_resume_forward_branch:
+ leaq (rPC, rINSTq, 2), rPC
+ FETCH_INST
+ GOTO_NEXT
+
+.L_check_osr_forward:
+ EXPORT_PC
+ movq rSELF, OUT_ARG0
+ leaq OFF_FP_SHADOWFRAME(rFP), OUT_ARG1
+ movq rINSTq, OUT_ARG2
+ call SYMBOL(MterpMaybeDoOnStackReplacement) # (self, shadow_frame, offset)
+ testb %al, %al
+ jz .L_resume_forward_branch
+ jmp MterpOnStackReplacement
+
+.L_add_batch:
+ movl rPROFILE, %eax
+ movq OFF_FP_METHOD(rFP), OUT_ARG0
+ leaq OFF_FP_SHADOWFRAME(rFP), OUT_ARG1
+ movw %ax, OFF_FP_COUNTDOWN_OFFSET(rFP)
+ movq rSELF, OUT_ARG2
+ call SYMBOL(MterpAddHotnessBatch) # (method, shadow_frame, self)
+ movswl %ax, rPROFILE
+ jmp .L_no_count_backwards
+
+/*
+ * Entered from the conditional branch handlers when OSR check request active on
+ * not-taken path. All Dalvik not-taken conditional branch offsets are 2.
+ */
+.L_check_not_taken_osr:
+ movq rSELF, OUT_ARG0
+ leaq OFF_FP_SHADOWFRAME(rFP), OUT_ARG1
+ movq rINSTq, OUT_ARG3
+ call SYMBOL(MterpMaybeDoOnStackReplacement) # (self, shadow_frame, offset)
+ testb %al, %al
+ jnz MterpOnStackReplacement
+ ADVANCE_PC_FETCH_AND_GOTO_NEXT 2
+
/*
* On-stack replacement has happened, and now we've returned from the compiled method.
*/
@@ -11943,7 +12008,28 @@
movq %rax, (%rdx)
movl $1, %eax
MterpDone:
+/*
+ * At this point, we expect rPROFILE to be non-zero. If negative, hotness is disabled or we're
+ * checking for OSR. If greater than zero, we might have unreported hotness to register
+ * (the difference between the ending rPROFILE and the cached hotness counter). rPROFILE
+ * should only reach zero immediately after a hotness decrement, and is then reset to either
+ * a negative special state or the new non-zero countdown value.
+ */
+ testl rPROFILE, rPROFILE
+ jle MRestoreFrame # if > 0, we may have some counts to report.
+
+ movl %eax, rINST # stash return value
+ /* Report cached hotness counts */
+ movl rPROFILE, %eax
+ movq OFF_FP_METHOD(rFP), OUT_ARG0
+ leaq OFF_FP_SHADOWFRAME(rFP), OUT_ARG1
+ movw %ax, OFF_FP_COUNTDOWN_OFFSET(rFP)
+ movq rSELF, OUT_ARG2
+ call SYMBOL(MterpAddHotnessBatch) # (method, shadow_frame, self)
+ movl rINST, %eax # restore return value
+
/* pop up frame */
+MRestoreFrame:
addq $FRAME_SIZE, %rsp
.cfi_adjust_cfa_offset -FRAME_SIZE
diff --git a/runtime/interpreter/mterp/x86/bincmp.S b/runtime/interpreter/mterp/x86/bincmp.S
index c72a5cf..ee32278 100644
--- a/runtime/interpreter/mterp/x86/bincmp.S
+++ b/runtime/interpreter/mterp/x86/bincmp.S
@@ -11,13 +11,11 @@
GET_VREG %eax, %ecx # eax <- vA
sarl $$4, rINST # rINST <- B
cmpl VREG_ADDRESS(rINST), %eax # compare (vA, vB)
- movl $$2, rINST
j${revcmp} 1f
movswl 2(rPC), rINST # Get signed branch offset
+ testl rINST, rINST
+ jmp MterpCommonTakenBranch
1:
- MTERP_PROFILE_BRANCH
- addl rINST, rINST # eax <- AA * 2
- leal (rPC, rINST), rPC
- FETCH_INST
- jle MterpCheckSuspendAndContinue # AA * 2 <= 0 => suspend check
- GOTO_NEXT
+ cmpw $$JIT_CHECK_OSR, rPROFILE
+ je .L_check_not_taken_osr
+ ADVANCE_PC_FETCH_AND_GOTO_NEXT 2
diff --git a/runtime/interpreter/mterp/x86/entry.S b/runtime/interpreter/mterp/x86/entry.S
index 785efdc..384dd9a 100644
--- a/runtime/interpreter/mterp/x86/entry.S
+++ b/runtime/interpreter/mterp/x86/entry.S
@@ -64,6 +64,13 @@
lea (rPC, %eax, 2), rPC
EXPORT_PC
+ /* Set up for backwards branches & osr profiling */
+ movl OFF_FP_METHOD(rFP), %eax
+ movl %eax, OUT_ARG0(%esp)
+ leal OFF_FP_SHADOWFRAME(rFP), %ecx
+ movl %ecx, OUT_ARG1(%esp)
+ call SYMBOL(MterpSetUpHotnessCountdown)
+
/* Starting ibase */
REFRESH_IBASE
diff --git a/runtime/interpreter/mterp/x86/footer.S b/runtime/interpreter/mterp/x86/footer.S
index 3965ecd..df10ff0 100644
--- a/runtime/interpreter/mterp/x86/footer.S
+++ b/runtime/interpreter/mterp/x86/footer.S
@@ -131,20 +131,120 @@
/* NOTE: no fallthrough */
/*
- * Check for suspend check request. Assumes rINST already loaded, rPC advanced and
- * still needs to get the opcode and branch to it, and flags are in lr.
+ * Common handling for branches with support for Jit profiling.
+ * On entry:
+ * rINST <= signed offset
+ * condition bits <= set to establish sign of offset (use "NoFlags" entry if not)
+ *
+ * We have quite a few different cases for branch profiling, OSR detection and
+ * suspend check support here.
+ *
+ * Taken backward branches:
+ * If profiling active, do hotness countdown and report if we hit zero.
+ * If in osr check mode, see if our target is a compiled loop header entry and do OSR if so.
+ * Is there a pending suspend request? If so, suspend.
+ *
+ * Taken forward branches and not-taken backward branches:
+ * If in osr check mode, see if our target is a compiled loop header entry and do OSR if so.
+ *
+ * Our most common case is expected to be a taken backward branch with active jit profiling,
+ * but no full OSR check and no pending suspend request.
+ * Next most common case is not-taken branch with no full OSR check.
+ *
*/
-MterpCheckSuspendAndContinue:
+MterpCommonTakenBranch:
+ jg .L_forward_branch # don't add forward branches to hotness
+/*
+ * We need to subtract 1 from positive values and we should not see 0 here,
+ * so we may use the result of the comparison with -1.
+ */
+#if JIT_CHECK_OSR != -1
+# error "JIT_CHECK_OSR must be -1."
+#endif
+ cmpw $$JIT_CHECK_OSR, rPROFILE
+ je .L_osr_check
+ decw rPROFILE
+ je .L_add_batch # counted down to zero - report
+.L_resume_backward_branch:
movl rSELF, %eax
- EXPORT_PC
testl $$(THREAD_SUSPEND_REQUEST | THREAD_CHECKPOINT_REQUEST), THREAD_FLAGS_OFFSET(%eax)
- jz 1f
- movl %eax, OUT_ARG0(%esp)
- call SYMBOL(MterpSuspendCheck)
+ leal (rPC, rINST, 2), rPC
+ FETCH_INST
+ jnz .L_suspend_request_pending
REFRESH_IBASE
-1:
GOTO_NEXT
+.L_suspend_request_pending:
+ EXPORT_PC
+ movl %eax, OUT_ARG0(%esp) # rSELF in eax
+ call SYMBOL(MterpSuspendCheck) # (self)
+ testb %al, %al
+ jnz MterpFallback
+ REFRESH_IBASE # might have changed during suspend
+ GOTO_NEXT
+
+.L_no_count_backwards:
+ cmpw $$JIT_CHECK_OSR, rPROFILE # possible OSR re-entry?
+ jne .L_resume_backward_branch
+.L_osr_check:
+ EXPORT_PC
+ movl rSELF, %eax
+ movl %eax, OUT_ARG0(%esp)
+ leal OFF_FP_SHADOWFRAME(rFP), %ecx
+ movl %ecx, OUT_ARG1(%esp)
+ movl rINST, OUT_ARG2(%esp)
+ call SYMBOL(MterpMaybeDoOnStackReplacement) # (self, shadow_frame, offset)
+ testb %al, %al
+ jz .L_resume_backward_branch
+ jmp MterpOnStackReplacement
+
+.L_forward_branch:
+ cmpw $$JIT_CHECK_OSR, rPROFILE # possible OSR re-entry?
+ je .L_check_osr_forward
+.L_resume_forward_branch:
+ leal (rPC, rINST, 2), rPC
+ FETCH_INST
+ GOTO_NEXT
+
+.L_check_osr_forward:
+ EXPORT_PC
+ movl rSELF, %eax
+ movl %eax, OUT_ARG0(%esp)
+ leal OFF_FP_SHADOWFRAME(rFP), %ecx
+ movl %ecx, OUT_ARG1(%esp)
+ movl rINST, OUT_ARG2(%esp)
+ call SYMBOL(MterpMaybeDoOnStackReplacement) # (self, shadow_frame, offset)
+ testb %al, %al
+ REFRESH_IBASE
+ jz .L_resume_forward_branch
+ jmp MterpOnStackReplacement
+
+.L_add_batch:
+ movl OFF_FP_METHOD(rFP), %eax
+ movl %eax, OUT_ARG0(%esp)
+ leal OFF_FP_SHADOWFRAME(rFP), %ecx
+ movl %ecx, OUT_ARG1(%esp)
+ movl rSELF, %eax
+ movl %eax, OUT_ARG2(%esp)
+ call SYMBOL(MterpAddHotnessBatch) # (method, shadow_frame, self)
+ jmp .L_no_count_backwards
+
+/*
+ * Entered from the conditional branch handlers when OSR check request active on
+ * not-taken path. All Dalvik not-taken conditional branch offsets are 2.
+ */
+.L_check_not_taken_osr:
+ movl rSELF, %eax
+ movl %eax, OUT_ARG0(%esp)
+ leal OFF_FP_SHADOWFRAME(rFP), %ecx
+ movl %ecx, OUT_ARG1(%esp)
+ movl rINST, OUT_ARG3(%esp)
+ call SYMBOL(MterpMaybeDoOnStackReplacement) # (self, shadow_frame, offset)
+ testb %al, %al
+ REFRESH_IBASE
+ jnz MterpOnStackReplacement
+ ADVANCE_PC_FETCH_AND_GOTO_NEXT 2
+
/*
* On-stack replacement has happened, and now we've returned from the compiled method.
*/
@@ -189,7 +289,29 @@
movl %ecx, 4(%edx)
mov $$1, %eax
MterpDone:
+/*
+ * At this point, we expect rPROFILE to be non-zero. If negative, hotness is disabled or we're
+ * checking for OSR. If greater than zero, we might have unreported hotness to register
+ * (the difference between the ending rPROFILE and the cached hotness counter). rPROFILE
+ * should only reach zero immediately after a hotness decrement, and is then reset to either
+ * a negative special state or the new non-zero countdown value.
+ */
+ cmpw $$0, rPROFILE
+ jle MRestoreFrame # if > 0, we may have some counts to report.
+
+ movl %eax, rINST # stash return value
+ /* Report cached hotness counts */
+ movl OFF_FP_METHOD(rFP), %eax
+ movl %eax, OUT_ARG0(%esp)
+ leal OFF_FP_SHADOWFRAME(rFP), %ecx
+ movl %ecx, OUT_ARG1(%esp)
+ movl rSELF, %eax
+ movl %eax, OUT_ARG2(%esp)
+ call SYMBOL(MterpAddHotnessBatch) # (method, shadow_frame, self)
+ movl rINST, %eax # restore return value
+
/* pop up frame */
+MRestoreFrame:
addl $$FRAME_SIZE, %esp
.cfi_adjust_cfa_offset -FRAME_SIZE
diff --git a/runtime/interpreter/mterp/x86/header.S b/runtime/interpreter/mterp/x86/header.S
index 5729b90..3a2dcb7 100644
--- a/runtime/interpreter/mterp/x86/header.S
+++ b/runtime/interpreter/mterp/x86/header.S
@@ -117,6 +117,21 @@
.cfi_restore \_reg
.endm
+/*
+ * Instead of holding a pointer to the shadow frame, we keep rFP at the base of the vregs. So,
+ * to access other shadow frame fields, we need to use a backwards offset. Define those here.
+ */
+#define OFF_FP(a) (a - SHADOWFRAME_VREGS_OFFSET)
+#define OFF_FP_NUMBER_OF_VREGS OFF_FP(SHADOWFRAME_NUMBER_OF_VREGS_OFFSET)
+#define OFF_FP_DEX_PC OFF_FP(SHADOWFRAME_DEX_PC_OFFSET)
+#define OFF_FP_LINK OFF_FP(SHADOWFRAME_LINK_OFFSET)
+#define OFF_FP_METHOD OFF_FP(SHADOWFRAME_METHOD_OFFSET)
+#define OFF_FP_RESULT_REGISTER OFF_FP(SHADOWFRAME_RESULT_REGISTER_OFFSET)
+#define OFF_FP_DEX_PC_PTR OFF_FP(SHADOWFRAME_DEX_PC_PTR_OFFSET)
+#define OFF_FP_CODE_ITEM OFF_FP(SHADOWFRAME_CODE_ITEM_OFFSET)
+#define OFF_FP_COUNTDOWN_OFFSET OFF_FP(SHADOWFRAME_HOTNESS_COUNTDOWN_OFFSET)
+#define OFF_FP_SHADOWFRAME OFF_FP(0)
+
/* Frame size must be 16-byte aligned.
* Remember about 4 bytes for return address + 4 * 4 for spills
*/
@@ -148,43 +163,11 @@
#define rINSTbl %bl
#define rIBASE %edx
#define rREFS %ebp
+#define rPROFILE OFF_FP_COUNTDOWN_OFFSET(rFP)
-/*
- * Instead of holding a pointer to the shadow frame, we keep rFP at the base of the vregs. So,
- * to access other shadow frame fields, we need to use a backwards offset. Define those here.
- */
-#define OFF_FP(a) (a - SHADOWFRAME_VREGS_OFFSET)
-#define OFF_FP_NUMBER_OF_VREGS OFF_FP(SHADOWFRAME_NUMBER_OF_VREGS_OFFSET)
-#define OFF_FP_DEX_PC OFF_FP(SHADOWFRAME_DEX_PC_OFFSET)
-#define OFF_FP_LINK OFF_FP(SHADOWFRAME_LINK_OFFSET)
-#define OFF_FP_METHOD OFF_FP(SHADOWFRAME_METHOD_OFFSET)
-#define OFF_FP_RESULT_REGISTER OFF_FP(SHADOWFRAME_RESULT_REGISTER_OFFSET)
-#define OFF_FP_DEX_PC_PTR OFF_FP(SHADOWFRAME_DEX_PC_PTR_OFFSET)
-#define OFF_FP_CODE_ITEM OFF_FP(SHADOWFRAME_CODE_ITEM_OFFSET)
-#define OFF_FP_SHADOWFRAME (-SHADOWFRAME_VREGS_OFFSET)
-
-#define MTERP_PROFILE_BRANCHES 1
#define MTERP_LOGGING 0
/*
- * Profile branch. rINST should contain the offset. %eax is scratch.
- */
-.macro MTERP_PROFILE_BRANCH
-#ifdef MTERP_PROFILE_BRANCHES
- EXPORT_PC
- movl rSELF, %eax
- movl %eax, OUT_ARG0(%esp)
- leal OFF_FP_SHADOWFRAME(rFP), %eax
- movl %eax, OUT_ARG1(%esp)
- movl rINST, OUT_ARG2(%esp)
- call SYMBOL(MterpProfileBranch)
- testb %al, %al
- jnz MterpOnStackReplacement
- RESTORE_IBASE
-#endif
-.endm
-
-/*
* "export" the PC to dex_pc field in the shadow frame, f/b/o future exception objects. Must
* be done *before* something throws.
*
diff --git a/runtime/interpreter/mterp/x86/op_goto.S b/runtime/interpreter/mterp/x86/op_goto.S
index 9a87361..1827d68 100644
--- a/runtime/interpreter/mterp/x86/op_goto.S
+++ b/runtime/interpreter/mterp/x86/op_goto.S
@@ -6,9 +6,5 @@
*/
/* goto +AA */
movsbl rINSTbl, rINST # rINST <- ssssssAA
- MTERP_PROFILE_BRANCH
- addl rINST, rINST # rINST <- AA * 2
- leal (rPC, rINST), rPC
- FETCH_INST
- jle MterpCheckSuspendAndContinue # AA * 2 <= 0 => suspend check
- GOTO_NEXT
+ testl rINST, rINST
+ jmp MterpCommonTakenBranch
diff --git a/runtime/interpreter/mterp/x86/op_goto_16.S b/runtime/interpreter/mterp/x86/op_goto_16.S
index a25c31b..ea5ea90 100644
--- a/runtime/interpreter/mterp/x86/op_goto_16.S
+++ b/runtime/interpreter/mterp/x86/op_goto_16.S
@@ -6,9 +6,5 @@
*/
/* goto/16 +AAAA */
movswl 2(rPC), rINST # rINST <- ssssAAAA
- MTERP_PROFILE_BRANCH
- addl rINST, rINST # rINST <- AA * 2
- leal (rPC, rINST), rPC
- FETCH_INST
- jle MterpCheckSuspendAndContinue # AA * 2 <= 0 => suspend check
- GOTO_NEXT
+ testl rINST, rINST
+ jmp MterpCommonTakenBranch
diff --git a/runtime/interpreter/mterp/x86/op_goto_32.S b/runtime/interpreter/mterp/x86/op_goto_32.S
index 159128b..4becaf3 100644
--- a/runtime/interpreter/mterp/x86/op_goto_32.S
+++ b/runtime/interpreter/mterp/x86/op_goto_32.S
@@ -11,9 +11,5 @@
*/
/* goto/32 +AAAAAAAA */
movl 2(rPC), rINST # rINST <- AAAAAAAA
- MTERP_PROFILE_BRANCH
- addl rINST, rINST # rINST <- AA * 2
- leal (rPC, rINST), rPC
- FETCH_INST
- jle MterpCheckSuspendAndContinue # AA * 2 <= 0 => suspend check
- GOTO_NEXT
+ testl rINST, rINST
+ jmp MterpCommonTakenBranch
diff --git a/runtime/interpreter/mterp/x86/op_packed_switch.S b/runtime/interpreter/mterp/x86/op_packed_switch.S
index e33cf75..fcb7509 100644
--- a/runtime/interpreter/mterp/x86/op_packed_switch.S
+++ b/runtime/interpreter/mterp/x86/op_packed_switch.S
@@ -15,11 +15,7 @@
movl %eax, OUT_ARG1(%esp) # ARG1 <- vAA
movl %ecx, OUT_ARG0(%esp) # ARG0 <- switchData
call SYMBOL($func)
- movl %eax, rINST
- MTERP_PROFILE_BRANCH
- addl rINST, rINST
- leal (rPC, rINST), rPC
- FETCH_INST
REFRESH_IBASE
- jle MterpCheckSuspendAndContinue
- GOTO_NEXT
+ testl %eax, %eax
+ movl %eax, rINST
+ jmp MterpCommonTakenBranch
diff --git a/runtime/interpreter/mterp/x86/zcmp.S b/runtime/interpreter/mterp/x86/zcmp.S
index 0f28d1a..c116159 100644
--- a/runtime/interpreter/mterp/x86/zcmp.S
+++ b/runtime/interpreter/mterp/x86/zcmp.S
@@ -7,13 +7,11 @@
*/
/* if-cmp vAA, +BBBB */
cmpl $$0, VREG_ADDRESS(rINST) # compare (vA, 0)
- movl $$2, rINST
j${revcmp} 1f
movswl 2(rPC), rINST # fetch signed displacement
+ testl rINST, rINST
+ jmp MterpCommonTakenBranch
1:
- MTERP_PROFILE_BRANCH
- addl rINST, rINST # eax <- AA * 2
- leal (rPC, rINST), rPC
- FETCH_INST
- jle MterpCheckSuspendAndContinue # AA * 2 <= 0 => suspend check
- GOTO_NEXT
+ cmpw $$JIT_CHECK_OSR, rPROFILE
+ je .L_check_not_taken_osr
+ ADVANCE_PC_FETCH_AND_GOTO_NEXT 2
diff --git a/runtime/interpreter/mterp/x86_64/bincmp.S b/runtime/interpreter/mterp/x86_64/bincmp.S
index a16050b..6601483 100644
--- a/runtime/interpreter/mterp/x86_64/bincmp.S
+++ b/runtime/interpreter/mterp/x86_64/bincmp.S
@@ -11,13 +11,11 @@
andb $$0xf, %cl # rcx <- A
GET_VREG %eax, %rcx # eax <- vA
cmpl VREG_ADDRESS(rINSTq), %eax # compare (vA, vB)
- movl $$2, rINST # assume not taken
j${revcmp} 1f
movswq 2(rPC), rINSTq # Get signed branch offset
+ testq rINSTq, rINSTq
+ jmp MterpCommonTakenBranch
1:
- MTERP_PROFILE_BRANCH
- addq rINSTq, rINSTq # rax <- AA * 2
- leaq (rPC, rINSTq), rPC
- FETCH_INST
- jle MterpCheckSuspendAndContinue # AA * 2 <= 0 => suspend check
- GOTO_NEXT
+ cmpl $$JIT_CHECK_OSR, rPROFILE
+ je .L_check_not_taken_osr
+ ADVANCE_PC_FETCH_AND_GOTO_NEXT 2
diff --git a/runtime/interpreter/mterp/x86_64/entry.S b/runtime/interpreter/mterp/x86_64/entry.S
index 69b2371..d992956 100644
--- a/runtime/interpreter/mterp/x86_64/entry.S
+++ b/runtime/interpreter/mterp/x86_64/entry.S
@@ -65,6 +65,12 @@
movq IN_ARG0, rSELF
REFRESH_IBASE
+ /* Set up for backwards branches & osr profiling */
+ movq OFF_FP_METHOD(rFP), OUT_ARG0
+ leaq OFF_FP_SHADOWFRAME(rFP), OUT_ARG1
+ call SYMBOL(MterpSetUpHotnessCountdown)
+ movswl %ax, rPROFILE
+
/* start executing the instruction at rPC */
FETCH_INST
GOTO_NEXT
diff --git a/runtime/interpreter/mterp/x86_64/footer.S b/runtime/interpreter/mterp/x86_64/footer.S
index 573256b..71130d1 100644
--- a/runtime/interpreter/mterp/x86_64/footer.S
+++ b/runtime/interpreter/mterp/x86_64/footer.S
@@ -71,7 +71,7 @@
#if MTERP_LOGGING
movq rSELF, OUT_ARG0
leaq OFF_FP_SHADOWFRAME(rFP), OUT_ARG1
- movl THREAD_FLAGS_OFFSET(rSELF), OUT_32_ARG2
+ movl THREAD_FLAGS_OFFSET(OUT_ARG0), OUT_32_ARG2
call SYMBOL(MterpLogSuspendFallback)
#endif
jmp MterpCommonFallback
@@ -82,7 +82,8 @@
* interpreter.
*/
MterpPossibleException:
- cmpq $$0, THREAD_EXCEPTION_OFFSET(rSELF)
+ movq rSELF, %rcx
+ cmpq $$0, THREAD_EXCEPTION_OFFSET(%rcx)
jz MterpFallback
/* intentional fallthrough - handle pending exception. */
@@ -113,19 +114,113 @@
/* NOTE: no fallthrough */
/*
- * Check for suspend check request. Assumes rINST already loaded, rPC advanced and
- * still needs to get the opcode and branch to it, and flags are in lr.
+ * Common handling for branches with support for Jit profiling.
+ * On entry:
+ * rINST <= signed offset
+ * rPROFILE <= signed hotness countdown (expanded to 32 bits)
+ * condition bits <= set to establish sign of offset (use "NoFlags" entry if not)
+ *
+ * We have quite a few different cases for branch profiling, OSR detection and
+ * suspend check support here.
+ *
+ * Taken backward branches:
+ * If profiling active, do hotness countdown and report if we hit zero.
+ * If in osr check mode, see if our target is a compiled loop header entry and do OSR if so.
+ * Is there a pending suspend request? If so, suspend.
+ *
+ * Taken forward branches and not-taken backward branches:
+ * If in osr check mode, see if our target is a compiled loop header entry and do OSR if so.
+ *
+ * Our most common case is expected to be a taken backward branch with active jit profiling,
+ * but no full OSR check and no pending suspend request.
+ * Next most common case is not-taken branch with no full OSR check.
+ *
*/
-MterpCheckSuspendAndContinue:
+MterpCommonTakenBranch:
+ jg .L_forward_branch # don't add forward branches to hotness
+/*
+ * We need to subtract 1 from positive values and we should not see 0 here,
+ * so we may use the result of the comparison with -1.
+ */
+#if JIT_CHECK_OSR != -1
+# error "JIT_CHECK_OSR must be -1."
+#endif
+ cmpl $$JIT_CHECK_OSR, rPROFILE
+ je .L_osr_check
+ decl rPROFILE
+ je .L_add_batch # counted down to zero - report
+.L_resume_backward_branch:
+ movq rSELF, %rax
+ testl $$(THREAD_SUSPEND_REQUEST | THREAD_CHECKPOINT_REQUEST), THREAD_FLAGS_OFFSET(%rax)
REFRESH_IBASE
- testl $$(THREAD_SUSPEND_REQUEST | THREAD_CHECKPOINT_REQUEST), THREAD_FLAGS_OFFSET(rSELF)
- jz 1f
+ leaq (rPC, rINSTq, 2), rPC
+ FETCH_INST
+ jnz .L_suspend_request_pending
+ GOTO_NEXT
+
+.L_suspend_request_pending:
EXPORT_PC
movq rSELF, OUT_ARG0
- call SYMBOL(MterpSuspendCheck)
-1:
+ call SYMBOL(MterpSuspendCheck) # (self)
+ testb %al, %al
+ jnz MterpFallback
+ REFRESH_IBASE # might have changed during suspend
GOTO_NEXT
+.L_no_count_backwards:
+ cmpl $$JIT_CHECK_OSR, rPROFILE # possible OSR re-entry?
+ jne .L_resume_backward_branch
+.L_osr_check:
+ EXPORT_PC
+ movq rSELF, OUT_ARG0
+ leaq OFF_FP_SHADOWFRAME(rFP), OUT_ARG1
+ movq rINSTq, OUT_ARG2
+ call SYMBOL(MterpMaybeDoOnStackReplacement) # (self, shadow_frame, offset)
+ testb %al, %al
+ jz .L_resume_backward_branch
+ jmp MterpOnStackReplacement
+
+.L_forward_branch:
+ cmpl $$JIT_CHECK_OSR, rPROFILE # possible OSR re-entry?
+ je .L_check_osr_forward
+.L_resume_forward_branch:
+ leaq (rPC, rINSTq, 2), rPC
+ FETCH_INST
+ GOTO_NEXT
+
+.L_check_osr_forward:
+ EXPORT_PC
+ movq rSELF, OUT_ARG0
+ leaq OFF_FP_SHADOWFRAME(rFP), OUT_ARG1
+ movq rINSTq, OUT_ARG2
+ call SYMBOL(MterpMaybeDoOnStackReplacement) # (self, shadow_frame, offset)
+ testb %al, %al
+ jz .L_resume_forward_branch
+ jmp MterpOnStackReplacement
+
+.L_add_batch:
+ movl rPROFILE, %eax
+ movq OFF_FP_METHOD(rFP), OUT_ARG0
+ leaq OFF_FP_SHADOWFRAME(rFP), OUT_ARG1
+ movw %ax, OFF_FP_COUNTDOWN_OFFSET(rFP)
+ movq rSELF, OUT_ARG2
+ call SYMBOL(MterpAddHotnessBatch) # (method, shadow_frame, self)
+ movswl %ax, rPROFILE
+ jmp .L_no_count_backwards
+
+/*
+ * Entered from the conditional branch handlers when OSR check request active on
+ * not-taken path. All Dalvik not-taken conditional branch offsets are 2.
+ */
+.L_check_not_taken_osr:
+ movq rSELF, OUT_ARG0
+ leaq OFF_FP_SHADOWFRAME(rFP), OUT_ARG1
+ movq rINSTq, OUT_ARG3
+ call SYMBOL(MterpMaybeDoOnStackReplacement) # (self, shadow_frame, offset)
+ testb %al, %al
+ jnz MterpOnStackReplacement
+ ADVANCE_PC_FETCH_AND_GOTO_NEXT 2
+
/*
* On-stack replacement has happened, and now we've returned from the compiled method.
*/
@@ -165,7 +260,28 @@
movq %rax, (%rdx)
movl $$1, %eax
MterpDone:
+/*
+ * At this point, we expect rPROFILE to be non-zero. If negative, hotness is disabled or we're
+ * checking for OSR. If greater than zero, we might have unreported hotness to register
+ * (the difference between the ending rPROFILE and the cached hotness counter). rPROFILE
+ * should only reach zero immediately after a hotness decrement, and is then reset to either
+ * a negative special state or the new non-zero countdown value.
+ */
+ testl rPROFILE, rPROFILE
+ jle MRestoreFrame # if > 0, we may have some counts to report.
+
+ movl %eax, rINST # stash return value
+ /* Report cached hotness counts */
+ movl rPROFILE, %eax
+ movq OFF_FP_METHOD(rFP), OUT_ARG0
+ leaq OFF_FP_SHADOWFRAME(rFP), OUT_ARG1
+ movw %ax, OFF_FP_COUNTDOWN_OFFSET(rFP)
+ movq rSELF, OUT_ARG2
+ call SYMBOL(MterpAddHotnessBatch) # (method, shadow_frame, self)
+ movl rINST, %eax # restore return value
+
/* pop up frame */
+MRestoreFrame:
addq $$FRAME_SIZE, %rsp
.cfi_adjust_cfa_offset -FRAME_SIZE
diff --git a/runtime/interpreter/mterp/x86_64/header.S b/runtime/interpreter/mterp/x86_64/header.S
index eb84ea1..47d30ec 100644
--- a/runtime/interpreter/mterp/x86_64/header.S
+++ b/runtime/interpreter/mterp/x86_64/header.S
@@ -113,6 +113,21 @@
.cfi_restore \_reg
.endm
+/*
+ * Instead of holding a pointer to the shadow frame, we keep rFP at the base of the vregs. So,
+ * to access other shadow frame fields, we need to use a backwards offset. Define those here.
+ */
+#define OFF_FP(a) (a - SHADOWFRAME_VREGS_OFFSET)
+#define OFF_FP_NUMBER_OF_VREGS OFF_FP(SHADOWFRAME_NUMBER_OF_VREGS_OFFSET)
+#define OFF_FP_DEX_PC OFF_FP(SHADOWFRAME_DEX_PC_OFFSET)
+#define OFF_FP_LINK OFF_FP(SHADOWFRAME_LINK_OFFSET)
+#define OFF_FP_METHOD OFF_FP(SHADOWFRAME_METHOD_OFFSET)
+#define OFF_FP_RESULT_REGISTER OFF_FP(SHADOWFRAME_RESULT_REGISTER_OFFSET)
+#define OFF_FP_DEX_PC_PTR OFF_FP(SHADOWFRAME_DEX_PC_PTR_OFFSET)
+#define OFF_FP_CODE_ITEM OFF_FP(SHADOWFRAME_CODE_ITEM_OFFSET)
+#define OFF_FP_COUNTDOWN_OFFSET OFF_FP(SHADOWFRAME_HOTNESS_COUNTDOWN_OFFSET)
+#define OFF_FP_SHADOWFRAME (-SHADOWFRAME_VREGS_OFFSET)
+
/* Frame size must be 16-byte aligned.
* Remember about 8 bytes for return address + 6 * 8 for spills.
*/
@@ -123,6 +138,8 @@
#define IN_ARG2 %rdx
#define IN_ARG1 %rsi
#define IN_ARG0 %rdi
+/* Spill offsets relative to %esp */
+#define SELF_SPILL (FRAME_SIZE - 8)
/* Out Args */
#define OUT_ARG3 %rcx
#define OUT_ARG2 %rdx
@@ -137,7 +154,7 @@
/* During bringup, we'll use the shadow frame model instead of rFP */
/* single-purpose registers, given names for clarity */
-#define rSELF %rbp
+#define rSELF SELF_SPILL(%rsp)
#define rPC %r12
#define rFP %r13
#define rINST %ebx
@@ -147,40 +164,11 @@
#define rINSTbl %bl
#define rIBASE %r14
#define rREFS %r15
+#define rPROFILE %ebp
-/*
- * Instead of holding a pointer to the shadow frame, we keep rFP at the base of the vregs. So,
- * to access other shadow frame fields, we need to use a backwards offset. Define those here.
- */
-#define OFF_FP(a) (a - SHADOWFRAME_VREGS_OFFSET)
-#define OFF_FP_NUMBER_OF_VREGS OFF_FP(SHADOWFRAME_NUMBER_OF_VREGS_OFFSET)
-#define OFF_FP_DEX_PC OFF_FP(SHADOWFRAME_DEX_PC_OFFSET)
-#define OFF_FP_LINK OFF_FP(SHADOWFRAME_LINK_OFFSET)
-#define OFF_FP_METHOD OFF_FP(SHADOWFRAME_METHOD_OFFSET)
-#define OFF_FP_RESULT_REGISTER OFF_FP(SHADOWFRAME_RESULT_REGISTER_OFFSET)
-#define OFF_FP_DEX_PC_PTR OFF_FP(SHADOWFRAME_DEX_PC_PTR_OFFSET)
-#define OFF_FP_CODE_ITEM OFF_FP(SHADOWFRAME_CODE_ITEM_OFFSET)
-#define OFF_FP_SHADOWFRAME (-SHADOWFRAME_VREGS_OFFSET)
-
-#define MTERP_PROFILE_BRANCHES 1
#define MTERP_LOGGING 0
/*
- * Profile branch. rINST should contain the offset. %eax is scratch.
- */
-.macro MTERP_PROFILE_BRANCH
-#ifdef MTERP_PROFILE_BRANCHES
- EXPORT_PC
- movq rSELF, OUT_ARG0
- leaq OFF_FP_SHADOWFRAME(rFP), OUT_ARG1
- movl rINST, OUT_32_ARG2
- call SYMBOL(MterpProfileBranch)
- testb %al, %al
- jnz MterpOnStackReplacement
-#endif
-.endm
-
-/*
* "export" the PC to dex_pc field in the shadow frame, f/b/o future exception objects. Must
* be done *before* something throws.
*
@@ -204,7 +192,8 @@
*
*/
.macro REFRESH_IBASE
- movq THREAD_CURRENT_IBASE_OFFSET(rSELF), rIBASE
+ movq rSELF, rIBASE
+ movq THREAD_CURRENT_IBASE_OFFSET(rIBASE), rIBASE
.endm
/*
diff --git a/runtime/interpreter/mterp/x86_64/op_aget_object.S b/runtime/interpreter/mterp/x86_64/op_aget_object.S
index 8baedea..5f77a97 100644
--- a/runtime/interpreter/mterp/x86_64/op_aget_object.S
+++ b/runtime/interpreter/mterp/x86_64/op_aget_object.S
@@ -10,7 +10,8 @@
GET_VREG OUT_32_ARG1, %rcx # ecx <- vCC (requested index)
EXPORT_PC
call SYMBOL(artAGetObjectFromMterp) # (array, index)
- cmpq $$0, THREAD_EXCEPTION_OFFSET(rSELF)
+ movq rSELF, %rcx
+ cmpq $$0, THREAD_EXCEPTION_OFFSET(%rcx)
jnz MterpException
SET_VREG_OBJECT %eax, rINSTq
ADVANCE_PC_FETCH_AND_GOTO_NEXT 2
diff --git a/runtime/interpreter/mterp/x86_64/op_goto.S b/runtime/interpreter/mterp/x86_64/op_goto.S
index c4fc976..9749901 100644
--- a/runtime/interpreter/mterp/x86_64/op_goto.S
+++ b/runtime/interpreter/mterp/x86_64/op_goto.S
@@ -6,9 +6,5 @@
*/
/* goto +AA */
movsbq rINSTbl, rINSTq # rINSTq <- ssssssAA
- MTERP_PROFILE_BRANCH
- addq rINSTq, rINSTq # rINSTq <- AA * 2
- leaq (rPC, rINSTq), rPC
- FETCH_INST
- jle MterpCheckSuspendAndContinue # AA * 2 <= 0 => suspend check
- GOTO_NEXT
+ testq rINSTq, rINSTq
+ jmp MterpCommonTakenBranch
diff --git a/runtime/interpreter/mterp/x86_64/op_goto_16.S b/runtime/interpreter/mterp/x86_64/op_goto_16.S
index 8cb9a5c..77688e0 100644
--- a/runtime/interpreter/mterp/x86_64/op_goto_16.S
+++ b/runtime/interpreter/mterp/x86_64/op_goto_16.S
@@ -6,9 +6,5 @@
*/
/* goto/16 +AAAA */
movswq 2(rPC), rINSTq # rINSTq <- ssssAAAA
- MTERP_PROFILE_BRANCH
- addq rINSTq, rINSTq # rINSTq <- AA * 2
- leaq (rPC, rINSTq), rPC
- FETCH_INST
- jle MterpCheckSuspendAndContinue # AA * 2 <= 0 => suspend check
- GOTO_NEXT
+ testq rINSTq, rINSTq
+ jmp MterpCommonTakenBranch
diff --git a/runtime/interpreter/mterp/x86_64/op_goto_32.S b/runtime/interpreter/mterp/x86_64/op_goto_32.S
index 4ecdacd..29d777b 100644
--- a/runtime/interpreter/mterp/x86_64/op_goto_32.S
+++ b/runtime/interpreter/mterp/x86_64/op_goto_32.S
@@ -9,9 +9,5 @@
*/
/* goto/32 +AAAAAAAA */
movslq 2(rPC), rINSTq # rINSTq <- AAAAAAAA
- MTERP_PROFILE_BRANCH
- addq rINSTq, rINSTq # rINSTq <- AA * 2
- leaq (rPC, rINSTq), rPC
- FETCH_INST
- jle MterpCheckSuspendAndContinue # AA * 2 <= 0 => suspend check
- GOTO_NEXT
+ testq rINSTq, rINSTq
+ jmp MterpCommonTakenBranch
diff --git a/runtime/interpreter/mterp/x86_64/op_iget.S b/runtime/interpreter/mterp/x86_64/op_iget.S
index a0d0faf..df43efe 100644
--- a/runtime/interpreter/mterp/x86_64/op_iget.S
+++ b/runtime/interpreter/mterp/x86_64/op_iget.S
@@ -12,7 +12,8 @@
movq OFF_FP_METHOD(rFP), OUT_ARG2 # referrer
movq rSELF, OUT_ARG3
call SYMBOL($helper)
- cmpq $$0, THREAD_EXCEPTION_OFFSET(rSELF)
+ movq rSELF, %rcx
+ cmpq $$0, THREAD_EXCEPTION_OFFSET(%rcx)
jnz MterpException # bail out
andb $$0xf, rINSTbl # rINST <- A
.if $is_object
diff --git a/runtime/interpreter/mterp/x86_64/op_iget_object_quick.S b/runtime/interpreter/mterp/x86_64/op_iget_object_quick.S
index 964d20a..176c954 100644
--- a/runtime/interpreter/mterp/x86_64/op_iget_object_quick.S
+++ b/runtime/interpreter/mterp/x86_64/op_iget_object_quick.S
@@ -7,7 +7,8 @@
movzwl 2(rPC), OUT_32_ARG1 # eax <- field byte offset
EXPORT_PC
callq SYMBOL(artIGetObjectFromMterp) # (obj, offset)
- cmpq $$0, THREAD_EXCEPTION_OFFSET(rSELF)
+ movq rSELF, %rcx
+ cmpq $$0, THREAD_EXCEPTION_OFFSET(%rcx)
jnz MterpException # bail out
andb $$0xf, rINSTbl # rINST <- A
SET_VREG_OBJECT %eax, rINSTq # fp[A] <- value
diff --git a/runtime/interpreter/mterp/x86_64/op_instance_of.S b/runtime/interpreter/mterp/x86_64/op_instance_of.S
index 6be37f9..4819833 100644
--- a/runtime/interpreter/mterp/x86_64/op_instance_of.S
+++ b/runtime/interpreter/mterp/x86_64/op_instance_of.S
@@ -14,7 +14,8 @@
movq rSELF, OUT_ARG3
call SYMBOL(MterpInstanceOf) # (index, &obj, method, self)
movsbl %al, %eax
- cmpq $$0, THREAD_EXCEPTION_OFFSET(rSELF)
+ movq rSELF, %rcx
+ cmpq $$0, THREAD_EXCEPTION_OFFSET(%rcx)
jnz MterpException
andb $$0xf, rINSTbl # rINSTbl <- A
SET_VREG %eax, rINSTq
diff --git a/runtime/interpreter/mterp/x86_64/op_move_exception.S b/runtime/interpreter/mterp/x86_64/op_move_exception.S
index d0a14fd..33db878 100644
--- a/runtime/interpreter/mterp/x86_64/op_move_exception.S
+++ b/runtime/interpreter/mterp/x86_64/op_move_exception.S
@@ -1,5 +1,6 @@
/* move-exception vAA */
- movl THREAD_EXCEPTION_OFFSET(rSELF), %eax
+ movq rSELF, %rcx
+ movl THREAD_EXCEPTION_OFFSET(%rcx), %eax
SET_VREG_OBJECT %eax, rINSTq # fp[AA] <- exception object
- movl $$0, THREAD_EXCEPTION_OFFSET(rSELF)
+ movl $$0, THREAD_EXCEPTION_OFFSET(%rcx)
ADVANCE_PC_FETCH_AND_GOTO_NEXT 1
diff --git a/runtime/interpreter/mterp/x86_64/op_packed_switch.S b/runtime/interpreter/mterp/x86_64/op_packed_switch.S
index cb0acb7..fdf5a50 100644
--- a/runtime/interpreter/mterp/x86_64/op_packed_switch.S
+++ b/runtime/interpreter/mterp/x86_64/op_packed_switch.S
@@ -13,10 +13,6 @@
leaq (rPC,OUT_ARG0,2), OUT_ARG0 # rcx <- PC + BBBBbbbb*2
GET_VREG OUT_32_ARG1, rINSTq # eax <- vAA
call SYMBOL($func)
+ testl %eax, %eax
movslq %eax, rINSTq
- MTERP_PROFILE_BRANCH
- addq rINSTq, rINSTq
- leaq (rPC, rINSTq), rPC
- FETCH_INST
- jle MterpCheckSuspendAndContinue
- GOTO_NEXT
+ jmp MterpCommonTakenBranch
diff --git a/runtime/interpreter/mterp/x86_64/op_return.S b/runtime/interpreter/mterp/x86_64/op_return.S
index 14f4f8a..07e0e53 100644
--- a/runtime/interpreter/mterp/x86_64/op_return.S
+++ b/runtime/interpreter/mterp/x86_64/op_return.S
@@ -6,9 +6,9 @@
/* op vAA */
.extern MterpThreadFenceForConstructor
call SYMBOL(MterpThreadFenceForConstructor)
- testl $$(THREAD_SUSPEND_REQUEST | THREAD_CHECKPOINT_REQUEST), THREAD_FLAGS_OFFSET(rSELF)
- jz 1f
movq rSELF, OUT_ARG0
+ testl $$(THREAD_SUSPEND_REQUEST | THREAD_CHECKPOINT_REQUEST), THREAD_FLAGS_OFFSET(OUT_ARG0)
+ jz 1f
call SYMBOL(MterpSuspendCheck)
1:
GET_VREG %eax, rINSTq # eax <- vAA
diff --git a/runtime/interpreter/mterp/x86_64/op_return_void.S b/runtime/interpreter/mterp/x86_64/op_return_void.S
index 46a5753..6a12df3 100644
--- a/runtime/interpreter/mterp/x86_64/op_return_void.S
+++ b/runtime/interpreter/mterp/x86_64/op_return_void.S
@@ -1,8 +1,8 @@
.extern MterpThreadFenceForConstructor
call SYMBOL(MterpThreadFenceForConstructor)
- testl $$(THREAD_SUSPEND_REQUEST | THREAD_CHECKPOINT_REQUEST), THREAD_FLAGS_OFFSET(rSELF)
- jz 1f
movq rSELF, OUT_ARG0
+ testl $$(THREAD_SUSPEND_REQUEST | THREAD_CHECKPOINT_REQUEST), THREAD_FLAGS_OFFSET(OUT_ARG0)
+ jz 1f
call SYMBOL(MterpSuspendCheck)
1:
xorq %rax, %rax
diff --git a/runtime/interpreter/mterp/x86_64/op_return_void_no_barrier.S b/runtime/interpreter/mterp/x86_64/op_return_void_no_barrier.S
index 92e3506..822b2e8 100644
--- a/runtime/interpreter/mterp/x86_64/op_return_void_no_barrier.S
+++ b/runtime/interpreter/mterp/x86_64/op_return_void_no_barrier.S
@@ -1,6 +1,6 @@
- testl $$(THREAD_SUSPEND_REQUEST | THREAD_CHECKPOINT_REQUEST), THREAD_FLAGS_OFFSET(rSELF)
- jz 1f
movq rSELF, OUT_ARG0
+ testl $$(THREAD_SUSPEND_REQUEST | THREAD_CHECKPOINT_REQUEST), THREAD_FLAGS_OFFSET(OUT_ARG0)
+ jz 1f
call SYMBOL(MterpSuspendCheck)
1:
xorq %rax, %rax
diff --git a/runtime/interpreter/mterp/x86_64/op_return_wide.S b/runtime/interpreter/mterp/x86_64/op_return_wide.S
index f2d6e04..288eb96 100644
--- a/runtime/interpreter/mterp/x86_64/op_return_wide.S
+++ b/runtime/interpreter/mterp/x86_64/op_return_wide.S
@@ -4,9 +4,9 @@
/* return-wide vAA */
.extern MterpThreadFenceForConstructor
call SYMBOL(MterpThreadFenceForConstructor)
- testl $$(THREAD_SUSPEND_REQUEST | THREAD_CHECKPOINT_REQUEST), THREAD_FLAGS_OFFSET(rSELF)
- jz 1f
movq rSELF, OUT_ARG0
+ testl $$(THREAD_SUSPEND_REQUEST | THREAD_CHECKPOINT_REQUEST), THREAD_FLAGS_OFFSET(OUT_ARG0)
+ jz 1f
call SYMBOL(MterpSuspendCheck)
1:
GET_WIDE_VREG %rax, rINSTq # eax <- v[AA]
diff --git a/runtime/interpreter/mterp/x86_64/op_sget.S b/runtime/interpreter/mterp/x86_64/op_sget.S
index 38d9a5e..d39e6c4 100644
--- a/runtime/interpreter/mterp/x86_64/op_sget.S
+++ b/runtime/interpreter/mterp/x86_64/op_sget.S
@@ -11,7 +11,8 @@
movq OFF_FP_METHOD(rFP), OUT_ARG1 # referrer
movq rSELF, OUT_ARG2 # self
call SYMBOL($helper)
- cmpl $$0, THREAD_EXCEPTION_OFFSET(rSELF)
+ movq rSELF, %rcx
+ cmpl $$0, THREAD_EXCEPTION_OFFSET(%rcx)
jnz MterpException
.if $is_object
SET_VREG_OBJECT %eax, rINSTq # fp[A] <- value
diff --git a/runtime/interpreter/mterp/x86_64/op_throw.S b/runtime/interpreter/mterp/x86_64/op_throw.S
index 22ed990..8095c25 100644
--- a/runtime/interpreter/mterp/x86_64/op_throw.S
+++ b/runtime/interpreter/mterp/x86_64/op_throw.S
@@ -6,5 +6,6 @@
GET_VREG %eax, rINSTq # eax<- vAA (exception object)
testb %al, %al
jz common_errNullObject
- movq %rax, THREAD_EXCEPTION_OFFSET(rSELF)
+ movq rSELF, %rcx
+ movq %rax, THREAD_EXCEPTION_OFFSET(%rcx)
jmp MterpException
diff --git a/runtime/interpreter/mterp/x86_64/zcmp.S b/runtime/interpreter/mterp/x86_64/zcmp.S
index 0051407..fb8ae6a 100644
--- a/runtime/interpreter/mterp/x86_64/zcmp.S
+++ b/runtime/interpreter/mterp/x86_64/zcmp.S
@@ -7,13 +7,11 @@
*/
/* if-cmp vAA, +BBBB */
cmpl $$0, VREG_ADDRESS(rINSTq) # compare (vA, 0)
- movl $$2, rINST # assume branch not taken
j${revcmp} 1f
movswq 2(rPC), rINSTq # fetch signed displacement
+ testq rINSTq, rINSTq
+ jmp MterpCommonTakenBranch
1:
- MTERP_PROFILE_BRANCH
- addq rINSTq, rINSTq # rINSTq <- AA * 2
- leaq (rPC, rINSTq), rPC
- FETCH_INST
- jle MterpCheckSuspendAndContinue # AA * 2 <= 0 => suspend check
- GOTO_NEXT
+ cmpl $$JIT_CHECK_OSR, rPROFILE
+ je .L_check_not_taken_osr
+ ADVANCE_PC_FETCH_AND_GOTO_NEXT 2
diff --git a/runtime/interpreter/unstarted_runtime.cc b/runtime/interpreter/unstarted_runtime.cc
index b21f1ec..78f34ff 100644
--- a/runtime/interpreter/unstarted_runtime.cc
+++ b/runtime/interpreter/unstarted_runtime.cc
@@ -444,46 +444,42 @@
UnstartedRuntime::UnstartedSystemArraycopy(self, shadow_frame, result, arg_offset);
}
+void UnstartedRuntime::UnstartedSystemGetSecurityManager(
+ Thread* self ATTRIBUTE_UNUSED, ShadowFrame* shadow_frame ATTRIBUTE_UNUSED,
+ JValue* result, size_t arg_offset ATTRIBUTE_UNUSED) {
+ result->SetL(nullptr);
+}
+
void UnstartedRuntime::UnstartedThreadLocalGet(
Thread* self, ShadowFrame* shadow_frame, JValue* result, size_t arg_offset ATTRIBUTE_UNUSED) {
std::string caller(PrettyMethod(shadow_frame->GetLink()->GetMethod()));
bool ok = false;
- if (caller == "java.lang.String java.lang.IntegralToString.convertInt"
- "(java.lang.AbstractStringBuilder, int)") {
+ if (caller == "void java.lang.FloatingDecimal.developLongDigits(int, long, long)" ||
+ caller == "java.lang.String java.lang.FloatingDecimal.toJavaFormatString()") {
// Allocate non-threadlocal buffer.
- result->SetL(mirror::CharArray::Alloc(self, 11));
+ result->SetL(mirror::CharArray::Alloc(self, 26));
ok = true;
- } else if (caller == "java.lang.RealToString java.lang.RealToString.getInstance()") {
- // Note: RealToString is implemented and used in a different fashion than IntegralToString.
- // Conversion is done over an actual object of RealToString (the conversion method is an
- // instance method). This means it is not as clear whether it is correct to return a new
- // object each time. The caller needs to be inspected by hand to see whether it (incorrectly)
- // stores the object for later use.
- // See also b/19548084 for a possible rewrite and bringing it in line with IntegralToString.
- if (shadow_frame->GetLink()->GetLink() != nullptr) {
- std::string caller2(PrettyMethod(shadow_frame->GetLink()->GetLink()->GetMethod()));
- if (caller2 == "java.lang.String java.lang.Double.toString(double)") {
- // Allocate new object.
- StackHandleScope<2> hs(self);
- Handle<mirror::Class> h_real_to_string_class(hs.NewHandle(
- shadow_frame->GetLink()->GetMethod()->GetDeclaringClass()));
- Handle<mirror::Object> h_real_to_string_obj(hs.NewHandle(
- h_real_to_string_class->AllocObject(self)));
- if (h_real_to_string_obj.Get() != nullptr) {
- auto* cl = Runtime::Current()->GetClassLinker();
- ArtMethod* init_method = h_real_to_string_class->FindDirectMethod(
- "<init>", "()V", cl->GetImagePointerSize());
- if (init_method == nullptr) {
- h_real_to_string_class->DumpClass(LOG(FATAL), mirror::Class::kDumpClassFullDetail);
- } else {
- JValue invoke_result;
- EnterInterpreterFromInvoke(self, init_method, h_real_to_string_obj.Get(), nullptr,
- nullptr);
- if (!self->IsExceptionPending()) {
- result->SetL(h_real_to_string_obj.Get());
- ok = true;
- }
- }
+ } else if (caller ==
+ "java.lang.FloatingDecimal java.lang.FloatingDecimal.getThreadLocalInstance()") {
+ // Allocate new object.
+ StackHandleScope<2> hs(self);
+ Handle<mirror::Class> h_real_to_string_class(hs.NewHandle(
+ shadow_frame->GetLink()->GetMethod()->GetDeclaringClass()));
+ Handle<mirror::Object> h_real_to_string_obj(hs.NewHandle(
+ h_real_to_string_class->AllocObject(self)));
+ if (h_real_to_string_obj.Get() != nullptr) {
+ auto* cl = Runtime::Current()->GetClassLinker();
+ ArtMethod* init_method = h_real_to_string_class->FindDirectMethod(
+ "<init>", "()V", cl->GetImagePointerSize());
+ if (init_method == nullptr) {
+ h_real_to_string_class->DumpClass(LOG(FATAL), mirror::Class::kDumpClassFullDetail);
+ } else {
+ JValue invoke_result;
+ EnterInterpreterFromInvoke(self, init_method, h_real_to_string_obj.Get(), nullptr,
+ nullptr);
+ if (!self->IsExceptionPending()) {
+ result->SetL(h_real_to_string_obj.Get());
+ ok = true;
}
}
}
@@ -690,32 +686,47 @@
Thread* self, ShadowFrame* shadow_frame ATTRIBUTE_UNUSED, JValue* result,
size_t arg_offset ATTRIBUTE_UNUSED) {
Runtime* runtime = Runtime::Current();
- const std::vector<const DexFile*>& path = runtime->GetClassLinker()->GetBootClassPath();
- std::string canonical(DexFile::GetDexCanonicalLocation(path[0]->GetLocation().c_str()));
+
+ std::vector<std::string> split;
+ Split(runtime->GetBootClassPathString(), ':', &split);
+ if (split.empty()) {
+ AbortTransactionOrFail(self,
+ "Boot classpath not set or split error:: %s",
+ runtime->GetBootClassPathString().c_str());
+ return;
+ }
+ const std::string& source = split[0];
+
mirror::String* string_data;
// Use a block to enclose the I/O and MemMap code so buffers are released early.
{
std::string error_msg;
- std::unique_ptr<ZipArchive> zip_archive(ZipArchive::Open(canonical.c_str(), &error_msg));
+ std::unique_ptr<ZipArchive> zip_archive(ZipArchive::Open(source.c_str(), &error_msg));
if (zip_archive.get() == nullptr) {
- AbortTransactionOrFail(self, "Could not open zip file %s: %s", canonical.c_str(),
+ AbortTransactionOrFail(self,
+ "Could not open zip file %s: %s",
+ source.c_str(),
error_msg.c_str());
return;
}
std::unique_ptr<ZipEntry> zip_entry(zip_archive->Find("java/security/security.properties",
&error_msg));
if (zip_entry.get() == nullptr) {
- AbortTransactionOrFail(self, "Could not find security.properties file in %s: %s",
- canonical.c_str(), error_msg.c_str());
+ AbortTransactionOrFail(self,
+ "Could not find security.properties file in %s: %s",
+ source.c_str(),
+ error_msg.c_str());
return;
}
- std::unique_ptr<MemMap> map(zip_entry->ExtractToMemMap(canonical.c_str(),
+ std::unique_ptr<MemMap> map(zip_entry->ExtractToMemMap(source.c_str(),
"java/security/security.properties",
&error_msg));
if (map.get() == nullptr) {
- AbortTransactionOrFail(self, "Could not unzip security.properties file in %s: %s",
- canonical.c_str(), error_msg.c_str());
+ AbortTransactionOrFail(self,
+ "Could not unzip security.properties file in %s: %s",
+ source.c_str(),
+ error_msg.c_str());
return;
}
@@ -728,8 +739,7 @@
}
if (string_data == nullptr) {
- AbortTransactionOrFail(self, "Could not create string from file content of %s",
- canonical.c_str());
+ AbortTransactionOrFail(self, "Could not create string from file content of %s", source.c_str());
return;
}
@@ -1220,6 +1230,19 @@
result->SetZ(success ? JNI_TRUE : JNI_FALSE);
}
+void UnstartedRuntime::UnstartedJNIUnsafeGetIntVolatile(
+ Thread* self, ArtMethod* method ATTRIBUTE_UNUSED, mirror::Object* receiver ATTRIBUTE_UNUSED,
+ uint32_t* args, JValue* result) {
+ mirror::Object* obj = reinterpret_cast<mirror::Object*>(args[0]);
+ if (obj == nullptr) {
+ AbortTransactionOrFail(self, "Cannot access null object, retry at runtime.");
+ return;
+ }
+
+ jlong offset = (static_cast<uint64_t>(args[2]) << 32) | args[1];
+ result->SetI(obj->GetField32Volatile(MemberOffset(offset)));
+}
+
void UnstartedRuntime::UnstartedJNIUnsafePutObject(
Thread* self ATTRIBUTE_UNUSED, ArtMethod* method ATTRIBUTE_UNUSED,
mirror::Object* receiver ATTRIBUTE_UNUSED, uint32_t* args, JValue* result ATTRIBUTE_UNUSED) {
diff --git a/runtime/interpreter/unstarted_runtime_list.h b/runtime/interpreter/unstarted_runtime_list.h
index 29f2197..d669b75 100644
--- a/runtime/interpreter/unstarted_runtime_list.h
+++ b/runtime/interpreter/unstarted_runtime_list.h
@@ -31,6 +31,7 @@
V(SystemArraycopy, "void java.lang.System.arraycopy(java.lang.Object, int, java.lang.Object, int, int)") \
V(SystemArraycopyChar, "void java.lang.System.arraycopy(char[], int, char[], int, int)") \
V(SystemArraycopyInt, "void java.lang.System.arraycopy(int[], int, int[], int, int)") \
+ V(SystemGetSecurityManager, "java.lang.SecurityManager java.lang.System.getSecurityManager()") \
V(ThreadLocalGet, "java.lang.Object java.lang.ThreadLocal.get()") \
V(MathCeil, "double java.lang.Math.ceil(double)") \
V(ObjectHashCode, "int java.lang.Object.hashCode()") \
@@ -79,6 +80,7 @@
V(SystemIdentityHashCode, "int java.lang.System.identityHashCode(java.lang.Object)") \
V(ByteOrderIsLittleEndian, "boolean java.nio.ByteOrder.isLittleEndian()") \
V(UnsafeCompareAndSwapInt, "boolean sun.misc.Unsafe.compareAndSwapInt(java.lang.Object, long, int, int)") \
+ V(UnsafeGetIntVolatile, "int sun.misc.Unsafe.getIntVolatile(java.lang.Object, long)") \
V(UnsafePutObject, "void sun.misc.Unsafe.putObject(java.lang.Object, long, java.lang.Object)") \
V(UnsafeGetArrayBaseOffsetForComponentType, "int sun.misc.Unsafe.getArrayBaseOffsetForComponentType(java.lang.Class)") \
V(UnsafeGetArrayIndexScaleForComponentType, "int sun.misc.Unsafe.getArrayIndexScaleForComponentType(java.lang.Class)")
diff --git a/runtime/jit/jit_instrumentation.cc b/runtime/jit/jit_instrumentation.cc
index d751e5a..b18d6a2 100644
--- a/runtime/jit/jit_instrumentation.cc
+++ b/runtime/jit/jit_instrumentation.cc
@@ -80,9 +80,9 @@
DISALLOW_IMPLICIT_CONSTRUCTORS(JitCompileTask);
};
-JitInstrumentationCache::JitInstrumentationCache(size_t hot_method_threshold,
- size_t warm_method_threshold,
- size_t osr_method_threshold)
+JitInstrumentationCache::JitInstrumentationCache(uint16_t hot_method_threshold,
+ uint16_t warm_method_threshold,
+ uint16_t osr_method_threshold)
: hot_method_threshold_(hot_method_threshold),
warm_method_threshold_(warm_method_threshold),
osr_method_threshold_(osr_method_threshold),
@@ -130,44 +130,62 @@
}
}
-void JitInstrumentationCache::AddSamples(Thread* self, ArtMethod* method, size_t) {
+void JitInstrumentationCache::AddSamples(Thread* self, ArtMethod* method, uint16_t count) {
// Since we don't have on-stack replacement, some methods can remain in the interpreter longer
- // than we want resulting in samples even after the method is compiled.
- if (method->IsClassInitializer() || method->IsNative()) {
+ // than we want resulting in samples even after the method is compiled. Also, if the
+ // jit is no longer interested in hotness samples because we're shutting down, just return.
+ if (method->IsClassInitializer() || method->IsNative() || (thread_pool_ == nullptr)) {
+ if (thread_pool_ == nullptr) {
+ // Should only see this when shutting down.
+ DCHECK(Runtime::Current()->IsShuttingDown(self));
+ }
return;
}
DCHECK(thread_pool_ != nullptr);
+ DCHECK_GT(warm_method_threshold_, 0);
+ DCHECK_GT(hot_method_threshold_, warm_method_threshold_);
+ DCHECK_GT(osr_method_threshold_, hot_method_threshold_);
- uint16_t sample_count = method->IncrementCounter();
- if (sample_count == warm_method_threshold_) {
- bool success = ProfilingInfo::Create(self, method, /* retry_allocation */ false);
- if (success) {
- VLOG(jit) << "Start profiling " << PrettyMethod(method);
+ int32_t starting_count = method->GetCounter();
+ int32_t new_count = starting_count + count; // int32 here to avoid wrap-around;
+ if (starting_count < warm_method_threshold_) {
+ if (new_count >= warm_method_threshold_) {
+ bool success = ProfilingInfo::Create(self, method, /* retry_allocation */ false);
+ if (success) {
+ VLOG(jit) << "Start profiling " << PrettyMethod(method);
+ }
+
+ if (thread_pool_ == nullptr) {
+ // Calling ProfilingInfo::Create might put us in a suspended state, which could
+ // lead to the thread pool being deleted when we are shutting down.
+ DCHECK(Runtime::Current()->IsShuttingDown(self));
+ return;
+ }
+
+ if (!success) {
+ // We failed allocating. Instead of doing the collection on the Java thread, we push
+ // an allocation to a compiler thread, that will do the collection.
+ thread_pool_->AddTask(self, new JitCompileTask(method, JitCompileTask::kAllocateProfile));
+ }
}
-
- if (thread_pool_ == nullptr) {
- // Calling ProfilingInfo::Create might put us in a suspended state, which could
- // lead to the thread pool being deleted when we are shutting down.
- DCHECK(Runtime::Current()->IsShuttingDown(self));
- return;
+ // Avoid jumping more than one state at a time.
+ new_count = std::min(new_count, hot_method_threshold_ - 1);
+ } else if (starting_count < hot_method_threshold_) {
+ if (new_count >= hot_method_threshold_) {
+ DCHECK(thread_pool_ != nullptr);
+ thread_pool_->AddTask(self, new JitCompileTask(method, JitCompileTask::kCompile));
}
-
- if (!success) {
- // We failed allocating. Instead of doing the collection on the Java thread, we push
- // an allocation to a compiler thread, that will do the collection.
- thread_pool_->AddTask(self, new JitCompileTask(method, JitCompileTask::kAllocateProfile));
+ // Avoid jumping more than one state at a time.
+ new_count = std::min(new_count, osr_method_threshold_ - 1);
+ } else if (starting_count < osr_method_threshold_) {
+ if (new_count >= osr_method_threshold_) {
+ DCHECK(thread_pool_ != nullptr);
+ thread_pool_->AddTask(self, new JitCompileTask(method, JitCompileTask::kCompileOsr));
}
}
-
- if (sample_count == hot_method_threshold_) {
- DCHECK(thread_pool_ != nullptr);
- thread_pool_->AddTask(self, new JitCompileTask(method, JitCompileTask::kCompile));
- }
-
- if (sample_count == osr_method_threshold_) {
- DCHECK(thread_pool_ != nullptr);
- thread_pool_->AddTask(self, new JitCompileTask(method, JitCompileTask::kCompileOsr));
- }
+ // Update hotness counter, but avoid wrap around.
+ method->SetCounter(
+ std::min(new_count, static_cast<int32_t>(std::numeric_limits<uint16_t>::max())));
}
JitInstrumentationListener::JitInstrumentationListener(JitInstrumentationCache* cache)
diff --git a/runtime/jit/jit_instrumentation.h b/runtime/jit/jit_instrumentation.h
index d1c5c44..7ffd4eb 100644
--- a/runtime/jit/jit_instrumentation.h
+++ b/runtime/jit/jit_instrumentation.h
@@ -40,6 +40,8 @@
class Thread;
namespace jit {
+static constexpr int16_t kJitCheckForOSR = -1;
+static constexpr int16_t kJitHotnessDisabled = -2;
class JitInstrumentationCache;
@@ -84,7 +86,6 @@
static constexpr uint32_t kJitEvents =
instrumentation::Instrumentation::kMethodEntered |
- instrumentation::Instrumentation::kBranch |
instrumentation::Instrumentation::kInvokeVirtualOrInterface;
private:
@@ -96,25 +97,33 @@
// Keeps track of which methods are hot.
class JitInstrumentationCache {
public:
- JitInstrumentationCache(size_t hot_method_threshold,
- size_t warm_method_threshold,
- size_t osr_method_threshold);
- void AddSamples(Thread* self, ArtMethod* method, size_t samples)
+ JitInstrumentationCache(uint16_t hot_method_threshold,
+ uint16_t warm_method_threshold,
+ uint16_t osr_method_threshold);
+ void AddSamples(Thread* self, ArtMethod* method, uint16_t samples)
SHARED_REQUIRES(Locks::mutator_lock_);
void CreateThreadPool();
void DeleteThreadPool(Thread* self);
+ size_t OSRMethodThreshold() const {
+ return osr_method_threshold_;
+ }
+
size_t HotMethodThreshold() const {
return hot_method_threshold_;
}
+ size_t WarmMethodThreshold() const {
+ return warm_method_threshold_;
+ }
+
// Wait until there is no more pending compilation tasks.
void WaitForCompilationToFinish(Thread* self);
private:
- size_t hot_method_threshold_;
- size_t warm_method_threshold_;
- size_t osr_method_threshold_;
+ uint16_t hot_method_threshold_;
+ uint16_t warm_method_threshold_;
+ uint16_t osr_method_threshold_;
JitInstrumentationListener listener_;
std::unique_ptr<ThreadPool> thread_pool_;
diff --git a/runtime/mirror/object.h b/runtime/mirror/object.h
index 3f739df..0ee46c3 100644
--- a/runtime/mirror/object.h
+++ b/runtime/mirror/object.h
@@ -17,6 +17,7 @@
#ifndef ART_RUNTIME_MIRROR_OBJECT_H_
#define ART_RUNTIME_MIRROR_OBJECT_H_
+#include "base/casts.h"
#include "globals.h"
#include "object_reference.h"
#include "offsets.h"
@@ -490,7 +491,7 @@
field_offset, static_cast<int32_t>(ptr));
} else {
SetField64<kTransactionActive, kCheckTransaction, kVerifyFlags>(
- field_offset, static_cast<int64_t>(reinterpret_cast<uintptr_t>(new_value)));
+ field_offset, reinterpret_cast64<int64_t>(new_value));
}
}
// TODO fix thread safety analysis broken by the use of template. This should be
@@ -531,9 +532,7 @@
return reinterpret_cast<T>(GetField32<kVerifyFlags, kIsVolatile>(field_offset));
} else {
int64_t v = GetField64<kVerifyFlags, kIsVolatile>(field_offset);
- // Check that we dont lose any non 0 bits.
- DCHECK_EQ(static_cast<int64_t>(static_cast<uintptr_t>(v)), v);
- return reinterpret_cast<T>(static_cast<uintptr_t>(v));
+ return reinterpret_cast64<T>(v);
}
}
diff --git a/runtime/native/dalvik_system_DexFile.cc b/runtime/native/dalvik_system_DexFile.cc
index 3397989..0abe39d 100644
--- a/runtime/native/dalvik_system_DexFile.cc
+++ b/runtime/native/dalvik_system_DexFile.cc
@@ -21,6 +21,7 @@
#include "base/stringprintf.h"
#include "class_linker.h"
#include "common_throws.h"
+#include "compiler_filter.h"
#include "dex_file-inl.h"
#include "jni_internal.h"
#include "mirror/class_loader.h"
@@ -390,7 +391,8 @@
jclass,
jstring javaFilename,
jstring javaInstructionSet,
- jint javaTargetCompilationTypeMask) {
+ jstring javaTargetCompilerFilter,
+ jboolean newProfile) {
ScopedUtfChars filename(env, javaFilename);
if (env->ExceptionCheck()) {
return -1;
@@ -401,18 +403,16 @@
return -1;
}
- // TODO: Take profile changed and compiler filter as arguments.
- // For now, we use "speed" by default, unless EXTRACT_ONLY = 0x4 was
- // included in the mask.
- const char* compiler_filter = "speed";
- if (javaTargetCompilationTypeMask & 0x4) {
- compiler_filter = "verify-at-runtime";
+ ScopedUtfChars target_compiler_filter(env, javaTargetCompilerFilter);
+ if (env->ExceptionCheck()) {
+ return -1;
}
+
return GetDexOptNeeded(env,
filename.c_str(),
instruction_set.c_str(),
- compiler_filter,
- /*profile_changed*/false);
+ target_compiler_filter.c_str(),
+ newProfile == JNI_TRUE);
}
// public API
@@ -428,6 +428,59 @@
return (status != OatFileAssistant::kNoDexOptNeeded) ? JNI_TRUE : JNI_FALSE;
}
+static jboolean DexFile_isValidCompilerFilter(JNIEnv* env,
+ jclass javeDexFileClass ATTRIBUTE_UNUSED,
+ jstring javaCompilerFilter) {
+ ScopedUtfChars compiler_filter(env, javaCompilerFilter);
+ if (env->ExceptionCheck()) {
+ return -1;
+ }
+
+ CompilerFilter::Filter filter;
+ return CompilerFilter::ParseCompilerFilter(compiler_filter.c_str(), &filter)
+ ? JNI_TRUE : JNI_FALSE;
+}
+
+static jboolean DexFile_isProfileGuidedCompilerFilter(JNIEnv* env,
+ jclass javeDexFileClass ATTRIBUTE_UNUSED,
+ jstring javaCompilerFilter) {
+ ScopedUtfChars compiler_filter(env, javaCompilerFilter);
+ if (env->ExceptionCheck()) {
+ return -1;
+ }
+
+ CompilerFilter::Filter filter;
+ if (!CompilerFilter::ParseCompilerFilter(compiler_filter.c_str(), &filter)) {
+ return JNI_FALSE;
+ }
+ return CompilerFilter::DependsOnProfile(filter) ? JNI_TRUE : JNI_FALSE;
+}
+
+static jstring DexFile_getNonProfileGuidedCompilerFilter(JNIEnv* env,
+ jclass javeDexFileClass ATTRIBUTE_UNUSED,
+ jstring javaCompilerFilter) {
+ ScopedUtfChars compiler_filter(env, javaCompilerFilter);
+ if (env->ExceptionCheck()) {
+ return nullptr;
+ }
+
+ CompilerFilter::Filter filter;
+ if (!CompilerFilter::ParseCompilerFilter(compiler_filter.c_str(), &filter)) {
+ return javaCompilerFilter;
+ }
+
+ CompilerFilter::Filter new_filter = CompilerFilter::GetNonProfileDependentFilterFrom(filter);
+
+ // Filter stayed the same, return input.
+ if (filter == new_filter) {
+ return javaCompilerFilter;
+ }
+
+ // Create a new string object and return.
+ std::string new_filter_str = CompilerFilter::NameOfFilter(new_filter);
+ return env->NewStringUTF(new_filter_str.c_str());
+}
+
static JNINativeMethod gMethods[] = {
NATIVE_METHOD(DexFile, closeDexFile, "(Ljava/lang/Object;)Z"),
NATIVE_METHOD(DexFile,
@@ -440,7 +493,7 @@
NATIVE_METHOD(DexFile, getClassNameList, "(Ljava/lang/Object;)[Ljava/lang/String;"),
NATIVE_METHOD(DexFile, isDexOptNeeded, "(Ljava/lang/String;)Z"),
NATIVE_METHOD(DexFile, getDexOptNeeded,
- "(Ljava/lang/String;Ljava/lang/String;I)I"),
+ "(Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;Z)I"),
NATIVE_METHOD(DexFile, openDexFileNative,
"(Ljava/lang/String;"
"Ljava/lang/String;"
@@ -448,6 +501,11 @@
"Ljava/lang/ClassLoader;"
"[Ldalvik/system/DexPathList$Element;"
")Ljava/lang/Object;"),
+ NATIVE_METHOD(DexFile, isValidCompilerFilter, "(Ljava/lang/String;)Z"),
+ NATIVE_METHOD(DexFile, isProfileGuidedCompilerFilter, "(Ljava/lang/String;)Z"),
+ NATIVE_METHOD(DexFile,
+ getNonProfileGuidedCompilerFilter,
+ "(Ljava/lang/String;)Ljava/lang/String;"),
};
void register_dalvik_system_DexFile(JNIEnv* env) {
diff --git a/runtime/oat.cc b/runtime/oat.cc
index d13999a..80231f3 100644
--- a/runtime/oat.cc
+++ b/runtime/oat.cc
@@ -462,6 +462,10 @@
return IsKeyEnabled(OatHeader::kPicKey);
}
+bool OatHeader::HasPatchInfo() const {
+ return IsKeyEnabled(OatHeader::kHasPatchInfoKey);
+}
+
bool OatHeader::IsDebuggable() const {
return IsKeyEnabled(OatHeader::kDebuggableKey);
}
diff --git a/runtime/oat.h b/runtime/oat.h
index 4e595a1..68e71c4 100644
--- a/runtime/oat.h
+++ b/runtime/oat.h
@@ -38,6 +38,7 @@
static constexpr const char* kDex2OatCmdLineKey = "dex2oat-cmdline";
static constexpr const char* kDex2OatHostKey = "dex2oat-host";
static constexpr const char* kPicKey = "pic";
+ static constexpr const char* kHasPatchInfoKey = "has-patch-info";
static constexpr const char* kDebuggableKey = "debuggable";
static constexpr const char* kNativeDebuggableKey = "native-debuggable";
static constexpr const char* kCompilerFilter = "compiler-filter";
@@ -109,6 +110,7 @@
size_t GetHeaderSize() const;
bool IsPic() const;
+ bool HasPatchInfo() const;
bool IsDebuggable() const;
bool IsNativeDebuggable() const;
CompilerFilter::Filter GetCompilerFilter() const;
diff --git a/runtime/oat_file.cc b/runtime/oat_file.cc
index 9ae033f..7c83715 100644
--- a/runtime/oat_file.cc
+++ b/runtime/oat_file.cc
@@ -1248,6 +1248,10 @@
method->SetEntryPointFromQuickCompiledCode(GetQuickCode());
}
+bool OatFile::HasPatchInfo() const {
+ return GetOatHeader().HasPatchInfo();
+}
+
bool OatFile::IsPic() const {
return GetOatHeader().IsPic();
// TODO: Check against oat_patches. b/18144996
diff --git a/runtime/oat_file.h b/runtime/oat_file.h
index 21aeab4..705ba0d 100644
--- a/runtime/oat_file.h
+++ b/runtime/oat_file.h
@@ -87,6 +87,8 @@
return is_executable_;
}
+ bool HasPatchInfo() const;
+
bool IsPic() const;
// Indicates whether the oat file was compiled with full debugging capability.
diff --git a/runtime/oat_file_assistant.cc b/runtime/oat_file_assistant.cc
index 096296b..ce892f3 100644
--- a/runtime/oat_file_assistant.cc
+++ b/runtime/oat_file_assistant.cc
@@ -166,15 +166,11 @@
// See if we can get an up-to-date file by running patchoat.
if (compilation_desired) {
- if (odex_okay && OdexFileNeedsRelocation()) {
- // TODO: don't return kPatchOatNeeded if the odex file contains no
- // patch information.
+ if (odex_okay && OdexFileNeedsRelocation() && OdexFileHasPatchInfo()) {
return kPatchOatNeeded;
}
- if (oat_okay && OatFileNeedsRelocation()) {
- // TODO: don't return kSelfPatchOatNeeded if the oat file contains no
- // patch information.
+ if (oat_okay && OatFileNeedsRelocation() && OatFileHasPatchInfo()) {
return kSelfPatchOatNeeded;
}
}
@@ -183,9 +179,10 @@
return HasOriginalDexFiles() ? kDex2OatNeeded : kNoDexOptNeeded;
}
-bool OatFileAssistant::MakeUpToDate(CompilerFilter::Filter target, std::string* error_msg) {
+OatFileAssistant::ResultOfAttemptToUpdate
+OatFileAssistant::MakeUpToDate(CompilerFilter::Filter target, std::string* error_msg) {
switch (GetDexOptNeeded(target)) {
- case kNoDexOptNeeded: return true;
+ case kNoDexOptNeeded: return kUpdateSucceeded;
case kDex2OatNeeded: return GenerateOatFile(target, error_msg);
case kPatchOatNeeded: return RelocateOatFile(OdexFileName(), error_msg);
case kSelfPatchOatNeeded: return RelocateOatFile(OatFileName(), error_msg);
@@ -573,21 +570,21 @@
return true;
}
-bool OatFileAssistant::RelocateOatFile(const std::string* input_file,
- std::string* error_msg) {
+OatFileAssistant::ResultOfAttemptToUpdate
+OatFileAssistant::RelocateOatFile(const std::string* input_file, std::string* error_msg) {
CHECK(error_msg != nullptr);
if (input_file == nullptr) {
*error_msg = "Patching of oat file for dex location " + dex_location_
+ " not attempted because the input file name could not be determined.";
- return false;
+ return kUpdateNotAttempted;
}
const std::string& input_file_name = *input_file;
if (OatFileName() == nullptr) {
*error_msg = "Patching of oat file for dex location " + dex_location_
+ " not attempted because the oat file name could not be determined.";
- return false;
+ return kUpdateNotAttempted;
}
const std::string& oat_file_name = *OatFileName();
@@ -596,13 +593,13 @@
if (image_info == nullptr) {
*error_msg = "Patching of oat file " + oat_file_name
+ " not attempted because no image location was found.";
- return false;
+ return kUpdateNotAttempted;
}
if (!runtime->IsDex2OatEnabled()) {
*error_msg = "Patching of oat file " + oat_file_name
+ " not attempted because dex2oat is disabled";
- return false;
+ return kUpdateNotAttempted;
}
std::vector<std::string> argv;
@@ -617,28 +614,29 @@
// Manually delete the file. This ensures there is no garbage left over if
// the process unexpectedly died.
TEMP_FAILURE_RETRY(unlink(oat_file_name.c_str()));
- return false;
+ return kUpdateFailed;
}
// Mark that the oat file has changed and we should try to reload.
ClearOatFileCache();
- return true;
+ return kUpdateSucceeded;
}
-bool OatFileAssistant::GenerateOatFile(CompilerFilter::Filter target, std::string* error_msg) {
+OatFileAssistant::ResultOfAttemptToUpdate
+OatFileAssistant::GenerateOatFile(CompilerFilter::Filter target, std::string* error_msg) {
CHECK(error_msg != nullptr);
Runtime* runtime = Runtime::Current();
if (!runtime->IsDex2OatEnabled()) {
*error_msg = "Generation of oat file for dex location " + dex_location_
+ " not attempted because dex2oat is disabled.";
- return false;
+ return kUpdateNotAttempted;
}
if (OatFileName() == nullptr) {
*error_msg = "Generation of oat file for dex location " + dex_location_
+ " not attempted because the oat file name could not be determined.";
- return false;
+ return kUpdateNotAttempted;
}
const std::string& oat_file_name = *OatFileName();
@@ -647,7 +645,7 @@
// TODO: Why does dex2oat behave that way?
if (!OS::FileExists(dex_location_.c_str())) {
*error_msg = "Dex location " + dex_location_ + " does not exists.";
- return false;
+ return kUpdateNotAttempted;
}
std::unique_ptr<File> oat_file;
@@ -655,14 +653,14 @@
if (oat_file.get() == nullptr) {
*error_msg = "Generation of oat file " + oat_file_name
+ " not attempted because the oat file could not be created.";
- return false;
+ return kUpdateNotAttempted;
}
if (fchmod(oat_file->Fd(), 0644) != 0) {
*error_msg = "Generation of oat file " + oat_file_name
+ " not attempted because the oat file could not be made world readable.";
oat_file->Erase();
- return false;
+ return kUpdateNotAttempted;
}
std::vector<std::string> args;
@@ -676,18 +674,18 @@
// the process unexpectedly died.
oat_file->Erase();
TEMP_FAILURE_RETRY(unlink(oat_file_name.c_str()));
- return false;
+ return kUpdateFailed;
}
if (oat_file->FlushCloseOrErase() != 0) {
*error_msg = "Unable to close oat file " + oat_file_name;
TEMP_FAILURE_RETRY(unlink(oat_file_name.c_str()));
- return false;
+ return kUpdateFailed;
}
// Mark that the oat file has changed and we should try to reload.
ClearOatFileCache();
- return true;
+ return kUpdateSucceeded;
}
bool OatFileAssistant::Dex2Oat(const std::vector<std::string>& args,
@@ -863,6 +861,11 @@
return (odex_file != nullptr && odex_file->IsExecutable());
}
+bool OatFileAssistant::OdexFileHasPatchInfo() {
+ const OatFile* odex_file = GetOdexFile();
+ return (odex_file != nullptr && odex_file->HasPatchInfo());
+}
+
void OatFileAssistant::ClearOdexFileCache() {
odex_file_load_attempted_ = false;
cached_odex_file_.reset();
@@ -899,6 +902,11 @@
return (oat_file != nullptr && oat_file->IsExecutable());
}
+bool OatFileAssistant::OatFileHasPatchInfo() {
+ const OatFile* oat_file = GetOatFile();
+ return (oat_file != nullptr && oat_file->HasPatchInfo());
+}
+
void OatFileAssistant::ClearOatFileCache() {
oat_file_load_attempted_ = false;
cached_oat_file_.reset();
diff --git a/runtime/oat_file_assistant.h b/runtime/oat_file_assistant.h
index 452cd84..17f72fe 100644
--- a/runtime/oat_file_assistant.h
+++ b/runtime/oat_file_assistant.h
@@ -148,14 +148,26 @@
// given compiler filter.
DexOptNeeded GetDexOptNeeded(CompilerFilter::Filter target_compiler_filter);
+ // Return code used when attempting to generate updated code.
+ enum ResultOfAttemptToUpdate {
+ kUpdateFailed, // We tried making the code up to date, but
+ // encountered an unexpected failure.
+ kUpdateNotAttempted, // We wanted to update the code, but determined we
+ // should not make the attempt.
+ kUpdateSucceeded // We successfully made the code up to date
+ // (possibly by doing nothing).
+ };
+
// Attempts to generate or relocate the oat file as needed to make it up to
// date with in a way that is at least as good as an oat file generated with
// the given compiler filter.
- // Returns true on success.
+ // Returns the result of attempting to update the code.
//
- // If there is a failure, the value of error_msg will be set to a string
- // describing why there was failure. error_msg must not be null.
- bool MakeUpToDate(CompilerFilter::Filter target_compiler_filter, std::string* error_msg);
+ // If the result is not kUpdateSucceeded, the value of error_msg will be set
+ // to a string describing why there was a failure or the update was not
+ // attempted. error_msg must not be null.
+ ResultOfAttemptToUpdate MakeUpToDate(CompilerFilter::Filter target_compiler_filter,
+ std::string* error_msg);
// Returns an oat file that can be used for loading dex files.
// Returns null if no suitable oat file was found.
@@ -232,22 +244,20 @@
// Generates the oat file by relocation from the named input file.
// This does not check the current status before attempting to relocate the
// oat file.
- // Returns true on success.
- // This will fail if dex2oat is not enabled in the current runtime.
//
- // If there is a failure, the value of error_msg will be set to a string
- // describing why there was failure. error_msg must not be null.
- bool RelocateOatFile(const std::string* input_file, std::string* error_msg);
+ // If the result is not kUpdateSucceeded, the value of error_msg will be set
+ // to a string describing why there was a failure or the update was not
+ // attempted. error_msg must not be null.
+ ResultOfAttemptToUpdate RelocateOatFile(const std::string* input_file, std::string* error_msg);
// Generate the oat file from the dex file using the given compiler filter.
// This does not check the current status before attempting to generate the
// oat file.
- // Returns true on success.
- // This will fail if dex2oat is not enabled in the current runtime.
//
- // If there is a failure, the value of error_msg will be set to a string
- // describing why there was failure. error_msg must not be null.
- bool GenerateOatFile(CompilerFilter::Filter filter, std::string* error_msg);
+ // If the result is not kUpdateSucceeded, the value of error_msg will be set
+ // to a string describing why there was a failure or the update was not
+ // attempted. error_msg must not be null.
+ ResultOfAttemptToUpdate GenerateOatFile(CompilerFilter::Filter filter, std::string* error_msg);
// Executes dex2oat using the current runtime configuration overridden with
// the given arguments. This does not check to see if dex2oat is enabled in
@@ -310,6 +320,9 @@
// Returns true if the odex file is opened executable.
bool OdexFileIsExecutable();
+ // Returns true if the odex file has patch info required to run patchoat.
+ bool OdexFileHasPatchInfo();
+
// Clear any cached information about the odex file that depends on the
// contents of the file.
void ClearOdexFileCache();
@@ -326,6 +339,9 @@
// Returns true if the oat file is opened executable.
bool OatFileIsExecutable();
+ // Returns true if the oat file has patch info required to run patchoat.
+ bool OatFileHasPatchInfo();
+
// Clear any cached information about the oat file that depends on the
// contents of the file.
void ClearOatFileCache();
diff --git a/runtime/oat_file_assistant_test.cc b/runtime/oat_file_assistant_test.cc
index 634e048..bddfa4f 100644
--- a/runtime/oat_file_assistant_test.cc
+++ b/runtime/oat_file_assistant_test.cc
@@ -230,6 +230,7 @@
&error_msg));
ASSERT_TRUE(odex_file.get() != nullptr) << error_msg;
EXPECT_FALSE(odex_file->IsPic());
+ EXPECT_TRUE(odex_file->HasPatchInfo());
EXPECT_EQ(filter, odex_file->GetCompilerFilter());
if (CompilerFilter::IsCompilationEnabled(filter)) {
@@ -277,6 +278,44 @@
EXPECT_EQ(filter, odex_file->GetCompilerFilter());
}
+ // Generate a non-PIC odex file without patch information for the purposes
+ // of test. The generated odex file will be un-relocated.
+ // TODO: This won't work correctly if we depend on the boot image being
+ // randomly relocated by a non-zero amount. We should have a better solution
+ // for avoiding that flakiness and duplicating code to generate odex and oat
+ // files for test.
+ void GenerateNoPatchOdexForTest(const std::string& dex_location,
+ const std::string& odex_location,
+ CompilerFilter::Filter filter) {
+ // Temporarily redirect the dalvik cache so dex2oat doesn't find the
+ // relocated image file.
+ std::string android_data_tmp = GetScratchDir() + "AndroidDataTmp";
+ setenv("ANDROID_DATA", android_data_tmp.c_str(), 1);
+ std::vector<std::string> args;
+ args.push_back("--dex-file=" + dex_location);
+ args.push_back("--oat-file=" + odex_location);
+ args.push_back("--compiler-filter=" + CompilerFilter::NameOfFilter(filter));
+ args.push_back("--runtime-arg");
+ args.push_back("-Xnorelocate");
+ std::string error_msg;
+ ASSERT_TRUE(OatFileAssistant::Dex2Oat(args, &error_msg)) << error_msg;
+ setenv("ANDROID_DATA", android_data_.c_str(), 1);
+
+ // Verify the odex file was generated as expected.
+ std::unique_ptr<OatFile> odex_file(OatFile::Open(odex_location.c_str(),
+ odex_location.c_str(),
+ nullptr,
+ nullptr,
+ false,
+ /*low_4gb*/false,
+ dex_location.c_str(),
+ &error_msg));
+ ASSERT_TRUE(odex_file.get() != nullptr) << error_msg;
+ EXPECT_FALSE(odex_file->IsPic());
+ EXPECT_FALSE(odex_file->HasPatchInfo());
+ EXPECT_EQ(filter, odex_file->GetCompilerFilter());
+ }
+
private:
// Reserve memory around where the image will be loaded so other memory
// won't conflict when it comes time to load the image.
@@ -413,7 +452,8 @@
// Trying to make the oat file up to date should not fail or crash.
std::string error_msg;
- EXPECT_TRUE(oat_file_assistant.MakeUpToDate(CompilerFilter::kSpeed, &error_msg));
+ EXPECT_EQ(OatFileAssistant::kUpdateSucceeded,
+ oat_file_assistant.MakeUpToDate(CompilerFilter::kSpeed, &error_msg));
// Trying to get the best oat file should fail, but not crash.
std::unique_ptr<OatFile> oat_file = oat_file_assistant.GetBestOatFile();
@@ -664,7 +704,8 @@
// Make the oat file up to date.
std::string error_msg;
- ASSERT_TRUE(oat_file_assistant.MakeUpToDate(CompilerFilter::kSpeed, &error_msg)) << error_msg;
+ ASSERT_EQ(OatFileAssistant::kUpdateSucceeded,
+ oat_file_assistant.MakeUpToDate(CompilerFilter::kSpeed, &error_msg)) << error_msg;
EXPECT_EQ(OatFileAssistant::kNoDexOptNeeded,
oat_file_assistant.GetDexOptNeeded(CompilerFilter::kSpeed));
@@ -726,7 +767,8 @@
// Make the oat file up to date.
std::string error_msg;
- ASSERT_TRUE(oat_file_assistant.MakeUpToDate(CompilerFilter::kSpeed, &error_msg)) << error_msg;
+ ASSERT_EQ(OatFileAssistant::kUpdateSucceeded,
+ oat_file_assistant.MakeUpToDate(CompilerFilter::kSpeed, &error_msg)) << error_msg;
EXPECT_EQ(OatFileAssistant::kNoDexOptNeeded,
oat_file_assistant.GetDexOptNeeded(CompilerFilter::kSpeed));
@@ -782,7 +824,8 @@
// Make the oat file up to date. This should have no effect.
std::string error_msg;
- EXPECT_TRUE(oat_file_assistant.MakeUpToDate(CompilerFilter::kSpeed, &error_msg)) << error_msg;
+ EXPECT_EQ(OatFileAssistant::kUpdateSucceeded,
+ oat_file_assistant.MakeUpToDate(CompilerFilter::kSpeed, &error_msg)) << error_msg;
EXPECT_EQ(OatFileAssistant::kNoDexOptNeeded,
oat_file_assistant.GetDexOptNeeded(CompilerFilter::kSpeed));
@@ -832,7 +875,8 @@
// Make the oat file up to date.
std::string error_msg;
- ASSERT_TRUE(oat_file_assistant.MakeUpToDate(CompilerFilter::kSpeed, &error_msg)) << error_msg;
+ ASSERT_EQ(OatFileAssistant::kUpdateSucceeded,
+ oat_file_assistant.MakeUpToDate(CompilerFilter::kSpeed, &error_msg)) << error_msg;
EXPECT_EQ(OatFileAssistant::kNoDexOptNeeded,
oat_file_assistant.GetDexOptNeeded(CompilerFilter::kSpeed));
@@ -856,6 +900,38 @@
EXPECT_EQ(1u, dex_files.size());
}
+// Case: We have a DEX file, no ODEX file and an OAT file that needs
+// relocation but doesn't have patch info.
+// Expect: The status is kDex2OatNeeded, because we can't run patchoat.
+TEST_F(OatFileAssistantTest, NoSelfRelocation) {
+ std::string dex_location = GetScratchDir() + "/NoSelfRelocation.jar";
+ std::string oat_location = GetOdexDir() + "/NoSelfRelocation.oat";
+
+ // Create the dex and odex files
+ Copy(GetDexSrc1(), dex_location);
+ GenerateNoPatchOdexForTest(dex_location, oat_location, CompilerFilter::kSpeed);
+
+ OatFileAssistant oat_file_assistant(dex_location.c_str(),
+ oat_location.c_str(), kRuntimeISA, false, true);
+
+ EXPECT_EQ(OatFileAssistant::kDex2OatNeeded,
+ oat_file_assistant.GetDexOptNeeded(CompilerFilter::kSpeed));
+
+ // Make the oat file up to date.
+ std::string error_msg;
+ ASSERT_EQ(OatFileAssistant::kUpdateSucceeded,
+ oat_file_assistant.MakeUpToDate(CompilerFilter::kSpeed, &error_msg)) << error_msg;
+ EXPECT_EQ(OatFileAssistant::kNoDexOptNeeded,
+ oat_file_assistant.GetDexOptNeeded(CompilerFilter::kSpeed));
+
+ std::unique_ptr<OatFile> oat_file = oat_file_assistant.GetBestOatFile();
+ ASSERT_TRUE(oat_file.get() != nullptr);
+ EXPECT_TRUE(oat_file->IsExecutable());
+ std::vector<std::unique_ptr<const DexFile>> dex_files;
+ dex_files = oat_file_assistant.LoadDexFiles(*oat_file, dex_location.c_str());
+ EXPECT_EQ(1u, dex_files.size());
+}
+
// Case: We have a DEX file, an ODEX file and an OAT file, where the ODEX and
// OAT files both have patch delta of 0.
// Expect: It shouldn't crash, and status is kPatchOatNeeded.
@@ -1023,7 +1099,8 @@
OatFileAssistant oat_file_assistant(
dex_location.c_str(), oat_location.c_str(), kRuntimeISA, false, true);
std::string error_msg;
- ASSERT_TRUE(oat_file_assistant.MakeUpToDate(CompilerFilter::kSpeed, &error_msg)) << error_msg;
+ ASSERT_EQ(OatFileAssistant::kUpdateSucceeded,
+ oat_file_assistant.MakeUpToDate(CompilerFilter::kSpeed, &error_msg)) << error_msg;
std::unique_ptr<OatFile> oat_file = oat_file_assistant.GetBestOatFile();
ASSERT_TRUE(oat_file.get() != nullptr);
@@ -1053,7 +1130,8 @@
OatFileAssistant oat_file_assistant(
dex_location.c_str(), oat_location.c_str(), kRuntimeISA, false, true);
std::string error_msg;
- ASSERT_FALSE(oat_file_assistant.MakeUpToDate(CompilerFilter::kSpeed, &error_msg));
+ ASSERT_EQ(OatFileAssistant::kUpdateNotAttempted,
+ oat_file_assistant.MakeUpToDate(CompilerFilter::kSpeed, &error_msg));
std::unique_ptr<OatFile> oat_file = oat_file_assistant.GetBestOatFile();
ASSERT_TRUE(oat_file.get() == nullptr);
@@ -1068,7 +1146,8 @@
OatFileAssistant oat_file_assistant(
dex_location.c_str(), oat_location.c_str(), kRuntimeISA, false, true);
std::string error_msg;
- ASSERT_FALSE(oat_file_assistant.GenerateOatFile(CompilerFilter::kSpeed, &error_msg));
+ EXPECT_EQ(OatFileAssistant::kUpdateNotAttempted,
+ oat_file_assistant.GenerateOatFile(CompilerFilter::kSpeed, &error_msg));
}
// Turn an absolute path into a path relative to the current working
@@ -1147,7 +1226,8 @@
// Trying to make it up to date should have no effect.
std::string error_msg;
- EXPECT_TRUE(oat_file_assistant.MakeUpToDate(CompilerFilter::kSpeed, &error_msg));
+ EXPECT_EQ(OatFileAssistant::kUpdateSucceeded,
+ oat_file_assistant.MakeUpToDate(CompilerFilter::kSpeed, &error_msg));
EXPECT_TRUE(error_msg.empty());
}
diff --git a/runtime/oat_file_manager.cc b/runtime/oat_file_manager.cc
index 2f13f55..94f6345 100644
--- a/runtime/oat_file_manager.cc
+++ b/runtime/oat_file_manager.cc
@@ -329,8 +329,20 @@
// Update the oat file on disk if we can. This may fail, but that's okay.
// Best effort is all that matters here.
- if (!oat_file_assistant.MakeUpToDate(filter_, /*out*/ &error_msg)) {
- LOG(INFO) << error_msg;
+ switch (oat_file_assistant.MakeUpToDate(filter_, /*out*/ &error_msg)) {
+ case OatFileAssistant::kUpdateFailed:
+ LOG(WARNING) << error_msg;
+ break;
+
+ case OatFileAssistant::kUpdateNotAttempted:
+ // Avoid spamming the logs if we decided not to attempt making the oat
+ // file up to date.
+ VLOG(oat) << error_msg;
+ break;
+
+ case OatFileAssistant::kUpdateSucceeded:
+ // Nothing to do.
+ break;
}
// Get the oat file on disk.
diff --git a/runtime/quick_exception_handler.cc b/runtime/quick_exception_handler.cc
index b9cd677..a785ecb 100644
--- a/runtime/quick_exception_handler.cc
+++ b/runtime/quick_exception_handler.cc
@@ -342,11 +342,7 @@
updated_vregs = GetThread()->GetUpdatedVRegFlags(frame_id);
DCHECK(updated_vregs != nullptr);
}
- if (GetCurrentOatQuickMethodHeader()->IsOptimized()) {
- HandleOptimizingDeoptimization(method, new_frame, updated_vregs);
- } else {
- HandleQuickDeoptimization(method, new_frame, updated_vregs);
- }
+ HandleOptimizingDeoptimization(method, new_frame, updated_vregs);
if (updated_vregs != nullptr) {
// Calling Thread::RemoveDebuggerShadowFrameMapping will also delete the updated_vregs
// array so this must come after we processed the frame.
@@ -475,132 +471,6 @@
return static_cast<VRegKind>(kinds.at(reg * 2));
}
- void HandleQuickDeoptimization(ArtMethod* m,
- ShadowFrame* new_frame,
- const bool* updated_vregs)
- SHARED_REQUIRES(Locks::mutator_lock_) {
- const DexFile::CodeItem* code_item = m->GetCodeItem();
- CHECK(code_item != nullptr) << "No code item for " << PrettyMethod(m);
- uint16_t num_regs = code_item->registers_size_;
- uint32_t dex_pc = GetDexPc();
- StackHandleScope<2> hs(GetThread()); // Dex cache and class loader.
- mirror::Class* declaring_class = m->GetDeclaringClass();
- Handle<mirror::DexCache> h_dex_cache(hs.NewHandle(declaring_class->GetDexCache()));
- Handle<mirror::ClassLoader> h_class_loader(hs.NewHandle(declaring_class->GetClassLoader()));
- verifier::MethodVerifier verifier(GetThread(), h_dex_cache->GetDexFile(), h_dex_cache,
- h_class_loader, &m->GetClassDef(), code_item,
- m->GetDexMethodIndex(), m, m->GetAccessFlags(), true, true,
- true, true);
- bool verifier_success = verifier.Verify();
- CHECK(verifier_success) << PrettyMethod(m);
- {
- ScopedStackedShadowFramePusher pusher(GetThread(), new_frame,
- StackedShadowFrameType::kShadowFrameUnderConstruction);
- const std::vector<int32_t> kinds(verifier.DescribeVRegs(dex_pc));
-
- // Markers for dead values, used when the verifier knows a Dex register is undefined,
- // or when the compiler knows the register has not been initialized, or is not used
- // anymore in the method.
- static constexpr uint32_t kDeadValue = 0xEBADDE09;
- static constexpr uint64_t kLongDeadValue = 0xEBADDE09EBADDE09;
- for (uint16_t reg = 0; reg < num_regs; ++reg) {
- if (updated_vregs != nullptr && updated_vregs[reg]) {
- // Keep the value set by debugger.
- continue;
- }
- VRegKind kind = GetVRegKind(reg, kinds);
- switch (kind) {
- case kUndefined:
- new_frame->SetVReg(reg, kDeadValue);
- break;
- case kConstant:
- new_frame->SetVReg(reg, kinds.at((reg * 2) + 1));
- break;
- case kReferenceVReg: {
- uint32_t value = 0;
- // Check IsReferenceVReg in case the compiled GC map doesn't agree with the verifier.
- // We don't want to copy a stale reference into the shadow frame as a reference.
- // b/20736048
- if (GetVReg(m, reg, kind, &value) && IsReferenceVReg(m, reg)) {
- new_frame->SetVRegReference(reg, reinterpret_cast<mirror::Object*>(value));
- } else {
- new_frame->SetVReg(reg, kDeadValue);
- }
- break;
- }
- case kLongLoVReg:
- if (GetVRegKind(reg + 1, kinds) == kLongHiVReg) {
- // Treat it as a "long" register pair.
- uint64_t value = 0;
- if (GetVRegPair(m, reg, kLongLoVReg, kLongHiVReg, &value)) {
- new_frame->SetVRegLong(reg, value);
- } else {
- new_frame->SetVRegLong(reg, kLongDeadValue);
- }
- } else {
- uint32_t value = 0;
- if (GetVReg(m, reg, kind, &value)) {
- new_frame->SetVReg(reg, value);
- } else {
- new_frame->SetVReg(reg, kDeadValue);
- }
- }
- break;
- case kLongHiVReg:
- if (GetVRegKind(reg - 1, kinds) == kLongLoVReg) {
- // Nothing to do: we treated it as a "long" register pair.
- } else {
- uint32_t value = 0;
- if (GetVReg(m, reg, kind, &value)) {
- new_frame->SetVReg(reg, value);
- } else {
- new_frame->SetVReg(reg, kDeadValue);
- }
- }
- break;
- case kDoubleLoVReg:
- if (GetVRegKind(reg + 1, kinds) == kDoubleHiVReg) {
- uint64_t value = 0;
- if (GetVRegPair(m, reg, kDoubleLoVReg, kDoubleHiVReg, &value)) {
- // Treat it as a "double" register pair.
- new_frame->SetVRegLong(reg, value);
- } else {
- new_frame->SetVRegLong(reg, kLongDeadValue);
- }
- } else {
- uint32_t value = 0;
- if (GetVReg(m, reg, kind, &value)) {
- new_frame->SetVReg(reg, value);
- } else {
- new_frame->SetVReg(reg, kDeadValue);
- }
- }
- break;
- case kDoubleHiVReg:
- if (GetVRegKind(reg - 1, kinds) == kDoubleLoVReg) {
- // Nothing to do: we treated it as a "double" register pair.
- } else {
- uint32_t value = 0;
- if (GetVReg(m, reg, kind, &value)) {
- new_frame->SetVReg(reg, value);
- } else {
- new_frame->SetVReg(reg, kDeadValue);
- }
- }
- break;
- default:
- uint32_t value = 0;
- if (GetVReg(m, reg, kind, &value)) {
- new_frame->SetVReg(reg, value);
- } else {
- new_frame->SetVReg(reg, kDeadValue);
- }
- break;
- }
- }
- }
- }
-
QuickExceptionHandler* const exception_handler_;
ShadowFrame* prev_shadow_frame_;
bool stacked_shadow_frame_pushed_;
diff --git a/runtime/runtime.cc b/runtime/runtime.cc
index 8a38f3a..630d101 100644
--- a/runtime/runtime.cc
+++ b/runtime/runtime.cc
@@ -554,19 +554,6 @@
started_ = true;
- // Use !IsAotCompiler so that we get test coverage, tests are never the zygote.
- if (!IsAotCompiler()) {
- ScopedObjectAccess soa(self);
- {
- ScopedTrace trace2("AddImageStringsToTable");
- GetInternTable()->AddImagesStringsToTable(heap_->GetBootImageSpaces());
- }
- {
- ScopedTrace trace2("MoveImageClassesToClassTable");
- GetClassLinker()->AddBootImageClassesToClassTable();
- }
- }
-
if (jit_options_->UseJIT()) {
std::string error_msg;
if (!IsZygote()) {
@@ -1140,6 +1127,14 @@
}
boot_class_path_string_ = Join(dex_locations, ':');
}
+ {
+ ScopedTrace trace2("AddImageStringsToTable");
+ GetInternTable()->AddImagesStringsToTable(heap_->GetBootImageSpaces());
+ }
+ {
+ ScopedTrace trace2("MoveImageClassesToClassTable");
+ GetClassLinker()->AddBootImageClassesToClassTable();
+ }
} else {
std::vector<std::string> dex_filenames;
Split(boot_class_path_string_, ':', &dex_filenames);
diff --git a/runtime/safe_map.h b/runtime/safe_map.h
index 0e5b503..49f80f3 100644
--- a/runtime/safe_map.h
+++ b/runtime/safe_map.h
@@ -19,6 +19,7 @@
#include <map>
#include <memory>
+#include <type_traits>
#include "base/allocator.h"
#include "base/logging.h"
@@ -124,6 +125,18 @@
return result.first;
}
+ template <typename CreateFn>
+ V GetOrCreate(const K& k, CreateFn create) {
+ static_assert(std::is_same<V, typename std::result_of<CreateFn()>::type>::value,
+ "Argument `create` should return a value of type V.");
+ auto lb = lower_bound(k);
+ if (lb != end() && !key_comp()(k, lb->first)) {
+ return lb->second;
+ }
+ auto it = PutBefore(lb, k, create());
+ return it->second;
+ }
+
bool Equals(const Self& rhs) const {
return map_ == rhs.map_;
}
diff --git a/runtime/stack.cc b/runtime/stack.cc
index 838678b..2336365 100644
--- a/runtime/stack.cc
+++ b/runtime/stack.cc
@@ -130,7 +130,11 @@
if (IsInInlinedFrame()) {
size_t depth_in_stack_map = current_inlining_depth_ - 1;
InlineInfo inline_info = GetCurrentInlineInfo();
- return GetResolvedMethod(*GetCurrentQuickFrame(), inline_info, depth_in_stack_map);
+ DCHECK(walk_kind_ != StackWalkKind::kSkipInlinedFrames);
+ bool allow_resolve = walk_kind_ != StackWalkKind::kIncludeInlinedFramesNoResolve;
+ return allow_resolve
+ ? GetResolvedMethod<true>(*GetCurrentQuickFrame(), inline_info, depth_in_stack_map)
+ : GetResolvedMethod<false>(*GetCurrentQuickFrame(), inline_info, depth_in_stack_map);
} else {
return *cur_quick_frame_;
}
@@ -859,7 +863,8 @@
cur_oat_quick_method_header_ = method->GetOatQuickMethodHeader(cur_quick_frame_pc_);
SanityCheckFrame();
- if ((walk_kind_ == StackWalkKind::kIncludeInlinedFrames)
+ if ((walk_kind_ == StackWalkKind::kIncludeInlinedFrames ||
+ walk_kind_ == StackWalkKind::kIncludeInlinedFramesNoResolve)
&& (cur_oat_quick_method_header_ != nullptr)
&& cur_oat_quick_method_header_->IsOptimized()) {
CodeInfo code_info = cur_oat_quick_method_header_->GetOptimizedCodeInfo();
diff --git a/runtime/stack.h b/runtime/stack.h
index 4fa1a4f..a25874e 100644
--- a/runtime/stack.h
+++ b/runtime/stack.h
@@ -187,6 +187,22 @@
return (dex_pc_ptr_ == nullptr) ? dex_pc_ : dex_pc_ptr_ - code_item_->insns_;
}
+ int16_t GetCachedHotnessCountdown() const {
+ return cached_hotness_countdown_;
+ }
+
+ void SetCachedHotnessCountdown(int16_t cached_hotness_countdown) {
+ cached_hotness_countdown_ = cached_hotness_countdown;
+ }
+
+ int16_t GetHotnessCountdown() const {
+ return hotness_countdown_;
+ }
+
+ void SetHotnessCountdown(int16_t hotness_countdown) {
+ hotness_countdown_ = hotness_countdown;
+ }
+
void SetDexPC(uint32_t dex_pc) {
dex_pc_ = dex_pc;
dex_pc_ptr_ = nullptr;
@@ -397,6 +413,14 @@
return OFFSETOF_MEMBER(ShadowFrame, code_item_);
}
+ static size_t CachedHotnessCountdownOffset() {
+ return OFFSETOF_MEMBER(ShadowFrame, cached_hotness_countdown_);
+ }
+
+ static size_t HotnessCountdownOffset() {
+ return OFFSETOF_MEMBER(ShadowFrame, hotness_countdown_);
+ }
+
// Create ShadowFrame for interpreter using provided memory.
static ShadowFrame* CreateShadowFrameImpl(uint32_t num_vregs,
ShadowFrame* link,
@@ -406,7 +430,7 @@
return new (memory) ShadowFrame(num_vregs, link, method, dex_pc, true);
}
- uint16_t* GetDexPCPtr() {
+ const uint16_t* GetDexPCPtr() {
return dex_pc_ptr_;
}
@@ -443,11 +467,13 @@
ShadowFrame* link_;
ArtMethod* method_;
JValue* result_register_;
- uint16_t* dex_pc_ptr_;
+ const uint16_t* dex_pc_ptr_;
const DexFile::CodeItem* code_item_;
LockCountData lock_count_data_; // This may contain GC roots when lock counting is active.
const uint32_t number_of_vregs_;
uint32_t dex_pc_;
+ int16_t cached_hotness_countdown_;
+ int16_t hotness_countdown_;
// This is a two-part array:
// - [0..number_of_vregs) holds the raw virtual registers, and each element here is always 4
@@ -569,6 +595,7 @@
// when walking the stack.
enum class StackWalkKind {
kIncludeInlinedFrames,
+ kIncludeInlinedFramesNoResolve,
kSkipInlinedFrames,
};
diff --git a/runtime/thread.cc b/runtime/thread.cc
index 8cc94e8..3ecb041 100644
--- a/runtime/thread.cc
+++ b/runtime/thread.cc
@@ -70,7 +70,6 @@
#include "thread_list.h"
#include "thread-inl.h"
#include "utils.h"
-#include "verifier/dex_gc_map.h"
#include "verifier/method_verifier.h"
#include "verify_object-inl.h"
#include "vmap_table.h"
@@ -91,8 +90,7 @@
pthread_key_t Thread::pthread_key_self_;
ConditionVariable* Thread::resume_cond_ = nullptr;
const size_t Thread::kStackOverflowImplicitCheckSize = GetStackOverflowReservedBytes(kRuntimeISA);
-// Enabled for b/27493510. TODO: disable when fixed.
-static constexpr bool kVerifyImageObjectsMarked = true;
+static constexpr bool kVerifyImageObjectsMarked = kIsDebugBuild;
// For implicit overflow checks we reserve an extra piece of memory at the bottom
// of the stack (lowest memory). The higher portion of the memory
diff --git a/runtime/utf.cc b/runtime/utf.cc
index a2d6363..5e9fdf7 100644
--- a/runtime/utf.cc
+++ b/runtime/utf.cc
@@ -178,6 +178,23 @@
return static_cast<int32_t>(hash);
}
+int32_t ComputeUtf16HashFromModifiedUtf8(const char* utf8, size_t utf16_length) {
+ uint32_t hash = 0;
+ while (utf16_length != 0u) {
+ const uint32_t pair = GetUtf16FromUtf8(&utf8);
+ const uint16_t first = GetLeadingUtf16Char(pair);
+ hash = hash * 31 + first;
+ --utf16_length;
+ const uint16_t second = GetTrailingUtf16Char(pair);
+ if (second != 0) {
+ hash = hash * 31 + second;
+ DCHECK_NE(utf16_length, 0u);
+ --utf16_length;
+ }
+ }
+ return static_cast<int32_t>(hash);
+}
+
uint32_t ComputeModifiedUtf8Hash(const char* chars) {
uint32_t hash = 0;
while (*chars != '\0') {
diff --git a/runtime/utf.h b/runtime/utf.h
index 4abd605..27d2fd5 100644
--- a/runtime/utf.h
+++ b/runtime/utf.h
@@ -83,6 +83,7 @@
int32_t ComputeUtf16Hash(mirror::CharArray* chars, int32_t offset, size_t char_count)
SHARED_REQUIRES(Locks::mutator_lock_);
int32_t ComputeUtf16Hash(const uint16_t* chars, size_t char_count);
+int32_t ComputeUtf16HashFromModifiedUtf8(const char* utf8, size_t utf16_length);
// Compute a hash code of a modified UTF-8 string. Not the standard java hash since it returns a
// uint32_t and hashes individual chars instead of codepoint words.
diff --git a/runtime/verifier/dex_gc_map.cc b/runtime/verifier/dex_gc_map.cc
deleted file mode 100644
index c435f9f..0000000
--- a/runtime/verifier/dex_gc_map.cc
+++ /dev/null
@@ -1,56 +0,0 @@
-/*
- * Copyright (C) 2012 The Android Open Source Project
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#include "verifier/dex_gc_map.h"
-
-#include "base/logging.h"
-
-namespace art {
-namespace verifier {
-
-const uint8_t* DexPcToReferenceMap::FindBitMap(uint16_t dex_pc, bool error_if_not_present) const {
- size_t num_entries = NumEntries();
- // Do linear or binary search?
- static const size_t kSearchThreshold = 8;
- if (num_entries < kSearchThreshold) {
- for (size_t i = 0; i < num_entries; i++) {
- if (GetDexPc(i) == dex_pc) {
- return GetBitMap(i);
- }
- }
- } else {
- int lo = 0;
- int hi = num_entries -1;
- while (hi >= lo) {
- int mid = (hi + lo) / 2;
- int mid_pc = GetDexPc(mid);
- if (dex_pc > mid_pc) {
- lo = mid + 1;
- } else if (dex_pc < mid_pc) {
- hi = mid - 1;
- } else {
- return GetBitMap(mid);
- }
- }
- }
- if (error_if_not_present) {
- LOG(ERROR) << "Didn't find reference bit map for dex_pc " << dex_pc;
- }
- return nullptr;
-}
-
-} // namespace verifier
-} // namespace art
diff --git a/runtime/verifier/dex_gc_map.h b/runtime/verifier/dex_gc_map.h
deleted file mode 100644
index 03a7821..0000000
--- a/runtime/verifier/dex_gc_map.h
+++ /dev/null
@@ -1,122 +0,0 @@
-/*
- * Copyright (C) 2012 The Android Open Source Project
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#ifndef ART_RUNTIME_VERIFIER_DEX_GC_MAP_H_
-#define ART_RUNTIME_VERIFIER_DEX_GC_MAP_H_
-
-#include <stdint.h>
-
-#include "base/logging.h"
-#include "base/macros.h"
-
-namespace art {
-namespace verifier {
-
-/*
- * Format enumeration for RegisterMap data area.
- */
-enum RegisterMapFormat {
- kRegMapFormatUnknown = 0,
- kRegMapFormatNone = 1, // Indicates no map data follows.
- kRegMapFormatCompact8 = 2, // Compact layout, 8-bit addresses.
- kRegMapFormatCompact16 = 3, // Compact layout, 16-bit addresses.
-};
-
-// Lightweight wrapper for Dex PC to reference bit maps.
-class DexPcToReferenceMap {
- public:
- explicit DexPcToReferenceMap(const uint8_t* data) : data_(data) {
- CHECK(data_ != nullptr);
- }
-
- // The total size of the reference bit map including header.
- size_t RawSize() const {
- return EntryWidth() * NumEntries() + 4u /* header */;
- }
-
- // The number of entries in the table
- size_t NumEntries() const {
- return GetData()[2] | (GetData()[3] << 8);
- }
-
- // Get the Dex PC at the given index
- uint16_t GetDexPc(size_t index) const {
- size_t entry_offset = index * EntryWidth();
- if (DexPcWidth() == 1) {
- return Table()[entry_offset];
- } else {
- return Table()[entry_offset] | (Table()[entry_offset + 1] << 8);
- }
- }
-
- // Return address of bitmap encoding what are live references
- const uint8_t* GetBitMap(size_t index) const {
- size_t entry_offset = index * EntryWidth();
- return &Table()[entry_offset + DexPcWidth()];
- }
-
- // Find the bitmap associated with the given dex pc
- const uint8_t* FindBitMap(uint16_t dex_pc, bool error_if_not_present = true) const;
-
- // The number of bytes used to encode registers
- size_t RegWidth() const {
- return GetData()[1] | ((GetData()[0] & ~kRegMapFormatMask) << kRegMapFormatShift);
- }
-
- private:
- // Table of num_entries * (dex pc, bitmap)
- const uint8_t* Table() const {
- return GetData() + 4;
- }
-
- // The format of the table of the PCs for the table
- RegisterMapFormat Format() const {
- return static_cast<RegisterMapFormat>(GetData()[0] & kRegMapFormatMask);
- }
-
- // Number of bytes used to encode a dex pc
- size_t DexPcWidth() const {
- RegisterMapFormat format = Format();
- switch (format) {
- case kRegMapFormatCompact8:
- return 1;
- case kRegMapFormatCompact16:
- return 2;
- default:
- LOG(FATAL) << "Invalid format " << static_cast<int>(format);
- return -1;
- }
- }
-
- // The width of an entry in the table
- size_t EntryWidth() const {
- return DexPcWidth() + RegWidth();
- }
-
- const uint8_t* GetData() const {
- return data_;
- }
-
- static const int kRegMapFormatShift = 5;
- static const uint8_t kRegMapFormatMask = 0x7;
-
- const uint8_t* const data_; // The header and table data
-};
-
-} // namespace verifier
-} // namespace art
-
-#endif // ART_RUNTIME_VERIFIER_DEX_GC_MAP_H_
diff --git a/runtime/verifier/method_verifier.cc b/runtime/verifier/method_verifier.cc
index 55d1720..83da6b7 100644
--- a/runtime/verifier/method_verifier.cc
+++ b/runtime/verifier/method_verifier.cc
@@ -47,7 +47,6 @@
#include "scoped_thread_state_change.h"
#include "utils.h"
#include "handle_scope-inl.h"
-#include "verifier/dex_gc_map.h"
namespace art {
namespace verifier {
@@ -123,7 +122,7 @@
mirror::Class* klass,
CompilerCallbacks* callbacks,
bool allow_soft_failures,
- bool log_hard_failures,
+ LogSeverity log_level,
std::string* error) {
if (klass->IsVerified()) {
return kNoFailure;
@@ -162,7 +161,7 @@
class_def,
callbacks,
allow_soft_failures,
- log_hard_failures,
+ log_level,
error);
}
@@ -196,7 +195,7 @@
Handle<mirror::ClassLoader> class_loader,
CompilerCallbacks* callbacks,
bool allow_soft_failures,
- bool log_hard_failures,
+ LogSeverity log_level,
bool need_precise_constants,
std::string* error_string) {
DCHECK(it != nullptr);
@@ -237,7 +236,7 @@
it->GetMethodAccessFlags(),
callbacks,
allow_soft_failures,
- log_hard_failures,
+ log_level,
need_precise_constants,
&hard_failure_msg);
if (result.kind == kHardFailure) {
@@ -267,7 +266,7 @@
const DexFile::ClassDef* class_def,
CompilerCallbacks* callbacks,
bool allow_soft_failures,
- bool log_hard_failures,
+ LogSeverity log_level,
std::string* error) {
DCHECK(class_def != nullptr);
ScopedTrace trace(__FUNCTION__);
@@ -300,7 +299,7 @@
class_loader,
callbacks,
allow_soft_failures,
- log_hard_failures,
+ log_level,
false /* need precise constants */,
error);
// Virtual methods.
@@ -313,7 +312,7 @@
class_loader,
callbacks,
allow_soft_failures,
- log_hard_failures,
+ log_level,
false /* need precise constants */,
error);
@@ -361,15 +360,26 @@
uint32_t method_access_flags,
CompilerCallbacks* callbacks,
bool allow_soft_failures,
- bool log_hard_failures,
+ LogSeverity log_level,
bool need_precise_constants,
std::string* hard_failure_msg) {
MethodVerifier::FailureData result;
uint64_t start_ns = kTimeVerifyMethod ? NanoTime() : 0;
- MethodVerifier verifier(self, dex_file, dex_cache, class_loader, class_def, code_item,
- method_idx, method, method_access_flags, true, allow_soft_failures,
- need_precise_constants, true);
+ MethodVerifier verifier(self,
+ dex_file,
+ dex_cache,
+ class_loader,
+ class_def,
+ code_item,
+ method_idx,
+ method,
+ method_access_flags,
+ true /* can_load_classes */,
+ allow_soft_failures,
+ need_precise_constants,
+ false /* verify to dump */,
+ true /* allow_thread_suspension */);
if (verifier.Verify()) {
// Verification completed, however failures may be pending that didn't cause the verification
// to hard fail.
@@ -397,9 +407,12 @@
result.kind = kSoftFailure;
} else {
CHECK(verifier.have_pending_hard_failure_);
- if (VLOG_IS_ON(verifier) || log_hard_failures) {
- verifier.DumpFailures(LOG(INFO) << "Verification error in "
- << PrettyMethod(method_idx, *dex_file) << "\n");
+ if (VLOG_IS_ON(verifier)) {
+ log_level = LogSeverity::VERBOSE;
+ }
+ if (log_level > LogSeverity::VERBOSE) {
+ verifier.DumpFailures(LOG(log_level) << "Verification error in "
+ << PrettyMethod(method_idx, *dex_file) << "\n");
}
if (hard_failure_msg != nullptr) {
CHECK(!verifier.failure_messages_.empty());
@@ -441,9 +454,20 @@
const DexFile::CodeItem* code_item,
ArtMethod* method,
uint32_t method_access_flags) {
- MethodVerifier* verifier = new MethodVerifier(self, dex_file, dex_cache, class_loader,
- class_def, code_item, dex_method_idx, method,
- method_access_flags, true, true, true, true);
+ MethodVerifier* verifier = new MethodVerifier(self,
+ dex_file,
+ dex_cache,
+ class_loader,
+ class_def,
+ code_item,
+ dex_method_idx,
+ method,
+ method_access_flags,
+ true /* can_load_classes */,
+ true /* allow_soft_failures */,
+ true /* need_precise_constants */,
+ true /* verify_to_dump */,
+ true /* allow_thread_suspension */);
verifier->Verify();
verifier->DumpFailures(vios->Stream());
vios->Stream() << verifier->info_messages_.str();
@@ -520,9 +544,20 @@
StackHandleScope<2> hs(Thread::Current());
Handle<mirror::DexCache> dex_cache(hs.NewHandle(m->GetDexCache()));
Handle<mirror::ClassLoader> class_loader(hs.NewHandle(m->GetClassLoader()));
- MethodVerifier verifier(hs.Self(), m->GetDexFile(), dex_cache, class_loader, &m->GetClassDef(),
- m->GetCodeItem(), m->GetDexMethodIndex(), m, m->GetAccessFlags(),
- false, true, false, false);
+ MethodVerifier verifier(hs.Self(),
+ m->GetDexFile(),
+ dex_cache,
+ class_loader,
+ &m->GetClassDef(),
+ m->GetCodeItem(),
+ m->GetDexMethodIndex(),
+ m,
+ m->GetAccessFlags(),
+ false /* can_load_classes */,
+ true /* allow_soft_failures */,
+ false /* need_precise_constants */,
+ false /* verify_to_dump */,
+ false /* allow_thread_suspension */);
verifier.interesting_dex_pc_ = dex_pc;
verifier.monitor_enter_dex_pcs_ = monitor_enter_dex_pcs;
verifier.FindLocksAtDexPc();
@@ -564,9 +599,20 @@
StackHandleScope<2> hs(Thread::Current());
Handle<mirror::DexCache> dex_cache(hs.NewHandle(m->GetDexCache()));
Handle<mirror::ClassLoader> class_loader(hs.NewHandle(m->GetClassLoader()));
- MethodVerifier verifier(hs.Self(), m->GetDexFile(), dex_cache, class_loader, &m->GetClassDef(),
- m->GetCodeItem(), m->GetDexMethodIndex(), m, m->GetAccessFlags(), true,
- true, false, true);
+ MethodVerifier verifier(hs.Self(),
+ m->GetDexFile(),
+ dex_cache,
+ class_loader,
+ &m->GetClassDef(),
+ m->GetCodeItem(),
+ m->GetDexMethodIndex(),
+ m,
+ m->GetAccessFlags(),
+ true /* can_load_classes */,
+ true /* allow_soft_failures */,
+ false /* need_precise_constants */,
+ false /* verify_to_dump */,
+ true /* allow_thread_suspension */);
return verifier.FindAccessedFieldAtDexPc(dex_pc);
}
@@ -593,9 +639,20 @@
StackHandleScope<2> hs(Thread::Current());
Handle<mirror::DexCache> dex_cache(hs.NewHandle(m->GetDexCache()));
Handle<mirror::ClassLoader> class_loader(hs.NewHandle(m->GetClassLoader()));
- MethodVerifier verifier(hs.Self(), m->GetDexFile(), dex_cache, class_loader, &m->GetClassDef(),
- m->GetCodeItem(), m->GetDexMethodIndex(), m, m->GetAccessFlags(), true,
- true, false, true);
+ MethodVerifier verifier(hs.Self(),
+ m->GetDexFile(),
+ dex_cache,
+ class_loader,
+ &m->GetClassDef(),
+ m->GetCodeItem(),
+ m->GetDexMethodIndex(),
+ m,
+ m->GetAccessFlags(),
+ true /* can_load_classes */,
+ true /* allow_soft_failures */,
+ false /* need_precise_constants */,
+ false /* verify_to_dump */,
+ true /* allow_thread_suspension */);
return verifier.FindInvokedMethodAtDexPc(dex_pc);
}
@@ -667,13 +724,15 @@
// If there aren't any instructions, make sure that's expected, then exit successfully.
if (code_item_ == nullptr) {
+ // Only native or abstract methods may not have code.
+ if ((method_access_flags_ & (kAccNative | kAccAbstract)) == 0) {
+ Fail(VERIFY_ERROR_BAD_CLASS_HARD) << "zero-length code in concrete non-native method";
+ return false;
+ }
+
// This should have been rejected by the dex file verifier. Only do in debug build.
+ // Note: the above will also be rejected in the dex file verifier, starting in dex version 37.
if (kIsDebugBuild) {
- // Only native or abstract methods may not have code.
- if ((method_access_flags_ & (kAccNative | kAccAbstract)) == 0) {
- Fail(VERIFY_ERROR_BAD_CLASS_HARD) << "zero-length code in concrete non-native method";
- return false;
- }
if ((method_access_flags_ & kAccAbstract) != 0) {
// Abstract methods are not allowed to have the following flags.
static constexpr uint32_t kForbidden =
diff --git a/runtime/verifier/method_verifier.h b/runtime/verifier/method_verifier.h
index 6d8e1ab..ebb0b8c 100644
--- a/runtime/verifier/method_verifier.h
+++ b/runtime/verifier/method_verifier.h
@@ -43,7 +43,6 @@
namespace verifier {
-class DexPcToReferenceMap;
class MethodVerifier;
class RegisterLine;
using RegisterLineArenaUniquePtr = std::unique_ptr<RegisterLine, RegisterLineArenaDelete>;
@@ -143,12 +142,12 @@
kHardFailure,
};
- /* Verify a class. Returns "kNoFailure" on success. */
+ // Verify a class. Returns "kNoFailure" on success.
static FailureKind VerifyClass(Thread* self,
mirror::Class* klass,
CompilerCallbacks* callbacks,
bool allow_soft_failures,
- bool log_hard_failures,
+ LogSeverity log_level,
std::string* error)
SHARED_REQUIRES(Locks::mutator_lock_);
static FailureKind VerifyClass(Thread* self,
@@ -158,7 +157,7 @@
const DexFile::ClassDef* class_def,
CompilerCallbacks* callbacks,
bool allow_soft_failures,
- bool log_hard_failures,
+ LogSeverity log_level,
std::string* error)
SHARED_REQUIRES(Locks::mutator_lock_);
@@ -220,35 +219,6 @@
return can_load_classes_;
}
- MethodVerifier(Thread* self,
- const DexFile* dex_file,
- Handle<mirror::DexCache> dex_cache,
- Handle<mirror::ClassLoader> class_loader,
- const DexFile::ClassDef* class_def,
- const DexFile::CodeItem* code_item,
- uint32_t method_idx,
- ArtMethod* method,
- uint32_t access_flags,
- bool can_load_classes,
- bool allow_soft_failures,
- bool need_precise_constants,
- bool allow_thread_suspension)
- SHARED_REQUIRES(Locks::mutator_lock_)
- : MethodVerifier(self,
- dex_file,
- dex_cache,
- class_loader,
- class_def,
- code_item,
- method_idx,
- method,
- access_flags,
- can_load_classes,
- allow_soft_failures,
- need_precise_constants,
- false,
- allow_thread_suspension) {}
-
~MethodVerifier();
// Run verification on the method. Returns true if verification completes and false if the input
@@ -304,20 +274,6 @@
}
private:
- void UninstantiableError(const char* descriptor);
- static bool IsInstantiableOrPrimitive(mirror::Class* klass) SHARED_REQUIRES(Locks::mutator_lock_);
-
- // Is the method being verified a constructor? See the comment on the field.
- bool IsConstructor() const {
- return is_constructor_;
- }
-
- // Is the method verified static?
- bool IsStatic() const {
- return (method_access_flags_ & kAccStatic) != 0;
- }
-
- // Private constructor for dumping.
MethodVerifier(Thread* self,
const DexFile* dex_file,
Handle<mirror::DexCache> dex_cache,
@@ -334,6 +290,19 @@
bool allow_thread_suspension)
SHARED_REQUIRES(Locks::mutator_lock_);
+ void UninstantiableError(const char* descriptor);
+ static bool IsInstantiableOrPrimitive(mirror::Class* klass) SHARED_REQUIRES(Locks::mutator_lock_);
+
+ // Is the method being verified a constructor? See the comment on the field.
+ bool IsConstructor() const {
+ return is_constructor_;
+ }
+
+ // Is the method verified static?
+ bool IsStatic() const {
+ return (method_access_flags_ & kAccStatic) != 0;
+ }
+
// Adds the given string to the beginning of the last failure message.
void PrependToLastFailMessage(std::string);
@@ -362,7 +331,7 @@
Handle<mirror::ClassLoader> class_loader,
CompilerCallbacks* callbacks,
bool allow_soft_failures,
- bool log_hard_failures,
+ LogSeverity log_level,
bool need_precise_constants,
std::string* error_string)
SHARED_REQUIRES(Locks::mutator_lock_);
@@ -388,7 +357,7 @@
uint32_t method_access_flags,
CompilerCallbacks* callbacks,
bool allow_soft_failures,
- bool log_hard_failures,
+ LogSeverity log_level,
bool need_precise_constants,
std::string* hard_failure_msg)
SHARED_REQUIRES(Locks::mutator_lock_);
diff --git a/runtime/verifier/method_verifier_test.cc b/runtime/verifier/method_verifier_test.cc
index 946f842..b036313 100644
--- a/runtime/verifier/method_verifier_test.cc
+++ b/runtime/verifier/method_verifier_test.cc
@@ -37,8 +37,13 @@
// Verify the class
std::string error_msg;
- ASSERT_TRUE(MethodVerifier::VerifyClass(self, klass, nullptr, true, true, &error_msg)
- == MethodVerifier::kNoFailure) << error_msg;
+ MethodVerifier::FailureKind failure = MethodVerifier::VerifyClass(self,
+ klass,
+ nullptr,
+ true,
+ LogSeverity::WARNING,
+ &error_msg);
+ ASSERT_TRUE(failure == MethodVerifier::kNoFailure) << error_msg;
}
void VerifyDexFile(const DexFile& dex)
diff --git a/runtime/verifier/register_line-inl.h b/runtime/verifier/register_line-inl.h
index 29d87c4..d2f3485 100644
--- a/runtime/verifier/register_line-inl.h
+++ b/runtime/verifier/register_line-inl.h
@@ -127,17 +127,6 @@
}
}
-inline size_t RegisterLine::GetMaxNonZeroReferenceReg(MethodVerifier* verifier,
- size_t max_ref_reg) const {
- size_t i = static_cast<int>(max_ref_reg) < 0 ? 0 : max_ref_reg;
- for (; i < num_regs_; i++) {
- if (GetRegisterType(verifier, i).IsNonZeroReferenceTypes()) {
- max_ref_reg = i;
- }
- }
- return max_ref_reg;
-}
-
inline bool RegisterLine::VerifyRegisterType(MethodVerifier* verifier, uint32_t vsrc,
const RegType& check_type) {
// Verify the src register type against the check type refining the type of the register
diff --git a/runtime/verifier/register_line.cc b/runtime/verifier/register_line.cc
index 82c371d..71aa94e 100644
--- a/runtime/verifier/register_line.cc
+++ b/runtime/verifier/register_line.cc
@@ -543,24 +543,5 @@
return changed;
}
-void RegisterLine::WriteReferenceBitMap(MethodVerifier* verifier,
- std::vector<uint8_t>* data, size_t max_bytes) {
- for (size_t i = 0; i < num_regs_; i += 8) {
- uint8_t val = 0;
- for (size_t j = 0; j < 8 && (i + j) < num_regs_; j++) {
- // Note: we write 1 for a Reference but not for Null
- if (GetRegisterType(verifier, i + j).IsNonZeroReferenceTypes()) {
- val |= 1 << j;
- }
- }
- if ((i / 8) >= max_bytes) {
- DCHECK_EQ(0, val);
- continue;
- }
- DCHECK_LT(i / 8, max_bytes) << "val=" << static_cast<uint32_t>(val);
- data->push_back(val);
- }
-}
-
} // namespace verifier
} // namespace art
diff --git a/runtime/verifier/register_line.h b/runtime/verifier/register_line.h
index 15ae202..56846c1 100644
--- a/runtime/verifier/register_line.h
+++ b/runtime/verifier/register_line.h
@@ -346,11 +346,6 @@
bool MergeRegisters(MethodVerifier* verifier, const RegisterLine* incoming_line)
SHARED_REQUIRES(Locks::mutator_lock_);
- size_t GetMaxNonZeroReferenceReg(MethodVerifier* verifier, size_t max_ref_reg) const;
-
- // Write a bit at each register location that holds a reference.
- void WriteReferenceBitMap(MethodVerifier* verifier, std::vector<uint8_t>* data, size_t max_bytes);
-
size_t GetMonitorEnterCount() const {
return monitors_.size();
}
diff --git a/test/370-dex-v37/build b/test/370-dex-v37/build
new file mode 100755
index 0000000..f472428
--- /dev/null
+++ b/test/370-dex-v37/build
@@ -0,0 +1,30 @@
+#!/bin/bash
+#
+# Copyright 2015 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# make us exit on a failure
+set -e
+
+./default-build "$@"
+
+if [[ $@ != *"--jvm"* ]]; then
+ # Change the generated dex file to have a v36 magic number if it is version 35
+ if test -f classes.dex && head -c 7 classes.dex | grep -q 035; then
+ # place ascii value '037' into the classes.dex file starting at byte 4.
+ printf '037' | dd status=none conv=notrunc of=classes.dex bs=1 seek=4 count=3
+ rm -f $TEST_NAME.jar
+ zip $TEST_NAME.jar classes.dex
+ fi
+fi
diff --git a/test/370-dex-v37/expected.txt b/test/370-dex-v37/expected.txt
new file mode 100644
index 0000000..af5626b
--- /dev/null
+++ b/test/370-dex-v37/expected.txt
@@ -0,0 +1 @@
+Hello, world!
diff --git a/test/370-dex-v37/info.txt b/test/370-dex-v37/info.txt
new file mode 100644
index 0000000..5ca9c76
--- /dev/null
+++ b/test/370-dex-v37/info.txt
@@ -0,0 +1 @@
+Print "Hello, World!" with a version 37 dex file.
diff --git a/test/370-dex-v37/src/Main.java b/test/370-dex-v37/src/Main.java
new file mode 100644
index 0000000..1ef6289
--- /dev/null
+++ b/test/370-dex-v37/src/Main.java
@@ -0,0 +1,21 @@
+/*
+ * Copyright (C) 2011 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+public class Main {
+ public static void main(String[] args) {
+ System.out.println("Hello, world!");
+ }
+}
diff --git a/test/496-checker-inlining-and-class-loader/src/Main.java b/test/496-checker-inlining-and-class-loader/src/Main.java
index ea6df62..8de6318 100644
--- a/test/496-checker-inlining-and-class-loader/src/Main.java
+++ b/test/496-checker-inlining-and-class-loader/src/Main.java
@@ -109,7 +109,8 @@
/* Load and initialize System */
/// CHECK-NEXT: LoadClass gen_clinit_check:true
/// CHECK-NEXT: StaticFieldGet
- /// CHECK-NEXT: LoadString
+ // There may be HArmDexCacheArraysBase or HX86ComputeBaseMethodAddress here.
+ /// CHECK: LoadString
/// CHECK-NEXT: NullCheck
/// CHECK-NEXT: InvokeVirtual
public static void bar() {
diff --git a/test/552-checker-sharpening/src/Main.java b/test/552-checker-sharpening/src/Main.java
index d50edd8..3d985bf 100644
--- a/test/552-checker-sharpening/src/Main.java
+++ b/test/552-checker-sharpening/src/Main.java
@@ -22,6 +22,12 @@
}
}
+ public static void assertStringEquals(String expected, String result) {
+ if (expected != null ? !expected.equals(result) : result != null) {
+ throw new Error("Expected: " + expected + ", found: " + result);
+ }
+ }
+
public static boolean doThrow = false;
private static int $noinline$foo(int x) {
@@ -185,6 +191,66 @@
return x;
}
+ /// CHECK-START: java.lang.String Main.$noinline$getBootImageString() sharpening (before)
+ /// CHECK: LoadString load_kind:DexCacheViaMethod
+
+ /// CHECK-START-X86: java.lang.String Main.$noinline$getBootImageString() sharpening (after)
+ // Note: load kind depends on PIC/non-PIC
+ // TODO: Remove DexCacheViaMethod when read barrier config supports BootImageAddress.
+ /// CHECK: LoadString load_kind:{{BootImageAddress|DexCachePcRelative|DexCacheViaMethod}}
+
+ /// CHECK-START-X86_64: java.lang.String Main.$noinline$getBootImageString() sharpening (after)
+ // Note: load kind depends on PIC/non-PIC
+ // TODO: Remove DexCacheViaMethod when read barrier config supports BootImageAddress.
+ /// CHECK: LoadString load_kind:{{BootImageAddress|DexCachePcRelative|DexCacheViaMethod}}
+
+ /// CHECK-START-ARM: java.lang.String Main.$noinline$getBootImageString() sharpening (after)
+ // Note: load kind depends on PIC/non-PIC
+ // TODO: Remove DexCacheViaMethod when read barrier config supports BootImageAddress.
+ /// CHECK: LoadString load_kind:{{BootImageAddress|DexCachePcRelative|DexCacheViaMethod}}
+
+ /// CHECK-START-ARM64: java.lang.String Main.$noinline$getBootImageString() sharpening (after)
+ // Note: load kind depends on PIC/non-PIC
+ // TODO: Remove DexCacheViaMethod when read barrier config supports BootImageAddress.
+ /// CHECK: LoadString load_kind:{{BootImageAddress|DexCachePcRelative|DexCacheViaMethod}}
+
+ public static String $noinline$getBootImageString() {
+ // Prevent inlining to avoid the string comparison being optimized away.
+ if (doThrow) { throw new Error(); }
+ // Empty string is known to be in the boot image.
+ return "";
+ }
+
+ /// CHECK-START: java.lang.String Main.$noinline$getNonBootImageString() sharpening (before)
+ /// CHECK: LoadString load_kind:DexCacheViaMethod
+
+ /// CHECK-START-X86: java.lang.String Main.$noinline$getNonBootImageString() sharpening (after)
+ /// CHECK: LoadString load_kind:DexCachePcRelative
+
+ /// CHECK-START-X86: java.lang.String Main.$noinline$getNonBootImageString() pc_relative_fixups_x86 (after)
+ /// CHECK-DAG: X86ComputeBaseMethodAddress
+ /// CHECK-DAG: LoadString load_kind:DexCachePcRelative
+
+ /// CHECK-START-X86_64: java.lang.String Main.$noinline$getNonBootImageString() sharpening (after)
+ /// CHECK: LoadString load_kind:DexCachePcRelative
+
+ /// CHECK-START-ARM: java.lang.String Main.$noinline$getNonBootImageString() sharpening (after)
+ /// CHECK: LoadString load_kind:DexCachePcRelative
+
+ /// CHECK-START-ARM: java.lang.String Main.$noinline$getNonBootImageString() dex_cache_array_fixups_arm (after)
+ /// CHECK-DAG: ArmDexCacheArraysBase
+ /// CHECK-DAG: LoadString load_kind:DexCachePcRelative
+
+ /// CHECK-START-ARM64: java.lang.String Main.$noinline$getNonBootImageString() sharpening (after)
+ /// CHECK: LoadString load_kind:DexCachePcRelative
+
+ public static String $noinline$getNonBootImageString() {
+ // Prevent inlining to avoid the string comparison being optimized away.
+ if (doThrow) { throw new Error(); }
+ // This string is not in the boot image.
+ return "non-boot-image-string";
+ }
+
public static void main(String[] args) {
assertIntEquals(1, testSimple(1));
assertIntEquals(1, testDiamond(false, 1));
@@ -194,5 +260,7 @@
assertIntEquals(1, testLoopWithDiamond(null, false, 1));
assertIntEquals(3, testLoopWithDiamond(new int[]{ 2 }, false, 1));
assertIntEquals(-6, testLoopWithDiamond(new int[]{ 3, 4 }, true, 1));
+ assertStringEquals("", $noinline$getBootImageString());
+ assertStringEquals("non-boot-image-string", $noinline$getNonBootImageString());
}
}
diff --git a/test/593-checker-boolean-to-integral-conv/expected.txt b/test/593-checker-boolean-to-integral-conv/expected.txt
new file mode 100644
index 0000000..b0aad4d
--- /dev/null
+++ b/test/593-checker-boolean-to-integral-conv/expected.txt
@@ -0,0 +1 @@
+passed
diff --git a/test/593-checker-boolean-to-integral-conv/info.txt b/test/593-checker-boolean-to-integral-conv/info.txt
new file mode 100644
index 0000000..2d883c7
--- /dev/null
+++ b/test/593-checker-boolean-to-integral-conv/info.txt
@@ -0,0 +1 @@
+Regression test for Boolean to integral types conversions.
diff --git a/test/593-checker-boolean-to-integral-conv/src/Main.java b/test/593-checker-boolean-to-integral-conv/src/Main.java
new file mode 100644
index 0000000..ba65839
--- /dev/null
+++ b/test/593-checker-boolean-to-integral-conv/src/Main.java
@@ -0,0 +1,232 @@
+/*
+ * Copyright (C) 2016 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+public class Main {
+
+ public static void main(String args[]) {
+ expectEqualsByte((byte)1, booleanToByte(true));
+ expectEqualsShort((short)1, booleanToShort(true));
+ expectEqualsChar((char)1, booleanToChar(true));
+ expectEqualsInt(1, booleanToInt(true));
+ expectEqualsLong(1L, booleanToLong(true));
+
+ expectEqualsInt(1, longToIntOfBoolean());
+
+ System.out.println("passed");
+ }
+
+ /// CHECK-START: byte Main.booleanToByte(boolean) builder (after)
+ /// CHECK: <<Arg:z\d+>> ParameterValue
+ /// CHECK-DAG: <<Zero:i\d+>> IntConstant 0
+ /// CHECK-DAG: <<One:i\d+>> IntConstant 1
+ /// CHECK-DAG: <<Cond:z\d+>> Equal [<<Arg>>,<<Zero>>]
+ /// CHECK-DAG: If [<<Cond>>]
+ /// CHECK-DAG: <<Phi:i\d+>> Phi [<<One>>,<<Zero>>]
+ /// CHECK-DAG: <<IToS:b\d+>> TypeConversion [<<Phi>>]
+ /// CHECK-DAG: Return [<<IToS>>]
+
+ /// CHECK-START: byte Main.booleanToByte(boolean) select_generator (after)
+ /// CHECK: <<Arg:z\d+>> ParameterValue
+ /// CHECK-DAG: <<Zero:i\d+>> IntConstant 0
+ /// CHECK-DAG: <<One:i\d+>> IntConstant 1
+ /// CHECK-DAG: <<Sel:i\d+>> Select [<<Zero>>,<<One>>,<<Arg>>]
+ /// CHECK-DAG: <<IToS:b\d+>> TypeConversion [<<Sel>>]
+ /// CHECK-DAG: Return [<<IToS>>]
+
+ /// CHECK-START: byte Main.booleanToByte(boolean) instruction_simplifier_after_bce (after)
+ /// CHECK: <<Arg:z\d+>> ParameterValue
+ /// CHECK-DAG: Return [<<Arg>>]
+
+ static byte booleanToByte(boolean b) {
+ return (byte)(b ? 1 : 0);
+ }
+
+ /// CHECK-START: short Main.booleanToShort(boolean) builder (after)
+ /// CHECK: <<Arg:z\d+>> ParameterValue
+ /// CHECK-DAG: <<Zero:i\d+>> IntConstant 0
+ /// CHECK-DAG: <<One:i\d+>> IntConstant 1
+ /// CHECK-DAG: <<Cond:z\d+>> Equal [<<Arg>>,<<Zero>>]
+ /// CHECK-DAG: If [<<Cond>>]
+ /// CHECK-DAG: <<Phi:i\d+>> Phi [<<One>>,<<Zero>>]
+ /// CHECK-DAG: <<IToS:s\d+>> TypeConversion [<<Phi>>]
+ /// CHECK-DAG: Return [<<IToS>>]
+
+ /// CHECK-START: short Main.booleanToShort(boolean) select_generator (after)
+ /// CHECK: <<Arg:z\d+>> ParameterValue
+ /// CHECK-DAG: <<Zero:i\d+>> IntConstant 0
+ /// CHECK-DAG: <<One:i\d+>> IntConstant 1
+ /// CHECK-DAG: <<Sel:i\d+>> Select [<<Zero>>,<<One>>,<<Arg>>]
+ /// CHECK-DAG: <<IToS:s\d+>> TypeConversion [<<Sel>>]
+ /// CHECK-DAG: Return [<<IToS>>]
+
+ /// CHECK-START: short Main.booleanToShort(boolean) instruction_simplifier_after_bce (after)
+ /// CHECK: <<Arg:z\d+>> ParameterValue
+ /// CHECK-DAG: Return [<<Arg>>]
+
+ static short booleanToShort(boolean b) {
+ return (short)(b ? 1 : 0);
+ }
+
+ /// CHECK-START: char Main.booleanToChar(boolean) builder (after)
+ /// CHECK: <<Arg:z\d+>> ParameterValue
+ /// CHECK-DAG: <<Zero:i\d+>> IntConstant 0
+ /// CHECK-DAG: <<One:i\d+>> IntConstant 1
+ /// CHECK-DAG: <<Cond:z\d+>> Equal [<<Arg>>,<<Zero>>]
+ /// CHECK-DAG: If [<<Cond>>]
+ /// CHECK-DAG: <<Phi:i\d+>> Phi [<<One>>,<<Zero>>]
+ /// CHECK-DAG: <<IToC:c\d+>> TypeConversion [<<Phi>>]
+ /// CHECK-DAG: Return [<<IToC>>]
+
+ /// CHECK-START: char Main.booleanToChar(boolean) select_generator (after)
+ /// CHECK: <<Arg:z\d+>> ParameterValue
+ /// CHECK-DAG: <<Zero:i\d+>> IntConstant 0
+ /// CHECK-DAG: <<One:i\d+>> IntConstant 1
+ /// CHECK-DAG: <<Sel:i\d+>> Select [<<Zero>>,<<One>>,<<Arg>>]
+ /// CHECK-DAG: <<IToC:c\d+>> TypeConversion [<<Sel>>]
+ /// CHECK-DAG: Return [<<IToC>>]
+
+ /// CHECK-START: char Main.booleanToChar(boolean) instruction_simplifier_after_bce (after)
+ /// CHECK: <<Arg:z\d+>> ParameterValue
+ /// CHECK-DAG: Return [<<Arg>>]
+
+ static char booleanToChar(boolean b) {
+ return (char)(b ? 1 : 0);
+ }
+
+ /// CHECK-START: int Main.booleanToInt(boolean) builder (after)
+ /// CHECK: <<Arg:z\d+>> ParameterValue
+ /// CHECK-DAG: <<Zero:i\d+>> IntConstant 0
+ /// CHECK-DAG: <<One:i\d+>> IntConstant 1
+ /// CHECK-DAG: <<Cond:z\d+>> Equal [<<Arg>>,<<Zero>>]
+ /// CHECK-DAG: If [<<Cond>>]
+ /// CHECK-DAG: <<Phi:i\d+>> Phi [<<One>>,<<Zero>>]
+ /// CHECK-DAG: Return [<<Phi>>]
+
+ /// CHECK-START: int Main.booleanToInt(boolean) select_generator (after)
+ /// CHECK: <<Arg:z\d+>> ParameterValue
+ /// CHECK-DAG: <<Zero:i\d+>> IntConstant 0
+ /// CHECK-DAG: <<One:i\d+>> IntConstant 1
+ /// CHECK-DAG: <<Sel:i\d+>> Select [<<Zero>>,<<One>>,<<Arg>>]
+ /// CHECK-DAG: Return [<<Sel>>]
+
+ /// CHECK-START: int Main.booleanToInt(boolean) instruction_simplifier_after_bce (after)
+ /// CHECK: <<Arg:z\d+>> ParameterValue
+ /// CHECK-DAG: Return [<<Arg>>]
+
+ static int booleanToInt(boolean b) {
+ return b ? 1 : 0;
+ }
+
+ /// CHECK-START: long Main.booleanToLong(boolean) builder (after)
+ /// CHECK: <<Arg:z\d+>> ParameterValue
+ /// CHECK-DAG: <<Zero:i\d+>> IntConstant 0
+ /// CHECK-DAG: <<One:i\d+>> IntConstant 1
+ /// CHECK-DAG: <<Cond:z\d+>> Equal [<<Arg>>,<<Zero>>]
+ /// CHECK-DAG: If [<<Cond>>]
+ /// CHECK-DAG: <<Phi:i\d+>> Phi [<<One>>,<<Zero>>]
+ /// CHECK-DAG: <<IToJ:j\d+>> TypeConversion [<<Phi>>]
+ /// CHECK-DAG: Return [<<IToJ>>]
+
+ /// CHECK-START: long Main.booleanToLong(boolean) select_generator (after)
+ /// CHECK: <<Arg:z\d+>> ParameterValue
+ /// CHECK-DAG: <<Zero:i\d+>> IntConstant 0
+ /// CHECK-DAG: <<One:i\d+>> IntConstant 1
+ /// CHECK-DAG: <<Sel:i\d+>> Select [<<Zero>>,<<One>>,<<Arg>>]
+ /// CHECK-DAG: <<IToJ:j\d+>> TypeConversion [<<Sel>>]
+ /// CHECK-DAG: Return [<<IToJ>>]
+
+ /// CHECK-START: long Main.booleanToLong(boolean) instruction_simplifier_after_bce (after)
+ /// CHECK: <<Arg:z\d+>> ParameterValue
+ /// CHECK-DAG: <<ZToJ:j\d+>> TypeConversion [<<Arg>>]
+ /// CHECK-DAG: Return [<<ZToJ>>]
+
+ static long booleanToLong(boolean b) {
+ return b ? 1 : 0;
+ }
+
+ /// CHECK-START: int Main.longToIntOfBoolean() builder (after)
+ /// CHECK-DAG: <<Method:[ij]\d+>> CurrentMethod
+ /// CHECK-DAG: <<Sget:z\d+>> StaticFieldGet
+ /// CHECK-DAG: <<ZToJ:j\d+>> InvokeStaticOrDirect [<<Sget>>,<<Method>>]
+ /// CHECK-DAG: <<JToI:i\d+>> TypeConversion [<<ZToJ>>]
+ /// CHECK-DAG: Return [<<JToI>>]
+
+ /// CHECK-START: int Main.longToIntOfBoolean() inliner (after)
+ /// CHECK-DAG: <<Method:[ij]\d+>> CurrentMethod
+ /// CHECK-DAG: <<Zero:i\d+>> IntConstant 0
+ /// CHECK-DAG: <<One:i\d+>> IntConstant 1
+ /// CHECK-DAG: <<Sget:z\d+>> StaticFieldGet
+ /// CHECK-DAG: If [<<Sget>>]
+ /// CHECK-DAG: <<Phi:i\d+>> Phi [<<One>>,<<Zero>>]
+ /// CHECK-DAG: <<IToJ:j\d+>> TypeConversion [<<Phi>>]
+ /// CHECK-DAG: <<JToI:i\d+>> TypeConversion [<<IToJ>>]
+ /// CHECK-DAG: Return [<<JToI>>]
+
+ /// CHECK-START: int Main.longToIntOfBoolean() select_generator (after)
+ /// CHECK-DAG: <<Method:[ij]\d+>> CurrentMethod
+ /// CHECK-DAG: <<Zero:i\d+>> IntConstant 0
+ /// CHECK-DAG: <<One:i\d+>> IntConstant 1
+ /// CHECK-DAG: <<Sget:z\d+>> StaticFieldGet
+ /// CHECK-DAG: <<Sel:i\d+>> Select [<<Zero>>,<<One>>,<<Sget>>]
+ /// CHECK-DAG: <<IToJ:j\d+>> TypeConversion [<<Sel>>]
+ /// CHECK-DAG: <<JToI:i\d+>> TypeConversion [<<IToJ>>]
+ /// CHECK-DAG: Return [<<JToI>>]
+
+ /// CHECK-START: int Main.longToIntOfBoolean() instruction_simplifier_after_bce (after)
+ /// CHECK-DAG: <<Method:[ij]\d+>> CurrentMethod
+ /// CHECK-DAG: <<Sget:z\d+>> StaticFieldGet
+ /// CHECK-DAG: Return [<<Sget>>]
+
+ static int longToIntOfBoolean() {
+ long l = booleanToLong(booleanField);
+ return (int) l;
+ }
+
+
+ private static void expectEqualsByte(byte expected, byte result) {
+ if (expected != result) {
+ throw new Error("Expected: " + expected + ", found: " + result);
+ }
+ }
+
+ private static void expectEqualsShort(short expected, short result) {
+ if (expected != result) {
+ throw new Error("Expected: " + expected + ", found: " + result);
+ }
+ }
+
+ private static void expectEqualsChar(char expected, char result) {
+ if (expected != result) {
+ throw new Error("Expected: " + expected + ", found: " + result);
+ }
+ }
+
+ private static void expectEqualsInt(int expected, int result) {
+ if (expected != result) {
+ throw new Error("Expected: " + expected + ", found: " + result);
+ }
+ }
+
+ private static void expectEqualsLong(long expected, long result) {
+ if (expected != result) {
+ throw new Error("Expected: " + expected + ", found: " + result);
+ }
+ }
+
+
+ public static boolean booleanField = true;
+
+}
diff --git a/test/593-checker-long-to-float-regression/expected.txt b/test/593-checker-long-to-float-regression/expected.txt
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/test/593-checker-long-to-float-regression/expected.txt
diff --git a/test/593-checker-long-to-float-regression/info.txt b/test/593-checker-long-to-float-regression/info.txt
new file mode 100644
index 0000000..39402e9
--- /dev/null
+++ b/test/593-checker-long-to-float-regression/info.txt
@@ -0,0 +1,3 @@
+Regression test for x86_64's code generator, which had a bug in
+the long-to-float implementation loading a constant as 64-bit double
+instead of 32-bit float.
diff --git a/test/593-checker-long-to-float-regression/src/Main.java b/test/593-checker-long-to-float-regression/src/Main.java
new file mode 100644
index 0000000..9c07f3d
--- /dev/null
+++ b/test/593-checker-long-to-float-regression/src/Main.java
@@ -0,0 +1,46 @@
+/*
+ * Copyright (C) 2016 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+public class Main {
+
+ static boolean doThrow = false;
+ static long longValue;
+
+ public static void assertEquals(float expected, float result) {
+ if (expected != result) {
+ throw new Error("Expected: " + expected + ", found: " + result);
+ }
+ }
+
+ public static void main(String[] args) {
+ assertEquals(1.0F, $noinline$longToFloat());
+ }
+
+ /// CHECK-START: float Main.$noinline$longToFloat() register (after)
+ /// CHECK-DAG: <<Const1:j\d+>> LongConstant 1
+ /// CHECK-DAG: <<Convert:f\d+>> TypeConversion [<<Const1>>]
+ /// CHECK-DAG: Return [<<Convert>>]
+
+ static float $noinline$longToFloat() {
+ if (doThrow) { throw new Error(); }
+ longValue = $inline$returnConst();
+ return (float) longValue;
+ }
+
+ static long $inline$returnConst() {
+ return 1L;
+ }
+}
diff --git a/test/593-checker-shift-and-simplifier/expected.txt b/test/593-checker-shift-and-simplifier/expected.txt
new file mode 100644
index 0000000..b0aad4d
--- /dev/null
+++ b/test/593-checker-shift-and-simplifier/expected.txt
@@ -0,0 +1 @@
+passed
diff --git a/test/593-checker-shift-and-simplifier/info.txt b/test/593-checker-shift-and-simplifier/info.txt
new file mode 100644
index 0000000..2fae678
--- /dev/null
+++ b/test/593-checker-shift-and-simplifier/info.txt
@@ -0,0 +1 @@
+Regression test on pattern that caused double removal of AND by AMD64 simplifier.
diff --git a/test/593-checker-shift-and-simplifier/src/Main.java b/test/593-checker-shift-and-simplifier/src/Main.java
new file mode 100644
index 0000000..65e809a
--- /dev/null
+++ b/test/593-checker-shift-and-simplifier/src/Main.java
@@ -0,0 +1,49 @@
+/*
+ * Copyright (C) 2016 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+public class Main {
+
+ private static int[] a = { 10 };
+
+ // A very particular set of operations that caused a double removal by the
+ // ARM64 simplifier doing "forward" removals (b/27851582).
+
+ /// CHECK-START-ARM64: int Main.operations() instruction_simplifier_arm64 (before)
+ /// CHECK-DAG: <<Get:i\d+>> ArrayGet
+ /// CHECK-DAG: <<Not:i\d+>> Not [<<Get>>]
+ /// CHECK-DAG: <<Shl:i\d+>> Shl [<<Get>>,i{{\d+}}]
+ /// CHECK-DAG: And [<<Not>>,<<Shl>>]
+ //
+ /// CHECK-START-ARM64: int Main.operations() instruction_simplifier_arm64 (after)
+ /// CHECK-DAG: <<Get:i\d+>> ArrayGet
+ /// CHECK-DAG: <<Not:i\d+>> Not [<<Get>>]
+ /// CHECK-DAG: Arm64DataProcWithShifterOp [<<Not>>,<<Get>>] kind:And+LSL shift:2
+ private static int operations() {
+ int r = a[0];
+ int n = ~r;
+ int s = r << 2;
+ int a = s & n;
+ return a;
+ }
+
+ public static void main(String[] args) {
+ if (operations() != 32) {
+ System.out.println("failed");
+ } else {
+ System.out.println("passed");
+ }
+ }
+}
diff --git a/test/Android.run-test.mk b/test/Android.run-test.mk
index 11a38cb..1edc599 100644
--- a/test/Android.run-test.mk
+++ b/test/Android.run-test.mk
@@ -37,14 +37,10 @@
$(DX) \
$(HOST_OUT_EXECUTABLES)/jasmin \
$(HOST_OUT_EXECUTABLES)/smali \
- $(HOST_OUT_EXECUTABLES)/dexmerger
-TEST_ART_RUN_TEST_ORDERONLY_DEPENDENCIES :=
+ $(HOST_OUT_EXECUTABLES)/dexmerger \
+ $(JACK)
-ifeq ($(ANDROID_COMPILE_WITH_JACK),true)
- TEST_ART_RUN_TEST_DEPENDENCIES += \
- $(JACK)
- TEST_ART_RUN_TEST_ORDERONLY_DEPENDENCIES += setup-jack-server
-endif
+TEST_ART_RUN_TEST_ORDERONLY_DEPENDENCIES := setup-jack-server
ifeq ($(ART_TEST_DEBUG_GC),true)
ART_TEST_WITH_STRACE := true
@@ -55,11 +51,6 @@
define define-build-art-run-test
dmart_target := $(art_run_tests_dir)/art-run-tests/$(1)/touch
run_test_options = --build-only
- ifeq ($(ANDROID_COMPILE_WITH_JACK),true)
- run_test_options += --build-with-jack
- else
- run_test_options += --build-with-javac-dx
- endif
ifeq ($(ART_TEST_QUIET),true)
run_test_options += --quiet
endif
@@ -670,11 +661,6 @@
test_groups :=
uc_host_or_target :=
jack_classpath :=
- ifeq ($(ANDROID_COMPILE_WITH_JACK),true)
- run_test_options += --build-with-jack
- else
- run_test_options += --build-with-javac-dx
- endif
ifeq ($(ART_TEST_WITH_STRACE),true)
run_test_options += --strace
endif