Fix ARM & ARM64 UnsafeGetObject intrinsics with read barriers.
The implementation was incorrectly interpreting the 'offset'
input as an index in a (4-byte) object reference array,
whereas it is a (1-byte) offset to an object reference field
within the 'base' (object) input.
Bug: 29516905
Change-Id: I4da5be0193217965f25e5d141c242592dea6ffe8
Test: Covered by test/004-UnsafeTest.
diff --git a/compiler/optimizing/code_generator_arm64.cc b/compiler/optimizing/code_generator_arm64.cc
index 5d3c8c5..dc5802a 100644
--- a/compiler/optimizing/code_generator_arm64.cc
+++ b/compiler/optimizing/code_generator_arm64.cc
@@ -4940,8 +4940,16 @@
// /* HeapReference<Object> */ ref = *(obj + offset)
Location no_index = Location::NoLocation();
- GenerateReferenceLoadWithBakerReadBarrier(
- instruction, ref, obj, offset, no_index, temp, needs_null_check, use_load_acquire);
+ size_t no_scale_factor = 0U;
+ GenerateReferenceLoadWithBakerReadBarrier(instruction,
+ ref,
+ obj,
+ offset,
+ no_index,
+ no_scale_factor,
+ temp,
+ needs_null_check,
+ use_load_acquire);
}
void CodeGeneratorARM64::GenerateArrayLoadWithBakerReadBarrier(HInstruction* instruction,
@@ -4958,10 +4966,21 @@
// never use Load-Acquire instructions on ARM64.
const bool use_load_acquire = false;
+ static_assert(
+ sizeof(mirror::HeapReference<mirror::Object>) == sizeof(int32_t),
+ "art::mirror::HeapReference<art::mirror::Object> and int32_t have different sizes.");
// /* HeapReference<Object> */ ref =
// *(obj + data_offset + index * sizeof(HeapReference<Object>))
- GenerateReferenceLoadWithBakerReadBarrier(
- instruction, ref, obj, data_offset, index, temp, needs_null_check, use_load_acquire);
+ size_t scale_factor = Primitive::ComponentSizeShift(Primitive::kPrimNot);
+ GenerateReferenceLoadWithBakerReadBarrier(instruction,
+ ref,
+ obj,
+ data_offset,
+ index,
+ scale_factor,
+ temp,
+ needs_null_check,
+ use_load_acquire);
}
void CodeGeneratorARM64::GenerateReferenceLoadWithBakerReadBarrier(HInstruction* instruction,
@@ -4969,15 +4988,16 @@
vixl::Register obj,
uint32_t offset,
Location index,
+ size_t scale_factor,
Register temp,
bool needs_null_check,
bool use_load_acquire) {
DCHECK(kEmitCompilerReadBarrier);
DCHECK(kUseBakerReadBarrier);
- // If `index` is a valid location, then we are emitting an array
- // load, so we shouldn't be using a Load Acquire instruction.
- // In other words: `index.IsValid()` => `!use_load_acquire`.
- DCHECK(!index.IsValid() || !use_load_acquire);
+ // If we are emitting an array load, we should not be using a
+ // Load Acquire instruction. In other words:
+ // `instruction->IsArrayGet()` => `!use_load_acquire`.
+ DCHECK(!instruction->IsArrayGet() || !use_load_acquire);
MacroAssembler* masm = GetVIXLAssembler();
UseScratchRegisterScope temps(masm);
@@ -5034,20 +5054,33 @@
// The actual reference load.
if (index.IsValid()) {
- static_assert(
- sizeof(mirror::HeapReference<mirror::Object>) == sizeof(int32_t),
- "art::mirror::HeapReference<art::mirror::Object> and int32_t have different sizes.");
- // /* HeapReference<Object> */ ref =
- // *(obj + offset + index * sizeof(HeapReference<Object>))
- const size_t shift_amount = Primitive::ComponentSizeShift(type);
- if (index.IsConstant()) {
- uint32_t computed_offset = offset + (Int64ConstantFrom(index) << shift_amount);
- Load(type, ref_reg, HeapOperand(obj, computed_offset));
+ // Load types involving an "index".
+ if (use_load_acquire) {
+ // UnsafeGetObjectVolatile intrinsic case.
+ // Register `index` is not an index in an object array, but an
+ // offset to an object reference field within object `obj`.
+ DCHECK(instruction->IsInvoke()) << instruction->DebugName();
+ DCHECK(instruction->GetLocations()->Intrinsified());
+ DCHECK(instruction->AsInvoke()->GetIntrinsic() == Intrinsics::kUnsafeGetObjectVolatile)
+ << instruction->AsInvoke()->GetIntrinsic();
+ DCHECK_EQ(offset, 0U);
+ DCHECK_EQ(scale_factor, 0U);
+ DCHECK_EQ(needs_null_check, 0U);
+ // /* HeapReference<Object> */ ref = *(obj + index)
+ MemOperand field = HeapOperand(obj, XRegisterFrom(index));
+ LoadAcquire(instruction, ref_reg, field, /* needs_null_check */ false);
} else {
- temp2 = temps.AcquireW();
- __ Add(temp2, obj, offset);
- Load(type, ref_reg, HeapOperand(temp2, XRegisterFrom(index), LSL, shift_amount));
- temps.Release(temp2);
+ // ArrayGet and UnsafeGetObject intrinsics cases.
+ // /* HeapReference<Object> */ ref = *(obj + offset + (index << scale_factor))
+ if (index.IsConstant()) {
+ uint32_t computed_offset = offset + (Int64ConstantFrom(index) << scale_factor);
+ Load(type, ref_reg, HeapOperand(obj, computed_offset));
+ } else {
+ temp2 = temps.AcquireW();
+ __ Add(temp2, obj, offset);
+ Load(type, ref_reg, HeapOperand(temp2, XRegisterFrom(index), LSL, scale_factor));
+ temps.Release(temp2);
+ }
}
} else {
// /* HeapReference<Object> */ ref = *(obj + offset)