ARM: Shorter fast-path for read barrier field load.
Reduces the aosp_hammerhead-userdebug boot.oat by 2.2MiB,
i.e. ~2.2%, in the ART_USE_READ_BARRIER=true configuration.
Test: Tested with ART_USE_READ_BARRIER=true on Nexus 5.
Bug: 29966877
Bug: 12687968
Change-Id: I4454150003e12a1aa7f0cf451627dc1ee9a495ae
diff --git a/compiler/optimizing/code_generator_arm.cc b/compiler/optimizing/code_generator_arm.cc
index 690ecc3..d636f8f 100644
--- a/compiler/optimizing/code_generator_arm.cc
+++ b/compiler/optimizing/code_generator_arm.cc
@@ -6277,21 +6277,12 @@
// /* LockWord */ lock_word = LockWord(monitor)
static_assert(sizeof(LockWord) == sizeof(int32_t),
"art::LockWord and int32_t have different sizes.");
- // /* uint32_t */ rb_state = lock_word.ReadBarrierState()
- __ Lsr(temp_reg, temp_reg, LockWord::kReadBarrierStateShift);
- __ and_(temp_reg, temp_reg, ShifterOperand(LockWord::kReadBarrierStateMask));
- static_assert(
- LockWord::kReadBarrierStateMask == ReadBarrier::rb_ptr_mask_,
- "art::LockWord::kReadBarrierStateMask is not equal to art::ReadBarrier::rb_ptr_mask_.");
- // Introduce a dependency on the high bits of rb_state, which shall
- // be all zeroes, to prevent load-load reordering, and without using
+ // Introduce a dependency on the lock_word including the rb_state,
+ // which shall prevent load-load reordering without using
// a memory barrier (which would be more expensive).
- // IP = rb_state & ~LockWord::kReadBarrierStateMask = 0
- __ bic(IP, temp_reg, ShifterOperand(LockWord::kReadBarrierStateMask));
- // obj is unchanged by this operation, but its value now depends on
- // IP, which depends on temp_reg.
- __ add(obj, obj, ShifterOperand(IP));
+ // obj is unchanged by this operation, but its value now depends on temp_reg.
+ __ add(obj, obj, ShifterOperand(temp_reg, LSR, 32));
// The actual reference load.
if (index.IsValid()) {
@@ -6328,8 +6319,14 @@
// if (rb_state == ReadBarrier::gray_ptr_)
// ref = ReadBarrier::Mark(ref);
- __ cmp(temp_reg, ShifterOperand(ReadBarrier::gray_ptr_));
- __ b(slow_path->GetEntryLabel(), EQ);
+ // Given the numeric representation, it's enough to check the low bit of the
+ // rb_state. We do that by shifting the bit out of the lock word with LSRS
+ // which can be a 16-bit instruction unlike the TST immediate.
+ static_assert(ReadBarrier::white_ptr_ == 0, "Expecting white to have value 0");
+ static_assert(ReadBarrier::gray_ptr_ == 1, "Expecting gray to have value 1");
+ static_assert(ReadBarrier::black_ptr_ == 2, "Expecting black to have value 2");
+ __ Lsrs(temp_reg, temp_reg, LockWord::kReadBarrierStateShift + 1);
+ __ b(slow_path->GetEntryLabel(), CS); // Carry flag is the last bit shifted out by LSRS.
__ Bind(slow_path->GetExitLabel());
}
diff --git a/disassembler/disassembler_arm.cc b/disassembler/disassembler_arm.cc
index 1a3e3f5..c410cd9 100644
--- a/disassembler/disassembler_arm.cc
+++ b/disassembler/disassembler_arm.cc
@@ -782,23 +782,13 @@
args << Rm;
// Shift operand.
- bool noShift = (imm5 == 0 && shift_type != 0x3);
+ bool noShift = (imm5 == 0 && shift_type == 0x0);
if (!noShift) {
args << ", ";
- switch (shift_type) {
- case 0x0: args << "lsl"; break;
- case 0x1: args << "lsr"; break;
- case 0x2: args << "asr"; break;
- case 0x3:
- if (imm5 == 0) {
- args << "rrx";
- } else {
- args << "ror #" << imm5;
- }
- break;
- }
- if (shift_type != 0x3 /* rrx */) {
- args << StringPrintf(" #%d", (0 != imm5 || 0 == shift_type) ? imm5 : 32);
+ if (shift_type == 0x3u && imm5 == 0u) {
+ args << "rrx";
+ } else {
+ args << kThumb2ShiftOperations[shift_type] << " #" << ((0 != imm5) ? imm5 : 32);
}
}
@@ -1516,82 +1506,82 @@
}
break;
}
- default: // more formats
- if ((op2 >> 4) == 2) { // 010xxxx
- // data processing (register)
- if ((instr & 0x0080f0f0) == 0x0000f000) {
- // LSL, LSR, ASR, ROR
- uint32_t shift_op = (instr >> 21) & 3;
- uint32_t S = (instr >> 20) & 1;
- ArmRegister Rd(instr, 8);
+ default: // more formats
+ if ((op2 >> 4) == 2) { // 010xxxx
+ // data processing (register)
+ if ((instr & 0x0080f0f0) == 0x0000f000) {
+ // LSL, LSR, ASR, ROR
+ uint32_t shift_op = (instr >> 21) & 3;
+ uint32_t S = (instr >> 20) & 1;
+ ArmRegister Rd(instr, 8);
+ ArmRegister Rn(instr, 16);
+ ArmRegister Rm(instr, 0);
+ opcode << kThumb2ShiftOperations[shift_op] << (S != 0 ? "s" : "");
+ args << Rd << ", " << Rn << ", " << Rm;
+ }
+ } else if ((op2 >> 3) == 6) { // 0110xxx
+ // Multiply, multiply accumulate, and absolute difference
+ op1 = (instr >> 20) & 0x7;
+ op2 = (instr >> 4) & 0x1;
+ ArmRegister Ra(instr, 12);
ArmRegister Rn(instr, 16);
ArmRegister Rm(instr, 0);
- opcode << kThumb2ShiftOperations[shift_op] << (S != 0 ? "s" : "");
- args << Rd << ", " << Rn << ", " << Rm;
- }
- } else if ((op2 >> 3) == 6) { // 0110xxx
- // Multiply, multiply accumulate, and absolute difference
- op1 = (instr >> 20) & 0x7;
- op2 = (instr >> 4) & 0x1;
- ArmRegister Ra(instr, 12);
- ArmRegister Rn(instr, 16);
- ArmRegister Rm(instr, 0);
- ArmRegister Rd(instr, 8);
- switch (op1) {
- case 0:
- if (op2 == 0) {
- if (Ra.r == 0xf) {
- opcode << "mul";
- args << Rd << ", " << Rn << ", " << Rm;
+ ArmRegister Rd(instr, 8);
+ switch (op1) {
+ case 0:
+ if (op2 == 0) {
+ if (Ra.r == 0xf) {
+ opcode << "mul";
+ args << Rd << ", " << Rn << ", " << Rm;
+ } else {
+ opcode << "mla";
+ args << Rd << ", " << Rn << ", " << Rm << ", " << Ra;
+ }
} else {
- opcode << "mla";
+ opcode << "mls";
args << Rd << ", " << Rn << ", " << Rm << ", " << Ra;
}
- } else {
- opcode << "mls";
- args << Rd << ", " << Rn << ", " << Rm << ", " << Ra;
+ break;
+ case 1:
+ case 2:
+ case 3:
+ case 4:
+ case 5:
+ case 6:
+ break; // do these sometime
}
- break;
- case 1:
- case 2:
- case 3:
- case 4:
- case 5:
- case 6:
- break; // do these sometime
+ } else if ((op2 >> 3) == 7) { // 0111xxx
+ // Long multiply, long multiply accumulate, and divide
+ op1 = (instr >> 20) & 0x7;
+ op2 = (instr >> 4) & 0xf;
+ ArmRegister Rn(instr, 16);
+ ArmRegister Rm(instr, 0);
+ ArmRegister Rd(instr, 8);
+ ArmRegister RdHi(instr, 8);
+ ArmRegister RdLo(instr, 12);
+ switch (op1) {
+ case 0:
+ opcode << "smull";
+ args << RdLo << ", " << RdHi << ", " << Rn << ", " << Rm;
+ break;
+ case 1:
+ opcode << "sdiv";
+ args << Rd << ", " << Rn << ", " << Rm;
+ break;
+ case 2:
+ opcode << "umull";
+ args << RdLo << ", " << RdHi << ", " << Rn << ", " << Rm;
+ break;
+ case 3:
+ opcode << "udiv";
+ args << Rd << ", " << Rn << ", " << Rm;
+ break;
+ case 4:
+ case 5:
+ case 6:
+ break; // TODO: when we generate these...
+ }
}
- } else if ((op2 >> 3) == 7) { // 0111xxx
- // Long multiply, long multiply accumulate, and divide
- op1 = (instr >> 20) & 0x7;
- op2 = (instr >> 4) & 0xf;
- ArmRegister Rn(instr, 16);
- ArmRegister Rm(instr, 0);
- ArmRegister Rd(instr, 8);
- ArmRegister RdHi(instr, 8);
- ArmRegister RdLo(instr, 12);
- switch (op1) {
- case 0:
- opcode << "smull";
- args << RdLo << ", " << RdHi << ", " << Rn << ", " << Rm;
- break;
- case 1:
- opcode << "sdiv";
- args << Rd << ", " << Rn << ", " << Rm;
- break;
- case 2:
- opcode << "umull";
- args << RdLo << ", " << RdHi << ", " << Rn << ", " << Rm;
- break;
- case 3:
- opcode << "udiv";
- args << Rd << ", " << Rn << ", " << Rm;
- break;
- case 4:
- case 5:
- case 6:
- break; // TODO: when we generate these...
- }
- }
}
break;
default: