Revert "Revert "Revert "Revert "ART: Enable Jit Profiling in Mterp for arm/arm64""""
This reverts commit 5d03317a834efdf3b5240c401f1bc2ceac7a2f25.
We need to catch all possible cases in which new instrumentation appears
or the debugger is attached, and then switch to the reference interpreter
if necessary. We may, in a future CL, use the alt-mterp mechanism to accompish
this (as did Dalvik).
Only enables Arm64 for now. Once it survives extended testing, will enable
arm and update x86.
Updated OSR handling to match other interpreters.
Change-Id: I076f1d752d6f59899876bab26b18e2221cd92f69
diff --git a/runtime/instrumentation.h b/runtime/instrumentation.h
index 56aeefc..e3cbf53 100644
--- a/runtime/instrumentation.h
+++ b/runtime/instrumentation.h
@@ -290,6 +290,14 @@
bool IsActive() const SHARED_REQUIRES(Locks::mutator_lock_) {
return have_dex_pc_listeners_ || have_method_entry_listeners_ || have_method_exit_listeners_ ||
have_field_read_listeners_ || have_field_write_listeners_ ||
+ have_exception_caught_listeners_ || have_method_unwind_listeners_ ||
+ have_branch_listeners_ || have_invoke_virtual_or_interface_listeners_;
+ }
+
+ // Any instrumentation *other* than what is needed for Jit profiling active?
+ bool NonJitProfilingActive() const SHARED_REQUIRES(Locks::mutator_lock_) {
+ return have_dex_pc_listeners_ || have_method_exit_listeners_ ||
+ have_field_read_listeners_ || have_field_write_listeners_ ||
have_exception_caught_listeners_ || have_method_unwind_listeners_;
}
diff --git a/runtime/interpreter/interpreter.cc b/runtime/interpreter/interpreter.cc
index 4fd3c78..a595d33 100644
--- a/runtime/interpreter/interpreter.cc
+++ b/runtime/interpreter/interpreter.cc
@@ -320,12 +320,13 @@
// No Mterp variant - just use the switch interpreter.
return ExecuteSwitchImpl<false, true>(self, code_item, shadow_frame, result_register,
false);
+ } else if (UNLIKELY(!Runtime::Current()->IsStarted())) {
+ return ExecuteSwitchImpl<false, false>(self, code_item, shadow_frame, result_register,
+ false);
} else {
- const instrumentation::Instrumentation* const instrumentation =
- Runtime::Current()->GetInstrumentation();
while (true) {
- if (instrumentation->IsActive() || !Runtime::Current()->IsStarted()) {
- // TODO: allow JIT profiling instrumentation. Now, just punt on all instrumentation.
+ // Mterp does not support all instrumentation/debugging.
+ if (MterpShouldSwitchInterpreters()) {
#if !defined(__clang__)
return ExecuteGotoImpl<false, false>(self, code_item, shadow_frame, result_register);
#else
diff --git a/runtime/interpreter/mterp/arm/bincmp.S b/runtime/interpreter/mterp/arm/bincmp.S
index 474bc3c..774e167 100644
--- a/runtime/interpreter/mterp/arm/bincmp.S
+++ b/runtime/interpreter/mterp/arm/bincmp.S
@@ -6,17 +6,29 @@
* For: if-eq, if-ne, if-lt, if-ge, if-gt, if-le
*/
/* if-cmp vA, vB, +CCCC */
-#if MTERP_SUSPEND
+#if MTERP_PROFILE_BRANCHES
mov r1, rINST, lsr #12 @ r1<- B
ubfx r0, rINST, #8, #4 @ r0<- A
GET_VREG r3, r1 @ r3<- vB
GET_VREG r2, r0 @ r2<- vA
- FETCH_S r1, 1 @ r1<- branch offset, in code units
+ FETCH_S rINST, 1 @ rINST<- branch offset, in code units
cmp r2, r3 @ compare (vA, vB)
- mov${revcmp} r1, #2 @ r1<- BYTE branch dist for not-taken
- adds r2, r1, r1 @ convert to bytes, check sign
+ b${revcmp} .L_${opcode}_not_taken
+ EXPORT_PC
+ mov r0, rSELF
+ add r1, rFP, #OFF_FP_SHADOWFRAME
+ mov r2, rINST
+ bl MterpProfileBranch @ (self, shadow_frame, offset)
+ cmp r0, #0
+ bne MterpOnStackReplacement @ Note: offset must be in rINST
+ adds r2, rINST, rINST @ convert to bytes, check sign
+ ldr lr, [rSELF, #THREAD_FLAGS_OFFSET]
FETCH_ADVANCE_INST_RB r2 @ update rPC, load rINST
- ldrmi rIBASE, [rSELF, #THREAD_CURRENT_IBASE_OFFSET] @ refresh rIBASE
+ bmi MterpCheckSuspendAndContinue
+ GET_INST_OPCODE ip @ extract opcode from rINST
+ GOTO_OPCODE ip @ jump to next instruction
+.L_${opcode}_not_taken:
+ FETCH_ADVANCE_INST 2 @ update rPC, load rINST
GET_INST_OPCODE ip @ extract opcode from rINST
GOTO_OPCODE ip @ jump to next instruction
#else
@@ -25,10 +37,10 @@
GET_VREG r3, r1 @ r3<- vB
GET_VREG r2, r0 @ r2<- vA
ldr lr, [rSELF, #THREAD_FLAGS_OFFSET]
- FETCH_S r1, 1 @ r1<- branch offset, in code units
+ FETCH_S rINST, 1 @ rINST<- branch offset, in code units
cmp r2, r3 @ compare (vA, vB)
- mov${revcmp} r1, #2 @ r1<- BYTE branch dist for not-taken
- adds r2, r1, r1 @ convert to bytes, check sign
+ mov${revcmp} rINST, #2 @ rINST<- BYTE branch dist for not-taken
+ adds r2, rINST, rINST @ convert to bytes, check sign
FETCH_ADVANCE_INST_RB r2 @ update rPC, load rINST
bmi MterpCheckSuspendAndContinue
GET_INST_OPCODE ip @ extract opcode from rINST
diff --git a/runtime/interpreter/mterp/arm/footer.S b/runtime/interpreter/mterp/arm/footer.S
index 1dba856..3456a75 100644
--- a/runtime/interpreter/mterp/arm/footer.S
+++ b/runtime/interpreter/mterp/arm/footer.S
@@ -12,7 +12,6 @@
* has not yet been thrown. Just bail out to the reference interpreter to deal with it.
* TUNING: for consistency, we may want to just go ahead and handle these here.
*/
-#define MTERP_LOGGING 0
common_errDivideByZero:
EXPORT_PC
#if MTERP_LOGGING
@@ -103,8 +102,12 @@
ldr rIBASE, [rSELF, #THREAD_CURRENT_IBASE_OFFSET]
add rPC, r0, #CODEITEM_INSNS_OFFSET
add rPC, rPC, r1, lsl #1 @ generate new dex_pc_ptr
- str rPC, [rFP, #OFF_FP_DEX_PC_PTR]
+ /* Do we need to switch interpreters? */
+ bl MterpShouldSwitchInterpreters
+ cmp r0, #0
+ bne MterpFallback
/* resume execution at catch block */
+ EXPORT_PC
FETCH_INST
GET_INST_OPCODE ip
GOTO_OPCODE ip
@@ -116,12 +119,31 @@
*/
MterpCheckSuspendAndContinue:
ldr rIBASE, [rSELF, #THREAD_CURRENT_IBASE_OFFSET] @ refresh rIBASE
- EXPORT_PC
- mov r0, rSELF
ands lr, #(THREAD_SUSPEND_REQUEST | THREAD_CHECKPOINT_REQUEST)
- blne MterpSuspendCheck @ (self)
+ bne 1f
GET_INST_OPCODE ip @ extract opcode from rINST
GOTO_OPCODE ip @ jump to next instruction
+1:
+ EXPORT_PC
+ mov r0, rSELF
+ bl MterpSuspendCheck @ (self)
+ cmp r0, #0
+ bne MterpFallback
+ GET_INST_OPCODE ip @ extract opcode from rINST
+ GOTO_OPCODE ip @ jump to next instruction
+
+/*
+ * On-stack replacement has happened, and now we've returned from the compiled method.
+ */
+MterpOnStackReplacement:
+#if MTERP_LOGGING
+ mov r0, rSELF
+ add r1, rFP, #OFF_FP_SHADOWFRAME
+ mov r2, rINST
+ bl MterpLogOSR
+#endif
+ mov r0, #1 @ Signal normal return
+ b MterpDone
/*
* Bail out to reference interpreter.
diff --git a/runtime/interpreter/mterp/arm/header.S b/runtime/interpreter/mterp/arm/header.S
index b2370bf..298af8a 100644
--- a/runtime/interpreter/mterp/arm/header.S
+++ b/runtime/interpreter/mterp/arm/header.S
@@ -85,6 +85,9 @@
*/
#include "asm_support.h"
+#define MTERP_PROFILE_BRANCHES 1
+#define MTERP_LOGGING 0
+
/* During bringup, we'll use the shadow frame model instead of rFP */
/* single-purpose registers, given names for clarity */
#define rPC r4
@@ -109,14 +112,6 @@
#define OFF_FP_SHADOWFRAME (-SHADOWFRAME_VREGS_OFFSET)
/*
- *
- * The reference interpreter performs explicit suspect checks, which is somewhat wasteful.
- * Dalvik's interpreter folded suspend checks into the jump table mechanism, and eventually
- * mterp should do so as well.
- */
-#define MTERP_SUSPEND 0
-
-/*
* "export" the PC to dex_pc field in the shadow frame, f/b/o future exception objects. Must
* be done *before* something throws.
*
diff --git a/runtime/interpreter/mterp/arm/invoke.S b/runtime/interpreter/mterp/arm/invoke.S
index 7575865..e47dd1b 100644
--- a/runtime/interpreter/mterp/arm/invoke.S
+++ b/runtime/interpreter/mterp/arm/invoke.S
@@ -14,6 +14,9 @@
cmp r0, #0
beq MterpException
FETCH_ADVANCE_INST 3
+ bl MterpShouldSwitchInterpreters
+ cmp r0, #0
+ bne MterpFallback
GET_INST_OPCODE ip
GOTO_OPCODE ip
diff --git a/runtime/interpreter/mterp/arm/op_goto.S b/runtime/interpreter/mterp/arm/op_goto.S
index 9b3632a..eb1d429 100644
--- a/runtime/interpreter/mterp/arm/op_goto.S
+++ b/runtime/interpreter/mterp/arm/op_goto.S
@@ -6,20 +6,28 @@
*/
/* goto +AA */
/* tuning: use sbfx for 6t2+ targets */
-#if MTERP_SUSPEND
+#if MTERP_PROFILE_BRANCHES
mov r0, rINST, lsl #16 @ r0<- AAxx0000
- movs r1, r0, asr #24 @ r1<- ssssssAA (sign-extended)
- add r2, r1, r1 @ r2<- byte offset, set flags
- @ If backwards branch refresh rIBASE
- ldrmi rIBASE, [rSELF, #THREAD_CURRENT_IBASE_OFFSET] @ refresh handler base
+ movs rINST, r0, asr #24 @ rINST<- ssssssAA (sign-extended)
+ EXPORT_PC
+ mov r0, rSELF
+ add r1, rFP, #OFF_FP_SHADOWFRAME
+ mov r2, rINST
+ bl MterpProfileBranch @ (self, shadow_frame, offset)
+ cmp r0, #0
+ bne MterpOnStackReplacement @ Note: offset must be in rINST
+ ldr lr, [rSELF, #THREAD_FLAGS_OFFSET]
+ add r2, rINST, rINST @ r2<- byte offset, set flags
FETCH_ADVANCE_INST_RB r2 @ update rPC, load rINST
+ @ If backwards branch refresh rIBASE
+ bmi MterpCheckSuspendAndContinue
GET_INST_OPCODE ip @ extract opcode from rINST
GOTO_OPCODE ip @ jump to next instruction
#else
- ldr lr, [rSELF, #THREAD_FLAGS_OFFSET]
mov r0, rINST, lsl #16 @ r0<- AAxx0000
- movs r1, r0, asr #24 @ r1<- ssssssAA (sign-extended)
- add r2, r1, r1 @ r2<- byte offset, set flags
+ movs rINST, r0, asr #24 @ rINST<- ssssssAA (sign-extended)
+ ldr lr, [rSELF, #THREAD_FLAGS_OFFSET]
+ add r2, rINST, rINST @ r2<- byte offset, set flags
FETCH_ADVANCE_INST_RB r2 @ update rPC, load rINST
@ If backwards branch refresh rIBASE
bmi MterpCheckSuspendAndContinue
diff --git a/runtime/interpreter/mterp/arm/op_goto_16.S b/runtime/interpreter/mterp/arm/op_goto_16.S
index 2231acd..91639ca 100644
--- a/runtime/interpreter/mterp/arm/op_goto_16.S
+++ b/runtime/interpreter/mterp/arm/op_goto_16.S
@@ -5,17 +5,25 @@
* double to get a byte offset.
*/
/* goto/16 +AAAA */
-#if MTERP_SUSPEND
- FETCH_S r0, 1 @ r0<- ssssAAAA (sign-extended)
- adds r1, r0, r0 @ r1<- byte offset, flags set
+#if MTERP_PROFILE_BRANCHES
+ FETCH_S rINST, 1 @ rINST<- ssssAAAA (sign-extended)
+ EXPORT_PC
+ mov r0, rSELF
+ add r1, rFP, #OFF_FP_SHADOWFRAME
+ mov r2, rINST
+ bl MterpProfileBranch @ (self, shadow_frame, offset)
+ cmp r0, #0
+ bne MterpOnStackReplacement @ Note: offset must be in rINST
+ ldr lr, [rSELF, #THREAD_FLAGS_OFFSET]
+ adds r1, rINST, rINST @ r1<- byte offset, flags set
FETCH_ADVANCE_INST_RB r1 @ update rPC, load rINST
- ldrmi rIBASE, [rSELF, #THREAD_CURRENT_IBASE_OFFSET] @ refresh handler base
+ bmi MterpCheckSuspendAndContinue
GET_INST_OPCODE ip @ extract opcode from rINST
GOTO_OPCODE ip @ jump to next instruction
#else
- FETCH_S r0, 1 @ r0<- ssssAAAA (sign-extended)
+ FETCH_S rINST, 1 @ rINST<- ssssAAAA (sign-extended)
ldr lr, [rSELF, #THREAD_FLAGS_OFFSET]
- adds r1, r0, r0 @ r1<- byte offset, flags set
+ adds r1, rINST, rINST @ r1<- byte offset, flags set
FETCH_ADVANCE_INST_RB r1 @ update rPC, load rINST
bmi MterpCheckSuspendAndContinue
GET_INST_OPCODE ip @ extract opcode from rINST
diff --git a/runtime/interpreter/mterp/arm/op_goto_32.S b/runtime/interpreter/mterp/arm/op_goto_32.S
index 6b72ff5..e730b52 100644
--- a/runtime/interpreter/mterp/arm/op_goto_32.S
+++ b/runtime/interpreter/mterp/arm/op_goto_32.S
@@ -10,21 +10,29 @@
* offset to byte offset.
*/
/* goto/32 +AAAAAAAA */
-#if MTERP_SUSPEND
+#if MTERP_PROFILE_BRANCHES
FETCH r0, 1 @ r0<- aaaa (lo)
FETCH r1, 2 @ r1<- AAAA (hi)
- orr r0, r0, r1, lsl #16 @ r0<- AAAAaaaa
- adds r1, r0, r0 @ r1<- byte offset
+ orr rINST, r0, r1, lsl #16 @ rINST<- AAAAaaaa
+ EXPORT_PC
+ mov r0, rSELF
+ add r1, rFP, #OFF_FP_SHADOWFRAME
+ mov r2, rINST
+ bl MterpProfileBranch @ (self, shadow_frame, offset)
+ cmp r0, #0
+ bne MterpOnStackReplacement @ Note: offset must be in rINST
+ ldr lr, [rSELF, #THREAD_FLAGS_OFFSET]
+ adds r1, rINST, rINST @ r1<- byte offset
FETCH_ADVANCE_INST_RB r1 @ update rPC, load rINST
- ldrle rIBASE, [rSELF, #THREAD_CURRENT_IBASE_OFFSET] @ refresh handler base
+ ble MterpCheckSuspendAndContinue
GET_INST_OPCODE ip @ extract opcode from rINST
GOTO_OPCODE ip @ jump to next instruction
#else
FETCH r0, 1 @ r0<- aaaa (lo)
FETCH r1, 2 @ r1<- AAAA (hi)
+ orr rINST, r0, r1, lsl #16 @ rINST<- AAAAaaaa
ldr lr, [rSELF, #THREAD_FLAGS_OFFSET]
- orr r0, r0, r1, lsl #16 @ r0<- AAAAaaaa
- adds r1, r0, r0 @ r1<- byte offset
+ adds r1, rINST, rINST @ r1<- byte offset
FETCH_ADVANCE_INST_RB r1 @ update rPC, load rINST
ble MterpCheckSuspendAndContinue
GET_INST_OPCODE ip @ extract opcode from rINST
diff --git a/runtime/interpreter/mterp/arm/op_packed_switch.S b/runtime/interpreter/mterp/arm/op_packed_switch.S
index 1e3370e..4c369cb 100644
--- a/runtime/interpreter/mterp/arm/op_packed_switch.S
+++ b/runtime/interpreter/mterp/arm/op_packed_switch.S
@@ -9,7 +9,7 @@
* for: packed-switch, sparse-switch
*/
/* op vAA, +BBBB */
-#if MTERP_SUSPEND
+#if MTERP_PROFILE_BRANCHES
FETCH r0, 1 @ r0<- bbbb (lo)
FETCH r1, 2 @ r1<- BBBB (hi)
mov r3, rINST, lsr #8 @ r3<- AA
@@ -17,9 +17,18 @@
GET_VREG r1, r3 @ r1<- vAA
add r0, rPC, r0, lsl #1 @ r0<- PC + BBBBbbbb*2
bl $func @ r0<- code-unit branch offset
- adds r1, r0, r0 @ r1<- byte offset; clear V
- ldrle rIBASE, [rSELF, #THREAD_CURRENT_IBASE_OFFSET] @ refresh handler base
+ mov rINST, r0
+ EXPORT_PC
+ mov r0, rSELF
+ add r1, rFP, #OFF_FP_SHADOWFRAME
+ mov r2, rINST
+ bl MterpProfileBranch @ (self, shadow_frame, offset)
+ cmp r0, #0
+ bne MterpOnStackReplacement @ Note: offset must be in rINST
+ ldr lr, [rSELF, #THREAD_FLAGS_OFFSET]
+ adds r1, rINST, rINST @ r1<- byte offset; clear V
FETCH_ADVANCE_INST_RB r1 @ update rPC, load rINST
+ ble MterpCheckSuspendAndContinue
GET_INST_OPCODE ip @ extract opcode from rINST
GOTO_OPCODE ip @ jump to next instruction
#else
@@ -30,8 +39,9 @@
GET_VREG r1, r3 @ r1<- vAA
add r0, rPC, r0, lsl #1 @ r0<- PC + BBBBbbbb*2
bl $func @ r0<- code-unit branch offset
+ mov rINST, r0
ldr lr, [rSELF, #THREAD_FLAGS_OFFSET]
- adds r1, r0, r0 @ r1<- byte offset; clear V
+ adds r1, rINST, rINST @ r1<- byte offset; clear V
FETCH_ADVANCE_INST_RB r1 @ update rPC, load rINST
ble MterpCheckSuspendAndContinue
GET_INST_OPCODE ip @ extract opcode from rINST
diff --git a/runtime/interpreter/mterp/arm/zcmp.S b/runtime/interpreter/mterp/arm/zcmp.S
index 6e9ef55..800804d 100644
--- a/runtime/interpreter/mterp/arm/zcmp.S
+++ b/runtime/interpreter/mterp/arm/zcmp.S
@@ -6,25 +6,37 @@
* for: if-eqz, if-nez, if-ltz, if-gez, if-gtz, if-lez
*/
/* if-cmp vAA, +BBBB */
-#if MTERP_SUSPEND
+#if MTERP_PROFILE_BRANCHES
mov r0, rINST, lsr #8 @ r0<- AA
GET_VREG r2, r0 @ r2<- vAA
- FETCH_S r1, 1 @ r1<- branch offset, in code units
+ FETCH_S rINST, 1 @ rINST<- branch offset, in code units
+ ldr lr, [rSELF, #THREAD_FLAGS_OFFSET]
cmp r2, #0 @ compare (vA, 0)
- mov${revcmp} r1, #2 @ r1<- inst branch dist for not-taken
- adds r1, r1, r1 @ convert to bytes & set flags
+ b${revcmp} .L_${opcode}_not_taken
+ EXPORT_PC
+ mov r0, rSELF
+ add r1, rFP, #OFF_FP_SHADOWFRAME
+ mov r2, rINST
+ bl MterpProfileBranch @ (self, shadow_frame, offset)
+ cmp r0, #0
+ bne MterpOnStackReplacement @ Note: offset must be in rINST
+ adds r1, rINST, rINST @ convert to bytes & set flags
FETCH_ADVANCE_INST_RB r1 @ update rPC, load rINST
- ldrmi rIBASE, [rSELF, #THREAD_CURRENT_IBASE_OFFSET] @ refresh table base
+ bmi MterpCheckSuspendAndContinue
+ GET_INST_OPCODE ip @ extract opcode from rINST
+ GOTO_OPCODE ip @ jump to next instruction
+.L_${opcode}_not_taken:
+ FETCH_ADVANCE_INST 2 @ update rPC, load rINST
GET_INST_OPCODE ip @ extract opcode from rINST
GOTO_OPCODE ip @ jump to next instruction
#else
mov r0, rINST, lsr #8 @ r0<- AA
GET_VREG r2, r0 @ r2<- vAA
- FETCH_S r1, 1 @ r1<- branch offset, in code units
+ FETCH_S rINST, 1 @ rINST<- branch offset, in code units
ldr lr, [rSELF, #THREAD_FLAGS_OFFSET]
cmp r2, #0 @ compare (vA, 0)
- mov${revcmp} r1, #2 @ r1<- inst branch dist for not-taken
- adds r1, r1, r1 @ convert to bytes & set flags
+ mov${revcmp} rINST, #2 @ rINST<- inst branch dist for not-taken
+ adds r1, rINST, rINST @ convert to bytes & set flags
FETCH_ADVANCE_INST_RB r1 @ update rPC, load rINST
bmi MterpCheckSuspendAndContinue
GET_INST_OPCODE ip @ extract opcode from rINST
diff --git a/runtime/interpreter/mterp/arm64/bincmp.S b/runtime/interpreter/mterp/arm64/bincmp.S
index ecab2ce..ed850fc 100644
--- a/runtime/interpreter/mterp/arm64/bincmp.S
+++ b/runtime/interpreter/mterp/arm64/bincmp.S
@@ -6,17 +6,28 @@
* For: if-eq, if-ne, if-lt, if-ge, if-gt, if-le
*/
/* if-cmp vA, vB, +CCCC */
-#if MTERP_SUSPEND
- mov w1, wINST, lsr #12 // w1<- B
+#if MTERP_PROFILE_BRANCHES
+ lsr w1, wINST, #12 // w1<- B
ubfx w0, wINST, #8, #4 // w0<- A
GET_VREG w3, w1 // w3<- vB
GET_VREG w2, w0 // w2<- vA
- FETCH_S w1, 1 // w1<- branch offset, in code units
+ FETCH_S wINST, 1 // wINST<- branch offset, in code units
cmp w2, w3 // compare (vA, vB)
- mov${condition} w1, #2 // w1<- BYTE branch dist for not-taken
- adds w2, w1, w1 // convert to bytes, check sign
+ b.${condition} .L_${opcode}_taken
+ FETCH_ADVANCE_INST 2 // update rPC, load wINST
+ GET_INST_OPCODE ip // extract opcode from wINST
+ GOTO_OPCODE ip // jump to next instruction
+.L_${opcode}_taken:
+ EXPORT_PC
+ mov x0, xSELF
+ add x1, xFP, #OFF_FP_SHADOWFRAME
+ sbfm x2, xINST, 0, 31 // Sign extend branch offset
+ bl MterpProfileBranch // (self, shadow_frame, offset)
+ cbnz w0, MterpOnStackReplacement // Note: offset must be in xINST
+ ldr w7, [xSELF, #THREAD_FLAGS_OFFSET]
+ adds w2, wINST, wINST // convert to bytes, check sign
FETCH_ADVANCE_INST_RB w2 // update rPC, load wINST
- ldrmi rIBASE, [xSELF, #THREAD_CURRENT_IBASE_OFFSET] // refresh rIBASE
+ b.mi MterpCheckSuspendAndContinue
GET_INST_OPCODE ip // extract opcode from wINST
GOTO_OPCODE ip // jump to next instruction
#else
@@ -25,11 +36,11 @@
GET_VREG w3, w1 // w3<- vB
GET_VREG w2, w0 // w2<- vA
FETCH_S w1, 1 // w1<- branch offset, in code units
- ldr w7, [xSELF, #THREAD_FLAGS_OFFSET]
mov w0, #2 // Offset if branch not taken
cmp w2, w3 // compare (vA, vB)
- csel w1, w1, w0, ${condition} // Branch if true
- adds w2, w1, w1 // convert to bytes, check sign
+ csel wINST, w1, w0, ${condition} // Branch if true, stashing result in callee save reg.
+ ldr w7, [xSELF, #THREAD_FLAGS_OFFSET]
+ adds w2, wINST, wINST // convert to bytes, check sign
FETCH_ADVANCE_INST_RB w2 // update rPC, load wINST
b.mi MterpCheckSuspendAndContinue
GET_INST_OPCODE ip // extract opcode from wINST
diff --git a/runtime/interpreter/mterp/arm64/footer.S b/runtime/interpreter/mterp/arm64/footer.S
index b360539..aae78de 100644
--- a/runtime/interpreter/mterp/arm64/footer.S
+++ b/runtime/interpreter/mterp/arm64/footer.S
@@ -10,7 +10,6 @@
* has not yet been thrown. Just bail out to the reference interpreter to deal with it.
* TUNING: for consistency, we may want to just go ahead and handle these here.
*/
-#define MTERP_LOGGING 0
common_errDivideByZero:
EXPORT_PC
#if MTERP_LOGGING
@@ -99,8 +98,11 @@
ldr xIBASE, [xSELF, #THREAD_CURRENT_IBASE_OFFSET]
add xPC, x0, #CODEITEM_INSNS_OFFSET
add xPC, xPC, x1, lsl #1 // generate new dex_pc_ptr
- str xPC, [xFP, #OFF_FP_DEX_PC_PTR]
+ /* Do we need to switch interpreters? */
+ bl MterpShouldSwitchInterpreters
+ cbnz w0, MterpFallback
/* resume execution at catch block */
+ EXPORT_PC
FETCH_INST
GET_INST_OPCODE ip
GOTO_OPCODE ip
@@ -120,10 +122,24 @@
EXPORT_PC
mov x0, xSELF
bl MterpSuspendCheck // (self)
+ cbnz x0, MterpFallback // Something in the environment changed, switch interpreters
GET_INST_OPCODE ip // extract opcode from wINST
GOTO_OPCODE ip // jump to next instruction
/*
+ * On-stack replacement has happened, and now we've returned from the compiled method.
+ */
+MterpOnStackReplacement:
+#if MTERP_LOGGING
+ mov x0, xSELF
+ add x1, xFP, #OFF_FP_SHADOWFRAME
+ sbfm x2, xINST, 0, 31
+ bl MterpLogOSR
+#endif
+ mov x0, #1 // Signal normal return
+ b MterpDone
+
+/*
* Bail out to reference interpreter.
*/
MterpFallback:
diff --git a/runtime/interpreter/mterp/arm64/header.S b/runtime/interpreter/mterp/arm64/header.S
index 351a607..7223750 100644
--- a/runtime/interpreter/mterp/arm64/header.S
+++ b/runtime/interpreter/mterp/arm64/header.S
@@ -87,6 +87,9 @@
*/
#include "asm_support.h"
+#define MTERP_PROFILE_BRANCHES 1
+#define MTERP_LOGGING 0
+
/* During bringup, we'll use the shadow frame model instead of xFP */
/* single-purpose registers, given names for clarity */
#define xPC x20
@@ -114,14 +117,6 @@
#define OFF_FP_SHADOWFRAME (-SHADOWFRAME_VREGS_OFFSET)
/*
- *
- * The reference interpreter performs explicit suspect checks, which is somewhat wasteful.
- * Dalvik's interpreter folded suspend checks into the jump table mechanism, and eventually
- * mterp should do so as well.
- */
-#define MTERP_SUSPEND 0
-
-/*
* "export" the PC to dex_pc field in the shadow frame, f/b/o future exception objects. Must
* be done *before* something throws.
*
diff --git a/runtime/interpreter/mterp/arm64/invoke.S b/runtime/interpreter/mterp/arm64/invoke.S
index ff1974c..7a32df7 100644
--- a/runtime/interpreter/mterp/arm64/invoke.S
+++ b/runtime/interpreter/mterp/arm64/invoke.S
@@ -9,11 +9,12 @@
mov x0, xSELF
add x1, xFP, #OFF_FP_SHADOWFRAME
mov x2, xPC
- // and x3, xINST, 0xFFFF
mov x3, xINST
bl $helper
cbz w0, MterpException
FETCH_ADVANCE_INST 3
+ bl MterpShouldSwitchInterpreters
+ cbnz w0, MterpFallback
GET_INST_OPCODE ip
GOTO_OPCODE ip
diff --git a/runtime/interpreter/mterp/arm64/op_goto.S b/runtime/interpreter/mterp/arm64/op_goto.S
index db98a45..7e2f6a9 100644
--- a/runtime/interpreter/mterp/arm64/op_goto.S
+++ b/runtime/interpreter/mterp/arm64/op_goto.S
@@ -6,23 +6,20 @@
*/
/* goto +AA */
/* tuning: use sbfx for 6t2+ targets */
-#if MTERP_SUSPEND
- mov w0, wINST, lsl #16 // w0<- AAxx0000
- movs w1, w0, asr #24 // w1<- ssssssAA (sign-extended)
- add w2, w1, w1 // w2<- byte offset, set flags
- // If backwards branch refresh rIBASE
- ldrmi rIBASE, [xSELF, #THREAD_CURRENT_IBASE_OFFSET] // refresh handler base
- FETCH_ADVANCE_INST_RB w2 // update rPC, load wINST
- GET_INST_OPCODE ip // extract opcode from wINST
- GOTO_OPCODE ip // jump to next instruction
-#else
- ldr w7, [xSELF, #THREAD_FLAGS_OFFSET] // Preload flags for MterpCheckSuspendAndContinue
lsl w0, wINST, #16 // w0<- AAxx0000
- asr w0, w0, #24 // w0<- ssssssAA (sign-extended)
- adds w1, w0, w0 // Convert dalvik offset to byte offset, setting flags
+ asr wINST, w0, #24 // wINST<- ssssssAA (sign-extended)
+#if MTERP_PROFILE_BRANCHES
+ EXPORT_PC
+ mov x0, xSELF
+ add x1, xFP, #OFF_FP_SHADOWFRAME
+ sbfm x2, xINST, 0, 31
+ bl MterpProfileBranch // (self, shadow_frame, offset)
+ cbnz w0, MterpOnStackReplacement // Note: offset must be in wINST
+#endif
+ ldr w7, [xSELF, #THREAD_FLAGS_OFFSET] // Preload flags for MterpCheckSuspendAndContinue
+ adds w1, wINST, wINST // Convert dalvik offset to byte offset, setting flags
FETCH_ADVANCE_INST_RB w1 // load wINST and advance xPC
// If backwards branch refresh rIBASE
b.mi MterpCheckSuspendAndContinue
GET_INST_OPCODE ip // extract opcode from wINST
GOTO_OPCODE ip // jump to next instruction
-#endif
diff --git a/runtime/interpreter/mterp/arm64/op_goto_16.S b/runtime/interpreter/mterp/arm64/op_goto_16.S
index ff66a23..b2b9924 100644
--- a/runtime/interpreter/mterp/arm64/op_goto_16.S
+++ b/runtime/interpreter/mterp/arm64/op_goto_16.S
@@ -5,19 +5,18 @@
* double to get a byte offset.
*/
/* goto/16 +AAAA */
-#if MTERP_SUSPEND
- FETCH_S w0, 1 // w0<- ssssAAAA (sign-extended)
- adds w1, w0, w0 // w1<- byte offset, flags set
- FETCH_ADVANCE_INST_RB w1 // update rPC, load rINST
- ldrmi xIBASE, [xSELF, #THREAD_CURRENT_IBASE_OFFSET] // refresh handler base
- GET_INST_OPCODE ip // extract opcode from rINST
- GOTO_OPCODE ip // jump to next instruction
-#else
- FETCH_S w0, 1 // w0<- ssssAAAA (sign-extended)
+ FETCH_S wINST, 1 // wINST<- ssssAAAA (sign-extended)
+#if MTERP_PROFILE_BRANCHES
+ EXPORT_PC
+ mov x0, xSELF
+ add x1, xFP, #OFF_FP_SHADOWFRAME
+ sbfm x2, xINST, 0, 31
+ bl MterpProfileBranch // (self, shadow_frame, offset)
+ cbnz w0, MterpOnStackReplacement // Note: offset must be in xINST
+#endif
ldr w7, [xSELF, #THREAD_FLAGS_OFFSET]
- adds w1, w0, w0 // w1<- byte offset, flags set
+ adds w1, wINST, wINST // w1<- byte offset, flags set
FETCH_ADVANCE_INST_RB w1 // update rPC, load rINST
b.mi MterpCheckSuspendAndContinue
GET_INST_OPCODE ip // extract opcode from rINST
GOTO_OPCODE ip // jump to next instruction
-#endif
diff --git a/runtime/interpreter/mterp/arm64/op_goto_32.S b/runtime/interpreter/mterp/arm64/op_goto_32.S
index 8a6980e..b785857 100644
--- a/runtime/interpreter/mterp/arm64/op_goto_32.S
+++ b/runtime/interpreter/mterp/arm64/op_goto_32.S
@@ -10,23 +10,20 @@
* offset to byte offset.
*/
/* goto/32 +AAAAAAAA */
-#if MTERP_SUSPEND
FETCH w0, 1 // w0<- aaaa (lo)
FETCH w1, 2 // w1<- AAAA (hi)
- orr w0, w0, w1, lsl #16 // w0<- AAAAaaaa
- adds w1, w0, w0 // w1<- byte offset
- FETCH_ADVANCE_INST_RB w1 // update rPC, load xINST
- ldrle xIBASE, [xSELF, #THREAD_CURRENT_IBASE_OFFSET] // refresh handler base
- GET_INST_OPCODE ip // extract opcode from xINST
- GOTO_OPCODE ip // jump to next instruction
-#else
- FETCH w0, 1 // w0<- aaaa (lo)
- FETCH w1, 2 // w1<- AAAA (hi)
+ orr wINST, w0, w1, lsl #16 // wINST<- AAAAaaaa
+#if MTERP_PROFILE_BRANCHES
+ EXPORT_PC
+ mov x0, xSELF
+ add x1, xFP, #OFF_FP_SHADOWFRAME
+ sbfm x2, xINST, 0, 31
+ bl MterpProfileBranch // (self, shadow_frame, offset)
+ cbnz w0, MterpOnStackReplacement // Note: offset must be in xINST
+#endif
ldr w7, [xSELF, #THREAD_FLAGS_OFFSET]
- orr w0, w0, w1, lsl #16 // w0<- AAAAaaaa
- adds w1, w0, w0 // w1<- byte offset
+ adds w1, wINST, wINST // w1<- byte offset
FETCH_ADVANCE_INST_RB w1 // update rPC, load xINST
b.le MterpCheckSuspendAndContinue
GET_INST_OPCODE ip // extract opcode from xINST
GOTO_OPCODE ip // jump to next instruction
-#endif
diff --git a/runtime/interpreter/mterp/arm64/op_iget.S b/runtime/interpreter/mterp/arm64/op_iget.S
index 165c730..88533bd 100644
--- a/runtime/interpreter/mterp/arm64/op_iget.S
+++ b/runtime/interpreter/mterp/arm64/op_iget.S
@@ -1,4 +1,4 @@
-%default { "is_object":"0", "helper":"artGet32InstanceFromCode"}
+%default { "extend":"", "is_object":"0", "helper":"artGet32InstanceFromCode"}
/*
* General instance field get.
*
@@ -12,6 +12,7 @@
mov x3, xSELF // w3<- self
bl $helper
ldr x3, [xSELF, #THREAD_EXCEPTION_OFFSET]
+ $extend
ubfx w2, wINST, #8, #4 // w2<- A
PREFETCH_INST 2
cbnz x3, MterpPossibleException // bail out
diff --git a/runtime/interpreter/mterp/arm64/op_packed_switch.S b/runtime/interpreter/mterp/arm64/op_packed_switch.S
index f087d23..e8b4f04 100644
--- a/runtime/interpreter/mterp/arm64/op_packed_switch.S
+++ b/runtime/interpreter/mterp/arm64/op_packed_switch.S
@@ -9,20 +9,6 @@
* for: packed-switch, sparse-switch
*/
/* op vAA, +BBBB */
-#if MTERP_SUSPEND
- FETCH w0, 1 // w0<- bbbb (lo)
- FETCH w1, 2 // w1<- BBBB (hi)
- mov w3, wINST, lsr #8 // w3<- AA
- orr w0, w0, w1, lsl #16 // w0<- BBBBbbbb
- GET_VREG w1, w3 // w1<- vAA
- add w0, rPC, w0, lsl #1 // w0<- PC + BBBBbbbb*2
- bl $func // w0<- code-unit branch offset
- adds w1, w0, w0 // w1<- byte offset; clear V
- ldrle rIBASE, [xSELF, #THREAD_CURRENT_IBASE_OFFSET] // refresh handler base
- FETCH_ADVANCE_INST_RB w1 // update rPC, load wINST
- GET_INST_OPCODE ip // extract opcode from wINST
- GOTO_OPCODE ip // jump to next instruction
-#else
FETCH w0, 1 // w0<- bbbb (lo)
FETCH w1, 2 // w1<- BBBB (hi)
lsr w3, wINST, #8 // w3<- AA
@@ -30,10 +16,18 @@
GET_VREG w1, w3 // w1<- vAA
add x0, xPC, w0, lsl #1 // w0<- PC + BBBBbbbb*2
bl $func // w0<- code-unit branch offset
+ sbfm xINST, x0, 0, 31
+#if MTERP_PROFILE_BRANCHES
+ EXPORT_PC
+ mov x0, xSELF
+ add x1, xFP, #OFF_FP_SHADOWFRAME
+ mov x2, xINST
+ bl MterpProfileBranch // (self, shadow_frame, offset)
+ cbnz w0, MterpOnStackReplacement
+#endif
ldr w7, [xSELF, #THREAD_FLAGS_OFFSET]
- adds w1, w0, w0 // w1<- byte offset; clear V
+ adds w1, wINST, wINST // w1<- byte offset; clear V
FETCH_ADVANCE_INST_RB w1 // update rPC, load wINST
b.le MterpCheckSuspendAndContinue
GET_INST_OPCODE ip // extract opcode from wINST
GOTO_OPCODE ip // jump to next instruction
-#endif
diff --git a/runtime/interpreter/mterp/arm64/zcmp.S b/runtime/interpreter/mterp/arm64/zcmp.S
index d4856d2..e528d9f 100644
--- a/runtime/interpreter/mterp/arm64/zcmp.S
+++ b/runtime/interpreter/mterp/arm64/zcmp.S
@@ -6,26 +6,37 @@
* for: if-eqz, if-nez, if-ltz, if-gez, if-gtz, if-lez
*/
/* if-cmp vAA, +BBBB */
-#if MTERP_SUSPEND
- mov w0, wINST, lsr #8 // w0<- AA
+#if MTERP_PROFILE_BRANCHES
+ lsr w0, wINST, #8 // w0<- AA
GET_VREG w2, w0 // w2<- vAA
- FETCH_S w1, 1 // w1<- branch offset, in code units
+ FETCH_S wINST, 1 // w1<- branch offset, in code units
cmp w2, #0 // compare (vA, 0)
- mov${condition} w1, #2 // w1<- inst branch dist for not-taken
- adds w1, w1, w1 // convert to bytes & set flags
- FETCH_ADVANCE_INST_RB w1 // update rPC, load wINST
- ldrmi rIBASE, [xSELF, #THREAD_CURRENT_IBASE_OFFSET] // refresh table base
+ b.${condition} .L_${opcode}_taken
+ FETCH_ADVANCE_INST 2 // update rPC, load wINST
+ GET_INST_OPCODE ip // extract opcode from wINST
+ GOTO_OPCODE ip // jump to next instruction
+.L_${opcode}_taken:
+ EXPORT_PC
+ mov x0, xSELF
+ add x1, xFP, #OFF_FP_SHADOWFRAME
+ sbfm x2, xINST, 0, 31
+ bl MterpProfileBranch // (self, shadow_frame, offset)
+ cbnz w0, MterpOnStackReplacement // Note: offset must be in wINST
+ ldr w7, [xSELF, #THREAD_FLAGS_OFFSET]
+ adds w2, wINST, wINST // convert to bytes & set flags
+ FETCH_ADVANCE_INST_RB w2 // update rPC, load wINST
+ b.mi MterpCheckSuspendAndContinue
GET_INST_OPCODE ip // extract opcode from wINST
GOTO_OPCODE ip // jump to next instruction
#else
lsr w0, wINST, #8 // w0<- AA
GET_VREG w2, w0 // w2<- vAA
FETCH_S w1, 1 // w1<- branch offset, in code units
- ldr w7, [xSELF, #THREAD_FLAGS_OFFSET]
mov w0, #2 // Branch offset if not taken
cmp w2, #0 // compare (vA, 0)
- csel w1, w1, w0, ${condition} // Branch if true
- adds w2, w1, w1 // convert to bytes & set flags
+ csel wINST, w1, w0, ${condition} // Branch if true, stashing result in callee save reg
+ ldr w7, [xSELF, #THREAD_FLAGS_OFFSET]
+ adds w2, wINST, wINST // convert to bytes & set flags
FETCH_ADVANCE_INST_RB w2 // update rPC, load wINST
b.mi MterpCheckSuspendAndContinue
GET_INST_OPCODE ip // extract opcode from wINST
diff --git a/runtime/interpreter/mterp/mterp.cc b/runtime/interpreter/mterp/mterp.cc
index 0afd276..15745d2 100644
--- a/runtime/interpreter/mterp/mterp.cc
+++ b/runtime/interpreter/mterp/mterp.cc
@@ -20,6 +20,8 @@
#include "interpreter/interpreter_common.h"
#include "entrypoints/entrypoint_utils-inl.h"
#include "mterp.h"
+#include "jit/jit.h"
+#include "debugger.h"
namespace art {
namespace interpreter {
@@ -139,6 +141,20 @@
return entries[index];
}
+extern "C" bool MterpShouldSwitchInterpreters()
+ SHARED_REQUIRES(Locks::mutator_lock_) {
+ const instrumentation::Instrumentation* const instrumentation =
+ Runtime::Current()->GetInstrumentation();
+ bool unhandled_instrumentation;
+ // TODO: enable for other targets after more extensive testing.
+ if (kRuntimeISA == kArm64) {
+ unhandled_instrumentation = instrumentation->NonJitProfilingActive();
+ } else {
+ unhandled_instrumentation = instrumentation->IsActive();
+ }
+ return unhandled_instrumentation || Dbg::IsDebuggerActive();
+}
+
extern "C" bool MterpInvokeVirtual(Thread* self, ShadowFrame* shadow_frame,
uint16_t* dex_pc_ptr, uint16_t inst_data )
@@ -488,6 +504,14 @@
<< self->IsExceptionPending();
}
+extern "C" void MterpLogOSR(Thread* self, ShadowFrame* shadow_frame, int32_t offset)
+ SHARED_REQUIRES(Locks::mutator_lock_) {
+ UNUSED(self);
+ const Instruction* inst = Instruction::At(shadow_frame->GetDexPCPtr());
+ uint16_t inst_data = inst->Fetch16(0);
+ LOG(INFO) << "OSR: " << inst->Opcode(inst_data) << ", offset = " << offset;
+}
+
extern "C" void MterpLogSuspendFallback(Thread* self, ShadowFrame* shadow_frame, uint32_t flags)
SHARED_REQUIRES(Locks::mutator_lock_) {
UNUSED(self);
@@ -500,9 +524,10 @@
}
}
-extern "C" void MterpSuspendCheck(Thread* self)
+extern "C" bool MterpSuspendCheck(Thread* self)
SHARED_REQUIRES(Locks::mutator_lock_) {
self->AllowThreadSuspension();
+ return MterpShouldSwitchInterpreters();
}
extern "C" int artSet64IndirectStaticFromMterp(uint32_t field_idx, ArtMethod* referrer,
@@ -618,5 +643,15 @@
return obj->GetFieldObject<mirror::Object>(MemberOffset(field_offset));
}
+extern "C" bool MterpProfileBranch(Thread* self, ShadowFrame* shadow_frame, int32_t offset)
+ SHARED_REQUIRES(Locks::mutator_lock_) {
+ ArtMethod* method = shadow_frame->GetMethod();
+ JValue* result = shadow_frame->GetResultRegister();
+ uint32_t dex_pc = shadow_frame->GetDexPC();
+ const auto* const instrumentation = Runtime::Current()->GetInstrumentation();
+ instrumentation->Branch(self, method, dex_pc, offset);
+ return jit::Jit::MaybeDoOnStackReplacement(self, method, dex_pc, offset, result);
+}
+
} // namespace interpreter
} // namespace art
diff --git a/runtime/interpreter/mterp/mterp.h b/runtime/interpreter/mterp/mterp.h
index 90d21e9..8d24641 100644
--- a/runtime/interpreter/mterp/mterp.h
+++ b/runtime/interpreter/mterp/mterp.h
@@ -30,6 +30,7 @@
void InitMterpTls(Thread* self);
void CheckMterpAsmConstants();
+extern "C" bool MterpShouldSwitchInterpreters();
} // namespace interpreter
} // namespace art
diff --git a/runtime/interpreter/mterp/out/mterp_arm.S b/runtime/interpreter/mterp/out/mterp_arm.S
index ee19559..511c35b 100644
--- a/runtime/interpreter/mterp/out/mterp_arm.S
+++ b/runtime/interpreter/mterp/out/mterp_arm.S
@@ -92,6 +92,9 @@
*/
#include "asm_support.h"
+#define MTERP_PROFILE_BRANCHES 1
+#define MTERP_LOGGING 0
+
/* During bringup, we'll use the shadow frame model instead of rFP */
/* single-purpose registers, given names for clarity */
#define rPC r4
@@ -116,14 +119,6 @@
#define OFF_FP_SHADOWFRAME (-SHADOWFRAME_VREGS_OFFSET)
/*
- *
- * The reference interpreter performs explicit suspect checks, which is somewhat wasteful.
- * Dalvik's interpreter folded suspend checks into the jump table mechanism, and eventually
- * mterp should do so as well.
- */
-#define MTERP_SUSPEND 0
-
-/*
* "export" the PC to dex_pc field in the shadow frame, f/b/o future exception objects. Must
* be done *before* something throws.
*
@@ -1111,20 +1106,28 @@
*/
/* goto +AA */
/* tuning: use sbfx for 6t2+ targets */
-#if MTERP_SUSPEND
+#if MTERP_PROFILE_BRANCHES
mov r0, rINST, lsl #16 @ r0<- AAxx0000
- movs r1, r0, asr #24 @ r1<- ssssssAA (sign-extended)
- add r2, r1, r1 @ r2<- byte offset, set flags
- @ If backwards branch refresh rIBASE
- ldrmi rIBASE, [rSELF, #THREAD_CURRENT_IBASE_OFFSET] @ refresh handler base
+ movs rINST, r0, asr #24 @ rINST<- ssssssAA (sign-extended)
+ EXPORT_PC
+ mov r0, rSELF
+ add r1, rFP, #OFF_FP_SHADOWFRAME
+ mov r2, rINST
+ bl MterpProfileBranch @ (self, shadow_frame, offset)
+ cmp r0, #0
+ bne MterpOnStackReplacement @ Note: offset must be in rINST
+ ldr lr, [rSELF, #THREAD_FLAGS_OFFSET]
+ add r2, rINST, rINST @ r2<- byte offset, set flags
FETCH_ADVANCE_INST_RB r2 @ update rPC, load rINST
+ @ If backwards branch refresh rIBASE
+ bmi MterpCheckSuspendAndContinue
GET_INST_OPCODE ip @ extract opcode from rINST
GOTO_OPCODE ip @ jump to next instruction
#else
- ldr lr, [rSELF, #THREAD_FLAGS_OFFSET]
mov r0, rINST, lsl #16 @ r0<- AAxx0000
- movs r1, r0, asr #24 @ r1<- ssssssAA (sign-extended)
- add r2, r1, r1 @ r2<- byte offset, set flags
+ movs rINST, r0, asr #24 @ rINST<- ssssssAA (sign-extended)
+ ldr lr, [rSELF, #THREAD_FLAGS_OFFSET]
+ add r2, rINST, rINST @ r2<- byte offset, set flags
FETCH_ADVANCE_INST_RB r2 @ update rPC, load rINST
@ If backwards branch refresh rIBASE
bmi MterpCheckSuspendAndContinue
@@ -1143,17 +1146,25 @@
* double to get a byte offset.
*/
/* goto/16 +AAAA */
-#if MTERP_SUSPEND
- FETCH_S r0, 1 @ r0<- ssssAAAA (sign-extended)
- adds r1, r0, r0 @ r1<- byte offset, flags set
+#if MTERP_PROFILE_BRANCHES
+ FETCH_S rINST, 1 @ rINST<- ssssAAAA (sign-extended)
+ EXPORT_PC
+ mov r0, rSELF
+ add r1, rFP, #OFF_FP_SHADOWFRAME
+ mov r2, rINST
+ bl MterpProfileBranch @ (self, shadow_frame, offset)
+ cmp r0, #0
+ bne MterpOnStackReplacement @ Note: offset must be in rINST
+ ldr lr, [rSELF, #THREAD_FLAGS_OFFSET]
+ adds r1, rINST, rINST @ r1<- byte offset, flags set
FETCH_ADVANCE_INST_RB r1 @ update rPC, load rINST
- ldrmi rIBASE, [rSELF, #THREAD_CURRENT_IBASE_OFFSET] @ refresh handler base
+ bmi MterpCheckSuspendAndContinue
GET_INST_OPCODE ip @ extract opcode from rINST
GOTO_OPCODE ip @ jump to next instruction
#else
- FETCH_S r0, 1 @ r0<- ssssAAAA (sign-extended)
+ FETCH_S rINST, 1 @ rINST<- ssssAAAA (sign-extended)
ldr lr, [rSELF, #THREAD_FLAGS_OFFSET]
- adds r1, r0, r0 @ r1<- byte offset, flags set
+ adds r1, rINST, rINST @ r1<- byte offset, flags set
FETCH_ADVANCE_INST_RB r1 @ update rPC, load rINST
bmi MterpCheckSuspendAndContinue
GET_INST_OPCODE ip @ extract opcode from rINST
@@ -1176,21 +1187,29 @@
* offset to byte offset.
*/
/* goto/32 +AAAAAAAA */
-#if MTERP_SUSPEND
+#if MTERP_PROFILE_BRANCHES
FETCH r0, 1 @ r0<- aaaa (lo)
FETCH r1, 2 @ r1<- AAAA (hi)
- orr r0, r0, r1, lsl #16 @ r0<- AAAAaaaa
- adds r1, r0, r0 @ r1<- byte offset
+ orr rINST, r0, r1, lsl #16 @ rINST<- AAAAaaaa
+ EXPORT_PC
+ mov r0, rSELF
+ add r1, rFP, #OFF_FP_SHADOWFRAME
+ mov r2, rINST
+ bl MterpProfileBranch @ (self, shadow_frame, offset)
+ cmp r0, #0
+ bne MterpOnStackReplacement @ Note: offset must be in rINST
+ ldr lr, [rSELF, #THREAD_FLAGS_OFFSET]
+ adds r1, rINST, rINST @ r1<- byte offset
FETCH_ADVANCE_INST_RB r1 @ update rPC, load rINST
- ldrle rIBASE, [rSELF, #THREAD_CURRENT_IBASE_OFFSET] @ refresh handler base
+ ble MterpCheckSuspendAndContinue
GET_INST_OPCODE ip @ extract opcode from rINST
GOTO_OPCODE ip @ jump to next instruction
#else
FETCH r0, 1 @ r0<- aaaa (lo)
FETCH r1, 2 @ r1<- AAAA (hi)
+ orr rINST, r0, r1, lsl #16 @ rINST<- AAAAaaaa
ldr lr, [rSELF, #THREAD_FLAGS_OFFSET]
- orr r0, r0, r1, lsl #16 @ r0<- AAAAaaaa
- adds r1, r0, r0 @ r1<- byte offset
+ adds r1, rINST, rINST @ r1<- byte offset
FETCH_ADVANCE_INST_RB r1 @ update rPC, load rINST
ble MterpCheckSuspendAndContinue
GET_INST_OPCODE ip @ extract opcode from rINST
@@ -1211,7 +1230,7 @@
* for: packed-switch, sparse-switch
*/
/* op vAA, +BBBB */
-#if MTERP_SUSPEND
+#if MTERP_PROFILE_BRANCHES
FETCH r0, 1 @ r0<- bbbb (lo)
FETCH r1, 2 @ r1<- BBBB (hi)
mov r3, rINST, lsr #8 @ r3<- AA
@@ -1219,9 +1238,18 @@
GET_VREG r1, r3 @ r1<- vAA
add r0, rPC, r0, lsl #1 @ r0<- PC + BBBBbbbb*2
bl MterpDoPackedSwitch @ r0<- code-unit branch offset
- adds r1, r0, r0 @ r1<- byte offset; clear V
- ldrle rIBASE, [rSELF, #THREAD_CURRENT_IBASE_OFFSET] @ refresh handler base
+ mov rINST, r0
+ EXPORT_PC
+ mov r0, rSELF
+ add r1, rFP, #OFF_FP_SHADOWFRAME
+ mov r2, rINST
+ bl MterpProfileBranch @ (self, shadow_frame, offset)
+ cmp r0, #0
+ bne MterpOnStackReplacement @ Note: offset must be in rINST
+ ldr lr, [rSELF, #THREAD_FLAGS_OFFSET]
+ adds r1, rINST, rINST @ r1<- byte offset; clear V
FETCH_ADVANCE_INST_RB r1 @ update rPC, load rINST
+ ble MterpCheckSuspendAndContinue
GET_INST_OPCODE ip @ extract opcode from rINST
GOTO_OPCODE ip @ jump to next instruction
#else
@@ -1232,8 +1260,9 @@
GET_VREG r1, r3 @ r1<- vAA
add r0, rPC, r0, lsl #1 @ r0<- PC + BBBBbbbb*2
bl MterpDoPackedSwitch @ r0<- code-unit branch offset
+ mov rINST, r0
ldr lr, [rSELF, #THREAD_FLAGS_OFFSET]
- adds r1, r0, r0 @ r1<- byte offset; clear V
+ adds r1, rINST, rINST @ r1<- byte offset; clear V
FETCH_ADVANCE_INST_RB r1 @ update rPC, load rINST
ble MterpCheckSuspendAndContinue
GET_INST_OPCODE ip @ extract opcode from rINST
@@ -1255,7 +1284,7 @@
* for: packed-switch, sparse-switch
*/
/* op vAA, +BBBB */
-#if MTERP_SUSPEND
+#if MTERP_PROFILE_BRANCHES
FETCH r0, 1 @ r0<- bbbb (lo)
FETCH r1, 2 @ r1<- BBBB (hi)
mov r3, rINST, lsr #8 @ r3<- AA
@@ -1263,9 +1292,18 @@
GET_VREG r1, r3 @ r1<- vAA
add r0, rPC, r0, lsl #1 @ r0<- PC + BBBBbbbb*2
bl MterpDoSparseSwitch @ r0<- code-unit branch offset
- adds r1, r0, r0 @ r1<- byte offset; clear V
- ldrle rIBASE, [rSELF, #THREAD_CURRENT_IBASE_OFFSET] @ refresh handler base
+ mov rINST, r0
+ EXPORT_PC
+ mov r0, rSELF
+ add r1, rFP, #OFF_FP_SHADOWFRAME
+ mov r2, rINST
+ bl MterpProfileBranch @ (self, shadow_frame, offset)
+ cmp r0, #0
+ bne MterpOnStackReplacement @ Note: offset must be in rINST
+ ldr lr, [rSELF, #THREAD_FLAGS_OFFSET]
+ adds r1, rINST, rINST @ r1<- byte offset; clear V
FETCH_ADVANCE_INST_RB r1 @ update rPC, load rINST
+ ble MterpCheckSuspendAndContinue
GET_INST_OPCODE ip @ extract opcode from rINST
GOTO_OPCODE ip @ jump to next instruction
#else
@@ -1276,8 +1314,9 @@
GET_VREG r1, r3 @ r1<- vAA
add r0, rPC, r0, lsl #1 @ r0<- PC + BBBBbbbb*2
bl MterpDoSparseSwitch @ r0<- code-unit branch offset
+ mov rINST, r0
ldr lr, [rSELF, #THREAD_FLAGS_OFFSET]
- adds r1, r0, r0 @ r1<- byte offset; clear V
+ adds r1, rINST, rINST @ r1<- byte offset; clear V
FETCH_ADVANCE_INST_RB r1 @ update rPC, load rINST
ble MterpCheckSuspendAndContinue
GET_INST_OPCODE ip @ extract opcode from rINST
@@ -1495,17 +1534,29 @@
* For: if-eq, if-ne, if-lt, if-ge, if-gt, if-le
*/
/* if-cmp vA, vB, +CCCC */
-#if MTERP_SUSPEND
+#if MTERP_PROFILE_BRANCHES
mov r1, rINST, lsr #12 @ r1<- B
ubfx r0, rINST, #8, #4 @ r0<- A
GET_VREG r3, r1 @ r3<- vB
GET_VREG r2, r0 @ r2<- vA
- FETCH_S r1, 1 @ r1<- branch offset, in code units
+ FETCH_S rINST, 1 @ rINST<- branch offset, in code units
cmp r2, r3 @ compare (vA, vB)
- movne r1, #2 @ r1<- BYTE branch dist for not-taken
- adds r2, r1, r1 @ convert to bytes, check sign
+ bne .L_op_if_eq_not_taken
+ EXPORT_PC
+ mov r0, rSELF
+ add r1, rFP, #OFF_FP_SHADOWFRAME
+ mov r2, rINST
+ bl MterpProfileBranch @ (self, shadow_frame, offset)
+ cmp r0, #0
+ bne MterpOnStackReplacement @ Note: offset must be in rINST
+ adds r2, rINST, rINST @ convert to bytes, check sign
+ ldr lr, [rSELF, #THREAD_FLAGS_OFFSET]
FETCH_ADVANCE_INST_RB r2 @ update rPC, load rINST
- ldrmi rIBASE, [rSELF, #THREAD_CURRENT_IBASE_OFFSET] @ refresh rIBASE
+ bmi MterpCheckSuspendAndContinue
+ GET_INST_OPCODE ip @ extract opcode from rINST
+ GOTO_OPCODE ip @ jump to next instruction
+.L_op_if_eq_not_taken:
+ FETCH_ADVANCE_INST 2 @ update rPC, load rINST
GET_INST_OPCODE ip @ extract opcode from rINST
GOTO_OPCODE ip @ jump to next instruction
#else
@@ -1514,10 +1565,10 @@
GET_VREG r3, r1 @ r3<- vB
GET_VREG r2, r0 @ r2<- vA
ldr lr, [rSELF, #THREAD_FLAGS_OFFSET]
- FETCH_S r1, 1 @ r1<- branch offset, in code units
+ FETCH_S rINST, 1 @ rINST<- branch offset, in code units
cmp r2, r3 @ compare (vA, vB)
- movne r1, #2 @ r1<- BYTE branch dist for not-taken
- adds r2, r1, r1 @ convert to bytes, check sign
+ movne rINST, #2 @ rINST<- BYTE branch dist for not-taken
+ adds r2, rINST, rINST @ convert to bytes, check sign
FETCH_ADVANCE_INST_RB r2 @ update rPC, load rINST
bmi MterpCheckSuspendAndContinue
GET_INST_OPCODE ip @ extract opcode from rINST
@@ -1538,17 +1589,29 @@
* For: if-eq, if-ne, if-lt, if-ge, if-gt, if-le
*/
/* if-cmp vA, vB, +CCCC */
-#if MTERP_SUSPEND
+#if MTERP_PROFILE_BRANCHES
mov r1, rINST, lsr #12 @ r1<- B
ubfx r0, rINST, #8, #4 @ r0<- A
GET_VREG r3, r1 @ r3<- vB
GET_VREG r2, r0 @ r2<- vA
- FETCH_S r1, 1 @ r1<- branch offset, in code units
+ FETCH_S rINST, 1 @ rINST<- branch offset, in code units
cmp r2, r3 @ compare (vA, vB)
- moveq r1, #2 @ r1<- BYTE branch dist for not-taken
- adds r2, r1, r1 @ convert to bytes, check sign
+ beq .L_op_if_ne_not_taken
+ EXPORT_PC
+ mov r0, rSELF
+ add r1, rFP, #OFF_FP_SHADOWFRAME
+ mov r2, rINST
+ bl MterpProfileBranch @ (self, shadow_frame, offset)
+ cmp r0, #0
+ bne MterpOnStackReplacement @ Note: offset must be in rINST
+ adds r2, rINST, rINST @ convert to bytes, check sign
+ ldr lr, [rSELF, #THREAD_FLAGS_OFFSET]
FETCH_ADVANCE_INST_RB r2 @ update rPC, load rINST
- ldrmi rIBASE, [rSELF, #THREAD_CURRENT_IBASE_OFFSET] @ refresh rIBASE
+ bmi MterpCheckSuspendAndContinue
+ GET_INST_OPCODE ip @ extract opcode from rINST
+ GOTO_OPCODE ip @ jump to next instruction
+.L_op_if_ne_not_taken:
+ FETCH_ADVANCE_INST 2 @ update rPC, load rINST
GET_INST_OPCODE ip @ extract opcode from rINST
GOTO_OPCODE ip @ jump to next instruction
#else
@@ -1557,10 +1620,10 @@
GET_VREG r3, r1 @ r3<- vB
GET_VREG r2, r0 @ r2<- vA
ldr lr, [rSELF, #THREAD_FLAGS_OFFSET]
- FETCH_S r1, 1 @ r1<- branch offset, in code units
+ FETCH_S rINST, 1 @ rINST<- branch offset, in code units
cmp r2, r3 @ compare (vA, vB)
- moveq r1, #2 @ r1<- BYTE branch dist for not-taken
- adds r2, r1, r1 @ convert to bytes, check sign
+ moveq rINST, #2 @ rINST<- BYTE branch dist for not-taken
+ adds r2, rINST, rINST @ convert to bytes, check sign
FETCH_ADVANCE_INST_RB r2 @ update rPC, load rINST
bmi MterpCheckSuspendAndContinue
GET_INST_OPCODE ip @ extract opcode from rINST
@@ -1581,17 +1644,29 @@
* For: if-eq, if-ne, if-lt, if-ge, if-gt, if-le
*/
/* if-cmp vA, vB, +CCCC */
-#if MTERP_SUSPEND
+#if MTERP_PROFILE_BRANCHES
mov r1, rINST, lsr #12 @ r1<- B
ubfx r0, rINST, #8, #4 @ r0<- A
GET_VREG r3, r1 @ r3<- vB
GET_VREG r2, r0 @ r2<- vA
- FETCH_S r1, 1 @ r1<- branch offset, in code units
+ FETCH_S rINST, 1 @ rINST<- branch offset, in code units
cmp r2, r3 @ compare (vA, vB)
- movge r1, #2 @ r1<- BYTE branch dist for not-taken
- adds r2, r1, r1 @ convert to bytes, check sign
+ bge .L_op_if_lt_not_taken
+ EXPORT_PC
+ mov r0, rSELF
+ add r1, rFP, #OFF_FP_SHADOWFRAME
+ mov r2, rINST
+ bl MterpProfileBranch @ (self, shadow_frame, offset)
+ cmp r0, #0
+ bne MterpOnStackReplacement @ Note: offset must be in rINST
+ adds r2, rINST, rINST @ convert to bytes, check sign
+ ldr lr, [rSELF, #THREAD_FLAGS_OFFSET]
FETCH_ADVANCE_INST_RB r2 @ update rPC, load rINST
- ldrmi rIBASE, [rSELF, #THREAD_CURRENT_IBASE_OFFSET] @ refresh rIBASE
+ bmi MterpCheckSuspendAndContinue
+ GET_INST_OPCODE ip @ extract opcode from rINST
+ GOTO_OPCODE ip @ jump to next instruction
+.L_op_if_lt_not_taken:
+ FETCH_ADVANCE_INST 2 @ update rPC, load rINST
GET_INST_OPCODE ip @ extract opcode from rINST
GOTO_OPCODE ip @ jump to next instruction
#else
@@ -1600,10 +1675,10 @@
GET_VREG r3, r1 @ r3<- vB
GET_VREG r2, r0 @ r2<- vA
ldr lr, [rSELF, #THREAD_FLAGS_OFFSET]
- FETCH_S r1, 1 @ r1<- branch offset, in code units
+ FETCH_S rINST, 1 @ rINST<- branch offset, in code units
cmp r2, r3 @ compare (vA, vB)
- movge r1, #2 @ r1<- BYTE branch dist for not-taken
- adds r2, r1, r1 @ convert to bytes, check sign
+ movge rINST, #2 @ rINST<- BYTE branch dist for not-taken
+ adds r2, rINST, rINST @ convert to bytes, check sign
FETCH_ADVANCE_INST_RB r2 @ update rPC, load rINST
bmi MterpCheckSuspendAndContinue
GET_INST_OPCODE ip @ extract opcode from rINST
@@ -1624,17 +1699,29 @@
* For: if-eq, if-ne, if-lt, if-ge, if-gt, if-le
*/
/* if-cmp vA, vB, +CCCC */
-#if MTERP_SUSPEND
+#if MTERP_PROFILE_BRANCHES
mov r1, rINST, lsr #12 @ r1<- B
ubfx r0, rINST, #8, #4 @ r0<- A
GET_VREG r3, r1 @ r3<- vB
GET_VREG r2, r0 @ r2<- vA
- FETCH_S r1, 1 @ r1<- branch offset, in code units
+ FETCH_S rINST, 1 @ rINST<- branch offset, in code units
cmp r2, r3 @ compare (vA, vB)
- movlt r1, #2 @ r1<- BYTE branch dist for not-taken
- adds r2, r1, r1 @ convert to bytes, check sign
+ blt .L_op_if_ge_not_taken
+ EXPORT_PC
+ mov r0, rSELF
+ add r1, rFP, #OFF_FP_SHADOWFRAME
+ mov r2, rINST
+ bl MterpProfileBranch @ (self, shadow_frame, offset)
+ cmp r0, #0
+ bne MterpOnStackReplacement @ Note: offset must be in rINST
+ adds r2, rINST, rINST @ convert to bytes, check sign
+ ldr lr, [rSELF, #THREAD_FLAGS_OFFSET]
FETCH_ADVANCE_INST_RB r2 @ update rPC, load rINST
- ldrmi rIBASE, [rSELF, #THREAD_CURRENT_IBASE_OFFSET] @ refresh rIBASE
+ bmi MterpCheckSuspendAndContinue
+ GET_INST_OPCODE ip @ extract opcode from rINST
+ GOTO_OPCODE ip @ jump to next instruction
+.L_op_if_ge_not_taken:
+ FETCH_ADVANCE_INST 2 @ update rPC, load rINST
GET_INST_OPCODE ip @ extract opcode from rINST
GOTO_OPCODE ip @ jump to next instruction
#else
@@ -1643,10 +1730,10 @@
GET_VREG r3, r1 @ r3<- vB
GET_VREG r2, r0 @ r2<- vA
ldr lr, [rSELF, #THREAD_FLAGS_OFFSET]
- FETCH_S r1, 1 @ r1<- branch offset, in code units
+ FETCH_S rINST, 1 @ rINST<- branch offset, in code units
cmp r2, r3 @ compare (vA, vB)
- movlt r1, #2 @ r1<- BYTE branch dist for not-taken
- adds r2, r1, r1 @ convert to bytes, check sign
+ movlt rINST, #2 @ rINST<- BYTE branch dist for not-taken
+ adds r2, rINST, rINST @ convert to bytes, check sign
FETCH_ADVANCE_INST_RB r2 @ update rPC, load rINST
bmi MterpCheckSuspendAndContinue
GET_INST_OPCODE ip @ extract opcode from rINST
@@ -1667,17 +1754,29 @@
* For: if-eq, if-ne, if-lt, if-ge, if-gt, if-le
*/
/* if-cmp vA, vB, +CCCC */
-#if MTERP_SUSPEND
+#if MTERP_PROFILE_BRANCHES
mov r1, rINST, lsr #12 @ r1<- B
ubfx r0, rINST, #8, #4 @ r0<- A
GET_VREG r3, r1 @ r3<- vB
GET_VREG r2, r0 @ r2<- vA
- FETCH_S r1, 1 @ r1<- branch offset, in code units
+ FETCH_S rINST, 1 @ rINST<- branch offset, in code units
cmp r2, r3 @ compare (vA, vB)
- movle r1, #2 @ r1<- BYTE branch dist for not-taken
- adds r2, r1, r1 @ convert to bytes, check sign
+ ble .L_op_if_gt_not_taken
+ EXPORT_PC
+ mov r0, rSELF
+ add r1, rFP, #OFF_FP_SHADOWFRAME
+ mov r2, rINST
+ bl MterpProfileBranch @ (self, shadow_frame, offset)
+ cmp r0, #0
+ bne MterpOnStackReplacement @ Note: offset must be in rINST
+ adds r2, rINST, rINST @ convert to bytes, check sign
+ ldr lr, [rSELF, #THREAD_FLAGS_OFFSET]
FETCH_ADVANCE_INST_RB r2 @ update rPC, load rINST
- ldrmi rIBASE, [rSELF, #THREAD_CURRENT_IBASE_OFFSET] @ refresh rIBASE
+ bmi MterpCheckSuspendAndContinue
+ GET_INST_OPCODE ip @ extract opcode from rINST
+ GOTO_OPCODE ip @ jump to next instruction
+.L_op_if_gt_not_taken:
+ FETCH_ADVANCE_INST 2 @ update rPC, load rINST
GET_INST_OPCODE ip @ extract opcode from rINST
GOTO_OPCODE ip @ jump to next instruction
#else
@@ -1686,10 +1785,10 @@
GET_VREG r3, r1 @ r3<- vB
GET_VREG r2, r0 @ r2<- vA
ldr lr, [rSELF, #THREAD_FLAGS_OFFSET]
- FETCH_S r1, 1 @ r1<- branch offset, in code units
+ FETCH_S rINST, 1 @ rINST<- branch offset, in code units
cmp r2, r3 @ compare (vA, vB)
- movle r1, #2 @ r1<- BYTE branch dist for not-taken
- adds r2, r1, r1 @ convert to bytes, check sign
+ movle rINST, #2 @ rINST<- BYTE branch dist for not-taken
+ adds r2, rINST, rINST @ convert to bytes, check sign
FETCH_ADVANCE_INST_RB r2 @ update rPC, load rINST
bmi MterpCheckSuspendAndContinue
GET_INST_OPCODE ip @ extract opcode from rINST
@@ -1710,17 +1809,29 @@
* For: if-eq, if-ne, if-lt, if-ge, if-gt, if-le
*/
/* if-cmp vA, vB, +CCCC */
-#if MTERP_SUSPEND
+#if MTERP_PROFILE_BRANCHES
mov r1, rINST, lsr #12 @ r1<- B
ubfx r0, rINST, #8, #4 @ r0<- A
GET_VREG r3, r1 @ r3<- vB
GET_VREG r2, r0 @ r2<- vA
- FETCH_S r1, 1 @ r1<- branch offset, in code units
+ FETCH_S rINST, 1 @ rINST<- branch offset, in code units
cmp r2, r3 @ compare (vA, vB)
- movgt r1, #2 @ r1<- BYTE branch dist for not-taken
- adds r2, r1, r1 @ convert to bytes, check sign
+ bgt .L_op_if_le_not_taken
+ EXPORT_PC
+ mov r0, rSELF
+ add r1, rFP, #OFF_FP_SHADOWFRAME
+ mov r2, rINST
+ bl MterpProfileBranch @ (self, shadow_frame, offset)
+ cmp r0, #0
+ bne MterpOnStackReplacement @ Note: offset must be in rINST
+ adds r2, rINST, rINST @ convert to bytes, check sign
+ ldr lr, [rSELF, #THREAD_FLAGS_OFFSET]
FETCH_ADVANCE_INST_RB r2 @ update rPC, load rINST
- ldrmi rIBASE, [rSELF, #THREAD_CURRENT_IBASE_OFFSET] @ refresh rIBASE
+ bmi MterpCheckSuspendAndContinue
+ GET_INST_OPCODE ip @ extract opcode from rINST
+ GOTO_OPCODE ip @ jump to next instruction
+.L_op_if_le_not_taken:
+ FETCH_ADVANCE_INST 2 @ update rPC, load rINST
GET_INST_OPCODE ip @ extract opcode from rINST
GOTO_OPCODE ip @ jump to next instruction
#else
@@ -1729,10 +1840,10 @@
GET_VREG r3, r1 @ r3<- vB
GET_VREG r2, r0 @ r2<- vA
ldr lr, [rSELF, #THREAD_FLAGS_OFFSET]
- FETCH_S r1, 1 @ r1<- branch offset, in code units
+ FETCH_S rINST, 1 @ rINST<- branch offset, in code units
cmp r2, r3 @ compare (vA, vB)
- movgt r1, #2 @ r1<- BYTE branch dist for not-taken
- adds r2, r1, r1 @ convert to bytes, check sign
+ movgt rINST, #2 @ rINST<- BYTE branch dist for not-taken
+ adds r2, rINST, rINST @ convert to bytes, check sign
FETCH_ADVANCE_INST_RB r2 @ update rPC, load rINST
bmi MterpCheckSuspendAndContinue
GET_INST_OPCODE ip @ extract opcode from rINST
@@ -1753,25 +1864,37 @@
* for: if-eqz, if-nez, if-ltz, if-gez, if-gtz, if-lez
*/
/* if-cmp vAA, +BBBB */
-#if MTERP_SUSPEND
+#if MTERP_PROFILE_BRANCHES
mov r0, rINST, lsr #8 @ r0<- AA
GET_VREG r2, r0 @ r2<- vAA
- FETCH_S r1, 1 @ r1<- branch offset, in code units
+ FETCH_S rINST, 1 @ rINST<- branch offset, in code units
+ ldr lr, [rSELF, #THREAD_FLAGS_OFFSET]
cmp r2, #0 @ compare (vA, 0)
- movne r1, #2 @ r1<- inst branch dist for not-taken
- adds r1, r1, r1 @ convert to bytes & set flags
+ bne .L_op_if_eqz_not_taken
+ EXPORT_PC
+ mov r0, rSELF
+ add r1, rFP, #OFF_FP_SHADOWFRAME
+ mov r2, rINST
+ bl MterpProfileBranch @ (self, shadow_frame, offset)
+ cmp r0, #0
+ bne MterpOnStackReplacement @ Note: offset must be in rINST
+ adds r1, rINST, rINST @ convert to bytes & set flags
FETCH_ADVANCE_INST_RB r1 @ update rPC, load rINST
- ldrmi rIBASE, [rSELF, #THREAD_CURRENT_IBASE_OFFSET] @ refresh table base
+ bmi MterpCheckSuspendAndContinue
+ GET_INST_OPCODE ip @ extract opcode from rINST
+ GOTO_OPCODE ip @ jump to next instruction
+.L_op_if_eqz_not_taken:
+ FETCH_ADVANCE_INST 2 @ update rPC, load rINST
GET_INST_OPCODE ip @ extract opcode from rINST
GOTO_OPCODE ip @ jump to next instruction
#else
mov r0, rINST, lsr #8 @ r0<- AA
GET_VREG r2, r0 @ r2<- vAA
- FETCH_S r1, 1 @ r1<- branch offset, in code units
+ FETCH_S rINST, 1 @ rINST<- branch offset, in code units
ldr lr, [rSELF, #THREAD_FLAGS_OFFSET]
cmp r2, #0 @ compare (vA, 0)
- movne r1, #2 @ r1<- inst branch dist for not-taken
- adds r1, r1, r1 @ convert to bytes & set flags
+ movne rINST, #2 @ rINST<- inst branch dist for not-taken
+ adds r1, rINST, rINST @ convert to bytes & set flags
FETCH_ADVANCE_INST_RB r1 @ update rPC, load rINST
bmi MterpCheckSuspendAndContinue
GET_INST_OPCODE ip @ extract opcode from rINST
@@ -1792,25 +1915,37 @@
* for: if-eqz, if-nez, if-ltz, if-gez, if-gtz, if-lez
*/
/* if-cmp vAA, +BBBB */
-#if MTERP_SUSPEND
+#if MTERP_PROFILE_BRANCHES
mov r0, rINST, lsr #8 @ r0<- AA
GET_VREG r2, r0 @ r2<- vAA
- FETCH_S r1, 1 @ r1<- branch offset, in code units
+ FETCH_S rINST, 1 @ rINST<- branch offset, in code units
+ ldr lr, [rSELF, #THREAD_FLAGS_OFFSET]
cmp r2, #0 @ compare (vA, 0)
- moveq r1, #2 @ r1<- inst branch dist for not-taken
- adds r1, r1, r1 @ convert to bytes & set flags
+ beq .L_op_if_nez_not_taken
+ EXPORT_PC
+ mov r0, rSELF
+ add r1, rFP, #OFF_FP_SHADOWFRAME
+ mov r2, rINST
+ bl MterpProfileBranch @ (self, shadow_frame, offset)
+ cmp r0, #0
+ bne MterpOnStackReplacement @ Note: offset must be in rINST
+ adds r1, rINST, rINST @ convert to bytes & set flags
FETCH_ADVANCE_INST_RB r1 @ update rPC, load rINST
- ldrmi rIBASE, [rSELF, #THREAD_CURRENT_IBASE_OFFSET] @ refresh table base
+ bmi MterpCheckSuspendAndContinue
+ GET_INST_OPCODE ip @ extract opcode from rINST
+ GOTO_OPCODE ip @ jump to next instruction
+.L_op_if_nez_not_taken:
+ FETCH_ADVANCE_INST 2 @ update rPC, load rINST
GET_INST_OPCODE ip @ extract opcode from rINST
GOTO_OPCODE ip @ jump to next instruction
#else
mov r0, rINST, lsr #8 @ r0<- AA
GET_VREG r2, r0 @ r2<- vAA
- FETCH_S r1, 1 @ r1<- branch offset, in code units
+ FETCH_S rINST, 1 @ rINST<- branch offset, in code units
ldr lr, [rSELF, #THREAD_FLAGS_OFFSET]
cmp r2, #0 @ compare (vA, 0)
- moveq r1, #2 @ r1<- inst branch dist for not-taken
- adds r1, r1, r1 @ convert to bytes & set flags
+ moveq rINST, #2 @ rINST<- inst branch dist for not-taken
+ adds r1, rINST, rINST @ convert to bytes & set flags
FETCH_ADVANCE_INST_RB r1 @ update rPC, load rINST
bmi MterpCheckSuspendAndContinue
GET_INST_OPCODE ip @ extract opcode from rINST
@@ -1831,25 +1966,37 @@
* for: if-eqz, if-nez, if-ltz, if-gez, if-gtz, if-lez
*/
/* if-cmp vAA, +BBBB */
-#if MTERP_SUSPEND
+#if MTERP_PROFILE_BRANCHES
mov r0, rINST, lsr #8 @ r0<- AA
GET_VREG r2, r0 @ r2<- vAA
- FETCH_S r1, 1 @ r1<- branch offset, in code units
+ FETCH_S rINST, 1 @ rINST<- branch offset, in code units
+ ldr lr, [rSELF, #THREAD_FLAGS_OFFSET]
cmp r2, #0 @ compare (vA, 0)
- movge r1, #2 @ r1<- inst branch dist for not-taken
- adds r1, r1, r1 @ convert to bytes & set flags
+ bge .L_op_if_ltz_not_taken
+ EXPORT_PC
+ mov r0, rSELF
+ add r1, rFP, #OFF_FP_SHADOWFRAME
+ mov r2, rINST
+ bl MterpProfileBranch @ (self, shadow_frame, offset)
+ cmp r0, #0
+ bne MterpOnStackReplacement @ Note: offset must be in rINST
+ adds r1, rINST, rINST @ convert to bytes & set flags
FETCH_ADVANCE_INST_RB r1 @ update rPC, load rINST
- ldrmi rIBASE, [rSELF, #THREAD_CURRENT_IBASE_OFFSET] @ refresh table base
+ bmi MterpCheckSuspendAndContinue
+ GET_INST_OPCODE ip @ extract opcode from rINST
+ GOTO_OPCODE ip @ jump to next instruction
+.L_op_if_ltz_not_taken:
+ FETCH_ADVANCE_INST 2 @ update rPC, load rINST
GET_INST_OPCODE ip @ extract opcode from rINST
GOTO_OPCODE ip @ jump to next instruction
#else
mov r0, rINST, lsr #8 @ r0<- AA
GET_VREG r2, r0 @ r2<- vAA
- FETCH_S r1, 1 @ r1<- branch offset, in code units
+ FETCH_S rINST, 1 @ rINST<- branch offset, in code units
ldr lr, [rSELF, #THREAD_FLAGS_OFFSET]
cmp r2, #0 @ compare (vA, 0)
- movge r1, #2 @ r1<- inst branch dist for not-taken
- adds r1, r1, r1 @ convert to bytes & set flags
+ movge rINST, #2 @ rINST<- inst branch dist for not-taken
+ adds r1, rINST, rINST @ convert to bytes & set flags
FETCH_ADVANCE_INST_RB r1 @ update rPC, load rINST
bmi MterpCheckSuspendAndContinue
GET_INST_OPCODE ip @ extract opcode from rINST
@@ -1870,25 +2017,37 @@
* for: if-eqz, if-nez, if-ltz, if-gez, if-gtz, if-lez
*/
/* if-cmp vAA, +BBBB */
-#if MTERP_SUSPEND
+#if MTERP_PROFILE_BRANCHES
mov r0, rINST, lsr #8 @ r0<- AA
GET_VREG r2, r0 @ r2<- vAA
- FETCH_S r1, 1 @ r1<- branch offset, in code units
+ FETCH_S rINST, 1 @ rINST<- branch offset, in code units
+ ldr lr, [rSELF, #THREAD_FLAGS_OFFSET]
cmp r2, #0 @ compare (vA, 0)
- movlt r1, #2 @ r1<- inst branch dist for not-taken
- adds r1, r1, r1 @ convert to bytes & set flags
+ blt .L_op_if_gez_not_taken
+ EXPORT_PC
+ mov r0, rSELF
+ add r1, rFP, #OFF_FP_SHADOWFRAME
+ mov r2, rINST
+ bl MterpProfileBranch @ (self, shadow_frame, offset)
+ cmp r0, #0
+ bne MterpOnStackReplacement @ Note: offset must be in rINST
+ adds r1, rINST, rINST @ convert to bytes & set flags
FETCH_ADVANCE_INST_RB r1 @ update rPC, load rINST
- ldrmi rIBASE, [rSELF, #THREAD_CURRENT_IBASE_OFFSET] @ refresh table base
+ bmi MterpCheckSuspendAndContinue
+ GET_INST_OPCODE ip @ extract opcode from rINST
+ GOTO_OPCODE ip @ jump to next instruction
+.L_op_if_gez_not_taken:
+ FETCH_ADVANCE_INST 2 @ update rPC, load rINST
GET_INST_OPCODE ip @ extract opcode from rINST
GOTO_OPCODE ip @ jump to next instruction
#else
mov r0, rINST, lsr #8 @ r0<- AA
GET_VREG r2, r0 @ r2<- vAA
- FETCH_S r1, 1 @ r1<- branch offset, in code units
+ FETCH_S rINST, 1 @ rINST<- branch offset, in code units
ldr lr, [rSELF, #THREAD_FLAGS_OFFSET]
cmp r2, #0 @ compare (vA, 0)
- movlt r1, #2 @ r1<- inst branch dist for not-taken
- adds r1, r1, r1 @ convert to bytes & set flags
+ movlt rINST, #2 @ rINST<- inst branch dist for not-taken
+ adds r1, rINST, rINST @ convert to bytes & set flags
FETCH_ADVANCE_INST_RB r1 @ update rPC, load rINST
bmi MterpCheckSuspendAndContinue
GET_INST_OPCODE ip @ extract opcode from rINST
@@ -1909,25 +2068,37 @@
* for: if-eqz, if-nez, if-ltz, if-gez, if-gtz, if-lez
*/
/* if-cmp vAA, +BBBB */
-#if MTERP_SUSPEND
+#if MTERP_PROFILE_BRANCHES
mov r0, rINST, lsr #8 @ r0<- AA
GET_VREG r2, r0 @ r2<- vAA
- FETCH_S r1, 1 @ r1<- branch offset, in code units
+ FETCH_S rINST, 1 @ rINST<- branch offset, in code units
+ ldr lr, [rSELF, #THREAD_FLAGS_OFFSET]
cmp r2, #0 @ compare (vA, 0)
- movle r1, #2 @ r1<- inst branch dist for not-taken
- adds r1, r1, r1 @ convert to bytes & set flags
+ ble .L_op_if_gtz_not_taken
+ EXPORT_PC
+ mov r0, rSELF
+ add r1, rFP, #OFF_FP_SHADOWFRAME
+ mov r2, rINST
+ bl MterpProfileBranch @ (self, shadow_frame, offset)
+ cmp r0, #0
+ bne MterpOnStackReplacement @ Note: offset must be in rINST
+ adds r1, rINST, rINST @ convert to bytes & set flags
FETCH_ADVANCE_INST_RB r1 @ update rPC, load rINST
- ldrmi rIBASE, [rSELF, #THREAD_CURRENT_IBASE_OFFSET] @ refresh table base
+ bmi MterpCheckSuspendAndContinue
+ GET_INST_OPCODE ip @ extract opcode from rINST
+ GOTO_OPCODE ip @ jump to next instruction
+.L_op_if_gtz_not_taken:
+ FETCH_ADVANCE_INST 2 @ update rPC, load rINST
GET_INST_OPCODE ip @ extract opcode from rINST
GOTO_OPCODE ip @ jump to next instruction
#else
mov r0, rINST, lsr #8 @ r0<- AA
GET_VREG r2, r0 @ r2<- vAA
- FETCH_S r1, 1 @ r1<- branch offset, in code units
+ FETCH_S rINST, 1 @ rINST<- branch offset, in code units
ldr lr, [rSELF, #THREAD_FLAGS_OFFSET]
cmp r2, #0 @ compare (vA, 0)
- movle r1, #2 @ r1<- inst branch dist for not-taken
- adds r1, r1, r1 @ convert to bytes & set flags
+ movle rINST, #2 @ rINST<- inst branch dist for not-taken
+ adds r1, rINST, rINST @ convert to bytes & set flags
FETCH_ADVANCE_INST_RB r1 @ update rPC, load rINST
bmi MterpCheckSuspendAndContinue
GET_INST_OPCODE ip @ extract opcode from rINST
@@ -1948,25 +2119,37 @@
* for: if-eqz, if-nez, if-ltz, if-gez, if-gtz, if-lez
*/
/* if-cmp vAA, +BBBB */
-#if MTERP_SUSPEND
+#if MTERP_PROFILE_BRANCHES
mov r0, rINST, lsr #8 @ r0<- AA
GET_VREG r2, r0 @ r2<- vAA
- FETCH_S r1, 1 @ r1<- branch offset, in code units
+ FETCH_S rINST, 1 @ rINST<- branch offset, in code units
+ ldr lr, [rSELF, #THREAD_FLAGS_OFFSET]
cmp r2, #0 @ compare (vA, 0)
- movgt r1, #2 @ r1<- inst branch dist for not-taken
- adds r1, r1, r1 @ convert to bytes & set flags
+ bgt .L_op_if_lez_not_taken
+ EXPORT_PC
+ mov r0, rSELF
+ add r1, rFP, #OFF_FP_SHADOWFRAME
+ mov r2, rINST
+ bl MterpProfileBranch @ (self, shadow_frame, offset)
+ cmp r0, #0
+ bne MterpOnStackReplacement @ Note: offset must be in rINST
+ adds r1, rINST, rINST @ convert to bytes & set flags
FETCH_ADVANCE_INST_RB r1 @ update rPC, load rINST
- ldrmi rIBASE, [rSELF, #THREAD_CURRENT_IBASE_OFFSET] @ refresh table base
+ bmi MterpCheckSuspendAndContinue
+ GET_INST_OPCODE ip @ extract opcode from rINST
+ GOTO_OPCODE ip @ jump to next instruction
+.L_op_if_lez_not_taken:
+ FETCH_ADVANCE_INST 2 @ update rPC, load rINST
GET_INST_OPCODE ip @ extract opcode from rINST
GOTO_OPCODE ip @ jump to next instruction
#else
mov r0, rINST, lsr #8 @ r0<- AA
GET_VREG r2, r0 @ r2<- vAA
- FETCH_S r1, 1 @ r1<- branch offset, in code units
+ FETCH_S rINST, 1 @ rINST<- branch offset, in code units
ldr lr, [rSELF, #THREAD_FLAGS_OFFSET]
cmp r2, #0 @ compare (vA, 0)
- movgt r1, #2 @ r1<- inst branch dist for not-taken
- adds r1, r1, r1 @ convert to bytes & set flags
+ movgt rINST, #2 @ rINST<- inst branch dist for not-taken
+ adds r1, rINST, rINST @ convert to bytes & set flags
FETCH_ADVANCE_INST_RB r1 @ update rPC, load rINST
bmi MterpCheckSuspendAndContinue
GET_INST_OPCODE ip @ extract opcode from rINST
@@ -3294,6 +3477,9 @@
cmp r0, #0
beq MterpException
FETCH_ADVANCE_INST 3
+ bl MterpShouldSwitchInterpreters
+ cmp r0, #0
+ bne MterpFallback
GET_INST_OPCODE ip
GOTO_OPCODE ip
@@ -3326,6 +3512,9 @@
cmp r0, #0
beq MterpException
FETCH_ADVANCE_INST 3
+ bl MterpShouldSwitchInterpreters
+ cmp r0, #0
+ bne MterpFallback
GET_INST_OPCODE ip
GOTO_OPCODE ip
@@ -3358,6 +3547,9 @@
cmp r0, #0
beq MterpException
FETCH_ADVANCE_INST 3
+ bl MterpShouldSwitchInterpreters
+ cmp r0, #0
+ bne MterpFallback
GET_INST_OPCODE ip
GOTO_OPCODE ip
@@ -3383,6 +3575,9 @@
cmp r0, #0
beq MterpException
FETCH_ADVANCE_INST 3
+ bl MterpShouldSwitchInterpreters
+ cmp r0, #0
+ bne MterpFallback
GET_INST_OPCODE ip
GOTO_OPCODE ip
@@ -3409,6 +3604,9 @@
cmp r0, #0
beq MterpException
FETCH_ADVANCE_INST 3
+ bl MterpShouldSwitchInterpreters
+ cmp r0, #0
+ bne MterpFallback
GET_INST_OPCODE ip
GOTO_OPCODE ip
@@ -3453,6 +3651,9 @@
cmp r0, #0
beq MterpException
FETCH_ADVANCE_INST 3
+ bl MterpShouldSwitchInterpreters
+ cmp r0, #0
+ bne MterpFallback
GET_INST_OPCODE ip
GOTO_OPCODE ip
@@ -3478,6 +3679,9 @@
cmp r0, #0
beq MterpException
FETCH_ADVANCE_INST 3
+ bl MterpShouldSwitchInterpreters
+ cmp r0, #0
+ bne MterpFallback
GET_INST_OPCODE ip
GOTO_OPCODE ip
@@ -3503,6 +3707,9 @@
cmp r0, #0
beq MterpException
FETCH_ADVANCE_INST 3
+ bl MterpShouldSwitchInterpreters
+ cmp r0, #0
+ bne MterpFallback
GET_INST_OPCODE ip
GOTO_OPCODE ip
@@ -3528,6 +3735,9 @@
cmp r0, #0
beq MterpException
FETCH_ADVANCE_INST 3
+ bl MterpShouldSwitchInterpreters
+ cmp r0, #0
+ bne MterpFallback
GET_INST_OPCODE ip
GOTO_OPCODE ip
@@ -3553,6 +3763,9 @@
cmp r0, #0
beq MterpException
FETCH_ADVANCE_INST 3
+ bl MterpShouldSwitchInterpreters
+ cmp r0, #0
+ bne MterpFallback
GET_INST_OPCODE ip
GOTO_OPCODE ip
@@ -7284,6 +7497,9 @@
cmp r0, #0
beq MterpException
FETCH_ADVANCE_INST 3
+ bl MterpShouldSwitchInterpreters
+ cmp r0, #0
+ bne MterpFallback
GET_INST_OPCODE ip
GOTO_OPCODE ip
@@ -7309,6 +7525,9 @@
cmp r0, #0
beq MterpException
FETCH_ADVANCE_INST 3
+ bl MterpShouldSwitchInterpreters
+ cmp r0, #0
+ bne MterpFallback
GET_INST_OPCODE ip
GOTO_OPCODE ip
@@ -12098,7 +12317,6 @@
* has not yet been thrown. Just bail out to the reference interpreter to deal with it.
* TUNING: for consistency, we may want to just go ahead and handle these here.
*/
-#define MTERP_LOGGING 0
common_errDivideByZero:
EXPORT_PC
#if MTERP_LOGGING
@@ -12189,8 +12407,12 @@
ldr rIBASE, [rSELF, #THREAD_CURRENT_IBASE_OFFSET]
add rPC, r0, #CODEITEM_INSNS_OFFSET
add rPC, rPC, r1, lsl #1 @ generate new dex_pc_ptr
- str rPC, [rFP, #OFF_FP_DEX_PC_PTR]
+ /* Do we need to switch interpreters? */
+ bl MterpShouldSwitchInterpreters
+ cmp r0, #0
+ bne MterpFallback
/* resume execution at catch block */
+ EXPORT_PC
FETCH_INST
GET_INST_OPCODE ip
GOTO_OPCODE ip
@@ -12202,12 +12424,31 @@
*/
MterpCheckSuspendAndContinue:
ldr rIBASE, [rSELF, #THREAD_CURRENT_IBASE_OFFSET] @ refresh rIBASE
- EXPORT_PC
- mov r0, rSELF
ands lr, #(THREAD_SUSPEND_REQUEST | THREAD_CHECKPOINT_REQUEST)
- blne MterpSuspendCheck @ (self)
+ bne 1f
GET_INST_OPCODE ip @ extract opcode from rINST
GOTO_OPCODE ip @ jump to next instruction
+1:
+ EXPORT_PC
+ mov r0, rSELF
+ bl MterpSuspendCheck @ (self)
+ cmp r0, #0
+ bne MterpFallback
+ GET_INST_OPCODE ip @ extract opcode from rINST
+ GOTO_OPCODE ip @ jump to next instruction
+
+/*
+ * On-stack replacement has happened, and now we've returned from the compiled method.
+ */
+MterpOnStackReplacement:
+#if MTERP_LOGGING
+ mov r0, rSELF
+ add r1, rFP, #OFF_FP_SHADOWFRAME
+ mov r2, rINST
+ bl MterpLogOSR
+#endif
+ mov r0, #1 @ Signal normal return
+ b MterpDone
/*
* Bail out to reference interpreter.
diff --git a/runtime/interpreter/mterp/out/mterp_arm64.S b/runtime/interpreter/mterp/out/mterp_arm64.S
index e9d28ab..e4825f0 100644
--- a/runtime/interpreter/mterp/out/mterp_arm64.S
+++ b/runtime/interpreter/mterp/out/mterp_arm64.S
@@ -94,6 +94,9 @@
*/
#include "asm_support.h"
+#define MTERP_PROFILE_BRANCHES 1
+#define MTERP_LOGGING 0
+
/* During bringup, we'll use the shadow frame model instead of xFP */
/* single-purpose registers, given names for clarity */
#define xPC x20
@@ -121,14 +124,6 @@
#define OFF_FP_SHADOWFRAME (-SHADOWFRAME_VREGS_OFFSET)
/*
- *
- * The reference interpreter performs explicit suspect checks, which is somewhat wasteful.
- * Dalvik's interpreter folded suspend checks into the jump table mechanism, and eventually
- * mterp should do so as well.
- */
-#define MTERP_SUSPEND 0
-
-/*
* "export" the PC to dex_pc field in the shadow frame, f/b/o future exception objects. Must
* be done *before* something throws.
*
@@ -1087,26 +1082,23 @@
*/
/* goto +AA */
/* tuning: use sbfx for 6t2+ targets */
-#if MTERP_SUSPEND
- mov w0, wINST, lsl #16 // w0<- AAxx0000
- movs w1, w0, asr #24 // w1<- ssssssAA (sign-extended)
- add w2, w1, w1 // w2<- byte offset, set flags
- // If backwards branch refresh rIBASE
- ldrmi rIBASE, [xSELF, #THREAD_CURRENT_IBASE_OFFSET] // refresh handler base
- FETCH_ADVANCE_INST_RB w2 // update rPC, load wINST
- GET_INST_OPCODE ip // extract opcode from wINST
- GOTO_OPCODE ip // jump to next instruction
-#else
- ldr w7, [xSELF, #THREAD_FLAGS_OFFSET] // Preload flags for MterpCheckSuspendAndContinue
lsl w0, wINST, #16 // w0<- AAxx0000
- asr w0, w0, #24 // w0<- ssssssAA (sign-extended)
- adds w1, w0, w0 // Convert dalvik offset to byte offset, setting flags
+ asr wINST, w0, #24 // wINST<- ssssssAA (sign-extended)
+#if MTERP_PROFILE_BRANCHES
+ EXPORT_PC
+ mov x0, xSELF
+ add x1, xFP, #OFF_FP_SHADOWFRAME
+ sbfm x2, xINST, 0, 31
+ bl MterpProfileBranch // (self, shadow_frame, offset)
+ cbnz w0, MterpOnStackReplacement // Note: offset must be in wINST
+#endif
+ ldr w7, [xSELF, #THREAD_FLAGS_OFFSET] // Preload flags for MterpCheckSuspendAndContinue
+ adds w1, wINST, wINST // Convert dalvik offset to byte offset, setting flags
FETCH_ADVANCE_INST_RB w1 // load wINST and advance xPC
// If backwards branch refresh rIBASE
b.mi MterpCheckSuspendAndContinue
GET_INST_OPCODE ip // extract opcode from wINST
GOTO_OPCODE ip // jump to next instruction
-#endif
/* ------------------------------ */
.balign 128
@@ -1119,22 +1111,21 @@
* double to get a byte offset.
*/
/* goto/16 +AAAA */
-#if MTERP_SUSPEND
- FETCH_S w0, 1 // w0<- ssssAAAA (sign-extended)
- adds w1, w0, w0 // w1<- byte offset, flags set
- FETCH_ADVANCE_INST_RB w1 // update rPC, load rINST
- ldrmi xIBASE, [xSELF, #THREAD_CURRENT_IBASE_OFFSET] // refresh handler base
- GET_INST_OPCODE ip // extract opcode from rINST
- GOTO_OPCODE ip // jump to next instruction
-#else
- FETCH_S w0, 1 // w0<- ssssAAAA (sign-extended)
+ FETCH_S wINST, 1 // wINST<- ssssAAAA (sign-extended)
+#if MTERP_PROFILE_BRANCHES
+ EXPORT_PC
+ mov x0, xSELF
+ add x1, xFP, #OFF_FP_SHADOWFRAME
+ sbfm x2, xINST, 0, 31
+ bl MterpProfileBranch // (self, shadow_frame, offset)
+ cbnz w0, MterpOnStackReplacement // Note: offset must be in xINST
+#endif
ldr w7, [xSELF, #THREAD_FLAGS_OFFSET]
- adds w1, w0, w0 // w1<- byte offset, flags set
+ adds w1, wINST, wINST // w1<- byte offset, flags set
FETCH_ADVANCE_INST_RB w1 // update rPC, load rINST
b.mi MterpCheckSuspendAndContinue
GET_INST_OPCODE ip // extract opcode from rINST
GOTO_OPCODE ip // jump to next instruction
-#endif
/* ------------------------------ */
.balign 128
@@ -1152,26 +1143,23 @@
* offset to byte offset.
*/
/* goto/32 +AAAAAAAA */
-#if MTERP_SUSPEND
FETCH w0, 1 // w0<- aaaa (lo)
FETCH w1, 2 // w1<- AAAA (hi)
- orr w0, w0, w1, lsl #16 // w0<- AAAAaaaa
- adds w1, w0, w0 // w1<- byte offset
- FETCH_ADVANCE_INST_RB w1 // update rPC, load xINST
- ldrle xIBASE, [xSELF, #THREAD_CURRENT_IBASE_OFFSET] // refresh handler base
- GET_INST_OPCODE ip // extract opcode from xINST
- GOTO_OPCODE ip // jump to next instruction
-#else
- FETCH w0, 1 // w0<- aaaa (lo)
- FETCH w1, 2 // w1<- AAAA (hi)
+ orr wINST, w0, w1, lsl #16 // wINST<- AAAAaaaa
+#if MTERP_PROFILE_BRANCHES
+ EXPORT_PC
+ mov x0, xSELF
+ add x1, xFP, #OFF_FP_SHADOWFRAME
+ sbfm x2, xINST, 0, 31
+ bl MterpProfileBranch // (self, shadow_frame, offset)
+ cbnz w0, MterpOnStackReplacement // Note: offset must be in xINST
+#endif
ldr w7, [xSELF, #THREAD_FLAGS_OFFSET]
- orr w0, w0, w1, lsl #16 // w0<- AAAAaaaa
- adds w1, w0, w0 // w1<- byte offset
+ adds w1, wINST, wINST // w1<- byte offset
FETCH_ADVANCE_INST_RB w1 // update rPC, load xINST
b.le MterpCheckSuspendAndContinue
GET_INST_OPCODE ip // extract opcode from xINST
GOTO_OPCODE ip // jump to next instruction
-#endif
/* ------------------------------ */
.balign 128
@@ -1187,20 +1175,6 @@
* for: packed-switch, sparse-switch
*/
/* op vAA, +BBBB */
-#if MTERP_SUSPEND
- FETCH w0, 1 // w0<- bbbb (lo)
- FETCH w1, 2 // w1<- BBBB (hi)
- mov w3, wINST, lsr #8 // w3<- AA
- orr w0, w0, w1, lsl #16 // w0<- BBBBbbbb
- GET_VREG w1, w3 // w1<- vAA
- add w0, rPC, w0, lsl #1 // w0<- PC + BBBBbbbb*2
- bl MterpDoPackedSwitch // w0<- code-unit branch offset
- adds w1, w0, w0 // w1<- byte offset; clear V
- ldrle rIBASE, [xSELF, #THREAD_CURRENT_IBASE_OFFSET] // refresh handler base
- FETCH_ADVANCE_INST_RB w1 // update rPC, load wINST
- GET_INST_OPCODE ip // extract opcode from wINST
- GOTO_OPCODE ip // jump to next instruction
-#else
FETCH w0, 1 // w0<- bbbb (lo)
FETCH w1, 2 // w1<- BBBB (hi)
lsr w3, wINST, #8 // w3<- AA
@@ -1208,13 +1182,21 @@
GET_VREG w1, w3 // w1<- vAA
add x0, xPC, w0, lsl #1 // w0<- PC + BBBBbbbb*2
bl MterpDoPackedSwitch // w0<- code-unit branch offset
+ sbfm xINST, x0, 0, 31
+#if MTERP_PROFILE_BRANCHES
+ EXPORT_PC
+ mov x0, xSELF
+ add x1, xFP, #OFF_FP_SHADOWFRAME
+ mov x2, xINST
+ bl MterpProfileBranch // (self, shadow_frame, offset)
+ cbnz w0, MterpOnStackReplacement
+#endif
ldr w7, [xSELF, #THREAD_FLAGS_OFFSET]
- adds w1, w0, w0 // w1<- byte offset; clear V
+ adds w1, wINST, wINST // w1<- byte offset; clear V
FETCH_ADVANCE_INST_RB w1 // update rPC, load wINST
b.le MterpCheckSuspendAndContinue
GET_INST_OPCODE ip // extract opcode from wINST
GOTO_OPCODE ip // jump to next instruction
-#endif
/* ------------------------------ */
.balign 128
@@ -1231,20 +1213,6 @@
* for: packed-switch, sparse-switch
*/
/* op vAA, +BBBB */
-#if MTERP_SUSPEND
- FETCH w0, 1 // w0<- bbbb (lo)
- FETCH w1, 2 // w1<- BBBB (hi)
- mov w3, wINST, lsr #8 // w3<- AA
- orr w0, w0, w1, lsl #16 // w0<- BBBBbbbb
- GET_VREG w1, w3 // w1<- vAA
- add w0, rPC, w0, lsl #1 // w0<- PC + BBBBbbbb*2
- bl MterpDoSparseSwitch // w0<- code-unit branch offset
- adds w1, w0, w0 // w1<- byte offset; clear V
- ldrle rIBASE, [xSELF, #THREAD_CURRENT_IBASE_OFFSET] // refresh handler base
- FETCH_ADVANCE_INST_RB w1 // update rPC, load wINST
- GET_INST_OPCODE ip // extract opcode from wINST
- GOTO_OPCODE ip // jump to next instruction
-#else
FETCH w0, 1 // w0<- bbbb (lo)
FETCH w1, 2 // w1<- BBBB (hi)
lsr w3, wINST, #8 // w3<- AA
@@ -1252,13 +1220,21 @@
GET_VREG w1, w3 // w1<- vAA
add x0, xPC, w0, lsl #1 // w0<- PC + BBBBbbbb*2
bl MterpDoSparseSwitch // w0<- code-unit branch offset
+ sbfm xINST, x0, 0, 31
+#if MTERP_PROFILE_BRANCHES
+ EXPORT_PC
+ mov x0, xSELF
+ add x1, xFP, #OFF_FP_SHADOWFRAME
+ mov x2, xINST
+ bl MterpProfileBranch // (self, shadow_frame, offset)
+ cbnz w0, MterpOnStackReplacement
+#endif
ldr w7, [xSELF, #THREAD_FLAGS_OFFSET]
- adds w1, w0, w0 // w1<- byte offset; clear V
+ adds w1, wINST, wINST // w1<- byte offset; clear V
FETCH_ADVANCE_INST_RB w1 // update rPC, load wINST
b.le MterpCheckSuspendAndContinue
GET_INST_OPCODE ip // extract opcode from wINST
GOTO_OPCODE ip // jump to next instruction
-#endif
/* ------------------------------ */
@@ -1396,17 +1372,28 @@
* For: if-eq, if-ne, if-lt, if-ge, if-gt, if-le
*/
/* if-cmp vA, vB, +CCCC */
-#if MTERP_SUSPEND
- mov w1, wINST, lsr #12 // w1<- B
+#if MTERP_PROFILE_BRANCHES
+ lsr w1, wINST, #12 // w1<- B
ubfx w0, wINST, #8, #4 // w0<- A
GET_VREG w3, w1 // w3<- vB
GET_VREG w2, w0 // w2<- vA
- FETCH_S w1, 1 // w1<- branch offset, in code units
+ FETCH_S wINST, 1 // wINST<- branch offset, in code units
cmp w2, w3 // compare (vA, vB)
- moveq w1, #2 // w1<- BYTE branch dist for not-taken
- adds w2, w1, w1 // convert to bytes, check sign
+ b.eq .L_op_if_eq_taken
+ FETCH_ADVANCE_INST 2 // update rPC, load wINST
+ GET_INST_OPCODE ip // extract opcode from wINST
+ GOTO_OPCODE ip // jump to next instruction
+.L_op_if_eq_taken:
+ EXPORT_PC
+ mov x0, xSELF
+ add x1, xFP, #OFF_FP_SHADOWFRAME
+ sbfm x2, xINST, 0, 31 // Sign extend branch offset
+ bl MterpProfileBranch // (self, shadow_frame, offset)
+ cbnz w0, MterpOnStackReplacement // Note: offset must be in xINST
+ ldr w7, [xSELF, #THREAD_FLAGS_OFFSET]
+ adds w2, wINST, wINST // convert to bytes, check sign
FETCH_ADVANCE_INST_RB w2 // update rPC, load wINST
- ldrmi rIBASE, [xSELF, #THREAD_CURRENT_IBASE_OFFSET] // refresh rIBASE
+ b.mi MterpCheckSuspendAndContinue
GET_INST_OPCODE ip // extract opcode from wINST
GOTO_OPCODE ip // jump to next instruction
#else
@@ -1415,11 +1402,11 @@
GET_VREG w3, w1 // w3<- vB
GET_VREG w2, w0 // w2<- vA
FETCH_S w1, 1 // w1<- branch offset, in code units
- ldr w7, [xSELF, #THREAD_FLAGS_OFFSET]
mov w0, #2 // Offset if branch not taken
cmp w2, w3 // compare (vA, vB)
- csel w1, w1, w0, eq // Branch if true
- adds w2, w1, w1 // convert to bytes, check sign
+ csel wINST, w1, w0, eq // Branch if true, stashing result in callee save reg.
+ ldr w7, [xSELF, #THREAD_FLAGS_OFFSET]
+ adds w2, wINST, wINST // convert to bytes, check sign
FETCH_ADVANCE_INST_RB w2 // update rPC, load wINST
b.mi MterpCheckSuspendAndContinue
GET_INST_OPCODE ip // extract opcode from wINST
@@ -1440,17 +1427,28 @@
* For: if-eq, if-ne, if-lt, if-ge, if-gt, if-le
*/
/* if-cmp vA, vB, +CCCC */
-#if MTERP_SUSPEND
- mov w1, wINST, lsr #12 // w1<- B
+#if MTERP_PROFILE_BRANCHES
+ lsr w1, wINST, #12 // w1<- B
ubfx w0, wINST, #8, #4 // w0<- A
GET_VREG w3, w1 // w3<- vB
GET_VREG w2, w0 // w2<- vA
- FETCH_S w1, 1 // w1<- branch offset, in code units
+ FETCH_S wINST, 1 // wINST<- branch offset, in code units
cmp w2, w3 // compare (vA, vB)
- movne w1, #2 // w1<- BYTE branch dist for not-taken
- adds w2, w1, w1 // convert to bytes, check sign
+ b.ne .L_op_if_ne_taken
+ FETCH_ADVANCE_INST 2 // update rPC, load wINST
+ GET_INST_OPCODE ip // extract opcode from wINST
+ GOTO_OPCODE ip // jump to next instruction
+.L_op_if_ne_taken:
+ EXPORT_PC
+ mov x0, xSELF
+ add x1, xFP, #OFF_FP_SHADOWFRAME
+ sbfm x2, xINST, 0, 31 // Sign extend branch offset
+ bl MterpProfileBranch // (self, shadow_frame, offset)
+ cbnz w0, MterpOnStackReplacement // Note: offset must be in xINST
+ ldr w7, [xSELF, #THREAD_FLAGS_OFFSET]
+ adds w2, wINST, wINST // convert to bytes, check sign
FETCH_ADVANCE_INST_RB w2 // update rPC, load wINST
- ldrmi rIBASE, [xSELF, #THREAD_CURRENT_IBASE_OFFSET] // refresh rIBASE
+ b.mi MterpCheckSuspendAndContinue
GET_INST_OPCODE ip // extract opcode from wINST
GOTO_OPCODE ip // jump to next instruction
#else
@@ -1459,11 +1457,11 @@
GET_VREG w3, w1 // w3<- vB
GET_VREG w2, w0 // w2<- vA
FETCH_S w1, 1 // w1<- branch offset, in code units
- ldr w7, [xSELF, #THREAD_FLAGS_OFFSET]
mov w0, #2 // Offset if branch not taken
cmp w2, w3 // compare (vA, vB)
- csel w1, w1, w0, ne // Branch if true
- adds w2, w1, w1 // convert to bytes, check sign
+ csel wINST, w1, w0, ne // Branch if true, stashing result in callee save reg.
+ ldr w7, [xSELF, #THREAD_FLAGS_OFFSET]
+ adds w2, wINST, wINST // convert to bytes, check sign
FETCH_ADVANCE_INST_RB w2 // update rPC, load wINST
b.mi MterpCheckSuspendAndContinue
GET_INST_OPCODE ip // extract opcode from wINST
@@ -1484,17 +1482,28 @@
* For: if-eq, if-ne, if-lt, if-ge, if-gt, if-le
*/
/* if-cmp vA, vB, +CCCC */
-#if MTERP_SUSPEND
- mov w1, wINST, lsr #12 // w1<- B
+#if MTERP_PROFILE_BRANCHES
+ lsr w1, wINST, #12 // w1<- B
ubfx w0, wINST, #8, #4 // w0<- A
GET_VREG w3, w1 // w3<- vB
GET_VREG w2, w0 // w2<- vA
- FETCH_S w1, 1 // w1<- branch offset, in code units
+ FETCH_S wINST, 1 // wINST<- branch offset, in code units
cmp w2, w3 // compare (vA, vB)
- movlt w1, #2 // w1<- BYTE branch dist for not-taken
- adds w2, w1, w1 // convert to bytes, check sign
+ b.lt .L_op_if_lt_taken
+ FETCH_ADVANCE_INST 2 // update rPC, load wINST
+ GET_INST_OPCODE ip // extract opcode from wINST
+ GOTO_OPCODE ip // jump to next instruction
+.L_op_if_lt_taken:
+ EXPORT_PC
+ mov x0, xSELF
+ add x1, xFP, #OFF_FP_SHADOWFRAME
+ sbfm x2, xINST, 0, 31 // Sign extend branch offset
+ bl MterpProfileBranch // (self, shadow_frame, offset)
+ cbnz w0, MterpOnStackReplacement // Note: offset must be in xINST
+ ldr w7, [xSELF, #THREAD_FLAGS_OFFSET]
+ adds w2, wINST, wINST // convert to bytes, check sign
FETCH_ADVANCE_INST_RB w2 // update rPC, load wINST
- ldrmi rIBASE, [xSELF, #THREAD_CURRENT_IBASE_OFFSET] // refresh rIBASE
+ b.mi MterpCheckSuspendAndContinue
GET_INST_OPCODE ip // extract opcode from wINST
GOTO_OPCODE ip // jump to next instruction
#else
@@ -1503,11 +1512,11 @@
GET_VREG w3, w1 // w3<- vB
GET_VREG w2, w0 // w2<- vA
FETCH_S w1, 1 // w1<- branch offset, in code units
- ldr w7, [xSELF, #THREAD_FLAGS_OFFSET]
mov w0, #2 // Offset if branch not taken
cmp w2, w3 // compare (vA, vB)
- csel w1, w1, w0, lt // Branch if true
- adds w2, w1, w1 // convert to bytes, check sign
+ csel wINST, w1, w0, lt // Branch if true, stashing result in callee save reg.
+ ldr w7, [xSELF, #THREAD_FLAGS_OFFSET]
+ adds w2, wINST, wINST // convert to bytes, check sign
FETCH_ADVANCE_INST_RB w2 // update rPC, load wINST
b.mi MterpCheckSuspendAndContinue
GET_INST_OPCODE ip // extract opcode from wINST
@@ -1528,17 +1537,28 @@
* For: if-eq, if-ne, if-lt, if-ge, if-gt, if-le
*/
/* if-cmp vA, vB, +CCCC */
-#if MTERP_SUSPEND
- mov w1, wINST, lsr #12 // w1<- B
+#if MTERP_PROFILE_BRANCHES
+ lsr w1, wINST, #12 // w1<- B
ubfx w0, wINST, #8, #4 // w0<- A
GET_VREG w3, w1 // w3<- vB
GET_VREG w2, w0 // w2<- vA
- FETCH_S w1, 1 // w1<- branch offset, in code units
+ FETCH_S wINST, 1 // wINST<- branch offset, in code units
cmp w2, w3 // compare (vA, vB)
- movge w1, #2 // w1<- BYTE branch dist for not-taken
- adds w2, w1, w1 // convert to bytes, check sign
+ b.ge .L_op_if_ge_taken
+ FETCH_ADVANCE_INST 2 // update rPC, load wINST
+ GET_INST_OPCODE ip // extract opcode from wINST
+ GOTO_OPCODE ip // jump to next instruction
+.L_op_if_ge_taken:
+ EXPORT_PC
+ mov x0, xSELF
+ add x1, xFP, #OFF_FP_SHADOWFRAME
+ sbfm x2, xINST, 0, 31 // Sign extend branch offset
+ bl MterpProfileBranch // (self, shadow_frame, offset)
+ cbnz w0, MterpOnStackReplacement // Note: offset must be in xINST
+ ldr w7, [xSELF, #THREAD_FLAGS_OFFSET]
+ adds w2, wINST, wINST // convert to bytes, check sign
FETCH_ADVANCE_INST_RB w2 // update rPC, load wINST
- ldrmi rIBASE, [xSELF, #THREAD_CURRENT_IBASE_OFFSET] // refresh rIBASE
+ b.mi MterpCheckSuspendAndContinue
GET_INST_OPCODE ip // extract opcode from wINST
GOTO_OPCODE ip // jump to next instruction
#else
@@ -1547,11 +1567,11 @@
GET_VREG w3, w1 // w3<- vB
GET_VREG w2, w0 // w2<- vA
FETCH_S w1, 1 // w1<- branch offset, in code units
- ldr w7, [xSELF, #THREAD_FLAGS_OFFSET]
mov w0, #2 // Offset if branch not taken
cmp w2, w3 // compare (vA, vB)
- csel w1, w1, w0, ge // Branch if true
- adds w2, w1, w1 // convert to bytes, check sign
+ csel wINST, w1, w0, ge // Branch if true, stashing result in callee save reg.
+ ldr w7, [xSELF, #THREAD_FLAGS_OFFSET]
+ adds w2, wINST, wINST // convert to bytes, check sign
FETCH_ADVANCE_INST_RB w2 // update rPC, load wINST
b.mi MterpCheckSuspendAndContinue
GET_INST_OPCODE ip // extract opcode from wINST
@@ -1572,17 +1592,28 @@
* For: if-eq, if-ne, if-lt, if-ge, if-gt, if-le
*/
/* if-cmp vA, vB, +CCCC */
-#if MTERP_SUSPEND
- mov w1, wINST, lsr #12 // w1<- B
+#if MTERP_PROFILE_BRANCHES
+ lsr w1, wINST, #12 // w1<- B
ubfx w0, wINST, #8, #4 // w0<- A
GET_VREG w3, w1 // w3<- vB
GET_VREG w2, w0 // w2<- vA
- FETCH_S w1, 1 // w1<- branch offset, in code units
+ FETCH_S wINST, 1 // wINST<- branch offset, in code units
cmp w2, w3 // compare (vA, vB)
- movgt w1, #2 // w1<- BYTE branch dist for not-taken
- adds w2, w1, w1 // convert to bytes, check sign
+ b.gt .L_op_if_gt_taken
+ FETCH_ADVANCE_INST 2 // update rPC, load wINST
+ GET_INST_OPCODE ip // extract opcode from wINST
+ GOTO_OPCODE ip // jump to next instruction
+.L_op_if_gt_taken:
+ EXPORT_PC
+ mov x0, xSELF
+ add x1, xFP, #OFF_FP_SHADOWFRAME
+ sbfm x2, xINST, 0, 31 // Sign extend branch offset
+ bl MterpProfileBranch // (self, shadow_frame, offset)
+ cbnz w0, MterpOnStackReplacement // Note: offset must be in xINST
+ ldr w7, [xSELF, #THREAD_FLAGS_OFFSET]
+ adds w2, wINST, wINST // convert to bytes, check sign
FETCH_ADVANCE_INST_RB w2 // update rPC, load wINST
- ldrmi rIBASE, [xSELF, #THREAD_CURRENT_IBASE_OFFSET] // refresh rIBASE
+ b.mi MterpCheckSuspendAndContinue
GET_INST_OPCODE ip // extract opcode from wINST
GOTO_OPCODE ip // jump to next instruction
#else
@@ -1591,11 +1622,11 @@
GET_VREG w3, w1 // w3<- vB
GET_VREG w2, w0 // w2<- vA
FETCH_S w1, 1 // w1<- branch offset, in code units
- ldr w7, [xSELF, #THREAD_FLAGS_OFFSET]
mov w0, #2 // Offset if branch not taken
cmp w2, w3 // compare (vA, vB)
- csel w1, w1, w0, gt // Branch if true
- adds w2, w1, w1 // convert to bytes, check sign
+ csel wINST, w1, w0, gt // Branch if true, stashing result in callee save reg.
+ ldr w7, [xSELF, #THREAD_FLAGS_OFFSET]
+ adds w2, wINST, wINST // convert to bytes, check sign
FETCH_ADVANCE_INST_RB w2 // update rPC, load wINST
b.mi MterpCheckSuspendAndContinue
GET_INST_OPCODE ip // extract opcode from wINST
@@ -1616,17 +1647,28 @@
* For: if-eq, if-ne, if-lt, if-ge, if-gt, if-le
*/
/* if-cmp vA, vB, +CCCC */
-#if MTERP_SUSPEND
- mov w1, wINST, lsr #12 // w1<- B
+#if MTERP_PROFILE_BRANCHES
+ lsr w1, wINST, #12 // w1<- B
ubfx w0, wINST, #8, #4 // w0<- A
GET_VREG w3, w1 // w3<- vB
GET_VREG w2, w0 // w2<- vA
- FETCH_S w1, 1 // w1<- branch offset, in code units
+ FETCH_S wINST, 1 // wINST<- branch offset, in code units
cmp w2, w3 // compare (vA, vB)
- movle w1, #2 // w1<- BYTE branch dist for not-taken
- adds w2, w1, w1 // convert to bytes, check sign
+ b.le .L_op_if_le_taken
+ FETCH_ADVANCE_INST 2 // update rPC, load wINST
+ GET_INST_OPCODE ip // extract opcode from wINST
+ GOTO_OPCODE ip // jump to next instruction
+.L_op_if_le_taken:
+ EXPORT_PC
+ mov x0, xSELF
+ add x1, xFP, #OFF_FP_SHADOWFRAME
+ sbfm x2, xINST, 0, 31 // Sign extend branch offset
+ bl MterpProfileBranch // (self, shadow_frame, offset)
+ cbnz w0, MterpOnStackReplacement // Note: offset must be in xINST
+ ldr w7, [xSELF, #THREAD_FLAGS_OFFSET]
+ adds w2, wINST, wINST // convert to bytes, check sign
FETCH_ADVANCE_INST_RB w2 // update rPC, load wINST
- ldrmi rIBASE, [xSELF, #THREAD_CURRENT_IBASE_OFFSET] // refresh rIBASE
+ b.mi MterpCheckSuspendAndContinue
GET_INST_OPCODE ip // extract opcode from wINST
GOTO_OPCODE ip // jump to next instruction
#else
@@ -1635,11 +1677,11 @@
GET_VREG w3, w1 // w3<- vB
GET_VREG w2, w0 // w2<- vA
FETCH_S w1, 1 // w1<- branch offset, in code units
- ldr w7, [xSELF, #THREAD_FLAGS_OFFSET]
mov w0, #2 // Offset if branch not taken
cmp w2, w3 // compare (vA, vB)
- csel w1, w1, w0, le // Branch if true
- adds w2, w1, w1 // convert to bytes, check sign
+ csel wINST, w1, w0, le // Branch if true, stashing result in callee save reg.
+ ldr w7, [xSELF, #THREAD_FLAGS_OFFSET]
+ adds w2, wINST, wINST // convert to bytes, check sign
FETCH_ADVANCE_INST_RB w2 // update rPC, load wINST
b.mi MterpCheckSuspendAndContinue
GET_INST_OPCODE ip // extract opcode from wINST
@@ -1660,26 +1702,37 @@
* for: if-eqz, if-nez, if-ltz, if-gez, if-gtz, if-lez
*/
/* if-cmp vAA, +BBBB */
-#if MTERP_SUSPEND
- mov w0, wINST, lsr #8 // w0<- AA
+#if MTERP_PROFILE_BRANCHES
+ lsr w0, wINST, #8 // w0<- AA
GET_VREG w2, w0 // w2<- vAA
- FETCH_S w1, 1 // w1<- branch offset, in code units
+ FETCH_S wINST, 1 // w1<- branch offset, in code units
cmp w2, #0 // compare (vA, 0)
- moveq w1, #2 // w1<- inst branch dist for not-taken
- adds w1, w1, w1 // convert to bytes & set flags
- FETCH_ADVANCE_INST_RB w1 // update rPC, load wINST
- ldrmi rIBASE, [xSELF, #THREAD_CURRENT_IBASE_OFFSET] // refresh table base
+ b.eq .L_op_if_eqz_taken
+ FETCH_ADVANCE_INST 2 // update rPC, load wINST
+ GET_INST_OPCODE ip // extract opcode from wINST
+ GOTO_OPCODE ip // jump to next instruction
+.L_op_if_eqz_taken:
+ EXPORT_PC
+ mov x0, xSELF
+ add x1, xFP, #OFF_FP_SHADOWFRAME
+ sbfm x2, xINST, 0, 31
+ bl MterpProfileBranch // (self, shadow_frame, offset)
+ cbnz w0, MterpOnStackReplacement // Note: offset must be in wINST
+ ldr w7, [xSELF, #THREAD_FLAGS_OFFSET]
+ adds w2, wINST, wINST // convert to bytes & set flags
+ FETCH_ADVANCE_INST_RB w2 // update rPC, load wINST
+ b.mi MterpCheckSuspendAndContinue
GET_INST_OPCODE ip // extract opcode from wINST
GOTO_OPCODE ip // jump to next instruction
#else
lsr w0, wINST, #8 // w0<- AA
GET_VREG w2, w0 // w2<- vAA
FETCH_S w1, 1 // w1<- branch offset, in code units
- ldr w7, [xSELF, #THREAD_FLAGS_OFFSET]
mov w0, #2 // Branch offset if not taken
cmp w2, #0 // compare (vA, 0)
- csel w1, w1, w0, eq // Branch if true
- adds w2, w1, w1 // convert to bytes & set flags
+ csel wINST, w1, w0, eq // Branch if true, stashing result in callee save reg
+ ldr w7, [xSELF, #THREAD_FLAGS_OFFSET]
+ adds w2, wINST, wINST // convert to bytes & set flags
FETCH_ADVANCE_INST_RB w2 // update rPC, load wINST
b.mi MterpCheckSuspendAndContinue
GET_INST_OPCODE ip // extract opcode from wINST
@@ -1700,26 +1753,37 @@
* for: if-eqz, if-nez, if-ltz, if-gez, if-gtz, if-lez
*/
/* if-cmp vAA, +BBBB */
-#if MTERP_SUSPEND
- mov w0, wINST, lsr #8 // w0<- AA
+#if MTERP_PROFILE_BRANCHES
+ lsr w0, wINST, #8 // w0<- AA
GET_VREG w2, w0 // w2<- vAA
- FETCH_S w1, 1 // w1<- branch offset, in code units
+ FETCH_S wINST, 1 // w1<- branch offset, in code units
cmp w2, #0 // compare (vA, 0)
- movne w1, #2 // w1<- inst branch dist for not-taken
- adds w1, w1, w1 // convert to bytes & set flags
- FETCH_ADVANCE_INST_RB w1 // update rPC, load wINST
- ldrmi rIBASE, [xSELF, #THREAD_CURRENT_IBASE_OFFSET] // refresh table base
+ b.ne .L_op_if_nez_taken
+ FETCH_ADVANCE_INST 2 // update rPC, load wINST
+ GET_INST_OPCODE ip // extract opcode from wINST
+ GOTO_OPCODE ip // jump to next instruction
+.L_op_if_nez_taken:
+ EXPORT_PC
+ mov x0, xSELF
+ add x1, xFP, #OFF_FP_SHADOWFRAME
+ sbfm x2, xINST, 0, 31
+ bl MterpProfileBranch // (self, shadow_frame, offset)
+ cbnz w0, MterpOnStackReplacement // Note: offset must be in wINST
+ ldr w7, [xSELF, #THREAD_FLAGS_OFFSET]
+ adds w2, wINST, wINST // convert to bytes & set flags
+ FETCH_ADVANCE_INST_RB w2 // update rPC, load wINST
+ b.mi MterpCheckSuspendAndContinue
GET_INST_OPCODE ip // extract opcode from wINST
GOTO_OPCODE ip // jump to next instruction
#else
lsr w0, wINST, #8 // w0<- AA
GET_VREG w2, w0 // w2<- vAA
FETCH_S w1, 1 // w1<- branch offset, in code units
- ldr w7, [xSELF, #THREAD_FLAGS_OFFSET]
mov w0, #2 // Branch offset if not taken
cmp w2, #0 // compare (vA, 0)
- csel w1, w1, w0, ne // Branch if true
- adds w2, w1, w1 // convert to bytes & set flags
+ csel wINST, w1, w0, ne // Branch if true, stashing result in callee save reg
+ ldr w7, [xSELF, #THREAD_FLAGS_OFFSET]
+ adds w2, wINST, wINST // convert to bytes & set flags
FETCH_ADVANCE_INST_RB w2 // update rPC, load wINST
b.mi MterpCheckSuspendAndContinue
GET_INST_OPCODE ip // extract opcode from wINST
@@ -1740,26 +1804,37 @@
* for: if-eqz, if-nez, if-ltz, if-gez, if-gtz, if-lez
*/
/* if-cmp vAA, +BBBB */
-#if MTERP_SUSPEND
- mov w0, wINST, lsr #8 // w0<- AA
+#if MTERP_PROFILE_BRANCHES
+ lsr w0, wINST, #8 // w0<- AA
GET_VREG w2, w0 // w2<- vAA
- FETCH_S w1, 1 // w1<- branch offset, in code units
+ FETCH_S wINST, 1 // w1<- branch offset, in code units
cmp w2, #0 // compare (vA, 0)
- movlt w1, #2 // w1<- inst branch dist for not-taken
- adds w1, w1, w1 // convert to bytes & set flags
- FETCH_ADVANCE_INST_RB w1 // update rPC, load wINST
- ldrmi rIBASE, [xSELF, #THREAD_CURRENT_IBASE_OFFSET] // refresh table base
+ b.lt .L_op_if_ltz_taken
+ FETCH_ADVANCE_INST 2 // update rPC, load wINST
+ GET_INST_OPCODE ip // extract opcode from wINST
+ GOTO_OPCODE ip // jump to next instruction
+.L_op_if_ltz_taken:
+ EXPORT_PC
+ mov x0, xSELF
+ add x1, xFP, #OFF_FP_SHADOWFRAME
+ sbfm x2, xINST, 0, 31
+ bl MterpProfileBranch // (self, shadow_frame, offset)
+ cbnz w0, MterpOnStackReplacement // Note: offset must be in wINST
+ ldr w7, [xSELF, #THREAD_FLAGS_OFFSET]
+ adds w2, wINST, wINST // convert to bytes & set flags
+ FETCH_ADVANCE_INST_RB w2 // update rPC, load wINST
+ b.mi MterpCheckSuspendAndContinue
GET_INST_OPCODE ip // extract opcode from wINST
GOTO_OPCODE ip // jump to next instruction
#else
lsr w0, wINST, #8 // w0<- AA
GET_VREG w2, w0 // w2<- vAA
FETCH_S w1, 1 // w1<- branch offset, in code units
- ldr w7, [xSELF, #THREAD_FLAGS_OFFSET]
mov w0, #2 // Branch offset if not taken
cmp w2, #0 // compare (vA, 0)
- csel w1, w1, w0, lt // Branch if true
- adds w2, w1, w1 // convert to bytes & set flags
+ csel wINST, w1, w0, lt // Branch if true, stashing result in callee save reg
+ ldr w7, [xSELF, #THREAD_FLAGS_OFFSET]
+ adds w2, wINST, wINST // convert to bytes & set flags
FETCH_ADVANCE_INST_RB w2 // update rPC, load wINST
b.mi MterpCheckSuspendAndContinue
GET_INST_OPCODE ip // extract opcode from wINST
@@ -1780,26 +1855,37 @@
* for: if-eqz, if-nez, if-ltz, if-gez, if-gtz, if-lez
*/
/* if-cmp vAA, +BBBB */
-#if MTERP_SUSPEND
- mov w0, wINST, lsr #8 // w0<- AA
+#if MTERP_PROFILE_BRANCHES
+ lsr w0, wINST, #8 // w0<- AA
GET_VREG w2, w0 // w2<- vAA
- FETCH_S w1, 1 // w1<- branch offset, in code units
+ FETCH_S wINST, 1 // w1<- branch offset, in code units
cmp w2, #0 // compare (vA, 0)
- movge w1, #2 // w1<- inst branch dist for not-taken
- adds w1, w1, w1 // convert to bytes & set flags
- FETCH_ADVANCE_INST_RB w1 // update rPC, load wINST
- ldrmi rIBASE, [xSELF, #THREAD_CURRENT_IBASE_OFFSET] // refresh table base
+ b.ge .L_op_if_gez_taken
+ FETCH_ADVANCE_INST 2 // update rPC, load wINST
+ GET_INST_OPCODE ip // extract opcode from wINST
+ GOTO_OPCODE ip // jump to next instruction
+.L_op_if_gez_taken:
+ EXPORT_PC
+ mov x0, xSELF
+ add x1, xFP, #OFF_FP_SHADOWFRAME
+ sbfm x2, xINST, 0, 31
+ bl MterpProfileBranch // (self, shadow_frame, offset)
+ cbnz w0, MterpOnStackReplacement // Note: offset must be in wINST
+ ldr w7, [xSELF, #THREAD_FLAGS_OFFSET]
+ adds w2, wINST, wINST // convert to bytes & set flags
+ FETCH_ADVANCE_INST_RB w2 // update rPC, load wINST
+ b.mi MterpCheckSuspendAndContinue
GET_INST_OPCODE ip // extract opcode from wINST
GOTO_OPCODE ip // jump to next instruction
#else
lsr w0, wINST, #8 // w0<- AA
GET_VREG w2, w0 // w2<- vAA
FETCH_S w1, 1 // w1<- branch offset, in code units
- ldr w7, [xSELF, #THREAD_FLAGS_OFFSET]
mov w0, #2 // Branch offset if not taken
cmp w2, #0 // compare (vA, 0)
- csel w1, w1, w0, ge // Branch if true
- adds w2, w1, w1 // convert to bytes & set flags
+ csel wINST, w1, w0, ge // Branch if true, stashing result in callee save reg
+ ldr w7, [xSELF, #THREAD_FLAGS_OFFSET]
+ adds w2, wINST, wINST // convert to bytes & set flags
FETCH_ADVANCE_INST_RB w2 // update rPC, load wINST
b.mi MterpCheckSuspendAndContinue
GET_INST_OPCODE ip // extract opcode from wINST
@@ -1820,26 +1906,37 @@
* for: if-eqz, if-nez, if-ltz, if-gez, if-gtz, if-lez
*/
/* if-cmp vAA, +BBBB */
-#if MTERP_SUSPEND
- mov w0, wINST, lsr #8 // w0<- AA
+#if MTERP_PROFILE_BRANCHES
+ lsr w0, wINST, #8 // w0<- AA
GET_VREG w2, w0 // w2<- vAA
- FETCH_S w1, 1 // w1<- branch offset, in code units
+ FETCH_S wINST, 1 // w1<- branch offset, in code units
cmp w2, #0 // compare (vA, 0)
- movgt w1, #2 // w1<- inst branch dist for not-taken
- adds w1, w1, w1 // convert to bytes & set flags
- FETCH_ADVANCE_INST_RB w1 // update rPC, load wINST
- ldrmi rIBASE, [xSELF, #THREAD_CURRENT_IBASE_OFFSET] // refresh table base
+ b.gt .L_op_if_gtz_taken
+ FETCH_ADVANCE_INST 2 // update rPC, load wINST
+ GET_INST_OPCODE ip // extract opcode from wINST
+ GOTO_OPCODE ip // jump to next instruction
+.L_op_if_gtz_taken:
+ EXPORT_PC
+ mov x0, xSELF
+ add x1, xFP, #OFF_FP_SHADOWFRAME
+ sbfm x2, xINST, 0, 31
+ bl MterpProfileBranch // (self, shadow_frame, offset)
+ cbnz w0, MterpOnStackReplacement // Note: offset must be in wINST
+ ldr w7, [xSELF, #THREAD_FLAGS_OFFSET]
+ adds w2, wINST, wINST // convert to bytes & set flags
+ FETCH_ADVANCE_INST_RB w2 // update rPC, load wINST
+ b.mi MterpCheckSuspendAndContinue
GET_INST_OPCODE ip // extract opcode from wINST
GOTO_OPCODE ip // jump to next instruction
#else
lsr w0, wINST, #8 // w0<- AA
GET_VREG w2, w0 // w2<- vAA
FETCH_S w1, 1 // w1<- branch offset, in code units
- ldr w7, [xSELF, #THREAD_FLAGS_OFFSET]
mov w0, #2 // Branch offset if not taken
cmp w2, #0 // compare (vA, 0)
- csel w1, w1, w0, gt // Branch if true
- adds w2, w1, w1 // convert to bytes & set flags
+ csel wINST, w1, w0, gt // Branch if true, stashing result in callee save reg
+ ldr w7, [xSELF, #THREAD_FLAGS_OFFSET]
+ adds w2, wINST, wINST // convert to bytes & set flags
FETCH_ADVANCE_INST_RB w2 // update rPC, load wINST
b.mi MterpCheckSuspendAndContinue
GET_INST_OPCODE ip // extract opcode from wINST
@@ -1860,26 +1957,37 @@
* for: if-eqz, if-nez, if-ltz, if-gez, if-gtz, if-lez
*/
/* if-cmp vAA, +BBBB */
-#if MTERP_SUSPEND
- mov w0, wINST, lsr #8 // w0<- AA
+#if MTERP_PROFILE_BRANCHES
+ lsr w0, wINST, #8 // w0<- AA
GET_VREG w2, w0 // w2<- vAA
- FETCH_S w1, 1 // w1<- branch offset, in code units
+ FETCH_S wINST, 1 // w1<- branch offset, in code units
cmp w2, #0 // compare (vA, 0)
- movle w1, #2 // w1<- inst branch dist for not-taken
- adds w1, w1, w1 // convert to bytes & set flags
- FETCH_ADVANCE_INST_RB w1 // update rPC, load wINST
- ldrmi rIBASE, [xSELF, #THREAD_CURRENT_IBASE_OFFSET] // refresh table base
+ b.le .L_op_if_lez_taken
+ FETCH_ADVANCE_INST 2 // update rPC, load wINST
+ GET_INST_OPCODE ip // extract opcode from wINST
+ GOTO_OPCODE ip // jump to next instruction
+.L_op_if_lez_taken:
+ EXPORT_PC
+ mov x0, xSELF
+ add x1, xFP, #OFF_FP_SHADOWFRAME
+ sbfm x2, xINST, 0, 31
+ bl MterpProfileBranch // (self, shadow_frame, offset)
+ cbnz w0, MterpOnStackReplacement // Note: offset must be in wINST
+ ldr w7, [xSELF, #THREAD_FLAGS_OFFSET]
+ adds w2, wINST, wINST // convert to bytes & set flags
+ FETCH_ADVANCE_INST_RB w2 // update rPC, load wINST
+ b.mi MterpCheckSuspendAndContinue
GET_INST_OPCODE ip // extract opcode from wINST
GOTO_OPCODE ip // jump to next instruction
#else
lsr w0, wINST, #8 // w0<- AA
GET_VREG w2, w0 // w2<- vAA
FETCH_S w1, 1 // w1<- branch offset, in code units
- ldr w7, [xSELF, #THREAD_FLAGS_OFFSET]
mov w0, #2 // Branch offset if not taken
cmp w2, #0 // compare (vA, 0)
- csel w1, w1, w0, le // Branch if true
- adds w2, w1, w1 // convert to bytes & set flags
+ csel wINST, w1, w0, le // Branch if true, stashing result in callee save reg
+ ldr w7, [xSELF, #THREAD_FLAGS_OFFSET]
+ adds w2, wINST, wINST // convert to bytes & set flags
FETCH_ADVANCE_INST_RB w2 // update rPC, load wINST
b.mi MterpCheckSuspendAndContinue
GET_INST_OPCODE ip // extract opcode from wINST
@@ -2401,6 +2509,7 @@
mov x3, xSELF // w3<- self
bl artGet32InstanceFromCode
ldr x3, [xSELF, #THREAD_EXCEPTION_OFFSET]
+
ubfx w2, wINST, #8, #4 // w2<- A
PREFETCH_INST 2
cbnz x3, MterpPossibleException // bail out
@@ -2457,6 +2566,7 @@
mov x3, xSELF // w3<- self
bl artGetObjInstanceFromCode
ldr x3, [xSELF, #THREAD_EXCEPTION_OFFSET]
+
ubfx w2, wINST, #8, #4 // w2<- A
PREFETCH_INST 2
cbnz x3, MterpPossibleException // bail out
@@ -2488,6 +2598,7 @@
mov x3, xSELF // w3<- self
bl artGetBooleanInstanceFromCode
ldr x3, [xSELF, #THREAD_EXCEPTION_OFFSET]
+ uxtb w0, w0
ubfx w2, wINST, #8, #4 // w2<- A
PREFETCH_INST 2
cbnz x3, MterpPossibleException // bail out
@@ -2519,6 +2630,7 @@
mov x3, xSELF // w3<- self
bl artGetByteInstanceFromCode
ldr x3, [xSELF, #THREAD_EXCEPTION_OFFSET]
+ sxtb w0, w0
ubfx w2, wINST, #8, #4 // w2<- A
PREFETCH_INST 2
cbnz x3, MterpPossibleException // bail out
@@ -2550,6 +2662,7 @@
mov x3, xSELF // w3<- self
bl artGetCharInstanceFromCode
ldr x3, [xSELF, #THREAD_EXCEPTION_OFFSET]
+ uxth w0, w0
ubfx w2, wINST, #8, #4 // w2<- A
PREFETCH_INST 2
cbnz x3, MterpPossibleException // bail out
@@ -2581,6 +2694,7 @@
mov x3, xSELF // w3<- self
bl artGetShortInstanceFromCode
ldr x3, [xSELF, #THREAD_EXCEPTION_OFFSET]
+ sxth w0, w0
ubfx w2, wINST, #8, #4 // w2<- A
PREFETCH_INST 2
cbnz x3, MterpPossibleException // bail out
@@ -3158,11 +3272,12 @@
mov x0, xSELF
add x1, xFP, #OFF_FP_SHADOWFRAME
mov x2, xPC
- // and x3, xINST, 0xFFFF
mov x3, xINST
bl MterpInvokeVirtual
cbz w0, MterpException
FETCH_ADVANCE_INST 3
+ bl MterpShouldSwitchInterpreters
+ cbnz w0, MterpFallback
GET_INST_OPCODE ip
GOTO_OPCODE ip
@@ -3190,11 +3305,12 @@
mov x0, xSELF
add x1, xFP, #OFF_FP_SHADOWFRAME
mov x2, xPC
- // and x3, xINST, 0xFFFF
mov x3, xINST
bl MterpInvokeSuper
cbz w0, MterpException
FETCH_ADVANCE_INST 3
+ bl MterpShouldSwitchInterpreters
+ cbnz w0, MterpFallback
GET_INST_OPCODE ip
GOTO_OPCODE ip
@@ -3222,11 +3338,12 @@
mov x0, xSELF
add x1, xFP, #OFF_FP_SHADOWFRAME
mov x2, xPC
- // and x3, xINST, 0xFFFF
mov x3, xINST
bl MterpInvokeDirect
cbz w0, MterpException
FETCH_ADVANCE_INST 3
+ bl MterpShouldSwitchInterpreters
+ cbnz w0, MterpFallback
GET_INST_OPCODE ip
GOTO_OPCODE ip
@@ -3247,11 +3364,12 @@
mov x0, xSELF
add x1, xFP, #OFF_FP_SHADOWFRAME
mov x2, xPC
- // and x3, xINST, 0xFFFF
mov x3, xINST
bl MterpInvokeStatic
cbz w0, MterpException
FETCH_ADVANCE_INST 3
+ bl MterpShouldSwitchInterpreters
+ cbnz w0, MterpFallback
GET_INST_OPCODE ip
GOTO_OPCODE ip
@@ -3273,11 +3391,12 @@
mov x0, xSELF
add x1, xFP, #OFF_FP_SHADOWFRAME
mov x2, xPC
- // and x3, xINST, 0xFFFF
mov x3, xINST
bl MterpInvokeInterface
cbz w0, MterpException
FETCH_ADVANCE_INST 3
+ bl MterpShouldSwitchInterpreters
+ cbnz w0, MterpFallback
GET_INST_OPCODE ip
GOTO_OPCODE ip
@@ -3320,11 +3439,12 @@
mov x0, xSELF
add x1, xFP, #OFF_FP_SHADOWFRAME
mov x2, xPC
- // and x3, xINST, 0xFFFF
mov x3, xINST
bl MterpInvokeVirtualRange
cbz w0, MterpException
FETCH_ADVANCE_INST 3
+ bl MterpShouldSwitchInterpreters
+ cbnz w0, MterpFallback
GET_INST_OPCODE ip
GOTO_OPCODE ip
@@ -3345,11 +3465,12 @@
mov x0, xSELF
add x1, xFP, #OFF_FP_SHADOWFRAME
mov x2, xPC
- // and x3, xINST, 0xFFFF
mov x3, xINST
bl MterpInvokeSuperRange
cbz w0, MterpException
FETCH_ADVANCE_INST 3
+ bl MterpShouldSwitchInterpreters
+ cbnz w0, MterpFallback
GET_INST_OPCODE ip
GOTO_OPCODE ip
@@ -3370,11 +3491,12 @@
mov x0, xSELF
add x1, xFP, #OFF_FP_SHADOWFRAME
mov x2, xPC
- // and x3, xINST, 0xFFFF
mov x3, xINST
bl MterpInvokeDirectRange
cbz w0, MterpException
FETCH_ADVANCE_INST 3
+ bl MterpShouldSwitchInterpreters
+ cbnz w0, MterpFallback
GET_INST_OPCODE ip
GOTO_OPCODE ip
@@ -3395,11 +3517,12 @@
mov x0, xSELF
add x1, xFP, #OFF_FP_SHADOWFRAME
mov x2, xPC
- // and x3, xINST, 0xFFFF
mov x3, xINST
bl MterpInvokeStaticRange
cbz w0, MterpException
FETCH_ADVANCE_INST 3
+ bl MterpShouldSwitchInterpreters
+ cbnz w0, MterpFallback
GET_INST_OPCODE ip
GOTO_OPCODE ip
@@ -3420,11 +3543,12 @@
mov x0, xSELF
add x1, xFP, #OFF_FP_SHADOWFRAME
mov x2, xPC
- // and x3, xINST, 0xFFFF
mov x3, xINST
bl MterpInvokeInterfaceRange
cbz w0, MterpException
FETCH_ADVANCE_INST 3
+ bl MterpShouldSwitchInterpreters
+ cbnz w0, MterpFallback
GET_INST_OPCODE ip
GOTO_OPCODE ip
@@ -6852,11 +6976,12 @@
mov x0, xSELF
add x1, xFP, #OFF_FP_SHADOWFRAME
mov x2, xPC
- // and x3, xINST, 0xFFFF
mov x3, xINST
bl MterpInvokeVirtualQuick
cbz w0, MterpException
FETCH_ADVANCE_INST 3
+ bl MterpShouldSwitchInterpreters
+ cbnz w0, MterpFallback
GET_INST_OPCODE ip
GOTO_OPCODE ip
@@ -6877,11 +7002,12 @@
mov x0, xSELF
add x1, xFP, #OFF_FP_SHADOWFRAME
mov x2, xPC
- // and x3, xINST, 0xFFFF
mov x3, xINST
bl MterpInvokeVirtualQuickRange
cbz w0, MterpException
FETCH_ADVANCE_INST 3
+ bl MterpShouldSwitchInterpreters
+ cbnz w0, MterpFallback
GET_INST_OPCODE ip
GOTO_OPCODE ip
@@ -11565,7 +11691,6 @@
* has not yet been thrown. Just bail out to the reference interpreter to deal with it.
* TUNING: for consistency, we may want to just go ahead and handle these here.
*/
-#define MTERP_LOGGING 0
common_errDivideByZero:
EXPORT_PC
#if MTERP_LOGGING
@@ -11654,8 +11779,11 @@
ldr xIBASE, [xSELF, #THREAD_CURRENT_IBASE_OFFSET]
add xPC, x0, #CODEITEM_INSNS_OFFSET
add xPC, xPC, x1, lsl #1 // generate new dex_pc_ptr
- str xPC, [xFP, #OFF_FP_DEX_PC_PTR]
+ /* Do we need to switch interpreters? */
+ bl MterpShouldSwitchInterpreters
+ cbnz w0, MterpFallback
/* resume execution at catch block */
+ EXPORT_PC
FETCH_INST
GET_INST_OPCODE ip
GOTO_OPCODE ip
@@ -11675,10 +11803,24 @@
EXPORT_PC
mov x0, xSELF
bl MterpSuspendCheck // (self)
+ cbnz x0, MterpFallback // Something in the environment changed, switch interpreters
GET_INST_OPCODE ip // extract opcode from wINST
GOTO_OPCODE ip // jump to next instruction
/*
+ * On-stack replacement has happened, and now we've returned from the compiled method.
+ */
+MterpOnStackReplacement:
+#if MTERP_LOGGING
+ mov x0, xSELF
+ add x1, xFP, #OFF_FP_SHADOWFRAME
+ sbfm x2, xINST, 0, 31
+ bl MterpLogOSR
+#endif
+ mov x0, #1 // Signal normal return
+ b MterpDone
+
+/*
* Bail out to reference interpreter.
*/
MterpFallback: