| /* |
| * Copyright (C) 2012 The Android Open Source Project |
| * |
| * Licensed under the Apache License, Version 2.0 (the "License"); |
| * you may not use this file except in compliance with the License. |
| * You may obtain a copy of the License at |
| * |
| * http://www.apache.org/licenses/LICENSE-2.0 |
| * |
| * Unless required by applicable law or agreed to in writing, software |
| * distributed under the License is distributed on an "AS IS" BASIS, |
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| * See the License for the specific language governing permissions and |
| * limitations under the License. |
| */ |
| |
| #include "asm_support_x86_64.S" |
| |
| #include "arch/quick_alloc_entrypoints.S" |
| |
| MACRO0(ASSERT_USE_READ_BARRIER) |
| #if !defined(USE_READ_BARRIER) |
| int3 |
| int3 |
| #endif |
| END_MACRO |
| |
| MACRO0(SETUP_FP_CALLEE_SAVE_FRAME) |
| // Create space for ART FP callee-saved registers |
| subq MACRO_LITERAL(4 * 8), %rsp |
| CFI_ADJUST_CFA_OFFSET(4 * 8) |
| movq %xmm12, 0(%rsp) |
| movq %xmm13, 8(%rsp) |
| movq %xmm14, 16(%rsp) |
| movq %xmm15, 24(%rsp) |
| END_MACRO |
| |
| MACRO0(RESTORE_FP_CALLEE_SAVE_FRAME) |
| // Restore ART FP callee-saved registers |
| movq 0(%rsp), %xmm12 |
| movq 8(%rsp), %xmm13 |
| movq 16(%rsp), %xmm14 |
| movq 24(%rsp), %xmm15 |
| addq MACRO_LITERAL(4 * 8), %rsp |
| CFI_ADJUST_CFA_OFFSET(- 4 * 8) |
| END_MACRO |
| |
| // For x86, the CFA is esp+4, the address above the pushed return address on the stack. |
| |
| /* |
| * Macro that sets up the callee save frame to conform with |
| * Runtime::CreateCalleeSaveMethod(kSaveAllCalleeSaves) |
| */ |
| MACRO0(SETUP_SAVE_ALL_CALLEE_SAVES_FRAME) |
| #if defined(__APPLE__) |
| int3 |
| int3 |
| #else |
| // R10 := Runtime::Current() |
| movq _ZN3art7Runtime9instance_E@GOTPCREL(%rip), %r10 |
| movq (%r10), %r10 |
| // Save callee save registers to agree with core spills bitmap. |
| PUSH r15 // Callee save. |
| PUSH r14 // Callee save. |
| PUSH r13 // Callee save. |
| PUSH r12 // Callee save. |
| PUSH rbp // Callee save. |
| PUSH rbx // Callee save. |
| // Create space for FPR args, plus space for ArtMethod*. |
| subq MACRO_LITERAL(4 * 8 + 8), %rsp |
| CFI_ADJUST_CFA_OFFSET(4 * 8 + 8) |
| // Save FPRs. |
| movq %xmm12, 8(%rsp) |
| movq %xmm13, 16(%rsp) |
| movq %xmm14, 24(%rsp) |
| movq %xmm15, 32(%rsp) |
| // R10 := ArtMethod* for save all callee save frame method. |
| movq RUNTIME_SAVE_ALL_CALLEE_SAVES_METHOD_OFFSET(%r10), %r10 |
| // Store ArtMethod* to bottom of stack. |
| movq %r10, 0(%rsp) |
| // Store rsp as the top quick frame. |
| movq %rsp, %gs:THREAD_TOP_QUICK_FRAME_OFFSET |
| |
| // Ugly compile-time check, but we only have the preprocessor. |
| // Last +8: implicit return address pushed on stack when caller made call. |
| #if (FRAME_SIZE_SAVE_ALL_CALLEE_SAVES != 6 * 8 + 4 * 8 + 8 + 8) |
| #error "FRAME_SIZE_SAVE_ALL_CALLEE_SAVES(X86_64) size not as expected." |
| #endif |
| #endif // __APPLE__ |
| END_MACRO |
| |
| /* |
| * Macro that sets up the callee save frame to conform with |
| * Runtime::CreateCalleeSaveMethod(kSaveRefsOnly) |
| */ |
| MACRO0(SETUP_SAVE_REFS_ONLY_FRAME) |
| #if defined(__APPLE__) |
| int3 |
| int3 |
| #else |
| // R10 := Runtime::Current() |
| movq _ZN3art7Runtime9instance_E@GOTPCREL(%rip), %r10 |
| movq (%r10), %r10 |
| // Save callee and GPR args, mixed together to agree with core spills bitmap. |
| PUSH r15 // Callee save. |
| PUSH r14 // Callee save. |
| PUSH r13 // Callee save. |
| PUSH r12 // Callee save. |
| PUSH rbp // Callee save. |
| PUSH rbx // Callee save. |
| // Create space for FPR args, plus space for ArtMethod*. |
| subq LITERAL(8 + 4 * 8), %rsp |
| CFI_ADJUST_CFA_OFFSET(8 + 4 * 8) |
| // Save FPRs. |
| movq %xmm12, 8(%rsp) |
| movq %xmm13, 16(%rsp) |
| movq %xmm14, 24(%rsp) |
| movq %xmm15, 32(%rsp) |
| // R10 := ArtMethod* for refs only callee save frame method. |
| movq RUNTIME_SAVE_REFS_ONLY_METHOD_OFFSET(%r10), %r10 |
| // Store ArtMethod* to bottom of stack. |
| movq %r10, 0(%rsp) |
| // Store rsp as the stop quick frame. |
| movq %rsp, %gs:THREAD_TOP_QUICK_FRAME_OFFSET |
| |
| // Ugly compile-time check, but we only have the preprocessor. |
| // Last +8: implicit return address pushed on stack when caller made call. |
| #if (FRAME_SIZE_SAVE_REFS_ONLY != 6 * 8 + 4 * 8 + 8 + 8) |
| #error "FRAME_SIZE_SAVE_REFS_ONLY(X86_64) size not as expected." |
| #endif |
| #endif // __APPLE__ |
| END_MACRO |
| |
| MACRO0(RESTORE_SAVE_REFS_ONLY_FRAME) |
| movq 8(%rsp), %xmm12 |
| movq 16(%rsp), %xmm13 |
| movq 24(%rsp), %xmm14 |
| movq 32(%rsp), %xmm15 |
| addq LITERAL(8 + 4*8), %rsp |
| CFI_ADJUST_CFA_OFFSET(-8 - 4*8) |
| // TODO: optimize by not restoring callee-saves restored by the ABI |
| POP rbx |
| POP rbp |
| POP r12 |
| POP r13 |
| POP r14 |
| POP r15 |
| END_MACRO |
| |
| /* |
| * Macro that sets up the callee save frame to conform with |
| * Runtime::CreateCalleeSaveMethod(kSaveRefsAndArgs) |
| */ |
| MACRO0(SETUP_SAVE_REFS_AND_ARGS_FRAME) |
| #if defined(__APPLE__) |
| int3 |
| int3 |
| #else |
| // R10 := Runtime::Current() |
| movq _ZN3art7Runtime9instance_E@GOTPCREL(%rip), %r10 |
| movq (%r10), %r10 |
| // Save callee and GPR args, mixed together to agree with core spills bitmap. |
| PUSH r15 // Callee save. |
| PUSH r14 // Callee save. |
| PUSH r13 // Callee save. |
| PUSH r12 // Callee save. |
| PUSH r9 // Quick arg 5. |
| PUSH r8 // Quick arg 4. |
| PUSH rsi // Quick arg 1. |
| PUSH rbp // Callee save. |
| PUSH rbx // Callee save. |
| PUSH rdx // Quick arg 2. |
| PUSH rcx // Quick arg 3. |
| // Create space for FPR args and create 2 slots for ArtMethod*. |
| subq MACRO_LITERAL(16 + 12 * 8), %rsp |
| CFI_ADJUST_CFA_OFFSET(16 + 12 * 8) |
| // R10 := ArtMethod* for ref and args callee save frame method. |
| movq RUNTIME_SAVE_REFS_AND_ARGS_METHOD_OFFSET(%r10), %r10 |
| // Save FPRs. |
| movq %xmm0, 16(%rsp) |
| movq %xmm1, 24(%rsp) |
| movq %xmm2, 32(%rsp) |
| movq %xmm3, 40(%rsp) |
| movq %xmm4, 48(%rsp) |
| movq %xmm5, 56(%rsp) |
| movq %xmm6, 64(%rsp) |
| movq %xmm7, 72(%rsp) |
| movq %xmm12, 80(%rsp) |
| movq %xmm13, 88(%rsp) |
| movq %xmm14, 96(%rsp) |
| movq %xmm15, 104(%rsp) |
| // Store ArtMethod* to bottom of stack. |
| movq %r10, 0(%rsp) |
| // Store rsp as the top quick frame. |
| movq %rsp, %gs:THREAD_TOP_QUICK_FRAME_OFFSET |
| |
| // Ugly compile-time check, but we only have the preprocessor. |
| // Last +8: implicit return address pushed on stack when caller made call. |
| #if (FRAME_SIZE_SAVE_REFS_AND_ARGS != 11 * 8 + 12 * 8 + 16 + 8) |
| #error "FRAME_SIZE_SAVE_REFS_AND_ARGS(X86_64) size not as expected." |
| #endif |
| #endif // __APPLE__ |
| END_MACRO |
| |
| MACRO0(SETUP_SAVE_REFS_AND_ARGS_FRAME_WITH_METHOD_IN_RDI) |
| // Save callee and GPR args, mixed together to agree with core spills bitmap. |
| PUSH r15 // Callee save. |
| PUSH r14 // Callee save. |
| PUSH r13 // Callee save. |
| PUSH r12 // Callee save. |
| PUSH r9 // Quick arg 5. |
| PUSH r8 // Quick arg 4. |
| PUSH rsi // Quick arg 1. |
| PUSH rbp // Callee save. |
| PUSH rbx // Callee save. |
| PUSH rdx // Quick arg 2. |
| PUSH rcx // Quick arg 3. |
| // Create space for FPR args and create 2 slots for ArtMethod*. |
| subq LITERAL(80 + 4 * 8), %rsp |
| CFI_ADJUST_CFA_OFFSET(80 + 4 * 8) |
| // Save FPRs. |
| movq %xmm0, 16(%rsp) |
| movq %xmm1, 24(%rsp) |
| movq %xmm2, 32(%rsp) |
| movq %xmm3, 40(%rsp) |
| movq %xmm4, 48(%rsp) |
| movq %xmm5, 56(%rsp) |
| movq %xmm6, 64(%rsp) |
| movq %xmm7, 72(%rsp) |
| movq %xmm12, 80(%rsp) |
| movq %xmm13, 88(%rsp) |
| movq %xmm14, 96(%rsp) |
| movq %xmm15, 104(%rsp) |
| // Store ArtMethod to bottom of stack. |
| movq %rdi, 0(%rsp) |
| // Store rsp as the stop quick frame. |
| movq %rsp, %gs:THREAD_TOP_QUICK_FRAME_OFFSET |
| END_MACRO |
| |
| MACRO0(RESTORE_SAVE_REFS_AND_ARGS_FRAME) |
| // Restore FPRs. |
| movq 16(%rsp), %xmm0 |
| movq 24(%rsp), %xmm1 |
| movq 32(%rsp), %xmm2 |
| movq 40(%rsp), %xmm3 |
| movq 48(%rsp), %xmm4 |
| movq 56(%rsp), %xmm5 |
| movq 64(%rsp), %xmm6 |
| movq 72(%rsp), %xmm7 |
| movq 80(%rsp), %xmm12 |
| movq 88(%rsp), %xmm13 |
| movq 96(%rsp), %xmm14 |
| movq 104(%rsp), %xmm15 |
| addq MACRO_LITERAL(80 + 4 * 8), %rsp |
| CFI_ADJUST_CFA_OFFSET(-(80 + 4 * 8)) |
| // Restore callee and GPR args, mixed together to agree with core spills bitmap. |
| POP rcx |
| POP rdx |
| POP rbx |
| POP rbp |
| POP rsi |
| POP r8 |
| POP r9 |
| POP r12 |
| POP r13 |
| POP r14 |
| POP r15 |
| END_MACRO |
| |
| /* |
| * Macro that sets up the callee save frame to conform with |
| * Runtime::CreateCalleeSaveMethod(kSaveEverything) |
| * when R14 and R15 are already saved. |
| */ |
| MACRO0(SETUP_SAVE_EVERYTHING_FRAME_R14_R15_SAVED) |
| #if defined(__APPLE__) |
| int3 |
| int3 |
| #else |
| // Save core registers from highest to lowest to agree with core spills bitmap. |
| // R14 and R15, or at least placeholders for them, are already on the stack. |
| PUSH r13 |
| PUSH r12 |
| PUSH r11 |
| PUSH r10 |
| PUSH r9 |
| PUSH r8 |
| PUSH rdi |
| PUSH rsi |
| PUSH rbp |
| PUSH rbx |
| PUSH rdx |
| PUSH rcx |
| PUSH rax |
| // Create space for FPRs and stack alignment padding. |
| subq MACRO_LITERAL(8 + 16 * 8), %rsp |
| CFI_ADJUST_CFA_OFFSET(8 + 16 * 8) |
| // R10 := Runtime::Current() |
| movq _ZN3art7Runtime9instance_E@GOTPCREL(%rip), %r10 |
| movq (%r10), %r10 |
| // Save FPRs. |
| movq %xmm0, 8(%rsp) |
| movq %xmm1, 16(%rsp) |
| movq %xmm2, 24(%rsp) |
| movq %xmm3, 32(%rsp) |
| movq %xmm4, 40(%rsp) |
| movq %xmm5, 48(%rsp) |
| movq %xmm6, 56(%rsp) |
| movq %xmm7, 64(%rsp) |
| movq %xmm8, 72(%rsp) |
| movq %xmm9, 80(%rsp) |
| movq %xmm10, 88(%rsp) |
| movq %xmm11, 96(%rsp) |
| movq %xmm12, 104(%rsp) |
| movq %xmm13, 112(%rsp) |
| movq %xmm14, 120(%rsp) |
| movq %xmm15, 128(%rsp) |
| // Push ArtMethod* for save everything frame method. |
| pushq RUNTIME_SAVE_EVERYTHING_METHOD_OFFSET(%r10) |
| CFI_ADJUST_CFA_OFFSET(8) |
| // Store rsp as the top quick frame. |
| movq %rsp, %gs:THREAD_TOP_QUICK_FRAME_OFFSET |
| |
| // Ugly compile-time check, but we only have the preprocessor. |
| // Last +8: implicit return address pushed on stack when caller made call. |
| #if (FRAME_SIZE_SAVE_EVERYTHING != 15 * 8 + 16 * 8 + 16 + 8) |
| #error "FRAME_SIZE_SAVE_EVERYTHING(X86_64) size not as expected." |
| #endif |
| #endif // __APPLE__ |
| END_MACRO |
| |
| /* |
| * Macro that sets up the callee save frame to conform with |
| * Runtime::CreateCalleeSaveMethod(kSaveEverything) |
| * when R15 is already saved. |
| */ |
| MACRO0(SETUP_SAVE_EVERYTHING_FRAME_R15_SAVED) |
| PUSH r14 |
| SETUP_SAVE_EVERYTHING_FRAME_R14_R15_SAVED |
| END_MACRO |
| |
| /* |
| * Macro that sets up the callee save frame to conform with |
| * Runtime::CreateCalleeSaveMethod(kSaveEverything) |
| */ |
| MACRO0(SETUP_SAVE_EVERYTHING_FRAME) |
| PUSH r15 |
| SETUP_SAVE_EVERYTHING_FRAME_R15_SAVED |
| END_MACRO |
| |
| MACRO0(RESTORE_SAVE_EVERYTHING_FRAME_FRPS) |
| // Restore FPRs. Method and padding is still on the stack. |
| movq 16(%rsp), %xmm0 |
| movq 24(%rsp), %xmm1 |
| movq 32(%rsp), %xmm2 |
| movq 40(%rsp), %xmm3 |
| movq 48(%rsp), %xmm4 |
| movq 56(%rsp), %xmm5 |
| movq 64(%rsp), %xmm6 |
| movq 72(%rsp), %xmm7 |
| movq 80(%rsp), %xmm8 |
| movq 88(%rsp), %xmm9 |
| movq 96(%rsp), %xmm10 |
| movq 104(%rsp), %xmm11 |
| movq 112(%rsp), %xmm12 |
| movq 120(%rsp), %xmm13 |
| movq 128(%rsp), %xmm14 |
| movq 136(%rsp), %xmm15 |
| END_MACRO |
| |
| MACRO0(RESTORE_SAVE_EVERYTHING_FRAME_GPRS_EXCEPT_RAX) |
| // Restore callee and GPR args (except RAX), mixed together to agree with core spills bitmap. |
| POP rcx |
| POP rdx |
| POP rbx |
| POP rbp |
| POP rsi |
| POP rdi |
| POP r8 |
| POP r9 |
| POP r10 |
| POP r11 |
| POP r12 |
| POP r13 |
| POP r14 |
| POP r15 |
| END_MACRO |
| |
| MACRO0(RESTORE_SAVE_EVERYTHING_FRAME) |
| RESTORE_SAVE_EVERYTHING_FRAME_FRPS |
| |
| // Remove save everything callee save method, stack alignment padding and FPRs. |
| addq MACRO_LITERAL(16 + 16 * 8), %rsp |
| CFI_ADJUST_CFA_OFFSET(-(16 + 16 * 8)) |
| |
| POP rax |
| RESTORE_SAVE_EVERYTHING_FRAME_GPRS_EXCEPT_RAX |
| END_MACRO |
| |
| MACRO0(RESTORE_SAVE_EVERYTHING_FRAME_KEEP_RAX) |
| RESTORE_SAVE_EVERYTHING_FRAME_FRPS |
| |
| // Remove save everything callee save method, stack alignment padding and FPRs, skip RAX. |
| addq MACRO_LITERAL(16 + 16 * 8 + 8), %rsp |
| CFI_ADJUST_CFA_OFFSET(-(16 + 16 * 8 + 8)) |
| |
| RESTORE_SAVE_EVERYTHING_FRAME_GPRS_EXCEPT_RAX |
| END_MACRO |
| |
| /* |
| * Macro that calls through to artDeliverPendingExceptionFromCode, where the pending |
| * exception is Thread::Current()->exception_ when the runtime method frame is ready. |
| */ |
| MACRO0(DELIVER_PENDING_EXCEPTION_FRAME_READY) |
| // (Thread*) setup |
| movq %gs:THREAD_SELF_OFFSET, %rdi |
| call SYMBOL(artDeliverPendingExceptionFromCode) // artDeliverPendingExceptionFromCode(Thread*) |
| UNREACHABLE |
| END_MACRO |
| |
| /* |
| * Macro that calls through to artDeliverPendingExceptionFromCode, where the pending |
| * exception is Thread::Current()->exception_. |
| */ |
| MACRO0(DELIVER_PENDING_EXCEPTION) |
| SETUP_SAVE_ALL_CALLEE_SAVES_FRAME // save callee saves for throw |
| DELIVER_PENDING_EXCEPTION_FRAME_READY |
| END_MACRO |
| |
| MACRO2(NO_ARG_RUNTIME_EXCEPTION, c_name, cxx_name) |
| DEFINE_FUNCTION VAR(c_name) |
| SETUP_SAVE_ALL_CALLEE_SAVES_FRAME // save all registers as basis for long jump context |
| // Outgoing argument set up |
| movq %gs:THREAD_SELF_OFFSET, %rdi // pass Thread::Current() |
| call CALLVAR(cxx_name) // cxx_name(Thread*) |
| UNREACHABLE |
| END_FUNCTION VAR(c_name) |
| END_MACRO |
| |
| MACRO2(NO_ARG_RUNTIME_EXCEPTION_SAVE_EVERYTHING, c_name, cxx_name) |
| DEFINE_FUNCTION VAR(c_name) |
| SETUP_SAVE_EVERYTHING_FRAME // save all registers as basis for long jump context |
| // Outgoing argument set up |
| movq %gs:THREAD_SELF_OFFSET, %rdi // pass Thread::Current() |
| call CALLVAR(cxx_name) // cxx_name(Thread*) |
| UNREACHABLE |
| END_FUNCTION VAR(c_name) |
| END_MACRO |
| |
| MACRO2(ONE_ARG_RUNTIME_EXCEPTION, c_name, cxx_name) |
| DEFINE_FUNCTION VAR(c_name) |
| SETUP_SAVE_ALL_CALLEE_SAVES_FRAME // save all registers as basis for long jump context |
| // Outgoing argument set up |
| movq %gs:THREAD_SELF_OFFSET, %rsi // pass Thread::Current() |
| call CALLVAR(cxx_name) // cxx_name(arg1, Thread*) |
| UNREACHABLE |
| END_FUNCTION VAR(c_name) |
| END_MACRO |
| |
| MACRO2(TWO_ARG_RUNTIME_EXCEPTION_SAVE_EVERYTHING, c_name, cxx_name) |
| DEFINE_FUNCTION VAR(c_name) |
| SETUP_SAVE_EVERYTHING_FRAME // save all registers as basis for long jump context |
| // Outgoing argument set up |
| movq %gs:THREAD_SELF_OFFSET, %rdx // pass Thread::Current() |
| call CALLVAR(cxx_name) // cxx_name(Thread*) |
| UNREACHABLE |
| END_FUNCTION VAR(c_name) |
| END_MACRO |
| |
| /* |
| * Called by managed code to create and deliver a NullPointerException. |
| */ |
| NO_ARG_RUNTIME_EXCEPTION_SAVE_EVERYTHING art_quick_throw_null_pointer_exception, artThrowNullPointerExceptionFromCode |
| |
| /* |
| * Call installed by a signal handler to create and deliver a NullPointerException. |
| */ |
| DEFINE_FUNCTION_CUSTOM_CFA art_quick_throw_null_pointer_exception_from_signal, 2 * __SIZEOF_POINTER__ |
| // Fault address and return address were saved by the fault handler. |
| // Save all registers as basis for long jump context; R15 will replace fault address later. |
| SETUP_SAVE_EVERYTHING_FRAME_R15_SAVED |
| // Retrieve fault address and save R15. |
| movq (FRAME_SIZE_SAVE_EVERYTHING - 2 * __SIZEOF_POINTER__)(%rsp), %rdi |
| movq %r15, (FRAME_SIZE_SAVE_EVERYTHING - 2 * __SIZEOF_POINTER__)(%rsp) |
| CFI_REL_OFFSET(%r15, (FRAME_SIZE_SAVE_EVERYTHING - 2 * __SIZEOF_POINTER__)) |
| // Outgoing argument set up; RDI already contains the fault address. |
| movq %gs:THREAD_SELF_OFFSET, %rsi // pass Thread::Current() |
| call SYMBOL(artThrowNullPointerExceptionFromSignal) // (addr, self) |
| UNREACHABLE |
| END_FUNCTION art_quick_throw_null_pointer_exception_from_signal |
| |
| /* |
| * Called by managed code to create and deliver an ArithmeticException. |
| */ |
| NO_ARG_RUNTIME_EXCEPTION_SAVE_EVERYTHING art_quick_throw_div_zero, artThrowDivZeroFromCode |
| |
| /* |
| * Called by managed code to create and deliver a StackOverflowError. |
| */ |
| NO_ARG_RUNTIME_EXCEPTION art_quick_throw_stack_overflow, artThrowStackOverflowFromCode |
| |
| /* |
| * Called by managed code, saves callee saves and then calls artThrowException |
| * that will place a mock Method* at the bottom of the stack. Arg1 holds the exception. |
| */ |
| ONE_ARG_RUNTIME_EXCEPTION art_quick_deliver_exception, artDeliverExceptionFromCode |
| |
| /* |
| * Called by managed code to create and deliver an ArrayIndexOutOfBoundsException. Arg1 holds |
| * index, arg2 holds limit. |
| */ |
| TWO_ARG_RUNTIME_EXCEPTION_SAVE_EVERYTHING art_quick_throw_array_bounds, artThrowArrayBoundsFromCode |
| |
| /* |
| * Called by managed code to create and deliver a StringIndexOutOfBoundsException |
| * as if thrown from a call to String.charAt(). Arg1 holds index, arg2 holds limit. |
| */ |
| TWO_ARG_RUNTIME_EXCEPTION_SAVE_EVERYTHING art_quick_throw_string_bounds, artThrowStringBoundsFromCode |
| |
| /* |
| * All generated callsites for interface invokes and invocation slow paths will load arguments |
| * as usual - except instead of loading arg0/rdi with the target Method*, arg0/rdi will contain |
| * the method_idx. This wrapper will save arg1-arg3, and call the appropriate C helper. |
| * NOTE: "this" is first visible argument of the target, and so can be found in arg1/rsi. |
| * |
| * The helper will attempt to locate the target and return a 128-bit result in rax/rdx consisting |
| * of the target Method* in rax and method->code_ in rdx. |
| * |
| * If unsuccessful, the helper will return null/????. There will be a pending exception in the |
| * thread and we branch to another stub to deliver it. |
| * |
| * On success this wrapper will restore arguments and *jump* to the target, leaving the return |
| * location on the stack. |
| * |
| * Adapted from x86 code. |
| */ |
| MACRO1(INVOKE_TRAMPOLINE_BODY, cxx_name) |
| SETUP_SAVE_REFS_AND_ARGS_FRAME // save callee saves in case allocation triggers GC |
| // Helper signature is always |
| // (method_idx, *this_object, *caller_method, *self, sp) |
| |
| movq %gs:THREAD_SELF_OFFSET, %rdx // pass Thread |
| movq %rsp, %rcx // pass SP |
| |
| call CALLVAR(cxx_name) // cxx_name(arg1, arg2, Thread*, SP) |
| // save the code pointer |
| movq %rax, %rdi |
| movq %rdx, %rax |
| RESTORE_SAVE_REFS_AND_ARGS_FRAME |
| |
| testq %rdi, %rdi |
| jz 1f |
| |
| // Tail call to intended method. |
| jmp *%rax |
| 1: |
| DELIVER_PENDING_EXCEPTION |
| END_MACRO |
| MACRO2(INVOKE_TRAMPOLINE, c_name, cxx_name) |
| DEFINE_FUNCTION VAR(c_name) |
| INVOKE_TRAMPOLINE_BODY RAW_VAR(cxx_name) |
| END_FUNCTION VAR(c_name) |
| END_MACRO |
| |
| INVOKE_TRAMPOLINE art_quick_invoke_interface_trampoline_with_access_check, artInvokeInterfaceTrampolineWithAccessCheck |
| |
| INVOKE_TRAMPOLINE art_quick_invoke_static_trampoline_with_access_check, artInvokeStaticTrampolineWithAccessCheck |
| INVOKE_TRAMPOLINE art_quick_invoke_direct_trampoline_with_access_check, artInvokeDirectTrampolineWithAccessCheck |
| INVOKE_TRAMPOLINE art_quick_invoke_super_trampoline_with_access_check, artInvokeSuperTrampolineWithAccessCheck |
| INVOKE_TRAMPOLINE art_quick_invoke_virtual_trampoline_with_access_check, artInvokeVirtualTrampolineWithAccessCheck |
| |
| |
| /* |
| * Helper for quick invocation stub to set up XMM registers. Assumes r10 == shorty, |
| * r11 == arg_array. Clobbers r10, r11 and al. Branches to xmm_setup_finished if it encounters |
| * the end of the shorty. |
| */ |
| MACRO2(LOOP_OVER_SHORTY_LOADING_XMMS, xmm_reg, finished) |
| 1: // LOOP |
| movb (%r10), %al // al := *shorty |
| addq MACRO_LITERAL(1), %r10 // shorty++ |
| cmpb MACRO_LITERAL(0), %al // if (al == '\0') goto xmm_setup_finished |
| je VAR(finished) |
| cmpb MACRO_LITERAL(68), %al // if (al == 'D') goto FOUND_DOUBLE |
| je 2f |
| cmpb MACRO_LITERAL(70), %al // if (al == 'F') goto FOUND_FLOAT |
| je 3f |
| addq MACRO_LITERAL(4), %r11 // arg_array++ |
| // Handle extra space in arg array taken by a long. |
| cmpb MACRO_LITERAL(74), %al // if (al != 'J') goto LOOP |
| jne 1b |
| addq MACRO_LITERAL(4), %r11 // arg_array++ |
| jmp 1b // goto LOOP |
| 2: // FOUND_DOUBLE |
| movsd (%r11), REG_VAR(xmm_reg) |
| addq MACRO_LITERAL(8), %r11 // arg_array+=2 |
| jmp 4f |
| 3: // FOUND_FLOAT |
| movss (%r11), REG_VAR(xmm_reg) |
| addq MACRO_LITERAL(4), %r11 // arg_array++ |
| 4: |
| END_MACRO |
| |
| /* |
| * Helper for quick invocation stub to set up GPR registers. Assumes r10 == shorty, |
| * r11 == arg_array. Clobbers r10, r11 and al. Branches to gpr_setup_finished if it encounters |
| * the end of the shorty. |
| */ |
| MACRO3(LOOP_OVER_SHORTY_LOADING_GPRS, gpr_reg64, gpr_reg32, finished) |
| 1: // LOOP |
| movb (%r10), %al // al := *shorty |
| addq MACRO_LITERAL(1), %r10 // shorty++ |
| cmpb MACRO_LITERAL(0), %al // if (al == '\0') goto gpr_setup_finished |
| je VAR(finished) |
| cmpb MACRO_LITERAL(74), %al // if (al == 'J') goto FOUND_LONG |
| je 2f |
| cmpb MACRO_LITERAL(70), %al // if (al == 'F') goto SKIP_FLOAT |
| je 3f |
| cmpb MACRO_LITERAL(68), %al // if (al == 'D') goto SKIP_DOUBLE |
| je 4f |
| movl (%r11), REG_VAR(gpr_reg32) |
| addq MACRO_LITERAL(4), %r11 // arg_array++ |
| jmp 5f |
| 2: // FOUND_LONG |
| movq (%r11), REG_VAR(gpr_reg64) |
| addq MACRO_LITERAL(8), %r11 // arg_array+=2 |
| jmp 5f |
| 3: // SKIP_FLOAT |
| addq MACRO_LITERAL(4), %r11 // arg_array++ |
| jmp 1b |
| 4: // SKIP_DOUBLE |
| addq MACRO_LITERAL(8), %r11 // arg_array+=2 |
| jmp 1b |
| 5: |
| END_MACRO |
| |
| /* |
| * Quick invocation stub. |
| * On entry: |
| * [sp] = return address |
| * rdi = method pointer |
| * rsi = argument array that must at least contain the this pointer. |
| * rdx = size of argument array in bytes |
| * rcx = (managed) thread pointer |
| * r8 = JValue* result |
| * r9 = char* shorty |
| */ |
| DEFINE_FUNCTION art_quick_invoke_stub |
| #if defined(__APPLE__) |
| int3 |
| int3 |
| #else |
| // Set up argument XMM registers. |
| leaq 1(%r9), %r10 // R10 := shorty + 1 ; ie skip return arg character. |
| leaq 4(%rsi), %r11 // R11 := arg_array + 4 ; ie skip this pointer. |
| LOOP_OVER_SHORTY_LOADING_XMMS xmm0, .Lxmm_setup_finished |
| LOOP_OVER_SHORTY_LOADING_XMMS xmm1, .Lxmm_setup_finished |
| LOOP_OVER_SHORTY_LOADING_XMMS xmm2, .Lxmm_setup_finished |
| LOOP_OVER_SHORTY_LOADING_XMMS xmm3, .Lxmm_setup_finished |
| LOOP_OVER_SHORTY_LOADING_XMMS xmm4, .Lxmm_setup_finished |
| LOOP_OVER_SHORTY_LOADING_XMMS xmm5, .Lxmm_setup_finished |
| LOOP_OVER_SHORTY_LOADING_XMMS xmm6, .Lxmm_setup_finished |
| LOOP_OVER_SHORTY_LOADING_XMMS xmm7, .Lxmm_setup_finished |
| .balign 16 |
| .Lxmm_setup_finished: |
| PUSH rbp // Save rbp. |
| PUSH r8 // Save r8/result*. |
| PUSH r9 // Save r9/shorty*. |
| PUSH rbx // Save native callee save rbx |
| PUSH r12 // Save native callee save r12 |
| PUSH r13 // Save native callee save r13 |
| PUSH r14 // Save native callee save r14 |
| PUSH r15 // Save native callee save r15 |
| movq %rsp, %rbp // Copy value of stack pointer into base pointer. |
| CFI_DEF_CFA_REGISTER(rbp) |
| |
| movl %edx, %r10d |
| addl LITERAL(100), %edx // Reserve space for return addr, StackReference<method>, rbp, |
| // r8, r9, rbx, r12, r13, r14, and r15 in frame. |
| andl LITERAL(0xFFFFFFF0), %edx // Align frame size to 16 bytes. |
| subl LITERAL(72), %edx // Remove space for return address, rbp, r8, r9, rbx, r12, |
| // r13, r14, and r15 |
| subq %rdx, %rsp // Reserve stack space for argument array. |
| |
| #if (STACK_REFERENCE_SIZE != 4) |
| #error "STACK_REFERENCE_SIZE(X86_64) size not as expected." |
| #endif |
| movq LITERAL(0), (%rsp) // Store null for method* |
| |
| movl %r10d, %ecx // Place size of args in rcx. |
| movq %rdi, %rax // rax := method to be called |
| movq %rsi, %r11 // r11 := arg_array |
| leaq 8(%rsp), %rdi // rdi is pointing just above the ArtMethod* in the stack |
| // arguments. |
| // Copy arg array into stack. |
| rep movsb // while (rcx--) { *rdi++ = *rsi++ } |
| leaq 1(%r9), %r10 // r10 := shorty + 1 ; ie skip return arg character |
| movq %rax, %rdi // rdi := method to be called |
| movl (%r11), %esi // rsi := this pointer |
| addq LITERAL(4), %r11 // arg_array++ |
| LOOP_OVER_SHORTY_LOADING_GPRS rdx, edx, .Lgpr_setup_finished |
| LOOP_OVER_SHORTY_LOADING_GPRS rcx, ecx, .Lgpr_setup_finished |
| LOOP_OVER_SHORTY_LOADING_GPRS r8, r8d, .Lgpr_setup_finished |
| LOOP_OVER_SHORTY_LOADING_GPRS r9, r9d, .Lgpr_setup_finished |
| .Lgpr_setup_finished: |
| call *ART_METHOD_QUICK_CODE_OFFSET_64(%rdi) // Call the method. |
| movq %rbp, %rsp // Restore stack pointer. |
| POP r15 // Pop r15 |
| POP r14 // Pop r14 |
| POP r13 // Pop r13 |
| POP r12 // Pop r12 |
| POP rbx // Pop rbx |
| POP r9 // Pop r9 - shorty* |
| POP r8 // Pop r8 - result*. |
| POP rbp // Pop rbp |
| cmpb LITERAL(68), (%r9) // Test if result type char == 'D'. |
| je .Lreturn_double_quick |
| cmpb LITERAL(70), (%r9) // Test if result type char == 'F'. |
| je .Lreturn_float_quick |
| movq %rax, (%r8) // Store the result assuming its a long, int or Object* |
| ret |
| .Lreturn_double_quick: |
| movsd %xmm0, (%r8) // Store the double floating point result. |
| ret |
| .Lreturn_float_quick: |
| movss %xmm0, (%r8) // Store the floating point result. |
| ret |
| #endif // __APPLE__ |
| END_FUNCTION art_quick_invoke_stub |
| |
| /* |
| * Quick invocation stub. |
| * On entry: |
| * [sp] = return address |
| * rdi = method pointer |
| * rsi = argument array or null if no arguments. |
| * rdx = size of argument array in bytes |
| * rcx = (managed) thread pointer |
| * r8 = JValue* result |
| * r9 = char* shorty |
| */ |
| DEFINE_FUNCTION art_quick_invoke_static_stub |
| #if defined(__APPLE__) |
| int3 |
| int3 |
| #else |
| // Set up argument XMM registers. |
| leaq 1(%r9), %r10 // R10 := shorty + 1 ; ie skip return arg character |
| movq %rsi, %r11 // R11 := arg_array |
| LOOP_OVER_SHORTY_LOADING_XMMS xmm0, .Lxmm_setup_finished2 |
| LOOP_OVER_SHORTY_LOADING_XMMS xmm1, .Lxmm_setup_finished2 |
| LOOP_OVER_SHORTY_LOADING_XMMS xmm2, .Lxmm_setup_finished2 |
| LOOP_OVER_SHORTY_LOADING_XMMS xmm3, .Lxmm_setup_finished2 |
| LOOP_OVER_SHORTY_LOADING_XMMS xmm4, .Lxmm_setup_finished2 |
| LOOP_OVER_SHORTY_LOADING_XMMS xmm5, .Lxmm_setup_finished2 |
| LOOP_OVER_SHORTY_LOADING_XMMS xmm6, .Lxmm_setup_finished2 |
| LOOP_OVER_SHORTY_LOADING_XMMS xmm7, .Lxmm_setup_finished2 |
| .balign 16 |
| .Lxmm_setup_finished2: |
| PUSH rbp // Save rbp. |
| PUSH r8 // Save r8/result*. |
| PUSH r9 // Save r9/shorty*. |
| PUSH rbx // Save rbx |
| PUSH r12 // Save r12 |
| PUSH r13 // Save r13 |
| PUSH r14 // Save r14 |
| PUSH r15 // Save r15 |
| movq %rsp, %rbp // Copy value of stack pointer into base pointer. |
| CFI_DEF_CFA_REGISTER(rbp) |
| |
| movl %edx, %r10d |
| addl LITERAL(100), %edx // Reserve space for return addr, StackReference<method>, rbp, |
| // r8, r9, r12, r13, r14, and r15 in frame. |
| andl LITERAL(0xFFFFFFF0), %edx // Align frame size to 16 bytes. |
| subl LITERAL(72), %edx // Remove space for return address, rbp, r8, r9, rbx, r12, |
| // r13, r14, and r15. |
| subq %rdx, %rsp // Reserve stack space for argument array. |
| |
| #if (STACK_REFERENCE_SIZE != 4) |
| #error "STACK_REFERENCE_SIZE(X86_64) size not as expected." |
| #endif |
| movq LITERAL(0), (%rsp) // Store null for method* |
| |
| movl %r10d, %ecx // Place size of args in rcx. |
| movq %rdi, %rax // rax := method to be called |
| movq %rsi, %r11 // r11 := arg_array |
| leaq 8(%rsp), %rdi // rdi is pointing just above the ArtMethod* in the |
| // stack arguments. |
| // Copy arg array into stack. |
| rep movsb // while (rcx--) { *rdi++ = *rsi++ } |
| leaq 1(%r9), %r10 // r10 := shorty + 1 ; ie skip return arg character |
| movq %rax, %rdi // rdi := method to be called |
| LOOP_OVER_SHORTY_LOADING_GPRS rsi, esi, .Lgpr_setup_finished2 |
| LOOP_OVER_SHORTY_LOADING_GPRS rdx, edx, .Lgpr_setup_finished2 |
| LOOP_OVER_SHORTY_LOADING_GPRS rcx, ecx, .Lgpr_setup_finished2 |
| LOOP_OVER_SHORTY_LOADING_GPRS r8, r8d, .Lgpr_setup_finished2 |
| LOOP_OVER_SHORTY_LOADING_GPRS r9, r9d, .Lgpr_setup_finished2 |
| .Lgpr_setup_finished2: |
| call *ART_METHOD_QUICK_CODE_OFFSET_64(%rdi) // Call the method. |
| movq %rbp, %rsp // Restore stack pointer. |
| POP r15 // Pop r15 |
| POP r14 // Pop r14 |
| POP r13 // Pop r13 |
| POP r12 // Pop r12 |
| POP rbx // Pop rbx |
| POP r9 // Pop r9 - shorty*. |
| POP r8 // Pop r8 - result*. |
| POP rbp // Pop rbp |
| cmpb LITERAL(68), (%r9) // Test if result type char == 'D'. |
| je .Lreturn_double_quick2 |
| cmpb LITERAL(70), (%r9) // Test if result type char == 'F'. |
| je .Lreturn_float_quick2 |
| movq %rax, (%r8) // Store the result assuming its a long, int or Object* |
| ret |
| .Lreturn_double_quick2: |
| movsd %xmm0, (%r8) // Store the double floating point result. |
| ret |
| .Lreturn_float_quick2: |
| movss %xmm0, (%r8) // Store the floating point result. |
| ret |
| #endif // __APPLE__ |
| END_FUNCTION art_quick_invoke_static_stub |
| |
| /* |
| * Long jump stub. |
| * On entry: |
| * rdi = gprs |
| * rsi = fprs |
| */ |
| DEFINE_FUNCTION art_quick_do_long_jump |
| #if defined(__APPLE__) |
| int3 |
| int3 |
| #else |
| // Restore FPRs. |
| movq 0(%rsi), %xmm0 |
| movq 8(%rsi), %xmm1 |
| movq 16(%rsi), %xmm2 |
| movq 24(%rsi), %xmm3 |
| movq 32(%rsi), %xmm4 |
| movq 40(%rsi), %xmm5 |
| movq 48(%rsi), %xmm6 |
| movq 56(%rsi), %xmm7 |
| movq 64(%rsi), %xmm8 |
| movq 72(%rsi), %xmm9 |
| movq 80(%rsi), %xmm10 |
| movq 88(%rsi), %xmm11 |
| movq 96(%rsi), %xmm12 |
| movq 104(%rsi), %xmm13 |
| movq 112(%rsi), %xmm14 |
| movq 120(%rsi), %xmm15 |
| // Restore FPRs. |
| movq %rdi, %rsp // RSP points to gprs. |
| // Load all registers except RSP and RIP with values in gprs. |
| popq %r15 |
| popq %r14 |
| popq %r13 |
| popq %r12 |
| popq %r11 |
| popq %r10 |
| popq %r9 |
| popq %r8 |
| popq %rdi |
| popq %rsi |
| popq %rbp |
| addq LITERAL(8), %rsp // Skip rsp |
| popq %rbx |
| popq %rdx |
| popq %rcx |
| popq %rax |
| popq %rsp // Load stack pointer. |
| ret // From higher in the stack pop rip. |
| #endif // __APPLE__ |
| END_FUNCTION art_quick_do_long_jump |
| |
| MACRO3(ONE_ARG_DOWNCALL, c_name, cxx_name, return_macro) |
| DEFINE_FUNCTION VAR(c_name) |
| SETUP_SAVE_REFS_ONLY_FRAME // save ref containing registers for GC |
| // Outgoing argument set up |
| movq %gs:THREAD_SELF_OFFSET, %rsi // pass Thread::Current() |
| call CALLVAR(cxx_name) // cxx_name(arg0, Thread*) |
| RESTORE_SAVE_REFS_ONLY_FRAME // restore frame up to return address |
| CALL_MACRO(return_macro) // return or deliver exception |
| END_FUNCTION VAR(c_name) |
| END_MACRO |
| |
| MACRO3(TWO_ARG_DOWNCALL, c_name, cxx_name, return_macro) |
| DEFINE_FUNCTION VAR(c_name) |
| SETUP_SAVE_REFS_ONLY_FRAME // save ref containing registers for GC |
| // Outgoing argument set up |
| movq %gs:THREAD_SELF_OFFSET, %rdx // pass Thread::Current() |
| call CALLVAR(cxx_name) // cxx_name(arg0, arg1, Thread*) |
| RESTORE_SAVE_REFS_ONLY_FRAME // restore frame up to return address |
| CALL_MACRO(return_macro) // return or deliver exception |
| END_FUNCTION VAR(c_name) |
| END_MACRO |
| |
| MACRO3(THREE_ARG_DOWNCALL, c_name, cxx_name, return_macro) |
| DEFINE_FUNCTION VAR(c_name) |
| SETUP_SAVE_REFS_ONLY_FRAME // save ref containing registers for GC |
| // Outgoing argument set up |
| movq %gs:THREAD_SELF_OFFSET, %rcx // pass Thread::Current() |
| call CALLVAR(cxx_name) // cxx_name(arg0, arg1, arg2, Thread*) |
| RESTORE_SAVE_REFS_ONLY_FRAME // restore frame up to return address |
| CALL_MACRO(return_macro) // return or deliver exception |
| END_FUNCTION VAR(c_name) |
| END_MACRO |
| |
| MACRO3(FOUR_ARG_DOWNCALL, c_name, cxx_name, return_macro) |
| DEFINE_FUNCTION VAR(c_name) |
| SETUP_SAVE_REFS_ONLY_FRAME // save ref containing registers for GC |
| // Outgoing argument set up |
| movq %gs:THREAD_SELF_OFFSET, %r8 // pass Thread::Current() |
| call CALLVAR(cxx_name) // cxx_name(arg1, arg2, arg3, arg4, Thread*) |
| RESTORE_SAVE_REFS_ONLY_FRAME // restore frame up to return address |
| CALL_MACRO(return_macro) // return or deliver exception |
| END_FUNCTION VAR(c_name) |
| END_MACRO |
| |
| MACRO3(ONE_ARG_REF_DOWNCALL, c_name, cxx_name, return_macro) |
| DEFINE_FUNCTION VAR(c_name) |
| movq 8(%rsp), %rsi // pass referrer |
| SETUP_SAVE_REFS_ONLY_FRAME |
| // arg0 is in rdi |
| movq %gs:THREAD_SELF_OFFSET, %rdx // pass Thread::Current() |
| call CALLVAR(cxx_name) // cxx_name(arg0, referrer, Thread*) |
| RESTORE_SAVE_REFS_ONLY_FRAME // restore frame up to return address |
| CALL_MACRO(return_macro) |
| END_FUNCTION VAR(c_name) |
| END_MACRO |
| |
| MACRO3(TWO_ARG_REF_DOWNCALL, c_name, cxx_name, return_macro) |
| DEFINE_FUNCTION VAR(c_name) |
| movq 8(%rsp), %rdx // pass referrer |
| SETUP_SAVE_REFS_ONLY_FRAME |
| // arg0 and arg1 are in rdi/rsi |
| movq %gs:THREAD_SELF_OFFSET, %rcx // pass Thread::Current() |
| call CALLVAR(cxx_name) // (arg0, arg1, referrer, Thread*) |
| RESTORE_SAVE_REFS_ONLY_FRAME // restore frame up to return address |
| CALL_MACRO(return_macro) |
| END_FUNCTION VAR(c_name) |
| END_MACRO |
| |
| MACRO3(THREE_ARG_REF_DOWNCALL, c_name, cxx_name, return_macro) |
| DEFINE_FUNCTION VAR(c_name) |
| movq 8(%rsp), %rcx // pass referrer |
| SETUP_SAVE_REFS_ONLY_FRAME |
| // arg0, arg1, and arg2 are in rdi/rsi/rdx |
| movq %gs:THREAD_SELF_OFFSET, %r8 // pass Thread::Current() |
| call CALLVAR(cxx_name) // cxx_name(arg0, arg1, arg2, referrer, Thread*) |
| RESTORE_SAVE_REFS_ONLY_FRAME // restore frame up to return address |
| CALL_MACRO(return_macro) // return or deliver exception |
| END_FUNCTION VAR(c_name) |
| END_MACRO |
| |
| MACRO0(RETURN_IF_RESULT_IS_NON_ZERO_OR_DELIVER) |
| testq %rax, %rax // rax == 0 ? |
| jz 1f // if rax == 0 goto 1 |
| ret // return |
| 1: // deliver exception on current thread |
| DELIVER_PENDING_EXCEPTION |
| END_MACRO |
| |
| MACRO0(RETURN_IF_EAX_ZERO) |
| testl %eax, %eax // eax == 0 ? |
| jnz 1f // if eax != 0 goto 1 |
| ret // return |
| 1: // deliver exception on current thread |
| DELIVER_PENDING_EXCEPTION |
| END_MACRO |
| |
| MACRO0(RETURN_OR_DELIVER_PENDING_EXCEPTION) |
| movq %gs:THREAD_EXCEPTION_OFFSET, %rcx // get exception field |
| testq %rcx, %rcx // rcx == 0 ? |
| jnz 1f // if rcx != 0 goto 1 |
| ret // return |
| 1: // deliver exception on current thread |
| DELIVER_PENDING_EXCEPTION |
| END_MACRO |
| |
| // Generate the allocation entrypoints for each allocator. |
| GENERATE_ALLOC_ENTRYPOINTS_FOR_NON_TLAB_ALLOCATORS |
| |
| // Comment out allocators that have x86_64 specific asm. |
| // Region TLAB: |
| // GENERATE_ALLOC_ENTRYPOINTS_ALLOC_OBJECT_RESOLVED(_region_tlab, RegionTLAB) |
| // GENERATE_ALLOC_ENTRYPOINTS_ALLOC_OBJECT_INITIALIZED(_region_tlab, RegionTLAB) |
| GENERATE_ALLOC_ENTRYPOINTS_ALLOC_OBJECT_WITH_ACCESS_CHECK(_region_tlab, RegionTLAB) |
| // GENERATE_ALLOC_ENTRYPOINTS_ALLOC_ARRAY_RESOLVED(_region_tlab, RegionTLAB) |
| GENERATE_ALLOC_ENTRYPOINTS_ALLOC_STRING_FROM_BYTES(_region_tlab, RegionTLAB) |
| GENERATE_ALLOC_ENTRYPOINTS_ALLOC_STRING_FROM_CHARS(_region_tlab, RegionTLAB) |
| GENERATE_ALLOC_ENTRYPOINTS_ALLOC_STRING_FROM_STRING(_region_tlab, RegionTLAB) |
| // Normal TLAB: |
| // GENERATE_ALLOC_ENTRYPOINTS_ALLOC_OBJECT_RESOLVED(_tlab, TLAB) |
| // GENERATE_ALLOC_ENTRYPOINTS_ALLOC_OBJECT_INITIALIZED(_tlab, TLAB) |
| GENERATE_ALLOC_ENTRYPOINTS_ALLOC_OBJECT_WITH_ACCESS_CHECK(_tlab, TLAB) |
| // GENERATE_ALLOC_ENTRYPOINTS_ALLOC_ARRAY_RESOLVED(_tlab, TLAB) |
| GENERATE_ALLOC_ENTRYPOINTS_ALLOC_STRING_FROM_BYTES(_tlab, TLAB) |
| GENERATE_ALLOC_ENTRYPOINTS_ALLOC_STRING_FROM_CHARS(_tlab, TLAB) |
| GENERATE_ALLOC_ENTRYPOINTS_ALLOC_STRING_FROM_STRING(_tlab, TLAB) |
| |
| |
| // A hand-written override for GENERATE_ALLOC_ENTRYPOINTS_ALLOC_OBJECT_RESOLVED(_rosalloc, RosAlloc). |
| DEFINE_FUNCTION art_quick_alloc_object_resolved_rosalloc |
| // Fast path rosalloc allocation. |
| // RDI: mirror::Class*, RAX: return value |
| // RSI, RDX, RCX, R8, R9: free. |
| // Check if the thread local |
| // allocation stack has room. |
| movq %gs:THREAD_SELF_OFFSET, %r8 // r8 = thread |
| movq THREAD_LOCAL_ALLOC_STACK_TOP_OFFSET(%r8), %rcx // rcx = alloc stack top. |
| cmpq THREAD_LOCAL_ALLOC_STACK_END_OFFSET(%r8), %rcx |
| jae .Lart_quick_alloc_object_resolved_rosalloc_slow_path |
| // Load the object size |
| movl MIRROR_CLASS_OBJECT_SIZE_ALLOC_FAST_PATH_OFFSET(%rdi), %eax |
| // Check if the size is for a thread |
| // local allocation. Also does the |
| // initialized and finalizable checks. |
| cmpl LITERAL(ROSALLOC_MAX_THREAD_LOCAL_BRACKET_SIZE), %eax |
| ja .Lart_quick_alloc_object_resolved_rosalloc_slow_path |
| // Compute the rosalloc bracket index |
| // from the size. |
| shrq LITERAL(ROSALLOC_BRACKET_QUANTUM_SIZE_SHIFT), %rax |
| // Load the rosalloc run (r9) |
| // Subtract __SIZEOF_POINTER__ to |
| // subtract one from edi as there is no |
| // 0 byte run and the size is already |
| // aligned. |
| movq (THREAD_ROSALLOC_RUNS_OFFSET - __SIZEOF_POINTER__)(%r8, %rax, __SIZEOF_POINTER__), %r9 |
| // Load the free list head (rax). This |
| // will be the return val. |
| movq (ROSALLOC_RUN_FREE_LIST_OFFSET + ROSALLOC_RUN_FREE_LIST_HEAD_OFFSET)(%r9), %rax |
| testq %rax, %rax |
| jz .Lart_quick_alloc_object_resolved_rosalloc_slow_path |
| // "Point of no slow path". Won't go to the slow path from here on. OK to clobber rdi and rsi. |
| // Push the new object onto the thread |
| // local allocation stack and |
| // increment the thread local |
| // allocation stack top. |
| movl %eax, (%rcx) |
| addq LITERAL(COMPRESSED_REFERENCE_SIZE), %rcx |
| movq %rcx, THREAD_LOCAL_ALLOC_STACK_TOP_OFFSET(%r8) |
| // Load the next pointer of the head |
| // and update the list head with the |
| // next pointer. |
| movq ROSALLOC_SLOT_NEXT_OFFSET(%rax), %rcx |
| movq %rcx, (ROSALLOC_RUN_FREE_LIST_OFFSET + ROSALLOC_RUN_FREE_LIST_HEAD_OFFSET)(%r9) |
| // Store the class pointer in the |
| // header. This also overwrites the |
| // next pointer. The offsets are |
| // asserted to match. |
| #if ROSALLOC_SLOT_NEXT_OFFSET != MIRROR_OBJECT_CLASS_OFFSET |
| #error "Class pointer needs to overwrite next pointer." |
| #endif |
| POISON_HEAP_REF edi |
| movl %edi, MIRROR_OBJECT_CLASS_OFFSET(%rax) |
| // Decrement the size of the free list |
| decl (ROSALLOC_RUN_FREE_LIST_OFFSET + ROSALLOC_RUN_FREE_LIST_SIZE_OFFSET)(%r9) |
| // No fence necessary for x86. |
| ret |
| .Lart_quick_alloc_object_resolved_rosalloc_slow_path: |
| SETUP_SAVE_REFS_ONLY_FRAME // save ref containing registers for GC |
| // Outgoing argument set up |
| movq %gs:THREAD_SELF_OFFSET, %rsi // pass Thread::Current() |
| call SYMBOL(artAllocObjectFromCodeResolvedRosAlloc) // cxx_name(arg0, Thread*) |
| RESTORE_SAVE_REFS_ONLY_FRAME // restore frame up to return address |
| RETURN_IF_RESULT_IS_NON_ZERO_OR_DELIVER // return or deliver exception |
| END_FUNCTION art_quick_alloc_object_rosalloc |
| |
| // The common fast path code for art_quick_alloc_object_tlab and art_quick_alloc_object_region_tlab. |
| // |
| // RDI: type_idx, RSI: ArtMethod*, RDX/EDX: the class, RAX: return value. |
| // RCX: scratch, r8: Thread::Current(). |
| MACRO1(ALLOC_OBJECT_TLAB_FAST_PATH, slowPathLabel) |
| testl %edx, %edx // Check null class |
| jz RAW_VAR(slowPathLabel) |
| ALLOC_OBJECT_RESOLVED_TLAB_FAST_PATH(RAW_VAR(slowPathLabel)) |
| END_MACRO |
| |
| // The common fast path code for art_quick_alloc_object_resolved_region_tlab. |
| // TODO: delete ALLOC_OBJECT_RESOLVED_TLAB_FAST_PATH since it is the same as |
| // ALLOC_OBJECT_INITIALIZED_TLAB_FAST_PATH. |
| // |
| // RDI: the class, RAX: return value. |
| // RCX, RSI, RDX: scratch, r8: Thread::Current(). |
| MACRO1(ALLOC_OBJECT_RESOLVED_TLAB_FAST_PATH, slowPathLabel) |
| ALLOC_OBJECT_INITIALIZED_TLAB_FAST_PATH(RAW_VAR(slowPathLabel)) |
| END_MACRO |
| |
| // The fast path code for art_quick_alloc_object_initialized_region_tlab. |
| // |
| // RDI: the class, RSI: ArtMethod*, RAX: return value. |
| // RCX, RSI, RDX: scratch, r8: Thread::Current(). |
| MACRO1(ALLOC_OBJECT_INITIALIZED_TLAB_FAST_PATH, slowPathLabel) |
| movq %gs:THREAD_SELF_OFFSET, %r8 // r8 = thread |
| movl MIRROR_CLASS_OBJECT_SIZE_ALLOC_FAST_PATH_OFFSET(%rdi), %ecx // Load the object size. |
| movq THREAD_LOCAL_POS_OFFSET(%r8), %rax |
| addq %rax, %rcx // Add size to pos, note that these |
| // are both 32 bit ints, overflow |
| // will cause the add to be past the |
| // end of the thread local region. |
| cmpq THREAD_LOCAL_END_OFFSET(%r8), %rcx // Check if it fits. |
| ja RAW_VAR(slowPathLabel) |
| movq %rcx, THREAD_LOCAL_POS_OFFSET(%r8) // Update thread_local_pos. |
| incq THREAD_LOCAL_OBJECTS_OFFSET(%r8) // Increase thread_local_objects. |
| // Store the class pointer in the |
| // header. |
| // No fence needed for x86. |
| POISON_HEAP_REF edi |
| movl %edi, MIRROR_OBJECT_CLASS_OFFSET(%rax) |
| ret // Fast path succeeded. |
| END_MACRO |
| |
| // The fast path code for art_quick_alloc_array_region_tlab. |
| // Inputs: RDI: the class, RSI: int32_t component_count |
| // Free temps: RCX, RDX, R8, R9 |
| // Output: RAX: return value. |
| MACRO1(ALLOC_ARRAY_TLAB_FAST_PATH_RESOLVED, slowPathLabel) |
| movl MIRROR_CLASS_COMPONENT_TYPE_OFFSET(%rdi), %ecx // Load component type. |
| UNPOISON_HEAP_REF ecx |
| movl MIRROR_CLASS_OBJECT_PRIMITIVE_TYPE_OFFSET(%rcx), %ecx // Load primitive type. |
| shrq LITERAL(PRIMITIVE_TYPE_SIZE_SHIFT_SHIFT), %rcx // Get component size shift. |
| movq %rsi, %r9 |
| salq %cl, %r9 // Calculate array count shifted. |
| // Add array header + alignment rounding. |
| addq LITERAL(MIRROR_INT_ARRAY_DATA_OFFSET + OBJECT_ALIGNMENT_MASK), %r9 |
| // Add 4 extra bytes if we are doing a long array. |
| addq LITERAL(1), %rcx |
| andq LITERAL(4), %rcx |
| addq %rcx, %r9 |
| movq %gs:THREAD_SELF_OFFSET, %rcx // rcx = thread |
| #if MIRROR_LONG_ARRAY_DATA_OFFSET != MIRROR_INT_ARRAY_DATA_OFFSET + 4 |
| #error Long array data offset must be 4 greater than int array data offset. |
| #endif |
| // Mask out the unaligned part to make sure we are 8 byte aligned. |
| andq LITERAL(OBJECT_ALIGNMENT_MASK_TOGGLED64), %r9 |
| movq THREAD_LOCAL_POS_OFFSET(%rcx), %rax |
| addq %rax, %r9 |
| cmpq THREAD_LOCAL_END_OFFSET(%rcx), %r9 // Check if it fits. |
| ja RAW_VAR(slowPathLabel) |
| movq %r9, THREAD_LOCAL_POS_OFFSET(%rcx) // Update thread_local_pos. |
| addq LITERAL(1), THREAD_LOCAL_OBJECTS_OFFSET(%rcx) // Increase thread_local_objects. |
| // Store the class pointer in the |
| // header. |
| // No fence needed for x86. |
| POISON_HEAP_REF edi |
| movl %edi, MIRROR_OBJECT_CLASS_OFFSET(%rax) |
| movl %esi, MIRROR_ARRAY_LENGTH_OFFSET(%rax) |
| ret // Fast path succeeded. |
| END_MACRO |
| |
| |
| // The common slow path code for art_quick_alloc_object_{resolved, initialized}_tlab |
| // and art_quick_alloc_object_{resolved, initialized}_region_tlab. |
| MACRO1(ALLOC_OBJECT_TLAB_SLOW_PATH, cxx_name) |
| SETUP_SAVE_REFS_ONLY_FRAME // save ref containing registers for GC |
| // Outgoing argument set up |
| movq %gs:THREAD_SELF_OFFSET, %rsi // pass Thread::Current() |
| call CALLVAR(cxx_name) // cxx_name(arg0, Thread*) |
| RESTORE_SAVE_REFS_ONLY_FRAME // restore frame up to return address |
| RETURN_IF_RESULT_IS_NON_ZERO_OR_DELIVER // return or deliver exception |
| END_MACRO |
| |
| // The slow path code for art_quick_alloc_array_region_tlab. |
| MACRO1(ALLOC_ARRAY_TLAB_SLOW_PATH, cxx_name) |
| SETUP_SAVE_REFS_ONLY_FRAME // save ref containing registers for GC |
| // Outgoing argument set up |
| movq %gs:THREAD_SELF_OFFSET, %rdx // pass Thread::Current() |
| call CALLVAR(cxx_name) // cxx_name(arg0, arg1, Thread*) |
| RESTORE_SAVE_REFS_ONLY_FRAME // restore frame up to return address |
| RETURN_IF_RESULT_IS_NON_ZERO_OR_DELIVER // return or deliver exception |
| END_MACRO |
| |
| // A hand-written override for GENERATE_ALLOC_ENTRYPOINTS_ALLOC_OBJECT_RESOLVED(_tlab, TLAB). May be |
| // called with CC if the GC is not active. |
| DEFINE_FUNCTION art_quick_alloc_object_resolved_tlab |
| // RDI: mirror::Class* klass |
| // RDX, RSI, RCX, R8, R9: free. RAX: return val. |
| ALLOC_OBJECT_RESOLVED_TLAB_FAST_PATH .Lart_quick_alloc_object_resolved_tlab_slow_path |
| .Lart_quick_alloc_object_resolved_tlab_slow_path: |
| ALLOC_OBJECT_TLAB_SLOW_PATH artAllocObjectFromCodeResolvedTLAB |
| END_FUNCTION art_quick_alloc_object_resolved_tlab |
| |
| // A hand-written override for GENERATE_ALLOC_ENTRYPOINTS_ALLOC_OBJECT_INITIALIZED(_tlab, TLAB). |
| // May be called with CC if the GC is not active. |
| DEFINE_FUNCTION art_quick_alloc_object_initialized_tlab |
| // RDI: mirror::Class* klass |
| // RDX, RSI, RCX, R8, R9: free. RAX: return val. |
| ALLOC_OBJECT_INITIALIZED_TLAB_FAST_PATH .Lart_quick_alloc_object_initialized_tlab_slow_path |
| .Lart_quick_alloc_object_initialized_tlab_slow_path: |
| ALLOC_OBJECT_TLAB_SLOW_PATH artAllocObjectFromCodeInitializedTLAB |
| END_FUNCTION art_quick_alloc_object_initialized_tlab |
| |
| // A hand-written override for GENERATE_ALLOC_ENTRYPOINTS_ALLOC_ARRAY_RESOLVED(_tlab, TLAB). |
| DEFINE_FUNCTION art_quick_alloc_array_resolved_tlab |
| // RDI: mirror::Class* klass, RSI: int32_t component_count |
| // RDX, RCX, R8, R9: free. RAX: return val. |
| ALLOC_ARRAY_TLAB_FAST_PATH_RESOLVED .Lart_quick_alloc_array_resolved_tlab_slow_path |
| .Lart_quick_alloc_array_resolved_tlab_slow_path: |
| ALLOC_ARRAY_TLAB_SLOW_PATH artAllocArrayFromCodeResolvedTLAB |
| END_FUNCTION art_quick_alloc_array_resolved_tlab |
| |
| // A hand-written override for GENERATE_ALLOC_ENTRYPOINTS_ALLOC_ARRAY_RESOLVED(_region_tlab, RegionTLAB). |
| DEFINE_FUNCTION art_quick_alloc_array_resolved_region_tlab |
| // Fast path region tlab allocation. |
| // RDI: mirror::Class* klass, RSI: int32_t component_count |
| // RCX, RDX, R8, R9: free. RAX: return val. |
| ASSERT_USE_READ_BARRIER |
| ALLOC_ARRAY_TLAB_FAST_PATH_RESOLVED .Lart_quick_alloc_array_resolved_region_tlab_slow_path |
| .Lart_quick_alloc_array_resolved_region_tlab_slow_path: |
| ALLOC_ARRAY_TLAB_SLOW_PATH artAllocArrayFromCodeResolvedRegionTLAB |
| END_FUNCTION art_quick_alloc_array_resolved_region_tlab |
| |
| // A hand-written override for GENERATE_ALLOC_ENTRYPOINTS_ALLOC_OBJECT_RESOLVED(_region_tlab, RegionTLAB). |
| DEFINE_FUNCTION art_quick_alloc_object_resolved_region_tlab |
| // Fast path region tlab allocation. |
| // RDI: mirror::Class* klass |
| // RDX, RSI, RCX, R8, R9: free. RAX: return val. |
| ASSERT_USE_READ_BARRIER |
| ALLOC_OBJECT_RESOLVED_TLAB_FAST_PATH .Lart_quick_alloc_object_resolved_region_tlab_slow_path |
| .Lart_quick_alloc_object_resolved_region_tlab_slow_path: |
| ALLOC_OBJECT_TLAB_SLOW_PATH artAllocObjectFromCodeResolvedRegionTLAB |
| END_FUNCTION art_quick_alloc_object_resolved_region_tlab |
| |
| // A hand-written override for GENERATE_ALLOC_ENTRYPOINTS_ALLOC_OBJECT_INITIALIZED(_region_tlab, RegionTLAB). |
| DEFINE_FUNCTION art_quick_alloc_object_initialized_region_tlab |
| // Fast path region tlab allocation. |
| // RDI: mirror::Class* klass |
| // RDX, RSI, RCX, R8, R9: free. RAX: return val. |
| ASSERT_USE_READ_BARRIER |
| // No read barrier since the caller is responsible for that. |
| ALLOC_OBJECT_INITIALIZED_TLAB_FAST_PATH .Lart_quick_alloc_object_initialized_region_tlab_slow_path |
| .Lart_quick_alloc_object_initialized_region_tlab_slow_path: |
| ALLOC_OBJECT_TLAB_SLOW_PATH artAllocObjectFromCodeInitializedRegionTLAB |
| END_FUNCTION art_quick_alloc_object_initialized_region_tlab |
| |
| DEFINE_FUNCTION art_quick_resolve_string |
| SETUP_SAVE_EVERYTHING_FRAME |
| // Outgoing argument set up |
| movl %eax, %edi // pass string index |
| movq %gs:THREAD_SELF_OFFSET, %rsi // pass Thread::Current() |
| call SYMBOL(artResolveStringFromCode) // artResolveStringFromCode(arg0, referrer, Thread*) |
| |
| testl %eax, %eax // If result is null, deliver the OOME. |
| jz 1f |
| CFI_REMEMBER_STATE |
| RESTORE_SAVE_EVERYTHING_FRAME_KEEP_RAX // restore frame up to return address |
| ret |
| CFI_RESTORE_STATE |
| CFI_DEF_CFA(rsp, FRAME_SIZE_SAVE_EVERYTHING) // workaround for clang bug: 31975598 |
| 1: |
| DELIVER_PENDING_EXCEPTION_FRAME_READY |
| END_FUNCTION art_quick_resolve_string |
| |
| ONE_ARG_DOWNCALL art_quick_initialize_static_storage, artInitializeStaticStorageFromCode, RETURN_IF_RESULT_IS_NON_ZERO_OR_DELIVER |
| ONE_ARG_DOWNCALL art_quick_initialize_type, artInitializeTypeFromCode, RETURN_IF_RESULT_IS_NON_ZERO_OR_DELIVER |
| ONE_ARG_DOWNCALL art_quick_initialize_type_and_verify_access, artInitializeTypeAndVerifyAccessFromCode, RETURN_IF_RESULT_IS_NON_ZERO_OR_DELIVER |
| |
| TWO_ARG_REF_DOWNCALL art_quick_handle_fill_data, artHandleFillArrayDataFromCode, RETURN_IF_EAX_ZERO |
| |
| DEFINE_FUNCTION art_quick_lock_object |
| testl %edi, %edi // Null check object/rdi. |
| jz .Lslow_lock |
| .Lretry_lock: |
| movl MIRROR_OBJECT_LOCK_WORD_OFFSET(%edi), %ecx // ecx := lock word. |
| test LITERAL(LOCK_WORD_STATE_MASK), %ecx // Test the 2 high bits. |
| jne .Lslow_lock // Slow path if either of the two high bits are set. |
| movl %ecx, %edx // save lock word (edx) to keep read barrier bits. |
| andl LITERAL(LOCK_WORD_GC_STATE_MASK_SHIFTED_TOGGLED), %ecx // zero the gc bits. |
| test %ecx, %ecx |
| jnz .Lalready_thin // Lock word contains a thin lock. |
| // unlocked case - edx: original lock word, edi: obj. |
| movl %edx, %eax // eax: lock word zero except for read barrier bits. |
| movl %gs:THREAD_ID_OFFSET, %edx // edx := thread id |
| or %eax, %edx // edx: thread id with count of 0 + read barrier bits. |
| lock cmpxchg %edx, MIRROR_OBJECT_LOCK_WORD_OFFSET(%edi) |
| jnz .Lretry_lock // cmpxchg failed retry |
| ret |
| .Lalready_thin: // edx: lock word (with high 2 bits zero and original rb bits), edi: obj. |
| movl %gs:THREAD_ID_OFFSET, %ecx // ecx := thread id |
| cmpw %cx, %dx // do we hold the lock already? |
| jne .Lslow_lock |
| movl %edx, %ecx // copy the lock word to check count overflow. |
| andl LITERAL(LOCK_WORD_GC_STATE_MASK_SHIFTED_TOGGLED), %ecx // zero the gc bits. |
| addl LITERAL(LOCK_WORD_THIN_LOCK_COUNT_ONE), %ecx // increment recursion count |
| test LITERAL(LOCK_WORD_READ_BARRIER_STATE_MASK), %ecx // overflowed if the upper bit (28) is set |
| jne .Lslow_lock // count overflowed so go slow |
| movl %edx, %eax // copy the lock word as the old val for cmpxchg. |
| addl LITERAL(LOCK_WORD_THIN_LOCK_COUNT_ONE), %edx // increment recursion count again for real. |
| // update lockword, cmpxchg necessary for read barrier bits. |
| lock cmpxchg %edx, MIRROR_OBJECT_LOCK_WORD_OFFSET(%edi) // eax: old val, edx: new val. |
| jnz .Lretry_lock // cmpxchg failed retry |
| ret |
| .Lslow_lock: |
| SETUP_SAVE_REFS_ONLY_FRAME |
| movq %gs:THREAD_SELF_OFFSET, %rsi // pass Thread::Current() |
| call SYMBOL(artLockObjectFromCode) // artLockObjectFromCode(object, Thread*) |
| RESTORE_SAVE_REFS_ONLY_FRAME // restore frame up to return address |
| RETURN_IF_EAX_ZERO |
| END_FUNCTION art_quick_lock_object |
| |
| DEFINE_FUNCTION art_quick_lock_object_no_inline |
| SETUP_SAVE_REFS_ONLY_FRAME |
| movq %gs:THREAD_SELF_OFFSET, %rsi // pass Thread::Current() |
| call SYMBOL(artLockObjectFromCode) // artLockObjectFromCode(object, Thread*) |
| RESTORE_SAVE_REFS_ONLY_FRAME // restore frame up to return address |
| RETURN_IF_EAX_ZERO |
| END_FUNCTION art_quick_lock_object_no_inline |
| |
| DEFINE_FUNCTION art_quick_unlock_object |
| testl %edi, %edi // null check object/edi |
| jz .Lslow_unlock |
| .Lretry_unlock: |
| movl MIRROR_OBJECT_LOCK_WORD_OFFSET(%edi), %ecx // ecx := lock word |
| movl %gs:THREAD_ID_OFFSET, %edx // edx := thread id |
| test LITERAL(LOCK_WORD_STATE_MASK), %ecx |
| jnz .Lslow_unlock // lock word contains a monitor |
| cmpw %cx, %dx // does the thread id match? |
| jne .Lslow_unlock |
| movl %ecx, %edx // copy the lock word to detect new count of 0. |
| andl LITERAL(LOCK_WORD_GC_STATE_MASK_SHIFTED_TOGGLED), %edx // zero the gc bits. |
| cmpl LITERAL(LOCK_WORD_THIN_LOCK_COUNT_ONE), %edx |
| jae .Lrecursive_thin_unlock |
| // update lockword, cmpxchg necessary for read barrier bits. |
| movl %ecx, %eax // eax: old lock word. |
| andl LITERAL(LOCK_WORD_GC_STATE_MASK_SHIFTED), %ecx // ecx: new lock word zero except original gc bits. |
| #ifndef USE_READ_BARRIER |
| movl %ecx, MIRROR_OBJECT_LOCK_WORD_OFFSET(%edi) |
| #else |
| lock cmpxchg %ecx, MIRROR_OBJECT_LOCK_WORD_OFFSET(%edi) // eax: old val, ecx: new val. |
| jnz .Lretry_unlock // cmpxchg failed retry |
| #endif |
| ret |
| .Lrecursive_thin_unlock: // ecx: original lock word, edi: obj |
| // update lockword, cmpxchg necessary for read barrier bits. |
| movl %ecx, %eax // eax: old lock word. |
| subl LITERAL(LOCK_WORD_THIN_LOCK_COUNT_ONE), %ecx |
| #ifndef USE_READ_BARRIER |
| mov %ecx, MIRROR_OBJECT_LOCK_WORD_OFFSET(%edi) |
| #else |
| lock cmpxchg %ecx, MIRROR_OBJECT_LOCK_WORD_OFFSET(%edi) // eax: old val, ecx: new val. |
| jnz .Lretry_unlock // cmpxchg failed retry |
| #endif |
| ret |
| .Lslow_unlock: |
| SETUP_SAVE_REFS_ONLY_FRAME |
| movq %gs:THREAD_SELF_OFFSET, %rsi // pass Thread::Current() |
| call SYMBOL(artUnlockObjectFromCode) // artUnlockObjectFromCode(object, Thread*) |
| RESTORE_SAVE_REFS_ONLY_FRAME // restore frame up to return address |
| RETURN_IF_EAX_ZERO |
| END_FUNCTION art_quick_unlock_object |
| |
| DEFINE_FUNCTION art_quick_unlock_object_no_inline |
| SETUP_SAVE_REFS_ONLY_FRAME |
| movq %gs:THREAD_SELF_OFFSET, %rsi // pass Thread::Current() |
| call SYMBOL(artUnlockObjectFromCode) // artUnlockObjectFromCode(object, Thread*) |
| RESTORE_SAVE_REFS_ONLY_FRAME // restore frame up to return address |
| RETURN_IF_EAX_ZERO |
| END_FUNCTION art_quick_unlock_object_no_inline |
| |
| DEFINE_FUNCTION art_quick_check_instance_of |
| // We could check the super classes here but that is usually already checked in the caller. |
| PUSH rdi // Save args for exc |
| PUSH rsi |
| subq LITERAL(8), %rsp // Alignment padding. |
| CFI_ADJUST_CFA_OFFSET(8) |
| SETUP_FP_CALLEE_SAVE_FRAME |
| call SYMBOL(artInstanceOfFromCode) // (Object* obj, Class* ref_klass) |
| testq %rax, %rax |
| jz 1f // jump forward if not assignable |
| RESTORE_FP_CALLEE_SAVE_FRAME |
| addq LITERAL(24), %rsp // pop arguments |
| CFI_ADJUST_CFA_OFFSET(-24) |
| |
| .Lreturn: |
| ret |
| |
| CFI_ADJUST_CFA_OFFSET(24 + 4 * 8) // Reset unwind info so following code unwinds. |
| 1: |
| RESTORE_FP_CALLEE_SAVE_FRAME |
| addq LITERAL(8), %rsp // pop padding |
| CFI_ADJUST_CFA_OFFSET(-8) |
| POP rsi // Pop arguments |
| POP rdi |
| SETUP_SAVE_ALL_CALLEE_SAVES_FRAME // save all registers as basis for long jump context |
| mov %gs:THREAD_SELF_OFFSET, %rdx // pass Thread::Current() |
| call SYMBOL(artThrowClassCastExceptionForObject) // (Object* src, Class* dest, Thread*) |
| UNREACHABLE |
| END_FUNCTION art_quick_check_instance_of |
| |
| |
| // Restore reg's value if reg is not the same as exclude_reg, otherwise just adjust stack. |
| MACRO2(POP_REG_NE, reg, exclude_reg) |
| .ifc RAW_VAR(reg), RAW_VAR(exclude_reg) |
| addq MACRO_LITERAL(8), %rsp |
| CFI_ADJUST_CFA_OFFSET(-8) |
| .else |
| POP RAW_VAR(reg) |
| .endif |
| END_MACRO |
| |
| /* |
| * Macro to insert read barrier, used in art_quick_aput_obj. |
| * obj_reg and dest_reg{32|64} are registers, offset is a defined literal such as |
| * MIRROR_OBJECT_CLASS_OFFSET. dest_reg needs two versions to handle the mismatch between |
| * 64b PUSH/POP and 32b argument. |
| * TODO: When read barrier has a fast path, add heap unpoisoning support for the fast path. |
| * |
| * As with art_quick_aput_obj function, the 64b versions are in comments. |
| */ |
| MACRO4(READ_BARRIER, obj_reg, offset, dest_reg32, dest_reg64) |
| #ifdef USE_READ_BARRIER |
| PUSH rax // save registers that might be used |
| PUSH rdi |
| PUSH rsi |
| PUSH rdx |
| PUSH rcx |
| SETUP_FP_CALLEE_SAVE_FRAME |
| // Outgoing argument set up |
| // movl REG_VAR(ref_reg32), %edi // pass ref, no-op for now since parameter ref is unused |
| // // movq REG_VAR(ref_reg64), %rdi |
| movl REG_VAR(obj_reg), %esi // pass obj_reg |
| // movq REG_VAR(obj_reg), %rsi |
| movl MACRO_LITERAL((RAW_VAR(offset))), %edx // pass offset, double parentheses are necessary |
| // movq MACRO_LITERAL((RAW_VAR(offset))), %rdx |
| call SYMBOL(artReadBarrierSlow) // artReadBarrierSlow(ref, obj_reg, offset) |
| // No need to unpoison return value in rax, artReadBarrierSlow() would do the unpoisoning. |
| .ifnc RAW_VAR(dest_reg32), eax |
| // .ifnc RAW_VAR(dest_reg64), rax |
| movl %eax, REG_VAR(dest_reg32) // save loaded ref in dest_reg |
| // movq %rax, REG_VAR(dest_reg64) |
| .endif |
| RESTORE_FP_CALLEE_SAVE_FRAME |
| POP_REG_NE rcx, RAW_VAR(dest_reg64) // Restore registers except dest_reg |
| POP_REG_NE rdx, RAW_VAR(dest_reg64) |
| POP_REG_NE rsi, RAW_VAR(dest_reg64) |
| POP_REG_NE rdi, RAW_VAR(dest_reg64) |
| POP_REG_NE rax, RAW_VAR(dest_reg64) |
| #else |
| movl RAW_VAR(offset)(REG_VAR(obj_reg)), REG_VAR(dest_reg32) |
| // movq RAW_VAR(offset)(REG_VAR(obj_reg)), REG_VAR(dest_reg64) |
| UNPOISON_HEAP_REF RAW_VAR(dest_reg32) // UNPOISON_HEAP_REF only takes a 32b register |
| #endif // USE_READ_BARRIER |
| END_MACRO |
| |
| DEFINE_FUNCTION art_quick_aput_obj |
| testl %edx, %edx // store of null |
| // test %rdx, %rdx |
| jz .Ldo_aput_null |
| READ_BARRIER edi, MIRROR_OBJECT_CLASS_OFFSET, ecx, rcx |
| // READ_BARRIER rdi, MIRROR_OBJECT_CLASS_OFFSET, ecx, rcx |
| READ_BARRIER ecx, MIRROR_CLASS_COMPONENT_TYPE_OFFSET, ecx, rcx |
| // READ_BARRIER rcx, MIRROR_CLASS_COMPONENT_TYPE_OFFSET, ecx, rcx |
| #if defined(USE_HEAP_POISONING) || defined(USE_READ_BARRIER) |
| READ_BARRIER edx, MIRROR_OBJECT_CLASS_OFFSET, eax, rax // rax is free. |
| // READ_BARRIER rdx, MIRROR_OBJECT_CLASS_OFFSET, eax, rax |
| cmpl %eax, %ecx // value's type == array's component type - trivial assignability |
| #else |
| cmpl MIRROR_OBJECT_CLASS_OFFSET(%edx), %ecx // value's type == array's component type - trivial assignability |
| // cmpq MIRROR_CLASS_OFFSET(%rdx), %rcx |
| #endif |
| jne .Lcheck_assignability |
| .Ldo_aput: |
| POISON_HEAP_REF edx |
| movl %edx, MIRROR_OBJECT_ARRAY_DATA_OFFSET(%edi, %esi, 4) |
| // movq %rdx, MIRROR_OBJECT_ARRAY_DATA_OFFSET(%rdi, %rsi, 4) |
| movq %gs:THREAD_CARD_TABLE_OFFSET, %rdx |
| shrl LITERAL(7), %edi |
| // shrl LITERAL(7), %rdi |
| movb %dl, (%rdx, %rdi) // Note: this assumes that top 32b of %rdi are zero |
| ret |
| .Ldo_aput_null: |
| movl %edx, MIRROR_OBJECT_ARRAY_DATA_OFFSET(%edi, %esi, 4) |
| // movq %rdx, MIRROR_OBJECT_ARRAY_DATA_OFFSET(%rdi, %rsi, 4) |
| ret |
| .Lcheck_assignability: |
| // Save arguments. |
| PUSH rdi |
| PUSH rsi |
| PUSH rdx |
| SETUP_FP_CALLEE_SAVE_FRAME |
| |
| #if defined(USE_HEAP_POISONING) || defined(USE_READ_BARRIER) |
| // The load of MIRROR_OBJECT_CLASS_OFFSET(%edx) is redundant, eax still holds the value. |
| movl %eax, %esi // Pass arg2 = value's class. |
| // movq %rax, %rsi |
| #else |
| // "Uncompress" = do nothing, as already zero-extended on load. |
| movl MIRROR_OBJECT_CLASS_OFFSET(%edx), %esi // Pass arg2 = value's class. |
| #endif |
| movq %rcx, %rdi // Pass arg1 = array's component type. |
| |
| call SYMBOL(artIsAssignableFromCode) // (Class* a, Class* b) |
| |
| // Exception? |
| testq %rax, %rax |
| jz .Lthrow_array_store_exception |
| |
| RESTORE_FP_CALLEE_SAVE_FRAME |
| // Restore arguments. |
| POP rdx |
| POP rsi |
| POP rdi |
| |
| POISON_HEAP_REF edx |
| movl %edx, MIRROR_OBJECT_ARRAY_DATA_OFFSET(%edi, %esi, 4) |
| // movq %rdx, MIRROR_OBJECT_ARRAY_DATA_OFFSET(%rdi, %rsi, 4) |
| movq %gs:THREAD_CARD_TABLE_OFFSET, %rdx |
| shrl LITERAL(7), %edi |
| // shrl LITERAL(7), %rdi |
| movb %dl, (%rdx, %rdi) // Note: this assumes that top 32b of %rdi are zero |
| // movb %dl, (%rdx, %rdi) |
| ret |
| CFI_ADJUST_CFA_OFFSET(24 + 4 * 8) // Reset unwind info so following code unwinds. |
| .Lthrow_array_store_exception: |
| RESTORE_FP_CALLEE_SAVE_FRAME |
| // Restore arguments. |
| POP rdx |
| POP rsi |
| POP rdi |
| |
| SETUP_SAVE_ALL_CALLEE_SAVES_FRAME // Save all registers as basis for long jump context. |
| |
| // Outgoing argument set up. |
| movq %rdx, %rsi // Pass arg 2 = value. |
| movq %gs:THREAD_SELF_OFFSET, %rdx // Pass arg 3 = Thread::Current(). |
| // Pass arg 1 = array. |
| call SYMBOL(artThrowArrayStoreException) // (array, value, Thread*) |
| UNREACHABLE |
| END_FUNCTION art_quick_aput_obj |
| |
| // TODO: This is quite silly on X86_64 now. |
| DEFINE_FUNCTION art_quick_memcpy |
| call PLT_SYMBOL(memcpy) // (void*, const void*, size_t) |
| ret |
| END_FUNCTION art_quick_memcpy |
| |
| DEFINE_FUNCTION art_quick_test_suspend |
| SETUP_SAVE_EVERYTHING_FRAME // save everything for GC |
| // Outgoing argument set up |
| movq %gs:THREAD_SELF_OFFSET, %rdi // pass Thread::Current() |
| call SYMBOL(artTestSuspendFromCode) // (Thread*) |
| RESTORE_SAVE_EVERYTHING_FRAME // restore frame up to return address |
| ret |
| END_FUNCTION art_quick_test_suspend |
| |
| UNIMPLEMENTED art_quick_ldiv |
| UNIMPLEMENTED art_quick_lmod |
| UNIMPLEMENTED art_quick_lmul |
| UNIMPLEMENTED art_quick_lshl |
| UNIMPLEMENTED art_quick_lshr |
| UNIMPLEMENTED art_quick_lushr |
| |
| THREE_ARG_REF_DOWNCALL art_quick_set8_instance, artSet8InstanceFromCode, RETURN_IF_EAX_ZERO |
| THREE_ARG_REF_DOWNCALL art_quick_set16_instance, artSet16InstanceFromCode, RETURN_IF_EAX_ZERO |
| THREE_ARG_REF_DOWNCALL art_quick_set32_instance, artSet32InstanceFromCode, RETURN_IF_EAX_ZERO |
| THREE_ARG_REF_DOWNCALL art_quick_set64_instance, artSet64InstanceFromCode, RETURN_IF_EAX_ZERO |
| THREE_ARG_REF_DOWNCALL art_quick_set_obj_instance, artSetObjInstanceFromCode, RETURN_IF_EAX_ZERO |
| |
| TWO_ARG_REF_DOWNCALL art_quick_get_byte_instance, artGetByteInstanceFromCode, RETURN_OR_DELIVER_PENDING_EXCEPTION |
| TWO_ARG_REF_DOWNCALL art_quick_get_boolean_instance, artGetBooleanInstanceFromCode, RETURN_OR_DELIVER_PENDING_EXCEPTION |
| TWO_ARG_REF_DOWNCALL art_quick_get_short_instance, artGetShortInstanceFromCode, RETURN_OR_DELIVER_PENDING_EXCEPTION |
| TWO_ARG_REF_DOWNCALL art_quick_get_char_instance, artGetCharInstanceFromCode, RETURN_OR_DELIVER_PENDING_EXCEPTION |
| TWO_ARG_REF_DOWNCALL art_quick_get32_instance, artGet32InstanceFromCode, RETURN_OR_DELIVER_PENDING_EXCEPTION |
| TWO_ARG_REF_DOWNCALL art_quick_get64_instance, artGet64InstanceFromCode, RETURN_OR_DELIVER_PENDING_EXCEPTION |
| TWO_ARG_REF_DOWNCALL art_quick_get_obj_instance, artGetObjInstanceFromCode, RETURN_OR_DELIVER_PENDING_EXCEPTION |
| |
| TWO_ARG_REF_DOWNCALL art_quick_set8_static, artSet8StaticFromCode, RETURN_IF_EAX_ZERO |
| TWO_ARG_REF_DOWNCALL art_quick_set16_static, artSet16StaticFromCode, RETURN_IF_EAX_ZERO |
| TWO_ARG_REF_DOWNCALL art_quick_set32_static, artSet32StaticFromCode, RETURN_IF_EAX_ZERO |
| TWO_ARG_REF_DOWNCALL art_quick_set_obj_static, artSetObjStaticFromCode, RETURN_IF_EAX_ZERO |
| |
| ONE_ARG_REF_DOWNCALL art_quick_get_byte_static, artGetByteStaticFromCode, RETURN_OR_DELIVER_PENDING_EXCEPTION |
| ONE_ARG_REF_DOWNCALL art_quick_get_boolean_static, artGetBooleanStaticFromCode, RETURN_OR_DELIVER_PENDING_EXCEPTION |
| ONE_ARG_REF_DOWNCALL art_quick_get_short_static, artGetShortStaticFromCode, RETURN_OR_DELIVER_PENDING_EXCEPTION |
| ONE_ARG_REF_DOWNCALL art_quick_get_char_static, artGetCharStaticFromCode, RETURN_OR_DELIVER_PENDING_EXCEPTION |
| ONE_ARG_REF_DOWNCALL art_quick_get32_static, artGet32StaticFromCode, RETURN_OR_DELIVER_PENDING_EXCEPTION |
| ONE_ARG_REF_DOWNCALL art_quick_get64_static, artGet64StaticFromCode, RETURN_OR_DELIVER_PENDING_EXCEPTION |
| ONE_ARG_REF_DOWNCALL art_quick_get_obj_static, artGetObjStaticFromCode, RETURN_OR_DELIVER_PENDING_EXCEPTION |
| |
| // This is singled out as the argument order is different. |
| DEFINE_FUNCTION art_quick_set64_static |
| // new_val is already in %rdx |
| movq 8(%rsp), %rsi // pass referrer |
| SETUP_SAVE_REFS_ONLY_FRAME |
| // field_idx is in rdi |
| movq %gs:THREAD_SELF_OFFSET, %rcx // pass Thread::Current() |
| call SYMBOL(artSet64StaticFromCode) // (field_idx, referrer, new_val, Thread*) |
| RESTORE_SAVE_REFS_ONLY_FRAME // restore frame up to return address |
| RETURN_IF_EAX_ZERO // return or deliver exception |
| END_FUNCTION art_quick_set64_static |
| |
| |
| DEFINE_FUNCTION art_quick_proxy_invoke_handler |
| SETUP_SAVE_REFS_AND_ARGS_FRAME_WITH_METHOD_IN_RDI |
| |
| movq %gs:THREAD_SELF_OFFSET, %rdx // Pass Thread::Current(). |
| movq %rsp, %rcx // Pass SP. |
| call SYMBOL(artQuickProxyInvokeHandler) // (proxy method, receiver, Thread*, SP) |
| RESTORE_SAVE_REFS_AND_ARGS_FRAME |
| movq %rax, %xmm0 // Copy return value in case of float returns. |
| RETURN_OR_DELIVER_PENDING_EXCEPTION |
| END_FUNCTION art_quick_proxy_invoke_handler |
| |
| /* |
| * Called to resolve an imt conflict. |
| * rdi is the conflict ArtMethod. |
| * rax is a hidden argument that holds the target interface method's dex method index. |
| * |
| * Note that this stub writes to r10 and rdi. |
| */ |
| DEFINE_FUNCTION art_quick_imt_conflict_trampoline |
| #if defined(__APPLE__) |
| int3 |
| int3 |
| #else |
| movq __SIZEOF_POINTER__(%rsp), %r10 // Load referrer |
| movq ART_METHOD_DEX_CACHE_METHODS_OFFSET_64(%r10), %r10 // Load dex cache methods array |
| movq 0(%r10, %rax, __SIZEOF_POINTER__), %r10 // Load interface method |
| movq ART_METHOD_JNI_OFFSET_64(%rdi), %rdi // Load ImtConflictTable |
| .Limt_table_iterate: |
| cmpq %r10, 0(%rdi) |
| jne .Limt_table_next_entry |
| // We successfully hit an entry in the table. Load the target method |
| // and jump to it. |
| movq __SIZEOF_POINTER__(%rdi), %rdi |
| jmp *ART_METHOD_QUICK_CODE_OFFSET_64(%rdi) |
| .Limt_table_next_entry: |
| // If the entry is null, the interface method is not in the ImtConflictTable. |
| cmpq LITERAL(0), 0(%rdi) |
| jz .Lconflict_trampoline |
| // Iterate over the entries of the ImtConflictTable. |
| addq LITERAL(2 * __SIZEOF_POINTER__), %rdi |
| jmp .Limt_table_iterate |
| .Lconflict_trampoline: |
| // Call the runtime stub to populate the ImtConflictTable and jump to the |
| // resolved method. |
| INVOKE_TRAMPOLINE_BODY artInvokeInterfaceTrampoline |
| #endif // __APPLE__ |
| END_FUNCTION art_quick_imt_conflict_trampoline |
| |
| DEFINE_FUNCTION art_quick_resolution_trampoline |
| SETUP_SAVE_REFS_AND_ARGS_FRAME |
| movq %gs:THREAD_SELF_OFFSET, %rdx |
| movq %rsp, %rcx |
| call SYMBOL(artQuickResolutionTrampoline) // (called, receiver, Thread*, SP) |
| movq %rax, %r10 // Remember returned code pointer in R10. |
| movq (%rsp), %rdi // Load called method into RDI. |
| RESTORE_SAVE_REFS_AND_ARGS_FRAME |
| testq %r10, %r10 // If code pointer is null goto deliver pending exception. |
| jz 1f |
| jmp *%r10 // Tail call into method. |
| 1: |
| DELIVER_PENDING_EXCEPTION |
| END_FUNCTION art_quick_resolution_trampoline |
| |
| /* Generic JNI frame layout: |
| * |
| * #-------------------# |
| * | | |
| * | caller method... | |
| * #-------------------# <--- SP on entry |
| * |
| * | |
| * V |
| * |
| * #-------------------# |
| * | caller method... | |
| * #-------------------# |
| * | Return | |
| * | R15 | callee save |
| * | R14 | callee save |
| * | R13 | callee save |
| * | R12 | callee save |
| * | R9 | arg5 |
| * | R8 | arg4 |
| * | RSI/R6 | arg1 |
| * | RBP/R5 | callee save |
| * | RBX/R3 | callee save |
| * | RDX/R2 | arg2 |
| * | RCX/R1 | arg3 |
| * | XMM7 | float arg 8 |
| * | XMM6 | float arg 7 |
| * | XMM5 | float arg 6 |
| * | XMM4 | float arg 5 |
| * | XMM3 | float arg 4 |
| * | XMM2 | float arg 3 |
| * | XMM1 | float arg 2 |
| * | XMM0 | float arg 1 |
| * | RDI/Method* | <- sp |
| * #-------------------# |
| * | Scratch Alloca | 5K scratch space |
| * #---------#---------# |
| * | | sp* | |
| * | Tramp. #---------# |
| * | args | thread | |
| * | Tramp. #---------# |
| * | | method | |
| * #-------------------# <--- SP on artQuickGenericJniTrampoline |
| * |
| * | |
| * v artQuickGenericJniTrampoline |
| * |
| * #-------------------# |
| * | caller method... | |
| * #-------------------# |
| * | Return | |
| * | Callee-Save Data | |
| * #-------------------# |
| * | handle scope | |
| * #-------------------# |
| * | Method* | <--- (1) |
| * #-------------------# |
| * | local ref cookie | // 4B |
| * | handle scope size | // 4B TODO: roll into call stack alignment? |
| * #-------------------# |
| * | JNI Call Stack | |
| * #-------------------# <--- SP on native call |
| * | | |
| * | Stack for Regs | The trampoline assembly will pop these values |
| * | | into registers for native call |
| * #-------------------# |
| * | Native code ptr | |
| * #-------------------# |
| * | Free scratch | |
| * #-------------------# |
| * | Ptr to (1) | <--- RSP |
| * #-------------------# |
| */ |
| /* |
| * Called to do a generic JNI down-call |
| */ |
| DEFINE_FUNCTION art_quick_generic_jni_trampoline |
| SETUP_SAVE_REFS_AND_ARGS_FRAME_WITH_METHOD_IN_RDI |
| |
| movq %rsp, %rbp // save SP at (old) callee-save frame |
| CFI_DEF_CFA_REGISTER(rbp) |
| |
| // |
| // reserve a lot of space |
| // |
| // 4 local state ref |
| // 4 padding |
| // 4196 4k scratch space, enough for 2x 256 8-byte parameters (TODO: handle scope overhead?) |
| // 16 handle scope member fields ? |
| // + 112 14x 8-byte stack-2-register space |
| // ------ |
| // 4332 |
| // 16-byte aligned: 4336 |
| // Note: 14x8 = 7*16, so the stack stays aligned for the native call... |
| // Also means: the padding is somewhere in the middle |
| // |
| // |
| // New test: use 5K and release |
| // 5k = 5120 |
| subq LITERAL(5120), %rsp |
| // prepare for artQuickGenericJniTrampoline call |
| // (Thread*, SP) |
| // rdi rsi <= C calling convention |
| // gs:... rbp <= where they are |
| movq %gs:THREAD_SELF_OFFSET, %rdi |
| movq %rbp, %rsi |
| call SYMBOL(artQuickGenericJniTrampoline) // (Thread*, sp) |
| |
| // The C call will have registered the complete save-frame on success. |
| // The result of the call is: |
| // %rax: pointer to native code, 0 on error. |
| // %rdx: pointer to the bottom of the used area of the alloca, can restore stack till there. |
| |
| // Check for error = 0. |
| test %rax, %rax |
| jz .Lexception_in_native |
| |
| // Release part of the alloca. |
| movq %rdx, %rsp |
| |
| // pop from the register-passing alloca region |
| // what's the right layout? |
| popq %rdi |
| popq %rsi |
| popq %rdx |
| popq %rcx |
| popq %r8 |
| popq %r9 |
| // TODO: skip floating point if unused, some flag. |
| movq 0(%rsp), %xmm0 |
| movq 8(%rsp), %xmm1 |
| movq 16(%rsp), %xmm2 |
| movq 24(%rsp), %xmm3 |
| movq 32(%rsp), %xmm4 |
| movq 40(%rsp), %xmm5 |
| movq 48(%rsp), %xmm6 |
| movq 56(%rsp), %xmm7 |
| addq LITERAL(64), %rsp // floating-point done |
| |
| // native call |
| call *%rax |
| |
| // result sign extension is handled in C code |
| // prepare for artQuickGenericJniEndTrampoline call |
| // (Thread*, result, result_f) |
| // rdi rsi rdx <= C calling convention |
| // gs:... rax xmm0 <= where they are |
| movq %gs:THREAD_SELF_OFFSET, %rdi |
| movq %rax, %rsi |
| movq %xmm0, %rdx |
| call SYMBOL(artQuickGenericJniEndTrampoline) |
| |
| // Pending exceptions possible. |
| // TODO: use cmpq, needs direct encoding because of gas bug |
| movq %gs:THREAD_EXCEPTION_OFFSET, %rcx |
| test %rcx, %rcx |
| jnz .Lexception_in_native |
| |
| // Tear down the alloca. |
| movq %rbp, %rsp |
| CFI_DEF_CFA_REGISTER(rsp) |
| |
| // Tear down the callee-save frame. |
| // Load FPRs. |
| // movq %xmm0, 16(%rsp) // doesn't make sense!!! |
| movq 24(%rsp), %xmm1 // neither does this!!! |
| movq 32(%rsp), %xmm2 |
| movq 40(%rsp), %xmm3 |
| movq 48(%rsp), %xmm4 |
| movq 56(%rsp), %xmm5 |
| movq 64(%rsp), %xmm6 |
| movq 72(%rsp), %xmm7 |
| movq 80(%rsp), %xmm12 |
| movq 88(%rsp), %xmm13 |
| movq 96(%rsp), %xmm14 |
| movq 104(%rsp), %xmm15 |
| // was 80 bytes |
| addq LITERAL(80 + 4*8), %rsp |
| CFI_ADJUST_CFA_OFFSET(-80 - 4*8) |
| // Save callee and GPR args, mixed together to agree with core spills bitmap. |
| POP rcx // Arg. |
| POP rdx // Arg. |
| POP rbx // Callee save. |
| POP rbp // Callee save. |
| POP rsi // Arg. |
| POP r8 // Arg. |
| POP r9 // Arg. |
| POP r12 // Callee save. |
| POP r13 // Callee save. |
| POP r14 // Callee save. |
| POP r15 // Callee save. |
| // store into fpr, for when it's a fpr return... |
| movq %rax, %xmm0 |
| ret |
| .Lexception_in_native: |
| movq %gs:THREAD_TOP_QUICK_FRAME_OFFSET, %rsp |
| CFI_DEF_CFA_REGISTER(rsp) |
| // Do a call to push a new save-all frame required by the runtime. |
| call .Lexception_call |
| .Lexception_call: |
| DELIVER_PENDING_EXCEPTION |
| END_FUNCTION art_quick_generic_jni_trampoline |
| |
| /* |
| * Called to bridge from the quick to interpreter ABI. On entry the arguments match those |
| * of a quick call: |
| * RDI = method being called / to bridge to. |
| * RSI, RDX, RCX, R8, R9 are arguments to that method. |
| */ |
| DEFINE_FUNCTION art_quick_to_interpreter_bridge |
| SETUP_SAVE_REFS_AND_ARGS_FRAME // Set up frame and save arguments. |
| movq %gs:THREAD_SELF_OFFSET, %rsi // RSI := Thread::Current() |
| movq %rsp, %rdx // RDX := sp |
| call SYMBOL(artQuickToInterpreterBridge) // (method, Thread*, SP) |
| RESTORE_SAVE_REFS_AND_ARGS_FRAME // TODO: no need to restore arguments in this case. |
| movq %rax, %xmm0 // Place return value also into floating point return value. |
| RETURN_OR_DELIVER_PENDING_EXCEPTION // return or deliver exception |
| END_FUNCTION art_quick_to_interpreter_bridge |
| |
| /* |
| * Routine that intercepts method calls and returns. |
| */ |
| DEFINE_FUNCTION art_quick_instrumentation_entry |
| #if defined(__APPLE__) |
| int3 |
| int3 |
| #else |
| SETUP_SAVE_REFS_AND_ARGS_FRAME |
| |
| movq %rdi, %r12 // Preserve method pointer in a callee-save. |
| |
| movq %gs:THREAD_SELF_OFFSET, %rdx // Pass thread. |
| movq FRAME_SIZE_SAVE_REFS_AND_ARGS-8(%rsp), %rcx // Pass return PC. |
| |
| call SYMBOL(artInstrumentationMethodEntryFromCode) // (Method*, Object*, Thread*, LR) |
| |
| // %rax = result of call. |
| movq %r12, %rdi // Reload method pointer. |
| |
| leaq art_quick_instrumentation_exit(%rip), %r12 // Set up return through instrumentation |
| movq %r12, FRAME_SIZE_SAVE_REFS_AND_ARGS-8(%rsp) // exit. |
| |
| RESTORE_SAVE_REFS_AND_ARGS_FRAME |
| |
| jmp *%rax // Tail call to intended method. |
| #endif // __APPLE__ |
| END_FUNCTION art_quick_instrumentation_entry |
| |
| DEFINE_FUNCTION art_quick_instrumentation_exit |
| pushq LITERAL(0) // Push a fake return PC as there will be none on the stack. |
| |
| SETUP_SAVE_REFS_ONLY_FRAME |
| |
| // We need to save rax and xmm0. We could use a callee-save from SETUP_REF_ONLY, but then |
| // we would need to fully restore it. As there are a good number of callee-save registers, it |
| // seems easier to have an extra small stack area. But this should be revisited. |
| |
| movq %rsp, %rsi // Pass SP. |
| |
| PUSH rax // Save integer result. |
| subq LITERAL(8), %rsp // Save floating-point result. |
| CFI_ADJUST_CFA_OFFSET(8) |
| movq %xmm0, (%rsp) |
| |
| movq %gs:THREAD_SELF_OFFSET, %rdi // Pass Thread. |
| movq %rax, %rdx // Pass integer result. |
| movq %xmm0, %rcx // Pass floating-point result. |
| |
| call SYMBOL(artInstrumentationMethodExitFromCode) // (Thread*, SP, gpr_res, fpr_res) |
| |
| movq %rax, %rdi // Store return PC |
| movq %rdx, %rsi // Store second return PC in hidden arg. |
| |
| movq (%rsp), %xmm0 // Restore floating-point result. |
| addq LITERAL(8), %rsp |
| CFI_ADJUST_CFA_OFFSET(-8) |
| POP rax // Restore integer result. |
| |
| RESTORE_SAVE_REFS_ONLY_FRAME |
| |
| addq LITERAL(8), %rsp // Drop fake return pc. |
| |
| jmp *%rdi // Return. |
| END_FUNCTION art_quick_instrumentation_exit |
| |
| /* |
| * Instrumentation has requested that we deoptimize into the interpreter. The deoptimization |
| * will long jump to the upcall with a special exception of -1. |
| */ |
| DEFINE_FUNCTION art_quick_deoptimize |
| pushq %rsi // Entry point for a jump. Fake that we were called. |
| // Use hidden arg. |
| SETUP_SAVE_ALL_CALLEE_SAVES_FRAME |
| // Stack should be aligned now. |
| movq %gs:THREAD_SELF_OFFSET, %rdi // Pass Thread. |
| call SYMBOL(artDeoptimize) // artDeoptimize(Thread*) |
| UNREACHABLE |
| END_FUNCTION art_quick_deoptimize |
| |
| /* |
| * Compiled code has requested that we deoptimize into the interpreter. The deoptimization |
| * will long jump to the interpreter bridge. |
| */ |
| DEFINE_FUNCTION art_quick_deoptimize_from_compiled_code |
| SETUP_SAVE_EVERYTHING_FRAME |
| // Stack should be aligned now. |
| movq %gs:THREAD_SELF_OFFSET, %rdi // Pass Thread. |
| call SYMBOL(artDeoptimizeFromCompiledCode) // artDeoptimizeFromCompiledCode(Thread*) |
| UNREACHABLE |
| END_FUNCTION art_quick_deoptimize_from_compiled_code |
| |
| /* |
| * String's compareTo. |
| * |
| * On entry: |
| * rdi: this string object (known non-null) |
| * rsi: comp string object (known non-null) |
| */ |
| DEFINE_FUNCTION art_quick_string_compareto |
| movl MIRROR_STRING_COUNT_OFFSET(%edi), %r8d |
| movl MIRROR_STRING_COUNT_OFFSET(%esi), %r9d |
| /* Build pointers to the start of string data */ |
| leal MIRROR_STRING_VALUE_OFFSET(%edi), %edi |
| leal MIRROR_STRING_VALUE_OFFSET(%esi), %esi |
| #if (STRING_COMPRESSION_FEATURE) |
| /* Differ cases */ |
| shrl LITERAL(1), %r8d |
| jnc .Lstring_compareto_this_is_compressed |
| shrl LITERAL(1), %r9d |
| jnc .Lstring_compareto_that_is_compressed |
| jmp .Lstring_compareto_both_not_compressed |
| .Lstring_compareto_this_is_compressed: |
| shrl LITERAL(1), %r9d |
| jnc .Lstring_compareto_both_compressed |
| /* Comparison this (8-bit) and that (16-bit) */ |
| mov %r8d, %eax |
| subl %r9d, %eax |
| mov %r8d, %ecx |
| cmovg %r9d, %ecx |
| /* Going into loop to compare each character */ |
| jecxz .Lstring_compareto_keep_length1 // check loop counter (if 0 then stop) |
| .Lstring_compareto_loop_comparison_this_compressed: |
| movzbl (%edi), %r8d // move *(this_cur_char) byte to long |
| movzwl (%esi), %r9d // move *(that_cur_char) word to long |
| addl LITERAL(1), %edi // ++this_cur_char (8-bit) |
| addl LITERAL(2), %esi // ++that_cur_char (16-bit) |
| subl %r9d, %r8d |
| loope .Lstring_compareto_loop_comparison_this_compressed |
| cmovne %r8d, %eax // return eax = *(this_cur_char) - *(that_cur_char) |
| .Lstring_compareto_keep_length1: |
| ret |
| .Lstring_compareto_that_is_compressed: |
| movl %r8d, %eax |
| subl %r9d, %eax |
| mov %r8d, %ecx |
| cmovg %r9d, %ecx |
| /* Comparison this (8-bit) and that (16-bit) */ |
| jecxz .Lstring_compareto_keep_length2 // check loop counter (if 0, don't compare) |
| .Lstring_compareto_loop_comparison_that_compressed: |
| movzwl (%edi), %r8d // move *(this_cur_char) word to long |
| movzbl (%esi), %r9d // move *(that_cur_chat) byte to long |
| addl LITERAL(2), %edi // ++this_cur_char (16-bit) |
| addl LITERAL(1), %esi // ++that_cur_char (8-bit) |
| subl %r9d, %r8d |
| loope .Lstring_compareto_loop_comparison_that_compressed |
| cmovne %r8d, %eax // return eax = *(this_cur_char) - *(that_cur_char) |
| .Lstring_compareto_keep_length2: |
| ret |
| .Lstring_compareto_both_compressed: |
| /* Calculate min length and count diff */ |
| movl %r8d, %ecx |
| movl %r8d, %eax |
| subl %r9d, %eax |
| cmovg %r9d, %ecx |
| jecxz .Lstring_compareto_keep_length3 |
| repe cmpsb |
| je .Lstring_compareto_keep_length3 |
| movzbl -1(%edi), %eax // get last compared char from this string (8-bit) |
| movzbl -1(%esi), %ecx // get last compared char from comp string (8-bit) |
| jmp .Lstring_compareto_count_difference |
| #endif // STRING_COMPRESSION_FEATURE |
| .Lstring_compareto_both_not_compressed: |
| /* Calculate min length and count diff */ |
| movl %r8d, %ecx |
| movl %r8d, %eax |
| subl %r9d, %eax |
| cmovg %r9d, %ecx |
| /* |
| * At this point we have: |
| * eax: value to return if first part of strings are equal |
| * ecx: minimum among the lengths of the two strings |
| * esi: pointer to comp string data |
| * edi: pointer to this string data |
| */ |
| jecxz .Lstring_compareto_keep_length3 |
| repe cmpsw // find nonmatching chars in [%esi] and [%edi], up to length %ecx |
| je .Lstring_compareto_keep_length3 |
| movzwl -2(%edi), %eax // get last compared char from this string (16-bit) |
| movzwl -2(%esi), %ecx // get last compared char from comp string (16-bit) |
| .Lstring_compareto_count_difference: |
| subl %ecx, %eax // return the difference |
| .Lstring_compareto_keep_length3: |
| ret |
| END_FUNCTION art_quick_string_compareto |
| |
| UNIMPLEMENTED art_quick_memcmp16 |
| |
| DEFINE_FUNCTION art_quick_instance_of |
| SETUP_FP_CALLEE_SAVE_FRAME |
| subq LITERAL(8), %rsp // Alignment padding. |
| CFI_ADJUST_CFA_OFFSET(8) |
| call SYMBOL(artInstanceOfFromCode) // (mirror::Object*, mirror::Class*) |
| addq LITERAL(8), %rsp |
| CFI_ADJUST_CFA_OFFSET(-8) |
| RESTORE_FP_CALLEE_SAVE_FRAME |
| ret |
| END_FUNCTION art_quick_instance_of |
| |
| |
| // Return from a nested signal: |
| // Entry: |
| // rdi: address of jmp_buf in TLS |
| |
| DEFINE_FUNCTION art_nested_signal_return |
| // first arg to longjmp is already in correct register |
| movq LITERAL(1), %rsi // second arg to longjmp (1) |
| call PLT_SYMBOL(longjmp) |
| UNREACHABLE |
| END_FUNCTION art_nested_signal_return |
| |
| // Create a function `name` calling the ReadBarrier::Mark routine, |
| // getting its argument and returning its result through register |
| // `reg`, saving and restoring all caller-save registers. |
| // |
| // The generated function follows a non-standard runtime calling |
| // convention: |
| // - register `reg` (which may be different from RDI) is used to pass |
| // the (sole) argument of this function; |
| // - register `reg` (which may be different from RAX) is used to return |
| // the result of this function (instead of RAX); |
| // - if `reg` is different from `rdi`, RDI is treated like a normal |
| // (non-argument) caller-save register; |
| // - if `reg` is different from `rax`, RAX is treated like a normal |
| // (non-result) caller-save register; |
| // - everything else is the same as in the standard runtime calling |
| // convention (e.g. standard callee-save registers are preserved). |
| MACRO2(READ_BARRIER_MARK_REG, name, reg) |
| DEFINE_FUNCTION VAR(name) |
| // Null check so that we can load the lock word. |
| testq REG_VAR(reg), REG_VAR(reg) |
| jz .Lret_rb_\name |
| .Lnot_null_\name: |
| // Check the mark bit, if it is 1 return. |
| testl LITERAL(LOCK_WORD_MARK_BIT_MASK_SHIFTED), MIRROR_OBJECT_LOCK_WORD_OFFSET(REG_VAR(reg)) |
| jz .Lslow_rb_\name |
| ret |
| .Lslow_rb_\name: |
| PUSH rax |
| movl MIRROR_OBJECT_LOCK_WORD_OFFSET(REG_VAR(reg)), %eax |
| addl LITERAL(LOCK_WORD_STATE_FORWARDING_ADDRESS_OVERFLOW), %eax |
| // Jump if the addl caused eax to unsigned overflow. The only case where it overflows is the |
| // forwarding address one. |
| // Taken ~25% of the time. |
| jnae .Lret_forwarding_address\name |
| |
| // Save all potentially live caller-save core registers. |
| movq 0(%rsp), %rax |
| PUSH rcx |
| PUSH rdx |
| PUSH rsi |
| PUSH rdi |
| PUSH r8 |
| PUSH r9 |
| PUSH r10 |
| PUSH r11 |
| // Create space for caller-save floating-point registers. |
| subq MACRO_LITERAL(12 * 8), %rsp |
| CFI_ADJUST_CFA_OFFSET(12 * 8) |
| // Save all potentially live caller-save floating-point registers. |
| movq %xmm0, 0(%rsp) |
| movq %xmm1, 8(%rsp) |
| movq %xmm2, 16(%rsp) |
| movq %xmm3, 24(%rsp) |
| movq %xmm4, 32(%rsp) |
| movq %xmm5, 40(%rsp) |
| movq %xmm6, 48(%rsp) |
| movq %xmm7, 56(%rsp) |
| movq %xmm8, 64(%rsp) |
| movq %xmm9, 72(%rsp) |
| movq %xmm10, 80(%rsp) |
| movq %xmm11, 88(%rsp) |
| SETUP_FP_CALLEE_SAVE_FRAME |
| |
| .ifnc RAW_VAR(reg), rdi |
| movq REG_VAR(reg), %rdi // Pass arg1 - obj from `reg`. |
| .endif |
| call SYMBOL(artReadBarrierMark) // artReadBarrierMark(obj) |
| .ifnc RAW_VAR(reg), rax |
| movq %rax, REG_VAR(reg) // Return result into `reg`. |
| .endif |
| |
| RESTORE_FP_CALLEE_SAVE_FRAME |
| // Restore floating-point registers. |
| movq 0(%rsp), %xmm0 |
| movq 8(%rsp), %xmm1 |
| movq 16(%rsp), %xmm2 |
| movq 24(%rsp), %xmm3 |
| movq 32(%rsp), %xmm4 |
| movq 40(%rsp), %xmm5 |
| movq 48(%rsp), %xmm6 |
| movq 56(%rsp), %xmm7 |
| movq 64(%rsp), %xmm8 |
| movq 72(%rsp), %xmm9 |
| movq 80(%rsp), %xmm10 |
| movq 88(%rsp), %xmm11 |
| // Remove floating-point registers. |
| addq MACRO_LITERAL(12 * 8), %rsp |
| CFI_ADJUST_CFA_OFFSET(-(12 * 8)) |
| // Restore core regs, except `reg`, as it is used to return the |
| // result of this function (simply remove it from the stack instead). |
| POP_REG_NE r11, RAW_VAR(reg) |
| POP_REG_NE r10, RAW_VAR(reg) |
| POP_REG_NE r9, RAW_VAR(reg) |
| POP_REG_NE r8, RAW_VAR(reg) |
| POP_REG_NE rdi, RAW_VAR(reg) |
| POP_REG_NE rsi, RAW_VAR(reg) |
| POP_REG_NE rdx, RAW_VAR(reg) |
| POP_REG_NE rcx, RAW_VAR(reg) |
| POP_REG_NE rax, RAW_VAR(reg) |
| .Lret_rb_\name: |
| ret |
| .Lret_forwarding_address\name: |
| // The overflow cleared the top bits. |
| sall LITERAL(LOCK_WORD_STATE_FORWARDING_ADDRESS_SHIFT), %eax |
| movq %rax, REG_VAR(reg) |
| POP_REG_NE rax, RAW_VAR(reg) |
| ret |
| END_FUNCTION VAR(name) |
| END_MACRO |
| |
| READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg00, rax |
| READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg01, rcx |
| READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg02, rdx |
| READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg03, rbx |
| // Note: There is no art_quick_read_barrier_mark_reg04, as register 4 (RSP) |
| // cannot be used to pass arguments. |
| READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg05, rbp |
| READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg06, rsi |
| READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg07, rdi |
| READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg08, r8 |
| READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg09, r9 |
| READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg10, r10 |
| READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg11, r11 |
| READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg12, r12 |
| READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg13, r13 |
| READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg14, r14 |
| READ_BARRIER_MARK_REG art_quick_read_barrier_mark_reg15, r15 |
| |
| DEFINE_FUNCTION art_quick_read_barrier_slow |
| SETUP_FP_CALLEE_SAVE_FRAME |
| subq LITERAL(8), %rsp // Alignment padding. |
| CFI_ADJUST_CFA_OFFSET(8) |
| call SYMBOL(artReadBarrierSlow) // artReadBarrierSlow(ref, obj, offset) |
| addq LITERAL(8), %rsp |
| CFI_ADJUST_CFA_OFFSET(-8) |
| RESTORE_FP_CALLEE_SAVE_FRAME |
| ret |
| END_FUNCTION art_quick_read_barrier_slow |
| |
| DEFINE_FUNCTION art_quick_read_barrier_for_root_slow |
| SETUP_FP_CALLEE_SAVE_FRAME |
| subq LITERAL(8), %rsp // Alignment padding. |
| CFI_ADJUST_CFA_OFFSET(8) |
| call SYMBOL(artReadBarrierForRootSlow) // artReadBarrierForRootSlow(root) |
| addq LITERAL(8), %rsp |
| CFI_ADJUST_CFA_OFFSET(-8) |
| RESTORE_FP_CALLEE_SAVE_FRAME |
| ret |
| END_FUNCTION art_quick_read_barrier_for_root_slow |
| |
| /* |
| * On stack replacement stub. |
| * On entry: |
| * [sp] = return address |
| * rdi = stack to copy |
| * rsi = size of stack |
| * rdx = pc to call |
| * rcx = JValue* result |
| * r8 = shorty |
| * r9 = thread |
| * |
| * Note that the native C ABI already aligned the stack to 16-byte. |
| */ |
| DEFINE_FUNCTION art_quick_osr_stub |
| // Save the non-volatiles. |
| PUSH rbp // Save rbp. |
| PUSH rcx // Save rcx/result*. |
| PUSH r8 // Save r8/shorty*. |
| |
| // Save callee saves. |
| PUSH rbx |
| PUSH r12 |
| PUSH r13 |
| PUSH r14 |
| PUSH r15 |
| |
| pushq LITERAL(0) // Push null for ArtMethod*. |
| movl %esi, %ecx // rcx := size of stack |
| movq %rdi, %rsi // rsi := stack to copy |
| call .Losr_entry |
| |
| // Restore stack and callee-saves. |
| addq LITERAL(8), %rsp |
| POP r15 |
| POP r14 |
| POP r13 |
| POP r12 |
| POP rbx |
| POP r8 |
| POP rcx |
| POP rbp |
| cmpb LITERAL(68), (%r8) // Test if result type char == 'D'. |
| je .Losr_return_double_quick |
| cmpb LITERAL(70), (%r8) // Test if result type char == 'F'. |
| je .Losr_return_float_quick |
| movq %rax, (%rcx) // Store the result assuming its a long, int or Object* |
| ret |
| .Losr_return_double_quick: |
| movsd %xmm0, (%rcx) // Store the double floating point result. |
| ret |
| .Losr_return_float_quick: |
| movss %xmm0, (%rcx) // Store the floating point result. |
| ret |
| .Losr_entry: |
| subl LITERAL(8), %ecx // Given stack size contains pushed frame pointer, substract it. |
| subq %rcx, %rsp |
| movq %rsp, %rdi // rdi := beginning of stack |
| rep movsb // while (rcx--) { *rdi++ = *rsi++ } |
| jmp *%rdx |
| END_FUNCTION art_quick_osr_stub |
| |
| DEFINE_FUNCTION art_quick_invoke_polymorphic |
| SETUP_SAVE_REFS_AND_ARGS_FRAME // save callee saves |
| movq %gs:THREAD_SELF_OFFSET, %rdx // pass Thread |
| movq %rsp, %rcx // pass SP |
| subq LITERAL(16), %rsp // make space for JValue result |
| CFI_ADJUST_CFA_OFFSET(16) |
| movq LITERAL(0), (%rsp) // initialize result |
| movq %rsp, %rdi // store pointer to JValue result |
| call SYMBOL(artInvokePolymorphic) // artInvokePolymorphic(result, receiver, Thread*, SP) |
| // save the code pointer |
| subq LITERAL('A'), %rax // Convert type descriptor character value to a zero based index. |
| cmpb LITERAL('Z' - 'A'), %al // Eliminate out of bounds options |
| ja .Lcleanup_and_return |
| movzbq %al, %rax |
| leaq .Lhandler_table(%rip), %rcx // Get the address of the handler table |
| movslq (%rcx, %rax, 4), %rax // Lookup handler offset relative to table |
| addq %rcx, %rax // Add table address to yield handler address. |
| jmpq *%rax // Jump to handler. |
| |
| .align 4 |
| .Lhandler_table: // Table of type descriptor to handlers. |
| MACRO1(HANDLER_TABLE_OFFSET, handle_label) |
| // NB some tools require 32-bits for relocations. Shouldn't need adjusting. |
| .long RAW_VAR(handle_label) - .Lhandler_table |
| END_MACRO |
| HANDLER_TABLE_OFFSET(.Lcleanup_and_return) // A |
| HANDLER_TABLE_OFFSET(.Lstore_long_result) // B (byte) |
| HANDLER_TABLE_OFFSET(.Lstore_char_result) // C (char) |
| HANDLER_TABLE_OFFSET(.Lstore_double_result) // D (double) |
| HANDLER_TABLE_OFFSET(.Lcleanup_and_return) // E |
| HANDLER_TABLE_OFFSET(.Lstore_float_result) // F (float) |
| HANDLER_TABLE_OFFSET(.Lcleanup_and_return) // G |
| HANDLER_TABLE_OFFSET(.Lcleanup_and_return) // H |
| HANDLER_TABLE_OFFSET(.Lstore_long_result) // I (int) |
| HANDLER_TABLE_OFFSET(.Lstore_long_result) // J (long) |
| HANDLER_TABLE_OFFSET(.Lcleanup_and_return) // K |
| HANDLER_TABLE_OFFSET(.Lstore_long_result) // L (object - references are compressed and only 32-bits) |
| HANDLER_TABLE_OFFSET(.Lcleanup_and_return) // M |
| HANDLER_TABLE_OFFSET(.Lcleanup_and_return) // N |
| HANDLER_TABLE_OFFSET(.Lcleanup_and_return) // O |
| HANDLER_TABLE_OFFSET(.Lcleanup_and_return) // P |
| HANDLER_TABLE_OFFSET(.Lcleanup_and_return) // Q |
| HANDLER_TABLE_OFFSET(.Lcleanup_and_return) // R |
| HANDLER_TABLE_OFFSET(.Lstore_long_result) // S (short) |
| HANDLER_TABLE_OFFSET(.Lcleanup_and_return) // T |
| HANDLER_TABLE_OFFSET(.Lcleanup_and_return) // U |
| HANDLER_TABLE_OFFSET(.Lcleanup_and_return) // V (void) |
| HANDLER_TABLE_OFFSET(.Lcleanup_and_return) // W |
| HANDLER_TABLE_OFFSET(.Lcleanup_and_return) // X |
| HANDLER_TABLE_OFFSET(.Lcleanup_and_return) // Y |
| HANDLER_TABLE_OFFSET(.Lstore_boolean_result) // Z (boolean) |
| |
| .Lstore_boolean_result: |
| movzbq (%rsp), %rax // Copy boolean result to the accumulator |
| jmp .Lcleanup_and_return |
| .Lstore_char_result: |
| movzwq (%rsp), %rax // Copy char result to the accumulator |
| jmp .Lcleanup_and_return |
| .Lstore_float_result: |
| movd (%rsp), %xmm0 // Copy float result to the context restored by |
| movd %xmm0, 32(%rsp) // RESTORE_SAVE_REFS_AND_ARGS_FRAME. |
| jmp .Lcleanup_and_return |
| .Lstore_double_result: |
| movsd (%rsp), %xmm0 // Copy double result to the context restored by |
| movsd %xmm0, 32(%rsp) // RESTORE_SAVE_REFS_AND_ARGS_FRAME. |
| jmp .Lcleanup_and_return |
| .Lstore_long_result: |
| movq (%rsp), %rax // Copy long result to the accumulator. |
| // Fall-through |
| .Lcleanup_and_return: |
| addq LITERAL(16), %rsp // Pop space for JValue result. |
| CFI_ADJUST_CFA_OFFSET(16) |
| RESTORE_SAVE_REFS_AND_ARGS_FRAME |
| RETURN_OR_DELIVER_PENDING_EXCEPTION |
| END_FUNCTION art_quick_invoke_polymorphic |