blob: d2378d54a2af4770961da983180a21955c41065f [file] [log] [blame]
Alexandre Rames5319def2014-10-23 10:03:10 +01001/*
2 * Copyright (C) 2014 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17#include "code_generator_arm64.h"
18
Serban Constantinescu579885a2015-02-22 20:51:33 +000019#include "arch/arm64/instruction_set_features_arm64.h"
Mathieu Chartiere401d142015-04-22 13:56:20 -070020#include "art_method.h"
Zheng Xuc6667102015-05-15 16:08:45 +080021#include "code_generator_utils.h"
Vladimir Marko58155012015-08-19 12:49:41 +000022#include "compiled_method.h"
Alexandre Rames5319def2014-10-23 10:03:10 +010023#include "entrypoints/quick/quick_entrypoints.h"
Andreas Gampe1cc7dba2014-12-17 18:43:01 -080024#include "entrypoints/quick/quick_entrypoints_enum.h"
Alexandre Rames5319def2014-10-23 10:03:10 +010025#include "gc/accounting/card_table.h"
Andreas Gampe878d58c2015-01-15 23:24:00 -080026#include "intrinsics.h"
27#include "intrinsics_arm64.h"
Alexandre Rames5319def2014-10-23 10:03:10 +010028#include "mirror/array-inl.h"
Mathieu Chartiere401d142015-04-22 13:56:20 -070029#include "mirror/class-inl.h"
Calin Juravlecd6dffe2015-01-08 17:35:35 +000030#include "offsets.h"
Alexandre Rames5319def2014-10-23 10:03:10 +010031#include "thread.h"
32#include "utils/arm64/assembler_arm64.h"
33#include "utils/assembler.h"
34#include "utils/stack_checks.h"
35
36
37using namespace vixl; // NOLINT(build/namespaces)
38
39#ifdef __
40#error "ARM64 Codegen VIXL macro-assembler macro already defined."
41#endif
42
Alexandre Rames5319def2014-10-23 10:03:10 +010043namespace art {
44
45namespace arm64 {
46
Andreas Gampe878d58c2015-01-15 23:24:00 -080047using helpers::CPURegisterFrom;
48using helpers::DRegisterFrom;
49using helpers::FPRegisterFrom;
50using helpers::HeapOperand;
51using helpers::HeapOperandFrom;
52using helpers::InputCPURegisterAt;
53using helpers::InputFPRegisterAt;
54using helpers::InputRegisterAt;
55using helpers::InputOperandAt;
56using helpers::Int64ConstantFrom;
Andreas Gampe878d58c2015-01-15 23:24:00 -080057using helpers::LocationFrom;
58using helpers::OperandFromMemOperand;
59using helpers::OutputCPURegister;
60using helpers::OutputFPRegister;
61using helpers::OutputRegister;
62using helpers::RegisterFrom;
63using helpers::StackOperandFrom;
64using helpers::VIXLRegCodeFromART;
65using helpers::WRegisterFrom;
66using helpers::XRegisterFrom;
Serban Constantinescu2d35d9d2015-02-22 22:08:01 +000067using helpers::ARM64EncodableConstantOrRegister;
Zheng Xuda403092015-04-24 17:35:39 +080068using helpers::ArtVixlRegCodeCoherentForRegSet;
Andreas Gampe878d58c2015-01-15 23:24:00 -080069
Alexandre Rames5319def2014-10-23 10:03:10 +010070static constexpr int kCurrentMethodStackOffset = 0;
71
Alexandre Rames5319def2014-10-23 10:03:10 +010072inline Condition ARM64Condition(IfCondition cond) {
73 switch (cond) {
74 case kCondEQ: return eq;
75 case kCondNE: return ne;
76 case kCondLT: return lt;
77 case kCondLE: return le;
78 case kCondGT: return gt;
79 case kCondGE: return ge;
Aart Bike9f37602015-10-09 11:15:55 -070080 case kCondB: return lo;
81 case kCondBE: return ls;
82 case kCondA: return hi;
83 case kCondAE: return hs;
Alexandre Rames5319def2014-10-23 10:03:10 +010084 }
Roland Levillain7f63c522015-07-13 15:54:55 +000085 LOG(FATAL) << "Unreachable";
86 UNREACHABLE();
Alexandre Rames5319def2014-10-23 10:03:10 +010087}
88
Alexandre Ramesa89086e2014-11-07 17:13:25 +000089Location ARM64ReturnLocation(Primitive::Type return_type) {
Alexandre Ramesa89086e2014-11-07 17:13:25 +000090 // Note that in practice, `LocationFrom(x0)` and `LocationFrom(w0)` create the
91 // same Location object, and so do `LocationFrom(d0)` and `LocationFrom(s0)`,
92 // but we use the exact registers for clarity.
93 if (return_type == Primitive::kPrimFloat) {
94 return LocationFrom(s0);
95 } else if (return_type == Primitive::kPrimDouble) {
96 return LocationFrom(d0);
97 } else if (return_type == Primitive::kPrimLong) {
98 return LocationFrom(x0);
Nicolas Geoffray925e5622015-06-03 12:23:32 +010099 } else if (return_type == Primitive::kPrimVoid) {
100 return Location::NoLocation();
Alexandre Ramesa89086e2014-11-07 17:13:25 +0000101 } else {
102 return LocationFrom(w0);
103 }
104}
105
Alexandre Rames5319def2014-10-23 10:03:10 +0100106Location InvokeRuntimeCallingConvention::GetReturnLocation(Primitive::Type return_type) {
Alexandre Ramesa89086e2014-11-07 17:13:25 +0000107 return ARM64ReturnLocation(return_type);
Alexandre Rames5319def2014-10-23 10:03:10 +0100108}
109
Alexandre Rames67555f72014-11-18 10:55:16 +0000110#define __ down_cast<CodeGeneratorARM64*>(codegen)->GetVIXLAssembler()->
111#define QUICK_ENTRY_POINT(x) QUICK_ENTRYPOINT_OFFSET(kArm64WordSize, x).Int32Value()
Alexandre Rames5319def2014-10-23 10:03:10 +0100112
Zheng Xuda403092015-04-24 17:35:39 +0800113// Calculate memory accessing operand for save/restore live registers.
114static void SaveRestoreLiveRegistersHelper(CodeGenerator* codegen,
115 RegisterSet* register_set,
116 int64_t spill_offset,
117 bool is_save) {
118 DCHECK(ArtVixlRegCodeCoherentForRegSet(register_set->GetCoreRegisters(),
119 codegen->GetNumberOfCoreRegisters(),
120 register_set->GetFloatingPointRegisters(),
121 codegen->GetNumberOfFloatingPointRegisters()));
122
123 CPURegList core_list = CPURegList(CPURegister::kRegister, kXRegSize,
124 register_set->GetCoreRegisters() & (~callee_saved_core_registers.list()));
Nicolas Geoffray75d5b9b2015-10-05 07:40:35 +0000125 CPURegList fp_list = CPURegList(CPURegister::kFPRegister, kDRegSize,
126 register_set->GetFloatingPointRegisters() & (~callee_saved_fp_registers.list()));
Zheng Xuda403092015-04-24 17:35:39 +0800127
128 MacroAssembler* masm = down_cast<CodeGeneratorARM64*>(codegen)->GetVIXLAssembler();
129 UseScratchRegisterScope temps(masm);
130
131 Register base = masm->StackPointer();
132 int64_t core_spill_size = core_list.TotalSizeInBytes();
133 int64_t fp_spill_size = fp_list.TotalSizeInBytes();
134 int64_t reg_size = kXRegSizeInBytes;
135 int64_t max_ls_pair_offset = spill_offset + core_spill_size + fp_spill_size - 2 * reg_size;
136 uint32_t ls_access_size = WhichPowerOf2(reg_size);
137 if (((core_list.Count() > 1) || (fp_list.Count() > 1)) &&
138 !masm->IsImmLSPair(max_ls_pair_offset, ls_access_size)) {
139 // If the offset does not fit in the instruction's immediate field, use an alternate register
140 // to compute the base address(float point registers spill base address).
141 Register new_base = temps.AcquireSameSizeAs(base);
142 __ Add(new_base, base, Operand(spill_offset + core_spill_size));
143 base = new_base;
144 spill_offset = -core_spill_size;
145 int64_t new_max_ls_pair_offset = fp_spill_size - 2 * reg_size;
146 DCHECK(masm->IsImmLSPair(spill_offset, ls_access_size));
147 DCHECK(masm->IsImmLSPair(new_max_ls_pair_offset, ls_access_size));
148 }
149
150 if (is_save) {
151 __ StoreCPURegList(core_list, MemOperand(base, spill_offset));
152 __ StoreCPURegList(fp_list, MemOperand(base, spill_offset + core_spill_size));
153 } else {
154 __ LoadCPURegList(core_list, MemOperand(base, spill_offset));
155 __ LoadCPURegList(fp_list, MemOperand(base, spill_offset + core_spill_size));
156 }
157}
158
159void SlowPathCodeARM64::SaveLiveRegisters(CodeGenerator* codegen, LocationSummary* locations) {
160 RegisterSet* register_set = locations->GetLiveRegisters();
161 size_t stack_offset = codegen->GetFirstRegisterSlotInSlowPath();
162 for (size_t i = 0, e = codegen->GetNumberOfCoreRegisters(); i < e; ++i) {
163 if (!codegen->IsCoreCalleeSaveRegister(i) && register_set->ContainsCoreRegister(i)) {
164 // If the register holds an object, update the stack mask.
165 if (locations->RegisterContainsObject(i)) {
166 locations->SetStackBit(stack_offset / kVRegSize);
167 }
168 DCHECK_LT(stack_offset, codegen->GetFrameSize() - codegen->FrameEntrySpillSize());
169 DCHECK_LT(i, kMaximumNumberOfExpectedRegisters);
170 saved_core_stack_offsets_[i] = stack_offset;
171 stack_offset += kXRegSizeInBytes;
172 }
173 }
174
175 for (size_t i = 0, e = codegen->GetNumberOfFloatingPointRegisters(); i < e; ++i) {
176 if (!codegen->IsFloatingPointCalleeSaveRegister(i) &&
177 register_set->ContainsFloatingPointRegister(i)) {
178 DCHECK_LT(stack_offset, codegen->GetFrameSize() - codegen->FrameEntrySpillSize());
179 DCHECK_LT(i, kMaximumNumberOfExpectedRegisters);
180 saved_fpu_stack_offsets_[i] = stack_offset;
181 stack_offset += kDRegSizeInBytes;
182 }
183 }
184
185 SaveRestoreLiveRegistersHelper(codegen, register_set,
186 codegen->GetFirstRegisterSlotInSlowPath(), true /* is_save */);
187}
188
189void SlowPathCodeARM64::RestoreLiveRegisters(CodeGenerator* codegen, LocationSummary* locations) {
190 RegisterSet* register_set = locations->GetLiveRegisters();
191 SaveRestoreLiveRegistersHelper(codegen, register_set,
192 codegen->GetFirstRegisterSlotInSlowPath(), false /* is_save */);
193}
194
Alexandre Rames5319def2014-10-23 10:03:10 +0100195class BoundsCheckSlowPathARM64 : public SlowPathCodeARM64 {
196 public:
Serban Constantinescu5a6cc492015-08-13 15:20:25 +0100197 explicit BoundsCheckSlowPathARM64(HBoundsCheck* instruction) : instruction_(instruction) {}
Alexandre Rames5319def2014-10-23 10:03:10 +0100198
Alexandre Rames67555f72014-11-18 10:55:16 +0000199 void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
Serban Constantinescu5a6cc492015-08-13 15:20:25 +0100200 LocationSummary* locations = instruction_->GetLocations();
Alexandre Rames3e69f162014-12-10 10:36:50 +0000201 CodeGeneratorARM64* arm64_codegen = down_cast<CodeGeneratorARM64*>(codegen);
Serban Constantinescu5a6cc492015-08-13 15:20:25 +0100202
Alexandre Rames5319def2014-10-23 10:03:10 +0100203 __ Bind(GetEntryLabel());
David Brazdil77a48ae2015-09-15 12:34:04 +0000204 if (instruction_->CanThrowIntoCatchBlock()) {
205 // Live registers will be restored in the catch block if caught.
206 SaveLiveRegisters(codegen, instruction_->GetLocations());
207 }
Alexandre Rames3e69f162014-12-10 10:36:50 +0000208 // We're moving two locations to locations that could overlap, so we need a parallel
209 // move resolver.
210 InvokeRuntimeCallingConvention calling_convention;
211 codegen->EmitParallelMoves(
Serban Constantinescu5a6cc492015-08-13 15:20:25 +0100212 locations->InAt(0), LocationFrom(calling_convention.GetRegisterAt(0)), Primitive::kPrimInt,
213 locations->InAt(1), LocationFrom(calling_convention.GetRegisterAt(1)), Primitive::kPrimInt);
Alexandre Rames3e69f162014-12-10 10:36:50 +0000214 arm64_codegen->InvokeRuntime(
Nicolas Geoffrayeeefa122015-03-13 18:52:59 +0000215 QUICK_ENTRY_POINT(pThrowArrayBounds), instruction_, instruction_->GetDexPc(), this);
Andreas Gampe1cc7dba2014-12-17 18:43:01 -0800216 CheckEntrypointTypes<kQuickThrowArrayBounds, void, int32_t, int32_t>();
Alexandre Rames5319def2014-10-23 10:03:10 +0100217 }
218
Alexandre Rames8158f282015-08-07 10:26:17 +0100219 bool IsFatal() const OVERRIDE { return true; }
220
Alexandre Rames9931f312015-06-19 14:47:01 +0100221 const char* GetDescription() const OVERRIDE { return "BoundsCheckSlowPathARM64"; }
222
Alexandre Rames5319def2014-10-23 10:03:10 +0100223 private:
Alexandre Rames3e69f162014-12-10 10:36:50 +0000224 HBoundsCheck* const instruction_;
Alexandre Rames3e69f162014-12-10 10:36:50 +0000225
Alexandre Rames5319def2014-10-23 10:03:10 +0100226 DISALLOW_COPY_AND_ASSIGN(BoundsCheckSlowPathARM64);
227};
228
Alexandre Rames67555f72014-11-18 10:55:16 +0000229class DivZeroCheckSlowPathARM64 : public SlowPathCodeARM64 {
230 public:
231 explicit DivZeroCheckSlowPathARM64(HDivZeroCheck* instruction) : instruction_(instruction) {}
232
233 void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
234 CodeGeneratorARM64* arm64_codegen = down_cast<CodeGeneratorARM64*>(codegen);
235 __ Bind(GetEntryLabel());
David Brazdil77a48ae2015-09-15 12:34:04 +0000236 if (instruction_->CanThrowIntoCatchBlock()) {
237 // Live registers will be restored in the catch block if caught.
238 SaveLiveRegisters(codegen, instruction_->GetLocations());
239 }
Alexandre Rames67555f72014-11-18 10:55:16 +0000240 arm64_codegen->InvokeRuntime(
Nicolas Geoffrayeeefa122015-03-13 18:52:59 +0000241 QUICK_ENTRY_POINT(pThrowDivZero), instruction_, instruction_->GetDexPc(), this);
Andreas Gampe1cc7dba2014-12-17 18:43:01 -0800242 CheckEntrypointTypes<kQuickThrowDivZero, void, void>();
Alexandre Rames67555f72014-11-18 10:55:16 +0000243 }
244
Alexandre Rames8158f282015-08-07 10:26:17 +0100245 bool IsFatal() const OVERRIDE { return true; }
246
Alexandre Rames9931f312015-06-19 14:47:01 +0100247 const char* GetDescription() const OVERRIDE { return "DivZeroCheckSlowPathARM64"; }
248
Alexandre Rames67555f72014-11-18 10:55:16 +0000249 private:
250 HDivZeroCheck* const instruction_;
251 DISALLOW_COPY_AND_ASSIGN(DivZeroCheckSlowPathARM64);
252};
253
254class LoadClassSlowPathARM64 : public SlowPathCodeARM64 {
255 public:
256 LoadClassSlowPathARM64(HLoadClass* cls,
257 HInstruction* at,
258 uint32_t dex_pc,
259 bool do_clinit)
260 : cls_(cls), at_(at), dex_pc_(dex_pc), do_clinit_(do_clinit) {
261 DCHECK(at->IsLoadClass() || at->IsClinitCheck());
262 }
263
264 void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
265 LocationSummary* locations = at_->GetLocations();
266 CodeGeneratorARM64* arm64_codegen = down_cast<CodeGeneratorARM64*>(codegen);
267
268 __ Bind(GetEntryLabel());
Nicolas Geoffraya8ac9132015-03-13 16:36:36 +0000269 SaveLiveRegisters(codegen, locations);
Alexandre Rames67555f72014-11-18 10:55:16 +0000270
271 InvokeRuntimeCallingConvention calling_convention;
272 __ Mov(calling_convention.GetRegisterAt(0).W(), cls_->GetTypeIndex());
Alexandre Rames67555f72014-11-18 10:55:16 +0000273 int32_t entry_point_offset = do_clinit_ ? QUICK_ENTRY_POINT(pInitializeStaticStorage)
274 : QUICK_ENTRY_POINT(pInitializeType);
Nicolas Geoffrayeeefa122015-03-13 18:52:59 +0000275 arm64_codegen->InvokeRuntime(entry_point_offset, at_, dex_pc_, this);
Andreas Gampe1cc7dba2014-12-17 18:43:01 -0800276 if (do_clinit_) {
Vladimir Marko5ea536a2015-04-20 20:11:30 +0100277 CheckEntrypointTypes<kQuickInitializeStaticStorage, void*, uint32_t>();
Andreas Gampe1cc7dba2014-12-17 18:43:01 -0800278 } else {
Vladimir Marko5ea536a2015-04-20 20:11:30 +0100279 CheckEntrypointTypes<kQuickInitializeType, void*, uint32_t>();
Andreas Gampe1cc7dba2014-12-17 18:43:01 -0800280 }
Alexandre Rames67555f72014-11-18 10:55:16 +0000281
282 // Move the class to the desired location.
283 Location out = locations->Out();
284 if (out.IsValid()) {
285 DCHECK(out.IsRegister() && !locations->GetLiveRegisters()->ContainsCoreRegister(out.reg()));
286 Primitive::Type type = at_->GetType();
Alexandre Rames3e69f162014-12-10 10:36:50 +0000287 arm64_codegen->MoveLocation(out, calling_convention.GetReturnLocation(type), type);
Alexandre Rames67555f72014-11-18 10:55:16 +0000288 }
289
Nicolas Geoffraya8ac9132015-03-13 16:36:36 +0000290 RestoreLiveRegisters(codegen, locations);
Alexandre Rames67555f72014-11-18 10:55:16 +0000291 __ B(GetExitLabel());
292 }
293
Alexandre Rames9931f312015-06-19 14:47:01 +0100294 const char* GetDescription() const OVERRIDE { return "LoadClassSlowPathARM64"; }
295
Alexandre Rames67555f72014-11-18 10:55:16 +0000296 private:
297 // The class this slow path will load.
298 HLoadClass* const cls_;
299
300 // The instruction where this slow path is happening.
301 // (Might be the load class or an initialization check).
302 HInstruction* const at_;
303
304 // The dex PC of `at_`.
305 const uint32_t dex_pc_;
306
307 // Whether to initialize the class.
308 const bool do_clinit_;
309
310 DISALLOW_COPY_AND_ASSIGN(LoadClassSlowPathARM64);
311};
312
313class LoadStringSlowPathARM64 : public SlowPathCodeARM64 {
314 public:
315 explicit LoadStringSlowPathARM64(HLoadString* instruction) : instruction_(instruction) {}
316
317 void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
318 LocationSummary* locations = instruction_->GetLocations();
319 DCHECK(!locations->GetLiveRegisters()->ContainsCoreRegister(locations->Out().reg()));
320 CodeGeneratorARM64* arm64_codegen = down_cast<CodeGeneratorARM64*>(codegen);
321
322 __ Bind(GetEntryLabel());
Nicolas Geoffraya8ac9132015-03-13 16:36:36 +0000323 SaveLiveRegisters(codegen, locations);
Alexandre Rames67555f72014-11-18 10:55:16 +0000324
325 InvokeRuntimeCallingConvention calling_convention;
Andreas Gampe1cc7dba2014-12-17 18:43:01 -0800326 __ Mov(calling_convention.GetRegisterAt(0).W(), instruction_->GetStringIndex());
Alexandre Rames67555f72014-11-18 10:55:16 +0000327 arm64_codegen->InvokeRuntime(
Nicolas Geoffrayeeefa122015-03-13 18:52:59 +0000328 QUICK_ENTRY_POINT(pResolveString), instruction_, instruction_->GetDexPc(), this);
Vladimir Marko5ea536a2015-04-20 20:11:30 +0100329 CheckEntrypointTypes<kQuickResolveString, void*, uint32_t>();
Alexandre Rames67555f72014-11-18 10:55:16 +0000330 Primitive::Type type = instruction_->GetType();
Alexandre Rames3e69f162014-12-10 10:36:50 +0000331 arm64_codegen->MoveLocation(locations->Out(), calling_convention.GetReturnLocation(type), type);
Alexandre Rames67555f72014-11-18 10:55:16 +0000332
Nicolas Geoffraya8ac9132015-03-13 16:36:36 +0000333 RestoreLiveRegisters(codegen, locations);
Alexandre Rames67555f72014-11-18 10:55:16 +0000334 __ B(GetExitLabel());
335 }
336
Alexandre Rames9931f312015-06-19 14:47:01 +0100337 const char* GetDescription() const OVERRIDE { return "LoadStringSlowPathARM64"; }
338
Alexandre Rames67555f72014-11-18 10:55:16 +0000339 private:
340 HLoadString* const instruction_;
341
342 DISALLOW_COPY_AND_ASSIGN(LoadStringSlowPathARM64);
343};
344
Alexandre Rames5319def2014-10-23 10:03:10 +0100345class NullCheckSlowPathARM64 : public SlowPathCodeARM64 {
346 public:
347 explicit NullCheckSlowPathARM64(HNullCheck* instr) : instruction_(instr) {}
348
Alexandre Rames67555f72014-11-18 10:55:16 +0000349 void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
350 CodeGeneratorARM64* arm64_codegen = down_cast<CodeGeneratorARM64*>(codegen);
Alexandre Rames5319def2014-10-23 10:03:10 +0100351 __ Bind(GetEntryLabel());
David Brazdil77a48ae2015-09-15 12:34:04 +0000352 if (instruction_->CanThrowIntoCatchBlock()) {
353 // Live registers will be restored in the catch block if caught.
354 SaveLiveRegisters(codegen, instruction_->GetLocations());
355 }
Alexandre Rames67555f72014-11-18 10:55:16 +0000356 arm64_codegen->InvokeRuntime(
Nicolas Geoffrayeeefa122015-03-13 18:52:59 +0000357 QUICK_ENTRY_POINT(pThrowNullPointer), instruction_, instruction_->GetDexPc(), this);
Andreas Gampe1cc7dba2014-12-17 18:43:01 -0800358 CheckEntrypointTypes<kQuickThrowNullPointer, void, void>();
Alexandre Rames5319def2014-10-23 10:03:10 +0100359 }
360
Alexandre Rames8158f282015-08-07 10:26:17 +0100361 bool IsFatal() const OVERRIDE { return true; }
362
Alexandre Rames9931f312015-06-19 14:47:01 +0100363 const char* GetDescription() const OVERRIDE { return "NullCheckSlowPathARM64"; }
364
Alexandre Rames5319def2014-10-23 10:03:10 +0100365 private:
366 HNullCheck* const instruction_;
367
368 DISALLOW_COPY_AND_ASSIGN(NullCheckSlowPathARM64);
369};
370
371class SuspendCheckSlowPathARM64 : public SlowPathCodeARM64 {
372 public:
Roland Levillain3887c462015-08-12 18:15:42 +0100373 SuspendCheckSlowPathARM64(HSuspendCheck* instruction, HBasicBlock* successor)
Alexandre Rames5319def2014-10-23 10:03:10 +0100374 : instruction_(instruction), successor_(successor) {}
375
Alexandre Rames67555f72014-11-18 10:55:16 +0000376 void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
377 CodeGeneratorARM64* arm64_codegen = down_cast<CodeGeneratorARM64*>(codegen);
Alexandre Rames5319def2014-10-23 10:03:10 +0100378 __ Bind(GetEntryLabel());
Nicolas Geoffraya8ac9132015-03-13 16:36:36 +0000379 SaveLiveRegisters(codegen, instruction_->GetLocations());
Alexandre Rames67555f72014-11-18 10:55:16 +0000380 arm64_codegen->InvokeRuntime(
Nicolas Geoffrayeeefa122015-03-13 18:52:59 +0000381 QUICK_ENTRY_POINT(pTestSuspend), instruction_, instruction_->GetDexPc(), this);
Andreas Gampe1cc7dba2014-12-17 18:43:01 -0800382 CheckEntrypointTypes<kQuickTestSuspend, void, void>();
Nicolas Geoffraya8ac9132015-03-13 16:36:36 +0000383 RestoreLiveRegisters(codegen, instruction_->GetLocations());
Alexandre Rames67555f72014-11-18 10:55:16 +0000384 if (successor_ == nullptr) {
385 __ B(GetReturnLabel());
386 } else {
387 __ B(arm64_codegen->GetLabelOf(successor_));
388 }
Alexandre Rames5319def2014-10-23 10:03:10 +0100389 }
390
391 vixl::Label* GetReturnLabel() {
392 DCHECK(successor_ == nullptr);
393 return &return_label_;
394 }
395
Nicolas Geoffraydb216f42015-05-05 17:02:20 +0100396 HBasicBlock* GetSuccessor() const {
397 return successor_;
398 }
399
Alexandre Rames9931f312015-06-19 14:47:01 +0100400 const char* GetDescription() const OVERRIDE { return "SuspendCheckSlowPathARM64"; }
401
Alexandre Rames5319def2014-10-23 10:03:10 +0100402 private:
403 HSuspendCheck* const instruction_;
404 // If not null, the block to branch to after the suspend check.
405 HBasicBlock* const successor_;
406
407 // If `successor_` is null, the label to branch to after the suspend check.
408 vixl::Label return_label_;
409
410 DISALLOW_COPY_AND_ASSIGN(SuspendCheckSlowPathARM64);
411};
412
Alexandre Rames67555f72014-11-18 10:55:16 +0000413class TypeCheckSlowPathARM64 : public SlowPathCodeARM64 {
414 public:
Nicolas Geoffray85c7bab2015-09-18 13:40:46 +0000415 TypeCheckSlowPathARM64(HInstruction* instruction, bool is_fatal)
416 : instruction_(instruction), is_fatal_(is_fatal) {}
Alexandre Rames67555f72014-11-18 10:55:16 +0000417
418 void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
Alexandre Rames3e69f162014-12-10 10:36:50 +0000419 LocationSummary* locations = instruction_->GetLocations();
Serban Constantinescu5a6cc492015-08-13 15:20:25 +0100420 Location class_to_check = locations->InAt(1);
421 Location object_class = instruction_->IsCheckCast() ? locations->GetTemp(0)
422 : locations->Out();
Alexandre Rames3e69f162014-12-10 10:36:50 +0000423 DCHECK(instruction_->IsCheckCast()
424 || !locations->GetLiveRegisters()->ContainsCoreRegister(locations->Out().reg()));
425 CodeGeneratorARM64* arm64_codegen = down_cast<CodeGeneratorARM64*>(codegen);
Serban Constantinescu5a6cc492015-08-13 15:20:25 +0100426 uint32_t dex_pc = instruction_->GetDexPc();
Alexandre Rames3e69f162014-12-10 10:36:50 +0000427
Alexandre Rames67555f72014-11-18 10:55:16 +0000428 __ Bind(GetEntryLabel());
Nicolas Geoffray85c7bab2015-09-18 13:40:46 +0000429
430 if (instruction_->IsCheckCast()) {
431 // The codegen for the instruction overwrites `temp`, so put it back in place.
432 Register obj = InputRegisterAt(instruction_, 0);
433 Register temp = WRegisterFrom(locations->GetTemp(0));
434 uint32_t class_offset = mirror::Object::ClassOffset().Int32Value();
435 __ Ldr(temp, HeapOperand(obj, class_offset));
436 arm64_codegen->GetAssembler()->MaybeUnpoisonHeapReference(temp);
437 }
438
439 if (!is_fatal_) {
440 SaveLiveRegisters(codegen, locations);
441 }
Alexandre Rames3e69f162014-12-10 10:36:50 +0000442
443 // We're moving two locations to locations that could overlap, so we need a parallel
444 // move resolver.
445 InvokeRuntimeCallingConvention calling_convention;
446 codegen->EmitParallelMoves(
Serban Constantinescu5a6cc492015-08-13 15:20:25 +0100447 class_to_check, LocationFrom(calling_convention.GetRegisterAt(0)), Primitive::kPrimNot,
448 object_class, LocationFrom(calling_convention.GetRegisterAt(1)), Primitive::kPrimNot);
Alexandre Rames3e69f162014-12-10 10:36:50 +0000449
450 if (instruction_->IsInstanceOf()) {
Nicolas Geoffrayeeefa122015-03-13 18:52:59 +0000451 arm64_codegen->InvokeRuntime(
Serban Constantinescu5a6cc492015-08-13 15:20:25 +0100452 QUICK_ENTRY_POINT(pInstanceofNonTrivial), instruction_, dex_pc, this);
Alexandre Rames3e69f162014-12-10 10:36:50 +0000453 Primitive::Type ret_type = instruction_->GetType();
454 Location ret_loc = calling_convention.GetReturnLocation(ret_type);
455 arm64_codegen->MoveLocation(locations->Out(), ret_loc, ret_type);
Andreas Gampe1cc7dba2014-12-17 18:43:01 -0800456 CheckEntrypointTypes<kQuickInstanceofNonTrivial, uint32_t,
457 const mirror::Class*, const mirror::Class*>();
Alexandre Rames3e69f162014-12-10 10:36:50 +0000458 } else {
459 DCHECK(instruction_->IsCheckCast());
Serban Constantinescu5a6cc492015-08-13 15:20:25 +0100460 arm64_codegen->InvokeRuntime(QUICK_ENTRY_POINT(pCheckCast), instruction_, dex_pc, this);
Andreas Gampe1cc7dba2014-12-17 18:43:01 -0800461 CheckEntrypointTypes<kQuickCheckCast, void, const mirror::Class*, const mirror::Class*>();
Alexandre Rames3e69f162014-12-10 10:36:50 +0000462 }
463
Nicolas Geoffray85c7bab2015-09-18 13:40:46 +0000464 if (!is_fatal_) {
465 RestoreLiveRegisters(codegen, locations);
466 __ B(GetExitLabel());
467 }
Alexandre Rames67555f72014-11-18 10:55:16 +0000468 }
469
Alexandre Rames9931f312015-06-19 14:47:01 +0100470 const char* GetDescription() const OVERRIDE { return "TypeCheckSlowPathARM64"; }
Nicolas Geoffray85c7bab2015-09-18 13:40:46 +0000471 bool IsFatal() const { return is_fatal_; }
Alexandre Rames9931f312015-06-19 14:47:01 +0100472
Alexandre Rames67555f72014-11-18 10:55:16 +0000473 private:
Alexandre Rames3e69f162014-12-10 10:36:50 +0000474 HInstruction* const instruction_;
Nicolas Geoffray85c7bab2015-09-18 13:40:46 +0000475 const bool is_fatal_;
Alexandre Rames3e69f162014-12-10 10:36:50 +0000476
Alexandre Rames67555f72014-11-18 10:55:16 +0000477 DISALLOW_COPY_AND_ASSIGN(TypeCheckSlowPathARM64);
478};
479
Mingyao Yangd43b3ac2015-04-01 14:03:04 -0700480class DeoptimizationSlowPathARM64 : public SlowPathCodeARM64 {
481 public:
482 explicit DeoptimizationSlowPathARM64(HInstruction* instruction)
Nicolas Geoffraye0395dd2015-09-25 11:04:45 +0100483 : instruction_(instruction) {}
Mingyao Yangd43b3ac2015-04-01 14:03:04 -0700484
485 void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
486 __ Bind(GetEntryLabel());
487 SaveLiveRegisters(codegen, instruction_->GetLocations());
488 DCHECK(instruction_->IsDeoptimize());
489 HDeoptimize* deoptimize = instruction_->AsDeoptimize();
490 uint32_t dex_pc = deoptimize->GetDexPc();
491 CodeGeneratorARM64* arm64_codegen = down_cast<CodeGeneratorARM64*>(codegen);
492 arm64_codegen->InvokeRuntime(QUICK_ENTRY_POINT(pDeoptimize), instruction_, dex_pc, this);
493 }
494
Alexandre Rames9931f312015-06-19 14:47:01 +0100495 const char* GetDescription() const OVERRIDE { return "DeoptimizationSlowPathARM64"; }
496
Mingyao Yangd43b3ac2015-04-01 14:03:04 -0700497 private:
498 HInstruction* const instruction_;
499 DISALLOW_COPY_AND_ASSIGN(DeoptimizationSlowPathARM64);
500};
501
Nicolas Geoffraye0395dd2015-09-25 11:04:45 +0100502class ArraySetSlowPathARM64 : public SlowPathCodeARM64 {
503 public:
504 explicit ArraySetSlowPathARM64(HInstruction* instruction) : instruction_(instruction) {}
505
506 void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
507 LocationSummary* locations = instruction_->GetLocations();
508 __ Bind(GetEntryLabel());
509 SaveLiveRegisters(codegen, locations);
510
511 InvokeRuntimeCallingConvention calling_convention;
512 HParallelMove parallel_move(codegen->GetGraph()->GetArena());
513 parallel_move.AddMove(
514 locations->InAt(0),
515 LocationFrom(calling_convention.GetRegisterAt(0)),
516 Primitive::kPrimNot,
517 nullptr);
518 parallel_move.AddMove(
519 locations->InAt(1),
520 LocationFrom(calling_convention.GetRegisterAt(1)),
521 Primitive::kPrimInt,
522 nullptr);
523 parallel_move.AddMove(
524 locations->InAt(2),
525 LocationFrom(calling_convention.GetRegisterAt(2)),
526 Primitive::kPrimNot,
527 nullptr);
528 codegen->GetMoveResolver()->EmitNativeCode(&parallel_move);
529
530 CodeGeneratorARM64* arm64_codegen = down_cast<CodeGeneratorARM64*>(codegen);
531 arm64_codegen->InvokeRuntime(QUICK_ENTRY_POINT(pAputObject),
532 instruction_,
533 instruction_->GetDexPc(),
534 this);
535 CheckEntrypointTypes<kQuickAputObject, void, mirror::Array*, int32_t, mirror::Object*>();
536 RestoreLiveRegisters(codegen, locations);
537 __ B(GetExitLabel());
538 }
539
540 const char* GetDescription() const OVERRIDE { return "ArraySetSlowPathARM64"; }
541
542 private:
543 HInstruction* const instruction_;
544
545 DISALLOW_COPY_AND_ASSIGN(ArraySetSlowPathARM64);
546};
547
Alexandre Rames5319def2014-10-23 10:03:10 +0100548#undef __
549
Roland Levillain2d27c8e2015-04-28 15:48:45 +0100550Location InvokeDexCallingConventionVisitorARM64::GetNextLocation(Primitive::Type type) {
Alexandre Rames5319def2014-10-23 10:03:10 +0100551 Location next_location;
552 if (type == Primitive::kPrimVoid) {
553 LOG(FATAL) << "Unreachable type " << type;
554 }
555
Alexandre Rames542361f2015-01-29 16:57:31 +0000556 if (Primitive::IsFloatingPointType(type) &&
Roland Levillain2d27c8e2015-04-28 15:48:45 +0100557 (float_index_ < calling_convention.GetNumberOfFpuRegisters())) {
558 next_location = LocationFrom(calling_convention.GetFpuRegisterAt(float_index_++));
Alexandre Rames542361f2015-01-29 16:57:31 +0000559 } else if (!Primitive::IsFloatingPointType(type) &&
560 (gp_index_ < calling_convention.GetNumberOfRegisters())) {
Alexandre Ramesa89086e2014-11-07 17:13:25 +0000561 next_location = LocationFrom(calling_convention.GetRegisterAt(gp_index_++));
562 } else {
563 size_t stack_offset = calling_convention.GetStackOffsetOf(stack_index_);
Alexandre Rames542361f2015-01-29 16:57:31 +0000564 next_location = Primitive::Is64BitType(type) ? Location::DoubleStackSlot(stack_offset)
565 : Location::StackSlot(stack_offset);
Alexandre Rames5319def2014-10-23 10:03:10 +0100566 }
567
Alexandre Ramesa89086e2014-11-07 17:13:25 +0000568 // Space on the stack is reserved for all arguments.
Alexandre Rames542361f2015-01-29 16:57:31 +0000569 stack_index_ += Primitive::Is64BitType(type) ? 2 : 1;
Alexandre Rames5319def2014-10-23 10:03:10 +0100570 return next_location;
571}
572
Nicolas Geoffrayfd88f162015-06-03 11:23:52 +0100573Location InvokeDexCallingConventionVisitorARM64::GetMethodLocation() const {
Nicolas Geoffray38207af2015-06-01 15:46:22 +0100574 return LocationFrom(kArtMethodRegister);
Nicolas Geoffrayfd88f162015-06-03 11:23:52 +0100575}
576
Serban Constantinescu579885a2015-02-22 20:51:33 +0000577CodeGeneratorARM64::CodeGeneratorARM64(HGraph* graph,
578 const Arm64InstructionSetFeatures& isa_features,
Serban Constantinescuecc43662015-08-13 13:33:12 +0100579 const CompilerOptions& compiler_options,
580 OptimizingCompilerStats* stats)
Alexandre Rames5319def2014-10-23 10:03:10 +0100581 : CodeGenerator(graph,
582 kNumberOfAllocatableRegisters,
Alexandre Ramesa89086e2014-11-07 17:13:25 +0000583 kNumberOfAllocatableFPRegisters,
Calin Juravlecd6dffe2015-01-08 17:35:35 +0000584 kNumberOfAllocatableRegisterPairs,
Serban Constantinescu3d087de2015-01-28 11:57:05 +0000585 callee_saved_core_registers.list(),
Nicolas Geoffray75d5b9b2015-10-05 07:40:35 +0000586 callee_saved_fp_registers.list(),
Serban Constantinescuecc43662015-08-13 13:33:12 +0100587 compiler_options,
588 stats),
Alexandre Rames5319def2014-10-23 10:03:10 +0100589 block_labels_(nullptr),
590 location_builder_(graph, this),
Alexandre Rames3e69f162014-12-10 10:36:50 +0000591 instruction_visitor_(graph, this),
Serban Constantinescu579885a2015-02-22 20:51:33 +0000592 move_resolver_(graph->GetArena(), this),
Vladimir Marko58155012015-08-19 12:49:41 +0000593 isa_features_(isa_features),
Vladimir Marko5233f932015-09-29 19:01:15 +0100594 uint64_literals_(std::less<uint64_t>(),
595 graph->GetArena()->Adapter(kArenaAllocCodeGenerator)),
596 method_patches_(MethodReferenceComparator(),
597 graph->GetArena()->Adapter(kArenaAllocCodeGenerator)),
598 call_patches_(MethodReferenceComparator(),
599 graph->GetArena()->Adapter(kArenaAllocCodeGenerator)),
600 relative_call_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)),
Vladimir Marko0f7dca42015-11-02 14:36:43 +0000601 pc_relative_dex_cache_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)) {
Nicolas Geoffrayd97dc402015-01-22 13:50:01 +0000602 // Save the link register (containing the return address) to mimic Quick.
Serban Constantinescu3d087de2015-01-28 11:57:05 +0000603 AddAllocatedRegister(LocationFrom(lr));
Nicolas Geoffrayd97dc402015-01-22 13:50:01 +0000604}
Alexandre Rames5319def2014-10-23 10:03:10 +0100605
Alexandre Rames67555f72014-11-18 10:55:16 +0000606#undef __
607#define __ GetVIXLAssembler()->
Alexandre Rames5319def2014-10-23 10:03:10 +0100608
Serban Constantinescu32f5b4d2014-11-25 20:05:46 +0000609void CodeGeneratorARM64::Finalize(CodeAllocator* allocator) {
610 // Ensure we emit the literal pool.
611 __ FinalizeCode();
Vladimir Marko58155012015-08-19 12:49:41 +0000612
Serban Constantinescu32f5b4d2014-11-25 20:05:46 +0000613 CodeGenerator::Finalize(allocator);
614}
615
Zheng Xuad4450e2015-04-17 18:48:56 +0800616void ParallelMoveResolverARM64::PrepareForEmitNativeCode() {
617 // Note: There are 6 kinds of moves:
618 // 1. constant -> GPR/FPR (non-cycle)
619 // 2. constant -> stack (non-cycle)
620 // 3. GPR/FPR -> GPR/FPR
621 // 4. GPR/FPR -> stack
622 // 5. stack -> GPR/FPR
623 // 6. stack -> stack (non-cycle)
624 // Case 1, 2 and 6 should never be included in a dependency cycle on ARM64. For case 3, 4, and 5
625 // VIXL uses at most 1 GPR. VIXL has 2 GPR and 1 FPR temps, and there should be no intersecting
626 // cycles on ARM64, so we always have 1 GPR and 1 FPR available VIXL temps to resolve the
627 // dependency.
628 vixl_temps_.Open(GetVIXLAssembler());
629}
630
631void ParallelMoveResolverARM64::FinishEmitNativeCode() {
632 vixl_temps_.Close();
633}
634
635Location ParallelMoveResolverARM64::AllocateScratchLocationFor(Location::Kind kind) {
636 DCHECK(kind == Location::kRegister || kind == Location::kFpuRegister ||
637 kind == Location::kStackSlot || kind == Location::kDoubleStackSlot);
638 kind = (kind == Location::kFpuRegister) ? Location::kFpuRegister : Location::kRegister;
639 Location scratch = GetScratchLocation(kind);
640 if (!scratch.Equals(Location::NoLocation())) {
641 return scratch;
642 }
643 // Allocate from VIXL temp registers.
644 if (kind == Location::kRegister) {
645 scratch = LocationFrom(vixl_temps_.AcquireX());
646 } else {
647 DCHECK(kind == Location::kFpuRegister);
648 scratch = LocationFrom(vixl_temps_.AcquireD());
649 }
650 AddScratchLocation(scratch);
651 return scratch;
652}
653
654void ParallelMoveResolverARM64::FreeScratchLocation(Location loc) {
655 if (loc.IsRegister()) {
656 vixl_temps_.Release(XRegisterFrom(loc));
657 } else {
658 DCHECK(loc.IsFpuRegister());
659 vixl_temps_.Release(DRegisterFrom(loc));
660 }
661 RemoveScratchLocation(loc);
662}
663
Alexandre Rames3e69f162014-12-10 10:36:50 +0000664void ParallelMoveResolverARM64::EmitMove(size_t index) {
Vladimir Marko225b6462015-09-28 12:17:40 +0100665 MoveOperands* move = moves_[index];
Calin Juravlee460d1d2015-09-29 04:52:17 +0100666 codegen_->MoveLocation(move->GetDestination(), move->GetSource(), Primitive::kPrimVoid);
Alexandre Rames3e69f162014-12-10 10:36:50 +0000667}
668
Alexandre Rames5319def2014-10-23 10:03:10 +0100669void CodeGeneratorARM64::GenerateFrameEntry() {
Alexandre Ramesd921d642015-04-16 15:07:16 +0100670 MacroAssembler* masm = GetVIXLAssembler();
671 BlockPoolsScope block_pools(masm);
Nicolas Geoffray1cf95282014-12-12 19:22:03 +0000672 __ Bind(&frame_entry_label_);
673
Serban Constantinescu02164b32014-11-13 14:05:07 +0000674 bool do_overflow_check = FrameNeedsStackCheck(GetFrameSize(), kArm64) || !IsLeafMethod();
675 if (do_overflow_check) {
Alexandre Ramesd921d642015-04-16 15:07:16 +0100676 UseScratchRegisterScope temps(masm);
Serban Constantinescu02164b32014-11-13 14:05:07 +0000677 Register temp = temps.AcquireX();
Nicolas Geoffrayd97dc402015-01-22 13:50:01 +0000678 DCHECK(GetCompilerOptions().GetImplicitStackOverflowChecks());
Serban Constantinescu3d087de2015-01-28 11:57:05 +0000679 __ Sub(temp, sp, static_cast<int32_t>(GetStackOverflowReservedBytes(kArm64)));
Nicolas Geoffrayd97dc402015-01-22 13:50:01 +0000680 __ Ldr(wzr, MemOperand(temp, 0));
681 RecordPcInfo(nullptr, 0);
Serban Constantinescu02164b32014-11-13 14:05:07 +0000682 }
Alexandre Rames5319def2014-10-23 10:03:10 +0100683
Nicolas Geoffrayc0572a42015-02-06 14:35:25 +0000684 if (!HasEmptyFrame()) {
685 int frame_size = GetFrameSize();
686 // Stack layout:
687 // sp[frame_size - 8] : lr.
688 // ... : other preserved core registers.
689 // ... : other preserved fp registers.
690 // ... : reserved frame space.
691 // sp[0] : current method.
692 __ Str(kArtMethodRegister, MemOperand(sp, -frame_size, PreIndex));
David Srbeckyc6b4dd82015-04-07 20:32:43 +0100693 GetAssembler()->cfi().AdjustCFAOffset(frame_size);
Zheng Xu69a50302015-04-14 20:04:41 +0800694 GetAssembler()->SpillRegisters(GetFramePreservedCoreRegisters(),
695 frame_size - GetCoreSpillSize());
696 GetAssembler()->SpillRegisters(GetFramePreservedFPRegisters(),
697 frame_size - FrameEntrySpillSize());
Nicolas Geoffrayc0572a42015-02-06 14:35:25 +0000698 }
Alexandre Rames5319def2014-10-23 10:03:10 +0100699}
700
701void CodeGeneratorARM64::GenerateFrameExit() {
Alexandre Ramesd921d642015-04-16 15:07:16 +0100702 BlockPoolsScope block_pools(GetVIXLAssembler());
David Srbeckyc34dc932015-04-12 09:27:43 +0100703 GetAssembler()->cfi().RememberState();
Nicolas Geoffrayc0572a42015-02-06 14:35:25 +0000704 if (!HasEmptyFrame()) {
705 int frame_size = GetFrameSize();
Zheng Xu69a50302015-04-14 20:04:41 +0800706 GetAssembler()->UnspillRegisters(GetFramePreservedFPRegisters(),
707 frame_size - FrameEntrySpillSize());
708 GetAssembler()->UnspillRegisters(GetFramePreservedCoreRegisters(),
709 frame_size - GetCoreSpillSize());
Nicolas Geoffrayc0572a42015-02-06 14:35:25 +0000710 __ Drop(frame_size);
David Srbeckyc6b4dd82015-04-07 20:32:43 +0100711 GetAssembler()->cfi().AdjustCFAOffset(-frame_size);
Nicolas Geoffrayc0572a42015-02-06 14:35:25 +0000712 }
David Srbeckyc34dc932015-04-12 09:27:43 +0100713 __ Ret();
714 GetAssembler()->cfi().RestoreState();
715 GetAssembler()->cfi().DefCFAOffset(GetFrameSize());
Alexandre Rames5319def2014-10-23 10:03:10 +0100716}
717
Zheng Xuda403092015-04-24 17:35:39 +0800718vixl::CPURegList CodeGeneratorARM64::GetFramePreservedCoreRegisters() const {
719 DCHECK(ArtVixlRegCodeCoherentForRegSet(core_spill_mask_, GetNumberOfCoreRegisters(), 0, 0));
720 return vixl::CPURegList(vixl::CPURegister::kRegister, vixl::kXRegSize,
721 core_spill_mask_);
722}
723
724vixl::CPURegList CodeGeneratorARM64::GetFramePreservedFPRegisters() const {
725 DCHECK(ArtVixlRegCodeCoherentForRegSet(0, 0, fpu_spill_mask_,
726 GetNumberOfFloatingPointRegisters()));
727 return vixl::CPURegList(vixl::CPURegister::kFPRegister, vixl::kDRegSize,
728 fpu_spill_mask_);
729}
730
Alexandre Rames5319def2014-10-23 10:03:10 +0100731void CodeGeneratorARM64::Bind(HBasicBlock* block) {
732 __ Bind(GetLabelOf(block));
733}
734
Alexandre Rames5319def2014-10-23 10:03:10 +0100735void CodeGeneratorARM64::Move(HInstruction* instruction,
736 Location location,
737 HInstruction* move_for) {
738 LocationSummary* locations = instruction->GetLocations();
Alexandre Rames5319def2014-10-23 10:03:10 +0100739 Primitive::Type type = instruction->GetType();
Alexandre Ramesa89086e2014-11-07 17:13:25 +0000740 DCHECK_NE(type, Primitive::kPrimVoid);
Alexandre Rames5319def2014-10-23 10:03:10 +0100741
Nicolas Geoffray9b1eba32015-07-13 15:55:26 +0100742 if (instruction->IsFakeString()) {
743 // The fake string is an alias for null.
744 DCHECK(IsBaseline());
745 instruction = locations->Out().GetConstant();
746 DCHECK(instruction->IsNullConstant()) << instruction->DebugName();
747 }
748
Nicolas Geoffray76b1e172015-05-27 17:18:33 +0100749 if (instruction->IsCurrentMethod()) {
Calin Juravlee460d1d2015-09-29 04:52:17 +0100750 MoveLocation(location,
751 Location::DoubleStackSlot(kCurrentMethodStackOffset),
752 Primitive::kPrimVoid);
Nicolas Geoffray76b1e172015-05-27 17:18:33 +0100753 } else if (locations != nullptr && locations->Out().Equals(location)) {
754 return;
755 } else if (instruction->IsIntConstant()
756 || instruction->IsLongConstant()
757 || instruction->IsNullConstant()) {
Nicolas Geoffrayd6138ef2015-02-18 14:48:53 +0000758 int64_t value = GetInt64ValueOf(instruction->AsConstant());
Alexandre Rames5319def2014-10-23 10:03:10 +0100759 if (location.IsRegister()) {
760 Register dst = RegisterFrom(location, type);
Nicolas Geoffrayd6138ef2015-02-18 14:48:53 +0000761 DCHECK(((instruction->IsIntConstant() || instruction->IsNullConstant()) && dst.Is32Bits()) ||
Alexandre Rames5319def2014-10-23 10:03:10 +0100762 (instruction->IsLongConstant() && dst.Is64Bits()));
763 __ Mov(dst, value);
764 } else {
765 DCHECK(location.IsStackSlot() || location.IsDoubleStackSlot());
Alexandre Rames67555f72014-11-18 10:55:16 +0000766 UseScratchRegisterScope temps(GetVIXLAssembler());
Nicolas Geoffrayd6138ef2015-02-18 14:48:53 +0000767 Register temp = (instruction->IsIntConstant() || instruction->IsNullConstant())
768 ? temps.AcquireW()
769 : temps.AcquireX();
Alexandre Rames5319def2014-10-23 10:03:10 +0100770 __ Mov(temp, value);
771 __ Str(temp, StackOperandFrom(location));
772 }
Nicolas Geoffrayf43083d2014-11-07 10:48:10 +0000773 } else if (instruction->IsTemporary()) {
774 Location temp_location = GetTemporaryLocation(instruction->AsTemporary());
Alexandre Rames3e69f162014-12-10 10:36:50 +0000775 MoveLocation(location, temp_location, type);
Alexandre Rames5319def2014-10-23 10:03:10 +0100776 } else if (instruction->IsLoadLocal()) {
777 uint32_t stack_slot = GetStackSlot(instruction->AsLoadLocal()->GetLocal());
Alexandre Rames542361f2015-01-29 16:57:31 +0000778 if (Primitive::Is64BitType(type)) {
Alexandre Rames3e69f162014-12-10 10:36:50 +0000779 MoveLocation(location, Location::DoubleStackSlot(stack_slot), type);
Alexandre Ramesa89086e2014-11-07 17:13:25 +0000780 } else {
Alexandre Rames3e69f162014-12-10 10:36:50 +0000781 MoveLocation(location, Location::StackSlot(stack_slot), type);
Alexandre Rames5319def2014-10-23 10:03:10 +0100782 }
783
784 } else {
785 DCHECK((instruction->GetNext() == move_for) || instruction->GetNext()->IsTemporary());
Alexandre Rames3e69f162014-12-10 10:36:50 +0000786 MoveLocation(location, locations->Out(), type);
Alexandre Rames5319def2014-10-23 10:03:10 +0100787 }
788}
789
Calin Juravle175dc732015-08-25 15:42:32 +0100790void CodeGeneratorARM64::MoveConstant(Location location, int32_t value) {
791 DCHECK(location.IsRegister());
792 __ Mov(RegisterFrom(location, Primitive::kPrimInt), value);
793}
794
Calin Juravlee460d1d2015-09-29 04:52:17 +0100795void CodeGeneratorARM64::AddLocationAsTemp(Location location, LocationSummary* locations) {
796 if (location.IsRegister()) {
797 locations->AddTemp(location);
798 } else {
799 UNIMPLEMENTED(FATAL) << "AddLocationAsTemp not implemented for location " << location;
800 }
801}
802
Alexandre Rames5319def2014-10-23 10:03:10 +0100803Location CodeGeneratorARM64::GetStackLocation(HLoadLocal* load) const {
804 Primitive::Type type = load->GetType();
Alexandre Ramesa89086e2014-11-07 17:13:25 +0000805
Alexandre Rames5319def2014-10-23 10:03:10 +0100806 switch (type) {
807 case Primitive::kPrimNot:
Alexandre Ramesa89086e2014-11-07 17:13:25 +0000808 case Primitive::kPrimInt:
809 case Primitive::kPrimFloat:
810 return Location::StackSlot(GetStackSlot(load->GetLocal()));
811
812 case Primitive::kPrimLong:
813 case Primitive::kPrimDouble:
814 return Location::DoubleStackSlot(GetStackSlot(load->GetLocal()));
815
Alexandre Rames5319def2014-10-23 10:03:10 +0100816 case Primitive::kPrimBoolean:
817 case Primitive::kPrimByte:
818 case Primitive::kPrimChar:
819 case Primitive::kPrimShort:
Alexandre Rames5319def2014-10-23 10:03:10 +0100820 case Primitive::kPrimVoid:
Alexandre Rames5319def2014-10-23 10:03:10 +0100821 LOG(FATAL) << "Unexpected type " << type;
822 }
Alexandre Ramesa89086e2014-11-07 17:13:25 +0000823
Alexandre Rames5319def2014-10-23 10:03:10 +0100824 LOG(FATAL) << "Unreachable";
825 return Location::NoLocation();
826}
827
Nicolas Geoffray07276db2015-05-18 14:22:09 +0100828void CodeGeneratorARM64::MarkGCCard(Register object, Register value, bool value_can_be_null) {
Alexandre Rames67555f72014-11-18 10:55:16 +0000829 UseScratchRegisterScope temps(GetVIXLAssembler());
Alexandre Rames5319def2014-10-23 10:03:10 +0100830 Register card = temps.AcquireX();
Serban Constantinescu02164b32014-11-13 14:05:07 +0000831 Register temp = temps.AcquireW(); // Index within the CardTable - 32bit.
Alexandre Rames5319def2014-10-23 10:03:10 +0100832 vixl::Label done;
Nicolas Geoffray07276db2015-05-18 14:22:09 +0100833 if (value_can_be_null) {
834 __ Cbz(value, &done);
835 }
Alexandre Rames5319def2014-10-23 10:03:10 +0100836 __ Ldr(card, MemOperand(tr, Thread::CardTableOffset<kArm64WordSize>().Int32Value()));
837 __ Lsr(temp, object, gc::accounting::CardTable::kCardShift);
Serban Constantinescu02164b32014-11-13 14:05:07 +0000838 __ Strb(card, MemOperand(card, temp.X()));
Nicolas Geoffray07276db2015-05-18 14:22:09 +0100839 if (value_can_be_null) {
840 __ Bind(&done);
841 }
Alexandre Rames5319def2014-10-23 10:03:10 +0100842}
843
Serban Constantinescu3d087de2015-01-28 11:57:05 +0000844void CodeGeneratorARM64::SetupBlockedRegisters(bool is_baseline) const {
845 // Blocked core registers:
846 // lr : Runtime reserved.
847 // tr : Runtime reserved.
848 // xSuspend : Runtime reserved. TODO: Unblock this when the runtime stops using it.
849 // ip1 : VIXL core temp.
850 // ip0 : VIXL core temp.
851 //
852 // Blocked fp registers:
853 // d31 : VIXL fp temp.
Alexandre Rames5319def2014-10-23 10:03:10 +0100854 CPURegList reserved_core_registers = vixl_reserved_core_registers;
855 reserved_core_registers.Combine(runtime_reserved_core_registers);
Alexandre Rames5319def2014-10-23 10:03:10 +0100856 while (!reserved_core_registers.IsEmpty()) {
857 blocked_core_registers_[reserved_core_registers.PopLowestIndex().code()] = true;
858 }
Serban Constantinescu3d087de2015-01-28 11:57:05 +0000859
Alexandre Ramesa89086e2014-11-07 17:13:25 +0000860 CPURegList reserved_fp_registers = vixl_reserved_fp_registers;
Zheng Xua3ec3942015-02-15 18:39:46 +0800861 while (!reserved_fp_registers.IsEmpty()) {
Alexandre Ramesa89086e2014-11-07 17:13:25 +0000862 blocked_fpu_registers_[reserved_fp_registers.PopLowestIndex().code()] = true;
863 }
Serban Constantinescu3d087de2015-01-28 11:57:05 +0000864
865 if (is_baseline) {
866 CPURegList reserved_core_baseline_registers = callee_saved_core_registers;
867 while (!reserved_core_baseline_registers.IsEmpty()) {
868 blocked_core_registers_[reserved_core_baseline_registers.PopLowestIndex().code()] = true;
869 }
Nicolas Geoffrayecf680d2015-10-05 11:15:37 +0100870 }
Serban Constantinescu3d087de2015-01-28 11:57:05 +0000871
Nicolas Geoffrayecf680d2015-10-05 11:15:37 +0100872 if (is_baseline || GetGraph()->IsDebuggable()) {
873 // Stubs do not save callee-save floating point registers. If the graph
874 // is debuggable, we need to deal with these registers differently. For
875 // now, just block them.
Serban Constantinescu3d087de2015-01-28 11:57:05 +0000876 CPURegList reserved_fp_baseline_registers = callee_saved_fp_registers;
877 while (!reserved_fp_baseline_registers.IsEmpty()) {
878 blocked_fpu_registers_[reserved_fp_baseline_registers.PopLowestIndex().code()] = true;
879 }
880 }
Alexandre Rames5319def2014-10-23 10:03:10 +0100881}
882
883Location CodeGeneratorARM64::AllocateFreeRegister(Primitive::Type type) const {
884 if (type == Primitive::kPrimVoid) {
885 LOG(FATAL) << "Unreachable type " << type;
886 }
887
Alexandre Rames542361f2015-01-29 16:57:31 +0000888 if (Primitive::IsFloatingPointType(type)) {
Alexandre Ramesa89086e2014-11-07 17:13:25 +0000889 ssize_t reg = FindFreeEntry(blocked_fpu_registers_, kNumberOfAllocatableFPRegisters);
890 DCHECK_NE(reg, -1);
Alexandre Rames5319def2014-10-23 10:03:10 +0100891 return Location::FpuRegisterLocation(reg);
892 } else {
Alexandre Ramesa89086e2014-11-07 17:13:25 +0000893 ssize_t reg = FindFreeEntry(blocked_core_registers_, kNumberOfAllocatableRegisters);
894 DCHECK_NE(reg, -1);
Alexandre Rames5319def2014-10-23 10:03:10 +0100895 return Location::RegisterLocation(reg);
896 }
897}
898
Alexandre Rames3e69f162014-12-10 10:36:50 +0000899size_t CodeGeneratorARM64::SaveCoreRegister(size_t stack_index, uint32_t reg_id) {
900 Register reg = Register(VIXLRegCodeFromART(reg_id), kXRegSize);
901 __ Str(reg, MemOperand(sp, stack_index));
902 return kArm64WordSize;
903}
904
905size_t CodeGeneratorARM64::RestoreCoreRegister(size_t stack_index, uint32_t reg_id) {
906 Register reg = Register(VIXLRegCodeFromART(reg_id), kXRegSize);
907 __ Ldr(reg, MemOperand(sp, stack_index));
908 return kArm64WordSize;
909}
910
911size_t CodeGeneratorARM64::SaveFloatingPointRegister(size_t stack_index, uint32_t reg_id) {
912 FPRegister reg = FPRegister(reg_id, kDRegSize);
913 __ Str(reg, MemOperand(sp, stack_index));
914 return kArm64WordSize;
915}
916
917size_t CodeGeneratorARM64::RestoreFloatingPointRegister(size_t stack_index, uint32_t reg_id) {
918 FPRegister reg = FPRegister(reg_id, kDRegSize);
919 __ Ldr(reg, MemOperand(sp, stack_index));
920 return kArm64WordSize;
921}
922
Alexandre Rames5319def2014-10-23 10:03:10 +0100923void CodeGeneratorARM64::DumpCoreRegister(std::ostream& stream, int reg) const {
David Brazdilc74652862015-05-13 17:50:09 +0100924 stream << XRegister(reg);
Alexandre Rames5319def2014-10-23 10:03:10 +0100925}
926
927void CodeGeneratorARM64::DumpFloatingPointRegister(std::ostream& stream, int reg) const {
David Brazdilc74652862015-05-13 17:50:09 +0100928 stream << DRegister(reg);
Alexandre Rames5319def2014-10-23 10:03:10 +0100929}
930
Alexandre Rames67555f72014-11-18 10:55:16 +0000931void CodeGeneratorARM64::MoveConstant(CPURegister destination, HConstant* constant) {
Nicolas Geoffrayd6138ef2015-02-18 14:48:53 +0000932 if (constant->IsIntConstant()) {
933 __ Mov(Register(destination), constant->AsIntConstant()->GetValue());
934 } else if (constant->IsLongConstant()) {
935 __ Mov(Register(destination), constant->AsLongConstant()->GetValue());
936 } else if (constant->IsNullConstant()) {
937 __ Mov(Register(destination), 0);
Alexandre Rames67555f72014-11-18 10:55:16 +0000938 } else if (constant->IsFloatConstant()) {
939 __ Fmov(FPRegister(destination), constant->AsFloatConstant()->GetValue());
940 } else {
941 DCHECK(constant->IsDoubleConstant());
942 __ Fmov(FPRegister(destination), constant->AsDoubleConstant()->GetValue());
943 }
944}
945
Alexandre Rames3e69f162014-12-10 10:36:50 +0000946
947static bool CoherentConstantAndType(Location constant, Primitive::Type type) {
948 DCHECK(constant.IsConstant());
949 HConstant* cst = constant.GetConstant();
950 return (cst->IsIntConstant() && type == Primitive::kPrimInt) ||
Nicolas Geoffrayd6138ef2015-02-18 14:48:53 +0000951 // Null is mapped to a core W register, which we associate with kPrimInt.
952 (cst->IsNullConstant() && type == Primitive::kPrimInt) ||
Alexandre Rames3e69f162014-12-10 10:36:50 +0000953 (cst->IsLongConstant() && type == Primitive::kPrimLong) ||
954 (cst->IsFloatConstant() && type == Primitive::kPrimFloat) ||
955 (cst->IsDoubleConstant() && type == Primitive::kPrimDouble);
956}
957
Calin Juravlee460d1d2015-09-29 04:52:17 +0100958void CodeGeneratorARM64::MoveLocation(Location destination,
959 Location source,
960 Primitive::Type dst_type) {
Alexandre Ramesfc19de82014-11-07 17:13:31 +0000961 if (source.Equals(destination)) {
962 return;
963 }
Alexandre Rames3e69f162014-12-10 10:36:50 +0000964
965 // A valid move can always be inferred from the destination and source
966 // locations. When moving from and to a register, the argument type can be
967 // used to generate 32bit instead of 64bit moves. In debug mode we also
968 // checks the coherency of the locations and the type.
Calin Juravlee460d1d2015-09-29 04:52:17 +0100969 bool unspecified_type = (dst_type == Primitive::kPrimVoid);
Alexandre Rames3e69f162014-12-10 10:36:50 +0000970
971 if (destination.IsRegister() || destination.IsFpuRegister()) {
972 if (unspecified_type) {
973 HConstant* src_cst = source.IsConstant() ? source.GetConstant() : nullptr;
974 if (source.IsStackSlot() ||
Nicolas Geoffrayd6138ef2015-02-18 14:48:53 +0000975 (src_cst != nullptr && (src_cst->IsIntConstant()
976 || src_cst->IsFloatConstant()
977 || src_cst->IsNullConstant()))) {
Alexandre Rames3e69f162014-12-10 10:36:50 +0000978 // For stack slots and 32bit constants, a 64bit type is appropriate.
Calin Juravlee460d1d2015-09-29 04:52:17 +0100979 dst_type = destination.IsRegister() ? Primitive::kPrimInt : Primitive::kPrimFloat;
Alexandre Rames67555f72014-11-18 10:55:16 +0000980 } else {
Alexandre Rames3e69f162014-12-10 10:36:50 +0000981 // If the source is a double stack slot or a 64bit constant, a 64bit
982 // type is appropriate. Else the source is a register, and since the
983 // type has not been specified, we chose a 64bit type to force a 64bit
984 // move.
Calin Juravlee460d1d2015-09-29 04:52:17 +0100985 dst_type = destination.IsRegister() ? Primitive::kPrimLong : Primitive::kPrimDouble;
Alexandre Rames67555f72014-11-18 10:55:16 +0000986 }
Alexandre Rames3e69f162014-12-10 10:36:50 +0000987 }
Calin Juravlee460d1d2015-09-29 04:52:17 +0100988 DCHECK((destination.IsFpuRegister() && Primitive::IsFloatingPointType(dst_type)) ||
989 (destination.IsRegister() && !Primitive::IsFloatingPointType(dst_type)));
990 CPURegister dst = CPURegisterFrom(destination, dst_type);
Alexandre Rames3e69f162014-12-10 10:36:50 +0000991 if (source.IsStackSlot() || source.IsDoubleStackSlot()) {
992 DCHECK(dst.Is64Bits() == source.IsDoubleStackSlot());
993 __ Ldr(dst, StackOperandFrom(source));
994 } else if (source.IsConstant()) {
Calin Juravlee460d1d2015-09-29 04:52:17 +0100995 DCHECK(CoherentConstantAndType(source, dst_type));
Alexandre Rames3e69f162014-12-10 10:36:50 +0000996 MoveConstant(dst, source.GetConstant());
Calin Juravlee460d1d2015-09-29 04:52:17 +0100997 } else if (source.IsRegister()) {
Alexandre Rames3e69f162014-12-10 10:36:50 +0000998 if (destination.IsRegister()) {
Calin Juravlee460d1d2015-09-29 04:52:17 +0100999 __ Mov(Register(dst), RegisterFrom(source, dst_type));
Alexandre Rames3e69f162014-12-10 10:36:50 +00001000 } else {
Zheng Xuad4450e2015-04-17 18:48:56 +08001001 DCHECK(destination.IsFpuRegister());
Calin Juravlee460d1d2015-09-29 04:52:17 +01001002 Primitive::Type source_type = Primitive::Is64BitType(dst_type)
1003 ? Primitive::kPrimLong
1004 : Primitive::kPrimInt;
1005 __ Fmov(FPRegisterFrom(destination, dst_type), RegisterFrom(source, source_type));
1006 }
1007 } else {
1008 DCHECK(source.IsFpuRegister());
1009 if (destination.IsRegister()) {
1010 Primitive::Type source_type = Primitive::Is64BitType(dst_type)
1011 ? Primitive::kPrimDouble
1012 : Primitive::kPrimFloat;
1013 __ Fmov(RegisterFrom(destination, dst_type), FPRegisterFrom(source, source_type));
1014 } else {
1015 DCHECK(destination.IsFpuRegister());
1016 __ Fmov(FPRegister(dst), FPRegisterFrom(source, dst_type));
Alexandre Rames3e69f162014-12-10 10:36:50 +00001017 }
1018 }
Alexandre Rames3e69f162014-12-10 10:36:50 +00001019 } else { // The destination is not a register. It must be a stack slot.
1020 DCHECK(destination.IsStackSlot() || destination.IsDoubleStackSlot());
1021 if (source.IsRegister() || source.IsFpuRegister()) {
1022 if (unspecified_type) {
1023 if (source.IsRegister()) {
Calin Juravlee460d1d2015-09-29 04:52:17 +01001024 dst_type = destination.IsStackSlot() ? Primitive::kPrimInt : Primitive::kPrimLong;
Alexandre Rames3e69f162014-12-10 10:36:50 +00001025 } else {
Calin Juravlee460d1d2015-09-29 04:52:17 +01001026 dst_type = destination.IsStackSlot() ? Primitive::kPrimFloat : Primitive::kPrimDouble;
Alexandre Rames3e69f162014-12-10 10:36:50 +00001027 }
1028 }
Calin Juravlee460d1d2015-09-29 04:52:17 +01001029 DCHECK((destination.IsDoubleStackSlot() == Primitive::Is64BitType(dst_type)) &&
1030 (source.IsFpuRegister() == Primitive::IsFloatingPointType(dst_type)));
1031 __ Str(CPURegisterFrom(source, dst_type), StackOperandFrom(destination));
Alexandre Rames3e69f162014-12-10 10:36:50 +00001032 } else if (source.IsConstant()) {
Calin Juravlee460d1d2015-09-29 04:52:17 +01001033 DCHECK(unspecified_type || CoherentConstantAndType(source, dst_type))
1034 << source << " " << dst_type;
Alexandre Rames3e69f162014-12-10 10:36:50 +00001035 UseScratchRegisterScope temps(GetVIXLAssembler());
1036 HConstant* src_cst = source.GetConstant();
1037 CPURegister temp;
Nicolas Geoffrayd6138ef2015-02-18 14:48:53 +00001038 if (src_cst->IsIntConstant() || src_cst->IsNullConstant()) {
Alexandre Rames3e69f162014-12-10 10:36:50 +00001039 temp = temps.AcquireW();
1040 } else if (src_cst->IsLongConstant()) {
1041 temp = temps.AcquireX();
1042 } else if (src_cst->IsFloatConstant()) {
1043 temp = temps.AcquireS();
1044 } else {
1045 DCHECK(src_cst->IsDoubleConstant());
1046 temp = temps.AcquireD();
1047 }
1048 MoveConstant(temp, src_cst);
Alexandre Rames67555f72014-11-18 10:55:16 +00001049 __ Str(temp, StackOperandFrom(destination));
Alexandre Ramesfc19de82014-11-07 17:13:31 +00001050 } else {
Alexandre Rames67555f72014-11-18 10:55:16 +00001051 DCHECK(source.IsStackSlot() || source.IsDoubleStackSlot());
Alexandre Rames3e69f162014-12-10 10:36:50 +00001052 DCHECK(source.IsDoubleStackSlot() == destination.IsDoubleStackSlot());
Alexandre Rames67555f72014-11-18 10:55:16 +00001053 UseScratchRegisterScope temps(GetVIXLAssembler());
Alexandre Rames3e69f162014-12-10 10:36:50 +00001054 // There is generally less pressure on FP registers.
1055 FPRegister temp = destination.IsDoubleStackSlot() ? temps.AcquireD() : temps.AcquireS();
Alexandre Ramesfc19de82014-11-07 17:13:31 +00001056 __ Ldr(temp, StackOperandFrom(source));
1057 __ Str(temp, StackOperandFrom(destination));
1058 }
1059 }
1060}
1061
1062void CodeGeneratorARM64::Load(Primitive::Type type,
Serban Constantinescu02d81cc2015-01-05 16:08:49 +00001063 CPURegister dst,
1064 const MemOperand& src) {
Alexandre Ramesfc19de82014-11-07 17:13:31 +00001065 switch (type) {
1066 case Primitive::kPrimBoolean:
Alexandre Rames67555f72014-11-18 10:55:16 +00001067 __ Ldrb(Register(dst), src);
Alexandre Ramesfc19de82014-11-07 17:13:31 +00001068 break;
1069 case Primitive::kPrimByte:
Alexandre Rames67555f72014-11-18 10:55:16 +00001070 __ Ldrsb(Register(dst), src);
Alexandre Ramesfc19de82014-11-07 17:13:31 +00001071 break;
1072 case Primitive::kPrimShort:
Alexandre Rames67555f72014-11-18 10:55:16 +00001073 __ Ldrsh(Register(dst), src);
Alexandre Ramesfc19de82014-11-07 17:13:31 +00001074 break;
1075 case Primitive::kPrimChar:
Alexandre Rames67555f72014-11-18 10:55:16 +00001076 __ Ldrh(Register(dst), src);
Alexandre Ramesfc19de82014-11-07 17:13:31 +00001077 break;
1078 case Primitive::kPrimInt:
1079 case Primitive::kPrimNot:
1080 case Primitive::kPrimLong:
Alexandre Ramesfc19de82014-11-07 17:13:31 +00001081 case Primitive::kPrimFloat:
1082 case Primitive::kPrimDouble:
Alexandre Rames542361f2015-01-29 16:57:31 +00001083 DCHECK_EQ(dst.Is64Bits(), Primitive::Is64BitType(type));
Alexandre Rames67555f72014-11-18 10:55:16 +00001084 __ Ldr(dst, src);
1085 break;
Alexandre Ramesfc19de82014-11-07 17:13:31 +00001086 case Primitive::kPrimVoid:
1087 LOG(FATAL) << "Unreachable type " << type;
1088 }
1089}
1090
Calin Juravle77520bc2015-01-12 18:45:46 +00001091void CodeGeneratorARM64::LoadAcquire(HInstruction* instruction,
Serban Constantinescu02d81cc2015-01-05 16:08:49 +00001092 CPURegister dst,
1093 const MemOperand& src) {
Alexandre Ramesd921d642015-04-16 15:07:16 +01001094 MacroAssembler* masm = GetVIXLAssembler();
1095 BlockPoolsScope block_pools(masm);
1096 UseScratchRegisterScope temps(masm);
Serban Constantinescu02d81cc2015-01-05 16:08:49 +00001097 Register temp_base = temps.AcquireX();
Calin Juravle77520bc2015-01-12 18:45:46 +00001098 Primitive::Type type = instruction->GetType();
Serban Constantinescu02d81cc2015-01-05 16:08:49 +00001099
Serban Constantinescu02d81cc2015-01-05 16:08:49 +00001100 DCHECK(!src.IsPreIndex());
1101 DCHECK(!src.IsPostIndex());
1102
1103 // TODO(vixl): Let the MacroAssembler handle MemOperand.
Andreas Gampe878d58c2015-01-15 23:24:00 -08001104 __ Add(temp_base, src.base(), OperandFromMemOperand(src));
Serban Constantinescu02d81cc2015-01-05 16:08:49 +00001105 MemOperand base = MemOperand(temp_base);
1106 switch (type) {
1107 case Primitive::kPrimBoolean:
1108 __ Ldarb(Register(dst), base);
Calin Juravle77520bc2015-01-12 18:45:46 +00001109 MaybeRecordImplicitNullCheck(instruction);
Serban Constantinescu02d81cc2015-01-05 16:08:49 +00001110 break;
1111 case Primitive::kPrimByte:
1112 __ Ldarb(Register(dst), base);
Calin Juravle77520bc2015-01-12 18:45:46 +00001113 MaybeRecordImplicitNullCheck(instruction);
Serban Constantinescu02d81cc2015-01-05 16:08:49 +00001114 __ Sbfx(Register(dst), Register(dst), 0, Primitive::ComponentSize(type) * kBitsPerByte);
1115 break;
1116 case Primitive::kPrimChar:
1117 __ Ldarh(Register(dst), base);
Calin Juravle77520bc2015-01-12 18:45:46 +00001118 MaybeRecordImplicitNullCheck(instruction);
Serban Constantinescu02d81cc2015-01-05 16:08:49 +00001119 break;
1120 case Primitive::kPrimShort:
1121 __ Ldarh(Register(dst), base);
Calin Juravle77520bc2015-01-12 18:45:46 +00001122 MaybeRecordImplicitNullCheck(instruction);
Serban Constantinescu02d81cc2015-01-05 16:08:49 +00001123 __ Sbfx(Register(dst), Register(dst), 0, Primitive::ComponentSize(type) * kBitsPerByte);
1124 break;
1125 case Primitive::kPrimInt:
1126 case Primitive::kPrimNot:
1127 case Primitive::kPrimLong:
Alexandre Rames542361f2015-01-29 16:57:31 +00001128 DCHECK_EQ(dst.Is64Bits(), Primitive::Is64BitType(type));
Serban Constantinescu02d81cc2015-01-05 16:08:49 +00001129 __ Ldar(Register(dst), base);
Calin Juravle77520bc2015-01-12 18:45:46 +00001130 MaybeRecordImplicitNullCheck(instruction);
Serban Constantinescu02d81cc2015-01-05 16:08:49 +00001131 break;
1132 case Primitive::kPrimFloat:
1133 case Primitive::kPrimDouble: {
1134 DCHECK(dst.IsFPRegister());
Alexandre Rames542361f2015-01-29 16:57:31 +00001135 DCHECK_EQ(dst.Is64Bits(), Primitive::Is64BitType(type));
Serban Constantinescu02d81cc2015-01-05 16:08:49 +00001136
1137 Register temp = dst.Is64Bits() ? temps.AcquireX() : temps.AcquireW();
1138 __ Ldar(temp, base);
Calin Juravle77520bc2015-01-12 18:45:46 +00001139 MaybeRecordImplicitNullCheck(instruction);
Serban Constantinescu02d81cc2015-01-05 16:08:49 +00001140 __ Fmov(FPRegister(dst), temp);
1141 break;
1142 }
1143 case Primitive::kPrimVoid:
1144 LOG(FATAL) << "Unreachable type " << type;
1145 }
1146}
1147
Alexandre Ramesfc19de82014-11-07 17:13:31 +00001148void CodeGeneratorARM64::Store(Primitive::Type type,
Serban Constantinescu02d81cc2015-01-05 16:08:49 +00001149 CPURegister src,
1150 const MemOperand& dst) {
Alexandre Ramesfc19de82014-11-07 17:13:31 +00001151 switch (type) {
1152 case Primitive::kPrimBoolean:
1153 case Primitive::kPrimByte:
Serban Constantinescu02d81cc2015-01-05 16:08:49 +00001154 __ Strb(Register(src), dst);
Alexandre Ramesfc19de82014-11-07 17:13:31 +00001155 break;
1156 case Primitive::kPrimChar:
1157 case Primitive::kPrimShort:
Serban Constantinescu02d81cc2015-01-05 16:08:49 +00001158 __ Strh(Register(src), dst);
Alexandre Ramesfc19de82014-11-07 17:13:31 +00001159 break;
1160 case Primitive::kPrimInt:
1161 case Primitive::kPrimNot:
1162 case Primitive::kPrimLong:
Alexandre Ramesfc19de82014-11-07 17:13:31 +00001163 case Primitive::kPrimFloat:
1164 case Primitive::kPrimDouble:
Alexandre Rames542361f2015-01-29 16:57:31 +00001165 DCHECK_EQ(src.Is64Bits(), Primitive::Is64BitType(type));
Serban Constantinescu02d81cc2015-01-05 16:08:49 +00001166 __ Str(src, dst);
Alexandre Rames67555f72014-11-18 10:55:16 +00001167 break;
Alexandre Ramesfc19de82014-11-07 17:13:31 +00001168 case Primitive::kPrimVoid:
1169 LOG(FATAL) << "Unreachable type " << type;
1170 }
1171}
1172
Serban Constantinescu02d81cc2015-01-05 16:08:49 +00001173void CodeGeneratorARM64::StoreRelease(Primitive::Type type,
1174 CPURegister src,
1175 const MemOperand& dst) {
1176 UseScratchRegisterScope temps(GetVIXLAssembler());
1177 Register temp_base = temps.AcquireX();
1178
Serban Constantinescu02d81cc2015-01-05 16:08:49 +00001179 DCHECK(!dst.IsPreIndex());
1180 DCHECK(!dst.IsPostIndex());
1181
1182 // TODO(vixl): Let the MacroAssembler handle this.
Andreas Gampe878d58c2015-01-15 23:24:00 -08001183 Operand op = OperandFromMemOperand(dst);
1184 __ Add(temp_base, dst.base(), op);
Serban Constantinescu02d81cc2015-01-05 16:08:49 +00001185 MemOperand base = MemOperand(temp_base);
1186 switch (type) {
1187 case Primitive::kPrimBoolean:
1188 case Primitive::kPrimByte:
1189 __ Stlrb(Register(src), base);
1190 break;
1191 case Primitive::kPrimChar:
1192 case Primitive::kPrimShort:
1193 __ Stlrh(Register(src), base);
1194 break;
1195 case Primitive::kPrimInt:
1196 case Primitive::kPrimNot:
1197 case Primitive::kPrimLong:
Alexandre Rames542361f2015-01-29 16:57:31 +00001198 DCHECK_EQ(src.Is64Bits(), Primitive::Is64BitType(type));
Serban Constantinescu02d81cc2015-01-05 16:08:49 +00001199 __ Stlr(Register(src), base);
1200 break;
1201 case Primitive::kPrimFloat:
1202 case Primitive::kPrimDouble: {
1203 DCHECK(src.IsFPRegister());
Alexandre Rames542361f2015-01-29 16:57:31 +00001204 DCHECK_EQ(src.Is64Bits(), Primitive::Is64BitType(type));
Serban Constantinescu02d81cc2015-01-05 16:08:49 +00001205
1206 Register temp = src.Is64Bits() ? temps.AcquireX() : temps.AcquireW();
1207 __ Fmov(temp, FPRegister(src));
1208 __ Stlr(temp, base);
1209 break;
1210 }
1211 case Primitive::kPrimVoid:
1212 LOG(FATAL) << "Unreachable type " << type;
1213 }
1214}
1215
Calin Juravle175dc732015-08-25 15:42:32 +01001216void CodeGeneratorARM64::InvokeRuntime(QuickEntrypointEnum entrypoint,
1217 HInstruction* instruction,
1218 uint32_t dex_pc,
1219 SlowPathCode* slow_path) {
1220 InvokeRuntime(GetThreadOffset<kArm64WordSize>(entrypoint).Int32Value(),
1221 instruction,
1222 dex_pc,
1223 slow_path);
1224}
1225
Alexandre Rames67555f72014-11-18 10:55:16 +00001226void CodeGeneratorARM64::InvokeRuntime(int32_t entry_point_offset,
1227 HInstruction* instruction,
Nicolas Geoffrayeeefa122015-03-13 18:52:59 +00001228 uint32_t dex_pc,
1229 SlowPathCode* slow_path) {
Alexandre Rames78e3ef62015-08-12 13:43:29 +01001230 ValidateInvokeRuntime(instruction, slow_path);
Alexandre Ramesd921d642015-04-16 15:07:16 +01001231 BlockPoolsScope block_pools(GetVIXLAssembler());
Alexandre Rames67555f72014-11-18 10:55:16 +00001232 __ Ldr(lr, MemOperand(tr, entry_point_offset));
1233 __ Blr(lr);
Roland Levillain896e32d2015-05-05 18:07:10 +01001234 RecordPcInfo(instruction, dex_pc, slow_path);
Alexandre Rames67555f72014-11-18 10:55:16 +00001235}
1236
1237void InstructionCodeGeneratorARM64::GenerateClassInitializationCheck(SlowPathCodeARM64* slow_path,
1238 vixl::Register class_reg) {
1239 UseScratchRegisterScope temps(GetVIXLAssembler());
1240 Register temp = temps.AcquireW();
Serban Constantinescu02d81cc2015-01-05 16:08:49 +00001241 size_t status_offset = mirror::Class::StatusOffset().SizeValue();
Serban Constantinescu579885a2015-02-22 20:51:33 +00001242 bool use_acquire_release = codegen_->GetInstructionSetFeatures().PreferAcquireRelease();
Serban Constantinescu02d81cc2015-01-05 16:08:49 +00001243
Serban Constantinescu02164b32014-11-13 14:05:07 +00001244 // Even if the initialized flag is set, we need to ensure consistent memory ordering.
Serban Constantinescu579885a2015-02-22 20:51:33 +00001245 if (use_acquire_release) {
Serban Constantinescu02d81cc2015-01-05 16:08:49 +00001246 // TODO(vixl): Let the MacroAssembler handle MemOperand.
1247 __ Add(temp, class_reg, status_offset);
1248 __ Ldar(temp, HeapOperand(temp));
1249 __ Cmp(temp, mirror::Class::kStatusInitialized);
1250 __ B(lt, slow_path->GetEntryLabel());
1251 } else {
1252 __ Ldr(temp, HeapOperand(class_reg, status_offset));
1253 __ Cmp(temp, mirror::Class::kStatusInitialized);
1254 __ B(lt, slow_path->GetEntryLabel());
1255 __ Dmb(InnerShareable, BarrierReads);
1256 }
Alexandre Rames67555f72014-11-18 10:55:16 +00001257 __ Bind(slow_path->GetExitLabel());
1258}
Alexandre Rames5319def2014-10-23 10:03:10 +01001259
Serban Constantinescu02d81cc2015-01-05 16:08:49 +00001260void InstructionCodeGeneratorARM64::GenerateMemoryBarrier(MemBarrierKind kind) {
1261 BarrierType type = BarrierAll;
1262
1263 switch (kind) {
1264 case MemBarrierKind::kAnyAny:
1265 case MemBarrierKind::kAnyStore: {
1266 type = BarrierAll;
1267 break;
1268 }
1269 case MemBarrierKind::kLoadAny: {
1270 type = BarrierReads;
1271 break;
1272 }
1273 case MemBarrierKind::kStoreStore: {
1274 type = BarrierWrites;
1275 break;
1276 }
1277 default:
1278 LOG(FATAL) << "Unexpected memory barrier " << kind;
1279 }
1280 __ Dmb(InnerShareable, type);
1281}
1282
Serban Constantinescu02164b32014-11-13 14:05:07 +00001283void InstructionCodeGeneratorARM64::GenerateSuspendCheck(HSuspendCheck* instruction,
1284 HBasicBlock* successor) {
1285 SuspendCheckSlowPathARM64* slow_path =
Nicolas Geoffraydb216f42015-05-05 17:02:20 +01001286 down_cast<SuspendCheckSlowPathARM64*>(instruction->GetSlowPath());
1287 if (slow_path == nullptr) {
1288 slow_path = new (GetGraph()->GetArena()) SuspendCheckSlowPathARM64(instruction, successor);
1289 instruction->SetSlowPath(slow_path);
1290 codegen_->AddSlowPath(slow_path);
1291 if (successor != nullptr) {
1292 DCHECK(successor->IsLoopHeader());
1293 codegen_->ClearSpillSlotsFromLoopPhisInStackMap(instruction);
1294 }
1295 } else {
1296 DCHECK_EQ(slow_path->GetSuccessor(), successor);
1297 }
1298
Serban Constantinescu02164b32014-11-13 14:05:07 +00001299 UseScratchRegisterScope temps(codegen_->GetVIXLAssembler());
1300 Register temp = temps.AcquireW();
1301
1302 __ Ldrh(temp, MemOperand(tr, Thread::ThreadFlagsOffset<kArm64WordSize>().SizeValue()));
1303 if (successor == nullptr) {
1304 __ Cbnz(temp, slow_path->GetEntryLabel());
1305 __ Bind(slow_path->GetReturnLabel());
1306 } else {
1307 __ Cbz(temp, codegen_->GetLabelOf(successor));
1308 __ B(slow_path->GetEntryLabel());
1309 // slow_path will return to GetLabelOf(successor).
1310 }
1311}
1312
Alexandre Rames5319def2014-10-23 10:03:10 +01001313InstructionCodeGeneratorARM64::InstructionCodeGeneratorARM64(HGraph* graph,
1314 CodeGeneratorARM64* codegen)
1315 : HGraphVisitor(graph),
1316 assembler_(codegen->GetAssembler()),
1317 codegen_(codegen) {}
1318
1319#define FOR_EACH_UNIMPLEMENTED_INSTRUCTION(M) \
Alexandre Rames3e69f162014-12-10 10:36:50 +00001320 /* No unimplemented IR. */
Alexandre Rames5319def2014-10-23 10:03:10 +01001321
1322#define UNIMPLEMENTED_INSTRUCTION_BREAK_CODE(name) name##UnimplementedInstructionBreakCode
1323
1324enum UnimplementedInstructionBreakCode {
Alexandre Rames67555f72014-11-18 10:55:16 +00001325 // Using a base helps identify when we hit such breakpoints.
1326 UnimplementedInstructionBreakCodeBaseCode = 0x900,
Alexandre Rames5319def2014-10-23 10:03:10 +01001327#define ENUM_UNIMPLEMENTED_INSTRUCTION(name) UNIMPLEMENTED_INSTRUCTION_BREAK_CODE(name),
1328 FOR_EACH_UNIMPLEMENTED_INSTRUCTION(ENUM_UNIMPLEMENTED_INSTRUCTION)
1329#undef ENUM_UNIMPLEMENTED_INSTRUCTION
1330};
1331
1332#define DEFINE_UNIMPLEMENTED_INSTRUCTION_VISITORS(name) \
Roland Levillain4b8f1ec2015-08-26 18:34:03 +01001333 void InstructionCodeGeneratorARM64::Visit##name(H##name* instr ATTRIBUTE_UNUSED) { \
Alexandre Rames5319def2014-10-23 10:03:10 +01001334 __ Brk(UNIMPLEMENTED_INSTRUCTION_BREAK_CODE(name)); \
1335 } \
1336 void LocationsBuilderARM64::Visit##name(H##name* instr) { \
1337 LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instr); \
1338 locations->SetOut(Location::Any()); \
1339 }
1340 FOR_EACH_UNIMPLEMENTED_INSTRUCTION(DEFINE_UNIMPLEMENTED_INSTRUCTION_VISITORS)
1341#undef DEFINE_UNIMPLEMENTED_INSTRUCTION_VISITORS
1342
1343#undef UNIMPLEMENTED_INSTRUCTION_BREAK_CODE
Alexandre Rames67555f72014-11-18 10:55:16 +00001344#undef FOR_EACH_UNIMPLEMENTED_INSTRUCTION
Alexandre Rames5319def2014-10-23 10:03:10 +01001345
Alexandre Rames67555f72014-11-18 10:55:16 +00001346void LocationsBuilderARM64::HandleBinaryOp(HBinaryOperation* instr) {
Alexandre Rames5319def2014-10-23 10:03:10 +01001347 DCHECK_EQ(instr->InputCount(), 2U);
1348 LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instr);
1349 Primitive::Type type = instr->GetResultType();
1350 switch (type) {
1351 case Primitive::kPrimInt:
Alexandre Ramesa89086e2014-11-07 17:13:25 +00001352 case Primitive::kPrimLong:
Alexandre Rames5319def2014-10-23 10:03:10 +01001353 locations->SetInAt(0, Location::RequiresRegister());
Serban Constantinescu2d35d9d2015-02-22 22:08:01 +00001354 locations->SetInAt(1, ARM64EncodableConstantOrRegister(instr->InputAt(1), instr));
Alexandre Ramesfb4e5fa2014-11-06 12:41:16 +00001355 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
Alexandre Rames5319def2014-10-23 10:03:10 +01001356 break;
Alexandre Ramesa89086e2014-11-07 17:13:25 +00001357
1358 case Primitive::kPrimFloat:
1359 case Primitive::kPrimDouble:
1360 locations->SetInAt(0, Location::RequiresFpuRegister());
1361 locations->SetInAt(1, Location::RequiresFpuRegister());
Alexandre Rames67555f72014-11-18 10:55:16 +00001362 locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap);
Alexandre Rames5319def2014-10-23 10:03:10 +01001363 break;
Alexandre Ramesa89086e2014-11-07 17:13:25 +00001364
Alexandre Rames5319def2014-10-23 10:03:10 +01001365 default:
Alexandre Ramesa89086e2014-11-07 17:13:25 +00001366 LOG(FATAL) << "Unexpected " << instr->DebugName() << " type " << type;
Alexandre Rames5319def2014-10-23 10:03:10 +01001367 }
1368}
1369
Alexandre Rames09a99962015-04-15 11:47:56 +01001370void LocationsBuilderARM64::HandleFieldGet(HInstruction* instruction) {
1371 LocationSummary* locations =
1372 new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall);
1373 locations->SetInAt(0, Location::RequiresRegister());
1374 if (Primitive::IsFloatingPointType(instruction->GetType())) {
1375 locations->SetOut(Location::RequiresFpuRegister());
1376 } else {
1377 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
1378 }
1379}
1380
1381void InstructionCodeGeneratorARM64::HandleFieldGet(HInstruction* instruction,
1382 const FieldInfo& field_info) {
1383 DCHECK(instruction->IsInstanceFieldGet() || instruction->IsStaticFieldGet());
Roland Levillain4d027112015-07-01 15:41:14 +01001384 Primitive::Type field_type = field_info.GetFieldType();
Alexandre Ramesd921d642015-04-16 15:07:16 +01001385 BlockPoolsScope block_pools(GetVIXLAssembler());
Alexandre Rames09a99962015-04-15 11:47:56 +01001386
1387 MemOperand field = HeapOperand(InputRegisterAt(instruction, 0), field_info.GetFieldOffset());
1388 bool use_acquire_release = codegen_->GetInstructionSetFeatures().PreferAcquireRelease();
1389
1390 if (field_info.IsVolatile()) {
1391 if (use_acquire_release) {
1392 // NB: LoadAcquire will record the pc info if needed.
1393 codegen_->LoadAcquire(instruction, OutputCPURegister(instruction), field);
1394 } else {
Roland Levillain4d027112015-07-01 15:41:14 +01001395 codegen_->Load(field_type, OutputCPURegister(instruction), field);
Alexandre Rames09a99962015-04-15 11:47:56 +01001396 codegen_->MaybeRecordImplicitNullCheck(instruction);
1397 // For IRIW sequential consistency kLoadAny is not sufficient.
1398 GenerateMemoryBarrier(MemBarrierKind::kAnyAny);
1399 }
1400 } else {
Roland Levillain4d027112015-07-01 15:41:14 +01001401 codegen_->Load(field_type, OutputCPURegister(instruction), field);
Alexandre Rames09a99962015-04-15 11:47:56 +01001402 codegen_->MaybeRecordImplicitNullCheck(instruction);
1403 }
Roland Levillain4d027112015-07-01 15:41:14 +01001404
1405 if (field_type == Primitive::kPrimNot) {
1406 GetAssembler()->MaybeUnpoisonHeapReference(OutputCPURegister(instruction).W());
1407 }
Alexandre Rames09a99962015-04-15 11:47:56 +01001408}
1409
1410void LocationsBuilderARM64::HandleFieldSet(HInstruction* instruction) {
1411 LocationSummary* locations =
1412 new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall);
1413 locations->SetInAt(0, Location::RequiresRegister());
1414 if (Primitive::IsFloatingPointType(instruction->InputAt(1)->GetType())) {
1415 locations->SetInAt(1, Location::RequiresFpuRegister());
1416 } else {
1417 locations->SetInAt(1, Location::RequiresRegister());
1418 }
1419}
1420
1421void InstructionCodeGeneratorARM64::HandleFieldSet(HInstruction* instruction,
Nicolas Geoffray07276db2015-05-18 14:22:09 +01001422 const FieldInfo& field_info,
1423 bool value_can_be_null) {
Alexandre Rames09a99962015-04-15 11:47:56 +01001424 DCHECK(instruction->IsInstanceFieldSet() || instruction->IsStaticFieldSet());
Alexandre Ramesd921d642015-04-16 15:07:16 +01001425 BlockPoolsScope block_pools(GetVIXLAssembler());
Alexandre Rames09a99962015-04-15 11:47:56 +01001426
1427 Register obj = InputRegisterAt(instruction, 0);
1428 CPURegister value = InputCPURegisterAt(instruction, 1);
Roland Levillain4d027112015-07-01 15:41:14 +01001429 CPURegister source = value;
Alexandre Rames09a99962015-04-15 11:47:56 +01001430 Offset offset = field_info.GetFieldOffset();
1431 Primitive::Type field_type = field_info.GetFieldType();
1432 bool use_acquire_release = codegen_->GetInstructionSetFeatures().PreferAcquireRelease();
1433
Roland Levillain4d027112015-07-01 15:41:14 +01001434 {
1435 // We use a block to end the scratch scope before the write barrier, thus
1436 // freeing the temporary registers so they can be used in `MarkGCCard`.
1437 UseScratchRegisterScope temps(GetVIXLAssembler());
1438
1439 if (kPoisonHeapReferences && field_type == Primitive::kPrimNot) {
1440 DCHECK(value.IsW());
1441 Register temp = temps.AcquireW();
1442 __ Mov(temp, value.W());
1443 GetAssembler()->PoisonHeapReference(temp.W());
1444 source = temp;
Alexandre Rames09a99962015-04-15 11:47:56 +01001445 }
Roland Levillain4d027112015-07-01 15:41:14 +01001446
1447 if (field_info.IsVolatile()) {
1448 if (use_acquire_release) {
1449 codegen_->StoreRelease(field_type, source, HeapOperand(obj, offset));
1450 codegen_->MaybeRecordImplicitNullCheck(instruction);
1451 } else {
1452 GenerateMemoryBarrier(MemBarrierKind::kAnyStore);
1453 codegen_->Store(field_type, source, HeapOperand(obj, offset));
1454 codegen_->MaybeRecordImplicitNullCheck(instruction);
1455 GenerateMemoryBarrier(MemBarrierKind::kAnyAny);
1456 }
1457 } else {
1458 codegen_->Store(field_type, source, HeapOperand(obj, offset));
1459 codegen_->MaybeRecordImplicitNullCheck(instruction);
1460 }
Alexandre Rames09a99962015-04-15 11:47:56 +01001461 }
1462
1463 if (CodeGenerator::StoreNeedsWriteBarrier(field_type, instruction->InputAt(1))) {
Nicolas Geoffray07276db2015-05-18 14:22:09 +01001464 codegen_->MarkGCCard(obj, Register(value), value_can_be_null);
Alexandre Rames09a99962015-04-15 11:47:56 +01001465 }
1466}
1467
Alexandre Rames67555f72014-11-18 10:55:16 +00001468void InstructionCodeGeneratorARM64::HandleBinaryOp(HBinaryOperation* instr) {
Alexandre Rames5319def2014-10-23 10:03:10 +01001469 Primitive::Type type = instr->GetType();
Alexandre Rames5319def2014-10-23 10:03:10 +01001470
1471 switch (type) {
1472 case Primitive::kPrimInt:
Alexandre Ramesa89086e2014-11-07 17:13:25 +00001473 case Primitive::kPrimLong: {
1474 Register dst = OutputRegister(instr);
1475 Register lhs = InputRegisterAt(instr, 0);
1476 Operand rhs = InputOperandAt(instr, 1);
Alexandre Rames5319def2014-10-23 10:03:10 +01001477 if (instr->IsAdd()) {
1478 __ Add(dst, lhs, rhs);
Alexandre Rames67555f72014-11-18 10:55:16 +00001479 } else if (instr->IsAnd()) {
1480 __ And(dst, lhs, rhs);
1481 } else if (instr->IsOr()) {
1482 __ Orr(dst, lhs, rhs);
1483 } else if (instr->IsSub()) {
Alexandre Rames5319def2014-10-23 10:03:10 +01001484 __ Sub(dst, lhs, rhs);
Alexandre Rames67555f72014-11-18 10:55:16 +00001485 } else {
1486 DCHECK(instr->IsXor());
1487 __ Eor(dst, lhs, rhs);
Alexandre Rames5319def2014-10-23 10:03:10 +01001488 }
1489 break;
Alexandre Ramesa89086e2014-11-07 17:13:25 +00001490 }
1491 case Primitive::kPrimFloat:
1492 case Primitive::kPrimDouble: {
1493 FPRegister dst = OutputFPRegister(instr);
1494 FPRegister lhs = InputFPRegisterAt(instr, 0);
1495 FPRegister rhs = InputFPRegisterAt(instr, 1);
1496 if (instr->IsAdd()) {
1497 __ Fadd(dst, lhs, rhs);
Alexandre Rames67555f72014-11-18 10:55:16 +00001498 } else if (instr->IsSub()) {
Alexandre Ramesa89086e2014-11-07 17:13:25 +00001499 __ Fsub(dst, lhs, rhs);
Alexandre Rames67555f72014-11-18 10:55:16 +00001500 } else {
1501 LOG(FATAL) << "Unexpected floating-point binary operation";
Alexandre Ramesa89086e2014-11-07 17:13:25 +00001502 }
Alexandre Rames5319def2014-10-23 10:03:10 +01001503 break;
Alexandre Ramesa89086e2014-11-07 17:13:25 +00001504 }
Alexandre Rames5319def2014-10-23 10:03:10 +01001505 default:
Alexandre Rames67555f72014-11-18 10:55:16 +00001506 LOG(FATAL) << "Unexpected binary operation type " << type;
Alexandre Rames5319def2014-10-23 10:03:10 +01001507 }
1508}
1509
Serban Constantinescu02164b32014-11-13 14:05:07 +00001510void LocationsBuilderARM64::HandleShift(HBinaryOperation* instr) {
1511 DCHECK(instr->IsShl() || instr->IsShr() || instr->IsUShr());
1512
1513 LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instr);
1514 Primitive::Type type = instr->GetResultType();
1515 switch (type) {
1516 case Primitive::kPrimInt:
1517 case Primitive::kPrimLong: {
1518 locations->SetInAt(0, Location::RequiresRegister());
1519 locations->SetInAt(1, Location::RegisterOrConstant(instr->InputAt(1)));
1520 locations->SetOut(Location::RequiresRegister());
1521 break;
1522 }
1523 default:
1524 LOG(FATAL) << "Unexpected shift type " << type;
1525 }
1526}
1527
1528void InstructionCodeGeneratorARM64::HandleShift(HBinaryOperation* instr) {
1529 DCHECK(instr->IsShl() || instr->IsShr() || instr->IsUShr());
1530
1531 Primitive::Type type = instr->GetType();
1532 switch (type) {
1533 case Primitive::kPrimInt:
1534 case Primitive::kPrimLong: {
1535 Register dst = OutputRegister(instr);
1536 Register lhs = InputRegisterAt(instr, 0);
1537 Operand rhs = InputOperandAt(instr, 1);
1538 if (rhs.IsImmediate()) {
1539 uint32_t shift_value = (type == Primitive::kPrimInt)
1540 ? static_cast<uint32_t>(rhs.immediate() & kMaxIntShiftValue)
1541 : static_cast<uint32_t>(rhs.immediate() & kMaxLongShiftValue);
1542 if (instr->IsShl()) {
1543 __ Lsl(dst, lhs, shift_value);
1544 } else if (instr->IsShr()) {
1545 __ Asr(dst, lhs, shift_value);
1546 } else {
1547 __ Lsr(dst, lhs, shift_value);
1548 }
1549 } else {
1550 Register rhs_reg = dst.IsX() ? rhs.reg().X() : rhs.reg().W();
1551
1552 if (instr->IsShl()) {
1553 __ Lsl(dst, lhs, rhs_reg);
1554 } else if (instr->IsShr()) {
1555 __ Asr(dst, lhs, rhs_reg);
1556 } else {
1557 __ Lsr(dst, lhs, rhs_reg);
1558 }
1559 }
1560 break;
1561 }
1562 default:
1563 LOG(FATAL) << "Unexpected shift operation type " << type;
1564 }
1565}
1566
Alexandre Rames5319def2014-10-23 10:03:10 +01001567void LocationsBuilderARM64::VisitAdd(HAdd* instruction) {
Alexandre Rames67555f72014-11-18 10:55:16 +00001568 HandleBinaryOp(instruction);
Alexandre Rames5319def2014-10-23 10:03:10 +01001569}
1570
1571void InstructionCodeGeneratorARM64::VisitAdd(HAdd* instruction) {
Alexandre Rames67555f72014-11-18 10:55:16 +00001572 HandleBinaryOp(instruction);
1573}
1574
1575void LocationsBuilderARM64::VisitAnd(HAnd* instruction) {
1576 HandleBinaryOp(instruction);
1577}
1578
1579void InstructionCodeGeneratorARM64::VisitAnd(HAnd* instruction) {
1580 HandleBinaryOp(instruction);
Alexandre Rames5319def2014-10-23 10:03:10 +01001581}
1582
Alexandre Ramese6dbf482015-10-19 10:10:41 +01001583void LocationsBuilderARM64::VisitArm64IntermediateAddress(HArm64IntermediateAddress* instruction) {
1584 LocationSummary* locations =
1585 new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall);
1586 locations->SetInAt(0, Location::RequiresRegister());
1587 locations->SetInAt(1, ARM64EncodableConstantOrRegister(instruction->GetOffset(), instruction));
1588 locations->SetOut(Location::RequiresRegister());
1589}
1590
1591void InstructionCodeGeneratorARM64::VisitArm64IntermediateAddress(
1592 HArm64IntermediateAddress* instruction) {
1593 __ Add(OutputRegister(instruction),
1594 InputRegisterAt(instruction, 0),
1595 Operand(InputOperandAt(instruction, 1)));
1596}
1597
Alexandre Ramesfc19de82014-11-07 17:13:31 +00001598void LocationsBuilderARM64::VisitArrayGet(HArrayGet* instruction) {
1599 LocationSummary* locations =
1600 new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall);
1601 locations->SetInAt(0, Location::RequiresRegister());
1602 locations->SetInAt(1, Location::RegisterOrConstant(instruction->InputAt(1)));
Alexandre Rames88c13cd2015-04-14 17:35:39 +01001603 if (Primitive::IsFloatingPointType(instruction->GetType())) {
1604 locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap);
1605 } else {
1606 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
1607 }
Alexandre Ramesfc19de82014-11-07 17:13:31 +00001608}
1609
1610void InstructionCodeGeneratorARM64::VisitArrayGet(HArrayGet* instruction) {
Alexandre Ramesfc19de82014-11-07 17:13:31 +00001611 Primitive::Type type = instruction->GetType();
1612 Register obj = InputRegisterAt(instruction, 0);
Alexandre Ramese6dbf482015-10-19 10:10:41 +01001613 Location index = instruction->GetLocations()->InAt(1);
Alexandre Ramesfc19de82014-11-07 17:13:31 +00001614 size_t offset = mirror::Array::DataOffset(Primitive::ComponentSize(type)).Uint32Value();
Serban Constantinescu02164b32014-11-13 14:05:07 +00001615 MemOperand source = HeapOperand(obj);
Alexandre Ramese6dbf482015-10-19 10:10:41 +01001616 CPURegister dest = OutputCPURegister(instruction);
1617
Alexandre Ramesd921d642015-04-16 15:07:16 +01001618 MacroAssembler* masm = GetVIXLAssembler();
1619 UseScratchRegisterScope temps(masm);
Alexandre Ramese6dbf482015-10-19 10:10:41 +01001620 // Block pools between `Load` and `MaybeRecordImplicitNullCheck`.
Alexandre Ramesd921d642015-04-16 15:07:16 +01001621 BlockPoolsScope block_pools(masm);
Alexandre Ramesfc19de82014-11-07 17:13:31 +00001622
1623 if (index.IsConstant()) {
1624 offset += Int64ConstantFrom(index) << Primitive::ComponentSizeShift(type);
Serban Constantinescu02164b32014-11-13 14:05:07 +00001625 source = HeapOperand(obj, offset);
Alexandre Ramesfc19de82014-11-07 17:13:31 +00001626 } else {
1627 Register temp = temps.AcquireSameSizeAs(obj);
Alexandre Ramese6dbf482015-10-19 10:10:41 +01001628 if (instruction->GetArray()->IsArm64IntermediateAddress()) {
1629 // We do not need to compute the intermediate address from the array: the
1630 // input instruction has done it already. See the comment in
1631 // `InstructionSimplifierArm64::TryExtractArrayAccessAddress()`.
1632 if (kIsDebugBuild) {
1633 HArm64IntermediateAddress* tmp = instruction->GetArray()->AsArm64IntermediateAddress();
1634 DCHECK(tmp->GetOffset()->AsIntConstant()->GetValueAsUint64() == offset);
1635 }
1636 temp = obj;
1637 } else {
1638 __ Add(temp, obj, offset);
1639 }
Alexandre Rames82000b02015-07-07 11:34:16 +01001640 source = HeapOperand(temp, XRegisterFrom(index), LSL, Primitive::ComponentSizeShift(type));
Alexandre Ramesfc19de82014-11-07 17:13:31 +00001641 }
1642
Alexandre Ramese6dbf482015-10-19 10:10:41 +01001643 codegen_->Load(type, dest, source);
Calin Juravle77520bc2015-01-12 18:45:46 +00001644 codegen_->MaybeRecordImplicitNullCheck(instruction);
Roland Levillain4d027112015-07-01 15:41:14 +01001645
Alexandre Ramese6dbf482015-10-19 10:10:41 +01001646 if (instruction->GetType() == Primitive::kPrimNot) {
1647 GetAssembler()->MaybeUnpoisonHeapReference(dest.W());
Roland Levillain4d027112015-07-01 15:41:14 +01001648 }
Alexandre Ramesfc19de82014-11-07 17:13:31 +00001649}
1650
Alexandre Rames5319def2014-10-23 10:03:10 +01001651void LocationsBuilderARM64::VisitArrayLength(HArrayLength* instruction) {
1652 LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instruction);
1653 locations->SetInAt(0, Location::RequiresRegister());
Alexandre Ramesfb4e5fa2014-11-06 12:41:16 +00001654 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
Alexandre Rames5319def2014-10-23 10:03:10 +01001655}
1656
1657void InstructionCodeGeneratorARM64::VisitArrayLength(HArrayLength* instruction) {
Alexandre Ramesd921d642015-04-16 15:07:16 +01001658 BlockPoolsScope block_pools(GetVIXLAssembler());
Alexandre Rames5319def2014-10-23 10:03:10 +01001659 __ Ldr(OutputRegister(instruction),
1660 HeapOperand(InputRegisterAt(instruction, 0), mirror::Array::LengthOffset()));
Calin Juravle77520bc2015-01-12 18:45:46 +00001661 codegen_->MaybeRecordImplicitNullCheck(instruction);
Alexandre Rames5319def2014-10-23 10:03:10 +01001662}
1663
Alexandre Ramesfc19de82014-11-07 17:13:31 +00001664void LocationsBuilderARM64::VisitArraySet(HArraySet* instruction) {
Nicolas Geoffraye0395dd2015-09-25 11:04:45 +01001665 LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(
1666 instruction,
1667 instruction->NeedsTypeCheck() ? LocationSummary::kCallOnSlowPath : LocationSummary::kNoCall);
1668 locations->SetInAt(0, Location::RequiresRegister());
1669 locations->SetInAt(1, Location::RegisterOrConstant(instruction->InputAt(1)));
1670 if (Primitive::IsFloatingPointType(instruction->InputAt(2)->GetType())) {
1671 locations->SetInAt(2, Location::RequiresFpuRegister());
Alexandre Ramesfc19de82014-11-07 17:13:31 +00001672 } else {
Nicolas Geoffraye0395dd2015-09-25 11:04:45 +01001673 locations->SetInAt(2, Location::RequiresRegister());
Alexandre Ramesfc19de82014-11-07 17:13:31 +00001674 }
1675}
1676
1677void InstructionCodeGeneratorARM64::VisitArraySet(HArraySet* instruction) {
1678 Primitive::Type value_type = instruction->GetComponentType();
Alexandre Rames97833a02015-04-16 15:07:12 +01001679 LocationSummary* locations = instruction->GetLocations();
Nicolas Geoffraye0395dd2015-09-25 11:04:45 +01001680 bool may_need_runtime_call = locations->CanCall();
1681 bool needs_write_barrier =
1682 CodeGenerator::StoreNeedsWriteBarrier(value_type, instruction->GetValue());
Alexandre Rames97833a02015-04-16 15:07:12 +01001683
Nicolas Geoffraye0395dd2015-09-25 11:04:45 +01001684 Register array = InputRegisterAt(instruction, 0);
1685 CPURegister value = InputCPURegisterAt(instruction, 2);
1686 CPURegister source = value;
1687 Location index = locations->InAt(1);
1688 size_t offset = mirror::Array::DataOffset(Primitive::ComponentSize(value_type)).Uint32Value();
1689 MemOperand destination = HeapOperand(array);
1690 MacroAssembler* masm = GetVIXLAssembler();
1691 BlockPoolsScope block_pools(masm);
1692
1693 if (!needs_write_barrier) {
1694 DCHECK(!may_need_runtime_call);
1695 if (index.IsConstant()) {
1696 offset += Int64ConstantFrom(index) << Primitive::ComponentSizeShift(value_type);
1697 destination = HeapOperand(array, offset);
1698 } else {
1699 UseScratchRegisterScope temps(masm);
1700 Register temp = temps.AcquireSameSizeAs(array);
Alexandre Ramese6dbf482015-10-19 10:10:41 +01001701 if (instruction->GetArray()->IsArm64IntermediateAddress()) {
1702 // We do not need to compute the intermediate address from the array: the
1703 // input instruction has done it already. See the comment in
1704 // `InstructionSimplifierArm64::TryExtractArrayAccessAddress()`.
1705 if (kIsDebugBuild) {
1706 HArm64IntermediateAddress* tmp = instruction->GetArray()->AsArm64IntermediateAddress();
1707 DCHECK(tmp->GetOffset()->AsIntConstant()->GetValueAsUint64() == offset);
1708 }
1709 temp = array;
1710 } else {
1711 __ Add(temp, array, offset);
1712 }
Nicolas Geoffraye0395dd2015-09-25 11:04:45 +01001713 destination = HeapOperand(temp,
1714 XRegisterFrom(index),
1715 LSL,
1716 Primitive::ComponentSizeShift(value_type));
1717 }
1718 codegen_->Store(value_type, value, destination);
1719 codegen_->MaybeRecordImplicitNullCheck(instruction);
Alexandre Ramesfc19de82014-11-07 17:13:31 +00001720 } else {
Nicolas Geoffraye0395dd2015-09-25 11:04:45 +01001721 DCHECK(needs_write_barrier);
Alexandre Ramese6dbf482015-10-19 10:10:41 +01001722 DCHECK(!instruction->GetArray()->IsArm64IntermediateAddress());
Nicolas Geoffraye0395dd2015-09-25 11:04:45 +01001723 vixl::Label done;
1724 SlowPathCodeARM64* slow_path = nullptr;
Alexandre Rames97833a02015-04-16 15:07:12 +01001725 {
1726 // We use a block to end the scratch scope before the write barrier, thus
1727 // freeing the temporary registers so they can be used in `MarkGCCard`.
1728 UseScratchRegisterScope temps(masm);
Nicolas Geoffraye0395dd2015-09-25 11:04:45 +01001729 Register temp = temps.AcquireSameSizeAs(array);
Alexandre Rames97833a02015-04-16 15:07:12 +01001730 if (index.IsConstant()) {
1731 offset += Int64ConstantFrom(index) << Primitive::ComponentSizeShift(value_type);
Nicolas Geoffraye0395dd2015-09-25 11:04:45 +01001732 destination = HeapOperand(array, offset);
Alexandre Rames97833a02015-04-16 15:07:12 +01001733 } else {
Alexandre Rames82000b02015-07-07 11:34:16 +01001734 destination = HeapOperand(temp,
1735 XRegisterFrom(index),
1736 LSL,
1737 Primitive::ComponentSizeShift(value_type));
Alexandre Rames97833a02015-04-16 15:07:12 +01001738 }
1739
Nicolas Geoffraye0395dd2015-09-25 11:04:45 +01001740 uint32_t class_offset = mirror::Object::ClassOffset().Int32Value();
1741 uint32_t super_offset = mirror::Class::SuperClassOffset().Int32Value();
1742 uint32_t component_offset = mirror::Class::ComponentTypeOffset().Int32Value();
1743
1744 if (may_need_runtime_call) {
1745 slow_path = new (GetGraph()->GetArena()) ArraySetSlowPathARM64(instruction);
1746 codegen_->AddSlowPath(slow_path);
1747 if (instruction->GetValueCanBeNull()) {
1748 vixl::Label non_zero;
1749 __ Cbnz(Register(value), &non_zero);
1750 if (!index.IsConstant()) {
1751 __ Add(temp, array, offset);
1752 }
1753 __ Str(wzr, destination);
1754 codegen_->MaybeRecordImplicitNullCheck(instruction);
1755 __ B(&done);
1756 __ Bind(&non_zero);
1757 }
1758
1759 Register temp2 = temps.AcquireSameSizeAs(array);
1760 __ Ldr(temp, HeapOperand(array, class_offset));
1761 codegen_->MaybeRecordImplicitNullCheck(instruction);
1762 GetAssembler()->MaybeUnpoisonHeapReference(temp);
1763 __ Ldr(temp, HeapOperand(temp, component_offset));
1764 __ Ldr(temp2, HeapOperand(Register(value), class_offset));
1765 // No need to poison/unpoison, we're comparing two poisoned references.
1766 __ Cmp(temp, temp2);
1767 if (instruction->StaticTypeOfArrayIsObjectArray()) {
1768 vixl::Label do_put;
1769 __ B(eq, &do_put);
1770 GetAssembler()->MaybeUnpoisonHeapReference(temp);
1771 __ Ldr(temp, HeapOperand(temp, super_offset));
1772 // No need to unpoison, we're comparing against null.
1773 __ Cbnz(temp, slow_path->GetEntryLabel());
1774 __ Bind(&do_put);
1775 } else {
1776 __ B(ne, slow_path->GetEntryLabel());
1777 }
Nicolas Geoffraya8a0fe22015-10-01 15:50:27 +01001778 temps.Release(temp2);
Nicolas Geoffraye0395dd2015-09-25 11:04:45 +01001779 }
1780
1781 if (kPoisonHeapReferences) {
Nicolas Geoffraya8a0fe22015-10-01 15:50:27 +01001782 Register temp2 = temps.AcquireSameSizeAs(array);
Nicolas Geoffraye0395dd2015-09-25 11:04:45 +01001783 DCHECK(value.IsW());
Nicolas Geoffraya8a0fe22015-10-01 15:50:27 +01001784 __ Mov(temp2, value.W());
1785 GetAssembler()->PoisonHeapReference(temp2);
1786 source = temp2;
Nicolas Geoffraye0395dd2015-09-25 11:04:45 +01001787 }
1788
1789 if (!index.IsConstant()) {
1790 __ Add(temp, array, offset);
1791 }
Nicolas Geoffray61b1dbe2015-10-01 10:27:52 +01001792 __ Str(source, destination);
Nicolas Geoffraye0395dd2015-09-25 11:04:45 +01001793
1794 if (!may_need_runtime_call) {
1795 codegen_->MaybeRecordImplicitNullCheck(instruction);
1796 }
Alexandre Ramesfc19de82014-11-07 17:13:31 +00001797 }
Nicolas Geoffraye0395dd2015-09-25 11:04:45 +01001798
1799 codegen_->MarkGCCard(array, value.W(), instruction->GetValueCanBeNull());
1800
1801 if (done.IsLinked()) {
1802 __ Bind(&done);
1803 }
1804
1805 if (slow_path != nullptr) {
1806 __ Bind(slow_path->GetExitLabel());
Alexandre Rames97833a02015-04-16 15:07:12 +01001807 }
Alexandre Ramesfc19de82014-11-07 17:13:31 +00001808 }
1809}
1810
Alexandre Rames67555f72014-11-18 10:55:16 +00001811void LocationsBuilderARM64::VisitBoundsCheck(HBoundsCheck* instruction) {
David Brazdil77a48ae2015-09-15 12:34:04 +00001812 LocationSummary::CallKind call_kind = instruction->CanThrowIntoCatchBlock()
1813 ? LocationSummary::kCallOnSlowPath
1814 : LocationSummary::kNoCall;
1815 LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instruction, call_kind);
Alexandre Rames67555f72014-11-18 10:55:16 +00001816 locations->SetInAt(0, Location::RequiresRegister());
Serban Constantinescu760d8ef2015-03-28 18:09:56 +00001817 locations->SetInAt(1, ARM64EncodableConstantOrRegister(instruction->InputAt(1), instruction));
Alexandre Rames67555f72014-11-18 10:55:16 +00001818 if (instruction->HasUses()) {
1819 locations->SetOut(Location::SameAsFirstInput());
1820 }
1821}
1822
1823void InstructionCodeGeneratorARM64::VisitBoundsCheck(HBoundsCheck* instruction) {
Serban Constantinescu5a6cc492015-08-13 15:20:25 +01001824 BoundsCheckSlowPathARM64* slow_path =
1825 new (GetGraph()->GetArena()) BoundsCheckSlowPathARM64(instruction);
Alexandre Rames67555f72014-11-18 10:55:16 +00001826 codegen_->AddSlowPath(slow_path);
1827
1828 __ Cmp(InputRegisterAt(instruction, 0), InputOperandAt(instruction, 1));
1829 __ B(slow_path->GetEntryLabel(), hs);
1830}
1831
Alexandre Rames67555f72014-11-18 10:55:16 +00001832void LocationsBuilderARM64::VisitClinitCheck(HClinitCheck* check) {
1833 LocationSummary* locations =
1834 new (GetGraph()->GetArena()) LocationSummary(check, LocationSummary::kCallOnSlowPath);
1835 locations->SetInAt(0, Location::RequiresRegister());
1836 if (check->HasUses()) {
1837 locations->SetOut(Location::SameAsFirstInput());
1838 }
1839}
1840
1841void InstructionCodeGeneratorARM64::VisitClinitCheck(HClinitCheck* check) {
1842 // We assume the class is not null.
1843 SlowPathCodeARM64* slow_path = new (GetGraph()->GetArena()) LoadClassSlowPathARM64(
1844 check->GetLoadClass(), check, check->GetDexPc(), true);
1845 codegen_->AddSlowPath(slow_path);
1846 GenerateClassInitializationCheck(slow_path, InputRegisterAt(check, 0));
1847}
1848
Roland Levillain7f63c522015-07-13 15:54:55 +00001849static bool IsFloatingPointZeroConstant(HInstruction* instruction) {
1850 return (instruction->IsFloatConstant() && (instruction->AsFloatConstant()->GetValue() == 0.0f))
1851 || (instruction->IsDoubleConstant() && (instruction->AsDoubleConstant()->GetValue() == 0.0));
1852}
1853
Serban Constantinescu02164b32014-11-13 14:05:07 +00001854void LocationsBuilderARM64::VisitCompare(HCompare* compare) {
Alexandre Rames5319def2014-10-23 10:03:10 +01001855 LocationSummary* locations =
Serban Constantinescu02164b32014-11-13 14:05:07 +00001856 new (GetGraph()->GetArena()) LocationSummary(compare, LocationSummary::kNoCall);
1857 Primitive::Type in_type = compare->InputAt(0)->GetType();
Alexandre Rames5319def2014-10-23 10:03:10 +01001858 switch (in_type) {
1859 case Primitive::kPrimLong: {
Serban Constantinescu02164b32014-11-13 14:05:07 +00001860 locations->SetInAt(0, Location::RequiresRegister());
Serban Constantinescu2d35d9d2015-02-22 22:08:01 +00001861 locations->SetInAt(1, ARM64EncodableConstantOrRegister(compare->InputAt(1), compare));
Serban Constantinescu02164b32014-11-13 14:05:07 +00001862 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
1863 break;
1864 }
1865 case Primitive::kPrimFloat:
1866 case Primitive::kPrimDouble: {
1867 locations->SetInAt(0, Location::RequiresFpuRegister());
Roland Levillain7f63c522015-07-13 15:54:55 +00001868 locations->SetInAt(1,
1869 IsFloatingPointZeroConstant(compare->InputAt(1))
1870 ? Location::ConstantLocation(compare->InputAt(1)->AsConstant())
1871 : Location::RequiresFpuRegister());
Serban Constantinescu02164b32014-11-13 14:05:07 +00001872 locations->SetOut(Location::RequiresRegister());
1873 break;
1874 }
1875 default:
1876 LOG(FATAL) << "Unexpected type for compare operation " << in_type;
1877 }
1878}
1879
1880void InstructionCodeGeneratorARM64::VisitCompare(HCompare* compare) {
1881 Primitive::Type in_type = compare->InputAt(0)->GetType();
1882
1883 // 0 if: left == right
1884 // 1 if: left > right
1885 // -1 if: left < right
1886 switch (in_type) {
1887 case Primitive::kPrimLong: {
1888 Register result = OutputRegister(compare);
1889 Register left = InputRegisterAt(compare, 0);
1890 Operand right = InputOperandAt(compare, 1);
1891
1892 __ Cmp(left, right);
1893 __ Cset(result, ne);
1894 __ Cneg(result, result, lt);
1895 break;
1896 }
1897 case Primitive::kPrimFloat:
1898 case Primitive::kPrimDouble: {
1899 Register result = OutputRegister(compare);
1900 FPRegister left = InputFPRegisterAt(compare, 0);
Alexandre Rames93415462015-02-17 15:08:20 +00001901 if (compare->GetLocations()->InAt(1).IsConstant()) {
Roland Levillain7f63c522015-07-13 15:54:55 +00001902 DCHECK(IsFloatingPointZeroConstant(compare->GetLocations()->InAt(1).GetConstant()));
1903 // 0.0 is the only immediate that can be encoded directly in an FCMP instruction.
Alexandre Rames93415462015-02-17 15:08:20 +00001904 __ Fcmp(left, 0.0);
1905 } else {
1906 __ Fcmp(left, InputFPRegisterAt(compare, 1));
1907 }
Serban Constantinescu02164b32014-11-13 14:05:07 +00001908 if (compare->IsGtBias()) {
1909 __ Cset(result, ne);
1910 } else {
1911 __ Csetm(result, ne);
1912 }
1913 __ Cneg(result, result, compare->IsGtBias() ? mi : gt);
Alexandre Rames5319def2014-10-23 10:03:10 +01001914 break;
1915 }
1916 default:
1917 LOG(FATAL) << "Unimplemented compare type " << in_type;
1918 }
1919}
1920
1921void LocationsBuilderARM64::VisitCondition(HCondition* instruction) {
1922 LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instruction);
Roland Levillain7f63c522015-07-13 15:54:55 +00001923
1924 if (Primitive::IsFloatingPointType(instruction->InputAt(0)->GetType())) {
1925 locations->SetInAt(0, Location::RequiresFpuRegister());
1926 locations->SetInAt(1,
1927 IsFloatingPointZeroConstant(instruction->InputAt(1))
1928 ? Location::ConstantLocation(instruction->InputAt(1)->AsConstant())
1929 : Location::RequiresFpuRegister());
1930 } else {
1931 // Integer cases.
1932 locations->SetInAt(0, Location::RequiresRegister());
1933 locations->SetInAt(1, ARM64EncodableConstantOrRegister(instruction->InputAt(1), instruction));
1934 }
1935
Alexandre Rames5319def2014-10-23 10:03:10 +01001936 if (instruction->NeedsMaterialization()) {
Alexandre Ramesfb4e5fa2014-11-06 12:41:16 +00001937 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
Alexandre Rames5319def2014-10-23 10:03:10 +01001938 }
1939}
1940
1941void InstructionCodeGeneratorARM64::VisitCondition(HCondition* instruction) {
1942 if (!instruction->NeedsMaterialization()) {
1943 return;
1944 }
1945
1946 LocationSummary* locations = instruction->GetLocations();
Alexandre Rames5319def2014-10-23 10:03:10 +01001947 Register res = RegisterFrom(locations->Out(), instruction->GetType());
Roland Levillain7f63c522015-07-13 15:54:55 +00001948 IfCondition if_cond = instruction->GetCondition();
1949 Condition arm64_cond = ARM64Condition(if_cond);
Alexandre Rames5319def2014-10-23 10:03:10 +01001950
Roland Levillain7f63c522015-07-13 15:54:55 +00001951 if (Primitive::IsFloatingPointType(instruction->InputAt(0)->GetType())) {
1952 FPRegister lhs = InputFPRegisterAt(instruction, 0);
1953 if (locations->InAt(1).IsConstant()) {
1954 DCHECK(IsFloatingPointZeroConstant(locations->InAt(1).GetConstant()));
1955 // 0.0 is the only immediate that can be encoded directly in an FCMP instruction.
1956 __ Fcmp(lhs, 0.0);
1957 } else {
1958 __ Fcmp(lhs, InputFPRegisterAt(instruction, 1));
1959 }
1960 __ Cset(res, arm64_cond);
1961 if (instruction->IsFPConditionTrueIfNaN()) {
1962 // res = IsUnordered(arm64_cond) ? 1 : res <=> res = IsNotUnordered(arm64_cond) ? res : 1
1963 __ Csel(res, res, Operand(1), vc); // VC for "not unordered".
1964 } else if (instruction->IsFPConditionFalseIfNaN()) {
1965 // res = IsUnordered(arm64_cond) ? 0 : res <=> res = IsNotUnordered(arm64_cond) ? res : 0
1966 __ Csel(res, res, Operand(0), vc); // VC for "not unordered".
1967 }
1968 } else {
1969 // Integer cases.
1970 Register lhs = InputRegisterAt(instruction, 0);
1971 Operand rhs = InputOperandAt(instruction, 1);
1972 __ Cmp(lhs, rhs);
1973 __ Cset(res, arm64_cond);
1974 }
Alexandre Rames5319def2014-10-23 10:03:10 +01001975}
1976
1977#define FOR_EACH_CONDITION_INSTRUCTION(M) \
1978 M(Equal) \
1979 M(NotEqual) \
1980 M(LessThan) \
1981 M(LessThanOrEqual) \
1982 M(GreaterThan) \
Aart Bike9f37602015-10-09 11:15:55 -07001983 M(GreaterThanOrEqual) \
1984 M(Below) \
1985 M(BelowOrEqual) \
1986 M(Above) \
1987 M(AboveOrEqual)
Alexandre Rames5319def2014-10-23 10:03:10 +01001988#define DEFINE_CONDITION_VISITORS(Name) \
1989void LocationsBuilderARM64::Visit##Name(H##Name* comp) { VisitCondition(comp); } \
1990void InstructionCodeGeneratorARM64::Visit##Name(H##Name* comp) { VisitCondition(comp); }
1991FOR_EACH_CONDITION_INSTRUCTION(DEFINE_CONDITION_VISITORS)
Alexandre Rames67555f72014-11-18 10:55:16 +00001992#undef DEFINE_CONDITION_VISITORS
Alexandre Rames5319def2014-10-23 10:03:10 +01001993#undef FOR_EACH_CONDITION_INSTRUCTION
1994
Zheng Xuc6667102015-05-15 16:08:45 +08001995void InstructionCodeGeneratorARM64::DivRemOneOrMinusOne(HBinaryOperation* instruction) {
1996 DCHECK(instruction->IsDiv() || instruction->IsRem());
1997
1998 LocationSummary* locations = instruction->GetLocations();
1999 Location second = locations->InAt(1);
2000 DCHECK(second.IsConstant());
2001
2002 Register out = OutputRegister(instruction);
2003 Register dividend = InputRegisterAt(instruction, 0);
2004 int64_t imm = Int64FromConstant(second.GetConstant());
2005 DCHECK(imm == 1 || imm == -1);
2006
2007 if (instruction->IsRem()) {
2008 __ Mov(out, 0);
2009 } else {
2010 if (imm == 1) {
2011 __ Mov(out, dividend);
2012 } else {
2013 __ Neg(out, dividend);
2014 }
2015 }
2016}
2017
2018void InstructionCodeGeneratorARM64::DivRemByPowerOfTwo(HBinaryOperation* instruction) {
2019 DCHECK(instruction->IsDiv() || instruction->IsRem());
2020
2021 LocationSummary* locations = instruction->GetLocations();
2022 Location second = locations->InAt(1);
2023 DCHECK(second.IsConstant());
2024
2025 Register out = OutputRegister(instruction);
2026 Register dividend = InputRegisterAt(instruction, 0);
2027 int64_t imm = Int64FromConstant(second.GetConstant());
Vladimir Marko80afd022015-05-19 18:08:00 +01002028 uint64_t abs_imm = static_cast<uint64_t>(std::abs(imm));
Zheng Xuc6667102015-05-15 16:08:45 +08002029 DCHECK(IsPowerOfTwo(abs_imm));
2030 int ctz_imm = CTZ(abs_imm);
2031
2032 UseScratchRegisterScope temps(GetVIXLAssembler());
2033 Register temp = temps.AcquireSameSizeAs(out);
2034
2035 if (instruction->IsDiv()) {
2036 __ Add(temp, dividend, abs_imm - 1);
2037 __ Cmp(dividend, 0);
2038 __ Csel(out, temp, dividend, lt);
2039 if (imm > 0) {
2040 __ Asr(out, out, ctz_imm);
2041 } else {
2042 __ Neg(out, Operand(out, ASR, ctz_imm));
2043 }
2044 } else {
2045 int bits = instruction->GetResultType() == Primitive::kPrimInt ? 32 : 64;
2046 __ Asr(temp, dividend, bits - 1);
2047 __ Lsr(temp, temp, bits - ctz_imm);
2048 __ Add(out, dividend, temp);
2049 __ And(out, out, abs_imm - 1);
2050 __ Sub(out, out, temp);
2051 }
2052}
2053
2054void InstructionCodeGeneratorARM64::GenerateDivRemWithAnyConstant(HBinaryOperation* instruction) {
2055 DCHECK(instruction->IsDiv() || instruction->IsRem());
2056
2057 LocationSummary* locations = instruction->GetLocations();
2058 Location second = locations->InAt(1);
2059 DCHECK(second.IsConstant());
2060
2061 Register out = OutputRegister(instruction);
2062 Register dividend = InputRegisterAt(instruction, 0);
2063 int64_t imm = Int64FromConstant(second.GetConstant());
2064
2065 Primitive::Type type = instruction->GetResultType();
2066 DCHECK(type == Primitive::kPrimInt || type == Primitive::kPrimLong);
2067
2068 int64_t magic;
2069 int shift;
2070 CalculateMagicAndShiftForDivRem(imm, type == Primitive::kPrimLong /* is_long */, &magic, &shift);
2071
2072 UseScratchRegisterScope temps(GetVIXLAssembler());
2073 Register temp = temps.AcquireSameSizeAs(out);
2074
2075 // temp = get_high(dividend * magic)
2076 __ Mov(temp, magic);
2077 if (type == Primitive::kPrimLong) {
2078 __ Smulh(temp, dividend, temp);
2079 } else {
2080 __ Smull(temp.X(), dividend, temp);
2081 __ Lsr(temp.X(), temp.X(), 32);
2082 }
2083
2084 if (imm > 0 && magic < 0) {
2085 __ Add(temp, temp, dividend);
2086 } else if (imm < 0 && magic > 0) {
2087 __ Sub(temp, temp, dividend);
2088 }
2089
2090 if (shift != 0) {
2091 __ Asr(temp, temp, shift);
2092 }
2093
2094 if (instruction->IsDiv()) {
2095 __ Sub(out, temp, Operand(temp, ASR, type == Primitive::kPrimLong ? 63 : 31));
2096 } else {
2097 __ Sub(temp, temp, Operand(temp, ASR, type == Primitive::kPrimLong ? 63 : 31));
2098 // TODO: Strength reduction for msub.
2099 Register temp_imm = temps.AcquireSameSizeAs(out);
2100 __ Mov(temp_imm, imm);
2101 __ Msub(out, temp, temp_imm, dividend);
2102 }
2103}
2104
2105void InstructionCodeGeneratorARM64::GenerateDivRemIntegral(HBinaryOperation* instruction) {
2106 DCHECK(instruction->IsDiv() || instruction->IsRem());
2107 Primitive::Type type = instruction->GetResultType();
2108 DCHECK(type == Primitive::kPrimInt || Primitive::kPrimLong);
2109
2110 LocationSummary* locations = instruction->GetLocations();
2111 Register out = OutputRegister(instruction);
2112 Location second = locations->InAt(1);
2113
2114 if (second.IsConstant()) {
2115 int64_t imm = Int64FromConstant(second.GetConstant());
2116
2117 if (imm == 0) {
2118 // Do not generate anything. DivZeroCheck would prevent any code to be executed.
2119 } else if (imm == 1 || imm == -1) {
2120 DivRemOneOrMinusOne(instruction);
2121 } else if (IsPowerOfTwo(std::abs(imm))) {
2122 DivRemByPowerOfTwo(instruction);
2123 } else {
2124 DCHECK(imm <= -2 || imm >= 2);
2125 GenerateDivRemWithAnyConstant(instruction);
2126 }
2127 } else {
2128 Register dividend = InputRegisterAt(instruction, 0);
2129 Register divisor = InputRegisterAt(instruction, 1);
2130 if (instruction->IsDiv()) {
2131 __ Sdiv(out, dividend, divisor);
2132 } else {
2133 UseScratchRegisterScope temps(GetVIXLAssembler());
2134 Register temp = temps.AcquireSameSizeAs(out);
2135 __ Sdiv(temp, dividend, divisor);
2136 __ Msub(out, temp, divisor, dividend);
2137 }
2138 }
2139}
2140
Alexandre Ramesfc19de82014-11-07 17:13:31 +00002141void LocationsBuilderARM64::VisitDiv(HDiv* div) {
2142 LocationSummary* locations =
2143 new (GetGraph()->GetArena()) LocationSummary(div, LocationSummary::kNoCall);
2144 switch (div->GetResultType()) {
2145 case Primitive::kPrimInt:
2146 case Primitive::kPrimLong:
2147 locations->SetInAt(0, Location::RequiresRegister());
Zheng Xuc6667102015-05-15 16:08:45 +08002148 locations->SetInAt(1, Location::RegisterOrConstant(div->InputAt(1)));
Alexandre Ramesfc19de82014-11-07 17:13:31 +00002149 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
2150 break;
2151
2152 case Primitive::kPrimFloat:
2153 case Primitive::kPrimDouble:
2154 locations->SetInAt(0, Location::RequiresFpuRegister());
2155 locations->SetInAt(1, Location::RequiresFpuRegister());
2156 locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap);
2157 break;
2158
2159 default:
2160 LOG(FATAL) << "Unexpected div type " << div->GetResultType();
2161 }
2162}
2163
2164void InstructionCodeGeneratorARM64::VisitDiv(HDiv* div) {
2165 Primitive::Type type = div->GetResultType();
2166 switch (type) {
2167 case Primitive::kPrimInt:
2168 case Primitive::kPrimLong:
Zheng Xuc6667102015-05-15 16:08:45 +08002169 GenerateDivRemIntegral(div);
Alexandre Ramesfc19de82014-11-07 17:13:31 +00002170 break;
2171
2172 case Primitive::kPrimFloat:
2173 case Primitive::kPrimDouble:
2174 __ Fdiv(OutputFPRegister(div), InputFPRegisterAt(div, 0), InputFPRegisterAt(div, 1));
2175 break;
2176
2177 default:
2178 LOG(FATAL) << "Unexpected div type " << type;
2179 }
2180}
2181
Alexandre Rames67555f72014-11-18 10:55:16 +00002182void LocationsBuilderARM64::VisitDivZeroCheck(HDivZeroCheck* instruction) {
David Brazdil77a48ae2015-09-15 12:34:04 +00002183 LocationSummary::CallKind call_kind = instruction->CanThrowIntoCatchBlock()
2184 ? LocationSummary::kCallOnSlowPath
2185 : LocationSummary::kNoCall;
2186 LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instruction, call_kind);
Alexandre Rames67555f72014-11-18 10:55:16 +00002187 locations->SetInAt(0, Location::RegisterOrConstant(instruction->InputAt(0)));
2188 if (instruction->HasUses()) {
2189 locations->SetOut(Location::SameAsFirstInput());
2190 }
2191}
2192
2193void InstructionCodeGeneratorARM64::VisitDivZeroCheck(HDivZeroCheck* instruction) {
2194 SlowPathCodeARM64* slow_path =
2195 new (GetGraph()->GetArena()) DivZeroCheckSlowPathARM64(instruction);
2196 codegen_->AddSlowPath(slow_path);
2197 Location value = instruction->GetLocations()->InAt(0);
2198
Alexandre Rames3e69f162014-12-10 10:36:50 +00002199 Primitive::Type type = instruction->GetType();
2200
Serguei Katkov8c0676c2015-08-03 13:55:33 +06002201 if ((type == Primitive::kPrimBoolean) || !Primitive::IsIntegralType(type)) {
2202 LOG(FATAL) << "Unexpected type " << type << " for DivZeroCheck.";
Alexandre Rames3e69f162014-12-10 10:36:50 +00002203 return;
2204 }
2205
Alexandre Rames67555f72014-11-18 10:55:16 +00002206 if (value.IsConstant()) {
2207 int64_t divisor = Int64ConstantFrom(value);
2208 if (divisor == 0) {
2209 __ B(slow_path->GetEntryLabel());
2210 } else {
Alexandre Rames3e69f162014-12-10 10:36:50 +00002211 // A division by a non-null constant is valid. We don't need to perform
2212 // any check, so simply fall through.
Alexandre Rames67555f72014-11-18 10:55:16 +00002213 }
2214 } else {
2215 __ Cbz(InputRegisterAt(instruction, 0), slow_path->GetEntryLabel());
2216 }
2217}
2218
Alexandre Ramesa89086e2014-11-07 17:13:25 +00002219void LocationsBuilderARM64::VisitDoubleConstant(HDoubleConstant* constant) {
2220 LocationSummary* locations =
2221 new (GetGraph()->GetArena()) LocationSummary(constant, LocationSummary::kNoCall);
2222 locations->SetOut(Location::ConstantLocation(constant));
2223}
2224
Roland Levillain4b8f1ec2015-08-26 18:34:03 +01002225void InstructionCodeGeneratorARM64::VisitDoubleConstant(
2226 HDoubleConstant* constant ATTRIBUTE_UNUSED) {
Alexandre Ramesa89086e2014-11-07 17:13:25 +00002227 // Will be generated at use site.
2228}
2229
Alexandre Rames5319def2014-10-23 10:03:10 +01002230void LocationsBuilderARM64::VisitExit(HExit* exit) {
2231 exit->SetLocations(nullptr);
2232}
2233
Roland Levillain4b8f1ec2015-08-26 18:34:03 +01002234void InstructionCodeGeneratorARM64::VisitExit(HExit* exit ATTRIBUTE_UNUSED) {
Alexandre Rames5319def2014-10-23 10:03:10 +01002235}
2236
Alexandre Ramesa89086e2014-11-07 17:13:25 +00002237void LocationsBuilderARM64::VisitFloatConstant(HFloatConstant* constant) {
2238 LocationSummary* locations =
2239 new (GetGraph()->GetArena()) LocationSummary(constant, LocationSummary::kNoCall);
2240 locations->SetOut(Location::ConstantLocation(constant));
2241}
2242
Roland Levillain4b8f1ec2015-08-26 18:34:03 +01002243void InstructionCodeGeneratorARM64::VisitFloatConstant(HFloatConstant* constant ATTRIBUTE_UNUSED) {
Alexandre Ramesa89086e2014-11-07 17:13:25 +00002244 // Will be generated at use site.
2245}
2246
David Brazdilfc6a86a2015-06-26 10:33:45 +00002247void InstructionCodeGeneratorARM64::HandleGoto(HInstruction* got, HBasicBlock* successor) {
Serban Constantinescu02164b32014-11-13 14:05:07 +00002248 DCHECK(!successor->IsExitBlock());
2249 HBasicBlock* block = got->GetBlock();
2250 HInstruction* previous = got->GetPrevious();
2251 HLoopInformation* info = block->GetLoopInformation();
2252
David Brazdil46e2a392015-03-16 17:31:52 +00002253 if (info != nullptr && info->IsBackEdge(*block) && info->HasSuspendCheck()) {
Serban Constantinescu02164b32014-11-13 14:05:07 +00002254 codegen_->ClearSpillSlotsFromLoopPhisInStackMap(info->GetSuspendCheck());
2255 GenerateSuspendCheck(info->GetSuspendCheck(), successor);
2256 return;
2257 }
2258 if (block->IsEntryBlock() && (previous != nullptr) && previous->IsSuspendCheck()) {
2259 GenerateSuspendCheck(previous->AsSuspendCheck(), nullptr);
2260 }
2261 if (!codegen_->GoesToNextBlock(block, successor)) {
Alexandre Rames5319def2014-10-23 10:03:10 +01002262 __ B(codegen_->GetLabelOf(successor));
2263 }
2264}
2265
David Brazdilfc6a86a2015-06-26 10:33:45 +00002266void LocationsBuilderARM64::VisitGoto(HGoto* got) {
2267 got->SetLocations(nullptr);
2268}
2269
2270void InstructionCodeGeneratorARM64::VisitGoto(HGoto* got) {
2271 HandleGoto(got, got->GetSuccessor());
2272}
2273
2274void LocationsBuilderARM64::VisitTryBoundary(HTryBoundary* try_boundary) {
2275 try_boundary->SetLocations(nullptr);
2276}
2277
2278void InstructionCodeGeneratorARM64::VisitTryBoundary(HTryBoundary* try_boundary) {
2279 HBasicBlock* successor = try_boundary->GetNormalFlowSuccessor();
2280 if (!successor->IsExitBlock()) {
2281 HandleGoto(try_boundary, successor);
2282 }
2283}
2284
Mingyao Yangd43b3ac2015-04-01 14:03:04 -07002285void InstructionCodeGeneratorARM64::GenerateTestAndBranch(HInstruction* instruction,
2286 vixl::Label* true_target,
2287 vixl::Label* false_target,
2288 vixl::Label* always_true_target) {
2289 HInstruction* cond = instruction->InputAt(0);
Alexandre Rames5319def2014-10-23 10:03:10 +01002290 HCondition* condition = cond->AsCondition();
Alexandre Rames5319def2014-10-23 10:03:10 +01002291
Serban Constantinescu02164b32014-11-13 14:05:07 +00002292 if (cond->IsIntConstant()) {
2293 int32_t cond_value = cond->AsIntConstant()->GetValue();
2294 if (cond_value == 1) {
Mingyao Yangd43b3ac2015-04-01 14:03:04 -07002295 if (always_true_target != nullptr) {
2296 __ B(always_true_target);
Serban Constantinescu02164b32014-11-13 14:05:07 +00002297 }
2298 return;
2299 } else {
2300 DCHECK_EQ(cond_value, 0);
2301 }
2302 } else if (!cond->IsCondition() || condition->NeedsMaterialization()) {
Alexandre Rames5319def2014-10-23 10:03:10 +01002303 // The condition instruction has been materialized, compare the output to 0.
Mingyao Yangd43b3ac2015-04-01 14:03:04 -07002304 Location cond_val = instruction->GetLocations()->InAt(0);
Alexandre Rames5319def2014-10-23 10:03:10 +01002305 DCHECK(cond_val.IsRegister());
Mingyao Yangd43b3ac2015-04-01 14:03:04 -07002306 __ Cbnz(InputRegisterAt(instruction, 0), true_target);
Alexandre Rames5319def2014-10-23 10:03:10 +01002307 } else {
2308 // The condition instruction has not been materialized, use its inputs as
2309 // the comparison and its condition as the branch condition.
Roland Levillain7f63c522015-07-13 15:54:55 +00002310 Primitive::Type type =
2311 cond->IsCondition() ? cond->InputAt(0)->GetType() : Primitive::kPrimInt;
2312
2313 if (Primitive::IsFloatingPointType(type)) {
2314 // FP compares don't like null false_targets.
2315 if (false_target == nullptr) {
2316 false_target = codegen_->GetLabelOf(instruction->AsIf()->IfFalseSuccessor());
Alexandre Rames5319def2014-10-23 10:03:10 +01002317 }
Roland Levillain7f63c522015-07-13 15:54:55 +00002318 FPRegister lhs = InputFPRegisterAt(condition, 0);
2319 if (condition->GetLocations()->InAt(1).IsConstant()) {
2320 DCHECK(IsFloatingPointZeroConstant(condition->GetLocations()->InAt(1).GetConstant()));
2321 // 0.0 is the only immediate that can be encoded directly in an FCMP instruction.
2322 __ Fcmp(lhs, 0.0);
2323 } else {
2324 __ Fcmp(lhs, InputFPRegisterAt(condition, 1));
2325 }
2326 if (condition->IsFPConditionTrueIfNaN()) {
2327 __ B(vs, true_target); // VS for unordered.
2328 } else if (condition->IsFPConditionFalseIfNaN()) {
2329 __ B(vs, false_target); // VS for unordered.
2330 }
2331 __ B(ARM64Condition(condition->GetCondition()), true_target);
Alexandre Rames5319def2014-10-23 10:03:10 +01002332 } else {
Roland Levillain7f63c522015-07-13 15:54:55 +00002333 // Integer cases.
2334 Register lhs = InputRegisterAt(condition, 0);
2335 Operand rhs = InputOperandAt(condition, 1);
2336 Condition arm64_cond = ARM64Condition(condition->GetCondition());
2337 if ((arm64_cond != gt && arm64_cond != le) && rhs.IsImmediate() && (rhs.immediate() == 0)) {
2338 switch (arm64_cond) {
2339 case eq:
2340 __ Cbz(lhs, true_target);
2341 break;
2342 case ne:
2343 __ Cbnz(lhs, true_target);
2344 break;
2345 case lt:
2346 // Test the sign bit and branch accordingly.
2347 __ Tbnz(lhs, (lhs.IsX() ? kXRegSize : kWRegSize) - 1, true_target);
2348 break;
2349 case ge:
2350 // Test the sign bit and branch accordingly.
2351 __ Tbz(lhs, (lhs.IsX() ? kXRegSize : kWRegSize) - 1, true_target);
2352 break;
2353 default:
2354 // Without the `static_cast` the compiler throws an error for
2355 // `-Werror=sign-promo`.
2356 LOG(FATAL) << "Unexpected condition: " << static_cast<int>(arm64_cond);
2357 }
2358 } else {
2359 __ Cmp(lhs, rhs);
2360 __ B(arm64_cond, true_target);
2361 }
Alexandre Rames5319def2014-10-23 10:03:10 +01002362 }
2363 }
Mingyao Yangd43b3ac2015-04-01 14:03:04 -07002364 if (false_target != nullptr) {
Alexandre Rames5319def2014-10-23 10:03:10 +01002365 __ B(false_target);
2366 }
2367}
2368
Mingyao Yangd43b3ac2015-04-01 14:03:04 -07002369void LocationsBuilderARM64::VisitIf(HIf* if_instr) {
2370 LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(if_instr);
2371 HInstruction* cond = if_instr->InputAt(0);
2372 if (!cond->IsCondition() || cond->AsCondition()->NeedsMaterialization()) {
2373 locations->SetInAt(0, Location::RequiresRegister());
2374 }
2375}
2376
2377void InstructionCodeGeneratorARM64::VisitIf(HIf* if_instr) {
2378 vixl::Label* true_target = codegen_->GetLabelOf(if_instr->IfTrueSuccessor());
2379 vixl::Label* false_target = codegen_->GetLabelOf(if_instr->IfFalseSuccessor());
2380 vixl::Label* always_true_target = true_target;
2381 if (codegen_->GoesToNextBlock(if_instr->GetBlock(),
2382 if_instr->IfTrueSuccessor())) {
2383 always_true_target = nullptr;
2384 }
2385 if (codegen_->GoesToNextBlock(if_instr->GetBlock(),
2386 if_instr->IfFalseSuccessor())) {
2387 false_target = nullptr;
2388 }
2389 GenerateTestAndBranch(if_instr, true_target, false_target, always_true_target);
2390}
2391
2392void LocationsBuilderARM64::VisitDeoptimize(HDeoptimize* deoptimize) {
2393 LocationSummary* locations = new (GetGraph()->GetArena())
2394 LocationSummary(deoptimize, LocationSummary::kCallOnSlowPath);
2395 HInstruction* cond = deoptimize->InputAt(0);
Aart Bikbb245d12015-10-19 11:05:03 -07002396 if (!cond->IsCondition() || cond->AsCondition()->NeedsMaterialization()) {
Mingyao Yangd43b3ac2015-04-01 14:03:04 -07002397 locations->SetInAt(0, Location::RequiresRegister());
2398 }
2399}
2400
2401void InstructionCodeGeneratorARM64::VisitDeoptimize(HDeoptimize* deoptimize) {
2402 SlowPathCodeARM64* slow_path = new (GetGraph()->GetArena())
2403 DeoptimizationSlowPathARM64(deoptimize);
2404 codegen_->AddSlowPath(slow_path);
2405 vixl::Label* slow_path_entry = slow_path->GetEntryLabel();
2406 GenerateTestAndBranch(deoptimize, slow_path_entry, nullptr, slow_path_entry);
2407}
2408
Alexandre Rames5319def2014-10-23 10:03:10 +01002409void LocationsBuilderARM64::VisitInstanceFieldGet(HInstanceFieldGet* instruction) {
Alexandre Rames09a99962015-04-15 11:47:56 +01002410 HandleFieldGet(instruction);
Alexandre Rames5319def2014-10-23 10:03:10 +01002411}
2412
2413void InstructionCodeGeneratorARM64::VisitInstanceFieldGet(HInstanceFieldGet* instruction) {
Alexandre Rames09a99962015-04-15 11:47:56 +01002414 HandleFieldGet(instruction, instruction->GetFieldInfo());
Alexandre Rames5319def2014-10-23 10:03:10 +01002415}
2416
2417void LocationsBuilderARM64::VisitInstanceFieldSet(HInstanceFieldSet* instruction) {
Alexandre Rames09a99962015-04-15 11:47:56 +01002418 HandleFieldSet(instruction);
Alexandre Rames5319def2014-10-23 10:03:10 +01002419}
2420
2421void InstructionCodeGeneratorARM64::VisitInstanceFieldSet(HInstanceFieldSet* instruction) {
Nicolas Geoffray07276db2015-05-18 14:22:09 +01002422 HandleFieldSet(instruction, instruction->GetFieldInfo(), instruction->GetValueCanBeNull());
Alexandre Rames5319def2014-10-23 10:03:10 +01002423}
2424
Alexandre Rames67555f72014-11-18 10:55:16 +00002425void LocationsBuilderARM64::VisitInstanceOf(HInstanceOf* instruction) {
Nicolas Geoffray85c7bab2015-09-18 13:40:46 +00002426 LocationSummary::CallKind call_kind = LocationSummary::kNoCall;
2427 switch (instruction->GetTypeCheckKind()) {
2428 case TypeCheckKind::kExactCheck:
2429 case TypeCheckKind::kAbstractClassCheck:
2430 case TypeCheckKind::kClassHierarchyCheck:
2431 case TypeCheckKind::kArrayObjectCheck:
2432 call_kind = LocationSummary::kNoCall;
2433 break;
Calin Juravle98893e12015-10-02 21:05:03 +01002434 case TypeCheckKind::kUnresolvedCheck:
Nicolas Geoffray85c7bab2015-09-18 13:40:46 +00002435 case TypeCheckKind::kInterfaceCheck:
2436 call_kind = LocationSummary::kCall;
2437 break;
2438 case TypeCheckKind::kArrayCheck:
2439 call_kind = LocationSummary::kCallOnSlowPath;
2440 break;
2441 }
Alexandre Rames67555f72014-11-18 10:55:16 +00002442 LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instruction, call_kind);
Nicolas Geoffray85c7bab2015-09-18 13:40:46 +00002443 if (call_kind != LocationSummary::kCall) {
2444 locations->SetInAt(0, Location::RequiresRegister());
2445 locations->SetInAt(1, Location::RequiresRegister());
2446 // The out register is used as a temporary, so it overlaps with the inputs.
2447 // Note that TypeCheckSlowPathARM64 uses this register too.
2448 locations->SetOut(Location::RequiresRegister(), Location::kOutputOverlap);
2449 } else {
2450 InvokeRuntimeCallingConvention calling_convention;
2451 locations->SetInAt(1, LocationFrom(calling_convention.GetRegisterAt(0)));
2452 locations->SetInAt(0, LocationFrom(calling_convention.GetRegisterAt(1)));
2453 locations->SetOut(calling_convention.GetReturnLocation(Primitive::kPrimInt));
2454 }
Alexandre Rames67555f72014-11-18 10:55:16 +00002455}
2456
2457void InstructionCodeGeneratorARM64::VisitInstanceOf(HInstanceOf* instruction) {
2458 LocationSummary* locations = instruction->GetLocations();
Nicolas Geoffray85c7bab2015-09-18 13:40:46 +00002459 Register obj = InputRegisterAt(instruction, 0);
2460 Register cls = InputRegisterAt(instruction, 1);
Alexandre Rames67555f72014-11-18 10:55:16 +00002461 Register out = OutputRegister(instruction);
Nicolas Geoffray85c7bab2015-09-18 13:40:46 +00002462 uint32_t class_offset = mirror::Object::ClassOffset().Int32Value();
2463 uint32_t super_offset = mirror::Class::SuperClassOffset().Int32Value();
2464 uint32_t component_offset = mirror::Class::ComponentTypeOffset().Int32Value();
2465 uint32_t primitive_offset = mirror::Class::PrimitiveTypeOffset().Int32Value();
Alexandre Rames67555f72014-11-18 10:55:16 +00002466
Nicolas Geoffray85c7bab2015-09-18 13:40:46 +00002467 vixl::Label done, zero;
2468 SlowPathCodeARM64* slow_path = nullptr;
Alexandre Rames67555f72014-11-18 10:55:16 +00002469
2470 // Return 0 if `obj` is null.
Guillaume "Vermeille" Sanchezaf888352015-04-20 14:41:30 +01002471 // Avoid null check if we know `obj` is not null.
2472 if (instruction->MustDoNullCheck()) {
Nicolas Geoffray85c7bab2015-09-18 13:40:46 +00002473 __ Cbz(obj, &zero);
2474 }
2475
Calin Juravle98893e12015-10-02 21:05:03 +01002476 // In case of an interface/unresolved check, we put the object class into the object register.
Nicolas Geoffray85c7bab2015-09-18 13:40:46 +00002477 // This is safe, as the register is caller-save, and the object must be in another
2478 // register if it survives the runtime call.
Calin Juravle98893e12015-10-02 21:05:03 +01002479 Register target = (instruction->GetTypeCheckKind() == TypeCheckKind::kInterfaceCheck) ||
2480 (instruction->GetTypeCheckKind() == TypeCheckKind::kUnresolvedCheck)
Nicolas Geoffray85c7bab2015-09-18 13:40:46 +00002481 ? obj
2482 : out;
2483 __ Ldr(target, HeapOperand(obj.W(), class_offset));
2484 GetAssembler()->MaybeUnpoisonHeapReference(target);
2485
2486 switch (instruction->GetTypeCheckKind()) {
2487 case TypeCheckKind::kExactCheck: {
2488 __ Cmp(out, cls);
2489 __ Cset(out, eq);
2490 if (zero.IsLinked()) {
2491 __ B(&done);
2492 }
2493 break;
2494 }
2495 case TypeCheckKind::kAbstractClassCheck: {
2496 // If the class is abstract, we eagerly fetch the super class of the
2497 // object to avoid doing a comparison we know will fail.
2498 vixl::Label loop, success;
2499 __ Bind(&loop);
2500 __ Ldr(out, HeapOperand(out, super_offset));
2501 GetAssembler()->MaybeUnpoisonHeapReference(out);
2502 // If `out` is null, we use it for the result, and jump to `done`.
2503 __ Cbz(out, &done);
2504 __ Cmp(out, cls);
2505 __ B(ne, &loop);
2506 __ Mov(out, 1);
2507 if (zero.IsLinked()) {
2508 __ B(&done);
2509 }
2510 break;
2511 }
2512 case TypeCheckKind::kClassHierarchyCheck: {
2513 // Walk over the class hierarchy to find a match.
2514 vixl::Label loop, success;
2515 __ Bind(&loop);
2516 __ Cmp(out, cls);
2517 __ B(eq, &success);
2518 __ Ldr(out, HeapOperand(out, super_offset));
2519 GetAssembler()->MaybeUnpoisonHeapReference(out);
2520 __ Cbnz(out, &loop);
2521 // If `out` is null, we use it for the result, and jump to `done`.
2522 __ B(&done);
2523 __ Bind(&success);
2524 __ Mov(out, 1);
2525 if (zero.IsLinked()) {
2526 __ B(&done);
2527 }
2528 break;
2529 }
2530 case TypeCheckKind::kArrayObjectCheck: {
Nicolas Geoffrayabfcf182015-09-21 18:41:21 +01002531 // Do an exact check.
2532 vixl::Label exact_check;
2533 __ Cmp(out, cls);
2534 __ B(eq, &exact_check);
2535 // Otherwise, we need to check that the object's class is a non primitive array.
Nicolas Geoffray85c7bab2015-09-18 13:40:46 +00002536 __ Ldr(out, HeapOperand(out, component_offset));
2537 GetAssembler()->MaybeUnpoisonHeapReference(out);
2538 // If `out` is null, we use it for the result, and jump to `done`.
2539 __ Cbz(out, &done);
2540 __ Ldrh(out, HeapOperand(out, primitive_offset));
2541 static_assert(Primitive::kPrimNot == 0, "Expected 0 for kPrimNot");
2542 __ Cbnz(out, &zero);
Nicolas Geoffrayabfcf182015-09-21 18:41:21 +01002543 __ Bind(&exact_check);
Nicolas Geoffray85c7bab2015-09-18 13:40:46 +00002544 __ Mov(out, 1);
2545 __ B(&done);
2546 break;
2547 }
2548 case TypeCheckKind::kArrayCheck: {
2549 __ Cmp(out, cls);
2550 DCHECK(locations->OnlyCallsOnSlowPath());
2551 slow_path = new (GetGraph()->GetArena()) TypeCheckSlowPathARM64(
2552 instruction, /* is_fatal */ false);
2553 codegen_->AddSlowPath(slow_path);
2554 __ B(ne, slow_path->GetEntryLabel());
2555 __ Mov(out, 1);
2556 if (zero.IsLinked()) {
2557 __ B(&done);
2558 }
2559 break;
2560 }
Calin Juravle98893e12015-10-02 21:05:03 +01002561 case TypeCheckKind::kUnresolvedCheck:
Nicolas Geoffray85c7bab2015-09-18 13:40:46 +00002562 case TypeCheckKind::kInterfaceCheck:
2563 default: {
2564 codegen_->InvokeRuntime(QUICK_ENTRY_POINT(pInstanceofNonTrivial),
2565 instruction,
2566 instruction->GetDexPc(),
2567 nullptr);
2568 if (zero.IsLinked()) {
2569 __ B(&done);
2570 }
2571 break;
2572 }
2573 }
2574
2575 if (zero.IsLinked()) {
2576 __ Bind(&zero);
Guillaume "Vermeille" Sanchezaf888352015-04-20 14:41:30 +01002577 __ Mov(out, 0);
Nicolas Geoffray85c7bab2015-09-18 13:40:46 +00002578 }
2579
2580 if (done.IsLinked()) {
2581 __ Bind(&done);
2582 }
2583
2584 if (slow_path != nullptr) {
2585 __ Bind(slow_path->GetExitLabel());
2586 }
2587}
2588
2589void LocationsBuilderARM64::VisitCheckCast(HCheckCast* instruction) {
2590 LocationSummary::CallKind call_kind = LocationSummary::kNoCall;
2591 bool throws_into_catch = instruction->CanThrowIntoCatchBlock();
2592
2593 switch (instruction->GetTypeCheckKind()) {
2594 case TypeCheckKind::kExactCheck:
2595 case TypeCheckKind::kAbstractClassCheck:
2596 case TypeCheckKind::kClassHierarchyCheck:
2597 case TypeCheckKind::kArrayObjectCheck:
2598 call_kind = throws_into_catch
2599 ? LocationSummary::kCallOnSlowPath
2600 : LocationSummary::kNoCall;
2601 break;
Calin Juravle98893e12015-10-02 21:05:03 +01002602 case TypeCheckKind::kUnresolvedCheck:
Nicolas Geoffray85c7bab2015-09-18 13:40:46 +00002603 case TypeCheckKind::kInterfaceCheck:
2604 call_kind = LocationSummary::kCall;
2605 break;
2606 case TypeCheckKind::kArrayCheck:
2607 call_kind = LocationSummary::kCallOnSlowPath;
2608 break;
2609 }
2610
2611 LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(
2612 instruction, call_kind);
2613 if (call_kind != LocationSummary::kCall) {
2614 locations->SetInAt(0, Location::RequiresRegister());
2615 locations->SetInAt(1, Location::RequiresRegister());
2616 // Note that TypeCheckSlowPathARM64 uses this register too.
2617 locations->AddTemp(Location::RequiresRegister());
2618 } else {
2619 InvokeRuntimeCallingConvention calling_convention;
2620 locations->SetInAt(1, LocationFrom(calling_convention.GetRegisterAt(0)));
2621 locations->SetInAt(0, LocationFrom(calling_convention.GetRegisterAt(1)));
2622 }
2623}
2624
2625void InstructionCodeGeneratorARM64::VisitCheckCast(HCheckCast* instruction) {
2626 LocationSummary* locations = instruction->GetLocations();
2627 Register obj = InputRegisterAt(instruction, 0);
2628 Register cls = InputRegisterAt(instruction, 1);
2629 Register temp;
2630 if (!locations->WillCall()) {
2631 temp = WRegisterFrom(instruction->GetLocations()->GetTemp(0));
2632 }
2633
2634 uint32_t class_offset = mirror::Object::ClassOffset().Int32Value();
2635 uint32_t super_offset = mirror::Class::SuperClassOffset().Int32Value();
2636 uint32_t component_offset = mirror::Class::ComponentTypeOffset().Int32Value();
2637 uint32_t primitive_offset = mirror::Class::PrimitiveTypeOffset().Int32Value();
2638 SlowPathCodeARM64* slow_path = nullptr;
2639
2640 if (!locations->WillCall()) {
2641 slow_path = new (GetGraph()->GetArena()) TypeCheckSlowPathARM64(
2642 instruction, !locations->CanCall());
2643 codegen_->AddSlowPath(slow_path);
2644 }
2645
2646 vixl::Label done;
2647 // Avoid null check if we know obj is not null.
2648 if (instruction->MustDoNullCheck()) {
Guillaume "Vermeille" Sanchezaf888352015-04-20 14:41:30 +01002649 __ Cbz(obj, &done);
2650 }
Alexandre Rames67555f72014-11-18 10:55:16 +00002651
Nicolas Geoffray85c7bab2015-09-18 13:40:46 +00002652 if (locations->WillCall()) {
2653 __ Ldr(obj, HeapOperand(obj, class_offset));
2654 GetAssembler()->MaybeUnpoisonHeapReference(obj);
Alexandre Rames67555f72014-11-18 10:55:16 +00002655 } else {
Nicolas Geoffray85c7bab2015-09-18 13:40:46 +00002656 __ Ldr(temp, HeapOperand(obj, class_offset));
2657 GetAssembler()->MaybeUnpoisonHeapReference(temp);
Nicolas Geoffray64acf302015-09-14 22:20:29 +01002658 }
Nicolas Geoffray75374372015-09-17 17:12:19 +00002659
Nicolas Geoffray85c7bab2015-09-18 13:40:46 +00002660 switch (instruction->GetTypeCheckKind()) {
2661 case TypeCheckKind::kExactCheck:
2662 case TypeCheckKind::kArrayCheck: {
2663 __ Cmp(temp, cls);
2664 // Jump to slow path for throwing the exception or doing a
2665 // more involved array check.
2666 __ B(ne, slow_path->GetEntryLabel());
2667 break;
2668 }
2669 case TypeCheckKind::kAbstractClassCheck: {
2670 // If the class is abstract, we eagerly fetch the super class of the
2671 // object to avoid doing a comparison we know will fail.
2672 vixl::Label loop;
2673 __ Bind(&loop);
2674 __ Ldr(temp, HeapOperand(temp, super_offset));
2675 GetAssembler()->MaybeUnpoisonHeapReference(temp);
2676 // Jump to the slow path to throw the exception.
2677 __ Cbz(temp, slow_path->GetEntryLabel());
2678 __ Cmp(temp, cls);
2679 __ B(ne, &loop);
2680 break;
2681 }
2682 case TypeCheckKind::kClassHierarchyCheck: {
2683 // Walk over the class hierarchy to find a match.
Nicolas Geoffrayabfcf182015-09-21 18:41:21 +01002684 vixl::Label loop;
Nicolas Geoffray85c7bab2015-09-18 13:40:46 +00002685 __ Bind(&loop);
2686 __ Cmp(temp, cls);
Nicolas Geoffrayabfcf182015-09-21 18:41:21 +01002687 __ B(eq, &done);
Nicolas Geoffray85c7bab2015-09-18 13:40:46 +00002688 __ Ldr(temp, HeapOperand(temp, super_offset));
2689 GetAssembler()->MaybeUnpoisonHeapReference(temp);
2690 __ Cbnz(temp, &loop);
2691 // Jump to the slow path to throw the exception.
2692 __ B(slow_path->GetEntryLabel());
Nicolas Geoffray85c7bab2015-09-18 13:40:46 +00002693 break;
2694 }
2695 case TypeCheckKind::kArrayObjectCheck: {
Nicolas Geoffrayabfcf182015-09-21 18:41:21 +01002696 // Do an exact check.
2697 __ Cmp(temp, cls);
2698 __ B(eq, &done);
2699 // Otherwise, we need to check that the object's class is a non primitive array.
Nicolas Geoffray85c7bab2015-09-18 13:40:46 +00002700 __ Ldr(temp, HeapOperand(temp, component_offset));
2701 GetAssembler()->MaybeUnpoisonHeapReference(temp);
2702 __ Cbz(temp, slow_path->GetEntryLabel());
2703 __ Ldrh(temp, HeapOperand(temp, primitive_offset));
2704 static_assert(Primitive::kPrimNot == 0, "Expected 0 for kPrimNot");
2705 __ Cbnz(temp, slow_path->GetEntryLabel());
2706 break;
2707 }
Calin Juravle98893e12015-10-02 21:05:03 +01002708 case TypeCheckKind::kUnresolvedCheck:
Nicolas Geoffray85c7bab2015-09-18 13:40:46 +00002709 case TypeCheckKind::kInterfaceCheck:
2710 default:
2711 codegen_->InvokeRuntime(QUICK_ENTRY_POINT(pCheckCast),
2712 instruction,
2713 instruction->GetDexPc(),
2714 nullptr);
2715 break;
2716 }
Nicolas Geoffray75374372015-09-17 17:12:19 +00002717 __ Bind(&done);
Nicolas Geoffray85c7bab2015-09-18 13:40:46 +00002718
2719 if (slow_path != nullptr) {
2720 __ Bind(slow_path->GetExitLabel());
2721 }
Alexandre Rames67555f72014-11-18 10:55:16 +00002722}
2723
Alexandre Rames5319def2014-10-23 10:03:10 +01002724void LocationsBuilderARM64::VisitIntConstant(HIntConstant* constant) {
2725 LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(constant);
2726 locations->SetOut(Location::ConstantLocation(constant));
2727}
2728
Roland Levillain4b8f1ec2015-08-26 18:34:03 +01002729void InstructionCodeGeneratorARM64::VisitIntConstant(HIntConstant* constant ATTRIBUTE_UNUSED) {
Alexandre Rames5319def2014-10-23 10:03:10 +01002730 // Will be generated at use site.
2731}
2732
Nicolas Geoffrayd6138ef2015-02-18 14:48:53 +00002733void LocationsBuilderARM64::VisitNullConstant(HNullConstant* constant) {
2734 LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(constant);
2735 locations->SetOut(Location::ConstantLocation(constant));
2736}
2737
Roland Levillain4b8f1ec2015-08-26 18:34:03 +01002738void InstructionCodeGeneratorARM64::VisitNullConstant(HNullConstant* constant ATTRIBUTE_UNUSED) {
Nicolas Geoffrayd6138ef2015-02-18 14:48:53 +00002739 // Will be generated at use site.
Nicolas Geoffrayd6138ef2015-02-18 14:48:53 +00002740}
2741
Calin Juravle175dc732015-08-25 15:42:32 +01002742void LocationsBuilderARM64::VisitInvokeUnresolved(HInvokeUnresolved* invoke) {
2743 // The trampoline uses the same calling convention as dex calling conventions,
2744 // except instead of loading arg0/r0 with the target Method*, arg0/r0 will contain
2745 // the method_idx.
2746 HandleInvoke(invoke);
2747}
2748
2749void InstructionCodeGeneratorARM64::VisitInvokeUnresolved(HInvokeUnresolved* invoke) {
2750 codegen_->GenerateInvokeUnresolvedRuntimeCall(invoke);
2751}
2752
Alexandre Rames5319def2014-10-23 10:03:10 +01002753void LocationsBuilderARM64::HandleInvoke(HInvoke* invoke) {
Roland Levillain2d27c8e2015-04-28 15:48:45 +01002754 InvokeDexCallingConventionVisitorARM64 calling_convention_visitor;
Nicolas Geoffrayfd88f162015-06-03 11:23:52 +01002755 CodeGenerator::CreateCommonInvokeLocationSummary(invoke, &calling_convention_visitor);
Alexandre Rames5319def2014-10-23 10:03:10 +01002756}
2757
Alexandre Rames67555f72014-11-18 10:55:16 +00002758void LocationsBuilderARM64::VisitInvokeInterface(HInvokeInterface* invoke) {
2759 HandleInvoke(invoke);
2760}
2761
2762void InstructionCodeGeneratorARM64::VisitInvokeInterface(HInvokeInterface* invoke) {
2763 // TODO: b/18116999, our IMTs can miss an IncompatibleClassChangeError.
Mathieu Chartiere401d142015-04-22 13:56:20 -07002764 Register temp = XRegisterFrom(invoke->GetLocations()->GetTemp(0));
2765 uint32_t method_offset = mirror::Class::EmbeddedImTableEntryOffset(
2766 invoke->GetImtIndex() % mirror::Class::kImtSize, kArm64PointerSize).Uint32Value();
Alexandre Rames67555f72014-11-18 10:55:16 +00002767 Location receiver = invoke->GetLocations()->InAt(0);
2768 Offset class_offset = mirror::Object::ClassOffset();
Mathieu Chartiere401d142015-04-22 13:56:20 -07002769 Offset entry_point = ArtMethod::EntryPointFromQuickCompiledCodeOffset(kArm64WordSize);
Alexandre Rames67555f72014-11-18 10:55:16 +00002770
2771 // The register ip1 is required to be used for the hidden argument in
2772 // art_quick_imt_conflict_trampoline, so prevent VIXL from using it.
Alexandre Ramesd921d642015-04-16 15:07:16 +01002773 MacroAssembler* masm = GetVIXLAssembler();
2774 UseScratchRegisterScope scratch_scope(masm);
2775 BlockPoolsScope block_pools(masm);
Alexandre Rames67555f72014-11-18 10:55:16 +00002776 scratch_scope.Exclude(ip1);
2777 __ Mov(ip1, invoke->GetDexMethodIndex());
2778
2779 // temp = object->GetClass();
2780 if (receiver.IsStackSlot()) {
Mathieu Chartiere401d142015-04-22 13:56:20 -07002781 __ Ldr(temp.W(), StackOperandFrom(receiver));
2782 __ Ldr(temp.W(), HeapOperand(temp.W(), class_offset));
Alexandre Rames67555f72014-11-18 10:55:16 +00002783 } else {
Mathieu Chartiere401d142015-04-22 13:56:20 -07002784 __ Ldr(temp.W(), HeapOperandFrom(receiver, class_offset));
Alexandre Rames67555f72014-11-18 10:55:16 +00002785 }
Calin Juravle77520bc2015-01-12 18:45:46 +00002786 codegen_->MaybeRecordImplicitNullCheck(invoke);
Roland Levillain4d027112015-07-01 15:41:14 +01002787 GetAssembler()->MaybeUnpoisonHeapReference(temp.W());
Alexandre Rames67555f72014-11-18 10:55:16 +00002788 // temp = temp->GetImtEntryAt(method_offset);
Mathieu Chartiere401d142015-04-22 13:56:20 -07002789 __ Ldr(temp, MemOperand(temp, method_offset));
Alexandre Rames67555f72014-11-18 10:55:16 +00002790 // lr = temp->GetEntryPoint();
Mathieu Chartiere401d142015-04-22 13:56:20 -07002791 __ Ldr(lr, MemOperand(temp, entry_point.Int32Value()));
Alexandre Rames67555f72014-11-18 10:55:16 +00002792 // lr();
2793 __ Blr(lr);
2794 DCHECK(!codegen_->IsLeafMethod());
2795 codegen_->RecordPcInfo(invoke, invoke->GetDexPc());
2796}
2797
2798void LocationsBuilderARM64::VisitInvokeVirtual(HInvokeVirtual* invoke) {
Andreas Gampe878d58c2015-01-15 23:24:00 -08002799 IntrinsicLocationsBuilderARM64 intrinsic(GetGraph()->GetArena());
2800 if (intrinsic.TryDispatch(invoke)) {
2801 return;
2802 }
2803
Alexandre Rames67555f72014-11-18 10:55:16 +00002804 HandleInvoke(invoke);
2805}
2806
Nicolas Geoffraye53798a2014-12-01 10:31:54 +00002807void LocationsBuilderARM64::VisitInvokeStaticOrDirect(HInvokeStaticOrDirect* invoke) {
Roland Levillain3e3d7332015-04-28 11:00:54 +01002808 // When we do not run baseline, explicit clinit checks triggered by static
2809 // invokes must have been pruned by art::PrepareForRegisterAllocation.
2810 DCHECK(codegen_->IsBaseline() || !invoke->IsStaticWithExplicitClinitCheck());
Roland Levillain4c0eb422015-04-24 16:43:49 +01002811
Andreas Gampe878d58c2015-01-15 23:24:00 -08002812 IntrinsicLocationsBuilderARM64 intrinsic(GetGraph()->GetArena());
2813 if (intrinsic.TryDispatch(invoke)) {
2814 return;
2815 }
2816
Alexandre Rames67555f72014-11-18 10:55:16 +00002817 HandleInvoke(invoke);
2818}
2819
Andreas Gampe878d58c2015-01-15 23:24:00 -08002820static bool TryGenerateIntrinsicCode(HInvoke* invoke, CodeGeneratorARM64* codegen) {
2821 if (invoke->GetLocations()->Intrinsified()) {
2822 IntrinsicCodeGeneratorARM64 intrinsic(codegen);
2823 intrinsic.Dispatch(invoke);
2824 return true;
2825 }
2826 return false;
2827}
2828
Vladimir Markodc151b22015-10-15 18:02:30 +01002829HInvokeStaticOrDirect::DispatchInfo CodeGeneratorARM64::GetSupportedInvokeStaticOrDirectDispatch(
2830 const HInvokeStaticOrDirect::DispatchInfo& desired_dispatch_info,
2831 MethodReference target_method ATTRIBUTE_UNUSED) {
2832 // On arm64 we support all dispatch types.
2833 return desired_dispatch_info;
2834}
2835
Nicolas Geoffray38207af2015-06-01 15:46:22 +01002836void CodeGeneratorARM64::GenerateStaticOrDirectCall(HInvokeStaticOrDirect* invoke, Location temp) {
Vladimir Marko58155012015-08-19 12:49:41 +00002837 // For better instruction scheduling we load the direct code pointer before the method pointer.
2838 bool direct_code_loaded = false;
2839 switch (invoke->GetCodePtrLocation()) {
2840 case HInvokeStaticOrDirect::CodePtrLocation::kCallDirectWithFixup:
2841 // LR = code address from literal pool with link-time patch.
2842 __ Ldr(lr, DeduplicateMethodCodeLiteral(invoke->GetTargetMethod()));
2843 direct_code_loaded = true;
2844 break;
2845 case HInvokeStaticOrDirect::CodePtrLocation::kCallDirect:
2846 // LR = invoke->GetDirectCodePtr();
2847 __ Ldr(lr, DeduplicateUint64Literal(invoke->GetDirectCodePtr()));
2848 direct_code_loaded = true;
2849 break;
2850 default:
2851 break;
2852 }
2853
Andreas Gampe878d58c2015-01-15 23:24:00 -08002854 // Make sure that ArtMethod* is passed in kArtMethodRegister as per the calling convention.
Vladimir Marko58155012015-08-19 12:49:41 +00002855 Location callee_method = temp; // For all kinds except kRecursive, callee will be in temp.
2856 switch (invoke->GetMethodLoadKind()) {
2857 case HInvokeStaticOrDirect::MethodLoadKind::kStringInit:
2858 // temp = thread->string_init_entrypoint
Alexandre Rames6dc01742015-11-12 14:44:19 +00002859 __ Ldr(XRegisterFrom(temp), MemOperand(tr, invoke->GetStringInitOffset()));
Vladimir Marko58155012015-08-19 12:49:41 +00002860 break;
2861 case HInvokeStaticOrDirect::MethodLoadKind::kRecursive:
2862 callee_method = invoke->GetLocations()->InAt(invoke->GetCurrentMethodInputIndex());
2863 break;
2864 case HInvokeStaticOrDirect::MethodLoadKind::kDirectAddress:
2865 // Load method address from literal pool.
Alexandre Rames6dc01742015-11-12 14:44:19 +00002866 __ Ldr(XRegisterFrom(temp), DeduplicateUint64Literal(invoke->GetMethodAddress()));
Vladimir Marko58155012015-08-19 12:49:41 +00002867 break;
2868 case HInvokeStaticOrDirect::MethodLoadKind::kDirectAddressWithFixup:
2869 // Load method address from literal pool with a link-time patch.
Alexandre Rames6dc01742015-11-12 14:44:19 +00002870 __ Ldr(XRegisterFrom(temp),
Vladimir Marko58155012015-08-19 12:49:41 +00002871 DeduplicateMethodAddressLiteral(invoke->GetTargetMethod()));
2872 break;
2873 case HInvokeStaticOrDirect::MethodLoadKind::kDexCachePcRelative: {
2874 // Add ADRP with its PC-relative DexCache access patch.
Vladimir Marko0f7dca42015-11-02 14:36:43 +00002875 pc_relative_dex_cache_patches_.emplace_back(*invoke->GetTargetMethod().dex_file,
2876 invoke->GetDexCacheArrayOffset());
2877 vixl::Label* pc_insn_label = &pc_relative_dex_cache_patches_.back().label;
Vladimir Marko58155012015-08-19 12:49:41 +00002878 {
2879 vixl::SingleEmissionCheckScope guard(GetVIXLAssembler());
Alexandre Rames6dc01742015-11-12 14:44:19 +00002880 __ Bind(pc_insn_label);
2881 __ adrp(XRegisterFrom(temp), 0);
Vladimir Marko58155012015-08-19 12:49:41 +00002882 }
Vladimir Marko0f7dca42015-11-02 14:36:43 +00002883 pc_relative_dex_cache_patches_.back().pc_insn_label = pc_insn_label;
Vladimir Marko58155012015-08-19 12:49:41 +00002884 // Add LDR with its PC-relative DexCache access patch.
Vladimir Marko0f7dca42015-11-02 14:36:43 +00002885 pc_relative_dex_cache_patches_.emplace_back(*invoke->GetTargetMethod().dex_file,
2886 invoke->GetDexCacheArrayOffset());
Alexandre Rames6dc01742015-11-12 14:44:19 +00002887 {
2888 vixl::SingleEmissionCheckScope guard(GetVIXLAssembler());
2889 __ Bind(&pc_relative_dex_cache_patches_.back().label);
2890 __ ldr(XRegisterFrom(temp), MemOperand(XRegisterFrom(temp), 0));
2891 pc_relative_dex_cache_patches_.back().pc_insn_label = pc_insn_label;
2892 }
Vladimir Marko58155012015-08-19 12:49:41 +00002893 break;
Vladimir Marko9b688a02015-05-06 14:12:42 +01002894 }
Vladimir Marko58155012015-08-19 12:49:41 +00002895 case HInvokeStaticOrDirect::MethodLoadKind::kDexCacheViaMethod: {
2896 Location current_method = invoke->GetLocations()->InAt(invoke->GetCurrentMethodInputIndex());
2897 Register reg = XRegisterFrom(temp);
2898 Register method_reg;
2899 if (current_method.IsRegister()) {
2900 method_reg = XRegisterFrom(current_method);
2901 } else {
2902 DCHECK(invoke->GetLocations()->Intrinsified());
2903 DCHECK(!current_method.IsValid());
2904 method_reg = reg;
2905 __ Ldr(reg.X(), MemOperand(sp, kCurrentMethodStackOffset));
2906 }
Vladimir Markob2c431e2015-08-19 12:45:42 +00002907
Vladimir Marko58155012015-08-19 12:49:41 +00002908 // temp = current_method->dex_cache_resolved_methods_;
Vladimir Marko05792b92015-08-03 11:56:49 +01002909 __ Ldr(reg.X(),
2910 MemOperand(method_reg.X(),
2911 ArtMethod::DexCacheResolvedMethodsOffset(kArm64WordSize).Int32Value()));
Vladimir Marko58155012015-08-19 12:49:41 +00002912 // temp = temp[index_in_cache];
2913 uint32_t index_in_cache = invoke->GetTargetMethod().dex_method_index;
2914 __ Ldr(reg.X(), MemOperand(reg.X(), GetCachePointerOffset(index_in_cache)));
2915 break;
2916 }
2917 }
2918
2919 switch (invoke->GetCodePtrLocation()) {
2920 case HInvokeStaticOrDirect::CodePtrLocation::kCallSelf:
2921 __ Bl(&frame_entry_label_);
2922 break;
2923 case HInvokeStaticOrDirect::CodePtrLocation::kCallPCRelative: {
2924 relative_call_patches_.emplace_back(invoke->GetTargetMethod());
2925 vixl::Label* label = &relative_call_patches_.back().label;
Alexandre Rames6dc01742015-11-12 14:44:19 +00002926 vixl::SingleEmissionCheckScope guard(GetVIXLAssembler());
2927 __ Bind(label);
2928 __ bl(0); // Branch and link to itself. This will be overriden at link time.
Vladimir Marko58155012015-08-19 12:49:41 +00002929 break;
2930 }
2931 case HInvokeStaticOrDirect::CodePtrLocation::kCallDirectWithFixup:
2932 case HInvokeStaticOrDirect::CodePtrLocation::kCallDirect:
2933 // LR prepared above for better instruction scheduling.
2934 DCHECK(direct_code_loaded);
2935 // lr()
2936 __ Blr(lr);
2937 break;
2938 case HInvokeStaticOrDirect::CodePtrLocation::kCallArtMethod:
2939 // LR = callee_method->entry_point_from_quick_compiled_code_;
2940 __ Ldr(lr, MemOperand(
Alexandre Rames6dc01742015-11-12 14:44:19 +00002941 XRegisterFrom(callee_method),
Vladimir Marko58155012015-08-19 12:49:41 +00002942 ArtMethod::EntryPointFromQuickCompiledCodeOffset(kArm64WordSize).Int32Value()));
2943 // lr()
2944 __ Blr(lr);
2945 break;
Nicolas Geoffray1cf95282014-12-12 19:22:03 +00002946 }
Alexandre Rames5319def2014-10-23 10:03:10 +01002947
Andreas Gampe878d58c2015-01-15 23:24:00 -08002948 DCHECK(!IsLeafMethod());
2949}
2950
Andreas Gampebfb5ba92015-09-01 15:45:02 +00002951void CodeGeneratorARM64::GenerateVirtualCall(HInvokeVirtual* invoke, Location temp_in) {
2952 LocationSummary* locations = invoke->GetLocations();
2953 Location receiver = locations->InAt(0);
2954 Register temp = XRegisterFrom(temp_in);
2955 size_t method_offset = mirror::Class::EmbeddedVTableEntryOffset(
2956 invoke->GetVTableIndex(), kArm64PointerSize).SizeValue();
2957 Offset class_offset = mirror::Object::ClassOffset();
2958 Offset entry_point = ArtMethod::EntryPointFromQuickCompiledCodeOffset(kArm64WordSize);
2959
2960 BlockPoolsScope block_pools(GetVIXLAssembler());
2961
2962 DCHECK(receiver.IsRegister());
2963 __ Ldr(temp.W(), HeapOperandFrom(receiver, class_offset));
2964 MaybeRecordImplicitNullCheck(invoke);
2965 GetAssembler()->MaybeUnpoisonHeapReference(temp.W());
2966 // temp = temp->GetMethodAt(method_offset);
2967 __ Ldr(temp, MemOperand(temp, method_offset));
2968 // lr = temp->GetEntryPoint();
2969 __ Ldr(lr, MemOperand(temp, entry_point.SizeValue()));
2970 // lr();
2971 __ Blr(lr);
2972}
2973
Vladimir Marko58155012015-08-19 12:49:41 +00002974void CodeGeneratorARM64::EmitLinkerPatches(ArenaVector<LinkerPatch>* linker_patches) {
2975 DCHECK(linker_patches->empty());
2976 size_t size =
2977 method_patches_.size() +
2978 call_patches_.size() +
2979 relative_call_patches_.size() +
Vladimir Marko0f7dca42015-11-02 14:36:43 +00002980 pc_relative_dex_cache_patches_.size();
Vladimir Marko58155012015-08-19 12:49:41 +00002981 linker_patches->reserve(size);
2982 for (const auto& entry : method_patches_) {
2983 const MethodReference& target_method = entry.first;
2984 vixl::Literal<uint64_t>* literal = entry.second;
2985 linker_patches->push_back(LinkerPatch::MethodPatch(literal->offset(),
2986 target_method.dex_file,
2987 target_method.dex_method_index));
2988 }
2989 for (const auto& entry : call_patches_) {
2990 const MethodReference& target_method = entry.first;
2991 vixl::Literal<uint64_t>* literal = entry.second;
2992 linker_patches->push_back(LinkerPatch::CodePatch(literal->offset(),
2993 target_method.dex_file,
2994 target_method.dex_method_index));
2995 }
2996 for (const MethodPatchInfo<vixl::Label>& info : relative_call_patches_) {
Alexandre Rames6dc01742015-11-12 14:44:19 +00002997 linker_patches->push_back(LinkerPatch::RelativeCodePatch(info.label.location(),
Vladimir Marko58155012015-08-19 12:49:41 +00002998 info.target_method.dex_file,
2999 info.target_method.dex_method_index));
3000 }
Vladimir Marko0f7dca42015-11-02 14:36:43 +00003001 for (const PcRelativeDexCacheAccessInfo& info : pc_relative_dex_cache_patches_) {
Alexandre Rames6dc01742015-11-12 14:44:19 +00003002 linker_patches->push_back(LinkerPatch::DexCacheArrayPatch(info.label.location(),
Vladimir Marko58155012015-08-19 12:49:41 +00003003 &info.target_dex_file,
Alexandre Rames6dc01742015-11-12 14:44:19 +00003004 info.pc_insn_label->location(),
Vladimir Marko58155012015-08-19 12:49:41 +00003005 info.element_offset));
3006 }
3007}
3008
3009vixl::Literal<uint64_t>* CodeGeneratorARM64::DeduplicateUint64Literal(uint64_t value) {
3010 // Look up the literal for value.
3011 auto lb = uint64_literals_.lower_bound(value);
3012 if (lb != uint64_literals_.end() && !uint64_literals_.key_comp()(value, lb->first)) {
3013 return lb->second;
3014 }
3015 // We don't have a literal for this value, insert a new one.
3016 vixl::Literal<uint64_t>* literal = __ CreateLiteralDestroyedWithPool<uint64_t>(value);
3017 uint64_literals_.PutBefore(lb, value, literal);
3018 return literal;
3019}
3020
3021vixl::Literal<uint64_t>* CodeGeneratorARM64::DeduplicateMethodLiteral(
3022 MethodReference target_method,
3023 MethodToLiteralMap* map) {
3024 // Look up the literal for target_method.
3025 auto lb = map->lower_bound(target_method);
3026 if (lb != map->end() && !map->key_comp()(target_method, lb->first)) {
3027 return lb->second;
3028 }
3029 // We don't have a literal for this method yet, insert a new one.
3030 vixl::Literal<uint64_t>* literal = __ CreateLiteralDestroyedWithPool<uint64_t>(0u);
3031 map->PutBefore(lb, target_method, literal);
3032 return literal;
3033}
3034
3035vixl::Literal<uint64_t>* CodeGeneratorARM64::DeduplicateMethodAddressLiteral(
3036 MethodReference target_method) {
3037 return DeduplicateMethodLiteral(target_method, &method_patches_);
3038}
3039
3040vixl::Literal<uint64_t>* CodeGeneratorARM64::DeduplicateMethodCodeLiteral(
3041 MethodReference target_method) {
3042 return DeduplicateMethodLiteral(target_method, &call_patches_);
3043}
3044
3045
Andreas Gampe878d58c2015-01-15 23:24:00 -08003046void InstructionCodeGeneratorARM64::VisitInvokeStaticOrDirect(HInvokeStaticOrDirect* invoke) {
Roland Levillain3e3d7332015-04-28 11:00:54 +01003047 // When we do not run baseline, explicit clinit checks triggered by static
3048 // invokes must have been pruned by art::PrepareForRegisterAllocation.
3049 DCHECK(codegen_->IsBaseline() || !invoke->IsStaticWithExplicitClinitCheck());
Roland Levillain4c0eb422015-04-24 16:43:49 +01003050
Andreas Gampe878d58c2015-01-15 23:24:00 -08003051 if (TryGenerateIntrinsicCode(invoke, codegen_)) {
3052 return;
3053 }
3054
Alexandre Ramesd921d642015-04-16 15:07:16 +01003055 BlockPoolsScope block_pools(GetVIXLAssembler());
Nicolas Geoffray38207af2015-06-01 15:46:22 +01003056 LocationSummary* locations = invoke->GetLocations();
3057 codegen_->GenerateStaticOrDirectCall(
3058 invoke, locations->HasTemps() ? locations->GetTemp(0) : Location::NoLocation());
Nicolas Geoffraya8ac9132015-03-13 16:36:36 +00003059 codegen_->RecordPcInfo(invoke, invoke->GetDexPc());
Alexandre Rames5319def2014-10-23 10:03:10 +01003060}
3061
3062void InstructionCodeGeneratorARM64::VisitInvokeVirtual(HInvokeVirtual* invoke) {
Andreas Gampe878d58c2015-01-15 23:24:00 -08003063 if (TryGenerateIntrinsicCode(invoke, codegen_)) {
3064 return;
3065 }
3066
Andreas Gampebfb5ba92015-09-01 15:45:02 +00003067 codegen_->GenerateVirtualCall(invoke, invoke->GetLocations()->GetTemp(0));
Alexandre Rames5319def2014-10-23 10:03:10 +01003068 DCHECK(!codegen_->IsLeafMethod());
3069 codegen_->RecordPcInfo(invoke, invoke->GetDexPc());
3070}
3071
Alexandre Rames67555f72014-11-18 10:55:16 +00003072void LocationsBuilderARM64::VisitLoadClass(HLoadClass* cls) {
Calin Juravle98893e12015-10-02 21:05:03 +01003073 InvokeRuntimeCallingConvention calling_convention;
3074 CodeGenerator::CreateLoadClassLocationSummary(
3075 cls,
3076 LocationFrom(calling_convention.GetRegisterAt(0)),
3077 LocationFrom(vixl::x0));
Alexandre Rames67555f72014-11-18 10:55:16 +00003078}
3079
3080void InstructionCodeGeneratorARM64::VisitLoadClass(HLoadClass* cls) {
Calin Juravle98893e12015-10-02 21:05:03 +01003081 if (cls->NeedsAccessCheck()) {
3082 codegen_->MoveConstant(cls->GetLocations()->GetTemp(0), cls->GetTypeIndex());
3083 codegen_->InvokeRuntime(QUICK_ENTRY_POINT(pInitializeTypeAndVerifyAccess),
3084 cls,
3085 cls->GetDexPc(),
3086 nullptr);
Calin Juravle580b6092015-10-06 17:35:58 +01003087 return;
3088 }
3089
3090 Register out = OutputRegister(cls);
3091 Register current_method = InputRegisterAt(cls, 0);
3092 if (cls->IsReferrersClass()) {
Alexandre Rames67555f72014-11-18 10:55:16 +00003093 DCHECK(!cls->CanCallRuntime());
3094 DCHECK(!cls->MustGenerateClinitCheck());
Mathieu Chartiere401d142015-04-22 13:56:20 -07003095 __ Ldr(out, MemOperand(current_method, ArtMethod::DeclaringClassOffset().Int32Value()));
Alexandre Rames67555f72014-11-18 10:55:16 +00003096 } else {
3097 DCHECK(cls->CanCallRuntime());
Vladimir Marko05792b92015-08-03 11:56:49 +01003098 MemberOffset resolved_types_offset = ArtMethod::DexCacheResolvedTypesOffset(kArm64PointerSize);
3099 __ Ldr(out.X(), MemOperand(current_method, resolved_types_offset.Int32Value()));
3100 __ Ldr(out, MemOperand(out.X(), CodeGenerator::GetCacheOffset(cls->GetTypeIndex())));
3101 // TODO: We will need a read barrier here.
Alexandre Rames67555f72014-11-18 10:55:16 +00003102
3103 SlowPathCodeARM64* slow_path = new (GetGraph()->GetArena()) LoadClassSlowPathARM64(
3104 cls, cls, cls->GetDexPc(), cls->MustGenerateClinitCheck());
3105 codegen_->AddSlowPath(slow_path);
3106 __ Cbz(out, slow_path->GetEntryLabel());
3107 if (cls->MustGenerateClinitCheck()) {
3108 GenerateClassInitializationCheck(slow_path, out);
3109 } else {
3110 __ Bind(slow_path->GetExitLabel());
3111 }
3112 }
3113}
3114
David Brazdilcb1c0552015-08-04 16:22:25 +01003115static MemOperand GetExceptionTlsAddress() {
3116 return MemOperand(tr, Thread::ExceptionOffset<kArm64WordSize>().Int32Value());
3117}
3118
Alexandre Rames67555f72014-11-18 10:55:16 +00003119void LocationsBuilderARM64::VisitLoadException(HLoadException* load) {
3120 LocationSummary* locations =
3121 new (GetGraph()->GetArena()) LocationSummary(load, LocationSummary::kNoCall);
3122 locations->SetOut(Location::RequiresRegister());
3123}
3124
3125void InstructionCodeGeneratorARM64::VisitLoadException(HLoadException* instruction) {
David Brazdilcb1c0552015-08-04 16:22:25 +01003126 __ Ldr(OutputRegister(instruction), GetExceptionTlsAddress());
3127}
3128
3129void LocationsBuilderARM64::VisitClearException(HClearException* clear) {
3130 new (GetGraph()->GetArena()) LocationSummary(clear, LocationSummary::kNoCall);
3131}
3132
3133void InstructionCodeGeneratorARM64::VisitClearException(HClearException* clear ATTRIBUTE_UNUSED) {
3134 __ Str(wzr, GetExceptionTlsAddress());
Alexandre Rames67555f72014-11-18 10:55:16 +00003135}
3136
Alexandre Rames5319def2014-10-23 10:03:10 +01003137void LocationsBuilderARM64::VisitLoadLocal(HLoadLocal* load) {
3138 load->SetLocations(nullptr);
3139}
3140
Roland Levillain4b8f1ec2015-08-26 18:34:03 +01003141void InstructionCodeGeneratorARM64::VisitLoadLocal(HLoadLocal* load ATTRIBUTE_UNUSED) {
Alexandre Rames5319def2014-10-23 10:03:10 +01003142 // Nothing to do, this is driven by the code generator.
3143}
3144
Alexandre Rames67555f72014-11-18 10:55:16 +00003145void LocationsBuilderARM64::VisitLoadString(HLoadString* load) {
3146 LocationSummary* locations =
3147 new (GetGraph()->GetArena()) LocationSummary(load, LocationSummary::kCallOnSlowPath);
Nicolas Geoffrayfbdaa302015-05-29 12:06:56 +01003148 locations->SetInAt(0, Location::RequiresRegister());
Alexandre Rames67555f72014-11-18 10:55:16 +00003149 locations->SetOut(Location::RequiresRegister());
3150}
3151
3152void InstructionCodeGeneratorARM64::VisitLoadString(HLoadString* load) {
3153 SlowPathCodeARM64* slow_path = new (GetGraph()->GetArena()) LoadStringSlowPathARM64(load);
3154 codegen_->AddSlowPath(slow_path);
3155
3156 Register out = OutputRegister(load);
Nicolas Geoffrayfbdaa302015-05-29 12:06:56 +01003157 Register current_method = InputRegisterAt(load, 0);
Mathieu Chartiere401d142015-04-22 13:56:20 -07003158 __ Ldr(out, MemOperand(current_method, ArtMethod::DeclaringClassOffset().Int32Value()));
Vladimir Marko05792b92015-08-03 11:56:49 +01003159 __ Ldr(out.X(), HeapOperand(out, mirror::Class::DexCacheStringsOffset()));
3160 __ Ldr(out, MemOperand(out.X(), CodeGenerator::GetCacheOffset(load->GetStringIndex())));
3161 // TODO: We will need a read barrier here.
Alexandre Rames67555f72014-11-18 10:55:16 +00003162 __ Cbz(out, slow_path->GetEntryLabel());
3163 __ Bind(slow_path->GetExitLabel());
3164}
3165
Alexandre Rames5319def2014-10-23 10:03:10 +01003166void LocationsBuilderARM64::VisitLocal(HLocal* local) {
3167 local->SetLocations(nullptr);
3168}
3169
3170void InstructionCodeGeneratorARM64::VisitLocal(HLocal* local) {
3171 DCHECK_EQ(local->GetBlock(), GetGraph()->GetEntryBlock());
3172}
3173
3174void LocationsBuilderARM64::VisitLongConstant(HLongConstant* constant) {
3175 LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(constant);
3176 locations->SetOut(Location::ConstantLocation(constant));
3177}
3178
Roland Levillain4b8f1ec2015-08-26 18:34:03 +01003179void InstructionCodeGeneratorARM64::VisitLongConstant(HLongConstant* constant ATTRIBUTE_UNUSED) {
Alexandre Rames5319def2014-10-23 10:03:10 +01003180 // Will be generated at use site.
3181}
3182
Alexandre Rames67555f72014-11-18 10:55:16 +00003183void LocationsBuilderARM64::VisitMonitorOperation(HMonitorOperation* instruction) {
3184 LocationSummary* locations =
3185 new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kCall);
3186 InvokeRuntimeCallingConvention calling_convention;
3187 locations->SetInAt(0, LocationFrom(calling_convention.GetRegisterAt(0)));
3188}
3189
3190void InstructionCodeGeneratorARM64::VisitMonitorOperation(HMonitorOperation* instruction) {
3191 codegen_->InvokeRuntime(instruction->IsEnter()
3192 ? QUICK_ENTRY_POINT(pLockObject) : QUICK_ENTRY_POINT(pUnlockObject),
3193 instruction,
Nicolas Geoffrayeeefa122015-03-13 18:52:59 +00003194 instruction->GetDexPc(),
3195 nullptr);
Andreas Gampe1cc7dba2014-12-17 18:43:01 -08003196 CheckEntrypointTypes<kQuickLockObject, void, mirror::Object*>();
Alexandre Rames67555f72014-11-18 10:55:16 +00003197}
3198
Alexandre Rames42d641b2014-10-27 14:00:51 +00003199void LocationsBuilderARM64::VisitMul(HMul* mul) {
3200 LocationSummary* locations =
3201 new (GetGraph()->GetArena()) LocationSummary(mul, LocationSummary::kNoCall);
3202 switch (mul->GetResultType()) {
3203 case Primitive::kPrimInt:
3204 case Primitive::kPrimLong:
3205 locations->SetInAt(0, Location::RequiresRegister());
3206 locations->SetInAt(1, Location::RequiresRegister());
Alexandre Ramesfb4e5fa2014-11-06 12:41:16 +00003207 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
Alexandre Rames42d641b2014-10-27 14:00:51 +00003208 break;
3209
3210 case Primitive::kPrimFloat:
3211 case Primitive::kPrimDouble:
Alexandre Ramesa89086e2014-11-07 17:13:25 +00003212 locations->SetInAt(0, Location::RequiresFpuRegister());
3213 locations->SetInAt(1, Location::RequiresFpuRegister());
Alexandre Rames67555f72014-11-18 10:55:16 +00003214 locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap);
Alexandre Rames42d641b2014-10-27 14:00:51 +00003215 break;
3216
3217 default:
3218 LOG(FATAL) << "Unexpected mul type " << mul->GetResultType();
3219 }
3220}
3221
3222void InstructionCodeGeneratorARM64::VisitMul(HMul* mul) {
3223 switch (mul->GetResultType()) {
3224 case Primitive::kPrimInt:
3225 case Primitive::kPrimLong:
3226 __ Mul(OutputRegister(mul), InputRegisterAt(mul, 0), InputRegisterAt(mul, 1));
3227 break;
3228
3229 case Primitive::kPrimFloat:
3230 case Primitive::kPrimDouble:
Alexandre Ramesa89086e2014-11-07 17:13:25 +00003231 __ Fmul(OutputFPRegister(mul), InputFPRegisterAt(mul, 0), InputFPRegisterAt(mul, 1));
Alexandre Rames42d641b2014-10-27 14:00:51 +00003232 break;
3233
3234 default:
3235 LOG(FATAL) << "Unexpected mul type " << mul->GetResultType();
3236 }
3237}
3238
Alexandre Ramesfc19de82014-11-07 17:13:31 +00003239void LocationsBuilderARM64::VisitNeg(HNeg* neg) {
3240 LocationSummary* locations =
3241 new (GetGraph()->GetArena()) LocationSummary(neg, LocationSummary::kNoCall);
3242 switch (neg->GetResultType()) {
3243 case Primitive::kPrimInt:
Alexandre Rames67555f72014-11-18 10:55:16 +00003244 case Primitive::kPrimLong:
Serban Constantinescu2d35d9d2015-02-22 22:08:01 +00003245 locations->SetInAt(0, ARM64EncodableConstantOrRegister(neg->InputAt(0), neg));
Alexandre Rames67555f72014-11-18 10:55:16 +00003246 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
Alexandre Ramesfc19de82014-11-07 17:13:31 +00003247 break;
Alexandre Ramesfc19de82014-11-07 17:13:31 +00003248
3249 case Primitive::kPrimFloat:
3250 case Primitive::kPrimDouble:
Alexandre Rames67555f72014-11-18 10:55:16 +00003251 locations->SetInAt(0, Location::RequiresFpuRegister());
3252 locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap);
Alexandre Ramesfc19de82014-11-07 17:13:31 +00003253 break;
3254
3255 default:
3256 LOG(FATAL) << "Unexpected neg type " << neg->GetResultType();
3257 }
3258}
3259
3260void InstructionCodeGeneratorARM64::VisitNeg(HNeg* neg) {
3261 switch (neg->GetResultType()) {
3262 case Primitive::kPrimInt:
3263 case Primitive::kPrimLong:
3264 __ Neg(OutputRegister(neg), InputOperandAt(neg, 0));
3265 break;
3266
3267 case Primitive::kPrimFloat:
3268 case Primitive::kPrimDouble:
Alexandre Rames67555f72014-11-18 10:55:16 +00003269 __ Fneg(OutputFPRegister(neg), InputFPRegisterAt(neg, 0));
Alexandre Ramesfc19de82014-11-07 17:13:31 +00003270 break;
3271
3272 default:
3273 LOG(FATAL) << "Unexpected neg type " << neg->GetResultType();
3274 }
3275}
3276
3277void LocationsBuilderARM64::VisitNewArray(HNewArray* instruction) {
3278 LocationSummary* locations =
3279 new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kCall);
3280 InvokeRuntimeCallingConvention calling_convention;
3281 locations->AddTemp(LocationFrom(calling_convention.GetRegisterAt(0)));
Alexandre Ramesfc19de82014-11-07 17:13:31 +00003282 locations->SetOut(LocationFrom(x0));
Andreas Gampe1cc7dba2014-12-17 18:43:01 -08003283 locations->SetInAt(0, LocationFrom(calling_convention.GetRegisterAt(1)));
Nicolas Geoffray69aa6012015-06-09 10:34:25 +01003284 locations->SetInAt(1, LocationFrom(calling_convention.GetRegisterAt(2)));
Andreas Gampe1cc7dba2014-12-17 18:43:01 -08003285 CheckEntrypointTypes<kQuickAllocArrayWithAccessCheck,
Mathieu Chartiere401d142015-04-22 13:56:20 -07003286 void*, uint32_t, int32_t, ArtMethod*>();
Alexandre Ramesfc19de82014-11-07 17:13:31 +00003287}
3288
3289void InstructionCodeGeneratorARM64::VisitNewArray(HNewArray* instruction) {
3290 LocationSummary* locations = instruction->GetLocations();
3291 InvokeRuntimeCallingConvention calling_convention;
3292 Register type_index = RegisterFrom(locations->GetTemp(0), Primitive::kPrimInt);
3293 DCHECK(type_index.Is(w0));
Alexandre Ramesfc19de82014-11-07 17:13:31 +00003294 __ Mov(type_index, instruction->GetTypeIndex());
Roland Levillain4d027112015-07-01 15:41:14 +01003295 // Note: if heap poisoning is enabled, the entry point takes cares
3296 // of poisoning the reference.
Calin Juravle175dc732015-08-25 15:42:32 +01003297 codegen_->InvokeRuntime(instruction->GetEntrypoint(),
3298 instruction,
3299 instruction->GetDexPc(),
3300 nullptr);
Mathieu Chartiere401d142015-04-22 13:56:20 -07003301 CheckEntrypointTypes<kQuickAllocArrayWithAccessCheck, void*, uint32_t, int32_t, ArtMethod*>();
Alexandre Ramesfc19de82014-11-07 17:13:31 +00003302}
3303
Alexandre Rames5319def2014-10-23 10:03:10 +01003304void LocationsBuilderARM64::VisitNewInstance(HNewInstance* instruction) {
3305 LocationSummary* locations =
3306 new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kCall);
3307 InvokeRuntimeCallingConvention calling_convention;
3308 locations->AddTemp(LocationFrom(calling_convention.GetRegisterAt(0)));
Nicolas Geoffray69aa6012015-06-09 10:34:25 +01003309 locations->SetInAt(0, LocationFrom(calling_convention.GetRegisterAt(1)));
Alexandre Rames5319def2014-10-23 10:03:10 +01003310 locations->SetOut(calling_convention.GetReturnLocation(Primitive::kPrimNot));
Mathieu Chartiere401d142015-04-22 13:56:20 -07003311 CheckEntrypointTypes<kQuickAllocObjectWithAccessCheck, void*, uint32_t, ArtMethod*>();
Alexandre Rames5319def2014-10-23 10:03:10 +01003312}
3313
3314void InstructionCodeGeneratorARM64::VisitNewInstance(HNewInstance* instruction) {
3315 LocationSummary* locations = instruction->GetLocations();
3316 Register type_index = RegisterFrom(locations->GetTemp(0), Primitive::kPrimInt);
3317 DCHECK(type_index.Is(w0));
Alexandre Rames5319def2014-10-23 10:03:10 +01003318 __ Mov(type_index, instruction->GetTypeIndex());
Roland Levillain4d027112015-07-01 15:41:14 +01003319 // Note: if heap poisoning is enabled, the entry point takes cares
3320 // of poisoning the reference.
Calin Juravle175dc732015-08-25 15:42:32 +01003321 codegen_->InvokeRuntime(instruction->GetEntrypoint(),
3322 instruction,
3323 instruction->GetDexPc(),
3324 nullptr);
Mathieu Chartiere401d142015-04-22 13:56:20 -07003325 CheckEntrypointTypes<kQuickAllocObjectWithAccessCheck, void*, uint32_t, ArtMethod*>();
Alexandre Rames5319def2014-10-23 10:03:10 +01003326}
3327
3328void LocationsBuilderARM64::VisitNot(HNot* instruction) {
3329 LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instruction);
Alexandre Rames4e596512014-11-07 15:56:50 +00003330 locations->SetInAt(0, Location::RequiresRegister());
Alexandre Ramesfb4e5fa2014-11-06 12:41:16 +00003331 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
Alexandre Rames5319def2014-10-23 10:03:10 +01003332}
3333
3334void InstructionCodeGeneratorARM64::VisitNot(HNot* instruction) {
Nicolas Geoffrayd8ef2e92015-02-24 16:02:06 +00003335 switch (instruction->GetResultType()) {
Alexandre Rames5319def2014-10-23 10:03:10 +01003336 case Primitive::kPrimInt:
Alexandre Rames5319def2014-10-23 10:03:10 +01003337 case Primitive::kPrimLong:
Roland Levillain55dcfb52014-10-24 18:09:09 +01003338 __ Mvn(OutputRegister(instruction), InputOperandAt(instruction, 0));
Alexandre Rames5319def2014-10-23 10:03:10 +01003339 break;
3340
3341 default:
3342 LOG(FATAL) << "Unexpected type for not operation " << instruction->GetResultType();
3343 }
3344}
3345
David Brazdil66d126e2015-04-03 16:02:44 +01003346void LocationsBuilderARM64::VisitBooleanNot(HBooleanNot* instruction) {
3347 LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instruction);
3348 locations->SetInAt(0, Location::RequiresRegister());
3349 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
3350}
3351
3352void InstructionCodeGeneratorARM64::VisitBooleanNot(HBooleanNot* instruction) {
David Brazdil66d126e2015-04-03 16:02:44 +01003353 __ Eor(OutputRegister(instruction), InputRegisterAt(instruction, 0), vixl::Operand(1));
3354}
3355
Alexandre Rames5319def2014-10-23 10:03:10 +01003356void LocationsBuilderARM64::VisitNullCheck(HNullCheck* instruction) {
David Brazdil77a48ae2015-09-15 12:34:04 +00003357 LocationSummary::CallKind call_kind = instruction->CanThrowIntoCatchBlock()
3358 ? LocationSummary::kCallOnSlowPath
3359 : LocationSummary::kNoCall;
3360 LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instruction, call_kind);
Alexandre Rames5319def2014-10-23 10:03:10 +01003361 locations->SetInAt(0, Location::RequiresRegister());
3362 if (instruction->HasUses()) {
3363 locations->SetOut(Location::SameAsFirstInput());
3364 }
3365}
3366
Calin Juravlecd6dffe2015-01-08 17:35:35 +00003367void InstructionCodeGeneratorARM64::GenerateImplicitNullCheck(HNullCheck* instruction) {
Calin Juravle77520bc2015-01-12 18:45:46 +00003368 if (codegen_->CanMoveNullCheckToUser(instruction)) {
3369 return;
3370 }
Calin Juravlecd6dffe2015-01-08 17:35:35 +00003371
Alexandre Ramesd921d642015-04-16 15:07:16 +01003372 BlockPoolsScope block_pools(GetVIXLAssembler());
3373 Location obj = instruction->GetLocations()->InAt(0);
Calin Juravlecd6dffe2015-01-08 17:35:35 +00003374 __ Ldr(wzr, HeapOperandFrom(obj, Offset(0)));
3375 codegen_->RecordPcInfo(instruction, instruction->GetDexPc());
3376}
3377
3378void InstructionCodeGeneratorARM64::GenerateExplicitNullCheck(HNullCheck* instruction) {
Alexandre Rames5319def2014-10-23 10:03:10 +01003379 SlowPathCodeARM64* slow_path = new (GetGraph()->GetArena()) NullCheckSlowPathARM64(instruction);
3380 codegen_->AddSlowPath(slow_path);
3381
3382 LocationSummary* locations = instruction->GetLocations();
3383 Location obj = locations->InAt(0);
Calin Juravle77520bc2015-01-12 18:45:46 +00003384
3385 __ Cbz(RegisterFrom(obj, instruction->InputAt(0)->GetType()), slow_path->GetEntryLabel());
Alexandre Rames5319def2014-10-23 10:03:10 +01003386}
3387
Calin Juravlecd6dffe2015-01-08 17:35:35 +00003388void InstructionCodeGeneratorARM64::VisitNullCheck(HNullCheck* instruction) {
David Brazdil77a48ae2015-09-15 12:34:04 +00003389 if (codegen_->IsImplicitNullCheckAllowed(instruction)) {
Calin Juravlecd6dffe2015-01-08 17:35:35 +00003390 GenerateImplicitNullCheck(instruction);
3391 } else {
3392 GenerateExplicitNullCheck(instruction);
3393 }
3394}
3395
Alexandre Rames67555f72014-11-18 10:55:16 +00003396void LocationsBuilderARM64::VisitOr(HOr* instruction) {
3397 HandleBinaryOp(instruction);
3398}
3399
3400void InstructionCodeGeneratorARM64::VisitOr(HOr* instruction) {
3401 HandleBinaryOp(instruction);
3402}
3403
Alexandre Rames3e69f162014-12-10 10:36:50 +00003404void LocationsBuilderARM64::VisitParallelMove(HParallelMove* instruction ATTRIBUTE_UNUSED) {
3405 LOG(FATAL) << "Unreachable";
3406}
3407
3408void InstructionCodeGeneratorARM64::VisitParallelMove(HParallelMove* instruction) {
3409 codegen_->GetMoveResolver()->EmitNativeCode(instruction);
3410}
3411
Alexandre Rames5319def2014-10-23 10:03:10 +01003412void LocationsBuilderARM64::VisitParameterValue(HParameterValue* instruction) {
3413 LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instruction);
3414 Location location = parameter_visitor_.GetNextLocation(instruction->GetType());
3415 if (location.IsStackSlot()) {
3416 location = Location::StackSlot(location.GetStackIndex() + codegen_->GetFrameSize());
3417 } else if (location.IsDoubleStackSlot()) {
3418 location = Location::DoubleStackSlot(location.GetStackIndex() + codegen_->GetFrameSize());
3419 }
3420 locations->SetOut(location);
3421}
3422
Nicolas Geoffray76b1e172015-05-27 17:18:33 +01003423void InstructionCodeGeneratorARM64::VisitParameterValue(
3424 HParameterValue* instruction ATTRIBUTE_UNUSED) {
Alexandre Rames5319def2014-10-23 10:03:10 +01003425 // Nothing to do, the parameter is already at its location.
Nicolas Geoffray76b1e172015-05-27 17:18:33 +01003426}
3427
3428void LocationsBuilderARM64::VisitCurrentMethod(HCurrentMethod* instruction) {
3429 LocationSummary* locations =
3430 new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall);
Nicolas Geoffray38207af2015-06-01 15:46:22 +01003431 locations->SetOut(LocationFrom(kArtMethodRegister));
Nicolas Geoffray76b1e172015-05-27 17:18:33 +01003432}
3433
3434void InstructionCodeGeneratorARM64::VisitCurrentMethod(
3435 HCurrentMethod* instruction ATTRIBUTE_UNUSED) {
3436 // Nothing to do, the method is already at its location.
Alexandre Rames5319def2014-10-23 10:03:10 +01003437}
3438
3439void LocationsBuilderARM64::VisitPhi(HPhi* instruction) {
3440 LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instruction);
3441 for (size_t i = 0, e = instruction->InputCount(); i < e; ++i) {
3442 locations->SetInAt(i, Location::Any());
3443 }
3444 locations->SetOut(Location::Any());
3445}
3446
Roland Levillain4b8f1ec2015-08-26 18:34:03 +01003447void InstructionCodeGeneratorARM64::VisitPhi(HPhi* instruction ATTRIBUTE_UNUSED) {
Alexandre Rames5319def2014-10-23 10:03:10 +01003448 LOG(FATAL) << "Unreachable";
3449}
3450
Serban Constantinescu02164b32014-11-13 14:05:07 +00003451void LocationsBuilderARM64::VisitRem(HRem* rem) {
Serban Constantinescu02d81cc2015-01-05 16:08:49 +00003452 Primitive::Type type = rem->GetResultType();
Alexandre Rames542361f2015-01-29 16:57:31 +00003453 LocationSummary::CallKind call_kind =
3454 Primitive::IsFloatingPointType(type) ? LocationSummary::kCall : LocationSummary::kNoCall;
Serban Constantinescu02d81cc2015-01-05 16:08:49 +00003455 LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(rem, call_kind);
3456
3457 switch (type) {
Serban Constantinescu02164b32014-11-13 14:05:07 +00003458 case Primitive::kPrimInt:
3459 case Primitive::kPrimLong:
3460 locations->SetInAt(0, Location::RequiresRegister());
Zheng Xuc6667102015-05-15 16:08:45 +08003461 locations->SetInAt(1, Location::RegisterOrConstant(rem->InputAt(1)));
Serban Constantinescu02164b32014-11-13 14:05:07 +00003462 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
3463 break;
3464
Serban Constantinescu02d81cc2015-01-05 16:08:49 +00003465 case Primitive::kPrimFloat:
3466 case Primitive::kPrimDouble: {
3467 InvokeRuntimeCallingConvention calling_convention;
3468 locations->SetInAt(0, LocationFrom(calling_convention.GetFpuRegisterAt(0)));
3469 locations->SetInAt(1, LocationFrom(calling_convention.GetFpuRegisterAt(1)));
3470 locations->SetOut(calling_convention.GetReturnLocation(type));
3471
3472 break;
3473 }
3474
Serban Constantinescu02164b32014-11-13 14:05:07 +00003475 default:
Serban Constantinescu02d81cc2015-01-05 16:08:49 +00003476 LOG(FATAL) << "Unexpected rem type " << type;
Serban Constantinescu02164b32014-11-13 14:05:07 +00003477 }
3478}
3479
3480void InstructionCodeGeneratorARM64::VisitRem(HRem* rem) {
3481 Primitive::Type type = rem->GetResultType();
Serban Constantinescu02d81cc2015-01-05 16:08:49 +00003482
Serban Constantinescu02164b32014-11-13 14:05:07 +00003483 switch (type) {
3484 case Primitive::kPrimInt:
3485 case Primitive::kPrimLong: {
Zheng Xuc6667102015-05-15 16:08:45 +08003486 GenerateDivRemIntegral(rem);
Serban Constantinescu02164b32014-11-13 14:05:07 +00003487 break;
3488 }
3489
Serban Constantinescu02d81cc2015-01-05 16:08:49 +00003490 case Primitive::kPrimFloat:
3491 case Primitive::kPrimDouble: {
3492 int32_t entry_offset = (type == Primitive::kPrimFloat) ? QUICK_ENTRY_POINT(pFmodf)
3493 : QUICK_ENTRY_POINT(pFmod);
Nicolas Geoffrayeeefa122015-03-13 18:52:59 +00003494 codegen_->InvokeRuntime(entry_offset, rem, rem->GetDexPc(), nullptr);
Serban Constantinescu02d81cc2015-01-05 16:08:49 +00003495 break;
3496 }
3497
Serban Constantinescu02164b32014-11-13 14:05:07 +00003498 default:
3499 LOG(FATAL) << "Unexpected rem type " << type;
3500 }
3501}
3502
Calin Juravle27df7582015-04-17 19:12:31 +01003503void LocationsBuilderARM64::VisitMemoryBarrier(HMemoryBarrier* memory_barrier) {
3504 memory_barrier->SetLocations(nullptr);
3505}
3506
3507void InstructionCodeGeneratorARM64::VisitMemoryBarrier(HMemoryBarrier* memory_barrier) {
3508 GenerateMemoryBarrier(memory_barrier->GetBarrierKind());
3509}
3510
Alexandre Rames5319def2014-10-23 10:03:10 +01003511void LocationsBuilderARM64::VisitReturn(HReturn* instruction) {
3512 LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instruction);
3513 Primitive::Type return_type = instruction->InputAt(0)->GetType();
Alexandre Ramesa89086e2014-11-07 17:13:25 +00003514 locations->SetInAt(0, ARM64ReturnLocation(return_type));
Alexandre Rames5319def2014-10-23 10:03:10 +01003515}
3516
Roland Levillain4b8f1ec2015-08-26 18:34:03 +01003517void InstructionCodeGeneratorARM64::VisitReturn(HReturn* instruction ATTRIBUTE_UNUSED) {
Alexandre Rames5319def2014-10-23 10:03:10 +01003518 codegen_->GenerateFrameExit();
Alexandre Rames5319def2014-10-23 10:03:10 +01003519}
3520
3521void LocationsBuilderARM64::VisitReturnVoid(HReturnVoid* instruction) {
3522 instruction->SetLocations(nullptr);
3523}
3524
Roland Levillain4b8f1ec2015-08-26 18:34:03 +01003525void InstructionCodeGeneratorARM64::VisitReturnVoid(HReturnVoid* instruction ATTRIBUTE_UNUSED) {
Alexandre Rames5319def2014-10-23 10:03:10 +01003526 codegen_->GenerateFrameExit();
Alexandre Rames5319def2014-10-23 10:03:10 +01003527}
3528
Serban Constantinescu02164b32014-11-13 14:05:07 +00003529void LocationsBuilderARM64::VisitShl(HShl* shl) {
3530 HandleShift(shl);
3531}
3532
3533void InstructionCodeGeneratorARM64::VisitShl(HShl* shl) {
3534 HandleShift(shl);
3535}
3536
3537void LocationsBuilderARM64::VisitShr(HShr* shr) {
3538 HandleShift(shr);
3539}
3540
3541void InstructionCodeGeneratorARM64::VisitShr(HShr* shr) {
3542 HandleShift(shr);
3543}
3544
Alexandre Rames5319def2014-10-23 10:03:10 +01003545void LocationsBuilderARM64::VisitStoreLocal(HStoreLocal* store) {
3546 LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(store);
3547 Primitive::Type field_type = store->InputAt(1)->GetType();
3548 switch (field_type) {
Alexandre Ramesa89086e2014-11-07 17:13:25 +00003549 case Primitive::kPrimNot:
Alexandre Rames5319def2014-10-23 10:03:10 +01003550 case Primitive::kPrimBoolean:
3551 case Primitive::kPrimByte:
3552 case Primitive::kPrimChar:
3553 case Primitive::kPrimShort:
3554 case Primitive::kPrimInt:
Alexandre Ramesa89086e2014-11-07 17:13:25 +00003555 case Primitive::kPrimFloat:
Alexandre Rames5319def2014-10-23 10:03:10 +01003556 locations->SetInAt(1, Location::StackSlot(codegen_->GetStackSlot(store->GetLocal())));
3557 break;
3558
3559 case Primitive::kPrimLong:
Alexandre Ramesa89086e2014-11-07 17:13:25 +00003560 case Primitive::kPrimDouble:
Alexandre Rames5319def2014-10-23 10:03:10 +01003561 locations->SetInAt(1, Location::DoubleStackSlot(codegen_->GetStackSlot(store->GetLocal())));
3562 break;
3563
3564 default:
3565 LOG(FATAL) << "Unimplemented local type " << field_type;
3566 }
3567}
3568
Roland Levillain4b8f1ec2015-08-26 18:34:03 +01003569void InstructionCodeGeneratorARM64::VisitStoreLocal(HStoreLocal* store ATTRIBUTE_UNUSED) {
Alexandre Rames5319def2014-10-23 10:03:10 +01003570}
3571
3572void LocationsBuilderARM64::VisitSub(HSub* instruction) {
Alexandre Rames67555f72014-11-18 10:55:16 +00003573 HandleBinaryOp(instruction);
Alexandre Rames5319def2014-10-23 10:03:10 +01003574}
3575
3576void InstructionCodeGeneratorARM64::VisitSub(HSub* instruction) {
Alexandre Rames67555f72014-11-18 10:55:16 +00003577 HandleBinaryOp(instruction);
Alexandre Rames5319def2014-10-23 10:03:10 +01003578}
3579
Alexandre Rames67555f72014-11-18 10:55:16 +00003580void LocationsBuilderARM64::VisitStaticFieldGet(HStaticFieldGet* instruction) {
Alexandre Rames09a99962015-04-15 11:47:56 +01003581 HandleFieldGet(instruction);
Alexandre Rames67555f72014-11-18 10:55:16 +00003582}
3583
3584void InstructionCodeGeneratorARM64::VisitStaticFieldGet(HStaticFieldGet* instruction) {
Alexandre Rames09a99962015-04-15 11:47:56 +01003585 HandleFieldGet(instruction, instruction->GetFieldInfo());
Alexandre Rames67555f72014-11-18 10:55:16 +00003586}
3587
3588void LocationsBuilderARM64::VisitStaticFieldSet(HStaticFieldSet* instruction) {
Alexandre Rames09a99962015-04-15 11:47:56 +01003589 HandleFieldSet(instruction);
Alexandre Rames5319def2014-10-23 10:03:10 +01003590}
3591
Alexandre Rames67555f72014-11-18 10:55:16 +00003592void InstructionCodeGeneratorARM64::VisitStaticFieldSet(HStaticFieldSet* instruction) {
Nicolas Geoffray07276db2015-05-18 14:22:09 +01003593 HandleFieldSet(instruction, instruction->GetFieldInfo(), instruction->GetValueCanBeNull());
Alexandre Rames5319def2014-10-23 10:03:10 +01003594}
3595
Calin Juravlee460d1d2015-09-29 04:52:17 +01003596void LocationsBuilderARM64::VisitUnresolvedInstanceFieldGet(
3597 HUnresolvedInstanceFieldGet* instruction) {
3598 FieldAccessCallingConventionARM64 calling_convention;
3599 codegen_->CreateUnresolvedFieldLocationSummary(
3600 instruction, instruction->GetFieldType(), calling_convention);
3601}
3602
3603void InstructionCodeGeneratorARM64::VisitUnresolvedInstanceFieldGet(
3604 HUnresolvedInstanceFieldGet* instruction) {
3605 FieldAccessCallingConventionARM64 calling_convention;
3606 codegen_->GenerateUnresolvedFieldAccess(instruction,
3607 instruction->GetFieldType(),
3608 instruction->GetFieldIndex(),
3609 instruction->GetDexPc(),
3610 calling_convention);
3611}
3612
3613void LocationsBuilderARM64::VisitUnresolvedInstanceFieldSet(
3614 HUnresolvedInstanceFieldSet* instruction) {
3615 FieldAccessCallingConventionARM64 calling_convention;
3616 codegen_->CreateUnresolvedFieldLocationSummary(
3617 instruction, instruction->GetFieldType(), calling_convention);
3618}
3619
3620void InstructionCodeGeneratorARM64::VisitUnresolvedInstanceFieldSet(
3621 HUnresolvedInstanceFieldSet* instruction) {
3622 FieldAccessCallingConventionARM64 calling_convention;
3623 codegen_->GenerateUnresolvedFieldAccess(instruction,
3624 instruction->GetFieldType(),
3625 instruction->GetFieldIndex(),
3626 instruction->GetDexPc(),
3627 calling_convention);
3628}
3629
3630void LocationsBuilderARM64::VisitUnresolvedStaticFieldGet(
3631 HUnresolvedStaticFieldGet* instruction) {
3632 FieldAccessCallingConventionARM64 calling_convention;
3633 codegen_->CreateUnresolvedFieldLocationSummary(
3634 instruction, instruction->GetFieldType(), calling_convention);
3635}
3636
3637void InstructionCodeGeneratorARM64::VisitUnresolvedStaticFieldGet(
3638 HUnresolvedStaticFieldGet* instruction) {
3639 FieldAccessCallingConventionARM64 calling_convention;
3640 codegen_->GenerateUnresolvedFieldAccess(instruction,
3641 instruction->GetFieldType(),
3642 instruction->GetFieldIndex(),
3643 instruction->GetDexPc(),
3644 calling_convention);
3645}
3646
3647void LocationsBuilderARM64::VisitUnresolvedStaticFieldSet(
3648 HUnresolvedStaticFieldSet* instruction) {
3649 FieldAccessCallingConventionARM64 calling_convention;
3650 codegen_->CreateUnresolvedFieldLocationSummary(
3651 instruction, instruction->GetFieldType(), calling_convention);
3652}
3653
3654void InstructionCodeGeneratorARM64::VisitUnresolvedStaticFieldSet(
3655 HUnresolvedStaticFieldSet* instruction) {
3656 FieldAccessCallingConventionARM64 calling_convention;
3657 codegen_->GenerateUnresolvedFieldAccess(instruction,
3658 instruction->GetFieldType(),
3659 instruction->GetFieldIndex(),
3660 instruction->GetDexPc(),
3661 calling_convention);
3662}
3663
Alexandre Rames5319def2014-10-23 10:03:10 +01003664void LocationsBuilderARM64::VisitSuspendCheck(HSuspendCheck* instruction) {
3665 new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kCallOnSlowPath);
3666}
3667
3668void InstructionCodeGeneratorARM64::VisitSuspendCheck(HSuspendCheck* instruction) {
Serban Constantinescu02164b32014-11-13 14:05:07 +00003669 HBasicBlock* block = instruction->GetBlock();
3670 if (block->GetLoopInformation() != nullptr) {
3671 DCHECK(block->GetLoopInformation()->GetSuspendCheck() == instruction);
3672 // The back edge will generate the suspend check.
3673 return;
3674 }
3675 if (block->IsEntryBlock() && instruction->GetNext()->IsGoto()) {
3676 // The goto will generate the suspend check.
3677 return;
3678 }
3679 GenerateSuspendCheck(instruction, nullptr);
Alexandre Rames5319def2014-10-23 10:03:10 +01003680}
3681
3682void LocationsBuilderARM64::VisitTemporary(HTemporary* temp) {
3683 temp->SetLocations(nullptr);
3684}
3685
Roland Levillain4b8f1ec2015-08-26 18:34:03 +01003686void InstructionCodeGeneratorARM64::VisitTemporary(HTemporary* temp ATTRIBUTE_UNUSED) {
Alexandre Rames5319def2014-10-23 10:03:10 +01003687 // Nothing to do, this is driven by the code generator.
Alexandre Rames5319def2014-10-23 10:03:10 +01003688}
3689
Alexandre Rames67555f72014-11-18 10:55:16 +00003690void LocationsBuilderARM64::VisitThrow(HThrow* instruction) {
3691 LocationSummary* locations =
3692 new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kCall);
3693 InvokeRuntimeCallingConvention calling_convention;
3694 locations->SetInAt(0, LocationFrom(calling_convention.GetRegisterAt(0)));
3695}
3696
3697void InstructionCodeGeneratorARM64::VisitThrow(HThrow* instruction) {
3698 codegen_->InvokeRuntime(
Nicolas Geoffrayeeefa122015-03-13 18:52:59 +00003699 QUICK_ENTRY_POINT(pDeliverException), instruction, instruction->GetDexPc(), nullptr);
Andreas Gampe1cc7dba2014-12-17 18:43:01 -08003700 CheckEntrypointTypes<kQuickDeliverException, void, mirror::Object*>();
Alexandre Rames67555f72014-11-18 10:55:16 +00003701}
3702
3703void LocationsBuilderARM64::VisitTypeConversion(HTypeConversion* conversion) {
3704 LocationSummary* locations =
3705 new (GetGraph()->GetArena()) LocationSummary(conversion, LocationSummary::kNoCall);
3706 Primitive::Type input_type = conversion->GetInputType();
3707 Primitive::Type result_type = conversion->GetResultType();
Nicolas Geoffray01fcc9e2014-12-01 14:16:20 +00003708 DCHECK_NE(input_type, result_type);
Alexandre Rames67555f72014-11-18 10:55:16 +00003709 if ((input_type == Primitive::kPrimNot) || (input_type == Primitive::kPrimVoid) ||
3710 (result_type == Primitive::kPrimNot) || (result_type == Primitive::kPrimVoid)) {
3711 LOG(FATAL) << "Unexpected type conversion from " << input_type << " to " << result_type;
3712 }
3713
Alexandre Rames542361f2015-01-29 16:57:31 +00003714 if (Primitive::IsFloatingPointType(input_type)) {
Alexandre Rames67555f72014-11-18 10:55:16 +00003715 locations->SetInAt(0, Location::RequiresFpuRegister());
3716 } else {
3717 locations->SetInAt(0, Location::RequiresRegister());
3718 }
3719
Alexandre Rames542361f2015-01-29 16:57:31 +00003720 if (Primitive::IsFloatingPointType(result_type)) {
Alexandre Rames67555f72014-11-18 10:55:16 +00003721 locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap);
3722 } else {
3723 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
3724 }
3725}
3726
3727void InstructionCodeGeneratorARM64::VisitTypeConversion(HTypeConversion* conversion) {
3728 Primitive::Type result_type = conversion->GetResultType();
3729 Primitive::Type input_type = conversion->GetInputType();
3730
3731 DCHECK_NE(input_type, result_type);
3732
Alexandre Rames542361f2015-01-29 16:57:31 +00003733 if (Primitive::IsIntegralType(result_type) && Primitive::IsIntegralType(input_type)) {
Alexandre Rames67555f72014-11-18 10:55:16 +00003734 int result_size = Primitive::ComponentSize(result_type);
3735 int input_size = Primitive::ComponentSize(input_type);
Alexandre Rames3e69f162014-12-10 10:36:50 +00003736 int min_size = std::min(result_size, input_size);
Serban Constantinescu02164b32014-11-13 14:05:07 +00003737 Register output = OutputRegister(conversion);
3738 Register source = InputRegisterAt(conversion, 0);
Alexandre Rames3e69f162014-12-10 10:36:50 +00003739 if ((result_type == Primitive::kPrimChar) && (input_size < result_size)) {
3740 __ Ubfx(output, source, 0, result_size * kBitsPerByte);
Alexandre Rames4dff2fd2015-08-20 13:36:35 +01003741 } else if (result_type == Primitive::kPrimInt && input_type == Primitive::kPrimLong) {
3742 // 'int' values are used directly as W registers, discarding the top
3743 // bits, so we don't need to sign-extend and can just perform a move.
3744 // We do not pass the `kDiscardForSameWReg` argument to force clearing the
3745 // top 32 bits of the target register. We theoretically could leave those
3746 // bits unchanged, but we would have to make sure that no code uses a
3747 // 32bit input value as a 64bit value assuming that the top 32 bits are
3748 // zero.
3749 __ Mov(output.W(), source.W());
Alexandre Rames3e69f162014-12-10 10:36:50 +00003750 } else if ((result_type == Primitive::kPrimChar) ||
3751 ((input_type == Primitive::kPrimChar) && (result_size > input_size))) {
3752 __ Ubfx(output, output.IsX() ? source.X() : source.W(), 0, min_size * kBitsPerByte);
Alexandre Rames67555f72014-11-18 10:55:16 +00003753 } else {
Alexandre Rames3e69f162014-12-10 10:36:50 +00003754 __ Sbfx(output, output.IsX() ? source.X() : source.W(), 0, min_size * kBitsPerByte);
Alexandre Rames67555f72014-11-18 10:55:16 +00003755 }
Alexandre Rames542361f2015-01-29 16:57:31 +00003756 } else if (Primitive::IsFloatingPointType(result_type) && Primitive::IsIntegralType(input_type)) {
Serban Constantinescu02164b32014-11-13 14:05:07 +00003757 __ Scvtf(OutputFPRegister(conversion), InputRegisterAt(conversion, 0));
Alexandre Rames542361f2015-01-29 16:57:31 +00003758 } else if (Primitive::IsIntegralType(result_type) && Primitive::IsFloatingPointType(input_type)) {
Serban Constantinescu02164b32014-11-13 14:05:07 +00003759 CHECK(result_type == Primitive::kPrimInt || result_type == Primitive::kPrimLong);
3760 __ Fcvtzs(OutputRegister(conversion), InputFPRegisterAt(conversion, 0));
Alexandre Rames542361f2015-01-29 16:57:31 +00003761 } else if (Primitive::IsFloatingPointType(result_type) &&
3762 Primitive::IsFloatingPointType(input_type)) {
Serban Constantinescu02164b32014-11-13 14:05:07 +00003763 __ Fcvt(OutputFPRegister(conversion), InputFPRegisterAt(conversion, 0));
3764 } else {
3765 LOG(FATAL) << "Unexpected or unimplemented type conversion from " << input_type
3766 << " to " << result_type;
Alexandre Rames67555f72014-11-18 10:55:16 +00003767 }
Serban Constantinescu02164b32014-11-13 14:05:07 +00003768}
Alexandre Rames67555f72014-11-18 10:55:16 +00003769
Serban Constantinescu02164b32014-11-13 14:05:07 +00003770void LocationsBuilderARM64::VisitUShr(HUShr* ushr) {
3771 HandleShift(ushr);
3772}
3773
3774void InstructionCodeGeneratorARM64::VisitUShr(HUShr* ushr) {
3775 HandleShift(ushr);
Alexandre Rames67555f72014-11-18 10:55:16 +00003776}
3777
3778void LocationsBuilderARM64::VisitXor(HXor* instruction) {
3779 HandleBinaryOp(instruction);
3780}
3781
3782void InstructionCodeGeneratorARM64::VisitXor(HXor* instruction) {
3783 HandleBinaryOp(instruction);
3784}
3785
Roland Levillain4b8f1ec2015-08-26 18:34:03 +01003786void LocationsBuilderARM64::VisitBoundType(HBoundType* instruction ATTRIBUTE_UNUSED) {
Calin Juravleb1498f62015-02-16 13:13:29 +00003787 // Nothing to do, this should be removed during prepare for register allocator.
Calin Juravleb1498f62015-02-16 13:13:29 +00003788 LOG(FATAL) << "Unreachable";
3789}
3790
Roland Levillain4b8f1ec2015-08-26 18:34:03 +01003791void InstructionCodeGeneratorARM64::VisitBoundType(HBoundType* instruction ATTRIBUTE_UNUSED) {
Calin Juravleb1498f62015-02-16 13:13:29 +00003792 // Nothing to do, this should be removed during prepare for register allocator.
Calin Juravleb1498f62015-02-16 13:13:29 +00003793 LOG(FATAL) << "Unreachable";
3794}
3795
Nicolas Geoffray2e7cd752015-07-10 11:38:52 +01003796void LocationsBuilderARM64::VisitFakeString(HFakeString* instruction) {
3797 DCHECK(codegen_->IsBaseline());
3798 LocationSummary* locations =
3799 new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall);
3800 locations->SetOut(Location::ConstantLocation(GetGraph()->GetNullConstant()));
3801}
3802
3803void InstructionCodeGeneratorARM64::VisitFakeString(HFakeString* instruction ATTRIBUTE_UNUSED) {
3804 DCHECK(codegen_->IsBaseline());
3805 // Will be generated at use site.
3806}
3807
Mark Mendellfe57faa2015-09-18 09:26:15 -04003808// Simple implementation of packed switch - generate cascaded compare/jumps.
3809void LocationsBuilderARM64::VisitPackedSwitch(HPackedSwitch* switch_instr) {
3810 LocationSummary* locations =
3811 new (GetGraph()->GetArena()) LocationSummary(switch_instr, LocationSummary::kNoCall);
3812 locations->SetInAt(0, Location::RequiresRegister());
3813}
3814
3815void InstructionCodeGeneratorARM64::VisitPackedSwitch(HPackedSwitch* switch_instr) {
3816 int32_t lower_bound = switch_instr->GetStartValue();
3817 int32_t num_entries = switch_instr->GetNumEntries();
3818 Register value_reg = InputRegisterAt(switch_instr, 0);
3819 HBasicBlock* default_block = switch_instr->GetDefaultBlock();
3820
3821 // Create a series of compare/jumps.
3822 const ArenaVector<HBasicBlock*>& successors = switch_instr->GetBlock()->GetSuccessors();
3823 for (int32_t i = 0; i < num_entries; i++) {
3824 int32_t case_value = lower_bound + i;
Vladimir Markoec7802a2015-10-01 20:57:57 +01003825 vixl::Label* succ = codegen_->GetLabelOf(successors[i]);
Mark Mendellfe57faa2015-09-18 09:26:15 -04003826 if (case_value == 0) {
3827 __ Cbz(value_reg, succ);
3828 } else {
3829 __ Cmp(value_reg, vixl::Operand(case_value));
3830 __ B(eq, succ);
3831 }
3832 }
3833
3834 // And the default for any other value.
3835 if (!codegen_->GoesToNextBlock(switch_instr->GetBlock(), default_block)) {
3836 __ B(codegen_->GetLabelOf(default_block));
3837 }
3838}
3839
Alexandre Rames67555f72014-11-18 10:55:16 +00003840#undef __
3841#undef QUICK_ENTRY_POINT
3842
Alexandre Rames5319def2014-10-23 10:03:10 +01003843} // namespace arm64
3844} // namespace art