blob: 225439345d00e84aa418434b25daeedd58034235 [file] [log] [blame]
Evan Cheng10043e22007-01-19 07:51:42 +00001//===-- ARMISelLowering.cpp - ARM DAG Lowering Implementation -------------===//
2//
3// The LLVM Compiler Infrastructure
4//
Chris Lattnerf3ebc3f2007-12-29 20:36:04 +00005// This file is distributed under the University of Illinois Open Source
6// License. See LICENSE.TXT for details.
Evan Cheng10043e22007-01-19 07:51:42 +00007//
8//===----------------------------------------------------------------------===//
9//
10// This file defines the interfaces that ARM uses to lower LLVM code into a
11// selection DAG.
12//
13//===----------------------------------------------------------------------===//
14
Craig Topper188ed9d2012-03-17 07:33:42 +000015#include "ARMISelLowering.h"
Eric Christopher1c069172010-09-10 22:42:06 +000016#include "ARMCallingConv.h"
Evan Cheng10043e22007-01-19 07:51:42 +000017#include "ARMConstantPoolValue.h"
Evan Cheng10043e22007-01-19 07:51:42 +000018#include "ARMMachineFunctionInfo.h"
Anton Korobeynikov9a232f42009-08-21 12:41:24 +000019#include "ARMPerfectShuffle.h"
Evan Cheng10043e22007-01-19 07:51:42 +000020#include "ARMSubtarget.h"
21#include "ARMTargetMachine.h"
Chris Lattner4e7dfaf2009-08-02 00:34:36 +000022#include "ARMTargetObjectFile.h"
Evan Chenga20cde32011-07-20 23:34:39 +000023#include "MCTargetDesc/ARMAddressingModes.h"
Chandler Carruthed0881b2012-12-03 16:50:05 +000024#include "llvm/ADT/Statistic.h"
25#include "llvm/ADT/StringExtras.h"
Pete Cooperef21bd42015-03-04 01:24:11 +000026#include "llvm/ADT/StringSwitch.h"
Bob Wilsona4c22902009-04-17 19:07:39 +000027#include "llvm/CodeGen/CallingConvLower.h"
Evan Cheng078b0b02011-01-08 01:24:27 +000028#include "llvm/CodeGen/IntrinsicLowering.h"
Evan Cheng10043e22007-01-19 07:51:42 +000029#include "llvm/CodeGen/MachineBasicBlock.h"
30#include "llvm/CodeGen/MachineFrameInfo.h"
31#include "llvm/CodeGen/MachineFunction.h"
32#include "llvm/CodeGen/MachineInstrBuilder.h"
Eric Christopher79cc1e32014-09-02 22:28:02 +000033#include "llvm/CodeGen/MachineJumpTableInfo.h"
Bill Wendling202803e2011-10-05 00:02:33 +000034#include "llvm/CodeGen/MachineModuleInfo.h"
Chris Lattnera10fff52007-12-31 04:13:23 +000035#include "llvm/CodeGen/MachineRegisterInfo.h"
Evan Cheng10043e22007-01-19 07:51:42 +000036#include "llvm/CodeGen/SelectionDAG.h"
Chandler Carruth9fb823b2013-01-02 11:36:10 +000037#include "llvm/IR/CallingConv.h"
38#include "llvm/IR/Constants.h"
39#include "llvm/IR/Function.h"
40#include "llvm/IR/GlobalValue.h"
Tim Northover037f26f22014-04-17 18:22:47 +000041#include "llvm/IR/IRBuilder.h"
Chandler Carruth9fb823b2013-01-02 11:36:10 +000042#include "llvm/IR/Instruction.h"
43#include "llvm/IR/Instructions.h"
John Brawn0dbcd652015-03-18 12:01:59 +000044#include "llvm/IR/IntrinsicInst.h"
Benjamin Kramer799003b2015-03-23 19:32:43 +000045#include "llvm/IR/Intrinsics.h"
Chandler Carruth9fb823b2013-01-02 11:36:10 +000046#include "llvm/IR/Type.h"
Bill Wendling46ffefc2010-03-09 02:46:12 +000047#include "llvm/MC/MCSectionMachO.h"
Jim Grosbach32bb3622010-04-14 22:28:31 +000048#include "llvm/Support/CommandLine.h"
Oliver Stannardc24f2172014-05-09 14:01:47 +000049#include "llvm/Support/Debug.h"
Torok Edwin6dd27302009-07-08 18:01:40 +000050#include "llvm/Support/ErrorHandling.h"
Evan Cheng2150b922007-03-12 23:30:29 +000051#include "llvm/Support/MathExtras.h"
Benjamin Kramer799003b2015-03-23 19:32:43 +000052#include "llvm/Support/raw_ostream.h"
Chandler Carruthed0881b2012-12-03 16:50:05 +000053#include "llvm/Target/TargetOptions.h"
David Peixottoc32e24a2013-10-17 19:49:22 +000054#include <utility>
Evan Cheng10043e22007-01-19 07:51:42 +000055using namespace llvm;
56
Chandler Carruth84e68b22014-04-22 02:41:26 +000057#define DEBUG_TYPE "arm-isel"
58
Dale Johannesend679ff72010-06-03 21:09:53 +000059STATISTIC(NumTailCalls, "Number of tail calls");
Evan Cheng68aec142011-01-19 02:16:49 +000060STATISTIC(NumMovwMovt, "Number of GAs materialized with movw + movt");
Manman Ren9f911162012-06-01 02:44:42 +000061STATISTIC(NumLoopByVals, "Number of loops generated for byval arguments");
James Molloy13065b02016-09-14 14:47:27 +000062STATISTIC(NumConstpoolPromoted,
63 "Number of constants with their storage promoted into constant pools");
Dale Johannesend679ff72010-06-03 21:09:53 +000064
Evan Chengf128bdc2010-06-16 07:35:02 +000065static cl::opt<bool>
66ARMInterworking("arm-interworking", cl::Hidden,
67 cl::desc("Enable / disable ARM interworking (for debugging only)"),
68 cl::init(true));
69
James Molloy13065b02016-09-14 14:47:27 +000070static cl::opt<bool> EnableConstpoolPromotion(
71 "arm-promote-constant", cl::Hidden,
72 cl::desc("Enable / disable promotion of unnamed_addr constants into "
73 "constant pools"),
74 cl::init(true));
75static cl::opt<unsigned> ConstpoolPromotionMaxSize(
76 "arm-promote-constant-max-size", cl::Hidden,
77 cl::desc("Maximum size of constant to promote into a constant pool"),
78 cl::init(64));
79
Benjamin Kramer7ba71be2011-11-26 23:01:57 +000080namespace {
Cameron Zwarich89019782011-06-10 20:59:24 +000081 class ARMCCState : public CCState {
82 public:
83 ARMCCState(CallingConv::ID CC, bool isVarArg, MachineFunction &MF,
Eric Christopherb5217502014-08-06 18:45:26 +000084 SmallVectorImpl<CCValAssign> &locs, LLVMContext &C,
85 ParmContext PC)
86 : CCState(CC, isVarArg, MF, locs, C) {
Cameron Zwarich89019782011-06-10 20:59:24 +000087 assert(((PC == Call) || (PC == Prologue)) &&
88 "ARMCCState users must specify whether their context is call"
89 "or prologue generation.");
90 CallOrPrologue = PC;
91 }
92 };
Alexander Kornienkof00654e2015-06-23 09:49:53 +000093}
Cameron Zwarich89019782011-06-10 20:59:24 +000094
Saleem Abdulrasool92e33a32016-09-09 20:11:31 +000095void ARMTargetLowering::InitLibcallCallingConvs() {
96 // The builtins on ARM always use AAPCS, irrespective of wheter C is AAPCS or
97 // AAPCS_VFP.
98 for (const auto LC : {
99 RTLIB::SHL_I16,
100 RTLIB::SHL_I32,
101 RTLIB::SHL_I64,
102 RTLIB::SHL_I128,
103 RTLIB::SRL_I16,
104 RTLIB::SRL_I32,
105 RTLIB::SRL_I64,
106 RTLIB::SRL_I128,
107 RTLIB::SRA_I16,
108 RTLIB::SRA_I32,
109 RTLIB::SRA_I64,
110 RTLIB::SRA_I128,
111 RTLIB::MUL_I8,
112 RTLIB::MUL_I16,
113 RTLIB::MUL_I32,
114 RTLIB::MUL_I64,
115 RTLIB::MUL_I128,
116 RTLIB::MULO_I32,
117 RTLIB::MULO_I64,
118 RTLIB::MULO_I128,
119 RTLIB::SDIV_I8,
120 RTLIB::SDIV_I16,
121 RTLIB::SDIV_I32,
122 RTLIB::SDIV_I64,
123 RTLIB::SDIV_I128,
124 RTLIB::UDIV_I8,
125 RTLIB::UDIV_I16,
126 RTLIB::UDIV_I32,
127 RTLIB::UDIV_I64,
128 RTLIB::UDIV_I128,
129 RTLIB::SREM_I8,
130 RTLIB::SREM_I16,
131 RTLIB::SREM_I32,
132 RTLIB::SREM_I64,
133 RTLIB::SREM_I128,
134 RTLIB::UREM_I8,
135 RTLIB::UREM_I16,
136 RTLIB::UREM_I32,
137 RTLIB::UREM_I64,
138 RTLIB::UREM_I128,
139 RTLIB::SDIVREM_I8,
140 RTLIB::SDIVREM_I16,
141 RTLIB::SDIVREM_I32,
142 RTLIB::SDIVREM_I64,
143 RTLIB::SDIVREM_I128,
144 RTLIB::UDIVREM_I8,
145 RTLIB::UDIVREM_I16,
146 RTLIB::UDIVREM_I32,
147 RTLIB::UDIVREM_I64,
148 RTLIB::UDIVREM_I128,
149 RTLIB::NEG_I32,
150 RTLIB::NEG_I64,
151 RTLIB::ADD_F32,
152 RTLIB::ADD_F64,
153 RTLIB::ADD_F80,
154 RTLIB::ADD_F128,
155 RTLIB::SUB_F32,
156 RTLIB::SUB_F64,
157 RTLIB::SUB_F80,
158 RTLIB::SUB_F128,
159 RTLIB::MUL_F32,
160 RTLIB::MUL_F64,
161 RTLIB::MUL_F80,
162 RTLIB::MUL_F128,
163 RTLIB::DIV_F32,
164 RTLIB::DIV_F64,
165 RTLIB::DIV_F80,
166 RTLIB::DIV_F128,
167 RTLIB::POWI_F32,
168 RTLIB::POWI_F64,
169 RTLIB::POWI_F80,
170 RTLIB::POWI_F128,
171 RTLIB::FPEXT_F64_F128,
172 RTLIB::FPEXT_F32_F128,
173 RTLIB::FPEXT_F32_F64,
174 RTLIB::FPEXT_F16_F32,
175 RTLIB::FPROUND_F32_F16,
176 RTLIB::FPROUND_F64_F16,
177 RTLIB::FPROUND_F80_F16,
178 RTLIB::FPROUND_F128_F16,
179 RTLIB::FPROUND_F64_F32,
180 RTLIB::FPROUND_F80_F32,
181 RTLIB::FPROUND_F128_F32,
182 RTLIB::FPROUND_F80_F64,
183 RTLIB::FPROUND_F128_F64,
184 RTLIB::FPTOSINT_F32_I32,
185 RTLIB::FPTOSINT_F32_I64,
186 RTLIB::FPTOSINT_F32_I128,
187 RTLIB::FPTOSINT_F64_I32,
188 RTLIB::FPTOSINT_F64_I64,
189 RTLIB::FPTOSINT_F64_I128,
190 RTLIB::FPTOSINT_F80_I32,
191 RTLIB::FPTOSINT_F80_I64,
192 RTLIB::FPTOSINT_F80_I128,
193 RTLIB::FPTOSINT_F128_I32,
194 RTLIB::FPTOSINT_F128_I64,
195 RTLIB::FPTOSINT_F128_I128,
196 RTLIB::FPTOUINT_F32_I32,
197 RTLIB::FPTOUINT_F32_I64,
198 RTLIB::FPTOUINT_F32_I128,
199 RTLIB::FPTOUINT_F64_I32,
200 RTLIB::FPTOUINT_F64_I64,
201 RTLIB::FPTOUINT_F64_I128,
202 RTLIB::FPTOUINT_F80_I32,
203 RTLIB::FPTOUINT_F80_I64,
204 RTLIB::FPTOUINT_F80_I128,
205 RTLIB::FPTOUINT_F128_I32,
206 RTLIB::FPTOUINT_F128_I64,
207 RTLIB::FPTOUINT_F128_I128,
208 RTLIB::SINTTOFP_I32_F32,
209 RTLIB::SINTTOFP_I32_F64,
210 RTLIB::SINTTOFP_I32_F80,
211 RTLIB::SINTTOFP_I32_F128,
212 RTLIB::SINTTOFP_I64_F32,
213 RTLIB::SINTTOFP_I64_F64,
214 RTLIB::SINTTOFP_I64_F80,
215 RTLIB::SINTTOFP_I64_F128,
216 RTLIB::SINTTOFP_I128_F32,
217 RTLIB::SINTTOFP_I128_F64,
218 RTLIB::SINTTOFP_I128_F80,
219 RTLIB::SINTTOFP_I128_F128,
220 RTLIB::UINTTOFP_I32_F32,
221 RTLIB::UINTTOFP_I32_F64,
222 RTLIB::UINTTOFP_I32_F80,
223 RTLIB::UINTTOFP_I32_F128,
224 RTLIB::UINTTOFP_I64_F32,
225 RTLIB::UINTTOFP_I64_F64,
226 RTLIB::UINTTOFP_I64_F80,
227 RTLIB::UINTTOFP_I64_F128,
228 RTLIB::UINTTOFP_I128_F32,
229 RTLIB::UINTTOFP_I128_F64,
230 RTLIB::UINTTOFP_I128_F80,
231 RTLIB::UINTTOFP_I128_F128,
232 RTLIB::OEQ_F32,
233 RTLIB::OEQ_F64,
234 RTLIB::OEQ_F128,
235 RTLIB::UNE_F32,
236 RTLIB::UNE_F64,
237 RTLIB::UNE_F128,
238 RTLIB::OGE_F32,
239 RTLIB::OGE_F64,
240 RTLIB::OGE_F128,
241 RTLIB::OLT_F32,
242 RTLIB::OLT_F64,
243 RTLIB::OLT_F128,
244 RTLIB::OLE_F32,
245 RTLIB::OLE_F64,
246 RTLIB::OLE_F128,
247 RTLIB::OGT_F32,
248 RTLIB::OGT_F64,
249 RTLIB::OGT_F128,
250 RTLIB::UO_F32,
251 RTLIB::UO_F64,
252 RTLIB::UO_F128,
253 RTLIB::O_F32,
254 RTLIB::O_F64,
255 RTLIB::O_F128,
256 })
257 setLibcallCallingConv(LC, CallingConv::ARM_AAPCS);
258}
259
Stuart Hastings45fe3c32011-04-20 16:47:52 +0000260// The APCS parameter registers.
Craig Topper840beec2014-04-04 05:16:06 +0000261static const MCPhysReg GPRArgRegs[] = {
Stuart Hastings45fe3c32011-04-20 16:47:52 +0000262 ARM::R0, ARM::R1, ARM::R2, ARM::R3
263};
264
Craig Topper4fa625f2012-08-12 03:16:37 +0000265void ARMTargetLowering::addTypeForNEON(MVT VT, MVT PromotedLdStVT,
266 MVT PromotedBitwiseVT) {
Bob Wilson2e076c42009-06-22 23:27:02 +0000267 if (VT != PromotedLdStVT) {
Craig Topper4fa625f2012-08-12 03:16:37 +0000268 setOperationAction(ISD::LOAD, VT, Promote);
269 AddPromotedToType (ISD::LOAD, VT, PromotedLdStVT);
Bob Wilson2e076c42009-06-22 23:27:02 +0000270
Craig Topper4fa625f2012-08-12 03:16:37 +0000271 setOperationAction(ISD::STORE, VT, Promote);
272 AddPromotedToType (ISD::STORE, VT, PromotedLdStVT);
Bob Wilson2e076c42009-06-22 23:27:02 +0000273 }
274
Craig Topper4fa625f2012-08-12 03:16:37 +0000275 MVT ElemTy = VT.getVectorElementType();
Owen Anderson9f944592009-08-11 20:47:22 +0000276 if (ElemTy != MVT::i64 && ElemTy != MVT::f64)
Craig Topper4fa625f2012-08-12 03:16:37 +0000277 setOperationAction(ISD::SETCC, VT, Custom);
278 setOperationAction(ISD::INSERT_VECTOR_ELT, VT, Custom);
279 setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT, Custom);
Eli Friedman2d4055b2011-11-09 23:36:02 +0000280 if (ElemTy == MVT::i32) {
Craig Topper4fa625f2012-08-12 03:16:37 +0000281 setOperationAction(ISD::SINT_TO_FP, VT, Custom);
282 setOperationAction(ISD::UINT_TO_FP, VT, Custom);
283 setOperationAction(ISD::FP_TO_SINT, VT, Custom);
284 setOperationAction(ISD::FP_TO_UINT, VT, Custom);
Eli Friedman2d4055b2011-11-09 23:36:02 +0000285 } else {
Craig Topper4fa625f2012-08-12 03:16:37 +0000286 setOperationAction(ISD::SINT_TO_FP, VT, Expand);
287 setOperationAction(ISD::UINT_TO_FP, VT, Expand);
288 setOperationAction(ISD::FP_TO_SINT, VT, Expand);
289 setOperationAction(ISD::FP_TO_UINT, VT, Expand);
Bob Wilson5d8cfb22009-09-16 20:20:44 +0000290 }
Craig Topper4fa625f2012-08-12 03:16:37 +0000291 setOperationAction(ISD::BUILD_VECTOR, VT, Custom);
292 setOperationAction(ISD::VECTOR_SHUFFLE, VT, Custom);
293 setOperationAction(ISD::CONCAT_VECTORS, VT, Legal);
294 setOperationAction(ISD::EXTRACT_SUBVECTOR, VT, Legal);
295 setOperationAction(ISD::SELECT, VT, Expand);
296 setOperationAction(ISD::SELECT_CC, VT, Expand);
Jim Grosbach30af4422012-10-12 22:59:21 +0000297 setOperationAction(ISD::VSELECT, VT, Expand);
Craig Topper4fa625f2012-08-12 03:16:37 +0000298 setOperationAction(ISD::SIGN_EXTEND_INREG, VT, Expand);
Bob Wilson2e076c42009-06-22 23:27:02 +0000299 if (VT.isInteger()) {
Craig Topper4fa625f2012-08-12 03:16:37 +0000300 setOperationAction(ISD::SHL, VT, Custom);
301 setOperationAction(ISD::SRA, VT, Custom);
302 setOperationAction(ISD::SRL, VT, Custom);
Bob Wilson2e076c42009-06-22 23:27:02 +0000303 }
304
305 // Promote all bit-wise operations.
306 if (VT.isInteger() && VT != PromotedBitwiseVT) {
Craig Topper4fa625f2012-08-12 03:16:37 +0000307 setOperationAction(ISD::AND, VT, Promote);
308 AddPromotedToType (ISD::AND, VT, PromotedBitwiseVT);
309 setOperationAction(ISD::OR, VT, Promote);
310 AddPromotedToType (ISD::OR, VT, PromotedBitwiseVT);
311 setOperationAction(ISD::XOR, VT, Promote);
312 AddPromotedToType (ISD::XOR, VT, PromotedBitwiseVT);
Bob Wilson2e076c42009-06-22 23:27:02 +0000313 }
Bob Wilson4ed397c2009-09-16 00:17:28 +0000314
315 // Neon does not support vector divide/remainder operations.
Craig Topper4fa625f2012-08-12 03:16:37 +0000316 setOperationAction(ISD::SDIV, VT, Expand);
317 setOperationAction(ISD::UDIV, VT, Expand);
318 setOperationAction(ISD::FDIV, VT, Expand);
319 setOperationAction(ISD::SREM, VT, Expand);
320 setOperationAction(ISD::UREM, VT, Expand);
321 setOperationAction(ISD::FREM, VT, Expand);
James Molloya6702e22015-07-17 17:10:55 +0000322
Silviu Barangaad1b19f2015-08-19 14:11:27 +0000323 if (!VT.isFloatingPoint() &&
324 VT != MVT::v2i64 && VT != MVT::v1i64)
325 for (unsigned Opcode : {ISD::SMIN, ISD::SMAX, ISD::UMIN, ISD::UMAX})
326 setOperationAction(Opcode, VT, Legal);
Bob Wilson2e076c42009-06-22 23:27:02 +0000327}
328
Craig Topper4fa625f2012-08-12 03:16:37 +0000329void ARMTargetLowering::addDRTypeForNEON(MVT VT) {
Craig Topperc7242e02012-04-20 07:30:17 +0000330 addRegisterClass(VT, &ARM::DPRRegClass);
Owen Anderson9f944592009-08-11 20:47:22 +0000331 addTypeForNEON(VT, MVT::f64, MVT::v2i32);
Bob Wilson2e076c42009-06-22 23:27:02 +0000332}
333
Craig Topper4fa625f2012-08-12 03:16:37 +0000334void ARMTargetLowering::addQRTypeForNEON(MVT VT) {
Jakob Stoklund Olesen20912062014-01-14 06:18:34 +0000335 addRegisterClass(VT, &ARM::DPairRegClass);
Owen Anderson9f944592009-08-11 20:47:22 +0000336 addTypeForNEON(VT, MVT::v2f64, MVT::v4i32);
Bob Wilson2e076c42009-06-22 23:27:02 +0000337}
338
Eric Christopher1889fdc2015-01-29 00:19:39 +0000339ARMTargetLowering::ARMTargetLowering(const TargetMachine &TM,
340 const ARMSubtarget &STI)
341 : TargetLowering(TM), Subtarget(&STI) {
342 RegInfo = Subtarget->getRegisterInfo();
343 Itins = Subtarget->getInstrItineraryData();
Evan Cheng10043e22007-01-19 07:51:42 +0000344
Duncan Sandsf2641e12011-09-06 19:07:46 +0000345 setBooleanVectorContents(ZeroOrNegativeOneBooleanContent);
346
Saleem Abdulrasool92e33a32016-09-09 20:11:31 +0000347 InitLibcallCallingConvs();
348
Tim Northoverd6a729b2014-01-06 14:28:05 +0000349 if (Subtarget->isTargetMachO()) {
Evan Chengc9f22fd12007-04-27 08:15:43 +0000350 // Uses VFP for Thumb libfuncs if available.
Jim Grosbach1d1d6d42013-10-24 23:07:11 +0000351 if (Subtarget->isThumb() && Subtarget->hasVFP2() &&
Eric Christopher824f42f2015-05-12 01:26:05 +0000352 Subtarget->hasARMOps() && !Subtarget->useSoftFloat()) {
Saleem Abdulrasool67697a72015-08-04 03:57:52 +0000353 static const struct {
354 const RTLIB::Libcall Op;
355 const char * const Name;
356 const ISD::CondCode Cond;
357 } LibraryCalls[] = {
358 // Single-precision floating-point arithmetic.
359 { RTLIB::ADD_F32, "__addsf3vfp", ISD::SETCC_INVALID },
360 { RTLIB::SUB_F32, "__subsf3vfp", ISD::SETCC_INVALID },
361 { RTLIB::MUL_F32, "__mulsf3vfp", ISD::SETCC_INVALID },
362 { RTLIB::DIV_F32, "__divsf3vfp", ISD::SETCC_INVALID },
Evan Cheng10043e22007-01-19 07:51:42 +0000363
Saleem Abdulrasool67697a72015-08-04 03:57:52 +0000364 // Double-precision floating-point arithmetic.
365 { RTLIB::ADD_F64, "__adddf3vfp", ISD::SETCC_INVALID },
366 { RTLIB::SUB_F64, "__subdf3vfp", ISD::SETCC_INVALID },
367 { RTLIB::MUL_F64, "__muldf3vfp", ISD::SETCC_INVALID },
368 { RTLIB::DIV_F64, "__divdf3vfp", ISD::SETCC_INVALID },
Evan Cheng143576d2007-01-31 09:30:58 +0000369
Saleem Abdulrasool67697a72015-08-04 03:57:52 +0000370 // Single-precision comparisons.
371 { RTLIB::OEQ_F32, "__eqsf2vfp", ISD::SETNE },
372 { RTLIB::UNE_F32, "__nesf2vfp", ISD::SETNE },
373 { RTLIB::OLT_F32, "__ltsf2vfp", ISD::SETNE },
374 { RTLIB::OLE_F32, "__lesf2vfp", ISD::SETNE },
375 { RTLIB::OGE_F32, "__gesf2vfp", ISD::SETNE },
376 { RTLIB::OGT_F32, "__gtsf2vfp", ISD::SETNE },
377 { RTLIB::UO_F32, "__unordsf2vfp", ISD::SETNE },
378 { RTLIB::O_F32, "__unordsf2vfp", ISD::SETEQ },
Evan Cheng10043e22007-01-19 07:51:42 +0000379
Saleem Abdulrasool67697a72015-08-04 03:57:52 +0000380 // Double-precision comparisons.
381 { RTLIB::OEQ_F64, "__eqdf2vfp", ISD::SETNE },
382 { RTLIB::UNE_F64, "__nedf2vfp", ISD::SETNE },
383 { RTLIB::OLT_F64, "__ltdf2vfp", ISD::SETNE },
384 { RTLIB::OLE_F64, "__ledf2vfp", ISD::SETNE },
385 { RTLIB::OGE_F64, "__gedf2vfp", ISD::SETNE },
386 { RTLIB::OGT_F64, "__gtdf2vfp", ISD::SETNE },
387 { RTLIB::UO_F64, "__unorddf2vfp", ISD::SETNE },
388 { RTLIB::O_F64, "__unorddf2vfp", ISD::SETEQ },
Evan Cheng143576d2007-01-31 09:30:58 +0000389
Saleem Abdulrasool67697a72015-08-04 03:57:52 +0000390 // Floating-point to integer conversions.
391 // i64 conversions are done via library routines even when generating VFP
392 // instructions, so use the same ones.
393 { RTLIB::FPTOSINT_F64_I32, "__fixdfsivfp", ISD::SETCC_INVALID },
394 { RTLIB::FPTOUINT_F64_I32, "__fixunsdfsivfp", ISD::SETCC_INVALID },
395 { RTLIB::FPTOSINT_F32_I32, "__fixsfsivfp", ISD::SETCC_INVALID },
396 { RTLIB::FPTOUINT_F32_I32, "__fixunssfsivfp", ISD::SETCC_INVALID },
Evan Cheng10043e22007-01-19 07:51:42 +0000397
Saleem Abdulrasool67697a72015-08-04 03:57:52 +0000398 // Conversions between floating types.
399 { RTLIB::FPROUND_F64_F32, "__truncdfsf2vfp", ISD::SETCC_INVALID },
400 { RTLIB::FPEXT_F32_F64, "__extendsfdf2vfp", ISD::SETCC_INVALID },
Evan Cheng10043e22007-01-19 07:51:42 +0000401
Saleem Abdulrasool67697a72015-08-04 03:57:52 +0000402 // Integer to floating-point conversions.
403 // i64 conversions are done via library routines even when generating VFP
404 // instructions, so use the same ones.
405 // FIXME: There appears to be some naming inconsistency in ARM libgcc:
406 // e.g., __floatunsidf vs. __floatunssidfvfp.
407 { RTLIB::SINTTOFP_I32_F64, "__floatsidfvfp", ISD::SETCC_INVALID },
408 { RTLIB::UINTTOFP_I32_F64, "__floatunssidfvfp", ISD::SETCC_INVALID },
409 { RTLIB::SINTTOFP_I32_F32, "__floatsisfvfp", ISD::SETCC_INVALID },
410 { RTLIB::UINTTOFP_I32_F32, "__floatunssisfvfp", ISD::SETCC_INVALID },
411 };
Evan Cheng10043e22007-01-19 07:51:42 +0000412
Saleem Abdulrasool67697a72015-08-04 03:57:52 +0000413 for (const auto &LC : LibraryCalls) {
414 setLibcallName(LC.Op, LC.Name);
415 if (LC.Cond != ISD::SETCC_INVALID)
416 setCmpLibcallCC(LC.Op, LC.Cond);
417 }
Evan Chengc9f22fd12007-04-27 08:15:43 +0000418 }
Tim Northover8b403662015-10-28 22:51:16 +0000419
420 // Set the correct calling convention for ARMv7k WatchOS. It's just
421 // AAPCS_VFP for functions as simple as libcalls.
Tim Northover042a6c12016-01-27 19:32:29 +0000422 if (Subtarget->isTargetWatchABI()) {
Tim Northover8b403662015-10-28 22:51:16 +0000423 for (int i = 0; i < RTLIB::UNKNOWN_LIBCALL; ++i)
424 setLibcallCallingConv((RTLIB::Libcall)i, CallingConv::ARM_AAPCS_VFP);
425 }
Evan Cheng10043e22007-01-19 07:51:42 +0000426 }
427
Bob Wilsonccbc17b2009-05-22 17:38:41 +0000428 // These libcalls are not available in 32-bit.
Craig Topper062a2ba2014-04-25 05:30:21 +0000429 setLibcallName(RTLIB::SHL_I128, nullptr);
430 setLibcallName(RTLIB::SRL_I128, nullptr);
431 setLibcallName(RTLIB::SRA_I128, nullptr);
Bob Wilsonccbc17b2009-05-22 17:38:41 +0000432
Renato Golin6d435f12015-11-09 12:40:30 +0000433 // RTLIB
434 if (Subtarget->isAAPCS_ABI() &&
435 (Subtarget->isTargetAEABI() || Subtarget->isTargetGNUAEABI() ||
Rafael Espindolaa895a0c2016-06-24 21:14:33 +0000436 Subtarget->isTargetMuslAEABI() || Subtarget->isTargetAndroid())) {
Saleem Abdulrasool8bfb1922014-05-18 16:39:11 +0000437 static const struct {
438 const RTLIB::Libcall Op;
439 const char * const Name;
440 const CallingConv::ID CC;
441 const ISD::CondCode Cond;
442 } LibraryCalls[] = {
443 // Double-precision floating-point arithmetic helper functions
444 // RTABI chapter 4.1.2, Table 2
445 { RTLIB::ADD_F64, "__aeabi_dadd", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID },
446 { RTLIB::DIV_F64, "__aeabi_ddiv", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID },
447 { RTLIB::MUL_F64, "__aeabi_dmul", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID },
448 { RTLIB::SUB_F64, "__aeabi_dsub", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID },
Anton Korobeynikov81bdc932010-09-28 21:39:26 +0000449
Saleem Abdulrasool8bfb1922014-05-18 16:39:11 +0000450 // Double-precision floating-point comparison helper functions
451 // RTABI chapter 4.1.2, Table 3
452 { RTLIB::OEQ_F64, "__aeabi_dcmpeq", CallingConv::ARM_AAPCS, ISD::SETNE },
453 { RTLIB::UNE_F64, "__aeabi_dcmpeq", CallingConv::ARM_AAPCS, ISD::SETEQ },
454 { RTLIB::OLT_F64, "__aeabi_dcmplt", CallingConv::ARM_AAPCS, ISD::SETNE },
455 { RTLIB::OLE_F64, "__aeabi_dcmple", CallingConv::ARM_AAPCS, ISD::SETNE },
456 { RTLIB::OGE_F64, "__aeabi_dcmpge", CallingConv::ARM_AAPCS, ISD::SETNE },
457 { RTLIB::OGT_F64, "__aeabi_dcmpgt", CallingConv::ARM_AAPCS, ISD::SETNE },
458 { RTLIB::UO_F64, "__aeabi_dcmpun", CallingConv::ARM_AAPCS, ISD::SETNE },
459 { RTLIB::O_F64, "__aeabi_dcmpun", CallingConv::ARM_AAPCS, ISD::SETEQ },
Anton Korobeynikov81bdc932010-09-28 21:39:26 +0000460
Saleem Abdulrasool8bfb1922014-05-18 16:39:11 +0000461 // Single-precision floating-point arithmetic helper functions
462 // RTABI chapter 4.1.2, Table 4
463 { RTLIB::ADD_F32, "__aeabi_fadd", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID },
464 { RTLIB::DIV_F32, "__aeabi_fdiv", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID },
465 { RTLIB::MUL_F32, "__aeabi_fmul", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID },
466 { RTLIB::SUB_F32, "__aeabi_fsub", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID },
Anton Korobeynikov81bdc932010-09-28 21:39:26 +0000467
Saleem Abdulrasool8bfb1922014-05-18 16:39:11 +0000468 // Single-precision floating-point comparison helper functions
469 // RTABI chapter 4.1.2, Table 5
470 { RTLIB::OEQ_F32, "__aeabi_fcmpeq", CallingConv::ARM_AAPCS, ISD::SETNE },
471 { RTLIB::UNE_F32, "__aeabi_fcmpeq", CallingConv::ARM_AAPCS, ISD::SETEQ },
472 { RTLIB::OLT_F32, "__aeabi_fcmplt", CallingConv::ARM_AAPCS, ISD::SETNE },
473 { RTLIB::OLE_F32, "__aeabi_fcmple", CallingConv::ARM_AAPCS, ISD::SETNE },
474 { RTLIB::OGE_F32, "__aeabi_fcmpge", CallingConv::ARM_AAPCS, ISD::SETNE },
475 { RTLIB::OGT_F32, "__aeabi_fcmpgt", CallingConv::ARM_AAPCS, ISD::SETNE },
476 { RTLIB::UO_F32, "__aeabi_fcmpun", CallingConv::ARM_AAPCS, ISD::SETNE },
477 { RTLIB::O_F32, "__aeabi_fcmpun", CallingConv::ARM_AAPCS, ISD::SETEQ },
Anton Korobeynikov81bdc932010-09-28 21:39:26 +0000478
Saleem Abdulrasool8bfb1922014-05-18 16:39:11 +0000479 // Floating-point to integer conversions.
480 // RTABI chapter 4.1.2, Table 6
481 { RTLIB::FPTOSINT_F64_I32, "__aeabi_d2iz", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID },
482 { RTLIB::FPTOUINT_F64_I32, "__aeabi_d2uiz", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID },
483 { RTLIB::FPTOSINT_F64_I64, "__aeabi_d2lz", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID },
484 { RTLIB::FPTOUINT_F64_I64, "__aeabi_d2ulz", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID },
485 { RTLIB::FPTOSINT_F32_I32, "__aeabi_f2iz", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID },
486 { RTLIB::FPTOUINT_F32_I32, "__aeabi_f2uiz", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID },
487 { RTLIB::FPTOSINT_F32_I64, "__aeabi_f2lz", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID },
488 { RTLIB::FPTOUINT_F32_I64, "__aeabi_f2ulz", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID },
Anton Korobeynikov81bdc932010-09-28 21:39:26 +0000489
Saleem Abdulrasool8bfb1922014-05-18 16:39:11 +0000490 // Conversions between floating types.
491 // RTABI chapter 4.1.2, Table 7
492 { RTLIB::FPROUND_F64_F32, "__aeabi_d2f", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID },
Saleem Abdulrasool017bd572014-08-17 22:51:02 +0000493 { RTLIB::FPROUND_F64_F16, "__aeabi_d2h", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID },
Chad Rosierad7c9102014-08-23 18:29:43 +0000494 { RTLIB::FPEXT_F32_F64, "__aeabi_f2d", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID },
Anton Korobeynikov81bdc932010-09-28 21:39:26 +0000495
Saleem Abdulrasool8bfb1922014-05-18 16:39:11 +0000496 // Integer to floating-point conversions.
497 // RTABI chapter 4.1.2, Table 8
498 { RTLIB::SINTTOFP_I32_F64, "__aeabi_i2d", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID },
499 { RTLIB::UINTTOFP_I32_F64, "__aeabi_ui2d", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID },
500 { RTLIB::SINTTOFP_I64_F64, "__aeabi_l2d", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID },
501 { RTLIB::UINTTOFP_I64_F64, "__aeabi_ul2d", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID },
502 { RTLIB::SINTTOFP_I32_F32, "__aeabi_i2f", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID },
503 { RTLIB::UINTTOFP_I32_F32, "__aeabi_ui2f", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID },
504 { RTLIB::SINTTOFP_I64_F32, "__aeabi_l2f", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID },
505 { RTLIB::UINTTOFP_I64_F32, "__aeabi_ul2f", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID },
Anton Korobeynikov81bdc932010-09-28 21:39:26 +0000506
Saleem Abdulrasool8bfb1922014-05-18 16:39:11 +0000507 // Long long helper functions
508 // RTABI chapter 4.2, Table 9
Chad Rosierad7c9102014-08-23 18:29:43 +0000509 { RTLIB::MUL_I64, "__aeabi_lmul", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID },
510 { RTLIB::SHL_I64, "__aeabi_llsl", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID },
511 { RTLIB::SRL_I64, "__aeabi_llsr", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID },
512 { RTLIB::SRA_I64, "__aeabi_lasr", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID },
Anton Korobeynikov81bdc932010-09-28 21:39:26 +0000513
Saleem Abdulrasool8bfb1922014-05-18 16:39:11 +0000514 // Integer division functions
515 // RTABI chapter 4.3.1
Chad Rosierad7c9102014-08-23 18:29:43 +0000516 { RTLIB::SDIV_I8, "__aeabi_idiv", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID },
517 { RTLIB::SDIV_I16, "__aeabi_idiv", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID },
518 { RTLIB::SDIV_I32, "__aeabi_idiv", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID },
519 { RTLIB::SDIV_I64, "__aeabi_ldivmod", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID },
520 { RTLIB::UDIV_I8, "__aeabi_uidiv", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID },
521 { RTLIB::UDIV_I16, "__aeabi_uidiv", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID },
522 { RTLIB::UDIV_I32, "__aeabi_uidiv", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID },
523 { RTLIB::UDIV_I64, "__aeabi_uldivmod", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID },
Saleem Abdulrasool8bfb1922014-05-18 16:39:11 +0000524 };
525
526 for (const auto &LC : LibraryCalls) {
527 setLibcallName(LC.Op, LC.Name);
528 setLibcallCallingConv(LC.Op, LC.CC);
529 if (LC.Cond != ISD::SETCC_INVALID)
530 setCmpLibcallCC(LC.Op, LC.Cond);
531 }
Renato Golin6d435f12015-11-09 12:40:30 +0000532
533 // EABI dependent RTLIB
534 if (TM.Options.EABIVersion == EABI::EABI4 ||
535 TM.Options.EABIVersion == EABI::EABI5) {
536 static const struct {
537 const RTLIB::Libcall Op;
538 const char *const Name;
539 const CallingConv::ID CC;
540 const ISD::CondCode Cond;
541 } MemOpsLibraryCalls[] = {
542 // Memory operations
543 // RTABI chapter 4.3.4
544 { RTLIB::MEMCPY, "__aeabi_memcpy", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID },
545 { RTLIB::MEMMOVE, "__aeabi_memmove", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID },
546 { RTLIB::MEMSET, "__aeabi_memset", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID },
547 };
548
549 for (const auto &LC : MemOpsLibraryCalls) {
550 setLibcallName(LC.Op, LC.Name);
551 setLibcallCallingConv(LC.Op, LC.CC);
552 if (LC.Cond != ISD::SETCC_INVALID)
553 setCmpLibcallCC(LC.Op, LC.Cond);
554 }
555 }
Anton Korobeynikova6b3ce22009-08-14 20:10:52 +0000556 }
557
Saleem Abdulrasool056fc3d2014-05-16 05:41:33 +0000558 if (Subtarget->isTargetWindows()) {
559 static const struct {
560 const RTLIB::Libcall Op;
561 const char * const Name;
562 const CallingConv::ID CC;
563 } LibraryCalls[] = {
564 { RTLIB::FPTOSINT_F32_I64, "__stoi64", CallingConv::ARM_AAPCS_VFP },
565 { RTLIB::FPTOSINT_F64_I64, "__dtoi64", CallingConv::ARM_AAPCS_VFP },
566 { RTLIB::FPTOUINT_F32_I64, "__stou64", CallingConv::ARM_AAPCS_VFP },
567 { RTLIB::FPTOUINT_F64_I64, "__dtou64", CallingConv::ARM_AAPCS_VFP },
568 { RTLIB::SINTTOFP_I64_F32, "__i64tos", CallingConv::ARM_AAPCS_VFP },
569 { RTLIB::SINTTOFP_I64_F64, "__i64tod", CallingConv::ARM_AAPCS_VFP },
570 { RTLIB::UINTTOFP_I64_F32, "__u64tos", CallingConv::ARM_AAPCS_VFP },
571 { RTLIB::UINTTOFP_I64_F64, "__u64tod", CallingConv::ARM_AAPCS_VFP },
572 };
573
574 for (const auto &LC : LibraryCalls) {
575 setLibcallName(LC.Op, LC.Name);
576 setLibcallCallingConv(LC.Op, LC.CC);
577 }
578 }
579
Bob Wilsonbc158992011-10-07 16:59:21 +0000580 // Use divmod compiler-rt calls for iOS 5.0 and later.
Tim Northover8b403662015-10-28 22:51:16 +0000581 if (Subtarget->isTargetWatchOS() ||
582 (Subtarget->isTargetIOS() &&
583 !Subtarget->getTargetTriple().isOSVersionLT(5, 0))) {
Bob Wilsonbc158992011-10-07 16:59:21 +0000584 setLibcallName(RTLIB::SDIVREM_I32, "__divmodsi4");
585 setLibcallName(RTLIB::UDIVREM_I32, "__udivmodsi4");
586 }
587
Ahmed Bougachaf0b46ee2016-06-24 00:08:01 +0000588 // The half <-> float conversion functions are always soft-float on
589 // non-watchos platforms, but are needed for some targets which use a
590 // hard-float calling convention by default.
591 if (!Subtarget->isTargetWatchABI()) {
592 if (Subtarget->isAAPCS_ABI()) {
593 setLibcallCallingConv(RTLIB::FPROUND_F32_F16, CallingConv::ARM_AAPCS);
594 setLibcallCallingConv(RTLIB::FPROUND_F64_F16, CallingConv::ARM_AAPCS);
595 setLibcallCallingConv(RTLIB::FPEXT_F16_F32, CallingConv::ARM_AAPCS);
596 } else {
597 setLibcallCallingConv(RTLIB::FPROUND_F32_F16, CallingConv::ARM_APCS);
598 setLibcallCallingConv(RTLIB::FPROUND_F64_F16, CallingConv::ARM_APCS);
599 setLibcallCallingConv(RTLIB::FPEXT_F16_F32, CallingConv::ARM_APCS);
600 }
Oliver Stannard11790b22014-08-11 09:12:32 +0000601 }
602
Oliver Stannardd3d114b2015-10-07 16:58:49 +0000603 // In EABI, these functions have an __aeabi_ prefix, but in GNUEABI they have
604 // a __gnu_ prefix (which is the default).
605 if (Subtarget->isTargetAEABI()) {
606 setLibcallName(RTLIB::FPROUND_F32_F16, "__aeabi_f2h");
607 setLibcallName(RTLIB::FPROUND_F64_F16, "__aeabi_d2h");
608 setLibcallName(RTLIB::FPEXT_F16_F32, "__aeabi_h2f");
609 }
610
David Goodwin22c2fba2009-07-08 23:10:31 +0000611 if (Subtarget->isThumb1Only())
Craig Topperc7242e02012-04-20 07:30:17 +0000612 addRegisterClass(MVT::i32, &ARM::tGPRRegClass);
Jim Grosbachfde21102009-04-07 20:34:09 +0000613 else
Craig Topperc7242e02012-04-20 07:30:17 +0000614 addRegisterClass(MVT::i32, &ARM::GPRRegClass);
Eric Christopher824f42f2015-05-12 01:26:05 +0000615 if (!Subtarget->useSoftFloat() && Subtarget->hasVFP2() &&
Nick Lewycky50f02cb2011-12-02 22:16:29 +0000616 !Subtarget->isThumb1Only()) {
Craig Topperc7242e02012-04-20 07:30:17 +0000617 addRegisterClass(MVT::f32, &ARM::SPRRegClass);
Oliver Stannard51b1d462014-08-21 12:50:31 +0000618 addRegisterClass(MVT::f64, &ARM::DPRRegClass);
Evan Cheng10043e22007-01-19 07:51:42 +0000619 }
Bob Wilson2e076c42009-06-22 23:27:02 +0000620
Ahmed Bougacha67dd2d22015-01-07 21:27:10 +0000621 for (MVT VT : MVT::vector_valuetypes()) {
Ahmed Bougacha2b6917b2015-01-08 00:51:32 +0000622 for (MVT InnerVT : MVT::vector_valuetypes()) {
Ahmed Bougacha67dd2d22015-01-07 21:27:10 +0000623 setTruncStoreAction(VT, InnerVT, Expand);
Ahmed Bougacha2b6917b2015-01-08 00:51:32 +0000624 setLoadExtAction(ISD::SEXTLOAD, VT, InnerVT, Expand);
625 setLoadExtAction(ISD::ZEXTLOAD, VT, InnerVT, Expand);
626 setLoadExtAction(ISD::EXTLOAD, VT, InnerVT, Expand);
627 }
Benjamin Kramer4dae5982014-04-26 12:06:28 +0000628
Ahmed Bougacha67dd2d22015-01-07 21:27:10 +0000629 setOperationAction(ISD::MULHS, VT, Expand);
630 setOperationAction(ISD::SMUL_LOHI, VT, Expand);
631 setOperationAction(ISD::MULHU, VT, Expand);
632 setOperationAction(ISD::UMUL_LOHI, VT, Expand);
Benjamin Kramerf3ad2352014-05-19 13:12:38 +0000633
Ahmed Bougacha67dd2d22015-01-07 21:27:10 +0000634 setOperationAction(ISD::BSWAP, VT, Expand);
Eli Friedman6f84fed2011-11-08 01:43:53 +0000635 }
636
Lang Hamesc35ee8b2012-03-15 18:49:02 +0000637 setOperationAction(ISD::ConstantFP, MVT::f32, Custom);
Tim Northoverf79c3a52013-08-20 08:57:11 +0000638 setOperationAction(ISD::ConstantFP, MVT::f64, Custom);
Lang Hamesc35ee8b2012-03-15 18:49:02 +0000639
Luke Cheeseman85fd06d2015-06-01 12:02:47 +0000640 setOperationAction(ISD::READ_REGISTER, MVT::i64, Custom);
641 setOperationAction(ISD::WRITE_REGISTER, MVT::i64, Custom);
642
Bob Wilson2e076c42009-06-22 23:27:02 +0000643 if (Subtarget->hasNEON()) {
Owen Anderson9f944592009-08-11 20:47:22 +0000644 addDRTypeForNEON(MVT::v2f32);
645 addDRTypeForNEON(MVT::v8i8);
646 addDRTypeForNEON(MVT::v4i16);
647 addDRTypeForNEON(MVT::v2i32);
648 addDRTypeForNEON(MVT::v1i64);
Bob Wilson2e076c42009-06-22 23:27:02 +0000649
Owen Anderson9f944592009-08-11 20:47:22 +0000650 addQRTypeForNEON(MVT::v4f32);
651 addQRTypeForNEON(MVT::v2f64);
652 addQRTypeForNEON(MVT::v16i8);
653 addQRTypeForNEON(MVT::v8i16);
654 addQRTypeForNEON(MVT::v4i32);
655 addQRTypeForNEON(MVT::v2i64);
Bob Wilson2e076c42009-06-22 23:27:02 +0000656
Bob Wilson194a2512009-09-15 23:55:57 +0000657 // v2f64 is legal so that QR subregs can be extracted as f64 elements, but
658 // neither Neon nor VFP support any arithmetic operations on it.
Stepan Dyatkovskiy46837402011-12-11 14:35:48 +0000659 // The same with v4f32. But keep in mind that vadd, vsub, vmul are natively
660 // supported for v4f32.
Bob Wilson194a2512009-09-15 23:55:57 +0000661 setOperationAction(ISD::FADD, MVT::v2f64, Expand);
662 setOperationAction(ISD::FSUB, MVT::v2f64, Expand);
663 setOperationAction(ISD::FMUL, MVT::v2f64, Expand);
Stepan Dyatkovskiy46837402011-12-11 14:35:48 +0000664 // FIXME: Code duplication: FDIV and FREM are expanded always, see
665 // ARMTargetLowering::addTypeForNEON method for details.
Bob Wilson194a2512009-09-15 23:55:57 +0000666 setOperationAction(ISD::FDIV, MVT::v2f64, Expand);
667 setOperationAction(ISD::FREM, MVT::v2f64, Expand);
Stepan Dyatkovskiy46837402011-12-11 14:35:48 +0000668 // FIXME: Create unittest.
669 // In another words, find a way when "copysign" appears in DAG with vector
670 // operands.
Bob Wilson194a2512009-09-15 23:55:57 +0000671 setOperationAction(ISD::FCOPYSIGN, MVT::v2f64, Expand);
Stepan Dyatkovskiy46837402011-12-11 14:35:48 +0000672 // FIXME: Code duplication: SETCC has custom operation action, see
673 // ARMTargetLowering::addTypeForNEON method for details.
Duncan Sandsf2641e12011-09-06 19:07:46 +0000674 setOperationAction(ISD::SETCC, MVT::v2f64, Expand);
Stepan Dyatkovskiy46837402011-12-11 14:35:48 +0000675 // FIXME: Create unittest for FNEG and for FABS.
Bob Wilson194a2512009-09-15 23:55:57 +0000676 setOperationAction(ISD::FNEG, MVT::v2f64, Expand);
677 setOperationAction(ISD::FABS, MVT::v2f64, Expand);
678 setOperationAction(ISD::FSQRT, MVT::v2f64, Expand);
679 setOperationAction(ISD::FSIN, MVT::v2f64, Expand);
680 setOperationAction(ISD::FCOS, MVT::v2f64, Expand);
681 setOperationAction(ISD::FPOWI, MVT::v2f64, Expand);
682 setOperationAction(ISD::FPOW, MVT::v2f64, Expand);
683 setOperationAction(ISD::FLOG, MVT::v2f64, Expand);
684 setOperationAction(ISD::FLOG2, MVT::v2f64, Expand);
685 setOperationAction(ISD::FLOG10, MVT::v2f64, Expand);
686 setOperationAction(ISD::FEXP, MVT::v2f64, Expand);
687 setOperationAction(ISD::FEXP2, MVT::v2f64, Expand);
Stepan Dyatkovskiy46837402011-12-11 14:35:48 +0000688 // FIXME: Create unittest for FCEIL, FTRUNC, FRINT, FNEARBYINT, FFLOOR.
Bob Wilson194a2512009-09-15 23:55:57 +0000689 setOperationAction(ISD::FCEIL, MVT::v2f64, Expand);
690 setOperationAction(ISD::FTRUNC, MVT::v2f64, Expand);
691 setOperationAction(ISD::FRINT, MVT::v2f64, Expand);
692 setOperationAction(ISD::FNEARBYINT, MVT::v2f64, Expand);
693 setOperationAction(ISD::FFLOOR, MVT::v2f64, Expand);
Arnold Schwaighofer99cba962013-03-02 19:38:33 +0000694 setOperationAction(ISD::FMA, MVT::v2f64, Expand);
Lang Hames591cdaf2012-03-29 21:56:11 +0000695
Stepan Dyatkovskiy46837402011-12-11 14:35:48 +0000696 setOperationAction(ISD::FSQRT, MVT::v4f32, Expand);
697 setOperationAction(ISD::FSIN, MVT::v4f32, Expand);
698 setOperationAction(ISD::FCOS, MVT::v4f32, Expand);
699 setOperationAction(ISD::FPOWI, MVT::v4f32, Expand);
700 setOperationAction(ISD::FPOW, MVT::v4f32, Expand);
701 setOperationAction(ISD::FLOG, MVT::v4f32, Expand);
702 setOperationAction(ISD::FLOG2, MVT::v4f32, Expand);
703 setOperationAction(ISD::FLOG10, MVT::v4f32, Expand);
704 setOperationAction(ISD::FEXP, MVT::v4f32, Expand);
705 setOperationAction(ISD::FEXP2, MVT::v4f32, Expand);
Craig Topper61d04572012-11-15 06:51:10 +0000706 setOperationAction(ISD::FCEIL, MVT::v4f32, Expand);
707 setOperationAction(ISD::FTRUNC, MVT::v4f32, Expand);
708 setOperationAction(ISD::FRINT, MVT::v4f32, Expand);
709 setOperationAction(ISD::FNEARBYINT, MVT::v4f32, Expand);
Craig Topper3e41a5b2012-09-08 04:58:43 +0000710 setOperationAction(ISD::FFLOOR, MVT::v4f32, Expand);
Bob Wilson194a2512009-09-15 23:55:57 +0000711
Arnold Schwaighofer99cba962013-03-02 19:38:33 +0000712 // Mark v2f32 intrinsics.
713 setOperationAction(ISD::FSQRT, MVT::v2f32, Expand);
714 setOperationAction(ISD::FSIN, MVT::v2f32, Expand);
715 setOperationAction(ISD::FCOS, MVT::v2f32, Expand);
716 setOperationAction(ISD::FPOWI, MVT::v2f32, Expand);
717 setOperationAction(ISD::FPOW, MVT::v2f32, Expand);
718 setOperationAction(ISD::FLOG, MVT::v2f32, Expand);
719 setOperationAction(ISD::FLOG2, MVT::v2f32, Expand);
720 setOperationAction(ISD::FLOG10, MVT::v2f32, Expand);
721 setOperationAction(ISD::FEXP, MVT::v2f32, Expand);
722 setOperationAction(ISD::FEXP2, MVT::v2f32, Expand);
723 setOperationAction(ISD::FCEIL, MVT::v2f32, Expand);
724 setOperationAction(ISD::FTRUNC, MVT::v2f32, Expand);
725 setOperationAction(ISD::FRINT, MVT::v2f32, Expand);
726 setOperationAction(ISD::FNEARBYINT, MVT::v2f32, Expand);
727 setOperationAction(ISD::FFLOOR, MVT::v2f32, Expand);
728
Bob Wilson6cc46572009-09-16 00:32:15 +0000729 // Neon does not support some operations on v1i64 and v2i64 types.
730 setOperationAction(ISD::MUL, MVT::v1i64, Expand);
Bob Wilson38ab35a2010-09-01 23:50:19 +0000731 // Custom handling for some quad-vector types to detect VMULL.
732 setOperationAction(ISD::MUL, MVT::v8i16, Custom);
733 setOperationAction(ISD::MUL, MVT::v4i32, Custom);
734 setOperationAction(ISD::MUL, MVT::v2i64, Custom);
Nate Begemanfa62d502011-02-11 20:53:29 +0000735 // Custom handling for some vector types to avoid expensive expansions
736 setOperationAction(ISD::SDIV, MVT::v4i16, Custom);
737 setOperationAction(ISD::SDIV, MVT::v8i8, Custom);
738 setOperationAction(ISD::UDIV, MVT::v4i16, Custom);
739 setOperationAction(ISD::UDIV, MVT::v8i8, Custom);
Duncan Sandsf2641e12011-09-06 19:07:46 +0000740 setOperationAction(ISD::SETCC, MVT::v1i64, Expand);
741 setOperationAction(ISD::SETCC, MVT::v2i64, Expand);
Cameron Zwarich143f9ae2011-03-29 21:41:55 +0000742 // Neon does not have single instruction SINT_TO_FP and UINT_TO_FP with
James Molloy547d4c02012-02-20 09:24:05 +0000743 // a destination type that is wider than the source, and nor does
744 // it have a FP_TO_[SU]INT instruction with a narrower destination than
745 // source.
Cameron Zwarich143f9ae2011-03-29 21:41:55 +0000746 setOperationAction(ISD::SINT_TO_FP, MVT::v4i16, Custom);
747 setOperationAction(ISD::UINT_TO_FP, MVT::v4i16, Custom);
James Molloy547d4c02012-02-20 09:24:05 +0000748 setOperationAction(ISD::FP_TO_UINT, MVT::v4i16, Custom);
749 setOperationAction(ISD::FP_TO_SINT, MVT::v4i16, Custom);
Bob Wilson6cc46572009-09-16 00:32:15 +0000750
Eli Friedmane6385e62012-11-15 22:44:27 +0000751 setOperationAction(ISD::FP_ROUND, MVT::v2f32, Expand);
Eli Friedman30834942012-11-17 01:52:46 +0000752 setOperationAction(ISD::FP_EXTEND, MVT::v2f64, Expand);
Eli Friedmane6385e62012-11-15 22:44:27 +0000753
Evan Chengb4eae132012-12-04 22:41:50 +0000754 // NEON does not have single instruction CTPOP for vectors with element
755 // types wider than 8-bits. However, custom lowering can leverage the
756 // v8i8/v16i8 vcnt instruction.
757 setOperationAction(ISD::CTPOP, MVT::v2i32, Custom);
758 setOperationAction(ISD::CTPOP, MVT::v4i32, Custom);
759 setOperationAction(ISD::CTPOP, MVT::v4i16, Custom);
760 setOperationAction(ISD::CTPOP, MVT::v8i16, Custom);
Benjamin Kramer569efd22016-03-31 19:42:04 +0000761 setOperationAction(ISD::CTPOP, MVT::v1i64, Expand);
762 setOperationAction(ISD::CTPOP, MVT::v2i64, Expand);
Evan Chengb4eae132012-12-04 22:41:50 +0000763
Craig Topperedb4a6b2016-04-26 05:04:33 +0000764 setOperationAction(ISD::CTLZ, MVT::v1i64, Expand);
765 setOperationAction(ISD::CTLZ, MVT::v2i64, Expand);
766
Logan Chien0a43abc2015-07-13 15:37:30 +0000767 // NEON does not have single instruction CTTZ for vectors.
768 setOperationAction(ISD::CTTZ, MVT::v8i8, Custom);
769 setOperationAction(ISD::CTTZ, MVT::v4i16, Custom);
770 setOperationAction(ISD::CTTZ, MVT::v2i32, Custom);
771 setOperationAction(ISD::CTTZ, MVT::v1i64, Custom);
772
773 setOperationAction(ISD::CTTZ, MVT::v16i8, Custom);
774 setOperationAction(ISD::CTTZ, MVT::v8i16, Custom);
775 setOperationAction(ISD::CTTZ, MVT::v4i32, Custom);
776 setOperationAction(ISD::CTTZ, MVT::v2i64, Custom);
777
778 setOperationAction(ISD::CTTZ_ZERO_UNDEF, MVT::v8i8, Custom);
779 setOperationAction(ISD::CTTZ_ZERO_UNDEF, MVT::v4i16, Custom);
780 setOperationAction(ISD::CTTZ_ZERO_UNDEF, MVT::v2i32, Custom);
781 setOperationAction(ISD::CTTZ_ZERO_UNDEF, MVT::v1i64, Custom);
782
783 setOperationAction(ISD::CTTZ_ZERO_UNDEF, MVT::v16i8, Custom);
784 setOperationAction(ISD::CTTZ_ZERO_UNDEF, MVT::v8i16, Custom);
785 setOperationAction(ISD::CTTZ_ZERO_UNDEF, MVT::v4i32, Custom);
786 setOperationAction(ISD::CTTZ_ZERO_UNDEF, MVT::v2i64, Custom);
787
Jim Grosbach5f215872013-02-27 21:31:12 +0000788 // NEON only has FMA instructions as of VFP4.
789 if (!Subtarget->hasVFP4()) {
790 setOperationAction(ISD::FMA, MVT::v2f32, Expand);
791 setOperationAction(ISD::FMA, MVT::v4f32, Expand);
792 }
793
Bob Wilson06fce872011-02-07 17:43:21 +0000794 setTargetDAGCombine(ISD::INTRINSIC_VOID);
795 setTargetDAGCombine(ISD::INTRINSIC_W_CHAIN);
Bob Wilson2e076c42009-06-22 23:27:02 +0000796 setTargetDAGCombine(ISD::INTRINSIC_WO_CHAIN);
797 setTargetDAGCombine(ISD::SHL);
798 setTargetDAGCombine(ISD::SRL);
799 setTargetDAGCombine(ISD::SRA);
800 setTargetDAGCombine(ISD::SIGN_EXTEND);
801 setTargetDAGCombine(ISD::ZERO_EXTEND);
802 setTargetDAGCombine(ISD::ANY_EXTEND);
Bob Wilsoncb6db982010-09-17 22:59:05 +0000803 setTargetDAGCombine(ISD::BUILD_VECTOR);
Bob Wilsonc7334a12010-10-27 20:38:28 +0000804 setTargetDAGCombine(ISD::VECTOR_SHUFFLE);
Bob Wilson1a20c2a2010-12-21 06:43:19 +0000805 setTargetDAGCombine(ISD::INSERT_VECTOR_ELT);
806 setTargetDAGCombine(ISD::STORE);
Chad Rosierfa8d8932011-06-24 19:23:04 +0000807 setTargetDAGCombine(ISD::FP_TO_SINT);
808 setTargetDAGCombine(ISD::FP_TO_UINT);
809 setTargetDAGCombine(ISD::FDIV);
Ahmed Bougachadb141ac2015-02-19 23:52:41 +0000810 setTargetDAGCombine(ISD::LOAD);
Nadav Rotem097106b2011-10-15 20:03:12 +0000811
James Molloy547d4c02012-02-20 09:24:05 +0000812 // It is legal to extload from v4i8 to v4i16 or v4i32.
Benjamin Kramer867bfc52015-03-07 17:41:00 +0000813 for (MVT Ty : {MVT::v8i8, MVT::v4i8, MVT::v2i8, MVT::v4i16, MVT::v2i16,
814 MVT::v2i32}) {
Ahmed Bougacha2b6917b2015-01-08 00:51:32 +0000815 for (MVT VT : MVT::integer_vector_valuetypes()) {
Benjamin Kramer867bfc52015-03-07 17:41:00 +0000816 setLoadExtAction(ISD::EXTLOAD, VT, Ty, Legal);
817 setLoadExtAction(ISD::ZEXTLOAD, VT, Ty, Legal);
818 setLoadExtAction(ISD::SEXTLOAD, VT, Ty, Legal);
Ahmed Bougacha2b6917b2015-01-08 00:51:32 +0000819 }
James Molloy547d4c02012-02-20 09:24:05 +0000820 }
Bob Wilson2e076c42009-06-22 23:27:02 +0000821 }
822
Arnold Schwaighoferf00fb1c2012-09-04 14:37:49 +0000823 // ARM and Thumb2 support UMLAL/SMLAL.
824 if (!Subtarget->isThumb1Only())
825 setTargetDAGCombine(ISD::ADDC);
826
Oliver Stannard51b1d462014-08-21 12:50:31 +0000827 if (Subtarget->isFPOnlySP()) {
Benjamin Kramerdf005cb2015-08-08 18:27:36 +0000828 // When targeting a floating-point unit with only single-precision
Oliver Stannard51b1d462014-08-21 12:50:31 +0000829 // operations, f64 is legal for the few double-precision instructions which
830 // are present However, no double-precision operations other than moves,
831 // loads and stores are provided by the hardware.
832 setOperationAction(ISD::FADD, MVT::f64, Expand);
833 setOperationAction(ISD::FSUB, MVT::f64, Expand);
834 setOperationAction(ISD::FMUL, MVT::f64, Expand);
835 setOperationAction(ISD::FMA, MVT::f64, Expand);
836 setOperationAction(ISD::FDIV, MVT::f64, Expand);
837 setOperationAction(ISD::FREM, MVT::f64, Expand);
838 setOperationAction(ISD::FCOPYSIGN, MVT::f64, Expand);
839 setOperationAction(ISD::FGETSIGN, MVT::f64, Expand);
840 setOperationAction(ISD::FNEG, MVT::f64, Expand);
841 setOperationAction(ISD::FABS, MVT::f64, Expand);
842 setOperationAction(ISD::FSQRT, MVT::f64, Expand);
843 setOperationAction(ISD::FSIN, MVT::f64, Expand);
844 setOperationAction(ISD::FCOS, MVT::f64, Expand);
845 setOperationAction(ISD::FPOWI, MVT::f64, Expand);
846 setOperationAction(ISD::FPOW, MVT::f64, Expand);
847 setOperationAction(ISD::FLOG, MVT::f64, Expand);
848 setOperationAction(ISD::FLOG2, MVT::f64, Expand);
849 setOperationAction(ISD::FLOG10, MVT::f64, Expand);
850 setOperationAction(ISD::FEXP, MVT::f64, Expand);
851 setOperationAction(ISD::FEXP2, MVT::f64, Expand);
852 setOperationAction(ISD::FCEIL, MVT::f64, Expand);
853 setOperationAction(ISD::FTRUNC, MVT::f64, Expand);
854 setOperationAction(ISD::FRINT, MVT::f64, Expand);
855 setOperationAction(ISD::FNEARBYINT, MVT::f64, Expand);
856 setOperationAction(ISD::FFLOOR, MVT::f64, Expand);
James Molloyfa041152015-03-23 16:15:16 +0000857 setOperationAction(ISD::SINT_TO_FP, MVT::i32, Custom);
858 setOperationAction(ISD::UINT_TO_FP, MVT::i32, Custom);
859 setOperationAction(ISD::FP_TO_SINT, MVT::i32, Custom);
860 setOperationAction(ISD::FP_TO_UINT, MVT::i32, Custom);
861 setOperationAction(ISD::FP_TO_SINT, MVT::f64, Custom);
862 setOperationAction(ISD::FP_TO_UINT, MVT::f64, Custom);
Oliver Stannard51b1d462014-08-21 12:50:31 +0000863 setOperationAction(ISD::FP_ROUND, MVT::f32, Custom);
864 setOperationAction(ISD::FP_EXTEND, MVT::f64, Custom);
865 }
Arnold Schwaighoferf00fb1c2012-09-04 14:37:49 +0000866
Eric Christopher23a3a7c2015-02-26 00:00:24 +0000867 computeRegisterProperties(Subtarget->getRegisterInfo());
Evan Cheng10043e22007-01-19 07:51:42 +0000868
Tim Northover4e80b582014-07-18 13:01:19 +0000869 // ARM does not have floating-point extending loads.
Ahmed Bougacha2b6917b2015-01-08 00:51:32 +0000870 for (MVT VT : MVT::fp_valuetypes()) {
871 setLoadExtAction(ISD::EXTLOAD, VT, MVT::f32, Expand);
872 setLoadExtAction(ISD::EXTLOAD, VT, MVT::f16, Expand);
873 }
Tim Northover4e80b582014-07-18 13:01:19 +0000874
875 // ... or truncating stores
876 setTruncStoreAction(MVT::f64, MVT::f32, Expand);
877 setTruncStoreAction(MVT::f32, MVT::f16, Expand);
878 setTruncStoreAction(MVT::f64, MVT::f16, Expand);
Evan Cheng10043e22007-01-19 07:51:42 +0000879
Duncan Sands95d46ef2008-01-23 20:39:46 +0000880 // ARM does not have i1 sign extending load.
Ahmed Bougacha2b6917b2015-01-08 00:51:32 +0000881 for (MVT VT : MVT::integer_valuetypes())
882 setLoadExtAction(ISD::SEXTLOAD, VT, MVT::i1, Promote);
Duncan Sands95d46ef2008-01-23 20:39:46 +0000883
Evan Cheng10043e22007-01-19 07:51:42 +0000884 // ARM supports all 4 flavors of integer indexed load / store.
Evan Cheng84c6cda2009-07-02 07:28:31 +0000885 if (!Subtarget->isThumb1Only()) {
886 for (unsigned im = (unsigned)ISD::PRE_INC;
887 im != (unsigned)ISD::LAST_INDEXED_MODE; ++im) {
Owen Anderson9f944592009-08-11 20:47:22 +0000888 setIndexedLoadAction(im, MVT::i1, Legal);
889 setIndexedLoadAction(im, MVT::i8, Legal);
890 setIndexedLoadAction(im, MVT::i16, Legal);
891 setIndexedLoadAction(im, MVT::i32, Legal);
892 setIndexedStoreAction(im, MVT::i1, Legal);
893 setIndexedStoreAction(im, MVT::i8, Legal);
894 setIndexedStoreAction(im, MVT::i16, Legal);
895 setIndexedStoreAction(im, MVT::i32, Legal);
Evan Cheng84c6cda2009-07-02 07:28:31 +0000896 }
James Molloyb3326df2016-07-15 08:03:56 +0000897 } else {
898 // Thumb-1 has limited post-inc load/store support - LDM r0!, {r1}.
899 setIndexedLoadAction(ISD::POST_INC, MVT::i32, Legal);
900 setIndexedStoreAction(ISD::POST_INC, MVT::i32, Legal);
Evan Cheng10043e22007-01-19 07:51:42 +0000901 }
902
Louis Gerbarg3342bf12014-05-09 17:02:49 +0000903 setOperationAction(ISD::SADDO, MVT::i32, Custom);
904 setOperationAction(ISD::UADDO, MVT::i32, Custom);
905 setOperationAction(ISD::SSUBO, MVT::i32, Custom);
906 setOperationAction(ISD::USUBO, MVT::i32, Custom);
907
Evan Cheng10043e22007-01-19 07:51:42 +0000908 // i64 operation support.
Eric Christopherc721b0db2011-04-19 18:49:19 +0000909 setOperationAction(ISD::MUL, MVT::i64, Expand);
910 setOperationAction(ISD::MULHU, MVT::i32, Expand);
Evan Chengb24e51e2009-07-07 01:17:28 +0000911 if (Subtarget->isThumb1Only()) {
Owen Anderson9f944592009-08-11 20:47:22 +0000912 setOperationAction(ISD::UMUL_LOHI, MVT::i32, Expand);
913 setOperationAction(ISD::SMUL_LOHI, MVT::i32, Expand);
Evan Cheng10043e22007-01-19 07:51:42 +0000914 }
Jim Grosbachcf1464d2011-07-01 21:12:19 +0000915 if (Subtarget->isThumb1Only() || !Subtarget->hasV6Ops()
Artyom Skrobovcf296442015-09-24 17:31:16 +0000916 || (Subtarget->isThumb2() && !Subtarget->hasDSP()))
Eric Christopherc721b0db2011-04-19 18:49:19 +0000917 setOperationAction(ISD::MULHS, MVT::i32, Expand);
918
Jim Grosbach5d994042009-10-31 19:38:01 +0000919 setOperationAction(ISD::SHL_PARTS, MVT::i32, Custom);
Jim Grosbach624fcb22009-10-31 21:00:56 +0000920 setOperationAction(ISD::SRA_PARTS, MVT::i32, Custom);
Jim Grosbach8fe6fd72009-10-31 21:42:19 +0000921 setOperationAction(ISD::SRL_PARTS, MVT::i32, Custom);
Owen Anderson9f944592009-08-11 20:47:22 +0000922 setOperationAction(ISD::SRL, MVT::i64, Custom);
923 setOperationAction(ISD::SRA, MVT::i64, Custom);
Evan Cheng10043e22007-01-19 07:51:42 +0000924
Evan Chenge8916542011-08-30 01:34:54 +0000925 if (!Subtarget->isThumb1Only()) {
926 // FIXME: We should do this for Thumb1 as well.
927 setOperationAction(ISD::ADDC, MVT::i32, Custom);
928 setOperationAction(ISD::ADDE, MVT::i32, Custom);
929 setOperationAction(ISD::SUBC, MVT::i32, Custom);
930 setOperationAction(ISD::SUBE, MVT::i32, Custom);
931 }
932
Weiming Zhao4b3b13d2016-01-08 18:43:41 +0000933 if (!Subtarget->isThumb1Only() && Subtarget->hasV6T2Ops())
James Molloyb5640982015-11-13 16:05:22 +0000934 setOperationAction(ISD::BITREVERSE, MVT::i32, Legal);
935
Evan Cheng10043e22007-01-19 07:51:42 +0000936 // ARM does not have ROTL.
Charlie Turner458e79b2015-10-27 10:25:20 +0000937 setOperationAction(ISD::ROTL, MVT::i32, Expand);
938 for (MVT VT : MVT::vector_valuetypes()) {
939 setOperationAction(ISD::ROTL, VT, Expand);
940 setOperationAction(ISD::ROTR, VT, Expand);
941 }
Jim Grosbach8546ec92010-01-18 19:58:49 +0000942 setOperationAction(ISD::CTTZ, MVT::i32, Custom);
Owen Anderson9f944592009-08-11 20:47:22 +0000943 setOperationAction(ISD::CTPOP, MVT::i32, Expand);
David Goodwinaa294c52009-06-26 20:47:43 +0000944 if (!Subtarget->hasV5TOps() || Subtarget->isThumb1Only())
Owen Anderson9f944592009-08-11 20:47:22 +0000945 setOperationAction(ISD::CTLZ, MVT::i32, Expand);
Evan Cheng10043e22007-01-19 07:51:42 +0000946
Ahmed Bougachaf9c19da2015-08-28 01:49:59 +0000947 // @llvm.readcyclecounter requires the Performance Monitors extension.
948 // Default to the 0 expansion on unsupported platforms.
949 // FIXME: Technically there are older ARM CPUs that have
950 // implementation-specific ways of obtaining this information.
951 if (Subtarget->hasPerfMon())
952 setOperationAction(ISD::READCYCLECOUNTER, MVT::i64, Custom);
Tim Northoverbc933082013-05-23 19:11:20 +0000953
Lauro Ramos Venancio25d40522007-03-16 22:54:16 +0000954 // Only ARMv6 has BSWAP.
955 if (!Subtarget->hasV6Ops())
Owen Anderson9f944592009-08-11 20:47:22 +0000956 setOperationAction(ISD::BSWAP, MVT::i32, Expand);
Lauro Ramos Venancio25d40522007-03-16 22:54:16 +0000957
Bradley Smith519563e2016-01-15 10:25:35 +0000958 bool hasDivide = Subtarget->isThumb() ? Subtarget->hasDivide()
959 : Subtarget->hasDivideInARMMode();
960 if (!hasDivide) {
Bob Wilsone8a549c2012-09-29 21:43:49 +0000961 // These are expanded into libcalls if the cpu doesn't have HW divider.
Artyom Skrobov7fd67e22015-10-20 13:14:52 +0000962 setOperationAction(ISD::SDIV, MVT::i32, LibCall);
963 setOperationAction(ISD::UDIV, MVT::i32, LibCall);
Jim Grosbach92d999002010-05-05 20:44:35 +0000964 }
Renato Golin87610692013-07-16 09:32:17 +0000965
Saleem Abdulrasool071a0992016-03-17 14:10:49 +0000966 if (Subtarget->isTargetWindows() && !Subtarget->hasDivide()) {
967 setOperationAction(ISD::SDIV, MVT::i32, Custom);
968 setOperationAction(ISD::UDIV, MVT::i32, Custom);
969
970 setOperationAction(ISD::SDIV, MVT::i64, Custom);
971 setOperationAction(ISD::UDIV, MVT::i64, Custom);
972 }
973
Chad Rosierad7c9102014-08-23 18:29:43 +0000974 setOperationAction(ISD::SREM, MVT::i32, Expand);
975 setOperationAction(ISD::UREM, MVT::i32, Expand);
976 // Register based DivRem for AEABI (RTABI 4.2)
Renato Golin6027dd38e2016-02-03 16:10:54 +0000977 if (Subtarget->isTargetAEABI() || Subtarget->isTargetAndroid() ||
Rafael Espindolaa895a0c2016-06-24 21:14:33 +0000978 Subtarget->isTargetGNUAEABI() || Subtarget->isTargetMuslAEABI()) {
Scott Douglassbdef6042015-08-24 09:17:18 +0000979 setOperationAction(ISD::SREM, MVT::i64, Custom);
980 setOperationAction(ISD::UREM, MVT::i64, Custom);
Diana Picus774d1572016-07-18 06:48:25 +0000981 HasStandaloneRem = false;
Scott Douglassbdef6042015-08-24 09:17:18 +0000982
Chad Rosierad7c9102014-08-23 18:29:43 +0000983 setLibcallName(RTLIB::SDIVREM_I8, "__aeabi_idivmod");
984 setLibcallName(RTLIB::SDIVREM_I16, "__aeabi_idivmod");
985 setLibcallName(RTLIB::SDIVREM_I32, "__aeabi_idivmod");
986 setLibcallName(RTLIB::SDIVREM_I64, "__aeabi_ldivmod");
987 setLibcallName(RTLIB::UDIVREM_I8, "__aeabi_uidivmod");
988 setLibcallName(RTLIB::UDIVREM_I16, "__aeabi_uidivmod");
989 setLibcallName(RTLIB::UDIVREM_I32, "__aeabi_uidivmod");
990 setLibcallName(RTLIB::UDIVREM_I64, "__aeabi_uldivmod");
991
992 setLibcallCallingConv(RTLIB::SDIVREM_I8, CallingConv::ARM_AAPCS);
993 setLibcallCallingConv(RTLIB::SDIVREM_I16, CallingConv::ARM_AAPCS);
994 setLibcallCallingConv(RTLIB::SDIVREM_I32, CallingConv::ARM_AAPCS);
995 setLibcallCallingConv(RTLIB::SDIVREM_I64, CallingConv::ARM_AAPCS);
996 setLibcallCallingConv(RTLIB::UDIVREM_I8, CallingConv::ARM_AAPCS);
997 setLibcallCallingConv(RTLIB::UDIVREM_I16, CallingConv::ARM_AAPCS);
998 setLibcallCallingConv(RTLIB::UDIVREM_I32, CallingConv::ARM_AAPCS);
999 setLibcallCallingConv(RTLIB::UDIVREM_I64, CallingConv::ARM_AAPCS);
1000
1001 setOperationAction(ISD::SDIVREM, MVT::i32, Custom);
1002 setOperationAction(ISD::UDIVREM, MVT::i32, Custom);
Renato Golin175c6d62016-03-04 19:19:36 +00001003 setOperationAction(ISD::SDIVREM, MVT::i64, Custom);
1004 setOperationAction(ISD::UDIVREM, MVT::i64, Custom);
Chad Rosierad7c9102014-08-23 18:29:43 +00001005 } else {
Renato Golin87610692013-07-16 09:32:17 +00001006 setOperationAction(ISD::SDIVREM, MVT::i32, Expand);
1007 setOperationAction(ISD::UDIVREM, MVT::i32, Expand);
1008 }
Bob Wilson7117a912009-03-20 22:42:55 +00001009
Owen Anderson9f944592009-08-11 20:47:22 +00001010 setOperationAction(ISD::GlobalAddress, MVT::i32, Custom);
1011 setOperationAction(ISD::ConstantPool, MVT::i32, Custom);
Owen Anderson9f944592009-08-11 20:47:22 +00001012 setOperationAction(ISD::GlobalTLSAddress, MVT::i32, Custom);
Bob Wilson1cf0b032009-10-30 05:45:42 +00001013 setOperationAction(ISD::BlockAddress, MVT::i32, Custom);
Evan Cheng10043e22007-01-19 07:51:42 +00001014
Evan Cheng74d92c12011-04-08 21:37:21 +00001015 setOperationAction(ISD::TRAP, MVT::Other, Legal);
Evan Cheng2fa5a7e2010-05-11 07:26:32 +00001016
Evan Cheng10043e22007-01-19 07:51:42 +00001017 // Use the default implementation.
Owen Anderson9f944592009-08-11 20:47:22 +00001018 setOperationAction(ISD::VASTART, MVT::Other, Custom);
1019 setOperationAction(ISD::VAARG, MVT::Other, Expand);
1020 setOperationAction(ISD::VACOPY, MVT::Other, Expand);
1021 setOperationAction(ISD::VAEND, MVT::Other, Expand);
1022 setOperationAction(ISD::STACKSAVE, MVT::Other, Expand);
1023 setOperationAction(ISD::STACKRESTORE, MVT::Other, Expand);
Bill Wendling05d6f2f2012-02-13 23:47:16 +00001024
Saleem Abdulrasoolabac6e92014-06-09 20:18:42 +00001025 if (Subtarget->getTargetTriple().isWindowsItaniumEnvironment())
1026 setOperationAction(ISD::DYNAMIC_STACKALLOC, MVT::i32, Custom);
1027 else
1028 setOperationAction(ISD::DYNAMIC_STACKALLOC, MVT::i32, Expand);
1029
Evan Cheng6e809de2010-08-11 06:22:01 +00001030 // ARMv6 Thumb1 (except for CPUs that support dmb / dsb) and earlier use
James Y Knighte6a46462016-04-01 19:33:19 +00001031 // the default expansion.
James Y Knightf44fc522016-03-16 22:12:04 +00001032 InsertFencesForAtomic = false;
James Y Knighte6a46462016-04-01 19:33:19 +00001033 if (Subtarget->hasAnyDataBarrier() &&
1034 (!Subtarget->isThumb() || Subtarget->hasV8MBaselineOps())) {
Tim Northoverc882eb02014-04-03 11:44:58 +00001035 // ATOMIC_FENCE needs custom lowering; the others should have been expanded
1036 // to ldrex/strex loops already.
Tim Northoverc7ea8042013-10-25 09:30:24 +00001037 setOperationAction(ISD::ATOMIC_FENCE, MVT::Other, Custom);
Tim Northoverb629c772016-04-18 21:48:55 +00001038 if (!Subtarget->isThumb() || !Subtarget->isMClass())
1039 setOperationAction(ISD::ATOMIC_CMP_SWAP, MVT::i64, Custom);
Tim Northoverc882eb02014-04-03 11:44:58 +00001040
Amara Emersonb4ad2f32013-09-26 12:22:36 +00001041 // On v8, we have particularly efficient implementations of atomic fences
1042 // if they can be combined with nearby atomic loads and stores.
Tim Northoverb629c772016-04-18 21:48:55 +00001043 if (!Subtarget->hasV8Ops() || getTargetMachine().getOptLevel() == 0) {
Robin Morissetd18cda62014-08-15 22:17:28 +00001044 // Automatically insert fences (dmb ish) around ATOMIC_SWAP etc.
James Y Knightf44fc522016-03-16 22:12:04 +00001045 InsertFencesForAtomic = true;
Amara Emersonb4ad2f32013-09-26 12:22:36 +00001046 }
Jim Grosbach6860bb72010-06-18 22:35:32 +00001047 } else {
Tim Northoverc7ea8042013-10-25 09:30:24 +00001048 // If there's anything we can use as a barrier, go through custom lowering
1049 // for ATOMIC_FENCE.
1050 setOperationAction(ISD::ATOMIC_FENCE, MVT::Other,
1051 Subtarget->hasAnyDataBarrier() ? Custom : Expand);
1052
Jim Grosbach6860bb72010-06-18 22:35:32 +00001053 // Set them all for expansion, which will force libcalls.
Jim Grosbach6860bb72010-06-18 22:35:32 +00001054 setOperationAction(ISD::ATOMIC_CMP_SWAP, MVT::i32, Expand);
Jim Grosbacha57c2882010-06-18 23:03:10 +00001055 setOperationAction(ISD::ATOMIC_SWAP, MVT::i32, Expand);
Jim Grosbach6860bb72010-06-18 22:35:32 +00001056 setOperationAction(ISD::ATOMIC_LOAD_ADD, MVT::i32, Expand);
Jim Grosbach6860bb72010-06-18 22:35:32 +00001057 setOperationAction(ISD::ATOMIC_LOAD_SUB, MVT::i32, Expand);
Jim Grosbach6860bb72010-06-18 22:35:32 +00001058 setOperationAction(ISD::ATOMIC_LOAD_AND, MVT::i32, Expand);
Jim Grosbach6860bb72010-06-18 22:35:32 +00001059 setOperationAction(ISD::ATOMIC_LOAD_OR, MVT::i32, Expand);
Jim Grosbach6860bb72010-06-18 22:35:32 +00001060 setOperationAction(ISD::ATOMIC_LOAD_XOR, MVT::i32, Expand);
Jim Grosbach6860bb72010-06-18 22:35:32 +00001061 setOperationAction(ISD::ATOMIC_LOAD_NAND, MVT::i32, Expand);
Jim Grosbachd4b733e2011-04-26 19:44:18 +00001062 setOperationAction(ISD::ATOMIC_LOAD_MIN, MVT::i32, Expand);
Jim Grosbachd4b733e2011-04-26 19:44:18 +00001063 setOperationAction(ISD::ATOMIC_LOAD_MAX, MVT::i32, Expand);
Jim Grosbachd4b733e2011-04-26 19:44:18 +00001064 setOperationAction(ISD::ATOMIC_LOAD_UMIN, MVT::i32, Expand);
Jim Grosbachd4b733e2011-04-26 19:44:18 +00001065 setOperationAction(ISD::ATOMIC_LOAD_UMAX, MVT::i32, Expand);
Eli Friedmanba912e02011-09-15 22:18:49 +00001066 // Mark ATOMIC_LOAD and ATOMIC_STORE custom so we can handle the
1067 // Unordered/Monotonic case.
1068 setOperationAction(ISD::ATOMIC_LOAD, MVT::i32, Custom);
1069 setOperationAction(ISD::ATOMIC_STORE, MVT::i32, Custom);
Jim Grosbach6860bb72010-06-18 22:35:32 +00001070 }
Evan Cheng10043e22007-01-19 07:51:42 +00001071
Evan Cheng21acf9f2010-11-04 05:19:35 +00001072 setOperationAction(ISD::PREFETCH, MVT::Other, Custom);
Evan Cheng6f360422010-11-03 05:14:24 +00001073
Eli Friedman8cfa7712010-06-26 04:36:50 +00001074 // Requires SXTB/SXTH, available on v6 and up in both ARM and Thumb modes.
1075 if (!Subtarget->hasV6Ops()) {
Owen Anderson9f944592009-08-11 20:47:22 +00001076 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i16, Expand);
1077 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i8, Expand);
Evan Cheng10043e22007-01-19 07:51:42 +00001078 }
Owen Anderson9f944592009-08-11 20:47:22 +00001079 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i1, Expand);
Evan Cheng10043e22007-01-19 07:51:42 +00001080
Eric Christopher824f42f2015-05-12 01:26:05 +00001081 if (!Subtarget->useSoftFloat() && Subtarget->hasVFP2() &&
Nick Lewycky50f02cb2011-12-02 22:16:29 +00001082 !Subtarget->isThumb1Only()) {
Bob Wilson6a4491b2010-01-19 22:56:26 +00001083 // Turn f64->i64 into VMOVRRD, i64 -> f64 to VMOVDRR
Sylvestre Ledru91ce36c2012-09-27 10:14:43 +00001084 // iff target supports vfp2.
Wesley Peck527da1b2010-11-23 03:31:01 +00001085 setOperationAction(ISD::BITCAST, MVT::i64, Custom);
Nate Begemanb69b1822010-08-03 21:31:55 +00001086 setOperationAction(ISD::FLT_ROUNDS_, MVT::i32, Custom);
1087 }
Lauro Ramos Venanciof6a67bf2007-11-08 17:20:05 +00001088
1089 // We want to custom lower some of our intrinsics.
Owen Anderson9f944592009-08-11 20:47:22 +00001090 setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::Other, Custom);
Matthias Braun3cd00c12015-07-16 22:34:16 +00001091 setOperationAction(ISD::EH_SJLJ_SETJMP, MVT::i32, Custom);
1092 setOperationAction(ISD::EH_SJLJ_LONGJMP, MVT::Other, Custom);
1093 setOperationAction(ISD::EH_SJLJ_SETUP_DISPATCH, MVT::Other, Custom);
Tim Northoverf8e47e42015-10-28 22:56:36 +00001094 if (Subtarget->useSjLjEH())
John McCall7d84ece2011-05-29 19:50:32 +00001095 setLibcallName(RTLIB::UNWIND_RESUME, "_Unwind_SjLj_Resume");
Lauro Ramos Venanciof6a67bf2007-11-08 17:20:05 +00001096
Owen Anderson9f944592009-08-11 20:47:22 +00001097 setOperationAction(ISD::SETCC, MVT::i32, Expand);
1098 setOperationAction(ISD::SETCC, MVT::f32, Expand);
1099 setOperationAction(ISD::SETCC, MVT::f64, Expand);
Bill Wendling6a981312010-08-11 08:43:16 +00001100 setOperationAction(ISD::SELECT, MVT::i32, Custom);
1101 setOperationAction(ISD::SELECT, MVT::f32, Custom);
1102 setOperationAction(ISD::SELECT, MVT::f64, Custom);
Owen Anderson9f944592009-08-11 20:47:22 +00001103 setOperationAction(ISD::SELECT_CC, MVT::i32, Custom);
1104 setOperationAction(ISD::SELECT_CC, MVT::f32, Custom);
1105 setOperationAction(ISD::SELECT_CC, MVT::f64, Custom);
Evan Cheng10043e22007-01-19 07:51:42 +00001106
Peter Collingbourne86b9fbe2016-03-21 18:00:02 +00001107 // Thumb-1 cannot currently select ARMISD::SUBE.
1108 if (!Subtarget->isThumb1Only())
1109 setOperationAction(ISD::SETCCE, MVT::i32, Custom);
1110
Owen Anderson9f944592009-08-11 20:47:22 +00001111 setOperationAction(ISD::BRCOND, MVT::Other, Expand);
1112 setOperationAction(ISD::BR_CC, MVT::i32, Custom);
1113 setOperationAction(ISD::BR_CC, MVT::f32, Custom);
1114 setOperationAction(ISD::BR_CC, MVT::f64, Custom);
1115 setOperationAction(ISD::BR_JT, MVT::Other, Custom);
Evan Cheng10043e22007-01-19 07:51:42 +00001116
Dan Gohman482732a2007-10-11 23:21:31 +00001117 // We don't support sin/cos/fmod/copysign/pow
Owen Anderson9f944592009-08-11 20:47:22 +00001118 setOperationAction(ISD::FSIN, MVT::f64, Expand);
1119 setOperationAction(ISD::FSIN, MVT::f32, Expand);
1120 setOperationAction(ISD::FCOS, MVT::f32, Expand);
1121 setOperationAction(ISD::FCOS, MVT::f64, Expand);
Evan Cheng0e88c7d2013-01-29 02:32:37 +00001122 setOperationAction(ISD::FSINCOS, MVT::f64, Expand);
1123 setOperationAction(ISD::FSINCOS, MVT::f32, Expand);
Owen Anderson9f944592009-08-11 20:47:22 +00001124 setOperationAction(ISD::FREM, MVT::f64, Expand);
1125 setOperationAction(ISD::FREM, MVT::f32, Expand);
Eric Christopher824f42f2015-05-12 01:26:05 +00001126 if (!Subtarget->useSoftFloat() && Subtarget->hasVFP2() &&
Nick Lewycky50f02cb2011-12-02 22:16:29 +00001127 !Subtarget->isThumb1Only()) {
Owen Anderson9f944592009-08-11 20:47:22 +00001128 setOperationAction(ISD::FCOPYSIGN, MVT::f64, Custom);
1129 setOperationAction(ISD::FCOPYSIGN, MVT::f32, Custom);
Evan Cheng86e476b2008-04-01 01:50:16 +00001130 }
Owen Anderson9f944592009-08-11 20:47:22 +00001131 setOperationAction(ISD::FPOW, MVT::f64, Expand);
1132 setOperationAction(ISD::FPOW, MVT::f32, Expand);
Bob Wilson7117a912009-03-20 22:42:55 +00001133
Evan Chengd0007f32012-04-10 21:40:28 +00001134 if (!Subtarget->hasVFP4()) {
1135 setOperationAction(ISD::FMA, MVT::f64, Expand);
1136 setOperationAction(ISD::FMA, MVT::f32, Expand);
1137 }
Cameron Zwarichf03fa182011-07-08 21:39:21 +00001138
Anton Korobeynikovd7fece32010-03-14 18:42:31 +00001139 // Various VFP goodness
Eric Christopher824f42f2015-05-12 01:26:05 +00001140 if (!Subtarget->useSoftFloat() && !Subtarget->isThumb1Only()) {
Oliver Stannardd4e0a4f2014-10-01 13:13:18 +00001141 // FP-ARMv8 adds f64 <-> f16 conversion. Before that it should be expanded.
1142 if (!Subtarget->hasFPARMv8() || Subtarget->isFPOnlySP()) {
Tim Northover53f3bcf2014-07-17 11:27:04 +00001143 setOperationAction(ISD::FP16_TO_FP, MVT::f64, Expand);
1144 setOperationAction(ISD::FP_TO_FP16, MVT::f64, Expand);
1145 }
1146
1147 // fp16 is a special v7 extension that adds f16 <-> f32 conversions.
Anton Korobeynikov64578d52010-03-18 22:35:37 +00001148 if (!Subtarget->hasFP16()) {
Tim Northoverfd7e4242014-07-17 10:51:23 +00001149 setOperationAction(ISD::FP16_TO_FP, MVT::f32, Expand);
1150 setOperationAction(ISD::FP_TO_FP16, MVT::f32, Expand);
Anton Korobeynikovd7fece32010-03-14 18:42:31 +00001151 }
Evan Cheng86e476b2008-04-01 01:50:16 +00001152 }
Jim Grosbach1a597112014-04-03 23:43:18 +00001153
Bob Wilsone7dde0c2013-11-03 06:14:38 +00001154 // Combine sin / cos into one node or libcall if possible.
1155 if (Subtarget->hasSinCos()) {
1156 setLibcallName(RTLIB::SINCOS_F32, "sincosf");
1157 setLibcallName(RTLIB::SINCOS_F64, "sincos");
Tim Northover042a6c12016-01-27 19:32:29 +00001158 if (Subtarget->isTargetWatchABI()) {
Tim Northover8b403662015-10-28 22:51:16 +00001159 setLibcallCallingConv(RTLIB::SINCOS_F32, CallingConv::ARM_AAPCS_VFP);
1160 setLibcallCallingConv(RTLIB::SINCOS_F64, CallingConv::ARM_AAPCS_VFP);
1161 }
1162 if (Subtarget->isTargetIOS() || Subtarget->isTargetWatchOS()) {
Bob Wilsone7dde0c2013-11-03 06:14:38 +00001163 // For iOS, we don't want to the normal expansion of a libcall to
1164 // sincos. We want to issue a libcall to __sincos_stret.
1165 setOperationAction(ISD::FSINCOS, MVT::f64, Custom);
1166 setOperationAction(ISD::FSINCOS, MVT::f32, Custom);
1167 }
1168 }
Evan Cheng10043e22007-01-19 07:51:42 +00001169
Oliver Stannardd4e0a4f2014-10-01 13:13:18 +00001170 // FP-ARMv8 implements a lot of rounding-like FP operations.
1171 if (Subtarget->hasFPARMv8()) {
1172 setOperationAction(ISD::FFLOOR, MVT::f32, Legal);
1173 setOperationAction(ISD::FCEIL, MVT::f32, Legal);
1174 setOperationAction(ISD::FROUND, MVT::f32, Legal);
1175 setOperationAction(ISD::FTRUNC, MVT::f32, Legal);
1176 setOperationAction(ISD::FNEARBYINT, MVT::f32, Legal);
1177 setOperationAction(ISD::FRINT, MVT::f32, Legal);
James Molloyea3a6872015-08-11 12:06:22 +00001178 setOperationAction(ISD::FMINNUM, MVT::f32, Legal);
1179 setOperationAction(ISD::FMAXNUM, MVT::f32, Legal);
James Molloyee868b22015-08-11 12:06:25 +00001180 setOperationAction(ISD::FMINNUM, MVT::v2f32, Legal);
1181 setOperationAction(ISD::FMAXNUM, MVT::v2f32, Legal);
1182 setOperationAction(ISD::FMINNUM, MVT::v4f32, Legal);
1183 setOperationAction(ISD::FMAXNUM, MVT::v4f32, Legal);
1184
Oliver Stannardd4e0a4f2014-10-01 13:13:18 +00001185 if (!Subtarget->isFPOnlySP()) {
1186 setOperationAction(ISD::FFLOOR, MVT::f64, Legal);
1187 setOperationAction(ISD::FCEIL, MVT::f64, Legal);
1188 setOperationAction(ISD::FROUND, MVT::f64, Legal);
1189 setOperationAction(ISD::FTRUNC, MVT::f64, Legal);
1190 setOperationAction(ISD::FNEARBYINT, MVT::f64, Legal);
1191 setOperationAction(ISD::FRINT, MVT::f64, Legal);
James Molloyea3a6872015-08-11 12:06:22 +00001192 setOperationAction(ISD::FMINNUM, MVT::f64, Legal);
1193 setOperationAction(ISD::FMAXNUM, MVT::f64, Legal);
Chad Rosierb1bbf6f2014-08-15 21:38:16 +00001194 }
1195 }
James Molloydb8ee4b2015-08-11 12:06:15 +00001196
James Molloy974838f2015-08-17 19:37:12 +00001197 if (Subtarget->hasNEON()) {
1198 // vmin and vmax aren't available in a scalar form, so we use
1199 // a NEON instruction with an undef lane instead.
James Molloydb8ee4b2015-08-11 12:06:15 +00001200 setOperationAction(ISD::FMINNAN, MVT::f32, Legal);
1201 setOperationAction(ISD::FMAXNAN, MVT::f32, Legal);
James Molloyd616c642015-08-11 12:06:28 +00001202 setOperationAction(ISD::FMINNAN, MVT::v2f32, Legal);
1203 setOperationAction(ISD::FMAXNAN, MVT::v2f32, Legal);
1204 setOperationAction(ISD::FMINNAN, MVT::v4f32, Legal);
1205 setOperationAction(ISD::FMAXNAN, MVT::v4f32, Legal);
1206 }
James Molloydb8ee4b2015-08-11 12:06:15 +00001207
Chris Lattnerf3f4ad92007-11-27 22:36:16 +00001208 // We have target-specific dag combine patterns for the following nodes:
Jim Grosbachd7cf55c2009-11-09 00:11:35 +00001209 // ARMISD::VMOVRRD - No need to call setTargetDAGCombine
Chris Lattner4147f082009-03-12 06:52:53 +00001210 setTargetDAGCombine(ISD::ADD);
1211 setTargetDAGCombine(ISD::SUB);
Anton Korobeynikov1bf28a12010-05-15 18:16:59 +00001212 setTargetDAGCombine(ISD::MUL);
Jakob Stoklund Olesene45e22b2012-09-07 17:34:15 +00001213 setTargetDAGCombine(ISD::AND);
1214 setTargetDAGCombine(ISD::OR);
1215 setTargetDAGCombine(ISD::XOR);
Jim Grosbach11013ed2010-07-16 23:05:05 +00001216
Evan Chengf258a152012-02-23 02:58:19 +00001217 if (Subtarget->hasV6Ops())
1218 setTargetDAGCombine(ISD::SRL);
1219
Evan Cheng10043e22007-01-19 07:51:42 +00001220 setStackPointerRegisterToSaveRestore(ARM::SP);
Evan Cheng4401f882010-05-20 23:26:43 +00001221
Eric Christopher824f42f2015-05-12 01:26:05 +00001222 if (Subtarget->useSoftFloat() || Subtarget->isThumb1Only() ||
Nick Lewycky50f02cb2011-12-02 22:16:29 +00001223 !Subtarget->hasVFP2())
Evan Cheng34c26042010-05-21 00:43:17 +00001224 setSchedulingPreference(Sched::RegPressure);
1225 else
1226 setSchedulingPreference(Sched::Hybrid);
Dale Johannesen58698d22007-05-17 21:31:21 +00001227
Evan Cheng3ae2b792011-01-06 06:52:41 +00001228 //// temporary - rewrite interface to use type
Jim Grosbach341ad3e2013-02-20 21:13:59 +00001229 MaxStoresPerMemset = 8;
Sanjay Patel1166f2f2015-07-30 21:41:50 +00001230 MaxStoresPerMemsetOptSize = 4;
Jim Grosbach341ad3e2013-02-20 21:13:59 +00001231 MaxStoresPerMemcpy = 4; // For @llvm.memcpy -> sequence of stores
Sanjay Patel1166f2f2015-07-30 21:41:50 +00001232 MaxStoresPerMemcpyOptSize = 2;
Jim Grosbach341ad3e2013-02-20 21:13:59 +00001233 MaxStoresPerMemmove = 4; // For @llvm.memmove -> sequence of stores
Sanjay Patel1166f2f2015-07-30 21:41:50 +00001234 MaxStoresPerMemmoveOptSize = 2;
Evan Chengb71233f2010-06-26 01:52:05 +00001235
Rafael Espindolaa76eccf2010-07-11 04:01:49 +00001236 // On ARM arguments smaller than 4 bytes are extended, so all arguments
1237 // are at least 4 bytes aligned.
1238 setMinStackArgumentAlignment(4);
1239
Benjamin Kramere31f31e2012-05-05 12:49:14 +00001240 // Prefer likely predicted branches to selects on out-of-order cores.
Junmo Park453f4aa2016-02-23 09:56:58 +00001241 PredictableSelectIsExpensive = Subtarget->getSchedModel().isOutOfOrder();
Benjamin Kramere31f31e2012-05-05 12:49:14 +00001242
Eli Friedman2518f832011-05-06 20:34:06 +00001243 setMinFunctionAlignment(Subtarget->isThumb() ? 1 : 2);
Evan Cheng10043e22007-01-19 07:51:42 +00001244}
1245
Eric Christopher824f42f2015-05-12 01:26:05 +00001246bool ARMTargetLowering::useSoftFloat() const {
1247 return Subtarget->useSoftFloat();
1248}
1249
Andrew Trick43f25632011-01-19 02:35:27 +00001250// FIXME: It might make sense to define the representative register class as the
1251// nearest super-register that has a non-null superset. For example, DPR_VFP2 is
1252// a super-register of SPR, and DPR is a superset if DPR_VFP2. Consequently,
1253// SPR's representative would be DPR_VFP2. This should work well if register
1254// pressure tracking were modified such that a register use would increment the
1255// pressure of the register class's representative and all of it's super
1256// classes' representatives transitively. We have not implemented this because
1257// of the difficulty prior to coalescing of modeling operand register classes
Chris Lattner0ab5e2c2011-04-15 05:18:47 +00001258// due to the common occurrence of cross class copies and subregister insertions
Andrew Trick43f25632011-01-19 02:35:27 +00001259// and extractions.
Eric Christopher23a3a7c2015-02-26 00:00:24 +00001260std::pair<const TargetRegisterClass *, uint8_t>
1261ARMTargetLowering::findRepresentativeClass(const TargetRegisterInfo *TRI,
1262 MVT VT) const {
Craig Topper062a2ba2014-04-25 05:30:21 +00001263 const TargetRegisterClass *RRC = nullptr;
Evan Chenga77f3d32010-07-21 06:09:07 +00001264 uint8_t Cost = 1;
Patrik Hagglundf9eb1682012-12-19 11:30:36 +00001265 switch (VT.SimpleTy) {
Evan Cheng10f99a32010-07-19 22:15:08 +00001266 default:
Eric Christopher23a3a7c2015-02-26 00:00:24 +00001267 return TargetLowering::findRepresentativeClass(TRI, VT);
Evan Cheng28590382010-07-21 23:53:58 +00001268 // Use DPR as representative register class for all floating point
1269 // and vector types. Since there are 32 SPR registers and 32 DPR registers so
1270 // the cost is 1 for both f32 and f64.
1271 case MVT::f32: case MVT::f64: case MVT::v8i8: case MVT::v4i16:
Evan Chenga77f3d32010-07-21 06:09:07 +00001272 case MVT::v2i32: case MVT::v1i64: case MVT::v2f32:
Craig Topperc7242e02012-04-20 07:30:17 +00001273 RRC = &ARM::DPRRegClass;
Andrew Trick43f25632011-01-19 02:35:27 +00001274 // When NEON is used for SP, only half of the register file is available
1275 // because operations that define both SP and DP results will be constrained
1276 // to the VFP2 class (D0-D15). We currently model this constraint prior to
1277 // coalescing by double-counting the SP regs. See the FIXME above.
1278 if (Subtarget->useNEONForSinglePrecisionFP())
1279 Cost = 2;
Evan Chenga77f3d32010-07-21 06:09:07 +00001280 break;
1281 case MVT::v16i8: case MVT::v8i16: case MVT::v4i32: case MVT::v2i64:
1282 case MVT::v4f32: case MVT::v2f64:
Craig Topperc7242e02012-04-20 07:30:17 +00001283 RRC = &ARM::DPRRegClass;
Evan Cheng28590382010-07-21 23:53:58 +00001284 Cost = 2;
Evan Chenga77f3d32010-07-21 06:09:07 +00001285 break;
1286 case MVT::v4i64:
Craig Topperc7242e02012-04-20 07:30:17 +00001287 RRC = &ARM::DPRRegClass;
Evan Cheng28590382010-07-21 23:53:58 +00001288 Cost = 4;
Evan Chenga77f3d32010-07-21 06:09:07 +00001289 break;
1290 case MVT::v8i64:
Craig Topperc7242e02012-04-20 07:30:17 +00001291 RRC = &ARM::DPRRegClass;
Evan Cheng28590382010-07-21 23:53:58 +00001292 Cost = 8;
Evan Chenga77f3d32010-07-21 06:09:07 +00001293 break;
Evan Cheng10f99a32010-07-19 22:15:08 +00001294 }
Evan Chenga77f3d32010-07-21 06:09:07 +00001295 return std::make_pair(RRC, Cost);
Evan Cheng10f99a32010-07-19 22:15:08 +00001296}
1297
Evan Cheng10043e22007-01-19 07:51:42 +00001298const char *ARMTargetLowering::getTargetNodeName(unsigned Opcode) const {
Matthias Braund04893f2015-05-07 21:33:59 +00001299 switch ((ARMISD::NodeType)Opcode) {
1300 case ARMISD::FIRST_NUMBER: break;
Evan Cheng10043e22007-01-19 07:51:42 +00001301 case ARMISD::Wrapper: return "ARMISD::Wrapper";
Evan Chengdfce83c2011-01-17 08:03:18 +00001302 case ARMISD::WrapperPIC: return "ARMISD::WrapperPIC";
Evan Cheng10043e22007-01-19 07:51:42 +00001303 case ARMISD::WrapperJT: return "ARMISD::WrapperJT";
Matthias Braunf45afee2015-05-07 22:16:10 +00001304 case ARMISD::COPY_STRUCT_BYVAL: return "ARMISD::COPY_STRUCT_BYVAL";
Evan Cheng10043e22007-01-19 07:51:42 +00001305 case ARMISD::CALL: return "ARMISD::CALL";
Evan Chengc3c949b42007-06-19 21:05:09 +00001306 case ARMISD::CALL_PRED: return "ARMISD::CALL_PRED";
Evan Cheng10043e22007-01-19 07:51:42 +00001307 case ARMISD::CALL_NOLINK: return "ARMISD::CALL_NOLINK";
Evan Cheng10043e22007-01-19 07:51:42 +00001308 case ARMISD::BRCOND: return "ARMISD::BRCOND";
1309 case ARMISD::BR_JT: return "ARMISD::BR_JT";
Evan Chengc6d70ae2009-07-29 02:18:14 +00001310 case ARMISD::BR2_JT: return "ARMISD::BR2_JT";
Evan Cheng10043e22007-01-19 07:51:42 +00001311 case ARMISD::RET_FLAG: return "ARMISD::RET_FLAG";
Tim Northoverd8407452013-10-01 14:33:28 +00001312 case ARMISD::INTRET_FLAG: return "ARMISD::INTRET_FLAG";
Evan Cheng10043e22007-01-19 07:51:42 +00001313 case ARMISD::PIC_ADD: return "ARMISD::PIC_ADD";
1314 case ARMISD::CMP: return "ARMISD::CMP";
Bill Wendling4b796472012-06-11 08:07:26 +00001315 case ARMISD::CMN: return "ARMISD::CMN";
David Goodwindbf11ba2009-06-29 15:33:01 +00001316 case ARMISD::CMPZ: return "ARMISD::CMPZ";
Evan Cheng10043e22007-01-19 07:51:42 +00001317 case ARMISD::CMPFP: return "ARMISD::CMPFP";
1318 case ARMISD::CMPFPw0: return "ARMISD::CMPFPw0";
Evan Cheng0cc4ad92010-07-13 19:27:42 +00001319 case ARMISD::BCC_i64: return "ARMISD::BCC_i64";
Evan Cheng10043e22007-01-19 07:51:42 +00001320 case ARMISD::FMSTAT: return "ARMISD::FMSTAT";
Evan Chenge87681c2012-02-23 01:19:06 +00001321
Evan Cheng10043e22007-01-19 07:51:42 +00001322 case ARMISD::CMOV: return "ARMISD::CMOV";
Bob Wilson7117a912009-03-20 22:42:55 +00001323
Pablo Barrio7a643462016-06-23 16:53:49 +00001324 case ARMISD::SSAT: return "ARMISD::SSAT";
1325
Evan Cheng10043e22007-01-19 07:51:42 +00001326 case ARMISD::SRL_FLAG: return "ARMISD::SRL_FLAG";
1327 case ARMISD::SRA_FLAG: return "ARMISD::SRA_FLAG";
1328 case ARMISD::RRX: return "ARMISD::RRX";
Bob Wilson7117a912009-03-20 22:42:55 +00001329
Evan Chenge8916542011-08-30 01:34:54 +00001330 case ARMISD::ADDC: return "ARMISD::ADDC";
1331 case ARMISD::ADDE: return "ARMISD::ADDE";
1332 case ARMISD::SUBC: return "ARMISD::SUBC";
1333 case ARMISD::SUBE: return "ARMISD::SUBE";
1334
Bob Wilson22806742010-09-22 22:09:21 +00001335 case ARMISD::VMOVRRD: return "ARMISD::VMOVRRD";
1336 case ARMISD::VMOVDRR: return "ARMISD::VMOVDRR";
Lauro Ramos Venancioc39c12a2007-04-27 13:54:47 +00001337
Evan Chengec6d7c92009-10-28 06:55:03 +00001338 case ARMISD::EH_SJLJ_SETJMP: return "ARMISD::EH_SJLJ_SETJMP";
Matthias Braun3cd00c12015-07-16 22:34:16 +00001339 case ARMISD::EH_SJLJ_LONGJMP: return "ARMISD::EH_SJLJ_LONGJMP";
1340 case ARMISD::EH_SJLJ_SETUP_DISPATCH: return "ARMISD::EH_SJLJ_SETUP_DISPATCH";
Evan Chengec6d7c92009-10-28 06:55:03 +00001341
Dale Johannesend679ff72010-06-03 21:09:53 +00001342 case ARMISD::TC_RETURN: return "ARMISD::TC_RETURN";
Jim Grosbach535d3b42010-09-08 03:54:02 +00001343
Lauro Ramos Venancioc39c12a2007-04-27 13:54:47 +00001344 case ARMISD::THREAD_POINTER:return "ARMISD::THREAD_POINTER";
Bob Wilson2e076c42009-06-22 23:27:02 +00001345
Evan Chengb972e562009-08-07 00:34:42 +00001346 case ARMISD::DYN_ALLOC: return "ARMISD::DYN_ALLOC";
1347
Bob Wilson7ed59712010-10-30 00:54:37 +00001348 case ARMISD::MEMBARRIER_MCR: return "ARMISD::MEMBARRIER_MCR";
Jim Grosbach53e88542009-12-10 00:11:09 +00001349
Evan Cheng8740ee32010-11-03 06:34:55 +00001350 case ARMISD::PRELOAD: return "ARMISD::PRELOAD";
1351
Saleem Abdulrasoolabac6e92014-06-09 20:18:42 +00001352 case ARMISD::WIN__CHKSTK: return "ARMISD:::WIN__CHKSTK";
Saleem Abdulrasoolfe83b502015-09-25 05:15:46 +00001353 case ARMISD::WIN__DBZCHK: return "ARMISD::WIN__DBZCHK";
Saleem Abdulrasoolabac6e92014-06-09 20:18:42 +00001354
Bob Wilson2e076c42009-06-22 23:27:02 +00001355 case ARMISD::VCEQ: return "ARMISD::VCEQ";
Bob Wilsonf268d032010-12-18 00:04:26 +00001356 case ARMISD::VCEQZ: return "ARMISD::VCEQZ";
Bob Wilson2e076c42009-06-22 23:27:02 +00001357 case ARMISD::VCGE: return "ARMISD::VCGE";
Bob Wilsonf268d032010-12-18 00:04:26 +00001358 case ARMISD::VCGEZ: return "ARMISD::VCGEZ";
1359 case ARMISD::VCLEZ: return "ARMISD::VCLEZ";
Bob Wilson2e076c42009-06-22 23:27:02 +00001360 case ARMISD::VCGEU: return "ARMISD::VCGEU";
1361 case ARMISD::VCGT: return "ARMISD::VCGT";
Bob Wilsonf268d032010-12-18 00:04:26 +00001362 case ARMISD::VCGTZ: return "ARMISD::VCGTZ";
1363 case ARMISD::VCLTZ: return "ARMISD::VCLTZ";
Bob Wilson2e076c42009-06-22 23:27:02 +00001364 case ARMISD::VCGTU: return "ARMISD::VCGTU";
1365 case ARMISD::VTST: return "ARMISD::VTST";
1366
1367 case ARMISD::VSHL: return "ARMISD::VSHL";
1368 case ARMISD::VSHRs: return "ARMISD::VSHRs";
1369 case ARMISD::VSHRu: return "ARMISD::VSHRu";
Bob Wilson2e076c42009-06-22 23:27:02 +00001370 case ARMISD::VRSHRs: return "ARMISD::VRSHRs";
1371 case ARMISD::VRSHRu: return "ARMISD::VRSHRu";
1372 case ARMISD::VRSHRN: return "ARMISD::VRSHRN";
1373 case ARMISD::VQSHLs: return "ARMISD::VQSHLs";
1374 case ARMISD::VQSHLu: return "ARMISD::VQSHLu";
1375 case ARMISD::VQSHLsu: return "ARMISD::VQSHLsu";
1376 case ARMISD::VQSHRNs: return "ARMISD::VQSHRNs";
1377 case ARMISD::VQSHRNu: return "ARMISD::VQSHRNu";
1378 case ARMISD::VQSHRNsu: return "ARMISD::VQSHRNsu";
1379 case ARMISD::VQRSHRNs: return "ARMISD::VQRSHRNs";
1380 case ARMISD::VQRSHRNu: return "ARMISD::VQRSHRNu";
1381 case ARMISD::VQRSHRNsu: return "ARMISD::VQRSHRNsu";
Matthias Braund04893f2015-05-07 21:33:59 +00001382 case ARMISD::VSLI: return "ARMISD::VSLI";
1383 case ARMISD::VSRI: return "ARMISD::VSRI";
Bob Wilson2e076c42009-06-22 23:27:02 +00001384 case ARMISD::VGETLANEu: return "ARMISD::VGETLANEu";
1385 case ARMISD::VGETLANEs: return "ARMISD::VGETLANEs";
Bob Wilsona3f19012010-07-13 21:16:48 +00001386 case ARMISD::VMOVIMM: return "ARMISD::VMOVIMM";
Bob Wilsonbad47f62010-07-14 06:31:50 +00001387 case ARMISD::VMVNIMM: return "ARMISD::VMVNIMM";
Evan Cheng7ca4b6e2011-11-15 02:12:34 +00001388 case ARMISD::VMOVFPIMM: return "ARMISD::VMOVFPIMM";
Bob Wilsoneb54d512009-08-14 05:13:08 +00001389 case ARMISD::VDUP: return "ARMISD::VDUP";
Bob Wilsoncce31f62009-08-14 05:08:32 +00001390 case ARMISD::VDUPLANE: return "ARMISD::VDUPLANE";
Bob Wilson32cd8552009-08-19 17:03:43 +00001391 case ARMISD::VEXT: return "ARMISD::VEXT";
Bob Wilsonea3a4022009-08-12 22:31:50 +00001392 case ARMISD::VREV64: return "ARMISD::VREV64";
1393 case ARMISD::VREV32: return "ARMISD::VREV32";
1394 case ARMISD::VREV16: return "ARMISD::VREV16";
Anton Korobeynikov232b19c2009-08-21 12:41:42 +00001395 case ARMISD::VZIP: return "ARMISD::VZIP";
1396 case ARMISD::VUZP: return "ARMISD::VUZP";
1397 case ARMISD::VTRN: return "ARMISD::VTRN";
Bill Wendlinge1fd78f2011-03-14 23:02:38 +00001398 case ARMISD::VTBL1: return "ARMISD::VTBL1";
1399 case ARMISD::VTBL2: return "ARMISD::VTBL2";
Bob Wilson38ab35a2010-09-01 23:50:19 +00001400 case ARMISD::VMULLs: return "ARMISD::VMULLs";
1401 case ARMISD::VMULLu: return "ARMISD::VMULLu";
Sam Parkerd616cf02016-06-20 16:47:09 +00001402 case ARMISD::UMAAL: return "ARMISD::UMAAL";
Arnold Schwaighoferf00fb1c2012-09-04 14:37:49 +00001403 case ARMISD::UMLAL: return "ARMISD::UMLAL";
1404 case ARMISD::SMLAL: return "ARMISD::SMLAL";
Bob Wilsond8a9a042010-06-04 00:04:02 +00001405 case ARMISD::BUILD_VECTOR: return "ARMISD::BUILD_VECTOR";
Jim Grosbach6e3b5fa2010-07-17 01:50:57 +00001406 case ARMISD::BFI: return "ARMISD::BFI";
Bob Wilson62a6f7e2010-11-28 06:51:11 +00001407 case ARMISD::VORRIMM: return "ARMISD::VORRIMM";
1408 case ARMISD::VBICIMM: return "ARMISD::VBICIMM";
Cameron Zwarich53dd03d2011-03-30 23:01:21 +00001409 case ARMISD::VBSL: return "ARMISD::VBSL";
Scott Douglass953f9082015-10-05 14:49:54 +00001410 case ARMISD::MEMCPY: return "ARMISD::MEMCPY";
Bob Wilson2d790df2010-11-28 06:51:26 +00001411 case ARMISD::VLD2DUP: return "ARMISD::VLD2DUP";
1412 case ARMISD::VLD3DUP: return "ARMISD::VLD3DUP";
1413 case ARMISD::VLD4DUP: return "ARMISD::VLD4DUP";
Bob Wilson06fce872011-02-07 17:43:21 +00001414 case ARMISD::VLD1_UPD: return "ARMISD::VLD1_UPD";
1415 case ARMISD::VLD2_UPD: return "ARMISD::VLD2_UPD";
1416 case ARMISD::VLD3_UPD: return "ARMISD::VLD3_UPD";
1417 case ARMISD::VLD4_UPD: return "ARMISD::VLD4_UPD";
1418 case ARMISD::VLD2LN_UPD: return "ARMISD::VLD2LN_UPD";
1419 case ARMISD::VLD3LN_UPD: return "ARMISD::VLD3LN_UPD";
1420 case ARMISD::VLD4LN_UPD: return "ARMISD::VLD4LN_UPD";
1421 case ARMISD::VLD2DUP_UPD: return "ARMISD::VLD2DUP_UPD";
1422 case ARMISD::VLD3DUP_UPD: return "ARMISD::VLD3DUP_UPD";
1423 case ARMISD::VLD4DUP_UPD: return "ARMISD::VLD4DUP_UPD";
1424 case ARMISD::VST1_UPD: return "ARMISD::VST1_UPD";
1425 case ARMISD::VST2_UPD: return "ARMISD::VST2_UPD";
1426 case ARMISD::VST3_UPD: return "ARMISD::VST3_UPD";
1427 case ARMISD::VST4_UPD: return "ARMISD::VST4_UPD";
1428 case ARMISD::VST2LN_UPD: return "ARMISD::VST2LN_UPD";
1429 case ARMISD::VST3LN_UPD: return "ARMISD::VST3LN_UPD";
1430 case ARMISD::VST4LN_UPD: return "ARMISD::VST4LN_UPD";
Evan Cheng10043e22007-01-19 07:51:42 +00001431 }
Matthias Braund04893f2015-05-07 21:33:59 +00001432 return nullptr;
Evan Cheng10043e22007-01-19 07:51:42 +00001433}
1434
Mehdi Amini44ede332015-07-09 02:09:04 +00001435EVT ARMTargetLowering::getSetCCResultType(const DataLayout &DL, LLVMContext &,
1436 EVT VT) const {
1437 if (!VT.isVector())
1438 return getPointerTy(DL);
Duncan Sandsf2641e12011-09-06 19:07:46 +00001439 return VT.changeVectorElementTypeToInteger();
1440}
1441
Evan Cheng4cad68e2010-05-15 02:18:07 +00001442/// getRegClassFor - Return the register class that should be used for the
1443/// specified value type.
Patrik Hagglund5e6c3612012-12-13 06:34:11 +00001444const TargetRegisterClass *ARMTargetLowering::getRegClassFor(MVT VT) const {
Evan Cheng4cad68e2010-05-15 02:18:07 +00001445 // Map v4i64 to QQ registers but do not make the type legal. Similarly map
1446 // v8i64 to QQQQ registers. v4i64 and v8i64 are only used for REG_SEQUENCE to
1447 // load / store 4 to 8 consecutive D registers.
Evan Cheng3d214cd2010-05-15 02:20:21 +00001448 if (Subtarget->hasNEON()) {
1449 if (VT == MVT::v4i64)
Craig Topperc7242e02012-04-20 07:30:17 +00001450 return &ARM::QQPRRegClass;
1451 if (VT == MVT::v8i64)
1452 return &ARM::QQQQPRRegClass;
Evan Cheng3d214cd2010-05-15 02:20:21 +00001453 }
Evan Cheng4cad68e2010-05-15 02:18:07 +00001454 return TargetLowering::getRegClassFor(VT);
1455}
1456
John Brawn0dbcd652015-03-18 12:01:59 +00001457// memcpy, and other memory intrinsics, typically tries to use LDM/STM if the
1458// source/dest is aligned and the copy size is large enough. We therefore want
1459// to align such objects passed to memory intrinsics.
1460bool ARMTargetLowering::shouldAlignPointerArgs(CallInst *CI, unsigned &MinSize,
1461 unsigned &PrefAlign) const {
1462 if (!isa<MemIntrinsic>(CI))
1463 return false;
1464 MinSize = 8;
1465 // On ARM11 onwards (excluding M class) 8-byte aligned LDM is typically 1
1466 // cycle faster than 4-byte aligned LDM.
1467 PrefAlign = (Subtarget->hasV6Ops() && !Subtarget->isMClass() ? 8 : 4);
1468 return true;
1469}
1470
Eric Christopher84bdfd82010-07-21 22:26:11 +00001471// Create a fast isel object.
1472FastISel *
Bob Wilson3e6fa462012-08-03 04:06:28 +00001473ARMTargetLowering::createFastISel(FunctionLoweringInfo &funcInfo,
1474 const TargetLibraryInfo *libInfo) const {
1475 return ARM::createFastISel(funcInfo, libInfo);
Eric Christopher84bdfd82010-07-21 22:26:11 +00001476}
1477
Evan Cheng4401f882010-05-20 23:26:43 +00001478Sched::Preference ARMTargetLowering::getSchedulingPreference(SDNode *N) const {
Evan Chengbf914992010-05-28 23:25:23 +00001479 unsigned NumVals = N->getNumValues();
1480 if (!NumVals)
1481 return Sched::RegPressure;
1482
1483 for (unsigned i = 0; i != NumVals; ++i) {
Evan Cheng4401f882010-05-20 23:26:43 +00001484 EVT VT = N->getValueType(i);
Chris Lattner3e5fbd72010-12-21 02:38:05 +00001485 if (VT == MVT::Glue || VT == MVT::Other)
Evan Cheng0c4c5ca2010-10-29 18:07:31 +00001486 continue;
Evan Cheng4401f882010-05-20 23:26:43 +00001487 if (VT.isFloatingPoint() || VT.isVector())
Dan Gohman4ed1afa2011-10-24 17:55:11 +00001488 return Sched::ILP;
Evan Cheng4401f882010-05-20 23:26:43 +00001489 }
Evan Chengbf914992010-05-28 23:25:23 +00001490
1491 if (!N->isMachineOpcode())
1492 return Sched::RegPressure;
1493
1494 // Load are scheduled for latency even if there instruction itinerary
1495 // is not available.
Eric Christopher1889fdc2015-01-29 00:19:39 +00001496 const TargetInstrInfo *TII = Subtarget->getInstrInfo();
Evan Cheng6cc775f2011-06-28 19:10:37 +00001497 const MCInstrDesc &MCID = TII->get(N->getMachineOpcode());
Evan Cheng0c4c5ca2010-10-29 18:07:31 +00001498
Evan Cheng6cc775f2011-06-28 19:10:37 +00001499 if (MCID.getNumDefs() == 0)
Evan Cheng0c4c5ca2010-10-29 18:07:31 +00001500 return Sched::RegPressure;
1501 if (!Itins->isEmpty() &&
Evan Cheng6cc775f2011-06-28 19:10:37 +00001502 Itins->getOperandCycle(MCID.getSchedClass(), 0) > 2)
Dan Gohman4ed1afa2011-10-24 17:55:11 +00001503 return Sched::ILP;
Evan Chengbf914992010-05-28 23:25:23 +00001504
Evan Cheng4401f882010-05-20 23:26:43 +00001505 return Sched::RegPressure;
1506}
1507
Evan Cheng10043e22007-01-19 07:51:42 +00001508//===----------------------------------------------------------------------===//
1509// Lowering Code
1510//===----------------------------------------------------------------------===//
1511
Evan Cheng10043e22007-01-19 07:51:42 +00001512/// IntCCToARMCC - Convert a DAG integer condition code to an ARM CC
1513static ARMCC::CondCodes IntCCToARMCC(ISD::CondCode CC) {
1514 switch (CC) {
Torok Edwinfbcc6632009-07-14 16:55:14 +00001515 default: llvm_unreachable("Unknown condition code!");
Evan Cheng10043e22007-01-19 07:51:42 +00001516 case ISD::SETNE: return ARMCC::NE;
1517 case ISD::SETEQ: return ARMCC::EQ;
1518 case ISD::SETGT: return ARMCC::GT;
1519 case ISD::SETGE: return ARMCC::GE;
1520 case ISD::SETLT: return ARMCC::LT;
1521 case ISD::SETLE: return ARMCC::LE;
1522 case ISD::SETUGT: return ARMCC::HI;
1523 case ISD::SETUGE: return ARMCC::HS;
1524 case ISD::SETULT: return ARMCC::LO;
1525 case ISD::SETULE: return ARMCC::LS;
1526 }
1527}
1528
Bob Wilsona2e83332009-09-09 23:14:54 +00001529/// FPCCToARMCC - Convert a DAG fp condition code to an ARM CC.
1530static void FPCCToARMCC(ISD::CondCode CC, ARMCC::CondCodes &CondCode,
Evan Cheng10043e22007-01-19 07:51:42 +00001531 ARMCC::CondCodes &CondCode2) {
Evan Cheng10043e22007-01-19 07:51:42 +00001532 CondCode2 = ARMCC::AL;
1533 switch (CC) {
Torok Edwinfbcc6632009-07-14 16:55:14 +00001534 default: llvm_unreachable("Unknown FP condition!");
Evan Cheng10043e22007-01-19 07:51:42 +00001535 case ISD::SETEQ:
1536 case ISD::SETOEQ: CondCode = ARMCC::EQ; break;
1537 case ISD::SETGT:
1538 case ISD::SETOGT: CondCode = ARMCC::GT; break;
1539 case ISD::SETGE:
1540 case ISD::SETOGE: CondCode = ARMCC::GE; break;
1541 case ISD::SETOLT: CondCode = ARMCC::MI; break;
Bob Wilsona2e83332009-09-09 23:14:54 +00001542 case ISD::SETOLE: CondCode = ARMCC::LS; break;
Evan Cheng10043e22007-01-19 07:51:42 +00001543 case ISD::SETONE: CondCode = ARMCC::MI; CondCode2 = ARMCC::GT; break;
1544 case ISD::SETO: CondCode = ARMCC::VC; break;
1545 case ISD::SETUO: CondCode = ARMCC::VS; break;
1546 case ISD::SETUEQ: CondCode = ARMCC::EQ; CondCode2 = ARMCC::VS; break;
1547 case ISD::SETUGT: CondCode = ARMCC::HI; break;
1548 case ISD::SETUGE: CondCode = ARMCC::PL; break;
1549 case ISD::SETLT:
1550 case ISD::SETULT: CondCode = ARMCC::LT; break;
1551 case ISD::SETLE:
1552 case ISD::SETULE: CondCode = ARMCC::LE; break;
1553 case ISD::SETNE:
1554 case ISD::SETUNE: CondCode = ARMCC::NE; break;
1555 }
Evan Cheng10043e22007-01-19 07:51:42 +00001556}
1557
Bob Wilsona4c22902009-04-17 19:07:39 +00001558//===----------------------------------------------------------------------===//
1559// Calling Convention Implementation
Bob Wilsona4c22902009-04-17 19:07:39 +00001560//===----------------------------------------------------------------------===//
1561
1562#include "ARMGenCallingConv.inc"
1563
Oliver Stannardc24f2172014-05-09 14:01:47 +00001564/// getEffectiveCallingConv - Get the effective calling convention, taking into
1565/// account presence of floating point hardware and calling convention
1566/// limitations, such as support for variadic functions.
1567CallingConv::ID
1568ARMTargetLowering::getEffectiveCallingConv(CallingConv::ID CC,
1569 bool isVarArg) const {
Anton Korobeynikova8fd40b2009-06-16 18:50:49 +00001570 switch (CC) {
1571 default:
Anton Korobeynikov22ef7512009-08-05 19:04:42 +00001572 llvm_unreachable("Unsupported calling convention");
Oliver Stannardc24f2172014-05-09 14:01:47 +00001573 case CallingConv::ARM_AAPCS:
1574 case CallingConv::ARM_APCS:
1575 case CallingConv::GHC:
1576 return CC;
Roman Levenstein2792b3f2016-03-10 04:35:09 +00001577 case CallingConv::PreserveMost:
1578 return CallingConv::PreserveMost;
Oliver Stannardc24f2172014-05-09 14:01:47 +00001579 case CallingConv::ARM_AAPCS_VFP:
Manman Ren802cd6f2016-04-05 22:44:44 +00001580 case CallingConv::Swift:
Oliver Stannardc24f2172014-05-09 14:01:47 +00001581 return isVarArg ? CallingConv::ARM_AAPCS : CallingConv::ARM_AAPCS_VFP;
1582 case CallingConv::C:
Evan Cheng08dd8c82010-10-22 18:23:05 +00001583 if (!Subtarget->isAAPCS_ABI())
Oliver Stannardc24f2172014-05-09 14:01:47 +00001584 return CallingConv::ARM_APCS;
Oliver Stannardb5e596f2014-06-13 08:33:03 +00001585 else if (Subtarget->hasVFP2() && !Subtarget->isThumb1Only() &&
Nick Lewycky50f02cb2011-12-02 22:16:29 +00001586 getTargetMachine().Options.FloatABIType == FloatABI::Hard &&
1587 !isVarArg)
Oliver Stannardc24f2172014-05-09 14:01:47 +00001588 return CallingConv::ARM_AAPCS_VFP;
1589 else
1590 return CallingConv::ARM_AAPCS;
1591 case CallingConv::Fast:
Manman Ren16026052016-01-11 23:50:43 +00001592 case CallingConv::CXX_FAST_TLS:
Oliver Stannardc24f2172014-05-09 14:01:47 +00001593 if (!Subtarget->isAAPCS_ABI()) {
Oliver Stannardb5e596f2014-06-13 08:33:03 +00001594 if (Subtarget->hasVFP2() && !Subtarget->isThumb1Only() && !isVarArg)
Oliver Stannardc24f2172014-05-09 14:01:47 +00001595 return CallingConv::Fast;
1596 return CallingConv::ARM_APCS;
Oliver Stannardb5e596f2014-06-13 08:33:03 +00001597 } else if (Subtarget->hasVFP2() && !Subtarget->isThumb1Only() && !isVarArg)
Oliver Stannardc24f2172014-05-09 14:01:47 +00001598 return CallingConv::ARM_AAPCS_VFP;
1599 else
1600 return CallingConv::ARM_AAPCS;
Evan Cheng08dd8c82010-10-22 18:23:05 +00001601 }
Oliver Stannardc24f2172014-05-09 14:01:47 +00001602}
1603
1604/// CCAssignFnForNode - Selects the correct CCAssignFn for the given
1605/// CallingConvention.
1606CCAssignFn *ARMTargetLowering::CCAssignFnForNode(CallingConv::ID CC,
1607 bool Return,
1608 bool isVarArg) const {
1609 switch (getEffectiveCallingConv(CC, isVarArg)) {
1610 default:
1611 llvm_unreachable("Unsupported calling convention");
Anton Korobeynikova8fd40b2009-06-16 18:50:49 +00001612 case CallingConv::ARM_APCS:
Evan Cheng08dd8c82010-10-22 18:23:05 +00001613 return (Return ? RetCC_ARM_APCS : CC_ARM_APCS);
Oliver Stannardc24f2172014-05-09 14:01:47 +00001614 case CallingConv::ARM_AAPCS:
1615 return (Return ? RetCC_ARM_AAPCS : CC_ARM_AAPCS);
1616 case CallingConv::ARM_AAPCS_VFP:
1617 return (Return ? RetCC_ARM_AAPCS_VFP : CC_ARM_AAPCS_VFP);
1618 case CallingConv::Fast:
1619 return (Return ? RetFastCC_ARM_APCS : FastCC_ARM_APCS);
Eric Christopherb3322362012-08-03 00:05:53 +00001620 case CallingConv::GHC:
1621 return (Return ? RetCC_ARM_APCS : CC_ARM_APCS_GHC);
Roman Levenstein2792b3f2016-03-10 04:35:09 +00001622 case CallingConv::PreserveMost:
1623 return (Return ? RetCC_ARM_AAPCS : CC_ARM_AAPCS);
Anton Korobeynikova8fd40b2009-06-16 18:50:49 +00001624 }
1625}
1626
Dan Gohmanf9bbcd12009-08-05 01:29:28 +00001627/// LowerCallResult - Lower the result values of a call into the
1628/// appropriate copies out of appropriate physical registers.
Benjamin Kramerbdc49562016-06-12 15:39:02 +00001629SDValue ARMTargetLowering::LowerCallResult(
1630 SDValue Chain, SDValue InFlag, CallingConv::ID CallConv, bool isVarArg,
1631 const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &dl,
1632 SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals, bool isThisReturn,
1633 SDValue ThisVal) const {
Bob Wilsona4c22902009-04-17 19:07:39 +00001634
Bob Wilsona4c22902009-04-17 19:07:39 +00001635 // Assign locations to each value returned by this call.
1636 SmallVector<CCValAssign, 16> RVLocs;
Eric Christopherb5217502014-08-06 18:45:26 +00001637 ARMCCState CCInfo(CallConv, isVarArg, DAG.getMachineFunction(), RVLocs,
1638 *DAG.getContext(), Call);
Dan Gohmanf9bbcd12009-08-05 01:29:28 +00001639 CCInfo.AnalyzeCallResult(Ins,
Anton Korobeynikov22ef7512009-08-05 19:04:42 +00001640 CCAssignFnForNode(CallConv, /* Return*/ true,
1641 isVarArg));
Bob Wilsona4c22902009-04-17 19:07:39 +00001642
1643 // Copy all of the result registers out of their specified physreg.
1644 for (unsigned i = 0; i != RVLocs.size(); ++i) {
1645 CCValAssign VA = RVLocs[i];
1646
Stephen Linb8bd2322013-04-20 05:14:40 +00001647 // Pass 'this' value directly from the argument to return value, to avoid
1648 // reg unit interference
David Majnemer5d261272016-07-20 04:13:01 +00001649 if (i == 0 && isThisReturn) {
Stephen Lin8118e0b2013-04-23 19:42:25 +00001650 assert(!VA.needsCustom() && VA.getLocVT() == MVT::i32 &&
1651 "unexpected return calling convention register assignment");
Stephen Linb8bd2322013-04-20 05:14:40 +00001652 InVals.push_back(ThisVal);
1653 continue;
1654 }
1655
Bob Wilson0041bd32009-04-25 00:33:20 +00001656 SDValue Val;
Bob Wilsona4c22902009-04-17 19:07:39 +00001657 if (VA.needsCustom()) {
Bob Wilson2e076c42009-06-22 23:27:02 +00001658 // Handle f64 or half of a v2f64.
Owen Anderson9f944592009-08-11 20:47:22 +00001659 SDValue Lo = DAG.getCopyFromReg(Chain, dl, VA.getLocReg(), MVT::i32,
Bob Wilsona4c22902009-04-17 19:07:39 +00001660 InFlag);
Bob Wilsonf134b2d2009-04-24 17:00:36 +00001661 Chain = Lo.getValue(1);
1662 InFlag = Lo.getValue(2);
Bob Wilsona4c22902009-04-17 19:07:39 +00001663 VA = RVLocs[++i]; // skip ahead to next loc
Owen Anderson9f944592009-08-11 20:47:22 +00001664 SDValue Hi = DAG.getCopyFromReg(Chain, dl, VA.getLocReg(), MVT::i32,
Bob Wilsonf134b2d2009-04-24 17:00:36 +00001665 InFlag);
1666 Chain = Hi.getValue(1);
1667 InFlag = Hi.getValue(2);
Christian Pirkerb5728192014-05-08 14:06:24 +00001668 if (!Subtarget->isLittle())
1669 std::swap (Lo, Hi);
Jim Grosbachd7cf55c2009-11-09 00:11:35 +00001670 Val = DAG.getNode(ARMISD::VMOVDRR, dl, MVT::f64, Lo, Hi);
Bob Wilson2e076c42009-06-22 23:27:02 +00001671
Owen Anderson9f944592009-08-11 20:47:22 +00001672 if (VA.getLocVT() == MVT::v2f64) {
1673 SDValue Vec = DAG.getNode(ISD::UNDEF, dl, MVT::v2f64);
1674 Vec = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, MVT::v2f64, Vec, Val,
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +00001675 DAG.getConstant(0, dl, MVT::i32));
Bob Wilson2e076c42009-06-22 23:27:02 +00001676
1677 VA = RVLocs[++i]; // skip ahead to next loc
Owen Anderson9f944592009-08-11 20:47:22 +00001678 Lo = DAG.getCopyFromReg(Chain, dl, VA.getLocReg(), MVT::i32, InFlag);
Bob Wilson2e076c42009-06-22 23:27:02 +00001679 Chain = Lo.getValue(1);
1680 InFlag = Lo.getValue(2);
1681 VA = RVLocs[++i]; // skip ahead to next loc
Owen Anderson9f944592009-08-11 20:47:22 +00001682 Hi = DAG.getCopyFromReg(Chain, dl, VA.getLocReg(), MVT::i32, InFlag);
Bob Wilson2e076c42009-06-22 23:27:02 +00001683 Chain = Hi.getValue(1);
1684 InFlag = Hi.getValue(2);
Christian Pirkerb5728192014-05-08 14:06:24 +00001685 if (!Subtarget->isLittle())
1686 std::swap (Lo, Hi);
Jim Grosbachd7cf55c2009-11-09 00:11:35 +00001687 Val = DAG.getNode(ARMISD::VMOVDRR, dl, MVT::f64, Lo, Hi);
Owen Anderson9f944592009-08-11 20:47:22 +00001688 Val = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, MVT::v2f64, Vec, Val,
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +00001689 DAG.getConstant(1, dl, MVT::i32));
Bob Wilson2e076c42009-06-22 23:27:02 +00001690 }
Bob Wilsona4c22902009-04-17 19:07:39 +00001691 } else {
Bob Wilson0041bd32009-04-25 00:33:20 +00001692 Val = DAG.getCopyFromReg(Chain, dl, VA.getLocReg(), VA.getLocVT(),
1693 InFlag);
Bob Wilsonf134b2d2009-04-24 17:00:36 +00001694 Chain = Val.getValue(1);
1695 InFlag = Val.getValue(2);
Bob Wilsona4c22902009-04-17 19:07:39 +00001696 }
Bob Wilson0041bd32009-04-25 00:33:20 +00001697
1698 switch (VA.getLocInfo()) {
Torok Edwinfbcc6632009-07-14 16:55:14 +00001699 default: llvm_unreachable("Unknown loc info!");
Bob Wilson0041bd32009-04-25 00:33:20 +00001700 case CCValAssign::Full: break;
1701 case CCValAssign::BCvt:
Wesley Peck527da1b2010-11-23 03:31:01 +00001702 Val = DAG.getNode(ISD::BITCAST, dl, VA.getValVT(), Val);
Bob Wilson0041bd32009-04-25 00:33:20 +00001703 break;
1704 }
1705
Dan Gohmanf9bbcd12009-08-05 01:29:28 +00001706 InVals.push_back(Val);
Bob Wilsona4c22902009-04-17 19:07:39 +00001707 }
1708
Dan Gohmanf9bbcd12009-08-05 01:29:28 +00001709 return Chain;
Bob Wilsona4c22902009-04-17 19:07:39 +00001710}
1711
Bob Wilsonea09d4a2009-04-17 20:35:10 +00001712/// LowerMemOpCallTo - Store the argument to the stack.
Benjamin Kramerbdc49562016-06-12 15:39:02 +00001713SDValue ARMTargetLowering::LowerMemOpCallTo(SDValue Chain, SDValue StackPtr,
1714 SDValue Arg, const SDLoc &dl,
1715 SelectionDAG &DAG,
1716 const CCValAssign &VA,
1717 ISD::ArgFlagsTy Flags) const {
Bob Wilsona4c22902009-04-17 19:07:39 +00001718 unsigned LocMemOffset = VA.getLocMemOffset();
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +00001719 SDValue PtrOff = DAG.getIntPtrConstant(LocMemOffset, dl);
Mehdi Amini44ede332015-07-09 02:09:04 +00001720 PtrOff = DAG.getNode(ISD::ADD, dl, getPointerTy(DAG.getDataLayout()),
1721 StackPtr, PtrOff);
Alex Lorenze40c8a22015-08-11 23:09:45 +00001722 return DAG.getStore(
1723 Chain, dl, Arg, PtrOff,
Justin Lebar9c375812016-07-15 18:27:10 +00001724 MachinePointerInfo::getStack(DAG.getMachineFunction(), LocMemOffset));
Evan Cheng10043e22007-01-19 07:51:42 +00001725}
1726
Benjamin Kramerbdc49562016-06-12 15:39:02 +00001727void ARMTargetLowering::PassF64ArgInRegs(const SDLoc &dl, SelectionDAG &DAG,
Bob Wilson2e076c42009-06-22 23:27:02 +00001728 SDValue Chain, SDValue &Arg,
1729 RegsToPassVector &RegsToPass,
1730 CCValAssign &VA, CCValAssign &NextVA,
1731 SDValue &StackPtr,
Craig Topperb94011f2013-07-14 04:42:23 +00001732 SmallVectorImpl<SDValue> &MemOpChains,
Dan Gohman21cea8a2010-04-17 15:26:15 +00001733 ISD::ArgFlagsTy Flags) const {
Bob Wilson2e076c42009-06-22 23:27:02 +00001734
Jim Grosbachd7cf55c2009-11-09 00:11:35 +00001735 SDValue fmrrd = DAG.getNode(ARMISD::VMOVRRD, dl,
Owen Anderson9f944592009-08-11 20:47:22 +00001736 DAG.getVTList(MVT::i32, MVT::i32), Arg);
Christian Pirkerb5728192014-05-08 14:06:24 +00001737 unsigned id = Subtarget->isLittle() ? 0 : 1;
1738 RegsToPass.push_back(std::make_pair(VA.getLocReg(), fmrrd.getValue(id)));
Bob Wilson2e076c42009-06-22 23:27:02 +00001739
1740 if (NextVA.isRegLoc())
Christian Pirkerb5728192014-05-08 14:06:24 +00001741 RegsToPass.push_back(std::make_pair(NextVA.getLocReg(), fmrrd.getValue(1-id)));
Bob Wilson2e076c42009-06-22 23:27:02 +00001742 else {
1743 assert(NextVA.isMemLoc());
Craig Topper062a2ba2014-04-25 05:30:21 +00001744 if (!StackPtr.getNode())
Mehdi Amini44ede332015-07-09 02:09:04 +00001745 StackPtr = DAG.getCopyFromReg(Chain, dl, ARM::SP,
1746 getPointerTy(DAG.getDataLayout()));
Bob Wilson2e076c42009-06-22 23:27:02 +00001747
Christian Pirkerb5728192014-05-08 14:06:24 +00001748 MemOpChains.push_back(LowerMemOpCallTo(Chain, StackPtr, fmrrd.getValue(1-id),
Dan Gohmanf9bbcd12009-08-05 01:29:28 +00001749 dl, DAG, NextVA,
1750 Flags));
Bob Wilson2e076c42009-06-22 23:27:02 +00001751 }
1752}
1753
Dan Gohmanf9bbcd12009-08-05 01:29:28 +00001754/// LowerCall - Lowering a call into a callseq_start <-
Evan Cheng4b6c8f72007-02-03 08:53:01 +00001755/// ARMISD:CALL <- callseq_end chain. Also add input and output parameter
1756/// nodes.
Dan Gohmanf9bbcd12009-08-05 01:29:28 +00001757SDValue
Justin Holewinskiaa583972012-05-25 16:35:28 +00001758ARMTargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
Dan Gohman21cea8a2010-04-17 15:26:15 +00001759 SmallVectorImpl<SDValue> &InVals) const {
Justin Holewinskiaa583972012-05-25 16:35:28 +00001760 SelectionDAG &DAG = CLI.DAG;
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +00001761 SDLoc &dl = CLI.DL;
Craig Topperb94011f2013-07-14 04:42:23 +00001762 SmallVectorImpl<ISD::OutputArg> &Outs = CLI.Outs;
1763 SmallVectorImpl<SDValue> &OutVals = CLI.OutVals;
1764 SmallVectorImpl<ISD::InputArg> &Ins = CLI.Ins;
Justin Holewinskiaa583972012-05-25 16:35:28 +00001765 SDValue Chain = CLI.Chain;
1766 SDValue Callee = CLI.Callee;
1767 bool &isTailCall = CLI.IsTailCall;
1768 CallingConv::ID CallConv = CLI.CallConv;
1769 bool doesNotRet = CLI.DoesNotReturn;
1770 bool isVarArg = CLI.IsVarArg;
1771
Dale Johannesend679ff72010-06-03 21:09:53 +00001772 MachineFunction &MF = DAG.getMachineFunction();
Stephen Lin4eedb292013-04-23 19:30:12 +00001773 bool isStructRet = (Outs.empty()) ? false : Outs[0].Flags.isSRet();
1774 bool isThisReturn = false;
1775 bool isSibCall = false;
Akira Hatanakad9699bc2015-06-09 19:07:19 +00001776 auto Attr = MF.getFunction()->getFnAttribute("disable-tail-calls");
Saleem Abdulrasool0d96f3d2014-03-11 15:09:54 +00001777
Bob Wilson8decdc42011-10-07 17:17:49 +00001778 // Disable tail calls if they're not supported.
Akira Hatanakad9699bc2015-06-09 19:07:19 +00001779 if (!Subtarget->supportsTailCall() || Attr.getValueAsString() == "true")
Bob Wilson3c9ed762010-08-13 22:43:33 +00001780 isTailCall = false;
Saleem Abdulrasool0d96f3d2014-03-11 15:09:54 +00001781
Dale Johannesend679ff72010-06-03 21:09:53 +00001782 if (isTailCall) {
1783 // Check if it's really possible to do a tail call.
1784 isTailCall = IsEligibleForTailCallOptimization(Callee, CallConv,
Stephen Lin4eedb292013-04-23 19:30:12 +00001785 isVarArg, isStructRet, MF.getFunction()->hasStructRetAttr(),
Dan Gohmanfe7532a2010-07-07 15:54:55 +00001786 Outs, OutVals, Ins, DAG);
Reid Kleckner5772b772014-04-24 20:14:34 +00001787 if (!isTailCall && CLI.CS && CLI.CS->isMustTailCall())
1788 report_fatal_error("failed to perform tail call elimination on a call "
1789 "site marked musttail");
Dale Johannesend679ff72010-06-03 21:09:53 +00001790 // We don't support GuaranteedTailCallOpt for ARM, only automatically
1791 // detected sibcalls.
1792 if (isTailCall) {
1793 ++NumTailCalls;
Stephen Lin4eedb292013-04-23 19:30:12 +00001794 isSibCall = true;
Dale Johannesend679ff72010-06-03 21:09:53 +00001795 }
1796 }
Evan Cheng10043e22007-01-19 07:51:42 +00001797
Bob Wilsona4c22902009-04-17 19:07:39 +00001798 // Analyze operands of the call, assigning locations to each operand.
1799 SmallVector<CCValAssign, 16> ArgLocs;
Eric Christopherb5217502014-08-06 18:45:26 +00001800 ARMCCState CCInfo(CallConv, isVarArg, DAG.getMachineFunction(), ArgLocs,
1801 *DAG.getContext(), Call);
Dan Gohmanf9bbcd12009-08-05 01:29:28 +00001802 CCInfo.AnalyzeCallOperands(Outs,
Anton Korobeynikov22ef7512009-08-05 19:04:42 +00001803 CCAssignFnForNode(CallConv, /* Return*/ false,
1804 isVarArg));
Evan Cheng10043e22007-01-19 07:51:42 +00001805
Bob Wilsona4c22902009-04-17 19:07:39 +00001806 // Get a count of how many bytes are to be pushed on the stack.
1807 unsigned NumBytes = CCInfo.getNextStackOffset();
Evan Cheng10043e22007-01-19 07:51:42 +00001808
Dale Johannesend679ff72010-06-03 21:09:53 +00001809 // For tail calls, memory operands are available in our caller's stack.
Stephen Lin4eedb292013-04-23 19:30:12 +00001810 if (isSibCall)
Dale Johannesend679ff72010-06-03 21:09:53 +00001811 NumBytes = 0;
1812
Evan Cheng10043e22007-01-19 07:51:42 +00001813 // Adjust the stack pointer for the new arguments...
1814 // These operations are automatically eliminated by the prolog/epilog pass
Stephen Lin4eedb292013-04-23 19:30:12 +00001815 if (!isSibCall)
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +00001816 Chain = DAG.getCALLSEQ_START(Chain,
1817 DAG.getIntPtrConstant(NumBytes, dl, true), dl);
Evan Cheng10043e22007-01-19 07:51:42 +00001818
Mehdi Amini44ede332015-07-09 02:09:04 +00001819 SDValue StackPtr =
1820 DAG.getCopyFromReg(Chain, dl, ARM::SP, getPointerTy(DAG.getDataLayout()));
Evan Cheng10043e22007-01-19 07:51:42 +00001821
Bob Wilson2e076c42009-06-22 23:27:02 +00001822 RegsToPassVector RegsToPass;
Bob Wilsona4c22902009-04-17 19:07:39 +00001823 SmallVector<SDValue, 8> MemOpChains;
Evan Cheng10043e22007-01-19 07:51:42 +00001824
Bob Wilsona4c22902009-04-17 19:07:39 +00001825 // Walk the register/memloc assignments, inserting copies/loads. In the case
Bob Wilsonea09d4a2009-04-17 20:35:10 +00001826 // of tail call optimization, arguments are handled later.
Bob Wilsona4c22902009-04-17 19:07:39 +00001827 for (unsigned i = 0, realArgIdx = 0, e = ArgLocs.size();
1828 i != e;
1829 ++i, ++realArgIdx) {
1830 CCValAssign &VA = ArgLocs[i];
Dan Gohmanfe7532a2010-07-07 15:54:55 +00001831 SDValue Arg = OutVals[realArgIdx];
Dan Gohmanf9bbcd12009-08-05 01:29:28 +00001832 ISD::ArgFlagsTy Flags = Outs[realArgIdx].Flags;
Stuart Hastings67c5c3e2011-02-28 17:17:53 +00001833 bool isByVal = Flags.isByVal();
Evan Cheng10043e22007-01-19 07:51:42 +00001834
Bob Wilsona4c22902009-04-17 19:07:39 +00001835 // Promote the value if needed.
1836 switch (VA.getLocInfo()) {
Torok Edwinfbcc6632009-07-14 16:55:14 +00001837 default: llvm_unreachable("Unknown loc info!");
Bob Wilsona4c22902009-04-17 19:07:39 +00001838 case CCValAssign::Full: break;
1839 case CCValAssign::SExt:
1840 Arg = DAG.getNode(ISD::SIGN_EXTEND, dl, VA.getLocVT(), Arg);
1841 break;
1842 case CCValAssign::ZExt:
1843 Arg = DAG.getNode(ISD::ZERO_EXTEND, dl, VA.getLocVT(), Arg);
1844 break;
1845 case CCValAssign::AExt:
1846 Arg = DAG.getNode(ISD::ANY_EXTEND, dl, VA.getLocVT(), Arg);
1847 break;
1848 case CCValAssign::BCvt:
Wesley Peck527da1b2010-11-23 03:31:01 +00001849 Arg = DAG.getNode(ISD::BITCAST, dl, VA.getLocVT(), Arg);
Bob Wilsona4c22902009-04-17 19:07:39 +00001850 break;
Evan Cheng10043e22007-01-19 07:51:42 +00001851 }
1852
Anton Korobeynikov22ef7512009-08-05 19:04:42 +00001853 // f64 and v2f64 might be passed in i32 pairs and must be split into pieces
Bob Wilsona4c22902009-04-17 19:07:39 +00001854 if (VA.needsCustom()) {
Owen Anderson9f944592009-08-11 20:47:22 +00001855 if (VA.getLocVT() == MVT::v2f64) {
1856 SDValue Op0 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::f64, Arg,
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +00001857 DAG.getConstant(0, dl, MVT::i32));
Owen Anderson9f944592009-08-11 20:47:22 +00001858 SDValue Op1 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::f64, Arg,
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +00001859 DAG.getConstant(1, dl, MVT::i32));
Bob Wilsona4c22902009-04-17 19:07:39 +00001860
Dan Gohmanf9bbcd12009-08-05 01:29:28 +00001861 PassF64ArgInRegs(dl, DAG, Chain, Op0, RegsToPass,
Bob Wilson2e076c42009-06-22 23:27:02 +00001862 VA, ArgLocs[++i], StackPtr, MemOpChains, Flags);
1863
1864 VA = ArgLocs[++i]; // skip ahead to next loc
1865 if (VA.isRegLoc()) {
Dan Gohmanf9bbcd12009-08-05 01:29:28 +00001866 PassF64ArgInRegs(dl, DAG, Chain, Op1, RegsToPass,
Bob Wilson2e076c42009-06-22 23:27:02 +00001867 VA, ArgLocs[++i], StackPtr, MemOpChains, Flags);
1868 } else {
1869 assert(VA.isMemLoc());
Bob Wilson2e076c42009-06-22 23:27:02 +00001870
Dan Gohmanf9bbcd12009-08-05 01:29:28 +00001871 MemOpChains.push_back(LowerMemOpCallTo(Chain, StackPtr, Op1,
1872 dl, DAG, VA, Flags));
Bob Wilson2e076c42009-06-22 23:27:02 +00001873 }
1874 } else {
Dan Gohmanf9bbcd12009-08-05 01:29:28 +00001875 PassF64ArgInRegs(dl, DAG, Chain, Arg, RegsToPass, VA, ArgLocs[++i],
Bob Wilson2e076c42009-06-22 23:27:02 +00001876 StackPtr, MemOpChains, Flags);
Bob Wilsona4c22902009-04-17 19:07:39 +00001877 }
1878 } else if (VA.isRegLoc()) {
Stephen Lin8118e0b2013-04-23 19:42:25 +00001879 if (realArgIdx == 0 && Flags.isReturned() && Outs[0].VT == MVT::i32) {
1880 assert(VA.getLocVT() == MVT::i32 &&
1881 "unexpected calling convention register assignment");
1882 assert(!Ins.empty() && Ins[0].VT == MVT::i32 &&
Stephen Linb8bd2322013-04-20 05:14:40 +00001883 "unexpected use of 'returned'");
Stephen Lin4eedb292013-04-23 19:30:12 +00001884 isThisReturn = true;
Stephen Linb8bd2322013-04-20 05:14:40 +00001885 }
Bob Wilsona4c22902009-04-17 19:07:39 +00001886 RegsToPass.push_back(std::make_pair(VA.getLocReg(), Arg));
Stuart Hastings45fe3c32011-04-20 16:47:52 +00001887 } else if (isByVal) {
1888 assert(VA.isMemLoc());
1889 unsigned offset = 0;
1890
1891 // True if this byval aggregate will be split between registers
1892 // and memory.
Stepan Dyatkovskiy8c02c982013-05-05 07:48:36 +00001893 unsigned ByValArgsCount = CCInfo.getInRegsParamsCount();
Daniel Sanders8104b752014-11-01 19:32:23 +00001894 unsigned CurByValIdx = CCInfo.getInRegsParamsProcessed();
Stepan Dyatkovskiy8c02c982013-05-05 07:48:36 +00001895
1896 if (CurByValIdx < ByValArgsCount) {
1897
1898 unsigned RegBegin, RegEnd;
1899 CCInfo.getInRegsParamInfo(CurByValIdx, RegBegin, RegEnd);
1900
Mehdi Amini44ede332015-07-09 02:09:04 +00001901 EVT PtrVT =
1902 DAG.getTargetLoweringInfo().getPointerTy(DAG.getDataLayout());
Stuart Hastings45fe3c32011-04-20 16:47:52 +00001903 unsigned int i, j;
Stepan Dyatkovskiy8c02c982013-05-05 07:48:36 +00001904 for (i = 0, j = RegBegin; j < RegEnd; i++, j++) {
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +00001905 SDValue Const = DAG.getConstant(4*i, dl, MVT::i32);
Stuart Hastings45fe3c32011-04-20 16:47:52 +00001906 SDValue AddArg = DAG.getNode(ISD::ADD, dl, PtrVT, Arg, Const);
1907 SDValue Load = DAG.getLoad(PtrVT, dl, Chain, AddArg,
1908 MachinePointerInfo(),
Manman Ren5a787552013-10-07 19:47:53 +00001909 DAG.InferPtrAlignment(AddArg));
Stuart Hastings45fe3c32011-04-20 16:47:52 +00001910 MemOpChains.push_back(Load.getValue(1));
1911 RegsToPass.push_back(std::make_pair(j, Load));
1912 }
Stepan Dyatkovskiy8c02c982013-05-05 07:48:36 +00001913
1914 // If parameter size outsides register area, "offset" value
1915 // helps us to calculate stack slot for remained part properly.
1916 offset = RegEnd - RegBegin;
1917
1918 CCInfo.nextInRegsParam();
Stuart Hastings45fe3c32011-04-20 16:47:52 +00001919 }
1920
Stepan Dyatkovskiy8c02c982013-05-05 07:48:36 +00001921 if (Flags.getByValSize() > 4*offset) {
Mehdi Amini44ede332015-07-09 02:09:04 +00001922 auto PtrVT = getPointerTy(DAG.getDataLayout());
Manman Ren9f911162012-06-01 02:44:42 +00001923 unsigned LocMemOffset = VA.getLocMemOffset();
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +00001924 SDValue StkPtrOff = DAG.getIntPtrConstant(LocMemOffset, dl);
Mehdi Amini44ede332015-07-09 02:09:04 +00001925 SDValue Dst = DAG.getNode(ISD::ADD, dl, PtrVT, StackPtr, StkPtrOff);
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +00001926 SDValue SrcOffset = DAG.getIntPtrConstant(4*offset, dl);
Mehdi Amini44ede332015-07-09 02:09:04 +00001927 SDValue Src = DAG.getNode(ISD::ADD, dl, PtrVT, Arg, SrcOffset);
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +00001928 SDValue SizeNode = DAG.getConstant(Flags.getByValSize() - 4*offset, dl,
Manman Ren9f911162012-06-01 02:44:42 +00001929 MVT::i32);
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +00001930 SDValue AlignNode = DAG.getConstant(Flags.getByValAlign(), dl,
1931 MVT::i32);
Stuart Hastings45fe3c32011-04-20 16:47:52 +00001932
Manman Ren9f911162012-06-01 02:44:42 +00001933 SDVTList VTs = DAG.getVTList(MVT::Other, MVT::Glue);
Manman Rene8735522012-06-01 19:33:18 +00001934 SDValue Ops[] = { Chain, Dst, Src, SizeNode, AlignNode};
Manman Ren9f911162012-06-01 02:44:42 +00001935 MemOpChains.push_back(DAG.getNode(ARMISD::COPY_STRUCT_BYVAL, dl, VTs,
Craig Topper48d114b2014-04-26 18:35:24 +00001936 Ops));
Manman Ren9f911162012-06-01 02:44:42 +00001937 }
Stephen Lin4eedb292013-04-23 19:30:12 +00001938 } else if (!isSibCall) {
Bob Wilsona4c22902009-04-17 19:07:39 +00001939 assert(VA.isMemLoc());
Bob Wilsona4c22902009-04-17 19:07:39 +00001940
Dan Gohmanf9bbcd12009-08-05 01:29:28 +00001941 MemOpChains.push_back(LowerMemOpCallTo(Chain, StackPtr, Arg,
1942 dl, DAG, VA, Flags));
Bob Wilsona4c22902009-04-17 19:07:39 +00001943 }
Evan Cheng10043e22007-01-19 07:51:42 +00001944 }
1945
1946 if (!MemOpChains.empty())
Craig Topper48d114b2014-04-26 18:35:24 +00001947 Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, MemOpChains);
Evan Cheng10043e22007-01-19 07:51:42 +00001948
1949 // Build a sequence of copy-to-reg nodes chained together with token chain
1950 // and flag operands which copy the outgoing args into the appropriate regs.
Dan Gohman2ce6f2a2008-07-27 21:46:04 +00001951 SDValue InFlag;
Dale Johannesen44f9dfc2010-06-15 22:08:33 +00001952 // Tail call byval lowering might overwrite argument registers so in case of
1953 // tail call optimization the copies to registers are lowered later.
1954 if (!isTailCall)
1955 for (unsigned i = 0, e = RegsToPass.size(); i != e; ++i) {
1956 Chain = DAG.getCopyToReg(Chain, dl, RegsToPass[i].first,
1957 RegsToPass[i].second, InFlag);
1958 InFlag = Chain.getValue(1);
1959 }
Evan Cheng10043e22007-01-19 07:51:42 +00001960
Dale Johannesend679ff72010-06-03 21:09:53 +00001961 // For tail calls lower the arguments to the 'real' stack slot.
1962 if (isTailCall) {
1963 // Force all the incoming stack arguments to be loaded from the stack
1964 // before any new outgoing arguments are stored to the stack, because the
1965 // outgoing stack slots may alias the incoming argument stack slots, and
1966 // the alias isn't otherwise explicit. This is slightly more conservative
1967 // than necessary, because it means that each store effectively depends
1968 // on every argument instead of just those arguments it would clobber.
1969
Chris Lattner0ab5e2c2011-04-15 05:18:47 +00001970 // Do not flag preceding copytoreg stuff together with the following stuff.
Dale Johannesend679ff72010-06-03 21:09:53 +00001971 InFlag = SDValue();
1972 for (unsigned i = 0, e = RegsToPass.size(); i != e; ++i) {
1973 Chain = DAG.getCopyToReg(Chain, dl, RegsToPass[i].first,
1974 RegsToPass[i].second, InFlag);
1975 InFlag = Chain.getValue(1);
1976 }
Stephen Lind36fd2c2013-04-20 00:47:48 +00001977 InFlag = SDValue();
Dale Johannesend679ff72010-06-03 21:09:53 +00001978 }
1979
Bill Wendling24c79f22008-09-16 21:48:12 +00001980 // If the callee is a GlobalAddress/ExternalSymbol node (quite common, every
1981 // direct call is) turn it into a TargetGlobalAddress/TargetExternalSymbol
1982 // node so that legalize doesn't hack it.
Evan Cheng10043e22007-01-19 07:51:42 +00001983 bool isDirect = false;
Rafael Espindola3888bdb2016-06-16 15:22:01 +00001984
1985 const TargetMachine &TM = getTargetMachine();
Rafael Espindolac1d739f2016-06-16 15:40:24 +00001986 const Module *Mod = MF.getFunction()->getParent();
Rafael Espindolac24f0ee2016-06-16 15:31:06 +00001987 const GlobalValue *GV = nullptr;
1988 if (GlobalAddressSDNode *G = dyn_cast<GlobalAddressSDNode>(Callee))
1989 GV = G->getGlobal();
Rafael Espindolac1d739f2016-06-16 15:40:24 +00001990 bool isStub =
Rafael Espindola3beef8d2016-06-27 23:15:57 +00001991 !TM.shouldAssumeDSOLocal(*Mod, GV) && Subtarget->isTargetMachO();
Rafael Espindola3888bdb2016-06-16 15:22:01 +00001992
Rafael Espindola9ba9c5b2016-06-16 15:44:06 +00001993 bool isARMFunc = !Subtarget->isThumb() || (isStub && !Subtarget->isMClass());
Evan Chengc3c949b42007-06-19 21:05:09 +00001994 bool isLocalARMFunc = false;
Evan Cheng408aa562009-11-06 22:24:13 +00001995 ARMFunctionInfo *AFI = MF.getInfo<ARMFunctionInfo>();
Mehdi Amini44ede332015-07-09 02:09:04 +00001996 auto PtrVt = getPointerTy(DAG.getDataLayout());
Jim Grosbach32bb3622010-04-14 22:28:31 +00001997
Akira Hatanaka1bc8af72015-07-07 06:54:42 +00001998 if (Subtarget->genLongCalls()) {
Saleem Abdulrasool4d950ef2016-07-05 18:30:52 +00001999 assert((!isPositionIndependent() || Subtarget->isTargetWindows()) &&
Rafael Espindola0f898332016-06-20 16:43:17 +00002000 "long-calls codegen is not position independent!");
Jim Grosbach32bb3622010-04-14 22:28:31 +00002001 // Handle a global address or an external symbol. If it's not one of
2002 // those, the target's already in a register, so we don't need to do
2003 // anything extra.
Rafael Espindolac24f0ee2016-06-16 15:31:06 +00002004 if (isa<GlobalAddressSDNode>(Callee)) {
Jim Grosbach32bb3622010-04-14 22:28:31 +00002005 // Create a constant pool entry for the callee address
Evan Chengdfce83c2011-01-17 08:03:18 +00002006 unsigned ARMPCLabelIndex = AFI->createPICLabelUId();
Bill Wendling7753d662011-10-01 08:00:54 +00002007 ARMConstantPoolValue *CPV =
2008 ARMConstantPoolConstant::Create(GV, ARMPCLabelIndex, ARMCP::CPValue, 0);
2009
Jim Grosbach32bb3622010-04-14 22:28:31 +00002010 // Get the address of the callee into a register
Mehdi Amini44ede332015-07-09 02:09:04 +00002011 SDValue CPAddr = DAG.getTargetConstantPool(CPV, PtrVt, 4);
Jim Grosbach32bb3622010-04-14 22:28:31 +00002012 CPAddr = DAG.getNode(ARMISD::Wrapper, dl, MVT::i32, CPAddr);
Alex Lorenze40c8a22015-08-11 23:09:45 +00002013 Callee = DAG.getLoad(
2014 PtrVt, dl, DAG.getEntryNode(), CPAddr,
Justin Lebar9c375812016-07-15 18:27:10 +00002015 MachinePointerInfo::getConstantPool(DAG.getMachineFunction()));
Jim Grosbach32bb3622010-04-14 22:28:31 +00002016 } else if (ExternalSymbolSDNode *S=dyn_cast<ExternalSymbolSDNode>(Callee)) {
2017 const char *Sym = S->getSymbol();
2018
2019 // Create a constant pool entry for the callee address
Evan Chengdfce83c2011-01-17 08:03:18 +00002020 unsigned ARMPCLabelIndex = AFI->createPICLabelUId();
Bill Wendlingc214cb02011-10-01 08:58:29 +00002021 ARMConstantPoolValue *CPV =
2022 ARMConstantPoolSymbol::Create(*DAG.getContext(), Sym,
2023 ARMPCLabelIndex, 0);
Jim Grosbach32bb3622010-04-14 22:28:31 +00002024 // Get the address of the callee into a register
Mehdi Amini44ede332015-07-09 02:09:04 +00002025 SDValue CPAddr = DAG.getTargetConstantPool(CPV, PtrVt, 4);
Jim Grosbach32bb3622010-04-14 22:28:31 +00002026 CPAddr = DAG.getNode(ARMISD::Wrapper, dl, MVT::i32, CPAddr);
Alex Lorenze40c8a22015-08-11 23:09:45 +00002027 Callee = DAG.getLoad(
2028 PtrVt, dl, DAG.getEntryNode(), CPAddr,
Justin Lebar9c375812016-07-15 18:27:10 +00002029 MachinePointerInfo::getConstantPool(DAG.getMachineFunction()));
Jim Grosbach32bb3622010-04-14 22:28:31 +00002030 }
Rafael Espindolac24f0ee2016-06-16 15:31:06 +00002031 } else if (isa<GlobalAddressSDNode>(Callee)) {
James Molloy2af08fa2016-07-15 07:57:35 +00002032 // If we're optimizing for minimum size and the function is called three or
2033 // more times in this block, we can improve codesize by calling indirectly
James Molloya454a112016-07-15 07:55:21 +00002034 // as BLXr has a 16-bit encoding.
2035 auto *GV = cast<GlobalAddressSDNode>(Callee)->getGlobal();
2036 auto *BB = CLI.CS->getParent();
2037 bool PreferIndirect =
2038 Subtarget->isThumb() && MF.getFunction()->optForMinSize() &&
David Majnemer0da5afe2016-08-12 04:32:29 +00002039 count_if(GV->users(), [&BB](const User *U) {
James Molloya454a112016-07-15 07:55:21 +00002040 return isa<Instruction>(U) && cast<Instruction>(U)->getParent() == BB;
2041 }) > 2;
Rafael Espindola41410cc2016-06-01 21:57:11 +00002042
James Molloya454a112016-07-15 07:55:21 +00002043 if (!PreferIndirect) {
2044 isDirect = true;
2045 bool isDef = GV->isStrongDefinitionForLinker();
2046
2047 // ARM call to a local ARM function is predicable.
2048 isLocalARMFunc = !Subtarget->isThumb() && (isDef || !ARMInterworking);
2049 // tBX takes a register source operand.
2050 if (isStub && Subtarget->isThumb1Only() && !Subtarget->hasV5TOps()) {
2051 assert(Subtarget->isTargetMachO() && "WrapperPIC use on non-MachO?");
2052 Callee = DAG.getNode(
2053 ARMISD::WrapperPIC, dl, PtrVt,
2054 DAG.getTargetGlobalAddress(GV, dl, PtrVt, 0, ARMII::MO_NONLAZY));
Justin Lebaradbf09e2016-09-11 01:38:58 +00002055 Callee = DAG.getLoad(
2056 PtrVt, dl, DAG.getEntryNode(), Callee,
2057 MachinePointerInfo::getGOT(DAG.getMachineFunction()),
2058 /* Alignment = */ 0, MachineMemOperand::MODereferenceable |
2059 MachineMemOperand::MOInvariant);
James Molloya454a112016-07-15 07:55:21 +00002060 } else if (Subtarget->isTargetCOFF()) {
2061 assert(Subtarget->isTargetWindows() &&
2062 "Windows is the only supported COFF target");
2063 unsigned TargetFlags = GV->hasDLLImportStorageClass()
2064 ? ARMII::MO_DLLIMPORT
2065 : ARMII::MO_NO_FLAG;
2066 Callee = DAG.getTargetGlobalAddress(GV, dl, PtrVt, /*Offset=*/0,
2067 TargetFlags);
2068 if (GV->hasDLLImportStorageClass())
2069 Callee =
2070 DAG.getLoad(PtrVt, dl, DAG.getEntryNode(),
2071 DAG.getNode(ARMISD::Wrapper, dl, PtrVt, Callee),
Justin Lebar9c375812016-07-15 18:27:10 +00002072 MachinePointerInfo::getGOT(DAG.getMachineFunction()));
James Molloya454a112016-07-15 07:55:21 +00002073 } else {
2074 Callee = DAG.getTargetGlobalAddress(GV, dl, PtrVt, 0, 0);
2075 }
Jim Grosbach85dcd3d2010-09-22 23:27:36 +00002076 }
Bill Wendling24c79f22008-09-16 21:48:12 +00002077 } else if (ExternalSymbolSDNode *S = dyn_cast<ExternalSymbolSDNode>(Callee)) {
Evan Cheng10043e22007-01-19 07:51:42 +00002078 isDirect = true;
Evan Cheng83f35172007-01-30 20:37:08 +00002079 // tBX takes a register source operand.
2080 const char *Sym = S->getSymbol();
David Goodwin22c2fba2009-07-08 23:10:31 +00002081 if (isARMFunc && Subtarget->isThumb1Only() && !Subtarget->hasV5TOps()) {
Evan Chengdfce83c2011-01-17 08:03:18 +00002082 unsigned ARMPCLabelIndex = AFI->createPICLabelUId();
Bill Wendlingc214cb02011-10-01 08:58:29 +00002083 ARMConstantPoolValue *CPV =
2084 ARMConstantPoolSymbol::Create(*DAG.getContext(), Sym,
2085 ARMPCLabelIndex, 4);
Mehdi Amini44ede332015-07-09 02:09:04 +00002086 SDValue CPAddr = DAG.getTargetConstantPool(CPV, PtrVt, 4);
Owen Anderson9f944592009-08-11 20:47:22 +00002087 CPAddr = DAG.getNode(ARMISD::Wrapper, dl, MVT::i32, CPAddr);
Alex Lorenze40c8a22015-08-11 23:09:45 +00002088 Callee = DAG.getLoad(
2089 PtrVt, dl, DAG.getEntryNode(), CPAddr,
Justin Lebar9c375812016-07-15 18:27:10 +00002090 MachinePointerInfo::getConstantPool(DAG.getMachineFunction()));
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +00002091 SDValue PICLabel = DAG.getConstant(ARMPCLabelIndex, dl, MVT::i32);
Mehdi Amini44ede332015-07-09 02:09:04 +00002092 Callee = DAG.getNode(ARMISD::PIC_ADD, dl, PtrVt, Callee, PICLabel);
Jim Grosbach85dcd3d2010-09-22 23:27:36 +00002093 } else {
Rafael Espindolaafade352016-06-16 16:09:53 +00002094 Callee = DAG.getTargetExternalSymbol(Sym, PtrVt, 0);
Jim Grosbach85dcd3d2010-09-22 23:27:36 +00002095 }
Evan Cheng10043e22007-01-19 07:51:42 +00002096 }
2097
Lauro Ramos Venancioa88c4a72007-03-20 17:57:23 +00002098 // FIXME: handle tail calls differently.
2099 unsigned CallOpc;
Evan Cheng6ab54fd2009-08-01 00:16:10 +00002100 if (Subtarget->isThumb()) {
2101 if ((!isDirect || isARMFunc) && !Subtarget->hasV5TOps())
Lauro Ramos Venancioa88c4a72007-03-20 17:57:23 +00002102 CallOpc = ARMISD::CALL_NOLINK;
2103 else
Tim Northoverb5ece522016-05-10 19:17:47 +00002104 CallOpc = ARMISD::CALL;
Lauro Ramos Venancioa88c4a72007-03-20 17:57:23 +00002105 } else {
Evan Cheng21b03482012-11-10 02:09:05 +00002106 if (!isDirect && !Subtarget->hasV5TOps())
Evan Cheng65f9d192012-02-28 18:51:51 +00002107 CallOpc = ARMISD::CALL_NOLINK;
Sjoerd Meijerd906bf12016-06-03 14:03:27 +00002108 else if (doesNotRet && isDirect && Subtarget->hasRetAddrStack() &&
Sanjay Patel924879a2015-08-04 15:49:57 +00002109 // Emit regular call when code size is the priority
2110 !MF.getFunction()->optForMinSize())
Evan Cheng65f9d192012-02-28 18:51:51 +00002111 // "mov lr, pc; b _foo" to avoid confusing the RSP
2112 CallOpc = ARMISD::CALL_NOLINK;
2113 else
2114 CallOpc = isLocalARMFunc ? ARMISD::CALL_PRED : ARMISD::CALL;
Lauro Ramos Venancioa88c4a72007-03-20 17:57:23 +00002115 }
Lauro Ramos Venancioa88c4a72007-03-20 17:57:23 +00002116
Dan Gohman2ce6f2a2008-07-27 21:46:04 +00002117 std::vector<SDValue> Ops;
Evan Cheng10043e22007-01-19 07:51:42 +00002118 Ops.push_back(Chain);
2119 Ops.push_back(Callee);
2120
2121 // Add argument registers to the end of the list so that they are known live
2122 // into the call.
2123 for (unsigned i = 0, e = RegsToPass.size(); i != e; ++i)
2124 Ops.push_back(DAG.getRegister(RegsToPass[i].first,
2125 RegsToPass[i].second.getValueType()));
2126
Jakob Stoklund Olesenfa7a5372012-02-24 01:19:29 +00002127 // Add a register mask operand representing the call-preserved registers.
Matthias Braunc22630e2013-10-04 16:52:54 +00002128 if (!isTailCall) {
2129 const uint32_t *Mask;
Eric Christopher1889fdc2015-01-29 00:19:39 +00002130 const ARMBaseRegisterInfo *ARI = Subtarget->getRegisterInfo();
Matthias Braunc22630e2013-10-04 16:52:54 +00002131 if (isThisReturn) {
2132 // For 'this' returns, use the R0-preserving mask if applicable
Eric Christopher9deb75d2015-03-11 22:42:13 +00002133 Mask = ARI->getThisReturnPreservedMask(MF, CallConv);
Matthias Braunc22630e2013-10-04 16:52:54 +00002134 if (!Mask) {
2135 // Set isThisReturn to false if the calling convention is not one that
2136 // allows 'returned' to be modeled in this way, so LowerCallResult does
2137 // not try to pass 'this' straight through
2138 isThisReturn = false;
Eric Christopher9deb75d2015-03-11 22:42:13 +00002139 Mask = ARI->getCallPreservedMask(MF, CallConv);
Matthias Braunc22630e2013-10-04 16:52:54 +00002140 }
2141 } else
Eric Christopher9deb75d2015-03-11 22:42:13 +00002142 Mask = ARI->getCallPreservedMask(MF, CallConv);
Stephen Linb8bd2322013-04-20 05:14:40 +00002143
Matthias Braunc22630e2013-10-04 16:52:54 +00002144 assert(Mask && "Missing call preserved mask for calling convention");
2145 Ops.push_back(DAG.getRegisterMask(Mask));
2146 }
Jakob Stoklund Olesenfa7a5372012-02-24 01:19:29 +00002147
Gabor Greiff304a7a2008-08-28 21:40:38 +00002148 if (InFlag.getNode())
Evan Cheng10043e22007-01-19 07:51:42 +00002149 Ops.push_back(InFlag);
Dale Johannesend679ff72010-06-03 21:09:53 +00002150
Chris Lattner3e5fbd72010-12-21 02:38:05 +00002151 SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);
Arnold Schwaighoferf54b73d2015-05-08 23:52:00 +00002152 if (isTailCall) {
Matthias Braun941a7052016-07-28 18:40:00 +00002153 MF.getFrameInfo().setHasTailCall();
Craig Topper48d114b2014-04-26 18:35:24 +00002154 return DAG.getNode(ARMISD::TC_RETURN, dl, NodeTys, Ops);
Arnold Schwaighoferf54b73d2015-05-08 23:52:00 +00002155 }
Dale Johannesend679ff72010-06-03 21:09:53 +00002156
Duncan Sands739a0542008-07-02 17:40:58 +00002157 // Returns a chain and a flag for retval copy to use.
Craig Topper48d114b2014-04-26 18:35:24 +00002158 Chain = DAG.getNode(CallOpc, dl, NodeTys, Ops);
Evan Cheng10043e22007-01-19 07:51:42 +00002159 InFlag = Chain.getValue(1);
2160
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +00002161 Chain = DAG.getCALLSEQ_END(Chain, DAG.getIntPtrConstant(NumBytes, dl, true),
2162 DAG.getIntPtrConstant(0, dl, true), InFlag, dl);
Dan Gohmanf9bbcd12009-08-05 01:29:28 +00002163 if (!Ins.empty())
Evan Cheng10043e22007-01-19 07:51:42 +00002164 InFlag = Chain.getValue(1);
2165
Bob Wilsona4c22902009-04-17 19:07:39 +00002166 // Handle result values, copying them out of physregs into vregs that we
2167 // return.
Stephen Linb8bd2322013-04-20 05:14:40 +00002168 return LowerCallResult(Chain, InFlag, CallConv, isVarArg, Ins, dl, DAG,
Stephen Lin4eedb292013-04-23 19:30:12 +00002169 InVals, isThisReturn,
2170 isThisReturn ? OutVals[0] : SDValue());
Evan Cheng10043e22007-01-19 07:51:42 +00002171}
2172
Stuart Hastings67c5c3e2011-02-28 17:17:53 +00002173/// HandleByVal - Every parameter *after* a byval parameter is passed
Stuart Hastings45fe3c32011-04-20 16:47:52 +00002174/// on the stack. Remember the next parameter register to allocate,
2175/// and then confiscate the rest of the parameter registers to insure
Stuart Hastings67c5c3e2011-02-28 17:17:53 +00002176/// this.
Tim Northover8cda34f2015-03-11 18:54:22 +00002177void ARMTargetLowering::HandleByVal(CCState *State, unsigned &Size,
2178 unsigned Align) const {
Stuart Hastings45fe3c32011-04-20 16:47:52 +00002179 assert((State->getCallOrPrologue() == Prologue ||
2180 State->getCallOrPrologue() == Call) &&
2181 "unhandled ParmContext");
Stepan Dyatkovskiy8c02c982013-05-05 07:48:36 +00002182
Tim Northover8cda34f2015-03-11 18:54:22 +00002183 // Byval (as with any stack) slots are always at least 4 byte aligned.
2184 Align = std::max(Align, 4U);
Stepan Dyatkovskiy8c02c982013-05-05 07:48:36 +00002185
Tim Northover8cda34f2015-03-11 18:54:22 +00002186 unsigned Reg = State->AllocateReg(GPRArgRegs);
2187 if (!Reg)
2188 return;
Stepan Dyatkovskiy8c02c982013-05-05 07:48:36 +00002189
Tim Northover8cda34f2015-03-11 18:54:22 +00002190 unsigned AlignInRegs = Align / 4;
2191 unsigned Waste = (ARM::R4 - Reg) % AlignInRegs;
2192 for (unsigned i = 0; i < Waste; ++i)
2193 Reg = State->AllocateReg(GPRArgRegs);
2194
2195 if (!Reg)
2196 return;
2197
2198 unsigned Excess = 4 * (ARM::R4 - Reg);
2199
2200 // Special case when NSAA != SP and parameter size greater than size of
2201 // all remained GPR regs. In that case we can't split parameter, we must
2202 // send it to stack. We also must set NCRN to R4, so waste all
2203 // remained registers.
2204 const unsigned NSAAOffset = State->getNextStackOffset();
2205 if (NSAAOffset != 0 && Size > Excess) {
2206 while (State->AllocateReg(GPRArgRegs))
2207 ;
2208 return;
Stuart Hastings45fe3c32011-04-20 16:47:52 +00002209 }
Tim Northover8cda34f2015-03-11 18:54:22 +00002210
2211 // First register for byval parameter is the first register that wasn't
2212 // allocated before this method call, so it would be "reg".
2213 // If parameter is small enough to be saved in range [reg, r4), then
2214 // the end (first after last) register would be reg + param-size-in-regs,
2215 // else parameter would be splitted between registers and stack,
2216 // end register would be r4 in this case.
2217 unsigned ByValRegBegin = Reg;
2218 unsigned ByValRegEnd = std::min<unsigned>(Reg + Size / 4, ARM::R4);
2219 State->addInRegsParamInfo(ByValRegBegin, ByValRegEnd);
2220 // Note, first register is allocated in the beginning of function already,
2221 // allocate remained amount of registers we need.
2222 for (unsigned i = Reg + 1; i != ByValRegEnd; ++i)
2223 State->AllocateReg(GPRArgRegs);
2224 // A byval parameter that is split between registers and memory needs its
2225 // size truncated here.
2226 // In the case where the entire structure fits in registers, we set the
2227 // size in memory to zero.
2228 Size = std::max<int>(Size - Excess, 0);
Stuart Hastings67c5c3e2011-02-28 17:17:53 +00002229}
2230
Dale Johannesend679ff72010-06-03 21:09:53 +00002231/// MatchingStackOffset - Return true if the given stack call argument is
2232/// already available in the same position (relatively) of the caller's
2233/// incoming argument stack.
2234static
2235bool MatchingStackOffset(SDValue Arg, unsigned Offset, ISD::ArgFlagsTy Flags,
Matthias Braun941a7052016-07-28 18:40:00 +00002236 MachineFrameInfo &MFI, const MachineRegisterInfo *MRI,
Craig Topper07720d82012-03-25 23:49:58 +00002237 const TargetInstrInfo *TII) {
Sanjay Patelb1f0a0f2016-09-14 16:05:51 +00002238 unsigned Bytes = Arg.getValueSizeInBits() / 8;
Dale Johannesend679ff72010-06-03 21:09:53 +00002239 int FI = INT_MAX;
2240 if (Arg.getOpcode() == ISD::CopyFromReg) {
2241 unsigned VR = cast<RegisterSDNode>(Arg.getOperand(1))->getReg();
Jakob Stoklund Olesen2fb5b312011-01-10 02:58:51 +00002242 if (!TargetRegisterInfo::isVirtualRegister(VR))
Dale Johannesend679ff72010-06-03 21:09:53 +00002243 return false;
2244 MachineInstr *Def = MRI->getVRegDef(VR);
2245 if (!Def)
2246 return false;
2247 if (!Flags.isByVal()) {
Duncan P. N. Exon Smith9cfc75c2016-06-30 00:01:54 +00002248 if (!TII->isLoadFromStackSlot(*Def, FI))
Dale Johannesend679ff72010-06-03 21:09:53 +00002249 return false;
2250 } else {
Dale Johannesene2289282010-07-08 01:18:23 +00002251 return false;
Dale Johannesend679ff72010-06-03 21:09:53 +00002252 }
2253 } else if (LoadSDNode *Ld = dyn_cast<LoadSDNode>(Arg)) {
2254 if (Flags.isByVal())
2255 // ByVal argument is passed in as a pointer but it's now being
2256 // dereferenced. e.g.
2257 // define @foo(%struct.X* %A) {
2258 // tail call @bar(%struct.X* byval %A)
2259 // }
2260 return false;
2261 SDValue Ptr = Ld->getBasePtr();
2262 FrameIndexSDNode *FINode = dyn_cast<FrameIndexSDNode>(Ptr);
2263 if (!FINode)
2264 return false;
2265 FI = FINode->getIndex();
2266 } else
2267 return false;
2268
2269 assert(FI != INT_MAX);
Matthias Braun941a7052016-07-28 18:40:00 +00002270 if (!MFI.isFixedObjectIndex(FI))
Dale Johannesend679ff72010-06-03 21:09:53 +00002271 return false;
Matthias Braun941a7052016-07-28 18:40:00 +00002272 return Offset == MFI.getObjectOffset(FI) && Bytes == MFI.getObjectSize(FI);
Dale Johannesend679ff72010-06-03 21:09:53 +00002273}
2274
2275/// IsEligibleForTailCallOptimization - Check whether the call is eligible
2276/// for tail call optimization. Targets which want to do tail call
2277/// optimization should implement this function.
2278bool
2279ARMTargetLowering::IsEligibleForTailCallOptimization(SDValue Callee,
2280 CallingConv::ID CalleeCC,
2281 bool isVarArg,
2282 bool isCalleeStructRet,
2283 bool isCallerStructRet,
2284 const SmallVectorImpl<ISD::OutputArg> &Outs,
Dan Gohmanfe7532a2010-07-07 15:54:55 +00002285 const SmallVectorImpl<SDValue> &OutVals,
Dale Johannesend679ff72010-06-03 21:09:53 +00002286 const SmallVectorImpl<ISD::InputArg> &Ins,
2287 SelectionDAG& DAG) const {
Matthias Braun8d414362016-03-30 22:46:04 +00002288 MachineFunction &MF = DAG.getMachineFunction();
2289 const Function *CallerF = MF.getFunction();
Dale Johannesend679ff72010-06-03 21:09:53 +00002290 CallingConv::ID CallerCC = CallerF->getCallingConv();
Manman Ren4865d892016-03-18 23:41:51 +00002291
Artyom Skrobovad8a0632015-09-28 09:44:11 +00002292 assert(Subtarget->supportsTailCall());
2293
Dale Johannesend679ff72010-06-03 21:09:53 +00002294 // Look for obvious safe cases to perform tail call optimization that do not
2295 // require ABI changes. This is what gcc calls sibcall.
2296
Jim Grosbache3864cc2010-06-16 23:45:49 +00002297 // Do not sibcall optimize vararg calls unless the call site is not passing
2298 // any arguments.
Dale Johannesend679ff72010-06-03 21:09:53 +00002299 if (isVarArg && !Outs.empty())
2300 return false;
2301
Tim Northoverd8407452013-10-01 14:33:28 +00002302 // Exception-handling functions need a special set of instructions to indicate
2303 // a return to the hardware. Tail-calling another function would probably
2304 // break this.
2305 if (CallerF->hasFnAttribute("interrupt"))
2306 return false;
2307
Dale Johannesend679ff72010-06-03 21:09:53 +00002308 // Also avoid sibcall optimization if either caller or callee uses struct
2309 // return semantics.
2310 if (isCalleeStructRet || isCallerStructRet)
2311 return false;
2312
Oliver Stannard12993dd2014-08-18 12:42:15 +00002313 // Externally-defined functions with weak linkage should not be
2314 // tail-called on ARM when the OS does not support dynamic
2315 // pre-emption of symbols, as the AAELF spec requires normal calls
2316 // to undefined weak functions to be replaced with a NOP or jump to the
2317 // next instruction. The behaviour of branch instructions in this
2318 // situation (as used for tail calls) is implementation-defined, so we
2319 // cannot rely on the linker replacing the tail call with a return.
2320 if (GlobalAddressSDNode *G = dyn_cast<GlobalAddressSDNode>(Callee)) {
2321 const GlobalValue *GV = G->getGlobal();
Daniel Sandersc81f4502015-06-16 15:44:21 +00002322 const Triple &TT = getTargetMachine().getTargetTriple();
Saleem Abdulrasool67f72992015-01-03 21:35:00 +00002323 if (GV->hasExternalWeakLinkage() &&
2324 (!TT.isOSWindows() || TT.isOSBinFormatELF() || TT.isOSBinFormatMachO()))
Oliver Stannard12993dd2014-08-18 12:42:15 +00002325 return false;
2326 }
2327
Matthias Braun8d414362016-03-30 22:46:04 +00002328 // Check that the call results are passed in the same way.
2329 LLVMContext &C = *DAG.getContext();
2330 if (!CCState::resultsCompatible(CalleeCC, CallerCC, MF, C, Ins,
2331 CCAssignFnForNode(CalleeCC, true, isVarArg),
2332 CCAssignFnForNode(CallerCC, true, isVarArg)))
2333 return false;
Matthias Braun870c34f2016-04-04 18:56:13 +00002334 // The callee has to preserve all registers the caller needs to preserve.
Matthias Braun707e02c2016-04-13 21:43:25 +00002335 const ARMBaseRegisterInfo *TRI = Subtarget->getRegisterInfo();
2336 const uint32_t *CallerPreserved = TRI->getCallPreservedMask(MF, CallerCC);
Matthias Braun870c34f2016-04-04 18:56:13 +00002337 if (CalleeCC != CallerCC) {
Matthias Braun707e02c2016-04-13 21:43:25 +00002338 const uint32_t *CalleePreserved = TRI->getCallPreservedMask(MF, CalleeCC);
2339 if (!TRI->regmaskSubsetEqual(CallerPreserved, CalleePreserved))
Matthias Braun870c34f2016-04-04 18:56:13 +00002340 return false;
2341 }
Dale Johannesend679ff72010-06-03 21:09:53 +00002342
Manman Ren7e48b252012-10-12 23:39:43 +00002343 // If Caller's vararg or byval argument has been split between registers and
2344 // stack, do not perform tail call, since part of the argument is in caller's
2345 // local frame.
Matthias Braun8d414362016-03-30 22:46:04 +00002346 const ARMFunctionInfo *AFI_Caller = MF.getInfo<ARMFunctionInfo>();
Stepan Dyatkovskiyf5aa83d2013-04-30 07:19:58 +00002347 if (AFI_Caller->getArgRegsSaveSize())
Manman Ren7e48b252012-10-12 23:39:43 +00002348 return false;
2349
Dale Johannesend679ff72010-06-03 21:09:53 +00002350 // If the callee takes no arguments then go on to check the results of the
2351 // call.
2352 if (!Outs.empty()) {
2353 // Check if stack adjustment is needed. For now, do not do this if any
2354 // argument is passed on the stack.
2355 SmallVector<CCValAssign, 16> ArgLocs;
Matthias Braun8d414362016-03-30 22:46:04 +00002356 ARMCCState CCInfo(CalleeCC, isVarArg, MF, ArgLocs, C, Call);
Dale Johannesend679ff72010-06-03 21:09:53 +00002357 CCInfo.AnalyzeCallOperands(Outs,
2358 CCAssignFnForNode(CalleeCC, false, isVarArg));
2359 if (CCInfo.getNextStackOffset()) {
Dale Johannesend679ff72010-06-03 21:09:53 +00002360 // Check if the arguments are already laid out in the right way as
2361 // the caller's fixed stack objects.
Matthias Braun941a7052016-07-28 18:40:00 +00002362 MachineFrameInfo &MFI = MF.getFrameInfo();
Dale Johannesend679ff72010-06-03 21:09:53 +00002363 const MachineRegisterInfo *MRI = &MF.getRegInfo();
Eric Christopher1889fdc2015-01-29 00:19:39 +00002364 const TargetInstrInfo *TII = Subtarget->getInstrInfo();
Dale Johannesen81ef35b2010-06-05 00:51:39 +00002365 for (unsigned i = 0, realArgIdx = 0, e = ArgLocs.size();
2366 i != e;
2367 ++i, ++realArgIdx) {
Dale Johannesend679ff72010-06-03 21:09:53 +00002368 CCValAssign &VA = ArgLocs[i];
2369 EVT RegVT = VA.getLocVT();
Dan Gohmanfe7532a2010-07-07 15:54:55 +00002370 SDValue Arg = OutVals[realArgIdx];
Dale Johannesen81ef35b2010-06-05 00:51:39 +00002371 ISD::ArgFlagsTy Flags = Outs[realArgIdx].Flags;
Dale Johannesend679ff72010-06-03 21:09:53 +00002372 if (VA.getLocInfo() == CCValAssign::Indirect)
2373 return false;
Dale Johannesen81ef35b2010-06-05 00:51:39 +00002374 if (VA.needsCustom()) {
2375 // f64 and vector types are split into multiple registers or
2376 // register/stack-slot combinations. The types will not match
2377 // the registers; give up on memory f64 refs until we figure
2378 // out what to do about this.
2379 if (!VA.isRegLoc())
2380 return false;
2381 if (!ArgLocs[++i].isRegLoc())
Jim Grosbach535d3b42010-09-08 03:54:02 +00002382 return false;
Dale Johannesen81ef35b2010-06-05 00:51:39 +00002383 if (RegVT == MVT::v2f64) {
2384 if (!ArgLocs[++i].isRegLoc())
2385 return false;
2386 if (!ArgLocs[++i].isRegLoc())
2387 return false;
2388 }
2389 } else if (!VA.isRegLoc()) {
Dale Johannesend679ff72010-06-03 21:09:53 +00002390 if (!MatchingStackOffset(Arg, VA.getLocMemOffset(), Flags,
2391 MFI, MRI, TII))
2392 return false;
2393 }
2394 }
2395 }
Matthias Braun707e02c2016-04-13 21:43:25 +00002396
Matthias Braun46b0f032016-04-14 01:10:42 +00002397 const MachineRegisterInfo &MRI = MF.getRegInfo();
2398 if (!parametersInCSRMatch(MRI, CallerPreserved, ArgLocs, OutVals))
2399 return false;
Dale Johannesend679ff72010-06-03 21:09:53 +00002400 }
2401
2402 return true;
2403}
2404
Benjamin Kramerb1996da2012-11-28 20:55:10 +00002405bool
2406ARMTargetLowering::CanLowerReturn(CallingConv::ID CallConv,
2407 MachineFunction &MF, bool isVarArg,
2408 const SmallVectorImpl<ISD::OutputArg> &Outs,
2409 LLVMContext &Context) const {
2410 SmallVector<CCValAssign, 16> RVLocs;
Eric Christopherb5217502014-08-06 18:45:26 +00002411 CCState CCInfo(CallConv, isVarArg, MF, RVLocs, Context);
Benjamin Kramerb1996da2012-11-28 20:55:10 +00002412 return CCInfo.CheckReturn(Outs, CCAssignFnForNode(CallConv, /*Return=*/true,
2413 isVarArg));
2414}
2415
Tim Northoverd8407452013-10-01 14:33:28 +00002416static SDValue LowerInterruptReturn(SmallVectorImpl<SDValue> &RetOps,
Benjamin Kramerbdc49562016-06-12 15:39:02 +00002417 const SDLoc &DL, SelectionDAG &DAG) {
Tim Northoverd8407452013-10-01 14:33:28 +00002418 const MachineFunction &MF = DAG.getMachineFunction();
2419 const Function *F = MF.getFunction();
2420
2421 StringRef IntKind = F->getFnAttribute("interrupt").getValueAsString();
2422
2423 // See ARM ARM v7 B1.8.3. On exception entry LR is set to a possibly offset
2424 // version of the "preferred return address". These offsets affect the return
2425 // instruction if this is a return from PL1 without hypervisor extensions.
2426 // IRQ/FIQ: +4 "subs pc, lr, #4"
2427 // SWI: 0 "subs pc, lr, #0"
2428 // ABORT: +4 "subs pc, lr, #4"
2429 // UNDEF: +4/+2 "subs pc, lr, #0"
2430 // UNDEF varies depending on where the exception came from ARM or Thumb
2431 // mode. Alongside GCC, we throw our hands up in disgust and pretend it's 0.
2432
2433 int64_t LROffset;
2434 if (IntKind == "" || IntKind == "IRQ" || IntKind == "FIQ" ||
2435 IntKind == "ABORT")
2436 LROffset = 4;
2437 else if (IntKind == "SWI" || IntKind == "UNDEF")
2438 LROffset = 0;
2439 else
2440 report_fatal_error("Unsupported interrupt attribute. If present, value "
2441 "must be one of: IRQ, FIQ, SWI, ABORT or UNDEF");
2442
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +00002443 RetOps.insert(RetOps.begin() + 1,
2444 DAG.getConstant(LROffset, DL, MVT::i32, false));
Tim Northoverd8407452013-10-01 14:33:28 +00002445
Craig Topper48d114b2014-04-26 18:35:24 +00002446 return DAG.getNode(ARMISD::INTRET_FLAG, DL, MVT::Other, RetOps);
Tim Northoverd8407452013-10-01 14:33:28 +00002447}
2448
Dan Gohmanf9bbcd12009-08-05 01:29:28 +00002449SDValue
Benjamin Kramerbdc49562016-06-12 15:39:02 +00002450ARMTargetLowering::LowerReturn(SDValue Chain, CallingConv::ID CallConv,
2451 bool isVarArg,
Dan Gohmanf9bbcd12009-08-05 01:29:28 +00002452 const SmallVectorImpl<ISD::OutputArg> &Outs,
Dan Gohmanfe7532a2010-07-07 15:54:55 +00002453 const SmallVectorImpl<SDValue> &OutVals,
Benjamin Kramerbdc49562016-06-12 15:39:02 +00002454 const SDLoc &dl, SelectionDAG &DAG) const {
Bob Wilson7117a912009-03-20 22:42:55 +00002455
Bob Wilsonea09d4a2009-04-17 20:35:10 +00002456 // CCValAssign - represent the assignment of the return value to a location.
Bob Wilsona4c22902009-04-17 19:07:39 +00002457 SmallVector<CCValAssign, 16> RVLocs;
Bob Wilsona4c22902009-04-17 19:07:39 +00002458
Bob Wilsonea09d4a2009-04-17 20:35:10 +00002459 // CCState - Info about the registers and stack slots.
Eric Christopherb5217502014-08-06 18:45:26 +00002460 ARMCCState CCInfo(CallConv, isVarArg, DAG.getMachineFunction(), RVLocs,
2461 *DAG.getContext(), Call);
Bob Wilsona4c22902009-04-17 19:07:39 +00002462
Dan Gohmanf9bbcd12009-08-05 01:29:28 +00002463 // Analyze outgoing return values.
Anton Korobeynikov22ef7512009-08-05 19:04:42 +00002464 CCInfo.AnalyzeReturn(Outs, CCAssignFnForNode(CallConv, /* Return */ true,
2465 isVarArg));
Bob Wilsona4c22902009-04-17 19:07:39 +00002466
Bob Wilsona4c22902009-04-17 19:07:39 +00002467 SDValue Flag;
Jakob Stoklund Olesenf90fb6e2013-02-05 18:08:40 +00002468 SmallVector<SDValue, 4> RetOps;
2469 RetOps.push_back(Chain); // Operand #0 = Chain (updated below)
Christian Pirkerb5728192014-05-08 14:06:24 +00002470 bool isLittleEndian = Subtarget->isLittle();
Bob Wilsona4c22902009-04-17 19:07:39 +00002471
Jonathan Roelofsef84bda2014-08-05 21:32:21 +00002472 MachineFunction &MF = DAG.getMachineFunction();
2473 ARMFunctionInfo *AFI = MF.getInfo<ARMFunctionInfo>();
2474 AFI->setReturnRegsCount(RVLocs.size());
2475
Bob Wilsona4c22902009-04-17 19:07:39 +00002476 // Copy the result values into the output registers.
2477 for (unsigned i = 0, realRVLocIdx = 0;
2478 i != RVLocs.size();
2479 ++i, ++realRVLocIdx) {
2480 CCValAssign &VA = RVLocs[i];
2481 assert(VA.isRegLoc() && "Can only return in registers!");
2482
Dan Gohmanfe7532a2010-07-07 15:54:55 +00002483 SDValue Arg = OutVals[realRVLocIdx];
Bob Wilsona4c22902009-04-17 19:07:39 +00002484
2485 switch (VA.getLocInfo()) {
Torok Edwinfbcc6632009-07-14 16:55:14 +00002486 default: llvm_unreachable("Unknown loc info!");
Bob Wilsona4c22902009-04-17 19:07:39 +00002487 case CCValAssign::Full: break;
2488 case CCValAssign::BCvt:
Wesley Peck527da1b2010-11-23 03:31:01 +00002489 Arg = DAG.getNode(ISD::BITCAST, dl, VA.getLocVT(), Arg);
Bob Wilsona4c22902009-04-17 19:07:39 +00002490 break;
2491 }
2492
Bob Wilsona4c22902009-04-17 19:07:39 +00002493 if (VA.needsCustom()) {
Owen Anderson9f944592009-08-11 20:47:22 +00002494 if (VA.getLocVT() == MVT::v2f64) {
Bob Wilson2e076c42009-06-22 23:27:02 +00002495 // Extract the first half and return it in two registers.
Owen Anderson9f944592009-08-11 20:47:22 +00002496 SDValue Half = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::f64, Arg,
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +00002497 DAG.getConstant(0, dl, MVT::i32));
Jim Grosbachd7cf55c2009-11-09 00:11:35 +00002498 SDValue HalfGPRs = DAG.getNode(ARMISD::VMOVRRD, dl,
Owen Anderson9f944592009-08-11 20:47:22 +00002499 DAG.getVTList(MVT::i32, MVT::i32), Half);
Bob Wilson2e076c42009-06-22 23:27:02 +00002500
Christian Pirkerb5728192014-05-08 14:06:24 +00002501 Chain = DAG.getCopyToReg(Chain, dl, VA.getLocReg(),
2502 HalfGPRs.getValue(isLittleEndian ? 0 : 1),
2503 Flag);
Bob Wilson2e076c42009-06-22 23:27:02 +00002504 Flag = Chain.getValue(1);
Jakob Stoklund Olesenf90fb6e2013-02-05 18:08:40 +00002505 RetOps.push_back(DAG.getRegister(VA.getLocReg(), VA.getLocVT()));
Bob Wilson2e076c42009-06-22 23:27:02 +00002506 VA = RVLocs[++i]; // skip ahead to next loc
2507 Chain = DAG.getCopyToReg(Chain, dl, VA.getLocReg(),
Christian Pirkerb5728192014-05-08 14:06:24 +00002508 HalfGPRs.getValue(isLittleEndian ? 1 : 0),
2509 Flag);
Bob Wilson2e076c42009-06-22 23:27:02 +00002510 Flag = Chain.getValue(1);
Jakob Stoklund Olesenf90fb6e2013-02-05 18:08:40 +00002511 RetOps.push_back(DAG.getRegister(VA.getLocReg(), VA.getLocVT()));
Bob Wilson2e076c42009-06-22 23:27:02 +00002512 VA = RVLocs[++i]; // skip ahead to next loc
2513
2514 // Extract the 2nd half and fall through to handle it as an f64 value.
Owen Anderson9f944592009-08-11 20:47:22 +00002515 Arg = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::f64, Arg,
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +00002516 DAG.getConstant(1, dl, MVT::i32));
Bob Wilson2e076c42009-06-22 23:27:02 +00002517 }
2518 // Legalize ret f64 -> ret 2 x i32. We always have fmrrd if f64 is
2519 // available.
Jim Grosbachd7cf55c2009-11-09 00:11:35 +00002520 SDValue fmrrd = DAG.getNode(ARMISD::VMOVRRD, dl,
Craig Topper48d114b2014-04-26 18:35:24 +00002521 DAG.getVTList(MVT::i32, MVT::i32), Arg);
Christian Pirkerb5728192014-05-08 14:06:24 +00002522 Chain = DAG.getCopyToReg(Chain, dl, VA.getLocReg(),
2523 fmrrd.getValue(isLittleEndian ? 0 : 1),
2524 Flag);
Bob Wilsonf134b2d2009-04-24 17:00:36 +00002525 Flag = Chain.getValue(1);
Jakob Stoklund Olesenf90fb6e2013-02-05 18:08:40 +00002526 RetOps.push_back(DAG.getRegister(VA.getLocReg(), VA.getLocVT()));
Bob Wilsona4c22902009-04-17 19:07:39 +00002527 VA = RVLocs[++i]; // skip ahead to next loc
Christian Pirkerb5728192014-05-08 14:06:24 +00002528 Chain = DAG.getCopyToReg(Chain, dl, VA.getLocReg(),
2529 fmrrd.getValue(isLittleEndian ? 1 : 0),
Bob Wilsona4c22902009-04-17 19:07:39 +00002530 Flag);
2531 } else
2532 Chain = DAG.getCopyToReg(Chain, dl, VA.getLocReg(), Arg, Flag);
2533
Bob Wilsonea09d4a2009-04-17 20:35:10 +00002534 // Guarantee that all emitted copies are
2535 // stuck together, avoiding something bad.
Bob Wilsona4c22902009-04-17 19:07:39 +00002536 Flag = Chain.getValue(1);
Jakob Stoklund Olesenf90fb6e2013-02-05 18:08:40 +00002537 RetOps.push_back(DAG.getRegister(VA.getLocReg(), VA.getLocVT()));
Bob Wilsona4c22902009-04-17 19:07:39 +00002538 }
Manman Ren5e9e65e2016-01-12 00:47:18 +00002539 const ARMBaseRegisterInfo *TRI = Subtarget->getRegisterInfo();
2540 const MCPhysReg *I =
2541 TRI->getCalleeSavedRegsViaCopy(&DAG.getMachineFunction());
2542 if (I) {
2543 for (; *I; ++I) {
2544 if (ARM::GPRRegClass.contains(*I))
2545 RetOps.push_back(DAG.getRegister(*I, MVT::i32));
2546 else if (ARM::DPRRegClass.contains(*I))
2547 RetOps.push_back(DAG.getRegister(*I, MVT::getFloatingPointVT(64)));
2548 else
2549 llvm_unreachable("Unexpected register class in CSRsViaCopy!");
2550 }
2551 }
Bob Wilsona4c22902009-04-17 19:07:39 +00002552
Jakob Stoklund Olesenf90fb6e2013-02-05 18:08:40 +00002553 // Update chain and glue.
2554 RetOps[0] = Chain;
Bob Wilsona4c22902009-04-17 19:07:39 +00002555 if (Flag.getNode())
Jakob Stoklund Olesenf90fb6e2013-02-05 18:08:40 +00002556 RetOps.push_back(Flag);
Bob Wilsona4c22902009-04-17 19:07:39 +00002557
Tim Northoverd8407452013-10-01 14:33:28 +00002558 // CPUs which aren't M-class use a special sequence to return from
2559 // exceptions (roughly, any instruction setting pc and cpsr simultaneously,
2560 // though we use "subs pc, lr, #N").
2561 //
2562 // M-class CPUs actually use a normal return sequence with a special
2563 // (hardware-provided) value in LR, so the normal code path works.
2564 if (DAG.getMachineFunction().getFunction()->hasFnAttribute("interrupt") &&
2565 !Subtarget->isMClass()) {
2566 if (Subtarget->isThumb1Only())
2567 report_fatal_error("interrupt attribute is not supported in Thumb1");
2568 return LowerInterruptReturn(RetOps, dl, DAG);
2569 }
2570
Craig Topper48d114b2014-04-26 18:35:24 +00002571 return DAG.getNode(ARMISD::RET_FLAG, dl, MVT::Other, RetOps);
Evan Cheng10043e22007-01-19 07:51:42 +00002572}
2573
Evan Chengf8bad082012-04-10 01:51:00 +00002574bool ARMTargetLowering::isUsedByReturnOnly(SDNode *N, SDValue &Chain) const {
Evan Chengd4b08732010-11-30 23:55:39 +00002575 if (N->getNumValues() != 1)
2576 return false;
2577 if (!N->hasNUsesOfValue(1, 0))
2578 return false;
2579
Evan Chengf8bad082012-04-10 01:51:00 +00002580 SDValue TCChain = Chain;
2581 SDNode *Copy = *N->use_begin();
2582 if (Copy->getOpcode() == ISD::CopyToReg) {
2583 // If the copy has a glue operand, we conservatively assume it isn't safe to
2584 // perform a tail call.
2585 if (Copy->getOperand(Copy->getNumOperands()-1).getValueType() == MVT::Glue)
2586 return false;
2587 TCChain = Copy->getOperand(0);
2588 } else if (Copy->getOpcode() == ARMISD::VMOVRRD) {
2589 SDNode *VMov = Copy;
Evan Chengd4b08732010-11-30 23:55:39 +00002590 // f64 returned in a pair of GPRs.
Evan Chengf8bad082012-04-10 01:51:00 +00002591 SmallPtrSet<SDNode*, 2> Copies;
2592 for (SDNode::use_iterator UI = VMov->use_begin(), UE = VMov->use_end();
Evan Chengd4b08732010-11-30 23:55:39 +00002593 UI != UE; ++UI) {
2594 if (UI->getOpcode() != ISD::CopyToReg)
2595 return false;
Evan Chengf8bad082012-04-10 01:51:00 +00002596 Copies.insert(*UI);
Evan Chengd4b08732010-11-30 23:55:39 +00002597 }
Evan Chengf8bad082012-04-10 01:51:00 +00002598 if (Copies.size() > 2)
2599 return false;
2600
2601 for (SDNode::use_iterator UI = VMov->use_begin(), UE = VMov->use_end();
2602 UI != UE; ++UI) {
2603 SDValue UseChain = UI->getOperand(0);
2604 if (Copies.count(UseChain.getNode()))
2605 // Second CopyToReg
2606 Copy = *UI;
Quentin Colombet17799fe2014-09-18 21:17:50 +00002607 else {
2608 // We are at the top of this chain.
2609 // If the copy has a glue operand, we conservatively assume it
2610 // isn't safe to perform a tail call.
2611 if (UI->getOperand(UI->getNumOperands()-1).getValueType() == MVT::Glue)
2612 return false;
Evan Chengf8bad082012-04-10 01:51:00 +00002613 // First CopyToReg
2614 TCChain = UseChain;
Quentin Colombet17799fe2014-09-18 21:17:50 +00002615 }
Evan Chengf8bad082012-04-10 01:51:00 +00002616 }
2617 } else if (Copy->getOpcode() == ISD::BITCAST) {
Evan Chengd4b08732010-11-30 23:55:39 +00002618 // f32 returned in a single GPR.
Evan Chengf8bad082012-04-10 01:51:00 +00002619 if (!Copy->hasOneUse())
Evan Chengd4b08732010-11-30 23:55:39 +00002620 return false;
Evan Chengf8bad082012-04-10 01:51:00 +00002621 Copy = *Copy->use_begin();
2622 if (Copy->getOpcode() != ISD::CopyToReg || !Copy->hasNUsesOfValue(1, 0))
Evan Chengd4b08732010-11-30 23:55:39 +00002623 return false;
Quentin Colombet17799fe2014-09-18 21:17:50 +00002624 // If the copy has a glue operand, we conservatively assume it isn't safe to
2625 // perform a tail call.
2626 if (Copy->getOperand(Copy->getNumOperands()-1).getValueType() == MVT::Glue)
2627 return false;
Lang Hames67c09b32013-05-13 10:21:19 +00002628 TCChain = Copy->getOperand(0);
Evan Chengd4b08732010-11-30 23:55:39 +00002629 } else {
2630 return false;
2631 }
2632
Evan Cheng419ea282010-12-01 22:59:46 +00002633 bool HasRet = false;
Evan Chengf8bad082012-04-10 01:51:00 +00002634 for (SDNode::use_iterator UI = Copy->use_begin(), UE = Copy->use_end();
2635 UI != UE; ++UI) {
Tim Northoverd8407452013-10-01 14:33:28 +00002636 if (UI->getOpcode() != ARMISD::RET_FLAG &&
2637 UI->getOpcode() != ARMISD::INTRET_FLAG)
Evan Chengf8bad082012-04-10 01:51:00 +00002638 return false;
2639 HasRet = true;
Evan Chengd4b08732010-11-30 23:55:39 +00002640 }
2641
Evan Chengf8bad082012-04-10 01:51:00 +00002642 if (!HasRet)
2643 return false;
2644
2645 Chain = TCChain;
2646 return true;
Evan Chengd4b08732010-11-30 23:55:39 +00002647}
2648
Evan Cheng0663f232011-03-21 01:19:09 +00002649bool ARMTargetLowering::mayBeEmittedAsTailCall(CallInst *CI) const {
Saleem Abdulrasoolb720a6b2014-03-11 15:09:49 +00002650 if (!Subtarget->supportsTailCall())
Evan Cheng0663f232011-03-21 01:19:09 +00002651 return false;
2652
Akira Hatanakad9699bc2015-06-09 19:07:19 +00002653 auto Attr =
2654 CI->getParent()->getParent()->getFnAttribute("disable-tail-calls");
2655 if (!CI->isTailCall() || Attr.getValueAsString() == "true")
Evan Cheng0663f232011-03-21 01:19:09 +00002656 return false;
2657
Artyom Skrobovad8a0632015-09-28 09:44:11 +00002658 return true;
Evan Cheng0663f232011-03-21 01:19:09 +00002659}
2660
Luke Cheeseman85fd06d2015-06-01 12:02:47 +00002661// Trying to write a 64 bit value so need to split into two 32 bit values first,
2662// and pass the lower and high parts through.
2663static SDValue LowerWRITE_REGISTER(SDValue Op, SelectionDAG &DAG) {
2664 SDLoc DL(Op);
2665 SDValue WriteValue = Op->getOperand(2);
2666
2667 // This function is only supposed to be called for i64 type argument.
2668 assert(WriteValue.getValueType() == MVT::i64
2669 && "LowerWRITE_REGISTER called for non-i64 type argument.");
2670
2671 SDValue Lo = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, MVT::i32, WriteValue,
2672 DAG.getConstant(0, DL, MVT::i32));
2673 SDValue Hi = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, MVT::i32, WriteValue,
2674 DAG.getConstant(1, DL, MVT::i32));
2675 SDValue Ops[] = { Op->getOperand(0), Op->getOperand(1), Lo, Hi };
2676 return DAG.getNode(ISD::WRITE_REGISTER, DL, MVT::Other, Ops);
2677}
2678
Bob Wilsonb389f2a2009-11-03 00:02:05 +00002679// ConstantPool, JumpTable, GlobalAddress, and ExternalSymbol are lowered as
2680// their target counterpart wrapped in the ARMISD::Wrapper node. Suppose N is
2681// one of the above mentioned nodes. It has to be wrapped because otherwise
2682// Select(N) returns N. So the raw TargetGlobalAddress nodes, etc. can only
2683// be used to form addressing mode. These wrapped nodes will be selected
2684// into MOVi.
Dan Gohman2ce6f2a2008-07-27 21:46:04 +00002685static SDValue LowerConstantPool(SDValue Op, SelectionDAG &DAG) {
Owen Anderson53aa7a92009-08-10 22:56:29 +00002686 EVT PtrVT = Op.getValueType();
Dale Johannesen62fd95d2009-02-07 00:55:49 +00002687 // FIXME there is no actual debug info here
Andrew Trickef9de2a2013-05-25 02:42:55 +00002688 SDLoc dl(Op);
Evan Cheng10043e22007-01-19 07:51:42 +00002689 ConstantPoolSDNode *CP = cast<ConstantPoolSDNode>(Op);
Dan Gohman2ce6f2a2008-07-27 21:46:04 +00002690 SDValue Res;
Evan Cheng10043e22007-01-19 07:51:42 +00002691 if (CP->isMachineConstantPoolEntry())
2692 Res = DAG.getTargetConstantPool(CP->getMachineCPVal(), PtrVT,
2693 CP->getAlignment());
2694 else
2695 Res = DAG.getTargetConstantPool(CP->getConstVal(), PtrVT,
2696 CP->getAlignment());
Owen Anderson9f944592009-08-11 20:47:22 +00002697 return DAG.getNode(ARMISD::Wrapper, dl, MVT::i32, Res);
Evan Cheng10043e22007-01-19 07:51:42 +00002698}
2699
Jim Grosbach8d3ba732010-07-19 17:20:38 +00002700unsigned ARMTargetLowering::getJumpTableEncoding() const {
2701 return MachineJumpTableInfo::EK_Inline;
2702}
2703
Dan Gohman21cea8a2010-04-17 15:26:15 +00002704SDValue ARMTargetLowering::LowerBlockAddress(SDValue Op,
2705 SelectionDAG &DAG) const {
Evan Cheng408aa562009-11-06 22:24:13 +00002706 MachineFunction &MF = DAG.getMachineFunction();
2707 ARMFunctionInfo *AFI = MF.getInfo<ARMFunctionInfo>();
2708 unsigned ARMPCLabelIndex = 0;
Andrew Trickef9de2a2013-05-25 02:42:55 +00002709 SDLoc DL(Op);
Mehdi Amini44ede332015-07-09 02:09:04 +00002710 EVT PtrVT = getPointerTy(DAG.getDataLayout());
Dan Gohmanbcaf6812010-04-15 01:51:59 +00002711 const BlockAddress *BA = cast<BlockAddressSDNode>(Op)->getBlockAddress();
Bob Wilson1c66e8a2009-11-02 20:59:23 +00002712 SDValue CPAddr;
Oliver Stannard8331aae2016-08-08 15:28:31 +00002713 bool IsPositionIndependent = isPositionIndependent() || Subtarget->isROPI();
Rafael Espindola0f898332016-06-20 16:43:17 +00002714 if (!IsPositionIndependent) {
Bob Wilson1c66e8a2009-11-02 20:59:23 +00002715 CPAddr = DAG.getTargetConstantPool(BA, PtrVT, 4);
2716 } else {
2717 unsigned PCAdj = Subtarget->isThumb() ? 4 : 8;
Evan Chengdfce83c2011-01-17 08:03:18 +00002718 ARMPCLabelIndex = AFI->createPICLabelUId();
Bill Wendling7753d662011-10-01 08:00:54 +00002719 ARMConstantPoolValue *CPV =
2720 ARMConstantPoolConstant::Create(BA, ARMPCLabelIndex,
2721 ARMCP::CPBlockAddress, PCAdj);
Bob Wilson1c66e8a2009-11-02 20:59:23 +00002722 CPAddr = DAG.getTargetConstantPool(CPV, PtrVT, 4);
2723 }
2724 CPAddr = DAG.getNode(ARMISD::Wrapper, DL, PtrVT, CPAddr);
Justin Lebar9c375812016-07-15 18:27:10 +00002725 SDValue Result = DAG.getLoad(
2726 PtrVT, DL, DAG.getEntryNode(), CPAddr,
2727 MachinePointerInfo::getConstantPool(DAG.getMachineFunction()));
Rafael Espindola0f898332016-06-20 16:43:17 +00002728 if (!IsPositionIndependent)
Bob Wilson1c66e8a2009-11-02 20:59:23 +00002729 return Result;
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +00002730 SDValue PICLabel = DAG.getConstant(ARMPCLabelIndex, DL, MVT::i32);
Bob Wilson1c66e8a2009-11-02 20:59:23 +00002731 return DAG.getNode(ARMISD::PIC_ADD, DL, PtrVT, Result, PICLabel);
Bob Wilson1cf0b032009-10-30 05:45:42 +00002732}
2733
Tim Northoverbd41cf82016-01-07 09:03:03 +00002734/// \brief Convert a TLS address reference into the correct sequence of loads
2735/// and calls to compute the variable's address for Darwin, and return an
2736/// SDValue containing the final node.
2737
2738/// Darwin only has one TLS scheme which must be capable of dealing with the
2739/// fully general situation, in the worst case. This means:
2740/// + "extern __thread" declaration.
2741/// + Defined in a possibly unknown dynamic library.
2742///
2743/// The general system is that each __thread variable has a [3 x i32] descriptor
2744/// which contains information used by the runtime to calculate the address. The
2745/// only part of this the compiler needs to know about is the first word, which
2746/// contains a function pointer that must be called with the address of the
2747/// entire descriptor in "r0".
2748///
2749/// Since this descriptor may be in a different unit, in general access must
2750/// proceed along the usual ARM rules. A common sequence to produce is:
2751///
2752/// movw rT1, :lower16:_var$non_lazy_ptr
2753/// movt rT1, :upper16:_var$non_lazy_ptr
2754/// ldr r0, [rT1]
2755/// ldr rT2, [r0]
2756/// blx rT2
2757/// [...address now in r0...]
2758SDValue
2759ARMTargetLowering::LowerGlobalTLSAddressDarwin(SDValue Op,
2760 SelectionDAG &DAG) const {
2761 assert(Subtarget->isTargetDarwin() && "TLS only supported on Darwin");
2762 SDLoc DL(Op);
2763
2764 // First step is to get the address of the actua global symbol. This is where
2765 // the TLS descriptor lives.
2766 SDValue DescAddr = LowerGlobalAddressDarwin(Op, DAG);
2767
2768 // The first entry in the descriptor is a function pointer that we must call
2769 // to obtain the address of the variable.
2770 SDValue Chain = DAG.getEntryNode();
Justin Lebaradbf09e2016-09-11 01:38:58 +00002771 SDValue FuncTLVGet = DAG.getLoad(
2772 MVT::i32, DL, Chain, DescAddr,
2773 MachinePointerInfo::getGOT(DAG.getMachineFunction()),
2774 /* Alignment = */ 4,
2775 MachineMemOperand::MONonTemporal | MachineMemOperand::MODereferenceable |
2776 MachineMemOperand::MOInvariant);
Tim Northoverbd41cf82016-01-07 09:03:03 +00002777 Chain = FuncTLVGet.getValue(1);
2778
2779 MachineFunction &F = DAG.getMachineFunction();
Matthias Braun941a7052016-07-28 18:40:00 +00002780 MachineFrameInfo &MFI = F.getFrameInfo();
2781 MFI.setAdjustsStack(true);
Tim Northoverbd41cf82016-01-07 09:03:03 +00002782
2783 // TLS calls preserve all registers except those that absolutely must be
2784 // trashed: R0 (it takes an argument), LR (it's a call) and CPSR (let's not be
2785 // silly).
2786 auto TRI =
2787 getTargetMachine().getSubtargetImpl(*F.getFunction())->getRegisterInfo();
2788 auto ARI = static_cast<const ARMRegisterInfo *>(TRI);
2789 const uint32_t *Mask = ARI->getTLSCallPreservedMask(DAG.getMachineFunction());
2790
2791 // Finally, we can make the call. This is just a degenerate version of a
2792 // normal AArch64 call node: r0 takes the address of the descriptor, and
2793 // returns the address of the variable in this thread.
2794 Chain = DAG.getCopyToReg(Chain, DL, ARM::R0, DescAddr, SDValue());
2795 Chain =
2796 DAG.getNode(ARMISD::CALL, DL, DAG.getVTList(MVT::Other, MVT::Glue),
2797 Chain, FuncTLVGet, DAG.getRegister(ARM::R0, MVT::i32),
2798 DAG.getRegisterMask(Mask), Chain.getValue(1));
2799 return DAG.getCopyFromReg(Chain, DL, ARM::R0, MVT::i32, Chain.getValue(1));
2800}
2801
Saleem Abdulrasoolf36005a2016-02-03 18:21:59 +00002802SDValue
2803ARMTargetLowering::LowerGlobalTLSAddressWindows(SDValue Op,
2804 SelectionDAG &DAG) const {
2805 assert(Subtarget->isTargetWindows() && "Windows specific TLS lowering");
Saleem Abdulrasool532dcbc2016-06-07 03:15:07 +00002806
Saleem Abdulrasoolf36005a2016-02-03 18:21:59 +00002807 SDValue Chain = DAG.getEntryNode();
2808 EVT PtrVT = getPointerTy(DAG.getDataLayout());
2809 SDLoc DL(Op);
2810
2811 // Load the current TEB (thread environment block)
2812 SDValue Ops[] = {Chain,
2813 DAG.getConstant(Intrinsic::arm_mrc, DL, MVT::i32),
2814 DAG.getConstant(15, DL, MVT::i32),
2815 DAG.getConstant(0, DL, MVT::i32),
2816 DAG.getConstant(13, DL, MVT::i32),
2817 DAG.getConstant(0, DL, MVT::i32),
2818 DAG.getConstant(2, DL, MVT::i32)};
2819 SDValue CurrentTEB = DAG.getNode(ISD::INTRINSIC_W_CHAIN, DL,
2820 DAG.getVTList(MVT::i32, MVT::Other), Ops);
2821
2822 SDValue TEB = CurrentTEB.getValue(0);
2823 Chain = CurrentTEB.getValue(1);
2824
2825 // Load the ThreadLocalStoragePointer from the TEB
2826 // A pointer to the TLS array is located at offset 0x2c from the TEB.
2827 SDValue TLSArray =
2828 DAG.getNode(ISD::ADD, DL, PtrVT, TEB, DAG.getIntPtrConstant(0x2c, DL));
Justin Lebar9c375812016-07-15 18:27:10 +00002829 TLSArray = DAG.getLoad(PtrVT, DL, Chain, TLSArray, MachinePointerInfo());
Saleem Abdulrasoolf36005a2016-02-03 18:21:59 +00002830
2831 // The pointer to the thread's TLS data area is at the TLS Index scaled by 4
2832 // offset into the TLSArray.
2833
2834 // Load the TLS index from the C runtime
2835 SDValue TLSIndex =
2836 DAG.getTargetExternalSymbol("_tls_index", PtrVT, ARMII::MO_NO_FLAG);
2837 TLSIndex = DAG.getNode(ARMISD::Wrapper, DL, PtrVT, TLSIndex);
Justin Lebar9c375812016-07-15 18:27:10 +00002838 TLSIndex = DAG.getLoad(PtrVT, DL, Chain, TLSIndex, MachinePointerInfo());
Saleem Abdulrasoolf36005a2016-02-03 18:21:59 +00002839
2840 SDValue Slot = DAG.getNode(ISD::SHL, DL, PtrVT, TLSIndex,
2841 DAG.getConstant(2, DL, MVT::i32));
2842 SDValue TLS = DAG.getLoad(PtrVT, DL, Chain,
2843 DAG.getNode(ISD::ADD, DL, PtrVT, TLSArray, Slot),
Justin Lebar9c375812016-07-15 18:27:10 +00002844 MachinePointerInfo());
Saleem Abdulrasoolf36005a2016-02-03 18:21:59 +00002845
Saleem Abdulrasool532dcbc2016-06-07 03:15:07 +00002846 // Get the offset of the start of the .tls section (section base)
2847 const auto *GA = cast<GlobalAddressSDNode>(Op);
2848 auto *CPV = ARMConstantPoolConstant::Create(GA->getGlobal(), ARMCP::SECREL);
Justin Lebar9c375812016-07-15 18:27:10 +00002849 SDValue Offset = DAG.getLoad(
2850 PtrVT, DL, Chain, DAG.getNode(ARMISD::Wrapper, DL, MVT::i32,
2851 DAG.getTargetConstantPool(CPV, PtrVT, 4)),
2852 MachinePointerInfo::getConstantPool(DAG.getMachineFunction()));
Saleem Abdulrasool532dcbc2016-06-07 03:15:07 +00002853
2854 return DAG.getNode(ISD::ADD, DL, PtrVT, TLS, Offset);
Saleem Abdulrasoolf36005a2016-02-03 18:21:59 +00002855}
2856
Lauro Ramos Venancioc39c12a2007-04-27 13:54:47 +00002857// Lower ISD::GlobalTLSAddress using the "general dynamic" model
Dan Gohman2ce6f2a2008-07-27 21:46:04 +00002858SDValue
Lauro Ramos Venancioc39c12a2007-04-27 13:54:47 +00002859ARMTargetLowering::LowerToTLSGeneralDynamicModel(GlobalAddressSDNode *GA,
Dan Gohman21cea8a2010-04-17 15:26:15 +00002860 SelectionDAG &DAG) const {
Andrew Trickef9de2a2013-05-25 02:42:55 +00002861 SDLoc dl(GA);
Mehdi Amini44ede332015-07-09 02:09:04 +00002862 EVT PtrVT = getPointerTy(DAG.getDataLayout());
Lauro Ramos Venancioc39c12a2007-04-27 13:54:47 +00002863 unsigned char PCAdj = Subtarget->isThumb() ? 4 : 8;
Evan Cheng408aa562009-11-06 22:24:13 +00002864 MachineFunction &MF = DAG.getMachineFunction();
2865 ARMFunctionInfo *AFI = MF.getInfo<ARMFunctionInfo>();
Evan Chengdfce83c2011-01-17 08:03:18 +00002866 unsigned ARMPCLabelIndex = AFI->createPICLabelUId();
Lauro Ramos Venancioc39c12a2007-04-27 13:54:47 +00002867 ARMConstantPoolValue *CPV =
Bill Wendling7753d662011-10-01 08:00:54 +00002868 ARMConstantPoolConstant::Create(GA->getGlobal(), ARMPCLabelIndex,
2869 ARMCP::CPValue, PCAdj, ARMCP::TLSGD, true);
Evan Cheng1fb8aed2009-03-13 07:51:59 +00002870 SDValue Argument = DAG.getTargetConstantPool(CPV, PtrVT, 4);
Owen Anderson9f944592009-08-11 20:47:22 +00002871 Argument = DAG.getNode(ARMISD::Wrapper, dl, MVT::i32, Argument);
Justin Lebar9c375812016-07-15 18:27:10 +00002872 Argument = DAG.getLoad(
2873 PtrVT, dl, DAG.getEntryNode(), Argument,
2874 MachinePointerInfo::getConstantPool(DAG.getMachineFunction()));
Dan Gohman2ce6f2a2008-07-27 21:46:04 +00002875 SDValue Chain = Argument.getValue(1);
Lauro Ramos Venancioc39c12a2007-04-27 13:54:47 +00002876
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +00002877 SDValue PICLabel = DAG.getConstant(ARMPCLabelIndex, dl, MVT::i32);
Dale Johannesen021052a2009-02-04 20:06:27 +00002878 Argument = DAG.getNode(ARMISD::PIC_ADD, dl, PtrVT, Argument, PICLabel);
Lauro Ramos Venancioc39c12a2007-04-27 13:54:47 +00002879
2880 // call __tls_get_addr.
2881 ArgListTy Args;
2882 ArgListEntry Entry;
2883 Entry.Node = Argument;
Chris Lattner229907c2011-07-18 04:54:35 +00002884 Entry.Ty = (Type *) Type::getInt32Ty(*DAG.getContext());
Lauro Ramos Venancioc39c12a2007-04-27 13:54:47 +00002885 Args.push_back(Entry);
Saleem Abdulrasoolf3a5a5c2014-05-17 21:50:17 +00002886
Dale Johannesen555a3752009-01-30 23:10:59 +00002887 // FIXME: is there useful debug info available here?
Saleem Abdulrasoolf3a5a5c2014-05-17 21:50:17 +00002888 TargetLowering::CallLoweringInfo CLI(DAG);
2889 CLI.setDebugLoc(dl).setChain(Chain)
2890 .setCallee(CallingConv::C, Type::getInt32Ty(*DAG.getContext()),
Krzysztof Parzyszeke116d5002016-06-22 12:54:25 +00002891 DAG.getExternalSymbol("__tls_get_addr", PtrVT), std::move(Args));
Saleem Abdulrasoolf3a5a5c2014-05-17 21:50:17 +00002892
Justin Holewinskiaa583972012-05-25 16:35:28 +00002893 std::pair<SDValue, SDValue> CallResult = LowerCallTo(CLI);
Lauro Ramos Venancioc39c12a2007-04-27 13:54:47 +00002894 return CallResult.first;
2895}
2896
2897// Lower ISD::GlobalTLSAddress using the "initial exec" or
2898// "local exec" model.
Dan Gohman2ce6f2a2008-07-27 21:46:04 +00002899SDValue
Lauro Ramos Venancioc39c12a2007-04-27 13:54:47 +00002900ARMTargetLowering::LowerToTLSExecModels(GlobalAddressSDNode *GA,
Hans Wennborgaea41202012-05-04 09:40:39 +00002901 SelectionDAG &DAG,
2902 TLSModel::Model model) const {
Dan Gohmanbcaf6812010-04-15 01:51:59 +00002903 const GlobalValue *GV = GA->getGlobal();
Andrew Trickef9de2a2013-05-25 02:42:55 +00002904 SDLoc dl(GA);
Dan Gohman2ce6f2a2008-07-27 21:46:04 +00002905 SDValue Offset;
2906 SDValue Chain = DAG.getEntryNode();
Mehdi Amini44ede332015-07-09 02:09:04 +00002907 EVT PtrVT = getPointerTy(DAG.getDataLayout());
Lauro Ramos Venancioc39c12a2007-04-27 13:54:47 +00002908 // Get the Thread Pointer
Dale Johannesen021052a2009-02-04 20:06:27 +00002909 SDValue ThreadPointer = DAG.getNode(ARMISD::THREAD_POINTER, dl, PtrVT);
Lauro Ramos Venancioc39c12a2007-04-27 13:54:47 +00002910
Hans Wennborgaea41202012-05-04 09:40:39 +00002911 if (model == TLSModel::InitialExec) {
Evan Cheng408aa562009-11-06 22:24:13 +00002912 MachineFunction &MF = DAG.getMachineFunction();
2913 ARMFunctionInfo *AFI = MF.getInfo<ARMFunctionInfo>();
Evan Chengdfce83c2011-01-17 08:03:18 +00002914 unsigned ARMPCLabelIndex = AFI->createPICLabelUId();
Evan Cheng408aa562009-11-06 22:24:13 +00002915 // Initial exec model.
Lauro Ramos Venancioc39c12a2007-04-27 13:54:47 +00002916 unsigned char PCAdj = Subtarget->isThumb() ? 4 : 8;
2917 ARMConstantPoolValue *CPV =
Bill Wendling7753d662011-10-01 08:00:54 +00002918 ARMConstantPoolConstant::Create(GA->getGlobal(), ARMPCLabelIndex,
2919 ARMCP::CPValue, PCAdj, ARMCP::GOTTPOFF,
2920 true);
Evan Cheng1fb8aed2009-03-13 07:51:59 +00002921 Offset = DAG.getTargetConstantPool(CPV, PtrVT, 4);
Owen Anderson9f944592009-08-11 20:47:22 +00002922 Offset = DAG.getNode(ARMISD::Wrapper, dl, MVT::i32, Offset);
Alex Lorenze40c8a22015-08-11 23:09:45 +00002923 Offset = DAG.getLoad(
2924 PtrVT, dl, Chain, Offset,
Justin Lebar9c375812016-07-15 18:27:10 +00002925 MachinePointerInfo::getConstantPool(DAG.getMachineFunction()));
Lauro Ramos Venancioc39c12a2007-04-27 13:54:47 +00002926 Chain = Offset.getValue(1);
2927
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +00002928 SDValue PICLabel = DAG.getConstant(ARMPCLabelIndex, dl, MVT::i32);
Dale Johannesen021052a2009-02-04 20:06:27 +00002929 Offset = DAG.getNode(ARMISD::PIC_ADD, dl, PtrVT, Offset, PICLabel);
Lauro Ramos Venancioc39c12a2007-04-27 13:54:47 +00002930
Alex Lorenze40c8a22015-08-11 23:09:45 +00002931 Offset = DAG.getLoad(
2932 PtrVT, dl, Chain, Offset,
Justin Lebar9c375812016-07-15 18:27:10 +00002933 MachinePointerInfo::getConstantPool(DAG.getMachineFunction()));
Lauro Ramos Venancioc39c12a2007-04-27 13:54:47 +00002934 } else {
2935 // local exec model
Hans Wennborgaea41202012-05-04 09:40:39 +00002936 assert(model == TLSModel::LocalExec);
Bill Wendling7753d662011-10-01 08:00:54 +00002937 ARMConstantPoolValue *CPV =
2938 ARMConstantPoolConstant::Create(GV, ARMCP::TPOFF);
Evan Cheng1fb8aed2009-03-13 07:51:59 +00002939 Offset = DAG.getTargetConstantPool(CPV, PtrVT, 4);
Owen Anderson9f944592009-08-11 20:47:22 +00002940 Offset = DAG.getNode(ARMISD::Wrapper, dl, MVT::i32, Offset);
Alex Lorenze40c8a22015-08-11 23:09:45 +00002941 Offset = DAG.getLoad(
2942 PtrVT, dl, Chain, Offset,
Justin Lebar9c375812016-07-15 18:27:10 +00002943 MachinePointerInfo::getConstantPool(DAG.getMachineFunction()));
Lauro Ramos Venancioc39c12a2007-04-27 13:54:47 +00002944 }
2945
2946 // The address of the thread local variable is the add of the thread
2947 // pointer with the offset of the variable.
Dale Johannesen021052a2009-02-04 20:06:27 +00002948 return DAG.getNode(ISD::ADD, dl, PtrVT, ThreadPointer, Offset);
Lauro Ramos Venancioc39c12a2007-04-27 13:54:47 +00002949}
2950
Dan Gohman2ce6f2a2008-07-27 21:46:04 +00002951SDValue
Dan Gohman21cea8a2010-04-17 15:26:15 +00002952ARMTargetLowering::LowerGlobalTLSAddress(SDValue Op, SelectionDAG &DAG) const {
Tim Northoverbd41cf82016-01-07 09:03:03 +00002953 if (Subtarget->isTargetDarwin())
2954 return LowerGlobalTLSAddressDarwin(Op, DAG);
2955
Saleem Abdulrasoolf36005a2016-02-03 18:21:59 +00002956 if (Subtarget->isTargetWindows())
2957 return LowerGlobalTLSAddressWindows(Op, DAG);
2958
Lauro Ramos Venancioc39c12a2007-04-27 13:54:47 +00002959 // TODO: implement the "local dynamic" model
Tim Northoverbd41cf82016-01-07 09:03:03 +00002960 assert(Subtarget->isTargetELF() && "Only ELF implemented here");
Lauro Ramos Venancioc39c12a2007-04-27 13:54:47 +00002961 GlobalAddressSDNode *GA = cast<GlobalAddressSDNode>(Op);
Chih-Hung Hsieh1e859582015-07-28 16:24:05 +00002962 if (DAG.getTarget().Options.EmulatedTLS)
2963 return LowerToTLSEmulatedModel(GA, DAG);
Hans Wennborgaea41202012-05-04 09:40:39 +00002964
2965 TLSModel::Model model = getTargetMachine().getTLSModel(GA->getGlobal());
2966
2967 switch (model) {
2968 case TLSModel::GeneralDynamic:
2969 case TLSModel::LocalDynamic:
2970 return LowerToTLSGeneralDynamicModel(GA, DAG);
2971 case TLSModel::InitialExec:
2972 case TLSModel::LocalExec:
2973 return LowerToTLSExecModels(GA, DAG, model);
2974 }
Matt Beaumont-Gaye82ab6b2012-05-04 18:34:27 +00002975 llvm_unreachable("bogus TLS model");
Lauro Ramos Venancioc39c12a2007-04-27 13:54:47 +00002976}
2977
James Molloy13065b02016-09-14 14:47:27 +00002978/// Return true if all users of V are within function F, looking through
2979/// ConstantExprs.
2980static bool allUsersAreInFunction(const Value *V, const Function *F) {
2981 SmallVector<const User*,4> Worklist;
2982 for (auto *U : V->users())
2983 Worklist.push_back(U);
2984 while (!Worklist.empty()) {
2985 auto *U = Worklist.pop_back_val();
2986 if (isa<ConstantExpr>(U)) {
2987 for (auto *UU : U->users())
2988 Worklist.push_back(UU);
2989 continue;
2990 }
2991
2992 auto *I = dyn_cast<Instruction>(U);
2993 if (!I || I->getParent()->getParent() != F)
2994 return false;
2995 }
2996 return true;
2997}
2998
2999/// Return true if all users of V are within some (any) function, looking through
3000/// ConstantExprs. In other words, are there any global constant users?
3001static bool allUsersAreInFunctions(const Value *V) {
3002 SmallVector<const User*,4> Worklist;
3003 for (auto *U : V->users())
3004 Worklist.push_back(U);
3005 while (!Worklist.empty()) {
3006 auto *U = Worklist.pop_back_val();
3007 if (isa<ConstantExpr>(U)) {
3008 for (auto *UU : U->users())
3009 Worklist.push_back(UU);
3010 continue;
3011 }
3012
3013 if (!isa<Instruction>(U))
3014 return false;
3015 }
3016 return true;
3017}
3018
3019static SDValue promoteToConstantPool(const GlobalValue *GV, SelectionDAG &DAG,
3020 EVT PtrVT, SDLoc dl) {
3021 // If we're creating a pool entry for a constant global with unnamed address,
3022 // and the global is small enough, we can emit it inline into the constant pool
3023 // to save ourselves an indirection.
3024 //
3025 // This is a win if the constant is only used in one function (so it doesn't
3026 // need to be duplicated) or duplicating the constant wouldn't increase code
3027 // size (implying the constant is no larger than 4 bytes).
3028 const Function *F = DAG.getMachineFunction().getFunction();
3029 auto *GVar = dyn_cast<GlobalVariable>(GV);
3030 if (EnableConstpoolPromotion && GVar && GVar->hasInitializer() &&
3031 GVar->isConstant() && GVar->hasGlobalUnnamedAddr() && GVar->hasLocalLinkage()) {
3032 // The constant islands pass can only really deal with alignment requests
3033 // <= 4 bytes and cannot pad constants itself. Therefore we cannot promote
3034 // any type wanting greater alignment requirements than 4 bytes. We also
3035 // can only promote constants that are multiples of 4 bytes in size or
3036 // are paddable to a multiple of 4. Currently we only try and pad constants
3037 // that are strings for simplicity.
3038 auto *Init = GVar->getInitializer();
3039 auto *CDAInit = dyn_cast<ConstantDataArray>(Init);
3040 unsigned Size = DAG.getDataLayout().getTypeAllocSize(Init->getType());
3041 unsigned Align = DAG.getDataLayout().getABITypeAlignment(Init->getType());
3042 unsigned RequiredPadding = 4 - (Size % 4);
3043 bool PaddingPossible =
3044 RequiredPadding == 4 || (CDAInit && CDAInit->isString());
3045
3046 if (PaddingPossible && Align <= 4 && Size <= ConstpoolPromotionMaxSize &&
3047 (allUsersAreInFunction(GVar, F) ||
3048 (Size <= 4 && allUsersAreInFunctions(GVar)))) {
3049 if (RequiredPadding != 4) {
3050 StringRef S = CDAInit->getAsString();
3051
3052 SmallVector<uint8_t,16> V(S.size());
3053 std::copy(S.bytes_begin(), S.bytes_end(), V.begin());
3054 while (RequiredPadding--)
3055 V.push_back(0);
3056 Init = ConstantDataArray::get(*DAG.getContext(), V);
3057 }
3058
3059 SDValue CPAddr =
3060 DAG.getTargetConstantPool(Init, PtrVT, Align);
3061
3062 MachineFunction &MF = DAG.getMachineFunction();
3063 ARMFunctionInfo *AFI = MF.getInfo<ARMFunctionInfo>();
3064 AFI->markGlobalAsPromotedToConstantPool(GVar);
3065 ++NumConstpoolPromoted;
3066 return DAG.getNode(ARMISD::Wrapper, dl, MVT::i32, CPAddr);
3067 }
3068 }
3069 return SDValue();
3070}
3071
Dan Gohman2ce6f2a2008-07-27 21:46:04 +00003072SDValue ARMTargetLowering::LowerGlobalAddressELF(SDValue Op,
Dan Gohman21cea8a2010-04-17 15:26:15 +00003073 SelectionDAG &DAG) const {
Mehdi Amini44ede332015-07-09 02:09:04 +00003074 EVT PtrVT = getPointerTy(DAG.getDataLayout());
Andrew Trickef9de2a2013-05-25 02:42:55 +00003075 SDLoc dl(Op);
Dan Gohmanbcaf6812010-04-15 01:51:59 +00003076 const GlobalValue *GV = cast<GlobalAddressSDNode>(Op)->getGlobal();
Rafael Espindola7ad97b22016-05-31 15:31:55 +00003077 const TargetMachine &TM = getTargetMachine();
Oliver Stannard8331aae2016-08-08 15:28:31 +00003078 if (const GlobalAlias *GA = dyn_cast<GlobalAlias>(GV))
3079 GV = GA->getBaseObject();
3080 bool IsRO =
3081 (isa<GlobalVariable>(GV) && cast<GlobalVariable>(GV)->isConstant()) ||
3082 isa<Function>(GV);
James Molloy13065b02016-09-14 14:47:27 +00003083
3084 if (TM.shouldAssumeDSOLocal(*GV->getParent(), GV))
3085 if (SDValue V = promoteToConstantPool(GV, DAG, PtrVT, dl))
3086 return V;
3087
Rafael Espindola0f898332016-06-20 16:43:17 +00003088 if (isPositionIndependent()) {
Rafael Espindola3beef8d2016-06-27 23:15:57 +00003089 bool UseGOT_PREL = !TM.shouldAssumeDSOLocal(*GV->getParent(), GV);
Peter Collingbourne97aae402015-10-26 18:23:16 +00003090
3091 MachineFunction &MF = DAG.getMachineFunction();
3092 ARMFunctionInfo *AFI = MF.getInfo<ARMFunctionInfo>();
3093 unsigned ARMPCLabelIndex = AFI->createPICLabelUId();
3094 EVT PtrVT = getPointerTy(DAG.getDataLayout());
3095 SDLoc dl(Op);
3096 unsigned PCAdj = Subtarget->isThumb() ? 4 : 8;
3097 ARMConstantPoolValue *CPV = ARMConstantPoolConstant::Create(
3098 GV, ARMPCLabelIndex, ARMCP::CPValue, PCAdj,
3099 UseGOT_PREL ? ARMCP::GOT_PREL : ARMCP::no_modifier,
3100 /*AddCurrentAddress=*/UseGOT_PREL);
Evan Cheng1fb8aed2009-03-13 07:51:59 +00003101 SDValue CPAddr = DAG.getTargetConstantPool(CPV, PtrVT, 4);
Owen Anderson9f944592009-08-11 20:47:22 +00003102 CPAddr = DAG.getNode(ARMISD::Wrapper, dl, MVT::i32, CPAddr);
Alex Lorenze40c8a22015-08-11 23:09:45 +00003103 SDValue Result = DAG.getLoad(
3104 PtrVT, dl, DAG.getEntryNode(), CPAddr,
Justin Lebar9c375812016-07-15 18:27:10 +00003105 MachinePointerInfo::getConstantPool(DAG.getMachineFunction()));
Dan Gohman2ce6f2a2008-07-27 21:46:04 +00003106 SDValue Chain = Result.getValue(1);
Peter Collingbourne97aae402015-10-26 18:23:16 +00003107 SDValue PICLabel = DAG.getConstant(ARMPCLabelIndex, dl, MVT::i32);
3108 Result = DAG.getNode(ARMISD::PIC_ADD, dl, PtrVT, Result, PICLabel);
3109 if (UseGOT_PREL)
Justin Lebar9c375812016-07-15 18:27:10 +00003110 Result =
3111 DAG.getLoad(PtrVT, dl, Chain, Result,
3112 MachinePointerInfo::getGOT(DAG.getMachineFunction()));
Lauro Ramos Venancioee2d1642007-04-22 00:04:12 +00003113 return Result;
Oliver Stannard8331aae2016-08-08 15:28:31 +00003114 } else if (Subtarget->isROPI() && IsRO) {
3115 // PC-relative.
3116 SDValue G = DAG.getTargetGlobalAddress(GV, dl, PtrVT);
3117 SDValue Result = DAG.getNode(ARMISD::WrapperPIC, dl, PtrVT, G);
3118 return Result;
3119 } else if (Subtarget->isRWPI() && !IsRO) {
3120 // SB-relative.
3121 ARMConstantPoolValue *CPV =
3122 ARMConstantPoolConstant::Create(GV, ARMCP::SBREL);
3123 SDValue CPAddr = DAG.getTargetConstantPool(CPV, PtrVT, 4);
3124 CPAddr = DAG.getNode(ARMISD::Wrapper, dl, MVT::i32, CPAddr);
3125 SDValue G = DAG.getLoad(
3126 PtrVT, dl, DAG.getEntryNode(), CPAddr,
3127 MachinePointerInfo::getConstantPool(DAG.getMachineFunction()));
3128 SDValue SB = DAG.getCopyFromReg(DAG.getEntryNode(), dl, ARM::R9, PtrVT);
3129 SDValue Result = DAG.getNode(ISD::ADD, dl, PtrVT, SB, G);
3130 return Result;
Evan Chengdfce83c2011-01-17 08:03:18 +00003131 }
3132
3133 // If we have T2 ops, we can materialize the address directly via movt/movw
James Molloydd9137a2011-10-26 08:53:19 +00003134 // pair. This is always cheaper.
Eric Christopherc1058df2014-07-04 01:55:26 +00003135 if (Subtarget->useMovt(DAG.getMachineFunction())) {
Evan Cheng68aec142011-01-19 02:16:49 +00003136 ++NumMovwMovt;
Evan Chengdfce83c2011-01-17 08:03:18 +00003137 // FIXME: Once remat is capable of dealing with instructions with register
3138 // operands, expand this into two nodes.
3139 return DAG.getNode(ARMISD::Wrapper, dl, PtrVT,
3140 DAG.getTargetGlobalAddress(GV, dl, PtrVT));
Lauro Ramos Venancioee2d1642007-04-22 00:04:12 +00003141 } else {
Evan Chengdfce83c2011-01-17 08:03:18 +00003142 SDValue CPAddr = DAG.getTargetConstantPool(GV, PtrVT, 4);
3143 CPAddr = DAG.getNode(ARMISD::Wrapper, dl, MVT::i32, CPAddr);
Alex Lorenze40c8a22015-08-11 23:09:45 +00003144 return DAG.getLoad(
3145 PtrVT, dl, DAG.getEntryNode(), CPAddr,
Justin Lebar9c375812016-07-15 18:27:10 +00003146 MachinePointerInfo::getConstantPool(DAG.getMachineFunction()));
Lauro Ramos Venancioee2d1642007-04-22 00:04:12 +00003147 }
3148}
3149
Dan Gohman2ce6f2a2008-07-27 21:46:04 +00003150SDValue ARMTargetLowering::LowerGlobalAddressDarwin(SDValue Op,
Dan Gohman21cea8a2010-04-17 15:26:15 +00003151 SelectionDAG &DAG) const {
Oliver Stannard8331aae2016-08-08 15:28:31 +00003152 assert(!Subtarget->isROPI() && !Subtarget->isRWPI() &&
3153 "ROPI/RWPI not currently supported for Darwin");
Mehdi Amini44ede332015-07-09 02:09:04 +00003154 EVT PtrVT = getPointerTy(DAG.getDataLayout());
Andrew Trickef9de2a2013-05-25 02:42:55 +00003155 SDLoc dl(Op);
Dan Gohmanbcaf6812010-04-15 01:51:59 +00003156 const GlobalValue *GV = cast<GlobalAddressSDNode>(Op)->getGlobal();
Evan Chengdfce83c2011-01-17 08:03:18 +00003157
Eric Christopherc1058df2014-07-04 01:55:26 +00003158 if (Subtarget->useMovt(DAG.getMachineFunction()))
Evan Cheng68aec142011-01-19 02:16:49 +00003159 ++NumMovwMovt;
Evan Chengdfce83c2011-01-17 08:03:18 +00003160
Tim Northover72360d22013-12-02 10:35:41 +00003161 // FIXME: Once remat is capable of dealing with instructions with register
3162 // operands, expand this into multiple nodes
3163 unsigned Wrapper =
Rafael Espindola0f898332016-06-20 16:43:17 +00003164 isPositionIndependent() ? ARMISD::WrapperPIC : ARMISD::Wrapper;
Tim Northoverdb962e2c2013-11-25 16:24:52 +00003165
Tim Northover72360d22013-12-02 10:35:41 +00003166 SDValue G = DAG.getTargetGlobalAddress(GV, dl, PtrVT, 0, ARMII::MO_NONLAZY);
3167 SDValue Result = DAG.getNode(Wrapper, dl, PtrVT, G);
Evan Cheng43b9ca62009-08-28 23:18:09 +00003168
Rafael Espindola5ac8f5c2016-06-28 15:38:13 +00003169 if (Subtarget->isGVIndirectSymbol(GV))
Tim Northover72360d22013-12-02 10:35:41 +00003170 Result = DAG.getLoad(PtrVT, dl, DAG.getEntryNode(), Result,
Justin Lebar9c375812016-07-15 18:27:10 +00003171 MachinePointerInfo::getGOT(DAG.getMachineFunction()));
Evan Cheng10043e22007-01-19 07:51:42 +00003172 return Result;
3173}
3174
Saleem Abdulrasool40bca0a2014-05-09 00:58:32 +00003175SDValue ARMTargetLowering::LowerGlobalAddressWindows(SDValue Op,
3176 SelectionDAG &DAG) const {
3177 assert(Subtarget->isTargetWindows() && "non-Windows COFF is not supported");
Eric Christopherc1058df2014-07-04 01:55:26 +00003178 assert(Subtarget->useMovt(DAG.getMachineFunction()) &&
3179 "Windows on ARM expects to use movw/movt");
Oliver Stannard8331aae2016-08-08 15:28:31 +00003180 assert(!Subtarget->isROPI() && !Subtarget->isRWPI() &&
3181 "ROPI/RWPI not currently supported for Windows");
Saleem Abdulrasool40bca0a2014-05-09 00:58:32 +00003182
3183 const GlobalValue *GV = cast<GlobalAddressSDNode>(Op)->getGlobal();
Reid Klecknerc35e7f52015-06-11 01:31:48 +00003184 const ARMII::TOF TargetFlags =
3185 (GV->hasDLLImportStorageClass() ? ARMII::MO_DLLIMPORT : ARMII::MO_NO_FLAG);
Mehdi Amini44ede332015-07-09 02:09:04 +00003186 EVT PtrVT = getPointerTy(DAG.getDataLayout());
Saleem Abdulrasool763f9a52014-07-07 05:18:35 +00003187 SDValue Result;
Saleem Abdulrasool40bca0a2014-05-09 00:58:32 +00003188 SDLoc DL(Op);
3189
3190 ++NumMovwMovt;
3191
3192 // FIXME: Once remat is capable of dealing with instructions with register
3193 // operands, expand this into two nodes.
Saleem Abdulrasool763f9a52014-07-07 05:18:35 +00003194 Result = DAG.getNode(ARMISD::Wrapper, DL, PtrVT,
3195 DAG.getTargetGlobalAddress(GV, DL, PtrVT, /*Offset=*/0,
Reid Klecknerc35e7f52015-06-11 01:31:48 +00003196 TargetFlags));
Saleem Abdulrasool763f9a52014-07-07 05:18:35 +00003197 if (GV->hasDLLImportStorageClass())
3198 Result = DAG.getLoad(PtrVT, DL, DAG.getEntryNode(), Result,
Justin Lebar9c375812016-07-15 18:27:10 +00003199 MachinePointerInfo::getGOT(DAG.getMachineFunction()));
Saleem Abdulrasool763f9a52014-07-07 05:18:35 +00003200 return Result;
Saleem Abdulrasool40bca0a2014-05-09 00:58:32 +00003201}
3202
Jim Grosbachaeca45d2009-05-12 23:59:14 +00003203SDValue
Jim Grosbachc98892f2010-05-26 20:22:18 +00003204ARMTargetLowering::LowerEH_SJLJ_SETJMP(SDValue Op, SelectionDAG &DAG) const {
Andrew Trickef9de2a2013-05-25 02:42:55 +00003205 SDLoc dl(Op);
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +00003206 SDValue Val = DAG.getConstant(0, dl, MVT::i32);
Bill Wendling7ecfbd92011-10-07 21:25:38 +00003207 return DAG.getNode(ARMISD::EH_SJLJ_SETJMP, dl,
3208 DAG.getVTList(MVT::i32, MVT::Other), Op.getOperand(0),
Jim Grosbachc98892f2010-05-26 20:22:18 +00003209 Op.getOperand(1), Val);
3210}
3211
3212SDValue
Jim Grosbachbd9485d2010-05-22 01:06:18 +00003213ARMTargetLowering::LowerEH_SJLJ_LONGJMP(SDValue Op, SelectionDAG &DAG) const {
Andrew Trickef9de2a2013-05-25 02:42:55 +00003214 SDLoc dl(Op);
Jim Grosbachbd9485d2010-05-22 01:06:18 +00003215 return DAG.getNode(ARMISD::EH_SJLJ_LONGJMP, dl, MVT::Other, Op.getOperand(0),
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +00003216 Op.getOperand(1), DAG.getConstant(0, dl, MVT::i32));
Jim Grosbachbd9485d2010-05-22 01:06:18 +00003217}
3218
Matthias Braun3cd00c12015-07-16 22:34:16 +00003219SDValue ARMTargetLowering::LowerEH_SJLJ_SETUP_DISPATCH(SDValue Op,
3220 SelectionDAG &DAG) const {
3221 SDLoc dl(Op);
3222 return DAG.getNode(ARMISD::EH_SJLJ_SETUP_DISPATCH, dl, MVT::Other,
3223 Op.getOperand(0));
3224}
3225
Jim Grosbachbd9485d2010-05-22 01:06:18 +00003226SDValue
Jim Grosbacha570d052010-02-08 23:22:00 +00003227ARMTargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op, SelectionDAG &DAG,
Jim Grosbache3864cc2010-06-16 23:45:49 +00003228 const ARMSubtarget *Subtarget) const {
Dan Gohmaneffb8942008-09-12 16:56:44 +00003229 unsigned IntNo = cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue();
Andrew Trickef9de2a2013-05-25 02:42:55 +00003230 SDLoc dl(Op);
Lauro Ramos Venanciof6a67bf2007-11-08 17:20:05 +00003231 switch (IntNo) {
Dan Gohman2ce6f2a2008-07-27 21:46:04 +00003232 default: return SDValue(); // Don't custom lower most intrinsics.
Jim Grosbach07393ba2014-06-16 21:55:30 +00003233 case Intrinsic::arm_rbit: {
Yi Kongc655f0c2014-08-20 10:40:20 +00003234 assert(Op.getOperand(1).getValueType() == MVT::i32 &&
Jim Grosbach07393ba2014-06-16 21:55:30 +00003235 "RBIT intrinsic must have i32 type!");
James Molloyb5640982015-11-13 16:05:22 +00003236 return DAG.getNode(ISD::BITREVERSE, dl, MVT::i32, Op.getOperand(1));
Jim Grosbach07393ba2014-06-16 21:55:30 +00003237 }
Marcin Koscielnicki3fdc2572016-04-19 20:51:05 +00003238 case Intrinsic::thread_pointer: {
Mehdi Amini44ede332015-07-09 02:09:04 +00003239 EVT PtrVT = getPointerTy(DAG.getDataLayout());
Bob Wilson17f88782009-08-04 00:25:01 +00003240 return DAG.getNode(ARMISD::THREAD_POINTER, dl, PtrVT);
3241 }
Jim Grosbach693e36a2009-08-11 00:09:57 +00003242 case Intrinsic::eh_sjlj_lsda: {
Jim Grosbach693e36a2009-08-11 00:09:57 +00003243 MachineFunction &MF = DAG.getMachineFunction();
Evan Cheng408aa562009-11-06 22:24:13 +00003244 ARMFunctionInfo *AFI = MF.getInfo<ARMFunctionInfo>();
Evan Chengdfce83c2011-01-17 08:03:18 +00003245 unsigned ARMPCLabelIndex = AFI->createPICLabelUId();
Mehdi Amini44ede332015-07-09 02:09:04 +00003246 EVT PtrVT = getPointerTy(DAG.getDataLayout());
Jim Grosbach693e36a2009-08-11 00:09:57 +00003247 SDValue CPAddr;
Rafael Espindola0f898332016-06-20 16:43:17 +00003248 bool IsPositionIndependent = isPositionIndependent();
3249 unsigned PCAdj = IsPositionIndependent ? (Subtarget->isThumb() ? 4 : 8) : 0;
Jim Grosbach693e36a2009-08-11 00:09:57 +00003250 ARMConstantPoolValue *CPV =
Bill Wendling7753d662011-10-01 08:00:54 +00003251 ARMConstantPoolConstant::Create(MF.getFunction(), ARMPCLabelIndex,
3252 ARMCP::CPLSDA, PCAdj);
Jim Grosbach693e36a2009-08-11 00:09:57 +00003253 CPAddr = DAG.getTargetConstantPool(CPV, PtrVT, 4);
Owen Anderson9f944592009-08-11 20:47:22 +00003254 CPAddr = DAG.getNode(ARMISD::Wrapper, dl, MVT::i32, CPAddr);
Alex Lorenze40c8a22015-08-11 23:09:45 +00003255 SDValue Result = DAG.getLoad(
3256 PtrVT, dl, DAG.getEntryNode(), CPAddr,
Justin Lebar9c375812016-07-15 18:27:10 +00003257 MachinePointerInfo::getConstantPool(DAG.getMachineFunction()));
Jim Grosbach693e36a2009-08-11 00:09:57 +00003258
Rafael Espindola0f898332016-06-20 16:43:17 +00003259 if (IsPositionIndependent) {
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +00003260 SDValue PICLabel = DAG.getConstant(ARMPCLabelIndex, dl, MVT::i32);
Jim Grosbach693e36a2009-08-11 00:09:57 +00003261 Result = DAG.getNode(ARMISD::PIC_ADD, dl, PtrVT, Result, PICLabel);
3262 }
3263 return Result;
3264 }
Evan Cheng18381b42011-03-29 23:06:19 +00003265 case Intrinsic::arm_neon_vmulls:
3266 case Intrinsic::arm_neon_vmullu: {
3267 unsigned NewOpc = (IntNo == Intrinsic::arm_neon_vmulls)
3268 ? ARMISD::VMULLs : ARMISD::VMULLu;
Andrew Trickef9de2a2013-05-25 02:42:55 +00003269 return DAG.getNode(NewOpc, SDLoc(Op), Op.getValueType(),
Evan Cheng18381b42011-03-29 23:06:19 +00003270 Op.getOperand(1), Op.getOperand(2));
3271 }
James Molloyee868b22015-08-11 12:06:25 +00003272 case Intrinsic::arm_neon_vminnm:
3273 case Intrinsic::arm_neon_vmaxnm: {
3274 unsigned NewOpc = (IntNo == Intrinsic::arm_neon_vminnm)
3275 ? ISD::FMINNUM : ISD::FMAXNUM;
3276 return DAG.getNode(NewOpc, SDLoc(Op), Op.getValueType(),
3277 Op.getOperand(1), Op.getOperand(2));
3278 }
Silviu Barangaad1b19f2015-08-19 14:11:27 +00003279 case Intrinsic::arm_neon_vminu:
3280 case Intrinsic::arm_neon_vmaxu: {
3281 if (Op.getValueType().isFloatingPoint())
3282 return SDValue();
3283 unsigned NewOpc = (IntNo == Intrinsic::arm_neon_vminu)
3284 ? ISD::UMIN : ISD::UMAX;
3285 return DAG.getNode(NewOpc, SDLoc(Op), Op.getValueType(),
3286 Op.getOperand(1), Op.getOperand(2));
3287 }
James Molloyd616c642015-08-11 12:06:28 +00003288 case Intrinsic::arm_neon_vmins:
3289 case Intrinsic::arm_neon_vmaxs: {
3290 // v{min,max}s is overloaded between signed integers and floats.
Silviu Barangaad1b19f2015-08-19 14:11:27 +00003291 if (!Op.getValueType().isFloatingPoint()) {
3292 unsigned NewOpc = (IntNo == Intrinsic::arm_neon_vmins)
3293 ? ISD::SMIN : ISD::SMAX;
3294 return DAG.getNode(NewOpc, SDLoc(Op), Op.getValueType(),
3295 Op.getOperand(1), Op.getOperand(2));
3296 }
James Molloyd616c642015-08-11 12:06:28 +00003297 unsigned NewOpc = (IntNo == Intrinsic::arm_neon_vmins)
3298 ? ISD::FMINNAN : ISD::FMAXNAN;
3299 return DAG.getNode(NewOpc, SDLoc(Op), Op.getValueType(),
3300 Op.getOperand(1), Op.getOperand(2));
3301 }
Lauro Ramos Venanciof6a67bf2007-11-08 17:20:05 +00003302 }
3303}
3304
Eli Friedman30a49e92011-08-03 21:06:02 +00003305static SDValue LowerATOMIC_FENCE(SDValue Op, SelectionDAG &DAG,
3306 const ARMSubtarget *Subtarget) {
3307 // FIXME: handle "fence singlethread" more efficiently.
Andrew Trickef9de2a2013-05-25 02:42:55 +00003308 SDLoc dl(Op);
Eli Friedman26a48482011-07-27 22:21:52 +00003309 if (!Subtarget->hasDataBarrier()) {
3310 // Some ARMv6 cpus can support data barriers with an mcr instruction.
3311 // Thumb1 and pre-v6 ARM mode use a libcall instead and should never get
3312 // here.
3313 assert(Subtarget->hasV6Ops() && !Subtarget->isThumb() &&
Tim Northoverc7ea8042013-10-25 09:30:24 +00003314 "Unexpected ISD::ATOMIC_FENCE encountered. Should be libcall!");
Eli Friedman30a49e92011-08-03 21:06:02 +00003315 return DAG.getNode(ARMISD::MEMBARRIER_MCR, dl, MVT::Other, Op.getOperand(0),
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +00003316 DAG.getConstant(0, dl, MVT::i32));
Eli Friedman26a48482011-07-27 22:21:52 +00003317 }
3318
Tim Northover36b24172013-07-03 09:20:36 +00003319 ConstantSDNode *OrdN = cast<ConstantSDNode>(Op.getOperand(1));
3320 AtomicOrdering Ord = static_cast<AtomicOrdering>(OrdN->getZExtValue());
Robin Morisseta47cb412014-09-03 21:01:03 +00003321 ARM_MB::MemBOpt Domain = ARM_MB::ISH;
Tim Northoverf5769882013-08-28 14:39:19 +00003322 if (Subtarget->isMClass()) {
3323 // Only a full system barrier exists in the M-class architectures.
3324 Domain = ARM_MB::SY;
Diana Picusc5baa432016-06-23 07:47:35 +00003325 } else if (Subtarget->preferISHSTBarriers() &&
3326 Ord == AtomicOrdering::Release) {
Tim Northover36b24172013-07-03 09:20:36 +00003327 // Swift happens to implement ISHST barriers in a way that's compatible with
3328 // Release semantics but weaker than ISH so we'd be fools not to use
3329 // it. Beware: other processors probably don't!
3330 Domain = ARM_MB::ISHST;
3331 }
3332
Joey Gouly926d3f52013-09-05 15:35:24 +00003333 return DAG.getNode(ISD::INTRINSIC_VOID, dl, MVT::Other, Op.getOperand(0),
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +00003334 DAG.getConstant(Intrinsic::arm_dmb, dl, MVT::i32),
3335 DAG.getConstant(Domain, dl, MVT::i32));
Eli Friedman26a48482011-07-27 22:21:52 +00003336}
3337
Evan Cheng8740ee32010-11-03 06:34:55 +00003338static SDValue LowerPREFETCH(SDValue Op, SelectionDAG &DAG,
3339 const ARMSubtarget *Subtarget) {
3340 // ARM pre v5TE and Thumb1 does not have preload instructions.
3341 if (!(Subtarget->isThumb2() ||
3342 (!Subtarget->isThumb1Only() && Subtarget->hasV5TEOps())))
3343 // Just preserve the chain.
3344 return Op.getOperand(0);
3345
Andrew Trickef9de2a2013-05-25 02:42:55 +00003346 SDLoc dl(Op);
Evan Cheng21acf9f2010-11-04 05:19:35 +00003347 unsigned isRead = ~cast<ConstantSDNode>(Op.getOperand(2))->getZExtValue() & 1;
3348 if (!isRead &&
3349 (!Subtarget->hasV7Ops() || !Subtarget->hasMPExtension()))
3350 // ARMv7 with MP extension has PLDW.
3351 return Op.getOperand(0);
Evan Cheng8740ee32010-11-03 06:34:55 +00003352
Bruno Cardoso Lopesdc9ff3a2011-06-14 04:58:37 +00003353 unsigned isData = cast<ConstantSDNode>(Op.getOperand(4))->getZExtValue();
3354 if (Subtarget->isThumb()) {
Evan Cheng8740ee32010-11-03 06:34:55 +00003355 // Invert the bits.
Evan Cheng21acf9f2010-11-04 05:19:35 +00003356 isRead = ~isRead & 1;
Bruno Cardoso Lopesdc9ff3a2011-06-14 04:58:37 +00003357 isData = ~isData & 1;
3358 }
Evan Cheng8740ee32010-11-03 06:34:55 +00003359
3360 return DAG.getNode(ARMISD::PRELOAD, dl, MVT::Other, Op.getOperand(0),
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +00003361 Op.getOperand(1), DAG.getConstant(isRead, dl, MVT::i32),
3362 DAG.getConstant(isData, dl, MVT::i32));
Evan Cheng8740ee32010-11-03 06:34:55 +00003363}
3364
Dan Gohman31ae5862010-04-17 14:41:14 +00003365static SDValue LowerVASTART(SDValue Op, SelectionDAG &DAG) {
3366 MachineFunction &MF = DAG.getMachineFunction();
3367 ARMFunctionInfo *FuncInfo = MF.getInfo<ARMFunctionInfo>();
3368
Evan Cheng10043e22007-01-19 07:51:42 +00003369 // vastart just stores the address of the VarArgsFrameIndex slot into the
3370 // memory location argument.
Andrew Trickef9de2a2013-05-25 02:42:55 +00003371 SDLoc dl(Op);
Mehdi Amini44ede332015-07-09 02:09:04 +00003372 EVT PtrVT = DAG.getTargetLoweringInfo().getPointerTy(DAG.getDataLayout());
Dan Gohman31ae5862010-04-17 14:41:14 +00003373 SDValue FR = DAG.getFrameIndex(FuncInfo->getVarArgsFrameIndex(), PtrVT);
Dan Gohman2d489b52008-02-06 22:27:42 +00003374 const Value *SV = cast<SrcValueSDNode>(Op.getOperand(2))->getValue();
Chris Lattner886250c2010-09-21 18:51:21 +00003375 return DAG.getStore(Op.getOperand(0), dl, FR, Op.getOperand(1),
Justin Lebar9c375812016-07-15 18:27:10 +00003376 MachinePointerInfo(SV));
Evan Cheng10043e22007-01-19 07:51:42 +00003377}
3378
Benjamin Kramerbdc49562016-06-12 15:39:02 +00003379SDValue ARMTargetLowering::GetF64FormalArgument(CCValAssign &VA,
3380 CCValAssign &NextVA,
3381 SDValue &Root,
3382 SelectionDAG &DAG,
3383 const SDLoc &dl) const {
Bob Wilson2e076c42009-06-22 23:27:02 +00003384 MachineFunction &MF = DAG.getMachineFunction();
3385 ARMFunctionInfo *AFI = MF.getInfo<ARMFunctionInfo>();
3386
Craig Topper760b1342012-02-22 05:59:10 +00003387 const TargetRegisterClass *RC;
David Goodwin22c2fba2009-07-08 23:10:31 +00003388 if (AFI->isThumb1OnlyFunction())
Craig Topperc7242e02012-04-20 07:30:17 +00003389 RC = &ARM::tGPRRegClass;
Bob Wilson2e076c42009-06-22 23:27:02 +00003390 else
Craig Topperc7242e02012-04-20 07:30:17 +00003391 RC = &ARM::GPRRegClass;
Bob Wilson2e076c42009-06-22 23:27:02 +00003392
3393 // Transform the arguments stored in physical registers into virtual ones.
Devang Patelf3292b22011-02-21 23:21:26 +00003394 unsigned Reg = MF.addLiveIn(VA.getLocReg(), RC);
Owen Anderson9f944592009-08-11 20:47:22 +00003395 SDValue ArgValue = DAG.getCopyFromReg(Root, dl, Reg, MVT::i32);
Bob Wilson2e076c42009-06-22 23:27:02 +00003396
3397 SDValue ArgValue2;
3398 if (NextVA.isMemLoc()) {
Matthias Braun941a7052016-07-28 18:40:00 +00003399 MachineFrameInfo &MFI = MF.getFrameInfo();
3400 int FI = MFI.CreateFixedObject(4, NextVA.getLocMemOffset(), true);
Bob Wilson2e076c42009-06-22 23:27:02 +00003401
3402 // Create load node to retrieve arguments from the stack.
Mehdi Amini44ede332015-07-09 02:09:04 +00003403 SDValue FIN = DAG.getFrameIndex(FI, getPointerTy(DAG.getDataLayout()));
Alex Lorenze40c8a22015-08-11 23:09:45 +00003404 ArgValue2 = DAG.getLoad(
3405 MVT::i32, dl, Root, FIN,
Justin Lebar9c375812016-07-15 18:27:10 +00003406 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FI));
Bob Wilson2e076c42009-06-22 23:27:02 +00003407 } else {
Devang Patelf3292b22011-02-21 23:21:26 +00003408 Reg = MF.addLiveIn(NextVA.getLocReg(), RC);
Owen Anderson9f944592009-08-11 20:47:22 +00003409 ArgValue2 = DAG.getCopyFromReg(Root, dl, Reg, MVT::i32);
Bob Wilson2e076c42009-06-22 23:27:02 +00003410 }
Christian Pirkerb5728192014-05-08 14:06:24 +00003411 if (!Subtarget->isLittle())
3412 std::swap (ArgValue, ArgValue2);
Jim Grosbachd7cf55c2009-11-09 00:11:35 +00003413 return DAG.getNode(ARMISD::VMOVDRR, dl, MVT::f64, ArgValue, ArgValue2);
Bob Wilson2e076c42009-06-22 23:27:02 +00003414}
3415
Stuart Hastings45fe3c32011-04-20 16:47:52 +00003416// The remaining GPRs hold either the beginning of variable-argument
David Peixotto4299cf82013-02-13 00:36:35 +00003417// data, or the beginning of an aggregate passed by value (usually
Stuart Hastings45fe3c32011-04-20 16:47:52 +00003418// byval). Either way, we allocate stack slots adjacent to the data
3419// provided by our caller, and store the unallocated registers there.
3420// If this is a variadic function, the va_list pointer will begin with
3421// these values; otherwise, this reassembles a (byval) structure that
3422// was split between registers and memory.
Stepan Dyatkovskiyf5aa83d2013-04-30 07:19:58 +00003423// Return: The frame index registers were stored into.
Benjamin Kramerbdc49562016-06-12 15:39:02 +00003424int ARMTargetLowering::StoreByValRegs(CCState &CCInfo, SelectionDAG &DAG,
3425 const SDLoc &dl, SDValue &Chain,
3426 const Value *OrigArg,
3427 unsigned InRegsParamRecordIdx,
3428 int ArgOffset, unsigned ArgSize) const {
Stepan Dyatkovskiyf5aa83d2013-04-30 07:19:58 +00003429 // Currently, two use-cases possible:
Alp Tokerf907b892013-12-05 05:44:44 +00003430 // Case #1. Non-var-args function, and we meet first byval parameter.
Stepan Dyatkovskiyf5aa83d2013-04-30 07:19:58 +00003431 // Setup first unallocated register as first byval register;
3432 // eat all remained registers
3433 // (these two actions are performed by HandleByVal method).
3434 // Then, here, we initialize stack frame with
3435 // "store-reg" instructions.
3436 // Case #2. Var-args function, that doesn't contain byval parameters.
3437 // The same: eat all remained unallocated registers,
3438 // initialize stack frame.
3439
Stuart Hastings45fe3c32011-04-20 16:47:52 +00003440 MachineFunction &MF = DAG.getMachineFunction();
Matthias Braun941a7052016-07-28 18:40:00 +00003441 MachineFrameInfo &MFI = MF.getFrameInfo();
Stuart Hastings45fe3c32011-04-20 16:47:52 +00003442 ARMFunctionInfo *AFI = MF.getInfo<ARMFunctionInfo>();
Stepan Dyatkovskiy8c02c982013-05-05 07:48:36 +00003443 unsigned RBegin, REnd;
3444 if (InRegsParamRecordIdx < CCInfo.getInRegsParamsCount()) {
3445 CCInfo.getInRegsParamInfo(InRegsParamRecordIdx, RBegin, REnd);
Stepan Dyatkovskiy8c02c982013-05-05 07:48:36 +00003446 } else {
Tim Northover8cda34f2015-03-11 18:54:22 +00003447 unsigned RBeginIdx = CCInfo.getFirstUnallocated(GPRArgRegs);
Aaron Ballmanc579d662015-03-12 13:24:06 +00003448 RBegin = RBeginIdx == 4 ? (unsigned)ARM::R4 : GPRArgRegs[RBeginIdx];
Tim Northover8cda34f2015-03-11 18:54:22 +00003449 REnd = ARM::R4;
Stuart Hastings45fe3c32011-04-20 16:47:52 +00003450 }
3451
Tim Northover8cda34f2015-03-11 18:54:22 +00003452 if (REnd != RBegin)
3453 ArgOffset = -4 * (ARM::R4 - RBegin);
Stepan Dyatkovskiyf5aa83d2013-04-30 07:19:58 +00003454
Mehdi Amini44ede332015-07-09 02:09:04 +00003455 auto PtrVT = getPointerTy(DAG.getDataLayout());
Matthias Braun941a7052016-07-28 18:40:00 +00003456 int FrameIndex = MFI.CreateFixedObject(ArgSize, ArgOffset, false);
Mehdi Amini44ede332015-07-09 02:09:04 +00003457 SDValue FIN = DAG.getFrameIndex(FrameIndex, PtrVT);
Stepan Dyatkovskiyd0e34a22013-05-20 08:01:34 +00003458
Tim Northover8cda34f2015-03-11 18:54:22 +00003459 SmallVector<SDValue, 4> MemOps;
3460 const TargetRegisterClass *RC =
3461 AFI->isThumb1OnlyFunction() ? &ARM::tGPRRegClass : &ARM::GPRRegClass;
Stepan Dyatkovskiyd0e34a22013-05-20 08:01:34 +00003462
Tim Northover8cda34f2015-03-11 18:54:22 +00003463 for (unsigned Reg = RBegin, i = 0; Reg < REnd; ++Reg, ++i) {
3464 unsigned VReg = MF.addLiveIn(Reg, RC);
3465 SDValue Val = DAG.getCopyFromReg(Chain, dl, VReg, MVT::i32);
Justin Lebar9c375812016-07-15 18:27:10 +00003466 SDValue Store = DAG.getStore(Val.getValue(1), dl, Val, FIN,
3467 MachinePointerInfo(OrigArg, 4 * i));
Tim Northover8cda34f2015-03-11 18:54:22 +00003468 MemOps.push_back(Store);
Mehdi Amini44ede332015-07-09 02:09:04 +00003469 FIN = DAG.getNode(ISD::ADD, dl, PtrVT, FIN, DAG.getConstant(4, dl, PtrVT));
Oliver Stannardd55e1152014-03-05 15:25:27 +00003470 }
Tim Northover8cda34f2015-03-11 18:54:22 +00003471
3472 if (!MemOps.empty())
3473 Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, MemOps);
3474 return FrameIndex;
Stepan Dyatkovskiyf5aa83d2013-04-30 07:19:58 +00003475}
3476
3477// Setup stack frame, the va_list pointer will start from.
Benjamin Kramerbdc49562016-06-12 15:39:02 +00003478void ARMTargetLowering::VarArgStyleRegisters(CCState &CCInfo, SelectionDAG &DAG,
3479 const SDLoc &dl, SDValue &Chain,
3480 unsigned ArgOffset,
3481 unsigned TotalArgRegsSaveSize,
3482 bool ForceMutable) const {
Stepan Dyatkovskiyf5aa83d2013-04-30 07:19:58 +00003483 MachineFunction &MF = DAG.getMachineFunction();
3484 ARMFunctionInfo *AFI = MF.getInfo<ARMFunctionInfo>();
3485
3486 // Try to store any remaining integer argument regs
Nick Lewycky99800752016-06-28 01:45:05 +00003487 // to their spots on the stack so that they may be loaded by dereferencing
Stepan Dyatkovskiyf5aa83d2013-04-30 07:19:58 +00003488 // the result of va_next.
3489 // If there is no regs to be stored, just point address after last
3490 // argument passed via stack.
Tim Northover8cda34f2015-03-11 18:54:22 +00003491 int FrameIndex = StoreByValRegs(CCInfo, DAG, dl, Chain, nullptr,
3492 CCInfo.getInRegsParamsCount(),
3493 CCInfo.getNextStackOffset(), 4);
Stepan Dyatkovskiyf5aa83d2013-04-30 07:19:58 +00003494 AFI->setVarArgsFrameIndex(FrameIndex);
Stuart Hastings45fe3c32011-04-20 16:47:52 +00003495}
3496
Benjamin Kramerbdc49562016-06-12 15:39:02 +00003497SDValue ARMTargetLowering::LowerFormalArguments(
3498 SDValue Chain, CallingConv::ID CallConv, bool isVarArg,
3499 const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &dl,
3500 SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals) const {
Bob Wilsona4c22902009-04-17 19:07:39 +00003501 MachineFunction &MF = DAG.getMachineFunction();
Matthias Braun941a7052016-07-28 18:40:00 +00003502 MachineFrameInfo &MFI = MF.getFrameInfo();
Bob Wilsona4c22902009-04-17 19:07:39 +00003503
Bob Wilsona4c22902009-04-17 19:07:39 +00003504 ARMFunctionInfo *AFI = MF.getInfo<ARMFunctionInfo>();
3505
3506 // Assign locations to all of the incoming arguments.
3507 SmallVector<CCValAssign, 16> ArgLocs;
Eric Christopherb5217502014-08-06 18:45:26 +00003508 ARMCCState CCInfo(CallConv, isVarArg, DAG.getMachineFunction(), ArgLocs,
3509 *DAG.getContext(), Prologue);
Dan Gohmanf9bbcd12009-08-05 01:29:28 +00003510 CCInfo.AnalyzeFormalArguments(Ins,
Anton Korobeynikov22ef7512009-08-05 19:04:42 +00003511 CCAssignFnForNode(CallConv, /* Return*/ false,
3512 isVarArg));
Jim Grosbach54efea02013-03-02 20:16:15 +00003513
Bob Wilsona4c22902009-04-17 19:07:39 +00003514 SmallVector<SDValue, 16> ArgValues;
Stuart Hastings67c5c3e2011-02-28 17:17:53 +00003515 SDValue ArgValue;
Stepan Dyatkovskiyf13dbb82012-10-10 11:37:36 +00003516 Function::const_arg_iterator CurOrigArg = MF.getFunction()->arg_begin();
3517 unsigned CurArgIdx = 0;
Stepan Dyatkovskiy8c02c982013-05-05 07:48:36 +00003518
3519 // Initially ArgRegsSaveSize is zero.
3520 // Then we increase this value each time we meet byval parameter.
3521 // We also increase this value in case of varargs function.
3522 AFI->setArgRegsSaveSize(0);
3523
Oliver Stannardd55e1152014-03-05 15:25:27 +00003524 // Calculate the amount of stack space that we need to allocate to store
3525 // byval and variadic arguments that are passed in registers.
3526 // We need to know this before we allocate the first byval or variadic
3527 // argument, as they will be allocated a stack slot below the CFA (Canonical
3528 // Frame Address, the stack pointer at entry to the function).
Tim Northover8cda34f2015-03-11 18:54:22 +00003529 unsigned ArgRegBegin = ARM::R4;
Oliver Stannardd55e1152014-03-05 15:25:27 +00003530 for (unsigned i = 0, e = ArgLocs.size(); i != e; ++i) {
Tim Northover8cda34f2015-03-11 18:54:22 +00003531 if (CCInfo.getInRegsParamsProcessed() >= CCInfo.getInRegsParamsCount())
3532 break;
Oliver Stannardd55e1152014-03-05 15:25:27 +00003533
Tim Northover8cda34f2015-03-11 18:54:22 +00003534 CCValAssign &VA = ArgLocs[i];
3535 unsigned Index = VA.getValNo();
3536 ISD::ArgFlagsTy Flags = Ins[Index].Flags;
3537 if (!Flags.isByVal())
3538 continue;
3539
3540 assert(VA.isMemLoc() && "unexpected byval pointer in reg");
3541 unsigned RBegin, REnd;
3542 CCInfo.getInRegsParamInfo(CCInfo.getInRegsParamsProcessed(), RBegin, REnd);
3543 ArgRegBegin = std::min(ArgRegBegin, RBegin);
3544
3545 CCInfo.nextInRegsParam();
Oliver Stannardd55e1152014-03-05 15:25:27 +00003546 }
3547 CCInfo.rewindByValRegsInfo();
Tim Northover8cda34f2015-03-11 18:54:22 +00003548
3549 int lastInsIndex = -1;
Matthias Braun941a7052016-07-28 18:40:00 +00003550 if (isVarArg && MFI.hasVAStart()) {
Tim Northover8cda34f2015-03-11 18:54:22 +00003551 unsigned RegIdx = CCInfo.getFirstUnallocated(GPRArgRegs);
3552 if (RegIdx != array_lengthof(GPRArgRegs))
3553 ArgRegBegin = std::min(ArgRegBegin, (unsigned)GPRArgRegs[RegIdx]);
Oliver Stannardd55e1152014-03-05 15:25:27 +00003554 }
Tim Northover8cda34f2015-03-11 18:54:22 +00003555
3556 unsigned TotalArgRegsSaveSize = 4 * (ARM::R4 - ArgRegBegin);
3557 AFI->setArgRegsSaveSize(TotalArgRegsSaveSize);
Mehdi Amini44ede332015-07-09 02:09:04 +00003558 auto PtrVT = getPointerTy(DAG.getDataLayout());
Oliver Stannardd55e1152014-03-05 15:25:27 +00003559
Bob Wilsona4c22902009-04-17 19:07:39 +00003560 for (unsigned i = 0, e = ArgLocs.size(); i != e; ++i) {
3561 CCValAssign &VA = ArgLocs[i];
Andrew Trick05938a52015-02-16 18:10:47 +00003562 if (Ins[VA.getValNo()].isOrigArg()) {
3563 std::advance(CurOrigArg,
3564 Ins[VA.getValNo()].getOrigArgIndex() - CurArgIdx);
3565 CurArgIdx = Ins[VA.getValNo()].getOrigArgIndex();
3566 }
Bob Wilsonea09d4a2009-04-17 20:35:10 +00003567 // Arguments stored in registers.
Bob Wilsona4c22902009-04-17 19:07:39 +00003568 if (VA.isRegLoc()) {
Owen Anderson53aa7a92009-08-10 22:56:29 +00003569 EVT RegVT = VA.getLocVT();
Bob Wilsona4c22902009-04-17 19:07:39 +00003570
Bob Wilsona4c22902009-04-17 19:07:39 +00003571 if (VA.needsCustom()) {
Bob Wilson2e076c42009-06-22 23:27:02 +00003572 // f64 and vector types are split up into multiple registers or
3573 // combinations of registers and stack slots.
Owen Anderson9f944592009-08-11 20:47:22 +00003574 if (VA.getLocVT() == MVT::v2f64) {
Bob Wilson2e076c42009-06-22 23:27:02 +00003575 SDValue ArgValue1 = GetF64FormalArgument(VA, ArgLocs[++i],
Dan Gohmanf9bbcd12009-08-05 01:29:28 +00003576 Chain, DAG, dl);
Bob Wilson2e076c42009-06-22 23:27:02 +00003577 VA = ArgLocs[++i]; // skip ahead to next loc
Bob Wilson699bdf72010-04-13 22:03:22 +00003578 SDValue ArgValue2;
3579 if (VA.isMemLoc()) {
Matthias Braun941a7052016-07-28 18:40:00 +00003580 int FI = MFI.CreateFixedObject(8, VA.getLocMemOffset(), true);
Mehdi Amini44ede332015-07-09 02:09:04 +00003581 SDValue FIN = DAG.getFrameIndex(FI, PtrVT);
Justin Lebar9c375812016-07-15 18:27:10 +00003582 ArgValue2 = DAG.getLoad(MVT::f64, dl, Chain, FIN,
3583 MachinePointerInfo::getFixedStack(
3584 DAG.getMachineFunction(), FI));
Bob Wilson699bdf72010-04-13 22:03:22 +00003585 } else {
3586 ArgValue2 = GetF64FormalArgument(VA, ArgLocs[++i],
3587 Chain, DAG, dl);
3588 }
Owen Anderson9f944592009-08-11 20:47:22 +00003589 ArgValue = DAG.getNode(ISD::UNDEF, dl, MVT::v2f64);
3590 ArgValue = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, MVT::v2f64,
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +00003591 ArgValue, ArgValue1,
3592 DAG.getIntPtrConstant(0, dl));
Owen Anderson9f944592009-08-11 20:47:22 +00003593 ArgValue = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, MVT::v2f64,
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +00003594 ArgValue, ArgValue2,
3595 DAG.getIntPtrConstant(1, dl));
Bob Wilson2e076c42009-06-22 23:27:02 +00003596 } else
Dan Gohmanf9bbcd12009-08-05 01:29:28 +00003597 ArgValue = GetF64FormalArgument(VA, ArgLocs[++i], Chain, DAG, dl);
Bob Wilsona4c22902009-04-17 19:07:39 +00003598
Bob Wilson2e076c42009-06-22 23:27:02 +00003599 } else {
Craig Topper760b1342012-02-22 05:59:10 +00003600 const TargetRegisterClass *RC;
Anton Korobeynikov22ef7512009-08-05 19:04:42 +00003601
Owen Anderson9f944592009-08-11 20:47:22 +00003602 if (RegVT == MVT::f32)
Craig Topperc7242e02012-04-20 07:30:17 +00003603 RC = &ARM::SPRRegClass;
Owen Anderson9f944592009-08-11 20:47:22 +00003604 else if (RegVT == MVT::f64)
Craig Topperc7242e02012-04-20 07:30:17 +00003605 RC = &ARM::DPRRegClass;
Owen Anderson9f944592009-08-11 20:47:22 +00003606 else if (RegVT == MVT::v2f64)
Craig Topperc7242e02012-04-20 07:30:17 +00003607 RC = &ARM::QPRRegClass;
Owen Anderson9f944592009-08-11 20:47:22 +00003608 else if (RegVT == MVT::i32)
Craig Topper61e88f42014-11-21 05:58:21 +00003609 RC = AFI->isThumb1OnlyFunction() ? &ARM::tGPRRegClass
3610 : &ARM::GPRRegClass;
Bob Wilson2e076c42009-06-22 23:27:02 +00003611 else
Anton Korobeynikovef98dbe2009-08-05 20:15:19 +00003612 llvm_unreachable("RegVT not supported by FORMAL_ARGUMENTS Lowering");
Bob Wilson2e076c42009-06-22 23:27:02 +00003613
3614 // Transform the arguments in physical registers into virtual ones.
Devang Patelf3292b22011-02-21 23:21:26 +00003615 unsigned Reg = MF.addLiveIn(VA.getLocReg(), RC);
Dan Gohmanf9bbcd12009-08-05 01:29:28 +00003616 ArgValue = DAG.getCopyFromReg(Chain, dl, Reg, RegVT);
Bob Wilsona4c22902009-04-17 19:07:39 +00003617 }
3618
3619 // If this is an 8 or 16-bit value, it is really passed promoted
3620 // to 32 bits. Insert an assert[sz]ext to capture this, then
3621 // truncate to the right size.
3622 switch (VA.getLocInfo()) {
Torok Edwinfbcc6632009-07-14 16:55:14 +00003623 default: llvm_unreachable("Unknown loc info!");
Bob Wilsona4c22902009-04-17 19:07:39 +00003624 case CCValAssign::Full: break;
3625 case CCValAssign::BCvt:
Wesley Peck527da1b2010-11-23 03:31:01 +00003626 ArgValue = DAG.getNode(ISD::BITCAST, dl, VA.getValVT(), ArgValue);
Bob Wilsona4c22902009-04-17 19:07:39 +00003627 break;
3628 case CCValAssign::SExt:
3629 ArgValue = DAG.getNode(ISD::AssertSext, dl, RegVT, ArgValue,
3630 DAG.getValueType(VA.getValVT()));
3631 ArgValue = DAG.getNode(ISD::TRUNCATE, dl, VA.getValVT(), ArgValue);
3632 break;
3633 case CCValAssign::ZExt:
3634 ArgValue = DAG.getNode(ISD::AssertZext, dl, RegVT, ArgValue,
3635 DAG.getValueType(VA.getValVT()));
3636 ArgValue = DAG.getNode(ISD::TRUNCATE, dl, VA.getValVT(), ArgValue);
3637 break;
3638 }
3639
Dan Gohmanf9bbcd12009-08-05 01:29:28 +00003640 InVals.push_back(ArgValue);
Bob Wilsona4c22902009-04-17 19:07:39 +00003641
3642 } else { // VA.isRegLoc()
3643
3644 // sanity check
3645 assert(VA.isMemLoc());
Owen Anderson9f944592009-08-11 20:47:22 +00003646 assert(VA.getValVT() != MVT::i64 && "i64 should already be lowered");
Bob Wilsona4c22902009-04-17 19:07:39 +00003647
Andrew Trick05938a52015-02-16 18:10:47 +00003648 int index = VA.getValNo();
Owen Anderson77aa2662011-04-05 21:48:57 +00003649
Stuart Hastings67c5c3e2011-02-28 17:17:53 +00003650 // Some Ins[] entries become multiple ArgLoc[] entries.
3651 // Process them only once.
3652 if (index != lastInsIndex)
3653 {
3654 ISD::ArgFlagsTy Flags = Ins[index].Flags;
Eric Christopher0713a9d2011-06-08 23:55:35 +00003655 // FIXME: For now, all byval parameter objects are marked mutable.
Eric Christophere02e07c2011-04-29 23:12:01 +00003656 // This can be changed with more analysis.
3657 // In case of tail call optimization mark all arguments mutable.
3658 // Since they could be overwritten by lowering of arguments in case of
3659 // a tail call.
Stuart Hastings67c5c3e2011-02-28 17:17:53 +00003660 if (Flags.isByVal()) {
Andrew Trick05938a52015-02-16 18:10:47 +00003661 assert(Ins[index].isOrigArg() &&
3662 "Byval arguments cannot be implicit");
Daniel Sanders8104b752014-11-01 19:32:23 +00003663 unsigned CurByValIndex = CCInfo.getInRegsParamsProcessed();
Oliver Stannardd55e1152014-03-05 15:25:27 +00003664
Duncan P. N. Exon Smith9f9559e2015-10-19 23:25:57 +00003665 int FrameIndex = StoreByValRegs(
3666 CCInfo, DAG, dl, Chain, &*CurOrigArg, CurByValIndex,
3667 VA.getLocMemOffset(), Flags.getByValSize());
Mehdi Amini44ede332015-07-09 02:09:04 +00003668 InVals.push_back(DAG.getFrameIndex(FrameIndex, PtrVT));
Stepan Dyatkovskiy8c02c982013-05-05 07:48:36 +00003669 CCInfo.nextInRegsParam();
Stuart Hastings67c5c3e2011-02-28 17:17:53 +00003670 } else {
Oliver Stannardd55e1152014-03-05 15:25:27 +00003671 unsigned FIOffset = VA.getLocMemOffset();
Matthias Braun941a7052016-07-28 18:40:00 +00003672 int FI = MFI.CreateFixedObject(VA.getLocVT().getSizeInBits()/8,
3673 FIOffset, true);
Bob Wilsona4c22902009-04-17 19:07:39 +00003674
Stuart Hastings67c5c3e2011-02-28 17:17:53 +00003675 // Create load nodes to retrieve arguments from the stack.
Mehdi Amini44ede332015-07-09 02:09:04 +00003676 SDValue FIN = DAG.getFrameIndex(FI, PtrVT);
Justin Lebar9c375812016-07-15 18:27:10 +00003677 InVals.push_back(DAG.getLoad(VA.getValVT(), dl, Chain, FIN,
3678 MachinePointerInfo::getFixedStack(
3679 DAG.getMachineFunction(), FI)));
Stuart Hastings67c5c3e2011-02-28 17:17:53 +00003680 }
3681 lastInsIndex = index;
3682 }
Bob Wilsona4c22902009-04-17 19:07:39 +00003683 }
3684 }
3685
3686 // varargs
Matthias Braun941a7052016-07-28 18:40:00 +00003687 if (isVarArg && MFI.hasVAStart())
Stepan Dyatkovskiyf5aa83d2013-04-30 07:19:58 +00003688 VarArgStyleRegisters(CCInfo, DAG, dl, Chain,
Oliver Stannardd55e1152014-03-05 15:25:27 +00003689 CCInfo.getNextStackOffset(),
3690 TotalArgRegsSaveSize);
Evan Cheng10043e22007-01-19 07:51:42 +00003691
Oliver Stannardb14c6252014-04-02 16:10:33 +00003692 AFI->setArgumentStackSize(CCInfo.getNextStackOffset());
3693
Dan Gohmanf9bbcd12009-08-05 01:29:28 +00003694 return Chain;
Evan Cheng10043e22007-01-19 07:51:42 +00003695}
3696
3697/// isFloatingPointZero - Return true if this is +0.0.
Dan Gohman2ce6f2a2008-07-27 21:46:04 +00003698static bool isFloatingPointZero(SDValue Op) {
Evan Cheng10043e22007-01-19 07:51:42 +00003699 if (ConstantFPSDNode *CFP = dyn_cast<ConstantFPSDNode>(Op))
Dale Johannesen3cf889f2007-08-31 04:03:46 +00003700 return CFP->getValueAPF().isPosZero();
Gabor Greiff304a7a2008-08-28 21:40:38 +00003701 else if (ISD::isEXTLoad(Op.getNode()) || ISD::isNON_EXTLoad(Op.getNode())) {
Evan Cheng10043e22007-01-19 07:51:42 +00003702 // Maybe this has already been legalized into the constant pool?
3703 if (Op.getOperand(1).getOpcode() == ARMISD::Wrapper) {
Dan Gohman2ce6f2a2008-07-27 21:46:04 +00003704 SDValue WrapperOp = Op.getOperand(1).getOperand(0);
Evan Cheng10043e22007-01-19 07:51:42 +00003705 if (ConstantPoolSDNode *CP = dyn_cast<ConstantPoolSDNode>(WrapperOp))
Dan Gohmanbcaf6812010-04-15 01:51:59 +00003706 if (const ConstantFP *CFP = dyn_cast<ConstantFP>(CP->getConstVal()))
Dale Johannesen3cf889f2007-08-31 04:03:46 +00003707 return CFP->getValueAPF().isPosZero();
Evan Cheng10043e22007-01-19 07:51:42 +00003708 }
Renato Golin6fb9c2e2014-10-23 15:31:50 +00003709 } else if (Op->getOpcode() == ISD::BITCAST &&
3710 Op->getValueType(0) == MVT::f64) {
3711 // Handle (ISD::BITCAST (ARMISD::VMOVIMM (ISD::TargetConstant 0)) MVT::f64)
3712 // created by LowerConstantFP().
3713 SDValue BitcastOp = Op->getOperand(0);
Artyom Skrobov314ee042015-11-25 19:41:11 +00003714 if (BitcastOp->getOpcode() == ARMISD::VMOVIMM &&
3715 isNullConstant(BitcastOp->getOperand(0)))
3716 return true;
Evan Cheng10043e22007-01-19 07:51:42 +00003717 }
3718 return false;
3719}
3720
Evan Cheng10043e22007-01-19 07:51:42 +00003721/// Returns appropriate ARM CMP (cmp) and corresponding condition code for
3722/// the given operands.
Benjamin Kramerbdc49562016-06-12 15:39:02 +00003723SDValue ARMTargetLowering::getARMCmp(SDValue LHS, SDValue RHS, ISD::CondCode CC,
3724 SDValue &ARMcc, SelectionDAG &DAG,
3725 const SDLoc &dl) const {
Gabor Greiff304a7a2008-08-28 21:40:38 +00003726 if (ConstantSDNode *RHSC = dyn_cast<ConstantSDNode>(RHS.getNode())) {
Dan Gohmaneffb8942008-09-12 16:56:44 +00003727 unsigned C = RHSC->getZExtValue();
Evan Cheng15b80e42009-11-12 07:13:11 +00003728 if (!isLegalICmpImmediate(C)) {
Evan Cheng10043e22007-01-19 07:51:42 +00003729 // Constant does not fit, try adjusting it by one?
3730 switch (CC) {
3731 default: break;
3732 case ISD::SETLT:
Evan Cheng10043e22007-01-19 07:51:42 +00003733 case ISD::SETGE:
Daniel Dunbara54a1b02010-08-25 16:58:05 +00003734 if (C != 0x80000000 && isLegalICmpImmediate(C-1)) {
Evan Cheng48b094d2007-02-02 01:53:26 +00003735 CC = (CC == ISD::SETLT) ? ISD::SETLE : ISD::SETGT;
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +00003736 RHS = DAG.getConstant(C - 1, dl, MVT::i32);
Evan Cheng48b094d2007-02-02 01:53:26 +00003737 }
3738 break;
3739 case ISD::SETULT:
3740 case ISD::SETUGE:
Daniel Dunbara54a1b02010-08-25 16:58:05 +00003741 if (C != 0 && isLegalICmpImmediate(C-1)) {
Evan Cheng48b094d2007-02-02 01:53:26 +00003742 CC = (CC == ISD::SETULT) ? ISD::SETULE : ISD::SETUGT;
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +00003743 RHS = DAG.getConstant(C - 1, dl, MVT::i32);
Evan Cheng10043e22007-01-19 07:51:42 +00003744 }
3745 break;
3746 case ISD::SETLE:
Evan Cheng10043e22007-01-19 07:51:42 +00003747 case ISD::SETGT:
Daniel Dunbara54a1b02010-08-25 16:58:05 +00003748 if (C != 0x7fffffff && isLegalICmpImmediate(C+1)) {
Evan Cheng48b094d2007-02-02 01:53:26 +00003749 CC = (CC == ISD::SETLE) ? ISD::SETLT : ISD::SETGE;
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +00003750 RHS = DAG.getConstant(C + 1, dl, MVT::i32);
Evan Cheng48b094d2007-02-02 01:53:26 +00003751 }
3752 break;
3753 case ISD::SETULE:
3754 case ISD::SETUGT:
Daniel Dunbara54a1b02010-08-25 16:58:05 +00003755 if (C != 0xffffffff && isLegalICmpImmediate(C+1)) {
Evan Cheng48b094d2007-02-02 01:53:26 +00003756 CC = (CC == ISD::SETULE) ? ISD::SETULT : ISD::SETUGE;
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +00003757 RHS = DAG.getConstant(C + 1, dl, MVT::i32);
Evan Cheng10043e22007-01-19 07:51:42 +00003758 }
3759 break;
3760 }
3761 }
3762 }
3763
3764 ARMCC::CondCodes CondCode = IntCCToARMCC(CC);
Lauro Ramos Venancio6be85332007-04-02 01:30:03 +00003765 ARMISD::NodeType CompareType;
3766 switch (CondCode) {
3767 default:
3768 CompareType = ARMISD::CMP;
3769 break;
3770 case ARMCC::EQ:
3771 case ARMCC::NE:
David Goodwindbf11ba2009-06-29 15:33:01 +00003772 // Uses only Z Flag
3773 CompareType = ARMISD::CMPZ;
Lauro Ramos Venancio6be85332007-04-02 01:30:03 +00003774 break;
3775 }
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +00003776 ARMcc = DAG.getConstant(CondCode, dl, MVT::i32);
Chris Lattner3e5fbd72010-12-21 02:38:05 +00003777 return DAG.getNode(CompareType, dl, MVT::Glue, LHS, RHS);
Evan Cheng10043e22007-01-19 07:51:42 +00003778}
3779
3780/// Returns a appropriate VFP CMP (fcmp{s|d}+fmstat) for the given operands.
Benjamin Kramerbdc49562016-06-12 15:39:02 +00003781SDValue ARMTargetLowering::getVFPCmp(SDValue LHS, SDValue RHS,
3782 SelectionDAG &DAG, const SDLoc &dl) const {
Oliver Stannard51b1d462014-08-21 12:50:31 +00003783 assert(!Subtarget->isFPOnlySP() || RHS.getValueType() != MVT::f64);
Dan Gohman2ce6f2a2008-07-27 21:46:04 +00003784 SDValue Cmp;
Evan Cheng10043e22007-01-19 07:51:42 +00003785 if (!isFloatingPointZero(RHS))
Chris Lattner3e5fbd72010-12-21 02:38:05 +00003786 Cmp = DAG.getNode(ARMISD::CMPFP, dl, MVT::Glue, LHS, RHS);
Evan Cheng10043e22007-01-19 07:51:42 +00003787 else
Chris Lattner3e5fbd72010-12-21 02:38:05 +00003788 Cmp = DAG.getNode(ARMISD::CMPFPw0, dl, MVT::Glue, LHS);
3789 return DAG.getNode(ARMISD::FMSTAT, dl, MVT::Glue, Cmp);
Evan Cheng10043e22007-01-19 07:51:42 +00003790}
3791
Bob Wilson45acbd02011-03-08 01:17:20 +00003792/// duplicateCmp - Glue values can have only one use, so this function
3793/// duplicates a comparison node.
3794SDValue
3795ARMTargetLowering::duplicateCmp(SDValue Cmp, SelectionDAG &DAG) const {
3796 unsigned Opc = Cmp.getOpcode();
Andrew Trickef9de2a2013-05-25 02:42:55 +00003797 SDLoc DL(Cmp);
Bob Wilson45acbd02011-03-08 01:17:20 +00003798 if (Opc == ARMISD::CMP || Opc == ARMISD::CMPZ)
3799 return DAG.getNode(Opc, DL, MVT::Glue, Cmp.getOperand(0),Cmp.getOperand(1));
3800
3801 assert(Opc == ARMISD::FMSTAT && "unexpected comparison operation");
3802 Cmp = Cmp.getOperand(0);
3803 Opc = Cmp.getOpcode();
3804 if (Opc == ARMISD::CMPFP)
3805 Cmp = DAG.getNode(Opc, DL, MVT::Glue, Cmp.getOperand(0),Cmp.getOperand(1));
3806 else {
3807 assert(Opc == ARMISD::CMPFPw0 && "unexpected operand of FMSTAT");
3808 Cmp = DAG.getNode(Opc, DL, MVT::Glue, Cmp.getOperand(0));
3809 }
3810 return DAG.getNode(ARMISD::FMSTAT, DL, MVT::Glue, Cmp);
3811}
3812
Louis Gerbarg3342bf12014-05-09 17:02:49 +00003813std::pair<SDValue, SDValue>
3814ARMTargetLowering::getARMXALUOOp(SDValue Op, SelectionDAG &DAG,
3815 SDValue &ARMcc) const {
3816 assert(Op.getValueType() == MVT::i32 && "Unsupported value type");
3817
3818 SDValue Value, OverflowCmp;
3819 SDValue LHS = Op.getOperand(0);
3820 SDValue RHS = Op.getOperand(1);
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +00003821 SDLoc dl(Op);
Louis Gerbarg3342bf12014-05-09 17:02:49 +00003822
3823 // FIXME: We are currently always generating CMPs because we don't support
3824 // generating CMN through the backend. This is not as good as the natural
3825 // CMP case because it causes a register dependency and cannot be folded
3826 // later.
3827
3828 switch (Op.getOpcode()) {
3829 default:
3830 llvm_unreachable("Unknown overflow instruction!");
3831 case ISD::SADDO:
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +00003832 ARMcc = DAG.getConstant(ARMCC::VC, dl, MVT::i32);
3833 Value = DAG.getNode(ISD::ADD, dl, Op.getValueType(), LHS, RHS);
3834 OverflowCmp = DAG.getNode(ARMISD::CMP, dl, MVT::Glue, Value, LHS);
Louis Gerbarg3342bf12014-05-09 17:02:49 +00003835 break;
3836 case ISD::UADDO:
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +00003837 ARMcc = DAG.getConstant(ARMCC::HS, dl, MVT::i32);
3838 Value = DAG.getNode(ISD::ADD, dl, Op.getValueType(), LHS, RHS);
3839 OverflowCmp = DAG.getNode(ARMISD::CMP, dl, MVT::Glue, Value, LHS);
Louis Gerbarg3342bf12014-05-09 17:02:49 +00003840 break;
3841 case ISD::SSUBO:
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +00003842 ARMcc = DAG.getConstant(ARMCC::VC, dl, MVT::i32);
3843 Value = DAG.getNode(ISD::SUB, dl, Op.getValueType(), LHS, RHS);
3844 OverflowCmp = DAG.getNode(ARMISD::CMP, dl, MVT::Glue, LHS, RHS);
Louis Gerbarg3342bf12014-05-09 17:02:49 +00003845 break;
3846 case ISD::USUBO:
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +00003847 ARMcc = DAG.getConstant(ARMCC::HS, dl, MVT::i32);
3848 Value = DAG.getNode(ISD::SUB, dl, Op.getValueType(), LHS, RHS);
3849 OverflowCmp = DAG.getNode(ARMISD::CMP, dl, MVT::Glue, LHS, RHS);
Louis Gerbarg3342bf12014-05-09 17:02:49 +00003850 break;
3851 } // switch (...)
3852
3853 return std::make_pair(Value, OverflowCmp);
3854}
3855
3856
3857SDValue
3858ARMTargetLowering::LowerXALUO(SDValue Op, SelectionDAG &DAG) const {
3859 // Let legalize expand this if it isn't a legal type yet.
3860 if (!DAG.getTargetLoweringInfo().isTypeLegal(Op.getValueType()))
3861 return SDValue();
3862
3863 SDValue Value, OverflowCmp;
3864 SDValue ARMcc;
3865 std::tie(Value, OverflowCmp) = getARMXALUOOp(Op, DAG, ARMcc);
3866 SDValue CCR = DAG.getRegister(ARM::CPSR, MVT::i32);
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +00003867 SDLoc dl(Op);
Louis Gerbarg3342bf12014-05-09 17:02:49 +00003868 // We use 0 and 1 as false and true values.
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +00003869 SDValue TVal = DAG.getConstant(1, dl, MVT::i32);
3870 SDValue FVal = DAG.getConstant(0, dl, MVT::i32);
Louis Gerbarg3342bf12014-05-09 17:02:49 +00003871 EVT VT = Op.getValueType();
3872
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +00003873 SDValue Overflow = DAG.getNode(ARMISD::CMOV, dl, VT, TVal, FVal,
Louis Gerbarg3342bf12014-05-09 17:02:49 +00003874 ARMcc, CCR, OverflowCmp);
3875
3876 SDVTList VTs = DAG.getVTList(Op.getValueType(), MVT::i32);
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +00003877 return DAG.getNode(ISD::MERGE_VALUES, dl, VTs, Value, Overflow);
Louis Gerbarg3342bf12014-05-09 17:02:49 +00003878}
3879
3880
Bill Wendling6a981312010-08-11 08:43:16 +00003881SDValue ARMTargetLowering::LowerSELECT(SDValue Op, SelectionDAG &DAG) const {
3882 SDValue Cond = Op.getOperand(0);
3883 SDValue SelectTrue = Op.getOperand(1);
3884 SDValue SelectFalse = Op.getOperand(2);
Andrew Trickef9de2a2013-05-25 02:42:55 +00003885 SDLoc dl(Op);
Louis Gerbarg3342bf12014-05-09 17:02:49 +00003886 unsigned Opc = Cond.getOpcode();
3887
3888 if (Cond.getResNo() == 1 &&
3889 (Opc == ISD::SADDO || Opc == ISD::UADDO || Opc == ISD::SSUBO ||
3890 Opc == ISD::USUBO)) {
3891 if (!DAG.getTargetLoweringInfo().isTypeLegal(Cond->getValueType(0)))
3892 return SDValue();
3893
3894 SDValue Value, OverflowCmp;
3895 SDValue ARMcc;
3896 std::tie(Value, OverflowCmp) = getARMXALUOOp(Cond, DAG, ARMcc);
3897 SDValue CCR = DAG.getRegister(ARM::CPSR, MVT::i32);
3898 EVT VT = Op.getValueType();
3899
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +00003900 return getCMOV(dl, VT, SelectTrue, SelectFalse, ARMcc, CCR,
Oliver Stannard51b1d462014-08-21 12:50:31 +00003901 OverflowCmp, DAG);
Louis Gerbarg3342bf12014-05-09 17:02:49 +00003902 }
Bill Wendling6a981312010-08-11 08:43:16 +00003903
3904 // Convert:
3905 //
3906 // (select (cmov 1, 0, cond), t, f) -> (cmov t, f, cond)
3907 // (select (cmov 0, 1, cond), t, f) -> (cmov f, t, cond)
3908 //
3909 if (Cond.getOpcode() == ARMISD::CMOV && Cond.hasOneUse()) {
3910 const ConstantSDNode *CMOVTrue =
3911 dyn_cast<ConstantSDNode>(Cond.getOperand(0));
3912 const ConstantSDNode *CMOVFalse =
3913 dyn_cast<ConstantSDNode>(Cond.getOperand(1));
3914
3915 if (CMOVTrue && CMOVFalse) {
3916 unsigned CMOVTrueVal = CMOVTrue->getZExtValue();
3917 unsigned CMOVFalseVal = CMOVFalse->getZExtValue();
3918
3919 SDValue True;
3920 SDValue False;
3921 if (CMOVTrueVal == 1 && CMOVFalseVal == 0) {
3922 True = SelectTrue;
3923 False = SelectFalse;
3924 } else if (CMOVTrueVal == 0 && CMOVFalseVal == 1) {
3925 True = SelectFalse;
3926 False = SelectTrue;
3927 }
3928
3929 if (True.getNode() && False.getNode()) {
Evan Cheng522fbfe2011-05-18 18:59:17 +00003930 EVT VT = Op.getValueType();
Bill Wendling6a981312010-08-11 08:43:16 +00003931 SDValue ARMcc = Cond.getOperand(2);
3932 SDValue CCR = Cond.getOperand(3);
Bob Wilson45acbd02011-03-08 01:17:20 +00003933 SDValue Cmp = duplicateCmp(Cond.getOperand(4), DAG);
Evan Cheng522fbfe2011-05-18 18:59:17 +00003934 assert(True.getValueType() == VT);
Oliver Stannard51b1d462014-08-21 12:50:31 +00003935 return getCMOV(dl, VT, True, False, ARMcc, CCR, Cmp, DAG);
Bill Wendling6a981312010-08-11 08:43:16 +00003936 }
3937 }
3938 }
3939
Dan Gohmand4a77c42012-02-24 00:09:36 +00003940 // ARM's BooleanContents value is UndefinedBooleanContent. Mask out the
3941 // undefined bits before doing a full-word comparison with zero.
3942 Cond = DAG.getNode(ISD::AND, dl, Cond.getValueType(), Cond,
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +00003943 DAG.getConstant(1, dl, Cond.getValueType()));
Dan Gohmand4a77c42012-02-24 00:09:36 +00003944
Bill Wendling6a981312010-08-11 08:43:16 +00003945 return DAG.getSelectCC(dl, Cond,
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +00003946 DAG.getConstant(0, dl, Cond.getValueType()),
Bill Wendling6a981312010-08-11 08:43:16 +00003947 SelectTrue, SelectFalse, ISD::SETNE);
3948}
3949
Joey Gouly881eab52013-08-22 15:29:11 +00003950static void checkVSELConstraints(ISD::CondCode CC, ARMCC::CondCodes &CondCode,
3951 bool &swpCmpOps, bool &swpVselOps) {
3952 // Start by selecting the GE condition code for opcodes that return true for
3953 // 'equality'
3954 if (CC == ISD::SETUGE || CC == ISD::SETOGE || CC == ISD::SETOLE ||
3955 CC == ISD::SETULE)
3956 CondCode = ARMCC::GE;
3957
3958 // and GT for opcodes that return false for 'equality'.
3959 else if (CC == ISD::SETUGT || CC == ISD::SETOGT || CC == ISD::SETOLT ||
3960 CC == ISD::SETULT)
3961 CondCode = ARMCC::GT;
3962
3963 // Since we are constrained to GE/GT, if the opcode contains 'less', we need
3964 // to swap the compare operands.
3965 if (CC == ISD::SETOLE || CC == ISD::SETULE || CC == ISD::SETOLT ||
3966 CC == ISD::SETULT)
3967 swpCmpOps = true;
3968
3969 // Both GT and GE are ordered comparisons, and return false for 'unordered'.
3970 // If we have an unordered opcode, we need to swap the operands to the VSEL
3971 // instruction (effectively negating the condition).
3972 //
3973 // This also has the effect of swapping which one of 'less' or 'greater'
3974 // returns true, so we also swap the compare operands. It also switches
3975 // whether we return true for 'equality', so we compensate by picking the
3976 // opposite condition code to our original choice.
3977 if (CC == ISD::SETULE || CC == ISD::SETULT || CC == ISD::SETUGE ||
3978 CC == ISD::SETUGT) {
3979 swpCmpOps = !swpCmpOps;
3980 swpVselOps = !swpVselOps;
3981 CondCode = CondCode == ARMCC::GT ? ARMCC::GE : ARMCC::GT;
3982 }
3983
3984 // 'ordered' is 'anything but unordered', so use the VS condition code and
3985 // swap the VSEL operands.
3986 if (CC == ISD::SETO) {
3987 CondCode = ARMCC::VS;
3988 swpVselOps = true;
3989 }
3990
3991 // 'unordered or not equal' is 'anything but equal', so use the EQ condition
3992 // code and swap the VSEL operands.
3993 if (CC == ISD::SETUNE) {
3994 CondCode = ARMCC::EQ;
3995 swpVselOps = true;
3996 }
3997}
3998
Benjamin Kramerbdc49562016-06-12 15:39:02 +00003999SDValue ARMTargetLowering::getCMOV(const SDLoc &dl, EVT VT, SDValue FalseVal,
Oliver Stannard51b1d462014-08-21 12:50:31 +00004000 SDValue TrueVal, SDValue ARMcc, SDValue CCR,
4001 SDValue Cmp, SelectionDAG &DAG) const {
4002 if (Subtarget->isFPOnlySP() && VT == MVT::f64) {
4003 FalseVal = DAG.getNode(ARMISD::VMOVRRD, dl,
4004 DAG.getVTList(MVT::i32, MVT::i32), FalseVal);
4005 TrueVal = DAG.getNode(ARMISD::VMOVRRD, dl,
4006 DAG.getVTList(MVT::i32, MVT::i32), TrueVal);
4007
4008 SDValue TrueLow = TrueVal.getValue(0);
4009 SDValue TrueHigh = TrueVal.getValue(1);
4010 SDValue FalseLow = FalseVal.getValue(0);
4011 SDValue FalseHigh = FalseVal.getValue(1);
4012
4013 SDValue Low = DAG.getNode(ARMISD::CMOV, dl, MVT::i32, FalseLow, TrueLow,
4014 ARMcc, CCR, Cmp);
4015 SDValue High = DAG.getNode(ARMISD::CMOV, dl, MVT::i32, FalseHigh, TrueHigh,
4016 ARMcc, CCR, duplicateCmp(Cmp, DAG));
4017
4018 return DAG.getNode(ARMISD::VMOVDRR, dl, MVT::f64, Low, High);
4019 } else {
4020 return DAG.getNode(ARMISD::CMOV, dl, VT, FalseVal, TrueVal, ARMcc, CCR,
4021 Cmp);
4022 }
4023}
4024
Benjamin Kramer4d098922016-07-10 11:28:51 +00004025static bool isGTorGE(ISD::CondCode CC) {
4026 return CC == ISD::SETGT || CC == ISD::SETGE;
4027}
Pablo Barrio7a643462016-06-23 16:53:49 +00004028
Benjamin Kramer4d098922016-07-10 11:28:51 +00004029static bool isLTorLE(ISD::CondCode CC) {
4030 return CC == ISD::SETLT || CC == ISD::SETLE;
4031}
Pablo Barrio7a643462016-06-23 16:53:49 +00004032
4033// See if a conditional (LHS CC RHS ? TrueVal : FalseVal) is lower-saturating.
4034// All of these conditions (and their <= and >= counterparts) will do:
4035// x < k ? k : x
4036// x > k ? x : k
4037// k < x ? x : k
4038// k > x ? k : x
Benjamin Kramer4d098922016-07-10 11:28:51 +00004039static bool isLowerSaturate(const SDValue LHS, const SDValue RHS,
4040 const SDValue TrueVal, const SDValue FalseVal,
4041 const ISD::CondCode CC, const SDValue K) {
Pablo Barrio7a643462016-06-23 16:53:49 +00004042 return (isGTorGE(CC) &&
4043 ((K == LHS && K == TrueVal) || (K == RHS && K == FalseVal))) ||
4044 (isLTorLE(CC) &&
4045 ((K == RHS && K == TrueVal) || (K == LHS && K == FalseVal)));
4046}
4047
4048// Similar to isLowerSaturate(), but checks for upper-saturating conditions.
Benjamin Kramer4d098922016-07-10 11:28:51 +00004049static bool isUpperSaturate(const SDValue LHS, const SDValue RHS,
4050 const SDValue TrueVal, const SDValue FalseVal,
4051 const ISD::CondCode CC, const SDValue K) {
Pablo Barrio7a643462016-06-23 16:53:49 +00004052 return (isGTorGE(CC) &&
4053 ((K == RHS && K == TrueVal) || (K == LHS && K == FalseVal))) ||
4054 (isLTorLE(CC) &&
4055 ((K == LHS && K == TrueVal) || (K == RHS && K == FalseVal)));
4056}
4057
4058// Check if two chained conditionals could be converted into SSAT.
4059//
4060// SSAT can replace a set of two conditional selectors that bound a number to an
4061// interval of type [k, ~k] when k + 1 is a power of 2. Here are some examples:
4062//
4063// x < -k ? -k : (x > k ? k : x)
4064// x < -k ? -k : (x < k ? x : k)
4065// x > -k ? (x > k ? k : x) : -k
4066// x < k ? (x < -k ? -k : x) : k
4067// etc.
4068//
4069// It returns true if the conversion can be done, false otherwise.
4070// Additionally, the variable is returned in parameter V and the constant in K.
Benjamin Kramer4d098922016-07-10 11:28:51 +00004071static bool isSaturatingConditional(const SDValue &Op, SDValue &V,
4072 uint64_t &K) {
Pablo Barrio7a643462016-06-23 16:53:49 +00004073
4074 SDValue LHS1 = Op.getOperand(0);
4075 SDValue RHS1 = Op.getOperand(1);
4076 SDValue TrueVal1 = Op.getOperand(2);
4077 SDValue FalseVal1 = Op.getOperand(3);
4078 ISD::CondCode CC1 = cast<CondCodeSDNode>(Op.getOperand(4))->get();
4079
4080 const SDValue Op2 = isa<ConstantSDNode>(TrueVal1) ? FalseVal1 : TrueVal1;
4081 if (Op2.getOpcode() != ISD::SELECT_CC)
4082 return false;
4083
4084 SDValue LHS2 = Op2.getOperand(0);
4085 SDValue RHS2 = Op2.getOperand(1);
4086 SDValue TrueVal2 = Op2.getOperand(2);
4087 SDValue FalseVal2 = Op2.getOperand(3);
4088 ISD::CondCode CC2 = cast<CondCodeSDNode>(Op2.getOperand(4))->get();
4089
4090 // Find out which are the constants and which are the variables
4091 // in each conditional
4092 SDValue *K1 = isa<ConstantSDNode>(LHS1) ? &LHS1 : isa<ConstantSDNode>(RHS1)
4093 ? &RHS1
4094 : NULL;
4095 SDValue *K2 = isa<ConstantSDNode>(LHS2) ? &LHS2 : isa<ConstantSDNode>(RHS2)
4096 ? &RHS2
4097 : NULL;
4098 SDValue K2Tmp = isa<ConstantSDNode>(TrueVal2) ? TrueVal2 : FalseVal2;
4099 SDValue V1Tmp = (K1 && *K1 == LHS1) ? RHS1 : LHS1;
4100 SDValue V2Tmp = (K2 && *K2 == LHS2) ? RHS2 : LHS2;
4101 SDValue V2 = (K2Tmp == TrueVal2) ? FalseVal2 : TrueVal2;
4102
4103 // We must detect cases where the original operations worked with 16- or
4104 // 8-bit values. In such case, V2Tmp != V2 because the comparison operations
4105 // must work with sign-extended values but the select operations return
4106 // the original non-extended value.
4107 SDValue V2TmpReg = V2Tmp;
4108 if (V2Tmp->getOpcode() == ISD::SIGN_EXTEND_INREG)
4109 V2TmpReg = V2Tmp->getOperand(0);
4110
4111 // Check that the registers and the constants have the correct values
4112 // in both conditionals
4113 if (!K1 || !K2 || *K1 == Op2 || *K2 != K2Tmp || V1Tmp != V2Tmp ||
4114 V2TmpReg != V2)
4115 return false;
4116
4117 // Figure out which conditional is saturating the lower/upper bound.
4118 const SDValue *LowerCheckOp =
4119 isLowerSaturate(LHS1, RHS1, TrueVal1, FalseVal1, CC1, *K1)
4120 ? &Op
4121 : isLowerSaturate(LHS2, RHS2, TrueVal2, FalseVal2, CC2, *K2) ? &Op2
4122 : NULL;
4123 const SDValue *UpperCheckOp =
4124 isUpperSaturate(LHS1, RHS1, TrueVal1, FalseVal1, CC1, *K1)
4125 ? &Op
4126 : isUpperSaturate(LHS2, RHS2, TrueVal2, FalseVal2, CC2, *K2) ? &Op2
4127 : NULL;
4128
4129 if (!UpperCheckOp || !LowerCheckOp || LowerCheckOp == UpperCheckOp)
4130 return false;
4131
4132 // Check that the constant in the lower-bound check is
4133 // the opposite of the constant in the upper-bound check
4134 // in 1's complement.
4135 int64_t Val1 = cast<ConstantSDNode>(*K1)->getSExtValue();
4136 int64_t Val2 = cast<ConstantSDNode>(*K2)->getSExtValue();
4137 int64_t PosVal = std::max(Val1, Val2);
4138
4139 if (((Val1 > Val2 && UpperCheckOp == &Op) ||
4140 (Val1 < Val2 && UpperCheckOp == &Op2)) &&
4141 Val1 == ~Val2 && isPowerOf2_64(PosVal + 1)) {
4142
4143 V = V2;
4144 K = (uint64_t)PosVal; // At this point, PosVal is guaranteed to be positive
4145 return true;
4146 }
4147
4148 return false;
4149}
4150
Dan Gohman21cea8a2010-04-17 15:26:15 +00004151SDValue ARMTargetLowering::LowerSELECT_CC(SDValue Op, SelectionDAG &DAG) const {
Pablo Barrio7a643462016-06-23 16:53:49 +00004152
Owen Anderson53aa7a92009-08-10 22:56:29 +00004153 EVT VT = Op.getValueType();
Pablo Barrio7a643462016-06-23 16:53:49 +00004154 SDLoc dl(Op);
4155
4156 // Try to convert two saturating conditional selects into a single SSAT
4157 SDValue SatValue;
4158 uint64_t SatConstant;
Bernard Ogden849f7372016-08-02 10:04:03 +00004159 if (((!Subtarget->isThumb() && Subtarget->hasV6Ops()) || Subtarget->isThumb2()) &&
Renato Golin32b165f2016-07-25 22:25:25 +00004160 isSaturatingConditional(Op, SatValue, SatConstant))
Pablo Barrio7a643462016-06-23 16:53:49 +00004161 return DAG.getNode(ARMISD::SSAT, dl, VT, SatValue,
4162 DAG.getConstant(countTrailingOnes(SatConstant), dl, VT));
4163
Dan Gohman2ce6f2a2008-07-27 21:46:04 +00004164 SDValue LHS = Op.getOperand(0);
4165 SDValue RHS = Op.getOperand(1);
Evan Cheng10043e22007-01-19 07:51:42 +00004166 ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(4))->get();
Dan Gohman2ce6f2a2008-07-27 21:46:04 +00004167 SDValue TrueVal = Op.getOperand(2);
4168 SDValue FalseVal = Op.getOperand(3);
Evan Cheng10043e22007-01-19 07:51:42 +00004169
Oliver Stannard51b1d462014-08-21 12:50:31 +00004170 if (Subtarget->isFPOnlySP() && LHS.getValueType() == MVT::f64) {
4171 DAG.getTargetLoweringInfo().softenSetCCOperands(DAG, MVT::f64, LHS, RHS, CC,
4172 dl);
4173
4174 // If softenSetCCOperands only returned one value, we should compare it to
4175 // zero.
4176 if (!RHS.getNode()) {
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +00004177 RHS = DAG.getConstant(0, dl, LHS.getValueType());
Oliver Stannard51b1d462014-08-21 12:50:31 +00004178 CC = ISD::SETNE;
4179 }
4180 }
4181
Owen Anderson9f944592009-08-11 20:47:22 +00004182 if (LHS.getValueType() == MVT::i32) {
Joey Gouly881eab52013-08-22 15:29:11 +00004183 // Try to generate VSEL on ARMv8.
4184 // The VSEL instruction can't use all the usual ARM condition
4185 // codes: it only has two bits to select the condition code, so it's
4186 // constrained to use only GE, GT, VS and EQ.
4187 //
4188 // To implement all the various ISD::SETXXX opcodes, we sometimes need to
4189 // swap the operands of the previous compare instruction (effectively
4190 // inverting the compare condition, swapping 'less' and 'greater') and
4191 // sometimes need to swap the operands to the VSEL (which inverts the
4192 // condition in the sense of firing whenever the previous condition didn't)
Eric Christopher1889fdc2015-01-29 00:19:39 +00004193 if (Subtarget->hasFPARMv8() && (TrueVal.getValueType() == MVT::f32 ||
4194 TrueVal.getValueType() == MVT::f64)) {
Joey Gouly881eab52013-08-22 15:29:11 +00004195 ARMCC::CondCodes CondCode = IntCCToARMCC(CC);
4196 if (CondCode == ARMCC::LT || CondCode == ARMCC::LE ||
4197 CondCode == ARMCC::VC || CondCode == ARMCC::NE) {
Artyom Skrobov3f8eae92015-05-06 11:44:10 +00004198 CC = ISD::getSetCCInverse(CC, true);
Joey Gouly881eab52013-08-22 15:29:11 +00004199 std::swap(TrueVal, FalseVal);
4200 }
4201 }
4202
Evan Cheng0cc4ad92010-07-13 19:27:42 +00004203 SDValue ARMcc;
Owen Anderson9f944592009-08-11 20:47:22 +00004204 SDValue CCR = DAG.getRegister(ARM::CPSR, MVT::i32);
Evan Cheng0cc4ad92010-07-13 19:27:42 +00004205 SDValue Cmp = getARMCmp(LHS, RHS, CC, ARMcc, DAG, dl);
Oliver Stannard51b1d462014-08-21 12:50:31 +00004206 return getCMOV(dl, VT, FalseVal, TrueVal, ARMcc, CCR, Cmp, DAG);
Evan Cheng10043e22007-01-19 07:51:42 +00004207 }
4208
4209 ARMCC::CondCodes CondCode, CondCode2;
Bob Wilsona2e83332009-09-09 23:14:54 +00004210 FPCCToARMCC(CC, CondCode, CondCode2);
Evan Cheng10043e22007-01-19 07:51:42 +00004211
Scott Douglass7ad77922015-04-08 17:18:28 +00004212 // Try to generate VMAXNM/VMINNM on ARMv8.
Eric Christopher1889fdc2015-01-29 00:19:39 +00004213 if (Subtarget->hasFPARMv8() && (TrueVal.getValueType() == MVT::f32 ||
4214 TrueVal.getValueType() == MVT::f64)) {
Joey Gouly881eab52013-08-22 15:29:11 +00004215 bool swpCmpOps = false;
4216 bool swpVselOps = false;
4217 checkVSELConstraints(CC, CondCode, swpCmpOps, swpVselOps);
4218
4219 if (CondCode == ARMCC::GT || CondCode == ARMCC::GE ||
4220 CondCode == ARMCC::VS || CondCode == ARMCC::EQ) {
4221 if (swpCmpOps)
4222 std::swap(LHS, RHS);
4223 if (swpVselOps)
4224 std::swap(TrueVal, FalseVal);
4225 }
4226 }
4227
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +00004228 SDValue ARMcc = DAG.getConstant(CondCode, dl, MVT::i32);
Evan Cheng0cc4ad92010-07-13 19:27:42 +00004229 SDValue Cmp = getVFPCmp(LHS, RHS, DAG, dl);
Owen Anderson9f944592009-08-11 20:47:22 +00004230 SDValue CCR = DAG.getRegister(ARM::CPSR, MVT::i32);
Oliver Stannard51b1d462014-08-21 12:50:31 +00004231 SDValue Result = getCMOV(dl, VT, FalseVal, TrueVal, ARMcc, CCR, Cmp, DAG);
Evan Cheng10043e22007-01-19 07:51:42 +00004232 if (CondCode2 != ARMCC::AL) {
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +00004233 SDValue ARMcc2 = DAG.getConstant(CondCode2, dl, MVT::i32);
Evan Cheng10043e22007-01-19 07:51:42 +00004234 // FIXME: Needs another CMP because flag can have but one use.
Evan Cheng0cc4ad92010-07-13 19:27:42 +00004235 SDValue Cmp2 = getVFPCmp(LHS, RHS, DAG, dl);
Oliver Stannard51b1d462014-08-21 12:50:31 +00004236 Result = getCMOV(dl, VT, Result, TrueVal, ARMcc2, CCR, Cmp2, DAG);
Evan Cheng10043e22007-01-19 07:51:42 +00004237 }
4238 return Result;
4239}
4240
Evan Cheng0cc4ad92010-07-13 19:27:42 +00004241/// canChangeToInt - Given the fp compare operand, return true if it is suitable
4242/// to morph to an integer compare sequence.
4243static bool canChangeToInt(SDValue Op, bool &SeenZero,
4244 const ARMSubtarget *Subtarget) {
4245 SDNode *N = Op.getNode();
4246 if (!N->hasOneUse())
4247 // Otherwise it requires moving the value from fp to integer registers.
4248 return false;
4249 if (!N->getNumValues())
4250 return false;
4251 EVT VT = Op.getValueType();
4252 if (VT != MVT::f32 && !Subtarget->isFPBrccSlow())
4253 // f32 case is generally profitable. f64 case only makes sense when vcmpe +
4254 // vmrs are very slow, e.g. cortex-a8.
4255 return false;
4256
4257 if (isFloatingPointZero(Op)) {
4258 SeenZero = true;
4259 return true;
4260 }
4261 return ISD::isNormalLoad(N);
4262}
4263
4264static SDValue bitcastf32Toi32(SDValue Op, SelectionDAG &DAG) {
4265 if (isFloatingPointZero(Op))
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +00004266 return DAG.getConstant(0, SDLoc(Op), MVT::i32);
Evan Cheng0cc4ad92010-07-13 19:27:42 +00004267
4268 if (LoadSDNode *Ld = dyn_cast<LoadSDNode>(Op))
Justin Lebar9c375812016-07-15 18:27:10 +00004269 return DAG.getLoad(MVT::i32, SDLoc(Op), Ld->getChain(), Ld->getBasePtr(),
4270 Ld->getPointerInfo(), Ld->getAlignment(),
4271 Ld->getMemOperand()->getFlags());
Evan Cheng0cc4ad92010-07-13 19:27:42 +00004272
4273 llvm_unreachable("Unknown VFP cmp argument!");
4274}
4275
4276static void expandf64Toi32(SDValue Op, SelectionDAG &DAG,
4277 SDValue &RetVal1, SDValue &RetVal2) {
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +00004278 SDLoc dl(Op);
4279
Evan Cheng0cc4ad92010-07-13 19:27:42 +00004280 if (isFloatingPointZero(Op)) {
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +00004281 RetVal1 = DAG.getConstant(0, dl, MVT::i32);
4282 RetVal2 = DAG.getConstant(0, dl, MVT::i32);
Evan Cheng0cc4ad92010-07-13 19:27:42 +00004283 return;
4284 }
4285
4286 if (LoadSDNode *Ld = dyn_cast<LoadSDNode>(Op)) {
4287 SDValue Ptr = Ld->getBasePtr();
Justin Lebar9c375812016-07-15 18:27:10 +00004288 RetVal1 =
4289 DAG.getLoad(MVT::i32, dl, Ld->getChain(), Ptr, Ld->getPointerInfo(),
4290 Ld->getAlignment(), Ld->getMemOperand()->getFlags());
Evan Cheng0cc4ad92010-07-13 19:27:42 +00004291
4292 EVT PtrType = Ptr.getValueType();
4293 unsigned NewAlign = MinAlign(Ld->getAlignment(), 4);
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +00004294 SDValue NewPtr = DAG.getNode(ISD::ADD, dl,
4295 PtrType, Ptr, DAG.getConstant(4, dl, PtrType));
Justin Lebar9c375812016-07-15 18:27:10 +00004296 RetVal2 = DAG.getLoad(MVT::i32, dl, Ld->getChain(), NewPtr,
4297 Ld->getPointerInfo().getWithOffset(4), NewAlign,
4298 Ld->getMemOperand()->getFlags());
Evan Cheng0cc4ad92010-07-13 19:27:42 +00004299 return;
4300 }
4301
4302 llvm_unreachable("Unknown VFP cmp argument!");
4303}
4304
4305/// OptimizeVFPBrcond - With -enable-unsafe-fp-math, it's legal to optimize some
4306/// f32 and even f64 comparisons to integer ones.
4307SDValue
4308ARMTargetLowering::OptimizeVFPBrcond(SDValue Op, SelectionDAG &DAG) const {
4309 SDValue Chain = Op.getOperand(0);
Evan Cheng10043e22007-01-19 07:51:42 +00004310 ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(1))->get();
Evan Cheng0cc4ad92010-07-13 19:27:42 +00004311 SDValue LHS = Op.getOperand(2);
4312 SDValue RHS = Op.getOperand(3);
4313 SDValue Dest = Op.getOperand(4);
Andrew Trickef9de2a2013-05-25 02:42:55 +00004314 SDLoc dl(Op);
Evan Cheng0cc4ad92010-07-13 19:27:42 +00004315
Evan Chengd12af5d2012-03-01 23:27:13 +00004316 bool LHSSeenZero = false;
4317 bool LHSOk = canChangeToInt(LHS, LHSSeenZero, Subtarget);
4318 bool RHSSeenZero = false;
4319 bool RHSOk = canChangeToInt(RHS, RHSSeenZero, Subtarget);
4320 if (LHSOk && RHSOk && (LHSSeenZero || RHSSeenZero)) {
Bob Wilson70bd3632011-03-08 01:17:16 +00004321 // If unsafe fp math optimization is enabled and there are no other uses of
4322 // the CMP operands, and the condition code is EQ or NE, we can optimize it
Evan Cheng0cc4ad92010-07-13 19:27:42 +00004323 // to an integer comparison.
4324 if (CC == ISD::SETOEQ)
4325 CC = ISD::SETEQ;
4326 else if (CC == ISD::SETUNE)
4327 CC = ISD::SETNE;
4328
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +00004329 SDValue Mask = DAG.getConstant(0x7fffffff, dl, MVT::i32);
Evan Cheng0cc4ad92010-07-13 19:27:42 +00004330 SDValue ARMcc;
4331 if (LHS.getValueType() == MVT::f32) {
Evan Chengd12af5d2012-03-01 23:27:13 +00004332 LHS = DAG.getNode(ISD::AND, dl, MVT::i32,
4333 bitcastf32Toi32(LHS, DAG), Mask);
4334 RHS = DAG.getNode(ISD::AND, dl, MVT::i32,
4335 bitcastf32Toi32(RHS, DAG), Mask);
Evan Cheng0cc4ad92010-07-13 19:27:42 +00004336 SDValue Cmp = getARMCmp(LHS, RHS, CC, ARMcc, DAG, dl);
4337 SDValue CCR = DAG.getRegister(ARM::CPSR, MVT::i32);
4338 return DAG.getNode(ARMISD::BRCOND, dl, MVT::Other,
4339 Chain, Dest, ARMcc, CCR, Cmp);
4340 }
4341
4342 SDValue LHS1, LHS2;
4343 SDValue RHS1, RHS2;
4344 expandf64Toi32(LHS, DAG, LHS1, LHS2);
4345 expandf64Toi32(RHS, DAG, RHS1, RHS2);
Evan Chengd12af5d2012-03-01 23:27:13 +00004346 LHS2 = DAG.getNode(ISD::AND, dl, MVT::i32, LHS2, Mask);
4347 RHS2 = DAG.getNode(ISD::AND, dl, MVT::i32, RHS2, Mask);
Evan Cheng0cc4ad92010-07-13 19:27:42 +00004348 ARMCC::CondCodes CondCode = IntCCToARMCC(CC);
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +00004349 ARMcc = DAG.getConstant(CondCode, dl, MVT::i32);
Chris Lattner3e5fbd72010-12-21 02:38:05 +00004350 SDVTList VTList = DAG.getVTList(MVT::Other, MVT::Glue);
Evan Cheng0cc4ad92010-07-13 19:27:42 +00004351 SDValue Ops[] = { Chain, ARMcc, LHS1, LHS2, RHS1, RHS2, Dest };
Craig Topper48d114b2014-04-26 18:35:24 +00004352 return DAG.getNode(ARMISD::BCC_i64, dl, VTList, Ops);
Evan Cheng0cc4ad92010-07-13 19:27:42 +00004353 }
4354
4355 return SDValue();
4356}
4357
4358SDValue ARMTargetLowering::LowerBR_CC(SDValue Op, SelectionDAG &DAG) const {
4359 SDValue Chain = Op.getOperand(0);
4360 ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(1))->get();
4361 SDValue LHS = Op.getOperand(2);
4362 SDValue RHS = Op.getOperand(3);
4363 SDValue Dest = Op.getOperand(4);
Andrew Trickef9de2a2013-05-25 02:42:55 +00004364 SDLoc dl(Op);
Evan Cheng10043e22007-01-19 07:51:42 +00004365
Oliver Stannard51b1d462014-08-21 12:50:31 +00004366 if (Subtarget->isFPOnlySP() && LHS.getValueType() == MVT::f64) {
4367 DAG.getTargetLoweringInfo().softenSetCCOperands(DAG, MVT::f64, LHS, RHS, CC,
4368 dl);
4369
4370 // If softenSetCCOperands only returned one value, we should compare it to
4371 // zero.
4372 if (!RHS.getNode()) {
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +00004373 RHS = DAG.getConstant(0, dl, LHS.getValueType());
Oliver Stannard51b1d462014-08-21 12:50:31 +00004374 CC = ISD::SETNE;
4375 }
4376 }
4377
Owen Anderson9f944592009-08-11 20:47:22 +00004378 if (LHS.getValueType() == MVT::i32) {
Evan Cheng0cc4ad92010-07-13 19:27:42 +00004379 SDValue ARMcc;
4380 SDValue Cmp = getARMCmp(LHS, RHS, CC, ARMcc, DAG, dl);
Owen Anderson9f944592009-08-11 20:47:22 +00004381 SDValue CCR = DAG.getRegister(ARM::CPSR, MVT::i32);
Owen Anderson9f944592009-08-11 20:47:22 +00004382 return DAG.getNode(ARMISD::BRCOND, dl, MVT::Other,
Evan Cheng0cc4ad92010-07-13 19:27:42 +00004383 Chain, Dest, ARMcc, CCR, Cmp);
Evan Cheng10043e22007-01-19 07:51:42 +00004384 }
4385
Owen Anderson9f944592009-08-11 20:47:22 +00004386 assert(LHS.getValueType() == MVT::f32 || LHS.getValueType() == MVT::f64);
Evan Cheng0cc4ad92010-07-13 19:27:42 +00004387
Nick Lewycky50f02cb2011-12-02 22:16:29 +00004388 if (getTargetMachine().Options.UnsafeFPMath &&
Evan Cheng0cc4ad92010-07-13 19:27:42 +00004389 (CC == ISD::SETEQ || CC == ISD::SETOEQ ||
4390 CC == ISD::SETNE || CC == ISD::SETUNE)) {
Ahmed Bougachaf8dfb472016-02-09 22:54:12 +00004391 if (SDValue Result = OptimizeVFPBrcond(Op, DAG))
Evan Cheng0cc4ad92010-07-13 19:27:42 +00004392 return Result;
4393 }
4394
Evan Cheng10043e22007-01-19 07:51:42 +00004395 ARMCC::CondCodes CondCode, CondCode2;
Bob Wilsona2e83332009-09-09 23:14:54 +00004396 FPCCToARMCC(CC, CondCode, CondCode2);
Bob Wilson7117a912009-03-20 22:42:55 +00004397
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +00004398 SDValue ARMcc = DAG.getConstant(CondCode, dl, MVT::i32);
Evan Cheng0cc4ad92010-07-13 19:27:42 +00004399 SDValue Cmp = getVFPCmp(LHS, RHS, DAG, dl);
Owen Anderson9f944592009-08-11 20:47:22 +00004400 SDValue CCR = DAG.getRegister(ARM::CPSR, MVT::i32);
Chris Lattner3e5fbd72010-12-21 02:38:05 +00004401 SDVTList VTList = DAG.getVTList(MVT::Other, MVT::Glue);
Evan Cheng0cc4ad92010-07-13 19:27:42 +00004402 SDValue Ops[] = { Chain, Dest, ARMcc, CCR, Cmp };
Craig Topper48d114b2014-04-26 18:35:24 +00004403 SDValue Res = DAG.getNode(ARMISD::BRCOND, dl, VTList, Ops);
Evan Cheng10043e22007-01-19 07:51:42 +00004404 if (CondCode2 != ARMCC::AL) {
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +00004405 ARMcc = DAG.getConstant(CondCode2, dl, MVT::i32);
Evan Cheng0cc4ad92010-07-13 19:27:42 +00004406 SDValue Ops[] = { Res, Dest, ARMcc, CCR, Res.getValue(1) };
Craig Topper48d114b2014-04-26 18:35:24 +00004407 Res = DAG.getNode(ARMISD::BRCOND, dl, VTList, Ops);
Evan Cheng10043e22007-01-19 07:51:42 +00004408 }
4409 return Res;
4410}
4411
Dan Gohman21cea8a2010-04-17 15:26:15 +00004412SDValue ARMTargetLowering::LowerBR_JT(SDValue Op, SelectionDAG &DAG) const {
Dan Gohman2ce6f2a2008-07-27 21:46:04 +00004413 SDValue Chain = Op.getOperand(0);
4414 SDValue Table = Op.getOperand(1);
4415 SDValue Index = Op.getOperand(2);
Andrew Trickef9de2a2013-05-25 02:42:55 +00004416 SDLoc dl(Op);
Evan Cheng10043e22007-01-19 07:51:42 +00004417
Mehdi Amini44ede332015-07-09 02:09:04 +00004418 EVT PTy = getPointerTy(DAG.getDataLayout());
Evan Cheng10043e22007-01-19 07:51:42 +00004419 JumpTableSDNode *JT = cast<JumpTableSDNode>(Table);
Dan Gohman2ce6f2a2008-07-27 21:46:04 +00004420 SDValue JTI = DAG.getTargetJumpTable(JT->getIndex(), PTy);
Tim Northover4998a472015-05-13 20:28:38 +00004421 Table = DAG.getNode(ARMISD::WrapperJT, dl, MVT::i32, JTI);
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +00004422 Index = DAG.getNode(ISD::MUL, dl, PTy, Index, DAG.getConstant(4, dl, PTy));
Evan Chengc8bed032009-07-28 20:53:24 +00004423 SDValue Addr = DAG.getNode(ISD::ADD, dl, PTy, Index, Table);
Evan Chengf3a1fce2009-07-25 00:33:29 +00004424 if (Subtarget->isThumb2()) {
4425 // Thumb2 uses a two-level jump. That is, it jumps into the jump table
4426 // which does another jump to the destination. This also makes it easier
4427 // to translate it to TBB / TBH later.
4428 // FIXME: This might not work if the function is extremely large.
Owen Anderson9f944592009-08-11 20:47:22 +00004429 return DAG.getNode(ARMISD::BR2_JT, dl, MVT::Other, Chain,
Tim Northover4998a472015-05-13 20:28:38 +00004430 Addr, Op.getOperand(2), JTI);
Evan Chengf3a1fce2009-07-25 00:33:29 +00004431 }
Oliver Stannard8331aae2016-08-08 15:28:31 +00004432 if (isPositionIndependent() || Subtarget->isROPI()) {
Alex Lorenze40c8a22015-08-11 23:09:45 +00004433 Addr =
4434 DAG.getLoad((EVT)MVT::i32, dl, Chain, Addr,
Justin Lebar9c375812016-07-15 18:27:10 +00004435 MachinePointerInfo::getJumpTable(DAG.getMachineFunction()));
Evan Chengf3a1fce2009-07-25 00:33:29 +00004436 Chain = Addr.getValue(1);
Dale Johannesen021052a2009-02-04 20:06:27 +00004437 Addr = DAG.getNode(ISD::ADD, dl, PTy, Addr, Table);
Tim Northover4998a472015-05-13 20:28:38 +00004438 return DAG.getNode(ARMISD::BR_JT, dl, MVT::Other, Chain, Addr, JTI);
Evan Chengf3a1fce2009-07-25 00:33:29 +00004439 } else {
Alex Lorenze40c8a22015-08-11 23:09:45 +00004440 Addr =
4441 DAG.getLoad(PTy, dl, Chain, Addr,
Justin Lebar9c375812016-07-15 18:27:10 +00004442 MachinePointerInfo::getJumpTable(DAG.getMachineFunction()));
Evan Chengf3a1fce2009-07-25 00:33:29 +00004443 Chain = Addr.getValue(1);
Tim Northover4998a472015-05-13 20:28:38 +00004444 return DAG.getNode(ARMISD::BR_JT, dl, MVT::Other, Chain, Addr, JTI);
Evan Chengf3a1fce2009-07-25 00:33:29 +00004445 }
Evan Cheng10043e22007-01-19 07:51:42 +00004446}
4447
Eli Friedman2d4055b2011-11-09 23:36:02 +00004448static SDValue LowerVectorFP_TO_INT(SDValue Op, SelectionDAG &DAG) {
James Molloy547d4c02012-02-20 09:24:05 +00004449 EVT VT = Op.getValueType();
Andrew Trickef9de2a2013-05-25 02:42:55 +00004450 SDLoc dl(Op);
Eli Friedman2d4055b2011-11-09 23:36:02 +00004451
James Molloy547d4c02012-02-20 09:24:05 +00004452 if (Op.getValueType().getVectorElementType() == MVT::i32) {
4453 if (Op.getOperand(0).getValueType().getVectorElementType() == MVT::f32)
4454 return Op;
4455 return DAG.UnrollVectorOp(Op.getNode());
4456 }
4457
4458 assert(Op.getOperand(0).getValueType() == MVT::v4f32 &&
4459 "Invalid type for custom lowering!");
4460 if (VT != MVT::v4i16)
4461 return DAG.UnrollVectorOp(Op.getNode());
4462
4463 Op = DAG.getNode(Op.getOpcode(), dl, MVT::v4i32, Op.getOperand(0));
4464 return DAG.getNode(ISD::TRUNCATE, dl, VT, Op);
Eli Friedman2d4055b2011-11-09 23:36:02 +00004465}
4466
Oliver Stannard51b1d462014-08-21 12:50:31 +00004467SDValue ARMTargetLowering::LowerFP_TO_INT(SDValue Op, SelectionDAG &DAG) const {
Eli Friedman2d4055b2011-11-09 23:36:02 +00004468 EVT VT = Op.getValueType();
4469 if (VT.isVector())
4470 return LowerVectorFP_TO_INT(Op, DAG);
Oliver Stannard51b1d462014-08-21 12:50:31 +00004471 if (Subtarget->isFPOnlySP() && Op.getOperand(0).getValueType() == MVT::f64) {
4472 RTLIB::Libcall LC;
4473 if (Op.getOpcode() == ISD::FP_TO_SINT)
4474 LC = RTLIB::getFPTOSINT(Op.getOperand(0).getValueType(),
4475 Op.getValueType());
4476 else
4477 LC = RTLIB::getFPTOUINT(Op.getOperand(0).getValueType(),
4478 Op.getValueType());
Craig Topper8fe40e02015-10-22 17:05:00 +00004479 return makeLibCall(DAG, LC, Op.getValueType(), Op.getOperand(0),
Oliver Stannard51b1d462014-08-21 12:50:31 +00004480 /*isSigned*/ false, SDLoc(Op)).first;
4481 }
4482
James Molloyfa041152015-03-23 16:15:16 +00004483 return Op;
Bob Wilsone4191e72010-03-19 22:51:32 +00004484}
4485
Cameron Zwarich143f9ae2011-03-29 21:41:55 +00004486static SDValue LowerVectorINT_TO_FP(SDValue Op, SelectionDAG &DAG) {
4487 EVT VT = Op.getValueType();
Andrew Trickef9de2a2013-05-25 02:42:55 +00004488 SDLoc dl(Op);
Cameron Zwarich143f9ae2011-03-29 21:41:55 +00004489
Eli Friedman2d4055b2011-11-09 23:36:02 +00004490 if (Op.getOperand(0).getValueType().getVectorElementType() == MVT::i32) {
4491 if (VT.getVectorElementType() == MVT::f32)
4492 return Op;
4493 return DAG.UnrollVectorOp(Op.getNode());
4494 }
4495
Duncan Sandsa41634e2011-08-12 14:54:45 +00004496 assert(Op.getOperand(0).getValueType() == MVT::v4i16 &&
4497 "Invalid type for custom lowering!");
Cameron Zwarich143f9ae2011-03-29 21:41:55 +00004498 if (VT != MVT::v4f32)
4499 return DAG.UnrollVectorOp(Op.getNode());
4500
4501 unsigned CastOpc;
4502 unsigned Opc;
4503 switch (Op.getOpcode()) {
Craig Toppere55c5562012-02-07 02:50:20 +00004504 default: llvm_unreachable("Invalid opcode!");
Cameron Zwarich143f9ae2011-03-29 21:41:55 +00004505 case ISD::SINT_TO_FP:
4506 CastOpc = ISD::SIGN_EXTEND;
4507 Opc = ISD::SINT_TO_FP;
4508 break;
4509 case ISD::UINT_TO_FP:
4510 CastOpc = ISD::ZERO_EXTEND;
4511 Opc = ISD::UINT_TO_FP;
4512 break;
4513 }
4514
4515 Op = DAG.getNode(CastOpc, dl, MVT::v4i32, Op.getOperand(0));
4516 return DAG.getNode(Opc, dl, VT, Op);
4517}
4518
Oliver Stannard51b1d462014-08-21 12:50:31 +00004519SDValue ARMTargetLowering::LowerINT_TO_FP(SDValue Op, SelectionDAG &DAG) const {
Bob Wilsone4191e72010-03-19 22:51:32 +00004520 EVT VT = Op.getValueType();
Cameron Zwarich143f9ae2011-03-29 21:41:55 +00004521 if (VT.isVector())
4522 return LowerVectorINT_TO_FP(Op, DAG);
Oliver Stannard51b1d462014-08-21 12:50:31 +00004523 if (Subtarget->isFPOnlySP() && Op.getValueType() == MVT::f64) {
4524 RTLIB::Libcall LC;
4525 if (Op.getOpcode() == ISD::SINT_TO_FP)
4526 LC = RTLIB::getSINTTOFP(Op.getOperand(0).getValueType(),
4527 Op.getValueType());
4528 else
4529 LC = RTLIB::getUINTTOFP(Op.getOperand(0).getValueType(),
4530 Op.getValueType());
Craig Topper8fe40e02015-10-22 17:05:00 +00004531 return makeLibCall(DAG, LC, Op.getValueType(), Op.getOperand(0),
Oliver Stannard51b1d462014-08-21 12:50:31 +00004532 /*isSigned*/ false, SDLoc(Op)).first;
4533 }
4534
James Molloyfa041152015-03-23 16:15:16 +00004535 return Op;
Bob Wilsone4191e72010-03-19 22:51:32 +00004536}
4537
Evan Cheng25f93642010-07-08 02:08:50 +00004538SDValue ARMTargetLowering::LowerFCOPYSIGN(SDValue Op, SelectionDAG &DAG) const {
Evan Cheng10043e22007-01-19 07:51:42 +00004539 // Implement fcopysign with a fabs and a conditional fneg.
Dan Gohman2ce6f2a2008-07-27 21:46:04 +00004540 SDValue Tmp0 = Op.getOperand(0);
4541 SDValue Tmp1 = Op.getOperand(1);
Andrew Trickef9de2a2013-05-25 02:42:55 +00004542 SDLoc dl(Op);
Owen Anderson53aa7a92009-08-10 22:56:29 +00004543 EVT VT = Op.getValueType();
4544 EVT SrcVT = Tmp1.getValueType();
Evan Chengd6b641e2011-02-23 02:24:55 +00004545 bool InGPR = Tmp0.getOpcode() == ISD::BITCAST ||
4546 Tmp0.getOpcode() == ARMISD::VMOVDRR;
4547 bool UseNEON = !InGPR && Subtarget->hasNEON();
4548
4549 if (UseNEON) {
4550 // Use VBSL to copy the sign bit.
4551 unsigned EncodedVal = ARM_AM::createNEONModImm(0x6, 0x80);
4552 SDValue Mask = DAG.getNode(ARMISD::VMOVIMM, dl, MVT::v2i32,
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +00004553 DAG.getTargetConstant(EncodedVal, dl, MVT::i32));
Evan Chengd6b641e2011-02-23 02:24:55 +00004554 EVT OpVT = (VT == MVT::f32) ? MVT::v2i32 : MVT::v1i64;
4555 if (VT == MVT::f64)
4556 Mask = DAG.getNode(ARMISD::VSHL, dl, OpVT,
4557 DAG.getNode(ISD::BITCAST, dl, OpVT, Mask),
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +00004558 DAG.getConstant(32, dl, MVT::i32));
Evan Chengd6b641e2011-02-23 02:24:55 +00004559 else /*if (VT == MVT::f32)*/
4560 Tmp0 = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v2f32, Tmp0);
4561 if (SrcVT == MVT::f32) {
4562 Tmp1 = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v2f32, Tmp1);
4563 if (VT == MVT::f64)
4564 Tmp1 = DAG.getNode(ARMISD::VSHL, dl, OpVT,
4565 DAG.getNode(ISD::BITCAST, dl, OpVT, Tmp1),
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +00004566 DAG.getConstant(32, dl, MVT::i32));
Evan Cheng12bb05b2011-04-15 01:31:00 +00004567 } else if (VT == MVT::f32)
4568 Tmp1 = DAG.getNode(ARMISD::VSHRu, dl, MVT::v1i64,
4569 DAG.getNode(ISD::BITCAST, dl, MVT::v1i64, Tmp1),
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +00004570 DAG.getConstant(32, dl, MVT::i32));
Evan Chengd6b641e2011-02-23 02:24:55 +00004571 Tmp0 = DAG.getNode(ISD::BITCAST, dl, OpVT, Tmp0);
4572 Tmp1 = DAG.getNode(ISD::BITCAST, dl, OpVT, Tmp1);
4573
4574 SDValue AllOnes = DAG.getTargetConstant(ARM_AM::createNEONModImm(0xe, 0xff),
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +00004575 dl, MVT::i32);
Evan Chengd6b641e2011-02-23 02:24:55 +00004576 AllOnes = DAG.getNode(ARMISD::VMOVIMM, dl, MVT::v8i8, AllOnes);
4577 SDValue MaskNot = DAG.getNode(ISD::XOR, dl, OpVT, Mask,
4578 DAG.getNode(ISD::BITCAST, dl, OpVT, AllOnes));
Owen Anderson77aa2662011-04-05 21:48:57 +00004579
Evan Chengd6b641e2011-02-23 02:24:55 +00004580 SDValue Res = DAG.getNode(ISD::OR, dl, OpVT,
4581 DAG.getNode(ISD::AND, dl, OpVT, Tmp1, Mask),
4582 DAG.getNode(ISD::AND, dl, OpVT, Tmp0, MaskNot));
Evan Cheng6e3d4432011-02-28 18:45:27 +00004583 if (VT == MVT::f32) {
Evan Chengd6b641e2011-02-23 02:24:55 +00004584 Res = DAG.getNode(ISD::BITCAST, dl, MVT::v2f32, Res);
4585 Res = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::f32, Res,
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +00004586 DAG.getConstant(0, dl, MVT::i32));
Evan Chengd6b641e2011-02-23 02:24:55 +00004587 } else {
4588 Res = DAG.getNode(ISD::BITCAST, dl, MVT::f64, Res);
4589 }
4590
4591 return Res;
4592 }
Evan Cheng2da1c952011-02-11 02:28:55 +00004593
4594 // Bitcast operand 1 to i32.
4595 if (SrcVT == MVT::f64)
4596 Tmp1 = DAG.getNode(ARMISD::VMOVRRD, dl, DAG.getVTList(MVT::i32, MVT::i32),
Craig Topper48d114b2014-04-26 18:35:24 +00004597 Tmp1).getValue(1);
Evan Cheng2da1c952011-02-11 02:28:55 +00004598 Tmp1 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, Tmp1);
4599
Evan Chengd6b641e2011-02-23 02:24:55 +00004600 // Or in the signbit with integer operations.
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +00004601 SDValue Mask1 = DAG.getConstant(0x80000000, dl, MVT::i32);
4602 SDValue Mask2 = DAG.getConstant(0x7fffffff, dl, MVT::i32);
Evan Chengd6b641e2011-02-23 02:24:55 +00004603 Tmp1 = DAG.getNode(ISD::AND, dl, MVT::i32, Tmp1, Mask1);
4604 if (VT == MVT::f32) {
4605 Tmp0 = DAG.getNode(ISD::AND, dl, MVT::i32,
4606 DAG.getNode(ISD::BITCAST, dl, MVT::i32, Tmp0), Mask2);
4607 return DAG.getNode(ISD::BITCAST, dl, MVT::f32,
4608 DAG.getNode(ISD::OR, dl, MVT::i32, Tmp0, Tmp1));
Evan Cheng2da1c952011-02-11 02:28:55 +00004609 }
4610
Evan Chengd6b641e2011-02-23 02:24:55 +00004611 // f64: Or the high part with signbit and then combine two parts.
4612 Tmp0 = DAG.getNode(ARMISD::VMOVRRD, dl, DAG.getVTList(MVT::i32, MVT::i32),
Craig Topper48d114b2014-04-26 18:35:24 +00004613 Tmp0);
Evan Chengd6b641e2011-02-23 02:24:55 +00004614 SDValue Lo = Tmp0.getValue(0);
4615 SDValue Hi = DAG.getNode(ISD::AND, dl, MVT::i32, Tmp0.getValue(1), Mask2);
4616 Hi = DAG.getNode(ISD::OR, dl, MVT::i32, Hi, Tmp1);
4617 return DAG.getNode(ARMISD::VMOVDRR, dl, MVT::f64, Lo, Hi);
Evan Cheng10043e22007-01-19 07:51:42 +00004618}
4619
Evan Cheng168ced92010-05-22 01:47:14 +00004620SDValue ARMTargetLowering::LowerRETURNADDR(SDValue Op, SelectionDAG &DAG) const{
4621 MachineFunction &MF = DAG.getMachineFunction();
Matthias Braun941a7052016-07-28 18:40:00 +00004622 MachineFrameInfo &MFI = MF.getFrameInfo();
4623 MFI.setReturnAddressIsTaken(true);
Evan Cheng168ced92010-05-22 01:47:14 +00004624
Bill Wendling908bf812014-01-06 00:43:20 +00004625 if (verifyReturnAddressArgumentIsConstant(Op, DAG))
Bill Wendlingdf7dd282014-01-05 01:47:20 +00004626 return SDValue();
Bill Wendlingdf7dd282014-01-05 01:47:20 +00004627
Evan Cheng168ced92010-05-22 01:47:14 +00004628 EVT VT = Op.getValueType();
Andrew Trickef9de2a2013-05-25 02:42:55 +00004629 SDLoc dl(Op);
Evan Cheng168ced92010-05-22 01:47:14 +00004630 unsigned Depth = cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue();
4631 if (Depth) {
4632 SDValue FrameAddr = LowerFRAMEADDR(Op, DAG);
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +00004633 SDValue Offset = DAG.getConstant(4, dl, MVT::i32);
Evan Cheng168ced92010-05-22 01:47:14 +00004634 return DAG.getLoad(VT, dl, DAG.getEntryNode(),
4635 DAG.getNode(ISD::ADD, dl, VT, FrameAddr, Offset),
Justin Lebar9c375812016-07-15 18:27:10 +00004636 MachinePointerInfo());
Evan Cheng168ced92010-05-22 01:47:14 +00004637 }
4638
4639 // Return LR, which contains the return address. Mark it an implicit live-in.
Devang Patelf3292b22011-02-21 23:21:26 +00004640 unsigned Reg = MF.addLiveIn(ARM::LR, getRegClassFor(MVT::i32));
Evan Cheng168ced92010-05-22 01:47:14 +00004641 return DAG.getCopyFromReg(DAG.getEntryNode(), dl, Reg, VT);
4642}
4643
Dan Gohman21cea8a2010-04-17 15:26:15 +00004644SDValue ARMTargetLowering::LowerFRAMEADDR(SDValue Op, SelectionDAG &DAG) const {
Saleem Abdulrasoolf11f4b42014-05-18 03:18:09 +00004645 const ARMBaseRegisterInfo &ARI =
4646 *static_cast<const ARMBaseRegisterInfo*>(RegInfo);
4647 MachineFunction &MF = DAG.getMachineFunction();
Matthias Braun941a7052016-07-28 18:40:00 +00004648 MachineFrameInfo &MFI = MF.getFrameInfo();
4649 MFI.setFrameAddressIsTaken(true);
Evan Cheng168ced92010-05-22 01:47:14 +00004650
Owen Anderson53aa7a92009-08-10 22:56:29 +00004651 EVT VT = Op.getValueType();
Andrew Trickef9de2a2013-05-25 02:42:55 +00004652 SDLoc dl(Op); // FIXME probably not meaningful
Jim Grosbachaeca45d2009-05-12 23:59:14 +00004653 unsigned Depth = cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue();
Saleem Abdulrasoolf11f4b42014-05-18 03:18:09 +00004654 unsigned FrameReg = ARI.getFrameRegister(MF);
Jim Grosbachaeca45d2009-05-12 23:59:14 +00004655 SDValue FrameAddr = DAG.getCopyFromReg(DAG.getEntryNode(), dl, FrameReg, VT);
4656 while (Depth--)
Chris Lattner7727d052010-09-21 06:44:06 +00004657 FrameAddr = DAG.getLoad(VT, dl, DAG.getEntryNode(), FrameAddr,
Justin Lebar9c375812016-07-15 18:27:10 +00004658 MachinePointerInfo());
Jim Grosbachaeca45d2009-05-12 23:59:14 +00004659 return FrameAddr;
4660}
4661
Renato Golinc7aea402014-05-06 16:51:25 +00004662// FIXME? Maybe this could be a TableGen attribute on some registers and
4663// this table could be generated automatically from RegInfo.
Pat Gavlina717f252015-07-09 17:40:29 +00004664unsigned ARMTargetLowering::getRegisterByName(const char* RegName, EVT VT,
4665 SelectionDAG &DAG) const {
Renato Golinc7aea402014-05-06 16:51:25 +00004666 unsigned Reg = StringSwitch<unsigned>(RegName)
4667 .Case("sp", ARM::SP)
4668 .Default(0);
4669 if (Reg)
4670 return Reg;
Luke Cheeseman85fd06d2015-06-01 12:02:47 +00004671 report_fatal_error(Twine("Invalid register name \""
4672 + StringRef(RegName) + "\"."));
4673}
4674
4675// Result is 64 bit value so split into two 32 bit values and return as a
4676// pair of values.
4677static void ExpandREAD_REGISTER(SDNode *N, SmallVectorImpl<SDValue> &Results,
4678 SelectionDAG &DAG) {
4679 SDLoc DL(N);
4680
4681 // This function is only supposed to be called for i64 type destination.
4682 assert(N->getValueType(0) == MVT::i64
4683 && "ExpandREAD_REGISTER called for non-i64 type result.");
4684
4685 SDValue Read = DAG.getNode(ISD::READ_REGISTER, DL,
4686 DAG.getVTList(MVT::i32, MVT::i32, MVT::Other),
4687 N->getOperand(0),
4688 N->getOperand(1));
4689
4690 Results.push_back(DAG.getNode(ISD::BUILD_PAIR, DL, MVT::i64, Read.getValue(0),
4691 Read.getValue(1)));
4692 Results.push_back(Read.getOperand(0));
Renato Golinc7aea402014-05-06 16:51:25 +00004693}
4694
Quentin Colombet901f0362015-12-04 01:53:14 +00004695/// \p BC is a bitcast that is about to be turned into a VMOVDRR.
4696/// When \p DstVT, the destination type of \p BC, is on the vector
4697/// register bank and the source of bitcast, \p Op, operates on the same bank,
4698/// it might be possible to combine them, such that everything stays on the
4699/// vector register bank.
4700/// \p return The node that would replace \p BT, if the combine
4701/// is possible.
4702static SDValue CombineVMOVDRRCandidateWithVecOp(const SDNode *BC,
4703 SelectionDAG &DAG) {
4704 SDValue Op = BC->getOperand(0);
4705 EVT DstVT = BC->getValueType(0);
4706
4707 // The only vector instruction that can produce a scalar (remember,
4708 // since the bitcast was about to be turned into VMOVDRR, the source
4709 // type is i64) from a vector is EXTRACT_VECTOR_ELT.
4710 // Moreover, we can do this combine only if there is one use.
4711 // Finally, if the destination type is not a vector, there is not
4712 // much point on forcing everything on the vector bank.
4713 if (!DstVT.isVector() || Op.getOpcode() != ISD::EXTRACT_VECTOR_ELT ||
4714 !Op.hasOneUse())
4715 return SDValue();
4716
4717 // If the index is not constant, we will introduce an additional
4718 // multiply that will stick.
4719 // Give up in that case.
4720 ConstantSDNode *Index = dyn_cast<ConstantSDNode>(Op.getOperand(1));
4721 if (!Index)
4722 return SDValue();
4723 unsigned DstNumElt = DstVT.getVectorNumElements();
4724
4725 // Compute the new index.
4726 const APInt &APIntIndex = Index->getAPIntValue();
4727 APInt NewIndex(APIntIndex.getBitWidth(), DstNumElt);
4728 NewIndex *= APIntIndex;
4729 // Check if the new constant index fits into i32.
4730 if (NewIndex.getBitWidth() > 32)
4731 return SDValue();
4732
4733 // vMTy bitcast(i64 extractelt vNi64 src, i32 index) ->
4734 // vMTy extractsubvector vNxMTy (bitcast vNi64 src), i32 index*M)
4735 SDLoc dl(Op);
4736 SDValue ExtractSrc = Op.getOperand(0);
4737 EVT VecVT = EVT::getVectorVT(
4738 *DAG.getContext(), DstVT.getScalarType(),
4739 ExtractSrc.getValueType().getVectorNumElements() * DstNumElt);
4740 SDValue BitCast = DAG.getNode(ISD::BITCAST, dl, VecVT, ExtractSrc);
4741 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, DstVT, BitCast,
4742 DAG.getConstant(NewIndex.getZExtValue(), dl, MVT::i32));
4743}
4744
Wesley Peck527da1b2010-11-23 03:31:01 +00004745/// ExpandBITCAST - If the target supports VFP, this function is called to
Bob Wilson59b70ea2010-04-17 05:30:19 +00004746/// expand a bit convert where either the source or destination type is i64 to
4747/// use a VMOVDRR or VMOVRRD node. This should not be done when the non-i64
4748/// operand type is illegal (e.g., v2f32 for a target that doesn't support
4749/// vectors), since the legalizer won't know what to do with that.
Wesley Peck527da1b2010-11-23 03:31:01 +00004750static SDValue ExpandBITCAST(SDNode *N, SelectionDAG &DAG) {
Bob Wilson59b70ea2010-04-17 05:30:19 +00004751 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
Andrew Trickef9de2a2013-05-25 02:42:55 +00004752 SDLoc dl(N);
Dan Gohman2ce6f2a2008-07-27 21:46:04 +00004753 SDValue Op = N->getOperand(0);
Bob Wilsonc05b8872010-04-14 20:45:23 +00004754
Bob Wilson59b70ea2010-04-17 05:30:19 +00004755 // This function is only supposed to be called for i64 types, either as the
4756 // source or destination of the bit convert.
4757 EVT SrcVT = Op.getValueType();
4758 EVT DstVT = N->getValueType(0);
4759 assert((SrcVT == MVT::i64 || DstVT == MVT::i64) &&
Wesley Peck527da1b2010-11-23 03:31:01 +00004760 "ExpandBITCAST called for non-i64 type");
Bob Wilsonc05b8872010-04-14 20:45:23 +00004761
Bob Wilson59b70ea2010-04-17 05:30:19 +00004762 // Turn i64->f64 into VMOVDRR.
4763 if (SrcVT == MVT::i64 && TLI.isTypeLegal(DstVT)) {
Quentin Colombet901f0362015-12-04 01:53:14 +00004764 // Do not force values to GPRs (this is what VMOVDRR does for the inputs)
4765 // if we can combine the bitcast with its source.
4766 if (SDValue Val = CombineVMOVDRRCandidateWithVecOp(N, DAG))
4767 return Val;
4768
Owen Anderson9f944592009-08-11 20:47:22 +00004769 SDValue Lo = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, MVT::i32, Op,
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +00004770 DAG.getConstant(0, dl, MVT::i32));
Owen Anderson9f944592009-08-11 20:47:22 +00004771 SDValue Hi = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, MVT::i32, Op,
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +00004772 DAG.getConstant(1, dl, MVT::i32));
Wesley Peck527da1b2010-11-23 03:31:01 +00004773 return DAG.getNode(ISD::BITCAST, dl, DstVT,
Bob Wilsonf07d33d2010-06-11 22:45:25 +00004774 DAG.getNode(ARMISD::VMOVDRR, dl, MVT::f64, Lo, Hi));
Evan Cheng297b32a2008-11-04 19:57:48 +00004775 }
Bob Wilson7117a912009-03-20 22:42:55 +00004776
Jim Grosbachd7cf55c2009-11-09 00:11:35 +00004777 // Turn f64->i64 into VMOVRRD.
Bob Wilson59b70ea2010-04-17 05:30:19 +00004778 if (DstVT == MVT::i64 && TLI.isTypeLegal(SrcVT)) {
Christian Pirker238c7c12014-05-12 11:19:20 +00004779 SDValue Cvt;
Mehdi Aminiffc14022015-07-08 01:00:38 +00004780 if (DAG.getDataLayout().isBigEndian() && SrcVT.isVector() &&
Christian Pirker6692e7c2014-05-14 16:59:44 +00004781 SrcVT.getVectorNumElements() > 1)
Christian Pirker238c7c12014-05-12 11:19:20 +00004782 Cvt = DAG.getNode(ARMISD::VMOVRRD, dl,
4783 DAG.getVTList(MVT::i32, MVT::i32),
4784 DAG.getNode(ARMISD::VREV64, dl, SrcVT, Op));
4785 else
4786 Cvt = DAG.getNode(ARMISD::VMOVRRD, dl,
4787 DAG.getVTList(MVT::i32, MVT::i32), Op);
Bob Wilson59b70ea2010-04-17 05:30:19 +00004788 // Merge the pieces into a single i64 value.
4789 return DAG.getNode(ISD::BUILD_PAIR, dl, MVT::i64, Cvt, Cvt.getValue(1));
4790 }
Bob Wilson7117a912009-03-20 22:42:55 +00004791
Bob Wilson59b70ea2010-04-17 05:30:19 +00004792 return SDValue();
Chris Lattnerf81d5882007-11-24 07:07:01 +00004793}
4794
Bob Wilson2e076c42009-06-22 23:27:02 +00004795/// getZeroVector - Returns a vector of specified type with all zero elements.
Bob Wilsona3f19012010-07-13 21:16:48 +00004796/// Zero vectors are used to represent vector negation and in those cases
4797/// will be implemented with the NEON VNEG instruction. However, VNEG does
4798/// not support i64 elements, so sometimes the zero vectors will need to be
4799/// explicitly constructed. Regardless, use a canonical VMOV to create the
4800/// zero vector.
Benjamin Kramerbdc49562016-06-12 15:39:02 +00004801static SDValue getZeroVector(EVT VT, SelectionDAG &DAG, const SDLoc &dl) {
Bob Wilson2e076c42009-06-22 23:27:02 +00004802 assert(VT.isVector() && "Expected a vector type");
Bob Wilsona3f19012010-07-13 21:16:48 +00004803 // The canonical modified immediate encoding of a zero vector is....0!
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +00004804 SDValue EncodedVal = DAG.getTargetConstant(0, dl, MVT::i32);
Bob Wilsona3f19012010-07-13 21:16:48 +00004805 EVT VmovVT = VT.is128BitVector() ? MVT::v4i32 : MVT::v2i32;
4806 SDValue Vmov = DAG.getNode(ARMISD::VMOVIMM, dl, VmovVT, EncodedVal);
Wesley Peck527da1b2010-11-23 03:31:01 +00004807 return DAG.getNode(ISD::BITCAST, dl, VT, Vmov);
Bob Wilson2e076c42009-06-22 23:27:02 +00004808}
4809
Jim Grosbach624fcb22009-10-31 21:00:56 +00004810/// LowerShiftRightParts - Lower SRA_PARTS, which returns two
4811/// i32 values and take a 2 x i32 value to shift plus a shift amount.
Dan Gohman21cea8a2010-04-17 15:26:15 +00004812SDValue ARMTargetLowering::LowerShiftRightParts(SDValue Op,
4813 SelectionDAG &DAG) const {
Jim Grosbach624fcb22009-10-31 21:00:56 +00004814 assert(Op.getNumOperands() == 3 && "Not a double-shift!");
4815 EVT VT = Op.getValueType();
4816 unsigned VTBits = VT.getSizeInBits();
Andrew Trickef9de2a2013-05-25 02:42:55 +00004817 SDLoc dl(Op);
Jim Grosbach624fcb22009-10-31 21:00:56 +00004818 SDValue ShOpLo = Op.getOperand(0);
4819 SDValue ShOpHi = Op.getOperand(1);
4820 SDValue ShAmt = Op.getOperand(2);
Evan Cheng0cc4ad92010-07-13 19:27:42 +00004821 SDValue ARMcc;
Jim Grosbach8fe6fd72009-10-31 21:42:19 +00004822 unsigned Opc = (Op.getOpcode() == ISD::SRA_PARTS) ? ISD::SRA : ISD::SRL;
Jim Grosbach624fcb22009-10-31 21:00:56 +00004823
Jim Grosbach8fe6fd72009-10-31 21:42:19 +00004824 assert(Op.getOpcode() == ISD::SRA_PARTS || Op.getOpcode() == ISD::SRL_PARTS);
4825
Jim Grosbach624fcb22009-10-31 21:00:56 +00004826 SDValue RevShAmt = DAG.getNode(ISD::SUB, dl, MVT::i32,
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +00004827 DAG.getConstant(VTBits, dl, MVT::i32), ShAmt);
Jim Grosbach624fcb22009-10-31 21:00:56 +00004828 SDValue Tmp1 = DAG.getNode(ISD::SRL, dl, VT, ShOpLo, ShAmt);
4829 SDValue ExtraShAmt = DAG.getNode(ISD::SUB, dl, MVT::i32, ShAmt,
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +00004830 DAG.getConstant(VTBits, dl, MVT::i32));
Jim Grosbach624fcb22009-10-31 21:00:56 +00004831 SDValue Tmp2 = DAG.getNode(ISD::SHL, dl, VT, ShOpHi, RevShAmt);
4832 SDValue FalseVal = DAG.getNode(ISD::OR, dl, VT, Tmp1, Tmp2);
Jim Grosbach8fe6fd72009-10-31 21:42:19 +00004833 SDValue TrueVal = DAG.getNode(Opc, dl, VT, ShOpHi, ExtraShAmt);
Jim Grosbach624fcb22009-10-31 21:00:56 +00004834
4835 SDValue CCR = DAG.getRegister(ARM::CPSR, MVT::i32);
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +00004836 SDValue Cmp = getARMCmp(ExtraShAmt, DAG.getConstant(0, dl, MVT::i32),
4837 ISD::SETGE, ARMcc, DAG, dl);
Jim Grosbach8fe6fd72009-10-31 21:42:19 +00004838 SDValue Hi = DAG.getNode(Opc, dl, VT, ShOpHi, ShAmt);
Evan Cheng0cc4ad92010-07-13 19:27:42 +00004839 SDValue Lo = DAG.getNode(ARMISD::CMOV, dl, VT, FalseVal, TrueVal, ARMcc,
Jim Grosbach624fcb22009-10-31 21:00:56 +00004840 CCR, Cmp);
4841
4842 SDValue Ops[2] = { Lo, Hi };
Craig Topper64941d92014-04-27 19:20:57 +00004843 return DAG.getMergeValues(Ops, dl);
Jim Grosbach624fcb22009-10-31 21:00:56 +00004844}
4845
Jim Grosbach5d994042009-10-31 19:38:01 +00004846/// LowerShiftLeftParts - Lower SHL_PARTS, which returns two
4847/// i32 values and take a 2 x i32 value to shift plus a shift amount.
Dan Gohman21cea8a2010-04-17 15:26:15 +00004848SDValue ARMTargetLowering::LowerShiftLeftParts(SDValue Op,
4849 SelectionDAG &DAG) const {
Jim Grosbach5d994042009-10-31 19:38:01 +00004850 assert(Op.getNumOperands() == 3 && "Not a double-shift!");
4851 EVT VT = Op.getValueType();
4852 unsigned VTBits = VT.getSizeInBits();
Andrew Trickef9de2a2013-05-25 02:42:55 +00004853 SDLoc dl(Op);
Jim Grosbach5d994042009-10-31 19:38:01 +00004854 SDValue ShOpLo = Op.getOperand(0);
4855 SDValue ShOpHi = Op.getOperand(1);
4856 SDValue ShAmt = Op.getOperand(2);
Evan Cheng0cc4ad92010-07-13 19:27:42 +00004857 SDValue ARMcc;
Jim Grosbach5d994042009-10-31 19:38:01 +00004858
4859 assert(Op.getOpcode() == ISD::SHL_PARTS);
4860 SDValue RevShAmt = DAG.getNode(ISD::SUB, dl, MVT::i32,
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +00004861 DAG.getConstant(VTBits, dl, MVT::i32), ShAmt);
Jim Grosbach5d994042009-10-31 19:38:01 +00004862 SDValue Tmp1 = DAG.getNode(ISD::SRL, dl, VT, ShOpLo, RevShAmt);
4863 SDValue ExtraShAmt = DAG.getNode(ISD::SUB, dl, MVT::i32, ShAmt,
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +00004864 DAG.getConstant(VTBits, dl, MVT::i32));
Jim Grosbach5d994042009-10-31 19:38:01 +00004865 SDValue Tmp2 = DAG.getNode(ISD::SHL, dl, VT, ShOpHi, ShAmt);
4866 SDValue Tmp3 = DAG.getNode(ISD::SHL, dl, VT, ShOpLo, ExtraShAmt);
4867
4868 SDValue FalseVal = DAG.getNode(ISD::OR, dl, VT, Tmp1, Tmp2);
4869 SDValue CCR = DAG.getRegister(ARM::CPSR, MVT::i32);
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +00004870 SDValue Cmp = getARMCmp(ExtraShAmt, DAG.getConstant(0, dl, MVT::i32),
4871 ISD::SETGE, ARMcc, DAG, dl);
Jim Grosbach5d994042009-10-31 19:38:01 +00004872 SDValue Lo = DAG.getNode(ISD::SHL, dl, VT, ShOpLo, ShAmt);
Evan Cheng0cc4ad92010-07-13 19:27:42 +00004873 SDValue Hi = DAG.getNode(ARMISD::CMOV, dl, VT, FalseVal, Tmp3, ARMcc,
Jim Grosbach5d994042009-10-31 19:38:01 +00004874 CCR, Cmp);
4875
4876 SDValue Ops[2] = { Lo, Hi };
Craig Topper64941d92014-04-27 19:20:57 +00004877 return DAG.getMergeValues(Ops, dl);
Jim Grosbach5d994042009-10-31 19:38:01 +00004878}
4879
Jim Grosbach535d3b42010-09-08 03:54:02 +00004880SDValue ARMTargetLowering::LowerFLT_ROUNDS_(SDValue Op,
Nate Begemanb69b1822010-08-03 21:31:55 +00004881 SelectionDAG &DAG) const {
4882 // The rounding mode is in bits 23:22 of the FPSCR.
4883 // The ARM rounding mode value to FLT_ROUNDS mapping is 0->1, 1->2, 2->3, 3->0
4884 // The formula we use to implement this is (((FPSCR + 1 << 22) >> 22) & 3)
4885 // so that the shift + and get folded into a bitfield extract.
Andrew Trickef9de2a2013-05-25 02:42:55 +00004886 SDLoc dl(Op);
Nate Begemanb69b1822010-08-03 21:31:55 +00004887 SDValue FPSCR = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, MVT::i32,
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +00004888 DAG.getConstant(Intrinsic::arm_get_fpscr, dl,
Nate Begemanb69b1822010-08-03 21:31:55 +00004889 MVT::i32));
Jim Grosbach535d3b42010-09-08 03:54:02 +00004890 SDValue FltRounds = DAG.getNode(ISD::ADD, dl, MVT::i32, FPSCR,
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +00004891 DAG.getConstant(1U << 22, dl, MVT::i32));
Nate Begemanb69b1822010-08-03 21:31:55 +00004892 SDValue RMODE = DAG.getNode(ISD::SRL, dl, MVT::i32, FltRounds,
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +00004893 DAG.getConstant(22, dl, MVT::i32));
Jim Grosbach535d3b42010-09-08 03:54:02 +00004894 return DAG.getNode(ISD::AND, dl, MVT::i32, RMODE,
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +00004895 DAG.getConstant(3, dl, MVT::i32));
Nate Begemanb69b1822010-08-03 21:31:55 +00004896}
4897
Jim Grosbach8546ec92010-01-18 19:58:49 +00004898static SDValue LowerCTTZ(SDNode *N, SelectionDAG &DAG,
4899 const ARMSubtarget *ST) {
Andrew Trickef9de2a2013-05-25 02:42:55 +00004900 SDLoc dl(N);
Logan Chien0a43abc2015-07-13 15:37:30 +00004901 EVT VT = N->getValueType(0);
4902 if (VT.isVector()) {
4903 assert(ST->hasNEON());
4904
4905 // Compute the least significant set bit: LSB = X & -X
4906 SDValue X = N->getOperand(0);
4907 SDValue NX = DAG.getNode(ISD::SUB, dl, VT, getZeroVector(VT, DAG, dl), X);
4908 SDValue LSB = DAG.getNode(ISD::AND, dl, VT, X, NX);
4909
4910 EVT ElemTy = VT.getVectorElementType();
4911
4912 if (ElemTy == MVT::i8) {
4913 // Compute with: cttz(x) = ctpop(lsb - 1)
4914 SDValue One = DAG.getNode(ARMISD::VMOVIMM, dl, VT,
4915 DAG.getTargetConstant(1, dl, ElemTy));
4916 SDValue Bits = DAG.getNode(ISD::SUB, dl, VT, LSB, One);
4917 return DAG.getNode(ISD::CTPOP, dl, VT, Bits);
4918 }
4919
4920 if ((ElemTy == MVT::i16 || ElemTy == MVT::i32) &&
4921 (N->getOpcode() == ISD::CTTZ_ZERO_UNDEF)) {
4922 // Compute with: cttz(x) = (width - 1) - ctlz(lsb), if x != 0
4923 unsigned NumBits = ElemTy.getSizeInBits();
4924 SDValue WidthMinus1 =
4925 DAG.getNode(ARMISD::VMOVIMM, dl, VT,
4926 DAG.getTargetConstant(NumBits - 1, dl, ElemTy));
4927 SDValue CTLZ = DAG.getNode(ISD::CTLZ, dl, VT, LSB);
4928 return DAG.getNode(ISD::SUB, dl, VT, WidthMinus1, CTLZ);
4929 }
4930
4931 // Compute with: cttz(x) = ctpop(lsb - 1)
4932
4933 // Since we can only compute the number of bits in a byte with vcnt.8, we
4934 // have to gather the result with pairwise addition (vpaddl) for i16, i32,
4935 // and i64.
4936
4937 // Compute LSB - 1.
4938 SDValue Bits;
4939 if (ElemTy == MVT::i64) {
4940 // Load constant 0xffff'ffff'ffff'ffff to register.
4941 SDValue FF = DAG.getNode(ARMISD::VMOVIMM, dl, VT,
4942 DAG.getTargetConstant(0x1eff, dl, MVT::i32));
4943 Bits = DAG.getNode(ISD::ADD, dl, VT, LSB, FF);
4944 } else {
4945 SDValue One = DAG.getNode(ARMISD::VMOVIMM, dl, VT,
4946 DAG.getTargetConstant(1, dl, ElemTy));
4947 Bits = DAG.getNode(ISD::SUB, dl, VT, LSB, One);
4948 }
4949
4950 // Count #bits with vcnt.8.
4951 EVT VT8Bit = VT.is64BitVector() ? MVT::v8i8 : MVT::v16i8;
4952 SDValue BitsVT8 = DAG.getNode(ISD::BITCAST, dl, VT8Bit, Bits);
4953 SDValue Cnt8 = DAG.getNode(ISD::CTPOP, dl, VT8Bit, BitsVT8);
4954
4955 // Gather the #bits with vpaddl (pairwise add.)
4956 EVT VT16Bit = VT.is64BitVector() ? MVT::v4i16 : MVT::v8i16;
4957 SDValue Cnt16 = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, VT16Bit,
4958 DAG.getTargetConstant(Intrinsic::arm_neon_vpaddlu, dl, MVT::i32),
4959 Cnt8);
4960 if (ElemTy == MVT::i16)
4961 return Cnt16;
4962
4963 EVT VT32Bit = VT.is64BitVector() ? MVT::v2i32 : MVT::v4i32;
4964 SDValue Cnt32 = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, VT32Bit,
4965 DAG.getTargetConstant(Intrinsic::arm_neon_vpaddlu, dl, MVT::i32),
4966 Cnt16);
4967 if (ElemTy == MVT::i32)
4968 return Cnt32;
4969
4970 assert(ElemTy == MVT::i64);
4971 SDValue Cnt64 = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, VT,
4972 DAG.getTargetConstant(Intrinsic::arm_neon_vpaddlu, dl, MVT::i32),
4973 Cnt32);
4974 return Cnt64;
4975 }
Jim Grosbach8546ec92010-01-18 19:58:49 +00004976
4977 if (!ST->hasV6T2Ops())
4978 return SDValue();
4979
James Molloyb5640982015-11-13 16:05:22 +00004980 SDValue rbit = DAG.getNode(ISD::BITREVERSE, dl, VT, N->getOperand(0));
Jim Grosbach8546ec92010-01-18 19:58:49 +00004981 return DAG.getNode(ISD::CTLZ, dl, VT, rbit);
4982}
4983
Evan Chengb4eae132012-12-04 22:41:50 +00004984/// getCTPOP16BitCounts - Returns a v8i8/v16i8 vector containing the bit-count
4985/// for each 16-bit element from operand, repeated. The basic idea is to
4986/// leverage vcnt to get the 8-bit counts, gather and add the results.
4987///
4988/// Trace for v4i16:
4989/// input = [v0 v1 v2 v3 ] (vi 16-bit element)
4990/// cast: N0 = [w0 w1 w2 w3 w4 w5 w6 w7] (v0 = [w0 w1], wi 8-bit element)
4991/// vcnt: N1 = [b0 b1 b2 b3 b4 b5 b6 b7] (bi = bit-count of 8-bit element wi)
Jim Grosbach54efea02013-03-02 20:16:15 +00004992/// vrev: N2 = [b1 b0 b3 b2 b5 b4 b7 b6]
Evan Chengb4eae132012-12-04 22:41:50 +00004993/// [b0 b1 b2 b3 b4 b5 b6 b7]
4994/// +[b1 b0 b3 b2 b5 b4 b7 b6]
4995/// N3=N1+N2 = [k0 k0 k1 k1 k2 k2 k3 k3] (k0 = b0+b1 = bit-count of 16-bit v0,
4996/// vuzp: = [k0 k1 k2 k3 k0 k1 k2 k3] each ki is 8-bits)
4997static SDValue getCTPOP16BitCounts(SDNode *N, SelectionDAG &DAG) {
4998 EVT VT = N->getValueType(0);
Andrew Trickef9de2a2013-05-25 02:42:55 +00004999 SDLoc DL(N);
Evan Chengb4eae132012-12-04 22:41:50 +00005000
5001 EVT VT8Bit = VT.is64BitVector() ? MVT::v8i8 : MVT::v16i8;
5002 SDValue N0 = DAG.getNode(ISD::BITCAST, DL, VT8Bit, N->getOperand(0));
5003 SDValue N1 = DAG.getNode(ISD::CTPOP, DL, VT8Bit, N0);
5004 SDValue N2 = DAG.getNode(ARMISD::VREV16, DL, VT8Bit, N1);
5005 SDValue N3 = DAG.getNode(ISD::ADD, DL, VT8Bit, N1, N2);
5006 return DAG.getNode(ARMISD::VUZP, DL, VT8Bit, N3, N3);
5007}
5008
5009/// lowerCTPOP16BitElements - Returns a v4i16/v8i16 vector containing the
5010/// bit-count for each 16-bit element from the operand. We need slightly
5011/// different sequencing for v4i16 and v8i16 to stay within NEON's available
5012/// 64/128-bit registers.
Jim Grosbach54efea02013-03-02 20:16:15 +00005013///
Evan Chengb4eae132012-12-04 22:41:50 +00005014/// Trace for v4i16:
5015/// input = [v0 v1 v2 v3 ] (vi 16-bit element)
5016/// v8i8: BitCounts = [k0 k1 k2 k3 k0 k1 k2 k3 ] (ki is the bit-count of vi)
5017/// v8i16:Extended = [k0 k1 k2 k3 k0 k1 k2 k3 ]
5018/// v4i16:Extracted = [k0 k1 k2 k3 ]
5019static SDValue lowerCTPOP16BitElements(SDNode *N, SelectionDAG &DAG) {
5020 EVT VT = N->getValueType(0);
Andrew Trickef9de2a2013-05-25 02:42:55 +00005021 SDLoc DL(N);
Evan Chengb4eae132012-12-04 22:41:50 +00005022
5023 SDValue BitCounts = getCTPOP16BitCounts(N, DAG);
5024 if (VT.is64BitVector()) {
5025 SDValue Extended = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::v8i16, BitCounts);
5026 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, MVT::v4i16, Extended,
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +00005027 DAG.getIntPtrConstant(0, DL));
Evan Chengb4eae132012-12-04 22:41:50 +00005028 } else {
5029 SDValue Extracted = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, MVT::v8i8,
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +00005030 BitCounts, DAG.getIntPtrConstant(0, DL));
Evan Chengb4eae132012-12-04 22:41:50 +00005031 return DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::v8i16, Extracted);
5032 }
5033}
5034
5035/// lowerCTPOP32BitElements - Returns a v2i32/v4i32 vector containing the
5036/// bit-count for each 32-bit element from the operand. The idea here is
5037/// to split the vector into 16-bit elements, leverage the 16-bit count
5038/// routine, and then combine the results.
5039///
5040/// Trace for v2i32 (v4i32 similar with Extracted/Extended exchanged):
5041/// input = [v0 v1 ] (vi: 32-bit elements)
5042/// Bitcast = [w0 w1 w2 w3 ] (wi: 16-bit elements, v0 = [w0 w1])
5043/// Counts16 = [k0 k1 k2 k3 ] (ki: 16-bit elements, bit-count of wi)
Jim Grosbach54efea02013-03-02 20:16:15 +00005044/// vrev: N0 = [k1 k0 k3 k2 ]
Evan Chengb4eae132012-12-04 22:41:50 +00005045/// [k0 k1 k2 k3 ]
5046/// N1 =+[k1 k0 k3 k2 ]
5047/// [k0 k2 k1 k3 ]
5048/// N2 =+[k1 k3 k0 k2 ]
5049/// [k0 k2 k1 k3 ]
5050/// Extended =+[k1 k3 k0 k2 ]
5051/// [k0 k2 ]
5052/// Extracted=+[k1 k3 ]
5053///
5054static SDValue lowerCTPOP32BitElements(SDNode *N, SelectionDAG &DAG) {
5055 EVT VT = N->getValueType(0);
Andrew Trickef9de2a2013-05-25 02:42:55 +00005056 SDLoc DL(N);
Evan Chengb4eae132012-12-04 22:41:50 +00005057
5058 EVT VT16Bit = VT.is64BitVector() ? MVT::v4i16 : MVT::v8i16;
5059
5060 SDValue Bitcast = DAG.getNode(ISD::BITCAST, DL, VT16Bit, N->getOperand(0));
5061 SDValue Counts16 = lowerCTPOP16BitElements(Bitcast.getNode(), DAG);
5062 SDValue N0 = DAG.getNode(ARMISD::VREV32, DL, VT16Bit, Counts16);
5063 SDValue N1 = DAG.getNode(ISD::ADD, DL, VT16Bit, Counts16, N0);
5064 SDValue N2 = DAG.getNode(ARMISD::VUZP, DL, VT16Bit, N1, N1);
5065
5066 if (VT.is64BitVector()) {
5067 SDValue Extended = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::v4i32, N2);
5068 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, MVT::v2i32, Extended,
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +00005069 DAG.getIntPtrConstant(0, DL));
Evan Chengb4eae132012-12-04 22:41:50 +00005070 } else {
5071 SDValue Extracted = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, MVT::v4i16, N2,
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +00005072 DAG.getIntPtrConstant(0, DL));
Evan Chengb4eae132012-12-04 22:41:50 +00005073 return DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::v4i32, Extracted);
5074 }
5075}
5076
5077static SDValue LowerCTPOP(SDNode *N, SelectionDAG &DAG,
5078 const ARMSubtarget *ST) {
5079 EVT VT = N->getValueType(0);
5080
5081 assert(ST->hasNEON() && "Custom ctpop lowering requires NEON.");
Matt Beaumont-Gay50f61b62012-12-04 23:54:02 +00005082 assert((VT == MVT::v2i32 || VT == MVT::v4i32 ||
5083 VT == MVT::v4i16 || VT == MVT::v8i16) &&
Evan Chengb4eae132012-12-04 22:41:50 +00005084 "Unexpected type for custom ctpop lowering");
5085
5086 if (VT.getVectorElementType() == MVT::i32)
5087 return lowerCTPOP32BitElements(N, DAG);
5088 else
5089 return lowerCTPOP16BitElements(N, DAG);
5090}
5091
Bob Wilson2e076c42009-06-22 23:27:02 +00005092static SDValue LowerShift(SDNode *N, SelectionDAG &DAG,
5093 const ARMSubtarget *ST) {
Owen Anderson53aa7a92009-08-10 22:56:29 +00005094 EVT VT = N->getValueType(0);
Andrew Trickef9de2a2013-05-25 02:42:55 +00005095 SDLoc dl(N);
Bob Wilson2e076c42009-06-22 23:27:02 +00005096
Bob Wilson7d471332010-11-18 21:16:28 +00005097 if (!VT.isVector())
5098 return SDValue();
5099
Bob Wilson2e076c42009-06-22 23:27:02 +00005100 // Lower vector shifts on NEON to use VSHL.
Bob Wilson7d471332010-11-18 21:16:28 +00005101 assert(ST->hasNEON() && "unexpected vector shift");
Bob Wilson2e076c42009-06-22 23:27:02 +00005102
Bob Wilson7d471332010-11-18 21:16:28 +00005103 // Left shifts translate directly to the vshiftu intrinsic.
5104 if (N->getOpcode() == ISD::SHL)
Bob Wilson2e076c42009-06-22 23:27:02 +00005105 return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, VT,
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +00005106 DAG.getConstant(Intrinsic::arm_neon_vshiftu, dl,
5107 MVT::i32),
Bob Wilson7d471332010-11-18 21:16:28 +00005108 N->getOperand(0), N->getOperand(1));
5109
5110 assert((N->getOpcode() == ISD::SRA ||
5111 N->getOpcode() == ISD::SRL) && "unexpected vector shift opcode");
5112
5113 // NEON uses the same intrinsics for both left and right shifts. For
5114 // right shifts, the shift amounts are negative, so negate the vector of
5115 // shift amounts.
5116 EVT ShiftVT = N->getOperand(1).getValueType();
5117 SDValue NegatedCount = DAG.getNode(ISD::SUB, dl, ShiftVT,
5118 getZeroVector(ShiftVT, DAG, dl),
5119 N->getOperand(1));
5120 Intrinsic::ID vshiftInt = (N->getOpcode() == ISD::SRA ?
5121 Intrinsic::arm_neon_vshifts :
5122 Intrinsic::arm_neon_vshiftu);
5123 return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, VT,
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +00005124 DAG.getConstant(vshiftInt, dl, MVT::i32),
Bob Wilson7d471332010-11-18 21:16:28 +00005125 N->getOperand(0), NegatedCount);
5126}
5127
5128static SDValue Expand64BitShift(SDNode *N, SelectionDAG &DAG,
5129 const ARMSubtarget *ST) {
5130 EVT VT = N->getValueType(0);
Andrew Trickef9de2a2013-05-25 02:42:55 +00005131 SDLoc dl(N);
Bob Wilson2e076c42009-06-22 23:27:02 +00005132
Eli Friedman682d8c12009-08-22 03:13:10 +00005133 // We can get here for a node like i32 = ISD::SHL i32, i64
5134 if (VT != MVT::i64)
5135 return SDValue();
5136
5137 assert((N->getOpcode() == ISD::SRL || N->getOpcode() == ISD::SRA) &&
Chris Lattnerf81d5882007-11-24 07:07:01 +00005138 "Unknown shift to lower!");
Duncan Sands6ed40142008-12-01 11:39:25 +00005139
Chris Lattnerf81d5882007-11-24 07:07:01 +00005140 // We only lower SRA, SRL of 1 here, all others use generic lowering.
Artyom Skrobov314ee042015-11-25 19:41:11 +00005141 if (!isOneConstant(N->getOperand(1)))
Duncan Sands6ed40142008-12-01 11:39:25 +00005142 return SDValue();
Bob Wilson7117a912009-03-20 22:42:55 +00005143
Chris Lattnerf81d5882007-11-24 07:07:01 +00005144 // If we are in thumb mode, we don't have RRX.
David Goodwin22c2fba2009-07-08 23:10:31 +00005145 if (ST->isThumb1Only()) return SDValue();
Bob Wilson7117a912009-03-20 22:42:55 +00005146
Chris Lattnerf81d5882007-11-24 07:07:01 +00005147 // Okay, we have a 64-bit SRA or SRL of 1. Lower this to an RRX expr.
Owen Anderson9f944592009-08-11 20:47:22 +00005148 SDValue Lo = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, MVT::i32, N->getOperand(0),
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +00005149 DAG.getConstant(0, dl, MVT::i32));
Owen Anderson9f944592009-08-11 20:47:22 +00005150 SDValue Hi = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, MVT::i32, N->getOperand(0),
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +00005151 DAG.getConstant(1, dl, MVT::i32));
Bob Wilson7117a912009-03-20 22:42:55 +00005152
Chris Lattnerf81d5882007-11-24 07:07:01 +00005153 // First, build a SRA_FLAG/SRL_FLAG op, which shifts the top part by one and
5154 // captures the result into a carry flag.
5155 unsigned Opc = N->getOpcode() == ISD::SRL ? ARMISD::SRL_FLAG:ARMISD::SRA_FLAG;
Craig Topper48d114b2014-04-26 18:35:24 +00005156 Hi = DAG.getNode(Opc, dl, DAG.getVTList(MVT::i32, MVT::Glue), Hi);
Bob Wilson7117a912009-03-20 22:42:55 +00005157
Chris Lattnerf81d5882007-11-24 07:07:01 +00005158 // The low part is an ARMISD::RRX operand, which shifts the carry in.
Owen Anderson9f944592009-08-11 20:47:22 +00005159 Lo = DAG.getNode(ARMISD::RRX, dl, MVT::i32, Lo, Hi.getValue(1));
Bob Wilson7117a912009-03-20 22:42:55 +00005160
Chris Lattnerf81d5882007-11-24 07:07:01 +00005161 // Merge the pieces into a single i64 value.
Owen Anderson9f944592009-08-11 20:47:22 +00005162 return DAG.getNode(ISD::BUILD_PAIR, dl, MVT::i64, Lo, Hi);
Chris Lattnerf81d5882007-11-24 07:07:01 +00005163}
5164
Bob Wilson2e076c42009-06-22 23:27:02 +00005165static SDValue LowerVSETCC(SDValue Op, SelectionDAG &DAG) {
5166 SDValue TmpOp0, TmpOp1;
5167 bool Invert = false;
5168 bool Swap = false;
5169 unsigned Opc = 0;
5170
5171 SDValue Op0 = Op.getOperand(0);
5172 SDValue Op1 = Op.getOperand(1);
5173 SDValue CC = Op.getOperand(2);
Tim Northover45aa89c2015-02-08 00:50:47 +00005174 EVT CmpVT = Op0.getValueType().changeVectorElementTypeToInteger();
Owen Anderson53aa7a92009-08-10 22:56:29 +00005175 EVT VT = Op.getValueType();
Bob Wilson2e076c42009-06-22 23:27:02 +00005176 ISD::CondCode SetCCOpcode = cast<CondCodeSDNode>(CC)->get();
Andrew Trickef9de2a2013-05-25 02:42:55 +00005177 SDLoc dl(Op);
Bob Wilson2e076c42009-06-22 23:27:02 +00005178
James Molloybf170092015-08-20 16:33:44 +00005179 if (CmpVT.getVectorElementType() == MVT::i64)
5180 // 64-bit comparisons are not legal. We've marked SETCC as non-Custom,
5181 // but it's possible that our operands are 64-bit but our result is 32-bit.
5182 // Bail in this case.
5183 return SDValue();
5184
Oliver Stannard51b1d462014-08-21 12:50:31 +00005185 if (Op1.getValueType().isFloatingPoint()) {
Bob Wilson2e076c42009-06-22 23:27:02 +00005186 switch (SetCCOpcode) {
David Blaikie46a9f012012-01-20 21:51:11 +00005187 default: llvm_unreachable("Illegal FP comparison");
Bob Wilson2e076c42009-06-22 23:27:02 +00005188 case ISD::SETUNE:
Justin Bognerb03fd122016-08-17 05:10:15 +00005189 case ISD::SETNE: Invert = true; LLVM_FALLTHROUGH;
Bob Wilson2e076c42009-06-22 23:27:02 +00005190 case ISD::SETOEQ:
5191 case ISD::SETEQ: Opc = ARMISD::VCEQ; break;
5192 case ISD::SETOLT:
Justin Bognerb03fd122016-08-17 05:10:15 +00005193 case ISD::SETLT: Swap = true; LLVM_FALLTHROUGH;
Bob Wilson2e076c42009-06-22 23:27:02 +00005194 case ISD::SETOGT:
5195 case ISD::SETGT: Opc = ARMISD::VCGT; break;
5196 case ISD::SETOLE:
Justin Bognerb03fd122016-08-17 05:10:15 +00005197 case ISD::SETLE: Swap = true; LLVM_FALLTHROUGH;
Bob Wilson2e076c42009-06-22 23:27:02 +00005198 case ISD::SETOGE:
5199 case ISD::SETGE: Opc = ARMISD::VCGE; break;
Justin Bognerb03fd122016-08-17 05:10:15 +00005200 case ISD::SETUGE: Swap = true; LLVM_FALLTHROUGH;
Bob Wilson2e076c42009-06-22 23:27:02 +00005201 case ISD::SETULE: Invert = true; Opc = ARMISD::VCGT; break;
Justin Bognerb03fd122016-08-17 05:10:15 +00005202 case ISD::SETUGT: Swap = true; LLVM_FALLTHROUGH;
Bob Wilson2e076c42009-06-22 23:27:02 +00005203 case ISD::SETULT: Invert = true; Opc = ARMISD::VCGE; break;
Justin Bognerb03fd122016-08-17 05:10:15 +00005204 case ISD::SETUEQ: Invert = true; LLVM_FALLTHROUGH;
Bob Wilson2e076c42009-06-22 23:27:02 +00005205 case ISD::SETONE:
5206 // Expand this to (OLT | OGT).
5207 TmpOp0 = Op0;
5208 TmpOp1 = Op1;
5209 Opc = ISD::OR;
Tim Northover45aa89c2015-02-08 00:50:47 +00005210 Op0 = DAG.getNode(ARMISD::VCGT, dl, CmpVT, TmpOp1, TmpOp0);
5211 Op1 = DAG.getNode(ARMISD::VCGT, dl, CmpVT, TmpOp0, TmpOp1);
Bob Wilson2e076c42009-06-22 23:27:02 +00005212 break;
Justin Bognerb03fd122016-08-17 05:10:15 +00005213 case ISD::SETUO:
5214 Invert = true;
5215 LLVM_FALLTHROUGH;
Bob Wilson2e076c42009-06-22 23:27:02 +00005216 case ISD::SETO:
5217 // Expand this to (OLT | OGE).
5218 TmpOp0 = Op0;
5219 TmpOp1 = Op1;
5220 Opc = ISD::OR;
Tim Northover45aa89c2015-02-08 00:50:47 +00005221 Op0 = DAG.getNode(ARMISD::VCGT, dl, CmpVT, TmpOp1, TmpOp0);
5222 Op1 = DAG.getNode(ARMISD::VCGE, dl, CmpVT, TmpOp0, TmpOp1);
Bob Wilson2e076c42009-06-22 23:27:02 +00005223 break;
5224 }
5225 } else {
5226 // Integer comparisons.
5227 switch (SetCCOpcode) {
David Blaikie46a9f012012-01-20 21:51:11 +00005228 default: llvm_unreachable("Illegal integer comparison");
Bob Wilson2e076c42009-06-22 23:27:02 +00005229 case ISD::SETNE: Invert = true;
5230 case ISD::SETEQ: Opc = ARMISD::VCEQ; break;
5231 case ISD::SETLT: Swap = true;
5232 case ISD::SETGT: Opc = ARMISD::VCGT; break;
5233 case ISD::SETLE: Swap = true;
5234 case ISD::SETGE: Opc = ARMISD::VCGE; break;
5235 case ISD::SETULT: Swap = true;
5236 case ISD::SETUGT: Opc = ARMISD::VCGTU; break;
5237 case ISD::SETULE: Swap = true;
5238 case ISD::SETUGE: Opc = ARMISD::VCGEU; break;
5239 }
5240
Nick Lewyckya21d3da2009-07-08 03:04:38 +00005241 // Detect VTST (Vector Test Bits) = icmp ne (and (op0, op1), zero).
Bob Wilson2e076c42009-06-22 23:27:02 +00005242 if (Opc == ARMISD::VCEQ) {
5243
5244 SDValue AndOp;
5245 if (ISD::isBuildVectorAllZeros(Op1.getNode()))
5246 AndOp = Op0;
5247 else if (ISD::isBuildVectorAllZeros(Op0.getNode()))
5248 AndOp = Op1;
5249
5250 // Ignore bitconvert.
Wesley Peck527da1b2010-11-23 03:31:01 +00005251 if (AndOp.getNode() && AndOp.getOpcode() == ISD::BITCAST)
Bob Wilson2e076c42009-06-22 23:27:02 +00005252 AndOp = AndOp.getOperand(0);
5253
5254 if (AndOp.getNode() && AndOp.getOpcode() == ISD::AND) {
5255 Opc = ARMISD::VTST;
Tim Northover45aa89c2015-02-08 00:50:47 +00005256 Op0 = DAG.getNode(ISD::BITCAST, dl, CmpVT, AndOp.getOperand(0));
5257 Op1 = DAG.getNode(ISD::BITCAST, dl, CmpVT, AndOp.getOperand(1));
Bob Wilson2e076c42009-06-22 23:27:02 +00005258 Invert = !Invert;
5259 }
5260 }
5261 }
5262
5263 if (Swap)
5264 std::swap(Op0, Op1);
5265
Owen Andersonc7baee32010-11-08 23:21:22 +00005266 // If one of the operands is a constant vector zero, attempt to fold the
5267 // comparison to a specialized compare-against-zero form.
5268 SDValue SingleOp;
5269 if (ISD::isBuildVectorAllZeros(Op1.getNode()))
5270 SingleOp = Op0;
5271 else if (ISD::isBuildVectorAllZeros(Op0.getNode())) {
5272 if (Opc == ARMISD::VCGE)
5273 Opc = ARMISD::VCLEZ;
5274 else if (Opc == ARMISD::VCGT)
5275 Opc = ARMISD::VCLTZ;
5276 SingleOp = Op1;
5277 }
Wesley Peck527da1b2010-11-23 03:31:01 +00005278
Owen Andersonc7baee32010-11-08 23:21:22 +00005279 SDValue Result;
5280 if (SingleOp.getNode()) {
5281 switch (Opc) {
5282 case ARMISD::VCEQ:
Tim Northover45aa89c2015-02-08 00:50:47 +00005283 Result = DAG.getNode(ARMISD::VCEQZ, dl, CmpVT, SingleOp); break;
Owen Andersonc7baee32010-11-08 23:21:22 +00005284 case ARMISD::VCGE:
Tim Northover45aa89c2015-02-08 00:50:47 +00005285 Result = DAG.getNode(ARMISD::VCGEZ, dl, CmpVT, SingleOp); break;
Owen Andersonc7baee32010-11-08 23:21:22 +00005286 case ARMISD::VCLEZ:
Tim Northover45aa89c2015-02-08 00:50:47 +00005287 Result = DAG.getNode(ARMISD::VCLEZ, dl, CmpVT, SingleOp); break;
Owen Andersonc7baee32010-11-08 23:21:22 +00005288 case ARMISD::VCGT:
Tim Northover45aa89c2015-02-08 00:50:47 +00005289 Result = DAG.getNode(ARMISD::VCGTZ, dl, CmpVT, SingleOp); break;
Owen Andersonc7baee32010-11-08 23:21:22 +00005290 case ARMISD::VCLTZ:
Tim Northover45aa89c2015-02-08 00:50:47 +00005291 Result = DAG.getNode(ARMISD::VCLTZ, dl, CmpVT, SingleOp); break;
Owen Andersonc7baee32010-11-08 23:21:22 +00005292 default:
Tim Northover45aa89c2015-02-08 00:50:47 +00005293 Result = DAG.getNode(Opc, dl, CmpVT, Op0, Op1);
Owen Andersonc7baee32010-11-08 23:21:22 +00005294 }
5295 } else {
Tim Northover45aa89c2015-02-08 00:50:47 +00005296 Result = DAG.getNode(Opc, dl, CmpVT, Op0, Op1);
Owen Andersonc7baee32010-11-08 23:21:22 +00005297 }
Bob Wilson2e076c42009-06-22 23:27:02 +00005298
Tim Northover45aa89c2015-02-08 00:50:47 +00005299 Result = DAG.getSExtOrTrunc(Result, dl, VT);
5300
Bob Wilson2e076c42009-06-22 23:27:02 +00005301 if (Invert)
5302 Result = DAG.getNOT(dl, Result, VT);
5303
5304 return Result;
5305}
5306
Peter Collingbourne86b9fbe2016-03-21 18:00:02 +00005307static SDValue LowerSETCCE(SDValue Op, SelectionDAG &DAG) {
5308 SDValue LHS = Op.getOperand(0);
5309 SDValue RHS = Op.getOperand(1);
5310 SDValue Carry = Op.getOperand(2);
5311 SDValue Cond = Op.getOperand(3);
5312 SDLoc DL(Op);
5313
5314 assert(LHS.getSimpleValueType().isInteger() && "SETCCE is integer only.");
5315
5316 assert(Carry.getOpcode() != ISD::CARRY_FALSE);
5317 SDVTList VTs = DAG.getVTList(LHS.getValueType(), MVT::i32);
5318 SDValue Cmp = DAG.getNode(ARMISD::SUBE, DL, VTs, LHS, RHS, Carry);
5319
5320 SDValue FVal = DAG.getConstant(0, DL, MVT::i32);
5321 SDValue TVal = DAG.getConstant(1, DL, MVT::i32);
5322 SDValue ARMcc = DAG.getConstant(
5323 IntCCToARMCC(cast<CondCodeSDNode>(Cond)->get()), DL, MVT::i32);
5324 SDValue CCR = DAG.getRegister(ARM::CPSR, MVT::i32);
5325 SDValue Chain = DAG.getCopyToReg(DAG.getEntryNode(), DL, ARM::CPSR,
5326 Cmp.getValue(1), SDValue());
5327 return DAG.getNode(ARMISD::CMOV, DL, Op.getValueType(), FVal, TVal, ARMcc,
5328 CCR, Chain.getValue(1));
5329}
5330
Bob Wilson5b2b5042010-06-14 22:19:57 +00005331/// isNEONModifiedImm - Check if the specified splat value corresponds to a
5332/// valid vector constant for a NEON instruction with a "modified immediate"
Bob Wilsona3f19012010-07-13 21:16:48 +00005333/// operand (e.g., VMOV). If so, return the encoded value.
Bob Wilson5b2b5042010-06-14 22:19:57 +00005334static SDValue isNEONModifiedImm(uint64_t SplatBits, uint64_t SplatUndef,
5335 unsigned SplatBitSize, SelectionDAG &DAG,
Benjamin Kramerbdc49562016-06-12 15:39:02 +00005336 const SDLoc &dl, EVT &VT, bool is128Bits,
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +00005337 NEONModImmType type) {
Bob Wilsonc1c6f472010-07-13 04:44:34 +00005338 unsigned OpCmode, Imm;
Bob Wilson6eae5202010-06-11 21:34:50 +00005339
Bob Wilsonf3f7a772010-06-15 19:05:35 +00005340 // SplatBitSize is set to the smallest size that splats the vector, so a
5341 // zero vector will always have SplatBitSize == 8. However, NEON modified
5342 // immediate instructions others than VMOV do not support the 8-bit encoding
5343 // of a zero vector, and the default encoding of zero is supposed to be the
5344 // 32-bit version.
5345 if (SplatBits == 0)
5346 SplatBitSize = 32;
5347
Bob Wilson2e076c42009-06-22 23:27:02 +00005348 switch (SplatBitSize) {
5349 case 8:
Owen Andersona4076922010-11-05 21:57:54 +00005350 if (type != VMOVModImm)
Bob Wilsonbad47f62010-07-14 06:31:50 +00005351 return SDValue();
Bob Wilson6eae5202010-06-11 21:34:50 +00005352 // Any 1-byte value is OK. Op=0, Cmode=1110.
Bob Wilson2e076c42009-06-22 23:27:02 +00005353 assert((SplatBits & ~0xff) == 0 && "one byte splat value is too big");
Bob Wilsonc1c6f472010-07-13 04:44:34 +00005354 OpCmode = 0xe;
Bob Wilson6eae5202010-06-11 21:34:50 +00005355 Imm = SplatBits;
Bob Wilsona3f19012010-07-13 21:16:48 +00005356 VT = is128Bits ? MVT::v16i8 : MVT::v8i8;
Bob Wilson6eae5202010-06-11 21:34:50 +00005357 break;
Bob Wilson2e076c42009-06-22 23:27:02 +00005358
5359 case 16:
5360 // NEON's 16-bit VMOV supports splat values where only one byte is nonzero.
Bob Wilsona3f19012010-07-13 21:16:48 +00005361 VT = is128Bits ? MVT::v8i16 : MVT::v4i16;
Bob Wilson6eae5202010-06-11 21:34:50 +00005362 if ((SplatBits & ~0xff) == 0) {
5363 // Value = 0x00nn: Op=x, Cmode=100x.
Bob Wilsonc1c6f472010-07-13 04:44:34 +00005364 OpCmode = 0x8;
Bob Wilson6eae5202010-06-11 21:34:50 +00005365 Imm = SplatBits;
5366 break;
5367 }
5368 if ((SplatBits & ~0xff00) == 0) {
5369 // Value = 0xnn00: Op=x, Cmode=101x.
Bob Wilsonc1c6f472010-07-13 04:44:34 +00005370 OpCmode = 0xa;
Bob Wilson6eae5202010-06-11 21:34:50 +00005371 Imm = SplatBits >> 8;
5372 break;
5373 }
5374 return SDValue();
Bob Wilson2e076c42009-06-22 23:27:02 +00005375
5376 case 32:
5377 // NEON's 32-bit VMOV supports splat values where:
5378 // * only one byte is nonzero, or
5379 // * the least significant byte is 0xff and the second byte is nonzero, or
5380 // * the least significant 2 bytes are 0xff and the third is nonzero.
Bob Wilsona3f19012010-07-13 21:16:48 +00005381 VT = is128Bits ? MVT::v4i32 : MVT::v2i32;
Bob Wilson6eae5202010-06-11 21:34:50 +00005382 if ((SplatBits & ~0xff) == 0) {
5383 // Value = 0x000000nn: Op=x, Cmode=000x.
Bob Wilsonc1c6f472010-07-13 04:44:34 +00005384 OpCmode = 0;
Bob Wilson6eae5202010-06-11 21:34:50 +00005385 Imm = SplatBits;
5386 break;
5387 }
5388 if ((SplatBits & ~0xff00) == 0) {
5389 // Value = 0x0000nn00: Op=x, Cmode=001x.
Bob Wilsonc1c6f472010-07-13 04:44:34 +00005390 OpCmode = 0x2;
Bob Wilson6eae5202010-06-11 21:34:50 +00005391 Imm = SplatBits >> 8;
5392 break;
5393 }
5394 if ((SplatBits & ~0xff0000) == 0) {
5395 // Value = 0x00nn0000: Op=x, Cmode=010x.
Bob Wilsonc1c6f472010-07-13 04:44:34 +00005396 OpCmode = 0x4;
Bob Wilson6eae5202010-06-11 21:34:50 +00005397 Imm = SplatBits >> 16;
5398 break;
5399 }
5400 if ((SplatBits & ~0xff000000) == 0) {
5401 // Value = 0xnn000000: Op=x, Cmode=011x.
Bob Wilsonc1c6f472010-07-13 04:44:34 +00005402 OpCmode = 0x6;
Bob Wilson6eae5202010-06-11 21:34:50 +00005403 Imm = SplatBits >> 24;
5404 break;
5405 }
Bob Wilson2e076c42009-06-22 23:27:02 +00005406
Owen Andersona4076922010-11-05 21:57:54 +00005407 // cmode == 0b1100 and cmode == 0b1101 are not supported for VORR or VBIC
5408 if (type == OtherModImm) return SDValue();
5409
Bob Wilson2e076c42009-06-22 23:27:02 +00005410 if ((SplatBits & ~0xffff) == 0 &&
Bob Wilson6eae5202010-06-11 21:34:50 +00005411 ((SplatBits | SplatUndef) & 0xff) == 0xff) {
5412 // Value = 0x0000nnff: Op=x, Cmode=1100.
Bob Wilsonc1c6f472010-07-13 04:44:34 +00005413 OpCmode = 0xc;
Bob Wilson6eae5202010-06-11 21:34:50 +00005414 Imm = SplatBits >> 8;
Bob Wilson6eae5202010-06-11 21:34:50 +00005415 break;
5416 }
Bob Wilson2e076c42009-06-22 23:27:02 +00005417
5418 if ((SplatBits & ~0xffffff) == 0 &&
Bob Wilson6eae5202010-06-11 21:34:50 +00005419 ((SplatBits | SplatUndef) & 0xffff) == 0xffff) {
5420 // Value = 0x00nnffff: Op=x, Cmode=1101.
Bob Wilsonc1c6f472010-07-13 04:44:34 +00005421 OpCmode = 0xd;
Bob Wilson6eae5202010-06-11 21:34:50 +00005422 Imm = SplatBits >> 16;
Bob Wilson6eae5202010-06-11 21:34:50 +00005423 break;
5424 }
Bob Wilson2e076c42009-06-22 23:27:02 +00005425
5426 // Note: there are a few 32-bit splat values (specifically: 00ffff00,
5427 // ff000000, ff0000ff, and ffff00ff) that are valid for VMOV.I64 but not
5428 // VMOV.I32. A (very) minor optimization would be to replicate the value
5429 // and fall through here to test for a valid 64-bit splat. But, then the
5430 // caller would also need to check and handle the change in size.
Bob Wilson6eae5202010-06-11 21:34:50 +00005431 return SDValue();
Bob Wilson2e076c42009-06-22 23:27:02 +00005432
5433 case 64: {
Owen Andersona4076922010-11-05 21:57:54 +00005434 if (type != VMOVModImm)
Bob Wilsonf3f7a772010-06-15 19:05:35 +00005435 return SDValue();
Bob Wilsonbad47f62010-07-14 06:31:50 +00005436 // NEON has a 64-bit VMOV splat where each byte is either 0 or 0xff.
Bob Wilson2e076c42009-06-22 23:27:02 +00005437 uint64_t BitMask = 0xff;
5438 uint64_t Val = 0;
Bob Wilson6eae5202010-06-11 21:34:50 +00005439 unsigned ImmMask = 1;
5440 Imm = 0;
Bob Wilson2e076c42009-06-22 23:27:02 +00005441 for (int ByteNum = 0; ByteNum < 8; ++ByteNum) {
Bob Wilson6eae5202010-06-11 21:34:50 +00005442 if (((SplatBits | SplatUndef) & BitMask) == BitMask) {
Bob Wilson2e076c42009-06-22 23:27:02 +00005443 Val |= BitMask;
Bob Wilson6eae5202010-06-11 21:34:50 +00005444 Imm |= ImmMask;
5445 } else if ((SplatBits & BitMask) != 0) {
Bob Wilson2e076c42009-06-22 23:27:02 +00005446 return SDValue();
Bob Wilson6eae5202010-06-11 21:34:50 +00005447 }
Bob Wilson2e076c42009-06-22 23:27:02 +00005448 BitMask <<= 8;
Bob Wilson6eae5202010-06-11 21:34:50 +00005449 ImmMask <<= 1;
Bob Wilson2e076c42009-06-22 23:27:02 +00005450 }
Christian Pirker6f81e752014-06-23 18:05:53 +00005451
Mehdi Aminiffc14022015-07-08 01:00:38 +00005452 if (DAG.getDataLayout().isBigEndian())
Christian Pirker6f81e752014-06-23 18:05:53 +00005453 // swap higher and lower 32 bit word
5454 Imm = ((Imm & 0xf) << 4) | ((Imm & 0xf0) >> 4);
5455
Bob Wilson6eae5202010-06-11 21:34:50 +00005456 // Op=1, Cmode=1110.
Bob Wilsonc1c6f472010-07-13 04:44:34 +00005457 OpCmode = 0x1e;
Bob Wilsona3f19012010-07-13 21:16:48 +00005458 VT = is128Bits ? MVT::v2i64 : MVT::v1i64;
Bob Wilson2e076c42009-06-22 23:27:02 +00005459 break;
5460 }
5461
Bob Wilson6eae5202010-06-11 21:34:50 +00005462 default:
Bob Wilson0ae08932010-06-19 05:32:09 +00005463 llvm_unreachable("unexpected size for isNEONModifiedImm");
Bob Wilson6eae5202010-06-11 21:34:50 +00005464 }
5465
Bob Wilsona3f19012010-07-13 21:16:48 +00005466 unsigned EncodedVal = ARM_AM::createNEONModImm(OpCmode, Imm);
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +00005467 return DAG.getTargetConstant(EncodedVal, dl, MVT::i32);
Bob Wilson2e076c42009-06-22 23:27:02 +00005468}
5469
Lang Hames591cdaf2012-03-29 21:56:11 +00005470SDValue ARMTargetLowering::LowerConstantFP(SDValue Op, SelectionDAG &DAG,
5471 const ARMSubtarget *ST) const {
Tim Northoverf79c3a52013-08-20 08:57:11 +00005472 if (!ST->hasVFP3())
Lang Hames591cdaf2012-03-29 21:56:11 +00005473 return SDValue();
5474
Tim Northoverf79c3a52013-08-20 08:57:11 +00005475 bool IsDouble = Op.getValueType() == MVT::f64;
Lang Hames591cdaf2012-03-29 21:56:11 +00005476 ConstantFPSDNode *CFP = cast<ConstantFPSDNode>(Op);
Lang Hames591cdaf2012-03-29 21:56:11 +00005477
Oliver Stannard51b1d462014-08-21 12:50:31 +00005478 // Use the default (constant pool) lowering for double constants when we have
5479 // an SP-only FPU
5480 if (IsDouble && Subtarget->isFPOnlySP())
5481 return SDValue();
5482
Lang Hames591cdaf2012-03-29 21:56:11 +00005483 // Try splatting with a VMOV.f32...
Benjamin Kramer46e38f32016-06-08 10:01:20 +00005484 const APFloat &FPVal = CFP->getValueAPF();
Tim Northoverf79c3a52013-08-20 08:57:11 +00005485 int ImmVal = IsDouble ? ARM_AM::getFP64Imm(FPVal) : ARM_AM::getFP32Imm(FPVal);
5486
Lang Hames591cdaf2012-03-29 21:56:11 +00005487 if (ImmVal != -1) {
Tim Northoverf79c3a52013-08-20 08:57:11 +00005488 if (IsDouble || !ST->useNEONForSinglePrecisionFP()) {
5489 // We have code in place to select a valid ConstantFP already, no need to
5490 // do any mangling.
5491 return Op;
5492 }
5493
5494 // It's a float and we are trying to use NEON operations where
5495 // possible. Lower it to a splat followed by an extract.
Andrew Trickef9de2a2013-05-25 02:42:55 +00005496 SDLoc DL(Op);
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +00005497 SDValue NewVal = DAG.getTargetConstant(ImmVal, DL, MVT::i32);
Lang Hames591cdaf2012-03-29 21:56:11 +00005498 SDValue VecConstant = DAG.getNode(ARMISD::VMOVFPIMM, DL, MVT::v2f32,
5499 NewVal);
5500 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::f32, VecConstant,
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +00005501 DAG.getConstant(0, DL, MVT::i32));
Lang Hames591cdaf2012-03-29 21:56:11 +00005502 }
5503
Tim Northoverf79c3a52013-08-20 08:57:11 +00005504 // The rest of our options are NEON only, make sure that's allowed before
5505 // proceeding..
5506 if (!ST->hasNEON() || (!IsDouble && !ST->useNEONForSinglePrecisionFP()))
5507 return SDValue();
5508
Lang Hames591cdaf2012-03-29 21:56:11 +00005509 EVT VMovVT;
Tim Northoverf79c3a52013-08-20 08:57:11 +00005510 uint64_t iVal = FPVal.bitcastToAPInt().getZExtValue();
5511
5512 // It wouldn't really be worth bothering for doubles except for one very
5513 // important value, which does happen to match: 0.0. So make sure we don't do
5514 // anything stupid.
5515 if (IsDouble && (iVal & 0xffffffff) != (iVal >> 32))
5516 return SDValue();
5517
5518 // Try a VMOV.i32 (FIXME: i8, i16, or i64 could work too).
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +00005519 SDValue NewVal = isNEONModifiedImm(iVal & 0xffffffffU, 0, 32, DAG, SDLoc(Op),
5520 VMovVT, false, VMOVModImm);
Lang Hames591cdaf2012-03-29 21:56:11 +00005521 if (NewVal != SDValue()) {
Andrew Trickef9de2a2013-05-25 02:42:55 +00005522 SDLoc DL(Op);
Lang Hames591cdaf2012-03-29 21:56:11 +00005523 SDValue VecConstant = DAG.getNode(ARMISD::VMOVIMM, DL, VMovVT,
5524 NewVal);
Tim Northoverf79c3a52013-08-20 08:57:11 +00005525 if (IsDouble)
5526 return DAG.getNode(ISD::BITCAST, DL, MVT::f64, VecConstant);
5527
5528 // It's a float: cast and extract a vector element.
Lang Hames591cdaf2012-03-29 21:56:11 +00005529 SDValue VecFConstant = DAG.getNode(ISD::BITCAST, DL, MVT::v2f32,
5530 VecConstant);
5531 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::f32, VecFConstant,
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +00005532 DAG.getConstant(0, DL, MVT::i32));
Lang Hames591cdaf2012-03-29 21:56:11 +00005533 }
5534
5535 // Finally, try a VMVN.i32
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +00005536 NewVal = isNEONModifiedImm(~iVal & 0xffffffffU, 0, 32, DAG, SDLoc(Op), VMovVT,
Tim Northoverf79c3a52013-08-20 08:57:11 +00005537 false, VMVNModImm);
Lang Hames591cdaf2012-03-29 21:56:11 +00005538 if (NewVal != SDValue()) {
Andrew Trickef9de2a2013-05-25 02:42:55 +00005539 SDLoc DL(Op);
Lang Hames591cdaf2012-03-29 21:56:11 +00005540 SDValue VecConstant = DAG.getNode(ARMISD::VMVNIMM, DL, VMovVT, NewVal);
Tim Northoverf79c3a52013-08-20 08:57:11 +00005541
5542 if (IsDouble)
5543 return DAG.getNode(ISD::BITCAST, DL, MVT::f64, VecConstant);
5544
5545 // It's a float: cast and extract a vector element.
Lang Hames591cdaf2012-03-29 21:56:11 +00005546 SDValue VecFConstant = DAG.getNode(ISD::BITCAST, DL, MVT::v2f32,
5547 VecConstant);
5548 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::f32, VecFConstant,
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +00005549 DAG.getConstant(0, DL, MVT::i32));
Lang Hames591cdaf2012-03-29 21:56:11 +00005550 }
5551
5552 return SDValue();
5553}
5554
Quentin Colombet8e1fe842012-11-02 21:32:17 +00005555// check if an VEXT instruction can handle the shuffle mask when the
5556// vector sources of the shuffle are the same.
5557static bool isSingletonVEXTMask(ArrayRef<int> M, EVT VT, unsigned &Imm) {
5558 unsigned NumElts = VT.getVectorNumElements();
5559
5560 // Assume that the first shuffle index is not UNDEF. Fail if it is.
5561 if (M[0] < 0)
5562 return false;
5563
5564 Imm = M[0];
5565
5566 // If this is a VEXT shuffle, the immediate value is the index of the first
5567 // element. The other shuffle indices must be the successive elements after
5568 // the first one.
5569 unsigned ExpectedElt = Imm;
5570 for (unsigned i = 1; i < NumElts; ++i) {
5571 // Increment the expected index. If it wraps around, just follow it
5572 // back to index zero and keep going.
5573 ++ExpectedElt;
5574 if (ExpectedElt == NumElts)
5575 ExpectedElt = 0;
5576
5577 if (M[i] < 0) continue; // ignore UNDEF indices
5578 if (ExpectedElt != static_cast<unsigned>(M[i]))
5579 return false;
5580 }
5581
5582 return true;
5583}
5584
Lang Hames591cdaf2012-03-29 21:56:11 +00005585
Benjamin Kramer339ced42012-01-15 13:16:05 +00005586static bool isVEXTMask(ArrayRef<int> M, EVT VT,
Anton Korobeynikovc32e99e2009-08-21 12:40:07 +00005587 bool &ReverseVEXT, unsigned &Imm) {
Bob Wilson32cd8552009-08-19 17:03:43 +00005588 unsigned NumElts = VT.getVectorNumElements();
5589 ReverseVEXT = false;
Bob Wilson411dfad2010-08-17 05:54:34 +00005590
5591 // Assume that the first shuffle index is not UNDEF. Fail if it is.
5592 if (M[0] < 0)
5593 return false;
5594
Anton Korobeynikovc32e99e2009-08-21 12:40:07 +00005595 Imm = M[0];
Bob Wilson32cd8552009-08-19 17:03:43 +00005596
5597 // If this is a VEXT shuffle, the immediate value is the index of the first
5598 // element. The other shuffle indices must be the successive elements after
5599 // the first one.
5600 unsigned ExpectedElt = Imm;
5601 for (unsigned i = 1; i < NumElts; ++i) {
Bob Wilson32cd8552009-08-19 17:03:43 +00005602 // Increment the expected index. If it wraps around, it may still be
5603 // a VEXT but the source vectors must be swapped.
5604 ExpectedElt += 1;
5605 if (ExpectedElt == NumElts * 2) {
5606 ExpectedElt = 0;
5607 ReverseVEXT = true;
5608 }
5609
Bob Wilson411dfad2010-08-17 05:54:34 +00005610 if (M[i] < 0) continue; // ignore UNDEF indices
Anton Korobeynikovc32e99e2009-08-21 12:40:07 +00005611 if (ExpectedElt != static_cast<unsigned>(M[i]))
Bob Wilson32cd8552009-08-19 17:03:43 +00005612 return false;
5613 }
5614
5615 // Adjust the index value if the source operands will be swapped.
5616 if (ReverseVEXT)
5617 Imm -= NumElts;
5618
Bob Wilson32cd8552009-08-19 17:03:43 +00005619 return true;
5620}
5621
Bob Wilson8a37bbe2009-07-26 00:39:34 +00005622/// isVREVMask - Check if a vector shuffle corresponds to a VREV
5623/// instruction with the specified blocksize. (The order of the elements
5624/// within each block of the vector is reversed.)
Benjamin Kramer339ced42012-01-15 13:16:05 +00005625static bool isVREVMask(ArrayRef<int> M, EVT VT, unsigned BlockSize) {
Bob Wilson8a37bbe2009-07-26 00:39:34 +00005626 assert((BlockSize==16 || BlockSize==32 || BlockSize==64) &&
5627 "Only possible block sizes for VREV are: 16, 32, 64");
5628
Bob Wilson8a37bbe2009-07-26 00:39:34 +00005629 unsigned EltSz = VT.getVectorElementType().getSizeInBits();
Bob Wilson854530a2009-10-21 21:36:27 +00005630 if (EltSz == 64)
5631 return false;
5632
5633 unsigned NumElts = VT.getVectorNumElements();
Anton Korobeynikovc32e99e2009-08-21 12:40:07 +00005634 unsigned BlockElts = M[0] + 1;
Bob Wilson411dfad2010-08-17 05:54:34 +00005635 // If the first shuffle index is UNDEF, be optimistic.
5636 if (M[0] < 0)
5637 BlockElts = BlockSize / EltSz;
Bob Wilson8a37bbe2009-07-26 00:39:34 +00005638
5639 if (BlockSize <= EltSz || BlockSize != BlockElts * EltSz)
5640 return false;
5641
5642 for (unsigned i = 0; i < NumElts; ++i) {
Bob Wilson411dfad2010-08-17 05:54:34 +00005643 if (M[i] < 0) continue; // ignore UNDEF indices
5644 if ((unsigned) M[i] != (i - i%BlockElts) + (BlockElts - 1 - i%BlockElts))
Bob Wilson8a37bbe2009-07-26 00:39:34 +00005645 return false;
5646 }
5647
5648 return true;
5649}
5650
Benjamin Kramer339ced42012-01-15 13:16:05 +00005651static bool isVTBLMask(ArrayRef<int> M, EVT VT) {
Bill Wendling865f8b52011-03-15 21:15:20 +00005652 // We can handle <8 x i8> vector shuffles. If the index in the mask is out of
5653 // range, then 0 is placed into the resulting vector. So pretty much any mask
5654 // of 8 elements can work here.
5655 return VT == MVT::v8i8 && M.size() == 8;
5656}
5657
Luke Cheeseman4d45ff22015-07-24 09:57:05 +00005658// Checks whether the shuffle mask represents a vector transpose (VTRN) by
5659// checking that pairs of elements in the shuffle mask represent the same index
5660// in each vector, incrementing the expected index by 2 at each step.
5661// e.g. For v1,v2 of type v4i32 a valid shuffle mask is: [0, 4, 2, 6]
5662// v1={a,b,c,d} => x=shufflevector v1, v2 shufflemask => x={a,e,c,g}
5663// v2={e,f,g,h}
5664// WhichResult gives the offset for each element in the mask based on which
5665// of the two results it belongs to.
5666//
5667// The transpose can be represented either as:
5668// result1 = shufflevector v1, v2, result1_shuffle_mask
5669// result2 = shufflevector v1, v2, result2_shuffle_mask
5670// where v1/v2 and the shuffle masks have the same number of elements
5671// (here WhichResult (see below) indicates which result is being checked)
5672//
5673// or as:
5674// results = shufflevector v1, v2, shuffle_mask
5675// where both results are returned in one vector and the shuffle mask has twice
5676// as many elements as v1/v2 (here WhichResult will always be 0 if true) here we
5677// want to check the low half and high half of the shuffle mask as if it were
5678// the other case
Benjamin Kramer339ced42012-01-15 13:16:05 +00005679static bool isVTRNMask(ArrayRef<int> M, EVT VT, unsigned &WhichResult) {
Bob Wilson854530a2009-10-21 21:36:27 +00005680 unsigned EltSz = VT.getVectorElementType().getSizeInBits();
5681 if (EltSz == 64)
5682 return false;
5683
Bob Wilsona7062312009-08-21 20:54:19 +00005684 unsigned NumElts = VT.getVectorNumElements();
Luke Cheeseman4d45ff22015-07-24 09:57:05 +00005685 if (M.size() != NumElts && M.size() != NumElts*2)
5686 return false;
5687
James Molloy8c995a92015-09-10 08:42:28 +00005688 // If the mask is twice as long as the input vector then we need to check the
5689 // upper and lower parts of the mask with a matching value for WhichResult
5690 // FIXME: A mask with only even values will be rejected in case the first
5691 // element is undefined, e.g. [-1, 4, 2, 6] will be rejected, because only
5692 // M[0] is used to determine WhichResult
Luke Cheeseman4d45ff22015-07-24 09:57:05 +00005693 for (unsigned i = 0; i < M.size(); i += NumElts) {
James Molloy8c995a92015-09-10 08:42:28 +00005694 if (M.size() == NumElts * 2)
5695 WhichResult = i / NumElts;
5696 else
5697 WhichResult = M[i] == 0 ? 0 : 1;
Luke Cheeseman4d45ff22015-07-24 09:57:05 +00005698 for (unsigned j = 0; j < NumElts; j += 2) {
5699 if ((M[i+j] >= 0 && (unsigned) M[i+j] != j + WhichResult) ||
5700 (M[i+j+1] >= 0 && (unsigned) M[i+j+1] != j + NumElts + WhichResult))
5701 return false;
5702 }
Bob Wilsona7062312009-08-21 20:54:19 +00005703 }
Luke Cheeseman4d45ff22015-07-24 09:57:05 +00005704
5705 if (M.size() == NumElts*2)
5706 WhichResult = 0;
5707
Bob Wilsona7062312009-08-21 20:54:19 +00005708 return true;
5709}
5710
Bob Wilson0bbd3072009-12-03 06:40:55 +00005711/// isVTRN_v_undef_Mask - Special case of isVTRNMask for canonical form of
5712/// "vector_shuffle v, v", i.e., "vector_shuffle v, undef".
5713/// Mask is e.g., <0, 0, 2, 2> instead of <0, 4, 2, 6>.
Benjamin Kramer339ced42012-01-15 13:16:05 +00005714static bool isVTRN_v_undef_Mask(ArrayRef<int> M, EVT VT, unsigned &WhichResult){
Bob Wilson0bbd3072009-12-03 06:40:55 +00005715 unsigned EltSz = VT.getVectorElementType().getSizeInBits();
5716 if (EltSz == 64)
5717 return false;
5718
5719 unsigned NumElts = VT.getVectorNumElements();
Luke Cheeseman4d45ff22015-07-24 09:57:05 +00005720 if (M.size() != NumElts && M.size() != NumElts*2)
5721 return false;
5722
5723 for (unsigned i = 0; i < M.size(); i += NumElts) {
James Molloy8c995a92015-09-10 08:42:28 +00005724 if (M.size() == NumElts * 2)
5725 WhichResult = i / NumElts;
5726 else
5727 WhichResult = M[i] == 0 ? 0 : 1;
Luke Cheeseman4d45ff22015-07-24 09:57:05 +00005728 for (unsigned j = 0; j < NumElts; j += 2) {
5729 if ((M[i+j] >= 0 && (unsigned) M[i+j] != j + WhichResult) ||
5730 (M[i+j+1] >= 0 && (unsigned) M[i+j+1] != j + WhichResult))
5731 return false;
5732 }
Bob Wilson0bbd3072009-12-03 06:40:55 +00005733 }
Luke Cheeseman4d45ff22015-07-24 09:57:05 +00005734
5735 if (M.size() == NumElts*2)
5736 WhichResult = 0;
5737
Bob Wilson0bbd3072009-12-03 06:40:55 +00005738 return true;
5739}
5740
Luke Cheeseman4d45ff22015-07-24 09:57:05 +00005741// Checks whether the shuffle mask represents a vector unzip (VUZP) by checking
5742// that the mask elements are either all even and in steps of size 2 or all odd
5743// and in steps of size 2.
5744// e.g. For v1,v2 of type v4i32 a valid shuffle mask is: [0, 2, 4, 6]
5745// v1={a,b,c,d} => x=shufflevector v1, v2 shufflemask => x={a,c,e,g}
5746// v2={e,f,g,h}
5747// Requires similar checks to that of isVTRNMask with
5748// respect the how results are returned.
Benjamin Kramer339ced42012-01-15 13:16:05 +00005749static bool isVUZPMask(ArrayRef<int> M, EVT VT, unsigned &WhichResult) {
Bob Wilson854530a2009-10-21 21:36:27 +00005750 unsigned EltSz = VT.getVectorElementType().getSizeInBits();
5751 if (EltSz == 64)
5752 return false;
5753
Bob Wilsona7062312009-08-21 20:54:19 +00005754 unsigned NumElts = VT.getVectorNumElements();
Luke Cheeseman4d45ff22015-07-24 09:57:05 +00005755 if (M.size() != NumElts && M.size() != NumElts*2)
5756 return false;
5757
5758 for (unsigned i = 0; i < M.size(); i += NumElts) {
5759 WhichResult = M[i] == 0 ? 0 : 1;
5760 for (unsigned j = 0; j < NumElts; ++j) {
5761 if (M[i+j] >= 0 && (unsigned) M[i+j] != 2 * j + WhichResult)
5762 return false;
5763 }
Bob Wilsona7062312009-08-21 20:54:19 +00005764 }
5765
Luke Cheeseman4d45ff22015-07-24 09:57:05 +00005766 if (M.size() == NumElts*2)
5767 WhichResult = 0;
5768
Bob Wilsona7062312009-08-21 20:54:19 +00005769 // VUZP.32 for 64-bit vectors is a pseudo-instruction alias for VTRN.32.
Bob Wilson854530a2009-10-21 21:36:27 +00005770 if (VT.is64BitVector() && EltSz == 32)
Bob Wilsona7062312009-08-21 20:54:19 +00005771 return false;
5772
5773 return true;
5774}
5775
Bob Wilson0bbd3072009-12-03 06:40:55 +00005776/// isVUZP_v_undef_Mask - Special case of isVUZPMask for canonical form of
5777/// "vector_shuffle v, v", i.e., "vector_shuffle v, undef".
5778/// Mask is e.g., <0, 2, 0, 2> instead of <0, 2, 4, 6>,
Benjamin Kramer339ced42012-01-15 13:16:05 +00005779static bool isVUZP_v_undef_Mask(ArrayRef<int> M, EVT VT, unsigned &WhichResult){
Bob Wilson0bbd3072009-12-03 06:40:55 +00005780 unsigned EltSz = VT.getVectorElementType().getSizeInBits();
5781 if (EltSz == 64)
5782 return false;
5783
Luke Cheeseman4d45ff22015-07-24 09:57:05 +00005784 unsigned NumElts = VT.getVectorNumElements();
5785 if (M.size() != NumElts && M.size() != NumElts*2)
5786 return false;
5787
5788 unsigned Half = NumElts / 2;
5789 for (unsigned i = 0; i < M.size(); i += NumElts) {
5790 WhichResult = M[i] == 0 ? 0 : 1;
5791 for (unsigned j = 0; j < NumElts; j += Half) {
5792 unsigned Idx = WhichResult;
5793 for (unsigned k = 0; k < Half; ++k) {
5794 int MIdx = M[i + j + k];
5795 if (MIdx >= 0 && (unsigned) MIdx != Idx)
5796 return false;
5797 Idx += 2;
5798 }
Bob Wilson0bbd3072009-12-03 06:40:55 +00005799 }
5800 }
5801
Luke Cheeseman4d45ff22015-07-24 09:57:05 +00005802 if (M.size() == NumElts*2)
5803 WhichResult = 0;
5804
Bob Wilson0bbd3072009-12-03 06:40:55 +00005805 // VUZP.32 for 64-bit vectors is a pseudo-instruction alias for VTRN.32.
5806 if (VT.is64BitVector() && EltSz == 32)
5807 return false;
5808
5809 return true;
5810}
5811
Luke Cheeseman4d45ff22015-07-24 09:57:05 +00005812// Checks whether the shuffle mask represents a vector zip (VZIP) by checking
5813// that pairs of elements of the shufflemask represent the same index in each
5814// vector incrementing sequentially through the vectors.
5815// e.g. For v1,v2 of type v4i32 a valid shuffle mask is: [0, 4, 1, 5]
5816// v1={a,b,c,d} => x=shufflevector v1, v2 shufflemask => x={a,e,b,f}
5817// v2={e,f,g,h}
5818// Requires similar checks to that of isVTRNMask with respect the how results
5819// are returned.
Benjamin Kramer339ced42012-01-15 13:16:05 +00005820static bool isVZIPMask(ArrayRef<int> M, EVT VT, unsigned &WhichResult) {
Bob Wilson854530a2009-10-21 21:36:27 +00005821 unsigned EltSz = VT.getVectorElementType().getSizeInBits();
5822 if (EltSz == 64)
5823 return false;
5824
Bob Wilsona7062312009-08-21 20:54:19 +00005825 unsigned NumElts = VT.getVectorNumElements();
Luke Cheeseman4d45ff22015-07-24 09:57:05 +00005826 if (M.size() != NumElts && M.size() != NumElts*2)
5827 return false;
5828
5829 for (unsigned i = 0; i < M.size(); i += NumElts) {
5830 WhichResult = M[i] == 0 ? 0 : 1;
5831 unsigned Idx = WhichResult * NumElts / 2;
5832 for (unsigned j = 0; j < NumElts; j += 2) {
5833 if ((M[i+j] >= 0 && (unsigned) M[i+j] != Idx) ||
5834 (M[i+j+1] >= 0 && (unsigned) M[i+j+1] != Idx + NumElts))
5835 return false;
5836 Idx += 1;
5837 }
Bob Wilsona7062312009-08-21 20:54:19 +00005838 }
5839
Luke Cheeseman4d45ff22015-07-24 09:57:05 +00005840 if (M.size() == NumElts*2)
5841 WhichResult = 0;
5842
Bob Wilsona7062312009-08-21 20:54:19 +00005843 // VZIP.32 for 64-bit vectors is a pseudo-instruction alias for VTRN.32.
Bob Wilson854530a2009-10-21 21:36:27 +00005844 if (VT.is64BitVector() && EltSz == 32)
Bob Wilsona7062312009-08-21 20:54:19 +00005845 return false;
5846
5847 return true;
5848}
5849
Bob Wilson0bbd3072009-12-03 06:40:55 +00005850/// isVZIP_v_undef_Mask - Special case of isVZIPMask for canonical form of
5851/// "vector_shuffle v, v", i.e., "vector_shuffle v, undef".
5852/// Mask is e.g., <0, 0, 1, 1> instead of <0, 4, 1, 5>.
Benjamin Kramer339ced42012-01-15 13:16:05 +00005853static bool isVZIP_v_undef_Mask(ArrayRef<int> M, EVT VT, unsigned &WhichResult){
Bob Wilson0bbd3072009-12-03 06:40:55 +00005854 unsigned EltSz = VT.getVectorElementType().getSizeInBits();
5855 if (EltSz == 64)
5856 return false;
5857
5858 unsigned NumElts = VT.getVectorNumElements();
Luke Cheeseman4d45ff22015-07-24 09:57:05 +00005859 if (M.size() != NumElts && M.size() != NumElts*2)
5860 return false;
5861
5862 for (unsigned i = 0; i < M.size(); i += NumElts) {
5863 WhichResult = M[i] == 0 ? 0 : 1;
5864 unsigned Idx = WhichResult * NumElts / 2;
5865 for (unsigned j = 0; j < NumElts; j += 2) {
5866 if ((M[i+j] >= 0 && (unsigned) M[i+j] != Idx) ||
5867 (M[i+j+1] >= 0 && (unsigned) M[i+j+1] != Idx))
5868 return false;
5869 Idx += 1;
5870 }
Bob Wilson0bbd3072009-12-03 06:40:55 +00005871 }
5872
Luke Cheeseman4d45ff22015-07-24 09:57:05 +00005873 if (M.size() == NumElts*2)
5874 WhichResult = 0;
5875
Bob Wilson0bbd3072009-12-03 06:40:55 +00005876 // VZIP.32 for 64-bit vectors is a pseudo-instruction alias for VTRN.32.
5877 if (VT.is64BitVector() && EltSz == 32)
5878 return false;
5879
5880 return true;
5881}
5882
Ahmed Bougacha2ffa91f2015-06-19 02:25:01 +00005883/// Check if \p ShuffleMask is a NEON two-result shuffle (VZIP, VUZP, VTRN),
5884/// and return the corresponding ARMISD opcode if it is, or 0 if it isn't.
5885static unsigned isNEONTwoResultShuffleMask(ArrayRef<int> ShuffleMask, EVT VT,
5886 unsigned &WhichResult,
5887 bool &isV_UNDEF) {
5888 isV_UNDEF = false;
5889 if (isVTRNMask(ShuffleMask, VT, WhichResult))
5890 return ARMISD::VTRN;
5891 if (isVUZPMask(ShuffleMask, VT, WhichResult))
5892 return ARMISD::VUZP;
5893 if (isVZIPMask(ShuffleMask, VT, WhichResult))
5894 return ARMISD::VZIP;
5895
5896 isV_UNDEF = true;
5897 if (isVTRN_v_undef_Mask(ShuffleMask, VT, WhichResult))
5898 return ARMISD::VTRN;
5899 if (isVUZP_v_undef_Mask(ShuffleMask, VT, WhichResult))
5900 return ARMISD::VUZP;
5901 if (isVZIP_v_undef_Mask(ShuffleMask, VT, WhichResult))
5902 return ARMISD::VZIP;
5903
5904 return 0;
5905}
5906
Arnold Schwaighofer1f3d3ca2013-02-12 01:58:32 +00005907/// \return true if this is a reverse operation on an vector.
5908static bool isReverseMask(ArrayRef<int> M, EVT VT) {
5909 unsigned NumElts = VT.getVectorNumElements();
5910 // Make sure the mask has the right size.
5911 if (NumElts != M.size())
5912 return false;
5913
5914 // Look for <15, ..., 3, -1, 1, 0>.
5915 for (unsigned i = 0; i != NumElts; ++i)
5916 if (M[i] >= 0 && M[i] != (int) (NumElts - 1 - i))
5917 return false;
5918
5919 return true;
5920}
5921
Dale Johannesen2bff5052010-07-29 20:10:08 +00005922// If N is an integer constant that can be moved into a register in one
5923// instruction, return an SDValue of such a constant (will become a MOV
5924// instruction). Otherwise return null.
5925static SDValue IsSingleInstrConstant(SDValue N, SelectionDAG &DAG,
Benjamin Kramerbdc49562016-06-12 15:39:02 +00005926 const ARMSubtarget *ST, const SDLoc &dl) {
Dale Johannesen2bff5052010-07-29 20:10:08 +00005927 uint64_t Val;
5928 if (!isa<ConstantSDNode>(N))
5929 return SDValue();
5930 Val = cast<ConstantSDNode>(N)->getZExtValue();
5931
5932 if (ST->isThumb1Only()) {
5933 if (Val <= 255 || ~Val <= 255)
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +00005934 return DAG.getConstant(Val, dl, MVT::i32);
Dale Johannesen2bff5052010-07-29 20:10:08 +00005935 } else {
5936 if (ARM_AM::getSOImmVal(Val) != -1 || ARM_AM::getSOImmVal(~Val) != -1)
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +00005937 return DAG.getConstant(Val, dl, MVT::i32);
Dale Johannesen2bff5052010-07-29 20:10:08 +00005938 }
5939 return SDValue();
5940}
5941
Bob Wilson2e076c42009-06-22 23:27:02 +00005942// If this is a case we can't handle, return null and let the default
5943// expansion code take care of it.
Bob Wilson6f2b8962011-01-07 21:37:30 +00005944SDValue ARMTargetLowering::LowerBUILD_VECTOR(SDValue Op, SelectionDAG &DAG,
5945 const ARMSubtarget *ST) const {
Bob Wilsonfcd63612009-08-13 01:57:47 +00005946 BuildVectorSDNode *BVN = cast<BuildVectorSDNode>(Op.getNode());
Andrew Trickef9de2a2013-05-25 02:42:55 +00005947 SDLoc dl(Op);
Owen Anderson53aa7a92009-08-10 22:56:29 +00005948 EVT VT = Op.getValueType();
Bob Wilson2e076c42009-06-22 23:27:02 +00005949
5950 APInt SplatBits, SplatUndef;
5951 unsigned SplatBitSize;
5952 bool HasAnyUndefs;
5953 if (BVN->isConstantSplat(SplatBits, SplatUndef, SplatBitSize, HasAnyUndefs)) {
Anton Korobeynikovece642a2009-08-29 00:08:18 +00005954 if (SplatBitSize <= 64) {
Bob Wilson5b2b5042010-06-14 22:19:57 +00005955 // Check if an immediate VMOV works.
Bob Wilsona3f19012010-07-13 21:16:48 +00005956 EVT VmovVT;
Bob Wilson5b2b5042010-06-14 22:19:57 +00005957 SDValue Val = isNEONModifiedImm(SplatBits.getZExtValue(),
Bob Wilsona3f19012010-07-13 21:16:48 +00005958 SplatUndef.getZExtValue(), SplatBitSize,
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +00005959 DAG, dl, VmovVT, VT.is128BitVector(),
Owen Andersona4076922010-11-05 21:57:54 +00005960 VMOVModImm);
Bob Wilsona3f19012010-07-13 21:16:48 +00005961 if (Val.getNode()) {
5962 SDValue Vmov = DAG.getNode(ARMISD::VMOVIMM, dl, VmovVT, Val);
Wesley Peck527da1b2010-11-23 03:31:01 +00005963 return DAG.getNode(ISD::BITCAST, dl, VT, Vmov);
Bob Wilsona3f19012010-07-13 21:16:48 +00005964 }
Bob Wilsonbad47f62010-07-14 06:31:50 +00005965
5966 // Try an immediate VMVN.
Eli Friedmanaa6ec392011-10-13 22:40:23 +00005967 uint64_t NegatedImm = (~SplatBits).getZExtValue();
Bob Wilsonbad47f62010-07-14 06:31:50 +00005968 Val = isNEONModifiedImm(NegatedImm,
5969 SplatUndef.getZExtValue(), SplatBitSize,
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +00005970 DAG, dl, VmovVT, VT.is128BitVector(),
Owen Andersona4076922010-11-05 21:57:54 +00005971 VMVNModImm);
Bob Wilsonbad47f62010-07-14 06:31:50 +00005972 if (Val.getNode()) {
5973 SDValue Vmov = DAG.getNode(ARMISD::VMVNIMM, dl, VmovVT, Val);
Wesley Peck527da1b2010-11-23 03:31:01 +00005974 return DAG.getNode(ISD::BITCAST, dl, VT, Vmov);
Bob Wilsonbad47f62010-07-14 06:31:50 +00005975 }
Evan Cheng7ca4b6e2011-11-15 02:12:34 +00005976
5977 // Use vmov.f32 to materialize other v2f32 and v4f32 splats.
Eli Friedmanc9bf1b12011-12-15 22:56:53 +00005978 if ((VT == MVT::v2f32 || VT == MVT::v4f32) && SplatBitSize == 32) {
Eli Friedman4e36a932011-12-09 23:54:42 +00005979 int ImmVal = ARM_AM::getFP32Imm(SplatBits);
Evan Cheng7ca4b6e2011-11-15 02:12:34 +00005980 if (ImmVal != -1) {
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +00005981 SDValue Val = DAG.getTargetConstant(ImmVal, dl, MVT::i32);
Evan Cheng7ca4b6e2011-11-15 02:12:34 +00005982 return DAG.getNode(ARMISD::VMOVFPIMM, dl, VT, Val);
5983 }
5984 }
Anton Korobeynikovece642a2009-08-29 00:08:18 +00005985 }
Bob Wilson0dbdec82009-07-30 00:31:25 +00005986 }
5987
Bob Wilson91fdf682010-05-22 00:23:12 +00005988 // Scan through the operands to see if only one value is used.
James Molloy49bdbce2012-09-06 09:55:02 +00005989 //
5990 // As an optimisation, even if more than one value is used it may be more
5991 // profitable to splat with one value then change some lanes.
5992 //
5993 // Heuristically we decide to do this if the vector has a "dominant" value,
5994 // defined as splatted to more than half of the lanes.
Bob Wilson91fdf682010-05-22 00:23:12 +00005995 unsigned NumElts = VT.getVectorNumElements();
5996 bool isOnlyLowElement = true;
5997 bool usesOnlyOneValue = true;
James Molloy49bdbce2012-09-06 09:55:02 +00005998 bool hasDominantValue = false;
Bob Wilson91fdf682010-05-22 00:23:12 +00005999 bool isConstant = true;
James Molloy49bdbce2012-09-06 09:55:02 +00006000
6001 // Map of the number of times a particular SDValue appears in the
6002 // element list.
James Molloy9d30dc22012-09-06 10:32:08 +00006003 DenseMap<SDValue, unsigned> ValueCounts;
Bob Wilson91fdf682010-05-22 00:23:12 +00006004 SDValue Value;
6005 for (unsigned i = 0; i < NumElts; ++i) {
6006 SDValue V = Op.getOperand(i);
Sanjay Patel57195842016-03-14 17:28:46 +00006007 if (V.isUndef())
Bob Wilson91fdf682010-05-22 00:23:12 +00006008 continue;
6009 if (i > 0)
6010 isOnlyLowElement = false;
6011 if (!isa<ConstantFPSDNode>(V) && !isa<ConstantSDNode>(V))
6012 isConstant = false;
6013
James Molloy49bdbce2012-09-06 09:55:02 +00006014 ValueCounts.insert(std::make_pair(V, 0));
James Molloy9d30dc22012-09-06 10:32:08 +00006015 unsigned &Count = ValueCounts[V];
Jim Grosbach54efea02013-03-02 20:16:15 +00006016
James Molloy49bdbce2012-09-06 09:55:02 +00006017 // Is this value dominant? (takes up more than half of the lanes)
6018 if (++Count > (NumElts / 2)) {
6019 hasDominantValue = true;
Bob Wilson91fdf682010-05-22 00:23:12 +00006020 Value = V;
James Molloy49bdbce2012-09-06 09:55:02 +00006021 }
Bob Wilson91fdf682010-05-22 00:23:12 +00006022 }
James Molloy49bdbce2012-09-06 09:55:02 +00006023 if (ValueCounts.size() != 1)
6024 usesOnlyOneValue = false;
6025 if (!Value.getNode() && ValueCounts.size() > 0)
6026 Value = ValueCounts.begin()->first;
Bob Wilson91fdf682010-05-22 00:23:12 +00006027
James Molloy49bdbce2012-09-06 09:55:02 +00006028 if (ValueCounts.size() == 0)
Bob Wilson91fdf682010-05-22 00:23:12 +00006029 return DAG.getUNDEF(VT);
6030
Quentin Colombet0f2fe742013-07-23 22:34:47 +00006031 // Loads are better lowered with insert_vector_elt/ARMISD::BUILD_VECTOR.
6032 // Keep going if we are hitting this case.
6033 if (isOnlyLowElement && !ISD::isNormalLoad(Value.getNode()))
Bob Wilson91fdf682010-05-22 00:23:12 +00006034 return DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Value);
6035
Dale Johannesen2bff5052010-07-29 20:10:08 +00006036 unsigned EltSize = VT.getVectorElementType().getSizeInBits();
6037
Dale Johannesen710a2d92010-10-19 20:00:17 +00006038 // Use VDUP for non-constant splats. For f32 constant splats, reduce to
6039 // i32 and try again.
James Molloy49bdbce2012-09-06 09:55:02 +00006040 if (hasDominantValue && EltSize <= 32) {
6041 if (!isConstant) {
6042 SDValue N;
6043
6044 // If we are VDUPing a value that comes directly from a vector, that will
6045 // cause an unnecessary move to and from a GPR, where instead we could
Jim Grosbacha3c5c762013-03-02 20:16:24 +00006046 // just use VDUPLANE. We can only do this if the lane being extracted
6047 // is at a constant index, as the VDUP from lane instructions only have
6048 // constant-index forms.
Artyom Skrobov314ee042015-11-25 19:41:11 +00006049 ConstantSDNode *constIndex;
Jim Grosbacha3c5c762013-03-02 20:16:24 +00006050 if (Value->getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
Artyom Skrobov314ee042015-11-25 19:41:11 +00006051 (constIndex = dyn_cast<ConstantSDNode>(Value->getOperand(1)))) {
Silviu Barangab1409702012-10-15 09:41:32 +00006052 // We need to create a new undef vector to use for the VDUPLANE if the
6053 // size of the vector from which we get the value is different than the
6054 // size of the vector that we need to create. We will insert the element
6055 // such that the register coalescer will remove unnecessary copies.
6056 if (VT != Value->getOperand(0).getValueType()) {
Silviu Barangab1409702012-10-15 09:41:32 +00006057 unsigned index = constIndex->getAPIntValue().getLimitedValue() %
6058 VT.getVectorNumElements();
6059 N = DAG.getNode(ARMISD::VDUPLANE, dl, VT,
6060 DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, VT, DAG.getUNDEF(VT),
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +00006061 Value, DAG.getConstant(index, dl, MVT::i32)),
6062 DAG.getConstant(index, dl, MVT::i32));
Jim Grosbachc6f19142013-03-02 20:16:19 +00006063 } else
Silviu Barangab1409702012-10-15 09:41:32 +00006064 N = DAG.getNode(ARMISD::VDUPLANE, dl, VT,
James Molloy49bdbce2012-09-06 09:55:02 +00006065 Value->getOperand(0), Value->getOperand(1));
Jim Grosbachc6f19142013-03-02 20:16:19 +00006066 } else
James Molloy49bdbce2012-09-06 09:55:02 +00006067 N = DAG.getNode(ARMISD::VDUP, dl, VT, Value);
6068
6069 if (!usesOnlyOneValue) {
6070 // The dominant value was splatted as 'N', but we now have to insert
6071 // all differing elements.
6072 for (unsigned I = 0; I < NumElts; ++I) {
6073 if (Op.getOperand(I) == Value)
6074 continue;
6075 SmallVector<SDValue, 3> Ops;
6076 Ops.push_back(N);
6077 Ops.push_back(Op.getOperand(I));
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +00006078 Ops.push_back(DAG.getConstant(I, dl, MVT::i32));
Craig Topper48d114b2014-04-26 18:35:24 +00006079 N = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, VT, Ops);
James Molloy49bdbce2012-09-06 09:55:02 +00006080 }
6081 }
6082 return N;
6083 }
Dale Johannesen710a2d92010-10-19 20:00:17 +00006084 if (VT.getVectorElementType().isFloatingPoint()) {
6085 SmallVector<SDValue, 8> Ops;
6086 for (unsigned i = 0; i < NumElts; ++i)
Wesley Peck527da1b2010-11-23 03:31:01 +00006087 Ops.push_back(DAG.getNode(ISD::BITCAST, dl, MVT::i32,
Dale Johannesen710a2d92010-10-19 20:00:17 +00006088 Op.getOperand(i)));
Nate Begemanca524112010-11-10 21:35:41 +00006089 EVT VecVT = EVT::getVectorVT(*DAG.getContext(), MVT::i32, NumElts);
Ahmed Bougacha128f8732016-04-26 21:15:30 +00006090 SDValue Val = DAG.getBuildVector(VecVT, dl, Ops);
Dale Johannesenff376752010-10-20 22:03:37 +00006091 Val = LowerBUILD_VECTOR(Val, DAG, ST);
6092 if (Val.getNode())
Wesley Peck527da1b2010-11-23 03:31:01 +00006093 return DAG.getNode(ISD::BITCAST, dl, VT, Val);
Dale Johannesen2bff5052010-07-29 20:10:08 +00006094 }
James Molloy49bdbce2012-09-06 09:55:02 +00006095 if (usesOnlyOneValue) {
6096 SDValue Val = IsSingleInstrConstant(Value, DAG, ST, dl);
6097 if (isConstant && Val.getNode())
Jim Grosbach54efea02013-03-02 20:16:15 +00006098 return DAG.getNode(ARMISD::VDUP, dl, VT, Val);
James Molloy49bdbce2012-09-06 09:55:02 +00006099 }
Dale Johannesen2bff5052010-07-29 20:10:08 +00006100 }
6101
6102 // If all elements are constants and the case above didn't get hit, fall back
6103 // to the default expansion, which will generate a load from the constant
6104 // pool.
Bob Wilson91fdf682010-05-22 00:23:12 +00006105 if (isConstant)
6106 return SDValue();
6107
Bob Wilson6f2b8962011-01-07 21:37:30 +00006108 // Empirical tests suggest this is rarely worth it for vectors of length <= 2.
6109 if (NumElts >= 4) {
6110 SDValue shuffle = ReconstructShuffle(Op, DAG);
6111 if (shuffle != SDValue())
6112 return shuffle;
6113 }
6114
Bob Wilson91fdf682010-05-22 00:23:12 +00006115 // Vectors with 32- or 64-bit elements can be built by directly assigning
Bob Wilsond8a9a042010-06-04 00:04:02 +00006116 // the subregisters. Lower it to an ARMISD::BUILD_VECTOR so the operands
6117 // will be legalized.
Bob Wilson91fdf682010-05-22 00:23:12 +00006118 if (EltSize >= 32) {
6119 // Do the expansion with floating-point types, since that is what the VFP
6120 // registers are defined to use, and since i64 is not legal.
6121 EVT EltVT = EVT::getFloatingPointVT(EltSize);
6122 EVT VecVT = EVT::getVectorVT(*DAG.getContext(), EltVT, NumElts);
Bob Wilsond8a9a042010-06-04 00:04:02 +00006123 SmallVector<SDValue, 8> Ops;
6124 for (unsigned i = 0; i < NumElts; ++i)
Wesley Peck527da1b2010-11-23 03:31:01 +00006125 Ops.push_back(DAG.getNode(ISD::BITCAST, dl, EltVT, Op.getOperand(i)));
Craig Topper48d114b2014-04-26 18:35:24 +00006126 SDValue Val = DAG.getNode(ARMISD::BUILD_VECTOR, dl, VecVT, Ops);
Wesley Peck527da1b2010-11-23 03:31:01 +00006127 return DAG.getNode(ISD::BITCAST, dl, VT, Val);
Bob Wilson2e076c42009-06-22 23:27:02 +00006128 }
6129
Jim Grosbach24e102a2013-07-08 18:18:52 +00006130 // If all else fails, just use a sequence of INSERT_VECTOR_ELT when we
6131 // know the default expansion would otherwise fall back on something even
6132 // worse. For a vector with one or two non-undef values, that's
6133 // scalar_to_vector for the elements followed by a shuffle (provided the
6134 // shuffle is valid for the target) and materialization element by element
6135 // on the stack followed by a load for everything else.
6136 if (!isConstant && !usesOnlyOneValue) {
6137 SDValue Vec = DAG.getUNDEF(VT);
6138 for (unsigned i = 0 ; i < NumElts; ++i) {
6139 SDValue V = Op.getOperand(i);
Sanjay Patel57195842016-03-14 17:28:46 +00006140 if (V.isUndef())
Jim Grosbach24e102a2013-07-08 18:18:52 +00006141 continue;
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +00006142 SDValue LaneIdx = DAG.getConstant(i, dl, MVT::i32);
Jim Grosbach24e102a2013-07-08 18:18:52 +00006143 Vec = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, VT, Vec, V, LaneIdx);
6144 }
6145 return Vec;
6146 }
6147
Bob Wilson2e076c42009-06-22 23:27:02 +00006148 return SDValue();
6149}
6150
Bob Wilson6f2b8962011-01-07 21:37:30 +00006151// Gather data to see if the operation can be modelled as a
Andrew Trick5eb0a302011-01-19 02:26:13 +00006152// shuffle in combination with VEXTs.
Eric Christopher2af95512011-01-14 23:50:53 +00006153SDValue ARMTargetLowering::ReconstructShuffle(SDValue Op,
6154 SelectionDAG &DAG) const {
Silviu Baranga3e8e51c2015-08-07 11:40:46 +00006155 assert(Op.getOpcode() == ISD::BUILD_VECTOR && "Unknown opcode!");
Andrew Trickef9de2a2013-05-25 02:42:55 +00006156 SDLoc dl(Op);
Bob Wilson6f2b8962011-01-07 21:37:30 +00006157 EVT VT = Op.getValueType();
6158 unsigned NumElts = VT.getVectorNumElements();
6159
Silviu Baranga3e8e51c2015-08-07 11:40:46 +00006160 struct ShuffleSourceInfo {
6161 SDValue Vec;
6162 unsigned MinElt;
6163 unsigned MaxElt;
Andrew Trick5eb0a302011-01-19 02:26:13 +00006164
Silviu Baranga3e8e51c2015-08-07 11:40:46 +00006165 // We may insert some combination of BITCASTs and VEXT nodes to force Vec to
6166 // be compatible with the shuffle we intend to construct. As a result
6167 // ShuffleVec will be some sliding window into the original Vec.
6168 SDValue ShuffleVec;
6169
6170 // Code should guarantee that element i in Vec starts at element "WindowBase
6171 // + i * WindowScale in ShuffleVec".
6172 int WindowBase;
6173 int WindowScale;
6174
6175 bool operator ==(SDValue OtherVec) { return Vec == OtherVec; }
6176 ShuffleSourceInfo(SDValue Vec)
6177 : Vec(Vec), MinElt(UINT_MAX), MaxElt(0), ShuffleVec(Vec), WindowBase(0),
6178 WindowScale(1) {}
6179 };
6180
6181 // First gather all vectors used as an immediate source for this BUILD_VECTOR
6182 // node.
6183 SmallVector<ShuffleSourceInfo, 2> Sources;
Bob Wilson6f2b8962011-01-07 21:37:30 +00006184 for (unsigned i = 0; i < NumElts; ++i) {
6185 SDValue V = Op.getOperand(i);
Sanjay Patel57195842016-03-14 17:28:46 +00006186 if (V.isUndef())
Bob Wilson6f2b8962011-01-07 21:37:30 +00006187 continue;
6188 else if (V.getOpcode() != ISD::EXTRACT_VECTOR_ELT) {
6189 // A shuffle can only come from building a vector from various
6190 // elements of other vectors.
6191 return SDValue();
Ahmed Bougacha699a9dd2015-09-01 21:56:00 +00006192 } else if (!isa<ConstantSDNode>(V.getOperand(1))) {
6193 // Furthermore, shuffles require a constant mask, whereas extractelts
6194 // accept variable indices.
6195 return SDValue();
Bob Wilson6f2b8962011-01-07 21:37:30 +00006196 }
Andrew Trick5eb0a302011-01-19 02:26:13 +00006197
Silviu Baranga3e8e51c2015-08-07 11:40:46 +00006198 // Add this element source to the list if it's not already there.
Bob Wilson6f2b8962011-01-07 21:37:30 +00006199 SDValue SourceVec = V.getOperand(0);
David Majnemer0d955d02016-08-11 22:21:41 +00006200 auto Source = find(Sources, SourceVec);
Silviu Baranga3e8e51c2015-08-07 11:40:46 +00006201 if (Source == Sources.end())
6202 Source = Sources.insert(Sources.end(), ShuffleSourceInfo(SourceVec));
Andrew Trick5eb0a302011-01-19 02:26:13 +00006203
Silviu Baranga3e8e51c2015-08-07 11:40:46 +00006204 // Update the minimum and maximum lane number seen.
6205 unsigned EltNo = cast<ConstantSDNode>(V.getOperand(1))->getZExtValue();
6206 Source->MinElt = std::min(Source->MinElt, EltNo);
6207 Source->MaxElt = std::max(Source->MaxElt, EltNo);
Bob Wilson6f2b8962011-01-07 21:37:30 +00006208 }
Andrew Trick5eb0a302011-01-19 02:26:13 +00006209
Bob Wilson6f2b8962011-01-07 21:37:30 +00006210 // Currently only do something sane when at most two source vectors
Silviu Baranga3e8e51c2015-08-07 11:40:46 +00006211 // are involved.
6212 if (Sources.size() > 2)
Bob Wilson6f2b8962011-01-07 21:37:30 +00006213 return SDValue();
6214
Silviu Baranga3e8e51c2015-08-07 11:40:46 +00006215 // Find out the smallest element size among result and two sources, and use
6216 // it as element size to build the shuffle_vector.
6217 EVT SmallestEltTy = VT.getVectorElementType();
6218 for (auto &Source : Sources) {
6219 EVT SrcEltTy = Source.Vec.getValueType().getVectorElementType();
6220 if (SrcEltTy.bitsLT(SmallestEltTy))
6221 SmallestEltTy = SrcEltTy;
6222 }
6223 unsigned ResMultiplier =
6224 VT.getVectorElementType().getSizeInBits() / SmallestEltTy.getSizeInBits();
6225 NumElts = VT.getSizeInBits() / SmallestEltTy.getSizeInBits();
6226 EVT ShuffleVT = EVT::getVectorVT(*DAG.getContext(), SmallestEltTy, NumElts);
Andrew Trick5eb0a302011-01-19 02:26:13 +00006227
Silviu Baranga3e8e51c2015-08-07 11:40:46 +00006228 // If the source vector is too wide or too narrow, we may nevertheless be able
6229 // to construct a compatible shuffle either by concatenating it with UNDEF or
6230 // extracting a suitable range of elements.
6231 for (auto &Src : Sources) {
6232 EVT SrcVT = Src.ShuffleVec.getValueType();
6233
6234 if (SrcVT.getSizeInBits() == VT.getSizeInBits())
Bob Wilson6f2b8962011-01-07 21:37:30 +00006235 continue;
Silviu Baranga3e8e51c2015-08-07 11:40:46 +00006236
6237 // This stage of the search produces a source with the same element type as
6238 // the original, but with a total width matching the BUILD_VECTOR output.
6239 EVT EltVT = SrcVT.getVectorElementType();
6240 unsigned NumSrcElts = VT.getSizeInBits() / EltVT.getSizeInBits();
6241 EVT DestVT = EVT::getVectorVT(*DAG.getContext(), EltVT, NumSrcElts);
6242
6243 if (SrcVT.getSizeInBits() < VT.getSizeInBits()) {
6244 if (2 * SrcVT.getSizeInBits() != VT.getSizeInBits())
6245 return SDValue();
6246 // We can pad out the smaller vector for free, so if it's part of a
6247 // shuffle...
6248 Src.ShuffleVec =
6249 DAG.getNode(ISD::CONCAT_VECTORS, dl, DestVT, Src.ShuffleVec,
6250 DAG.getUNDEF(Src.ShuffleVec.getValueType()));
6251 continue;
Bob Wilson6f2b8962011-01-07 21:37:30 +00006252 }
Andrew Trick5eb0a302011-01-19 02:26:13 +00006253
Silviu Baranga3e8e51c2015-08-07 11:40:46 +00006254 if (SrcVT.getSizeInBits() != 2 * VT.getSizeInBits())
6255 return SDValue();
Andrew Trick5eb0a302011-01-19 02:26:13 +00006256
Silviu Baranga3e8e51c2015-08-07 11:40:46 +00006257 if (Src.MaxElt - Src.MinElt >= NumSrcElts) {
Bob Wilson6f2b8962011-01-07 21:37:30 +00006258 // Span too large for a VEXT to cope
6259 return SDValue();
Andrew Trick5eb0a302011-01-19 02:26:13 +00006260 }
6261
Silviu Baranga3e8e51c2015-08-07 11:40:46 +00006262 if (Src.MinElt >= NumSrcElts) {
Bob Wilson6f2b8962011-01-07 21:37:30 +00006263 // The extraction can just take the second half
Silviu Baranga3e8e51c2015-08-07 11:40:46 +00006264 Src.ShuffleVec =
6265 DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, DestVT, Src.ShuffleVec,
6266 DAG.getConstant(NumSrcElts, dl, MVT::i32));
6267 Src.WindowBase = -NumSrcElts;
6268 } else if (Src.MaxElt < NumSrcElts) {
Bob Wilson6f2b8962011-01-07 21:37:30 +00006269 // The extraction can just take the first half
Silviu Baranga3e8e51c2015-08-07 11:40:46 +00006270 Src.ShuffleVec =
6271 DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, DestVT, Src.ShuffleVec,
6272 DAG.getConstant(0, dl, MVT::i32));
Bob Wilson6f2b8962011-01-07 21:37:30 +00006273 } else {
6274 // An actual VEXT is needed
Silviu Baranga3e8e51c2015-08-07 11:40:46 +00006275 SDValue VEXTSrc1 =
6276 DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, DestVT, Src.ShuffleVec,
6277 DAG.getConstant(0, dl, MVT::i32));
6278 SDValue VEXTSrc2 =
6279 DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, DestVT, Src.ShuffleVec,
6280 DAG.getConstant(NumSrcElts, dl, MVT::i32));
Silviu Baranga3e8e51c2015-08-07 11:40:46 +00006281
6282 Src.ShuffleVec = DAG.getNode(ARMISD::VEXT, dl, DestVT, VEXTSrc1,
6283 VEXTSrc2,
Jeroen Ketema41681a52015-09-21 20:28:04 +00006284 DAG.getConstant(Src.MinElt, dl, MVT::i32));
Silviu Baranga3e8e51c2015-08-07 11:40:46 +00006285 Src.WindowBase = -Src.MinElt;
Bob Wilson6f2b8962011-01-07 21:37:30 +00006286 }
6287 }
Andrew Trick5eb0a302011-01-19 02:26:13 +00006288
Silviu Baranga3e8e51c2015-08-07 11:40:46 +00006289 // Another possible incompatibility occurs from the vector element types. We
6290 // can fix this by bitcasting the source vectors to the same type we intend
6291 // for the shuffle.
6292 for (auto &Src : Sources) {
6293 EVT SrcEltTy = Src.ShuffleVec.getValueType().getVectorElementType();
6294 if (SrcEltTy == SmallestEltTy)
Bob Wilson6f2b8962011-01-07 21:37:30 +00006295 continue;
Silviu Baranga3e8e51c2015-08-07 11:40:46 +00006296 assert(ShuffleVT.getVectorElementType() == SmallestEltTy);
6297 Src.ShuffleVec = DAG.getNode(ISD::BITCAST, dl, ShuffleVT, Src.ShuffleVec);
6298 Src.WindowScale = SrcEltTy.getSizeInBits() / SmallestEltTy.getSizeInBits();
6299 Src.WindowBase *= Src.WindowScale;
6300 }
Andrew Trick5eb0a302011-01-19 02:26:13 +00006301
Silviu Baranga3e8e51c2015-08-07 11:40:46 +00006302 // Final sanity check before we try to actually produce a shuffle.
Silviu Barangaa07090f2015-08-07 12:05:46 +00006303 DEBUG(
6304 for (auto Src : Sources)
6305 assert(Src.ShuffleVec.getValueType() == ShuffleVT);
6306 );
Silviu Baranga3e8e51c2015-08-07 11:40:46 +00006307
6308 // The stars all align, our next step is to produce the mask for the shuffle.
6309 SmallVector<int, 8> Mask(ShuffleVT.getVectorNumElements(), -1);
6310 int BitsPerShuffleLane = ShuffleVT.getVectorElementType().getSizeInBits();
6311 for (unsigned i = 0; i < VT.getVectorNumElements(); ++i) {
6312 SDValue Entry = Op.getOperand(i);
Sanjay Patel57195842016-03-14 17:28:46 +00006313 if (Entry.isUndef())
Silviu Baranga3e8e51c2015-08-07 11:40:46 +00006314 continue;
6315
David Majnemer0d955d02016-08-11 22:21:41 +00006316 auto Src = find(Sources, Entry.getOperand(0));
Silviu Baranga3e8e51c2015-08-07 11:40:46 +00006317 int EltNo = cast<ConstantSDNode>(Entry.getOperand(1))->getSExtValue();
6318
6319 // EXTRACT_VECTOR_ELT performs an implicit any_ext; BUILD_VECTOR an implicit
6320 // trunc. So only std::min(SrcBits, DestBits) actually get defined in this
6321 // segment.
6322 EVT OrigEltTy = Entry.getOperand(0).getValueType().getVectorElementType();
6323 int BitsDefined = std::min(OrigEltTy.getSizeInBits(),
6324 VT.getVectorElementType().getSizeInBits());
6325 int LanesDefined = BitsDefined / BitsPerShuffleLane;
6326
6327 // This source is expected to fill ResMultiplier lanes of the final shuffle,
6328 // starting at the appropriate offset.
6329 int *LaneMask = &Mask[i * ResMultiplier];
6330
6331 int ExtractBase = EltNo * Src->WindowScale + Src->WindowBase;
6332 ExtractBase += NumElts * (Src - Sources.begin());
6333 for (int j = 0; j < LanesDefined; ++j)
6334 LaneMask[j] = ExtractBase + j;
Bob Wilson6f2b8962011-01-07 21:37:30 +00006335 }
Andrew Trick5eb0a302011-01-19 02:26:13 +00006336
Bob Wilson6f2b8962011-01-07 21:37:30 +00006337 // Final check before we try to produce nonsense...
Silviu Baranga3e8e51c2015-08-07 11:40:46 +00006338 if (!isShuffleMaskLegal(Mask, ShuffleVT))
6339 return SDValue();
Andrew Trick5eb0a302011-01-19 02:26:13 +00006340
Silviu Baranga3e8e51c2015-08-07 11:40:46 +00006341 // We can't handle more than two sources. This should have already
6342 // been checked before this point.
6343 assert(Sources.size() <= 2 && "Too many sources!");
6344
6345 SDValue ShuffleOps[] = { DAG.getUNDEF(ShuffleVT), DAG.getUNDEF(ShuffleVT) };
6346 for (unsigned i = 0; i < Sources.size(); ++i)
6347 ShuffleOps[i] = Sources[i].ShuffleVec;
6348
6349 SDValue Shuffle = DAG.getVectorShuffle(ShuffleVT, dl, ShuffleOps[0],
Craig Topper2bd8b4b2016-07-01 06:54:47 +00006350 ShuffleOps[1], Mask);
Silviu Baranga3e8e51c2015-08-07 11:40:46 +00006351 return DAG.getNode(ISD::BITCAST, dl, VT, Shuffle);
Bob Wilson6f2b8962011-01-07 21:37:30 +00006352}
6353
Anton Korobeynikovc32e99e2009-08-21 12:40:07 +00006354/// isShuffleMaskLegal - Targets can use this to indicate that they only
6355/// support *some* VECTOR_SHUFFLE operations, those with specific masks.
6356/// By default, if a target supports the VECTOR_SHUFFLE node, all mask values
6357/// are assumed to be legal.
6358bool
6359ARMTargetLowering::isShuffleMaskLegal(const SmallVectorImpl<int> &M,
6360 EVT VT) const {
Anton Korobeynikov9a232f42009-08-21 12:41:24 +00006361 if (VT.getVectorNumElements() == 4 &&
6362 (VT.is128BitVector() || VT.is64BitVector())) {
6363 unsigned PFIndexes[4];
6364 for (unsigned i = 0; i != 4; ++i) {
6365 if (M[i] < 0)
6366 PFIndexes[i] = 8;
6367 else
6368 PFIndexes[i] = M[i];
6369 }
6370
6371 // Compute the index in the perfect shuffle table.
6372 unsigned PFTableIndex =
6373 PFIndexes[0]*9*9*9+PFIndexes[1]*9*9+PFIndexes[2]*9+PFIndexes[3];
6374 unsigned PFEntry = PerfectShuffleTable[PFTableIndex];
6375 unsigned Cost = (PFEntry >> 30);
6376
6377 if (Cost <= 4)
6378 return true;
6379 }
6380
Ahmed Bougacha2ffa91f2015-06-19 02:25:01 +00006381 bool ReverseVEXT, isV_UNDEF;
Bob Wilsona7062312009-08-21 20:54:19 +00006382 unsigned Imm, WhichResult;
Anton Korobeynikovc32e99e2009-08-21 12:40:07 +00006383
Bob Wilson846bd792010-06-07 23:53:38 +00006384 unsigned EltSize = VT.getVectorElementType().getSizeInBits();
6385 return (EltSize >= 32 ||
6386 ShuffleVectorSDNode::isSplatMask(&M[0], VT) ||
Anton Korobeynikovc32e99e2009-08-21 12:40:07 +00006387 isVREVMask(M, VT, 64) ||
6388 isVREVMask(M, VT, 32) ||
6389 isVREVMask(M, VT, 16) ||
Bob Wilsona7062312009-08-21 20:54:19 +00006390 isVEXTMask(M, VT, ReverseVEXT, Imm) ||
Bill Wendling865f8b52011-03-15 21:15:20 +00006391 isVTBLMask(M, VT) ||
Ahmed Bougacha2ffa91f2015-06-19 02:25:01 +00006392 isNEONTwoResultShuffleMask(M, VT, WhichResult, isV_UNDEF) ||
Arnold Schwaighofer1f3d3ca2013-02-12 01:58:32 +00006393 ((VT == MVT::v8i16 || VT == MVT::v16i8) && isReverseMask(M, VT)));
Anton Korobeynikovc32e99e2009-08-21 12:40:07 +00006394}
6395
Anton Korobeynikov9a232f42009-08-21 12:41:24 +00006396/// GeneratePerfectShuffle - Given an entry in the perfect-shuffle table, emit
6397/// the specified operations to build the shuffle.
6398static SDValue GeneratePerfectShuffle(unsigned PFEntry, SDValue LHS,
6399 SDValue RHS, SelectionDAG &DAG,
Benjamin Kramerbdc49562016-06-12 15:39:02 +00006400 const SDLoc &dl) {
Anton Korobeynikov9a232f42009-08-21 12:41:24 +00006401 unsigned OpNum = (PFEntry >> 26) & 0x0F;
6402 unsigned LHSID = (PFEntry >> 13) & ((1 << 13)-1);
6403 unsigned RHSID = (PFEntry >> 0) & ((1 << 13)-1);
6404
6405 enum {
6406 OP_COPY = 0, // Copy, used for things like <u,u,u,3> to say it is <0,1,2,3>
6407 OP_VREV,
6408 OP_VDUP0,
6409 OP_VDUP1,
6410 OP_VDUP2,
6411 OP_VDUP3,
6412 OP_VEXT1,
6413 OP_VEXT2,
6414 OP_VEXT3,
6415 OP_VUZPL, // VUZP, left result
6416 OP_VUZPR, // VUZP, right result
6417 OP_VZIPL, // VZIP, left result
6418 OP_VZIPR, // VZIP, right result
6419 OP_VTRNL, // VTRN, left result
6420 OP_VTRNR // VTRN, right result
6421 };
6422
6423 if (OpNum == OP_COPY) {
6424 if (LHSID == (1*9+2)*9+3) return LHS;
6425 assert(LHSID == ((4*9+5)*9+6)*9+7 && "Illegal OP_COPY!");
6426 return RHS;
6427 }
6428
6429 SDValue OpLHS, OpRHS;
6430 OpLHS = GeneratePerfectShuffle(PerfectShuffleTable[LHSID], LHS, RHS, DAG, dl);
6431 OpRHS = GeneratePerfectShuffle(PerfectShuffleTable[RHSID], LHS, RHS, DAG, dl);
6432 EVT VT = OpLHS.getValueType();
6433
6434 switch (OpNum) {
6435 default: llvm_unreachable("Unknown shuffle opcode!");
6436 case OP_VREV:
Tanya Lattner48b182c2011-05-18 06:42:21 +00006437 // VREV divides the vector in half and swaps within the half.
Tanya Lattner1d117202011-05-18 21:44:54 +00006438 if (VT.getVectorElementType() == MVT::i32 ||
6439 VT.getVectorElementType() == MVT::f32)
Tanya Lattner48b182c2011-05-18 06:42:21 +00006440 return DAG.getNode(ARMISD::VREV64, dl, VT, OpLHS);
6441 // vrev <4 x i16> -> VREV32
6442 if (VT.getVectorElementType() == MVT::i16)
6443 return DAG.getNode(ARMISD::VREV32, dl, VT, OpLHS);
6444 // vrev <4 x i8> -> VREV16
6445 assert(VT.getVectorElementType() == MVT::i8);
6446 return DAG.getNode(ARMISD::VREV16, dl, VT, OpLHS);
Anton Korobeynikov9a232f42009-08-21 12:41:24 +00006447 case OP_VDUP0:
6448 case OP_VDUP1:
6449 case OP_VDUP2:
6450 case OP_VDUP3:
6451 return DAG.getNode(ARMISD::VDUPLANE, dl, VT,
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +00006452 OpLHS, DAG.getConstant(OpNum-OP_VDUP0, dl, MVT::i32));
Anton Korobeynikov9a232f42009-08-21 12:41:24 +00006453 case OP_VEXT1:
6454 case OP_VEXT2:
6455 case OP_VEXT3:
6456 return DAG.getNode(ARMISD::VEXT, dl, VT,
6457 OpLHS, OpRHS,
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +00006458 DAG.getConstant(OpNum - OP_VEXT1 + 1, dl, MVT::i32));
Anton Korobeynikov9a232f42009-08-21 12:41:24 +00006459 case OP_VUZPL:
6460 case OP_VUZPR:
Anton Korobeynikov232b19c2009-08-21 12:41:42 +00006461 return DAG.getNode(ARMISD::VUZP, dl, DAG.getVTList(VT, VT),
Anton Korobeynikov9a232f42009-08-21 12:41:24 +00006462 OpLHS, OpRHS).getValue(OpNum-OP_VUZPL);
6463 case OP_VZIPL:
6464 case OP_VZIPR:
Anton Korobeynikov232b19c2009-08-21 12:41:42 +00006465 return DAG.getNode(ARMISD::VZIP, dl, DAG.getVTList(VT, VT),
Anton Korobeynikov9a232f42009-08-21 12:41:24 +00006466 OpLHS, OpRHS).getValue(OpNum-OP_VZIPL);
6467 case OP_VTRNL:
6468 case OP_VTRNR:
Anton Korobeynikov232b19c2009-08-21 12:41:42 +00006469 return DAG.getNode(ARMISD::VTRN, dl, DAG.getVTList(VT, VT),
6470 OpLHS, OpRHS).getValue(OpNum-OP_VTRNL);
Anton Korobeynikov9a232f42009-08-21 12:41:24 +00006471 }
6472}
6473
Bill Wendlinge1fd78f2011-03-14 23:02:38 +00006474static SDValue LowerVECTOR_SHUFFLEv8i8(SDValue Op,
Benjamin Kramer339ced42012-01-15 13:16:05 +00006475 ArrayRef<int> ShuffleMask,
Bill Wendlinge1fd78f2011-03-14 23:02:38 +00006476 SelectionDAG &DAG) {
6477 // Check to see if we can use the VTBL instruction.
6478 SDValue V1 = Op.getOperand(0);
6479 SDValue V2 = Op.getOperand(1);
Andrew Trickef9de2a2013-05-25 02:42:55 +00006480 SDLoc DL(Op);
Bill Wendlinge1fd78f2011-03-14 23:02:38 +00006481
6482 SmallVector<SDValue, 8> VTBLMask;
Benjamin Kramer339ced42012-01-15 13:16:05 +00006483 for (ArrayRef<int>::iterator
Bill Wendlinge1fd78f2011-03-14 23:02:38 +00006484 I = ShuffleMask.begin(), E = ShuffleMask.end(); I != E; ++I)
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +00006485 VTBLMask.push_back(DAG.getConstant(*I, DL, MVT::i32));
Bill Wendlinge1fd78f2011-03-14 23:02:38 +00006486
Sanjay Patel57195842016-03-14 17:28:46 +00006487 if (V2.getNode()->isUndef())
Bill Wendlinge1fd78f2011-03-14 23:02:38 +00006488 return DAG.getNode(ARMISD::VTBL1, DL, MVT::v8i8, V1,
Ahmed Bougacha128f8732016-04-26 21:15:30 +00006489 DAG.getBuildVector(MVT::v8i8, DL, VTBLMask));
Bill Wendlingebecb332011-03-15 20:47:26 +00006490
Owen Anderson77aa2662011-04-05 21:48:57 +00006491 return DAG.getNode(ARMISD::VTBL2, DL, MVT::v8i8, V1, V2,
Ahmed Bougacha128f8732016-04-26 21:15:30 +00006492 DAG.getBuildVector(MVT::v8i8, DL, VTBLMask));
Bill Wendlinge1fd78f2011-03-14 23:02:38 +00006493}
6494
Arnold Schwaighofer1f3d3ca2013-02-12 01:58:32 +00006495static SDValue LowerReverse_VECTOR_SHUFFLEv16i8_v8i16(SDValue Op,
6496 SelectionDAG &DAG) {
Andrew Trickef9de2a2013-05-25 02:42:55 +00006497 SDLoc DL(Op);
Arnold Schwaighofer1f3d3ca2013-02-12 01:58:32 +00006498 SDValue OpLHS = Op.getOperand(0);
6499 EVT VT = OpLHS.getValueType();
6500
6501 assert((VT == MVT::v8i16 || VT == MVT::v16i8) &&
6502 "Expect an v8i16/v16i8 type");
6503 OpLHS = DAG.getNode(ARMISD::VREV64, DL, VT, OpLHS);
6504 // For a v16i8 type: After the VREV, we have got <8, ...15, 8, ..., 0>. Now,
6505 // extract the first 8 bytes into the top double word and the last 8 bytes
6506 // into the bottom double word. The v8i16 case is similar.
6507 unsigned ExtractNum = (VT == MVT::v16i8) ? 8 : 4;
6508 return DAG.getNode(ARMISD::VEXT, DL, VT, OpLHS, OpLHS,
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +00006509 DAG.getConstant(ExtractNum, DL, MVT::i32));
Arnold Schwaighofer1f3d3ca2013-02-12 01:58:32 +00006510}
6511
Bob Wilson2e076c42009-06-22 23:27:02 +00006512static SDValue LowerVECTOR_SHUFFLE(SDValue Op, SelectionDAG &DAG) {
Anton Korobeynikov9a232f42009-08-21 12:41:24 +00006513 SDValue V1 = Op.getOperand(0);
6514 SDValue V2 = Op.getOperand(1);
Andrew Trickef9de2a2013-05-25 02:42:55 +00006515 SDLoc dl(Op);
Bob Wilsonea3a4022009-08-12 22:31:50 +00006516 EVT VT = Op.getValueType();
Anton Korobeynikov9a232f42009-08-21 12:41:24 +00006517 ShuffleVectorSDNode *SVN = cast<ShuffleVectorSDNode>(Op.getNode());
Bob Wilsonea3a4022009-08-12 22:31:50 +00006518
Bob Wilsonc6800b52009-08-13 02:13:04 +00006519 // Convert shuffles that are directly supported on NEON to target-specific
6520 // DAG nodes, instead of keeping them as shuffles and matching them again
6521 // during code selection. This is more efficient and avoids the possibility
6522 // of inconsistencies between legalization and selection.
Bob Wilson3e4c0122009-08-13 06:01:30 +00006523 // FIXME: floating-point vectors should be canonicalized to integer vectors
6524 // of the same time so that they get CSEd properly.
Benjamin Kramer339ced42012-01-15 13:16:05 +00006525 ArrayRef<int> ShuffleMask = SVN->getMask();
Anton Korobeynikovc32e99e2009-08-21 12:40:07 +00006526
Bob Wilson846bd792010-06-07 23:53:38 +00006527 unsigned EltSize = VT.getVectorElementType().getSizeInBits();
6528 if (EltSize <= 32) {
Craig Topperbc56e3b2016-06-30 04:38:51 +00006529 if (SVN->isSplat()) {
Bob Wilson846bd792010-06-07 23:53:38 +00006530 int Lane = SVN->getSplatIndex();
6531 // If this is undef splat, generate it via "just" vdup, if possible.
6532 if (Lane == -1) Lane = 0;
Anton Korobeynikov4d237542009-11-02 00:12:06 +00006533
Dan Gohman198b7ff2011-11-03 21:49:52 +00006534 // Test if V1 is a SCALAR_TO_VECTOR.
Bob Wilson846bd792010-06-07 23:53:38 +00006535 if (Lane == 0 && V1.getOpcode() == ISD::SCALAR_TO_VECTOR) {
6536 return DAG.getNode(ARMISD::VDUP, dl, VT, V1.getOperand(0));
6537 }
Dan Gohman198b7ff2011-11-03 21:49:52 +00006538 // Test if V1 is a BUILD_VECTOR which is equivalent to a SCALAR_TO_VECTOR
6539 // (and probably will turn into a SCALAR_TO_VECTOR once legalization
6540 // reaches it).
6541 if (Lane == 0 && V1.getOpcode() == ISD::BUILD_VECTOR &&
6542 !isa<ConstantSDNode>(V1.getOperand(0))) {
6543 bool IsScalarToVector = true;
6544 for (unsigned i = 1, e = V1.getNumOperands(); i != e; ++i)
Sanjay Patel75068522016-03-14 18:09:43 +00006545 if (!V1.getOperand(i).isUndef()) {
Dan Gohman198b7ff2011-11-03 21:49:52 +00006546 IsScalarToVector = false;
6547 break;
6548 }
6549 if (IsScalarToVector)
6550 return DAG.getNode(ARMISD::VDUP, dl, VT, V1.getOperand(0));
6551 }
Bob Wilson846bd792010-06-07 23:53:38 +00006552 return DAG.getNode(ARMISD::VDUPLANE, dl, VT, V1,
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +00006553 DAG.getConstant(Lane, dl, MVT::i32));
Bob Wilsoneb54d512009-08-14 05:13:08 +00006554 }
Bob Wilson846bd792010-06-07 23:53:38 +00006555
6556 bool ReverseVEXT;
6557 unsigned Imm;
6558 if (isVEXTMask(ShuffleMask, VT, ReverseVEXT, Imm)) {
6559 if (ReverseVEXT)
6560 std::swap(V1, V2);
6561 return DAG.getNode(ARMISD::VEXT, dl, VT, V1, V2,
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +00006562 DAG.getConstant(Imm, dl, MVT::i32));
Bob Wilson846bd792010-06-07 23:53:38 +00006563 }
6564
6565 if (isVREVMask(ShuffleMask, VT, 64))
6566 return DAG.getNode(ARMISD::VREV64, dl, VT, V1);
6567 if (isVREVMask(ShuffleMask, VT, 32))
6568 return DAG.getNode(ARMISD::VREV32, dl, VT, V1);
6569 if (isVREVMask(ShuffleMask, VT, 16))
6570 return DAG.getNode(ARMISD::VREV16, dl, VT, V1);
6571
Sanjay Patel57195842016-03-14 17:28:46 +00006572 if (V2->isUndef() && isSingletonVEXTMask(ShuffleMask, VT, Imm)) {
Quentin Colombet8e1fe842012-11-02 21:32:17 +00006573 return DAG.getNode(ARMISD::VEXT, dl, VT, V1, V1,
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +00006574 DAG.getConstant(Imm, dl, MVT::i32));
Quentin Colombet8e1fe842012-11-02 21:32:17 +00006575 }
6576
Bob Wilson846bd792010-06-07 23:53:38 +00006577 // Check for Neon shuffles that modify both input vectors in place.
6578 // If both results are used, i.e., if there are two shuffles with the same
6579 // source operands and with masks corresponding to both results of one of
6580 // these operations, DAG memoization will ensure that a single node is
6581 // used for both shuffles.
6582 unsigned WhichResult;
Ahmed Bougacha2ffa91f2015-06-19 02:25:01 +00006583 bool isV_UNDEF;
6584 if (unsigned ShuffleOpc = isNEONTwoResultShuffleMask(
6585 ShuffleMask, VT, WhichResult, isV_UNDEF)) {
6586 if (isV_UNDEF)
6587 V2 = V1;
6588 return DAG.getNode(ShuffleOpc, dl, DAG.getVTList(VT, VT), V1, V2)
6589 .getValue(WhichResult);
6590 }
Bob Wilson846bd792010-06-07 23:53:38 +00006591
Ahmed Bougacha9a909422015-06-19 02:32:35 +00006592 // Also check for these shuffles through CONCAT_VECTORS: we canonicalize
6593 // shuffles that produce a result larger than their operands with:
6594 // shuffle(concat(v1, undef), concat(v2, undef))
6595 // ->
6596 // shuffle(concat(v1, v2), undef)
6597 // because we can access quad vectors (see PerformVECTOR_SHUFFLECombine).
6598 //
6599 // This is useful in the general case, but there are special cases where
6600 // native shuffles produce larger results: the two-result ops.
6601 //
6602 // Look through the concat when lowering them:
6603 // shuffle(concat(v1, v2), undef)
6604 // ->
6605 // concat(VZIP(v1, v2):0, :1)
6606 //
Sanjay Patel57195842016-03-14 17:28:46 +00006607 if (V1->getOpcode() == ISD::CONCAT_VECTORS && V2->isUndef()) {
Ahmed Bougacha9a909422015-06-19 02:32:35 +00006608 SDValue SubV1 = V1->getOperand(0);
6609 SDValue SubV2 = V1->getOperand(1);
6610 EVT SubVT = SubV1.getValueType();
6611
6612 // We expect these to have been canonicalized to -1.
David Majnemer0a16c222016-08-11 21:15:00 +00006613 assert(all_of(ShuffleMask, [&](int i) {
Ahmed Bougacha9a909422015-06-19 02:32:35 +00006614 return i < (int)VT.getVectorNumElements();
6615 }) && "Unexpected shuffle index into UNDEF operand!");
6616
6617 if (unsigned ShuffleOpc = isNEONTwoResultShuffleMask(
6618 ShuffleMask, SubVT, WhichResult, isV_UNDEF)) {
6619 if (isV_UNDEF)
6620 SubV2 = SubV1;
6621 assert((WhichResult == 0) &&
6622 "In-place shuffle of concat can only have one result!");
6623 SDValue Res = DAG.getNode(ShuffleOpc, dl, DAG.getVTList(SubVT, SubVT),
6624 SubV1, SubV2);
6625 return DAG.getNode(ISD::CONCAT_VECTORS, dl, VT, Res.getValue(0),
6626 Res.getValue(1));
6627 }
6628 }
Bob Wilsoncce31f62009-08-14 05:08:32 +00006629 }
Bob Wilson32cd8552009-08-19 17:03:43 +00006630
Bob Wilsona7062312009-08-21 20:54:19 +00006631 // If the shuffle is not directly supported and it has 4 elements, use
6632 // the PerfectShuffle-generated table to synthesize it from other shuffles.
Bob Wilson91fdf682010-05-22 00:23:12 +00006633 unsigned NumElts = VT.getVectorNumElements();
6634 if (NumElts == 4) {
Anton Korobeynikov9a232f42009-08-21 12:41:24 +00006635 unsigned PFIndexes[4];
6636 for (unsigned i = 0; i != 4; ++i) {
6637 if (ShuffleMask[i] < 0)
6638 PFIndexes[i] = 8;
6639 else
6640 PFIndexes[i] = ShuffleMask[i];
6641 }
6642
6643 // Compute the index in the perfect shuffle table.
6644 unsigned PFTableIndex =
6645 PFIndexes[0]*9*9*9+PFIndexes[1]*9*9+PFIndexes[2]*9+PFIndexes[3];
Anton Korobeynikov9a232f42009-08-21 12:41:24 +00006646 unsigned PFEntry = PerfectShuffleTable[PFTableIndex];
6647 unsigned Cost = (PFEntry >> 30);
6648
6649 if (Cost <= 4)
6650 return GeneratePerfectShuffle(PFEntry, V1, V2, DAG, dl);
6651 }
Bob Wilsonea3a4022009-08-12 22:31:50 +00006652
Bob Wilsond8a9a042010-06-04 00:04:02 +00006653 // Implement shuffles with 32- or 64-bit elements as ARMISD::BUILD_VECTORs.
Bob Wilson91fdf682010-05-22 00:23:12 +00006654 if (EltSize >= 32) {
6655 // Do the expansion with floating-point types, since that is what the VFP
6656 // registers are defined to use, and since i64 is not legal.
6657 EVT EltVT = EVT::getFloatingPointVT(EltSize);
6658 EVT VecVT = EVT::getVectorVT(*DAG.getContext(), EltVT, NumElts);
Wesley Peck527da1b2010-11-23 03:31:01 +00006659 V1 = DAG.getNode(ISD::BITCAST, dl, VecVT, V1);
6660 V2 = DAG.getNode(ISD::BITCAST, dl, VecVT, V2);
Bob Wilsond8a9a042010-06-04 00:04:02 +00006661 SmallVector<SDValue, 8> Ops;
Bob Wilson91fdf682010-05-22 00:23:12 +00006662 for (unsigned i = 0; i < NumElts; ++i) {
Bob Wilson59549942010-05-20 18:39:53 +00006663 if (ShuffleMask[i] < 0)
Bob Wilsond8a9a042010-06-04 00:04:02 +00006664 Ops.push_back(DAG.getUNDEF(EltVT));
6665 else
6666 Ops.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT,
6667 ShuffleMask[i] < (int)NumElts ? V1 : V2,
6668 DAG.getConstant(ShuffleMask[i] & (NumElts-1),
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +00006669 dl, MVT::i32)));
Bob Wilson59549942010-05-20 18:39:53 +00006670 }
Craig Topper48d114b2014-04-26 18:35:24 +00006671 SDValue Val = DAG.getNode(ARMISD::BUILD_VECTOR, dl, VecVT, Ops);
Wesley Peck527da1b2010-11-23 03:31:01 +00006672 return DAG.getNode(ISD::BITCAST, dl, VT, Val);
Bob Wilson59549942010-05-20 18:39:53 +00006673 }
6674
Arnold Schwaighofer1f3d3ca2013-02-12 01:58:32 +00006675 if ((VT == MVT::v8i16 || VT == MVT::v16i8) && isReverseMask(ShuffleMask, VT))
6676 return LowerReverse_VECTOR_SHUFFLEv16i8_v8i16(Op, DAG);
6677
Ahmed Bougachaf8dfb472016-02-09 22:54:12 +00006678 if (VT == MVT::v8i8)
6679 if (SDValue NewOp = LowerVECTOR_SHUFFLEv8i8(Op, ShuffleMask, DAG))
Bill Wendlinge1fd78f2011-03-14 23:02:38 +00006680 return NewOp;
Bill Wendlinge1fd78f2011-03-14 23:02:38 +00006681
Bob Wilson6f34e272009-08-14 05:16:33 +00006682 return SDValue();
Bob Wilson2e076c42009-06-22 23:27:02 +00006683}
6684
Eli Friedmana5e244c2011-10-24 23:08:52 +00006685static SDValue LowerINSERT_VECTOR_ELT(SDValue Op, SelectionDAG &DAG) {
6686 // INSERT_VECTOR_ELT is legal only for immediate indexes.
6687 SDValue Lane = Op.getOperand(2);
6688 if (!isa<ConstantSDNode>(Lane))
6689 return SDValue();
6690
6691 return Op;
6692}
6693
Bob Wilson2e076c42009-06-22 23:27:02 +00006694static SDValue LowerEXTRACT_VECTOR_ELT(SDValue Op, SelectionDAG &DAG) {
Bob Wilsonceb49292010-11-03 16:24:50 +00006695 // EXTRACT_VECTOR_ELT is legal only for immediate indexes.
Bob Wilson2e076c42009-06-22 23:27:02 +00006696 SDValue Lane = Op.getOperand(1);
Bob Wilsonceb49292010-11-03 16:24:50 +00006697 if (!isa<ConstantSDNode>(Lane))
6698 return SDValue();
6699
6700 SDValue Vec = Op.getOperand(0);
6701 if (Op.getValueType() == MVT::i32 &&
6702 Vec.getValueType().getVectorElementType().getSizeInBits() < 32) {
Andrew Trickef9de2a2013-05-25 02:42:55 +00006703 SDLoc dl(Op);
Bob Wilsonceb49292010-11-03 16:24:50 +00006704 return DAG.getNode(ARMISD::VGETLANEu, dl, MVT::i32, Vec, Lane);
6705 }
6706
6707 return Op;
Bob Wilson2e076c42009-06-22 23:27:02 +00006708}
6709
Bob Wilsonf307e0b2009-08-03 20:36:38 +00006710static SDValue LowerCONCAT_VECTORS(SDValue Op, SelectionDAG &DAG) {
6711 // The only time a CONCAT_VECTORS operation can have legal types is when
6712 // two 64-bit vectors are concatenated to a 128-bit vector.
6713 assert(Op.getValueType().is128BitVector() && Op.getNumOperands() == 2 &&
6714 "unexpected CONCAT_VECTORS");
Andrew Trickef9de2a2013-05-25 02:42:55 +00006715 SDLoc dl(Op);
Owen Anderson9f944592009-08-11 20:47:22 +00006716 SDValue Val = DAG.getUNDEF(MVT::v2f64);
Bob Wilsonf307e0b2009-08-03 20:36:38 +00006717 SDValue Op0 = Op.getOperand(0);
6718 SDValue Op1 = Op.getOperand(1);
Sanjay Patel75068522016-03-14 18:09:43 +00006719 if (!Op0.isUndef())
Owen Anderson9f944592009-08-11 20:47:22 +00006720 Val = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, MVT::v2f64, Val,
Wesley Peck527da1b2010-11-23 03:31:01 +00006721 DAG.getNode(ISD::BITCAST, dl, MVT::f64, Op0),
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +00006722 DAG.getIntPtrConstant(0, dl));
Sanjay Patel75068522016-03-14 18:09:43 +00006723 if (!Op1.isUndef())
Owen Anderson9f944592009-08-11 20:47:22 +00006724 Val = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, MVT::v2f64, Val,
Wesley Peck527da1b2010-11-23 03:31:01 +00006725 DAG.getNode(ISD::BITCAST, dl, MVT::f64, Op1),
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +00006726 DAG.getIntPtrConstant(1, dl));
Wesley Peck527da1b2010-11-23 03:31:01 +00006727 return DAG.getNode(ISD::BITCAST, dl, Op.getValueType(), Val);
Bob Wilson2e076c42009-06-22 23:27:02 +00006728}
6729
Bob Wilsond7d2cf72010-11-23 19:38:38 +00006730/// isExtendedBUILD_VECTOR - Check if N is a constant BUILD_VECTOR where each
6731/// element has been zero/sign-extended, depending on the isSigned parameter,
6732/// from an integer type half its size.
6733static bool isExtendedBUILD_VECTOR(SDNode *N, SelectionDAG &DAG,
6734 bool isSigned) {
6735 // A v2i64 BUILD_VECTOR will have been legalized to a BITCAST from v4i32.
6736 EVT VT = N->getValueType(0);
6737 if (VT == MVT::v2i64 && N->getOpcode() == ISD::BITCAST) {
6738 SDNode *BVN = N->getOperand(0).getNode();
6739 if (BVN->getValueType(0) != MVT::v4i32 ||
6740 BVN->getOpcode() != ISD::BUILD_VECTOR)
6741 return false;
Mehdi Aminiffc14022015-07-08 01:00:38 +00006742 unsigned LoElt = DAG.getDataLayout().isBigEndian() ? 1 : 0;
Bob Wilsond7d2cf72010-11-23 19:38:38 +00006743 unsigned HiElt = 1 - LoElt;
6744 ConstantSDNode *Lo0 = dyn_cast<ConstantSDNode>(BVN->getOperand(LoElt));
6745 ConstantSDNode *Hi0 = dyn_cast<ConstantSDNode>(BVN->getOperand(HiElt));
6746 ConstantSDNode *Lo1 = dyn_cast<ConstantSDNode>(BVN->getOperand(LoElt+2));
6747 ConstantSDNode *Hi1 = dyn_cast<ConstantSDNode>(BVN->getOperand(HiElt+2));
6748 if (!Lo0 || !Hi0 || !Lo1 || !Hi1)
6749 return false;
6750 if (isSigned) {
6751 if (Hi0->getSExtValue() == Lo0->getSExtValue() >> 32 &&
6752 Hi1->getSExtValue() == Lo1->getSExtValue() >> 32)
6753 return true;
6754 } else {
6755 if (Hi0->isNullValue() && Hi1->isNullValue())
6756 return true;
6757 }
6758 return false;
6759 }
6760
6761 if (N->getOpcode() != ISD::BUILD_VECTOR)
6762 return false;
6763
6764 for (unsigned i = 0, e = N->getNumOperands(); i != e; ++i) {
6765 SDNode *Elt = N->getOperand(i).getNode();
6766 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Elt)) {
6767 unsigned EltSize = VT.getVectorElementType().getSizeInBits();
6768 unsigned HalfSize = EltSize / 2;
6769 if (isSigned) {
Bob Wilson93b0f7b2011-10-18 18:46:49 +00006770 if (!isIntN(HalfSize, C->getSExtValue()))
Bob Wilsond7d2cf72010-11-23 19:38:38 +00006771 return false;
6772 } else {
Bob Wilson93b0f7b2011-10-18 18:46:49 +00006773 if (!isUIntN(HalfSize, C->getZExtValue()))
Bob Wilsond7d2cf72010-11-23 19:38:38 +00006774 return false;
6775 }
6776 continue;
6777 }
6778 return false;
6779 }
6780
6781 return true;
6782}
6783
6784/// isSignExtended - Check if a node is a vector value that is sign-extended
6785/// or a constant BUILD_VECTOR with sign-extended elements.
6786static bool isSignExtended(SDNode *N, SelectionDAG &DAG) {
6787 if (N->getOpcode() == ISD::SIGN_EXTEND || ISD::isSEXTLoad(N))
6788 return true;
6789 if (isExtendedBUILD_VECTOR(N, DAG, true))
6790 return true;
6791 return false;
6792}
6793
6794/// isZeroExtended - Check if a node is a vector value that is zero-extended
6795/// or a constant BUILD_VECTOR with zero-extended elements.
6796static bool isZeroExtended(SDNode *N, SelectionDAG &DAG) {
6797 if (N->getOpcode() == ISD::ZERO_EXTEND || ISD::isZEXTLoad(N))
6798 return true;
6799 if (isExtendedBUILD_VECTOR(N, DAG, false))
6800 return true;
6801 return false;
6802}
6803
Arnold Schwaighoferaf85f602013-05-14 22:33:24 +00006804static EVT getExtensionTo64Bits(const EVT &OrigVT) {
6805 if (OrigVT.getSizeInBits() >= 64)
6806 return OrigVT;
6807
6808 assert(OrigVT.isSimple() && "Expecting a simple value type");
6809
6810 MVT::SimpleValueType OrigSimpleTy = OrigVT.getSimpleVT().SimpleTy;
6811 switch (OrigSimpleTy) {
6812 default: llvm_unreachable("Unexpected Vector Type");
6813 case MVT::v2i8:
6814 case MVT::v2i16:
6815 return MVT::v2i32;
6816 case MVT::v4i8:
6817 return MVT::v4i16;
6818 }
6819}
6820
Sebastian Popa204f722012-11-30 19:08:04 +00006821/// AddRequiredExtensionForVMULL - Add a sign/zero extension to extend the total
6822/// value size to 64 bits. We need a 64-bit D register as an operand to VMULL.
6823/// We insert the required extension here to get the vector to fill a D register.
6824static SDValue AddRequiredExtensionForVMULL(SDValue N, SelectionDAG &DAG,
6825 const EVT &OrigTy,
6826 const EVT &ExtTy,
6827 unsigned ExtOpcode) {
6828 // The vector originally had a size of OrigTy. It was then extended to ExtTy.
6829 // We expect the ExtTy to be 128-bits total. If the OrigTy is less than
6830 // 64-bits we need to insert a new extension so that it will be 64-bits.
6831 assert(ExtTy.is128BitVector() && "Unexpected extension size");
6832 if (OrigTy.getSizeInBits() >= 64)
6833 return N;
6834
6835 // Must extend size to at least 64 bits to be used as an operand for VMULL.
Arnold Schwaighoferaf85f602013-05-14 22:33:24 +00006836 EVT NewVT = getExtensionTo64Bits(OrigTy);
6837
Andrew Trickef9de2a2013-05-25 02:42:55 +00006838 return DAG.getNode(ExtOpcode, SDLoc(N), NewVT, N);
Sebastian Popa204f722012-11-30 19:08:04 +00006839}
6840
6841/// SkipLoadExtensionForVMULL - return a load of the original vector size that
6842/// does not do any sign/zero extension. If the original vector is less
6843/// than 64 bits, an appropriate extension will be added after the load to
6844/// reach a total size of 64 bits. We have to add the extension separately
6845/// because ARM does not have a sign/zero extending load for vectors.
6846static SDValue SkipLoadExtensionForVMULL(LoadSDNode *LD, SelectionDAG& DAG) {
Arnold Schwaighoferaf85f602013-05-14 22:33:24 +00006847 EVT ExtendedTy = getExtensionTo64Bits(LD->getMemoryVT());
6848
6849 // The load already has the right type.
6850 if (ExtendedTy == LD->getMemoryVT())
Andrew Trickef9de2a2013-05-25 02:42:55 +00006851 return DAG.getLoad(LD->getMemoryVT(), SDLoc(LD), LD->getChain(),
Justin Lebar9c375812016-07-15 18:27:10 +00006852 LD->getBasePtr(), LD->getPointerInfo(),
6853 LD->getAlignment(), LD->getMemOperand()->getFlags());
Arnold Schwaighoferaf85f602013-05-14 22:33:24 +00006854
6855 // We need to create a zextload/sextload. We cannot just create a load
6856 // followed by a zext/zext node because LowerMUL is also run during normal
6857 // operation legalization where we can't create illegal types.
Andrew Trickef9de2a2013-05-25 02:42:55 +00006858 return DAG.getExtLoad(LD->getExtensionType(), SDLoc(LD), ExtendedTy,
Arnold Schwaighoferaf85f602013-05-14 22:33:24 +00006859 LD->getChain(), LD->getBasePtr(), LD->getPointerInfo(),
Justin Lebar9c375812016-07-15 18:27:10 +00006860 LD->getMemoryVT(), LD->getAlignment(),
6861 LD->getMemOperand()->getFlags());
Sebastian Popa204f722012-11-30 19:08:04 +00006862}
6863
6864/// SkipExtensionForVMULL - For a node that is a SIGN_EXTEND, ZERO_EXTEND,
6865/// extending load, or BUILD_VECTOR with extended elements, return the
6866/// unextended value. The unextended vector should be 64 bits so that it can
6867/// be used as an operand to a VMULL instruction. If the original vector size
6868/// before extension is less than 64 bits we add a an extension to resize
6869/// the vector to 64 bits.
6870static SDValue SkipExtensionForVMULL(SDNode *N, SelectionDAG &DAG) {
Bob Wilson38ab35a2010-09-01 23:50:19 +00006871 if (N->getOpcode() == ISD::SIGN_EXTEND || N->getOpcode() == ISD::ZERO_EXTEND)
Sebastian Popa204f722012-11-30 19:08:04 +00006872 return AddRequiredExtensionForVMULL(N->getOperand(0), DAG,
6873 N->getOperand(0)->getValueType(0),
6874 N->getValueType(0),
6875 N->getOpcode());
6876
Bob Wilsond7d2cf72010-11-23 19:38:38 +00006877 if (LoadSDNode *LD = dyn_cast<LoadSDNode>(N))
Sebastian Popa204f722012-11-30 19:08:04 +00006878 return SkipLoadExtensionForVMULL(LD, DAG);
6879
Bob Wilsond7d2cf72010-11-23 19:38:38 +00006880 // Otherwise, the value must be a BUILD_VECTOR. For v2i64, it will
6881 // have been legalized as a BITCAST from v4i32.
6882 if (N->getOpcode() == ISD::BITCAST) {
6883 SDNode *BVN = N->getOperand(0).getNode();
6884 assert(BVN->getOpcode() == ISD::BUILD_VECTOR &&
6885 BVN->getValueType(0) == MVT::v4i32 && "expected v4i32 BUILD_VECTOR");
Mehdi Aminiffc14022015-07-08 01:00:38 +00006886 unsigned LowElt = DAG.getDataLayout().isBigEndian() ? 1 : 0;
Ahmed Bougacha128f8732016-04-26 21:15:30 +00006887 return DAG.getBuildVector(
6888 MVT::v2i32, SDLoc(N),
6889 {BVN->getOperand(LowElt), BVN->getOperand(LowElt + 2)});
Bob Wilsond7d2cf72010-11-23 19:38:38 +00006890 }
6891 // Construct a new BUILD_VECTOR with elements truncated to half the size.
6892 assert(N->getOpcode() == ISD::BUILD_VECTOR && "expected BUILD_VECTOR");
6893 EVT VT = N->getValueType(0);
6894 unsigned EltSize = VT.getVectorElementType().getSizeInBits() / 2;
6895 unsigned NumElts = VT.getVectorNumElements();
6896 MVT TruncVT = MVT::getIntegerVT(EltSize);
6897 SmallVector<SDValue, 8> Ops;
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +00006898 SDLoc dl(N);
Bob Wilsond7d2cf72010-11-23 19:38:38 +00006899 for (unsigned i = 0; i != NumElts; ++i) {
6900 ConstantSDNode *C = cast<ConstantSDNode>(N->getOperand(i));
6901 const APInt &CInt = C->getAPIntValue();
Bob Wilson9245c932012-04-30 16:53:34 +00006902 // Element types smaller than 32 bits are not legal, so use i32 elements.
6903 // The values are implicitly truncated so sext vs. zext doesn't matter.
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +00006904 Ops.push_back(DAG.getConstant(CInt.zextOrTrunc(32), dl, MVT::i32));
Bob Wilsond7d2cf72010-11-23 19:38:38 +00006905 }
Ahmed Bougacha128f8732016-04-26 21:15:30 +00006906 return DAG.getBuildVector(MVT::getVectorVT(TruncVT, NumElts), dl, Ops);
Bob Wilson38ab35a2010-09-01 23:50:19 +00006907}
6908
Evan Chenge2086e72011-03-29 01:56:09 +00006909static bool isAddSubSExt(SDNode *N, SelectionDAG &DAG) {
6910 unsigned Opcode = N->getOpcode();
6911 if (Opcode == ISD::ADD || Opcode == ISD::SUB) {
6912 SDNode *N0 = N->getOperand(0).getNode();
6913 SDNode *N1 = N->getOperand(1).getNode();
6914 return N0->hasOneUse() && N1->hasOneUse() &&
6915 isSignExtended(N0, DAG) && isSignExtended(N1, DAG);
6916 }
6917 return false;
6918}
6919
6920static bool isAddSubZExt(SDNode *N, SelectionDAG &DAG) {
6921 unsigned Opcode = N->getOpcode();
6922 if (Opcode == ISD::ADD || Opcode == ISD::SUB) {
6923 SDNode *N0 = N->getOperand(0).getNode();
6924 SDNode *N1 = N->getOperand(1).getNode();
6925 return N0->hasOneUse() && N1->hasOneUse() &&
6926 isZeroExtended(N0, DAG) && isZeroExtended(N1, DAG);
6927 }
6928 return false;
6929}
6930
Bob Wilson38ab35a2010-09-01 23:50:19 +00006931static SDValue LowerMUL(SDValue Op, SelectionDAG &DAG) {
6932 // Multiplications are only custom-lowered for 128-bit vectors so that
6933 // VMULL can be detected. Otherwise v2i64 multiplications are not legal.
6934 EVT VT = Op.getValueType();
Sebastian Popa204f722012-11-30 19:08:04 +00006935 assert(VT.is128BitVector() && VT.isInteger() &&
6936 "unexpected type for custom-lowering ISD::MUL");
Bob Wilson38ab35a2010-09-01 23:50:19 +00006937 SDNode *N0 = Op.getOperand(0).getNode();
6938 SDNode *N1 = Op.getOperand(1).getNode();
6939 unsigned NewOpc = 0;
Evan Chenge2086e72011-03-29 01:56:09 +00006940 bool isMLA = false;
6941 bool isN0SExt = isSignExtended(N0, DAG);
6942 bool isN1SExt = isSignExtended(N1, DAG);
6943 if (isN0SExt && isN1SExt)
Bob Wilson38ab35a2010-09-01 23:50:19 +00006944 NewOpc = ARMISD::VMULLs;
Evan Chenge2086e72011-03-29 01:56:09 +00006945 else {
6946 bool isN0ZExt = isZeroExtended(N0, DAG);
6947 bool isN1ZExt = isZeroExtended(N1, DAG);
6948 if (isN0ZExt && isN1ZExt)
6949 NewOpc = ARMISD::VMULLu;
6950 else if (isN1SExt || isN1ZExt) {
6951 // Look for (s/zext A + s/zext B) * (s/zext C). We want to turn these
6952 // into (s/zext A * s/zext C) + (s/zext B * s/zext C)
6953 if (isN1SExt && isAddSubSExt(N0, DAG)) {
6954 NewOpc = ARMISD::VMULLs;
6955 isMLA = true;
6956 } else if (isN1ZExt && isAddSubZExt(N0, DAG)) {
6957 NewOpc = ARMISD::VMULLu;
6958 isMLA = true;
6959 } else if (isN0ZExt && isAddSubZExt(N1, DAG)) {
6960 std::swap(N0, N1);
6961 NewOpc = ARMISD::VMULLu;
6962 isMLA = true;
6963 }
6964 }
6965
6966 if (!NewOpc) {
6967 if (VT == MVT::v2i64)
6968 // Fall through to expand this. It is not legal.
6969 return SDValue();
6970 else
6971 // Other vector multiplications are legal.
6972 return Op;
6973 }
6974 }
Bob Wilson38ab35a2010-09-01 23:50:19 +00006975
6976 // Legalize to a VMULL instruction.
Andrew Trickef9de2a2013-05-25 02:42:55 +00006977 SDLoc DL(Op);
Evan Chenge2086e72011-03-29 01:56:09 +00006978 SDValue Op0;
Sebastian Popa204f722012-11-30 19:08:04 +00006979 SDValue Op1 = SkipExtensionForVMULL(N1, DAG);
Evan Chenge2086e72011-03-29 01:56:09 +00006980 if (!isMLA) {
Sebastian Popa204f722012-11-30 19:08:04 +00006981 Op0 = SkipExtensionForVMULL(N0, DAG);
Evan Chenge2086e72011-03-29 01:56:09 +00006982 assert(Op0.getValueType().is64BitVector() &&
6983 Op1.getValueType().is64BitVector() &&
6984 "unexpected types for extended operands to VMULL");
6985 return DAG.getNode(NewOpc, DL, VT, Op0, Op1);
6986 }
Bob Wilson38ab35a2010-09-01 23:50:19 +00006987
Evan Chenge2086e72011-03-29 01:56:09 +00006988 // Optimizing (zext A + zext B) * C, to (VMULL A, C) + (VMULL B, C) during
6989 // isel lowering to take advantage of no-stall back to back vmul + vmla.
6990 // vmull q0, d4, d6
6991 // vmlal q0, d5, d6
6992 // is faster than
6993 // vaddl q0, d4, d5
6994 // vmovl q1, d6
6995 // vmul q0, q0, q1
Sebastian Popa204f722012-11-30 19:08:04 +00006996 SDValue N00 = SkipExtensionForVMULL(N0->getOperand(0).getNode(), DAG);
6997 SDValue N01 = SkipExtensionForVMULL(N0->getOperand(1).getNode(), DAG);
Evan Chenge2086e72011-03-29 01:56:09 +00006998 EVT Op1VT = Op1.getValueType();
6999 return DAG.getNode(N0->getOpcode(), DL, VT,
7000 DAG.getNode(NewOpc, DL, VT,
7001 DAG.getNode(ISD::BITCAST, DL, Op1VT, N00), Op1),
7002 DAG.getNode(NewOpc, DL, VT,
7003 DAG.getNode(ISD::BITCAST, DL, Op1VT, N01), Op1));
Bob Wilson38ab35a2010-09-01 23:50:19 +00007004}
7005
Benjamin Kramerbdc49562016-06-12 15:39:02 +00007006static SDValue LowerSDIV_v4i8(SDValue X, SDValue Y, const SDLoc &dl,
7007 SelectionDAG &DAG) {
Sanjay Patela2607012015-09-16 16:31:21 +00007008 // TODO: Should this propagate fast-math-flags?
7009
Nate Begemanfa62d502011-02-11 20:53:29 +00007010 // Convert to float
7011 // float4 xf = vcvt_f32_s32(vmovl_s16(a.lo));
7012 // float4 yf = vcvt_f32_s32(vmovl_s16(b.lo));
7013 X = DAG.getNode(ISD::SIGN_EXTEND, dl, MVT::v4i32, X);
7014 Y = DAG.getNode(ISD::SIGN_EXTEND, dl, MVT::v4i32, Y);
7015 X = DAG.getNode(ISD::SINT_TO_FP, dl, MVT::v4f32, X);
7016 Y = DAG.getNode(ISD::SINT_TO_FP, dl, MVT::v4f32, Y);
7017 // Get reciprocal estimate.
7018 // float4 recip = vrecpeq_f32(yf);
Owen Anderson77aa2662011-04-05 21:48:57 +00007019 Y = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, MVT::v4f32,
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +00007020 DAG.getConstant(Intrinsic::arm_neon_vrecpe, dl, MVT::i32),
7021 Y);
Nate Begemanfa62d502011-02-11 20:53:29 +00007022 // Because char has a smaller range than uchar, we can actually get away
7023 // without any newton steps. This requires that we use a weird bias
7024 // of 0xb000, however (again, this has been exhaustively tested).
7025 // float4 result = as_float4(as_int4(xf*recip) + 0xb000);
7026 X = DAG.getNode(ISD::FMUL, dl, MVT::v4f32, X, Y);
7027 X = DAG.getNode(ISD::BITCAST, dl, MVT::v4i32, X);
Ahmed Bougacha93cff7f2016-02-15 18:07:29 +00007028 Y = DAG.getConstant(0xb000, dl, MVT::v4i32);
Nate Begemanfa62d502011-02-11 20:53:29 +00007029 X = DAG.getNode(ISD::ADD, dl, MVT::v4i32, X, Y);
7030 X = DAG.getNode(ISD::BITCAST, dl, MVT::v4f32, X);
7031 // Convert back to short.
7032 X = DAG.getNode(ISD::FP_TO_SINT, dl, MVT::v4i32, X);
7033 X = DAG.getNode(ISD::TRUNCATE, dl, MVT::v4i16, X);
7034 return X;
7035}
7036
Benjamin Kramerbdc49562016-06-12 15:39:02 +00007037static SDValue LowerSDIV_v4i16(SDValue N0, SDValue N1, const SDLoc &dl,
7038 SelectionDAG &DAG) {
Sanjay Patela2607012015-09-16 16:31:21 +00007039 // TODO: Should this propagate fast-math-flags?
7040
Nate Begemanfa62d502011-02-11 20:53:29 +00007041 SDValue N2;
7042 // Convert to float.
7043 // float4 yf = vcvt_f32_s32(vmovl_s16(y));
7044 // float4 xf = vcvt_f32_s32(vmovl_s16(x));
7045 N0 = DAG.getNode(ISD::SIGN_EXTEND, dl, MVT::v4i32, N0);
7046 N1 = DAG.getNode(ISD::SIGN_EXTEND, dl, MVT::v4i32, N1);
7047 N0 = DAG.getNode(ISD::SINT_TO_FP, dl, MVT::v4f32, N0);
7048 N1 = DAG.getNode(ISD::SINT_TO_FP, dl, MVT::v4f32, N1);
Owen Anderson77aa2662011-04-05 21:48:57 +00007049
Nate Begemanfa62d502011-02-11 20:53:29 +00007050 // Use reciprocal estimate and one refinement step.
7051 // float4 recip = vrecpeq_f32(yf);
7052 // recip *= vrecpsq_f32(yf, recip);
Owen Anderson77aa2662011-04-05 21:48:57 +00007053 N2 = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, MVT::v4f32,
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +00007054 DAG.getConstant(Intrinsic::arm_neon_vrecpe, dl, MVT::i32),
7055 N1);
Owen Anderson77aa2662011-04-05 21:48:57 +00007056 N1 = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, MVT::v4f32,
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +00007057 DAG.getConstant(Intrinsic::arm_neon_vrecps, dl, MVT::i32),
Nate Begemanfa62d502011-02-11 20:53:29 +00007058 N1, N2);
7059 N2 = DAG.getNode(ISD::FMUL, dl, MVT::v4f32, N1, N2);
7060 // Because short has a smaller range than ushort, we can actually get away
7061 // with only a single newton step. This requires that we use a weird bias
7062 // of 89, however (again, this has been exhaustively tested).
Mon P Wang6d9e1c72011-05-19 04:15:07 +00007063 // float4 result = as_float4(as_int4(xf*recip) + 0x89);
Nate Begemanfa62d502011-02-11 20:53:29 +00007064 N0 = DAG.getNode(ISD::FMUL, dl, MVT::v4f32, N0, N2);
7065 N0 = DAG.getNode(ISD::BITCAST, dl, MVT::v4i32, N0);
Ahmed Bougacha93cff7f2016-02-15 18:07:29 +00007066 N1 = DAG.getConstant(0x89, dl, MVT::v4i32);
Nate Begemanfa62d502011-02-11 20:53:29 +00007067 N0 = DAG.getNode(ISD::ADD, dl, MVT::v4i32, N0, N1);
7068 N0 = DAG.getNode(ISD::BITCAST, dl, MVT::v4f32, N0);
7069 // Convert back to integer and return.
7070 // return vmovn_s32(vcvt_s32_f32(result));
7071 N0 = DAG.getNode(ISD::FP_TO_SINT, dl, MVT::v4i32, N0);
7072 N0 = DAG.getNode(ISD::TRUNCATE, dl, MVT::v4i16, N0);
7073 return N0;
7074}
7075
7076static SDValue LowerSDIV(SDValue Op, SelectionDAG &DAG) {
7077 EVT VT = Op.getValueType();
7078 assert((VT == MVT::v4i16 || VT == MVT::v8i8) &&
7079 "unexpected type for custom-lowering ISD::SDIV");
7080
Andrew Trickef9de2a2013-05-25 02:42:55 +00007081 SDLoc dl(Op);
Nate Begemanfa62d502011-02-11 20:53:29 +00007082 SDValue N0 = Op.getOperand(0);
7083 SDValue N1 = Op.getOperand(1);
7084 SDValue N2, N3;
Owen Anderson77aa2662011-04-05 21:48:57 +00007085
Nate Begemanfa62d502011-02-11 20:53:29 +00007086 if (VT == MVT::v8i8) {
7087 N0 = DAG.getNode(ISD::SIGN_EXTEND, dl, MVT::v8i16, N0);
7088 N1 = DAG.getNode(ISD::SIGN_EXTEND, dl, MVT::v8i16, N1);
Owen Anderson77aa2662011-04-05 21:48:57 +00007089
Nate Begemanfa62d502011-02-11 20:53:29 +00007090 N2 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, MVT::v4i16, N0,
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +00007091 DAG.getIntPtrConstant(4, dl));
Nate Begemanfa62d502011-02-11 20:53:29 +00007092 N3 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, MVT::v4i16, N1,
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +00007093 DAG.getIntPtrConstant(4, dl));
Nate Begemanfa62d502011-02-11 20:53:29 +00007094 N0 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, MVT::v4i16, N0,
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +00007095 DAG.getIntPtrConstant(0, dl));
Nate Begemanfa62d502011-02-11 20:53:29 +00007096 N1 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, MVT::v4i16, N1,
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +00007097 DAG.getIntPtrConstant(0, dl));
Nate Begemanfa62d502011-02-11 20:53:29 +00007098
7099 N0 = LowerSDIV_v4i8(N0, N1, dl, DAG); // v4i16
7100 N2 = LowerSDIV_v4i8(N2, N3, dl, DAG); // v4i16
7101
7102 N0 = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v8i16, N0, N2);
7103 N0 = LowerCONCAT_VECTORS(N0, DAG);
Owen Anderson77aa2662011-04-05 21:48:57 +00007104
Nate Begemanfa62d502011-02-11 20:53:29 +00007105 N0 = DAG.getNode(ISD::TRUNCATE, dl, MVT::v8i8, N0);
7106 return N0;
7107 }
7108 return LowerSDIV_v4i16(N0, N1, dl, DAG);
7109}
7110
7111static SDValue LowerUDIV(SDValue Op, SelectionDAG &DAG) {
Sanjay Patela2607012015-09-16 16:31:21 +00007112 // TODO: Should this propagate fast-math-flags?
Nate Begemanfa62d502011-02-11 20:53:29 +00007113 EVT VT = Op.getValueType();
7114 assert((VT == MVT::v4i16 || VT == MVT::v8i8) &&
7115 "unexpected type for custom-lowering ISD::UDIV");
7116
Andrew Trickef9de2a2013-05-25 02:42:55 +00007117 SDLoc dl(Op);
Nate Begemanfa62d502011-02-11 20:53:29 +00007118 SDValue N0 = Op.getOperand(0);
7119 SDValue N1 = Op.getOperand(1);
7120 SDValue N2, N3;
Owen Anderson77aa2662011-04-05 21:48:57 +00007121
Nate Begemanfa62d502011-02-11 20:53:29 +00007122 if (VT == MVT::v8i8) {
7123 N0 = DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::v8i16, N0);
7124 N1 = DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::v8i16, N1);
Owen Anderson77aa2662011-04-05 21:48:57 +00007125
Nate Begemanfa62d502011-02-11 20:53:29 +00007126 N2 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, MVT::v4i16, N0,
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +00007127 DAG.getIntPtrConstant(4, dl));
Nate Begemanfa62d502011-02-11 20:53:29 +00007128 N3 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, MVT::v4i16, N1,
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +00007129 DAG.getIntPtrConstant(4, dl));
Nate Begemanfa62d502011-02-11 20:53:29 +00007130 N0 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, MVT::v4i16, N0,
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +00007131 DAG.getIntPtrConstant(0, dl));
Nate Begemanfa62d502011-02-11 20:53:29 +00007132 N1 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, MVT::v4i16, N1,
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +00007133 DAG.getIntPtrConstant(0, dl));
Owen Anderson77aa2662011-04-05 21:48:57 +00007134
Nate Begemanfa62d502011-02-11 20:53:29 +00007135 N0 = LowerSDIV_v4i16(N0, N1, dl, DAG); // v4i16
7136 N2 = LowerSDIV_v4i16(N2, N3, dl, DAG); // v4i16
Owen Anderson77aa2662011-04-05 21:48:57 +00007137
Nate Begemanfa62d502011-02-11 20:53:29 +00007138 N0 = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v8i16, N0, N2);
7139 N0 = LowerCONCAT_VECTORS(N0, DAG);
Owen Anderson77aa2662011-04-05 21:48:57 +00007140
7141 N0 = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, MVT::v8i8,
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +00007142 DAG.getConstant(Intrinsic::arm_neon_vqmovnsu, dl,
7143 MVT::i32),
Nate Begemanfa62d502011-02-11 20:53:29 +00007144 N0);
7145 return N0;
7146 }
Owen Anderson77aa2662011-04-05 21:48:57 +00007147
Nate Begemanfa62d502011-02-11 20:53:29 +00007148 // v4i16 sdiv ... Convert to float.
7149 // float4 yf = vcvt_f32_s32(vmovl_u16(y));
7150 // float4 xf = vcvt_f32_s32(vmovl_u16(x));
7151 N0 = DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::v4i32, N0);
7152 N1 = DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::v4i32, N1);
7153 N0 = DAG.getNode(ISD::SINT_TO_FP, dl, MVT::v4f32, N0);
Mon P Wang6d9e1c72011-05-19 04:15:07 +00007154 SDValue BN1 = DAG.getNode(ISD::SINT_TO_FP, dl, MVT::v4f32, N1);
Nate Begemanfa62d502011-02-11 20:53:29 +00007155
7156 // Use reciprocal estimate and two refinement steps.
7157 // float4 recip = vrecpeq_f32(yf);
7158 // recip *= vrecpsq_f32(yf, recip);
7159 // recip *= vrecpsq_f32(yf, recip);
Owen Anderson77aa2662011-04-05 21:48:57 +00007160 N2 = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, MVT::v4f32,
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +00007161 DAG.getConstant(Intrinsic::arm_neon_vrecpe, dl, MVT::i32),
7162 BN1);
Owen Anderson77aa2662011-04-05 21:48:57 +00007163 N1 = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, MVT::v4f32,
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +00007164 DAG.getConstant(Intrinsic::arm_neon_vrecps, dl, MVT::i32),
Mon P Wang6d9e1c72011-05-19 04:15:07 +00007165 BN1, N2);
Nate Begemanfa62d502011-02-11 20:53:29 +00007166 N2 = DAG.getNode(ISD::FMUL, dl, MVT::v4f32, N1, N2);
Owen Anderson77aa2662011-04-05 21:48:57 +00007167 N1 = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, MVT::v4f32,
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +00007168 DAG.getConstant(Intrinsic::arm_neon_vrecps, dl, MVT::i32),
Mon P Wang6d9e1c72011-05-19 04:15:07 +00007169 BN1, N2);
Nate Begemanfa62d502011-02-11 20:53:29 +00007170 N2 = DAG.getNode(ISD::FMUL, dl, MVT::v4f32, N1, N2);
7171 // Simply multiplying by the reciprocal estimate can leave us a few ulps
7172 // too low, so we add 2 ulps (exhaustive testing shows that this is enough,
7173 // and that it will never cause us to return an answer too large).
Mon P Wang6d9e1c72011-05-19 04:15:07 +00007174 // float4 result = as_float4(as_int4(xf*recip) + 2);
Nate Begemanfa62d502011-02-11 20:53:29 +00007175 N0 = DAG.getNode(ISD::FMUL, dl, MVT::v4f32, N0, N2);
7176 N0 = DAG.getNode(ISD::BITCAST, dl, MVT::v4i32, N0);
Ahmed Bougacha93cff7f2016-02-15 18:07:29 +00007177 N1 = DAG.getConstant(2, dl, MVT::v4i32);
Nate Begemanfa62d502011-02-11 20:53:29 +00007178 N0 = DAG.getNode(ISD::ADD, dl, MVT::v4i32, N0, N1);
7179 N0 = DAG.getNode(ISD::BITCAST, dl, MVT::v4f32, N0);
7180 // Convert back to integer and return.
7181 // return vmovn_u32(vcvt_s32_f32(result));
7182 N0 = DAG.getNode(ISD::FP_TO_SINT, dl, MVT::v4i32, N0);
7183 N0 = DAG.getNode(ISD::TRUNCATE, dl, MVT::v4i16, N0);
7184 return N0;
7185}
7186
Evan Chenge8916542011-08-30 01:34:54 +00007187static SDValue LowerADDC_ADDE_SUBC_SUBE(SDValue Op, SelectionDAG &DAG) {
7188 EVT VT = Op.getNode()->getValueType(0);
7189 SDVTList VTs = DAG.getVTList(VT, MVT::i32);
7190
7191 unsigned Opc;
7192 bool ExtraOp = false;
7193 switch (Op.getOpcode()) {
Craig Toppere55c5562012-02-07 02:50:20 +00007194 default: llvm_unreachable("Invalid code");
Evan Chenge8916542011-08-30 01:34:54 +00007195 case ISD::ADDC: Opc = ARMISD::ADDC; break;
7196 case ISD::ADDE: Opc = ARMISD::ADDE; ExtraOp = true; break;
7197 case ISD::SUBC: Opc = ARMISD::SUBC; break;
7198 case ISD::SUBE: Opc = ARMISD::SUBE; ExtraOp = true; break;
7199 }
7200
7201 if (!ExtraOp)
Andrew Trickef9de2a2013-05-25 02:42:55 +00007202 return DAG.getNode(Opc, SDLoc(Op), VTs, Op.getOperand(0),
Evan Chenge8916542011-08-30 01:34:54 +00007203 Op.getOperand(1));
Andrew Trickef9de2a2013-05-25 02:42:55 +00007204 return DAG.getNode(Opc, SDLoc(Op), VTs, Op.getOperand(0),
Evan Chenge8916542011-08-30 01:34:54 +00007205 Op.getOperand(1), Op.getOperand(2));
7206}
7207
Bob Wilsone7dde0c2013-11-03 06:14:38 +00007208SDValue ARMTargetLowering::LowerFSINCOS(SDValue Op, SelectionDAG &DAG) const {
7209 assert(Subtarget->isTargetDarwin());
7210
7211 // For iOS, we want to call an alternative entry point: __sincos_stret,
7212 // return values are passed via sret.
7213 SDLoc dl(Op);
7214 SDValue Arg = Op.getOperand(0);
7215 EVT ArgVT = Arg.getValueType();
7216 Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext());
Mehdi Amini44ede332015-07-09 02:09:04 +00007217 auto PtrVT = getPointerTy(DAG.getDataLayout());
Bob Wilsone7dde0c2013-11-03 06:14:38 +00007218
Matthias Braun941a7052016-07-28 18:40:00 +00007219 MachineFrameInfo &MFI = DAG.getMachineFunction().getFrameInfo();
Tim Northover8b403662015-10-28 22:51:16 +00007220 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
Bob Wilsone7dde0c2013-11-03 06:14:38 +00007221
7222 // Pair of floats / doubles used to pass the result.
Tim Northover8b403662015-10-28 22:51:16 +00007223 Type *RetTy = StructType::get(ArgTy, ArgTy, nullptr);
Mehdi Amini44ede332015-07-09 02:09:04 +00007224 auto &DL = DAG.getDataLayout();
Bob Wilsone7dde0c2013-11-03 06:14:38 +00007225
7226 ArgListTy Args;
Tim Northover8b403662015-10-28 22:51:16 +00007227 bool ShouldUseSRet = Subtarget->isAPCS_ABI();
7228 SDValue SRet;
7229 if (ShouldUseSRet) {
7230 // Create stack object for sret.
7231 const uint64_t ByteSize = DL.getTypeAllocSize(RetTy);
7232 const unsigned StackAlign = DL.getPrefTypeAlignment(RetTy);
Matthias Braun941a7052016-07-28 18:40:00 +00007233 int FrameIdx = MFI.CreateStackObject(ByteSize, StackAlign, false);
Tim Northover8b403662015-10-28 22:51:16 +00007234 SRet = DAG.getFrameIndex(FrameIdx, TLI.getPointerTy(DL));
7235
7236 ArgListEntry Entry;
7237 Entry.Node = SRet;
7238 Entry.Ty = RetTy->getPointerTo();
7239 Entry.isSExt = false;
7240 Entry.isZExt = false;
7241 Entry.isSRet = true;
7242 Args.push_back(Entry);
7243 RetTy = Type::getVoidTy(*DAG.getContext());
7244 }
7245
Bob Wilsone7dde0c2013-11-03 06:14:38 +00007246 ArgListEntry Entry;
Bob Wilsone7dde0c2013-11-03 06:14:38 +00007247 Entry.Node = Arg;
7248 Entry.Ty = ArgTy;
7249 Entry.isSExt = false;
7250 Entry.isZExt = false;
7251 Args.push_back(Entry);
7252
Saleem Abdulrasool4966f582015-09-20 03:19:09 +00007253 const char *LibcallName =
7254 (ArgVT == MVT::f64) ? "__sincos_stret" : "__sincosf_stret";
Tim Northover8b403662015-10-28 22:51:16 +00007255 RTLIB::Libcall LC =
7256 (ArgVT == MVT::f64) ? RTLIB::SINCOS_F64 : RTLIB::SINCOS_F32;
7257 CallingConv::ID CC = getLibcallCallingConv(LC);
Mehdi Amini44ede332015-07-09 02:09:04 +00007258 SDValue Callee = DAG.getExternalSymbol(LibcallName, getPointerTy(DL));
Bob Wilsone7dde0c2013-11-03 06:14:38 +00007259
Saleem Abdulrasoolf3a5a5c2014-05-17 21:50:17 +00007260 TargetLowering::CallLoweringInfo CLI(DAG);
Tim Northover8b403662015-10-28 22:51:16 +00007261 CLI.setDebugLoc(dl)
7262 .setChain(DAG.getEntryNode())
Krzysztof Parzyszeke116d5002016-06-22 12:54:25 +00007263 .setCallee(CC, RetTy, Callee, std::move(Args))
Tim Northover8b403662015-10-28 22:51:16 +00007264 .setDiscardResult(ShouldUseSRet);
Bob Wilsone7dde0c2013-11-03 06:14:38 +00007265 std::pair<SDValue, SDValue> CallResult = LowerCallTo(CLI);
7266
Tim Northover8b403662015-10-28 22:51:16 +00007267 if (!ShouldUseSRet)
7268 return CallResult.first;
7269
Justin Lebar9c375812016-07-15 18:27:10 +00007270 SDValue LoadSin =
7271 DAG.getLoad(ArgVT, dl, CallResult.second, SRet, MachinePointerInfo());
Bob Wilsone7dde0c2013-11-03 06:14:38 +00007272
7273 // Address of cos field.
Mehdi Amini44ede332015-07-09 02:09:04 +00007274 SDValue Add = DAG.getNode(ISD::ADD, dl, PtrVT, SRet,
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +00007275 DAG.getIntPtrConstant(ArgVT.getStoreSize(), dl));
Justin Lebar9c375812016-07-15 18:27:10 +00007276 SDValue LoadCos =
7277 DAG.getLoad(ArgVT, dl, LoadSin.getValue(1), Add, MachinePointerInfo());
Bob Wilsone7dde0c2013-11-03 06:14:38 +00007278
7279 SDVTList Tys = DAG.getVTList(ArgVT, ArgVT);
7280 return DAG.getNode(ISD::MERGE_VALUES, dl, Tys,
7281 LoadSin.getValue(0), LoadCos.getValue(0));
7282}
7283
Saleem Abdulrasoolfe83b502015-09-25 05:15:46 +00007284SDValue ARMTargetLowering::LowerWindowsDIVLibCall(SDValue Op, SelectionDAG &DAG,
Martell Maloned1229242015-11-26 15:34:03 +00007285 bool Signed,
Saleem Abdulrasoolfe83b502015-09-25 05:15:46 +00007286 SDValue &Chain) const {
7287 EVT VT = Op.getValueType();
7288 assert((VT == MVT::i32 || VT == MVT::i64) &&
7289 "unexpected type for custom lowering DIV");
7290 SDLoc dl(Op);
7291
7292 const auto &DL = DAG.getDataLayout();
7293 const auto &TLI = DAG.getTargetLoweringInfo();
7294
7295 const char *Name = nullptr;
Martell Maloned1229242015-11-26 15:34:03 +00007296 if (Signed)
7297 Name = (VT == MVT::i32) ? "__rt_sdiv" : "__rt_sdiv64";
7298 else
7299 Name = (VT == MVT::i32) ? "__rt_udiv" : "__rt_udiv64";
Saleem Abdulrasoolfe83b502015-09-25 05:15:46 +00007300
7301 SDValue ES = DAG.getExternalSymbol(Name, TLI.getPointerTy(DL));
7302
7303 ARMTargetLowering::ArgListTy Args;
7304
7305 for (auto AI : {1, 0}) {
7306 ArgListEntry Arg;
7307 Arg.Node = Op.getOperand(AI);
7308 Arg.Ty = Arg.Node.getValueType().getTypeForEVT(*DAG.getContext());
7309 Args.push_back(Arg);
7310 }
7311
7312 CallLoweringInfo CLI(DAG);
7313 CLI.setDebugLoc(dl)
7314 .setChain(Chain)
7315 .setCallee(CallingConv::ARM_AAPCS_VFP, VT.getTypeForEVT(*DAG.getContext()),
Krzysztof Parzyszeke116d5002016-06-22 12:54:25 +00007316 ES, std::move(Args));
Saleem Abdulrasoolfe83b502015-09-25 05:15:46 +00007317
7318 return LowerCallTo(CLI).first;
7319}
7320
Martell Maloned1229242015-11-26 15:34:03 +00007321SDValue ARMTargetLowering::LowerDIV_Windows(SDValue Op, SelectionDAG &DAG,
7322 bool Signed) const {
Saleem Abdulrasool8e99f502015-09-25 05:41:02 +00007323 assert(Op.getValueType() == MVT::i32 &&
7324 "unexpected type for custom lowering DIV");
Saleem Abdulrasoolfe83b502015-09-25 05:15:46 +00007325 SDLoc dl(Op);
7326
7327 SDValue DBZCHK = DAG.getNode(ARMISD::WIN__DBZCHK, dl, MVT::Other,
7328 DAG.getEntryNode(), Op.getOperand(1));
7329
Martell Maloned1229242015-11-26 15:34:03 +00007330 return LowerWindowsDIVLibCall(Op, DAG, Signed, DBZCHK);
Saleem Abdulrasoolfe83b502015-09-25 05:15:46 +00007331}
7332
7333void ARMTargetLowering::ExpandDIV_Windows(
Martell Maloned1229242015-11-26 15:34:03 +00007334 SDValue Op, SelectionDAG &DAG, bool Signed,
Saleem Abdulrasoolfe83b502015-09-25 05:15:46 +00007335 SmallVectorImpl<SDValue> &Results) const {
7336 const auto &DL = DAG.getDataLayout();
7337 const auto &TLI = DAG.getTargetLoweringInfo();
7338
Saleem Abdulrasool8e99f502015-09-25 05:41:02 +00007339 assert(Op.getValueType() == MVT::i64 &&
7340 "unexpected type for custom lowering DIV");
Saleem Abdulrasoolfe83b502015-09-25 05:15:46 +00007341 SDLoc dl(Op);
7342
7343 SDValue Lo = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, MVT::i32, Op.getOperand(1),
7344 DAG.getConstant(0, dl, MVT::i32));
7345 SDValue Hi = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, MVT::i32, Op.getOperand(1),
7346 DAG.getConstant(1, dl, MVT::i32));
7347 SDValue Or = DAG.getNode(ISD::OR, dl, MVT::i32, Lo, Hi);
7348
7349 SDValue DBZCHK =
7350 DAG.getNode(ARMISD::WIN__DBZCHK, dl, MVT::Other, DAG.getEntryNode(), Or);
7351
Martell Maloned1229242015-11-26 15:34:03 +00007352 SDValue Result = LowerWindowsDIVLibCall(Op, DAG, Signed, DBZCHK);
Saleem Abdulrasoolfe83b502015-09-25 05:15:46 +00007353
7354 SDValue Lower = DAG.getNode(ISD::TRUNCATE, dl, MVT::i32, Result);
7355 SDValue Upper = DAG.getNode(ISD::SRL, dl, MVT::i64, Result,
7356 DAG.getConstant(32, dl, TLI.getPointerTy(DL)));
7357 Upper = DAG.getNode(ISD::TRUNCATE, dl, MVT::i32, Upper);
7358
7359 Results.push_back(Lower);
7360 Results.push_back(Upper);
7361}
7362
Eli Friedman10f9ce22011-09-15 22:26:18 +00007363static SDValue LowerAtomicLoadStore(SDValue Op, SelectionDAG &DAG) {
JF Bastien800f87a2016-04-06 21:19:33 +00007364 if (isStrongerThanMonotonic(cast<AtomicSDNode>(Op)->getOrdering()))
7365 // Acquire/Release load/store is not legal for targets without a dmb or
7366 // equivalent available.
7367 return SDValue();
Eli Friedmanba912e02011-09-15 22:18:49 +00007368
JF Bastien800f87a2016-04-06 21:19:33 +00007369 // Monotonic load/store is legal for all targets.
7370 return Op;
Eli Friedmanba912e02011-09-15 22:18:49 +00007371}
7372
Tim Northoverbc933082013-05-23 19:11:20 +00007373static void ReplaceREADCYCLECOUNTER(SDNode *N,
7374 SmallVectorImpl<SDValue> &Results,
7375 SelectionDAG &DAG,
7376 const ARMSubtarget *Subtarget) {
Andrew Trickef9de2a2013-05-25 02:42:55 +00007377 SDLoc DL(N);
Ahmed Bougachaf9c19da2015-08-28 01:49:59 +00007378 // Under Power Management extensions, the cycle-count is:
7379 // mrc p15, #0, <Rt>, c9, c13, #0
7380 SDValue Ops[] = { N->getOperand(0), // Chain
7381 DAG.getConstant(Intrinsic::arm_mrc, DL, MVT::i32),
7382 DAG.getConstant(15, DL, MVT::i32),
7383 DAG.getConstant(0, DL, MVT::i32),
7384 DAG.getConstant(9, DL, MVT::i32),
7385 DAG.getConstant(13, DL, MVT::i32),
7386 DAG.getConstant(0, DL, MVT::i32)
7387 };
Tim Northoverbc933082013-05-23 19:11:20 +00007388
Ahmed Bougachaf9c19da2015-08-28 01:49:59 +00007389 SDValue Cycles32 = DAG.getNode(ISD::INTRINSIC_W_CHAIN, DL,
7390 DAG.getVTList(MVT::i32, MVT::Other), Ops);
7391 Results.push_back(DAG.getNode(ISD::BUILD_PAIR, DL, MVT::i64, Cycles32,
7392 DAG.getConstant(0, DL, MVT::i32)));
7393 Results.push_back(Cycles32.getValue(1));
Tim Northoverbc933082013-05-23 19:11:20 +00007394}
7395
Tim Northover1ee27c72016-04-19 22:25:02 +00007396static SDValue createGPRPairNode(SelectionDAG &DAG, SDValue V) {
7397 SDLoc dl(V.getNode());
7398 SDValue VLo = DAG.getAnyExtOrTrunc(V, dl, MVT::i32);
7399 SDValue VHi = DAG.getAnyExtOrTrunc(
7400 DAG.getNode(ISD::SRL, dl, MVT::i64, V, DAG.getConstant(32, dl, MVT::i32)),
7401 dl, MVT::i32);
Tim Northoverb629c772016-04-18 21:48:55 +00007402 SDValue RegClass =
7403 DAG.getTargetConstant(ARM::GPRPairRegClassID, dl, MVT::i32);
7404 SDValue SubReg0 = DAG.getTargetConstant(ARM::gsub_0, dl, MVT::i32);
7405 SDValue SubReg1 = DAG.getTargetConstant(ARM::gsub_1, dl, MVT::i32);
Tim Northover1ee27c72016-04-19 22:25:02 +00007406 const SDValue Ops[] = { RegClass, VLo, SubReg0, VHi, SubReg1 };
Tim Northoverb629c772016-04-18 21:48:55 +00007407 return SDValue(
7408 DAG.getMachineNode(TargetOpcode::REG_SEQUENCE, dl, MVT::Untyped, Ops), 0);
7409}
7410
7411static void ReplaceCMP_SWAP_64Results(SDNode *N,
7412 SmallVectorImpl<SDValue> & Results,
7413 SelectionDAG &DAG) {
7414 assert(N->getValueType(0) == MVT::i64 &&
7415 "AtomicCmpSwap on types less than 64 should be legal");
7416 SDValue Ops[] = {N->getOperand(1),
Tim Northover1ee27c72016-04-19 22:25:02 +00007417 createGPRPairNode(DAG, N->getOperand(2)),
7418 createGPRPairNode(DAG, N->getOperand(3)),
Tim Northoverb629c772016-04-18 21:48:55 +00007419 N->getOperand(0)};
7420 SDNode *CmpSwap = DAG.getMachineNode(
7421 ARM::CMP_SWAP_64, SDLoc(N),
7422 DAG.getVTList(MVT::Untyped, MVT::i32, MVT::Other), Ops);
7423
7424 MachineFunction &MF = DAG.getMachineFunction();
7425 MachineSDNode::mmo_iterator MemOp = MF.allocateMemRefsArray(1);
7426 MemOp[0] = cast<MemSDNode>(N)->getMemOperand();
7427 cast<MachineSDNode>(CmpSwap)->setMemRefs(MemOp, MemOp + 1);
7428
7429 Results.push_back(DAG.getTargetExtractSubreg(ARM::gsub_0, SDLoc(N), MVT::i32,
7430 SDValue(CmpSwap, 0)));
7431 Results.push_back(DAG.getTargetExtractSubreg(ARM::gsub_1, SDLoc(N), MVT::i32,
7432 SDValue(CmpSwap, 0)));
7433 Results.push_back(SDValue(CmpSwap, 2));
7434}
7435
Dan Gohman21cea8a2010-04-17 15:26:15 +00007436SDValue ARMTargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const {
Evan Cheng10043e22007-01-19 07:51:42 +00007437 switch (Op.getOpcode()) {
Torok Edwinfbcc6632009-07-14 16:55:14 +00007438 default: llvm_unreachable("Don't know how to custom lower this!");
Luke Cheeseman85fd06d2015-06-01 12:02:47 +00007439 case ISD::WRITE_REGISTER: return LowerWRITE_REGISTER(Op, DAG);
Evan Cheng10043e22007-01-19 07:51:42 +00007440 case ISD::ConstantPool: return LowerConstantPool(Op, DAG);
Bob Wilson1cf0b032009-10-30 05:45:42 +00007441 case ISD::BlockAddress: return LowerBlockAddress(Op, DAG);
Lauro Ramos Venancioee2d1642007-04-22 00:04:12 +00007442 case ISD::GlobalAddress:
Saleem Abdulrasool40bca0a2014-05-09 00:58:32 +00007443 switch (Subtarget->getTargetTriple().getObjectFormat()) {
7444 default: llvm_unreachable("unknown object format");
7445 case Triple::COFF:
7446 return LowerGlobalAddressWindows(Op, DAG);
7447 case Triple::ELF:
7448 return LowerGlobalAddressELF(Op, DAG);
7449 case Triple::MachO:
7450 return LowerGlobalAddressDarwin(Op, DAG);
7451 }
Bill Wendlinge1fd78f2011-03-14 23:02:38 +00007452 case ISD::GlobalTLSAddress: return LowerGlobalTLSAddress(Op, DAG);
Bill Wendling6a981312010-08-11 08:43:16 +00007453 case ISD::SELECT: return LowerSELECT(Op, DAG);
Evan Cheng15b80e42009-11-12 07:13:11 +00007454 case ISD::SELECT_CC: return LowerSELECT_CC(Op, DAG);
7455 case ISD::BR_CC: return LowerBR_CC(Op, DAG);
Evan Cheng10043e22007-01-19 07:51:42 +00007456 case ISD::BR_JT: return LowerBR_JT(Op, DAG);
Dan Gohman31ae5862010-04-17 14:41:14 +00007457 case ISD::VASTART: return LowerVASTART(Op, DAG);
Eli Friedman26a48482011-07-27 22:21:52 +00007458 case ISD::ATOMIC_FENCE: return LowerATOMIC_FENCE(Op, DAG, Subtarget);
Evan Cheng8740ee32010-11-03 06:34:55 +00007459 case ISD::PREFETCH: return LowerPREFETCH(Op, DAG, Subtarget);
Bob Wilsone4191e72010-03-19 22:51:32 +00007460 case ISD::SINT_TO_FP:
7461 case ISD::UINT_TO_FP: return LowerINT_TO_FP(Op, DAG);
7462 case ISD::FP_TO_SINT:
7463 case ISD::FP_TO_UINT: return LowerFP_TO_INT(Op, DAG);
Evan Cheng10043e22007-01-19 07:51:42 +00007464 case ISD::FCOPYSIGN: return LowerFCOPYSIGN(Op, DAG);
Evan Cheng168ced92010-05-22 01:47:14 +00007465 case ISD::RETURNADDR: return LowerRETURNADDR(Op, DAG);
Jim Grosbachaeca45d2009-05-12 23:59:14 +00007466 case ISD::FRAMEADDR: return LowerFRAMEADDR(Op, DAG);
Jim Grosbachc98892f2010-05-26 20:22:18 +00007467 case ISD::EH_SJLJ_SETJMP: return LowerEH_SJLJ_SETJMP(Op, DAG);
Jim Grosbachbd9485d2010-05-22 01:06:18 +00007468 case ISD::EH_SJLJ_LONGJMP: return LowerEH_SJLJ_LONGJMP(Op, DAG);
Matthias Braun3cd00c12015-07-16 22:34:16 +00007469 case ISD::EH_SJLJ_SETUP_DISPATCH: return LowerEH_SJLJ_SETUP_DISPATCH(Op, DAG);
Jim Grosbacha570d052010-02-08 23:22:00 +00007470 case ISD::INTRINSIC_WO_CHAIN: return LowerINTRINSIC_WO_CHAIN(Op, DAG,
7471 Subtarget);
Evan Cheng383ecd82011-03-14 18:02:30 +00007472 case ISD::BITCAST: return ExpandBITCAST(Op.getNode(), DAG);
Bob Wilson2e076c42009-06-22 23:27:02 +00007473 case ISD::SHL:
Chris Lattnerf81d5882007-11-24 07:07:01 +00007474 case ISD::SRL:
Bob Wilson2e076c42009-06-22 23:27:02 +00007475 case ISD::SRA: return LowerShift(Op.getNode(), DAG, Subtarget);
Scott Douglassbdef6042015-08-24 09:17:18 +00007476 case ISD::SREM: return LowerREM(Op.getNode(), DAG);
7477 case ISD::UREM: return LowerREM(Op.getNode(), DAG);
Evan Cheng15b80e42009-11-12 07:13:11 +00007478 case ISD::SHL_PARTS: return LowerShiftLeftParts(Op, DAG);
Jim Grosbach8fe6fd72009-10-31 21:42:19 +00007479 case ISD::SRL_PARTS:
Evan Cheng15b80e42009-11-12 07:13:11 +00007480 case ISD::SRA_PARTS: return LowerShiftRightParts(Op, DAG);
Logan Chien0a43abc2015-07-13 15:37:30 +00007481 case ISD::CTTZ:
7482 case ISD::CTTZ_ZERO_UNDEF: return LowerCTTZ(Op.getNode(), DAG, Subtarget);
Evan Chengb4eae132012-12-04 22:41:50 +00007483 case ISD::CTPOP: return LowerCTPOP(Op.getNode(), DAG, Subtarget);
Duncan Sandsf2641e12011-09-06 19:07:46 +00007484 case ISD::SETCC: return LowerVSETCC(Op, DAG);
Peter Collingbourne86b9fbe2016-03-21 18:00:02 +00007485 case ISD::SETCCE: return LowerSETCCE(Op, DAG);
Lang Hamesc35ee8b2012-03-15 18:49:02 +00007486 case ISD::ConstantFP: return LowerConstantFP(Op, DAG, Subtarget);
Dale Johannesen2bff5052010-07-29 20:10:08 +00007487 case ISD::BUILD_VECTOR: return LowerBUILD_VECTOR(Op, DAG, Subtarget);
Bob Wilson2e076c42009-06-22 23:27:02 +00007488 case ISD::VECTOR_SHUFFLE: return LowerVECTOR_SHUFFLE(Op, DAG);
Eli Friedmana5e244c2011-10-24 23:08:52 +00007489 case ISD::INSERT_VECTOR_ELT: return LowerINSERT_VECTOR_ELT(Op, DAG);
Bob Wilson2e076c42009-06-22 23:27:02 +00007490 case ISD::EXTRACT_VECTOR_ELT: return LowerEXTRACT_VECTOR_ELT(Op, DAG);
Bob Wilsonf307e0b2009-08-03 20:36:38 +00007491 case ISD::CONCAT_VECTORS: return LowerCONCAT_VECTORS(Op, DAG);
Bob Wilson9a511c02010-08-20 04:54:02 +00007492 case ISD::FLT_ROUNDS_: return LowerFLT_ROUNDS_(Op, DAG);
Bob Wilson38ab35a2010-09-01 23:50:19 +00007493 case ISD::MUL: return LowerMUL(Op, DAG);
Saleem Abdulrasool071a0992016-03-17 14:10:49 +00007494 case ISD::SDIV:
7495 if (Subtarget->isTargetWindows())
7496 return LowerDIV_Windows(Op, DAG, /* Signed */ true);
7497 return LowerSDIV(Op, DAG);
7498 case ISD::UDIV:
7499 if (Subtarget->isTargetWindows())
7500 return LowerDIV_Windows(Op, DAG, /* Signed */ false);
7501 return LowerUDIV(Op, DAG);
Evan Chenge8916542011-08-30 01:34:54 +00007502 case ISD::ADDC:
7503 case ISD::ADDE:
7504 case ISD::SUBC:
7505 case ISD::SUBE: return LowerADDC_ADDE_SUBC_SUBE(Op, DAG);
Louis Gerbarg3342bf12014-05-09 17:02:49 +00007506 case ISD::SADDO:
7507 case ISD::UADDO:
7508 case ISD::SSUBO:
7509 case ISD::USUBO:
7510 return LowerXALUO(Op, DAG);
Eli Friedmanba912e02011-09-15 22:18:49 +00007511 case ISD::ATOMIC_LOAD:
Eli Friedman10f9ce22011-09-15 22:26:18 +00007512 case ISD::ATOMIC_STORE: return LowerAtomicLoadStore(Op, DAG);
Bob Wilsone7dde0c2013-11-03 06:14:38 +00007513 case ISD::FSINCOS: return LowerFSINCOS(Op, DAG);
Renato Golin87610692013-07-16 09:32:17 +00007514 case ISD::SDIVREM:
7515 case ISD::UDIVREM: return LowerDivRem(Op, DAG);
Saleem Abdulrasoolabac6e92014-06-09 20:18:42 +00007516 case ISD::DYNAMIC_STACKALLOC:
7517 if (Subtarget->getTargetTriple().isWindowsItaniumEnvironment())
7518 return LowerDYNAMIC_STACKALLOC(Op, DAG);
7519 llvm_unreachable("Don't know how to custom lower this!");
Oliver Stannard51b1d462014-08-21 12:50:31 +00007520 case ISD::FP_ROUND: return LowerFP_ROUND(Op, DAG);
7521 case ISD::FP_EXTEND: return LowerFP_EXTEND(Op, DAG);
Saleem Abdulrasoolfe83b502015-09-25 05:15:46 +00007522 case ARMISD::WIN__DBZCHK: return SDValue();
Evan Cheng10043e22007-01-19 07:51:42 +00007523 }
Evan Cheng10043e22007-01-19 07:51:42 +00007524}
7525
Duncan Sands6ed40142008-12-01 11:39:25 +00007526/// ReplaceNodeResults - Replace the results of node with an illegal result
7527/// type with new values built out of custom code.
Duncan Sands6ed40142008-12-01 11:39:25 +00007528void ARMTargetLowering::ReplaceNodeResults(SDNode *N,
Saleem Abdulrasoolfe83b502015-09-25 05:15:46 +00007529 SmallVectorImpl<SDValue> &Results,
Dan Gohman21cea8a2010-04-17 15:26:15 +00007530 SelectionDAG &DAG) const {
Bob Wilsonc05b8872010-04-14 20:45:23 +00007531 SDValue Res;
Chris Lattnerf81d5882007-11-24 07:07:01 +00007532 switch (N->getOpcode()) {
Duncan Sands6ed40142008-12-01 11:39:25 +00007533 default:
Torok Edwinfbcc6632009-07-14 16:55:14 +00007534 llvm_unreachable("Don't know how to custom expand this!");
Luke Cheeseman85fd06d2015-06-01 12:02:47 +00007535 case ISD::READ_REGISTER:
7536 ExpandREAD_REGISTER(N, Results, DAG);
7537 break;
Wesley Peck527da1b2010-11-23 03:31:01 +00007538 case ISD::BITCAST:
7539 Res = ExpandBITCAST(N, DAG);
Bob Wilsonc05b8872010-04-14 20:45:23 +00007540 break;
Chris Lattnerf81d5882007-11-24 07:07:01 +00007541 case ISD::SRL:
Bob Wilsonc05b8872010-04-14 20:45:23 +00007542 case ISD::SRA:
Bob Wilson7d471332010-11-18 21:16:28 +00007543 Res = Expand64BitShift(N, DAG, Subtarget);
Bob Wilsonc05b8872010-04-14 20:45:23 +00007544 break;
Scott Douglassbdef6042015-08-24 09:17:18 +00007545 case ISD::SREM:
7546 case ISD::UREM:
7547 Res = LowerREM(N, DAG);
7548 break;
Renato Golin175c6d62016-03-04 19:19:36 +00007549 case ISD::SDIVREM:
7550 case ISD::UDIVREM:
7551 Res = LowerDivRem(SDValue(N, 0), DAG);
7552 assert(Res.getNumOperands() == 2 && "DivRem needs two values");
7553 Results.push_back(Res.getValue(0));
7554 Results.push_back(Res.getValue(1));
7555 return;
Tim Northoverbc933082013-05-23 19:11:20 +00007556 case ISD::READCYCLECOUNTER:
7557 ReplaceREADCYCLECOUNTER(N, Results, DAG, Subtarget);
7558 return;
Saleem Abdulrasoolfe83b502015-09-25 05:15:46 +00007559 case ISD::UDIV:
Martell Maloned1229242015-11-26 15:34:03 +00007560 case ISD::SDIV:
Saleem Abdulrasoolfe83b502015-09-25 05:15:46 +00007561 assert(Subtarget->isTargetWindows() && "can only expand DIV on Windows");
Martell Maloned1229242015-11-26 15:34:03 +00007562 return ExpandDIV_Windows(SDValue(N, 0), DAG, N->getOpcode() == ISD::SDIV,
7563 Results);
Tim Northoverb629c772016-04-18 21:48:55 +00007564 case ISD::ATOMIC_CMP_SWAP:
7565 ReplaceCMP_SWAP_64Results(N, Results, DAG);
7566 return;
Duncan Sands6ed40142008-12-01 11:39:25 +00007567 }
Bob Wilsonc05b8872010-04-14 20:45:23 +00007568 if (Res.getNode())
7569 Results.push_back(Res);
Chris Lattnerf81d5882007-11-24 07:07:01 +00007570}
Chris Lattnerf81d5882007-11-24 07:07:01 +00007571
Evan Cheng10043e22007-01-19 07:51:42 +00007572//===----------------------------------------------------------------------===//
7573// ARM Scheduler Hooks
7574//===----------------------------------------------------------------------===//
7575
Bill Wendling030b58e2011-10-06 22:18:16 +00007576/// SetupEntryBlockForSjLj - Insert code into the entry block that creates and
7577/// registers the function context.
Duncan P. N. Exon Smithe4f5e4f2016-06-30 22:52:52 +00007578void ARMTargetLowering::SetupEntryBlockForSjLj(MachineInstr &MI,
7579 MachineBasicBlock *MBB,
7580 MachineBasicBlock *DispatchBB,
7581 int FI) const {
Oliver Stannard8331aae2016-08-08 15:28:31 +00007582 assert(!Subtarget->isROPI() && !Subtarget->isRWPI() &&
7583 "ROPI/RWPI not currently supported with SjLj");
Eric Christopher1889fdc2015-01-29 00:19:39 +00007584 const TargetInstrInfo *TII = Subtarget->getInstrInfo();
Duncan P. N. Exon Smithe4f5e4f2016-06-30 22:52:52 +00007585 DebugLoc dl = MI.getDebugLoc();
Bill Wendling374ee192011-10-03 21:25:38 +00007586 MachineFunction *MF = MBB->getParent();
7587 MachineRegisterInfo *MRI = &MF->getRegInfo();
7588 MachineConstantPool *MCP = MF->getConstantPool();
7589 ARMFunctionInfo *AFI = MF->getInfo<ARMFunctionInfo>();
7590 const Function *F = MF->getFunction();
Bill Wendling374ee192011-10-03 21:25:38 +00007591
Bill Wendling374ee192011-10-03 21:25:38 +00007592 bool isThumb = Subtarget->isThumb();
Bill Wendling1eab54f2011-10-03 22:44:15 +00007593 bool isThumb2 = Subtarget->isThumb2();
Bill Wendling030b58e2011-10-06 22:18:16 +00007594
Bill Wendling374ee192011-10-03 21:25:38 +00007595 unsigned PCLabelId = AFI->createPICLabelUId();
Bill Wendling1eab54f2011-10-03 22:44:15 +00007596 unsigned PCAdj = (isThumb || isThumb2) ? 4 : 8;
Bill Wendling374ee192011-10-03 21:25:38 +00007597 ARMConstantPoolValue *CPV =
7598 ARMConstantPoolMBB::Create(F->getContext(), DispatchBB, PCLabelId, PCAdj);
7599 unsigned CPI = MCP->getConstantPoolIndex(CPV, 4);
7600
Craig Topper61e88f42014-11-21 05:58:21 +00007601 const TargetRegisterClass *TRC = isThumb ? &ARM::tGPRRegClass
7602 : &ARM::GPRRegClass;
Bill Wendling374ee192011-10-03 21:25:38 +00007603
Bill Wendling030b58e2011-10-06 22:18:16 +00007604 // Grab constant pool and fixed stack memory operands.
7605 MachineMemOperand *CPMMO =
Alex Lorenze40c8a22015-08-11 23:09:45 +00007606 MF->getMachineMemOperand(MachinePointerInfo::getConstantPool(*MF),
7607 MachineMemOperand::MOLoad, 4, 4);
Bill Wendling030b58e2011-10-06 22:18:16 +00007608
7609 MachineMemOperand *FIMMOSt =
Alex Lorenze40c8a22015-08-11 23:09:45 +00007610 MF->getMachineMemOperand(MachinePointerInfo::getFixedStack(*MF, FI),
7611 MachineMemOperand::MOStore, 4, 4);
Bill Wendling030b58e2011-10-06 22:18:16 +00007612
7613 // Load the address of the dispatch MBB into the jump buffer.
7614 if (isThumb2) {
7615 // Incoming value: jbuf
7616 // ldr.n r5, LCPI1_1
7617 // orr r5, r5, #1
7618 // add r5, pc
7619 // str r5, [$jbuf, #+4] ; &jbuf[1]
7620 unsigned NewVReg1 = MRI->createVirtualRegister(TRC);
7621 AddDefaultPred(BuildMI(*MBB, MI, dl, TII->get(ARM::t2LDRpci), NewVReg1)
7622 .addConstantPoolIndex(CPI)
7623 .addMemOperand(CPMMO));
7624 // Set the low bit because of thumb mode.
7625 unsigned NewVReg2 = MRI->createVirtualRegister(TRC);
7626 AddDefaultCC(
7627 AddDefaultPred(BuildMI(*MBB, MI, dl, TII->get(ARM::t2ORRri), NewVReg2)
7628 .addReg(NewVReg1, RegState::Kill)
7629 .addImm(0x01)));
7630 unsigned NewVReg3 = MRI->createVirtualRegister(TRC);
7631 BuildMI(*MBB, MI, dl, TII->get(ARM::tPICADD), NewVReg3)
7632 .addReg(NewVReg2, RegState::Kill)
7633 .addImm(PCLabelId);
7634 AddDefaultPred(BuildMI(*MBB, MI, dl, TII->get(ARM::t2STRi12))
7635 .addReg(NewVReg3, RegState::Kill)
7636 .addFrameIndex(FI)
7637 .addImm(36) // &jbuf[1] :: pc
7638 .addMemOperand(FIMMOSt));
7639 } else if (isThumb) {
7640 // Incoming value: jbuf
7641 // ldr.n r1, LCPI1_4
7642 // add r1, pc
7643 // mov r2, #1
7644 // orrs r1, r2
7645 // add r2, $jbuf, #+4 ; &jbuf[1]
7646 // str r1, [r2]
7647 unsigned NewVReg1 = MRI->createVirtualRegister(TRC);
7648 AddDefaultPred(BuildMI(*MBB, MI, dl, TII->get(ARM::tLDRpci), NewVReg1)
7649 .addConstantPoolIndex(CPI)
7650 .addMemOperand(CPMMO));
7651 unsigned NewVReg2 = MRI->createVirtualRegister(TRC);
7652 BuildMI(*MBB, MI, dl, TII->get(ARM::tPICADD), NewVReg2)
7653 .addReg(NewVReg1, RegState::Kill)
7654 .addImm(PCLabelId);
7655 // Set the low bit because of thumb mode.
7656 unsigned NewVReg3 = MRI->createVirtualRegister(TRC);
7657 AddDefaultPred(BuildMI(*MBB, MI, dl, TII->get(ARM::tMOVi8), NewVReg3)
7658 .addReg(ARM::CPSR, RegState::Define)
7659 .addImm(1));
7660 unsigned NewVReg4 = MRI->createVirtualRegister(TRC);
7661 AddDefaultPred(BuildMI(*MBB, MI, dl, TII->get(ARM::tORR), NewVReg4)
7662 .addReg(ARM::CPSR, RegState::Define)
7663 .addReg(NewVReg2, RegState::Kill)
7664 .addReg(NewVReg3, RegState::Kill));
7665 unsigned NewVReg5 = MRI->createVirtualRegister(TRC);
Tim Northover23075cc2014-10-20 21:28:41 +00007666 BuildMI(*MBB, MI, dl, TII->get(ARM::tADDframe), NewVReg5)
7667 .addFrameIndex(FI)
7668 .addImm(36); // &jbuf[1] :: pc
Bill Wendling030b58e2011-10-06 22:18:16 +00007669 AddDefaultPred(BuildMI(*MBB, MI, dl, TII->get(ARM::tSTRi))
7670 .addReg(NewVReg4, RegState::Kill)
7671 .addReg(NewVReg5, RegState::Kill)
7672 .addImm(0)
7673 .addMemOperand(FIMMOSt));
7674 } else {
7675 // Incoming value: jbuf
7676 // ldr r1, LCPI1_1
7677 // add r1, pc, r1
7678 // str r1, [$jbuf, #+4] ; &jbuf[1]
7679 unsigned NewVReg1 = MRI->createVirtualRegister(TRC);
7680 AddDefaultPred(BuildMI(*MBB, MI, dl, TII->get(ARM::LDRi12), NewVReg1)
7681 .addConstantPoolIndex(CPI)
7682 .addImm(0)
7683 .addMemOperand(CPMMO));
7684 unsigned NewVReg2 = MRI->createVirtualRegister(TRC);
7685 AddDefaultPred(BuildMI(*MBB, MI, dl, TII->get(ARM::PICADD), NewVReg2)
7686 .addReg(NewVReg1, RegState::Kill)
7687 .addImm(PCLabelId));
7688 AddDefaultPred(BuildMI(*MBB, MI, dl, TII->get(ARM::STRi12))
7689 .addReg(NewVReg2, RegState::Kill)
7690 .addFrameIndex(FI)
7691 .addImm(36) // &jbuf[1] :: pc
7692 .addMemOperand(FIMMOSt));
7693 }
7694}
7695
Duncan P. N. Exon Smithe4f5e4f2016-06-30 22:52:52 +00007696void ARMTargetLowering::EmitSjLjDispatchBlock(MachineInstr &MI,
Matthias Brauneec4efc2015-04-28 00:37:05 +00007697 MachineBasicBlock *MBB) const {
Eric Christopher1889fdc2015-01-29 00:19:39 +00007698 const TargetInstrInfo *TII = Subtarget->getInstrInfo();
Duncan P. N. Exon Smithe4f5e4f2016-06-30 22:52:52 +00007699 DebugLoc dl = MI.getDebugLoc();
Bill Wendling030b58e2011-10-06 22:18:16 +00007700 MachineFunction *MF = MBB->getParent();
7701 MachineRegisterInfo *MRI = &MF->getRegInfo();
Matthias Braun941a7052016-07-28 18:40:00 +00007702 MachineFrameInfo &MFI = MF->getFrameInfo();
7703 int FI = MFI.getFunctionContextIndex();
Bill Wendling030b58e2011-10-06 22:18:16 +00007704
Craig Topper61e88f42014-11-21 05:58:21 +00007705 const TargetRegisterClass *TRC = Subtarget->isThumb() ? &ARM::tGPRRegClass
7706 : &ARM::GPRnopcRegClass;
Bill Wendling030b58e2011-10-06 22:18:16 +00007707
Bill Wendling362c1b02011-10-06 21:29:56 +00007708 // Get a mapping of the call site numbers to all of the landing pads they're
7709 // associated with.
Bill Wendling202803e2011-10-05 00:02:33 +00007710 DenseMap<unsigned, SmallVector<MachineBasicBlock*, 2> > CallSiteNumToLPad;
7711 unsigned MaxCSNum = 0;
7712 MachineModuleInfo &MMI = MF->getMMI();
Jim Grosbach0c509fa2012-04-06 23:43:50 +00007713 for (MachineFunction::iterator BB = MF->begin(), E = MF->end(); BB != E;
7714 ++BB) {
Reid Kleckner0e288232015-08-27 23:27:47 +00007715 if (!BB->isEHPad()) continue;
Bill Wendling202803e2011-10-05 00:02:33 +00007716
7717 // FIXME: We should assert that the EH_LABEL is the first MI in the landing
7718 // pad.
7719 for (MachineBasicBlock::iterator
7720 II = BB->begin(), IE = BB->end(); II != IE; ++II) {
7721 if (!II->isEHLabel()) continue;
7722
7723 MCSymbol *Sym = II->getOperand(0).getMCSymbol();
Bill Wendlingf793e7e2011-10-05 23:28:57 +00007724 if (!MMI.hasCallSiteLandingPad(Sym)) continue;
Bill Wendling202803e2011-10-05 00:02:33 +00007725
Bill Wendlingf793e7e2011-10-05 23:28:57 +00007726 SmallVectorImpl<unsigned> &CallSiteIdxs = MMI.getCallSiteLandingPad(Sym);
7727 for (SmallVectorImpl<unsigned>::iterator
7728 CSI = CallSiteIdxs.begin(), CSE = CallSiteIdxs.end();
7729 CSI != CSE; ++CSI) {
Duncan P. N. Exon Smith9f9559e2015-10-19 23:25:57 +00007730 CallSiteNumToLPad[*CSI].push_back(&*BB);
Bill Wendlingf793e7e2011-10-05 23:28:57 +00007731 MaxCSNum = std::max(MaxCSNum, *CSI);
7732 }
Bill Wendling202803e2011-10-05 00:02:33 +00007733 break;
7734 }
7735 }
7736
7737 // Get an ordered list of the machine basic blocks for the jump table.
7738 std::vector<MachineBasicBlock*> LPadList;
Matthias Braunb30f2f512016-01-30 01:24:31 +00007739 SmallPtrSet<MachineBasicBlock*, 32> InvokeBBs;
Bill Wendling202803e2011-10-05 00:02:33 +00007740 LPadList.reserve(CallSiteNumToLPad.size());
7741 for (unsigned I = 1; I <= MaxCSNum; ++I) {
7742 SmallVectorImpl<MachineBasicBlock*> &MBBList = CallSiteNumToLPad[I];
7743 for (SmallVectorImpl<MachineBasicBlock*>::iterator
Bill Wendling883ec972011-10-07 23:18:02 +00007744 II = MBBList.begin(), IE = MBBList.end(); II != IE; ++II) {
Bill Wendling202803e2011-10-05 00:02:33 +00007745 LPadList.push_back(*II);
Bill Wendling883ec972011-10-07 23:18:02 +00007746 InvokeBBs.insert((*II)->pred_begin(), (*II)->pred_end());
7747 }
Bill Wendling202803e2011-10-05 00:02:33 +00007748 }
7749
Bill Wendlingf793e7e2011-10-05 23:28:57 +00007750 assert(!LPadList.empty() &&
7751 "No landing pad destinations for the dispatch jump table!");
7752
Bill Wendling362c1b02011-10-06 21:29:56 +00007753 // Create the jump table and associated information.
Bill Wendling202803e2011-10-05 00:02:33 +00007754 MachineJumpTableInfo *JTI =
7755 MF->getOrCreateJumpTableInfo(MachineJumpTableInfo::EK_Inline);
7756 unsigned MJTI = JTI->createJumpTableIndex(LPadList);
Bill Wendling202803e2011-10-05 00:02:33 +00007757
Bill Wendling362c1b02011-10-06 21:29:56 +00007758 // Create the MBBs for the dispatch code.
Bill Wendling030b58e2011-10-06 22:18:16 +00007759
7760 // Shove the dispatch's address into the return slot in the function context.
7761 MachineBasicBlock *DispatchBB = MF->CreateMachineBasicBlock();
Reid Kleckner0e288232015-08-27 23:27:47 +00007762 DispatchBB->setIsEHPad();
Bill Wendling030b58e2011-10-06 22:18:16 +00007763
Bill Wendling324be982011-10-05 00:39:32 +00007764 MachineBasicBlock *TrapBB = MF->CreateMachineBasicBlock();
Eli Bendersky2e2ce492013-01-30 16:30:19 +00007765 unsigned trap_opcode;
Chad Rosier11a98282013-02-28 18:54:27 +00007766 if (Subtarget->isThumb())
Eli Bendersky2e2ce492013-01-30 16:30:19 +00007767 trap_opcode = ARM::tTRAP;
Chad Rosier11a98282013-02-28 18:54:27 +00007768 else
7769 trap_opcode = Subtarget->useNaClTrap() ? ARM::TRAPNaCl : ARM::TRAP;
7770
Eli Bendersky2e2ce492013-01-30 16:30:19 +00007771 BuildMI(TrapBB, dl, TII->get(trap_opcode));
Bill Wendling324be982011-10-05 00:39:32 +00007772 DispatchBB->addSuccessor(TrapBB);
7773
7774 MachineBasicBlock *DispContBB = MF->CreateMachineBasicBlock();
7775 DispatchBB->addSuccessor(DispContBB);
Bill Wendling202803e2011-10-05 00:02:33 +00007776
Bill Wendling510fbcd2011-10-17 21:32:56 +00007777 // Insert and MBBs.
Bill Wendling61346552011-10-06 00:53:33 +00007778 MF->insert(MF->end(), DispatchBB);
7779 MF->insert(MF->end(), DispContBB);
7780 MF->insert(MF->end(), TrapBB);
Bill Wendling61346552011-10-06 00:53:33 +00007781
Bill Wendling030b58e2011-10-06 22:18:16 +00007782 // Insert code into the entry block that creates and registers the function
7783 // context.
7784 SetupEntryBlockForSjLj(MI, MBB, DispatchBB, FI);
7785
Alex Lorenze40c8a22015-08-11 23:09:45 +00007786 MachineMemOperand *FIMMOLd = MF->getMachineMemOperand(
7787 MachinePointerInfo::getFixedStack(*MF, FI),
7788 MachineMemOperand::MOLoad | MachineMemOperand::MOVolatile, 4, 4);
Bill Wendling61346552011-10-06 00:53:33 +00007789
Chad Rosier1ec8e402012-11-06 23:05:24 +00007790 MachineInstrBuilder MIB;
7791 MIB = BuildMI(DispatchBB, dl, TII->get(ARM::Int_eh_sjlj_dispatchsetup));
7792
7793 const ARMBaseInstrInfo *AII = static_cast<const ARMBaseInstrInfo*>(TII);
7794 const ARMBaseRegisterInfo &RI = AII->getRegisterInfo();
7795
7796 // Add a register mask with no preserved registers. This results in all
7797 // registers being marked as clobbered.
7798 MIB.addRegMask(RI.getNoPreservedMask());
Bob Wilsonf6d17282011-11-16 07:11:57 +00007799
Rafael Espindola0f898332016-06-20 16:43:17 +00007800 bool IsPositionIndependent = isPositionIndependent();
Bill Wendling85833f72011-10-18 22:49:07 +00007801 unsigned NumLPads = LPadList.size();
Bill Wendling5626c662011-10-06 22:53:00 +00007802 if (Subtarget->isThumb2()) {
7803 unsigned NewVReg1 = MRI->createVirtualRegister(TRC);
7804 AddDefaultPred(BuildMI(DispatchBB, dl, TII->get(ARM::t2LDRi12), NewVReg1)
7805 .addFrameIndex(FI)
7806 .addImm(4)
7807 .addMemOperand(FIMMOLd));
Bill Wendlingb2a703d2011-10-18 21:55:58 +00007808
Bill Wendling85833f72011-10-18 22:49:07 +00007809 if (NumLPads < 256) {
7810 AddDefaultPred(BuildMI(DispatchBB, dl, TII->get(ARM::t2CMPri))
7811 .addReg(NewVReg1)
7812 .addImm(LPadList.size()));
7813 } else {
7814 unsigned VReg1 = MRI->createVirtualRegister(TRC);
7815 AddDefaultPred(BuildMI(DispatchBB, dl, TII->get(ARM::t2MOVi16), VReg1)
Bill Wendling94f60012011-10-18 23:19:55 +00007816 .addImm(NumLPads & 0xFFFF));
7817
7818 unsigned VReg2 = VReg1;
7819 if ((NumLPads & 0xFFFF0000) != 0) {
7820 VReg2 = MRI->createVirtualRegister(TRC);
7821 AddDefaultPred(BuildMI(DispatchBB, dl, TII->get(ARM::t2MOVTi16), VReg2)
7822 .addReg(VReg1)
7823 .addImm(NumLPads >> 16));
7824 }
7825
Bill Wendling85833f72011-10-18 22:49:07 +00007826 AddDefaultPred(BuildMI(DispatchBB, dl, TII->get(ARM::t2CMPrr))
7827 .addReg(NewVReg1)
7828 .addReg(VReg2));
7829 }
Bill Wendlingb2a703d2011-10-18 21:55:58 +00007830
Bill Wendling5626c662011-10-06 22:53:00 +00007831 BuildMI(DispatchBB, dl, TII->get(ARM::t2Bcc))
7832 .addMBB(TrapBB)
7833 .addImm(ARMCC::HI)
7834 .addReg(ARM::CPSR);
Bill Wendling324be982011-10-05 00:39:32 +00007835
Bill Wendlingb2a703d2011-10-18 21:55:58 +00007836 unsigned NewVReg3 = MRI->createVirtualRegister(TRC);
7837 AddDefaultPred(BuildMI(DispContBB, dl, TII->get(ARM::t2LEApcrelJT),NewVReg3)
Tim Northover4998a472015-05-13 20:28:38 +00007838 .addJumpTableIndex(MJTI));
Bill Wendling202803e2011-10-05 00:02:33 +00007839
Bill Wendlingb2a703d2011-10-18 21:55:58 +00007840 unsigned NewVReg4 = MRI->createVirtualRegister(TRC);
Bill Wendling5626c662011-10-06 22:53:00 +00007841 AddDefaultCC(
7842 AddDefaultPred(
Bill Wendlingb2a703d2011-10-18 21:55:58 +00007843 BuildMI(DispContBB, dl, TII->get(ARM::t2ADDrs), NewVReg4)
7844 .addReg(NewVReg3, RegState::Kill)
Bill Wendling5626c662011-10-06 22:53:00 +00007845 .addReg(NewVReg1)
7846 .addImm(ARM_AM::getSORegOpc(ARM_AM::lsl, 2))));
7847
7848 BuildMI(DispContBB, dl, TII->get(ARM::t2BR_JT))
Bill Wendlingb2a703d2011-10-18 21:55:58 +00007849 .addReg(NewVReg4, RegState::Kill)
Bill Wendling202803e2011-10-05 00:02:33 +00007850 .addReg(NewVReg1)
Tim Northover4998a472015-05-13 20:28:38 +00007851 .addJumpTableIndex(MJTI);
Bill Wendling5626c662011-10-06 22:53:00 +00007852 } else if (Subtarget->isThumb()) {
Bill Wendlingb3d46782011-10-06 23:37:36 +00007853 unsigned NewVReg1 = MRI->createVirtualRegister(TRC);
7854 AddDefaultPred(BuildMI(DispatchBB, dl, TII->get(ARM::tLDRspi), NewVReg1)
7855 .addFrameIndex(FI)
7856 .addImm(1)
7857 .addMemOperand(FIMMOLd));
Bill Wendlingf9f5e452011-10-07 22:08:37 +00007858
Bill Wendling64e6bfc2011-10-18 23:11:05 +00007859 if (NumLPads < 256) {
7860 AddDefaultPred(BuildMI(DispatchBB, dl, TII->get(ARM::tCMPi8))
7861 .addReg(NewVReg1)
7862 .addImm(NumLPads));
7863 } else {
7864 MachineConstantPool *ConstantPool = MF->getConstantPool();
Bill Wendling2977a152011-10-19 09:24:02 +00007865 Type *Int32Ty = Type::getInt32Ty(MF->getFunction()->getContext());
7866 const Constant *C = ConstantInt::get(Int32Ty, NumLPads);
7867
7868 // MachineConstantPool wants an explicit alignment.
Mehdi Aminia749f2a2015-07-09 02:09:52 +00007869 unsigned Align = MF->getDataLayout().getPrefTypeAlignment(Int32Ty);
Bill Wendling2977a152011-10-19 09:24:02 +00007870 if (Align == 0)
Mehdi Aminia749f2a2015-07-09 02:09:52 +00007871 Align = MF->getDataLayout().getTypeAllocSize(C->getType());
Bill Wendling2977a152011-10-19 09:24:02 +00007872 unsigned Idx = ConstantPool->getConstantPoolIndex(C, Align);
Bill Wendling64e6bfc2011-10-18 23:11:05 +00007873
7874 unsigned VReg1 = MRI->createVirtualRegister(TRC);
7875 AddDefaultPred(BuildMI(DispatchBB, dl, TII->get(ARM::tLDRpci))
7876 .addReg(VReg1, RegState::Define)
7877 .addConstantPoolIndex(Idx));
7878 AddDefaultPred(BuildMI(DispatchBB, dl, TII->get(ARM::tCMPr))
7879 .addReg(NewVReg1)
7880 .addReg(VReg1));
7881 }
7882
Bill Wendlingb3d46782011-10-06 23:37:36 +00007883 BuildMI(DispatchBB, dl, TII->get(ARM::tBcc))
7884 .addMBB(TrapBB)
7885 .addImm(ARMCC::HI)
7886 .addReg(ARM::CPSR);
7887
7888 unsigned NewVReg2 = MRI->createVirtualRegister(TRC);
7889 AddDefaultPred(BuildMI(DispContBB, dl, TII->get(ARM::tLSLri), NewVReg2)
7890 .addReg(ARM::CPSR, RegState::Define)
7891 .addReg(NewVReg1)
7892 .addImm(2));
7893
7894 unsigned NewVReg3 = MRI->createVirtualRegister(TRC);
Bill Wendling8d50ea02011-10-06 23:41:14 +00007895 AddDefaultPred(BuildMI(DispContBB, dl, TII->get(ARM::tLEApcrelJT), NewVReg3)
Tim Northover4998a472015-05-13 20:28:38 +00007896 .addJumpTableIndex(MJTI));
Bill Wendlingb3d46782011-10-06 23:37:36 +00007897
7898 unsigned NewVReg4 = MRI->createVirtualRegister(TRC);
7899 AddDefaultPred(BuildMI(DispContBB, dl, TII->get(ARM::tADDrr), NewVReg4)
7900 .addReg(ARM::CPSR, RegState::Define)
7901 .addReg(NewVReg2, RegState::Kill)
7902 .addReg(NewVReg3));
7903
Alex Lorenze40c8a22015-08-11 23:09:45 +00007904 MachineMemOperand *JTMMOLd = MF->getMachineMemOperand(
7905 MachinePointerInfo::getJumpTable(*MF), MachineMemOperand::MOLoad, 4, 4);
Bill Wendlingb3d46782011-10-06 23:37:36 +00007906
7907 unsigned NewVReg5 = MRI->createVirtualRegister(TRC);
7908 AddDefaultPred(BuildMI(DispContBB, dl, TII->get(ARM::tLDRi), NewVReg5)
7909 .addReg(NewVReg4, RegState::Kill)
7910 .addImm(0)
7911 .addMemOperand(JTMMOLd));
7912
Chad Rosier96603432013-03-01 18:30:38 +00007913 unsigned NewVReg6 = NewVReg5;
Rafael Espindola0f898332016-06-20 16:43:17 +00007914 if (IsPositionIndependent) {
Chad Rosier96603432013-03-01 18:30:38 +00007915 NewVReg6 = MRI->createVirtualRegister(TRC);
7916 AddDefaultPred(BuildMI(DispContBB, dl, TII->get(ARM::tADDrr), NewVReg6)
7917 .addReg(ARM::CPSR, RegState::Define)
7918 .addReg(NewVReg5, RegState::Kill)
7919 .addReg(NewVReg3));
7920 }
Bill Wendlingb3d46782011-10-06 23:37:36 +00007921
7922 BuildMI(DispContBB, dl, TII->get(ARM::tBR_JTr))
7923 .addReg(NewVReg6, RegState::Kill)
Tim Northover4998a472015-05-13 20:28:38 +00007924 .addJumpTableIndex(MJTI);
Bill Wendling5626c662011-10-06 22:53:00 +00007925 } else {
7926 unsigned NewVReg1 = MRI->createVirtualRegister(TRC);
7927 AddDefaultPred(BuildMI(DispatchBB, dl, TII->get(ARM::LDRi12), NewVReg1)
7928 .addFrameIndex(FI)
7929 .addImm(4)
7930 .addMemOperand(FIMMOLd));
Bill Wendling973c8172011-10-18 22:11:18 +00007931
Bill Wendling4969dcd2011-10-18 22:52:20 +00007932 if (NumLPads < 256) {
7933 AddDefaultPred(BuildMI(DispatchBB, dl, TII->get(ARM::CMPri))
7934 .addReg(NewVReg1)
7935 .addImm(NumLPads));
Bill Wendling2977a152011-10-19 09:24:02 +00007936 } else if (Subtarget->hasV6T2Ops() && isUInt<16>(NumLPads)) {
Bill Wendling4969dcd2011-10-18 22:52:20 +00007937 unsigned VReg1 = MRI->createVirtualRegister(TRC);
7938 AddDefaultPred(BuildMI(DispatchBB, dl, TII->get(ARM::MOVi16), VReg1)
Bill Wendling94f60012011-10-18 23:19:55 +00007939 .addImm(NumLPads & 0xFFFF));
7940
7941 unsigned VReg2 = VReg1;
7942 if ((NumLPads & 0xFFFF0000) != 0) {
7943 VReg2 = MRI->createVirtualRegister(TRC);
7944 AddDefaultPred(BuildMI(DispatchBB, dl, TII->get(ARM::MOVTi16), VReg2)
7945 .addReg(VReg1)
7946 .addImm(NumLPads >> 16));
7947 }
7948
Bill Wendling4969dcd2011-10-18 22:52:20 +00007949 AddDefaultPred(BuildMI(DispatchBB, dl, TII->get(ARM::CMPrr))
7950 .addReg(NewVReg1)
7951 .addReg(VReg2));
Bill Wendling2977a152011-10-19 09:24:02 +00007952 } else {
7953 MachineConstantPool *ConstantPool = MF->getConstantPool();
7954 Type *Int32Ty = Type::getInt32Ty(MF->getFunction()->getContext());
7955 const Constant *C = ConstantInt::get(Int32Ty, NumLPads);
7956
7957 // MachineConstantPool wants an explicit alignment.
Mehdi Aminia749f2a2015-07-09 02:09:52 +00007958 unsigned Align = MF->getDataLayout().getPrefTypeAlignment(Int32Ty);
Bill Wendling2977a152011-10-19 09:24:02 +00007959 if (Align == 0)
Mehdi Aminia749f2a2015-07-09 02:09:52 +00007960 Align = MF->getDataLayout().getTypeAllocSize(C->getType());
Bill Wendling2977a152011-10-19 09:24:02 +00007961 unsigned Idx = ConstantPool->getConstantPoolIndex(C, Align);
7962
7963 unsigned VReg1 = MRI->createVirtualRegister(TRC);
7964 AddDefaultPred(BuildMI(DispatchBB, dl, TII->get(ARM::LDRcp))
7965 .addReg(VReg1, RegState::Define)
Bill Wendlingcf7bdf42011-10-20 20:37:11 +00007966 .addConstantPoolIndex(Idx)
7967 .addImm(0));
Bill Wendling2977a152011-10-19 09:24:02 +00007968 AddDefaultPred(BuildMI(DispatchBB, dl, TII->get(ARM::CMPrr))
7969 .addReg(NewVReg1)
7970 .addReg(VReg1, RegState::Kill));
Bill Wendling4969dcd2011-10-18 22:52:20 +00007971 }
7972
Bill Wendling5626c662011-10-06 22:53:00 +00007973 BuildMI(DispatchBB, dl, TII->get(ARM::Bcc))
7974 .addMBB(TrapBB)
7975 .addImm(ARMCC::HI)
7976 .addReg(ARM::CPSR);
Bill Wendling202803e2011-10-05 00:02:33 +00007977
Bill Wendling973c8172011-10-18 22:11:18 +00007978 unsigned NewVReg3 = MRI->createVirtualRegister(TRC);
Bill Wendling5626c662011-10-06 22:53:00 +00007979 AddDefaultCC(
Bill Wendling973c8172011-10-18 22:11:18 +00007980 AddDefaultPred(BuildMI(DispContBB, dl, TII->get(ARM::MOVsi), NewVReg3)
Bill Wendling5626c662011-10-06 22:53:00 +00007981 .addReg(NewVReg1)
7982 .addImm(ARM_AM::getSORegOpc(ARM_AM::lsl, 2))));
Bill Wendling973c8172011-10-18 22:11:18 +00007983 unsigned NewVReg4 = MRI->createVirtualRegister(TRC);
7984 AddDefaultPred(BuildMI(DispContBB, dl, TII->get(ARM::LEApcrelJT), NewVReg4)
Tim Northover4998a472015-05-13 20:28:38 +00007985 .addJumpTableIndex(MJTI));
Bill Wendling5626c662011-10-06 22:53:00 +00007986
Alex Lorenze40c8a22015-08-11 23:09:45 +00007987 MachineMemOperand *JTMMOLd = MF->getMachineMemOperand(
7988 MachinePointerInfo::getJumpTable(*MF), MachineMemOperand::MOLoad, 4, 4);
Bill Wendling973c8172011-10-18 22:11:18 +00007989 unsigned NewVReg5 = MRI->createVirtualRegister(TRC);
Bill Wendling5626c662011-10-06 22:53:00 +00007990 AddDefaultPred(
Bill Wendling973c8172011-10-18 22:11:18 +00007991 BuildMI(DispContBB, dl, TII->get(ARM::LDRrs), NewVReg5)
7992 .addReg(NewVReg3, RegState::Kill)
7993 .addReg(NewVReg4)
Bill Wendling5626c662011-10-06 22:53:00 +00007994 .addImm(0)
7995 .addMemOperand(JTMMOLd));
7996
Rafael Espindola0f898332016-06-20 16:43:17 +00007997 if (IsPositionIndependent) {
Chad Rosier96603432013-03-01 18:30:38 +00007998 BuildMI(DispContBB, dl, TII->get(ARM::BR_JTadd))
7999 .addReg(NewVReg5, RegState::Kill)
8000 .addReg(NewVReg4)
Tim Northover4998a472015-05-13 20:28:38 +00008001 .addJumpTableIndex(MJTI);
Chad Rosier96603432013-03-01 18:30:38 +00008002 } else {
8003 BuildMI(DispContBB, dl, TII->get(ARM::BR_JTr))
8004 .addReg(NewVReg5, RegState::Kill)
Tim Northover4998a472015-05-13 20:28:38 +00008005 .addJumpTableIndex(MJTI);
Chad Rosier96603432013-03-01 18:30:38 +00008006 }
Bill Wendling5626c662011-10-06 22:53:00 +00008007 }
Bill Wendling202803e2011-10-05 00:02:33 +00008008
Bill Wendling324be982011-10-05 00:39:32 +00008009 // Add the jump table entries as successors to the MBB.
Jakob Stoklund Olesen710093e2012-08-20 20:52:03 +00008010 SmallPtrSet<MachineBasicBlock*, 8> SeenMBBs;
Bill Wendling324be982011-10-05 00:39:32 +00008011 for (std::vector<MachineBasicBlock*>::iterator
Bill Wendling883ec972011-10-07 23:18:02 +00008012 I = LPadList.begin(), E = LPadList.end(); I != E; ++I) {
8013 MachineBasicBlock *CurMBB = *I;
David Blaikie70573dc2014-11-19 07:49:26 +00008014 if (SeenMBBs.insert(CurMBB).second)
Bill Wendling883ec972011-10-07 23:18:02 +00008015 DispContBB->addSuccessor(CurMBB);
Bill Wendling883ec972011-10-07 23:18:02 +00008016 }
8017
Bill Wendling26d27802011-10-17 05:25:09 +00008018 // N.B. the order the invoke BBs are processed in doesn't matter here.
Craig Topper840beec2014-04-04 05:16:06 +00008019 const MCPhysReg *SavedRegs = RI.getCalleeSavedRegs(MF);
Bill Wendling617075f2011-10-18 18:30:49 +00008020 SmallVector<MachineBasicBlock*, 64> MBBLPads;
Craig Topper46276792014-08-24 23:23:06 +00008021 for (MachineBasicBlock *BB : InvokeBBs) {
Bill Wendling6f3f9a32011-10-14 23:34:37 +00008022
8023 // Remove the landing pad successor from the invoke block and replace it
8024 // with the new dispatch block.
Bill Wendling1414bc52011-10-26 07:16:18 +00008025 SmallVector<MachineBasicBlock*, 4> Successors(BB->succ_begin(),
8026 BB->succ_end());
8027 while (!Successors.empty()) {
8028 MachineBasicBlock *SMBB = Successors.pop_back_val();
Reid Kleckner0e288232015-08-27 23:27:47 +00008029 if (SMBB->isEHPad()) {
Bill Wendling883ec972011-10-07 23:18:02 +00008030 BB->removeSuccessor(SMBB);
Bill Wendling617075f2011-10-18 18:30:49 +00008031 MBBLPads.push_back(SMBB);
Bill Wendling883ec972011-10-07 23:18:02 +00008032 }
8033 }
8034
Cong Houd97c1002015-12-01 05:29:22 +00008035 BB->addSuccessor(DispatchBB, BranchProbability::getZero());
Cong Houc1069892015-12-13 09:26:17 +00008036 BB->normalizeSuccProbs();
Bill Wendling6f3f9a32011-10-14 23:34:37 +00008037
8038 // Find the invoke call and mark all of the callee-saved registers as
8039 // 'implicit defined' so that they're spilled. This prevents code from
8040 // moving instructions to before the EH block, where they will never be
8041 // executed.
8042 for (MachineBasicBlock::reverse_iterator
8043 II = BB->rbegin(), IE = BB->rend(); II != IE; ++II) {
Evan Cheng7f8e5632011-12-07 07:15:52 +00008044 if (!II->isCall()) continue;
Bill Wendling6f3f9a32011-10-14 23:34:37 +00008045
8046 DenseMap<unsigned, bool> DefRegs;
8047 for (MachineInstr::mop_iterator
8048 OI = II->operands_begin(), OE = II->operands_end();
8049 OI != OE; ++OI) {
8050 if (!OI->isReg()) continue;
8051 DefRegs[OI->getReg()] = true;
8052 }
8053
Jakob Stoklund Olesenb159b5f2012-12-19 21:31:56 +00008054 MachineInstrBuilder MIB(*MF, &*II);
Bill Wendling6f3f9a32011-10-14 23:34:37 +00008055
Bill Wendling9e0cd1e2011-10-14 23:55:44 +00008056 for (unsigned i = 0; SavedRegs[i] != 0; ++i) {
Bill Wendling94e66432011-10-22 00:29:28 +00008057 unsigned Reg = SavedRegs[i];
8058 if (Subtarget->isThumb2() &&
Craig Topperc7242e02012-04-20 07:30:17 +00008059 !ARM::tGPRRegClass.contains(Reg) &&
8060 !ARM::hGPRRegClass.contains(Reg))
Bill Wendling94e66432011-10-22 00:29:28 +00008061 continue;
Craig Topperc7242e02012-04-20 07:30:17 +00008062 if (Subtarget->isThumb1Only() && !ARM::tGPRRegClass.contains(Reg))
Bill Wendling94e66432011-10-22 00:29:28 +00008063 continue;
Craig Topperc7242e02012-04-20 07:30:17 +00008064 if (!Subtarget->isThumb() && !ARM::GPRRegClass.contains(Reg))
Bill Wendling94e66432011-10-22 00:29:28 +00008065 continue;
8066 if (!DefRegs[Reg])
8067 MIB.addReg(Reg, RegState::ImplicitDefine | RegState::Dead);
Bill Wendling9e0cd1e2011-10-14 23:55:44 +00008068 }
Bill Wendling6f3f9a32011-10-14 23:34:37 +00008069
8070 break;
8071 }
Bill Wendling883ec972011-10-07 23:18:02 +00008072 }
Bill Wendling324be982011-10-05 00:39:32 +00008073
Bill Wendling617075f2011-10-18 18:30:49 +00008074 // Mark all former landing pads as non-landing pads. The dispatch is the only
8075 // landing pad now.
8076 for (SmallVectorImpl<MachineBasicBlock*>::iterator
8077 I = MBBLPads.begin(), E = MBBLPads.end(); I != E; ++I)
Reid Kleckner0e288232015-08-27 23:27:47 +00008078 (*I)->setIsEHPad(false);
Bill Wendling617075f2011-10-18 18:30:49 +00008079
Bill Wendling324be982011-10-05 00:39:32 +00008080 // The instruction is gone now.
Duncan P. N. Exon Smithe4f5e4f2016-06-30 22:52:52 +00008081 MI.eraseFromParent();
Bill Wendling374ee192011-10-03 21:25:38 +00008082}
8083
Evan Cheng0cc4ad92010-07-13 19:27:42 +00008084static
8085MachineBasicBlock *OtherSucc(MachineBasicBlock *MBB, MachineBasicBlock *Succ) {
8086 for (MachineBasicBlock::succ_iterator I = MBB->succ_begin(),
8087 E = MBB->succ_end(); I != E; ++I)
8088 if (*I != Succ)
8089 return *I;
8090 llvm_unreachable("Expecting a BB with two successors!");
8091}
8092
Manman Renb504f492013-10-29 22:27:32 +00008093/// Return the load opcode for a given load size. If load size >= 8,
8094/// neon opcode will be returned.
8095static unsigned getLdOpcode(unsigned LdSize, bool IsThumb1, bool IsThumb2) {
8096 if (LdSize >= 8)
8097 return LdSize == 16 ? ARM::VLD1q32wb_fixed
8098 : LdSize == 8 ? ARM::VLD1d32wb_fixed : 0;
8099 if (IsThumb1)
8100 return LdSize == 4 ? ARM::tLDRi
8101 : LdSize == 2 ? ARM::tLDRHi
8102 : LdSize == 1 ? ARM::tLDRBi : 0;
8103 if (IsThumb2)
8104 return LdSize == 4 ? ARM::t2LDR_POST
8105 : LdSize == 2 ? ARM::t2LDRH_POST
8106 : LdSize == 1 ? ARM::t2LDRB_POST : 0;
8107 return LdSize == 4 ? ARM::LDR_POST_IMM
8108 : LdSize == 2 ? ARM::LDRH_POST
8109 : LdSize == 1 ? ARM::LDRB_POST_IMM : 0;
8110}
8111
8112/// Return the store opcode for a given store size. If store size >= 8,
8113/// neon opcode will be returned.
8114static unsigned getStOpcode(unsigned StSize, bool IsThumb1, bool IsThumb2) {
8115 if (StSize >= 8)
8116 return StSize == 16 ? ARM::VST1q32wb_fixed
8117 : StSize == 8 ? ARM::VST1d32wb_fixed : 0;
8118 if (IsThumb1)
8119 return StSize == 4 ? ARM::tSTRi
8120 : StSize == 2 ? ARM::tSTRHi
8121 : StSize == 1 ? ARM::tSTRBi : 0;
8122 if (IsThumb2)
8123 return StSize == 4 ? ARM::t2STR_POST
8124 : StSize == 2 ? ARM::t2STRH_POST
8125 : StSize == 1 ? ARM::t2STRB_POST : 0;
8126 return StSize == 4 ? ARM::STR_POST_IMM
8127 : StSize == 2 ? ARM::STRH_POST
8128 : StSize == 1 ? ARM::STRB_POST_IMM : 0;
8129}
8130
8131/// Emit a post-increment load operation with given size. The instructions
8132/// will be added to BB at Pos.
Duncan P. N. Exon Smithe4f5e4f2016-06-30 22:52:52 +00008133static void emitPostLd(MachineBasicBlock *BB, MachineBasicBlock::iterator Pos,
Benjamin Kramerbdc49562016-06-12 15:39:02 +00008134 const TargetInstrInfo *TII, const DebugLoc &dl,
Manman Renb504f492013-10-29 22:27:32 +00008135 unsigned LdSize, unsigned Data, unsigned AddrIn,
8136 unsigned AddrOut, bool IsThumb1, bool IsThumb2) {
8137 unsigned LdOpc = getLdOpcode(LdSize, IsThumb1, IsThumb2);
8138 assert(LdOpc != 0 && "Should have a load opcode");
8139 if (LdSize >= 8) {
8140 AddDefaultPred(BuildMI(*BB, Pos, dl, TII->get(LdOpc), Data)
8141 .addReg(AddrOut, RegState::Define).addReg(AddrIn)
8142 .addImm(0));
8143 } else if (IsThumb1) {
8144 // load + update AddrIn
8145 AddDefaultPred(BuildMI(*BB, Pos, dl, TII->get(LdOpc), Data)
8146 .addReg(AddrIn).addImm(0));
8147 MachineInstrBuilder MIB =
8148 BuildMI(*BB, Pos, dl, TII->get(ARM::tADDi8), AddrOut);
8149 MIB = AddDefaultT1CC(MIB);
8150 MIB.addReg(AddrIn).addImm(LdSize);
8151 AddDefaultPred(MIB);
8152 } else if (IsThumb2) {
8153 AddDefaultPred(BuildMI(*BB, Pos, dl, TII->get(LdOpc), Data)
8154 .addReg(AddrOut, RegState::Define).addReg(AddrIn)
8155 .addImm(LdSize));
8156 } else { // arm
8157 AddDefaultPred(BuildMI(*BB, Pos, dl, TII->get(LdOpc), Data)
8158 .addReg(AddrOut, RegState::Define).addReg(AddrIn)
8159 .addReg(0).addImm(LdSize));
8160 }
8161}
8162
8163/// Emit a post-increment store operation with given size. The instructions
8164/// will be added to BB at Pos.
Duncan P. N. Exon Smithe4f5e4f2016-06-30 22:52:52 +00008165static void emitPostSt(MachineBasicBlock *BB, MachineBasicBlock::iterator Pos,
Benjamin Kramerbdc49562016-06-12 15:39:02 +00008166 const TargetInstrInfo *TII, const DebugLoc &dl,
Manman Renb504f492013-10-29 22:27:32 +00008167 unsigned StSize, unsigned Data, unsigned AddrIn,
8168 unsigned AddrOut, bool IsThumb1, bool IsThumb2) {
8169 unsigned StOpc = getStOpcode(StSize, IsThumb1, IsThumb2);
8170 assert(StOpc != 0 && "Should have a store opcode");
8171 if (StSize >= 8) {
8172 AddDefaultPred(BuildMI(*BB, Pos, dl, TII->get(StOpc), AddrOut)
8173 .addReg(AddrIn).addImm(0).addReg(Data));
8174 } else if (IsThumb1) {
8175 // store + update AddrIn
8176 AddDefaultPred(BuildMI(*BB, Pos, dl, TII->get(StOpc)).addReg(Data)
8177 .addReg(AddrIn).addImm(0));
8178 MachineInstrBuilder MIB =
8179 BuildMI(*BB, Pos, dl, TII->get(ARM::tADDi8), AddrOut);
8180 MIB = AddDefaultT1CC(MIB);
8181 MIB.addReg(AddrIn).addImm(StSize);
8182 AddDefaultPred(MIB);
8183 } else if (IsThumb2) {
8184 AddDefaultPred(BuildMI(*BB, Pos, dl, TII->get(StOpc), AddrOut)
8185 .addReg(Data).addReg(AddrIn).addImm(StSize));
8186 } else { // arm
8187 AddDefaultPred(BuildMI(*BB, Pos, dl, TII->get(StOpc), AddrOut)
8188 .addReg(Data).addReg(AddrIn).addReg(0)
8189 .addImm(StSize));
8190 }
8191}
8192
David Peixottoc32e24a2013-10-17 19:49:22 +00008193MachineBasicBlock *
Duncan P. N. Exon Smithe4f5e4f2016-06-30 22:52:52 +00008194ARMTargetLowering::EmitStructByval(MachineInstr &MI,
David Peixottoc32e24a2013-10-17 19:49:22 +00008195 MachineBasicBlock *BB) const {
Manman Rene8735522012-06-01 19:33:18 +00008196 // This pseudo instruction has 3 operands: dst, src, size
8197 // We expand it to a loop if size > Subtarget->getMaxInlineSizeThreshold().
8198 // Otherwise, we will generate unrolled scalar copies.
Eric Christopher1889fdc2015-01-29 00:19:39 +00008199 const TargetInstrInfo *TII = Subtarget->getInstrInfo();
Manman Rene8735522012-06-01 19:33:18 +00008200 const BasicBlock *LLVM_BB = BB->getBasicBlock();
Duncan P. N. Exon Smith9f9559e2015-10-19 23:25:57 +00008201 MachineFunction::iterator It = ++BB->getIterator();
Manman Rene8735522012-06-01 19:33:18 +00008202
Duncan P. N. Exon Smithe4f5e4f2016-06-30 22:52:52 +00008203 unsigned dest = MI.getOperand(0).getReg();
8204 unsigned src = MI.getOperand(1).getReg();
8205 unsigned SizeVal = MI.getOperand(2).getImm();
8206 unsigned Align = MI.getOperand(3).getImm();
8207 DebugLoc dl = MI.getDebugLoc();
Manman Rene8735522012-06-01 19:33:18 +00008208
Manman Rene8735522012-06-01 19:33:18 +00008209 MachineFunction *MF = BB->getParent();
8210 MachineRegisterInfo &MRI = MF->getRegInfo();
David Peixottoc32e24a2013-10-17 19:49:22 +00008211 unsigned UnitSize = 0;
Craig Topper062a2ba2014-04-25 05:30:21 +00008212 const TargetRegisterClass *TRC = nullptr;
8213 const TargetRegisterClass *VecTRC = nullptr;
David Peixottob0653e532013-10-24 16:39:36 +00008214
8215 bool IsThumb1 = Subtarget->isThumb1Only();
8216 bool IsThumb2 = Subtarget->isThumb2();
Prakhar Bahugunad1233e82016-07-29 09:16:46 +00008217 bool IsThumb = Subtarget->isThumb();
Manman Rene8735522012-06-01 19:33:18 +00008218
8219 if (Align & 1) {
Manman Rene8735522012-06-01 19:33:18 +00008220 UnitSize = 1;
8221 } else if (Align & 2) {
Manman Rene8735522012-06-01 19:33:18 +00008222 UnitSize = 2;
8223 } else {
Manman Ren6e1fd462012-06-18 22:23:48 +00008224 // Check whether we can use NEON instructions.
Duncan P. N. Exon Smith2cff9e12015-02-14 02:24:44 +00008225 if (!MF->getFunction()->hasFnAttribute(Attribute::NoImplicitFloat) &&
Manman Ren6e1fd462012-06-18 22:23:48 +00008226 Subtarget->hasNEON()) {
David Peixottoc32e24a2013-10-17 19:49:22 +00008227 if ((Align % 16 == 0) && SizeVal >= 16)
Manman Ren6e1fd462012-06-18 22:23:48 +00008228 UnitSize = 16;
David Peixottoc32e24a2013-10-17 19:49:22 +00008229 else if ((Align % 8 == 0) && SizeVal >= 8)
Manman Ren6e1fd462012-06-18 22:23:48 +00008230 UnitSize = 8;
Manman Ren6e1fd462012-06-18 22:23:48 +00008231 }
8232 // Can't use NEON instructions.
David Peixottoc32e24a2013-10-17 19:49:22 +00008233 if (UnitSize == 0)
Manman Ren6e1fd462012-06-18 22:23:48 +00008234 UnitSize = 4;
Manman Rene8735522012-06-01 19:33:18 +00008235 }
Manman Ren6e1fd462012-06-18 22:23:48 +00008236
David Peixottob0653e532013-10-24 16:39:36 +00008237 // Select the correct opcode and register class for unit size load/store
8238 bool IsNeon = UnitSize >= 8;
Prakhar Bahugunad1233e82016-07-29 09:16:46 +00008239 TRC = IsThumb ? &ARM::tGPRRegClass : &ARM::GPRRegClass;
Manman Renb504f492013-10-29 22:27:32 +00008240 if (IsNeon)
Craig Topper61e88f42014-11-21 05:58:21 +00008241 VecTRC = UnitSize == 16 ? &ARM::DPairRegClass
8242 : UnitSize == 8 ? &ARM::DPRRegClass
8243 : nullptr;
David Peixottob0653e532013-10-24 16:39:36 +00008244
Manman Rene8735522012-06-01 19:33:18 +00008245 unsigned BytesLeft = SizeVal % UnitSize;
8246 unsigned LoopSize = SizeVal - BytesLeft;
8247
8248 if (SizeVal <= Subtarget->getMaxInlineSizeThreshold()) {
8249 // Use LDR and STR to copy.
8250 // [scratch, srcOut] = LDR_POST(srcIn, UnitSize)
8251 // [destOut] = STR_POST(scratch, destIn, UnitSize)
8252 unsigned srcIn = src;
8253 unsigned destIn = dest;
8254 for (unsigned i = 0; i < LoopSize; i+=UnitSize) {
David Peixottob0653e532013-10-24 16:39:36 +00008255 unsigned srcOut = MRI.createVirtualRegister(TRC);
8256 unsigned destOut = MRI.createVirtualRegister(TRC);
8257 unsigned scratch = MRI.createVirtualRegister(IsNeon ? VecTRC : TRC);
Manman Renb504f492013-10-29 22:27:32 +00008258 emitPostLd(BB, MI, TII, dl, UnitSize, scratch, srcIn, srcOut,
8259 IsThumb1, IsThumb2);
8260 emitPostSt(BB, MI, TII, dl, UnitSize, scratch, destIn, destOut,
8261 IsThumb1, IsThumb2);
David Peixottob0653e532013-10-24 16:39:36 +00008262 srcIn = srcOut;
8263 destIn = destOut;
Manman Rene8735522012-06-01 19:33:18 +00008264 }
8265
8266 // Handle the leftover bytes with LDRB and STRB.
8267 // [scratch, srcOut] = LDRB_POST(srcIn, 1)
8268 // [destOut] = STRB_POST(scratch, destIn, 1)
Manman Rene8735522012-06-01 19:33:18 +00008269 for (unsigned i = 0; i < BytesLeft; i++) {
David Peixottob0653e532013-10-24 16:39:36 +00008270 unsigned srcOut = MRI.createVirtualRegister(TRC);
8271 unsigned destOut = MRI.createVirtualRegister(TRC);
8272 unsigned scratch = MRI.createVirtualRegister(TRC);
Manman Renb504f492013-10-29 22:27:32 +00008273 emitPostLd(BB, MI, TII, dl, 1, scratch, srcIn, srcOut,
8274 IsThumb1, IsThumb2);
8275 emitPostSt(BB, MI, TII, dl, 1, scratch, destIn, destOut,
8276 IsThumb1, IsThumb2);
David Peixottob0653e532013-10-24 16:39:36 +00008277 srcIn = srcOut;
8278 destIn = destOut;
Manman Rene8735522012-06-01 19:33:18 +00008279 }
Duncan P. N. Exon Smithe4f5e4f2016-06-30 22:52:52 +00008280 MI.eraseFromParent(); // The instruction is gone now.
Manman Rene8735522012-06-01 19:33:18 +00008281 return BB;
8282 }
8283
8284 // Expand the pseudo op to a loop.
8285 // thisMBB:
8286 // ...
8287 // movw varEnd, # --> with thumb2
8288 // movt varEnd, #
8289 // ldrcp varEnd, idx --> without thumb2
8290 // fallthrough --> loopMBB
8291 // loopMBB:
8292 // PHI varPhi, varEnd, varLoop
8293 // PHI srcPhi, src, srcLoop
8294 // PHI destPhi, dst, destLoop
8295 // [scratch, srcLoop] = LDR_POST(srcPhi, UnitSize)
8296 // [destLoop] = STR_POST(scratch, destPhi, UnitSize)
8297 // subs varLoop, varPhi, #UnitSize
8298 // bne loopMBB
8299 // fallthrough --> exitMBB
8300 // exitMBB:
8301 // epilogue to handle left-over bytes
8302 // [scratch, srcOut] = LDRB_POST(srcLoop, 1)
8303 // [destOut] = STRB_POST(scratch, destLoop, 1)
8304 MachineBasicBlock *loopMBB = MF->CreateMachineBasicBlock(LLVM_BB);
8305 MachineBasicBlock *exitMBB = MF->CreateMachineBasicBlock(LLVM_BB);
8306 MF->insert(It, loopMBB);
8307 MF->insert(It, exitMBB);
8308
8309 // Transfer the remainder of BB and its successor edges to exitMBB.
8310 exitMBB->splice(exitMBB->begin(), BB,
Benjamin Kramerb6d0bd42014-03-02 12:27:27 +00008311 std::next(MachineBasicBlock::iterator(MI)), BB->end());
Manman Rene8735522012-06-01 19:33:18 +00008312 exitMBB->transferSuccessorsAndUpdatePHIs(BB);
8313
8314 // Load an immediate to varEnd.
David Peixottob0653e532013-10-24 16:39:36 +00008315 unsigned varEnd = MRI.createVirtualRegister(TRC);
Derek Schuffb0513892015-03-26 22:11:00 +00008316 if (Subtarget->useMovt(*MF)) {
David Peixottob0653e532013-10-24 16:39:36 +00008317 unsigned Vtmp = varEnd;
8318 if ((LoopSize & 0xFFFF0000) != 0)
8319 Vtmp = MRI.createVirtualRegister(TRC);
Derek Schuffb0513892015-03-26 22:11:00 +00008320 AddDefaultPred(BuildMI(BB, dl,
Prakhar Bahugunad1233e82016-07-29 09:16:46 +00008321 TII->get(IsThumb ? ARM::t2MOVi16 : ARM::MOVi16),
Derek Schuffb0513892015-03-26 22:11:00 +00008322 Vtmp).addImm(LoopSize & 0xFFFF));
David Peixottob0653e532013-10-24 16:39:36 +00008323
8324 if ((LoopSize & 0xFFFF0000) != 0)
Derek Schuffb0513892015-03-26 22:11:00 +00008325 AddDefaultPred(BuildMI(BB, dl,
Prakhar Bahugunad1233e82016-07-29 09:16:46 +00008326 TII->get(IsThumb ? ARM::t2MOVTi16 : ARM::MOVTi16),
Derek Schuffb0513892015-03-26 22:11:00 +00008327 varEnd)
8328 .addReg(Vtmp)
8329 .addImm(LoopSize >> 16));
David Peixottob0653e532013-10-24 16:39:36 +00008330 } else {
8331 MachineConstantPool *ConstantPool = MF->getConstantPool();
8332 Type *Int32Ty = Type::getInt32Ty(MF->getFunction()->getContext());
8333 const Constant *C = ConstantInt::get(Int32Ty, LoopSize);
8334
8335 // MachineConstantPool wants an explicit alignment.
Mehdi Aminia749f2a2015-07-09 02:09:52 +00008336 unsigned Align = MF->getDataLayout().getPrefTypeAlignment(Int32Ty);
David Peixottob0653e532013-10-24 16:39:36 +00008337 if (Align == 0)
Mehdi Aminia749f2a2015-07-09 02:09:52 +00008338 Align = MF->getDataLayout().getTypeAllocSize(C->getType());
David Peixottob0653e532013-10-24 16:39:36 +00008339 unsigned Idx = ConstantPool->getConstantPoolIndex(C, Align);
8340
Prakhar Bahugunad1233e82016-07-29 09:16:46 +00008341 if (IsThumb)
David Peixottob0653e532013-10-24 16:39:36 +00008342 AddDefaultPred(BuildMI(*BB, MI, dl, TII->get(ARM::tLDRpci)).addReg(
8343 varEnd, RegState::Define).addConstantPoolIndex(Idx));
8344 else
8345 AddDefaultPred(BuildMI(*BB, MI, dl, TII->get(ARM::LDRcp)).addReg(
8346 varEnd, RegState::Define).addConstantPoolIndex(Idx).addImm(0));
8347 }
Manman Rene8735522012-06-01 19:33:18 +00008348 BB->addSuccessor(loopMBB);
8349
8350 // Generate the loop body:
8351 // varPhi = PHI(varLoop, varEnd)
8352 // srcPhi = PHI(srcLoop, src)
8353 // destPhi = PHI(destLoop, dst)
8354 MachineBasicBlock *entryBB = BB;
8355 BB = loopMBB;
David Peixottob0653e532013-10-24 16:39:36 +00008356 unsigned varLoop = MRI.createVirtualRegister(TRC);
8357 unsigned varPhi = MRI.createVirtualRegister(TRC);
8358 unsigned srcLoop = MRI.createVirtualRegister(TRC);
8359 unsigned srcPhi = MRI.createVirtualRegister(TRC);
8360 unsigned destLoop = MRI.createVirtualRegister(TRC);
8361 unsigned destPhi = MRI.createVirtualRegister(TRC);
Manman Rene8735522012-06-01 19:33:18 +00008362
8363 BuildMI(*BB, BB->begin(), dl, TII->get(ARM::PHI), varPhi)
8364 .addReg(varLoop).addMBB(loopMBB)
8365 .addReg(varEnd).addMBB(entryBB);
8366 BuildMI(BB, dl, TII->get(ARM::PHI), srcPhi)
8367 .addReg(srcLoop).addMBB(loopMBB)
8368 .addReg(src).addMBB(entryBB);
8369 BuildMI(BB, dl, TII->get(ARM::PHI), destPhi)
8370 .addReg(destLoop).addMBB(loopMBB)
8371 .addReg(dest).addMBB(entryBB);
8372
8373 // [scratch, srcLoop] = LDR_POST(srcPhi, UnitSize)
8374 // [destLoop] = STR_POST(scratch, destPhi, UnitSiz)
David Peixottob0653e532013-10-24 16:39:36 +00008375 unsigned scratch = MRI.createVirtualRegister(IsNeon ? VecTRC : TRC);
Manman Renb504f492013-10-29 22:27:32 +00008376 emitPostLd(BB, BB->end(), TII, dl, UnitSize, scratch, srcPhi, srcLoop,
8377 IsThumb1, IsThumb2);
8378 emitPostSt(BB, BB->end(), TII, dl, UnitSize, scratch, destPhi, destLoop,
8379 IsThumb1, IsThumb2);
Manman Rene8735522012-06-01 19:33:18 +00008380
8381 // Decrement loop variable by UnitSize.
David Peixottob0653e532013-10-24 16:39:36 +00008382 if (IsThumb1) {
8383 MachineInstrBuilder MIB =
8384 BuildMI(*BB, BB->end(), dl, TII->get(ARM::tSUBi8), varLoop);
8385 MIB = AddDefaultT1CC(MIB);
8386 MIB.addReg(varPhi).addImm(UnitSize);
8387 AddDefaultPred(MIB);
8388 } else {
8389 MachineInstrBuilder MIB =
8390 BuildMI(*BB, BB->end(), dl,
8391 TII->get(IsThumb2 ? ARM::t2SUBri : ARM::SUBri), varLoop);
8392 AddDefaultCC(AddDefaultPred(MIB.addReg(varPhi).addImm(UnitSize)));
8393 MIB->getOperand(5).setReg(ARM::CPSR);
8394 MIB->getOperand(5).setIsDef(true);
8395 }
8396 BuildMI(*BB, BB->end(), dl,
8397 TII->get(IsThumb1 ? ARM::tBcc : IsThumb2 ? ARM::t2Bcc : ARM::Bcc))
8398 .addMBB(loopMBB).addImm(ARMCC::NE).addReg(ARM::CPSR);
Manman Rene8735522012-06-01 19:33:18 +00008399
8400 // loopMBB can loop back to loopMBB or fall through to exitMBB.
8401 BB->addSuccessor(loopMBB);
8402 BB->addSuccessor(exitMBB);
8403
8404 // Add epilogue to handle BytesLeft.
8405 BB = exitMBB;
Duncan P. N. Exon Smith29c52492016-07-08 20:21:17 +00008406 auto StartOfExit = exitMBB->begin();
Manman Rene8735522012-06-01 19:33:18 +00008407
8408 // [scratch, srcOut] = LDRB_POST(srcLoop, 1)
8409 // [destOut] = STRB_POST(scratch, destLoop, 1)
8410 unsigned srcIn = srcLoop;
8411 unsigned destIn = destLoop;
8412 for (unsigned i = 0; i < BytesLeft; i++) {
David Peixottob0653e532013-10-24 16:39:36 +00008413 unsigned srcOut = MRI.createVirtualRegister(TRC);
8414 unsigned destOut = MRI.createVirtualRegister(TRC);
8415 unsigned scratch = MRI.createVirtualRegister(TRC);
Manman Renb504f492013-10-29 22:27:32 +00008416 emitPostLd(BB, StartOfExit, TII, dl, 1, scratch, srcIn, srcOut,
8417 IsThumb1, IsThumb2);
8418 emitPostSt(BB, StartOfExit, TII, dl, 1, scratch, destIn, destOut,
8419 IsThumb1, IsThumb2);
David Peixottob0653e532013-10-24 16:39:36 +00008420 srcIn = srcOut;
8421 destIn = destOut;
Manman Rene8735522012-06-01 19:33:18 +00008422 }
8423
Duncan P. N. Exon Smithe4f5e4f2016-06-30 22:52:52 +00008424 MI.eraseFromParent(); // The instruction is gone now.
Manman Rene8735522012-06-01 19:33:18 +00008425 return BB;
8426}
8427
Jim Grosbach8f9a3ac2009-12-12 01:40:06 +00008428MachineBasicBlock *
Duncan P. N. Exon Smithe4f5e4f2016-06-30 22:52:52 +00008429ARMTargetLowering::EmitLowered__chkstk(MachineInstr &MI,
Saleem Abdulrasoolabac6e92014-06-09 20:18:42 +00008430 MachineBasicBlock *MBB) const {
8431 const TargetMachine &TM = getTargetMachine();
Eric Christopher1889fdc2015-01-29 00:19:39 +00008432 const TargetInstrInfo &TII = *Subtarget->getInstrInfo();
Duncan P. N. Exon Smithe4f5e4f2016-06-30 22:52:52 +00008433 DebugLoc DL = MI.getDebugLoc();
Saleem Abdulrasoolabac6e92014-06-09 20:18:42 +00008434
8435 assert(Subtarget->isTargetWindows() &&
8436 "__chkstk is only supported on Windows");
8437 assert(Subtarget->isThumb2() && "Windows on ARM requires Thumb-2 mode");
8438
8439 // __chkstk takes the number of words to allocate on the stack in R4, and
8440 // returns the stack adjustment in number of bytes in R4. This will not
8441 // clober any other registers (other than the obvious lr).
8442 //
8443 // Although, technically, IP should be considered a register which may be
8444 // clobbered, the call itself will not touch it. Windows on ARM is a pure
8445 // thumb-2 environment, so there is no interworking required. As a result, we
8446 // do not expect a veneer to be emitted by the linker, clobbering IP.
8447 //
Alp Toker1d099d92014-06-19 19:41:26 +00008448 // Each module receives its own copy of __chkstk, so no import thunk is
Saleem Abdulrasoolabac6e92014-06-09 20:18:42 +00008449 // required, again, ensuring that IP is not clobbered.
8450 //
8451 // Finally, although some linkers may theoretically provide a trampoline for
8452 // out of range calls (which is quite common due to a 32M range limitation of
8453 // branches for Thumb), we can generate the long-call version via
8454 // -mcmodel=large, alleviating the need for the trampoline which may clobber
8455 // IP.
8456
8457 switch (TM.getCodeModel()) {
8458 case CodeModel::Small:
8459 case CodeModel::Medium:
8460 case CodeModel::Default:
8461 case CodeModel::Kernel:
8462 BuildMI(*MBB, MI, DL, TII.get(ARM::tBL))
8463 .addImm((unsigned)ARMCC::AL).addReg(0)
8464 .addExternalSymbol("__chkstk")
8465 .addReg(ARM::R4, RegState::Implicit | RegState::Kill)
8466 .addReg(ARM::R4, RegState::Implicit | RegState::Define)
8467 .addReg(ARM::R12, RegState::Implicit | RegState::Define | RegState::Dead);
8468 break;
8469 case CodeModel::Large:
8470 case CodeModel::JITDefault: {
8471 MachineRegisterInfo &MRI = MBB->getParent()->getRegInfo();
8472 unsigned Reg = MRI.createVirtualRegister(&ARM::rGPRRegClass);
8473
8474 BuildMI(*MBB, MI, DL, TII.get(ARM::t2MOVi32imm), Reg)
8475 .addExternalSymbol("__chkstk");
8476 BuildMI(*MBB, MI, DL, TII.get(ARM::tBLXr))
8477 .addImm((unsigned)ARMCC::AL).addReg(0)
8478 .addReg(Reg, RegState::Kill)
8479 .addReg(ARM::R4, RegState::Implicit | RegState::Kill)
8480 .addReg(ARM::R4, RegState::Implicit | RegState::Define)
8481 .addReg(ARM::R12, RegState::Implicit | RegState::Define | RegState::Dead);
8482 break;
8483 }
8484 }
8485
8486 AddDefaultCC(AddDefaultPred(BuildMI(*MBB, MI, DL, TII.get(ARM::t2SUBrr),
8487 ARM::SP)
Saleem Abdulrasool96115182016-04-24 20:12:48 +00008488 .addReg(ARM::SP, RegState::Kill)
8489 .addReg(ARM::R4, RegState::Kill)
8490 .setMIFlags(MachineInstr::FrameSetup)));
Saleem Abdulrasoolabac6e92014-06-09 20:18:42 +00008491
Duncan P. N. Exon Smithe4f5e4f2016-06-30 22:52:52 +00008492 MI.eraseFromParent();
Saleem Abdulrasoolabac6e92014-06-09 20:18:42 +00008493 return MBB;
8494}
8495
8496MachineBasicBlock *
Duncan P. N. Exon Smithe4f5e4f2016-06-30 22:52:52 +00008497ARMTargetLowering::EmitLowered__dbzchk(MachineInstr &MI,
Saleem Abdulrasoolfe83b502015-09-25 05:15:46 +00008498 MachineBasicBlock *MBB) const {
Duncan P. N. Exon Smithe4f5e4f2016-06-30 22:52:52 +00008499 DebugLoc DL = MI.getDebugLoc();
Saleem Abdulrasoolfe83b502015-09-25 05:15:46 +00008500 MachineFunction *MF = MBB->getParent();
8501 const TargetInstrInfo *TII = Subtarget->getInstrInfo();
8502
8503 MachineBasicBlock *ContBB = MF->CreateMachineBasicBlock();
Saleem Abdulrasool750a90d2016-03-25 19:48:06 +00008504 MF->insert(++MBB->getIterator(), ContBB);
Saleem Abdulrasoolfe83b502015-09-25 05:15:46 +00008505 ContBB->splice(ContBB->begin(), MBB,
8506 std::next(MachineBasicBlock::iterator(MI)), MBB->end());
Saleem Abdulrasool071a0992016-03-17 14:10:49 +00008507 ContBB->transferSuccessorsAndUpdatePHIs(MBB);
Saleem Abdulrasoolfe83b502015-09-25 05:15:46 +00008508
8509 MachineBasicBlock *TrapBB = MF->CreateMachineBasicBlock();
8510 MF->push_back(TrapBB);
8511 BuildMI(TrapBB, DL, TII->get(ARM::t2UDF)).addImm(249);
8512 MBB->addSuccessor(TrapBB);
8513
8514 BuildMI(*MBB, MI, DL, TII->get(ARM::tCBZ))
Duncan P. N. Exon Smithe4f5e4f2016-06-30 22:52:52 +00008515 .addReg(MI.getOperand(0).getReg())
Saleem Abdulrasoolfe83b502015-09-25 05:15:46 +00008516 .addMBB(TrapBB);
Saleem Abdulrasool0dab98d2016-03-25 00:34:11 +00008517 AddDefaultPred(BuildMI(*MBB, MI, DL, TII->get(ARM::t2B)).addMBB(ContBB));
Saleem Abdulrasool071a0992016-03-17 14:10:49 +00008518 MBB->addSuccessor(ContBB);
Saleem Abdulrasoolfe83b502015-09-25 05:15:46 +00008519
Duncan P. N. Exon Smithe4f5e4f2016-06-30 22:52:52 +00008520 MI.eraseFromParent();
Saleem Abdulrasoolfe83b502015-09-25 05:15:46 +00008521 return ContBB;
8522}
8523
8524MachineBasicBlock *
Duncan P. N. Exon Smithe4f5e4f2016-06-30 22:52:52 +00008525ARMTargetLowering::EmitInstrWithCustomInserter(MachineInstr &MI,
Dan Gohman25c16532010-05-01 00:01:06 +00008526 MachineBasicBlock *BB) const {
Eric Christopher1889fdc2015-01-29 00:19:39 +00008527 const TargetInstrInfo *TII = Subtarget->getInstrInfo();
Duncan P. N. Exon Smithe4f5e4f2016-06-30 22:52:52 +00008528 DebugLoc dl = MI.getDebugLoc();
Jim Grosbach57ccc192009-12-14 20:14:59 +00008529 bool isThumb2 = Subtarget->isThumb2();
Duncan P. N. Exon Smithe4f5e4f2016-06-30 22:52:52 +00008530 switch (MI.getOpcode()) {
Andrew Trick0ed57782011-04-23 03:55:32 +00008531 default: {
Duncan P. N. Exon Smithe4f5e4f2016-06-30 22:52:52 +00008532 MI.dump();
Evan Chengb972e562009-08-07 00:34:42 +00008533 llvm_unreachable("Unexpected instr type to insert");
Andrew Trick0ed57782011-04-23 03:55:32 +00008534 }
James Molloyb3326df2016-07-15 08:03:56 +00008535
8536 // Thumb1 post-indexed loads are really just single-register LDMs.
8537 case ARM::tLDR_postidx: {
8538 BuildMI(*BB, MI, dl, TII->get(ARM::tLDMIA_UPD))
James Molloy8f16dff2016-07-15 08:12:44 +00008539 .addOperand(MI.getOperand(1)) // Rn_wb
8540 .addOperand(MI.getOperand(2)) // Rn
8541 .addOperand(MI.getOperand(3)) // PredImm
8542 .addOperand(MI.getOperand(4)) // PredReg
8543 .addOperand(MI.getOperand(0)); // Rt
8544 MI.eraseFromParent();
James Molloyb3326df2016-07-15 08:03:56 +00008545 return BB;
8546 }
8547
Jim Grosbach9c0b86a2011-09-16 21:55:56 +00008548 // The Thumb2 pre-indexed stores have the same MI operands, they just
8549 // define them differently in the .td files from the isel patterns, so
8550 // they need pseudos.
8551 case ARM::t2STR_preidx:
Duncan P. N. Exon Smithe4f5e4f2016-06-30 22:52:52 +00008552 MI.setDesc(TII->get(ARM::t2STR_PRE));
Jim Grosbach9c0b86a2011-09-16 21:55:56 +00008553 return BB;
8554 case ARM::t2STRB_preidx:
Duncan P. N. Exon Smithe4f5e4f2016-06-30 22:52:52 +00008555 MI.setDesc(TII->get(ARM::t2STRB_PRE));
Jim Grosbach9c0b86a2011-09-16 21:55:56 +00008556 return BB;
8557 case ARM::t2STRH_preidx:
Duncan P. N. Exon Smithe4f5e4f2016-06-30 22:52:52 +00008558 MI.setDesc(TII->get(ARM::t2STRH_PRE));
Jim Grosbach9c0b86a2011-09-16 21:55:56 +00008559 return BB;
8560
Jim Grosbachf0c95ca2011-08-05 20:35:44 +00008561 case ARM::STRi_preidx:
8562 case ARM::STRBi_preidx: {
Duncan P. N. Exon Smithe4f5e4f2016-06-30 22:52:52 +00008563 unsigned NewOpc = MI.getOpcode() == ARM::STRi_preidx ? ARM::STR_PRE_IMM
8564 : ARM::STRB_PRE_IMM;
Jim Grosbachf0c95ca2011-08-05 20:35:44 +00008565 // Decode the offset.
Duncan P. N. Exon Smithe4f5e4f2016-06-30 22:52:52 +00008566 unsigned Offset = MI.getOperand(4).getImm();
Jim Grosbachf0c95ca2011-08-05 20:35:44 +00008567 bool isSub = ARM_AM::getAM2Op(Offset) == ARM_AM::sub;
8568 Offset = ARM_AM::getAM2Offset(Offset);
8569 if (isSub)
8570 Offset = -Offset;
8571
Duncan P. N. Exon Smithe4f5e4f2016-06-30 22:52:52 +00008572 MachineMemOperand *MMO = *MI.memoperands_begin();
Benjamin Kramer61a1ff52011-08-27 17:36:14 +00008573 BuildMI(*BB, MI, dl, TII->get(NewOpc))
Duncan P. N. Exon Smithe4f5e4f2016-06-30 22:52:52 +00008574 .addOperand(MI.getOperand(0)) // Rn_wb
8575 .addOperand(MI.getOperand(1)) // Rt
8576 .addOperand(MI.getOperand(2)) // Rn
8577 .addImm(Offset) // offset (skip GPR==zero_reg)
8578 .addOperand(MI.getOperand(5)) // pred
8579 .addOperand(MI.getOperand(6))
8580 .addMemOperand(MMO);
8581 MI.eraseFromParent();
Jim Grosbachf0c95ca2011-08-05 20:35:44 +00008582 return BB;
8583 }
8584 case ARM::STRr_preidx:
Jim Grosbachd886f8c2011-08-11 21:17:22 +00008585 case ARM::STRBr_preidx:
8586 case ARM::STRH_preidx: {
8587 unsigned NewOpc;
Duncan P. N. Exon Smithe4f5e4f2016-06-30 22:52:52 +00008588 switch (MI.getOpcode()) {
Jim Grosbachd886f8c2011-08-11 21:17:22 +00008589 default: llvm_unreachable("unexpected opcode!");
8590 case ARM::STRr_preidx: NewOpc = ARM::STR_PRE_REG; break;
8591 case ARM::STRBr_preidx: NewOpc = ARM::STRB_PRE_REG; break;
8592 case ARM::STRH_preidx: NewOpc = ARM::STRH_PRE; break;
8593 }
Jim Grosbachf0c95ca2011-08-05 20:35:44 +00008594 MachineInstrBuilder MIB = BuildMI(*BB, MI, dl, TII->get(NewOpc));
Duncan P. N. Exon Smithe4f5e4f2016-06-30 22:52:52 +00008595 for (unsigned i = 0; i < MI.getNumOperands(); ++i)
8596 MIB.addOperand(MI.getOperand(i));
8597 MI.eraseFromParent();
Jim Grosbachf0c95ca2011-08-05 20:35:44 +00008598 return BB;
8599 }
Eli Friedmanc3f9c4a2011-08-31 00:31:29 +00008600
Evan Chengbb2af352009-08-12 05:17:19 +00008601 case ARM::tMOVCCr_pseudo: {
Evan Cheng10043e22007-01-19 07:51:42 +00008602 // To "insert" a SELECT_CC instruction, we actually have to insert the
8603 // diamond control-flow pattern. The incoming instruction knows the
8604 // destination vreg to set, the condition code register to branch on, the
8605 // true/false values to select between, and a branch opcode to use.
8606 const BasicBlock *LLVM_BB = BB->getBasicBlock();
Duncan P. N. Exon Smith9f9559e2015-10-19 23:25:57 +00008607 MachineFunction::iterator It = ++BB->getIterator();
Evan Cheng10043e22007-01-19 07:51:42 +00008608
8609 // thisMBB:
8610 // ...
8611 // TrueVal = ...
8612 // cmpTY ccX, r1, r2
8613 // bCC copy1MBB
8614 // fallthrough --> copy0MBB
8615 MachineBasicBlock *thisMBB = BB;
Dan Gohman3b460302008-07-07 23:14:23 +00008616 MachineFunction *F = BB->getParent();
8617 MachineBasicBlock *copy0MBB = F->CreateMachineBasicBlock(LLVM_BB);
8618 MachineBasicBlock *sinkMBB = F->CreateMachineBasicBlock(LLVM_BB);
Dan Gohmanf4f04102010-07-06 15:49:48 +00008619 F->insert(It, copy0MBB);
8620 F->insert(It, sinkMBB);
Dan Gohman34396292010-07-06 20:24:04 +00008621
8622 // Transfer the remainder of BB and its successor edges to sinkMBB.
8623 sinkMBB->splice(sinkMBB->begin(), BB,
Benjamin Kramerb6d0bd42014-03-02 12:27:27 +00008624 std::next(MachineBasicBlock::iterator(MI)), BB->end());
Dan Gohman34396292010-07-06 20:24:04 +00008625 sinkMBB->transferSuccessorsAndUpdatePHIs(BB);
8626
Dan Gohmanf4f04102010-07-06 15:49:48 +00008627 BB->addSuccessor(copy0MBB);
8628 BB->addSuccessor(sinkMBB);
Dan Gohman12205642010-07-06 15:18:19 +00008629
Duncan P. N. Exon Smithe4f5e4f2016-06-30 22:52:52 +00008630 BuildMI(BB, dl, TII->get(ARM::tBcc))
8631 .addMBB(sinkMBB)
8632 .addImm(MI.getOperand(3).getImm())
8633 .addReg(MI.getOperand(4).getReg());
Dan Gohman34396292010-07-06 20:24:04 +00008634
Evan Cheng10043e22007-01-19 07:51:42 +00008635 // copy0MBB:
8636 // %FalseValue = ...
8637 // # fallthrough to sinkMBB
8638 BB = copy0MBB;
8639
8640 // Update machine-CFG edges
8641 BB->addSuccessor(sinkMBB);
8642
8643 // sinkMBB:
8644 // %Result = phi [ %FalseValue, copy0MBB ], [ %TrueValue, thisMBB ]
8645 // ...
8646 BB = sinkMBB;
Duncan P. N. Exon Smithe4f5e4f2016-06-30 22:52:52 +00008647 BuildMI(*BB, BB->begin(), dl, TII->get(ARM::PHI), MI.getOperand(0).getReg())
8648 .addReg(MI.getOperand(1).getReg())
8649 .addMBB(copy0MBB)
8650 .addReg(MI.getOperand(2).getReg())
8651 .addMBB(thisMBB);
Evan Cheng10043e22007-01-19 07:51:42 +00008652
Duncan P. N. Exon Smithe4f5e4f2016-06-30 22:52:52 +00008653 MI.eraseFromParent(); // The pseudo instruction is gone now.
Evan Cheng10043e22007-01-19 07:51:42 +00008654 return BB;
8655 }
Evan Chengb972e562009-08-07 00:34:42 +00008656
Evan Cheng0cc4ad92010-07-13 19:27:42 +00008657 case ARM::BCCi64:
8658 case ARM::BCCZi64: {
Bob Wilson36be00c2010-12-23 22:45:49 +00008659 // If there is an unconditional branch to the other successor, remove it.
Benjamin Kramerb6d0bd42014-03-02 12:27:27 +00008660 BB->erase(std::next(MachineBasicBlock::iterator(MI)), BB->end());
Andrew Trick5eb0a302011-01-19 02:26:13 +00008661
Evan Cheng0cc4ad92010-07-13 19:27:42 +00008662 // Compare both parts that make up the double comparison separately for
8663 // equality.
Duncan P. N. Exon Smithe4f5e4f2016-06-30 22:52:52 +00008664 bool RHSisZero = MI.getOpcode() == ARM::BCCZi64;
Evan Cheng0cc4ad92010-07-13 19:27:42 +00008665
Duncan P. N. Exon Smithe4f5e4f2016-06-30 22:52:52 +00008666 unsigned LHS1 = MI.getOperand(1).getReg();
8667 unsigned LHS2 = MI.getOperand(2).getReg();
Evan Cheng0cc4ad92010-07-13 19:27:42 +00008668 if (RHSisZero) {
8669 AddDefaultPred(BuildMI(BB, dl,
8670 TII->get(isThumb2 ? ARM::t2CMPri : ARM::CMPri))
8671 .addReg(LHS1).addImm(0));
8672 BuildMI(BB, dl, TII->get(isThumb2 ? ARM::t2CMPri : ARM::CMPri))
8673 .addReg(LHS2).addImm(0)
8674 .addImm(ARMCC::EQ).addReg(ARM::CPSR);
8675 } else {
Duncan P. N. Exon Smithe4f5e4f2016-06-30 22:52:52 +00008676 unsigned RHS1 = MI.getOperand(3).getReg();
8677 unsigned RHS2 = MI.getOperand(4).getReg();
Evan Cheng0cc4ad92010-07-13 19:27:42 +00008678 AddDefaultPred(BuildMI(BB, dl,
8679 TII->get(isThumb2 ? ARM::t2CMPrr : ARM::CMPrr))
8680 .addReg(LHS1).addReg(RHS1));
8681 BuildMI(BB, dl, TII->get(isThumb2 ? ARM::t2CMPrr : ARM::CMPrr))
8682 .addReg(LHS2).addReg(RHS2)
8683 .addImm(ARMCC::EQ).addReg(ARM::CPSR);
8684 }
8685
Duncan P. N. Exon Smithe4f5e4f2016-06-30 22:52:52 +00008686 MachineBasicBlock *destMBB = MI.getOperand(RHSisZero ? 3 : 5).getMBB();
Evan Cheng0cc4ad92010-07-13 19:27:42 +00008687 MachineBasicBlock *exitMBB = OtherSucc(BB, destMBB);
Duncan P. N. Exon Smithe4f5e4f2016-06-30 22:52:52 +00008688 if (MI.getOperand(0).getImm() == ARMCC::NE)
Evan Cheng0cc4ad92010-07-13 19:27:42 +00008689 std::swap(destMBB, exitMBB);
8690
8691 BuildMI(BB, dl, TII->get(isThumb2 ? ARM::t2Bcc : ARM::Bcc))
8692 .addMBB(destMBB).addImm(ARMCC::EQ).addReg(ARM::CPSR);
Owen Anderson29cfe6c2011-09-09 21:48:23 +00008693 if (isThumb2)
8694 AddDefaultPred(BuildMI(BB, dl, TII->get(ARM::t2B)).addMBB(exitMBB));
8695 else
8696 BuildMI(BB, dl, TII->get(ARM::B)) .addMBB(exitMBB);
Evan Cheng0cc4ad92010-07-13 19:27:42 +00008697
Duncan P. N. Exon Smithe4f5e4f2016-06-30 22:52:52 +00008698 MI.eraseFromParent(); // The pseudo instruction is gone now.
Evan Cheng0cc4ad92010-07-13 19:27:42 +00008699 return BB;
8700 }
Bill Wendlinga7d697e2011-10-10 22:59:55 +00008701
Bill Wendlingf7f223f2011-10-17 20:37:20 +00008702 case ARM::Int_eh_sjlj_setjmp:
8703 case ARM::Int_eh_sjlj_setjmp_nofp:
8704 case ARM::tInt_eh_sjlj_setjmp:
8705 case ARM::t2Int_eh_sjlj_setjmp:
8706 case ARM::t2Int_eh_sjlj_setjmp_nofp:
Matthias Braun3cd00c12015-07-16 22:34:16 +00008707 return BB;
8708
8709 case ARM::Int_eh_sjlj_setup_dispatch:
Bill Wendlingf7f223f2011-10-17 20:37:20 +00008710 EmitSjLjDispatchBlock(MI, BB);
8711 return BB;
8712
Bill Wendlinga7d697e2011-10-10 22:59:55 +00008713 case ARM::ABS:
8714 case ARM::t2ABS: {
8715 // To insert an ABS instruction, we have to insert the
8716 // diamond control-flow pattern. The incoming instruction knows the
8717 // source vreg to test against 0, the destination vreg to set,
8718 // the condition code register to branch on, the
Andrew Trick3f07c422011-10-18 18:40:53 +00008719 // true/false values to select between, and a branch opcode to use.
Bill Wendlinga7d697e2011-10-10 22:59:55 +00008720 // It transforms
8721 // V1 = ABS V0
8722 // into
8723 // V2 = MOVS V0
8724 // BCC (branch to SinkBB if V0 >= 0)
8725 // RSBBB: V3 = RSBri V2, 0 (compute ABS if V2 < 0)
Andrew Trick3f07c422011-10-18 18:40:53 +00008726 // SinkBB: V1 = PHI(V2, V3)
Bill Wendlinga7d697e2011-10-10 22:59:55 +00008727 const BasicBlock *LLVM_BB = BB->getBasicBlock();
Duncan P. N. Exon Smith9f9559e2015-10-19 23:25:57 +00008728 MachineFunction::iterator BBI = ++BB->getIterator();
Bill Wendlinga7d697e2011-10-10 22:59:55 +00008729 MachineFunction *Fn = BB->getParent();
8730 MachineBasicBlock *RSBBB = Fn->CreateMachineBasicBlock(LLVM_BB);
8731 MachineBasicBlock *SinkBB = Fn->CreateMachineBasicBlock(LLVM_BB);
8732 Fn->insert(BBI, RSBBB);
8733 Fn->insert(BBI, SinkBB);
8734
Duncan P. N. Exon Smithe4f5e4f2016-06-30 22:52:52 +00008735 unsigned int ABSSrcReg = MI.getOperand(1).getReg();
8736 unsigned int ABSDstReg = MI.getOperand(0).getReg();
8737 bool ABSSrcKIll = MI.getOperand(1).isKill();
Bill Wendlinga7d697e2011-10-10 22:59:55 +00008738 bool isThumb2 = Subtarget->isThumb2();
8739 MachineRegisterInfo &MRI = Fn->getRegInfo();
8740 // In Thumb mode S must not be specified if source register is the SP or
8741 // PC and if destination register is the SP, so restrict register class
Craig Topper61e88f42014-11-21 05:58:21 +00008742 unsigned NewRsbDstReg =
8743 MRI.createVirtualRegister(isThumb2 ? &ARM::rGPRRegClass : &ARM::GPRRegClass);
Bill Wendlinga7d697e2011-10-10 22:59:55 +00008744
8745 // Transfer the remainder of BB and its successor edges to sinkMBB.
8746 SinkBB->splice(SinkBB->begin(), BB,
Benjamin Kramerb6d0bd42014-03-02 12:27:27 +00008747 std::next(MachineBasicBlock::iterator(MI)), BB->end());
Bill Wendlinga7d697e2011-10-10 22:59:55 +00008748 SinkBB->transferSuccessorsAndUpdatePHIs(BB);
8749
8750 BB->addSuccessor(RSBBB);
8751 BB->addSuccessor(SinkBB);
8752
8753 // fall through to SinkMBB
8754 RSBBB->addSuccessor(SinkBB);
8755
Manman Rene0763c72012-06-15 21:32:12 +00008756 // insert a cmp at the end of BB
Andrew Trickbc325162012-07-18 18:34:24 +00008757 AddDefaultPred(BuildMI(BB, dl,
Manman Rene0763c72012-06-15 21:32:12 +00008758 TII->get(isThumb2 ? ARM::t2CMPri : ARM::CMPri))
8759 .addReg(ABSSrcReg).addImm(0));
Bill Wendlinga7d697e2011-10-10 22:59:55 +00008760
8761 // insert a bcc with opposite CC to ARMCC::MI at the end of BB
Andrew Trick3f07c422011-10-18 18:40:53 +00008762 BuildMI(BB, dl,
Bill Wendlinga7d697e2011-10-10 22:59:55 +00008763 TII->get(isThumb2 ? ARM::t2Bcc : ARM::Bcc)).addMBB(SinkBB)
8764 .addImm(ARMCC::getOppositeCondition(ARMCC::MI)).addReg(ARM::CPSR);
8765
8766 // insert rsbri in RSBBB
8767 // Note: BCC and rsbri will be converted into predicated rsbmi
8768 // by if-conversion pass
Andrew Trick3f07c422011-10-18 18:40:53 +00008769 BuildMI(*RSBBB, RSBBB->begin(), dl,
Bill Wendlinga7d697e2011-10-10 22:59:55 +00008770 TII->get(isThumb2 ? ARM::t2RSBri : ARM::RSBri), NewRsbDstReg)
Pete Cooper51118812015-04-30 22:15:59 +00008771 .addReg(ABSSrcReg, ABSSrcKIll ? RegState::Kill : 0)
Bill Wendlinga7d697e2011-10-10 22:59:55 +00008772 .addImm(0).addImm((unsigned)ARMCC::AL).addReg(0).addReg(0);
8773
Andrew Trick3f07c422011-10-18 18:40:53 +00008774 // insert PHI in SinkBB,
Bill Wendlinga7d697e2011-10-10 22:59:55 +00008775 // reuse ABSDstReg to not change uses of ABS instruction
8776 BuildMI(*SinkBB, SinkBB->begin(), dl,
8777 TII->get(ARM::PHI), ABSDstReg)
8778 .addReg(NewRsbDstReg).addMBB(RSBBB)
Manman Rene0763c72012-06-15 21:32:12 +00008779 .addReg(ABSSrcReg).addMBB(BB);
Bill Wendlinga7d697e2011-10-10 22:59:55 +00008780
8781 // remove ABS instruction
Duncan P. N. Exon Smithe4f5e4f2016-06-30 22:52:52 +00008782 MI.eraseFromParent();
Bill Wendlinga7d697e2011-10-10 22:59:55 +00008783
8784 // return last added BB
8785 return SinkBB;
8786 }
Manman Rene8735522012-06-01 19:33:18 +00008787 case ARM::COPY_STRUCT_BYVAL_I32:
Manman Ren9f911162012-06-01 02:44:42 +00008788 ++NumLoopByVals;
Manman Rene8735522012-06-01 19:33:18 +00008789 return EmitStructByval(MI, BB);
Saleem Abdulrasoolabac6e92014-06-09 20:18:42 +00008790 case ARM::WIN__CHKSTK:
8791 return EmitLowered__chkstk(MI, BB);
Saleem Abdulrasoolfe83b502015-09-25 05:15:46 +00008792 case ARM::WIN__DBZCHK:
8793 return EmitLowered__dbzchk(MI, BB);
Evan Cheng10043e22007-01-19 07:51:42 +00008794 }
8795}
8796
Scott Douglass953f9082015-10-05 14:49:54 +00008797/// \brief Attaches vregs to MEMCPY that it will use as scratch registers
8798/// when it is expanded into LDM/STM. This is done as a post-isel lowering
8799/// instead of as a custom inserter because we need the use list from the SDNode.
8800static void attachMEMCPYScratchRegs(const ARMSubtarget *Subtarget,
Duncan P. N. Exon Smithe4f5e4f2016-06-30 22:52:52 +00008801 MachineInstr &MI, const SDNode *Node) {
Scott Douglass953f9082015-10-05 14:49:54 +00008802 bool isThumb1 = Subtarget->isThumb1Only();
8803
Duncan P. N. Exon Smithe4f5e4f2016-06-30 22:52:52 +00008804 DebugLoc DL = MI.getDebugLoc();
8805 MachineFunction *MF = MI.getParent()->getParent();
Scott Douglass953f9082015-10-05 14:49:54 +00008806 MachineRegisterInfo &MRI = MF->getRegInfo();
8807 MachineInstrBuilder MIB(*MF, MI);
8808
8809 // If the new dst/src is unused mark it as dead.
8810 if (!Node->hasAnyUseOfValue(0)) {
Duncan P. N. Exon Smithe4f5e4f2016-06-30 22:52:52 +00008811 MI.getOperand(0).setIsDead(true);
Scott Douglass953f9082015-10-05 14:49:54 +00008812 }
8813 if (!Node->hasAnyUseOfValue(1)) {
Duncan P. N. Exon Smithe4f5e4f2016-06-30 22:52:52 +00008814 MI.getOperand(1).setIsDead(true);
Scott Douglass953f9082015-10-05 14:49:54 +00008815 }
8816
8817 // The MEMCPY both defines and kills the scratch registers.
Duncan P. N. Exon Smithe4f5e4f2016-06-30 22:52:52 +00008818 for (unsigned I = 0; I != MI.getOperand(4).getImm(); ++I) {
Scott Douglass953f9082015-10-05 14:49:54 +00008819 unsigned TmpReg = MRI.createVirtualRegister(isThumb1 ? &ARM::tGPRRegClass
8820 : &ARM::GPRRegClass);
8821 MIB.addReg(TmpReg, RegState::Define|RegState::Dead);
8822 }
8823}
8824
Duncan P. N. Exon Smithe4f5e4f2016-06-30 22:52:52 +00008825void ARMTargetLowering::AdjustInstrPostInstrSelection(MachineInstr &MI,
Evan Chenge6fba772011-08-30 19:09:48 +00008826 SDNode *Node) const {
Duncan P. N. Exon Smithe4f5e4f2016-06-30 22:52:52 +00008827 if (MI.getOpcode() == ARM::MEMCPY) {
Scott Douglass953f9082015-10-05 14:49:54 +00008828 attachMEMCPYScratchRegs(Subtarget, MI, Node);
8829 return;
8830 }
8831
Duncan P. N. Exon Smithe4f5e4f2016-06-30 22:52:52 +00008832 const MCInstrDesc *MCID = &MI.getDesc();
Andrew Trick8586e622011-09-20 03:17:40 +00008833 // Adjust potentially 's' setting instructions after isel, i.e. ADC, SBC, RSB,
8834 // RSC. Coming out of isel, they have an implicit CPSR def, but the optional
8835 // operand is still set to noreg. If needed, set the optional operand's
8836 // register to CPSR, and remove the redundant implicit def.
Andrew Trick924123a2011-09-21 02:20:46 +00008837 //
Andrew Trick88b24502011-10-18 19:18:52 +00008838 // e.g. ADCS (..., CPSR<imp-def>) -> ADC (... opt:CPSR<def>).
Andrew Trick8586e622011-09-20 03:17:40 +00008839
Andrew Trick924123a2011-09-21 02:20:46 +00008840 // Rename pseudo opcodes.
Duncan P. N. Exon Smithe4f5e4f2016-06-30 22:52:52 +00008841 unsigned NewOpc = convertAddSubFlagsOpcode(MI.getOpcode());
Andrew Trick924123a2011-09-21 02:20:46 +00008842 if (NewOpc) {
Eric Christopher1889fdc2015-01-29 00:19:39 +00008843 const ARMBaseInstrInfo *TII = Subtarget->getInstrInfo();
Andrew Trick88b24502011-10-18 19:18:52 +00008844 MCID = &TII->get(NewOpc);
8845
Duncan P. N. Exon Smithe4f5e4f2016-06-30 22:52:52 +00008846 assert(MCID->getNumOperands() == MI.getDesc().getNumOperands() + 1 &&
Andrew Trick88b24502011-10-18 19:18:52 +00008847 "converted opcode should be the same except for cc_out");
8848
Duncan P. N. Exon Smithe4f5e4f2016-06-30 22:52:52 +00008849 MI.setDesc(*MCID);
Andrew Trick88b24502011-10-18 19:18:52 +00008850
8851 // Add the optional cc_out operand
Duncan P. N. Exon Smithe4f5e4f2016-06-30 22:52:52 +00008852 MI.addOperand(MachineOperand::CreateReg(0, /*isDef=*/true));
Andrew Trick924123a2011-09-21 02:20:46 +00008853 }
Andrew Trick88b24502011-10-18 19:18:52 +00008854 unsigned ccOutIdx = MCID->getNumOperands() - 1;
Andrew Trick8586e622011-09-20 03:17:40 +00008855
8856 // Any ARM instruction that sets the 's' bit should specify an optional
8857 // "cc_out" operand in the last operand position.
Duncan P. N. Exon Smithe4f5e4f2016-06-30 22:52:52 +00008858 if (!MI.hasOptionalDef() || !MCID->OpInfo[ccOutIdx].isOptionalDef()) {
Andrew Trick924123a2011-09-21 02:20:46 +00008859 assert(!NewOpc && "Optional cc_out operand required");
Andrew Trick8586e622011-09-20 03:17:40 +00008860 return;
8861 }
Andrew Trick924123a2011-09-21 02:20:46 +00008862 // Look for an implicit def of CPSR added by MachineInstr ctor. Remove it
8863 // since we already have an optional CPSR def.
Andrew Trick8586e622011-09-20 03:17:40 +00008864 bool definesCPSR = false;
8865 bool deadCPSR = false;
Duncan P. N. Exon Smithe4f5e4f2016-06-30 22:52:52 +00008866 for (unsigned i = MCID->getNumOperands(), e = MI.getNumOperands(); i != e;
8867 ++i) {
8868 const MachineOperand &MO = MI.getOperand(i);
Andrew Trick8586e622011-09-20 03:17:40 +00008869 if (MO.isReg() && MO.isDef() && MO.getReg() == ARM::CPSR) {
8870 definesCPSR = true;
8871 if (MO.isDead())
8872 deadCPSR = true;
Duncan P. N. Exon Smithe4f5e4f2016-06-30 22:52:52 +00008873 MI.RemoveOperand(i);
Andrew Trick8586e622011-09-20 03:17:40 +00008874 break;
Evan Chenge6fba772011-08-30 19:09:48 +00008875 }
8876 }
Andrew Trick8586e622011-09-20 03:17:40 +00008877 if (!definesCPSR) {
Andrew Trick924123a2011-09-21 02:20:46 +00008878 assert(!NewOpc && "Optional cc_out operand required");
Andrew Trick8586e622011-09-20 03:17:40 +00008879 return;
8880 }
8881 assert(deadCPSR == !Node->hasAnyUseOfValue(1) && "inconsistent dead flag");
Andrew Trick924123a2011-09-21 02:20:46 +00008882 if (deadCPSR) {
Duncan P. N. Exon Smithe4f5e4f2016-06-30 22:52:52 +00008883 assert(!MI.getOperand(ccOutIdx).getReg() &&
Andrew Trick924123a2011-09-21 02:20:46 +00008884 "expect uninitialized optional cc_out operand");
Andrew Trick8586e622011-09-20 03:17:40 +00008885 return;
Andrew Trick924123a2011-09-21 02:20:46 +00008886 }
Andrew Trick8586e622011-09-20 03:17:40 +00008887
Andrew Trick924123a2011-09-21 02:20:46 +00008888 // If this instruction was defined with an optional CPSR def and its dag node
8889 // had a live implicit CPSR def, then activate the optional CPSR def.
Duncan P. N. Exon Smithe4f5e4f2016-06-30 22:52:52 +00008890 MachineOperand &MO = MI.getOperand(ccOutIdx);
Andrew Trick8586e622011-09-20 03:17:40 +00008891 MO.setReg(ARM::CPSR);
8892 MO.setIsDef(true);
Evan Chenge6fba772011-08-30 19:09:48 +00008893}
8894
Evan Cheng10043e22007-01-19 07:51:42 +00008895//===----------------------------------------------------------------------===//
8896// ARM Optimization Hooks
8897//===----------------------------------------------------------------------===//
8898
Jakob Stoklund Olesenc1dee482012-08-17 16:59:09 +00008899// Helper function that checks if N is a null or all ones constant.
8900static inline bool isZeroOrAllOnes(SDValue N, bool AllOnes) {
Artyom Skrobov314ee042015-11-25 19:41:11 +00008901 return AllOnes ? isAllOnesConstant(N) : isNullConstant(N);
Jakob Stoklund Olesenc1dee482012-08-17 16:59:09 +00008902}
8903
Jakob Stoklund Olesendded0612012-08-18 21:25:22 +00008904// Return true if N is conditionally 0 or all ones.
8905// Detects these expressions where cc is an i1 value:
8906//
8907// (select cc 0, y) [AllOnes=0]
8908// (select cc y, 0) [AllOnes=0]
8909// (zext cc) [AllOnes=0]
8910// (sext cc) [AllOnes=0/1]
8911// (select cc -1, y) [AllOnes=1]
8912// (select cc y, -1) [AllOnes=1]
8913//
8914// Invert is set when N is the null/all ones constant when CC is false.
8915// OtherOp is set to the alternative value of N.
8916static bool isConditionalZeroOrAllOnes(SDNode *N, bool AllOnes,
8917 SDValue &CC, bool &Invert,
8918 SDValue &OtherOp,
8919 SelectionDAG &DAG) {
8920 switch (N->getOpcode()) {
8921 default: return false;
8922 case ISD::SELECT: {
8923 CC = N->getOperand(0);
8924 SDValue N1 = N->getOperand(1);
8925 SDValue N2 = N->getOperand(2);
8926 if (isZeroOrAllOnes(N1, AllOnes)) {
8927 Invert = false;
8928 OtherOp = N2;
8929 return true;
8930 }
8931 if (isZeroOrAllOnes(N2, AllOnes)) {
8932 Invert = true;
8933 OtherOp = N1;
8934 return true;
8935 }
8936 return false;
8937 }
8938 case ISD::ZERO_EXTEND:
8939 // (zext cc) can never be the all ones value.
8940 if (AllOnes)
8941 return false;
Justin Bognercd1d5aa2016-08-17 20:30:52 +00008942 LLVM_FALLTHROUGH;
Jakob Stoklund Olesendded0612012-08-18 21:25:22 +00008943 case ISD::SIGN_EXTEND: {
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +00008944 SDLoc dl(N);
Jakob Stoklund Olesendded0612012-08-18 21:25:22 +00008945 EVT VT = N->getValueType(0);
8946 CC = N->getOperand(0);
8947 if (CC.getValueType() != MVT::i1)
8948 return false;
8949 Invert = !AllOnes;
8950 if (AllOnes)
8951 // When looking for an AllOnes constant, N is an sext, and the 'other'
8952 // value is 0.
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +00008953 OtherOp = DAG.getConstant(0, dl, VT);
Jakob Stoklund Olesendded0612012-08-18 21:25:22 +00008954 else if (N->getOpcode() == ISD::ZERO_EXTEND)
8955 // When looking for a 0 constant, N can be zext or sext.
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +00008956 OtherOp = DAG.getConstant(1, dl, VT);
Jakob Stoklund Olesendded0612012-08-18 21:25:22 +00008957 else
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +00008958 OtherOp = DAG.getConstant(APInt::getAllOnesValue(VT.getSizeInBits()), dl,
8959 VT);
Jakob Stoklund Olesendded0612012-08-18 21:25:22 +00008960 return true;
8961 }
8962 }
8963}
8964
Jakob Stoklund Olesenc1dee482012-08-17 16:59:09 +00008965// Combine a constant select operand into its use:
8966//
Jakob Stoklund Olesenaab43db2012-08-18 21:25:16 +00008967// (add (select cc, 0, c), x) -> (select cc, x, (add, x, c))
8968// (sub x, (select cc, 0, c)) -> (select cc, x, (sub, x, c))
8969// (and (select cc, -1, c), x) -> (select cc, x, (and, x, c)) [AllOnes=1]
8970// (or (select cc, 0, c), x) -> (select cc, x, (or, x, c))
8971// (xor (select cc, 0, c), x) -> (select cc, x, (xor, x, c))
Jakob Stoklund Olesenc1dee482012-08-17 16:59:09 +00008972//
8973// The transform is rejected if the select doesn't have a constant operand that
Jakob Stoklund Olesenaab43db2012-08-18 21:25:16 +00008974// is null, or all ones when AllOnes is set.
Jakob Stoklund Olesenc1dee482012-08-17 16:59:09 +00008975//
Jakob Stoklund Olesendded0612012-08-18 21:25:22 +00008976// Also recognize sext/zext from i1:
8977//
8978// (add (zext cc), x) -> (select cc (add x, 1), x)
8979// (add (sext cc), x) -> (select cc (add x, -1), x)
8980//
8981// These transformations eventually create predicated instructions.
8982//
Jakob Stoklund Olesenc1dee482012-08-17 16:59:09 +00008983// @param N The node to transform.
8984// @param Slct The N operand that is a select.
8985// @param OtherOp The other N operand (x above).
8986// @param DCI Context.
Jakob Stoklund Olesenaab43db2012-08-18 21:25:16 +00008987// @param AllOnes Require the select constant to be all ones instead of null.
Jakob Stoklund Olesenc1dee482012-08-17 16:59:09 +00008988// @returns The new node, or SDValue() on failure.
Chris Lattner4147f082009-03-12 06:52:53 +00008989static
8990SDValue combineSelectAndUse(SDNode *N, SDValue Slct, SDValue OtherOp,
Jakob Stoklund Olesenaab43db2012-08-18 21:25:16 +00008991 TargetLowering::DAGCombinerInfo &DCI,
8992 bool AllOnes = false) {
Chris Lattner4147f082009-03-12 06:52:53 +00008993 SelectionDAG &DAG = DCI.DAG;
Owen Anderson53aa7a92009-08-10 22:56:29 +00008994 EVT VT = N->getValueType(0);
Jakob Stoklund Olesendded0612012-08-18 21:25:22 +00008995 SDValue NonConstantVal;
8996 SDValue CCOp;
8997 bool SwapSelectOps;
8998 if (!isConditionalZeroOrAllOnes(Slct.getNode(), AllOnes, CCOp, SwapSelectOps,
8999 NonConstantVal, DAG))
Jakob Stoklund Olesenc1dee482012-08-17 16:59:09 +00009000 return SDValue();
9001
Jakob Stoklund Olesendded0612012-08-18 21:25:22 +00009002 // Slct is now know to be the desired identity constant when CC is true.
9003 SDValue TrueVal = OtherOp;
Andrew Trickef9de2a2013-05-25 02:42:55 +00009004 SDValue FalseVal = DAG.getNode(N->getOpcode(), SDLoc(N), VT,
Jakob Stoklund Olesendded0612012-08-18 21:25:22 +00009005 OtherOp, NonConstantVal);
9006 // Unless SwapSelectOps says CC should be false.
9007 if (SwapSelectOps)
9008 std::swap(TrueVal, FalseVal);
9009
Andrew Trickef9de2a2013-05-25 02:42:55 +00009010 return DAG.getNode(ISD::SELECT, SDLoc(N), VT,
Jakob Stoklund Olesendded0612012-08-18 21:25:22 +00009011 CCOp, TrueVal, FalseVal);
Chris Lattner4147f082009-03-12 06:52:53 +00009012}
9013
Jakob Stoklund Olesenaab43db2012-08-18 21:25:16 +00009014// Attempt combineSelectAndUse on each operand of a commutative operator N.
9015static
9016SDValue combineSelectAndUseCommutative(SDNode *N, bool AllOnes,
9017 TargetLowering::DAGCombinerInfo &DCI) {
9018 SDValue N0 = N->getOperand(0);
9019 SDValue N1 = N->getOperand(1);
Ahmed Bougachaf8dfb472016-02-09 22:54:12 +00009020 if (N0.getNode()->hasOneUse())
9021 if (SDValue Result = combineSelectAndUse(N, N0, N1, DCI, AllOnes))
Jakob Stoklund Olesenaab43db2012-08-18 21:25:16 +00009022 return Result;
Ahmed Bougachaf8dfb472016-02-09 22:54:12 +00009023 if (N1.getNode()->hasOneUse())
9024 if (SDValue Result = combineSelectAndUse(N, N1, N0, DCI, AllOnes))
Jakob Stoklund Olesenaab43db2012-08-18 21:25:16 +00009025 return Result;
Jakob Stoklund Olesenaab43db2012-08-18 21:25:16 +00009026 return SDValue();
9027}
9028
Eric Christopher1b8b94192011-06-29 21:10:36 +00009029// AddCombineToVPADDL- For pair-wise add on neon, use the vpaddl instruction
Tanya Lattnere9e67052011-06-14 23:48:48 +00009030// (only after legalization).
9031static SDValue AddCombineToVPADDL(SDNode *N, SDValue N0, SDValue N1,
9032 TargetLowering::DAGCombinerInfo &DCI,
9033 const ARMSubtarget *Subtarget) {
9034
9035 // Only perform optimization if after legalize, and if NEON is available. We
9036 // also expected both operands to be BUILD_VECTORs.
9037 if (DCI.isBeforeLegalize() || !Subtarget->hasNEON()
9038 || N0.getOpcode() != ISD::BUILD_VECTOR
9039 || N1.getOpcode() != ISD::BUILD_VECTOR)
9040 return SDValue();
9041
9042 // Check output type since VPADDL operand elements can only be 8, 16, or 32.
9043 EVT VT = N->getValueType(0);
9044 if (!VT.isInteger() || VT.getVectorElementType() == MVT::i64)
9045 return SDValue();
9046
9047 // Check that the vector operands are of the right form.
9048 // N0 and N1 are BUILD_VECTOR nodes with N number of EXTRACT_VECTOR
9049 // operands, where N is the size of the formed vector.
9050 // Each EXTRACT_VECTOR should have the same input vector and odd or even
9051 // index such that we have a pair wise add pattern.
Tanya Lattnere9e67052011-06-14 23:48:48 +00009052
9053 // Grab the vector that all EXTRACT_VECTOR nodes should be referencing.
Bob Wilson4b12a112011-06-15 06:04:34 +00009054 if (N0->getOperand(0)->getOpcode() != ISD::EXTRACT_VECTOR_ELT)
Tanya Lattnere9e67052011-06-14 23:48:48 +00009055 return SDValue();
Bob Wilson4b12a112011-06-15 06:04:34 +00009056 SDValue Vec = N0->getOperand(0)->getOperand(0);
9057 SDNode *V = Vec.getNode();
9058 unsigned nextIndex = 0;
Tanya Lattnere9e67052011-06-14 23:48:48 +00009059
Eric Christopher1b8b94192011-06-29 21:10:36 +00009060 // For each operands to the ADD which are BUILD_VECTORs,
Tanya Lattnere9e67052011-06-14 23:48:48 +00009061 // check to see if each of their operands are an EXTRACT_VECTOR with
9062 // the same vector and appropriate index.
9063 for (unsigned i = 0, e = N0->getNumOperands(); i != e; ++i) {
9064 if (N0->getOperand(i)->getOpcode() == ISD::EXTRACT_VECTOR_ELT
9065 && N1->getOperand(i)->getOpcode() == ISD::EXTRACT_VECTOR_ELT) {
Eric Christopher1b8b94192011-06-29 21:10:36 +00009066
Tanya Lattnere9e67052011-06-14 23:48:48 +00009067 SDValue ExtVec0 = N0->getOperand(i);
9068 SDValue ExtVec1 = N1->getOperand(i);
Eric Christopher1b8b94192011-06-29 21:10:36 +00009069
Tanya Lattnere9e67052011-06-14 23:48:48 +00009070 // First operand is the vector, verify its the same.
9071 if (V != ExtVec0->getOperand(0).getNode() ||
9072 V != ExtVec1->getOperand(0).getNode())
9073 return SDValue();
Eric Christopher1b8b94192011-06-29 21:10:36 +00009074
Tanya Lattnere9e67052011-06-14 23:48:48 +00009075 // Second is the constant, verify its correct.
9076 ConstantSDNode *C0 = dyn_cast<ConstantSDNode>(ExtVec0->getOperand(1));
9077 ConstantSDNode *C1 = dyn_cast<ConstantSDNode>(ExtVec1->getOperand(1));
Eric Christopher1b8b94192011-06-29 21:10:36 +00009078
Tanya Lattnere9e67052011-06-14 23:48:48 +00009079 // For the constant, we want to see all the even or all the odd.
9080 if (!C0 || !C1 || C0->getZExtValue() != nextIndex
9081 || C1->getZExtValue() != nextIndex+1)
9082 return SDValue();
9083
9084 // Increment index.
9085 nextIndex+=2;
Eric Christopher1b8b94192011-06-29 21:10:36 +00009086 } else
Tanya Lattnere9e67052011-06-14 23:48:48 +00009087 return SDValue();
9088 }
9089
9090 // Create VPADDL node.
9091 SelectionDAG &DAG = DCI.DAG;
9092 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
Tanya Lattnere9e67052011-06-14 23:48:48 +00009093
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +00009094 SDLoc dl(N);
9095
Tanya Lattnere9e67052011-06-14 23:48:48 +00009096 // Build operand list.
9097 SmallVector<SDValue, 8> Ops;
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +00009098 Ops.push_back(DAG.getConstant(Intrinsic::arm_neon_vpaddls, dl,
Mehdi Amini44ede332015-07-09 02:09:04 +00009099 TLI.getPointerTy(DAG.getDataLayout())));
Tanya Lattnere9e67052011-06-14 23:48:48 +00009100
9101 // Input is the vector.
9102 Ops.push_back(Vec);
Eric Christopher1b8b94192011-06-29 21:10:36 +00009103
Tanya Lattnere9e67052011-06-14 23:48:48 +00009104 // Get widened type and narrowed type.
9105 MVT widenType;
9106 unsigned numElem = VT.getVectorNumElements();
Junmo Park1108ab02016-02-19 01:46:04 +00009107
Silviu Barangaa3106e62014-04-03 10:44:27 +00009108 EVT inputLaneType = Vec.getValueType().getVectorElementType();
9109 switch (inputLaneType.getSimpleVT().SimpleTy) {
Tanya Lattnere9e67052011-06-14 23:48:48 +00009110 case MVT::i8: widenType = MVT::getVectorVT(MVT::i16, numElem); break;
9111 case MVT::i16: widenType = MVT::getVectorVT(MVT::i32, numElem); break;
9112 case MVT::i32: widenType = MVT::getVectorVT(MVT::i64, numElem); break;
9113 default:
Craig Toppere55c5562012-02-07 02:50:20 +00009114 llvm_unreachable("Invalid vector element type for padd optimization.");
Tanya Lattnere9e67052011-06-14 23:48:48 +00009115 }
9116
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +00009117 SDValue tmp = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, widenType, Ops);
Silviu Barangaa3106e62014-04-03 10:44:27 +00009118 unsigned ExtOp = VT.bitsGT(tmp.getValueType()) ? ISD::ANY_EXTEND : ISD::TRUNCATE;
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +00009119 return DAG.getNode(ExtOp, dl, VT, tmp);
Tanya Lattnere9e67052011-06-14 23:48:48 +00009120}
9121
Arnold Schwaighoferf00fb1c2012-09-04 14:37:49 +00009122static SDValue findMUL_LOHI(SDValue V) {
9123 if (V->getOpcode() == ISD::UMUL_LOHI ||
9124 V->getOpcode() == ISD::SMUL_LOHI)
9125 return V;
9126 return SDValue();
9127}
9128
9129static SDValue AddCombineTo64bitMLAL(SDNode *AddcNode,
9130 TargetLowering::DAGCombinerInfo &DCI,
9131 const ARMSubtarget *Subtarget) {
9132
Arnold Schwaighoferf00fb1c2012-09-04 14:37:49 +00009133 // Look for multiply add opportunities.
9134 // The pattern is a ISD::UMUL_LOHI followed by two add nodes, where
9135 // each add nodes consumes a value from ISD::UMUL_LOHI and there is
9136 // a glue link from the first add to the second add.
9137 // If we find this pattern, we can replace the U/SMUL_LOHI, ADDC, and ADDE by
9138 // a S/UMLAL instruction.
Matthias Braun60912082015-05-20 18:40:06 +00009139 // UMUL_LOHI
9140 // / :lo \ :hi
9141 // / \ [no multiline comment]
9142 // loAdd -> ADDE |
9143 // \ :glue /
9144 // \ /
9145 // ADDC <- hiAdd
Arnold Schwaighoferf00fb1c2012-09-04 14:37:49 +00009146 //
9147 assert(AddcNode->getOpcode() == ISD::ADDC && "Expect an ADDC");
9148 SDValue AddcOp0 = AddcNode->getOperand(0);
9149 SDValue AddcOp1 = AddcNode->getOperand(1);
9150
9151 // Check if the two operands are from the same mul_lohi node.
9152 if (AddcOp0.getNode() == AddcOp1.getNode())
9153 return SDValue();
9154
9155 assert(AddcNode->getNumValues() == 2 &&
9156 AddcNode->getValueType(0) == MVT::i32 &&
Michael Gottesmanb2a70562013-06-18 20:49:40 +00009157 "Expect ADDC with two result values. First: i32");
9158
9159 // Check that we have a glued ADDC node.
9160 if (AddcNode->getValueType(1) != MVT::Glue)
9161 return SDValue();
Arnold Schwaighoferf00fb1c2012-09-04 14:37:49 +00009162
9163 // Check that the ADDC adds the low result of the S/UMUL_LOHI.
9164 if (AddcOp0->getOpcode() != ISD::UMUL_LOHI &&
9165 AddcOp0->getOpcode() != ISD::SMUL_LOHI &&
9166 AddcOp1->getOpcode() != ISD::UMUL_LOHI &&
9167 AddcOp1->getOpcode() != ISD::SMUL_LOHI)
9168 return SDValue();
9169
9170 // Look for the glued ADDE.
9171 SDNode* AddeNode = AddcNode->getGluedUser();
Craig Topper062a2ba2014-04-25 05:30:21 +00009172 if (!AddeNode)
Arnold Schwaighoferf00fb1c2012-09-04 14:37:49 +00009173 return SDValue();
9174
9175 // Make sure it is really an ADDE.
9176 if (AddeNode->getOpcode() != ISD::ADDE)
9177 return SDValue();
9178
9179 assert(AddeNode->getNumOperands() == 3 &&
9180 AddeNode->getOperand(2).getValueType() == MVT::Glue &&
9181 "ADDE node has the wrong inputs");
9182
9183 // Check for the triangle shape.
9184 SDValue AddeOp0 = AddeNode->getOperand(0);
9185 SDValue AddeOp1 = AddeNode->getOperand(1);
9186
9187 // Make sure that the ADDE operands are not coming from the same node.
9188 if (AddeOp0.getNode() == AddeOp1.getNode())
9189 return SDValue();
9190
9191 // Find the MUL_LOHI node walking up ADDE's operands.
9192 bool IsLeftOperandMUL = false;
9193 SDValue MULOp = findMUL_LOHI(AddeOp0);
9194 if (MULOp == SDValue())
9195 MULOp = findMUL_LOHI(AddeOp1);
9196 else
9197 IsLeftOperandMUL = true;
9198 if (MULOp == SDValue())
Jyoti Allurf1d70502015-01-23 09:10:03 +00009199 return SDValue();
Arnold Schwaighoferf00fb1c2012-09-04 14:37:49 +00009200
9201 // Figure out the right opcode.
9202 unsigned Opc = MULOp->getOpcode();
9203 unsigned FinalOpc = (Opc == ISD::SMUL_LOHI) ? ARMISD::SMLAL : ARMISD::UMLAL;
9204
9205 // Figure out the high and low input values to the MLAL node.
Craig Topper062a2ba2014-04-25 05:30:21 +00009206 SDValue* HiAdd = nullptr;
9207 SDValue* LoMul = nullptr;
9208 SDValue* LowAdd = nullptr;
Arnold Schwaighoferf00fb1c2012-09-04 14:37:49 +00009209
Jyoti Allurf1d70502015-01-23 09:10:03 +00009210 // Ensure that ADDE is from high result of ISD::SMUL_LOHI.
9211 if ((AddeOp0 != MULOp.getValue(1)) && (AddeOp1 != MULOp.getValue(1)))
9212 return SDValue();
9213
Arnold Schwaighoferf00fb1c2012-09-04 14:37:49 +00009214 if (IsLeftOperandMUL)
9215 HiAdd = &AddeOp1;
9216 else
9217 HiAdd = &AddeOp0;
9218
9219
Jyoti Allurf1d70502015-01-23 09:10:03 +00009220 // Ensure that LoMul and LowAdd are taken from correct ISD::SMUL_LOHI node
9221 // whose low result is fed to the ADDC we are checking.
9222
9223 if (AddcOp0 == MULOp.getValue(0)) {
Arnold Schwaighoferf00fb1c2012-09-04 14:37:49 +00009224 LoMul = &AddcOp0;
9225 LowAdd = &AddcOp1;
9226 }
Jyoti Allurf1d70502015-01-23 09:10:03 +00009227 if (AddcOp1 == MULOp.getValue(0)) {
Arnold Schwaighoferf00fb1c2012-09-04 14:37:49 +00009228 LoMul = &AddcOp1;
9229 LowAdd = &AddcOp0;
9230 }
9231
Craig Topper062a2ba2014-04-25 05:30:21 +00009232 if (!LoMul)
Arnold Schwaighoferf00fb1c2012-09-04 14:37:49 +00009233 return SDValue();
9234
Arnold Schwaighoferf00fb1c2012-09-04 14:37:49 +00009235 // Create the merged node.
9236 SelectionDAG &DAG = DCI.DAG;
9237
9238 // Build operand list.
9239 SmallVector<SDValue, 8> Ops;
9240 Ops.push_back(LoMul->getOperand(0));
9241 Ops.push_back(LoMul->getOperand(1));
9242 Ops.push_back(*LowAdd);
9243 Ops.push_back(*HiAdd);
9244
Andrew Trickef9de2a2013-05-25 02:42:55 +00009245 SDValue MLALNode = DAG.getNode(FinalOpc, SDLoc(AddcNode),
Craig Topper48d114b2014-04-26 18:35:24 +00009246 DAG.getVTList(MVT::i32, MVT::i32), Ops);
Arnold Schwaighoferf00fb1c2012-09-04 14:37:49 +00009247
9248 // Replace the ADDs' nodes uses by the MLA node's values.
9249 SDValue HiMLALResult(MLALNode.getNode(), 1);
9250 DAG.ReplaceAllUsesOfValueWith(SDValue(AddeNode, 0), HiMLALResult);
9251
9252 SDValue LoMLALResult(MLALNode.getNode(), 0);
9253 DAG.ReplaceAllUsesOfValueWith(SDValue(AddcNode, 0), LoMLALResult);
9254
9255 // Return original node to notify the driver to stop replacing.
9256 SDValue resNode(AddcNode, 0);
9257 return resNode;
9258}
9259
Sam Parkerd616cf02016-06-20 16:47:09 +00009260static SDValue AddCombineTo64bitUMAAL(SDNode *AddcNode,
9261 TargetLowering::DAGCombinerInfo &DCI,
9262 const ARMSubtarget *Subtarget) {
9263 // UMAAL is similar to UMLAL except that it adds two unsigned values.
9264 // While trying to combine for the other MLAL nodes, first search for the
9265 // chance to use UMAAL. Check if Addc uses another addc node which can first
9266 // be combined into a UMLAL. The other pattern is AddcNode being combined
9267 // into an UMLAL and then using another addc is handled in ISelDAGToDAG.
9268
Sam Parkerd5ca0a62016-07-25 10:11:00 +00009269 if (!Subtarget->hasV6Ops() ||
9270 (Subtarget->isThumb() && !Subtarget->hasThumb2()))
Sam Parkerd616cf02016-06-20 16:47:09 +00009271 return AddCombineTo64bitMLAL(AddcNode, DCI, Subtarget);
9272
9273 SDNode *PrevAddc = nullptr;
9274 if (AddcNode->getOperand(0).getOpcode() == ISD::ADDC)
9275 PrevAddc = AddcNode->getOperand(0).getNode();
9276 else if (AddcNode->getOperand(1).getOpcode() == ISD::ADDC)
9277 PrevAddc = AddcNode->getOperand(1).getNode();
9278
9279 // If there's no addc chains, just return a search for any MLAL.
9280 if (PrevAddc == nullptr)
9281 return AddCombineTo64bitMLAL(AddcNode, DCI, Subtarget);
9282
9283 // Try to convert the addc operand to an MLAL and if that fails try to
9284 // combine AddcNode.
9285 SDValue MLAL = AddCombineTo64bitMLAL(PrevAddc, DCI, Subtarget);
9286 if (MLAL != SDValue(PrevAddc, 0))
9287 return AddCombineTo64bitMLAL(AddcNode, DCI, Subtarget);
9288
9289 // Find the converted UMAAL or quit if it doesn't exist.
9290 SDNode *UmlalNode = nullptr;
9291 SDValue AddHi;
9292 if (AddcNode->getOperand(0).getOpcode() == ARMISD::UMLAL) {
9293 UmlalNode = AddcNode->getOperand(0).getNode();
9294 AddHi = AddcNode->getOperand(1);
9295 } else if (AddcNode->getOperand(1).getOpcode() == ARMISD::UMLAL) {
9296 UmlalNode = AddcNode->getOperand(1).getNode();
9297 AddHi = AddcNode->getOperand(0);
9298 } else {
9299 return SDValue();
9300 }
9301
9302 // The ADDC should be glued to an ADDE node, which uses the same UMLAL as
9303 // the ADDC as well as Zero.
9304 auto *Zero = dyn_cast<ConstantSDNode>(UmlalNode->getOperand(3));
9305
9306 if (!Zero || Zero->getZExtValue() != 0)
9307 return SDValue();
9308
9309 // Check that we have a glued ADDC node.
9310 if (AddcNode->getValueType(1) != MVT::Glue)
9311 return SDValue();
9312
9313 // Look for the glued ADDE.
9314 SDNode* AddeNode = AddcNode->getGluedUser();
9315 if (!AddeNode)
9316 return SDValue();
9317
9318 if ((AddeNode->getOperand(0).getNode() == Zero &&
9319 AddeNode->getOperand(1).getNode() == UmlalNode) ||
9320 (AddeNode->getOperand(0).getNode() == UmlalNode &&
9321 AddeNode->getOperand(1).getNode() == Zero)) {
9322
9323 SelectionDAG &DAG = DCI.DAG;
9324 SDValue Ops[] = { UmlalNode->getOperand(0), UmlalNode->getOperand(1),
9325 UmlalNode->getOperand(2), AddHi };
9326 SDValue UMAAL = DAG.getNode(ARMISD::UMAAL, SDLoc(AddcNode),
9327 DAG.getVTList(MVT::i32, MVT::i32), Ops);
9328
9329 // Replace the ADDs' nodes uses by the UMAAL node's values.
9330 DAG.ReplaceAllUsesOfValueWith(SDValue(AddeNode, 0), SDValue(UMAAL.getNode(), 1));
9331 DAG.ReplaceAllUsesOfValueWith(SDValue(AddcNode, 0), SDValue(UMAAL.getNode(), 0));
9332
9333 // Return original node to notify the driver to stop replacing.
9334 return SDValue(AddcNode, 0);
9335 }
9336 return SDValue();
9337}
9338
Arnold Schwaighoferf00fb1c2012-09-04 14:37:49 +00009339/// PerformADDCCombine - Target-specific dag combine transform from
Sam Parkerd616cf02016-06-20 16:47:09 +00009340/// ISD::ADDC, ISD::ADDE, and ISD::MUL_LOHI to MLAL or
9341/// ISD::ADDC, ISD::ADDE and ARMISD::UMLAL to ARMISD::UMAAL
Arnold Schwaighoferf00fb1c2012-09-04 14:37:49 +00009342static SDValue PerformADDCCombine(SDNode *N,
9343 TargetLowering::DAGCombinerInfo &DCI,
9344 const ARMSubtarget *Subtarget) {
9345
Sam Parkerd616cf02016-06-20 16:47:09 +00009346 if (Subtarget->isThumb1Only()) return SDValue();
Arnold Schwaighoferf00fb1c2012-09-04 14:37:49 +00009347
Sam Parkerd616cf02016-06-20 16:47:09 +00009348 // Only perform the checks after legalize when the pattern is available.
9349 if (DCI.isBeforeLegalize()) return SDValue();
9350
9351 return AddCombineTo64bitUMAAL(N, DCI, Subtarget);
Arnold Schwaighoferf00fb1c2012-09-04 14:37:49 +00009352}
9353
Bob Wilson728eb292010-07-29 20:34:14 +00009354/// PerformADDCombineWithOperands - Try DAG combinations for an ADD with
9355/// operands N0 and N1. This is a helper for PerformADDCombine that is
9356/// called with the default operands, and if that fails, with commuted
9357/// operands.
9358static SDValue PerformADDCombineWithOperands(SDNode *N, SDValue N0, SDValue N1,
Tanya Lattnere9e67052011-06-14 23:48:48 +00009359 TargetLowering::DAGCombinerInfo &DCI,
9360 const ARMSubtarget *Subtarget){
9361
9362 // Attempt to create vpaddl for this add.
Ahmed Bougachaf8dfb472016-02-09 22:54:12 +00009363 if (SDValue Result = AddCombineToVPADDL(N, N0, N1, DCI, Subtarget))
Tanya Lattnere9e67052011-06-14 23:48:48 +00009364 return Result;
Eric Christopher1b8b94192011-06-29 21:10:36 +00009365
Chris Lattner4147f082009-03-12 06:52:53 +00009366 // fold (add (select cc, 0, c), x) -> (select cc, x, (add, x, c))
Ahmed Bougachaf8dfb472016-02-09 22:54:12 +00009367 if (N0.getNode()->hasOneUse())
9368 if (SDValue Result = combineSelectAndUse(N, N0, N1, DCI))
9369 return Result;
Chris Lattner4147f082009-03-12 06:52:53 +00009370 return SDValue();
9371}
9372
Bob Wilson728eb292010-07-29 20:34:14 +00009373/// PerformADDCombine - Target-specific dag combine xforms for ISD::ADD.
9374///
9375static SDValue PerformADDCombine(SDNode *N,
Tanya Lattnere9e67052011-06-14 23:48:48 +00009376 TargetLowering::DAGCombinerInfo &DCI,
9377 const ARMSubtarget *Subtarget) {
Bob Wilson728eb292010-07-29 20:34:14 +00009378 SDValue N0 = N->getOperand(0);
9379 SDValue N1 = N->getOperand(1);
9380
9381 // First try with the default operand order.
Ahmed Bougachaf8dfb472016-02-09 22:54:12 +00009382 if (SDValue Result = PerformADDCombineWithOperands(N, N0, N1, DCI, Subtarget))
Bob Wilson728eb292010-07-29 20:34:14 +00009383 return Result;
9384
9385 // If that didn't work, try again with the operands commuted.
Tanya Lattnere9e67052011-06-14 23:48:48 +00009386 return PerformADDCombineWithOperands(N, N1, N0, DCI, Subtarget);
Bob Wilson728eb292010-07-29 20:34:14 +00009387}
9388
Chris Lattner4147f082009-03-12 06:52:53 +00009389/// PerformSUBCombine - Target-specific dag combine xforms for ISD::SUB.
Bob Wilson728eb292010-07-29 20:34:14 +00009390///
Chris Lattner4147f082009-03-12 06:52:53 +00009391static SDValue PerformSUBCombine(SDNode *N,
9392 TargetLowering::DAGCombinerInfo &DCI) {
Bob Wilson728eb292010-07-29 20:34:14 +00009393 SDValue N0 = N->getOperand(0);
9394 SDValue N1 = N->getOperand(1);
Bob Wilson7117a912009-03-20 22:42:55 +00009395
Chris Lattner4147f082009-03-12 06:52:53 +00009396 // fold (sub x, (select cc, 0, c)) -> (select cc, x, (sub, x, c))
Ahmed Bougachaf8dfb472016-02-09 22:54:12 +00009397 if (N1.getNode()->hasOneUse())
9398 if (SDValue Result = combineSelectAndUse(N, N1, N0, DCI))
9399 return Result;
Bob Wilson7117a912009-03-20 22:42:55 +00009400
Chris Lattner4147f082009-03-12 06:52:53 +00009401 return SDValue();
9402}
9403
Evan Cheng38bf5ad2011-03-31 19:38:48 +00009404/// PerformVMULCombine
9405/// Distribute (A + B) * C to (A * C) + (B * C) to take advantage of the
9406/// special multiplier accumulator forwarding.
9407/// vmul d3, d0, d2
9408/// vmla d3, d1, d2
9409/// is faster than
9410/// vadd d3, d0, d1
9411/// vmul d3, d3, d2
Weiming Zhao2052f482013-09-25 23:12:06 +00009412// However, for (A + B) * (A + B),
9413// vadd d2, d0, d1
9414// vmul d3, d0, d2
9415// vmla d3, d1, d2
9416// is slower than
9417// vadd d2, d0, d1
9418// vmul d3, d2, d2
Evan Cheng38bf5ad2011-03-31 19:38:48 +00009419static SDValue PerformVMULCombine(SDNode *N,
9420 TargetLowering::DAGCombinerInfo &DCI,
9421 const ARMSubtarget *Subtarget) {
9422 if (!Subtarget->hasVMLxForwarding())
9423 return SDValue();
9424
9425 SelectionDAG &DAG = DCI.DAG;
9426 SDValue N0 = N->getOperand(0);
9427 SDValue N1 = N->getOperand(1);
9428 unsigned Opcode = N0.getOpcode();
9429 if (Opcode != ISD::ADD && Opcode != ISD::SUB &&
9430 Opcode != ISD::FADD && Opcode != ISD::FSUB) {
Chad Rosier27301622011-06-16 01:21:54 +00009431 Opcode = N1.getOpcode();
Evan Cheng38bf5ad2011-03-31 19:38:48 +00009432 if (Opcode != ISD::ADD && Opcode != ISD::SUB &&
9433 Opcode != ISD::FADD && Opcode != ISD::FSUB)
9434 return SDValue();
9435 std::swap(N0, N1);
9436 }
9437
Weiming Zhao2052f482013-09-25 23:12:06 +00009438 if (N0 == N1)
9439 return SDValue();
9440
Evan Cheng38bf5ad2011-03-31 19:38:48 +00009441 EVT VT = N->getValueType(0);
Andrew Trickef9de2a2013-05-25 02:42:55 +00009442 SDLoc DL(N);
Evan Cheng38bf5ad2011-03-31 19:38:48 +00009443 SDValue N00 = N0->getOperand(0);
9444 SDValue N01 = N0->getOperand(1);
9445 return DAG.getNode(Opcode, DL, VT,
9446 DAG.getNode(ISD::MUL, DL, VT, N00, N1),
9447 DAG.getNode(ISD::MUL, DL, VT, N01, N1));
9448}
9449
Anton Korobeynikov1bf28a12010-05-15 18:16:59 +00009450static SDValue PerformMULCombine(SDNode *N,
9451 TargetLowering::DAGCombinerInfo &DCI,
9452 const ARMSubtarget *Subtarget) {
9453 SelectionDAG &DAG = DCI.DAG;
9454
9455 if (Subtarget->isThumb1Only())
9456 return SDValue();
9457
Anton Korobeynikov1bf28a12010-05-15 18:16:59 +00009458 if (DCI.isBeforeLegalize() || DCI.isCalledByLegalizer())
9459 return SDValue();
9460
9461 EVT VT = N->getValueType(0);
Evan Cheng38bf5ad2011-03-31 19:38:48 +00009462 if (VT.is64BitVector() || VT.is128BitVector())
9463 return PerformVMULCombine(N, DCI, Subtarget);
Anton Korobeynikov1bf28a12010-05-15 18:16:59 +00009464 if (VT != MVT::i32)
9465 return SDValue();
9466
9467 ConstantSDNode *C = dyn_cast<ConstantSDNode>(N->getOperand(1));
9468 if (!C)
9469 return SDValue();
9470
Anton Korobeynikov3edd854d2012-03-19 19:19:50 +00009471 int64_t MulAmt = C->getSExtValue();
Michael J. Spencerdf1ecbd72013-05-24 22:23:49 +00009472 unsigned ShiftAmt = countTrailingZeros<uint64_t>(MulAmt);
Anton Korobeynikov3edd854d2012-03-19 19:19:50 +00009473
Anton Korobeynikov1bf28a12010-05-15 18:16:59 +00009474 ShiftAmt = ShiftAmt & (32 - 1);
9475 SDValue V = N->getOperand(0);
Andrew Trickef9de2a2013-05-25 02:42:55 +00009476 SDLoc DL(N);
Anton Korobeynikov1bf28a12010-05-15 18:16:59 +00009477
Anton Korobeynikov4c719c42010-05-16 08:54:20 +00009478 SDValue Res;
9479 MulAmt >>= ShiftAmt;
Anton Korobeynikov3edd854d2012-03-19 19:19:50 +00009480
9481 if (MulAmt >= 0) {
9482 if (isPowerOf2_32(MulAmt - 1)) {
9483 // (mul x, 2^N + 1) => (add (shl x, N), x)
9484 Res = DAG.getNode(ISD::ADD, DL, VT,
9485 V,
9486 DAG.getNode(ISD::SHL, DL, VT,
9487 V,
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +00009488 DAG.getConstant(Log2_32(MulAmt - 1), DL,
Anton Korobeynikov3edd854d2012-03-19 19:19:50 +00009489 MVT::i32)));
9490 } else if (isPowerOf2_32(MulAmt + 1)) {
9491 // (mul x, 2^N - 1) => (sub (shl x, N), x)
9492 Res = DAG.getNode(ISD::SUB, DL, VT,
9493 DAG.getNode(ISD::SHL, DL, VT,
9494 V,
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +00009495 DAG.getConstant(Log2_32(MulAmt + 1), DL,
Anton Korobeynikov3edd854d2012-03-19 19:19:50 +00009496 MVT::i32)),
9497 V);
9498 } else
9499 return SDValue();
9500 } else {
9501 uint64_t MulAmtAbs = -MulAmt;
9502 if (isPowerOf2_32(MulAmtAbs + 1)) {
9503 // (mul x, -(2^N - 1)) => (sub x, (shl x, N))
9504 Res = DAG.getNode(ISD::SUB, DL, VT,
9505 V,
9506 DAG.getNode(ISD::SHL, DL, VT,
9507 V,
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +00009508 DAG.getConstant(Log2_32(MulAmtAbs + 1), DL,
Anton Korobeynikov3edd854d2012-03-19 19:19:50 +00009509 MVT::i32)));
9510 } else if (isPowerOf2_32(MulAmtAbs - 1)) {
9511 // (mul x, -(2^N + 1)) => - (add (shl x, N), x)
9512 Res = DAG.getNode(ISD::ADD, DL, VT,
9513 V,
9514 DAG.getNode(ISD::SHL, DL, VT,
9515 V,
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +00009516 DAG.getConstant(Log2_32(MulAmtAbs - 1), DL,
Anton Korobeynikov3edd854d2012-03-19 19:19:50 +00009517 MVT::i32)));
9518 Res = DAG.getNode(ISD::SUB, DL, VT,
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +00009519 DAG.getConstant(0, DL, MVT::i32), Res);
Anton Korobeynikov3edd854d2012-03-19 19:19:50 +00009520
9521 } else
9522 return SDValue();
9523 }
Anton Korobeynikov4c719c42010-05-16 08:54:20 +00009524
9525 if (ShiftAmt != 0)
Anton Korobeynikov3edd854d2012-03-19 19:19:50 +00009526 Res = DAG.getNode(ISD::SHL, DL, VT,
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +00009527 Res, DAG.getConstant(ShiftAmt, DL, MVT::i32));
Anton Korobeynikov1bf28a12010-05-15 18:16:59 +00009528
9529 // Do not add new nodes to DAG combiner worklist.
Anton Korobeynikov4c719c42010-05-16 08:54:20 +00009530 DCI.CombineTo(N, Res, false);
Anton Korobeynikov1bf28a12010-05-15 18:16:59 +00009531 return SDValue();
9532}
9533
Owen Anderson30c48922010-11-05 19:27:46 +00009534static SDValue PerformANDCombine(SDNode *N,
Evan Chenge87681c2012-02-23 01:19:06 +00009535 TargetLowering::DAGCombinerInfo &DCI,
9536 const ARMSubtarget *Subtarget) {
Owen Anderson77aa2662011-04-05 21:48:57 +00009537
Owen Anderson30c48922010-11-05 19:27:46 +00009538 // Attempt to use immediate-form VBIC
9539 BuildVectorSDNode *BVN = dyn_cast<BuildVectorSDNode>(N->getOperand(1));
Andrew Trickef9de2a2013-05-25 02:42:55 +00009540 SDLoc dl(N);
Owen Anderson30c48922010-11-05 19:27:46 +00009541 EVT VT = N->getValueType(0);
9542 SelectionDAG &DAG = DCI.DAG;
Wesley Peck527da1b2010-11-23 03:31:01 +00009543
Tanya Lattner266792a2011-04-07 15:24:20 +00009544 if(!DAG.getTargetLoweringInfo().isTypeLegal(VT))
9545 return SDValue();
Andrew Trick0ed57782011-04-23 03:55:32 +00009546
Owen Anderson30c48922010-11-05 19:27:46 +00009547 APInt SplatBits, SplatUndef;
9548 unsigned SplatBitSize;
9549 bool HasAnyUndefs;
9550 if (BVN &&
9551 BVN->isConstantSplat(SplatBits, SplatUndef, SplatBitSize, HasAnyUndefs)) {
9552 if (SplatBitSize <= 64) {
9553 EVT VbicVT;
9554 SDValue Val = isNEONModifiedImm((~SplatBits).getZExtValue(),
9555 SplatUndef.getZExtValue(), SplatBitSize,
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +00009556 DAG, dl, VbicVT, VT.is128BitVector(),
Owen Andersona4076922010-11-05 21:57:54 +00009557 OtherModImm);
Owen Anderson30c48922010-11-05 19:27:46 +00009558 if (Val.getNode()) {
9559 SDValue Input =
Wesley Peck527da1b2010-11-23 03:31:01 +00009560 DAG.getNode(ISD::BITCAST, dl, VbicVT, N->getOperand(0));
Owen Anderson30c48922010-11-05 19:27:46 +00009561 SDValue Vbic = DAG.getNode(ARMISD::VBICIMM, dl, VbicVT, Input, Val);
Wesley Peck527da1b2010-11-23 03:31:01 +00009562 return DAG.getNode(ISD::BITCAST, dl, VT, Vbic);
Owen Anderson30c48922010-11-05 19:27:46 +00009563 }
9564 }
9565 }
Wesley Peck527da1b2010-11-23 03:31:01 +00009566
Evan Chenge87681c2012-02-23 01:19:06 +00009567 if (!Subtarget->isThumb1Only()) {
Jakob Stoklund Olesenaab43db2012-08-18 21:25:16 +00009568 // fold (and (select cc, -1, c), x) -> (select cc, x, (and, x, c))
Ahmed Bougachaf8dfb472016-02-09 22:54:12 +00009569 if (SDValue Result = combineSelectAndUseCommutative(N, true, DCI))
Jakob Stoklund Olesenaab43db2012-08-18 21:25:16 +00009570 return Result;
Evan Chenge87681c2012-02-23 01:19:06 +00009571 }
9572
Owen Anderson30c48922010-11-05 19:27:46 +00009573 return SDValue();
9574}
9575
Jim Grosbach11013ed2010-07-16 23:05:05 +00009576/// PerformORCombine - Target-specific dag combine xforms for ISD::OR
9577static SDValue PerformORCombine(SDNode *N,
9578 TargetLowering::DAGCombinerInfo &DCI,
9579 const ARMSubtarget *Subtarget) {
Owen Andersonbc9b31c2010-11-03 23:15:26 +00009580 // Attempt to use immediate-form VORR
9581 BuildVectorSDNode *BVN = dyn_cast<BuildVectorSDNode>(N->getOperand(1));
Andrew Trickef9de2a2013-05-25 02:42:55 +00009582 SDLoc dl(N);
Owen Andersonbc9b31c2010-11-03 23:15:26 +00009583 EVT VT = N->getValueType(0);
9584 SelectionDAG &DAG = DCI.DAG;
Wesley Peck527da1b2010-11-23 03:31:01 +00009585
Tanya Lattner266792a2011-04-07 15:24:20 +00009586 if(!DAG.getTargetLoweringInfo().isTypeLegal(VT))
9587 return SDValue();
Andrew Trick0ed57782011-04-23 03:55:32 +00009588
Owen Andersonbc9b31c2010-11-03 23:15:26 +00009589 APInt SplatBits, SplatUndef;
9590 unsigned SplatBitSize;
9591 bool HasAnyUndefs;
9592 if (BVN && Subtarget->hasNEON() &&
9593 BVN->isConstantSplat(SplatBits, SplatUndef, SplatBitSize, HasAnyUndefs)) {
9594 if (SplatBitSize <= 64) {
9595 EVT VorrVT;
9596 SDValue Val = isNEONModifiedImm(SplatBits.getZExtValue(),
9597 SplatUndef.getZExtValue(), SplatBitSize,
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +00009598 DAG, dl, VorrVT, VT.is128BitVector(),
Owen Andersona4076922010-11-05 21:57:54 +00009599 OtherModImm);
Owen Andersonbc9b31c2010-11-03 23:15:26 +00009600 if (Val.getNode()) {
9601 SDValue Input =
Wesley Peck527da1b2010-11-23 03:31:01 +00009602 DAG.getNode(ISD::BITCAST, dl, VorrVT, N->getOperand(0));
Owen Andersonbc9b31c2010-11-03 23:15:26 +00009603 SDValue Vorr = DAG.getNode(ARMISD::VORRIMM, dl, VorrVT, Input, Val);
Wesley Peck527da1b2010-11-23 03:31:01 +00009604 return DAG.getNode(ISD::BITCAST, dl, VT, Vorr);
Owen Andersonbc9b31c2010-11-03 23:15:26 +00009605 }
9606 }
9607 }
9608
Evan Chenge87681c2012-02-23 01:19:06 +00009609 if (!Subtarget->isThumb1Only()) {
Jakob Stoklund Olesenaab43db2012-08-18 21:25:16 +00009610 // fold (or (select cc, 0, c), x) -> (select cc, x, (or, x, c))
Ahmed Bougachaf8dfb472016-02-09 22:54:12 +00009611 if (SDValue Result = combineSelectAndUseCommutative(N, false, DCI))
Jakob Stoklund Olesenaab43db2012-08-18 21:25:16 +00009612 return Result;
Evan Chenge87681c2012-02-23 01:19:06 +00009613 }
9614
Nadav Rotem3a94c542012-08-13 18:52:44 +00009615 // The code below optimizes (or (and X, Y), Z).
9616 // The AND operand needs to have a single user to make these optimizations
9617 // profitable.
Cameron Zwarich53dd03d2011-03-30 23:01:21 +00009618 SDValue N0 = N->getOperand(0);
Nadav Rotem3a94c542012-08-13 18:52:44 +00009619 if (N0.getOpcode() != ISD::AND || !N0.hasOneUse())
Cameron Zwarich53dd03d2011-03-30 23:01:21 +00009620 return SDValue();
9621 SDValue N1 = N->getOperand(1);
9622
9623 // (or (and B, A), (and C, ~A)) => (VBSL A, B, C) when A is a constant.
9624 if (Subtarget->hasNEON() && N1.getOpcode() == ISD::AND && VT.isVector() &&
9625 DAG.getTargetLoweringInfo().isTypeLegal(VT)) {
9626 APInt SplatUndef;
9627 unsigned SplatBitSize;
9628 bool HasAnyUndefs;
9629
Saleem Abdulrasool0c2ee5a2013-07-30 04:43:08 +00009630 APInt SplatBits0, SplatBits1;
Cameron Zwarich53dd03d2011-03-30 23:01:21 +00009631 BuildVectorSDNode *BVN0 = dyn_cast<BuildVectorSDNode>(N0->getOperand(1));
Saleem Abdulrasool0c2ee5a2013-07-30 04:43:08 +00009632 BuildVectorSDNode *BVN1 = dyn_cast<BuildVectorSDNode>(N1->getOperand(1));
9633 // Ensure that the second operand of both ands are constants
Cameron Zwarich53dd03d2011-03-30 23:01:21 +00009634 if (BVN0 && BVN0->isConstantSplat(SplatBits0, SplatUndef, SplatBitSize,
Saleem Abdulrasool0c2ee5a2013-07-30 04:43:08 +00009635 HasAnyUndefs) && !HasAnyUndefs) {
9636 if (BVN1 && BVN1->isConstantSplat(SplatBits1, SplatUndef, SplatBitSize,
9637 HasAnyUndefs) && !HasAnyUndefs) {
9638 // Ensure that the bit width of the constants are the same and that
9639 // the splat arguments are logical inverses as per the pattern we
9640 // are trying to simplify.
9641 if (SplatBits0.getBitWidth() == SplatBits1.getBitWidth() &&
9642 SplatBits0 == ~SplatBits1) {
9643 // Canonicalize the vector type to make instruction selection
9644 // simpler.
9645 EVT CanonicalVT = VT.is128BitVector() ? MVT::v4i32 : MVT::v2i32;
9646 SDValue Result = DAG.getNode(ARMISD::VBSL, dl, CanonicalVT,
9647 N0->getOperand(1),
9648 N0->getOperand(0),
9649 N1->getOperand(0));
9650 return DAG.getNode(ISD::BITCAST, dl, VT, Result);
9651 }
9652 }
Cameron Zwarich53dd03d2011-03-30 23:01:21 +00009653 }
9654 }
9655
Jim Grosbachb97e2bb2010-07-17 03:30:54 +00009656 // Try to use the ARM/Thumb2 BFI (bitfield insert) instruction when
9657 // reasonable.
9658
Jim Grosbach11013ed2010-07-16 23:05:05 +00009659 // BFI is only available on V6T2+
9660 if (Subtarget->isThumb1Only() || !Subtarget->hasV6T2Ops())
9661 return SDValue();
9662
Andrew Trickef9de2a2013-05-25 02:42:55 +00009663 SDLoc DL(N);
Jim Grosbachb97e2bb2010-07-17 03:30:54 +00009664 // 1) or (and A, mask), val => ARMbfi A, val, mask
Sylvestre Ledru91ce36c2012-09-27 10:14:43 +00009665 // iff (val & mask) == val
Jim Grosbachb97e2bb2010-07-17 03:30:54 +00009666 //
9667 // 2) or (and A, mask), (and B, mask2) => ARMbfi A, (lsr B, amt), mask
Sylvestre Ledru91ce36c2012-09-27 10:14:43 +00009668 // 2a) iff isBitFieldInvertedMask(mask) && isBitFieldInvertedMask(~mask2)
Eric Christopherd5530962011-03-26 01:21:03 +00009669 // && mask == ~mask2
Sylvestre Ledru91ce36c2012-09-27 10:14:43 +00009670 // 2b) iff isBitFieldInvertedMask(~mask) && isBitFieldInvertedMask(mask2)
Eric Christopherd5530962011-03-26 01:21:03 +00009671 // && ~mask == mask2
Jim Grosbachb97e2bb2010-07-17 03:30:54 +00009672 // (i.e., copy a bitfield value into another bitfield of the same width)
Jim Grosbach11013ed2010-07-16 23:05:05 +00009673
Jim Grosbach11013ed2010-07-16 23:05:05 +00009674 if (VT != MVT::i32)
9675 return SDValue();
9676
Evan Cheng2e51bb42010-12-13 20:32:54 +00009677 SDValue N00 = N0.getOperand(0);
Jim Grosbachb97e2bb2010-07-17 03:30:54 +00009678
Jim Grosbach11013ed2010-07-16 23:05:05 +00009679 // The value and the mask need to be constants so we can verify this is
9680 // actually a bitfield set. If the mask is 0xffff, we can do better
9681 // via a movt instruction, so don't use BFI in that case.
Evan Cheng2e51bb42010-12-13 20:32:54 +00009682 SDValue MaskOp = N0.getOperand(1);
9683 ConstantSDNode *MaskC = dyn_cast<ConstantSDNode>(MaskOp);
9684 if (!MaskC)
Jim Grosbach11013ed2010-07-16 23:05:05 +00009685 return SDValue();
Evan Cheng2e51bb42010-12-13 20:32:54 +00009686 unsigned Mask = MaskC->getZExtValue();
Jim Grosbach11013ed2010-07-16 23:05:05 +00009687 if (Mask == 0xffff)
9688 return SDValue();
Jim Grosbachb97e2bb2010-07-17 03:30:54 +00009689 SDValue Res;
9690 // Case (1): or (and A, mask), val => ARMbfi A, val, mask
Evan Cheng2e51bb42010-12-13 20:32:54 +00009691 ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1);
9692 if (N1C) {
9693 unsigned Val = N1C->getZExtValue();
Evan Cheng34345752010-12-11 04:11:38 +00009694 if ((Val & ~Mask) != Val)
Jim Grosbachb97e2bb2010-07-17 03:30:54 +00009695 return SDValue();
Jim Grosbach11013ed2010-07-16 23:05:05 +00009696
Evan Cheng34345752010-12-11 04:11:38 +00009697 if (ARM::isBitFieldInvertedMask(Mask)) {
Michael J. Spencerdf1ecbd72013-05-24 22:23:49 +00009698 Val >>= countTrailingZeros(~Mask);
Jim Grosbach11013ed2010-07-16 23:05:05 +00009699
Evan Cheng2e51bb42010-12-13 20:32:54 +00009700 Res = DAG.getNode(ARMISD::BFI, DL, VT, N00,
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +00009701 DAG.getConstant(Val, DL, MVT::i32),
9702 DAG.getConstant(Mask, DL, MVT::i32));
Evan Cheng34345752010-12-11 04:11:38 +00009703
9704 // Do not add new nodes to DAG combiner worklist.
9705 DCI.CombineTo(N, Res, false);
Evan Cheng2e51bb42010-12-13 20:32:54 +00009706 return SDValue();
Evan Cheng34345752010-12-11 04:11:38 +00009707 }
Jim Grosbachb97e2bb2010-07-17 03:30:54 +00009708 } else if (N1.getOpcode() == ISD::AND) {
9709 // case (2) or (and A, mask), (and B, mask2) => ARMbfi A, (lsr B, amt), mask
Evan Cheng2e51bb42010-12-13 20:32:54 +00009710 ConstantSDNode *N11C = dyn_cast<ConstantSDNode>(N1.getOperand(1));
9711 if (!N11C)
Jim Grosbachb97e2bb2010-07-17 03:30:54 +00009712 return SDValue();
Evan Cheng2e51bb42010-12-13 20:32:54 +00009713 unsigned Mask2 = N11C->getZExtValue();
Jim Grosbachb97e2bb2010-07-17 03:30:54 +00009714
Eric Christopherd5530962011-03-26 01:21:03 +00009715 // Mask and ~Mask2 (or reverse) must be equivalent for the BFI pattern
9716 // as is to match.
Jim Grosbachb97e2bb2010-07-17 03:30:54 +00009717 if (ARM::isBitFieldInvertedMask(Mask) &&
Eric Christopherd5530962011-03-26 01:21:03 +00009718 (Mask == ~Mask2)) {
Jim Grosbachb97e2bb2010-07-17 03:30:54 +00009719 // The pack halfword instruction works better for masks that fit it,
9720 // so use that when it's available.
9721 if (Subtarget->hasT2ExtractPack() &&
9722 (Mask == 0xffff || Mask == 0xffff0000))
9723 return SDValue();
9724 // 2a
Michael J. Spencerdf1ecbd72013-05-24 22:23:49 +00009725 unsigned amt = countTrailingZeros(Mask2);
Jim Grosbachb97e2bb2010-07-17 03:30:54 +00009726 Res = DAG.getNode(ISD::SRL, DL, VT, N1.getOperand(0),
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +00009727 DAG.getConstant(amt, DL, MVT::i32));
Evan Cheng2e51bb42010-12-13 20:32:54 +00009728 Res = DAG.getNode(ARMISD::BFI, DL, VT, N00, Res,
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +00009729 DAG.getConstant(Mask, DL, MVT::i32));
Jim Grosbachb97e2bb2010-07-17 03:30:54 +00009730 // Do not add new nodes to DAG combiner worklist.
9731 DCI.CombineTo(N, Res, false);
Evan Cheng2e51bb42010-12-13 20:32:54 +00009732 return SDValue();
Jim Grosbachb97e2bb2010-07-17 03:30:54 +00009733 } else if (ARM::isBitFieldInvertedMask(~Mask) &&
Eric Christopherd5530962011-03-26 01:21:03 +00009734 (~Mask == Mask2)) {
Jim Grosbachb97e2bb2010-07-17 03:30:54 +00009735 // The pack halfword instruction works better for masks that fit it,
9736 // so use that when it's available.
9737 if (Subtarget->hasT2ExtractPack() &&
9738 (Mask2 == 0xffff || Mask2 == 0xffff0000))
9739 return SDValue();
9740 // 2b
Michael J. Spencerdf1ecbd72013-05-24 22:23:49 +00009741 unsigned lsb = countTrailingZeros(Mask);
Evan Cheng2e51bb42010-12-13 20:32:54 +00009742 Res = DAG.getNode(ISD::SRL, DL, VT, N00,
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +00009743 DAG.getConstant(lsb, DL, MVT::i32));
Jim Grosbachb97e2bb2010-07-17 03:30:54 +00009744 Res = DAG.getNode(ARMISD::BFI, DL, VT, N1.getOperand(0), Res,
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +00009745 DAG.getConstant(Mask2, DL, MVT::i32));
Jim Grosbachb97e2bb2010-07-17 03:30:54 +00009746 // Do not add new nodes to DAG combiner worklist.
9747 DCI.CombineTo(N, Res, false);
Evan Cheng2e51bb42010-12-13 20:32:54 +00009748 return SDValue();
Jim Grosbachb97e2bb2010-07-17 03:30:54 +00009749 }
9750 }
Wesley Peck527da1b2010-11-23 03:31:01 +00009751
Evan Cheng2e51bb42010-12-13 20:32:54 +00009752 if (DAG.MaskedValueIsZero(N1, MaskC->getAPIntValue()) &&
9753 N00.getOpcode() == ISD::SHL && isa<ConstantSDNode>(N00.getOperand(1)) &&
9754 ARM::isBitFieldInvertedMask(~Mask)) {
9755 // Case (3): or (and (shl A, #shamt), mask), B => ARMbfi B, A, ~mask
9756 // where lsb(mask) == #shamt and masked bits of B are known zero.
9757 SDValue ShAmt = N00.getOperand(1);
9758 unsigned ShAmtC = cast<ConstantSDNode>(ShAmt)->getZExtValue();
Michael J. Spencerdf1ecbd72013-05-24 22:23:49 +00009759 unsigned LSB = countTrailingZeros(Mask);
Evan Cheng2e51bb42010-12-13 20:32:54 +00009760 if (ShAmtC != LSB)
9761 return SDValue();
9762
9763 Res = DAG.getNode(ARMISD::BFI, DL, VT, N1, N00.getOperand(0),
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +00009764 DAG.getConstant(~Mask, DL, MVT::i32));
Evan Cheng2e51bb42010-12-13 20:32:54 +00009765
9766 // Do not add new nodes to DAG combiner worklist.
9767 DCI.CombineTo(N, Res, false);
9768 }
9769
Jim Grosbach11013ed2010-07-16 23:05:05 +00009770 return SDValue();
9771}
9772
Evan Chenge87681c2012-02-23 01:19:06 +00009773static SDValue PerformXORCombine(SDNode *N,
9774 TargetLowering::DAGCombinerInfo &DCI,
9775 const ARMSubtarget *Subtarget) {
9776 EVT VT = N->getValueType(0);
9777 SelectionDAG &DAG = DCI.DAG;
9778
9779 if(!DAG.getTargetLoweringInfo().isTypeLegal(VT))
9780 return SDValue();
9781
9782 if (!Subtarget->isThumb1Only()) {
Jakob Stoklund Olesenaab43db2012-08-18 21:25:16 +00009783 // fold (xor (select cc, 0, c), x) -> (select cc, x, (xor, x, c))
Ahmed Bougachaf8dfb472016-02-09 22:54:12 +00009784 if (SDValue Result = combineSelectAndUseCommutative(N, false, DCI))
Jakob Stoklund Olesenaab43db2012-08-18 21:25:16 +00009785 return Result;
Evan Chenge87681c2012-02-23 01:19:06 +00009786 }
9787
9788 return SDValue();
9789}
9790
James Molloyce12c922015-11-11 15:40:40 +00009791// ParseBFI - given a BFI instruction in N, extract the "from" value (Rn) and return it,
9792// and fill in FromMask and ToMask with (consecutive) bits in "from" to be extracted and
9793// their position in "to" (Rd).
9794static SDValue ParseBFI(SDNode *N, APInt &ToMask, APInt &FromMask) {
9795 assert(N->getOpcode() == ARMISD::BFI);
Chad Rosier353d7192015-12-21 18:08:05 +00009796
James Molloyce12c922015-11-11 15:40:40 +00009797 SDValue From = N->getOperand(1);
9798 ToMask = ~cast<ConstantSDNode>(N->getOperand(2))->getAPIntValue();
9799 FromMask = APInt::getLowBitsSet(ToMask.getBitWidth(), ToMask.countPopulation());
9800
9801 // If the Base came from a SHR #C, we can deduce that it is really testing bit
9802 // #C in the base of the SHR.
9803 if (From->getOpcode() == ISD::SRL &&
9804 isa<ConstantSDNode>(From->getOperand(1))) {
9805 APInt Shift = cast<ConstantSDNode>(From->getOperand(1))->getAPIntValue();
9806 assert(Shift.getLimitedValue() < 32 && "Shift too large!");
9807 FromMask <<= Shift.getLimitedValue(31);
9808 From = From->getOperand(0);
9809 }
9810
9811 return From;
9812}
9813
9814// If A and B contain one contiguous set of bits, does A | B == A . B?
9815//
9816// Neither A nor B must be zero.
9817static bool BitsProperlyConcatenate(const APInt &A, const APInt &B) {
9818 unsigned LastActiveBitInA = A.countTrailingZeros();
9819 unsigned FirstActiveBitInB = B.getBitWidth() - B.countLeadingZeros() - 1;
9820 return LastActiveBitInA - 1 == FirstActiveBitInB;
9821}
9822
9823static SDValue FindBFIToCombineWith(SDNode *N) {
9824 // We have a BFI in N. Follow a possible chain of BFIs and find a BFI it can combine with,
9825 // if one exists.
9826 APInt ToMask, FromMask;
9827 SDValue From = ParseBFI(N, ToMask, FromMask);
9828 SDValue To = N->getOperand(0);
9829
9830 // Now check for a compatible BFI to merge with. We can pass through BFIs that
9831 // aren't compatible, but not if they set the same bit in their destination as
9832 // we do (or that of any BFI we're going to combine with).
9833 SDValue V = To;
9834 APInt CombinedToMask = ToMask;
9835 while (V.getOpcode() == ARMISD::BFI) {
9836 APInt NewToMask, NewFromMask;
9837 SDValue NewFrom = ParseBFI(V.getNode(), NewToMask, NewFromMask);
9838 if (NewFrom != From) {
9839 // This BFI has a different base. Keep going.
9840 CombinedToMask |= NewToMask;
9841 V = V.getOperand(0);
9842 continue;
9843 }
9844
9845 // Do the written bits conflict with any we've seen so far?
9846 if ((NewToMask & CombinedToMask).getBoolValue())
9847 // Conflicting bits - bail out because going further is unsafe.
9848 return SDValue();
9849
9850 // Are the new bits contiguous when combined with the old bits?
9851 if (BitsProperlyConcatenate(ToMask, NewToMask) &&
9852 BitsProperlyConcatenate(FromMask, NewFromMask))
9853 return V;
9854 if (BitsProperlyConcatenate(NewToMask, ToMask) &&
9855 BitsProperlyConcatenate(NewFromMask, FromMask))
9856 return V;
Chad Rosier353d7192015-12-21 18:08:05 +00009857
James Molloyce12c922015-11-11 15:40:40 +00009858 // We've seen a write to some bits, so track it.
9859 CombinedToMask |= NewToMask;
9860 // Keep going...
9861 V = V.getOperand(0);
9862 }
9863
9864 return SDValue();
9865}
9866
Evan Chengc1778132010-12-14 03:22:07 +00009867static SDValue PerformBFICombine(SDNode *N,
9868 TargetLowering::DAGCombinerInfo &DCI) {
9869 SDValue N1 = N->getOperand(1);
9870 if (N1.getOpcode() == ISD::AND) {
James Molloyce12c922015-11-11 15:40:40 +00009871 // (bfi A, (and B, Mask1), Mask2) -> (bfi A, B, Mask2) iff
9872 // the bits being cleared by the AND are not demanded by the BFI.
Evan Chengc1778132010-12-14 03:22:07 +00009873 ConstantSDNode *N11C = dyn_cast<ConstantSDNode>(N1.getOperand(1));
9874 if (!N11C)
9875 return SDValue();
Evan Cheng6d02d902011-06-15 01:12:31 +00009876 unsigned InvMask = cast<ConstantSDNode>(N->getOperand(2))->getZExtValue();
Michael J. Spencerdf1ecbd72013-05-24 22:23:49 +00009877 unsigned LSB = countTrailingZeros(~InvMask);
9878 unsigned Width = (32 - countLeadingZeros(~InvMask)) - LSB;
Aaron Ballman0d6a0102014-12-16 14:04:11 +00009879 assert(Width <
9880 static_cast<unsigned>(std::numeric_limits<unsigned>::digits) &&
Michael Ilsemanaddddc42014-12-15 18:48:43 +00009881 "undefined behavior");
9882 unsigned Mask = (1u << Width) - 1;
Evan Chengc1778132010-12-14 03:22:07 +00009883 unsigned Mask2 = N11C->getZExtValue();
Evan Cheng6d02d902011-06-15 01:12:31 +00009884 if ((Mask & (~Mask2)) == 0)
Andrew Trickef9de2a2013-05-25 02:42:55 +00009885 return DCI.DAG.getNode(ARMISD::BFI, SDLoc(N), N->getValueType(0),
Evan Chengc1778132010-12-14 03:22:07 +00009886 N->getOperand(0), N1.getOperand(0),
9887 N->getOperand(2));
James Molloyce12c922015-11-11 15:40:40 +00009888 } else if (N->getOperand(0).getOpcode() == ARMISD::BFI) {
9889 // We have a BFI of a BFI. Walk up the BFI chain to see how long it goes.
9890 // Keep track of any consecutive bits set that all come from the same base
9891 // value. We can combine these together into a single BFI.
9892 SDValue CombineBFI = FindBFIToCombineWith(N);
9893 if (CombineBFI == SDValue())
9894 return SDValue();
9895
9896 // We've found a BFI.
9897 APInt ToMask1, FromMask1;
9898 SDValue From1 = ParseBFI(N, ToMask1, FromMask1);
9899
9900 APInt ToMask2, FromMask2;
Diego Novillo0767ae52015-11-11 16:39:22 +00009901 SDValue From2 = ParseBFI(CombineBFI.getNode(), ToMask2, FromMask2);
9902 assert(From1 == From2);
9903 (void)From2;
Chad Rosier353d7192015-12-21 18:08:05 +00009904
James Molloyce12c922015-11-11 15:40:40 +00009905 // First, unlink CombineBFI.
9906 DCI.DAG.ReplaceAllUsesWith(CombineBFI, CombineBFI.getOperand(0));
9907 // Then create a new BFI, combining the two together.
9908 APInt NewFromMask = FromMask1 | FromMask2;
9909 APInt NewToMask = ToMask1 | ToMask2;
9910
9911 EVT VT = N->getValueType(0);
9912 SDLoc dl(N);
9913
9914 if (NewFromMask[0] == 0)
9915 From1 = DCI.DAG.getNode(
9916 ISD::SRL, dl, VT, From1,
9917 DCI.DAG.getConstant(NewFromMask.countTrailingZeros(), dl, VT));
9918 return DCI.DAG.getNode(ARMISD::BFI, dl, VT, N->getOperand(0), From1,
9919 DCI.DAG.getConstant(~NewToMask, dl, VT));
Evan Chengc1778132010-12-14 03:22:07 +00009920 }
9921 return SDValue();
9922}
9923
Bob Wilson22806742010-09-22 22:09:21 +00009924/// PerformVMOVRRDCombine - Target-specific dag combine xforms for
9925/// ARMISD::VMOVRRD.
9926static SDValue PerformVMOVRRDCombine(SDNode *N,
Oliver Stannard51b1d462014-08-21 12:50:31 +00009927 TargetLowering::DAGCombinerInfo &DCI,
9928 const ARMSubtarget *Subtarget) {
Bob Wilson22806742010-09-22 22:09:21 +00009929 // vmovrrd(vmovdrr x, y) -> x,y
9930 SDValue InDouble = N->getOperand(0);
Oliver Stannard51b1d462014-08-21 12:50:31 +00009931 if (InDouble.getOpcode() == ARMISD::VMOVDRR && !Subtarget->isFPOnlySP())
Bob Wilson22806742010-09-22 22:09:21 +00009932 return DCI.CombineTo(N, InDouble.getOperand(0), InDouble.getOperand(1));
Cameron Zwarich6fe5c292011-04-02 02:40:43 +00009933
9934 // vmovrrd(load f64) -> (load i32), (load i32)
9935 SDNode *InNode = InDouble.getNode();
9936 if (ISD::isNormalLoad(InNode) && InNode->hasOneUse() &&
9937 InNode->getValueType(0) == MVT::f64 &&
9938 InNode->getOperand(1).getOpcode() == ISD::FrameIndex &&
9939 !cast<LoadSDNode>(InNode)->isVolatile()) {
9940 // TODO: Should this be done for non-FrameIndex operands?
9941 LoadSDNode *LD = cast<LoadSDNode>(InNode);
9942
9943 SelectionDAG &DAG = DCI.DAG;
Andrew Trickef9de2a2013-05-25 02:42:55 +00009944 SDLoc DL(LD);
Cameron Zwarich6fe5c292011-04-02 02:40:43 +00009945 SDValue BasePtr = LD->getBasePtr();
Justin Lebar9c375812016-07-15 18:27:10 +00009946 SDValue NewLD1 =
9947 DAG.getLoad(MVT::i32, DL, LD->getChain(), BasePtr, LD->getPointerInfo(),
9948 LD->getAlignment(), LD->getMemOperand()->getFlags());
Cameron Zwarich6fe5c292011-04-02 02:40:43 +00009949
9950 SDValue OffsetPtr = DAG.getNode(ISD::ADD, DL, MVT::i32, BasePtr,
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +00009951 DAG.getConstant(4, DL, MVT::i32));
Justin Lebar9c375812016-07-15 18:27:10 +00009952 SDValue NewLD2 = DAG.getLoad(
9953 MVT::i32, DL, NewLD1.getValue(1), OffsetPtr, LD->getPointerInfo(),
9954 std::min(4U, LD->getAlignment() / 2), LD->getMemOperand()->getFlags());
Cameron Zwarich6fe5c292011-04-02 02:40:43 +00009955
9956 DAG.ReplaceAllUsesOfValueWith(SDValue(LD, 1), NewLD2.getValue(1));
Mehdi Aminiffc14022015-07-08 01:00:38 +00009957 if (DCI.DAG.getDataLayout().isBigEndian())
Christian Pirker762b2c62014-06-01 09:30:52 +00009958 std::swap (NewLD1, NewLD2);
Cameron Zwarich6fe5c292011-04-02 02:40:43 +00009959 SDValue Result = DCI.CombineTo(N, NewLD1, NewLD2);
Cameron Zwarich6fe5c292011-04-02 02:40:43 +00009960 return Result;
9961 }
9962
Bob Wilson22806742010-09-22 22:09:21 +00009963 return SDValue();
9964}
9965
9966/// PerformVMOVDRRCombine - Target-specific dag combine xforms for
9967/// ARMISD::VMOVDRR. This is also used for BUILD_VECTORs with 2 operands.
9968static SDValue PerformVMOVDRRCombine(SDNode *N, SelectionDAG &DAG) {
9969 // N=vmovrrd(X); vmovdrr(N:0, N:1) -> bit_convert(X)
9970 SDValue Op0 = N->getOperand(0);
9971 SDValue Op1 = N->getOperand(1);
Wesley Peck527da1b2010-11-23 03:31:01 +00009972 if (Op0.getOpcode() == ISD::BITCAST)
Bob Wilson22806742010-09-22 22:09:21 +00009973 Op0 = Op0.getOperand(0);
Wesley Peck527da1b2010-11-23 03:31:01 +00009974 if (Op1.getOpcode() == ISD::BITCAST)
Bob Wilson22806742010-09-22 22:09:21 +00009975 Op1 = Op1.getOperand(0);
9976 if (Op0.getOpcode() == ARMISD::VMOVRRD &&
9977 Op0.getNode() == Op1.getNode() &&
9978 Op0.getResNo() == 0 && Op1.getResNo() == 1)
Andrew Trickef9de2a2013-05-25 02:42:55 +00009979 return DAG.getNode(ISD::BITCAST, SDLoc(N),
Bob Wilson22806742010-09-22 22:09:21 +00009980 N->getValueType(0), Op0.getOperand(0));
9981 return SDValue();
9982}
9983
Bob Wilson1a20c2a2010-12-21 06:43:19 +00009984/// hasNormalLoadOperand - Check if any of the operands of a BUILD_VECTOR node
9985/// are normal, non-volatile loads. If so, it is profitable to bitcast an
9986/// i64 vector to have f64 elements, since the value can then be loaded
9987/// directly into a VFP register.
9988static bool hasNormalLoadOperand(SDNode *N) {
9989 unsigned NumElts = N->getValueType(0).getVectorNumElements();
9990 for (unsigned i = 0; i < NumElts; ++i) {
9991 SDNode *Elt = N->getOperand(i).getNode();
9992 if (ISD::isNormalLoad(Elt) && !cast<LoadSDNode>(Elt)->isVolatile())
9993 return true;
9994 }
9995 return false;
9996}
9997
Bob Wilsoncb6db982010-09-17 22:59:05 +00009998/// PerformBUILD_VECTORCombine - Target-specific dag combine xforms for
9999/// ISD::BUILD_VECTOR.
Bob Wilson1a20c2a2010-12-21 06:43:19 +000010000static SDValue PerformBUILD_VECTORCombine(SDNode *N,
Oliver Stannard51b1d462014-08-21 12:50:31 +000010001 TargetLowering::DAGCombinerInfo &DCI,
10002 const ARMSubtarget *Subtarget) {
Bob Wilsoncb6db982010-09-17 22:59:05 +000010003 // build_vector(N=ARMISD::VMOVRRD(X), N:1) -> bit_convert(X):
10004 // VMOVRRD is introduced when legalizing i64 types. It forces the i64 value
10005 // into a pair of GPRs, which is fine when the value is used as a scalar,
10006 // but if the i64 value is converted to a vector, we need to undo the VMOVRRD.
Bob Wilson1a20c2a2010-12-21 06:43:19 +000010007 SelectionDAG &DAG = DCI.DAG;
Ahmed Bougachaf8dfb472016-02-09 22:54:12 +000010008 if (N->getNumOperands() == 2)
10009 if (SDValue RV = PerformVMOVDRRCombine(N, DAG))
Bob Wilson1a20c2a2010-12-21 06:43:19 +000010010 return RV;
Bob Wilsoncb6db982010-09-17 22:59:05 +000010011
Bob Wilson1a20c2a2010-12-21 06:43:19 +000010012 // Load i64 elements as f64 values so that type legalization does not split
10013 // them up into i32 values.
10014 EVT VT = N->getValueType(0);
10015 if (VT.getVectorElementType() != MVT::i64 || !hasNormalLoadOperand(N))
10016 return SDValue();
Andrew Trickef9de2a2013-05-25 02:42:55 +000010017 SDLoc dl(N);
Bob Wilson1a20c2a2010-12-21 06:43:19 +000010018 SmallVector<SDValue, 8> Ops;
10019 unsigned NumElts = VT.getVectorNumElements();
10020 for (unsigned i = 0; i < NumElts; ++i) {
10021 SDValue V = DAG.getNode(ISD::BITCAST, dl, MVT::f64, N->getOperand(i));
10022 Ops.push_back(V);
10023 // Make the DAGCombiner fold the bitcast.
10024 DCI.AddToWorklist(V.getNode());
10025 }
10026 EVT FloatVT = EVT::getVectorVT(*DAG.getContext(), MVT::f64, NumElts);
Ahmed Bougacha128f8732016-04-26 21:15:30 +000010027 SDValue BV = DAG.getBuildVector(FloatVT, dl, Ops);
Bob Wilson1a20c2a2010-12-21 06:43:19 +000010028 return DAG.getNode(ISD::BITCAST, dl, VT, BV);
10029}
10030
Quentin Colombet04b3a0f2013-07-03 21:42:57 +000010031/// \brief Target-specific dag combine xforms for ARMISD::BUILD_VECTOR.
10032static SDValue
10033PerformARMBUILD_VECTORCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI) {
10034 // ARMISD::BUILD_VECTOR is introduced when legalizing ISD::BUILD_VECTOR.
10035 // At that time, we may have inserted bitcasts from integer to float.
10036 // If these bitcasts have survived DAGCombine, change the lowering of this
10037 // BUILD_VECTOR in something more vector friendly, i.e., that does not
10038 // force to use floating point types.
10039
10040 // Make sure we can change the type of the vector.
10041 // This is possible iff:
10042 // 1. The vector is only used in a bitcast to a integer type. I.e.,
10043 // 1.1. Vector is used only once.
10044 // 1.2. Use is a bit convert to an integer type.
10045 // 2. The size of its operands are 32-bits (64-bits are not legal).
10046 EVT VT = N->getValueType(0);
10047 EVT EltVT = VT.getVectorElementType();
10048
10049 // Check 1.1. and 2.
10050 if (EltVT.getSizeInBits() != 32 || !N->hasOneUse())
10051 return SDValue();
10052
10053 // By construction, the input type must be float.
10054 assert(EltVT == MVT::f32 && "Unexpected type!");
10055
10056 // Check 1.2.
10057 SDNode *Use = *N->use_begin();
10058 if (Use->getOpcode() != ISD::BITCAST ||
10059 Use->getValueType(0).isFloatingPoint())
10060 return SDValue();
10061
10062 // Check profitability.
10063 // Model is, if more than half of the relevant operands are bitcast from
10064 // i32, turn the build_vector into a sequence of insert_vector_elt.
10065 // Relevant operands are everything that is not statically
10066 // (i.e., at compile time) bitcasted.
10067 unsigned NumOfBitCastedElts = 0;
10068 unsigned NumElts = VT.getVectorNumElements();
10069 unsigned NumOfRelevantElts = NumElts;
10070 for (unsigned Idx = 0; Idx < NumElts; ++Idx) {
10071 SDValue Elt = N->getOperand(Idx);
10072 if (Elt->getOpcode() == ISD::BITCAST) {
10073 // Assume only bit cast to i32 will go away.
10074 if (Elt->getOperand(0).getValueType() == MVT::i32)
10075 ++NumOfBitCastedElts;
Sanjay Patel57195842016-03-14 17:28:46 +000010076 } else if (Elt.isUndef() || isa<ConstantSDNode>(Elt))
Quentin Colombet04b3a0f2013-07-03 21:42:57 +000010077 // Constants are statically casted, thus do not count them as
10078 // relevant operands.
10079 --NumOfRelevantElts;
10080 }
10081
10082 // Check if more than half of the elements require a non-free bitcast.
10083 if (NumOfBitCastedElts <= NumOfRelevantElts / 2)
10084 return SDValue();
10085
10086 SelectionDAG &DAG = DCI.DAG;
10087 // Create the new vector type.
10088 EVT VecVT = EVT::getVectorVT(*DAG.getContext(), MVT::i32, NumElts);
10089 // Check if the type is legal.
10090 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
10091 if (!TLI.isTypeLegal(VecVT))
10092 return SDValue();
10093
10094 // Combine:
10095 // ARMISD::BUILD_VECTOR E1, E2, ..., EN.
10096 // => BITCAST INSERT_VECTOR_ELT
10097 // (INSERT_VECTOR_ELT (...), (BITCAST EN-1), N-1),
10098 // (BITCAST EN), N.
10099 SDValue Vec = DAG.getUNDEF(VecVT);
10100 SDLoc dl(N);
10101 for (unsigned Idx = 0 ; Idx < NumElts; ++Idx) {
10102 SDValue V = N->getOperand(Idx);
Sanjay Patel57195842016-03-14 17:28:46 +000010103 if (V.isUndef())
Quentin Colombet04b3a0f2013-07-03 21:42:57 +000010104 continue;
10105 if (V.getOpcode() == ISD::BITCAST &&
10106 V->getOperand(0).getValueType() == MVT::i32)
10107 // Fold obvious case.
10108 V = V.getOperand(0);
10109 else {
Jim Grosbach1a597112014-04-03 23:43:18 +000010110 V = DAG.getNode(ISD::BITCAST, SDLoc(V), MVT::i32, V);
Quentin Colombet04b3a0f2013-07-03 21:42:57 +000010111 // Make the DAGCombiner fold the bitcasts.
10112 DCI.AddToWorklist(V.getNode());
10113 }
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +000010114 SDValue LaneIdx = DAG.getConstant(Idx, dl, MVT::i32);
Quentin Colombet04b3a0f2013-07-03 21:42:57 +000010115 Vec = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, VecVT, Vec, V, LaneIdx);
10116 }
10117 Vec = DAG.getNode(ISD::BITCAST, dl, VT, Vec);
10118 // Make the DAGCombiner fold the bitcasts.
10119 DCI.AddToWorklist(Vec.getNode());
10120 return Vec;
10121}
10122
Bob Wilson1a20c2a2010-12-21 06:43:19 +000010123/// PerformInsertEltCombine - Target-specific dag combine xforms for
10124/// ISD::INSERT_VECTOR_ELT.
10125static SDValue PerformInsertEltCombine(SDNode *N,
10126 TargetLowering::DAGCombinerInfo &DCI) {
10127 // Bitcast an i64 load inserted into a vector to f64.
10128 // Otherwise, the i64 value will be legalized to a pair of i32 values.
10129 EVT VT = N->getValueType(0);
10130 SDNode *Elt = N->getOperand(1).getNode();
10131 if (VT.getVectorElementType() != MVT::i64 ||
10132 !ISD::isNormalLoad(Elt) || cast<LoadSDNode>(Elt)->isVolatile())
10133 return SDValue();
10134
10135 SelectionDAG &DAG = DCI.DAG;
Andrew Trickef9de2a2013-05-25 02:42:55 +000010136 SDLoc dl(N);
Bob Wilson1a20c2a2010-12-21 06:43:19 +000010137 EVT FloatVT = EVT::getVectorVT(*DAG.getContext(), MVT::f64,
10138 VT.getVectorNumElements());
10139 SDValue Vec = DAG.getNode(ISD::BITCAST, dl, FloatVT, N->getOperand(0));
10140 SDValue V = DAG.getNode(ISD::BITCAST, dl, MVT::f64, N->getOperand(1));
10141 // Make the DAGCombiner fold the bitcasts.
10142 DCI.AddToWorklist(Vec.getNode());
10143 DCI.AddToWorklist(V.getNode());
10144 SDValue InsElt = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, FloatVT,
10145 Vec, V, N->getOperand(2));
10146 return DAG.getNode(ISD::BITCAST, dl, VT, InsElt);
Bob Wilsoncb6db982010-09-17 22:59:05 +000010147}
10148
Bob Wilsonc7334a12010-10-27 20:38:28 +000010149/// PerformVECTOR_SHUFFLECombine - Target-specific dag combine xforms for
10150/// ISD::VECTOR_SHUFFLE.
10151static SDValue PerformVECTOR_SHUFFLECombine(SDNode *N, SelectionDAG &DAG) {
10152 // The LLVM shufflevector instruction does not require the shuffle mask
10153 // length to match the operand vector length, but ISD::VECTOR_SHUFFLE does
10154 // have that requirement. When translating to ISD::VECTOR_SHUFFLE, if the
10155 // operands do not match the mask length, they are extended by concatenating
10156 // them with undef vectors. That is probably the right thing for other
10157 // targets, but for NEON it is better to concatenate two double-register
10158 // size vector operands into a single quad-register size vector. Do that
10159 // transformation here:
10160 // shuffle(concat(v1, undef), concat(v2, undef)) ->
10161 // shuffle(concat(v1, v2), undef)
10162 SDValue Op0 = N->getOperand(0);
10163 SDValue Op1 = N->getOperand(1);
10164 if (Op0.getOpcode() != ISD::CONCAT_VECTORS ||
10165 Op1.getOpcode() != ISD::CONCAT_VECTORS ||
10166 Op0.getNumOperands() != 2 ||
10167 Op1.getNumOperands() != 2)
10168 return SDValue();
10169 SDValue Concat0Op1 = Op0.getOperand(1);
10170 SDValue Concat1Op1 = Op1.getOperand(1);
Sanjay Patel75068522016-03-14 18:09:43 +000010171 if (!Concat0Op1.isUndef() || !Concat1Op1.isUndef())
Bob Wilsonc7334a12010-10-27 20:38:28 +000010172 return SDValue();
10173 // Skip the transformation if any of the types are illegal.
10174 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
10175 EVT VT = N->getValueType(0);
10176 if (!TLI.isTypeLegal(VT) ||
10177 !TLI.isTypeLegal(Concat0Op1.getValueType()) ||
10178 !TLI.isTypeLegal(Concat1Op1.getValueType()))
10179 return SDValue();
10180
Andrew Trickef9de2a2013-05-25 02:42:55 +000010181 SDValue NewConcat = DAG.getNode(ISD::CONCAT_VECTORS, SDLoc(N), VT,
Bob Wilsonc7334a12010-10-27 20:38:28 +000010182 Op0.getOperand(0), Op1.getOperand(0));
10183 // Translate the shuffle mask.
10184 SmallVector<int, 16> NewMask;
10185 unsigned NumElts = VT.getVectorNumElements();
10186 unsigned HalfElts = NumElts/2;
10187 ShuffleVectorSDNode *SVN = cast<ShuffleVectorSDNode>(N);
10188 for (unsigned n = 0; n < NumElts; ++n) {
10189 int MaskElt = SVN->getMaskElt(n);
10190 int NewElt = -1;
Bob Wilson6c550072010-10-27 23:49:00 +000010191 if (MaskElt < (int)HalfElts)
Bob Wilsonc7334a12010-10-27 20:38:28 +000010192 NewElt = MaskElt;
Bob Wilson6c550072010-10-27 23:49:00 +000010193 else if (MaskElt >= (int)NumElts && MaskElt < (int)(NumElts + HalfElts))
Bob Wilsonc7334a12010-10-27 20:38:28 +000010194 NewElt = HalfElts + MaskElt - NumElts;
10195 NewMask.push_back(NewElt);
10196 }
Andrew Trickef9de2a2013-05-25 02:42:55 +000010197 return DAG.getVectorShuffle(VT, SDLoc(N), NewConcat,
Craig Topper2bd8b4b2016-07-01 06:54:47 +000010198 DAG.getUNDEF(VT), NewMask);
Bob Wilsonc7334a12010-10-27 20:38:28 +000010199}
10200
Ahmed Bougachadb141ac2015-02-19 23:52:41 +000010201/// CombineBaseUpdate - Target-specific DAG combine function for VLDDUP,
10202/// NEON load/store intrinsics, and generic vector load/stores, to merge
10203/// base address updates.
10204/// For generic load/stores, the memory type is assumed to be a vector.
10205/// The caller is assumed to have checked legality.
Bob Wilson06fce872011-02-07 17:43:21 +000010206static SDValue CombineBaseUpdate(SDNode *N,
10207 TargetLowering::DAGCombinerInfo &DCI) {
Bob Wilson06fce872011-02-07 17:43:21 +000010208 SelectionDAG &DAG = DCI.DAG;
Ahmed Bougachadfdf54b2015-02-19 23:30:37 +000010209 const bool isIntrinsic = (N->getOpcode() == ISD::INTRINSIC_VOID ||
10210 N->getOpcode() == ISD::INTRINSIC_W_CHAIN);
Ahmed Bougachadb141ac2015-02-19 23:52:41 +000010211 const bool isStore = N->getOpcode() == ISD::STORE;
10212 const unsigned AddrOpIdx = ((isIntrinsic || isStore) ? 2 : 1);
Bob Wilson06fce872011-02-07 17:43:21 +000010213 SDValue Addr = N->getOperand(AddrOpIdx);
Ahmed Bougachadfdf54b2015-02-19 23:30:37 +000010214 MemSDNode *MemN = cast<MemSDNode>(N);
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +000010215 SDLoc dl(N);
Bob Wilson06fce872011-02-07 17:43:21 +000010216
10217 // Search for a use of the address operand that is an increment.
10218 for (SDNode::use_iterator UI = Addr.getNode()->use_begin(),
10219 UE = Addr.getNode()->use_end(); UI != UE; ++UI) {
10220 SDNode *User = *UI;
10221 if (User->getOpcode() != ISD::ADD ||
10222 UI.getUse().getResNo() != Addr.getResNo())
10223 continue;
10224
10225 // Check that the add is independent of the load/store. Otherwise, folding
10226 // it would create a cycle.
10227 if (User->isPredecessorOf(N) || N->isPredecessorOf(User))
10228 continue;
10229
10230 // Find the new opcode for the updating load/store.
Ahmed Bougachadfdf54b2015-02-19 23:30:37 +000010231 bool isLoadOp = true;
Bob Wilson06fce872011-02-07 17:43:21 +000010232 bool isLaneOp = false;
10233 unsigned NewOpc = 0;
10234 unsigned NumVecs = 0;
10235 if (isIntrinsic) {
10236 unsigned IntNo = cast<ConstantSDNode>(N->getOperand(1))->getZExtValue();
10237 switch (IntNo) {
Craig Toppere55c5562012-02-07 02:50:20 +000010238 default: llvm_unreachable("unexpected intrinsic for Neon base update");
Bob Wilson06fce872011-02-07 17:43:21 +000010239 case Intrinsic::arm_neon_vld1: NewOpc = ARMISD::VLD1_UPD;
10240 NumVecs = 1; break;
10241 case Intrinsic::arm_neon_vld2: NewOpc = ARMISD::VLD2_UPD;
10242 NumVecs = 2; break;
10243 case Intrinsic::arm_neon_vld3: NewOpc = ARMISD::VLD3_UPD;
10244 NumVecs = 3; break;
10245 case Intrinsic::arm_neon_vld4: NewOpc = ARMISD::VLD4_UPD;
10246 NumVecs = 4; break;
10247 case Intrinsic::arm_neon_vld2lane: NewOpc = ARMISD::VLD2LN_UPD;
10248 NumVecs = 2; isLaneOp = true; break;
10249 case Intrinsic::arm_neon_vld3lane: NewOpc = ARMISD::VLD3LN_UPD;
10250 NumVecs = 3; isLaneOp = true; break;
10251 case Intrinsic::arm_neon_vld4lane: NewOpc = ARMISD::VLD4LN_UPD;
10252 NumVecs = 4; isLaneOp = true; break;
10253 case Intrinsic::arm_neon_vst1: NewOpc = ARMISD::VST1_UPD;
Ahmed Bougachadfdf54b2015-02-19 23:30:37 +000010254 NumVecs = 1; isLoadOp = false; break;
Bob Wilson06fce872011-02-07 17:43:21 +000010255 case Intrinsic::arm_neon_vst2: NewOpc = ARMISD::VST2_UPD;
Ahmed Bougachadfdf54b2015-02-19 23:30:37 +000010256 NumVecs = 2; isLoadOp = false; break;
Bob Wilson06fce872011-02-07 17:43:21 +000010257 case Intrinsic::arm_neon_vst3: NewOpc = ARMISD::VST3_UPD;
Ahmed Bougachadfdf54b2015-02-19 23:30:37 +000010258 NumVecs = 3; isLoadOp = false; break;
Bob Wilson06fce872011-02-07 17:43:21 +000010259 case Intrinsic::arm_neon_vst4: NewOpc = ARMISD::VST4_UPD;
Ahmed Bougachadfdf54b2015-02-19 23:30:37 +000010260 NumVecs = 4; isLoadOp = false; break;
Bob Wilson06fce872011-02-07 17:43:21 +000010261 case Intrinsic::arm_neon_vst2lane: NewOpc = ARMISD::VST2LN_UPD;
Ahmed Bougachadfdf54b2015-02-19 23:30:37 +000010262 NumVecs = 2; isLoadOp = false; isLaneOp = true; break;
Bob Wilson06fce872011-02-07 17:43:21 +000010263 case Intrinsic::arm_neon_vst3lane: NewOpc = ARMISD::VST3LN_UPD;
Ahmed Bougachadfdf54b2015-02-19 23:30:37 +000010264 NumVecs = 3; isLoadOp = false; isLaneOp = true; break;
Bob Wilson06fce872011-02-07 17:43:21 +000010265 case Intrinsic::arm_neon_vst4lane: NewOpc = ARMISD::VST4LN_UPD;
Ahmed Bougachadfdf54b2015-02-19 23:30:37 +000010266 NumVecs = 4; isLoadOp = false; isLaneOp = true; break;
Bob Wilson06fce872011-02-07 17:43:21 +000010267 }
10268 } else {
10269 isLaneOp = true;
10270 switch (N->getOpcode()) {
Craig Toppere55c5562012-02-07 02:50:20 +000010271 default: llvm_unreachable("unexpected opcode for Neon base update");
Bob Wilson06fce872011-02-07 17:43:21 +000010272 case ARMISD::VLD2DUP: NewOpc = ARMISD::VLD2DUP_UPD; NumVecs = 2; break;
10273 case ARMISD::VLD3DUP: NewOpc = ARMISD::VLD3DUP_UPD; NumVecs = 3; break;
10274 case ARMISD::VLD4DUP: NewOpc = ARMISD::VLD4DUP_UPD; NumVecs = 4; break;
Ahmed Bougachadb141ac2015-02-19 23:52:41 +000010275 case ISD::LOAD: NewOpc = ARMISD::VLD1_UPD;
10276 NumVecs = 1; isLaneOp = false; break;
10277 case ISD::STORE: NewOpc = ARMISD::VST1_UPD;
10278 NumVecs = 1; isLaneOp = false; isLoadOp = false; break;
Bob Wilson06fce872011-02-07 17:43:21 +000010279 }
10280 }
10281
10282 // Find the size of memory referenced by the load/store.
10283 EVT VecTy;
Ahmed Bougachadb141ac2015-02-19 23:52:41 +000010284 if (isLoadOp) {
Bob Wilson06fce872011-02-07 17:43:21 +000010285 VecTy = N->getValueType(0);
Ahmed Bougachadb141ac2015-02-19 23:52:41 +000010286 } else if (isIntrinsic) {
Renato Golin2a5c0a52015-02-04 10:11:59 +000010287 VecTy = N->getOperand(AddrOpIdx+1).getValueType();
Ahmed Bougachadb141ac2015-02-19 23:52:41 +000010288 } else {
10289 assert(isStore && "Node has to be a load, a store, or an intrinsic!");
10290 VecTy = N->getOperand(1).getValueType();
10291 }
10292
Bob Wilson06fce872011-02-07 17:43:21 +000010293 unsigned NumBytes = NumVecs * VecTy.getSizeInBits() / 8;
10294 if (isLaneOp)
10295 NumBytes /= VecTy.getVectorNumElements();
10296
10297 // If the increment is a constant, it must match the memory ref size.
10298 SDValue Inc = User->getOperand(User->getOperand(0) == Addr ? 1 : 0);
10299 if (ConstantSDNode *CInc = dyn_cast<ConstantSDNode>(Inc.getNode())) {
10300 uint64_t IncVal = CInc->getZExtValue();
10301 if (IncVal != NumBytes)
10302 continue;
10303 } else if (NumBytes >= 3 * 16) {
10304 // VLD3/4 and VST3/4 for 128-bit vectors are implemented with two
10305 // separate instructions that make it harder to use a non-constant update.
10306 continue;
10307 }
10308
Ahmed Bougachadb141ac2015-02-19 23:52:41 +000010309 // OK, we found an ADD we can fold into the base update.
10310 // Now, create a _UPD node, taking care of not breaking alignment.
10311
10312 EVT AlignedVecTy = VecTy;
10313 unsigned Alignment = MemN->getAlignment();
10314
10315 // If this is a less-than-standard-aligned load/store, change the type to
10316 // match the standard alignment.
10317 // The alignment is overlooked when selecting _UPD variants; and it's
10318 // easier to introduce bitcasts here than fix that.
10319 // There are 3 ways to get to this base-update combine:
10320 // - intrinsics: they are assumed to be properly aligned (to the standard
10321 // alignment of the memory type), so we don't need to do anything.
10322 // - ARMISD::VLDx nodes: they are only generated from the aforementioned
10323 // intrinsics, so, likewise, there's nothing to do.
10324 // - generic load/store instructions: the alignment is specified as an
10325 // explicit operand, rather than implicitly as the standard alignment
10326 // of the memory type (like the intrisics). We need to change the
10327 // memory type to match the explicit alignment. That way, we don't
10328 // generate non-standard-aligned ARMISD::VLDx nodes.
10329 if (isa<LSBaseSDNode>(N)) {
10330 if (Alignment == 0)
10331 Alignment = 1;
10332 if (Alignment < VecTy.getScalarSizeInBits() / 8) {
10333 MVT EltTy = MVT::getIntegerVT(Alignment * 8);
10334 assert(NumVecs == 1 && "Unexpected multi-element generic load/store.");
10335 assert(!isLaneOp && "Unexpected generic load/store lane.");
10336 unsigned NumElts = NumBytes / (EltTy.getSizeInBits() / 8);
10337 AlignedVecTy = MVT::getVectorVT(EltTy, NumElts);
10338 }
10339 // Don't set an explicit alignment on regular load/stores that we want
10340 // to transform to VLD/VST 1_UPD nodes.
10341 // This matches the behavior of regular load/stores, which only get an
10342 // explicit alignment if the MMO alignment is larger than the standard
10343 // alignment of the memory type.
10344 // Intrinsics, however, always get an explicit alignment, set to the
10345 // alignment of the MMO.
10346 Alignment = 1;
10347 }
10348
Bob Wilson06fce872011-02-07 17:43:21 +000010349 // Create the new updating load/store node.
Ahmed Bougachadfdf54b2015-02-19 23:30:37 +000010350 // First, create an SDVTList for the new updating node's results.
Bob Wilson06fce872011-02-07 17:43:21 +000010351 EVT Tys[6];
Ahmed Bougachadfdf54b2015-02-19 23:30:37 +000010352 unsigned NumResultVecs = (isLoadOp ? NumVecs : 0);
Bob Wilson06fce872011-02-07 17:43:21 +000010353 unsigned n;
10354 for (n = 0; n < NumResultVecs; ++n)
Ahmed Bougachadb141ac2015-02-19 23:52:41 +000010355 Tys[n] = AlignedVecTy;
Bob Wilson06fce872011-02-07 17:43:21 +000010356 Tys[n++] = MVT::i32;
10357 Tys[n] = MVT::Other;
Craig Toppere1d12942014-08-27 05:25:25 +000010358 SDVTList SDTys = DAG.getVTList(makeArrayRef(Tys, NumResultVecs+2));
Ahmed Bougachadfdf54b2015-02-19 23:30:37 +000010359
10360 // Then, gather the new node's operands.
Bob Wilson06fce872011-02-07 17:43:21 +000010361 SmallVector<SDValue, 8> Ops;
10362 Ops.push_back(N->getOperand(0)); // incoming chain
10363 Ops.push_back(N->getOperand(AddrOpIdx));
10364 Ops.push_back(Inc);
Ahmed Bougachadb141ac2015-02-19 23:52:41 +000010365
10366 if (StoreSDNode *StN = dyn_cast<StoreSDNode>(N)) {
10367 // Try to match the intrinsic's signature
10368 Ops.push_back(StN->getValue());
10369 } else {
10370 // Loads (and of course intrinsics) match the intrinsics' signature,
10371 // so just add all but the alignment operand.
10372 for (unsigned i = AddrOpIdx + 1; i < N->getNumOperands() - 1; ++i)
10373 Ops.push_back(N->getOperand(i));
10374 }
10375
10376 // For all node types, the alignment operand is always the last one.
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +000010377 Ops.push_back(DAG.getConstant(Alignment, dl, MVT::i32));
Ahmed Bougachadb141ac2015-02-19 23:52:41 +000010378
10379 // If this is a non-standard-aligned STORE, the penultimate operand is the
10380 // stored value. Bitcast it to the aligned type.
10381 if (AlignedVecTy != VecTy && N->getOpcode() == ISD::STORE) {
10382 SDValue &StVal = Ops[Ops.size()-2];
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +000010383 StVal = DAG.getNode(ISD::BITCAST, dl, AlignedVecTy, StVal);
Ahmed Bougachadb141ac2015-02-19 23:52:41 +000010384 }
Ahmed Bougachadfdf54b2015-02-19 23:30:37 +000010385
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +000010386 SDValue UpdN = DAG.getMemIntrinsicNode(NewOpc, dl, SDTys,
Ahmed Bougachadb141ac2015-02-19 23:52:41 +000010387 Ops, AlignedVecTy,
Ahmed Bougachadfdf54b2015-02-19 23:30:37 +000010388 MemN->getMemOperand());
Bob Wilson06fce872011-02-07 17:43:21 +000010389
10390 // Update the uses.
Ahmed Bougacha4c2b0782015-02-19 23:13:10 +000010391 SmallVector<SDValue, 5> NewResults;
Ahmed Bougachadfdf54b2015-02-19 23:30:37 +000010392 for (unsigned i = 0; i < NumResultVecs; ++i)
Bob Wilson06fce872011-02-07 17:43:21 +000010393 NewResults.push_back(SDValue(UpdN.getNode(), i));
Ahmed Bougachadb141ac2015-02-19 23:52:41 +000010394
10395 // If this is an non-standard-aligned LOAD, the first result is the loaded
10396 // value. Bitcast it to the expected result type.
10397 if (AlignedVecTy != VecTy && N->getOpcode() == ISD::LOAD) {
10398 SDValue &LdVal = NewResults[0];
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +000010399 LdVal = DAG.getNode(ISD::BITCAST, dl, VecTy, LdVal);
Ahmed Bougachadb141ac2015-02-19 23:52:41 +000010400 }
10401
Bob Wilson06fce872011-02-07 17:43:21 +000010402 NewResults.push_back(SDValue(UpdN.getNode(), NumResultVecs+1)); // chain
10403 DCI.CombineTo(N, NewResults);
10404 DCI.CombineTo(User, SDValue(UpdN.getNode(), NumResultVecs));
10405
10406 break;
Owen Anderson77aa2662011-04-05 21:48:57 +000010407 }
Bob Wilson06fce872011-02-07 17:43:21 +000010408 return SDValue();
10409}
10410
Ahmed Bougachadb141ac2015-02-19 23:52:41 +000010411static SDValue PerformVLDCombine(SDNode *N,
10412 TargetLowering::DAGCombinerInfo &DCI) {
10413 if (DCI.isBeforeLegalize() || DCI.isCalledByLegalizer())
10414 return SDValue();
10415
10416 return CombineBaseUpdate(N, DCI);
10417}
10418
Bob Wilson2d790df2010-11-28 06:51:26 +000010419/// CombineVLDDUP - For a VDUPLANE node N, check if its source operand is a
10420/// vldN-lane (N > 1) intrinsic, and if all the other uses of that intrinsic
10421/// are also VDUPLANEs. If so, combine them to a vldN-dup operation and
10422/// return true.
10423static bool CombineVLDDUP(SDNode *N, TargetLowering::DAGCombinerInfo &DCI) {
10424 SelectionDAG &DAG = DCI.DAG;
10425 EVT VT = N->getValueType(0);
10426 // vldN-dup instructions only support 64-bit vectors for N > 1.
10427 if (!VT.is64BitVector())
10428 return false;
10429
10430 // Check if the VDUPLANE operand is a vldN-dup intrinsic.
10431 SDNode *VLD = N->getOperand(0).getNode();
10432 if (VLD->getOpcode() != ISD::INTRINSIC_W_CHAIN)
10433 return false;
10434 unsigned NumVecs = 0;
10435 unsigned NewOpc = 0;
10436 unsigned IntNo = cast<ConstantSDNode>(VLD->getOperand(1))->getZExtValue();
10437 if (IntNo == Intrinsic::arm_neon_vld2lane) {
10438 NumVecs = 2;
10439 NewOpc = ARMISD::VLD2DUP;
10440 } else if (IntNo == Intrinsic::arm_neon_vld3lane) {
10441 NumVecs = 3;
10442 NewOpc = ARMISD::VLD3DUP;
10443 } else if (IntNo == Intrinsic::arm_neon_vld4lane) {
10444 NumVecs = 4;
10445 NewOpc = ARMISD::VLD4DUP;
10446 } else {
10447 return false;
10448 }
10449
10450 // First check that all the vldN-lane uses are VDUPLANEs and that the lane
10451 // numbers match the load.
10452 unsigned VLDLaneNo =
10453 cast<ConstantSDNode>(VLD->getOperand(NumVecs+3))->getZExtValue();
10454 for (SDNode::use_iterator UI = VLD->use_begin(), UE = VLD->use_end();
10455 UI != UE; ++UI) {
10456 // Ignore uses of the chain result.
10457 if (UI.getUse().getResNo() == NumVecs)
10458 continue;
10459 SDNode *User = *UI;
10460 if (User->getOpcode() != ARMISD::VDUPLANE ||
10461 VLDLaneNo != cast<ConstantSDNode>(User->getOperand(1))->getZExtValue())
10462 return false;
10463 }
10464
10465 // Create the vldN-dup node.
10466 EVT Tys[5];
10467 unsigned n;
10468 for (n = 0; n < NumVecs; ++n)
10469 Tys[n] = VT;
10470 Tys[n] = MVT::Other;
Craig Toppere1d12942014-08-27 05:25:25 +000010471 SDVTList SDTys = DAG.getVTList(makeArrayRef(Tys, NumVecs+1));
Bob Wilson2d790df2010-11-28 06:51:26 +000010472 SDValue Ops[] = { VLD->getOperand(0), VLD->getOperand(2) };
10473 MemIntrinsicSDNode *VLDMemInt = cast<MemIntrinsicSDNode>(VLD);
Andrew Trickef9de2a2013-05-25 02:42:55 +000010474 SDValue VLDDup = DAG.getMemIntrinsicNode(NewOpc, SDLoc(VLD), SDTys,
Craig Topper206fcd42014-04-26 19:29:41 +000010475 Ops, VLDMemInt->getMemoryVT(),
Bob Wilson2d790df2010-11-28 06:51:26 +000010476 VLDMemInt->getMemOperand());
10477
10478 // Update the uses.
10479 for (SDNode::use_iterator UI = VLD->use_begin(), UE = VLD->use_end();
10480 UI != UE; ++UI) {
10481 unsigned ResNo = UI.getUse().getResNo();
10482 // Ignore uses of the chain result.
10483 if (ResNo == NumVecs)
10484 continue;
10485 SDNode *User = *UI;
10486 DCI.CombineTo(User, SDValue(VLDDup.getNode(), ResNo));
10487 }
10488
10489 // Now the vldN-lane intrinsic is dead except for its chain result.
10490 // Update uses of the chain.
10491 std::vector<SDValue> VLDDupResults;
10492 for (unsigned n = 0; n < NumVecs; ++n)
10493 VLDDupResults.push_back(SDValue(VLDDup.getNode(), n));
10494 VLDDupResults.push_back(SDValue(VLDDup.getNode(), NumVecs));
10495 DCI.CombineTo(VLD, VLDDupResults);
10496
10497 return true;
10498}
10499
Bob Wilson103a0dc2010-07-14 01:22:12 +000010500/// PerformVDUPLANECombine - Target-specific dag combine xforms for
10501/// ARMISD::VDUPLANE.
Bob Wilson2d790df2010-11-28 06:51:26 +000010502static SDValue PerformVDUPLANECombine(SDNode *N,
10503 TargetLowering::DAGCombinerInfo &DCI) {
Bob Wilson103a0dc2010-07-14 01:22:12 +000010504 SDValue Op = N->getOperand(0);
Bob Wilson103a0dc2010-07-14 01:22:12 +000010505
Bob Wilson2d790df2010-11-28 06:51:26 +000010506 // If the source is a vldN-lane (N > 1) intrinsic, and all the other uses
10507 // of that intrinsic are also VDUPLANEs, combine them to a vldN-dup operation.
10508 if (CombineVLDDUP(N, DCI))
10509 return SDValue(N, 0);
10510
10511 // If the source is already a VMOVIMM or VMVNIMM splat, the VDUPLANE is
10512 // redundant. Ignore bit_converts for now; element sizes are checked below.
Wesley Peck527da1b2010-11-23 03:31:01 +000010513 while (Op.getOpcode() == ISD::BITCAST)
Bob Wilson103a0dc2010-07-14 01:22:12 +000010514 Op = Op.getOperand(0);
Bob Wilsonbad47f62010-07-14 06:31:50 +000010515 if (Op.getOpcode() != ARMISD::VMOVIMM && Op.getOpcode() != ARMISD::VMVNIMM)
Bob Wilson103a0dc2010-07-14 01:22:12 +000010516 return SDValue();
10517
10518 // Make sure the VMOV element size is not bigger than the VDUPLANE elements.
10519 unsigned EltSize = Op.getValueType().getVectorElementType().getSizeInBits();
10520 // The canonical VMOV for a zero vector uses a 32-bit element size.
10521 unsigned Imm = cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue();
10522 unsigned EltBits;
10523 if (ARM_AM::decodeNEONModImm(Imm, EltBits) == 0)
10524 EltSize = 8;
Bob Wilson2d790df2010-11-28 06:51:26 +000010525 EVT VT = N->getValueType(0);
Bob Wilson103a0dc2010-07-14 01:22:12 +000010526 if (EltSize > VT.getVectorElementType().getSizeInBits())
10527 return SDValue();
10528
Andrew Trickef9de2a2013-05-25 02:42:55 +000010529 return DCI.DAG.getNode(ISD::BITCAST, SDLoc(N), VT, Op);
Bob Wilson103a0dc2010-07-14 01:22:12 +000010530}
10531
Ahmed Bougachadb141ac2015-02-19 23:52:41 +000010532static SDValue PerformLOADCombine(SDNode *N,
10533 TargetLowering::DAGCombinerInfo &DCI) {
10534 EVT VT = N->getValueType(0);
10535
10536 // If this is a legal vector load, try to combine it into a VLD1_UPD.
10537 if (ISD::isNormalLoad(N) && VT.isVector() &&
10538 DCI.DAG.getTargetLoweringInfo().isTypeLegal(VT))
10539 return CombineBaseUpdate(N, DCI);
10540
10541 return SDValue();
10542}
10543
Ahmed Bougacha23167462014-12-09 21:26:53 +000010544/// PerformSTORECombine - Target-specific dag combine xforms for
10545/// ISD::STORE.
10546static SDValue PerformSTORECombine(SDNode *N,
10547 TargetLowering::DAGCombinerInfo &DCI) {
10548 StoreSDNode *St = cast<StoreSDNode>(N);
10549 if (St->isVolatile())
10550 return SDValue();
10551
10552 // Optimize trunc store (of multiple scalars) to shuffle and store. First,
10553 // pack all of the elements in one place. Next, store to memory in fewer
10554 // chunks.
10555 SDValue StVal = St->getValue();
10556 EVT VT = StVal.getValueType();
10557 if (St->isTruncatingStore() && VT.isVector()) {
10558 SelectionDAG &DAG = DCI.DAG;
10559 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
10560 EVT StVT = St->getMemoryVT();
10561 unsigned NumElems = VT.getVectorNumElements();
10562 assert(StVT != VT && "Cannot truncate to the same type");
10563 unsigned FromEltSz = VT.getVectorElementType().getSizeInBits();
10564 unsigned ToEltSz = StVT.getVectorElementType().getSizeInBits();
10565
10566 // From, To sizes and ElemCount must be pow of two
10567 if (!isPowerOf2_32(NumElems * FromEltSz * ToEltSz)) return SDValue();
10568
10569 // We are going to use the original vector elt for storing.
10570 // Accumulated smaller vector elements must be a multiple of the store size.
10571 if (0 != (NumElems * FromEltSz) % ToEltSz) return SDValue();
10572
10573 unsigned SizeRatio = FromEltSz / ToEltSz;
10574 assert(SizeRatio * NumElems * ToEltSz == VT.getSizeInBits());
10575
10576 // Create a type on which we perform the shuffle.
10577 EVT WideVecVT = EVT::getVectorVT(*DAG.getContext(), StVT.getScalarType(),
10578 NumElems*SizeRatio);
10579 assert(WideVecVT.getSizeInBits() == VT.getSizeInBits());
10580
10581 SDLoc DL(St);
10582 SDValue WideVec = DAG.getNode(ISD::BITCAST, DL, WideVecVT, StVal);
10583 SmallVector<int, 8> ShuffleVec(NumElems * SizeRatio, -1);
10584 for (unsigned i = 0; i < NumElems; ++i)
Mehdi Aminiffc14022015-07-08 01:00:38 +000010585 ShuffleVec[i] = DAG.getDataLayout().isBigEndian()
10586 ? (i + 1) * SizeRatio - 1
10587 : i * SizeRatio;
Ahmed Bougacha23167462014-12-09 21:26:53 +000010588
10589 // Can't shuffle using an illegal type.
10590 if (!TLI.isTypeLegal(WideVecVT)) return SDValue();
10591
10592 SDValue Shuff = DAG.getVectorShuffle(WideVecVT, DL, WideVec,
10593 DAG.getUNDEF(WideVec.getValueType()),
Craig Topper2bd8b4b2016-07-01 06:54:47 +000010594 ShuffleVec);
Ahmed Bougacha23167462014-12-09 21:26:53 +000010595 // At this point all of the data is stored at the bottom of the
10596 // register. We now need to save it to mem.
10597
10598 // Find the largest store unit
10599 MVT StoreType = MVT::i8;
Ahmed Bougacha67dd2d22015-01-07 21:27:10 +000010600 for (MVT Tp : MVT::integer_valuetypes()) {
Ahmed Bougacha23167462014-12-09 21:26:53 +000010601 if (TLI.isTypeLegal(Tp) && Tp.getSizeInBits() <= NumElems * ToEltSz)
10602 StoreType = Tp;
10603 }
10604 // Didn't find a legal store type.
10605 if (!TLI.isTypeLegal(StoreType))
10606 return SDValue();
10607
10608 // Bitcast the original vector into a vector of store-size units
10609 EVT StoreVecVT = EVT::getVectorVT(*DAG.getContext(),
10610 StoreType, VT.getSizeInBits()/EVT(StoreType).getSizeInBits());
10611 assert(StoreVecVT.getSizeInBits() == VT.getSizeInBits());
10612 SDValue ShuffWide = DAG.getNode(ISD::BITCAST, DL, StoreVecVT, Shuff);
10613 SmallVector<SDValue, 8> Chains;
Mehdi Amini44ede332015-07-09 02:09:04 +000010614 SDValue Increment = DAG.getConstant(StoreType.getSizeInBits() / 8, DL,
10615 TLI.getPointerTy(DAG.getDataLayout()));
Ahmed Bougacha23167462014-12-09 21:26:53 +000010616 SDValue BasePtr = St->getBasePtr();
10617
10618 // Perform one or more big stores into memory.
10619 unsigned E = (ToEltSz*NumElems)/StoreType.getSizeInBits();
10620 for (unsigned I = 0; I < E; I++) {
10621 SDValue SubVec = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL,
10622 StoreType, ShuffWide,
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +000010623 DAG.getIntPtrConstant(I, DL));
Ahmed Bougacha23167462014-12-09 21:26:53 +000010624 SDValue Ch = DAG.getStore(St->getChain(), DL, SubVec, BasePtr,
Justin Lebar9c375812016-07-15 18:27:10 +000010625 St->getPointerInfo(), St->getAlignment(),
10626 St->getMemOperand()->getFlags());
Ahmed Bougacha23167462014-12-09 21:26:53 +000010627 BasePtr = DAG.getNode(ISD::ADD, DL, BasePtr.getValueType(), BasePtr,
10628 Increment);
10629 Chains.push_back(Ch);
10630 }
10631 return DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Chains);
10632 }
10633
10634 if (!ISD::isNormalStore(St))
10635 return SDValue();
10636
10637 // Split a store of a VMOVDRR into two integer stores to avoid mixing NEON and
10638 // ARM stores of arguments in the same cache line.
10639 if (StVal.getNode()->getOpcode() == ARMISD::VMOVDRR &&
10640 StVal.getNode()->hasOneUse()) {
10641 SelectionDAG &DAG = DCI.DAG;
Mehdi Aminiffc14022015-07-08 01:00:38 +000010642 bool isBigEndian = DAG.getDataLayout().isBigEndian();
Ahmed Bougacha23167462014-12-09 21:26:53 +000010643 SDLoc DL(St);
10644 SDValue BasePtr = St->getBasePtr();
Justin Lebar9c375812016-07-15 18:27:10 +000010645 SDValue NewST1 = DAG.getStore(
10646 St->getChain(), DL, StVal.getNode()->getOperand(isBigEndian ? 1 : 0),
10647 BasePtr, St->getPointerInfo(), St->getAlignment(),
10648 St->getMemOperand()->getFlags());
Ahmed Bougacha23167462014-12-09 21:26:53 +000010649
10650 SDValue OffsetPtr = DAG.getNode(ISD::ADD, DL, MVT::i32, BasePtr,
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +000010651 DAG.getConstant(4, DL, MVT::i32));
Ahmed Bougacha23167462014-12-09 21:26:53 +000010652 return DAG.getStore(NewST1.getValue(0), DL,
10653 StVal.getNode()->getOperand(isBigEndian ? 0 : 1),
Justin Lebar9c375812016-07-15 18:27:10 +000010654 OffsetPtr, St->getPointerInfo(),
10655 std::min(4U, St->getAlignment() / 2),
10656 St->getMemOperand()->getFlags());
Ahmed Bougacha23167462014-12-09 21:26:53 +000010657 }
10658
10659 if (StVal.getValueType() == MVT::i64 &&
10660 StVal.getNode()->getOpcode() == ISD::EXTRACT_VECTOR_ELT) {
10661
10662 // Bitcast an i64 store extracted from a vector to f64.
10663 // Otherwise, the i64 value will be legalized to a pair of i32 values.
10664 SelectionDAG &DAG = DCI.DAG;
10665 SDLoc dl(StVal);
10666 SDValue IntVec = StVal.getOperand(0);
10667 EVT FloatVT = EVT::getVectorVT(*DAG.getContext(), MVT::f64,
10668 IntVec.getValueType().getVectorNumElements());
10669 SDValue Vec = DAG.getNode(ISD::BITCAST, dl, FloatVT, IntVec);
10670 SDValue ExtElt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::f64,
10671 Vec, StVal.getOperand(1));
10672 dl = SDLoc(N);
10673 SDValue V = DAG.getNode(ISD::BITCAST, dl, MVT::i64, ExtElt);
10674 // Make the DAGCombiner fold the bitcasts.
10675 DCI.AddToWorklist(Vec.getNode());
10676 DCI.AddToWorklist(ExtElt.getNode());
10677 DCI.AddToWorklist(V.getNode());
10678 return DAG.getStore(St->getChain(), dl, V, St->getBasePtr(),
Justin Lebar9c375812016-07-15 18:27:10 +000010679 St->getPointerInfo(), St->getAlignment(),
10680 St->getMemOperand()->getFlags(), St->getAAInfo());
Ahmed Bougacha23167462014-12-09 21:26:53 +000010681 }
10682
Ahmed Bougachadb141ac2015-02-19 23:52:41 +000010683 // If this is a legal vector store, try to combine it into a VST1_UPD.
10684 if (ISD::isNormalStore(N) && VT.isVector() &&
10685 DCI.DAG.getTargetLoweringInfo().isTypeLegal(VT))
10686 return CombineBaseUpdate(N, DCI);
10687
Ahmed Bougacha23167462014-12-09 21:26:53 +000010688 return SDValue();
10689}
10690
Chad Rosierfa8d8932011-06-24 19:23:04 +000010691/// PerformVCVTCombine - VCVT (floating-point to fixed-point, Advanced SIMD)
10692/// can replace combinations of VMUL and VCVT (floating-point to integer)
10693/// when the VMUL has a constant operand that is a power of 2.
10694///
10695/// Example (assume d17 = <float 8.000000e+00, float 8.000000e+00>):
10696/// vmul.f32 d16, d17, d16
10697/// vcvt.s32.f32 d16, d16
10698/// becomes:
10699/// vcvt.s32.f32 d16, d16, #3
Chad Rosiera087fd22015-10-06 20:23:42 +000010700static SDValue PerformVCVTCombine(SDNode *N, SelectionDAG &DAG,
Chad Rosierfa8d8932011-06-24 19:23:04 +000010701 const ARMSubtarget *Subtarget) {
Chad Rosiera087fd22015-10-06 20:23:42 +000010702 if (!Subtarget->hasNEON())
10703 return SDValue();
Chad Rosierfa8d8932011-06-24 19:23:04 +000010704
Chad Rosiera087fd22015-10-06 20:23:42 +000010705 SDValue Op = N->getOperand(0);
Tim Northover498c56c2016-03-17 20:10:28 +000010706 if (!Op.getValueType().isVector() || !Op.getValueType().isSimple() ||
10707 Op.getOpcode() != ISD::FMUL)
Chad Rosierfa8d8932011-06-24 19:23:04 +000010708 return SDValue();
10709
Chad Rosierfa8d8932011-06-24 19:23:04 +000010710 SDValue ConstVec = Op->getOperand(1);
Chad Rosieraed910b2015-10-06 20:51:26 +000010711 if (!isa<BuildVectorSDNode>(ConstVec))
10712 return SDValue();
Chad Rosierfa8d8932011-06-24 19:23:04 +000010713
Tim Northover7cbc2152013-06-28 15:29:25 +000010714 MVT FloatTy = Op.getSimpleValueType().getVectorElementType();
Chad Rosier9df4aff2015-10-06 20:45:45 +000010715 uint32_t FloatBits = FloatTy.getSizeInBits();
Tim Northover7cbc2152013-06-28 15:29:25 +000010716 MVT IntTy = N->getSimpleValueType(0).getVectorElementType();
Chad Rosier9df4aff2015-10-06 20:45:45 +000010717 uint32_t IntBits = IntTy.getSizeInBits();
Bradley Smithececb7f2014-12-16 10:59:27 +000010718 unsigned NumLanes = Op.getValueType().getVectorNumElements();
Chad Rosier9df4aff2015-10-06 20:45:45 +000010719 if (FloatBits != 32 || IntBits > 32 || NumLanes > 4) {
Tim Northover7cbc2152013-06-28 15:29:25 +000010720 // These instructions only exist converting from f32 to i32. We can handle
10721 // smaller integers by generating an extra truncate, but larger ones would
Bradley Smithececb7f2014-12-16 10:59:27 +000010722 // be lossy. We also can't handle more then 4 lanes, since these intructions
10723 // only support v2i32/v4i32 types.
Tim Northover7cbc2152013-06-28 15:29:25 +000010724 return SDValue();
10725 }
10726
Chad Rosier169865f2015-10-07 17:28:58 +000010727 BitVector UndefElements;
10728 BuildVectorSDNode *BV = cast<BuildVectorSDNode>(ConstVec);
10729 int32_t C = BV->getConstantFPSplatPow2ToLog2Int(&UndefElements, 33);
10730 if (C == -1 || C == 0 || C > 32)
Chad Rosierdb71abf2015-10-07 13:40:44 +000010731 return SDValue();
10732
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +000010733 SDLoc dl(N);
Chad Rosier169865f2015-10-07 17:28:58 +000010734 bool isSigned = N->getOpcode() == ISD::FP_TO_SINT;
Chad Rosierfa8d8932011-06-24 19:23:04 +000010735 unsigned IntrinsicOpcode = isSigned ? Intrinsic::arm_neon_vcvtfp2fxs :
10736 Intrinsic::arm_neon_vcvtfp2fxu;
Chad Rosier9df4aff2015-10-06 20:45:45 +000010737 SDValue FixConv = DAG.getNode(
10738 ISD::INTRINSIC_WO_CHAIN, dl, NumLanes == 2 ? MVT::v2i32 : MVT::v4i32,
10739 DAG.getConstant(IntrinsicOpcode, dl, MVT::i32), Op->getOperand(0),
Chad Rosier169865f2015-10-07 17:28:58 +000010740 DAG.getConstant(C, dl, MVT::i32));
Tim Northover7cbc2152013-06-28 15:29:25 +000010741
Chad Rosier9df4aff2015-10-06 20:45:45 +000010742 if (IntBits < FloatBits)
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +000010743 FixConv = DAG.getNode(ISD::TRUNCATE, dl, N->getValueType(0), FixConv);
Tim Northover7cbc2152013-06-28 15:29:25 +000010744
10745 return FixConv;
Chad Rosierfa8d8932011-06-24 19:23:04 +000010746}
10747
10748/// PerformVDIVCombine - VCVT (fixed-point to floating-point, Advanced SIMD)
10749/// can replace combinations of VCVT (integer to floating-point) and VDIV
10750/// when the VDIV has a constant operand that is a power of 2.
10751///
10752/// Example (assume d17 = <float 8.000000e+00, float 8.000000e+00>):
10753/// vcvt.f32.s32 d16, d16
10754/// vdiv.f32 d16, d17, d16
10755/// becomes:
10756/// vcvt.f32.s32 d16, d16, #3
Chad Rosiera087fd22015-10-06 20:23:42 +000010757static SDValue PerformVDIVCombine(SDNode *N, SelectionDAG &DAG,
Chad Rosierfa8d8932011-06-24 19:23:04 +000010758 const ARMSubtarget *Subtarget) {
Chad Rosiera087fd22015-10-06 20:23:42 +000010759 if (!Subtarget->hasNEON())
10760 return SDValue();
10761
Chad Rosierfa8d8932011-06-24 19:23:04 +000010762 SDValue Op = N->getOperand(0);
10763 unsigned OpOpcode = Op.getNode()->getOpcode();
Tim Northover498c56c2016-03-17 20:10:28 +000010764 if (!N->getValueType(0).isVector() || !N->getValueType(0).isSimple() ||
Chad Rosierfa8d8932011-06-24 19:23:04 +000010765 (OpOpcode != ISD::SINT_TO_FP && OpOpcode != ISD::UINT_TO_FP))
10766 return SDValue();
10767
Chad Rosierfa8d8932011-06-24 19:23:04 +000010768 SDValue ConstVec = N->getOperand(1);
Chad Rosieraed910b2015-10-06 20:51:26 +000010769 if (!isa<BuildVectorSDNode>(ConstVec))
10770 return SDValue();
Chad Rosierfa8d8932011-06-24 19:23:04 +000010771
Tim Northover7cbc2152013-06-28 15:29:25 +000010772 MVT FloatTy = N->getSimpleValueType(0).getVectorElementType();
Chad Rosierdca46b42015-10-06 20:58:42 +000010773 uint32_t FloatBits = FloatTy.getSizeInBits();
Tim Northover7cbc2152013-06-28 15:29:25 +000010774 MVT IntTy = Op.getOperand(0).getSimpleValueType().getVectorElementType();
Chad Rosierdca46b42015-10-06 20:58:42 +000010775 uint32_t IntBits = IntTy.getSizeInBits();
Chad Rosier17436bf2015-10-07 16:15:40 +000010776 unsigned NumLanes = Op.getValueType().getVectorNumElements();
10777 if (FloatBits != 32 || IntBits > 32 || NumLanes > 4) {
Tim Northover7cbc2152013-06-28 15:29:25 +000010778 // These instructions only exist converting from i32 to f32. We can handle
10779 // smaller integers by generating an extra extend, but larger ones would
Chad Rosier17436bf2015-10-07 16:15:40 +000010780 // be lossy. We also can't handle more then 4 lanes, since these intructions
10781 // only support v2i32/v4i32 types.
Tim Northover7cbc2152013-06-28 15:29:25 +000010782 return SDValue();
10783 }
10784
Chad Rosier169865f2015-10-07 17:28:58 +000010785 BitVector UndefElements;
10786 BuildVectorSDNode *BV = cast<BuildVectorSDNode>(ConstVec);
10787 int32_t C = BV->getConstantFPSplatPow2ToLog2Int(&UndefElements, 33);
10788 if (C == -1 || C == 0 || C > 32)
Chad Rosierdb71abf2015-10-07 13:40:44 +000010789 return SDValue();
10790
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +000010791 SDLoc dl(N);
Chad Rosier169865f2015-10-07 17:28:58 +000010792 bool isSigned = OpOpcode == ISD::SINT_TO_FP;
Tim Northover7cbc2152013-06-28 15:29:25 +000010793 SDValue ConvInput = Op.getOperand(0);
Chad Rosierdca46b42015-10-06 20:58:42 +000010794 if (IntBits < FloatBits)
Tim Northover7cbc2152013-06-28 15:29:25 +000010795 ConvInput = DAG.getNode(isSigned ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND,
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +000010796 dl, NumLanes == 2 ? MVT::v2i32 : MVT::v4i32,
Tim Northover7cbc2152013-06-28 15:29:25 +000010797 ConvInput);
10798
Eric Christopher1b8b94192011-06-29 21:10:36 +000010799 unsigned IntrinsicOpcode = isSigned ? Intrinsic::arm_neon_vcvtfxs2fp :
Chad Rosierfa8d8932011-06-24 19:23:04 +000010800 Intrinsic::arm_neon_vcvtfxu2fp;
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +000010801 return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl,
Chad Rosierfa8d8932011-06-24 19:23:04 +000010802 Op.getValueType(),
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +000010803 DAG.getConstant(IntrinsicOpcode, dl, MVT::i32),
Chad Rosier169865f2015-10-07 17:28:58 +000010804 ConvInput, DAG.getConstant(C, dl, MVT::i32));
Chad Rosierfa8d8932011-06-24 19:23:04 +000010805}
10806
10807/// Getvshiftimm - Check if this is a valid build_vector for the immediate
Bob Wilson2e076c42009-06-22 23:27:02 +000010808/// operand of a vector shift operation, where all the elements of the
10809/// build_vector must have the same constant integer value.
10810static bool getVShiftImm(SDValue Op, unsigned ElementBits, int64_t &Cnt) {
10811 // Ignore bit_converts.
Wesley Peck527da1b2010-11-23 03:31:01 +000010812 while (Op.getOpcode() == ISD::BITCAST)
Bob Wilson2e076c42009-06-22 23:27:02 +000010813 Op = Op.getOperand(0);
10814 BuildVectorSDNode *BVN = dyn_cast<BuildVectorSDNode>(Op.getNode());
10815 APInt SplatBits, SplatUndef;
10816 unsigned SplatBitSize;
10817 bool HasAnyUndefs;
10818 if (! BVN || ! BVN->isConstantSplat(SplatBits, SplatUndef, SplatBitSize,
10819 HasAnyUndefs, ElementBits) ||
10820 SplatBitSize > ElementBits)
10821 return false;
10822 Cnt = SplatBits.getSExtValue();
10823 return true;
10824}
10825
10826/// isVShiftLImm - Check if this is a valid build_vector for the immediate
10827/// operand of a vector shift left operation. That value must be in the range:
10828/// 0 <= Value < ElementBits for a left shift; or
10829/// 0 <= Value <= ElementBits for a long left shift.
Owen Anderson53aa7a92009-08-10 22:56:29 +000010830static bool isVShiftLImm(SDValue Op, EVT VT, bool isLong, int64_t &Cnt) {
Bob Wilson2e076c42009-06-22 23:27:02 +000010831 assert(VT.isVector() && "vector shift count is not a vector type");
Luke Cheesemanb5c627a2015-07-24 09:31:48 +000010832 int64_t ElementBits = VT.getVectorElementType().getSizeInBits();
Bob Wilson2e076c42009-06-22 23:27:02 +000010833 if (! getVShiftImm(Op, ElementBits, Cnt))
10834 return false;
10835 return (Cnt >= 0 && (isLong ? Cnt-1 : Cnt) < ElementBits);
10836}
10837
10838/// isVShiftRImm - Check if this is a valid build_vector for the immediate
10839/// operand of a vector shift right operation. For a shift opcode, the value
10840/// is positive, but for an intrinsic the value count must be negative. The
10841/// absolute value must be in the range:
10842/// 1 <= |Value| <= ElementBits for a right shift; or
10843/// 1 <= |Value| <= ElementBits/2 for a narrow right shift.
Owen Anderson53aa7a92009-08-10 22:56:29 +000010844static bool isVShiftRImm(SDValue Op, EVT VT, bool isNarrow, bool isIntrinsic,
Bob Wilson2e076c42009-06-22 23:27:02 +000010845 int64_t &Cnt) {
10846 assert(VT.isVector() && "vector shift count is not a vector type");
Luke Cheesemanb5c627a2015-07-24 09:31:48 +000010847 int64_t ElementBits = VT.getVectorElementType().getSizeInBits();
Bob Wilson2e076c42009-06-22 23:27:02 +000010848 if (! getVShiftImm(Op, ElementBits, Cnt))
10849 return false;
Luke Cheesemanb5c627a2015-07-24 09:31:48 +000010850 if (!isIntrinsic)
10851 return (Cnt >= 1 && Cnt <= (isNarrow ? ElementBits/2 : ElementBits));
10852 if (Cnt >= -(isNarrow ? ElementBits/2 : ElementBits) && Cnt <= -1) {
Bob Wilson2e076c42009-06-22 23:27:02 +000010853 Cnt = -Cnt;
Luke Cheesemanb5c627a2015-07-24 09:31:48 +000010854 return true;
10855 }
10856 return false;
Bob Wilson2e076c42009-06-22 23:27:02 +000010857}
10858
10859/// PerformIntrinsicCombine - ARM-specific DAG combining for intrinsics.
10860static SDValue PerformIntrinsicCombine(SDNode *N, SelectionDAG &DAG) {
10861 unsigned IntNo = cast<ConstantSDNode>(N->getOperand(0))->getZExtValue();
10862 switch (IntNo) {
10863 default:
10864 // Don't do anything for most intrinsics.
10865 break;
10866
10867 // Vector shifts: check for immediate versions and lower them.
10868 // Note: This is done during DAG combining instead of DAG legalizing because
10869 // the build_vectors for 64-bit vector element shift counts are generally
10870 // not legal, and it is hard to see their values after they get legalized to
10871 // loads from a constant pool.
10872 case Intrinsic::arm_neon_vshifts:
10873 case Intrinsic::arm_neon_vshiftu:
Bob Wilson2e076c42009-06-22 23:27:02 +000010874 case Intrinsic::arm_neon_vrshifts:
10875 case Intrinsic::arm_neon_vrshiftu:
10876 case Intrinsic::arm_neon_vrshiftn:
10877 case Intrinsic::arm_neon_vqshifts:
10878 case Intrinsic::arm_neon_vqshiftu:
10879 case Intrinsic::arm_neon_vqshiftsu:
10880 case Intrinsic::arm_neon_vqshiftns:
10881 case Intrinsic::arm_neon_vqshiftnu:
10882 case Intrinsic::arm_neon_vqshiftnsu:
10883 case Intrinsic::arm_neon_vqrshiftns:
10884 case Intrinsic::arm_neon_vqrshiftnu:
10885 case Intrinsic::arm_neon_vqrshiftnsu: {
Owen Anderson53aa7a92009-08-10 22:56:29 +000010886 EVT VT = N->getOperand(1).getValueType();
Bob Wilson2e076c42009-06-22 23:27:02 +000010887 int64_t Cnt;
10888 unsigned VShiftOpc = 0;
10889
10890 switch (IntNo) {
10891 case Intrinsic::arm_neon_vshifts:
10892 case Intrinsic::arm_neon_vshiftu:
10893 if (isVShiftLImm(N->getOperand(2), VT, false, Cnt)) {
10894 VShiftOpc = ARMISD::VSHL;
10895 break;
10896 }
10897 if (isVShiftRImm(N->getOperand(2), VT, false, true, Cnt)) {
10898 VShiftOpc = (IntNo == Intrinsic::arm_neon_vshifts ?
10899 ARMISD::VSHRs : ARMISD::VSHRu);
10900 break;
10901 }
10902 return SDValue();
10903
Bob Wilson2e076c42009-06-22 23:27:02 +000010904 case Intrinsic::arm_neon_vrshifts:
10905 case Intrinsic::arm_neon_vrshiftu:
10906 if (isVShiftRImm(N->getOperand(2), VT, false, true, Cnt))
10907 break;
10908 return SDValue();
10909
10910 case Intrinsic::arm_neon_vqshifts:
10911 case Intrinsic::arm_neon_vqshiftu:
10912 if (isVShiftLImm(N->getOperand(2), VT, false, Cnt))
10913 break;
10914 return SDValue();
10915
10916 case Intrinsic::arm_neon_vqshiftsu:
10917 if (isVShiftLImm(N->getOperand(2), VT, false, Cnt))
10918 break;
Torok Edwinfbcc6632009-07-14 16:55:14 +000010919 llvm_unreachable("invalid shift count for vqshlu intrinsic");
Bob Wilson2e076c42009-06-22 23:27:02 +000010920
Bob Wilson2e076c42009-06-22 23:27:02 +000010921 case Intrinsic::arm_neon_vrshiftn:
10922 case Intrinsic::arm_neon_vqshiftns:
10923 case Intrinsic::arm_neon_vqshiftnu:
10924 case Intrinsic::arm_neon_vqshiftnsu:
10925 case Intrinsic::arm_neon_vqrshiftns:
10926 case Intrinsic::arm_neon_vqrshiftnu:
10927 case Intrinsic::arm_neon_vqrshiftnsu:
10928 // Narrowing shifts require an immediate right shift.
10929 if (isVShiftRImm(N->getOperand(2), VT, true, true, Cnt))
10930 break;
Jim Grosbach84511e12010-06-02 21:53:11 +000010931 llvm_unreachable("invalid shift count for narrowing vector shift "
10932 "intrinsic");
Bob Wilson2e076c42009-06-22 23:27:02 +000010933
10934 default:
Torok Edwinfbcc6632009-07-14 16:55:14 +000010935 llvm_unreachable("unhandled vector shift");
Bob Wilson2e076c42009-06-22 23:27:02 +000010936 }
10937
10938 switch (IntNo) {
10939 case Intrinsic::arm_neon_vshifts:
10940 case Intrinsic::arm_neon_vshiftu:
10941 // Opcode already set above.
10942 break;
Bob Wilson2e076c42009-06-22 23:27:02 +000010943 case Intrinsic::arm_neon_vrshifts:
10944 VShiftOpc = ARMISD::VRSHRs; break;
10945 case Intrinsic::arm_neon_vrshiftu:
10946 VShiftOpc = ARMISD::VRSHRu; break;
10947 case Intrinsic::arm_neon_vrshiftn:
10948 VShiftOpc = ARMISD::VRSHRN; break;
10949 case Intrinsic::arm_neon_vqshifts:
10950 VShiftOpc = ARMISD::VQSHLs; break;
10951 case Intrinsic::arm_neon_vqshiftu:
10952 VShiftOpc = ARMISD::VQSHLu; break;
10953 case Intrinsic::arm_neon_vqshiftsu:
10954 VShiftOpc = ARMISD::VQSHLsu; break;
10955 case Intrinsic::arm_neon_vqshiftns:
10956 VShiftOpc = ARMISD::VQSHRNs; break;
10957 case Intrinsic::arm_neon_vqshiftnu:
10958 VShiftOpc = ARMISD::VQSHRNu; break;
10959 case Intrinsic::arm_neon_vqshiftnsu:
10960 VShiftOpc = ARMISD::VQSHRNsu; break;
10961 case Intrinsic::arm_neon_vqrshiftns:
10962 VShiftOpc = ARMISD::VQRSHRNs; break;
10963 case Intrinsic::arm_neon_vqrshiftnu:
10964 VShiftOpc = ARMISD::VQRSHRNu; break;
10965 case Intrinsic::arm_neon_vqrshiftnsu:
10966 VShiftOpc = ARMISD::VQRSHRNsu; break;
10967 }
10968
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +000010969 SDLoc dl(N);
10970 return DAG.getNode(VShiftOpc, dl, N->getValueType(0),
10971 N->getOperand(1), DAG.getConstant(Cnt, dl, MVT::i32));
Bob Wilson2e076c42009-06-22 23:27:02 +000010972 }
10973
10974 case Intrinsic::arm_neon_vshiftins: {
Owen Anderson53aa7a92009-08-10 22:56:29 +000010975 EVT VT = N->getOperand(1).getValueType();
Bob Wilson2e076c42009-06-22 23:27:02 +000010976 int64_t Cnt;
10977 unsigned VShiftOpc = 0;
10978
10979 if (isVShiftLImm(N->getOperand(3), VT, false, Cnt))
10980 VShiftOpc = ARMISD::VSLI;
10981 else if (isVShiftRImm(N->getOperand(3), VT, false, true, Cnt))
10982 VShiftOpc = ARMISD::VSRI;
10983 else {
Torok Edwinfbcc6632009-07-14 16:55:14 +000010984 llvm_unreachable("invalid shift count for vsli/vsri intrinsic");
Bob Wilson2e076c42009-06-22 23:27:02 +000010985 }
10986
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +000010987 SDLoc dl(N);
10988 return DAG.getNode(VShiftOpc, dl, N->getValueType(0),
Bob Wilson2e076c42009-06-22 23:27:02 +000010989 N->getOperand(1), N->getOperand(2),
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +000010990 DAG.getConstant(Cnt, dl, MVT::i32));
Bob Wilson2e076c42009-06-22 23:27:02 +000010991 }
10992
10993 case Intrinsic::arm_neon_vqrshifts:
10994 case Intrinsic::arm_neon_vqrshiftu:
10995 // No immediate versions of these to check for.
10996 break;
10997 }
10998
10999 return SDValue();
11000}
11001
11002/// PerformShiftCombine - Checks for immediate versions of vector shifts and
11003/// lowers them. As with the vector shift intrinsics, this is done during DAG
11004/// combining instead of DAG legalizing because the build_vectors for 64-bit
11005/// vector element shift counts are generally not legal, and it is hard to see
11006/// their values after they get legalized to loads from a constant pool.
11007static SDValue PerformShiftCombine(SDNode *N, SelectionDAG &DAG,
11008 const ARMSubtarget *ST) {
Owen Anderson53aa7a92009-08-10 22:56:29 +000011009 EVT VT = N->getValueType(0);
Evan Chengf258a152012-02-23 02:58:19 +000011010 if (N->getOpcode() == ISD::SRL && VT == MVT::i32 && ST->hasV6Ops()) {
11011 // Canonicalize (srl (bswap x), 16) to (rotr (bswap x), 16) if the high
11012 // 16-bits of x is zero. This optimizes rev + lsr 16 to rev16.
11013 SDValue N1 = N->getOperand(1);
11014 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(N1)) {
11015 SDValue N0 = N->getOperand(0);
11016 if (C->getZExtValue() == 16 && N0.getOpcode() == ISD::BSWAP &&
11017 DAG.MaskedValueIsZero(N0.getOperand(0),
11018 APInt::getHighBitsSet(32, 16)))
Andrew Trickef9de2a2013-05-25 02:42:55 +000011019 return DAG.getNode(ISD::ROTR, SDLoc(N), VT, N0, N1);
Evan Chengf258a152012-02-23 02:58:19 +000011020 }
11021 }
Bob Wilson2e076c42009-06-22 23:27:02 +000011022
11023 // Nothing to be done for scalar shifts.
Tanya Lattnercd680952010-11-18 22:06:46 +000011024 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
11025 if (!VT.isVector() || !TLI.isTypeLegal(VT))
Bob Wilson2e076c42009-06-22 23:27:02 +000011026 return SDValue();
11027
11028 assert(ST->hasNEON() && "unexpected vector shift");
11029 int64_t Cnt;
11030
11031 switch (N->getOpcode()) {
Torok Edwinfbcc6632009-07-14 16:55:14 +000011032 default: llvm_unreachable("unexpected shift opcode");
Bob Wilson2e076c42009-06-22 23:27:02 +000011033
11034 case ISD::SHL:
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +000011035 if (isVShiftLImm(N->getOperand(1), VT, false, Cnt)) {
11036 SDLoc dl(N);
11037 return DAG.getNode(ARMISD::VSHL, dl, VT, N->getOperand(0),
11038 DAG.getConstant(Cnt, dl, MVT::i32));
11039 }
Bob Wilson2e076c42009-06-22 23:27:02 +000011040 break;
11041
11042 case ISD::SRA:
11043 case ISD::SRL:
11044 if (isVShiftRImm(N->getOperand(1), VT, false, false, Cnt)) {
11045 unsigned VShiftOpc = (N->getOpcode() == ISD::SRA ?
11046 ARMISD::VSHRs : ARMISD::VSHRu);
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +000011047 SDLoc dl(N);
11048 return DAG.getNode(VShiftOpc, dl, VT, N->getOperand(0),
11049 DAG.getConstant(Cnt, dl, MVT::i32));
Bob Wilson2e076c42009-06-22 23:27:02 +000011050 }
11051 }
11052 return SDValue();
11053}
11054
11055/// PerformExtendCombine - Target-specific DAG combining for ISD::SIGN_EXTEND,
11056/// ISD::ZERO_EXTEND, and ISD::ANY_EXTEND.
11057static SDValue PerformExtendCombine(SDNode *N, SelectionDAG &DAG,
11058 const ARMSubtarget *ST) {
11059 SDValue N0 = N->getOperand(0);
11060
11061 // Check for sign- and zero-extensions of vector extract operations of 8-
11062 // and 16-bit vector elements. NEON supports these directly. They are
11063 // handled during DAG combining because type legalization will promote them
11064 // to 32-bit types and it is messy to recognize the operations after that.
11065 if (ST->hasNEON() && N0.getOpcode() == ISD::EXTRACT_VECTOR_ELT) {
11066 SDValue Vec = N0.getOperand(0);
11067 SDValue Lane = N0.getOperand(1);
Owen Anderson53aa7a92009-08-10 22:56:29 +000011068 EVT VT = N->getValueType(0);
11069 EVT EltVT = N0.getValueType();
Bob Wilson2e076c42009-06-22 23:27:02 +000011070 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
11071
Owen Anderson9f944592009-08-11 20:47:22 +000011072 if (VT == MVT::i32 &&
11073 (EltVT == MVT::i8 || EltVT == MVT::i16) &&
Bob Wilsonceb49292010-11-03 16:24:50 +000011074 TLI.isTypeLegal(Vec.getValueType()) &&
11075 isa<ConstantSDNode>(Lane)) {
Bob Wilson2e076c42009-06-22 23:27:02 +000011076
11077 unsigned Opc = 0;
11078 switch (N->getOpcode()) {
Torok Edwinfbcc6632009-07-14 16:55:14 +000011079 default: llvm_unreachable("unexpected opcode");
Bob Wilson2e076c42009-06-22 23:27:02 +000011080 case ISD::SIGN_EXTEND:
11081 Opc = ARMISD::VGETLANEs;
11082 break;
11083 case ISD::ZERO_EXTEND:
11084 case ISD::ANY_EXTEND:
11085 Opc = ARMISD::VGETLANEu;
11086 break;
11087 }
Andrew Trickef9de2a2013-05-25 02:42:55 +000011088 return DAG.getNode(Opc, SDLoc(N), VT, Vec, Lane);
Bob Wilson2e076c42009-06-22 23:27:02 +000011089 }
11090 }
11091
11092 return SDValue();
11093}
11094
James Molloy9d55f192015-11-10 14:22:05 +000011095static void computeKnownBits(SelectionDAG &DAG, SDValue Op, APInt &KnownZero,
11096 APInt &KnownOne) {
11097 if (Op.getOpcode() == ARMISD::BFI) {
11098 // Conservatively, we can recurse down the first operand
11099 // and just mask out all affected bits.
11100 computeKnownBits(DAG, Op.getOperand(0), KnownZero, KnownOne);
11101
11102 // The operand to BFI is already a mask suitable for removing the bits it
11103 // sets.
11104 ConstantSDNode *CI = cast<ConstantSDNode>(Op.getOperand(2));
Benjamin Kramer46e38f32016-06-08 10:01:20 +000011105 const APInt &Mask = CI->getAPIntValue();
James Molloy9d55f192015-11-10 14:22:05 +000011106 KnownZero &= Mask;
11107 KnownOne &= Mask;
11108 return;
11109 }
11110 if (Op.getOpcode() == ARMISD::CMOV) {
11111 APInt KZ2(KnownZero.getBitWidth(), 0);
11112 APInt KO2(KnownOne.getBitWidth(), 0);
11113 computeKnownBits(DAG, Op.getOperand(1), KnownZero, KnownOne);
11114 computeKnownBits(DAG, Op.getOperand(2), KZ2, KO2);
11115
11116 KnownZero &= KZ2;
11117 KnownOne &= KO2;
11118 return;
11119 }
11120 return DAG.computeKnownBits(Op, KnownZero, KnownOne);
11121}
11122
11123SDValue ARMTargetLowering::PerformCMOVToBFICombine(SDNode *CMOV, SelectionDAG &DAG) const {
11124 // If we have a CMOV, OR and AND combination such as:
11125 // if (x & CN)
11126 // y |= CM;
11127 //
11128 // And:
11129 // * CN is a single bit;
11130 // * All bits covered by CM are known zero in y
11131 //
11132 // Then we can convert this into a sequence of BFI instructions. This will
11133 // always be a win if CM is a single bit, will always be no worse than the
11134 // TST&OR sequence if CM is two bits, and for thumb will be no worse if CM is
11135 // three bits (due to the extra IT instruction).
11136
11137 SDValue Op0 = CMOV->getOperand(0);
11138 SDValue Op1 = CMOV->getOperand(1);
James Molloy8e99e972015-11-12 13:49:17 +000011139 auto CCNode = cast<ConstantSDNode>(CMOV->getOperand(2));
11140 auto CC = CCNode->getAPIntValue().getLimitedValue();
James Molloy9d55f192015-11-10 14:22:05 +000011141 SDValue CmpZ = CMOV->getOperand(4);
11142
James Molloy20180912015-11-16 10:49:25 +000011143 // The compare must be against zero.
Artyom Skrobov314ee042015-11-25 19:41:11 +000011144 if (!isNullConstant(CmpZ->getOperand(1)))
James Molloy20180912015-11-16 10:49:25 +000011145 return SDValue();
11146
James Molloy9d55f192015-11-10 14:22:05 +000011147 assert(CmpZ->getOpcode() == ARMISD::CMPZ);
11148 SDValue And = CmpZ->getOperand(0);
11149 if (And->getOpcode() != ISD::AND)
11150 return SDValue();
11151 ConstantSDNode *AndC = dyn_cast<ConstantSDNode>(And->getOperand(1));
11152 if (!AndC || !AndC->getAPIntValue().isPowerOf2())
11153 return SDValue();
11154 SDValue X = And->getOperand(0);
11155
James Molloy8e99e972015-11-12 13:49:17 +000011156 if (CC == ARMCC::EQ) {
11157 // We're performing an "equal to zero" compare. Swap the operands so we
11158 // canonicalize on a "not equal to zero" compare.
11159 std::swap(Op0, Op1);
11160 } else {
11161 assert(CC == ARMCC::NE && "How can a CMPZ node not be EQ or NE?");
11162 }
Junmo Park1108ab02016-02-19 01:46:04 +000011163
James Molloy9d55f192015-11-10 14:22:05 +000011164 if (Op1->getOpcode() != ISD::OR)
11165 return SDValue();
11166
11167 ConstantSDNode *OrC = dyn_cast<ConstantSDNode>(Op1->getOperand(1));
11168 if (!OrC)
11169 return SDValue();
11170 SDValue Y = Op1->getOperand(0);
11171
11172 if (Op0 != Y)
11173 return SDValue();
11174
11175 // Now, is it profitable to continue?
11176 APInt OrCI = OrC->getAPIntValue();
11177 unsigned Heuristic = Subtarget->isThumb() ? 3 : 2;
11178 if (OrCI.countPopulation() > Heuristic)
11179 return SDValue();
11180
11181 // Lastly, can we determine that the bits defined by OrCI
11182 // are zero in Y?
11183 APInt KnownZero, KnownOne;
11184 computeKnownBits(DAG, Y, KnownZero, KnownOne);
11185 if ((OrCI & KnownZero) != OrCI)
11186 return SDValue();
11187
11188 // OK, we can do the combine.
11189 SDValue V = Y;
11190 SDLoc dl(X);
11191 EVT VT = X.getValueType();
11192 unsigned BitInX = AndC->getAPIntValue().logBase2();
Junmo Park1108ab02016-02-19 01:46:04 +000011193
James Molloy9d55f192015-11-10 14:22:05 +000011194 if (BitInX != 0) {
11195 // We must shift X first.
11196 X = DAG.getNode(ISD::SRL, dl, VT, X,
11197 DAG.getConstant(BitInX, dl, VT));
11198 }
11199
11200 for (unsigned BitInY = 0, NumActiveBits = OrCI.getActiveBits();
11201 BitInY < NumActiveBits; ++BitInY) {
11202 if (OrCI[BitInY] == 0)
11203 continue;
11204 APInt Mask(VT.getSizeInBits(), 0);
11205 Mask.setBit(BitInY);
11206 V = DAG.getNode(ARMISD::BFI, dl, VT, V, X,
11207 // Confusingly, the operand is an *inverted* mask.
11208 DAG.getConstant(~Mask, dl, VT));
11209 }
11210
11211 return V;
11212}
11213
Peter Collingbourne86b9fbe2016-03-21 18:00:02 +000011214/// PerformBRCONDCombine - Target-specific DAG combining for ARMISD::BRCOND.
11215SDValue
11216ARMTargetLowering::PerformBRCONDCombine(SDNode *N, SelectionDAG &DAG) const {
11217 SDValue Cmp = N->getOperand(4);
11218 if (Cmp.getOpcode() != ARMISD::CMPZ)
11219 // Only looking at NE cases.
11220 return SDValue();
11221
11222 EVT VT = N->getValueType(0);
11223 SDLoc dl(N);
11224 SDValue LHS = Cmp.getOperand(0);
11225 SDValue RHS = Cmp.getOperand(1);
11226 SDValue Chain = N->getOperand(0);
11227 SDValue BB = N->getOperand(1);
11228 SDValue ARMcc = N->getOperand(2);
11229 ARMCC::CondCodes CC =
11230 (ARMCC::CondCodes)cast<ConstantSDNode>(ARMcc)->getZExtValue();
11231
11232 // (brcond Chain BB ne CPSR (cmpz (and (cmov 0 1 CC CPSR Cmp) 1) 0))
11233 // -> (brcond Chain BB CC CPSR Cmp)
11234 if (CC == ARMCC::NE && LHS.getOpcode() == ISD::AND && LHS->hasOneUse() &&
11235 LHS->getOperand(0)->getOpcode() == ARMISD::CMOV &&
11236 LHS->getOperand(0)->hasOneUse()) {
11237 auto *LHS00C = dyn_cast<ConstantSDNode>(LHS->getOperand(0)->getOperand(0));
11238 auto *LHS01C = dyn_cast<ConstantSDNode>(LHS->getOperand(0)->getOperand(1));
11239 auto *LHS1C = dyn_cast<ConstantSDNode>(LHS->getOperand(1));
11240 auto *RHSC = dyn_cast<ConstantSDNode>(RHS);
11241 if ((LHS00C && LHS00C->getZExtValue() == 0) &&
11242 (LHS01C && LHS01C->getZExtValue() == 1) &&
11243 (LHS1C && LHS1C->getZExtValue() == 1) &&
11244 (RHSC && RHSC->getZExtValue() == 0)) {
11245 return DAG.getNode(
11246 ARMISD::BRCOND, dl, VT, Chain, BB, LHS->getOperand(0)->getOperand(2),
11247 LHS->getOperand(0)->getOperand(3), LHS->getOperand(0)->getOperand(4));
11248 }
11249 }
11250
11251 return SDValue();
11252}
11253
Evan Chengf863e3f2011-07-13 00:42:17 +000011254/// PerformCMOVCombine - Target-specific DAG combining for ARMISD::CMOV.
11255SDValue
11256ARMTargetLowering::PerformCMOVCombine(SDNode *N, SelectionDAG &DAG) const {
11257 SDValue Cmp = N->getOperand(4);
11258 if (Cmp.getOpcode() != ARMISD::CMPZ)
11259 // Only looking at EQ and NE cases.
11260 return SDValue();
11261
11262 EVT VT = N->getValueType(0);
Andrew Trickef9de2a2013-05-25 02:42:55 +000011263 SDLoc dl(N);
Evan Chengf863e3f2011-07-13 00:42:17 +000011264 SDValue LHS = Cmp.getOperand(0);
11265 SDValue RHS = Cmp.getOperand(1);
11266 SDValue FalseVal = N->getOperand(0);
11267 SDValue TrueVal = N->getOperand(1);
11268 SDValue ARMcc = N->getOperand(2);
Jim Grosbache7e2aca2011-09-13 20:30:37 +000011269 ARMCC::CondCodes CC =
11270 (ARMCC::CondCodes)cast<ConstantSDNode>(ARMcc)->getZExtValue();
Evan Chengf863e3f2011-07-13 00:42:17 +000011271
James Molloy9d55f192015-11-10 14:22:05 +000011272 // BFI is only available on V6T2+.
11273 if (!Subtarget->isThumb1Only() && Subtarget->hasV6T2Ops()) {
11274 SDValue R = PerformCMOVToBFICombine(N, DAG);
11275 if (R)
11276 return R;
11277 }
11278
Evan Chengf863e3f2011-07-13 00:42:17 +000011279 // Simplify
11280 // mov r1, r0
11281 // cmp r1, x
11282 // mov r0, y
11283 // moveq r0, x
11284 // to
11285 // cmp r0, x
11286 // movne r0, y
11287 //
11288 // mov r1, r0
11289 // cmp r1, x
11290 // mov r0, x
11291 // movne r0, y
11292 // to
11293 // cmp r0, x
11294 // movne r0, y
11295 /// FIXME: Turn this into a target neutral optimization?
11296 SDValue Res;
Evan Cheng81563762011-09-28 23:16:31 +000011297 if (CC == ARMCC::NE && FalseVal == RHS && FalseVal != LHS) {
Evan Chengf863e3f2011-07-13 00:42:17 +000011298 Res = DAG.getNode(ARMISD::CMOV, dl, VT, LHS, TrueVal, ARMcc,
11299 N->getOperand(3), Cmp);
11300 } else if (CC == ARMCC::EQ && TrueVal == RHS) {
11301 SDValue ARMcc;
11302 SDValue NewCmp = getARMCmp(LHS, RHS, ISD::SETNE, ARMcc, DAG, dl);
11303 Res = DAG.getNode(ARMISD::CMOV, dl, VT, LHS, FalseVal, ARMcc,
11304 N->getOperand(3), NewCmp);
11305 }
11306
Peter Collingbourne86b9fbe2016-03-21 18:00:02 +000011307 // (cmov F T ne CPSR (cmpz (cmov 0 1 CC CPSR Cmp) 0))
11308 // -> (cmov F T CC CPSR Cmp)
11309 if (CC == ARMCC::NE && LHS.getOpcode() == ARMISD::CMOV && LHS->hasOneUse()) {
11310 auto *LHS0C = dyn_cast<ConstantSDNode>(LHS->getOperand(0));
11311 auto *LHS1C = dyn_cast<ConstantSDNode>(LHS->getOperand(1));
11312 auto *RHSC = dyn_cast<ConstantSDNode>(RHS);
11313 if ((LHS0C && LHS0C->getZExtValue() == 0) &&
11314 (LHS1C && LHS1C->getZExtValue() == 1) &&
11315 (RHSC && RHSC->getZExtValue() == 0)) {
11316 return DAG.getNode(ARMISD::CMOV, dl, VT, FalseVal, TrueVal,
11317 LHS->getOperand(2), LHS->getOperand(3),
11318 LHS->getOperand(4));
11319 }
11320 }
11321
Evan Chengf863e3f2011-07-13 00:42:17 +000011322 if (Res.getNode()) {
11323 APInt KnownZero, KnownOne;
Jay Foada0653a32014-05-14 21:14:37 +000011324 DAG.computeKnownBits(SDValue(N,0), KnownZero, KnownOne);
Evan Chengf863e3f2011-07-13 00:42:17 +000011325 // Capture demanded bits information that would be otherwise lost.
11326 if (KnownZero == 0xfffffffe)
11327 Res = DAG.getNode(ISD::AssertZext, dl, MVT::i32, Res,
11328 DAG.getValueType(MVT::i1));
11329 else if (KnownZero == 0xffffff00)
11330 Res = DAG.getNode(ISD::AssertZext, dl, MVT::i32, Res,
11331 DAG.getValueType(MVT::i8));
11332 else if (KnownZero == 0xffff0000)
11333 Res = DAG.getNode(ISD::AssertZext, dl, MVT::i32, Res,
11334 DAG.getValueType(MVT::i16));
11335 }
11336
11337 return Res;
11338}
11339
Dan Gohman2ce6f2a2008-07-27 21:46:04 +000011340SDValue ARMTargetLowering::PerformDAGCombine(SDNode *N,
Bob Wilson7117a912009-03-20 22:42:55 +000011341 DAGCombinerInfo &DCI) const {
Chris Lattnerf3f4ad92007-11-27 22:36:16 +000011342 switch (N->getOpcode()) {
11343 default: break;
Arnold Schwaighoferf00fb1c2012-09-04 14:37:49 +000011344 case ISD::ADDC: return PerformADDCCombine(N, DCI, Subtarget);
Tanya Lattnere9e67052011-06-14 23:48:48 +000011345 case ISD::ADD: return PerformADDCombine(N, DCI, Subtarget);
Bob Wilsonc6c13a32010-02-18 06:05:53 +000011346 case ISD::SUB: return PerformSUBCombine(N, DCI);
Anton Korobeynikov1bf28a12010-05-15 18:16:59 +000011347 case ISD::MUL: return PerformMULCombine(N, DCI, Subtarget);
Jim Grosbach11013ed2010-07-16 23:05:05 +000011348 case ISD::OR: return PerformORCombine(N, DCI, Subtarget);
Evan Chenge87681c2012-02-23 01:19:06 +000011349 case ISD::XOR: return PerformXORCombine(N, DCI, Subtarget);
11350 case ISD::AND: return PerformANDCombine(N, DCI, Subtarget);
Evan Chengc1778132010-12-14 03:22:07 +000011351 case ARMISD::BFI: return PerformBFICombine(N, DCI);
Oliver Stannard51b1d462014-08-21 12:50:31 +000011352 case ARMISD::VMOVRRD: return PerformVMOVRRDCombine(N, DCI, Subtarget);
Bob Wilson22806742010-09-22 22:09:21 +000011353 case ARMISD::VMOVDRR: return PerformVMOVDRRCombine(N, DCI.DAG);
Bob Wilson1a20c2a2010-12-21 06:43:19 +000011354 case ISD::STORE: return PerformSTORECombine(N, DCI);
Oliver Stannard51b1d462014-08-21 12:50:31 +000011355 case ISD::BUILD_VECTOR: return PerformBUILD_VECTORCombine(N, DCI, Subtarget);
Bob Wilson1a20c2a2010-12-21 06:43:19 +000011356 case ISD::INSERT_VECTOR_ELT: return PerformInsertEltCombine(N, DCI);
Bob Wilsonc7334a12010-10-27 20:38:28 +000011357 case ISD::VECTOR_SHUFFLE: return PerformVECTOR_SHUFFLECombine(N, DCI.DAG);
Bob Wilson2d790df2010-11-28 06:51:26 +000011358 case ARMISD::VDUPLANE: return PerformVDUPLANECombine(N, DCI);
Chad Rosierfa8d8932011-06-24 19:23:04 +000011359 case ISD::FP_TO_SINT:
Chad Rosiera087fd22015-10-06 20:23:42 +000011360 case ISD::FP_TO_UINT:
11361 return PerformVCVTCombine(N, DCI.DAG, Subtarget);
11362 case ISD::FDIV:
11363 return PerformVDIVCombine(N, DCI.DAG, Subtarget);
Bob Wilsonc6c13a32010-02-18 06:05:53 +000011364 case ISD::INTRINSIC_WO_CHAIN: return PerformIntrinsicCombine(N, DCI.DAG);
Bob Wilson2e076c42009-06-22 23:27:02 +000011365 case ISD::SHL:
11366 case ISD::SRA:
Bob Wilsonc6c13a32010-02-18 06:05:53 +000011367 case ISD::SRL: return PerformShiftCombine(N, DCI.DAG, Subtarget);
Bob Wilson2e076c42009-06-22 23:27:02 +000011368 case ISD::SIGN_EXTEND:
11369 case ISD::ZERO_EXTEND:
Bob Wilsonc6c13a32010-02-18 06:05:53 +000011370 case ISD::ANY_EXTEND: return PerformExtendCombine(N, DCI.DAG, Subtarget);
Evan Chengf863e3f2011-07-13 00:42:17 +000011371 case ARMISD::CMOV: return PerformCMOVCombine(N, DCI.DAG);
Peter Collingbourne86b9fbe2016-03-21 18:00:02 +000011372 case ARMISD::BRCOND: return PerformBRCONDCombine(N, DCI.DAG);
Ahmed Bougachadb141ac2015-02-19 23:52:41 +000011373 case ISD::LOAD: return PerformLOADCombine(N, DCI);
Bob Wilson06fce872011-02-07 17:43:21 +000011374 case ARMISD::VLD2DUP:
11375 case ARMISD::VLD3DUP:
11376 case ARMISD::VLD4DUP:
Ahmed Bougachadb141ac2015-02-19 23:52:41 +000011377 return PerformVLDCombine(N, DCI);
Quentin Colombet04b3a0f2013-07-03 21:42:57 +000011378 case ARMISD::BUILD_VECTOR:
11379 return PerformARMBUILD_VECTORCombine(N, DCI);
Bob Wilson06fce872011-02-07 17:43:21 +000011380 case ISD::INTRINSIC_VOID:
11381 case ISD::INTRINSIC_W_CHAIN:
11382 switch (cast<ConstantSDNode>(N->getOperand(1))->getZExtValue()) {
11383 case Intrinsic::arm_neon_vld1:
11384 case Intrinsic::arm_neon_vld2:
11385 case Intrinsic::arm_neon_vld3:
11386 case Intrinsic::arm_neon_vld4:
11387 case Intrinsic::arm_neon_vld2lane:
11388 case Intrinsic::arm_neon_vld3lane:
11389 case Intrinsic::arm_neon_vld4lane:
11390 case Intrinsic::arm_neon_vst1:
11391 case Intrinsic::arm_neon_vst2:
11392 case Intrinsic::arm_neon_vst3:
11393 case Intrinsic::arm_neon_vst4:
11394 case Intrinsic::arm_neon_vst2lane:
11395 case Intrinsic::arm_neon_vst3lane:
11396 case Intrinsic::arm_neon_vst4lane:
Ahmed Bougachadb141ac2015-02-19 23:52:41 +000011397 return PerformVLDCombine(N, DCI);
Bob Wilson06fce872011-02-07 17:43:21 +000011398 default: break;
11399 }
11400 break;
Chris Lattnerf3f4ad92007-11-27 22:36:16 +000011401 }
Dan Gohman2ce6f2a2008-07-27 21:46:04 +000011402 return SDValue();
Chris Lattnerf3f4ad92007-11-27 22:36:16 +000011403}
11404
Evan Chengd42641c2011-02-02 01:06:55 +000011405bool ARMTargetLowering::isDesirableToTransformToIntegerOp(unsigned Opc,
11406 EVT VT) const {
11407 return (VT == MVT::f32) && (Opc == ISD::LOAD || Opc == ISD::STORE);
11408}
11409
Matt Arsenault6f2a5262014-07-27 17:46:40 +000011410bool ARMTargetLowering::allowsMisalignedMemoryAccesses(EVT VT,
11411 unsigned,
11412 unsigned,
11413 bool *Fast) const {
Evan Cheng90ae8f82012-09-18 01:42:45 +000011414 // The AllowsUnaliged flag models the SCTLR.A setting in ARM cpus
Chad Rosier66bb1782012-11-09 18:25:27 +000011415 bool AllowsUnaligned = Subtarget->allowsUnalignedMem();
Bill Wendlingbae6b2c2009-08-15 21:21:19 +000011416
11417 switch (VT.getSimpleVT().SimpleTy) {
11418 default:
11419 return false;
11420 case MVT::i8:
11421 case MVT::i16:
Evan Cheng79e2ca92012-12-10 23:21:26 +000011422 case MVT::i32: {
Evan Cheng90ae8f82012-09-18 01:42:45 +000011423 // Unaligned access can use (for example) LRDB, LRDH, LDR
Evan Cheng79e2ca92012-12-10 23:21:26 +000011424 if (AllowsUnaligned) {
11425 if (Fast)
11426 *Fast = Subtarget->hasV7Ops();
11427 return true;
11428 }
11429 return false;
11430 }
Evan Chengeec6bc62012-08-15 17:44:53 +000011431 case MVT::f64:
Evan Cheng79e2ca92012-12-10 23:21:26 +000011432 case MVT::v2f64: {
Evan Cheng90ae8f82012-09-18 01:42:45 +000011433 // For any little-endian targets with neon, we can support unaligned ld/st
11434 // of D and Q (e.g. {D0,D1}) registers by using vld1.i8/vst1.i8.
Alp Tokercb402912014-01-24 17:20:08 +000011435 // A big-endian target may also explicitly support unaligned accesses
Mehdi Aminiffc14022015-07-08 01:00:38 +000011436 if (Subtarget->hasNEON() && (AllowsUnaligned || Subtarget->isLittle())) {
Evan Cheng79e2ca92012-12-10 23:21:26 +000011437 if (Fast)
11438 *Fast = true;
11439 return true;
11440 }
11441 return false;
11442 }
Bill Wendlingbae6b2c2009-08-15 21:21:19 +000011443 }
11444}
11445
Lang Hames9929c422011-11-02 22:52:45 +000011446static bool memOpAlign(unsigned DstAlign, unsigned SrcAlign,
11447 unsigned AlignCheck) {
11448 return ((SrcAlign == 0 || SrcAlign % AlignCheck == 0) &&
11449 (DstAlign == 0 || DstAlign % AlignCheck == 0));
11450}
11451
11452EVT ARMTargetLowering::getOptimalMemOpType(uint64_t Size,
11453 unsigned DstAlign, unsigned SrcAlign,
Evan Cheng962711e2012-12-12 02:34:41 +000011454 bool IsMemset, bool ZeroMemset,
Lang Hames9929c422011-11-02 22:52:45 +000011455 bool MemcpyStrSrc,
11456 MachineFunction &MF) const {
11457 const Function *F = MF.getFunction();
11458
11459 // See if we can use NEON instructions for this...
Duncan P. N. Exon Smith2cff9e12015-02-14 02:24:44 +000011460 if ((!IsMemset || ZeroMemset) && Subtarget->hasNEON() &&
11461 !F->hasFnAttribute(Attribute::NoImplicitFloat)) {
Evan Cheng79e2ca92012-12-10 23:21:26 +000011462 bool Fast;
Evan Chengc2bd6202012-12-11 02:31:57 +000011463 if (Size >= 16 &&
11464 (memOpAlign(SrcAlign, DstAlign, 16) ||
Matt Arsenault6f2a5262014-07-27 17:46:40 +000011465 (allowsMisalignedMemoryAccesses(MVT::v2f64, 0, 1, &Fast) && Fast))) {
Evan Cheng79e2ca92012-12-10 23:21:26 +000011466 return MVT::v2f64;
Evan Chengc2bd6202012-12-11 02:31:57 +000011467 } else if (Size >= 8 &&
11468 (memOpAlign(SrcAlign, DstAlign, 8) ||
Matt Arsenault6f2a5262014-07-27 17:46:40 +000011469 (allowsMisalignedMemoryAccesses(MVT::f64, 0, 1, &Fast) &&
11470 Fast))) {
Evan Cheng79e2ca92012-12-10 23:21:26 +000011471 return MVT::f64;
Lang Hames9929c422011-11-02 22:52:45 +000011472 }
11473 }
11474
Lang Hamesb85fcd02011-11-08 18:56:23 +000011475 // Lowering to i32/i16 if the size permits.
Evan Chengc2bd6202012-12-11 02:31:57 +000011476 if (Size >= 4)
Lang Hamesb85fcd02011-11-08 18:56:23 +000011477 return MVT::i32;
Evan Chengc2bd6202012-12-11 02:31:57 +000011478 else if (Size >= 2)
Lang Hamesb85fcd02011-11-08 18:56:23 +000011479 return MVT::i16;
Lang Hamesb85fcd02011-11-08 18:56:23 +000011480
Lang Hames9929c422011-11-02 22:52:45 +000011481 // Let the target-independent logic figure it out.
11482 return MVT::Other;
11483}
11484
Evan Cheng9ec512d2012-12-06 19:13:27 +000011485bool ARMTargetLowering::isZExtFree(SDValue Val, EVT VT2) const {
11486 if (Val.getOpcode() != ISD::LOAD)
11487 return false;
11488
11489 EVT VT1 = Val.getValueType();
11490 if (!VT1.isSimple() || !VT1.isInteger() ||
11491 !VT2.isSimple() || !VT2.isInteger())
11492 return false;
11493
11494 switch (VT1.getSimpleVT().SimpleTy) {
11495 default: break;
11496 case MVT::i1:
11497 case MVT::i8:
11498 case MVT::i16:
11499 // 8-bit and 16-bit loads implicitly zero-extend to 32-bits.
11500 return true;
11501 }
11502
11503 return false;
11504}
11505
Ahmed Bougacha4200cc92015-03-05 19:37:53 +000011506bool ARMTargetLowering::isVectorLoadExtDesirable(SDValue ExtVal) const {
11507 EVT VT = ExtVal.getValueType();
11508
11509 if (!isTypeLegal(VT))
11510 return false;
11511
11512 // Don't create a loadext if we can fold the extension into a wide/long
11513 // instruction.
11514 // If there's more than one user instruction, the loadext is desirable no
11515 // matter what. There can be two uses by the same instruction.
11516 if (ExtVal->use_empty() ||
11517 !ExtVal->use_begin()->isOnlyUserOf(ExtVal.getNode()))
11518 return true;
11519
11520 SDNode *U = *ExtVal->use_begin();
11521 if ((U->getOpcode() == ISD::ADD || U->getOpcode() == ISD::SUB ||
11522 U->getOpcode() == ISD::SHL || U->getOpcode() == ARMISD::VSHL))
11523 return false;
11524
11525 return true;
11526}
11527
Tim Northovercc2e9032013-08-06 13:58:03 +000011528bool ARMTargetLowering::allowTruncateForTailCall(Type *Ty1, Type *Ty2) const {
11529 if (!Ty1->isIntegerTy() || !Ty2->isIntegerTy())
11530 return false;
11531
11532 if (!isTypeLegal(EVT::getEVT(Ty1)))
11533 return false;
11534
11535 assert(Ty1->getPrimitiveSizeInBits() <= 64 && "i128 is probably not a noop");
11536
11537 // Assuming the caller doesn't have a zeroext or signext return parameter,
11538 // truncation all the way down to i1 is valid.
11539 return true;
11540}
11541
11542
Evan Chengdc49a8d2009-08-14 20:09:37 +000011543static bool isLegalT1AddressImmediate(int64_t V, EVT VT) {
11544 if (V < 0)
11545 return false;
11546
11547 unsigned Scale = 1;
11548 switch (VT.getSimpleVT().SimpleTy) {
11549 default: return false;
11550 case MVT::i1:
11551 case MVT::i8:
11552 // Scale == 1;
11553 break;
11554 case MVT::i16:
11555 // Scale == 2;
11556 Scale = 2;
11557 break;
11558 case MVT::i32:
11559 // Scale == 4;
11560 Scale = 4;
11561 break;
11562 }
11563
11564 if ((V & (Scale - 1)) != 0)
11565 return false;
11566 V /= Scale;
11567 return V == (V & ((1LL << 5) - 1));
11568}
11569
11570static bool isLegalT2AddressImmediate(int64_t V, EVT VT,
11571 const ARMSubtarget *Subtarget) {
11572 bool isNeg = false;
11573 if (V < 0) {
11574 isNeg = true;
11575 V = - V;
11576 }
11577
11578 switch (VT.getSimpleVT().SimpleTy) {
11579 default: return false;
11580 case MVT::i1:
11581 case MVT::i8:
11582 case MVT::i16:
11583 case MVT::i32:
11584 // + imm12 or - imm8
11585 if (isNeg)
11586 return V == (V & ((1LL << 8) - 1));
11587 return V == (V & ((1LL << 12) - 1));
11588 case MVT::f32:
11589 case MVT::f64:
11590 // Same as ARM mode. FIXME: NEON?
11591 if (!Subtarget->hasVFP2())
11592 return false;
11593 if ((V & 3) != 0)
11594 return false;
11595 V >>= 2;
11596 return V == (V & ((1LL << 8) - 1));
11597 }
11598}
11599
Evan Cheng2150b922007-03-12 23:30:29 +000011600/// isLegalAddressImmediate - Return true if the integer value can be used
11601/// as the offset of the target addressing mode for load / store of the
11602/// given type.
Owen Anderson53aa7a92009-08-10 22:56:29 +000011603static bool isLegalAddressImmediate(int64_t V, EVT VT,
Chris Lattnerd44e24c2007-04-09 23:33:39 +000011604 const ARMSubtarget *Subtarget) {
Evan Cheng507eefa2007-03-13 20:37:59 +000011605 if (V == 0)
11606 return true;
11607
Evan Chengce5dfb62009-03-09 19:15:00 +000011608 if (!VT.isSimple())
11609 return false;
11610
Evan Chengdc49a8d2009-08-14 20:09:37 +000011611 if (Subtarget->isThumb1Only())
11612 return isLegalT1AddressImmediate(V, VT);
11613 else if (Subtarget->isThumb2())
11614 return isLegalT2AddressImmediate(V, VT, Subtarget);
Evan Cheng2150b922007-03-12 23:30:29 +000011615
Evan Chengdc49a8d2009-08-14 20:09:37 +000011616 // ARM mode.
Evan Cheng2150b922007-03-12 23:30:29 +000011617 if (V < 0)
11618 V = - V;
Owen Anderson9f944592009-08-11 20:47:22 +000011619 switch (VT.getSimpleVT().SimpleTy) {
Evan Cheng2150b922007-03-12 23:30:29 +000011620 default: return false;
Owen Anderson9f944592009-08-11 20:47:22 +000011621 case MVT::i1:
11622 case MVT::i8:
11623 case MVT::i32:
Evan Cheng2150b922007-03-12 23:30:29 +000011624 // +- imm12
Anton Korobeynikov40d67c52008-02-20 11:22:39 +000011625 return V == (V & ((1LL << 12) - 1));
Owen Anderson9f944592009-08-11 20:47:22 +000011626 case MVT::i16:
Evan Cheng2150b922007-03-12 23:30:29 +000011627 // +- imm8
Anton Korobeynikov40d67c52008-02-20 11:22:39 +000011628 return V == (V & ((1LL << 8) - 1));
Owen Anderson9f944592009-08-11 20:47:22 +000011629 case MVT::f32:
11630 case MVT::f64:
Evan Chengdc49a8d2009-08-14 20:09:37 +000011631 if (!Subtarget->hasVFP2()) // FIXME: NEON?
Evan Cheng2150b922007-03-12 23:30:29 +000011632 return false;
Evan Chengbef131de2007-05-03 02:00:18 +000011633 if ((V & 3) != 0)
Evan Cheng2150b922007-03-12 23:30:29 +000011634 return false;
11635 V >>= 2;
Anton Korobeynikov40d67c52008-02-20 11:22:39 +000011636 return V == (V & ((1LL << 8) - 1));
Evan Cheng2150b922007-03-12 23:30:29 +000011637 }
Evan Cheng10043e22007-01-19 07:51:42 +000011638}
11639
Evan Chengdc49a8d2009-08-14 20:09:37 +000011640bool ARMTargetLowering::isLegalT2ScaledAddressingMode(const AddrMode &AM,
11641 EVT VT) const {
11642 int Scale = AM.Scale;
11643 if (Scale < 0)
11644 return false;
11645
11646 switch (VT.getSimpleVT().SimpleTy) {
11647 default: return false;
11648 case MVT::i1:
11649 case MVT::i8:
11650 case MVT::i16:
11651 case MVT::i32:
11652 if (Scale == 1)
11653 return true;
11654 // r + r << imm
11655 Scale = Scale & ~1;
11656 return Scale == 2 || Scale == 4 || Scale == 8;
11657 case MVT::i64:
11658 // r + r
11659 if (((unsigned)AM.HasBaseReg + Scale) <= 2)
11660 return true;
11661 return false;
11662 case MVT::isVoid:
11663 // Note, we allow "void" uses (basically, uses that aren't loads or
11664 // stores), because arm allows folding a scale into many arithmetic
11665 // operations. This should be made more precise and revisited later.
11666
11667 // Allow r << imm, but the imm has to be a multiple of two.
11668 if (Scale & 1) return false;
11669 return isPowerOf2_32(Scale);
11670 }
11671}
11672
Chris Lattnerd44e24c2007-04-09 23:33:39 +000011673/// isLegalAddressingMode - Return true if the addressing mode represented
11674/// by AM is legal for this target, for a load/store of the specified type.
Mehdi Amini0cdec1e2015-07-09 02:09:40 +000011675bool ARMTargetLowering::isLegalAddressingMode(const DataLayout &DL,
11676 const AddrMode &AM, Type *Ty,
Matt Arsenaultbd7d80a2015-06-01 05:31:59 +000011677 unsigned AS) const {
Mehdi Amini0cdec1e2015-07-09 02:09:40 +000011678 EVT VT = getValueType(DL, Ty, true);
Bob Wilson866c1742009-04-08 17:55:28 +000011679 if (!isLegalAddressImmediate(AM.BaseOffs, VT, Subtarget))
Evan Cheng2150b922007-03-12 23:30:29 +000011680 return false;
Bob Wilson7117a912009-03-20 22:42:55 +000011681
Chris Lattnerd44e24c2007-04-09 23:33:39 +000011682 // Can never fold addr of global into load/store.
Bob Wilson7117a912009-03-20 22:42:55 +000011683 if (AM.BaseGV)
Chris Lattnerd44e24c2007-04-09 23:33:39 +000011684 return false;
Bob Wilson7117a912009-03-20 22:42:55 +000011685
Chris Lattnerd44e24c2007-04-09 23:33:39 +000011686 switch (AM.Scale) {
11687 case 0: // no scale reg, must be "r+i" or "r", or "i".
11688 break;
11689 case 1:
Evan Chengdc49a8d2009-08-14 20:09:37 +000011690 if (Subtarget->isThumb1Only())
Chris Lattnerd44e24c2007-04-09 23:33:39 +000011691 return false;
Justin Bognercd1d5aa2016-08-17 20:30:52 +000011692 LLVM_FALLTHROUGH;
Chris Lattnerd44e24c2007-04-09 23:33:39 +000011693 default:
Chris Lattner502c3f42007-04-13 06:50:55 +000011694 // ARM doesn't support any R+R*scale+imm addr modes.
11695 if (AM.BaseOffs)
11696 return false;
Bob Wilson7117a912009-03-20 22:42:55 +000011697
Bob Wilson866c1742009-04-08 17:55:28 +000011698 if (!VT.isSimple())
11699 return false;
11700
Evan Chengdc49a8d2009-08-14 20:09:37 +000011701 if (Subtarget->isThumb2())
11702 return isLegalT2ScaledAddressingMode(AM, VT);
11703
Chris Lattner9b6d69e2007-04-10 03:48:29 +000011704 int Scale = AM.Scale;
Owen Anderson9f944592009-08-11 20:47:22 +000011705 switch (VT.getSimpleVT().SimpleTy) {
Chris Lattnerd44e24c2007-04-09 23:33:39 +000011706 default: return false;
Owen Anderson9f944592009-08-11 20:47:22 +000011707 case MVT::i1:
11708 case MVT::i8:
11709 case MVT::i32:
Chris Lattner9b6d69e2007-04-10 03:48:29 +000011710 if (Scale < 0) Scale = -Scale;
11711 if (Scale == 1)
Chris Lattnerd44e24c2007-04-09 23:33:39 +000011712 return true;
11713 // r + r << imm
Chris Lattnerfe926e22007-04-11 16:17:12 +000011714 return isPowerOf2_32(Scale & ~1);
Owen Anderson9f944592009-08-11 20:47:22 +000011715 case MVT::i16:
Evan Chengdc49a8d2009-08-14 20:09:37 +000011716 case MVT::i64:
Chris Lattnerd44e24c2007-04-09 23:33:39 +000011717 // r + r
Chris Lattner9b6d69e2007-04-10 03:48:29 +000011718 if (((unsigned)AM.HasBaseReg + Scale) <= 2)
Chris Lattnerd44e24c2007-04-09 23:33:39 +000011719 return true;
Chris Lattnerfe926e22007-04-11 16:17:12 +000011720 return false;
Bob Wilson7117a912009-03-20 22:42:55 +000011721
Owen Anderson9f944592009-08-11 20:47:22 +000011722 case MVT::isVoid:
Chris Lattnerd44e24c2007-04-09 23:33:39 +000011723 // Note, we allow "void" uses (basically, uses that aren't loads or
11724 // stores), because arm allows folding a scale into many arithmetic
11725 // operations. This should be made more precise and revisited later.
Bob Wilson7117a912009-03-20 22:42:55 +000011726
Chris Lattnerd44e24c2007-04-09 23:33:39 +000011727 // Allow r << imm, but the imm has to be a multiple of two.
Evan Chengdc49a8d2009-08-14 20:09:37 +000011728 if (Scale & 1) return false;
11729 return isPowerOf2_32(Scale);
Chris Lattnerd44e24c2007-04-09 23:33:39 +000011730 }
Evan Cheng2150b922007-03-12 23:30:29 +000011731 }
Chris Lattnerd44e24c2007-04-09 23:33:39 +000011732 return true;
Evan Cheng2150b922007-03-12 23:30:29 +000011733}
11734
Evan Cheng3d3c24a2009-11-11 19:05:52 +000011735/// isLegalICmpImmediate - Return true if the specified immediate is legal
11736/// icmp immediate, that is the target has icmp instructions which can compare
11737/// a register against the immediate without having to materialize the
11738/// immediate into a register.
Evan Cheng15b80e42009-11-12 07:13:11 +000011739bool ARMTargetLowering::isLegalICmpImmediate(int64_t Imm) const {
Jakob Stoklund Olesen967b86a2012-04-06 17:45:04 +000011740 // Thumb2 and ARM modes can use cmn for negative immediates.
Evan Cheng3d3c24a2009-11-11 19:05:52 +000011741 if (!Subtarget->isThumb())
Benjamin Kramer7bd1f7c2015-03-09 20:20:16 +000011742 return ARM_AM::getSOImmVal(std::abs(Imm)) != -1;
Evan Cheng3d3c24a2009-11-11 19:05:52 +000011743 if (Subtarget->isThumb2())
Benjamin Kramer7bd1f7c2015-03-09 20:20:16 +000011744 return ARM_AM::getT2SOImmVal(std::abs(Imm)) != -1;
Jakob Stoklund Olesen967b86a2012-04-06 17:45:04 +000011745 // Thumb1 doesn't have cmn, and only 8-bit immediates.
Evan Cheng15b80e42009-11-12 07:13:11 +000011746 return Imm >= 0 && Imm <= 255;
Evan Cheng3d3c24a2009-11-11 19:05:52 +000011747}
11748
Andrew Tricka22cdb72012-07-18 18:34:27 +000011749/// isLegalAddImmediate - Return true if the specified immediate is a legal add
11750/// *or sub* immediate, that is the target has add or sub instructions which can
11751/// add a register with the immediate without having to materialize the
Dan Gohman6136e942011-05-03 00:46:49 +000011752/// immediate into a register.
11753bool ARMTargetLowering::isLegalAddImmediate(int64_t Imm) const {
Andrew Tricka22cdb72012-07-18 18:34:27 +000011754 // Same encoding for add/sub, just flip the sign.
Benjamin Kramer7bd1f7c2015-03-09 20:20:16 +000011755 int64_t AbsImm = std::abs(Imm);
Andrew Tricka22cdb72012-07-18 18:34:27 +000011756 if (!Subtarget->isThumb())
11757 return ARM_AM::getSOImmVal(AbsImm) != -1;
11758 if (Subtarget->isThumb2())
11759 return ARM_AM::getT2SOImmVal(AbsImm) != -1;
11760 // Thumb1 only has 8-bit unsigned immediate.
11761 return AbsImm >= 0 && AbsImm <= 255;
Dan Gohman6136e942011-05-03 00:46:49 +000011762}
11763
Owen Anderson53aa7a92009-08-10 22:56:29 +000011764static bool getARMIndexedAddressParts(SDNode *Ptr, EVT VT,
Evan Cheng84c6cda2009-07-02 07:28:31 +000011765 bool isSEXTLoad, SDValue &Base,
11766 SDValue &Offset, bool &isInc,
11767 SelectionDAG &DAG) {
Evan Cheng10043e22007-01-19 07:51:42 +000011768 if (Ptr->getOpcode() != ISD::ADD && Ptr->getOpcode() != ISD::SUB)
11769 return false;
11770
Owen Anderson9f944592009-08-11 20:47:22 +000011771 if (VT == MVT::i16 || ((VT == MVT::i8 || VT == MVT::i1) && isSEXTLoad)) {
Evan Cheng10043e22007-01-19 07:51:42 +000011772 // AddressingMode 3
11773 Base = Ptr->getOperand(0);
11774 if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(Ptr->getOperand(1))) {
Dan Gohmaneffb8942008-09-12 16:56:44 +000011775 int RHSC = (int)RHS->getZExtValue();
Evan Cheng10043e22007-01-19 07:51:42 +000011776 if (RHSC < 0 && RHSC > -256) {
Evan Cheng84c6cda2009-07-02 07:28:31 +000011777 assert(Ptr->getOpcode() == ISD::ADD);
Evan Cheng10043e22007-01-19 07:51:42 +000011778 isInc = false;
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +000011779 Offset = DAG.getConstant(-RHSC, SDLoc(Ptr), RHS->getValueType(0));
Evan Cheng10043e22007-01-19 07:51:42 +000011780 return true;
11781 }
11782 }
11783 isInc = (Ptr->getOpcode() == ISD::ADD);
11784 Offset = Ptr->getOperand(1);
11785 return true;
Owen Anderson9f944592009-08-11 20:47:22 +000011786 } else if (VT == MVT::i32 || VT == MVT::i8 || VT == MVT::i1) {
Evan Cheng10043e22007-01-19 07:51:42 +000011787 // AddressingMode 2
11788 if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(Ptr->getOperand(1))) {
Dan Gohmaneffb8942008-09-12 16:56:44 +000011789 int RHSC = (int)RHS->getZExtValue();
Evan Cheng10043e22007-01-19 07:51:42 +000011790 if (RHSC < 0 && RHSC > -0x1000) {
Evan Cheng84c6cda2009-07-02 07:28:31 +000011791 assert(Ptr->getOpcode() == ISD::ADD);
Evan Cheng10043e22007-01-19 07:51:42 +000011792 isInc = false;
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +000011793 Offset = DAG.getConstant(-RHSC, SDLoc(Ptr), RHS->getValueType(0));
Evan Cheng10043e22007-01-19 07:51:42 +000011794 Base = Ptr->getOperand(0);
11795 return true;
11796 }
11797 }
11798
11799 if (Ptr->getOpcode() == ISD::ADD) {
11800 isInc = true;
Evan Chenga20cde32011-07-20 23:34:39 +000011801 ARM_AM::ShiftOpc ShOpcVal=
11802 ARM_AM::getShiftOpcForNode(Ptr->getOperand(0).getOpcode());
Evan Cheng10043e22007-01-19 07:51:42 +000011803 if (ShOpcVal != ARM_AM::no_shift) {
11804 Base = Ptr->getOperand(1);
11805 Offset = Ptr->getOperand(0);
11806 } else {
11807 Base = Ptr->getOperand(0);
11808 Offset = Ptr->getOperand(1);
11809 }
11810 return true;
11811 }
11812
11813 isInc = (Ptr->getOpcode() == ISD::ADD);
11814 Base = Ptr->getOperand(0);
11815 Offset = Ptr->getOperand(1);
11816 return true;
11817 }
11818
Jim Grosbachd7cf55c2009-11-09 00:11:35 +000011819 // FIXME: Use VLDM / VSTM to emulate indexed FP load / store.
Evan Cheng10043e22007-01-19 07:51:42 +000011820 return false;
11821}
11822
Owen Anderson53aa7a92009-08-10 22:56:29 +000011823static bool getT2IndexedAddressParts(SDNode *Ptr, EVT VT,
Evan Cheng84c6cda2009-07-02 07:28:31 +000011824 bool isSEXTLoad, SDValue &Base,
11825 SDValue &Offset, bool &isInc,
11826 SelectionDAG &DAG) {
11827 if (Ptr->getOpcode() != ISD::ADD && Ptr->getOpcode() != ISD::SUB)
11828 return false;
11829
11830 Base = Ptr->getOperand(0);
11831 if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(Ptr->getOperand(1))) {
11832 int RHSC = (int)RHS->getZExtValue();
11833 if (RHSC < 0 && RHSC > -0x100) { // 8 bits.
11834 assert(Ptr->getOpcode() == ISD::ADD);
11835 isInc = false;
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +000011836 Offset = DAG.getConstant(-RHSC, SDLoc(Ptr), RHS->getValueType(0));
Evan Cheng84c6cda2009-07-02 07:28:31 +000011837 return true;
11838 } else if (RHSC > 0 && RHSC < 0x100) { // 8 bit, no zero.
11839 isInc = Ptr->getOpcode() == ISD::ADD;
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +000011840 Offset = DAG.getConstant(RHSC, SDLoc(Ptr), RHS->getValueType(0));
Evan Cheng84c6cda2009-07-02 07:28:31 +000011841 return true;
11842 }
11843 }
11844
11845 return false;
11846}
11847
Evan Cheng10043e22007-01-19 07:51:42 +000011848/// getPreIndexedAddressParts - returns true by value, base pointer and
11849/// offset pointer and addressing mode by reference if the node's address
11850/// can be legally represented as pre-indexed load / store address.
11851bool
Dan Gohman2ce6f2a2008-07-27 21:46:04 +000011852ARMTargetLowering::getPreIndexedAddressParts(SDNode *N, SDValue &Base,
11853 SDValue &Offset,
Evan Cheng10043e22007-01-19 07:51:42 +000011854 ISD::MemIndexedMode &AM,
Dan Gohman02b93132009-01-15 16:29:45 +000011855 SelectionDAG &DAG) const {
Evan Cheng84c6cda2009-07-02 07:28:31 +000011856 if (Subtarget->isThumb1Only())
Evan Cheng10043e22007-01-19 07:51:42 +000011857 return false;
11858
Owen Anderson53aa7a92009-08-10 22:56:29 +000011859 EVT VT;
Dan Gohman2ce6f2a2008-07-27 21:46:04 +000011860 SDValue Ptr;
Evan Cheng10043e22007-01-19 07:51:42 +000011861 bool isSEXTLoad = false;
11862 if (LoadSDNode *LD = dyn_cast<LoadSDNode>(N)) {
11863 Ptr = LD->getBasePtr();
Dan Gohman47a7d6f2008-01-30 00:15:11 +000011864 VT = LD->getMemoryVT();
Evan Cheng10043e22007-01-19 07:51:42 +000011865 isSEXTLoad = LD->getExtensionType() == ISD::SEXTLOAD;
11866 } else if (StoreSDNode *ST = dyn_cast<StoreSDNode>(N)) {
11867 Ptr = ST->getBasePtr();
Dan Gohman47a7d6f2008-01-30 00:15:11 +000011868 VT = ST->getMemoryVT();
Evan Cheng10043e22007-01-19 07:51:42 +000011869 } else
11870 return false;
11871
11872 bool isInc;
Evan Cheng84c6cda2009-07-02 07:28:31 +000011873 bool isLegal = false;
Evan Chengdc49a8d2009-08-14 20:09:37 +000011874 if (Subtarget->isThumb2())
Evan Cheng84c6cda2009-07-02 07:28:31 +000011875 isLegal = getT2IndexedAddressParts(Ptr.getNode(), VT, isSEXTLoad, Base,
11876 Offset, isInc, DAG);
Jim Grosbachf24f9d92009-08-11 15:33:49 +000011877 else
Evan Cheng84c6cda2009-07-02 07:28:31 +000011878 isLegal = getARMIndexedAddressParts(Ptr.getNode(), VT, isSEXTLoad, Base,
Evan Cheng844f0b42009-07-02 06:44:30 +000011879 Offset, isInc, DAG);
Evan Cheng84c6cda2009-07-02 07:28:31 +000011880 if (!isLegal)
11881 return false;
11882
11883 AM = isInc ? ISD::PRE_INC : ISD::PRE_DEC;
11884 return true;
Evan Cheng10043e22007-01-19 07:51:42 +000011885}
11886
11887/// getPostIndexedAddressParts - returns true by value, base pointer and
11888/// offset pointer and addressing mode by reference if this node can be
11889/// combined with a load / store to form a post-indexed load / store.
11890bool ARMTargetLowering::getPostIndexedAddressParts(SDNode *N, SDNode *Op,
Dan Gohman2ce6f2a2008-07-27 21:46:04 +000011891 SDValue &Base,
11892 SDValue &Offset,
Evan Cheng10043e22007-01-19 07:51:42 +000011893 ISD::MemIndexedMode &AM,
Dan Gohman02b93132009-01-15 16:29:45 +000011894 SelectionDAG &DAG) const {
Owen Anderson53aa7a92009-08-10 22:56:29 +000011895 EVT VT;
Dan Gohman2ce6f2a2008-07-27 21:46:04 +000011896 SDValue Ptr;
James Molloyb3326df2016-07-15 08:03:56 +000011897 bool isSEXTLoad = false, isNonExt;
Evan Cheng10043e22007-01-19 07:51:42 +000011898 if (LoadSDNode *LD = dyn_cast<LoadSDNode>(N)) {
Dan Gohman47a7d6f2008-01-30 00:15:11 +000011899 VT = LD->getMemoryVT();
Evan Chengf19384d2010-05-18 21:31:17 +000011900 Ptr = LD->getBasePtr();
Evan Cheng10043e22007-01-19 07:51:42 +000011901 isSEXTLoad = LD->getExtensionType() == ISD::SEXTLOAD;
James Molloyb3326df2016-07-15 08:03:56 +000011902 isNonExt = LD->getExtensionType() == ISD::NON_EXTLOAD;
Evan Cheng10043e22007-01-19 07:51:42 +000011903 } else if (StoreSDNode *ST = dyn_cast<StoreSDNode>(N)) {
Dan Gohman47a7d6f2008-01-30 00:15:11 +000011904 VT = ST->getMemoryVT();
Evan Chengf19384d2010-05-18 21:31:17 +000011905 Ptr = ST->getBasePtr();
James Molloyb3326df2016-07-15 08:03:56 +000011906 isNonExt = !ST->isTruncatingStore();
Evan Cheng10043e22007-01-19 07:51:42 +000011907 } else
11908 return false;
11909
James Molloyb3326df2016-07-15 08:03:56 +000011910 if (Subtarget->isThumb1Only()) {
11911 // Thumb-1 can do a limited post-inc load or store as an updating LDM. It
11912 // must be non-extending/truncating, i32, with an offset of 4.
11913 assert(Op->getValueType(0) == MVT::i32 && "Non-i32 post-inc op?!");
11914 if (Op->getOpcode() != ISD::ADD || !isNonExt)
11915 return false;
11916 auto *RHS = dyn_cast<ConstantSDNode>(Op->getOperand(1));
11917 if (!RHS || RHS->getZExtValue() != 4)
11918 return false;
11919
11920 Offset = Op->getOperand(1);
11921 Base = Op->getOperand(0);
11922 AM = ISD::POST_INC;
11923 return true;
11924 }
11925
Evan Cheng10043e22007-01-19 07:51:42 +000011926 bool isInc;
Evan Cheng84c6cda2009-07-02 07:28:31 +000011927 bool isLegal = false;
Evan Chengdc49a8d2009-08-14 20:09:37 +000011928 if (Subtarget->isThumb2())
Evan Cheng84c6cda2009-07-02 07:28:31 +000011929 isLegal = getT2IndexedAddressParts(Op, VT, isSEXTLoad, Base, Offset,
Evan Chengf19384d2010-05-18 21:31:17 +000011930 isInc, DAG);
Jim Grosbachf24f9d92009-08-11 15:33:49 +000011931 else
Evan Cheng84c6cda2009-07-02 07:28:31 +000011932 isLegal = getARMIndexedAddressParts(Op, VT, isSEXTLoad, Base, Offset,
11933 isInc, DAG);
11934 if (!isLegal)
11935 return false;
11936
Evan Chengf19384d2010-05-18 21:31:17 +000011937 if (Ptr != Base) {
11938 // Swap base ptr and offset to catch more post-index load / store when
11939 // it's legal. In Thumb2 mode, offset must be an immediate.
11940 if (Ptr == Offset && Op->getOpcode() == ISD::ADD &&
11941 !Subtarget->isThumb2())
11942 std::swap(Base, Offset);
11943
11944 // Post-indexed load / store update the base pointer.
11945 if (Ptr != Base)
11946 return false;
11947 }
11948
Evan Cheng84c6cda2009-07-02 07:28:31 +000011949 AM = isInc ? ISD::POST_INC : ISD::POST_DEC;
11950 return true;
Evan Cheng10043e22007-01-19 07:51:42 +000011951}
11952
Jay Foada0653a32014-05-14 21:14:37 +000011953void ARMTargetLowering::computeKnownBitsForTargetNode(const SDValue Op,
11954 APInt &KnownZero,
11955 APInt &KnownOne,
11956 const SelectionDAG &DAG,
11957 unsigned Depth) const {
Michael Gottesman696e44e2013-06-18 20:49:45 +000011958 unsigned BitWidth = KnownOne.getBitWidth();
11959 KnownZero = KnownOne = APInt(BitWidth, 0);
Evan Cheng10043e22007-01-19 07:51:42 +000011960 switch (Op.getOpcode()) {
11961 default: break;
Michael Gottesman696e44e2013-06-18 20:49:45 +000011962 case ARMISD::ADDC:
11963 case ARMISD::ADDE:
11964 case ARMISD::SUBC:
11965 case ARMISD::SUBE:
11966 // These nodes' second result is a boolean
11967 if (Op.getResNo() == 0)
11968 break;
11969 KnownZero |= APInt::getHighBitsSet(BitWidth, BitWidth - 1);
11970 break;
Evan Cheng10043e22007-01-19 07:51:42 +000011971 case ARMISD::CMOV: {
11972 // Bits are known zero/one if known on the LHS and RHS.
Jay Foada0653a32014-05-14 21:14:37 +000011973 DAG.computeKnownBits(Op.getOperand(0), KnownZero, KnownOne, Depth+1);
Evan Cheng10043e22007-01-19 07:51:42 +000011974 if (KnownZero == 0 && KnownOne == 0) return;
11975
Dan Gohmanf990faf2008-02-13 00:35:47 +000011976 APInt KnownZeroRHS, KnownOneRHS;
Jay Foada0653a32014-05-14 21:14:37 +000011977 DAG.computeKnownBits(Op.getOperand(1), KnownZeroRHS, KnownOneRHS, Depth+1);
Evan Cheng10043e22007-01-19 07:51:42 +000011978 KnownZero &= KnownZeroRHS;
11979 KnownOne &= KnownOneRHS;
11980 return;
11981 }
Tim Northover01b4aa92014-04-03 15:10:35 +000011982 case ISD::INTRINSIC_W_CHAIN: {
11983 ConstantSDNode *CN = cast<ConstantSDNode>(Op->getOperand(1));
11984 Intrinsic::ID IntID = static_cast<Intrinsic::ID>(CN->getZExtValue());
11985 switch (IntID) {
11986 default: return;
11987 case Intrinsic::arm_ldaex:
11988 case Intrinsic::arm_ldrex: {
11989 EVT VT = cast<MemIntrinsicSDNode>(Op)->getMemoryVT();
Sanjay Patelbd6fca12016-09-14 15:21:00 +000011990 unsigned MemBits = VT.getScalarSizeInBits();
Tim Northover01b4aa92014-04-03 15:10:35 +000011991 KnownZero |= APInt::getHighBitsSet(BitWidth, BitWidth - MemBits);
11992 return;
11993 }
11994 }
11995 }
Evan Cheng10043e22007-01-19 07:51:42 +000011996 }
11997}
11998
11999//===----------------------------------------------------------------------===//
12000// ARM Inline Assembly Support
12001//===----------------------------------------------------------------------===//
12002
Evan Cheng078b0b02011-01-08 01:24:27 +000012003bool ARMTargetLowering::ExpandInlineAsm(CallInst *CI) const {
12004 // Looking for "rev" which is V6+.
12005 if (!Subtarget->hasV6Ops())
12006 return false;
12007
12008 InlineAsm *IA = cast<InlineAsm>(CI->getCalledValue());
12009 std::string AsmStr = IA->getAsmString();
12010 SmallVector<StringRef, 4> AsmPieces;
12011 SplitString(AsmStr, AsmPieces, ";\n");
12012
12013 switch (AsmPieces.size()) {
12014 default: return false;
12015 case 1:
12016 AsmStr = AsmPieces[0];
12017 AsmPieces.clear();
12018 SplitString(AsmStr, AsmPieces, " \t,");
12019
12020 // rev $0, $1
12021 if (AsmPieces.size() == 3 &&
12022 AsmPieces[0] == "rev" && AsmPieces[1] == "$0" && AsmPieces[2] == "$1" &&
12023 IA->getConstraintString().compare(0, 4, "=l,l") == 0) {
Chris Lattner229907c2011-07-18 04:54:35 +000012024 IntegerType *Ty = dyn_cast<IntegerType>(CI->getType());
Evan Cheng078b0b02011-01-08 01:24:27 +000012025 if (Ty && Ty->getBitWidth() == 32)
12026 return IntrinsicLowering::LowerToByteSwap(CI);
12027 }
12028 break;
12029 }
12030
12031 return false;
12032}
12033
Silviu Baranga82d04262016-04-25 14:29:18 +000012034const char *ARMTargetLowering::LowerXConstraint(EVT ConstraintVT) const {
12035 // At this point, we have to lower this constraint to something else, so we
12036 // lower it to an "r" or "w". However, by doing this we will force the result
12037 // to be in register, while the X constraint is much more permissive.
12038 //
12039 // Although we are correct (we are free to emit anything, without
12040 // constraints), we might break use cases that would expect us to be more
12041 // efficient and emit something else.
12042 if (!Subtarget->hasVFP2())
12043 return "r";
12044 if (ConstraintVT.isFloatingPoint())
12045 return "w";
12046 if (ConstraintVT.isVector() && Subtarget->hasNEON() &&
12047 (ConstraintVT.getSizeInBits() == 64 ||
12048 ConstraintVT.getSizeInBits() == 128))
12049 return "w";
12050
12051 return "r";
12052}
12053
Evan Cheng10043e22007-01-19 07:51:42 +000012054/// getConstraintType - Given a constraint letter, return the type of
12055/// constraint it is for this target.
12056ARMTargetLowering::ConstraintType
Benjamin Kramer9bfb6272015-07-05 19:29:18 +000012057ARMTargetLowering::getConstraintType(StringRef Constraint) const {
Chris Lattnerd6855142007-03-25 02:14:49 +000012058 if (Constraint.size() == 1) {
12059 switch (Constraint[0]) {
12060 default: break;
12061 case 'l': return C_RegisterClass;
Chris Lattner6223e832007-04-02 17:24:08 +000012062 case 'w': return C_RegisterClass;
Eric Christopherf45daac2011-06-30 23:23:01 +000012063 case 'h': return C_RegisterClass;
Eric Christopherf1c74592011-07-01 00:14:47 +000012064 case 'x': return C_RegisterClass;
Eric Christopherc011d312011-07-01 00:30:46 +000012065 case 't': return C_RegisterClass;
Eric Christopher29f1db82011-07-01 01:00:07 +000012066 case 'j': return C_Other; // Constant for movw.
Eric Christopheraa503002011-07-29 21:18:58 +000012067 // An address with a single base register. Due to the way we
12068 // currently handle addresses it is the same as an 'r' memory constraint.
12069 case 'Q': return C_Memory;
Chris Lattnerd6855142007-03-25 02:14:49 +000012070 }
Eric Christophere256cd02011-06-21 22:10:57 +000012071 } else if (Constraint.size() == 2) {
12072 switch (Constraint[0]) {
12073 default: break;
12074 // All 'U+' constraints are addresses.
12075 case 'U': return C_Memory;
12076 }
Evan Cheng10043e22007-01-19 07:51:42 +000012077 }
Chris Lattnerd6855142007-03-25 02:14:49 +000012078 return TargetLowering::getConstraintType(Constraint);
Evan Cheng10043e22007-01-19 07:51:42 +000012079}
12080
John Thompsone8360b72010-10-29 17:29:13 +000012081/// Examine constraint type and operand type and determine a weight value.
12082/// This object must already have been set up with the operand type
12083/// and the current alternative constraint selected.
12084TargetLowering::ConstraintWeight
12085ARMTargetLowering::getSingleConstraintMatchWeight(
12086 AsmOperandInfo &info, const char *constraint) const {
12087 ConstraintWeight weight = CW_Invalid;
12088 Value *CallOperandVal = info.CallOperandVal;
12089 // If we don't have a value, we can't do a match,
12090 // but allow it at the lowest weight.
Craig Topper062a2ba2014-04-25 05:30:21 +000012091 if (!CallOperandVal)
John Thompsone8360b72010-10-29 17:29:13 +000012092 return CW_Default;
Chris Lattner229907c2011-07-18 04:54:35 +000012093 Type *type = CallOperandVal->getType();
John Thompsone8360b72010-10-29 17:29:13 +000012094 // Look at the constraint type.
12095 switch (*constraint) {
12096 default:
12097 weight = TargetLowering::getSingleConstraintMatchWeight(info, constraint);
12098 break;
12099 case 'l':
12100 if (type->isIntegerTy()) {
12101 if (Subtarget->isThumb())
12102 weight = CW_SpecificReg;
12103 else
12104 weight = CW_Register;
12105 }
12106 break;
12107 case 'w':
12108 if (type->isFloatingPointTy())
12109 weight = CW_Register;
12110 break;
12111 }
12112 return weight;
12113}
12114
Eric Christophercf2007c2011-06-30 23:50:52 +000012115typedef std::pair<unsigned, const TargetRegisterClass*> RCPair;
Benjamin Kramer9bfb6272015-07-05 19:29:18 +000012116RCPair ARMTargetLowering::getRegForInlineAsmConstraint(
12117 const TargetRegisterInfo *TRI, StringRef Constraint, MVT VT) const {
Evan Cheng10043e22007-01-19 07:51:42 +000012118 if (Constraint.size() == 1) {
Jakob Stoklund Olesen0ca14e42010-01-14 18:19:56 +000012119 // GCC ARM Constraint Letters
Evan Cheng10043e22007-01-19 07:51:42 +000012120 switch (Constraint[0]) {
Eric Christopherf45daac2011-06-30 23:23:01 +000012121 case 'l': // Low regs or general regs.
Jakob Stoklund Olesen0ca14e42010-01-14 18:19:56 +000012122 if (Subtarget->isThumb())
Craig Topperc7242e02012-04-20 07:30:17 +000012123 return RCPair(0U, &ARM::tGPRRegClass);
12124 return RCPair(0U, &ARM::GPRRegClass);
Eric Christopherf45daac2011-06-30 23:23:01 +000012125 case 'h': // High regs or no regs.
12126 if (Subtarget->isThumb())
Craig Topperc7242e02012-04-20 07:30:17 +000012127 return RCPair(0U, &ARM::hGPRRegClass);
Eric Christopherf09b0f12011-07-01 00:19:27 +000012128 break;
Chris Lattner6223e832007-04-02 17:24:08 +000012129 case 'r':
Akira Hatanakab9615342014-11-03 20:37:04 +000012130 if (Subtarget->isThumb1Only())
12131 return RCPair(0U, &ARM::tGPRRegClass);
Craig Topperc7242e02012-04-20 07:30:17 +000012132 return RCPair(0U, &ARM::GPRRegClass);
Chris Lattner6223e832007-04-02 17:24:08 +000012133 case 'w':
Tim Northover28adfbb2013-11-14 17:15:39 +000012134 if (VT == MVT::Other)
12135 break;
Owen Anderson9f944592009-08-11 20:47:22 +000012136 if (VT == MVT::f32)
Craig Topperc7242e02012-04-20 07:30:17 +000012137 return RCPair(0U, &ARM::SPRRegClass);
Bob Wilson3152b0472009-12-18 01:03:29 +000012138 if (VT.getSizeInBits() == 64)
Craig Topperc7242e02012-04-20 07:30:17 +000012139 return RCPair(0U, &ARM::DPRRegClass);
Evan Cheng0c2544f2009-12-08 23:06:22 +000012140 if (VT.getSizeInBits() == 128)
Craig Topperc7242e02012-04-20 07:30:17 +000012141 return RCPair(0U, &ARM::QPRRegClass);
Chris Lattner6223e832007-04-02 17:24:08 +000012142 break;
Eric Christopherf1c74592011-07-01 00:14:47 +000012143 case 'x':
Tim Northover28adfbb2013-11-14 17:15:39 +000012144 if (VT == MVT::Other)
12145 break;
Eric Christopherf1c74592011-07-01 00:14:47 +000012146 if (VT == MVT::f32)
Craig Topperc7242e02012-04-20 07:30:17 +000012147 return RCPair(0U, &ARM::SPR_8RegClass);
Eric Christopherf1c74592011-07-01 00:14:47 +000012148 if (VT.getSizeInBits() == 64)
Craig Topperc7242e02012-04-20 07:30:17 +000012149 return RCPair(0U, &ARM::DPR_8RegClass);
Eric Christopherf1c74592011-07-01 00:14:47 +000012150 if (VT.getSizeInBits() == 128)
Craig Topperc7242e02012-04-20 07:30:17 +000012151 return RCPair(0U, &ARM::QPR_8RegClass);
Eric Christopherf1c74592011-07-01 00:14:47 +000012152 break;
Eric Christopherc011d312011-07-01 00:30:46 +000012153 case 't':
12154 if (VT == MVT::f32)
Craig Topperc7242e02012-04-20 07:30:17 +000012155 return RCPair(0U, &ARM::SPRRegClass);
Eric Christopherc011d312011-07-01 00:30:46 +000012156 break;
Evan Cheng10043e22007-01-19 07:51:42 +000012157 }
12158 }
Bob Wilson3f2293b2010-03-15 23:09:18 +000012159 if (StringRef("{cc}").equals_lower(Constraint))
Craig Topperc7242e02012-04-20 07:30:17 +000012160 return std::make_pair(unsigned(ARM::CPSR), &ARM::CCRRegClass);
Bob Wilson3f2293b2010-03-15 23:09:18 +000012161
Eric Christopher11e4df72015-02-26 22:38:43 +000012162 return TargetLowering::getRegForInlineAsmConstraint(TRI, Constraint, VT);
Evan Cheng10043e22007-01-19 07:51:42 +000012163}
12164
Bob Wilsoncf1ec2c2009-04-01 17:58:54 +000012165/// LowerAsmOperandForConstraint - Lower the specified operand into the Ops
12166/// vector. If it is invalid, don't add anything to Ops.
12167void ARMTargetLowering::LowerAsmOperandForConstraint(SDValue Op,
Eric Christopherde9399b2011-06-02 23:16:42 +000012168 std::string &Constraint,
Bob Wilsoncf1ec2c2009-04-01 17:58:54 +000012169 std::vector<SDValue>&Ops,
12170 SelectionDAG &DAG) const {
Craig Topper062a2ba2014-04-25 05:30:21 +000012171 SDValue Result;
Bob Wilsoncf1ec2c2009-04-01 17:58:54 +000012172
Eric Christopherde9399b2011-06-02 23:16:42 +000012173 // Currently only support length 1 constraints.
12174 if (Constraint.length() != 1) return;
Eric Christopher0713a9d2011-06-08 23:55:35 +000012175
Eric Christopherde9399b2011-06-02 23:16:42 +000012176 char ConstraintLetter = Constraint[0];
12177 switch (ConstraintLetter) {
Bob Wilsoncf1ec2c2009-04-01 17:58:54 +000012178 default: break;
Eric Christopher29f1db82011-07-01 01:00:07 +000012179 case 'j':
Bob Wilsoncf1ec2c2009-04-01 17:58:54 +000012180 case 'I': case 'J': case 'K': case 'L':
12181 case 'M': case 'N': case 'O':
12182 ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op);
12183 if (!C)
12184 return;
12185
12186 int64_t CVal64 = C->getSExtValue();
12187 int CVal = (int) CVal64;
12188 // None of these constraints allow values larger than 32 bits. Check
12189 // that the value fits in an int.
12190 if (CVal != CVal64)
12191 return;
12192
Eric Christopherde9399b2011-06-02 23:16:42 +000012193 switch (ConstraintLetter) {
Eric Christopher29f1db82011-07-01 01:00:07 +000012194 case 'j':
Andrew Trick53df4b62011-09-20 03:06:13 +000012195 // Constant suitable for movw, must be between 0 and
12196 // 65535.
12197 if (Subtarget->hasV6T2Ops())
12198 if (CVal >= 0 && CVal <= 65535)
12199 break;
12200 return;
Bob Wilsoncf1ec2c2009-04-01 17:58:54 +000012201 case 'I':
David Goodwin22c2fba2009-07-08 23:10:31 +000012202 if (Subtarget->isThumb1Only()) {
12203 // This must be a constant between 0 and 255, for ADD
12204 // immediates.
Bob Wilsoncf1ec2c2009-04-01 17:58:54 +000012205 if (CVal >= 0 && CVal <= 255)
12206 break;
David Goodwin22c2fba2009-07-08 23:10:31 +000012207 } else if (Subtarget->isThumb2()) {
12208 // A constant that can be used as an immediate value in a
12209 // data-processing instruction.
12210 if (ARM_AM::getT2SOImmVal(CVal) != -1)
12211 break;
Bob Wilsoncf1ec2c2009-04-01 17:58:54 +000012212 } else {
12213 // A constant that can be used as an immediate value in a
12214 // data-processing instruction.
12215 if (ARM_AM::getSOImmVal(CVal) != -1)
12216 break;
12217 }
12218 return;
12219
12220 case 'J':
Eric Christopherb7932302016-01-08 00:34:44 +000012221 if (Subtarget->isThumb1Only()) {
Bob Wilsoncf1ec2c2009-04-01 17:58:54 +000012222 // This must be a constant between -255 and -1, for negated ADD
12223 // immediates. This can be used in GCC with an "n" modifier that
12224 // prints the negated value, for use with SUB instructions. It is
12225 // not useful otherwise but is implemented for compatibility.
12226 if (CVal >= -255 && CVal <= -1)
12227 break;
12228 } else {
12229 // This must be a constant between -4095 and 4095. It is not clear
12230 // what this constraint is intended for. Implemented for
12231 // compatibility with GCC.
12232 if (CVal >= -4095 && CVal <= 4095)
12233 break;
12234 }
12235 return;
12236
12237 case 'K':
David Goodwin22c2fba2009-07-08 23:10:31 +000012238 if (Subtarget->isThumb1Only()) {
Bob Wilsoncf1ec2c2009-04-01 17:58:54 +000012239 // A 32-bit value where only one byte has a nonzero value. Exclude
12240 // zero to match GCC. This constraint is used by GCC internally for
12241 // constants that can be loaded with a move/shift combination.
12242 // It is not useful otherwise but is implemented for compatibility.
12243 if (CVal != 0 && ARM_AM::isThumbImmShiftedVal(CVal))
12244 break;
David Goodwin22c2fba2009-07-08 23:10:31 +000012245 } else if (Subtarget->isThumb2()) {
12246 // A constant whose bitwise inverse can be used as an immediate
12247 // value in a data-processing instruction. This can be used in GCC
12248 // with a "B" modifier that prints the inverted value, for use with
12249 // BIC and MVN instructions. It is not useful otherwise but is
12250 // implemented for compatibility.
12251 if (ARM_AM::getT2SOImmVal(~CVal) != -1)
12252 break;
Bob Wilsoncf1ec2c2009-04-01 17:58:54 +000012253 } else {
12254 // A constant whose bitwise inverse can be used as an immediate
12255 // value in a data-processing instruction. This can be used in GCC
12256 // with a "B" modifier that prints the inverted value, for use with
12257 // BIC and MVN instructions. It is not useful otherwise but is
12258 // implemented for compatibility.
12259 if (ARM_AM::getSOImmVal(~CVal) != -1)
12260 break;
12261 }
12262 return;
12263
12264 case 'L':
David Goodwin22c2fba2009-07-08 23:10:31 +000012265 if (Subtarget->isThumb1Only()) {
Bob Wilsoncf1ec2c2009-04-01 17:58:54 +000012266 // This must be a constant between -7 and 7,
12267 // for 3-operand ADD/SUB immediate instructions.
12268 if (CVal >= -7 && CVal < 7)
12269 break;
David Goodwin22c2fba2009-07-08 23:10:31 +000012270 } else if (Subtarget->isThumb2()) {
12271 // A constant whose negation can be used as an immediate value in a
12272 // data-processing instruction. This can be used in GCC with an "n"
12273 // modifier that prints the negated value, for use with SUB
12274 // instructions. It is not useful otherwise but is implemented for
12275 // compatibility.
12276 if (ARM_AM::getT2SOImmVal(-CVal) != -1)
12277 break;
Bob Wilsoncf1ec2c2009-04-01 17:58:54 +000012278 } else {
12279 // A constant whose negation can be used as an immediate value in a
12280 // data-processing instruction. This can be used in GCC with an "n"
12281 // modifier that prints the negated value, for use with SUB
12282 // instructions. It is not useful otherwise but is implemented for
12283 // compatibility.
12284 if (ARM_AM::getSOImmVal(-CVal) != -1)
12285 break;
12286 }
12287 return;
12288
12289 case 'M':
Eric Christopherb7932302016-01-08 00:34:44 +000012290 if (Subtarget->isThumb1Only()) {
Bob Wilsoncf1ec2c2009-04-01 17:58:54 +000012291 // This must be a multiple of 4 between 0 and 1020, for
12292 // ADD sp + immediate.
12293 if ((CVal >= 0 && CVal <= 1020) && ((CVal & 3) == 0))
12294 break;
12295 } else {
12296 // A power of two or a constant between 0 and 32. This is used in
12297 // GCC for the shift amount on shifted register operands, but it is
12298 // useful in general for any shift amounts.
12299 if ((CVal >= 0 && CVal <= 32) || ((CVal & (CVal - 1)) == 0))
12300 break;
12301 }
12302 return;
12303
12304 case 'N':
David Goodwin22c2fba2009-07-08 23:10:31 +000012305 if (Subtarget->isThumb()) { // FIXME thumb2
Bob Wilsoncf1ec2c2009-04-01 17:58:54 +000012306 // This must be a constant between 0 and 31, for shift amounts.
12307 if (CVal >= 0 && CVal <= 31)
12308 break;
12309 }
12310 return;
12311
12312 case 'O':
David Goodwin22c2fba2009-07-08 23:10:31 +000012313 if (Subtarget->isThumb()) { // FIXME thumb2
Bob Wilsoncf1ec2c2009-04-01 17:58:54 +000012314 // This must be a multiple of 4 between -508 and 508, for
12315 // ADD/SUB sp = sp + immediate.
12316 if ((CVal >= -508 && CVal <= 508) && ((CVal & 3) == 0))
12317 break;
12318 }
12319 return;
12320 }
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +000012321 Result = DAG.getTargetConstant(CVal, SDLoc(Op), Op.getValueType());
Bob Wilsoncf1ec2c2009-04-01 17:58:54 +000012322 break;
12323 }
12324
12325 if (Result.getNode()) {
12326 Ops.push_back(Result);
12327 return;
12328 }
Dale Johannesence97d552010-06-25 21:55:36 +000012329 return TargetLowering::LowerAsmOperandForConstraint(Op, Constraint, Ops, DAG);
Bob Wilsoncf1ec2c2009-04-01 17:58:54 +000012330}
Anton Korobeynikov29a44df2009-09-23 19:04:09 +000012331
Scott Douglassd2974a62015-08-24 09:17:11 +000012332static RTLIB::Libcall getDivRemLibcall(
12333 const SDNode *N, MVT::SimpleValueType SVT) {
Scott Douglassbdef6042015-08-24 09:17:18 +000012334 assert((N->getOpcode() == ISD::SDIVREM || N->getOpcode() == ISD::UDIVREM ||
12335 N->getOpcode() == ISD::SREM || N->getOpcode() == ISD::UREM) &&
Scott Douglassd2974a62015-08-24 09:17:11 +000012336 "Unhandled Opcode in getDivRemLibcall");
Scott Douglassbdef6042015-08-24 09:17:18 +000012337 bool isSigned = N->getOpcode() == ISD::SDIVREM ||
12338 N->getOpcode() == ISD::SREM;
Scott Douglassd2974a62015-08-24 09:17:11 +000012339 RTLIB::Libcall LC;
12340 switch (SVT) {
12341 default: llvm_unreachable("Unexpected request for libcall!");
12342 case MVT::i8: LC = isSigned ? RTLIB::SDIVREM_I8 : RTLIB::UDIVREM_I8; break;
12343 case MVT::i16: LC = isSigned ? RTLIB::SDIVREM_I16 : RTLIB::UDIVREM_I16; break;
12344 case MVT::i32: LC = isSigned ? RTLIB::SDIVREM_I32 : RTLIB::UDIVREM_I32; break;
12345 case MVT::i64: LC = isSigned ? RTLIB::SDIVREM_I64 : RTLIB::UDIVREM_I64; break;
12346 }
12347 return LC;
12348}
12349
12350static TargetLowering::ArgListTy getDivRemArgList(
12351 const SDNode *N, LLVMContext *Context) {
Scott Douglassbdef6042015-08-24 09:17:18 +000012352 assert((N->getOpcode() == ISD::SDIVREM || N->getOpcode() == ISD::UDIVREM ||
12353 N->getOpcode() == ISD::SREM || N->getOpcode() == ISD::UREM) &&
Scott Douglassd2974a62015-08-24 09:17:11 +000012354 "Unhandled Opcode in getDivRemArgList");
Scott Douglassbdef6042015-08-24 09:17:18 +000012355 bool isSigned = N->getOpcode() == ISD::SDIVREM ||
12356 N->getOpcode() == ISD::SREM;
Scott Douglassd2974a62015-08-24 09:17:11 +000012357 TargetLowering::ArgListTy Args;
12358 TargetLowering::ArgListEntry Entry;
12359 for (unsigned i = 0, e = N->getNumOperands(); i != e; ++i) {
12360 EVT ArgVT = N->getOperand(i).getValueType();
12361 Type *ArgTy = ArgVT.getTypeForEVT(*Context);
12362 Entry.Node = N->getOperand(i);
12363 Entry.Ty = ArgTy;
12364 Entry.isSExt = isSigned;
12365 Entry.isZExt = !isSigned;
12366 Args.push_back(Entry);
12367 }
12368 return Args;
12369}
12370
Renato Golin87610692013-07-16 09:32:17 +000012371SDValue ARMTargetLowering::LowerDivRem(SDValue Op, SelectionDAG &DAG) const {
Renato Golin6027dd38e2016-02-03 16:10:54 +000012372 assert((Subtarget->isTargetAEABI() || Subtarget->isTargetAndroid() ||
Rafael Espindolaa895a0c2016-06-24 21:14:33 +000012373 Subtarget->isTargetGNUAEABI() || Subtarget->isTargetMuslAEABI()) &&
Sumanth Gundapaneni532a1362015-07-31 00:45:12 +000012374 "Register-based DivRem lowering only");
Renato Golin87610692013-07-16 09:32:17 +000012375 unsigned Opcode = Op->getOpcode();
12376 assert((Opcode == ISD::SDIVREM || Opcode == ISD::UDIVREM) &&
Saleem Abdulrasool740be892014-08-17 22:50:59 +000012377 "Invalid opcode for Div/Rem lowering");
Renato Golin87610692013-07-16 09:32:17 +000012378 bool isSigned = (Opcode == ISD::SDIVREM);
12379 EVT VT = Op->getValueType(0);
12380 Type *Ty = VT.getTypeForEVT(*DAG.getContext());
12381
Scott Douglassd2974a62015-08-24 09:17:11 +000012382 RTLIB::Libcall LC = getDivRemLibcall(Op.getNode(),
12383 VT.getSimpleVT().SimpleTy);
Renato Golin87610692013-07-16 09:32:17 +000012384 SDValue InChain = DAG.getEntryNode();
12385
Scott Douglassd2974a62015-08-24 09:17:11 +000012386 TargetLowering::ArgListTy Args = getDivRemArgList(Op.getNode(),
12387 DAG.getContext());
Renato Golin87610692013-07-16 09:32:17 +000012388
12389 SDValue Callee = DAG.getExternalSymbol(getLibcallName(LC),
Mehdi Amini44ede332015-07-09 02:09:04 +000012390 getPointerTy(DAG.getDataLayout()));
Renato Golin87610692013-07-16 09:32:17 +000012391
Reid Kleckner343c3952014-11-20 23:51:47 +000012392 Type *RetTy = (Type*)StructType::get(Ty, Ty, nullptr);
Renato Golin87610692013-07-16 09:32:17 +000012393
Pablo Barrio2b7ed132016-09-08 10:05:57 +000012394 SDLoc dl(Op);
Saleem Abdulrasoolf3a5a5c2014-05-17 21:50:17 +000012395 TargetLowering::CallLoweringInfo CLI(DAG);
12396 CLI.setDebugLoc(dl).setChain(InChain)
Krzysztof Parzyszeke116d5002016-06-22 12:54:25 +000012397 .setCallee(getLibcallCallingConv(LC), RetTy, Callee, std::move(Args))
Saleem Abdulrasoolf3a5a5c2014-05-17 21:50:17 +000012398 .setInRegister().setSExtResult(isSigned).setZExtResult(!isSigned);
Renato Golin87610692013-07-16 09:32:17 +000012399
Saleem Abdulrasoolf3a5a5c2014-05-17 21:50:17 +000012400 std::pair<SDValue, SDValue> CallInfo = LowerCallTo(CLI);
Renato Golin87610692013-07-16 09:32:17 +000012401 return CallInfo.first;
12402}
12403
Scott Douglassbdef6042015-08-24 09:17:18 +000012404// Lowers REM using divmod helpers
12405// see RTABI section 4.2/4.3
12406SDValue ARMTargetLowering::LowerREM(SDNode *N, SelectionDAG &DAG) const {
12407 // Build return types (div and rem)
12408 std::vector<Type*> RetTyParams;
12409 Type *RetTyElement;
12410
12411 switch (N->getValueType(0).getSimpleVT().SimpleTy) {
12412 default: llvm_unreachable("Unexpected request for libcall!");
12413 case MVT::i8: RetTyElement = Type::getInt8Ty(*DAG.getContext()); break;
12414 case MVT::i16: RetTyElement = Type::getInt16Ty(*DAG.getContext()); break;
12415 case MVT::i32: RetTyElement = Type::getInt32Ty(*DAG.getContext()); break;
12416 case MVT::i64: RetTyElement = Type::getInt64Ty(*DAG.getContext()); break;
12417 }
12418
12419 RetTyParams.push_back(RetTyElement);
12420 RetTyParams.push_back(RetTyElement);
12421 ArrayRef<Type*> ret = ArrayRef<Type*>(RetTyParams);
12422 Type *RetTy = StructType::get(*DAG.getContext(), ret);
12423
12424 RTLIB::Libcall LC = getDivRemLibcall(N, N->getValueType(0).getSimpleVT().
12425 SimpleTy);
12426 SDValue InChain = DAG.getEntryNode();
12427 TargetLowering::ArgListTy Args = getDivRemArgList(N, DAG.getContext());
12428 bool isSigned = N->getOpcode() == ISD::SREM;
12429 SDValue Callee = DAG.getExternalSymbol(getLibcallName(LC),
12430 getPointerTy(DAG.getDataLayout()));
12431
12432 // Lower call
12433 CallLoweringInfo CLI(DAG);
12434 CLI.setChain(InChain)
Krzysztof Parzyszeke116d5002016-06-22 12:54:25 +000012435 .setCallee(CallingConv::ARM_AAPCS, RetTy, Callee, std::move(Args))
Scott Douglassbdef6042015-08-24 09:17:18 +000012436 .setSExtResult(isSigned).setZExtResult(!isSigned).setDebugLoc(SDLoc(N));
12437 std::pair<SDValue, SDValue> CallResult = LowerCallTo(CLI);
12438
12439 // Return second (rem) result operand (first contains div)
12440 SDNode *ResNode = CallResult.first.getNode();
12441 assert(ResNode->getNumOperands() == 2 && "divmod should return two operands");
12442 return ResNode->getOperand(1);
12443}
12444
Saleem Abdulrasoolabac6e92014-06-09 20:18:42 +000012445SDValue
12446ARMTargetLowering::LowerDYNAMIC_STACKALLOC(SDValue Op, SelectionDAG &DAG) const {
12447 assert(Subtarget->isTargetWindows() && "unsupported target platform");
12448 SDLoc DL(Op);
12449
12450 // Get the inputs.
12451 SDValue Chain = Op.getOperand(0);
12452 SDValue Size = Op.getOperand(1);
12453
12454 SDValue Words = DAG.getNode(ISD::SRL, DL, MVT::i32, Size,
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +000012455 DAG.getConstant(2, DL, MVT::i32));
Saleem Abdulrasoolabac6e92014-06-09 20:18:42 +000012456
12457 SDValue Flag;
12458 Chain = DAG.getCopyToReg(Chain, DL, ARM::R4, Words, Flag);
12459 Flag = Chain.getValue(1);
12460
Saleem Abdulrasoolc4e00282014-07-19 01:29:51 +000012461 SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);
Saleem Abdulrasoolabac6e92014-06-09 20:18:42 +000012462 Chain = DAG.getNode(ARMISD::WIN__CHKSTK, DL, NodeTys, Chain, Flag);
12463
12464 SDValue NewSP = DAG.getCopyFromReg(Chain, DL, ARM::SP, MVT::i32);
12465 Chain = NewSP.getValue(1);
12466
12467 SDValue Ops[2] = { NewSP, Chain };
12468 return DAG.getMergeValues(Ops, DL);
12469}
12470
Oliver Stannard51b1d462014-08-21 12:50:31 +000012471SDValue ARMTargetLowering::LowerFP_EXTEND(SDValue Op, SelectionDAG &DAG) const {
12472 assert(Op.getValueType() == MVT::f64 && Subtarget->isFPOnlySP() &&
12473 "Unexpected type for custom-lowering FP_EXTEND");
12474
12475 RTLIB::Libcall LC;
12476 LC = RTLIB::getFPEXT(Op.getOperand(0).getValueType(), Op.getValueType());
12477
12478 SDValue SrcVal = Op.getOperand(0);
Craig Topper8fe40e02015-10-22 17:05:00 +000012479 return makeLibCall(DAG, LC, Op.getValueType(), SrcVal, /*isSigned*/ false,
12480 SDLoc(Op)).first;
Oliver Stannard51b1d462014-08-21 12:50:31 +000012481}
12482
12483SDValue ARMTargetLowering::LowerFP_ROUND(SDValue Op, SelectionDAG &DAG) const {
12484 assert(Op.getOperand(0).getValueType() == MVT::f64 &&
12485 Subtarget->isFPOnlySP() &&
12486 "Unexpected type for custom-lowering FP_ROUND");
12487
12488 RTLIB::Libcall LC;
12489 LC = RTLIB::getFPROUND(Op.getOperand(0).getValueType(), Op.getValueType());
12490
12491 SDValue SrcVal = Op.getOperand(0);
Craig Topper8fe40e02015-10-22 17:05:00 +000012492 return makeLibCall(DAG, LC, Op.getValueType(), SrcVal, /*isSigned*/ false,
12493 SDLoc(Op)).first;
Oliver Stannard51b1d462014-08-21 12:50:31 +000012494}
12495
Anton Korobeynikov29a44df2009-09-23 19:04:09 +000012496bool
12497ARMTargetLowering::isOffsetFoldingLegal(const GlobalAddressSDNode *GA) const {
12498 // The ARM target isn't yet aware of offsets.
12499 return false;
12500}
Evan Cheng4a609f3c2009-10-28 01:44:26 +000012501
Jim Grosbach11013ed2010-07-16 23:05:05 +000012502bool ARM::isBitFieldInvertedMask(unsigned v) {
12503 if (v == 0xffffffff)
Benjamin Kramer8bad66e2013-05-19 22:01:57 +000012504 return false;
12505
Jim Grosbach11013ed2010-07-16 23:05:05 +000012506 // there can be 1's on either or both "outsides", all the "inside"
12507 // bits must be 0's
Benjamin Kramer5f6a9072015-02-12 15:35:40 +000012508 return isShiftedMask_32(~v);
Jim Grosbach11013ed2010-07-16 23:05:05 +000012509}
12510
Evan Cheng4a609f3c2009-10-28 01:44:26 +000012511/// isFPImmLegal - Returns true if the target can instruction select the
12512/// specified FP immediate natively. If false, the legalizer will
12513/// materialize the FP immediate as a load from a constant pool.
12514bool ARMTargetLowering::isFPImmLegal(const APFloat &Imm, EVT VT) const {
12515 if (!Subtarget->hasVFP3())
12516 return false;
12517 if (VT == MVT::f32)
Jim Grosbachefc761a2011-09-30 00:50:06 +000012518 return ARM_AM::getFP32Imm(Imm) != -1;
Oliver Stannard51b1d462014-08-21 12:50:31 +000012519 if (VT == MVT::f64 && !Subtarget->isFPOnlySP())
Jim Grosbachefc761a2011-09-30 00:50:06 +000012520 return ARM_AM::getFP64Imm(Imm) != -1;
Evan Cheng4a609f3c2009-10-28 01:44:26 +000012521 return false;
12522}
Bob Wilson5549d492010-09-21 17:56:22 +000012523
Wesley Peck527da1b2010-11-23 03:31:01 +000012524/// getTgtMemIntrinsic - Represent NEON load and store intrinsics as
Bob Wilson5549d492010-09-21 17:56:22 +000012525/// MemIntrinsicNodes. The associated MachineMemOperands record the alignment
12526/// specified in the intrinsic calls.
12527bool ARMTargetLowering::getTgtMemIntrinsic(IntrinsicInfo &Info,
12528 const CallInst &I,
12529 unsigned Intrinsic) const {
12530 switch (Intrinsic) {
12531 case Intrinsic::arm_neon_vld1:
12532 case Intrinsic::arm_neon_vld2:
12533 case Intrinsic::arm_neon_vld3:
12534 case Intrinsic::arm_neon_vld4:
12535 case Intrinsic::arm_neon_vld2lane:
12536 case Intrinsic::arm_neon_vld3lane:
12537 case Intrinsic::arm_neon_vld4lane: {
12538 Info.opc = ISD::INTRINSIC_W_CHAIN;
12539 // Conservatively set memVT to the entire set of vectors loaded.
Mehdi Aminia749f2a2015-07-09 02:09:52 +000012540 auto &DL = I.getCalledFunction()->getParent()->getDataLayout();
Ahmed Bougacha97564c32015-12-09 01:19:50 +000012541 uint64_t NumElts = DL.getTypeSizeInBits(I.getType()) / 64;
Bob Wilson5549d492010-09-21 17:56:22 +000012542 Info.memVT = EVT::getVectorVT(I.getType()->getContext(), MVT::i64, NumElts);
12543 Info.ptrVal = I.getArgOperand(0);
12544 Info.offset = 0;
12545 Value *AlignArg = I.getArgOperand(I.getNumArgOperands() - 1);
12546 Info.align = cast<ConstantInt>(AlignArg)->getZExtValue();
12547 Info.vol = false; // volatile loads with NEON intrinsics not supported
12548 Info.readMem = true;
12549 Info.writeMem = false;
12550 return true;
12551 }
12552 case Intrinsic::arm_neon_vst1:
12553 case Intrinsic::arm_neon_vst2:
12554 case Intrinsic::arm_neon_vst3:
12555 case Intrinsic::arm_neon_vst4:
12556 case Intrinsic::arm_neon_vst2lane:
12557 case Intrinsic::arm_neon_vst3lane:
12558 case Intrinsic::arm_neon_vst4lane: {
12559 Info.opc = ISD::INTRINSIC_VOID;
12560 // Conservatively set memVT to the entire set of vectors stored.
Mehdi Aminia749f2a2015-07-09 02:09:52 +000012561 auto &DL = I.getCalledFunction()->getParent()->getDataLayout();
Bob Wilson5549d492010-09-21 17:56:22 +000012562 unsigned NumElts = 0;
12563 for (unsigned ArgI = 1, ArgE = I.getNumArgOperands(); ArgI < ArgE; ++ArgI) {
Chris Lattner229907c2011-07-18 04:54:35 +000012564 Type *ArgTy = I.getArgOperand(ArgI)->getType();
Bob Wilson5549d492010-09-21 17:56:22 +000012565 if (!ArgTy->isVectorTy())
12566 break;
Ahmed Bougacha97564c32015-12-09 01:19:50 +000012567 NumElts += DL.getTypeSizeInBits(ArgTy) / 64;
Bob Wilson5549d492010-09-21 17:56:22 +000012568 }
12569 Info.memVT = EVT::getVectorVT(I.getType()->getContext(), MVT::i64, NumElts);
12570 Info.ptrVal = I.getArgOperand(0);
12571 Info.offset = 0;
12572 Value *AlignArg = I.getArgOperand(I.getNumArgOperands() - 1);
12573 Info.align = cast<ConstantInt>(AlignArg)->getZExtValue();
12574 Info.vol = false; // volatile stores with NEON intrinsics not supported
12575 Info.readMem = false;
12576 Info.writeMem = true;
12577 return true;
12578 }
Tim Northover1ff5f292014-03-26 14:39:31 +000012579 case Intrinsic::arm_ldaex:
Tim Northovera7ecd242013-07-16 09:46:55 +000012580 case Intrinsic::arm_ldrex: {
Mehdi Aminia749f2a2015-07-09 02:09:52 +000012581 auto &DL = I.getCalledFunction()->getParent()->getDataLayout();
Tim Northovera7ecd242013-07-16 09:46:55 +000012582 PointerType *PtrTy = cast<PointerType>(I.getArgOperand(0)->getType());
12583 Info.opc = ISD::INTRINSIC_W_CHAIN;
12584 Info.memVT = MVT::getVT(PtrTy->getElementType());
12585 Info.ptrVal = I.getArgOperand(0);
12586 Info.offset = 0;
Mehdi Aminia749f2a2015-07-09 02:09:52 +000012587 Info.align = DL.getABITypeAlignment(PtrTy->getElementType());
Tim Northovera7ecd242013-07-16 09:46:55 +000012588 Info.vol = true;
12589 Info.readMem = true;
12590 Info.writeMem = false;
12591 return true;
12592 }
Tim Northover1ff5f292014-03-26 14:39:31 +000012593 case Intrinsic::arm_stlex:
Tim Northovera7ecd242013-07-16 09:46:55 +000012594 case Intrinsic::arm_strex: {
Mehdi Aminia749f2a2015-07-09 02:09:52 +000012595 auto &DL = I.getCalledFunction()->getParent()->getDataLayout();
Tim Northovera7ecd242013-07-16 09:46:55 +000012596 PointerType *PtrTy = cast<PointerType>(I.getArgOperand(1)->getType());
12597 Info.opc = ISD::INTRINSIC_W_CHAIN;
12598 Info.memVT = MVT::getVT(PtrTy->getElementType());
12599 Info.ptrVal = I.getArgOperand(1);
12600 Info.offset = 0;
Mehdi Aminia749f2a2015-07-09 02:09:52 +000012601 Info.align = DL.getABITypeAlignment(PtrTy->getElementType());
Tim Northovera7ecd242013-07-16 09:46:55 +000012602 Info.vol = true;
12603 Info.readMem = false;
12604 Info.writeMem = true;
12605 return true;
12606 }
Tim Northover1ff5f292014-03-26 14:39:31 +000012607 case Intrinsic::arm_stlexd:
Bruno Cardoso Lopes325110f2011-05-28 04:07:29 +000012608 case Intrinsic::arm_strexd: {
12609 Info.opc = ISD::INTRINSIC_W_CHAIN;
12610 Info.memVT = MVT::i64;
12611 Info.ptrVal = I.getArgOperand(2);
12612 Info.offset = 0;
12613 Info.align = 8;
Bruno Cardoso Lopesd66ab9e2011-06-16 18:11:32 +000012614 Info.vol = true;
Bruno Cardoso Lopes325110f2011-05-28 04:07:29 +000012615 Info.readMem = false;
12616 Info.writeMem = true;
12617 return true;
12618 }
Tim Northover1ff5f292014-03-26 14:39:31 +000012619 case Intrinsic::arm_ldaexd:
Bruno Cardoso Lopes325110f2011-05-28 04:07:29 +000012620 case Intrinsic::arm_ldrexd: {
12621 Info.opc = ISD::INTRINSIC_W_CHAIN;
12622 Info.memVT = MVT::i64;
12623 Info.ptrVal = I.getArgOperand(0);
12624 Info.offset = 0;
12625 Info.align = 8;
Bruno Cardoso Lopesd66ab9e2011-06-16 18:11:32 +000012626 Info.vol = true;
Bruno Cardoso Lopes325110f2011-05-28 04:07:29 +000012627 Info.readMem = true;
12628 Info.writeMem = false;
12629 return true;
12630 }
Bob Wilson5549d492010-09-21 17:56:22 +000012631 default:
12632 break;
12633 }
12634
12635 return false;
12636}
Juergen Ributzka659ce002014-01-28 01:20:14 +000012637
12638/// \brief Returns true if it is beneficial to convert a load of a constant
12639/// to just the constant itself.
12640bool ARMTargetLowering::shouldConvertConstantLoadToIntImm(const APInt &Imm,
12641 Type *Ty) const {
12642 assert(Ty->isIntegerTy());
12643
12644 unsigned Bits = Ty->getPrimitiveSizeInBits();
12645 if (Bits == 0 || Bits > 32)
12646 return false;
12647 return true;
12648}
Tim Northover037f26f22014-04-17 18:22:47 +000012649
Robin Morisset5349e8e2014-09-18 18:56:04 +000012650Instruction* ARMTargetLowering::makeDMB(IRBuilder<> &Builder,
12651 ARM_MB::MemBOpt Domain) const {
Robin Morisseta47cb412014-09-03 21:01:03 +000012652 Module *M = Builder.GetInsertBlock()->getParent()->getParent();
Robin Morisset5349e8e2014-09-18 18:56:04 +000012653
12654 // First, if the target has no DMB, see what fallback we can use.
12655 if (!Subtarget->hasDataBarrier()) {
12656 // Some ARMv6 cpus can support data barriers with an mcr instruction.
12657 // Thumb1 and pre-v6 ARM mode use a libcall instead and should never get
12658 // here.
12659 if (Subtarget->hasV6Ops() && !Subtarget->isThumb()) {
12660 Function *MCR = llvm::Intrinsic::getDeclaration(M, Intrinsic::arm_mcr);
12661 Value* args[6] = {Builder.getInt32(15), Builder.getInt32(0),
12662 Builder.getInt32(0), Builder.getInt32(7),
12663 Builder.getInt32(10), Builder.getInt32(5)};
12664 return Builder.CreateCall(MCR, args);
12665 } else {
12666 // Instead of using barriers, atomic accesses on these subtargets use
12667 // libcalls.
12668 llvm_unreachable("makeDMB on a target so old that it has no barriers");
12669 }
12670 } else {
12671 Function *DMB = llvm::Intrinsic::getDeclaration(M, Intrinsic::arm_dmb);
12672 // Only a full system barrier exists in the M-class architectures.
12673 Domain = Subtarget->isMClass() ? ARM_MB::SY : Domain;
12674 Constant *CDomain = Builder.getInt32(Domain);
12675 return Builder.CreateCall(DMB, CDomain);
12676 }
Robin Morisseta47cb412014-09-03 21:01:03 +000012677}
12678
12679// Based on http://www.cl.cam.ac.uk/~pes20/cpp/cpp0xmappings.html
Robin Morissetdedef332014-09-23 20:31:14 +000012680Instruction* ARMTargetLowering::emitLeadingFence(IRBuilder<> &Builder,
Robin Morisseta47cb412014-09-03 21:01:03 +000012681 AtomicOrdering Ord, bool IsStore,
12682 bool IsLoad) const {
Robin Morisseta47cb412014-09-03 21:01:03 +000012683 switch (Ord) {
JF Bastien800f87a2016-04-06 21:19:33 +000012684 case AtomicOrdering::NotAtomic:
12685 case AtomicOrdering::Unordered:
Robin Morisseta47cb412014-09-03 21:01:03 +000012686 llvm_unreachable("Invalid fence: unordered/non-atomic");
JF Bastien800f87a2016-04-06 21:19:33 +000012687 case AtomicOrdering::Monotonic:
12688 case AtomicOrdering::Acquire:
Robin Morissetdedef332014-09-23 20:31:14 +000012689 return nullptr; // Nothing to do
JF Bastien800f87a2016-04-06 21:19:33 +000012690 case AtomicOrdering::SequentiallyConsistent:
Robin Morisseta47cb412014-09-03 21:01:03 +000012691 if (!IsStore)
Robin Morissetdedef332014-09-23 20:31:14 +000012692 return nullptr; // Nothing to do
12693 /*FALLTHROUGH*/
JF Bastien800f87a2016-04-06 21:19:33 +000012694 case AtomicOrdering::Release:
12695 case AtomicOrdering::AcquireRelease:
Diana Picusc5baa432016-06-23 07:47:35 +000012696 if (Subtarget->preferISHSTBarriers())
Robin Morissetdedef332014-09-23 20:31:14 +000012697 return makeDMB(Builder, ARM_MB::ISHST);
Robin Morisseta47cb412014-09-03 21:01:03 +000012698 // FIXME: add a comment with a link to documentation justifying this.
12699 else
Robin Morissetdedef332014-09-23 20:31:14 +000012700 return makeDMB(Builder, ARM_MB::ISH);
Robin Morisseta47cb412014-09-03 21:01:03 +000012701 }
Robin Morissetdedef332014-09-23 20:31:14 +000012702 llvm_unreachable("Unknown fence ordering in emitLeadingFence");
Robin Morisseta47cb412014-09-03 21:01:03 +000012703}
12704
Robin Morissetdedef332014-09-23 20:31:14 +000012705Instruction* ARMTargetLowering::emitTrailingFence(IRBuilder<> &Builder,
Robin Morisseta47cb412014-09-03 21:01:03 +000012706 AtomicOrdering Ord, bool IsStore,
12707 bool IsLoad) const {
Robin Morisseta47cb412014-09-03 21:01:03 +000012708 switch (Ord) {
JF Bastien800f87a2016-04-06 21:19:33 +000012709 case AtomicOrdering::NotAtomic:
12710 case AtomicOrdering::Unordered:
Robin Morisseta47cb412014-09-03 21:01:03 +000012711 llvm_unreachable("Invalid fence: unordered/not-atomic");
JF Bastien800f87a2016-04-06 21:19:33 +000012712 case AtomicOrdering::Monotonic:
12713 case AtomicOrdering::Release:
Robin Morissetdedef332014-09-23 20:31:14 +000012714 return nullptr; // Nothing to do
JF Bastien800f87a2016-04-06 21:19:33 +000012715 case AtomicOrdering::Acquire:
12716 case AtomicOrdering::AcquireRelease:
12717 case AtomicOrdering::SequentiallyConsistent:
Robin Morissetdedef332014-09-23 20:31:14 +000012718 return makeDMB(Builder, ARM_MB::ISH);
Robin Morisseta47cb412014-09-03 21:01:03 +000012719 }
Robin Morissetdedef332014-09-23 20:31:14 +000012720 llvm_unreachable("Unknown fence ordering in emitTrailingFence");
Robin Morisseta47cb412014-09-03 21:01:03 +000012721}
12722
Robin Morisseted3d48f2014-09-03 21:29:59 +000012723// Loads and stores less than 64-bits are already atomic; ones above that
12724// are doomed anyway, so defer to the default libcall and blame the OS when
12725// things go wrong. Cortex M doesn't have ldrexd/strexd though, so don't emit
12726// anything for those.
12727bool ARMTargetLowering::shouldExpandAtomicStoreInIR(StoreInst *SI) const {
12728 unsigned Size = SI->getValueOperand()->getType()->getPrimitiveSizeInBits();
12729 return (Size == 64) && !Subtarget->isMClass();
12730}
Tim Northover037f26f22014-04-17 18:22:47 +000012731
Robin Morisseted3d48f2014-09-03 21:29:59 +000012732// Loads and stores less than 64-bits are already atomic; ones above that
12733// are doomed anyway, so defer to the default libcall and blame the OS when
12734// things go wrong. Cortex M doesn't have ldrexd/strexd though, so don't emit
12735// anything for those.
Robin Morisseta7b357f2014-09-23 18:33:21 +000012736// FIXME: ldrd and strd are atomic if the CPU has LPAE (e.g. A15 has that
12737// guarantee, see DDI0406C ARM architecture reference manual,
12738// sections A8.8.72-74 LDRD)
Ahmed Bougacha52468672015-09-11 17:08:28 +000012739TargetLowering::AtomicExpansionKind
12740ARMTargetLowering::shouldExpandAtomicLoadInIR(LoadInst *LI) const {
Robin Morisseted3d48f2014-09-03 21:29:59 +000012741 unsigned Size = LI->getType()->getPrimitiveSizeInBits();
Tim Northoverf520eff2015-12-02 18:12:57 +000012742 return ((Size == 64) && !Subtarget->isMClass()) ? AtomicExpansionKind::LLOnly
Ahmed Bougacha52468672015-09-11 17:08:28 +000012743 : AtomicExpansionKind::None;
Robin Morisseted3d48f2014-09-03 21:29:59 +000012744}
12745
12746// For the real atomic operations, we have ldrex/strex up to 32 bits,
12747// and up to 64 bits on the non-M profiles
Ahmed Bougacha52468672015-09-11 17:08:28 +000012748TargetLowering::AtomicExpansionKind
JF Bastienf14889e2015-03-04 15:47:57 +000012749ARMTargetLowering::shouldExpandAtomicRMWInIR(AtomicRMWInst *AI) const {
Robin Morisseted3d48f2014-09-03 21:29:59 +000012750 unsigned Size = AI->getType()->getPrimitiveSizeInBits();
JF Bastienf14889e2015-03-04 15:47:57 +000012751 return (Size <= (Subtarget->isMClass() ? 32U : 64U))
Ahmed Bougacha9d677132015-09-11 17:08:17 +000012752 ? AtomicExpansionKind::LLSC
12753 : AtomicExpansionKind::None;
Tim Northover037f26f22014-04-17 18:22:47 +000012754}
12755
Ahmed Bougacha52468672015-09-11 17:08:28 +000012756bool ARMTargetLowering::shouldExpandAtomicCmpXchgInIR(
12757 AtomicCmpXchgInst *AI) const {
Tim Northoverb629c772016-04-18 21:48:55 +000012758 // At -O0, fast-regalloc cannot cope with the live vregs necessary to
12759 // implement cmpxchg without spilling. If the address being exchanged is also
12760 // on the stack and close enough to the spill slot, this can lead to a
12761 // situation where the monitor always gets cleared and the atomic operation
12762 // can never succeed. So at -O0 we need a late-expanded pseudo-inst instead.
12763 return getTargetMachine().getOptLevel() != 0;
Ahmed Bougacha52468672015-09-11 17:08:28 +000012764}
12765
James Y Knightf44fc522016-03-16 22:12:04 +000012766bool ARMTargetLowering::shouldInsertFencesForAtomic(
12767 const Instruction *I) const {
12768 return InsertFencesForAtomic;
12769}
12770
Akira Hatanakae5b6e0d2014-07-25 19:31:34 +000012771// This has so far only been implemented for MachO.
12772bool ARMTargetLowering::useLoadStackGuardNode() const {
Eric Christopher66322e82014-12-05 00:22:35 +000012773 return Subtarget->isTargetMachO();
Akira Hatanakae5b6e0d2014-07-25 19:31:34 +000012774}
12775
Quentin Colombetc32615d2014-10-31 17:52:53 +000012776bool ARMTargetLowering::canCombineStoreAndExtract(Type *VectorTy, Value *Idx,
12777 unsigned &Cost) const {
12778 // If we do not have NEON, vector types are not natively supported.
12779 if (!Subtarget->hasNEON())
12780 return false;
12781
12782 // Floating point values and vector values map to the same register file.
Benjamin Kramerdf005cb2015-08-08 18:27:36 +000012783 // Therefore, although we could do a store extract of a vector type, this is
Quentin Colombetc32615d2014-10-31 17:52:53 +000012784 // better to leave at float as we have more freedom in the addressing mode for
12785 // those.
12786 if (VectorTy->isFPOrFPVectorTy())
12787 return false;
12788
12789 // If the index is unknown at compile time, this is very expensive to lower
12790 // and it is not possible to combine the store with the extract.
12791 if (!isa<ConstantInt>(Idx))
12792 return false;
12793
12794 assert(VectorTy->isVectorTy() && "VectorTy is not a vector type");
12795 unsigned BitWidth = cast<VectorType>(VectorTy)->getBitWidth();
12796 // We can do a store + vector extract on any vector that fits perfectly in a D
12797 // or Q register.
12798 if (BitWidth == 64 || BitWidth == 128) {
12799 Cost = 0;
12800 return true;
12801 }
12802 return false;
12803}
12804
Sanjay Patelaf1b48b2015-11-10 19:24:31 +000012805bool ARMTargetLowering::isCheapToSpeculateCttz() const {
12806 return Subtarget->hasV6T2Ops();
12807}
12808
12809bool ARMTargetLowering::isCheapToSpeculateCtlz() const {
12810 return Subtarget->hasV6T2Ops();
12811}
12812
Tim Northover037f26f22014-04-17 18:22:47 +000012813Value *ARMTargetLowering::emitLoadLinked(IRBuilder<> &Builder, Value *Addr,
12814 AtomicOrdering Ord) const {
12815 Module *M = Builder.GetInsertBlock()->getParent()->getParent();
12816 Type *ValTy = cast<PointerType>(Addr->getType())->getElementType();
JF Bastien800f87a2016-04-06 21:19:33 +000012817 bool IsAcquire = isAcquireOrStronger(Ord);
Tim Northover037f26f22014-04-17 18:22:47 +000012818
12819 // Since i64 isn't legal and intrinsics don't get type-lowered, the ldrexd
12820 // intrinsic must return {i32, i32} and we have to recombine them into a
12821 // single i64 here.
12822 if (ValTy->getPrimitiveSizeInBits() == 64) {
12823 Intrinsic::ID Int =
12824 IsAcquire ? Intrinsic::arm_ldaexd : Intrinsic::arm_ldrexd;
12825 Function *Ldrex = llvm::Intrinsic::getDeclaration(M, Int);
12826
12827 Addr = Builder.CreateBitCast(Addr, Type::getInt8PtrTy(M->getContext()));
12828 Value *LoHi = Builder.CreateCall(Ldrex, Addr, "lohi");
12829
12830 Value *Lo = Builder.CreateExtractValue(LoHi, 0, "lo");
12831 Value *Hi = Builder.CreateExtractValue(LoHi, 1, "hi");
Christian Pirkerb5728192014-05-08 14:06:24 +000012832 if (!Subtarget->isLittle())
12833 std::swap (Lo, Hi);
Tim Northover037f26f22014-04-17 18:22:47 +000012834 Lo = Builder.CreateZExt(Lo, ValTy, "lo64");
12835 Hi = Builder.CreateZExt(Hi, ValTy, "hi64");
12836 return Builder.CreateOr(
12837 Lo, Builder.CreateShl(Hi, ConstantInt::get(ValTy, 32)), "val64");
12838 }
12839
12840 Type *Tys[] = { Addr->getType() };
12841 Intrinsic::ID Int = IsAcquire ? Intrinsic::arm_ldaex : Intrinsic::arm_ldrex;
12842 Function *Ldrex = llvm::Intrinsic::getDeclaration(M, Int, Tys);
12843
12844 return Builder.CreateTruncOrBitCast(
12845 Builder.CreateCall(Ldrex, Addr),
12846 cast<PointerType>(Addr->getType())->getElementType());
12847}
12848
Ahmed Bougacha81616a72015-09-22 17:22:58 +000012849void ARMTargetLowering::emitAtomicCmpXchgNoStoreLLBalance(
12850 IRBuilder<> &Builder) const {
Ahmed Bougachae81610f2015-09-26 00:14:02 +000012851 if (!Subtarget->hasV7Ops())
12852 return;
Ahmed Bougacha81616a72015-09-22 17:22:58 +000012853 Module *M = Builder.GetInsertBlock()->getParent()->getParent();
12854 Builder.CreateCall(llvm::Intrinsic::getDeclaration(M, Intrinsic::arm_clrex));
12855}
12856
Tim Northover037f26f22014-04-17 18:22:47 +000012857Value *ARMTargetLowering::emitStoreConditional(IRBuilder<> &Builder, Value *Val,
12858 Value *Addr,
12859 AtomicOrdering Ord) const {
12860 Module *M = Builder.GetInsertBlock()->getParent()->getParent();
JF Bastien800f87a2016-04-06 21:19:33 +000012861 bool IsRelease = isReleaseOrStronger(Ord);
Tim Northover037f26f22014-04-17 18:22:47 +000012862
12863 // Since the intrinsics must have legal type, the i64 intrinsics take two
12864 // parameters: "i32, i32". We must marshal Val into the appropriate form
12865 // before the call.
12866 if (Val->getType()->getPrimitiveSizeInBits() == 64) {
12867 Intrinsic::ID Int =
12868 IsRelease ? Intrinsic::arm_stlexd : Intrinsic::arm_strexd;
12869 Function *Strex = Intrinsic::getDeclaration(M, Int);
12870 Type *Int32Ty = Type::getInt32Ty(M->getContext());
12871
12872 Value *Lo = Builder.CreateTrunc(Val, Int32Ty, "lo");
12873 Value *Hi = Builder.CreateTrunc(Builder.CreateLShr(Val, 32), Int32Ty, "hi");
Christian Pirkerb5728192014-05-08 14:06:24 +000012874 if (!Subtarget->isLittle())
12875 std::swap (Lo, Hi);
Tim Northover037f26f22014-04-17 18:22:47 +000012876 Addr = Builder.CreateBitCast(Addr, Type::getInt8PtrTy(M->getContext()));
David Blaikieff6409d2015-05-18 22:13:54 +000012877 return Builder.CreateCall(Strex, {Lo, Hi, Addr});
Tim Northover037f26f22014-04-17 18:22:47 +000012878 }
12879
12880 Intrinsic::ID Int = IsRelease ? Intrinsic::arm_stlex : Intrinsic::arm_strex;
12881 Type *Tys[] = { Addr->getType() };
12882 Function *Strex = Intrinsic::getDeclaration(M, Int, Tys);
12883
David Blaikieff6409d2015-05-18 22:13:54 +000012884 return Builder.CreateCall(
12885 Strex, {Builder.CreateZExtOrBitCast(
12886 Val, Strex->getFunctionType()->getParamType(0)),
12887 Addr});
Tim Northover037f26f22014-04-17 18:22:47 +000012888}
Oliver Stannardc24f2172014-05-09 14:01:47 +000012889
Hao Liu2cd34bb2015-06-26 02:45:36 +000012890/// \brief Lower an interleaved load into a vldN intrinsic.
12891///
12892/// E.g. Lower an interleaved load (Factor = 2):
12893/// %wide.vec = load <8 x i32>, <8 x i32>* %ptr, align 4
12894/// %v0 = shuffle %wide.vec, undef, <0, 2, 4, 6> ; Extract even elements
12895/// %v1 = shuffle %wide.vec, undef, <1, 3, 5, 7> ; Extract odd elements
12896///
12897/// Into:
12898/// %vld2 = { <4 x i32>, <4 x i32> } call llvm.arm.neon.vld2(%ptr, 4)
12899/// %vec0 = extractelement { <4 x i32>, <4 x i32> } %vld2, i32 0
12900/// %vec1 = extractelement { <4 x i32>, <4 x i32> } %vld2, i32 1
12901bool ARMTargetLowering::lowerInterleavedLoad(
12902 LoadInst *LI, ArrayRef<ShuffleVectorInst *> Shuffles,
12903 ArrayRef<unsigned> Indices, unsigned Factor) const {
12904 assert(Factor >= 2 && Factor <= getMaxSupportedInterleaveFactor() &&
12905 "Invalid interleave factor");
12906 assert(!Shuffles.empty() && "Empty shufflevector input");
12907 assert(Shuffles.size() == Indices.size() &&
12908 "Unmatched number of shufflevectors and indices");
12909
12910 VectorType *VecTy = Shuffles[0]->getType();
12911 Type *EltTy = VecTy->getVectorElementType();
12912
Mehdi Aminia749f2a2015-07-09 02:09:52 +000012913 const DataLayout &DL = LI->getModule()->getDataLayout();
Ahmed Bougacha97564c32015-12-09 01:19:50 +000012914 unsigned VecSize = DL.getTypeSizeInBits(VecTy);
12915 bool EltIs64Bits = DL.getTypeSizeInBits(EltTy) == 64;
Hao Liu2cd34bb2015-06-26 02:45:36 +000012916
Jeroen Ketemaaebca092015-10-07 14:53:29 +000012917 // Skip if we do not have NEON and skip illegal vector types and vector types
12918 // with i64/f64 elements (vldN doesn't support i64/f64 elements).
12919 if (!Subtarget->hasNEON() || (VecSize != 64 && VecSize != 128) || EltIs64Bits)
Hao Liu2cd34bb2015-06-26 02:45:36 +000012920 return false;
12921
12922 // A pointer vector can not be the return type of the ldN intrinsics. Need to
12923 // load integer vectors first and then convert to pointer vectors.
12924 if (EltTy->isPointerTy())
Mehdi Aminia749f2a2015-07-09 02:09:52 +000012925 VecTy =
12926 VectorType::get(DL.getIntPtrType(EltTy), VecTy->getVectorNumElements());
Hao Liu2cd34bb2015-06-26 02:45:36 +000012927
12928 static const Intrinsic::ID LoadInts[3] = {Intrinsic::arm_neon_vld2,
12929 Intrinsic::arm_neon_vld3,
12930 Intrinsic::arm_neon_vld4};
12931
Hao Liu2cd34bb2015-06-26 02:45:36 +000012932 IRBuilder<> Builder(LI);
12933 SmallVector<Value *, 2> Ops;
12934
12935 Type *Int8Ptr = Builder.getInt8PtrTy(LI->getPointerAddressSpace());
12936 Ops.push_back(Builder.CreateBitCast(LI->getPointerOperand(), Int8Ptr));
12937 Ops.push_back(Builder.getInt32(LI->getAlignment()));
12938
Jeroen Ketemaab99b592015-09-30 10:56:37 +000012939 Type *Tys[] = { VecTy, Int8Ptr };
12940 Function *VldnFunc =
12941 Intrinsic::getDeclaration(LI->getModule(), LoadInts[Factor - 2], Tys);
Hao Liu2cd34bb2015-06-26 02:45:36 +000012942 CallInst *VldN = Builder.CreateCall(VldnFunc, Ops, "vldN");
12943
12944 // Replace uses of each shufflevector with the corresponding vector loaded
12945 // by ldN.
12946 for (unsigned i = 0; i < Shuffles.size(); i++) {
12947 ShuffleVectorInst *SV = Shuffles[i];
12948 unsigned Index = Indices[i];
12949
12950 Value *SubVec = Builder.CreateExtractValue(VldN, Index);
12951
12952 // Convert the integer vector to pointer vector if the element is pointer.
12953 if (EltTy->isPointerTy())
12954 SubVec = Builder.CreateIntToPtr(SubVec, SV->getType());
12955
12956 SV->replaceAllUsesWith(SubVec);
12957 }
12958
12959 return true;
12960}
12961
12962/// \brief Get a mask consisting of sequential integers starting from \p Start.
12963///
12964/// I.e. <Start, Start + 1, ..., Start + NumElts - 1>
12965static Constant *getSequentialMask(IRBuilder<> &Builder, unsigned Start,
12966 unsigned NumElts) {
12967 SmallVector<Constant *, 16> Mask;
12968 for (unsigned i = 0; i < NumElts; i++)
12969 Mask.push_back(Builder.getInt32(Start + i));
12970
12971 return ConstantVector::get(Mask);
12972}
12973
12974/// \brief Lower an interleaved store into a vstN intrinsic.
12975///
12976/// E.g. Lower an interleaved store (Factor = 3):
12977/// %i.vec = shuffle <8 x i32> %v0, <8 x i32> %v1,
12978/// <0, 4, 8, 1, 5, 9, 2, 6, 10, 3, 7, 11>
12979/// store <12 x i32> %i.vec, <12 x i32>* %ptr, align 4
12980///
12981/// Into:
12982/// %sub.v0 = shuffle <8 x i32> %v0, <8 x i32> v1, <0, 1, 2, 3>
12983/// %sub.v1 = shuffle <8 x i32> %v0, <8 x i32> v1, <4, 5, 6, 7>
12984/// %sub.v2 = shuffle <8 x i32> %v0, <8 x i32> v1, <8, 9, 10, 11>
12985/// call void llvm.arm.neon.vst3(%ptr, %sub.v0, %sub.v1, %sub.v2, 4)
12986///
12987/// Note that the new shufflevectors will be removed and we'll only generate one
12988/// vst3 instruction in CodeGen.
12989bool ARMTargetLowering::lowerInterleavedStore(StoreInst *SI,
12990 ShuffleVectorInst *SVI,
12991 unsigned Factor) const {
12992 assert(Factor >= 2 && Factor <= getMaxSupportedInterleaveFactor() &&
12993 "Invalid interleave factor");
12994
12995 VectorType *VecTy = SVI->getType();
12996 assert(VecTy->getVectorNumElements() % Factor == 0 &&
12997 "Invalid interleaved store");
12998
12999 unsigned NumSubElts = VecTy->getVectorNumElements() / Factor;
13000 Type *EltTy = VecTy->getVectorElementType();
13001 VectorType *SubVecTy = VectorType::get(EltTy, NumSubElts);
13002
Mehdi Aminia749f2a2015-07-09 02:09:52 +000013003 const DataLayout &DL = SI->getModule()->getDataLayout();
Ahmed Bougacha97564c32015-12-09 01:19:50 +000013004 unsigned SubVecSize = DL.getTypeSizeInBits(SubVecTy);
13005 bool EltIs64Bits = DL.getTypeSizeInBits(EltTy) == 64;
Hao Liu2cd34bb2015-06-26 02:45:36 +000013006
Jeroen Ketemaaebca092015-10-07 14:53:29 +000013007 // Skip if we do not have NEON and skip illegal vector types and vector types
13008 // with i64/f64 elements (vstN doesn't support i64/f64 elements).
13009 if (!Subtarget->hasNEON() || (SubVecSize != 64 && SubVecSize != 128) ||
13010 EltIs64Bits)
Hao Liu2cd34bb2015-06-26 02:45:36 +000013011 return false;
13012
13013 Value *Op0 = SVI->getOperand(0);
13014 Value *Op1 = SVI->getOperand(1);
13015 IRBuilder<> Builder(SI);
13016
13017 // StN intrinsics don't support pointer vectors as arguments. Convert pointer
13018 // vectors to integer vectors.
13019 if (EltTy->isPointerTy()) {
Mehdi Aminia749f2a2015-07-09 02:09:52 +000013020 Type *IntTy = DL.getIntPtrType(EltTy);
Hao Liu2cd34bb2015-06-26 02:45:36 +000013021
13022 // Convert to the corresponding integer vector.
13023 Type *IntVecTy =
13024 VectorType::get(IntTy, Op0->getType()->getVectorNumElements());
13025 Op0 = Builder.CreatePtrToInt(Op0, IntVecTy);
13026 Op1 = Builder.CreatePtrToInt(Op1, IntVecTy);
13027
13028 SubVecTy = VectorType::get(IntTy, NumSubElts);
13029 }
13030
Craig Topper26260942015-10-18 05:15:34 +000013031 static const Intrinsic::ID StoreInts[3] = {Intrinsic::arm_neon_vst2,
13032 Intrinsic::arm_neon_vst3,
13033 Intrinsic::arm_neon_vst4};
Hao Liu2cd34bb2015-06-26 02:45:36 +000013034 SmallVector<Value *, 6> Ops;
13035
13036 Type *Int8Ptr = Builder.getInt8PtrTy(SI->getPointerAddressSpace());
13037 Ops.push_back(Builder.CreateBitCast(SI->getPointerOperand(), Int8Ptr));
13038
Jeroen Ketemaab99b592015-09-30 10:56:37 +000013039 Type *Tys[] = { Int8Ptr, SubVecTy };
13040 Function *VstNFunc = Intrinsic::getDeclaration(
13041 SI->getModule(), StoreInts[Factor - 2], Tys);
13042
Hao Liu2cd34bb2015-06-26 02:45:36 +000013043 // Split the shufflevector operands into sub vectors for the new vstN call.
13044 for (unsigned i = 0; i < Factor; i++)
13045 Ops.push_back(Builder.CreateShuffleVector(
13046 Op0, Op1, getSequentialMask(Builder, NumSubElts * i, NumSubElts)));
13047
13048 Ops.push_back(Builder.getInt32(SI->getAlignment()));
13049 Builder.CreateCall(VstNFunc, Ops);
13050 return true;
13051}
13052
Oliver Stannardc24f2172014-05-09 14:01:47 +000013053enum HABaseType {
13054 HA_UNKNOWN = 0,
13055 HA_FLOAT,
13056 HA_DOUBLE,
13057 HA_VECT64,
13058 HA_VECT128
13059};
13060
13061static bool isHomogeneousAggregate(Type *Ty, HABaseType &Base,
13062 uint64_t &Members) {
Craig Toppere3dcce92015-08-01 22:20:21 +000013063 if (auto *ST = dyn_cast<StructType>(Ty)) {
Oliver Stannardc24f2172014-05-09 14:01:47 +000013064 for (unsigned i = 0; i < ST->getNumElements(); ++i) {
13065 uint64_t SubMembers = 0;
13066 if (!isHomogeneousAggregate(ST->getElementType(i), Base, SubMembers))
13067 return false;
13068 Members += SubMembers;
13069 }
Craig Toppere3dcce92015-08-01 22:20:21 +000013070 } else if (auto *AT = dyn_cast<ArrayType>(Ty)) {
Oliver Stannardc24f2172014-05-09 14:01:47 +000013071 uint64_t SubMembers = 0;
13072 if (!isHomogeneousAggregate(AT->getElementType(), Base, SubMembers))
13073 return false;
13074 Members += SubMembers * AT->getNumElements();
13075 } else if (Ty->isFloatTy()) {
13076 if (Base != HA_UNKNOWN && Base != HA_FLOAT)
13077 return false;
13078 Members = 1;
13079 Base = HA_FLOAT;
13080 } else if (Ty->isDoubleTy()) {
13081 if (Base != HA_UNKNOWN && Base != HA_DOUBLE)
13082 return false;
13083 Members = 1;
13084 Base = HA_DOUBLE;
Craig Toppere3dcce92015-08-01 22:20:21 +000013085 } else if (auto *VT = dyn_cast<VectorType>(Ty)) {
Oliver Stannardc24f2172014-05-09 14:01:47 +000013086 Members = 1;
13087 switch (Base) {
13088 case HA_FLOAT:
13089 case HA_DOUBLE:
13090 return false;
13091 case HA_VECT64:
13092 return VT->getBitWidth() == 64;
13093 case HA_VECT128:
13094 return VT->getBitWidth() == 128;
13095 case HA_UNKNOWN:
13096 switch (VT->getBitWidth()) {
13097 case 64:
13098 Base = HA_VECT64;
13099 return true;
13100 case 128:
13101 Base = HA_VECT128;
13102 return true;
13103 default:
13104 return false;
13105 }
13106 }
13107 }
13108
13109 return (Members > 0 && Members <= 4);
13110}
13111
Tim Northovere95c5b32015-02-24 17:22:34 +000013112/// \brief Return true if a type is an AAPCS-VFP homogeneous aggregate or one of
13113/// [N x i32] or [N x i64]. This allows front-ends to skip emitting padding when
13114/// passing according to AAPCS rules.
Oliver Stannardc24f2172014-05-09 14:01:47 +000013115bool ARMTargetLowering::functionArgumentNeedsConsecutiveRegisters(
13116 Type *Ty, CallingConv::ID CallConv, bool isVarArg) const {
Tim Northover4f1909f2014-05-27 10:43:38 +000013117 if (getEffectiveCallingConv(CallConv, isVarArg) !=
13118 CallingConv::ARM_AAPCS_VFP)
Oliver Stannardc24f2172014-05-09 14:01:47 +000013119 return false;
Tim Northover4f1909f2014-05-27 10:43:38 +000013120
13121 HABaseType Base = HA_UNKNOWN;
13122 uint64_t Members = 0;
Tim Northovere95c5b32015-02-24 17:22:34 +000013123 bool IsHA = isHomogeneousAggregate(Ty, Base, Members);
13124 DEBUG(dbgs() << "isHA: " << IsHA << " "; Ty->dump());
13125
13126 bool IsIntArray = Ty->isArrayTy() && Ty->getArrayElementType()->isIntegerTy();
13127 return IsHA || IsIntArray;
Oliver Stannardc24f2172014-05-09 14:01:47 +000013128}
Joseph Tremouletf748c892015-11-07 01:11:31 +000013129
13130unsigned ARMTargetLowering::getExceptionPointerRegister(
13131 const Constant *PersonalityFn) const {
13132 // Platforms which do not use SjLj EH may return values in these registers
13133 // via the personality function.
13134 return Subtarget->useSjLjEH() ? ARM::NoRegister : ARM::R0;
13135}
13136
13137unsigned ARMTargetLowering::getExceptionSelectorRegister(
13138 const Constant *PersonalityFn) const {
13139 // Platforms which do not use SjLj EH may return values in these registers
13140 // via the personality function.
13141 return Subtarget->useSjLjEH() ? ARM::NoRegister : ARM::R1;
13142}
Manman Ren5e9e65e2016-01-12 00:47:18 +000013143
13144void ARMTargetLowering::initializeSplitCSR(MachineBasicBlock *Entry) const {
13145 // Update IsSplitCSR in ARMFunctionInfo.
13146 ARMFunctionInfo *AFI = Entry->getParent()->getInfo<ARMFunctionInfo>();
13147 AFI->setIsSplitCSR(true);
13148}
13149
13150void ARMTargetLowering::insertCopiesSplitCSR(
13151 MachineBasicBlock *Entry,
13152 const SmallVectorImpl<MachineBasicBlock *> &Exits) const {
13153 const ARMBaseRegisterInfo *TRI = Subtarget->getRegisterInfo();
13154 const MCPhysReg *IStart = TRI->getCalleeSavedRegsViaCopy(Entry->getParent());
13155 if (!IStart)
13156 return;
13157
13158 const TargetInstrInfo *TII = Subtarget->getInstrInfo();
13159 MachineRegisterInfo *MRI = &Entry->getParent()->getRegInfo();
Manman Rene5f807f2016-01-15 20:24:11 +000013160 MachineBasicBlock::iterator MBBI = Entry->begin();
Manman Ren5e9e65e2016-01-12 00:47:18 +000013161 for (const MCPhysReg *I = IStart; *I; ++I) {
13162 const TargetRegisterClass *RC = nullptr;
13163 if (ARM::GPRRegClass.contains(*I))
13164 RC = &ARM::GPRRegClass;
13165 else if (ARM::DPRRegClass.contains(*I))
13166 RC = &ARM::DPRRegClass;
13167 else
13168 llvm_unreachable("Unexpected register class in CSRsViaCopy!");
13169
13170 unsigned NewVR = MRI->createVirtualRegister(RC);
13171 // Create copy from CSR to a virtual register.
13172 // FIXME: this currently does not emit CFI pseudo-instructions, it works
13173 // fine for CXX_FAST_TLS since the C++-style TLS access functions should be
13174 // nounwind. If we want to generalize this later, we may need to emit
13175 // CFI pseudo-instructions.
13176 assert(Entry->getParent()->getFunction()->hasFnAttribute(
13177 Attribute::NoUnwind) &&
13178 "Function should be nounwind in insertCopiesSplitCSR!");
13179 Entry->addLiveIn(*I);
Manman Rene5f807f2016-01-15 20:24:11 +000013180 BuildMI(*Entry, MBBI, DebugLoc(), TII->get(TargetOpcode::COPY), NewVR)
Manman Ren5e9e65e2016-01-12 00:47:18 +000013181 .addReg(*I);
13182
Manman Rene5f807f2016-01-15 20:24:11 +000013183 // Insert the copy-back instructions right before the terminator.
Manman Ren5e9e65e2016-01-12 00:47:18 +000013184 for (auto *Exit : Exits)
Manman Rene5f807f2016-01-15 20:24:11 +000013185 BuildMI(*Exit, Exit->getFirstTerminator(), DebugLoc(),
13186 TII->get(TargetOpcode::COPY), *I)
Manman Ren5e9e65e2016-01-12 00:47:18 +000013187 .addReg(NewVR);
13188 }
13189}