Gitiles
Code Review Sign In
gerrit-public.fairphone.software / toolchain / llvm-project / ac451066f48820a0be4bccba0a64b7c2e2dc0c35 / . / llvm / lib / Target / NVPTX / NVPTXISelLowering.cpp
blob: 91e24bb617d8ba0ede44a96d3b54d8b52b04b50a [file] [log] [blame]
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]1//
2// The LLVM Compiler Infrastructure
3//
4// This file is distributed under the University of Illinois Open Source
5// License. See LICENSE.TXT for details.
6//
7//===----------------------------------------------------------------------===//
8//
9// This file defines the interfaces that NVPTX uses to lower LLVM code into a
10// selection DAG.
11//
12//===----------------------------------------------------------------------===//
13
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]14#include "NVPTXISelLowering.h"
Chandler Carruthed0881b2012-12-03 16:50:05 +0000[diff] [blame]15#include "NVPTX.h"
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]16#include "NVPTXTargetMachine.h"
17#include "NVPTXTargetObjectFile.h"
18#include "NVPTXUtilities.h"
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]19#include "llvm/CodeGen/Analysis.h"
20#include "llvm/CodeGen/MachineFrameInfo.h"
21#include "llvm/CodeGen/MachineFunction.h"
22#include "llvm/CodeGen/MachineInstrBuilder.h"
23#include "llvm/CodeGen/MachineRegisterInfo.h"
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]24#include "llvm/CodeGen/TargetLoweringObjectFileImpl.h"
Chandler Carruth219b89b2014-03-04 11:01:28 +0000[diff] [blame]25#include "llvm/IR/CallSite.h"
Chandler Carruth9fb823b2013-01-02 11:36:10 +0000[diff] [blame]26#include "llvm/IR/DerivedTypes.h"
27#include "llvm/IR/Function.h"
28#include "llvm/IR/GlobalValue.h"
29#include "llvm/IR/IntrinsicInst.h"
30#include "llvm/IR/Intrinsics.h"
31#include "llvm/IR/Module.h"
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]32#include "llvm/MC/MCSectionELF.h"
Chandler Carruthed0881b2012-12-03 16:50:05 +0000[diff] [blame]33#include "llvm/Support/CommandLine.h"
34#include "llvm/Support/Debug.h"
35#include "llvm/Support/ErrorHandling.h"
Justin Holewinski9982f062014-06-27 19:36:25 +0000[diff] [blame]36#include "llvm/Support/MathExtras.h"
Chandler Carruthed0881b2012-12-03 16:50:05 +0000[diff] [blame]37#include "llvm/Support/raw_ostream.h"
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]38#include <sstream>
39
40#undef DEBUG_TYPE
41#define DEBUG_TYPE "nvptx-lower"
42
43using namespace llvm;
44
45static unsigned int uniqueCallSite = 0;
46
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]47static cl::opt<bool> sched4reg(
48 "nvptx-sched4reg",
49 cl::desc("NVPTX Specific: schedule for register pressue"), cl::init(false));
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]50
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]51static bool IsPTXVectorType(MVT VT) {
52 switch (VT.SimpleTy) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]53 default:
54 return false;
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]55 case MVT::v2i1:
56 case MVT::v4i1:
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]57 case MVT::v2i8:
58 case MVT::v4i8:
59 case MVT::v2i16:
60 case MVT::v4i16:
61 case MVT::v2i32:
62 case MVT::v4i32:
63 case MVT::v2i64:
64 case MVT::v2f32:
65 case MVT::v4f32:
66 case MVT::v2f64:
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]67 return true;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]68 }
69}
70
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]71/// ComputePTXValueVTs - For the given Type \p Ty, returns the set of primitive
72/// EVTs that compose it. Unlike ComputeValueVTs, this will break apart vectors
73/// into their primitive components.
74/// NOTE: This is a band-aid for code that expects ComputeValueVTs to return the
75/// same number of types as the Ins/Outs arrays in LowerFormalArguments,
76/// LowerCall, and LowerReturn.
77static void ComputePTXValueVTs(const TargetLowering &TLI, Type *Ty,
78 SmallVectorImpl<EVT> &ValueVTs,
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]79 SmallVectorImpl<uint64_t> *Offsets = nullptr,
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]80 uint64_t StartingOffset = 0) {
81 SmallVector<EVT, 16> TempVTs;
82 SmallVector<uint64_t, 16> TempOffsets;
83
84 ComputeValueVTs(TLI, Ty, TempVTs, &TempOffsets, StartingOffset);
85 for (unsigned i = 0, e = TempVTs.size(); i != e; ++i) {
86 EVT VT = TempVTs[i];
87 uint64_t Off = TempOffsets[i];
88 if (VT.isVector())
89 for (unsigned j = 0, je = VT.getVectorNumElements(); j != je; ++j) {
90 ValueVTs.push_back(VT.getVectorElementType());
91 if (Offsets)
92 Offsets->push_back(Off+j*VT.getVectorElementType().getStoreSize());
93 }
94 else {
95 ValueVTs.push_back(VT);
96 if (Offsets)
97 Offsets->push_back(Off);
98 }
99 }
100}
101
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]102// NVPTXTargetLowering Constructor.
103NVPTXTargetLowering::NVPTXTargetLowering(NVPTXTargetMachine &TM)
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]104 : TargetLowering(TM, new NVPTXTargetObjectFile()), nvTM(&TM),
105 nvptxSubtarget(TM.getSubtarget<NVPTXSubtarget>()) {
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]106
107 // always lower memset, memcpy, and memmove intrinsics to load/store
108 // instructions, rather
109 // then generating calls to memset, mempcy or memmove.
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]110 MaxStoresPerMemset = (unsigned) 0xFFFFFFFF;
111 MaxStoresPerMemcpy = (unsigned) 0xFFFFFFFF;
112 MaxStoresPerMemmove = (unsigned) 0xFFFFFFFF;
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]113
114 setBooleanContents(ZeroOrNegativeOneBooleanContent);
Justin Holewinskid7d8fe02014-06-27 18:35:42 +0000[diff] [blame]115 setBooleanVectorContents(ZeroOrNegativeOneBooleanContent);
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]116
117 // Jump is Expensive. Don't create extra control flow for 'and', 'or'
118 // condition branches.
119 setJumpIsExpensive(true);
120
121 // By default, use the Source scheduling
122 if (sched4reg)
123 setSchedulingPreference(Sched::RegPressure);
124 else
125 setSchedulingPreference(Sched::Source);
126
127 addRegisterClass(MVT::i1, &NVPTX::Int1RegsRegClass);
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]128 addRegisterClass(MVT::i16, &NVPTX::Int16RegsRegClass);
129 addRegisterClass(MVT::i32, &NVPTX::Int32RegsRegClass);
130 addRegisterClass(MVT::i64, &NVPTX::Int64RegsRegClass);
131 addRegisterClass(MVT::f32, &NVPTX::Float32RegsRegClass);
132 addRegisterClass(MVT::f64, &NVPTX::Float64RegsRegClass);
133
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]134 // Operations not directly supported by NVPTX.
Tom Stellard3787b122014-06-10 16:01:29 +0000[diff] [blame]135 setOperationAction(ISD::SELECT_CC, MVT::f32, Expand);
136 setOperationAction(ISD::SELECT_CC, MVT::f64, Expand);
137 setOperationAction(ISD::SELECT_CC, MVT::i1, Expand);
138 setOperationAction(ISD::SELECT_CC, MVT::i8, Expand);
139 setOperationAction(ISD::SELECT_CC, MVT::i16, Expand);
140 setOperationAction(ISD::SELECT_CC, MVT::i32, Expand);
141 setOperationAction(ISD::SELECT_CC, MVT::i64, Expand);
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]142 setOperationAction(ISD::BR_CC, MVT::f32, Expand);
143 setOperationAction(ISD::BR_CC, MVT::f64, Expand);
144 setOperationAction(ISD::BR_CC, MVT::i1, Expand);
145 setOperationAction(ISD::BR_CC, MVT::i8, Expand);
146 setOperationAction(ISD::BR_CC, MVT::i16, Expand);
147 setOperationAction(ISD::BR_CC, MVT::i32, Expand);
148 setOperationAction(ISD::BR_CC, MVT::i64, Expand);
Justin Holewinski318c6252013-07-01 12:58:56 +0000[diff] [blame]149 // Some SIGN_EXTEND_INREG can be done using cvt instruction.
150 // For others we will expand to a SHL/SRA pair.
151 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i64, Legal);
152 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i32, Legal);
153 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i16, Legal);
154 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i8 , Legal);
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]155 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i1, Expand);
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]156
Justin Holewinski360a5cf2014-06-27 18:35:40 +0000[diff] [blame]157 setOperationAction(ISD::SHL_PARTS, MVT::i32 , Custom);
158 setOperationAction(ISD::SRA_PARTS, MVT::i32 , Custom);
159 setOperationAction(ISD::SRL_PARTS, MVT::i32 , Custom);
160 setOperationAction(ISD::SHL_PARTS, MVT::i64 , Custom);
161 setOperationAction(ISD::SRA_PARTS, MVT::i64 , Custom);
162 setOperationAction(ISD::SRL_PARTS, MVT::i64 , Custom);
163
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]164 if (nvptxSubtarget.hasROT64()) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]165 setOperationAction(ISD::ROTL, MVT::i64, Legal);
166 setOperationAction(ISD::ROTR, MVT::i64, Legal);
167 } else {
168 setOperationAction(ISD::ROTL, MVT::i64, Expand);
169 setOperationAction(ISD::ROTR, MVT::i64, Expand);
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]170 }
171 if (nvptxSubtarget.hasROT32()) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]172 setOperationAction(ISD::ROTL, MVT::i32, Legal);
173 setOperationAction(ISD::ROTR, MVT::i32, Legal);
174 } else {
175 setOperationAction(ISD::ROTL, MVT::i32, Expand);
176 setOperationAction(ISD::ROTR, MVT::i32, Expand);
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]177 }
178
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]179 setOperationAction(ISD::ROTL, MVT::i16, Expand);
180 setOperationAction(ISD::ROTR, MVT::i16, Expand);
181 setOperationAction(ISD::ROTL, MVT::i8, Expand);
182 setOperationAction(ISD::ROTR, MVT::i8, Expand);
183 setOperationAction(ISD::BSWAP, MVT::i16, Expand);
184 setOperationAction(ISD::BSWAP, MVT::i32, Expand);
185 setOperationAction(ISD::BSWAP, MVT::i64, Expand);
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]186
187 // Indirect branch is not supported.
188 // This also disables Jump Table creation.
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]189 setOperationAction(ISD::BR_JT, MVT::Other, Expand);
190 setOperationAction(ISD::BRIND, MVT::Other, Expand);
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]191
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]192 setOperationAction(ISD::GlobalAddress, MVT::i32, Custom);
193 setOperationAction(ISD::GlobalAddress, MVT::i64, Custom);
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]194
195 // We want to legalize constant related memmove and memcopy
196 // intrinsics.
197 setOperationAction(ISD::INTRINSIC_W_CHAIN, MVT::Other, Custom);
198
199 // Turn FP extload into load/fextend
200 setLoadExtAction(ISD::EXTLOAD, MVT::f32, Expand);
201 // Turn FP truncstore into trunc + store.
202 setTruncStoreAction(MVT::f64, MVT::f32, Expand);
203
204 // PTX does not support load / store predicate registers
Justin Holewinskic6462aa2012-11-14 19:19:16 +0000[diff] [blame]205 setOperationAction(ISD::LOAD, MVT::i1, Custom);
206 setOperationAction(ISD::STORE, MVT::i1, Custom);
207
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]208 setLoadExtAction(ISD::SEXTLOAD, MVT::i1, Promote);
209 setLoadExtAction(ISD::ZEXTLOAD, MVT::i1, Promote);
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]210 setTruncStoreAction(MVT::i64, MVT::i1, Expand);
211 setTruncStoreAction(MVT::i32, MVT::i1, Expand);
212 setTruncStoreAction(MVT::i16, MVT::i1, Expand);
213 setTruncStoreAction(MVT::i8, MVT::i1, Expand);
214
215 // This is legal in NVPTX
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]216 setOperationAction(ISD::ConstantFP, MVT::f64, Legal);
217 setOperationAction(ISD::ConstantFP, MVT::f32, Legal);
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]218
219 // TRAP can be lowered to PTX trap
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]220 setOperationAction(ISD::TRAP, MVT::Other, Legal);
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]221
Justin Holewinski51cb1342013-07-01 12:59:04 +0000[diff] [blame]222 setOperationAction(ISD::ADDC, MVT::i64, Expand);
223 setOperationAction(ISD::ADDE, MVT::i64, Expand);
224
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]225 // Register custom handling for vector loads/stores
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]226 for (int i = MVT::FIRST_VECTOR_VALUETYPE; i <= MVT::LAST_VECTOR_VALUETYPE;
227 ++i) {
228 MVT VT = (MVT::SimpleValueType) i;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]229 if (IsPTXVectorType(VT)) {
230 setOperationAction(ISD::LOAD, VT, Custom);
231 setOperationAction(ISD::STORE, VT, Custom);
232 setOperationAction(ISD::INTRINSIC_W_CHAIN, VT, Custom);
233 }
234 }
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]235
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]236 // Custom handling for i8 intrinsics
237 setOperationAction(ISD::INTRINSIC_W_CHAIN, MVT::i8, Custom);
238
Justin Holewinskidc372df2013-06-28 17:58:07 +0000[diff] [blame]239 setOperationAction(ISD::CTLZ, MVT::i16, Legal);
240 setOperationAction(ISD::CTLZ, MVT::i32, Legal);
241 setOperationAction(ISD::CTLZ, MVT::i64, Legal);
242 setOperationAction(ISD::CTLZ_ZERO_UNDEF, MVT::i16, Legal);
243 setOperationAction(ISD::CTLZ_ZERO_UNDEF, MVT::i32, Legal);
244 setOperationAction(ISD::CTLZ_ZERO_UNDEF, MVT::i64, Legal);
245 setOperationAction(ISD::CTTZ, MVT::i16, Expand);
246 setOperationAction(ISD::CTTZ, MVT::i32, Expand);
247 setOperationAction(ISD::CTTZ, MVT::i64, Expand);
248 setOperationAction(ISD::CTTZ_ZERO_UNDEF, MVT::i16, Expand);
249 setOperationAction(ISD::CTTZ_ZERO_UNDEF, MVT::i32, Expand);
250 setOperationAction(ISD::CTTZ_ZERO_UNDEF, MVT::i64, Expand);
251 setOperationAction(ISD::CTPOP, MVT::i16, Legal);
252 setOperationAction(ISD::CTPOP, MVT::i32, Legal);
253 setOperationAction(ISD::CTPOP, MVT::i64, Legal);
254
Justin Holewinskieafe26d2014-06-27 18:35:37 +0000[diff] [blame]255 // We have some custom DAG combine patterns for these nodes
256 setTargetDAGCombine(ISD::ADD);
257 setTargetDAGCombine(ISD::AND);
258 setTargetDAGCombine(ISD::FADD);
259 setTargetDAGCombine(ISD::MUL);
260 setTargetDAGCombine(ISD::SHL);
261
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]262 // Now deduce the information based on the above mentioned
263 // actions
264 computeRegisterProperties();
265}
266
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]267const char *NVPTXTargetLowering::getTargetNodeName(unsigned Opcode) const {
268 switch (Opcode) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]269 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]270 return nullptr;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]271 case NVPTXISD::CALL:
272 return "NVPTXISD::CALL";
273 case NVPTXISD::RET_FLAG:
274 return "NVPTXISD::RET_FLAG";
275 case NVPTXISD::Wrapper:
276 return "NVPTXISD::Wrapper";
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]277 case NVPTXISD::DeclareParam:
278 return "NVPTXISD::DeclareParam";
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]279 case NVPTXISD::DeclareScalarParam:
280 return "NVPTXISD::DeclareScalarParam";
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]281 case NVPTXISD::DeclareRet:
282 return "NVPTXISD::DeclareRet";
283 case NVPTXISD::DeclareRetParam:
284 return "NVPTXISD::DeclareRetParam";
285 case NVPTXISD::PrintCall:
286 return "NVPTXISD::PrintCall";
287 case NVPTXISD::LoadParam:
288 return "NVPTXISD::LoadParam";
Justin Holewinskife44314f2013-06-28 17:57:51 +0000[diff] [blame]289 case NVPTXISD::LoadParamV2:
290 return "NVPTXISD::LoadParamV2";
291 case NVPTXISD::LoadParamV4:
292 return "NVPTXISD::LoadParamV4";
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]293 case NVPTXISD::StoreParam:
294 return "NVPTXISD::StoreParam";
Justin Holewinskife44314f2013-06-28 17:57:51 +0000[diff] [blame]295 case NVPTXISD::StoreParamV2:
296 return "NVPTXISD::StoreParamV2";
297 case NVPTXISD::StoreParamV4:
298 return "NVPTXISD::StoreParamV4";
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]299 case NVPTXISD::StoreParamS32:
300 return "NVPTXISD::StoreParamS32";
301 case NVPTXISD::StoreParamU32:
302 return "NVPTXISD::StoreParamU32";
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]303 case NVPTXISD::CallArgBegin:
304 return "NVPTXISD::CallArgBegin";
305 case NVPTXISD::CallArg:
306 return "NVPTXISD::CallArg";
307 case NVPTXISD::LastCallArg:
308 return "NVPTXISD::LastCallArg";
309 case NVPTXISD::CallArgEnd:
310 return "NVPTXISD::CallArgEnd";
311 case NVPTXISD::CallVoid:
312 return "NVPTXISD::CallVoid";
313 case NVPTXISD::CallVal:
314 return "NVPTXISD::CallVal";
315 case NVPTXISD::CallSymbol:
316 return "NVPTXISD::CallSymbol";
317 case NVPTXISD::Prototype:
318 return "NVPTXISD::Prototype";
319 case NVPTXISD::MoveParam:
320 return "NVPTXISD::MoveParam";
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]321 case NVPTXISD::StoreRetval:
322 return "NVPTXISD::StoreRetval";
Justin Holewinskife44314f2013-06-28 17:57:51 +0000[diff] [blame]323 case NVPTXISD::StoreRetvalV2:
324 return "NVPTXISD::StoreRetvalV2";
325 case NVPTXISD::StoreRetvalV4:
326 return "NVPTXISD::StoreRetvalV4";
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]327 case NVPTXISD::PseudoUseParam:
328 return "NVPTXISD::PseudoUseParam";
329 case NVPTXISD::RETURN:
330 return "NVPTXISD::RETURN";
331 case NVPTXISD::CallSeqBegin:
332 return "NVPTXISD::CallSeqBegin";
333 case NVPTXISD::CallSeqEnd:
334 return "NVPTXISD::CallSeqEnd";
Justin Holewinski3d49e5c2013-11-15 12:30:04 +0000[diff] [blame]335 case NVPTXISD::CallPrototype:
336 return "NVPTXISD::CallPrototype";
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]337 case NVPTXISD::LoadV2:
338 return "NVPTXISD::LoadV2";
339 case NVPTXISD::LoadV4:
340 return "NVPTXISD::LoadV4";
341 case NVPTXISD::LDGV2:
342 return "NVPTXISD::LDGV2";
343 case NVPTXISD::LDGV4:
344 return "NVPTXISD::LDGV4";
345 case NVPTXISD::LDUV2:
346 return "NVPTXISD::LDUV2";
347 case NVPTXISD::LDUV4:
348 return "NVPTXISD::LDUV4";
349 case NVPTXISD::StoreV2:
350 return "NVPTXISD::StoreV2";
351 case NVPTXISD::StoreV4:
352 return "NVPTXISD::StoreV4";
Justin Holewinskieafe26d2014-06-27 18:35:37 +0000[diff] [blame]353 case NVPTXISD::FUN_SHFL_CLAMP:
354 return "NVPTXISD::FUN_SHFL_CLAMP";
355 case NVPTXISD::FUN_SHFR_CLAMP:
356 return "NVPTXISD::FUN_SHFR_CLAMP";
Justin Holewinski360a5cf2014-06-27 18:35:40 +0000[diff] [blame]357 case NVPTXISD::IMAD:
358 return "NVPTXISD::IMAD";
359 case NVPTXISD::MUL_WIDE_SIGNED:
360 return "NVPTXISD::MUL_WIDE_SIGNED";
361 case NVPTXISD::MUL_WIDE_UNSIGNED:
362 return "NVPTXISD::MUL_WIDE_UNSIGNED";
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]363 case NVPTXISD::Tex1DFloatI32: return "NVPTXISD::Tex1DFloatI32";
364 case NVPTXISD::Tex1DFloatFloat: return "NVPTXISD::Tex1DFloatFloat";
365 case NVPTXISD::Tex1DFloatFloatLevel:
366 return "NVPTXISD::Tex1DFloatFloatLevel";
367 case NVPTXISD::Tex1DFloatFloatGrad:
368 return "NVPTXISD::Tex1DFloatFloatGrad";
369 case NVPTXISD::Tex1DI32I32: return "NVPTXISD::Tex1DI32I32";
370 case NVPTXISD::Tex1DI32Float: return "NVPTXISD::Tex1DI32Float";
371 case NVPTXISD::Tex1DI32FloatLevel:
372 return "NVPTXISD::Tex1DI32FloatLevel";
373 case NVPTXISD::Tex1DI32FloatGrad:
374 return "NVPTXISD::Tex1DI32FloatGrad";
375 case NVPTXISD::Tex1DArrayFloatI32: return "NVPTXISD::Tex2DArrayFloatI32";
376 case NVPTXISD::Tex1DArrayFloatFloat: return "NVPTXISD::Tex2DArrayFloatFloat";
377 case NVPTXISD::Tex1DArrayFloatFloatLevel:
378 return "NVPTXISD::Tex2DArrayFloatFloatLevel";
379 case NVPTXISD::Tex1DArrayFloatFloatGrad:
380 return "NVPTXISD::Tex2DArrayFloatFloatGrad";
381 case NVPTXISD::Tex1DArrayI32I32: return "NVPTXISD::Tex2DArrayI32I32";
382 case NVPTXISD::Tex1DArrayI32Float: return "NVPTXISD::Tex2DArrayI32Float";
383 case NVPTXISD::Tex1DArrayI32FloatLevel:
384 return "NVPTXISD::Tex2DArrayI32FloatLevel";
385 case NVPTXISD::Tex1DArrayI32FloatGrad:
386 return "NVPTXISD::Tex2DArrayI32FloatGrad";
387 case NVPTXISD::Tex2DFloatI32: return "NVPTXISD::Tex2DFloatI32";
388 case NVPTXISD::Tex2DFloatFloat: return "NVPTXISD::Tex2DFloatFloat";
389 case NVPTXISD::Tex2DFloatFloatLevel:
390 return "NVPTXISD::Tex2DFloatFloatLevel";
391 case NVPTXISD::Tex2DFloatFloatGrad:
392 return "NVPTXISD::Tex2DFloatFloatGrad";
393 case NVPTXISD::Tex2DI32I32: return "NVPTXISD::Tex2DI32I32";
394 case NVPTXISD::Tex2DI32Float: return "NVPTXISD::Tex2DI32Float";
395 case NVPTXISD::Tex2DI32FloatLevel:
396 return "NVPTXISD::Tex2DI32FloatLevel";
397 case NVPTXISD::Tex2DI32FloatGrad:
398 return "NVPTXISD::Tex2DI32FloatGrad";
399 case NVPTXISD::Tex2DArrayFloatI32: return "NVPTXISD::Tex2DArrayFloatI32";
400 case NVPTXISD::Tex2DArrayFloatFloat: return "NVPTXISD::Tex2DArrayFloatFloat";
401 case NVPTXISD::Tex2DArrayFloatFloatLevel:
402 return "NVPTXISD::Tex2DArrayFloatFloatLevel";
403 case NVPTXISD::Tex2DArrayFloatFloatGrad:
404 return "NVPTXISD::Tex2DArrayFloatFloatGrad";
405 case NVPTXISD::Tex2DArrayI32I32: return "NVPTXISD::Tex2DArrayI32I32";
406 case NVPTXISD::Tex2DArrayI32Float: return "NVPTXISD::Tex2DArrayI32Float";
407 case NVPTXISD::Tex2DArrayI32FloatLevel:
408 return "NVPTXISD::Tex2DArrayI32FloatLevel";
409 case NVPTXISD::Tex2DArrayI32FloatGrad:
410 return "NVPTXISD::Tex2DArrayI32FloatGrad";
411 case NVPTXISD::Tex3DFloatI32: return "NVPTXISD::Tex3DFloatI32";
412 case NVPTXISD::Tex3DFloatFloat: return "NVPTXISD::Tex3DFloatFloat";
413 case NVPTXISD::Tex3DFloatFloatLevel:
414 return "NVPTXISD::Tex3DFloatFloatLevel";
415 case NVPTXISD::Tex3DFloatFloatGrad:
416 return "NVPTXISD::Tex3DFloatFloatGrad";
417 case NVPTXISD::Tex3DI32I32: return "NVPTXISD::Tex3DI32I32";
418 case NVPTXISD::Tex3DI32Float: return "NVPTXISD::Tex3DI32Float";
419 case NVPTXISD::Tex3DI32FloatLevel:
420 return "NVPTXISD::Tex3DI32FloatLevel";
421 case NVPTXISD::Tex3DI32FloatGrad:
422 return "NVPTXISD::Tex3DI32FloatGrad";
423
424 case NVPTXISD::Suld1DI8Trap: return "NVPTXISD::Suld1DI8Trap";
425 case NVPTXISD::Suld1DI16Trap: return "NVPTXISD::Suld1DI16Trap";
426 case NVPTXISD::Suld1DI32Trap: return "NVPTXISD::Suld1DI32Trap";
427 case NVPTXISD::Suld1DV2I8Trap: return "NVPTXISD::Suld1DV2I8Trap";
428 case NVPTXISD::Suld1DV2I16Trap: return "NVPTXISD::Suld1DV2I16Trap";
429 case NVPTXISD::Suld1DV2I32Trap: return "NVPTXISD::Suld1DV2I32Trap";
430 case NVPTXISD::Suld1DV4I8Trap: return "NVPTXISD::Suld1DV4I8Trap";
431 case NVPTXISD::Suld1DV4I16Trap: return "NVPTXISD::Suld1DV4I16Trap";
432 case NVPTXISD::Suld1DV4I32Trap: return "NVPTXISD::Suld1DV4I32Trap";
433
434 case NVPTXISD::Suld1DArrayI8Trap: return "NVPTXISD::Suld1DArrayI8Trap";
435 case NVPTXISD::Suld1DArrayI16Trap: return "NVPTXISD::Suld1DArrayI16Trap";
436 case NVPTXISD::Suld1DArrayI32Trap: return "NVPTXISD::Suld1DArrayI32Trap";
437 case NVPTXISD::Suld1DArrayV2I8Trap: return "NVPTXISD::Suld1DArrayV2I8Trap";
438 case NVPTXISD::Suld1DArrayV2I16Trap: return "NVPTXISD::Suld1DArrayV2I16Trap";
439 case NVPTXISD::Suld1DArrayV2I32Trap: return "NVPTXISD::Suld1DArrayV2I32Trap";
440 case NVPTXISD::Suld1DArrayV4I8Trap: return "NVPTXISD::Suld1DArrayV4I8Trap";
441 case NVPTXISD::Suld1DArrayV4I16Trap: return "NVPTXISD::Suld1DArrayV4I16Trap";
442 case NVPTXISD::Suld1DArrayV4I32Trap: return "NVPTXISD::Suld1DArrayV4I32Trap";
443
444 case NVPTXISD::Suld2DI8Trap: return "NVPTXISD::Suld2DI8Trap";
445 case NVPTXISD::Suld2DI16Trap: return "NVPTXISD::Suld2DI16Trap";
446 case NVPTXISD::Suld2DI32Trap: return "NVPTXISD::Suld2DI32Trap";
447 case NVPTXISD::Suld2DV2I8Trap: return "NVPTXISD::Suld2DV2I8Trap";
448 case NVPTXISD::Suld2DV2I16Trap: return "NVPTXISD::Suld2DV2I16Trap";
449 case NVPTXISD::Suld2DV2I32Trap: return "NVPTXISD::Suld2DV2I32Trap";
450 case NVPTXISD::Suld2DV4I8Trap: return "NVPTXISD::Suld2DV4I8Trap";
451 case NVPTXISD::Suld2DV4I16Trap: return "NVPTXISD::Suld2DV4I16Trap";
452 case NVPTXISD::Suld2DV4I32Trap: return "NVPTXISD::Suld2DV4I32Trap";
453
454 case NVPTXISD::Suld2DArrayI8Trap: return "NVPTXISD::Suld2DArrayI8Trap";
455 case NVPTXISD::Suld2DArrayI16Trap: return "NVPTXISD::Suld2DArrayI16Trap";
456 case NVPTXISD::Suld2DArrayI32Trap: return "NVPTXISD::Suld2DArrayI32Trap";
457 case NVPTXISD::Suld2DArrayV2I8Trap: return "NVPTXISD::Suld2DArrayV2I8Trap";
458 case NVPTXISD::Suld2DArrayV2I16Trap: return "NVPTXISD::Suld2DArrayV2I16Trap";
459 case NVPTXISD::Suld2DArrayV2I32Trap: return "NVPTXISD::Suld2DArrayV2I32Trap";
460 case NVPTXISD::Suld2DArrayV4I8Trap: return "NVPTXISD::Suld2DArrayV4I8Trap";
461 case NVPTXISD::Suld2DArrayV4I16Trap: return "NVPTXISD::Suld2DArrayV4I16Trap";
462 case NVPTXISD::Suld2DArrayV4I32Trap: return "NVPTXISD::Suld2DArrayV4I32Trap";
463
464 case NVPTXISD::Suld3DI8Trap: return "NVPTXISD::Suld3DI8Trap";
465 case NVPTXISD::Suld3DI16Trap: return "NVPTXISD::Suld3DI16Trap";
466 case NVPTXISD::Suld3DI32Trap: return "NVPTXISD::Suld3DI32Trap";
467 case NVPTXISD::Suld3DV2I8Trap: return "NVPTXISD::Suld3DV2I8Trap";
468 case NVPTXISD::Suld3DV2I16Trap: return "NVPTXISD::Suld3DV2I16Trap";
469 case NVPTXISD::Suld3DV2I32Trap: return "NVPTXISD::Suld3DV2I32Trap";
470 case NVPTXISD::Suld3DV4I8Trap: return "NVPTXISD::Suld3DV4I8Trap";
471 case NVPTXISD::Suld3DV4I16Trap: return "NVPTXISD::Suld3DV4I16Trap";
472 case NVPTXISD::Suld3DV4I32Trap: return "NVPTXISD::Suld3DV4I32Trap";
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]473 }
474}
475
Chandler Carruth9d010ff2014-07-03 00:23:43 +0000[diff] [blame]476TargetLoweringBase::LegalizeTypeAction
477NVPTXTargetLowering::getPreferredVectorAction(EVT VT) const {
478 if (VT.getVectorNumElements() != 1 && VT.getScalarType() == MVT::i1)
479 return TypeSplitVector;
480
481 return TargetLoweringBase::getPreferredVectorAction(VT);
Justin Holewinskibc451192012-11-29 14:26:24 +0000[diff] [blame]482}
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]483
484SDValue
485NVPTXTargetLowering::LowerGlobalAddress(SDValue Op, SelectionDAG &DAG) const {
Andrew Trickef9de2a2013-05-25 02:42:55 +0000[diff] [blame]486 SDLoc dl(Op);
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]487 const GlobalValue *GV = cast<GlobalAddressSDNode>(Op)->getGlobal();
488 Op = DAG.getTargetGlobalAddress(GV, dl, getPointerTy());
489 return DAG.getNode(NVPTXISD::Wrapper, dl, getPointerTy(), Op);
490}
491
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]492std::string
493NVPTXTargetLowering::getPrototype(Type *retTy, const ArgListTy &Args,
494 const SmallVectorImpl<ISD::OutputArg> &Outs,
495 unsigned retAlignment,
496 const ImmutableCallSite *CS) const {
497
498 bool isABI = (nvptxSubtarget.getSmVersion() >= 20);
499 assert(isABI && "Non-ABI compilation is not supported");
500 if (!isABI)
501 return "";
502
503 std::stringstream O;
504 O << "prototype_" << uniqueCallSite << " : .callprototype ";
505
506 if (retTy->getTypeID() == Type::VoidTyID) {
507 O << "()";
508 } else {
509 O << "(";
Rafael Espindola08013342013-12-07 19:34:20 +0000[diff] [blame]510 if (retTy->isFloatingPointTy() || retTy->isIntegerTy()) {
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]511 unsigned size = 0;
512 if (const IntegerType *ITy = dyn_cast<IntegerType>(retTy)) {
513 size = ITy->getBitWidth();
514 if (size < 32)
515 size = 32;
516 } else {
517 assert(retTy->isFloatingPointTy() &&
518 "Floating point type expected here");
519 size = retTy->getPrimitiveSizeInBits();
520 }
521
522 O << ".param .b" << size << " _";
523 } else if (isa<PointerType>(retTy)) {
524 O << ".param .b" << getPointerTy().getSizeInBits() << " _";
525 } else {
Justin Holewinski6e40f632014-06-27 18:35:44 +0000[diff] [blame]526 if((retTy->getTypeID() == Type::StructTyID) ||
527 isa<VectorType>(retTy)) {
528 O << ".param .align "
529 << retAlignment
530 << " .b8 _["
531 << getDataLayout()->getTypeAllocSize(retTy) << "]";
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]532 } else {
533 assert(false && "Unknown return type");
534 }
535 }
536 O << ") ";
537 }
538 O << "_ (";
539
540 bool first = true;
541 MVT thePointerTy = getPointerTy();
542
543 unsigned OIdx = 0;
544 for (unsigned i = 0, e = Args.size(); i != e; ++i, ++OIdx) {
545 Type *Ty = Args[i].Ty;
546 if (!first) {
547 O << ", ";
548 }
549 first = false;
550
551 if (Outs[OIdx].Flags.isByVal() == false) {
552 if (Ty->isAggregateType() || Ty->isVectorTy()) {
553 unsigned align = 0;
554 const CallInst *CallI = cast<CallInst>(CS->getInstruction());
555 const DataLayout *TD = getDataLayout();
556 // +1 because index 0 is reserved for return type alignment
557 if (!llvm::getAlign(*CallI, i + 1, align))
558 align = TD->getABITypeAlignment(Ty);
559 unsigned sz = TD->getTypeAllocSize(Ty);
560 O << ".param .align " << align << " .b8 ";
561 O << "_";
562 O << "[" << sz << "]";
563 // update the index for Outs
564 SmallVector<EVT, 16> vtparts;
565 ComputeValueVTs(*this, Ty, vtparts);
566 if (unsigned len = vtparts.size())
567 OIdx += len - 1;
568 continue;
569 }
Justin Holewinskidff28d22013-07-01 12:59:01 +0000[diff] [blame]570 // i8 types in IR will be i16 types in SDAG
571 assert((getValueType(Ty) == Outs[OIdx].VT ||
572 (getValueType(Ty) == MVT::i8 && Outs[OIdx].VT == MVT::i16)) &&
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]573 "type mismatch between callee prototype and arguments");
574 // scalar type
575 unsigned sz = 0;
576 if (isa<IntegerType>(Ty)) {
577 sz = cast<IntegerType>(Ty)->getBitWidth();
578 if (sz < 32)
579 sz = 32;
580 } else if (isa<PointerType>(Ty))
581 sz = thePointerTy.getSizeInBits();
582 else
583 sz = Ty->getPrimitiveSizeInBits();
584 O << ".param .b" << sz << " ";
585 O << "_";
586 continue;
587 }
588 const PointerType *PTy = dyn_cast<PointerType>(Ty);
589 assert(PTy && "Param with byval attribute should be a pointer type");
590 Type *ETy = PTy->getElementType();
591
592 unsigned align = Outs[OIdx].Flags.getByValAlign();
593 unsigned sz = getDataLayout()->getTypeAllocSize(ETy);
594 O << ".param .align " << align << " .b8 ";
595 O << "_";
596 O << "[" << sz << "]";
597 }
598 O << ");";
599 return O.str();
600}
601
602unsigned
603NVPTXTargetLowering::getArgumentAlignment(SDValue Callee,
604 const ImmutableCallSite *CS,
605 Type *Ty,
606 unsigned Idx) const {
607 const DataLayout *TD = getDataLayout();
Justin Holewinski124e93d2013-11-11 19:28:19 +0000[diff] [blame]608 unsigned Align = 0;
609 const Value *DirectCallee = CS->getCalledFunction();
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]610
Justin Holewinski124e93d2013-11-11 19:28:19 +0000[diff] [blame]611 if (!DirectCallee) {
612 // We don't have a direct function symbol, but that may be because of
613 // constant cast instructions in the call.
614 const Instruction *CalleeI = CS->getInstruction();
615 assert(CalleeI && "Call target is not a function or derived value?");
616
617 // With bitcast'd call targets, the instruction will be the call
618 if (isa<CallInst>(CalleeI)) {
619 // Check if we have call alignment metadata
620 if (llvm::getAlign(*cast<CallInst>(CalleeI), Idx, Align))
621 return Align;
622
623 const Value *CalleeV = cast<CallInst>(CalleeI)->getCalledValue();
624 // Ignore any bitcast instructions
625 while(isa<ConstantExpr>(CalleeV)) {
626 const ConstantExpr *CE = cast<ConstantExpr>(CalleeV);
627 if (!CE->isCast())
628 break;
629 // Look through the bitcast
630 CalleeV = cast<ConstantExpr>(CalleeV)->getOperand(0);
631 }
632
633 // We have now looked past all of the bitcasts. Do we finally have a
634 // Function?
635 if (isa<Function>(CalleeV))
636 DirectCallee = CalleeV;
637 }
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]638 }
639
Justin Holewinski124e93d2013-11-11 19:28:19 +0000[diff] [blame]640 // Check for function alignment information if we found that the
641 // ultimate target is a Function
642 if (DirectCallee)
643 if (llvm::getAlign(*cast<Function>(DirectCallee), Idx, Align))
644 return Align;
645
646 // Call is indirect or alignment information is not available, fall back to
647 // the ABI type alignment
648 return TD->getABITypeAlignment(Ty);
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]649}
650
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]651SDValue NVPTXTargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
652 SmallVectorImpl<SDValue> &InVals) const {
653 SelectionDAG &DAG = CLI.DAG;
Andrew Trickef9de2a2013-05-25 02:42:55 +0000[diff] [blame]654 SDLoc dl = CLI.DL;
Craig Topperb94011f2013-07-14 04:42:23 +0000[diff] [blame]655 SmallVectorImpl<ISD::OutputArg> &Outs = CLI.Outs;
656 SmallVectorImpl<SDValue> &OutVals = CLI.OutVals;
657 SmallVectorImpl<ISD::InputArg> &Ins = CLI.Ins;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]658 SDValue Chain = CLI.Chain;
659 SDValue Callee = CLI.Callee;
660 bool &isTailCall = CLI.IsTailCall;
Saleem Abdulrasool9f664c12014-05-17 21:50:01 +0000[diff] [blame]661 ArgListTy &Args = CLI.getArgs();
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]662 Type *retTy = CLI.RetTy;
663 ImmutableCallSite *CS = CLI.CS;
Justin Holewinskiaa583972012-05-25 16:35:28 +0000[diff] [blame]664
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]665 bool isABI = (nvptxSubtarget.getSmVersion() >= 20);
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]666 assert(isABI && "Non-ABI compilation is not supported");
667 if (!isABI)
668 return Chain;
669 const DataLayout *TD = getDataLayout();
670 MachineFunction &MF = DAG.getMachineFunction();
671 const Function *F = MF.getFunction();
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]672
673 SDValue tempChain = Chain;
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]674 Chain =
675 DAG.getCALLSEQ_START(Chain, DAG.getIntPtrConstant(uniqueCallSite, true),
676 dl);
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]677 SDValue InFlag = Chain.getValue(1);
678
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]679 unsigned paramCount = 0;
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]680 // Args.size() and Outs.size() need not match.
681 // Outs.size() will be larger
682 // * if there is an aggregate argument with multiple fields (each field
683 // showing up separately in Outs)
684 // * if there is a vector argument with more than typical vector-length
685 // elements (generally if more than 4) where each vector element is
686 // individually present in Outs.
687 // So a different index should be used for indexing into Outs/OutVals.
688 // See similar issue in LowerFormalArguments.
689 unsigned OIdx = 0;
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]690 // Declare the .params or .reg need to pass values
691 // to the function
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]692 for (unsigned i = 0, e = Args.size(); i != e; ++i, ++OIdx) {
693 EVT VT = Outs[OIdx].VT;
694 Type *Ty = Args[i].Ty;
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]695
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]696 if (Outs[OIdx].Flags.isByVal() == false) {
697 if (Ty->isAggregateType()) {
698 // aggregate
699 SmallVector<EVT, 16> vtparts;
Justin Holewinski6e40f632014-06-27 18:35:44 +0000[diff] [blame]700 SmallVector<uint64_t, 16> Offsets;
701 ComputePTXValueVTs(*this, Ty, vtparts, &Offsets, 0);
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]702
703 unsigned align = getArgumentAlignment(Callee, CS, Ty, paramCount + 1);
704 // declare .param .align <align> .b8 .param<n>[<size>];
705 unsigned sz = TD->getTypeAllocSize(Ty);
706 SDVTList DeclareParamVTs = DAG.getVTList(MVT::Other, MVT::Glue);
707 SDValue DeclareParamOps[] = { Chain, DAG.getConstant(align, MVT::i32),
708 DAG.getConstant(paramCount, MVT::i32),
709 DAG.getConstant(sz, MVT::i32), InFlag };
710 Chain = DAG.getNode(NVPTXISD::DeclareParam, dl, DeclareParamVTs,
Craig Topper48d114b2014-04-26 18:35:24 +0000[diff] [blame]711 DeclareParamOps);
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]712 InFlag = Chain.getValue(1);
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]713 for (unsigned j = 0, je = vtparts.size(); j != je; ++j) {
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]714 EVT elemtype = vtparts[j];
Justin Holewinski9982f062014-06-27 19:36:25 +0000[diff] [blame]715 unsigned ArgAlign = GreatestCommonDivisor64(align, Offsets[j]);
Justin Holewinski6e40f632014-06-27 18:35:44 +0000[diff] [blame]716 if (elemtype.isInteger() && (sz < 8))
717 sz = 8;
718 SDValue StVal = OutVals[OIdx];
719 if (elemtype.getSizeInBits() < 16) {
720 StVal = DAG.getNode(ISD::ANY_EXTEND, dl, MVT::i16, StVal);
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]721 }
Justin Holewinski6e40f632014-06-27 18:35:44 +0000[diff] [blame]722 SDVTList CopyParamVTs = DAG.getVTList(MVT::Other, MVT::Glue);
723 SDValue CopyParamOps[] = { Chain,
724 DAG.getConstant(paramCount, MVT::i32),
725 DAG.getConstant(Offsets[j], MVT::i32),
726 StVal, InFlag };
727 Chain = DAG.getMemIntrinsicNode(NVPTXISD::StoreParam, dl,
728 CopyParamVTs, CopyParamOps,
729 elemtype, MachinePointerInfo(),
730 ArgAlign);
731 InFlag = Chain.getValue(1);
732 ++OIdx;
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]733 }
734 if (vtparts.size() > 0)
735 --OIdx;
736 ++paramCount;
737 continue;
738 }
739 if (Ty->isVectorTy()) {
740 EVT ObjectVT = getValueType(Ty);
741 unsigned align = getArgumentAlignment(Callee, CS, Ty, paramCount + 1);
742 // declare .param .align <align> .b8 .param<n>[<size>];
743 unsigned sz = TD->getTypeAllocSize(Ty);
744 SDVTList DeclareParamVTs = DAG.getVTList(MVT::Other, MVT::Glue);
745 SDValue DeclareParamOps[] = { Chain, DAG.getConstant(align, MVT::i32),
746 DAG.getConstant(paramCount, MVT::i32),
747 DAG.getConstant(sz, MVT::i32), InFlag };
748 Chain = DAG.getNode(NVPTXISD::DeclareParam, dl, DeclareParamVTs,
Craig Topper48d114b2014-04-26 18:35:24 +0000[diff] [blame]749 DeclareParamOps);
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]750 InFlag = Chain.getValue(1);
751 unsigned NumElts = ObjectVT.getVectorNumElements();
752 EVT EltVT = ObjectVT.getVectorElementType();
753 EVT MemVT = EltVT;
754 bool NeedExtend = false;
755 if (EltVT.getSizeInBits() < 16) {
756 NeedExtend = true;
757 EltVT = MVT::i16;
758 }
759
760 // V1 store
761 if (NumElts == 1) {
762 SDValue Elt = OutVals[OIdx++];
763 if (NeedExtend)
764 Elt = DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i16, Elt);
765
766 SDVTList CopyParamVTs = DAG.getVTList(MVT::Other, MVT::Glue);
767 SDValue CopyParamOps[] = { Chain,
768 DAG.getConstant(paramCount, MVT::i32),
769 DAG.getConstant(0, MVT::i32), Elt,
770 InFlag };
771 Chain = DAG.getMemIntrinsicNode(NVPTXISD::StoreParam, dl,
Craig Topper206fcd42014-04-26 19:29:41 +0000[diff] [blame]772 CopyParamVTs, CopyParamOps,
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]773 MemVT, MachinePointerInfo());
774 InFlag = Chain.getValue(1);
775 } else if (NumElts == 2) {
776 SDValue Elt0 = OutVals[OIdx++];
777 SDValue Elt1 = OutVals[OIdx++];
778 if (NeedExtend) {
779 Elt0 = DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i16, Elt0);
780 Elt1 = DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i16, Elt1);
781 }
782
783 SDVTList CopyParamVTs = DAG.getVTList(MVT::Other, MVT::Glue);
784 SDValue CopyParamOps[] = { Chain,
785 DAG.getConstant(paramCount, MVT::i32),
786 DAG.getConstant(0, MVT::i32), Elt0, Elt1,
787 InFlag };
788 Chain = DAG.getMemIntrinsicNode(NVPTXISD::StoreParamV2, dl,
Craig Topper206fcd42014-04-26 19:29:41 +0000[diff] [blame]789 CopyParamVTs, CopyParamOps,
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]790 MemVT, MachinePointerInfo());
791 InFlag = Chain.getValue(1);
792 } else {
793 unsigned curOffset = 0;
794 // V4 stores
795 // We have at least 4 elements (<3 x Ty> expands to 4 elements) and
796 // the
797 // vector will be expanded to a power of 2 elements, so we know we can
798 // always round up to the next multiple of 4 when creating the vector
799 // stores.
800 // e.g. 4 elem => 1 st.v4
801 // 6 elem => 2 st.v4
802 // 8 elem => 2 st.v4
803 // 11 elem => 3 st.v4
804 unsigned VecSize = 4;
805 if (EltVT.getSizeInBits() == 64)
806 VecSize = 2;
807
808 // This is potentially only part of a vector, so assume all elements
809 // are packed together.
810 unsigned PerStoreOffset = MemVT.getStoreSizeInBits() / 8 * VecSize;
811
812 for (unsigned i = 0; i < NumElts; i += VecSize) {
813 // Get values
814 SDValue StoreVal;
815 SmallVector<SDValue, 8> Ops;
816 Ops.push_back(Chain);
817 Ops.push_back(DAG.getConstant(paramCount, MVT::i32));
818 Ops.push_back(DAG.getConstant(curOffset, MVT::i32));
819
820 unsigned Opc = NVPTXISD::StoreParamV2;
821
822 StoreVal = OutVals[OIdx++];
823 if (NeedExtend)
824 StoreVal = DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i16, StoreVal);
825 Ops.push_back(StoreVal);
826
827 if (i + 1 < NumElts) {
828 StoreVal = OutVals[OIdx++];
829 if (NeedExtend)
830 StoreVal =
831 DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i16, StoreVal);
832 } else {
833 StoreVal = DAG.getUNDEF(EltVT);
834 }
835 Ops.push_back(StoreVal);
836
837 if (VecSize == 4) {
838 Opc = NVPTXISD::StoreParamV4;
839 if (i + 2 < NumElts) {
840 StoreVal = OutVals[OIdx++];
841 if (NeedExtend)
842 StoreVal =
843 DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i16, StoreVal);
844 } else {
845 StoreVal = DAG.getUNDEF(EltVT);
846 }
847 Ops.push_back(StoreVal);
848
849 if (i + 3 < NumElts) {
850 StoreVal = OutVals[OIdx++];
851 if (NeedExtend)
852 StoreVal =
853 DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i16, StoreVal);
854 } else {
855 StoreVal = DAG.getUNDEF(EltVT);
856 }
857 Ops.push_back(StoreVal);
858 }
859
Justin Holewinskidff28d22013-07-01 12:59:01 +0000[diff] [blame]860 Ops.push_back(InFlag);
861
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]862 SDVTList CopyParamVTs = DAG.getVTList(MVT::Other, MVT::Glue);
Craig Topper206fcd42014-04-26 19:29:41 +0000[diff] [blame]863 Chain = DAG.getMemIntrinsicNode(Opc, dl, CopyParamVTs, Ops,
864 MemVT, MachinePointerInfo());
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]865 InFlag = Chain.getValue(1);
866 curOffset += PerStoreOffset;
867 }
868 }
869 ++paramCount;
870 --OIdx;
871 continue;
872 }
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]873 // Plain scalar
874 // for ABI, declare .param .b<size> .param<n>;
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]875 unsigned sz = VT.getSizeInBits();
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]876 bool needExtend = false;
877 if (VT.isInteger()) {
878 if (sz < 16)
879 needExtend = true;
880 if (sz < 32)
881 sz = 32;
882 }
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]883 SDVTList DeclareParamVTs = DAG.getVTList(MVT::Other, MVT::Glue);
884 SDValue DeclareParamOps[] = { Chain,
885 DAG.getConstant(paramCount, MVT::i32),
886 DAG.getConstant(sz, MVT::i32),
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]887 DAG.getConstant(0, MVT::i32), InFlag };
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]888 Chain = DAG.getNode(NVPTXISD::DeclareScalarParam, dl, DeclareParamVTs,
Craig Topper48d114b2014-04-26 18:35:24 +0000[diff] [blame]889 DeclareParamOps);
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]890 InFlag = Chain.getValue(1);
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]891 SDValue OutV = OutVals[OIdx];
892 if (needExtend) {
893 // zext/sext i1 to i16
894 unsigned opc = ISD::ZERO_EXTEND;
895 if (Outs[OIdx].Flags.isSExt())
896 opc = ISD::SIGN_EXTEND;
897 OutV = DAG.getNode(opc, dl, MVT::i16, OutV);
898 }
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]899 SDVTList CopyParamVTs = DAG.getVTList(MVT::Other, MVT::Glue);
900 SDValue CopyParamOps[] = { Chain, DAG.getConstant(paramCount, MVT::i32),
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]901 DAG.getConstant(0, MVT::i32), OutV, InFlag };
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]902
903 unsigned opcode = NVPTXISD::StoreParam;
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]904 if (Outs[OIdx].Flags.isZExt())
905 opcode = NVPTXISD::StoreParamU32;
906 else if (Outs[OIdx].Flags.isSExt())
907 opcode = NVPTXISD::StoreParamS32;
Craig Topper206fcd42014-04-26 19:29:41 +0000[diff] [blame]908 Chain = DAG.getMemIntrinsicNode(opcode, dl, CopyParamVTs, CopyParamOps,
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]909 VT, MachinePointerInfo());
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]910
911 InFlag = Chain.getValue(1);
912 ++paramCount;
913 continue;
914 }
915 // struct or vector
916 SmallVector<EVT, 16> vtparts;
Justin Holewinski6e40f632014-06-27 18:35:44 +0000[diff] [blame]917 SmallVector<uint64_t, 16> Offsets;
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]918 const PointerType *PTy = dyn_cast<PointerType>(Args[i].Ty);
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]919 assert(PTy && "Type of a byval parameter should be pointer");
Justin Holewinski6e40f632014-06-27 18:35:44 +0000[diff] [blame]920 ComputePTXValueVTs(*this, PTy->getElementType(), vtparts, &Offsets, 0);
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]921
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]922 // declare .param .align <align> .b8 .param<n>[<size>];
923 unsigned sz = Outs[OIdx].Flags.getByValSize();
924 SDVTList DeclareParamVTs = DAG.getVTList(MVT::Other, MVT::Glue);
Justin Holewinski6e40f632014-06-27 18:35:44 +0000[diff] [blame]925 unsigned ArgAlign = Outs[OIdx].Flags.getByValAlign();
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]926 // The ByValAlign in the Outs[OIdx].Flags is alway set at this point,
927 // so we don't need to worry about natural alignment or not.
928 // See TargetLowering::LowerCallTo().
929 SDValue DeclareParamOps[] = {
930 Chain, DAG.getConstant(Outs[OIdx].Flags.getByValAlign(), MVT::i32),
931 DAG.getConstant(paramCount, MVT::i32), DAG.getConstant(sz, MVT::i32),
932 InFlag
933 };
934 Chain = DAG.getNode(NVPTXISD::DeclareParam, dl, DeclareParamVTs,
Craig Topper48d114b2014-04-26 18:35:24 +0000[diff] [blame]935 DeclareParamOps);
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]936 InFlag = Chain.getValue(1);
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]937 for (unsigned j = 0, je = vtparts.size(); j != je; ++j) {
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]938 EVT elemtype = vtparts[j];
Justin Holewinski6e40f632014-06-27 18:35:44 +0000[diff] [blame]939 int curOffset = Offsets[j];
Justin Holewinski9982f062014-06-27 19:36:25 +0000[diff] [blame]940 unsigned PartAlign = GreatestCommonDivisor64(ArgAlign, curOffset);
Justin Holewinski6e40f632014-06-27 18:35:44 +0000[diff] [blame]941 SDValue srcAddr =
942 DAG.getNode(ISD::ADD, dl, getPointerTy(), OutVals[OIdx],
943 DAG.getConstant(curOffset, getPointerTy()));
944 SDValue theVal = DAG.getLoad(elemtype, dl, tempChain, srcAddr,
945 MachinePointerInfo(), false, false, false,
946 PartAlign);
947 if (elemtype.getSizeInBits() < 16) {
948 theVal = DAG.getNode(ISD::ANY_EXTEND, dl, MVT::i16, theVal);
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]949 }
Justin Holewinski6e40f632014-06-27 18:35:44 +0000[diff] [blame]950 SDVTList CopyParamVTs = DAG.getVTList(MVT::Other, MVT::Glue);
951 SDValue CopyParamOps[] = { Chain, DAG.getConstant(paramCount, MVT::i32),
952 DAG.getConstant(curOffset, MVT::i32), theVal,
953 InFlag };
954 Chain = DAG.getMemIntrinsicNode(NVPTXISD::StoreParam, dl, CopyParamVTs,
955 CopyParamOps, elemtype,
956 MachinePointerInfo());
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]957
Justin Holewinski6e40f632014-06-27 18:35:44 +0000[diff] [blame]958 InFlag = Chain.getValue(1);
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]959 }
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]960 ++paramCount;
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]961 }
962
963 GlobalAddressSDNode *Func = dyn_cast<GlobalAddressSDNode>(Callee.getNode());
964 unsigned retAlignment = 0;
965
966 // Handle Result
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]967 if (Ins.size() > 0) {
968 SmallVector<EVT, 16> resvtparts;
969 ComputeValueVTs(*this, retTy, resvtparts);
970
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]971 // Declare
972 // .param .align 16 .b8 retval0[<size-in-bytes>], or
973 // .param .b<size-in-bits> retval0
974 unsigned resultsz = TD->getTypeAllocSizeInBits(retTy);
Rafael Espindola08013342013-12-07 19:34:20 +0000[diff] [blame]975 if (retTy->isSingleValueType()) {
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]976 // Scalar needs to be at least 32bit wide
977 if (resultsz < 32)
978 resultsz = 32;
979 SDVTList DeclareRetVTs = DAG.getVTList(MVT::Other, MVT::Glue);
980 SDValue DeclareRetOps[] = { Chain, DAG.getConstant(1, MVT::i32),
981 DAG.getConstant(resultsz, MVT::i32),
982 DAG.getConstant(0, MVT::i32), InFlag };
983 Chain = DAG.getNode(NVPTXISD::DeclareRet, dl, DeclareRetVTs,
Craig Topper48d114b2014-04-26 18:35:24 +0000[diff] [blame]984 DeclareRetOps);
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]985 InFlag = Chain.getValue(1);
986 } else {
987 retAlignment = getArgumentAlignment(Callee, CS, retTy, 0);
988 SDVTList DeclareRetVTs = DAG.getVTList(MVT::Other, MVT::Glue);
989 SDValue DeclareRetOps[] = { Chain,
990 DAG.getConstant(retAlignment, MVT::i32),
991 DAG.getConstant(resultsz / 8, MVT::i32),
992 DAG.getConstant(0, MVT::i32), InFlag };
993 Chain = DAG.getNode(NVPTXISD::DeclareRetParam, dl, DeclareRetVTs,
Craig Topper48d114b2014-04-26 18:35:24 +0000[diff] [blame]994 DeclareRetOps);
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]995 InFlag = Chain.getValue(1);
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]996 }
997 }
998
999 if (!Func) {
1000 // This is indirect function call case : PTX requires a prototype of the
1001 // form
1002 // proto_0 : .callprototype(.param .b32 _) _ (.param .b32 _);
1003 // to be emitted, and the label has to used as the last arg of call
1004 // instruction.
Justin Holewinski3d49e5c2013-11-15 12:30:04 +0000[diff] [blame]1005 // The prototype is embedded in a string and put as the operand for a
1006 // CallPrototype SDNode which will print out to the value of the string.
1007 SDVTList ProtoVTs = DAG.getVTList(MVT::Other, MVT::Glue);
1008 std::string Proto = getPrototype(retTy, Args, Outs, retAlignment, CS);
1009 const char *ProtoStr =
1010 nvTM->getManagedStrPool()->getManagedString(Proto.c_str())->c_str();
1011 SDValue ProtoOps[] = {
1012 Chain, DAG.getTargetExternalSymbol(ProtoStr, MVT::i32), InFlag,
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1013 };
Craig Topper48d114b2014-04-26 18:35:24 +0000[diff] [blame]1014 Chain = DAG.getNode(NVPTXISD::CallPrototype, dl, ProtoVTs, ProtoOps);
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]1015 InFlag = Chain.getValue(1);
1016 }
1017 // Op to just print "call"
1018 SDVTList PrintCallVTs = DAG.getVTList(MVT::Other, MVT::Glue);
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1019 SDValue PrintCallOps[] = {
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]1020 Chain, DAG.getConstant((Ins.size() == 0) ? 0 : 1, MVT::i32), InFlag
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1021 };
1022 Chain = DAG.getNode(Func ? (NVPTXISD::PrintCallUni) : (NVPTXISD::PrintCall),
Craig Topper48d114b2014-04-26 18:35:24 +0000[diff] [blame]1023 dl, PrintCallVTs, PrintCallOps);
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]1024 InFlag = Chain.getValue(1);
1025
1026 // Ops to print out the function name
1027 SDVTList CallVoidVTs = DAG.getVTList(MVT::Other, MVT::Glue);
1028 SDValue CallVoidOps[] = { Chain, Callee, InFlag };
Craig Topper48d114b2014-04-26 18:35:24 +0000[diff] [blame]1029 Chain = DAG.getNode(NVPTXISD::CallVoid, dl, CallVoidVTs, CallVoidOps);
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]1030 InFlag = Chain.getValue(1);
1031
1032 // Ops to print out the param list
1033 SDVTList CallArgBeginVTs = DAG.getVTList(MVT::Other, MVT::Glue);
1034 SDValue CallArgBeginOps[] = { Chain, InFlag };
1035 Chain = DAG.getNode(NVPTXISD::CallArgBegin, dl, CallArgBeginVTs,
Craig Topper48d114b2014-04-26 18:35:24 +0000[diff] [blame]1036 CallArgBeginOps);
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]1037 InFlag = Chain.getValue(1);
1038
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1039 for (unsigned i = 0, e = paramCount; i != e; ++i) {
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]1040 unsigned opcode;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1041 if (i == (e - 1))
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]1042 opcode = NVPTXISD::LastCallArg;
1043 else
1044 opcode = NVPTXISD::CallArg;
1045 SDVTList CallArgVTs = DAG.getVTList(MVT::Other, MVT::Glue);
1046 SDValue CallArgOps[] = { Chain, DAG.getConstant(1, MVT::i32),
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1047 DAG.getConstant(i, MVT::i32), InFlag };
Craig Topper48d114b2014-04-26 18:35:24 +0000[diff] [blame]1048 Chain = DAG.getNode(opcode, dl, CallArgVTs, CallArgOps);
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]1049 InFlag = Chain.getValue(1);
1050 }
1051 SDVTList CallArgEndVTs = DAG.getVTList(MVT::Other, MVT::Glue);
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1052 SDValue CallArgEndOps[] = { Chain, DAG.getConstant(Func ? 1 : 0, MVT::i32),
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]1053 InFlag };
Craig Topper48d114b2014-04-26 18:35:24 +0000[diff] [blame]1054 Chain = DAG.getNode(NVPTXISD::CallArgEnd, dl, CallArgEndVTs, CallArgEndOps);
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]1055 InFlag = Chain.getValue(1);
1056
1057 if (!Func) {
1058 SDVTList PrototypeVTs = DAG.getVTList(MVT::Other, MVT::Glue);
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1059 SDValue PrototypeOps[] = { Chain, DAG.getConstant(uniqueCallSite, MVT::i32),
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]1060 InFlag };
Craig Topper48d114b2014-04-26 18:35:24 +0000[diff] [blame]1061 Chain = DAG.getNode(NVPTXISD::Prototype, dl, PrototypeVTs, PrototypeOps);
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]1062 InFlag = Chain.getValue(1);
1063 }
1064
1065 // Generate loads from param memory/moves from registers for result
1066 if (Ins.size() > 0) {
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]1067 if (retTy && retTy->isVectorTy()) {
1068 EVT ObjectVT = getValueType(retTy);
1069 unsigned NumElts = ObjectVT.getVectorNumElements();
1070 EVT EltVT = ObjectVT.getVectorElementType();
Benjamin Kramer3cc579a2013-06-29 22:51:12 +0000[diff] [blame]1071 assert(nvTM->getTargetLowering()->getNumRegisters(F->getContext(),
1072 ObjectVT) == NumElts &&
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]1073 "Vector was not scalarized");
1074 unsigned sz = EltVT.getSizeInBits();
Justin Holewinski6e40f632014-06-27 18:35:44 +0000[diff] [blame]1075 bool needTruncate = sz < 8 ? true : false;
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]1076
1077 if (NumElts == 1) {
1078 // Just a simple load
Craig Topper59f626d2014-04-26 19:29:47 +0000[diff] [blame]1079 SmallVector<EVT, 4> LoadRetVTs;
Justin Holewinski6e40f632014-06-27 18:35:44 +0000[diff] [blame]1080 if (EltVT == MVT::i1 || EltVT == MVT::i8) {
1081 // If loading i1/i8 result, generate
1082 // load.b8 i16
1083 // if i1
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]1084 // trunc i16 to i1
1085 LoadRetVTs.push_back(MVT::i16);
1086 } else
1087 LoadRetVTs.push_back(EltVT);
1088 LoadRetVTs.push_back(MVT::Other);
1089 LoadRetVTs.push_back(MVT::Glue);
Craig Topper59f626d2014-04-26 19:29:47 +0000[diff] [blame]1090 SmallVector<SDValue, 4> LoadRetOps;
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]1091 LoadRetOps.push_back(Chain);
1092 LoadRetOps.push_back(DAG.getConstant(1, MVT::i32));
1093 LoadRetOps.push_back(DAG.getConstant(0, MVT::i32));
1094 LoadRetOps.push_back(InFlag);
1095 SDValue retval = DAG.getMemIntrinsicNode(
1096 NVPTXISD::LoadParam, dl,
Craig Topper206fcd42014-04-26 19:29:41 +0000[diff] [blame]1097 DAG.getVTList(LoadRetVTs), LoadRetOps, EltVT, MachinePointerInfo());
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]1098 Chain = retval.getValue(1);
1099 InFlag = retval.getValue(2);
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]1100 SDValue Ret0 = retval;
1101 if (needTruncate)
1102 Ret0 = DAG.getNode(ISD::TRUNCATE, dl, EltVT, Ret0);
1103 InVals.push_back(Ret0);
1104 } else if (NumElts == 2) {
1105 // LoadV2
Craig Topper59f626d2014-04-26 19:29:47 +0000[diff] [blame]1106 SmallVector<EVT, 4> LoadRetVTs;
Justin Holewinski6e40f632014-06-27 18:35:44 +0000[diff] [blame]1107 if (EltVT == MVT::i1 || EltVT == MVT::i8) {
1108 // If loading i1/i8 result, generate
1109 // load.b8 i16
1110 // if i1
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]1111 // trunc i16 to i1
1112 LoadRetVTs.push_back(MVT::i16);
1113 LoadRetVTs.push_back(MVT::i16);
1114 } else {
1115 LoadRetVTs.push_back(EltVT);
1116 LoadRetVTs.push_back(EltVT);
1117 }
1118 LoadRetVTs.push_back(MVT::Other);
1119 LoadRetVTs.push_back(MVT::Glue);
Craig Topper59f626d2014-04-26 19:29:47 +0000[diff] [blame]1120 SmallVector<SDValue, 4> LoadRetOps;
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]1121 LoadRetOps.push_back(Chain);
1122 LoadRetOps.push_back(DAG.getConstant(1, MVT::i32));
1123 LoadRetOps.push_back(DAG.getConstant(0, MVT::i32));
1124 LoadRetOps.push_back(InFlag);
1125 SDValue retval = DAG.getMemIntrinsicNode(
1126 NVPTXISD::LoadParamV2, dl,
Craig Topper206fcd42014-04-26 19:29:41 +0000[diff] [blame]1127 DAG.getVTList(LoadRetVTs), LoadRetOps, EltVT, MachinePointerInfo());
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]1128 Chain = retval.getValue(2);
1129 InFlag = retval.getValue(3);
1130 SDValue Ret0 = retval.getValue(0);
1131 SDValue Ret1 = retval.getValue(1);
1132 if (needTruncate) {
1133 Ret0 = DAG.getNode(ISD::TRUNCATE, dl, MVT::i1, Ret0);
1134 InVals.push_back(Ret0);
1135 Ret1 = DAG.getNode(ISD::TRUNCATE, dl, MVT::i1, Ret1);
1136 InVals.push_back(Ret1);
1137 } else {
1138 InVals.push_back(Ret0);
1139 InVals.push_back(Ret1);
1140 }
1141 } else {
1142 // Split into N LoadV4
1143 unsigned Ofst = 0;
1144 unsigned VecSize = 4;
1145 unsigned Opc = NVPTXISD::LoadParamV4;
1146 if (EltVT.getSizeInBits() == 64) {
1147 VecSize = 2;
1148 Opc = NVPTXISD::LoadParamV2;
1149 }
1150 EVT VecVT = EVT::getVectorVT(F->getContext(), EltVT, VecSize);
1151 for (unsigned i = 0; i < NumElts; i += VecSize) {
1152 SmallVector<EVT, 8> LoadRetVTs;
Justin Holewinski6e40f632014-06-27 18:35:44 +0000[diff] [blame]1153 if (EltVT == MVT::i1 || EltVT == MVT::i8) {
1154 // If loading i1/i8 result, generate
1155 // load.b8 i16
1156 // if i1
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]1157 // trunc i16 to i1
1158 for (unsigned j = 0; j < VecSize; ++j)
1159 LoadRetVTs.push_back(MVT::i16);
1160 } else {
1161 for (unsigned j = 0; j < VecSize; ++j)
1162 LoadRetVTs.push_back(EltVT);
1163 }
1164 LoadRetVTs.push_back(MVT::Other);
1165 LoadRetVTs.push_back(MVT::Glue);
1166 SmallVector<SDValue, 4> LoadRetOps;
1167 LoadRetOps.push_back(Chain);
1168 LoadRetOps.push_back(DAG.getConstant(1, MVT::i32));
1169 LoadRetOps.push_back(DAG.getConstant(Ofst, MVT::i32));
1170 LoadRetOps.push_back(InFlag);
1171 SDValue retval = DAG.getMemIntrinsicNode(
Craig Topperabb4ac72014-04-16 06:10:51 +0000[diff] [blame]1172 Opc, dl, DAG.getVTList(LoadRetVTs),
Craig Topper206fcd42014-04-26 19:29:41 +0000[diff] [blame]1173 LoadRetOps, EltVT, MachinePointerInfo());
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]1174 if (VecSize == 2) {
1175 Chain = retval.getValue(2);
1176 InFlag = retval.getValue(3);
1177 } else {
1178 Chain = retval.getValue(4);
1179 InFlag = retval.getValue(5);
1180 }
1181
1182 for (unsigned j = 0; j < VecSize; ++j) {
1183 if (i + j >= NumElts)
1184 break;
1185 SDValue Elt = retval.getValue(j);
1186 if (needTruncate)
1187 Elt = DAG.getNode(ISD::TRUNCATE, dl, EltVT, Elt);
1188 InVals.push_back(Elt);
1189 }
1190 Ofst += TD->getTypeAllocSize(VecVT.getTypeForEVT(F->getContext()));
1191 }
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]1192 }
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1193 } else {
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]1194 SmallVector<EVT, 16> VTs;
Justin Holewinski6e40f632014-06-27 18:35:44 +0000[diff] [blame]1195 SmallVector<uint64_t, 16> Offsets;
1196 ComputePTXValueVTs(*this, retTy, VTs, &Offsets, 0);
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]1197 assert(VTs.size() == Ins.size() && "Bad value decomposition");
Justin Holewinski6e40f632014-06-27 18:35:44 +0000[diff] [blame]1198 unsigned RetAlign = getArgumentAlignment(Callee, CS, retTy, 0);
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1199 for (unsigned i = 0, e = Ins.size(); i != e; ++i) {
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]1200 unsigned sz = VTs[i].getSizeInBits();
Justin Holewinski9982f062014-06-27 19:36:25 +0000[diff] [blame]1201 unsigned AlignI = GreatestCommonDivisor64(RetAlign, Offsets[i]);
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]1202 bool needTruncate = sz < 8 ? true : false;
1203 if (VTs[i].isInteger() && (sz < 8))
1204 sz = 8;
1205
1206 SmallVector<EVT, 4> LoadRetVTs;
Justin Holewinskie04e4bd2013-06-28 17:58:10 +0000[diff] [blame]1207 EVT TheLoadType = VTs[i];
1208 if (retTy->isIntegerTy() &&
1209 TD->getTypeAllocSizeInBits(retTy) < 32) {
1210 // This is for integer types only, and specifically not for
1211 // aggregates.
1212 LoadRetVTs.push_back(MVT::i32);
1213 TheLoadType = MVT::i32;
1214 } else if (sz < 16) {
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]1215 // If loading i1/i8 result, generate
1216 // load i8 (-> i16)
1217 // trunc i16 to i1/i8
1218 LoadRetVTs.push_back(MVT::i16);
1219 } else
1220 LoadRetVTs.push_back(Ins[i].VT);
1221 LoadRetVTs.push_back(MVT::Other);
1222 LoadRetVTs.push_back(MVT::Glue);
1223
1224 SmallVector<SDValue, 4> LoadRetOps;
1225 LoadRetOps.push_back(Chain);
1226 LoadRetOps.push_back(DAG.getConstant(1, MVT::i32));
Justin Holewinski6e40f632014-06-27 18:35:44 +0000[diff] [blame]1227 LoadRetOps.push_back(DAG.getConstant(Offsets[i], MVT::i32));
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]1228 LoadRetOps.push_back(InFlag);
1229 SDValue retval = DAG.getMemIntrinsicNode(
1230 NVPTXISD::LoadParam, dl,
Craig Topper206fcd42014-04-26 19:29:41 +0000[diff] [blame]1231 DAG.getVTList(LoadRetVTs), LoadRetOps,
Justin Holewinski6e40f632014-06-27 18:35:44 +0000[diff] [blame]1232 TheLoadType, MachinePointerInfo(), AlignI);
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]1233 Chain = retval.getValue(1);
1234 InFlag = retval.getValue(2);
1235 SDValue Ret0 = retval.getValue(0);
1236 if (needTruncate)
1237 Ret0 = DAG.getNode(ISD::TRUNCATE, dl, Ins[i].VT, Ret0);
1238 InVals.push_back(Ret0);
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]1239 }
1240 }
1241 }
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]1242
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1243 Chain = DAG.getCALLSEQ_END(Chain, DAG.getIntPtrConstant(uniqueCallSite, true),
1244 DAG.getIntPtrConstant(uniqueCallSite + 1, true),
Andrew Trickad6d08a2013-05-29 22:03:55 +0000[diff] [blame]1245 InFlag, dl);
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]1246 uniqueCallSite++;
1247
1248 // set isTailCall to false for now, until we figure out how to express
1249 // tail call optimization in PTX
1250 isTailCall = false;
1251 return Chain;
1252}
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]1253
1254// By default CONCAT_VECTORS is lowered by ExpandVectorBuildThroughStack()
1255// (see LegalizeDAG.cpp). This is slow and uses local memory.
1256// We use extract/insert/build vector just as what LegalizeOp() does in llvm 2.5
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1257SDValue
1258NVPTXTargetLowering::LowerCONCAT_VECTORS(SDValue Op, SelectionDAG &DAG) const {
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]1259 SDNode *Node = Op.getNode();
Andrew Trickef9de2a2013-05-25 02:42:55 +0000[diff] [blame]1260 SDLoc dl(Node);
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]1261 SmallVector<SDValue, 8> Ops;
1262 unsigned NumOperands = Node->getNumOperands();
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1263 for (unsigned i = 0; i < NumOperands; ++i) {
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]1264 SDValue SubOp = Node->getOperand(i);
1265 EVT VVT = SubOp.getNode()->getValueType(0);
1266 EVT EltVT = VVT.getVectorElementType();
1267 unsigned NumSubElem = VVT.getVectorNumElements();
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1268 for (unsigned j = 0; j < NumSubElem; ++j) {
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]1269 Ops.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, SubOp,
1270 DAG.getIntPtrConstant(j)));
1271 }
1272 }
Craig Topper48d114b2014-04-26 18:35:24 +0000[diff] [blame]1273 return DAG.getNode(ISD::BUILD_VECTOR, dl, Node->getValueType(0), Ops);
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]1274}
1275
Justin Holewinski360a5cf2014-06-27 18:35:40 +0000[diff] [blame]1276/// LowerShiftRightParts - Lower SRL_PARTS, SRA_PARTS, which
1277/// 1) returns two i32 values and take a 2 x i32 value to shift plus a shift
1278/// amount, or
1279/// 2) returns two i64 values and take a 2 x i64 value to shift plus a shift
1280/// amount.
1281SDValue NVPTXTargetLowering::LowerShiftRightParts(SDValue Op,
1282 SelectionDAG &DAG) const {
1283 assert(Op.getNumOperands() == 3 && "Not a double-shift!");
1284 assert(Op.getOpcode() == ISD::SRA_PARTS || Op.getOpcode() == ISD::SRL_PARTS);
1285
1286 EVT VT = Op.getValueType();
1287 unsigned VTBits = VT.getSizeInBits();
1288 SDLoc dl(Op);
1289 SDValue ShOpLo = Op.getOperand(0);
1290 SDValue ShOpHi = Op.getOperand(1);
1291 SDValue ShAmt = Op.getOperand(2);
1292 unsigned Opc = (Op.getOpcode() == ISD::SRA_PARTS) ? ISD::SRA : ISD::SRL;
1293
1294 if (VTBits == 32 && nvptxSubtarget.getSmVersion() >= 35) {
1295
1296 // For 32bit and sm35, we can use the funnel shift 'shf' instruction.
1297 // {dHi, dLo} = {aHi, aLo} >> Amt
1298 // dHi = aHi >> Amt
1299 // dLo = shf.r.clamp aLo, aHi, Amt
1300
1301 SDValue Hi = DAG.getNode(Opc, dl, VT, ShOpHi, ShAmt);
1302 SDValue Lo = DAG.getNode(NVPTXISD::FUN_SHFR_CLAMP, dl, VT, ShOpLo, ShOpHi,
1303 ShAmt);
1304
1305 SDValue Ops[2] = { Lo, Hi };
1306 return DAG.getMergeValues(Ops, dl);
1307 }
1308 else {
1309
1310 // {dHi, dLo} = {aHi, aLo} >> Amt
1311 // - if (Amt>=size) then
1312 // dLo = aHi >> (Amt-size)
1313 // dHi = aHi >> Amt (this is either all 0 or all 1)
1314 // else
1315 // dLo = (aLo >>logic Amt) | (aHi << (size-Amt))
1316 // dHi = aHi >> Amt
1317
1318 SDValue RevShAmt = DAG.getNode(ISD::SUB, dl, MVT::i32,
1319 DAG.getConstant(VTBits, MVT::i32), ShAmt);
1320 SDValue Tmp1 = DAG.getNode(ISD::SRL, dl, VT, ShOpLo, ShAmt);
1321 SDValue ExtraShAmt = DAG.getNode(ISD::SUB, dl, MVT::i32, ShAmt,
1322 DAG.getConstant(VTBits, MVT::i32));
1323 SDValue Tmp2 = DAG.getNode(ISD::SHL, dl, VT, ShOpHi, RevShAmt);
1324 SDValue FalseVal = DAG.getNode(ISD::OR, dl, VT, Tmp1, Tmp2);
1325 SDValue TrueVal = DAG.getNode(Opc, dl, VT, ShOpHi, ExtraShAmt);
1326
1327 SDValue Cmp = DAG.getSetCC(dl, MVT::i1, ShAmt,
1328 DAG.getConstant(VTBits, MVT::i32), ISD::SETGE);
1329 SDValue Hi = DAG.getNode(Opc, dl, VT, ShOpHi, ShAmt);
1330 SDValue Lo = DAG.getNode(ISD::SELECT, dl, VT, Cmp, TrueVal, FalseVal);
1331
1332 SDValue Ops[2] = { Lo, Hi };
1333 return DAG.getMergeValues(Ops, dl);
1334 }
1335}
1336
1337/// LowerShiftLeftParts - Lower SHL_PARTS, which
1338/// 1) returns two i32 values and take a 2 x i32 value to shift plus a shift
1339/// amount, or
1340/// 2) returns two i64 values and take a 2 x i64 value to shift plus a shift
1341/// amount.
1342SDValue NVPTXTargetLowering::LowerShiftLeftParts(SDValue Op,
1343 SelectionDAG &DAG) const {
1344 assert(Op.getNumOperands() == 3 && "Not a double-shift!");
1345 assert(Op.getOpcode() == ISD::SHL_PARTS);
1346
1347 EVT VT = Op.getValueType();
1348 unsigned VTBits = VT.getSizeInBits();
1349 SDLoc dl(Op);
1350 SDValue ShOpLo = Op.getOperand(0);
1351 SDValue ShOpHi = Op.getOperand(1);
1352 SDValue ShAmt = Op.getOperand(2);
1353
1354 if (VTBits == 32 && nvptxSubtarget.getSmVersion() >= 35) {
1355
1356 // For 32bit and sm35, we can use the funnel shift 'shf' instruction.
1357 // {dHi, dLo} = {aHi, aLo} << Amt
1358 // dHi = shf.l.clamp aLo, aHi, Amt
1359 // dLo = aLo << Amt
1360
1361 SDValue Hi = DAG.getNode(NVPTXISD::FUN_SHFL_CLAMP, dl, VT, ShOpLo, ShOpHi,
1362 ShAmt);
1363 SDValue Lo = DAG.getNode(ISD::SHL, dl, VT, ShOpLo, ShAmt);
1364
1365 SDValue Ops[2] = { Lo, Hi };
1366 return DAG.getMergeValues(Ops, dl);
1367 }
1368 else {
1369
1370 // {dHi, dLo} = {aHi, aLo} << Amt
1371 // - if (Amt>=size) then
1372 // dLo = aLo << Amt (all 0)
1373 // dLo = aLo << (Amt-size)
1374 // else
1375 // dLo = aLo << Amt
1376 // dHi = (aHi << Amt) | (aLo >> (size-Amt))
1377
1378 SDValue RevShAmt = DAG.getNode(ISD::SUB, dl, MVT::i32,
1379 DAG.getConstant(VTBits, MVT::i32), ShAmt);
1380 SDValue Tmp1 = DAG.getNode(ISD::SHL, dl, VT, ShOpHi, ShAmt);
1381 SDValue ExtraShAmt = DAG.getNode(ISD::SUB, dl, MVT::i32, ShAmt,
1382 DAG.getConstant(VTBits, MVT::i32));
1383 SDValue Tmp2 = DAG.getNode(ISD::SRL, dl, VT, ShOpLo, RevShAmt);
1384 SDValue FalseVal = DAG.getNode(ISD::OR, dl, VT, Tmp1, Tmp2);
1385 SDValue TrueVal = DAG.getNode(ISD::SHL, dl, VT, ShOpLo, ExtraShAmt);
1386
1387 SDValue Cmp = DAG.getSetCC(dl, MVT::i1, ShAmt,
1388 DAG.getConstant(VTBits, MVT::i32), ISD::SETGE);
1389 SDValue Lo = DAG.getNode(ISD::SHL, dl, VT, ShOpLo, ShAmt);
1390 SDValue Hi = DAG.getNode(ISD::SELECT, dl, VT, Cmp, TrueVal, FalseVal);
1391
1392 SDValue Ops[2] = { Lo, Hi };
1393 return DAG.getMergeValues(Ops, dl);
1394 }
1395}
1396
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1397SDValue
1398NVPTXTargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const {
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]1399 switch (Op.getOpcode()) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1400 case ISD::RETURNADDR:
1401 return SDValue();
1402 case ISD::FRAMEADDR:
1403 return SDValue();
1404 case ISD::GlobalAddress:
1405 return LowerGlobalAddress(Op, DAG);
1406 case ISD::INTRINSIC_W_CHAIN:
1407 return Op;
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]1408 case ISD::BUILD_VECTOR:
1409 case ISD::EXTRACT_SUBVECTOR:
1410 return Op;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1411 case ISD::CONCAT_VECTORS:
1412 return LowerCONCAT_VECTORS(Op, DAG);
1413 case ISD::STORE:
1414 return LowerSTORE(Op, DAG);
1415 case ISD::LOAD:
1416 return LowerLOAD(Op, DAG);
Justin Holewinski360a5cf2014-06-27 18:35:40 +0000[diff] [blame]1417 case ISD::SHL_PARTS:
1418 return LowerShiftLeftParts(Op, DAG);
1419 case ISD::SRA_PARTS:
1420 case ISD::SRL_PARTS:
1421 return LowerShiftRightParts(Op, DAG);
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]1422 default:
David Blaikie891d0a32012-05-04 22:34:16 +0000[diff] [blame]1423 llvm_unreachable("Custom lowering not defined for operation");
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]1424 }
1425}
1426
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]1427SDValue NVPTXTargetLowering::LowerLOAD(SDValue Op, SelectionDAG &DAG) const {
1428 if (Op.getValueType() == MVT::i1)
1429 return LowerLOADi1(Op, DAG);
1430 else
1431 return SDValue();
1432}
1433
Justin Holewinskic6462aa2012-11-14 19:19:16 +0000[diff] [blame]1434// v = ld i1* addr
1435// =>
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]1436// v1 = ld i8* addr (-> i16)
1437// v = trunc i16 to i1
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1438SDValue NVPTXTargetLowering::LowerLOADi1(SDValue Op, SelectionDAG &DAG) const {
Justin Holewinskic6462aa2012-11-14 19:19:16 +0000[diff] [blame]1439 SDNode *Node = Op.getNode();
1440 LoadSDNode *LD = cast<LoadSDNode>(Node);
Andrew Trickef9de2a2013-05-25 02:42:55 +0000[diff] [blame]1441 SDLoc dl(Node);
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1442 assert(LD->getExtensionType() == ISD::NON_EXTLOAD);
NAKAMURA Takumi5bbe0e12012-11-14 23:46:15 +0000[diff] [blame]1443 assert(Node->getValueType(0) == MVT::i1 &&
1444 "Custom lowering for i1 load only");
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1445 SDValue newLD =
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]1446 DAG.getLoad(MVT::i16, dl, LD->getChain(), LD->getBasePtr(),
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1447 LD->getPointerInfo(), LD->isVolatile(), LD->isNonTemporal(),
1448 LD->isInvariant(), LD->getAlignment());
Justin Holewinskic6462aa2012-11-14 19:19:16 +0000[diff] [blame]1449 SDValue result = DAG.getNode(ISD::TRUNCATE, dl, MVT::i1, newLD);
1450 // The legalizer (the caller) is expecting two values from the legalized
1451 // load, so we build a MergeValues node for it. See ExpandUnalignedLoad()
1452 // in LegalizeDAG.cpp which also uses MergeValues.
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1453 SDValue Ops[] = { result, LD->getChain() };
Craig Topper64941d92014-04-27 19:20:57 +0000[diff] [blame]1454 return DAG.getMergeValues(Ops, dl);
Justin Holewinskic6462aa2012-11-14 19:19:16 +0000[diff] [blame]1455}
1456
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]1457SDValue NVPTXTargetLowering::LowerSTORE(SDValue Op, SelectionDAG &DAG) const {
1458 EVT ValVT = Op.getOperand(1).getValueType();
1459 if (ValVT == MVT::i1)
1460 return LowerSTOREi1(Op, DAG);
1461 else if (ValVT.isVector())
1462 return LowerSTOREVector(Op, DAG);
1463 else
1464 return SDValue();
1465}
1466
1467SDValue
1468NVPTXTargetLowering::LowerSTOREVector(SDValue Op, SelectionDAG &DAG) const {
1469 SDNode *N = Op.getNode();
1470 SDValue Val = N->getOperand(1);
Andrew Trickef9de2a2013-05-25 02:42:55 +0000[diff] [blame]1471 SDLoc DL(N);
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]1472 EVT ValVT = Val.getValueType();
1473
1474 if (ValVT.isVector()) {
1475 // We only handle "native" vector sizes for now, e.g. <4 x double> is not
1476 // legal. We can (and should) split that into 2 stores of <2 x double> here
1477 // but I'm leaving that as a TODO for now.
1478 if (!ValVT.isSimple())
1479 return SDValue();
1480 switch (ValVT.getSimpleVT().SimpleTy) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1481 default:
1482 return SDValue();
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]1483 case MVT::v2i8:
1484 case MVT::v2i16:
1485 case MVT::v2i32:
1486 case MVT::v2i64:
1487 case MVT::v2f32:
1488 case MVT::v2f64:
1489 case MVT::v4i8:
1490 case MVT::v4i16:
1491 case MVT::v4i32:
1492 case MVT::v4f32:
1493 // This is a "native" vector type
1494 break;
1495 }
1496
Justin Holewinskiac451062014-07-16 19:45:35 +0000[diff] [blame^]1497 MemSDNode *MemSD = cast<MemSDNode>(N);
1498 const DataLayout *TD = getDataLayout();
1499
1500 unsigned Align = MemSD->getAlignment();
1501 unsigned PrefAlign =
1502 TD->getPrefTypeAlignment(ValVT.getTypeForEVT(*DAG.getContext()));
1503 if (Align < PrefAlign) {
1504 // This store is not sufficiently aligned, so bail out and let this vector
1505 // store be scalarized. Note that we may still be able to emit smaller
1506 // vector stores. For example, if we are storing a <4 x float> with an
1507 // alignment of 8, this check will fail but the legalizer will try again
1508 // with 2 x <2 x float>, which will succeed with an alignment of 8.
1509 return SDValue();
1510 }
1511
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]1512 unsigned Opcode = 0;
1513 EVT EltVT = ValVT.getVectorElementType();
1514 unsigned NumElts = ValVT.getVectorNumElements();
1515
1516 // Since StoreV2 is a target node, we cannot rely on DAG type legalization.
1517 // Therefore, we must ensure the type is legal. For i1 and i8, we set the
Alp Tokercb402912014-01-24 17:20:08 +0000[diff] [blame]1518 // stored type to i16 and propagate the "real" type as the memory type.
Justin Holewinskia2911282013-07-01 12:58:58 +0000[diff] [blame]1519 bool NeedExt = false;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]1520 if (EltVT.getSizeInBits() < 16)
Justin Holewinskia2911282013-07-01 12:58:58 +0000[diff] [blame]1521 NeedExt = true;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]1522
1523 switch (NumElts) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1524 default:
1525 return SDValue();
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]1526 case 2:
1527 Opcode = NVPTXISD::StoreV2;
1528 break;
1529 case 4: {
1530 Opcode = NVPTXISD::StoreV4;
1531 break;
1532 }
1533 }
1534
1535 SmallVector<SDValue, 8> Ops;
1536
1537 // First is the chain
1538 Ops.push_back(N->getOperand(0));
1539
1540 // Then the split values
1541 for (unsigned i = 0; i < NumElts; ++i) {
1542 SDValue ExtVal = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, EltVT, Val,
1543 DAG.getIntPtrConstant(i));
Justin Holewinskia2911282013-07-01 12:58:58 +0000[diff] [blame]1544 if (NeedExt)
1545 ExtVal = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i16, ExtVal);
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]1546 Ops.push_back(ExtVal);
1547 }
1548
1549 // Then any remaining arguments
1550 for (unsigned i = 2, e = N->getNumOperands(); i != e; ++i) {
1551 Ops.push_back(N->getOperand(i));
1552 }
1553
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1554 SDValue NewSt = DAG.getMemIntrinsicNode(
Craig Topper206fcd42014-04-26 19:29:41 +0000[diff] [blame]1555 Opcode, DL, DAG.getVTList(MVT::Other), Ops,
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1556 MemSD->getMemoryVT(), MemSD->getMemOperand());
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]1557
1558 //return DCI.CombineTo(N, NewSt, true);
1559 return NewSt;
1560 }
1561
1562 return SDValue();
1563}
1564
Justin Holewinskic6462aa2012-11-14 19:19:16 +0000[diff] [blame]1565// st i1 v, addr
1566// =>
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]1567// v1 = zxt v to i16
1568// st.u8 i16, addr
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1569SDValue NVPTXTargetLowering::LowerSTOREi1(SDValue Op, SelectionDAG &DAG) const {
Justin Holewinskic6462aa2012-11-14 19:19:16 +0000[diff] [blame]1570 SDNode *Node = Op.getNode();
Andrew Trickef9de2a2013-05-25 02:42:55 +0000[diff] [blame]1571 SDLoc dl(Node);
Justin Holewinskic6462aa2012-11-14 19:19:16 +0000[diff] [blame]1572 StoreSDNode *ST = cast<StoreSDNode>(Node);
1573 SDValue Tmp1 = ST->getChain();
1574 SDValue Tmp2 = ST->getBasePtr();
1575 SDValue Tmp3 = ST->getValue();
NAKAMURA Takumi5bbe0e12012-11-14 23:46:15 +0000[diff] [blame]1576 assert(Tmp3.getValueType() == MVT::i1 && "Custom lowering for i1 store only");
Justin Holewinskic6462aa2012-11-14 19:19:16 +0000[diff] [blame]1577 unsigned Alignment = ST->getAlignment();
1578 bool isVolatile = ST->isVolatile();
1579 bool isNonTemporal = ST->isNonTemporal();
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]1580 Tmp3 = DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i16, Tmp3);
1581 SDValue Result = DAG.getTruncStore(Tmp1, dl, Tmp3, Tmp2,
1582 ST->getPointerInfo(), MVT::i8, isNonTemporal,
1583 isVolatile, Alignment);
Justin Holewinskic6462aa2012-11-14 19:19:16 +0000[diff] [blame]1584 return Result;
1585}
1586
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1587SDValue NVPTXTargetLowering::getExtSymb(SelectionDAG &DAG, const char *inname,
1588 int idx, EVT v) const {
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]1589 std::string *name = nvTM->getManagedStrPool()->getManagedString(inname);
1590 std::stringstream suffix;
1591 suffix << idx;
1592 *name += suffix.str();
1593 return DAG.getTargetExternalSymbol(name->c_str(), v);
1594}
1595
1596SDValue
1597NVPTXTargetLowering::getParamSymbol(SelectionDAG &DAG, int idx, EVT v) const {
Justin Holewinskia2a63d22013-08-06 14:13:27 +0000[diff] [blame]1598 std::string ParamSym;
1599 raw_string_ostream ParamStr(ParamSym);
1600
1601 ParamStr << DAG.getMachineFunction().getName() << "_param_" << idx;
1602 ParamStr.flush();
1603
1604 std::string *SavedStr =
1605 nvTM->getManagedStrPool()->getManagedString(ParamSym.c_str());
1606 return DAG.getTargetExternalSymbol(SavedStr->c_str(), v);
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]1607}
1608
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1609SDValue NVPTXTargetLowering::getParamHelpSymbol(SelectionDAG &DAG, int idx) {
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]1610 return getExtSymb(DAG, ".HLPPARAM", idx);
1611}
1612
1613// Check to see if the kernel argument is image*_t or sampler_t
1614
1615bool llvm::isImageOrSamplerVal(const Value *arg, const Module *context) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1616 static const char *const specialTypes[] = { "struct._image2d_t",
1617 "struct._image3d_t",
1618 "struct._sampler_t" };
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]1619
1620 const Type *Ty = arg->getType();
1621 const PointerType *PTy = dyn_cast<PointerType>(Ty);
1622
1623 if (!PTy)
1624 return false;
1625
1626 if (!context)
1627 return false;
1628
1629 const StructType *STy = dyn_cast<StructType>(PTy->getElementType());
Justin Holewinskifb711152012-12-05 20:50:28 +0000[diff] [blame]1630 const std::string TypeName = STy && !STy->isLiteral() ? STy->getName() : "";
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]1631
Craig Toppere4260f92012-05-24 04:22:05 +0000[diff] [blame]1632 for (int i = 0, e = array_lengthof(specialTypes); i != e; ++i)
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]1633 if (TypeName == specialTypes[i])
1634 return true;
1635
1636 return false;
1637}
1638
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1639SDValue NVPTXTargetLowering::LowerFormalArguments(
1640 SDValue Chain, CallingConv::ID CallConv, bool isVarArg,
Andrew Trickef9de2a2013-05-25 02:42:55 +0000[diff] [blame]1641 const SmallVectorImpl<ISD::InputArg> &Ins, SDLoc dl, SelectionDAG &DAG,
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1642 SmallVectorImpl<SDValue> &InVals) const {
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]1643 MachineFunction &MF = DAG.getMachineFunction();
Micah Villmowcdfe20b2012-10-08 16:38:25 +0000[diff] [blame]1644 const DataLayout *TD = getDataLayout();
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]1645
1646 const Function *F = MF.getFunction();
Bill Wendlinge94d8432012-12-07 23:16:57 +0000[diff] [blame]1647 const AttributeSet &PAL = F->getAttributes();
Eric Christopher2ecb77e2014-06-27 03:45:49 +0000[diff] [blame]1648 const TargetLowering *TLI = DAG.getTarget().getTargetLowering();
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]1649
1650 SDValue Root = DAG.getRoot();
1651 std::vector<SDValue> OutChains;
1652
1653 bool isKernel = llvm::isKernelFunction(*F);
1654 bool isABI = (nvptxSubtarget.getSmVersion() >= 20);
Justin Holewinski44f5c602013-06-28 17:57:53 +0000[diff] [blame]1655 assert(isABI && "Non-ABI compilation is not supported");
1656 if (!isABI)
1657 return Chain;
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]1658
1659 std::vector<Type *> argTypes;
1660 std::vector<const Argument *> theArgs;
1661 for (Function::const_arg_iterator I = F->arg_begin(), E = F->arg_end();
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1662 I != E; ++I) {
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]1663 theArgs.push_back(I);
1664 argTypes.push_back(I->getType());
1665 }
Justin Holewinski44f5c602013-06-28 17:57:53 +0000[diff] [blame]1666 // argTypes.size() (or theArgs.size()) and Ins.size() need not match.
1667 // Ins.size() will be larger
1668 // * if there is an aggregate argument with multiple fields (each field
1669 // showing up separately in Ins)
1670 // * if there is a vector argument with more than typical vector-length
1671 // elements (generally if more than 4) where each vector element is
1672 // individually present in Ins.
1673 // So a different index should be used for indexing into Ins.
1674 // See similar issue in LowerCall.
1675 unsigned InsIdx = 0;
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]1676
1677 int idx = 0;
Justin Holewinski44f5c602013-06-28 17:57:53 +0000[diff] [blame]1678 for (unsigned i = 0, e = theArgs.size(); i != e; ++i, ++idx, ++InsIdx) {
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]1679 Type *Ty = argTypes[i];
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]1680
1681 // If the kernel argument is image*_t or sampler_t, convert it to
1682 // a i32 constant holding the parameter position. This can later
1683 // matched in the AsmPrinter to output the correct mangled name.
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1684 if (isImageOrSamplerVal(
1685 theArgs[i],
1686 (theArgs[i]->getParent() ? theArgs[i]->getParent()->getParent()
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]1687 : nullptr))) {
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]1688 assert(isKernel && "Only kernels can have image/sampler params");
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1689 InVals.push_back(DAG.getConstant(i + 1, MVT::i32));
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]1690 continue;
1691 }
1692
1693 if (theArgs[i]->use_empty()) {
1694 // argument is dead
Justin Holewinski44f5c602013-06-28 17:57:53 +0000[diff] [blame]1695 if (Ty->isAggregateType()) {
1696 SmallVector<EVT, 16> vtparts;
1697
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]1698 ComputePTXValueVTs(*this, Ty, vtparts);
Justin Holewinski44f5c602013-06-28 17:57:53 +0000[diff] [blame]1699 assert(vtparts.size() > 0 && "empty aggregate type not expected");
1700 for (unsigned parti = 0, parte = vtparts.size(); parti != parte;
1701 ++parti) {
Justin Holewinskib5db95e2014-06-27 18:36:04 +0000[diff] [blame]1702 InVals.push_back(DAG.getNode(ISD::UNDEF, dl, Ins[InsIdx].VT));
Justin Holewinski44f5c602013-06-28 17:57:53 +0000[diff] [blame]1703 ++InsIdx;
Justin Holewinskie9884092013-03-24 21:17:47 +0000[diff] [blame]1704 }
Justin Holewinski44f5c602013-06-28 17:57:53 +0000[diff] [blame]1705 if (vtparts.size() > 0)
1706 --InsIdx;
1707 continue;
Justin Holewinskie9884092013-03-24 21:17:47 +0000[diff] [blame]1708 }
Justin Holewinski44f5c602013-06-28 17:57:53 +0000[diff] [blame]1709 if (Ty->isVectorTy()) {
1710 EVT ObjectVT = getValueType(Ty);
1711 unsigned NumRegs = TLI->getNumRegisters(F->getContext(), ObjectVT);
1712 for (unsigned parti = 0; parti < NumRegs; ++parti) {
1713 InVals.push_back(DAG.getNode(ISD::UNDEF, dl, Ins[InsIdx].VT));
1714 ++InsIdx;
1715 }
1716 if (NumRegs > 0)
1717 --InsIdx;
1718 continue;
1719 }
1720 InVals.push_back(DAG.getNode(ISD::UNDEF, dl, Ins[InsIdx].VT));
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]1721 continue;
1722 }
1723
1724 // In the following cases, assign a node order of "idx+1"
Justin Holewinski44f5c602013-06-28 17:57:53 +0000[diff] [blame]1725 // to newly created nodes. The SDNodes for params have to
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]1726 // appear in the same order as their order of appearance
1727 // in the original function. "idx+1" holds that order.
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1728 if (PAL.hasAttribute(i + 1, Attribute::ByVal) == false) {
Justin Holewinski44f5c602013-06-28 17:57:53 +0000[diff] [blame]1729 if (Ty->isAggregateType()) {
1730 SmallVector<EVT, 16> vtparts;
1731 SmallVector<uint64_t, 16> offsets;
1732
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]1733 // NOTE: Here, we lose the ability to issue vector loads for vectors
1734 // that are a part of a struct. This should be investigated in the
1735 // future.
1736 ComputePTXValueVTs(*this, Ty, vtparts, &offsets, 0);
Justin Holewinski44f5c602013-06-28 17:57:53 +0000[diff] [blame]1737 assert(vtparts.size() > 0 && "empty aggregate type not expected");
1738 bool aggregateIsPacked = false;
1739 if (StructType *STy = llvm::dyn_cast<StructType>(Ty))
1740 aggregateIsPacked = STy->isPacked();
1741
1742 SDValue Arg = getParamSymbol(DAG, idx, getPointerTy());
1743 for (unsigned parti = 0, parte = vtparts.size(); parti != parte;
1744 ++parti) {
1745 EVT partVT = vtparts[parti];
1746 Value *srcValue = Constant::getNullValue(
1747 PointerType::get(partVT.getTypeForEVT(F->getContext()),
1748 llvm::ADDRESS_SPACE_PARAM));
1749 SDValue srcAddr =
1750 DAG.getNode(ISD::ADD, dl, getPointerTy(), Arg,
1751 DAG.getConstant(offsets[parti], getPointerTy()));
1752 unsigned partAlign =
1753 aggregateIsPacked ? 1
1754 : TD->getABITypeAlignment(
1755 partVT.getTypeForEVT(F->getContext()));
Justin Holewinskia2911282013-07-01 12:58:58 +0000[diff] [blame]1756 SDValue p;
1757 if (Ins[InsIdx].VT.getSizeInBits() > partVT.getSizeInBits()) {
1758 ISD::LoadExtType ExtOp = Ins[InsIdx].Flags.isSExt() ?
1759 ISD::SEXTLOAD : ISD::ZEXTLOAD;
1760 p = DAG.getExtLoad(ExtOp, dl, Ins[InsIdx].VT, Root, srcAddr,
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]1761 MachinePointerInfo(srcValue), partVT, false,
1762 false, partAlign);
Justin Holewinskia2911282013-07-01 12:58:58 +0000[diff] [blame]1763 } else {
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]1764 p = DAG.getLoad(partVT, dl, Root, srcAddr,
1765 MachinePointerInfo(srcValue), false, false, false,
1766 partAlign);
Justin Holewinskia2911282013-07-01 12:58:58 +0000[diff] [blame]1767 }
Justin Holewinski44f5c602013-06-28 17:57:53 +0000[diff] [blame]1768 if (p.getNode())
1769 p.getNode()->setIROrder(idx + 1);
1770 InVals.push_back(p);
1771 ++InsIdx;
Justin Holewinskie9884092013-03-24 21:17:47 +0000[diff] [blame]1772 }
Justin Holewinski44f5c602013-06-28 17:57:53 +0000[diff] [blame]1773 if (vtparts.size() > 0)
1774 --InsIdx;
Justin Holewinskie9884092013-03-24 21:17:47 +0000[diff] [blame]1775 continue;
1776 }
Justin Holewinski44f5c602013-06-28 17:57:53 +0000[diff] [blame]1777 if (Ty->isVectorTy()) {
1778 EVT ObjectVT = getValueType(Ty);
Justin Holewinskiaaaf2892013-06-25 12:22:21 +0000[diff] [blame]1779 SDValue Arg = getParamSymbol(DAG, idx, getPointerTy());
Justin Holewinski44f5c602013-06-28 17:57:53 +0000[diff] [blame]1780 unsigned NumElts = ObjectVT.getVectorNumElements();
1781 assert(TLI->getNumRegisters(F->getContext(), ObjectVT) == NumElts &&
1782 "Vector was not scalarized");
1783 unsigned Ofst = 0;
1784 EVT EltVT = ObjectVT.getVectorElementType();
1785
1786 // V1 load
1787 // f32 = load ...
1788 if (NumElts == 1) {
1789 // We only have one element, so just directly load it
1790 Value *SrcValue = Constant::getNullValue(PointerType::get(
1791 EltVT.getTypeForEVT(F->getContext()), llvm::ADDRESS_SPACE_PARAM));
1792 SDValue SrcAddr = DAG.getNode(ISD::ADD, dl, getPointerTy(), Arg,
1793 DAG.getConstant(Ofst, getPointerTy()));
1794 SDValue P = DAG.getLoad(
1795 EltVT, dl, Root, SrcAddr, MachinePointerInfo(SrcValue), false,
1796 false, true,
1797 TD->getABITypeAlignment(EltVT.getTypeForEVT(F->getContext())));
1798 if (P.getNode())
1799 P.getNode()->setIROrder(idx + 1);
1800
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]1801 if (Ins[InsIdx].VT.getSizeInBits() > EltVT.getSizeInBits())
Justin Holewinskia2911282013-07-01 12:58:58 +0000[diff] [blame]1802 P = DAG.getNode(ISD::ANY_EXTEND, dl, Ins[InsIdx].VT, P);
Justin Holewinski44f5c602013-06-28 17:57:53 +0000[diff] [blame]1803 InVals.push_back(P);
1804 Ofst += TD->getTypeAllocSize(EltVT.getTypeForEVT(F->getContext()));
1805 ++InsIdx;
1806 } else if (NumElts == 2) {
1807 // V2 load
1808 // f32,f32 = load ...
1809 EVT VecVT = EVT::getVectorVT(F->getContext(), EltVT, 2);
1810 Value *SrcValue = Constant::getNullValue(PointerType::get(
1811 VecVT.getTypeForEVT(F->getContext()), llvm::ADDRESS_SPACE_PARAM));
1812 SDValue SrcAddr = DAG.getNode(ISD::ADD, dl, getPointerTy(), Arg,
1813 DAG.getConstant(Ofst, getPointerTy()));
1814 SDValue P = DAG.getLoad(
1815 VecVT, dl, Root, SrcAddr, MachinePointerInfo(SrcValue), false,
1816 false, true,
1817 TD->getABITypeAlignment(VecVT.getTypeForEVT(F->getContext())));
1818 if (P.getNode())
1819 P.getNode()->setIROrder(idx + 1);
1820
1821 SDValue Elt0 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, P,
1822 DAG.getIntPtrConstant(0));
1823 SDValue Elt1 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, P,
1824 DAG.getIntPtrConstant(1));
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]1825
1826 if (Ins[InsIdx].VT.getSizeInBits() > EltVT.getSizeInBits()) {
Justin Holewinskia2911282013-07-01 12:58:58 +0000[diff] [blame]1827 Elt0 = DAG.getNode(ISD::ANY_EXTEND, dl, Ins[InsIdx].VT, Elt0);
1828 Elt1 = DAG.getNode(ISD::ANY_EXTEND, dl, Ins[InsIdx].VT, Elt1);
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]1829 }
1830
Justin Holewinski44f5c602013-06-28 17:57:53 +0000[diff] [blame]1831 InVals.push_back(Elt0);
1832 InVals.push_back(Elt1);
1833 Ofst += TD->getTypeAllocSize(VecVT.getTypeForEVT(F->getContext()));
1834 InsIdx += 2;
1835 } else {
1836 // V4 loads
1837 // We have at least 4 elements (<3 x Ty> expands to 4 elements) and
1838 // the
1839 // vector will be expanded to a power of 2 elements, so we know we can
1840 // always round up to the next multiple of 4 when creating the vector
1841 // loads.
1842 // e.g. 4 elem => 1 ld.v4
1843 // 6 elem => 2 ld.v4
1844 // 8 elem => 2 ld.v4
1845 // 11 elem => 3 ld.v4
1846 unsigned VecSize = 4;
1847 if (EltVT.getSizeInBits() == 64) {
1848 VecSize = 2;
1849 }
1850 EVT VecVT = EVT::getVectorVT(F->getContext(), EltVT, VecSize);
1851 for (unsigned i = 0; i < NumElts; i += VecSize) {
1852 Value *SrcValue = Constant::getNullValue(
1853 PointerType::get(VecVT.getTypeForEVT(F->getContext()),
1854 llvm::ADDRESS_SPACE_PARAM));
1855 SDValue SrcAddr =
1856 DAG.getNode(ISD::ADD, dl, getPointerTy(), Arg,
1857 DAG.getConstant(Ofst, getPointerTy()));
1858 SDValue P = DAG.getLoad(
1859 VecVT, dl, Root, SrcAddr, MachinePointerInfo(SrcValue), false,
1860 false, true,
1861 TD->getABITypeAlignment(VecVT.getTypeForEVT(F->getContext())));
1862 if (P.getNode())
1863 P.getNode()->setIROrder(idx + 1);
1864
1865 for (unsigned j = 0; j < VecSize; ++j) {
1866 if (i + j >= NumElts)
1867 break;
1868 SDValue Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, P,
1869 DAG.getIntPtrConstant(j));
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]1870 if (Ins[InsIdx].VT.getSizeInBits() > EltVT.getSizeInBits())
Justin Holewinskia2911282013-07-01 12:58:58 +0000[diff] [blame]1871 Elt = DAG.getNode(ISD::ANY_EXTEND, dl, Ins[InsIdx].VT, Elt);
Justin Holewinski44f5c602013-06-28 17:57:53 +0000[diff] [blame]1872 InVals.push_back(Elt);
1873 }
1874 Ofst += TD->getTypeAllocSize(VecVT.getTypeForEVT(F->getContext()));
Justin Holewinski44f5c602013-06-28 17:57:53 +0000[diff] [blame]1875 }
Justin Holewinski4f5bc9b2013-11-11 19:28:16 +0000[diff] [blame]1876 InsIdx += NumElts;
Justin Holewinski44f5c602013-06-28 17:57:53 +0000[diff] [blame]1877 }
1878
1879 if (NumElts > 0)
1880 --InsIdx;
1881 continue;
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]1882 }
Justin Holewinski44f5c602013-06-28 17:57:53 +0000[diff] [blame]1883 // A plain scalar.
1884 EVT ObjectVT = getValueType(Ty);
Justin Holewinski44f5c602013-06-28 17:57:53 +0000[diff] [blame]1885 // If ABI, load from the param symbol
1886 SDValue Arg = getParamSymbol(DAG, idx, getPointerTy());
1887 Value *srcValue = Constant::getNullValue(PointerType::get(
1888 ObjectVT.getTypeForEVT(F->getContext()), llvm::ADDRESS_SPACE_PARAM));
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]1889 SDValue p;
Justin Holewinskia2911282013-07-01 12:58:58 +0000[diff] [blame]1890 if (ObjectVT.getSizeInBits() < Ins[InsIdx].VT.getSizeInBits()) {
1891 ISD::LoadExtType ExtOp = Ins[InsIdx].Flags.isSExt() ?
1892 ISD::SEXTLOAD : ISD::ZEXTLOAD;
1893 p = DAG.getExtLoad(ExtOp, dl, Ins[InsIdx].VT, Root, Arg,
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]1894 MachinePointerInfo(srcValue), ObjectVT, false, false,
Justin Holewinskia2911282013-07-01 12:58:58 +0000[diff] [blame]1895 TD->getABITypeAlignment(ObjectVT.getTypeForEVT(F->getContext())));
1896 } else {
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]1897 p = DAG.getLoad(Ins[InsIdx].VT, dl, Root, Arg,
1898 MachinePointerInfo(srcValue), false, false, false,
Justin Holewinskia2911282013-07-01 12:58:58 +0000[diff] [blame]1899 TD->getABITypeAlignment(ObjectVT.getTypeForEVT(F->getContext())));
1900 }
Justin Holewinski44f5c602013-06-28 17:57:53 +0000[diff] [blame]1901 if (p.getNode())
1902 p.getNode()->setIROrder(idx + 1);
1903 InVals.push_back(p);
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]1904 continue;
1905 }
1906
1907 // Param has ByVal attribute
Justin Holewinski44f5c602013-06-28 17:57:53 +0000[diff] [blame]1908 // Return MoveParam(param symbol).
1909 // Ideally, the param symbol can be returned directly,
1910 // but when SDNode builder decides to use it in a CopyToReg(),
1911 // machine instruction fails because TargetExternalSymbol
1912 // (not lowered) is target dependent, and CopyToReg assumes
1913 // the source is lowered.
1914 EVT ObjectVT = getValueType(Ty);
1915 assert(ObjectVT == Ins[InsIdx].VT &&
1916 "Ins type did not match function type");
1917 SDValue Arg = getParamSymbol(DAG, idx, getPointerTy());
1918 SDValue p = DAG.getNode(NVPTXISD::MoveParam, dl, ObjectVT, Arg);
1919 if (p.getNode())
1920 p.getNode()->setIROrder(idx + 1);
1921 if (isKernel)
1922 InVals.push_back(p);
1923 else {
1924 SDValue p2 = DAG.getNode(
1925 ISD::INTRINSIC_WO_CHAIN, dl, ObjectVT,
1926 DAG.getConstant(Intrinsic::nvvm_ptr_local_to_gen, MVT::i32), p);
1927 InVals.push_back(p2);
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]1928 }
1929 }
1930
1931 // Clang will check explicit VarArg and issue error if any. However, Clang
1932 // will let code with
Justin Holewinski44f5c602013-06-28 17:57:53 +0000[diff] [blame]1933 // implicit var arg like f() pass. See bug 617733.
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]1934 // We treat this case as if the arg list is empty.
Justin Holewinski44f5c602013-06-28 17:57:53 +0000[diff] [blame]1935 // if (F.isVarArg()) {
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]1936 // assert(0 && "VarArg not supported yet!");
1937 //}
1938
1939 if (!OutChains.empty())
Craig Topper48d114b2014-04-26 18:35:24 +0000[diff] [blame]1940 DAG.setRoot(DAG.getNode(ISD::TokenFactor, dl, MVT::Other, OutChains));
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]1941
1942 return Chain;
1943}
1944
Justin Holewinski44f5c602013-06-28 17:57:53 +0000[diff] [blame]1945
Justin Holewinski120baee2013-06-28 17:57:55 +0000[diff] [blame]1946SDValue
1947NVPTXTargetLowering::LowerReturn(SDValue Chain, CallingConv::ID CallConv,
1948 bool isVarArg,
1949 const SmallVectorImpl<ISD::OutputArg> &Outs,
1950 const SmallVectorImpl<SDValue> &OutVals,
1951 SDLoc dl, SelectionDAG &DAG) const {
1952 MachineFunction &MF = DAG.getMachineFunction();
1953 const Function *F = MF.getFunction();
Justin Holewinskie04e4bd2013-06-28 17:58:10 +0000[diff] [blame]1954 Type *RetTy = F->getReturnType();
Justin Holewinski120baee2013-06-28 17:57:55 +0000[diff] [blame]1955 const DataLayout *TD = getDataLayout();
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]1956
1957 bool isABI = (nvptxSubtarget.getSmVersion() >= 20);
Justin Holewinski120baee2013-06-28 17:57:55 +0000[diff] [blame]1958 assert(isABI && "Non-ABI compilation is not supported");
1959 if (!isABI)
1960 return Chain;
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]1961
Justin Holewinskie04e4bd2013-06-28 17:58:10 +0000[diff] [blame]1962 if (VectorType *VTy = dyn_cast<VectorType>(RetTy)) {
Justin Holewinski120baee2013-06-28 17:57:55 +0000[diff] [blame]1963 // If we have a vector type, the OutVals array will be the scalarized
1964 // components and we have combine them into 1 or more vector stores.
1965 unsigned NumElts = VTy->getNumElements();
1966 assert(NumElts == Outs.size() && "Bad scalarization of return value");
1967
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]1968 // const_cast can be removed in later LLVM versions
Justin Holewinskie04e4bd2013-06-28 17:58:10 +0000[diff] [blame]1969 EVT EltVT = getValueType(RetTy).getVectorElementType();
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]1970 bool NeedExtend = false;
1971 if (EltVT.getSizeInBits() < 16)
1972 NeedExtend = true;
1973
Justin Holewinski120baee2013-06-28 17:57:55 +0000[diff] [blame]1974 // V1 store
1975 if (NumElts == 1) {
1976 SDValue StoreVal = OutVals[0];
1977 // We only have one element, so just directly store it
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]1978 if (NeedExtend)
1979 StoreVal = DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i16, StoreVal);
1980 SDValue Ops[] = { Chain, DAG.getConstant(0, MVT::i32), StoreVal };
1981 Chain = DAG.getMemIntrinsicNode(NVPTXISD::StoreRetval, dl,
Craig Topper206fcd42014-04-26 19:29:41 +0000[diff] [blame]1982 DAG.getVTList(MVT::Other), Ops,
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]1983 EltVT, MachinePointerInfo());
1984
Justin Holewinski120baee2013-06-28 17:57:55 +0000[diff] [blame]1985 } else if (NumElts == 2) {
1986 // V2 store
1987 SDValue StoreVal0 = OutVals[0];
1988 SDValue StoreVal1 = OutVals[1];
1989
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]1990 if (NeedExtend) {
1991 StoreVal0 = DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i16, StoreVal0);
1992 StoreVal1 = DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i16, StoreVal1);
Justin Holewinski120baee2013-06-28 17:57:55 +0000[diff] [blame]1993 }
1994
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]1995 SDValue Ops[] = { Chain, DAG.getConstant(0, MVT::i32), StoreVal0,
1996 StoreVal1 };
1997 Chain = DAG.getMemIntrinsicNode(NVPTXISD::StoreRetvalV2, dl,
Craig Topper206fcd42014-04-26 19:29:41 +0000[diff] [blame]1998 DAG.getVTList(MVT::Other), Ops,
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]1999 EltVT, MachinePointerInfo());
Justin Holewinski120baee2013-06-28 17:57:55 +0000[diff] [blame]2000 } else {
2001 // V4 stores
2002 // We have at least 4 elements (<3 x Ty> expands to 4 elements) and the
2003 // vector will be expanded to a power of 2 elements, so we know we can
2004 // always round up to the next multiple of 4 when creating the vector
2005 // stores.
2006 // e.g. 4 elem => 1 st.v4
2007 // 6 elem => 2 st.v4
2008 // 8 elem => 2 st.v4
2009 // 11 elem => 3 st.v4
2010
2011 unsigned VecSize = 4;
2012 if (OutVals[0].getValueType().getSizeInBits() == 64)
2013 VecSize = 2;
2014
2015 unsigned Offset = 0;
2016
2017 EVT VecVT =
Justin Holewinskib5db95e2014-06-27 18:36:04 +0000[diff] [blame]2018 EVT::getVectorVT(F->getContext(), EltVT, VecSize);
Justin Holewinski120baee2013-06-28 17:57:55 +0000[diff] [blame]2019 unsigned PerStoreOffset =
2020 TD->getTypeAllocSize(VecVT.getTypeForEVT(F->getContext()));
2021
Justin Holewinski120baee2013-06-28 17:57:55 +0000[diff] [blame]2022 for (unsigned i = 0; i < NumElts; i += VecSize) {
2023 // Get values
2024 SDValue StoreVal;
2025 SmallVector<SDValue, 8> Ops;
2026 Ops.push_back(Chain);
2027 Ops.push_back(DAG.getConstant(Offset, MVT::i32));
2028 unsigned Opc = NVPTXISD::StoreRetvalV2;
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]2029 EVT ExtendedVT = (NeedExtend) ? MVT::i16 : OutVals[0].getValueType();
Justin Holewinski120baee2013-06-28 17:57:55 +0000[diff] [blame]2030
2031 StoreVal = OutVals[i];
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]2032 if (NeedExtend)
2033 StoreVal = DAG.getNode(ISD::ZERO_EXTEND, dl, ExtendedVT, StoreVal);
Justin Holewinski120baee2013-06-28 17:57:55 +0000[diff] [blame]2034 Ops.push_back(StoreVal);
2035
2036 if (i + 1 < NumElts) {
2037 StoreVal = OutVals[i + 1];
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]2038 if (NeedExtend)
2039 StoreVal = DAG.getNode(ISD::ZERO_EXTEND, dl, ExtendedVT, StoreVal);
Justin Holewinski120baee2013-06-28 17:57:55 +0000[diff] [blame]2040 } else {
2041 StoreVal = DAG.getUNDEF(ExtendedVT);
2042 }
2043 Ops.push_back(StoreVal);
2044
2045 if (VecSize == 4) {
2046 Opc = NVPTXISD::StoreRetvalV4;
2047 if (i + 2 < NumElts) {
2048 StoreVal = OutVals[i + 2];
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]2049 if (NeedExtend)
2050 StoreVal =
2051 DAG.getNode(ISD::ZERO_EXTEND, dl, ExtendedVT, StoreVal);
Justin Holewinski120baee2013-06-28 17:57:55 +0000[diff] [blame]2052 } else {
2053 StoreVal = DAG.getUNDEF(ExtendedVT);
2054 }
2055 Ops.push_back(StoreVal);
2056
2057 if (i + 3 < NumElts) {
2058 StoreVal = OutVals[i + 3];
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]2059 if (NeedExtend)
2060 StoreVal =
2061 DAG.getNode(ISD::ZERO_EXTEND, dl, ExtendedVT, StoreVal);
Justin Holewinski120baee2013-06-28 17:57:55 +0000[diff] [blame]2062 } else {
2063 StoreVal = DAG.getUNDEF(ExtendedVT);
2064 }
2065 Ops.push_back(StoreVal);
2066 }
2067
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]2068 // Chain = DAG.getNode(Opc, dl, MVT::Other, &Ops[0], Ops.size());
2069 Chain =
Craig Topper206fcd42014-04-26 19:29:41 +0000[diff] [blame]2070 DAG.getMemIntrinsicNode(Opc, dl, DAG.getVTList(MVT::Other), Ops,
2071 EltVT, MachinePointerInfo());
Justin Holewinski120baee2013-06-28 17:57:55 +0000[diff] [blame]2072 Offset += PerStoreOffset;
2073 }
2074 }
2075 } else {
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]2076 SmallVector<EVT, 16> ValVTs;
Justin Holewinskib5db95e2014-06-27 18:36:04 +0000[diff] [blame]2077 SmallVector<uint64_t, 16> Offsets;
2078 ComputePTXValueVTs(*this, RetTy, ValVTs, &Offsets, 0);
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]2079 assert(ValVTs.size() == OutVals.size() && "Bad return value decomposition");
2080
Justin Holewinski120baee2013-06-28 17:57:55 +0000[diff] [blame]2081 for (unsigned i = 0, e = Outs.size(); i != e; ++i) {
2082 SDValue theVal = OutVals[i];
Justin Holewinskie04e4bd2013-06-28 17:58:10 +0000[diff] [blame]2083 EVT TheValType = theVal.getValueType();
Justin Holewinski120baee2013-06-28 17:57:55 +0000[diff] [blame]2084 unsigned numElems = 1;
Justin Holewinskie04e4bd2013-06-28 17:58:10 +0000[diff] [blame]2085 if (TheValType.isVector())
2086 numElems = TheValType.getVectorNumElements();
Justin Holewinski120baee2013-06-28 17:57:55 +0000[diff] [blame]2087 for (unsigned j = 0, je = numElems; j != je; ++j) {
Justin Holewinskie04e4bd2013-06-28 17:58:10 +0000[diff] [blame]2088 SDValue TmpVal = theVal;
2089 if (TheValType.isVector())
2090 TmpVal = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl,
2091 TheValType.getVectorElementType(), TmpVal,
Justin Holewinski120baee2013-06-28 17:57:55 +0000[diff] [blame]2092 DAG.getIntPtrConstant(j));
Justin Holewinskie04e4bd2013-06-28 17:58:10 +0000[diff] [blame]2093 EVT TheStoreType = ValVTs[i];
2094 if (RetTy->isIntegerTy() &&
2095 TD->getTypeAllocSizeInBits(RetTy) < 32) {
2096 // The following zero-extension is for integer types only, and
2097 // specifically not for aggregates.
2098 TmpVal = DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i32, TmpVal);
2099 TheStoreType = MVT::i32;
2100 }
2101 else if (TmpVal.getValueType().getSizeInBits() < 16)
2102 TmpVal = DAG.getNode(ISD::ANY_EXTEND, dl, MVT::i16, TmpVal);
2103
Justin Holewinskib5db95e2014-06-27 18:36:04 +0000[diff] [blame]2104 SDValue Ops[] = {
2105 Chain,
2106 DAG.getConstant(Offsets[i], MVT::i32),
2107 TmpVal };
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]2108 Chain = DAG.getMemIntrinsicNode(NVPTXISD::StoreRetval, dl,
Craig Topper206fcd42014-04-26 19:29:41 +0000[diff] [blame]2109 DAG.getVTList(MVT::Other), Ops,
2110 TheStoreType,
Justin Holewinskie04e4bd2013-06-28 17:58:10 +0000[diff] [blame]2111 MachinePointerInfo());
Justin Holewinski120baee2013-06-28 17:57:55 +0000[diff] [blame]2112 }
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]2113 }
2114 }
2115
2116 return DAG.getNode(NVPTXISD::RET_FLAG, dl, MVT::Other, Chain);
2117}
2118
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]2119
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2120void NVPTXTargetLowering::LowerAsmOperandForConstraint(
2121 SDValue Op, std::string &Constraint, std::vector<SDValue> &Ops,
2122 SelectionDAG &DAG) const {
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]2123 if (Constraint.length() > 1)
2124 return;
2125 else
2126 TargetLowering::LowerAsmOperandForConstraint(Op, Constraint, Ops, DAG);
2127}
2128
2129// NVPTX suuport vector of legal types of any length in Intrinsics because the
2130// NVPTX specific type legalizer
2131// will legalize them to the PTX supported length.
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2132bool NVPTXTargetLowering::isTypeSupportedInIntrinsic(MVT VT) const {
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]2133 if (isTypeLegal(VT))
2134 return true;
2135 if (VT.isVector()) {
2136 MVT eVT = VT.getVectorElementType();
2137 if (isTypeLegal(eVT))
2138 return true;
2139 }
2140 return false;
2141}
2142
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]2143static unsigned getOpcForTextureInstr(unsigned Intrinsic) {
2144 switch (Intrinsic) {
2145 default:
2146 return 0;
2147
2148 case Intrinsic::nvvm_tex_1d_v4f32_i32:
2149 return NVPTXISD::Tex1DFloatI32;
2150 case Intrinsic::nvvm_tex_1d_v4f32_f32:
2151 return NVPTXISD::Tex1DFloatFloat;
2152 case Intrinsic::nvvm_tex_1d_level_v4f32_f32:
2153 return NVPTXISD::Tex1DFloatFloatLevel;
2154 case Intrinsic::nvvm_tex_1d_grad_v4f32_f32:
2155 return NVPTXISD::Tex1DFloatFloatGrad;
2156 case Intrinsic::nvvm_tex_1d_v4i32_i32:
2157 return NVPTXISD::Tex1DI32I32;
2158 case Intrinsic::nvvm_tex_1d_v4i32_f32:
2159 return NVPTXISD::Tex1DI32Float;
2160 case Intrinsic::nvvm_tex_1d_level_v4i32_f32:
2161 return NVPTXISD::Tex1DI32FloatLevel;
2162 case Intrinsic::nvvm_tex_1d_grad_v4i32_f32:
2163 return NVPTXISD::Tex1DI32FloatGrad;
2164
2165 case Intrinsic::nvvm_tex_1d_array_v4f32_i32:
2166 return NVPTXISD::Tex1DArrayFloatI32;
2167 case Intrinsic::nvvm_tex_1d_array_v4f32_f32:
2168 return NVPTXISD::Tex1DArrayFloatFloat;
2169 case Intrinsic::nvvm_tex_1d_array_level_v4f32_f32:
2170 return NVPTXISD::Tex1DArrayFloatFloatLevel;
2171 case Intrinsic::nvvm_tex_1d_array_grad_v4f32_f32:
2172 return NVPTXISD::Tex1DArrayFloatFloatGrad;
2173 case Intrinsic::nvvm_tex_1d_array_v4i32_i32:
2174 return NVPTXISD::Tex1DArrayI32I32;
2175 case Intrinsic::nvvm_tex_1d_array_v4i32_f32:
2176 return NVPTXISD::Tex1DArrayI32Float;
2177 case Intrinsic::nvvm_tex_1d_array_level_v4i32_f32:
2178 return NVPTXISD::Tex1DArrayI32FloatLevel;
2179 case Intrinsic::nvvm_tex_1d_array_grad_v4i32_f32:
2180 return NVPTXISD::Tex1DArrayI32FloatGrad;
2181
2182 case Intrinsic::nvvm_tex_2d_v4f32_i32:
2183 return NVPTXISD::Tex2DFloatI32;
2184 case Intrinsic::nvvm_tex_2d_v4f32_f32:
2185 return NVPTXISD::Tex2DFloatFloat;
2186 case Intrinsic::nvvm_tex_2d_level_v4f32_f32:
2187 return NVPTXISD::Tex2DFloatFloatLevel;
2188 case Intrinsic::nvvm_tex_2d_grad_v4f32_f32:
2189 return NVPTXISD::Tex2DFloatFloatGrad;
2190 case Intrinsic::nvvm_tex_2d_v4i32_i32:
2191 return NVPTXISD::Tex2DI32I32;
2192 case Intrinsic::nvvm_tex_2d_v4i32_f32:
2193 return NVPTXISD::Tex2DI32Float;
2194 case Intrinsic::nvvm_tex_2d_level_v4i32_f32:
2195 return NVPTXISD::Tex2DI32FloatLevel;
2196 case Intrinsic::nvvm_tex_2d_grad_v4i32_f32:
2197 return NVPTXISD::Tex2DI32FloatGrad;
2198
2199 case Intrinsic::nvvm_tex_2d_array_v4f32_i32:
2200 return NVPTXISD::Tex2DArrayFloatI32;
2201 case Intrinsic::nvvm_tex_2d_array_v4f32_f32:
2202 return NVPTXISD::Tex2DArrayFloatFloat;
2203 case Intrinsic::nvvm_tex_2d_array_level_v4f32_f32:
2204 return NVPTXISD::Tex2DArrayFloatFloatLevel;
2205 case Intrinsic::nvvm_tex_2d_array_grad_v4f32_f32:
2206 return NVPTXISD::Tex2DArrayFloatFloatGrad;
2207 case Intrinsic::nvvm_tex_2d_array_v4i32_i32:
2208 return NVPTXISD::Tex2DArrayI32I32;
2209 case Intrinsic::nvvm_tex_2d_array_v4i32_f32:
2210 return NVPTXISD::Tex2DArrayI32Float;
2211 case Intrinsic::nvvm_tex_2d_array_level_v4i32_f32:
2212 return NVPTXISD::Tex2DArrayI32FloatLevel;
2213 case Intrinsic::nvvm_tex_2d_array_grad_v4i32_f32:
2214 return NVPTXISD::Tex2DArrayI32FloatGrad;
2215
2216 case Intrinsic::nvvm_tex_3d_v4f32_i32:
2217 return NVPTXISD::Tex3DFloatI32;
2218 case Intrinsic::nvvm_tex_3d_v4f32_f32:
2219 return NVPTXISD::Tex3DFloatFloat;
2220 case Intrinsic::nvvm_tex_3d_level_v4f32_f32:
2221 return NVPTXISD::Tex3DFloatFloatLevel;
2222 case Intrinsic::nvvm_tex_3d_grad_v4f32_f32:
2223 return NVPTXISD::Tex3DFloatFloatGrad;
2224 case Intrinsic::nvvm_tex_3d_v4i32_i32:
2225 return NVPTXISD::Tex3DI32I32;
2226 case Intrinsic::nvvm_tex_3d_v4i32_f32:
2227 return NVPTXISD::Tex3DI32Float;
2228 case Intrinsic::nvvm_tex_3d_level_v4i32_f32:
2229 return NVPTXISD::Tex3DI32FloatLevel;
2230 case Intrinsic::nvvm_tex_3d_grad_v4i32_f32:
2231 return NVPTXISD::Tex3DI32FloatGrad;
2232 }
2233}
2234
2235static unsigned getOpcForSurfaceInstr(unsigned Intrinsic) {
2236 switch (Intrinsic) {
2237 default:
2238 return 0;
2239 case Intrinsic::nvvm_suld_1d_i8_trap:
2240 return NVPTXISD::Suld1DI8Trap;
2241 case Intrinsic::nvvm_suld_1d_i16_trap:
2242 return NVPTXISD::Suld1DI16Trap;
2243 case Intrinsic::nvvm_suld_1d_i32_trap:
2244 return NVPTXISD::Suld1DI32Trap;
2245 case Intrinsic::nvvm_suld_1d_v2i8_trap:
2246 return NVPTXISD::Suld1DV2I8Trap;
2247 case Intrinsic::nvvm_suld_1d_v2i16_trap:
2248 return NVPTXISD::Suld1DV2I16Trap;
2249 case Intrinsic::nvvm_suld_1d_v2i32_trap:
2250 return NVPTXISD::Suld1DV2I32Trap;
2251 case Intrinsic::nvvm_suld_1d_v4i8_trap:
2252 return NVPTXISD::Suld1DV4I8Trap;
2253 case Intrinsic::nvvm_suld_1d_v4i16_trap:
2254 return NVPTXISD::Suld1DV4I16Trap;
2255 case Intrinsic::nvvm_suld_1d_v4i32_trap:
2256 return NVPTXISD::Suld1DV4I32Trap;
2257 case Intrinsic::nvvm_suld_1d_array_i8_trap:
2258 return NVPTXISD::Suld1DArrayI8Trap;
2259 case Intrinsic::nvvm_suld_1d_array_i16_trap:
2260 return NVPTXISD::Suld1DArrayI16Trap;
2261 case Intrinsic::nvvm_suld_1d_array_i32_trap:
2262 return NVPTXISD::Suld1DArrayI32Trap;
2263 case Intrinsic::nvvm_suld_1d_array_v2i8_trap:
2264 return NVPTXISD::Suld1DArrayV2I8Trap;
2265 case Intrinsic::nvvm_suld_1d_array_v2i16_trap:
2266 return NVPTXISD::Suld1DArrayV2I16Trap;
2267 case Intrinsic::nvvm_suld_1d_array_v2i32_trap:
2268 return NVPTXISD::Suld1DArrayV2I32Trap;
2269 case Intrinsic::nvvm_suld_1d_array_v4i8_trap:
2270 return NVPTXISD::Suld1DArrayV4I8Trap;
2271 case Intrinsic::nvvm_suld_1d_array_v4i16_trap:
2272 return NVPTXISD::Suld1DArrayV4I16Trap;
2273 case Intrinsic::nvvm_suld_1d_array_v4i32_trap:
2274 return NVPTXISD::Suld1DArrayV4I32Trap;
2275 case Intrinsic::nvvm_suld_2d_i8_trap:
2276 return NVPTXISD::Suld2DI8Trap;
2277 case Intrinsic::nvvm_suld_2d_i16_trap:
2278 return NVPTXISD::Suld2DI16Trap;
2279 case Intrinsic::nvvm_suld_2d_i32_trap:
2280 return NVPTXISD::Suld2DI32Trap;
2281 case Intrinsic::nvvm_suld_2d_v2i8_trap:
2282 return NVPTXISD::Suld2DV2I8Trap;
2283 case Intrinsic::nvvm_suld_2d_v2i16_trap:
2284 return NVPTXISD::Suld2DV2I16Trap;
2285 case Intrinsic::nvvm_suld_2d_v2i32_trap:
2286 return NVPTXISD::Suld2DV2I32Trap;
2287 case Intrinsic::nvvm_suld_2d_v4i8_trap:
2288 return NVPTXISD::Suld2DV4I8Trap;
2289 case Intrinsic::nvvm_suld_2d_v4i16_trap:
2290 return NVPTXISD::Suld2DV4I16Trap;
2291 case Intrinsic::nvvm_suld_2d_v4i32_trap:
2292 return NVPTXISD::Suld2DV4I32Trap;
2293 case Intrinsic::nvvm_suld_2d_array_i8_trap:
2294 return NVPTXISD::Suld2DArrayI8Trap;
2295 case Intrinsic::nvvm_suld_2d_array_i16_trap:
2296 return NVPTXISD::Suld2DArrayI16Trap;
2297 case Intrinsic::nvvm_suld_2d_array_i32_trap:
2298 return NVPTXISD::Suld2DArrayI32Trap;
2299 case Intrinsic::nvvm_suld_2d_array_v2i8_trap:
2300 return NVPTXISD::Suld2DArrayV2I8Trap;
2301 case Intrinsic::nvvm_suld_2d_array_v2i16_trap:
2302 return NVPTXISD::Suld2DArrayV2I16Trap;
2303 case Intrinsic::nvvm_suld_2d_array_v2i32_trap:
2304 return NVPTXISD::Suld2DArrayV2I32Trap;
2305 case Intrinsic::nvvm_suld_2d_array_v4i8_trap:
2306 return NVPTXISD::Suld2DArrayV4I8Trap;
2307 case Intrinsic::nvvm_suld_2d_array_v4i16_trap:
2308 return NVPTXISD::Suld2DArrayV4I16Trap;
2309 case Intrinsic::nvvm_suld_2d_array_v4i32_trap:
2310 return NVPTXISD::Suld2DArrayV4I32Trap;
2311 case Intrinsic::nvvm_suld_3d_i8_trap:
2312 return NVPTXISD::Suld3DI8Trap;
2313 case Intrinsic::nvvm_suld_3d_i16_trap:
2314 return NVPTXISD::Suld3DI16Trap;
2315 case Intrinsic::nvvm_suld_3d_i32_trap:
2316 return NVPTXISD::Suld3DI32Trap;
2317 case Intrinsic::nvvm_suld_3d_v2i8_trap:
2318 return NVPTXISD::Suld3DV2I8Trap;
2319 case Intrinsic::nvvm_suld_3d_v2i16_trap:
2320 return NVPTXISD::Suld3DV2I16Trap;
2321 case Intrinsic::nvvm_suld_3d_v2i32_trap:
2322 return NVPTXISD::Suld3DV2I32Trap;
2323 case Intrinsic::nvvm_suld_3d_v4i8_trap:
2324 return NVPTXISD::Suld3DV4I8Trap;
2325 case Intrinsic::nvvm_suld_3d_v4i16_trap:
2326 return NVPTXISD::Suld3DV4I16Trap;
2327 case Intrinsic::nvvm_suld_3d_v4i32_trap:
2328 return NVPTXISD::Suld3DV4I32Trap;
2329 }
2330}
2331
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]2332// llvm.ptx.memcpy.const and llvm.ptx.memmove.const need to be modeled as
2333// TgtMemIntrinsic
2334// because we need the information that is only available in the "Value" type
2335// of destination
2336// pointer. In particular, the address space information.
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2337bool NVPTXTargetLowering::getTgtMemIntrinsic(
2338 IntrinsicInfo &Info, const CallInst &I, unsigned Intrinsic) const {
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]2339 switch (Intrinsic) {
2340 default:
2341 return false;
2342
2343 case Intrinsic::nvvm_atomic_load_add_f32:
2344 Info.opc = ISD::INTRINSIC_W_CHAIN;
2345 Info.memVT = MVT::f32;
2346 Info.ptrVal = I.getArgOperand(0);
2347 Info.offset = 0;
2348 Info.vol = 0;
2349 Info.readMem = true;
2350 Info.writeMem = true;
2351 Info.align = 0;
2352 return true;
2353
2354 case Intrinsic::nvvm_atomic_load_inc_32:
2355 case Intrinsic::nvvm_atomic_load_dec_32:
2356 Info.opc = ISD::INTRINSIC_W_CHAIN;
2357 Info.memVT = MVT::i32;
2358 Info.ptrVal = I.getArgOperand(0);
2359 Info.offset = 0;
2360 Info.vol = 0;
2361 Info.readMem = true;
2362 Info.writeMem = true;
2363 Info.align = 0;
2364 return true;
2365
2366 case Intrinsic::nvvm_ldu_global_i:
2367 case Intrinsic::nvvm_ldu_global_f:
Justin Holewinskib926d9d2014-06-27 18:35:51 +0000[diff] [blame]2368 case Intrinsic::nvvm_ldu_global_p: {
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]2369
2370 Info.opc = ISD::INTRINSIC_W_CHAIN;
2371 if (Intrinsic == Intrinsic::nvvm_ldu_global_i)
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]2372 Info.memVT = getValueType(I.getType());
Justin Holewinskib926d9d2014-06-27 18:35:51 +0000[diff] [blame]2373 else if(Intrinsic == Intrinsic::nvvm_ldu_global_p)
2374 Info.memVT = getPointerTy();
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]2375 else
Justin Holewinskib926d9d2014-06-27 18:35:51 +0000[diff] [blame]2376 Info.memVT = getValueType(I.getType());
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]2377 Info.ptrVal = I.getArgOperand(0);
2378 Info.offset = 0;
2379 Info.vol = 0;
2380 Info.readMem = true;
2381 Info.writeMem = false;
Justin Holewinskib926d9d2014-06-27 18:35:51 +0000[diff] [blame]2382
2383 // alignment is available as metadata.
2384 // Grab it and set the alignment.
2385 assert(I.hasMetadataOtherThanDebugLoc() && "Must have alignment metadata");
2386 MDNode *AlignMD = I.getMetadata("align");
2387 assert(AlignMD && "Must have a non-null MDNode");
2388 assert(AlignMD->getNumOperands() == 1 && "Must have a single operand");
2389 Value *Align = AlignMD->getOperand(0);
2390 int64_t Alignment = cast<ConstantInt>(Align)->getZExtValue();
2391 Info.align = Alignment;
2392
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]2393 return true;
Justin Holewinskib926d9d2014-06-27 18:35:51 +0000[diff] [blame]2394 }
2395 case Intrinsic::nvvm_ldg_global_i:
2396 case Intrinsic::nvvm_ldg_global_f:
2397 case Intrinsic::nvvm_ldg_global_p: {
2398
2399 Info.opc = ISD::INTRINSIC_W_CHAIN;
2400 if (Intrinsic == Intrinsic::nvvm_ldg_global_i)
2401 Info.memVT = getValueType(I.getType());
2402 else if(Intrinsic == Intrinsic::nvvm_ldg_global_p)
2403 Info.memVT = getPointerTy();
2404 else
2405 Info.memVT = getValueType(I.getType());
2406 Info.ptrVal = I.getArgOperand(0);
2407 Info.offset = 0;
2408 Info.vol = 0;
2409 Info.readMem = true;
2410 Info.writeMem = false;
2411
2412 // alignment is available as metadata.
2413 // Grab it and set the alignment.
2414 assert(I.hasMetadataOtherThanDebugLoc() && "Must have alignment metadata");
2415 MDNode *AlignMD = I.getMetadata("align");
2416 assert(AlignMD && "Must have a non-null MDNode");
2417 assert(AlignMD->getNumOperands() == 1 && "Must have a single operand");
2418 Value *Align = AlignMD->getOperand(0);
2419 int64_t Alignment = cast<ConstantInt>(Align)->getZExtValue();
2420 Info.align = Alignment;
2421
2422 return true;
2423 }
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]2424
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]2425 case Intrinsic::nvvm_tex_1d_v4f32_i32:
2426 case Intrinsic::nvvm_tex_1d_v4f32_f32:
2427 case Intrinsic::nvvm_tex_1d_level_v4f32_f32:
2428 case Intrinsic::nvvm_tex_1d_grad_v4f32_f32:
2429 case Intrinsic::nvvm_tex_1d_array_v4f32_i32:
2430 case Intrinsic::nvvm_tex_1d_array_v4f32_f32:
2431 case Intrinsic::nvvm_tex_1d_array_level_v4f32_f32:
2432 case Intrinsic::nvvm_tex_1d_array_grad_v4f32_f32:
2433 case Intrinsic::nvvm_tex_2d_v4f32_i32:
2434 case Intrinsic::nvvm_tex_2d_v4f32_f32:
2435 case Intrinsic::nvvm_tex_2d_level_v4f32_f32:
2436 case Intrinsic::nvvm_tex_2d_grad_v4f32_f32:
2437 case Intrinsic::nvvm_tex_2d_array_v4f32_i32:
2438 case Intrinsic::nvvm_tex_2d_array_v4f32_f32:
2439 case Intrinsic::nvvm_tex_2d_array_level_v4f32_f32:
2440 case Intrinsic::nvvm_tex_2d_array_grad_v4f32_f32:
2441 case Intrinsic::nvvm_tex_3d_v4f32_i32:
2442 case Intrinsic::nvvm_tex_3d_v4f32_f32:
2443 case Intrinsic::nvvm_tex_3d_level_v4f32_f32:
2444 case Intrinsic::nvvm_tex_3d_grad_v4f32_f32: {
2445 Info.opc = getOpcForTextureInstr(Intrinsic);
2446 Info.memVT = MVT::f32;
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2447 Info.ptrVal = nullptr;
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]2448 Info.offset = 0;
2449 Info.vol = 0;
2450 Info.readMem = true;
2451 Info.writeMem = false;
2452 Info.align = 16;
2453 return true;
2454 }
2455 case Intrinsic::nvvm_tex_1d_v4i32_i32:
2456 case Intrinsic::nvvm_tex_1d_v4i32_f32:
2457 case Intrinsic::nvvm_tex_1d_level_v4i32_f32:
2458 case Intrinsic::nvvm_tex_1d_grad_v4i32_f32:
2459 case Intrinsic::nvvm_tex_1d_array_v4i32_i32:
2460 case Intrinsic::nvvm_tex_1d_array_v4i32_f32:
2461 case Intrinsic::nvvm_tex_1d_array_level_v4i32_f32:
2462 case Intrinsic::nvvm_tex_1d_array_grad_v4i32_f32:
2463 case Intrinsic::nvvm_tex_2d_v4i32_i32:
2464 case Intrinsic::nvvm_tex_2d_v4i32_f32:
2465 case Intrinsic::nvvm_tex_2d_level_v4i32_f32:
2466 case Intrinsic::nvvm_tex_2d_grad_v4i32_f32:
2467 case Intrinsic::nvvm_tex_2d_array_v4i32_i32:
2468 case Intrinsic::nvvm_tex_2d_array_v4i32_f32:
2469 case Intrinsic::nvvm_tex_2d_array_level_v4i32_f32:
2470 case Intrinsic::nvvm_tex_2d_array_grad_v4i32_f32:
2471 case Intrinsic::nvvm_tex_3d_v4i32_i32:
2472 case Intrinsic::nvvm_tex_3d_v4i32_f32:
2473 case Intrinsic::nvvm_tex_3d_level_v4i32_f32:
2474 case Intrinsic::nvvm_tex_3d_grad_v4i32_f32: {
2475 Info.opc = getOpcForTextureInstr(Intrinsic);
2476 Info.memVT = MVT::i32;
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2477 Info.ptrVal = nullptr;
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]2478 Info.offset = 0;
2479 Info.vol = 0;
2480 Info.readMem = true;
2481 Info.writeMem = false;
2482 Info.align = 16;
2483 return true;
2484 }
2485 case Intrinsic::nvvm_suld_1d_i8_trap:
2486 case Intrinsic::nvvm_suld_1d_v2i8_trap:
2487 case Intrinsic::nvvm_suld_1d_v4i8_trap:
2488 case Intrinsic::nvvm_suld_1d_array_i8_trap:
2489 case Intrinsic::nvvm_suld_1d_array_v2i8_trap:
2490 case Intrinsic::nvvm_suld_1d_array_v4i8_trap:
2491 case Intrinsic::nvvm_suld_2d_i8_trap:
2492 case Intrinsic::nvvm_suld_2d_v2i8_trap:
2493 case Intrinsic::nvvm_suld_2d_v4i8_trap:
2494 case Intrinsic::nvvm_suld_2d_array_i8_trap:
2495 case Intrinsic::nvvm_suld_2d_array_v2i8_trap:
2496 case Intrinsic::nvvm_suld_2d_array_v4i8_trap:
2497 case Intrinsic::nvvm_suld_3d_i8_trap:
2498 case Intrinsic::nvvm_suld_3d_v2i8_trap:
2499 case Intrinsic::nvvm_suld_3d_v4i8_trap: {
2500 Info.opc = getOpcForSurfaceInstr(Intrinsic);
2501 Info.memVT = MVT::i8;
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2502 Info.ptrVal = nullptr;
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]2503 Info.offset = 0;
2504 Info.vol = 0;
2505 Info.readMem = true;
2506 Info.writeMem = false;
2507 Info.align = 16;
2508 return true;
2509 }
2510 case Intrinsic::nvvm_suld_1d_i16_trap:
2511 case Intrinsic::nvvm_suld_1d_v2i16_trap:
2512 case Intrinsic::nvvm_suld_1d_v4i16_trap:
2513 case Intrinsic::nvvm_suld_1d_array_i16_trap:
2514 case Intrinsic::nvvm_suld_1d_array_v2i16_trap:
2515 case Intrinsic::nvvm_suld_1d_array_v4i16_trap:
2516 case Intrinsic::nvvm_suld_2d_i16_trap:
2517 case Intrinsic::nvvm_suld_2d_v2i16_trap:
2518 case Intrinsic::nvvm_suld_2d_v4i16_trap:
2519 case Intrinsic::nvvm_suld_2d_array_i16_trap:
2520 case Intrinsic::nvvm_suld_2d_array_v2i16_trap:
2521 case Intrinsic::nvvm_suld_2d_array_v4i16_trap:
2522 case Intrinsic::nvvm_suld_3d_i16_trap:
2523 case Intrinsic::nvvm_suld_3d_v2i16_trap:
2524 case Intrinsic::nvvm_suld_3d_v4i16_trap: {
2525 Info.opc = getOpcForSurfaceInstr(Intrinsic);
2526 Info.memVT = MVT::i16;
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2527 Info.ptrVal = nullptr;
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]2528 Info.offset = 0;
2529 Info.vol = 0;
2530 Info.readMem = true;
2531 Info.writeMem = false;
2532 Info.align = 16;
2533 return true;
2534 }
2535 case Intrinsic::nvvm_suld_1d_i32_trap:
2536 case Intrinsic::nvvm_suld_1d_v2i32_trap:
2537 case Intrinsic::nvvm_suld_1d_v4i32_trap:
2538 case Intrinsic::nvvm_suld_1d_array_i32_trap:
2539 case Intrinsic::nvvm_suld_1d_array_v2i32_trap:
2540 case Intrinsic::nvvm_suld_1d_array_v4i32_trap:
2541 case Intrinsic::nvvm_suld_2d_i32_trap:
2542 case Intrinsic::nvvm_suld_2d_v2i32_trap:
2543 case Intrinsic::nvvm_suld_2d_v4i32_trap:
2544 case Intrinsic::nvvm_suld_2d_array_i32_trap:
2545 case Intrinsic::nvvm_suld_2d_array_v2i32_trap:
2546 case Intrinsic::nvvm_suld_2d_array_v4i32_trap:
2547 case Intrinsic::nvvm_suld_3d_i32_trap:
2548 case Intrinsic::nvvm_suld_3d_v2i32_trap:
2549 case Intrinsic::nvvm_suld_3d_v4i32_trap: {
2550 Info.opc = getOpcForSurfaceInstr(Intrinsic);
2551 Info.memVT = MVT::i32;
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2552 Info.ptrVal = nullptr;
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]2553 Info.offset = 0;
2554 Info.vol = 0;
2555 Info.readMem = true;
2556 Info.writeMem = false;
2557 Info.align = 16;
2558 return true;
2559 }
2560
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]2561 }
2562 return false;
2563}
2564
2565/// isLegalAddressingMode - Return true if the addressing mode represented
2566/// by AM is legal for this target, for a load/store of the specified type.
2567/// Used to guide target specific optimizations, like loop strength reduction
2568/// (LoopStrengthReduce.cpp) and memory optimization for address mode
2569/// (CodeGenPrepare.cpp)
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2570bool NVPTXTargetLowering::isLegalAddressingMode(const AddrMode &AM,
2571 Type *Ty) const {
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]2572
2573 // AddrMode - This represents an addressing mode of:
2574 // BaseGV + BaseOffs + BaseReg + Scale*ScaleReg
2575 //
2576 // The legal address modes are
2577 // - [avar]
2578 // - [areg]
2579 // - [areg+immoff]
2580 // - [immAddr]
2581
2582 if (AM.BaseGV) {
2583 if (AM.BaseOffs || AM.HasBaseReg || AM.Scale)
2584 return false;
2585 return true;
2586 }
2587
2588 switch (AM.Scale) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2589 case 0: // "r", "r+i" or "i" is allowed
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]2590 break;
2591 case 1:
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2592 if (AM.HasBaseReg) // "r+r+i" or "r+r" is not allowed.
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]2593 return false;
2594 // Otherwise we have r+i.
2595 break;
2596 default:
2597 // No scale > 1 is allowed
2598 return false;
2599 }
2600 return true;
2601}
2602
2603//===----------------------------------------------------------------------===//
2604// NVPTX Inline Assembly Support
2605//===----------------------------------------------------------------------===//
2606
2607/// getConstraintType - Given a constraint letter, return the type of
2608/// constraint it is for this target.
2609NVPTXTargetLowering::ConstraintType
2610NVPTXTargetLowering::getConstraintType(const std::string &Constraint) const {
2611 if (Constraint.size() == 1) {
2612 switch (Constraint[0]) {
2613 default:
2614 break;
Justin Holewinski2739c012014-06-27 18:36:06 +0000[diff] [blame]2615 case 'b':
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]2616 case 'r':
2617 case 'h':
2618 case 'c':
2619 case 'l':
2620 case 'f':
2621 case 'd':
2622 case '0':
2623 case 'N':
2624 return C_RegisterClass;
2625 }
2626 }
2627 return TargetLowering::getConstraintType(Constraint);
2628}
2629
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2630std::pair<unsigned, const TargetRegisterClass *>
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]2631NVPTXTargetLowering::getRegForInlineAsmConstraint(const std::string &Constraint,
Chad Rosier295bd432013-06-22 18:37:38 +0000[diff] [blame]2632 MVT VT) const {
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]2633 if (Constraint.size() == 1) {
2634 switch (Constraint[0]) {
Justin Holewinski2739c012014-06-27 18:36:06 +0000[diff] [blame]2635 case 'b':
2636 return std::make_pair(0U, &NVPTX::Int1RegsRegClass);
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]2637 case 'c':
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]2638 return std::make_pair(0U, &NVPTX::Int16RegsRegClass);
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]2639 case 'h':
2640 return std::make_pair(0U, &NVPTX::Int16RegsRegClass);
2641 case 'r':
2642 return std::make_pair(0U, &NVPTX::Int32RegsRegClass);
2643 case 'l':
2644 case 'N':
2645 return std::make_pair(0U, &NVPTX::Int64RegsRegClass);
2646 case 'f':
2647 return std::make_pair(0U, &NVPTX::Float32RegsRegClass);
2648 case 'd':
2649 return std::make_pair(0U, &NVPTX::Float64RegsRegClass);
2650 }
2651 }
2652 return TargetLowering::getRegForInlineAsmConstraint(Constraint, VT);
2653}
2654
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]2655/// getFunctionAlignment - Return the Log2 alignment of this function.
2656unsigned NVPTXTargetLowering::getFunctionAlignment(const Function *) const {
2657 return 4;
2658}
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]2659
Justin Holewinskieafe26d2014-06-27 18:35:37 +0000[diff] [blame]2660//===----------------------------------------------------------------------===//
2661// NVPTX DAG Combining
2662//===----------------------------------------------------------------------===//
2663
2664extern unsigned FMAContractLevel;
2665
2666/// PerformADDCombineWithOperands - Try DAG combinations for an ADD with
2667/// operands N0 and N1. This is a helper for PerformADDCombine that is
2668/// called with the default operands, and if that fails, with commuted
2669/// operands.
2670static SDValue PerformADDCombineWithOperands(SDNode *N, SDValue N0, SDValue N1,
2671 TargetLowering::DAGCombinerInfo &DCI,
2672 const NVPTXSubtarget &Subtarget,
2673 CodeGenOpt::Level OptLevel) {
2674 SelectionDAG &DAG = DCI.DAG;
2675 // Skip non-integer, non-scalar case
2676 EVT VT=N0.getValueType();
2677 if (VT.isVector())
2678 return SDValue();
2679
2680 // fold (add (mul a, b), c) -> (mad a, b, c)
2681 //
2682 if (N0.getOpcode() == ISD::MUL) {
2683 assert (VT.isInteger());
2684 // For integer:
2685 // Since integer multiply-add costs the same as integer multiply
2686 // but is more costly than integer add, do the fusion only when
2687 // the mul is only used in the add.
2688 if (OptLevel==CodeGenOpt::None || VT != MVT::i32 ||
2689 !N0.getNode()->hasOneUse())
2690 return SDValue();
2691
2692 // Do the folding
2693 return DAG.getNode(NVPTXISD::IMAD, SDLoc(N), VT,
2694 N0.getOperand(0), N0.getOperand(1), N1);
2695 }
2696 else if (N0.getOpcode() == ISD::FMUL) {
2697 if (VT == MVT::f32 || VT == MVT::f64) {
2698 if (FMAContractLevel == 0)
2699 return SDValue();
2700
2701 // For floating point:
2702 // Do the fusion only when the mul has less than 5 uses and all
2703 // are add.
2704 // The heuristic is that if a use is not an add, then that use
2705 // cannot be fused into fma, therefore mul is still needed anyway.
2706 // If there are more than 4 uses, even if they are all add, fusing
2707 // them will increase register pressue.
2708 //
2709 int numUses = 0;
2710 int nonAddCount = 0;
2711 for (SDNode::use_iterator UI = N0.getNode()->use_begin(),
2712 UE = N0.getNode()->use_end();
2713 UI != UE; ++UI) {
2714 numUses++;
2715 SDNode *User = *UI;
2716 if (User->getOpcode() != ISD::FADD)
2717 ++nonAddCount;
2718 }
2719 if (numUses >= 5)
2720 return SDValue();
2721 if (nonAddCount) {
2722 int orderNo = N->getIROrder();
2723 int orderNo2 = N0.getNode()->getIROrder();
2724 // simple heuristics here for considering potential register
2725 // pressure, the logics here is that the differnce are used
2726 // to measure the distance between def and use, the longer distance
2727 // more likely cause register pressure.
2728 if (orderNo - orderNo2 < 500)
2729 return SDValue();
2730
2731 // Now, check if at least one of the FMUL's operands is live beyond the node N,
2732 // which guarantees that the FMA will not increase register pressure at node N.
2733 bool opIsLive = false;
2734 const SDNode *left = N0.getOperand(0).getNode();
2735 const SDNode *right = N0.getOperand(1).getNode();
2736
2737 if (dyn_cast<ConstantSDNode>(left) || dyn_cast<ConstantSDNode>(right))
2738 opIsLive = true;
2739
2740 if (!opIsLive)
2741 for (SDNode::use_iterator UI = left->use_begin(), UE = left->use_end(); UI != UE; ++UI) {
2742 SDNode *User = *UI;
2743 int orderNo3 = User->getIROrder();
2744 if (orderNo3 > orderNo) {
2745 opIsLive = true;
2746 break;
2747 }
2748 }
2749
2750 if (!opIsLive)
2751 for (SDNode::use_iterator UI = right->use_begin(), UE = right->use_end(); UI != UE; ++UI) {
2752 SDNode *User = *UI;
2753 int orderNo3 = User->getIROrder();
2754 if (orderNo3 > orderNo) {
2755 opIsLive = true;
2756 break;
2757 }
2758 }
2759
2760 if (!opIsLive)
2761 return SDValue();
2762 }
2763
2764 return DAG.getNode(ISD::FMA, SDLoc(N), VT,
2765 N0.getOperand(0), N0.getOperand(1), N1);
2766 }
2767 }
2768
2769 return SDValue();
2770}
2771
2772/// PerformADDCombine - Target-specific dag combine xforms for ISD::ADD.
2773///
2774static SDValue PerformADDCombine(SDNode *N,
2775 TargetLowering::DAGCombinerInfo &DCI,
2776 const NVPTXSubtarget &Subtarget,
2777 CodeGenOpt::Level OptLevel) {
2778 SDValue N0 = N->getOperand(0);
2779 SDValue N1 = N->getOperand(1);
2780
2781 // First try with the default operand order.
2782 SDValue Result = PerformADDCombineWithOperands(N, N0, N1, DCI, Subtarget,
2783 OptLevel);
2784 if (Result.getNode())
2785 return Result;
2786
2787 // If that didn't work, try again with the operands commuted.
2788 return PerformADDCombineWithOperands(N, N1, N0, DCI, Subtarget, OptLevel);
2789}
2790
2791static SDValue PerformANDCombine(SDNode *N,
2792 TargetLowering::DAGCombinerInfo &DCI) {
2793 // The type legalizer turns a vector load of i8 values into a zextload to i16
2794 // registers, optionally ANY_EXTENDs it (if target type is integer),
2795 // and ANDs off the high 8 bits. Since we turn this load into a
2796 // target-specific DAG node, the DAG combiner fails to eliminate these AND
2797 // nodes. Do that here.
2798 SDValue Val = N->getOperand(0);
2799 SDValue Mask = N->getOperand(1);
2800
2801 if (isa<ConstantSDNode>(Val)) {
2802 std::swap(Val, Mask);
2803 }
2804
2805 SDValue AExt;
2806 // Generally, we will see zextload -> IMOV16rr -> ANY_EXTEND -> and
2807 if (Val.getOpcode() == ISD::ANY_EXTEND) {
2808 AExt = Val;
2809 Val = Val->getOperand(0);
2810 }
2811
2812 if (Val->isMachineOpcode() && Val->getMachineOpcode() == NVPTX::IMOV16rr) {
2813 Val = Val->getOperand(0);
2814 }
2815
2816 if (Val->getOpcode() == NVPTXISD::LoadV2 ||
2817 Val->getOpcode() == NVPTXISD::LoadV4) {
2818 ConstantSDNode *MaskCnst = dyn_cast<ConstantSDNode>(Mask);
2819 if (!MaskCnst) {
2820 // Not an AND with a constant
2821 return SDValue();
2822 }
2823
2824 uint64_t MaskVal = MaskCnst->getZExtValue();
2825 if (MaskVal != 0xff) {
2826 // Not an AND that chops off top 8 bits
2827 return SDValue();
2828 }
2829
2830 MemSDNode *Mem = dyn_cast<MemSDNode>(Val);
2831 if (!Mem) {
2832 // Not a MemSDNode?!?
2833 return SDValue();
2834 }
2835
2836 EVT MemVT = Mem->getMemoryVT();
2837 if (MemVT != MVT::v2i8 && MemVT != MVT::v4i8) {
2838 // We only handle the i8 case
2839 return SDValue();
2840 }
2841
2842 unsigned ExtType =
2843 cast<ConstantSDNode>(Val->getOperand(Val->getNumOperands()-1))->
2844 getZExtValue();
2845 if (ExtType == ISD::SEXTLOAD) {
2846 // If for some reason the load is a sextload, the and is needed to zero
2847 // out the high 8 bits
2848 return SDValue();
2849 }
2850
2851 bool AddTo = false;
2852 if (AExt.getNode() != 0) {
2853 // Re-insert the ext as a zext.
2854 Val = DCI.DAG.getNode(ISD::ZERO_EXTEND, SDLoc(N),
2855 AExt.getValueType(), Val);
2856 AddTo = true;
2857 }
2858
2859 // If we get here, the AND is unnecessary. Just replace it with the load
2860 DCI.CombineTo(N, Val, AddTo);
2861 }
2862
2863 return SDValue();
2864}
2865
2866enum OperandSignedness {
2867 Signed = 0,
2868 Unsigned,
2869 Unknown
2870};
2871
2872/// IsMulWideOperandDemotable - Checks if the provided DAG node is an operand
2873/// that can be demoted to \p OptSize bits without loss of information. The
2874/// signedness of the operand, if determinable, is placed in \p S.
2875static bool IsMulWideOperandDemotable(SDValue Op,
2876 unsigned OptSize,
2877 OperandSignedness &S) {
2878 S = Unknown;
2879
2880 if (Op.getOpcode() == ISD::SIGN_EXTEND ||
2881 Op.getOpcode() == ISD::SIGN_EXTEND_INREG) {
2882 EVT OrigVT = Op.getOperand(0).getValueType();
2883 if (OrigVT.getSizeInBits() == OptSize) {
2884 S = Signed;
2885 return true;
2886 }
2887 } else if (Op.getOpcode() == ISD::ZERO_EXTEND) {
2888 EVT OrigVT = Op.getOperand(0).getValueType();
2889 if (OrigVT.getSizeInBits() == OptSize) {
2890 S = Unsigned;
2891 return true;
2892 }
2893 }
2894
2895 return false;
2896}
2897
2898/// AreMulWideOperandsDemotable - Checks if the given LHS and RHS operands can
2899/// be demoted to \p OptSize bits without loss of information. If the operands
2900/// contain a constant, it should appear as the RHS operand. The signedness of
2901/// the operands is placed in \p IsSigned.
2902static bool AreMulWideOperandsDemotable(SDValue LHS, SDValue RHS,
2903 unsigned OptSize,
2904 bool &IsSigned) {
2905
2906 OperandSignedness LHSSign;
2907
2908 // The LHS operand must be a demotable op
2909 if (!IsMulWideOperandDemotable(LHS, OptSize, LHSSign))
2910 return false;
2911
2912 // We should have been able to determine the signedness from the LHS
2913 if (LHSSign == Unknown)
2914 return false;
2915
2916 IsSigned = (LHSSign == Signed);
2917
2918 // The RHS can be a demotable op or a constant
2919 if (ConstantSDNode *CI = dyn_cast<ConstantSDNode>(RHS)) {
2920 APInt Val = CI->getAPIntValue();
2921 if (LHSSign == Unsigned) {
2922 if (Val.isIntN(OptSize)) {
2923 return true;
2924 }
2925 return false;
2926 } else {
2927 if (Val.isSignedIntN(OptSize)) {
2928 return true;
2929 }
2930 return false;
2931 }
2932 } else {
2933 OperandSignedness RHSSign;
2934 if (!IsMulWideOperandDemotable(RHS, OptSize, RHSSign))
2935 return false;
2936
2937 if (LHSSign != RHSSign)
2938 return false;
2939
2940 return true;
2941 }
2942}
2943
2944/// TryMULWIDECombine - Attempt to replace a multiply of M bits with a multiply
2945/// of M/2 bits that produces an M-bit result (i.e. mul.wide). This transform
2946/// works on both multiply DAG nodes and SHL DAG nodes with a constant shift
2947/// amount.
2948static SDValue TryMULWIDECombine(SDNode *N,
2949 TargetLowering::DAGCombinerInfo &DCI) {
2950 EVT MulType = N->getValueType(0);
2951 if (MulType != MVT::i32 && MulType != MVT::i64) {
2952 return SDValue();
2953 }
2954
2955 unsigned OptSize = MulType.getSizeInBits() >> 1;
2956 SDValue LHS = N->getOperand(0);
2957 SDValue RHS = N->getOperand(1);
2958
2959 // Canonicalize the multiply so the constant (if any) is on the right
2960 if (N->getOpcode() == ISD::MUL) {
2961 if (isa<ConstantSDNode>(LHS)) {
2962 std::swap(LHS, RHS);
2963 }
2964 }
2965
2966 // If we have a SHL, determine the actual multiply amount
2967 if (N->getOpcode() == ISD::SHL) {
2968 ConstantSDNode *ShlRHS = dyn_cast<ConstantSDNode>(RHS);
2969 if (!ShlRHS) {
2970 return SDValue();
2971 }
2972
2973 APInt ShiftAmt = ShlRHS->getAPIntValue();
2974 unsigned BitWidth = MulType.getSizeInBits();
2975 if (ShiftAmt.sge(0) && ShiftAmt.slt(BitWidth)) {
2976 APInt MulVal = APInt(BitWidth, 1) << ShiftAmt;
2977 RHS = DCI.DAG.getConstant(MulVal, MulType);
2978 } else {
2979 return SDValue();
2980 }
2981 }
2982
2983 bool Signed;
2984 // Verify that our operands are demotable
2985 if (!AreMulWideOperandsDemotable(LHS, RHS, OptSize, Signed)) {
2986 return SDValue();
2987 }
2988
2989 EVT DemotedVT;
2990 if (MulType == MVT::i32) {
2991 DemotedVT = MVT::i16;
2992 } else {
2993 DemotedVT = MVT::i32;
2994 }
2995
2996 // Truncate the operands to the correct size. Note that these are just for
2997 // type consistency and will (likely) be eliminated in later phases.
2998 SDValue TruncLHS =
2999 DCI.DAG.getNode(ISD::TRUNCATE, SDLoc(N), DemotedVT, LHS);
3000 SDValue TruncRHS =
3001 DCI.DAG.getNode(ISD::TRUNCATE, SDLoc(N), DemotedVT, RHS);
3002
3003 unsigned Opc;
3004 if (Signed) {
3005 Opc = NVPTXISD::MUL_WIDE_SIGNED;
3006 } else {
3007 Opc = NVPTXISD::MUL_WIDE_UNSIGNED;
3008 }
3009
3010 return DCI.DAG.getNode(Opc, SDLoc(N), MulType, TruncLHS, TruncRHS);
3011}
3012
3013/// PerformMULCombine - Runs PTX-specific DAG combine patterns on MUL nodes.
3014static SDValue PerformMULCombine(SDNode *N,
3015 TargetLowering::DAGCombinerInfo &DCI,
3016 CodeGenOpt::Level OptLevel) {
3017 if (OptLevel > 0) {
3018 // Try mul.wide combining at OptLevel > 0
3019 SDValue Ret = TryMULWIDECombine(N, DCI);
3020 if (Ret.getNode())
3021 return Ret;
3022 }
3023
3024 return SDValue();
3025}
3026
3027/// PerformSHLCombine - Runs PTX-specific DAG combine patterns on SHL nodes.
3028static SDValue PerformSHLCombine(SDNode *N,
3029 TargetLowering::DAGCombinerInfo &DCI,
3030 CodeGenOpt::Level OptLevel) {
3031 if (OptLevel > 0) {
3032 // Try mul.wide combining at OptLevel > 0
3033 SDValue Ret = TryMULWIDECombine(N, DCI);
3034 if (Ret.getNode())
3035 return Ret;
3036 }
3037
3038 return SDValue();
3039}
3040
3041SDValue NVPTXTargetLowering::PerformDAGCombine(SDNode *N,
3042 DAGCombinerInfo &DCI) const {
3043 // FIXME: Get this from the DAG somehow
3044 CodeGenOpt::Level OptLevel = CodeGenOpt::Aggressive;
3045 switch (N->getOpcode()) {
3046 default: break;
3047 case ISD::ADD:
3048 case ISD::FADD:
3049 return PerformADDCombine(N, DCI, nvptxSubtarget, OptLevel);
3050 case ISD::MUL:
3051 return PerformMULCombine(N, DCI, OptLevel);
3052 case ISD::SHL:
3053 return PerformSHLCombine(N, DCI, OptLevel);
3054 case ISD::AND:
3055 return PerformANDCombine(N, DCI);
3056 }
3057 return SDValue();
3058}
3059
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]3060/// ReplaceVectorLoad - Convert vector loads into multi-output scalar loads.
3061static void ReplaceLoadVector(SDNode *N, SelectionDAG &DAG,
Justin Holewinskiac451062014-07-16 19:45:35 +0000[diff] [blame^]3062 const DataLayout *TD,
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]3063 SmallVectorImpl<SDValue> &Results) {
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]3064 EVT ResVT = N->getValueType(0);
Andrew Trickef9de2a2013-05-25 02:42:55 +0000[diff] [blame]3065 SDLoc DL(N);
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]3066
3067 assert(ResVT.isVector() && "Vector load must have vector type");
3068
3069 // We only handle "native" vector sizes for now, e.g. <4 x double> is not
3070 // legal. We can (and should) split that into 2 loads of <2 x double> here
3071 // but I'm leaving that as a TODO for now.
3072 assert(ResVT.isSimple() && "Can only handle simple types");
3073 switch (ResVT.getSimpleVT().SimpleTy) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]3074 default:
3075 return;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]3076 case MVT::v2i8:
3077 case MVT::v2i16:
3078 case MVT::v2i32:
3079 case MVT::v2i64:
3080 case MVT::v2f32:
3081 case MVT::v2f64:
3082 case MVT::v4i8:
3083 case MVT::v4i16:
3084 case MVT::v4i32:
3085 case MVT::v4f32:
3086 // This is a "native" vector type
3087 break;
3088 }
3089
Justin Holewinskiac451062014-07-16 19:45:35 +0000[diff] [blame^]3090 LoadSDNode *LD = cast<LoadSDNode>(N);
3091
3092 unsigned Align = LD->getAlignment();
3093 unsigned PrefAlign =
3094 TD->getPrefTypeAlignment(ResVT.getTypeForEVT(*DAG.getContext()));
3095 if (Align < PrefAlign) {
3096 // This load is not sufficiently aligned, so bail out and let this vector
3097 // load be scalarized. Note that we may still be able to emit smaller
3098 // vector loads. For example, if we are loading a <4 x float> with an
3099 // alignment of 8, this check will fail but the legalizer will try again
3100 // with 2 x <2 x float>, which will succeed with an alignment of 8.
3101 return;
3102 }
3103
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]3104 EVT EltVT = ResVT.getVectorElementType();
3105 unsigned NumElts = ResVT.getVectorNumElements();
3106
3107 // Since LoadV2 is a target node, we cannot rely on DAG type legalization.
3108 // Therefore, we must ensure the type is legal. For i1 and i8, we set the
Alp Tokercb402912014-01-24 17:20:08 +0000[diff] [blame]3109 // loaded type to i16 and propagate the "real" type as the memory type.
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]3110 bool NeedTrunc = false;
3111 if (EltVT.getSizeInBits() < 16) {
3112 EltVT = MVT::i16;
3113 NeedTrunc = true;
3114 }
3115
3116 unsigned Opcode = 0;
3117 SDVTList LdResVTs;
3118
3119 switch (NumElts) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]3120 default:
3121 return;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]3122 case 2:
3123 Opcode = NVPTXISD::LoadV2;
3124 LdResVTs = DAG.getVTList(EltVT, EltVT, MVT::Other);
3125 break;
3126 case 4: {
3127 Opcode = NVPTXISD::LoadV4;
3128 EVT ListVTs[] = { EltVT, EltVT, EltVT, EltVT, MVT::Other };
Craig Topperabb4ac72014-04-16 06:10:51 +0000[diff] [blame]3129 LdResVTs = DAG.getVTList(ListVTs);
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]3130 break;
3131 }
3132 }
3133
3134 SmallVector<SDValue, 8> OtherOps;
3135
3136 // Copy regular operands
3137 for (unsigned i = 0, e = N->getNumOperands(); i != e; ++i)
3138 OtherOps.push_back(N->getOperand(i));
3139
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]3140 // The select routine does not have access to the LoadSDNode instance, so
3141 // pass along the extension information
3142 OtherOps.push_back(DAG.getIntPtrConstant(LD->getExtensionType()));
3143
Craig Topper206fcd42014-04-26 19:29:41 +0000[diff] [blame]3144 SDValue NewLD = DAG.getMemIntrinsicNode(Opcode, DL, LdResVTs, OtherOps,
3145 LD->getMemoryVT(),
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]3146 LD->getMemOperand());
3147
3148 SmallVector<SDValue, 4> ScalarRes;
3149
3150 for (unsigned i = 0; i < NumElts; ++i) {
3151 SDValue Res = NewLD.getValue(i);
3152 if (NeedTrunc)
3153 Res = DAG.getNode(ISD::TRUNCATE, DL, ResVT.getVectorElementType(), Res);
3154 ScalarRes.push_back(Res);
3155 }
3156
3157 SDValue LoadChain = NewLD.getValue(NumElts);
3158
Craig Topper48d114b2014-04-26 18:35:24 +0000[diff] [blame]3159 SDValue BuildVec = DAG.getNode(ISD::BUILD_VECTOR, DL, ResVT, ScalarRes);
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]3160
3161 Results.push_back(BuildVec);
3162 Results.push_back(LoadChain);
3163}
3164
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]3165static void ReplaceINTRINSIC_W_CHAIN(SDNode *N, SelectionDAG &DAG,
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]3166 SmallVectorImpl<SDValue> &Results) {
3167 SDValue Chain = N->getOperand(0);
3168 SDValue Intrin = N->getOperand(1);
Andrew Trickef9de2a2013-05-25 02:42:55 +0000[diff] [blame]3169 SDLoc DL(N);
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]3170
3171 // Get the intrinsic ID
3172 unsigned IntrinNo = cast<ConstantSDNode>(Intrin.getNode())->getZExtValue();
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]3173 switch (IntrinNo) {
3174 default:
3175 return;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]3176 case Intrinsic::nvvm_ldg_global_i:
3177 case Intrinsic::nvvm_ldg_global_f:
3178 case Intrinsic::nvvm_ldg_global_p:
3179 case Intrinsic::nvvm_ldu_global_i:
3180 case Intrinsic::nvvm_ldu_global_f:
3181 case Intrinsic::nvvm_ldu_global_p: {
3182 EVT ResVT = N->getValueType(0);
3183
3184 if (ResVT.isVector()) {
3185 // Vector LDG/LDU
3186
3187 unsigned NumElts = ResVT.getVectorNumElements();
3188 EVT EltVT = ResVT.getVectorElementType();
3189
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]3190 // Since LDU/LDG are target nodes, we cannot rely on DAG type
3191 // legalization.
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]3192 // Therefore, we must ensure the type is legal. For i1 and i8, we set the
Alp Tokercb402912014-01-24 17:20:08 +0000[diff] [blame]3193 // loaded type to i16 and propagate the "real" type as the memory type.
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]3194 bool NeedTrunc = false;
3195 if (EltVT.getSizeInBits() < 16) {
3196 EltVT = MVT::i16;
3197 NeedTrunc = true;
3198 }
3199
3200 unsigned Opcode = 0;
3201 SDVTList LdResVTs;
3202
3203 switch (NumElts) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]3204 default:
3205 return;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]3206 case 2:
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]3207 switch (IntrinNo) {
3208 default:
3209 return;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]3210 case Intrinsic::nvvm_ldg_global_i:
3211 case Intrinsic::nvvm_ldg_global_f:
3212 case Intrinsic::nvvm_ldg_global_p:
3213 Opcode = NVPTXISD::LDGV2;
3214 break;
3215 case Intrinsic::nvvm_ldu_global_i:
3216 case Intrinsic::nvvm_ldu_global_f:
3217 case Intrinsic::nvvm_ldu_global_p:
3218 Opcode = NVPTXISD::LDUV2;
3219 break;
3220 }
3221 LdResVTs = DAG.getVTList(EltVT, EltVT, MVT::Other);
3222 break;
3223 case 4: {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]3224 switch (IntrinNo) {
3225 default:
3226 return;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]3227 case Intrinsic::nvvm_ldg_global_i:
3228 case Intrinsic::nvvm_ldg_global_f:
3229 case Intrinsic::nvvm_ldg_global_p:
3230 Opcode = NVPTXISD::LDGV4;
3231 break;
3232 case Intrinsic::nvvm_ldu_global_i:
3233 case Intrinsic::nvvm_ldu_global_f:
3234 case Intrinsic::nvvm_ldu_global_p:
3235 Opcode = NVPTXISD::LDUV4;
3236 break;
3237 }
3238 EVT ListVTs[] = { EltVT, EltVT, EltVT, EltVT, MVT::Other };
Craig Topperabb4ac72014-04-16 06:10:51 +0000[diff] [blame]3239 LdResVTs = DAG.getVTList(ListVTs);
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]3240 break;
3241 }
3242 }
3243
3244 SmallVector<SDValue, 8> OtherOps;
3245
3246 // Copy regular operands
3247
3248 OtherOps.push_back(Chain); // Chain
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]3249 // Skip operand 1 (intrinsic ID)
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]3250 // Others
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]3251 for (unsigned i = 2, e = N->getNumOperands(); i != e; ++i)
3252 OtherOps.push_back(N->getOperand(i));
3253
3254 MemIntrinsicSDNode *MemSD = cast<MemIntrinsicSDNode>(N);
3255
Craig Topper206fcd42014-04-26 19:29:41 +0000[diff] [blame]3256 SDValue NewLD = DAG.getMemIntrinsicNode(Opcode, DL, LdResVTs, OtherOps,
3257 MemSD->getMemoryVT(),
3258 MemSD->getMemOperand());
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]3259
3260 SmallVector<SDValue, 4> ScalarRes;
3261
3262 for (unsigned i = 0; i < NumElts; ++i) {
3263 SDValue Res = NewLD.getValue(i);
3264 if (NeedTrunc)
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]3265 Res =
3266 DAG.getNode(ISD::TRUNCATE, DL, ResVT.getVectorElementType(), Res);
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]3267 ScalarRes.push_back(Res);
3268 }
3269
3270 SDValue LoadChain = NewLD.getValue(NumElts);
3271
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]3272 SDValue BuildVec =
Craig Topper48d114b2014-04-26 18:35:24 +0000[diff] [blame]3273 DAG.getNode(ISD::BUILD_VECTOR, DL, ResVT, ScalarRes);
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]3274
3275 Results.push_back(BuildVec);
3276 Results.push_back(LoadChain);
3277 } else {
3278 // i8 LDG/LDU
3279 assert(ResVT.isSimple() && ResVT.getSimpleVT().SimpleTy == MVT::i8 &&
3280 "Custom handling of non-i8 ldu/ldg?");
3281
3282 // Just copy all operands as-is
3283 SmallVector<SDValue, 4> Ops;
3284 for (unsigned i = 0, e = N->getNumOperands(); i != e; ++i)
3285 Ops.push_back(N->getOperand(i));
3286
3287 // Force output to i16
3288 SDVTList LdResVTs = DAG.getVTList(MVT::i16, MVT::Other);
3289
3290 MemIntrinsicSDNode *MemSD = cast<MemIntrinsicSDNode>(N);
3291
3292 // We make sure the memory type is i8, which will be used during isel
3293 // to select the proper instruction.
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]3294 SDValue NewLD =
Craig Topper206fcd42014-04-26 19:29:41 +0000[diff] [blame]3295 DAG.getMemIntrinsicNode(ISD::INTRINSIC_W_CHAIN, DL, LdResVTs, Ops,
3296 MVT::i8, MemSD->getMemOperand());
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]3297
Justin Holewinskie8c93e32013-07-01 12:58:48 +0000[diff] [blame]3298 Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i8,
3299 NewLD.getValue(0)));
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]3300 Results.push_back(NewLD.getValue(1));
3301 }
3302 }
3303 }
3304}
3305
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]3306void NVPTXTargetLowering::ReplaceNodeResults(
3307 SDNode *N, SmallVectorImpl<SDValue> &Results, SelectionDAG &DAG) const {
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]3308 switch (N->getOpcode()) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]3309 default:
3310 report_fatal_error("Unhandled custom legalization");
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]3311 case ISD::LOAD:
Justin Holewinskiac451062014-07-16 19:45:35 +0000[diff] [blame^]3312 ReplaceLoadVector(N, DAG, getDataLayout(), Results);
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]3313 return;
3314 case ISD::INTRINSIC_W_CHAIN:
3315 ReplaceINTRINSIC_W_CHAIN(N, DAG, Results);
3316 return;
3317 }
3318}
Juergen Ributzkad12ccbd2013-11-19 00:57:56 +0000[diff] [blame]3319
3320// Pin NVPTXSection's and NVPTXTargetObjectFile's vtables to this file.
3321void NVPTXSection::anchor() {}
3322
3323NVPTXTargetObjectFile::~NVPTXTargetObjectFile() {
3324 delete TextSection;
3325 delete DataSection;
3326 delete BSSSection;
3327 delete ReadOnlySection;
3328
3329 delete StaticCtorSection;
3330 delete StaticDtorSection;
3331 delete LSDASection;
3332 delete EHFrameSection;
3333 delete DwarfAbbrevSection;
3334 delete DwarfInfoSection;
3335 delete DwarfLineSection;
3336 delete DwarfFrameSection;
3337 delete DwarfPubTypesSection;
3338 delete DwarfDebugInlineSection;
3339 delete DwarfStrSection;
3340 delete DwarfLocSection;
3341 delete DwarfARangesSection;
3342 delete DwarfRangesSection;
3343 delete DwarfMacroInfoSection;
3344}