blob: f2578584bc649de283637223238c5272bbc29634 [file] [log] [blame]
Justin Holewinskiae556d32012-05-04 20:18:50 +00001//
2// The LLVM Compiler Infrastructure
3//
4// This file is distributed under the University of Illinois Open Source
5// License. See LICENSE.TXT for details.
6//
7//===----------------------------------------------------------------------===//
8//
9// This file defines the interfaces that NVPTX uses to lower LLVM code into a
10// selection DAG.
11//
12//===----------------------------------------------------------------------===//
13
Justin Holewinskiae556d32012-05-04 20:18:50 +000014#include "NVPTXISelLowering.h"
Chandler Carruthed0881b2012-12-03 16:50:05 +000015#include "NVPTX.h"
Justin Holewinskiae556d32012-05-04 20:18:50 +000016#include "NVPTXTargetMachine.h"
17#include "NVPTXTargetObjectFile.h"
18#include "NVPTXUtilities.h"
Justin Holewinskiae556d32012-05-04 20:18:50 +000019#include "llvm/CodeGen/Analysis.h"
20#include "llvm/CodeGen/MachineFrameInfo.h"
21#include "llvm/CodeGen/MachineFunction.h"
22#include "llvm/CodeGen/MachineInstrBuilder.h"
23#include "llvm/CodeGen/MachineRegisterInfo.h"
Justin Holewinskiae556d32012-05-04 20:18:50 +000024#include "llvm/CodeGen/TargetLoweringObjectFileImpl.h"
Chandler Carruth9fb823b2013-01-02 11:36:10 +000025#include "llvm/IR/DerivedTypes.h"
26#include "llvm/IR/Function.h"
27#include "llvm/IR/GlobalValue.h"
28#include "llvm/IR/IntrinsicInst.h"
29#include "llvm/IR/Intrinsics.h"
30#include "llvm/IR/Module.h"
Justin Holewinskiae556d32012-05-04 20:18:50 +000031#include "llvm/MC/MCSectionELF.h"
Chandler Carruthed0881b2012-12-03 16:50:05 +000032#include "llvm/Support/CallSite.h"
33#include "llvm/Support/CommandLine.h"
34#include "llvm/Support/Debug.h"
35#include "llvm/Support/ErrorHandling.h"
36#include "llvm/Support/raw_ostream.h"
Justin Holewinskiae556d32012-05-04 20:18:50 +000037#include <sstream>
38
39#undef DEBUG_TYPE
40#define DEBUG_TYPE "nvptx-lower"
41
42using namespace llvm;
43
44static unsigned int uniqueCallSite = 0;
45
Justin Holewinski0497ab12013-03-30 14:29:21 +000046static cl::opt<bool> sched4reg(
47 "nvptx-sched4reg",
48 cl::desc("NVPTX Specific: schedule for register pressue"), cl::init(false));
Justin Holewinskiae556d32012-05-04 20:18:50 +000049
Justin Holewinskibe8dc642013-02-12 14:18:49 +000050static bool IsPTXVectorType(MVT VT) {
51 switch (VT.SimpleTy) {
Justin Holewinski0497ab12013-03-30 14:29:21 +000052 default:
53 return false;
Justin Holewinskif8f70912013-06-28 17:57:59 +000054 case MVT::v2i1:
55 case MVT::v4i1:
Justin Holewinskibe8dc642013-02-12 14:18:49 +000056 case MVT::v2i8:
57 case MVT::v4i8:
58 case MVT::v2i16:
59 case MVT::v4i16:
60 case MVT::v2i32:
61 case MVT::v4i32:
62 case MVT::v2i64:
63 case MVT::v2f32:
64 case MVT::v4f32:
65 case MVT::v2f64:
Justin Holewinski0497ab12013-03-30 14:29:21 +000066 return true;
Justin Holewinskibe8dc642013-02-12 14:18:49 +000067 }
68}
69
Justin Holewinskif8f70912013-06-28 17:57:59 +000070/// ComputePTXValueVTs - For the given Type \p Ty, returns the set of primitive
71/// EVTs that compose it. Unlike ComputeValueVTs, this will break apart vectors
72/// into their primitive components.
73/// NOTE: This is a band-aid for code that expects ComputeValueVTs to return the
74/// same number of types as the Ins/Outs arrays in LowerFormalArguments,
75/// LowerCall, and LowerReturn.
76static void ComputePTXValueVTs(const TargetLowering &TLI, Type *Ty,
77 SmallVectorImpl<EVT> &ValueVTs,
78 SmallVectorImpl<uint64_t> *Offsets = 0,
79 uint64_t StartingOffset = 0) {
80 SmallVector<EVT, 16> TempVTs;
81 SmallVector<uint64_t, 16> TempOffsets;
82
83 ComputeValueVTs(TLI, Ty, TempVTs, &TempOffsets, StartingOffset);
84 for (unsigned i = 0, e = TempVTs.size(); i != e; ++i) {
85 EVT VT = TempVTs[i];
86 uint64_t Off = TempOffsets[i];
87 if (VT.isVector())
88 for (unsigned j = 0, je = VT.getVectorNumElements(); j != je; ++j) {
89 ValueVTs.push_back(VT.getVectorElementType());
90 if (Offsets)
91 Offsets->push_back(Off+j*VT.getVectorElementType().getStoreSize());
92 }
93 else {
94 ValueVTs.push_back(VT);
95 if (Offsets)
96 Offsets->push_back(Off);
97 }
98 }
99}
100
Justin Holewinskiae556d32012-05-04 20:18:50 +0000101// NVPTXTargetLowering Constructor.
102NVPTXTargetLowering::NVPTXTargetLowering(NVPTXTargetMachine &TM)
Justin Holewinski0497ab12013-03-30 14:29:21 +0000103 : TargetLowering(TM, new NVPTXTargetObjectFile()), nvTM(&TM),
104 nvptxSubtarget(TM.getSubtarget<NVPTXSubtarget>()) {
Justin Holewinskiae556d32012-05-04 20:18:50 +0000105
106 // always lower memset, memcpy, and memmove intrinsics to load/store
107 // instructions, rather
108 // then generating calls to memset, mempcy or memmove.
Justin Holewinski0497ab12013-03-30 14:29:21 +0000109 MaxStoresPerMemset = (unsigned) 0xFFFFFFFF;
110 MaxStoresPerMemcpy = (unsigned) 0xFFFFFFFF;
111 MaxStoresPerMemmove = (unsigned) 0xFFFFFFFF;
Justin Holewinskiae556d32012-05-04 20:18:50 +0000112
113 setBooleanContents(ZeroOrNegativeOneBooleanContent);
114
115 // Jump is Expensive. Don't create extra control flow for 'and', 'or'
116 // condition branches.
117 setJumpIsExpensive(true);
118
119 // By default, use the Source scheduling
120 if (sched4reg)
121 setSchedulingPreference(Sched::RegPressure);
122 else
123 setSchedulingPreference(Sched::Source);
124
125 addRegisterClass(MVT::i1, &NVPTX::Int1RegsRegClass);
Justin Holewinskiae556d32012-05-04 20:18:50 +0000126 addRegisterClass(MVT::i16, &NVPTX::Int16RegsRegClass);
127 addRegisterClass(MVT::i32, &NVPTX::Int32RegsRegClass);
128 addRegisterClass(MVT::i64, &NVPTX::Int64RegsRegClass);
129 addRegisterClass(MVT::f32, &NVPTX::Float32RegsRegClass);
130 addRegisterClass(MVT::f64, &NVPTX::Float64RegsRegClass);
131
Justin Holewinskiae556d32012-05-04 20:18:50 +0000132 // Operations not directly supported by NVPTX.
Justin Holewinski0497ab12013-03-30 14:29:21 +0000133 setOperationAction(ISD::SELECT_CC, MVT::Other, Expand);
134 setOperationAction(ISD::BR_CC, MVT::f32, Expand);
135 setOperationAction(ISD::BR_CC, MVT::f64, Expand);
136 setOperationAction(ISD::BR_CC, MVT::i1, Expand);
137 setOperationAction(ISD::BR_CC, MVT::i8, Expand);
138 setOperationAction(ISD::BR_CC, MVT::i16, Expand);
139 setOperationAction(ISD::BR_CC, MVT::i32, Expand);
140 setOperationAction(ISD::BR_CC, MVT::i64, Expand);
Justin Holewinski318c6252013-07-01 12:58:56 +0000141 // Some SIGN_EXTEND_INREG can be done using cvt instruction.
142 // For others we will expand to a SHL/SRA pair.
143 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i64, Legal);
144 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i32, Legal);
145 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i16, Legal);
146 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i8 , Legal);
Justin Holewinski0497ab12013-03-30 14:29:21 +0000147 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i1, Expand);
Justin Holewinskiae556d32012-05-04 20:18:50 +0000148
149 if (nvptxSubtarget.hasROT64()) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000150 setOperationAction(ISD::ROTL, MVT::i64, Legal);
151 setOperationAction(ISD::ROTR, MVT::i64, Legal);
152 } else {
153 setOperationAction(ISD::ROTL, MVT::i64, Expand);
154 setOperationAction(ISD::ROTR, MVT::i64, Expand);
Justin Holewinskiae556d32012-05-04 20:18:50 +0000155 }
156 if (nvptxSubtarget.hasROT32()) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000157 setOperationAction(ISD::ROTL, MVT::i32, Legal);
158 setOperationAction(ISD::ROTR, MVT::i32, Legal);
159 } else {
160 setOperationAction(ISD::ROTL, MVT::i32, Expand);
161 setOperationAction(ISD::ROTR, MVT::i32, Expand);
Justin Holewinskiae556d32012-05-04 20:18:50 +0000162 }
163
Justin Holewinski0497ab12013-03-30 14:29:21 +0000164 setOperationAction(ISD::ROTL, MVT::i16, Expand);
165 setOperationAction(ISD::ROTR, MVT::i16, Expand);
166 setOperationAction(ISD::ROTL, MVT::i8, Expand);
167 setOperationAction(ISD::ROTR, MVT::i8, Expand);
168 setOperationAction(ISD::BSWAP, MVT::i16, Expand);
169 setOperationAction(ISD::BSWAP, MVT::i32, Expand);
170 setOperationAction(ISD::BSWAP, MVT::i64, Expand);
Justin Holewinskiae556d32012-05-04 20:18:50 +0000171
172 // Indirect branch is not supported.
173 // This also disables Jump Table creation.
Justin Holewinski0497ab12013-03-30 14:29:21 +0000174 setOperationAction(ISD::BR_JT, MVT::Other, Expand);
175 setOperationAction(ISD::BRIND, MVT::Other, Expand);
Justin Holewinskiae556d32012-05-04 20:18:50 +0000176
Justin Holewinski0497ab12013-03-30 14:29:21 +0000177 setOperationAction(ISD::GlobalAddress, MVT::i32, Custom);
178 setOperationAction(ISD::GlobalAddress, MVT::i64, Custom);
Justin Holewinskiae556d32012-05-04 20:18:50 +0000179
180 // We want to legalize constant related memmove and memcopy
181 // intrinsics.
182 setOperationAction(ISD::INTRINSIC_W_CHAIN, MVT::Other, Custom);
183
184 // Turn FP extload into load/fextend
185 setLoadExtAction(ISD::EXTLOAD, MVT::f32, Expand);
186 // Turn FP truncstore into trunc + store.
187 setTruncStoreAction(MVT::f64, MVT::f32, Expand);
188
189 // PTX does not support load / store predicate registers
Justin Holewinskic6462aa2012-11-14 19:19:16 +0000190 setOperationAction(ISD::LOAD, MVT::i1, Custom);
191 setOperationAction(ISD::STORE, MVT::i1, Custom);
192
Justin Holewinskiae556d32012-05-04 20:18:50 +0000193 setLoadExtAction(ISD::SEXTLOAD, MVT::i1, Promote);
194 setLoadExtAction(ISD::ZEXTLOAD, MVT::i1, Promote);
Justin Holewinskiae556d32012-05-04 20:18:50 +0000195 setTruncStoreAction(MVT::i64, MVT::i1, Expand);
196 setTruncStoreAction(MVT::i32, MVT::i1, Expand);
197 setTruncStoreAction(MVT::i16, MVT::i1, Expand);
198 setTruncStoreAction(MVT::i8, MVT::i1, Expand);
199
200 // This is legal in NVPTX
Justin Holewinski0497ab12013-03-30 14:29:21 +0000201 setOperationAction(ISD::ConstantFP, MVT::f64, Legal);
202 setOperationAction(ISD::ConstantFP, MVT::f32, Legal);
Justin Holewinskiae556d32012-05-04 20:18:50 +0000203
204 // TRAP can be lowered to PTX trap
Justin Holewinski0497ab12013-03-30 14:29:21 +0000205 setOperationAction(ISD::TRAP, MVT::Other, Legal);
Justin Holewinskiae556d32012-05-04 20:18:50 +0000206
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000207 // Register custom handling for vector loads/stores
Justin Holewinski0497ab12013-03-30 14:29:21 +0000208 for (int i = MVT::FIRST_VECTOR_VALUETYPE; i <= MVT::LAST_VECTOR_VALUETYPE;
209 ++i) {
210 MVT VT = (MVT::SimpleValueType) i;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000211 if (IsPTXVectorType(VT)) {
212 setOperationAction(ISD::LOAD, VT, Custom);
213 setOperationAction(ISD::STORE, VT, Custom);
214 setOperationAction(ISD::INTRINSIC_W_CHAIN, VT, Custom);
215 }
216 }
Justin Holewinskiae556d32012-05-04 20:18:50 +0000217
Justin Holewinskif8f70912013-06-28 17:57:59 +0000218 // Custom handling for i8 intrinsics
219 setOperationAction(ISD::INTRINSIC_W_CHAIN, MVT::i8, Custom);
220
Justin Holewinskidc372df2013-06-28 17:58:07 +0000221 setOperationAction(ISD::CTLZ, MVT::i16, Legal);
222 setOperationAction(ISD::CTLZ, MVT::i32, Legal);
223 setOperationAction(ISD::CTLZ, MVT::i64, Legal);
224 setOperationAction(ISD::CTLZ_ZERO_UNDEF, MVT::i16, Legal);
225 setOperationAction(ISD::CTLZ_ZERO_UNDEF, MVT::i32, Legal);
226 setOperationAction(ISD::CTLZ_ZERO_UNDEF, MVT::i64, Legal);
227 setOperationAction(ISD::CTTZ, MVT::i16, Expand);
228 setOperationAction(ISD::CTTZ, MVT::i32, Expand);
229 setOperationAction(ISD::CTTZ, MVT::i64, Expand);
230 setOperationAction(ISD::CTTZ_ZERO_UNDEF, MVT::i16, Expand);
231 setOperationAction(ISD::CTTZ_ZERO_UNDEF, MVT::i32, Expand);
232 setOperationAction(ISD::CTTZ_ZERO_UNDEF, MVT::i64, Expand);
233 setOperationAction(ISD::CTPOP, MVT::i16, Legal);
234 setOperationAction(ISD::CTPOP, MVT::i32, Legal);
235 setOperationAction(ISD::CTPOP, MVT::i64, Legal);
236
Justin Holewinskiae556d32012-05-04 20:18:50 +0000237 // Now deduce the information based on the above mentioned
238 // actions
239 computeRegisterProperties();
240}
241
Justin Holewinskiae556d32012-05-04 20:18:50 +0000242const char *NVPTXTargetLowering::getTargetNodeName(unsigned Opcode) const {
243 switch (Opcode) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000244 default:
245 return 0;
246 case NVPTXISD::CALL:
247 return "NVPTXISD::CALL";
248 case NVPTXISD::RET_FLAG:
249 return "NVPTXISD::RET_FLAG";
250 case NVPTXISD::Wrapper:
251 return "NVPTXISD::Wrapper";
252 case NVPTXISD::NVBuiltin:
253 return "NVPTXISD::NVBuiltin";
254 case NVPTXISD::DeclareParam:
255 return "NVPTXISD::DeclareParam";
Justin Holewinskiae556d32012-05-04 20:18:50 +0000256 case NVPTXISD::DeclareScalarParam:
257 return "NVPTXISD::DeclareScalarParam";
Justin Holewinski0497ab12013-03-30 14:29:21 +0000258 case NVPTXISD::DeclareRet:
259 return "NVPTXISD::DeclareRet";
260 case NVPTXISD::DeclareRetParam:
261 return "NVPTXISD::DeclareRetParam";
262 case NVPTXISD::PrintCall:
263 return "NVPTXISD::PrintCall";
264 case NVPTXISD::LoadParam:
265 return "NVPTXISD::LoadParam";
Justin Holewinskife44314f2013-06-28 17:57:51 +0000266 case NVPTXISD::LoadParamV2:
267 return "NVPTXISD::LoadParamV2";
268 case NVPTXISD::LoadParamV4:
269 return "NVPTXISD::LoadParamV4";
Justin Holewinski0497ab12013-03-30 14:29:21 +0000270 case NVPTXISD::StoreParam:
271 return "NVPTXISD::StoreParam";
Justin Holewinskife44314f2013-06-28 17:57:51 +0000272 case NVPTXISD::StoreParamV2:
273 return "NVPTXISD::StoreParamV2";
274 case NVPTXISD::StoreParamV4:
275 return "NVPTXISD::StoreParamV4";
Justin Holewinski0497ab12013-03-30 14:29:21 +0000276 case NVPTXISD::StoreParamS32:
277 return "NVPTXISD::StoreParamS32";
278 case NVPTXISD::StoreParamU32:
279 return "NVPTXISD::StoreParamU32";
Justin Holewinski0497ab12013-03-30 14:29:21 +0000280 case NVPTXISD::CallArgBegin:
281 return "NVPTXISD::CallArgBegin";
282 case NVPTXISD::CallArg:
283 return "NVPTXISD::CallArg";
284 case NVPTXISD::LastCallArg:
285 return "NVPTXISD::LastCallArg";
286 case NVPTXISD::CallArgEnd:
287 return "NVPTXISD::CallArgEnd";
288 case NVPTXISD::CallVoid:
289 return "NVPTXISD::CallVoid";
290 case NVPTXISD::CallVal:
291 return "NVPTXISD::CallVal";
292 case NVPTXISD::CallSymbol:
293 return "NVPTXISD::CallSymbol";
294 case NVPTXISD::Prototype:
295 return "NVPTXISD::Prototype";
296 case NVPTXISD::MoveParam:
297 return "NVPTXISD::MoveParam";
Justin Holewinski0497ab12013-03-30 14:29:21 +0000298 case NVPTXISD::StoreRetval:
299 return "NVPTXISD::StoreRetval";
Justin Holewinskife44314f2013-06-28 17:57:51 +0000300 case NVPTXISD::StoreRetvalV2:
301 return "NVPTXISD::StoreRetvalV2";
302 case NVPTXISD::StoreRetvalV4:
303 return "NVPTXISD::StoreRetvalV4";
Justin Holewinski0497ab12013-03-30 14:29:21 +0000304 case NVPTXISD::PseudoUseParam:
305 return "NVPTXISD::PseudoUseParam";
306 case NVPTXISD::RETURN:
307 return "NVPTXISD::RETURN";
308 case NVPTXISD::CallSeqBegin:
309 return "NVPTXISD::CallSeqBegin";
310 case NVPTXISD::CallSeqEnd:
311 return "NVPTXISD::CallSeqEnd";
312 case NVPTXISD::LoadV2:
313 return "NVPTXISD::LoadV2";
314 case NVPTXISD::LoadV4:
315 return "NVPTXISD::LoadV4";
316 case NVPTXISD::LDGV2:
317 return "NVPTXISD::LDGV2";
318 case NVPTXISD::LDGV4:
319 return "NVPTXISD::LDGV4";
320 case NVPTXISD::LDUV2:
321 return "NVPTXISD::LDUV2";
322 case NVPTXISD::LDUV4:
323 return "NVPTXISD::LDUV4";
324 case NVPTXISD::StoreV2:
325 return "NVPTXISD::StoreV2";
326 case NVPTXISD::StoreV4:
327 return "NVPTXISD::StoreV4";
Justin Holewinskiae556d32012-05-04 20:18:50 +0000328 }
329}
330
Justin Holewinskibc451192012-11-29 14:26:24 +0000331bool NVPTXTargetLowering::shouldSplitVectorElementType(EVT VT) const {
332 return VT == MVT::i1;
333}
Justin Holewinskiae556d32012-05-04 20:18:50 +0000334
335SDValue
336NVPTXTargetLowering::LowerGlobalAddress(SDValue Op, SelectionDAG &DAG) const {
Andrew Trickef9de2a2013-05-25 02:42:55 +0000337 SDLoc dl(Op);
Justin Holewinskiae556d32012-05-04 20:18:50 +0000338 const GlobalValue *GV = cast<GlobalAddressSDNode>(Op)->getGlobal();
339 Op = DAG.getTargetGlobalAddress(GV, dl, getPointerTy());
340 return DAG.getNode(NVPTXISD::Wrapper, dl, getPointerTy(), Op);
341}
342
Justin Holewinskif8f70912013-06-28 17:57:59 +0000343std::string
344NVPTXTargetLowering::getPrototype(Type *retTy, const ArgListTy &Args,
345 const SmallVectorImpl<ISD::OutputArg> &Outs,
346 unsigned retAlignment,
347 const ImmutableCallSite *CS) const {
348
349 bool isABI = (nvptxSubtarget.getSmVersion() >= 20);
350 assert(isABI && "Non-ABI compilation is not supported");
351 if (!isABI)
352 return "";
353
354 std::stringstream O;
355 O << "prototype_" << uniqueCallSite << " : .callprototype ";
356
357 if (retTy->getTypeID() == Type::VoidTyID) {
358 O << "()";
359 } else {
360 O << "(";
361 if (retTy->isPrimitiveType() || retTy->isIntegerTy()) {
362 unsigned size = 0;
363 if (const IntegerType *ITy = dyn_cast<IntegerType>(retTy)) {
364 size = ITy->getBitWidth();
365 if (size < 32)
366 size = 32;
367 } else {
368 assert(retTy->isFloatingPointTy() &&
369 "Floating point type expected here");
370 size = retTy->getPrimitiveSizeInBits();
371 }
372
373 O << ".param .b" << size << " _";
374 } else if (isa<PointerType>(retTy)) {
375 O << ".param .b" << getPointerTy().getSizeInBits() << " _";
376 } else {
377 if ((retTy->getTypeID() == Type::StructTyID) || isa<VectorType>(retTy)) {
378 SmallVector<EVT, 16> vtparts;
379 ComputeValueVTs(*this, retTy, vtparts);
380 unsigned totalsz = 0;
381 for (unsigned i = 0, e = vtparts.size(); i != e; ++i) {
382 unsigned elems = 1;
383 EVT elemtype = vtparts[i];
384 if (vtparts[i].isVector()) {
385 elems = vtparts[i].getVectorNumElements();
386 elemtype = vtparts[i].getVectorElementType();
387 }
388 // TODO: no need to loop
389 for (unsigned j = 0, je = elems; j != je; ++j) {
390 unsigned sz = elemtype.getSizeInBits();
391 if (elemtype.isInteger() && (sz < 8))
392 sz = 8;
393 totalsz += sz / 8;
394 }
395 }
396 O << ".param .align " << retAlignment << " .b8 _[" << totalsz << "]";
397 } else {
398 assert(false && "Unknown return type");
399 }
400 }
401 O << ") ";
402 }
403 O << "_ (";
404
405 bool first = true;
406 MVT thePointerTy = getPointerTy();
407
408 unsigned OIdx = 0;
409 for (unsigned i = 0, e = Args.size(); i != e; ++i, ++OIdx) {
410 Type *Ty = Args[i].Ty;
411 if (!first) {
412 O << ", ";
413 }
414 first = false;
415
416 if (Outs[OIdx].Flags.isByVal() == false) {
417 if (Ty->isAggregateType() || Ty->isVectorTy()) {
418 unsigned align = 0;
419 const CallInst *CallI = cast<CallInst>(CS->getInstruction());
420 const DataLayout *TD = getDataLayout();
421 // +1 because index 0 is reserved for return type alignment
422 if (!llvm::getAlign(*CallI, i + 1, align))
423 align = TD->getABITypeAlignment(Ty);
424 unsigned sz = TD->getTypeAllocSize(Ty);
425 O << ".param .align " << align << " .b8 ";
426 O << "_";
427 O << "[" << sz << "]";
428 // update the index for Outs
429 SmallVector<EVT, 16> vtparts;
430 ComputeValueVTs(*this, Ty, vtparts);
431 if (unsigned len = vtparts.size())
432 OIdx += len - 1;
433 continue;
434 }
Justin Holewinskidff28d22013-07-01 12:59:01 +0000435 // i8 types in IR will be i16 types in SDAG
436 assert((getValueType(Ty) == Outs[OIdx].VT ||
437 (getValueType(Ty) == MVT::i8 && Outs[OIdx].VT == MVT::i16)) &&
Justin Holewinskif8f70912013-06-28 17:57:59 +0000438 "type mismatch between callee prototype and arguments");
439 // scalar type
440 unsigned sz = 0;
441 if (isa<IntegerType>(Ty)) {
442 sz = cast<IntegerType>(Ty)->getBitWidth();
443 if (sz < 32)
444 sz = 32;
445 } else if (isa<PointerType>(Ty))
446 sz = thePointerTy.getSizeInBits();
447 else
448 sz = Ty->getPrimitiveSizeInBits();
449 O << ".param .b" << sz << " ";
450 O << "_";
451 continue;
452 }
453 const PointerType *PTy = dyn_cast<PointerType>(Ty);
454 assert(PTy && "Param with byval attribute should be a pointer type");
455 Type *ETy = PTy->getElementType();
456
457 unsigned align = Outs[OIdx].Flags.getByValAlign();
458 unsigned sz = getDataLayout()->getTypeAllocSize(ETy);
459 O << ".param .align " << align << " .b8 ";
460 O << "_";
461 O << "[" << sz << "]";
462 }
463 O << ");";
464 return O.str();
465}
466
467unsigned
468NVPTXTargetLowering::getArgumentAlignment(SDValue Callee,
469 const ImmutableCallSite *CS,
470 Type *Ty,
471 unsigned Idx) const {
472 const DataLayout *TD = getDataLayout();
473 unsigned align = 0;
474 GlobalAddressSDNode *Func = dyn_cast<GlobalAddressSDNode>(Callee.getNode());
475
476 if (Func) { // direct call
477 assert(CS->getCalledFunction() &&
478 "direct call cannot find callee");
479 if (!llvm::getAlign(*(CS->getCalledFunction()), Idx, align))
480 align = TD->getABITypeAlignment(Ty);
481 }
482 else { // indirect call
483 const CallInst *CallI = dyn_cast<CallInst>(CS->getInstruction());
484 if (!llvm::getAlign(*CallI, Idx, align))
485 align = TD->getABITypeAlignment(Ty);
486 }
487
488 return align;
Justin Holewinskiae556d32012-05-04 20:18:50 +0000489}
490
Justin Holewinski0497ab12013-03-30 14:29:21 +0000491SDValue NVPTXTargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
492 SmallVectorImpl<SDValue> &InVals) const {
493 SelectionDAG &DAG = CLI.DAG;
Andrew Trickef9de2a2013-05-25 02:42:55 +0000494 SDLoc dl = CLI.DL;
Justin Holewinskiaa583972012-05-25 16:35:28 +0000495 SmallVector<ISD::OutputArg, 32> &Outs = CLI.Outs;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000496 SmallVector<SDValue, 32> &OutVals = CLI.OutVals;
497 SmallVector<ISD::InputArg, 32> &Ins = CLI.Ins;
498 SDValue Chain = CLI.Chain;
499 SDValue Callee = CLI.Callee;
500 bool &isTailCall = CLI.IsTailCall;
501 ArgListTy &Args = CLI.Args;
502 Type *retTy = CLI.RetTy;
503 ImmutableCallSite *CS = CLI.CS;
Justin Holewinskiaa583972012-05-25 16:35:28 +0000504
Justin Holewinskiae556d32012-05-04 20:18:50 +0000505 bool isABI = (nvptxSubtarget.getSmVersion() >= 20);
Justin Holewinskif8f70912013-06-28 17:57:59 +0000506 assert(isABI && "Non-ABI compilation is not supported");
507 if (!isABI)
508 return Chain;
509 const DataLayout *TD = getDataLayout();
510 MachineFunction &MF = DAG.getMachineFunction();
511 const Function *F = MF.getFunction();
Justin Holewinskiae556d32012-05-04 20:18:50 +0000512
513 SDValue tempChain = Chain;
Justin Holewinskif8f70912013-06-28 17:57:59 +0000514 Chain =
515 DAG.getCALLSEQ_START(Chain, DAG.getIntPtrConstant(uniqueCallSite, true),
516 dl);
Justin Holewinskiae556d32012-05-04 20:18:50 +0000517 SDValue InFlag = Chain.getValue(1);
518
Justin Holewinskiae556d32012-05-04 20:18:50 +0000519 unsigned paramCount = 0;
Justin Holewinskif8f70912013-06-28 17:57:59 +0000520 // Args.size() and Outs.size() need not match.
521 // Outs.size() will be larger
522 // * if there is an aggregate argument with multiple fields (each field
523 // showing up separately in Outs)
524 // * if there is a vector argument with more than typical vector-length
525 // elements (generally if more than 4) where each vector element is
526 // individually present in Outs.
527 // So a different index should be used for indexing into Outs/OutVals.
528 // See similar issue in LowerFormalArguments.
529 unsigned OIdx = 0;
Justin Holewinskiae556d32012-05-04 20:18:50 +0000530 // Declare the .params or .reg need to pass values
531 // to the function
Justin Holewinskif8f70912013-06-28 17:57:59 +0000532 for (unsigned i = 0, e = Args.size(); i != e; ++i, ++OIdx) {
533 EVT VT = Outs[OIdx].VT;
534 Type *Ty = Args[i].Ty;
Justin Holewinskiae556d32012-05-04 20:18:50 +0000535
Justin Holewinskif8f70912013-06-28 17:57:59 +0000536 if (Outs[OIdx].Flags.isByVal() == false) {
537 if (Ty->isAggregateType()) {
538 // aggregate
539 SmallVector<EVT, 16> vtparts;
540 ComputeValueVTs(*this, Ty, vtparts);
541
542 unsigned align = getArgumentAlignment(Callee, CS, Ty, paramCount + 1);
543 // declare .param .align <align> .b8 .param<n>[<size>];
544 unsigned sz = TD->getTypeAllocSize(Ty);
545 SDVTList DeclareParamVTs = DAG.getVTList(MVT::Other, MVT::Glue);
546 SDValue DeclareParamOps[] = { Chain, DAG.getConstant(align, MVT::i32),
547 DAG.getConstant(paramCount, MVT::i32),
548 DAG.getConstant(sz, MVT::i32), InFlag };
549 Chain = DAG.getNode(NVPTXISD::DeclareParam, dl, DeclareParamVTs,
550 DeclareParamOps, 5);
551 InFlag = Chain.getValue(1);
552 unsigned curOffset = 0;
553 for (unsigned j = 0, je = vtparts.size(); j != je; ++j) {
554 unsigned elems = 1;
555 EVT elemtype = vtparts[j];
556 if (vtparts[j].isVector()) {
557 elems = vtparts[j].getVectorNumElements();
558 elemtype = vtparts[j].getVectorElementType();
559 }
560 for (unsigned k = 0, ke = elems; k != ke; ++k) {
561 unsigned sz = elemtype.getSizeInBits();
562 if (elemtype.isInteger() && (sz < 8))
563 sz = 8;
564 SDValue StVal = OutVals[OIdx];
565 if (elemtype.getSizeInBits() < 16) {
Justin Holewinskia2911282013-07-01 12:58:58 +0000566 StVal = DAG.getNode(ISD::ANY_EXTEND, dl, MVT::i16, StVal);
Justin Holewinskif8f70912013-06-28 17:57:59 +0000567 }
568 SDVTList CopyParamVTs = DAG.getVTList(MVT::Other, MVT::Glue);
569 SDValue CopyParamOps[] = { Chain,
570 DAG.getConstant(paramCount, MVT::i32),
571 DAG.getConstant(curOffset, MVT::i32),
572 StVal, InFlag };
573 Chain = DAG.getMemIntrinsicNode(NVPTXISD::StoreParam, dl,
574 CopyParamVTs, &CopyParamOps[0], 5,
575 elemtype, MachinePointerInfo());
576 InFlag = Chain.getValue(1);
577 curOffset += sz / 8;
578 ++OIdx;
579 }
580 }
581 if (vtparts.size() > 0)
582 --OIdx;
583 ++paramCount;
584 continue;
585 }
586 if (Ty->isVectorTy()) {
587 EVT ObjectVT = getValueType(Ty);
588 unsigned align = getArgumentAlignment(Callee, CS, Ty, paramCount + 1);
589 // declare .param .align <align> .b8 .param<n>[<size>];
590 unsigned sz = TD->getTypeAllocSize(Ty);
591 SDVTList DeclareParamVTs = DAG.getVTList(MVT::Other, MVT::Glue);
592 SDValue DeclareParamOps[] = { Chain, DAG.getConstant(align, MVT::i32),
593 DAG.getConstant(paramCount, MVT::i32),
594 DAG.getConstant(sz, MVT::i32), InFlag };
595 Chain = DAG.getNode(NVPTXISD::DeclareParam, dl, DeclareParamVTs,
596 DeclareParamOps, 5);
597 InFlag = Chain.getValue(1);
598 unsigned NumElts = ObjectVT.getVectorNumElements();
599 EVT EltVT = ObjectVT.getVectorElementType();
600 EVT MemVT = EltVT;
601 bool NeedExtend = false;
602 if (EltVT.getSizeInBits() < 16) {
603 NeedExtend = true;
604 EltVT = MVT::i16;
605 }
606
607 // V1 store
608 if (NumElts == 1) {
609 SDValue Elt = OutVals[OIdx++];
610 if (NeedExtend)
611 Elt = DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i16, Elt);
612
613 SDVTList CopyParamVTs = DAG.getVTList(MVT::Other, MVT::Glue);
614 SDValue CopyParamOps[] = { Chain,
615 DAG.getConstant(paramCount, MVT::i32),
616 DAG.getConstant(0, MVT::i32), Elt,
617 InFlag };
618 Chain = DAG.getMemIntrinsicNode(NVPTXISD::StoreParam, dl,
619 CopyParamVTs, &CopyParamOps[0], 5,
620 MemVT, MachinePointerInfo());
621 InFlag = Chain.getValue(1);
622 } else if (NumElts == 2) {
623 SDValue Elt0 = OutVals[OIdx++];
624 SDValue Elt1 = OutVals[OIdx++];
625 if (NeedExtend) {
626 Elt0 = DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i16, Elt0);
627 Elt1 = DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i16, Elt1);
628 }
629
630 SDVTList CopyParamVTs = DAG.getVTList(MVT::Other, MVT::Glue);
631 SDValue CopyParamOps[] = { Chain,
632 DAG.getConstant(paramCount, MVT::i32),
633 DAG.getConstant(0, MVT::i32), Elt0, Elt1,
634 InFlag };
635 Chain = DAG.getMemIntrinsicNode(NVPTXISD::StoreParamV2, dl,
636 CopyParamVTs, &CopyParamOps[0], 6,
637 MemVT, MachinePointerInfo());
638 InFlag = Chain.getValue(1);
639 } else {
640 unsigned curOffset = 0;
641 // V4 stores
642 // We have at least 4 elements (<3 x Ty> expands to 4 elements) and
643 // the
644 // vector will be expanded to a power of 2 elements, so we know we can
645 // always round up to the next multiple of 4 when creating the vector
646 // stores.
647 // e.g. 4 elem => 1 st.v4
648 // 6 elem => 2 st.v4
649 // 8 elem => 2 st.v4
650 // 11 elem => 3 st.v4
651 unsigned VecSize = 4;
652 if (EltVT.getSizeInBits() == 64)
653 VecSize = 2;
654
655 // This is potentially only part of a vector, so assume all elements
656 // are packed together.
657 unsigned PerStoreOffset = MemVT.getStoreSizeInBits() / 8 * VecSize;
658
659 for (unsigned i = 0; i < NumElts; i += VecSize) {
660 // Get values
661 SDValue StoreVal;
662 SmallVector<SDValue, 8> Ops;
663 Ops.push_back(Chain);
664 Ops.push_back(DAG.getConstant(paramCount, MVT::i32));
665 Ops.push_back(DAG.getConstant(curOffset, MVT::i32));
666
667 unsigned Opc = NVPTXISD::StoreParamV2;
668
669 StoreVal = OutVals[OIdx++];
670 if (NeedExtend)
671 StoreVal = DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i16, StoreVal);
672 Ops.push_back(StoreVal);
673
674 if (i + 1 < NumElts) {
675 StoreVal = OutVals[OIdx++];
676 if (NeedExtend)
677 StoreVal =
678 DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i16, StoreVal);
679 } else {
680 StoreVal = DAG.getUNDEF(EltVT);
681 }
682 Ops.push_back(StoreVal);
683
684 if (VecSize == 4) {
685 Opc = NVPTXISD::StoreParamV4;
686 if (i + 2 < NumElts) {
687 StoreVal = OutVals[OIdx++];
688 if (NeedExtend)
689 StoreVal =
690 DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i16, StoreVal);
691 } else {
692 StoreVal = DAG.getUNDEF(EltVT);
693 }
694 Ops.push_back(StoreVal);
695
696 if (i + 3 < NumElts) {
697 StoreVal = OutVals[OIdx++];
698 if (NeedExtend)
699 StoreVal =
700 DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i16, StoreVal);
701 } else {
702 StoreVal = DAG.getUNDEF(EltVT);
703 }
704 Ops.push_back(StoreVal);
705 }
706
Justin Holewinskidff28d22013-07-01 12:59:01 +0000707 Ops.push_back(InFlag);
708
Justin Holewinskif8f70912013-06-28 17:57:59 +0000709 SDVTList CopyParamVTs = DAG.getVTList(MVT::Other, MVT::Glue);
710 Chain = DAG.getMemIntrinsicNode(Opc, dl, CopyParamVTs, &Ops[0],
711 Ops.size(), MemVT,
712 MachinePointerInfo());
713 InFlag = Chain.getValue(1);
714 curOffset += PerStoreOffset;
715 }
716 }
717 ++paramCount;
718 --OIdx;
719 continue;
720 }
Justin Holewinskiae556d32012-05-04 20:18:50 +0000721 // Plain scalar
722 // for ABI, declare .param .b<size> .param<n>;
Justin Holewinskiae556d32012-05-04 20:18:50 +0000723 unsigned sz = VT.getSizeInBits();
Justin Holewinskif8f70912013-06-28 17:57:59 +0000724 bool needExtend = false;
725 if (VT.isInteger()) {
726 if (sz < 16)
727 needExtend = true;
728 if (sz < 32)
729 sz = 32;
730 }
Justin Holewinskiae556d32012-05-04 20:18:50 +0000731 SDVTList DeclareParamVTs = DAG.getVTList(MVT::Other, MVT::Glue);
732 SDValue DeclareParamOps[] = { Chain,
733 DAG.getConstant(paramCount, MVT::i32),
734 DAG.getConstant(sz, MVT::i32),
Justin Holewinskif8f70912013-06-28 17:57:59 +0000735 DAG.getConstant(0, MVT::i32), InFlag };
Justin Holewinskiae556d32012-05-04 20:18:50 +0000736 Chain = DAG.getNode(NVPTXISD::DeclareScalarParam, dl, DeclareParamVTs,
737 DeclareParamOps, 5);
738 InFlag = Chain.getValue(1);
Justin Holewinskif8f70912013-06-28 17:57:59 +0000739 SDValue OutV = OutVals[OIdx];
740 if (needExtend) {
741 // zext/sext i1 to i16
742 unsigned opc = ISD::ZERO_EXTEND;
743 if (Outs[OIdx].Flags.isSExt())
744 opc = ISD::SIGN_EXTEND;
745 OutV = DAG.getNode(opc, dl, MVT::i16, OutV);
746 }
Justin Holewinskiae556d32012-05-04 20:18:50 +0000747 SDVTList CopyParamVTs = DAG.getVTList(MVT::Other, MVT::Glue);
748 SDValue CopyParamOps[] = { Chain, DAG.getConstant(paramCount, MVT::i32),
Justin Holewinskif8f70912013-06-28 17:57:59 +0000749 DAG.getConstant(0, MVT::i32), OutV, InFlag };
Justin Holewinskiae556d32012-05-04 20:18:50 +0000750
751 unsigned opcode = NVPTXISD::StoreParam;
Justin Holewinskif8f70912013-06-28 17:57:59 +0000752 if (Outs[OIdx].Flags.isZExt())
753 opcode = NVPTXISD::StoreParamU32;
754 else if (Outs[OIdx].Flags.isSExt())
755 opcode = NVPTXISD::StoreParamS32;
756 Chain = DAG.getMemIntrinsicNode(opcode, dl, CopyParamVTs, CopyParamOps, 5,
757 VT, MachinePointerInfo());
Justin Holewinskiae556d32012-05-04 20:18:50 +0000758
759 InFlag = Chain.getValue(1);
760 ++paramCount;
761 continue;
762 }
763 // struct or vector
764 SmallVector<EVT, 16> vtparts;
765 const PointerType *PTy = dyn_cast<PointerType>(Args[i].Ty);
Justin Holewinski0497ab12013-03-30 14:29:21 +0000766 assert(PTy && "Type of a byval parameter should be pointer");
Justin Holewinskiae556d32012-05-04 20:18:50 +0000767 ComputeValueVTs(*this, PTy->getElementType(), vtparts);
768
Justin Holewinskif8f70912013-06-28 17:57:59 +0000769 // declare .param .align <align> .b8 .param<n>[<size>];
770 unsigned sz = Outs[OIdx].Flags.getByValSize();
771 SDVTList DeclareParamVTs = DAG.getVTList(MVT::Other, MVT::Glue);
772 // The ByValAlign in the Outs[OIdx].Flags is alway set at this point,
773 // so we don't need to worry about natural alignment or not.
774 // See TargetLowering::LowerCallTo().
775 SDValue DeclareParamOps[] = {
776 Chain, DAG.getConstant(Outs[OIdx].Flags.getByValAlign(), MVT::i32),
777 DAG.getConstant(paramCount, MVT::i32), DAG.getConstant(sz, MVT::i32),
778 InFlag
779 };
780 Chain = DAG.getNode(NVPTXISD::DeclareParam, dl, DeclareParamVTs,
781 DeclareParamOps, 5);
782 InFlag = Chain.getValue(1);
Justin Holewinskiae556d32012-05-04 20:18:50 +0000783 unsigned curOffset = 0;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000784 for (unsigned j = 0, je = vtparts.size(); j != je; ++j) {
Justin Holewinskiae556d32012-05-04 20:18:50 +0000785 unsigned elems = 1;
786 EVT elemtype = vtparts[j];
787 if (vtparts[j].isVector()) {
788 elems = vtparts[j].getVectorNumElements();
789 elemtype = vtparts[j].getVectorElementType();
790 }
Justin Holewinski0497ab12013-03-30 14:29:21 +0000791 for (unsigned k = 0, ke = elems; k != ke; ++k) {
Justin Holewinskiae556d32012-05-04 20:18:50 +0000792 unsigned sz = elemtype.getSizeInBits();
Justin Holewinskif8f70912013-06-28 17:57:59 +0000793 if (elemtype.isInteger() && (sz < 8))
794 sz = 8;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000795 SDValue srcAddr =
Justin Holewinskif8f70912013-06-28 17:57:59 +0000796 DAG.getNode(ISD::ADD, dl, getPointerTy(), OutVals[OIdx],
Justin Holewinski0497ab12013-03-30 14:29:21 +0000797 DAG.getConstant(curOffset, getPointerTy()));
Justin Holewinskif8f70912013-06-28 17:57:59 +0000798 SDValue theVal = DAG.getLoad(elemtype, dl, tempChain, srcAddr,
799 MachinePointerInfo(), false, false, false,
800 0);
801 if (elemtype.getSizeInBits() < 16) {
Justin Holewinskia2911282013-07-01 12:58:58 +0000802 theVal = DAG.getNode(ISD::ANY_EXTEND, dl, MVT::i16, theVal);
Justin Holewinskif8f70912013-06-28 17:57:59 +0000803 }
Justin Holewinskiae556d32012-05-04 20:18:50 +0000804 SDVTList CopyParamVTs = DAG.getVTList(MVT::Other, MVT::Glue);
805 SDValue CopyParamOps[] = { Chain, DAG.getConstant(paramCount, MVT::i32),
Justin Holewinskif8f70912013-06-28 17:57:59 +0000806 DAG.getConstant(curOffset, MVT::i32), theVal,
Justin Holewinskiae556d32012-05-04 20:18:50 +0000807 InFlag };
Justin Holewinskif8f70912013-06-28 17:57:59 +0000808 Chain = DAG.getMemIntrinsicNode(NVPTXISD::StoreParam, dl, CopyParamVTs,
809 CopyParamOps, 5, elemtype,
810 MachinePointerInfo());
811
Justin Holewinskiae556d32012-05-04 20:18:50 +0000812 InFlag = Chain.getValue(1);
Justin Holewinskif8f70912013-06-28 17:57:59 +0000813 curOffset += sz / 8;
Justin Holewinskiae556d32012-05-04 20:18:50 +0000814 }
815 }
Justin Holewinskif8f70912013-06-28 17:57:59 +0000816 ++paramCount;
Justin Holewinskiae556d32012-05-04 20:18:50 +0000817 }
818
819 GlobalAddressSDNode *Func = dyn_cast<GlobalAddressSDNode>(Callee.getNode());
820 unsigned retAlignment = 0;
821
822 // Handle Result
Justin Holewinskiae556d32012-05-04 20:18:50 +0000823 if (Ins.size() > 0) {
824 SmallVector<EVT, 16> resvtparts;
825 ComputeValueVTs(*this, retTy, resvtparts);
826
Justin Holewinskif8f70912013-06-28 17:57:59 +0000827 // Declare
828 // .param .align 16 .b8 retval0[<size-in-bytes>], or
829 // .param .b<size-in-bits> retval0
830 unsigned resultsz = TD->getTypeAllocSizeInBits(retTy);
831 if (retTy->isPrimitiveType() || retTy->isIntegerTy() ||
832 retTy->isPointerTy()) {
833 // Scalar needs to be at least 32bit wide
834 if (resultsz < 32)
835 resultsz = 32;
836 SDVTList DeclareRetVTs = DAG.getVTList(MVT::Other, MVT::Glue);
837 SDValue DeclareRetOps[] = { Chain, DAG.getConstant(1, MVT::i32),
838 DAG.getConstant(resultsz, MVT::i32),
839 DAG.getConstant(0, MVT::i32), InFlag };
840 Chain = DAG.getNode(NVPTXISD::DeclareRet, dl, DeclareRetVTs,
841 DeclareRetOps, 5);
842 InFlag = Chain.getValue(1);
843 } else {
844 retAlignment = getArgumentAlignment(Callee, CS, retTy, 0);
845 SDVTList DeclareRetVTs = DAG.getVTList(MVT::Other, MVT::Glue);
846 SDValue DeclareRetOps[] = { Chain,
847 DAG.getConstant(retAlignment, MVT::i32),
848 DAG.getConstant(resultsz / 8, MVT::i32),
849 DAG.getConstant(0, MVT::i32), InFlag };
850 Chain = DAG.getNode(NVPTXISD::DeclareRetParam, dl, DeclareRetVTs,
851 DeclareRetOps, 5);
852 InFlag = Chain.getValue(1);
Justin Holewinskiae556d32012-05-04 20:18:50 +0000853 }
854 }
855
856 if (!Func) {
857 // This is indirect function call case : PTX requires a prototype of the
858 // form
859 // proto_0 : .callprototype(.param .b32 _) _ (.param .b32 _);
860 // to be emitted, and the label has to used as the last arg of call
861 // instruction.
862 // The prototype is embedded in a string and put as the operand for an
863 // INLINEASM SDNode.
864 SDVTList InlineAsmVTs = DAG.getVTList(MVT::Other, MVT::Glue);
Justin Holewinskif8f70912013-06-28 17:57:59 +0000865 std::string proto_string =
866 getPrototype(retTy, Args, Outs, retAlignment, CS);
Justin Holewinski0497ab12013-03-30 14:29:21 +0000867 const char *asmstr = nvTM->getManagedStrPool()
868 ->getManagedString(proto_string.c_str())->c_str();
869 SDValue InlineAsmOps[] = {
870 Chain, DAG.getTargetExternalSymbol(asmstr, getPointerTy()),
871 DAG.getMDNode(0), DAG.getTargetConstant(0, MVT::i32), InFlag
872 };
Justin Holewinskiae556d32012-05-04 20:18:50 +0000873 Chain = DAG.getNode(ISD::INLINEASM, dl, InlineAsmVTs, InlineAsmOps, 5);
874 InFlag = Chain.getValue(1);
875 }
876 // Op to just print "call"
877 SDVTList PrintCallVTs = DAG.getVTList(MVT::Other, MVT::Glue);
Justin Holewinski0497ab12013-03-30 14:29:21 +0000878 SDValue PrintCallOps[] = {
Justin Holewinskif8f70912013-06-28 17:57:59 +0000879 Chain, DAG.getConstant((Ins.size() == 0) ? 0 : 1, MVT::i32), InFlag
Justin Holewinski0497ab12013-03-30 14:29:21 +0000880 };
881 Chain = DAG.getNode(Func ? (NVPTXISD::PrintCallUni) : (NVPTXISD::PrintCall),
882 dl, PrintCallVTs, PrintCallOps, 3);
Justin Holewinskiae556d32012-05-04 20:18:50 +0000883 InFlag = Chain.getValue(1);
884
885 // Ops to print out the function name
886 SDVTList CallVoidVTs = DAG.getVTList(MVT::Other, MVT::Glue);
887 SDValue CallVoidOps[] = { Chain, Callee, InFlag };
888 Chain = DAG.getNode(NVPTXISD::CallVoid, dl, CallVoidVTs, CallVoidOps, 3);
889 InFlag = Chain.getValue(1);
890
891 // Ops to print out the param list
892 SDVTList CallArgBeginVTs = DAG.getVTList(MVT::Other, MVT::Glue);
893 SDValue CallArgBeginOps[] = { Chain, InFlag };
894 Chain = DAG.getNode(NVPTXISD::CallArgBegin, dl, CallArgBeginVTs,
895 CallArgBeginOps, 2);
896 InFlag = Chain.getValue(1);
897
Justin Holewinski0497ab12013-03-30 14:29:21 +0000898 for (unsigned i = 0, e = paramCount; i != e; ++i) {
Justin Holewinskiae556d32012-05-04 20:18:50 +0000899 unsigned opcode;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000900 if (i == (e - 1))
Justin Holewinskiae556d32012-05-04 20:18:50 +0000901 opcode = NVPTXISD::LastCallArg;
902 else
903 opcode = NVPTXISD::CallArg;
904 SDVTList CallArgVTs = DAG.getVTList(MVT::Other, MVT::Glue);
905 SDValue CallArgOps[] = { Chain, DAG.getConstant(1, MVT::i32),
Justin Holewinski0497ab12013-03-30 14:29:21 +0000906 DAG.getConstant(i, MVT::i32), InFlag };
Justin Holewinskiae556d32012-05-04 20:18:50 +0000907 Chain = DAG.getNode(opcode, dl, CallArgVTs, CallArgOps, 4);
908 InFlag = Chain.getValue(1);
909 }
910 SDVTList CallArgEndVTs = DAG.getVTList(MVT::Other, MVT::Glue);
Justin Holewinski0497ab12013-03-30 14:29:21 +0000911 SDValue CallArgEndOps[] = { Chain, DAG.getConstant(Func ? 1 : 0, MVT::i32),
Justin Holewinskiae556d32012-05-04 20:18:50 +0000912 InFlag };
Justin Holewinski0497ab12013-03-30 14:29:21 +0000913 Chain =
914 DAG.getNode(NVPTXISD::CallArgEnd, dl, CallArgEndVTs, CallArgEndOps, 3);
Justin Holewinskiae556d32012-05-04 20:18:50 +0000915 InFlag = Chain.getValue(1);
916
917 if (!Func) {
918 SDVTList PrototypeVTs = DAG.getVTList(MVT::Other, MVT::Glue);
Justin Holewinski0497ab12013-03-30 14:29:21 +0000919 SDValue PrototypeOps[] = { Chain, DAG.getConstant(uniqueCallSite, MVT::i32),
Justin Holewinskiae556d32012-05-04 20:18:50 +0000920 InFlag };
921 Chain = DAG.getNode(NVPTXISD::Prototype, dl, PrototypeVTs, PrototypeOps, 3);
922 InFlag = Chain.getValue(1);
923 }
924
925 // Generate loads from param memory/moves from registers for result
926 if (Ins.size() > 0) {
Justin Holewinskif8f70912013-06-28 17:57:59 +0000927 unsigned resoffset = 0;
928 if (retTy && retTy->isVectorTy()) {
929 EVT ObjectVT = getValueType(retTy);
930 unsigned NumElts = ObjectVT.getVectorNumElements();
931 EVT EltVT = ObjectVT.getVectorElementType();
Benjamin Kramer3cc579a2013-06-29 22:51:12 +0000932 assert(nvTM->getTargetLowering()->getNumRegisters(F->getContext(),
933 ObjectVT) == NumElts &&
Justin Holewinskif8f70912013-06-28 17:57:59 +0000934 "Vector was not scalarized");
935 unsigned sz = EltVT.getSizeInBits();
936 bool needTruncate = sz < 16 ? true : false;
937
938 if (NumElts == 1) {
939 // Just a simple load
940 std::vector<EVT> LoadRetVTs;
941 if (needTruncate) {
942 // If loading i1 result, generate
943 // load i16
944 // trunc i16 to i1
945 LoadRetVTs.push_back(MVT::i16);
946 } else
947 LoadRetVTs.push_back(EltVT);
948 LoadRetVTs.push_back(MVT::Other);
949 LoadRetVTs.push_back(MVT::Glue);
950 std::vector<SDValue> LoadRetOps;
951 LoadRetOps.push_back(Chain);
952 LoadRetOps.push_back(DAG.getConstant(1, MVT::i32));
953 LoadRetOps.push_back(DAG.getConstant(0, MVT::i32));
954 LoadRetOps.push_back(InFlag);
955 SDValue retval = DAG.getMemIntrinsicNode(
956 NVPTXISD::LoadParam, dl,
957 DAG.getVTList(&LoadRetVTs[0], LoadRetVTs.size()), &LoadRetOps[0],
958 LoadRetOps.size(), EltVT, MachinePointerInfo());
Justin Holewinskiae556d32012-05-04 20:18:50 +0000959 Chain = retval.getValue(1);
960 InFlag = retval.getValue(2);
Justin Holewinskif8f70912013-06-28 17:57:59 +0000961 SDValue Ret0 = retval;
962 if (needTruncate)
963 Ret0 = DAG.getNode(ISD::TRUNCATE, dl, EltVT, Ret0);
964 InVals.push_back(Ret0);
965 } else if (NumElts == 2) {
966 // LoadV2
967 std::vector<EVT> LoadRetVTs;
968 if (needTruncate) {
969 // If loading i1 result, generate
970 // load i16
971 // trunc i16 to i1
972 LoadRetVTs.push_back(MVT::i16);
973 LoadRetVTs.push_back(MVT::i16);
974 } else {
975 LoadRetVTs.push_back(EltVT);
976 LoadRetVTs.push_back(EltVT);
977 }
978 LoadRetVTs.push_back(MVT::Other);
979 LoadRetVTs.push_back(MVT::Glue);
980 std::vector<SDValue> LoadRetOps;
981 LoadRetOps.push_back(Chain);
982 LoadRetOps.push_back(DAG.getConstant(1, MVT::i32));
983 LoadRetOps.push_back(DAG.getConstant(0, MVT::i32));
984 LoadRetOps.push_back(InFlag);
985 SDValue retval = DAG.getMemIntrinsicNode(
986 NVPTXISD::LoadParamV2, dl,
987 DAG.getVTList(&LoadRetVTs[0], LoadRetVTs.size()), &LoadRetOps[0],
988 LoadRetOps.size(), EltVT, MachinePointerInfo());
989 Chain = retval.getValue(2);
990 InFlag = retval.getValue(3);
991 SDValue Ret0 = retval.getValue(0);
992 SDValue Ret1 = retval.getValue(1);
993 if (needTruncate) {
994 Ret0 = DAG.getNode(ISD::TRUNCATE, dl, MVT::i1, Ret0);
995 InVals.push_back(Ret0);
996 Ret1 = DAG.getNode(ISD::TRUNCATE, dl, MVT::i1, Ret1);
997 InVals.push_back(Ret1);
998 } else {
999 InVals.push_back(Ret0);
1000 InVals.push_back(Ret1);
1001 }
1002 } else {
1003 // Split into N LoadV4
1004 unsigned Ofst = 0;
1005 unsigned VecSize = 4;
1006 unsigned Opc = NVPTXISD::LoadParamV4;
1007 if (EltVT.getSizeInBits() == 64) {
1008 VecSize = 2;
1009 Opc = NVPTXISD::LoadParamV2;
1010 }
1011 EVT VecVT = EVT::getVectorVT(F->getContext(), EltVT, VecSize);
1012 for (unsigned i = 0; i < NumElts; i += VecSize) {
1013 SmallVector<EVT, 8> LoadRetVTs;
1014 if (needTruncate) {
1015 // If loading i1 result, generate
1016 // load i16
1017 // trunc i16 to i1
1018 for (unsigned j = 0; j < VecSize; ++j)
1019 LoadRetVTs.push_back(MVT::i16);
1020 } else {
1021 for (unsigned j = 0; j < VecSize; ++j)
1022 LoadRetVTs.push_back(EltVT);
1023 }
1024 LoadRetVTs.push_back(MVT::Other);
1025 LoadRetVTs.push_back(MVT::Glue);
1026 SmallVector<SDValue, 4> LoadRetOps;
1027 LoadRetOps.push_back(Chain);
1028 LoadRetOps.push_back(DAG.getConstant(1, MVT::i32));
1029 LoadRetOps.push_back(DAG.getConstant(Ofst, MVT::i32));
1030 LoadRetOps.push_back(InFlag);
1031 SDValue retval = DAG.getMemIntrinsicNode(
1032 Opc, dl, DAG.getVTList(&LoadRetVTs[0], LoadRetVTs.size()),
1033 &LoadRetOps[0], LoadRetOps.size(), EltVT, MachinePointerInfo());
1034 if (VecSize == 2) {
1035 Chain = retval.getValue(2);
1036 InFlag = retval.getValue(3);
1037 } else {
1038 Chain = retval.getValue(4);
1039 InFlag = retval.getValue(5);
1040 }
1041
1042 for (unsigned j = 0; j < VecSize; ++j) {
1043 if (i + j >= NumElts)
1044 break;
1045 SDValue Elt = retval.getValue(j);
1046 if (needTruncate)
1047 Elt = DAG.getNode(ISD::TRUNCATE, dl, EltVT, Elt);
1048 InVals.push_back(Elt);
1049 }
1050 Ofst += TD->getTypeAllocSize(VecVT.getTypeForEVT(F->getContext()));
1051 }
Justin Holewinskiae556d32012-05-04 20:18:50 +00001052 }
Justin Holewinski0497ab12013-03-30 14:29:21 +00001053 } else {
Justin Holewinskif8f70912013-06-28 17:57:59 +00001054 SmallVector<EVT, 16> VTs;
1055 ComputePTXValueVTs(*this, retTy, VTs);
1056 assert(VTs.size() == Ins.size() && "Bad value decomposition");
Justin Holewinski0497ab12013-03-30 14:29:21 +00001057 for (unsigned i = 0, e = Ins.size(); i != e; ++i) {
Justin Holewinskif8f70912013-06-28 17:57:59 +00001058 unsigned sz = VTs[i].getSizeInBits();
1059 bool needTruncate = sz < 8 ? true : false;
1060 if (VTs[i].isInteger() && (sz < 8))
1061 sz = 8;
1062
1063 SmallVector<EVT, 4> LoadRetVTs;
Justin Holewinskie04e4bd2013-06-28 17:58:10 +00001064 EVT TheLoadType = VTs[i];
1065 if (retTy->isIntegerTy() &&
1066 TD->getTypeAllocSizeInBits(retTy) < 32) {
1067 // This is for integer types only, and specifically not for
1068 // aggregates.
1069 LoadRetVTs.push_back(MVT::i32);
1070 TheLoadType = MVT::i32;
1071 } else if (sz < 16) {
Justin Holewinskif8f70912013-06-28 17:57:59 +00001072 // If loading i1/i8 result, generate
1073 // load i8 (-> i16)
1074 // trunc i16 to i1/i8
1075 LoadRetVTs.push_back(MVT::i16);
1076 } else
1077 LoadRetVTs.push_back(Ins[i].VT);
1078 LoadRetVTs.push_back(MVT::Other);
1079 LoadRetVTs.push_back(MVT::Glue);
1080
1081 SmallVector<SDValue, 4> LoadRetOps;
1082 LoadRetOps.push_back(Chain);
1083 LoadRetOps.push_back(DAG.getConstant(1, MVT::i32));
1084 LoadRetOps.push_back(DAG.getConstant(resoffset, MVT::i32));
1085 LoadRetOps.push_back(InFlag);
1086 SDValue retval = DAG.getMemIntrinsicNode(
1087 NVPTXISD::LoadParam, dl,
1088 DAG.getVTList(&LoadRetVTs[0], LoadRetVTs.size()), &LoadRetOps[0],
Justin Holewinskie04e4bd2013-06-28 17:58:10 +00001089 LoadRetOps.size(), TheLoadType, MachinePointerInfo());
Justin Holewinskif8f70912013-06-28 17:57:59 +00001090 Chain = retval.getValue(1);
1091 InFlag = retval.getValue(2);
1092 SDValue Ret0 = retval.getValue(0);
1093 if (needTruncate)
1094 Ret0 = DAG.getNode(ISD::TRUNCATE, dl, Ins[i].VT, Ret0);
1095 InVals.push_back(Ret0);
1096 resoffset += sz / 8;
Justin Holewinskiae556d32012-05-04 20:18:50 +00001097 }
1098 }
1099 }
Justin Holewinskif8f70912013-06-28 17:57:59 +00001100
Justin Holewinski0497ab12013-03-30 14:29:21 +00001101 Chain = DAG.getCALLSEQ_END(Chain, DAG.getIntPtrConstant(uniqueCallSite, true),
1102 DAG.getIntPtrConstant(uniqueCallSite + 1, true),
Andrew Trickad6d08a2013-05-29 22:03:55 +00001103 InFlag, dl);
Justin Holewinskiae556d32012-05-04 20:18:50 +00001104 uniqueCallSite++;
1105
1106 // set isTailCall to false for now, until we figure out how to express
1107 // tail call optimization in PTX
1108 isTailCall = false;
1109 return Chain;
1110}
Justin Holewinskiae556d32012-05-04 20:18:50 +00001111
1112// By default CONCAT_VECTORS is lowered by ExpandVectorBuildThroughStack()
1113// (see LegalizeDAG.cpp). This is slow and uses local memory.
1114// We use extract/insert/build vector just as what LegalizeOp() does in llvm 2.5
Justin Holewinski0497ab12013-03-30 14:29:21 +00001115SDValue
1116NVPTXTargetLowering::LowerCONCAT_VECTORS(SDValue Op, SelectionDAG &DAG) const {
Justin Holewinskiae556d32012-05-04 20:18:50 +00001117 SDNode *Node = Op.getNode();
Andrew Trickef9de2a2013-05-25 02:42:55 +00001118 SDLoc dl(Node);
Justin Holewinskiae556d32012-05-04 20:18:50 +00001119 SmallVector<SDValue, 8> Ops;
1120 unsigned NumOperands = Node->getNumOperands();
Justin Holewinski0497ab12013-03-30 14:29:21 +00001121 for (unsigned i = 0; i < NumOperands; ++i) {
Justin Holewinskiae556d32012-05-04 20:18:50 +00001122 SDValue SubOp = Node->getOperand(i);
1123 EVT VVT = SubOp.getNode()->getValueType(0);
1124 EVT EltVT = VVT.getVectorElementType();
1125 unsigned NumSubElem = VVT.getVectorNumElements();
Justin Holewinski0497ab12013-03-30 14:29:21 +00001126 for (unsigned j = 0; j < NumSubElem; ++j) {
Justin Holewinskiae556d32012-05-04 20:18:50 +00001127 Ops.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, SubOp,
1128 DAG.getIntPtrConstant(j)));
1129 }
1130 }
Justin Holewinski0497ab12013-03-30 14:29:21 +00001131 return DAG.getNode(ISD::BUILD_VECTOR, dl, Node->getValueType(0), &Ops[0],
1132 Ops.size());
Justin Holewinskiae556d32012-05-04 20:18:50 +00001133}
1134
Justin Holewinski0497ab12013-03-30 14:29:21 +00001135SDValue
1136NVPTXTargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const {
Justin Holewinskiae556d32012-05-04 20:18:50 +00001137 switch (Op.getOpcode()) {
Justin Holewinski0497ab12013-03-30 14:29:21 +00001138 case ISD::RETURNADDR:
1139 return SDValue();
1140 case ISD::FRAMEADDR:
1141 return SDValue();
1142 case ISD::GlobalAddress:
1143 return LowerGlobalAddress(Op, DAG);
1144 case ISD::INTRINSIC_W_CHAIN:
1145 return Op;
Justin Holewinskiae556d32012-05-04 20:18:50 +00001146 case ISD::BUILD_VECTOR:
1147 case ISD::EXTRACT_SUBVECTOR:
1148 return Op;
Justin Holewinski0497ab12013-03-30 14:29:21 +00001149 case ISD::CONCAT_VECTORS:
1150 return LowerCONCAT_VECTORS(Op, DAG);
1151 case ISD::STORE:
1152 return LowerSTORE(Op, DAG);
1153 case ISD::LOAD:
1154 return LowerLOAD(Op, DAG);
Justin Holewinskiae556d32012-05-04 20:18:50 +00001155 default:
David Blaikie891d0a32012-05-04 22:34:16 +00001156 llvm_unreachable("Custom lowering not defined for operation");
Justin Holewinskiae556d32012-05-04 20:18:50 +00001157 }
1158}
1159
Justin Holewinskibe8dc642013-02-12 14:18:49 +00001160SDValue NVPTXTargetLowering::LowerLOAD(SDValue Op, SelectionDAG &DAG) const {
1161 if (Op.getValueType() == MVT::i1)
1162 return LowerLOADi1(Op, DAG);
1163 else
1164 return SDValue();
1165}
1166
Justin Holewinskic6462aa2012-11-14 19:19:16 +00001167// v = ld i1* addr
1168// =>
Justin Holewinskif8f70912013-06-28 17:57:59 +00001169// v1 = ld i8* addr (-> i16)
1170// v = trunc i16 to i1
Justin Holewinski0497ab12013-03-30 14:29:21 +00001171SDValue NVPTXTargetLowering::LowerLOADi1(SDValue Op, SelectionDAG &DAG) const {
Justin Holewinskic6462aa2012-11-14 19:19:16 +00001172 SDNode *Node = Op.getNode();
1173 LoadSDNode *LD = cast<LoadSDNode>(Node);
Andrew Trickef9de2a2013-05-25 02:42:55 +00001174 SDLoc dl(Node);
Justin Holewinski0497ab12013-03-30 14:29:21 +00001175 assert(LD->getExtensionType() == ISD::NON_EXTLOAD);
NAKAMURA Takumi5bbe0e12012-11-14 23:46:15 +00001176 assert(Node->getValueType(0) == MVT::i1 &&
1177 "Custom lowering for i1 load only");
Justin Holewinski0497ab12013-03-30 14:29:21 +00001178 SDValue newLD =
Justin Holewinskif8f70912013-06-28 17:57:59 +00001179 DAG.getLoad(MVT::i16, dl, LD->getChain(), LD->getBasePtr(),
Justin Holewinski0497ab12013-03-30 14:29:21 +00001180 LD->getPointerInfo(), LD->isVolatile(), LD->isNonTemporal(),
1181 LD->isInvariant(), LD->getAlignment());
Justin Holewinskic6462aa2012-11-14 19:19:16 +00001182 SDValue result = DAG.getNode(ISD::TRUNCATE, dl, MVT::i1, newLD);
1183 // The legalizer (the caller) is expecting two values from the legalized
1184 // load, so we build a MergeValues node for it. See ExpandUnalignedLoad()
1185 // in LegalizeDAG.cpp which also uses MergeValues.
Justin Holewinski0497ab12013-03-30 14:29:21 +00001186 SDValue Ops[] = { result, LD->getChain() };
Justin Holewinskic6462aa2012-11-14 19:19:16 +00001187 return DAG.getMergeValues(Ops, 2, dl);
1188}
1189
Justin Holewinskibe8dc642013-02-12 14:18:49 +00001190SDValue NVPTXTargetLowering::LowerSTORE(SDValue Op, SelectionDAG &DAG) const {
1191 EVT ValVT = Op.getOperand(1).getValueType();
1192 if (ValVT == MVT::i1)
1193 return LowerSTOREi1(Op, DAG);
1194 else if (ValVT.isVector())
1195 return LowerSTOREVector(Op, DAG);
1196 else
1197 return SDValue();
1198}
1199
1200SDValue
1201NVPTXTargetLowering::LowerSTOREVector(SDValue Op, SelectionDAG &DAG) const {
1202 SDNode *N = Op.getNode();
1203 SDValue Val = N->getOperand(1);
Andrew Trickef9de2a2013-05-25 02:42:55 +00001204 SDLoc DL(N);
Justin Holewinskibe8dc642013-02-12 14:18:49 +00001205 EVT ValVT = Val.getValueType();
1206
1207 if (ValVT.isVector()) {
1208 // We only handle "native" vector sizes for now, e.g. <4 x double> is not
1209 // legal. We can (and should) split that into 2 stores of <2 x double> here
1210 // but I'm leaving that as a TODO for now.
1211 if (!ValVT.isSimple())
1212 return SDValue();
1213 switch (ValVT.getSimpleVT().SimpleTy) {
Justin Holewinski0497ab12013-03-30 14:29:21 +00001214 default:
1215 return SDValue();
Justin Holewinskibe8dc642013-02-12 14:18:49 +00001216 case MVT::v2i8:
1217 case MVT::v2i16:
1218 case MVT::v2i32:
1219 case MVT::v2i64:
1220 case MVT::v2f32:
1221 case MVT::v2f64:
1222 case MVT::v4i8:
1223 case MVT::v4i16:
1224 case MVT::v4i32:
1225 case MVT::v4f32:
1226 // This is a "native" vector type
1227 break;
1228 }
1229
1230 unsigned Opcode = 0;
1231 EVT EltVT = ValVT.getVectorElementType();
1232 unsigned NumElts = ValVT.getVectorNumElements();
1233
1234 // Since StoreV2 is a target node, we cannot rely on DAG type legalization.
1235 // Therefore, we must ensure the type is legal. For i1 and i8, we set the
1236 // stored type to i16 and propogate the "real" type as the memory type.
Justin Holewinskia2911282013-07-01 12:58:58 +00001237 bool NeedExt = false;
Justin Holewinskibe8dc642013-02-12 14:18:49 +00001238 if (EltVT.getSizeInBits() < 16)
Justin Holewinskia2911282013-07-01 12:58:58 +00001239 NeedExt = true;
Justin Holewinskibe8dc642013-02-12 14:18:49 +00001240
1241 switch (NumElts) {
Justin Holewinski0497ab12013-03-30 14:29:21 +00001242 default:
1243 return SDValue();
Justin Holewinskibe8dc642013-02-12 14:18:49 +00001244 case 2:
1245 Opcode = NVPTXISD::StoreV2;
1246 break;
1247 case 4: {
1248 Opcode = NVPTXISD::StoreV4;
1249 break;
1250 }
1251 }
1252
1253 SmallVector<SDValue, 8> Ops;
1254
1255 // First is the chain
1256 Ops.push_back(N->getOperand(0));
1257
1258 // Then the split values
1259 for (unsigned i = 0; i < NumElts; ++i) {
1260 SDValue ExtVal = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, EltVT, Val,
1261 DAG.getIntPtrConstant(i));
Justin Holewinskia2911282013-07-01 12:58:58 +00001262 if (NeedExt)
1263 ExtVal = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i16, ExtVal);
Justin Holewinskibe8dc642013-02-12 14:18:49 +00001264 Ops.push_back(ExtVal);
1265 }
1266
1267 // Then any remaining arguments
1268 for (unsigned i = 2, e = N->getNumOperands(); i != e; ++i) {
1269 Ops.push_back(N->getOperand(i));
1270 }
1271
1272 MemSDNode *MemSD = cast<MemSDNode>(N);
1273
Justin Holewinski0497ab12013-03-30 14:29:21 +00001274 SDValue NewSt = DAG.getMemIntrinsicNode(
1275 Opcode, DL, DAG.getVTList(MVT::Other), &Ops[0], Ops.size(),
1276 MemSD->getMemoryVT(), MemSD->getMemOperand());
Justin Holewinskibe8dc642013-02-12 14:18:49 +00001277
1278 //return DCI.CombineTo(N, NewSt, true);
1279 return NewSt;
1280 }
1281
1282 return SDValue();
1283}
1284
Justin Holewinskic6462aa2012-11-14 19:19:16 +00001285// st i1 v, addr
1286// =>
Justin Holewinskif8f70912013-06-28 17:57:59 +00001287// v1 = zxt v to i16
1288// st.u8 i16, addr
Justin Holewinski0497ab12013-03-30 14:29:21 +00001289SDValue NVPTXTargetLowering::LowerSTOREi1(SDValue Op, SelectionDAG &DAG) const {
Justin Holewinskic6462aa2012-11-14 19:19:16 +00001290 SDNode *Node = Op.getNode();
Andrew Trickef9de2a2013-05-25 02:42:55 +00001291 SDLoc dl(Node);
Justin Holewinskic6462aa2012-11-14 19:19:16 +00001292 StoreSDNode *ST = cast<StoreSDNode>(Node);
1293 SDValue Tmp1 = ST->getChain();
1294 SDValue Tmp2 = ST->getBasePtr();
1295 SDValue Tmp3 = ST->getValue();
NAKAMURA Takumi5bbe0e12012-11-14 23:46:15 +00001296 assert(Tmp3.getValueType() == MVT::i1 && "Custom lowering for i1 store only");
Justin Holewinskic6462aa2012-11-14 19:19:16 +00001297 unsigned Alignment = ST->getAlignment();
1298 bool isVolatile = ST->isVolatile();
1299 bool isNonTemporal = ST->isNonTemporal();
Justin Holewinskif8f70912013-06-28 17:57:59 +00001300 Tmp3 = DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i16, Tmp3);
1301 SDValue Result = DAG.getTruncStore(Tmp1, dl, Tmp3, Tmp2,
1302 ST->getPointerInfo(), MVT::i8, isNonTemporal,
1303 isVolatile, Alignment);
Justin Holewinskic6462aa2012-11-14 19:19:16 +00001304 return Result;
1305}
1306
Justin Holewinski0497ab12013-03-30 14:29:21 +00001307SDValue NVPTXTargetLowering::getExtSymb(SelectionDAG &DAG, const char *inname,
1308 int idx, EVT v) const {
Justin Holewinskiae556d32012-05-04 20:18:50 +00001309 std::string *name = nvTM->getManagedStrPool()->getManagedString(inname);
1310 std::stringstream suffix;
1311 suffix << idx;
1312 *name += suffix.str();
1313 return DAG.getTargetExternalSymbol(name->c_str(), v);
1314}
1315
1316SDValue
1317NVPTXTargetLowering::getParamSymbol(SelectionDAG &DAG, int idx, EVT v) const {
1318 return getExtSymb(DAG, ".PARAM", idx, v);
1319}
1320
Justin Holewinski0497ab12013-03-30 14:29:21 +00001321SDValue NVPTXTargetLowering::getParamHelpSymbol(SelectionDAG &DAG, int idx) {
Justin Holewinskiae556d32012-05-04 20:18:50 +00001322 return getExtSymb(DAG, ".HLPPARAM", idx);
1323}
1324
1325// Check to see if the kernel argument is image*_t or sampler_t
1326
1327bool llvm::isImageOrSamplerVal(const Value *arg, const Module *context) {
Justin Holewinski0497ab12013-03-30 14:29:21 +00001328 static const char *const specialTypes[] = { "struct._image2d_t",
1329 "struct._image3d_t",
1330 "struct._sampler_t" };
Justin Holewinskiae556d32012-05-04 20:18:50 +00001331
1332 const Type *Ty = arg->getType();
1333 const PointerType *PTy = dyn_cast<PointerType>(Ty);
1334
1335 if (!PTy)
1336 return false;
1337
1338 if (!context)
1339 return false;
1340
1341 const StructType *STy = dyn_cast<StructType>(PTy->getElementType());
Justin Holewinskifb711152012-12-05 20:50:28 +00001342 const std::string TypeName = STy && !STy->isLiteral() ? STy->getName() : "";
Justin Holewinskiae556d32012-05-04 20:18:50 +00001343
Craig Toppere4260f92012-05-24 04:22:05 +00001344 for (int i = 0, e = array_lengthof(specialTypes); i != e; ++i)
Justin Holewinskiae556d32012-05-04 20:18:50 +00001345 if (TypeName == specialTypes[i])
1346 return true;
1347
1348 return false;
1349}
1350
Justin Holewinski0497ab12013-03-30 14:29:21 +00001351SDValue NVPTXTargetLowering::LowerFormalArguments(
1352 SDValue Chain, CallingConv::ID CallConv, bool isVarArg,
Andrew Trickef9de2a2013-05-25 02:42:55 +00001353 const SmallVectorImpl<ISD::InputArg> &Ins, SDLoc dl, SelectionDAG &DAG,
Justin Holewinski0497ab12013-03-30 14:29:21 +00001354 SmallVectorImpl<SDValue> &InVals) const {
Justin Holewinskiae556d32012-05-04 20:18:50 +00001355 MachineFunction &MF = DAG.getMachineFunction();
Micah Villmowcdfe20b2012-10-08 16:38:25 +00001356 const DataLayout *TD = getDataLayout();
Justin Holewinskiae556d32012-05-04 20:18:50 +00001357
1358 const Function *F = MF.getFunction();
Bill Wendlinge94d8432012-12-07 23:16:57 +00001359 const AttributeSet &PAL = F->getAttributes();
Justin Holewinski44f5c602013-06-28 17:57:53 +00001360 const TargetLowering *TLI = nvTM->getTargetLowering();
Justin Holewinskiae556d32012-05-04 20:18:50 +00001361
1362 SDValue Root = DAG.getRoot();
1363 std::vector<SDValue> OutChains;
1364
1365 bool isKernel = llvm::isKernelFunction(*F);
1366 bool isABI = (nvptxSubtarget.getSmVersion() >= 20);
Justin Holewinski44f5c602013-06-28 17:57:53 +00001367 assert(isABI && "Non-ABI compilation is not supported");
1368 if (!isABI)
1369 return Chain;
Justin Holewinskiae556d32012-05-04 20:18:50 +00001370
1371 std::vector<Type *> argTypes;
1372 std::vector<const Argument *> theArgs;
1373 for (Function::const_arg_iterator I = F->arg_begin(), E = F->arg_end();
Justin Holewinski0497ab12013-03-30 14:29:21 +00001374 I != E; ++I) {
Justin Holewinskiae556d32012-05-04 20:18:50 +00001375 theArgs.push_back(I);
1376 argTypes.push_back(I->getType());
1377 }
Justin Holewinski44f5c602013-06-28 17:57:53 +00001378 // argTypes.size() (or theArgs.size()) and Ins.size() need not match.
1379 // Ins.size() will be larger
1380 // * if there is an aggregate argument with multiple fields (each field
1381 // showing up separately in Ins)
1382 // * if there is a vector argument with more than typical vector-length
1383 // elements (generally if more than 4) where each vector element is
1384 // individually present in Ins.
1385 // So a different index should be used for indexing into Ins.
1386 // See similar issue in LowerCall.
1387 unsigned InsIdx = 0;
Justin Holewinskiae556d32012-05-04 20:18:50 +00001388
1389 int idx = 0;
Justin Holewinski44f5c602013-06-28 17:57:53 +00001390 for (unsigned i = 0, e = theArgs.size(); i != e; ++i, ++idx, ++InsIdx) {
Justin Holewinskiae556d32012-05-04 20:18:50 +00001391 Type *Ty = argTypes[i];
Justin Holewinskiae556d32012-05-04 20:18:50 +00001392
1393 // If the kernel argument is image*_t or sampler_t, convert it to
1394 // a i32 constant holding the parameter position. This can later
1395 // matched in the AsmPrinter to output the correct mangled name.
Justin Holewinski0497ab12013-03-30 14:29:21 +00001396 if (isImageOrSamplerVal(
1397 theArgs[i],
1398 (theArgs[i]->getParent() ? theArgs[i]->getParent()->getParent()
1399 : 0))) {
Justin Holewinskiae556d32012-05-04 20:18:50 +00001400 assert(isKernel && "Only kernels can have image/sampler params");
Justin Holewinski0497ab12013-03-30 14:29:21 +00001401 InVals.push_back(DAG.getConstant(i + 1, MVT::i32));
Justin Holewinskiae556d32012-05-04 20:18:50 +00001402 continue;
1403 }
1404
1405 if (theArgs[i]->use_empty()) {
1406 // argument is dead
Justin Holewinski44f5c602013-06-28 17:57:53 +00001407 if (Ty->isAggregateType()) {
1408 SmallVector<EVT, 16> vtparts;
1409
Justin Holewinskif8f70912013-06-28 17:57:59 +00001410 ComputePTXValueVTs(*this, Ty, vtparts);
Justin Holewinski44f5c602013-06-28 17:57:53 +00001411 assert(vtparts.size() > 0 && "empty aggregate type not expected");
1412 for (unsigned parti = 0, parte = vtparts.size(); parti != parte;
1413 ++parti) {
1414 EVT partVT = vtparts[parti];
1415 InVals.push_back(DAG.getNode(ISD::UNDEF, dl, partVT));
1416 ++InsIdx;
Justin Holewinskie9884092013-03-24 21:17:47 +00001417 }
Justin Holewinski44f5c602013-06-28 17:57:53 +00001418 if (vtparts.size() > 0)
1419 --InsIdx;
1420 continue;
Justin Holewinskie9884092013-03-24 21:17:47 +00001421 }
Justin Holewinski44f5c602013-06-28 17:57:53 +00001422 if (Ty->isVectorTy()) {
1423 EVT ObjectVT = getValueType(Ty);
1424 unsigned NumRegs = TLI->getNumRegisters(F->getContext(), ObjectVT);
1425 for (unsigned parti = 0; parti < NumRegs; ++parti) {
1426 InVals.push_back(DAG.getNode(ISD::UNDEF, dl, Ins[InsIdx].VT));
1427 ++InsIdx;
1428 }
1429 if (NumRegs > 0)
1430 --InsIdx;
1431 continue;
1432 }
1433 InVals.push_back(DAG.getNode(ISD::UNDEF, dl, Ins[InsIdx].VT));
Justin Holewinskiae556d32012-05-04 20:18:50 +00001434 continue;
1435 }
1436
1437 // In the following cases, assign a node order of "idx+1"
Justin Holewinski44f5c602013-06-28 17:57:53 +00001438 // to newly created nodes. The SDNodes for params have to
Justin Holewinskiae556d32012-05-04 20:18:50 +00001439 // appear in the same order as their order of appearance
1440 // in the original function. "idx+1" holds that order.
Justin Holewinski0497ab12013-03-30 14:29:21 +00001441 if (PAL.hasAttribute(i + 1, Attribute::ByVal) == false) {
Justin Holewinski44f5c602013-06-28 17:57:53 +00001442 if (Ty->isAggregateType()) {
1443 SmallVector<EVT, 16> vtparts;
1444 SmallVector<uint64_t, 16> offsets;
1445
Justin Holewinskif8f70912013-06-28 17:57:59 +00001446 // NOTE: Here, we lose the ability to issue vector loads for vectors
1447 // that are a part of a struct. This should be investigated in the
1448 // future.
1449 ComputePTXValueVTs(*this, Ty, vtparts, &offsets, 0);
Justin Holewinski44f5c602013-06-28 17:57:53 +00001450 assert(vtparts.size() > 0 && "empty aggregate type not expected");
1451 bool aggregateIsPacked = false;
1452 if (StructType *STy = llvm::dyn_cast<StructType>(Ty))
1453 aggregateIsPacked = STy->isPacked();
1454
1455 SDValue Arg = getParamSymbol(DAG, idx, getPointerTy());
1456 for (unsigned parti = 0, parte = vtparts.size(); parti != parte;
1457 ++parti) {
1458 EVT partVT = vtparts[parti];
1459 Value *srcValue = Constant::getNullValue(
1460 PointerType::get(partVT.getTypeForEVT(F->getContext()),
1461 llvm::ADDRESS_SPACE_PARAM));
1462 SDValue srcAddr =
1463 DAG.getNode(ISD::ADD, dl, getPointerTy(), Arg,
1464 DAG.getConstant(offsets[parti], getPointerTy()));
1465 unsigned partAlign =
1466 aggregateIsPacked ? 1
1467 : TD->getABITypeAlignment(
1468 partVT.getTypeForEVT(F->getContext()));
Justin Holewinskia2911282013-07-01 12:58:58 +00001469 SDValue p;
1470 if (Ins[InsIdx].VT.getSizeInBits() > partVT.getSizeInBits()) {
1471 ISD::LoadExtType ExtOp = Ins[InsIdx].Flags.isSExt() ?
1472 ISD::SEXTLOAD : ISD::ZEXTLOAD;
1473 p = DAG.getExtLoad(ExtOp, dl, Ins[InsIdx].VT, Root, srcAddr,
Justin Holewinskif8f70912013-06-28 17:57:59 +00001474 MachinePointerInfo(srcValue), partVT, false,
1475 false, partAlign);
Justin Holewinskia2911282013-07-01 12:58:58 +00001476 } else {
Justin Holewinskif8f70912013-06-28 17:57:59 +00001477 p = DAG.getLoad(partVT, dl, Root, srcAddr,
1478 MachinePointerInfo(srcValue), false, false, false,
1479 partAlign);
Justin Holewinskia2911282013-07-01 12:58:58 +00001480 }
Justin Holewinski44f5c602013-06-28 17:57:53 +00001481 if (p.getNode())
1482 p.getNode()->setIROrder(idx + 1);
1483 InVals.push_back(p);
1484 ++InsIdx;
Justin Holewinskie9884092013-03-24 21:17:47 +00001485 }
Justin Holewinski44f5c602013-06-28 17:57:53 +00001486 if (vtparts.size() > 0)
1487 --InsIdx;
Justin Holewinskie9884092013-03-24 21:17:47 +00001488 continue;
1489 }
Justin Holewinski44f5c602013-06-28 17:57:53 +00001490 if (Ty->isVectorTy()) {
1491 EVT ObjectVT = getValueType(Ty);
Justin Holewinskiaaaf2892013-06-25 12:22:21 +00001492 SDValue Arg = getParamSymbol(DAG, idx, getPointerTy());
Justin Holewinski44f5c602013-06-28 17:57:53 +00001493 unsigned NumElts = ObjectVT.getVectorNumElements();
1494 assert(TLI->getNumRegisters(F->getContext(), ObjectVT) == NumElts &&
1495 "Vector was not scalarized");
1496 unsigned Ofst = 0;
1497 EVT EltVT = ObjectVT.getVectorElementType();
1498
1499 // V1 load
1500 // f32 = load ...
1501 if (NumElts == 1) {
1502 // We only have one element, so just directly load it
1503 Value *SrcValue = Constant::getNullValue(PointerType::get(
1504 EltVT.getTypeForEVT(F->getContext()), llvm::ADDRESS_SPACE_PARAM));
1505 SDValue SrcAddr = DAG.getNode(ISD::ADD, dl, getPointerTy(), Arg,
1506 DAG.getConstant(Ofst, getPointerTy()));
1507 SDValue P = DAG.getLoad(
1508 EltVT, dl, Root, SrcAddr, MachinePointerInfo(SrcValue), false,
1509 false, true,
1510 TD->getABITypeAlignment(EltVT.getTypeForEVT(F->getContext())));
1511 if (P.getNode())
1512 P.getNode()->setIROrder(idx + 1);
1513
Justin Holewinskif8f70912013-06-28 17:57:59 +00001514 if (Ins[InsIdx].VT.getSizeInBits() > EltVT.getSizeInBits())
Justin Holewinskia2911282013-07-01 12:58:58 +00001515 P = DAG.getNode(ISD::ANY_EXTEND, dl, Ins[InsIdx].VT, P);
Justin Holewinski44f5c602013-06-28 17:57:53 +00001516 InVals.push_back(P);
1517 Ofst += TD->getTypeAllocSize(EltVT.getTypeForEVT(F->getContext()));
1518 ++InsIdx;
1519 } else if (NumElts == 2) {
1520 // V2 load
1521 // f32,f32 = load ...
1522 EVT VecVT = EVT::getVectorVT(F->getContext(), EltVT, 2);
1523 Value *SrcValue = Constant::getNullValue(PointerType::get(
1524 VecVT.getTypeForEVT(F->getContext()), llvm::ADDRESS_SPACE_PARAM));
1525 SDValue SrcAddr = DAG.getNode(ISD::ADD, dl, getPointerTy(), Arg,
1526 DAG.getConstant(Ofst, getPointerTy()));
1527 SDValue P = DAG.getLoad(
1528 VecVT, dl, Root, SrcAddr, MachinePointerInfo(SrcValue), false,
1529 false, true,
1530 TD->getABITypeAlignment(VecVT.getTypeForEVT(F->getContext())));
1531 if (P.getNode())
1532 P.getNode()->setIROrder(idx + 1);
1533
1534 SDValue Elt0 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, P,
1535 DAG.getIntPtrConstant(0));
1536 SDValue Elt1 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, P,
1537 DAG.getIntPtrConstant(1));
Justin Holewinskif8f70912013-06-28 17:57:59 +00001538
1539 if (Ins[InsIdx].VT.getSizeInBits() > EltVT.getSizeInBits()) {
Justin Holewinskia2911282013-07-01 12:58:58 +00001540 Elt0 = DAG.getNode(ISD::ANY_EXTEND, dl, Ins[InsIdx].VT, Elt0);
1541 Elt1 = DAG.getNode(ISD::ANY_EXTEND, dl, Ins[InsIdx].VT, Elt1);
Justin Holewinskif8f70912013-06-28 17:57:59 +00001542 }
1543
Justin Holewinski44f5c602013-06-28 17:57:53 +00001544 InVals.push_back(Elt0);
1545 InVals.push_back(Elt1);
1546 Ofst += TD->getTypeAllocSize(VecVT.getTypeForEVT(F->getContext()));
1547 InsIdx += 2;
1548 } else {
1549 // V4 loads
1550 // We have at least 4 elements (<3 x Ty> expands to 4 elements) and
1551 // the
1552 // vector will be expanded to a power of 2 elements, so we know we can
1553 // always round up to the next multiple of 4 when creating the vector
1554 // loads.
1555 // e.g. 4 elem => 1 ld.v4
1556 // 6 elem => 2 ld.v4
1557 // 8 elem => 2 ld.v4
1558 // 11 elem => 3 ld.v4
1559 unsigned VecSize = 4;
1560 if (EltVT.getSizeInBits() == 64) {
1561 VecSize = 2;
1562 }
1563 EVT VecVT = EVT::getVectorVT(F->getContext(), EltVT, VecSize);
1564 for (unsigned i = 0; i < NumElts; i += VecSize) {
1565 Value *SrcValue = Constant::getNullValue(
1566 PointerType::get(VecVT.getTypeForEVT(F->getContext()),
1567 llvm::ADDRESS_SPACE_PARAM));
1568 SDValue SrcAddr =
1569 DAG.getNode(ISD::ADD, dl, getPointerTy(), Arg,
1570 DAG.getConstant(Ofst, getPointerTy()));
1571 SDValue P = DAG.getLoad(
1572 VecVT, dl, Root, SrcAddr, MachinePointerInfo(SrcValue), false,
1573 false, true,
1574 TD->getABITypeAlignment(VecVT.getTypeForEVT(F->getContext())));
1575 if (P.getNode())
1576 P.getNode()->setIROrder(idx + 1);
1577
1578 for (unsigned j = 0; j < VecSize; ++j) {
1579 if (i + j >= NumElts)
1580 break;
1581 SDValue Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, P,
1582 DAG.getIntPtrConstant(j));
Justin Holewinskif8f70912013-06-28 17:57:59 +00001583 if (Ins[InsIdx].VT.getSizeInBits() > EltVT.getSizeInBits())
Justin Holewinskia2911282013-07-01 12:58:58 +00001584 Elt = DAG.getNode(ISD::ANY_EXTEND, dl, Ins[InsIdx].VT, Elt);
Justin Holewinski44f5c602013-06-28 17:57:53 +00001585 InVals.push_back(Elt);
1586 }
1587 Ofst += TD->getTypeAllocSize(VecVT.getTypeForEVT(F->getContext()));
Justin Holewinski44f5c602013-06-28 17:57:53 +00001588 }
Justin Holewinskidff28d22013-07-01 12:59:01 +00001589 InsIdx += VecSize;
Justin Holewinski44f5c602013-06-28 17:57:53 +00001590 }
1591
1592 if (NumElts > 0)
1593 --InsIdx;
1594 continue;
Justin Holewinskiae556d32012-05-04 20:18:50 +00001595 }
Justin Holewinski44f5c602013-06-28 17:57:53 +00001596 // A plain scalar.
1597 EVT ObjectVT = getValueType(Ty);
Justin Holewinski44f5c602013-06-28 17:57:53 +00001598 // If ABI, load from the param symbol
1599 SDValue Arg = getParamSymbol(DAG, idx, getPointerTy());
1600 Value *srcValue = Constant::getNullValue(PointerType::get(
1601 ObjectVT.getTypeForEVT(F->getContext()), llvm::ADDRESS_SPACE_PARAM));
Justin Holewinskif8f70912013-06-28 17:57:59 +00001602 SDValue p;
Justin Holewinskia2911282013-07-01 12:58:58 +00001603 if (ObjectVT.getSizeInBits() < Ins[InsIdx].VT.getSizeInBits()) {
1604 ISD::LoadExtType ExtOp = Ins[InsIdx].Flags.isSExt() ?
1605 ISD::SEXTLOAD : ISD::ZEXTLOAD;
1606 p = DAG.getExtLoad(ExtOp, dl, Ins[InsIdx].VT, Root, Arg,
Justin Holewinskif8f70912013-06-28 17:57:59 +00001607 MachinePointerInfo(srcValue), ObjectVT, false, false,
Justin Holewinskia2911282013-07-01 12:58:58 +00001608 TD->getABITypeAlignment(ObjectVT.getTypeForEVT(F->getContext())));
1609 } else {
Justin Holewinskif8f70912013-06-28 17:57:59 +00001610 p = DAG.getLoad(Ins[InsIdx].VT, dl, Root, Arg,
1611 MachinePointerInfo(srcValue), false, false, false,
Justin Holewinskia2911282013-07-01 12:58:58 +00001612 TD->getABITypeAlignment(ObjectVT.getTypeForEVT(F->getContext())));
1613 }
Justin Holewinski44f5c602013-06-28 17:57:53 +00001614 if (p.getNode())
1615 p.getNode()->setIROrder(idx + 1);
1616 InVals.push_back(p);
Justin Holewinskiae556d32012-05-04 20:18:50 +00001617 continue;
1618 }
1619
1620 // Param has ByVal attribute
Justin Holewinski44f5c602013-06-28 17:57:53 +00001621 // Return MoveParam(param symbol).
1622 // Ideally, the param symbol can be returned directly,
1623 // but when SDNode builder decides to use it in a CopyToReg(),
1624 // machine instruction fails because TargetExternalSymbol
1625 // (not lowered) is target dependent, and CopyToReg assumes
1626 // the source is lowered.
1627 EVT ObjectVT = getValueType(Ty);
1628 assert(ObjectVT == Ins[InsIdx].VT &&
1629 "Ins type did not match function type");
1630 SDValue Arg = getParamSymbol(DAG, idx, getPointerTy());
1631 SDValue p = DAG.getNode(NVPTXISD::MoveParam, dl, ObjectVT, Arg);
1632 if (p.getNode())
1633 p.getNode()->setIROrder(idx + 1);
1634 if (isKernel)
1635 InVals.push_back(p);
1636 else {
1637 SDValue p2 = DAG.getNode(
1638 ISD::INTRINSIC_WO_CHAIN, dl, ObjectVT,
1639 DAG.getConstant(Intrinsic::nvvm_ptr_local_to_gen, MVT::i32), p);
1640 InVals.push_back(p2);
Justin Holewinskiae556d32012-05-04 20:18:50 +00001641 }
1642 }
1643
1644 // Clang will check explicit VarArg and issue error if any. However, Clang
1645 // will let code with
Justin Holewinski44f5c602013-06-28 17:57:53 +00001646 // implicit var arg like f() pass. See bug 617733.
Justin Holewinskiae556d32012-05-04 20:18:50 +00001647 // We treat this case as if the arg list is empty.
Justin Holewinski44f5c602013-06-28 17:57:53 +00001648 // if (F.isVarArg()) {
Justin Holewinskiae556d32012-05-04 20:18:50 +00001649 // assert(0 && "VarArg not supported yet!");
1650 //}
1651
1652 if (!OutChains.empty())
Justin Holewinski0497ab12013-03-30 14:29:21 +00001653 DAG.setRoot(DAG.getNode(ISD::TokenFactor, dl, MVT::Other, &OutChains[0],
1654 OutChains.size()));
Justin Holewinskiae556d32012-05-04 20:18:50 +00001655
1656 return Chain;
1657}
1658
Justin Holewinski44f5c602013-06-28 17:57:53 +00001659
Justin Holewinski120baee2013-06-28 17:57:55 +00001660SDValue
1661NVPTXTargetLowering::LowerReturn(SDValue Chain, CallingConv::ID CallConv,
1662 bool isVarArg,
1663 const SmallVectorImpl<ISD::OutputArg> &Outs,
1664 const SmallVectorImpl<SDValue> &OutVals,
1665 SDLoc dl, SelectionDAG &DAG) const {
1666 MachineFunction &MF = DAG.getMachineFunction();
1667 const Function *F = MF.getFunction();
Justin Holewinskie04e4bd2013-06-28 17:58:10 +00001668 Type *RetTy = F->getReturnType();
Justin Holewinski120baee2013-06-28 17:57:55 +00001669 const DataLayout *TD = getDataLayout();
Justin Holewinskiae556d32012-05-04 20:18:50 +00001670
1671 bool isABI = (nvptxSubtarget.getSmVersion() >= 20);
Justin Holewinski120baee2013-06-28 17:57:55 +00001672 assert(isABI && "Non-ABI compilation is not supported");
1673 if (!isABI)
1674 return Chain;
Justin Holewinskiae556d32012-05-04 20:18:50 +00001675
Justin Holewinskie04e4bd2013-06-28 17:58:10 +00001676 if (VectorType *VTy = dyn_cast<VectorType>(RetTy)) {
Justin Holewinski120baee2013-06-28 17:57:55 +00001677 // If we have a vector type, the OutVals array will be the scalarized
1678 // components and we have combine them into 1 or more vector stores.
1679 unsigned NumElts = VTy->getNumElements();
1680 assert(NumElts == Outs.size() && "Bad scalarization of return value");
1681
Justin Holewinskif8f70912013-06-28 17:57:59 +00001682 // const_cast can be removed in later LLVM versions
Justin Holewinskie04e4bd2013-06-28 17:58:10 +00001683 EVT EltVT = getValueType(RetTy).getVectorElementType();
Justin Holewinskif8f70912013-06-28 17:57:59 +00001684 bool NeedExtend = false;
1685 if (EltVT.getSizeInBits() < 16)
1686 NeedExtend = true;
1687
Justin Holewinski120baee2013-06-28 17:57:55 +00001688 // V1 store
1689 if (NumElts == 1) {
1690 SDValue StoreVal = OutVals[0];
1691 // We only have one element, so just directly store it
Justin Holewinskif8f70912013-06-28 17:57:59 +00001692 if (NeedExtend)
1693 StoreVal = DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i16, StoreVal);
1694 SDValue Ops[] = { Chain, DAG.getConstant(0, MVT::i32), StoreVal };
1695 Chain = DAG.getMemIntrinsicNode(NVPTXISD::StoreRetval, dl,
1696 DAG.getVTList(MVT::Other), &Ops[0], 3,
1697 EltVT, MachinePointerInfo());
1698
Justin Holewinski120baee2013-06-28 17:57:55 +00001699 } else if (NumElts == 2) {
1700 // V2 store
1701 SDValue StoreVal0 = OutVals[0];
1702 SDValue StoreVal1 = OutVals[1];
1703
Justin Holewinskif8f70912013-06-28 17:57:59 +00001704 if (NeedExtend) {
1705 StoreVal0 = DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i16, StoreVal0);
1706 StoreVal1 = DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i16, StoreVal1);
Justin Holewinski120baee2013-06-28 17:57:55 +00001707 }
1708
Justin Holewinskif8f70912013-06-28 17:57:59 +00001709 SDValue Ops[] = { Chain, DAG.getConstant(0, MVT::i32), StoreVal0,
1710 StoreVal1 };
1711 Chain = DAG.getMemIntrinsicNode(NVPTXISD::StoreRetvalV2, dl,
1712 DAG.getVTList(MVT::Other), &Ops[0], 4,
1713 EltVT, MachinePointerInfo());
Justin Holewinski120baee2013-06-28 17:57:55 +00001714 } else {
1715 // V4 stores
1716 // We have at least 4 elements (<3 x Ty> expands to 4 elements) and the
1717 // vector will be expanded to a power of 2 elements, so we know we can
1718 // always round up to the next multiple of 4 when creating the vector
1719 // stores.
1720 // e.g. 4 elem => 1 st.v4
1721 // 6 elem => 2 st.v4
1722 // 8 elem => 2 st.v4
1723 // 11 elem => 3 st.v4
1724
1725 unsigned VecSize = 4;
1726 if (OutVals[0].getValueType().getSizeInBits() == 64)
1727 VecSize = 2;
1728
1729 unsigned Offset = 0;
1730
1731 EVT VecVT =
1732 EVT::getVectorVT(F->getContext(), OutVals[0].getValueType(), VecSize);
1733 unsigned PerStoreOffset =
1734 TD->getTypeAllocSize(VecVT.getTypeForEVT(F->getContext()));
1735
Justin Holewinski120baee2013-06-28 17:57:55 +00001736 for (unsigned i = 0; i < NumElts; i += VecSize) {
1737 // Get values
1738 SDValue StoreVal;
1739 SmallVector<SDValue, 8> Ops;
1740 Ops.push_back(Chain);
1741 Ops.push_back(DAG.getConstant(Offset, MVT::i32));
1742 unsigned Opc = NVPTXISD::StoreRetvalV2;
Justin Holewinskif8f70912013-06-28 17:57:59 +00001743 EVT ExtendedVT = (NeedExtend) ? MVT::i16 : OutVals[0].getValueType();
Justin Holewinski120baee2013-06-28 17:57:55 +00001744
1745 StoreVal = OutVals[i];
Justin Holewinskif8f70912013-06-28 17:57:59 +00001746 if (NeedExtend)
1747 StoreVal = DAG.getNode(ISD::ZERO_EXTEND, dl, ExtendedVT, StoreVal);
Justin Holewinski120baee2013-06-28 17:57:55 +00001748 Ops.push_back(StoreVal);
1749
1750 if (i + 1 < NumElts) {
1751 StoreVal = OutVals[i + 1];
Justin Holewinskif8f70912013-06-28 17:57:59 +00001752 if (NeedExtend)
1753 StoreVal = DAG.getNode(ISD::ZERO_EXTEND, dl, ExtendedVT, StoreVal);
Justin Holewinski120baee2013-06-28 17:57:55 +00001754 } else {
1755 StoreVal = DAG.getUNDEF(ExtendedVT);
1756 }
1757 Ops.push_back(StoreVal);
1758
1759 if (VecSize == 4) {
1760 Opc = NVPTXISD::StoreRetvalV4;
1761 if (i + 2 < NumElts) {
1762 StoreVal = OutVals[i + 2];
Justin Holewinskif8f70912013-06-28 17:57:59 +00001763 if (NeedExtend)
1764 StoreVal =
1765 DAG.getNode(ISD::ZERO_EXTEND, dl, ExtendedVT, StoreVal);
Justin Holewinski120baee2013-06-28 17:57:55 +00001766 } else {
1767 StoreVal = DAG.getUNDEF(ExtendedVT);
1768 }
1769 Ops.push_back(StoreVal);
1770
1771 if (i + 3 < NumElts) {
1772 StoreVal = OutVals[i + 3];
Justin Holewinskif8f70912013-06-28 17:57:59 +00001773 if (NeedExtend)
1774 StoreVal =
1775 DAG.getNode(ISD::ZERO_EXTEND, dl, ExtendedVT, StoreVal);
Justin Holewinski120baee2013-06-28 17:57:55 +00001776 } else {
1777 StoreVal = DAG.getUNDEF(ExtendedVT);
1778 }
1779 Ops.push_back(StoreVal);
1780 }
1781
Justin Holewinskif8f70912013-06-28 17:57:59 +00001782 // Chain = DAG.getNode(Opc, dl, MVT::Other, &Ops[0], Ops.size());
1783 Chain =
1784 DAG.getMemIntrinsicNode(Opc, dl, DAG.getVTList(MVT::Other), &Ops[0],
1785 Ops.size(), EltVT, MachinePointerInfo());
Justin Holewinski120baee2013-06-28 17:57:55 +00001786 Offset += PerStoreOffset;
1787 }
1788 }
1789 } else {
Justin Holewinskif8f70912013-06-28 17:57:59 +00001790 SmallVector<EVT, 16> ValVTs;
1791 // const_cast is necessary since we are still using an LLVM version from
1792 // before the type system re-write.
Justin Holewinskie04e4bd2013-06-28 17:58:10 +00001793 ComputePTXValueVTs(*this, RetTy, ValVTs);
Justin Holewinskif8f70912013-06-28 17:57:59 +00001794 assert(ValVTs.size() == OutVals.size() && "Bad return value decomposition");
1795
Justin Holewinskie04e4bd2013-06-28 17:58:10 +00001796 unsigned SizeSoFar = 0;
Justin Holewinski120baee2013-06-28 17:57:55 +00001797 for (unsigned i = 0, e = Outs.size(); i != e; ++i) {
1798 SDValue theVal = OutVals[i];
Justin Holewinskie04e4bd2013-06-28 17:58:10 +00001799 EVT TheValType = theVal.getValueType();
Justin Holewinski120baee2013-06-28 17:57:55 +00001800 unsigned numElems = 1;
Justin Holewinskie04e4bd2013-06-28 17:58:10 +00001801 if (TheValType.isVector())
1802 numElems = TheValType.getVectorNumElements();
Justin Holewinski120baee2013-06-28 17:57:55 +00001803 for (unsigned j = 0, je = numElems; j != je; ++j) {
Justin Holewinskie04e4bd2013-06-28 17:58:10 +00001804 SDValue TmpVal = theVal;
1805 if (TheValType.isVector())
1806 TmpVal = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl,
1807 TheValType.getVectorElementType(), TmpVal,
Justin Holewinski120baee2013-06-28 17:57:55 +00001808 DAG.getIntPtrConstant(j));
Justin Holewinskie04e4bd2013-06-28 17:58:10 +00001809 EVT TheStoreType = ValVTs[i];
1810 if (RetTy->isIntegerTy() &&
1811 TD->getTypeAllocSizeInBits(RetTy) < 32) {
1812 // The following zero-extension is for integer types only, and
1813 // specifically not for aggregates.
1814 TmpVal = DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i32, TmpVal);
1815 TheStoreType = MVT::i32;
1816 }
1817 else if (TmpVal.getValueType().getSizeInBits() < 16)
1818 TmpVal = DAG.getNode(ISD::ANY_EXTEND, dl, MVT::i16, TmpVal);
1819
1820 SDValue Ops[] = { Chain, DAG.getConstant(SizeSoFar, MVT::i32), TmpVal };
Justin Holewinskif8f70912013-06-28 17:57:59 +00001821 Chain = DAG.getMemIntrinsicNode(NVPTXISD::StoreRetval, dl,
Justin Holewinskie04e4bd2013-06-28 17:58:10 +00001822 DAG.getVTList(MVT::Other), &Ops[0],
1823 3, TheStoreType,
1824 MachinePointerInfo());
1825 if(TheValType.isVector())
1826 SizeSoFar +=
1827 TheStoreType.getVectorElementType().getStoreSizeInBits() / 8;
Justin Holewinski120baee2013-06-28 17:57:55 +00001828 else
Justin Holewinskie04e4bd2013-06-28 17:58:10 +00001829 SizeSoFar += TheStoreType.getStoreSizeInBits()/8;
Justin Holewinski120baee2013-06-28 17:57:55 +00001830 }
Justin Holewinskiae556d32012-05-04 20:18:50 +00001831 }
1832 }
1833
1834 return DAG.getNode(NVPTXISD::RET_FLAG, dl, MVT::Other, Chain);
1835}
1836
Justin Holewinskif8f70912013-06-28 17:57:59 +00001837
Justin Holewinski0497ab12013-03-30 14:29:21 +00001838void NVPTXTargetLowering::LowerAsmOperandForConstraint(
1839 SDValue Op, std::string &Constraint, std::vector<SDValue> &Ops,
1840 SelectionDAG &DAG) const {
Justin Holewinskiae556d32012-05-04 20:18:50 +00001841 if (Constraint.length() > 1)
1842 return;
1843 else
1844 TargetLowering::LowerAsmOperandForConstraint(Op, Constraint, Ops, DAG);
1845}
1846
1847// NVPTX suuport vector of legal types of any length in Intrinsics because the
1848// NVPTX specific type legalizer
1849// will legalize them to the PTX supported length.
Justin Holewinski0497ab12013-03-30 14:29:21 +00001850bool NVPTXTargetLowering::isTypeSupportedInIntrinsic(MVT VT) const {
Justin Holewinskiae556d32012-05-04 20:18:50 +00001851 if (isTypeLegal(VT))
1852 return true;
1853 if (VT.isVector()) {
1854 MVT eVT = VT.getVectorElementType();
1855 if (isTypeLegal(eVT))
1856 return true;
1857 }
1858 return false;
1859}
1860
Justin Holewinskiae556d32012-05-04 20:18:50 +00001861// llvm.ptx.memcpy.const and llvm.ptx.memmove.const need to be modeled as
1862// TgtMemIntrinsic
1863// because we need the information that is only available in the "Value" type
1864// of destination
1865// pointer. In particular, the address space information.
Justin Holewinski0497ab12013-03-30 14:29:21 +00001866bool NVPTXTargetLowering::getTgtMemIntrinsic(
1867 IntrinsicInfo &Info, const CallInst &I, unsigned Intrinsic) const {
Justin Holewinskiae556d32012-05-04 20:18:50 +00001868 switch (Intrinsic) {
1869 default:
1870 return false;
1871
1872 case Intrinsic::nvvm_atomic_load_add_f32:
1873 Info.opc = ISD::INTRINSIC_W_CHAIN;
1874 Info.memVT = MVT::f32;
1875 Info.ptrVal = I.getArgOperand(0);
1876 Info.offset = 0;
1877 Info.vol = 0;
1878 Info.readMem = true;
1879 Info.writeMem = true;
1880 Info.align = 0;
1881 return true;
1882
1883 case Intrinsic::nvvm_atomic_load_inc_32:
1884 case Intrinsic::nvvm_atomic_load_dec_32:
1885 Info.opc = ISD::INTRINSIC_W_CHAIN;
1886 Info.memVT = MVT::i32;
1887 Info.ptrVal = I.getArgOperand(0);
1888 Info.offset = 0;
1889 Info.vol = 0;
1890 Info.readMem = true;
1891 Info.writeMem = true;
1892 Info.align = 0;
1893 return true;
1894
1895 case Intrinsic::nvvm_ldu_global_i:
1896 case Intrinsic::nvvm_ldu_global_f:
1897 case Intrinsic::nvvm_ldu_global_p:
1898
1899 Info.opc = ISD::INTRINSIC_W_CHAIN;
1900 if (Intrinsic == Intrinsic::nvvm_ldu_global_i)
Justin Holewinskif8f70912013-06-28 17:57:59 +00001901 Info.memVT = getValueType(I.getType());
Justin Holewinskiae556d32012-05-04 20:18:50 +00001902 else if (Intrinsic == Intrinsic::nvvm_ldu_global_p)
Justin Holewinskif8f70912013-06-28 17:57:59 +00001903 Info.memVT = getValueType(I.getType());
Justin Holewinskiae556d32012-05-04 20:18:50 +00001904 else
1905 Info.memVT = MVT::f32;
1906 Info.ptrVal = I.getArgOperand(0);
1907 Info.offset = 0;
1908 Info.vol = 0;
1909 Info.readMem = true;
1910 Info.writeMem = false;
1911 Info.align = 0;
1912 return true;
1913
1914 }
1915 return false;
1916}
1917
1918/// isLegalAddressingMode - Return true if the addressing mode represented
1919/// by AM is legal for this target, for a load/store of the specified type.
1920/// Used to guide target specific optimizations, like loop strength reduction
1921/// (LoopStrengthReduce.cpp) and memory optimization for address mode
1922/// (CodeGenPrepare.cpp)
Justin Holewinski0497ab12013-03-30 14:29:21 +00001923bool NVPTXTargetLowering::isLegalAddressingMode(const AddrMode &AM,
1924 Type *Ty) const {
Justin Holewinskiae556d32012-05-04 20:18:50 +00001925
1926 // AddrMode - This represents an addressing mode of:
1927 // BaseGV + BaseOffs + BaseReg + Scale*ScaleReg
1928 //
1929 // The legal address modes are
1930 // - [avar]
1931 // - [areg]
1932 // - [areg+immoff]
1933 // - [immAddr]
1934
1935 if (AM.BaseGV) {
1936 if (AM.BaseOffs || AM.HasBaseReg || AM.Scale)
1937 return false;
1938 return true;
1939 }
1940
1941 switch (AM.Scale) {
Justin Holewinski0497ab12013-03-30 14:29:21 +00001942 case 0: // "r", "r+i" or "i" is allowed
Justin Holewinskiae556d32012-05-04 20:18:50 +00001943 break;
1944 case 1:
Justin Holewinski0497ab12013-03-30 14:29:21 +00001945 if (AM.HasBaseReg) // "r+r+i" or "r+r" is not allowed.
Justin Holewinskiae556d32012-05-04 20:18:50 +00001946 return false;
1947 // Otherwise we have r+i.
1948 break;
1949 default:
1950 // No scale > 1 is allowed
1951 return false;
1952 }
1953 return true;
1954}
1955
1956//===----------------------------------------------------------------------===//
1957// NVPTX Inline Assembly Support
1958//===----------------------------------------------------------------------===//
1959
1960/// getConstraintType - Given a constraint letter, return the type of
1961/// constraint it is for this target.
1962NVPTXTargetLowering::ConstraintType
1963NVPTXTargetLowering::getConstraintType(const std::string &Constraint) const {
1964 if (Constraint.size() == 1) {
1965 switch (Constraint[0]) {
1966 default:
1967 break;
1968 case 'r':
1969 case 'h':
1970 case 'c':
1971 case 'l':
1972 case 'f':
1973 case 'd':
1974 case '0':
1975 case 'N':
1976 return C_RegisterClass;
1977 }
1978 }
1979 return TargetLowering::getConstraintType(Constraint);
1980}
1981
Justin Holewinski0497ab12013-03-30 14:29:21 +00001982std::pair<unsigned, const TargetRegisterClass *>
Justin Holewinskiae556d32012-05-04 20:18:50 +00001983NVPTXTargetLowering::getRegForInlineAsmConstraint(const std::string &Constraint,
Chad Rosier295bd432013-06-22 18:37:38 +00001984 MVT VT) const {
Justin Holewinskiae556d32012-05-04 20:18:50 +00001985 if (Constraint.size() == 1) {
1986 switch (Constraint[0]) {
1987 case 'c':
Justin Holewinskif8f70912013-06-28 17:57:59 +00001988 return std::make_pair(0U, &NVPTX::Int16RegsRegClass);
Justin Holewinskiae556d32012-05-04 20:18:50 +00001989 case 'h':
1990 return std::make_pair(0U, &NVPTX::Int16RegsRegClass);
1991 case 'r':
1992 return std::make_pair(0U, &NVPTX::Int32RegsRegClass);
1993 case 'l':
1994 case 'N':
1995 return std::make_pair(0U, &NVPTX::Int64RegsRegClass);
1996 case 'f':
1997 return std::make_pair(0U, &NVPTX::Float32RegsRegClass);
1998 case 'd':
1999 return std::make_pair(0U, &NVPTX::Float64RegsRegClass);
2000 }
2001 }
2002 return TargetLowering::getRegForInlineAsmConstraint(Constraint, VT);
2003}
2004
Justin Holewinskiae556d32012-05-04 20:18:50 +00002005/// getFunctionAlignment - Return the Log2 alignment of this function.
2006unsigned NVPTXTargetLowering::getFunctionAlignment(const Function *) const {
2007 return 4;
2008}
Justin Holewinskibe8dc642013-02-12 14:18:49 +00002009
2010/// ReplaceVectorLoad - Convert vector loads into multi-output scalar loads.
2011static void ReplaceLoadVector(SDNode *N, SelectionDAG &DAG,
Justin Holewinski0497ab12013-03-30 14:29:21 +00002012 SmallVectorImpl<SDValue> &Results) {
Justin Holewinskibe8dc642013-02-12 14:18:49 +00002013 EVT ResVT = N->getValueType(0);
Andrew Trickef9de2a2013-05-25 02:42:55 +00002014 SDLoc DL(N);
Justin Holewinskibe8dc642013-02-12 14:18:49 +00002015
2016 assert(ResVT.isVector() && "Vector load must have vector type");
2017
2018 // We only handle "native" vector sizes for now, e.g. <4 x double> is not
2019 // legal. We can (and should) split that into 2 loads of <2 x double> here
2020 // but I'm leaving that as a TODO for now.
2021 assert(ResVT.isSimple() && "Can only handle simple types");
2022 switch (ResVT.getSimpleVT().SimpleTy) {
Justin Holewinski0497ab12013-03-30 14:29:21 +00002023 default:
2024 return;
Justin Holewinskibe8dc642013-02-12 14:18:49 +00002025 case MVT::v2i8:
2026 case MVT::v2i16:
2027 case MVT::v2i32:
2028 case MVT::v2i64:
2029 case MVT::v2f32:
2030 case MVT::v2f64:
2031 case MVT::v4i8:
2032 case MVT::v4i16:
2033 case MVT::v4i32:
2034 case MVT::v4f32:
2035 // This is a "native" vector type
2036 break;
2037 }
2038
2039 EVT EltVT = ResVT.getVectorElementType();
2040 unsigned NumElts = ResVT.getVectorNumElements();
2041
2042 // Since LoadV2 is a target node, we cannot rely on DAG type legalization.
2043 // Therefore, we must ensure the type is legal. For i1 and i8, we set the
2044 // loaded type to i16 and propogate the "real" type as the memory type.
2045 bool NeedTrunc = false;
2046 if (EltVT.getSizeInBits() < 16) {
2047 EltVT = MVT::i16;
2048 NeedTrunc = true;
2049 }
2050
2051 unsigned Opcode = 0;
2052 SDVTList LdResVTs;
2053
2054 switch (NumElts) {
Justin Holewinski0497ab12013-03-30 14:29:21 +00002055 default:
2056 return;
Justin Holewinskibe8dc642013-02-12 14:18:49 +00002057 case 2:
2058 Opcode = NVPTXISD::LoadV2;
2059 LdResVTs = DAG.getVTList(EltVT, EltVT, MVT::Other);
2060 break;
2061 case 4: {
2062 Opcode = NVPTXISD::LoadV4;
2063 EVT ListVTs[] = { EltVT, EltVT, EltVT, EltVT, MVT::Other };
2064 LdResVTs = DAG.getVTList(ListVTs, 5);
2065 break;
2066 }
2067 }
2068
2069 SmallVector<SDValue, 8> OtherOps;
2070
2071 // Copy regular operands
2072 for (unsigned i = 0, e = N->getNumOperands(); i != e; ++i)
2073 OtherOps.push_back(N->getOperand(i));
2074
2075 LoadSDNode *LD = cast<LoadSDNode>(N);
2076
2077 // The select routine does not have access to the LoadSDNode instance, so
2078 // pass along the extension information
2079 OtherOps.push_back(DAG.getIntPtrConstant(LD->getExtensionType()));
2080
2081 SDValue NewLD = DAG.getMemIntrinsicNode(Opcode, DL, LdResVTs, &OtherOps[0],
2082 OtherOps.size(), LD->getMemoryVT(),
2083 LD->getMemOperand());
2084
2085 SmallVector<SDValue, 4> ScalarRes;
2086
2087 for (unsigned i = 0; i < NumElts; ++i) {
2088 SDValue Res = NewLD.getValue(i);
2089 if (NeedTrunc)
2090 Res = DAG.getNode(ISD::TRUNCATE, DL, ResVT.getVectorElementType(), Res);
2091 ScalarRes.push_back(Res);
2092 }
2093
2094 SDValue LoadChain = NewLD.getValue(NumElts);
2095
Justin Holewinski0497ab12013-03-30 14:29:21 +00002096 SDValue BuildVec =
2097 DAG.getNode(ISD::BUILD_VECTOR, DL, ResVT, &ScalarRes[0], NumElts);
Justin Holewinskibe8dc642013-02-12 14:18:49 +00002098
2099 Results.push_back(BuildVec);
2100 Results.push_back(LoadChain);
2101}
2102
Justin Holewinski0497ab12013-03-30 14:29:21 +00002103static void ReplaceINTRINSIC_W_CHAIN(SDNode *N, SelectionDAG &DAG,
Justin Holewinskibe8dc642013-02-12 14:18:49 +00002104 SmallVectorImpl<SDValue> &Results) {
2105 SDValue Chain = N->getOperand(0);
2106 SDValue Intrin = N->getOperand(1);
Andrew Trickef9de2a2013-05-25 02:42:55 +00002107 SDLoc DL(N);
Justin Holewinskibe8dc642013-02-12 14:18:49 +00002108
2109 // Get the intrinsic ID
2110 unsigned IntrinNo = cast<ConstantSDNode>(Intrin.getNode())->getZExtValue();
Justin Holewinski0497ab12013-03-30 14:29:21 +00002111 switch (IntrinNo) {
2112 default:
2113 return;
Justin Holewinskibe8dc642013-02-12 14:18:49 +00002114 case Intrinsic::nvvm_ldg_global_i:
2115 case Intrinsic::nvvm_ldg_global_f:
2116 case Intrinsic::nvvm_ldg_global_p:
2117 case Intrinsic::nvvm_ldu_global_i:
2118 case Intrinsic::nvvm_ldu_global_f:
2119 case Intrinsic::nvvm_ldu_global_p: {
2120 EVT ResVT = N->getValueType(0);
2121
2122 if (ResVT.isVector()) {
2123 // Vector LDG/LDU
2124
2125 unsigned NumElts = ResVT.getVectorNumElements();
2126 EVT EltVT = ResVT.getVectorElementType();
2127
Justin Holewinskif8f70912013-06-28 17:57:59 +00002128 // Since LDU/LDG are target nodes, we cannot rely on DAG type
2129 // legalization.
Justin Holewinskibe8dc642013-02-12 14:18:49 +00002130 // Therefore, we must ensure the type is legal. For i1 and i8, we set the
2131 // loaded type to i16 and propogate the "real" type as the memory type.
2132 bool NeedTrunc = false;
2133 if (EltVT.getSizeInBits() < 16) {
2134 EltVT = MVT::i16;
2135 NeedTrunc = true;
2136 }
2137
2138 unsigned Opcode = 0;
2139 SDVTList LdResVTs;
2140
2141 switch (NumElts) {
Justin Holewinski0497ab12013-03-30 14:29:21 +00002142 default:
2143 return;
Justin Holewinskibe8dc642013-02-12 14:18:49 +00002144 case 2:
Justin Holewinski0497ab12013-03-30 14:29:21 +00002145 switch (IntrinNo) {
2146 default:
2147 return;
Justin Holewinskibe8dc642013-02-12 14:18:49 +00002148 case Intrinsic::nvvm_ldg_global_i:
2149 case Intrinsic::nvvm_ldg_global_f:
2150 case Intrinsic::nvvm_ldg_global_p:
2151 Opcode = NVPTXISD::LDGV2;
2152 break;
2153 case Intrinsic::nvvm_ldu_global_i:
2154 case Intrinsic::nvvm_ldu_global_f:
2155 case Intrinsic::nvvm_ldu_global_p:
2156 Opcode = NVPTXISD::LDUV2;
2157 break;
2158 }
2159 LdResVTs = DAG.getVTList(EltVT, EltVT, MVT::Other);
2160 break;
2161 case 4: {
Justin Holewinski0497ab12013-03-30 14:29:21 +00002162 switch (IntrinNo) {
2163 default:
2164 return;
Justin Holewinskibe8dc642013-02-12 14:18:49 +00002165 case Intrinsic::nvvm_ldg_global_i:
2166 case Intrinsic::nvvm_ldg_global_f:
2167 case Intrinsic::nvvm_ldg_global_p:
2168 Opcode = NVPTXISD::LDGV4;
2169 break;
2170 case Intrinsic::nvvm_ldu_global_i:
2171 case Intrinsic::nvvm_ldu_global_f:
2172 case Intrinsic::nvvm_ldu_global_p:
2173 Opcode = NVPTXISD::LDUV4;
2174 break;
2175 }
2176 EVT ListVTs[] = { EltVT, EltVT, EltVT, EltVT, MVT::Other };
2177 LdResVTs = DAG.getVTList(ListVTs, 5);
2178 break;
2179 }
2180 }
2181
2182 SmallVector<SDValue, 8> OtherOps;
2183
2184 // Copy regular operands
2185
2186 OtherOps.push_back(Chain); // Chain
Justin Holewinski0497ab12013-03-30 14:29:21 +00002187 // Skip operand 1 (intrinsic ID)
Justin Holewinskif8f70912013-06-28 17:57:59 +00002188 // Others
Justin Holewinskibe8dc642013-02-12 14:18:49 +00002189 for (unsigned i = 2, e = N->getNumOperands(); i != e; ++i)
2190 OtherOps.push_back(N->getOperand(i));
2191
2192 MemIntrinsicSDNode *MemSD = cast<MemIntrinsicSDNode>(N);
2193
Justin Holewinski0497ab12013-03-30 14:29:21 +00002194 SDValue NewLD = DAG.getMemIntrinsicNode(
2195 Opcode, DL, LdResVTs, &OtherOps[0], OtherOps.size(),
2196 MemSD->getMemoryVT(), MemSD->getMemOperand());
Justin Holewinskibe8dc642013-02-12 14:18:49 +00002197
2198 SmallVector<SDValue, 4> ScalarRes;
2199
2200 for (unsigned i = 0; i < NumElts; ++i) {
2201 SDValue Res = NewLD.getValue(i);
2202 if (NeedTrunc)
Justin Holewinski0497ab12013-03-30 14:29:21 +00002203 Res =
2204 DAG.getNode(ISD::TRUNCATE, DL, ResVT.getVectorElementType(), Res);
Justin Holewinskibe8dc642013-02-12 14:18:49 +00002205 ScalarRes.push_back(Res);
2206 }
2207
2208 SDValue LoadChain = NewLD.getValue(NumElts);
2209
Justin Holewinski0497ab12013-03-30 14:29:21 +00002210 SDValue BuildVec =
2211 DAG.getNode(ISD::BUILD_VECTOR, DL, ResVT, &ScalarRes[0], NumElts);
Justin Holewinskibe8dc642013-02-12 14:18:49 +00002212
2213 Results.push_back(BuildVec);
2214 Results.push_back(LoadChain);
2215 } else {
2216 // i8 LDG/LDU
2217 assert(ResVT.isSimple() && ResVT.getSimpleVT().SimpleTy == MVT::i8 &&
2218 "Custom handling of non-i8 ldu/ldg?");
2219
2220 // Just copy all operands as-is
2221 SmallVector<SDValue, 4> Ops;
2222 for (unsigned i = 0, e = N->getNumOperands(); i != e; ++i)
2223 Ops.push_back(N->getOperand(i));
2224
2225 // Force output to i16
2226 SDVTList LdResVTs = DAG.getVTList(MVT::i16, MVT::Other);
2227
2228 MemIntrinsicSDNode *MemSD = cast<MemIntrinsicSDNode>(N);
2229
2230 // We make sure the memory type is i8, which will be used during isel
2231 // to select the proper instruction.
Justin Holewinski0497ab12013-03-30 14:29:21 +00002232 SDValue NewLD =
2233 DAG.getMemIntrinsicNode(ISD::INTRINSIC_W_CHAIN, DL, LdResVTs, &Ops[0],
2234 Ops.size(), MVT::i8, MemSD->getMemOperand());
Justin Holewinskibe8dc642013-02-12 14:18:49 +00002235
Justin Holewinskie8c93e32013-07-01 12:58:48 +00002236 Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i8,
2237 NewLD.getValue(0)));
Justin Holewinskibe8dc642013-02-12 14:18:49 +00002238 Results.push_back(NewLD.getValue(1));
2239 }
2240 }
2241 }
2242}
2243
Justin Holewinski0497ab12013-03-30 14:29:21 +00002244void NVPTXTargetLowering::ReplaceNodeResults(
2245 SDNode *N, SmallVectorImpl<SDValue> &Results, SelectionDAG &DAG) const {
Justin Holewinskibe8dc642013-02-12 14:18:49 +00002246 switch (N->getOpcode()) {
Justin Holewinski0497ab12013-03-30 14:29:21 +00002247 default:
2248 report_fatal_error("Unhandled custom legalization");
Justin Holewinskibe8dc642013-02-12 14:18:49 +00002249 case ISD::LOAD:
2250 ReplaceLoadVector(N, DAG, Results);
2251 return;
2252 case ISD::INTRINSIC_W_CHAIN:
2253 ReplaceINTRINSIC_W_CHAIN(N, DAG, Results);
2254 return;
2255 }
2256}