Gitiles
Code Review Sign In
gerrit-public.fairphone.software / toolchain / llvm-project / 9f664c108396a70fd3ab984c1ddceeda305efbf9 / . / llvm / lib / Target / NVPTX / NVPTXISelLowering.cpp
blob: b0943befa88f9ff282e9d5d1bb105e607065471f [file] [log] [blame]
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]1//
2// The LLVM Compiler Infrastructure
3//
4// This file is distributed under the University of Illinois Open Source
5// License. See LICENSE.TXT for details.
6//
7//===----------------------------------------------------------------------===//
8//
9// This file defines the interfaces that NVPTX uses to lower LLVM code into a
10// selection DAG.
11//
12//===----------------------------------------------------------------------===//
13
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]14#include "NVPTXISelLowering.h"
Chandler Carruthed0881b2012-12-03 16:50:05 +0000[diff] [blame]15#include "NVPTX.h"
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]16#include "NVPTXTargetMachine.h"
17#include "NVPTXTargetObjectFile.h"
18#include "NVPTXUtilities.h"
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]19#include "llvm/CodeGen/Analysis.h"
20#include "llvm/CodeGen/MachineFrameInfo.h"
21#include "llvm/CodeGen/MachineFunction.h"
22#include "llvm/CodeGen/MachineInstrBuilder.h"
23#include "llvm/CodeGen/MachineRegisterInfo.h"
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]24#include "llvm/CodeGen/TargetLoweringObjectFileImpl.h"
Chandler Carruth219b89b2014-03-04 11:01:28 +0000[diff] [blame]25#include "llvm/IR/CallSite.h"
Chandler Carruth9fb823b2013-01-02 11:36:10 +0000[diff] [blame]26#include "llvm/IR/DerivedTypes.h"
27#include "llvm/IR/Function.h"
28#include "llvm/IR/GlobalValue.h"
29#include "llvm/IR/IntrinsicInst.h"
30#include "llvm/IR/Intrinsics.h"
31#include "llvm/IR/Module.h"
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]32#include "llvm/MC/MCSectionELF.h"
Chandler Carruthed0881b2012-12-03 16:50:05 +0000[diff] [blame]33#include "llvm/Support/CommandLine.h"
34#include "llvm/Support/Debug.h"
35#include "llvm/Support/ErrorHandling.h"
36#include "llvm/Support/raw_ostream.h"
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]37#include <sstream>
38
39#undef DEBUG_TYPE
40#define DEBUG_TYPE "nvptx-lower"
41
42using namespace llvm;
43
44static unsigned int uniqueCallSite = 0;
45
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]46static cl::opt<bool> sched4reg(
47 "nvptx-sched4reg",
48 cl::desc("NVPTX Specific: schedule for register pressue"), cl::init(false));
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]49
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]50static bool IsPTXVectorType(MVT VT) {
51 switch (VT.SimpleTy) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]52 default:
53 return false;
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]54 case MVT::v2i1:
55 case MVT::v4i1:
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]56 case MVT::v2i8:
57 case MVT::v4i8:
58 case MVT::v2i16:
59 case MVT::v4i16:
60 case MVT::v2i32:
61 case MVT::v4i32:
62 case MVT::v2i64:
63 case MVT::v2f32:
64 case MVT::v4f32:
65 case MVT::v2f64:
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]66 return true;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]67 }
68}
69
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]70/// ComputePTXValueVTs - For the given Type \p Ty, returns the set of primitive
71/// EVTs that compose it. Unlike ComputeValueVTs, this will break apart vectors
72/// into their primitive components.
73/// NOTE: This is a band-aid for code that expects ComputeValueVTs to return the
74/// same number of types as the Ins/Outs arrays in LowerFormalArguments,
75/// LowerCall, and LowerReturn.
76static void ComputePTXValueVTs(const TargetLowering &TLI, Type *Ty,
77 SmallVectorImpl<EVT> &ValueVTs,
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]78 SmallVectorImpl<uint64_t> *Offsets = nullptr,
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]79 uint64_t StartingOffset = 0) {
80 SmallVector<EVT, 16> TempVTs;
81 SmallVector<uint64_t, 16> TempOffsets;
82
83 ComputeValueVTs(TLI, Ty, TempVTs, &TempOffsets, StartingOffset);
84 for (unsigned i = 0, e = TempVTs.size(); i != e; ++i) {
85 EVT VT = TempVTs[i];
86 uint64_t Off = TempOffsets[i];
87 if (VT.isVector())
88 for (unsigned j = 0, je = VT.getVectorNumElements(); j != je; ++j) {
89 ValueVTs.push_back(VT.getVectorElementType());
90 if (Offsets)
91 Offsets->push_back(Off+j*VT.getVectorElementType().getStoreSize());
92 }
93 else {
94 ValueVTs.push_back(VT);
95 if (Offsets)
96 Offsets->push_back(Off);
97 }
98 }
99}
100
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]101// NVPTXTargetLowering Constructor.
102NVPTXTargetLowering::NVPTXTargetLowering(NVPTXTargetMachine &TM)
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]103 : TargetLowering(TM, new NVPTXTargetObjectFile()), nvTM(&TM),
104 nvptxSubtarget(TM.getSubtarget<NVPTXSubtarget>()) {
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]105
106 // always lower memset, memcpy, and memmove intrinsics to load/store
107 // instructions, rather
108 // then generating calls to memset, mempcy or memmove.
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]109 MaxStoresPerMemset = (unsigned) 0xFFFFFFFF;
110 MaxStoresPerMemcpy = (unsigned) 0xFFFFFFFF;
111 MaxStoresPerMemmove = (unsigned) 0xFFFFFFFF;
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]112
113 setBooleanContents(ZeroOrNegativeOneBooleanContent);
114
115 // Jump is Expensive. Don't create extra control flow for 'and', 'or'
116 // condition branches.
117 setJumpIsExpensive(true);
118
119 // By default, use the Source scheduling
120 if (sched4reg)
121 setSchedulingPreference(Sched::RegPressure);
122 else
123 setSchedulingPreference(Sched::Source);
124
125 addRegisterClass(MVT::i1, &NVPTX::Int1RegsRegClass);
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]126 addRegisterClass(MVT::i16, &NVPTX::Int16RegsRegClass);
127 addRegisterClass(MVT::i32, &NVPTX::Int32RegsRegClass);
128 addRegisterClass(MVT::i64, &NVPTX::Int64RegsRegClass);
129 addRegisterClass(MVT::f32, &NVPTX::Float32RegsRegClass);
130 addRegisterClass(MVT::f64, &NVPTX::Float64RegsRegClass);
131
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]132 // Operations not directly supported by NVPTX.
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]133 setOperationAction(ISD::SELECT_CC, MVT::Other, Expand);
134 setOperationAction(ISD::BR_CC, MVT::f32, Expand);
135 setOperationAction(ISD::BR_CC, MVT::f64, Expand);
136 setOperationAction(ISD::BR_CC, MVT::i1, Expand);
137 setOperationAction(ISD::BR_CC, MVT::i8, Expand);
138 setOperationAction(ISD::BR_CC, MVT::i16, Expand);
139 setOperationAction(ISD::BR_CC, MVT::i32, Expand);
140 setOperationAction(ISD::BR_CC, MVT::i64, Expand);
Justin Holewinski318c6252013-07-01 12:58:56 +0000[diff] [blame]141 // Some SIGN_EXTEND_INREG can be done using cvt instruction.
142 // For others we will expand to a SHL/SRA pair.
143 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i64, Legal);
144 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i32, Legal);
145 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i16, Legal);
146 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i8 , Legal);
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]147 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i1, Expand);
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]148
149 if (nvptxSubtarget.hasROT64()) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]150 setOperationAction(ISD::ROTL, MVT::i64, Legal);
151 setOperationAction(ISD::ROTR, MVT::i64, Legal);
152 } else {
153 setOperationAction(ISD::ROTL, MVT::i64, Expand);
154 setOperationAction(ISD::ROTR, MVT::i64, Expand);
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]155 }
156 if (nvptxSubtarget.hasROT32()) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]157 setOperationAction(ISD::ROTL, MVT::i32, Legal);
158 setOperationAction(ISD::ROTR, MVT::i32, Legal);
159 } else {
160 setOperationAction(ISD::ROTL, MVT::i32, Expand);
161 setOperationAction(ISD::ROTR, MVT::i32, Expand);
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]162 }
163
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]164 setOperationAction(ISD::ROTL, MVT::i16, Expand);
165 setOperationAction(ISD::ROTR, MVT::i16, Expand);
166 setOperationAction(ISD::ROTL, MVT::i8, Expand);
167 setOperationAction(ISD::ROTR, MVT::i8, Expand);
168 setOperationAction(ISD::BSWAP, MVT::i16, Expand);
169 setOperationAction(ISD::BSWAP, MVT::i32, Expand);
170 setOperationAction(ISD::BSWAP, MVT::i64, Expand);
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]171
172 // Indirect branch is not supported.
173 // This also disables Jump Table creation.
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]174 setOperationAction(ISD::BR_JT, MVT::Other, Expand);
175 setOperationAction(ISD::BRIND, MVT::Other, Expand);
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]176
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]177 setOperationAction(ISD::GlobalAddress, MVT::i32, Custom);
178 setOperationAction(ISD::GlobalAddress, MVT::i64, Custom);
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]179
180 // We want to legalize constant related memmove and memcopy
181 // intrinsics.
182 setOperationAction(ISD::INTRINSIC_W_CHAIN, MVT::Other, Custom);
183
184 // Turn FP extload into load/fextend
185 setLoadExtAction(ISD::EXTLOAD, MVT::f32, Expand);
186 // Turn FP truncstore into trunc + store.
187 setTruncStoreAction(MVT::f64, MVT::f32, Expand);
188
189 // PTX does not support load / store predicate registers
Justin Holewinskic6462aa2012-11-14 19:19:16 +0000[diff] [blame]190 setOperationAction(ISD::LOAD, MVT::i1, Custom);
191 setOperationAction(ISD::STORE, MVT::i1, Custom);
192
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]193 setLoadExtAction(ISD::SEXTLOAD, MVT::i1, Promote);
194 setLoadExtAction(ISD::ZEXTLOAD, MVT::i1, Promote);
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]195 setTruncStoreAction(MVT::i64, MVT::i1, Expand);
196 setTruncStoreAction(MVT::i32, MVT::i1, Expand);
197 setTruncStoreAction(MVT::i16, MVT::i1, Expand);
198 setTruncStoreAction(MVT::i8, MVT::i1, Expand);
199
200 // This is legal in NVPTX
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]201 setOperationAction(ISD::ConstantFP, MVT::f64, Legal);
202 setOperationAction(ISD::ConstantFP, MVT::f32, Legal);
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]203
204 // TRAP can be lowered to PTX trap
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]205 setOperationAction(ISD::TRAP, MVT::Other, Legal);
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]206
Justin Holewinski51cb1342013-07-01 12:59:04 +0000[diff] [blame]207 setOperationAction(ISD::ADDC, MVT::i64, Expand);
208 setOperationAction(ISD::ADDE, MVT::i64, Expand);
209
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]210 // Register custom handling for vector loads/stores
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]211 for (int i = MVT::FIRST_VECTOR_VALUETYPE; i <= MVT::LAST_VECTOR_VALUETYPE;
212 ++i) {
213 MVT VT = (MVT::SimpleValueType) i;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]214 if (IsPTXVectorType(VT)) {
215 setOperationAction(ISD::LOAD, VT, Custom);
216 setOperationAction(ISD::STORE, VT, Custom);
217 setOperationAction(ISD::INTRINSIC_W_CHAIN, VT, Custom);
218 }
219 }
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]220
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]221 // Custom handling for i8 intrinsics
222 setOperationAction(ISD::INTRINSIC_W_CHAIN, MVT::i8, Custom);
223
Justin Holewinskidc372df2013-06-28 17:58:07 +0000[diff] [blame]224 setOperationAction(ISD::CTLZ, MVT::i16, Legal);
225 setOperationAction(ISD::CTLZ, MVT::i32, Legal);
226 setOperationAction(ISD::CTLZ, MVT::i64, Legal);
227 setOperationAction(ISD::CTLZ_ZERO_UNDEF, MVT::i16, Legal);
228 setOperationAction(ISD::CTLZ_ZERO_UNDEF, MVT::i32, Legal);
229 setOperationAction(ISD::CTLZ_ZERO_UNDEF, MVT::i64, Legal);
230 setOperationAction(ISD::CTTZ, MVT::i16, Expand);
231 setOperationAction(ISD::CTTZ, MVT::i32, Expand);
232 setOperationAction(ISD::CTTZ, MVT::i64, Expand);
233 setOperationAction(ISD::CTTZ_ZERO_UNDEF, MVT::i16, Expand);
234 setOperationAction(ISD::CTTZ_ZERO_UNDEF, MVT::i32, Expand);
235 setOperationAction(ISD::CTTZ_ZERO_UNDEF, MVT::i64, Expand);
236 setOperationAction(ISD::CTPOP, MVT::i16, Legal);
237 setOperationAction(ISD::CTPOP, MVT::i32, Legal);
238 setOperationAction(ISD::CTPOP, MVT::i64, Legal);
239
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]240 // Now deduce the information based on the above mentioned
241 // actions
242 computeRegisterProperties();
243}
244
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]245const char *NVPTXTargetLowering::getTargetNodeName(unsigned Opcode) const {
246 switch (Opcode) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]247 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]248 return nullptr;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]249 case NVPTXISD::CALL:
250 return "NVPTXISD::CALL";
251 case NVPTXISD::RET_FLAG:
252 return "NVPTXISD::RET_FLAG";
253 case NVPTXISD::Wrapper:
254 return "NVPTXISD::Wrapper";
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]255 case NVPTXISD::DeclareParam:
256 return "NVPTXISD::DeclareParam";
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]257 case NVPTXISD::DeclareScalarParam:
258 return "NVPTXISD::DeclareScalarParam";
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]259 case NVPTXISD::DeclareRet:
260 return "NVPTXISD::DeclareRet";
261 case NVPTXISD::DeclareRetParam:
262 return "NVPTXISD::DeclareRetParam";
263 case NVPTXISD::PrintCall:
264 return "NVPTXISD::PrintCall";
265 case NVPTXISD::LoadParam:
266 return "NVPTXISD::LoadParam";
Justin Holewinskife44314f2013-06-28 17:57:51 +0000[diff] [blame]267 case NVPTXISD::LoadParamV2:
268 return "NVPTXISD::LoadParamV2";
269 case NVPTXISD::LoadParamV4:
270 return "NVPTXISD::LoadParamV4";
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]271 case NVPTXISD::StoreParam:
272 return "NVPTXISD::StoreParam";
Justin Holewinskife44314f2013-06-28 17:57:51 +0000[diff] [blame]273 case NVPTXISD::StoreParamV2:
274 return "NVPTXISD::StoreParamV2";
275 case NVPTXISD::StoreParamV4:
276 return "NVPTXISD::StoreParamV4";
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]277 case NVPTXISD::StoreParamS32:
278 return "NVPTXISD::StoreParamS32";
279 case NVPTXISD::StoreParamU32:
280 return "NVPTXISD::StoreParamU32";
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]281 case NVPTXISD::CallArgBegin:
282 return "NVPTXISD::CallArgBegin";
283 case NVPTXISD::CallArg:
284 return "NVPTXISD::CallArg";
285 case NVPTXISD::LastCallArg:
286 return "NVPTXISD::LastCallArg";
287 case NVPTXISD::CallArgEnd:
288 return "NVPTXISD::CallArgEnd";
289 case NVPTXISD::CallVoid:
290 return "NVPTXISD::CallVoid";
291 case NVPTXISD::CallVal:
292 return "NVPTXISD::CallVal";
293 case NVPTXISD::CallSymbol:
294 return "NVPTXISD::CallSymbol";
295 case NVPTXISD::Prototype:
296 return "NVPTXISD::Prototype";
297 case NVPTXISD::MoveParam:
298 return "NVPTXISD::MoveParam";
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]299 case NVPTXISD::StoreRetval:
300 return "NVPTXISD::StoreRetval";
Justin Holewinskife44314f2013-06-28 17:57:51 +0000[diff] [blame]301 case NVPTXISD::StoreRetvalV2:
302 return "NVPTXISD::StoreRetvalV2";
303 case NVPTXISD::StoreRetvalV4:
304 return "NVPTXISD::StoreRetvalV4";
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]305 case NVPTXISD::PseudoUseParam:
306 return "NVPTXISD::PseudoUseParam";
307 case NVPTXISD::RETURN:
308 return "NVPTXISD::RETURN";
309 case NVPTXISD::CallSeqBegin:
310 return "NVPTXISD::CallSeqBegin";
311 case NVPTXISD::CallSeqEnd:
312 return "NVPTXISD::CallSeqEnd";
Justin Holewinski3d49e5c2013-11-15 12:30:04 +0000[diff] [blame]313 case NVPTXISD::CallPrototype:
314 return "NVPTXISD::CallPrototype";
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]315 case NVPTXISD::LoadV2:
316 return "NVPTXISD::LoadV2";
317 case NVPTXISD::LoadV4:
318 return "NVPTXISD::LoadV4";
319 case NVPTXISD::LDGV2:
320 return "NVPTXISD::LDGV2";
321 case NVPTXISD::LDGV4:
322 return "NVPTXISD::LDGV4";
323 case NVPTXISD::LDUV2:
324 return "NVPTXISD::LDUV2";
325 case NVPTXISD::LDUV4:
326 return "NVPTXISD::LDUV4";
327 case NVPTXISD::StoreV2:
328 return "NVPTXISD::StoreV2";
329 case NVPTXISD::StoreV4:
330 return "NVPTXISD::StoreV4";
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]331 case NVPTXISD::Tex1DFloatI32: return "NVPTXISD::Tex1DFloatI32";
332 case NVPTXISD::Tex1DFloatFloat: return "NVPTXISD::Tex1DFloatFloat";
333 case NVPTXISD::Tex1DFloatFloatLevel:
334 return "NVPTXISD::Tex1DFloatFloatLevel";
335 case NVPTXISD::Tex1DFloatFloatGrad:
336 return "NVPTXISD::Tex1DFloatFloatGrad";
337 case NVPTXISD::Tex1DI32I32: return "NVPTXISD::Tex1DI32I32";
338 case NVPTXISD::Tex1DI32Float: return "NVPTXISD::Tex1DI32Float";
339 case NVPTXISD::Tex1DI32FloatLevel:
340 return "NVPTXISD::Tex1DI32FloatLevel";
341 case NVPTXISD::Tex1DI32FloatGrad:
342 return "NVPTXISD::Tex1DI32FloatGrad";
343 case NVPTXISD::Tex1DArrayFloatI32: return "NVPTXISD::Tex2DArrayFloatI32";
344 case NVPTXISD::Tex1DArrayFloatFloat: return "NVPTXISD::Tex2DArrayFloatFloat";
345 case NVPTXISD::Tex1DArrayFloatFloatLevel:
346 return "NVPTXISD::Tex2DArrayFloatFloatLevel";
347 case NVPTXISD::Tex1DArrayFloatFloatGrad:
348 return "NVPTXISD::Tex2DArrayFloatFloatGrad";
349 case NVPTXISD::Tex1DArrayI32I32: return "NVPTXISD::Tex2DArrayI32I32";
350 case NVPTXISD::Tex1DArrayI32Float: return "NVPTXISD::Tex2DArrayI32Float";
351 case NVPTXISD::Tex1DArrayI32FloatLevel:
352 return "NVPTXISD::Tex2DArrayI32FloatLevel";
353 case NVPTXISD::Tex1DArrayI32FloatGrad:
354 return "NVPTXISD::Tex2DArrayI32FloatGrad";
355 case NVPTXISD::Tex2DFloatI32: return "NVPTXISD::Tex2DFloatI32";
356 case NVPTXISD::Tex2DFloatFloat: return "NVPTXISD::Tex2DFloatFloat";
357 case NVPTXISD::Tex2DFloatFloatLevel:
358 return "NVPTXISD::Tex2DFloatFloatLevel";
359 case NVPTXISD::Tex2DFloatFloatGrad:
360 return "NVPTXISD::Tex2DFloatFloatGrad";
361 case NVPTXISD::Tex2DI32I32: return "NVPTXISD::Tex2DI32I32";
362 case NVPTXISD::Tex2DI32Float: return "NVPTXISD::Tex2DI32Float";
363 case NVPTXISD::Tex2DI32FloatLevel:
364 return "NVPTXISD::Tex2DI32FloatLevel";
365 case NVPTXISD::Tex2DI32FloatGrad:
366 return "NVPTXISD::Tex2DI32FloatGrad";
367 case NVPTXISD::Tex2DArrayFloatI32: return "NVPTXISD::Tex2DArrayFloatI32";
368 case NVPTXISD::Tex2DArrayFloatFloat: return "NVPTXISD::Tex2DArrayFloatFloat";
369 case NVPTXISD::Tex2DArrayFloatFloatLevel:
370 return "NVPTXISD::Tex2DArrayFloatFloatLevel";
371 case NVPTXISD::Tex2DArrayFloatFloatGrad:
372 return "NVPTXISD::Tex2DArrayFloatFloatGrad";
373 case NVPTXISD::Tex2DArrayI32I32: return "NVPTXISD::Tex2DArrayI32I32";
374 case NVPTXISD::Tex2DArrayI32Float: return "NVPTXISD::Tex2DArrayI32Float";
375 case NVPTXISD::Tex2DArrayI32FloatLevel:
376 return "NVPTXISD::Tex2DArrayI32FloatLevel";
377 case NVPTXISD::Tex2DArrayI32FloatGrad:
378 return "NVPTXISD::Tex2DArrayI32FloatGrad";
379 case NVPTXISD::Tex3DFloatI32: return "NVPTXISD::Tex3DFloatI32";
380 case NVPTXISD::Tex3DFloatFloat: return "NVPTXISD::Tex3DFloatFloat";
381 case NVPTXISD::Tex3DFloatFloatLevel:
382 return "NVPTXISD::Tex3DFloatFloatLevel";
383 case NVPTXISD::Tex3DFloatFloatGrad:
384 return "NVPTXISD::Tex3DFloatFloatGrad";
385 case NVPTXISD::Tex3DI32I32: return "NVPTXISD::Tex3DI32I32";
386 case NVPTXISD::Tex3DI32Float: return "NVPTXISD::Tex3DI32Float";
387 case NVPTXISD::Tex3DI32FloatLevel:
388 return "NVPTXISD::Tex3DI32FloatLevel";
389 case NVPTXISD::Tex3DI32FloatGrad:
390 return "NVPTXISD::Tex3DI32FloatGrad";
391
392 case NVPTXISD::Suld1DI8Trap: return "NVPTXISD::Suld1DI8Trap";
393 case NVPTXISD::Suld1DI16Trap: return "NVPTXISD::Suld1DI16Trap";
394 case NVPTXISD::Suld1DI32Trap: return "NVPTXISD::Suld1DI32Trap";
395 case NVPTXISD::Suld1DV2I8Trap: return "NVPTXISD::Suld1DV2I8Trap";
396 case NVPTXISD::Suld1DV2I16Trap: return "NVPTXISD::Suld1DV2I16Trap";
397 case NVPTXISD::Suld1DV2I32Trap: return "NVPTXISD::Suld1DV2I32Trap";
398 case NVPTXISD::Suld1DV4I8Trap: return "NVPTXISD::Suld1DV4I8Trap";
399 case NVPTXISD::Suld1DV4I16Trap: return "NVPTXISD::Suld1DV4I16Trap";
400 case NVPTXISD::Suld1DV4I32Trap: return "NVPTXISD::Suld1DV4I32Trap";
401
402 case NVPTXISD::Suld1DArrayI8Trap: return "NVPTXISD::Suld1DArrayI8Trap";
403 case NVPTXISD::Suld1DArrayI16Trap: return "NVPTXISD::Suld1DArrayI16Trap";
404 case NVPTXISD::Suld1DArrayI32Trap: return "NVPTXISD::Suld1DArrayI32Trap";
405 case NVPTXISD::Suld1DArrayV2I8Trap: return "NVPTXISD::Suld1DArrayV2I8Trap";
406 case NVPTXISD::Suld1DArrayV2I16Trap: return "NVPTXISD::Suld1DArrayV2I16Trap";
407 case NVPTXISD::Suld1DArrayV2I32Trap: return "NVPTXISD::Suld1DArrayV2I32Trap";
408 case NVPTXISD::Suld1DArrayV4I8Trap: return "NVPTXISD::Suld1DArrayV4I8Trap";
409 case NVPTXISD::Suld1DArrayV4I16Trap: return "NVPTXISD::Suld1DArrayV4I16Trap";
410 case NVPTXISD::Suld1DArrayV4I32Trap: return "NVPTXISD::Suld1DArrayV4I32Trap";
411
412 case NVPTXISD::Suld2DI8Trap: return "NVPTXISD::Suld2DI8Trap";
413 case NVPTXISD::Suld2DI16Trap: return "NVPTXISD::Suld2DI16Trap";
414 case NVPTXISD::Suld2DI32Trap: return "NVPTXISD::Suld2DI32Trap";
415 case NVPTXISD::Suld2DV2I8Trap: return "NVPTXISD::Suld2DV2I8Trap";
416 case NVPTXISD::Suld2DV2I16Trap: return "NVPTXISD::Suld2DV2I16Trap";
417 case NVPTXISD::Suld2DV2I32Trap: return "NVPTXISD::Suld2DV2I32Trap";
418 case NVPTXISD::Suld2DV4I8Trap: return "NVPTXISD::Suld2DV4I8Trap";
419 case NVPTXISD::Suld2DV4I16Trap: return "NVPTXISD::Suld2DV4I16Trap";
420 case NVPTXISD::Suld2DV4I32Trap: return "NVPTXISD::Suld2DV4I32Trap";
421
422 case NVPTXISD::Suld2DArrayI8Trap: return "NVPTXISD::Suld2DArrayI8Trap";
423 case NVPTXISD::Suld2DArrayI16Trap: return "NVPTXISD::Suld2DArrayI16Trap";
424 case NVPTXISD::Suld2DArrayI32Trap: return "NVPTXISD::Suld2DArrayI32Trap";
425 case NVPTXISD::Suld2DArrayV2I8Trap: return "NVPTXISD::Suld2DArrayV2I8Trap";
426 case NVPTXISD::Suld2DArrayV2I16Trap: return "NVPTXISD::Suld2DArrayV2I16Trap";
427 case NVPTXISD::Suld2DArrayV2I32Trap: return "NVPTXISD::Suld2DArrayV2I32Trap";
428 case NVPTXISD::Suld2DArrayV4I8Trap: return "NVPTXISD::Suld2DArrayV4I8Trap";
429 case NVPTXISD::Suld2DArrayV4I16Trap: return "NVPTXISD::Suld2DArrayV4I16Trap";
430 case NVPTXISD::Suld2DArrayV4I32Trap: return "NVPTXISD::Suld2DArrayV4I32Trap";
431
432 case NVPTXISD::Suld3DI8Trap: return "NVPTXISD::Suld3DI8Trap";
433 case NVPTXISD::Suld3DI16Trap: return "NVPTXISD::Suld3DI16Trap";
434 case NVPTXISD::Suld3DI32Trap: return "NVPTXISD::Suld3DI32Trap";
435 case NVPTXISD::Suld3DV2I8Trap: return "NVPTXISD::Suld3DV2I8Trap";
436 case NVPTXISD::Suld3DV2I16Trap: return "NVPTXISD::Suld3DV2I16Trap";
437 case NVPTXISD::Suld3DV2I32Trap: return "NVPTXISD::Suld3DV2I32Trap";
438 case NVPTXISD::Suld3DV4I8Trap: return "NVPTXISD::Suld3DV4I8Trap";
439 case NVPTXISD::Suld3DV4I16Trap: return "NVPTXISD::Suld3DV4I16Trap";
440 case NVPTXISD::Suld3DV4I32Trap: return "NVPTXISD::Suld3DV4I32Trap";
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]441 }
442}
443
Matt Arsenaultf751d622014-03-31 20:54:58 +0000[diff] [blame]444bool NVPTXTargetLowering::shouldSplitVectorType(EVT VT) const {
445 return VT.getScalarType() == MVT::i1;
Justin Holewinskibc451192012-11-29 14:26:24 +0000[diff] [blame]446}
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]447
448SDValue
449NVPTXTargetLowering::LowerGlobalAddress(SDValue Op, SelectionDAG &DAG) const {
Andrew Trickef9de2a2013-05-25 02:42:55 +0000[diff] [blame]450 SDLoc dl(Op);
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]451 const GlobalValue *GV = cast<GlobalAddressSDNode>(Op)->getGlobal();
452 Op = DAG.getTargetGlobalAddress(GV, dl, getPointerTy());
453 return DAG.getNode(NVPTXISD::Wrapper, dl, getPointerTy(), Op);
454}
455
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]456std::string
457NVPTXTargetLowering::getPrototype(Type *retTy, const ArgListTy &Args,
458 const SmallVectorImpl<ISD::OutputArg> &Outs,
459 unsigned retAlignment,
460 const ImmutableCallSite *CS) const {
461
462 bool isABI = (nvptxSubtarget.getSmVersion() >= 20);
463 assert(isABI && "Non-ABI compilation is not supported");
464 if (!isABI)
465 return "";
466
467 std::stringstream O;
468 O << "prototype_" << uniqueCallSite << " : .callprototype ";
469
470 if (retTy->getTypeID() == Type::VoidTyID) {
471 O << "()";
472 } else {
473 O << "(";
Rafael Espindola08013342013-12-07 19:34:20 +0000[diff] [blame]474 if (retTy->isFloatingPointTy() || retTy->isIntegerTy()) {
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]475 unsigned size = 0;
476 if (const IntegerType *ITy = dyn_cast<IntegerType>(retTy)) {
477 size = ITy->getBitWidth();
478 if (size < 32)
479 size = 32;
480 } else {
481 assert(retTy->isFloatingPointTy() &&
482 "Floating point type expected here");
483 size = retTy->getPrimitiveSizeInBits();
484 }
485
486 O << ".param .b" << size << " _";
487 } else if (isa<PointerType>(retTy)) {
488 O << ".param .b" << getPointerTy().getSizeInBits() << " _";
489 } else {
490 if ((retTy->getTypeID() == Type::StructTyID) || isa<VectorType>(retTy)) {
491 SmallVector<EVT, 16> vtparts;
492 ComputeValueVTs(*this, retTy, vtparts);
493 unsigned totalsz = 0;
494 for (unsigned i = 0, e = vtparts.size(); i != e; ++i) {
495 unsigned elems = 1;
496 EVT elemtype = vtparts[i];
497 if (vtparts[i].isVector()) {
498 elems = vtparts[i].getVectorNumElements();
499 elemtype = vtparts[i].getVectorElementType();
500 }
501 // TODO: no need to loop
502 for (unsigned j = 0, je = elems; j != je; ++j) {
503 unsigned sz = elemtype.getSizeInBits();
504 if (elemtype.isInteger() && (sz < 8))
505 sz = 8;
506 totalsz += sz / 8;
507 }
508 }
509 O << ".param .align " << retAlignment << " .b8 _[" << totalsz << "]";
510 } else {
511 assert(false && "Unknown return type");
512 }
513 }
514 O << ") ";
515 }
516 O << "_ (";
517
518 bool first = true;
519 MVT thePointerTy = getPointerTy();
520
521 unsigned OIdx = 0;
522 for (unsigned i = 0, e = Args.size(); i != e; ++i, ++OIdx) {
523 Type *Ty = Args[i].Ty;
524 if (!first) {
525 O << ", ";
526 }
527 first = false;
528
529 if (Outs[OIdx].Flags.isByVal() == false) {
530 if (Ty->isAggregateType() || Ty->isVectorTy()) {
531 unsigned align = 0;
532 const CallInst *CallI = cast<CallInst>(CS->getInstruction());
533 const DataLayout *TD = getDataLayout();
534 // +1 because index 0 is reserved for return type alignment
535 if (!llvm::getAlign(*CallI, i + 1, align))
536 align = TD->getABITypeAlignment(Ty);
537 unsigned sz = TD->getTypeAllocSize(Ty);
538 O << ".param .align " << align << " .b8 ";
539 O << "_";
540 O << "[" << sz << "]";
541 // update the index for Outs
542 SmallVector<EVT, 16> vtparts;
543 ComputeValueVTs(*this, Ty, vtparts);
544 if (unsigned len = vtparts.size())
545 OIdx += len - 1;
546 continue;
547 }
Justin Holewinskidff28d22013-07-01 12:59:01 +0000[diff] [blame]548 // i8 types in IR will be i16 types in SDAG
549 assert((getValueType(Ty) == Outs[OIdx].VT ||
550 (getValueType(Ty) == MVT::i8 && Outs[OIdx].VT == MVT::i16)) &&
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]551 "type mismatch between callee prototype and arguments");
552 // scalar type
553 unsigned sz = 0;
554 if (isa<IntegerType>(Ty)) {
555 sz = cast<IntegerType>(Ty)->getBitWidth();
556 if (sz < 32)
557 sz = 32;
558 } else if (isa<PointerType>(Ty))
559 sz = thePointerTy.getSizeInBits();
560 else
561 sz = Ty->getPrimitiveSizeInBits();
562 O << ".param .b" << sz << " ";
563 O << "_";
564 continue;
565 }
566 const PointerType *PTy = dyn_cast<PointerType>(Ty);
567 assert(PTy && "Param with byval attribute should be a pointer type");
568 Type *ETy = PTy->getElementType();
569
570 unsigned align = Outs[OIdx].Flags.getByValAlign();
571 unsigned sz = getDataLayout()->getTypeAllocSize(ETy);
572 O << ".param .align " << align << " .b8 ";
573 O << "_";
574 O << "[" << sz << "]";
575 }
576 O << ");";
577 return O.str();
578}
579
580unsigned
581NVPTXTargetLowering::getArgumentAlignment(SDValue Callee,
582 const ImmutableCallSite *CS,
583 Type *Ty,
584 unsigned Idx) const {
585 const DataLayout *TD = getDataLayout();
Justin Holewinski124e93d2013-11-11 19:28:19 +0000[diff] [blame]586 unsigned Align = 0;
587 const Value *DirectCallee = CS->getCalledFunction();
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]588
Justin Holewinski124e93d2013-11-11 19:28:19 +0000[diff] [blame]589 if (!DirectCallee) {
590 // We don't have a direct function symbol, but that may be because of
591 // constant cast instructions in the call.
592 const Instruction *CalleeI = CS->getInstruction();
593 assert(CalleeI && "Call target is not a function or derived value?");
594
595 // With bitcast'd call targets, the instruction will be the call
596 if (isa<CallInst>(CalleeI)) {
597 // Check if we have call alignment metadata
598 if (llvm::getAlign(*cast<CallInst>(CalleeI), Idx, Align))
599 return Align;
600
601 const Value *CalleeV = cast<CallInst>(CalleeI)->getCalledValue();
602 // Ignore any bitcast instructions
603 while(isa<ConstantExpr>(CalleeV)) {
604 const ConstantExpr *CE = cast<ConstantExpr>(CalleeV);
605 if (!CE->isCast())
606 break;
607 // Look through the bitcast
608 CalleeV = cast<ConstantExpr>(CalleeV)->getOperand(0);
609 }
610
611 // We have now looked past all of the bitcasts. Do we finally have a
612 // Function?
613 if (isa<Function>(CalleeV))
614 DirectCallee = CalleeV;
615 }
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]616 }
617
Justin Holewinski124e93d2013-11-11 19:28:19 +0000[diff] [blame]618 // Check for function alignment information if we found that the
619 // ultimate target is a Function
620 if (DirectCallee)
621 if (llvm::getAlign(*cast<Function>(DirectCallee), Idx, Align))
622 return Align;
623
624 // Call is indirect or alignment information is not available, fall back to
625 // the ABI type alignment
626 return TD->getABITypeAlignment(Ty);
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]627}
628
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]629SDValue NVPTXTargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
630 SmallVectorImpl<SDValue> &InVals) const {
631 SelectionDAG &DAG = CLI.DAG;
Andrew Trickef9de2a2013-05-25 02:42:55 +0000[diff] [blame]632 SDLoc dl = CLI.DL;
Craig Topperb94011f2013-07-14 04:42:23 +0000[diff] [blame]633 SmallVectorImpl<ISD::OutputArg> &Outs = CLI.Outs;
634 SmallVectorImpl<SDValue> &OutVals = CLI.OutVals;
635 SmallVectorImpl<ISD::InputArg> &Ins = CLI.Ins;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]636 SDValue Chain = CLI.Chain;
637 SDValue Callee = CLI.Callee;
638 bool &isTailCall = CLI.IsTailCall;
Saleem Abdulrasool9f664c12014-05-17 21:50:01 +0000[diff] [blame^]639 ArgListTy &Args = CLI.getArgs();
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]640 Type *retTy = CLI.RetTy;
641 ImmutableCallSite *CS = CLI.CS;
Justin Holewinskiaa583972012-05-25 16:35:28 +0000[diff] [blame]642
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]643 bool isABI = (nvptxSubtarget.getSmVersion() >= 20);
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]644 assert(isABI && "Non-ABI compilation is not supported");
645 if (!isABI)
646 return Chain;
647 const DataLayout *TD = getDataLayout();
648 MachineFunction &MF = DAG.getMachineFunction();
649 const Function *F = MF.getFunction();
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]650
651 SDValue tempChain = Chain;
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]652 Chain =
653 DAG.getCALLSEQ_START(Chain, DAG.getIntPtrConstant(uniqueCallSite, true),
654 dl);
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]655 SDValue InFlag = Chain.getValue(1);
656
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]657 unsigned paramCount = 0;
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]658 // Args.size() and Outs.size() need not match.
659 // Outs.size() will be larger
660 // * if there is an aggregate argument with multiple fields (each field
661 // showing up separately in Outs)
662 // * if there is a vector argument with more than typical vector-length
663 // elements (generally if more than 4) where each vector element is
664 // individually present in Outs.
665 // So a different index should be used for indexing into Outs/OutVals.
666 // See similar issue in LowerFormalArguments.
667 unsigned OIdx = 0;
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]668 // Declare the .params or .reg need to pass values
669 // to the function
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]670 for (unsigned i = 0, e = Args.size(); i != e; ++i, ++OIdx) {
671 EVT VT = Outs[OIdx].VT;
672 Type *Ty = Args[i].Ty;
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]673
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]674 if (Outs[OIdx].Flags.isByVal() == false) {
675 if (Ty->isAggregateType()) {
676 // aggregate
677 SmallVector<EVT, 16> vtparts;
678 ComputeValueVTs(*this, Ty, vtparts);
679
680 unsigned align = getArgumentAlignment(Callee, CS, Ty, paramCount + 1);
681 // declare .param .align <align> .b8 .param<n>[<size>];
682 unsigned sz = TD->getTypeAllocSize(Ty);
683 SDVTList DeclareParamVTs = DAG.getVTList(MVT::Other, MVT::Glue);
684 SDValue DeclareParamOps[] = { Chain, DAG.getConstant(align, MVT::i32),
685 DAG.getConstant(paramCount, MVT::i32),
686 DAG.getConstant(sz, MVT::i32), InFlag };
687 Chain = DAG.getNode(NVPTXISD::DeclareParam, dl, DeclareParamVTs,
Craig Topper48d114b2014-04-26 18:35:24 +0000[diff] [blame]688 DeclareParamOps);
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]689 InFlag = Chain.getValue(1);
690 unsigned curOffset = 0;
691 for (unsigned j = 0, je = vtparts.size(); j != je; ++j) {
692 unsigned elems = 1;
693 EVT elemtype = vtparts[j];
694 if (vtparts[j].isVector()) {
695 elems = vtparts[j].getVectorNumElements();
696 elemtype = vtparts[j].getVectorElementType();
697 }
698 for (unsigned k = 0, ke = elems; k != ke; ++k) {
699 unsigned sz = elemtype.getSizeInBits();
700 if (elemtype.isInteger() && (sz < 8))
701 sz = 8;
702 SDValue StVal = OutVals[OIdx];
703 if (elemtype.getSizeInBits() < 16) {
Justin Holewinskia2911282013-07-01 12:58:58 +0000[diff] [blame]704 StVal = DAG.getNode(ISD::ANY_EXTEND, dl, MVT::i16, StVal);
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]705 }
706 SDVTList CopyParamVTs = DAG.getVTList(MVT::Other, MVT::Glue);
707 SDValue CopyParamOps[] = { Chain,
708 DAG.getConstant(paramCount, MVT::i32),
709 DAG.getConstant(curOffset, MVT::i32),
710 StVal, InFlag };
711 Chain = DAG.getMemIntrinsicNode(NVPTXISD::StoreParam, dl,
Craig Topper206fcd42014-04-26 19:29:41 +0000[diff] [blame]712 CopyParamVTs, CopyParamOps,
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]713 elemtype, MachinePointerInfo());
714 InFlag = Chain.getValue(1);
715 curOffset += sz / 8;
716 ++OIdx;
717 }
718 }
719 if (vtparts.size() > 0)
720 --OIdx;
721 ++paramCount;
722 continue;
723 }
724 if (Ty->isVectorTy()) {
725 EVT ObjectVT = getValueType(Ty);
726 unsigned align = getArgumentAlignment(Callee, CS, Ty, paramCount + 1);
727 // declare .param .align <align> .b8 .param<n>[<size>];
728 unsigned sz = TD->getTypeAllocSize(Ty);
729 SDVTList DeclareParamVTs = DAG.getVTList(MVT::Other, MVT::Glue);
730 SDValue DeclareParamOps[] = { Chain, DAG.getConstant(align, MVT::i32),
731 DAG.getConstant(paramCount, MVT::i32),
732 DAG.getConstant(sz, MVT::i32), InFlag };
733 Chain = DAG.getNode(NVPTXISD::DeclareParam, dl, DeclareParamVTs,
Craig Topper48d114b2014-04-26 18:35:24 +0000[diff] [blame]734 DeclareParamOps);
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]735 InFlag = Chain.getValue(1);
736 unsigned NumElts = ObjectVT.getVectorNumElements();
737 EVT EltVT = ObjectVT.getVectorElementType();
738 EVT MemVT = EltVT;
739 bool NeedExtend = false;
740 if (EltVT.getSizeInBits() < 16) {
741 NeedExtend = true;
742 EltVT = MVT::i16;
743 }
744
745 // V1 store
746 if (NumElts == 1) {
747 SDValue Elt = OutVals[OIdx++];
748 if (NeedExtend)
749 Elt = DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i16, Elt);
750
751 SDVTList CopyParamVTs = DAG.getVTList(MVT::Other, MVT::Glue);
752 SDValue CopyParamOps[] = { Chain,
753 DAG.getConstant(paramCount, MVT::i32),
754 DAG.getConstant(0, MVT::i32), Elt,
755 InFlag };
756 Chain = DAG.getMemIntrinsicNode(NVPTXISD::StoreParam, dl,
Craig Topper206fcd42014-04-26 19:29:41 +0000[diff] [blame]757 CopyParamVTs, CopyParamOps,
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]758 MemVT, MachinePointerInfo());
759 InFlag = Chain.getValue(1);
760 } else if (NumElts == 2) {
761 SDValue Elt0 = OutVals[OIdx++];
762 SDValue Elt1 = OutVals[OIdx++];
763 if (NeedExtend) {
764 Elt0 = DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i16, Elt0);
765 Elt1 = DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i16, Elt1);
766 }
767
768 SDVTList CopyParamVTs = DAG.getVTList(MVT::Other, MVT::Glue);
769 SDValue CopyParamOps[] = { Chain,
770 DAG.getConstant(paramCount, MVT::i32),
771 DAG.getConstant(0, MVT::i32), Elt0, Elt1,
772 InFlag };
773 Chain = DAG.getMemIntrinsicNode(NVPTXISD::StoreParamV2, dl,
Craig Topper206fcd42014-04-26 19:29:41 +0000[diff] [blame]774 CopyParamVTs, CopyParamOps,
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]775 MemVT, MachinePointerInfo());
776 InFlag = Chain.getValue(1);
777 } else {
778 unsigned curOffset = 0;
779 // V4 stores
780 // We have at least 4 elements (<3 x Ty> expands to 4 elements) and
781 // the
782 // vector will be expanded to a power of 2 elements, so we know we can
783 // always round up to the next multiple of 4 when creating the vector
784 // stores.
785 // e.g. 4 elem => 1 st.v4
786 // 6 elem => 2 st.v4
787 // 8 elem => 2 st.v4
788 // 11 elem => 3 st.v4
789 unsigned VecSize = 4;
790 if (EltVT.getSizeInBits() == 64)
791 VecSize = 2;
792
793 // This is potentially only part of a vector, so assume all elements
794 // are packed together.
795 unsigned PerStoreOffset = MemVT.getStoreSizeInBits() / 8 * VecSize;
796
797 for (unsigned i = 0; i < NumElts; i += VecSize) {
798 // Get values
799 SDValue StoreVal;
800 SmallVector<SDValue, 8> Ops;
801 Ops.push_back(Chain);
802 Ops.push_back(DAG.getConstant(paramCount, MVT::i32));
803 Ops.push_back(DAG.getConstant(curOffset, MVT::i32));
804
805 unsigned Opc = NVPTXISD::StoreParamV2;
806
807 StoreVal = OutVals[OIdx++];
808 if (NeedExtend)
809 StoreVal = DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i16, StoreVal);
810 Ops.push_back(StoreVal);
811
812 if (i + 1 < NumElts) {
813 StoreVal = OutVals[OIdx++];
814 if (NeedExtend)
815 StoreVal =
816 DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i16, StoreVal);
817 } else {
818 StoreVal = DAG.getUNDEF(EltVT);
819 }
820 Ops.push_back(StoreVal);
821
822 if (VecSize == 4) {
823 Opc = NVPTXISD::StoreParamV4;
824 if (i + 2 < NumElts) {
825 StoreVal = OutVals[OIdx++];
826 if (NeedExtend)
827 StoreVal =
828 DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i16, StoreVal);
829 } else {
830 StoreVal = DAG.getUNDEF(EltVT);
831 }
832 Ops.push_back(StoreVal);
833
834 if (i + 3 < NumElts) {
835 StoreVal = OutVals[OIdx++];
836 if (NeedExtend)
837 StoreVal =
838 DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i16, StoreVal);
839 } else {
840 StoreVal = DAG.getUNDEF(EltVT);
841 }
842 Ops.push_back(StoreVal);
843 }
844
Justin Holewinskidff28d22013-07-01 12:59:01 +0000[diff] [blame]845 Ops.push_back(InFlag);
846
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]847 SDVTList CopyParamVTs = DAG.getVTList(MVT::Other, MVT::Glue);
Craig Topper206fcd42014-04-26 19:29:41 +0000[diff] [blame]848 Chain = DAG.getMemIntrinsicNode(Opc, dl, CopyParamVTs, Ops,
849 MemVT, MachinePointerInfo());
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]850 InFlag = Chain.getValue(1);
851 curOffset += PerStoreOffset;
852 }
853 }
854 ++paramCount;
855 --OIdx;
856 continue;
857 }
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]858 // Plain scalar
859 // for ABI, declare .param .b<size> .param<n>;
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]860 unsigned sz = VT.getSizeInBits();
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]861 bool needExtend = false;
862 if (VT.isInteger()) {
863 if (sz < 16)
864 needExtend = true;
865 if (sz < 32)
866 sz = 32;
867 }
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]868 SDVTList DeclareParamVTs = DAG.getVTList(MVT::Other, MVT::Glue);
869 SDValue DeclareParamOps[] = { Chain,
870 DAG.getConstant(paramCount, MVT::i32),
871 DAG.getConstant(sz, MVT::i32),
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]872 DAG.getConstant(0, MVT::i32), InFlag };
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]873 Chain = DAG.getNode(NVPTXISD::DeclareScalarParam, dl, DeclareParamVTs,
Craig Topper48d114b2014-04-26 18:35:24 +0000[diff] [blame]874 DeclareParamOps);
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]875 InFlag = Chain.getValue(1);
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]876 SDValue OutV = OutVals[OIdx];
877 if (needExtend) {
878 // zext/sext i1 to i16
879 unsigned opc = ISD::ZERO_EXTEND;
880 if (Outs[OIdx].Flags.isSExt())
881 opc = ISD::SIGN_EXTEND;
882 OutV = DAG.getNode(opc, dl, MVT::i16, OutV);
883 }
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]884 SDVTList CopyParamVTs = DAG.getVTList(MVT::Other, MVT::Glue);
885 SDValue CopyParamOps[] = { Chain, DAG.getConstant(paramCount, MVT::i32),
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]886 DAG.getConstant(0, MVT::i32), OutV, InFlag };
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]887
888 unsigned opcode = NVPTXISD::StoreParam;
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]889 if (Outs[OIdx].Flags.isZExt())
890 opcode = NVPTXISD::StoreParamU32;
891 else if (Outs[OIdx].Flags.isSExt())
892 opcode = NVPTXISD::StoreParamS32;
Craig Topper206fcd42014-04-26 19:29:41 +0000[diff] [blame]893 Chain = DAG.getMemIntrinsicNode(opcode, dl, CopyParamVTs, CopyParamOps,
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]894 VT, MachinePointerInfo());
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]895
896 InFlag = Chain.getValue(1);
897 ++paramCount;
898 continue;
899 }
900 // struct or vector
901 SmallVector<EVT, 16> vtparts;
902 const PointerType *PTy = dyn_cast<PointerType>(Args[i].Ty);
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]903 assert(PTy && "Type of a byval parameter should be pointer");
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]904 ComputeValueVTs(*this, PTy->getElementType(), vtparts);
905
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]906 // declare .param .align <align> .b8 .param<n>[<size>];
907 unsigned sz = Outs[OIdx].Flags.getByValSize();
908 SDVTList DeclareParamVTs = DAG.getVTList(MVT::Other, MVT::Glue);
909 // The ByValAlign in the Outs[OIdx].Flags is alway set at this point,
910 // so we don't need to worry about natural alignment or not.
911 // See TargetLowering::LowerCallTo().
912 SDValue DeclareParamOps[] = {
913 Chain, DAG.getConstant(Outs[OIdx].Flags.getByValAlign(), MVT::i32),
914 DAG.getConstant(paramCount, MVT::i32), DAG.getConstant(sz, MVT::i32),
915 InFlag
916 };
917 Chain = DAG.getNode(NVPTXISD::DeclareParam, dl, DeclareParamVTs,
Craig Topper48d114b2014-04-26 18:35:24 +0000[diff] [blame]918 DeclareParamOps);
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]919 InFlag = Chain.getValue(1);
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]920 unsigned curOffset = 0;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]921 for (unsigned j = 0, je = vtparts.size(); j != je; ++j) {
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]922 unsigned elems = 1;
923 EVT elemtype = vtparts[j];
924 if (vtparts[j].isVector()) {
925 elems = vtparts[j].getVectorNumElements();
926 elemtype = vtparts[j].getVectorElementType();
927 }
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]928 for (unsigned k = 0, ke = elems; k != ke; ++k) {
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]929 unsigned sz = elemtype.getSizeInBits();
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]930 if (elemtype.isInteger() && (sz < 8))
931 sz = 8;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]932 SDValue srcAddr =
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]933 DAG.getNode(ISD::ADD, dl, getPointerTy(), OutVals[OIdx],
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]934 DAG.getConstant(curOffset, getPointerTy()));
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]935 SDValue theVal = DAG.getLoad(elemtype, dl, tempChain, srcAddr,
936 MachinePointerInfo(), false, false, false,
937 0);
938 if (elemtype.getSizeInBits() < 16) {
Justin Holewinskia2911282013-07-01 12:58:58 +0000[diff] [blame]939 theVal = DAG.getNode(ISD::ANY_EXTEND, dl, MVT::i16, theVal);
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]940 }
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]941 SDVTList CopyParamVTs = DAG.getVTList(MVT::Other, MVT::Glue);
942 SDValue CopyParamOps[] = { Chain, DAG.getConstant(paramCount, MVT::i32),
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]943 DAG.getConstant(curOffset, MVT::i32), theVal,
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]944 InFlag };
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]945 Chain = DAG.getMemIntrinsicNode(NVPTXISD::StoreParam, dl, CopyParamVTs,
Craig Topper206fcd42014-04-26 19:29:41 +0000[diff] [blame]946 CopyParamOps, elemtype,
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]947 MachinePointerInfo());
948
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]949 InFlag = Chain.getValue(1);
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]950 curOffset += sz / 8;
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]951 }
952 }
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]953 ++paramCount;
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]954 }
955
956 GlobalAddressSDNode *Func = dyn_cast<GlobalAddressSDNode>(Callee.getNode());
957 unsigned retAlignment = 0;
958
959 // Handle Result
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]960 if (Ins.size() > 0) {
961 SmallVector<EVT, 16> resvtparts;
962 ComputeValueVTs(*this, retTy, resvtparts);
963
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]964 // Declare
965 // .param .align 16 .b8 retval0[<size-in-bytes>], or
966 // .param .b<size-in-bits> retval0
967 unsigned resultsz = TD->getTypeAllocSizeInBits(retTy);
Rafael Espindola08013342013-12-07 19:34:20 +0000[diff] [blame]968 if (retTy->isSingleValueType()) {
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]969 // Scalar needs to be at least 32bit wide
970 if (resultsz < 32)
971 resultsz = 32;
972 SDVTList DeclareRetVTs = DAG.getVTList(MVT::Other, MVT::Glue);
973 SDValue DeclareRetOps[] = { Chain, DAG.getConstant(1, MVT::i32),
974 DAG.getConstant(resultsz, MVT::i32),
975 DAG.getConstant(0, MVT::i32), InFlag };
976 Chain = DAG.getNode(NVPTXISD::DeclareRet, dl, DeclareRetVTs,
Craig Topper48d114b2014-04-26 18:35:24 +0000[diff] [blame]977 DeclareRetOps);
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]978 InFlag = Chain.getValue(1);
979 } else {
980 retAlignment = getArgumentAlignment(Callee, CS, retTy, 0);
981 SDVTList DeclareRetVTs = DAG.getVTList(MVT::Other, MVT::Glue);
982 SDValue DeclareRetOps[] = { Chain,
983 DAG.getConstant(retAlignment, MVT::i32),
984 DAG.getConstant(resultsz / 8, MVT::i32),
985 DAG.getConstant(0, MVT::i32), InFlag };
986 Chain = DAG.getNode(NVPTXISD::DeclareRetParam, dl, DeclareRetVTs,
Craig Topper48d114b2014-04-26 18:35:24 +0000[diff] [blame]987 DeclareRetOps);
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]988 InFlag = Chain.getValue(1);
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]989 }
990 }
991
992 if (!Func) {
993 // This is indirect function call case : PTX requires a prototype of the
994 // form
995 // proto_0 : .callprototype(.param .b32 _) _ (.param .b32 _);
996 // to be emitted, and the label has to used as the last arg of call
997 // instruction.
Justin Holewinski3d49e5c2013-11-15 12:30:04 +0000[diff] [blame]998 // The prototype is embedded in a string and put as the operand for a
999 // CallPrototype SDNode which will print out to the value of the string.
1000 SDVTList ProtoVTs = DAG.getVTList(MVT::Other, MVT::Glue);
1001 std::string Proto = getPrototype(retTy, Args, Outs, retAlignment, CS);
1002 const char *ProtoStr =
1003 nvTM->getManagedStrPool()->getManagedString(Proto.c_str())->c_str();
1004 SDValue ProtoOps[] = {
1005 Chain, DAG.getTargetExternalSymbol(ProtoStr, MVT::i32), InFlag,
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1006 };
Craig Topper48d114b2014-04-26 18:35:24 +0000[diff] [blame]1007 Chain = DAG.getNode(NVPTXISD::CallPrototype, dl, ProtoVTs, ProtoOps);
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]1008 InFlag = Chain.getValue(1);
1009 }
1010 // Op to just print "call"
1011 SDVTList PrintCallVTs = DAG.getVTList(MVT::Other, MVT::Glue);
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1012 SDValue PrintCallOps[] = {
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]1013 Chain, DAG.getConstant((Ins.size() == 0) ? 0 : 1, MVT::i32), InFlag
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1014 };
1015 Chain = DAG.getNode(Func ? (NVPTXISD::PrintCallUni) : (NVPTXISD::PrintCall),
Craig Topper48d114b2014-04-26 18:35:24 +0000[diff] [blame]1016 dl, PrintCallVTs, PrintCallOps);
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]1017 InFlag = Chain.getValue(1);
1018
1019 // Ops to print out the function name
1020 SDVTList CallVoidVTs = DAG.getVTList(MVT::Other, MVT::Glue);
1021 SDValue CallVoidOps[] = { Chain, Callee, InFlag };
Craig Topper48d114b2014-04-26 18:35:24 +0000[diff] [blame]1022 Chain = DAG.getNode(NVPTXISD::CallVoid, dl, CallVoidVTs, CallVoidOps);
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]1023 InFlag = Chain.getValue(1);
1024
1025 // Ops to print out the param list
1026 SDVTList CallArgBeginVTs = DAG.getVTList(MVT::Other, MVT::Glue);
1027 SDValue CallArgBeginOps[] = { Chain, InFlag };
1028 Chain = DAG.getNode(NVPTXISD::CallArgBegin, dl, CallArgBeginVTs,
Craig Topper48d114b2014-04-26 18:35:24 +0000[diff] [blame]1029 CallArgBeginOps);
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]1030 InFlag = Chain.getValue(1);
1031
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1032 for (unsigned i = 0, e = paramCount; i != e; ++i) {
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]1033 unsigned opcode;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1034 if (i == (e - 1))
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]1035 opcode = NVPTXISD::LastCallArg;
1036 else
1037 opcode = NVPTXISD::CallArg;
1038 SDVTList CallArgVTs = DAG.getVTList(MVT::Other, MVT::Glue);
1039 SDValue CallArgOps[] = { Chain, DAG.getConstant(1, MVT::i32),
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1040 DAG.getConstant(i, MVT::i32), InFlag };
Craig Topper48d114b2014-04-26 18:35:24 +0000[diff] [blame]1041 Chain = DAG.getNode(opcode, dl, CallArgVTs, CallArgOps);
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]1042 InFlag = Chain.getValue(1);
1043 }
1044 SDVTList CallArgEndVTs = DAG.getVTList(MVT::Other, MVT::Glue);
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1045 SDValue CallArgEndOps[] = { Chain, DAG.getConstant(Func ? 1 : 0, MVT::i32),
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]1046 InFlag };
Craig Topper48d114b2014-04-26 18:35:24 +0000[diff] [blame]1047 Chain = DAG.getNode(NVPTXISD::CallArgEnd, dl, CallArgEndVTs, CallArgEndOps);
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]1048 InFlag = Chain.getValue(1);
1049
1050 if (!Func) {
1051 SDVTList PrototypeVTs = DAG.getVTList(MVT::Other, MVT::Glue);
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1052 SDValue PrototypeOps[] = { Chain, DAG.getConstant(uniqueCallSite, MVT::i32),
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]1053 InFlag };
Craig Topper48d114b2014-04-26 18:35:24 +0000[diff] [blame]1054 Chain = DAG.getNode(NVPTXISD::Prototype, dl, PrototypeVTs, PrototypeOps);
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]1055 InFlag = Chain.getValue(1);
1056 }
1057
1058 // Generate loads from param memory/moves from registers for result
1059 if (Ins.size() > 0) {
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]1060 unsigned resoffset = 0;
1061 if (retTy && retTy->isVectorTy()) {
1062 EVT ObjectVT = getValueType(retTy);
1063 unsigned NumElts = ObjectVT.getVectorNumElements();
1064 EVT EltVT = ObjectVT.getVectorElementType();
Benjamin Kramer3cc579a2013-06-29 22:51:12 +0000[diff] [blame]1065 assert(nvTM->getTargetLowering()->getNumRegisters(F->getContext(),
1066 ObjectVT) == NumElts &&
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]1067 "Vector was not scalarized");
1068 unsigned sz = EltVT.getSizeInBits();
1069 bool needTruncate = sz < 16 ? true : false;
1070
1071 if (NumElts == 1) {
1072 // Just a simple load
Craig Topper59f626d2014-04-26 19:29:47 +0000[diff] [blame]1073 SmallVector<EVT, 4> LoadRetVTs;
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]1074 if (needTruncate) {
1075 // If loading i1 result, generate
1076 // load i16
1077 // trunc i16 to i1
1078 LoadRetVTs.push_back(MVT::i16);
1079 } else
1080 LoadRetVTs.push_back(EltVT);
1081 LoadRetVTs.push_back(MVT::Other);
1082 LoadRetVTs.push_back(MVT::Glue);
Craig Topper59f626d2014-04-26 19:29:47 +0000[diff] [blame]1083 SmallVector<SDValue, 4> LoadRetOps;
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]1084 LoadRetOps.push_back(Chain);
1085 LoadRetOps.push_back(DAG.getConstant(1, MVT::i32));
1086 LoadRetOps.push_back(DAG.getConstant(0, MVT::i32));
1087 LoadRetOps.push_back(InFlag);
1088 SDValue retval = DAG.getMemIntrinsicNode(
1089 NVPTXISD::LoadParam, dl,
Craig Topper206fcd42014-04-26 19:29:41 +0000[diff] [blame]1090 DAG.getVTList(LoadRetVTs), LoadRetOps, EltVT, MachinePointerInfo());
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]1091 Chain = retval.getValue(1);
1092 InFlag = retval.getValue(2);
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]1093 SDValue Ret0 = retval;
1094 if (needTruncate)
1095 Ret0 = DAG.getNode(ISD::TRUNCATE, dl, EltVT, Ret0);
1096 InVals.push_back(Ret0);
1097 } else if (NumElts == 2) {
1098 // LoadV2
Craig Topper59f626d2014-04-26 19:29:47 +0000[diff] [blame]1099 SmallVector<EVT, 4> LoadRetVTs;
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]1100 if (needTruncate) {
1101 // If loading i1 result, generate
1102 // load i16
1103 // trunc i16 to i1
1104 LoadRetVTs.push_back(MVT::i16);
1105 LoadRetVTs.push_back(MVT::i16);
1106 } else {
1107 LoadRetVTs.push_back(EltVT);
1108 LoadRetVTs.push_back(EltVT);
1109 }
1110 LoadRetVTs.push_back(MVT::Other);
1111 LoadRetVTs.push_back(MVT::Glue);
Craig Topper59f626d2014-04-26 19:29:47 +0000[diff] [blame]1112 SmallVector<SDValue, 4> LoadRetOps;
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]1113 LoadRetOps.push_back(Chain);
1114 LoadRetOps.push_back(DAG.getConstant(1, MVT::i32));
1115 LoadRetOps.push_back(DAG.getConstant(0, MVT::i32));
1116 LoadRetOps.push_back(InFlag);
1117 SDValue retval = DAG.getMemIntrinsicNode(
1118 NVPTXISD::LoadParamV2, dl,
Craig Topper206fcd42014-04-26 19:29:41 +0000[diff] [blame]1119 DAG.getVTList(LoadRetVTs), LoadRetOps, EltVT, MachinePointerInfo());
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]1120 Chain = retval.getValue(2);
1121 InFlag = retval.getValue(3);
1122 SDValue Ret0 = retval.getValue(0);
1123 SDValue Ret1 = retval.getValue(1);
1124 if (needTruncate) {
1125 Ret0 = DAG.getNode(ISD::TRUNCATE, dl, MVT::i1, Ret0);
1126 InVals.push_back(Ret0);
1127 Ret1 = DAG.getNode(ISD::TRUNCATE, dl, MVT::i1, Ret1);
1128 InVals.push_back(Ret1);
1129 } else {
1130 InVals.push_back(Ret0);
1131 InVals.push_back(Ret1);
1132 }
1133 } else {
1134 // Split into N LoadV4
1135 unsigned Ofst = 0;
1136 unsigned VecSize = 4;
1137 unsigned Opc = NVPTXISD::LoadParamV4;
1138 if (EltVT.getSizeInBits() == 64) {
1139 VecSize = 2;
1140 Opc = NVPTXISD::LoadParamV2;
1141 }
1142 EVT VecVT = EVT::getVectorVT(F->getContext(), EltVT, VecSize);
1143 for (unsigned i = 0; i < NumElts; i += VecSize) {
1144 SmallVector<EVT, 8> LoadRetVTs;
1145 if (needTruncate) {
1146 // If loading i1 result, generate
1147 // load i16
1148 // trunc i16 to i1
1149 for (unsigned j = 0; j < VecSize; ++j)
1150 LoadRetVTs.push_back(MVT::i16);
1151 } else {
1152 for (unsigned j = 0; j < VecSize; ++j)
1153 LoadRetVTs.push_back(EltVT);
1154 }
1155 LoadRetVTs.push_back(MVT::Other);
1156 LoadRetVTs.push_back(MVT::Glue);
1157 SmallVector<SDValue, 4> LoadRetOps;
1158 LoadRetOps.push_back(Chain);
1159 LoadRetOps.push_back(DAG.getConstant(1, MVT::i32));
1160 LoadRetOps.push_back(DAG.getConstant(Ofst, MVT::i32));
1161 LoadRetOps.push_back(InFlag);
1162 SDValue retval = DAG.getMemIntrinsicNode(
Craig Topperabb4ac72014-04-16 06:10:51 +0000[diff] [blame]1163 Opc, dl, DAG.getVTList(LoadRetVTs),
Craig Topper206fcd42014-04-26 19:29:41 +0000[diff] [blame]1164 LoadRetOps, EltVT, MachinePointerInfo());
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]1165 if (VecSize == 2) {
1166 Chain = retval.getValue(2);
1167 InFlag = retval.getValue(3);
1168 } else {
1169 Chain = retval.getValue(4);
1170 InFlag = retval.getValue(5);
1171 }
1172
1173 for (unsigned j = 0; j < VecSize; ++j) {
1174 if (i + j >= NumElts)
1175 break;
1176 SDValue Elt = retval.getValue(j);
1177 if (needTruncate)
1178 Elt = DAG.getNode(ISD::TRUNCATE, dl, EltVT, Elt);
1179 InVals.push_back(Elt);
1180 }
1181 Ofst += TD->getTypeAllocSize(VecVT.getTypeForEVT(F->getContext()));
1182 }
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]1183 }
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1184 } else {
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]1185 SmallVector<EVT, 16> VTs;
1186 ComputePTXValueVTs(*this, retTy, VTs);
1187 assert(VTs.size() == Ins.size() && "Bad value decomposition");
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1188 for (unsigned i = 0, e = Ins.size(); i != e; ++i) {
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]1189 unsigned sz = VTs[i].getSizeInBits();
1190 bool needTruncate = sz < 8 ? true : false;
1191 if (VTs[i].isInteger() && (sz < 8))
1192 sz = 8;
1193
1194 SmallVector<EVT, 4> LoadRetVTs;
Justin Holewinskie04e4bd2013-06-28 17:58:10 +0000[diff] [blame]1195 EVT TheLoadType = VTs[i];
1196 if (retTy->isIntegerTy() &&
1197 TD->getTypeAllocSizeInBits(retTy) < 32) {
1198 // This is for integer types only, and specifically not for
1199 // aggregates.
1200 LoadRetVTs.push_back(MVT::i32);
1201 TheLoadType = MVT::i32;
1202 } else if (sz < 16) {
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]1203 // If loading i1/i8 result, generate
1204 // load i8 (-> i16)
1205 // trunc i16 to i1/i8
1206 LoadRetVTs.push_back(MVT::i16);
1207 } else
1208 LoadRetVTs.push_back(Ins[i].VT);
1209 LoadRetVTs.push_back(MVT::Other);
1210 LoadRetVTs.push_back(MVT::Glue);
1211
1212 SmallVector<SDValue, 4> LoadRetOps;
1213 LoadRetOps.push_back(Chain);
1214 LoadRetOps.push_back(DAG.getConstant(1, MVT::i32));
1215 LoadRetOps.push_back(DAG.getConstant(resoffset, MVT::i32));
1216 LoadRetOps.push_back(InFlag);
1217 SDValue retval = DAG.getMemIntrinsicNode(
1218 NVPTXISD::LoadParam, dl,
Craig Topper206fcd42014-04-26 19:29:41 +0000[diff] [blame]1219 DAG.getVTList(LoadRetVTs), LoadRetOps,
1220 TheLoadType, MachinePointerInfo());
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]1221 Chain = retval.getValue(1);
1222 InFlag = retval.getValue(2);
1223 SDValue Ret0 = retval.getValue(0);
1224 if (needTruncate)
1225 Ret0 = DAG.getNode(ISD::TRUNCATE, dl, Ins[i].VT, Ret0);
1226 InVals.push_back(Ret0);
1227 resoffset += sz / 8;
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]1228 }
1229 }
1230 }
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]1231
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1232 Chain = DAG.getCALLSEQ_END(Chain, DAG.getIntPtrConstant(uniqueCallSite, true),
1233 DAG.getIntPtrConstant(uniqueCallSite + 1, true),
Andrew Trickad6d08a2013-05-29 22:03:55 +0000[diff] [blame]1234 InFlag, dl);
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]1235 uniqueCallSite++;
1236
1237 // set isTailCall to false for now, until we figure out how to express
1238 // tail call optimization in PTX
1239 isTailCall = false;
1240 return Chain;
1241}
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]1242
1243// By default CONCAT_VECTORS is lowered by ExpandVectorBuildThroughStack()
1244// (see LegalizeDAG.cpp). This is slow and uses local memory.
1245// We use extract/insert/build vector just as what LegalizeOp() does in llvm 2.5
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1246SDValue
1247NVPTXTargetLowering::LowerCONCAT_VECTORS(SDValue Op, SelectionDAG &DAG) const {
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]1248 SDNode *Node = Op.getNode();
Andrew Trickef9de2a2013-05-25 02:42:55 +0000[diff] [blame]1249 SDLoc dl(Node);
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]1250 SmallVector<SDValue, 8> Ops;
1251 unsigned NumOperands = Node->getNumOperands();
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1252 for (unsigned i = 0; i < NumOperands; ++i) {
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]1253 SDValue SubOp = Node->getOperand(i);
1254 EVT VVT = SubOp.getNode()->getValueType(0);
1255 EVT EltVT = VVT.getVectorElementType();
1256 unsigned NumSubElem = VVT.getVectorNumElements();
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1257 for (unsigned j = 0; j < NumSubElem; ++j) {
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]1258 Ops.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, SubOp,
1259 DAG.getIntPtrConstant(j)));
1260 }
1261 }
Craig Topper48d114b2014-04-26 18:35:24 +0000[diff] [blame]1262 return DAG.getNode(ISD::BUILD_VECTOR, dl, Node->getValueType(0), Ops);
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]1263}
1264
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1265SDValue
1266NVPTXTargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const {
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]1267 switch (Op.getOpcode()) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1268 case ISD::RETURNADDR:
1269 return SDValue();
1270 case ISD::FRAMEADDR:
1271 return SDValue();
1272 case ISD::GlobalAddress:
1273 return LowerGlobalAddress(Op, DAG);
1274 case ISD::INTRINSIC_W_CHAIN:
1275 return Op;
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]1276 case ISD::BUILD_VECTOR:
1277 case ISD::EXTRACT_SUBVECTOR:
1278 return Op;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1279 case ISD::CONCAT_VECTORS:
1280 return LowerCONCAT_VECTORS(Op, DAG);
1281 case ISD::STORE:
1282 return LowerSTORE(Op, DAG);
1283 case ISD::LOAD:
1284 return LowerLOAD(Op, DAG);
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]1285 default:
David Blaikie891d0a32012-05-04 22:34:16 +0000[diff] [blame]1286 llvm_unreachable("Custom lowering not defined for operation");
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]1287 }
1288}
1289
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]1290SDValue NVPTXTargetLowering::LowerLOAD(SDValue Op, SelectionDAG &DAG) const {
1291 if (Op.getValueType() == MVT::i1)
1292 return LowerLOADi1(Op, DAG);
1293 else
1294 return SDValue();
1295}
1296
Justin Holewinskic6462aa2012-11-14 19:19:16 +0000[diff] [blame]1297// v = ld i1* addr
1298// =>
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]1299// v1 = ld i8* addr (-> i16)
1300// v = trunc i16 to i1
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1301SDValue NVPTXTargetLowering::LowerLOADi1(SDValue Op, SelectionDAG &DAG) const {
Justin Holewinskic6462aa2012-11-14 19:19:16 +0000[diff] [blame]1302 SDNode *Node = Op.getNode();
1303 LoadSDNode *LD = cast<LoadSDNode>(Node);
Andrew Trickef9de2a2013-05-25 02:42:55 +0000[diff] [blame]1304 SDLoc dl(Node);
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1305 assert(LD->getExtensionType() == ISD::NON_EXTLOAD);
NAKAMURA Takumi5bbe0e12012-11-14 23:46:15 +0000[diff] [blame]1306 assert(Node->getValueType(0) == MVT::i1 &&
1307 "Custom lowering for i1 load only");
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1308 SDValue newLD =
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]1309 DAG.getLoad(MVT::i16, dl, LD->getChain(), LD->getBasePtr(),
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1310 LD->getPointerInfo(), LD->isVolatile(), LD->isNonTemporal(),
1311 LD->isInvariant(), LD->getAlignment());
Justin Holewinskic6462aa2012-11-14 19:19:16 +0000[diff] [blame]1312 SDValue result = DAG.getNode(ISD::TRUNCATE, dl, MVT::i1, newLD);
1313 // The legalizer (the caller) is expecting two values from the legalized
1314 // load, so we build a MergeValues node for it. See ExpandUnalignedLoad()
1315 // in LegalizeDAG.cpp which also uses MergeValues.
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1316 SDValue Ops[] = { result, LD->getChain() };
Craig Topper64941d92014-04-27 19:20:57 +0000[diff] [blame]1317 return DAG.getMergeValues(Ops, dl);
Justin Holewinskic6462aa2012-11-14 19:19:16 +0000[diff] [blame]1318}
1319
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]1320SDValue NVPTXTargetLowering::LowerSTORE(SDValue Op, SelectionDAG &DAG) const {
1321 EVT ValVT = Op.getOperand(1).getValueType();
1322 if (ValVT == MVT::i1)
1323 return LowerSTOREi1(Op, DAG);
1324 else if (ValVT.isVector())
1325 return LowerSTOREVector(Op, DAG);
1326 else
1327 return SDValue();
1328}
1329
1330SDValue
1331NVPTXTargetLowering::LowerSTOREVector(SDValue Op, SelectionDAG &DAG) const {
1332 SDNode *N = Op.getNode();
1333 SDValue Val = N->getOperand(1);
Andrew Trickef9de2a2013-05-25 02:42:55 +0000[diff] [blame]1334 SDLoc DL(N);
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]1335 EVT ValVT = Val.getValueType();
1336
1337 if (ValVT.isVector()) {
1338 // We only handle "native" vector sizes for now, e.g. <4 x double> is not
1339 // legal. We can (and should) split that into 2 stores of <2 x double> here
1340 // but I'm leaving that as a TODO for now.
1341 if (!ValVT.isSimple())
1342 return SDValue();
1343 switch (ValVT.getSimpleVT().SimpleTy) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1344 default:
1345 return SDValue();
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]1346 case MVT::v2i8:
1347 case MVT::v2i16:
1348 case MVT::v2i32:
1349 case MVT::v2i64:
1350 case MVT::v2f32:
1351 case MVT::v2f64:
1352 case MVT::v4i8:
1353 case MVT::v4i16:
1354 case MVT::v4i32:
1355 case MVT::v4f32:
1356 // This is a "native" vector type
1357 break;
1358 }
1359
1360 unsigned Opcode = 0;
1361 EVT EltVT = ValVT.getVectorElementType();
1362 unsigned NumElts = ValVT.getVectorNumElements();
1363
1364 // Since StoreV2 is a target node, we cannot rely on DAG type legalization.
1365 // Therefore, we must ensure the type is legal. For i1 and i8, we set the
Alp Tokercb402912014-01-24 17:20:08 +0000[diff] [blame]1366 // stored type to i16 and propagate the "real" type as the memory type.
Justin Holewinskia2911282013-07-01 12:58:58 +0000[diff] [blame]1367 bool NeedExt = false;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]1368 if (EltVT.getSizeInBits() < 16)
Justin Holewinskia2911282013-07-01 12:58:58 +0000[diff] [blame]1369 NeedExt = true;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]1370
1371 switch (NumElts) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1372 default:
1373 return SDValue();
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]1374 case 2:
1375 Opcode = NVPTXISD::StoreV2;
1376 break;
1377 case 4: {
1378 Opcode = NVPTXISD::StoreV4;
1379 break;
1380 }
1381 }
1382
1383 SmallVector<SDValue, 8> Ops;
1384
1385 // First is the chain
1386 Ops.push_back(N->getOperand(0));
1387
1388 // Then the split values
1389 for (unsigned i = 0; i < NumElts; ++i) {
1390 SDValue ExtVal = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, EltVT, Val,
1391 DAG.getIntPtrConstant(i));
Justin Holewinskia2911282013-07-01 12:58:58 +0000[diff] [blame]1392 if (NeedExt)
1393 ExtVal = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i16, ExtVal);
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]1394 Ops.push_back(ExtVal);
1395 }
1396
1397 // Then any remaining arguments
1398 for (unsigned i = 2, e = N->getNumOperands(); i != e; ++i) {
1399 Ops.push_back(N->getOperand(i));
1400 }
1401
1402 MemSDNode *MemSD = cast<MemSDNode>(N);
1403
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1404 SDValue NewSt = DAG.getMemIntrinsicNode(
Craig Topper206fcd42014-04-26 19:29:41 +0000[diff] [blame]1405 Opcode, DL, DAG.getVTList(MVT::Other), Ops,
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1406 MemSD->getMemoryVT(), MemSD->getMemOperand());
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]1407
1408 //return DCI.CombineTo(N, NewSt, true);
1409 return NewSt;
1410 }
1411
1412 return SDValue();
1413}
1414
Justin Holewinskic6462aa2012-11-14 19:19:16 +0000[diff] [blame]1415// st i1 v, addr
1416// =>
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]1417// v1 = zxt v to i16
1418// st.u8 i16, addr
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1419SDValue NVPTXTargetLowering::LowerSTOREi1(SDValue Op, SelectionDAG &DAG) const {
Justin Holewinskic6462aa2012-11-14 19:19:16 +0000[diff] [blame]1420 SDNode *Node = Op.getNode();
Andrew Trickef9de2a2013-05-25 02:42:55 +0000[diff] [blame]1421 SDLoc dl(Node);
Justin Holewinskic6462aa2012-11-14 19:19:16 +0000[diff] [blame]1422 StoreSDNode *ST = cast<StoreSDNode>(Node);
1423 SDValue Tmp1 = ST->getChain();
1424 SDValue Tmp2 = ST->getBasePtr();
1425 SDValue Tmp3 = ST->getValue();
NAKAMURA Takumi5bbe0e12012-11-14 23:46:15 +0000[diff] [blame]1426 assert(Tmp3.getValueType() == MVT::i1 && "Custom lowering for i1 store only");
Justin Holewinskic6462aa2012-11-14 19:19:16 +0000[diff] [blame]1427 unsigned Alignment = ST->getAlignment();
1428 bool isVolatile = ST->isVolatile();
1429 bool isNonTemporal = ST->isNonTemporal();
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]1430 Tmp3 = DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i16, Tmp3);
1431 SDValue Result = DAG.getTruncStore(Tmp1, dl, Tmp3, Tmp2,
1432 ST->getPointerInfo(), MVT::i8, isNonTemporal,
1433 isVolatile, Alignment);
Justin Holewinskic6462aa2012-11-14 19:19:16 +0000[diff] [blame]1434 return Result;
1435}
1436
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1437SDValue NVPTXTargetLowering::getExtSymb(SelectionDAG &DAG, const char *inname,
1438 int idx, EVT v) const {
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]1439 std::string *name = nvTM->getManagedStrPool()->getManagedString(inname);
1440 std::stringstream suffix;
1441 suffix << idx;
1442 *name += suffix.str();
1443 return DAG.getTargetExternalSymbol(name->c_str(), v);
1444}
1445
1446SDValue
1447NVPTXTargetLowering::getParamSymbol(SelectionDAG &DAG, int idx, EVT v) const {
Justin Holewinskia2a63d22013-08-06 14:13:27 +0000[diff] [blame]1448 std::string ParamSym;
1449 raw_string_ostream ParamStr(ParamSym);
1450
1451 ParamStr << DAG.getMachineFunction().getName() << "_param_" << idx;
1452 ParamStr.flush();
1453
1454 std::string *SavedStr =
1455 nvTM->getManagedStrPool()->getManagedString(ParamSym.c_str());
1456 return DAG.getTargetExternalSymbol(SavedStr->c_str(), v);
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]1457}
1458
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1459SDValue NVPTXTargetLowering::getParamHelpSymbol(SelectionDAG &DAG, int idx) {
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]1460 return getExtSymb(DAG, ".HLPPARAM", idx);
1461}
1462
1463// Check to see if the kernel argument is image*_t or sampler_t
1464
1465bool llvm::isImageOrSamplerVal(const Value *arg, const Module *context) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1466 static const char *const specialTypes[] = { "struct._image2d_t",
1467 "struct._image3d_t",
1468 "struct._sampler_t" };
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]1469
1470 const Type *Ty = arg->getType();
1471 const PointerType *PTy = dyn_cast<PointerType>(Ty);
1472
1473 if (!PTy)
1474 return false;
1475
1476 if (!context)
1477 return false;
1478
1479 const StructType *STy = dyn_cast<StructType>(PTy->getElementType());
Justin Holewinskifb711152012-12-05 20:50:28 +0000[diff] [blame]1480 const std::string TypeName = STy && !STy->isLiteral() ? STy->getName() : "";
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]1481
Craig Toppere4260f92012-05-24 04:22:05 +0000[diff] [blame]1482 for (int i = 0, e = array_lengthof(specialTypes); i != e; ++i)
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]1483 if (TypeName == specialTypes[i])
1484 return true;
1485
1486 return false;
1487}
1488
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1489SDValue NVPTXTargetLowering::LowerFormalArguments(
1490 SDValue Chain, CallingConv::ID CallConv, bool isVarArg,
Andrew Trickef9de2a2013-05-25 02:42:55 +0000[diff] [blame]1491 const SmallVectorImpl<ISD::InputArg> &Ins, SDLoc dl, SelectionDAG &DAG,
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1492 SmallVectorImpl<SDValue> &InVals) const {
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]1493 MachineFunction &MF = DAG.getMachineFunction();
Micah Villmowcdfe20b2012-10-08 16:38:25 +0000[diff] [blame]1494 const DataLayout *TD = getDataLayout();
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]1495
1496 const Function *F = MF.getFunction();
Bill Wendlinge94d8432012-12-07 23:16:57 +0000[diff] [blame]1497 const AttributeSet &PAL = F->getAttributes();
Justin Holewinski44f5c602013-06-28 17:57:53 +0000[diff] [blame]1498 const TargetLowering *TLI = nvTM->getTargetLowering();
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]1499
1500 SDValue Root = DAG.getRoot();
1501 std::vector<SDValue> OutChains;
1502
1503 bool isKernel = llvm::isKernelFunction(*F);
1504 bool isABI = (nvptxSubtarget.getSmVersion() >= 20);
Justin Holewinski44f5c602013-06-28 17:57:53 +0000[diff] [blame]1505 assert(isABI && "Non-ABI compilation is not supported");
1506 if (!isABI)
1507 return Chain;
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]1508
1509 std::vector<Type *> argTypes;
1510 std::vector<const Argument *> theArgs;
1511 for (Function::const_arg_iterator I = F->arg_begin(), E = F->arg_end();
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1512 I != E; ++I) {
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]1513 theArgs.push_back(I);
1514 argTypes.push_back(I->getType());
1515 }
Justin Holewinski44f5c602013-06-28 17:57:53 +0000[diff] [blame]1516 // argTypes.size() (or theArgs.size()) and Ins.size() need not match.
1517 // Ins.size() will be larger
1518 // * if there is an aggregate argument with multiple fields (each field
1519 // showing up separately in Ins)
1520 // * if there is a vector argument with more than typical vector-length
1521 // elements (generally if more than 4) where each vector element is
1522 // individually present in Ins.
1523 // So a different index should be used for indexing into Ins.
1524 // See similar issue in LowerCall.
1525 unsigned InsIdx = 0;
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]1526
1527 int idx = 0;
Justin Holewinski44f5c602013-06-28 17:57:53 +0000[diff] [blame]1528 for (unsigned i = 0, e = theArgs.size(); i != e; ++i, ++idx, ++InsIdx) {
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]1529 Type *Ty = argTypes[i];
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]1530
1531 // If the kernel argument is image*_t or sampler_t, convert it to
1532 // a i32 constant holding the parameter position. This can later
1533 // matched in the AsmPrinter to output the correct mangled name.
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1534 if (isImageOrSamplerVal(
1535 theArgs[i],
1536 (theArgs[i]->getParent() ? theArgs[i]->getParent()->getParent()
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]1537 : nullptr))) {
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]1538 assert(isKernel && "Only kernels can have image/sampler params");
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1539 InVals.push_back(DAG.getConstant(i + 1, MVT::i32));
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]1540 continue;
1541 }
1542
1543 if (theArgs[i]->use_empty()) {
1544 // argument is dead
Justin Holewinski44f5c602013-06-28 17:57:53 +0000[diff] [blame]1545 if (Ty->isAggregateType()) {
1546 SmallVector<EVT, 16> vtparts;
1547
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]1548 ComputePTXValueVTs(*this, Ty, vtparts);
Justin Holewinski44f5c602013-06-28 17:57:53 +0000[diff] [blame]1549 assert(vtparts.size() > 0 && "empty aggregate type not expected");
1550 for (unsigned parti = 0, parte = vtparts.size(); parti != parte;
1551 ++parti) {
1552 EVT partVT = vtparts[parti];
1553 InVals.push_back(DAG.getNode(ISD::UNDEF, dl, partVT));
1554 ++InsIdx;
Justin Holewinskie9884092013-03-24 21:17:47 +0000[diff] [blame]1555 }
Justin Holewinski44f5c602013-06-28 17:57:53 +0000[diff] [blame]1556 if (vtparts.size() > 0)
1557 --InsIdx;
1558 continue;
Justin Holewinskie9884092013-03-24 21:17:47 +0000[diff] [blame]1559 }
Justin Holewinski44f5c602013-06-28 17:57:53 +0000[diff] [blame]1560 if (Ty->isVectorTy()) {
1561 EVT ObjectVT = getValueType(Ty);
1562 unsigned NumRegs = TLI->getNumRegisters(F->getContext(), ObjectVT);
1563 for (unsigned parti = 0; parti < NumRegs; ++parti) {
1564 InVals.push_back(DAG.getNode(ISD::UNDEF, dl, Ins[InsIdx].VT));
1565 ++InsIdx;
1566 }
1567 if (NumRegs > 0)
1568 --InsIdx;
1569 continue;
1570 }
1571 InVals.push_back(DAG.getNode(ISD::UNDEF, dl, Ins[InsIdx].VT));
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]1572 continue;
1573 }
1574
1575 // In the following cases, assign a node order of "idx+1"
Justin Holewinski44f5c602013-06-28 17:57:53 +0000[diff] [blame]1576 // to newly created nodes. The SDNodes for params have to
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]1577 // appear in the same order as their order of appearance
1578 // in the original function. "idx+1" holds that order.
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1579 if (PAL.hasAttribute(i + 1, Attribute::ByVal) == false) {
Justin Holewinski44f5c602013-06-28 17:57:53 +0000[diff] [blame]1580 if (Ty->isAggregateType()) {
1581 SmallVector<EVT, 16> vtparts;
1582 SmallVector<uint64_t, 16> offsets;
1583
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]1584 // NOTE: Here, we lose the ability to issue vector loads for vectors
1585 // that are a part of a struct. This should be investigated in the
1586 // future.
1587 ComputePTXValueVTs(*this, Ty, vtparts, &offsets, 0);
Justin Holewinski44f5c602013-06-28 17:57:53 +0000[diff] [blame]1588 assert(vtparts.size() > 0 && "empty aggregate type not expected");
1589 bool aggregateIsPacked = false;
1590 if (StructType *STy = llvm::dyn_cast<StructType>(Ty))
1591 aggregateIsPacked = STy->isPacked();
1592
1593 SDValue Arg = getParamSymbol(DAG, idx, getPointerTy());
1594 for (unsigned parti = 0, parte = vtparts.size(); parti != parte;
1595 ++parti) {
1596 EVT partVT = vtparts[parti];
1597 Value *srcValue = Constant::getNullValue(
1598 PointerType::get(partVT.getTypeForEVT(F->getContext()),
1599 llvm::ADDRESS_SPACE_PARAM));
1600 SDValue srcAddr =
1601 DAG.getNode(ISD::ADD, dl, getPointerTy(), Arg,
1602 DAG.getConstant(offsets[parti], getPointerTy()));
1603 unsigned partAlign =
1604 aggregateIsPacked ? 1
1605 : TD->getABITypeAlignment(
1606 partVT.getTypeForEVT(F->getContext()));
Justin Holewinskia2911282013-07-01 12:58:58 +0000[diff] [blame]1607 SDValue p;
1608 if (Ins[InsIdx].VT.getSizeInBits() > partVT.getSizeInBits()) {
1609 ISD::LoadExtType ExtOp = Ins[InsIdx].Flags.isSExt() ?
1610 ISD::SEXTLOAD : ISD::ZEXTLOAD;
1611 p = DAG.getExtLoad(ExtOp, dl, Ins[InsIdx].VT, Root, srcAddr,
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]1612 MachinePointerInfo(srcValue), partVT, false,
1613 false, partAlign);
Justin Holewinskia2911282013-07-01 12:58:58 +0000[diff] [blame]1614 } else {
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]1615 p = DAG.getLoad(partVT, dl, Root, srcAddr,
1616 MachinePointerInfo(srcValue), false, false, false,
1617 partAlign);
Justin Holewinskia2911282013-07-01 12:58:58 +0000[diff] [blame]1618 }
Justin Holewinski44f5c602013-06-28 17:57:53 +0000[diff] [blame]1619 if (p.getNode())
1620 p.getNode()->setIROrder(idx + 1);
1621 InVals.push_back(p);
1622 ++InsIdx;
Justin Holewinskie9884092013-03-24 21:17:47 +0000[diff] [blame]1623 }
Justin Holewinski44f5c602013-06-28 17:57:53 +0000[diff] [blame]1624 if (vtparts.size() > 0)
1625 --InsIdx;
Justin Holewinskie9884092013-03-24 21:17:47 +0000[diff] [blame]1626 continue;
1627 }
Justin Holewinski44f5c602013-06-28 17:57:53 +0000[diff] [blame]1628 if (Ty->isVectorTy()) {
1629 EVT ObjectVT = getValueType(Ty);
Justin Holewinskiaaaf2892013-06-25 12:22:21 +0000[diff] [blame]1630 SDValue Arg = getParamSymbol(DAG, idx, getPointerTy());
Justin Holewinski44f5c602013-06-28 17:57:53 +0000[diff] [blame]1631 unsigned NumElts = ObjectVT.getVectorNumElements();
1632 assert(TLI->getNumRegisters(F->getContext(), ObjectVT) == NumElts &&
1633 "Vector was not scalarized");
1634 unsigned Ofst = 0;
1635 EVT EltVT = ObjectVT.getVectorElementType();
1636
1637 // V1 load
1638 // f32 = load ...
1639 if (NumElts == 1) {
1640 // We only have one element, so just directly load it
1641 Value *SrcValue = Constant::getNullValue(PointerType::get(
1642 EltVT.getTypeForEVT(F->getContext()), llvm::ADDRESS_SPACE_PARAM));
1643 SDValue SrcAddr = DAG.getNode(ISD::ADD, dl, getPointerTy(), Arg,
1644 DAG.getConstant(Ofst, getPointerTy()));
1645 SDValue P = DAG.getLoad(
1646 EltVT, dl, Root, SrcAddr, MachinePointerInfo(SrcValue), false,
1647 false, true,
1648 TD->getABITypeAlignment(EltVT.getTypeForEVT(F->getContext())));
1649 if (P.getNode())
1650 P.getNode()->setIROrder(idx + 1);
1651
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]1652 if (Ins[InsIdx].VT.getSizeInBits() > EltVT.getSizeInBits())
Justin Holewinskia2911282013-07-01 12:58:58 +0000[diff] [blame]1653 P = DAG.getNode(ISD::ANY_EXTEND, dl, Ins[InsIdx].VT, P);
Justin Holewinski44f5c602013-06-28 17:57:53 +0000[diff] [blame]1654 InVals.push_back(P);
1655 Ofst += TD->getTypeAllocSize(EltVT.getTypeForEVT(F->getContext()));
1656 ++InsIdx;
1657 } else if (NumElts == 2) {
1658 // V2 load
1659 // f32,f32 = load ...
1660 EVT VecVT = EVT::getVectorVT(F->getContext(), EltVT, 2);
1661 Value *SrcValue = Constant::getNullValue(PointerType::get(
1662 VecVT.getTypeForEVT(F->getContext()), llvm::ADDRESS_SPACE_PARAM));
1663 SDValue SrcAddr = DAG.getNode(ISD::ADD, dl, getPointerTy(), Arg,
1664 DAG.getConstant(Ofst, getPointerTy()));
1665 SDValue P = DAG.getLoad(
1666 VecVT, dl, Root, SrcAddr, MachinePointerInfo(SrcValue), false,
1667 false, true,
1668 TD->getABITypeAlignment(VecVT.getTypeForEVT(F->getContext())));
1669 if (P.getNode())
1670 P.getNode()->setIROrder(idx + 1);
1671
1672 SDValue Elt0 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, P,
1673 DAG.getIntPtrConstant(0));
1674 SDValue Elt1 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, P,
1675 DAG.getIntPtrConstant(1));
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]1676
1677 if (Ins[InsIdx].VT.getSizeInBits() > EltVT.getSizeInBits()) {
Justin Holewinskia2911282013-07-01 12:58:58 +0000[diff] [blame]1678 Elt0 = DAG.getNode(ISD::ANY_EXTEND, dl, Ins[InsIdx].VT, Elt0);
1679 Elt1 = DAG.getNode(ISD::ANY_EXTEND, dl, Ins[InsIdx].VT, Elt1);
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]1680 }
1681
Justin Holewinski44f5c602013-06-28 17:57:53 +0000[diff] [blame]1682 InVals.push_back(Elt0);
1683 InVals.push_back(Elt1);
1684 Ofst += TD->getTypeAllocSize(VecVT.getTypeForEVT(F->getContext()));
1685 InsIdx += 2;
1686 } else {
1687 // V4 loads
1688 // We have at least 4 elements (<3 x Ty> expands to 4 elements) and
1689 // the
1690 // vector will be expanded to a power of 2 elements, so we know we can
1691 // always round up to the next multiple of 4 when creating the vector
1692 // loads.
1693 // e.g. 4 elem => 1 ld.v4
1694 // 6 elem => 2 ld.v4
1695 // 8 elem => 2 ld.v4
1696 // 11 elem => 3 ld.v4
1697 unsigned VecSize = 4;
1698 if (EltVT.getSizeInBits() == 64) {
1699 VecSize = 2;
1700 }
1701 EVT VecVT = EVT::getVectorVT(F->getContext(), EltVT, VecSize);
1702 for (unsigned i = 0; i < NumElts; i += VecSize) {
1703 Value *SrcValue = Constant::getNullValue(
1704 PointerType::get(VecVT.getTypeForEVT(F->getContext()),
1705 llvm::ADDRESS_SPACE_PARAM));
1706 SDValue SrcAddr =
1707 DAG.getNode(ISD::ADD, dl, getPointerTy(), Arg,
1708 DAG.getConstant(Ofst, getPointerTy()));
1709 SDValue P = DAG.getLoad(
1710 VecVT, dl, Root, SrcAddr, MachinePointerInfo(SrcValue), false,
1711 false, true,
1712 TD->getABITypeAlignment(VecVT.getTypeForEVT(F->getContext())));
1713 if (P.getNode())
1714 P.getNode()->setIROrder(idx + 1);
1715
1716 for (unsigned j = 0; j < VecSize; ++j) {
1717 if (i + j >= NumElts)
1718 break;
1719 SDValue Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, P,
1720 DAG.getIntPtrConstant(j));
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]1721 if (Ins[InsIdx].VT.getSizeInBits() > EltVT.getSizeInBits())
Justin Holewinskia2911282013-07-01 12:58:58 +0000[diff] [blame]1722 Elt = DAG.getNode(ISD::ANY_EXTEND, dl, Ins[InsIdx].VT, Elt);
Justin Holewinski44f5c602013-06-28 17:57:53 +0000[diff] [blame]1723 InVals.push_back(Elt);
1724 }
1725 Ofst += TD->getTypeAllocSize(VecVT.getTypeForEVT(F->getContext()));
Justin Holewinski44f5c602013-06-28 17:57:53 +0000[diff] [blame]1726 }
Justin Holewinski4f5bc9b2013-11-11 19:28:16 +0000[diff] [blame]1727 InsIdx += NumElts;
Justin Holewinski44f5c602013-06-28 17:57:53 +0000[diff] [blame]1728 }
1729
1730 if (NumElts > 0)
1731 --InsIdx;
1732 continue;
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]1733 }
Justin Holewinski44f5c602013-06-28 17:57:53 +0000[diff] [blame]1734 // A plain scalar.
1735 EVT ObjectVT = getValueType(Ty);
Justin Holewinski44f5c602013-06-28 17:57:53 +0000[diff] [blame]1736 // If ABI, load from the param symbol
1737 SDValue Arg = getParamSymbol(DAG, idx, getPointerTy());
1738 Value *srcValue = Constant::getNullValue(PointerType::get(
1739 ObjectVT.getTypeForEVT(F->getContext()), llvm::ADDRESS_SPACE_PARAM));
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]1740 SDValue p;
Justin Holewinskia2911282013-07-01 12:58:58 +0000[diff] [blame]1741 if (ObjectVT.getSizeInBits() < Ins[InsIdx].VT.getSizeInBits()) {
1742 ISD::LoadExtType ExtOp = Ins[InsIdx].Flags.isSExt() ?
1743 ISD::SEXTLOAD : ISD::ZEXTLOAD;
1744 p = DAG.getExtLoad(ExtOp, dl, Ins[InsIdx].VT, Root, Arg,
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]1745 MachinePointerInfo(srcValue), ObjectVT, false, false,
Justin Holewinskia2911282013-07-01 12:58:58 +0000[diff] [blame]1746 TD->getABITypeAlignment(ObjectVT.getTypeForEVT(F->getContext())));
1747 } else {
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]1748 p = DAG.getLoad(Ins[InsIdx].VT, dl, Root, Arg,
1749 MachinePointerInfo(srcValue), false, false, false,
Justin Holewinskia2911282013-07-01 12:58:58 +0000[diff] [blame]1750 TD->getABITypeAlignment(ObjectVT.getTypeForEVT(F->getContext())));
1751 }
Justin Holewinski44f5c602013-06-28 17:57:53 +0000[diff] [blame]1752 if (p.getNode())
1753 p.getNode()->setIROrder(idx + 1);
1754 InVals.push_back(p);
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]1755 continue;
1756 }
1757
1758 // Param has ByVal attribute
Justin Holewinski44f5c602013-06-28 17:57:53 +0000[diff] [blame]1759 // Return MoveParam(param symbol).
1760 // Ideally, the param symbol can be returned directly,
1761 // but when SDNode builder decides to use it in a CopyToReg(),
1762 // machine instruction fails because TargetExternalSymbol
1763 // (not lowered) is target dependent, and CopyToReg assumes
1764 // the source is lowered.
1765 EVT ObjectVT = getValueType(Ty);
1766 assert(ObjectVT == Ins[InsIdx].VT &&
1767 "Ins type did not match function type");
1768 SDValue Arg = getParamSymbol(DAG, idx, getPointerTy());
1769 SDValue p = DAG.getNode(NVPTXISD::MoveParam, dl, ObjectVT, Arg);
1770 if (p.getNode())
1771 p.getNode()->setIROrder(idx + 1);
1772 if (isKernel)
1773 InVals.push_back(p);
1774 else {
1775 SDValue p2 = DAG.getNode(
1776 ISD::INTRINSIC_WO_CHAIN, dl, ObjectVT,
1777 DAG.getConstant(Intrinsic::nvvm_ptr_local_to_gen, MVT::i32), p);
1778 InVals.push_back(p2);
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]1779 }
1780 }
1781
1782 // Clang will check explicit VarArg and issue error if any. However, Clang
1783 // will let code with
Justin Holewinski44f5c602013-06-28 17:57:53 +0000[diff] [blame]1784 // implicit var arg like f() pass. See bug 617733.
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]1785 // We treat this case as if the arg list is empty.
Justin Holewinski44f5c602013-06-28 17:57:53 +0000[diff] [blame]1786 // if (F.isVarArg()) {
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]1787 // assert(0 && "VarArg not supported yet!");
1788 //}
1789
1790 if (!OutChains.empty())
Craig Topper48d114b2014-04-26 18:35:24 +0000[diff] [blame]1791 DAG.setRoot(DAG.getNode(ISD::TokenFactor, dl, MVT::Other, OutChains));
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]1792
1793 return Chain;
1794}
1795
Justin Holewinski44f5c602013-06-28 17:57:53 +0000[diff] [blame]1796
Justin Holewinski120baee2013-06-28 17:57:55 +0000[diff] [blame]1797SDValue
1798NVPTXTargetLowering::LowerReturn(SDValue Chain, CallingConv::ID CallConv,
1799 bool isVarArg,
1800 const SmallVectorImpl<ISD::OutputArg> &Outs,
1801 const SmallVectorImpl<SDValue> &OutVals,
1802 SDLoc dl, SelectionDAG &DAG) const {
1803 MachineFunction &MF = DAG.getMachineFunction();
1804 const Function *F = MF.getFunction();
Justin Holewinskie04e4bd2013-06-28 17:58:10 +0000[diff] [blame]1805 Type *RetTy = F->getReturnType();
Justin Holewinski120baee2013-06-28 17:57:55 +0000[diff] [blame]1806 const DataLayout *TD = getDataLayout();
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]1807
1808 bool isABI = (nvptxSubtarget.getSmVersion() >= 20);
Justin Holewinski120baee2013-06-28 17:57:55 +0000[diff] [blame]1809 assert(isABI && "Non-ABI compilation is not supported");
1810 if (!isABI)
1811 return Chain;
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]1812
Justin Holewinskie04e4bd2013-06-28 17:58:10 +0000[diff] [blame]1813 if (VectorType *VTy = dyn_cast<VectorType>(RetTy)) {
Justin Holewinski120baee2013-06-28 17:57:55 +0000[diff] [blame]1814 // If we have a vector type, the OutVals array will be the scalarized
1815 // components and we have combine them into 1 or more vector stores.
1816 unsigned NumElts = VTy->getNumElements();
1817 assert(NumElts == Outs.size() && "Bad scalarization of return value");
1818
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]1819 // const_cast can be removed in later LLVM versions
Justin Holewinskie04e4bd2013-06-28 17:58:10 +0000[diff] [blame]1820 EVT EltVT = getValueType(RetTy).getVectorElementType();
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]1821 bool NeedExtend = false;
1822 if (EltVT.getSizeInBits() < 16)
1823 NeedExtend = true;
1824
Justin Holewinski120baee2013-06-28 17:57:55 +0000[diff] [blame]1825 // V1 store
1826 if (NumElts == 1) {
1827 SDValue StoreVal = OutVals[0];
1828 // We only have one element, so just directly store it
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]1829 if (NeedExtend)
1830 StoreVal = DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i16, StoreVal);
1831 SDValue Ops[] = { Chain, DAG.getConstant(0, MVT::i32), StoreVal };
1832 Chain = DAG.getMemIntrinsicNode(NVPTXISD::StoreRetval, dl,
Craig Topper206fcd42014-04-26 19:29:41 +0000[diff] [blame]1833 DAG.getVTList(MVT::Other), Ops,
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]1834 EltVT, MachinePointerInfo());
1835
Justin Holewinski120baee2013-06-28 17:57:55 +0000[diff] [blame]1836 } else if (NumElts == 2) {
1837 // V2 store
1838 SDValue StoreVal0 = OutVals[0];
1839 SDValue StoreVal1 = OutVals[1];
1840
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]1841 if (NeedExtend) {
1842 StoreVal0 = DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i16, StoreVal0);
1843 StoreVal1 = DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i16, StoreVal1);
Justin Holewinski120baee2013-06-28 17:57:55 +0000[diff] [blame]1844 }
1845
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]1846 SDValue Ops[] = { Chain, DAG.getConstant(0, MVT::i32), StoreVal0,
1847 StoreVal1 };
1848 Chain = DAG.getMemIntrinsicNode(NVPTXISD::StoreRetvalV2, dl,
Craig Topper206fcd42014-04-26 19:29:41 +0000[diff] [blame]1849 DAG.getVTList(MVT::Other), Ops,
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]1850 EltVT, MachinePointerInfo());
Justin Holewinski120baee2013-06-28 17:57:55 +0000[diff] [blame]1851 } else {
1852 // V4 stores
1853 // We have at least 4 elements (<3 x Ty> expands to 4 elements) and the
1854 // vector will be expanded to a power of 2 elements, so we know we can
1855 // always round up to the next multiple of 4 when creating the vector
1856 // stores.
1857 // e.g. 4 elem => 1 st.v4
1858 // 6 elem => 2 st.v4
1859 // 8 elem => 2 st.v4
1860 // 11 elem => 3 st.v4
1861
1862 unsigned VecSize = 4;
1863 if (OutVals[0].getValueType().getSizeInBits() == 64)
1864 VecSize = 2;
1865
1866 unsigned Offset = 0;
1867
1868 EVT VecVT =
1869 EVT::getVectorVT(F->getContext(), OutVals[0].getValueType(), VecSize);
1870 unsigned PerStoreOffset =
1871 TD->getTypeAllocSize(VecVT.getTypeForEVT(F->getContext()));
1872
Justin Holewinski120baee2013-06-28 17:57:55 +0000[diff] [blame]1873 for (unsigned i = 0; i < NumElts; i += VecSize) {
1874 // Get values
1875 SDValue StoreVal;
1876 SmallVector<SDValue, 8> Ops;
1877 Ops.push_back(Chain);
1878 Ops.push_back(DAG.getConstant(Offset, MVT::i32));
1879 unsigned Opc = NVPTXISD::StoreRetvalV2;
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]1880 EVT ExtendedVT = (NeedExtend) ? MVT::i16 : OutVals[0].getValueType();
Justin Holewinski120baee2013-06-28 17:57:55 +0000[diff] [blame]1881
1882 StoreVal = OutVals[i];
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]1883 if (NeedExtend)
1884 StoreVal = DAG.getNode(ISD::ZERO_EXTEND, dl, ExtendedVT, StoreVal);
Justin Holewinski120baee2013-06-28 17:57:55 +0000[diff] [blame]1885 Ops.push_back(StoreVal);
1886
1887 if (i + 1 < NumElts) {
1888 StoreVal = OutVals[i + 1];
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]1889 if (NeedExtend)
1890 StoreVal = DAG.getNode(ISD::ZERO_EXTEND, dl, ExtendedVT, StoreVal);
Justin Holewinski120baee2013-06-28 17:57:55 +0000[diff] [blame]1891 } else {
1892 StoreVal = DAG.getUNDEF(ExtendedVT);
1893 }
1894 Ops.push_back(StoreVal);
1895
1896 if (VecSize == 4) {
1897 Opc = NVPTXISD::StoreRetvalV4;
1898 if (i + 2 < NumElts) {
1899 StoreVal = OutVals[i + 2];
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]1900 if (NeedExtend)
1901 StoreVal =
1902 DAG.getNode(ISD::ZERO_EXTEND, dl, ExtendedVT, StoreVal);
Justin Holewinski120baee2013-06-28 17:57:55 +0000[diff] [blame]1903 } else {
1904 StoreVal = DAG.getUNDEF(ExtendedVT);
1905 }
1906 Ops.push_back(StoreVal);
1907
1908 if (i + 3 < NumElts) {
1909 StoreVal = OutVals[i + 3];
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]1910 if (NeedExtend)
1911 StoreVal =
1912 DAG.getNode(ISD::ZERO_EXTEND, dl, ExtendedVT, StoreVal);
Justin Holewinski120baee2013-06-28 17:57:55 +0000[diff] [blame]1913 } else {
1914 StoreVal = DAG.getUNDEF(ExtendedVT);
1915 }
1916 Ops.push_back(StoreVal);
1917 }
1918
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]1919 // Chain = DAG.getNode(Opc, dl, MVT::Other, &Ops[0], Ops.size());
1920 Chain =
Craig Topper206fcd42014-04-26 19:29:41 +0000[diff] [blame]1921 DAG.getMemIntrinsicNode(Opc, dl, DAG.getVTList(MVT::Other), Ops,
1922 EltVT, MachinePointerInfo());
Justin Holewinski120baee2013-06-28 17:57:55 +0000[diff] [blame]1923 Offset += PerStoreOffset;
1924 }
1925 }
1926 } else {
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]1927 SmallVector<EVT, 16> ValVTs;
1928 // const_cast is necessary since we are still using an LLVM version from
1929 // before the type system re-write.
Justin Holewinskie04e4bd2013-06-28 17:58:10 +0000[diff] [blame]1930 ComputePTXValueVTs(*this, RetTy, ValVTs);
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]1931 assert(ValVTs.size() == OutVals.size() && "Bad return value decomposition");
1932
Justin Holewinskie04e4bd2013-06-28 17:58:10 +0000[diff] [blame]1933 unsigned SizeSoFar = 0;
Justin Holewinski120baee2013-06-28 17:57:55 +0000[diff] [blame]1934 for (unsigned i = 0, e = Outs.size(); i != e; ++i) {
1935 SDValue theVal = OutVals[i];
Justin Holewinskie04e4bd2013-06-28 17:58:10 +0000[diff] [blame]1936 EVT TheValType = theVal.getValueType();
Justin Holewinski120baee2013-06-28 17:57:55 +0000[diff] [blame]1937 unsigned numElems = 1;
Justin Holewinskie04e4bd2013-06-28 17:58:10 +0000[diff] [blame]1938 if (TheValType.isVector())
1939 numElems = TheValType.getVectorNumElements();
Justin Holewinski120baee2013-06-28 17:57:55 +0000[diff] [blame]1940 for (unsigned j = 0, je = numElems; j != je; ++j) {
Justin Holewinskie04e4bd2013-06-28 17:58:10 +0000[diff] [blame]1941 SDValue TmpVal = theVal;
1942 if (TheValType.isVector())
1943 TmpVal = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl,
1944 TheValType.getVectorElementType(), TmpVal,
Justin Holewinski120baee2013-06-28 17:57:55 +0000[diff] [blame]1945 DAG.getIntPtrConstant(j));
Justin Holewinskie04e4bd2013-06-28 17:58:10 +0000[diff] [blame]1946 EVT TheStoreType = ValVTs[i];
1947 if (RetTy->isIntegerTy() &&
1948 TD->getTypeAllocSizeInBits(RetTy) < 32) {
1949 // The following zero-extension is for integer types only, and
1950 // specifically not for aggregates.
1951 TmpVal = DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i32, TmpVal);
1952 TheStoreType = MVT::i32;
1953 }
1954 else if (TmpVal.getValueType().getSizeInBits() < 16)
1955 TmpVal = DAG.getNode(ISD::ANY_EXTEND, dl, MVT::i16, TmpVal);
1956
1957 SDValue Ops[] = { Chain, DAG.getConstant(SizeSoFar, MVT::i32), TmpVal };
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]1958 Chain = DAG.getMemIntrinsicNode(NVPTXISD::StoreRetval, dl,
Craig Topper206fcd42014-04-26 19:29:41 +0000[diff] [blame]1959 DAG.getVTList(MVT::Other), Ops,
1960 TheStoreType,
Justin Holewinskie04e4bd2013-06-28 17:58:10 +0000[diff] [blame]1961 MachinePointerInfo());
1962 if(TheValType.isVector())
1963 SizeSoFar +=
1964 TheStoreType.getVectorElementType().getStoreSizeInBits() / 8;
Justin Holewinski120baee2013-06-28 17:57:55 +0000[diff] [blame]1965 else
Justin Holewinskie04e4bd2013-06-28 17:58:10 +0000[diff] [blame]1966 SizeSoFar += TheStoreType.getStoreSizeInBits()/8;
Justin Holewinski120baee2013-06-28 17:57:55 +0000[diff] [blame]1967 }
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]1968 }
1969 }
1970
1971 return DAG.getNode(NVPTXISD::RET_FLAG, dl, MVT::Other, Chain);
1972}
1973
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]1974
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1975void NVPTXTargetLowering::LowerAsmOperandForConstraint(
1976 SDValue Op, std::string &Constraint, std::vector<SDValue> &Ops,
1977 SelectionDAG &DAG) const {
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]1978 if (Constraint.length() > 1)
1979 return;
1980 else
1981 TargetLowering::LowerAsmOperandForConstraint(Op, Constraint, Ops, DAG);
1982}
1983
1984// NVPTX suuport vector of legal types of any length in Intrinsics because the
1985// NVPTX specific type legalizer
1986// will legalize them to the PTX supported length.
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1987bool NVPTXTargetLowering::isTypeSupportedInIntrinsic(MVT VT) const {
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]1988 if (isTypeLegal(VT))
1989 return true;
1990 if (VT.isVector()) {
1991 MVT eVT = VT.getVectorElementType();
1992 if (isTypeLegal(eVT))
1993 return true;
1994 }
1995 return false;
1996}
1997
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]1998static unsigned getOpcForTextureInstr(unsigned Intrinsic) {
1999 switch (Intrinsic) {
2000 default:
2001 return 0;
2002
2003 case Intrinsic::nvvm_tex_1d_v4f32_i32:
2004 return NVPTXISD::Tex1DFloatI32;
2005 case Intrinsic::nvvm_tex_1d_v4f32_f32:
2006 return NVPTXISD::Tex1DFloatFloat;
2007 case Intrinsic::nvvm_tex_1d_level_v4f32_f32:
2008 return NVPTXISD::Tex1DFloatFloatLevel;
2009 case Intrinsic::nvvm_tex_1d_grad_v4f32_f32:
2010 return NVPTXISD::Tex1DFloatFloatGrad;
2011 case Intrinsic::nvvm_tex_1d_v4i32_i32:
2012 return NVPTXISD::Tex1DI32I32;
2013 case Intrinsic::nvvm_tex_1d_v4i32_f32:
2014 return NVPTXISD::Tex1DI32Float;
2015 case Intrinsic::nvvm_tex_1d_level_v4i32_f32:
2016 return NVPTXISD::Tex1DI32FloatLevel;
2017 case Intrinsic::nvvm_tex_1d_grad_v4i32_f32:
2018 return NVPTXISD::Tex1DI32FloatGrad;
2019
2020 case Intrinsic::nvvm_tex_1d_array_v4f32_i32:
2021 return NVPTXISD::Tex1DArrayFloatI32;
2022 case Intrinsic::nvvm_tex_1d_array_v4f32_f32:
2023 return NVPTXISD::Tex1DArrayFloatFloat;
2024 case Intrinsic::nvvm_tex_1d_array_level_v4f32_f32:
2025 return NVPTXISD::Tex1DArrayFloatFloatLevel;
2026 case Intrinsic::nvvm_tex_1d_array_grad_v4f32_f32:
2027 return NVPTXISD::Tex1DArrayFloatFloatGrad;
2028 case Intrinsic::nvvm_tex_1d_array_v4i32_i32:
2029 return NVPTXISD::Tex1DArrayI32I32;
2030 case Intrinsic::nvvm_tex_1d_array_v4i32_f32:
2031 return NVPTXISD::Tex1DArrayI32Float;
2032 case Intrinsic::nvvm_tex_1d_array_level_v4i32_f32:
2033 return NVPTXISD::Tex1DArrayI32FloatLevel;
2034 case Intrinsic::nvvm_tex_1d_array_grad_v4i32_f32:
2035 return NVPTXISD::Tex1DArrayI32FloatGrad;
2036
2037 case Intrinsic::nvvm_tex_2d_v4f32_i32:
2038 return NVPTXISD::Tex2DFloatI32;
2039 case Intrinsic::nvvm_tex_2d_v4f32_f32:
2040 return NVPTXISD::Tex2DFloatFloat;
2041 case Intrinsic::nvvm_tex_2d_level_v4f32_f32:
2042 return NVPTXISD::Tex2DFloatFloatLevel;
2043 case Intrinsic::nvvm_tex_2d_grad_v4f32_f32:
2044 return NVPTXISD::Tex2DFloatFloatGrad;
2045 case Intrinsic::nvvm_tex_2d_v4i32_i32:
2046 return NVPTXISD::Tex2DI32I32;
2047 case Intrinsic::nvvm_tex_2d_v4i32_f32:
2048 return NVPTXISD::Tex2DI32Float;
2049 case Intrinsic::nvvm_tex_2d_level_v4i32_f32:
2050 return NVPTXISD::Tex2DI32FloatLevel;
2051 case Intrinsic::nvvm_tex_2d_grad_v4i32_f32:
2052 return NVPTXISD::Tex2DI32FloatGrad;
2053
2054 case Intrinsic::nvvm_tex_2d_array_v4f32_i32:
2055 return NVPTXISD::Tex2DArrayFloatI32;
2056 case Intrinsic::nvvm_tex_2d_array_v4f32_f32:
2057 return NVPTXISD::Tex2DArrayFloatFloat;
2058 case Intrinsic::nvvm_tex_2d_array_level_v4f32_f32:
2059 return NVPTXISD::Tex2DArrayFloatFloatLevel;
2060 case Intrinsic::nvvm_tex_2d_array_grad_v4f32_f32:
2061 return NVPTXISD::Tex2DArrayFloatFloatGrad;
2062 case Intrinsic::nvvm_tex_2d_array_v4i32_i32:
2063 return NVPTXISD::Tex2DArrayI32I32;
2064 case Intrinsic::nvvm_tex_2d_array_v4i32_f32:
2065 return NVPTXISD::Tex2DArrayI32Float;
2066 case Intrinsic::nvvm_tex_2d_array_level_v4i32_f32:
2067 return NVPTXISD::Tex2DArrayI32FloatLevel;
2068 case Intrinsic::nvvm_tex_2d_array_grad_v4i32_f32:
2069 return NVPTXISD::Tex2DArrayI32FloatGrad;
2070
2071 case Intrinsic::nvvm_tex_3d_v4f32_i32:
2072 return NVPTXISD::Tex3DFloatI32;
2073 case Intrinsic::nvvm_tex_3d_v4f32_f32:
2074 return NVPTXISD::Tex3DFloatFloat;
2075 case Intrinsic::nvvm_tex_3d_level_v4f32_f32:
2076 return NVPTXISD::Tex3DFloatFloatLevel;
2077 case Intrinsic::nvvm_tex_3d_grad_v4f32_f32:
2078 return NVPTXISD::Tex3DFloatFloatGrad;
2079 case Intrinsic::nvvm_tex_3d_v4i32_i32:
2080 return NVPTXISD::Tex3DI32I32;
2081 case Intrinsic::nvvm_tex_3d_v4i32_f32:
2082 return NVPTXISD::Tex3DI32Float;
2083 case Intrinsic::nvvm_tex_3d_level_v4i32_f32:
2084 return NVPTXISD::Tex3DI32FloatLevel;
2085 case Intrinsic::nvvm_tex_3d_grad_v4i32_f32:
2086 return NVPTXISD::Tex3DI32FloatGrad;
2087 }
2088}
2089
2090static unsigned getOpcForSurfaceInstr(unsigned Intrinsic) {
2091 switch (Intrinsic) {
2092 default:
2093 return 0;
2094 case Intrinsic::nvvm_suld_1d_i8_trap:
2095 return NVPTXISD::Suld1DI8Trap;
2096 case Intrinsic::nvvm_suld_1d_i16_trap:
2097 return NVPTXISD::Suld1DI16Trap;
2098 case Intrinsic::nvvm_suld_1d_i32_trap:
2099 return NVPTXISD::Suld1DI32Trap;
2100 case Intrinsic::nvvm_suld_1d_v2i8_trap:
2101 return NVPTXISD::Suld1DV2I8Trap;
2102 case Intrinsic::nvvm_suld_1d_v2i16_trap:
2103 return NVPTXISD::Suld1DV2I16Trap;
2104 case Intrinsic::nvvm_suld_1d_v2i32_trap:
2105 return NVPTXISD::Suld1DV2I32Trap;
2106 case Intrinsic::nvvm_suld_1d_v4i8_trap:
2107 return NVPTXISD::Suld1DV4I8Trap;
2108 case Intrinsic::nvvm_suld_1d_v4i16_trap:
2109 return NVPTXISD::Suld1DV4I16Trap;
2110 case Intrinsic::nvvm_suld_1d_v4i32_trap:
2111 return NVPTXISD::Suld1DV4I32Trap;
2112 case Intrinsic::nvvm_suld_1d_array_i8_trap:
2113 return NVPTXISD::Suld1DArrayI8Trap;
2114 case Intrinsic::nvvm_suld_1d_array_i16_trap:
2115 return NVPTXISD::Suld1DArrayI16Trap;
2116 case Intrinsic::nvvm_suld_1d_array_i32_trap:
2117 return NVPTXISD::Suld1DArrayI32Trap;
2118 case Intrinsic::nvvm_suld_1d_array_v2i8_trap:
2119 return NVPTXISD::Suld1DArrayV2I8Trap;
2120 case Intrinsic::nvvm_suld_1d_array_v2i16_trap:
2121 return NVPTXISD::Suld1DArrayV2I16Trap;
2122 case Intrinsic::nvvm_suld_1d_array_v2i32_trap:
2123 return NVPTXISD::Suld1DArrayV2I32Trap;
2124 case Intrinsic::nvvm_suld_1d_array_v4i8_trap:
2125 return NVPTXISD::Suld1DArrayV4I8Trap;
2126 case Intrinsic::nvvm_suld_1d_array_v4i16_trap:
2127 return NVPTXISD::Suld1DArrayV4I16Trap;
2128 case Intrinsic::nvvm_suld_1d_array_v4i32_trap:
2129 return NVPTXISD::Suld1DArrayV4I32Trap;
2130 case Intrinsic::nvvm_suld_2d_i8_trap:
2131 return NVPTXISD::Suld2DI8Trap;
2132 case Intrinsic::nvvm_suld_2d_i16_trap:
2133 return NVPTXISD::Suld2DI16Trap;
2134 case Intrinsic::nvvm_suld_2d_i32_trap:
2135 return NVPTXISD::Suld2DI32Trap;
2136 case Intrinsic::nvvm_suld_2d_v2i8_trap:
2137 return NVPTXISD::Suld2DV2I8Trap;
2138 case Intrinsic::nvvm_suld_2d_v2i16_trap:
2139 return NVPTXISD::Suld2DV2I16Trap;
2140 case Intrinsic::nvvm_suld_2d_v2i32_trap:
2141 return NVPTXISD::Suld2DV2I32Trap;
2142 case Intrinsic::nvvm_suld_2d_v4i8_trap:
2143 return NVPTXISD::Suld2DV4I8Trap;
2144 case Intrinsic::nvvm_suld_2d_v4i16_trap:
2145 return NVPTXISD::Suld2DV4I16Trap;
2146 case Intrinsic::nvvm_suld_2d_v4i32_trap:
2147 return NVPTXISD::Suld2DV4I32Trap;
2148 case Intrinsic::nvvm_suld_2d_array_i8_trap:
2149 return NVPTXISD::Suld2DArrayI8Trap;
2150 case Intrinsic::nvvm_suld_2d_array_i16_trap:
2151 return NVPTXISD::Suld2DArrayI16Trap;
2152 case Intrinsic::nvvm_suld_2d_array_i32_trap:
2153 return NVPTXISD::Suld2DArrayI32Trap;
2154 case Intrinsic::nvvm_suld_2d_array_v2i8_trap:
2155 return NVPTXISD::Suld2DArrayV2I8Trap;
2156 case Intrinsic::nvvm_suld_2d_array_v2i16_trap:
2157 return NVPTXISD::Suld2DArrayV2I16Trap;
2158 case Intrinsic::nvvm_suld_2d_array_v2i32_trap:
2159 return NVPTXISD::Suld2DArrayV2I32Trap;
2160 case Intrinsic::nvvm_suld_2d_array_v4i8_trap:
2161 return NVPTXISD::Suld2DArrayV4I8Trap;
2162 case Intrinsic::nvvm_suld_2d_array_v4i16_trap:
2163 return NVPTXISD::Suld2DArrayV4I16Trap;
2164 case Intrinsic::nvvm_suld_2d_array_v4i32_trap:
2165 return NVPTXISD::Suld2DArrayV4I32Trap;
2166 case Intrinsic::nvvm_suld_3d_i8_trap:
2167 return NVPTXISD::Suld3DI8Trap;
2168 case Intrinsic::nvvm_suld_3d_i16_trap:
2169 return NVPTXISD::Suld3DI16Trap;
2170 case Intrinsic::nvvm_suld_3d_i32_trap:
2171 return NVPTXISD::Suld3DI32Trap;
2172 case Intrinsic::nvvm_suld_3d_v2i8_trap:
2173 return NVPTXISD::Suld3DV2I8Trap;
2174 case Intrinsic::nvvm_suld_3d_v2i16_trap:
2175 return NVPTXISD::Suld3DV2I16Trap;
2176 case Intrinsic::nvvm_suld_3d_v2i32_trap:
2177 return NVPTXISD::Suld3DV2I32Trap;
2178 case Intrinsic::nvvm_suld_3d_v4i8_trap:
2179 return NVPTXISD::Suld3DV4I8Trap;
2180 case Intrinsic::nvvm_suld_3d_v4i16_trap:
2181 return NVPTXISD::Suld3DV4I16Trap;
2182 case Intrinsic::nvvm_suld_3d_v4i32_trap:
2183 return NVPTXISD::Suld3DV4I32Trap;
2184 }
2185}
2186
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]2187// llvm.ptx.memcpy.const and llvm.ptx.memmove.const need to be modeled as
2188// TgtMemIntrinsic
2189// because we need the information that is only available in the "Value" type
2190// of destination
2191// pointer. In particular, the address space information.
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2192bool NVPTXTargetLowering::getTgtMemIntrinsic(
2193 IntrinsicInfo &Info, const CallInst &I, unsigned Intrinsic) const {
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]2194 switch (Intrinsic) {
2195 default:
2196 return false;
2197
2198 case Intrinsic::nvvm_atomic_load_add_f32:
2199 Info.opc = ISD::INTRINSIC_W_CHAIN;
2200 Info.memVT = MVT::f32;
2201 Info.ptrVal = I.getArgOperand(0);
2202 Info.offset = 0;
2203 Info.vol = 0;
2204 Info.readMem = true;
2205 Info.writeMem = true;
2206 Info.align = 0;
2207 return true;
2208
2209 case Intrinsic::nvvm_atomic_load_inc_32:
2210 case Intrinsic::nvvm_atomic_load_dec_32:
2211 Info.opc = ISD::INTRINSIC_W_CHAIN;
2212 Info.memVT = MVT::i32;
2213 Info.ptrVal = I.getArgOperand(0);
2214 Info.offset = 0;
2215 Info.vol = 0;
2216 Info.readMem = true;
2217 Info.writeMem = true;
2218 Info.align = 0;
2219 return true;
2220
2221 case Intrinsic::nvvm_ldu_global_i:
2222 case Intrinsic::nvvm_ldu_global_f:
2223 case Intrinsic::nvvm_ldu_global_p:
2224
2225 Info.opc = ISD::INTRINSIC_W_CHAIN;
2226 if (Intrinsic == Intrinsic::nvvm_ldu_global_i)
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]2227 Info.memVT = getValueType(I.getType());
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]2228 else if (Intrinsic == Intrinsic::nvvm_ldu_global_p)
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]2229 Info.memVT = getValueType(I.getType());
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]2230 else
2231 Info.memVT = MVT::f32;
2232 Info.ptrVal = I.getArgOperand(0);
2233 Info.offset = 0;
2234 Info.vol = 0;
2235 Info.readMem = true;
2236 Info.writeMem = false;
2237 Info.align = 0;
2238 return true;
2239
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]2240 case Intrinsic::nvvm_tex_1d_v4f32_i32:
2241 case Intrinsic::nvvm_tex_1d_v4f32_f32:
2242 case Intrinsic::nvvm_tex_1d_level_v4f32_f32:
2243 case Intrinsic::nvvm_tex_1d_grad_v4f32_f32:
2244 case Intrinsic::nvvm_tex_1d_array_v4f32_i32:
2245 case Intrinsic::nvvm_tex_1d_array_v4f32_f32:
2246 case Intrinsic::nvvm_tex_1d_array_level_v4f32_f32:
2247 case Intrinsic::nvvm_tex_1d_array_grad_v4f32_f32:
2248 case Intrinsic::nvvm_tex_2d_v4f32_i32:
2249 case Intrinsic::nvvm_tex_2d_v4f32_f32:
2250 case Intrinsic::nvvm_tex_2d_level_v4f32_f32:
2251 case Intrinsic::nvvm_tex_2d_grad_v4f32_f32:
2252 case Intrinsic::nvvm_tex_2d_array_v4f32_i32:
2253 case Intrinsic::nvvm_tex_2d_array_v4f32_f32:
2254 case Intrinsic::nvvm_tex_2d_array_level_v4f32_f32:
2255 case Intrinsic::nvvm_tex_2d_array_grad_v4f32_f32:
2256 case Intrinsic::nvvm_tex_3d_v4f32_i32:
2257 case Intrinsic::nvvm_tex_3d_v4f32_f32:
2258 case Intrinsic::nvvm_tex_3d_level_v4f32_f32:
2259 case Intrinsic::nvvm_tex_3d_grad_v4f32_f32: {
2260 Info.opc = getOpcForTextureInstr(Intrinsic);
2261 Info.memVT = MVT::f32;
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2262 Info.ptrVal = nullptr;
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]2263 Info.offset = 0;
2264 Info.vol = 0;
2265 Info.readMem = true;
2266 Info.writeMem = false;
2267 Info.align = 16;
2268 return true;
2269 }
2270 case Intrinsic::nvvm_tex_1d_v4i32_i32:
2271 case Intrinsic::nvvm_tex_1d_v4i32_f32:
2272 case Intrinsic::nvvm_tex_1d_level_v4i32_f32:
2273 case Intrinsic::nvvm_tex_1d_grad_v4i32_f32:
2274 case Intrinsic::nvvm_tex_1d_array_v4i32_i32:
2275 case Intrinsic::nvvm_tex_1d_array_v4i32_f32:
2276 case Intrinsic::nvvm_tex_1d_array_level_v4i32_f32:
2277 case Intrinsic::nvvm_tex_1d_array_grad_v4i32_f32:
2278 case Intrinsic::nvvm_tex_2d_v4i32_i32:
2279 case Intrinsic::nvvm_tex_2d_v4i32_f32:
2280 case Intrinsic::nvvm_tex_2d_level_v4i32_f32:
2281 case Intrinsic::nvvm_tex_2d_grad_v4i32_f32:
2282 case Intrinsic::nvvm_tex_2d_array_v4i32_i32:
2283 case Intrinsic::nvvm_tex_2d_array_v4i32_f32:
2284 case Intrinsic::nvvm_tex_2d_array_level_v4i32_f32:
2285 case Intrinsic::nvvm_tex_2d_array_grad_v4i32_f32:
2286 case Intrinsic::nvvm_tex_3d_v4i32_i32:
2287 case Intrinsic::nvvm_tex_3d_v4i32_f32:
2288 case Intrinsic::nvvm_tex_3d_level_v4i32_f32:
2289 case Intrinsic::nvvm_tex_3d_grad_v4i32_f32: {
2290 Info.opc = getOpcForTextureInstr(Intrinsic);
2291 Info.memVT = MVT::i32;
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2292 Info.ptrVal = nullptr;
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]2293 Info.offset = 0;
2294 Info.vol = 0;
2295 Info.readMem = true;
2296 Info.writeMem = false;
2297 Info.align = 16;
2298 return true;
2299 }
2300 case Intrinsic::nvvm_suld_1d_i8_trap:
2301 case Intrinsic::nvvm_suld_1d_v2i8_trap:
2302 case Intrinsic::nvvm_suld_1d_v4i8_trap:
2303 case Intrinsic::nvvm_suld_1d_array_i8_trap:
2304 case Intrinsic::nvvm_suld_1d_array_v2i8_trap:
2305 case Intrinsic::nvvm_suld_1d_array_v4i8_trap:
2306 case Intrinsic::nvvm_suld_2d_i8_trap:
2307 case Intrinsic::nvvm_suld_2d_v2i8_trap:
2308 case Intrinsic::nvvm_suld_2d_v4i8_trap:
2309 case Intrinsic::nvvm_suld_2d_array_i8_trap:
2310 case Intrinsic::nvvm_suld_2d_array_v2i8_trap:
2311 case Intrinsic::nvvm_suld_2d_array_v4i8_trap:
2312 case Intrinsic::nvvm_suld_3d_i8_trap:
2313 case Intrinsic::nvvm_suld_3d_v2i8_trap:
2314 case Intrinsic::nvvm_suld_3d_v4i8_trap: {
2315 Info.opc = getOpcForSurfaceInstr(Intrinsic);
2316 Info.memVT = MVT::i8;
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2317 Info.ptrVal = nullptr;
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]2318 Info.offset = 0;
2319 Info.vol = 0;
2320 Info.readMem = true;
2321 Info.writeMem = false;
2322 Info.align = 16;
2323 return true;
2324 }
2325 case Intrinsic::nvvm_suld_1d_i16_trap:
2326 case Intrinsic::nvvm_suld_1d_v2i16_trap:
2327 case Intrinsic::nvvm_suld_1d_v4i16_trap:
2328 case Intrinsic::nvvm_suld_1d_array_i16_trap:
2329 case Intrinsic::nvvm_suld_1d_array_v2i16_trap:
2330 case Intrinsic::nvvm_suld_1d_array_v4i16_trap:
2331 case Intrinsic::nvvm_suld_2d_i16_trap:
2332 case Intrinsic::nvvm_suld_2d_v2i16_trap:
2333 case Intrinsic::nvvm_suld_2d_v4i16_trap:
2334 case Intrinsic::nvvm_suld_2d_array_i16_trap:
2335 case Intrinsic::nvvm_suld_2d_array_v2i16_trap:
2336 case Intrinsic::nvvm_suld_2d_array_v4i16_trap:
2337 case Intrinsic::nvvm_suld_3d_i16_trap:
2338 case Intrinsic::nvvm_suld_3d_v2i16_trap:
2339 case Intrinsic::nvvm_suld_3d_v4i16_trap: {
2340 Info.opc = getOpcForSurfaceInstr(Intrinsic);
2341 Info.memVT = MVT::i16;
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2342 Info.ptrVal = nullptr;
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]2343 Info.offset = 0;
2344 Info.vol = 0;
2345 Info.readMem = true;
2346 Info.writeMem = false;
2347 Info.align = 16;
2348 return true;
2349 }
2350 case Intrinsic::nvvm_suld_1d_i32_trap:
2351 case Intrinsic::nvvm_suld_1d_v2i32_trap:
2352 case Intrinsic::nvvm_suld_1d_v4i32_trap:
2353 case Intrinsic::nvvm_suld_1d_array_i32_trap:
2354 case Intrinsic::nvvm_suld_1d_array_v2i32_trap:
2355 case Intrinsic::nvvm_suld_1d_array_v4i32_trap:
2356 case Intrinsic::nvvm_suld_2d_i32_trap:
2357 case Intrinsic::nvvm_suld_2d_v2i32_trap:
2358 case Intrinsic::nvvm_suld_2d_v4i32_trap:
2359 case Intrinsic::nvvm_suld_2d_array_i32_trap:
2360 case Intrinsic::nvvm_suld_2d_array_v2i32_trap:
2361 case Intrinsic::nvvm_suld_2d_array_v4i32_trap:
2362 case Intrinsic::nvvm_suld_3d_i32_trap:
2363 case Intrinsic::nvvm_suld_3d_v2i32_trap:
2364 case Intrinsic::nvvm_suld_3d_v4i32_trap: {
2365 Info.opc = getOpcForSurfaceInstr(Intrinsic);
2366 Info.memVT = MVT::i32;
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2367 Info.ptrVal = nullptr;
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]2368 Info.offset = 0;
2369 Info.vol = 0;
2370 Info.readMem = true;
2371 Info.writeMem = false;
2372 Info.align = 16;
2373 return true;
2374 }
2375
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]2376 }
2377 return false;
2378}
2379
2380/// isLegalAddressingMode - Return true if the addressing mode represented
2381/// by AM is legal for this target, for a load/store of the specified type.
2382/// Used to guide target specific optimizations, like loop strength reduction
2383/// (LoopStrengthReduce.cpp) and memory optimization for address mode
2384/// (CodeGenPrepare.cpp)
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2385bool NVPTXTargetLowering::isLegalAddressingMode(const AddrMode &AM,
2386 Type *Ty) const {
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]2387
2388 // AddrMode - This represents an addressing mode of:
2389 // BaseGV + BaseOffs + BaseReg + Scale*ScaleReg
2390 //
2391 // The legal address modes are
2392 // - [avar]
2393 // - [areg]
2394 // - [areg+immoff]
2395 // - [immAddr]
2396
2397 if (AM.BaseGV) {
2398 if (AM.BaseOffs || AM.HasBaseReg || AM.Scale)
2399 return false;
2400 return true;
2401 }
2402
2403 switch (AM.Scale) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2404 case 0: // "r", "r+i" or "i" is allowed
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]2405 break;
2406 case 1:
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2407 if (AM.HasBaseReg) // "r+r+i" or "r+r" is not allowed.
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]2408 return false;
2409 // Otherwise we have r+i.
2410 break;
2411 default:
2412 // No scale > 1 is allowed
2413 return false;
2414 }
2415 return true;
2416}
2417
2418//===----------------------------------------------------------------------===//
2419// NVPTX Inline Assembly Support
2420//===----------------------------------------------------------------------===//
2421
2422/// getConstraintType - Given a constraint letter, return the type of
2423/// constraint it is for this target.
2424NVPTXTargetLowering::ConstraintType
2425NVPTXTargetLowering::getConstraintType(const std::string &Constraint) const {
2426 if (Constraint.size() == 1) {
2427 switch (Constraint[0]) {
2428 default:
2429 break;
2430 case 'r':
2431 case 'h':
2432 case 'c':
2433 case 'l':
2434 case 'f':
2435 case 'd':
2436 case '0':
2437 case 'N':
2438 return C_RegisterClass;
2439 }
2440 }
2441 return TargetLowering::getConstraintType(Constraint);
2442}
2443
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2444std::pair<unsigned, const TargetRegisterClass *>
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]2445NVPTXTargetLowering::getRegForInlineAsmConstraint(const std::string &Constraint,
Chad Rosier295bd432013-06-22 18:37:38 +0000[diff] [blame]2446 MVT VT) const {
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]2447 if (Constraint.size() == 1) {
2448 switch (Constraint[0]) {
2449 case 'c':
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]2450 return std::make_pair(0U, &NVPTX::Int16RegsRegClass);
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]2451 case 'h':
2452 return std::make_pair(0U, &NVPTX::Int16RegsRegClass);
2453 case 'r':
2454 return std::make_pair(0U, &NVPTX::Int32RegsRegClass);
2455 case 'l':
2456 case 'N':
2457 return std::make_pair(0U, &NVPTX::Int64RegsRegClass);
2458 case 'f':
2459 return std::make_pair(0U, &NVPTX::Float32RegsRegClass);
2460 case 'd':
2461 return std::make_pair(0U, &NVPTX::Float64RegsRegClass);
2462 }
2463 }
2464 return TargetLowering::getRegForInlineAsmConstraint(Constraint, VT);
2465}
2466
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]2467/// getFunctionAlignment - Return the Log2 alignment of this function.
2468unsigned NVPTXTargetLowering::getFunctionAlignment(const Function *) const {
2469 return 4;
2470}
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]2471
2472/// ReplaceVectorLoad - Convert vector loads into multi-output scalar loads.
2473static void ReplaceLoadVector(SDNode *N, SelectionDAG &DAG,
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2474 SmallVectorImpl<SDValue> &Results) {
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]2475 EVT ResVT = N->getValueType(0);
Andrew Trickef9de2a2013-05-25 02:42:55 +0000[diff] [blame]2476 SDLoc DL(N);
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]2477
2478 assert(ResVT.isVector() && "Vector load must have vector type");
2479
2480 // We only handle "native" vector sizes for now, e.g. <4 x double> is not
2481 // legal. We can (and should) split that into 2 loads of <2 x double> here
2482 // but I'm leaving that as a TODO for now.
2483 assert(ResVT.isSimple() && "Can only handle simple types");
2484 switch (ResVT.getSimpleVT().SimpleTy) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2485 default:
2486 return;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]2487 case MVT::v2i8:
2488 case MVT::v2i16:
2489 case MVT::v2i32:
2490 case MVT::v2i64:
2491 case MVT::v2f32:
2492 case MVT::v2f64:
2493 case MVT::v4i8:
2494 case MVT::v4i16:
2495 case MVT::v4i32:
2496 case MVT::v4f32:
2497 // This is a "native" vector type
2498 break;
2499 }
2500
2501 EVT EltVT = ResVT.getVectorElementType();
2502 unsigned NumElts = ResVT.getVectorNumElements();
2503
2504 // Since LoadV2 is a target node, we cannot rely on DAG type legalization.
2505 // Therefore, we must ensure the type is legal. For i1 and i8, we set the
Alp Tokercb402912014-01-24 17:20:08 +0000[diff] [blame]2506 // loaded type to i16 and propagate the "real" type as the memory type.
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]2507 bool NeedTrunc = false;
2508 if (EltVT.getSizeInBits() < 16) {
2509 EltVT = MVT::i16;
2510 NeedTrunc = true;
2511 }
2512
2513 unsigned Opcode = 0;
2514 SDVTList LdResVTs;
2515
2516 switch (NumElts) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2517 default:
2518 return;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]2519 case 2:
2520 Opcode = NVPTXISD::LoadV2;
2521 LdResVTs = DAG.getVTList(EltVT, EltVT, MVT::Other);
2522 break;
2523 case 4: {
2524 Opcode = NVPTXISD::LoadV4;
2525 EVT ListVTs[] = { EltVT, EltVT, EltVT, EltVT, MVT::Other };
Craig Topperabb4ac72014-04-16 06:10:51 +0000[diff] [blame]2526 LdResVTs = DAG.getVTList(ListVTs);
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]2527 break;
2528 }
2529 }
2530
2531 SmallVector<SDValue, 8> OtherOps;
2532
2533 // Copy regular operands
2534 for (unsigned i = 0, e = N->getNumOperands(); i != e; ++i)
2535 OtherOps.push_back(N->getOperand(i));
2536
2537 LoadSDNode *LD = cast<LoadSDNode>(N);
2538
2539 // The select routine does not have access to the LoadSDNode instance, so
2540 // pass along the extension information
2541 OtherOps.push_back(DAG.getIntPtrConstant(LD->getExtensionType()));
2542
Craig Topper206fcd42014-04-26 19:29:41 +0000[diff] [blame]2543 SDValue NewLD = DAG.getMemIntrinsicNode(Opcode, DL, LdResVTs, OtherOps,
2544 LD->getMemoryVT(),
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]2545 LD->getMemOperand());
2546
2547 SmallVector<SDValue, 4> ScalarRes;
2548
2549 for (unsigned i = 0; i < NumElts; ++i) {
2550 SDValue Res = NewLD.getValue(i);
2551 if (NeedTrunc)
2552 Res = DAG.getNode(ISD::TRUNCATE, DL, ResVT.getVectorElementType(), Res);
2553 ScalarRes.push_back(Res);
2554 }
2555
2556 SDValue LoadChain = NewLD.getValue(NumElts);
2557
Craig Topper48d114b2014-04-26 18:35:24 +0000[diff] [blame]2558 SDValue BuildVec = DAG.getNode(ISD::BUILD_VECTOR, DL, ResVT, ScalarRes);
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]2559
2560 Results.push_back(BuildVec);
2561 Results.push_back(LoadChain);
2562}
2563
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2564static void ReplaceINTRINSIC_W_CHAIN(SDNode *N, SelectionDAG &DAG,
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]2565 SmallVectorImpl<SDValue> &Results) {
2566 SDValue Chain = N->getOperand(0);
2567 SDValue Intrin = N->getOperand(1);
Andrew Trickef9de2a2013-05-25 02:42:55 +0000[diff] [blame]2568 SDLoc DL(N);
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]2569
2570 // Get the intrinsic ID
2571 unsigned IntrinNo = cast<ConstantSDNode>(Intrin.getNode())->getZExtValue();
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2572 switch (IntrinNo) {
2573 default:
2574 return;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]2575 case Intrinsic::nvvm_ldg_global_i:
2576 case Intrinsic::nvvm_ldg_global_f:
2577 case Intrinsic::nvvm_ldg_global_p:
2578 case Intrinsic::nvvm_ldu_global_i:
2579 case Intrinsic::nvvm_ldu_global_f:
2580 case Intrinsic::nvvm_ldu_global_p: {
2581 EVT ResVT = N->getValueType(0);
2582
2583 if (ResVT.isVector()) {
2584 // Vector LDG/LDU
2585
2586 unsigned NumElts = ResVT.getVectorNumElements();
2587 EVT EltVT = ResVT.getVectorElementType();
2588
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]2589 // Since LDU/LDG are target nodes, we cannot rely on DAG type
2590 // legalization.
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]2591 // Therefore, we must ensure the type is legal. For i1 and i8, we set the
Alp Tokercb402912014-01-24 17:20:08 +0000[diff] [blame]2592 // loaded type to i16 and propagate the "real" type as the memory type.
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]2593 bool NeedTrunc = false;
2594 if (EltVT.getSizeInBits() < 16) {
2595 EltVT = MVT::i16;
2596 NeedTrunc = true;
2597 }
2598
2599 unsigned Opcode = 0;
2600 SDVTList LdResVTs;
2601
2602 switch (NumElts) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2603 default:
2604 return;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]2605 case 2:
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2606 switch (IntrinNo) {
2607 default:
2608 return;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]2609 case Intrinsic::nvvm_ldg_global_i:
2610 case Intrinsic::nvvm_ldg_global_f:
2611 case Intrinsic::nvvm_ldg_global_p:
2612 Opcode = NVPTXISD::LDGV2;
2613 break;
2614 case Intrinsic::nvvm_ldu_global_i:
2615 case Intrinsic::nvvm_ldu_global_f:
2616 case Intrinsic::nvvm_ldu_global_p:
2617 Opcode = NVPTXISD::LDUV2;
2618 break;
2619 }
2620 LdResVTs = DAG.getVTList(EltVT, EltVT, MVT::Other);
2621 break;
2622 case 4: {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2623 switch (IntrinNo) {
2624 default:
2625 return;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]2626 case Intrinsic::nvvm_ldg_global_i:
2627 case Intrinsic::nvvm_ldg_global_f:
2628 case Intrinsic::nvvm_ldg_global_p:
2629 Opcode = NVPTXISD::LDGV4;
2630 break;
2631 case Intrinsic::nvvm_ldu_global_i:
2632 case Intrinsic::nvvm_ldu_global_f:
2633 case Intrinsic::nvvm_ldu_global_p:
2634 Opcode = NVPTXISD::LDUV4;
2635 break;
2636 }
2637 EVT ListVTs[] = { EltVT, EltVT, EltVT, EltVT, MVT::Other };
Craig Topperabb4ac72014-04-16 06:10:51 +0000[diff] [blame]2638 LdResVTs = DAG.getVTList(ListVTs);
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]2639 break;
2640 }
2641 }
2642
2643 SmallVector<SDValue, 8> OtherOps;
2644
2645 // Copy regular operands
2646
2647 OtherOps.push_back(Chain); // Chain
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2648 // Skip operand 1 (intrinsic ID)
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]2649 // Others
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]2650 for (unsigned i = 2, e = N->getNumOperands(); i != e; ++i)
2651 OtherOps.push_back(N->getOperand(i));
2652
2653 MemIntrinsicSDNode *MemSD = cast<MemIntrinsicSDNode>(N);
2654
Craig Topper206fcd42014-04-26 19:29:41 +0000[diff] [blame]2655 SDValue NewLD = DAG.getMemIntrinsicNode(Opcode, DL, LdResVTs, OtherOps,
2656 MemSD->getMemoryVT(),
2657 MemSD->getMemOperand());
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]2658
2659 SmallVector<SDValue, 4> ScalarRes;
2660
2661 for (unsigned i = 0; i < NumElts; ++i) {
2662 SDValue Res = NewLD.getValue(i);
2663 if (NeedTrunc)
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2664 Res =
2665 DAG.getNode(ISD::TRUNCATE, DL, ResVT.getVectorElementType(), Res);
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]2666 ScalarRes.push_back(Res);
2667 }
2668
2669 SDValue LoadChain = NewLD.getValue(NumElts);
2670
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2671 SDValue BuildVec =
Craig Topper48d114b2014-04-26 18:35:24 +0000[diff] [blame]2672 DAG.getNode(ISD::BUILD_VECTOR, DL, ResVT, ScalarRes);
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]2673
2674 Results.push_back(BuildVec);
2675 Results.push_back(LoadChain);
2676 } else {
2677 // i8 LDG/LDU
2678 assert(ResVT.isSimple() && ResVT.getSimpleVT().SimpleTy == MVT::i8 &&
2679 "Custom handling of non-i8 ldu/ldg?");
2680
2681 // Just copy all operands as-is
2682 SmallVector<SDValue, 4> Ops;
2683 for (unsigned i = 0, e = N->getNumOperands(); i != e; ++i)
2684 Ops.push_back(N->getOperand(i));
2685
2686 // Force output to i16
2687 SDVTList LdResVTs = DAG.getVTList(MVT::i16, MVT::Other);
2688
2689 MemIntrinsicSDNode *MemSD = cast<MemIntrinsicSDNode>(N);
2690
2691 // We make sure the memory type is i8, which will be used during isel
2692 // to select the proper instruction.
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2693 SDValue NewLD =
Craig Topper206fcd42014-04-26 19:29:41 +0000[diff] [blame]2694 DAG.getMemIntrinsicNode(ISD::INTRINSIC_W_CHAIN, DL, LdResVTs, Ops,
2695 MVT::i8, MemSD->getMemOperand());
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]2696
Justin Holewinskie8c93e32013-07-01 12:58:48 +0000[diff] [blame]2697 Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i8,
2698 NewLD.getValue(0)));
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]2699 Results.push_back(NewLD.getValue(1));
2700 }
2701 }
2702 }
2703}
2704
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2705void NVPTXTargetLowering::ReplaceNodeResults(
2706 SDNode *N, SmallVectorImpl<SDValue> &Results, SelectionDAG &DAG) const {
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]2707 switch (N->getOpcode()) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2708 default:
2709 report_fatal_error("Unhandled custom legalization");
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]2710 case ISD::LOAD:
2711 ReplaceLoadVector(N, DAG, Results);
2712 return;
2713 case ISD::INTRINSIC_W_CHAIN:
2714 ReplaceINTRINSIC_W_CHAIN(N, DAG, Results);
2715 return;
2716 }
2717}
Juergen Ributzkad12ccbd2013-11-19 00:57:56 +0000[diff] [blame]2718
2719// Pin NVPTXSection's and NVPTXTargetObjectFile's vtables to this file.
2720void NVPTXSection::anchor() {}
2721
2722NVPTXTargetObjectFile::~NVPTXTargetObjectFile() {
2723 delete TextSection;
2724 delete DataSection;
2725 delete BSSSection;
2726 delete ReadOnlySection;
2727
2728 delete StaticCtorSection;
2729 delete StaticDtorSection;
2730 delete LSDASection;
2731 delete EHFrameSection;
2732 delete DwarfAbbrevSection;
2733 delete DwarfInfoSection;
2734 delete DwarfLineSection;
2735 delete DwarfFrameSection;
2736 delete DwarfPubTypesSection;
2737 delete DwarfDebugInlineSection;
2738 delete DwarfStrSection;
2739 delete DwarfLocSection;
2740 delete DwarfARangesSection;
2741 delete DwarfRangesSection;
2742 delete DwarfMacroInfoSection;
2743}