Gitiles
Code Review Sign In
gerrit-public.fairphone.software / toolchain / llvm-project / 60f1db0525003e15485a84fa4cc8ec475df6f403 / . / llvm / lib / Target / NVPTX / NVPTXISelDAGToDAG.cpp
blob: 62ae664804d8af55adf8b662b41c16b7cbff798a [file] [log] [blame]
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]1//===-- NVPTXISelDAGToDAG.cpp - A dag to dag inst selector for NVPTX ------===//
2//
3// The LLVM Compiler Infrastructure
4//
5// This file is distributed under the University of Illinois Open Source
6// License. See LICENSE.TXT for details.
7//
8//===----------------------------------------------------------------------===//
9//
10// This file defines an instruction selector for the NVPTX target.
11//
12//===----------------------------------------------------------------------===//
13
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]14#include "NVPTXISelDAGToDAG.h"
Chandler Carruth9fb823b2013-01-02 11:36:10 +0000[diff] [blame]15#include "llvm/IR/GlobalValue.h"
16#include "llvm/IR/Instructions.h"
Chandler Carruthed0881b2012-12-03 16:50:05 +0000[diff] [blame]17#include "llvm/Support/CommandLine.h"
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]18#include "llvm/Support/Debug.h"
19#include "llvm/Support/ErrorHandling.h"
Chandler Carruthed0881b2012-12-03 16:50:05 +0000[diff] [blame]20#include "llvm/Support/raw_ostream.h"
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]21#include "llvm/Target/TargetIntrinsicInfo.h"
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]22
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]23using namespace llvm;
24
Chandler Carruth84e68b22014-04-22 02:41:26 +0000[diff] [blame]25#define DEBUG_TYPE "nvptx-isel"
26
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]27static cl::opt<int> UsePrecDivF32(
Nadav Rotem7f27e0b2013-10-18 23:38:13 +0000[diff] [blame]28 "nvptx-prec-divf32", cl::ZeroOrMore, cl::Hidden,
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]29 cl::desc("NVPTX Specifies: 0 use div.approx, 1 use div.full, 2 use"
Sylvestre Ledru469de192014-08-11 18:04:46 +0000[diff] [blame]30 " IEEE Compliant F32 div.rnd if available."),
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]31 cl::init(2));
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]32
Justin Holewinski48f4ad32013-05-21 16:51:30 +0000[diff] [blame]33static cl::opt<bool>
Nadav Rotem7f27e0b2013-10-18 23:38:13 +0000[diff] [blame]34UsePrecSqrtF32("nvptx-prec-sqrtf32", cl::Hidden,
Justin Holewinski48f4ad32013-05-21 16:51:30 +0000[diff] [blame]35 cl::desc("NVPTX Specific: 0 use sqrt.approx, 1 use sqrt.rn."),
36 cl::init(true));
37
Justin Holewinskicd069e62013-07-22 12:18:04 +0000[diff] [blame]38static cl::opt<bool>
Nadav Rotem7f27e0b2013-10-18 23:38:13 +0000[diff] [blame]39FtzEnabled("nvptx-f32ftz", cl::ZeroOrMore, cl::Hidden,
Justin Holewinskicd069e62013-07-22 12:18:04 +0000[diff] [blame]40 cl::desc("NVPTX Specific: Flush f32 subnormals to sign-preserving zero."),
41 cl::init(false));
42
43
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]44/// createNVPTXISelDag - This pass converts a legalized DAG into a
45/// NVPTX-specific DAG, ready for instruction scheduling.
46FunctionPass *llvm::createNVPTXISelDag(NVPTXTargetMachine &TM,
47 llvm::CodeGenOpt::Level OptLevel) {
48 return new NVPTXDAGToDAGISel(TM, OptLevel);
49}
50
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]51NVPTXDAGToDAGISel::NVPTXDAGToDAGISel(NVPTXTargetMachine &tm,
52 CodeGenOpt::Level OptLevel)
Eric Christopher02389e32015-02-19 00:08:27 +0000[diff] [blame]53 : SelectionDAGISel(tm, OptLevel), TM(tm) {
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]54 doMulWide = (OptLevel > 0);
Justin Holewinskicd069e62013-07-22 12:18:04 +0000[diff] [blame]55}
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]56
Eric Christopher147bba22015-01-30 01:40:59 +0000[diff] [blame]57bool NVPTXDAGToDAGISel::runOnMachineFunction(MachineFunction &MF) {
58 Subtarget = &static_cast<const NVPTXSubtarget &>(MF.getSubtarget());
59 return SelectionDAGISel::runOnMachineFunction(MF);
60}
61
Justin Holewinskicd069e62013-07-22 12:18:04 +0000[diff] [blame]62int NVPTXDAGToDAGISel::getDivF32Level() const {
63 if (UsePrecDivF32.getNumOccurrences() > 0) {
64 // If nvptx-prec-div32=N is used on the command-line, always honor it
65 return UsePrecDivF32;
66 } else {
67 // Otherwise, use div.approx if fast math is enabled
68 if (TM.Options.UnsafeFPMath)
69 return 0;
70 else
71 return 2;
72 }
73}
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]74
Justin Holewinskicd069e62013-07-22 12:18:04 +0000[diff] [blame]75bool NVPTXDAGToDAGISel::usePrecSqrtF32() const {
76 if (UsePrecSqrtF32.getNumOccurrences() > 0) {
77 // If nvptx-prec-sqrtf32 is used on the command-line, always honor it
78 return UsePrecSqrtF32;
79 } else {
80 // Otherwise, use sqrt.approx if fast math is enabled
81 if (TM.Options.UnsafeFPMath)
82 return false;
83 else
84 return true;
85 }
86}
87
88bool NVPTXDAGToDAGISel::useF32FTZ() const {
89 if (FtzEnabled.getNumOccurrences() > 0) {
90 // If nvptx-f32ftz is used on the command-line, always honor it
91 return FtzEnabled;
92 } else {
93 const Function *F = MF->getFunction();
94 // Otherwise, check for an nvptx-f32ftz attribute on the function
95 if (F->hasFnAttribute("nvptx-f32ftz"))
Duncan P. N. Exon Smithb5054332015-02-14 15:35:43 +0000[diff] [blame]96 return F->getFnAttribute("nvptx-f32ftz").getValueAsString() == "true";
Justin Holewinskicd069e62013-07-22 12:18:04 +0000[diff] [blame]97 else
98 return false;
99 }
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]100}
101
Justin Holewinski428cf0e2014-07-17 18:10:09 +0000[diff] [blame]102bool NVPTXDAGToDAGISel::allowFMA() const {
Eric Christopher147bba22015-01-30 01:40:59 +0000[diff] [blame]103 const NVPTXTargetLowering *TL = Subtarget->getTargetLowering();
Justin Holewinski428cf0e2014-07-17 18:10:09 +0000[diff] [blame]104 return TL->allowFMA(*MF, OptLevel);
105}
106
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]107/// Select - Select instructions not customized! Used for
108/// expanded, promoted and normal instructions.
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]109SDNode *NVPTXDAGToDAGISel::Select(SDNode *N) {
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]110
Tim Northover31d093c2013-09-22 08:21:56 +0000[diff] [blame]111 if (N->isMachineOpcode()) {
112 N->setNodeId(-1);
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]113 return nullptr; // Already selected.
Tim Northover31d093c2013-09-22 08:21:56 +0000[diff] [blame]114 }
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]115
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]116 SDNode *ResNode = nullptr;
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]117 switch (N->getOpcode()) {
118 case ISD::LOAD:
119 ResNode = SelectLoad(N);
120 break;
121 case ISD::STORE:
122 ResNode = SelectStore(N);
123 break;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]124 case NVPTXISD::LoadV2:
125 case NVPTXISD::LoadV4:
126 ResNode = SelectLoadVector(N);
127 break;
128 case NVPTXISD::LDGV2:
129 case NVPTXISD::LDGV4:
130 case NVPTXISD::LDUV2:
131 case NVPTXISD::LDUV4:
Justin Holewinskib926d9d2014-06-27 18:35:51 +0000[diff] [blame]132 ResNode = SelectLDGLDU(N);
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]133 break;
134 case NVPTXISD::StoreV2:
135 case NVPTXISD::StoreV4:
136 ResNode = SelectStoreVector(N);
137 break;
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]138 case NVPTXISD::LoadParam:
139 case NVPTXISD::LoadParamV2:
140 case NVPTXISD::LoadParamV4:
141 ResNode = SelectLoadParam(N);
142 break;
143 case NVPTXISD::StoreRetval:
144 case NVPTXISD::StoreRetvalV2:
145 case NVPTXISD::StoreRetvalV4:
146 ResNode = SelectStoreRetval(N);
147 break;
148 case NVPTXISD::StoreParam:
149 case NVPTXISD::StoreParamV2:
150 case NVPTXISD::StoreParamV4:
151 case NVPTXISD::StoreParamS32:
152 case NVPTXISD::StoreParamU32:
153 ResNode = SelectStoreParam(N);
154 break;
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]155 case ISD::INTRINSIC_WO_CHAIN:
156 ResNode = SelectIntrinsicNoChain(N);
157 break;
Justin Holewinskib926d9d2014-06-27 18:35:51 +0000[diff] [blame]158 case ISD::INTRINSIC_W_CHAIN:
159 ResNode = SelectIntrinsicChain(N);
160 break;
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]161 case NVPTXISD::Tex1DFloatS32:
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]162 case NVPTXISD::Tex1DFloatFloat:
163 case NVPTXISD::Tex1DFloatFloatLevel:
164 case NVPTXISD::Tex1DFloatFloatGrad:
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]165 case NVPTXISD::Tex1DS32S32:
166 case NVPTXISD::Tex1DS32Float:
167 case NVPTXISD::Tex1DS32FloatLevel:
168 case NVPTXISD::Tex1DS32FloatGrad:
169 case NVPTXISD::Tex1DU32S32:
170 case NVPTXISD::Tex1DU32Float:
171 case NVPTXISD::Tex1DU32FloatLevel:
172 case NVPTXISD::Tex1DU32FloatGrad:
173 case NVPTXISD::Tex1DArrayFloatS32:
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]174 case NVPTXISD::Tex1DArrayFloatFloat:
175 case NVPTXISD::Tex1DArrayFloatFloatLevel:
176 case NVPTXISD::Tex1DArrayFloatFloatGrad:
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]177 case NVPTXISD::Tex1DArrayS32S32:
178 case NVPTXISD::Tex1DArrayS32Float:
179 case NVPTXISD::Tex1DArrayS32FloatLevel:
180 case NVPTXISD::Tex1DArrayS32FloatGrad:
181 case NVPTXISD::Tex1DArrayU32S32:
182 case NVPTXISD::Tex1DArrayU32Float:
183 case NVPTXISD::Tex1DArrayU32FloatLevel:
184 case NVPTXISD::Tex1DArrayU32FloatGrad:
185 case NVPTXISD::Tex2DFloatS32:
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]186 case NVPTXISD::Tex2DFloatFloat:
187 case NVPTXISD::Tex2DFloatFloatLevel:
188 case NVPTXISD::Tex2DFloatFloatGrad:
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]189 case NVPTXISD::Tex2DS32S32:
190 case NVPTXISD::Tex2DS32Float:
191 case NVPTXISD::Tex2DS32FloatLevel:
192 case NVPTXISD::Tex2DS32FloatGrad:
193 case NVPTXISD::Tex2DU32S32:
194 case NVPTXISD::Tex2DU32Float:
195 case NVPTXISD::Tex2DU32FloatLevel:
196 case NVPTXISD::Tex2DU32FloatGrad:
197 case NVPTXISD::Tex2DArrayFloatS32:
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]198 case NVPTXISD::Tex2DArrayFloatFloat:
199 case NVPTXISD::Tex2DArrayFloatFloatLevel:
200 case NVPTXISD::Tex2DArrayFloatFloatGrad:
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]201 case NVPTXISD::Tex2DArrayS32S32:
202 case NVPTXISD::Tex2DArrayS32Float:
203 case NVPTXISD::Tex2DArrayS32FloatLevel:
204 case NVPTXISD::Tex2DArrayS32FloatGrad:
205 case NVPTXISD::Tex2DArrayU32S32:
206 case NVPTXISD::Tex2DArrayU32Float:
207 case NVPTXISD::Tex2DArrayU32FloatLevel:
208 case NVPTXISD::Tex2DArrayU32FloatGrad:
209 case NVPTXISD::Tex3DFloatS32:
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]210 case NVPTXISD::Tex3DFloatFloat:
211 case NVPTXISD::Tex3DFloatFloatLevel:
212 case NVPTXISD::Tex3DFloatFloatGrad:
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]213 case NVPTXISD::Tex3DS32S32:
214 case NVPTXISD::Tex3DS32Float:
215 case NVPTXISD::Tex3DS32FloatLevel:
216 case NVPTXISD::Tex3DS32FloatGrad:
217 case NVPTXISD::Tex3DU32S32:
218 case NVPTXISD::Tex3DU32Float:
219 case NVPTXISD::Tex3DU32FloatLevel:
220 case NVPTXISD::Tex3DU32FloatGrad:
221 case NVPTXISD::TexCubeFloatFloat:
222 case NVPTXISD::TexCubeFloatFloatLevel:
223 case NVPTXISD::TexCubeS32Float:
224 case NVPTXISD::TexCubeS32FloatLevel:
225 case NVPTXISD::TexCubeU32Float:
226 case NVPTXISD::TexCubeU32FloatLevel:
227 case NVPTXISD::TexCubeArrayFloatFloat:
228 case NVPTXISD::TexCubeArrayFloatFloatLevel:
229 case NVPTXISD::TexCubeArrayS32Float:
230 case NVPTXISD::TexCubeArrayS32FloatLevel:
231 case NVPTXISD::TexCubeArrayU32Float:
232 case NVPTXISD::TexCubeArrayU32FloatLevel:
233 case NVPTXISD::Tld4R2DFloatFloat:
234 case NVPTXISD::Tld4G2DFloatFloat:
235 case NVPTXISD::Tld4B2DFloatFloat:
236 case NVPTXISD::Tld4A2DFloatFloat:
237 case NVPTXISD::Tld4R2DS64Float:
238 case NVPTXISD::Tld4G2DS64Float:
239 case NVPTXISD::Tld4B2DS64Float:
240 case NVPTXISD::Tld4A2DS64Float:
241 case NVPTXISD::Tld4R2DU64Float:
242 case NVPTXISD::Tld4G2DU64Float:
243 case NVPTXISD::Tld4B2DU64Float:
244 case NVPTXISD::Tld4A2DU64Float:
245 case NVPTXISD::TexUnified1DFloatS32:
246 case NVPTXISD::TexUnified1DFloatFloat:
247 case NVPTXISD::TexUnified1DFloatFloatLevel:
248 case NVPTXISD::TexUnified1DFloatFloatGrad:
249 case NVPTXISD::TexUnified1DS32S32:
250 case NVPTXISD::TexUnified1DS32Float:
251 case NVPTXISD::TexUnified1DS32FloatLevel:
252 case NVPTXISD::TexUnified1DS32FloatGrad:
253 case NVPTXISD::TexUnified1DU32S32:
254 case NVPTXISD::TexUnified1DU32Float:
255 case NVPTXISD::TexUnified1DU32FloatLevel:
256 case NVPTXISD::TexUnified1DU32FloatGrad:
257 case NVPTXISD::TexUnified1DArrayFloatS32:
258 case NVPTXISD::TexUnified1DArrayFloatFloat:
259 case NVPTXISD::TexUnified1DArrayFloatFloatLevel:
260 case NVPTXISD::TexUnified1DArrayFloatFloatGrad:
261 case NVPTXISD::TexUnified1DArrayS32S32:
262 case NVPTXISD::TexUnified1DArrayS32Float:
263 case NVPTXISD::TexUnified1DArrayS32FloatLevel:
264 case NVPTXISD::TexUnified1DArrayS32FloatGrad:
265 case NVPTXISD::TexUnified1DArrayU32S32:
266 case NVPTXISD::TexUnified1DArrayU32Float:
267 case NVPTXISD::TexUnified1DArrayU32FloatLevel:
268 case NVPTXISD::TexUnified1DArrayU32FloatGrad:
269 case NVPTXISD::TexUnified2DFloatS32:
270 case NVPTXISD::TexUnified2DFloatFloat:
271 case NVPTXISD::TexUnified2DFloatFloatLevel:
272 case NVPTXISD::TexUnified2DFloatFloatGrad:
273 case NVPTXISD::TexUnified2DS32S32:
274 case NVPTXISD::TexUnified2DS32Float:
275 case NVPTXISD::TexUnified2DS32FloatLevel:
276 case NVPTXISD::TexUnified2DS32FloatGrad:
277 case NVPTXISD::TexUnified2DU32S32:
278 case NVPTXISD::TexUnified2DU32Float:
279 case NVPTXISD::TexUnified2DU32FloatLevel:
280 case NVPTXISD::TexUnified2DU32FloatGrad:
281 case NVPTXISD::TexUnified2DArrayFloatS32:
282 case NVPTXISD::TexUnified2DArrayFloatFloat:
283 case NVPTXISD::TexUnified2DArrayFloatFloatLevel:
284 case NVPTXISD::TexUnified2DArrayFloatFloatGrad:
285 case NVPTXISD::TexUnified2DArrayS32S32:
286 case NVPTXISD::TexUnified2DArrayS32Float:
287 case NVPTXISD::TexUnified2DArrayS32FloatLevel:
288 case NVPTXISD::TexUnified2DArrayS32FloatGrad:
289 case NVPTXISD::TexUnified2DArrayU32S32:
290 case NVPTXISD::TexUnified2DArrayU32Float:
291 case NVPTXISD::TexUnified2DArrayU32FloatLevel:
292 case NVPTXISD::TexUnified2DArrayU32FloatGrad:
293 case NVPTXISD::TexUnified3DFloatS32:
294 case NVPTXISD::TexUnified3DFloatFloat:
295 case NVPTXISD::TexUnified3DFloatFloatLevel:
296 case NVPTXISD::TexUnified3DFloatFloatGrad:
297 case NVPTXISD::TexUnified3DS32S32:
298 case NVPTXISD::TexUnified3DS32Float:
299 case NVPTXISD::TexUnified3DS32FloatLevel:
300 case NVPTXISD::TexUnified3DS32FloatGrad:
301 case NVPTXISD::TexUnified3DU32S32:
302 case NVPTXISD::TexUnified3DU32Float:
303 case NVPTXISD::TexUnified3DU32FloatLevel:
304 case NVPTXISD::TexUnified3DU32FloatGrad:
305 case NVPTXISD::TexUnifiedCubeFloatFloat:
306 case NVPTXISD::TexUnifiedCubeFloatFloatLevel:
307 case NVPTXISD::TexUnifiedCubeS32Float:
308 case NVPTXISD::TexUnifiedCubeS32FloatLevel:
309 case NVPTXISD::TexUnifiedCubeU32Float:
310 case NVPTXISD::TexUnifiedCubeU32FloatLevel:
311 case NVPTXISD::TexUnifiedCubeArrayFloatFloat:
312 case NVPTXISD::TexUnifiedCubeArrayFloatFloatLevel:
313 case NVPTXISD::TexUnifiedCubeArrayS32Float:
314 case NVPTXISD::TexUnifiedCubeArrayS32FloatLevel:
315 case NVPTXISD::TexUnifiedCubeArrayU32Float:
316 case NVPTXISD::TexUnifiedCubeArrayU32FloatLevel:
317 case NVPTXISD::Tld4UnifiedR2DFloatFloat:
318 case NVPTXISD::Tld4UnifiedG2DFloatFloat:
319 case NVPTXISD::Tld4UnifiedB2DFloatFloat:
320 case NVPTXISD::Tld4UnifiedA2DFloatFloat:
321 case NVPTXISD::Tld4UnifiedR2DS64Float:
322 case NVPTXISD::Tld4UnifiedG2DS64Float:
323 case NVPTXISD::Tld4UnifiedB2DS64Float:
324 case NVPTXISD::Tld4UnifiedA2DS64Float:
325 case NVPTXISD::Tld4UnifiedR2DU64Float:
326 case NVPTXISD::Tld4UnifiedG2DU64Float:
327 case NVPTXISD::Tld4UnifiedB2DU64Float:
328 case NVPTXISD::Tld4UnifiedA2DU64Float:
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]329 ResNode = SelectTextureIntrinsic(N);
330 break;
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]331 case NVPTXISD::Suld1DI8Clamp:
332 case NVPTXISD::Suld1DI16Clamp:
333 case NVPTXISD::Suld1DI32Clamp:
334 case NVPTXISD::Suld1DI64Clamp:
335 case NVPTXISD::Suld1DV2I8Clamp:
336 case NVPTXISD::Suld1DV2I16Clamp:
337 case NVPTXISD::Suld1DV2I32Clamp:
338 case NVPTXISD::Suld1DV2I64Clamp:
339 case NVPTXISD::Suld1DV4I8Clamp:
340 case NVPTXISD::Suld1DV4I16Clamp:
341 case NVPTXISD::Suld1DV4I32Clamp:
342 case NVPTXISD::Suld1DArrayI8Clamp:
343 case NVPTXISD::Suld1DArrayI16Clamp:
344 case NVPTXISD::Suld1DArrayI32Clamp:
345 case NVPTXISD::Suld1DArrayI64Clamp:
346 case NVPTXISD::Suld1DArrayV2I8Clamp:
347 case NVPTXISD::Suld1DArrayV2I16Clamp:
348 case NVPTXISD::Suld1DArrayV2I32Clamp:
349 case NVPTXISD::Suld1DArrayV2I64Clamp:
350 case NVPTXISD::Suld1DArrayV4I8Clamp:
351 case NVPTXISD::Suld1DArrayV4I16Clamp:
352 case NVPTXISD::Suld1DArrayV4I32Clamp:
353 case NVPTXISD::Suld2DI8Clamp:
354 case NVPTXISD::Suld2DI16Clamp:
355 case NVPTXISD::Suld2DI32Clamp:
356 case NVPTXISD::Suld2DI64Clamp:
357 case NVPTXISD::Suld2DV2I8Clamp:
358 case NVPTXISD::Suld2DV2I16Clamp:
359 case NVPTXISD::Suld2DV2I32Clamp:
360 case NVPTXISD::Suld2DV2I64Clamp:
361 case NVPTXISD::Suld2DV4I8Clamp:
362 case NVPTXISD::Suld2DV4I16Clamp:
363 case NVPTXISD::Suld2DV4I32Clamp:
364 case NVPTXISD::Suld2DArrayI8Clamp:
365 case NVPTXISD::Suld2DArrayI16Clamp:
366 case NVPTXISD::Suld2DArrayI32Clamp:
367 case NVPTXISD::Suld2DArrayI64Clamp:
368 case NVPTXISD::Suld2DArrayV2I8Clamp:
369 case NVPTXISD::Suld2DArrayV2I16Clamp:
370 case NVPTXISD::Suld2DArrayV2I32Clamp:
371 case NVPTXISD::Suld2DArrayV2I64Clamp:
372 case NVPTXISD::Suld2DArrayV4I8Clamp:
373 case NVPTXISD::Suld2DArrayV4I16Clamp:
374 case NVPTXISD::Suld2DArrayV4I32Clamp:
375 case NVPTXISD::Suld3DI8Clamp:
376 case NVPTXISD::Suld3DI16Clamp:
377 case NVPTXISD::Suld3DI32Clamp:
378 case NVPTXISD::Suld3DI64Clamp:
379 case NVPTXISD::Suld3DV2I8Clamp:
380 case NVPTXISD::Suld3DV2I16Clamp:
381 case NVPTXISD::Suld3DV2I32Clamp:
382 case NVPTXISD::Suld3DV2I64Clamp:
383 case NVPTXISD::Suld3DV4I8Clamp:
384 case NVPTXISD::Suld3DV4I16Clamp:
385 case NVPTXISD::Suld3DV4I32Clamp:
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]386 case NVPTXISD::Suld1DI8Trap:
387 case NVPTXISD::Suld1DI16Trap:
388 case NVPTXISD::Suld1DI32Trap:
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]389 case NVPTXISD::Suld1DI64Trap:
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]390 case NVPTXISD::Suld1DV2I8Trap:
391 case NVPTXISD::Suld1DV2I16Trap:
392 case NVPTXISD::Suld1DV2I32Trap:
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]393 case NVPTXISD::Suld1DV2I64Trap:
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]394 case NVPTXISD::Suld1DV4I8Trap:
395 case NVPTXISD::Suld1DV4I16Trap:
396 case NVPTXISD::Suld1DV4I32Trap:
397 case NVPTXISD::Suld1DArrayI8Trap:
398 case NVPTXISD::Suld1DArrayI16Trap:
399 case NVPTXISD::Suld1DArrayI32Trap:
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]400 case NVPTXISD::Suld1DArrayI64Trap:
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]401 case NVPTXISD::Suld1DArrayV2I8Trap:
402 case NVPTXISD::Suld1DArrayV2I16Trap:
403 case NVPTXISD::Suld1DArrayV2I32Trap:
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]404 case NVPTXISD::Suld1DArrayV2I64Trap:
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]405 case NVPTXISD::Suld1DArrayV4I8Trap:
406 case NVPTXISD::Suld1DArrayV4I16Trap:
407 case NVPTXISD::Suld1DArrayV4I32Trap:
408 case NVPTXISD::Suld2DI8Trap:
409 case NVPTXISD::Suld2DI16Trap:
410 case NVPTXISD::Suld2DI32Trap:
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]411 case NVPTXISD::Suld2DI64Trap:
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]412 case NVPTXISD::Suld2DV2I8Trap:
413 case NVPTXISD::Suld2DV2I16Trap:
414 case NVPTXISD::Suld2DV2I32Trap:
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]415 case NVPTXISD::Suld2DV2I64Trap:
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]416 case NVPTXISD::Suld2DV4I8Trap:
417 case NVPTXISD::Suld2DV4I16Trap:
418 case NVPTXISD::Suld2DV4I32Trap:
419 case NVPTXISD::Suld2DArrayI8Trap:
420 case NVPTXISD::Suld2DArrayI16Trap:
421 case NVPTXISD::Suld2DArrayI32Trap:
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]422 case NVPTXISD::Suld2DArrayI64Trap:
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]423 case NVPTXISD::Suld2DArrayV2I8Trap:
424 case NVPTXISD::Suld2DArrayV2I16Trap:
425 case NVPTXISD::Suld2DArrayV2I32Trap:
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]426 case NVPTXISD::Suld2DArrayV2I64Trap:
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]427 case NVPTXISD::Suld2DArrayV4I8Trap:
428 case NVPTXISD::Suld2DArrayV4I16Trap:
429 case NVPTXISD::Suld2DArrayV4I32Trap:
430 case NVPTXISD::Suld3DI8Trap:
431 case NVPTXISD::Suld3DI16Trap:
432 case NVPTXISD::Suld3DI32Trap:
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]433 case NVPTXISD::Suld3DI64Trap:
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]434 case NVPTXISD::Suld3DV2I8Trap:
435 case NVPTXISD::Suld3DV2I16Trap:
436 case NVPTXISD::Suld3DV2I32Trap:
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]437 case NVPTXISD::Suld3DV2I64Trap:
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]438 case NVPTXISD::Suld3DV4I8Trap:
439 case NVPTXISD::Suld3DV4I16Trap:
440 case NVPTXISD::Suld3DV4I32Trap:
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]441 case NVPTXISD::Suld1DI8Zero:
442 case NVPTXISD::Suld1DI16Zero:
443 case NVPTXISD::Suld1DI32Zero:
444 case NVPTXISD::Suld1DI64Zero:
445 case NVPTXISD::Suld1DV2I8Zero:
446 case NVPTXISD::Suld1DV2I16Zero:
447 case NVPTXISD::Suld1DV2I32Zero:
448 case NVPTXISD::Suld1DV2I64Zero:
449 case NVPTXISD::Suld1DV4I8Zero:
450 case NVPTXISD::Suld1DV4I16Zero:
451 case NVPTXISD::Suld1DV4I32Zero:
452 case NVPTXISD::Suld1DArrayI8Zero:
453 case NVPTXISD::Suld1DArrayI16Zero:
454 case NVPTXISD::Suld1DArrayI32Zero:
455 case NVPTXISD::Suld1DArrayI64Zero:
456 case NVPTXISD::Suld1DArrayV2I8Zero:
457 case NVPTXISD::Suld1DArrayV2I16Zero:
458 case NVPTXISD::Suld1DArrayV2I32Zero:
459 case NVPTXISD::Suld1DArrayV2I64Zero:
460 case NVPTXISD::Suld1DArrayV4I8Zero:
461 case NVPTXISD::Suld1DArrayV4I16Zero:
462 case NVPTXISD::Suld1DArrayV4I32Zero:
463 case NVPTXISD::Suld2DI8Zero:
464 case NVPTXISD::Suld2DI16Zero:
465 case NVPTXISD::Suld2DI32Zero:
466 case NVPTXISD::Suld2DI64Zero:
467 case NVPTXISD::Suld2DV2I8Zero:
468 case NVPTXISD::Suld2DV2I16Zero:
469 case NVPTXISD::Suld2DV2I32Zero:
470 case NVPTXISD::Suld2DV2I64Zero:
471 case NVPTXISD::Suld2DV4I8Zero:
472 case NVPTXISD::Suld2DV4I16Zero:
473 case NVPTXISD::Suld2DV4I32Zero:
474 case NVPTXISD::Suld2DArrayI8Zero:
475 case NVPTXISD::Suld2DArrayI16Zero:
476 case NVPTXISD::Suld2DArrayI32Zero:
477 case NVPTXISD::Suld2DArrayI64Zero:
478 case NVPTXISD::Suld2DArrayV2I8Zero:
479 case NVPTXISD::Suld2DArrayV2I16Zero:
480 case NVPTXISD::Suld2DArrayV2I32Zero:
481 case NVPTXISD::Suld2DArrayV2I64Zero:
482 case NVPTXISD::Suld2DArrayV4I8Zero:
483 case NVPTXISD::Suld2DArrayV4I16Zero:
484 case NVPTXISD::Suld2DArrayV4I32Zero:
485 case NVPTXISD::Suld3DI8Zero:
486 case NVPTXISD::Suld3DI16Zero:
487 case NVPTXISD::Suld3DI32Zero:
488 case NVPTXISD::Suld3DI64Zero:
489 case NVPTXISD::Suld3DV2I8Zero:
490 case NVPTXISD::Suld3DV2I16Zero:
491 case NVPTXISD::Suld3DV2I32Zero:
492 case NVPTXISD::Suld3DV2I64Zero:
493 case NVPTXISD::Suld3DV4I8Zero:
494 case NVPTXISD::Suld3DV4I16Zero:
495 case NVPTXISD::Suld3DV4I32Zero:
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]496 ResNode = SelectSurfaceIntrinsic(N);
497 break;
Justin Holewinskica7a4f12014-06-27 18:35:27 +0000[diff] [blame]498 case ISD::AND:
499 case ISD::SRA:
500 case ISD::SRL:
501 // Try to select BFE
502 ResNode = SelectBFE(N);
503 break;
Justin Holewinskiba2fa6d2014-03-24 11:17:53 +0000[diff] [blame]504 case ISD::ADDRSPACECAST:
505 ResNode = SelectAddrSpaceCast(N);
506 break;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]507 default:
508 break;
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]509 }
510 if (ResNode)
511 return ResNode;
512 return SelectCode(N);
513}
514
Justin Holewinskib926d9d2014-06-27 18:35:51 +0000[diff] [blame]515SDNode *NVPTXDAGToDAGISel::SelectIntrinsicChain(SDNode *N) {
516 unsigned IID = cast<ConstantSDNode>(N->getOperand(1))->getZExtValue();
517 switch (IID) {
518 default:
519 return NULL;
520 case Intrinsic::nvvm_ldg_global_f:
521 case Intrinsic::nvvm_ldg_global_i:
522 case Intrinsic::nvvm_ldg_global_p:
523 case Intrinsic::nvvm_ldu_global_f:
524 case Intrinsic::nvvm_ldu_global_i:
525 case Intrinsic::nvvm_ldu_global_p:
526 return SelectLDGLDU(N);
527 }
528}
529
Eric Christopher9745b3a2015-01-30 01:41:01 +0000[diff] [blame]530static unsigned int getCodeAddrSpace(MemSDNode *N) {
Nick Lewyckyaad475b2014-04-15 07:22:52 +0000[diff] [blame]531 const Value *Src = N->getMemOperand()->getValue();
Justin Holewinskib96d1392013-06-10 13:29:47 +0000[diff] [blame]532
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]533 if (!Src)
Justin Holewinskib96d1392013-06-10 13:29:47 +0000[diff] [blame]534 return NVPTX::PTXLdStInstCode::GENERIC;
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]535
536 if (const PointerType *PT = dyn_cast<PointerType>(Src->getType())) {
537 switch (PT->getAddressSpace()) {
Justin Holewinskib96d1392013-06-10 13:29:47 +0000[diff] [blame]538 case llvm::ADDRESS_SPACE_LOCAL: return NVPTX::PTXLdStInstCode::LOCAL;
539 case llvm::ADDRESS_SPACE_GLOBAL: return NVPTX::PTXLdStInstCode::GLOBAL;
540 case llvm::ADDRESS_SPACE_SHARED: return NVPTX::PTXLdStInstCode::SHARED;
541 case llvm::ADDRESS_SPACE_GENERIC: return NVPTX::PTXLdStInstCode::GENERIC;
542 case llvm::ADDRESS_SPACE_PARAM: return NVPTX::PTXLdStInstCode::PARAM;
543 case llvm::ADDRESS_SPACE_CONST: return NVPTX::PTXLdStInstCode::CONSTANT;
544 default: break;
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]545 }
546 }
Justin Holewinskib96d1392013-06-10 13:29:47 +0000[diff] [blame]547 return NVPTX::PTXLdStInstCode::GENERIC;
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]548}
549
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]550SDNode *NVPTXDAGToDAGISel::SelectIntrinsicNoChain(SDNode *N) {
551 unsigned IID = cast<ConstantSDNode>(N->getOperand(0))->getZExtValue();
552 switch (IID) {
553 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]554 return nullptr;
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]555 case Intrinsic::nvvm_texsurf_handle_internal:
556 return SelectTexSurfHandle(N);
557 }
558}
559
560SDNode *NVPTXDAGToDAGISel::SelectTexSurfHandle(SDNode *N) {
561 // Op 0 is the intrinsic ID
562 SDValue Wrapper = N->getOperand(1);
563 SDValue GlobalVal = Wrapper.getOperand(0);
564 return CurDAG->getMachineNode(NVPTX::texsurf_handles, SDLoc(N), MVT::i64,
565 GlobalVal);
566}
567
Justin Holewinskiba2fa6d2014-03-24 11:17:53 +0000[diff] [blame]568SDNode *NVPTXDAGToDAGISel::SelectAddrSpaceCast(SDNode *N) {
569 SDValue Src = N->getOperand(0);
570 AddrSpaceCastSDNode *CastN = cast<AddrSpaceCastSDNode>(N);
571 unsigned SrcAddrSpace = CastN->getSrcAddressSpace();
572 unsigned DstAddrSpace = CastN->getDestAddressSpace();
573
574 assert(SrcAddrSpace != DstAddrSpace &&
575 "addrspacecast must be between different address spaces");
576
577 if (DstAddrSpace == ADDRESS_SPACE_GENERIC) {
578 // Specific to generic
579 unsigned Opc;
580 switch (SrcAddrSpace) {
581 default: report_fatal_error("Bad address space in addrspacecast");
582 case ADDRESS_SPACE_GLOBAL:
Eric Christopher02389e32015-02-19 00:08:27 +0000[diff] [blame]583 Opc = TM.is64Bit() ? NVPTX::cvta_global_yes_64 : NVPTX::cvta_global_yes;
Justin Holewinskiba2fa6d2014-03-24 11:17:53 +0000[diff] [blame]584 break;
585 case ADDRESS_SPACE_SHARED:
Eric Christopher02389e32015-02-19 00:08:27 +0000[diff] [blame]586 Opc = TM.is64Bit() ? NVPTX::cvta_shared_yes_64 : NVPTX::cvta_shared_yes;
Justin Holewinskiba2fa6d2014-03-24 11:17:53 +0000[diff] [blame]587 break;
588 case ADDRESS_SPACE_CONST:
Eric Christopher02389e32015-02-19 00:08:27 +0000[diff] [blame]589 Opc = TM.is64Bit() ? NVPTX::cvta_const_yes_64 : NVPTX::cvta_const_yes;
Justin Holewinskiba2fa6d2014-03-24 11:17:53 +0000[diff] [blame]590 break;
591 case ADDRESS_SPACE_LOCAL:
Eric Christopher02389e32015-02-19 00:08:27 +0000[diff] [blame]592 Opc = TM.is64Bit() ? NVPTX::cvta_local_yes_64 : NVPTX::cvta_local_yes;
Justin Holewinskiba2fa6d2014-03-24 11:17:53 +0000[diff] [blame]593 break;
594 }
595 return CurDAG->getMachineNode(Opc, SDLoc(N), N->getValueType(0), Src);
596 } else {
597 // Generic to specific
598 if (SrcAddrSpace != 0)
599 report_fatal_error("Cannot cast between two non-generic address spaces");
600 unsigned Opc;
601 switch (DstAddrSpace) {
602 default: report_fatal_error("Bad address space in addrspacecast");
603 case ADDRESS_SPACE_GLOBAL:
Eric Christopher02389e32015-02-19 00:08:27 +0000[diff] [blame]604 Opc = TM.is64Bit() ? NVPTX::cvta_to_global_yes_64
605 : NVPTX::cvta_to_global_yes;
Justin Holewinskiba2fa6d2014-03-24 11:17:53 +0000[diff] [blame]606 break;
607 case ADDRESS_SPACE_SHARED:
Eric Christopher02389e32015-02-19 00:08:27 +0000[diff] [blame]608 Opc = TM.is64Bit() ? NVPTX::cvta_to_shared_yes_64
609 : NVPTX::cvta_to_shared_yes;
Justin Holewinskiba2fa6d2014-03-24 11:17:53 +0000[diff] [blame]610 break;
611 case ADDRESS_SPACE_CONST:
Eric Christopher02389e32015-02-19 00:08:27 +0000[diff] [blame]612 Opc =
613 TM.is64Bit() ? NVPTX::cvta_to_const_yes_64 : NVPTX::cvta_to_const_yes;
Justin Holewinskiba2fa6d2014-03-24 11:17:53 +0000[diff] [blame]614 break;
615 case ADDRESS_SPACE_LOCAL:
Eric Christopher02389e32015-02-19 00:08:27 +0000[diff] [blame]616 Opc =
617 TM.is64Bit() ? NVPTX::cvta_to_local_yes_64 : NVPTX::cvta_to_local_yes;
Justin Holewinskiba2fa6d2014-03-24 11:17:53 +0000[diff] [blame]618 break;
619 }
620 return CurDAG->getMachineNode(Opc, SDLoc(N), N->getValueType(0), Src);
621 }
622}
623
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]624SDNode *NVPTXDAGToDAGISel::SelectLoad(SDNode *N) {
Andrew Trickef9de2a2013-05-25 02:42:55 +0000[diff] [blame]625 SDLoc dl(N);
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]626 LoadSDNode *LD = cast<LoadSDNode>(N);
627 EVT LoadedVT = LD->getMemoryVT();
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]628 SDNode *NVPTXLD = nullptr;
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]629
630 // do not support pre/post inc/dec
631 if (LD->isIndexed())
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]632 return nullptr;
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]633
634 if (!LoadedVT.isSimple())
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]635 return nullptr;
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]636
637 // Address Space Setting
Eric Christopher9745b3a2015-01-30 01:41:01 +0000[diff] [blame]638 unsigned int codeAddrSpace = getCodeAddrSpace(LD);
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]639
640 // Volatile Setting
641 // - .volatile is only availalble for .global and .shared
642 bool isVolatile = LD->isVolatile();
643 if (codeAddrSpace != NVPTX::PTXLdStInstCode::GLOBAL &&
644 codeAddrSpace != NVPTX::PTXLdStInstCode::SHARED &&
645 codeAddrSpace != NVPTX::PTXLdStInstCode::GENERIC)
646 isVolatile = false;
647
648 // Vector Setting
649 MVT SimpleVT = LoadedVT.getSimpleVT();
650 unsigned vecType = NVPTX::PTXLdStInstCode::Scalar;
651 if (SimpleVT.isVector()) {
652 unsigned num = SimpleVT.getVectorNumElements();
653 if (num == 2)
654 vecType = NVPTX::PTXLdStInstCode::V2;
655 else if (num == 4)
656 vecType = NVPTX::PTXLdStInstCode::V4;
657 else
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]658 return nullptr;
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]659 }
660
661 // Type Setting: fromType + fromTypeWidth
662 //
663 // Sign : ISD::SEXTLOAD
664 // Unsign : ISD::ZEXTLOAD, ISD::NON_EXTLOAD or ISD::EXTLOAD and the
665 // type is integer
666 // Float : ISD::NON_EXTLOAD or ISD::EXTLOAD and the type is float
667 MVT ScalarVT = SimpleVT.getScalarType();
Justin Holewinski994d66a2013-05-30 12:22:39 +0000[diff] [blame]668 // Read at least 8 bits (predicates are stored as 8-bit values)
669 unsigned fromTypeWidth = std::max(8U, ScalarVT.getSizeInBits());
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]670 unsigned int fromType;
671 if ((LD->getExtensionType() == ISD::SEXTLOAD))
672 fromType = NVPTX::PTXLdStInstCode::Signed;
673 else if (ScalarVT.isFloatingPoint())
674 fromType = NVPTX::PTXLdStInstCode::Float;
675 else
676 fromType = NVPTX::PTXLdStInstCode::Unsigned;
677
678 // Create the machine instruction DAG
679 SDValue Chain = N->getOperand(0);
680 SDValue N1 = N->getOperand(1);
681 SDValue Addr;
682 SDValue Offset, Base;
683 unsigned Opcode;
Craig Topperd9c27832013-08-15 02:44:19 +0000[diff] [blame]684 MVT::SimpleValueType TargetVT = LD->getSimpleValueType(0).SimpleTy;
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]685
686 if (SelectDirectAddr(N1, Addr)) {
687 switch (TargetVT) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]688 case MVT::i8:
689 Opcode = NVPTX::LD_i8_avar;
690 break;
691 case MVT::i16:
692 Opcode = NVPTX::LD_i16_avar;
693 break;
694 case MVT::i32:
695 Opcode = NVPTX::LD_i32_avar;
696 break;
697 case MVT::i64:
698 Opcode = NVPTX::LD_i64_avar;
699 break;
700 case MVT::f32:
701 Opcode = NVPTX::LD_f32_avar;
702 break;
703 case MVT::f64:
704 Opcode = NVPTX::LD_f64_avar;
705 break;
706 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]707 return nullptr;
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]708 }
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]709 SDValue Ops[] = { getI32Imm(isVolatile), getI32Imm(codeAddrSpace),
710 getI32Imm(vecType), getI32Imm(fromType),
711 getI32Imm(fromTypeWidth), Addr, Chain };
Michael Liaob53d8962013-04-19 22:22:57 +0000[diff] [blame]712 NVPTXLD = CurDAG->getMachineNode(Opcode, dl, TargetVT, MVT::Other, Ops);
Eric Christopher02389e32015-02-19 00:08:27 +0000[diff] [blame]713 } else if (TM.is64Bit() ? SelectADDRsi64(N1.getNode(), N1, Base, Offset)
714 : SelectADDRsi(N1.getNode(), N1, Base, Offset)) {
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]715 switch (TargetVT) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]716 case MVT::i8:
717 Opcode = NVPTX::LD_i8_asi;
718 break;
719 case MVT::i16:
720 Opcode = NVPTX::LD_i16_asi;
721 break;
722 case MVT::i32:
723 Opcode = NVPTX::LD_i32_asi;
724 break;
725 case MVT::i64:
726 Opcode = NVPTX::LD_i64_asi;
727 break;
728 case MVT::f32:
729 Opcode = NVPTX::LD_f32_asi;
730 break;
731 case MVT::f64:
732 Opcode = NVPTX::LD_f64_asi;
733 break;
734 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]735 return nullptr;
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]736 }
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]737 SDValue Ops[] = { getI32Imm(isVolatile), getI32Imm(codeAddrSpace),
738 getI32Imm(vecType), getI32Imm(fromType),
739 getI32Imm(fromTypeWidth), Base, Offset, Chain };
Michael Liaob53d8962013-04-19 22:22:57 +0000[diff] [blame]740 NVPTXLD = CurDAG->getMachineNode(Opcode, dl, TargetVT, MVT::Other, Ops);
Eric Christopher02389e32015-02-19 00:08:27 +0000[diff] [blame]741 } else if (TM.is64Bit() ? SelectADDRri64(N1.getNode(), N1, Base, Offset)
742 : SelectADDRri(N1.getNode(), N1, Base, Offset)) {
743 if (TM.is64Bit()) {
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]744 switch (TargetVT) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]745 case MVT::i8:
746 Opcode = NVPTX::LD_i8_ari_64;
747 break;
748 case MVT::i16:
749 Opcode = NVPTX::LD_i16_ari_64;
750 break;
751 case MVT::i32:
752 Opcode = NVPTX::LD_i32_ari_64;
753 break;
754 case MVT::i64:
755 Opcode = NVPTX::LD_i64_ari_64;
756 break;
757 case MVT::f32:
758 Opcode = NVPTX::LD_f32_ari_64;
759 break;
760 case MVT::f64:
761 Opcode = NVPTX::LD_f64_ari_64;
762 break;
763 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]764 return nullptr;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]765 }
766 } else {
767 switch (TargetVT) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]768 case MVT::i8:
769 Opcode = NVPTX::LD_i8_ari;
770 break;
771 case MVT::i16:
772 Opcode = NVPTX::LD_i16_ari;
773 break;
774 case MVT::i32:
775 Opcode = NVPTX::LD_i32_ari;
776 break;
777 case MVT::i64:
778 Opcode = NVPTX::LD_i64_ari;
779 break;
780 case MVT::f32:
781 Opcode = NVPTX::LD_f32_ari;
782 break;
783 case MVT::f64:
784 Opcode = NVPTX::LD_f64_ari;
785 break;
786 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]787 return nullptr;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]788 }
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]789 }
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]790 SDValue Ops[] = { getI32Imm(isVolatile), getI32Imm(codeAddrSpace),
791 getI32Imm(vecType), getI32Imm(fromType),
792 getI32Imm(fromTypeWidth), Base, Offset, Chain };
Michael Liaob53d8962013-04-19 22:22:57 +0000[diff] [blame]793 NVPTXLD = CurDAG->getMachineNode(Opcode, dl, TargetVT, MVT::Other, Ops);
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]794 } else {
Eric Christopher02389e32015-02-19 00:08:27 +0000[diff] [blame]795 if (TM.is64Bit()) {
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]796 switch (TargetVT) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]797 case MVT::i8:
798 Opcode = NVPTX::LD_i8_areg_64;
799 break;
800 case MVT::i16:
801 Opcode = NVPTX::LD_i16_areg_64;
802 break;
803 case MVT::i32:
804 Opcode = NVPTX::LD_i32_areg_64;
805 break;
806 case MVT::i64:
807 Opcode = NVPTX::LD_i64_areg_64;
808 break;
809 case MVT::f32:
810 Opcode = NVPTX::LD_f32_areg_64;
811 break;
812 case MVT::f64:
813 Opcode = NVPTX::LD_f64_areg_64;
814 break;
815 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]816 return nullptr;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]817 }
818 } else {
819 switch (TargetVT) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]820 case MVT::i8:
821 Opcode = NVPTX::LD_i8_areg;
822 break;
823 case MVT::i16:
824 Opcode = NVPTX::LD_i16_areg;
825 break;
826 case MVT::i32:
827 Opcode = NVPTX::LD_i32_areg;
828 break;
829 case MVT::i64:
830 Opcode = NVPTX::LD_i64_areg;
831 break;
832 case MVT::f32:
833 Opcode = NVPTX::LD_f32_areg;
834 break;
835 case MVT::f64:
836 Opcode = NVPTX::LD_f64_areg;
837 break;
838 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]839 return nullptr;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]840 }
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]841 }
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]842 SDValue Ops[] = { getI32Imm(isVolatile), getI32Imm(codeAddrSpace),
843 getI32Imm(vecType), getI32Imm(fromType),
844 getI32Imm(fromTypeWidth), N1, Chain };
Michael Liaob53d8962013-04-19 22:22:57 +0000[diff] [blame]845 NVPTXLD = CurDAG->getMachineNode(Opcode, dl, TargetVT, MVT::Other, Ops);
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]846 }
847
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]848 if (NVPTXLD) {
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]849 MachineSDNode::mmo_iterator MemRefs0 = MF->allocateMemRefsArray(1);
850 MemRefs0[0] = cast<MemSDNode>(N)->getMemOperand();
851 cast<MachineSDNode>(NVPTXLD)->setMemRefs(MemRefs0, MemRefs0 + 1);
852 }
853
854 return NVPTXLD;
855}
856
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]857SDNode *NVPTXDAGToDAGISel::SelectLoadVector(SDNode *N) {
858
859 SDValue Chain = N->getOperand(0);
860 SDValue Op1 = N->getOperand(1);
861 SDValue Addr, Offset, Base;
862 unsigned Opcode;
Andrew Trickef9de2a2013-05-25 02:42:55 +0000[diff] [blame]863 SDLoc DL(N);
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]864 SDNode *LD;
865 MemSDNode *MemSD = cast<MemSDNode>(N);
866 EVT LoadedVT = MemSD->getMemoryVT();
867
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]868 if (!LoadedVT.isSimple())
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]869 return nullptr;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]870
871 // Address Space Setting
Eric Christopher9745b3a2015-01-30 01:41:01 +0000[diff] [blame]872 unsigned int CodeAddrSpace = getCodeAddrSpace(MemSD);
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]873
874 // Volatile Setting
875 // - .volatile is only availalble for .global and .shared
876 bool IsVolatile = MemSD->isVolatile();
877 if (CodeAddrSpace != NVPTX::PTXLdStInstCode::GLOBAL &&
878 CodeAddrSpace != NVPTX::PTXLdStInstCode::SHARED &&
879 CodeAddrSpace != NVPTX::PTXLdStInstCode::GENERIC)
880 IsVolatile = false;
881
882 // Vector Setting
883 MVT SimpleVT = LoadedVT.getSimpleVT();
884
885 // Type Setting: fromType + fromTypeWidth
886 //
887 // Sign : ISD::SEXTLOAD
888 // Unsign : ISD::ZEXTLOAD, ISD::NON_EXTLOAD or ISD::EXTLOAD and the
889 // type is integer
890 // Float : ISD::NON_EXTLOAD or ISD::EXTLOAD and the type is float
891 MVT ScalarVT = SimpleVT.getScalarType();
Justin Holewinski994d66a2013-05-30 12:22:39 +0000[diff] [blame]892 // Read at least 8 bits (predicates are stored as 8-bit values)
893 unsigned FromTypeWidth = std::max(8U, ScalarVT.getSizeInBits());
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]894 unsigned int FromType;
895 // The last operand holds the original LoadSDNode::getExtensionType() value
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]896 unsigned ExtensionType = cast<ConstantSDNode>(
897 N->getOperand(N->getNumOperands() - 1))->getZExtValue();
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]898 if (ExtensionType == ISD::SEXTLOAD)
899 FromType = NVPTX::PTXLdStInstCode::Signed;
900 else if (ScalarVT.isFloatingPoint())
901 FromType = NVPTX::PTXLdStInstCode::Float;
902 else
903 FromType = NVPTX::PTXLdStInstCode::Unsigned;
904
905 unsigned VecType;
906
907 switch (N->getOpcode()) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]908 case NVPTXISD::LoadV2:
909 VecType = NVPTX::PTXLdStInstCode::V2;
910 break;
911 case NVPTXISD::LoadV4:
912 VecType = NVPTX::PTXLdStInstCode::V4;
913 break;
914 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]915 return nullptr;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]916 }
917
918 EVT EltVT = N->getValueType(0);
919
920 if (SelectDirectAddr(Op1, Addr)) {
921 switch (N->getOpcode()) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]922 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]923 return nullptr;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]924 case NVPTXISD::LoadV2:
925 switch (EltVT.getSimpleVT().SimpleTy) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]926 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]927 return nullptr;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]928 case MVT::i8:
929 Opcode = NVPTX::LDV_i8_v2_avar;
930 break;
931 case MVT::i16:
932 Opcode = NVPTX::LDV_i16_v2_avar;
933 break;
934 case MVT::i32:
935 Opcode = NVPTX::LDV_i32_v2_avar;
936 break;
937 case MVT::i64:
938 Opcode = NVPTX::LDV_i64_v2_avar;
939 break;
940 case MVT::f32:
941 Opcode = NVPTX::LDV_f32_v2_avar;
942 break;
943 case MVT::f64:
944 Opcode = NVPTX::LDV_f64_v2_avar;
945 break;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]946 }
947 break;
948 case NVPTXISD::LoadV4:
949 switch (EltVT.getSimpleVT().SimpleTy) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]950 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]951 return nullptr;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]952 case MVT::i8:
953 Opcode = NVPTX::LDV_i8_v4_avar;
954 break;
955 case MVT::i16:
956 Opcode = NVPTX::LDV_i16_v4_avar;
957 break;
958 case MVT::i32:
959 Opcode = NVPTX::LDV_i32_v4_avar;
960 break;
961 case MVT::f32:
962 Opcode = NVPTX::LDV_f32_v4_avar;
963 break;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]964 }
965 break;
966 }
967
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]968 SDValue Ops[] = { getI32Imm(IsVolatile), getI32Imm(CodeAddrSpace),
969 getI32Imm(VecType), getI32Imm(FromType),
970 getI32Imm(FromTypeWidth), Addr, Chain };
Michael Liaob53d8962013-04-19 22:22:57 +0000[diff] [blame]971 LD = CurDAG->getMachineNode(Opcode, DL, N->getVTList(), Ops);
Eric Christopher02389e32015-02-19 00:08:27 +0000[diff] [blame]972 } else if (TM.is64Bit() ? SelectADDRsi64(Op1.getNode(), Op1, Base, Offset)
973 : SelectADDRsi(Op1.getNode(), Op1, Base, Offset)) {
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]974 switch (N->getOpcode()) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]975 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]976 return nullptr;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]977 case NVPTXISD::LoadV2:
978 switch (EltVT.getSimpleVT().SimpleTy) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]979 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]980 return nullptr;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]981 case MVT::i8:
982 Opcode = NVPTX::LDV_i8_v2_asi;
983 break;
984 case MVT::i16:
985 Opcode = NVPTX::LDV_i16_v2_asi;
986 break;
987 case MVT::i32:
988 Opcode = NVPTX::LDV_i32_v2_asi;
989 break;
990 case MVT::i64:
991 Opcode = NVPTX::LDV_i64_v2_asi;
992 break;
993 case MVT::f32:
994 Opcode = NVPTX::LDV_f32_v2_asi;
995 break;
996 case MVT::f64:
997 Opcode = NVPTX::LDV_f64_v2_asi;
998 break;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]999 }
1000 break;
1001 case NVPTXISD::LoadV4:
1002 switch (EltVT.getSimpleVT().SimpleTy) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1003 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]1004 return nullptr;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1005 case MVT::i8:
1006 Opcode = NVPTX::LDV_i8_v4_asi;
1007 break;
1008 case MVT::i16:
1009 Opcode = NVPTX::LDV_i16_v4_asi;
1010 break;
1011 case MVT::i32:
1012 Opcode = NVPTX::LDV_i32_v4_asi;
1013 break;
1014 case MVT::f32:
1015 Opcode = NVPTX::LDV_f32_v4_asi;
1016 break;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]1017 }
1018 break;
1019 }
1020
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1021 SDValue Ops[] = { getI32Imm(IsVolatile), getI32Imm(CodeAddrSpace),
1022 getI32Imm(VecType), getI32Imm(FromType),
1023 getI32Imm(FromTypeWidth), Base, Offset, Chain };
Michael Liaob53d8962013-04-19 22:22:57 +0000[diff] [blame]1024 LD = CurDAG->getMachineNode(Opcode, DL, N->getVTList(), Ops);
Eric Christopher02389e32015-02-19 00:08:27 +0000[diff] [blame]1025 } else if (TM.is64Bit() ? SelectADDRri64(Op1.getNode(), Op1, Base, Offset)
1026 : SelectADDRri(Op1.getNode(), Op1, Base, Offset)) {
1027 if (TM.is64Bit()) {
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]1028 switch (N->getOpcode()) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1029 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]1030 return nullptr;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]1031 case NVPTXISD::LoadV2:
1032 switch (EltVT.getSimpleVT().SimpleTy) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1033 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]1034 return nullptr;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1035 case MVT::i8:
1036 Opcode = NVPTX::LDV_i8_v2_ari_64;
1037 break;
1038 case MVT::i16:
1039 Opcode = NVPTX::LDV_i16_v2_ari_64;
1040 break;
1041 case MVT::i32:
1042 Opcode = NVPTX::LDV_i32_v2_ari_64;
1043 break;
1044 case MVT::i64:
1045 Opcode = NVPTX::LDV_i64_v2_ari_64;
1046 break;
1047 case MVT::f32:
1048 Opcode = NVPTX::LDV_f32_v2_ari_64;
1049 break;
1050 case MVT::f64:
1051 Opcode = NVPTX::LDV_f64_v2_ari_64;
1052 break;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]1053 }
1054 break;
1055 case NVPTXISD::LoadV4:
1056 switch (EltVT.getSimpleVT().SimpleTy) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1057 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]1058 return nullptr;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1059 case MVT::i8:
1060 Opcode = NVPTX::LDV_i8_v4_ari_64;
1061 break;
1062 case MVT::i16:
1063 Opcode = NVPTX::LDV_i16_v4_ari_64;
1064 break;
1065 case MVT::i32:
1066 Opcode = NVPTX::LDV_i32_v4_ari_64;
1067 break;
1068 case MVT::f32:
1069 Opcode = NVPTX::LDV_f32_v4_ari_64;
1070 break;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]1071 }
1072 break;
1073 }
1074 } else {
1075 switch (N->getOpcode()) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1076 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]1077 return nullptr;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]1078 case NVPTXISD::LoadV2:
1079 switch (EltVT.getSimpleVT().SimpleTy) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1080 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]1081 return nullptr;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1082 case MVT::i8:
1083 Opcode = NVPTX::LDV_i8_v2_ari;
1084 break;
1085 case MVT::i16:
1086 Opcode = NVPTX::LDV_i16_v2_ari;
1087 break;
1088 case MVT::i32:
1089 Opcode = NVPTX::LDV_i32_v2_ari;
1090 break;
1091 case MVT::i64:
1092 Opcode = NVPTX::LDV_i64_v2_ari;
1093 break;
1094 case MVT::f32:
1095 Opcode = NVPTX::LDV_f32_v2_ari;
1096 break;
1097 case MVT::f64:
1098 Opcode = NVPTX::LDV_f64_v2_ari;
1099 break;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]1100 }
1101 break;
1102 case NVPTXISD::LoadV4:
1103 switch (EltVT.getSimpleVT().SimpleTy) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1104 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]1105 return nullptr;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1106 case MVT::i8:
1107 Opcode = NVPTX::LDV_i8_v4_ari;
1108 break;
1109 case MVT::i16:
1110 Opcode = NVPTX::LDV_i16_v4_ari;
1111 break;
1112 case MVT::i32:
1113 Opcode = NVPTX::LDV_i32_v4_ari;
1114 break;
1115 case MVT::f32:
1116 Opcode = NVPTX::LDV_f32_v4_ari;
1117 break;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]1118 }
1119 break;
1120 }
1121 }
1122
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1123 SDValue Ops[] = { getI32Imm(IsVolatile), getI32Imm(CodeAddrSpace),
1124 getI32Imm(VecType), getI32Imm(FromType),
1125 getI32Imm(FromTypeWidth), Base, Offset, Chain };
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]1126
Michael Liaob53d8962013-04-19 22:22:57 +0000[diff] [blame]1127 LD = CurDAG->getMachineNode(Opcode, DL, N->getVTList(), Ops);
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]1128 } else {
Eric Christopher02389e32015-02-19 00:08:27 +0000[diff] [blame]1129 if (TM.is64Bit()) {
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]1130 switch (N->getOpcode()) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1131 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]1132 return nullptr;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]1133 case NVPTXISD::LoadV2:
1134 switch (EltVT.getSimpleVT().SimpleTy) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1135 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]1136 return nullptr;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1137 case MVT::i8:
1138 Opcode = NVPTX::LDV_i8_v2_areg_64;
1139 break;
1140 case MVT::i16:
1141 Opcode = NVPTX::LDV_i16_v2_areg_64;
1142 break;
1143 case MVT::i32:
1144 Opcode = NVPTX::LDV_i32_v2_areg_64;
1145 break;
1146 case MVT::i64:
1147 Opcode = NVPTX::LDV_i64_v2_areg_64;
1148 break;
1149 case MVT::f32:
1150 Opcode = NVPTX::LDV_f32_v2_areg_64;
1151 break;
1152 case MVT::f64:
1153 Opcode = NVPTX::LDV_f64_v2_areg_64;
1154 break;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]1155 }
1156 break;
1157 case NVPTXISD::LoadV4:
1158 switch (EltVT.getSimpleVT().SimpleTy) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1159 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]1160 return nullptr;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1161 case MVT::i8:
1162 Opcode = NVPTX::LDV_i8_v4_areg_64;
1163 break;
1164 case MVT::i16:
1165 Opcode = NVPTX::LDV_i16_v4_areg_64;
1166 break;
1167 case MVT::i32:
1168 Opcode = NVPTX::LDV_i32_v4_areg_64;
1169 break;
1170 case MVT::f32:
1171 Opcode = NVPTX::LDV_f32_v4_areg_64;
1172 break;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]1173 }
1174 break;
1175 }
1176 } else {
1177 switch (N->getOpcode()) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1178 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]1179 return nullptr;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]1180 case NVPTXISD::LoadV2:
1181 switch (EltVT.getSimpleVT().SimpleTy) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1182 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]1183 return nullptr;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1184 case MVT::i8:
1185 Opcode = NVPTX::LDV_i8_v2_areg;
1186 break;
1187 case MVT::i16:
1188 Opcode = NVPTX::LDV_i16_v2_areg;
1189 break;
1190 case MVT::i32:
1191 Opcode = NVPTX::LDV_i32_v2_areg;
1192 break;
1193 case MVT::i64:
1194 Opcode = NVPTX::LDV_i64_v2_areg;
1195 break;
1196 case MVT::f32:
1197 Opcode = NVPTX::LDV_f32_v2_areg;
1198 break;
1199 case MVT::f64:
1200 Opcode = NVPTX::LDV_f64_v2_areg;
1201 break;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]1202 }
1203 break;
1204 case NVPTXISD::LoadV4:
1205 switch (EltVT.getSimpleVT().SimpleTy) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1206 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]1207 return nullptr;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1208 case MVT::i8:
1209 Opcode = NVPTX::LDV_i8_v4_areg;
1210 break;
1211 case MVT::i16:
1212 Opcode = NVPTX::LDV_i16_v4_areg;
1213 break;
1214 case MVT::i32:
1215 Opcode = NVPTX::LDV_i32_v4_areg;
1216 break;
1217 case MVT::f32:
1218 Opcode = NVPTX::LDV_f32_v4_areg;
1219 break;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]1220 }
1221 break;
1222 }
1223 }
1224
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1225 SDValue Ops[] = { getI32Imm(IsVolatile), getI32Imm(CodeAddrSpace),
1226 getI32Imm(VecType), getI32Imm(FromType),
1227 getI32Imm(FromTypeWidth), Op1, Chain };
Michael Liaob53d8962013-04-19 22:22:57 +0000[diff] [blame]1228 LD = CurDAG->getMachineNode(Opcode, DL, N->getVTList(), Ops);
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]1229 }
1230
1231 MachineSDNode::mmo_iterator MemRefs0 = MF->allocateMemRefsArray(1);
1232 MemRefs0[0] = cast<MemSDNode>(N)->getMemOperand();
1233 cast<MachineSDNode>(LD)->setMemRefs(MemRefs0, MemRefs0 + 1);
1234
1235 return LD;
1236}
1237
Justin Holewinskib926d9d2014-06-27 18:35:51 +0000[diff] [blame]1238SDNode *NVPTXDAGToDAGISel::SelectLDGLDU(SDNode *N) {
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]1239
1240 SDValue Chain = N->getOperand(0);
Justin Holewinskib926d9d2014-06-27 18:35:51 +0000[diff] [blame]1241 SDValue Op1;
1242 MemSDNode *Mem;
1243 bool IsLDG = true;
1244
1245 // If this is an LDG intrinsic, the address is the third operand. Its its an
1246 // LDG/LDU SD node (from custom vector handling), then its the second operand
1247 if (N->getOpcode() == ISD::INTRINSIC_W_CHAIN) {
1248 Op1 = N->getOperand(2);
1249 Mem = cast<MemIntrinsicSDNode>(N);
1250 unsigned IID = cast<ConstantSDNode>(N->getOperand(1))->getZExtValue();
1251 switch (IID) {
1252 default:
1253 return NULL;
1254 case Intrinsic::nvvm_ldg_global_f:
1255 case Intrinsic::nvvm_ldg_global_i:
1256 case Intrinsic::nvvm_ldg_global_p:
1257 IsLDG = true;
1258 break;
1259 case Intrinsic::nvvm_ldu_global_f:
1260 case Intrinsic::nvvm_ldu_global_i:
1261 case Intrinsic::nvvm_ldu_global_p:
1262 IsLDG = false;
1263 break;
1264 }
1265 } else {
1266 Op1 = N->getOperand(1);
1267 Mem = cast<MemSDNode>(N);
1268 }
1269
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]1270 unsigned Opcode;
Andrew Trickef9de2a2013-05-25 02:42:55 +0000[diff] [blame]1271 SDLoc DL(N);
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]1272 SDNode *LD;
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1273 SDValue Base, Offset, Addr;
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]1274
Justin Holewinskib926d9d2014-06-27 18:35:51 +0000[diff] [blame]1275 EVT EltVT = Mem->getMemoryVT();
1276 if (EltVT.isVector()) {
1277 EltVT = EltVT.getVectorElementType();
1278 }
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]1279
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1280 if (SelectDirectAddr(Op1, Addr)) {
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]1281 switch (N->getOpcode()) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1282 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]1283 return nullptr;
Justin Holewinskib926d9d2014-06-27 18:35:51 +0000[diff] [blame]1284 case ISD::INTRINSIC_W_CHAIN:
1285 if (IsLDG) {
1286 switch (EltVT.getSimpleVT().SimpleTy) {
1287 default:
1288 return nullptr;
1289 case MVT::i8:
1290 Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i8avar;
1291 break;
1292 case MVT::i16:
1293 Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i16avar;
1294 break;
1295 case MVT::i32:
1296 Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i32avar;
1297 break;
1298 case MVT::i64:
1299 Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i64avar;
1300 break;
1301 case MVT::f32:
1302 Opcode = NVPTX::INT_PTX_LDG_GLOBAL_f32avar;
1303 break;
1304 case MVT::f64:
1305 Opcode = NVPTX::INT_PTX_LDG_GLOBAL_f64avar;
1306 break;
1307 }
1308 } else {
1309 switch (EltVT.getSimpleVT().SimpleTy) {
1310 default:
1311 return nullptr;
1312 case MVT::i8:
1313 Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i8avar;
1314 break;
1315 case MVT::i16:
1316 Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i16avar;
1317 break;
1318 case MVT::i32:
1319 Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i32avar;
1320 break;
1321 case MVT::i64:
1322 Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i64avar;
1323 break;
1324 case MVT::f32:
1325 Opcode = NVPTX::INT_PTX_LDU_GLOBAL_f32avar;
1326 break;
1327 case MVT::f64:
1328 Opcode = NVPTX::INT_PTX_LDU_GLOBAL_f64avar;
1329 break;
1330 }
1331 }
1332 break;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]1333 case NVPTXISD::LDGV2:
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1334 switch (EltVT.getSimpleVT().SimpleTy) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1335 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]1336 return nullptr;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1337 case MVT::i8:
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1338 Opcode = NVPTX::INT_PTX_LDG_G_v2i8_ELE_avar;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1339 break;
1340 case MVT::i16:
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1341 Opcode = NVPTX::INT_PTX_LDG_G_v2i16_ELE_avar;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1342 break;
1343 case MVT::i32:
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1344 Opcode = NVPTX::INT_PTX_LDG_G_v2i32_ELE_avar;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1345 break;
1346 case MVT::i64:
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1347 Opcode = NVPTX::INT_PTX_LDG_G_v2i64_ELE_avar;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1348 break;
1349 case MVT::f32:
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1350 Opcode = NVPTX::INT_PTX_LDG_G_v2f32_ELE_avar;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1351 break;
1352 case MVT::f64:
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1353 Opcode = NVPTX::INT_PTX_LDG_G_v2f64_ELE_avar;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1354 break;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]1355 }
1356 break;
1357 case NVPTXISD::LDUV2:
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1358 switch (EltVT.getSimpleVT().SimpleTy) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1359 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]1360 return nullptr;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1361 case MVT::i8:
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1362 Opcode = NVPTX::INT_PTX_LDU_G_v2i8_ELE_avar;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1363 break;
1364 case MVT::i16:
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1365 Opcode = NVPTX::INT_PTX_LDU_G_v2i16_ELE_avar;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1366 break;
1367 case MVT::i32:
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1368 Opcode = NVPTX::INT_PTX_LDU_G_v2i32_ELE_avar;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1369 break;
1370 case MVT::i64:
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1371 Opcode = NVPTX::INT_PTX_LDU_G_v2i64_ELE_avar;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1372 break;
1373 case MVT::f32:
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1374 Opcode = NVPTX::INT_PTX_LDU_G_v2f32_ELE_avar;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1375 break;
1376 case MVT::f64:
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1377 Opcode = NVPTX::INT_PTX_LDU_G_v2f64_ELE_avar;
1378 break;
1379 }
1380 break;
1381 case NVPTXISD::LDGV4:
1382 switch (EltVT.getSimpleVT().SimpleTy) {
1383 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]1384 return nullptr;
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1385 case MVT::i8:
1386 Opcode = NVPTX::INT_PTX_LDG_G_v4i8_ELE_avar;
1387 break;
1388 case MVT::i16:
1389 Opcode = NVPTX::INT_PTX_LDG_G_v4i16_ELE_avar;
1390 break;
1391 case MVT::i32:
1392 Opcode = NVPTX::INT_PTX_LDG_G_v4i32_ELE_avar;
1393 break;
1394 case MVT::f32:
1395 Opcode = NVPTX::INT_PTX_LDG_G_v4f32_ELE_avar;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1396 break;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]1397 }
1398 break;
1399 case NVPTXISD::LDUV4:
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1400 switch (EltVT.getSimpleVT().SimpleTy) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1401 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]1402 return nullptr;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1403 case MVT::i8:
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1404 Opcode = NVPTX::INT_PTX_LDU_G_v4i8_ELE_avar;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1405 break;
1406 case MVT::i16:
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1407 Opcode = NVPTX::INT_PTX_LDU_G_v4i16_ELE_avar;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1408 break;
1409 case MVT::i32:
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1410 Opcode = NVPTX::INT_PTX_LDU_G_v4i32_ELE_avar;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1411 break;
1412 case MVT::f32:
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1413 Opcode = NVPTX::INT_PTX_LDU_G_v4f32_ELE_avar;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1414 break;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]1415 }
1416 break;
1417 }
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1418
1419 SDValue Ops[] = { Addr, Chain };
Craig Topper2d2aa0c2014-04-30 07:17:30 +0000[diff] [blame]1420 LD = CurDAG->getMachineNode(Opcode, DL, N->getVTList(), Ops);
Eric Christopher02389e32015-02-19 00:08:27 +0000[diff] [blame]1421 } else if (TM.is64Bit() ? SelectADDRri64(Op1.getNode(), Op1, Base, Offset)
1422 : SelectADDRri(Op1.getNode(), Op1, Base, Offset)) {
1423 if (TM.is64Bit()) {
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1424 switch (N->getOpcode()) {
1425 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]1426 return nullptr;
Justin Holewinskib926d9d2014-06-27 18:35:51 +0000[diff] [blame]1427 case ISD::INTRINSIC_W_CHAIN:
1428 if (IsLDG) {
1429 switch (EltVT.getSimpleVT().SimpleTy) {
1430 default:
1431 return nullptr;
1432 case MVT::i8:
1433 Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i8ari64;
1434 break;
1435 case MVT::i16:
1436 Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i16ari64;
1437 break;
1438 case MVT::i32:
1439 Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i32ari64;
1440 break;
1441 case MVT::i64:
1442 Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i64ari64;
1443 break;
1444 case MVT::f32:
1445 Opcode = NVPTX::INT_PTX_LDG_GLOBAL_f32ari64;
1446 break;
1447 case MVT::f64:
1448 Opcode = NVPTX::INT_PTX_LDG_GLOBAL_f64ari64;
1449 break;
1450 }
1451 } else {
1452 switch (EltVT.getSimpleVT().SimpleTy) {
1453 default:
1454 return nullptr;
1455 case MVT::i8:
1456 Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i8ari64;
1457 break;
1458 case MVT::i16:
1459 Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i16ari64;
1460 break;
1461 case MVT::i32:
1462 Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i32ari64;
1463 break;
1464 case MVT::i64:
1465 Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i64ari64;
1466 break;
1467 case MVT::f32:
1468 Opcode = NVPTX::INT_PTX_LDU_GLOBAL_f32ari64;
1469 break;
1470 case MVT::f64:
1471 Opcode = NVPTX::INT_PTX_LDU_GLOBAL_f64ari64;
1472 break;
1473 }
1474 }
1475 break;
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1476 case NVPTXISD::LDGV2:
1477 switch (EltVT.getSimpleVT().SimpleTy) {
1478 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]1479 return nullptr;
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1480 case MVT::i8:
1481 Opcode = NVPTX::INT_PTX_LDG_G_v2i8_ELE_ari64;
1482 break;
1483 case MVT::i16:
1484 Opcode = NVPTX::INT_PTX_LDG_G_v2i16_ELE_ari64;
1485 break;
1486 case MVT::i32:
1487 Opcode = NVPTX::INT_PTX_LDG_G_v2i32_ELE_ari64;
1488 break;
1489 case MVT::i64:
1490 Opcode = NVPTX::INT_PTX_LDG_G_v2i64_ELE_ari64;
1491 break;
1492 case MVT::f32:
1493 Opcode = NVPTX::INT_PTX_LDG_G_v2f32_ELE_ari64;
1494 break;
1495 case MVT::f64:
1496 Opcode = NVPTX::INT_PTX_LDG_G_v2f64_ELE_ari64;
1497 break;
1498 }
1499 break;
1500 case NVPTXISD::LDUV2:
1501 switch (EltVT.getSimpleVT().SimpleTy) {
1502 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]1503 return nullptr;
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1504 case MVT::i8:
1505 Opcode = NVPTX::INT_PTX_LDU_G_v2i8_ELE_ari64;
1506 break;
1507 case MVT::i16:
1508 Opcode = NVPTX::INT_PTX_LDU_G_v2i16_ELE_ari64;
1509 break;
1510 case MVT::i32:
1511 Opcode = NVPTX::INT_PTX_LDU_G_v2i32_ELE_ari64;
1512 break;
1513 case MVT::i64:
1514 Opcode = NVPTX::INT_PTX_LDU_G_v2i64_ELE_ari64;
1515 break;
1516 case MVT::f32:
1517 Opcode = NVPTX::INT_PTX_LDU_G_v2f32_ELE_ari64;
1518 break;
1519 case MVT::f64:
1520 Opcode = NVPTX::INT_PTX_LDU_G_v2f64_ELE_ari64;
1521 break;
1522 }
1523 break;
1524 case NVPTXISD::LDGV4:
1525 switch (EltVT.getSimpleVT().SimpleTy) {
1526 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]1527 return nullptr;
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1528 case MVT::i8:
1529 Opcode = NVPTX::INT_PTX_LDG_G_v4i8_ELE_ari64;
1530 break;
1531 case MVT::i16:
1532 Opcode = NVPTX::INT_PTX_LDG_G_v4i16_ELE_ari64;
1533 break;
1534 case MVT::i32:
1535 Opcode = NVPTX::INT_PTX_LDG_G_v4i32_ELE_ari64;
1536 break;
1537 case MVT::f32:
1538 Opcode = NVPTX::INT_PTX_LDG_G_v4f32_ELE_ari64;
1539 break;
1540 }
1541 break;
1542 case NVPTXISD::LDUV4:
1543 switch (EltVT.getSimpleVT().SimpleTy) {
1544 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]1545 return nullptr;
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1546 case MVT::i8:
1547 Opcode = NVPTX::INT_PTX_LDU_G_v4i8_ELE_ari64;
1548 break;
1549 case MVT::i16:
1550 Opcode = NVPTX::INT_PTX_LDU_G_v4i16_ELE_ari64;
1551 break;
1552 case MVT::i32:
1553 Opcode = NVPTX::INT_PTX_LDU_G_v4i32_ELE_ari64;
1554 break;
1555 case MVT::f32:
1556 Opcode = NVPTX::INT_PTX_LDU_G_v4f32_ELE_ari64;
1557 break;
1558 }
1559 break;
1560 }
1561 } else {
1562 switch (N->getOpcode()) {
1563 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]1564 return nullptr;
Justin Holewinskib926d9d2014-06-27 18:35:51 +0000[diff] [blame]1565 case ISD::INTRINSIC_W_CHAIN:
1566 if (IsLDG) {
1567 switch (EltVT.getSimpleVT().SimpleTy) {
1568 default:
1569 return nullptr;
1570 case MVT::i8:
1571 Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i8ari;
1572 break;
1573 case MVT::i16:
1574 Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i16ari;
1575 break;
1576 case MVT::i32:
1577 Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i32ari;
1578 break;
1579 case MVT::i64:
1580 Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i64ari;
1581 break;
1582 case MVT::f32:
1583 Opcode = NVPTX::INT_PTX_LDG_GLOBAL_f32ari;
1584 break;
1585 case MVT::f64:
1586 Opcode = NVPTX::INT_PTX_LDG_GLOBAL_f64ari;
1587 break;
1588 }
1589 } else {
1590 switch (EltVT.getSimpleVT().SimpleTy) {
1591 default:
1592 return nullptr;
1593 case MVT::i8:
1594 Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i8ari;
1595 break;
1596 case MVT::i16:
1597 Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i16ari;
1598 break;
1599 case MVT::i32:
1600 Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i32ari;
1601 break;
1602 case MVT::i64:
1603 Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i64ari;
1604 break;
1605 case MVT::f32:
1606 Opcode = NVPTX::INT_PTX_LDU_GLOBAL_f32ari;
1607 break;
1608 case MVT::f64:
1609 Opcode = NVPTX::INT_PTX_LDU_GLOBAL_f64ari;
1610 break;
1611 }
1612 }
1613 break;
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1614 case NVPTXISD::LDGV2:
1615 switch (EltVT.getSimpleVT().SimpleTy) {
1616 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]1617 return nullptr;
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1618 case MVT::i8:
1619 Opcode = NVPTX::INT_PTX_LDG_G_v2i8_ELE_ari32;
1620 break;
1621 case MVT::i16:
1622 Opcode = NVPTX::INT_PTX_LDG_G_v2i16_ELE_ari32;
1623 break;
1624 case MVT::i32:
1625 Opcode = NVPTX::INT_PTX_LDG_G_v2i32_ELE_ari32;
1626 break;
1627 case MVT::i64:
1628 Opcode = NVPTX::INT_PTX_LDG_G_v2i64_ELE_ari32;
1629 break;
1630 case MVT::f32:
1631 Opcode = NVPTX::INT_PTX_LDG_G_v2f32_ELE_ari32;
1632 break;
1633 case MVT::f64:
1634 Opcode = NVPTX::INT_PTX_LDG_G_v2f64_ELE_ari32;
1635 break;
1636 }
1637 break;
1638 case NVPTXISD::LDUV2:
1639 switch (EltVT.getSimpleVT().SimpleTy) {
1640 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]1641 return nullptr;
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1642 case MVT::i8:
1643 Opcode = NVPTX::INT_PTX_LDU_G_v2i8_ELE_ari32;
1644 break;
1645 case MVT::i16:
1646 Opcode = NVPTX::INT_PTX_LDU_G_v2i16_ELE_ari32;
1647 break;
1648 case MVT::i32:
1649 Opcode = NVPTX::INT_PTX_LDU_G_v2i32_ELE_ari32;
1650 break;
1651 case MVT::i64:
1652 Opcode = NVPTX::INT_PTX_LDU_G_v2i64_ELE_ari32;
1653 break;
1654 case MVT::f32:
1655 Opcode = NVPTX::INT_PTX_LDU_G_v2f32_ELE_ari32;
1656 break;
1657 case MVT::f64:
1658 Opcode = NVPTX::INT_PTX_LDU_G_v2f64_ELE_ari32;
1659 break;
1660 }
1661 break;
1662 case NVPTXISD::LDGV4:
1663 switch (EltVT.getSimpleVT().SimpleTy) {
1664 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]1665 return nullptr;
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1666 case MVT::i8:
1667 Opcode = NVPTX::INT_PTX_LDG_G_v4i8_ELE_ari32;
1668 break;
1669 case MVT::i16:
1670 Opcode = NVPTX::INT_PTX_LDG_G_v4i16_ELE_ari32;
1671 break;
1672 case MVT::i32:
1673 Opcode = NVPTX::INT_PTX_LDG_G_v4i32_ELE_ari32;
1674 break;
1675 case MVT::f32:
1676 Opcode = NVPTX::INT_PTX_LDG_G_v4f32_ELE_ari32;
1677 break;
1678 }
1679 break;
1680 case NVPTXISD::LDUV4:
1681 switch (EltVT.getSimpleVT().SimpleTy) {
1682 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]1683 return nullptr;
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1684 case MVT::i8:
1685 Opcode = NVPTX::INT_PTX_LDU_G_v4i8_ELE_ari32;
1686 break;
1687 case MVT::i16:
1688 Opcode = NVPTX::INT_PTX_LDU_G_v4i16_ELE_ari32;
1689 break;
1690 case MVT::i32:
1691 Opcode = NVPTX::INT_PTX_LDU_G_v4i32_ELE_ari32;
1692 break;
1693 case MVT::f32:
1694 Opcode = NVPTX::INT_PTX_LDU_G_v4f32_ELE_ari32;
1695 break;
1696 }
1697 break;
1698 }
1699 }
1700
1701 SDValue Ops[] = { Base, Offset, Chain };
1702
Craig Topper2d2aa0c2014-04-30 07:17:30 +0000[diff] [blame]1703 LD = CurDAG->getMachineNode(Opcode, DL, N->getVTList(), Ops);
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]1704 } else {
Eric Christopher02389e32015-02-19 00:08:27 +0000[diff] [blame]1705 if (TM.is64Bit()) {
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1706 switch (N->getOpcode()) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1707 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]1708 return nullptr;
Justin Holewinskib926d9d2014-06-27 18:35:51 +0000[diff] [blame]1709 case ISD::INTRINSIC_W_CHAIN:
1710 if (IsLDG) {
1711 switch (EltVT.getSimpleVT().SimpleTy) {
1712 default:
1713 return nullptr;
1714 case MVT::i8:
1715 Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i8areg64;
1716 break;
1717 case MVT::i16:
1718 Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i16areg64;
1719 break;
1720 case MVT::i32:
1721 Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i32areg64;
1722 break;
1723 case MVT::i64:
1724 Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i64areg64;
1725 break;
1726 case MVT::f32:
1727 Opcode = NVPTX::INT_PTX_LDG_GLOBAL_f32areg64;
1728 break;
1729 case MVT::f64:
1730 Opcode = NVPTX::INT_PTX_LDG_GLOBAL_f64areg64;
1731 break;
1732 }
1733 } else {
1734 switch (EltVT.getSimpleVT().SimpleTy) {
1735 default:
1736 return nullptr;
1737 case MVT::i8:
1738 Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i8areg64;
1739 break;
1740 case MVT::i16:
1741 Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i16areg64;
1742 break;
1743 case MVT::i32:
1744 Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i32areg64;
1745 break;
1746 case MVT::i64:
1747 Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i64areg64;
1748 break;
1749 case MVT::f32:
1750 Opcode = NVPTX::INT_PTX_LDU_GLOBAL_f32areg64;
1751 break;
1752 case MVT::f64:
1753 Opcode = NVPTX::INT_PTX_LDU_GLOBAL_f64areg64;
1754 break;
1755 }
1756 }
1757 break;
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1758 case NVPTXISD::LDGV2:
1759 switch (EltVT.getSimpleVT().SimpleTy) {
1760 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]1761 return nullptr;
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1762 case MVT::i8:
1763 Opcode = NVPTX::INT_PTX_LDG_G_v2i8_ELE_areg64;
1764 break;
1765 case MVT::i16:
1766 Opcode = NVPTX::INT_PTX_LDG_G_v2i16_ELE_areg64;
1767 break;
1768 case MVT::i32:
1769 Opcode = NVPTX::INT_PTX_LDG_G_v2i32_ELE_areg64;
1770 break;
1771 case MVT::i64:
1772 Opcode = NVPTX::INT_PTX_LDG_G_v2i64_ELE_areg64;
1773 break;
1774 case MVT::f32:
1775 Opcode = NVPTX::INT_PTX_LDG_G_v2f32_ELE_areg64;
1776 break;
1777 case MVT::f64:
1778 Opcode = NVPTX::INT_PTX_LDG_G_v2f64_ELE_areg64;
1779 break;
1780 }
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1781 break;
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1782 case NVPTXISD::LDUV2:
1783 switch (EltVT.getSimpleVT().SimpleTy) {
1784 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]1785 return nullptr;
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1786 case MVT::i8:
1787 Opcode = NVPTX::INT_PTX_LDU_G_v2i8_ELE_areg64;
1788 break;
1789 case MVT::i16:
1790 Opcode = NVPTX::INT_PTX_LDU_G_v2i16_ELE_areg64;
1791 break;
1792 case MVT::i32:
1793 Opcode = NVPTX::INT_PTX_LDU_G_v2i32_ELE_areg64;
1794 break;
1795 case MVT::i64:
1796 Opcode = NVPTX::INT_PTX_LDU_G_v2i64_ELE_areg64;
1797 break;
1798 case MVT::f32:
1799 Opcode = NVPTX::INT_PTX_LDU_G_v2f32_ELE_areg64;
1800 break;
1801 case MVT::f64:
1802 Opcode = NVPTX::INT_PTX_LDU_G_v2f64_ELE_areg64;
1803 break;
1804 }
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1805 break;
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1806 case NVPTXISD::LDGV4:
1807 switch (EltVT.getSimpleVT().SimpleTy) {
1808 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]1809 return nullptr;
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1810 case MVT::i8:
1811 Opcode = NVPTX::INT_PTX_LDG_G_v4i8_ELE_areg64;
1812 break;
1813 case MVT::i16:
1814 Opcode = NVPTX::INT_PTX_LDG_G_v4i16_ELE_areg64;
1815 break;
1816 case MVT::i32:
1817 Opcode = NVPTX::INT_PTX_LDG_G_v4i32_ELE_areg64;
1818 break;
1819 case MVT::f32:
1820 Opcode = NVPTX::INT_PTX_LDG_G_v4f32_ELE_areg64;
1821 break;
1822 }
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1823 break;
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1824 case NVPTXISD::LDUV4:
1825 switch (EltVT.getSimpleVT().SimpleTy) {
1826 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]1827 return nullptr;
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1828 case MVT::i8:
1829 Opcode = NVPTX::INT_PTX_LDU_G_v4i8_ELE_areg64;
1830 break;
1831 case MVT::i16:
1832 Opcode = NVPTX::INT_PTX_LDU_G_v4i16_ELE_areg64;
1833 break;
1834 case MVT::i32:
1835 Opcode = NVPTX::INT_PTX_LDU_G_v4i32_ELE_areg64;
1836 break;
1837 case MVT::f32:
1838 Opcode = NVPTX::INT_PTX_LDU_G_v4f32_ELE_areg64;
1839 break;
1840 }
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1841 break;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]1842 }
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1843 } else {
1844 switch (N->getOpcode()) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1845 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]1846 return nullptr;
Justin Holewinskib926d9d2014-06-27 18:35:51 +0000[diff] [blame]1847 case ISD::INTRINSIC_W_CHAIN:
1848 if (IsLDG) {
1849 switch (EltVT.getSimpleVT().SimpleTy) {
1850 default:
1851 return nullptr;
1852 case MVT::i8:
1853 Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i8areg;
1854 break;
1855 case MVT::i16:
1856 Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i16areg;
1857 break;
1858 case MVT::i32:
1859 Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i32areg;
1860 break;
1861 case MVT::i64:
1862 Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i64areg;
1863 break;
1864 case MVT::f32:
1865 Opcode = NVPTX::INT_PTX_LDG_GLOBAL_f32areg;
1866 break;
1867 case MVT::f64:
1868 Opcode = NVPTX::INT_PTX_LDG_GLOBAL_f64areg;
1869 break;
1870 }
1871 } else {
1872 switch (EltVT.getSimpleVT().SimpleTy) {
1873 default:
1874 return nullptr;
1875 case MVT::i8:
1876 Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i8areg;
1877 break;
1878 case MVT::i16:
1879 Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i16areg;
1880 break;
1881 case MVT::i32:
1882 Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i32areg;
1883 break;
1884 case MVT::i64:
1885 Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i64areg;
1886 break;
1887 case MVT::f32:
1888 Opcode = NVPTX::INT_PTX_LDU_GLOBAL_f32areg;
1889 break;
1890 case MVT::f64:
1891 Opcode = NVPTX::INT_PTX_LDU_GLOBAL_f64areg;
1892 break;
1893 }
1894 }
1895 break;
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1896 case NVPTXISD::LDGV2:
1897 switch (EltVT.getSimpleVT().SimpleTy) {
1898 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]1899 return nullptr;
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1900 case MVT::i8:
1901 Opcode = NVPTX::INT_PTX_LDG_G_v2i8_ELE_areg32;
1902 break;
1903 case MVT::i16:
1904 Opcode = NVPTX::INT_PTX_LDG_G_v2i16_ELE_areg32;
1905 break;
1906 case MVT::i32:
1907 Opcode = NVPTX::INT_PTX_LDG_G_v2i32_ELE_areg32;
1908 break;
1909 case MVT::i64:
1910 Opcode = NVPTX::INT_PTX_LDG_G_v2i64_ELE_areg32;
1911 break;
1912 case MVT::f32:
1913 Opcode = NVPTX::INT_PTX_LDG_G_v2f32_ELE_areg32;
1914 break;
1915 case MVT::f64:
1916 Opcode = NVPTX::INT_PTX_LDG_G_v2f64_ELE_areg32;
1917 break;
1918 }
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1919 break;
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1920 case NVPTXISD::LDUV2:
1921 switch (EltVT.getSimpleVT().SimpleTy) {
1922 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]1923 return nullptr;
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1924 case MVT::i8:
1925 Opcode = NVPTX::INT_PTX_LDU_G_v2i8_ELE_areg32;
1926 break;
1927 case MVT::i16:
1928 Opcode = NVPTX::INT_PTX_LDU_G_v2i16_ELE_areg32;
1929 break;
1930 case MVT::i32:
1931 Opcode = NVPTX::INT_PTX_LDU_G_v2i32_ELE_areg32;
1932 break;
1933 case MVT::i64:
1934 Opcode = NVPTX::INT_PTX_LDU_G_v2i64_ELE_areg32;
1935 break;
1936 case MVT::f32:
1937 Opcode = NVPTX::INT_PTX_LDU_G_v2f32_ELE_areg32;
1938 break;
1939 case MVT::f64:
1940 Opcode = NVPTX::INT_PTX_LDU_G_v2f64_ELE_areg32;
1941 break;
1942 }
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1943 break;
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1944 case NVPTXISD::LDGV4:
1945 switch (EltVT.getSimpleVT().SimpleTy) {
1946 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]1947 return nullptr;
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1948 case MVT::i8:
1949 Opcode = NVPTX::INT_PTX_LDG_G_v4i8_ELE_areg32;
1950 break;
1951 case MVT::i16:
1952 Opcode = NVPTX::INT_PTX_LDG_G_v4i16_ELE_areg32;
1953 break;
1954 case MVT::i32:
1955 Opcode = NVPTX::INT_PTX_LDG_G_v4i32_ELE_areg32;
1956 break;
1957 case MVT::f32:
1958 Opcode = NVPTX::INT_PTX_LDG_G_v4f32_ELE_areg32;
1959 break;
1960 }
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1961 break;
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1962 case NVPTXISD::LDUV4:
1963 switch (EltVT.getSimpleVT().SimpleTy) {
1964 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]1965 return nullptr;
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1966 case MVT::i8:
1967 Opcode = NVPTX::INT_PTX_LDU_G_v4i8_ELE_areg32;
1968 break;
1969 case MVT::i16:
1970 Opcode = NVPTX::INT_PTX_LDU_G_v4i16_ELE_areg32;
1971 break;
1972 case MVT::i32:
1973 Opcode = NVPTX::INT_PTX_LDU_G_v4i32_ELE_areg32;
1974 break;
1975 case MVT::f32:
1976 Opcode = NVPTX::INT_PTX_LDU_G_v4f32_ELE_areg32;
1977 break;
1978 }
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1979 break;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]1980 }
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]1981 }
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]1982
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1983 SDValue Ops[] = { Op1, Chain };
Craig Topper2d2aa0c2014-04-30 07:17:30 +0000[diff] [blame]1984 LD = CurDAG->getMachineNode(Opcode, DL, N->getVTList(), Ops);
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1985 }
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]1986
1987 MachineSDNode::mmo_iterator MemRefs0 = MF->allocateMemRefsArray(1);
Justin Holewinskib926d9d2014-06-27 18:35:51 +0000[diff] [blame]1988 MemRefs0[0] = Mem->getMemOperand();
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]1989 cast<MachineSDNode>(LD)->setMemRefs(MemRefs0, MemRefs0 + 1);
1990
1991 return LD;
1992}
1993
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1994SDNode *NVPTXDAGToDAGISel::SelectStore(SDNode *N) {
Andrew Trickef9de2a2013-05-25 02:42:55 +0000[diff] [blame]1995 SDLoc dl(N);
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]1996 StoreSDNode *ST = cast<StoreSDNode>(N);
1997 EVT StoreVT = ST->getMemoryVT();
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]1998 SDNode *NVPTXST = nullptr;
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]1999
2000 // do not support pre/post inc/dec
2001 if (ST->isIndexed())
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2002 return nullptr;
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]2003
2004 if (!StoreVT.isSimple())
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2005 return nullptr;
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]2006
2007 // Address Space Setting
Eric Christopher9745b3a2015-01-30 01:41:01 +0000[diff] [blame]2008 unsigned int codeAddrSpace = getCodeAddrSpace(ST);
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]2009
2010 // Volatile Setting
2011 // - .volatile is only availalble for .global and .shared
2012 bool isVolatile = ST->isVolatile();
2013 if (codeAddrSpace != NVPTX::PTXLdStInstCode::GLOBAL &&
2014 codeAddrSpace != NVPTX::PTXLdStInstCode::SHARED &&
2015 codeAddrSpace != NVPTX::PTXLdStInstCode::GENERIC)
2016 isVolatile = false;
2017
2018 // Vector Setting
2019 MVT SimpleVT = StoreVT.getSimpleVT();
2020 unsigned vecType = NVPTX::PTXLdStInstCode::Scalar;
2021 if (SimpleVT.isVector()) {
2022 unsigned num = SimpleVT.getVectorNumElements();
2023 if (num == 2)
2024 vecType = NVPTX::PTXLdStInstCode::V2;
2025 else if (num == 4)
2026 vecType = NVPTX::PTXLdStInstCode::V4;
2027 else
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2028 return nullptr;
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]2029 }
2030
2031 // Type Setting: toType + toTypeWidth
2032 // - for integer type, always use 'u'
2033 //
2034 MVT ScalarVT = SimpleVT.getScalarType();
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2035 unsigned toTypeWidth = ScalarVT.getSizeInBits();
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]2036 unsigned int toType;
2037 if (ScalarVT.isFloatingPoint())
2038 toType = NVPTX::PTXLdStInstCode::Float;
2039 else
2040 toType = NVPTX::PTXLdStInstCode::Unsigned;
2041
2042 // Create the machine instruction DAG
2043 SDValue Chain = N->getOperand(0);
2044 SDValue N1 = N->getOperand(1);
2045 SDValue N2 = N->getOperand(2);
2046 SDValue Addr;
2047 SDValue Offset, Base;
2048 unsigned Opcode;
Craig Topperd9c27832013-08-15 02:44:19 +0000[diff] [blame]2049 MVT::SimpleValueType SourceVT = N1.getNode()->getSimpleValueType(0).SimpleTy;
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]2050
2051 if (SelectDirectAddr(N2, Addr)) {
2052 switch (SourceVT) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2053 case MVT::i8:
2054 Opcode = NVPTX::ST_i8_avar;
2055 break;
2056 case MVT::i16:
2057 Opcode = NVPTX::ST_i16_avar;
2058 break;
2059 case MVT::i32:
2060 Opcode = NVPTX::ST_i32_avar;
2061 break;
2062 case MVT::i64:
2063 Opcode = NVPTX::ST_i64_avar;
2064 break;
2065 case MVT::f32:
2066 Opcode = NVPTX::ST_f32_avar;
2067 break;
2068 case MVT::f64:
2069 Opcode = NVPTX::ST_f64_avar;
2070 break;
2071 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2072 return nullptr;
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]2073 }
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2074 SDValue Ops[] = { N1, getI32Imm(isVolatile), getI32Imm(codeAddrSpace),
2075 getI32Imm(vecType), getI32Imm(toType),
2076 getI32Imm(toTypeWidth), Addr, Chain };
Michael Liaob53d8962013-04-19 22:22:57 +0000[diff] [blame]2077 NVPTXST = CurDAG->getMachineNode(Opcode, dl, MVT::Other, Ops);
Eric Christopher02389e32015-02-19 00:08:27 +0000[diff] [blame]2078 } else if (TM.is64Bit() ? SelectADDRsi64(N2.getNode(), N2, Base, Offset)
2079 : SelectADDRsi(N2.getNode(), N2, Base, Offset)) {
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]2080 switch (SourceVT) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2081 case MVT::i8:
2082 Opcode = NVPTX::ST_i8_asi;
2083 break;
2084 case MVT::i16:
2085 Opcode = NVPTX::ST_i16_asi;
2086 break;
2087 case MVT::i32:
2088 Opcode = NVPTX::ST_i32_asi;
2089 break;
2090 case MVT::i64:
2091 Opcode = NVPTX::ST_i64_asi;
2092 break;
2093 case MVT::f32:
2094 Opcode = NVPTX::ST_f32_asi;
2095 break;
2096 case MVT::f64:
2097 Opcode = NVPTX::ST_f64_asi;
2098 break;
2099 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2100 return nullptr;
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]2101 }
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2102 SDValue Ops[] = { N1, getI32Imm(isVolatile), getI32Imm(codeAddrSpace),
2103 getI32Imm(vecType), getI32Imm(toType),
2104 getI32Imm(toTypeWidth), Base, Offset, Chain };
Michael Liaob53d8962013-04-19 22:22:57 +0000[diff] [blame]2105 NVPTXST = CurDAG->getMachineNode(Opcode, dl, MVT::Other, Ops);
Eric Christopher02389e32015-02-19 00:08:27 +0000[diff] [blame]2106 } else if (TM.is64Bit() ? SelectADDRri64(N2.getNode(), N2, Base, Offset)
2107 : SelectADDRri(N2.getNode(), N2, Base, Offset)) {
2108 if (TM.is64Bit()) {
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]2109 switch (SourceVT) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2110 case MVT::i8:
2111 Opcode = NVPTX::ST_i8_ari_64;
2112 break;
2113 case MVT::i16:
2114 Opcode = NVPTX::ST_i16_ari_64;
2115 break;
2116 case MVT::i32:
2117 Opcode = NVPTX::ST_i32_ari_64;
2118 break;
2119 case MVT::i64:
2120 Opcode = NVPTX::ST_i64_ari_64;
2121 break;
2122 case MVT::f32:
2123 Opcode = NVPTX::ST_f32_ari_64;
2124 break;
2125 case MVT::f64:
2126 Opcode = NVPTX::ST_f64_ari_64;
2127 break;
2128 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2129 return nullptr;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]2130 }
2131 } else {
2132 switch (SourceVT) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2133 case MVT::i8:
2134 Opcode = NVPTX::ST_i8_ari;
2135 break;
2136 case MVT::i16:
2137 Opcode = NVPTX::ST_i16_ari;
2138 break;
2139 case MVT::i32:
2140 Opcode = NVPTX::ST_i32_ari;
2141 break;
2142 case MVT::i64:
2143 Opcode = NVPTX::ST_i64_ari;
2144 break;
2145 case MVT::f32:
2146 Opcode = NVPTX::ST_f32_ari;
2147 break;
2148 case MVT::f64:
2149 Opcode = NVPTX::ST_f64_ari;
2150 break;
2151 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2152 return nullptr;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]2153 }
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]2154 }
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2155 SDValue Ops[] = { N1, getI32Imm(isVolatile), getI32Imm(codeAddrSpace),
2156 getI32Imm(vecType), getI32Imm(toType),
2157 getI32Imm(toTypeWidth), Base, Offset, Chain };
Michael Liaob53d8962013-04-19 22:22:57 +0000[diff] [blame]2158 NVPTXST = CurDAG->getMachineNode(Opcode, dl, MVT::Other, Ops);
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]2159 } else {
Eric Christopher02389e32015-02-19 00:08:27 +0000[diff] [blame]2160 if (TM.is64Bit()) {
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]2161 switch (SourceVT) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2162 case MVT::i8:
2163 Opcode = NVPTX::ST_i8_areg_64;
2164 break;
2165 case MVT::i16:
2166 Opcode = NVPTX::ST_i16_areg_64;
2167 break;
2168 case MVT::i32:
2169 Opcode = NVPTX::ST_i32_areg_64;
2170 break;
2171 case MVT::i64:
2172 Opcode = NVPTX::ST_i64_areg_64;
2173 break;
2174 case MVT::f32:
2175 Opcode = NVPTX::ST_f32_areg_64;
2176 break;
2177 case MVT::f64:
2178 Opcode = NVPTX::ST_f64_areg_64;
2179 break;
2180 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2181 return nullptr;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]2182 }
2183 } else {
2184 switch (SourceVT) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2185 case MVT::i8:
2186 Opcode = NVPTX::ST_i8_areg;
2187 break;
2188 case MVT::i16:
2189 Opcode = NVPTX::ST_i16_areg;
2190 break;
2191 case MVT::i32:
2192 Opcode = NVPTX::ST_i32_areg;
2193 break;
2194 case MVT::i64:
2195 Opcode = NVPTX::ST_i64_areg;
2196 break;
2197 case MVT::f32:
2198 Opcode = NVPTX::ST_f32_areg;
2199 break;
2200 case MVT::f64:
2201 Opcode = NVPTX::ST_f64_areg;
2202 break;
2203 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2204 return nullptr;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]2205 }
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]2206 }
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2207 SDValue Ops[] = { N1, getI32Imm(isVolatile), getI32Imm(codeAddrSpace),
2208 getI32Imm(vecType), getI32Imm(toType),
2209 getI32Imm(toTypeWidth), N2, Chain };
Michael Liaob53d8962013-04-19 22:22:57 +0000[diff] [blame]2210 NVPTXST = CurDAG->getMachineNode(Opcode, dl, MVT::Other, Ops);
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]2211 }
2212
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2213 if (NVPTXST) {
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]2214 MachineSDNode::mmo_iterator MemRefs0 = MF->allocateMemRefsArray(1);
2215 MemRefs0[0] = cast<MemSDNode>(N)->getMemOperand();
2216 cast<MachineSDNode>(NVPTXST)->setMemRefs(MemRefs0, MemRefs0 + 1);
2217 }
2218
2219 return NVPTXST;
2220}
2221
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]2222SDNode *NVPTXDAGToDAGISel::SelectStoreVector(SDNode *N) {
2223 SDValue Chain = N->getOperand(0);
2224 SDValue Op1 = N->getOperand(1);
2225 SDValue Addr, Offset, Base;
2226 unsigned Opcode;
Andrew Trickef9de2a2013-05-25 02:42:55 +0000[diff] [blame]2227 SDLoc DL(N);
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]2228 SDNode *ST;
2229 EVT EltVT = Op1.getValueType();
2230 MemSDNode *MemSD = cast<MemSDNode>(N);
2231 EVT StoreVT = MemSD->getMemoryVT();
2232
2233 // Address Space Setting
Eric Christopher9745b3a2015-01-30 01:41:01 +0000[diff] [blame]2234 unsigned CodeAddrSpace = getCodeAddrSpace(MemSD);
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]2235
2236 if (CodeAddrSpace == NVPTX::PTXLdStInstCode::CONSTANT) {
2237 report_fatal_error("Cannot store to pointer that points to constant "
2238 "memory space");
2239 }
2240
2241 // Volatile Setting
2242 // - .volatile is only availalble for .global and .shared
2243 bool IsVolatile = MemSD->isVolatile();
2244 if (CodeAddrSpace != NVPTX::PTXLdStInstCode::GLOBAL &&
2245 CodeAddrSpace != NVPTX::PTXLdStInstCode::SHARED &&
2246 CodeAddrSpace != NVPTX::PTXLdStInstCode::GENERIC)
2247 IsVolatile = false;
2248
2249 // Type Setting: toType + toTypeWidth
2250 // - for integer type, always use 'u'
2251 assert(StoreVT.isSimple() && "Store value is not simple");
2252 MVT ScalarVT = StoreVT.getSimpleVT().getScalarType();
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2253 unsigned ToTypeWidth = ScalarVT.getSizeInBits();
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]2254 unsigned ToType;
2255 if (ScalarVT.isFloatingPoint())
2256 ToType = NVPTX::PTXLdStInstCode::Float;
2257 else
2258 ToType = NVPTX::PTXLdStInstCode::Unsigned;
2259
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]2260 SmallVector<SDValue, 12> StOps;
2261 SDValue N2;
2262 unsigned VecType;
2263
2264 switch (N->getOpcode()) {
2265 case NVPTXISD::StoreV2:
2266 VecType = NVPTX::PTXLdStInstCode::V2;
2267 StOps.push_back(N->getOperand(1));
2268 StOps.push_back(N->getOperand(2));
2269 N2 = N->getOperand(3);
2270 break;
2271 case NVPTXISD::StoreV4:
2272 VecType = NVPTX::PTXLdStInstCode::V4;
2273 StOps.push_back(N->getOperand(1));
2274 StOps.push_back(N->getOperand(2));
2275 StOps.push_back(N->getOperand(3));
2276 StOps.push_back(N->getOperand(4));
2277 N2 = N->getOperand(5);
2278 break;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2279 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2280 return nullptr;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]2281 }
2282
2283 StOps.push_back(getI32Imm(IsVolatile));
2284 StOps.push_back(getI32Imm(CodeAddrSpace));
2285 StOps.push_back(getI32Imm(VecType));
2286 StOps.push_back(getI32Imm(ToType));
2287 StOps.push_back(getI32Imm(ToTypeWidth));
2288
2289 if (SelectDirectAddr(N2, Addr)) {
2290 switch (N->getOpcode()) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2291 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2292 return nullptr;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]2293 case NVPTXISD::StoreV2:
2294 switch (EltVT.getSimpleVT().SimpleTy) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2295 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2296 return nullptr;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2297 case MVT::i8:
2298 Opcode = NVPTX::STV_i8_v2_avar;
2299 break;
2300 case MVT::i16:
2301 Opcode = NVPTX::STV_i16_v2_avar;
2302 break;
2303 case MVT::i32:
2304 Opcode = NVPTX::STV_i32_v2_avar;
2305 break;
2306 case MVT::i64:
2307 Opcode = NVPTX::STV_i64_v2_avar;
2308 break;
2309 case MVT::f32:
2310 Opcode = NVPTX::STV_f32_v2_avar;
2311 break;
2312 case MVT::f64:
2313 Opcode = NVPTX::STV_f64_v2_avar;
2314 break;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]2315 }
2316 break;
2317 case NVPTXISD::StoreV4:
2318 switch (EltVT.getSimpleVT().SimpleTy) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2319 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2320 return nullptr;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2321 case MVT::i8:
2322 Opcode = NVPTX::STV_i8_v4_avar;
2323 break;
2324 case MVT::i16:
2325 Opcode = NVPTX::STV_i16_v4_avar;
2326 break;
2327 case MVT::i32:
2328 Opcode = NVPTX::STV_i32_v4_avar;
2329 break;
2330 case MVT::f32:
2331 Opcode = NVPTX::STV_f32_v4_avar;
2332 break;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]2333 }
2334 break;
2335 }
2336 StOps.push_back(Addr);
Eric Christopher02389e32015-02-19 00:08:27 +0000[diff] [blame]2337 } else if (TM.is64Bit() ? SelectADDRsi64(N2.getNode(), N2, Base, Offset)
2338 : SelectADDRsi(N2.getNode(), N2, Base, Offset)) {
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]2339 switch (N->getOpcode()) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2340 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2341 return nullptr;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]2342 case NVPTXISD::StoreV2:
2343 switch (EltVT.getSimpleVT().SimpleTy) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2344 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2345 return nullptr;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2346 case MVT::i8:
2347 Opcode = NVPTX::STV_i8_v2_asi;
2348 break;
2349 case MVT::i16:
2350 Opcode = NVPTX::STV_i16_v2_asi;
2351 break;
2352 case MVT::i32:
2353 Opcode = NVPTX::STV_i32_v2_asi;
2354 break;
2355 case MVT::i64:
2356 Opcode = NVPTX::STV_i64_v2_asi;
2357 break;
2358 case MVT::f32:
2359 Opcode = NVPTX::STV_f32_v2_asi;
2360 break;
2361 case MVT::f64:
2362 Opcode = NVPTX::STV_f64_v2_asi;
2363 break;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]2364 }
2365 break;
2366 case NVPTXISD::StoreV4:
2367 switch (EltVT.getSimpleVT().SimpleTy) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2368 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2369 return nullptr;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2370 case MVT::i8:
2371 Opcode = NVPTX::STV_i8_v4_asi;
2372 break;
2373 case MVT::i16:
2374 Opcode = NVPTX::STV_i16_v4_asi;
2375 break;
2376 case MVT::i32:
2377 Opcode = NVPTX::STV_i32_v4_asi;
2378 break;
2379 case MVT::f32:
2380 Opcode = NVPTX::STV_f32_v4_asi;
2381 break;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]2382 }
2383 break;
2384 }
2385 StOps.push_back(Base);
2386 StOps.push_back(Offset);
Eric Christopher02389e32015-02-19 00:08:27 +0000[diff] [blame]2387 } else if (TM.is64Bit() ? SelectADDRri64(N2.getNode(), N2, Base, Offset)
2388 : SelectADDRri(N2.getNode(), N2, Base, Offset)) {
2389 if (TM.is64Bit()) {
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]2390 switch (N->getOpcode()) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2391 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2392 return nullptr;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]2393 case NVPTXISD::StoreV2:
2394 switch (EltVT.getSimpleVT().SimpleTy) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2395 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2396 return nullptr;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2397 case MVT::i8:
2398 Opcode = NVPTX::STV_i8_v2_ari_64;
2399 break;
2400 case MVT::i16:
2401 Opcode = NVPTX::STV_i16_v2_ari_64;
2402 break;
2403 case MVT::i32:
2404 Opcode = NVPTX::STV_i32_v2_ari_64;
2405 break;
2406 case MVT::i64:
2407 Opcode = NVPTX::STV_i64_v2_ari_64;
2408 break;
2409 case MVT::f32:
2410 Opcode = NVPTX::STV_f32_v2_ari_64;
2411 break;
2412 case MVT::f64:
2413 Opcode = NVPTX::STV_f64_v2_ari_64;
2414 break;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]2415 }
2416 break;
2417 case NVPTXISD::StoreV4:
2418 switch (EltVT.getSimpleVT().SimpleTy) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2419 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2420 return nullptr;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2421 case MVT::i8:
2422 Opcode = NVPTX::STV_i8_v4_ari_64;
2423 break;
2424 case MVT::i16:
2425 Opcode = NVPTX::STV_i16_v4_ari_64;
2426 break;
2427 case MVT::i32:
2428 Opcode = NVPTX::STV_i32_v4_ari_64;
2429 break;
2430 case MVT::f32:
2431 Opcode = NVPTX::STV_f32_v4_ari_64;
2432 break;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]2433 }
2434 break;
2435 }
2436 } else {
2437 switch (N->getOpcode()) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2438 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2439 return nullptr;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]2440 case NVPTXISD::StoreV2:
2441 switch (EltVT.getSimpleVT().SimpleTy) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2442 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2443 return nullptr;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2444 case MVT::i8:
2445 Opcode = NVPTX::STV_i8_v2_ari;
2446 break;
2447 case MVT::i16:
2448 Opcode = NVPTX::STV_i16_v2_ari;
2449 break;
2450 case MVT::i32:
2451 Opcode = NVPTX::STV_i32_v2_ari;
2452 break;
2453 case MVT::i64:
2454 Opcode = NVPTX::STV_i64_v2_ari;
2455 break;
2456 case MVT::f32:
2457 Opcode = NVPTX::STV_f32_v2_ari;
2458 break;
2459 case MVT::f64:
2460 Opcode = NVPTX::STV_f64_v2_ari;
2461 break;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]2462 }
2463 break;
2464 case NVPTXISD::StoreV4:
2465 switch (EltVT.getSimpleVT().SimpleTy) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2466 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2467 return nullptr;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2468 case MVT::i8:
2469 Opcode = NVPTX::STV_i8_v4_ari;
2470 break;
2471 case MVT::i16:
2472 Opcode = NVPTX::STV_i16_v4_ari;
2473 break;
2474 case MVT::i32:
2475 Opcode = NVPTX::STV_i32_v4_ari;
2476 break;
2477 case MVT::f32:
2478 Opcode = NVPTX::STV_f32_v4_ari;
2479 break;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]2480 }
2481 break;
2482 }
2483 }
2484 StOps.push_back(Base);
2485 StOps.push_back(Offset);
2486 } else {
Eric Christopher02389e32015-02-19 00:08:27 +0000[diff] [blame]2487 if (TM.is64Bit()) {
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]2488 switch (N->getOpcode()) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2489 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2490 return nullptr;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]2491 case NVPTXISD::StoreV2:
2492 switch (EltVT.getSimpleVT().SimpleTy) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2493 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2494 return nullptr;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2495 case MVT::i8:
2496 Opcode = NVPTX::STV_i8_v2_areg_64;
2497 break;
2498 case MVT::i16:
2499 Opcode = NVPTX::STV_i16_v2_areg_64;
2500 break;
2501 case MVT::i32:
2502 Opcode = NVPTX::STV_i32_v2_areg_64;
2503 break;
2504 case MVT::i64:
2505 Opcode = NVPTX::STV_i64_v2_areg_64;
2506 break;
2507 case MVT::f32:
2508 Opcode = NVPTX::STV_f32_v2_areg_64;
2509 break;
2510 case MVT::f64:
2511 Opcode = NVPTX::STV_f64_v2_areg_64;
2512 break;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]2513 }
2514 break;
2515 case NVPTXISD::StoreV4:
2516 switch (EltVT.getSimpleVT().SimpleTy) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2517 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2518 return nullptr;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2519 case MVT::i8:
2520 Opcode = NVPTX::STV_i8_v4_areg_64;
2521 break;
2522 case MVT::i16:
2523 Opcode = NVPTX::STV_i16_v4_areg_64;
2524 break;
2525 case MVT::i32:
2526 Opcode = NVPTX::STV_i32_v4_areg_64;
2527 break;
2528 case MVT::f32:
2529 Opcode = NVPTX::STV_f32_v4_areg_64;
2530 break;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]2531 }
2532 break;
2533 }
2534 } else {
2535 switch (N->getOpcode()) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2536 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2537 return nullptr;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]2538 case NVPTXISD::StoreV2:
2539 switch (EltVT.getSimpleVT().SimpleTy) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2540 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2541 return nullptr;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2542 case MVT::i8:
2543 Opcode = NVPTX::STV_i8_v2_areg;
2544 break;
2545 case MVT::i16:
2546 Opcode = NVPTX::STV_i16_v2_areg;
2547 break;
2548 case MVT::i32:
2549 Opcode = NVPTX::STV_i32_v2_areg;
2550 break;
2551 case MVT::i64:
2552 Opcode = NVPTX::STV_i64_v2_areg;
2553 break;
2554 case MVT::f32:
2555 Opcode = NVPTX::STV_f32_v2_areg;
2556 break;
2557 case MVT::f64:
2558 Opcode = NVPTX::STV_f64_v2_areg;
2559 break;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]2560 }
2561 break;
2562 case NVPTXISD::StoreV4:
2563 switch (EltVT.getSimpleVT().SimpleTy) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2564 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2565 return nullptr;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2566 case MVT::i8:
2567 Opcode = NVPTX::STV_i8_v4_areg;
2568 break;
2569 case MVT::i16:
2570 Opcode = NVPTX::STV_i16_v4_areg;
2571 break;
2572 case MVT::i32:
2573 Opcode = NVPTX::STV_i32_v4_areg;
2574 break;
2575 case MVT::f32:
2576 Opcode = NVPTX::STV_f32_v4_areg;
2577 break;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]2578 }
2579 break;
2580 }
2581 }
2582 StOps.push_back(N2);
2583 }
2584
2585 StOps.push_back(Chain);
2586
Michael Liaob53d8962013-04-19 22:22:57 +0000[diff] [blame]2587 ST = CurDAG->getMachineNode(Opcode, DL, MVT::Other, StOps);
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]2588
2589 MachineSDNode::mmo_iterator MemRefs0 = MF->allocateMemRefsArray(1);
2590 MemRefs0[0] = cast<MemSDNode>(N)->getMemOperand();
2591 cast<MachineSDNode>(ST)->setMemRefs(MemRefs0, MemRefs0 + 1);
2592
2593 return ST;
2594}
2595
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]2596SDNode *NVPTXDAGToDAGISel::SelectLoadParam(SDNode *Node) {
2597 SDValue Chain = Node->getOperand(0);
2598 SDValue Offset = Node->getOperand(2);
2599 SDValue Flag = Node->getOperand(3);
2600 SDLoc DL(Node);
2601 MemSDNode *Mem = cast<MemSDNode>(Node);
2602
2603 unsigned VecSize;
2604 switch (Node->getOpcode()) {
2605 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2606 return nullptr;
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]2607 case NVPTXISD::LoadParam:
2608 VecSize = 1;
2609 break;
2610 case NVPTXISD::LoadParamV2:
2611 VecSize = 2;
2612 break;
2613 case NVPTXISD::LoadParamV4:
2614 VecSize = 4;
2615 break;
2616 }
2617
2618 EVT EltVT = Node->getValueType(0);
2619 EVT MemVT = Mem->getMemoryVT();
2620
2621 unsigned Opc = 0;
2622
2623 switch (VecSize) {
2624 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2625 return nullptr;
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]2626 case 1:
2627 switch (MemVT.getSimpleVT().SimpleTy) {
2628 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2629 return nullptr;
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]2630 case MVT::i1:
2631 Opc = NVPTX::LoadParamMemI8;
2632 break;
2633 case MVT::i8:
2634 Opc = NVPTX::LoadParamMemI8;
2635 break;
2636 case MVT::i16:
2637 Opc = NVPTX::LoadParamMemI16;
2638 break;
2639 case MVT::i32:
2640 Opc = NVPTX::LoadParamMemI32;
2641 break;
2642 case MVT::i64:
2643 Opc = NVPTX::LoadParamMemI64;
2644 break;
2645 case MVT::f32:
2646 Opc = NVPTX::LoadParamMemF32;
2647 break;
2648 case MVT::f64:
2649 Opc = NVPTX::LoadParamMemF64;
2650 break;
2651 }
2652 break;
2653 case 2:
2654 switch (MemVT.getSimpleVT().SimpleTy) {
2655 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2656 return nullptr;
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]2657 case MVT::i1:
2658 Opc = NVPTX::LoadParamMemV2I8;
2659 break;
2660 case MVT::i8:
2661 Opc = NVPTX::LoadParamMemV2I8;
2662 break;
2663 case MVT::i16:
2664 Opc = NVPTX::LoadParamMemV2I16;
2665 break;
2666 case MVT::i32:
2667 Opc = NVPTX::LoadParamMemV2I32;
2668 break;
2669 case MVT::i64:
2670 Opc = NVPTX::LoadParamMemV2I64;
2671 break;
2672 case MVT::f32:
2673 Opc = NVPTX::LoadParamMemV2F32;
2674 break;
2675 case MVT::f64:
2676 Opc = NVPTX::LoadParamMemV2F64;
2677 break;
2678 }
2679 break;
2680 case 4:
2681 switch (MemVT.getSimpleVT().SimpleTy) {
2682 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2683 return nullptr;
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]2684 case MVT::i1:
2685 Opc = NVPTX::LoadParamMemV4I8;
2686 break;
2687 case MVT::i8:
2688 Opc = NVPTX::LoadParamMemV4I8;
2689 break;
2690 case MVT::i16:
2691 Opc = NVPTX::LoadParamMemV4I16;
2692 break;
2693 case MVT::i32:
2694 Opc = NVPTX::LoadParamMemV4I32;
2695 break;
2696 case MVT::f32:
2697 Opc = NVPTX::LoadParamMemV4F32;
2698 break;
2699 }
2700 break;
2701 }
2702
2703 SDVTList VTs;
2704 if (VecSize == 1) {
2705 VTs = CurDAG->getVTList(EltVT, MVT::Other, MVT::Glue);
2706 } else if (VecSize == 2) {
2707 VTs = CurDAG->getVTList(EltVT, EltVT, MVT::Other, MVT::Glue);
2708 } else {
2709 EVT EVTs[] = { EltVT, EltVT, EltVT, EltVT, MVT::Other, MVT::Glue };
Craig Topperabb4ac72014-04-16 06:10:51 +0000[diff] [blame]2710 VTs = CurDAG->getVTList(EVTs);
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]2711 }
2712
2713 unsigned OffsetVal = cast<ConstantSDNode>(Offset)->getZExtValue();
2714
2715 SmallVector<SDValue, 2> Ops;
2716 Ops.push_back(CurDAG->getTargetConstant(OffsetVal, MVT::i32));
2717 Ops.push_back(Chain);
2718 Ops.push_back(Flag);
2719
2720 SDNode *Ret =
Justin Holewinskidff28d22013-07-01 12:59:01 +0000[diff] [blame]2721 CurDAG->getMachineNode(Opc, DL, VTs, Ops);
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]2722 return Ret;
2723}
2724
2725SDNode *NVPTXDAGToDAGISel::SelectStoreRetval(SDNode *N) {
2726 SDLoc DL(N);
2727 SDValue Chain = N->getOperand(0);
2728 SDValue Offset = N->getOperand(1);
2729 unsigned OffsetVal = cast<ConstantSDNode>(Offset)->getZExtValue();
2730 MemSDNode *Mem = cast<MemSDNode>(N);
2731
2732 // How many elements do we have?
2733 unsigned NumElts = 1;
2734 switch (N->getOpcode()) {
2735 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2736 return nullptr;
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]2737 case NVPTXISD::StoreRetval:
2738 NumElts = 1;
2739 break;
2740 case NVPTXISD::StoreRetvalV2:
2741 NumElts = 2;
2742 break;
2743 case NVPTXISD::StoreRetvalV4:
2744 NumElts = 4;
2745 break;
2746 }
2747
2748 // Build vector of operands
2749 SmallVector<SDValue, 6> Ops;
2750 for (unsigned i = 0; i < NumElts; ++i)
2751 Ops.push_back(N->getOperand(i + 2));
2752 Ops.push_back(CurDAG->getTargetConstant(OffsetVal, MVT::i32));
2753 Ops.push_back(Chain);
2754
2755 // Determine target opcode
2756 // If we have an i1, use an 8-bit store. The lowering code in
2757 // NVPTXISelLowering will have already emitted an upcast.
2758 unsigned Opcode = 0;
2759 switch (NumElts) {
2760 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2761 return nullptr;
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]2762 case 1:
2763 switch (Mem->getMemoryVT().getSimpleVT().SimpleTy) {
2764 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2765 return nullptr;
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]2766 case MVT::i1:
2767 Opcode = NVPTX::StoreRetvalI8;
2768 break;
2769 case MVT::i8:
2770 Opcode = NVPTX::StoreRetvalI8;
2771 break;
2772 case MVT::i16:
2773 Opcode = NVPTX::StoreRetvalI16;
2774 break;
2775 case MVT::i32:
2776 Opcode = NVPTX::StoreRetvalI32;
2777 break;
2778 case MVT::i64:
2779 Opcode = NVPTX::StoreRetvalI64;
2780 break;
2781 case MVT::f32:
2782 Opcode = NVPTX::StoreRetvalF32;
2783 break;
2784 case MVT::f64:
2785 Opcode = NVPTX::StoreRetvalF64;
2786 break;
2787 }
2788 break;
2789 case 2:
2790 switch (Mem->getMemoryVT().getSimpleVT().SimpleTy) {
2791 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2792 return nullptr;
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]2793 case MVT::i1:
2794 Opcode = NVPTX::StoreRetvalV2I8;
2795 break;
2796 case MVT::i8:
2797 Opcode = NVPTX::StoreRetvalV2I8;
2798 break;
2799 case MVT::i16:
2800 Opcode = NVPTX::StoreRetvalV2I16;
2801 break;
2802 case MVT::i32:
2803 Opcode = NVPTX::StoreRetvalV2I32;
2804 break;
2805 case MVT::i64:
2806 Opcode = NVPTX::StoreRetvalV2I64;
2807 break;
2808 case MVT::f32:
2809 Opcode = NVPTX::StoreRetvalV2F32;
2810 break;
2811 case MVT::f64:
2812 Opcode = NVPTX::StoreRetvalV2F64;
2813 break;
2814 }
2815 break;
2816 case 4:
2817 switch (Mem->getMemoryVT().getSimpleVT().SimpleTy) {
2818 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2819 return nullptr;
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]2820 case MVT::i1:
2821 Opcode = NVPTX::StoreRetvalV4I8;
2822 break;
2823 case MVT::i8:
2824 Opcode = NVPTX::StoreRetvalV4I8;
2825 break;
2826 case MVT::i16:
2827 Opcode = NVPTX::StoreRetvalV4I16;
2828 break;
2829 case MVT::i32:
2830 Opcode = NVPTX::StoreRetvalV4I32;
2831 break;
2832 case MVT::f32:
2833 Opcode = NVPTX::StoreRetvalV4F32;
2834 break;
2835 }
2836 break;
2837 }
2838
2839 SDNode *Ret =
2840 CurDAG->getMachineNode(Opcode, DL, MVT::Other, Ops);
2841 MachineSDNode::mmo_iterator MemRefs0 = MF->allocateMemRefsArray(1);
2842 MemRefs0[0] = cast<MemSDNode>(N)->getMemOperand();
2843 cast<MachineSDNode>(Ret)->setMemRefs(MemRefs0, MemRefs0 + 1);
2844
2845 return Ret;
2846}
2847
2848SDNode *NVPTXDAGToDAGISel::SelectStoreParam(SDNode *N) {
2849 SDLoc DL(N);
2850 SDValue Chain = N->getOperand(0);
2851 SDValue Param = N->getOperand(1);
2852 unsigned ParamVal = cast<ConstantSDNode>(Param)->getZExtValue();
2853 SDValue Offset = N->getOperand(2);
2854 unsigned OffsetVal = cast<ConstantSDNode>(Offset)->getZExtValue();
2855 MemSDNode *Mem = cast<MemSDNode>(N);
2856 SDValue Flag = N->getOperand(N->getNumOperands() - 1);
2857
2858 // How many elements do we have?
2859 unsigned NumElts = 1;
2860 switch (N->getOpcode()) {
2861 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2862 return nullptr;
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]2863 case NVPTXISD::StoreParamU32:
2864 case NVPTXISD::StoreParamS32:
2865 case NVPTXISD::StoreParam:
2866 NumElts = 1;
2867 break;
2868 case NVPTXISD::StoreParamV2:
2869 NumElts = 2;
2870 break;
2871 case NVPTXISD::StoreParamV4:
2872 NumElts = 4;
2873 break;
2874 }
2875
2876 // Build vector of operands
2877 SmallVector<SDValue, 8> Ops;
2878 for (unsigned i = 0; i < NumElts; ++i)
2879 Ops.push_back(N->getOperand(i + 3));
2880 Ops.push_back(CurDAG->getTargetConstant(ParamVal, MVT::i32));
2881 Ops.push_back(CurDAG->getTargetConstant(OffsetVal, MVT::i32));
2882 Ops.push_back(Chain);
2883 Ops.push_back(Flag);
2884
2885 // Determine target opcode
2886 // If we have an i1, use an 8-bit store. The lowering code in
2887 // NVPTXISelLowering will have already emitted an upcast.
2888 unsigned Opcode = 0;
2889 switch (N->getOpcode()) {
2890 default:
2891 switch (NumElts) {
2892 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2893 return nullptr;
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]2894 case 1:
2895 switch (Mem->getMemoryVT().getSimpleVT().SimpleTy) {
2896 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2897 return nullptr;
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]2898 case MVT::i1:
2899 Opcode = NVPTX::StoreParamI8;
2900 break;
2901 case MVT::i8:
2902 Opcode = NVPTX::StoreParamI8;
2903 break;
2904 case MVT::i16:
2905 Opcode = NVPTX::StoreParamI16;
2906 break;
2907 case MVT::i32:
2908 Opcode = NVPTX::StoreParamI32;
2909 break;
2910 case MVT::i64:
2911 Opcode = NVPTX::StoreParamI64;
2912 break;
2913 case MVT::f32:
2914 Opcode = NVPTX::StoreParamF32;
2915 break;
2916 case MVT::f64:
2917 Opcode = NVPTX::StoreParamF64;
2918 break;
2919 }
2920 break;
2921 case 2:
2922 switch (Mem->getMemoryVT().getSimpleVT().SimpleTy) {
2923 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2924 return nullptr;
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]2925 case MVT::i1:
2926 Opcode = NVPTX::StoreParamV2I8;
2927 break;
2928 case MVT::i8:
2929 Opcode = NVPTX::StoreParamV2I8;
2930 break;
2931 case MVT::i16:
2932 Opcode = NVPTX::StoreParamV2I16;
2933 break;
2934 case MVT::i32:
2935 Opcode = NVPTX::StoreParamV2I32;
2936 break;
2937 case MVT::i64:
2938 Opcode = NVPTX::StoreParamV2I64;
2939 break;
2940 case MVT::f32:
2941 Opcode = NVPTX::StoreParamV2F32;
2942 break;
2943 case MVT::f64:
2944 Opcode = NVPTX::StoreParamV2F64;
2945 break;
2946 }
2947 break;
2948 case 4:
2949 switch (Mem->getMemoryVT().getSimpleVT().SimpleTy) {
2950 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2951 return nullptr;
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]2952 case MVT::i1:
2953 Opcode = NVPTX::StoreParamV4I8;
2954 break;
2955 case MVT::i8:
2956 Opcode = NVPTX::StoreParamV4I8;
2957 break;
2958 case MVT::i16:
2959 Opcode = NVPTX::StoreParamV4I16;
2960 break;
2961 case MVT::i32:
2962 Opcode = NVPTX::StoreParamV4I32;
2963 break;
2964 case MVT::f32:
2965 Opcode = NVPTX::StoreParamV4F32;
2966 break;
2967 }
2968 break;
2969 }
2970 break;
Justin Holewinskidc5e3b62013-06-28 17:58:04 +0000[diff] [blame]2971 // Special case: if we have a sign-extend/zero-extend node, insert the
2972 // conversion instruction first, and use that as the value operand to
2973 // the selected StoreParam node.
2974 case NVPTXISD::StoreParamU32: {
2975 Opcode = NVPTX::StoreParamI32;
2976 SDValue CvtNone = CurDAG->getTargetConstant(NVPTX::PTXCvtMode::NONE,
2977 MVT::i32);
2978 SDNode *Cvt = CurDAG->getMachineNode(NVPTX::CVT_u32_u16, DL,
2979 MVT::i32, Ops[0], CvtNone);
2980 Ops[0] = SDValue(Cvt, 0);
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]2981 break;
Justin Holewinskidc5e3b62013-06-28 17:58:04 +0000[diff] [blame]2982 }
2983 case NVPTXISD::StoreParamS32: {
2984 Opcode = NVPTX::StoreParamI32;
2985 SDValue CvtNone = CurDAG->getTargetConstant(NVPTX::PTXCvtMode::NONE,
2986 MVT::i32);
2987 SDNode *Cvt = CurDAG->getMachineNode(NVPTX::CVT_s32_s16, DL,
2988 MVT::i32, Ops[0], CvtNone);
2989 Ops[0] = SDValue(Cvt, 0);
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]2990 break;
2991 }
Justin Holewinskidc5e3b62013-06-28 17:58:04 +0000[diff] [blame]2992 }
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]2993
Justin Holewinskidff28d22013-07-01 12:59:01 +0000[diff] [blame]2994 SDVTList RetVTs = CurDAG->getVTList(MVT::Other, MVT::Glue);
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]2995 SDNode *Ret =
Justin Holewinskidff28d22013-07-01 12:59:01 +0000[diff] [blame]2996 CurDAG->getMachineNode(Opcode, DL, RetVTs, Ops);
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]2997 MachineSDNode::mmo_iterator MemRefs0 = MF->allocateMemRefsArray(1);
2998 MemRefs0[0] = cast<MemSDNode>(N)->getMemOperand();
2999 cast<MachineSDNode>(Ret)->setMemRefs(MemRefs0, MemRefs0 + 1);
3000
3001 return Ret;
3002}
3003
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]3004SDNode *NVPTXDAGToDAGISel::SelectTextureIntrinsic(SDNode *N) {
3005 SDValue Chain = N->getOperand(0);
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]3006 SDNode *Ret = nullptr;
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]3007 unsigned Opc = 0;
3008 SmallVector<SDValue, 8> Ops;
3009
3010 switch (N->getOpcode()) {
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]3011 default: return nullptr;
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]3012 case NVPTXISD::Tex1DFloatS32:
3013 Opc = NVPTX::TEX_1D_F32_S32;
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]3014 break;
3015 case NVPTXISD::Tex1DFloatFloat:
3016 Opc = NVPTX::TEX_1D_F32_F32;
3017 break;
3018 case NVPTXISD::Tex1DFloatFloatLevel:
3019 Opc = NVPTX::TEX_1D_F32_F32_LEVEL;
3020 break;
3021 case NVPTXISD::Tex1DFloatFloatGrad:
3022 Opc = NVPTX::TEX_1D_F32_F32_GRAD;
3023 break;
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]3024 case NVPTXISD::Tex1DS32S32:
3025 Opc = NVPTX::TEX_1D_S32_S32;
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]3026 break;
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]3027 case NVPTXISD::Tex1DS32Float:
3028 Opc = NVPTX::TEX_1D_S32_F32;
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]3029 break;
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]3030 case NVPTXISD::Tex1DS32FloatLevel:
3031 Opc = NVPTX::TEX_1D_S32_F32_LEVEL;
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]3032 break;
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]3033 case NVPTXISD::Tex1DS32FloatGrad:
3034 Opc = NVPTX::TEX_1D_S32_F32_GRAD;
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]3035 break;
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]3036 case NVPTXISD::Tex1DU32S32:
3037 Opc = NVPTX::TEX_1D_U32_S32;
3038 break;
3039 case NVPTXISD::Tex1DU32Float:
3040 Opc = NVPTX::TEX_1D_U32_F32;
3041 break;
3042 case NVPTXISD::Tex1DU32FloatLevel:
3043 Opc = NVPTX::TEX_1D_U32_F32_LEVEL;
3044 break;
3045 case NVPTXISD::Tex1DU32FloatGrad:
3046 Opc = NVPTX::TEX_1D_U32_F32_GRAD;
3047 break;
3048 case NVPTXISD::Tex1DArrayFloatS32:
3049 Opc = NVPTX::TEX_1D_ARRAY_F32_S32;
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]3050 break;
3051 case NVPTXISD::Tex1DArrayFloatFloat:
3052 Opc = NVPTX::TEX_1D_ARRAY_F32_F32;
3053 break;
3054 case NVPTXISD::Tex1DArrayFloatFloatLevel:
3055 Opc = NVPTX::TEX_1D_ARRAY_F32_F32_LEVEL;
3056 break;
3057 case NVPTXISD::Tex1DArrayFloatFloatGrad:
3058 Opc = NVPTX::TEX_1D_ARRAY_F32_F32_GRAD;
3059 break;
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]3060 case NVPTXISD::Tex1DArrayS32S32:
3061 Opc = NVPTX::TEX_1D_ARRAY_S32_S32;
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]3062 break;
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]3063 case NVPTXISD::Tex1DArrayS32Float:
3064 Opc = NVPTX::TEX_1D_ARRAY_S32_F32;
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]3065 break;
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]3066 case NVPTXISD::Tex1DArrayS32FloatLevel:
3067 Opc = NVPTX::TEX_1D_ARRAY_S32_F32_LEVEL;
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]3068 break;
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]3069 case NVPTXISD::Tex1DArrayS32FloatGrad:
3070 Opc = NVPTX::TEX_1D_ARRAY_S32_F32_GRAD;
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]3071 break;
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]3072 case NVPTXISD::Tex1DArrayU32S32:
3073 Opc = NVPTX::TEX_1D_ARRAY_U32_S32;
3074 break;
3075 case NVPTXISD::Tex1DArrayU32Float:
3076 Opc = NVPTX::TEX_1D_ARRAY_U32_F32;
3077 break;
3078 case NVPTXISD::Tex1DArrayU32FloatLevel:
3079 Opc = NVPTX::TEX_1D_ARRAY_U32_F32_LEVEL;
3080 break;
3081 case NVPTXISD::Tex1DArrayU32FloatGrad:
3082 Opc = NVPTX::TEX_1D_ARRAY_U32_F32_GRAD;
3083 break;
3084 case NVPTXISD::Tex2DFloatS32:
3085 Opc = NVPTX::TEX_2D_F32_S32;
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]3086 break;
3087 case NVPTXISD::Tex2DFloatFloat:
3088 Opc = NVPTX::TEX_2D_F32_F32;
3089 break;
3090 case NVPTXISD::Tex2DFloatFloatLevel:
3091 Opc = NVPTX::TEX_2D_F32_F32_LEVEL;
3092 break;
3093 case NVPTXISD::Tex2DFloatFloatGrad:
3094 Opc = NVPTX::TEX_2D_F32_F32_GRAD;
3095 break;
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]3096 case NVPTXISD::Tex2DS32S32:
3097 Opc = NVPTX::TEX_2D_S32_S32;
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]3098 break;
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]3099 case NVPTXISD::Tex2DS32Float:
3100 Opc = NVPTX::TEX_2D_S32_F32;
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]3101 break;
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]3102 case NVPTXISD::Tex2DS32FloatLevel:
3103 Opc = NVPTX::TEX_2D_S32_F32_LEVEL;
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]3104 break;
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]3105 case NVPTXISD::Tex2DS32FloatGrad:
3106 Opc = NVPTX::TEX_2D_S32_F32_GRAD;
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]3107 break;
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]3108 case NVPTXISD::Tex2DU32S32:
3109 Opc = NVPTX::TEX_2D_U32_S32;
3110 break;
3111 case NVPTXISD::Tex2DU32Float:
3112 Opc = NVPTX::TEX_2D_U32_F32;
3113 break;
3114 case NVPTXISD::Tex2DU32FloatLevel:
3115 Opc = NVPTX::TEX_2D_U32_F32_LEVEL;
3116 break;
3117 case NVPTXISD::Tex2DU32FloatGrad:
3118 Opc = NVPTX::TEX_2D_U32_F32_GRAD;
3119 break;
3120 case NVPTXISD::Tex2DArrayFloatS32:
3121 Opc = NVPTX::TEX_2D_ARRAY_F32_S32;
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]3122 break;
3123 case NVPTXISD::Tex2DArrayFloatFloat:
3124 Opc = NVPTX::TEX_2D_ARRAY_F32_F32;
3125 break;
3126 case NVPTXISD::Tex2DArrayFloatFloatLevel:
3127 Opc = NVPTX::TEX_2D_ARRAY_F32_F32_LEVEL;
3128 break;
3129 case NVPTXISD::Tex2DArrayFloatFloatGrad:
3130 Opc = NVPTX::TEX_2D_ARRAY_F32_F32_GRAD;
3131 break;
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]3132 case NVPTXISD::Tex2DArrayS32S32:
3133 Opc = NVPTX::TEX_2D_ARRAY_S32_S32;
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]3134 break;
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]3135 case NVPTXISD::Tex2DArrayS32Float:
3136 Opc = NVPTX::TEX_2D_ARRAY_S32_F32;
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]3137 break;
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]3138 case NVPTXISD::Tex2DArrayS32FloatLevel:
3139 Opc = NVPTX::TEX_2D_ARRAY_S32_F32_LEVEL;
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]3140 break;
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]3141 case NVPTXISD::Tex2DArrayS32FloatGrad:
3142 Opc = NVPTX::TEX_2D_ARRAY_S32_F32_GRAD;
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]3143 break;
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]3144 case NVPTXISD::Tex2DArrayU32S32:
3145 Opc = NVPTX::TEX_2D_ARRAY_U32_S32;
3146 break;
3147 case NVPTXISD::Tex2DArrayU32Float:
3148 Opc = NVPTX::TEX_2D_ARRAY_U32_F32;
3149 break;
3150 case NVPTXISD::Tex2DArrayU32FloatLevel:
3151 Opc = NVPTX::TEX_2D_ARRAY_U32_F32_LEVEL;
3152 break;
3153 case NVPTXISD::Tex2DArrayU32FloatGrad:
3154 Opc = NVPTX::TEX_2D_ARRAY_U32_F32_GRAD;
3155 break;
3156 case NVPTXISD::Tex3DFloatS32:
3157 Opc = NVPTX::TEX_3D_F32_S32;
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]3158 break;
3159 case NVPTXISD::Tex3DFloatFloat:
3160 Opc = NVPTX::TEX_3D_F32_F32;
3161 break;
3162 case NVPTXISD::Tex3DFloatFloatLevel:
3163 Opc = NVPTX::TEX_3D_F32_F32_LEVEL;
3164 break;
3165 case NVPTXISD::Tex3DFloatFloatGrad:
3166 Opc = NVPTX::TEX_3D_F32_F32_GRAD;
3167 break;
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]3168 case NVPTXISD::Tex3DS32S32:
3169 Opc = NVPTX::TEX_3D_S32_S32;
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]3170 break;
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]3171 case NVPTXISD::Tex3DS32Float:
3172 Opc = NVPTX::TEX_3D_S32_F32;
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]3173 break;
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]3174 case NVPTXISD::Tex3DS32FloatLevel:
3175 Opc = NVPTX::TEX_3D_S32_F32_LEVEL;
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]3176 break;
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]3177 case NVPTXISD::Tex3DS32FloatGrad:
3178 Opc = NVPTX::TEX_3D_S32_F32_GRAD;
3179 break;
3180 case NVPTXISD::Tex3DU32S32:
3181 Opc = NVPTX::TEX_3D_U32_S32;
3182 break;
3183 case NVPTXISD::Tex3DU32Float:
3184 Opc = NVPTX::TEX_3D_U32_F32;
3185 break;
3186 case NVPTXISD::Tex3DU32FloatLevel:
3187 Opc = NVPTX::TEX_3D_U32_F32_LEVEL;
3188 break;
3189 case NVPTXISD::Tex3DU32FloatGrad:
3190 Opc = NVPTX::TEX_3D_U32_F32_GRAD;
3191 break;
3192 case NVPTXISD::TexCubeFloatFloat:
3193 Opc = NVPTX::TEX_CUBE_F32_F32;
3194 break;
3195 case NVPTXISD::TexCubeFloatFloatLevel:
3196 Opc = NVPTX::TEX_CUBE_F32_F32_LEVEL;
3197 break;
3198 case NVPTXISD::TexCubeS32Float:
3199 Opc = NVPTX::TEX_CUBE_S32_F32;
3200 break;
3201 case NVPTXISD::TexCubeS32FloatLevel:
3202 Opc = NVPTX::TEX_CUBE_S32_F32_LEVEL;
3203 break;
3204 case NVPTXISD::TexCubeU32Float:
3205 Opc = NVPTX::TEX_CUBE_U32_F32;
3206 break;
3207 case NVPTXISD::TexCubeU32FloatLevel:
3208 Opc = NVPTX::TEX_CUBE_U32_F32_LEVEL;
3209 break;
3210 case NVPTXISD::TexCubeArrayFloatFloat:
3211 Opc = NVPTX::TEX_CUBE_ARRAY_F32_F32;
3212 break;
3213 case NVPTXISD::TexCubeArrayFloatFloatLevel:
3214 Opc = NVPTX::TEX_CUBE_ARRAY_F32_F32_LEVEL;
3215 break;
3216 case NVPTXISD::TexCubeArrayS32Float:
3217 Opc = NVPTX::TEX_CUBE_ARRAY_S32_F32;
3218 break;
3219 case NVPTXISD::TexCubeArrayS32FloatLevel:
3220 Opc = NVPTX::TEX_CUBE_ARRAY_S32_F32_LEVEL;
3221 break;
3222 case NVPTXISD::TexCubeArrayU32Float:
3223 Opc = NVPTX::TEX_CUBE_ARRAY_U32_F32;
3224 break;
3225 case NVPTXISD::TexCubeArrayU32FloatLevel:
3226 Opc = NVPTX::TEX_CUBE_ARRAY_U32_F32_LEVEL;
3227 break;
3228 case NVPTXISD::Tld4R2DFloatFloat:
3229 Opc = NVPTX::TLD4_R_2D_F32_F32;
3230 break;
3231 case NVPTXISD::Tld4G2DFloatFloat:
3232 Opc = NVPTX::TLD4_G_2D_F32_F32;
3233 break;
3234 case NVPTXISD::Tld4B2DFloatFloat:
3235 Opc = NVPTX::TLD4_B_2D_F32_F32;
3236 break;
3237 case NVPTXISD::Tld4A2DFloatFloat:
3238 Opc = NVPTX::TLD4_A_2D_F32_F32;
3239 break;
3240 case NVPTXISD::Tld4R2DS64Float:
3241 Opc = NVPTX::TLD4_R_2D_S32_F32;
3242 break;
3243 case NVPTXISD::Tld4G2DS64Float:
3244 Opc = NVPTX::TLD4_G_2D_S32_F32;
3245 break;
3246 case NVPTXISD::Tld4B2DS64Float:
3247 Opc = NVPTX::TLD4_B_2D_S32_F32;
3248 break;
3249 case NVPTXISD::Tld4A2DS64Float:
3250 Opc = NVPTX::TLD4_A_2D_S32_F32;
3251 break;
3252 case NVPTXISD::Tld4R2DU64Float:
3253 Opc = NVPTX::TLD4_R_2D_U32_F32;
3254 break;
3255 case NVPTXISD::Tld4G2DU64Float:
3256 Opc = NVPTX::TLD4_G_2D_U32_F32;
3257 break;
3258 case NVPTXISD::Tld4B2DU64Float:
3259 Opc = NVPTX::TLD4_B_2D_U32_F32;
3260 break;
3261 case NVPTXISD::Tld4A2DU64Float:
3262 Opc = NVPTX::TLD4_A_2D_U32_F32;
3263 break;
3264 case NVPTXISD::TexUnified1DFloatS32:
3265 Opc = NVPTX::TEX_UNIFIED_1D_F32_S32;
3266 break;
3267 case NVPTXISD::TexUnified1DFloatFloat:
3268 Opc = NVPTX::TEX_UNIFIED_1D_F32_F32;
3269 break;
3270 case NVPTXISD::TexUnified1DFloatFloatLevel:
3271 Opc = NVPTX::TEX_UNIFIED_1D_F32_F32_LEVEL;
3272 break;
3273 case NVPTXISD::TexUnified1DFloatFloatGrad:
3274 Opc = NVPTX::TEX_UNIFIED_1D_F32_F32_GRAD;
3275 break;
3276 case NVPTXISD::TexUnified1DS32S32:
3277 Opc = NVPTX::TEX_UNIFIED_1D_S32_S32;
3278 break;
3279 case NVPTXISD::TexUnified1DS32Float:
3280 Opc = NVPTX::TEX_UNIFIED_1D_S32_F32;
3281 break;
3282 case NVPTXISD::TexUnified1DS32FloatLevel:
3283 Opc = NVPTX::TEX_UNIFIED_1D_S32_F32_LEVEL;
3284 break;
3285 case NVPTXISD::TexUnified1DS32FloatGrad:
3286 Opc = NVPTX::TEX_UNIFIED_1D_S32_F32_GRAD;
3287 break;
3288 case NVPTXISD::TexUnified1DU32S32:
3289 Opc = NVPTX::TEX_UNIFIED_1D_U32_S32;
3290 break;
3291 case NVPTXISD::TexUnified1DU32Float:
3292 Opc = NVPTX::TEX_UNIFIED_1D_U32_F32;
3293 break;
3294 case NVPTXISD::TexUnified1DU32FloatLevel:
3295 Opc = NVPTX::TEX_UNIFIED_1D_U32_F32_LEVEL;
3296 break;
3297 case NVPTXISD::TexUnified1DU32FloatGrad:
3298 Opc = NVPTX::TEX_UNIFIED_1D_U32_F32_GRAD;
3299 break;
3300 case NVPTXISD::TexUnified1DArrayFloatS32:
3301 Opc = NVPTX::TEX_UNIFIED_1D_ARRAY_F32_S32;
3302 break;
3303 case NVPTXISD::TexUnified1DArrayFloatFloat:
3304 Opc = NVPTX::TEX_UNIFIED_1D_ARRAY_F32_F32;
3305 break;
3306 case NVPTXISD::TexUnified1DArrayFloatFloatLevel:
3307 Opc = NVPTX::TEX_UNIFIED_1D_ARRAY_F32_F32_LEVEL;
3308 break;
3309 case NVPTXISD::TexUnified1DArrayFloatFloatGrad:
3310 Opc = NVPTX::TEX_UNIFIED_1D_ARRAY_F32_F32_GRAD;
3311 break;
3312 case NVPTXISD::TexUnified1DArrayS32S32:
3313 Opc = NVPTX::TEX_UNIFIED_1D_ARRAY_S32_S32;
3314 break;
3315 case NVPTXISD::TexUnified1DArrayS32Float:
3316 Opc = NVPTX::TEX_UNIFIED_1D_ARRAY_S32_F32;
3317 break;
3318 case NVPTXISD::TexUnified1DArrayS32FloatLevel:
3319 Opc = NVPTX::TEX_UNIFIED_1D_ARRAY_S32_F32_LEVEL;
3320 break;
3321 case NVPTXISD::TexUnified1DArrayS32FloatGrad:
3322 Opc = NVPTX::TEX_UNIFIED_1D_ARRAY_S32_F32_GRAD;
3323 break;
3324 case NVPTXISD::TexUnified1DArrayU32S32:
3325 Opc = NVPTX::TEX_UNIFIED_1D_ARRAY_U32_S32;
3326 break;
3327 case NVPTXISD::TexUnified1DArrayU32Float:
3328 Opc = NVPTX::TEX_UNIFIED_1D_ARRAY_U32_F32;
3329 break;
3330 case NVPTXISD::TexUnified1DArrayU32FloatLevel:
3331 Opc = NVPTX::TEX_UNIFIED_1D_ARRAY_U32_F32_LEVEL;
3332 break;
3333 case NVPTXISD::TexUnified1DArrayU32FloatGrad:
3334 Opc = NVPTX::TEX_UNIFIED_1D_ARRAY_U32_F32_GRAD;
3335 break;
3336 case NVPTXISD::TexUnified2DFloatS32:
3337 Opc = NVPTX::TEX_UNIFIED_2D_F32_S32;
3338 break;
3339 case NVPTXISD::TexUnified2DFloatFloat:
3340 Opc = NVPTX::TEX_UNIFIED_2D_F32_F32;
3341 break;
3342 case NVPTXISD::TexUnified2DFloatFloatLevel:
3343 Opc = NVPTX::TEX_UNIFIED_2D_F32_F32_LEVEL;
3344 break;
3345 case NVPTXISD::TexUnified2DFloatFloatGrad:
3346 Opc = NVPTX::TEX_UNIFIED_2D_F32_F32_GRAD;
3347 break;
3348 case NVPTXISD::TexUnified2DS32S32:
3349 Opc = NVPTX::TEX_UNIFIED_2D_S32_S32;
3350 break;
3351 case NVPTXISD::TexUnified2DS32Float:
3352 Opc = NVPTX::TEX_UNIFIED_2D_S32_F32;
3353 break;
3354 case NVPTXISD::TexUnified2DS32FloatLevel:
3355 Opc = NVPTX::TEX_UNIFIED_2D_S32_F32_LEVEL;
3356 break;
3357 case NVPTXISD::TexUnified2DS32FloatGrad:
3358 Opc = NVPTX::TEX_UNIFIED_2D_S32_F32_GRAD;
3359 break;
3360 case NVPTXISD::TexUnified2DU32S32:
3361 Opc = NVPTX::TEX_UNIFIED_2D_U32_S32;
3362 break;
3363 case NVPTXISD::TexUnified2DU32Float:
3364 Opc = NVPTX::TEX_UNIFIED_2D_U32_F32;
3365 break;
3366 case NVPTXISD::TexUnified2DU32FloatLevel:
3367 Opc = NVPTX::TEX_UNIFIED_2D_U32_F32_LEVEL;
3368 break;
3369 case NVPTXISD::TexUnified2DU32FloatGrad:
3370 Opc = NVPTX::TEX_UNIFIED_2D_U32_F32_GRAD;
3371 break;
3372 case NVPTXISD::TexUnified2DArrayFloatS32:
3373 Opc = NVPTX::TEX_UNIFIED_2D_ARRAY_F32_S32;
3374 break;
3375 case NVPTXISD::TexUnified2DArrayFloatFloat:
3376 Opc = NVPTX::TEX_UNIFIED_2D_ARRAY_F32_F32;
3377 break;
3378 case NVPTXISD::TexUnified2DArrayFloatFloatLevel:
3379 Opc = NVPTX::TEX_UNIFIED_2D_ARRAY_F32_F32_LEVEL;
3380 break;
3381 case NVPTXISD::TexUnified2DArrayFloatFloatGrad:
3382 Opc = NVPTX::TEX_UNIFIED_2D_ARRAY_F32_F32_GRAD;
3383 break;
3384 case NVPTXISD::TexUnified2DArrayS32S32:
3385 Opc = NVPTX::TEX_UNIFIED_2D_ARRAY_S32_S32;
3386 break;
3387 case NVPTXISD::TexUnified2DArrayS32Float:
3388 Opc = NVPTX::TEX_UNIFIED_2D_ARRAY_S32_F32;
3389 break;
3390 case NVPTXISD::TexUnified2DArrayS32FloatLevel:
3391 Opc = NVPTX::TEX_UNIFIED_2D_ARRAY_S32_F32_LEVEL;
3392 break;
3393 case NVPTXISD::TexUnified2DArrayS32FloatGrad:
3394 Opc = NVPTX::TEX_UNIFIED_2D_ARRAY_S32_F32_GRAD;
3395 break;
3396 case NVPTXISD::TexUnified2DArrayU32S32:
3397 Opc = NVPTX::TEX_UNIFIED_2D_ARRAY_U32_S32;
3398 break;
3399 case NVPTXISD::TexUnified2DArrayU32Float:
3400 Opc = NVPTX::TEX_UNIFIED_2D_ARRAY_U32_F32;
3401 break;
3402 case NVPTXISD::TexUnified2DArrayU32FloatLevel:
3403 Opc = NVPTX::TEX_UNIFIED_2D_ARRAY_U32_F32_LEVEL;
3404 break;
3405 case NVPTXISD::TexUnified2DArrayU32FloatGrad:
3406 Opc = NVPTX::TEX_UNIFIED_2D_ARRAY_U32_F32_GRAD;
3407 break;
3408 case NVPTXISD::TexUnified3DFloatS32:
3409 Opc = NVPTX::TEX_UNIFIED_3D_F32_S32;
3410 break;
3411 case NVPTXISD::TexUnified3DFloatFloat:
3412 Opc = NVPTX::TEX_UNIFIED_3D_F32_F32;
3413 break;
3414 case NVPTXISD::TexUnified3DFloatFloatLevel:
3415 Opc = NVPTX::TEX_UNIFIED_3D_F32_F32_LEVEL;
3416 break;
3417 case NVPTXISD::TexUnified3DFloatFloatGrad:
3418 Opc = NVPTX::TEX_UNIFIED_3D_F32_F32_GRAD;
3419 break;
3420 case NVPTXISD::TexUnified3DS32S32:
3421 Opc = NVPTX::TEX_UNIFIED_3D_S32_S32;
3422 break;
3423 case NVPTXISD::TexUnified3DS32Float:
3424 Opc = NVPTX::TEX_UNIFIED_3D_S32_F32;
3425 break;
3426 case NVPTXISD::TexUnified3DS32FloatLevel:
3427 Opc = NVPTX::TEX_UNIFIED_3D_S32_F32_LEVEL;
3428 break;
3429 case NVPTXISD::TexUnified3DS32FloatGrad:
3430 Opc = NVPTX::TEX_UNIFIED_3D_S32_F32_GRAD;
3431 break;
3432 case NVPTXISD::TexUnified3DU32S32:
3433 Opc = NVPTX::TEX_UNIFIED_3D_U32_S32;
3434 break;
3435 case NVPTXISD::TexUnified3DU32Float:
3436 Opc = NVPTX::TEX_UNIFIED_3D_U32_F32;
3437 break;
3438 case NVPTXISD::TexUnified3DU32FloatLevel:
3439 Opc = NVPTX::TEX_UNIFIED_3D_U32_F32_LEVEL;
3440 break;
3441 case NVPTXISD::TexUnified3DU32FloatGrad:
3442 Opc = NVPTX::TEX_UNIFIED_3D_U32_F32_GRAD;
3443 break;
3444 case NVPTXISD::TexUnifiedCubeFloatFloat:
3445 Opc = NVPTX::TEX_UNIFIED_CUBE_F32_F32;
3446 break;
3447 case NVPTXISD::TexUnifiedCubeFloatFloatLevel:
3448 Opc = NVPTX::TEX_UNIFIED_CUBE_F32_F32_LEVEL;
3449 break;
3450 case NVPTXISD::TexUnifiedCubeS32Float:
3451 Opc = NVPTX::TEX_UNIFIED_CUBE_S32_F32;
3452 break;
3453 case NVPTXISD::TexUnifiedCubeS32FloatLevel:
3454 Opc = NVPTX::TEX_UNIFIED_CUBE_S32_F32_LEVEL;
3455 break;
3456 case NVPTXISD::TexUnifiedCubeU32Float:
3457 Opc = NVPTX::TEX_UNIFIED_CUBE_U32_F32;
3458 break;
3459 case NVPTXISD::TexUnifiedCubeU32FloatLevel:
3460 Opc = NVPTX::TEX_UNIFIED_CUBE_U32_F32_LEVEL;
3461 break;
3462 case NVPTXISD::TexUnifiedCubeArrayFloatFloat:
3463 Opc = NVPTX::TEX_UNIFIED_CUBE_ARRAY_F32_F32;
3464 break;
3465 case NVPTXISD::TexUnifiedCubeArrayFloatFloatLevel:
3466 Opc = NVPTX::TEX_UNIFIED_CUBE_ARRAY_F32_F32_LEVEL;
3467 break;
3468 case NVPTXISD::TexUnifiedCubeArrayS32Float:
3469 Opc = NVPTX::TEX_UNIFIED_CUBE_ARRAY_S32_F32;
3470 break;
3471 case NVPTXISD::TexUnifiedCubeArrayS32FloatLevel:
3472 Opc = NVPTX::TEX_UNIFIED_CUBE_ARRAY_S32_F32_LEVEL;
3473 break;
3474 case NVPTXISD::TexUnifiedCubeArrayU32Float:
3475 Opc = NVPTX::TEX_UNIFIED_CUBE_ARRAY_U32_F32;
3476 break;
3477 case NVPTXISD::TexUnifiedCubeArrayU32FloatLevel:
3478 Opc = NVPTX::TEX_UNIFIED_CUBE_ARRAY_U32_F32_LEVEL;
3479 break;
3480 case NVPTXISD::Tld4UnifiedR2DFloatFloat:
3481 Opc = NVPTX::TLD4_UNIFIED_R_2D_F32_F32;
3482 break;
3483 case NVPTXISD::Tld4UnifiedG2DFloatFloat:
3484 Opc = NVPTX::TLD4_UNIFIED_G_2D_F32_F32;
3485 break;
3486 case NVPTXISD::Tld4UnifiedB2DFloatFloat:
3487 Opc = NVPTX::TLD4_UNIFIED_B_2D_F32_F32;
3488 break;
3489 case NVPTXISD::Tld4UnifiedA2DFloatFloat:
3490 Opc = NVPTX::TLD4_UNIFIED_A_2D_F32_F32;
3491 break;
3492 case NVPTXISD::Tld4UnifiedR2DS64Float:
3493 Opc = NVPTX::TLD4_UNIFIED_R_2D_S32_F32;
3494 break;
3495 case NVPTXISD::Tld4UnifiedG2DS64Float:
3496 Opc = NVPTX::TLD4_UNIFIED_G_2D_S32_F32;
3497 break;
3498 case NVPTXISD::Tld4UnifiedB2DS64Float:
3499 Opc = NVPTX::TLD4_UNIFIED_B_2D_S32_F32;
3500 break;
3501 case NVPTXISD::Tld4UnifiedA2DS64Float:
3502 Opc = NVPTX::TLD4_UNIFIED_A_2D_S32_F32;
3503 break;
3504 case NVPTXISD::Tld4UnifiedR2DU64Float:
3505 Opc = NVPTX::TLD4_UNIFIED_R_2D_U32_F32;
3506 break;
3507 case NVPTXISD::Tld4UnifiedG2DU64Float:
3508 Opc = NVPTX::TLD4_UNIFIED_G_2D_U32_F32;
3509 break;
3510 case NVPTXISD::Tld4UnifiedB2DU64Float:
3511 Opc = NVPTX::TLD4_UNIFIED_B_2D_U32_F32;
3512 break;
3513 case NVPTXISD::Tld4UnifiedA2DU64Float:
3514 Opc = NVPTX::TLD4_UNIFIED_A_2D_U32_F32;
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]3515 break;
3516 }
3517
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]3518 // Copy over operands
3519 for (unsigned i = 1; i < N->getNumOperands(); ++i) {
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]3520 Ops.push_back(N->getOperand(i));
3521 }
3522
3523 Ops.push_back(Chain);
3524 Ret = CurDAG->getMachineNode(Opc, SDLoc(N), N->getVTList(), Ops);
3525 return Ret;
3526}
3527
3528SDNode *NVPTXDAGToDAGISel::SelectSurfaceIntrinsic(SDNode *N) {
3529 SDValue Chain = N->getOperand(0);
3530 SDValue TexHandle = N->getOperand(1);
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]3531 SDNode *Ret = nullptr;
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]3532 unsigned Opc = 0;
3533 SmallVector<SDValue, 8> Ops;
3534 switch (N->getOpcode()) {
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]3535 default: return nullptr;
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]3536 case NVPTXISD::Suld1DI8Clamp:
3537 Opc = NVPTX::SULD_1D_I8_CLAMP;
3538 Ops.push_back(TexHandle);
3539 Ops.push_back(N->getOperand(2));
3540 Ops.push_back(Chain);
3541 break;
3542 case NVPTXISD::Suld1DI16Clamp:
3543 Opc = NVPTX::SULD_1D_I16_CLAMP;
3544 Ops.push_back(TexHandle);
3545 Ops.push_back(N->getOperand(2));
3546 Ops.push_back(Chain);
3547 break;
3548 case NVPTXISD::Suld1DI32Clamp:
3549 Opc = NVPTX::SULD_1D_I32_CLAMP;
3550 Ops.push_back(TexHandle);
3551 Ops.push_back(N->getOperand(2));
3552 Ops.push_back(Chain);
3553 break;
3554 case NVPTXISD::Suld1DI64Clamp:
3555 Opc = NVPTX::SULD_1D_I64_CLAMP;
3556 Ops.push_back(TexHandle);
3557 Ops.push_back(N->getOperand(2));
3558 Ops.push_back(Chain);
3559 break;
3560 case NVPTXISD::Suld1DV2I8Clamp:
3561 Opc = NVPTX::SULD_1D_V2I8_CLAMP;
3562 Ops.push_back(TexHandle);
3563 Ops.push_back(N->getOperand(2));
3564 Ops.push_back(Chain);
3565 break;
3566 case NVPTXISD::Suld1DV2I16Clamp:
3567 Opc = NVPTX::SULD_1D_V2I16_CLAMP;
3568 Ops.push_back(TexHandle);
3569 Ops.push_back(N->getOperand(2));
3570 Ops.push_back(Chain);
3571 break;
3572 case NVPTXISD::Suld1DV2I32Clamp:
3573 Opc = NVPTX::SULD_1D_V2I32_CLAMP;
3574 Ops.push_back(TexHandle);
3575 Ops.push_back(N->getOperand(2));
3576 Ops.push_back(Chain);
3577 break;
3578 case NVPTXISD::Suld1DV2I64Clamp:
3579 Opc = NVPTX::SULD_1D_V2I64_CLAMP;
3580 Ops.push_back(TexHandle);
3581 Ops.push_back(N->getOperand(2));
3582 Ops.push_back(Chain);
3583 break;
3584 case NVPTXISD::Suld1DV4I8Clamp:
3585 Opc = NVPTX::SULD_1D_V4I8_CLAMP;
3586 Ops.push_back(TexHandle);
3587 Ops.push_back(N->getOperand(2));
3588 Ops.push_back(Chain);
3589 break;
3590 case NVPTXISD::Suld1DV4I16Clamp:
3591 Opc = NVPTX::SULD_1D_V4I16_CLAMP;
3592 Ops.push_back(TexHandle);
3593 Ops.push_back(N->getOperand(2));
3594 Ops.push_back(Chain);
3595 break;
3596 case NVPTXISD::Suld1DV4I32Clamp:
3597 Opc = NVPTX::SULD_1D_V4I32_CLAMP;
3598 Ops.push_back(TexHandle);
3599 Ops.push_back(N->getOperand(2));
3600 Ops.push_back(Chain);
3601 break;
3602 case NVPTXISD::Suld1DArrayI8Clamp:
3603 Opc = NVPTX::SULD_1D_ARRAY_I8_CLAMP;
3604 Ops.push_back(TexHandle);
3605 Ops.push_back(N->getOperand(2));
3606 Ops.push_back(N->getOperand(3));
3607 Ops.push_back(Chain);
3608 break;
3609 case NVPTXISD::Suld1DArrayI16Clamp:
3610 Opc = NVPTX::SULD_1D_ARRAY_I16_CLAMP;
3611 Ops.push_back(TexHandle);
3612 Ops.push_back(N->getOperand(2));
3613 Ops.push_back(N->getOperand(3));
3614 Ops.push_back(Chain);
3615 break;
3616 case NVPTXISD::Suld1DArrayI32Clamp:
3617 Opc = NVPTX::SULD_1D_ARRAY_I32_CLAMP;
3618 Ops.push_back(TexHandle);
3619 Ops.push_back(N->getOperand(2));
3620 Ops.push_back(N->getOperand(3));
3621 Ops.push_back(Chain);
3622 break;
3623 case NVPTXISD::Suld1DArrayI64Clamp:
3624 Opc = NVPTX::SULD_1D_ARRAY_I64_CLAMP;
3625 Ops.push_back(TexHandle);
3626 Ops.push_back(N->getOperand(2));
3627 Ops.push_back(N->getOperand(3));
3628 Ops.push_back(Chain);
3629 break;
3630 case NVPTXISD::Suld1DArrayV2I8Clamp:
3631 Opc = NVPTX::SULD_1D_ARRAY_V2I8_CLAMP;
3632 Ops.push_back(TexHandle);
3633 Ops.push_back(N->getOperand(2));
3634 Ops.push_back(N->getOperand(3));
3635 Ops.push_back(Chain);
3636 break;
3637 case NVPTXISD::Suld1DArrayV2I16Clamp:
3638 Opc = NVPTX::SULD_1D_ARRAY_V2I16_CLAMP;
3639 Ops.push_back(TexHandle);
3640 Ops.push_back(N->getOperand(2));
3641 Ops.push_back(N->getOperand(3));
3642 Ops.push_back(Chain);
3643 break;
3644 case NVPTXISD::Suld1DArrayV2I32Clamp:
3645 Opc = NVPTX::SULD_1D_ARRAY_V2I32_CLAMP;
3646 Ops.push_back(TexHandle);
3647 Ops.push_back(N->getOperand(2));
3648 Ops.push_back(N->getOperand(3));
3649 Ops.push_back(Chain);
3650 break;
3651 case NVPTXISD::Suld1DArrayV2I64Clamp:
3652 Opc = NVPTX::SULD_1D_ARRAY_V2I64_CLAMP;
3653 Ops.push_back(TexHandle);
3654 Ops.push_back(N->getOperand(2));
3655 Ops.push_back(N->getOperand(3));
3656 Ops.push_back(Chain);
3657 break;
3658 case NVPTXISD::Suld1DArrayV4I8Clamp:
3659 Opc = NVPTX::SULD_1D_ARRAY_V4I8_CLAMP;
3660 Ops.push_back(TexHandle);
3661 Ops.push_back(N->getOperand(2));
3662 Ops.push_back(N->getOperand(3));
3663 Ops.push_back(Chain);
3664 break;
3665 case NVPTXISD::Suld1DArrayV4I16Clamp:
3666 Opc = NVPTX::SULD_1D_ARRAY_V4I16_CLAMP;
3667 Ops.push_back(TexHandle);
3668 Ops.push_back(N->getOperand(2));
3669 Ops.push_back(N->getOperand(3));
3670 Ops.push_back(Chain);
3671 break;
3672 case NVPTXISD::Suld1DArrayV4I32Clamp:
3673 Opc = NVPTX::SULD_1D_ARRAY_V4I32_CLAMP;
3674 Ops.push_back(TexHandle);
3675 Ops.push_back(N->getOperand(2));
3676 Ops.push_back(N->getOperand(3));
3677 Ops.push_back(Chain);
3678 break;
3679 case NVPTXISD::Suld2DI8Clamp:
3680 Opc = NVPTX::SULD_2D_I8_CLAMP;
3681 Ops.push_back(TexHandle);
3682 Ops.push_back(N->getOperand(2));
3683 Ops.push_back(N->getOperand(3));
3684 Ops.push_back(Chain);
3685 break;
3686 case NVPTXISD::Suld2DI16Clamp:
3687 Opc = NVPTX::SULD_2D_I16_CLAMP;
3688 Ops.push_back(TexHandle);
3689 Ops.push_back(N->getOperand(2));
3690 Ops.push_back(N->getOperand(3));
3691 Ops.push_back(Chain);
3692 break;
3693 case NVPTXISD::Suld2DI32Clamp:
3694 Opc = NVPTX::SULD_2D_I32_CLAMP;
3695 Ops.push_back(TexHandle);
3696 Ops.push_back(N->getOperand(2));
3697 Ops.push_back(N->getOperand(3));
3698 Ops.push_back(Chain);
3699 break;
3700 case NVPTXISD::Suld2DI64Clamp:
3701 Opc = NVPTX::SULD_2D_I64_CLAMP;
3702 Ops.push_back(TexHandle);
3703 Ops.push_back(N->getOperand(2));
3704 Ops.push_back(N->getOperand(3));
3705 Ops.push_back(Chain);
3706 break;
3707 case NVPTXISD::Suld2DV2I8Clamp:
3708 Opc = NVPTX::SULD_2D_V2I8_CLAMP;
3709 Ops.push_back(TexHandle);
3710 Ops.push_back(N->getOperand(2));
3711 Ops.push_back(N->getOperand(3));
3712 Ops.push_back(Chain);
3713 break;
3714 case NVPTXISD::Suld2DV2I16Clamp:
3715 Opc = NVPTX::SULD_2D_V2I16_CLAMP;
3716 Ops.push_back(TexHandle);
3717 Ops.push_back(N->getOperand(2));
3718 Ops.push_back(N->getOperand(3));
3719 Ops.push_back(Chain);
3720 break;
3721 case NVPTXISD::Suld2DV2I32Clamp:
3722 Opc = NVPTX::SULD_2D_V2I32_CLAMP;
3723 Ops.push_back(TexHandle);
3724 Ops.push_back(N->getOperand(2));
3725 Ops.push_back(N->getOperand(3));
3726 Ops.push_back(Chain);
3727 break;
3728 case NVPTXISD::Suld2DV2I64Clamp:
3729 Opc = NVPTX::SULD_2D_V2I64_CLAMP;
3730 Ops.push_back(TexHandle);
3731 Ops.push_back(N->getOperand(2));
3732 Ops.push_back(N->getOperand(3));
3733 Ops.push_back(Chain);
3734 break;
3735 case NVPTXISD::Suld2DV4I8Clamp:
3736 Opc = NVPTX::SULD_2D_V4I8_CLAMP;
3737 Ops.push_back(TexHandle);
3738 Ops.push_back(N->getOperand(2));
3739 Ops.push_back(N->getOperand(3));
3740 Ops.push_back(Chain);
3741 break;
3742 case NVPTXISD::Suld2DV4I16Clamp:
3743 Opc = NVPTX::SULD_2D_V4I16_CLAMP;
3744 Ops.push_back(TexHandle);
3745 Ops.push_back(N->getOperand(2));
3746 Ops.push_back(N->getOperand(3));
3747 Ops.push_back(Chain);
3748 break;
3749 case NVPTXISD::Suld2DV4I32Clamp:
3750 Opc = NVPTX::SULD_2D_V4I32_CLAMP;
3751 Ops.push_back(TexHandle);
3752 Ops.push_back(N->getOperand(2));
3753 Ops.push_back(N->getOperand(3));
3754 Ops.push_back(Chain);
3755 break;
3756 case NVPTXISD::Suld2DArrayI8Clamp:
3757 Opc = NVPTX::SULD_2D_ARRAY_I8_CLAMP;
3758 Ops.push_back(TexHandle);
3759 Ops.push_back(N->getOperand(2));
3760 Ops.push_back(N->getOperand(3));
3761 Ops.push_back(N->getOperand(4));
3762 Ops.push_back(Chain);
3763 break;
3764 case NVPTXISD::Suld2DArrayI16Clamp:
3765 Opc = NVPTX::SULD_2D_ARRAY_I16_CLAMP;
3766 Ops.push_back(TexHandle);
3767 Ops.push_back(N->getOperand(2));
3768 Ops.push_back(N->getOperand(3));
3769 Ops.push_back(N->getOperand(4));
3770 Ops.push_back(Chain);
3771 break;
3772 case NVPTXISD::Suld2DArrayI32Clamp:
3773 Opc = NVPTX::SULD_2D_ARRAY_I32_CLAMP;
3774 Ops.push_back(TexHandle);
3775 Ops.push_back(N->getOperand(2));
3776 Ops.push_back(N->getOperand(3));
3777 Ops.push_back(N->getOperand(4));
3778 Ops.push_back(Chain);
3779 break;
3780 case NVPTXISD::Suld2DArrayI64Clamp:
3781 Opc = NVPTX::SULD_2D_ARRAY_I64_CLAMP;
3782 Ops.push_back(TexHandle);
3783 Ops.push_back(N->getOperand(2));
3784 Ops.push_back(N->getOperand(3));
3785 Ops.push_back(N->getOperand(4));
3786 Ops.push_back(Chain);
3787 break;
3788 case NVPTXISD::Suld2DArrayV2I8Clamp:
3789 Opc = NVPTX::SULD_2D_ARRAY_V2I8_CLAMP;
3790 Ops.push_back(TexHandle);
3791 Ops.push_back(N->getOperand(2));
3792 Ops.push_back(N->getOperand(3));
3793 Ops.push_back(N->getOperand(4));
3794 Ops.push_back(Chain);
3795 break;
3796 case NVPTXISD::Suld2DArrayV2I16Clamp:
3797 Opc = NVPTX::SULD_2D_ARRAY_V2I16_CLAMP;
3798 Ops.push_back(TexHandle);
3799 Ops.push_back(N->getOperand(2));
3800 Ops.push_back(N->getOperand(3));
3801 Ops.push_back(N->getOperand(4));
3802 Ops.push_back(Chain);
3803 break;
3804 case NVPTXISD::Suld2DArrayV2I32Clamp:
3805 Opc = NVPTX::SULD_2D_ARRAY_V2I32_CLAMP;
3806 Ops.push_back(TexHandle);
3807 Ops.push_back(N->getOperand(2));
3808 Ops.push_back(N->getOperand(3));
3809 Ops.push_back(N->getOperand(4));
3810 Ops.push_back(Chain);
3811 break;
3812 case NVPTXISD::Suld2DArrayV2I64Clamp:
3813 Opc = NVPTX::SULD_2D_ARRAY_V2I64_CLAMP;
3814 Ops.push_back(TexHandle);
3815 Ops.push_back(N->getOperand(2));
3816 Ops.push_back(N->getOperand(3));
3817 Ops.push_back(N->getOperand(4));
3818 Ops.push_back(Chain);
3819 break;
3820 case NVPTXISD::Suld2DArrayV4I8Clamp:
3821 Opc = NVPTX::SULD_2D_ARRAY_V4I8_CLAMP;
3822 Ops.push_back(TexHandle);
3823 Ops.push_back(N->getOperand(2));
3824 Ops.push_back(N->getOperand(3));
3825 Ops.push_back(N->getOperand(4));
3826 Ops.push_back(Chain);
3827 break;
3828 case NVPTXISD::Suld2DArrayV4I16Clamp:
3829 Opc = NVPTX::SULD_2D_ARRAY_V4I16_CLAMP;
3830 Ops.push_back(TexHandle);
3831 Ops.push_back(N->getOperand(2));
3832 Ops.push_back(N->getOperand(3));
3833 Ops.push_back(N->getOperand(4));
3834 Ops.push_back(Chain);
3835 break;
3836 case NVPTXISD::Suld2DArrayV4I32Clamp:
3837 Opc = NVPTX::SULD_2D_ARRAY_V4I32_CLAMP;
3838 Ops.push_back(TexHandle);
3839 Ops.push_back(N->getOperand(2));
3840 Ops.push_back(N->getOperand(3));
3841 Ops.push_back(N->getOperand(4));
3842 Ops.push_back(Chain);
3843 break;
3844 case NVPTXISD::Suld3DI8Clamp:
3845 Opc = NVPTX::SULD_3D_I8_CLAMP;
3846 Ops.push_back(TexHandle);
3847 Ops.push_back(N->getOperand(2));
3848 Ops.push_back(N->getOperand(3));
3849 Ops.push_back(N->getOperand(4));
3850 Ops.push_back(Chain);
3851 break;
3852 case NVPTXISD::Suld3DI16Clamp:
3853 Opc = NVPTX::SULD_3D_I16_CLAMP;
3854 Ops.push_back(TexHandle);
3855 Ops.push_back(N->getOperand(2));
3856 Ops.push_back(N->getOperand(3));
3857 Ops.push_back(N->getOperand(4));
3858 Ops.push_back(Chain);
3859 break;
3860 case NVPTXISD::Suld3DI32Clamp:
3861 Opc = NVPTX::SULD_3D_I32_CLAMP;
3862 Ops.push_back(TexHandle);
3863 Ops.push_back(N->getOperand(2));
3864 Ops.push_back(N->getOperand(3));
3865 Ops.push_back(N->getOperand(4));
3866 Ops.push_back(Chain);
3867 break;
3868 case NVPTXISD::Suld3DI64Clamp:
3869 Opc = NVPTX::SULD_3D_I64_CLAMP;
3870 Ops.push_back(TexHandle);
3871 Ops.push_back(N->getOperand(2));
3872 Ops.push_back(N->getOperand(3));
3873 Ops.push_back(N->getOperand(4));
3874 Ops.push_back(Chain);
3875 break;
3876 case NVPTXISD::Suld3DV2I8Clamp:
3877 Opc = NVPTX::SULD_3D_V2I8_CLAMP;
3878 Ops.push_back(TexHandle);
3879 Ops.push_back(N->getOperand(2));
3880 Ops.push_back(N->getOperand(3));
3881 Ops.push_back(N->getOperand(4));
3882 Ops.push_back(Chain);
3883 break;
3884 case NVPTXISD::Suld3DV2I16Clamp:
3885 Opc = NVPTX::SULD_3D_V2I16_CLAMP;
3886 Ops.push_back(TexHandle);
3887 Ops.push_back(N->getOperand(2));
3888 Ops.push_back(N->getOperand(3));
3889 Ops.push_back(N->getOperand(4));
3890 Ops.push_back(Chain);
3891 break;
3892 case NVPTXISD::Suld3DV2I32Clamp:
3893 Opc = NVPTX::SULD_3D_V2I32_CLAMP;
3894 Ops.push_back(TexHandle);
3895 Ops.push_back(N->getOperand(2));
3896 Ops.push_back(N->getOperand(3));
3897 Ops.push_back(N->getOperand(4));
3898 Ops.push_back(Chain);
3899 break;
3900 case NVPTXISD::Suld3DV2I64Clamp:
3901 Opc = NVPTX::SULD_3D_V2I64_CLAMP;
3902 Ops.push_back(TexHandle);
3903 Ops.push_back(N->getOperand(2));
3904 Ops.push_back(N->getOperand(3));
3905 Ops.push_back(N->getOperand(4));
3906 Ops.push_back(Chain);
3907 break;
3908 case NVPTXISD::Suld3DV4I8Clamp:
3909 Opc = NVPTX::SULD_3D_V4I8_CLAMP;
3910 Ops.push_back(TexHandle);
3911 Ops.push_back(N->getOperand(2));
3912 Ops.push_back(N->getOperand(3));
3913 Ops.push_back(N->getOperand(4));
3914 Ops.push_back(Chain);
3915 break;
3916 case NVPTXISD::Suld3DV4I16Clamp:
3917 Opc = NVPTX::SULD_3D_V4I16_CLAMP;
3918 Ops.push_back(TexHandle);
3919 Ops.push_back(N->getOperand(2));
3920 Ops.push_back(N->getOperand(3));
3921 Ops.push_back(N->getOperand(4));
3922 Ops.push_back(Chain);
3923 break;
3924 case NVPTXISD::Suld3DV4I32Clamp:
3925 Opc = NVPTX::SULD_3D_V4I32_CLAMP;
3926 Ops.push_back(TexHandle);
3927 Ops.push_back(N->getOperand(2));
3928 Ops.push_back(N->getOperand(3));
3929 Ops.push_back(N->getOperand(4));
3930 Ops.push_back(Chain);
3931 break;
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]3932 case NVPTXISD::Suld1DI8Trap:
3933 Opc = NVPTX::SULD_1D_I8_TRAP;
3934 Ops.push_back(TexHandle);
3935 Ops.push_back(N->getOperand(2));
3936 Ops.push_back(Chain);
3937 break;
3938 case NVPTXISD::Suld1DI16Trap:
3939 Opc = NVPTX::SULD_1D_I16_TRAP;
3940 Ops.push_back(TexHandle);
3941 Ops.push_back(N->getOperand(2));
3942 Ops.push_back(Chain);
3943 break;
3944 case NVPTXISD::Suld1DI32Trap:
3945 Opc = NVPTX::SULD_1D_I32_TRAP;
3946 Ops.push_back(TexHandle);
3947 Ops.push_back(N->getOperand(2));
3948 Ops.push_back(Chain);
3949 break;
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]3950 case NVPTXISD::Suld1DI64Trap:
3951 Opc = NVPTX::SULD_1D_I64_TRAP;
3952 Ops.push_back(TexHandle);
3953 Ops.push_back(N->getOperand(2));
3954 Ops.push_back(Chain);
3955 break;
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]3956 case NVPTXISD::Suld1DV2I8Trap:
3957 Opc = NVPTX::SULD_1D_V2I8_TRAP;
3958 Ops.push_back(TexHandle);
3959 Ops.push_back(N->getOperand(2));
3960 Ops.push_back(Chain);
3961 break;
3962 case NVPTXISD::Suld1DV2I16Trap:
3963 Opc = NVPTX::SULD_1D_V2I16_TRAP;
3964 Ops.push_back(TexHandle);
3965 Ops.push_back(N->getOperand(2));
3966 Ops.push_back(Chain);
3967 break;
3968 case NVPTXISD::Suld1DV2I32Trap:
3969 Opc = NVPTX::SULD_1D_V2I32_TRAP;
3970 Ops.push_back(TexHandle);
3971 Ops.push_back(N->getOperand(2));
3972 Ops.push_back(Chain);
3973 break;
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]3974 case NVPTXISD::Suld1DV2I64Trap:
3975 Opc = NVPTX::SULD_1D_V2I64_TRAP;
3976 Ops.push_back(TexHandle);
3977 Ops.push_back(N->getOperand(2));
3978 Ops.push_back(Chain);
3979 break;
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]3980 case NVPTXISD::Suld1DV4I8Trap:
3981 Opc = NVPTX::SULD_1D_V4I8_TRAP;
3982 Ops.push_back(TexHandle);
3983 Ops.push_back(N->getOperand(2));
3984 Ops.push_back(Chain);
3985 break;
3986 case NVPTXISD::Suld1DV4I16Trap:
3987 Opc = NVPTX::SULD_1D_V4I16_TRAP;
3988 Ops.push_back(TexHandle);
3989 Ops.push_back(N->getOperand(2));
3990 Ops.push_back(Chain);
3991 break;
3992 case NVPTXISD::Suld1DV4I32Trap:
3993 Opc = NVPTX::SULD_1D_V4I32_TRAP;
3994 Ops.push_back(TexHandle);
3995 Ops.push_back(N->getOperand(2));
3996 Ops.push_back(Chain);
3997 break;
3998 case NVPTXISD::Suld1DArrayI8Trap:
3999 Opc = NVPTX::SULD_1D_ARRAY_I8_TRAP;
4000 Ops.push_back(TexHandle);
4001 Ops.push_back(N->getOperand(2));
4002 Ops.push_back(N->getOperand(3));
4003 Ops.push_back(Chain);
4004 break;
4005 case NVPTXISD::Suld1DArrayI16Trap:
4006 Opc = NVPTX::SULD_1D_ARRAY_I16_TRAP;
4007 Ops.push_back(TexHandle);
4008 Ops.push_back(N->getOperand(2));
4009 Ops.push_back(N->getOperand(3));
4010 Ops.push_back(Chain);
4011 break;
4012 case NVPTXISD::Suld1DArrayI32Trap:
4013 Opc = NVPTX::SULD_1D_ARRAY_I32_TRAP;
4014 Ops.push_back(TexHandle);
4015 Ops.push_back(N->getOperand(2));
4016 Ops.push_back(N->getOperand(3));
4017 Ops.push_back(Chain);
4018 break;
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]4019 case NVPTXISD::Suld1DArrayI64Trap:
4020 Opc = NVPTX::SULD_1D_ARRAY_I64_TRAP;
4021 Ops.push_back(TexHandle);
4022 Ops.push_back(N->getOperand(2));
4023 Ops.push_back(N->getOperand(3));
4024 Ops.push_back(Chain);
4025 break;
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]4026 case NVPTXISD::Suld1DArrayV2I8Trap:
4027 Opc = NVPTX::SULD_1D_ARRAY_V2I8_TRAP;
4028 Ops.push_back(TexHandle);
4029 Ops.push_back(N->getOperand(2));
4030 Ops.push_back(N->getOperand(3));
4031 Ops.push_back(Chain);
4032 break;
4033 case NVPTXISD::Suld1DArrayV2I16Trap:
4034 Opc = NVPTX::SULD_1D_ARRAY_V2I16_TRAP;
4035 Ops.push_back(TexHandle);
4036 Ops.push_back(N->getOperand(2));
4037 Ops.push_back(N->getOperand(3));
4038 Ops.push_back(Chain);
4039 break;
4040 case NVPTXISD::Suld1DArrayV2I32Trap:
4041 Opc = NVPTX::SULD_1D_ARRAY_V2I32_TRAP;
4042 Ops.push_back(TexHandle);
4043 Ops.push_back(N->getOperand(2));
4044 Ops.push_back(N->getOperand(3));
4045 Ops.push_back(Chain);
4046 break;
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]4047 case NVPTXISD::Suld1DArrayV2I64Trap:
4048 Opc = NVPTX::SULD_1D_ARRAY_V2I64_TRAP;
4049 Ops.push_back(TexHandle);
4050 Ops.push_back(N->getOperand(2));
4051 Ops.push_back(N->getOperand(3));
4052 Ops.push_back(Chain);
4053 break;
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]4054 case NVPTXISD::Suld1DArrayV4I8Trap:
4055 Opc = NVPTX::SULD_1D_ARRAY_V4I8_TRAP;
4056 Ops.push_back(TexHandle);
4057 Ops.push_back(N->getOperand(2));
4058 Ops.push_back(N->getOperand(3));
4059 Ops.push_back(Chain);
4060 break;
4061 case NVPTXISD::Suld1DArrayV4I16Trap:
4062 Opc = NVPTX::SULD_1D_ARRAY_V4I16_TRAP;
4063 Ops.push_back(TexHandle);
4064 Ops.push_back(N->getOperand(2));
4065 Ops.push_back(N->getOperand(3));
4066 Ops.push_back(Chain);
4067 break;
4068 case NVPTXISD::Suld1DArrayV4I32Trap:
4069 Opc = NVPTX::SULD_1D_ARRAY_V4I32_TRAP;
4070 Ops.push_back(TexHandle);
4071 Ops.push_back(N->getOperand(2));
4072 Ops.push_back(N->getOperand(3));
4073 Ops.push_back(Chain);
4074 break;
4075 case NVPTXISD::Suld2DI8Trap:
4076 Opc = NVPTX::SULD_2D_I8_TRAP;
4077 Ops.push_back(TexHandle);
4078 Ops.push_back(N->getOperand(2));
4079 Ops.push_back(N->getOperand(3));
4080 Ops.push_back(Chain);
4081 break;
4082 case NVPTXISD::Suld2DI16Trap:
4083 Opc = NVPTX::SULD_2D_I16_TRAP;
4084 Ops.push_back(TexHandle);
4085 Ops.push_back(N->getOperand(2));
4086 Ops.push_back(N->getOperand(3));
4087 Ops.push_back(Chain);
4088 break;
4089 case NVPTXISD::Suld2DI32Trap:
4090 Opc = NVPTX::SULD_2D_I32_TRAP;
4091 Ops.push_back(TexHandle);
4092 Ops.push_back(N->getOperand(2));
4093 Ops.push_back(N->getOperand(3));
4094 Ops.push_back(Chain);
4095 break;
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]4096 case NVPTXISD::Suld2DI64Trap:
4097 Opc = NVPTX::SULD_2D_I64_TRAP;
4098 Ops.push_back(TexHandle);
4099 Ops.push_back(N->getOperand(2));
4100 Ops.push_back(N->getOperand(3));
4101 Ops.push_back(Chain);
4102 break;
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]4103 case NVPTXISD::Suld2DV2I8Trap:
4104 Opc = NVPTX::SULD_2D_V2I8_TRAP;
4105 Ops.push_back(TexHandle);
4106 Ops.push_back(N->getOperand(2));
4107 Ops.push_back(N->getOperand(3));
4108 Ops.push_back(Chain);
4109 break;
4110 case NVPTXISD::Suld2DV2I16Trap:
4111 Opc = NVPTX::SULD_2D_V2I16_TRAP;
4112 Ops.push_back(TexHandle);
4113 Ops.push_back(N->getOperand(2));
4114 Ops.push_back(N->getOperand(3));
4115 Ops.push_back(Chain);
4116 break;
4117 case NVPTXISD::Suld2DV2I32Trap:
4118 Opc = NVPTX::SULD_2D_V2I32_TRAP;
4119 Ops.push_back(TexHandle);
4120 Ops.push_back(N->getOperand(2));
4121 Ops.push_back(N->getOperand(3));
4122 Ops.push_back(Chain);
4123 break;
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]4124 case NVPTXISD::Suld2DV2I64Trap:
4125 Opc = NVPTX::SULD_2D_V2I64_TRAP;
4126 Ops.push_back(TexHandle);
4127 Ops.push_back(N->getOperand(2));
4128 Ops.push_back(N->getOperand(3));
4129 Ops.push_back(Chain);
4130 break;
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]4131 case NVPTXISD::Suld2DV4I8Trap:
4132 Opc = NVPTX::SULD_2D_V4I8_TRAP;
4133 Ops.push_back(TexHandle);
4134 Ops.push_back(N->getOperand(2));
4135 Ops.push_back(N->getOperand(3));
4136 Ops.push_back(Chain);
4137 break;
4138 case NVPTXISD::Suld2DV4I16Trap:
4139 Opc = NVPTX::SULD_2D_V4I16_TRAP;
4140 Ops.push_back(TexHandle);
4141 Ops.push_back(N->getOperand(2));
4142 Ops.push_back(N->getOperand(3));
4143 Ops.push_back(Chain);
4144 break;
4145 case NVPTXISD::Suld2DV4I32Trap:
4146 Opc = NVPTX::SULD_2D_V4I32_TRAP;
4147 Ops.push_back(TexHandle);
4148 Ops.push_back(N->getOperand(2));
4149 Ops.push_back(N->getOperand(3));
4150 Ops.push_back(Chain);
4151 break;
4152 case NVPTXISD::Suld2DArrayI8Trap:
4153 Opc = NVPTX::SULD_2D_ARRAY_I8_TRAP;
4154 Ops.push_back(TexHandle);
4155 Ops.push_back(N->getOperand(2));
4156 Ops.push_back(N->getOperand(3));
4157 Ops.push_back(N->getOperand(4));
4158 Ops.push_back(Chain);
4159 break;
4160 case NVPTXISD::Suld2DArrayI16Trap:
4161 Opc = NVPTX::SULD_2D_ARRAY_I16_TRAP;
4162 Ops.push_back(TexHandle);
4163 Ops.push_back(N->getOperand(2));
4164 Ops.push_back(N->getOperand(3));
4165 Ops.push_back(N->getOperand(4));
4166 Ops.push_back(Chain);
4167 break;
4168 case NVPTXISD::Suld2DArrayI32Trap:
4169 Opc = NVPTX::SULD_2D_ARRAY_I32_TRAP;
4170 Ops.push_back(TexHandle);
4171 Ops.push_back(N->getOperand(2));
4172 Ops.push_back(N->getOperand(3));
4173 Ops.push_back(N->getOperand(4));
4174 Ops.push_back(Chain);
4175 break;
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]4176 case NVPTXISD::Suld2DArrayI64Trap:
4177 Opc = NVPTX::SULD_2D_ARRAY_I64_TRAP;
4178 Ops.push_back(TexHandle);
4179 Ops.push_back(N->getOperand(2));
4180 Ops.push_back(N->getOperand(3));
4181 Ops.push_back(N->getOperand(4));
4182 Ops.push_back(Chain);
4183 break;
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]4184 case NVPTXISD::Suld2DArrayV2I8Trap:
4185 Opc = NVPTX::SULD_2D_ARRAY_V2I8_TRAP;
4186 Ops.push_back(TexHandle);
4187 Ops.push_back(N->getOperand(2));
4188 Ops.push_back(N->getOperand(3));
4189 Ops.push_back(N->getOperand(4));
4190 Ops.push_back(Chain);
4191 break;
4192 case NVPTXISD::Suld2DArrayV2I16Trap:
4193 Opc = NVPTX::SULD_2D_ARRAY_V2I16_TRAP;
4194 Ops.push_back(TexHandle);
4195 Ops.push_back(N->getOperand(2));
4196 Ops.push_back(N->getOperand(3));
4197 Ops.push_back(N->getOperand(4));
4198 Ops.push_back(Chain);
4199 break;
4200 case NVPTXISD::Suld2DArrayV2I32Trap:
4201 Opc = NVPTX::SULD_2D_ARRAY_V2I32_TRAP;
4202 Ops.push_back(TexHandle);
4203 Ops.push_back(N->getOperand(2));
4204 Ops.push_back(N->getOperand(3));
4205 Ops.push_back(N->getOperand(4));
4206 Ops.push_back(Chain);
4207 break;
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]4208 case NVPTXISD::Suld2DArrayV2I64Trap:
4209 Opc = NVPTX::SULD_2D_ARRAY_V2I64_TRAP;
4210 Ops.push_back(TexHandle);
4211 Ops.push_back(N->getOperand(2));
4212 Ops.push_back(N->getOperand(3));
4213 Ops.push_back(N->getOperand(4));
4214 Ops.push_back(Chain);
4215 break;
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]4216 case NVPTXISD::Suld2DArrayV4I8Trap:
4217 Opc = NVPTX::SULD_2D_ARRAY_V4I8_TRAP;
4218 Ops.push_back(TexHandle);
4219 Ops.push_back(N->getOperand(2));
4220 Ops.push_back(N->getOperand(3));
4221 Ops.push_back(N->getOperand(4));
4222 Ops.push_back(Chain);
4223 break;
4224 case NVPTXISD::Suld2DArrayV4I16Trap:
4225 Opc = NVPTX::SULD_2D_ARRAY_V4I16_TRAP;
4226 Ops.push_back(TexHandle);
4227 Ops.push_back(N->getOperand(2));
4228 Ops.push_back(N->getOperand(3));
4229 Ops.push_back(N->getOperand(4));
4230 Ops.push_back(Chain);
4231 break;
4232 case NVPTXISD::Suld2DArrayV4I32Trap:
4233 Opc = NVPTX::SULD_2D_ARRAY_V4I32_TRAP;
4234 Ops.push_back(TexHandle);
4235 Ops.push_back(N->getOperand(2));
4236 Ops.push_back(N->getOperand(3));
4237 Ops.push_back(N->getOperand(4));
4238 Ops.push_back(Chain);
4239 break;
4240 case NVPTXISD::Suld3DI8Trap:
4241 Opc = NVPTX::SULD_3D_I8_TRAP;
4242 Ops.push_back(TexHandle);
4243 Ops.push_back(N->getOperand(2));
4244 Ops.push_back(N->getOperand(3));
4245 Ops.push_back(N->getOperand(4));
4246 Ops.push_back(Chain);
4247 break;
4248 case NVPTXISD::Suld3DI16Trap:
4249 Opc = NVPTX::SULD_3D_I16_TRAP;
4250 Ops.push_back(TexHandle);
4251 Ops.push_back(N->getOperand(2));
4252 Ops.push_back(N->getOperand(3));
4253 Ops.push_back(N->getOperand(4));
4254 Ops.push_back(Chain);
4255 break;
4256 case NVPTXISD::Suld3DI32Trap:
4257 Opc = NVPTX::SULD_3D_I32_TRAP;
4258 Ops.push_back(TexHandle);
4259 Ops.push_back(N->getOperand(2));
4260 Ops.push_back(N->getOperand(3));
4261 Ops.push_back(N->getOperand(4));
4262 Ops.push_back(Chain);
4263 break;
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]4264 case NVPTXISD::Suld3DI64Trap:
4265 Opc = NVPTX::SULD_3D_I64_TRAP;
4266 Ops.push_back(TexHandle);
4267 Ops.push_back(N->getOperand(2));
4268 Ops.push_back(N->getOperand(3));
4269 Ops.push_back(N->getOperand(4));
4270 Ops.push_back(Chain);
4271 break;
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]4272 case NVPTXISD::Suld3DV2I8Trap:
4273 Opc = NVPTX::SULD_3D_V2I8_TRAP;
4274 Ops.push_back(TexHandle);
4275 Ops.push_back(N->getOperand(2));
4276 Ops.push_back(N->getOperand(3));
4277 Ops.push_back(N->getOperand(4));
4278 Ops.push_back(Chain);
4279 break;
4280 case NVPTXISD::Suld3DV2I16Trap:
4281 Opc = NVPTX::SULD_3D_V2I16_TRAP;
4282 Ops.push_back(TexHandle);
4283 Ops.push_back(N->getOperand(2));
4284 Ops.push_back(N->getOperand(3));
4285 Ops.push_back(N->getOperand(4));
4286 Ops.push_back(Chain);
4287 break;
4288 case NVPTXISD::Suld3DV2I32Trap:
4289 Opc = NVPTX::SULD_3D_V2I32_TRAP;
4290 Ops.push_back(TexHandle);
4291 Ops.push_back(N->getOperand(2));
4292 Ops.push_back(N->getOperand(3));
4293 Ops.push_back(N->getOperand(4));
4294 Ops.push_back(Chain);
4295 break;
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]4296 case NVPTXISD::Suld3DV2I64Trap:
4297 Opc = NVPTX::SULD_3D_V2I64_TRAP;
4298 Ops.push_back(TexHandle);
4299 Ops.push_back(N->getOperand(2));
4300 Ops.push_back(N->getOperand(3));
4301 Ops.push_back(N->getOperand(4));
4302 Ops.push_back(Chain);
4303 break;
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]4304 case NVPTXISD::Suld3DV4I8Trap:
4305 Opc = NVPTX::SULD_3D_V4I8_TRAP;
4306 Ops.push_back(TexHandle);
4307 Ops.push_back(N->getOperand(2));
4308 Ops.push_back(N->getOperand(3));
4309 Ops.push_back(N->getOperand(4));
4310 Ops.push_back(Chain);
4311 break;
4312 case NVPTXISD::Suld3DV4I16Trap:
4313 Opc = NVPTX::SULD_3D_V4I16_TRAP;
4314 Ops.push_back(TexHandle);
4315 Ops.push_back(N->getOperand(2));
4316 Ops.push_back(N->getOperand(3));
4317 Ops.push_back(N->getOperand(4));
4318 Ops.push_back(Chain);
4319 break;
4320 case NVPTXISD::Suld3DV4I32Trap:
4321 Opc = NVPTX::SULD_3D_V4I32_TRAP;
4322 Ops.push_back(TexHandle);
4323 Ops.push_back(N->getOperand(2));
4324 Ops.push_back(N->getOperand(3));
4325 Ops.push_back(N->getOperand(4));
4326 Ops.push_back(Chain);
4327 break;
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]4328 case NVPTXISD::Suld1DI8Zero:
4329 Opc = NVPTX::SULD_1D_I8_ZERO;
4330 Ops.push_back(TexHandle);
4331 Ops.push_back(N->getOperand(2));
4332 Ops.push_back(Chain);
4333 break;
4334 case NVPTXISD::Suld1DI16Zero:
4335 Opc = NVPTX::SULD_1D_I16_ZERO;
4336 Ops.push_back(TexHandle);
4337 Ops.push_back(N->getOperand(2));
4338 Ops.push_back(Chain);
4339 break;
4340 case NVPTXISD::Suld1DI32Zero:
4341 Opc = NVPTX::SULD_1D_I32_ZERO;
4342 Ops.push_back(TexHandle);
4343 Ops.push_back(N->getOperand(2));
4344 Ops.push_back(Chain);
4345 break;
4346 case NVPTXISD::Suld1DI64Zero:
4347 Opc = NVPTX::SULD_1D_I64_ZERO;
4348 Ops.push_back(TexHandle);
4349 Ops.push_back(N->getOperand(2));
4350 Ops.push_back(Chain);
4351 break;
4352 case NVPTXISD::Suld1DV2I8Zero:
4353 Opc = NVPTX::SULD_1D_V2I8_ZERO;
4354 Ops.push_back(TexHandle);
4355 Ops.push_back(N->getOperand(2));
4356 Ops.push_back(Chain);
4357 break;
4358 case NVPTXISD::Suld1DV2I16Zero:
4359 Opc = NVPTX::SULD_1D_V2I16_ZERO;
4360 Ops.push_back(TexHandle);
4361 Ops.push_back(N->getOperand(2));
4362 Ops.push_back(Chain);
4363 break;
4364 case NVPTXISD::Suld1DV2I32Zero:
4365 Opc = NVPTX::SULD_1D_V2I32_ZERO;
4366 Ops.push_back(TexHandle);
4367 Ops.push_back(N->getOperand(2));
4368 Ops.push_back(Chain);
4369 break;
4370 case NVPTXISD::Suld1DV2I64Zero:
4371 Opc = NVPTX::SULD_1D_V2I64_ZERO;
4372 Ops.push_back(TexHandle);
4373 Ops.push_back(N->getOperand(2));
4374 Ops.push_back(Chain);
4375 break;
4376 case NVPTXISD::Suld1DV4I8Zero:
4377 Opc = NVPTX::SULD_1D_V4I8_ZERO;
4378 Ops.push_back(TexHandle);
4379 Ops.push_back(N->getOperand(2));
4380 Ops.push_back(Chain);
4381 break;
4382 case NVPTXISD::Suld1DV4I16Zero:
4383 Opc = NVPTX::SULD_1D_V4I16_ZERO;
4384 Ops.push_back(TexHandle);
4385 Ops.push_back(N->getOperand(2));
4386 Ops.push_back(Chain);
4387 break;
4388 case NVPTXISD::Suld1DV4I32Zero:
4389 Opc = NVPTX::SULD_1D_V4I32_ZERO;
4390 Ops.push_back(TexHandle);
4391 Ops.push_back(N->getOperand(2));
4392 Ops.push_back(Chain);
4393 break;
4394 case NVPTXISD::Suld1DArrayI8Zero:
4395 Opc = NVPTX::SULD_1D_ARRAY_I8_ZERO;
4396 Ops.push_back(TexHandle);
4397 Ops.push_back(N->getOperand(2));
4398 Ops.push_back(N->getOperand(3));
4399 Ops.push_back(Chain);
4400 break;
4401 case NVPTXISD::Suld1DArrayI16Zero:
4402 Opc = NVPTX::SULD_1D_ARRAY_I16_ZERO;
4403 Ops.push_back(TexHandle);
4404 Ops.push_back(N->getOperand(2));
4405 Ops.push_back(N->getOperand(3));
4406 Ops.push_back(Chain);
4407 break;
4408 case NVPTXISD::Suld1DArrayI32Zero:
4409 Opc = NVPTX::SULD_1D_ARRAY_I32_ZERO;
4410 Ops.push_back(TexHandle);
4411 Ops.push_back(N->getOperand(2));
4412 Ops.push_back(N->getOperand(3));
4413 Ops.push_back(Chain);
4414 break;
4415 case NVPTXISD::Suld1DArrayI64Zero:
4416 Opc = NVPTX::SULD_1D_ARRAY_I64_ZERO;
4417 Ops.push_back(TexHandle);
4418 Ops.push_back(N->getOperand(2));
4419 Ops.push_back(N->getOperand(3));
4420 Ops.push_back(Chain);
4421 break;
4422 case NVPTXISD::Suld1DArrayV2I8Zero:
4423 Opc = NVPTX::SULD_1D_ARRAY_V2I8_ZERO;
4424 Ops.push_back(TexHandle);
4425 Ops.push_back(N->getOperand(2));
4426 Ops.push_back(N->getOperand(3));
4427 Ops.push_back(Chain);
4428 break;
4429 case NVPTXISD::Suld1DArrayV2I16Zero:
4430 Opc = NVPTX::SULD_1D_ARRAY_V2I16_ZERO;
4431 Ops.push_back(TexHandle);
4432 Ops.push_back(N->getOperand(2));
4433 Ops.push_back(N->getOperand(3));
4434 Ops.push_back(Chain);
4435 break;
4436 case NVPTXISD::Suld1DArrayV2I32Zero:
4437 Opc = NVPTX::SULD_1D_ARRAY_V2I32_ZERO;
4438 Ops.push_back(TexHandle);
4439 Ops.push_back(N->getOperand(2));
4440 Ops.push_back(N->getOperand(3));
4441 Ops.push_back(Chain);
4442 break;
4443 case NVPTXISD::Suld1DArrayV2I64Zero:
4444 Opc = NVPTX::SULD_1D_ARRAY_V2I64_ZERO;
4445 Ops.push_back(TexHandle);
4446 Ops.push_back(N->getOperand(2));
4447 Ops.push_back(N->getOperand(3));
4448 Ops.push_back(Chain);
4449 break;
4450 case NVPTXISD::Suld1DArrayV4I8Zero:
4451 Opc = NVPTX::SULD_1D_ARRAY_V4I8_ZERO;
4452 Ops.push_back(TexHandle);
4453 Ops.push_back(N->getOperand(2));
4454 Ops.push_back(N->getOperand(3));
4455 Ops.push_back(Chain);
4456 break;
4457 case NVPTXISD::Suld1DArrayV4I16Zero:
4458 Opc = NVPTX::SULD_1D_ARRAY_V4I16_ZERO;
4459 Ops.push_back(TexHandle);
4460 Ops.push_back(N->getOperand(2));
4461 Ops.push_back(N->getOperand(3));
4462 Ops.push_back(Chain);
4463 break;
4464 case NVPTXISD::Suld1DArrayV4I32Zero:
4465 Opc = NVPTX::SULD_1D_ARRAY_V4I32_ZERO;
4466 Ops.push_back(TexHandle);
4467 Ops.push_back(N->getOperand(2));
4468 Ops.push_back(N->getOperand(3));
4469 Ops.push_back(Chain);
4470 break;
4471 case NVPTXISD::Suld2DI8Zero:
4472 Opc = NVPTX::SULD_2D_I8_ZERO;
4473 Ops.push_back(TexHandle);
4474 Ops.push_back(N->getOperand(2));
4475 Ops.push_back(N->getOperand(3));
4476 Ops.push_back(Chain);
4477 break;
4478 case NVPTXISD::Suld2DI16Zero:
4479 Opc = NVPTX::SULD_2D_I16_ZERO;
4480 Ops.push_back(TexHandle);
4481 Ops.push_back(N->getOperand(2));
4482 Ops.push_back(N->getOperand(3));
4483 Ops.push_back(Chain);
4484 break;
4485 case NVPTXISD::Suld2DI32Zero:
4486 Opc = NVPTX::SULD_2D_I32_ZERO;
4487 Ops.push_back(TexHandle);
4488 Ops.push_back(N->getOperand(2));
4489 Ops.push_back(N->getOperand(3));
4490 Ops.push_back(Chain);
4491 break;
4492 case NVPTXISD::Suld2DI64Zero:
4493 Opc = NVPTX::SULD_2D_I64_ZERO;
4494 Ops.push_back(TexHandle);
4495 Ops.push_back(N->getOperand(2));
4496 Ops.push_back(N->getOperand(3));
4497 Ops.push_back(Chain);
4498 break;
4499 case NVPTXISD::Suld2DV2I8Zero:
4500 Opc = NVPTX::SULD_2D_V2I8_ZERO;
4501 Ops.push_back(TexHandle);
4502 Ops.push_back(N->getOperand(2));
4503 Ops.push_back(N->getOperand(3));
4504 Ops.push_back(Chain);
4505 break;
4506 case NVPTXISD::Suld2DV2I16Zero:
4507 Opc = NVPTX::SULD_2D_V2I16_ZERO;
4508 Ops.push_back(TexHandle);
4509 Ops.push_back(N->getOperand(2));
4510 Ops.push_back(N->getOperand(3));
4511 Ops.push_back(Chain);
4512 break;
4513 case NVPTXISD::Suld2DV2I32Zero:
4514 Opc = NVPTX::SULD_2D_V2I32_ZERO;
4515 Ops.push_back(TexHandle);
4516 Ops.push_back(N->getOperand(2));
4517 Ops.push_back(N->getOperand(3));
4518 Ops.push_back(Chain);
4519 break;
4520 case NVPTXISD::Suld2DV2I64Zero:
4521 Opc = NVPTX::SULD_2D_V2I64_ZERO;
4522 Ops.push_back(TexHandle);
4523 Ops.push_back(N->getOperand(2));
4524 Ops.push_back(N->getOperand(3));
4525 Ops.push_back(Chain);
4526 break;
4527 case NVPTXISD::Suld2DV4I8Zero:
4528 Opc = NVPTX::SULD_2D_V4I8_ZERO;
4529 Ops.push_back(TexHandle);
4530 Ops.push_back(N->getOperand(2));
4531 Ops.push_back(N->getOperand(3));
4532 Ops.push_back(Chain);
4533 break;
4534 case NVPTXISD::Suld2DV4I16Zero:
4535 Opc = NVPTX::SULD_2D_V4I16_ZERO;
4536 Ops.push_back(TexHandle);
4537 Ops.push_back(N->getOperand(2));
4538 Ops.push_back(N->getOperand(3));
4539 Ops.push_back(Chain);
4540 break;
4541 case NVPTXISD::Suld2DV4I32Zero:
4542 Opc = NVPTX::SULD_2D_V4I32_ZERO;
4543 Ops.push_back(TexHandle);
4544 Ops.push_back(N->getOperand(2));
4545 Ops.push_back(N->getOperand(3));
4546 Ops.push_back(Chain);
4547 break;
4548 case NVPTXISD::Suld2DArrayI8Zero:
4549 Opc = NVPTX::SULD_2D_ARRAY_I8_ZERO;
4550 Ops.push_back(TexHandle);
4551 Ops.push_back(N->getOperand(2));
4552 Ops.push_back(N->getOperand(3));
4553 Ops.push_back(N->getOperand(4));
4554 Ops.push_back(Chain);
4555 break;
4556 case NVPTXISD::Suld2DArrayI16Zero:
4557 Opc = NVPTX::SULD_2D_ARRAY_I16_ZERO;
4558 Ops.push_back(TexHandle);
4559 Ops.push_back(N->getOperand(2));
4560 Ops.push_back(N->getOperand(3));
4561 Ops.push_back(N->getOperand(4));
4562 Ops.push_back(Chain);
4563 break;
4564 case NVPTXISD::Suld2DArrayI32Zero:
4565 Opc = NVPTX::SULD_2D_ARRAY_I32_ZERO;
4566 Ops.push_back(TexHandle);
4567 Ops.push_back(N->getOperand(2));
4568 Ops.push_back(N->getOperand(3));
4569 Ops.push_back(N->getOperand(4));
4570 Ops.push_back(Chain);
4571 break;
4572 case NVPTXISD::Suld2DArrayI64Zero:
4573 Opc = NVPTX::SULD_2D_ARRAY_I64_ZERO;
4574 Ops.push_back(TexHandle);
4575 Ops.push_back(N->getOperand(2));
4576 Ops.push_back(N->getOperand(3));
4577 Ops.push_back(N->getOperand(4));
4578 Ops.push_back(Chain);
4579 break;
4580 case NVPTXISD::Suld2DArrayV2I8Zero:
4581 Opc = NVPTX::SULD_2D_ARRAY_V2I8_ZERO;
4582 Ops.push_back(TexHandle);
4583 Ops.push_back(N->getOperand(2));
4584 Ops.push_back(N->getOperand(3));
4585 Ops.push_back(N->getOperand(4));
4586 Ops.push_back(Chain);
4587 break;
4588 case NVPTXISD::Suld2DArrayV2I16Zero:
4589 Opc = NVPTX::SULD_2D_ARRAY_V2I16_ZERO;
4590 Ops.push_back(TexHandle);
4591 Ops.push_back(N->getOperand(2));
4592 Ops.push_back(N->getOperand(3));
4593 Ops.push_back(N->getOperand(4));
4594 Ops.push_back(Chain);
4595 break;
4596 case NVPTXISD::Suld2DArrayV2I32Zero:
4597 Opc = NVPTX::SULD_2D_ARRAY_V2I32_ZERO;
4598 Ops.push_back(TexHandle);
4599 Ops.push_back(N->getOperand(2));
4600 Ops.push_back(N->getOperand(3));
4601 Ops.push_back(N->getOperand(4));
4602 Ops.push_back(Chain);
4603 break;
4604 case NVPTXISD::Suld2DArrayV2I64Zero:
4605 Opc = NVPTX::SULD_2D_ARRAY_V2I64_ZERO;
4606 Ops.push_back(TexHandle);
4607 Ops.push_back(N->getOperand(2));
4608 Ops.push_back(N->getOperand(3));
4609 Ops.push_back(N->getOperand(4));
4610 Ops.push_back(Chain);
4611 break;
4612 case NVPTXISD::Suld2DArrayV4I8Zero:
4613 Opc = NVPTX::SULD_2D_ARRAY_V4I8_ZERO;
4614 Ops.push_back(TexHandle);
4615 Ops.push_back(N->getOperand(2));
4616 Ops.push_back(N->getOperand(3));
4617 Ops.push_back(N->getOperand(4));
4618 Ops.push_back(Chain);
4619 break;
4620 case NVPTXISD::Suld2DArrayV4I16Zero:
4621 Opc = NVPTX::SULD_2D_ARRAY_V4I16_ZERO;
4622 Ops.push_back(TexHandle);
4623 Ops.push_back(N->getOperand(2));
4624 Ops.push_back(N->getOperand(3));
4625 Ops.push_back(N->getOperand(4));
4626 Ops.push_back(Chain);
4627 break;
4628 case NVPTXISD::Suld2DArrayV4I32Zero:
4629 Opc = NVPTX::SULD_2D_ARRAY_V4I32_ZERO;
4630 Ops.push_back(TexHandle);
4631 Ops.push_back(N->getOperand(2));
4632 Ops.push_back(N->getOperand(3));
4633 Ops.push_back(N->getOperand(4));
4634 Ops.push_back(Chain);
4635 break;
4636 case NVPTXISD::Suld3DI8Zero:
4637 Opc = NVPTX::SULD_3D_I8_ZERO;
4638 Ops.push_back(TexHandle);
4639 Ops.push_back(N->getOperand(2));
4640 Ops.push_back(N->getOperand(3));
4641 Ops.push_back(N->getOperand(4));
4642 Ops.push_back(Chain);
4643 break;
4644 case NVPTXISD::Suld3DI16Zero:
4645 Opc = NVPTX::SULD_3D_I16_ZERO;
4646 Ops.push_back(TexHandle);
4647 Ops.push_back(N->getOperand(2));
4648 Ops.push_back(N->getOperand(3));
4649 Ops.push_back(N->getOperand(4));
4650 Ops.push_back(Chain);
4651 break;
4652 case NVPTXISD::Suld3DI32Zero:
4653 Opc = NVPTX::SULD_3D_I32_ZERO;
4654 Ops.push_back(TexHandle);
4655 Ops.push_back(N->getOperand(2));
4656 Ops.push_back(N->getOperand(3));
4657 Ops.push_back(N->getOperand(4));
4658 Ops.push_back(Chain);
4659 break;
4660 case NVPTXISD::Suld3DI64Zero:
4661 Opc = NVPTX::SULD_3D_I64_ZERO;
4662 Ops.push_back(TexHandle);
4663 Ops.push_back(N->getOperand(2));
4664 Ops.push_back(N->getOperand(3));
4665 Ops.push_back(N->getOperand(4));
4666 Ops.push_back(Chain);
4667 break;
4668 case NVPTXISD::Suld3DV2I8Zero:
4669 Opc = NVPTX::SULD_3D_V2I8_ZERO;
4670 Ops.push_back(TexHandle);
4671 Ops.push_back(N->getOperand(2));
4672 Ops.push_back(N->getOperand(3));
4673 Ops.push_back(N->getOperand(4));
4674 Ops.push_back(Chain);
4675 break;
4676 case NVPTXISD::Suld3DV2I16Zero:
4677 Opc = NVPTX::SULD_3D_V2I16_ZERO;
4678 Ops.push_back(TexHandle);
4679 Ops.push_back(N->getOperand(2));
4680 Ops.push_back(N->getOperand(3));
4681 Ops.push_back(N->getOperand(4));
4682 Ops.push_back(Chain);
4683 break;
4684 case NVPTXISD::Suld3DV2I32Zero:
4685 Opc = NVPTX::SULD_3D_V2I32_ZERO;
4686 Ops.push_back(TexHandle);
4687 Ops.push_back(N->getOperand(2));
4688 Ops.push_back(N->getOperand(3));
4689 Ops.push_back(N->getOperand(4));
4690 Ops.push_back(Chain);
4691 break;
4692 case NVPTXISD::Suld3DV2I64Zero:
4693 Opc = NVPTX::SULD_3D_V2I64_ZERO;
4694 Ops.push_back(TexHandle);
4695 Ops.push_back(N->getOperand(2));
4696 Ops.push_back(N->getOperand(3));
4697 Ops.push_back(N->getOperand(4));
4698 Ops.push_back(Chain);
4699 break;
4700 case NVPTXISD::Suld3DV4I8Zero:
4701 Opc = NVPTX::SULD_3D_V4I8_ZERO;
4702 Ops.push_back(TexHandle);
4703 Ops.push_back(N->getOperand(2));
4704 Ops.push_back(N->getOperand(3));
4705 Ops.push_back(N->getOperand(4));
4706 Ops.push_back(Chain);
4707 break;
4708 case NVPTXISD::Suld3DV4I16Zero:
4709 Opc = NVPTX::SULD_3D_V4I16_ZERO;
4710 Ops.push_back(TexHandle);
4711 Ops.push_back(N->getOperand(2));
4712 Ops.push_back(N->getOperand(3));
4713 Ops.push_back(N->getOperand(4));
4714 Ops.push_back(Chain);
4715 break;
4716 case NVPTXISD::Suld3DV4I32Zero:
4717 Opc = NVPTX::SULD_3D_V4I32_ZERO;
4718 Ops.push_back(TexHandle);
4719 Ops.push_back(N->getOperand(2));
4720 Ops.push_back(N->getOperand(3));
4721 Ops.push_back(N->getOperand(4));
4722 Ops.push_back(Chain);
4723 break;
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]4724 }
4725 Ret = CurDAG->getMachineNode(Opc, SDLoc(N), N->getVTList(), Ops);
4726 return Ret;
4727}
4728
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]4729
Justin Holewinskica7a4f12014-06-27 18:35:27 +0000[diff] [blame]4730/// SelectBFE - Look for instruction sequences that can be made more efficient
4731/// by using the 'bfe' (bit-field extract) PTX instruction
4732SDNode *NVPTXDAGToDAGISel::SelectBFE(SDNode *N) {
4733 SDValue LHS = N->getOperand(0);
4734 SDValue RHS = N->getOperand(1);
4735 SDValue Len;
4736 SDValue Start;
4737 SDValue Val;
4738 bool IsSigned = false;
4739
4740 if (N->getOpcode() == ISD::AND) {
4741 // Canonicalize the operands
4742 // We want 'and %val, %mask'
4743 if (isa<ConstantSDNode>(LHS) && !isa<ConstantSDNode>(RHS)) {
4744 std::swap(LHS, RHS);
4745 }
4746
4747 ConstantSDNode *Mask = dyn_cast<ConstantSDNode>(RHS);
4748 if (!Mask) {
4749 // We need a constant mask on the RHS of the AND
4750 return NULL;
4751 }
4752
4753 // Extract the mask bits
4754 uint64_t MaskVal = Mask->getZExtValue();
4755 if (!isMask_64(MaskVal)) {
4756 // We *could* handle shifted masks here, but doing so would require an
4757 // 'and' operation to fix up the low-order bits so we would trade
4758 // shr+and for bfe+and, which has the same throughput
4759 return NULL;
4760 }
4761
4762 // How many bits are in our mask?
Benjamin Kramer5f6a9072015-02-12 15:35:40 +0000[diff] [blame]4763 uint64_t NumBits = countTrailingOnes(MaskVal);
Justin Holewinskica7a4f12014-06-27 18:35:27 +0000[diff] [blame]4764 Len = CurDAG->getTargetConstant(NumBits, MVT::i32);
4765
4766 if (LHS.getOpcode() == ISD::SRL || LHS.getOpcode() == ISD::SRA) {
4767 // We have a 'srl/and' pair, extract the effective start bit and length
4768 Val = LHS.getNode()->getOperand(0);
4769 Start = LHS.getNode()->getOperand(1);
4770 ConstantSDNode *StartConst = dyn_cast<ConstantSDNode>(Start);
4771 if (StartConst) {
4772 uint64_t StartVal = StartConst->getZExtValue();
4773 // How many "good" bits do we have left? "good" is defined here as bits
4774 // that exist in the original value, not shifted in.
4775 uint64_t GoodBits = Start.getValueType().getSizeInBits() - StartVal;
4776 if (NumBits > GoodBits) {
4777 // Do not handle the case where bits have been shifted in. In theory
4778 // we could handle this, but the cost is likely higher than just
4779 // emitting the srl/and pair.
4780 return NULL;
4781 }
4782 Start = CurDAG->getTargetConstant(StartVal, MVT::i32);
4783 } else {
4784 // Do not handle the case where the shift amount (can be zero if no srl
4785 // was found) is not constant. We could handle this case, but it would
4786 // require run-time logic that would be more expensive than just
4787 // emitting the srl/and pair.
4788 return NULL;
4789 }
4790 } else {
4791 // Do not handle the case where the LHS of the and is not a shift. While
4792 // it would be trivial to handle this case, it would just transform
4793 // 'and' -> 'bfe', but 'and' has higher-throughput.
4794 return NULL;
4795 }
4796 } else if (N->getOpcode() == ISD::SRL || N->getOpcode() == ISD::SRA) {
4797 if (LHS->getOpcode() == ISD::AND) {
4798 ConstantSDNode *ShiftCnst = dyn_cast<ConstantSDNode>(RHS);
4799 if (!ShiftCnst) {
4800 // Shift amount must be constant
4801 return NULL;
4802 }
4803
4804 uint64_t ShiftAmt = ShiftCnst->getZExtValue();
4805
4806 SDValue AndLHS = LHS->getOperand(0);
4807 SDValue AndRHS = LHS->getOperand(1);
4808
4809 // Canonicalize the AND to have the mask on the RHS
4810 if (isa<ConstantSDNode>(AndLHS)) {
4811 std::swap(AndLHS, AndRHS);
4812 }
4813
4814 ConstantSDNode *MaskCnst = dyn_cast<ConstantSDNode>(AndRHS);
4815 if (!MaskCnst) {
4816 // Mask must be constant
4817 return NULL;
4818 }
4819
4820 uint64_t MaskVal = MaskCnst->getZExtValue();
4821 uint64_t NumZeros;
4822 uint64_t NumBits;
4823 if (isMask_64(MaskVal)) {
4824 NumZeros = 0;
4825 // The number of bits in the result bitfield will be the number of
4826 // trailing ones (the AND) minus the number of bits we shift off
Benjamin Kramer5f6a9072015-02-12 15:35:40 +0000[diff] [blame]4827 NumBits = countTrailingOnes(MaskVal) - ShiftAmt;
Justin Holewinskica7a4f12014-06-27 18:35:27 +0000[diff] [blame]4828 } else if (isShiftedMask_64(MaskVal)) {
4829 NumZeros = countTrailingZeros(MaskVal);
Benjamin Kramer5f6a9072015-02-12 15:35:40 +0000[diff] [blame]4830 unsigned NumOnes = countTrailingOnes(MaskVal >> NumZeros);
Justin Holewinskica7a4f12014-06-27 18:35:27 +0000[diff] [blame]4831 // The number of bits in the result bitfield will be the number of
4832 // trailing zeros plus the number of set bits in the mask minus the
4833 // number of bits we shift off
4834 NumBits = NumZeros + NumOnes - ShiftAmt;
4835 } else {
4836 // This is not a mask we can handle
4837 return NULL;
4838 }
4839
4840 if (ShiftAmt < NumZeros) {
4841 // Handling this case would require extra logic that would make this
4842 // transformation non-profitable
4843 return NULL;
4844 }
4845
4846 Val = AndLHS;
4847 Start = CurDAG->getTargetConstant(ShiftAmt, MVT::i32);
4848 Len = CurDAG->getTargetConstant(NumBits, MVT::i32);
4849 } else if (LHS->getOpcode() == ISD::SHL) {
4850 // Here, we have a pattern like:
4851 //
4852 // (sra (shl val, NN), MM)
4853 // or
4854 // (srl (shl val, NN), MM)
4855 //
4856 // If MM >= NN, we can efficiently optimize this with bfe
4857 Val = LHS->getOperand(0);
4858
4859 SDValue ShlRHS = LHS->getOperand(1);
4860 ConstantSDNode *ShlCnst = dyn_cast<ConstantSDNode>(ShlRHS);
4861 if (!ShlCnst) {
4862 // Shift amount must be constant
4863 return NULL;
4864 }
4865 uint64_t InnerShiftAmt = ShlCnst->getZExtValue();
4866
4867 SDValue ShrRHS = RHS;
4868 ConstantSDNode *ShrCnst = dyn_cast<ConstantSDNode>(ShrRHS);
4869 if (!ShrCnst) {
4870 // Shift amount must be constant
4871 return NULL;
4872 }
4873 uint64_t OuterShiftAmt = ShrCnst->getZExtValue();
4874
4875 // To avoid extra codegen and be profitable, we need Outer >= Inner
4876 if (OuterShiftAmt < InnerShiftAmt) {
4877 return NULL;
4878 }
4879
4880 // If the outer shift is more than the type size, we have no bitfield to
4881 // extract (since we also check that the inner shift is <= the outer shift
4882 // then this also implies that the inner shift is < the type size)
4883 if (OuterShiftAmt >= Val.getValueType().getSizeInBits()) {
4884 return NULL;
4885 }
4886
4887 Start =
4888 CurDAG->getTargetConstant(OuterShiftAmt - InnerShiftAmt, MVT::i32);
4889 Len =
4890 CurDAG->getTargetConstant(Val.getValueType().getSizeInBits() -
4891 OuterShiftAmt, MVT::i32);
4892
4893 if (N->getOpcode() == ISD::SRA) {
4894 // If we have a arithmetic right shift, we need to use the signed bfe
4895 // variant
4896 IsSigned = true;
4897 }
4898 } else {
4899 // No can do...
4900 return NULL;
4901 }
4902 } else {
4903 // No can do...
4904 return NULL;
4905 }
4906
4907
4908 unsigned Opc;
4909 // For the BFE operations we form here from "and" and "srl", always use the
4910 // unsigned variants.
4911 if (Val.getValueType() == MVT::i32) {
4912 if (IsSigned) {
4913 Opc = NVPTX::BFE_S32rii;
4914 } else {
4915 Opc = NVPTX::BFE_U32rii;
4916 }
4917 } else if (Val.getValueType() == MVT::i64) {
4918 if (IsSigned) {
4919 Opc = NVPTX::BFE_S64rii;
4920 } else {
4921 Opc = NVPTX::BFE_U64rii;
4922 }
4923 } else {
4924 // We cannot handle this type
4925 return NULL;
4926 }
4927
4928 SDValue Ops[] = {
4929 Val, Start, Len
4930 };
4931
4932 SDNode *Ret =
4933 CurDAG->getMachineNode(Opc, SDLoc(N), N->getVTList(), Ops);
4934
4935 return Ret;
4936}
4937
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]4938// SelectDirectAddr - Match a direct address for DAG.
4939// A direct address could be a globaladdress or externalsymbol.
4940bool NVPTXDAGToDAGISel::SelectDirectAddr(SDValue N, SDValue &Address) {
4941 // Return true if TGA or ES.
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]4942 if (N.getOpcode() == ISD::TargetGlobalAddress ||
4943 N.getOpcode() == ISD::TargetExternalSymbol) {
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]4944 Address = N;
4945 return true;
4946 }
4947 if (N.getOpcode() == NVPTXISD::Wrapper) {
4948 Address = N.getOperand(0);
4949 return true;
4950 }
4951 if (N.getOpcode() == ISD::INTRINSIC_WO_CHAIN) {
4952 unsigned IID = cast<ConstantSDNode>(N.getOperand(0))->getZExtValue();
4953 if (IID == Intrinsic::nvvm_ptr_gen_to_param)
4954 if (N.getOperand(1).getOpcode() == NVPTXISD::MoveParam)
4955 return (SelectDirectAddr(N.getOperand(1).getOperand(0), Address));
4956 }
4957 return false;
4958}
4959
4960// symbol+offset
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]4961bool NVPTXDAGToDAGISel::SelectADDRsi_imp(
4962 SDNode *OpNode, SDValue Addr, SDValue &Base, SDValue &Offset, MVT mvt) {
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]4963 if (Addr.getOpcode() == ISD::ADD) {
4964 if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(Addr.getOperand(1))) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]4965 SDValue base = Addr.getOperand(0);
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]4966 if (SelectDirectAddr(base, Base)) {
4967 Offset = CurDAG->getTargetConstant(CN->getZExtValue(), mvt);
4968 return true;
4969 }
4970 }
4971 }
4972 return false;
4973}
4974
4975// symbol+offset
4976bool NVPTXDAGToDAGISel::SelectADDRsi(SDNode *OpNode, SDValue Addr,
4977 SDValue &Base, SDValue &Offset) {
4978 return SelectADDRsi_imp(OpNode, Addr, Base, Offset, MVT::i32);
4979}
4980
4981// symbol+offset
4982bool NVPTXDAGToDAGISel::SelectADDRsi64(SDNode *OpNode, SDValue Addr,
4983 SDValue &Base, SDValue &Offset) {
4984 return SelectADDRsi_imp(OpNode, Addr, Base, Offset, MVT::i64);
4985}
4986
4987// register+offset
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]4988bool NVPTXDAGToDAGISel::SelectADDRri_imp(
4989 SDNode *OpNode, SDValue Addr, SDValue &Base, SDValue &Offset, MVT mvt) {
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]4990 if (FrameIndexSDNode *FIN = dyn_cast<FrameIndexSDNode>(Addr)) {
4991 Base = CurDAG->getTargetFrameIndex(FIN->getIndex(), mvt);
4992 Offset = CurDAG->getTargetConstant(0, mvt);
4993 return true;
4994 }
4995 if (Addr.getOpcode() == ISD::TargetExternalSymbol ||
4996 Addr.getOpcode() == ISD::TargetGlobalAddress)
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]4997 return false; // direct calls.
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]4998
4999 if (Addr.getOpcode() == ISD::ADD) {
5000 if (SelectDirectAddr(Addr.getOperand(0), Addr)) {
5001 return false;
5002 }
5003 if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(Addr.getOperand(1))) {
5004 if (FrameIndexSDNode *FIN =
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]5005 dyn_cast<FrameIndexSDNode>(Addr.getOperand(0)))
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]5006 // Constant offset from frame ref.
5007 Base = CurDAG->getTargetFrameIndex(FIN->getIndex(), mvt);
5008 else
5009 Base = Addr.getOperand(0);
5010 Offset = CurDAG->getTargetConstant(CN->getZExtValue(), mvt);
5011 return true;
5012 }
5013 }
5014 return false;
5015}
5016
5017// register+offset
5018bool NVPTXDAGToDAGISel::SelectADDRri(SDNode *OpNode, SDValue Addr,
5019 SDValue &Base, SDValue &Offset) {
5020 return SelectADDRri_imp(OpNode, Addr, Base, Offset, MVT::i32);
5021}
5022
5023// register+offset
5024bool NVPTXDAGToDAGISel::SelectADDRri64(SDNode *OpNode, SDValue Addr,
5025 SDValue &Base, SDValue &Offset) {
5026 return SelectADDRri_imp(OpNode, Addr, Base, Offset, MVT::i64);
5027}
5028
5029bool NVPTXDAGToDAGISel::ChkMemSDNodeAddressSpace(SDNode *N,
5030 unsigned int spN) const {
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]5031 const Value *Src = nullptr;
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]5032 if (MemSDNode *mN = dyn_cast<MemSDNode>(N)) {
Nick Lewyckyaad475b2014-04-15 07:22:52 +0000[diff] [blame]5033 if (spN == 0 && mN->getMemOperand()->getPseudoValue())
5034 return true;
5035 Src = mN->getMemOperand()->getValue();
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]5036 }
5037 if (!Src)
5038 return false;
5039 if (const PointerType *PT = dyn_cast<PointerType>(Src->getType()))
5040 return (PT->getAddressSpace() == spN);
5041 return false;
5042}
5043
5044/// SelectInlineAsmMemoryOperand - Implement addressing mode selection for
5045/// inline asm expressions.
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]5046bool NVPTXDAGToDAGISel::SelectInlineAsmMemoryOperand(
Daniel Sanders60f1db02015-03-13 12:45:09 +0000[diff] [blame^]5047 const SDValue &Op, unsigned ConstraintID, std::vector<SDValue> &OutOps) {
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]5048 SDValue Op0, Op1;
Daniel Sanders60f1db02015-03-13 12:45:09 +0000[diff] [blame^]5049 switch (ConstraintID) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]5050 default:
5051 return true;
Daniel Sanders60f1db02015-03-13 12:45:09 +0000[diff] [blame^]5052 case InlineAsm::Constraint_m: // memory
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]5053 if (SelectDirectAddr(Op, Op0)) {
5054 OutOps.push_back(Op0);
5055 OutOps.push_back(CurDAG->getTargetConstant(0, MVT::i32));
5056 return false;
5057 }
5058 if (SelectADDRri(Op.getNode(), Op, Op0, Op1)) {
5059 OutOps.push_back(Op0);
5060 OutOps.push_back(Op1);
5061 return false;
5062 }
5063 break;
5064 }
5065 return true;
5066}