Gitiles
Code Review Sign In
gerrit-public.fairphone.software / toolchain / llvm-project / 85aebc8c019bae882534790d960b24f5c27051ba / . / llvm / lib / Target / NVPTX / NVPTXISelDAGToDAG.cpp
blob: fa38a686fcbf89e3626a708e00eb5e47dcbd0673 [file] [log] [blame]
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]1//===-- NVPTXISelDAGToDAG.cpp - A dag to dag inst selector for NVPTX ------===//
2//
3// The LLVM Compiler Infrastructure
4//
5// This file is distributed under the University of Illinois Open Source
6// License. See LICENSE.TXT for details.
7//
8//===----------------------------------------------------------------------===//
9//
10// This file defines an instruction selector for the NVPTX target.
11//
12//===----------------------------------------------------------------------===//
13
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]14#include "NVPTXISelDAGToDAG.h"
Chandler Carruth9fb823b2013-01-02 11:36:10 +0000[diff] [blame]15#include "llvm/IR/GlobalValue.h"
16#include "llvm/IR/Instructions.h"
Chandler Carruthed0881b2012-12-03 16:50:05 +0000[diff] [blame]17#include "llvm/Support/CommandLine.h"
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]18#include "llvm/Support/Debug.h"
19#include "llvm/Support/ErrorHandling.h"
Chandler Carruthed0881b2012-12-03 16:50:05 +0000[diff] [blame]20#include "llvm/Support/raw_ostream.h"
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]21#include "llvm/Target/TargetIntrinsicInfo.h"
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]22
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]23using namespace llvm;
24
Chandler Carruth84e68b22014-04-22 02:41:26 +0000[diff] [blame]25#define DEBUG_TYPE "nvptx-isel"
26
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]27static cl::opt<int> UsePrecDivF32(
Nadav Rotem7f27e0b2013-10-18 23:38:13 +0000[diff] [blame]28 "nvptx-prec-divf32", cl::ZeroOrMore, cl::Hidden,
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]29 cl::desc("NVPTX Specifies: 0 use div.approx, 1 use div.full, 2 use"
Sylvestre Ledru469de192014-08-11 18:04:46 +0000[diff] [blame]30 " IEEE Compliant F32 div.rnd if available."),
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]31 cl::init(2));
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]32
Justin Holewinski48f4ad32013-05-21 16:51:30 +0000[diff] [blame]33static cl::opt<bool>
Nadav Rotem7f27e0b2013-10-18 23:38:13 +0000[diff] [blame]34UsePrecSqrtF32("nvptx-prec-sqrtf32", cl::Hidden,
Justin Holewinski48f4ad32013-05-21 16:51:30 +0000[diff] [blame]35 cl::desc("NVPTX Specific: 0 use sqrt.approx, 1 use sqrt.rn."),
36 cl::init(true));
37
Justin Holewinskicd069e62013-07-22 12:18:04 +0000[diff] [blame]38static cl::opt<bool>
Nadav Rotem7f27e0b2013-10-18 23:38:13 +0000[diff] [blame]39FtzEnabled("nvptx-f32ftz", cl::ZeroOrMore, cl::Hidden,
Justin Holewinskicd069e62013-07-22 12:18:04 +0000[diff] [blame]40 cl::desc("NVPTX Specific: Flush f32 subnormals to sign-preserving zero."),
41 cl::init(false));
42
43
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]44/// createNVPTXISelDag - This pass converts a legalized DAG into a
45/// NVPTX-specific DAG, ready for instruction scheduling.
46FunctionPass *llvm::createNVPTXISelDag(NVPTXTargetMachine &TM,
47 llvm::CodeGenOpt::Level OptLevel) {
48 return new NVPTXDAGToDAGISel(TM, OptLevel);
49}
50
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]51NVPTXDAGToDAGISel::NVPTXDAGToDAGISel(NVPTXTargetMachine &tm,
52 CodeGenOpt::Level OptLevel)
Eric Christopher02389e32015-02-19 00:08:27 +0000[diff] [blame]53 : SelectionDAGISel(tm, OptLevel), TM(tm) {
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]54 doMulWide = (OptLevel > 0);
Justin Holewinskicd069e62013-07-22 12:18:04 +0000[diff] [blame]55}
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]56
Eric Christopher147bba22015-01-30 01:40:59 +0000[diff] [blame]57bool NVPTXDAGToDAGISel::runOnMachineFunction(MachineFunction &MF) {
58 Subtarget = &static_cast<const NVPTXSubtarget &>(MF.getSubtarget());
59 return SelectionDAGISel::runOnMachineFunction(MF);
60}
61
Justin Holewinskicd069e62013-07-22 12:18:04 +0000[diff] [blame]62int NVPTXDAGToDAGISel::getDivF32Level() const {
63 if (UsePrecDivF32.getNumOccurrences() > 0) {
64 // If nvptx-prec-div32=N is used on the command-line, always honor it
65 return UsePrecDivF32;
66 } else {
67 // Otherwise, use div.approx if fast math is enabled
68 if (TM.Options.UnsafeFPMath)
69 return 0;
70 else
71 return 2;
72 }
73}
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]74
Justin Holewinskicd069e62013-07-22 12:18:04 +0000[diff] [blame]75bool NVPTXDAGToDAGISel::usePrecSqrtF32() const {
76 if (UsePrecSqrtF32.getNumOccurrences() > 0) {
77 // If nvptx-prec-sqrtf32 is used on the command-line, always honor it
78 return UsePrecSqrtF32;
79 } else {
80 // Otherwise, use sqrt.approx if fast math is enabled
Eli Bendersky3e840192015-03-23 16:26:23 +0000[diff] [blame]81 return !TM.Options.UnsafeFPMath;
Justin Holewinskicd069e62013-07-22 12:18:04 +0000[diff] [blame]82 }
83}
84
85bool NVPTXDAGToDAGISel::useF32FTZ() const {
86 if (FtzEnabled.getNumOccurrences() > 0) {
87 // If nvptx-f32ftz is used on the command-line, always honor it
88 return FtzEnabled;
89 } else {
90 const Function *F = MF->getFunction();
91 // Otherwise, check for an nvptx-f32ftz attribute on the function
92 if (F->hasFnAttribute("nvptx-f32ftz"))
Duncan P. N. Exon Smithb5054332015-02-14 15:35:43 +0000[diff] [blame]93 return F->getFnAttribute("nvptx-f32ftz").getValueAsString() == "true";
Justin Holewinskicd069e62013-07-22 12:18:04 +0000[diff] [blame]94 else
95 return false;
96 }
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]97}
98
Justin Holewinski428cf0e2014-07-17 18:10:09 +0000[diff] [blame]99bool NVPTXDAGToDAGISel::allowFMA() const {
Eric Christopher147bba22015-01-30 01:40:59 +0000[diff] [blame]100 const NVPTXTargetLowering *TL = Subtarget->getTargetLowering();
Justin Holewinski428cf0e2014-07-17 18:10:09 +0000[diff] [blame]101 return TL->allowFMA(*MF, OptLevel);
102}
103
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]104/// Select - Select instructions not customized! Used for
105/// expanded, promoted and normal instructions.
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]106SDNode *NVPTXDAGToDAGISel::Select(SDNode *N) {
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]107
Tim Northover31d093c2013-09-22 08:21:56 +0000[diff] [blame]108 if (N->isMachineOpcode()) {
109 N->setNodeId(-1);
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]110 return nullptr; // Already selected.
Tim Northover31d093c2013-09-22 08:21:56 +0000[diff] [blame]111 }
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]112
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]113 SDNode *ResNode = nullptr;
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]114 switch (N->getOpcode()) {
115 case ISD::LOAD:
116 ResNode = SelectLoad(N);
117 break;
118 case ISD::STORE:
119 ResNode = SelectStore(N);
120 break;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]121 case NVPTXISD::LoadV2:
122 case NVPTXISD::LoadV4:
123 ResNode = SelectLoadVector(N);
124 break;
125 case NVPTXISD::LDGV2:
126 case NVPTXISD::LDGV4:
127 case NVPTXISD::LDUV2:
128 case NVPTXISD::LDUV4:
Justin Holewinskib926d9d2014-06-27 18:35:51 +0000[diff] [blame]129 ResNode = SelectLDGLDU(N);
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]130 break;
131 case NVPTXISD::StoreV2:
132 case NVPTXISD::StoreV4:
133 ResNode = SelectStoreVector(N);
134 break;
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]135 case NVPTXISD::LoadParam:
136 case NVPTXISD::LoadParamV2:
137 case NVPTXISD::LoadParamV4:
138 ResNode = SelectLoadParam(N);
139 break;
140 case NVPTXISD::StoreRetval:
141 case NVPTXISD::StoreRetvalV2:
142 case NVPTXISD::StoreRetvalV4:
143 ResNode = SelectStoreRetval(N);
144 break;
145 case NVPTXISD::StoreParam:
146 case NVPTXISD::StoreParamV2:
147 case NVPTXISD::StoreParamV4:
148 case NVPTXISD::StoreParamS32:
149 case NVPTXISD::StoreParamU32:
150 ResNode = SelectStoreParam(N);
151 break;
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]152 case ISD::INTRINSIC_WO_CHAIN:
153 ResNode = SelectIntrinsicNoChain(N);
154 break;
Justin Holewinskib926d9d2014-06-27 18:35:51 +0000[diff] [blame]155 case ISD::INTRINSIC_W_CHAIN:
156 ResNode = SelectIntrinsicChain(N);
157 break;
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]158 case NVPTXISD::Tex1DFloatS32:
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]159 case NVPTXISD::Tex1DFloatFloat:
160 case NVPTXISD::Tex1DFloatFloatLevel:
161 case NVPTXISD::Tex1DFloatFloatGrad:
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]162 case NVPTXISD::Tex1DS32S32:
163 case NVPTXISD::Tex1DS32Float:
164 case NVPTXISD::Tex1DS32FloatLevel:
165 case NVPTXISD::Tex1DS32FloatGrad:
166 case NVPTXISD::Tex1DU32S32:
167 case NVPTXISD::Tex1DU32Float:
168 case NVPTXISD::Tex1DU32FloatLevel:
169 case NVPTXISD::Tex1DU32FloatGrad:
170 case NVPTXISD::Tex1DArrayFloatS32:
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]171 case NVPTXISD::Tex1DArrayFloatFloat:
172 case NVPTXISD::Tex1DArrayFloatFloatLevel:
173 case NVPTXISD::Tex1DArrayFloatFloatGrad:
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]174 case NVPTXISD::Tex1DArrayS32S32:
175 case NVPTXISD::Tex1DArrayS32Float:
176 case NVPTXISD::Tex1DArrayS32FloatLevel:
177 case NVPTXISD::Tex1DArrayS32FloatGrad:
178 case NVPTXISD::Tex1DArrayU32S32:
179 case NVPTXISD::Tex1DArrayU32Float:
180 case NVPTXISD::Tex1DArrayU32FloatLevel:
181 case NVPTXISD::Tex1DArrayU32FloatGrad:
182 case NVPTXISD::Tex2DFloatS32:
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]183 case NVPTXISD::Tex2DFloatFloat:
184 case NVPTXISD::Tex2DFloatFloatLevel:
185 case NVPTXISD::Tex2DFloatFloatGrad:
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]186 case NVPTXISD::Tex2DS32S32:
187 case NVPTXISD::Tex2DS32Float:
188 case NVPTXISD::Tex2DS32FloatLevel:
189 case NVPTXISD::Tex2DS32FloatGrad:
190 case NVPTXISD::Tex2DU32S32:
191 case NVPTXISD::Tex2DU32Float:
192 case NVPTXISD::Tex2DU32FloatLevel:
193 case NVPTXISD::Tex2DU32FloatGrad:
194 case NVPTXISD::Tex2DArrayFloatS32:
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]195 case NVPTXISD::Tex2DArrayFloatFloat:
196 case NVPTXISD::Tex2DArrayFloatFloatLevel:
197 case NVPTXISD::Tex2DArrayFloatFloatGrad:
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]198 case NVPTXISD::Tex2DArrayS32S32:
199 case NVPTXISD::Tex2DArrayS32Float:
200 case NVPTXISD::Tex2DArrayS32FloatLevel:
201 case NVPTXISD::Tex2DArrayS32FloatGrad:
202 case NVPTXISD::Tex2DArrayU32S32:
203 case NVPTXISD::Tex2DArrayU32Float:
204 case NVPTXISD::Tex2DArrayU32FloatLevel:
205 case NVPTXISD::Tex2DArrayU32FloatGrad:
206 case NVPTXISD::Tex3DFloatS32:
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]207 case NVPTXISD::Tex3DFloatFloat:
208 case NVPTXISD::Tex3DFloatFloatLevel:
209 case NVPTXISD::Tex3DFloatFloatGrad:
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]210 case NVPTXISD::Tex3DS32S32:
211 case NVPTXISD::Tex3DS32Float:
212 case NVPTXISD::Tex3DS32FloatLevel:
213 case NVPTXISD::Tex3DS32FloatGrad:
214 case NVPTXISD::Tex3DU32S32:
215 case NVPTXISD::Tex3DU32Float:
216 case NVPTXISD::Tex3DU32FloatLevel:
217 case NVPTXISD::Tex3DU32FloatGrad:
218 case NVPTXISD::TexCubeFloatFloat:
219 case NVPTXISD::TexCubeFloatFloatLevel:
220 case NVPTXISD::TexCubeS32Float:
221 case NVPTXISD::TexCubeS32FloatLevel:
222 case NVPTXISD::TexCubeU32Float:
223 case NVPTXISD::TexCubeU32FloatLevel:
224 case NVPTXISD::TexCubeArrayFloatFloat:
225 case NVPTXISD::TexCubeArrayFloatFloatLevel:
226 case NVPTXISD::TexCubeArrayS32Float:
227 case NVPTXISD::TexCubeArrayS32FloatLevel:
228 case NVPTXISD::TexCubeArrayU32Float:
229 case NVPTXISD::TexCubeArrayU32FloatLevel:
230 case NVPTXISD::Tld4R2DFloatFloat:
231 case NVPTXISD::Tld4G2DFloatFloat:
232 case NVPTXISD::Tld4B2DFloatFloat:
233 case NVPTXISD::Tld4A2DFloatFloat:
234 case NVPTXISD::Tld4R2DS64Float:
235 case NVPTXISD::Tld4G2DS64Float:
236 case NVPTXISD::Tld4B2DS64Float:
237 case NVPTXISD::Tld4A2DS64Float:
238 case NVPTXISD::Tld4R2DU64Float:
239 case NVPTXISD::Tld4G2DU64Float:
240 case NVPTXISD::Tld4B2DU64Float:
241 case NVPTXISD::Tld4A2DU64Float:
242 case NVPTXISD::TexUnified1DFloatS32:
243 case NVPTXISD::TexUnified1DFloatFloat:
244 case NVPTXISD::TexUnified1DFloatFloatLevel:
245 case NVPTXISD::TexUnified1DFloatFloatGrad:
246 case NVPTXISD::TexUnified1DS32S32:
247 case NVPTXISD::TexUnified1DS32Float:
248 case NVPTXISD::TexUnified1DS32FloatLevel:
249 case NVPTXISD::TexUnified1DS32FloatGrad:
250 case NVPTXISD::TexUnified1DU32S32:
251 case NVPTXISD::TexUnified1DU32Float:
252 case NVPTXISD::TexUnified1DU32FloatLevel:
253 case NVPTXISD::TexUnified1DU32FloatGrad:
254 case NVPTXISD::TexUnified1DArrayFloatS32:
255 case NVPTXISD::TexUnified1DArrayFloatFloat:
256 case NVPTXISD::TexUnified1DArrayFloatFloatLevel:
257 case NVPTXISD::TexUnified1DArrayFloatFloatGrad:
258 case NVPTXISD::TexUnified1DArrayS32S32:
259 case NVPTXISD::TexUnified1DArrayS32Float:
260 case NVPTXISD::TexUnified1DArrayS32FloatLevel:
261 case NVPTXISD::TexUnified1DArrayS32FloatGrad:
262 case NVPTXISD::TexUnified1DArrayU32S32:
263 case NVPTXISD::TexUnified1DArrayU32Float:
264 case NVPTXISD::TexUnified1DArrayU32FloatLevel:
265 case NVPTXISD::TexUnified1DArrayU32FloatGrad:
266 case NVPTXISD::TexUnified2DFloatS32:
267 case NVPTXISD::TexUnified2DFloatFloat:
268 case NVPTXISD::TexUnified2DFloatFloatLevel:
269 case NVPTXISD::TexUnified2DFloatFloatGrad:
270 case NVPTXISD::TexUnified2DS32S32:
271 case NVPTXISD::TexUnified2DS32Float:
272 case NVPTXISD::TexUnified2DS32FloatLevel:
273 case NVPTXISD::TexUnified2DS32FloatGrad:
274 case NVPTXISD::TexUnified2DU32S32:
275 case NVPTXISD::TexUnified2DU32Float:
276 case NVPTXISD::TexUnified2DU32FloatLevel:
277 case NVPTXISD::TexUnified2DU32FloatGrad:
278 case NVPTXISD::TexUnified2DArrayFloatS32:
279 case NVPTXISD::TexUnified2DArrayFloatFloat:
280 case NVPTXISD::TexUnified2DArrayFloatFloatLevel:
281 case NVPTXISD::TexUnified2DArrayFloatFloatGrad:
282 case NVPTXISD::TexUnified2DArrayS32S32:
283 case NVPTXISD::TexUnified2DArrayS32Float:
284 case NVPTXISD::TexUnified2DArrayS32FloatLevel:
285 case NVPTXISD::TexUnified2DArrayS32FloatGrad:
286 case NVPTXISD::TexUnified2DArrayU32S32:
287 case NVPTXISD::TexUnified2DArrayU32Float:
288 case NVPTXISD::TexUnified2DArrayU32FloatLevel:
289 case NVPTXISD::TexUnified2DArrayU32FloatGrad:
290 case NVPTXISD::TexUnified3DFloatS32:
291 case NVPTXISD::TexUnified3DFloatFloat:
292 case NVPTXISD::TexUnified3DFloatFloatLevel:
293 case NVPTXISD::TexUnified3DFloatFloatGrad:
294 case NVPTXISD::TexUnified3DS32S32:
295 case NVPTXISD::TexUnified3DS32Float:
296 case NVPTXISD::TexUnified3DS32FloatLevel:
297 case NVPTXISD::TexUnified3DS32FloatGrad:
298 case NVPTXISD::TexUnified3DU32S32:
299 case NVPTXISD::TexUnified3DU32Float:
300 case NVPTXISD::TexUnified3DU32FloatLevel:
301 case NVPTXISD::TexUnified3DU32FloatGrad:
302 case NVPTXISD::TexUnifiedCubeFloatFloat:
303 case NVPTXISD::TexUnifiedCubeFloatFloatLevel:
304 case NVPTXISD::TexUnifiedCubeS32Float:
305 case NVPTXISD::TexUnifiedCubeS32FloatLevel:
306 case NVPTXISD::TexUnifiedCubeU32Float:
307 case NVPTXISD::TexUnifiedCubeU32FloatLevel:
308 case NVPTXISD::TexUnifiedCubeArrayFloatFloat:
309 case NVPTXISD::TexUnifiedCubeArrayFloatFloatLevel:
310 case NVPTXISD::TexUnifiedCubeArrayS32Float:
311 case NVPTXISD::TexUnifiedCubeArrayS32FloatLevel:
312 case NVPTXISD::TexUnifiedCubeArrayU32Float:
313 case NVPTXISD::TexUnifiedCubeArrayU32FloatLevel:
314 case NVPTXISD::Tld4UnifiedR2DFloatFloat:
315 case NVPTXISD::Tld4UnifiedG2DFloatFloat:
316 case NVPTXISD::Tld4UnifiedB2DFloatFloat:
317 case NVPTXISD::Tld4UnifiedA2DFloatFloat:
318 case NVPTXISD::Tld4UnifiedR2DS64Float:
319 case NVPTXISD::Tld4UnifiedG2DS64Float:
320 case NVPTXISD::Tld4UnifiedB2DS64Float:
321 case NVPTXISD::Tld4UnifiedA2DS64Float:
322 case NVPTXISD::Tld4UnifiedR2DU64Float:
323 case NVPTXISD::Tld4UnifiedG2DU64Float:
324 case NVPTXISD::Tld4UnifiedB2DU64Float:
325 case NVPTXISD::Tld4UnifiedA2DU64Float:
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]326 ResNode = SelectTextureIntrinsic(N);
327 break;
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]328 case NVPTXISD::Suld1DI8Clamp:
329 case NVPTXISD::Suld1DI16Clamp:
330 case NVPTXISD::Suld1DI32Clamp:
331 case NVPTXISD::Suld1DI64Clamp:
332 case NVPTXISD::Suld1DV2I8Clamp:
333 case NVPTXISD::Suld1DV2I16Clamp:
334 case NVPTXISD::Suld1DV2I32Clamp:
335 case NVPTXISD::Suld1DV2I64Clamp:
336 case NVPTXISD::Suld1DV4I8Clamp:
337 case NVPTXISD::Suld1DV4I16Clamp:
338 case NVPTXISD::Suld1DV4I32Clamp:
339 case NVPTXISD::Suld1DArrayI8Clamp:
340 case NVPTXISD::Suld1DArrayI16Clamp:
341 case NVPTXISD::Suld1DArrayI32Clamp:
342 case NVPTXISD::Suld1DArrayI64Clamp:
343 case NVPTXISD::Suld1DArrayV2I8Clamp:
344 case NVPTXISD::Suld1DArrayV2I16Clamp:
345 case NVPTXISD::Suld1DArrayV2I32Clamp:
346 case NVPTXISD::Suld1DArrayV2I64Clamp:
347 case NVPTXISD::Suld1DArrayV4I8Clamp:
348 case NVPTXISD::Suld1DArrayV4I16Clamp:
349 case NVPTXISD::Suld1DArrayV4I32Clamp:
350 case NVPTXISD::Suld2DI8Clamp:
351 case NVPTXISD::Suld2DI16Clamp:
352 case NVPTXISD::Suld2DI32Clamp:
353 case NVPTXISD::Suld2DI64Clamp:
354 case NVPTXISD::Suld2DV2I8Clamp:
355 case NVPTXISD::Suld2DV2I16Clamp:
356 case NVPTXISD::Suld2DV2I32Clamp:
357 case NVPTXISD::Suld2DV2I64Clamp:
358 case NVPTXISD::Suld2DV4I8Clamp:
359 case NVPTXISD::Suld2DV4I16Clamp:
360 case NVPTXISD::Suld2DV4I32Clamp:
361 case NVPTXISD::Suld2DArrayI8Clamp:
362 case NVPTXISD::Suld2DArrayI16Clamp:
363 case NVPTXISD::Suld2DArrayI32Clamp:
364 case NVPTXISD::Suld2DArrayI64Clamp:
365 case NVPTXISD::Suld2DArrayV2I8Clamp:
366 case NVPTXISD::Suld2DArrayV2I16Clamp:
367 case NVPTXISD::Suld2DArrayV2I32Clamp:
368 case NVPTXISD::Suld2DArrayV2I64Clamp:
369 case NVPTXISD::Suld2DArrayV4I8Clamp:
370 case NVPTXISD::Suld2DArrayV4I16Clamp:
371 case NVPTXISD::Suld2DArrayV4I32Clamp:
372 case NVPTXISD::Suld3DI8Clamp:
373 case NVPTXISD::Suld3DI16Clamp:
374 case NVPTXISD::Suld3DI32Clamp:
375 case NVPTXISD::Suld3DI64Clamp:
376 case NVPTXISD::Suld3DV2I8Clamp:
377 case NVPTXISD::Suld3DV2I16Clamp:
378 case NVPTXISD::Suld3DV2I32Clamp:
379 case NVPTXISD::Suld3DV2I64Clamp:
380 case NVPTXISD::Suld3DV4I8Clamp:
381 case NVPTXISD::Suld3DV4I16Clamp:
382 case NVPTXISD::Suld3DV4I32Clamp:
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]383 case NVPTXISD::Suld1DI8Trap:
384 case NVPTXISD::Suld1DI16Trap:
385 case NVPTXISD::Suld1DI32Trap:
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]386 case NVPTXISD::Suld1DI64Trap:
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]387 case NVPTXISD::Suld1DV2I8Trap:
388 case NVPTXISD::Suld1DV2I16Trap:
389 case NVPTXISD::Suld1DV2I32Trap:
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]390 case NVPTXISD::Suld1DV2I64Trap:
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]391 case NVPTXISD::Suld1DV4I8Trap:
392 case NVPTXISD::Suld1DV4I16Trap:
393 case NVPTXISD::Suld1DV4I32Trap:
394 case NVPTXISD::Suld1DArrayI8Trap:
395 case NVPTXISD::Suld1DArrayI16Trap:
396 case NVPTXISD::Suld1DArrayI32Trap:
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]397 case NVPTXISD::Suld1DArrayI64Trap:
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]398 case NVPTXISD::Suld1DArrayV2I8Trap:
399 case NVPTXISD::Suld1DArrayV2I16Trap:
400 case NVPTXISD::Suld1DArrayV2I32Trap:
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]401 case NVPTXISD::Suld1DArrayV2I64Trap:
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]402 case NVPTXISD::Suld1DArrayV4I8Trap:
403 case NVPTXISD::Suld1DArrayV4I16Trap:
404 case NVPTXISD::Suld1DArrayV4I32Trap:
405 case NVPTXISD::Suld2DI8Trap:
406 case NVPTXISD::Suld2DI16Trap:
407 case NVPTXISD::Suld2DI32Trap:
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]408 case NVPTXISD::Suld2DI64Trap:
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]409 case NVPTXISD::Suld2DV2I8Trap:
410 case NVPTXISD::Suld2DV2I16Trap:
411 case NVPTXISD::Suld2DV2I32Trap:
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]412 case NVPTXISD::Suld2DV2I64Trap:
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]413 case NVPTXISD::Suld2DV4I8Trap:
414 case NVPTXISD::Suld2DV4I16Trap:
415 case NVPTXISD::Suld2DV4I32Trap:
416 case NVPTXISD::Suld2DArrayI8Trap:
417 case NVPTXISD::Suld2DArrayI16Trap:
418 case NVPTXISD::Suld2DArrayI32Trap:
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]419 case NVPTXISD::Suld2DArrayI64Trap:
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]420 case NVPTXISD::Suld2DArrayV2I8Trap:
421 case NVPTXISD::Suld2DArrayV2I16Trap:
422 case NVPTXISD::Suld2DArrayV2I32Trap:
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]423 case NVPTXISD::Suld2DArrayV2I64Trap:
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]424 case NVPTXISD::Suld2DArrayV4I8Trap:
425 case NVPTXISD::Suld2DArrayV4I16Trap:
426 case NVPTXISD::Suld2DArrayV4I32Trap:
427 case NVPTXISD::Suld3DI8Trap:
428 case NVPTXISD::Suld3DI16Trap:
429 case NVPTXISD::Suld3DI32Trap:
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]430 case NVPTXISD::Suld3DI64Trap:
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]431 case NVPTXISD::Suld3DV2I8Trap:
432 case NVPTXISD::Suld3DV2I16Trap:
433 case NVPTXISD::Suld3DV2I32Trap:
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]434 case NVPTXISD::Suld3DV2I64Trap:
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]435 case NVPTXISD::Suld3DV4I8Trap:
436 case NVPTXISD::Suld3DV4I16Trap:
437 case NVPTXISD::Suld3DV4I32Trap:
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]438 case NVPTXISD::Suld1DI8Zero:
439 case NVPTXISD::Suld1DI16Zero:
440 case NVPTXISD::Suld1DI32Zero:
441 case NVPTXISD::Suld1DI64Zero:
442 case NVPTXISD::Suld1DV2I8Zero:
443 case NVPTXISD::Suld1DV2I16Zero:
444 case NVPTXISD::Suld1DV2I32Zero:
445 case NVPTXISD::Suld1DV2I64Zero:
446 case NVPTXISD::Suld1DV4I8Zero:
447 case NVPTXISD::Suld1DV4I16Zero:
448 case NVPTXISD::Suld1DV4I32Zero:
449 case NVPTXISD::Suld1DArrayI8Zero:
450 case NVPTXISD::Suld1DArrayI16Zero:
451 case NVPTXISD::Suld1DArrayI32Zero:
452 case NVPTXISD::Suld1DArrayI64Zero:
453 case NVPTXISD::Suld1DArrayV2I8Zero:
454 case NVPTXISD::Suld1DArrayV2I16Zero:
455 case NVPTXISD::Suld1DArrayV2I32Zero:
456 case NVPTXISD::Suld1DArrayV2I64Zero:
457 case NVPTXISD::Suld1DArrayV4I8Zero:
458 case NVPTXISD::Suld1DArrayV4I16Zero:
459 case NVPTXISD::Suld1DArrayV4I32Zero:
460 case NVPTXISD::Suld2DI8Zero:
461 case NVPTXISD::Suld2DI16Zero:
462 case NVPTXISD::Suld2DI32Zero:
463 case NVPTXISD::Suld2DI64Zero:
464 case NVPTXISD::Suld2DV2I8Zero:
465 case NVPTXISD::Suld2DV2I16Zero:
466 case NVPTXISD::Suld2DV2I32Zero:
467 case NVPTXISD::Suld2DV2I64Zero:
468 case NVPTXISD::Suld2DV4I8Zero:
469 case NVPTXISD::Suld2DV4I16Zero:
470 case NVPTXISD::Suld2DV4I32Zero:
471 case NVPTXISD::Suld2DArrayI8Zero:
472 case NVPTXISD::Suld2DArrayI16Zero:
473 case NVPTXISD::Suld2DArrayI32Zero:
474 case NVPTXISD::Suld2DArrayI64Zero:
475 case NVPTXISD::Suld2DArrayV2I8Zero:
476 case NVPTXISD::Suld2DArrayV2I16Zero:
477 case NVPTXISD::Suld2DArrayV2I32Zero:
478 case NVPTXISD::Suld2DArrayV2I64Zero:
479 case NVPTXISD::Suld2DArrayV4I8Zero:
480 case NVPTXISD::Suld2DArrayV4I16Zero:
481 case NVPTXISD::Suld2DArrayV4I32Zero:
482 case NVPTXISD::Suld3DI8Zero:
483 case NVPTXISD::Suld3DI16Zero:
484 case NVPTXISD::Suld3DI32Zero:
485 case NVPTXISD::Suld3DI64Zero:
486 case NVPTXISD::Suld3DV2I8Zero:
487 case NVPTXISD::Suld3DV2I16Zero:
488 case NVPTXISD::Suld3DV2I32Zero:
489 case NVPTXISD::Suld3DV2I64Zero:
490 case NVPTXISD::Suld3DV4I8Zero:
491 case NVPTXISD::Suld3DV4I16Zero:
492 case NVPTXISD::Suld3DV4I32Zero:
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]493 ResNode = SelectSurfaceIntrinsic(N);
494 break;
Justin Holewinskica7a4f12014-06-27 18:35:27 +0000[diff] [blame]495 case ISD::AND:
496 case ISD::SRA:
497 case ISD::SRL:
498 // Try to select BFE
499 ResNode = SelectBFE(N);
500 break;
Justin Holewinskiba2fa6d2014-03-24 11:17:53 +0000[diff] [blame]501 case ISD::ADDRSPACECAST:
502 ResNode = SelectAddrSpaceCast(N);
503 break;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]504 default:
505 break;
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]506 }
507 if (ResNode)
508 return ResNode;
509 return SelectCode(N);
510}
511
Justin Holewinskib926d9d2014-06-27 18:35:51 +0000[diff] [blame]512SDNode *NVPTXDAGToDAGISel::SelectIntrinsicChain(SDNode *N) {
513 unsigned IID = cast<ConstantSDNode>(N->getOperand(1))->getZExtValue();
514 switch (IID) {
515 default:
516 return NULL;
517 case Intrinsic::nvvm_ldg_global_f:
518 case Intrinsic::nvvm_ldg_global_i:
519 case Intrinsic::nvvm_ldg_global_p:
520 case Intrinsic::nvvm_ldu_global_f:
521 case Intrinsic::nvvm_ldu_global_i:
522 case Intrinsic::nvvm_ldu_global_p:
523 return SelectLDGLDU(N);
524 }
525}
526
Eric Christopher9745b3a2015-01-30 01:41:01 +0000[diff] [blame]527static unsigned int getCodeAddrSpace(MemSDNode *N) {
Nick Lewyckyaad475b2014-04-15 07:22:52 +0000[diff] [blame]528 const Value *Src = N->getMemOperand()->getValue();
Justin Holewinskib96d1392013-06-10 13:29:47 +0000[diff] [blame]529
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]530 if (!Src)
Justin Holewinskib96d1392013-06-10 13:29:47 +0000[diff] [blame]531 return NVPTX::PTXLdStInstCode::GENERIC;
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]532
533 if (const PointerType *PT = dyn_cast<PointerType>(Src->getType())) {
534 switch (PT->getAddressSpace()) {
Justin Holewinskib96d1392013-06-10 13:29:47 +0000[diff] [blame]535 case llvm::ADDRESS_SPACE_LOCAL: return NVPTX::PTXLdStInstCode::LOCAL;
536 case llvm::ADDRESS_SPACE_GLOBAL: return NVPTX::PTXLdStInstCode::GLOBAL;
537 case llvm::ADDRESS_SPACE_SHARED: return NVPTX::PTXLdStInstCode::SHARED;
538 case llvm::ADDRESS_SPACE_GENERIC: return NVPTX::PTXLdStInstCode::GENERIC;
539 case llvm::ADDRESS_SPACE_PARAM: return NVPTX::PTXLdStInstCode::PARAM;
540 case llvm::ADDRESS_SPACE_CONST: return NVPTX::PTXLdStInstCode::CONSTANT;
541 default: break;
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]542 }
543 }
Justin Holewinskib96d1392013-06-10 13:29:47 +0000[diff] [blame]544 return NVPTX::PTXLdStInstCode::GENERIC;
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]545}
546
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]547SDNode *NVPTXDAGToDAGISel::SelectIntrinsicNoChain(SDNode *N) {
548 unsigned IID = cast<ConstantSDNode>(N->getOperand(0))->getZExtValue();
549 switch (IID) {
550 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]551 return nullptr;
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]552 case Intrinsic::nvvm_texsurf_handle_internal:
553 return SelectTexSurfHandle(N);
554 }
555}
556
557SDNode *NVPTXDAGToDAGISel::SelectTexSurfHandle(SDNode *N) {
558 // Op 0 is the intrinsic ID
559 SDValue Wrapper = N->getOperand(1);
560 SDValue GlobalVal = Wrapper.getOperand(0);
561 return CurDAG->getMachineNode(NVPTX::texsurf_handles, SDLoc(N), MVT::i64,
562 GlobalVal);
563}
564
Justin Holewinskiba2fa6d2014-03-24 11:17:53 +0000[diff] [blame]565SDNode *NVPTXDAGToDAGISel::SelectAddrSpaceCast(SDNode *N) {
566 SDValue Src = N->getOperand(0);
567 AddrSpaceCastSDNode *CastN = cast<AddrSpaceCastSDNode>(N);
568 unsigned SrcAddrSpace = CastN->getSrcAddressSpace();
569 unsigned DstAddrSpace = CastN->getDestAddressSpace();
570
571 assert(SrcAddrSpace != DstAddrSpace &&
572 "addrspacecast must be between different address spaces");
573
574 if (DstAddrSpace == ADDRESS_SPACE_GENERIC) {
575 // Specific to generic
576 unsigned Opc;
577 switch (SrcAddrSpace) {
578 default: report_fatal_error("Bad address space in addrspacecast");
579 case ADDRESS_SPACE_GLOBAL:
Eric Christopher02389e32015-02-19 00:08:27 +0000[diff] [blame]580 Opc = TM.is64Bit() ? NVPTX::cvta_global_yes_64 : NVPTX::cvta_global_yes;
Justin Holewinskiba2fa6d2014-03-24 11:17:53 +0000[diff] [blame]581 break;
582 case ADDRESS_SPACE_SHARED:
Eric Christopher02389e32015-02-19 00:08:27 +0000[diff] [blame]583 Opc = TM.is64Bit() ? NVPTX::cvta_shared_yes_64 : NVPTX::cvta_shared_yes;
Justin Holewinskiba2fa6d2014-03-24 11:17:53 +0000[diff] [blame]584 break;
585 case ADDRESS_SPACE_CONST:
Eric Christopher02389e32015-02-19 00:08:27 +0000[diff] [blame]586 Opc = TM.is64Bit() ? NVPTX::cvta_const_yes_64 : NVPTX::cvta_const_yes;
Justin Holewinskiba2fa6d2014-03-24 11:17:53 +0000[diff] [blame]587 break;
588 case ADDRESS_SPACE_LOCAL:
Eric Christopher02389e32015-02-19 00:08:27 +0000[diff] [blame]589 Opc = TM.is64Bit() ? NVPTX::cvta_local_yes_64 : NVPTX::cvta_local_yes;
Justin Holewinskiba2fa6d2014-03-24 11:17:53 +0000[diff] [blame]590 break;
591 }
592 return CurDAG->getMachineNode(Opc, SDLoc(N), N->getValueType(0), Src);
593 } else {
594 // Generic to specific
595 if (SrcAddrSpace != 0)
596 report_fatal_error("Cannot cast between two non-generic address spaces");
597 unsigned Opc;
598 switch (DstAddrSpace) {
599 default: report_fatal_error("Bad address space in addrspacecast");
600 case ADDRESS_SPACE_GLOBAL:
Eric Christopher02389e32015-02-19 00:08:27 +0000[diff] [blame]601 Opc = TM.is64Bit() ? NVPTX::cvta_to_global_yes_64
602 : NVPTX::cvta_to_global_yes;
Justin Holewinskiba2fa6d2014-03-24 11:17:53 +0000[diff] [blame]603 break;
604 case ADDRESS_SPACE_SHARED:
Eric Christopher02389e32015-02-19 00:08:27 +0000[diff] [blame]605 Opc = TM.is64Bit() ? NVPTX::cvta_to_shared_yes_64
606 : NVPTX::cvta_to_shared_yes;
Justin Holewinskiba2fa6d2014-03-24 11:17:53 +0000[diff] [blame]607 break;
608 case ADDRESS_SPACE_CONST:
Eric Christopher02389e32015-02-19 00:08:27 +0000[diff] [blame]609 Opc =
610 TM.is64Bit() ? NVPTX::cvta_to_const_yes_64 : NVPTX::cvta_to_const_yes;
Justin Holewinskiba2fa6d2014-03-24 11:17:53 +0000[diff] [blame]611 break;
612 case ADDRESS_SPACE_LOCAL:
Eric Christopher02389e32015-02-19 00:08:27 +0000[diff] [blame]613 Opc =
614 TM.is64Bit() ? NVPTX::cvta_to_local_yes_64 : NVPTX::cvta_to_local_yes;
Justin Holewinskiba2fa6d2014-03-24 11:17:53 +0000[diff] [blame]615 break;
616 }
617 return CurDAG->getMachineNode(Opc, SDLoc(N), N->getValueType(0), Src);
618 }
619}
620
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]621SDNode *NVPTXDAGToDAGISel::SelectLoad(SDNode *N) {
Andrew Trickef9de2a2013-05-25 02:42:55 +0000[diff] [blame]622 SDLoc dl(N);
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]623 LoadSDNode *LD = cast<LoadSDNode>(N);
624 EVT LoadedVT = LD->getMemoryVT();
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]625 SDNode *NVPTXLD = nullptr;
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]626
627 // do not support pre/post inc/dec
628 if (LD->isIndexed())
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]629 return nullptr;
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]630
631 if (!LoadedVT.isSimple())
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]632 return nullptr;
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]633
634 // Address Space Setting
Eric Christopher9745b3a2015-01-30 01:41:01 +0000[diff] [blame]635 unsigned int codeAddrSpace = getCodeAddrSpace(LD);
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]636
637 // Volatile Setting
638 // - .volatile is only availalble for .global and .shared
639 bool isVolatile = LD->isVolatile();
640 if (codeAddrSpace != NVPTX::PTXLdStInstCode::GLOBAL &&
641 codeAddrSpace != NVPTX::PTXLdStInstCode::SHARED &&
642 codeAddrSpace != NVPTX::PTXLdStInstCode::GENERIC)
643 isVolatile = false;
644
645 // Vector Setting
646 MVT SimpleVT = LoadedVT.getSimpleVT();
647 unsigned vecType = NVPTX::PTXLdStInstCode::Scalar;
648 if (SimpleVT.isVector()) {
649 unsigned num = SimpleVT.getVectorNumElements();
650 if (num == 2)
651 vecType = NVPTX::PTXLdStInstCode::V2;
652 else if (num == 4)
653 vecType = NVPTX::PTXLdStInstCode::V4;
654 else
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]655 return nullptr;
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]656 }
657
658 // Type Setting: fromType + fromTypeWidth
659 //
660 // Sign : ISD::SEXTLOAD
661 // Unsign : ISD::ZEXTLOAD, ISD::NON_EXTLOAD or ISD::EXTLOAD and the
662 // type is integer
663 // Float : ISD::NON_EXTLOAD or ISD::EXTLOAD and the type is float
664 MVT ScalarVT = SimpleVT.getScalarType();
Justin Holewinski994d66a2013-05-30 12:22:39 +0000[diff] [blame]665 // Read at least 8 bits (predicates are stored as 8-bit values)
666 unsigned fromTypeWidth = std::max(8U, ScalarVT.getSizeInBits());
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]667 unsigned int fromType;
668 if ((LD->getExtensionType() == ISD::SEXTLOAD))
669 fromType = NVPTX::PTXLdStInstCode::Signed;
670 else if (ScalarVT.isFloatingPoint())
671 fromType = NVPTX::PTXLdStInstCode::Float;
672 else
673 fromType = NVPTX::PTXLdStInstCode::Unsigned;
674
675 // Create the machine instruction DAG
676 SDValue Chain = N->getOperand(0);
677 SDValue N1 = N->getOperand(1);
678 SDValue Addr;
679 SDValue Offset, Base;
680 unsigned Opcode;
Craig Topperd9c27832013-08-15 02:44:19 +0000[diff] [blame]681 MVT::SimpleValueType TargetVT = LD->getSimpleValueType(0).SimpleTy;
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]682
683 if (SelectDirectAddr(N1, Addr)) {
684 switch (TargetVT) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]685 case MVT::i8:
686 Opcode = NVPTX::LD_i8_avar;
687 break;
688 case MVT::i16:
689 Opcode = NVPTX::LD_i16_avar;
690 break;
691 case MVT::i32:
692 Opcode = NVPTX::LD_i32_avar;
693 break;
694 case MVT::i64:
695 Opcode = NVPTX::LD_i64_avar;
696 break;
697 case MVT::f32:
698 Opcode = NVPTX::LD_f32_avar;
699 break;
700 case MVT::f64:
701 Opcode = NVPTX::LD_f64_avar;
702 break;
703 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]704 return nullptr;
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]705 }
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +0000[diff] [blame]706 SDValue Ops[] = { getI32Imm(isVolatile, dl), getI32Imm(codeAddrSpace, dl),
707 getI32Imm(vecType, dl), getI32Imm(fromType, dl),
708 getI32Imm(fromTypeWidth, dl), Addr, Chain };
Michael Liaob53d8962013-04-19 22:22:57 +0000[diff] [blame]709 NVPTXLD = CurDAG->getMachineNode(Opcode, dl, TargetVT, MVT::Other, Ops);
Eric Christopher02389e32015-02-19 00:08:27 +0000[diff] [blame]710 } else if (TM.is64Bit() ? SelectADDRsi64(N1.getNode(), N1, Base, Offset)
711 : SelectADDRsi(N1.getNode(), N1, Base, Offset)) {
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]712 switch (TargetVT) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]713 case MVT::i8:
714 Opcode = NVPTX::LD_i8_asi;
715 break;
716 case MVT::i16:
717 Opcode = NVPTX::LD_i16_asi;
718 break;
719 case MVT::i32:
720 Opcode = NVPTX::LD_i32_asi;
721 break;
722 case MVT::i64:
723 Opcode = NVPTX::LD_i64_asi;
724 break;
725 case MVT::f32:
726 Opcode = NVPTX::LD_f32_asi;
727 break;
728 case MVT::f64:
729 Opcode = NVPTX::LD_f64_asi;
730 break;
731 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]732 return nullptr;
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]733 }
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +0000[diff] [blame]734 SDValue Ops[] = { getI32Imm(isVolatile, dl), getI32Imm(codeAddrSpace, dl),
735 getI32Imm(vecType, dl), getI32Imm(fromType, dl),
736 getI32Imm(fromTypeWidth, dl), Base, Offset, Chain };
Michael Liaob53d8962013-04-19 22:22:57 +0000[diff] [blame]737 NVPTXLD = CurDAG->getMachineNode(Opcode, dl, TargetVT, MVT::Other, Ops);
Eric Christopher02389e32015-02-19 00:08:27 +0000[diff] [blame]738 } else if (TM.is64Bit() ? SelectADDRri64(N1.getNode(), N1, Base, Offset)
739 : SelectADDRri(N1.getNode(), N1, Base, Offset)) {
740 if (TM.is64Bit()) {
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]741 switch (TargetVT) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]742 case MVT::i8:
743 Opcode = NVPTX::LD_i8_ari_64;
744 break;
745 case MVT::i16:
746 Opcode = NVPTX::LD_i16_ari_64;
747 break;
748 case MVT::i32:
749 Opcode = NVPTX::LD_i32_ari_64;
750 break;
751 case MVT::i64:
752 Opcode = NVPTX::LD_i64_ari_64;
753 break;
754 case MVT::f32:
755 Opcode = NVPTX::LD_f32_ari_64;
756 break;
757 case MVT::f64:
758 Opcode = NVPTX::LD_f64_ari_64;
759 break;
760 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]761 return nullptr;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]762 }
763 } else {
764 switch (TargetVT) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]765 case MVT::i8:
766 Opcode = NVPTX::LD_i8_ari;
767 break;
768 case MVT::i16:
769 Opcode = NVPTX::LD_i16_ari;
770 break;
771 case MVT::i32:
772 Opcode = NVPTX::LD_i32_ari;
773 break;
774 case MVT::i64:
775 Opcode = NVPTX::LD_i64_ari;
776 break;
777 case MVT::f32:
778 Opcode = NVPTX::LD_f32_ari;
779 break;
780 case MVT::f64:
781 Opcode = NVPTX::LD_f64_ari;
782 break;
783 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]784 return nullptr;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]785 }
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]786 }
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +0000[diff] [blame]787 SDValue Ops[] = { getI32Imm(isVolatile, dl), getI32Imm(codeAddrSpace, dl),
788 getI32Imm(vecType, dl), getI32Imm(fromType, dl),
789 getI32Imm(fromTypeWidth, dl), Base, Offset, Chain };
Michael Liaob53d8962013-04-19 22:22:57 +0000[diff] [blame]790 NVPTXLD = CurDAG->getMachineNode(Opcode, dl, TargetVT, MVT::Other, Ops);
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]791 } else {
Eric Christopher02389e32015-02-19 00:08:27 +0000[diff] [blame]792 if (TM.is64Bit()) {
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]793 switch (TargetVT) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]794 case MVT::i8:
795 Opcode = NVPTX::LD_i8_areg_64;
796 break;
797 case MVT::i16:
798 Opcode = NVPTX::LD_i16_areg_64;
799 break;
800 case MVT::i32:
801 Opcode = NVPTX::LD_i32_areg_64;
802 break;
803 case MVT::i64:
804 Opcode = NVPTX::LD_i64_areg_64;
805 break;
806 case MVT::f32:
807 Opcode = NVPTX::LD_f32_areg_64;
808 break;
809 case MVT::f64:
810 Opcode = NVPTX::LD_f64_areg_64;
811 break;
812 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]813 return nullptr;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]814 }
815 } else {
816 switch (TargetVT) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]817 case MVT::i8:
818 Opcode = NVPTX::LD_i8_areg;
819 break;
820 case MVT::i16:
821 Opcode = NVPTX::LD_i16_areg;
822 break;
823 case MVT::i32:
824 Opcode = NVPTX::LD_i32_areg;
825 break;
826 case MVT::i64:
827 Opcode = NVPTX::LD_i64_areg;
828 break;
829 case MVT::f32:
830 Opcode = NVPTX::LD_f32_areg;
831 break;
832 case MVT::f64:
833 Opcode = NVPTX::LD_f64_areg;
834 break;
835 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]836 return nullptr;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]837 }
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]838 }
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +0000[diff] [blame]839 SDValue Ops[] = { getI32Imm(isVolatile, dl), getI32Imm(codeAddrSpace, dl),
840 getI32Imm(vecType, dl), getI32Imm(fromType, dl),
841 getI32Imm(fromTypeWidth, dl), N1, Chain };
Michael Liaob53d8962013-04-19 22:22:57 +0000[diff] [blame]842 NVPTXLD = CurDAG->getMachineNode(Opcode, dl, TargetVT, MVT::Other, Ops);
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]843 }
844
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]845 if (NVPTXLD) {
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]846 MachineSDNode::mmo_iterator MemRefs0 = MF->allocateMemRefsArray(1);
847 MemRefs0[0] = cast<MemSDNode>(N)->getMemOperand();
848 cast<MachineSDNode>(NVPTXLD)->setMemRefs(MemRefs0, MemRefs0 + 1);
849 }
850
851 return NVPTXLD;
852}
853
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]854SDNode *NVPTXDAGToDAGISel::SelectLoadVector(SDNode *N) {
855
856 SDValue Chain = N->getOperand(0);
857 SDValue Op1 = N->getOperand(1);
858 SDValue Addr, Offset, Base;
859 unsigned Opcode;
Andrew Trickef9de2a2013-05-25 02:42:55 +0000[diff] [blame]860 SDLoc DL(N);
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]861 SDNode *LD;
862 MemSDNode *MemSD = cast<MemSDNode>(N);
863 EVT LoadedVT = MemSD->getMemoryVT();
864
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]865 if (!LoadedVT.isSimple())
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]866 return nullptr;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]867
868 // Address Space Setting
Eric Christopher9745b3a2015-01-30 01:41:01 +0000[diff] [blame]869 unsigned int CodeAddrSpace = getCodeAddrSpace(MemSD);
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]870
871 // Volatile Setting
872 // - .volatile is only availalble for .global and .shared
873 bool IsVolatile = MemSD->isVolatile();
874 if (CodeAddrSpace != NVPTX::PTXLdStInstCode::GLOBAL &&
875 CodeAddrSpace != NVPTX::PTXLdStInstCode::SHARED &&
876 CodeAddrSpace != NVPTX::PTXLdStInstCode::GENERIC)
877 IsVolatile = false;
878
879 // Vector Setting
880 MVT SimpleVT = LoadedVT.getSimpleVT();
881
882 // Type Setting: fromType + fromTypeWidth
883 //
884 // Sign : ISD::SEXTLOAD
885 // Unsign : ISD::ZEXTLOAD, ISD::NON_EXTLOAD or ISD::EXTLOAD and the
886 // type is integer
887 // Float : ISD::NON_EXTLOAD or ISD::EXTLOAD and the type is float
888 MVT ScalarVT = SimpleVT.getScalarType();
Justin Holewinski994d66a2013-05-30 12:22:39 +0000[diff] [blame]889 // Read at least 8 bits (predicates are stored as 8-bit values)
890 unsigned FromTypeWidth = std::max(8U, ScalarVT.getSizeInBits());
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]891 unsigned int FromType;
892 // The last operand holds the original LoadSDNode::getExtensionType() value
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]893 unsigned ExtensionType = cast<ConstantSDNode>(
894 N->getOperand(N->getNumOperands() - 1))->getZExtValue();
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]895 if (ExtensionType == ISD::SEXTLOAD)
896 FromType = NVPTX::PTXLdStInstCode::Signed;
897 else if (ScalarVT.isFloatingPoint())
898 FromType = NVPTX::PTXLdStInstCode::Float;
899 else
900 FromType = NVPTX::PTXLdStInstCode::Unsigned;
901
902 unsigned VecType;
903
904 switch (N->getOpcode()) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]905 case NVPTXISD::LoadV2:
906 VecType = NVPTX::PTXLdStInstCode::V2;
907 break;
908 case NVPTXISD::LoadV4:
909 VecType = NVPTX::PTXLdStInstCode::V4;
910 break;
911 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]912 return nullptr;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]913 }
914
915 EVT EltVT = N->getValueType(0);
916
917 if (SelectDirectAddr(Op1, Addr)) {
918 switch (N->getOpcode()) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]919 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]920 return nullptr;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]921 case NVPTXISD::LoadV2:
922 switch (EltVT.getSimpleVT().SimpleTy) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]923 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]924 return nullptr;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]925 case MVT::i8:
926 Opcode = NVPTX::LDV_i8_v2_avar;
927 break;
928 case MVT::i16:
929 Opcode = NVPTX::LDV_i16_v2_avar;
930 break;
931 case MVT::i32:
932 Opcode = NVPTX::LDV_i32_v2_avar;
933 break;
934 case MVT::i64:
935 Opcode = NVPTX::LDV_i64_v2_avar;
936 break;
937 case MVT::f32:
938 Opcode = NVPTX::LDV_f32_v2_avar;
939 break;
940 case MVT::f64:
941 Opcode = NVPTX::LDV_f64_v2_avar;
942 break;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]943 }
944 break;
945 case NVPTXISD::LoadV4:
946 switch (EltVT.getSimpleVT().SimpleTy) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]947 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]948 return nullptr;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]949 case MVT::i8:
950 Opcode = NVPTX::LDV_i8_v4_avar;
951 break;
952 case MVT::i16:
953 Opcode = NVPTX::LDV_i16_v4_avar;
954 break;
955 case MVT::i32:
956 Opcode = NVPTX::LDV_i32_v4_avar;
957 break;
958 case MVT::f32:
959 Opcode = NVPTX::LDV_f32_v4_avar;
960 break;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]961 }
962 break;
963 }
964
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +0000[diff] [blame]965 SDValue Ops[] = { getI32Imm(IsVolatile, DL), getI32Imm(CodeAddrSpace, DL),
966 getI32Imm(VecType, DL), getI32Imm(FromType, DL),
967 getI32Imm(FromTypeWidth, DL), Addr, Chain };
Michael Liaob53d8962013-04-19 22:22:57 +0000[diff] [blame]968 LD = CurDAG->getMachineNode(Opcode, DL, N->getVTList(), Ops);
Eric Christopher02389e32015-02-19 00:08:27 +0000[diff] [blame]969 } else if (TM.is64Bit() ? SelectADDRsi64(Op1.getNode(), Op1, Base, Offset)
970 : SelectADDRsi(Op1.getNode(), Op1, Base, Offset)) {
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]971 switch (N->getOpcode()) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]972 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]973 return nullptr;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]974 case NVPTXISD::LoadV2:
975 switch (EltVT.getSimpleVT().SimpleTy) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]976 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]977 return nullptr;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]978 case MVT::i8:
979 Opcode = NVPTX::LDV_i8_v2_asi;
980 break;
981 case MVT::i16:
982 Opcode = NVPTX::LDV_i16_v2_asi;
983 break;
984 case MVT::i32:
985 Opcode = NVPTX::LDV_i32_v2_asi;
986 break;
987 case MVT::i64:
988 Opcode = NVPTX::LDV_i64_v2_asi;
989 break;
990 case MVT::f32:
991 Opcode = NVPTX::LDV_f32_v2_asi;
992 break;
993 case MVT::f64:
994 Opcode = NVPTX::LDV_f64_v2_asi;
995 break;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]996 }
997 break;
998 case NVPTXISD::LoadV4:
999 switch (EltVT.getSimpleVT().SimpleTy) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1000 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]1001 return nullptr;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1002 case MVT::i8:
1003 Opcode = NVPTX::LDV_i8_v4_asi;
1004 break;
1005 case MVT::i16:
1006 Opcode = NVPTX::LDV_i16_v4_asi;
1007 break;
1008 case MVT::i32:
1009 Opcode = NVPTX::LDV_i32_v4_asi;
1010 break;
1011 case MVT::f32:
1012 Opcode = NVPTX::LDV_f32_v4_asi;
1013 break;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]1014 }
1015 break;
1016 }
1017
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +0000[diff] [blame]1018 SDValue Ops[] = { getI32Imm(IsVolatile, DL), getI32Imm(CodeAddrSpace, DL),
1019 getI32Imm(VecType, DL), getI32Imm(FromType, DL),
1020 getI32Imm(FromTypeWidth, DL), Base, Offset, Chain };
Michael Liaob53d8962013-04-19 22:22:57 +0000[diff] [blame]1021 LD = CurDAG->getMachineNode(Opcode, DL, N->getVTList(), Ops);
Eric Christopher02389e32015-02-19 00:08:27 +0000[diff] [blame]1022 } else if (TM.is64Bit() ? SelectADDRri64(Op1.getNode(), Op1, Base, Offset)
1023 : SelectADDRri(Op1.getNode(), Op1, Base, Offset)) {
1024 if (TM.is64Bit()) {
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]1025 switch (N->getOpcode()) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1026 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]1027 return nullptr;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]1028 case NVPTXISD::LoadV2:
1029 switch (EltVT.getSimpleVT().SimpleTy) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1030 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]1031 return nullptr;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1032 case MVT::i8:
1033 Opcode = NVPTX::LDV_i8_v2_ari_64;
1034 break;
1035 case MVT::i16:
1036 Opcode = NVPTX::LDV_i16_v2_ari_64;
1037 break;
1038 case MVT::i32:
1039 Opcode = NVPTX::LDV_i32_v2_ari_64;
1040 break;
1041 case MVT::i64:
1042 Opcode = NVPTX::LDV_i64_v2_ari_64;
1043 break;
1044 case MVT::f32:
1045 Opcode = NVPTX::LDV_f32_v2_ari_64;
1046 break;
1047 case MVT::f64:
1048 Opcode = NVPTX::LDV_f64_v2_ari_64;
1049 break;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]1050 }
1051 break;
1052 case NVPTXISD::LoadV4:
1053 switch (EltVT.getSimpleVT().SimpleTy) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1054 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]1055 return nullptr;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1056 case MVT::i8:
1057 Opcode = NVPTX::LDV_i8_v4_ari_64;
1058 break;
1059 case MVT::i16:
1060 Opcode = NVPTX::LDV_i16_v4_ari_64;
1061 break;
1062 case MVT::i32:
1063 Opcode = NVPTX::LDV_i32_v4_ari_64;
1064 break;
1065 case MVT::f32:
1066 Opcode = NVPTX::LDV_f32_v4_ari_64;
1067 break;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]1068 }
1069 break;
1070 }
1071 } else {
1072 switch (N->getOpcode()) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1073 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]1074 return nullptr;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]1075 case NVPTXISD::LoadV2:
1076 switch (EltVT.getSimpleVT().SimpleTy) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1077 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]1078 return nullptr;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1079 case MVT::i8:
1080 Opcode = NVPTX::LDV_i8_v2_ari;
1081 break;
1082 case MVT::i16:
1083 Opcode = NVPTX::LDV_i16_v2_ari;
1084 break;
1085 case MVT::i32:
1086 Opcode = NVPTX::LDV_i32_v2_ari;
1087 break;
1088 case MVT::i64:
1089 Opcode = NVPTX::LDV_i64_v2_ari;
1090 break;
1091 case MVT::f32:
1092 Opcode = NVPTX::LDV_f32_v2_ari;
1093 break;
1094 case MVT::f64:
1095 Opcode = NVPTX::LDV_f64_v2_ari;
1096 break;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]1097 }
1098 break;
1099 case NVPTXISD::LoadV4:
1100 switch (EltVT.getSimpleVT().SimpleTy) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1101 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]1102 return nullptr;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1103 case MVT::i8:
1104 Opcode = NVPTX::LDV_i8_v4_ari;
1105 break;
1106 case MVT::i16:
1107 Opcode = NVPTX::LDV_i16_v4_ari;
1108 break;
1109 case MVT::i32:
1110 Opcode = NVPTX::LDV_i32_v4_ari;
1111 break;
1112 case MVT::f32:
1113 Opcode = NVPTX::LDV_f32_v4_ari;
1114 break;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]1115 }
1116 break;
1117 }
1118 }
1119
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +0000[diff] [blame]1120 SDValue Ops[] = { getI32Imm(IsVolatile, DL), getI32Imm(CodeAddrSpace, DL),
1121 getI32Imm(VecType, DL), getI32Imm(FromType, DL),
1122 getI32Imm(FromTypeWidth, DL), Base, Offset, Chain };
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]1123
Michael Liaob53d8962013-04-19 22:22:57 +0000[diff] [blame]1124 LD = CurDAG->getMachineNode(Opcode, DL, N->getVTList(), Ops);
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]1125 } else {
Eric Christopher02389e32015-02-19 00:08:27 +0000[diff] [blame]1126 if (TM.is64Bit()) {
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]1127 switch (N->getOpcode()) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1128 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]1129 return nullptr;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]1130 case NVPTXISD::LoadV2:
1131 switch (EltVT.getSimpleVT().SimpleTy) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1132 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]1133 return nullptr;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1134 case MVT::i8:
1135 Opcode = NVPTX::LDV_i8_v2_areg_64;
1136 break;
1137 case MVT::i16:
1138 Opcode = NVPTX::LDV_i16_v2_areg_64;
1139 break;
1140 case MVT::i32:
1141 Opcode = NVPTX::LDV_i32_v2_areg_64;
1142 break;
1143 case MVT::i64:
1144 Opcode = NVPTX::LDV_i64_v2_areg_64;
1145 break;
1146 case MVT::f32:
1147 Opcode = NVPTX::LDV_f32_v2_areg_64;
1148 break;
1149 case MVT::f64:
1150 Opcode = NVPTX::LDV_f64_v2_areg_64;
1151 break;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]1152 }
1153 break;
1154 case NVPTXISD::LoadV4:
1155 switch (EltVT.getSimpleVT().SimpleTy) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1156 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]1157 return nullptr;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1158 case MVT::i8:
1159 Opcode = NVPTX::LDV_i8_v4_areg_64;
1160 break;
1161 case MVT::i16:
1162 Opcode = NVPTX::LDV_i16_v4_areg_64;
1163 break;
1164 case MVT::i32:
1165 Opcode = NVPTX::LDV_i32_v4_areg_64;
1166 break;
1167 case MVT::f32:
1168 Opcode = NVPTX::LDV_f32_v4_areg_64;
1169 break;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]1170 }
1171 break;
1172 }
1173 } else {
1174 switch (N->getOpcode()) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1175 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]1176 return nullptr;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]1177 case NVPTXISD::LoadV2:
1178 switch (EltVT.getSimpleVT().SimpleTy) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1179 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]1180 return nullptr;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1181 case MVT::i8:
1182 Opcode = NVPTX::LDV_i8_v2_areg;
1183 break;
1184 case MVT::i16:
1185 Opcode = NVPTX::LDV_i16_v2_areg;
1186 break;
1187 case MVT::i32:
1188 Opcode = NVPTX::LDV_i32_v2_areg;
1189 break;
1190 case MVT::i64:
1191 Opcode = NVPTX::LDV_i64_v2_areg;
1192 break;
1193 case MVT::f32:
1194 Opcode = NVPTX::LDV_f32_v2_areg;
1195 break;
1196 case MVT::f64:
1197 Opcode = NVPTX::LDV_f64_v2_areg;
1198 break;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]1199 }
1200 break;
1201 case NVPTXISD::LoadV4:
1202 switch (EltVT.getSimpleVT().SimpleTy) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1203 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]1204 return nullptr;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1205 case MVT::i8:
1206 Opcode = NVPTX::LDV_i8_v4_areg;
1207 break;
1208 case MVT::i16:
1209 Opcode = NVPTX::LDV_i16_v4_areg;
1210 break;
1211 case MVT::i32:
1212 Opcode = NVPTX::LDV_i32_v4_areg;
1213 break;
1214 case MVT::f32:
1215 Opcode = NVPTX::LDV_f32_v4_areg;
1216 break;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]1217 }
1218 break;
1219 }
1220 }
1221
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +0000[diff] [blame]1222 SDValue Ops[] = { getI32Imm(IsVolatile, DL), getI32Imm(CodeAddrSpace, DL),
1223 getI32Imm(VecType, DL), getI32Imm(FromType, DL),
1224 getI32Imm(FromTypeWidth, DL), Op1, Chain };
Michael Liaob53d8962013-04-19 22:22:57 +0000[diff] [blame]1225 LD = CurDAG->getMachineNode(Opcode, DL, N->getVTList(), Ops);
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]1226 }
1227
1228 MachineSDNode::mmo_iterator MemRefs0 = MF->allocateMemRefsArray(1);
1229 MemRefs0[0] = cast<MemSDNode>(N)->getMemOperand();
1230 cast<MachineSDNode>(LD)->setMemRefs(MemRefs0, MemRefs0 + 1);
1231
1232 return LD;
1233}
1234
Justin Holewinskib926d9d2014-06-27 18:35:51 +0000[diff] [blame]1235SDNode *NVPTXDAGToDAGISel::SelectLDGLDU(SDNode *N) {
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]1236
1237 SDValue Chain = N->getOperand(0);
Justin Holewinskib926d9d2014-06-27 18:35:51 +0000[diff] [blame]1238 SDValue Op1;
1239 MemSDNode *Mem;
1240 bool IsLDG = true;
1241
1242 // If this is an LDG intrinsic, the address is the third operand. Its its an
1243 // LDG/LDU SD node (from custom vector handling), then its the second operand
1244 if (N->getOpcode() == ISD::INTRINSIC_W_CHAIN) {
1245 Op1 = N->getOperand(2);
1246 Mem = cast<MemIntrinsicSDNode>(N);
1247 unsigned IID = cast<ConstantSDNode>(N->getOperand(1))->getZExtValue();
1248 switch (IID) {
1249 default:
1250 return NULL;
1251 case Intrinsic::nvvm_ldg_global_f:
1252 case Intrinsic::nvvm_ldg_global_i:
1253 case Intrinsic::nvvm_ldg_global_p:
1254 IsLDG = true;
1255 break;
1256 case Intrinsic::nvvm_ldu_global_f:
1257 case Intrinsic::nvvm_ldu_global_i:
1258 case Intrinsic::nvvm_ldu_global_p:
1259 IsLDG = false;
1260 break;
1261 }
1262 } else {
1263 Op1 = N->getOperand(1);
1264 Mem = cast<MemSDNode>(N);
1265 }
1266
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]1267 unsigned Opcode;
Andrew Trickef9de2a2013-05-25 02:42:55 +0000[diff] [blame]1268 SDLoc DL(N);
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]1269 SDNode *LD;
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1270 SDValue Base, Offset, Addr;
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]1271
Justin Holewinskib926d9d2014-06-27 18:35:51 +0000[diff] [blame]1272 EVT EltVT = Mem->getMemoryVT();
1273 if (EltVT.isVector()) {
1274 EltVT = EltVT.getVectorElementType();
1275 }
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]1276
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1277 if (SelectDirectAddr(Op1, Addr)) {
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]1278 switch (N->getOpcode()) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1279 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]1280 return nullptr;
Justin Holewinskib926d9d2014-06-27 18:35:51 +0000[diff] [blame]1281 case ISD::INTRINSIC_W_CHAIN:
1282 if (IsLDG) {
1283 switch (EltVT.getSimpleVT().SimpleTy) {
1284 default:
1285 return nullptr;
1286 case MVT::i8:
1287 Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i8avar;
1288 break;
1289 case MVT::i16:
1290 Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i16avar;
1291 break;
1292 case MVT::i32:
1293 Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i32avar;
1294 break;
1295 case MVT::i64:
1296 Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i64avar;
1297 break;
1298 case MVT::f32:
1299 Opcode = NVPTX::INT_PTX_LDG_GLOBAL_f32avar;
1300 break;
1301 case MVT::f64:
1302 Opcode = NVPTX::INT_PTX_LDG_GLOBAL_f64avar;
1303 break;
1304 }
1305 } else {
1306 switch (EltVT.getSimpleVT().SimpleTy) {
1307 default:
1308 return nullptr;
1309 case MVT::i8:
1310 Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i8avar;
1311 break;
1312 case MVT::i16:
1313 Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i16avar;
1314 break;
1315 case MVT::i32:
1316 Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i32avar;
1317 break;
1318 case MVT::i64:
1319 Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i64avar;
1320 break;
1321 case MVT::f32:
1322 Opcode = NVPTX::INT_PTX_LDU_GLOBAL_f32avar;
1323 break;
1324 case MVT::f64:
1325 Opcode = NVPTX::INT_PTX_LDU_GLOBAL_f64avar;
1326 break;
1327 }
1328 }
1329 break;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]1330 case NVPTXISD::LDGV2:
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1331 switch (EltVT.getSimpleVT().SimpleTy) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1332 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]1333 return nullptr;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1334 case MVT::i8:
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1335 Opcode = NVPTX::INT_PTX_LDG_G_v2i8_ELE_avar;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1336 break;
1337 case MVT::i16:
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1338 Opcode = NVPTX::INT_PTX_LDG_G_v2i16_ELE_avar;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1339 break;
1340 case MVT::i32:
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1341 Opcode = NVPTX::INT_PTX_LDG_G_v2i32_ELE_avar;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1342 break;
1343 case MVT::i64:
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1344 Opcode = NVPTX::INT_PTX_LDG_G_v2i64_ELE_avar;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1345 break;
1346 case MVT::f32:
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1347 Opcode = NVPTX::INT_PTX_LDG_G_v2f32_ELE_avar;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1348 break;
1349 case MVT::f64:
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1350 Opcode = NVPTX::INT_PTX_LDG_G_v2f64_ELE_avar;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1351 break;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]1352 }
1353 break;
1354 case NVPTXISD::LDUV2:
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1355 switch (EltVT.getSimpleVT().SimpleTy) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1356 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]1357 return nullptr;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1358 case MVT::i8:
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1359 Opcode = NVPTX::INT_PTX_LDU_G_v2i8_ELE_avar;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1360 break;
1361 case MVT::i16:
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1362 Opcode = NVPTX::INT_PTX_LDU_G_v2i16_ELE_avar;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1363 break;
1364 case MVT::i32:
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1365 Opcode = NVPTX::INT_PTX_LDU_G_v2i32_ELE_avar;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1366 break;
1367 case MVT::i64:
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1368 Opcode = NVPTX::INT_PTX_LDU_G_v2i64_ELE_avar;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1369 break;
1370 case MVT::f32:
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1371 Opcode = NVPTX::INT_PTX_LDU_G_v2f32_ELE_avar;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1372 break;
1373 case MVT::f64:
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1374 Opcode = NVPTX::INT_PTX_LDU_G_v2f64_ELE_avar;
1375 break;
1376 }
1377 break;
1378 case NVPTXISD::LDGV4:
1379 switch (EltVT.getSimpleVT().SimpleTy) {
1380 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]1381 return nullptr;
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1382 case MVT::i8:
1383 Opcode = NVPTX::INT_PTX_LDG_G_v4i8_ELE_avar;
1384 break;
1385 case MVT::i16:
1386 Opcode = NVPTX::INT_PTX_LDG_G_v4i16_ELE_avar;
1387 break;
1388 case MVT::i32:
1389 Opcode = NVPTX::INT_PTX_LDG_G_v4i32_ELE_avar;
1390 break;
1391 case MVT::f32:
1392 Opcode = NVPTX::INT_PTX_LDG_G_v4f32_ELE_avar;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1393 break;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]1394 }
1395 break;
1396 case NVPTXISD::LDUV4:
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1397 switch (EltVT.getSimpleVT().SimpleTy) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1398 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]1399 return nullptr;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1400 case MVT::i8:
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1401 Opcode = NVPTX::INT_PTX_LDU_G_v4i8_ELE_avar;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1402 break;
1403 case MVT::i16:
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1404 Opcode = NVPTX::INT_PTX_LDU_G_v4i16_ELE_avar;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1405 break;
1406 case MVT::i32:
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1407 Opcode = NVPTX::INT_PTX_LDU_G_v4i32_ELE_avar;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1408 break;
1409 case MVT::f32:
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1410 Opcode = NVPTX::INT_PTX_LDU_G_v4f32_ELE_avar;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1411 break;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]1412 }
1413 break;
1414 }
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1415
1416 SDValue Ops[] = { Addr, Chain };
Craig Topper2d2aa0c2014-04-30 07:17:30 +0000[diff] [blame]1417 LD = CurDAG->getMachineNode(Opcode, DL, N->getVTList(), Ops);
Eric Christopher02389e32015-02-19 00:08:27 +0000[diff] [blame]1418 } else if (TM.is64Bit() ? SelectADDRri64(Op1.getNode(), Op1, Base, Offset)
1419 : SelectADDRri(Op1.getNode(), Op1, Base, Offset)) {
1420 if (TM.is64Bit()) {
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1421 switch (N->getOpcode()) {
1422 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]1423 return nullptr;
Justin Holewinskib926d9d2014-06-27 18:35:51 +0000[diff] [blame]1424 case ISD::INTRINSIC_W_CHAIN:
1425 if (IsLDG) {
1426 switch (EltVT.getSimpleVT().SimpleTy) {
1427 default:
1428 return nullptr;
1429 case MVT::i8:
1430 Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i8ari64;
1431 break;
1432 case MVT::i16:
1433 Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i16ari64;
1434 break;
1435 case MVT::i32:
1436 Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i32ari64;
1437 break;
1438 case MVT::i64:
1439 Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i64ari64;
1440 break;
1441 case MVT::f32:
1442 Opcode = NVPTX::INT_PTX_LDG_GLOBAL_f32ari64;
1443 break;
1444 case MVT::f64:
1445 Opcode = NVPTX::INT_PTX_LDG_GLOBAL_f64ari64;
1446 break;
1447 }
1448 } else {
1449 switch (EltVT.getSimpleVT().SimpleTy) {
1450 default:
1451 return nullptr;
1452 case MVT::i8:
1453 Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i8ari64;
1454 break;
1455 case MVT::i16:
1456 Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i16ari64;
1457 break;
1458 case MVT::i32:
1459 Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i32ari64;
1460 break;
1461 case MVT::i64:
1462 Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i64ari64;
1463 break;
1464 case MVT::f32:
1465 Opcode = NVPTX::INT_PTX_LDU_GLOBAL_f32ari64;
1466 break;
1467 case MVT::f64:
1468 Opcode = NVPTX::INT_PTX_LDU_GLOBAL_f64ari64;
1469 break;
1470 }
1471 }
1472 break;
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1473 case NVPTXISD::LDGV2:
1474 switch (EltVT.getSimpleVT().SimpleTy) {
1475 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]1476 return nullptr;
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1477 case MVT::i8:
1478 Opcode = NVPTX::INT_PTX_LDG_G_v2i8_ELE_ari64;
1479 break;
1480 case MVT::i16:
1481 Opcode = NVPTX::INT_PTX_LDG_G_v2i16_ELE_ari64;
1482 break;
1483 case MVT::i32:
1484 Opcode = NVPTX::INT_PTX_LDG_G_v2i32_ELE_ari64;
1485 break;
1486 case MVT::i64:
1487 Opcode = NVPTX::INT_PTX_LDG_G_v2i64_ELE_ari64;
1488 break;
1489 case MVT::f32:
1490 Opcode = NVPTX::INT_PTX_LDG_G_v2f32_ELE_ari64;
1491 break;
1492 case MVT::f64:
1493 Opcode = NVPTX::INT_PTX_LDG_G_v2f64_ELE_ari64;
1494 break;
1495 }
1496 break;
1497 case NVPTXISD::LDUV2:
1498 switch (EltVT.getSimpleVT().SimpleTy) {
1499 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]1500 return nullptr;
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1501 case MVT::i8:
1502 Opcode = NVPTX::INT_PTX_LDU_G_v2i8_ELE_ari64;
1503 break;
1504 case MVT::i16:
1505 Opcode = NVPTX::INT_PTX_LDU_G_v2i16_ELE_ari64;
1506 break;
1507 case MVT::i32:
1508 Opcode = NVPTX::INT_PTX_LDU_G_v2i32_ELE_ari64;
1509 break;
1510 case MVT::i64:
1511 Opcode = NVPTX::INT_PTX_LDU_G_v2i64_ELE_ari64;
1512 break;
1513 case MVT::f32:
1514 Opcode = NVPTX::INT_PTX_LDU_G_v2f32_ELE_ari64;
1515 break;
1516 case MVT::f64:
1517 Opcode = NVPTX::INT_PTX_LDU_G_v2f64_ELE_ari64;
1518 break;
1519 }
1520 break;
1521 case NVPTXISD::LDGV4:
1522 switch (EltVT.getSimpleVT().SimpleTy) {
1523 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]1524 return nullptr;
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1525 case MVT::i8:
1526 Opcode = NVPTX::INT_PTX_LDG_G_v4i8_ELE_ari64;
1527 break;
1528 case MVT::i16:
1529 Opcode = NVPTX::INT_PTX_LDG_G_v4i16_ELE_ari64;
1530 break;
1531 case MVT::i32:
1532 Opcode = NVPTX::INT_PTX_LDG_G_v4i32_ELE_ari64;
1533 break;
1534 case MVT::f32:
1535 Opcode = NVPTX::INT_PTX_LDG_G_v4f32_ELE_ari64;
1536 break;
1537 }
1538 break;
1539 case NVPTXISD::LDUV4:
1540 switch (EltVT.getSimpleVT().SimpleTy) {
1541 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]1542 return nullptr;
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1543 case MVT::i8:
1544 Opcode = NVPTX::INT_PTX_LDU_G_v4i8_ELE_ari64;
1545 break;
1546 case MVT::i16:
1547 Opcode = NVPTX::INT_PTX_LDU_G_v4i16_ELE_ari64;
1548 break;
1549 case MVT::i32:
1550 Opcode = NVPTX::INT_PTX_LDU_G_v4i32_ELE_ari64;
1551 break;
1552 case MVT::f32:
1553 Opcode = NVPTX::INT_PTX_LDU_G_v4f32_ELE_ari64;
1554 break;
1555 }
1556 break;
1557 }
1558 } else {
1559 switch (N->getOpcode()) {
1560 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]1561 return nullptr;
Justin Holewinskib926d9d2014-06-27 18:35:51 +0000[diff] [blame]1562 case ISD::INTRINSIC_W_CHAIN:
1563 if (IsLDG) {
1564 switch (EltVT.getSimpleVT().SimpleTy) {
1565 default:
1566 return nullptr;
1567 case MVT::i8:
1568 Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i8ari;
1569 break;
1570 case MVT::i16:
1571 Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i16ari;
1572 break;
1573 case MVT::i32:
1574 Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i32ari;
1575 break;
1576 case MVT::i64:
1577 Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i64ari;
1578 break;
1579 case MVT::f32:
1580 Opcode = NVPTX::INT_PTX_LDG_GLOBAL_f32ari;
1581 break;
1582 case MVT::f64:
1583 Opcode = NVPTX::INT_PTX_LDG_GLOBAL_f64ari;
1584 break;
1585 }
1586 } else {
1587 switch (EltVT.getSimpleVT().SimpleTy) {
1588 default:
1589 return nullptr;
1590 case MVT::i8:
1591 Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i8ari;
1592 break;
1593 case MVT::i16:
1594 Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i16ari;
1595 break;
1596 case MVT::i32:
1597 Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i32ari;
1598 break;
1599 case MVT::i64:
1600 Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i64ari;
1601 break;
1602 case MVT::f32:
1603 Opcode = NVPTX::INT_PTX_LDU_GLOBAL_f32ari;
1604 break;
1605 case MVT::f64:
1606 Opcode = NVPTX::INT_PTX_LDU_GLOBAL_f64ari;
1607 break;
1608 }
1609 }
1610 break;
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1611 case NVPTXISD::LDGV2:
1612 switch (EltVT.getSimpleVT().SimpleTy) {
1613 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]1614 return nullptr;
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1615 case MVT::i8:
1616 Opcode = NVPTX::INT_PTX_LDG_G_v2i8_ELE_ari32;
1617 break;
1618 case MVT::i16:
1619 Opcode = NVPTX::INT_PTX_LDG_G_v2i16_ELE_ari32;
1620 break;
1621 case MVT::i32:
1622 Opcode = NVPTX::INT_PTX_LDG_G_v2i32_ELE_ari32;
1623 break;
1624 case MVT::i64:
1625 Opcode = NVPTX::INT_PTX_LDG_G_v2i64_ELE_ari32;
1626 break;
1627 case MVT::f32:
1628 Opcode = NVPTX::INT_PTX_LDG_G_v2f32_ELE_ari32;
1629 break;
1630 case MVT::f64:
1631 Opcode = NVPTX::INT_PTX_LDG_G_v2f64_ELE_ari32;
1632 break;
1633 }
1634 break;
1635 case NVPTXISD::LDUV2:
1636 switch (EltVT.getSimpleVT().SimpleTy) {
1637 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]1638 return nullptr;
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1639 case MVT::i8:
1640 Opcode = NVPTX::INT_PTX_LDU_G_v2i8_ELE_ari32;
1641 break;
1642 case MVT::i16:
1643 Opcode = NVPTX::INT_PTX_LDU_G_v2i16_ELE_ari32;
1644 break;
1645 case MVT::i32:
1646 Opcode = NVPTX::INT_PTX_LDU_G_v2i32_ELE_ari32;
1647 break;
1648 case MVT::i64:
1649 Opcode = NVPTX::INT_PTX_LDU_G_v2i64_ELE_ari32;
1650 break;
1651 case MVT::f32:
1652 Opcode = NVPTX::INT_PTX_LDU_G_v2f32_ELE_ari32;
1653 break;
1654 case MVT::f64:
1655 Opcode = NVPTX::INT_PTX_LDU_G_v2f64_ELE_ari32;
1656 break;
1657 }
1658 break;
1659 case NVPTXISD::LDGV4:
1660 switch (EltVT.getSimpleVT().SimpleTy) {
1661 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]1662 return nullptr;
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1663 case MVT::i8:
1664 Opcode = NVPTX::INT_PTX_LDG_G_v4i8_ELE_ari32;
1665 break;
1666 case MVT::i16:
1667 Opcode = NVPTX::INT_PTX_LDG_G_v4i16_ELE_ari32;
1668 break;
1669 case MVT::i32:
1670 Opcode = NVPTX::INT_PTX_LDG_G_v4i32_ELE_ari32;
1671 break;
1672 case MVT::f32:
1673 Opcode = NVPTX::INT_PTX_LDG_G_v4f32_ELE_ari32;
1674 break;
1675 }
1676 break;
1677 case NVPTXISD::LDUV4:
1678 switch (EltVT.getSimpleVT().SimpleTy) {
1679 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]1680 return nullptr;
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1681 case MVT::i8:
1682 Opcode = NVPTX::INT_PTX_LDU_G_v4i8_ELE_ari32;
1683 break;
1684 case MVT::i16:
1685 Opcode = NVPTX::INT_PTX_LDU_G_v4i16_ELE_ari32;
1686 break;
1687 case MVT::i32:
1688 Opcode = NVPTX::INT_PTX_LDU_G_v4i32_ELE_ari32;
1689 break;
1690 case MVT::f32:
1691 Opcode = NVPTX::INT_PTX_LDU_G_v4f32_ELE_ari32;
1692 break;
1693 }
1694 break;
1695 }
1696 }
1697
1698 SDValue Ops[] = { Base, Offset, Chain };
1699
Craig Topper2d2aa0c2014-04-30 07:17:30 +0000[diff] [blame]1700 LD = CurDAG->getMachineNode(Opcode, DL, N->getVTList(), Ops);
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]1701 } else {
Eric Christopher02389e32015-02-19 00:08:27 +0000[diff] [blame]1702 if (TM.is64Bit()) {
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1703 switch (N->getOpcode()) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1704 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]1705 return nullptr;
Justin Holewinskib926d9d2014-06-27 18:35:51 +0000[diff] [blame]1706 case ISD::INTRINSIC_W_CHAIN:
1707 if (IsLDG) {
1708 switch (EltVT.getSimpleVT().SimpleTy) {
1709 default:
1710 return nullptr;
1711 case MVT::i8:
1712 Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i8areg64;
1713 break;
1714 case MVT::i16:
1715 Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i16areg64;
1716 break;
1717 case MVT::i32:
1718 Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i32areg64;
1719 break;
1720 case MVT::i64:
1721 Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i64areg64;
1722 break;
1723 case MVT::f32:
1724 Opcode = NVPTX::INT_PTX_LDG_GLOBAL_f32areg64;
1725 break;
1726 case MVT::f64:
1727 Opcode = NVPTX::INT_PTX_LDG_GLOBAL_f64areg64;
1728 break;
1729 }
1730 } else {
1731 switch (EltVT.getSimpleVT().SimpleTy) {
1732 default:
1733 return nullptr;
1734 case MVT::i8:
1735 Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i8areg64;
1736 break;
1737 case MVT::i16:
1738 Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i16areg64;
1739 break;
1740 case MVT::i32:
1741 Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i32areg64;
1742 break;
1743 case MVT::i64:
1744 Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i64areg64;
1745 break;
1746 case MVT::f32:
1747 Opcode = NVPTX::INT_PTX_LDU_GLOBAL_f32areg64;
1748 break;
1749 case MVT::f64:
1750 Opcode = NVPTX::INT_PTX_LDU_GLOBAL_f64areg64;
1751 break;
1752 }
1753 }
1754 break;
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1755 case NVPTXISD::LDGV2:
1756 switch (EltVT.getSimpleVT().SimpleTy) {
1757 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]1758 return nullptr;
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1759 case MVT::i8:
1760 Opcode = NVPTX::INT_PTX_LDG_G_v2i8_ELE_areg64;
1761 break;
1762 case MVT::i16:
1763 Opcode = NVPTX::INT_PTX_LDG_G_v2i16_ELE_areg64;
1764 break;
1765 case MVT::i32:
1766 Opcode = NVPTX::INT_PTX_LDG_G_v2i32_ELE_areg64;
1767 break;
1768 case MVT::i64:
1769 Opcode = NVPTX::INT_PTX_LDG_G_v2i64_ELE_areg64;
1770 break;
1771 case MVT::f32:
1772 Opcode = NVPTX::INT_PTX_LDG_G_v2f32_ELE_areg64;
1773 break;
1774 case MVT::f64:
1775 Opcode = NVPTX::INT_PTX_LDG_G_v2f64_ELE_areg64;
1776 break;
1777 }
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1778 break;
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1779 case NVPTXISD::LDUV2:
1780 switch (EltVT.getSimpleVT().SimpleTy) {
1781 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]1782 return nullptr;
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1783 case MVT::i8:
1784 Opcode = NVPTX::INT_PTX_LDU_G_v2i8_ELE_areg64;
1785 break;
1786 case MVT::i16:
1787 Opcode = NVPTX::INT_PTX_LDU_G_v2i16_ELE_areg64;
1788 break;
1789 case MVT::i32:
1790 Opcode = NVPTX::INT_PTX_LDU_G_v2i32_ELE_areg64;
1791 break;
1792 case MVT::i64:
1793 Opcode = NVPTX::INT_PTX_LDU_G_v2i64_ELE_areg64;
1794 break;
1795 case MVT::f32:
1796 Opcode = NVPTX::INT_PTX_LDU_G_v2f32_ELE_areg64;
1797 break;
1798 case MVT::f64:
1799 Opcode = NVPTX::INT_PTX_LDU_G_v2f64_ELE_areg64;
1800 break;
1801 }
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1802 break;
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1803 case NVPTXISD::LDGV4:
1804 switch (EltVT.getSimpleVT().SimpleTy) {
1805 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]1806 return nullptr;
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1807 case MVT::i8:
1808 Opcode = NVPTX::INT_PTX_LDG_G_v4i8_ELE_areg64;
1809 break;
1810 case MVT::i16:
1811 Opcode = NVPTX::INT_PTX_LDG_G_v4i16_ELE_areg64;
1812 break;
1813 case MVT::i32:
1814 Opcode = NVPTX::INT_PTX_LDG_G_v4i32_ELE_areg64;
1815 break;
1816 case MVT::f32:
1817 Opcode = NVPTX::INT_PTX_LDG_G_v4f32_ELE_areg64;
1818 break;
1819 }
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1820 break;
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1821 case NVPTXISD::LDUV4:
1822 switch (EltVT.getSimpleVT().SimpleTy) {
1823 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]1824 return nullptr;
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1825 case MVT::i8:
1826 Opcode = NVPTX::INT_PTX_LDU_G_v4i8_ELE_areg64;
1827 break;
1828 case MVT::i16:
1829 Opcode = NVPTX::INT_PTX_LDU_G_v4i16_ELE_areg64;
1830 break;
1831 case MVT::i32:
1832 Opcode = NVPTX::INT_PTX_LDU_G_v4i32_ELE_areg64;
1833 break;
1834 case MVT::f32:
1835 Opcode = NVPTX::INT_PTX_LDU_G_v4f32_ELE_areg64;
1836 break;
1837 }
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1838 break;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]1839 }
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1840 } else {
1841 switch (N->getOpcode()) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1842 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]1843 return nullptr;
Justin Holewinskib926d9d2014-06-27 18:35:51 +0000[diff] [blame]1844 case ISD::INTRINSIC_W_CHAIN:
1845 if (IsLDG) {
1846 switch (EltVT.getSimpleVT().SimpleTy) {
1847 default:
1848 return nullptr;
1849 case MVT::i8:
1850 Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i8areg;
1851 break;
1852 case MVT::i16:
1853 Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i16areg;
1854 break;
1855 case MVT::i32:
1856 Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i32areg;
1857 break;
1858 case MVT::i64:
1859 Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i64areg;
1860 break;
1861 case MVT::f32:
1862 Opcode = NVPTX::INT_PTX_LDG_GLOBAL_f32areg;
1863 break;
1864 case MVT::f64:
1865 Opcode = NVPTX::INT_PTX_LDG_GLOBAL_f64areg;
1866 break;
1867 }
1868 } else {
1869 switch (EltVT.getSimpleVT().SimpleTy) {
1870 default:
1871 return nullptr;
1872 case MVT::i8:
1873 Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i8areg;
1874 break;
1875 case MVT::i16:
1876 Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i16areg;
1877 break;
1878 case MVT::i32:
1879 Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i32areg;
1880 break;
1881 case MVT::i64:
1882 Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i64areg;
1883 break;
1884 case MVT::f32:
1885 Opcode = NVPTX::INT_PTX_LDU_GLOBAL_f32areg;
1886 break;
1887 case MVT::f64:
1888 Opcode = NVPTX::INT_PTX_LDU_GLOBAL_f64areg;
1889 break;
1890 }
1891 }
1892 break;
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1893 case NVPTXISD::LDGV2:
1894 switch (EltVT.getSimpleVT().SimpleTy) {
1895 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]1896 return nullptr;
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1897 case MVT::i8:
1898 Opcode = NVPTX::INT_PTX_LDG_G_v2i8_ELE_areg32;
1899 break;
1900 case MVT::i16:
1901 Opcode = NVPTX::INT_PTX_LDG_G_v2i16_ELE_areg32;
1902 break;
1903 case MVT::i32:
1904 Opcode = NVPTX::INT_PTX_LDG_G_v2i32_ELE_areg32;
1905 break;
1906 case MVT::i64:
1907 Opcode = NVPTX::INT_PTX_LDG_G_v2i64_ELE_areg32;
1908 break;
1909 case MVT::f32:
1910 Opcode = NVPTX::INT_PTX_LDG_G_v2f32_ELE_areg32;
1911 break;
1912 case MVT::f64:
1913 Opcode = NVPTX::INT_PTX_LDG_G_v2f64_ELE_areg32;
1914 break;
1915 }
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1916 break;
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1917 case NVPTXISD::LDUV2:
1918 switch (EltVT.getSimpleVT().SimpleTy) {
1919 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]1920 return nullptr;
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1921 case MVT::i8:
1922 Opcode = NVPTX::INT_PTX_LDU_G_v2i8_ELE_areg32;
1923 break;
1924 case MVT::i16:
1925 Opcode = NVPTX::INT_PTX_LDU_G_v2i16_ELE_areg32;
1926 break;
1927 case MVT::i32:
1928 Opcode = NVPTX::INT_PTX_LDU_G_v2i32_ELE_areg32;
1929 break;
1930 case MVT::i64:
1931 Opcode = NVPTX::INT_PTX_LDU_G_v2i64_ELE_areg32;
1932 break;
1933 case MVT::f32:
1934 Opcode = NVPTX::INT_PTX_LDU_G_v2f32_ELE_areg32;
1935 break;
1936 case MVT::f64:
1937 Opcode = NVPTX::INT_PTX_LDU_G_v2f64_ELE_areg32;
1938 break;
1939 }
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1940 break;
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1941 case NVPTXISD::LDGV4:
1942 switch (EltVT.getSimpleVT().SimpleTy) {
1943 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]1944 return nullptr;
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1945 case MVT::i8:
1946 Opcode = NVPTX::INT_PTX_LDG_G_v4i8_ELE_areg32;
1947 break;
1948 case MVT::i16:
1949 Opcode = NVPTX::INT_PTX_LDG_G_v4i16_ELE_areg32;
1950 break;
1951 case MVT::i32:
1952 Opcode = NVPTX::INT_PTX_LDG_G_v4i32_ELE_areg32;
1953 break;
1954 case MVT::f32:
1955 Opcode = NVPTX::INT_PTX_LDG_G_v4f32_ELE_areg32;
1956 break;
1957 }
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1958 break;
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1959 case NVPTXISD::LDUV4:
1960 switch (EltVT.getSimpleVT().SimpleTy) {
1961 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]1962 return nullptr;
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1963 case MVT::i8:
1964 Opcode = NVPTX::INT_PTX_LDU_G_v4i8_ELE_areg32;
1965 break;
1966 case MVT::i16:
1967 Opcode = NVPTX::INT_PTX_LDU_G_v4i16_ELE_areg32;
1968 break;
1969 case MVT::i32:
1970 Opcode = NVPTX::INT_PTX_LDU_G_v4i32_ELE_areg32;
1971 break;
1972 case MVT::f32:
1973 Opcode = NVPTX::INT_PTX_LDU_G_v4f32_ELE_areg32;
1974 break;
1975 }
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1976 break;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]1977 }
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]1978 }
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]1979
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1980 SDValue Ops[] = { Op1, Chain };
Craig Topper2d2aa0c2014-04-30 07:17:30 +0000[diff] [blame]1981 LD = CurDAG->getMachineNode(Opcode, DL, N->getVTList(), Ops);
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1982 }
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]1983
1984 MachineSDNode::mmo_iterator MemRefs0 = MF->allocateMemRefsArray(1);
Justin Holewinskib926d9d2014-06-27 18:35:51 +0000[diff] [blame]1985 MemRefs0[0] = Mem->getMemOperand();
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]1986 cast<MachineSDNode>(LD)->setMemRefs(MemRefs0, MemRefs0 + 1);
1987
1988 return LD;
1989}
1990
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1991SDNode *NVPTXDAGToDAGISel::SelectStore(SDNode *N) {
Andrew Trickef9de2a2013-05-25 02:42:55 +0000[diff] [blame]1992 SDLoc dl(N);
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]1993 StoreSDNode *ST = cast<StoreSDNode>(N);
1994 EVT StoreVT = ST->getMemoryVT();
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]1995 SDNode *NVPTXST = nullptr;
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]1996
1997 // do not support pre/post inc/dec
1998 if (ST->isIndexed())
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]1999 return nullptr;
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]2000
2001 if (!StoreVT.isSimple())
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2002 return nullptr;
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]2003
2004 // Address Space Setting
Eric Christopher9745b3a2015-01-30 01:41:01 +0000[diff] [blame]2005 unsigned int codeAddrSpace = getCodeAddrSpace(ST);
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]2006
2007 // Volatile Setting
2008 // - .volatile is only availalble for .global and .shared
2009 bool isVolatile = ST->isVolatile();
2010 if (codeAddrSpace != NVPTX::PTXLdStInstCode::GLOBAL &&
2011 codeAddrSpace != NVPTX::PTXLdStInstCode::SHARED &&
2012 codeAddrSpace != NVPTX::PTXLdStInstCode::GENERIC)
2013 isVolatile = false;
2014
2015 // Vector Setting
2016 MVT SimpleVT = StoreVT.getSimpleVT();
2017 unsigned vecType = NVPTX::PTXLdStInstCode::Scalar;
2018 if (SimpleVT.isVector()) {
2019 unsigned num = SimpleVT.getVectorNumElements();
2020 if (num == 2)
2021 vecType = NVPTX::PTXLdStInstCode::V2;
2022 else if (num == 4)
2023 vecType = NVPTX::PTXLdStInstCode::V4;
2024 else
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2025 return nullptr;
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]2026 }
2027
2028 // Type Setting: toType + toTypeWidth
2029 // - for integer type, always use 'u'
2030 //
2031 MVT ScalarVT = SimpleVT.getScalarType();
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2032 unsigned toTypeWidth = ScalarVT.getSizeInBits();
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]2033 unsigned int toType;
2034 if (ScalarVT.isFloatingPoint())
2035 toType = NVPTX::PTXLdStInstCode::Float;
2036 else
2037 toType = NVPTX::PTXLdStInstCode::Unsigned;
2038
2039 // Create the machine instruction DAG
2040 SDValue Chain = N->getOperand(0);
2041 SDValue N1 = N->getOperand(1);
2042 SDValue N2 = N->getOperand(2);
2043 SDValue Addr;
2044 SDValue Offset, Base;
2045 unsigned Opcode;
Craig Topperd9c27832013-08-15 02:44:19 +0000[diff] [blame]2046 MVT::SimpleValueType SourceVT = N1.getNode()->getSimpleValueType(0).SimpleTy;
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]2047
2048 if (SelectDirectAddr(N2, Addr)) {
2049 switch (SourceVT) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2050 case MVT::i8:
2051 Opcode = NVPTX::ST_i8_avar;
2052 break;
2053 case MVT::i16:
2054 Opcode = NVPTX::ST_i16_avar;
2055 break;
2056 case MVT::i32:
2057 Opcode = NVPTX::ST_i32_avar;
2058 break;
2059 case MVT::i64:
2060 Opcode = NVPTX::ST_i64_avar;
2061 break;
2062 case MVT::f32:
2063 Opcode = NVPTX::ST_f32_avar;
2064 break;
2065 case MVT::f64:
2066 Opcode = NVPTX::ST_f64_avar;
2067 break;
2068 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2069 return nullptr;
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]2070 }
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +0000[diff] [blame]2071 SDValue Ops[] = { N1, getI32Imm(isVolatile, dl),
2072 getI32Imm(codeAddrSpace, dl), getI32Imm(vecType, dl),
2073 getI32Imm(toType, dl), getI32Imm(toTypeWidth, dl), Addr,
2074 Chain };
Michael Liaob53d8962013-04-19 22:22:57 +0000[diff] [blame]2075 NVPTXST = CurDAG->getMachineNode(Opcode, dl, MVT::Other, Ops);
Eric Christopher02389e32015-02-19 00:08:27 +0000[diff] [blame]2076 } else if (TM.is64Bit() ? SelectADDRsi64(N2.getNode(), N2, Base, Offset)
2077 : SelectADDRsi(N2.getNode(), N2, Base, Offset)) {
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]2078 switch (SourceVT) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2079 case MVT::i8:
2080 Opcode = NVPTX::ST_i8_asi;
2081 break;
2082 case MVT::i16:
2083 Opcode = NVPTX::ST_i16_asi;
2084 break;
2085 case MVT::i32:
2086 Opcode = NVPTX::ST_i32_asi;
2087 break;
2088 case MVT::i64:
2089 Opcode = NVPTX::ST_i64_asi;
2090 break;
2091 case MVT::f32:
2092 Opcode = NVPTX::ST_f32_asi;
2093 break;
2094 case MVT::f64:
2095 Opcode = NVPTX::ST_f64_asi;
2096 break;
2097 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2098 return nullptr;
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]2099 }
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +0000[diff] [blame]2100 SDValue Ops[] = { N1, getI32Imm(isVolatile, dl),
2101 getI32Imm(codeAddrSpace, dl), getI32Imm(vecType, dl),
2102 getI32Imm(toType, dl), getI32Imm(toTypeWidth, dl), Base,
2103 Offset, Chain };
Michael Liaob53d8962013-04-19 22:22:57 +0000[diff] [blame]2104 NVPTXST = CurDAG->getMachineNode(Opcode, dl, MVT::Other, Ops);
Eric Christopher02389e32015-02-19 00:08:27 +0000[diff] [blame]2105 } else if (TM.is64Bit() ? SelectADDRri64(N2.getNode(), N2, Base, Offset)
2106 : SelectADDRri(N2.getNode(), N2, Base, Offset)) {
2107 if (TM.is64Bit()) {
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]2108 switch (SourceVT) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2109 case MVT::i8:
2110 Opcode = NVPTX::ST_i8_ari_64;
2111 break;
2112 case MVT::i16:
2113 Opcode = NVPTX::ST_i16_ari_64;
2114 break;
2115 case MVT::i32:
2116 Opcode = NVPTX::ST_i32_ari_64;
2117 break;
2118 case MVT::i64:
2119 Opcode = NVPTX::ST_i64_ari_64;
2120 break;
2121 case MVT::f32:
2122 Opcode = NVPTX::ST_f32_ari_64;
2123 break;
2124 case MVT::f64:
2125 Opcode = NVPTX::ST_f64_ari_64;
2126 break;
2127 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2128 return nullptr;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]2129 }
2130 } else {
2131 switch (SourceVT) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2132 case MVT::i8:
2133 Opcode = NVPTX::ST_i8_ari;
2134 break;
2135 case MVT::i16:
2136 Opcode = NVPTX::ST_i16_ari;
2137 break;
2138 case MVT::i32:
2139 Opcode = NVPTX::ST_i32_ari;
2140 break;
2141 case MVT::i64:
2142 Opcode = NVPTX::ST_i64_ari;
2143 break;
2144 case MVT::f32:
2145 Opcode = NVPTX::ST_f32_ari;
2146 break;
2147 case MVT::f64:
2148 Opcode = NVPTX::ST_f64_ari;
2149 break;
2150 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2151 return nullptr;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]2152 }
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]2153 }
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +0000[diff] [blame]2154 SDValue Ops[] = { N1, getI32Imm(isVolatile, dl),
2155 getI32Imm(codeAddrSpace, dl), getI32Imm(vecType, dl),
2156 getI32Imm(toType, dl), getI32Imm(toTypeWidth, dl), Base,
2157 Offset, Chain };
Michael Liaob53d8962013-04-19 22:22:57 +0000[diff] [blame]2158 NVPTXST = CurDAG->getMachineNode(Opcode, dl, MVT::Other, Ops);
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]2159 } else {
Eric Christopher02389e32015-02-19 00:08:27 +0000[diff] [blame]2160 if (TM.is64Bit()) {
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]2161 switch (SourceVT) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2162 case MVT::i8:
2163 Opcode = NVPTX::ST_i8_areg_64;
2164 break;
2165 case MVT::i16:
2166 Opcode = NVPTX::ST_i16_areg_64;
2167 break;
2168 case MVT::i32:
2169 Opcode = NVPTX::ST_i32_areg_64;
2170 break;
2171 case MVT::i64:
2172 Opcode = NVPTX::ST_i64_areg_64;
2173 break;
2174 case MVT::f32:
2175 Opcode = NVPTX::ST_f32_areg_64;
2176 break;
2177 case MVT::f64:
2178 Opcode = NVPTX::ST_f64_areg_64;
2179 break;
2180 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2181 return nullptr;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]2182 }
2183 } else {
2184 switch (SourceVT) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2185 case MVT::i8:
2186 Opcode = NVPTX::ST_i8_areg;
2187 break;
2188 case MVT::i16:
2189 Opcode = NVPTX::ST_i16_areg;
2190 break;
2191 case MVT::i32:
2192 Opcode = NVPTX::ST_i32_areg;
2193 break;
2194 case MVT::i64:
2195 Opcode = NVPTX::ST_i64_areg;
2196 break;
2197 case MVT::f32:
2198 Opcode = NVPTX::ST_f32_areg;
2199 break;
2200 case MVT::f64:
2201 Opcode = NVPTX::ST_f64_areg;
2202 break;
2203 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2204 return nullptr;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]2205 }
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]2206 }
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +0000[diff] [blame]2207 SDValue Ops[] = { N1, getI32Imm(isVolatile, dl),
2208 getI32Imm(codeAddrSpace, dl), getI32Imm(vecType, dl),
2209 getI32Imm(toType, dl), getI32Imm(toTypeWidth, dl), N2,
2210 Chain };
Michael Liaob53d8962013-04-19 22:22:57 +0000[diff] [blame]2211 NVPTXST = CurDAG->getMachineNode(Opcode, dl, MVT::Other, Ops);
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]2212 }
2213
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2214 if (NVPTXST) {
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]2215 MachineSDNode::mmo_iterator MemRefs0 = MF->allocateMemRefsArray(1);
2216 MemRefs0[0] = cast<MemSDNode>(N)->getMemOperand();
2217 cast<MachineSDNode>(NVPTXST)->setMemRefs(MemRefs0, MemRefs0 + 1);
2218 }
2219
2220 return NVPTXST;
2221}
2222
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]2223SDNode *NVPTXDAGToDAGISel::SelectStoreVector(SDNode *N) {
2224 SDValue Chain = N->getOperand(0);
2225 SDValue Op1 = N->getOperand(1);
2226 SDValue Addr, Offset, Base;
2227 unsigned Opcode;
Andrew Trickef9de2a2013-05-25 02:42:55 +0000[diff] [blame]2228 SDLoc DL(N);
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]2229 SDNode *ST;
2230 EVT EltVT = Op1.getValueType();
2231 MemSDNode *MemSD = cast<MemSDNode>(N);
2232 EVT StoreVT = MemSD->getMemoryVT();
2233
2234 // Address Space Setting
Eric Christopher9745b3a2015-01-30 01:41:01 +0000[diff] [blame]2235 unsigned CodeAddrSpace = getCodeAddrSpace(MemSD);
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]2236
2237 if (CodeAddrSpace == NVPTX::PTXLdStInstCode::CONSTANT) {
2238 report_fatal_error("Cannot store to pointer that points to constant "
2239 "memory space");
2240 }
2241
2242 // Volatile Setting
2243 // - .volatile is only availalble for .global and .shared
2244 bool IsVolatile = MemSD->isVolatile();
2245 if (CodeAddrSpace != NVPTX::PTXLdStInstCode::GLOBAL &&
2246 CodeAddrSpace != NVPTX::PTXLdStInstCode::SHARED &&
2247 CodeAddrSpace != NVPTX::PTXLdStInstCode::GENERIC)
2248 IsVolatile = false;
2249
2250 // Type Setting: toType + toTypeWidth
2251 // - for integer type, always use 'u'
2252 assert(StoreVT.isSimple() && "Store value is not simple");
2253 MVT ScalarVT = StoreVT.getSimpleVT().getScalarType();
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2254 unsigned ToTypeWidth = ScalarVT.getSizeInBits();
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]2255 unsigned ToType;
2256 if (ScalarVT.isFloatingPoint())
2257 ToType = NVPTX::PTXLdStInstCode::Float;
2258 else
2259 ToType = NVPTX::PTXLdStInstCode::Unsigned;
2260
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]2261 SmallVector<SDValue, 12> StOps;
2262 SDValue N2;
2263 unsigned VecType;
2264
2265 switch (N->getOpcode()) {
2266 case NVPTXISD::StoreV2:
2267 VecType = NVPTX::PTXLdStInstCode::V2;
2268 StOps.push_back(N->getOperand(1));
2269 StOps.push_back(N->getOperand(2));
2270 N2 = N->getOperand(3);
2271 break;
2272 case NVPTXISD::StoreV4:
2273 VecType = NVPTX::PTXLdStInstCode::V4;
2274 StOps.push_back(N->getOperand(1));
2275 StOps.push_back(N->getOperand(2));
2276 StOps.push_back(N->getOperand(3));
2277 StOps.push_back(N->getOperand(4));
2278 N2 = N->getOperand(5);
2279 break;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2280 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2281 return nullptr;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]2282 }
2283
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +0000[diff] [blame]2284 StOps.push_back(getI32Imm(IsVolatile, DL));
2285 StOps.push_back(getI32Imm(CodeAddrSpace, DL));
2286 StOps.push_back(getI32Imm(VecType, DL));
2287 StOps.push_back(getI32Imm(ToType, DL));
2288 StOps.push_back(getI32Imm(ToTypeWidth, DL));
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]2289
2290 if (SelectDirectAddr(N2, Addr)) {
2291 switch (N->getOpcode()) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2292 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2293 return nullptr;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]2294 case NVPTXISD::StoreV2:
2295 switch (EltVT.getSimpleVT().SimpleTy) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2296 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2297 return nullptr;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2298 case MVT::i8:
2299 Opcode = NVPTX::STV_i8_v2_avar;
2300 break;
2301 case MVT::i16:
2302 Opcode = NVPTX::STV_i16_v2_avar;
2303 break;
2304 case MVT::i32:
2305 Opcode = NVPTX::STV_i32_v2_avar;
2306 break;
2307 case MVT::i64:
2308 Opcode = NVPTX::STV_i64_v2_avar;
2309 break;
2310 case MVT::f32:
2311 Opcode = NVPTX::STV_f32_v2_avar;
2312 break;
2313 case MVT::f64:
2314 Opcode = NVPTX::STV_f64_v2_avar;
2315 break;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]2316 }
2317 break;
2318 case NVPTXISD::StoreV4:
2319 switch (EltVT.getSimpleVT().SimpleTy) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2320 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2321 return nullptr;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2322 case MVT::i8:
2323 Opcode = NVPTX::STV_i8_v4_avar;
2324 break;
2325 case MVT::i16:
2326 Opcode = NVPTX::STV_i16_v4_avar;
2327 break;
2328 case MVT::i32:
2329 Opcode = NVPTX::STV_i32_v4_avar;
2330 break;
2331 case MVT::f32:
2332 Opcode = NVPTX::STV_f32_v4_avar;
2333 break;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]2334 }
2335 break;
2336 }
2337 StOps.push_back(Addr);
Eric Christopher02389e32015-02-19 00:08:27 +0000[diff] [blame]2338 } else if (TM.is64Bit() ? SelectADDRsi64(N2.getNode(), N2, Base, Offset)
2339 : SelectADDRsi(N2.getNode(), N2, Base, Offset)) {
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]2340 switch (N->getOpcode()) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2341 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2342 return nullptr;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]2343 case NVPTXISD::StoreV2:
2344 switch (EltVT.getSimpleVT().SimpleTy) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2345 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2346 return nullptr;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2347 case MVT::i8:
2348 Opcode = NVPTX::STV_i8_v2_asi;
2349 break;
2350 case MVT::i16:
2351 Opcode = NVPTX::STV_i16_v2_asi;
2352 break;
2353 case MVT::i32:
2354 Opcode = NVPTX::STV_i32_v2_asi;
2355 break;
2356 case MVT::i64:
2357 Opcode = NVPTX::STV_i64_v2_asi;
2358 break;
2359 case MVT::f32:
2360 Opcode = NVPTX::STV_f32_v2_asi;
2361 break;
2362 case MVT::f64:
2363 Opcode = NVPTX::STV_f64_v2_asi;
2364 break;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]2365 }
2366 break;
2367 case NVPTXISD::StoreV4:
2368 switch (EltVT.getSimpleVT().SimpleTy) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2369 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2370 return nullptr;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2371 case MVT::i8:
2372 Opcode = NVPTX::STV_i8_v4_asi;
2373 break;
2374 case MVT::i16:
2375 Opcode = NVPTX::STV_i16_v4_asi;
2376 break;
2377 case MVT::i32:
2378 Opcode = NVPTX::STV_i32_v4_asi;
2379 break;
2380 case MVT::f32:
2381 Opcode = NVPTX::STV_f32_v4_asi;
2382 break;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]2383 }
2384 break;
2385 }
2386 StOps.push_back(Base);
2387 StOps.push_back(Offset);
Eric Christopher02389e32015-02-19 00:08:27 +0000[diff] [blame]2388 } else if (TM.is64Bit() ? SelectADDRri64(N2.getNode(), N2, Base, Offset)
2389 : SelectADDRri(N2.getNode(), N2, Base, Offset)) {
2390 if (TM.is64Bit()) {
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]2391 switch (N->getOpcode()) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2392 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2393 return nullptr;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]2394 case NVPTXISD::StoreV2:
2395 switch (EltVT.getSimpleVT().SimpleTy) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2396 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2397 return nullptr;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2398 case MVT::i8:
2399 Opcode = NVPTX::STV_i8_v2_ari_64;
2400 break;
2401 case MVT::i16:
2402 Opcode = NVPTX::STV_i16_v2_ari_64;
2403 break;
2404 case MVT::i32:
2405 Opcode = NVPTX::STV_i32_v2_ari_64;
2406 break;
2407 case MVT::i64:
2408 Opcode = NVPTX::STV_i64_v2_ari_64;
2409 break;
2410 case MVT::f32:
2411 Opcode = NVPTX::STV_f32_v2_ari_64;
2412 break;
2413 case MVT::f64:
2414 Opcode = NVPTX::STV_f64_v2_ari_64;
2415 break;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]2416 }
2417 break;
2418 case NVPTXISD::StoreV4:
2419 switch (EltVT.getSimpleVT().SimpleTy) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2420 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2421 return nullptr;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2422 case MVT::i8:
2423 Opcode = NVPTX::STV_i8_v4_ari_64;
2424 break;
2425 case MVT::i16:
2426 Opcode = NVPTX::STV_i16_v4_ari_64;
2427 break;
2428 case MVT::i32:
2429 Opcode = NVPTX::STV_i32_v4_ari_64;
2430 break;
2431 case MVT::f32:
2432 Opcode = NVPTX::STV_f32_v4_ari_64;
2433 break;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]2434 }
2435 break;
2436 }
2437 } else {
2438 switch (N->getOpcode()) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2439 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2440 return nullptr;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]2441 case NVPTXISD::StoreV2:
2442 switch (EltVT.getSimpleVT().SimpleTy) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2443 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2444 return nullptr;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2445 case MVT::i8:
2446 Opcode = NVPTX::STV_i8_v2_ari;
2447 break;
2448 case MVT::i16:
2449 Opcode = NVPTX::STV_i16_v2_ari;
2450 break;
2451 case MVT::i32:
2452 Opcode = NVPTX::STV_i32_v2_ari;
2453 break;
2454 case MVT::i64:
2455 Opcode = NVPTX::STV_i64_v2_ari;
2456 break;
2457 case MVT::f32:
2458 Opcode = NVPTX::STV_f32_v2_ari;
2459 break;
2460 case MVT::f64:
2461 Opcode = NVPTX::STV_f64_v2_ari;
2462 break;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]2463 }
2464 break;
2465 case NVPTXISD::StoreV4:
2466 switch (EltVT.getSimpleVT().SimpleTy) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2467 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2468 return nullptr;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2469 case MVT::i8:
2470 Opcode = NVPTX::STV_i8_v4_ari;
2471 break;
2472 case MVT::i16:
2473 Opcode = NVPTX::STV_i16_v4_ari;
2474 break;
2475 case MVT::i32:
2476 Opcode = NVPTX::STV_i32_v4_ari;
2477 break;
2478 case MVT::f32:
2479 Opcode = NVPTX::STV_f32_v4_ari;
2480 break;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]2481 }
2482 break;
2483 }
2484 }
2485 StOps.push_back(Base);
2486 StOps.push_back(Offset);
2487 } else {
Eric Christopher02389e32015-02-19 00:08:27 +0000[diff] [blame]2488 if (TM.is64Bit()) {
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]2489 switch (N->getOpcode()) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2490 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2491 return nullptr;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]2492 case NVPTXISD::StoreV2:
2493 switch (EltVT.getSimpleVT().SimpleTy) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2494 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2495 return nullptr;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2496 case MVT::i8:
2497 Opcode = NVPTX::STV_i8_v2_areg_64;
2498 break;
2499 case MVT::i16:
2500 Opcode = NVPTX::STV_i16_v2_areg_64;
2501 break;
2502 case MVT::i32:
2503 Opcode = NVPTX::STV_i32_v2_areg_64;
2504 break;
2505 case MVT::i64:
2506 Opcode = NVPTX::STV_i64_v2_areg_64;
2507 break;
2508 case MVT::f32:
2509 Opcode = NVPTX::STV_f32_v2_areg_64;
2510 break;
2511 case MVT::f64:
2512 Opcode = NVPTX::STV_f64_v2_areg_64;
2513 break;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]2514 }
2515 break;
2516 case NVPTXISD::StoreV4:
2517 switch (EltVT.getSimpleVT().SimpleTy) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2518 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2519 return nullptr;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2520 case MVT::i8:
2521 Opcode = NVPTX::STV_i8_v4_areg_64;
2522 break;
2523 case MVT::i16:
2524 Opcode = NVPTX::STV_i16_v4_areg_64;
2525 break;
2526 case MVT::i32:
2527 Opcode = NVPTX::STV_i32_v4_areg_64;
2528 break;
2529 case MVT::f32:
2530 Opcode = NVPTX::STV_f32_v4_areg_64;
2531 break;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]2532 }
2533 break;
2534 }
2535 } else {
2536 switch (N->getOpcode()) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2537 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2538 return nullptr;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]2539 case NVPTXISD::StoreV2:
2540 switch (EltVT.getSimpleVT().SimpleTy) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2541 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2542 return nullptr;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2543 case MVT::i8:
2544 Opcode = NVPTX::STV_i8_v2_areg;
2545 break;
2546 case MVT::i16:
2547 Opcode = NVPTX::STV_i16_v2_areg;
2548 break;
2549 case MVT::i32:
2550 Opcode = NVPTX::STV_i32_v2_areg;
2551 break;
2552 case MVT::i64:
2553 Opcode = NVPTX::STV_i64_v2_areg;
2554 break;
2555 case MVT::f32:
2556 Opcode = NVPTX::STV_f32_v2_areg;
2557 break;
2558 case MVT::f64:
2559 Opcode = NVPTX::STV_f64_v2_areg;
2560 break;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]2561 }
2562 break;
2563 case NVPTXISD::StoreV4:
2564 switch (EltVT.getSimpleVT().SimpleTy) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2565 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2566 return nullptr;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2567 case MVT::i8:
2568 Opcode = NVPTX::STV_i8_v4_areg;
2569 break;
2570 case MVT::i16:
2571 Opcode = NVPTX::STV_i16_v4_areg;
2572 break;
2573 case MVT::i32:
2574 Opcode = NVPTX::STV_i32_v4_areg;
2575 break;
2576 case MVT::f32:
2577 Opcode = NVPTX::STV_f32_v4_areg;
2578 break;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]2579 }
2580 break;
2581 }
2582 }
2583 StOps.push_back(N2);
2584 }
2585
2586 StOps.push_back(Chain);
2587
Michael Liaob53d8962013-04-19 22:22:57 +0000[diff] [blame]2588 ST = CurDAG->getMachineNode(Opcode, DL, MVT::Other, StOps);
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]2589
2590 MachineSDNode::mmo_iterator MemRefs0 = MF->allocateMemRefsArray(1);
2591 MemRefs0[0] = cast<MemSDNode>(N)->getMemOperand();
2592 cast<MachineSDNode>(ST)->setMemRefs(MemRefs0, MemRefs0 + 1);
2593
2594 return ST;
2595}
2596
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]2597SDNode *NVPTXDAGToDAGISel::SelectLoadParam(SDNode *Node) {
2598 SDValue Chain = Node->getOperand(0);
2599 SDValue Offset = Node->getOperand(2);
2600 SDValue Flag = Node->getOperand(3);
2601 SDLoc DL(Node);
2602 MemSDNode *Mem = cast<MemSDNode>(Node);
2603
2604 unsigned VecSize;
2605 switch (Node->getOpcode()) {
2606 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2607 return nullptr;
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]2608 case NVPTXISD::LoadParam:
2609 VecSize = 1;
2610 break;
2611 case NVPTXISD::LoadParamV2:
2612 VecSize = 2;
2613 break;
2614 case NVPTXISD::LoadParamV4:
2615 VecSize = 4;
2616 break;
2617 }
2618
2619 EVT EltVT = Node->getValueType(0);
2620 EVT MemVT = Mem->getMemoryVT();
2621
2622 unsigned Opc = 0;
2623
2624 switch (VecSize) {
2625 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2626 return nullptr;
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]2627 case 1:
2628 switch (MemVT.getSimpleVT().SimpleTy) {
2629 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2630 return nullptr;
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]2631 case MVT::i1:
2632 Opc = NVPTX::LoadParamMemI8;
2633 break;
2634 case MVT::i8:
2635 Opc = NVPTX::LoadParamMemI8;
2636 break;
2637 case MVT::i16:
2638 Opc = NVPTX::LoadParamMemI16;
2639 break;
2640 case MVT::i32:
2641 Opc = NVPTX::LoadParamMemI32;
2642 break;
2643 case MVT::i64:
2644 Opc = NVPTX::LoadParamMemI64;
2645 break;
2646 case MVT::f32:
2647 Opc = NVPTX::LoadParamMemF32;
2648 break;
2649 case MVT::f64:
2650 Opc = NVPTX::LoadParamMemF64;
2651 break;
2652 }
2653 break;
2654 case 2:
2655 switch (MemVT.getSimpleVT().SimpleTy) {
2656 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2657 return nullptr;
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]2658 case MVT::i1:
2659 Opc = NVPTX::LoadParamMemV2I8;
2660 break;
2661 case MVT::i8:
2662 Opc = NVPTX::LoadParamMemV2I8;
2663 break;
2664 case MVT::i16:
2665 Opc = NVPTX::LoadParamMemV2I16;
2666 break;
2667 case MVT::i32:
2668 Opc = NVPTX::LoadParamMemV2I32;
2669 break;
2670 case MVT::i64:
2671 Opc = NVPTX::LoadParamMemV2I64;
2672 break;
2673 case MVT::f32:
2674 Opc = NVPTX::LoadParamMemV2F32;
2675 break;
2676 case MVT::f64:
2677 Opc = NVPTX::LoadParamMemV2F64;
2678 break;
2679 }
2680 break;
2681 case 4:
2682 switch (MemVT.getSimpleVT().SimpleTy) {
2683 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2684 return nullptr;
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]2685 case MVT::i1:
2686 Opc = NVPTX::LoadParamMemV4I8;
2687 break;
2688 case MVT::i8:
2689 Opc = NVPTX::LoadParamMemV4I8;
2690 break;
2691 case MVT::i16:
2692 Opc = NVPTX::LoadParamMemV4I16;
2693 break;
2694 case MVT::i32:
2695 Opc = NVPTX::LoadParamMemV4I32;
2696 break;
2697 case MVT::f32:
2698 Opc = NVPTX::LoadParamMemV4F32;
2699 break;
2700 }
2701 break;
2702 }
2703
2704 SDVTList VTs;
2705 if (VecSize == 1) {
2706 VTs = CurDAG->getVTList(EltVT, MVT::Other, MVT::Glue);
2707 } else if (VecSize == 2) {
2708 VTs = CurDAG->getVTList(EltVT, EltVT, MVT::Other, MVT::Glue);
2709 } else {
2710 EVT EVTs[] = { EltVT, EltVT, EltVT, EltVT, MVT::Other, MVT::Glue };
Craig Topperabb4ac72014-04-16 06:10:51 +0000[diff] [blame]2711 VTs = CurDAG->getVTList(EVTs);
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]2712 }
2713
2714 unsigned OffsetVal = cast<ConstantSDNode>(Offset)->getZExtValue();
2715
2716 SmallVector<SDValue, 2> Ops;
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +0000[diff] [blame]2717 Ops.push_back(CurDAG->getTargetConstant(OffsetVal, DL, MVT::i32));
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]2718 Ops.push_back(Chain);
2719 Ops.push_back(Flag);
2720
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +0000[diff] [blame]2721 return CurDAG->getMachineNode(Opc, DL, VTs, Ops);
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]2722}
2723
2724SDNode *NVPTXDAGToDAGISel::SelectStoreRetval(SDNode *N) {
2725 SDLoc DL(N);
2726 SDValue Chain = N->getOperand(0);
2727 SDValue Offset = N->getOperand(1);
2728 unsigned OffsetVal = cast<ConstantSDNode>(Offset)->getZExtValue();
2729 MemSDNode *Mem = cast<MemSDNode>(N);
2730
2731 // How many elements do we have?
2732 unsigned NumElts = 1;
2733 switch (N->getOpcode()) {
2734 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2735 return nullptr;
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]2736 case NVPTXISD::StoreRetval:
2737 NumElts = 1;
2738 break;
2739 case NVPTXISD::StoreRetvalV2:
2740 NumElts = 2;
2741 break;
2742 case NVPTXISD::StoreRetvalV4:
2743 NumElts = 4;
2744 break;
2745 }
2746
2747 // Build vector of operands
2748 SmallVector<SDValue, 6> Ops;
2749 for (unsigned i = 0; i < NumElts; ++i)
2750 Ops.push_back(N->getOperand(i + 2));
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +0000[diff] [blame]2751 Ops.push_back(CurDAG->getTargetConstant(OffsetVal, DL, MVT::i32));
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]2752 Ops.push_back(Chain);
2753
2754 // Determine target opcode
2755 // If we have an i1, use an 8-bit store. The lowering code in
2756 // NVPTXISelLowering will have already emitted an upcast.
2757 unsigned Opcode = 0;
2758 switch (NumElts) {
2759 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2760 return nullptr;
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]2761 case 1:
2762 switch (Mem->getMemoryVT().getSimpleVT().SimpleTy) {
2763 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2764 return nullptr;
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]2765 case MVT::i1:
2766 Opcode = NVPTX::StoreRetvalI8;
2767 break;
2768 case MVT::i8:
2769 Opcode = NVPTX::StoreRetvalI8;
2770 break;
2771 case MVT::i16:
2772 Opcode = NVPTX::StoreRetvalI16;
2773 break;
2774 case MVT::i32:
2775 Opcode = NVPTX::StoreRetvalI32;
2776 break;
2777 case MVT::i64:
2778 Opcode = NVPTX::StoreRetvalI64;
2779 break;
2780 case MVT::f32:
2781 Opcode = NVPTX::StoreRetvalF32;
2782 break;
2783 case MVT::f64:
2784 Opcode = NVPTX::StoreRetvalF64;
2785 break;
2786 }
2787 break;
2788 case 2:
2789 switch (Mem->getMemoryVT().getSimpleVT().SimpleTy) {
2790 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2791 return nullptr;
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]2792 case MVT::i1:
2793 Opcode = NVPTX::StoreRetvalV2I8;
2794 break;
2795 case MVT::i8:
2796 Opcode = NVPTX::StoreRetvalV2I8;
2797 break;
2798 case MVT::i16:
2799 Opcode = NVPTX::StoreRetvalV2I16;
2800 break;
2801 case MVT::i32:
2802 Opcode = NVPTX::StoreRetvalV2I32;
2803 break;
2804 case MVT::i64:
2805 Opcode = NVPTX::StoreRetvalV2I64;
2806 break;
2807 case MVT::f32:
2808 Opcode = NVPTX::StoreRetvalV2F32;
2809 break;
2810 case MVT::f64:
2811 Opcode = NVPTX::StoreRetvalV2F64;
2812 break;
2813 }
2814 break;
2815 case 4:
2816 switch (Mem->getMemoryVT().getSimpleVT().SimpleTy) {
2817 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2818 return nullptr;
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]2819 case MVT::i1:
2820 Opcode = NVPTX::StoreRetvalV4I8;
2821 break;
2822 case MVT::i8:
2823 Opcode = NVPTX::StoreRetvalV4I8;
2824 break;
2825 case MVT::i16:
2826 Opcode = NVPTX::StoreRetvalV4I16;
2827 break;
2828 case MVT::i32:
2829 Opcode = NVPTX::StoreRetvalV4I32;
2830 break;
2831 case MVT::f32:
2832 Opcode = NVPTX::StoreRetvalV4F32;
2833 break;
2834 }
2835 break;
2836 }
2837
2838 SDNode *Ret =
2839 CurDAG->getMachineNode(Opcode, DL, MVT::Other, Ops);
2840 MachineSDNode::mmo_iterator MemRefs0 = MF->allocateMemRefsArray(1);
2841 MemRefs0[0] = cast<MemSDNode>(N)->getMemOperand();
2842 cast<MachineSDNode>(Ret)->setMemRefs(MemRefs0, MemRefs0 + 1);
2843
2844 return Ret;
2845}
2846
2847SDNode *NVPTXDAGToDAGISel::SelectStoreParam(SDNode *N) {
2848 SDLoc DL(N);
2849 SDValue Chain = N->getOperand(0);
2850 SDValue Param = N->getOperand(1);
2851 unsigned ParamVal = cast<ConstantSDNode>(Param)->getZExtValue();
2852 SDValue Offset = N->getOperand(2);
2853 unsigned OffsetVal = cast<ConstantSDNode>(Offset)->getZExtValue();
2854 MemSDNode *Mem = cast<MemSDNode>(N);
2855 SDValue Flag = N->getOperand(N->getNumOperands() - 1);
2856
2857 // How many elements do we have?
2858 unsigned NumElts = 1;
2859 switch (N->getOpcode()) {
2860 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2861 return nullptr;
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]2862 case NVPTXISD::StoreParamU32:
2863 case NVPTXISD::StoreParamS32:
2864 case NVPTXISD::StoreParam:
2865 NumElts = 1;
2866 break;
2867 case NVPTXISD::StoreParamV2:
2868 NumElts = 2;
2869 break;
2870 case NVPTXISD::StoreParamV4:
2871 NumElts = 4;
2872 break;
2873 }
2874
2875 // Build vector of operands
2876 SmallVector<SDValue, 8> Ops;
2877 for (unsigned i = 0; i < NumElts; ++i)
2878 Ops.push_back(N->getOperand(i + 3));
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +0000[diff] [blame]2879 Ops.push_back(CurDAG->getTargetConstant(ParamVal, DL, MVT::i32));
2880 Ops.push_back(CurDAG->getTargetConstant(OffsetVal, DL, MVT::i32));
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]2881 Ops.push_back(Chain);
2882 Ops.push_back(Flag);
2883
2884 // Determine target opcode
2885 // If we have an i1, use an 8-bit store. The lowering code in
2886 // NVPTXISelLowering will have already emitted an upcast.
2887 unsigned Opcode = 0;
2888 switch (N->getOpcode()) {
2889 default:
2890 switch (NumElts) {
2891 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2892 return nullptr;
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]2893 case 1:
2894 switch (Mem->getMemoryVT().getSimpleVT().SimpleTy) {
2895 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2896 return nullptr;
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]2897 case MVT::i1:
2898 Opcode = NVPTX::StoreParamI8;
2899 break;
2900 case MVT::i8:
2901 Opcode = NVPTX::StoreParamI8;
2902 break;
2903 case MVT::i16:
2904 Opcode = NVPTX::StoreParamI16;
2905 break;
2906 case MVT::i32:
2907 Opcode = NVPTX::StoreParamI32;
2908 break;
2909 case MVT::i64:
2910 Opcode = NVPTX::StoreParamI64;
2911 break;
2912 case MVT::f32:
2913 Opcode = NVPTX::StoreParamF32;
2914 break;
2915 case MVT::f64:
2916 Opcode = NVPTX::StoreParamF64;
2917 break;
2918 }
2919 break;
2920 case 2:
2921 switch (Mem->getMemoryVT().getSimpleVT().SimpleTy) {
2922 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2923 return nullptr;
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]2924 case MVT::i1:
2925 Opcode = NVPTX::StoreParamV2I8;
2926 break;
2927 case MVT::i8:
2928 Opcode = NVPTX::StoreParamV2I8;
2929 break;
2930 case MVT::i16:
2931 Opcode = NVPTX::StoreParamV2I16;
2932 break;
2933 case MVT::i32:
2934 Opcode = NVPTX::StoreParamV2I32;
2935 break;
2936 case MVT::i64:
2937 Opcode = NVPTX::StoreParamV2I64;
2938 break;
2939 case MVT::f32:
2940 Opcode = NVPTX::StoreParamV2F32;
2941 break;
2942 case MVT::f64:
2943 Opcode = NVPTX::StoreParamV2F64;
2944 break;
2945 }
2946 break;
2947 case 4:
2948 switch (Mem->getMemoryVT().getSimpleVT().SimpleTy) {
2949 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2950 return nullptr;
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]2951 case MVT::i1:
2952 Opcode = NVPTX::StoreParamV4I8;
2953 break;
2954 case MVT::i8:
2955 Opcode = NVPTX::StoreParamV4I8;
2956 break;
2957 case MVT::i16:
2958 Opcode = NVPTX::StoreParamV4I16;
2959 break;
2960 case MVT::i32:
2961 Opcode = NVPTX::StoreParamV4I32;
2962 break;
2963 case MVT::f32:
2964 Opcode = NVPTX::StoreParamV4F32;
2965 break;
2966 }
2967 break;
2968 }
2969 break;
Justin Holewinskidc5e3b62013-06-28 17:58:04 +0000[diff] [blame]2970 // Special case: if we have a sign-extend/zero-extend node, insert the
2971 // conversion instruction first, and use that as the value operand to
2972 // the selected StoreParam node.
2973 case NVPTXISD::StoreParamU32: {
2974 Opcode = NVPTX::StoreParamI32;
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +0000[diff] [blame]2975 SDValue CvtNone = CurDAG->getTargetConstant(NVPTX::PTXCvtMode::NONE, DL,
Justin Holewinskidc5e3b62013-06-28 17:58:04 +0000[diff] [blame]2976 MVT::i32);
2977 SDNode *Cvt = CurDAG->getMachineNode(NVPTX::CVT_u32_u16, DL,
2978 MVT::i32, Ops[0], CvtNone);
2979 Ops[0] = SDValue(Cvt, 0);
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]2980 break;
Justin Holewinskidc5e3b62013-06-28 17:58:04 +0000[diff] [blame]2981 }
2982 case NVPTXISD::StoreParamS32: {
2983 Opcode = NVPTX::StoreParamI32;
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +0000[diff] [blame]2984 SDValue CvtNone = CurDAG->getTargetConstant(NVPTX::PTXCvtMode::NONE, DL,
Justin Holewinskidc5e3b62013-06-28 17:58:04 +0000[diff] [blame]2985 MVT::i32);
2986 SDNode *Cvt = CurDAG->getMachineNode(NVPTX::CVT_s32_s16, DL,
2987 MVT::i32, Ops[0], CvtNone);
2988 Ops[0] = SDValue(Cvt, 0);
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]2989 break;
2990 }
Justin Holewinskidc5e3b62013-06-28 17:58:04 +0000[diff] [blame]2991 }
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]2992
Justin Holewinskidff28d22013-07-01 12:59:01 +0000[diff] [blame]2993 SDVTList RetVTs = CurDAG->getVTList(MVT::Other, MVT::Glue);
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]2994 SDNode *Ret =
Justin Holewinskidff28d22013-07-01 12:59:01 +0000[diff] [blame]2995 CurDAG->getMachineNode(Opcode, DL, RetVTs, Ops);
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]2996 MachineSDNode::mmo_iterator MemRefs0 = MF->allocateMemRefsArray(1);
2997 MemRefs0[0] = cast<MemSDNode>(N)->getMemOperand();
2998 cast<MachineSDNode>(Ret)->setMemRefs(MemRefs0, MemRefs0 + 1);
2999
3000 return Ret;
3001}
3002
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]3003SDNode *NVPTXDAGToDAGISel::SelectTextureIntrinsic(SDNode *N) {
3004 SDValue Chain = N->getOperand(0);
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]3005 SDNode *Ret = nullptr;
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]3006 unsigned Opc = 0;
3007 SmallVector<SDValue, 8> Ops;
3008
3009 switch (N->getOpcode()) {
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]3010 default: return nullptr;
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]3011 case NVPTXISD::Tex1DFloatS32:
3012 Opc = NVPTX::TEX_1D_F32_S32;
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]3013 break;
3014 case NVPTXISD::Tex1DFloatFloat:
3015 Opc = NVPTX::TEX_1D_F32_F32;
3016 break;
3017 case NVPTXISD::Tex1DFloatFloatLevel:
3018 Opc = NVPTX::TEX_1D_F32_F32_LEVEL;
3019 break;
3020 case NVPTXISD::Tex1DFloatFloatGrad:
3021 Opc = NVPTX::TEX_1D_F32_F32_GRAD;
3022 break;
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]3023 case NVPTXISD::Tex1DS32S32:
3024 Opc = NVPTX::TEX_1D_S32_S32;
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]3025 break;
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]3026 case NVPTXISD::Tex1DS32Float:
3027 Opc = NVPTX::TEX_1D_S32_F32;
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]3028 break;
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]3029 case NVPTXISD::Tex1DS32FloatLevel:
3030 Opc = NVPTX::TEX_1D_S32_F32_LEVEL;
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]3031 break;
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]3032 case NVPTXISD::Tex1DS32FloatGrad:
3033 Opc = NVPTX::TEX_1D_S32_F32_GRAD;
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]3034 break;
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]3035 case NVPTXISD::Tex1DU32S32:
3036 Opc = NVPTX::TEX_1D_U32_S32;
3037 break;
3038 case NVPTXISD::Tex1DU32Float:
3039 Opc = NVPTX::TEX_1D_U32_F32;
3040 break;
3041 case NVPTXISD::Tex1DU32FloatLevel:
3042 Opc = NVPTX::TEX_1D_U32_F32_LEVEL;
3043 break;
3044 case NVPTXISD::Tex1DU32FloatGrad:
3045 Opc = NVPTX::TEX_1D_U32_F32_GRAD;
3046 break;
3047 case NVPTXISD::Tex1DArrayFloatS32:
3048 Opc = NVPTX::TEX_1D_ARRAY_F32_S32;
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]3049 break;
3050 case NVPTXISD::Tex1DArrayFloatFloat:
3051 Opc = NVPTX::TEX_1D_ARRAY_F32_F32;
3052 break;
3053 case NVPTXISD::Tex1DArrayFloatFloatLevel:
3054 Opc = NVPTX::TEX_1D_ARRAY_F32_F32_LEVEL;
3055 break;
3056 case NVPTXISD::Tex1DArrayFloatFloatGrad:
3057 Opc = NVPTX::TEX_1D_ARRAY_F32_F32_GRAD;
3058 break;
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]3059 case NVPTXISD::Tex1DArrayS32S32:
3060 Opc = NVPTX::TEX_1D_ARRAY_S32_S32;
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]3061 break;
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]3062 case NVPTXISD::Tex1DArrayS32Float:
3063 Opc = NVPTX::TEX_1D_ARRAY_S32_F32;
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]3064 break;
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]3065 case NVPTXISD::Tex1DArrayS32FloatLevel:
3066 Opc = NVPTX::TEX_1D_ARRAY_S32_F32_LEVEL;
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]3067 break;
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]3068 case NVPTXISD::Tex1DArrayS32FloatGrad:
3069 Opc = NVPTX::TEX_1D_ARRAY_S32_F32_GRAD;
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]3070 break;
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]3071 case NVPTXISD::Tex1DArrayU32S32:
3072 Opc = NVPTX::TEX_1D_ARRAY_U32_S32;
3073 break;
3074 case NVPTXISD::Tex1DArrayU32Float:
3075 Opc = NVPTX::TEX_1D_ARRAY_U32_F32;
3076 break;
3077 case NVPTXISD::Tex1DArrayU32FloatLevel:
3078 Opc = NVPTX::TEX_1D_ARRAY_U32_F32_LEVEL;
3079 break;
3080 case NVPTXISD::Tex1DArrayU32FloatGrad:
3081 Opc = NVPTX::TEX_1D_ARRAY_U32_F32_GRAD;
3082 break;
3083 case NVPTXISD::Tex2DFloatS32:
3084 Opc = NVPTX::TEX_2D_F32_S32;
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]3085 break;
3086 case NVPTXISD::Tex2DFloatFloat:
3087 Opc = NVPTX::TEX_2D_F32_F32;
3088 break;
3089 case NVPTXISD::Tex2DFloatFloatLevel:
3090 Opc = NVPTX::TEX_2D_F32_F32_LEVEL;
3091 break;
3092 case NVPTXISD::Tex2DFloatFloatGrad:
3093 Opc = NVPTX::TEX_2D_F32_F32_GRAD;
3094 break;
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]3095 case NVPTXISD::Tex2DS32S32:
3096 Opc = NVPTX::TEX_2D_S32_S32;
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]3097 break;
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]3098 case NVPTXISD::Tex2DS32Float:
3099 Opc = NVPTX::TEX_2D_S32_F32;
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]3100 break;
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]3101 case NVPTXISD::Tex2DS32FloatLevel:
3102 Opc = NVPTX::TEX_2D_S32_F32_LEVEL;
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]3103 break;
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]3104 case NVPTXISD::Tex2DS32FloatGrad:
3105 Opc = NVPTX::TEX_2D_S32_F32_GRAD;
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]3106 break;
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]3107 case NVPTXISD::Tex2DU32S32:
3108 Opc = NVPTX::TEX_2D_U32_S32;
3109 break;
3110 case NVPTXISD::Tex2DU32Float:
3111 Opc = NVPTX::TEX_2D_U32_F32;
3112 break;
3113 case NVPTXISD::Tex2DU32FloatLevel:
3114 Opc = NVPTX::TEX_2D_U32_F32_LEVEL;
3115 break;
3116 case NVPTXISD::Tex2DU32FloatGrad:
3117 Opc = NVPTX::TEX_2D_U32_F32_GRAD;
3118 break;
3119 case NVPTXISD::Tex2DArrayFloatS32:
3120 Opc = NVPTX::TEX_2D_ARRAY_F32_S32;
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]3121 break;
3122 case NVPTXISD::Tex2DArrayFloatFloat:
3123 Opc = NVPTX::TEX_2D_ARRAY_F32_F32;
3124 break;
3125 case NVPTXISD::Tex2DArrayFloatFloatLevel:
3126 Opc = NVPTX::TEX_2D_ARRAY_F32_F32_LEVEL;
3127 break;
3128 case NVPTXISD::Tex2DArrayFloatFloatGrad:
3129 Opc = NVPTX::TEX_2D_ARRAY_F32_F32_GRAD;
3130 break;
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]3131 case NVPTXISD::Tex2DArrayS32S32:
3132 Opc = NVPTX::TEX_2D_ARRAY_S32_S32;
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]3133 break;
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]3134 case NVPTXISD::Tex2DArrayS32Float:
3135 Opc = NVPTX::TEX_2D_ARRAY_S32_F32;
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]3136 break;
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]3137 case NVPTXISD::Tex2DArrayS32FloatLevel:
3138 Opc = NVPTX::TEX_2D_ARRAY_S32_F32_LEVEL;
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]3139 break;
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]3140 case NVPTXISD::Tex2DArrayS32FloatGrad:
3141 Opc = NVPTX::TEX_2D_ARRAY_S32_F32_GRAD;
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]3142 break;
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]3143 case NVPTXISD::Tex2DArrayU32S32:
3144 Opc = NVPTX::TEX_2D_ARRAY_U32_S32;
3145 break;
3146 case NVPTXISD::Tex2DArrayU32Float:
3147 Opc = NVPTX::TEX_2D_ARRAY_U32_F32;
3148 break;
3149 case NVPTXISD::Tex2DArrayU32FloatLevel:
3150 Opc = NVPTX::TEX_2D_ARRAY_U32_F32_LEVEL;
3151 break;
3152 case NVPTXISD::Tex2DArrayU32FloatGrad:
3153 Opc = NVPTX::TEX_2D_ARRAY_U32_F32_GRAD;
3154 break;
3155 case NVPTXISD::Tex3DFloatS32:
3156 Opc = NVPTX::TEX_3D_F32_S32;
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]3157 break;
3158 case NVPTXISD::Tex3DFloatFloat:
3159 Opc = NVPTX::TEX_3D_F32_F32;
3160 break;
3161 case NVPTXISD::Tex3DFloatFloatLevel:
3162 Opc = NVPTX::TEX_3D_F32_F32_LEVEL;
3163 break;
3164 case NVPTXISD::Tex3DFloatFloatGrad:
3165 Opc = NVPTX::TEX_3D_F32_F32_GRAD;
3166 break;
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]3167 case NVPTXISD::Tex3DS32S32:
3168 Opc = NVPTX::TEX_3D_S32_S32;
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]3169 break;
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]3170 case NVPTXISD::Tex3DS32Float:
3171 Opc = NVPTX::TEX_3D_S32_F32;
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]3172 break;
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]3173 case NVPTXISD::Tex3DS32FloatLevel:
3174 Opc = NVPTX::TEX_3D_S32_F32_LEVEL;
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]3175 break;
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]3176 case NVPTXISD::Tex3DS32FloatGrad:
3177 Opc = NVPTX::TEX_3D_S32_F32_GRAD;
3178 break;
3179 case NVPTXISD::Tex3DU32S32:
3180 Opc = NVPTX::TEX_3D_U32_S32;
3181 break;
3182 case NVPTXISD::Tex3DU32Float:
3183 Opc = NVPTX::TEX_3D_U32_F32;
3184 break;
3185 case NVPTXISD::Tex3DU32FloatLevel:
3186 Opc = NVPTX::TEX_3D_U32_F32_LEVEL;
3187 break;
3188 case NVPTXISD::Tex3DU32FloatGrad:
3189 Opc = NVPTX::TEX_3D_U32_F32_GRAD;
3190 break;
3191 case NVPTXISD::TexCubeFloatFloat:
3192 Opc = NVPTX::TEX_CUBE_F32_F32;
3193 break;
3194 case NVPTXISD::TexCubeFloatFloatLevel:
3195 Opc = NVPTX::TEX_CUBE_F32_F32_LEVEL;
3196 break;
3197 case NVPTXISD::TexCubeS32Float:
3198 Opc = NVPTX::TEX_CUBE_S32_F32;
3199 break;
3200 case NVPTXISD::TexCubeS32FloatLevel:
3201 Opc = NVPTX::TEX_CUBE_S32_F32_LEVEL;
3202 break;
3203 case NVPTXISD::TexCubeU32Float:
3204 Opc = NVPTX::TEX_CUBE_U32_F32;
3205 break;
3206 case NVPTXISD::TexCubeU32FloatLevel:
3207 Opc = NVPTX::TEX_CUBE_U32_F32_LEVEL;
3208 break;
3209 case NVPTXISD::TexCubeArrayFloatFloat:
3210 Opc = NVPTX::TEX_CUBE_ARRAY_F32_F32;
3211 break;
3212 case NVPTXISD::TexCubeArrayFloatFloatLevel:
3213 Opc = NVPTX::TEX_CUBE_ARRAY_F32_F32_LEVEL;
3214 break;
3215 case NVPTXISD::TexCubeArrayS32Float:
3216 Opc = NVPTX::TEX_CUBE_ARRAY_S32_F32;
3217 break;
3218 case NVPTXISD::TexCubeArrayS32FloatLevel:
3219 Opc = NVPTX::TEX_CUBE_ARRAY_S32_F32_LEVEL;
3220 break;
3221 case NVPTXISD::TexCubeArrayU32Float:
3222 Opc = NVPTX::TEX_CUBE_ARRAY_U32_F32;
3223 break;
3224 case NVPTXISD::TexCubeArrayU32FloatLevel:
3225 Opc = NVPTX::TEX_CUBE_ARRAY_U32_F32_LEVEL;
3226 break;
3227 case NVPTXISD::Tld4R2DFloatFloat:
3228 Opc = NVPTX::TLD4_R_2D_F32_F32;
3229 break;
3230 case NVPTXISD::Tld4G2DFloatFloat:
3231 Opc = NVPTX::TLD4_G_2D_F32_F32;
3232 break;
3233 case NVPTXISD::Tld4B2DFloatFloat:
3234 Opc = NVPTX::TLD4_B_2D_F32_F32;
3235 break;
3236 case NVPTXISD::Tld4A2DFloatFloat:
3237 Opc = NVPTX::TLD4_A_2D_F32_F32;
3238 break;
3239 case NVPTXISD::Tld4R2DS64Float:
3240 Opc = NVPTX::TLD4_R_2D_S32_F32;
3241 break;
3242 case NVPTXISD::Tld4G2DS64Float:
3243 Opc = NVPTX::TLD4_G_2D_S32_F32;
3244 break;
3245 case NVPTXISD::Tld4B2DS64Float:
3246 Opc = NVPTX::TLD4_B_2D_S32_F32;
3247 break;
3248 case NVPTXISD::Tld4A2DS64Float:
3249 Opc = NVPTX::TLD4_A_2D_S32_F32;
3250 break;
3251 case NVPTXISD::Tld4R2DU64Float:
3252 Opc = NVPTX::TLD4_R_2D_U32_F32;
3253 break;
3254 case NVPTXISD::Tld4G2DU64Float:
3255 Opc = NVPTX::TLD4_G_2D_U32_F32;
3256 break;
3257 case NVPTXISD::Tld4B2DU64Float:
3258 Opc = NVPTX::TLD4_B_2D_U32_F32;
3259 break;
3260 case NVPTXISD::Tld4A2DU64Float:
3261 Opc = NVPTX::TLD4_A_2D_U32_F32;
3262 break;
3263 case NVPTXISD::TexUnified1DFloatS32:
3264 Opc = NVPTX::TEX_UNIFIED_1D_F32_S32;
3265 break;
3266 case NVPTXISD::TexUnified1DFloatFloat:
3267 Opc = NVPTX::TEX_UNIFIED_1D_F32_F32;
3268 break;
3269 case NVPTXISD::TexUnified1DFloatFloatLevel:
3270 Opc = NVPTX::TEX_UNIFIED_1D_F32_F32_LEVEL;
3271 break;
3272 case NVPTXISD::TexUnified1DFloatFloatGrad:
3273 Opc = NVPTX::TEX_UNIFIED_1D_F32_F32_GRAD;
3274 break;
3275 case NVPTXISD::TexUnified1DS32S32:
3276 Opc = NVPTX::TEX_UNIFIED_1D_S32_S32;
3277 break;
3278 case NVPTXISD::TexUnified1DS32Float:
3279 Opc = NVPTX::TEX_UNIFIED_1D_S32_F32;
3280 break;
3281 case NVPTXISD::TexUnified1DS32FloatLevel:
3282 Opc = NVPTX::TEX_UNIFIED_1D_S32_F32_LEVEL;
3283 break;
3284 case NVPTXISD::TexUnified1DS32FloatGrad:
3285 Opc = NVPTX::TEX_UNIFIED_1D_S32_F32_GRAD;
3286 break;
3287 case NVPTXISD::TexUnified1DU32S32:
3288 Opc = NVPTX::TEX_UNIFIED_1D_U32_S32;
3289 break;
3290 case NVPTXISD::TexUnified1DU32Float:
3291 Opc = NVPTX::TEX_UNIFIED_1D_U32_F32;
3292 break;
3293 case NVPTXISD::TexUnified1DU32FloatLevel:
3294 Opc = NVPTX::TEX_UNIFIED_1D_U32_F32_LEVEL;
3295 break;
3296 case NVPTXISD::TexUnified1DU32FloatGrad:
3297 Opc = NVPTX::TEX_UNIFIED_1D_U32_F32_GRAD;
3298 break;
3299 case NVPTXISD::TexUnified1DArrayFloatS32:
3300 Opc = NVPTX::TEX_UNIFIED_1D_ARRAY_F32_S32;
3301 break;
3302 case NVPTXISD::TexUnified1DArrayFloatFloat:
3303 Opc = NVPTX::TEX_UNIFIED_1D_ARRAY_F32_F32;
3304 break;
3305 case NVPTXISD::TexUnified1DArrayFloatFloatLevel:
3306 Opc = NVPTX::TEX_UNIFIED_1D_ARRAY_F32_F32_LEVEL;
3307 break;
3308 case NVPTXISD::TexUnified1DArrayFloatFloatGrad:
3309 Opc = NVPTX::TEX_UNIFIED_1D_ARRAY_F32_F32_GRAD;
3310 break;
3311 case NVPTXISD::TexUnified1DArrayS32S32:
3312 Opc = NVPTX::TEX_UNIFIED_1D_ARRAY_S32_S32;
3313 break;
3314 case NVPTXISD::TexUnified1DArrayS32Float:
3315 Opc = NVPTX::TEX_UNIFIED_1D_ARRAY_S32_F32;
3316 break;
3317 case NVPTXISD::TexUnified1DArrayS32FloatLevel:
3318 Opc = NVPTX::TEX_UNIFIED_1D_ARRAY_S32_F32_LEVEL;
3319 break;
3320 case NVPTXISD::TexUnified1DArrayS32FloatGrad:
3321 Opc = NVPTX::TEX_UNIFIED_1D_ARRAY_S32_F32_GRAD;
3322 break;
3323 case NVPTXISD::TexUnified1DArrayU32S32:
3324 Opc = NVPTX::TEX_UNIFIED_1D_ARRAY_U32_S32;
3325 break;
3326 case NVPTXISD::TexUnified1DArrayU32Float:
3327 Opc = NVPTX::TEX_UNIFIED_1D_ARRAY_U32_F32;
3328 break;
3329 case NVPTXISD::TexUnified1DArrayU32FloatLevel:
3330 Opc = NVPTX::TEX_UNIFIED_1D_ARRAY_U32_F32_LEVEL;
3331 break;
3332 case NVPTXISD::TexUnified1DArrayU32FloatGrad:
3333 Opc = NVPTX::TEX_UNIFIED_1D_ARRAY_U32_F32_GRAD;
3334 break;
3335 case NVPTXISD::TexUnified2DFloatS32:
3336 Opc = NVPTX::TEX_UNIFIED_2D_F32_S32;
3337 break;
3338 case NVPTXISD::TexUnified2DFloatFloat:
3339 Opc = NVPTX::TEX_UNIFIED_2D_F32_F32;
3340 break;
3341 case NVPTXISD::TexUnified2DFloatFloatLevel:
3342 Opc = NVPTX::TEX_UNIFIED_2D_F32_F32_LEVEL;
3343 break;
3344 case NVPTXISD::TexUnified2DFloatFloatGrad:
3345 Opc = NVPTX::TEX_UNIFIED_2D_F32_F32_GRAD;
3346 break;
3347 case NVPTXISD::TexUnified2DS32S32:
3348 Opc = NVPTX::TEX_UNIFIED_2D_S32_S32;
3349 break;
3350 case NVPTXISD::TexUnified2DS32Float:
3351 Opc = NVPTX::TEX_UNIFIED_2D_S32_F32;
3352 break;
3353 case NVPTXISD::TexUnified2DS32FloatLevel:
3354 Opc = NVPTX::TEX_UNIFIED_2D_S32_F32_LEVEL;
3355 break;
3356 case NVPTXISD::TexUnified2DS32FloatGrad:
3357 Opc = NVPTX::TEX_UNIFIED_2D_S32_F32_GRAD;
3358 break;
3359 case NVPTXISD::TexUnified2DU32S32:
3360 Opc = NVPTX::TEX_UNIFIED_2D_U32_S32;
3361 break;
3362 case NVPTXISD::TexUnified2DU32Float:
3363 Opc = NVPTX::TEX_UNIFIED_2D_U32_F32;
3364 break;
3365 case NVPTXISD::TexUnified2DU32FloatLevel:
3366 Opc = NVPTX::TEX_UNIFIED_2D_U32_F32_LEVEL;
3367 break;
3368 case NVPTXISD::TexUnified2DU32FloatGrad:
3369 Opc = NVPTX::TEX_UNIFIED_2D_U32_F32_GRAD;
3370 break;
3371 case NVPTXISD::TexUnified2DArrayFloatS32:
3372 Opc = NVPTX::TEX_UNIFIED_2D_ARRAY_F32_S32;
3373 break;
3374 case NVPTXISD::TexUnified2DArrayFloatFloat:
3375 Opc = NVPTX::TEX_UNIFIED_2D_ARRAY_F32_F32;
3376 break;
3377 case NVPTXISD::TexUnified2DArrayFloatFloatLevel:
3378 Opc = NVPTX::TEX_UNIFIED_2D_ARRAY_F32_F32_LEVEL;
3379 break;
3380 case NVPTXISD::TexUnified2DArrayFloatFloatGrad:
3381 Opc = NVPTX::TEX_UNIFIED_2D_ARRAY_F32_F32_GRAD;
3382 break;
3383 case NVPTXISD::TexUnified2DArrayS32S32:
3384 Opc = NVPTX::TEX_UNIFIED_2D_ARRAY_S32_S32;
3385 break;
3386 case NVPTXISD::TexUnified2DArrayS32Float:
3387 Opc = NVPTX::TEX_UNIFIED_2D_ARRAY_S32_F32;
3388 break;
3389 case NVPTXISD::TexUnified2DArrayS32FloatLevel:
3390 Opc = NVPTX::TEX_UNIFIED_2D_ARRAY_S32_F32_LEVEL;
3391 break;
3392 case NVPTXISD::TexUnified2DArrayS32FloatGrad:
3393 Opc = NVPTX::TEX_UNIFIED_2D_ARRAY_S32_F32_GRAD;
3394 break;
3395 case NVPTXISD::TexUnified2DArrayU32S32:
3396 Opc = NVPTX::TEX_UNIFIED_2D_ARRAY_U32_S32;
3397 break;
3398 case NVPTXISD::TexUnified2DArrayU32Float:
3399 Opc = NVPTX::TEX_UNIFIED_2D_ARRAY_U32_F32;
3400 break;
3401 case NVPTXISD::TexUnified2DArrayU32FloatLevel:
3402 Opc = NVPTX::TEX_UNIFIED_2D_ARRAY_U32_F32_LEVEL;
3403 break;
3404 case NVPTXISD::TexUnified2DArrayU32FloatGrad:
3405 Opc = NVPTX::TEX_UNIFIED_2D_ARRAY_U32_F32_GRAD;
3406 break;
3407 case NVPTXISD::TexUnified3DFloatS32:
3408 Opc = NVPTX::TEX_UNIFIED_3D_F32_S32;
3409 break;
3410 case NVPTXISD::TexUnified3DFloatFloat:
3411 Opc = NVPTX::TEX_UNIFIED_3D_F32_F32;
3412 break;
3413 case NVPTXISD::TexUnified3DFloatFloatLevel:
3414 Opc = NVPTX::TEX_UNIFIED_3D_F32_F32_LEVEL;
3415 break;
3416 case NVPTXISD::TexUnified3DFloatFloatGrad:
3417 Opc = NVPTX::TEX_UNIFIED_3D_F32_F32_GRAD;
3418 break;
3419 case NVPTXISD::TexUnified3DS32S32:
3420 Opc = NVPTX::TEX_UNIFIED_3D_S32_S32;
3421 break;
3422 case NVPTXISD::TexUnified3DS32Float:
3423 Opc = NVPTX::TEX_UNIFIED_3D_S32_F32;
3424 break;
3425 case NVPTXISD::TexUnified3DS32FloatLevel:
3426 Opc = NVPTX::TEX_UNIFIED_3D_S32_F32_LEVEL;
3427 break;
3428 case NVPTXISD::TexUnified3DS32FloatGrad:
3429 Opc = NVPTX::TEX_UNIFIED_3D_S32_F32_GRAD;
3430 break;
3431 case NVPTXISD::TexUnified3DU32S32:
3432 Opc = NVPTX::TEX_UNIFIED_3D_U32_S32;
3433 break;
3434 case NVPTXISD::TexUnified3DU32Float:
3435 Opc = NVPTX::TEX_UNIFIED_3D_U32_F32;
3436 break;
3437 case NVPTXISD::TexUnified3DU32FloatLevel:
3438 Opc = NVPTX::TEX_UNIFIED_3D_U32_F32_LEVEL;
3439 break;
3440 case NVPTXISD::TexUnified3DU32FloatGrad:
3441 Opc = NVPTX::TEX_UNIFIED_3D_U32_F32_GRAD;
3442 break;
3443 case NVPTXISD::TexUnifiedCubeFloatFloat:
3444 Opc = NVPTX::TEX_UNIFIED_CUBE_F32_F32;
3445 break;
3446 case NVPTXISD::TexUnifiedCubeFloatFloatLevel:
3447 Opc = NVPTX::TEX_UNIFIED_CUBE_F32_F32_LEVEL;
3448 break;
3449 case NVPTXISD::TexUnifiedCubeS32Float:
3450 Opc = NVPTX::TEX_UNIFIED_CUBE_S32_F32;
3451 break;
3452 case NVPTXISD::TexUnifiedCubeS32FloatLevel:
3453 Opc = NVPTX::TEX_UNIFIED_CUBE_S32_F32_LEVEL;
3454 break;
3455 case NVPTXISD::TexUnifiedCubeU32Float:
3456 Opc = NVPTX::TEX_UNIFIED_CUBE_U32_F32;
3457 break;
3458 case NVPTXISD::TexUnifiedCubeU32FloatLevel:
3459 Opc = NVPTX::TEX_UNIFIED_CUBE_U32_F32_LEVEL;
3460 break;
3461 case NVPTXISD::TexUnifiedCubeArrayFloatFloat:
3462 Opc = NVPTX::TEX_UNIFIED_CUBE_ARRAY_F32_F32;
3463 break;
3464 case NVPTXISD::TexUnifiedCubeArrayFloatFloatLevel:
3465 Opc = NVPTX::TEX_UNIFIED_CUBE_ARRAY_F32_F32_LEVEL;
3466 break;
3467 case NVPTXISD::TexUnifiedCubeArrayS32Float:
3468 Opc = NVPTX::TEX_UNIFIED_CUBE_ARRAY_S32_F32;
3469 break;
3470 case NVPTXISD::TexUnifiedCubeArrayS32FloatLevel:
3471 Opc = NVPTX::TEX_UNIFIED_CUBE_ARRAY_S32_F32_LEVEL;
3472 break;
3473 case NVPTXISD::TexUnifiedCubeArrayU32Float:
3474 Opc = NVPTX::TEX_UNIFIED_CUBE_ARRAY_U32_F32;
3475 break;
3476 case NVPTXISD::TexUnifiedCubeArrayU32FloatLevel:
3477 Opc = NVPTX::TEX_UNIFIED_CUBE_ARRAY_U32_F32_LEVEL;
3478 break;
3479 case NVPTXISD::Tld4UnifiedR2DFloatFloat:
3480 Opc = NVPTX::TLD4_UNIFIED_R_2D_F32_F32;
3481 break;
3482 case NVPTXISD::Tld4UnifiedG2DFloatFloat:
3483 Opc = NVPTX::TLD4_UNIFIED_G_2D_F32_F32;
3484 break;
3485 case NVPTXISD::Tld4UnifiedB2DFloatFloat:
3486 Opc = NVPTX::TLD4_UNIFIED_B_2D_F32_F32;
3487 break;
3488 case NVPTXISD::Tld4UnifiedA2DFloatFloat:
3489 Opc = NVPTX::TLD4_UNIFIED_A_2D_F32_F32;
3490 break;
3491 case NVPTXISD::Tld4UnifiedR2DS64Float:
3492 Opc = NVPTX::TLD4_UNIFIED_R_2D_S32_F32;
3493 break;
3494 case NVPTXISD::Tld4UnifiedG2DS64Float:
3495 Opc = NVPTX::TLD4_UNIFIED_G_2D_S32_F32;
3496 break;
3497 case NVPTXISD::Tld4UnifiedB2DS64Float:
3498 Opc = NVPTX::TLD4_UNIFIED_B_2D_S32_F32;
3499 break;
3500 case NVPTXISD::Tld4UnifiedA2DS64Float:
3501 Opc = NVPTX::TLD4_UNIFIED_A_2D_S32_F32;
3502 break;
3503 case NVPTXISD::Tld4UnifiedR2DU64Float:
3504 Opc = NVPTX::TLD4_UNIFIED_R_2D_U32_F32;
3505 break;
3506 case NVPTXISD::Tld4UnifiedG2DU64Float:
3507 Opc = NVPTX::TLD4_UNIFIED_G_2D_U32_F32;
3508 break;
3509 case NVPTXISD::Tld4UnifiedB2DU64Float:
3510 Opc = NVPTX::TLD4_UNIFIED_B_2D_U32_F32;
3511 break;
3512 case NVPTXISD::Tld4UnifiedA2DU64Float:
3513 Opc = NVPTX::TLD4_UNIFIED_A_2D_U32_F32;
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]3514 break;
3515 }
3516
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]3517 // Copy over operands
3518 for (unsigned i = 1; i < N->getNumOperands(); ++i) {
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]3519 Ops.push_back(N->getOperand(i));
3520 }
3521
3522 Ops.push_back(Chain);
3523 Ret = CurDAG->getMachineNode(Opc, SDLoc(N), N->getVTList(), Ops);
3524 return Ret;
3525}
3526
3527SDNode *NVPTXDAGToDAGISel::SelectSurfaceIntrinsic(SDNode *N) {
3528 SDValue Chain = N->getOperand(0);
3529 SDValue TexHandle = N->getOperand(1);
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]3530 SDNode *Ret = nullptr;
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]3531 unsigned Opc = 0;
3532 SmallVector<SDValue, 8> Ops;
3533 switch (N->getOpcode()) {
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]3534 default: return nullptr;
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]3535 case NVPTXISD::Suld1DI8Clamp:
3536 Opc = NVPTX::SULD_1D_I8_CLAMP;
3537 Ops.push_back(TexHandle);
3538 Ops.push_back(N->getOperand(2));
3539 Ops.push_back(Chain);
3540 break;
3541 case NVPTXISD::Suld1DI16Clamp:
3542 Opc = NVPTX::SULD_1D_I16_CLAMP;
3543 Ops.push_back(TexHandle);
3544 Ops.push_back(N->getOperand(2));
3545 Ops.push_back(Chain);
3546 break;
3547 case NVPTXISD::Suld1DI32Clamp:
3548 Opc = NVPTX::SULD_1D_I32_CLAMP;
3549 Ops.push_back(TexHandle);
3550 Ops.push_back(N->getOperand(2));
3551 Ops.push_back(Chain);
3552 break;
3553 case NVPTXISD::Suld1DI64Clamp:
3554 Opc = NVPTX::SULD_1D_I64_CLAMP;
3555 Ops.push_back(TexHandle);
3556 Ops.push_back(N->getOperand(2));
3557 Ops.push_back(Chain);
3558 break;
3559 case NVPTXISD::Suld1DV2I8Clamp:
3560 Opc = NVPTX::SULD_1D_V2I8_CLAMP;
3561 Ops.push_back(TexHandle);
3562 Ops.push_back(N->getOperand(2));
3563 Ops.push_back(Chain);
3564 break;
3565 case NVPTXISD::Suld1DV2I16Clamp:
3566 Opc = NVPTX::SULD_1D_V2I16_CLAMP;
3567 Ops.push_back(TexHandle);
3568 Ops.push_back(N->getOperand(2));
3569 Ops.push_back(Chain);
3570 break;
3571 case NVPTXISD::Suld1DV2I32Clamp:
3572 Opc = NVPTX::SULD_1D_V2I32_CLAMP;
3573 Ops.push_back(TexHandle);
3574 Ops.push_back(N->getOperand(2));
3575 Ops.push_back(Chain);
3576 break;
3577 case NVPTXISD::Suld1DV2I64Clamp:
3578 Opc = NVPTX::SULD_1D_V2I64_CLAMP;
3579 Ops.push_back(TexHandle);
3580 Ops.push_back(N->getOperand(2));
3581 Ops.push_back(Chain);
3582 break;
3583 case NVPTXISD::Suld1DV4I8Clamp:
3584 Opc = NVPTX::SULD_1D_V4I8_CLAMP;
3585 Ops.push_back(TexHandle);
3586 Ops.push_back(N->getOperand(2));
3587 Ops.push_back(Chain);
3588 break;
3589 case NVPTXISD::Suld1DV4I16Clamp:
3590 Opc = NVPTX::SULD_1D_V4I16_CLAMP;
3591 Ops.push_back(TexHandle);
3592 Ops.push_back(N->getOperand(2));
3593 Ops.push_back(Chain);
3594 break;
3595 case NVPTXISD::Suld1DV4I32Clamp:
3596 Opc = NVPTX::SULD_1D_V4I32_CLAMP;
3597 Ops.push_back(TexHandle);
3598 Ops.push_back(N->getOperand(2));
3599 Ops.push_back(Chain);
3600 break;
3601 case NVPTXISD::Suld1DArrayI8Clamp:
3602 Opc = NVPTX::SULD_1D_ARRAY_I8_CLAMP;
3603 Ops.push_back(TexHandle);
3604 Ops.push_back(N->getOperand(2));
3605 Ops.push_back(N->getOperand(3));
3606 Ops.push_back(Chain);
3607 break;
3608 case NVPTXISD::Suld1DArrayI16Clamp:
3609 Opc = NVPTX::SULD_1D_ARRAY_I16_CLAMP;
3610 Ops.push_back(TexHandle);
3611 Ops.push_back(N->getOperand(2));
3612 Ops.push_back(N->getOperand(3));
3613 Ops.push_back(Chain);
3614 break;
3615 case NVPTXISD::Suld1DArrayI32Clamp:
3616 Opc = NVPTX::SULD_1D_ARRAY_I32_CLAMP;
3617 Ops.push_back(TexHandle);
3618 Ops.push_back(N->getOperand(2));
3619 Ops.push_back(N->getOperand(3));
3620 Ops.push_back(Chain);
3621 break;
3622 case NVPTXISD::Suld1DArrayI64Clamp:
3623 Opc = NVPTX::SULD_1D_ARRAY_I64_CLAMP;
3624 Ops.push_back(TexHandle);
3625 Ops.push_back(N->getOperand(2));
3626 Ops.push_back(N->getOperand(3));
3627 Ops.push_back(Chain);
3628 break;
3629 case NVPTXISD::Suld1DArrayV2I8Clamp:
3630 Opc = NVPTX::SULD_1D_ARRAY_V2I8_CLAMP;
3631 Ops.push_back(TexHandle);
3632 Ops.push_back(N->getOperand(2));
3633 Ops.push_back(N->getOperand(3));
3634 Ops.push_back(Chain);
3635 break;
3636 case NVPTXISD::Suld1DArrayV2I16Clamp:
3637 Opc = NVPTX::SULD_1D_ARRAY_V2I16_CLAMP;
3638 Ops.push_back(TexHandle);
3639 Ops.push_back(N->getOperand(2));
3640 Ops.push_back(N->getOperand(3));
3641 Ops.push_back(Chain);
3642 break;
3643 case NVPTXISD::Suld1DArrayV2I32Clamp:
3644 Opc = NVPTX::SULD_1D_ARRAY_V2I32_CLAMP;
3645 Ops.push_back(TexHandle);
3646 Ops.push_back(N->getOperand(2));
3647 Ops.push_back(N->getOperand(3));
3648 Ops.push_back(Chain);
3649 break;
3650 case NVPTXISD::Suld1DArrayV2I64Clamp:
3651 Opc = NVPTX::SULD_1D_ARRAY_V2I64_CLAMP;
3652 Ops.push_back(TexHandle);
3653 Ops.push_back(N->getOperand(2));
3654 Ops.push_back(N->getOperand(3));
3655 Ops.push_back(Chain);
3656 break;
3657 case NVPTXISD::Suld1DArrayV4I8Clamp:
3658 Opc = NVPTX::SULD_1D_ARRAY_V4I8_CLAMP;
3659 Ops.push_back(TexHandle);
3660 Ops.push_back(N->getOperand(2));
3661 Ops.push_back(N->getOperand(3));
3662 Ops.push_back(Chain);
3663 break;
3664 case NVPTXISD::Suld1DArrayV4I16Clamp:
3665 Opc = NVPTX::SULD_1D_ARRAY_V4I16_CLAMP;
3666 Ops.push_back(TexHandle);
3667 Ops.push_back(N->getOperand(2));
3668 Ops.push_back(N->getOperand(3));
3669 Ops.push_back(Chain);
3670 break;
3671 case NVPTXISD::Suld1DArrayV4I32Clamp:
3672 Opc = NVPTX::SULD_1D_ARRAY_V4I32_CLAMP;
3673 Ops.push_back(TexHandle);
3674 Ops.push_back(N->getOperand(2));
3675 Ops.push_back(N->getOperand(3));
3676 Ops.push_back(Chain);
3677 break;
3678 case NVPTXISD::Suld2DI8Clamp:
3679 Opc = NVPTX::SULD_2D_I8_CLAMP;
3680 Ops.push_back(TexHandle);
3681 Ops.push_back(N->getOperand(2));
3682 Ops.push_back(N->getOperand(3));
3683 Ops.push_back(Chain);
3684 break;
3685 case NVPTXISD::Suld2DI16Clamp:
3686 Opc = NVPTX::SULD_2D_I16_CLAMP;
3687 Ops.push_back(TexHandle);
3688 Ops.push_back(N->getOperand(2));
3689 Ops.push_back(N->getOperand(3));
3690 Ops.push_back(Chain);
3691 break;
3692 case NVPTXISD::Suld2DI32Clamp:
3693 Opc = NVPTX::SULD_2D_I32_CLAMP;
3694 Ops.push_back(TexHandle);
3695 Ops.push_back(N->getOperand(2));
3696 Ops.push_back(N->getOperand(3));
3697 Ops.push_back(Chain);
3698 break;
3699 case NVPTXISD::Suld2DI64Clamp:
3700 Opc = NVPTX::SULD_2D_I64_CLAMP;
3701 Ops.push_back(TexHandle);
3702 Ops.push_back(N->getOperand(2));
3703 Ops.push_back(N->getOperand(3));
3704 Ops.push_back(Chain);
3705 break;
3706 case NVPTXISD::Suld2DV2I8Clamp:
3707 Opc = NVPTX::SULD_2D_V2I8_CLAMP;
3708 Ops.push_back(TexHandle);
3709 Ops.push_back(N->getOperand(2));
3710 Ops.push_back(N->getOperand(3));
3711 Ops.push_back(Chain);
3712 break;
3713 case NVPTXISD::Suld2DV2I16Clamp:
3714 Opc = NVPTX::SULD_2D_V2I16_CLAMP;
3715 Ops.push_back(TexHandle);
3716 Ops.push_back(N->getOperand(2));
3717 Ops.push_back(N->getOperand(3));
3718 Ops.push_back(Chain);
3719 break;
3720 case NVPTXISD::Suld2DV2I32Clamp:
3721 Opc = NVPTX::SULD_2D_V2I32_CLAMP;
3722 Ops.push_back(TexHandle);
3723 Ops.push_back(N->getOperand(2));
3724 Ops.push_back(N->getOperand(3));
3725 Ops.push_back(Chain);
3726 break;
3727 case NVPTXISD::Suld2DV2I64Clamp:
3728 Opc = NVPTX::SULD_2D_V2I64_CLAMP;
3729 Ops.push_back(TexHandle);
3730 Ops.push_back(N->getOperand(2));
3731 Ops.push_back(N->getOperand(3));
3732 Ops.push_back(Chain);
3733 break;
3734 case NVPTXISD::Suld2DV4I8Clamp:
3735 Opc = NVPTX::SULD_2D_V4I8_CLAMP;
3736 Ops.push_back(TexHandle);
3737 Ops.push_back(N->getOperand(2));
3738 Ops.push_back(N->getOperand(3));
3739 Ops.push_back(Chain);
3740 break;
3741 case NVPTXISD::Suld2DV4I16Clamp:
3742 Opc = NVPTX::SULD_2D_V4I16_CLAMP;
3743 Ops.push_back(TexHandle);
3744 Ops.push_back(N->getOperand(2));
3745 Ops.push_back(N->getOperand(3));
3746 Ops.push_back(Chain);
3747 break;
3748 case NVPTXISD::Suld2DV4I32Clamp:
3749 Opc = NVPTX::SULD_2D_V4I32_CLAMP;
3750 Ops.push_back(TexHandle);
3751 Ops.push_back(N->getOperand(2));
3752 Ops.push_back(N->getOperand(3));
3753 Ops.push_back(Chain);
3754 break;
3755 case NVPTXISD::Suld2DArrayI8Clamp:
3756 Opc = NVPTX::SULD_2D_ARRAY_I8_CLAMP;
3757 Ops.push_back(TexHandle);
3758 Ops.push_back(N->getOperand(2));
3759 Ops.push_back(N->getOperand(3));
3760 Ops.push_back(N->getOperand(4));
3761 Ops.push_back(Chain);
3762 break;
3763 case NVPTXISD::Suld2DArrayI16Clamp:
3764 Opc = NVPTX::SULD_2D_ARRAY_I16_CLAMP;
3765 Ops.push_back(TexHandle);
3766 Ops.push_back(N->getOperand(2));
3767 Ops.push_back(N->getOperand(3));
3768 Ops.push_back(N->getOperand(4));
3769 Ops.push_back(Chain);
3770 break;
3771 case NVPTXISD::Suld2DArrayI32Clamp:
3772 Opc = NVPTX::SULD_2D_ARRAY_I32_CLAMP;
3773 Ops.push_back(TexHandle);
3774 Ops.push_back(N->getOperand(2));
3775 Ops.push_back(N->getOperand(3));
3776 Ops.push_back(N->getOperand(4));
3777 Ops.push_back(Chain);
3778 break;
3779 case NVPTXISD::Suld2DArrayI64Clamp:
3780 Opc = NVPTX::SULD_2D_ARRAY_I64_CLAMP;
3781 Ops.push_back(TexHandle);
3782 Ops.push_back(N->getOperand(2));
3783 Ops.push_back(N->getOperand(3));
3784 Ops.push_back(N->getOperand(4));
3785 Ops.push_back(Chain);
3786 break;
3787 case NVPTXISD::Suld2DArrayV2I8Clamp:
3788 Opc = NVPTX::SULD_2D_ARRAY_V2I8_CLAMP;
3789 Ops.push_back(TexHandle);
3790 Ops.push_back(N->getOperand(2));
3791 Ops.push_back(N->getOperand(3));
3792 Ops.push_back(N->getOperand(4));
3793 Ops.push_back(Chain);
3794 break;
3795 case NVPTXISD::Suld2DArrayV2I16Clamp:
3796 Opc = NVPTX::SULD_2D_ARRAY_V2I16_CLAMP;
3797 Ops.push_back(TexHandle);
3798 Ops.push_back(N->getOperand(2));
3799 Ops.push_back(N->getOperand(3));
3800 Ops.push_back(N->getOperand(4));
3801 Ops.push_back(Chain);
3802 break;
3803 case NVPTXISD::Suld2DArrayV2I32Clamp:
3804 Opc = NVPTX::SULD_2D_ARRAY_V2I32_CLAMP;
3805 Ops.push_back(TexHandle);
3806 Ops.push_back(N->getOperand(2));
3807 Ops.push_back(N->getOperand(3));
3808 Ops.push_back(N->getOperand(4));
3809 Ops.push_back(Chain);
3810 break;
3811 case NVPTXISD::Suld2DArrayV2I64Clamp:
3812 Opc = NVPTX::SULD_2D_ARRAY_V2I64_CLAMP;
3813 Ops.push_back(TexHandle);
3814 Ops.push_back(N->getOperand(2));
3815 Ops.push_back(N->getOperand(3));
3816 Ops.push_back(N->getOperand(4));
3817 Ops.push_back(Chain);
3818 break;
3819 case NVPTXISD::Suld2DArrayV4I8Clamp:
3820 Opc = NVPTX::SULD_2D_ARRAY_V4I8_CLAMP;
3821 Ops.push_back(TexHandle);
3822 Ops.push_back(N->getOperand(2));
3823 Ops.push_back(N->getOperand(3));
3824 Ops.push_back(N->getOperand(4));
3825 Ops.push_back(Chain);
3826 break;
3827 case NVPTXISD::Suld2DArrayV4I16Clamp:
3828 Opc = NVPTX::SULD_2D_ARRAY_V4I16_CLAMP;
3829 Ops.push_back(TexHandle);
3830 Ops.push_back(N->getOperand(2));
3831 Ops.push_back(N->getOperand(3));
3832 Ops.push_back(N->getOperand(4));
3833 Ops.push_back(Chain);
3834 break;
3835 case NVPTXISD::Suld2DArrayV4I32Clamp:
3836 Opc = NVPTX::SULD_2D_ARRAY_V4I32_CLAMP;
3837 Ops.push_back(TexHandle);
3838 Ops.push_back(N->getOperand(2));
3839 Ops.push_back(N->getOperand(3));
3840 Ops.push_back(N->getOperand(4));
3841 Ops.push_back(Chain);
3842 break;
3843 case NVPTXISD::Suld3DI8Clamp:
3844 Opc = NVPTX::SULD_3D_I8_CLAMP;
3845 Ops.push_back(TexHandle);
3846 Ops.push_back(N->getOperand(2));
3847 Ops.push_back(N->getOperand(3));
3848 Ops.push_back(N->getOperand(4));
3849 Ops.push_back(Chain);
3850 break;
3851 case NVPTXISD::Suld3DI16Clamp:
3852 Opc = NVPTX::SULD_3D_I16_CLAMP;
3853 Ops.push_back(TexHandle);
3854 Ops.push_back(N->getOperand(2));
3855 Ops.push_back(N->getOperand(3));
3856 Ops.push_back(N->getOperand(4));
3857 Ops.push_back(Chain);
3858 break;
3859 case NVPTXISD::Suld3DI32Clamp:
3860 Opc = NVPTX::SULD_3D_I32_CLAMP;
3861 Ops.push_back(TexHandle);
3862 Ops.push_back(N->getOperand(2));
3863 Ops.push_back(N->getOperand(3));
3864 Ops.push_back(N->getOperand(4));
3865 Ops.push_back(Chain);
3866 break;
3867 case NVPTXISD::Suld3DI64Clamp:
3868 Opc = NVPTX::SULD_3D_I64_CLAMP;
3869 Ops.push_back(TexHandle);
3870 Ops.push_back(N->getOperand(2));
3871 Ops.push_back(N->getOperand(3));
3872 Ops.push_back(N->getOperand(4));
3873 Ops.push_back(Chain);
3874 break;
3875 case NVPTXISD::Suld3DV2I8Clamp:
3876 Opc = NVPTX::SULD_3D_V2I8_CLAMP;
3877 Ops.push_back(TexHandle);
3878 Ops.push_back(N->getOperand(2));
3879 Ops.push_back(N->getOperand(3));
3880 Ops.push_back(N->getOperand(4));
3881 Ops.push_back(Chain);
3882 break;
3883 case NVPTXISD::Suld3DV2I16Clamp:
3884 Opc = NVPTX::SULD_3D_V2I16_CLAMP;
3885 Ops.push_back(TexHandle);
3886 Ops.push_back(N->getOperand(2));
3887 Ops.push_back(N->getOperand(3));
3888 Ops.push_back(N->getOperand(4));
3889 Ops.push_back(Chain);
3890 break;
3891 case NVPTXISD::Suld3DV2I32Clamp:
3892 Opc = NVPTX::SULD_3D_V2I32_CLAMP;
3893 Ops.push_back(TexHandle);
3894 Ops.push_back(N->getOperand(2));
3895 Ops.push_back(N->getOperand(3));
3896 Ops.push_back(N->getOperand(4));
3897 Ops.push_back(Chain);
3898 break;
3899 case NVPTXISD::Suld3DV2I64Clamp:
3900 Opc = NVPTX::SULD_3D_V2I64_CLAMP;
3901 Ops.push_back(TexHandle);
3902 Ops.push_back(N->getOperand(2));
3903 Ops.push_back(N->getOperand(3));
3904 Ops.push_back(N->getOperand(4));
3905 Ops.push_back(Chain);
3906 break;
3907 case NVPTXISD::Suld3DV4I8Clamp:
3908 Opc = NVPTX::SULD_3D_V4I8_CLAMP;
3909 Ops.push_back(TexHandle);
3910 Ops.push_back(N->getOperand(2));
3911 Ops.push_back(N->getOperand(3));
3912 Ops.push_back(N->getOperand(4));
3913 Ops.push_back(Chain);
3914 break;
3915 case NVPTXISD::Suld3DV4I16Clamp:
3916 Opc = NVPTX::SULD_3D_V4I16_CLAMP;
3917 Ops.push_back(TexHandle);
3918 Ops.push_back(N->getOperand(2));
3919 Ops.push_back(N->getOperand(3));
3920 Ops.push_back(N->getOperand(4));
3921 Ops.push_back(Chain);
3922 break;
3923 case NVPTXISD::Suld3DV4I32Clamp:
3924 Opc = NVPTX::SULD_3D_V4I32_CLAMP;
3925 Ops.push_back(TexHandle);
3926 Ops.push_back(N->getOperand(2));
3927 Ops.push_back(N->getOperand(3));
3928 Ops.push_back(N->getOperand(4));
3929 Ops.push_back(Chain);
3930 break;
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]3931 case NVPTXISD::Suld1DI8Trap:
3932 Opc = NVPTX::SULD_1D_I8_TRAP;
3933 Ops.push_back(TexHandle);
3934 Ops.push_back(N->getOperand(2));
3935 Ops.push_back(Chain);
3936 break;
3937 case NVPTXISD::Suld1DI16Trap:
3938 Opc = NVPTX::SULD_1D_I16_TRAP;
3939 Ops.push_back(TexHandle);
3940 Ops.push_back(N->getOperand(2));
3941 Ops.push_back(Chain);
3942 break;
3943 case NVPTXISD::Suld1DI32Trap:
3944 Opc = NVPTX::SULD_1D_I32_TRAP;
3945 Ops.push_back(TexHandle);
3946 Ops.push_back(N->getOperand(2));
3947 Ops.push_back(Chain);
3948 break;
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]3949 case NVPTXISD::Suld1DI64Trap:
3950 Opc = NVPTX::SULD_1D_I64_TRAP;
3951 Ops.push_back(TexHandle);
3952 Ops.push_back(N->getOperand(2));
3953 Ops.push_back(Chain);
3954 break;
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]3955 case NVPTXISD::Suld1DV2I8Trap:
3956 Opc = NVPTX::SULD_1D_V2I8_TRAP;
3957 Ops.push_back(TexHandle);
3958 Ops.push_back(N->getOperand(2));
3959 Ops.push_back(Chain);
3960 break;
3961 case NVPTXISD::Suld1DV2I16Trap:
3962 Opc = NVPTX::SULD_1D_V2I16_TRAP;
3963 Ops.push_back(TexHandle);
3964 Ops.push_back(N->getOperand(2));
3965 Ops.push_back(Chain);
3966 break;
3967 case NVPTXISD::Suld1DV2I32Trap:
3968 Opc = NVPTX::SULD_1D_V2I32_TRAP;
3969 Ops.push_back(TexHandle);
3970 Ops.push_back(N->getOperand(2));
3971 Ops.push_back(Chain);
3972 break;
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]3973 case NVPTXISD::Suld1DV2I64Trap:
3974 Opc = NVPTX::SULD_1D_V2I64_TRAP;
3975 Ops.push_back(TexHandle);
3976 Ops.push_back(N->getOperand(2));
3977 Ops.push_back(Chain);
3978 break;
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]3979 case NVPTXISD::Suld1DV4I8Trap:
3980 Opc = NVPTX::SULD_1D_V4I8_TRAP;
3981 Ops.push_back(TexHandle);
3982 Ops.push_back(N->getOperand(2));
3983 Ops.push_back(Chain);
3984 break;
3985 case NVPTXISD::Suld1DV4I16Trap:
3986 Opc = NVPTX::SULD_1D_V4I16_TRAP;
3987 Ops.push_back(TexHandle);
3988 Ops.push_back(N->getOperand(2));
3989 Ops.push_back(Chain);
3990 break;
3991 case NVPTXISD::Suld1DV4I32Trap:
3992 Opc = NVPTX::SULD_1D_V4I32_TRAP;
3993 Ops.push_back(TexHandle);
3994 Ops.push_back(N->getOperand(2));
3995 Ops.push_back(Chain);
3996 break;
3997 case NVPTXISD::Suld1DArrayI8Trap:
3998 Opc = NVPTX::SULD_1D_ARRAY_I8_TRAP;
3999 Ops.push_back(TexHandle);
4000 Ops.push_back(N->getOperand(2));
4001 Ops.push_back(N->getOperand(3));
4002 Ops.push_back(Chain);
4003 break;
4004 case NVPTXISD::Suld1DArrayI16Trap:
4005 Opc = NVPTX::SULD_1D_ARRAY_I16_TRAP;
4006 Ops.push_back(TexHandle);
4007 Ops.push_back(N->getOperand(2));
4008 Ops.push_back(N->getOperand(3));
4009 Ops.push_back(Chain);
4010 break;
4011 case NVPTXISD::Suld1DArrayI32Trap:
4012 Opc = NVPTX::SULD_1D_ARRAY_I32_TRAP;
4013 Ops.push_back(TexHandle);
4014 Ops.push_back(N->getOperand(2));
4015 Ops.push_back(N->getOperand(3));
4016 Ops.push_back(Chain);
4017 break;
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]4018 case NVPTXISD::Suld1DArrayI64Trap:
4019 Opc = NVPTX::SULD_1D_ARRAY_I64_TRAP;
4020 Ops.push_back(TexHandle);
4021 Ops.push_back(N->getOperand(2));
4022 Ops.push_back(N->getOperand(3));
4023 Ops.push_back(Chain);
4024 break;
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]4025 case NVPTXISD::Suld1DArrayV2I8Trap:
4026 Opc = NVPTX::SULD_1D_ARRAY_V2I8_TRAP;
4027 Ops.push_back(TexHandle);
4028 Ops.push_back(N->getOperand(2));
4029 Ops.push_back(N->getOperand(3));
4030 Ops.push_back(Chain);
4031 break;
4032 case NVPTXISD::Suld1DArrayV2I16Trap:
4033 Opc = NVPTX::SULD_1D_ARRAY_V2I16_TRAP;
4034 Ops.push_back(TexHandle);
4035 Ops.push_back(N->getOperand(2));
4036 Ops.push_back(N->getOperand(3));
4037 Ops.push_back(Chain);
4038 break;
4039 case NVPTXISD::Suld1DArrayV2I32Trap:
4040 Opc = NVPTX::SULD_1D_ARRAY_V2I32_TRAP;
4041 Ops.push_back(TexHandle);
4042 Ops.push_back(N->getOperand(2));
4043 Ops.push_back(N->getOperand(3));
4044 Ops.push_back(Chain);
4045 break;
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]4046 case NVPTXISD::Suld1DArrayV2I64Trap:
4047 Opc = NVPTX::SULD_1D_ARRAY_V2I64_TRAP;
4048 Ops.push_back(TexHandle);
4049 Ops.push_back(N->getOperand(2));
4050 Ops.push_back(N->getOperand(3));
4051 Ops.push_back(Chain);
4052 break;
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]4053 case NVPTXISD::Suld1DArrayV4I8Trap:
4054 Opc = NVPTX::SULD_1D_ARRAY_V4I8_TRAP;
4055 Ops.push_back(TexHandle);
4056 Ops.push_back(N->getOperand(2));
4057 Ops.push_back(N->getOperand(3));
4058 Ops.push_back(Chain);
4059 break;
4060 case NVPTXISD::Suld1DArrayV4I16Trap:
4061 Opc = NVPTX::SULD_1D_ARRAY_V4I16_TRAP;
4062 Ops.push_back(TexHandle);
4063 Ops.push_back(N->getOperand(2));
4064 Ops.push_back(N->getOperand(3));
4065 Ops.push_back(Chain);
4066 break;
4067 case NVPTXISD::Suld1DArrayV4I32Trap:
4068 Opc = NVPTX::SULD_1D_ARRAY_V4I32_TRAP;
4069 Ops.push_back(TexHandle);
4070 Ops.push_back(N->getOperand(2));
4071 Ops.push_back(N->getOperand(3));
4072 Ops.push_back(Chain);
4073 break;
4074 case NVPTXISD::Suld2DI8Trap:
4075 Opc = NVPTX::SULD_2D_I8_TRAP;
4076 Ops.push_back(TexHandle);
4077 Ops.push_back(N->getOperand(2));
4078 Ops.push_back(N->getOperand(3));
4079 Ops.push_back(Chain);
4080 break;
4081 case NVPTXISD::Suld2DI16Trap:
4082 Opc = NVPTX::SULD_2D_I16_TRAP;
4083 Ops.push_back(TexHandle);
4084 Ops.push_back(N->getOperand(2));
4085 Ops.push_back(N->getOperand(3));
4086 Ops.push_back(Chain);
4087 break;
4088 case NVPTXISD::Suld2DI32Trap:
4089 Opc = NVPTX::SULD_2D_I32_TRAP;
4090 Ops.push_back(TexHandle);
4091 Ops.push_back(N->getOperand(2));
4092 Ops.push_back(N->getOperand(3));
4093 Ops.push_back(Chain);
4094 break;
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]4095 case NVPTXISD::Suld2DI64Trap:
4096 Opc = NVPTX::SULD_2D_I64_TRAP;
4097 Ops.push_back(TexHandle);
4098 Ops.push_back(N->getOperand(2));
4099 Ops.push_back(N->getOperand(3));
4100 Ops.push_back(Chain);
4101 break;
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]4102 case NVPTXISD::Suld2DV2I8Trap:
4103 Opc = NVPTX::SULD_2D_V2I8_TRAP;
4104 Ops.push_back(TexHandle);
4105 Ops.push_back(N->getOperand(2));
4106 Ops.push_back(N->getOperand(3));
4107 Ops.push_back(Chain);
4108 break;
4109 case NVPTXISD::Suld2DV2I16Trap:
4110 Opc = NVPTX::SULD_2D_V2I16_TRAP;
4111 Ops.push_back(TexHandle);
4112 Ops.push_back(N->getOperand(2));
4113 Ops.push_back(N->getOperand(3));
4114 Ops.push_back(Chain);
4115 break;
4116 case NVPTXISD::Suld2DV2I32Trap:
4117 Opc = NVPTX::SULD_2D_V2I32_TRAP;
4118 Ops.push_back(TexHandle);
4119 Ops.push_back(N->getOperand(2));
4120 Ops.push_back(N->getOperand(3));
4121 Ops.push_back(Chain);
4122 break;
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]4123 case NVPTXISD::Suld2DV2I64Trap:
4124 Opc = NVPTX::SULD_2D_V2I64_TRAP;
4125 Ops.push_back(TexHandle);
4126 Ops.push_back(N->getOperand(2));
4127 Ops.push_back(N->getOperand(3));
4128 Ops.push_back(Chain);
4129 break;
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]4130 case NVPTXISD::Suld2DV4I8Trap:
4131 Opc = NVPTX::SULD_2D_V4I8_TRAP;
4132 Ops.push_back(TexHandle);
4133 Ops.push_back(N->getOperand(2));
4134 Ops.push_back(N->getOperand(3));
4135 Ops.push_back(Chain);
4136 break;
4137 case NVPTXISD::Suld2DV4I16Trap:
4138 Opc = NVPTX::SULD_2D_V4I16_TRAP;
4139 Ops.push_back(TexHandle);
4140 Ops.push_back(N->getOperand(2));
4141 Ops.push_back(N->getOperand(3));
4142 Ops.push_back(Chain);
4143 break;
4144 case NVPTXISD::Suld2DV4I32Trap:
4145 Opc = NVPTX::SULD_2D_V4I32_TRAP;
4146 Ops.push_back(TexHandle);
4147 Ops.push_back(N->getOperand(2));
4148 Ops.push_back(N->getOperand(3));
4149 Ops.push_back(Chain);
4150 break;
4151 case NVPTXISD::Suld2DArrayI8Trap:
4152 Opc = NVPTX::SULD_2D_ARRAY_I8_TRAP;
4153 Ops.push_back(TexHandle);
4154 Ops.push_back(N->getOperand(2));
4155 Ops.push_back(N->getOperand(3));
4156 Ops.push_back(N->getOperand(4));
4157 Ops.push_back(Chain);
4158 break;
4159 case NVPTXISD::Suld2DArrayI16Trap:
4160 Opc = NVPTX::SULD_2D_ARRAY_I16_TRAP;
4161 Ops.push_back(TexHandle);
4162 Ops.push_back(N->getOperand(2));
4163 Ops.push_back(N->getOperand(3));
4164 Ops.push_back(N->getOperand(4));
4165 Ops.push_back(Chain);
4166 break;
4167 case NVPTXISD::Suld2DArrayI32Trap:
4168 Opc = NVPTX::SULD_2D_ARRAY_I32_TRAP;
4169 Ops.push_back(TexHandle);
4170 Ops.push_back(N->getOperand(2));
4171 Ops.push_back(N->getOperand(3));
4172 Ops.push_back(N->getOperand(4));
4173 Ops.push_back(Chain);
4174 break;
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]4175 case NVPTXISD::Suld2DArrayI64Trap:
4176 Opc = NVPTX::SULD_2D_ARRAY_I64_TRAP;
4177 Ops.push_back(TexHandle);
4178 Ops.push_back(N->getOperand(2));
4179 Ops.push_back(N->getOperand(3));
4180 Ops.push_back(N->getOperand(4));
4181 Ops.push_back(Chain);
4182 break;
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]4183 case NVPTXISD::Suld2DArrayV2I8Trap:
4184 Opc = NVPTX::SULD_2D_ARRAY_V2I8_TRAP;
4185 Ops.push_back(TexHandle);
4186 Ops.push_back(N->getOperand(2));
4187 Ops.push_back(N->getOperand(3));
4188 Ops.push_back(N->getOperand(4));
4189 Ops.push_back(Chain);
4190 break;
4191 case NVPTXISD::Suld2DArrayV2I16Trap:
4192 Opc = NVPTX::SULD_2D_ARRAY_V2I16_TRAP;
4193 Ops.push_back(TexHandle);
4194 Ops.push_back(N->getOperand(2));
4195 Ops.push_back(N->getOperand(3));
4196 Ops.push_back(N->getOperand(4));
4197 Ops.push_back(Chain);
4198 break;
4199 case NVPTXISD::Suld2DArrayV2I32Trap:
4200 Opc = NVPTX::SULD_2D_ARRAY_V2I32_TRAP;
4201 Ops.push_back(TexHandle);
4202 Ops.push_back(N->getOperand(2));
4203 Ops.push_back(N->getOperand(3));
4204 Ops.push_back(N->getOperand(4));
4205 Ops.push_back(Chain);
4206 break;
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]4207 case NVPTXISD::Suld2DArrayV2I64Trap:
4208 Opc = NVPTX::SULD_2D_ARRAY_V2I64_TRAP;
4209 Ops.push_back(TexHandle);
4210 Ops.push_back(N->getOperand(2));
4211 Ops.push_back(N->getOperand(3));
4212 Ops.push_back(N->getOperand(4));
4213 Ops.push_back(Chain);
4214 break;
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]4215 case NVPTXISD::Suld2DArrayV4I8Trap:
4216 Opc = NVPTX::SULD_2D_ARRAY_V4I8_TRAP;
4217 Ops.push_back(TexHandle);
4218 Ops.push_back(N->getOperand(2));
4219 Ops.push_back(N->getOperand(3));
4220 Ops.push_back(N->getOperand(4));
4221 Ops.push_back(Chain);
4222 break;
4223 case NVPTXISD::Suld2DArrayV4I16Trap:
4224 Opc = NVPTX::SULD_2D_ARRAY_V4I16_TRAP;
4225 Ops.push_back(TexHandle);
4226 Ops.push_back(N->getOperand(2));
4227 Ops.push_back(N->getOperand(3));
4228 Ops.push_back(N->getOperand(4));
4229 Ops.push_back(Chain);
4230 break;
4231 case NVPTXISD::Suld2DArrayV4I32Trap:
4232 Opc = NVPTX::SULD_2D_ARRAY_V4I32_TRAP;
4233 Ops.push_back(TexHandle);
4234 Ops.push_back(N->getOperand(2));
4235 Ops.push_back(N->getOperand(3));
4236 Ops.push_back(N->getOperand(4));
4237 Ops.push_back(Chain);
4238 break;
4239 case NVPTXISD::Suld3DI8Trap:
4240 Opc = NVPTX::SULD_3D_I8_TRAP;
4241 Ops.push_back(TexHandle);
4242 Ops.push_back(N->getOperand(2));
4243 Ops.push_back(N->getOperand(3));
4244 Ops.push_back(N->getOperand(4));
4245 Ops.push_back(Chain);
4246 break;
4247 case NVPTXISD::Suld3DI16Trap:
4248 Opc = NVPTX::SULD_3D_I16_TRAP;
4249 Ops.push_back(TexHandle);
4250 Ops.push_back(N->getOperand(2));
4251 Ops.push_back(N->getOperand(3));
4252 Ops.push_back(N->getOperand(4));
4253 Ops.push_back(Chain);
4254 break;
4255 case NVPTXISD::Suld3DI32Trap:
4256 Opc = NVPTX::SULD_3D_I32_TRAP;
4257 Ops.push_back(TexHandle);
4258 Ops.push_back(N->getOperand(2));
4259 Ops.push_back(N->getOperand(3));
4260 Ops.push_back(N->getOperand(4));
4261 Ops.push_back(Chain);
4262 break;
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]4263 case NVPTXISD::Suld3DI64Trap:
4264 Opc = NVPTX::SULD_3D_I64_TRAP;
4265 Ops.push_back(TexHandle);
4266 Ops.push_back(N->getOperand(2));
4267 Ops.push_back(N->getOperand(3));
4268 Ops.push_back(N->getOperand(4));
4269 Ops.push_back(Chain);
4270 break;
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]4271 case NVPTXISD::Suld3DV2I8Trap:
4272 Opc = NVPTX::SULD_3D_V2I8_TRAP;
4273 Ops.push_back(TexHandle);
4274 Ops.push_back(N->getOperand(2));
4275 Ops.push_back(N->getOperand(3));
4276 Ops.push_back(N->getOperand(4));
4277 Ops.push_back(Chain);
4278 break;
4279 case NVPTXISD::Suld3DV2I16Trap:
4280 Opc = NVPTX::SULD_3D_V2I16_TRAP;
4281 Ops.push_back(TexHandle);
4282 Ops.push_back(N->getOperand(2));
4283 Ops.push_back(N->getOperand(3));
4284 Ops.push_back(N->getOperand(4));
4285 Ops.push_back(Chain);
4286 break;
4287 case NVPTXISD::Suld3DV2I32Trap:
4288 Opc = NVPTX::SULD_3D_V2I32_TRAP;
4289 Ops.push_back(TexHandle);
4290 Ops.push_back(N->getOperand(2));
4291 Ops.push_back(N->getOperand(3));
4292 Ops.push_back(N->getOperand(4));
4293 Ops.push_back(Chain);
4294 break;
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]4295 case NVPTXISD::Suld3DV2I64Trap:
4296 Opc = NVPTX::SULD_3D_V2I64_TRAP;
4297 Ops.push_back(TexHandle);
4298 Ops.push_back(N->getOperand(2));
4299 Ops.push_back(N->getOperand(3));
4300 Ops.push_back(N->getOperand(4));
4301 Ops.push_back(Chain);
4302 break;
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]4303 case NVPTXISD::Suld3DV4I8Trap:
4304 Opc = NVPTX::SULD_3D_V4I8_TRAP;
4305 Ops.push_back(TexHandle);
4306 Ops.push_back(N->getOperand(2));
4307 Ops.push_back(N->getOperand(3));
4308 Ops.push_back(N->getOperand(4));
4309 Ops.push_back(Chain);
4310 break;
4311 case NVPTXISD::Suld3DV4I16Trap:
4312 Opc = NVPTX::SULD_3D_V4I16_TRAP;
4313 Ops.push_back(TexHandle);
4314 Ops.push_back(N->getOperand(2));
4315 Ops.push_back(N->getOperand(3));
4316 Ops.push_back(N->getOperand(4));
4317 Ops.push_back(Chain);
4318 break;
4319 case NVPTXISD::Suld3DV4I32Trap:
4320 Opc = NVPTX::SULD_3D_V4I32_TRAP;
4321 Ops.push_back(TexHandle);
4322 Ops.push_back(N->getOperand(2));
4323 Ops.push_back(N->getOperand(3));
4324 Ops.push_back(N->getOperand(4));
4325 Ops.push_back(Chain);
4326 break;
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]4327 case NVPTXISD::Suld1DI8Zero:
4328 Opc = NVPTX::SULD_1D_I8_ZERO;
4329 Ops.push_back(TexHandle);
4330 Ops.push_back(N->getOperand(2));
4331 Ops.push_back(Chain);
4332 break;
4333 case NVPTXISD::Suld1DI16Zero:
4334 Opc = NVPTX::SULD_1D_I16_ZERO;
4335 Ops.push_back(TexHandle);
4336 Ops.push_back(N->getOperand(2));
4337 Ops.push_back(Chain);
4338 break;
4339 case NVPTXISD::Suld1DI32Zero:
4340 Opc = NVPTX::SULD_1D_I32_ZERO;
4341 Ops.push_back(TexHandle);
4342 Ops.push_back(N->getOperand(2));
4343 Ops.push_back(Chain);
4344 break;
4345 case NVPTXISD::Suld1DI64Zero:
4346 Opc = NVPTX::SULD_1D_I64_ZERO;
4347 Ops.push_back(TexHandle);
4348 Ops.push_back(N->getOperand(2));
4349 Ops.push_back(Chain);
4350 break;
4351 case NVPTXISD::Suld1DV2I8Zero:
4352 Opc = NVPTX::SULD_1D_V2I8_ZERO;
4353 Ops.push_back(TexHandle);
4354 Ops.push_back(N->getOperand(2));
4355 Ops.push_back(Chain);
4356 break;
4357 case NVPTXISD::Suld1DV2I16Zero:
4358 Opc = NVPTX::SULD_1D_V2I16_ZERO;
4359 Ops.push_back(TexHandle);
4360 Ops.push_back(N->getOperand(2));
4361 Ops.push_back(Chain);
4362 break;
4363 case NVPTXISD::Suld1DV2I32Zero:
4364 Opc = NVPTX::SULD_1D_V2I32_ZERO;
4365 Ops.push_back(TexHandle);
4366 Ops.push_back(N->getOperand(2));
4367 Ops.push_back(Chain);
4368 break;
4369 case NVPTXISD::Suld1DV2I64Zero:
4370 Opc = NVPTX::SULD_1D_V2I64_ZERO;
4371 Ops.push_back(TexHandle);
4372 Ops.push_back(N->getOperand(2));
4373 Ops.push_back(Chain);
4374 break;
4375 case NVPTXISD::Suld1DV4I8Zero:
4376 Opc = NVPTX::SULD_1D_V4I8_ZERO;
4377 Ops.push_back(TexHandle);
4378 Ops.push_back(N->getOperand(2));
4379 Ops.push_back(Chain);
4380 break;
4381 case NVPTXISD::Suld1DV4I16Zero:
4382 Opc = NVPTX::SULD_1D_V4I16_ZERO;
4383 Ops.push_back(TexHandle);
4384 Ops.push_back(N->getOperand(2));
4385 Ops.push_back(Chain);
4386 break;
4387 case NVPTXISD::Suld1DV4I32Zero:
4388 Opc = NVPTX::SULD_1D_V4I32_ZERO;
4389 Ops.push_back(TexHandle);
4390 Ops.push_back(N->getOperand(2));
4391 Ops.push_back(Chain);
4392 break;
4393 case NVPTXISD::Suld1DArrayI8Zero:
4394 Opc = NVPTX::SULD_1D_ARRAY_I8_ZERO;
4395 Ops.push_back(TexHandle);
4396 Ops.push_back(N->getOperand(2));
4397 Ops.push_back(N->getOperand(3));
4398 Ops.push_back(Chain);
4399 break;
4400 case NVPTXISD::Suld1DArrayI16Zero:
4401 Opc = NVPTX::SULD_1D_ARRAY_I16_ZERO;
4402 Ops.push_back(TexHandle);
4403 Ops.push_back(N->getOperand(2));
4404 Ops.push_back(N->getOperand(3));
4405 Ops.push_back(Chain);
4406 break;
4407 case NVPTXISD::Suld1DArrayI32Zero:
4408 Opc = NVPTX::SULD_1D_ARRAY_I32_ZERO;
4409 Ops.push_back(TexHandle);
4410 Ops.push_back(N->getOperand(2));
4411 Ops.push_back(N->getOperand(3));
4412 Ops.push_back(Chain);
4413 break;
4414 case NVPTXISD::Suld1DArrayI64Zero:
4415 Opc = NVPTX::SULD_1D_ARRAY_I64_ZERO;
4416 Ops.push_back(TexHandle);
4417 Ops.push_back(N->getOperand(2));
4418 Ops.push_back(N->getOperand(3));
4419 Ops.push_back(Chain);
4420 break;
4421 case NVPTXISD::Suld1DArrayV2I8Zero:
4422 Opc = NVPTX::SULD_1D_ARRAY_V2I8_ZERO;
4423 Ops.push_back(TexHandle);
4424 Ops.push_back(N->getOperand(2));
4425 Ops.push_back(N->getOperand(3));
4426 Ops.push_back(Chain);
4427 break;
4428 case NVPTXISD::Suld1DArrayV2I16Zero:
4429 Opc = NVPTX::SULD_1D_ARRAY_V2I16_ZERO;
4430 Ops.push_back(TexHandle);
4431 Ops.push_back(N->getOperand(2));
4432 Ops.push_back(N->getOperand(3));
4433 Ops.push_back(Chain);
4434 break;
4435 case NVPTXISD::Suld1DArrayV2I32Zero:
4436 Opc = NVPTX::SULD_1D_ARRAY_V2I32_ZERO;
4437 Ops.push_back(TexHandle);
4438 Ops.push_back(N->getOperand(2));
4439 Ops.push_back(N->getOperand(3));
4440 Ops.push_back(Chain);
4441 break;
4442 case NVPTXISD::Suld1DArrayV2I64Zero:
4443 Opc = NVPTX::SULD_1D_ARRAY_V2I64_ZERO;
4444 Ops.push_back(TexHandle);
4445 Ops.push_back(N->getOperand(2));
4446 Ops.push_back(N->getOperand(3));
4447 Ops.push_back(Chain);
4448 break;
4449 case NVPTXISD::Suld1DArrayV4I8Zero:
4450 Opc = NVPTX::SULD_1D_ARRAY_V4I8_ZERO;
4451 Ops.push_back(TexHandle);
4452 Ops.push_back(N->getOperand(2));
4453 Ops.push_back(N->getOperand(3));
4454 Ops.push_back(Chain);
4455 break;
4456 case NVPTXISD::Suld1DArrayV4I16Zero:
4457 Opc = NVPTX::SULD_1D_ARRAY_V4I16_ZERO;
4458 Ops.push_back(TexHandle);
4459 Ops.push_back(N->getOperand(2));
4460 Ops.push_back(N->getOperand(3));
4461 Ops.push_back(Chain);
4462 break;
4463 case NVPTXISD::Suld1DArrayV4I32Zero:
4464 Opc = NVPTX::SULD_1D_ARRAY_V4I32_ZERO;
4465 Ops.push_back(TexHandle);
4466 Ops.push_back(N->getOperand(2));
4467 Ops.push_back(N->getOperand(3));
4468 Ops.push_back(Chain);
4469 break;
4470 case NVPTXISD::Suld2DI8Zero:
4471 Opc = NVPTX::SULD_2D_I8_ZERO;
4472 Ops.push_back(TexHandle);
4473 Ops.push_back(N->getOperand(2));
4474 Ops.push_back(N->getOperand(3));
4475 Ops.push_back(Chain);
4476 break;
4477 case NVPTXISD::Suld2DI16Zero:
4478 Opc = NVPTX::SULD_2D_I16_ZERO;
4479 Ops.push_back(TexHandle);
4480 Ops.push_back(N->getOperand(2));
4481 Ops.push_back(N->getOperand(3));
4482 Ops.push_back(Chain);
4483 break;
4484 case NVPTXISD::Suld2DI32Zero:
4485 Opc = NVPTX::SULD_2D_I32_ZERO;
4486 Ops.push_back(TexHandle);
4487 Ops.push_back(N->getOperand(2));
4488 Ops.push_back(N->getOperand(3));
4489 Ops.push_back(Chain);
4490 break;
4491 case NVPTXISD::Suld2DI64Zero:
4492 Opc = NVPTX::SULD_2D_I64_ZERO;
4493 Ops.push_back(TexHandle);
4494 Ops.push_back(N->getOperand(2));
4495 Ops.push_back(N->getOperand(3));
4496 Ops.push_back(Chain);
4497 break;
4498 case NVPTXISD::Suld2DV2I8Zero:
4499 Opc = NVPTX::SULD_2D_V2I8_ZERO;
4500 Ops.push_back(TexHandle);
4501 Ops.push_back(N->getOperand(2));
4502 Ops.push_back(N->getOperand(3));
4503 Ops.push_back(Chain);
4504 break;
4505 case NVPTXISD::Suld2DV2I16Zero:
4506 Opc = NVPTX::SULD_2D_V2I16_ZERO;
4507 Ops.push_back(TexHandle);
4508 Ops.push_back(N->getOperand(2));
4509 Ops.push_back(N->getOperand(3));
4510 Ops.push_back(Chain);
4511 break;
4512 case NVPTXISD::Suld2DV2I32Zero:
4513 Opc = NVPTX::SULD_2D_V2I32_ZERO;
4514 Ops.push_back(TexHandle);
4515 Ops.push_back(N->getOperand(2));
4516 Ops.push_back(N->getOperand(3));
4517 Ops.push_back(Chain);
4518 break;
4519 case NVPTXISD::Suld2DV2I64Zero:
4520 Opc = NVPTX::SULD_2D_V2I64_ZERO;
4521 Ops.push_back(TexHandle);
4522 Ops.push_back(N->getOperand(2));
4523 Ops.push_back(N->getOperand(3));
4524 Ops.push_back(Chain);
4525 break;
4526 case NVPTXISD::Suld2DV4I8Zero:
4527 Opc = NVPTX::SULD_2D_V4I8_ZERO;
4528 Ops.push_back(TexHandle);
4529 Ops.push_back(N->getOperand(2));
4530 Ops.push_back(N->getOperand(3));
4531 Ops.push_back(Chain);
4532 break;
4533 case NVPTXISD::Suld2DV4I16Zero:
4534 Opc = NVPTX::SULD_2D_V4I16_ZERO;
4535 Ops.push_back(TexHandle);
4536 Ops.push_back(N->getOperand(2));
4537 Ops.push_back(N->getOperand(3));
4538 Ops.push_back(Chain);
4539 break;
4540 case NVPTXISD::Suld2DV4I32Zero:
4541 Opc = NVPTX::SULD_2D_V4I32_ZERO;
4542 Ops.push_back(TexHandle);
4543 Ops.push_back(N->getOperand(2));
4544 Ops.push_back(N->getOperand(3));
4545 Ops.push_back(Chain);
4546 break;
4547 case NVPTXISD::Suld2DArrayI8Zero:
4548 Opc = NVPTX::SULD_2D_ARRAY_I8_ZERO;
4549 Ops.push_back(TexHandle);
4550 Ops.push_back(N->getOperand(2));
4551 Ops.push_back(N->getOperand(3));
4552 Ops.push_back(N->getOperand(4));
4553 Ops.push_back(Chain);
4554 break;
4555 case NVPTXISD::Suld2DArrayI16Zero:
4556 Opc = NVPTX::SULD_2D_ARRAY_I16_ZERO;
4557 Ops.push_back(TexHandle);
4558 Ops.push_back(N->getOperand(2));
4559 Ops.push_back(N->getOperand(3));
4560 Ops.push_back(N->getOperand(4));
4561 Ops.push_back(Chain);
4562 break;
4563 case NVPTXISD::Suld2DArrayI32Zero:
4564 Opc = NVPTX::SULD_2D_ARRAY_I32_ZERO;
4565 Ops.push_back(TexHandle);
4566 Ops.push_back(N->getOperand(2));
4567 Ops.push_back(N->getOperand(3));
4568 Ops.push_back(N->getOperand(4));
4569 Ops.push_back(Chain);
4570 break;
4571 case NVPTXISD::Suld2DArrayI64Zero:
4572 Opc = NVPTX::SULD_2D_ARRAY_I64_ZERO;
4573 Ops.push_back(TexHandle);
4574 Ops.push_back(N->getOperand(2));
4575 Ops.push_back(N->getOperand(3));
4576 Ops.push_back(N->getOperand(4));
4577 Ops.push_back(Chain);
4578 break;
4579 case NVPTXISD::Suld2DArrayV2I8Zero:
4580 Opc = NVPTX::SULD_2D_ARRAY_V2I8_ZERO;
4581 Ops.push_back(TexHandle);
4582 Ops.push_back(N->getOperand(2));
4583 Ops.push_back(N->getOperand(3));
4584 Ops.push_back(N->getOperand(4));
4585 Ops.push_back(Chain);
4586 break;
4587 case NVPTXISD::Suld2DArrayV2I16Zero:
4588 Opc = NVPTX::SULD_2D_ARRAY_V2I16_ZERO;
4589 Ops.push_back(TexHandle);
4590 Ops.push_back(N->getOperand(2));
4591 Ops.push_back(N->getOperand(3));
4592 Ops.push_back(N->getOperand(4));
4593 Ops.push_back(Chain);
4594 break;
4595 case NVPTXISD::Suld2DArrayV2I32Zero:
4596 Opc = NVPTX::SULD_2D_ARRAY_V2I32_ZERO;
4597 Ops.push_back(TexHandle);
4598 Ops.push_back(N->getOperand(2));
4599 Ops.push_back(N->getOperand(3));
4600 Ops.push_back(N->getOperand(4));
4601 Ops.push_back(Chain);
4602 break;
4603 case NVPTXISD::Suld2DArrayV2I64Zero:
4604 Opc = NVPTX::SULD_2D_ARRAY_V2I64_ZERO;
4605 Ops.push_back(TexHandle);
4606 Ops.push_back(N->getOperand(2));
4607 Ops.push_back(N->getOperand(3));
4608 Ops.push_back(N->getOperand(4));
4609 Ops.push_back(Chain);
4610 break;
4611 case NVPTXISD::Suld2DArrayV4I8Zero:
4612 Opc = NVPTX::SULD_2D_ARRAY_V4I8_ZERO;
4613 Ops.push_back(TexHandle);
4614 Ops.push_back(N->getOperand(2));
4615 Ops.push_back(N->getOperand(3));
4616 Ops.push_back(N->getOperand(4));
4617 Ops.push_back(Chain);
4618 break;
4619 case NVPTXISD::Suld2DArrayV4I16Zero:
4620 Opc = NVPTX::SULD_2D_ARRAY_V4I16_ZERO;
4621 Ops.push_back(TexHandle);
4622 Ops.push_back(N->getOperand(2));
4623 Ops.push_back(N->getOperand(3));
4624 Ops.push_back(N->getOperand(4));
4625 Ops.push_back(Chain);
4626 break;
4627 case NVPTXISD::Suld2DArrayV4I32Zero:
4628 Opc = NVPTX::SULD_2D_ARRAY_V4I32_ZERO;
4629 Ops.push_back(TexHandle);
4630 Ops.push_back(N->getOperand(2));
4631 Ops.push_back(N->getOperand(3));
4632 Ops.push_back(N->getOperand(4));
4633 Ops.push_back(Chain);
4634 break;
4635 case NVPTXISD::Suld3DI8Zero:
4636 Opc = NVPTX::SULD_3D_I8_ZERO;
4637 Ops.push_back(TexHandle);
4638 Ops.push_back(N->getOperand(2));
4639 Ops.push_back(N->getOperand(3));
4640 Ops.push_back(N->getOperand(4));
4641 Ops.push_back(Chain);
4642 break;
4643 case NVPTXISD::Suld3DI16Zero:
4644 Opc = NVPTX::SULD_3D_I16_ZERO;
4645 Ops.push_back(TexHandle);
4646 Ops.push_back(N->getOperand(2));
4647 Ops.push_back(N->getOperand(3));
4648 Ops.push_back(N->getOperand(4));
4649 Ops.push_back(Chain);
4650 break;
4651 case NVPTXISD::Suld3DI32Zero:
4652 Opc = NVPTX::SULD_3D_I32_ZERO;
4653 Ops.push_back(TexHandle);
4654 Ops.push_back(N->getOperand(2));
4655 Ops.push_back(N->getOperand(3));
4656 Ops.push_back(N->getOperand(4));
4657 Ops.push_back(Chain);
4658 break;
4659 case NVPTXISD::Suld3DI64Zero:
4660 Opc = NVPTX::SULD_3D_I64_ZERO;
4661 Ops.push_back(TexHandle);
4662 Ops.push_back(N->getOperand(2));
4663 Ops.push_back(N->getOperand(3));
4664 Ops.push_back(N->getOperand(4));
4665 Ops.push_back(Chain);
4666 break;
4667 case NVPTXISD::Suld3DV2I8Zero:
4668 Opc = NVPTX::SULD_3D_V2I8_ZERO;
4669 Ops.push_back(TexHandle);
4670 Ops.push_back(N->getOperand(2));
4671 Ops.push_back(N->getOperand(3));
4672 Ops.push_back(N->getOperand(4));
4673 Ops.push_back(Chain);
4674 break;
4675 case NVPTXISD::Suld3DV2I16Zero:
4676 Opc = NVPTX::SULD_3D_V2I16_ZERO;
4677 Ops.push_back(TexHandle);
4678 Ops.push_back(N->getOperand(2));
4679 Ops.push_back(N->getOperand(3));
4680 Ops.push_back(N->getOperand(4));
4681 Ops.push_back(Chain);
4682 break;
4683 case NVPTXISD::Suld3DV2I32Zero:
4684 Opc = NVPTX::SULD_3D_V2I32_ZERO;
4685 Ops.push_back(TexHandle);
4686 Ops.push_back(N->getOperand(2));
4687 Ops.push_back(N->getOperand(3));
4688 Ops.push_back(N->getOperand(4));
4689 Ops.push_back(Chain);
4690 break;
4691 case NVPTXISD::Suld3DV2I64Zero:
4692 Opc = NVPTX::SULD_3D_V2I64_ZERO;
4693 Ops.push_back(TexHandle);
4694 Ops.push_back(N->getOperand(2));
4695 Ops.push_back(N->getOperand(3));
4696 Ops.push_back(N->getOperand(4));
4697 Ops.push_back(Chain);
4698 break;
4699 case NVPTXISD::Suld3DV4I8Zero:
4700 Opc = NVPTX::SULD_3D_V4I8_ZERO;
4701 Ops.push_back(TexHandle);
4702 Ops.push_back(N->getOperand(2));
4703 Ops.push_back(N->getOperand(3));
4704 Ops.push_back(N->getOperand(4));
4705 Ops.push_back(Chain);
4706 break;
4707 case NVPTXISD::Suld3DV4I16Zero:
4708 Opc = NVPTX::SULD_3D_V4I16_ZERO;
4709 Ops.push_back(TexHandle);
4710 Ops.push_back(N->getOperand(2));
4711 Ops.push_back(N->getOperand(3));
4712 Ops.push_back(N->getOperand(4));
4713 Ops.push_back(Chain);
4714 break;
4715 case NVPTXISD::Suld3DV4I32Zero:
4716 Opc = NVPTX::SULD_3D_V4I32_ZERO;
4717 Ops.push_back(TexHandle);
4718 Ops.push_back(N->getOperand(2));
4719 Ops.push_back(N->getOperand(3));
4720 Ops.push_back(N->getOperand(4));
4721 Ops.push_back(Chain);
4722 break;
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]4723 }
4724 Ret = CurDAG->getMachineNode(Opc, SDLoc(N), N->getVTList(), Ops);
4725 return Ret;
4726}
4727
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]4728
Justin Holewinskica7a4f12014-06-27 18:35:27 +0000[diff] [blame]4729/// SelectBFE - Look for instruction sequences that can be made more efficient
4730/// by using the 'bfe' (bit-field extract) PTX instruction
4731SDNode *NVPTXDAGToDAGISel::SelectBFE(SDNode *N) {
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +0000[diff] [blame]4732 SDLoc DL(N);
Justin Holewinskica7a4f12014-06-27 18:35:27 +0000[diff] [blame]4733 SDValue LHS = N->getOperand(0);
4734 SDValue RHS = N->getOperand(1);
4735 SDValue Len;
4736 SDValue Start;
4737 SDValue Val;
4738 bool IsSigned = false;
4739
4740 if (N->getOpcode() == ISD::AND) {
4741 // Canonicalize the operands
4742 // We want 'and %val, %mask'
4743 if (isa<ConstantSDNode>(LHS) && !isa<ConstantSDNode>(RHS)) {
4744 std::swap(LHS, RHS);
4745 }
4746
4747 ConstantSDNode *Mask = dyn_cast<ConstantSDNode>(RHS);
4748 if (!Mask) {
4749 // We need a constant mask on the RHS of the AND
4750 return NULL;
4751 }
4752
4753 // Extract the mask bits
4754 uint64_t MaskVal = Mask->getZExtValue();
4755 if (!isMask_64(MaskVal)) {
4756 // We *could* handle shifted masks here, but doing so would require an
4757 // 'and' operation to fix up the low-order bits so we would trade
4758 // shr+and for bfe+and, which has the same throughput
4759 return NULL;
4760 }
4761
4762 // How many bits are in our mask?
Benjamin Kramer5f6a9072015-02-12 15:35:40 +0000[diff] [blame]4763 uint64_t NumBits = countTrailingOnes(MaskVal);
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +0000[diff] [blame]4764 Len = CurDAG->getTargetConstant(NumBits, DL, MVT::i32);
Justin Holewinskica7a4f12014-06-27 18:35:27 +0000[diff] [blame]4765
4766 if (LHS.getOpcode() == ISD::SRL || LHS.getOpcode() == ISD::SRA) {
4767 // We have a 'srl/and' pair, extract the effective start bit and length
4768 Val = LHS.getNode()->getOperand(0);
4769 Start = LHS.getNode()->getOperand(1);
4770 ConstantSDNode *StartConst = dyn_cast<ConstantSDNode>(Start);
4771 if (StartConst) {
4772 uint64_t StartVal = StartConst->getZExtValue();
4773 // How many "good" bits do we have left? "good" is defined here as bits
4774 // that exist in the original value, not shifted in.
4775 uint64_t GoodBits = Start.getValueType().getSizeInBits() - StartVal;
4776 if (NumBits > GoodBits) {
4777 // Do not handle the case where bits have been shifted in. In theory
4778 // we could handle this, but the cost is likely higher than just
4779 // emitting the srl/and pair.
4780 return NULL;
4781 }
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +0000[diff] [blame]4782 Start = CurDAG->getTargetConstant(StartVal, DL, MVT::i32);
Justin Holewinskica7a4f12014-06-27 18:35:27 +0000[diff] [blame]4783 } else {
4784 // Do not handle the case where the shift amount (can be zero if no srl
4785 // was found) is not constant. We could handle this case, but it would
4786 // require run-time logic that would be more expensive than just
4787 // emitting the srl/and pair.
4788 return NULL;
4789 }
4790 } else {
4791 // Do not handle the case where the LHS of the and is not a shift. While
4792 // it would be trivial to handle this case, it would just transform
4793 // 'and' -> 'bfe', but 'and' has higher-throughput.
4794 return NULL;
4795 }
4796 } else if (N->getOpcode() == ISD::SRL || N->getOpcode() == ISD::SRA) {
4797 if (LHS->getOpcode() == ISD::AND) {
4798 ConstantSDNode *ShiftCnst = dyn_cast<ConstantSDNode>(RHS);
4799 if (!ShiftCnst) {
4800 // Shift amount must be constant
4801 return NULL;
4802 }
4803
4804 uint64_t ShiftAmt = ShiftCnst->getZExtValue();
4805
4806 SDValue AndLHS = LHS->getOperand(0);
4807 SDValue AndRHS = LHS->getOperand(1);
4808
4809 // Canonicalize the AND to have the mask on the RHS
4810 if (isa<ConstantSDNode>(AndLHS)) {
4811 std::swap(AndLHS, AndRHS);
4812 }
4813
4814 ConstantSDNode *MaskCnst = dyn_cast<ConstantSDNode>(AndRHS);
4815 if (!MaskCnst) {
4816 // Mask must be constant
4817 return NULL;
4818 }
4819
4820 uint64_t MaskVal = MaskCnst->getZExtValue();
4821 uint64_t NumZeros;
4822 uint64_t NumBits;
4823 if (isMask_64(MaskVal)) {
4824 NumZeros = 0;
4825 // The number of bits in the result bitfield will be the number of
4826 // trailing ones (the AND) minus the number of bits we shift off
Benjamin Kramer5f6a9072015-02-12 15:35:40 +0000[diff] [blame]4827 NumBits = countTrailingOnes(MaskVal) - ShiftAmt;
Justin Holewinskica7a4f12014-06-27 18:35:27 +0000[diff] [blame]4828 } else if (isShiftedMask_64(MaskVal)) {
4829 NumZeros = countTrailingZeros(MaskVal);
Benjamin Kramer5f6a9072015-02-12 15:35:40 +0000[diff] [blame]4830 unsigned NumOnes = countTrailingOnes(MaskVal >> NumZeros);
Justin Holewinskica7a4f12014-06-27 18:35:27 +0000[diff] [blame]4831 // The number of bits in the result bitfield will be the number of
4832 // trailing zeros plus the number of set bits in the mask minus the
4833 // number of bits we shift off
4834 NumBits = NumZeros + NumOnes - ShiftAmt;
4835 } else {
4836 // This is not a mask we can handle
4837 return NULL;
4838 }
4839
4840 if (ShiftAmt < NumZeros) {
4841 // Handling this case would require extra logic that would make this
4842 // transformation non-profitable
4843 return NULL;
4844 }
4845
4846 Val = AndLHS;
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +0000[diff] [blame]4847 Start = CurDAG->getTargetConstant(ShiftAmt, DL, MVT::i32);
4848 Len = CurDAG->getTargetConstant(NumBits, DL, MVT::i32);
Justin Holewinskica7a4f12014-06-27 18:35:27 +0000[diff] [blame]4849 } else if (LHS->getOpcode() == ISD::SHL) {
4850 // Here, we have a pattern like:
4851 //
4852 // (sra (shl val, NN), MM)
4853 // or
4854 // (srl (shl val, NN), MM)
4855 //
4856 // If MM >= NN, we can efficiently optimize this with bfe
4857 Val = LHS->getOperand(0);
4858
4859 SDValue ShlRHS = LHS->getOperand(1);
4860 ConstantSDNode *ShlCnst = dyn_cast<ConstantSDNode>(ShlRHS);
4861 if (!ShlCnst) {
4862 // Shift amount must be constant
4863 return NULL;
4864 }
4865 uint64_t InnerShiftAmt = ShlCnst->getZExtValue();
4866
4867 SDValue ShrRHS = RHS;
4868 ConstantSDNode *ShrCnst = dyn_cast<ConstantSDNode>(ShrRHS);
4869 if (!ShrCnst) {
4870 // Shift amount must be constant
4871 return NULL;
4872 }
4873 uint64_t OuterShiftAmt = ShrCnst->getZExtValue();
4874
4875 // To avoid extra codegen and be profitable, we need Outer >= Inner
4876 if (OuterShiftAmt < InnerShiftAmt) {
4877 return NULL;
4878 }
4879
4880 // If the outer shift is more than the type size, we have no bitfield to
4881 // extract (since we also check that the inner shift is <= the outer shift
4882 // then this also implies that the inner shift is < the type size)
4883 if (OuterShiftAmt >= Val.getValueType().getSizeInBits()) {
4884 return NULL;
4885 }
4886
4887 Start =
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +0000[diff] [blame]4888 CurDAG->getTargetConstant(OuterShiftAmt - InnerShiftAmt, DL, MVT::i32);
Justin Holewinskica7a4f12014-06-27 18:35:27 +0000[diff] [blame]4889 Len =
4890 CurDAG->getTargetConstant(Val.getValueType().getSizeInBits() -
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +0000[diff] [blame]4891 OuterShiftAmt, DL, MVT::i32);
Justin Holewinskica7a4f12014-06-27 18:35:27 +0000[diff] [blame]4892
4893 if (N->getOpcode() == ISD::SRA) {
4894 // If we have a arithmetic right shift, we need to use the signed bfe
4895 // variant
4896 IsSigned = true;
4897 }
4898 } else {
4899 // No can do...
4900 return NULL;
4901 }
4902 } else {
4903 // No can do...
4904 return NULL;
4905 }
4906
4907
4908 unsigned Opc;
4909 // For the BFE operations we form here from "and" and "srl", always use the
4910 // unsigned variants.
4911 if (Val.getValueType() == MVT::i32) {
4912 if (IsSigned) {
4913 Opc = NVPTX::BFE_S32rii;
4914 } else {
4915 Opc = NVPTX::BFE_U32rii;
4916 }
4917 } else if (Val.getValueType() == MVT::i64) {
4918 if (IsSigned) {
4919 Opc = NVPTX::BFE_S64rii;
4920 } else {
4921 Opc = NVPTX::BFE_U64rii;
4922 }
4923 } else {
4924 // We cannot handle this type
4925 return NULL;
4926 }
4927
4928 SDValue Ops[] = {
4929 Val, Start, Len
4930 };
4931
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +0000[diff] [blame]4932 return CurDAG->getMachineNode(Opc, DL, N->getVTList(), Ops);
Justin Holewinskica7a4f12014-06-27 18:35:27 +0000[diff] [blame]4933}
4934
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]4935// SelectDirectAddr - Match a direct address for DAG.
4936// A direct address could be a globaladdress or externalsymbol.
4937bool NVPTXDAGToDAGISel::SelectDirectAddr(SDValue N, SDValue &Address) {
4938 // Return true if TGA or ES.
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]4939 if (N.getOpcode() == ISD::TargetGlobalAddress ||
4940 N.getOpcode() == ISD::TargetExternalSymbol) {
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]4941 Address = N;
4942 return true;
4943 }
4944 if (N.getOpcode() == NVPTXISD::Wrapper) {
4945 Address = N.getOperand(0);
4946 return true;
4947 }
4948 if (N.getOpcode() == ISD::INTRINSIC_WO_CHAIN) {
4949 unsigned IID = cast<ConstantSDNode>(N.getOperand(0))->getZExtValue();
4950 if (IID == Intrinsic::nvvm_ptr_gen_to_param)
4951 if (N.getOperand(1).getOpcode() == NVPTXISD::MoveParam)
4952 return (SelectDirectAddr(N.getOperand(1).getOperand(0), Address));
4953 }
4954 return false;
4955}
4956
4957// symbol+offset
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]4958bool NVPTXDAGToDAGISel::SelectADDRsi_imp(
4959 SDNode *OpNode, SDValue Addr, SDValue &Base, SDValue &Offset, MVT mvt) {
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]4960 if (Addr.getOpcode() == ISD::ADD) {
4961 if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(Addr.getOperand(1))) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]4962 SDValue base = Addr.getOperand(0);
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]4963 if (SelectDirectAddr(base, Base)) {
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +0000[diff] [blame]4964 Offset = CurDAG->getTargetConstant(CN->getZExtValue(), SDLoc(OpNode),
4965 mvt);
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]4966 return true;
4967 }
4968 }
4969 }
4970 return false;
4971}
4972
4973// symbol+offset
4974bool NVPTXDAGToDAGISel::SelectADDRsi(SDNode *OpNode, SDValue Addr,
4975 SDValue &Base, SDValue &Offset) {
4976 return SelectADDRsi_imp(OpNode, Addr, Base, Offset, MVT::i32);
4977}
4978
4979// symbol+offset
4980bool NVPTXDAGToDAGISel::SelectADDRsi64(SDNode *OpNode, SDValue Addr,
4981 SDValue &Base, SDValue &Offset) {
4982 return SelectADDRsi_imp(OpNode, Addr, Base, Offset, MVT::i64);
4983}
4984
4985// register+offset
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]4986bool NVPTXDAGToDAGISel::SelectADDRri_imp(
4987 SDNode *OpNode, SDValue Addr, SDValue &Base, SDValue &Offset, MVT mvt) {
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]4988 if (FrameIndexSDNode *FIN = dyn_cast<FrameIndexSDNode>(Addr)) {
4989 Base = CurDAG->getTargetFrameIndex(FIN->getIndex(), mvt);
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +0000[diff] [blame]4990 Offset = CurDAG->getTargetConstant(0, SDLoc(OpNode), mvt);
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]4991 return true;
4992 }
4993 if (Addr.getOpcode() == ISD::TargetExternalSymbol ||
4994 Addr.getOpcode() == ISD::TargetGlobalAddress)
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]4995 return false; // direct calls.
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]4996
4997 if (Addr.getOpcode() == ISD::ADD) {
4998 if (SelectDirectAddr(Addr.getOperand(0), Addr)) {
4999 return false;
5000 }
5001 if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(Addr.getOperand(1))) {
5002 if (FrameIndexSDNode *FIN =
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]5003 dyn_cast<FrameIndexSDNode>(Addr.getOperand(0)))
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]5004 // Constant offset from frame ref.
5005 Base = CurDAG->getTargetFrameIndex(FIN->getIndex(), mvt);
5006 else
5007 Base = Addr.getOperand(0);
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +0000[diff] [blame]5008 Offset = CurDAG->getTargetConstant(CN->getZExtValue(), SDLoc(OpNode),
5009 mvt);
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]5010 return true;
5011 }
5012 }
5013 return false;
5014}
5015
5016// register+offset
5017bool NVPTXDAGToDAGISel::SelectADDRri(SDNode *OpNode, SDValue Addr,
5018 SDValue &Base, SDValue &Offset) {
5019 return SelectADDRri_imp(OpNode, Addr, Base, Offset, MVT::i32);
5020}
5021
5022// register+offset
5023bool NVPTXDAGToDAGISel::SelectADDRri64(SDNode *OpNode, SDValue Addr,
5024 SDValue &Base, SDValue &Offset) {
5025 return SelectADDRri_imp(OpNode, Addr, Base, Offset, MVT::i64);
5026}
5027
5028bool NVPTXDAGToDAGISel::ChkMemSDNodeAddressSpace(SDNode *N,
5029 unsigned int spN) const {
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]5030 const Value *Src = nullptr;
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]5031 if (MemSDNode *mN = dyn_cast<MemSDNode>(N)) {
Nick Lewyckyaad475b2014-04-15 07:22:52 +0000[diff] [blame]5032 if (spN == 0 && mN->getMemOperand()->getPseudoValue())
5033 return true;
5034 Src = mN->getMemOperand()->getValue();
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]5035 }
5036 if (!Src)
5037 return false;
5038 if (const PointerType *PT = dyn_cast<PointerType>(Src->getType()))
5039 return (PT->getAddressSpace() == spN);
5040 return false;
5041}
5042
5043/// SelectInlineAsmMemoryOperand - Implement addressing mode selection for
5044/// inline asm expressions.
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]5045bool NVPTXDAGToDAGISel::SelectInlineAsmMemoryOperand(
Daniel Sanders60f1db02015-03-13 12:45:09 +0000[diff] [blame]5046 const SDValue &Op, unsigned ConstraintID, std::vector<SDValue> &OutOps) {
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]5047 SDValue Op0, Op1;
Daniel Sanders60f1db02015-03-13 12:45:09 +0000[diff] [blame]5048 switch (ConstraintID) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]5049 default:
5050 return true;
Daniel Sanders60f1db02015-03-13 12:45:09 +0000[diff] [blame]5051 case InlineAsm::Constraint_m: // memory
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]5052 if (SelectDirectAddr(Op, Op0)) {
5053 OutOps.push_back(Op0);
Sergey Dmitrouk842a51b2015-04-28 14:05:47 +0000[diff] [blame]5054 OutOps.push_back(CurDAG->getTargetConstant(0, SDLoc(Op), MVT::i32));
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]5055 return false;
5056 }
5057 if (SelectADDRri(Op.getNode(), Op, Op0, Op1)) {
5058 OutOps.push_back(Op0);
5059 OutOps.push_back(Op1);
5060 return false;
5061 }
5062 break;
5063 }
5064 return true;
5065}