Gitiles
Code Review Sign In
gerrit-public.fairphone.software / toolchain / llvm-project / 147bba23853ff4a81b864bfeffd74b2e8c4ea622 / . / llvm / lib / Target / NVPTX / NVPTXISelDAGToDAG.cpp
blob: 9244693c347a161772f1e8e7e3f3e32c1bfda549 [file] [log] [blame]
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]1//===-- NVPTXISelDAGToDAG.cpp - A dag to dag inst selector for NVPTX ------===//
2//
3// The LLVM Compiler Infrastructure
4//
5// This file is distributed under the University of Illinois Open Source
6// License. See LICENSE.TXT for details.
7//
8//===----------------------------------------------------------------------===//
9//
10// This file defines an instruction selector for the NVPTX target.
11//
12//===----------------------------------------------------------------------===//
13
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]14#include "NVPTXISelDAGToDAG.h"
Chandler Carruth9fb823b2013-01-02 11:36:10 +0000[diff] [blame]15#include "llvm/IR/GlobalValue.h"
16#include "llvm/IR/Instructions.h"
Chandler Carruthed0881b2012-12-03 16:50:05 +0000[diff] [blame]17#include "llvm/Support/CommandLine.h"
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]18#include "llvm/Support/Debug.h"
19#include "llvm/Support/ErrorHandling.h"
Chandler Carruthed0881b2012-12-03 16:50:05 +0000[diff] [blame]20#include "llvm/Support/raw_ostream.h"
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]21#include "llvm/Target/TargetIntrinsicInfo.h"
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]22
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]23using namespace llvm;
24
Chandler Carruth84e68b22014-04-22 02:41:26 +0000[diff] [blame]25#define DEBUG_TYPE "nvptx-isel"
26
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]27static cl::opt<int> UsePrecDivF32(
Nadav Rotem7f27e0b2013-10-18 23:38:13 +0000[diff] [blame]28 "nvptx-prec-divf32", cl::ZeroOrMore, cl::Hidden,
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]29 cl::desc("NVPTX Specifies: 0 use div.approx, 1 use div.full, 2 use"
Sylvestre Ledru469de192014-08-11 18:04:46 +0000[diff] [blame]30 " IEEE Compliant F32 div.rnd if available."),
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]31 cl::init(2));
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]32
Justin Holewinski48f4ad32013-05-21 16:51:30 +0000[diff] [blame]33static cl::opt<bool>
Nadav Rotem7f27e0b2013-10-18 23:38:13 +0000[diff] [blame]34UsePrecSqrtF32("nvptx-prec-sqrtf32", cl::Hidden,
Justin Holewinski48f4ad32013-05-21 16:51:30 +0000[diff] [blame]35 cl::desc("NVPTX Specific: 0 use sqrt.approx, 1 use sqrt.rn."),
36 cl::init(true));
37
Justin Holewinskicd069e62013-07-22 12:18:04 +0000[diff] [blame]38static cl::opt<bool>
Nadav Rotem7f27e0b2013-10-18 23:38:13 +0000[diff] [blame]39FtzEnabled("nvptx-f32ftz", cl::ZeroOrMore, cl::Hidden,
Justin Holewinskicd069e62013-07-22 12:18:04 +0000[diff] [blame]40 cl::desc("NVPTX Specific: Flush f32 subnormals to sign-preserving zero."),
41 cl::init(false));
42
43
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]44/// createNVPTXISelDag - This pass converts a legalized DAG into a
45/// NVPTX-specific DAG, ready for instruction scheduling.
46FunctionPass *llvm::createNVPTXISelDag(NVPTXTargetMachine &TM,
47 llvm::CodeGenOpt::Level OptLevel) {
48 return new NVPTXDAGToDAGISel(TM, OptLevel);
49}
50
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]51NVPTXDAGToDAGISel::NVPTXDAGToDAGISel(NVPTXTargetMachine &tm,
52 CodeGenOpt::Level OptLevel)
Eric Christopher147bba22015-01-30 01:40:59 +0000[diff] [blame^]53 : SelectionDAGISel(tm, OptLevel) {
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]54 doMulWide = (OptLevel > 0);
Justin Holewinskicd069e62013-07-22 12:18:04 +0000[diff] [blame]55}
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]56
Eric Christopher147bba22015-01-30 01:40:59 +0000[diff] [blame^]57bool NVPTXDAGToDAGISel::runOnMachineFunction(MachineFunction &MF) {
58 Subtarget = &static_cast<const NVPTXSubtarget &>(MF.getSubtarget());
59 return SelectionDAGISel::runOnMachineFunction(MF);
60}
61
Justin Holewinskicd069e62013-07-22 12:18:04 +0000[diff] [blame]62int NVPTXDAGToDAGISel::getDivF32Level() const {
63 if (UsePrecDivF32.getNumOccurrences() > 0) {
64 // If nvptx-prec-div32=N is used on the command-line, always honor it
65 return UsePrecDivF32;
66 } else {
67 // Otherwise, use div.approx if fast math is enabled
68 if (TM.Options.UnsafeFPMath)
69 return 0;
70 else
71 return 2;
72 }
73}
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]74
Justin Holewinskicd069e62013-07-22 12:18:04 +0000[diff] [blame]75bool NVPTXDAGToDAGISel::usePrecSqrtF32() const {
76 if (UsePrecSqrtF32.getNumOccurrences() > 0) {
77 // If nvptx-prec-sqrtf32 is used on the command-line, always honor it
78 return UsePrecSqrtF32;
79 } else {
80 // Otherwise, use sqrt.approx if fast math is enabled
81 if (TM.Options.UnsafeFPMath)
82 return false;
83 else
84 return true;
85 }
86}
87
88bool NVPTXDAGToDAGISel::useF32FTZ() const {
89 if (FtzEnabled.getNumOccurrences() > 0) {
90 // If nvptx-f32ftz is used on the command-line, always honor it
91 return FtzEnabled;
92 } else {
93 const Function *F = MF->getFunction();
94 // Otherwise, check for an nvptx-f32ftz attribute on the function
95 if (F->hasFnAttribute("nvptx-f32ftz"))
96 return (F->getAttributes().getAttribute(AttributeSet::FunctionIndex,
97 "nvptx-f32ftz")
98 .getValueAsString() == "true");
99 else
100 return false;
101 }
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]102}
103
Justin Holewinski428cf0e2014-07-17 18:10:09 +0000[diff] [blame]104bool NVPTXDAGToDAGISel::allowFMA() const {
Eric Christopher147bba22015-01-30 01:40:59 +0000[diff] [blame^]105 const NVPTXTargetLowering *TL = Subtarget->getTargetLowering();
Justin Holewinski428cf0e2014-07-17 18:10:09 +0000[diff] [blame]106 return TL->allowFMA(*MF, OptLevel);
107}
108
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]109/// Select - Select instructions not customized! Used for
110/// expanded, promoted and normal instructions.
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]111SDNode *NVPTXDAGToDAGISel::Select(SDNode *N) {
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]112
Tim Northover31d093c2013-09-22 08:21:56 +0000[diff] [blame]113 if (N->isMachineOpcode()) {
114 N->setNodeId(-1);
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]115 return nullptr; // Already selected.
Tim Northover31d093c2013-09-22 08:21:56 +0000[diff] [blame]116 }
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]117
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]118 SDNode *ResNode = nullptr;
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]119 switch (N->getOpcode()) {
120 case ISD::LOAD:
121 ResNode = SelectLoad(N);
122 break;
123 case ISD::STORE:
124 ResNode = SelectStore(N);
125 break;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]126 case NVPTXISD::LoadV2:
127 case NVPTXISD::LoadV4:
128 ResNode = SelectLoadVector(N);
129 break;
130 case NVPTXISD::LDGV2:
131 case NVPTXISD::LDGV4:
132 case NVPTXISD::LDUV2:
133 case NVPTXISD::LDUV4:
Justin Holewinskib926d9d2014-06-27 18:35:51 +0000[diff] [blame]134 ResNode = SelectLDGLDU(N);
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]135 break;
136 case NVPTXISD::StoreV2:
137 case NVPTXISD::StoreV4:
138 ResNode = SelectStoreVector(N);
139 break;
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]140 case NVPTXISD::LoadParam:
141 case NVPTXISD::LoadParamV2:
142 case NVPTXISD::LoadParamV4:
143 ResNode = SelectLoadParam(N);
144 break;
145 case NVPTXISD::StoreRetval:
146 case NVPTXISD::StoreRetvalV2:
147 case NVPTXISD::StoreRetvalV4:
148 ResNode = SelectStoreRetval(N);
149 break;
150 case NVPTXISD::StoreParam:
151 case NVPTXISD::StoreParamV2:
152 case NVPTXISD::StoreParamV4:
153 case NVPTXISD::StoreParamS32:
154 case NVPTXISD::StoreParamU32:
155 ResNode = SelectStoreParam(N);
156 break;
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]157 case ISD::INTRINSIC_WO_CHAIN:
158 ResNode = SelectIntrinsicNoChain(N);
159 break;
Justin Holewinskib926d9d2014-06-27 18:35:51 +0000[diff] [blame]160 case ISD::INTRINSIC_W_CHAIN:
161 ResNode = SelectIntrinsicChain(N);
162 break;
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]163 case NVPTXISD::Tex1DFloatS32:
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]164 case NVPTXISD::Tex1DFloatFloat:
165 case NVPTXISD::Tex1DFloatFloatLevel:
166 case NVPTXISD::Tex1DFloatFloatGrad:
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]167 case NVPTXISD::Tex1DS32S32:
168 case NVPTXISD::Tex1DS32Float:
169 case NVPTXISD::Tex1DS32FloatLevel:
170 case NVPTXISD::Tex1DS32FloatGrad:
171 case NVPTXISD::Tex1DU32S32:
172 case NVPTXISD::Tex1DU32Float:
173 case NVPTXISD::Tex1DU32FloatLevel:
174 case NVPTXISD::Tex1DU32FloatGrad:
175 case NVPTXISD::Tex1DArrayFloatS32:
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]176 case NVPTXISD::Tex1DArrayFloatFloat:
177 case NVPTXISD::Tex1DArrayFloatFloatLevel:
178 case NVPTXISD::Tex1DArrayFloatFloatGrad:
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]179 case NVPTXISD::Tex1DArrayS32S32:
180 case NVPTXISD::Tex1DArrayS32Float:
181 case NVPTXISD::Tex1DArrayS32FloatLevel:
182 case NVPTXISD::Tex1DArrayS32FloatGrad:
183 case NVPTXISD::Tex1DArrayU32S32:
184 case NVPTXISD::Tex1DArrayU32Float:
185 case NVPTXISD::Tex1DArrayU32FloatLevel:
186 case NVPTXISD::Tex1DArrayU32FloatGrad:
187 case NVPTXISD::Tex2DFloatS32:
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]188 case NVPTXISD::Tex2DFloatFloat:
189 case NVPTXISD::Tex2DFloatFloatLevel:
190 case NVPTXISD::Tex2DFloatFloatGrad:
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]191 case NVPTXISD::Tex2DS32S32:
192 case NVPTXISD::Tex2DS32Float:
193 case NVPTXISD::Tex2DS32FloatLevel:
194 case NVPTXISD::Tex2DS32FloatGrad:
195 case NVPTXISD::Tex2DU32S32:
196 case NVPTXISD::Tex2DU32Float:
197 case NVPTXISD::Tex2DU32FloatLevel:
198 case NVPTXISD::Tex2DU32FloatGrad:
199 case NVPTXISD::Tex2DArrayFloatS32:
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]200 case NVPTXISD::Tex2DArrayFloatFloat:
201 case NVPTXISD::Tex2DArrayFloatFloatLevel:
202 case NVPTXISD::Tex2DArrayFloatFloatGrad:
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]203 case NVPTXISD::Tex2DArrayS32S32:
204 case NVPTXISD::Tex2DArrayS32Float:
205 case NVPTXISD::Tex2DArrayS32FloatLevel:
206 case NVPTXISD::Tex2DArrayS32FloatGrad:
207 case NVPTXISD::Tex2DArrayU32S32:
208 case NVPTXISD::Tex2DArrayU32Float:
209 case NVPTXISD::Tex2DArrayU32FloatLevel:
210 case NVPTXISD::Tex2DArrayU32FloatGrad:
211 case NVPTXISD::Tex3DFloatS32:
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]212 case NVPTXISD::Tex3DFloatFloat:
213 case NVPTXISD::Tex3DFloatFloatLevel:
214 case NVPTXISD::Tex3DFloatFloatGrad:
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]215 case NVPTXISD::Tex3DS32S32:
216 case NVPTXISD::Tex3DS32Float:
217 case NVPTXISD::Tex3DS32FloatLevel:
218 case NVPTXISD::Tex3DS32FloatGrad:
219 case NVPTXISD::Tex3DU32S32:
220 case NVPTXISD::Tex3DU32Float:
221 case NVPTXISD::Tex3DU32FloatLevel:
222 case NVPTXISD::Tex3DU32FloatGrad:
223 case NVPTXISD::TexCubeFloatFloat:
224 case NVPTXISD::TexCubeFloatFloatLevel:
225 case NVPTXISD::TexCubeS32Float:
226 case NVPTXISD::TexCubeS32FloatLevel:
227 case NVPTXISD::TexCubeU32Float:
228 case NVPTXISD::TexCubeU32FloatLevel:
229 case NVPTXISD::TexCubeArrayFloatFloat:
230 case NVPTXISD::TexCubeArrayFloatFloatLevel:
231 case NVPTXISD::TexCubeArrayS32Float:
232 case NVPTXISD::TexCubeArrayS32FloatLevel:
233 case NVPTXISD::TexCubeArrayU32Float:
234 case NVPTXISD::TexCubeArrayU32FloatLevel:
235 case NVPTXISD::Tld4R2DFloatFloat:
236 case NVPTXISD::Tld4G2DFloatFloat:
237 case NVPTXISD::Tld4B2DFloatFloat:
238 case NVPTXISD::Tld4A2DFloatFloat:
239 case NVPTXISD::Tld4R2DS64Float:
240 case NVPTXISD::Tld4G2DS64Float:
241 case NVPTXISD::Tld4B2DS64Float:
242 case NVPTXISD::Tld4A2DS64Float:
243 case NVPTXISD::Tld4R2DU64Float:
244 case NVPTXISD::Tld4G2DU64Float:
245 case NVPTXISD::Tld4B2DU64Float:
246 case NVPTXISD::Tld4A2DU64Float:
247 case NVPTXISD::TexUnified1DFloatS32:
248 case NVPTXISD::TexUnified1DFloatFloat:
249 case NVPTXISD::TexUnified1DFloatFloatLevel:
250 case NVPTXISD::TexUnified1DFloatFloatGrad:
251 case NVPTXISD::TexUnified1DS32S32:
252 case NVPTXISD::TexUnified1DS32Float:
253 case NVPTXISD::TexUnified1DS32FloatLevel:
254 case NVPTXISD::TexUnified1DS32FloatGrad:
255 case NVPTXISD::TexUnified1DU32S32:
256 case NVPTXISD::TexUnified1DU32Float:
257 case NVPTXISD::TexUnified1DU32FloatLevel:
258 case NVPTXISD::TexUnified1DU32FloatGrad:
259 case NVPTXISD::TexUnified1DArrayFloatS32:
260 case NVPTXISD::TexUnified1DArrayFloatFloat:
261 case NVPTXISD::TexUnified1DArrayFloatFloatLevel:
262 case NVPTXISD::TexUnified1DArrayFloatFloatGrad:
263 case NVPTXISD::TexUnified1DArrayS32S32:
264 case NVPTXISD::TexUnified1DArrayS32Float:
265 case NVPTXISD::TexUnified1DArrayS32FloatLevel:
266 case NVPTXISD::TexUnified1DArrayS32FloatGrad:
267 case NVPTXISD::TexUnified1DArrayU32S32:
268 case NVPTXISD::TexUnified1DArrayU32Float:
269 case NVPTXISD::TexUnified1DArrayU32FloatLevel:
270 case NVPTXISD::TexUnified1DArrayU32FloatGrad:
271 case NVPTXISD::TexUnified2DFloatS32:
272 case NVPTXISD::TexUnified2DFloatFloat:
273 case NVPTXISD::TexUnified2DFloatFloatLevel:
274 case NVPTXISD::TexUnified2DFloatFloatGrad:
275 case NVPTXISD::TexUnified2DS32S32:
276 case NVPTXISD::TexUnified2DS32Float:
277 case NVPTXISD::TexUnified2DS32FloatLevel:
278 case NVPTXISD::TexUnified2DS32FloatGrad:
279 case NVPTXISD::TexUnified2DU32S32:
280 case NVPTXISD::TexUnified2DU32Float:
281 case NVPTXISD::TexUnified2DU32FloatLevel:
282 case NVPTXISD::TexUnified2DU32FloatGrad:
283 case NVPTXISD::TexUnified2DArrayFloatS32:
284 case NVPTXISD::TexUnified2DArrayFloatFloat:
285 case NVPTXISD::TexUnified2DArrayFloatFloatLevel:
286 case NVPTXISD::TexUnified2DArrayFloatFloatGrad:
287 case NVPTXISD::TexUnified2DArrayS32S32:
288 case NVPTXISD::TexUnified2DArrayS32Float:
289 case NVPTXISD::TexUnified2DArrayS32FloatLevel:
290 case NVPTXISD::TexUnified2DArrayS32FloatGrad:
291 case NVPTXISD::TexUnified2DArrayU32S32:
292 case NVPTXISD::TexUnified2DArrayU32Float:
293 case NVPTXISD::TexUnified2DArrayU32FloatLevel:
294 case NVPTXISD::TexUnified2DArrayU32FloatGrad:
295 case NVPTXISD::TexUnified3DFloatS32:
296 case NVPTXISD::TexUnified3DFloatFloat:
297 case NVPTXISD::TexUnified3DFloatFloatLevel:
298 case NVPTXISD::TexUnified3DFloatFloatGrad:
299 case NVPTXISD::TexUnified3DS32S32:
300 case NVPTXISD::TexUnified3DS32Float:
301 case NVPTXISD::TexUnified3DS32FloatLevel:
302 case NVPTXISD::TexUnified3DS32FloatGrad:
303 case NVPTXISD::TexUnified3DU32S32:
304 case NVPTXISD::TexUnified3DU32Float:
305 case NVPTXISD::TexUnified3DU32FloatLevel:
306 case NVPTXISD::TexUnified3DU32FloatGrad:
307 case NVPTXISD::TexUnifiedCubeFloatFloat:
308 case NVPTXISD::TexUnifiedCubeFloatFloatLevel:
309 case NVPTXISD::TexUnifiedCubeS32Float:
310 case NVPTXISD::TexUnifiedCubeS32FloatLevel:
311 case NVPTXISD::TexUnifiedCubeU32Float:
312 case NVPTXISD::TexUnifiedCubeU32FloatLevel:
313 case NVPTXISD::TexUnifiedCubeArrayFloatFloat:
314 case NVPTXISD::TexUnifiedCubeArrayFloatFloatLevel:
315 case NVPTXISD::TexUnifiedCubeArrayS32Float:
316 case NVPTXISD::TexUnifiedCubeArrayS32FloatLevel:
317 case NVPTXISD::TexUnifiedCubeArrayU32Float:
318 case NVPTXISD::TexUnifiedCubeArrayU32FloatLevel:
319 case NVPTXISD::Tld4UnifiedR2DFloatFloat:
320 case NVPTXISD::Tld4UnifiedG2DFloatFloat:
321 case NVPTXISD::Tld4UnifiedB2DFloatFloat:
322 case NVPTXISD::Tld4UnifiedA2DFloatFloat:
323 case NVPTXISD::Tld4UnifiedR2DS64Float:
324 case NVPTXISD::Tld4UnifiedG2DS64Float:
325 case NVPTXISD::Tld4UnifiedB2DS64Float:
326 case NVPTXISD::Tld4UnifiedA2DS64Float:
327 case NVPTXISD::Tld4UnifiedR2DU64Float:
328 case NVPTXISD::Tld4UnifiedG2DU64Float:
329 case NVPTXISD::Tld4UnifiedB2DU64Float:
330 case NVPTXISD::Tld4UnifiedA2DU64Float:
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]331 ResNode = SelectTextureIntrinsic(N);
332 break;
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]333 case NVPTXISD::Suld1DI8Clamp:
334 case NVPTXISD::Suld1DI16Clamp:
335 case NVPTXISD::Suld1DI32Clamp:
336 case NVPTXISD::Suld1DI64Clamp:
337 case NVPTXISD::Suld1DV2I8Clamp:
338 case NVPTXISD::Suld1DV2I16Clamp:
339 case NVPTXISD::Suld1DV2I32Clamp:
340 case NVPTXISD::Suld1DV2I64Clamp:
341 case NVPTXISD::Suld1DV4I8Clamp:
342 case NVPTXISD::Suld1DV4I16Clamp:
343 case NVPTXISD::Suld1DV4I32Clamp:
344 case NVPTXISD::Suld1DArrayI8Clamp:
345 case NVPTXISD::Suld1DArrayI16Clamp:
346 case NVPTXISD::Suld1DArrayI32Clamp:
347 case NVPTXISD::Suld1DArrayI64Clamp:
348 case NVPTXISD::Suld1DArrayV2I8Clamp:
349 case NVPTXISD::Suld1DArrayV2I16Clamp:
350 case NVPTXISD::Suld1DArrayV2I32Clamp:
351 case NVPTXISD::Suld1DArrayV2I64Clamp:
352 case NVPTXISD::Suld1DArrayV4I8Clamp:
353 case NVPTXISD::Suld1DArrayV4I16Clamp:
354 case NVPTXISD::Suld1DArrayV4I32Clamp:
355 case NVPTXISD::Suld2DI8Clamp:
356 case NVPTXISD::Suld2DI16Clamp:
357 case NVPTXISD::Suld2DI32Clamp:
358 case NVPTXISD::Suld2DI64Clamp:
359 case NVPTXISD::Suld2DV2I8Clamp:
360 case NVPTXISD::Suld2DV2I16Clamp:
361 case NVPTXISD::Suld2DV2I32Clamp:
362 case NVPTXISD::Suld2DV2I64Clamp:
363 case NVPTXISD::Suld2DV4I8Clamp:
364 case NVPTXISD::Suld2DV4I16Clamp:
365 case NVPTXISD::Suld2DV4I32Clamp:
366 case NVPTXISD::Suld2DArrayI8Clamp:
367 case NVPTXISD::Suld2DArrayI16Clamp:
368 case NVPTXISD::Suld2DArrayI32Clamp:
369 case NVPTXISD::Suld2DArrayI64Clamp:
370 case NVPTXISD::Suld2DArrayV2I8Clamp:
371 case NVPTXISD::Suld2DArrayV2I16Clamp:
372 case NVPTXISD::Suld2DArrayV2I32Clamp:
373 case NVPTXISD::Suld2DArrayV2I64Clamp:
374 case NVPTXISD::Suld2DArrayV4I8Clamp:
375 case NVPTXISD::Suld2DArrayV4I16Clamp:
376 case NVPTXISD::Suld2DArrayV4I32Clamp:
377 case NVPTXISD::Suld3DI8Clamp:
378 case NVPTXISD::Suld3DI16Clamp:
379 case NVPTXISD::Suld3DI32Clamp:
380 case NVPTXISD::Suld3DI64Clamp:
381 case NVPTXISD::Suld3DV2I8Clamp:
382 case NVPTXISD::Suld3DV2I16Clamp:
383 case NVPTXISD::Suld3DV2I32Clamp:
384 case NVPTXISD::Suld3DV2I64Clamp:
385 case NVPTXISD::Suld3DV4I8Clamp:
386 case NVPTXISD::Suld3DV4I16Clamp:
387 case NVPTXISD::Suld3DV4I32Clamp:
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]388 case NVPTXISD::Suld1DI8Trap:
389 case NVPTXISD::Suld1DI16Trap:
390 case NVPTXISD::Suld1DI32Trap:
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]391 case NVPTXISD::Suld1DI64Trap:
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]392 case NVPTXISD::Suld1DV2I8Trap:
393 case NVPTXISD::Suld1DV2I16Trap:
394 case NVPTXISD::Suld1DV2I32Trap:
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]395 case NVPTXISD::Suld1DV2I64Trap:
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]396 case NVPTXISD::Suld1DV4I8Trap:
397 case NVPTXISD::Suld1DV4I16Trap:
398 case NVPTXISD::Suld1DV4I32Trap:
399 case NVPTXISD::Suld1DArrayI8Trap:
400 case NVPTXISD::Suld1DArrayI16Trap:
401 case NVPTXISD::Suld1DArrayI32Trap:
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]402 case NVPTXISD::Suld1DArrayI64Trap:
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]403 case NVPTXISD::Suld1DArrayV2I8Trap:
404 case NVPTXISD::Suld1DArrayV2I16Trap:
405 case NVPTXISD::Suld1DArrayV2I32Trap:
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]406 case NVPTXISD::Suld1DArrayV2I64Trap:
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]407 case NVPTXISD::Suld1DArrayV4I8Trap:
408 case NVPTXISD::Suld1DArrayV4I16Trap:
409 case NVPTXISD::Suld1DArrayV4I32Trap:
410 case NVPTXISD::Suld2DI8Trap:
411 case NVPTXISD::Suld2DI16Trap:
412 case NVPTXISD::Suld2DI32Trap:
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]413 case NVPTXISD::Suld2DI64Trap:
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]414 case NVPTXISD::Suld2DV2I8Trap:
415 case NVPTXISD::Suld2DV2I16Trap:
416 case NVPTXISD::Suld2DV2I32Trap:
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]417 case NVPTXISD::Suld2DV2I64Trap:
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]418 case NVPTXISD::Suld2DV4I8Trap:
419 case NVPTXISD::Suld2DV4I16Trap:
420 case NVPTXISD::Suld2DV4I32Trap:
421 case NVPTXISD::Suld2DArrayI8Trap:
422 case NVPTXISD::Suld2DArrayI16Trap:
423 case NVPTXISD::Suld2DArrayI32Trap:
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]424 case NVPTXISD::Suld2DArrayI64Trap:
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]425 case NVPTXISD::Suld2DArrayV2I8Trap:
426 case NVPTXISD::Suld2DArrayV2I16Trap:
427 case NVPTXISD::Suld2DArrayV2I32Trap:
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]428 case NVPTXISD::Suld2DArrayV2I64Trap:
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]429 case NVPTXISD::Suld2DArrayV4I8Trap:
430 case NVPTXISD::Suld2DArrayV4I16Trap:
431 case NVPTXISD::Suld2DArrayV4I32Trap:
432 case NVPTXISD::Suld3DI8Trap:
433 case NVPTXISD::Suld3DI16Trap:
434 case NVPTXISD::Suld3DI32Trap:
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]435 case NVPTXISD::Suld3DI64Trap:
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]436 case NVPTXISD::Suld3DV2I8Trap:
437 case NVPTXISD::Suld3DV2I16Trap:
438 case NVPTXISD::Suld3DV2I32Trap:
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]439 case NVPTXISD::Suld3DV2I64Trap:
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]440 case NVPTXISD::Suld3DV4I8Trap:
441 case NVPTXISD::Suld3DV4I16Trap:
442 case NVPTXISD::Suld3DV4I32Trap:
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]443 case NVPTXISD::Suld1DI8Zero:
444 case NVPTXISD::Suld1DI16Zero:
445 case NVPTXISD::Suld1DI32Zero:
446 case NVPTXISD::Suld1DI64Zero:
447 case NVPTXISD::Suld1DV2I8Zero:
448 case NVPTXISD::Suld1DV2I16Zero:
449 case NVPTXISD::Suld1DV2I32Zero:
450 case NVPTXISD::Suld1DV2I64Zero:
451 case NVPTXISD::Suld1DV4I8Zero:
452 case NVPTXISD::Suld1DV4I16Zero:
453 case NVPTXISD::Suld1DV4I32Zero:
454 case NVPTXISD::Suld1DArrayI8Zero:
455 case NVPTXISD::Suld1DArrayI16Zero:
456 case NVPTXISD::Suld1DArrayI32Zero:
457 case NVPTXISD::Suld1DArrayI64Zero:
458 case NVPTXISD::Suld1DArrayV2I8Zero:
459 case NVPTXISD::Suld1DArrayV2I16Zero:
460 case NVPTXISD::Suld1DArrayV2I32Zero:
461 case NVPTXISD::Suld1DArrayV2I64Zero:
462 case NVPTXISD::Suld1DArrayV4I8Zero:
463 case NVPTXISD::Suld1DArrayV4I16Zero:
464 case NVPTXISD::Suld1DArrayV4I32Zero:
465 case NVPTXISD::Suld2DI8Zero:
466 case NVPTXISD::Suld2DI16Zero:
467 case NVPTXISD::Suld2DI32Zero:
468 case NVPTXISD::Suld2DI64Zero:
469 case NVPTXISD::Suld2DV2I8Zero:
470 case NVPTXISD::Suld2DV2I16Zero:
471 case NVPTXISD::Suld2DV2I32Zero:
472 case NVPTXISD::Suld2DV2I64Zero:
473 case NVPTXISD::Suld2DV4I8Zero:
474 case NVPTXISD::Suld2DV4I16Zero:
475 case NVPTXISD::Suld2DV4I32Zero:
476 case NVPTXISD::Suld2DArrayI8Zero:
477 case NVPTXISD::Suld2DArrayI16Zero:
478 case NVPTXISD::Suld2DArrayI32Zero:
479 case NVPTXISD::Suld2DArrayI64Zero:
480 case NVPTXISD::Suld2DArrayV2I8Zero:
481 case NVPTXISD::Suld2DArrayV2I16Zero:
482 case NVPTXISD::Suld2DArrayV2I32Zero:
483 case NVPTXISD::Suld2DArrayV2I64Zero:
484 case NVPTXISD::Suld2DArrayV4I8Zero:
485 case NVPTXISD::Suld2DArrayV4I16Zero:
486 case NVPTXISD::Suld2DArrayV4I32Zero:
487 case NVPTXISD::Suld3DI8Zero:
488 case NVPTXISD::Suld3DI16Zero:
489 case NVPTXISD::Suld3DI32Zero:
490 case NVPTXISD::Suld3DI64Zero:
491 case NVPTXISD::Suld3DV2I8Zero:
492 case NVPTXISD::Suld3DV2I16Zero:
493 case NVPTXISD::Suld3DV2I32Zero:
494 case NVPTXISD::Suld3DV2I64Zero:
495 case NVPTXISD::Suld3DV4I8Zero:
496 case NVPTXISD::Suld3DV4I16Zero:
497 case NVPTXISD::Suld3DV4I32Zero:
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]498 ResNode = SelectSurfaceIntrinsic(N);
499 break;
Justin Holewinskica7a4f12014-06-27 18:35:27 +0000[diff] [blame]500 case ISD::AND:
501 case ISD::SRA:
502 case ISD::SRL:
503 // Try to select BFE
504 ResNode = SelectBFE(N);
505 break;
Justin Holewinskiba2fa6d2014-03-24 11:17:53 +0000[diff] [blame]506 case ISD::ADDRSPACECAST:
507 ResNode = SelectAddrSpaceCast(N);
508 break;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]509 default:
510 break;
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]511 }
512 if (ResNode)
513 return ResNode;
514 return SelectCode(N);
515}
516
Justin Holewinskib926d9d2014-06-27 18:35:51 +0000[diff] [blame]517SDNode *NVPTXDAGToDAGISel::SelectIntrinsicChain(SDNode *N) {
518 unsigned IID = cast<ConstantSDNode>(N->getOperand(1))->getZExtValue();
519 switch (IID) {
520 default:
521 return NULL;
522 case Intrinsic::nvvm_ldg_global_f:
523 case Intrinsic::nvvm_ldg_global_i:
524 case Intrinsic::nvvm_ldg_global_p:
525 case Intrinsic::nvvm_ldu_global_f:
526 case Intrinsic::nvvm_ldu_global_i:
527 case Intrinsic::nvvm_ldu_global_p:
528 return SelectLDGLDU(N);
529 }
530}
531
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]532static unsigned int getCodeAddrSpace(MemSDNode *N,
Eric Christopher147bba22015-01-30 01:40:59 +0000[diff] [blame^]533 const NVPTXSubtarget *Subtarget) {
Nick Lewyckyaad475b2014-04-15 07:22:52 +0000[diff] [blame]534 const Value *Src = N->getMemOperand()->getValue();
Justin Holewinskib96d1392013-06-10 13:29:47 +0000[diff] [blame]535
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]536 if (!Src)
Justin Holewinskib96d1392013-06-10 13:29:47 +0000[diff] [blame]537 return NVPTX::PTXLdStInstCode::GENERIC;
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]538
539 if (const PointerType *PT = dyn_cast<PointerType>(Src->getType())) {
540 switch (PT->getAddressSpace()) {
Justin Holewinskib96d1392013-06-10 13:29:47 +0000[diff] [blame]541 case llvm::ADDRESS_SPACE_LOCAL: return NVPTX::PTXLdStInstCode::LOCAL;
542 case llvm::ADDRESS_SPACE_GLOBAL: return NVPTX::PTXLdStInstCode::GLOBAL;
543 case llvm::ADDRESS_SPACE_SHARED: return NVPTX::PTXLdStInstCode::SHARED;
544 case llvm::ADDRESS_SPACE_GENERIC: return NVPTX::PTXLdStInstCode::GENERIC;
545 case llvm::ADDRESS_SPACE_PARAM: return NVPTX::PTXLdStInstCode::PARAM;
546 case llvm::ADDRESS_SPACE_CONST: return NVPTX::PTXLdStInstCode::CONSTANT;
547 default: break;
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]548 }
549 }
Justin Holewinskib96d1392013-06-10 13:29:47 +0000[diff] [blame]550 return NVPTX::PTXLdStInstCode::GENERIC;
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]551}
552
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]553SDNode *NVPTXDAGToDAGISel::SelectIntrinsicNoChain(SDNode *N) {
554 unsigned IID = cast<ConstantSDNode>(N->getOperand(0))->getZExtValue();
555 switch (IID) {
556 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]557 return nullptr;
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]558 case Intrinsic::nvvm_texsurf_handle_internal:
559 return SelectTexSurfHandle(N);
560 }
561}
562
563SDNode *NVPTXDAGToDAGISel::SelectTexSurfHandle(SDNode *N) {
564 // Op 0 is the intrinsic ID
565 SDValue Wrapper = N->getOperand(1);
566 SDValue GlobalVal = Wrapper.getOperand(0);
567 return CurDAG->getMachineNode(NVPTX::texsurf_handles, SDLoc(N), MVT::i64,
568 GlobalVal);
569}
570
Justin Holewinskiba2fa6d2014-03-24 11:17:53 +0000[diff] [blame]571SDNode *NVPTXDAGToDAGISel::SelectAddrSpaceCast(SDNode *N) {
572 SDValue Src = N->getOperand(0);
573 AddrSpaceCastSDNode *CastN = cast<AddrSpaceCastSDNode>(N);
574 unsigned SrcAddrSpace = CastN->getSrcAddressSpace();
575 unsigned DstAddrSpace = CastN->getDestAddressSpace();
576
577 assert(SrcAddrSpace != DstAddrSpace &&
578 "addrspacecast must be between different address spaces");
579
580 if (DstAddrSpace == ADDRESS_SPACE_GENERIC) {
581 // Specific to generic
582 unsigned Opc;
583 switch (SrcAddrSpace) {
584 default: report_fatal_error("Bad address space in addrspacecast");
585 case ADDRESS_SPACE_GLOBAL:
Eric Christopher147bba22015-01-30 01:40:59 +0000[diff] [blame^]586 Opc = Subtarget->is64Bit() ? NVPTX::cvta_global_yes_64
Justin Holewinskiba2fa6d2014-03-24 11:17:53 +0000[diff] [blame]587 : NVPTX::cvta_global_yes;
588 break;
589 case ADDRESS_SPACE_SHARED:
Eric Christopher147bba22015-01-30 01:40:59 +0000[diff] [blame^]590 Opc = Subtarget->is64Bit() ? NVPTX::cvta_shared_yes_64
Justin Holewinskiba2fa6d2014-03-24 11:17:53 +0000[diff] [blame]591 : NVPTX::cvta_shared_yes;
592 break;
593 case ADDRESS_SPACE_CONST:
Eric Christopher147bba22015-01-30 01:40:59 +0000[diff] [blame^]594 Opc = Subtarget->is64Bit() ? NVPTX::cvta_const_yes_64
Justin Holewinskiba2fa6d2014-03-24 11:17:53 +0000[diff] [blame]595 : NVPTX::cvta_const_yes;
596 break;
597 case ADDRESS_SPACE_LOCAL:
Eric Christopher147bba22015-01-30 01:40:59 +0000[diff] [blame^]598 Opc = Subtarget->is64Bit() ? NVPTX::cvta_local_yes_64
Justin Holewinskiba2fa6d2014-03-24 11:17:53 +0000[diff] [blame]599 : NVPTX::cvta_local_yes;
600 break;
601 }
602 return CurDAG->getMachineNode(Opc, SDLoc(N), N->getValueType(0), Src);
603 } else {
604 // Generic to specific
605 if (SrcAddrSpace != 0)
606 report_fatal_error("Cannot cast between two non-generic address spaces");
607 unsigned Opc;
608 switch (DstAddrSpace) {
609 default: report_fatal_error("Bad address space in addrspacecast");
610 case ADDRESS_SPACE_GLOBAL:
Eric Christopher147bba22015-01-30 01:40:59 +0000[diff] [blame^]611 Opc = Subtarget->is64Bit() ? NVPTX::cvta_to_global_yes_64
Justin Holewinskiba2fa6d2014-03-24 11:17:53 +0000[diff] [blame]612 : NVPTX::cvta_to_global_yes;
613 break;
614 case ADDRESS_SPACE_SHARED:
Eric Christopher147bba22015-01-30 01:40:59 +0000[diff] [blame^]615 Opc = Subtarget->is64Bit() ? NVPTX::cvta_to_shared_yes_64
Justin Holewinskiba2fa6d2014-03-24 11:17:53 +0000[diff] [blame]616 : NVPTX::cvta_to_shared_yes;
617 break;
618 case ADDRESS_SPACE_CONST:
Eric Christopher147bba22015-01-30 01:40:59 +0000[diff] [blame^]619 Opc = Subtarget->is64Bit() ? NVPTX::cvta_to_const_yes_64
Justin Holewinskiba2fa6d2014-03-24 11:17:53 +0000[diff] [blame]620 : NVPTX::cvta_to_const_yes;
621 break;
622 case ADDRESS_SPACE_LOCAL:
Eric Christopher147bba22015-01-30 01:40:59 +0000[diff] [blame^]623 Opc = Subtarget->is64Bit() ? NVPTX::cvta_to_local_yes_64
Justin Holewinskiba2fa6d2014-03-24 11:17:53 +0000[diff] [blame]624 : NVPTX::cvta_to_local_yes;
625 break;
626 }
627 return CurDAG->getMachineNode(Opc, SDLoc(N), N->getValueType(0), Src);
628 }
629}
630
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]631SDNode *NVPTXDAGToDAGISel::SelectLoad(SDNode *N) {
Andrew Trickef9de2a2013-05-25 02:42:55 +0000[diff] [blame]632 SDLoc dl(N);
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]633 LoadSDNode *LD = cast<LoadSDNode>(N);
634 EVT LoadedVT = LD->getMemoryVT();
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]635 SDNode *NVPTXLD = nullptr;
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]636
637 // do not support pre/post inc/dec
638 if (LD->isIndexed())
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]639 return nullptr;
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]640
641 if (!LoadedVT.isSimple())
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]642 return nullptr;
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]643
644 // Address Space Setting
645 unsigned int codeAddrSpace = getCodeAddrSpace(LD, Subtarget);
646
647 // Volatile Setting
648 // - .volatile is only availalble for .global and .shared
649 bool isVolatile = LD->isVolatile();
650 if (codeAddrSpace != NVPTX::PTXLdStInstCode::GLOBAL &&
651 codeAddrSpace != NVPTX::PTXLdStInstCode::SHARED &&
652 codeAddrSpace != NVPTX::PTXLdStInstCode::GENERIC)
653 isVolatile = false;
654
655 // Vector Setting
656 MVT SimpleVT = LoadedVT.getSimpleVT();
657 unsigned vecType = NVPTX::PTXLdStInstCode::Scalar;
658 if (SimpleVT.isVector()) {
659 unsigned num = SimpleVT.getVectorNumElements();
660 if (num == 2)
661 vecType = NVPTX::PTXLdStInstCode::V2;
662 else if (num == 4)
663 vecType = NVPTX::PTXLdStInstCode::V4;
664 else
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]665 return nullptr;
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]666 }
667
668 // Type Setting: fromType + fromTypeWidth
669 //
670 // Sign : ISD::SEXTLOAD
671 // Unsign : ISD::ZEXTLOAD, ISD::NON_EXTLOAD or ISD::EXTLOAD and the
672 // type is integer
673 // Float : ISD::NON_EXTLOAD or ISD::EXTLOAD and the type is float
674 MVT ScalarVT = SimpleVT.getScalarType();
Justin Holewinski994d66a2013-05-30 12:22:39 +0000[diff] [blame]675 // Read at least 8 bits (predicates are stored as 8-bit values)
676 unsigned fromTypeWidth = std::max(8U, ScalarVT.getSizeInBits());
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]677 unsigned int fromType;
678 if ((LD->getExtensionType() == ISD::SEXTLOAD))
679 fromType = NVPTX::PTXLdStInstCode::Signed;
680 else if (ScalarVT.isFloatingPoint())
681 fromType = NVPTX::PTXLdStInstCode::Float;
682 else
683 fromType = NVPTX::PTXLdStInstCode::Unsigned;
684
685 // Create the machine instruction DAG
686 SDValue Chain = N->getOperand(0);
687 SDValue N1 = N->getOperand(1);
688 SDValue Addr;
689 SDValue Offset, Base;
690 unsigned Opcode;
Craig Topperd9c27832013-08-15 02:44:19 +0000[diff] [blame]691 MVT::SimpleValueType TargetVT = LD->getSimpleValueType(0).SimpleTy;
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]692
693 if (SelectDirectAddr(N1, Addr)) {
694 switch (TargetVT) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]695 case MVT::i8:
696 Opcode = NVPTX::LD_i8_avar;
697 break;
698 case MVT::i16:
699 Opcode = NVPTX::LD_i16_avar;
700 break;
701 case MVT::i32:
702 Opcode = NVPTX::LD_i32_avar;
703 break;
704 case MVT::i64:
705 Opcode = NVPTX::LD_i64_avar;
706 break;
707 case MVT::f32:
708 Opcode = NVPTX::LD_f32_avar;
709 break;
710 case MVT::f64:
711 Opcode = NVPTX::LD_f64_avar;
712 break;
713 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]714 return nullptr;
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]715 }
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]716 SDValue Ops[] = { getI32Imm(isVolatile), getI32Imm(codeAddrSpace),
717 getI32Imm(vecType), getI32Imm(fromType),
718 getI32Imm(fromTypeWidth), Addr, Chain };
Michael Liaob53d8962013-04-19 22:22:57 +0000[diff] [blame]719 NVPTXLD = CurDAG->getMachineNode(Opcode, dl, TargetVT, MVT::Other, Ops);
Eric Christopher147bba22015-01-30 01:40:59 +0000[diff] [blame^]720 } else if (Subtarget->is64Bit()
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]721 ? SelectADDRsi64(N1.getNode(), N1, Base, Offset)
722 : SelectADDRsi(N1.getNode(), N1, Base, Offset)) {
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]723 switch (TargetVT) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]724 case MVT::i8:
725 Opcode = NVPTX::LD_i8_asi;
726 break;
727 case MVT::i16:
728 Opcode = NVPTX::LD_i16_asi;
729 break;
730 case MVT::i32:
731 Opcode = NVPTX::LD_i32_asi;
732 break;
733 case MVT::i64:
734 Opcode = NVPTX::LD_i64_asi;
735 break;
736 case MVT::f32:
737 Opcode = NVPTX::LD_f32_asi;
738 break;
739 case MVT::f64:
740 Opcode = NVPTX::LD_f64_asi;
741 break;
742 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]743 return nullptr;
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]744 }
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]745 SDValue Ops[] = { getI32Imm(isVolatile), getI32Imm(codeAddrSpace),
746 getI32Imm(vecType), getI32Imm(fromType),
747 getI32Imm(fromTypeWidth), Base, Offset, Chain };
Michael Liaob53d8962013-04-19 22:22:57 +0000[diff] [blame]748 NVPTXLD = CurDAG->getMachineNode(Opcode, dl, TargetVT, MVT::Other, Ops);
Eric Christopher147bba22015-01-30 01:40:59 +0000[diff] [blame^]749 } else if (Subtarget->is64Bit()
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]750 ? SelectADDRri64(N1.getNode(), N1, Base, Offset)
751 : SelectADDRri(N1.getNode(), N1, Base, Offset)) {
Eric Christopher147bba22015-01-30 01:40:59 +0000[diff] [blame^]752 if (Subtarget->is64Bit()) {
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]753 switch (TargetVT) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]754 case MVT::i8:
755 Opcode = NVPTX::LD_i8_ari_64;
756 break;
757 case MVT::i16:
758 Opcode = NVPTX::LD_i16_ari_64;
759 break;
760 case MVT::i32:
761 Opcode = NVPTX::LD_i32_ari_64;
762 break;
763 case MVT::i64:
764 Opcode = NVPTX::LD_i64_ari_64;
765 break;
766 case MVT::f32:
767 Opcode = NVPTX::LD_f32_ari_64;
768 break;
769 case MVT::f64:
770 Opcode = NVPTX::LD_f64_ari_64;
771 break;
772 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]773 return nullptr;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]774 }
775 } else {
776 switch (TargetVT) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]777 case MVT::i8:
778 Opcode = NVPTX::LD_i8_ari;
779 break;
780 case MVT::i16:
781 Opcode = NVPTX::LD_i16_ari;
782 break;
783 case MVT::i32:
784 Opcode = NVPTX::LD_i32_ari;
785 break;
786 case MVT::i64:
787 Opcode = NVPTX::LD_i64_ari;
788 break;
789 case MVT::f32:
790 Opcode = NVPTX::LD_f32_ari;
791 break;
792 case MVT::f64:
793 Opcode = NVPTX::LD_f64_ari;
794 break;
795 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]796 return nullptr;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]797 }
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]798 }
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]799 SDValue Ops[] = { getI32Imm(isVolatile), getI32Imm(codeAddrSpace),
800 getI32Imm(vecType), getI32Imm(fromType),
801 getI32Imm(fromTypeWidth), Base, Offset, Chain };
Michael Liaob53d8962013-04-19 22:22:57 +0000[diff] [blame]802 NVPTXLD = CurDAG->getMachineNode(Opcode, dl, TargetVT, MVT::Other, Ops);
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]803 } else {
Eric Christopher147bba22015-01-30 01:40:59 +0000[diff] [blame^]804 if (Subtarget->is64Bit()) {
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]805 switch (TargetVT) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]806 case MVT::i8:
807 Opcode = NVPTX::LD_i8_areg_64;
808 break;
809 case MVT::i16:
810 Opcode = NVPTX::LD_i16_areg_64;
811 break;
812 case MVT::i32:
813 Opcode = NVPTX::LD_i32_areg_64;
814 break;
815 case MVT::i64:
816 Opcode = NVPTX::LD_i64_areg_64;
817 break;
818 case MVT::f32:
819 Opcode = NVPTX::LD_f32_areg_64;
820 break;
821 case MVT::f64:
822 Opcode = NVPTX::LD_f64_areg_64;
823 break;
824 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]825 return nullptr;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]826 }
827 } else {
828 switch (TargetVT) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]829 case MVT::i8:
830 Opcode = NVPTX::LD_i8_areg;
831 break;
832 case MVT::i16:
833 Opcode = NVPTX::LD_i16_areg;
834 break;
835 case MVT::i32:
836 Opcode = NVPTX::LD_i32_areg;
837 break;
838 case MVT::i64:
839 Opcode = NVPTX::LD_i64_areg;
840 break;
841 case MVT::f32:
842 Opcode = NVPTX::LD_f32_areg;
843 break;
844 case MVT::f64:
845 Opcode = NVPTX::LD_f64_areg;
846 break;
847 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]848 return nullptr;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]849 }
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]850 }
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]851 SDValue Ops[] = { getI32Imm(isVolatile), getI32Imm(codeAddrSpace),
852 getI32Imm(vecType), getI32Imm(fromType),
853 getI32Imm(fromTypeWidth), N1, Chain };
Michael Liaob53d8962013-04-19 22:22:57 +0000[diff] [blame]854 NVPTXLD = CurDAG->getMachineNode(Opcode, dl, TargetVT, MVT::Other, Ops);
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]855 }
856
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]857 if (NVPTXLD) {
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]858 MachineSDNode::mmo_iterator MemRefs0 = MF->allocateMemRefsArray(1);
859 MemRefs0[0] = cast<MemSDNode>(N)->getMemOperand();
860 cast<MachineSDNode>(NVPTXLD)->setMemRefs(MemRefs0, MemRefs0 + 1);
861 }
862
863 return NVPTXLD;
864}
865
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]866SDNode *NVPTXDAGToDAGISel::SelectLoadVector(SDNode *N) {
867
868 SDValue Chain = N->getOperand(0);
869 SDValue Op1 = N->getOperand(1);
870 SDValue Addr, Offset, Base;
871 unsigned Opcode;
Andrew Trickef9de2a2013-05-25 02:42:55 +0000[diff] [blame]872 SDLoc DL(N);
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]873 SDNode *LD;
874 MemSDNode *MemSD = cast<MemSDNode>(N);
875 EVT LoadedVT = MemSD->getMemoryVT();
876
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]877 if (!LoadedVT.isSimple())
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]878 return nullptr;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]879
880 // Address Space Setting
881 unsigned int CodeAddrSpace = getCodeAddrSpace(MemSD, Subtarget);
882
883 // Volatile Setting
884 // - .volatile is only availalble for .global and .shared
885 bool IsVolatile = MemSD->isVolatile();
886 if (CodeAddrSpace != NVPTX::PTXLdStInstCode::GLOBAL &&
887 CodeAddrSpace != NVPTX::PTXLdStInstCode::SHARED &&
888 CodeAddrSpace != NVPTX::PTXLdStInstCode::GENERIC)
889 IsVolatile = false;
890
891 // Vector Setting
892 MVT SimpleVT = LoadedVT.getSimpleVT();
893
894 // Type Setting: fromType + fromTypeWidth
895 //
896 // Sign : ISD::SEXTLOAD
897 // Unsign : ISD::ZEXTLOAD, ISD::NON_EXTLOAD or ISD::EXTLOAD and the
898 // type is integer
899 // Float : ISD::NON_EXTLOAD or ISD::EXTLOAD and the type is float
900 MVT ScalarVT = SimpleVT.getScalarType();
Justin Holewinski994d66a2013-05-30 12:22:39 +0000[diff] [blame]901 // Read at least 8 bits (predicates are stored as 8-bit values)
902 unsigned FromTypeWidth = std::max(8U, ScalarVT.getSizeInBits());
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]903 unsigned int FromType;
904 // The last operand holds the original LoadSDNode::getExtensionType() value
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]905 unsigned ExtensionType = cast<ConstantSDNode>(
906 N->getOperand(N->getNumOperands() - 1))->getZExtValue();
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]907 if (ExtensionType == ISD::SEXTLOAD)
908 FromType = NVPTX::PTXLdStInstCode::Signed;
909 else if (ScalarVT.isFloatingPoint())
910 FromType = NVPTX::PTXLdStInstCode::Float;
911 else
912 FromType = NVPTX::PTXLdStInstCode::Unsigned;
913
914 unsigned VecType;
915
916 switch (N->getOpcode()) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]917 case NVPTXISD::LoadV2:
918 VecType = NVPTX::PTXLdStInstCode::V2;
919 break;
920 case NVPTXISD::LoadV4:
921 VecType = NVPTX::PTXLdStInstCode::V4;
922 break;
923 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]924 return nullptr;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]925 }
926
927 EVT EltVT = N->getValueType(0);
928
929 if (SelectDirectAddr(Op1, Addr)) {
930 switch (N->getOpcode()) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]931 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]932 return nullptr;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]933 case NVPTXISD::LoadV2:
934 switch (EltVT.getSimpleVT().SimpleTy) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]935 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]936 return nullptr;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]937 case MVT::i8:
938 Opcode = NVPTX::LDV_i8_v2_avar;
939 break;
940 case MVT::i16:
941 Opcode = NVPTX::LDV_i16_v2_avar;
942 break;
943 case MVT::i32:
944 Opcode = NVPTX::LDV_i32_v2_avar;
945 break;
946 case MVT::i64:
947 Opcode = NVPTX::LDV_i64_v2_avar;
948 break;
949 case MVT::f32:
950 Opcode = NVPTX::LDV_f32_v2_avar;
951 break;
952 case MVT::f64:
953 Opcode = NVPTX::LDV_f64_v2_avar;
954 break;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]955 }
956 break;
957 case NVPTXISD::LoadV4:
958 switch (EltVT.getSimpleVT().SimpleTy) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]959 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]960 return nullptr;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]961 case MVT::i8:
962 Opcode = NVPTX::LDV_i8_v4_avar;
963 break;
964 case MVT::i16:
965 Opcode = NVPTX::LDV_i16_v4_avar;
966 break;
967 case MVT::i32:
968 Opcode = NVPTX::LDV_i32_v4_avar;
969 break;
970 case MVT::f32:
971 Opcode = NVPTX::LDV_f32_v4_avar;
972 break;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]973 }
974 break;
975 }
976
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]977 SDValue Ops[] = { getI32Imm(IsVolatile), getI32Imm(CodeAddrSpace),
978 getI32Imm(VecType), getI32Imm(FromType),
979 getI32Imm(FromTypeWidth), Addr, Chain };
Michael Liaob53d8962013-04-19 22:22:57 +0000[diff] [blame]980 LD = CurDAG->getMachineNode(Opcode, DL, N->getVTList(), Ops);
Eric Christopher147bba22015-01-30 01:40:59 +0000[diff] [blame^]981 } else if (Subtarget->is64Bit()
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]982 ? SelectADDRsi64(Op1.getNode(), Op1, Base, Offset)
983 : SelectADDRsi(Op1.getNode(), Op1, Base, Offset)) {
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]984 switch (N->getOpcode()) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]985 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]986 return nullptr;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]987 case NVPTXISD::LoadV2:
988 switch (EltVT.getSimpleVT().SimpleTy) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]989 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]990 return nullptr;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]991 case MVT::i8:
992 Opcode = NVPTX::LDV_i8_v2_asi;
993 break;
994 case MVT::i16:
995 Opcode = NVPTX::LDV_i16_v2_asi;
996 break;
997 case MVT::i32:
998 Opcode = NVPTX::LDV_i32_v2_asi;
999 break;
1000 case MVT::i64:
1001 Opcode = NVPTX::LDV_i64_v2_asi;
1002 break;
1003 case MVT::f32:
1004 Opcode = NVPTX::LDV_f32_v2_asi;
1005 break;
1006 case MVT::f64:
1007 Opcode = NVPTX::LDV_f64_v2_asi;
1008 break;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]1009 }
1010 break;
1011 case NVPTXISD::LoadV4:
1012 switch (EltVT.getSimpleVT().SimpleTy) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1013 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]1014 return nullptr;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1015 case MVT::i8:
1016 Opcode = NVPTX::LDV_i8_v4_asi;
1017 break;
1018 case MVT::i16:
1019 Opcode = NVPTX::LDV_i16_v4_asi;
1020 break;
1021 case MVT::i32:
1022 Opcode = NVPTX::LDV_i32_v4_asi;
1023 break;
1024 case MVT::f32:
1025 Opcode = NVPTX::LDV_f32_v4_asi;
1026 break;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]1027 }
1028 break;
1029 }
1030
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1031 SDValue Ops[] = { getI32Imm(IsVolatile), getI32Imm(CodeAddrSpace),
1032 getI32Imm(VecType), getI32Imm(FromType),
1033 getI32Imm(FromTypeWidth), Base, Offset, Chain };
Michael Liaob53d8962013-04-19 22:22:57 +0000[diff] [blame]1034 LD = CurDAG->getMachineNode(Opcode, DL, N->getVTList(), Ops);
Eric Christopher147bba22015-01-30 01:40:59 +0000[diff] [blame^]1035 } else if (Subtarget->is64Bit()
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1036 ? SelectADDRri64(Op1.getNode(), Op1, Base, Offset)
1037 : SelectADDRri(Op1.getNode(), Op1, Base, Offset)) {
Eric Christopher147bba22015-01-30 01:40:59 +0000[diff] [blame^]1038 if (Subtarget->is64Bit()) {
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]1039 switch (N->getOpcode()) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1040 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]1041 return nullptr;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]1042 case NVPTXISD::LoadV2:
1043 switch (EltVT.getSimpleVT().SimpleTy) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1044 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]1045 return nullptr;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1046 case MVT::i8:
1047 Opcode = NVPTX::LDV_i8_v2_ari_64;
1048 break;
1049 case MVT::i16:
1050 Opcode = NVPTX::LDV_i16_v2_ari_64;
1051 break;
1052 case MVT::i32:
1053 Opcode = NVPTX::LDV_i32_v2_ari_64;
1054 break;
1055 case MVT::i64:
1056 Opcode = NVPTX::LDV_i64_v2_ari_64;
1057 break;
1058 case MVT::f32:
1059 Opcode = NVPTX::LDV_f32_v2_ari_64;
1060 break;
1061 case MVT::f64:
1062 Opcode = NVPTX::LDV_f64_v2_ari_64;
1063 break;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]1064 }
1065 break;
1066 case NVPTXISD::LoadV4:
1067 switch (EltVT.getSimpleVT().SimpleTy) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1068 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]1069 return nullptr;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1070 case MVT::i8:
1071 Opcode = NVPTX::LDV_i8_v4_ari_64;
1072 break;
1073 case MVT::i16:
1074 Opcode = NVPTX::LDV_i16_v4_ari_64;
1075 break;
1076 case MVT::i32:
1077 Opcode = NVPTX::LDV_i32_v4_ari_64;
1078 break;
1079 case MVT::f32:
1080 Opcode = NVPTX::LDV_f32_v4_ari_64;
1081 break;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]1082 }
1083 break;
1084 }
1085 } else {
1086 switch (N->getOpcode()) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1087 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]1088 return nullptr;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]1089 case NVPTXISD::LoadV2:
1090 switch (EltVT.getSimpleVT().SimpleTy) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1091 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]1092 return nullptr;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1093 case MVT::i8:
1094 Opcode = NVPTX::LDV_i8_v2_ari;
1095 break;
1096 case MVT::i16:
1097 Opcode = NVPTX::LDV_i16_v2_ari;
1098 break;
1099 case MVT::i32:
1100 Opcode = NVPTX::LDV_i32_v2_ari;
1101 break;
1102 case MVT::i64:
1103 Opcode = NVPTX::LDV_i64_v2_ari;
1104 break;
1105 case MVT::f32:
1106 Opcode = NVPTX::LDV_f32_v2_ari;
1107 break;
1108 case MVT::f64:
1109 Opcode = NVPTX::LDV_f64_v2_ari;
1110 break;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]1111 }
1112 break;
1113 case NVPTXISD::LoadV4:
1114 switch (EltVT.getSimpleVT().SimpleTy) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1115 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]1116 return nullptr;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1117 case MVT::i8:
1118 Opcode = NVPTX::LDV_i8_v4_ari;
1119 break;
1120 case MVT::i16:
1121 Opcode = NVPTX::LDV_i16_v4_ari;
1122 break;
1123 case MVT::i32:
1124 Opcode = NVPTX::LDV_i32_v4_ari;
1125 break;
1126 case MVT::f32:
1127 Opcode = NVPTX::LDV_f32_v4_ari;
1128 break;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]1129 }
1130 break;
1131 }
1132 }
1133
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1134 SDValue Ops[] = { getI32Imm(IsVolatile), getI32Imm(CodeAddrSpace),
1135 getI32Imm(VecType), getI32Imm(FromType),
1136 getI32Imm(FromTypeWidth), Base, Offset, Chain };
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]1137
Michael Liaob53d8962013-04-19 22:22:57 +0000[diff] [blame]1138 LD = CurDAG->getMachineNode(Opcode, DL, N->getVTList(), Ops);
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]1139 } else {
Eric Christopher147bba22015-01-30 01:40:59 +0000[diff] [blame^]1140 if (Subtarget->is64Bit()) {
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]1141 switch (N->getOpcode()) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1142 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]1143 return nullptr;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]1144 case NVPTXISD::LoadV2:
1145 switch (EltVT.getSimpleVT().SimpleTy) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1146 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]1147 return nullptr;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1148 case MVT::i8:
1149 Opcode = NVPTX::LDV_i8_v2_areg_64;
1150 break;
1151 case MVT::i16:
1152 Opcode = NVPTX::LDV_i16_v2_areg_64;
1153 break;
1154 case MVT::i32:
1155 Opcode = NVPTX::LDV_i32_v2_areg_64;
1156 break;
1157 case MVT::i64:
1158 Opcode = NVPTX::LDV_i64_v2_areg_64;
1159 break;
1160 case MVT::f32:
1161 Opcode = NVPTX::LDV_f32_v2_areg_64;
1162 break;
1163 case MVT::f64:
1164 Opcode = NVPTX::LDV_f64_v2_areg_64;
1165 break;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]1166 }
1167 break;
1168 case NVPTXISD::LoadV4:
1169 switch (EltVT.getSimpleVT().SimpleTy) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1170 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]1171 return nullptr;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1172 case MVT::i8:
1173 Opcode = NVPTX::LDV_i8_v4_areg_64;
1174 break;
1175 case MVT::i16:
1176 Opcode = NVPTX::LDV_i16_v4_areg_64;
1177 break;
1178 case MVT::i32:
1179 Opcode = NVPTX::LDV_i32_v4_areg_64;
1180 break;
1181 case MVT::f32:
1182 Opcode = NVPTX::LDV_f32_v4_areg_64;
1183 break;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]1184 }
1185 break;
1186 }
1187 } else {
1188 switch (N->getOpcode()) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1189 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]1190 return nullptr;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]1191 case NVPTXISD::LoadV2:
1192 switch (EltVT.getSimpleVT().SimpleTy) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1193 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]1194 return nullptr;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1195 case MVT::i8:
1196 Opcode = NVPTX::LDV_i8_v2_areg;
1197 break;
1198 case MVT::i16:
1199 Opcode = NVPTX::LDV_i16_v2_areg;
1200 break;
1201 case MVT::i32:
1202 Opcode = NVPTX::LDV_i32_v2_areg;
1203 break;
1204 case MVT::i64:
1205 Opcode = NVPTX::LDV_i64_v2_areg;
1206 break;
1207 case MVT::f32:
1208 Opcode = NVPTX::LDV_f32_v2_areg;
1209 break;
1210 case MVT::f64:
1211 Opcode = NVPTX::LDV_f64_v2_areg;
1212 break;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]1213 }
1214 break;
1215 case NVPTXISD::LoadV4:
1216 switch (EltVT.getSimpleVT().SimpleTy) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1217 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]1218 return nullptr;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1219 case MVT::i8:
1220 Opcode = NVPTX::LDV_i8_v4_areg;
1221 break;
1222 case MVT::i16:
1223 Opcode = NVPTX::LDV_i16_v4_areg;
1224 break;
1225 case MVT::i32:
1226 Opcode = NVPTX::LDV_i32_v4_areg;
1227 break;
1228 case MVT::f32:
1229 Opcode = NVPTX::LDV_f32_v4_areg;
1230 break;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]1231 }
1232 break;
1233 }
1234 }
1235
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1236 SDValue Ops[] = { getI32Imm(IsVolatile), getI32Imm(CodeAddrSpace),
1237 getI32Imm(VecType), getI32Imm(FromType),
1238 getI32Imm(FromTypeWidth), Op1, Chain };
Michael Liaob53d8962013-04-19 22:22:57 +0000[diff] [blame]1239 LD = CurDAG->getMachineNode(Opcode, DL, N->getVTList(), Ops);
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]1240 }
1241
1242 MachineSDNode::mmo_iterator MemRefs0 = MF->allocateMemRefsArray(1);
1243 MemRefs0[0] = cast<MemSDNode>(N)->getMemOperand();
1244 cast<MachineSDNode>(LD)->setMemRefs(MemRefs0, MemRefs0 + 1);
1245
1246 return LD;
1247}
1248
Justin Holewinskib926d9d2014-06-27 18:35:51 +0000[diff] [blame]1249SDNode *NVPTXDAGToDAGISel::SelectLDGLDU(SDNode *N) {
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]1250
1251 SDValue Chain = N->getOperand(0);
Justin Holewinskib926d9d2014-06-27 18:35:51 +0000[diff] [blame]1252 SDValue Op1;
1253 MemSDNode *Mem;
1254 bool IsLDG = true;
1255
1256 // If this is an LDG intrinsic, the address is the third operand. Its its an
1257 // LDG/LDU SD node (from custom vector handling), then its the second operand
1258 if (N->getOpcode() == ISD::INTRINSIC_W_CHAIN) {
1259 Op1 = N->getOperand(2);
1260 Mem = cast<MemIntrinsicSDNode>(N);
1261 unsigned IID = cast<ConstantSDNode>(N->getOperand(1))->getZExtValue();
1262 switch (IID) {
1263 default:
1264 return NULL;
1265 case Intrinsic::nvvm_ldg_global_f:
1266 case Intrinsic::nvvm_ldg_global_i:
1267 case Intrinsic::nvvm_ldg_global_p:
1268 IsLDG = true;
1269 break;
1270 case Intrinsic::nvvm_ldu_global_f:
1271 case Intrinsic::nvvm_ldu_global_i:
1272 case Intrinsic::nvvm_ldu_global_p:
1273 IsLDG = false;
1274 break;
1275 }
1276 } else {
1277 Op1 = N->getOperand(1);
1278 Mem = cast<MemSDNode>(N);
1279 }
1280
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]1281 unsigned Opcode;
Andrew Trickef9de2a2013-05-25 02:42:55 +0000[diff] [blame]1282 SDLoc DL(N);
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]1283 SDNode *LD;
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1284 SDValue Base, Offset, Addr;
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]1285
Justin Holewinskib926d9d2014-06-27 18:35:51 +0000[diff] [blame]1286 EVT EltVT = Mem->getMemoryVT();
1287 if (EltVT.isVector()) {
1288 EltVT = EltVT.getVectorElementType();
1289 }
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]1290
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1291 if (SelectDirectAddr(Op1, Addr)) {
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]1292 switch (N->getOpcode()) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1293 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]1294 return nullptr;
Justin Holewinskib926d9d2014-06-27 18:35:51 +0000[diff] [blame]1295 case ISD::INTRINSIC_W_CHAIN:
1296 if (IsLDG) {
1297 switch (EltVT.getSimpleVT().SimpleTy) {
1298 default:
1299 return nullptr;
1300 case MVT::i8:
1301 Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i8avar;
1302 break;
1303 case MVT::i16:
1304 Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i16avar;
1305 break;
1306 case MVT::i32:
1307 Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i32avar;
1308 break;
1309 case MVT::i64:
1310 Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i64avar;
1311 break;
1312 case MVT::f32:
1313 Opcode = NVPTX::INT_PTX_LDG_GLOBAL_f32avar;
1314 break;
1315 case MVT::f64:
1316 Opcode = NVPTX::INT_PTX_LDG_GLOBAL_f64avar;
1317 break;
1318 }
1319 } else {
1320 switch (EltVT.getSimpleVT().SimpleTy) {
1321 default:
1322 return nullptr;
1323 case MVT::i8:
1324 Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i8avar;
1325 break;
1326 case MVT::i16:
1327 Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i16avar;
1328 break;
1329 case MVT::i32:
1330 Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i32avar;
1331 break;
1332 case MVT::i64:
1333 Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i64avar;
1334 break;
1335 case MVT::f32:
1336 Opcode = NVPTX::INT_PTX_LDU_GLOBAL_f32avar;
1337 break;
1338 case MVT::f64:
1339 Opcode = NVPTX::INT_PTX_LDU_GLOBAL_f64avar;
1340 break;
1341 }
1342 }
1343 break;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]1344 case NVPTXISD::LDGV2:
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1345 switch (EltVT.getSimpleVT().SimpleTy) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1346 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]1347 return nullptr;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1348 case MVT::i8:
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1349 Opcode = NVPTX::INT_PTX_LDG_G_v2i8_ELE_avar;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1350 break;
1351 case MVT::i16:
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1352 Opcode = NVPTX::INT_PTX_LDG_G_v2i16_ELE_avar;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1353 break;
1354 case MVT::i32:
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1355 Opcode = NVPTX::INT_PTX_LDG_G_v2i32_ELE_avar;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1356 break;
1357 case MVT::i64:
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1358 Opcode = NVPTX::INT_PTX_LDG_G_v2i64_ELE_avar;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1359 break;
1360 case MVT::f32:
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1361 Opcode = NVPTX::INT_PTX_LDG_G_v2f32_ELE_avar;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1362 break;
1363 case MVT::f64:
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1364 Opcode = NVPTX::INT_PTX_LDG_G_v2f64_ELE_avar;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1365 break;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]1366 }
1367 break;
1368 case NVPTXISD::LDUV2:
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1369 switch (EltVT.getSimpleVT().SimpleTy) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1370 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]1371 return nullptr;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1372 case MVT::i8:
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1373 Opcode = NVPTX::INT_PTX_LDU_G_v2i8_ELE_avar;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1374 break;
1375 case MVT::i16:
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1376 Opcode = NVPTX::INT_PTX_LDU_G_v2i16_ELE_avar;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1377 break;
1378 case MVT::i32:
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1379 Opcode = NVPTX::INT_PTX_LDU_G_v2i32_ELE_avar;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1380 break;
1381 case MVT::i64:
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1382 Opcode = NVPTX::INT_PTX_LDU_G_v2i64_ELE_avar;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1383 break;
1384 case MVT::f32:
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1385 Opcode = NVPTX::INT_PTX_LDU_G_v2f32_ELE_avar;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1386 break;
1387 case MVT::f64:
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1388 Opcode = NVPTX::INT_PTX_LDU_G_v2f64_ELE_avar;
1389 break;
1390 }
1391 break;
1392 case NVPTXISD::LDGV4:
1393 switch (EltVT.getSimpleVT().SimpleTy) {
1394 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]1395 return nullptr;
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1396 case MVT::i8:
1397 Opcode = NVPTX::INT_PTX_LDG_G_v4i8_ELE_avar;
1398 break;
1399 case MVT::i16:
1400 Opcode = NVPTX::INT_PTX_LDG_G_v4i16_ELE_avar;
1401 break;
1402 case MVT::i32:
1403 Opcode = NVPTX::INT_PTX_LDG_G_v4i32_ELE_avar;
1404 break;
1405 case MVT::f32:
1406 Opcode = NVPTX::INT_PTX_LDG_G_v4f32_ELE_avar;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1407 break;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]1408 }
1409 break;
1410 case NVPTXISD::LDUV4:
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1411 switch (EltVT.getSimpleVT().SimpleTy) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1412 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]1413 return nullptr;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1414 case MVT::i8:
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1415 Opcode = NVPTX::INT_PTX_LDU_G_v4i8_ELE_avar;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1416 break;
1417 case MVT::i16:
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1418 Opcode = NVPTX::INT_PTX_LDU_G_v4i16_ELE_avar;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1419 break;
1420 case MVT::i32:
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1421 Opcode = NVPTX::INT_PTX_LDU_G_v4i32_ELE_avar;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1422 break;
1423 case MVT::f32:
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1424 Opcode = NVPTX::INT_PTX_LDU_G_v4f32_ELE_avar;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1425 break;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]1426 }
1427 break;
1428 }
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1429
1430 SDValue Ops[] = { Addr, Chain };
Craig Topper2d2aa0c2014-04-30 07:17:30 +0000[diff] [blame]1431 LD = CurDAG->getMachineNode(Opcode, DL, N->getVTList(), Ops);
Eric Christopher147bba22015-01-30 01:40:59 +0000[diff] [blame^]1432 } else if (Subtarget->is64Bit()
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1433 ? SelectADDRri64(Op1.getNode(), Op1, Base, Offset)
1434 : SelectADDRri(Op1.getNode(), Op1, Base, Offset)) {
Eric Christopher147bba22015-01-30 01:40:59 +0000[diff] [blame^]1435 if (Subtarget->is64Bit()) {
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1436 switch (N->getOpcode()) {
1437 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]1438 return nullptr;
Justin Holewinskib926d9d2014-06-27 18:35:51 +0000[diff] [blame]1439 case ISD::INTRINSIC_W_CHAIN:
1440 if (IsLDG) {
1441 switch (EltVT.getSimpleVT().SimpleTy) {
1442 default:
1443 return nullptr;
1444 case MVT::i8:
1445 Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i8ari64;
1446 break;
1447 case MVT::i16:
1448 Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i16ari64;
1449 break;
1450 case MVT::i32:
1451 Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i32ari64;
1452 break;
1453 case MVT::i64:
1454 Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i64ari64;
1455 break;
1456 case MVT::f32:
1457 Opcode = NVPTX::INT_PTX_LDG_GLOBAL_f32ari64;
1458 break;
1459 case MVT::f64:
1460 Opcode = NVPTX::INT_PTX_LDG_GLOBAL_f64ari64;
1461 break;
1462 }
1463 } else {
1464 switch (EltVT.getSimpleVT().SimpleTy) {
1465 default:
1466 return nullptr;
1467 case MVT::i8:
1468 Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i8ari64;
1469 break;
1470 case MVT::i16:
1471 Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i16ari64;
1472 break;
1473 case MVT::i32:
1474 Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i32ari64;
1475 break;
1476 case MVT::i64:
1477 Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i64ari64;
1478 break;
1479 case MVT::f32:
1480 Opcode = NVPTX::INT_PTX_LDU_GLOBAL_f32ari64;
1481 break;
1482 case MVT::f64:
1483 Opcode = NVPTX::INT_PTX_LDU_GLOBAL_f64ari64;
1484 break;
1485 }
1486 }
1487 break;
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1488 case NVPTXISD::LDGV2:
1489 switch (EltVT.getSimpleVT().SimpleTy) {
1490 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]1491 return nullptr;
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1492 case MVT::i8:
1493 Opcode = NVPTX::INT_PTX_LDG_G_v2i8_ELE_ari64;
1494 break;
1495 case MVT::i16:
1496 Opcode = NVPTX::INT_PTX_LDG_G_v2i16_ELE_ari64;
1497 break;
1498 case MVT::i32:
1499 Opcode = NVPTX::INT_PTX_LDG_G_v2i32_ELE_ari64;
1500 break;
1501 case MVT::i64:
1502 Opcode = NVPTX::INT_PTX_LDG_G_v2i64_ELE_ari64;
1503 break;
1504 case MVT::f32:
1505 Opcode = NVPTX::INT_PTX_LDG_G_v2f32_ELE_ari64;
1506 break;
1507 case MVT::f64:
1508 Opcode = NVPTX::INT_PTX_LDG_G_v2f64_ELE_ari64;
1509 break;
1510 }
1511 break;
1512 case NVPTXISD::LDUV2:
1513 switch (EltVT.getSimpleVT().SimpleTy) {
1514 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]1515 return nullptr;
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1516 case MVT::i8:
1517 Opcode = NVPTX::INT_PTX_LDU_G_v2i8_ELE_ari64;
1518 break;
1519 case MVT::i16:
1520 Opcode = NVPTX::INT_PTX_LDU_G_v2i16_ELE_ari64;
1521 break;
1522 case MVT::i32:
1523 Opcode = NVPTX::INT_PTX_LDU_G_v2i32_ELE_ari64;
1524 break;
1525 case MVT::i64:
1526 Opcode = NVPTX::INT_PTX_LDU_G_v2i64_ELE_ari64;
1527 break;
1528 case MVT::f32:
1529 Opcode = NVPTX::INT_PTX_LDU_G_v2f32_ELE_ari64;
1530 break;
1531 case MVT::f64:
1532 Opcode = NVPTX::INT_PTX_LDU_G_v2f64_ELE_ari64;
1533 break;
1534 }
1535 break;
1536 case NVPTXISD::LDGV4:
1537 switch (EltVT.getSimpleVT().SimpleTy) {
1538 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]1539 return nullptr;
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1540 case MVT::i8:
1541 Opcode = NVPTX::INT_PTX_LDG_G_v4i8_ELE_ari64;
1542 break;
1543 case MVT::i16:
1544 Opcode = NVPTX::INT_PTX_LDG_G_v4i16_ELE_ari64;
1545 break;
1546 case MVT::i32:
1547 Opcode = NVPTX::INT_PTX_LDG_G_v4i32_ELE_ari64;
1548 break;
1549 case MVT::f32:
1550 Opcode = NVPTX::INT_PTX_LDG_G_v4f32_ELE_ari64;
1551 break;
1552 }
1553 break;
1554 case NVPTXISD::LDUV4:
1555 switch (EltVT.getSimpleVT().SimpleTy) {
1556 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]1557 return nullptr;
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1558 case MVT::i8:
1559 Opcode = NVPTX::INT_PTX_LDU_G_v4i8_ELE_ari64;
1560 break;
1561 case MVT::i16:
1562 Opcode = NVPTX::INT_PTX_LDU_G_v4i16_ELE_ari64;
1563 break;
1564 case MVT::i32:
1565 Opcode = NVPTX::INT_PTX_LDU_G_v4i32_ELE_ari64;
1566 break;
1567 case MVT::f32:
1568 Opcode = NVPTX::INT_PTX_LDU_G_v4f32_ELE_ari64;
1569 break;
1570 }
1571 break;
1572 }
1573 } else {
1574 switch (N->getOpcode()) {
1575 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]1576 return nullptr;
Justin Holewinskib926d9d2014-06-27 18:35:51 +0000[diff] [blame]1577 case ISD::INTRINSIC_W_CHAIN:
1578 if (IsLDG) {
1579 switch (EltVT.getSimpleVT().SimpleTy) {
1580 default:
1581 return nullptr;
1582 case MVT::i8:
1583 Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i8ari;
1584 break;
1585 case MVT::i16:
1586 Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i16ari;
1587 break;
1588 case MVT::i32:
1589 Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i32ari;
1590 break;
1591 case MVT::i64:
1592 Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i64ari;
1593 break;
1594 case MVT::f32:
1595 Opcode = NVPTX::INT_PTX_LDG_GLOBAL_f32ari;
1596 break;
1597 case MVT::f64:
1598 Opcode = NVPTX::INT_PTX_LDG_GLOBAL_f64ari;
1599 break;
1600 }
1601 } else {
1602 switch (EltVT.getSimpleVT().SimpleTy) {
1603 default:
1604 return nullptr;
1605 case MVT::i8:
1606 Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i8ari;
1607 break;
1608 case MVT::i16:
1609 Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i16ari;
1610 break;
1611 case MVT::i32:
1612 Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i32ari;
1613 break;
1614 case MVT::i64:
1615 Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i64ari;
1616 break;
1617 case MVT::f32:
1618 Opcode = NVPTX::INT_PTX_LDU_GLOBAL_f32ari;
1619 break;
1620 case MVT::f64:
1621 Opcode = NVPTX::INT_PTX_LDU_GLOBAL_f64ari;
1622 break;
1623 }
1624 }
1625 break;
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1626 case NVPTXISD::LDGV2:
1627 switch (EltVT.getSimpleVT().SimpleTy) {
1628 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]1629 return nullptr;
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1630 case MVT::i8:
1631 Opcode = NVPTX::INT_PTX_LDG_G_v2i8_ELE_ari32;
1632 break;
1633 case MVT::i16:
1634 Opcode = NVPTX::INT_PTX_LDG_G_v2i16_ELE_ari32;
1635 break;
1636 case MVT::i32:
1637 Opcode = NVPTX::INT_PTX_LDG_G_v2i32_ELE_ari32;
1638 break;
1639 case MVT::i64:
1640 Opcode = NVPTX::INT_PTX_LDG_G_v2i64_ELE_ari32;
1641 break;
1642 case MVT::f32:
1643 Opcode = NVPTX::INT_PTX_LDG_G_v2f32_ELE_ari32;
1644 break;
1645 case MVT::f64:
1646 Opcode = NVPTX::INT_PTX_LDG_G_v2f64_ELE_ari32;
1647 break;
1648 }
1649 break;
1650 case NVPTXISD::LDUV2:
1651 switch (EltVT.getSimpleVT().SimpleTy) {
1652 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]1653 return nullptr;
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1654 case MVT::i8:
1655 Opcode = NVPTX::INT_PTX_LDU_G_v2i8_ELE_ari32;
1656 break;
1657 case MVT::i16:
1658 Opcode = NVPTX::INT_PTX_LDU_G_v2i16_ELE_ari32;
1659 break;
1660 case MVT::i32:
1661 Opcode = NVPTX::INT_PTX_LDU_G_v2i32_ELE_ari32;
1662 break;
1663 case MVT::i64:
1664 Opcode = NVPTX::INT_PTX_LDU_G_v2i64_ELE_ari32;
1665 break;
1666 case MVT::f32:
1667 Opcode = NVPTX::INT_PTX_LDU_G_v2f32_ELE_ari32;
1668 break;
1669 case MVT::f64:
1670 Opcode = NVPTX::INT_PTX_LDU_G_v2f64_ELE_ari32;
1671 break;
1672 }
1673 break;
1674 case NVPTXISD::LDGV4:
1675 switch (EltVT.getSimpleVT().SimpleTy) {
1676 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]1677 return nullptr;
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1678 case MVT::i8:
1679 Opcode = NVPTX::INT_PTX_LDG_G_v4i8_ELE_ari32;
1680 break;
1681 case MVT::i16:
1682 Opcode = NVPTX::INT_PTX_LDG_G_v4i16_ELE_ari32;
1683 break;
1684 case MVT::i32:
1685 Opcode = NVPTX::INT_PTX_LDG_G_v4i32_ELE_ari32;
1686 break;
1687 case MVT::f32:
1688 Opcode = NVPTX::INT_PTX_LDG_G_v4f32_ELE_ari32;
1689 break;
1690 }
1691 break;
1692 case NVPTXISD::LDUV4:
1693 switch (EltVT.getSimpleVT().SimpleTy) {
1694 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]1695 return nullptr;
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1696 case MVT::i8:
1697 Opcode = NVPTX::INT_PTX_LDU_G_v4i8_ELE_ari32;
1698 break;
1699 case MVT::i16:
1700 Opcode = NVPTX::INT_PTX_LDU_G_v4i16_ELE_ari32;
1701 break;
1702 case MVT::i32:
1703 Opcode = NVPTX::INT_PTX_LDU_G_v4i32_ELE_ari32;
1704 break;
1705 case MVT::f32:
1706 Opcode = NVPTX::INT_PTX_LDU_G_v4f32_ELE_ari32;
1707 break;
1708 }
1709 break;
1710 }
1711 }
1712
1713 SDValue Ops[] = { Base, Offset, Chain };
1714
Craig Topper2d2aa0c2014-04-30 07:17:30 +0000[diff] [blame]1715 LD = CurDAG->getMachineNode(Opcode, DL, N->getVTList(), Ops);
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]1716 } else {
Eric Christopher147bba22015-01-30 01:40:59 +0000[diff] [blame^]1717 if (Subtarget->is64Bit()) {
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1718 switch (N->getOpcode()) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1719 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]1720 return nullptr;
Justin Holewinskib926d9d2014-06-27 18:35:51 +0000[diff] [blame]1721 case ISD::INTRINSIC_W_CHAIN:
1722 if (IsLDG) {
1723 switch (EltVT.getSimpleVT().SimpleTy) {
1724 default:
1725 return nullptr;
1726 case MVT::i8:
1727 Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i8areg64;
1728 break;
1729 case MVT::i16:
1730 Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i16areg64;
1731 break;
1732 case MVT::i32:
1733 Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i32areg64;
1734 break;
1735 case MVT::i64:
1736 Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i64areg64;
1737 break;
1738 case MVT::f32:
1739 Opcode = NVPTX::INT_PTX_LDG_GLOBAL_f32areg64;
1740 break;
1741 case MVT::f64:
1742 Opcode = NVPTX::INT_PTX_LDG_GLOBAL_f64areg64;
1743 break;
1744 }
1745 } else {
1746 switch (EltVT.getSimpleVT().SimpleTy) {
1747 default:
1748 return nullptr;
1749 case MVT::i8:
1750 Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i8areg64;
1751 break;
1752 case MVT::i16:
1753 Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i16areg64;
1754 break;
1755 case MVT::i32:
1756 Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i32areg64;
1757 break;
1758 case MVT::i64:
1759 Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i64areg64;
1760 break;
1761 case MVT::f32:
1762 Opcode = NVPTX::INT_PTX_LDU_GLOBAL_f32areg64;
1763 break;
1764 case MVT::f64:
1765 Opcode = NVPTX::INT_PTX_LDU_GLOBAL_f64areg64;
1766 break;
1767 }
1768 }
1769 break;
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1770 case NVPTXISD::LDGV2:
1771 switch (EltVT.getSimpleVT().SimpleTy) {
1772 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]1773 return nullptr;
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1774 case MVT::i8:
1775 Opcode = NVPTX::INT_PTX_LDG_G_v2i8_ELE_areg64;
1776 break;
1777 case MVT::i16:
1778 Opcode = NVPTX::INT_PTX_LDG_G_v2i16_ELE_areg64;
1779 break;
1780 case MVT::i32:
1781 Opcode = NVPTX::INT_PTX_LDG_G_v2i32_ELE_areg64;
1782 break;
1783 case MVT::i64:
1784 Opcode = NVPTX::INT_PTX_LDG_G_v2i64_ELE_areg64;
1785 break;
1786 case MVT::f32:
1787 Opcode = NVPTX::INT_PTX_LDG_G_v2f32_ELE_areg64;
1788 break;
1789 case MVT::f64:
1790 Opcode = NVPTX::INT_PTX_LDG_G_v2f64_ELE_areg64;
1791 break;
1792 }
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1793 break;
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1794 case NVPTXISD::LDUV2:
1795 switch (EltVT.getSimpleVT().SimpleTy) {
1796 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]1797 return nullptr;
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1798 case MVT::i8:
1799 Opcode = NVPTX::INT_PTX_LDU_G_v2i8_ELE_areg64;
1800 break;
1801 case MVT::i16:
1802 Opcode = NVPTX::INT_PTX_LDU_G_v2i16_ELE_areg64;
1803 break;
1804 case MVT::i32:
1805 Opcode = NVPTX::INT_PTX_LDU_G_v2i32_ELE_areg64;
1806 break;
1807 case MVT::i64:
1808 Opcode = NVPTX::INT_PTX_LDU_G_v2i64_ELE_areg64;
1809 break;
1810 case MVT::f32:
1811 Opcode = NVPTX::INT_PTX_LDU_G_v2f32_ELE_areg64;
1812 break;
1813 case MVT::f64:
1814 Opcode = NVPTX::INT_PTX_LDU_G_v2f64_ELE_areg64;
1815 break;
1816 }
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1817 break;
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1818 case NVPTXISD::LDGV4:
1819 switch (EltVT.getSimpleVT().SimpleTy) {
1820 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]1821 return nullptr;
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1822 case MVT::i8:
1823 Opcode = NVPTX::INT_PTX_LDG_G_v4i8_ELE_areg64;
1824 break;
1825 case MVT::i16:
1826 Opcode = NVPTX::INT_PTX_LDG_G_v4i16_ELE_areg64;
1827 break;
1828 case MVT::i32:
1829 Opcode = NVPTX::INT_PTX_LDG_G_v4i32_ELE_areg64;
1830 break;
1831 case MVT::f32:
1832 Opcode = NVPTX::INT_PTX_LDG_G_v4f32_ELE_areg64;
1833 break;
1834 }
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1835 break;
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1836 case NVPTXISD::LDUV4:
1837 switch (EltVT.getSimpleVT().SimpleTy) {
1838 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]1839 return nullptr;
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1840 case MVT::i8:
1841 Opcode = NVPTX::INT_PTX_LDU_G_v4i8_ELE_areg64;
1842 break;
1843 case MVT::i16:
1844 Opcode = NVPTX::INT_PTX_LDU_G_v4i16_ELE_areg64;
1845 break;
1846 case MVT::i32:
1847 Opcode = NVPTX::INT_PTX_LDU_G_v4i32_ELE_areg64;
1848 break;
1849 case MVT::f32:
1850 Opcode = NVPTX::INT_PTX_LDU_G_v4f32_ELE_areg64;
1851 break;
1852 }
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1853 break;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]1854 }
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1855 } else {
1856 switch (N->getOpcode()) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1857 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]1858 return nullptr;
Justin Holewinskib926d9d2014-06-27 18:35:51 +0000[diff] [blame]1859 case ISD::INTRINSIC_W_CHAIN:
1860 if (IsLDG) {
1861 switch (EltVT.getSimpleVT().SimpleTy) {
1862 default:
1863 return nullptr;
1864 case MVT::i8:
1865 Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i8areg;
1866 break;
1867 case MVT::i16:
1868 Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i16areg;
1869 break;
1870 case MVT::i32:
1871 Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i32areg;
1872 break;
1873 case MVT::i64:
1874 Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i64areg;
1875 break;
1876 case MVT::f32:
1877 Opcode = NVPTX::INT_PTX_LDG_GLOBAL_f32areg;
1878 break;
1879 case MVT::f64:
1880 Opcode = NVPTX::INT_PTX_LDG_GLOBAL_f64areg;
1881 break;
1882 }
1883 } else {
1884 switch (EltVT.getSimpleVT().SimpleTy) {
1885 default:
1886 return nullptr;
1887 case MVT::i8:
1888 Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i8areg;
1889 break;
1890 case MVT::i16:
1891 Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i16areg;
1892 break;
1893 case MVT::i32:
1894 Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i32areg;
1895 break;
1896 case MVT::i64:
1897 Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i64areg;
1898 break;
1899 case MVT::f32:
1900 Opcode = NVPTX::INT_PTX_LDU_GLOBAL_f32areg;
1901 break;
1902 case MVT::f64:
1903 Opcode = NVPTX::INT_PTX_LDU_GLOBAL_f64areg;
1904 break;
1905 }
1906 }
1907 break;
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1908 case NVPTXISD::LDGV2:
1909 switch (EltVT.getSimpleVT().SimpleTy) {
1910 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]1911 return nullptr;
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1912 case MVT::i8:
1913 Opcode = NVPTX::INT_PTX_LDG_G_v2i8_ELE_areg32;
1914 break;
1915 case MVT::i16:
1916 Opcode = NVPTX::INT_PTX_LDG_G_v2i16_ELE_areg32;
1917 break;
1918 case MVT::i32:
1919 Opcode = NVPTX::INT_PTX_LDG_G_v2i32_ELE_areg32;
1920 break;
1921 case MVT::i64:
1922 Opcode = NVPTX::INT_PTX_LDG_G_v2i64_ELE_areg32;
1923 break;
1924 case MVT::f32:
1925 Opcode = NVPTX::INT_PTX_LDG_G_v2f32_ELE_areg32;
1926 break;
1927 case MVT::f64:
1928 Opcode = NVPTX::INT_PTX_LDG_G_v2f64_ELE_areg32;
1929 break;
1930 }
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1931 break;
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1932 case NVPTXISD::LDUV2:
1933 switch (EltVT.getSimpleVT().SimpleTy) {
1934 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]1935 return nullptr;
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1936 case MVT::i8:
1937 Opcode = NVPTX::INT_PTX_LDU_G_v2i8_ELE_areg32;
1938 break;
1939 case MVT::i16:
1940 Opcode = NVPTX::INT_PTX_LDU_G_v2i16_ELE_areg32;
1941 break;
1942 case MVT::i32:
1943 Opcode = NVPTX::INT_PTX_LDU_G_v2i32_ELE_areg32;
1944 break;
1945 case MVT::i64:
1946 Opcode = NVPTX::INT_PTX_LDU_G_v2i64_ELE_areg32;
1947 break;
1948 case MVT::f32:
1949 Opcode = NVPTX::INT_PTX_LDU_G_v2f32_ELE_areg32;
1950 break;
1951 case MVT::f64:
1952 Opcode = NVPTX::INT_PTX_LDU_G_v2f64_ELE_areg32;
1953 break;
1954 }
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1955 break;
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1956 case NVPTXISD::LDGV4:
1957 switch (EltVT.getSimpleVT().SimpleTy) {
1958 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]1959 return nullptr;
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1960 case MVT::i8:
1961 Opcode = NVPTX::INT_PTX_LDG_G_v4i8_ELE_areg32;
1962 break;
1963 case MVT::i16:
1964 Opcode = NVPTX::INT_PTX_LDG_G_v4i16_ELE_areg32;
1965 break;
1966 case MVT::i32:
1967 Opcode = NVPTX::INT_PTX_LDG_G_v4i32_ELE_areg32;
1968 break;
1969 case MVT::f32:
1970 Opcode = NVPTX::INT_PTX_LDG_G_v4f32_ELE_areg32;
1971 break;
1972 }
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1973 break;
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1974 case NVPTXISD::LDUV4:
1975 switch (EltVT.getSimpleVT().SimpleTy) {
1976 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]1977 return nullptr;
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1978 case MVT::i8:
1979 Opcode = NVPTX::INT_PTX_LDU_G_v4i8_ELE_areg32;
1980 break;
1981 case MVT::i16:
1982 Opcode = NVPTX::INT_PTX_LDU_G_v4i16_ELE_areg32;
1983 break;
1984 case MVT::i32:
1985 Opcode = NVPTX::INT_PTX_LDU_G_v4i32_ELE_areg32;
1986 break;
1987 case MVT::f32:
1988 Opcode = NVPTX::INT_PTX_LDU_G_v4f32_ELE_areg32;
1989 break;
1990 }
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1991 break;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]1992 }
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]1993 }
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]1994
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1995 SDValue Ops[] = { Op1, Chain };
Craig Topper2d2aa0c2014-04-30 07:17:30 +0000[diff] [blame]1996 LD = CurDAG->getMachineNode(Opcode, DL, N->getVTList(), Ops);
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1997 }
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]1998
1999 MachineSDNode::mmo_iterator MemRefs0 = MF->allocateMemRefsArray(1);
Justin Holewinskib926d9d2014-06-27 18:35:51 +0000[diff] [blame]2000 MemRefs0[0] = Mem->getMemOperand();
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]2001 cast<MachineSDNode>(LD)->setMemRefs(MemRefs0, MemRefs0 + 1);
2002
2003 return LD;
2004}
2005
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2006SDNode *NVPTXDAGToDAGISel::SelectStore(SDNode *N) {
Andrew Trickef9de2a2013-05-25 02:42:55 +0000[diff] [blame]2007 SDLoc dl(N);
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]2008 StoreSDNode *ST = cast<StoreSDNode>(N);
2009 EVT StoreVT = ST->getMemoryVT();
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2010 SDNode *NVPTXST = nullptr;
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]2011
2012 // do not support pre/post inc/dec
2013 if (ST->isIndexed())
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2014 return nullptr;
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]2015
2016 if (!StoreVT.isSimple())
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2017 return nullptr;
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]2018
2019 // Address Space Setting
2020 unsigned int codeAddrSpace = getCodeAddrSpace(ST, Subtarget);
2021
2022 // Volatile Setting
2023 // - .volatile is only availalble for .global and .shared
2024 bool isVolatile = ST->isVolatile();
2025 if (codeAddrSpace != NVPTX::PTXLdStInstCode::GLOBAL &&
2026 codeAddrSpace != NVPTX::PTXLdStInstCode::SHARED &&
2027 codeAddrSpace != NVPTX::PTXLdStInstCode::GENERIC)
2028 isVolatile = false;
2029
2030 // Vector Setting
2031 MVT SimpleVT = StoreVT.getSimpleVT();
2032 unsigned vecType = NVPTX::PTXLdStInstCode::Scalar;
2033 if (SimpleVT.isVector()) {
2034 unsigned num = SimpleVT.getVectorNumElements();
2035 if (num == 2)
2036 vecType = NVPTX::PTXLdStInstCode::V2;
2037 else if (num == 4)
2038 vecType = NVPTX::PTXLdStInstCode::V4;
2039 else
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2040 return nullptr;
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]2041 }
2042
2043 // Type Setting: toType + toTypeWidth
2044 // - for integer type, always use 'u'
2045 //
2046 MVT ScalarVT = SimpleVT.getScalarType();
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2047 unsigned toTypeWidth = ScalarVT.getSizeInBits();
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]2048 unsigned int toType;
2049 if (ScalarVT.isFloatingPoint())
2050 toType = NVPTX::PTXLdStInstCode::Float;
2051 else
2052 toType = NVPTX::PTXLdStInstCode::Unsigned;
2053
2054 // Create the machine instruction DAG
2055 SDValue Chain = N->getOperand(0);
2056 SDValue N1 = N->getOperand(1);
2057 SDValue N2 = N->getOperand(2);
2058 SDValue Addr;
2059 SDValue Offset, Base;
2060 unsigned Opcode;
Craig Topperd9c27832013-08-15 02:44:19 +0000[diff] [blame]2061 MVT::SimpleValueType SourceVT = N1.getNode()->getSimpleValueType(0).SimpleTy;
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]2062
2063 if (SelectDirectAddr(N2, Addr)) {
2064 switch (SourceVT) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2065 case MVT::i8:
2066 Opcode = NVPTX::ST_i8_avar;
2067 break;
2068 case MVT::i16:
2069 Opcode = NVPTX::ST_i16_avar;
2070 break;
2071 case MVT::i32:
2072 Opcode = NVPTX::ST_i32_avar;
2073 break;
2074 case MVT::i64:
2075 Opcode = NVPTX::ST_i64_avar;
2076 break;
2077 case MVT::f32:
2078 Opcode = NVPTX::ST_f32_avar;
2079 break;
2080 case MVT::f64:
2081 Opcode = NVPTX::ST_f64_avar;
2082 break;
2083 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2084 return nullptr;
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]2085 }
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2086 SDValue Ops[] = { N1, getI32Imm(isVolatile), getI32Imm(codeAddrSpace),
2087 getI32Imm(vecType), getI32Imm(toType),
2088 getI32Imm(toTypeWidth), Addr, Chain };
Michael Liaob53d8962013-04-19 22:22:57 +0000[diff] [blame]2089 NVPTXST = CurDAG->getMachineNode(Opcode, dl, MVT::Other, Ops);
Eric Christopher147bba22015-01-30 01:40:59 +0000[diff] [blame^]2090 } else if (Subtarget->is64Bit()
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2091 ? SelectADDRsi64(N2.getNode(), N2, Base, Offset)
2092 : SelectADDRsi(N2.getNode(), N2, Base, Offset)) {
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]2093 switch (SourceVT) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2094 case MVT::i8:
2095 Opcode = NVPTX::ST_i8_asi;
2096 break;
2097 case MVT::i16:
2098 Opcode = NVPTX::ST_i16_asi;
2099 break;
2100 case MVT::i32:
2101 Opcode = NVPTX::ST_i32_asi;
2102 break;
2103 case MVT::i64:
2104 Opcode = NVPTX::ST_i64_asi;
2105 break;
2106 case MVT::f32:
2107 Opcode = NVPTX::ST_f32_asi;
2108 break;
2109 case MVT::f64:
2110 Opcode = NVPTX::ST_f64_asi;
2111 break;
2112 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2113 return nullptr;
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]2114 }
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2115 SDValue Ops[] = { N1, getI32Imm(isVolatile), getI32Imm(codeAddrSpace),
2116 getI32Imm(vecType), getI32Imm(toType),
2117 getI32Imm(toTypeWidth), Base, Offset, Chain };
Michael Liaob53d8962013-04-19 22:22:57 +0000[diff] [blame]2118 NVPTXST = CurDAG->getMachineNode(Opcode, dl, MVT::Other, Ops);
Eric Christopher147bba22015-01-30 01:40:59 +0000[diff] [blame^]2119 } else if (Subtarget->is64Bit()
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2120 ? SelectADDRri64(N2.getNode(), N2, Base, Offset)
2121 : SelectADDRri(N2.getNode(), N2, Base, Offset)) {
Eric Christopher147bba22015-01-30 01:40:59 +0000[diff] [blame^]2122 if (Subtarget->is64Bit()) {
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]2123 switch (SourceVT) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2124 case MVT::i8:
2125 Opcode = NVPTX::ST_i8_ari_64;
2126 break;
2127 case MVT::i16:
2128 Opcode = NVPTX::ST_i16_ari_64;
2129 break;
2130 case MVT::i32:
2131 Opcode = NVPTX::ST_i32_ari_64;
2132 break;
2133 case MVT::i64:
2134 Opcode = NVPTX::ST_i64_ari_64;
2135 break;
2136 case MVT::f32:
2137 Opcode = NVPTX::ST_f32_ari_64;
2138 break;
2139 case MVT::f64:
2140 Opcode = NVPTX::ST_f64_ari_64;
2141 break;
2142 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2143 return nullptr;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]2144 }
2145 } else {
2146 switch (SourceVT) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2147 case MVT::i8:
2148 Opcode = NVPTX::ST_i8_ari;
2149 break;
2150 case MVT::i16:
2151 Opcode = NVPTX::ST_i16_ari;
2152 break;
2153 case MVT::i32:
2154 Opcode = NVPTX::ST_i32_ari;
2155 break;
2156 case MVT::i64:
2157 Opcode = NVPTX::ST_i64_ari;
2158 break;
2159 case MVT::f32:
2160 Opcode = NVPTX::ST_f32_ari;
2161 break;
2162 case MVT::f64:
2163 Opcode = NVPTX::ST_f64_ari;
2164 break;
2165 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2166 return nullptr;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]2167 }
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]2168 }
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2169 SDValue Ops[] = { N1, getI32Imm(isVolatile), getI32Imm(codeAddrSpace),
2170 getI32Imm(vecType), getI32Imm(toType),
2171 getI32Imm(toTypeWidth), Base, Offset, Chain };
Michael Liaob53d8962013-04-19 22:22:57 +0000[diff] [blame]2172 NVPTXST = CurDAG->getMachineNode(Opcode, dl, MVT::Other, Ops);
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]2173 } else {
Eric Christopher147bba22015-01-30 01:40:59 +0000[diff] [blame^]2174 if (Subtarget->is64Bit()) {
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]2175 switch (SourceVT) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2176 case MVT::i8:
2177 Opcode = NVPTX::ST_i8_areg_64;
2178 break;
2179 case MVT::i16:
2180 Opcode = NVPTX::ST_i16_areg_64;
2181 break;
2182 case MVT::i32:
2183 Opcode = NVPTX::ST_i32_areg_64;
2184 break;
2185 case MVT::i64:
2186 Opcode = NVPTX::ST_i64_areg_64;
2187 break;
2188 case MVT::f32:
2189 Opcode = NVPTX::ST_f32_areg_64;
2190 break;
2191 case MVT::f64:
2192 Opcode = NVPTX::ST_f64_areg_64;
2193 break;
2194 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2195 return nullptr;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]2196 }
2197 } else {
2198 switch (SourceVT) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2199 case MVT::i8:
2200 Opcode = NVPTX::ST_i8_areg;
2201 break;
2202 case MVT::i16:
2203 Opcode = NVPTX::ST_i16_areg;
2204 break;
2205 case MVT::i32:
2206 Opcode = NVPTX::ST_i32_areg;
2207 break;
2208 case MVT::i64:
2209 Opcode = NVPTX::ST_i64_areg;
2210 break;
2211 case MVT::f32:
2212 Opcode = NVPTX::ST_f32_areg;
2213 break;
2214 case MVT::f64:
2215 Opcode = NVPTX::ST_f64_areg;
2216 break;
2217 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2218 return nullptr;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]2219 }
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]2220 }
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2221 SDValue Ops[] = { N1, getI32Imm(isVolatile), getI32Imm(codeAddrSpace),
2222 getI32Imm(vecType), getI32Imm(toType),
2223 getI32Imm(toTypeWidth), N2, Chain };
Michael Liaob53d8962013-04-19 22:22:57 +0000[diff] [blame]2224 NVPTXST = CurDAG->getMachineNode(Opcode, dl, MVT::Other, Ops);
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]2225 }
2226
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2227 if (NVPTXST) {
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]2228 MachineSDNode::mmo_iterator MemRefs0 = MF->allocateMemRefsArray(1);
2229 MemRefs0[0] = cast<MemSDNode>(N)->getMemOperand();
2230 cast<MachineSDNode>(NVPTXST)->setMemRefs(MemRefs0, MemRefs0 + 1);
2231 }
2232
2233 return NVPTXST;
2234}
2235
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]2236SDNode *NVPTXDAGToDAGISel::SelectStoreVector(SDNode *N) {
2237 SDValue Chain = N->getOperand(0);
2238 SDValue Op1 = N->getOperand(1);
2239 SDValue Addr, Offset, Base;
2240 unsigned Opcode;
Andrew Trickef9de2a2013-05-25 02:42:55 +0000[diff] [blame]2241 SDLoc DL(N);
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]2242 SDNode *ST;
2243 EVT EltVT = Op1.getValueType();
2244 MemSDNode *MemSD = cast<MemSDNode>(N);
2245 EVT StoreVT = MemSD->getMemoryVT();
2246
2247 // Address Space Setting
2248 unsigned CodeAddrSpace = getCodeAddrSpace(MemSD, Subtarget);
2249
2250 if (CodeAddrSpace == NVPTX::PTXLdStInstCode::CONSTANT) {
2251 report_fatal_error("Cannot store to pointer that points to constant "
2252 "memory space");
2253 }
2254
2255 // Volatile Setting
2256 // - .volatile is only availalble for .global and .shared
2257 bool IsVolatile = MemSD->isVolatile();
2258 if (CodeAddrSpace != NVPTX::PTXLdStInstCode::GLOBAL &&
2259 CodeAddrSpace != NVPTX::PTXLdStInstCode::SHARED &&
2260 CodeAddrSpace != NVPTX::PTXLdStInstCode::GENERIC)
2261 IsVolatile = false;
2262
2263 // Type Setting: toType + toTypeWidth
2264 // - for integer type, always use 'u'
2265 assert(StoreVT.isSimple() && "Store value is not simple");
2266 MVT ScalarVT = StoreVT.getSimpleVT().getScalarType();
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2267 unsigned ToTypeWidth = ScalarVT.getSizeInBits();
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]2268 unsigned ToType;
2269 if (ScalarVT.isFloatingPoint())
2270 ToType = NVPTX::PTXLdStInstCode::Float;
2271 else
2272 ToType = NVPTX::PTXLdStInstCode::Unsigned;
2273
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]2274 SmallVector<SDValue, 12> StOps;
2275 SDValue N2;
2276 unsigned VecType;
2277
2278 switch (N->getOpcode()) {
2279 case NVPTXISD::StoreV2:
2280 VecType = NVPTX::PTXLdStInstCode::V2;
2281 StOps.push_back(N->getOperand(1));
2282 StOps.push_back(N->getOperand(2));
2283 N2 = N->getOperand(3);
2284 break;
2285 case NVPTXISD::StoreV4:
2286 VecType = NVPTX::PTXLdStInstCode::V4;
2287 StOps.push_back(N->getOperand(1));
2288 StOps.push_back(N->getOperand(2));
2289 StOps.push_back(N->getOperand(3));
2290 StOps.push_back(N->getOperand(4));
2291 N2 = N->getOperand(5);
2292 break;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2293 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2294 return nullptr;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]2295 }
2296
2297 StOps.push_back(getI32Imm(IsVolatile));
2298 StOps.push_back(getI32Imm(CodeAddrSpace));
2299 StOps.push_back(getI32Imm(VecType));
2300 StOps.push_back(getI32Imm(ToType));
2301 StOps.push_back(getI32Imm(ToTypeWidth));
2302
2303 if (SelectDirectAddr(N2, Addr)) {
2304 switch (N->getOpcode()) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2305 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2306 return nullptr;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]2307 case NVPTXISD::StoreV2:
2308 switch (EltVT.getSimpleVT().SimpleTy) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2309 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2310 return nullptr;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2311 case MVT::i8:
2312 Opcode = NVPTX::STV_i8_v2_avar;
2313 break;
2314 case MVT::i16:
2315 Opcode = NVPTX::STV_i16_v2_avar;
2316 break;
2317 case MVT::i32:
2318 Opcode = NVPTX::STV_i32_v2_avar;
2319 break;
2320 case MVT::i64:
2321 Opcode = NVPTX::STV_i64_v2_avar;
2322 break;
2323 case MVT::f32:
2324 Opcode = NVPTX::STV_f32_v2_avar;
2325 break;
2326 case MVT::f64:
2327 Opcode = NVPTX::STV_f64_v2_avar;
2328 break;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]2329 }
2330 break;
2331 case NVPTXISD::StoreV4:
2332 switch (EltVT.getSimpleVT().SimpleTy) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2333 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2334 return nullptr;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2335 case MVT::i8:
2336 Opcode = NVPTX::STV_i8_v4_avar;
2337 break;
2338 case MVT::i16:
2339 Opcode = NVPTX::STV_i16_v4_avar;
2340 break;
2341 case MVT::i32:
2342 Opcode = NVPTX::STV_i32_v4_avar;
2343 break;
2344 case MVT::f32:
2345 Opcode = NVPTX::STV_f32_v4_avar;
2346 break;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]2347 }
2348 break;
2349 }
2350 StOps.push_back(Addr);
Eric Christopher147bba22015-01-30 01:40:59 +0000[diff] [blame^]2351 } else if (Subtarget->is64Bit()
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2352 ? SelectADDRsi64(N2.getNode(), N2, Base, Offset)
2353 : SelectADDRsi(N2.getNode(), N2, Base, Offset)) {
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]2354 switch (N->getOpcode()) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2355 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2356 return nullptr;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]2357 case NVPTXISD::StoreV2:
2358 switch (EltVT.getSimpleVT().SimpleTy) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2359 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2360 return nullptr;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2361 case MVT::i8:
2362 Opcode = NVPTX::STV_i8_v2_asi;
2363 break;
2364 case MVT::i16:
2365 Opcode = NVPTX::STV_i16_v2_asi;
2366 break;
2367 case MVT::i32:
2368 Opcode = NVPTX::STV_i32_v2_asi;
2369 break;
2370 case MVT::i64:
2371 Opcode = NVPTX::STV_i64_v2_asi;
2372 break;
2373 case MVT::f32:
2374 Opcode = NVPTX::STV_f32_v2_asi;
2375 break;
2376 case MVT::f64:
2377 Opcode = NVPTX::STV_f64_v2_asi;
2378 break;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]2379 }
2380 break;
2381 case NVPTXISD::StoreV4:
2382 switch (EltVT.getSimpleVT().SimpleTy) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2383 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2384 return nullptr;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2385 case MVT::i8:
2386 Opcode = NVPTX::STV_i8_v4_asi;
2387 break;
2388 case MVT::i16:
2389 Opcode = NVPTX::STV_i16_v4_asi;
2390 break;
2391 case MVT::i32:
2392 Opcode = NVPTX::STV_i32_v4_asi;
2393 break;
2394 case MVT::f32:
2395 Opcode = NVPTX::STV_f32_v4_asi;
2396 break;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]2397 }
2398 break;
2399 }
2400 StOps.push_back(Base);
2401 StOps.push_back(Offset);
Eric Christopher147bba22015-01-30 01:40:59 +0000[diff] [blame^]2402 } else if (Subtarget->is64Bit()
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2403 ? SelectADDRri64(N2.getNode(), N2, Base, Offset)
2404 : SelectADDRri(N2.getNode(), N2, Base, Offset)) {
Eric Christopher147bba22015-01-30 01:40:59 +0000[diff] [blame^]2405 if (Subtarget->is64Bit()) {
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]2406 switch (N->getOpcode()) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2407 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2408 return nullptr;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]2409 case NVPTXISD::StoreV2:
2410 switch (EltVT.getSimpleVT().SimpleTy) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2411 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2412 return nullptr;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2413 case MVT::i8:
2414 Opcode = NVPTX::STV_i8_v2_ari_64;
2415 break;
2416 case MVT::i16:
2417 Opcode = NVPTX::STV_i16_v2_ari_64;
2418 break;
2419 case MVT::i32:
2420 Opcode = NVPTX::STV_i32_v2_ari_64;
2421 break;
2422 case MVT::i64:
2423 Opcode = NVPTX::STV_i64_v2_ari_64;
2424 break;
2425 case MVT::f32:
2426 Opcode = NVPTX::STV_f32_v2_ari_64;
2427 break;
2428 case MVT::f64:
2429 Opcode = NVPTX::STV_f64_v2_ari_64;
2430 break;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]2431 }
2432 break;
2433 case NVPTXISD::StoreV4:
2434 switch (EltVT.getSimpleVT().SimpleTy) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2435 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2436 return nullptr;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2437 case MVT::i8:
2438 Opcode = NVPTX::STV_i8_v4_ari_64;
2439 break;
2440 case MVT::i16:
2441 Opcode = NVPTX::STV_i16_v4_ari_64;
2442 break;
2443 case MVT::i32:
2444 Opcode = NVPTX::STV_i32_v4_ari_64;
2445 break;
2446 case MVT::f32:
2447 Opcode = NVPTX::STV_f32_v4_ari_64;
2448 break;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]2449 }
2450 break;
2451 }
2452 } else {
2453 switch (N->getOpcode()) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2454 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2455 return nullptr;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]2456 case NVPTXISD::StoreV2:
2457 switch (EltVT.getSimpleVT().SimpleTy) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2458 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2459 return nullptr;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2460 case MVT::i8:
2461 Opcode = NVPTX::STV_i8_v2_ari;
2462 break;
2463 case MVT::i16:
2464 Opcode = NVPTX::STV_i16_v2_ari;
2465 break;
2466 case MVT::i32:
2467 Opcode = NVPTX::STV_i32_v2_ari;
2468 break;
2469 case MVT::i64:
2470 Opcode = NVPTX::STV_i64_v2_ari;
2471 break;
2472 case MVT::f32:
2473 Opcode = NVPTX::STV_f32_v2_ari;
2474 break;
2475 case MVT::f64:
2476 Opcode = NVPTX::STV_f64_v2_ari;
2477 break;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]2478 }
2479 break;
2480 case NVPTXISD::StoreV4:
2481 switch (EltVT.getSimpleVT().SimpleTy) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2482 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2483 return nullptr;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2484 case MVT::i8:
2485 Opcode = NVPTX::STV_i8_v4_ari;
2486 break;
2487 case MVT::i16:
2488 Opcode = NVPTX::STV_i16_v4_ari;
2489 break;
2490 case MVT::i32:
2491 Opcode = NVPTX::STV_i32_v4_ari;
2492 break;
2493 case MVT::f32:
2494 Opcode = NVPTX::STV_f32_v4_ari;
2495 break;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]2496 }
2497 break;
2498 }
2499 }
2500 StOps.push_back(Base);
2501 StOps.push_back(Offset);
2502 } else {
Eric Christopher147bba22015-01-30 01:40:59 +0000[diff] [blame^]2503 if (Subtarget->is64Bit()) {
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]2504 switch (N->getOpcode()) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2505 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2506 return nullptr;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]2507 case NVPTXISD::StoreV2:
2508 switch (EltVT.getSimpleVT().SimpleTy) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2509 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2510 return nullptr;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2511 case MVT::i8:
2512 Opcode = NVPTX::STV_i8_v2_areg_64;
2513 break;
2514 case MVT::i16:
2515 Opcode = NVPTX::STV_i16_v2_areg_64;
2516 break;
2517 case MVT::i32:
2518 Opcode = NVPTX::STV_i32_v2_areg_64;
2519 break;
2520 case MVT::i64:
2521 Opcode = NVPTX::STV_i64_v2_areg_64;
2522 break;
2523 case MVT::f32:
2524 Opcode = NVPTX::STV_f32_v2_areg_64;
2525 break;
2526 case MVT::f64:
2527 Opcode = NVPTX::STV_f64_v2_areg_64;
2528 break;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]2529 }
2530 break;
2531 case NVPTXISD::StoreV4:
2532 switch (EltVT.getSimpleVT().SimpleTy) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2533 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2534 return nullptr;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2535 case MVT::i8:
2536 Opcode = NVPTX::STV_i8_v4_areg_64;
2537 break;
2538 case MVT::i16:
2539 Opcode = NVPTX::STV_i16_v4_areg_64;
2540 break;
2541 case MVT::i32:
2542 Opcode = NVPTX::STV_i32_v4_areg_64;
2543 break;
2544 case MVT::f32:
2545 Opcode = NVPTX::STV_f32_v4_areg_64;
2546 break;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]2547 }
2548 break;
2549 }
2550 } else {
2551 switch (N->getOpcode()) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2552 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2553 return nullptr;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]2554 case NVPTXISD::StoreV2:
2555 switch (EltVT.getSimpleVT().SimpleTy) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2556 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2557 return nullptr;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2558 case MVT::i8:
2559 Opcode = NVPTX::STV_i8_v2_areg;
2560 break;
2561 case MVT::i16:
2562 Opcode = NVPTX::STV_i16_v2_areg;
2563 break;
2564 case MVT::i32:
2565 Opcode = NVPTX::STV_i32_v2_areg;
2566 break;
2567 case MVT::i64:
2568 Opcode = NVPTX::STV_i64_v2_areg;
2569 break;
2570 case MVT::f32:
2571 Opcode = NVPTX::STV_f32_v2_areg;
2572 break;
2573 case MVT::f64:
2574 Opcode = NVPTX::STV_f64_v2_areg;
2575 break;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]2576 }
2577 break;
2578 case NVPTXISD::StoreV4:
2579 switch (EltVT.getSimpleVT().SimpleTy) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2580 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2581 return nullptr;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2582 case MVT::i8:
2583 Opcode = NVPTX::STV_i8_v4_areg;
2584 break;
2585 case MVT::i16:
2586 Opcode = NVPTX::STV_i16_v4_areg;
2587 break;
2588 case MVT::i32:
2589 Opcode = NVPTX::STV_i32_v4_areg;
2590 break;
2591 case MVT::f32:
2592 Opcode = NVPTX::STV_f32_v4_areg;
2593 break;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]2594 }
2595 break;
2596 }
2597 }
2598 StOps.push_back(N2);
2599 }
2600
2601 StOps.push_back(Chain);
2602
Michael Liaob53d8962013-04-19 22:22:57 +0000[diff] [blame]2603 ST = CurDAG->getMachineNode(Opcode, DL, MVT::Other, StOps);
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]2604
2605 MachineSDNode::mmo_iterator MemRefs0 = MF->allocateMemRefsArray(1);
2606 MemRefs0[0] = cast<MemSDNode>(N)->getMemOperand();
2607 cast<MachineSDNode>(ST)->setMemRefs(MemRefs0, MemRefs0 + 1);
2608
2609 return ST;
2610}
2611
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]2612SDNode *NVPTXDAGToDAGISel::SelectLoadParam(SDNode *Node) {
2613 SDValue Chain = Node->getOperand(0);
2614 SDValue Offset = Node->getOperand(2);
2615 SDValue Flag = Node->getOperand(3);
2616 SDLoc DL(Node);
2617 MemSDNode *Mem = cast<MemSDNode>(Node);
2618
2619 unsigned VecSize;
2620 switch (Node->getOpcode()) {
2621 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2622 return nullptr;
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]2623 case NVPTXISD::LoadParam:
2624 VecSize = 1;
2625 break;
2626 case NVPTXISD::LoadParamV2:
2627 VecSize = 2;
2628 break;
2629 case NVPTXISD::LoadParamV4:
2630 VecSize = 4;
2631 break;
2632 }
2633
2634 EVT EltVT = Node->getValueType(0);
2635 EVT MemVT = Mem->getMemoryVT();
2636
2637 unsigned Opc = 0;
2638
2639 switch (VecSize) {
2640 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2641 return nullptr;
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]2642 case 1:
2643 switch (MemVT.getSimpleVT().SimpleTy) {
2644 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2645 return nullptr;
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]2646 case MVT::i1:
2647 Opc = NVPTX::LoadParamMemI8;
2648 break;
2649 case MVT::i8:
2650 Opc = NVPTX::LoadParamMemI8;
2651 break;
2652 case MVT::i16:
2653 Opc = NVPTX::LoadParamMemI16;
2654 break;
2655 case MVT::i32:
2656 Opc = NVPTX::LoadParamMemI32;
2657 break;
2658 case MVT::i64:
2659 Opc = NVPTX::LoadParamMemI64;
2660 break;
2661 case MVT::f32:
2662 Opc = NVPTX::LoadParamMemF32;
2663 break;
2664 case MVT::f64:
2665 Opc = NVPTX::LoadParamMemF64;
2666 break;
2667 }
2668 break;
2669 case 2:
2670 switch (MemVT.getSimpleVT().SimpleTy) {
2671 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2672 return nullptr;
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]2673 case MVT::i1:
2674 Opc = NVPTX::LoadParamMemV2I8;
2675 break;
2676 case MVT::i8:
2677 Opc = NVPTX::LoadParamMemV2I8;
2678 break;
2679 case MVT::i16:
2680 Opc = NVPTX::LoadParamMemV2I16;
2681 break;
2682 case MVT::i32:
2683 Opc = NVPTX::LoadParamMemV2I32;
2684 break;
2685 case MVT::i64:
2686 Opc = NVPTX::LoadParamMemV2I64;
2687 break;
2688 case MVT::f32:
2689 Opc = NVPTX::LoadParamMemV2F32;
2690 break;
2691 case MVT::f64:
2692 Opc = NVPTX::LoadParamMemV2F64;
2693 break;
2694 }
2695 break;
2696 case 4:
2697 switch (MemVT.getSimpleVT().SimpleTy) {
2698 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2699 return nullptr;
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]2700 case MVT::i1:
2701 Opc = NVPTX::LoadParamMemV4I8;
2702 break;
2703 case MVT::i8:
2704 Opc = NVPTX::LoadParamMemV4I8;
2705 break;
2706 case MVT::i16:
2707 Opc = NVPTX::LoadParamMemV4I16;
2708 break;
2709 case MVT::i32:
2710 Opc = NVPTX::LoadParamMemV4I32;
2711 break;
2712 case MVT::f32:
2713 Opc = NVPTX::LoadParamMemV4F32;
2714 break;
2715 }
2716 break;
2717 }
2718
2719 SDVTList VTs;
2720 if (VecSize == 1) {
2721 VTs = CurDAG->getVTList(EltVT, MVT::Other, MVT::Glue);
2722 } else if (VecSize == 2) {
2723 VTs = CurDAG->getVTList(EltVT, EltVT, MVT::Other, MVT::Glue);
2724 } else {
2725 EVT EVTs[] = { EltVT, EltVT, EltVT, EltVT, MVT::Other, MVT::Glue };
Craig Topperabb4ac72014-04-16 06:10:51 +0000[diff] [blame]2726 VTs = CurDAG->getVTList(EVTs);
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]2727 }
2728
2729 unsigned OffsetVal = cast<ConstantSDNode>(Offset)->getZExtValue();
2730
2731 SmallVector<SDValue, 2> Ops;
2732 Ops.push_back(CurDAG->getTargetConstant(OffsetVal, MVT::i32));
2733 Ops.push_back(Chain);
2734 Ops.push_back(Flag);
2735
2736 SDNode *Ret =
Justin Holewinskidff28d22013-07-01 12:59:01 +0000[diff] [blame]2737 CurDAG->getMachineNode(Opc, DL, VTs, Ops);
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]2738 return Ret;
2739}
2740
2741SDNode *NVPTXDAGToDAGISel::SelectStoreRetval(SDNode *N) {
2742 SDLoc DL(N);
2743 SDValue Chain = N->getOperand(0);
2744 SDValue Offset = N->getOperand(1);
2745 unsigned OffsetVal = cast<ConstantSDNode>(Offset)->getZExtValue();
2746 MemSDNode *Mem = cast<MemSDNode>(N);
2747
2748 // How many elements do we have?
2749 unsigned NumElts = 1;
2750 switch (N->getOpcode()) {
2751 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2752 return nullptr;
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]2753 case NVPTXISD::StoreRetval:
2754 NumElts = 1;
2755 break;
2756 case NVPTXISD::StoreRetvalV2:
2757 NumElts = 2;
2758 break;
2759 case NVPTXISD::StoreRetvalV4:
2760 NumElts = 4;
2761 break;
2762 }
2763
2764 // Build vector of operands
2765 SmallVector<SDValue, 6> Ops;
2766 for (unsigned i = 0; i < NumElts; ++i)
2767 Ops.push_back(N->getOperand(i + 2));
2768 Ops.push_back(CurDAG->getTargetConstant(OffsetVal, MVT::i32));
2769 Ops.push_back(Chain);
2770
2771 // Determine target opcode
2772 // If we have an i1, use an 8-bit store. The lowering code in
2773 // NVPTXISelLowering will have already emitted an upcast.
2774 unsigned Opcode = 0;
2775 switch (NumElts) {
2776 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2777 return nullptr;
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]2778 case 1:
2779 switch (Mem->getMemoryVT().getSimpleVT().SimpleTy) {
2780 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2781 return nullptr;
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]2782 case MVT::i1:
2783 Opcode = NVPTX::StoreRetvalI8;
2784 break;
2785 case MVT::i8:
2786 Opcode = NVPTX::StoreRetvalI8;
2787 break;
2788 case MVT::i16:
2789 Opcode = NVPTX::StoreRetvalI16;
2790 break;
2791 case MVT::i32:
2792 Opcode = NVPTX::StoreRetvalI32;
2793 break;
2794 case MVT::i64:
2795 Opcode = NVPTX::StoreRetvalI64;
2796 break;
2797 case MVT::f32:
2798 Opcode = NVPTX::StoreRetvalF32;
2799 break;
2800 case MVT::f64:
2801 Opcode = NVPTX::StoreRetvalF64;
2802 break;
2803 }
2804 break;
2805 case 2:
2806 switch (Mem->getMemoryVT().getSimpleVT().SimpleTy) {
2807 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2808 return nullptr;
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]2809 case MVT::i1:
2810 Opcode = NVPTX::StoreRetvalV2I8;
2811 break;
2812 case MVT::i8:
2813 Opcode = NVPTX::StoreRetvalV2I8;
2814 break;
2815 case MVT::i16:
2816 Opcode = NVPTX::StoreRetvalV2I16;
2817 break;
2818 case MVT::i32:
2819 Opcode = NVPTX::StoreRetvalV2I32;
2820 break;
2821 case MVT::i64:
2822 Opcode = NVPTX::StoreRetvalV2I64;
2823 break;
2824 case MVT::f32:
2825 Opcode = NVPTX::StoreRetvalV2F32;
2826 break;
2827 case MVT::f64:
2828 Opcode = NVPTX::StoreRetvalV2F64;
2829 break;
2830 }
2831 break;
2832 case 4:
2833 switch (Mem->getMemoryVT().getSimpleVT().SimpleTy) {
2834 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2835 return nullptr;
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]2836 case MVT::i1:
2837 Opcode = NVPTX::StoreRetvalV4I8;
2838 break;
2839 case MVT::i8:
2840 Opcode = NVPTX::StoreRetvalV4I8;
2841 break;
2842 case MVT::i16:
2843 Opcode = NVPTX::StoreRetvalV4I16;
2844 break;
2845 case MVT::i32:
2846 Opcode = NVPTX::StoreRetvalV4I32;
2847 break;
2848 case MVT::f32:
2849 Opcode = NVPTX::StoreRetvalV4F32;
2850 break;
2851 }
2852 break;
2853 }
2854
2855 SDNode *Ret =
2856 CurDAG->getMachineNode(Opcode, DL, MVT::Other, Ops);
2857 MachineSDNode::mmo_iterator MemRefs0 = MF->allocateMemRefsArray(1);
2858 MemRefs0[0] = cast<MemSDNode>(N)->getMemOperand();
2859 cast<MachineSDNode>(Ret)->setMemRefs(MemRefs0, MemRefs0 + 1);
2860
2861 return Ret;
2862}
2863
2864SDNode *NVPTXDAGToDAGISel::SelectStoreParam(SDNode *N) {
2865 SDLoc DL(N);
2866 SDValue Chain = N->getOperand(0);
2867 SDValue Param = N->getOperand(1);
2868 unsigned ParamVal = cast<ConstantSDNode>(Param)->getZExtValue();
2869 SDValue Offset = N->getOperand(2);
2870 unsigned OffsetVal = cast<ConstantSDNode>(Offset)->getZExtValue();
2871 MemSDNode *Mem = cast<MemSDNode>(N);
2872 SDValue Flag = N->getOperand(N->getNumOperands() - 1);
2873
2874 // How many elements do we have?
2875 unsigned NumElts = 1;
2876 switch (N->getOpcode()) {
2877 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2878 return nullptr;
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]2879 case NVPTXISD::StoreParamU32:
2880 case NVPTXISD::StoreParamS32:
2881 case NVPTXISD::StoreParam:
2882 NumElts = 1;
2883 break;
2884 case NVPTXISD::StoreParamV2:
2885 NumElts = 2;
2886 break;
2887 case NVPTXISD::StoreParamV4:
2888 NumElts = 4;
2889 break;
2890 }
2891
2892 // Build vector of operands
2893 SmallVector<SDValue, 8> Ops;
2894 for (unsigned i = 0; i < NumElts; ++i)
2895 Ops.push_back(N->getOperand(i + 3));
2896 Ops.push_back(CurDAG->getTargetConstant(ParamVal, MVT::i32));
2897 Ops.push_back(CurDAG->getTargetConstant(OffsetVal, MVT::i32));
2898 Ops.push_back(Chain);
2899 Ops.push_back(Flag);
2900
2901 // Determine target opcode
2902 // If we have an i1, use an 8-bit store. The lowering code in
2903 // NVPTXISelLowering will have already emitted an upcast.
2904 unsigned Opcode = 0;
2905 switch (N->getOpcode()) {
2906 default:
2907 switch (NumElts) {
2908 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2909 return nullptr;
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]2910 case 1:
2911 switch (Mem->getMemoryVT().getSimpleVT().SimpleTy) {
2912 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2913 return nullptr;
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]2914 case MVT::i1:
2915 Opcode = NVPTX::StoreParamI8;
2916 break;
2917 case MVT::i8:
2918 Opcode = NVPTX::StoreParamI8;
2919 break;
2920 case MVT::i16:
2921 Opcode = NVPTX::StoreParamI16;
2922 break;
2923 case MVT::i32:
2924 Opcode = NVPTX::StoreParamI32;
2925 break;
2926 case MVT::i64:
2927 Opcode = NVPTX::StoreParamI64;
2928 break;
2929 case MVT::f32:
2930 Opcode = NVPTX::StoreParamF32;
2931 break;
2932 case MVT::f64:
2933 Opcode = NVPTX::StoreParamF64;
2934 break;
2935 }
2936 break;
2937 case 2:
2938 switch (Mem->getMemoryVT().getSimpleVT().SimpleTy) {
2939 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2940 return nullptr;
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]2941 case MVT::i1:
2942 Opcode = NVPTX::StoreParamV2I8;
2943 break;
2944 case MVT::i8:
2945 Opcode = NVPTX::StoreParamV2I8;
2946 break;
2947 case MVT::i16:
2948 Opcode = NVPTX::StoreParamV2I16;
2949 break;
2950 case MVT::i32:
2951 Opcode = NVPTX::StoreParamV2I32;
2952 break;
2953 case MVT::i64:
2954 Opcode = NVPTX::StoreParamV2I64;
2955 break;
2956 case MVT::f32:
2957 Opcode = NVPTX::StoreParamV2F32;
2958 break;
2959 case MVT::f64:
2960 Opcode = NVPTX::StoreParamV2F64;
2961 break;
2962 }
2963 break;
2964 case 4:
2965 switch (Mem->getMemoryVT().getSimpleVT().SimpleTy) {
2966 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2967 return nullptr;
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]2968 case MVT::i1:
2969 Opcode = NVPTX::StoreParamV4I8;
2970 break;
2971 case MVT::i8:
2972 Opcode = NVPTX::StoreParamV4I8;
2973 break;
2974 case MVT::i16:
2975 Opcode = NVPTX::StoreParamV4I16;
2976 break;
2977 case MVT::i32:
2978 Opcode = NVPTX::StoreParamV4I32;
2979 break;
2980 case MVT::f32:
2981 Opcode = NVPTX::StoreParamV4F32;
2982 break;
2983 }
2984 break;
2985 }
2986 break;
Justin Holewinskidc5e3b62013-06-28 17:58:04 +0000[diff] [blame]2987 // Special case: if we have a sign-extend/zero-extend node, insert the
2988 // conversion instruction first, and use that as the value operand to
2989 // the selected StoreParam node.
2990 case NVPTXISD::StoreParamU32: {
2991 Opcode = NVPTX::StoreParamI32;
2992 SDValue CvtNone = CurDAG->getTargetConstant(NVPTX::PTXCvtMode::NONE,
2993 MVT::i32);
2994 SDNode *Cvt = CurDAG->getMachineNode(NVPTX::CVT_u32_u16, DL,
2995 MVT::i32, Ops[0], CvtNone);
2996 Ops[0] = SDValue(Cvt, 0);
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]2997 break;
Justin Holewinskidc5e3b62013-06-28 17:58:04 +0000[diff] [blame]2998 }
2999 case NVPTXISD::StoreParamS32: {
3000 Opcode = NVPTX::StoreParamI32;
3001 SDValue CvtNone = CurDAG->getTargetConstant(NVPTX::PTXCvtMode::NONE,
3002 MVT::i32);
3003 SDNode *Cvt = CurDAG->getMachineNode(NVPTX::CVT_s32_s16, DL,
3004 MVT::i32, Ops[0], CvtNone);
3005 Ops[0] = SDValue(Cvt, 0);
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]3006 break;
3007 }
Justin Holewinskidc5e3b62013-06-28 17:58:04 +0000[diff] [blame]3008 }
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]3009
Justin Holewinskidff28d22013-07-01 12:59:01 +0000[diff] [blame]3010 SDVTList RetVTs = CurDAG->getVTList(MVT::Other, MVT::Glue);
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]3011 SDNode *Ret =
Justin Holewinskidff28d22013-07-01 12:59:01 +0000[diff] [blame]3012 CurDAG->getMachineNode(Opcode, DL, RetVTs, Ops);
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]3013 MachineSDNode::mmo_iterator MemRefs0 = MF->allocateMemRefsArray(1);
3014 MemRefs0[0] = cast<MemSDNode>(N)->getMemOperand();
3015 cast<MachineSDNode>(Ret)->setMemRefs(MemRefs0, MemRefs0 + 1);
3016
3017 return Ret;
3018}
3019
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]3020SDNode *NVPTXDAGToDAGISel::SelectTextureIntrinsic(SDNode *N) {
3021 SDValue Chain = N->getOperand(0);
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]3022 SDNode *Ret = nullptr;
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]3023 unsigned Opc = 0;
3024 SmallVector<SDValue, 8> Ops;
3025
3026 switch (N->getOpcode()) {
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]3027 default: return nullptr;
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]3028 case NVPTXISD::Tex1DFloatS32:
3029 Opc = NVPTX::TEX_1D_F32_S32;
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]3030 break;
3031 case NVPTXISD::Tex1DFloatFloat:
3032 Opc = NVPTX::TEX_1D_F32_F32;
3033 break;
3034 case NVPTXISD::Tex1DFloatFloatLevel:
3035 Opc = NVPTX::TEX_1D_F32_F32_LEVEL;
3036 break;
3037 case NVPTXISD::Tex1DFloatFloatGrad:
3038 Opc = NVPTX::TEX_1D_F32_F32_GRAD;
3039 break;
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]3040 case NVPTXISD::Tex1DS32S32:
3041 Opc = NVPTX::TEX_1D_S32_S32;
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]3042 break;
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]3043 case NVPTXISD::Tex1DS32Float:
3044 Opc = NVPTX::TEX_1D_S32_F32;
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]3045 break;
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]3046 case NVPTXISD::Tex1DS32FloatLevel:
3047 Opc = NVPTX::TEX_1D_S32_F32_LEVEL;
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]3048 break;
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]3049 case NVPTXISD::Tex1DS32FloatGrad:
3050 Opc = NVPTX::TEX_1D_S32_F32_GRAD;
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]3051 break;
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]3052 case NVPTXISD::Tex1DU32S32:
3053 Opc = NVPTX::TEX_1D_U32_S32;
3054 break;
3055 case NVPTXISD::Tex1DU32Float:
3056 Opc = NVPTX::TEX_1D_U32_F32;
3057 break;
3058 case NVPTXISD::Tex1DU32FloatLevel:
3059 Opc = NVPTX::TEX_1D_U32_F32_LEVEL;
3060 break;
3061 case NVPTXISD::Tex1DU32FloatGrad:
3062 Opc = NVPTX::TEX_1D_U32_F32_GRAD;
3063 break;
3064 case NVPTXISD::Tex1DArrayFloatS32:
3065 Opc = NVPTX::TEX_1D_ARRAY_F32_S32;
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]3066 break;
3067 case NVPTXISD::Tex1DArrayFloatFloat:
3068 Opc = NVPTX::TEX_1D_ARRAY_F32_F32;
3069 break;
3070 case NVPTXISD::Tex1DArrayFloatFloatLevel:
3071 Opc = NVPTX::TEX_1D_ARRAY_F32_F32_LEVEL;
3072 break;
3073 case NVPTXISD::Tex1DArrayFloatFloatGrad:
3074 Opc = NVPTX::TEX_1D_ARRAY_F32_F32_GRAD;
3075 break;
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]3076 case NVPTXISD::Tex1DArrayS32S32:
3077 Opc = NVPTX::TEX_1D_ARRAY_S32_S32;
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]3078 break;
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]3079 case NVPTXISD::Tex1DArrayS32Float:
3080 Opc = NVPTX::TEX_1D_ARRAY_S32_F32;
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]3081 break;
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]3082 case NVPTXISD::Tex1DArrayS32FloatLevel:
3083 Opc = NVPTX::TEX_1D_ARRAY_S32_F32_LEVEL;
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]3084 break;
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]3085 case NVPTXISD::Tex1DArrayS32FloatGrad:
3086 Opc = NVPTX::TEX_1D_ARRAY_S32_F32_GRAD;
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]3087 break;
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]3088 case NVPTXISD::Tex1DArrayU32S32:
3089 Opc = NVPTX::TEX_1D_ARRAY_U32_S32;
3090 break;
3091 case NVPTXISD::Tex1DArrayU32Float:
3092 Opc = NVPTX::TEX_1D_ARRAY_U32_F32;
3093 break;
3094 case NVPTXISD::Tex1DArrayU32FloatLevel:
3095 Opc = NVPTX::TEX_1D_ARRAY_U32_F32_LEVEL;
3096 break;
3097 case NVPTXISD::Tex1DArrayU32FloatGrad:
3098 Opc = NVPTX::TEX_1D_ARRAY_U32_F32_GRAD;
3099 break;
3100 case NVPTXISD::Tex2DFloatS32:
3101 Opc = NVPTX::TEX_2D_F32_S32;
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]3102 break;
3103 case NVPTXISD::Tex2DFloatFloat:
3104 Opc = NVPTX::TEX_2D_F32_F32;
3105 break;
3106 case NVPTXISD::Tex2DFloatFloatLevel:
3107 Opc = NVPTX::TEX_2D_F32_F32_LEVEL;
3108 break;
3109 case NVPTXISD::Tex2DFloatFloatGrad:
3110 Opc = NVPTX::TEX_2D_F32_F32_GRAD;
3111 break;
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]3112 case NVPTXISD::Tex2DS32S32:
3113 Opc = NVPTX::TEX_2D_S32_S32;
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]3114 break;
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]3115 case NVPTXISD::Tex2DS32Float:
3116 Opc = NVPTX::TEX_2D_S32_F32;
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]3117 break;
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]3118 case NVPTXISD::Tex2DS32FloatLevel:
3119 Opc = NVPTX::TEX_2D_S32_F32_LEVEL;
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]3120 break;
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]3121 case NVPTXISD::Tex2DS32FloatGrad:
3122 Opc = NVPTX::TEX_2D_S32_F32_GRAD;
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]3123 break;
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]3124 case NVPTXISD::Tex2DU32S32:
3125 Opc = NVPTX::TEX_2D_U32_S32;
3126 break;
3127 case NVPTXISD::Tex2DU32Float:
3128 Opc = NVPTX::TEX_2D_U32_F32;
3129 break;
3130 case NVPTXISD::Tex2DU32FloatLevel:
3131 Opc = NVPTX::TEX_2D_U32_F32_LEVEL;
3132 break;
3133 case NVPTXISD::Tex2DU32FloatGrad:
3134 Opc = NVPTX::TEX_2D_U32_F32_GRAD;
3135 break;
3136 case NVPTXISD::Tex2DArrayFloatS32:
3137 Opc = NVPTX::TEX_2D_ARRAY_F32_S32;
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]3138 break;
3139 case NVPTXISD::Tex2DArrayFloatFloat:
3140 Opc = NVPTX::TEX_2D_ARRAY_F32_F32;
3141 break;
3142 case NVPTXISD::Tex2DArrayFloatFloatLevel:
3143 Opc = NVPTX::TEX_2D_ARRAY_F32_F32_LEVEL;
3144 break;
3145 case NVPTXISD::Tex2DArrayFloatFloatGrad:
3146 Opc = NVPTX::TEX_2D_ARRAY_F32_F32_GRAD;
3147 break;
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]3148 case NVPTXISD::Tex2DArrayS32S32:
3149 Opc = NVPTX::TEX_2D_ARRAY_S32_S32;
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]3150 break;
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]3151 case NVPTXISD::Tex2DArrayS32Float:
3152 Opc = NVPTX::TEX_2D_ARRAY_S32_F32;
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]3153 break;
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]3154 case NVPTXISD::Tex2DArrayS32FloatLevel:
3155 Opc = NVPTX::TEX_2D_ARRAY_S32_F32_LEVEL;
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]3156 break;
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]3157 case NVPTXISD::Tex2DArrayS32FloatGrad:
3158 Opc = NVPTX::TEX_2D_ARRAY_S32_F32_GRAD;
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]3159 break;
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]3160 case NVPTXISD::Tex2DArrayU32S32:
3161 Opc = NVPTX::TEX_2D_ARRAY_U32_S32;
3162 break;
3163 case NVPTXISD::Tex2DArrayU32Float:
3164 Opc = NVPTX::TEX_2D_ARRAY_U32_F32;
3165 break;
3166 case NVPTXISD::Tex2DArrayU32FloatLevel:
3167 Opc = NVPTX::TEX_2D_ARRAY_U32_F32_LEVEL;
3168 break;
3169 case NVPTXISD::Tex2DArrayU32FloatGrad:
3170 Opc = NVPTX::TEX_2D_ARRAY_U32_F32_GRAD;
3171 break;
3172 case NVPTXISD::Tex3DFloatS32:
3173 Opc = NVPTX::TEX_3D_F32_S32;
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]3174 break;
3175 case NVPTXISD::Tex3DFloatFloat:
3176 Opc = NVPTX::TEX_3D_F32_F32;
3177 break;
3178 case NVPTXISD::Tex3DFloatFloatLevel:
3179 Opc = NVPTX::TEX_3D_F32_F32_LEVEL;
3180 break;
3181 case NVPTXISD::Tex3DFloatFloatGrad:
3182 Opc = NVPTX::TEX_3D_F32_F32_GRAD;
3183 break;
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]3184 case NVPTXISD::Tex3DS32S32:
3185 Opc = NVPTX::TEX_3D_S32_S32;
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]3186 break;
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]3187 case NVPTXISD::Tex3DS32Float:
3188 Opc = NVPTX::TEX_3D_S32_F32;
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]3189 break;
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]3190 case NVPTXISD::Tex3DS32FloatLevel:
3191 Opc = NVPTX::TEX_3D_S32_F32_LEVEL;
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]3192 break;
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]3193 case NVPTXISD::Tex3DS32FloatGrad:
3194 Opc = NVPTX::TEX_3D_S32_F32_GRAD;
3195 break;
3196 case NVPTXISD::Tex3DU32S32:
3197 Opc = NVPTX::TEX_3D_U32_S32;
3198 break;
3199 case NVPTXISD::Tex3DU32Float:
3200 Opc = NVPTX::TEX_3D_U32_F32;
3201 break;
3202 case NVPTXISD::Tex3DU32FloatLevel:
3203 Opc = NVPTX::TEX_3D_U32_F32_LEVEL;
3204 break;
3205 case NVPTXISD::Tex3DU32FloatGrad:
3206 Opc = NVPTX::TEX_3D_U32_F32_GRAD;
3207 break;
3208 case NVPTXISD::TexCubeFloatFloat:
3209 Opc = NVPTX::TEX_CUBE_F32_F32;
3210 break;
3211 case NVPTXISD::TexCubeFloatFloatLevel:
3212 Opc = NVPTX::TEX_CUBE_F32_F32_LEVEL;
3213 break;
3214 case NVPTXISD::TexCubeS32Float:
3215 Opc = NVPTX::TEX_CUBE_S32_F32;
3216 break;
3217 case NVPTXISD::TexCubeS32FloatLevel:
3218 Opc = NVPTX::TEX_CUBE_S32_F32_LEVEL;
3219 break;
3220 case NVPTXISD::TexCubeU32Float:
3221 Opc = NVPTX::TEX_CUBE_U32_F32;
3222 break;
3223 case NVPTXISD::TexCubeU32FloatLevel:
3224 Opc = NVPTX::TEX_CUBE_U32_F32_LEVEL;
3225 break;
3226 case NVPTXISD::TexCubeArrayFloatFloat:
3227 Opc = NVPTX::TEX_CUBE_ARRAY_F32_F32;
3228 break;
3229 case NVPTXISD::TexCubeArrayFloatFloatLevel:
3230 Opc = NVPTX::TEX_CUBE_ARRAY_F32_F32_LEVEL;
3231 break;
3232 case NVPTXISD::TexCubeArrayS32Float:
3233 Opc = NVPTX::TEX_CUBE_ARRAY_S32_F32;
3234 break;
3235 case NVPTXISD::TexCubeArrayS32FloatLevel:
3236 Opc = NVPTX::TEX_CUBE_ARRAY_S32_F32_LEVEL;
3237 break;
3238 case NVPTXISD::TexCubeArrayU32Float:
3239 Opc = NVPTX::TEX_CUBE_ARRAY_U32_F32;
3240 break;
3241 case NVPTXISD::TexCubeArrayU32FloatLevel:
3242 Opc = NVPTX::TEX_CUBE_ARRAY_U32_F32_LEVEL;
3243 break;
3244 case NVPTXISD::Tld4R2DFloatFloat:
3245 Opc = NVPTX::TLD4_R_2D_F32_F32;
3246 break;
3247 case NVPTXISD::Tld4G2DFloatFloat:
3248 Opc = NVPTX::TLD4_G_2D_F32_F32;
3249 break;
3250 case NVPTXISD::Tld4B2DFloatFloat:
3251 Opc = NVPTX::TLD4_B_2D_F32_F32;
3252 break;
3253 case NVPTXISD::Tld4A2DFloatFloat:
3254 Opc = NVPTX::TLD4_A_2D_F32_F32;
3255 break;
3256 case NVPTXISD::Tld4R2DS64Float:
3257 Opc = NVPTX::TLD4_R_2D_S32_F32;
3258 break;
3259 case NVPTXISD::Tld4G2DS64Float:
3260 Opc = NVPTX::TLD4_G_2D_S32_F32;
3261 break;
3262 case NVPTXISD::Tld4B2DS64Float:
3263 Opc = NVPTX::TLD4_B_2D_S32_F32;
3264 break;
3265 case NVPTXISD::Tld4A2DS64Float:
3266 Opc = NVPTX::TLD4_A_2D_S32_F32;
3267 break;
3268 case NVPTXISD::Tld4R2DU64Float:
3269 Opc = NVPTX::TLD4_R_2D_U32_F32;
3270 break;
3271 case NVPTXISD::Tld4G2DU64Float:
3272 Opc = NVPTX::TLD4_G_2D_U32_F32;
3273 break;
3274 case NVPTXISD::Tld4B2DU64Float:
3275 Opc = NVPTX::TLD4_B_2D_U32_F32;
3276 break;
3277 case NVPTXISD::Tld4A2DU64Float:
3278 Opc = NVPTX::TLD4_A_2D_U32_F32;
3279 break;
3280 case NVPTXISD::TexUnified1DFloatS32:
3281 Opc = NVPTX::TEX_UNIFIED_1D_F32_S32;
3282 break;
3283 case NVPTXISD::TexUnified1DFloatFloat:
3284 Opc = NVPTX::TEX_UNIFIED_1D_F32_F32;
3285 break;
3286 case NVPTXISD::TexUnified1DFloatFloatLevel:
3287 Opc = NVPTX::TEX_UNIFIED_1D_F32_F32_LEVEL;
3288 break;
3289 case NVPTXISD::TexUnified1DFloatFloatGrad:
3290 Opc = NVPTX::TEX_UNIFIED_1D_F32_F32_GRAD;
3291 break;
3292 case NVPTXISD::TexUnified1DS32S32:
3293 Opc = NVPTX::TEX_UNIFIED_1D_S32_S32;
3294 break;
3295 case NVPTXISD::TexUnified1DS32Float:
3296 Opc = NVPTX::TEX_UNIFIED_1D_S32_F32;
3297 break;
3298 case NVPTXISD::TexUnified1DS32FloatLevel:
3299 Opc = NVPTX::TEX_UNIFIED_1D_S32_F32_LEVEL;
3300 break;
3301 case NVPTXISD::TexUnified1DS32FloatGrad:
3302 Opc = NVPTX::TEX_UNIFIED_1D_S32_F32_GRAD;
3303 break;
3304 case NVPTXISD::TexUnified1DU32S32:
3305 Opc = NVPTX::TEX_UNIFIED_1D_U32_S32;
3306 break;
3307 case NVPTXISD::TexUnified1DU32Float:
3308 Opc = NVPTX::TEX_UNIFIED_1D_U32_F32;
3309 break;
3310 case NVPTXISD::TexUnified1DU32FloatLevel:
3311 Opc = NVPTX::TEX_UNIFIED_1D_U32_F32_LEVEL;
3312 break;
3313 case NVPTXISD::TexUnified1DU32FloatGrad:
3314 Opc = NVPTX::TEX_UNIFIED_1D_U32_F32_GRAD;
3315 break;
3316 case NVPTXISD::TexUnified1DArrayFloatS32:
3317 Opc = NVPTX::TEX_UNIFIED_1D_ARRAY_F32_S32;
3318 break;
3319 case NVPTXISD::TexUnified1DArrayFloatFloat:
3320 Opc = NVPTX::TEX_UNIFIED_1D_ARRAY_F32_F32;
3321 break;
3322 case NVPTXISD::TexUnified1DArrayFloatFloatLevel:
3323 Opc = NVPTX::TEX_UNIFIED_1D_ARRAY_F32_F32_LEVEL;
3324 break;
3325 case NVPTXISD::TexUnified1DArrayFloatFloatGrad:
3326 Opc = NVPTX::TEX_UNIFIED_1D_ARRAY_F32_F32_GRAD;
3327 break;
3328 case NVPTXISD::TexUnified1DArrayS32S32:
3329 Opc = NVPTX::TEX_UNIFIED_1D_ARRAY_S32_S32;
3330 break;
3331 case NVPTXISD::TexUnified1DArrayS32Float:
3332 Opc = NVPTX::TEX_UNIFIED_1D_ARRAY_S32_F32;
3333 break;
3334 case NVPTXISD::TexUnified1DArrayS32FloatLevel:
3335 Opc = NVPTX::TEX_UNIFIED_1D_ARRAY_S32_F32_LEVEL;
3336 break;
3337 case NVPTXISD::TexUnified1DArrayS32FloatGrad:
3338 Opc = NVPTX::TEX_UNIFIED_1D_ARRAY_S32_F32_GRAD;
3339 break;
3340 case NVPTXISD::TexUnified1DArrayU32S32:
3341 Opc = NVPTX::TEX_UNIFIED_1D_ARRAY_U32_S32;
3342 break;
3343 case NVPTXISD::TexUnified1DArrayU32Float:
3344 Opc = NVPTX::TEX_UNIFIED_1D_ARRAY_U32_F32;
3345 break;
3346 case NVPTXISD::TexUnified1DArrayU32FloatLevel:
3347 Opc = NVPTX::TEX_UNIFIED_1D_ARRAY_U32_F32_LEVEL;
3348 break;
3349 case NVPTXISD::TexUnified1DArrayU32FloatGrad:
3350 Opc = NVPTX::TEX_UNIFIED_1D_ARRAY_U32_F32_GRAD;
3351 break;
3352 case NVPTXISD::TexUnified2DFloatS32:
3353 Opc = NVPTX::TEX_UNIFIED_2D_F32_S32;
3354 break;
3355 case NVPTXISD::TexUnified2DFloatFloat:
3356 Opc = NVPTX::TEX_UNIFIED_2D_F32_F32;
3357 break;
3358 case NVPTXISD::TexUnified2DFloatFloatLevel:
3359 Opc = NVPTX::TEX_UNIFIED_2D_F32_F32_LEVEL;
3360 break;
3361 case NVPTXISD::TexUnified2DFloatFloatGrad:
3362 Opc = NVPTX::TEX_UNIFIED_2D_F32_F32_GRAD;
3363 break;
3364 case NVPTXISD::TexUnified2DS32S32:
3365 Opc = NVPTX::TEX_UNIFIED_2D_S32_S32;
3366 break;
3367 case NVPTXISD::TexUnified2DS32Float:
3368 Opc = NVPTX::TEX_UNIFIED_2D_S32_F32;
3369 break;
3370 case NVPTXISD::TexUnified2DS32FloatLevel:
3371 Opc = NVPTX::TEX_UNIFIED_2D_S32_F32_LEVEL;
3372 break;
3373 case NVPTXISD::TexUnified2DS32FloatGrad:
3374 Opc = NVPTX::TEX_UNIFIED_2D_S32_F32_GRAD;
3375 break;
3376 case NVPTXISD::TexUnified2DU32S32:
3377 Opc = NVPTX::TEX_UNIFIED_2D_U32_S32;
3378 break;
3379 case NVPTXISD::TexUnified2DU32Float:
3380 Opc = NVPTX::TEX_UNIFIED_2D_U32_F32;
3381 break;
3382 case NVPTXISD::TexUnified2DU32FloatLevel:
3383 Opc = NVPTX::TEX_UNIFIED_2D_U32_F32_LEVEL;
3384 break;
3385 case NVPTXISD::TexUnified2DU32FloatGrad:
3386 Opc = NVPTX::TEX_UNIFIED_2D_U32_F32_GRAD;
3387 break;
3388 case NVPTXISD::TexUnified2DArrayFloatS32:
3389 Opc = NVPTX::TEX_UNIFIED_2D_ARRAY_F32_S32;
3390 break;
3391 case NVPTXISD::TexUnified2DArrayFloatFloat:
3392 Opc = NVPTX::TEX_UNIFIED_2D_ARRAY_F32_F32;
3393 break;
3394 case NVPTXISD::TexUnified2DArrayFloatFloatLevel:
3395 Opc = NVPTX::TEX_UNIFIED_2D_ARRAY_F32_F32_LEVEL;
3396 break;
3397 case NVPTXISD::TexUnified2DArrayFloatFloatGrad:
3398 Opc = NVPTX::TEX_UNIFIED_2D_ARRAY_F32_F32_GRAD;
3399 break;
3400 case NVPTXISD::TexUnified2DArrayS32S32:
3401 Opc = NVPTX::TEX_UNIFIED_2D_ARRAY_S32_S32;
3402 break;
3403 case NVPTXISD::TexUnified2DArrayS32Float:
3404 Opc = NVPTX::TEX_UNIFIED_2D_ARRAY_S32_F32;
3405 break;
3406 case NVPTXISD::TexUnified2DArrayS32FloatLevel:
3407 Opc = NVPTX::TEX_UNIFIED_2D_ARRAY_S32_F32_LEVEL;
3408 break;
3409 case NVPTXISD::TexUnified2DArrayS32FloatGrad:
3410 Opc = NVPTX::TEX_UNIFIED_2D_ARRAY_S32_F32_GRAD;
3411 break;
3412 case NVPTXISD::TexUnified2DArrayU32S32:
3413 Opc = NVPTX::TEX_UNIFIED_2D_ARRAY_U32_S32;
3414 break;
3415 case NVPTXISD::TexUnified2DArrayU32Float:
3416 Opc = NVPTX::TEX_UNIFIED_2D_ARRAY_U32_F32;
3417 break;
3418 case NVPTXISD::TexUnified2DArrayU32FloatLevel:
3419 Opc = NVPTX::TEX_UNIFIED_2D_ARRAY_U32_F32_LEVEL;
3420 break;
3421 case NVPTXISD::TexUnified2DArrayU32FloatGrad:
3422 Opc = NVPTX::TEX_UNIFIED_2D_ARRAY_U32_F32_GRAD;
3423 break;
3424 case NVPTXISD::TexUnified3DFloatS32:
3425 Opc = NVPTX::TEX_UNIFIED_3D_F32_S32;
3426 break;
3427 case NVPTXISD::TexUnified3DFloatFloat:
3428 Opc = NVPTX::TEX_UNIFIED_3D_F32_F32;
3429 break;
3430 case NVPTXISD::TexUnified3DFloatFloatLevel:
3431 Opc = NVPTX::TEX_UNIFIED_3D_F32_F32_LEVEL;
3432 break;
3433 case NVPTXISD::TexUnified3DFloatFloatGrad:
3434 Opc = NVPTX::TEX_UNIFIED_3D_F32_F32_GRAD;
3435 break;
3436 case NVPTXISD::TexUnified3DS32S32:
3437 Opc = NVPTX::TEX_UNIFIED_3D_S32_S32;
3438 break;
3439 case NVPTXISD::TexUnified3DS32Float:
3440 Opc = NVPTX::TEX_UNIFIED_3D_S32_F32;
3441 break;
3442 case NVPTXISD::TexUnified3DS32FloatLevel:
3443 Opc = NVPTX::TEX_UNIFIED_3D_S32_F32_LEVEL;
3444 break;
3445 case NVPTXISD::TexUnified3DS32FloatGrad:
3446 Opc = NVPTX::TEX_UNIFIED_3D_S32_F32_GRAD;
3447 break;
3448 case NVPTXISD::TexUnified3DU32S32:
3449 Opc = NVPTX::TEX_UNIFIED_3D_U32_S32;
3450 break;
3451 case NVPTXISD::TexUnified3DU32Float:
3452 Opc = NVPTX::TEX_UNIFIED_3D_U32_F32;
3453 break;
3454 case NVPTXISD::TexUnified3DU32FloatLevel:
3455 Opc = NVPTX::TEX_UNIFIED_3D_U32_F32_LEVEL;
3456 break;
3457 case NVPTXISD::TexUnified3DU32FloatGrad:
3458 Opc = NVPTX::TEX_UNIFIED_3D_U32_F32_GRAD;
3459 break;
3460 case NVPTXISD::TexUnifiedCubeFloatFloat:
3461 Opc = NVPTX::TEX_UNIFIED_CUBE_F32_F32;
3462 break;
3463 case NVPTXISD::TexUnifiedCubeFloatFloatLevel:
3464 Opc = NVPTX::TEX_UNIFIED_CUBE_F32_F32_LEVEL;
3465 break;
3466 case NVPTXISD::TexUnifiedCubeS32Float:
3467 Opc = NVPTX::TEX_UNIFIED_CUBE_S32_F32;
3468 break;
3469 case NVPTXISD::TexUnifiedCubeS32FloatLevel:
3470 Opc = NVPTX::TEX_UNIFIED_CUBE_S32_F32_LEVEL;
3471 break;
3472 case NVPTXISD::TexUnifiedCubeU32Float:
3473 Opc = NVPTX::TEX_UNIFIED_CUBE_U32_F32;
3474 break;
3475 case NVPTXISD::TexUnifiedCubeU32FloatLevel:
3476 Opc = NVPTX::TEX_UNIFIED_CUBE_U32_F32_LEVEL;
3477 break;
3478 case NVPTXISD::TexUnifiedCubeArrayFloatFloat:
3479 Opc = NVPTX::TEX_UNIFIED_CUBE_ARRAY_F32_F32;
3480 break;
3481 case NVPTXISD::TexUnifiedCubeArrayFloatFloatLevel:
3482 Opc = NVPTX::TEX_UNIFIED_CUBE_ARRAY_F32_F32_LEVEL;
3483 break;
3484 case NVPTXISD::TexUnifiedCubeArrayS32Float:
3485 Opc = NVPTX::TEX_UNIFIED_CUBE_ARRAY_S32_F32;
3486 break;
3487 case NVPTXISD::TexUnifiedCubeArrayS32FloatLevel:
3488 Opc = NVPTX::TEX_UNIFIED_CUBE_ARRAY_S32_F32_LEVEL;
3489 break;
3490 case NVPTXISD::TexUnifiedCubeArrayU32Float:
3491 Opc = NVPTX::TEX_UNIFIED_CUBE_ARRAY_U32_F32;
3492 break;
3493 case NVPTXISD::TexUnifiedCubeArrayU32FloatLevel:
3494 Opc = NVPTX::TEX_UNIFIED_CUBE_ARRAY_U32_F32_LEVEL;
3495 break;
3496 case NVPTXISD::Tld4UnifiedR2DFloatFloat:
3497 Opc = NVPTX::TLD4_UNIFIED_R_2D_F32_F32;
3498 break;
3499 case NVPTXISD::Tld4UnifiedG2DFloatFloat:
3500 Opc = NVPTX::TLD4_UNIFIED_G_2D_F32_F32;
3501 break;
3502 case NVPTXISD::Tld4UnifiedB2DFloatFloat:
3503 Opc = NVPTX::TLD4_UNIFIED_B_2D_F32_F32;
3504 break;
3505 case NVPTXISD::Tld4UnifiedA2DFloatFloat:
3506 Opc = NVPTX::TLD4_UNIFIED_A_2D_F32_F32;
3507 break;
3508 case NVPTXISD::Tld4UnifiedR2DS64Float:
3509 Opc = NVPTX::TLD4_UNIFIED_R_2D_S32_F32;
3510 break;
3511 case NVPTXISD::Tld4UnifiedG2DS64Float:
3512 Opc = NVPTX::TLD4_UNIFIED_G_2D_S32_F32;
3513 break;
3514 case NVPTXISD::Tld4UnifiedB2DS64Float:
3515 Opc = NVPTX::TLD4_UNIFIED_B_2D_S32_F32;
3516 break;
3517 case NVPTXISD::Tld4UnifiedA2DS64Float:
3518 Opc = NVPTX::TLD4_UNIFIED_A_2D_S32_F32;
3519 break;
3520 case NVPTXISD::Tld4UnifiedR2DU64Float:
3521 Opc = NVPTX::TLD4_UNIFIED_R_2D_U32_F32;
3522 break;
3523 case NVPTXISD::Tld4UnifiedG2DU64Float:
3524 Opc = NVPTX::TLD4_UNIFIED_G_2D_U32_F32;
3525 break;
3526 case NVPTXISD::Tld4UnifiedB2DU64Float:
3527 Opc = NVPTX::TLD4_UNIFIED_B_2D_U32_F32;
3528 break;
3529 case NVPTXISD::Tld4UnifiedA2DU64Float:
3530 Opc = NVPTX::TLD4_UNIFIED_A_2D_U32_F32;
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]3531 break;
3532 }
3533
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]3534 // Copy over operands
3535 for (unsigned i = 1; i < N->getNumOperands(); ++i) {
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]3536 Ops.push_back(N->getOperand(i));
3537 }
3538
3539 Ops.push_back(Chain);
3540 Ret = CurDAG->getMachineNode(Opc, SDLoc(N), N->getVTList(), Ops);
3541 return Ret;
3542}
3543
3544SDNode *NVPTXDAGToDAGISel::SelectSurfaceIntrinsic(SDNode *N) {
3545 SDValue Chain = N->getOperand(0);
3546 SDValue TexHandle = N->getOperand(1);
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]3547 SDNode *Ret = nullptr;
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]3548 unsigned Opc = 0;
3549 SmallVector<SDValue, 8> Ops;
3550 switch (N->getOpcode()) {
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]3551 default: return nullptr;
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]3552 case NVPTXISD::Suld1DI8Clamp:
3553 Opc = NVPTX::SULD_1D_I8_CLAMP;
3554 Ops.push_back(TexHandle);
3555 Ops.push_back(N->getOperand(2));
3556 Ops.push_back(Chain);
3557 break;
3558 case NVPTXISD::Suld1DI16Clamp:
3559 Opc = NVPTX::SULD_1D_I16_CLAMP;
3560 Ops.push_back(TexHandle);
3561 Ops.push_back(N->getOperand(2));
3562 Ops.push_back(Chain);
3563 break;
3564 case NVPTXISD::Suld1DI32Clamp:
3565 Opc = NVPTX::SULD_1D_I32_CLAMP;
3566 Ops.push_back(TexHandle);
3567 Ops.push_back(N->getOperand(2));
3568 Ops.push_back(Chain);
3569 break;
3570 case NVPTXISD::Suld1DI64Clamp:
3571 Opc = NVPTX::SULD_1D_I64_CLAMP;
3572 Ops.push_back(TexHandle);
3573 Ops.push_back(N->getOperand(2));
3574 Ops.push_back(Chain);
3575 break;
3576 case NVPTXISD::Suld1DV2I8Clamp:
3577 Opc = NVPTX::SULD_1D_V2I8_CLAMP;
3578 Ops.push_back(TexHandle);
3579 Ops.push_back(N->getOperand(2));
3580 Ops.push_back(Chain);
3581 break;
3582 case NVPTXISD::Suld1DV2I16Clamp:
3583 Opc = NVPTX::SULD_1D_V2I16_CLAMP;
3584 Ops.push_back(TexHandle);
3585 Ops.push_back(N->getOperand(2));
3586 Ops.push_back(Chain);
3587 break;
3588 case NVPTXISD::Suld1DV2I32Clamp:
3589 Opc = NVPTX::SULD_1D_V2I32_CLAMP;
3590 Ops.push_back(TexHandle);
3591 Ops.push_back(N->getOperand(2));
3592 Ops.push_back(Chain);
3593 break;
3594 case NVPTXISD::Suld1DV2I64Clamp:
3595 Opc = NVPTX::SULD_1D_V2I64_CLAMP;
3596 Ops.push_back(TexHandle);
3597 Ops.push_back(N->getOperand(2));
3598 Ops.push_back(Chain);
3599 break;
3600 case NVPTXISD::Suld1DV4I8Clamp:
3601 Opc = NVPTX::SULD_1D_V4I8_CLAMP;
3602 Ops.push_back(TexHandle);
3603 Ops.push_back(N->getOperand(2));
3604 Ops.push_back(Chain);
3605 break;
3606 case NVPTXISD::Suld1DV4I16Clamp:
3607 Opc = NVPTX::SULD_1D_V4I16_CLAMP;
3608 Ops.push_back(TexHandle);
3609 Ops.push_back(N->getOperand(2));
3610 Ops.push_back(Chain);
3611 break;
3612 case NVPTXISD::Suld1DV4I32Clamp:
3613 Opc = NVPTX::SULD_1D_V4I32_CLAMP;
3614 Ops.push_back(TexHandle);
3615 Ops.push_back(N->getOperand(2));
3616 Ops.push_back(Chain);
3617 break;
3618 case NVPTXISD::Suld1DArrayI8Clamp:
3619 Opc = NVPTX::SULD_1D_ARRAY_I8_CLAMP;
3620 Ops.push_back(TexHandle);
3621 Ops.push_back(N->getOperand(2));
3622 Ops.push_back(N->getOperand(3));
3623 Ops.push_back(Chain);
3624 break;
3625 case NVPTXISD::Suld1DArrayI16Clamp:
3626 Opc = NVPTX::SULD_1D_ARRAY_I16_CLAMP;
3627 Ops.push_back(TexHandle);
3628 Ops.push_back(N->getOperand(2));
3629 Ops.push_back(N->getOperand(3));
3630 Ops.push_back(Chain);
3631 break;
3632 case NVPTXISD::Suld1DArrayI32Clamp:
3633 Opc = NVPTX::SULD_1D_ARRAY_I32_CLAMP;
3634 Ops.push_back(TexHandle);
3635 Ops.push_back(N->getOperand(2));
3636 Ops.push_back(N->getOperand(3));
3637 Ops.push_back(Chain);
3638 break;
3639 case NVPTXISD::Suld1DArrayI64Clamp:
3640 Opc = NVPTX::SULD_1D_ARRAY_I64_CLAMP;
3641 Ops.push_back(TexHandle);
3642 Ops.push_back(N->getOperand(2));
3643 Ops.push_back(N->getOperand(3));
3644 Ops.push_back(Chain);
3645 break;
3646 case NVPTXISD::Suld1DArrayV2I8Clamp:
3647 Opc = NVPTX::SULD_1D_ARRAY_V2I8_CLAMP;
3648 Ops.push_back(TexHandle);
3649 Ops.push_back(N->getOperand(2));
3650 Ops.push_back(N->getOperand(3));
3651 Ops.push_back(Chain);
3652 break;
3653 case NVPTXISD::Suld1DArrayV2I16Clamp:
3654 Opc = NVPTX::SULD_1D_ARRAY_V2I16_CLAMP;
3655 Ops.push_back(TexHandle);
3656 Ops.push_back(N->getOperand(2));
3657 Ops.push_back(N->getOperand(3));
3658 Ops.push_back(Chain);
3659 break;
3660 case NVPTXISD::Suld1DArrayV2I32Clamp:
3661 Opc = NVPTX::SULD_1D_ARRAY_V2I32_CLAMP;
3662 Ops.push_back(TexHandle);
3663 Ops.push_back(N->getOperand(2));
3664 Ops.push_back(N->getOperand(3));
3665 Ops.push_back(Chain);
3666 break;
3667 case NVPTXISD::Suld1DArrayV2I64Clamp:
3668 Opc = NVPTX::SULD_1D_ARRAY_V2I64_CLAMP;
3669 Ops.push_back(TexHandle);
3670 Ops.push_back(N->getOperand(2));
3671 Ops.push_back(N->getOperand(3));
3672 Ops.push_back(Chain);
3673 break;
3674 case NVPTXISD::Suld1DArrayV4I8Clamp:
3675 Opc = NVPTX::SULD_1D_ARRAY_V4I8_CLAMP;
3676 Ops.push_back(TexHandle);
3677 Ops.push_back(N->getOperand(2));
3678 Ops.push_back(N->getOperand(3));
3679 Ops.push_back(Chain);
3680 break;
3681 case NVPTXISD::Suld1DArrayV4I16Clamp:
3682 Opc = NVPTX::SULD_1D_ARRAY_V4I16_CLAMP;
3683 Ops.push_back(TexHandle);
3684 Ops.push_back(N->getOperand(2));
3685 Ops.push_back(N->getOperand(3));
3686 Ops.push_back(Chain);
3687 break;
3688 case NVPTXISD::Suld1DArrayV4I32Clamp:
3689 Opc = NVPTX::SULD_1D_ARRAY_V4I32_CLAMP;
3690 Ops.push_back(TexHandle);
3691 Ops.push_back(N->getOperand(2));
3692 Ops.push_back(N->getOperand(3));
3693 Ops.push_back(Chain);
3694 break;
3695 case NVPTXISD::Suld2DI8Clamp:
3696 Opc = NVPTX::SULD_2D_I8_CLAMP;
3697 Ops.push_back(TexHandle);
3698 Ops.push_back(N->getOperand(2));
3699 Ops.push_back(N->getOperand(3));
3700 Ops.push_back(Chain);
3701 break;
3702 case NVPTXISD::Suld2DI16Clamp:
3703 Opc = NVPTX::SULD_2D_I16_CLAMP;
3704 Ops.push_back(TexHandle);
3705 Ops.push_back(N->getOperand(2));
3706 Ops.push_back(N->getOperand(3));
3707 Ops.push_back(Chain);
3708 break;
3709 case NVPTXISD::Suld2DI32Clamp:
3710 Opc = NVPTX::SULD_2D_I32_CLAMP;
3711 Ops.push_back(TexHandle);
3712 Ops.push_back(N->getOperand(2));
3713 Ops.push_back(N->getOperand(3));
3714 Ops.push_back(Chain);
3715 break;
3716 case NVPTXISD::Suld2DI64Clamp:
3717 Opc = NVPTX::SULD_2D_I64_CLAMP;
3718 Ops.push_back(TexHandle);
3719 Ops.push_back(N->getOperand(2));
3720 Ops.push_back(N->getOperand(3));
3721 Ops.push_back(Chain);
3722 break;
3723 case NVPTXISD::Suld2DV2I8Clamp:
3724 Opc = NVPTX::SULD_2D_V2I8_CLAMP;
3725 Ops.push_back(TexHandle);
3726 Ops.push_back(N->getOperand(2));
3727 Ops.push_back(N->getOperand(3));
3728 Ops.push_back(Chain);
3729 break;
3730 case NVPTXISD::Suld2DV2I16Clamp:
3731 Opc = NVPTX::SULD_2D_V2I16_CLAMP;
3732 Ops.push_back(TexHandle);
3733 Ops.push_back(N->getOperand(2));
3734 Ops.push_back(N->getOperand(3));
3735 Ops.push_back(Chain);
3736 break;
3737 case NVPTXISD::Suld2DV2I32Clamp:
3738 Opc = NVPTX::SULD_2D_V2I32_CLAMP;
3739 Ops.push_back(TexHandle);
3740 Ops.push_back(N->getOperand(2));
3741 Ops.push_back(N->getOperand(3));
3742 Ops.push_back(Chain);
3743 break;
3744 case NVPTXISD::Suld2DV2I64Clamp:
3745 Opc = NVPTX::SULD_2D_V2I64_CLAMP;
3746 Ops.push_back(TexHandle);
3747 Ops.push_back(N->getOperand(2));
3748 Ops.push_back(N->getOperand(3));
3749 Ops.push_back(Chain);
3750 break;
3751 case NVPTXISD::Suld2DV4I8Clamp:
3752 Opc = NVPTX::SULD_2D_V4I8_CLAMP;
3753 Ops.push_back(TexHandle);
3754 Ops.push_back(N->getOperand(2));
3755 Ops.push_back(N->getOperand(3));
3756 Ops.push_back(Chain);
3757 break;
3758 case NVPTXISD::Suld2DV4I16Clamp:
3759 Opc = NVPTX::SULD_2D_V4I16_CLAMP;
3760 Ops.push_back(TexHandle);
3761 Ops.push_back(N->getOperand(2));
3762 Ops.push_back(N->getOperand(3));
3763 Ops.push_back(Chain);
3764 break;
3765 case NVPTXISD::Suld2DV4I32Clamp:
3766 Opc = NVPTX::SULD_2D_V4I32_CLAMP;
3767 Ops.push_back(TexHandle);
3768 Ops.push_back(N->getOperand(2));
3769 Ops.push_back(N->getOperand(3));
3770 Ops.push_back(Chain);
3771 break;
3772 case NVPTXISD::Suld2DArrayI8Clamp:
3773 Opc = NVPTX::SULD_2D_ARRAY_I8_CLAMP;
3774 Ops.push_back(TexHandle);
3775 Ops.push_back(N->getOperand(2));
3776 Ops.push_back(N->getOperand(3));
3777 Ops.push_back(N->getOperand(4));
3778 Ops.push_back(Chain);
3779 break;
3780 case NVPTXISD::Suld2DArrayI16Clamp:
3781 Opc = NVPTX::SULD_2D_ARRAY_I16_CLAMP;
3782 Ops.push_back(TexHandle);
3783 Ops.push_back(N->getOperand(2));
3784 Ops.push_back(N->getOperand(3));
3785 Ops.push_back(N->getOperand(4));
3786 Ops.push_back(Chain);
3787 break;
3788 case NVPTXISD::Suld2DArrayI32Clamp:
3789 Opc = NVPTX::SULD_2D_ARRAY_I32_CLAMP;
3790 Ops.push_back(TexHandle);
3791 Ops.push_back(N->getOperand(2));
3792 Ops.push_back(N->getOperand(3));
3793 Ops.push_back(N->getOperand(4));
3794 Ops.push_back(Chain);
3795 break;
3796 case NVPTXISD::Suld2DArrayI64Clamp:
3797 Opc = NVPTX::SULD_2D_ARRAY_I64_CLAMP;
3798 Ops.push_back(TexHandle);
3799 Ops.push_back(N->getOperand(2));
3800 Ops.push_back(N->getOperand(3));
3801 Ops.push_back(N->getOperand(4));
3802 Ops.push_back(Chain);
3803 break;
3804 case NVPTXISD::Suld2DArrayV2I8Clamp:
3805 Opc = NVPTX::SULD_2D_ARRAY_V2I8_CLAMP;
3806 Ops.push_back(TexHandle);
3807 Ops.push_back(N->getOperand(2));
3808 Ops.push_back(N->getOperand(3));
3809 Ops.push_back(N->getOperand(4));
3810 Ops.push_back(Chain);
3811 break;
3812 case NVPTXISD::Suld2DArrayV2I16Clamp:
3813 Opc = NVPTX::SULD_2D_ARRAY_V2I16_CLAMP;
3814 Ops.push_back(TexHandle);
3815 Ops.push_back(N->getOperand(2));
3816 Ops.push_back(N->getOperand(3));
3817 Ops.push_back(N->getOperand(4));
3818 Ops.push_back(Chain);
3819 break;
3820 case NVPTXISD::Suld2DArrayV2I32Clamp:
3821 Opc = NVPTX::SULD_2D_ARRAY_V2I32_CLAMP;
3822 Ops.push_back(TexHandle);
3823 Ops.push_back(N->getOperand(2));
3824 Ops.push_back(N->getOperand(3));
3825 Ops.push_back(N->getOperand(4));
3826 Ops.push_back(Chain);
3827 break;
3828 case NVPTXISD::Suld2DArrayV2I64Clamp:
3829 Opc = NVPTX::SULD_2D_ARRAY_V2I64_CLAMP;
3830 Ops.push_back(TexHandle);
3831 Ops.push_back(N->getOperand(2));
3832 Ops.push_back(N->getOperand(3));
3833 Ops.push_back(N->getOperand(4));
3834 Ops.push_back(Chain);
3835 break;
3836 case NVPTXISD::Suld2DArrayV4I8Clamp:
3837 Opc = NVPTX::SULD_2D_ARRAY_V4I8_CLAMP;
3838 Ops.push_back(TexHandle);
3839 Ops.push_back(N->getOperand(2));
3840 Ops.push_back(N->getOperand(3));
3841 Ops.push_back(N->getOperand(4));
3842 Ops.push_back(Chain);
3843 break;
3844 case NVPTXISD::Suld2DArrayV4I16Clamp:
3845 Opc = NVPTX::SULD_2D_ARRAY_V4I16_CLAMP;
3846 Ops.push_back(TexHandle);
3847 Ops.push_back(N->getOperand(2));
3848 Ops.push_back(N->getOperand(3));
3849 Ops.push_back(N->getOperand(4));
3850 Ops.push_back(Chain);
3851 break;
3852 case NVPTXISD::Suld2DArrayV4I32Clamp:
3853 Opc = NVPTX::SULD_2D_ARRAY_V4I32_CLAMP;
3854 Ops.push_back(TexHandle);
3855 Ops.push_back(N->getOperand(2));
3856 Ops.push_back(N->getOperand(3));
3857 Ops.push_back(N->getOperand(4));
3858 Ops.push_back(Chain);
3859 break;
3860 case NVPTXISD::Suld3DI8Clamp:
3861 Opc = NVPTX::SULD_3D_I8_CLAMP;
3862 Ops.push_back(TexHandle);
3863 Ops.push_back(N->getOperand(2));
3864 Ops.push_back(N->getOperand(3));
3865 Ops.push_back(N->getOperand(4));
3866 Ops.push_back(Chain);
3867 break;
3868 case NVPTXISD::Suld3DI16Clamp:
3869 Opc = NVPTX::SULD_3D_I16_CLAMP;
3870 Ops.push_back(TexHandle);
3871 Ops.push_back(N->getOperand(2));
3872 Ops.push_back(N->getOperand(3));
3873 Ops.push_back(N->getOperand(4));
3874 Ops.push_back(Chain);
3875 break;
3876 case NVPTXISD::Suld3DI32Clamp:
3877 Opc = NVPTX::SULD_3D_I32_CLAMP;
3878 Ops.push_back(TexHandle);
3879 Ops.push_back(N->getOperand(2));
3880 Ops.push_back(N->getOperand(3));
3881 Ops.push_back(N->getOperand(4));
3882 Ops.push_back(Chain);
3883 break;
3884 case NVPTXISD::Suld3DI64Clamp:
3885 Opc = NVPTX::SULD_3D_I64_CLAMP;
3886 Ops.push_back(TexHandle);
3887 Ops.push_back(N->getOperand(2));
3888 Ops.push_back(N->getOperand(3));
3889 Ops.push_back(N->getOperand(4));
3890 Ops.push_back(Chain);
3891 break;
3892 case NVPTXISD::Suld3DV2I8Clamp:
3893 Opc = NVPTX::SULD_3D_V2I8_CLAMP;
3894 Ops.push_back(TexHandle);
3895 Ops.push_back(N->getOperand(2));
3896 Ops.push_back(N->getOperand(3));
3897 Ops.push_back(N->getOperand(4));
3898 Ops.push_back(Chain);
3899 break;
3900 case NVPTXISD::Suld3DV2I16Clamp:
3901 Opc = NVPTX::SULD_3D_V2I16_CLAMP;
3902 Ops.push_back(TexHandle);
3903 Ops.push_back(N->getOperand(2));
3904 Ops.push_back(N->getOperand(3));
3905 Ops.push_back(N->getOperand(4));
3906 Ops.push_back(Chain);
3907 break;
3908 case NVPTXISD::Suld3DV2I32Clamp:
3909 Opc = NVPTX::SULD_3D_V2I32_CLAMP;
3910 Ops.push_back(TexHandle);
3911 Ops.push_back(N->getOperand(2));
3912 Ops.push_back(N->getOperand(3));
3913 Ops.push_back(N->getOperand(4));
3914 Ops.push_back(Chain);
3915 break;
3916 case NVPTXISD::Suld3DV2I64Clamp:
3917 Opc = NVPTX::SULD_3D_V2I64_CLAMP;
3918 Ops.push_back(TexHandle);
3919 Ops.push_back(N->getOperand(2));
3920 Ops.push_back(N->getOperand(3));
3921 Ops.push_back(N->getOperand(4));
3922 Ops.push_back(Chain);
3923 break;
3924 case NVPTXISD::Suld3DV4I8Clamp:
3925 Opc = NVPTX::SULD_3D_V4I8_CLAMP;
3926 Ops.push_back(TexHandle);
3927 Ops.push_back(N->getOperand(2));
3928 Ops.push_back(N->getOperand(3));
3929 Ops.push_back(N->getOperand(4));
3930 Ops.push_back(Chain);
3931 break;
3932 case NVPTXISD::Suld3DV4I16Clamp:
3933 Opc = NVPTX::SULD_3D_V4I16_CLAMP;
3934 Ops.push_back(TexHandle);
3935 Ops.push_back(N->getOperand(2));
3936 Ops.push_back(N->getOperand(3));
3937 Ops.push_back(N->getOperand(4));
3938 Ops.push_back(Chain);
3939 break;
3940 case NVPTXISD::Suld3DV4I32Clamp:
3941 Opc = NVPTX::SULD_3D_V4I32_CLAMP;
3942 Ops.push_back(TexHandle);
3943 Ops.push_back(N->getOperand(2));
3944 Ops.push_back(N->getOperand(3));
3945 Ops.push_back(N->getOperand(4));
3946 Ops.push_back(Chain);
3947 break;
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]3948 case NVPTXISD::Suld1DI8Trap:
3949 Opc = NVPTX::SULD_1D_I8_TRAP;
3950 Ops.push_back(TexHandle);
3951 Ops.push_back(N->getOperand(2));
3952 Ops.push_back(Chain);
3953 break;
3954 case NVPTXISD::Suld1DI16Trap:
3955 Opc = NVPTX::SULD_1D_I16_TRAP;
3956 Ops.push_back(TexHandle);
3957 Ops.push_back(N->getOperand(2));
3958 Ops.push_back(Chain);
3959 break;
3960 case NVPTXISD::Suld1DI32Trap:
3961 Opc = NVPTX::SULD_1D_I32_TRAP;
3962 Ops.push_back(TexHandle);
3963 Ops.push_back(N->getOperand(2));
3964 Ops.push_back(Chain);
3965 break;
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]3966 case NVPTXISD::Suld1DI64Trap:
3967 Opc = NVPTX::SULD_1D_I64_TRAP;
3968 Ops.push_back(TexHandle);
3969 Ops.push_back(N->getOperand(2));
3970 Ops.push_back(Chain);
3971 break;
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]3972 case NVPTXISD::Suld1DV2I8Trap:
3973 Opc = NVPTX::SULD_1D_V2I8_TRAP;
3974 Ops.push_back(TexHandle);
3975 Ops.push_back(N->getOperand(2));
3976 Ops.push_back(Chain);
3977 break;
3978 case NVPTXISD::Suld1DV2I16Trap:
3979 Opc = NVPTX::SULD_1D_V2I16_TRAP;
3980 Ops.push_back(TexHandle);
3981 Ops.push_back(N->getOperand(2));
3982 Ops.push_back(Chain);
3983 break;
3984 case NVPTXISD::Suld1DV2I32Trap:
3985 Opc = NVPTX::SULD_1D_V2I32_TRAP;
3986 Ops.push_back(TexHandle);
3987 Ops.push_back(N->getOperand(2));
3988 Ops.push_back(Chain);
3989 break;
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]3990 case NVPTXISD::Suld1DV2I64Trap:
3991 Opc = NVPTX::SULD_1D_V2I64_TRAP;
3992 Ops.push_back(TexHandle);
3993 Ops.push_back(N->getOperand(2));
3994 Ops.push_back(Chain);
3995 break;
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]3996 case NVPTXISD::Suld1DV4I8Trap:
3997 Opc = NVPTX::SULD_1D_V4I8_TRAP;
3998 Ops.push_back(TexHandle);
3999 Ops.push_back(N->getOperand(2));
4000 Ops.push_back(Chain);
4001 break;
4002 case NVPTXISD::Suld1DV4I16Trap:
4003 Opc = NVPTX::SULD_1D_V4I16_TRAP;
4004 Ops.push_back(TexHandle);
4005 Ops.push_back(N->getOperand(2));
4006 Ops.push_back(Chain);
4007 break;
4008 case NVPTXISD::Suld1DV4I32Trap:
4009 Opc = NVPTX::SULD_1D_V4I32_TRAP;
4010 Ops.push_back(TexHandle);
4011 Ops.push_back(N->getOperand(2));
4012 Ops.push_back(Chain);
4013 break;
4014 case NVPTXISD::Suld1DArrayI8Trap:
4015 Opc = NVPTX::SULD_1D_ARRAY_I8_TRAP;
4016 Ops.push_back(TexHandle);
4017 Ops.push_back(N->getOperand(2));
4018 Ops.push_back(N->getOperand(3));
4019 Ops.push_back(Chain);
4020 break;
4021 case NVPTXISD::Suld1DArrayI16Trap:
4022 Opc = NVPTX::SULD_1D_ARRAY_I16_TRAP;
4023 Ops.push_back(TexHandle);
4024 Ops.push_back(N->getOperand(2));
4025 Ops.push_back(N->getOperand(3));
4026 Ops.push_back(Chain);
4027 break;
4028 case NVPTXISD::Suld1DArrayI32Trap:
4029 Opc = NVPTX::SULD_1D_ARRAY_I32_TRAP;
4030 Ops.push_back(TexHandle);
4031 Ops.push_back(N->getOperand(2));
4032 Ops.push_back(N->getOperand(3));
4033 Ops.push_back(Chain);
4034 break;
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]4035 case NVPTXISD::Suld1DArrayI64Trap:
4036 Opc = NVPTX::SULD_1D_ARRAY_I64_TRAP;
4037 Ops.push_back(TexHandle);
4038 Ops.push_back(N->getOperand(2));
4039 Ops.push_back(N->getOperand(3));
4040 Ops.push_back(Chain);
4041 break;
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]4042 case NVPTXISD::Suld1DArrayV2I8Trap:
4043 Opc = NVPTX::SULD_1D_ARRAY_V2I8_TRAP;
4044 Ops.push_back(TexHandle);
4045 Ops.push_back(N->getOperand(2));
4046 Ops.push_back(N->getOperand(3));
4047 Ops.push_back(Chain);
4048 break;
4049 case NVPTXISD::Suld1DArrayV2I16Trap:
4050 Opc = NVPTX::SULD_1D_ARRAY_V2I16_TRAP;
4051 Ops.push_back(TexHandle);
4052 Ops.push_back(N->getOperand(2));
4053 Ops.push_back(N->getOperand(3));
4054 Ops.push_back(Chain);
4055 break;
4056 case NVPTXISD::Suld1DArrayV2I32Trap:
4057 Opc = NVPTX::SULD_1D_ARRAY_V2I32_TRAP;
4058 Ops.push_back(TexHandle);
4059 Ops.push_back(N->getOperand(2));
4060 Ops.push_back(N->getOperand(3));
4061 Ops.push_back(Chain);
4062 break;
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]4063 case NVPTXISD::Suld1DArrayV2I64Trap:
4064 Opc = NVPTX::SULD_1D_ARRAY_V2I64_TRAP;
4065 Ops.push_back(TexHandle);
4066 Ops.push_back(N->getOperand(2));
4067 Ops.push_back(N->getOperand(3));
4068 Ops.push_back(Chain);
4069 break;
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]4070 case NVPTXISD::Suld1DArrayV4I8Trap:
4071 Opc = NVPTX::SULD_1D_ARRAY_V4I8_TRAP;
4072 Ops.push_back(TexHandle);
4073 Ops.push_back(N->getOperand(2));
4074 Ops.push_back(N->getOperand(3));
4075 Ops.push_back(Chain);
4076 break;
4077 case NVPTXISD::Suld1DArrayV4I16Trap:
4078 Opc = NVPTX::SULD_1D_ARRAY_V4I16_TRAP;
4079 Ops.push_back(TexHandle);
4080 Ops.push_back(N->getOperand(2));
4081 Ops.push_back(N->getOperand(3));
4082 Ops.push_back(Chain);
4083 break;
4084 case NVPTXISD::Suld1DArrayV4I32Trap:
4085 Opc = NVPTX::SULD_1D_ARRAY_V4I32_TRAP;
4086 Ops.push_back(TexHandle);
4087 Ops.push_back(N->getOperand(2));
4088 Ops.push_back(N->getOperand(3));
4089 Ops.push_back(Chain);
4090 break;
4091 case NVPTXISD::Suld2DI8Trap:
4092 Opc = NVPTX::SULD_2D_I8_TRAP;
4093 Ops.push_back(TexHandle);
4094 Ops.push_back(N->getOperand(2));
4095 Ops.push_back(N->getOperand(3));
4096 Ops.push_back(Chain);
4097 break;
4098 case NVPTXISD::Suld2DI16Trap:
4099 Opc = NVPTX::SULD_2D_I16_TRAP;
4100 Ops.push_back(TexHandle);
4101 Ops.push_back(N->getOperand(2));
4102 Ops.push_back(N->getOperand(3));
4103 Ops.push_back(Chain);
4104 break;
4105 case NVPTXISD::Suld2DI32Trap:
4106 Opc = NVPTX::SULD_2D_I32_TRAP;
4107 Ops.push_back(TexHandle);
4108 Ops.push_back(N->getOperand(2));
4109 Ops.push_back(N->getOperand(3));
4110 Ops.push_back(Chain);
4111 break;
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]4112 case NVPTXISD::Suld2DI64Trap:
4113 Opc = NVPTX::SULD_2D_I64_TRAP;
4114 Ops.push_back(TexHandle);
4115 Ops.push_back(N->getOperand(2));
4116 Ops.push_back(N->getOperand(3));
4117 Ops.push_back(Chain);
4118 break;
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]4119 case NVPTXISD::Suld2DV2I8Trap:
4120 Opc = NVPTX::SULD_2D_V2I8_TRAP;
4121 Ops.push_back(TexHandle);
4122 Ops.push_back(N->getOperand(2));
4123 Ops.push_back(N->getOperand(3));
4124 Ops.push_back(Chain);
4125 break;
4126 case NVPTXISD::Suld2DV2I16Trap:
4127 Opc = NVPTX::SULD_2D_V2I16_TRAP;
4128 Ops.push_back(TexHandle);
4129 Ops.push_back(N->getOperand(2));
4130 Ops.push_back(N->getOperand(3));
4131 Ops.push_back(Chain);
4132 break;
4133 case NVPTXISD::Suld2DV2I32Trap:
4134 Opc = NVPTX::SULD_2D_V2I32_TRAP;
4135 Ops.push_back(TexHandle);
4136 Ops.push_back(N->getOperand(2));
4137 Ops.push_back(N->getOperand(3));
4138 Ops.push_back(Chain);
4139 break;
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]4140 case NVPTXISD::Suld2DV2I64Trap:
4141 Opc = NVPTX::SULD_2D_V2I64_TRAP;
4142 Ops.push_back(TexHandle);
4143 Ops.push_back(N->getOperand(2));
4144 Ops.push_back(N->getOperand(3));
4145 Ops.push_back(Chain);
4146 break;
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]4147 case NVPTXISD::Suld2DV4I8Trap:
4148 Opc = NVPTX::SULD_2D_V4I8_TRAP;
4149 Ops.push_back(TexHandle);
4150 Ops.push_back(N->getOperand(2));
4151 Ops.push_back(N->getOperand(3));
4152 Ops.push_back(Chain);
4153 break;
4154 case NVPTXISD::Suld2DV4I16Trap:
4155 Opc = NVPTX::SULD_2D_V4I16_TRAP;
4156 Ops.push_back(TexHandle);
4157 Ops.push_back(N->getOperand(2));
4158 Ops.push_back(N->getOperand(3));
4159 Ops.push_back(Chain);
4160 break;
4161 case NVPTXISD::Suld2DV4I32Trap:
4162 Opc = NVPTX::SULD_2D_V4I32_TRAP;
4163 Ops.push_back(TexHandle);
4164 Ops.push_back(N->getOperand(2));
4165 Ops.push_back(N->getOperand(3));
4166 Ops.push_back(Chain);
4167 break;
4168 case NVPTXISD::Suld2DArrayI8Trap:
4169 Opc = NVPTX::SULD_2D_ARRAY_I8_TRAP;
4170 Ops.push_back(TexHandle);
4171 Ops.push_back(N->getOperand(2));
4172 Ops.push_back(N->getOperand(3));
4173 Ops.push_back(N->getOperand(4));
4174 Ops.push_back(Chain);
4175 break;
4176 case NVPTXISD::Suld2DArrayI16Trap:
4177 Opc = NVPTX::SULD_2D_ARRAY_I16_TRAP;
4178 Ops.push_back(TexHandle);
4179 Ops.push_back(N->getOperand(2));
4180 Ops.push_back(N->getOperand(3));
4181 Ops.push_back(N->getOperand(4));
4182 Ops.push_back(Chain);
4183 break;
4184 case NVPTXISD::Suld2DArrayI32Trap:
4185 Opc = NVPTX::SULD_2D_ARRAY_I32_TRAP;
4186 Ops.push_back(TexHandle);
4187 Ops.push_back(N->getOperand(2));
4188 Ops.push_back(N->getOperand(3));
4189 Ops.push_back(N->getOperand(4));
4190 Ops.push_back(Chain);
4191 break;
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]4192 case NVPTXISD::Suld2DArrayI64Trap:
4193 Opc = NVPTX::SULD_2D_ARRAY_I64_TRAP;
4194 Ops.push_back(TexHandle);
4195 Ops.push_back(N->getOperand(2));
4196 Ops.push_back(N->getOperand(3));
4197 Ops.push_back(N->getOperand(4));
4198 Ops.push_back(Chain);
4199 break;
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]4200 case NVPTXISD::Suld2DArrayV2I8Trap:
4201 Opc = NVPTX::SULD_2D_ARRAY_V2I8_TRAP;
4202 Ops.push_back(TexHandle);
4203 Ops.push_back(N->getOperand(2));
4204 Ops.push_back(N->getOperand(3));
4205 Ops.push_back(N->getOperand(4));
4206 Ops.push_back(Chain);
4207 break;
4208 case NVPTXISD::Suld2DArrayV2I16Trap:
4209 Opc = NVPTX::SULD_2D_ARRAY_V2I16_TRAP;
4210 Ops.push_back(TexHandle);
4211 Ops.push_back(N->getOperand(2));
4212 Ops.push_back(N->getOperand(3));
4213 Ops.push_back(N->getOperand(4));
4214 Ops.push_back(Chain);
4215 break;
4216 case NVPTXISD::Suld2DArrayV2I32Trap:
4217 Opc = NVPTX::SULD_2D_ARRAY_V2I32_TRAP;
4218 Ops.push_back(TexHandle);
4219 Ops.push_back(N->getOperand(2));
4220 Ops.push_back(N->getOperand(3));
4221 Ops.push_back(N->getOperand(4));
4222 Ops.push_back(Chain);
4223 break;
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]4224 case NVPTXISD::Suld2DArrayV2I64Trap:
4225 Opc = NVPTX::SULD_2D_ARRAY_V2I64_TRAP;
4226 Ops.push_back(TexHandle);
4227 Ops.push_back(N->getOperand(2));
4228 Ops.push_back(N->getOperand(3));
4229 Ops.push_back(N->getOperand(4));
4230 Ops.push_back(Chain);
4231 break;
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]4232 case NVPTXISD::Suld2DArrayV4I8Trap:
4233 Opc = NVPTX::SULD_2D_ARRAY_V4I8_TRAP;
4234 Ops.push_back(TexHandle);
4235 Ops.push_back(N->getOperand(2));
4236 Ops.push_back(N->getOperand(3));
4237 Ops.push_back(N->getOperand(4));
4238 Ops.push_back(Chain);
4239 break;
4240 case NVPTXISD::Suld2DArrayV4I16Trap:
4241 Opc = NVPTX::SULD_2D_ARRAY_V4I16_TRAP;
4242 Ops.push_back(TexHandle);
4243 Ops.push_back(N->getOperand(2));
4244 Ops.push_back(N->getOperand(3));
4245 Ops.push_back(N->getOperand(4));
4246 Ops.push_back(Chain);
4247 break;
4248 case NVPTXISD::Suld2DArrayV4I32Trap:
4249 Opc = NVPTX::SULD_2D_ARRAY_V4I32_TRAP;
4250 Ops.push_back(TexHandle);
4251 Ops.push_back(N->getOperand(2));
4252 Ops.push_back(N->getOperand(3));
4253 Ops.push_back(N->getOperand(4));
4254 Ops.push_back(Chain);
4255 break;
4256 case NVPTXISD::Suld3DI8Trap:
4257 Opc = NVPTX::SULD_3D_I8_TRAP;
4258 Ops.push_back(TexHandle);
4259 Ops.push_back(N->getOperand(2));
4260 Ops.push_back(N->getOperand(3));
4261 Ops.push_back(N->getOperand(4));
4262 Ops.push_back(Chain);
4263 break;
4264 case NVPTXISD::Suld3DI16Trap:
4265 Opc = NVPTX::SULD_3D_I16_TRAP;
4266 Ops.push_back(TexHandle);
4267 Ops.push_back(N->getOperand(2));
4268 Ops.push_back(N->getOperand(3));
4269 Ops.push_back(N->getOperand(4));
4270 Ops.push_back(Chain);
4271 break;
4272 case NVPTXISD::Suld3DI32Trap:
4273 Opc = NVPTX::SULD_3D_I32_TRAP;
4274 Ops.push_back(TexHandle);
4275 Ops.push_back(N->getOperand(2));
4276 Ops.push_back(N->getOperand(3));
4277 Ops.push_back(N->getOperand(4));
4278 Ops.push_back(Chain);
4279 break;
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]4280 case NVPTXISD::Suld3DI64Trap:
4281 Opc = NVPTX::SULD_3D_I64_TRAP;
4282 Ops.push_back(TexHandle);
4283 Ops.push_back(N->getOperand(2));
4284 Ops.push_back(N->getOperand(3));
4285 Ops.push_back(N->getOperand(4));
4286 Ops.push_back(Chain);
4287 break;
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]4288 case NVPTXISD::Suld3DV2I8Trap:
4289 Opc = NVPTX::SULD_3D_V2I8_TRAP;
4290 Ops.push_back(TexHandle);
4291 Ops.push_back(N->getOperand(2));
4292 Ops.push_back(N->getOperand(3));
4293 Ops.push_back(N->getOperand(4));
4294 Ops.push_back(Chain);
4295 break;
4296 case NVPTXISD::Suld3DV2I16Trap:
4297 Opc = NVPTX::SULD_3D_V2I16_TRAP;
4298 Ops.push_back(TexHandle);
4299 Ops.push_back(N->getOperand(2));
4300 Ops.push_back(N->getOperand(3));
4301 Ops.push_back(N->getOperand(4));
4302 Ops.push_back(Chain);
4303 break;
4304 case NVPTXISD::Suld3DV2I32Trap:
4305 Opc = NVPTX::SULD_3D_V2I32_TRAP;
4306 Ops.push_back(TexHandle);
4307 Ops.push_back(N->getOperand(2));
4308 Ops.push_back(N->getOperand(3));
4309 Ops.push_back(N->getOperand(4));
4310 Ops.push_back(Chain);
4311 break;
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]4312 case NVPTXISD::Suld3DV2I64Trap:
4313 Opc = NVPTX::SULD_3D_V2I64_TRAP;
4314 Ops.push_back(TexHandle);
4315 Ops.push_back(N->getOperand(2));
4316 Ops.push_back(N->getOperand(3));
4317 Ops.push_back(N->getOperand(4));
4318 Ops.push_back(Chain);
4319 break;
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]4320 case NVPTXISD::Suld3DV4I8Trap:
4321 Opc = NVPTX::SULD_3D_V4I8_TRAP;
4322 Ops.push_back(TexHandle);
4323 Ops.push_back(N->getOperand(2));
4324 Ops.push_back(N->getOperand(3));
4325 Ops.push_back(N->getOperand(4));
4326 Ops.push_back(Chain);
4327 break;
4328 case NVPTXISD::Suld3DV4I16Trap:
4329 Opc = NVPTX::SULD_3D_V4I16_TRAP;
4330 Ops.push_back(TexHandle);
4331 Ops.push_back(N->getOperand(2));
4332 Ops.push_back(N->getOperand(3));
4333 Ops.push_back(N->getOperand(4));
4334 Ops.push_back(Chain);
4335 break;
4336 case NVPTXISD::Suld3DV4I32Trap:
4337 Opc = NVPTX::SULD_3D_V4I32_TRAP;
4338 Ops.push_back(TexHandle);
4339 Ops.push_back(N->getOperand(2));
4340 Ops.push_back(N->getOperand(3));
4341 Ops.push_back(N->getOperand(4));
4342 Ops.push_back(Chain);
4343 break;
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]4344 case NVPTXISD::Suld1DI8Zero:
4345 Opc = NVPTX::SULD_1D_I8_ZERO;
4346 Ops.push_back(TexHandle);
4347 Ops.push_back(N->getOperand(2));
4348 Ops.push_back(Chain);
4349 break;
4350 case NVPTXISD::Suld1DI16Zero:
4351 Opc = NVPTX::SULD_1D_I16_ZERO;
4352 Ops.push_back(TexHandle);
4353 Ops.push_back(N->getOperand(2));
4354 Ops.push_back(Chain);
4355 break;
4356 case NVPTXISD::Suld1DI32Zero:
4357 Opc = NVPTX::SULD_1D_I32_ZERO;
4358 Ops.push_back(TexHandle);
4359 Ops.push_back(N->getOperand(2));
4360 Ops.push_back(Chain);
4361 break;
4362 case NVPTXISD::Suld1DI64Zero:
4363 Opc = NVPTX::SULD_1D_I64_ZERO;
4364 Ops.push_back(TexHandle);
4365 Ops.push_back(N->getOperand(2));
4366 Ops.push_back(Chain);
4367 break;
4368 case NVPTXISD::Suld1DV2I8Zero:
4369 Opc = NVPTX::SULD_1D_V2I8_ZERO;
4370 Ops.push_back(TexHandle);
4371 Ops.push_back(N->getOperand(2));
4372 Ops.push_back(Chain);
4373 break;
4374 case NVPTXISD::Suld1DV2I16Zero:
4375 Opc = NVPTX::SULD_1D_V2I16_ZERO;
4376 Ops.push_back(TexHandle);
4377 Ops.push_back(N->getOperand(2));
4378 Ops.push_back(Chain);
4379 break;
4380 case NVPTXISD::Suld1DV2I32Zero:
4381 Opc = NVPTX::SULD_1D_V2I32_ZERO;
4382 Ops.push_back(TexHandle);
4383 Ops.push_back(N->getOperand(2));
4384 Ops.push_back(Chain);
4385 break;
4386 case NVPTXISD::Suld1DV2I64Zero:
4387 Opc = NVPTX::SULD_1D_V2I64_ZERO;
4388 Ops.push_back(TexHandle);
4389 Ops.push_back(N->getOperand(2));
4390 Ops.push_back(Chain);
4391 break;
4392 case NVPTXISD::Suld1DV4I8Zero:
4393 Opc = NVPTX::SULD_1D_V4I8_ZERO;
4394 Ops.push_back(TexHandle);
4395 Ops.push_back(N->getOperand(2));
4396 Ops.push_back(Chain);
4397 break;
4398 case NVPTXISD::Suld1DV4I16Zero:
4399 Opc = NVPTX::SULD_1D_V4I16_ZERO;
4400 Ops.push_back(TexHandle);
4401 Ops.push_back(N->getOperand(2));
4402 Ops.push_back(Chain);
4403 break;
4404 case NVPTXISD::Suld1DV4I32Zero:
4405 Opc = NVPTX::SULD_1D_V4I32_ZERO;
4406 Ops.push_back(TexHandle);
4407 Ops.push_back(N->getOperand(2));
4408 Ops.push_back(Chain);
4409 break;
4410 case NVPTXISD::Suld1DArrayI8Zero:
4411 Opc = NVPTX::SULD_1D_ARRAY_I8_ZERO;
4412 Ops.push_back(TexHandle);
4413 Ops.push_back(N->getOperand(2));
4414 Ops.push_back(N->getOperand(3));
4415 Ops.push_back(Chain);
4416 break;
4417 case NVPTXISD::Suld1DArrayI16Zero:
4418 Opc = NVPTX::SULD_1D_ARRAY_I16_ZERO;
4419 Ops.push_back(TexHandle);
4420 Ops.push_back(N->getOperand(2));
4421 Ops.push_back(N->getOperand(3));
4422 Ops.push_back(Chain);
4423 break;
4424 case NVPTXISD::Suld1DArrayI32Zero:
4425 Opc = NVPTX::SULD_1D_ARRAY_I32_ZERO;
4426 Ops.push_back(TexHandle);
4427 Ops.push_back(N->getOperand(2));
4428 Ops.push_back(N->getOperand(3));
4429 Ops.push_back(Chain);
4430 break;
4431 case NVPTXISD::Suld1DArrayI64Zero:
4432 Opc = NVPTX::SULD_1D_ARRAY_I64_ZERO;
4433 Ops.push_back(TexHandle);
4434 Ops.push_back(N->getOperand(2));
4435 Ops.push_back(N->getOperand(3));
4436 Ops.push_back(Chain);
4437 break;
4438 case NVPTXISD::Suld1DArrayV2I8Zero:
4439 Opc = NVPTX::SULD_1D_ARRAY_V2I8_ZERO;
4440 Ops.push_back(TexHandle);
4441 Ops.push_back(N->getOperand(2));
4442 Ops.push_back(N->getOperand(3));
4443 Ops.push_back(Chain);
4444 break;
4445 case NVPTXISD::Suld1DArrayV2I16Zero:
4446 Opc = NVPTX::SULD_1D_ARRAY_V2I16_ZERO;
4447 Ops.push_back(TexHandle);
4448 Ops.push_back(N->getOperand(2));
4449 Ops.push_back(N->getOperand(3));
4450 Ops.push_back(Chain);
4451 break;
4452 case NVPTXISD::Suld1DArrayV2I32Zero:
4453 Opc = NVPTX::SULD_1D_ARRAY_V2I32_ZERO;
4454 Ops.push_back(TexHandle);
4455 Ops.push_back(N->getOperand(2));
4456 Ops.push_back(N->getOperand(3));
4457 Ops.push_back(Chain);
4458 break;
4459 case NVPTXISD::Suld1DArrayV2I64Zero:
4460 Opc = NVPTX::SULD_1D_ARRAY_V2I64_ZERO;
4461 Ops.push_back(TexHandle);
4462 Ops.push_back(N->getOperand(2));
4463 Ops.push_back(N->getOperand(3));
4464 Ops.push_back(Chain);
4465 break;
4466 case NVPTXISD::Suld1DArrayV4I8Zero:
4467 Opc = NVPTX::SULD_1D_ARRAY_V4I8_ZERO;
4468 Ops.push_back(TexHandle);
4469 Ops.push_back(N->getOperand(2));
4470 Ops.push_back(N->getOperand(3));
4471 Ops.push_back(Chain);
4472 break;
4473 case NVPTXISD::Suld1DArrayV4I16Zero:
4474 Opc = NVPTX::SULD_1D_ARRAY_V4I16_ZERO;
4475 Ops.push_back(TexHandle);
4476 Ops.push_back(N->getOperand(2));
4477 Ops.push_back(N->getOperand(3));
4478 Ops.push_back(Chain);
4479 break;
4480 case NVPTXISD::Suld1DArrayV4I32Zero:
4481 Opc = NVPTX::SULD_1D_ARRAY_V4I32_ZERO;
4482 Ops.push_back(TexHandle);
4483 Ops.push_back(N->getOperand(2));
4484 Ops.push_back(N->getOperand(3));
4485 Ops.push_back(Chain);
4486 break;
4487 case NVPTXISD::Suld2DI8Zero:
4488 Opc = NVPTX::SULD_2D_I8_ZERO;
4489 Ops.push_back(TexHandle);
4490 Ops.push_back(N->getOperand(2));
4491 Ops.push_back(N->getOperand(3));
4492 Ops.push_back(Chain);
4493 break;
4494 case NVPTXISD::Suld2DI16Zero:
4495 Opc = NVPTX::SULD_2D_I16_ZERO;
4496 Ops.push_back(TexHandle);
4497 Ops.push_back(N->getOperand(2));
4498 Ops.push_back(N->getOperand(3));
4499 Ops.push_back(Chain);
4500 break;
4501 case NVPTXISD::Suld2DI32Zero:
4502 Opc = NVPTX::SULD_2D_I32_ZERO;
4503 Ops.push_back(TexHandle);
4504 Ops.push_back(N->getOperand(2));
4505 Ops.push_back(N->getOperand(3));
4506 Ops.push_back(Chain);
4507 break;
4508 case NVPTXISD::Suld2DI64Zero:
4509 Opc = NVPTX::SULD_2D_I64_ZERO;
4510 Ops.push_back(TexHandle);
4511 Ops.push_back(N->getOperand(2));
4512 Ops.push_back(N->getOperand(3));
4513 Ops.push_back(Chain);
4514 break;
4515 case NVPTXISD::Suld2DV2I8Zero:
4516 Opc = NVPTX::SULD_2D_V2I8_ZERO;
4517 Ops.push_back(TexHandle);
4518 Ops.push_back(N->getOperand(2));
4519 Ops.push_back(N->getOperand(3));
4520 Ops.push_back(Chain);
4521 break;
4522 case NVPTXISD::Suld2DV2I16Zero:
4523 Opc = NVPTX::SULD_2D_V2I16_ZERO;
4524 Ops.push_back(TexHandle);
4525 Ops.push_back(N->getOperand(2));
4526 Ops.push_back(N->getOperand(3));
4527 Ops.push_back(Chain);
4528 break;
4529 case NVPTXISD::Suld2DV2I32Zero:
4530 Opc = NVPTX::SULD_2D_V2I32_ZERO;
4531 Ops.push_back(TexHandle);
4532 Ops.push_back(N->getOperand(2));
4533 Ops.push_back(N->getOperand(3));
4534 Ops.push_back(Chain);
4535 break;
4536 case NVPTXISD::Suld2DV2I64Zero:
4537 Opc = NVPTX::SULD_2D_V2I64_ZERO;
4538 Ops.push_back(TexHandle);
4539 Ops.push_back(N->getOperand(2));
4540 Ops.push_back(N->getOperand(3));
4541 Ops.push_back(Chain);
4542 break;
4543 case NVPTXISD::Suld2DV4I8Zero:
4544 Opc = NVPTX::SULD_2D_V4I8_ZERO;
4545 Ops.push_back(TexHandle);
4546 Ops.push_back(N->getOperand(2));
4547 Ops.push_back(N->getOperand(3));
4548 Ops.push_back(Chain);
4549 break;
4550 case NVPTXISD::Suld2DV4I16Zero:
4551 Opc = NVPTX::SULD_2D_V4I16_ZERO;
4552 Ops.push_back(TexHandle);
4553 Ops.push_back(N->getOperand(2));
4554 Ops.push_back(N->getOperand(3));
4555 Ops.push_back(Chain);
4556 break;
4557 case NVPTXISD::Suld2DV4I32Zero:
4558 Opc = NVPTX::SULD_2D_V4I32_ZERO;
4559 Ops.push_back(TexHandle);
4560 Ops.push_back(N->getOperand(2));
4561 Ops.push_back(N->getOperand(3));
4562 Ops.push_back(Chain);
4563 break;
4564 case NVPTXISD::Suld2DArrayI8Zero:
4565 Opc = NVPTX::SULD_2D_ARRAY_I8_ZERO;
4566 Ops.push_back(TexHandle);
4567 Ops.push_back(N->getOperand(2));
4568 Ops.push_back(N->getOperand(3));
4569 Ops.push_back(N->getOperand(4));
4570 Ops.push_back(Chain);
4571 break;
4572 case NVPTXISD::Suld2DArrayI16Zero:
4573 Opc = NVPTX::SULD_2D_ARRAY_I16_ZERO;
4574 Ops.push_back(TexHandle);
4575 Ops.push_back(N->getOperand(2));
4576 Ops.push_back(N->getOperand(3));
4577 Ops.push_back(N->getOperand(4));
4578 Ops.push_back(Chain);
4579 break;
4580 case NVPTXISD::Suld2DArrayI32Zero:
4581 Opc = NVPTX::SULD_2D_ARRAY_I32_ZERO;
4582 Ops.push_back(TexHandle);
4583 Ops.push_back(N->getOperand(2));
4584 Ops.push_back(N->getOperand(3));
4585 Ops.push_back(N->getOperand(4));
4586 Ops.push_back(Chain);
4587 break;
4588 case NVPTXISD::Suld2DArrayI64Zero:
4589 Opc = NVPTX::SULD_2D_ARRAY_I64_ZERO;
4590 Ops.push_back(TexHandle);
4591 Ops.push_back(N->getOperand(2));
4592 Ops.push_back(N->getOperand(3));
4593 Ops.push_back(N->getOperand(4));
4594 Ops.push_back(Chain);
4595 break;
4596 case NVPTXISD::Suld2DArrayV2I8Zero:
4597 Opc = NVPTX::SULD_2D_ARRAY_V2I8_ZERO;
4598 Ops.push_back(TexHandle);
4599 Ops.push_back(N->getOperand(2));
4600 Ops.push_back(N->getOperand(3));
4601 Ops.push_back(N->getOperand(4));
4602 Ops.push_back(Chain);
4603 break;
4604 case NVPTXISD::Suld2DArrayV2I16Zero:
4605 Opc = NVPTX::SULD_2D_ARRAY_V2I16_ZERO;
4606 Ops.push_back(TexHandle);
4607 Ops.push_back(N->getOperand(2));
4608 Ops.push_back(N->getOperand(3));
4609 Ops.push_back(N->getOperand(4));
4610 Ops.push_back(Chain);
4611 break;
4612 case NVPTXISD::Suld2DArrayV2I32Zero:
4613 Opc = NVPTX::SULD_2D_ARRAY_V2I32_ZERO;
4614 Ops.push_back(TexHandle);
4615 Ops.push_back(N->getOperand(2));
4616 Ops.push_back(N->getOperand(3));
4617 Ops.push_back(N->getOperand(4));
4618 Ops.push_back(Chain);
4619 break;
4620 case NVPTXISD::Suld2DArrayV2I64Zero:
4621 Opc = NVPTX::SULD_2D_ARRAY_V2I64_ZERO;
4622 Ops.push_back(TexHandle);
4623 Ops.push_back(N->getOperand(2));
4624 Ops.push_back(N->getOperand(3));
4625 Ops.push_back(N->getOperand(4));
4626 Ops.push_back(Chain);
4627 break;
4628 case NVPTXISD::Suld2DArrayV4I8Zero:
4629 Opc = NVPTX::SULD_2D_ARRAY_V4I8_ZERO;
4630 Ops.push_back(TexHandle);
4631 Ops.push_back(N->getOperand(2));
4632 Ops.push_back(N->getOperand(3));
4633 Ops.push_back(N->getOperand(4));
4634 Ops.push_back(Chain);
4635 break;
4636 case NVPTXISD::Suld2DArrayV4I16Zero:
4637 Opc = NVPTX::SULD_2D_ARRAY_V4I16_ZERO;
4638 Ops.push_back(TexHandle);
4639 Ops.push_back(N->getOperand(2));
4640 Ops.push_back(N->getOperand(3));
4641 Ops.push_back(N->getOperand(4));
4642 Ops.push_back(Chain);
4643 break;
4644 case NVPTXISD::Suld2DArrayV4I32Zero:
4645 Opc = NVPTX::SULD_2D_ARRAY_V4I32_ZERO;
4646 Ops.push_back(TexHandle);
4647 Ops.push_back(N->getOperand(2));
4648 Ops.push_back(N->getOperand(3));
4649 Ops.push_back(N->getOperand(4));
4650 Ops.push_back(Chain);
4651 break;
4652 case NVPTXISD::Suld3DI8Zero:
4653 Opc = NVPTX::SULD_3D_I8_ZERO;
4654 Ops.push_back(TexHandle);
4655 Ops.push_back(N->getOperand(2));
4656 Ops.push_back(N->getOperand(3));
4657 Ops.push_back(N->getOperand(4));
4658 Ops.push_back(Chain);
4659 break;
4660 case NVPTXISD::Suld3DI16Zero:
4661 Opc = NVPTX::SULD_3D_I16_ZERO;
4662 Ops.push_back(TexHandle);
4663 Ops.push_back(N->getOperand(2));
4664 Ops.push_back(N->getOperand(3));
4665 Ops.push_back(N->getOperand(4));
4666 Ops.push_back(Chain);
4667 break;
4668 case NVPTXISD::Suld3DI32Zero:
4669 Opc = NVPTX::SULD_3D_I32_ZERO;
4670 Ops.push_back(TexHandle);
4671 Ops.push_back(N->getOperand(2));
4672 Ops.push_back(N->getOperand(3));
4673 Ops.push_back(N->getOperand(4));
4674 Ops.push_back(Chain);
4675 break;
4676 case NVPTXISD::Suld3DI64Zero:
4677 Opc = NVPTX::SULD_3D_I64_ZERO;
4678 Ops.push_back(TexHandle);
4679 Ops.push_back(N->getOperand(2));
4680 Ops.push_back(N->getOperand(3));
4681 Ops.push_back(N->getOperand(4));
4682 Ops.push_back(Chain);
4683 break;
4684 case NVPTXISD::Suld3DV2I8Zero:
4685 Opc = NVPTX::SULD_3D_V2I8_ZERO;
4686 Ops.push_back(TexHandle);
4687 Ops.push_back(N->getOperand(2));
4688 Ops.push_back(N->getOperand(3));
4689 Ops.push_back(N->getOperand(4));
4690 Ops.push_back(Chain);
4691 break;
4692 case NVPTXISD::Suld3DV2I16Zero:
4693 Opc = NVPTX::SULD_3D_V2I16_ZERO;
4694 Ops.push_back(TexHandle);
4695 Ops.push_back(N->getOperand(2));
4696 Ops.push_back(N->getOperand(3));
4697 Ops.push_back(N->getOperand(4));
4698 Ops.push_back(Chain);
4699 break;
4700 case NVPTXISD::Suld3DV2I32Zero:
4701 Opc = NVPTX::SULD_3D_V2I32_ZERO;
4702 Ops.push_back(TexHandle);
4703 Ops.push_back(N->getOperand(2));
4704 Ops.push_back(N->getOperand(3));
4705 Ops.push_back(N->getOperand(4));
4706 Ops.push_back(Chain);
4707 break;
4708 case NVPTXISD::Suld3DV2I64Zero:
4709 Opc = NVPTX::SULD_3D_V2I64_ZERO;
4710 Ops.push_back(TexHandle);
4711 Ops.push_back(N->getOperand(2));
4712 Ops.push_back(N->getOperand(3));
4713 Ops.push_back(N->getOperand(4));
4714 Ops.push_back(Chain);
4715 break;
4716 case NVPTXISD::Suld3DV4I8Zero:
4717 Opc = NVPTX::SULD_3D_V4I8_ZERO;
4718 Ops.push_back(TexHandle);
4719 Ops.push_back(N->getOperand(2));
4720 Ops.push_back(N->getOperand(3));
4721 Ops.push_back(N->getOperand(4));
4722 Ops.push_back(Chain);
4723 break;
4724 case NVPTXISD::Suld3DV4I16Zero:
4725 Opc = NVPTX::SULD_3D_V4I16_ZERO;
4726 Ops.push_back(TexHandle);
4727 Ops.push_back(N->getOperand(2));
4728 Ops.push_back(N->getOperand(3));
4729 Ops.push_back(N->getOperand(4));
4730 Ops.push_back(Chain);
4731 break;
4732 case NVPTXISD::Suld3DV4I32Zero:
4733 Opc = NVPTX::SULD_3D_V4I32_ZERO;
4734 Ops.push_back(TexHandle);
4735 Ops.push_back(N->getOperand(2));
4736 Ops.push_back(N->getOperand(3));
4737 Ops.push_back(N->getOperand(4));
4738 Ops.push_back(Chain);
4739 break;
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]4740 }
4741 Ret = CurDAG->getMachineNode(Opc, SDLoc(N), N->getVTList(), Ops);
4742 return Ret;
4743}
4744
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]4745
Justin Holewinskica7a4f12014-06-27 18:35:27 +0000[diff] [blame]4746/// SelectBFE - Look for instruction sequences that can be made more efficient
4747/// by using the 'bfe' (bit-field extract) PTX instruction
4748SDNode *NVPTXDAGToDAGISel::SelectBFE(SDNode *N) {
4749 SDValue LHS = N->getOperand(0);
4750 SDValue RHS = N->getOperand(1);
4751 SDValue Len;
4752 SDValue Start;
4753 SDValue Val;
4754 bool IsSigned = false;
4755
4756 if (N->getOpcode() == ISD::AND) {
4757 // Canonicalize the operands
4758 // We want 'and %val, %mask'
4759 if (isa<ConstantSDNode>(LHS) && !isa<ConstantSDNode>(RHS)) {
4760 std::swap(LHS, RHS);
4761 }
4762
4763 ConstantSDNode *Mask = dyn_cast<ConstantSDNode>(RHS);
4764 if (!Mask) {
4765 // We need a constant mask on the RHS of the AND
4766 return NULL;
4767 }
4768
4769 // Extract the mask bits
4770 uint64_t MaskVal = Mask->getZExtValue();
4771 if (!isMask_64(MaskVal)) {
4772 // We *could* handle shifted masks here, but doing so would require an
4773 // 'and' operation to fix up the low-order bits so we would trade
4774 // shr+and for bfe+and, which has the same throughput
4775 return NULL;
4776 }
4777
4778 // How many bits are in our mask?
4779 uint64_t NumBits = CountTrailingOnes_64(MaskVal);
4780 Len = CurDAG->getTargetConstant(NumBits, MVT::i32);
4781
4782 if (LHS.getOpcode() == ISD::SRL || LHS.getOpcode() == ISD::SRA) {
4783 // We have a 'srl/and' pair, extract the effective start bit and length
4784 Val = LHS.getNode()->getOperand(0);
4785 Start = LHS.getNode()->getOperand(1);
4786 ConstantSDNode *StartConst = dyn_cast<ConstantSDNode>(Start);
4787 if (StartConst) {
4788 uint64_t StartVal = StartConst->getZExtValue();
4789 // How many "good" bits do we have left? "good" is defined here as bits
4790 // that exist in the original value, not shifted in.
4791 uint64_t GoodBits = Start.getValueType().getSizeInBits() - StartVal;
4792 if (NumBits > GoodBits) {
4793 // Do not handle the case where bits have been shifted in. In theory
4794 // we could handle this, but the cost is likely higher than just
4795 // emitting the srl/and pair.
4796 return NULL;
4797 }
4798 Start = CurDAG->getTargetConstant(StartVal, MVT::i32);
4799 } else {
4800 // Do not handle the case where the shift amount (can be zero if no srl
4801 // was found) is not constant. We could handle this case, but it would
4802 // require run-time logic that would be more expensive than just
4803 // emitting the srl/and pair.
4804 return NULL;
4805 }
4806 } else {
4807 // Do not handle the case where the LHS of the and is not a shift. While
4808 // it would be trivial to handle this case, it would just transform
4809 // 'and' -> 'bfe', but 'and' has higher-throughput.
4810 return NULL;
4811 }
4812 } else if (N->getOpcode() == ISD::SRL || N->getOpcode() == ISD::SRA) {
4813 if (LHS->getOpcode() == ISD::AND) {
4814 ConstantSDNode *ShiftCnst = dyn_cast<ConstantSDNode>(RHS);
4815 if (!ShiftCnst) {
4816 // Shift amount must be constant
4817 return NULL;
4818 }
4819
4820 uint64_t ShiftAmt = ShiftCnst->getZExtValue();
4821
4822 SDValue AndLHS = LHS->getOperand(0);
4823 SDValue AndRHS = LHS->getOperand(1);
4824
4825 // Canonicalize the AND to have the mask on the RHS
4826 if (isa<ConstantSDNode>(AndLHS)) {
4827 std::swap(AndLHS, AndRHS);
4828 }
4829
4830 ConstantSDNode *MaskCnst = dyn_cast<ConstantSDNode>(AndRHS);
4831 if (!MaskCnst) {
4832 // Mask must be constant
4833 return NULL;
4834 }
4835
4836 uint64_t MaskVal = MaskCnst->getZExtValue();
4837 uint64_t NumZeros;
4838 uint64_t NumBits;
4839 if (isMask_64(MaskVal)) {
4840 NumZeros = 0;
4841 // The number of bits in the result bitfield will be the number of
4842 // trailing ones (the AND) minus the number of bits we shift off
4843 NumBits = CountTrailingOnes_64(MaskVal) - ShiftAmt;
4844 } else if (isShiftedMask_64(MaskVal)) {
4845 NumZeros = countTrailingZeros(MaskVal);
4846 unsigned NumOnes = CountTrailingOnes_64(MaskVal >> NumZeros);
4847 // The number of bits in the result bitfield will be the number of
4848 // trailing zeros plus the number of set bits in the mask minus the
4849 // number of bits we shift off
4850 NumBits = NumZeros + NumOnes - ShiftAmt;
4851 } else {
4852 // This is not a mask we can handle
4853 return NULL;
4854 }
4855
4856 if (ShiftAmt < NumZeros) {
4857 // Handling this case would require extra logic that would make this
4858 // transformation non-profitable
4859 return NULL;
4860 }
4861
4862 Val = AndLHS;
4863 Start = CurDAG->getTargetConstant(ShiftAmt, MVT::i32);
4864 Len = CurDAG->getTargetConstant(NumBits, MVT::i32);
4865 } else if (LHS->getOpcode() == ISD::SHL) {
4866 // Here, we have a pattern like:
4867 //
4868 // (sra (shl val, NN), MM)
4869 // or
4870 // (srl (shl val, NN), MM)
4871 //
4872 // If MM >= NN, we can efficiently optimize this with bfe
4873 Val = LHS->getOperand(0);
4874
4875 SDValue ShlRHS = LHS->getOperand(1);
4876 ConstantSDNode *ShlCnst = dyn_cast<ConstantSDNode>(ShlRHS);
4877 if (!ShlCnst) {
4878 // Shift amount must be constant
4879 return NULL;
4880 }
4881 uint64_t InnerShiftAmt = ShlCnst->getZExtValue();
4882
4883 SDValue ShrRHS = RHS;
4884 ConstantSDNode *ShrCnst = dyn_cast<ConstantSDNode>(ShrRHS);
4885 if (!ShrCnst) {
4886 // Shift amount must be constant
4887 return NULL;
4888 }
4889 uint64_t OuterShiftAmt = ShrCnst->getZExtValue();
4890
4891 // To avoid extra codegen and be profitable, we need Outer >= Inner
4892 if (OuterShiftAmt < InnerShiftAmt) {
4893 return NULL;
4894 }
4895
4896 // If the outer shift is more than the type size, we have no bitfield to
4897 // extract (since we also check that the inner shift is <= the outer shift
4898 // then this also implies that the inner shift is < the type size)
4899 if (OuterShiftAmt >= Val.getValueType().getSizeInBits()) {
4900 return NULL;
4901 }
4902
4903 Start =
4904 CurDAG->getTargetConstant(OuterShiftAmt - InnerShiftAmt, MVT::i32);
4905 Len =
4906 CurDAG->getTargetConstant(Val.getValueType().getSizeInBits() -
4907 OuterShiftAmt, MVT::i32);
4908
4909 if (N->getOpcode() == ISD::SRA) {
4910 // If we have a arithmetic right shift, we need to use the signed bfe
4911 // variant
4912 IsSigned = true;
4913 }
4914 } else {
4915 // No can do...
4916 return NULL;
4917 }
4918 } else {
4919 // No can do...
4920 return NULL;
4921 }
4922
4923
4924 unsigned Opc;
4925 // For the BFE operations we form here from "and" and "srl", always use the
4926 // unsigned variants.
4927 if (Val.getValueType() == MVT::i32) {
4928 if (IsSigned) {
4929 Opc = NVPTX::BFE_S32rii;
4930 } else {
4931 Opc = NVPTX::BFE_U32rii;
4932 }
4933 } else if (Val.getValueType() == MVT::i64) {
4934 if (IsSigned) {
4935 Opc = NVPTX::BFE_S64rii;
4936 } else {
4937 Opc = NVPTX::BFE_U64rii;
4938 }
4939 } else {
4940 // We cannot handle this type
4941 return NULL;
4942 }
4943
4944 SDValue Ops[] = {
4945 Val, Start, Len
4946 };
4947
4948 SDNode *Ret =
4949 CurDAG->getMachineNode(Opc, SDLoc(N), N->getVTList(), Ops);
4950
4951 return Ret;
4952}
4953
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]4954// SelectDirectAddr - Match a direct address for DAG.
4955// A direct address could be a globaladdress or externalsymbol.
4956bool NVPTXDAGToDAGISel::SelectDirectAddr(SDValue N, SDValue &Address) {
4957 // Return true if TGA or ES.
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]4958 if (N.getOpcode() == ISD::TargetGlobalAddress ||
4959 N.getOpcode() == ISD::TargetExternalSymbol) {
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]4960 Address = N;
4961 return true;
4962 }
4963 if (N.getOpcode() == NVPTXISD::Wrapper) {
4964 Address = N.getOperand(0);
4965 return true;
4966 }
4967 if (N.getOpcode() == ISD::INTRINSIC_WO_CHAIN) {
4968 unsigned IID = cast<ConstantSDNode>(N.getOperand(0))->getZExtValue();
4969 if (IID == Intrinsic::nvvm_ptr_gen_to_param)
4970 if (N.getOperand(1).getOpcode() == NVPTXISD::MoveParam)
4971 return (SelectDirectAddr(N.getOperand(1).getOperand(0), Address));
4972 }
4973 return false;
4974}
4975
4976// symbol+offset
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]4977bool NVPTXDAGToDAGISel::SelectADDRsi_imp(
4978 SDNode *OpNode, SDValue Addr, SDValue &Base, SDValue &Offset, MVT mvt) {
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]4979 if (Addr.getOpcode() == ISD::ADD) {
4980 if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(Addr.getOperand(1))) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]4981 SDValue base = Addr.getOperand(0);
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]4982 if (SelectDirectAddr(base, Base)) {
4983 Offset = CurDAG->getTargetConstant(CN->getZExtValue(), mvt);
4984 return true;
4985 }
4986 }
4987 }
4988 return false;
4989}
4990
4991// symbol+offset
4992bool NVPTXDAGToDAGISel::SelectADDRsi(SDNode *OpNode, SDValue Addr,
4993 SDValue &Base, SDValue &Offset) {
4994 return SelectADDRsi_imp(OpNode, Addr, Base, Offset, MVT::i32);
4995}
4996
4997// symbol+offset
4998bool NVPTXDAGToDAGISel::SelectADDRsi64(SDNode *OpNode, SDValue Addr,
4999 SDValue &Base, SDValue &Offset) {
5000 return SelectADDRsi_imp(OpNode, Addr, Base, Offset, MVT::i64);
5001}
5002
5003// register+offset
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]5004bool NVPTXDAGToDAGISel::SelectADDRri_imp(
5005 SDNode *OpNode, SDValue Addr, SDValue &Base, SDValue &Offset, MVT mvt) {
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]5006 if (FrameIndexSDNode *FIN = dyn_cast<FrameIndexSDNode>(Addr)) {
5007 Base = CurDAG->getTargetFrameIndex(FIN->getIndex(), mvt);
5008 Offset = CurDAG->getTargetConstant(0, mvt);
5009 return true;
5010 }
5011 if (Addr.getOpcode() == ISD::TargetExternalSymbol ||
5012 Addr.getOpcode() == ISD::TargetGlobalAddress)
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]5013 return false; // direct calls.
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]5014
5015 if (Addr.getOpcode() == ISD::ADD) {
5016 if (SelectDirectAddr(Addr.getOperand(0), Addr)) {
5017 return false;
5018 }
5019 if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(Addr.getOperand(1))) {
5020 if (FrameIndexSDNode *FIN =
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]5021 dyn_cast<FrameIndexSDNode>(Addr.getOperand(0)))
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]5022 // Constant offset from frame ref.
5023 Base = CurDAG->getTargetFrameIndex(FIN->getIndex(), mvt);
5024 else
5025 Base = Addr.getOperand(0);
5026 Offset = CurDAG->getTargetConstant(CN->getZExtValue(), mvt);
5027 return true;
5028 }
5029 }
5030 return false;
5031}
5032
5033// register+offset
5034bool NVPTXDAGToDAGISel::SelectADDRri(SDNode *OpNode, SDValue Addr,
5035 SDValue &Base, SDValue &Offset) {
5036 return SelectADDRri_imp(OpNode, Addr, Base, Offset, MVT::i32);
5037}
5038
5039// register+offset
5040bool NVPTXDAGToDAGISel::SelectADDRri64(SDNode *OpNode, SDValue Addr,
5041 SDValue &Base, SDValue &Offset) {
5042 return SelectADDRri_imp(OpNode, Addr, Base, Offset, MVT::i64);
5043}
5044
5045bool NVPTXDAGToDAGISel::ChkMemSDNodeAddressSpace(SDNode *N,
5046 unsigned int spN) const {
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]5047 const Value *Src = nullptr;
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]5048 if (MemSDNode *mN = dyn_cast<MemSDNode>(N)) {
Nick Lewyckyaad475b2014-04-15 07:22:52 +0000[diff] [blame]5049 if (spN == 0 && mN->getMemOperand()->getPseudoValue())
5050 return true;
5051 Src = mN->getMemOperand()->getValue();
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]5052 }
5053 if (!Src)
5054 return false;
5055 if (const PointerType *PT = dyn_cast<PointerType>(Src->getType()))
5056 return (PT->getAddressSpace() == spN);
5057 return false;
5058}
5059
5060/// SelectInlineAsmMemoryOperand - Implement addressing mode selection for
5061/// inline asm expressions.
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]5062bool NVPTXDAGToDAGISel::SelectInlineAsmMemoryOperand(
5063 const SDValue &Op, char ConstraintCode, std::vector<SDValue> &OutOps) {
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]5064 SDValue Op0, Op1;
5065 switch (ConstraintCode) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]5066 default:
5067 return true;
5068 case 'm': // memory
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]5069 if (SelectDirectAddr(Op, Op0)) {
5070 OutOps.push_back(Op0);
5071 OutOps.push_back(CurDAG->getTargetConstant(0, MVT::i32));
5072 return false;
5073 }
5074 if (SelectADDRri(Op.getNode(), Op, Op0, Op1)) {
5075 OutOps.push_back(Op0);
5076 OutOps.push_back(Op1);
5077 return false;
5078 }
5079 break;
5080 }
5081 return true;
5082}