Gitiles
Code Review Sign In
gerrit-public.fairphone.software / toolchain / llvm-project / 9745b3aae07a959e1284355e07d1ffbc0f5e33fc / . / llvm / lib / Target / NVPTX / NVPTXISelDAGToDAG.cpp
blob: 39750f446b2c4ffd3c394fe010eabf072a3c7a56 [file] [log] [blame]
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]1//===-- NVPTXISelDAGToDAG.cpp - A dag to dag inst selector for NVPTX ------===//
2//
3// The LLVM Compiler Infrastructure
4//
5// This file is distributed under the University of Illinois Open Source
6// License. See LICENSE.TXT for details.
7//
8//===----------------------------------------------------------------------===//
9//
10// This file defines an instruction selector for the NVPTX target.
11//
12//===----------------------------------------------------------------------===//
13
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]14#include "NVPTXISelDAGToDAG.h"
Chandler Carruth9fb823b2013-01-02 11:36:10 +0000[diff] [blame]15#include "llvm/IR/GlobalValue.h"
16#include "llvm/IR/Instructions.h"
Chandler Carruthed0881b2012-12-03 16:50:05 +0000[diff] [blame]17#include "llvm/Support/CommandLine.h"
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]18#include "llvm/Support/Debug.h"
19#include "llvm/Support/ErrorHandling.h"
Chandler Carruthed0881b2012-12-03 16:50:05 +0000[diff] [blame]20#include "llvm/Support/raw_ostream.h"
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]21#include "llvm/Target/TargetIntrinsicInfo.h"
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]22
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]23using namespace llvm;
24
Chandler Carruth84e68b22014-04-22 02:41:26 +0000[diff] [blame]25#define DEBUG_TYPE "nvptx-isel"
26
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]27static cl::opt<int> UsePrecDivF32(
Nadav Rotem7f27e0b2013-10-18 23:38:13 +0000[diff] [blame]28 "nvptx-prec-divf32", cl::ZeroOrMore, cl::Hidden,
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]29 cl::desc("NVPTX Specifies: 0 use div.approx, 1 use div.full, 2 use"
Sylvestre Ledru469de192014-08-11 18:04:46 +0000[diff] [blame]30 " IEEE Compliant F32 div.rnd if available."),
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]31 cl::init(2));
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]32
Justin Holewinski48f4ad32013-05-21 16:51:30 +0000[diff] [blame]33static cl::opt<bool>
Nadav Rotem7f27e0b2013-10-18 23:38:13 +0000[diff] [blame]34UsePrecSqrtF32("nvptx-prec-sqrtf32", cl::Hidden,
Justin Holewinski48f4ad32013-05-21 16:51:30 +0000[diff] [blame]35 cl::desc("NVPTX Specific: 0 use sqrt.approx, 1 use sqrt.rn."),
36 cl::init(true));
37
Justin Holewinskicd069e62013-07-22 12:18:04 +0000[diff] [blame]38static cl::opt<bool>
Nadav Rotem7f27e0b2013-10-18 23:38:13 +0000[diff] [blame]39FtzEnabled("nvptx-f32ftz", cl::ZeroOrMore, cl::Hidden,
Justin Holewinskicd069e62013-07-22 12:18:04 +0000[diff] [blame]40 cl::desc("NVPTX Specific: Flush f32 subnormals to sign-preserving zero."),
41 cl::init(false));
42
43
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]44/// createNVPTXISelDag - This pass converts a legalized DAG into a
45/// NVPTX-specific DAG, ready for instruction scheduling.
46FunctionPass *llvm::createNVPTXISelDag(NVPTXTargetMachine &TM,
47 llvm::CodeGenOpt::Level OptLevel) {
48 return new NVPTXDAGToDAGISel(TM, OptLevel);
49}
50
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]51NVPTXDAGToDAGISel::NVPTXDAGToDAGISel(NVPTXTargetMachine &tm,
52 CodeGenOpt::Level OptLevel)
Eric Christopher147bba22015-01-30 01:40:59 +0000[diff] [blame]53 : SelectionDAGISel(tm, OptLevel) {
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]54 doMulWide = (OptLevel > 0);
Justin Holewinskicd069e62013-07-22 12:18:04 +0000[diff] [blame]55}
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]56
Eric Christopher147bba22015-01-30 01:40:59 +0000[diff] [blame]57bool NVPTXDAGToDAGISel::runOnMachineFunction(MachineFunction &MF) {
58 Subtarget = &static_cast<const NVPTXSubtarget &>(MF.getSubtarget());
59 return SelectionDAGISel::runOnMachineFunction(MF);
60}
61
Justin Holewinskicd069e62013-07-22 12:18:04 +0000[diff] [blame]62int NVPTXDAGToDAGISel::getDivF32Level() const {
63 if (UsePrecDivF32.getNumOccurrences() > 0) {
64 // If nvptx-prec-div32=N is used on the command-line, always honor it
65 return UsePrecDivF32;
66 } else {
67 // Otherwise, use div.approx if fast math is enabled
68 if (TM.Options.UnsafeFPMath)
69 return 0;
70 else
71 return 2;
72 }
73}
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]74
Justin Holewinskicd069e62013-07-22 12:18:04 +0000[diff] [blame]75bool NVPTXDAGToDAGISel::usePrecSqrtF32() const {
76 if (UsePrecSqrtF32.getNumOccurrences() > 0) {
77 // If nvptx-prec-sqrtf32 is used on the command-line, always honor it
78 return UsePrecSqrtF32;
79 } else {
80 // Otherwise, use sqrt.approx if fast math is enabled
81 if (TM.Options.UnsafeFPMath)
82 return false;
83 else
84 return true;
85 }
86}
87
88bool NVPTXDAGToDAGISel::useF32FTZ() const {
89 if (FtzEnabled.getNumOccurrences() > 0) {
90 // If nvptx-f32ftz is used on the command-line, always honor it
91 return FtzEnabled;
92 } else {
93 const Function *F = MF->getFunction();
94 // Otherwise, check for an nvptx-f32ftz attribute on the function
95 if (F->hasFnAttribute("nvptx-f32ftz"))
96 return (F->getAttributes().getAttribute(AttributeSet::FunctionIndex,
97 "nvptx-f32ftz")
98 .getValueAsString() == "true");
99 else
100 return false;
101 }
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]102}
103
Justin Holewinski428cf0e2014-07-17 18:10:09 +0000[diff] [blame]104bool NVPTXDAGToDAGISel::allowFMA() const {
Eric Christopher147bba22015-01-30 01:40:59 +0000[diff] [blame]105 const NVPTXTargetLowering *TL = Subtarget->getTargetLowering();
Justin Holewinski428cf0e2014-07-17 18:10:09 +0000[diff] [blame]106 return TL->allowFMA(*MF, OptLevel);
107}
108
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]109/// Select - Select instructions not customized! Used for
110/// expanded, promoted and normal instructions.
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]111SDNode *NVPTXDAGToDAGISel::Select(SDNode *N) {
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]112
Tim Northover31d093c2013-09-22 08:21:56 +0000[diff] [blame]113 if (N->isMachineOpcode()) {
114 N->setNodeId(-1);
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]115 return nullptr; // Already selected.
Tim Northover31d093c2013-09-22 08:21:56 +0000[diff] [blame]116 }
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]117
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]118 SDNode *ResNode = nullptr;
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]119 switch (N->getOpcode()) {
120 case ISD::LOAD:
121 ResNode = SelectLoad(N);
122 break;
123 case ISD::STORE:
124 ResNode = SelectStore(N);
125 break;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]126 case NVPTXISD::LoadV2:
127 case NVPTXISD::LoadV4:
128 ResNode = SelectLoadVector(N);
129 break;
130 case NVPTXISD::LDGV2:
131 case NVPTXISD::LDGV4:
132 case NVPTXISD::LDUV2:
133 case NVPTXISD::LDUV4:
Justin Holewinskib926d9d2014-06-27 18:35:51 +0000[diff] [blame]134 ResNode = SelectLDGLDU(N);
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]135 break;
136 case NVPTXISD::StoreV2:
137 case NVPTXISD::StoreV4:
138 ResNode = SelectStoreVector(N);
139 break;
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]140 case NVPTXISD::LoadParam:
141 case NVPTXISD::LoadParamV2:
142 case NVPTXISD::LoadParamV4:
143 ResNode = SelectLoadParam(N);
144 break;
145 case NVPTXISD::StoreRetval:
146 case NVPTXISD::StoreRetvalV2:
147 case NVPTXISD::StoreRetvalV4:
148 ResNode = SelectStoreRetval(N);
149 break;
150 case NVPTXISD::StoreParam:
151 case NVPTXISD::StoreParamV2:
152 case NVPTXISD::StoreParamV4:
153 case NVPTXISD::StoreParamS32:
154 case NVPTXISD::StoreParamU32:
155 ResNode = SelectStoreParam(N);
156 break;
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]157 case ISD::INTRINSIC_WO_CHAIN:
158 ResNode = SelectIntrinsicNoChain(N);
159 break;
Justin Holewinskib926d9d2014-06-27 18:35:51 +0000[diff] [blame]160 case ISD::INTRINSIC_W_CHAIN:
161 ResNode = SelectIntrinsicChain(N);
162 break;
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]163 case NVPTXISD::Tex1DFloatS32:
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]164 case NVPTXISD::Tex1DFloatFloat:
165 case NVPTXISD::Tex1DFloatFloatLevel:
166 case NVPTXISD::Tex1DFloatFloatGrad:
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]167 case NVPTXISD::Tex1DS32S32:
168 case NVPTXISD::Tex1DS32Float:
169 case NVPTXISD::Tex1DS32FloatLevel:
170 case NVPTXISD::Tex1DS32FloatGrad:
171 case NVPTXISD::Tex1DU32S32:
172 case NVPTXISD::Tex1DU32Float:
173 case NVPTXISD::Tex1DU32FloatLevel:
174 case NVPTXISD::Tex1DU32FloatGrad:
175 case NVPTXISD::Tex1DArrayFloatS32:
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]176 case NVPTXISD::Tex1DArrayFloatFloat:
177 case NVPTXISD::Tex1DArrayFloatFloatLevel:
178 case NVPTXISD::Tex1DArrayFloatFloatGrad:
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]179 case NVPTXISD::Tex1DArrayS32S32:
180 case NVPTXISD::Tex1DArrayS32Float:
181 case NVPTXISD::Tex1DArrayS32FloatLevel:
182 case NVPTXISD::Tex1DArrayS32FloatGrad:
183 case NVPTXISD::Tex1DArrayU32S32:
184 case NVPTXISD::Tex1DArrayU32Float:
185 case NVPTXISD::Tex1DArrayU32FloatLevel:
186 case NVPTXISD::Tex1DArrayU32FloatGrad:
187 case NVPTXISD::Tex2DFloatS32:
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]188 case NVPTXISD::Tex2DFloatFloat:
189 case NVPTXISD::Tex2DFloatFloatLevel:
190 case NVPTXISD::Tex2DFloatFloatGrad:
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]191 case NVPTXISD::Tex2DS32S32:
192 case NVPTXISD::Tex2DS32Float:
193 case NVPTXISD::Tex2DS32FloatLevel:
194 case NVPTXISD::Tex2DS32FloatGrad:
195 case NVPTXISD::Tex2DU32S32:
196 case NVPTXISD::Tex2DU32Float:
197 case NVPTXISD::Tex2DU32FloatLevel:
198 case NVPTXISD::Tex2DU32FloatGrad:
199 case NVPTXISD::Tex2DArrayFloatS32:
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]200 case NVPTXISD::Tex2DArrayFloatFloat:
201 case NVPTXISD::Tex2DArrayFloatFloatLevel:
202 case NVPTXISD::Tex2DArrayFloatFloatGrad:
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]203 case NVPTXISD::Tex2DArrayS32S32:
204 case NVPTXISD::Tex2DArrayS32Float:
205 case NVPTXISD::Tex2DArrayS32FloatLevel:
206 case NVPTXISD::Tex2DArrayS32FloatGrad:
207 case NVPTXISD::Tex2DArrayU32S32:
208 case NVPTXISD::Tex2DArrayU32Float:
209 case NVPTXISD::Tex2DArrayU32FloatLevel:
210 case NVPTXISD::Tex2DArrayU32FloatGrad:
211 case NVPTXISD::Tex3DFloatS32:
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]212 case NVPTXISD::Tex3DFloatFloat:
213 case NVPTXISD::Tex3DFloatFloatLevel:
214 case NVPTXISD::Tex3DFloatFloatGrad:
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]215 case NVPTXISD::Tex3DS32S32:
216 case NVPTXISD::Tex3DS32Float:
217 case NVPTXISD::Tex3DS32FloatLevel:
218 case NVPTXISD::Tex3DS32FloatGrad:
219 case NVPTXISD::Tex3DU32S32:
220 case NVPTXISD::Tex3DU32Float:
221 case NVPTXISD::Tex3DU32FloatLevel:
222 case NVPTXISD::Tex3DU32FloatGrad:
223 case NVPTXISD::TexCubeFloatFloat:
224 case NVPTXISD::TexCubeFloatFloatLevel:
225 case NVPTXISD::TexCubeS32Float:
226 case NVPTXISD::TexCubeS32FloatLevel:
227 case NVPTXISD::TexCubeU32Float:
228 case NVPTXISD::TexCubeU32FloatLevel:
229 case NVPTXISD::TexCubeArrayFloatFloat:
230 case NVPTXISD::TexCubeArrayFloatFloatLevel:
231 case NVPTXISD::TexCubeArrayS32Float:
232 case NVPTXISD::TexCubeArrayS32FloatLevel:
233 case NVPTXISD::TexCubeArrayU32Float:
234 case NVPTXISD::TexCubeArrayU32FloatLevel:
235 case NVPTXISD::Tld4R2DFloatFloat:
236 case NVPTXISD::Tld4G2DFloatFloat:
237 case NVPTXISD::Tld4B2DFloatFloat:
238 case NVPTXISD::Tld4A2DFloatFloat:
239 case NVPTXISD::Tld4R2DS64Float:
240 case NVPTXISD::Tld4G2DS64Float:
241 case NVPTXISD::Tld4B2DS64Float:
242 case NVPTXISD::Tld4A2DS64Float:
243 case NVPTXISD::Tld4R2DU64Float:
244 case NVPTXISD::Tld4G2DU64Float:
245 case NVPTXISD::Tld4B2DU64Float:
246 case NVPTXISD::Tld4A2DU64Float:
247 case NVPTXISD::TexUnified1DFloatS32:
248 case NVPTXISD::TexUnified1DFloatFloat:
249 case NVPTXISD::TexUnified1DFloatFloatLevel:
250 case NVPTXISD::TexUnified1DFloatFloatGrad:
251 case NVPTXISD::TexUnified1DS32S32:
252 case NVPTXISD::TexUnified1DS32Float:
253 case NVPTXISD::TexUnified1DS32FloatLevel:
254 case NVPTXISD::TexUnified1DS32FloatGrad:
255 case NVPTXISD::TexUnified1DU32S32:
256 case NVPTXISD::TexUnified1DU32Float:
257 case NVPTXISD::TexUnified1DU32FloatLevel:
258 case NVPTXISD::TexUnified1DU32FloatGrad:
259 case NVPTXISD::TexUnified1DArrayFloatS32:
260 case NVPTXISD::TexUnified1DArrayFloatFloat:
261 case NVPTXISD::TexUnified1DArrayFloatFloatLevel:
262 case NVPTXISD::TexUnified1DArrayFloatFloatGrad:
263 case NVPTXISD::TexUnified1DArrayS32S32:
264 case NVPTXISD::TexUnified1DArrayS32Float:
265 case NVPTXISD::TexUnified1DArrayS32FloatLevel:
266 case NVPTXISD::TexUnified1DArrayS32FloatGrad:
267 case NVPTXISD::TexUnified1DArrayU32S32:
268 case NVPTXISD::TexUnified1DArrayU32Float:
269 case NVPTXISD::TexUnified1DArrayU32FloatLevel:
270 case NVPTXISD::TexUnified1DArrayU32FloatGrad:
271 case NVPTXISD::TexUnified2DFloatS32:
272 case NVPTXISD::TexUnified2DFloatFloat:
273 case NVPTXISD::TexUnified2DFloatFloatLevel:
274 case NVPTXISD::TexUnified2DFloatFloatGrad:
275 case NVPTXISD::TexUnified2DS32S32:
276 case NVPTXISD::TexUnified2DS32Float:
277 case NVPTXISD::TexUnified2DS32FloatLevel:
278 case NVPTXISD::TexUnified2DS32FloatGrad:
279 case NVPTXISD::TexUnified2DU32S32:
280 case NVPTXISD::TexUnified2DU32Float:
281 case NVPTXISD::TexUnified2DU32FloatLevel:
282 case NVPTXISD::TexUnified2DU32FloatGrad:
283 case NVPTXISD::TexUnified2DArrayFloatS32:
284 case NVPTXISD::TexUnified2DArrayFloatFloat:
285 case NVPTXISD::TexUnified2DArrayFloatFloatLevel:
286 case NVPTXISD::TexUnified2DArrayFloatFloatGrad:
287 case NVPTXISD::TexUnified2DArrayS32S32:
288 case NVPTXISD::TexUnified2DArrayS32Float:
289 case NVPTXISD::TexUnified2DArrayS32FloatLevel:
290 case NVPTXISD::TexUnified2DArrayS32FloatGrad:
291 case NVPTXISD::TexUnified2DArrayU32S32:
292 case NVPTXISD::TexUnified2DArrayU32Float:
293 case NVPTXISD::TexUnified2DArrayU32FloatLevel:
294 case NVPTXISD::TexUnified2DArrayU32FloatGrad:
295 case NVPTXISD::TexUnified3DFloatS32:
296 case NVPTXISD::TexUnified3DFloatFloat:
297 case NVPTXISD::TexUnified3DFloatFloatLevel:
298 case NVPTXISD::TexUnified3DFloatFloatGrad:
299 case NVPTXISD::TexUnified3DS32S32:
300 case NVPTXISD::TexUnified3DS32Float:
301 case NVPTXISD::TexUnified3DS32FloatLevel:
302 case NVPTXISD::TexUnified3DS32FloatGrad:
303 case NVPTXISD::TexUnified3DU32S32:
304 case NVPTXISD::TexUnified3DU32Float:
305 case NVPTXISD::TexUnified3DU32FloatLevel:
306 case NVPTXISD::TexUnified3DU32FloatGrad:
307 case NVPTXISD::TexUnifiedCubeFloatFloat:
308 case NVPTXISD::TexUnifiedCubeFloatFloatLevel:
309 case NVPTXISD::TexUnifiedCubeS32Float:
310 case NVPTXISD::TexUnifiedCubeS32FloatLevel:
311 case NVPTXISD::TexUnifiedCubeU32Float:
312 case NVPTXISD::TexUnifiedCubeU32FloatLevel:
313 case NVPTXISD::TexUnifiedCubeArrayFloatFloat:
314 case NVPTXISD::TexUnifiedCubeArrayFloatFloatLevel:
315 case NVPTXISD::TexUnifiedCubeArrayS32Float:
316 case NVPTXISD::TexUnifiedCubeArrayS32FloatLevel:
317 case NVPTXISD::TexUnifiedCubeArrayU32Float:
318 case NVPTXISD::TexUnifiedCubeArrayU32FloatLevel:
319 case NVPTXISD::Tld4UnifiedR2DFloatFloat:
320 case NVPTXISD::Tld4UnifiedG2DFloatFloat:
321 case NVPTXISD::Tld4UnifiedB2DFloatFloat:
322 case NVPTXISD::Tld4UnifiedA2DFloatFloat:
323 case NVPTXISD::Tld4UnifiedR2DS64Float:
324 case NVPTXISD::Tld4UnifiedG2DS64Float:
325 case NVPTXISD::Tld4UnifiedB2DS64Float:
326 case NVPTXISD::Tld4UnifiedA2DS64Float:
327 case NVPTXISD::Tld4UnifiedR2DU64Float:
328 case NVPTXISD::Tld4UnifiedG2DU64Float:
329 case NVPTXISD::Tld4UnifiedB2DU64Float:
330 case NVPTXISD::Tld4UnifiedA2DU64Float:
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]331 ResNode = SelectTextureIntrinsic(N);
332 break;
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]333 case NVPTXISD::Suld1DI8Clamp:
334 case NVPTXISD::Suld1DI16Clamp:
335 case NVPTXISD::Suld1DI32Clamp:
336 case NVPTXISD::Suld1DI64Clamp:
337 case NVPTXISD::Suld1DV2I8Clamp:
338 case NVPTXISD::Suld1DV2I16Clamp:
339 case NVPTXISD::Suld1DV2I32Clamp:
340 case NVPTXISD::Suld1DV2I64Clamp:
341 case NVPTXISD::Suld1DV4I8Clamp:
342 case NVPTXISD::Suld1DV4I16Clamp:
343 case NVPTXISD::Suld1DV4I32Clamp:
344 case NVPTXISD::Suld1DArrayI8Clamp:
345 case NVPTXISD::Suld1DArrayI16Clamp:
346 case NVPTXISD::Suld1DArrayI32Clamp:
347 case NVPTXISD::Suld1DArrayI64Clamp:
348 case NVPTXISD::Suld1DArrayV2I8Clamp:
349 case NVPTXISD::Suld1DArrayV2I16Clamp:
350 case NVPTXISD::Suld1DArrayV2I32Clamp:
351 case NVPTXISD::Suld1DArrayV2I64Clamp:
352 case NVPTXISD::Suld1DArrayV4I8Clamp:
353 case NVPTXISD::Suld1DArrayV4I16Clamp:
354 case NVPTXISD::Suld1DArrayV4I32Clamp:
355 case NVPTXISD::Suld2DI8Clamp:
356 case NVPTXISD::Suld2DI16Clamp:
357 case NVPTXISD::Suld2DI32Clamp:
358 case NVPTXISD::Suld2DI64Clamp:
359 case NVPTXISD::Suld2DV2I8Clamp:
360 case NVPTXISD::Suld2DV2I16Clamp:
361 case NVPTXISD::Suld2DV2I32Clamp:
362 case NVPTXISD::Suld2DV2I64Clamp:
363 case NVPTXISD::Suld2DV4I8Clamp:
364 case NVPTXISD::Suld2DV4I16Clamp:
365 case NVPTXISD::Suld2DV4I32Clamp:
366 case NVPTXISD::Suld2DArrayI8Clamp:
367 case NVPTXISD::Suld2DArrayI16Clamp:
368 case NVPTXISD::Suld2DArrayI32Clamp:
369 case NVPTXISD::Suld2DArrayI64Clamp:
370 case NVPTXISD::Suld2DArrayV2I8Clamp:
371 case NVPTXISD::Suld2DArrayV2I16Clamp:
372 case NVPTXISD::Suld2DArrayV2I32Clamp:
373 case NVPTXISD::Suld2DArrayV2I64Clamp:
374 case NVPTXISD::Suld2DArrayV4I8Clamp:
375 case NVPTXISD::Suld2DArrayV4I16Clamp:
376 case NVPTXISD::Suld2DArrayV4I32Clamp:
377 case NVPTXISD::Suld3DI8Clamp:
378 case NVPTXISD::Suld3DI16Clamp:
379 case NVPTXISD::Suld3DI32Clamp:
380 case NVPTXISD::Suld3DI64Clamp:
381 case NVPTXISD::Suld3DV2I8Clamp:
382 case NVPTXISD::Suld3DV2I16Clamp:
383 case NVPTXISD::Suld3DV2I32Clamp:
384 case NVPTXISD::Suld3DV2I64Clamp:
385 case NVPTXISD::Suld3DV4I8Clamp:
386 case NVPTXISD::Suld3DV4I16Clamp:
387 case NVPTXISD::Suld3DV4I32Clamp:
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]388 case NVPTXISD::Suld1DI8Trap:
389 case NVPTXISD::Suld1DI16Trap:
390 case NVPTXISD::Suld1DI32Trap:
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]391 case NVPTXISD::Suld1DI64Trap:
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]392 case NVPTXISD::Suld1DV2I8Trap:
393 case NVPTXISD::Suld1DV2I16Trap:
394 case NVPTXISD::Suld1DV2I32Trap:
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]395 case NVPTXISD::Suld1DV2I64Trap:
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]396 case NVPTXISD::Suld1DV4I8Trap:
397 case NVPTXISD::Suld1DV4I16Trap:
398 case NVPTXISD::Suld1DV4I32Trap:
399 case NVPTXISD::Suld1DArrayI8Trap:
400 case NVPTXISD::Suld1DArrayI16Trap:
401 case NVPTXISD::Suld1DArrayI32Trap:
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]402 case NVPTXISD::Suld1DArrayI64Trap:
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]403 case NVPTXISD::Suld1DArrayV2I8Trap:
404 case NVPTXISD::Suld1DArrayV2I16Trap:
405 case NVPTXISD::Suld1DArrayV2I32Trap:
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]406 case NVPTXISD::Suld1DArrayV2I64Trap:
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]407 case NVPTXISD::Suld1DArrayV4I8Trap:
408 case NVPTXISD::Suld1DArrayV4I16Trap:
409 case NVPTXISD::Suld1DArrayV4I32Trap:
410 case NVPTXISD::Suld2DI8Trap:
411 case NVPTXISD::Suld2DI16Trap:
412 case NVPTXISD::Suld2DI32Trap:
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]413 case NVPTXISD::Suld2DI64Trap:
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]414 case NVPTXISD::Suld2DV2I8Trap:
415 case NVPTXISD::Suld2DV2I16Trap:
416 case NVPTXISD::Suld2DV2I32Trap:
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]417 case NVPTXISD::Suld2DV2I64Trap:
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]418 case NVPTXISD::Suld2DV4I8Trap:
419 case NVPTXISD::Suld2DV4I16Trap:
420 case NVPTXISD::Suld2DV4I32Trap:
421 case NVPTXISD::Suld2DArrayI8Trap:
422 case NVPTXISD::Suld2DArrayI16Trap:
423 case NVPTXISD::Suld2DArrayI32Trap:
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]424 case NVPTXISD::Suld2DArrayI64Trap:
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]425 case NVPTXISD::Suld2DArrayV2I8Trap:
426 case NVPTXISD::Suld2DArrayV2I16Trap:
427 case NVPTXISD::Suld2DArrayV2I32Trap:
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]428 case NVPTXISD::Suld2DArrayV2I64Trap:
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]429 case NVPTXISD::Suld2DArrayV4I8Trap:
430 case NVPTXISD::Suld2DArrayV4I16Trap:
431 case NVPTXISD::Suld2DArrayV4I32Trap:
432 case NVPTXISD::Suld3DI8Trap:
433 case NVPTXISD::Suld3DI16Trap:
434 case NVPTXISD::Suld3DI32Trap:
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]435 case NVPTXISD::Suld3DI64Trap:
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]436 case NVPTXISD::Suld3DV2I8Trap:
437 case NVPTXISD::Suld3DV2I16Trap:
438 case NVPTXISD::Suld3DV2I32Trap:
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]439 case NVPTXISD::Suld3DV2I64Trap:
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]440 case NVPTXISD::Suld3DV4I8Trap:
441 case NVPTXISD::Suld3DV4I16Trap:
442 case NVPTXISD::Suld3DV4I32Trap:
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]443 case NVPTXISD::Suld1DI8Zero:
444 case NVPTXISD::Suld1DI16Zero:
445 case NVPTXISD::Suld1DI32Zero:
446 case NVPTXISD::Suld1DI64Zero:
447 case NVPTXISD::Suld1DV2I8Zero:
448 case NVPTXISD::Suld1DV2I16Zero:
449 case NVPTXISD::Suld1DV2I32Zero:
450 case NVPTXISD::Suld1DV2I64Zero:
451 case NVPTXISD::Suld1DV4I8Zero:
452 case NVPTXISD::Suld1DV4I16Zero:
453 case NVPTXISD::Suld1DV4I32Zero:
454 case NVPTXISD::Suld1DArrayI8Zero:
455 case NVPTXISD::Suld1DArrayI16Zero:
456 case NVPTXISD::Suld1DArrayI32Zero:
457 case NVPTXISD::Suld1DArrayI64Zero:
458 case NVPTXISD::Suld1DArrayV2I8Zero:
459 case NVPTXISD::Suld1DArrayV2I16Zero:
460 case NVPTXISD::Suld1DArrayV2I32Zero:
461 case NVPTXISD::Suld1DArrayV2I64Zero:
462 case NVPTXISD::Suld1DArrayV4I8Zero:
463 case NVPTXISD::Suld1DArrayV4I16Zero:
464 case NVPTXISD::Suld1DArrayV4I32Zero:
465 case NVPTXISD::Suld2DI8Zero:
466 case NVPTXISD::Suld2DI16Zero:
467 case NVPTXISD::Suld2DI32Zero:
468 case NVPTXISD::Suld2DI64Zero:
469 case NVPTXISD::Suld2DV2I8Zero:
470 case NVPTXISD::Suld2DV2I16Zero:
471 case NVPTXISD::Suld2DV2I32Zero:
472 case NVPTXISD::Suld2DV2I64Zero:
473 case NVPTXISD::Suld2DV4I8Zero:
474 case NVPTXISD::Suld2DV4I16Zero:
475 case NVPTXISD::Suld2DV4I32Zero:
476 case NVPTXISD::Suld2DArrayI8Zero:
477 case NVPTXISD::Suld2DArrayI16Zero:
478 case NVPTXISD::Suld2DArrayI32Zero:
479 case NVPTXISD::Suld2DArrayI64Zero:
480 case NVPTXISD::Suld2DArrayV2I8Zero:
481 case NVPTXISD::Suld2DArrayV2I16Zero:
482 case NVPTXISD::Suld2DArrayV2I32Zero:
483 case NVPTXISD::Suld2DArrayV2I64Zero:
484 case NVPTXISD::Suld2DArrayV4I8Zero:
485 case NVPTXISD::Suld2DArrayV4I16Zero:
486 case NVPTXISD::Suld2DArrayV4I32Zero:
487 case NVPTXISD::Suld3DI8Zero:
488 case NVPTXISD::Suld3DI16Zero:
489 case NVPTXISD::Suld3DI32Zero:
490 case NVPTXISD::Suld3DI64Zero:
491 case NVPTXISD::Suld3DV2I8Zero:
492 case NVPTXISD::Suld3DV2I16Zero:
493 case NVPTXISD::Suld3DV2I32Zero:
494 case NVPTXISD::Suld3DV2I64Zero:
495 case NVPTXISD::Suld3DV4I8Zero:
496 case NVPTXISD::Suld3DV4I16Zero:
497 case NVPTXISD::Suld3DV4I32Zero:
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]498 ResNode = SelectSurfaceIntrinsic(N);
499 break;
Justin Holewinskica7a4f12014-06-27 18:35:27 +0000[diff] [blame]500 case ISD::AND:
501 case ISD::SRA:
502 case ISD::SRL:
503 // Try to select BFE
504 ResNode = SelectBFE(N);
505 break;
Justin Holewinskiba2fa6d2014-03-24 11:17:53 +0000[diff] [blame]506 case ISD::ADDRSPACECAST:
507 ResNode = SelectAddrSpaceCast(N);
508 break;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]509 default:
510 break;
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]511 }
512 if (ResNode)
513 return ResNode;
514 return SelectCode(N);
515}
516
Justin Holewinskib926d9d2014-06-27 18:35:51 +0000[diff] [blame]517SDNode *NVPTXDAGToDAGISel::SelectIntrinsicChain(SDNode *N) {
518 unsigned IID = cast<ConstantSDNode>(N->getOperand(1))->getZExtValue();
519 switch (IID) {
520 default:
521 return NULL;
522 case Intrinsic::nvvm_ldg_global_f:
523 case Intrinsic::nvvm_ldg_global_i:
524 case Intrinsic::nvvm_ldg_global_p:
525 case Intrinsic::nvvm_ldu_global_f:
526 case Intrinsic::nvvm_ldu_global_i:
527 case Intrinsic::nvvm_ldu_global_p:
528 return SelectLDGLDU(N);
529 }
530}
531
Eric Christopher9745b3a2015-01-30 01:41:01 +0000[diff] [blame^]532static unsigned int getCodeAddrSpace(MemSDNode *N) {
Nick Lewyckyaad475b2014-04-15 07:22:52 +0000[diff] [blame]533 const Value *Src = N->getMemOperand()->getValue();
Justin Holewinskib96d1392013-06-10 13:29:47 +0000[diff] [blame]534
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]535 if (!Src)
Justin Holewinskib96d1392013-06-10 13:29:47 +0000[diff] [blame]536 return NVPTX::PTXLdStInstCode::GENERIC;
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]537
538 if (const PointerType *PT = dyn_cast<PointerType>(Src->getType())) {
539 switch (PT->getAddressSpace()) {
Justin Holewinskib96d1392013-06-10 13:29:47 +0000[diff] [blame]540 case llvm::ADDRESS_SPACE_LOCAL: return NVPTX::PTXLdStInstCode::LOCAL;
541 case llvm::ADDRESS_SPACE_GLOBAL: return NVPTX::PTXLdStInstCode::GLOBAL;
542 case llvm::ADDRESS_SPACE_SHARED: return NVPTX::PTXLdStInstCode::SHARED;
543 case llvm::ADDRESS_SPACE_GENERIC: return NVPTX::PTXLdStInstCode::GENERIC;
544 case llvm::ADDRESS_SPACE_PARAM: return NVPTX::PTXLdStInstCode::PARAM;
545 case llvm::ADDRESS_SPACE_CONST: return NVPTX::PTXLdStInstCode::CONSTANT;
546 default: break;
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]547 }
548 }
Justin Holewinskib96d1392013-06-10 13:29:47 +0000[diff] [blame]549 return NVPTX::PTXLdStInstCode::GENERIC;
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]550}
551
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]552SDNode *NVPTXDAGToDAGISel::SelectIntrinsicNoChain(SDNode *N) {
553 unsigned IID = cast<ConstantSDNode>(N->getOperand(0))->getZExtValue();
554 switch (IID) {
555 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]556 return nullptr;
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]557 case Intrinsic::nvvm_texsurf_handle_internal:
558 return SelectTexSurfHandle(N);
559 }
560}
561
562SDNode *NVPTXDAGToDAGISel::SelectTexSurfHandle(SDNode *N) {
563 // Op 0 is the intrinsic ID
564 SDValue Wrapper = N->getOperand(1);
565 SDValue GlobalVal = Wrapper.getOperand(0);
566 return CurDAG->getMachineNode(NVPTX::texsurf_handles, SDLoc(N), MVT::i64,
567 GlobalVal);
568}
569
Justin Holewinskiba2fa6d2014-03-24 11:17:53 +0000[diff] [blame]570SDNode *NVPTXDAGToDAGISel::SelectAddrSpaceCast(SDNode *N) {
571 SDValue Src = N->getOperand(0);
572 AddrSpaceCastSDNode *CastN = cast<AddrSpaceCastSDNode>(N);
573 unsigned SrcAddrSpace = CastN->getSrcAddressSpace();
574 unsigned DstAddrSpace = CastN->getDestAddressSpace();
575
576 assert(SrcAddrSpace != DstAddrSpace &&
577 "addrspacecast must be between different address spaces");
578
579 if (DstAddrSpace == ADDRESS_SPACE_GENERIC) {
580 // Specific to generic
581 unsigned Opc;
582 switch (SrcAddrSpace) {
583 default: report_fatal_error("Bad address space in addrspacecast");
584 case ADDRESS_SPACE_GLOBAL:
Eric Christopher147bba22015-01-30 01:40:59 +0000[diff] [blame]585 Opc = Subtarget->is64Bit() ? NVPTX::cvta_global_yes_64
Justin Holewinskiba2fa6d2014-03-24 11:17:53 +0000[diff] [blame]586 : NVPTX::cvta_global_yes;
587 break;
588 case ADDRESS_SPACE_SHARED:
Eric Christopher147bba22015-01-30 01:40:59 +0000[diff] [blame]589 Opc = Subtarget->is64Bit() ? NVPTX::cvta_shared_yes_64
Justin Holewinskiba2fa6d2014-03-24 11:17:53 +0000[diff] [blame]590 : NVPTX::cvta_shared_yes;
591 break;
592 case ADDRESS_SPACE_CONST:
Eric Christopher147bba22015-01-30 01:40:59 +0000[diff] [blame]593 Opc = Subtarget->is64Bit() ? NVPTX::cvta_const_yes_64
Justin Holewinskiba2fa6d2014-03-24 11:17:53 +0000[diff] [blame]594 : NVPTX::cvta_const_yes;
595 break;
596 case ADDRESS_SPACE_LOCAL:
Eric Christopher147bba22015-01-30 01:40:59 +0000[diff] [blame]597 Opc = Subtarget->is64Bit() ? NVPTX::cvta_local_yes_64
Justin Holewinskiba2fa6d2014-03-24 11:17:53 +0000[diff] [blame]598 : NVPTX::cvta_local_yes;
599 break;
600 }
601 return CurDAG->getMachineNode(Opc, SDLoc(N), N->getValueType(0), Src);
602 } else {
603 // Generic to specific
604 if (SrcAddrSpace != 0)
605 report_fatal_error("Cannot cast between two non-generic address spaces");
606 unsigned Opc;
607 switch (DstAddrSpace) {
608 default: report_fatal_error("Bad address space in addrspacecast");
609 case ADDRESS_SPACE_GLOBAL:
Eric Christopher147bba22015-01-30 01:40:59 +0000[diff] [blame]610 Opc = Subtarget->is64Bit() ? NVPTX::cvta_to_global_yes_64
Justin Holewinskiba2fa6d2014-03-24 11:17:53 +0000[diff] [blame]611 : NVPTX::cvta_to_global_yes;
612 break;
613 case ADDRESS_SPACE_SHARED:
Eric Christopher147bba22015-01-30 01:40:59 +0000[diff] [blame]614 Opc = Subtarget->is64Bit() ? NVPTX::cvta_to_shared_yes_64
Justin Holewinskiba2fa6d2014-03-24 11:17:53 +0000[diff] [blame]615 : NVPTX::cvta_to_shared_yes;
616 break;
617 case ADDRESS_SPACE_CONST:
Eric Christopher147bba22015-01-30 01:40:59 +0000[diff] [blame]618 Opc = Subtarget->is64Bit() ? NVPTX::cvta_to_const_yes_64
Justin Holewinskiba2fa6d2014-03-24 11:17:53 +0000[diff] [blame]619 : NVPTX::cvta_to_const_yes;
620 break;
621 case ADDRESS_SPACE_LOCAL:
Eric Christopher147bba22015-01-30 01:40:59 +0000[diff] [blame]622 Opc = Subtarget->is64Bit() ? NVPTX::cvta_to_local_yes_64
Justin Holewinskiba2fa6d2014-03-24 11:17:53 +0000[diff] [blame]623 : NVPTX::cvta_to_local_yes;
624 break;
625 }
626 return CurDAG->getMachineNode(Opc, SDLoc(N), N->getValueType(0), Src);
627 }
628}
629
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]630SDNode *NVPTXDAGToDAGISel::SelectLoad(SDNode *N) {
Andrew Trickef9de2a2013-05-25 02:42:55 +0000[diff] [blame]631 SDLoc dl(N);
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]632 LoadSDNode *LD = cast<LoadSDNode>(N);
633 EVT LoadedVT = LD->getMemoryVT();
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]634 SDNode *NVPTXLD = nullptr;
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]635
636 // do not support pre/post inc/dec
637 if (LD->isIndexed())
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]638 return nullptr;
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]639
640 if (!LoadedVT.isSimple())
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]641 return nullptr;
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]642
643 // Address Space Setting
Eric Christopher9745b3a2015-01-30 01:41:01 +0000[diff] [blame^]644 unsigned int codeAddrSpace = getCodeAddrSpace(LD);
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]645
646 // Volatile Setting
647 // - .volatile is only availalble for .global and .shared
648 bool isVolatile = LD->isVolatile();
649 if (codeAddrSpace != NVPTX::PTXLdStInstCode::GLOBAL &&
650 codeAddrSpace != NVPTX::PTXLdStInstCode::SHARED &&
651 codeAddrSpace != NVPTX::PTXLdStInstCode::GENERIC)
652 isVolatile = false;
653
654 // Vector Setting
655 MVT SimpleVT = LoadedVT.getSimpleVT();
656 unsigned vecType = NVPTX::PTXLdStInstCode::Scalar;
657 if (SimpleVT.isVector()) {
658 unsigned num = SimpleVT.getVectorNumElements();
659 if (num == 2)
660 vecType = NVPTX::PTXLdStInstCode::V2;
661 else if (num == 4)
662 vecType = NVPTX::PTXLdStInstCode::V4;
663 else
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]664 return nullptr;
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]665 }
666
667 // Type Setting: fromType + fromTypeWidth
668 //
669 // Sign : ISD::SEXTLOAD
670 // Unsign : ISD::ZEXTLOAD, ISD::NON_EXTLOAD or ISD::EXTLOAD and the
671 // type is integer
672 // Float : ISD::NON_EXTLOAD or ISD::EXTLOAD and the type is float
673 MVT ScalarVT = SimpleVT.getScalarType();
Justin Holewinski994d66a2013-05-30 12:22:39 +0000[diff] [blame]674 // Read at least 8 bits (predicates are stored as 8-bit values)
675 unsigned fromTypeWidth = std::max(8U, ScalarVT.getSizeInBits());
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]676 unsigned int fromType;
677 if ((LD->getExtensionType() == ISD::SEXTLOAD))
678 fromType = NVPTX::PTXLdStInstCode::Signed;
679 else if (ScalarVT.isFloatingPoint())
680 fromType = NVPTX::PTXLdStInstCode::Float;
681 else
682 fromType = NVPTX::PTXLdStInstCode::Unsigned;
683
684 // Create the machine instruction DAG
685 SDValue Chain = N->getOperand(0);
686 SDValue N1 = N->getOperand(1);
687 SDValue Addr;
688 SDValue Offset, Base;
689 unsigned Opcode;
Craig Topperd9c27832013-08-15 02:44:19 +0000[diff] [blame]690 MVT::SimpleValueType TargetVT = LD->getSimpleValueType(0).SimpleTy;
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]691
692 if (SelectDirectAddr(N1, Addr)) {
693 switch (TargetVT) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]694 case MVT::i8:
695 Opcode = NVPTX::LD_i8_avar;
696 break;
697 case MVT::i16:
698 Opcode = NVPTX::LD_i16_avar;
699 break;
700 case MVT::i32:
701 Opcode = NVPTX::LD_i32_avar;
702 break;
703 case MVT::i64:
704 Opcode = NVPTX::LD_i64_avar;
705 break;
706 case MVT::f32:
707 Opcode = NVPTX::LD_f32_avar;
708 break;
709 case MVT::f64:
710 Opcode = NVPTX::LD_f64_avar;
711 break;
712 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]713 return nullptr;
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]714 }
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]715 SDValue Ops[] = { getI32Imm(isVolatile), getI32Imm(codeAddrSpace),
716 getI32Imm(vecType), getI32Imm(fromType),
717 getI32Imm(fromTypeWidth), Addr, Chain };
Michael Liaob53d8962013-04-19 22:22:57 +0000[diff] [blame]718 NVPTXLD = CurDAG->getMachineNode(Opcode, dl, TargetVT, MVT::Other, Ops);
Eric Christopher147bba22015-01-30 01:40:59 +0000[diff] [blame]719 } else if (Subtarget->is64Bit()
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]720 ? SelectADDRsi64(N1.getNode(), N1, Base, Offset)
721 : SelectADDRsi(N1.getNode(), N1, Base, Offset)) {
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]722 switch (TargetVT) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]723 case MVT::i8:
724 Opcode = NVPTX::LD_i8_asi;
725 break;
726 case MVT::i16:
727 Opcode = NVPTX::LD_i16_asi;
728 break;
729 case MVT::i32:
730 Opcode = NVPTX::LD_i32_asi;
731 break;
732 case MVT::i64:
733 Opcode = NVPTX::LD_i64_asi;
734 break;
735 case MVT::f32:
736 Opcode = NVPTX::LD_f32_asi;
737 break;
738 case MVT::f64:
739 Opcode = NVPTX::LD_f64_asi;
740 break;
741 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]742 return nullptr;
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]743 }
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]744 SDValue Ops[] = { getI32Imm(isVolatile), getI32Imm(codeAddrSpace),
745 getI32Imm(vecType), getI32Imm(fromType),
746 getI32Imm(fromTypeWidth), Base, Offset, Chain };
Michael Liaob53d8962013-04-19 22:22:57 +0000[diff] [blame]747 NVPTXLD = CurDAG->getMachineNode(Opcode, dl, TargetVT, MVT::Other, Ops);
Eric Christopher147bba22015-01-30 01:40:59 +0000[diff] [blame]748 } else if (Subtarget->is64Bit()
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]749 ? SelectADDRri64(N1.getNode(), N1, Base, Offset)
750 : SelectADDRri(N1.getNode(), N1, Base, Offset)) {
Eric Christopher147bba22015-01-30 01:40:59 +0000[diff] [blame]751 if (Subtarget->is64Bit()) {
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]752 switch (TargetVT) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]753 case MVT::i8:
754 Opcode = NVPTX::LD_i8_ari_64;
755 break;
756 case MVT::i16:
757 Opcode = NVPTX::LD_i16_ari_64;
758 break;
759 case MVT::i32:
760 Opcode = NVPTX::LD_i32_ari_64;
761 break;
762 case MVT::i64:
763 Opcode = NVPTX::LD_i64_ari_64;
764 break;
765 case MVT::f32:
766 Opcode = NVPTX::LD_f32_ari_64;
767 break;
768 case MVT::f64:
769 Opcode = NVPTX::LD_f64_ari_64;
770 break;
771 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]772 return nullptr;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]773 }
774 } else {
775 switch (TargetVT) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]776 case MVT::i8:
777 Opcode = NVPTX::LD_i8_ari;
778 break;
779 case MVT::i16:
780 Opcode = NVPTX::LD_i16_ari;
781 break;
782 case MVT::i32:
783 Opcode = NVPTX::LD_i32_ari;
784 break;
785 case MVT::i64:
786 Opcode = NVPTX::LD_i64_ari;
787 break;
788 case MVT::f32:
789 Opcode = NVPTX::LD_f32_ari;
790 break;
791 case MVT::f64:
792 Opcode = NVPTX::LD_f64_ari;
793 break;
794 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]795 return nullptr;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]796 }
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]797 }
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]798 SDValue Ops[] = { getI32Imm(isVolatile), getI32Imm(codeAddrSpace),
799 getI32Imm(vecType), getI32Imm(fromType),
800 getI32Imm(fromTypeWidth), Base, Offset, Chain };
Michael Liaob53d8962013-04-19 22:22:57 +0000[diff] [blame]801 NVPTXLD = CurDAG->getMachineNode(Opcode, dl, TargetVT, MVT::Other, Ops);
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]802 } else {
Eric Christopher147bba22015-01-30 01:40:59 +0000[diff] [blame]803 if (Subtarget->is64Bit()) {
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]804 switch (TargetVT) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]805 case MVT::i8:
806 Opcode = NVPTX::LD_i8_areg_64;
807 break;
808 case MVT::i16:
809 Opcode = NVPTX::LD_i16_areg_64;
810 break;
811 case MVT::i32:
812 Opcode = NVPTX::LD_i32_areg_64;
813 break;
814 case MVT::i64:
815 Opcode = NVPTX::LD_i64_areg_64;
816 break;
817 case MVT::f32:
818 Opcode = NVPTX::LD_f32_areg_64;
819 break;
820 case MVT::f64:
821 Opcode = NVPTX::LD_f64_areg_64;
822 break;
823 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]824 return nullptr;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]825 }
826 } else {
827 switch (TargetVT) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]828 case MVT::i8:
829 Opcode = NVPTX::LD_i8_areg;
830 break;
831 case MVT::i16:
832 Opcode = NVPTX::LD_i16_areg;
833 break;
834 case MVT::i32:
835 Opcode = NVPTX::LD_i32_areg;
836 break;
837 case MVT::i64:
838 Opcode = NVPTX::LD_i64_areg;
839 break;
840 case MVT::f32:
841 Opcode = NVPTX::LD_f32_areg;
842 break;
843 case MVT::f64:
844 Opcode = NVPTX::LD_f64_areg;
845 break;
846 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]847 return nullptr;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]848 }
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]849 }
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]850 SDValue Ops[] = { getI32Imm(isVolatile), getI32Imm(codeAddrSpace),
851 getI32Imm(vecType), getI32Imm(fromType),
852 getI32Imm(fromTypeWidth), N1, Chain };
Michael Liaob53d8962013-04-19 22:22:57 +0000[diff] [blame]853 NVPTXLD = CurDAG->getMachineNode(Opcode, dl, TargetVT, MVT::Other, Ops);
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]854 }
855
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]856 if (NVPTXLD) {
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]857 MachineSDNode::mmo_iterator MemRefs0 = MF->allocateMemRefsArray(1);
858 MemRefs0[0] = cast<MemSDNode>(N)->getMemOperand();
859 cast<MachineSDNode>(NVPTXLD)->setMemRefs(MemRefs0, MemRefs0 + 1);
860 }
861
862 return NVPTXLD;
863}
864
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]865SDNode *NVPTXDAGToDAGISel::SelectLoadVector(SDNode *N) {
866
867 SDValue Chain = N->getOperand(0);
868 SDValue Op1 = N->getOperand(1);
869 SDValue Addr, Offset, Base;
870 unsigned Opcode;
Andrew Trickef9de2a2013-05-25 02:42:55 +0000[diff] [blame]871 SDLoc DL(N);
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]872 SDNode *LD;
873 MemSDNode *MemSD = cast<MemSDNode>(N);
874 EVT LoadedVT = MemSD->getMemoryVT();
875
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]876 if (!LoadedVT.isSimple())
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]877 return nullptr;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]878
879 // Address Space Setting
Eric Christopher9745b3a2015-01-30 01:41:01 +0000[diff] [blame^]880 unsigned int CodeAddrSpace = getCodeAddrSpace(MemSD);
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]881
882 // Volatile Setting
883 // - .volatile is only availalble for .global and .shared
884 bool IsVolatile = MemSD->isVolatile();
885 if (CodeAddrSpace != NVPTX::PTXLdStInstCode::GLOBAL &&
886 CodeAddrSpace != NVPTX::PTXLdStInstCode::SHARED &&
887 CodeAddrSpace != NVPTX::PTXLdStInstCode::GENERIC)
888 IsVolatile = false;
889
890 // Vector Setting
891 MVT SimpleVT = LoadedVT.getSimpleVT();
892
893 // Type Setting: fromType + fromTypeWidth
894 //
895 // Sign : ISD::SEXTLOAD
896 // Unsign : ISD::ZEXTLOAD, ISD::NON_EXTLOAD or ISD::EXTLOAD and the
897 // type is integer
898 // Float : ISD::NON_EXTLOAD or ISD::EXTLOAD and the type is float
899 MVT ScalarVT = SimpleVT.getScalarType();
Justin Holewinski994d66a2013-05-30 12:22:39 +0000[diff] [blame]900 // Read at least 8 bits (predicates are stored as 8-bit values)
901 unsigned FromTypeWidth = std::max(8U, ScalarVT.getSizeInBits());
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]902 unsigned int FromType;
903 // The last operand holds the original LoadSDNode::getExtensionType() value
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]904 unsigned ExtensionType = cast<ConstantSDNode>(
905 N->getOperand(N->getNumOperands() - 1))->getZExtValue();
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]906 if (ExtensionType == ISD::SEXTLOAD)
907 FromType = NVPTX::PTXLdStInstCode::Signed;
908 else if (ScalarVT.isFloatingPoint())
909 FromType = NVPTX::PTXLdStInstCode::Float;
910 else
911 FromType = NVPTX::PTXLdStInstCode::Unsigned;
912
913 unsigned VecType;
914
915 switch (N->getOpcode()) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]916 case NVPTXISD::LoadV2:
917 VecType = NVPTX::PTXLdStInstCode::V2;
918 break;
919 case NVPTXISD::LoadV4:
920 VecType = NVPTX::PTXLdStInstCode::V4;
921 break;
922 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]923 return nullptr;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]924 }
925
926 EVT EltVT = N->getValueType(0);
927
928 if (SelectDirectAddr(Op1, Addr)) {
929 switch (N->getOpcode()) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]930 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]931 return nullptr;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]932 case NVPTXISD::LoadV2:
933 switch (EltVT.getSimpleVT().SimpleTy) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]934 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]935 return nullptr;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]936 case MVT::i8:
937 Opcode = NVPTX::LDV_i8_v2_avar;
938 break;
939 case MVT::i16:
940 Opcode = NVPTX::LDV_i16_v2_avar;
941 break;
942 case MVT::i32:
943 Opcode = NVPTX::LDV_i32_v2_avar;
944 break;
945 case MVT::i64:
946 Opcode = NVPTX::LDV_i64_v2_avar;
947 break;
948 case MVT::f32:
949 Opcode = NVPTX::LDV_f32_v2_avar;
950 break;
951 case MVT::f64:
952 Opcode = NVPTX::LDV_f64_v2_avar;
953 break;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]954 }
955 break;
956 case NVPTXISD::LoadV4:
957 switch (EltVT.getSimpleVT().SimpleTy) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]958 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]959 return nullptr;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]960 case MVT::i8:
961 Opcode = NVPTX::LDV_i8_v4_avar;
962 break;
963 case MVT::i16:
964 Opcode = NVPTX::LDV_i16_v4_avar;
965 break;
966 case MVT::i32:
967 Opcode = NVPTX::LDV_i32_v4_avar;
968 break;
969 case MVT::f32:
970 Opcode = NVPTX::LDV_f32_v4_avar;
971 break;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]972 }
973 break;
974 }
975
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]976 SDValue Ops[] = { getI32Imm(IsVolatile), getI32Imm(CodeAddrSpace),
977 getI32Imm(VecType), getI32Imm(FromType),
978 getI32Imm(FromTypeWidth), Addr, Chain };
Michael Liaob53d8962013-04-19 22:22:57 +0000[diff] [blame]979 LD = CurDAG->getMachineNode(Opcode, DL, N->getVTList(), Ops);
Eric Christopher147bba22015-01-30 01:40:59 +0000[diff] [blame]980 } else if (Subtarget->is64Bit()
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]981 ? SelectADDRsi64(Op1.getNode(), Op1, Base, Offset)
982 : SelectADDRsi(Op1.getNode(), Op1, Base, Offset)) {
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]983 switch (N->getOpcode()) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]984 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]985 return nullptr;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]986 case NVPTXISD::LoadV2:
987 switch (EltVT.getSimpleVT().SimpleTy) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]988 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]989 return nullptr;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]990 case MVT::i8:
991 Opcode = NVPTX::LDV_i8_v2_asi;
992 break;
993 case MVT::i16:
994 Opcode = NVPTX::LDV_i16_v2_asi;
995 break;
996 case MVT::i32:
997 Opcode = NVPTX::LDV_i32_v2_asi;
998 break;
999 case MVT::i64:
1000 Opcode = NVPTX::LDV_i64_v2_asi;
1001 break;
1002 case MVT::f32:
1003 Opcode = NVPTX::LDV_f32_v2_asi;
1004 break;
1005 case MVT::f64:
1006 Opcode = NVPTX::LDV_f64_v2_asi;
1007 break;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]1008 }
1009 break;
1010 case NVPTXISD::LoadV4:
1011 switch (EltVT.getSimpleVT().SimpleTy) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1012 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]1013 return nullptr;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1014 case MVT::i8:
1015 Opcode = NVPTX::LDV_i8_v4_asi;
1016 break;
1017 case MVT::i16:
1018 Opcode = NVPTX::LDV_i16_v4_asi;
1019 break;
1020 case MVT::i32:
1021 Opcode = NVPTX::LDV_i32_v4_asi;
1022 break;
1023 case MVT::f32:
1024 Opcode = NVPTX::LDV_f32_v4_asi;
1025 break;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]1026 }
1027 break;
1028 }
1029
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1030 SDValue Ops[] = { getI32Imm(IsVolatile), getI32Imm(CodeAddrSpace),
1031 getI32Imm(VecType), getI32Imm(FromType),
1032 getI32Imm(FromTypeWidth), Base, Offset, Chain };
Michael Liaob53d8962013-04-19 22:22:57 +0000[diff] [blame]1033 LD = CurDAG->getMachineNode(Opcode, DL, N->getVTList(), Ops);
Eric Christopher147bba22015-01-30 01:40:59 +0000[diff] [blame]1034 } else if (Subtarget->is64Bit()
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1035 ? SelectADDRri64(Op1.getNode(), Op1, Base, Offset)
1036 : SelectADDRri(Op1.getNode(), Op1, Base, Offset)) {
Eric Christopher147bba22015-01-30 01:40:59 +0000[diff] [blame]1037 if (Subtarget->is64Bit()) {
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]1038 switch (N->getOpcode()) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1039 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]1040 return nullptr;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]1041 case NVPTXISD::LoadV2:
1042 switch (EltVT.getSimpleVT().SimpleTy) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1043 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]1044 return nullptr;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1045 case MVT::i8:
1046 Opcode = NVPTX::LDV_i8_v2_ari_64;
1047 break;
1048 case MVT::i16:
1049 Opcode = NVPTX::LDV_i16_v2_ari_64;
1050 break;
1051 case MVT::i32:
1052 Opcode = NVPTX::LDV_i32_v2_ari_64;
1053 break;
1054 case MVT::i64:
1055 Opcode = NVPTX::LDV_i64_v2_ari_64;
1056 break;
1057 case MVT::f32:
1058 Opcode = NVPTX::LDV_f32_v2_ari_64;
1059 break;
1060 case MVT::f64:
1061 Opcode = NVPTX::LDV_f64_v2_ari_64;
1062 break;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]1063 }
1064 break;
1065 case NVPTXISD::LoadV4:
1066 switch (EltVT.getSimpleVT().SimpleTy) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1067 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]1068 return nullptr;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1069 case MVT::i8:
1070 Opcode = NVPTX::LDV_i8_v4_ari_64;
1071 break;
1072 case MVT::i16:
1073 Opcode = NVPTX::LDV_i16_v4_ari_64;
1074 break;
1075 case MVT::i32:
1076 Opcode = NVPTX::LDV_i32_v4_ari_64;
1077 break;
1078 case MVT::f32:
1079 Opcode = NVPTX::LDV_f32_v4_ari_64;
1080 break;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]1081 }
1082 break;
1083 }
1084 } else {
1085 switch (N->getOpcode()) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1086 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]1087 return nullptr;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]1088 case NVPTXISD::LoadV2:
1089 switch (EltVT.getSimpleVT().SimpleTy) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1090 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]1091 return nullptr;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1092 case MVT::i8:
1093 Opcode = NVPTX::LDV_i8_v2_ari;
1094 break;
1095 case MVT::i16:
1096 Opcode = NVPTX::LDV_i16_v2_ari;
1097 break;
1098 case MVT::i32:
1099 Opcode = NVPTX::LDV_i32_v2_ari;
1100 break;
1101 case MVT::i64:
1102 Opcode = NVPTX::LDV_i64_v2_ari;
1103 break;
1104 case MVT::f32:
1105 Opcode = NVPTX::LDV_f32_v2_ari;
1106 break;
1107 case MVT::f64:
1108 Opcode = NVPTX::LDV_f64_v2_ari;
1109 break;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]1110 }
1111 break;
1112 case NVPTXISD::LoadV4:
1113 switch (EltVT.getSimpleVT().SimpleTy) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1114 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]1115 return nullptr;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1116 case MVT::i8:
1117 Opcode = NVPTX::LDV_i8_v4_ari;
1118 break;
1119 case MVT::i16:
1120 Opcode = NVPTX::LDV_i16_v4_ari;
1121 break;
1122 case MVT::i32:
1123 Opcode = NVPTX::LDV_i32_v4_ari;
1124 break;
1125 case MVT::f32:
1126 Opcode = NVPTX::LDV_f32_v4_ari;
1127 break;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]1128 }
1129 break;
1130 }
1131 }
1132
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1133 SDValue Ops[] = { getI32Imm(IsVolatile), getI32Imm(CodeAddrSpace),
1134 getI32Imm(VecType), getI32Imm(FromType),
1135 getI32Imm(FromTypeWidth), Base, Offset, Chain };
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]1136
Michael Liaob53d8962013-04-19 22:22:57 +0000[diff] [blame]1137 LD = CurDAG->getMachineNode(Opcode, DL, N->getVTList(), Ops);
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]1138 } else {
Eric Christopher147bba22015-01-30 01:40:59 +0000[diff] [blame]1139 if (Subtarget->is64Bit()) {
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]1140 switch (N->getOpcode()) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1141 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]1142 return nullptr;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]1143 case NVPTXISD::LoadV2:
1144 switch (EltVT.getSimpleVT().SimpleTy) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1145 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]1146 return nullptr;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1147 case MVT::i8:
1148 Opcode = NVPTX::LDV_i8_v2_areg_64;
1149 break;
1150 case MVT::i16:
1151 Opcode = NVPTX::LDV_i16_v2_areg_64;
1152 break;
1153 case MVT::i32:
1154 Opcode = NVPTX::LDV_i32_v2_areg_64;
1155 break;
1156 case MVT::i64:
1157 Opcode = NVPTX::LDV_i64_v2_areg_64;
1158 break;
1159 case MVT::f32:
1160 Opcode = NVPTX::LDV_f32_v2_areg_64;
1161 break;
1162 case MVT::f64:
1163 Opcode = NVPTX::LDV_f64_v2_areg_64;
1164 break;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]1165 }
1166 break;
1167 case NVPTXISD::LoadV4:
1168 switch (EltVT.getSimpleVT().SimpleTy) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1169 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]1170 return nullptr;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1171 case MVT::i8:
1172 Opcode = NVPTX::LDV_i8_v4_areg_64;
1173 break;
1174 case MVT::i16:
1175 Opcode = NVPTX::LDV_i16_v4_areg_64;
1176 break;
1177 case MVT::i32:
1178 Opcode = NVPTX::LDV_i32_v4_areg_64;
1179 break;
1180 case MVT::f32:
1181 Opcode = NVPTX::LDV_f32_v4_areg_64;
1182 break;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]1183 }
1184 break;
1185 }
1186 } else {
1187 switch (N->getOpcode()) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1188 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]1189 return nullptr;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]1190 case NVPTXISD::LoadV2:
1191 switch (EltVT.getSimpleVT().SimpleTy) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1192 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]1193 return nullptr;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1194 case MVT::i8:
1195 Opcode = NVPTX::LDV_i8_v2_areg;
1196 break;
1197 case MVT::i16:
1198 Opcode = NVPTX::LDV_i16_v2_areg;
1199 break;
1200 case MVT::i32:
1201 Opcode = NVPTX::LDV_i32_v2_areg;
1202 break;
1203 case MVT::i64:
1204 Opcode = NVPTX::LDV_i64_v2_areg;
1205 break;
1206 case MVT::f32:
1207 Opcode = NVPTX::LDV_f32_v2_areg;
1208 break;
1209 case MVT::f64:
1210 Opcode = NVPTX::LDV_f64_v2_areg;
1211 break;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]1212 }
1213 break;
1214 case NVPTXISD::LoadV4:
1215 switch (EltVT.getSimpleVT().SimpleTy) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1216 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]1217 return nullptr;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1218 case MVT::i8:
1219 Opcode = NVPTX::LDV_i8_v4_areg;
1220 break;
1221 case MVT::i16:
1222 Opcode = NVPTX::LDV_i16_v4_areg;
1223 break;
1224 case MVT::i32:
1225 Opcode = NVPTX::LDV_i32_v4_areg;
1226 break;
1227 case MVT::f32:
1228 Opcode = NVPTX::LDV_f32_v4_areg;
1229 break;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]1230 }
1231 break;
1232 }
1233 }
1234
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1235 SDValue Ops[] = { getI32Imm(IsVolatile), getI32Imm(CodeAddrSpace),
1236 getI32Imm(VecType), getI32Imm(FromType),
1237 getI32Imm(FromTypeWidth), Op1, Chain };
Michael Liaob53d8962013-04-19 22:22:57 +0000[diff] [blame]1238 LD = CurDAG->getMachineNode(Opcode, DL, N->getVTList(), Ops);
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]1239 }
1240
1241 MachineSDNode::mmo_iterator MemRefs0 = MF->allocateMemRefsArray(1);
1242 MemRefs0[0] = cast<MemSDNode>(N)->getMemOperand();
1243 cast<MachineSDNode>(LD)->setMemRefs(MemRefs0, MemRefs0 + 1);
1244
1245 return LD;
1246}
1247
Justin Holewinskib926d9d2014-06-27 18:35:51 +0000[diff] [blame]1248SDNode *NVPTXDAGToDAGISel::SelectLDGLDU(SDNode *N) {
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]1249
1250 SDValue Chain = N->getOperand(0);
Justin Holewinskib926d9d2014-06-27 18:35:51 +0000[diff] [blame]1251 SDValue Op1;
1252 MemSDNode *Mem;
1253 bool IsLDG = true;
1254
1255 // If this is an LDG intrinsic, the address is the third operand. Its its an
1256 // LDG/LDU SD node (from custom vector handling), then its the second operand
1257 if (N->getOpcode() == ISD::INTRINSIC_W_CHAIN) {
1258 Op1 = N->getOperand(2);
1259 Mem = cast<MemIntrinsicSDNode>(N);
1260 unsigned IID = cast<ConstantSDNode>(N->getOperand(1))->getZExtValue();
1261 switch (IID) {
1262 default:
1263 return NULL;
1264 case Intrinsic::nvvm_ldg_global_f:
1265 case Intrinsic::nvvm_ldg_global_i:
1266 case Intrinsic::nvvm_ldg_global_p:
1267 IsLDG = true;
1268 break;
1269 case Intrinsic::nvvm_ldu_global_f:
1270 case Intrinsic::nvvm_ldu_global_i:
1271 case Intrinsic::nvvm_ldu_global_p:
1272 IsLDG = false;
1273 break;
1274 }
1275 } else {
1276 Op1 = N->getOperand(1);
1277 Mem = cast<MemSDNode>(N);
1278 }
1279
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]1280 unsigned Opcode;
Andrew Trickef9de2a2013-05-25 02:42:55 +0000[diff] [blame]1281 SDLoc DL(N);
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]1282 SDNode *LD;
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1283 SDValue Base, Offset, Addr;
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]1284
Justin Holewinskib926d9d2014-06-27 18:35:51 +0000[diff] [blame]1285 EVT EltVT = Mem->getMemoryVT();
1286 if (EltVT.isVector()) {
1287 EltVT = EltVT.getVectorElementType();
1288 }
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]1289
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1290 if (SelectDirectAddr(Op1, Addr)) {
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]1291 switch (N->getOpcode()) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1292 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]1293 return nullptr;
Justin Holewinskib926d9d2014-06-27 18:35:51 +0000[diff] [blame]1294 case ISD::INTRINSIC_W_CHAIN:
1295 if (IsLDG) {
1296 switch (EltVT.getSimpleVT().SimpleTy) {
1297 default:
1298 return nullptr;
1299 case MVT::i8:
1300 Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i8avar;
1301 break;
1302 case MVT::i16:
1303 Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i16avar;
1304 break;
1305 case MVT::i32:
1306 Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i32avar;
1307 break;
1308 case MVT::i64:
1309 Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i64avar;
1310 break;
1311 case MVT::f32:
1312 Opcode = NVPTX::INT_PTX_LDG_GLOBAL_f32avar;
1313 break;
1314 case MVT::f64:
1315 Opcode = NVPTX::INT_PTX_LDG_GLOBAL_f64avar;
1316 break;
1317 }
1318 } else {
1319 switch (EltVT.getSimpleVT().SimpleTy) {
1320 default:
1321 return nullptr;
1322 case MVT::i8:
1323 Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i8avar;
1324 break;
1325 case MVT::i16:
1326 Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i16avar;
1327 break;
1328 case MVT::i32:
1329 Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i32avar;
1330 break;
1331 case MVT::i64:
1332 Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i64avar;
1333 break;
1334 case MVT::f32:
1335 Opcode = NVPTX::INT_PTX_LDU_GLOBAL_f32avar;
1336 break;
1337 case MVT::f64:
1338 Opcode = NVPTX::INT_PTX_LDU_GLOBAL_f64avar;
1339 break;
1340 }
1341 }
1342 break;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]1343 case NVPTXISD::LDGV2:
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1344 switch (EltVT.getSimpleVT().SimpleTy) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1345 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]1346 return nullptr;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1347 case MVT::i8:
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1348 Opcode = NVPTX::INT_PTX_LDG_G_v2i8_ELE_avar;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1349 break;
1350 case MVT::i16:
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1351 Opcode = NVPTX::INT_PTX_LDG_G_v2i16_ELE_avar;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1352 break;
1353 case MVT::i32:
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1354 Opcode = NVPTX::INT_PTX_LDG_G_v2i32_ELE_avar;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1355 break;
1356 case MVT::i64:
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1357 Opcode = NVPTX::INT_PTX_LDG_G_v2i64_ELE_avar;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1358 break;
1359 case MVT::f32:
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1360 Opcode = NVPTX::INT_PTX_LDG_G_v2f32_ELE_avar;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1361 break;
1362 case MVT::f64:
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1363 Opcode = NVPTX::INT_PTX_LDG_G_v2f64_ELE_avar;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1364 break;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]1365 }
1366 break;
1367 case NVPTXISD::LDUV2:
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1368 switch (EltVT.getSimpleVT().SimpleTy) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1369 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]1370 return nullptr;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1371 case MVT::i8:
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1372 Opcode = NVPTX::INT_PTX_LDU_G_v2i8_ELE_avar;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1373 break;
1374 case MVT::i16:
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1375 Opcode = NVPTX::INT_PTX_LDU_G_v2i16_ELE_avar;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1376 break;
1377 case MVT::i32:
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1378 Opcode = NVPTX::INT_PTX_LDU_G_v2i32_ELE_avar;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1379 break;
1380 case MVT::i64:
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1381 Opcode = NVPTX::INT_PTX_LDU_G_v2i64_ELE_avar;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1382 break;
1383 case MVT::f32:
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1384 Opcode = NVPTX::INT_PTX_LDU_G_v2f32_ELE_avar;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1385 break;
1386 case MVT::f64:
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1387 Opcode = NVPTX::INT_PTX_LDU_G_v2f64_ELE_avar;
1388 break;
1389 }
1390 break;
1391 case NVPTXISD::LDGV4:
1392 switch (EltVT.getSimpleVT().SimpleTy) {
1393 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]1394 return nullptr;
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1395 case MVT::i8:
1396 Opcode = NVPTX::INT_PTX_LDG_G_v4i8_ELE_avar;
1397 break;
1398 case MVT::i16:
1399 Opcode = NVPTX::INT_PTX_LDG_G_v4i16_ELE_avar;
1400 break;
1401 case MVT::i32:
1402 Opcode = NVPTX::INT_PTX_LDG_G_v4i32_ELE_avar;
1403 break;
1404 case MVT::f32:
1405 Opcode = NVPTX::INT_PTX_LDG_G_v4f32_ELE_avar;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1406 break;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]1407 }
1408 break;
1409 case NVPTXISD::LDUV4:
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1410 switch (EltVT.getSimpleVT().SimpleTy) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1411 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]1412 return nullptr;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1413 case MVT::i8:
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1414 Opcode = NVPTX::INT_PTX_LDU_G_v4i8_ELE_avar;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1415 break;
1416 case MVT::i16:
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1417 Opcode = NVPTX::INT_PTX_LDU_G_v4i16_ELE_avar;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1418 break;
1419 case MVT::i32:
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1420 Opcode = NVPTX::INT_PTX_LDU_G_v4i32_ELE_avar;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1421 break;
1422 case MVT::f32:
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1423 Opcode = NVPTX::INT_PTX_LDU_G_v4f32_ELE_avar;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1424 break;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]1425 }
1426 break;
1427 }
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1428
1429 SDValue Ops[] = { Addr, Chain };
Craig Topper2d2aa0c2014-04-30 07:17:30 +0000[diff] [blame]1430 LD = CurDAG->getMachineNode(Opcode, DL, N->getVTList(), Ops);
Eric Christopher147bba22015-01-30 01:40:59 +0000[diff] [blame]1431 } else if (Subtarget->is64Bit()
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1432 ? SelectADDRri64(Op1.getNode(), Op1, Base, Offset)
1433 : SelectADDRri(Op1.getNode(), Op1, Base, Offset)) {
Eric Christopher147bba22015-01-30 01:40:59 +0000[diff] [blame]1434 if (Subtarget->is64Bit()) {
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1435 switch (N->getOpcode()) {
1436 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]1437 return nullptr;
Justin Holewinskib926d9d2014-06-27 18:35:51 +0000[diff] [blame]1438 case ISD::INTRINSIC_W_CHAIN:
1439 if (IsLDG) {
1440 switch (EltVT.getSimpleVT().SimpleTy) {
1441 default:
1442 return nullptr;
1443 case MVT::i8:
1444 Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i8ari64;
1445 break;
1446 case MVT::i16:
1447 Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i16ari64;
1448 break;
1449 case MVT::i32:
1450 Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i32ari64;
1451 break;
1452 case MVT::i64:
1453 Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i64ari64;
1454 break;
1455 case MVT::f32:
1456 Opcode = NVPTX::INT_PTX_LDG_GLOBAL_f32ari64;
1457 break;
1458 case MVT::f64:
1459 Opcode = NVPTX::INT_PTX_LDG_GLOBAL_f64ari64;
1460 break;
1461 }
1462 } else {
1463 switch (EltVT.getSimpleVT().SimpleTy) {
1464 default:
1465 return nullptr;
1466 case MVT::i8:
1467 Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i8ari64;
1468 break;
1469 case MVT::i16:
1470 Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i16ari64;
1471 break;
1472 case MVT::i32:
1473 Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i32ari64;
1474 break;
1475 case MVT::i64:
1476 Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i64ari64;
1477 break;
1478 case MVT::f32:
1479 Opcode = NVPTX::INT_PTX_LDU_GLOBAL_f32ari64;
1480 break;
1481 case MVT::f64:
1482 Opcode = NVPTX::INT_PTX_LDU_GLOBAL_f64ari64;
1483 break;
1484 }
1485 }
1486 break;
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1487 case NVPTXISD::LDGV2:
1488 switch (EltVT.getSimpleVT().SimpleTy) {
1489 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]1490 return nullptr;
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1491 case MVT::i8:
1492 Opcode = NVPTX::INT_PTX_LDG_G_v2i8_ELE_ari64;
1493 break;
1494 case MVT::i16:
1495 Opcode = NVPTX::INT_PTX_LDG_G_v2i16_ELE_ari64;
1496 break;
1497 case MVT::i32:
1498 Opcode = NVPTX::INT_PTX_LDG_G_v2i32_ELE_ari64;
1499 break;
1500 case MVT::i64:
1501 Opcode = NVPTX::INT_PTX_LDG_G_v2i64_ELE_ari64;
1502 break;
1503 case MVT::f32:
1504 Opcode = NVPTX::INT_PTX_LDG_G_v2f32_ELE_ari64;
1505 break;
1506 case MVT::f64:
1507 Opcode = NVPTX::INT_PTX_LDG_G_v2f64_ELE_ari64;
1508 break;
1509 }
1510 break;
1511 case NVPTXISD::LDUV2:
1512 switch (EltVT.getSimpleVT().SimpleTy) {
1513 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]1514 return nullptr;
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1515 case MVT::i8:
1516 Opcode = NVPTX::INT_PTX_LDU_G_v2i8_ELE_ari64;
1517 break;
1518 case MVT::i16:
1519 Opcode = NVPTX::INT_PTX_LDU_G_v2i16_ELE_ari64;
1520 break;
1521 case MVT::i32:
1522 Opcode = NVPTX::INT_PTX_LDU_G_v2i32_ELE_ari64;
1523 break;
1524 case MVT::i64:
1525 Opcode = NVPTX::INT_PTX_LDU_G_v2i64_ELE_ari64;
1526 break;
1527 case MVT::f32:
1528 Opcode = NVPTX::INT_PTX_LDU_G_v2f32_ELE_ari64;
1529 break;
1530 case MVT::f64:
1531 Opcode = NVPTX::INT_PTX_LDU_G_v2f64_ELE_ari64;
1532 break;
1533 }
1534 break;
1535 case NVPTXISD::LDGV4:
1536 switch (EltVT.getSimpleVT().SimpleTy) {
1537 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]1538 return nullptr;
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1539 case MVT::i8:
1540 Opcode = NVPTX::INT_PTX_LDG_G_v4i8_ELE_ari64;
1541 break;
1542 case MVT::i16:
1543 Opcode = NVPTX::INT_PTX_LDG_G_v4i16_ELE_ari64;
1544 break;
1545 case MVT::i32:
1546 Opcode = NVPTX::INT_PTX_LDG_G_v4i32_ELE_ari64;
1547 break;
1548 case MVT::f32:
1549 Opcode = NVPTX::INT_PTX_LDG_G_v4f32_ELE_ari64;
1550 break;
1551 }
1552 break;
1553 case NVPTXISD::LDUV4:
1554 switch (EltVT.getSimpleVT().SimpleTy) {
1555 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]1556 return nullptr;
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1557 case MVT::i8:
1558 Opcode = NVPTX::INT_PTX_LDU_G_v4i8_ELE_ari64;
1559 break;
1560 case MVT::i16:
1561 Opcode = NVPTX::INT_PTX_LDU_G_v4i16_ELE_ari64;
1562 break;
1563 case MVT::i32:
1564 Opcode = NVPTX::INT_PTX_LDU_G_v4i32_ELE_ari64;
1565 break;
1566 case MVT::f32:
1567 Opcode = NVPTX::INT_PTX_LDU_G_v4f32_ELE_ari64;
1568 break;
1569 }
1570 break;
1571 }
1572 } else {
1573 switch (N->getOpcode()) {
1574 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]1575 return nullptr;
Justin Holewinskib926d9d2014-06-27 18:35:51 +0000[diff] [blame]1576 case ISD::INTRINSIC_W_CHAIN:
1577 if (IsLDG) {
1578 switch (EltVT.getSimpleVT().SimpleTy) {
1579 default:
1580 return nullptr;
1581 case MVT::i8:
1582 Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i8ari;
1583 break;
1584 case MVT::i16:
1585 Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i16ari;
1586 break;
1587 case MVT::i32:
1588 Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i32ari;
1589 break;
1590 case MVT::i64:
1591 Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i64ari;
1592 break;
1593 case MVT::f32:
1594 Opcode = NVPTX::INT_PTX_LDG_GLOBAL_f32ari;
1595 break;
1596 case MVT::f64:
1597 Opcode = NVPTX::INT_PTX_LDG_GLOBAL_f64ari;
1598 break;
1599 }
1600 } else {
1601 switch (EltVT.getSimpleVT().SimpleTy) {
1602 default:
1603 return nullptr;
1604 case MVT::i8:
1605 Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i8ari;
1606 break;
1607 case MVT::i16:
1608 Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i16ari;
1609 break;
1610 case MVT::i32:
1611 Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i32ari;
1612 break;
1613 case MVT::i64:
1614 Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i64ari;
1615 break;
1616 case MVT::f32:
1617 Opcode = NVPTX::INT_PTX_LDU_GLOBAL_f32ari;
1618 break;
1619 case MVT::f64:
1620 Opcode = NVPTX::INT_PTX_LDU_GLOBAL_f64ari;
1621 break;
1622 }
1623 }
1624 break;
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1625 case NVPTXISD::LDGV2:
1626 switch (EltVT.getSimpleVT().SimpleTy) {
1627 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]1628 return nullptr;
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1629 case MVT::i8:
1630 Opcode = NVPTX::INT_PTX_LDG_G_v2i8_ELE_ari32;
1631 break;
1632 case MVT::i16:
1633 Opcode = NVPTX::INT_PTX_LDG_G_v2i16_ELE_ari32;
1634 break;
1635 case MVT::i32:
1636 Opcode = NVPTX::INT_PTX_LDG_G_v2i32_ELE_ari32;
1637 break;
1638 case MVT::i64:
1639 Opcode = NVPTX::INT_PTX_LDG_G_v2i64_ELE_ari32;
1640 break;
1641 case MVT::f32:
1642 Opcode = NVPTX::INT_PTX_LDG_G_v2f32_ELE_ari32;
1643 break;
1644 case MVT::f64:
1645 Opcode = NVPTX::INT_PTX_LDG_G_v2f64_ELE_ari32;
1646 break;
1647 }
1648 break;
1649 case NVPTXISD::LDUV2:
1650 switch (EltVT.getSimpleVT().SimpleTy) {
1651 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]1652 return nullptr;
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1653 case MVT::i8:
1654 Opcode = NVPTX::INT_PTX_LDU_G_v2i8_ELE_ari32;
1655 break;
1656 case MVT::i16:
1657 Opcode = NVPTX::INT_PTX_LDU_G_v2i16_ELE_ari32;
1658 break;
1659 case MVT::i32:
1660 Opcode = NVPTX::INT_PTX_LDU_G_v2i32_ELE_ari32;
1661 break;
1662 case MVT::i64:
1663 Opcode = NVPTX::INT_PTX_LDU_G_v2i64_ELE_ari32;
1664 break;
1665 case MVT::f32:
1666 Opcode = NVPTX::INT_PTX_LDU_G_v2f32_ELE_ari32;
1667 break;
1668 case MVT::f64:
1669 Opcode = NVPTX::INT_PTX_LDU_G_v2f64_ELE_ari32;
1670 break;
1671 }
1672 break;
1673 case NVPTXISD::LDGV4:
1674 switch (EltVT.getSimpleVT().SimpleTy) {
1675 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]1676 return nullptr;
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1677 case MVT::i8:
1678 Opcode = NVPTX::INT_PTX_LDG_G_v4i8_ELE_ari32;
1679 break;
1680 case MVT::i16:
1681 Opcode = NVPTX::INT_PTX_LDG_G_v4i16_ELE_ari32;
1682 break;
1683 case MVT::i32:
1684 Opcode = NVPTX::INT_PTX_LDG_G_v4i32_ELE_ari32;
1685 break;
1686 case MVT::f32:
1687 Opcode = NVPTX::INT_PTX_LDG_G_v4f32_ELE_ari32;
1688 break;
1689 }
1690 break;
1691 case NVPTXISD::LDUV4:
1692 switch (EltVT.getSimpleVT().SimpleTy) {
1693 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]1694 return nullptr;
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1695 case MVT::i8:
1696 Opcode = NVPTX::INT_PTX_LDU_G_v4i8_ELE_ari32;
1697 break;
1698 case MVT::i16:
1699 Opcode = NVPTX::INT_PTX_LDU_G_v4i16_ELE_ari32;
1700 break;
1701 case MVT::i32:
1702 Opcode = NVPTX::INT_PTX_LDU_G_v4i32_ELE_ari32;
1703 break;
1704 case MVT::f32:
1705 Opcode = NVPTX::INT_PTX_LDU_G_v4f32_ELE_ari32;
1706 break;
1707 }
1708 break;
1709 }
1710 }
1711
1712 SDValue Ops[] = { Base, Offset, Chain };
1713
Craig Topper2d2aa0c2014-04-30 07:17:30 +0000[diff] [blame]1714 LD = CurDAG->getMachineNode(Opcode, DL, N->getVTList(), Ops);
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]1715 } else {
Eric Christopher147bba22015-01-30 01:40:59 +0000[diff] [blame]1716 if (Subtarget->is64Bit()) {
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1717 switch (N->getOpcode()) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1718 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]1719 return nullptr;
Justin Holewinskib926d9d2014-06-27 18:35:51 +0000[diff] [blame]1720 case ISD::INTRINSIC_W_CHAIN:
1721 if (IsLDG) {
1722 switch (EltVT.getSimpleVT().SimpleTy) {
1723 default:
1724 return nullptr;
1725 case MVT::i8:
1726 Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i8areg64;
1727 break;
1728 case MVT::i16:
1729 Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i16areg64;
1730 break;
1731 case MVT::i32:
1732 Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i32areg64;
1733 break;
1734 case MVT::i64:
1735 Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i64areg64;
1736 break;
1737 case MVT::f32:
1738 Opcode = NVPTX::INT_PTX_LDG_GLOBAL_f32areg64;
1739 break;
1740 case MVT::f64:
1741 Opcode = NVPTX::INT_PTX_LDG_GLOBAL_f64areg64;
1742 break;
1743 }
1744 } else {
1745 switch (EltVT.getSimpleVT().SimpleTy) {
1746 default:
1747 return nullptr;
1748 case MVT::i8:
1749 Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i8areg64;
1750 break;
1751 case MVT::i16:
1752 Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i16areg64;
1753 break;
1754 case MVT::i32:
1755 Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i32areg64;
1756 break;
1757 case MVT::i64:
1758 Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i64areg64;
1759 break;
1760 case MVT::f32:
1761 Opcode = NVPTX::INT_PTX_LDU_GLOBAL_f32areg64;
1762 break;
1763 case MVT::f64:
1764 Opcode = NVPTX::INT_PTX_LDU_GLOBAL_f64areg64;
1765 break;
1766 }
1767 }
1768 break;
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1769 case NVPTXISD::LDGV2:
1770 switch (EltVT.getSimpleVT().SimpleTy) {
1771 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]1772 return nullptr;
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1773 case MVT::i8:
1774 Opcode = NVPTX::INT_PTX_LDG_G_v2i8_ELE_areg64;
1775 break;
1776 case MVT::i16:
1777 Opcode = NVPTX::INT_PTX_LDG_G_v2i16_ELE_areg64;
1778 break;
1779 case MVT::i32:
1780 Opcode = NVPTX::INT_PTX_LDG_G_v2i32_ELE_areg64;
1781 break;
1782 case MVT::i64:
1783 Opcode = NVPTX::INT_PTX_LDG_G_v2i64_ELE_areg64;
1784 break;
1785 case MVT::f32:
1786 Opcode = NVPTX::INT_PTX_LDG_G_v2f32_ELE_areg64;
1787 break;
1788 case MVT::f64:
1789 Opcode = NVPTX::INT_PTX_LDG_G_v2f64_ELE_areg64;
1790 break;
1791 }
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1792 break;
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1793 case NVPTXISD::LDUV2:
1794 switch (EltVT.getSimpleVT().SimpleTy) {
1795 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]1796 return nullptr;
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1797 case MVT::i8:
1798 Opcode = NVPTX::INT_PTX_LDU_G_v2i8_ELE_areg64;
1799 break;
1800 case MVT::i16:
1801 Opcode = NVPTX::INT_PTX_LDU_G_v2i16_ELE_areg64;
1802 break;
1803 case MVT::i32:
1804 Opcode = NVPTX::INT_PTX_LDU_G_v2i32_ELE_areg64;
1805 break;
1806 case MVT::i64:
1807 Opcode = NVPTX::INT_PTX_LDU_G_v2i64_ELE_areg64;
1808 break;
1809 case MVT::f32:
1810 Opcode = NVPTX::INT_PTX_LDU_G_v2f32_ELE_areg64;
1811 break;
1812 case MVT::f64:
1813 Opcode = NVPTX::INT_PTX_LDU_G_v2f64_ELE_areg64;
1814 break;
1815 }
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1816 break;
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1817 case NVPTXISD::LDGV4:
1818 switch (EltVT.getSimpleVT().SimpleTy) {
1819 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]1820 return nullptr;
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1821 case MVT::i8:
1822 Opcode = NVPTX::INT_PTX_LDG_G_v4i8_ELE_areg64;
1823 break;
1824 case MVT::i16:
1825 Opcode = NVPTX::INT_PTX_LDG_G_v4i16_ELE_areg64;
1826 break;
1827 case MVT::i32:
1828 Opcode = NVPTX::INT_PTX_LDG_G_v4i32_ELE_areg64;
1829 break;
1830 case MVT::f32:
1831 Opcode = NVPTX::INT_PTX_LDG_G_v4f32_ELE_areg64;
1832 break;
1833 }
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1834 break;
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1835 case NVPTXISD::LDUV4:
1836 switch (EltVT.getSimpleVT().SimpleTy) {
1837 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]1838 return nullptr;
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1839 case MVT::i8:
1840 Opcode = NVPTX::INT_PTX_LDU_G_v4i8_ELE_areg64;
1841 break;
1842 case MVT::i16:
1843 Opcode = NVPTX::INT_PTX_LDU_G_v4i16_ELE_areg64;
1844 break;
1845 case MVT::i32:
1846 Opcode = NVPTX::INT_PTX_LDU_G_v4i32_ELE_areg64;
1847 break;
1848 case MVT::f32:
1849 Opcode = NVPTX::INT_PTX_LDU_G_v4f32_ELE_areg64;
1850 break;
1851 }
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1852 break;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]1853 }
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1854 } else {
1855 switch (N->getOpcode()) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1856 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]1857 return nullptr;
Justin Holewinskib926d9d2014-06-27 18:35:51 +0000[diff] [blame]1858 case ISD::INTRINSIC_W_CHAIN:
1859 if (IsLDG) {
1860 switch (EltVT.getSimpleVT().SimpleTy) {
1861 default:
1862 return nullptr;
1863 case MVT::i8:
1864 Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i8areg;
1865 break;
1866 case MVT::i16:
1867 Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i16areg;
1868 break;
1869 case MVT::i32:
1870 Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i32areg;
1871 break;
1872 case MVT::i64:
1873 Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i64areg;
1874 break;
1875 case MVT::f32:
1876 Opcode = NVPTX::INT_PTX_LDG_GLOBAL_f32areg;
1877 break;
1878 case MVT::f64:
1879 Opcode = NVPTX::INT_PTX_LDG_GLOBAL_f64areg;
1880 break;
1881 }
1882 } else {
1883 switch (EltVT.getSimpleVT().SimpleTy) {
1884 default:
1885 return nullptr;
1886 case MVT::i8:
1887 Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i8areg;
1888 break;
1889 case MVT::i16:
1890 Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i16areg;
1891 break;
1892 case MVT::i32:
1893 Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i32areg;
1894 break;
1895 case MVT::i64:
1896 Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i64areg;
1897 break;
1898 case MVT::f32:
1899 Opcode = NVPTX::INT_PTX_LDU_GLOBAL_f32areg;
1900 break;
1901 case MVT::f64:
1902 Opcode = NVPTX::INT_PTX_LDU_GLOBAL_f64areg;
1903 break;
1904 }
1905 }
1906 break;
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1907 case NVPTXISD::LDGV2:
1908 switch (EltVT.getSimpleVT().SimpleTy) {
1909 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]1910 return nullptr;
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1911 case MVT::i8:
1912 Opcode = NVPTX::INT_PTX_LDG_G_v2i8_ELE_areg32;
1913 break;
1914 case MVT::i16:
1915 Opcode = NVPTX::INT_PTX_LDG_G_v2i16_ELE_areg32;
1916 break;
1917 case MVT::i32:
1918 Opcode = NVPTX::INT_PTX_LDG_G_v2i32_ELE_areg32;
1919 break;
1920 case MVT::i64:
1921 Opcode = NVPTX::INT_PTX_LDG_G_v2i64_ELE_areg32;
1922 break;
1923 case MVT::f32:
1924 Opcode = NVPTX::INT_PTX_LDG_G_v2f32_ELE_areg32;
1925 break;
1926 case MVT::f64:
1927 Opcode = NVPTX::INT_PTX_LDG_G_v2f64_ELE_areg32;
1928 break;
1929 }
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1930 break;
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1931 case NVPTXISD::LDUV2:
1932 switch (EltVT.getSimpleVT().SimpleTy) {
1933 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]1934 return nullptr;
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1935 case MVT::i8:
1936 Opcode = NVPTX::INT_PTX_LDU_G_v2i8_ELE_areg32;
1937 break;
1938 case MVT::i16:
1939 Opcode = NVPTX::INT_PTX_LDU_G_v2i16_ELE_areg32;
1940 break;
1941 case MVT::i32:
1942 Opcode = NVPTX::INT_PTX_LDU_G_v2i32_ELE_areg32;
1943 break;
1944 case MVT::i64:
1945 Opcode = NVPTX::INT_PTX_LDU_G_v2i64_ELE_areg32;
1946 break;
1947 case MVT::f32:
1948 Opcode = NVPTX::INT_PTX_LDU_G_v2f32_ELE_areg32;
1949 break;
1950 case MVT::f64:
1951 Opcode = NVPTX::INT_PTX_LDU_G_v2f64_ELE_areg32;
1952 break;
1953 }
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1954 break;
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1955 case NVPTXISD::LDGV4:
1956 switch (EltVT.getSimpleVT().SimpleTy) {
1957 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]1958 return nullptr;
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1959 case MVT::i8:
1960 Opcode = NVPTX::INT_PTX_LDG_G_v4i8_ELE_areg32;
1961 break;
1962 case MVT::i16:
1963 Opcode = NVPTX::INT_PTX_LDG_G_v4i16_ELE_areg32;
1964 break;
1965 case MVT::i32:
1966 Opcode = NVPTX::INT_PTX_LDG_G_v4i32_ELE_areg32;
1967 break;
1968 case MVT::f32:
1969 Opcode = NVPTX::INT_PTX_LDG_G_v4f32_ELE_areg32;
1970 break;
1971 }
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1972 break;
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1973 case NVPTXISD::LDUV4:
1974 switch (EltVT.getSimpleVT().SimpleTy) {
1975 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]1976 return nullptr;
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1977 case MVT::i8:
1978 Opcode = NVPTX::INT_PTX_LDU_G_v4i8_ELE_areg32;
1979 break;
1980 case MVT::i16:
1981 Opcode = NVPTX::INT_PTX_LDU_G_v4i16_ELE_areg32;
1982 break;
1983 case MVT::i32:
1984 Opcode = NVPTX::INT_PTX_LDU_G_v4i32_ELE_areg32;
1985 break;
1986 case MVT::f32:
1987 Opcode = NVPTX::INT_PTX_LDU_G_v4f32_ELE_areg32;
1988 break;
1989 }
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]1990 break;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]1991 }
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]1992 }
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]1993
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1994 SDValue Ops[] = { Op1, Chain };
Craig Topper2d2aa0c2014-04-30 07:17:30 +0000[diff] [blame]1995 LD = CurDAG->getMachineNode(Opcode, DL, N->getVTList(), Ops);
Justin Holewinskie40e9292013-07-01 12:58:52 +0000[diff] [blame]1996 }
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]1997
1998 MachineSDNode::mmo_iterator MemRefs0 = MF->allocateMemRefsArray(1);
Justin Holewinskib926d9d2014-06-27 18:35:51 +0000[diff] [blame]1999 MemRefs0[0] = Mem->getMemOperand();
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]2000 cast<MachineSDNode>(LD)->setMemRefs(MemRefs0, MemRefs0 + 1);
2001
2002 return LD;
2003}
2004
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2005SDNode *NVPTXDAGToDAGISel::SelectStore(SDNode *N) {
Andrew Trickef9de2a2013-05-25 02:42:55 +0000[diff] [blame]2006 SDLoc dl(N);
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]2007 StoreSDNode *ST = cast<StoreSDNode>(N);
2008 EVT StoreVT = ST->getMemoryVT();
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2009 SDNode *NVPTXST = nullptr;
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]2010
2011 // do not support pre/post inc/dec
2012 if (ST->isIndexed())
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2013 return nullptr;
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]2014
2015 if (!StoreVT.isSimple())
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2016 return nullptr;
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]2017
2018 // Address Space Setting
Eric Christopher9745b3a2015-01-30 01:41:01 +0000[diff] [blame^]2019 unsigned int codeAddrSpace = getCodeAddrSpace(ST);
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]2020
2021 // Volatile Setting
2022 // - .volatile is only availalble for .global and .shared
2023 bool isVolatile = ST->isVolatile();
2024 if (codeAddrSpace != NVPTX::PTXLdStInstCode::GLOBAL &&
2025 codeAddrSpace != NVPTX::PTXLdStInstCode::SHARED &&
2026 codeAddrSpace != NVPTX::PTXLdStInstCode::GENERIC)
2027 isVolatile = false;
2028
2029 // Vector Setting
2030 MVT SimpleVT = StoreVT.getSimpleVT();
2031 unsigned vecType = NVPTX::PTXLdStInstCode::Scalar;
2032 if (SimpleVT.isVector()) {
2033 unsigned num = SimpleVT.getVectorNumElements();
2034 if (num == 2)
2035 vecType = NVPTX::PTXLdStInstCode::V2;
2036 else if (num == 4)
2037 vecType = NVPTX::PTXLdStInstCode::V4;
2038 else
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2039 return nullptr;
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]2040 }
2041
2042 // Type Setting: toType + toTypeWidth
2043 // - for integer type, always use 'u'
2044 //
2045 MVT ScalarVT = SimpleVT.getScalarType();
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2046 unsigned toTypeWidth = ScalarVT.getSizeInBits();
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]2047 unsigned int toType;
2048 if (ScalarVT.isFloatingPoint())
2049 toType = NVPTX::PTXLdStInstCode::Float;
2050 else
2051 toType = NVPTX::PTXLdStInstCode::Unsigned;
2052
2053 // Create the machine instruction DAG
2054 SDValue Chain = N->getOperand(0);
2055 SDValue N1 = N->getOperand(1);
2056 SDValue N2 = N->getOperand(2);
2057 SDValue Addr;
2058 SDValue Offset, Base;
2059 unsigned Opcode;
Craig Topperd9c27832013-08-15 02:44:19 +0000[diff] [blame]2060 MVT::SimpleValueType SourceVT = N1.getNode()->getSimpleValueType(0).SimpleTy;
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]2061
2062 if (SelectDirectAddr(N2, Addr)) {
2063 switch (SourceVT) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2064 case MVT::i8:
2065 Opcode = NVPTX::ST_i8_avar;
2066 break;
2067 case MVT::i16:
2068 Opcode = NVPTX::ST_i16_avar;
2069 break;
2070 case MVT::i32:
2071 Opcode = NVPTX::ST_i32_avar;
2072 break;
2073 case MVT::i64:
2074 Opcode = NVPTX::ST_i64_avar;
2075 break;
2076 case MVT::f32:
2077 Opcode = NVPTX::ST_f32_avar;
2078 break;
2079 case MVT::f64:
2080 Opcode = NVPTX::ST_f64_avar;
2081 break;
2082 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2083 return nullptr;
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]2084 }
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2085 SDValue Ops[] = { N1, getI32Imm(isVolatile), getI32Imm(codeAddrSpace),
2086 getI32Imm(vecType), getI32Imm(toType),
2087 getI32Imm(toTypeWidth), Addr, Chain };
Michael Liaob53d8962013-04-19 22:22:57 +0000[diff] [blame]2088 NVPTXST = CurDAG->getMachineNode(Opcode, dl, MVT::Other, Ops);
Eric Christopher147bba22015-01-30 01:40:59 +0000[diff] [blame]2089 } else if (Subtarget->is64Bit()
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2090 ? SelectADDRsi64(N2.getNode(), N2, Base, Offset)
2091 : SelectADDRsi(N2.getNode(), N2, Base, Offset)) {
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]2092 switch (SourceVT) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2093 case MVT::i8:
2094 Opcode = NVPTX::ST_i8_asi;
2095 break;
2096 case MVT::i16:
2097 Opcode = NVPTX::ST_i16_asi;
2098 break;
2099 case MVT::i32:
2100 Opcode = NVPTX::ST_i32_asi;
2101 break;
2102 case MVT::i64:
2103 Opcode = NVPTX::ST_i64_asi;
2104 break;
2105 case MVT::f32:
2106 Opcode = NVPTX::ST_f32_asi;
2107 break;
2108 case MVT::f64:
2109 Opcode = NVPTX::ST_f64_asi;
2110 break;
2111 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2112 return nullptr;
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]2113 }
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2114 SDValue Ops[] = { N1, getI32Imm(isVolatile), getI32Imm(codeAddrSpace),
2115 getI32Imm(vecType), getI32Imm(toType),
2116 getI32Imm(toTypeWidth), Base, Offset, Chain };
Michael Liaob53d8962013-04-19 22:22:57 +0000[diff] [blame]2117 NVPTXST = CurDAG->getMachineNode(Opcode, dl, MVT::Other, Ops);
Eric Christopher147bba22015-01-30 01:40:59 +0000[diff] [blame]2118 } else if (Subtarget->is64Bit()
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2119 ? SelectADDRri64(N2.getNode(), N2, Base, Offset)
2120 : SelectADDRri(N2.getNode(), N2, Base, Offset)) {
Eric Christopher147bba22015-01-30 01:40:59 +0000[diff] [blame]2121 if (Subtarget->is64Bit()) {
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]2122 switch (SourceVT) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2123 case MVT::i8:
2124 Opcode = NVPTX::ST_i8_ari_64;
2125 break;
2126 case MVT::i16:
2127 Opcode = NVPTX::ST_i16_ari_64;
2128 break;
2129 case MVT::i32:
2130 Opcode = NVPTX::ST_i32_ari_64;
2131 break;
2132 case MVT::i64:
2133 Opcode = NVPTX::ST_i64_ari_64;
2134 break;
2135 case MVT::f32:
2136 Opcode = NVPTX::ST_f32_ari_64;
2137 break;
2138 case MVT::f64:
2139 Opcode = NVPTX::ST_f64_ari_64;
2140 break;
2141 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2142 return nullptr;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]2143 }
2144 } else {
2145 switch (SourceVT) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2146 case MVT::i8:
2147 Opcode = NVPTX::ST_i8_ari;
2148 break;
2149 case MVT::i16:
2150 Opcode = NVPTX::ST_i16_ari;
2151 break;
2152 case MVT::i32:
2153 Opcode = NVPTX::ST_i32_ari;
2154 break;
2155 case MVT::i64:
2156 Opcode = NVPTX::ST_i64_ari;
2157 break;
2158 case MVT::f32:
2159 Opcode = NVPTX::ST_f32_ari;
2160 break;
2161 case MVT::f64:
2162 Opcode = NVPTX::ST_f64_ari;
2163 break;
2164 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2165 return nullptr;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]2166 }
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]2167 }
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2168 SDValue Ops[] = { N1, getI32Imm(isVolatile), getI32Imm(codeAddrSpace),
2169 getI32Imm(vecType), getI32Imm(toType),
2170 getI32Imm(toTypeWidth), Base, Offset, Chain };
Michael Liaob53d8962013-04-19 22:22:57 +0000[diff] [blame]2171 NVPTXST = CurDAG->getMachineNode(Opcode, dl, MVT::Other, Ops);
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]2172 } else {
Eric Christopher147bba22015-01-30 01:40:59 +0000[diff] [blame]2173 if (Subtarget->is64Bit()) {
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]2174 switch (SourceVT) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2175 case MVT::i8:
2176 Opcode = NVPTX::ST_i8_areg_64;
2177 break;
2178 case MVT::i16:
2179 Opcode = NVPTX::ST_i16_areg_64;
2180 break;
2181 case MVT::i32:
2182 Opcode = NVPTX::ST_i32_areg_64;
2183 break;
2184 case MVT::i64:
2185 Opcode = NVPTX::ST_i64_areg_64;
2186 break;
2187 case MVT::f32:
2188 Opcode = NVPTX::ST_f32_areg_64;
2189 break;
2190 case MVT::f64:
2191 Opcode = NVPTX::ST_f64_areg_64;
2192 break;
2193 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2194 return nullptr;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]2195 }
2196 } else {
2197 switch (SourceVT) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2198 case MVT::i8:
2199 Opcode = NVPTX::ST_i8_areg;
2200 break;
2201 case MVT::i16:
2202 Opcode = NVPTX::ST_i16_areg;
2203 break;
2204 case MVT::i32:
2205 Opcode = NVPTX::ST_i32_areg;
2206 break;
2207 case MVT::i64:
2208 Opcode = NVPTX::ST_i64_areg;
2209 break;
2210 case MVT::f32:
2211 Opcode = NVPTX::ST_f32_areg;
2212 break;
2213 case MVT::f64:
2214 Opcode = NVPTX::ST_f64_areg;
2215 break;
2216 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2217 return nullptr;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]2218 }
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]2219 }
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2220 SDValue Ops[] = { N1, getI32Imm(isVolatile), getI32Imm(codeAddrSpace),
2221 getI32Imm(vecType), getI32Imm(toType),
2222 getI32Imm(toTypeWidth), N2, Chain };
Michael Liaob53d8962013-04-19 22:22:57 +0000[diff] [blame]2223 NVPTXST = CurDAG->getMachineNode(Opcode, dl, MVT::Other, Ops);
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]2224 }
2225
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2226 if (NVPTXST) {
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]2227 MachineSDNode::mmo_iterator MemRefs0 = MF->allocateMemRefsArray(1);
2228 MemRefs0[0] = cast<MemSDNode>(N)->getMemOperand();
2229 cast<MachineSDNode>(NVPTXST)->setMemRefs(MemRefs0, MemRefs0 + 1);
2230 }
2231
2232 return NVPTXST;
2233}
2234
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]2235SDNode *NVPTXDAGToDAGISel::SelectStoreVector(SDNode *N) {
2236 SDValue Chain = N->getOperand(0);
2237 SDValue Op1 = N->getOperand(1);
2238 SDValue Addr, Offset, Base;
2239 unsigned Opcode;
Andrew Trickef9de2a2013-05-25 02:42:55 +0000[diff] [blame]2240 SDLoc DL(N);
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]2241 SDNode *ST;
2242 EVT EltVT = Op1.getValueType();
2243 MemSDNode *MemSD = cast<MemSDNode>(N);
2244 EVT StoreVT = MemSD->getMemoryVT();
2245
2246 // Address Space Setting
Eric Christopher9745b3a2015-01-30 01:41:01 +0000[diff] [blame^]2247 unsigned CodeAddrSpace = getCodeAddrSpace(MemSD);
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]2248
2249 if (CodeAddrSpace == NVPTX::PTXLdStInstCode::CONSTANT) {
2250 report_fatal_error("Cannot store to pointer that points to constant "
2251 "memory space");
2252 }
2253
2254 // Volatile Setting
2255 // - .volatile is only availalble for .global and .shared
2256 bool IsVolatile = MemSD->isVolatile();
2257 if (CodeAddrSpace != NVPTX::PTXLdStInstCode::GLOBAL &&
2258 CodeAddrSpace != NVPTX::PTXLdStInstCode::SHARED &&
2259 CodeAddrSpace != NVPTX::PTXLdStInstCode::GENERIC)
2260 IsVolatile = false;
2261
2262 // Type Setting: toType + toTypeWidth
2263 // - for integer type, always use 'u'
2264 assert(StoreVT.isSimple() && "Store value is not simple");
2265 MVT ScalarVT = StoreVT.getSimpleVT().getScalarType();
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2266 unsigned ToTypeWidth = ScalarVT.getSizeInBits();
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]2267 unsigned ToType;
2268 if (ScalarVT.isFloatingPoint())
2269 ToType = NVPTX::PTXLdStInstCode::Float;
2270 else
2271 ToType = NVPTX::PTXLdStInstCode::Unsigned;
2272
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]2273 SmallVector<SDValue, 12> StOps;
2274 SDValue N2;
2275 unsigned VecType;
2276
2277 switch (N->getOpcode()) {
2278 case NVPTXISD::StoreV2:
2279 VecType = NVPTX::PTXLdStInstCode::V2;
2280 StOps.push_back(N->getOperand(1));
2281 StOps.push_back(N->getOperand(2));
2282 N2 = N->getOperand(3);
2283 break;
2284 case NVPTXISD::StoreV4:
2285 VecType = NVPTX::PTXLdStInstCode::V4;
2286 StOps.push_back(N->getOperand(1));
2287 StOps.push_back(N->getOperand(2));
2288 StOps.push_back(N->getOperand(3));
2289 StOps.push_back(N->getOperand(4));
2290 N2 = N->getOperand(5);
2291 break;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2292 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2293 return nullptr;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]2294 }
2295
2296 StOps.push_back(getI32Imm(IsVolatile));
2297 StOps.push_back(getI32Imm(CodeAddrSpace));
2298 StOps.push_back(getI32Imm(VecType));
2299 StOps.push_back(getI32Imm(ToType));
2300 StOps.push_back(getI32Imm(ToTypeWidth));
2301
2302 if (SelectDirectAddr(N2, Addr)) {
2303 switch (N->getOpcode()) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2304 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2305 return nullptr;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]2306 case NVPTXISD::StoreV2:
2307 switch (EltVT.getSimpleVT().SimpleTy) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2308 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2309 return nullptr;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2310 case MVT::i8:
2311 Opcode = NVPTX::STV_i8_v2_avar;
2312 break;
2313 case MVT::i16:
2314 Opcode = NVPTX::STV_i16_v2_avar;
2315 break;
2316 case MVT::i32:
2317 Opcode = NVPTX::STV_i32_v2_avar;
2318 break;
2319 case MVT::i64:
2320 Opcode = NVPTX::STV_i64_v2_avar;
2321 break;
2322 case MVT::f32:
2323 Opcode = NVPTX::STV_f32_v2_avar;
2324 break;
2325 case MVT::f64:
2326 Opcode = NVPTX::STV_f64_v2_avar;
2327 break;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]2328 }
2329 break;
2330 case NVPTXISD::StoreV4:
2331 switch (EltVT.getSimpleVT().SimpleTy) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2332 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2333 return nullptr;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2334 case MVT::i8:
2335 Opcode = NVPTX::STV_i8_v4_avar;
2336 break;
2337 case MVT::i16:
2338 Opcode = NVPTX::STV_i16_v4_avar;
2339 break;
2340 case MVT::i32:
2341 Opcode = NVPTX::STV_i32_v4_avar;
2342 break;
2343 case MVT::f32:
2344 Opcode = NVPTX::STV_f32_v4_avar;
2345 break;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]2346 }
2347 break;
2348 }
2349 StOps.push_back(Addr);
Eric Christopher147bba22015-01-30 01:40:59 +0000[diff] [blame]2350 } else if (Subtarget->is64Bit()
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2351 ? SelectADDRsi64(N2.getNode(), N2, Base, Offset)
2352 : SelectADDRsi(N2.getNode(), N2, Base, Offset)) {
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]2353 switch (N->getOpcode()) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2354 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2355 return nullptr;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]2356 case NVPTXISD::StoreV2:
2357 switch (EltVT.getSimpleVT().SimpleTy) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2358 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2359 return nullptr;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2360 case MVT::i8:
2361 Opcode = NVPTX::STV_i8_v2_asi;
2362 break;
2363 case MVT::i16:
2364 Opcode = NVPTX::STV_i16_v2_asi;
2365 break;
2366 case MVT::i32:
2367 Opcode = NVPTX::STV_i32_v2_asi;
2368 break;
2369 case MVT::i64:
2370 Opcode = NVPTX::STV_i64_v2_asi;
2371 break;
2372 case MVT::f32:
2373 Opcode = NVPTX::STV_f32_v2_asi;
2374 break;
2375 case MVT::f64:
2376 Opcode = NVPTX::STV_f64_v2_asi;
2377 break;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]2378 }
2379 break;
2380 case NVPTXISD::StoreV4:
2381 switch (EltVT.getSimpleVT().SimpleTy) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2382 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2383 return nullptr;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2384 case MVT::i8:
2385 Opcode = NVPTX::STV_i8_v4_asi;
2386 break;
2387 case MVT::i16:
2388 Opcode = NVPTX::STV_i16_v4_asi;
2389 break;
2390 case MVT::i32:
2391 Opcode = NVPTX::STV_i32_v4_asi;
2392 break;
2393 case MVT::f32:
2394 Opcode = NVPTX::STV_f32_v4_asi;
2395 break;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]2396 }
2397 break;
2398 }
2399 StOps.push_back(Base);
2400 StOps.push_back(Offset);
Eric Christopher147bba22015-01-30 01:40:59 +0000[diff] [blame]2401 } else if (Subtarget->is64Bit()
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2402 ? SelectADDRri64(N2.getNode(), N2, Base, Offset)
2403 : SelectADDRri(N2.getNode(), N2, Base, Offset)) {
Eric Christopher147bba22015-01-30 01:40:59 +0000[diff] [blame]2404 if (Subtarget->is64Bit()) {
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]2405 switch (N->getOpcode()) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2406 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2407 return nullptr;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]2408 case NVPTXISD::StoreV2:
2409 switch (EltVT.getSimpleVT().SimpleTy) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2410 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2411 return nullptr;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2412 case MVT::i8:
2413 Opcode = NVPTX::STV_i8_v2_ari_64;
2414 break;
2415 case MVT::i16:
2416 Opcode = NVPTX::STV_i16_v2_ari_64;
2417 break;
2418 case MVT::i32:
2419 Opcode = NVPTX::STV_i32_v2_ari_64;
2420 break;
2421 case MVT::i64:
2422 Opcode = NVPTX::STV_i64_v2_ari_64;
2423 break;
2424 case MVT::f32:
2425 Opcode = NVPTX::STV_f32_v2_ari_64;
2426 break;
2427 case MVT::f64:
2428 Opcode = NVPTX::STV_f64_v2_ari_64;
2429 break;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]2430 }
2431 break;
2432 case NVPTXISD::StoreV4:
2433 switch (EltVT.getSimpleVT().SimpleTy) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2434 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2435 return nullptr;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2436 case MVT::i8:
2437 Opcode = NVPTX::STV_i8_v4_ari_64;
2438 break;
2439 case MVT::i16:
2440 Opcode = NVPTX::STV_i16_v4_ari_64;
2441 break;
2442 case MVT::i32:
2443 Opcode = NVPTX::STV_i32_v4_ari_64;
2444 break;
2445 case MVT::f32:
2446 Opcode = NVPTX::STV_f32_v4_ari_64;
2447 break;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]2448 }
2449 break;
2450 }
2451 } else {
2452 switch (N->getOpcode()) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2453 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2454 return nullptr;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]2455 case NVPTXISD::StoreV2:
2456 switch (EltVT.getSimpleVT().SimpleTy) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2457 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2458 return nullptr;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2459 case MVT::i8:
2460 Opcode = NVPTX::STV_i8_v2_ari;
2461 break;
2462 case MVT::i16:
2463 Opcode = NVPTX::STV_i16_v2_ari;
2464 break;
2465 case MVT::i32:
2466 Opcode = NVPTX::STV_i32_v2_ari;
2467 break;
2468 case MVT::i64:
2469 Opcode = NVPTX::STV_i64_v2_ari;
2470 break;
2471 case MVT::f32:
2472 Opcode = NVPTX::STV_f32_v2_ari;
2473 break;
2474 case MVT::f64:
2475 Opcode = NVPTX::STV_f64_v2_ari;
2476 break;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]2477 }
2478 break;
2479 case NVPTXISD::StoreV4:
2480 switch (EltVT.getSimpleVT().SimpleTy) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2481 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2482 return nullptr;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2483 case MVT::i8:
2484 Opcode = NVPTX::STV_i8_v4_ari;
2485 break;
2486 case MVT::i16:
2487 Opcode = NVPTX::STV_i16_v4_ari;
2488 break;
2489 case MVT::i32:
2490 Opcode = NVPTX::STV_i32_v4_ari;
2491 break;
2492 case MVT::f32:
2493 Opcode = NVPTX::STV_f32_v4_ari;
2494 break;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]2495 }
2496 break;
2497 }
2498 }
2499 StOps.push_back(Base);
2500 StOps.push_back(Offset);
2501 } else {
Eric Christopher147bba22015-01-30 01:40:59 +0000[diff] [blame]2502 if (Subtarget->is64Bit()) {
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]2503 switch (N->getOpcode()) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2504 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2505 return nullptr;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]2506 case NVPTXISD::StoreV2:
2507 switch (EltVT.getSimpleVT().SimpleTy) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2508 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2509 return nullptr;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2510 case MVT::i8:
2511 Opcode = NVPTX::STV_i8_v2_areg_64;
2512 break;
2513 case MVT::i16:
2514 Opcode = NVPTX::STV_i16_v2_areg_64;
2515 break;
2516 case MVT::i32:
2517 Opcode = NVPTX::STV_i32_v2_areg_64;
2518 break;
2519 case MVT::i64:
2520 Opcode = NVPTX::STV_i64_v2_areg_64;
2521 break;
2522 case MVT::f32:
2523 Opcode = NVPTX::STV_f32_v2_areg_64;
2524 break;
2525 case MVT::f64:
2526 Opcode = NVPTX::STV_f64_v2_areg_64;
2527 break;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]2528 }
2529 break;
2530 case NVPTXISD::StoreV4:
2531 switch (EltVT.getSimpleVT().SimpleTy) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2532 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2533 return nullptr;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2534 case MVT::i8:
2535 Opcode = NVPTX::STV_i8_v4_areg_64;
2536 break;
2537 case MVT::i16:
2538 Opcode = NVPTX::STV_i16_v4_areg_64;
2539 break;
2540 case MVT::i32:
2541 Opcode = NVPTX::STV_i32_v4_areg_64;
2542 break;
2543 case MVT::f32:
2544 Opcode = NVPTX::STV_f32_v4_areg_64;
2545 break;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]2546 }
2547 break;
2548 }
2549 } else {
2550 switch (N->getOpcode()) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2551 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2552 return nullptr;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]2553 case NVPTXISD::StoreV2:
2554 switch (EltVT.getSimpleVT().SimpleTy) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2555 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2556 return nullptr;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2557 case MVT::i8:
2558 Opcode = NVPTX::STV_i8_v2_areg;
2559 break;
2560 case MVT::i16:
2561 Opcode = NVPTX::STV_i16_v2_areg;
2562 break;
2563 case MVT::i32:
2564 Opcode = NVPTX::STV_i32_v2_areg;
2565 break;
2566 case MVT::i64:
2567 Opcode = NVPTX::STV_i64_v2_areg;
2568 break;
2569 case MVT::f32:
2570 Opcode = NVPTX::STV_f32_v2_areg;
2571 break;
2572 case MVT::f64:
2573 Opcode = NVPTX::STV_f64_v2_areg;
2574 break;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]2575 }
2576 break;
2577 case NVPTXISD::StoreV4:
2578 switch (EltVT.getSimpleVT().SimpleTy) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2579 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2580 return nullptr;
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]2581 case MVT::i8:
2582 Opcode = NVPTX::STV_i8_v4_areg;
2583 break;
2584 case MVT::i16:
2585 Opcode = NVPTX::STV_i16_v4_areg;
2586 break;
2587 case MVT::i32:
2588 Opcode = NVPTX::STV_i32_v4_areg;
2589 break;
2590 case MVT::f32:
2591 Opcode = NVPTX::STV_f32_v4_areg;
2592 break;
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]2593 }
2594 break;
2595 }
2596 }
2597 StOps.push_back(N2);
2598 }
2599
2600 StOps.push_back(Chain);
2601
Michael Liaob53d8962013-04-19 22:22:57 +0000[diff] [blame]2602 ST = CurDAG->getMachineNode(Opcode, DL, MVT::Other, StOps);
Justin Holewinskibe8dc642013-02-12 14:18:49 +0000[diff] [blame]2603
2604 MachineSDNode::mmo_iterator MemRefs0 = MF->allocateMemRefsArray(1);
2605 MemRefs0[0] = cast<MemSDNode>(N)->getMemOperand();
2606 cast<MachineSDNode>(ST)->setMemRefs(MemRefs0, MemRefs0 + 1);
2607
2608 return ST;
2609}
2610
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]2611SDNode *NVPTXDAGToDAGISel::SelectLoadParam(SDNode *Node) {
2612 SDValue Chain = Node->getOperand(0);
2613 SDValue Offset = Node->getOperand(2);
2614 SDValue Flag = Node->getOperand(3);
2615 SDLoc DL(Node);
2616 MemSDNode *Mem = cast<MemSDNode>(Node);
2617
2618 unsigned VecSize;
2619 switch (Node->getOpcode()) {
2620 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2621 return nullptr;
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]2622 case NVPTXISD::LoadParam:
2623 VecSize = 1;
2624 break;
2625 case NVPTXISD::LoadParamV2:
2626 VecSize = 2;
2627 break;
2628 case NVPTXISD::LoadParamV4:
2629 VecSize = 4;
2630 break;
2631 }
2632
2633 EVT EltVT = Node->getValueType(0);
2634 EVT MemVT = Mem->getMemoryVT();
2635
2636 unsigned Opc = 0;
2637
2638 switch (VecSize) {
2639 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2640 return nullptr;
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]2641 case 1:
2642 switch (MemVT.getSimpleVT().SimpleTy) {
2643 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2644 return nullptr;
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]2645 case MVT::i1:
2646 Opc = NVPTX::LoadParamMemI8;
2647 break;
2648 case MVT::i8:
2649 Opc = NVPTX::LoadParamMemI8;
2650 break;
2651 case MVT::i16:
2652 Opc = NVPTX::LoadParamMemI16;
2653 break;
2654 case MVT::i32:
2655 Opc = NVPTX::LoadParamMemI32;
2656 break;
2657 case MVT::i64:
2658 Opc = NVPTX::LoadParamMemI64;
2659 break;
2660 case MVT::f32:
2661 Opc = NVPTX::LoadParamMemF32;
2662 break;
2663 case MVT::f64:
2664 Opc = NVPTX::LoadParamMemF64;
2665 break;
2666 }
2667 break;
2668 case 2:
2669 switch (MemVT.getSimpleVT().SimpleTy) {
2670 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2671 return nullptr;
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]2672 case MVT::i1:
2673 Opc = NVPTX::LoadParamMemV2I8;
2674 break;
2675 case MVT::i8:
2676 Opc = NVPTX::LoadParamMemV2I8;
2677 break;
2678 case MVT::i16:
2679 Opc = NVPTX::LoadParamMemV2I16;
2680 break;
2681 case MVT::i32:
2682 Opc = NVPTX::LoadParamMemV2I32;
2683 break;
2684 case MVT::i64:
2685 Opc = NVPTX::LoadParamMemV2I64;
2686 break;
2687 case MVT::f32:
2688 Opc = NVPTX::LoadParamMemV2F32;
2689 break;
2690 case MVT::f64:
2691 Opc = NVPTX::LoadParamMemV2F64;
2692 break;
2693 }
2694 break;
2695 case 4:
2696 switch (MemVT.getSimpleVT().SimpleTy) {
2697 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2698 return nullptr;
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]2699 case MVT::i1:
2700 Opc = NVPTX::LoadParamMemV4I8;
2701 break;
2702 case MVT::i8:
2703 Opc = NVPTX::LoadParamMemV4I8;
2704 break;
2705 case MVT::i16:
2706 Opc = NVPTX::LoadParamMemV4I16;
2707 break;
2708 case MVT::i32:
2709 Opc = NVPTX::LoadParamMemV4I32;
2710 break;
2711 case MVT::f32:
2712 Opc = NVPTX::LoadParamMemV4F32;
2713 break;
2714 }
2715 break;
2716 }
2717
2718 SDVTList VTs;
2719 if (VecSize == 1) {
2720 VTs = CurDAG->getVTList(EltVT, MVT::Other, MVT::Glue);
2721 } else if (VecSize == 2) {
2722 VTs = CurDAG->getVTList(EltVT, EltVT, MVT::Other, MVT::Glue);
2723 } else {
2724 EVT EVTs[] = { EltVT, EltVT, EltVT, EltVT, MVT::Other, MVT::Glue };
Craig Topperabb4ac72014-04-16 06:10:51 +0000[diff] [blame]2725 VTs = CurDAG->getVTList(EVTs);
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]2726 }
2727
2728 unsigned OffsetVal = cast<ConstantSDNode>(Offset)->getZExtValue();
2729
2730 SmallVector<SDValue, 2> Ops;
2731 Ops.push_back(CurDAG->getTargetConstant(OffsetVal, MVT::i32));
2732 Ops.push_back(Chain);
2733 Ops.push_back(Flag);
2734
2735 SDNode *Ret =
Justin Holewinskidff28d22013-07-01 12:59:01 +0000[diff] [blame]2736 CurDAG->getMachineNode(Opc, DL, VTs, Ops);
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]2737 return Ret;
2738}
2739
2740SDNode *NVPTXDAGToDAGISel::SelectStoreRetval(SDNode *N) {
2741 SDLoc DL(N);
2742 SDValue Chain = N->getOperand(0);
2743 SDValue Offset = N->getOperand(1);
2744 unsigned OffsetVal = cast<ConstantSDNode>(Offset)->getZExtValue();
2745 MemSDNode *Mem = cast<MemSDNode>(N);
2746
2747 // How many elements do we have?
2748 unsigned NumElts = 1;
2749 switch (N->getOpcode()) {
2750 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2751 return nullptr;
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]2752 case NVPTXISD::StoreRetval:
2753 NumElts = 1;
2754 break;
2755 case NVPTXISD::StoreRetvalV2:
2756 NumElts = 2;
2757 break;
2758 case NVPTXISD::StoreRetvalV4:
2759 NumElts = 4;
2760 break;
2761 }
2762
2763 // Build vector of operands
2764 SmallVector<SDValue, 6> Ops;
2765 for (unsigned i = 0; i < NumElts; ++i)
2766 Ops.push_back(N->getOperand(i + 2));
2767 Ops.push_back(CurDAG->getTargetConstant(OffsetVal, MVT::i32));
2768 Ops.push_back(Chain);
2769
2770 // Determine target opcode
2771 // If we have an i1, use an 8-bit store. The lowering code in
2772 // NVPTXISelLowering will have already emitted an upcast.
2773 unsigned Opcode = 0;
2774 switch (NumElts) {
2775 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2776 return nullptr;
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]2777 case 1:
2778 switch (Mem->getMemoryVT().getSimpleVT().SimpleTy) {
2779 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2780 return nullptr;
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]2781 case MVT::i1:
2782 Opcode = NVPTX::StoreRetvalI8;
2783 break;
2784 case MVT::i8:
2785 Opcode = NVPTX::StoreRetvalI8;
2786 break;
2787 case MVT::i16:
2788 Opcode = NVPTX::StoreRetvalI16;
2789 break;
2790 case MVT::i32:
2791 Opcode = NVPTX::StoreRetvalI32;
2792 break;
2793 case MVT::i64:
2794 Opcode = NVPTX::StoreRetvalI64;
2795 break;
2796 case MVT::f32:
2797 Opcode = NVPTX::StoreRetvalF32;
2798 break;
2799 case MVT::f64:
2800 Opcode = NVPTX::StoreRetvalF64;
2801 break;
2802 }
2803 break;
2804 case 2:
2805 switch (Mem->getMemoryVT().getSimpleVT().SimpleTy) {
2806 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2807 return nullptr;
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]2808 case MVT::i1:
2809 Opcode = NVPTX::StoreRetvalV2I8;
2810 break;
2811 case MVT::i8:
2812 Opcode = NVPTX::StoreRetvalV2I8;
2813 break;
2814 case MVT::i16:
2815 Opcode = NVPTX::StoreRetvalV2I16;
2816 break;
2817 case MVT::i32:
2818 Opcode = NVPTX::StoreRetvalV2I32;
2819 break;
2820 case MVT::i64:
2821 Opcode = NVPTX::StoreRetvalV2I64;
2822 break;
2823 case MVT::f32:
2824 Opcode = NVPTX::StoreRetvalV2F32;
2825 break;
2826 case MVT::f64:
2827 Opcode = NVPTX::StoreRetvalV2F64;
2828 break;
2829 }
2830 break;
2831 case 4:
2832 switch (Mem->getMemoryVT().getSimpleVT().SimpleTy) {
2833 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2834 return nullptr;
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]2835 case MVT::i1:
2836 Opcode = NVPTX::StoreRetvalV4I8;
2837 break;
2838 case MVT::i8:
2839 Opcode = NVPTX::StoreRetvalV4I8;
2840 break;
2841 case MVT::i16:
2842 Opcode = NVPTX::StoreRetvalV4I16;
2843 break;
2844 case MVT::i32:
2845 Opcode = NVPTX::StoreRetvalV4I32;
2846 break;
2847 case MVT::f32:
2848 Opcode = NVPTX::StoreRetvalV4F32;
2849 break;
2850 }
2851 break;
2852 }
2853
2854 SDNode *Ret =
2855 CurDAG->getMachineNode(Opcode, DL, MVT::Other, Ops);
2856 MachineSDNode::mmo_iterator MemRefs0 = MF->allocateMemRefsArray(1);
2857 MemRefs0[0] = cast<MemSDNode>(N)->getMemOperand();
2858 cast<MachineSDNode>(Ret)->setMemRefs(MemRefs0, MemRefs0 + 1);
2859
2860 return Ret;
2861}
2862
2863SDNode *NVPTXDAGToDAGISel::SelectStoreParam(SDNode *N) {
2864 SDLoc DL(N);
2865 SDValue Chain = N->getOperand(0);
2866 SDValue Param = N->getOperand(1);
2867 unsigned ParamVal = cast<ConstantSDNode>(Param)->getZExtValue();
2868 SDValue Offset = N->getOperand(2);
2869 unsigned OffsetVal = cast<ConstantSDNode>(Offset)->getZExtValue();
2870 MemSDNode *Mem = cast<MemSDNode>(N);
2871 SDValue Flag = N->getOperand(N->getNumOperands() - 1);
2872
2873 // How many elements do we have?
2874 unsigned NumElts = 1;
2875 switch (N->getOpcode()) {
2876 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2877 return nullptr;
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]2878 case NVPTXISD::StoreParamU32:
2879 case NVPTXISD::StoreParamS32:
2880 case NVPTXISD::StoreParam:
2881 NumElts = 1;
2882 break;
2883 case NVPTXISD::StoreParamV2:
2884 NumElts = 2;
2885 break;
2886 case NVPTXISD::StoreParamV4:
2887 NumElts = 4;
2888 break;
2889 }
2890
2891 // Build vector of operands
2892 SmallVector<SDValue, 8> Ops;
2893 for (unsigned i = 0; i < NumElts; ++i)
2894 Ops.push_back(N->getOperand(i + 3));
2895 Ops.push_back(CurDAG->getTargetConstant(ParamVal, MVT::i32));
2896 Ops.push_back(CurDAG->getTargetConstant(OffsetVal, MVT::i32));
2897 Ops.push_back(Chain);
2898 Ops.push_back(Flag);
2899
2900 // Determine target opcode
2901 // If we have an i1, use an 8-bit store. The lowering code in
2902 // NVPTXISelLowering will have already emitted an upcast.
2903 unsigned Opcode = 0;
2904 switch (N->getOpcode()) {
2905 default:
2906 switch (NumElts) {
2907 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2908 return nullptr;
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]2909 case 1:
2910 switch (Mem->getMemoryVT().getSimpleVT().SimpleTy) {
2911 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2912 return nullptr;
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]2913 case MVT::i1:
2914 Opcode = NVPTX::StoreParamI8;
2915 break;
2916 case MVT::i8:
2917 Opcode = NVPTX::StoreParamI8;
2918 break;
2919 case MVT::i16:
2920 Opcode = NVPTX::StoreParamI16;
2921 break;
2922 case MVT::i32:
2923 Opcode = NVPTX::StoreParamI32;
2924 break;
2925 case MVT::i64:
2926 Opcode = NVPTX::StoreParamI64;
2927 break;
2928 case MVT::f32:
2929 Opcode = NVPTX::StoreParamF32;
2930 break;
2931 case MVT::f64:
2932 Opcode = NVPTX::StoreParamF64;
2933 break;
2934 }
2935 break;
2936 case 2:
2937 switch (Mem->getMemoryVT().getSimpleVT().SimpleTy) {
2938 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2939 return nullptr;
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]2940 case MVT::i1:
2941 Opcode = NVPTX::StoreParamV2I8;
2942 break;
2943 case MVT::i8:
2944 Opcode = NVPTX::StoreParamV2I8;
2945 break;
2946 case MVT::i16:
2947 Opcode = NVPTX::StoreParamV2I16;
2948 break;
2949 case MVT::i32:
2950 Opcode = NVPTX::StoreParamV2I32;
2951 break;
2952 case MVT::i64:
2953 Opcode = NVPTX::StoreParamV2I64;
2954 break;
2955 case MVT::f32:
2956 Opcode = NVPTX::StoreParamV2F32;
2957 break;
2958 case MVT::f64:
2959 Opcode = NVPTX::StoreParamV2F64;
2960 break;
2961 }
2962 break;
2963 case 4:
2964 switch (Mem->getMemoryVT().getSimpleVT().SimpleTy) {
2965 default:
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]2966 return nullptr;
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]2967 case MVT::i1:
2968 Opcode = NVPTX::StoreParamV4I8;
2969 break;
2970 case MVT::i8:
2971 Opcode = NVPTX::StoreParamV4I8;
2972 break;
2973 case MVT::i16:
2974 Opcode = NVPTX::StoreParamV4I16;
2975 break;
2976 case MVT::i32:
2977 Opcode = NVPTX::StoreParamV4I32;
2978 break;
2979 case MVT::f32:
2980 Opcode = NVPTX::StoreParamV4F32;
2981 break;
2982 }
2983 break;
2984 }
2985 break;
Justin Holewinskidc5e3b62013-06-28 17:58:04 +0000[diff] [blame]2986 // Special case: if we have a sign-extend/zero-extend node, insert the
2987 // conversion instruction first, and use that as the value operand to
2988 // the selected StoreParam node.
2989 case NVPTXISD::StoreParamU32: {
2990 Opcode = NVPTX::StoreParamI32;
2991 SDValue CvtNone = CurDAG->getTargetConstant(NVPTX::PTXCvtMode::NONE,
2992 MVT::i32);
2993 SDNode *Cvt = CurDAG->getMachineNode(NVPTX::CVT_u32_u16, DL,
2994 MVT::i32, Ops[0], CvtNone);
2995 Ops[0] = SDValue(Cvt, 0);
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]2996 break;
Justin Holewinskidc5e3b62013-06-28 17:58:04 +0000[diff] [blame]2997 }
2998 case NVPTXISD::StoreParamS32: {
2999 Opcode = NVPTX::StoreParamI32;
3000 SDValue CvtNone = CurDAG->getTargetConstant(NVPTX::PTXCvtMode::NONE,
3001 MVT::i32);
3002 SDNode *Cvt = CurDAG->getMachineNode(NVPTX::CVT_s32_s16, DL,
3003 MVT::i32, Ops[0], CvtNone);
3004 Ops[0] = SDValue(Cvt, 0);
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]3005 break;
3006 }
Justin Holewinskidc5e3b62013-06-28 17:58:04 +0000[diff] [blame]3007 }
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]3008
Justin Holewinskidff28d22013-07-01 12:59:01 +0000[diff] [blame]3009 SDVTList RetVTs = CurDAG->getVTList(MVT::Other, MVT::Glue);
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]3010 SDNode *Ret =
Justin Holewinskidff28d22013-07-01 12:59:01 +0000[diff] [blame]3011 CurDAG->getMachineNode(Opcode, DL, RetVTs, Ops);
Justin Holewinskif8f70912013-06-28 17:57:59 +0000[diff] [blame]3012 MachineSDNode::mmo_iterator MemRefs0 = MF->allocateMemRefsArray(1);
3013 MemRefs0[0] = cast<MemSDNode>(N)->getMemOperand();
3014 cast<MachineSDNode>(Ret)->setMemRefs(MemRefs0, MemRefs0 + 1);
3015
3016 return Ret;
3017}
3018
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]3019SDNode *NVPTXDAGToDAGISel::SelectTextureIntrinsic(SDNode *N) {
3020 SDValue Chain = N->getOperand(0);
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]3021 SDNode *Ret = nullptr;
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]3022 unsigned Opc = 0;
3023 SmallVector<SDValue, 8> Ops;
3024
3025 switch (N->getOpcode()) {
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]3026 default: return nullptr;
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]3027 case NVPTXISD::Tex1DFloatS32:
3028 Opc = NVPTX::TEX_1D_F32_S32;
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]3029 break;
3030 case NVPTXISD::Tex1DFloatFloat:
3031 Opc = NVPTX::TEX_1D_F32_F32;
3032 break;
3033 case NVPTXISD::Tex1DFloatFloatLevel:
3034 Opc = NVPTX::TEX_1D_F32_F32_LEVEL;
3035 break;
3036 case NVPTXISD::Tex1DFloatFloatGrad:
3037 Opc = NVPTX::TEX_1D_F32_F32_GRAD;
3038 break;
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]3039 case NVPTXISD::Tex1DS32S32:
3040 Opc = NVPTX::TEX_1D_S32_S32;
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]3041 break;
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]3042 case NVPTXISD::Tex1DS32Float:
3043 Opc = NVPTX::TEX_1D_S32_F32;
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]3044 break;
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]3045 case NVPTXISD::Tex1DS32FloatLevel:
3046 Opc = NVPTX::TEX_1D_S32_F32_LEVEL;
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]3047 break;
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]3048 case NVPTXISD::Tex1DS32FloatGrad:
3049 Opc = NVPTX::TEX_1D_S32_F32_GRAD;
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]3050 break;
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]3051 case NVPTXISD::Tex1DU32S32:
3052 Opc = NVPTX::TEX_1D_U32_S32;
3053 break;
3054 case NVPTXISD::Tex1DU32Float:
3055 Opc = NVPTX::TEX_1D_U32_F32;
3056 break;
3057 case NVPTXISD::Tex1DU32FloatLevel:
3058 Opc = NVPTX::TEX_1D_U32_F32_LEVEL;
3059 break;
3060 case NVPTXISD::Tex1DU32FloatGrad:
3061 Opc = NVPTX::TEX_1D_U32_F32_GRAD;
3062 break;
3063 case NVPTXISD::Tex1DArrayFloatS32:
3064 Opc = NVPTX::TEX_1D_ARRAY_F32_S32;
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]3065 break;
3066 case NVPTXISD::Tex1DArrayFloatFloat:
3067 Opc = NVPTX::TEX_1D_ARRAY_F32_F32;
3068 break;
3069 case NVPTXISD::Tex1DArrayFloatFloatLevel:
3070 Opc = NVPTX::TEX_1D_ARRAY_F32_F32_LEVEL;
3071 break;
3072 case NVPTXISD::Tex1DArrayFloatFloatGrad:
3073 Opc = NVPTX::TEX_1D_ARRAY_F32_F32_GRAD;
3074 break;
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]3075 case NVPTXISD::Tex1DArrayS32S32:
3076 Opc = NVPTX::TEX_1D_ARRAY_S32_S32;
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]3077 break;
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]3078 case NVPTXISD::Tex1DArrayS32Float:
3079 Opc = NVPTX::TEX_1D_ARRAY_S32_F32;
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]3080 break;
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]3081 case NVPTXISD::Tex1DArrayS32FloatLevel:
3082 Opc = NVPTX::TEX_1D_ARRAY_S32_F32_LEVEL;
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]3083 break;
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]3084 case NVPTXISD::Tex1DArrayS32FloatGrad:
3085 Opc = NVPTX::TEX_1D_ARRAY_S32_F32_GRAD;
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]3086 break;
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]3087 case NVPTXISD::Tex1DArrayU32S32:
3088 Opc = NVPTX::TEX_1D_ARRAY_U32_S32;
3089 break;
3090 case NVPTXISD::Tex1DArrayU32Float:
3091 Opc = NVPTX::TEX_1D_ARRAY_U32_F32;
3092 break;
3093 case NVPTXISD::Tex1DArrayU32FloatLevel:
3094 Opc = NVPTX::TEX_1D_ARRAY_U32_F32_LEVEL;
3095 break;
3096 case NVPTXISD::Tex1DArrayU32FloatGrad:
3097 Opc = NVPTX::TEX_1D_ARRAY_U32_F32_GRAD;
3098 break;
3099 case NVPTXISD::Tex2DFloatS32:
3100 Opc = NVPTX::TEX_2D_F32_S32;
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]3101 break;
3102 case NVPTXISD::Tex2DFloatFloat:
3103 Opc = NVPTX::TEX_2D_F32_F32;
3104 break;
3105 case NVPTXISD::Tex2DFloatFloatLevel:
3106 Opc = NVPTX::TEX_2D_F32_F32_LEVEL;
3107 break;
3108 case NVPTXISD::Tex2DFloatFloatGrad:
3109 Opc = NVPTX::TEX_2D_F32_F32_GRAD;
3110 break;
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]3111 case NVPTXISD::Tex2DS32S32:
3112 Opc = NVPTX::TEX_2D_S32_S32;
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]3113 break;
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]3114 case NVPTXISD::Tex2DS32Float:
3115 Opc = NVPTX::TEX_2D_S32_F32;
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]3116 break;
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]3117 case NVPTXISD::Tex2DS32FloatLevel:
3118 Opc = NVPTX::TEX_2D_S32_F32_LEVEL;
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]3119 break;
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]3120 case NVPTXISD::Tex2DS32FloatGrad:
3121 Opc = NVPTX::TEX_2D_S32_F32_GRAD;
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]3122 break;
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]3123 case NVPTXISD::Tex2DU32S32:
3124 Opc = NVPTX::TEX_2D_U32_S32;
3125 break;
3126 case NVPTXISD::Tex2DU32Float:
3127 Opc = NVPTX::TEX_2D_U32_F32;
3128 break;
3129 case NVPTXISD::Tex2DU32FloatLevel:
3130 Opc = NVPTX::TEX_2D_U32_F32_LEVEL;
3131 break;
3132 case NVPTXISD::Tex2DU32FloatGrad:
3133 Opc = NVPTX::TEX_2D_U32_F32_GRAD;
3134 break;
3135 case NVPTXISD::Tex2DArrayFloatS32:
3136 Opc = NVPTX::TEX_2D_ARRAY_F32_S32;
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]3137 break;
3138 case NVPTXISD::Tex2DArrayFloatFloat:
3139 Opc = NVPTX::TEX_2D_ARRAY_F32_F32;
3140 break;
3141 case NVPTXISD::Tex2DArrayFloatFloatLevel:
3142 Opc = NVPTX::TEX_2D_ARRAY_F32_F32_LEVEL;
3143 break;
3144 case NVPTXISD::Tex2DArrayFloatFloatGrad:
3145 Opc = NVPTX::TEX_2D_ARRAY_F32_F32_GRAD;
3146 break;
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]3147 case NVPTXISD::Tex2DArrayS32S32:
3148 Opc = NVPTX::TEX_2D_ARRAY_S32_S32;
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]3149 break;
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]3150 case NVPTXISD::Tex2DArrayS32Float:
3151 Opc = NVPTX::TEX_2D_ARRAY_S32_F32;
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]3152 break;
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]3153 case NVPTXISD::Tex2DArrayS32FloatLevel:
3154 Opc = NVPTX::TEX_2D_ARRAY_S32_F32_LEVEL;
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]3155 break;
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]3156 case NVPTXISD::Tex2DArrayS32FloatGrad:
3157 Opc = NVPTX::TEX_2D_ARRAY_S32_F32_GRAD;
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]3158 break;
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]3159 case NVPTXISD::Tex2DArrayU32S32:
3160 Opc = NVPTX::TEX_2D_ARRAY_U32_S32;
3161 break;
3162 case NVPTXISD::Tex2DArrayU32Float:
3163 Opc = NVPTX::TEX_2D_ARRAY_U32_F32;
3164 break;
3165 case NVPTXISD::Tex2DArrayU32FloatLevel:
3166 Opc = NVPTX::TEX_2D_ARRAY_U32_F32_LEVEL;
3167 break;
3168 case NVPTXISD::Tex2DArrayU32FloatGrad:
3169 Opc = NVPTX::TEX_2D_ARRAY_U32_F32_GRAD;
3170 break;
3171 case NVPTXISD::Tex3DFloatS32:
3172 Opc = NVPTX::TEX_3D_F32_S32;
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]3173 break;
3174 case NVPTXISD::Tex3DFloatFloat:
3175 Opc = NVPTX::TEX_3D_F32_F32;
3176 break;
3177 case NVPTXISD::Tex3DFloatFloatLevel:
3178 Opc = NVPTX::TEX_3D_F32_F32_LEVEL;
3179 break;
3180 case NVPTXISD::Tex3DFloatFloatGrad:
3181 Opc = NVPTX::TEX_3D_F32_F32_GRAD;
3182 break;
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]3183 case NVPTXISD::Tex3DS32S32:
3184 Opc = NVPTX::TEX_3D_S32_S32;
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]3185 break;
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]3186 case NVPTXISD::Tex3DS32Float:
3187 Opc = NVPTX::TEX_3D_S32_F32;
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]3188 break;
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]3189 case NVPTXISD::Tex3DS32FloatLevel:
3190 Opc = NVPTX::TEX_3D_S32_F32_LEVEL;
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]3191 break;
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]3192 case NVPTXISD::Tex3DS32FloatGrad:
3193 Opc = NVPTX::TEX_3D_S32_F32_GRAD;
3194 break;
3195 case NVPTXISD::Tex3DU32S32:
3196 Opc = NVPTX::TEX_3D_U32_S32;
3197 break;
3198 case NVPTXISD::Tex3DU32Float:
3199 Opc = NVPTX::TEX_3D_U32_F32;
3200 break;
3201 case NVPTXISD::Tex3DU32FloatLevel:
3202 Opc = NVPTX::TEX_3D_U32_F32_LEVEL;
3203 break;
3204 case NVPTXISD::Tex3DU32FloatGrad:
3205 Opc = NVPTX::TEX_3D_U32_F32_GRAD;
3206 break;
3207 case NVPTXISD::TexCubeFloatFloat:
3208 Opc = NVPTX::TEX_CUBE_F32_F32;
3209 break;
3210 case NVPTXISD::TexCubeFloatFloatLevel:
3211 Opc = NVPTX::TEX_CUBE_F32_F32_LEVEL;
3212 break;
3213 case NVPTXISD::TexCubeS32Float:
3214 Opc = NVPTX::TEX_CUBE_S32_F32;
3215 break;
3216 case NVPTXISD::TexCubeS32FloatLevel:
3217 Opc = NVPTX::TEX_CUBE_S32_F32_LEVEL;
3218 break;
3219 case NVPTXISD::TexCubeU32Float:
3220 Opc = NVPTX::TEX_CUBE_U32_F32;
3221 break;
3222 case NVPTXISD::TexCubeU32FloatLevel:
3223 Opc = NVPTX::TEX_CUBE_U32_F32_LEVEL;
3224 break;
3225 case NVPTXISD::TexCubeArrayFloatFloat:
3226 Opc = NVPTX::TEX_CUBE_ARRAY_F32_F32;
3227 break;
3228 case NVPTXISD::TexCubeArrayFloatFloatLevel:
3229 Opc = NVPTX::TEX_CUBE_ARRAY_F32_F32_LEVEL;
3230 break;
3231 case NVPTXISD::TexCubeArrayS32Float:
3232 Opc = NVPTX::TEX_CUBE_ARRAY_S32_F32;
3233 break;
3234 case NVPTXISD::TexCubeArrayS32FloatLevel:
3235 Opc = NVPTX::TEX_CUBE_ARRAY_S32_F32_LEVEL;
3236 break;
3237 case NVPTXISD::TexCubeArrayU32Float:
3238 Opc = NVPTX::TEX_CUBE_ARRAY_U32_F32;
3239 break;
3240 case NVPTXISD::TexCubeArrayU32FloatLevel:
3241 Opc = NVPTX::TEX_CUBE_ARRAY_U32_F32_LEVEL;
3242 break;
3243 case NVPTXISD::Tld4R2DFloatFloat:
3244 Opc = NVPTX::TLD4_R_2D_F32_F32;
3245 break;
3246 case NVPTXISD::Tld4G2DFloatFloat:
3247 Opc = NVPTX::TLD4_G_2D_F32_F32;
3248 break;
3249 case NVPTXISD::Tld4B2DFloatFloat:
3250 Opc = NVPTX::TLD4_B_2D_F32_F32;
3251 break;
3252 case NVPTXISD::Tld4A2DFloatFloat:
3253 Opc = NVPTX::TLD4_A_2D_F32_F32;
3254 break;
3255 case NVPTXISD::Tld4R2DS64Float:
3256 Opc = NVPTX::TLD4_R_2D_S32_F32;
3257 break;
3258 case NVPTXISD::Tld4G2DS64Float:
3259 Opc = NVPTX::TLD4_G_2D_S32_F32;
3260 break;
3261 case NVPTXISD::Tld4B2DS64Float:
3262 Opc = NVPTX::TLD4_B_2D_S32_F32;
3263 break;
3264 case NVPTXISD::Tld4A2DS64Float:
3265 Opc = NVPTX::TLD4_A_2D_S32_F32;
3266 break;
3267 case NVPTXISD::Tld4R2DU64Float:
3268 Opc = NVPTX::TLD4_R_2D_U32_F32;
3269 break;
3270 case NVPTXISD::Tld4G2DU64Float:
3271 Opc = NVPTX::TLD4_G_2D_U32_F32;
3272 break;
3273 case NVPTXISD::Tld4B2DU64Float:
3274 Opc = NVPTX::TLD4_B_2D_U32_F32;
3275 break;
3276 case NVPTXISD::Tld4A2DU64Float:
3277 Opc = NVPTX::TLD4_A_2D_U32_F32;
3278 break;
3279 case NVPTXISD::TexUnified1DFloatS32:
3280 Opc = NVPTX::TEX_UNIFIED_1D_F32_S32;
3281 break;
3282 case NVPTXISD::TexUnified1DFloatFloat:
3283 Opc = NVPTX::TEX_UNIFIED_1D_F32_F32;
3284 break;
3285 case NVPTXISD::TexUnified1DFloatFloatLevel:
3286 Opc = NVPTX::TEX_UNIFIED_1D_F32_F32_LEVEL;
3287 break;
3288 case NVPTXISD::TexUnified1DFloatFloatGrad:
3289 Opc = NVPTX::TEX_UNIFIED_1D_F32_F32_GRAD;
3290 break;
3291 case NVPTXISD::TexUnified1DS32S32:
3292 Opc = NVPTX::TEX_UNIFIED_1D_S32_S32;
3293 break;
3294 case NVPTXISD::TexUnified1DS32Float:
3295 Opc = NVPTX::TEX_UNIFIED_1D_S32_F32;
3296 break;
3297 case NVPTXISD::TexUnified1DS32FloatLevel:
3298 Opc = NVPTX::TEX_UNIFIED_1D_S32_F32_LEVEL;
3299 break;
3300 case NVPTXISD::TexUnified1DS32FloatGrad:
3301 Opc = NVPTX::TEX_UNIFIED_1D_S32_F32_GRAD;
3302 break;
3303 case NVPTXISD::TexUnified1DU32S32:
3304 Opc = NVPTX::TEX_UNIFIED_1D_U32_S32;
3305 break;
3306 case NVPTXISD::TexUnified1DU32Float:
3307 Opc = NVPTX::TEX_UNIFIED_1D_U32_F32;
3308 break;
3309 case NVPTXISD::TexUnified1DU32FloatLevel:
3310 Opc = NVPTX::TEX_UNIFIED_1D_U32_F32_LEVEL;
3311 break;
3312 case NVPTXISD::TexUnified1DU32FloatGrad:
3313 Opc = NVPTX::TEX_UNIFIED_1D_U32_F32_GRAD;
3314 break;
3315 case NVPTXISD::TexUnified1DArrayFloatS32:
3316 Opc = NVPTX::TEX_UNIFIED_1D_ARRAY_F32_S32;
3317 break;
3318 case NVPTXISD::TexUnified1DArrayFloatFloat:
3319 Opc = NVPTX::TEX_UNIFIED_1D_ARRAY_F32_F32;
3320 break;
3321 case NVPTXISD::TexUnified1DArrayFloatFloatLevel:
3322 Opc = NVPTX::TEX_UNIFIED_1D_ARRAY_F32_F32_LEVEL;
3323 break;
3324 case NVPTXISD::TexUnified1DArrayFloatFloatGrad:
3325 Opc = NVPTX::TEX_UNIFIED_1D_ARRAY_F32_F32_GRAD;
3326 break;
3327 case NVPTXISD::TexUnified1DArrayS32S32:
3328 Opc = NVPTX::TEX_UNIFIED_1D_ARRAY_S32_S32;
3329 break;
3330 case NVPTXISD::TexUnified1DArrayS32Float:
3331 Opc = NVPTX::TEX_UNIFIED_1D_ARRAY_S32_F32;
3332 break;
3333 case NVPTXISD::TexUnified1DArrayS32FloatLevel:
3334 Opc = NVPTX::TEX_UNIFIED_1D_ARRAY_S32_F32_LEVEL;
3335 break;
3336 case NVPTXISD::TexUnified1DArrayS32FloatGrad:
3337 Opc = NVPTX::TEX_UNIFIED_1D_ARRAY_S32_F32_GRAD;
3338 break;
3339 case NVPTXISD::TexUnified1DArrayU32S32:
3340 Opc = NVPTX::TEX_UNIFIED_1D_ARRAY_U32_S32;
3341 break;
3342 case NVPTXISD::TexUnified1DArrayU32Float:
3343 Opc = NVPTX::TEX_UNIFIED_1D_ARRAY_U32_F32;
3344 break;
3345 case NVPTXISD::TexUnified1DArrayU32FloatLevel:
3346 Opc = NVPTX::TEX_UNIFIED_1D_ARRAY_U32_F32_LEVEL;
3347 break;
3348 case NVPTXISD::TexUnified1DArrayU32FloatGrad:
3349 Opc = NVPTX::TEX_UNIFIED_1D_ARRAY_U32_F32_GRAD;
3350 break;
3351 case NVPTXISD::TexUnified2DFloatS32:
3352 Opc = NVPTX::TEX_UNIFIED_2D_F32_S32;
3353 break;
3354 case NVPTXISD::TexUnified2DFloatFloat:
3355 Opc = NVPTX::TEX_UNIFIED_2D_F32_F32;
3356 break;
3357 case NVPTXISD::TexUnified2DFloatFloatLevel:
3358 Opc = NVPTX::TEX_UNIFIED_2D_F32_F32_LEVEL;
3359 break;
3360 case NVPTXISD::TexUnified2DFloatFloatGrad:
3361 Opc = NVPTX::TEX_UNIFIED_2D_F32_F32_GRAD;
3362 break;
3363 case NVPTXISD::TexUnified2DS32S32:
3364 Opc = NVPTX::TEX_UNIFIED_2D_S32_S32;
3365 break;
3366 case NVPTXISD::TexUnified2DS32Float:
3367 Opc = NVPTX::TEX_UNIFIED_2D_S32_F32;
3368 break;
3369 case NVPTXISD::TexUnified2DS32FloatLevel:
3370 Opc = NVPTX::TEX_UNIFIED_2D_S32_F32_LEVEL;
3371 break;
3372 case NVPTXISD::TexUnified2DS32FloatGrad:
3373 Opc = NVPTX::TEX_UNIFIED_2D_S32_F32_GRAD;
3374 break;
3375 case NVPTXISD::TexUnified2DU32S32:
3376 Opc = NVPTX::TEX_UNIFIED_2D_U32_S32;
3377 break;
3378 case NVPTXISD::TexUnified2DU32Float:
3379 Opc = NVPTX::TEX_UNIFIED_2D_U32_F32;
3380 break;
3381 case NVPTXISD::TexUnified2DU32FloatLevel:
3382 Opc = NVPTX::TEX_UNIFIED_2D_U32_F32_LEVEL;
3383 break;
3384 case NVPTXISD::TexUnified2DU32FloatGrad:
3385 Opc = NVPTX::TEX_UNIFIED_2D_U32_F32_GRAD;
3386 break;
3387 case NVPTXISD::TexUnified2DArrayFloatS32:
3388 Opc = NVPTX::TEX_UNIFIED_2D_ARRAY_F32_S32;
3389 break;
3390 case NVPTXISD::TexUnified2DArrayFloatFloat:
3391 Opc = NVPTX::TEX_UNIFIED_2D_ARRAY_F32_F32;
3392 break;
3393 case NVPTXISD::TexUnified2DArrayFloatFloatLevel:
3394 Opc = NVPTX::TEX_UNIFIED_2D_ARRAY_F32_F32_LEVEL;
3395 break;
3396 case NVPTXISD::TexUnified2DArrayFloatFloatGrad:
3397 Opc = NVPTX::TEX_UNIFIED_2D_ARRAY_F32_F32_GRAD;
3398 break;
3399 case NVPTXISD::TexUnified2DArrayS32S32:
3400 Opc = NVPTX::TEX_UNIFIED_2D_ARRAY_S32_S32;
3401 break;
3402 case NVPTXISD::TexUnified2DArrayS32Float:
3403 Opc = NVPTX::TEX_UNIFIED_2D_ARRAY_S32_F32;
3404 break;
3405 case NVPTXISD::TexUnified2DArrayS32FloatLevel:
3406 Opc = NVPTX::TEX_UNIFIED_2D_ARRAY_S32_F32_LEVEL;
3407 break;
3408 case NVPTXISD::TexUnified2DArrayS32FloatGrad:
3409 Opc = NVPTX::TEX_UNIFIED_2D_ARRAY_S32_F32_GRAD;
3410 break;
3411 case NVPTXISD::TexUnified2DArrayU32S32:
3412 Opc = NVPTX::TEX_UNIFIED_2D_ARRAY_U32_S32;
3413 break;
3414 case NVPTXISD::TexUnified2DArrayU32Float:
3415 Opc = NVPTX::TEX_UNIFIED_2D_ARRAY_U32_F32;
3416 break;
3417 case NVPTXISD::TexUnified2DArrayU32FloatLevel:
3418 Opc = NVPTX::TEX_UNIFIED_2D_ARRAY_U32_F32_LEVEL;
3419 break;
3420 case NVPTXISD::TexUnified2DArrayU32FloatGrad:
3421 Opc = NVPTX::TEX_UNIFIED_2D_ARRAY_U32_F32_GRAD;
3422 break;
3423 case NVPTXISD::TexUnified3DFloatS32:
3424 Opc = NVPTX::TEX_UNIFIED_3D_F32_S32;
3425 break;
3426 case NVPTXISD::TexUnified3DFloatFloat:
3427 Opc = NVPTX::TEX_UNIFIED_3D_F32_F32;
3428 break;
3429 case NVPTXISD::TexUnified3DFloatFloatLevel:
3430 Opc = NVPTX::TEX_UNIFIED_3D_F32_F32_LEVEL;
3431 break;
3432 case NVPTXISD::TexUnified3DFloatFloatGrad:
3433 Opc = NVPTX::TEX_UNIFIED_3D_F32_F32_GRAD;
3434 break;
3435 case NVPTXISD::TexUnified3DS32S32:
3436 Opc = NVPTX::TEX_UNIFIED_3D_S32_S32;
3437 break;
3438 case NVPTXISD::TexUnified3DS32Float:
3439 Opc = NVPTX::TEX_UNIFIED_3D_S32_F32;
3440 break;
3441 case NVPTXISD::TexUnified3DS32FloatLevel:
3442 Opc = NVPTX::TEX_UNIFIED_3D_S32_F32_LEVEL;
3443 break;
3444 case NVPTXISD::TexUnified3DS32FloatGrad:
3445 Opc = NVPTX::TEX_UNIFIED_3D_S32_F32_GRAD;
3446 break;
3447 case NVPTXISD::TexUnified3DU32S32:
3448 Opc = NVPTX::TEX_UNIFIED_3D_U32_S32;
3449 break;
3450 case NVPTXISD::TexUnified3DU32Float:
3451 Opc = NVPTX::TEX_UNIFIED_3D_U32_F32;
3452 break;
3453 case NVPTXISD::TexUnified3DU32FloatLevel:
3454 Opc = NVPTX::TEX_UNIFIED_3D_U32_F32_LEVEL;
3455 break;
3456 case NVPTXISD::TexUnified3DU32FloatGrad:
3457 Opc = NVPTX::TEX_UNIFIED_3D_U32_F32_GRAD;
3458 break;
3459 case NVPTXISD::TexUnifiedCubeFloatFloat:
3460 Opc = NVPTX::TEX_UNIFIED_CUBE_F32_F32;
3461 break;
3462 case NVPTXISD::TexUnifiedCubeFloatFloatLevel:
3463 Opc = NVPTX::TEX_UNIFIED_CUBE_F32_F32_LEVEL;
3464 break;
3465 case NVPTXISD::TexUnifiedCubeS32Float:
3466 Opc = NVPTX::TEX_UNIFIED_CUBE_S32_F32;
3467 break;
3468 case NVPTXISD::TexUnifiedCubeS32FloatLevel:
3469 Opc = NVPTX::TEX_UNIFIED_CUBE_S32_F32_LEVEL;
3470 break;
3471 case NVPTXISD::TexUnifiedCubeU32Float:
3472 Opc = NVPTX::TEX_UNIFIED_CUBE_U32_F32;
3473 break;
3474 case NVPTXISD::TexUnifiedCubeU32FloatLevel:
3475 Opc = NVPTX::TEX_UNIFIED_CUBE_U32_F32_LEVEL;
3476 break;
3477 case NVPTXISD::TexUnifiedCubeArrayFloatFloat:
3478 Opc = NVPTX::TEX_UNIFIED_CUBE_ARRAY_F32_F32;
3479 break;
3480 case NVPTXISD::TexUnifiedCubeArrayFloatFloatLevel:
3481 Opc = NVPTX::TEX_UNIFIED_CUBE_ARRAY_F32_F32_LEVEL;
3482 break;
3483 case NVPTXISD::TexUnifiedCubeArrayS32Float:
3484 Opc = NVPTX::TEX_UNIFIED_CUBE_ARRAY_S32_F32;
3485 break;
3486 case NVPTXISD::TexUnifiedCubeArrayS32FloatLevel:
3487 Opc = NVPTX::TEX_UNIFIED_CUBE_ARRAY_S32_F32_LEVEL;
3488 break;
3489 case NVPTXISD::TexUnifiedCubeArrayU32Float:
3490 Opc = NVPTX::TEX_UNIFIED_CUBE_ARRAY_U32_F32;
3491 break;
3492 case NVPTXISD::TexUnifiedCubeArrayU32FloatLevel:
3493 Opc = NVPTX::TEX_UNIFIED_CUBE_ARRAY_U32_F32_LEVEL;
3494 break;
3495 case NVPTXISD::Tld4UnifiedR2DFloatFloat:
3496 Opc = NVPTX::TLD4_UNIFIED_R_2D_F32_F32;
3497 break;
3498 case NVPTXISD::Tld4UnifiedG2DFloatFloat:
3499 Opc = NVPTX::TLD4_UNIFIED_G_2D_F32_F32;
3500 break;
3501 case NVPTXISD::Tld4UnifiedB2DFloatFloat:
3502 Opc = NVPTX::TLD4_UNIFIED_B_2D_F32_F32;
3503 break;
3504 case NVPTXISD::Tld4UnifiedA2DFloatFloat:
3505 Opc = NVPTX::TLD4_UNIFIED_A_2D_F32_F32;
3506 break;
3507 case NVPTXISD::Tld4UnifiedR2DS64Float:
3508 Opc = NVPTX::TLD4_UNIFIED_R_2D_S32_F32;
3509 break;
3510 case NVPTXISD::Tld4UnifiedG2DS64Float:
3511 Opc = NVPTX::TLD4_UNIFIED_G_2D_S32_F32;
3512 break;
3513 case NVPTXISD::Tld4UnifiedB2DS64Float:
3514 Opc = NVPTX::TLD4_UNIFIED_B_2D_S32_F32;
3515 break;
3516 case NVPTXISD::Tld4UnifiedA2DS64Float:
3517 Opc = NVPTX::TLD4_UNIFIED_A_2D_S32_F32;
3518 break;
3519 case NVPTXISD::Tld4UnifiedR2DU64Float:
3520 Opc = NVPTX::TLD4_UNIFIED_R_2D_U32_F32;
3521 break;
3522 case NVPTXISD::Tld4UnifiedG2DU64Float:
3523 Opc = NVPTX::TLD4_UNIFIED_G_2D_U32_F32;
3524 break;
3525 case NVPTXISD::Tld4UnifiedB2DU64Float:
3526 Opc = NVPTX::TLD4_UNIFIED_B_2D_U32_F32;
3527 break;
3528 case NVPTXISD::Tld4UnifiedA2DU64Float:
3529 Opc = NVPTX::TLD4_UNIFIED_A_2D_U32_F32;
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]3530 break;
3531 }
3532
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]3533 // Copy over operands
3534 for (unsigned i = 1; i < N->getNumOperands(); ++i) {
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]3535 Ops.push_back(N->getOperand(i));
3536 }
3537
3538 Ops.push_back(Chain);
3539 Ret = CurDAG->getMachineNode(Opc, SDLoc(N), N->getVTList(), Ops);
3540 return Ret;
3541}
3542
3543SDNode *NVPTXDAGToDAGISel::SelectSurfaceIntrinsic(SDNode *N) {
3544 SDValue Chain = N->getOperand(0);
3545 SDValue TexHandle = N->getOperand(1);
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]3546 SDNode *Ret = nullptr;
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]3547 unsigned Opc = 0;
3548 SmallVector<SDValue, 8> Ops;
3549 switch (N->getOpcode()) {
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]3550 default: return nullptr;
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]3551 case NVPTXISD::Suld1DI8Clamp:
3552 Opc = NVPTX::SULD_1D_I8_CLAMP;
3553 Ops.push_back(TexHandle);
3554 Ops.push_back(N->getOperand(2));
3555 Ops.push_back(Chain);
3556 break;
3557 case NVPTXISD::Suld1DI16Clamp:
3558 Opc = NVPTX::SULD_1D_I16_CLAMP;
3559 Ops.push_back(TexHandle);
3560 Ops.push_back(N->getOperand(2));
3561 Ops.push_back(Chain);
3562 break;
3563 case NVPTXISD::Suld1DI32Clamp:
3564 Opc = NVPTX::SULD_1D_I32_CLAMP;
3565 Ops.push_back(TexHandle);
3566 Ops.push_back(N->getOperand(2));
3567 Ops.push_back(Chain);
3568 break;
3569 case NVPTXISD::Suld1DI64Clamp:
3570 Opc = NVPTX::SULD_1D_I64_CLAMP;
3571 Ops.push_back(TexHandle);
3572 Ops.push_back(N->getOperand(2));
3573 Ops.push_back(Chain);
3574 break;
3575 case NVPTXISD::Suld1DV2I8Clamp:
3576 Opc = NVPTX::SULD_1D_V2I8_CLAMP;
3577 Ops.push_back(TexHandle);
3578 Ops.push_back(N->getOperand(2));
3579 Ops.push_back(Chain);
3580 break;
3581 case NVPTXISD::Suld1DV2I16Clamp:
3582 Opc = NVPTX::SULD_1D_V2I16_CLAMP;
3583 Ops.push_back(TexHandle);
3584 Ops.push_back(N->getOperand(2));
3585 Ops.push_back(Chain);
3586 break;
3587 case NVPTXISD::Suld1DV2I32Clamp:
3588 Opc = NVPTX::SULD_1D_V2I32_CLAMP;
3589 Ops.push_back(TexHandle);
3590 Ops.push_back(N->getOperand(2));
3591 Ops.push_back(Chain);
3592 break;
3593 case NVPTXISD::Suld1DV2I64Clamp:
3594 Opc = NVPTX::SULD_1D_V2I64_CLAMP;
3595 Ops.push_back(TexHandle);
3596 Ops.push_back(N->getOperand(2));
3597 Ops.push_back(Chain);
3598 break;
3599 case NVPTXISD::Suld1DV4I8Clamp:
3600 Opc = NVPTX::SULD_1D_V4I8_CLAMP;
3601 Ops.push_back(TexHandle);
3602 Ops.push_back(N->getOperand(2));
3603 Ops.push_back(Chain);
3604 break;
3605 case NVPTXISD::Suld1DV4I16Clamp:
3606 Opc = NVPTX::SULD_1D_V4I16_CLAMP;
3607 Ops.push_back(TexHandle);
3608 Ops.push_back(N->getOperand(2));
3609 Ops.push_back(Chain);
3610 break;
3611 case NVPTXISD::Suld1DV4I32Clamp:
3612 Opc = NVPTX::SULD_1D_V4I32_CLAMP;
3613 Ops.push_back(TexHandle);
3614 Ops.push_back(N->getOperand(2));
3615 Ops.push_back(Chain);
3616 break;
3617 case NVPTXISD::Suld1DArrayI8Clamp:
3618 Opc = NVPTX::SULD_1D_ARRAY_I8_CLAMP;
3619 Ops.push_back(TexHandle);
3620 Ops.push_back(N->getOperand(2));
3621 Ops.push_back(N->getOperand(3));
3622 Ops.push_back(Chain);
3623 break;
3624 case NVPTXISD::Suld1DArrayI16Clamp:
3625 Opc = NVPTX::SULD_1D_ARRAY_I16_CLAMP;
3626 Ops.push_back(TexHandle);
3627 Ops.push_back(N->getOperand(2));
3628 Ops.push_back(N->getOperand(3));
3629 Ops.push_back(Chain);
3630 break;
3631 case NVPTXISD::Suld1DArrayI32Clamp:
3632 Opc = NVPTX::SULD_1D_ARRAY_I32_CLAMP;
3633 Ops.push_back(TexHandle);
3634 Ops.push_back(N->getOperand(2));
3635 Ops.push_back(N->getOperand(3));
3636 Ops.push_back(Chain);
3637 break;
3638 case NVPTXISD::Suld1DArrayI64Clamp:
3639 Opc = NVPTX::SULD_1D_ARRAY_I64_CLAMP;
3640 Ops.push_back(TexHandle);
3641 Ops.push_back(N->getOperand(2));
3642 Ops.push_back(N->getOperand(3));
3643 Ops.push_back(Chain);
3644 break;
3645 case NVPTXISD::Suld1DArrayV2I8Clamp:
3646 Opc = NVPTX::SULD_1D_ARRAY_V2I8_CLAMP;
3647 Ops.push_back(TexHandle);
3648 Ops.push_back(N->getOperand(2));
3649 Ops.push_back(N->getOperand(3));
3650 Ops.push_back(Chain);
3651 break;
3652 case NVPTXISD::Suld1DArrayV2I16Clamp:
3653 Opc = NVPTX::SULD_1D_ARRAY_V2I16_CLAMP;
3654 Ops.push_back(TexHandle);
3655 Ops.push_back(N->getOperand(2));
3656 Ops.push_back(N->getOperand(3));
3657 Ops.push_back(Chain);
3658 break;
3659 case NVPTXISD::Suld1DArrayV2I32Clamp:
3660 Opc = NVPTX::SULD_1D_ARRAY_V2I32_CLAMP;
3661 Ops.push_back(TexHandle);
3662 Ops.push_back(N->getOperand(2));
3663 Ops.push_back(N->getOperand(3));
3664 Ops.push_back(Chain);
3665 break;
3666 case NVPTXISD::Suld1DArrayV2I64Clamp:
3667 Opc = NVPTX::SULD_1D_ARRAY_V2I64_CLAMP;
3668 Ops.push_back(TexHandle);
3669 Ops.push_back(N->getOperand(2));
3670 Ops.push_back(N->getOperand(3));
3671 Ops.push_back(Chain);
3672 break;
3673 case NVPTXISD::Suld1DArrayV4I8Clamp:
3674 Opc = NVPTX::SULD_1D_ARRAY_V4I8_CLAMP;
3675 Ops.push_back(TexHandle);
3676 Ops.push_back(N->getOperand(2));
3677 Ops.push_back(N->getOperand(3));
3678 Ops.push_back(Chain);
3679 break;
3680 case NVPTXISD::Suld1DArrayV4I16Clamp:
3681 Opc = NVPTX::SULD_1D_ARRAY_V4I16_CLAMP;
3682 Ops.push_back(TexHandle);
3683 Ops.push_back(N->getOperand(2));
3684 Ops.push_back(N->getOperand(3));
3685 Ops.push_back(Chain);
3686 break;
3687 case NVPTXISD::Suld1DArrayV4I32Clamp:
3688 Opc = NVPTX::SULD_1D_ARRAY_V4I32_CLAMP;
3689 Ops.push_back(TexHandle);
3690 Ops.push_back(N->getOperand(2));
3691 Ops.push_back(N->getOperand(3));
3692 Ops.push_back(Chain);
3693 break;
3694 case NVPTXISD::Suld2DI8Clamp:
3695 Opc = NVPTX::SULD_2D_I8_CLAMP;
3696 Ops.push_back(TexHandle);
3697 Ops.push_back(N->getOperand(2));
3698 Ops.push_back(N->getOperand(3));
3699 Ops.push_back(Chain);
3700 break;
3701 case NVPTXISD::Suld2DI16Clamp:
3702 Opc = NVPTX::SULD_2D_I16_CLAMP;
3703 Ops.push_back(TexHandle);
3704 Ops.push_back(N->getOperand(2));
3705 Ops.push_back(N->getOperand(3));
3706 Ops.push_back(Chain);
3707 break;
3708 case NVPTXISD::Suld2DI32Clamp:
3709 Opc = NVPTX::SULD_2D_I32_CLAMP;
3710 Ops.push_back(TexHandle);
3711 Ops.push_back(N->getOperand(2));
3712 Ops.push_back(N->getOperand(3));
3713 Ops.push_back(Chain);
3714 break;
3715 case NVPTXISD::Suld2DI64Clamp:
3716 Opc = NVPTX::SULD_2D_I64_CLAMP;
3717 Ops.push_back(TexHandle);
3718 Ops.push_back(N->getOperand(2));
3719 Ops.push_back(N->getOperand(3));
3720 Ops.push_back(Chain);
3721 break;
3722 case NVPTXISD::Suld2DV2I8Clamp:
3723 Opc = NVPTX::SULD_2D_V2I8_CLAMP;
3724 Ops.push_back(TexHandle);
3725 Ops.push_back(N->getOperand(2));
3726 Ops.push_back(N->getOperand(3));
3727 Ops.push_back(Chain);
3728 break;
3729 case NVPTXISD::Suld2DV2I16Clamp:
3730 Opc = NVPTX::SULD_2D_V2I16_CLAMP;
3731 Ops.push_back(TexHandle);
3732 Ops.push_back(N->getOperand(2));
3733 Ops.push_back(N->getOperand(3));
3734 Ops.push_back(Chain);
3735 break;
3736 case NVPTXISD::Suld2DV2I32Clamp:
3737 Opc = NVPTX::SULD_2D_V2I32_CLAMP;
3738 Ops.push_back(TexHandle);
3739 Ops.push_back(N->getOperand(2));
3740 Ops.push_back(N->getOperand(3));
3741 Ops.push_back(Chain);
3742 break;
3743 case NVPTXISD::Suld2DV2I64Clamp:
3744 Opc = NVPTX::SULD_2D_V2I64_CLAMP;
3745 Ops.push_back(TexHandle);
3746 Ops.push_back(N->getOperand(2));
3747 Ops.push_back(N->getOperand(3));
3748 Ops.push_back(Chain);
3749 break;
3750 case NVPTXISD::Suld2DV4I8Clamp:
3751 Opc = NVPTX::SULD_2D_V4I8_CLAMP;
3752 Ops.push_back(TexHandle);
3753 Ops.push_back(N->getOperand(2));
3754 Ops.push_back(N->getOperand(3));
3755 Ops.push_back(Chain);
3756 break;
3757 case NVPTXISD::Suld2DV4I16Clamp:
3758 Opc = NVPTX::SULD_2D_V4I16_CLAMP;
3759 Ops.push_back(TexHandle);
3760 Ops.push_back(N->getOperand(2));
3761 Ops.push_back(N->getOperand(3));
3762 Ops.push_back(Chain);
3763 break;
3764 case NVPTXISD::Suld2DV4I32Clamp:
3765 Opc = NVPTX::SULD_2D_V4I32_CLAMP;
3766 Ops.push_back(TexHandle);
3767 Ops.push_back(N->getOperand(2));
3768 Ops.push_back(N->getOperand(3));
3769 Ops.push_back(Chain);
3770 break;
3771 case NVPTXISD::Suld2DArrayI8Clamp:
3772 Opc = NVPTX::SULD_2D_ARRAY_I8_CLAMP;
3773 Ops.push_back(TexHandle);
3774 Ops.push_back(N->getOperand(2));
3775 Ops.push_back(N->getOperand(3));
3776 Ops.push_back(N->getOperand(4));
3777 Ops.push_back(Chain);
3778 break;
3779 case NVPTXISD::Suld2DArrayI16Clamp:
3780 Opc = NVPTX::SULD_2D_ARRAY_I16_CLAMP;
3781 Ops.push_back(TexHandle);
3782 Ops.push_back(N->getOperand(2));
3783 Ops.push_back(N->getOperand(3));
3784 Ops.push_back(N->getOperand(4));
3785 Ops.push_back(Chain);
3786 break;
3787 case NVPTXISD::Suld2DArrayI32Clamp:
3788 Opc = NVPTX::SULD_2D_ARRAY_I32_CLAMP;
3789 Ops.push_back(TexHandle);
3790 Ops.push_back(N->getOperand(2));
3791 Ops.push_back(N->getOperand(3));
3792 Ops.push_back(N->getOperand(4));
3793 Ops.push_back(Chain);
3794 break;
3795 case NVPTXISD::Suld2DArrayI64Clamp:
3796 Opc = NVPTX::SULD_2D_ARRAY_I64_CLAMP;
3797 Ops.push_back(TexHandle);
3798 Ops.push_back(N->getOperand(2));
3799 Ops.push_back(N->getOperand(3));
3800 Ops.push_back(N->getOperand(4));
3801 Ops.push_back(Chain);
3802 break;
3803 case NVPTXISD::Suld2DArrayV2I8Clamp:
3804 Opc = NVPTX::SULD_2D_ARRAY_V2I8_CLAMP;
3805 Ops.push_back(TexHandle);
3806 Ops.push_back(N->getOperand(2));
3807 Ops.push_back(N->getOperand(3));
3808 Ops.push_back(N->getOperand(4));
3809 Ops.push_back(Chain);
3810 break;
3811 case NVPTXISD::Suld2DArrayV2I16Clamp:
3812 Opc = NVPTX::SULD_2D_ARRAY_V2I16_CLAMP;
3813 Ops.push_back(TexHandle);
3814 Ops.push_back(N->getOperand(2));
3815 Ops.push_back(N->getOperand(3));
3816 Ops.push_back(N->getOperand(4));
3817 Ops.push_back(Chain);
3818 break;
3819 case NVPTXISD::Suld2DArrayV2I32Clamp:
3820 Opc = NVPTX::SULD_2D_ARRAY_V2I32_CLAMP;
3821 Ops.push_back(TexHandle);
3822 Ops.push_back(N->getOperand(2));
3823 Ops.push_back(N->getOperand(3));
3824 Ops.push_back(N->getOperand(4));
3825 Ops.push_back(Chain);
3826 break;
3827 case NVPTXISD::Suld2DArrayV2I64Clamp:
3828 Opc = NVPTX::SULD_2D_ARRAY_V2I64_CLAMP;
3829 Ops.push_back(TexHandle);
3830 Ops.push_back(N->getOperand(2));
3831 Ops.push_back(N->getOperand(3));
3832 Ops.push_back(N->getOperand(4));
3833 Ops.push_back(Chain);
3834 break;
3835 case NVPTXISD::Suld2DArrayV4I8Clamp:
3836 Opc = NVPTX::SULD_2D_ARRAY_V4I8_CLAMP;
3837 Ops.push_back(TexHandle);
3838 Ops.push_back(N->getOperand(2));
3839 Ops.push_back(N->getOperand(3));
3840 Ops.push_back(N->getOperand(4));
3841 Ops.push_back(Chain);
3842 break;
3843 case NVPTXISD::Suld2DArrayV4I16Clamp:
3844 Opc = NVPTX::SULD_2D_ARRAY_V4I16_CLAMP;
3845 Ops.push_back(TexHandle);
3846 Ops.push_back(N->getOperand(2));
3847 Ops.push_back(N->getOperand(3));
3848 Ops.push_back(N->getOperand(4));
3849 Ops.push_back(Chain);
3850 break;
3851 case NVPTXISD::Suld2DArrayV4I32Clamp:
3852 Opc = NVPTX::SULD_2D_ARRAY_V4I32_CLAMP;
3853 Ops.push_back(TexHandle);
3854 Ops.push_back(N->getOperand(2));
3855 Ops.push_back(N->getOperand(3));
3856 Ops.push_back(N->getOperand(4));
3857 Ops.push_back(Chain);
3858 break;
3859 case NVPTXISD::Suld3DI8Clamp:
3860 Opc = NVPTX::SULD_3D_I8_CLAMP;
3861 Ops.push_back(TexHandle);
3862 Ops.push_back(N->getOperand(2));
3863 Ops.push_back(N->getOperand(3));
3864 Ops.push_back(N->getOperand(4));
3865 Ops.push_back(Chain);
3866 break;
3867 case NVPTXISD::Suld3DI16Clamp:
3868 Opc = NVPTX::SULD_3D_I16_CLAMP;
3869 Ops.push_back(TexHandle);
3870 Ops.push_back(N->getOperand(2));
3871 Ops.push_back(N->getOperand(3));
3872 Ops.push_back(N->getOperand(4));
3873 Ops.push_back(Chain);
3874 break;
3875 case NVPTXISD::Suld3DI32Clamp:
3876 Opc = NVPTX::SULD_3D_I32_CLAMP;
3877 Ops.push_back(TexHandle);
3878 Ops.push_back(N->getOperand(2));
3879 Ops.push_back(N->getOperand(3));
3880 Ops.push_back(N->getOperand(4));
3881 Ops.push_back(Chain);
3882 break;
3883 case NVPTXISD::Suld3DI64Clamp:
3884 Opc = NVPTX::SULD_3D_I64_CLAMP;
3885 Ops.push_back(TexHandle);
3886 Ops.push_back(N->getOperand(2));
3887 Ops.push_back(N->getOperand(3));
3888 Ops.push_back(N->getOperand(4));
3889 Ops.push_back(Chain);
3890 break;
3891 case NVPTXISD::Suld3DV2I8Clamp:
3892 Opc = NVPTX::SULD_3D_V2I8_CLAMP;
3893 Ops.push_back(TexHandle);
3894 Ops.push_back(N->getOperand(2));
3895 Ops.push_back(N->getOperand(3));
3896 Ops.push_back(N->getOperand(4));
3897 Ops.push_back(Chain);
3898 break;
3899 case NVPTXISD::Suld3DV2I16Clamp:
3900 Opc = NVPTX::SULD_3D_V2I16_CLAMP;
3901 Ops.push_back(TexHandle);
3902 Ops.push_back(N->getOperand(2));
3903 Ops.push_back(N->getOperand(3));
3904 Ops.push_back(N->getOperand(4));
3905 Ops.push_back(Chain);
3906 break;
3907 case NVPTXISD::Suld3DV2I32Clamp:
3908 Opc = NVPTX::SULD_3D_V2I32_CLAMP;
3909 Ops.push_back(TexHandle);
3910 Ops.push_back(N->getOperand(2));
3911 Ops.push_back(N->getOperand(3));
3912 Ops.push_back(N->getOperand(4));
3913 Ops.push_back(Chain);
3914 break;
3915 case NVPTXISD::Suld3DV2I64Clamp:
3916 Opc = NVPTX::SULD_3D_V2I64_CLAMP;
3917 Ops.push_back(TexHandle);
3918 Ops.push_back(N->getOperand(2));
3919 Ops.push_back(N->getOperand(3));
3920 Ops.push_back(N->getOperand(4));
3921 Ops.push_back(Chain);
3922 break;
3923 case NVPTXISD::Suld3DV4I8Clamp:
3924 Opc = NVPTX::SULD_3D_V4I8_CLAMP;
3925 Ops.push_back(TexHandle);
3926 Ops.push_back(N->getOperand(2));
3927 Ops.push_back(N->getOperand(3));
3928 Ops.push_back(N->getOperand(4));
3929 Ops.push_back(Chain);
3930 break;
3931 case NVPTXISD::Suld3DV4I16Clamp:
3932 Opc = NVPTX::SULD_3D_V4I16_CLAMP;
3933 Ops.push_back(TexHandle);
3934 Ops.push_back(N->getOperand(2));
3935 Ops.push_back(N->getOperand(3));
3936 Ops.push_back(N->getOperand(4));
3937 Ops.push_back(Chain);
3938 break;
3939 case NVPTXISD::Suld3DV4I32Clamp:
3940 Opc = NVPTX::SULD_3D_V4I32_CLAMP;
3941 Ops.push_back(TexHandle);
3942 Ops.push_back(N->getOperand(2));
3943 Ops.push_back(N->getOperand(3));
3944 Ops.push_back(N->getOperand(4));
3945 Ops.push_back(Chain);
3946 break;
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]3947 case NVPTXISD::Suld1DI8Trap:
3948 Opc = NVPTX::SULD_1D_I8_TRAP;
3949 Ops.push_back(TexHandle);
3950 Ops.push_back(N->getOperand(2));
3951 Ops.push_back(Chain);
3952 break;
3953 case NVPTXISD::Suld1DI16Trap:
3954 Opc = NVPTX::SULD_1D_I16_TRAP;
3955 Ops.push_back(TexHandle);
3956 Ops.push_back(N->getOperand(2));
3957 Ops.push_back(Chain);
3958 break;
3959 case NVPTXISD::Suld1DI32Trap:
3960 Opc = NVPTX::SULD_1D_I32_TRAP;
3961 Ops.push_back(TexHandle);
3962 Ops.push_back(N->getOperand(2));
3963 Ops.push_back(Chain);
3964 break;
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]3965 case NVPTXISD::Suld1DI64Trap:
3966 Opc = NVPTX::SULD_1D_I64_TRAP;
3967 Ops.push_back(TexHandle);
3968 Ops.push_back(N->getOperand(2));
3969 Ops.push_back(Chain);
3970 break;
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]3971 case NVPTXISD::Suld1DV2I8Trap:
3972 Opc = NVPTX::SULD_1D_V2I8_TRAP;
3973 Ops.push_back(TexHandle);
3974 Ops.push_back(N->getOperand(2));
3975 Ops.push_back(Chain);
3976 break;
3977 case NVPTXISD::Suld1DV2I16Trap:
3978 Opc = NVPTX::SULD_1D_V2I16_TRAP;
3979 Ops.push_back(TexHandle);
3980 Ops.push_back(N->getOperand(2));
3981 Ops.push_back(Chain);
3982 break;
3983 case NVPTXISD::Suld1DV2I32Trap:
3984 Opc = NVPTX::SULD_1D_V2I32_TRAP;
3985 Ops.push_back(TexHandle);
3986 Ops.push_back(N->getOperand(2));
3987 Ops.push_back(Chain);
3988 break;
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]3989 case NVPTXISD::Suld1DV2I64Trap:
3990 Opc = NVPTX::SULD_1D_V2I64_TRAP;
3991 Ops.push_back(TexHandle);
3992 Ops.push_back(N->getOperand(2));
3993 Ops.push_back(Chain);
3994 break;
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]3995 case NVPTXISD::Suld1DV4I8Trap:
3996 Opc = NVPTX::SULD_1D_V4I8_TRAP;
3997 Ops.push_back(TexHandle);
3998 Ops.push_back(N->getOperand(2));
3999 Ops.push_back(Chain);
4000 break;
4001 case NVPTXISD::Suld1DV4I16Trap:
4002 Opc = NVPTX::SULD_1D_V4I16_TRAP;
4003 Ops.push_back(TexHandle);
4004 Ops.push_back(N->getOperand(2));
4005 Ops.push_back(Chain);
4006 break;
4007 case NVPTXISD::Suld1DV4I32Trap:
4008 Opc = NVPTX::SULD_1D_V4I32_TRAP;
4009 Ops.push_back(TexHandle);
4010 Ops.push_back(N->getOperand(2));
4011 Ops.push_back(Chain);
4012 break;
4013 case NVPTXISD::Suld1DArrayI8Trap:
4014 Opc = NVPTX::SULD_1D_ARRAY_I8_TRAP;
4015 Ops.push_back(TexHandle);
4016 Ops.push_back(N->getOperand(2));
4017 Ops.push_back(N->getOperand(3));
4018 Ops.push_back(Chain);
4019 break;
4020 case NVPTXISD::Suld1DArrayI16Trap:
4021 Opc = NVPTX::SULD_1D_ARRAY_I16_TRAP;
4022 Ops.push_back(TexHandle);
4023 Ops.push_back(N->getOperand(2));
4024 Ops.push_back(N->getOperand(3));
4025 Ops.push_back(Chain);
4026 break;
4027 case NVPTXISD::Suld1DArrayI32Trap:
4028 Opc = NVPTX::SULD_1D_ARRAY_I32_TRAP;
4029 Ops.push_back(TexHandle);
4030 Ops.push_back(N->getOperand(2));
4031 Ops.push_back(N->getOperand(3));
4032 Ops.push_back(Chain);
4033 break;
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]4034 case NVPTXISD::Suld1DArrayI64Trap:
4035 Opc = NVPTX::SULD_1D_ARRAY_I64_TRAP;
4036 Ops.push_back(TexHandle);
4037 Ops.push_back(N->getOperand(2));
4038 Ops.push_back(N->getOperand(3));
4039 Ops.push_back(Chain);
4040 break;
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]4041 case NVPTXISD::Suld1DArrayV2I8Trap:
4042 Opc = NVPTX::SULD_1D_ARRAY_V2I8_TRAP;
4043 Ops.push_back(TexHandle);
4044 Ops.push_back(N->getOperand(2));
4045 Ops.push_back(N->getOperand(3));
4046 Ops.push_back(Chain);
4047 break;
4048 case NVPTXISD::Suld1DArrayV2I16Trap:
4049 Opc = NVPTX::SULD_1D_ARRAY_V2I16_TRAP;
4050 Ops.push_back(TexHandle);
4051 Ops.push_back(N->getOperand(2));
4052 Ops.push_back(N->getOperand(3));
4053 Ops.push_back(Chain);
4054 break;
4055 case NVPTXISD::Suld1DArrayV2I32Trap:
4056 Opc = NVPTX::SULD_1D_ARRAY_V2I32_TRAP;
4057 Ops.push_back(TexHandle);
4058 Ops.push_back(N->getOperand(2));
4059 Ops.push_back(N->getOperand(3));
4060 Ops.push_back(Chain);
4061 break;
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]4062 case NVPTXISD::Suld1DArrayV2I64Trap:
4063 Opc = NVPTX::SULD_1D_ARRAY_V2I64_TRAP;
4064 Ops.push_back(TexHandle);
4065 Ops.push_back(N->getOperand(2));
4066 Ops.push_back(N->getOperand(3));
4067 Ops.push_back(Chain);
4068 break;
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]4069 case NVPTXISD::Suld1DArrayV4I8Trap:
4070 Opc = NVPTX::SULD_1D_ARRAY_V4I8_TRAP;
4071 Ops.push_back(TexHandle);
4072 Ops.push_back(N->getOperand(2));
4073 Ops.push_back(N->getOperand(3));
4074 Ops.push_back(Chain);
4075 break;
4076 case NVPTXISD::Suld1DArrayV4I16Trap:
4077 Opc = NVPTX::SULD_1D_ARRAY_V4I16_TRAP;
4078 Ops.push_back(TexHandle);
4079 Ops.push_back(N->getOperand(2));
4080 Ops.push_back(N->getOperand(3));
4081 Ops.push_back(Chain);
4082 break;
4083 case NVPTXISD::Suld1DArrayV4I32Trap:
4084 Opc = NVPTX::SULD_1D_ARRAY_V4I32_TRAP;
4085 Ops.push_back(TexHandle);
4086 Ops.push_back(N->getOperand(2));
4087 Ops.push_back(N->getOperand(3));
4088 Ops.push_back(Chain);
4089 break;
4090 case NVPTXISD::Suld2DI8Trap:
4091 Opc = NVPTX::SULD_2D_I8_TRAP;
4092 Ops.push_back(TexHandle);
4093 Ops.push_back(N->getOperand(2));
4094 Ops.push_back(N->getOperand(3));
4095 Ops.push_back(Chain);
4096 break;
4097 case NVPTXISD::Suld2DI16Trap:
4098 Opc = NVPTX::SULD_2D_I16_TRAP;
4099 Ops.push_back(TexHandle);
4100 Ops.push_back(N->getOperand(2));
4101 Ops.push_back(N->getOperand(3));
4102 Ops.push_back(Chain);
4103 break;
4104 case NVPTXISD::Suld2DI32Trap:
4105 Opc = NVPTX::SULD_2D_I32_TRAP;
4106 Ops.push_back(TexHandle);
4107 Ops.push_back(N->getOperand(2));
4108 Ops.push_back(N->getOperand(3));
4109 Ops.push_back(Chain);
4110 break;
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]4111 case NVPTXISD::Suld2DI64Trap:
4112 Opc = NVPTX::SULD_2D_I64_TRAP;
4113 Ops.push_back(TexHandle);
4114 Ops.push_back(N->getOperand(2));
4115 Ops.push_back(N->getOperand(3));
4116 Ops.push_back(Chain);
4117 break;
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]4118 case NVPTXISD::Suld2DV2I8Trap:
4119 Opc = NVPTX::SULD_2D_V2I8_TRAP;
4120 Ops.push_back(TexHandle);
4121 Ops.push_back(N->getOperand(2));
4122 Ops.push_back(N->getOperand(3));
4123 Ops.push_back(Chain);
4124 break;
4125 case NVPTXISD::Suld2DV2I16Trap:
4126 Opc = NVPTX::SULD_2D_V2I16_TRAP;
4127 Ops.push_back(TexHandle);
4128 Ops.push_back(N->getOperand(2));
4129 Ops.push_back(N->getOperand(3));
4130 Ops.push_back(Chain);
4131 break;
4132 case NVPTXISD::Suld2DV2I32Trap:
4133 Opc = NVPTX::SULD_2D_V2I32_TRAP;
4134 Ops.push_back(TexHandle);
4135 Ops.push_back(N->getOperand(2));
4136 Ops.push_back(N->getOperand(3));
4137 Ops.push_back(Chain);
4138 break;
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]4139 case NVPTXISD::Suld2DV2I64Trap:
4140 Opc = NVPTX::SULD_2D_V2I64_TRAP;
4141 Ops.push_back(TexHandle);
4142 Ops.push_back(N->getOperand(2));
4143 Ops.push_back(N->getOperand(3));
4144 Ops.push_back(Chain);
4145 break;
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]4146 case NVPTXISD::Suld2DV4I8Trap:
4147 Opc = NVPTX::SULD_2D_V4I8_TRAP;
4148 Ops.push_back(TexHandle);
4149 Ops.push_back(N->getOperand(2));
4150 Ops.push_back(N->getOperand(3));
4151 Ops.push_back(Chain);
4152 break;
4153 case NVPTXISD::Suld2DV4I16Trap:
4154 Opc = NVPTX::SULD_2D_V4I16_TRAP;
4155 Ops.push_back(TexHandle);
4156 Ops.push_back(N->getOperand(2));
4157 Ops.push_back(N->getOperand(3));
4158 Ops.push_back(Chain);
4159 break;
4160 case NVPTXISD::Suld2DV4I32Trap:
4161 Opc = NVPTX::SULD_2D_V4I32_TRAP;
4162 Ops.push_back(TexHandle);
4163 Ops.push_back(N->getOperand(2));
4164 Ops.push_back(N->getOperand(3));
4165 Ops.push_back(Chain);
4166 break;
4167 case NVPTXISD::Suld2DArrayI8Trap:
4168 Opc = NVPTX::SULD_2D_ARRAY_I8_TRAP;
4169 Ops.push_back(TexHandle);
4170 Ops.push_back(N->getOperand(2));
4171 Ops.push_back(N->getOperand(3));
4172 Ops.push_back(N->getOperand(4));
4173 Ops.push_back(Chain);
4174 break;
4175 case NVPTXISD::Suld2DArrayI16Trap:
4176 Opc = NVPTX::SULD_2D_ARRAY_I16_TRAP;
4177 Ops.push_back(TexHandle);
4178 Ops.push_back(N->getOperand(2));
4179 Ops.push_back(N->getOperand(3));
4180 Ops.push_back(N->getOperand(4));
4181 Ops.push_back(Chain);
4182 break;
4183 case NVPTXISD::Suld2DArrayI32Trap:
4184 Opc = NVPTX::SULD_2D_ARRAY_I32_TRAP;
4185 Ops.push_back(TexHandle);
4186 Ops.push_back(N->getOperand(2));
4187 Ops.push_back(N->getOperand(3));
4188 Ops.push_back(N->getOperand(4));
4189 Ops.push_back(Chain);
4190 break;
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]4191 case NVPTXISD::Suld2DArrayI64Trap:
4192 Opc = NVPTX::SULD_2D_ARRAY_I64_TRAP;
4193 Ops.push_back(TexHandle);
4194 Ops.push_back(N->getOperand(2));
4195 Ops.push_back(N->getOperand(3));
4196 Ops.push_back(N->getOperand(4));
4197 Ops.push_back(Chain);
4198 break;
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]4199 case NVPTXISD::Suld2DArrayV2I8Trap:
4200 Opc = NVPTX::SULD_2D_ARRAY_V2I8_TRAP;
4201 Ops.push_back(TexHandle);
4202 Ops.push_back(N->getOperand(2));
4203 Ops.push_back(N->getOperand(3));
4204 Ops.push_back(N->getOperand(4));
4205 Ops.push_back(Chain);
4206 break;
4207 case NVPTXISD::Suld2DArrayV2I16Trap:
4208 Opc = NVPTX::SULD_2D_ARRAY_V2I16_TRAP;
4209 Ops.push_back(TexHandle);
4210 Ops.push_back(N->getOperand(2));
4211 Ops.push_back(N->getOperand(3));
4212 Ops.push_back(N->getOperand(4));
4213 Ops.push_back(Chain);
4214 break;
4215 case NVPTXISD::Suld2DArrayV2I32Trap:
4216 Opc = NVPTX::SULD_2D_ARRAY_V2I32_TRAP;
4217 Ops.push_back(TexHandle);
4218 Ops.push_back(N->getOperand(2));
4219 Ops.push_back(N->getOperand(3));
4220 Ops.push_back(N->getOperand(4));
4221 Ops.push_back(Chain);
4222 break;
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]4223 case NVPTXISD::Suld2DArrayV2I64Trap:
4224 Opc = NVPTX::SULD_2D_ARRAY_V2I64_TRAP;
4225 Ops.push_back(TexHandle);
4226 Ops.push_back(N->getOperand(2));
4227 Ops.push_back(N->getOperand(3));
4228 Ops.push_back(N->getOperand(4));
4229 Ops.push_back(Chain);
4230 break;
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]4231 case NVPTXISD::Suld2DArrayV4I8Trap:
4232 Opc = NVPTX::SULD_2D_ARRAY_V4I8_TRAP;
4233 Ops.push_back(TexHandle);
4234 Ops.push_back(N->getOperand(2));
4235 Ops.push_back(N->getOperand(3));
4236 Ops.push_back(N->getOperand(4));
4237 Ops.push_back(Chain);
4238 break;
4239 case NVPTXISD::Suld2DArrayV4I16Trap:
4240 Opc = NVPTX::SULD_2D_ARRAY_V4I16_TRAP;
4241 Ops.push_back(TexHandle);
4242 Ops.push_back(N->getOperand(2));
4243 Ops.push_back(N->getOperand(3));
4244 Ops.push_back(N->getOperand(4));
4245 Ops.push_back(Chain);
4246 break;
4247 case NVPTXISD::Suld2DArrayV4I32Trap:
4248 Opc = NVPTX::SULD_2D_ARRAY_V4I32_TRAP;
4249 Ops.push_back(TexHandle);
4250 Ops.push_back(N->getOperand(2));
4251 Ops.push_back(N->getOperand(3));
4252 Ops.push_back(N->getOperand(4));
4253 Ops.push_back(Chain);
4254 break;
4255 case NVPTXISD::Suld3DI8Trap:
4256 Opc = NVPTX::SULD_3D_I8_TRAP;
4257 Ops.push_back(TexHandle);
4258 Ops.push_back(N->getOperand(2));
4259 Ops.push_back(N->getOperand(3));
4260 Ops.push_back(N->getOperand(4));
4261 Ops.push_back(Chain);
4262 break;
4263 case NVPTXISD::Suld3DI16Trap:
4264 Opc = NVPTX::SULD_3D_I16_TRAP;
4265 Ops.push_back(TexHandle);
4266 Ops.push_back(N->getOperand(2));
4267 Ops.push_back(N->getOperand(3));
4268 Ops.push_back(N->getOperand(4));
4269 Ops.push_back(Chain);
4270 break;
4271 case NVPTXISD::Suld3DI32Trap:
4272 Opc = NVPTX::SULD_3D_I32_TRAP;
4273 Ops.push_back(TexHandle);
4274 Ops.push_back(N->getOperand(2));
4275 Ops.push_back(N->getOperand(3));
4276 Ops.push_back(N->getOperand(4));
4277 Ops.push_back(Chain);
4278 break;
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]4279 case NVPTXISD::Suld3DI64Trap:
4280 Opc = NVPTX::SULD_3D_I64_TRAP;
4281 Ops.push_back(TexHandle);
4282 Ops.push_back(N->getOperand(2));
4283 Ops.push_back(N->getOperand(3));
4284 Ops.push_back(N->getOperand(4));
4285 Ops.push_back(Chain);
4286 break;
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]4287 case NVPTXISD::Suld3DV2I8Trap:
4288 Opc = NVPTX::SULD_3D_V2I8_TRAP;
4289 Ops.push_back(TexHandle);
4290 Ops.push_back(N->getOperand(2));
4291 Ops.push_back(N->getOperand(3));
4292 Ops.push_back(N->getOperand(4));
4293 Ops.push_back(Chain);
4294 break;
4295 case NVPTXISD::Suld3DV2I16Trap:
4296 Opc = NVPTX::SULD_3D_V2I16_TRAP;
4297 Ops.push_back(TexHandle);
4298 Ops.push_back(N->getOperand(2));
4299 Ops.push_back(N->getOperand(3));
4300 Ops.push_back(N->getOperand(4));
4301 Ops.push_back(Chain);
4302 break;
4303 case NVPTXISD::Suld3DV2I32Trap:
4304 Opc = NVPTX::SULD_3D_V2I32_TRAP;
4305 Ops.push_back(TexHandle);
4306 Ops.push_back(N->getOperand(2));
4307 Ops.push_back(N->getOperand(3));
4308 Ops.push_back(N->getOperand(4));
4309 Ops.push_back(Chain);
4310 break;
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]4311 case NVPTXISD::Suld3DV2I64Trap:
4312 Opc = NVPTX::SULD_3D_V2I64_TRAP;
4313 Ops.push_back(TexHandle);
4314 Ops.push_back(N->getOperand(2));
4315 Ops.push_back(N->getOperand(3));
4316 Ops.push_back(N->getOperand(4));
4317 Ops.push_back(Chain);
4318 break;
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]4319 case NVPTXISD::Suld3DV4I8Trap:
4320 Opc = NVPTX::SULD_3D_V4I8_TRAP;
4321 Ops.push_back(TexHandle);
4322 Ops.push_back(N->getOperand(2));
4323 Ops.push_back(N->getOperand(3));
4324 Ops.push_back(N->getOperand(4));
4325 Ops.push_back(Chain);
4326 break;
4327 case NVPTXISD::Suld3DV4I16Trap:
4328 Opc = NVPTX::SULD_3D_V4I16_TRAP;
4329 Ops.push_back(TexHandle);
4330 Ops.push_back(N->getOperand(2));
4331 Ops.push_back(N->getOperand(3));
4332 Ops.push_back(N->getOperand(4));
4333 Ops.push_back(Chain);
4334 break;
4335 case NVPTXISD::Suld3DV4I32Trap:
4336 Opc = NVPTX::SULD_3D_V4I32_TRAP;
4337 Ops.push_back(TexHandle);
4338 Ops.push_back(N->getOperand(2));
4339 Ops.push_back(N->getOperand(3));
4340 Ops.push_back(N->getOperand(4));
4341 Ops.push_back(Chain);
4342 break;
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]4343 case NVPTXISD::Suld1DI8Zero:
4344 Opc = NVPTX::SULD_1D_I8_ZERO;
4345 Ops.push_back(TexHandle);
4346 Ops.push_back(N->getOperand(2));
4347 Ops.push_back(Chain);
4348 break;
4349 case NVPTXISD::Suld1DI16Zero:
4350 Opc = NVPTX::SULD_1D_I16_ZERO;
4351 Ops.push_back(TexHandle);
4352 Ops.push_back(N->getOperand(2));
4353 Ops.push_back(Chain);
4354 break;
4355 case NVPTXISD::Suld1DI32Zero:
4356 Opc = NVPTX::SULD_1D_I32_ZERO;
4357 Ops.push_back(TexHandle);
4358 Ops.push_back(N->getOperand(2));
4359 Ops.push_back(Chain);
4360 break;
4361 case NVPTXISD::Suld1DI64Zero:
4362 Opc = NVPTX::SULD_1D_I64_ZERO;
4363 Ops.push_back(TexHandle);
4364 Ops.push_back(N->getOperand(2));
4365 Ops.push_back(Chain);
4366 break;
4367 case NVPTXISD::Suld1DV2I8Zero:
4368 Opc = NVPTX::SULD_1D_V2I8_ZERO;
4369 Ops.push_back(TexHandle);
4370 Ops.push_back(N->getOperand(2));
4371 Ops.push_back(Chain);
4372 break;
4373 case NVPTXISD::Suld1DV2I16Zero:
4374 Opc = NVPTX::SULD_1D_V2I16_ZERO;
4375 Ops.push_back(TexHandle);
4376 Ops.push_back(N->getOperand(2));
4377 Ops.push_back(Chain);
4378 break;
4379 case NVPTXISD::Suld1DV2I32Zero:
4380 Opc = NVPTX::SULD_1D_V2I32_ZERO;
4381 Ops.push_back(TexHandle);
4382 Ops.push_back(N->getOperand(2));
4383 Ops.push_back(Chain);
4384 break;
4385 case NVPTXISD::Suld1DV2I64Zero:
4386 Opc = NVPTX::SULD_1D_V2I64_ZERO;
4387 Ops.push_back(TexHandle);
4388 Ops.push_back(N->getOperand(2));
4389 Ops.push_back(Chain);
4390 break;
4391 case NVPTXISD::Suld1DV4I8Zero:
4392 Opc = NVPTX::SULD_1D_V4I8_ZERO;
4393 Ops.push_back(TexHandle);
4394 Ops.push_back(N->getOperand(2));
4395 Ops.push_back(Chain);
4396 break;
4397 case NVPTXISD::Suld1DV4I16Zero:
4398 Opc = NVPTX::SULD_1D_V4I16_ZERO;
4399 Ops.push_back(TexHandle);
4400 Ops.push_back(N->getOperand(2));
4401 Ops.push_back(Chain);
4402 break;
4403 case NVPTXISD::Suld1DV4I32Zero:
4404 Opc = NVPTX::SULD_1D_V4I32_ZERO;
4405 Ops.push_back(TexHandle);
4406 Ops.push_back(N->getOperand(2));
4407 Ops.push_back(Chain);
4408 break;
4409 case NVPTXISD::Suld1DArrayI8Zero:
4410 Opc = NVPTX::SULD_1D_ARRAY_I8_ZERO;
4411 Ops.push_back(TexHandle);
4412 Ops.push_back(N->getOperand(2));
4413 Ops.push_back(N->getOperand(3));
4414 Ops.push_back(Chain);
4415 break;
4416 case NVPTXISD::Suld1DArrayI16Zero:
4417 Opc = NVPTX::SULD_1D_ARRAY_I16_ZERO;
4418 Ops.push_back(TexHandle);
4419 Ops.push_back(N->getOperand(2));
4420 Ops.push_back(N->getOperand(3));
4421 Ops.push_back(Chain);
4422 break;
4423 case NVPTXISD::Suld1DArrayI32Zero:
4424 Opc = NVPTX::SULD_1D_ARRAY_I32_ZERO;
4425 Ops.push_back(TexHandle);
4426 Ops.push_back(N->getOperand(2));
4427 Ops.push_back(N->getOperand(3));
4428 Ops.push_back(Chain);
4429 break;
4430 case NVPTXISD::Suld1DArrayI64Zero:
4431 Opc = NVPTX::SULD_1D_ARRAY_I64_ZERO;
4432 Ops.push_back(TexHandle);
4433 Ops.push_back(N->getOperand(2));
4434 Ops.push_back(N->getOperand(3));
4435 Ops.push_back(Chain);
4436 break;
4437 case NVPTXISD::Suld1DArrayV2I8Zero:
4438 Opc = NVPTX::SULD_1D_ARRAY_V2I8_ZERO;
4439 Ops.push_back(TexHandle);
4440 Ops.push_back(N->getOperand(2));
4441 Ops.push_back(N->getOperand(3));
4442 Ops.push_back(Chain);
4443 break;
4444 case NVPTXISD::Suld1DArrayV2I16Zero:
4445 Opc = NVPTX::SULD_1D_ARRAY_V2I16_ZERO;
4446 Ops.push_back(TexHandle);
4447 Ops.push_back(N->getOperand(2));
4448 Ops.push_back(N->getOperand(3));
4449 Ops.push_back(Chain);
4450 break;
4451 case NVPTXISD::Suld1DArrayV2I32Zero:
4452 Opc = NVPTX::SULD_1D_ARRAY_V2I32_ZERO;
4453 Ops.push_back(TexHandle);
4454 Ops.push_back(N->getOperand(2));
4455 Ops.push_back(N->getOperand(3));
4456 Ops.push_back(Chain);
4457 break;
4458 case NVPTXISD::Suld1DArrayV2I64Zero:
4459 Opc = NVPTX::SULD_1D_ARRAY_V2I64_ZERO;
4460 Ops.push_back(TexHandle);
4461 Ops.push_back(N->getOperand(2));
4462 Ops.push_back(N->getOperand(3));
4463 Ops.push_back(Chain);
4464 break;
4465 case NVPTXISD::Suld1DArrayV4I8Zero:
4466 Opc = NVPTX::SULD_1D_ARRAY_V4I8_ZERO;
4467 Ops.push_back(TexHandle);
4468 Ops.push_back(N->getOperand(2));
4469 Ops.push_back(N->getOperand(3));
4470 Ops.push_back(Chain);
4471 break;
4472 case NVPTXISD::Suld1DArrayV4I16Zero:
4473 Opc = NVPTX::SULD_1D_ARRAY_V4I16_ZERO;
4474 Ops.push_back(TexHandle);
4475 Ops.push_back(N->getOperand(2));
4476 Ops.push_back(N->getOperand(3));
4477 Ops.push_back(Chain);
4478 break;
4479 case NVPTXISD::Suld1DArrayV4I32Zero:
4480 Opc = NVPTX::SULD_1D_ARRAY_V4I32_ZERO;
4481 Ops.push_back(TexHandle);
4482 Ops.push_back(N->getOperand(2));
4483 Ops.push_back(N->getOperand(3));
4484 Ops.push_back(Chain);
4485 break;
4486 case NVPTXISD::Suld2DI8Zero:
4487 Opc = NVPTX::SULD_2D_I8_ZERO;
4488 Ops.push_back(TexHandle);
4489 Ops.push_back(N->getOperand(2));
4490 Ops.push_back(N->getOperand(3));
4491 Ops.push_back(Chain);
4492 break;
4493 case NVPTXISD::Suld2DI16Zero:
4494 Opc = NVPTX::SULD_2D_I16_ZERO;
4495 Ops.push_back(TexHandle);
4496 Ops.push_back(N->getOperand(2));
4497 Ops.push_back(N->getOperand(3));
4498 Ops.push_back(Chain);
4499 break;
4500 case NVPTXISD::Suld2DI32Zero:
4501 Opc = NVPTX::SULD_2D_I32_ZERO;
4502 Ops.push_back(TexHandle);
4503 Ops.push_back(N->getOperand(2));
4504 Ops.push_back(N->getOperand(3));
4505 Ops.push_back(Chain);
4506 break;
4507 case NVPTXISD::Suld2DI64Zero:
4508 Opc = NVPTX::SULD_2D_I64_ZERO;
4509 Ops.push_back(TexHandle);
4510 Ops.push_back(N->getOperand(2));
4511 Ops.push_back(N->getOperand(3));
4512 Ops.push_back(Chain);
4513 break;
4514 case NVPTXISD::Suld2DV2I8Zero:
4515 Opc = NVPTX::SULD_2D_V2I8_ZERO;
4516 Ops.push_back(TexHandle);
4517 Ops.push_back(N->getOperand(2));
4518 Ops.push_back(N->getOperand(3));
4519 Ops.push_back(Chain);
4520 break;
4521 case NVPTXISD::Suld2DV2I16Zero:
4522 Opc = NVPTX::SULD_2D_V2I16_ZERO;
4523 Ops.push_back(TexHandle);
4524 Ops.push_back(N->getOperand(2));
4525 Ops.push_back(N->getOperand(3));
4526 Ops.push_back(Chain);
4527 break;
4528 case NVPTXISD::Suld2DV2I32Zero:
4529 Opc = NVPTX::SULD_2D_V2I32_ZERO;
4530 Ops.push_back(TexHandle);
4531 Ops.push_back(N->getOperand(2));
4532 Ops.push_back(N->getOperand(3));
4533 Ops.push_back(Chain);
4534 break;
4535 case NVPTXISD::Suld2DV2I64Zero:
4536 Opc = NVPTX::SULD_2D_V2I64_ZERO;
4537 Ops.push_back(TexHandle);
4538 Ops.push_back(N->getOperand(2));
4539 Ops.push_back(N->getOperand(3));
4540 Ops.push_back(Chain);
4541 break;
4542 case NVPTXISD::Suld2DV4I8Zero:
4543 Opc = NVPTX::SULD_2D_V4I8_ZERO;
4544 Ops.push_back(TexHandle);
4545 Ops.push_back(N->getOperand(2));
4546 Ops.push_back(N->getOperand(3));
4547 Ops.push_back(Chain);
4548 break;
4549 case NVPTXISD::Suld2DV4I16Zero:
4550 Opc = NVPTX::SULD_2D_V4I16_ZERO;
4551 Ops.push_back(TexHandle);
4552 Ops.push_back(N->getOperand(2));
4553 Ops.push_back(N->getOperand(3));
4554 Ops.push_back(Chain);
4555 break;
4556 case NVPTXISD::Suld2DV4I32Zero:
4557 Opc = NVPTX::SULD_2D_V4I32_ZERO;
4558 Ops.push_back(TexHandle);
4559 Ops.push_back(N->getOperand(2));
4560 Ops.push_back(N->getOperand(3));
4561 Ops.push_back(Chain);
4562 break;
4563 case NVPTXISD::Suld2DArrayI8Zero:
4564 Opc = NVPTX::SULD_2D_ARRAY_I8_ZERO;
4565 Ops.push_back(TexHandle);
4566 Ops.push_back(N->getOperand(2));
4567 Ops.push_back(N->getOperand(3));
4568 Ops.push_back(N->getOperand(4));
4569 Ops.push_back(Chain);
4570 break;
4571 case NVPTXISD::Suld2DArrayI16Zero:
4572 Opc = NVPTX::SULD_2D_ARRAY_I16_ZERO;
4573 Ops.push_back(TexHandle);
4574 Ops.push_back(N->getOperand(2));
4575 Ops.push_back(N->getOperand(3));
4576 Ops.push_back(N->getOperand(4));
4577 Ops.push_back(Chain);
4578 break;
4579 case NVPTXISD::Suld2DArrayI32Zero:
4580 Opc = NVPTX::SULD_2D_ARRAY_I32_ZERO;
4581 Ops.push_back(TexHandle);
4582 Ops.push_back(N->getOperand(2));
4583 Ops.push_back(N->getOperand(3));
4584 Ops.push_back(N->getOperand(4));
4585 Ops.push_back(Chain);
4586 break;
4587 case NVPTXISD::Suld2DArrayI64Zero:
4588 Opc = NVPTX::SULD_2D_ARRAY_I64_ZERO;
4589 Ops.push_back(TexHandle);
4590 Ops.push_back(N->getOperand(2));
4591 Ops.push_back(N->getOperand(3));
4592 Ops.push_back(N->getOperand(4));
4593 Ops.push_back(Chain);
4594 break;
4595 case NVPTXISD::Suld2DArrayV2I8Zero:
4596 Opc = NVPTX::SULD_2D_ARRAY_V2I8_ZERO;
4597 Ops.push_back(TexHandle);
4598 Ops.push_back(N->getOperand(2));
4599 Ops.push_back(N->getOperand(3));
4600 Ops.push_back(N->getOperand(4));
4601 Ops.push_back(Chain);
4602 break;
4603 case NVPTXISD::Suld2DArrayV2I16Zero:
4604 Opc = NVPTX::SULD_2D_ARRAY_V2I16_ZERO;
4605 Ops.push_back(TexHandle);
4606 Ops.push_back(N->getOperand(2));
4607 Ops.push_back(N->getOperand(3));
4608 Ops.push_back(N->getOperand(4));
4609 Ops.push_back(Chain);
4610 break;
4611 case NVPTXISD::Suld2DArrayV2I32Zero:
4612 Opc = NVPTX::SULD_2D_ARRAY_V2I32_ZERO;
4613 Ops.push_back(TexHandle);
4614 Ops.push_back(N->getOperand(2));
4615 Ops.push_back(N->getOperand(3));
4616 Ops.push_back(N->getOperand(4));
4617 Ops.push_back(Chain);
4618 break;
4619 case NVPTXISD::Suld2DArrayV2I64Zero:
4620 Opc = NVPTX::SULD_2D_ARRAY_V2I64_ZERO;
4621 Ops.push_back(TexHandle);
4622 Ops.push_back(N->getOperand(2));
4623 Ops.push_back(N->getOperand(3));
4624 Ops.push_back(N->getOperand(4));
4625 Ops.push_back(Chain);
4626 break;
4627 case NVPTXISD::Suld2DArrayV4I8Zero:
4628 Opc = NVPTX::SULD_2D_ARRAY_V4I8_ZERO;
4629 Ops.push_back(TexHandle);
4630 Ops.push_back(N->getOperand(2));
4631 Ops.push_back(N->getOperand(3));
4632 Ops.push_back(N->getOperand(4));
4633 Ops.push_back(Chain);
4634 break;
4635 case NVPTXISD::Suld2DArrayV4I16Zero:
4636 Opc = NVPTX::SULD_2D_ARRAY_V4I16_ZERO;
4637 Ops.push_back(TexHandle);
4638 Ops.push_back(N->getOperand(2));
4639 Ops.push_back(N->getOperand(3));
4640 Ops.push_back(N->getOperand(4));
4641 Ops.push_back(Chain);
4642 break;
4643 case NVPTXISD::Suld2DArrayV4I32Zero:
4644 Opc = NVPTX::SULD_2D_ARRAY_V4I32_ZERO;
4645 Ops.push_back(TexHandle);
4646 Ops.push_back(N->getOperand(2));
4647 Ops.push_back(N->getOperand(3));
4648 Ops.push_back(N->getOperand(4));
4649 Ops.push_back(Chain);
4650 break;
4651 case NVPTXISD::Suld3DI8Zero:
4652 Opc = NVPTX::SULD_3D_I8_ZERO;
4653 Ops.push_back(TexHandle);
4654 Ops.push_back(N->getOperand(2));
4655 Ops.push_back(N->getOperand(3));
4656 Ops.push_back(N->getOperand(4));
4657 Ops.push_back(Chain);
4658 break;
4659 case NVPTXISD::Suld3DI16Zero:
4660 Opc = NVPTX::SULD_3D_I16_ZERO;
4661 Ops.push_back(TexHandle);
4662 Ops.push_back(N->getOperand(2));
4663 Ops.push_back(N->getOperand(3));
4664 Ops.push_back(N->getOperand(4));
4665 Ops.push_back(Chain);
4666 break;
4667 case NVPTXISD::Suld3DI32Zero:
4668 Opc = NVPTX::SULD_3D_I32_ZERO;
4669 Ops.push_back(TexHandle);
4670 Ops.push_back(N->getOperand(2));
4671 Ops.push_back(N->getOperand(3));
4672 Ops.push_back(N->getOperand(4));
4673 Ops.push_back(Chain);
4674 break;
4675 case NVPTXISD::Suld3DI64Zero:
4676 Opc = NVPTX::SULD_3D_I64_ZERO;
4677 Ops.push_back(TexHandle);
4678 Ops.push_back(N->getOperand(2));
4679 Ops.push_back(N->getOperand(3));
4680 Ops.push_back(N->getOperand(4));
4681 Ops.push_back(Chain);
4682 break;
4683 case NVPTXISD::Suld3DV2I8Zero:
4684 Opc = NVPTX::SULD_3D_V2I8_ZERO;
4685 Ops.push_back(TexHandle);
4686 Ops.push_back(N->getOperand(2));
4687 Ops.push_back(N->getOperand(3));
4688 Ops.push_back(N->getOperand(4));
4689 Ops.push_back(Chain);
4690 break;
4691 case NVPTXISD::Suld3DV2I16Zero:
4692 Opc = NVPTX::SULD_3D_V2I16_ZERO;
4693 Ops.push_back(TexHandle);
4694 Ops.push_back(N->getOperand(2));
4695 Ops.push_back(N->getOperand(3));
4696 Ops.push_back(N->getOperand(4));
4697 Ops.push_back(Chain);
4698 break;
4699 case NVPTXISD::Suld3DV2I32Zero:
4700 Opc = NVPTX::SULD_3D_V2I32_ZERO;
4701 Ops.push_back(TexHandle);
4702 Ops.push_back(N->getOperand(2));
4703 Ops.push_back(N->getOperand(3));
4704 Ops.push_back(N->getOperand(4));
4705 Ops.push_back(Chain);
4706 break;
4707 case NVPTXISD::Suld3DV2I64Zero:
4708 Opc = NVPTX::SULD_3D_V2I64_ZERO;
4709 Ops.push_back(TexHandle);
4710 Ops.push_back(N->getOperand(2));
4711 Ops.push_back(N->getOperand(3));
4712 Ops.push_back(N->getOperand(4));
4713 Ops.push_back(Chain);
4714 break;
4715 case NVPTXISD::Suld3DV4I8Zero:
4716 Opc = NVPTX::SULD_3D_V4I8_ZERO;
4717 Ops.push_back(TexHandle);
4718 Ops.push_back(N->getOperand(2));
4719 Ops.push_back(N->getOperand(3));
4720 Ops.push_back(N->getOperand(4));
4721 Ops.push_back(Chain);
4722 break;
4723 case NVPTXISD::Suld3DV4I16Zero:
4724 Opc = NVPTX::SULD_3D_V4I16_ZERO;
4725 Ops.push_back(TexHandle);
4726 Ops.push_back(N->getOperand(2));
4727 Ops.push_back(N->getOperand(3));
4728 Ops.push_back(N->getOperand(4));
4729 Ops.push_back(Chain);
4730 break;
4731 case NVPTXISD::Suld3DV4I32Zero:
4732 Opc = NVPTX::SULD_3D_V4I32_ZERO;
4733 Ops.push_back(TexHandle);
4734 Ops.push_back(N->getOperand(2));
4735 Ops.push_back(N->getOperand(3));
4736 Ops.push_back(N->getOperand(4));
4737 Ops.push_back(Chain);
4738 break;
Justin Holewinski30d56a72014-04-09 15:39:15 +0000[diff] [blame]4739 }
4740 Ret = CurDAG->getMachineNode(Opc, SDLoc(N), N->getVTList(), Ops);
4741 return Ret;
4742}
4743
Justin Holewinski9a2350e2014-07-17 11:59:04 +0000[diff] [blame]4744
Justin Holewinskica7a4f12014-06-27 18:35:27 +0000[diff] [blame]4745/// SelectBFE - Look for instruction sequences that can be made more efficient
4746/// by using the 'bfe' (bit-field extract) PTX instruction
4747SDNode *NVPTXDAGToDAGISel::SelectBFE(SDNode *N) {
4748 SDValue LHS = N->getOperand(0);
4749 SDValue RHS = N->getOperand(1);
4750 SDValue Len;
4751 SDValue Start;
4752 SDValue Val;
4753 bool IsSigned = false;
4754
4755 if (N->getOpcode() == ISD::AND) {
4756 // Canonicalize the operands
4757 // We want 'and %val, %mask'
4758 if (isa<ConstantSDNode>(LHS) && !isa<ConstantSDNode>(RHS)) {
4759 std::swap(LHS, RHS);
4760 }
4761
4762 ConstantSDNode *Mask = dyn_cast<ConstantSDNode>(RHS);
4763 if (!Mask) {
4764 // We need a constant mask on the RHS of the AND
4765 return NULL;
4766 }
4767
4768 // Extract the mask bits
4769 uint64_t MaskVal = Mask->getZExtValue();
4770 if (!isMask_64(MaskVal)) {
4771 // We *could* handle shifted masks here, but doing so would require an
4772 // 'and' operation to fix up the low-order bits so we would trade
4773 // shr+and for bfe+and, which has the same throughput
4774 return NULL;
4775 }
4776
4777 // How many bits are in our mask?
4778 uint64_t NumBits = CountTrailingOnes_64(MaskVal);
4779 Len = CurDAG->getTargetConstant(NumBits, MVT::i32);
4780
4781 if (LHS.getOpcode() == ISD::SRL || LHS.getOpcode() == ISD::SRA) {
4782 // We have a 'srl/and' pair, extract the effective start bit and length
4783 Val = LHS.getNode()->getOperand(0);
4784 Start = LHS.getNode()->getOperand(1);
4785 ConstantSDNode *StartConst = dyn_cast<ConstantSDNode>(Start);
4786 if (StartConst) {
4787 uint64_t StartVal = StartConst->getZExtValue();
4788 // How many "good" bits do we have left? "good" is defined here as bits
4789 // that exist in the original value, not shifted in.
4790 uint64_t GoodBits = Start.getValueType().getSizeInBits() - StartVal;
4791 if (NumBits > GoodBits) {
4792 // Do not handle the case where bits have been shifted in. In theory
4793 // we could handle this, but the cost is likely higher than just
4794 // emitting the srl/and pair.
4795 return NULL;
4796 }
4797 Start = CurDAG->getTargetConstant(StartVal, MVT::i32);
4798 } else {
4799 // Do not handle the case where the shift amount (can be zero if no srl
4800 // was found) is not constant. We could handle this case, but it would
4801 // require run-time logic that would be more expensive than just
4802 // emitting the srl/and pair.
4803 return NULL;
4804 }
4805 } else {
4806 // Do not handle the case where the LHS of the and is not a shift. While
4807 // it would be trivial to handle this case, it would just transform
4808 // 'and' -> 'bfe', but 'and' has higher-throughput.
4809 return NULL;
4810 }
4811 } else if (N->getOpcode() == ISD::SRL || N->getOpcode() == ISD::SRA) {
4812 if (LHS->getOpcode() == ISD::AND) {
4813 ConstantSDNode *ShiftCnst = dyn_cast<ConstantSDNode>(RHS);
4814 if (!ShiftCnst) {
4815 // Shift amount must be constant
4816 return NULL;
4817 }
4818
4819 uint64_t ShiftAmt = ShiftCnst->getZExtValue();
4820
4821 SDValue AndLHS = LHS->getOperand(0);
4822 SDValue AndRHS = LHS->getOperand(1);
4823
4824 // Canonicalize the AND to have the mask on the RHS
4825 if (isa<ConstantSDNode>(AndLHS)) {
4826 std::swap(AndLHS, AndRHS);
4827 }
4828
4829 ConstantSDNode *MaskCnst = dyn_cast<ConstantSDNode>(AndRHS);
4830 if (!MaskCnst) {
4831 // Mask must be constant
4832 return NULL;
4833 }
4834
4835 uint64_t MaskVal = MaskCnst->getZExtValue();
4836 uint64_t NumZeros;
4837 uint64_t NumBits;
4838 if (isMask_64(MaskVal)) {
4839 NumZeros = 0;
4840 // The number of bits in the result bitfield will be the number of
4841 // trailing ones (the AND) minus the number of bits we shift off
4842 NumBits = CountTrailingOnes_64(MaskVal) - ShiftAmt;
4843 } else if (isShiftedMask_64(MaskVal)) {
4844 NumZeros = countTrailingZeros(MaskVal);
4845 unsigned NumOnes = CountTrailingOnes_64(MaskVal >> NumZeros);
4846 // The number of bits in the result bitfield will be the number of
4847 // trailing zeros plus the number of set bits in the mask minus the
4848 // number of bits we shift off
4849 NumBits = NumZeros + NumOnes - ShiftAmt;
4850 } else {
4851 // This is not a mask we can handle
4852 return NULL;
4853 }
4854
4855 if (ShiftAmt < NumZeros) {
4856 // Handling this case would require extra logic that would make this
4857 // transformation non-profitable
4858 return NULL;
4859 }
4860
4861 Val = AndLHS;
4862 Start = CurDAG->getTargetConstant(ShiftAmt, MVT::i32);
4863 Len = CurDAG->getTargetConstant(NumBits, MVT::i32);
4864 } else if (LHS->getOpcode() == ISD::SHL) {
4865 // Here, we have a pattern like:
4866 //
4867 // (sra (shl val, NN), MM)
4868 // or
4869 // (srl (shl val, NN), MM)
4870 //
4871 // If MM >= NN, we can efficiently optimize this with bfe
4872 Val = LHS->getOperand(0);
4873
4874 SDValue ShlRHS = LHS->getOperand(1);
4875 ConstantSDNode *ShlCnst = dyn_cast<ConstantSDNode>(ShlRHS);
4876 if (!ShlCnst) {
4877 // Shift amount must be constant
4878 return NULL;
4879 }
4880 uint64_t InnerShiftAmt = ShlCnst->getZExtValue();
4881
4882 SDValue ShrRHS = RHS;
4883 ConstantSDNode *ShrCnst = dyn_cast<ConstantSDNode>(ShrRHS);
4884 if (!ShrCnst) {
4885 // Shift amount must be constant
4886 return NULL;
4887 }
4888 uint64_t OuterShiftAmt = ShrCnst->getZExtValue();
4889
4890 // To avoid extra codegen and be profitable, we need Outer >= Inner
4891 if (OuterShiftAmt < InnerShiftAmt) {
4892 return NULL;
4893 }
4894
4895 // If the outer shift is more than the type size, we have no bitfield to
4896 // extract (since we also check that the inner shift is <= the outer shift
4897 // then this also implies that the inner shift is < the type size)
4898 if (OuterShiftAmt >= Val.getValueType().getSizeInBits()) {
4899 return NULL;
4900 }
4901
4902 Start =
4903 CurDAG->getTargetConstant(OuterShiftAmt - InnerShiftAmt, MVT::i32);
4904 Len =
4905 CurDAG->getTargetConstant(Val.getValueType().getSizeInBits() -
4906 OuterShiftAmt, MVT::i32);
4907
4908 if (N->getOpcode() == ISD::SRA) {
4909 // If we have a arithmetic right shift, we need to use the signed bfe
4910 // variant
4911 IsSigned = true;
4912 }
4913 } else {
4914 // No can do...
4915 return NULL;
4916 }
4917 } else {
4918 // No can do...
4919 return NULL;
4920 }
4921
4922
4923 unsigned Opc;
4924 // For the BFE operations we form here from "and" and "srl", always use the
4925 // unsigned variants.
4926 if (Val.getValueType() == MVT::i32) {
4927 if (IsSigned) {
4928 Opc = NVPTX::BFE_S32rii;
4929 } else {
4930 Opc = NVPTX::BFE_U32rii;
4931 }
4932 } else if (Val.getValueType() == MVT::i64) {
4933 if (IsSigned) {
4934 Opc = NVPTX::BFE_S64rii;
4935 } else {
4936 Opc = NVPTX::BFE_U64rii;
4937 }
4938 } else {
4939 // We cannot handle this type
4940 return NULL;
4941 }
4942
4943 SDValue Ops[] = {
4944 Val, Start, Len
4945 };
4946
4947 SDNode *Ret =
4948 CurDAG->getMachineNode(Opc, SDLoc(N), N->getVTList(), Ops);
4949
4950 return Ret;
4951}
4952
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]4953// SelectDirectAddr - Match a direct address for DAG.
4954// A direct address could be a globaladdress or externalsymbol.
4955bool NVPTXDAGToDAGISel::SelectDirectAddr(SDValue N, SDValue &Address) {
4956 // Return true if TGA or ES.
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]4957 if (N.getOpcode() == ISD::TargetGlobalAddress ||
4958 N.getOpcode() == ISD::TargetExternalSymbol) {
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]4959 Address = N;
4960 return true;
4961 }
4962 if (N.getOpcode() == NVPTXISD::Wrapper) {
4963 Address = N.getOperand(0);
4964 return true;
4965 }
4966 if (N.getOpcode() == ISD::INTRINSIC_WO_CHAIN) {
4967 unsigned IID = cast<ConstantSDNode>(N.getOperand(0))->getZExtValue();
4968 if (IID == Intrinsic::nvvm_ptr_gen_to_param)
4969 if (N.getOperand(1).getOpcode() == NVPTXISD::MoveParam)
4970 return (SelectDirectAddr(N.getOperand(1).getOperand(0), Address));
4971 }
4972 return false;
4973}
4974
4975// symbol+offset
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]4976bool NVPTXDAGToDAGISel::SelectADDRsi_imp(
4977 SDNode *OpNode, SDValue Addr, SDValue &Base, SDValue &Offset, MVT mvt) {
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]4978 if (Addr.getOpcode() == ISD::ADD) {
4979 if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(Addr.getOperand(1))) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]4980 SDValue base = Addr.getOperand(0);
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]4981 if (SelectDirectAddr(base, Base)) {
4982 Offset = CurDAG->getTargetConstant(CN->getZExtValue(), mvt);
4983 return true;
4984 }
4985 }
4986 }
4987 return false;
4988}
4989
4990// symbol+offset
4991bool NVPTXDAGToDAGISel::SelectADDRsi(SDNode *OpNode, SDValue Addr,
4992 SDValue &Base, SDValue &Offset) {
4993 return SelectADDRsi_imp(OpNode, Addr, Base, Offset, MVT::i32);
4994}
4995
4996// symbol+offset
4997bool NVPTXDAGToDAGISel::SelectADDRsi64(SDNode *OpNode, SDValue Addr,
4998 SDValue &Base, SDValue &Offset) {
4999 return SelectADDRsi_imp(OpNode, Addr, Base, Offset, MVT::i64);
5000}
5001
5002// register+offset
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]5003bool NVPTXDAGToDAGISel::SelectADDRri_imp(
5004 SDNode *OpNode, SDValue Addr, SDValue &Base, SDValue &Offset, MVT mvt) {
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]5005 if (FrameIndexSDNode *FIN = dyn_cast<FrameIndexSDNode>(Addr)) {
5006 Base = CurDAG->getTargetFrameIndex(FIN->getIndex(), mvt);
5007 Offset = CurDAG->getTargetConstant(0, mvt);
5008 return true;
5009 }
5010 if (Addr.getOpcode() == ISD::TargetExternalSymbol ||
5011 Addr.getOpcode() == ISD::TargetGlobalAddress)
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]5012 return false; // direct calls.
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]5013
5014 if (Addr.getOpcode() == ISD::ADD) {
5015 if (SelectDirectAddr(Addr.getOperand(0), Addr)) {
5016 return false;
5017 }
5018 if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(Addr.getOperand(1))) {
5019 if (FrameIndexSDNode *FIN =
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]5020 dyn_cast<FrameIndexSDNode>(Addr.getOperand(0)))
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]5021 // Constant offset from frame ref.
5022 Base = CurDAG->getTargetFrameIndex(FIN->getIndex(), mvt);
5023 else
5024 Base = Addr.getOperand(0);
5025 Offset = CurDAG->getTargetConstant(CN->getZExtValue(), mvt);
5026 return true;
5027 }
5028 }
5029 return false;
5030}
5031
5032// register+offset
5033bool NVPTXDAGToDAGISel::SelectADDRri(SDNode *OpNode, SDValue Addr,
5034 SDValue &Base, SDValue &Offset) {
5035 return SelectADDRri_imp(OpNode, Addr, Base, Offset, MVT::i32);
5036}
5037
5038// register+offset
5039bool NVPTXDAGToDAGISel::SelectADDRri64(SDNode *OpNode, SDValue Addr,
5040 SDValue &Base, SDValue &Offset) {
5041 return SelectADDRri_imp(OpNode, Addr, Base, Offset, MVT::i64);
5042}
5043
5044bool NVPTXDAGToDAGISel::ChkMemSDNodeAddressSpace(SDNode *N,
5045 unsigned int spN) const {
Craig Topper062a2ba2014-04-25 05:30:21 +0000[diff] [blame]5046 const Value *Src = nullptr;
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]5047 if (MemSDNode *mN = dyn_cast<MemSDNode>(N)) {
Nick Lewyckyaad475b2014-04-15 07:22:52 +0000[diff] [blame]5048 if (spN == 0 && mN->getMemOperand()->getPseudoValue())
5049 return true;
5050 Src = mN->getMemOperand()->getValue();
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]5051 }
5052 if (!Src)
5053 return false;
5054 if (const PointerType *PT = dyn_cast<PointerType>(Src->getType()))
5055 return (PT->getAddressSpace() == spN);
5056 return false;
5057}
5058
5059/// SelectInlineAsmMemoryOperand - Implement addressing mode selection for
5060/// inline asm expressions.
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]5061bool NVPTXDAGToDAGISel::SelectInlineAsmMemoryOperand(
5062 const SDValue &Op, char ConstraintCode, std::vector<SDValue> &OutOps) {
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]5063 SDValue Op0, Op1;
5064 switch (ConstraintCode) {
Justin Holewinski0497ab12013-03-30 14:29:21 +0000[diff] [blame]5065 default:
5066 return true;
5067 case 'm': // memory
Justin Holewinskiae556d32012-05-04 20:18:50 +0000[diff] [blame]5068 if (SelectDirectAddr(Op, Op0)) {
5069 OutOps.push_back(Op0);
5070 OutOps.push_back(CurDAG->getTargetConstant(0, MVT::i32));
5071 return false;
5072 }
5073 if (SelectADDRri(Op.getNode(), Op, Op0, Op1)) {
5074 OutOps.push_back(Op0);
5075 OutOps.push_back(Op1);
5076 return false;
5077 }
5078 break;
5079 }
5080 return true;
5081}